From 7e0df8663137dd5254f76c3ad2247b43553cc75e Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Fri, 29 May 2026 15:16:38 +0300
Subject: [PATCH 1/6] XDB-546 merge from main upstream

---
 .clang-tidy                                   |   21 +
 .github/workflows/stale.yml                   |   24 +
 .github/workflows/windows-boost-test.yml      |    2 +-
 .gitignore                                    |    4 +-
 CMakeLists.txt                                |    5 +-
 README.md                                     |   11 +-
 bindings/c/CMakeLists.txt                     |    5 +-
 bindings/c/fdb_c.cpp                          |    4 +-
 .../c/test/apitester/TesterApiWorkload.cpp    |    6 +-
 .../TesterAtomicOpsCorrectnessWorkload.cpp    |    6 +-
 .../apitester/TesterCorrectnessWorkload.cpp   |    4 +-
 .../c/test/apitester/TesterKeyValueStore.cpp  |   10 +-
 bindings/c/test/apitester/TesterTestSpec.cpp  |    2 +-
 .../apitester/TesterTransactionExecutor.cpp   |   14 +-
 bindings/c/test/apitester/TesterUtil.h        |    6 +-
 bindings/c/test/apitester/TesterWorkload.cpp  |   12 +-
 .../c/test/apitester/fdb_c_api_tester.cpp     |    2 +-
 bindings/c/test/mako/async.cpp                |   11 +-
 bindings/c/test/mako/async.hpp                |    2 +-
 bindings/c/test/mako/mako.cpp                 |  291 +-
 bindings/c/test/mako/mako.hpp                 |   10 +-
 bindings/c/test/mako/mako.rst                 |    9 +
 bindings/c/test/mako/operations.cpp           |   14 +
 bindings/c/test/mako/stats.hpp                |   18 +
 bindings/c/test/performance_test.c            |   11 +-
 bindings/c/test/ryw_benchmark.c               |   17 +-
 bindings/c/test/test.h                        |   17 +-
 bindings/c/test/workloads/workloads.cpp       |    2 +-
 bindings/flow/CMakeLists.txt                  |   10 +-
 ...toryLayer.actor.cpp => DirectoryLayer.cpp} |  301 +-
 bindings/flow/DirectoryPartition.h            |   12 +-
 ....actor.cpp => HighContentionAllocator.cpp} |   34 +-
 bindings/flow/{Node.actor.cpp => Node.cpp}    |   10 +-
 bindings/flow/Subspace.cpp                    |    4 +-
 .../flow/{fdb_flow.actor.cpp => fdb_flow.cpp} |   49 +-
 bindings/flow/tester/CMakeLists.txt           |    6 +-
 ...ryTester.actor.cpp => DirectoryTester.cpp} |  249 +-
 .../tester/{Tester.actor.cpp => Tester.cpp}   |  846 ++-
 .../flow/tester/{Tester.actor.h => Tester.h}  |   60 +-
 bindings/go/CMakeLists.txt                    |   37 +-
 bindings/go/README.md                         |   10 +
 bindings/go/src/fdb/database.go               |    5 +-
 bindings/go/src/fdb/doc.go                    |   48 +-
 bindings/go/src/fdb/error_codes_generated.go  |  677 ++
 bindings/go/src/fdb/errors.go                 |   35 +-
 bindings/go/src/fdb/errors_test.go            |   27 +
 bindings/go/src/fdb/fdb.go                    |   20 +-
 bindings/go/src/fdb/generated.go              |   73 +-
 bindings/go/src/internal/gen_errors/main.go   |  176 +
 .../translate_fdb_options.go                  |   18 +-
 bindings/java/JavaWorkload.cpp                |   20 +-
 bindings/java/fdbJNI.cpp                      |  106 +-
 .../MappedRangeQueryIntegrationTest.java      |  113 +-
 .../apple/foundationdb/FDBTransaction.java    |    5 +
 .../com/apple/foundationdb/Transaction.java   |   18 +
 .../test/ConcurrentGetSetGet.java             |    1 -
 .../foundationdb/test/SerialIteration.java    |    2 -
 .../apple/foundationdb/test/WatchTest.java    |    1 -
 cmake/AddFdbServerLinkTest.cmake              |    8 +
 cmake/CompileBoost.cmake                      |   42 +-
 cmake/CompileRocksDB.cmake                    |   13 +-
 cmake/ConfigureCompiler.cmake                 |   30 +
 cmake/FDBBenchmark.cmake                      |   61 +
 cmake/FDBComponents.cmake                     |   33 +-
 cmake/FindCoroutines.cmake                    |    2 +-
 cmake/GetFmt.cmake                            |   23 +-
 cmake/RocksDBVersion.cmake                    |    2 +-
 .../benchmark-download.cmake                  |    7 +-
 cmake/utils.cmake                             |    5 +
 contrib/Joshua/scripts/correctnessTest.sh     |   11 +
 contrib/TestHarness2/README.md                |    6 +
 contrib/TestHarness2/test_harness/config.py   |   29 +-
 contrib/TestHarness2/test_harness/results.py  |   19 +-
 contrib/TestHarness2/test_harness/run.py      |   46 +
 .../TestHarness2/test_harness/summarize.py    |   17 +-
 contrib/benchmark_comparison.py               |   12 +-
 contrib/joshua_logtool.py                     |    2 +-
 contrib/linenoise/CMakeLists.txt              |    1 +
 contrib/lsan.suppressions                     |   11 +
 contrib/mockkms/CMakeLists.txt                |   18 -
 contrib/mockkms/fault_injection.go            |  179 -
 contrib/mockkms/get_encryption_keys.go        |  321 -
 contrib/mockkms/mock_kms.go                   |   75 -
 contrib/mockkms/mockkms_test.go               |  302 -
 contrib/monitoring/actor_flamegraph.cpp       |    4 +-
 contrib/pkg_tester/requirements.txt           |    2 +-
 contrib/replay/CMakeLists.txt                 |   28 +-
 contrib/sqlite/CMakeLists.txt                 |    4 +-
 contrib/stacktrace/CMakeLists.txt             |    1 +
 design/AI-generated/FDB_NETWORK_PROTOCOL.md   | 2666 ++++++++
 design/AI-generated/README.md                 |   63 +
 .../AI-generated/diagram_01_flow_runtime.md   |   76 +
 .../AI-generated/diagram_02_rpc_transport.md  |  109 +
 .../AI-generated/diagram_03_client_library.md |  104 +
 .../diagram_04_cluster_controller.md          |  107 +
 .../diagram_05_commit_pipeline.md             |  107 +
 .../AI-generated/diagram_06_tlog_logsystem.md |  139 +
 .../AI-generated/diagram_07_storage_server.md |  119 +
 .../diagram_08_data_distribution.md           |  129 +
 .../diagram_09_cluster_recovery.md            |  149 +
 design/AI-generated/diagram_10_ratekeeper.md  |   91 +
 design/AI-generated/diagram_11_backup_dr.md   |  114 +
 .../diagram_12_simulation_testing.md          |  155 +
 design/AI-generated/diagram_overview.md       |  163 +
 .../foundationdb_subsystem_map.md             |  359 ++
 .../AI-generated/subsystem_01_flow_runtime.md |  378 ++
 .../subsystem_02_rpc_transport.md             |  340 +
 .../subsystem_03_client_library.md            |  288 +
 .../subsystem_04_cluster_controller.md        |  319 +
 .../subsystem_05_commit_pipeline.md           |  305 +
 .../subsystem_06_tlog_logsystem.md            |  243 +
 .../subsystem_07_storage_server.md            |  766 +++
 .../subsystem_08_data_distribution.md         |  238 +
 .../subsystem_09_cluster_recovery.md          |  215 +
 .../AI-generated/subsystem_10_ratekeeper.md   |  203 +
 design/AI-generated/subsystem_11_backup_dr.md |  290 +
 .../subsystem_12_simulation_testing.md        |  345 +
 design/AcAC.md                                |   15 +-
 design/Commit/commit.sequence                 |    2 +-
 design/backup_v2_partitioned_logs.md          |    4 +-
 design/cluster_health_metric.md               |  171 +
 design/coroutines.md                          |  111 +-
 design/data-distributor-internals.md          |  806 ++-
 design/fdb-coroutines-internals.md            |  727 +++
 design/file-level-encryption-backup.md        |  370 ++
 design/global-tag-throttling.md               |  131 -
 design/mocks3server_chaos_design.md           |    3 +-
 design/recovery-internals.md                  |    6 +-
 design/restore_v1.md                          |    2 +-
 design/transaction-state-store.md             |    2 +-
 ...idating_restored_data_using_one_cluster.md |    2 -
 documentation/coro_tutorial/tutorial.cpp      |   21 +-
 documentation/sphinx/source/api-c.rst         |    4 +-
 documentation/sphinx/source/backups.rst       |    4 +-
 documentation/sphinx/source/clang-format.rst  |   68 +
 documentation/sphinx/source/clang-tidy.rst    |  186 +
 .../sphinx/source/command-line-interface.rst  |    2 +-
 .../sphinx/source/developer-tools.rst         |   13 +
 .../sphinx/source/global-configuration.rst    |    4 +-
 documentation/sphinx/source/ha-write-path.rst |    6 +-
 documentation/sphinx/source/index.rst         |    5 +-
 .../source/mr-status-json-schemas.rst.inc     |    4 +-
 .../sphinx/source/read-write-path.rst         |    2 +-
 .../release-notes/release-notes-700.rst       |    2 +-
 .../release-notes/release-notes-730.rst       |   10 +
 .../tutorial/dining_philosophers.actor.cpp    |   10 +-
 documentation/tutorial/make_h2o.actor.cpp     |    2 +-
 documentation/tutorial/play.actor.cpp         |    2 +-
 documentation/tutorial/play_network.actor.cpp |    2 +-
 .../tutorial/print_in_order.actor.cpp         |    2 +-
 documentation/tutorial/tutorial.actor.cpp     |   21 +-
 fdbbackup/CMakeLists.txt                      |   30 +-
 ...eConverter.actor.cpp => FileConverter.cpp} |  119 +-
 ...{FileDecoder.actor.cpp => FileDecoder.cpp} |  178 +-
 fdbbackup/{backup.actor.cpp => backup.cpp}    | 1219 ++--
 fdbbackup/include/fdbbackup/FileConverter.h   |    4 +-
 fdbbackup/tests/backup_tests_common.sh        |  114 +-
 ...up_test.sh => blob_backup_restore_test.sh} |   66 +-
 fdbbackup/tests/dir_backup_test.sh            |   41 +-
 .../tests/s3_backup_bulkdump_bulkload.sh      |    6 +-
 ...nd.actor.cpp => AdvanceVersionCommand.cpp} |   28 +-
 ...mand.actor.cpp => AuditStorageCommand.cpp} |   34 +-
 ...pCommand.actor.cpp => BulkDumpCommand.cpp} |  133 +-
 ...dCommand.actor.cpp => BulkLoadCommand.cpp} |  226 +-
 fdbcli/CMakeLists.txt                         |    2 +-
 ...Command.actor.cpp => ConfigureCommand.cpp} |   69 +-
 ....actor.cpp => ConsistencyCheckCommand.cpp} |   26 +-
 ...d.actor.cpp => ConsistencyScanCommand.cpp} |   52 +-
 ...mand.actor.cpp => CoordinatorsCommand.cpp} |  110 +-
 ....actor.cpp => DataDistributionCommand.cpp} |   75 +-
 ...ugCommands.actor.cpp => DebugCommands.cpp} |  189 +-
 ...deCommand.actor.cpp => ExcludeCommand.cpp} |  260 +-
 ...ctor.cpp => ExpensiveDataCheckCommand.cpp} |   27 +-
 ...and.actor.cpp => FileConfigureCommand.cpp} |   40 +-
 ...wLineNoise.actor.cpp => FlowLineNoise.cpp} |   12 +-
 ...p => ForceRecoveryWithDataLossCommand.cpp} |   14 +-
 ...nd.actor.cpp => GetAuditStatusCommand.cpp} |  155 +-
 ...eCommand.actor.cpp => HotRangeCommand.cpp} |   36 +-
 ...nd.actor.cpp => IdempotencyIdsCommand.cpp} |   28 +-
 ...deCommand.actor.cpp => IncludeCommand.cpp} |   78 +-
 ...{KillCommand.actor.cpp => KillCommand.cpp} |   37 +-
 ....actor.cpp => LocationMetadataCommand.cpp} |  155 +-
 ...{LockCommand.actor.cpp => LockCommand.cpp} |   78 +-
 ...mmand.actor.cpp => MaintenanceCommand.cpp} |   75 +-
 ...leCommand.actor.cpp => ProfileCommand.cpp} |   36 +-
 ...mmand.actor.cpp => RangeConfigCommand.cpp} |   56 +-
 ...sCommand.actor.cpp => SetClassCommand.cpp} |   74 +-
 ...tCommand.actor.cpp => SnapshotCommand.cpp} |   16 +-
 ...tusCommand.actor.cpp => StatusCommand.cpp} |  148 +-
 ...ndCommand.actor.cpp => SuspendCommand.cpp} |   34 +-
 ...eCommand.actor.cpp => ThrottleCommand.cpp} |   65 +-
 ...or.cpp => TriggerDDTeamInfoLogCommand.cpp} |   20 +-
 ...{TssqCommand.actor.cpp => TssqCommand.cpp} |   75 +-
 fdbcli/Util.actor.cpp                         |  193 -
 fdbcli/Util.cpp                               |  465 ++
 ...mand.actor.cpp => VersionEpochCommand.cpp} |   91 +-
 fdbcli/{fdbcli.actor.cpp => fdbcli.cpp}       |  357 +-
 fdbcli/include/fdbcli/fdbcli.actor.h          |  272 -
 fdbcli/include/fdbcli/fdbcli.h                |  342 +
 fdbcli/tests/fdbcli_tests.py                  |  137 +-
 fdbclient/ActorLineageProfiler.cpp            |    4 +-
 ...Store.actor.cpp => AsyncFileBlobStore.cpp} |   41 +-
 fdbclient/AsyncTaskThread.actor.cpp           |  120 -
 .../{AuditUtils.actor.cpp => AuditUtils.cpp}  |  751 ++-
 ...gentBase.actor.cpp => BackupAgentBase.cpp} |  539 +-
 ...ontainer.actor.cpp => BackupContainer.cpp} |  165 +-
 ...actor.cpp => BackupContainerBlobStore.cpp} |  144 +-
 ...BlobStore.h => BackupContainerBlobStore.h} |   33 +-
 ...ctor.cpp => BackupContainerFileSystem.cpp} | 1021 ++-
 ....cpp => BackupContainerLocalDirectory.cpp} |   67 +-
 .../BackupContainerLocalDirectory.h           |    4 +-
 fdbclient/BackupTLSConfig.cpp                 |   16 +-
 fdbclient/BlobStoreCommon.cpp                 | 1015 +++
 fdbclient/BulkLoading.cpp                     |   10 +-
 fdbclient/CMakeLists.txt                      |   31 +-
 fdbclient/ClientKnobCollection.cpp            |   60 -
 fdbclient/ClientKnobs.cpp                     |  112 +-
 fdbclient/ClientStatusReport.cpp              |    4 +-
 ...le.actor.cpp => ClusterConnectionFile.cpp} |    9 +-
 ...Key.actor.cpp => ClusterConnectionKey.cpp} |   81 +-
 ...tionKey.actor.h => ClusterConnectionKey.h} |   23 +-
 ....cpp => ClusterConnectionMemoryRecord.cpp} |    6 +-
 fdbclient/CoordinationInterface.cpp           |    3 -
 ...g.actor.cpp => DataDistributionConfig.cpp} |    5 +-
 ...gent.actor.cpp => DatabaseBackupAgent.cpp} | 2297 +++----
 fdbclient/DatabaseConfiguration.cpp           |  105 +-
 ...eContext.actor.cpp => DatabaseContext.cpp} |  337 +-
 fdbclient/FDBAWSCredentialsProvider.cpp       |    2 +-
 .../FDBAWSCredentialsProvider.h               |    0
 fdbclient/FDBTypes.cpp                        |    7 +-
 ...kupAgent.actor.cpp => FileBackupAgent.cpp} | 5615 +++++++++--------
 fdbclient/FluentDSampleIngestor.cpp           |    4 +-
 fdbclient/GCSBlobStore.cpp                    |  289 +
 fdbclient/GCSBlobStore.h                      |   59 +
 ...lobalConfig.actor.cpp => GlobalConfig.cpp} |  154 +-
 fdbclient/IKnobCollection.cpp                 |  126 -
 ...mpotencyId.actor.cpp => IdempotencyId.cpp} |  105 +-
 fdbclient/JsonBuilder.cpp                     |    1 -
 ...{KeyRangeMap.actor.cpp => KeyRangeMap.cpp} |  107 +-
 fdbclient/KnobValue.cpp                       |    1 -
 fdbclient/LocalClientAPI.cpp                  |    4 +-
 .../{include/fdbclient => }/LocalClientAPI.h  |    2 +-
 ...agementAPI.actor.cpp => ManagementAPI.cpp} | 2022 +++---
 ...itorLeader.actor.cpp => MonitorLeader.cpp} |  274 +-
 ....actor.cpp => MultiVersionTransaction.cpp} |  231 +-
 ...Reader.actor.cpp => MutationLogReader.cpp} |   70 +-
 fdbclient/NameLineage.cpp                     |    2 +-
 .../{include/fdbclient => }/NameLineage.h     |    0
 fdbclient/NativeAPI.actor.cpp                 | 1259 ++--
 fdbclient/ParallelStream.actor.cpp            |   78 -
 .../fdbclient => }/PartitionedLogIterator.h   |    2 +-
 fdbclient/ProxyLoadBalance.h                  |   95 +
 .../{RESTClient.actor.cpp => RESTClient.cpp}  |   64 +-
 .../{include/fdbclient => }/RESTClient.h      |    2 +-
 .../{RESTUtils.actor.cpp => RESTUtils.cpp}    |   26 +-
 fdbclient/{include/fdbclient => }/RESTUtils.h |    4 +-
 fdbclient/ReadYourWrites.actor.cpp            |  317 +-
 fdbclient/RestoreInterface.cpp                |    2 +-
 .../fdbclient => }/RestoreInterface.h         |    0
 fdbclient/S3BlobStore.actor.cpp               | 2600 --------
 fdbclient/S3BlobStore.cpp                     | 1698 +++++
 .../{S3Client.actor.cpp => S3Client.cpp}      |  732 ++-
 ...3Client_cli.actor.cpp => S3Client_cli.cpp} |   54 +-
 fdbclient/Schemas.cpp                         |   29 +-
 fdbclient/ServerKnobCollection.cpp            |   61 -
 ...KeySpace.actor.cpp => SpecialKeySpace.cpp} |  677 +-
 ...tatusClient.actor.cpp => StatusClient.cpp} |  184 +-
 fdbclient/StorageServerInterface.cpp          |   69 +
 fdbclient/Subspace.cpp                        |    4 +-
 fdbclient/SystemData.cpp                      |  255 +-
 ...{TagThrottle.actor.cpp => TagThrottle.cpp} |   52 +-
 .../{TaskBucket.actor.cpp => TaskBucket.cpp}  |  648 +-
 fdbclient/TestKnobCollection.cpp              |   80 -
 fdbclient/ThreadSafeTransaction.cpp           |   18 +-
 fdbclient/{Tracing.actor.cpp => Tracing.cpp}  |   84 +-
 fdbclient/VersionedMap.cpp                    |   77 -
 fdbclient/WriteMap.cpp                        |  129 +-
 .../bench}/BenchIdempotencyIds.cpp            |    0
 .../bench}/BenchIterate.cpp                   |    2 +-
 .../bench/BenchMain.cpp                       |   12 +-
 .../bench}/BenchMetadataCheck.cpp             |    0
 .../bench}/BenchPopulate.cpp                  |    2 +-
 .../bench}/BenchTempTagMessages.cpp           |    0
 .../bench}/BenchVersionVector.cpp             |    0
 .../BenchVersionVectorSerialization.cpp       |    0
 fdbclient/bench/BenchVersionedMap.cpp         |  564 ++
 fdbclient/bench/CMakeLists.txt                |   12 +
 {flowbench => fdbclient/bench}/GlobalData.cpp |    1 +
 .../bench}/GlobalData.h                       |    7 +-
 fdbclient/include/fdbclient/AnnotateActor.h   |    2 +-
 ...BlobStore.actor.h => AsyncFileBlobStore.h} |  148 +-
 fdbclient/include/fdbclient/AsyncTaskThread.h |  100 -
 .../include/fdbclient/AuditUtils.actor.h      |  122 -
 fdbclient/include/fdbclient/AuditUtils.h      |  203 +
 .../{BackupAgent.actor.h => BackupAgent.h}    |  216 +-
 fdbclient/include/fdbclient/BackupContainer.h |   49 +-
 .../fdbclient/BackupContainerFileSystem.h     |   30 +-
 .../fdbclient/BuildIdempotencyIdMutations.h   |    2 +-
 fdbclient/include/fdbclient/BulkDumping.h     |    2 +
 fdbclient/include/fdbclient/BulkLoading.h     |   21 +-
 .../include/fdbclient/ClientKnobCollection.h  |   51 -
 fdbclient/include/fdbclient/ClientKnobs.h     |  352 --
 fdbclient/include/fdbclient/ClientVersion.h   |    2 +-
 .../include/fdbclient/CommitProxyInterface.h  |  172 +-
 .../include/fdbclient/CommitTransaction.h     |   10 -
 ...ace.actor.h => ConsistencyScanInterface.h} |   73 +-
 .../include/fdbclient/CoordinationInterface.h |   12 +-
 ...onfig.actor.h => DataDistributionConfig.h} |   22 +-
 .../include/fdbclient/DatabaseConfiguration.h |   37 +-
 fdbclient/include/fdbclient/DatabaseContext.h |   40 +-
 .../fdbclient/EncryptKeyProxyInterface.h      |  317 -
 fdbclient/include/fdbclient/EventTypes.h      |   34 +
 fdbclient/include/fdbclient/FDBTypes.h        |   41 +-
 ...mentAPI.actor.h => GenericManagementAPI.h} |  211 +-
 .../include/fdbclient/GlobalConfig.actor.h    |  186 -
 fdbclient/include/fdbclient/GlobalConfig.h    |  159 +-
 .../include/fdbclient/GrvProxyInterface.h     |  148 +
 ...ctor.h => HighContentionPrefixAllocator.h} |   50 +-
 fdbclient/include/fdbclient/IBlobStore.h      |  522 ++
 fdbclient/include/fdbclient/IKnobCollection.h |   79 -
 ...{IdempotencyId.actor.h => IdempotencyId.h} |   22 +-
 fdbclient/include/fdbclient/JSONDoc.h         |    8 +-
 ...edRangeMap.actor.h => KeyBackedRangeMap.h} |   74 +-
 ...eyBackedTypes.actor.h => KeyBackedTypes.h} |  141 +-
 fdbclient/include/fdbclient/KeyRangeMap.h     |   49 +-
 fdbclient/include/fdbclient/Knobs.h           |  338 +-
 .../include/fdbclient/ManagementAPI.actor.h   |  296 -
 fdbclient/include/fdbclient/ManagementAPI.h   |  419 ++
 fdbclient/include/fdbclient/MonitorLeader.h   |   33 +-
 .../fdbclient/MultiVersionTransaction.h       |    4 +-
 ...nLogReader.actor.h => MutationLogReader.h} |   36 +-
 fdbclient/include/fdbclient/NativeAPI.actor.h |   72 +-
 fdbclient/include/fdbclient/Notified.h        |    4 +-
 .../include/fdbclient/ParallelStream.actor.h  |  131 -
 .../include/fdbclient/RandomKeyValueUtils.h   |   30 +-
 fdbclient/include/fdbclient/ReadYourWrites.h  |    7 +-
 .../fdbclient/RunRYWTransaction.actor.h       |  135 -
 .../include/fdbclient/RunRYWTransaction.h     |  137 +
 ...unTransaction.actor.h => RunTransaction.h} |   64 +-
 fdbclient/include/fdbclient/S3BlobStore.h     |  449 +-
 .../{S3Client.actor.h => S3Client.h}          |   34 +-
 .../include/fdbclient/ServerKnobCollection.h  |   49 -
 fdbclient/include/fdbclient/SimpleIni.h       |   54 +-
 fdbclient/include/fdbclient/SnapshotCache.h   |    7 +-
 ...cialKeySpace.actor.h => SpecialKeySpace.h} |   72 +-
 fdbclient/include/fdbclient/Status.h          |    6 +-
 fdbclient/include/fdbclient/StatusClient.h    |    2 +-
 .../include/fdbclient/StorageCheckpoint.h     |    2 +-
 .../fdbclient/StorageServerInterface.h        |   87 +-
 ...Metrics.actor.h => StorageWiggleMetrics.h} |   33 +-
 fdbclient/include/fdbclient/Subspace.h        |    6 +-
 fdbclient/include/fdbclient/SystemData.h      |  159 +-
 .../{TagThrottle.actor.h => TagThrottle.h}    |  198 +-
 fdbclient/include/fdbclient/TaskBucket.h      |   26 +-
 .../include/fdbclient/TestKnobCollection.h    |   66 -
 .../include/fdbclient/ThreadSafeTransaction.h |    4 +-
 fdbclient/include/fdbclient/Tracing.h         |    4 +-
 .../include/fdbclient/TransactionLineage.h    |    3 +-
 fdbclient/include/fdbclient/Tuple.h           |    2 +-
 .../include/fdbclient/TupleVersionstamp.h     |    2 +-
 fdbclient/include/fdbclient/VersionVector.h   |    2 +-
 .../include/fdbclient/VersionedMap.actor.h    |   54 -
 fdbclient/include/fdbclient/VersionedMap.h    |  260 +-
 fdbclient/include/fdbclient/WriteMap.h        |   50 +-
 .../json_spirit/json_spirit_reader_template.h |    8 +-
 .../fdbclient/json_spirit/json_spirit_value.h |   23 +-
 fdbclient/sha1/SHA1.cpp                       |    4 +-
 fdbclient/{include/fdbclient => }/sha1/SHA1.h |    0
 fdbclient/tests/gcp_fixture.sh                |   94 +
 fdbclient/tests/s3client_test.sh              |   49 +-
 fdbclient/tests/tests_common.sh               |   85 +-
 fdbclient/vexillographer/fdb.options          |    5 +-
 fdbctl/CMakeLists.txt                         |    2 +-
 fdbctl/ControlCommands.cpp                    |   27 +-
 fdbctl/ExcludeCommand.cpp                     |   13 +-
 fdbctl/include/fdbctl/ControlService.h        |    3 +-
 fdbctl/protos/control_service.proto           |    2 +-
 fdbkubernetesmonitor/CMakeLists.txt           |   43 +
 fdbkubernetesmonitor/README.md                |    6 +-
 fdbkubernetesmonitor/api/version_test.go      |   17 +-
 fdbkubernetesmonitor/go.mod                   |   63 +-
 fdbkubernetesmonitor/go.sum                   |  185 +-
 .../internal/certloader/certloader.go         |   26 +-
 fdbkubernetesmonitor/kubernetes.go            |  169 +-
 fdbkubernetesmonitor/kubernetes_test.go       |  139 +-
 fdbkubernetesmonitor/main.go                  |    6 +-
 fdbkubernetesmonitor/monitor.go               |  183 +-
 fdbkubernetesmonitor/monitor_test.go          |   46 +-
 fdbmonitor/fdbmonitor.cpp                     |   17 +-
 fdbmonitor/fdbmonitor_lib.cpp                 |   22 +-
 ...leCached.actor.cpp => AsyncFileCached.cpp} |    4 +-
 fdbrpc/{include/fdbrpc => }/AsyncFileChaos.h  |    0
 .../AsyncFileEIO.actor.h => AsyncFileEIO.h}   |  210 +-
 ...ypted.actor.cpp => AsyncFileEncrypted.cpp} |  163 +-
 .../AsyncFileKAIO.actor.h => AsyncFileKAIO.h} |  150 +-
 fdbrpc/AsyncFileNonDurable.actor.cpp          |  226 -
 fdbrpc/AsyncFileNonDurable.cpp                |  215 +
 ...FileWinASIO.actor.h => AsyncFileWinASIO.h} |   16 +-
 fdbrpc/AsyncFileWriteChecker.cpp              |   12 +-
 ...hecker.actor.h => AsyncFileWriteChecker.h} |   33 +-
 fdbrpc/Base64Decode.cpp                       |    4 +-
 fdbrpc/{include/fdbrpc => }/Base64Decode.h    |    0
 fdbrpc/Base64Encode.cpp                       |    2 +-
 fdbrpc/{include/fdbrpc => }/Base64Encode.h    |    0
 fdbrpc/CMakeLists.txt                         |   25 +-
 .../{include/fdbrpc => }/ContinuousSample.h   |    0
 ...DSketchTest.actor.cpp => DDSketchTest.cpp} |    1 -
 ...reMonitor.actor.cpp => FailureMonitor.cpp} |   48 +-
 fdbrpc/FileTransfer.cpp                       |    2 +-
 fdbrpc/{include/fdbrpc => }/FileTransfer.h    |    3 +-
 fdbrpc/FlowGrpc.cpp                           |    4 +-
 fdbrpc/FlowGrpcTests.actor.cpp                |    2 +-
 fdbrpc/FlowGrpcTests.cpp                      |    6 +-
 fdbrpc/{include/fdbrpc => }/FlowGrpcTests.h   |    2 +-
 fdbrpc/FlowTests.actor.cpp                    |  143 +-
 ...wTransport.actor.cpp => FlowTransport.cpp} |  478 +-
 fdbrpc/{HTTP.actor.cpp => HTTP.cpp}           |  255 +-
 .../{HTTPServer.actor.cpp => HTTPServer.cpp}  |  202 +-
 ...lthMonitor.actor.cpp => HealthMonitor.cpp} |    4 +-
 fdbrpc/IPAllowList.cpp                        |    4 +-
 fdbrpc/JsonWebKeySet.cpp                      |    6 +-
 fdbrpc/{include/fdbrpc => }/JsonWebKeySet.h   |    4 +-
 fdbrpc/Locality.cpp                           |  182 +
 fdbrpc/Net2FileSystem.cpp                     |  111 +-
 fdbrpc/Net2FileSystemTests.cpp                |  156 +
 fdbrpc/ReplicationPolicy.cpp                  |   29 +-
 fdbrpc/ReplicationUtils.cpp                   |  221 +-
 ...on.actor.cpp => SimExternalConnection.cpp} |   99 +-
 .../fdbrpc => }/SimExternalConnection.h       |    4 +-
 fdbrpc/{Stats.actor.cpp => Stats.cpp}         |   37 +-
 .../bench}/BenchAsyncFileWriteCheckerLRU.cpp  |    9 +-
 .../bench}/BenchIONet2.actor.cpp              |    2 +-
 fdbrpc/bench/BenchMain.cpp                    |   35 +
 {flowbench => fdbrpc/bench}/BenchSamples.cpp  |    5 +-
 .../bench}/BenchSelectReplicas.cpp            |    5 +-
 .../utils.go => fdbrpc/bench/BenchSupport.h   |   53 +-
 fdbrpc/bench/CMakeLists.txt                   |   14 +
 fdbrpc/dsltest.actor.cpp                      |   19 +-
 ...ncFileCached.actor.h => AsyncFileCached.h} |  102 +-
 fdbrpc/include/fdbrpc/AsyncFileEncrypted.h    |   18 +-
 ...nDurable.actor.h => AsyncFileNonDurable.h} |  241 +-
 ...ReadAhead.actor.h => AsyncFileReadAhead.h} |   50 +-
 fdbrpc/include/fdbrpc/DDSketch.h              |    2 +-
 fdbrpc/include/fdbrpc/FlowGrpc.h              |    8 +-
 fdbrpc/include/fdbrpc/FlowProcess.actor.h     |   94 -
 fdbrpc/include/fdbrpc/LoadBalance.actor.h     |   76 +-
 fdbrpc/include/fdbrpc/Locality.h              |  125 +-
 fdbrpc/include/fdbrpc/MultiInterface.h        |    6 +-
 fdbrpc/include/fdbrpc/Net2FileSystem.h        |    2 +-
 fdbrpc/include/fdbrpc/PerfMetric.h            |    4 +-
 fdbrpc/include/fdbrpc/RangeMap.h              |    2 +-
 fdbrpc/include/fdbrpc/Replication.h           |   10 +-
 fdbrpc/include/fdbrpc/ReplicationPolicy.h     |    3 +-
 fdbrpc/include/fdbrpc/ReplicationTypes.h      |   10 +-
 fdbrpc/include/fdbrpc/SimulatorKillType.h     |    4 +-
 fdbrpc/include/fdbrpc/SimulatorProcessInfo.h  |    2 +-
 fdbrpc/include/fdbrpc/Stats.h                 |    6 +-
 fdbrpc/include/fdbrpc/TimedRequest.h          |    2 +
 fdbrpc/include/fdbrpc/fdbrpc.h                |   33 +-
 ...{genericactors.actor.h => genericactors.h} |  371 +-
 fdbrpc/include/fdbrpc/grpc/AsyncGrpcClient.h  |    6 +-
 .../include/fdbrpc/grpc/AsyncTaskExecutor.h   |    7 +-
 fdbrpc/include/fdbrpc/grpc/Credentials.h      |    4 +-
 ...{networksender.actor.h => networksender.h} |   25 +-
 fdbrpc/include/fdbrpc/simulator.h             |  181 +-
 fdbrpc/libeio/eio.c                           |    3 +-
 fdbrpc/{include/fdbrpc => }/linux_kaio.h      |    0
 fdbrpc/{sim2.actor.cpp => sim2.cpp}           |  679 +-
 fdbrpc/sim_validation.cpp                     |   38 +-
 fdbrpc/swift_sim2_hooks.cpp                   |    2 +-
 ...uthzTlsTest.actor.cpp => AuthzTlsTest.cpp} |   19 +-
 fdbrpc/tests/CMakeLists.txt                   |    6 +-
 ...dbrpc_bench.actor.cpp => fdbrpc_bench.cpp} |  100 +-
 .../BackupWorkerRangePartitioned.actor.cpp    |  252 -
 fdbserver/CMakeLists.txt                      |   79 +-
 fdbserver/GlobalTagThrottler.actor.cpp        | 1146 ----
 fdbserver/GrvProxyTagThrottler.actor.cpp      |  512 --
 fdbserver/LatencyBandsMap.actor.cpp           |  131 -
 fdbserver/RemoteIKeyValueStore.actor.cpp      |  251 -
 fdbserver/RestoreApplier.actor.cpp            |  825 ---
 fdbserver/RestoreCommon.actor.cpp             |  367 --
 fdbserver/RestoreController.actor.cpp         | 1222 ----
 fdbserver/RestoreLoader.actor.cpp             | 1583 -----
 fdbserver/RestoreRoleCommon.actor.cpp         |  219 -
 fdbserver/RestoreUtil.actor.cpp               |   79 -
 fdbserver/RestoreWorker.actor.cpp             |  423 --
 fdbserver/RestoreWorkerInterface.actor.cpp    |  102 -
 fdbserver/ServerThroughputTracker.cpp         |  192 -
 ...Cluster.actor.cpp => SimulatedCluster.cpp} |  640 +-
 fdbserver/TSSMappingUtil.actor.cpp            |   76 -
 .../BackupPartitionMap.cpp}                   |  124 +-
 fdbserver/backupworker/BackupPartitionMap.h   |   44 +
 .../BackupRangePartitionedProgress.cpp        |   73 +
 .../BackupRangePartitionedProgress.h          |   65 +
 .../BackupWorker.cpp}                         |  674 +-
 .../BackupWorkerRangePartitioned.cpp          | 1024 +++
 fdbserver/backupworker/CMakeLists.txt         |   15 +
 .../fdbserver/backupworker/BackupWorker.h     |   29 +
 fdbserver/clustercontroller/CMakeLists.txt    |   15 +
 .../ClusterController.actor.cpp               |  747 ++-
 .../ClusterController.h}                      |  214 +-
 .../ClusterHealthIFactor.cpp                  |  287 +
 .../clustercontroller/ClusterHealthIFactor.h  |   94 +
 .../ClusterHealthMonitor.cpp                  |  274 +
 .../clustercontroller/ClusterHealthMonitor.h  |  107 +
 .../ClusterHealthMonitorTesting.cpp           |  407 ++
 .../ClusterRecovery.cpp}                      |  859 +--
 .../ClusterRecovery.h}                        |   80 +-
 .../clustercontroller/RatekeeperMonitor.cpp   |   62 +
 .../clustercontroller/RatekeeperMonitor.h     |   46 +
 .../SingletonRoles.h                          |   12 +-
 .../Status.cpp}                               | 1432 +++--
 .../Status.h}                                 |   48 +-
 .../clustercontroller/ClusterController.h     |   34 +
 fdbserver/commitproxy/CMakeLists.txt          |   17 +
 .../CommitProxyServer.cpp}                    | 1369 ++--
 .../ProxyCommitData.h}                        |   88 +-
 .../fdbserver/commitproxy/CommitProxyServer.h |   32 +
 fdbserver/consistencyscan/CMakeLists.txt      |   12 +
 .../ConsistencyScan.cpp}                      |  871 +--
 .../consistencyscan/ConsistencyScan.h         |   64 +
 fdbserver/coordinator/CMakeLists.txt          |   16 +
 .../Coordination.cpp}                         |  737 ++-
 .../OnDemandStore.cpp}                        |   13 +-
 .../fdbserver => coordinator}/OnDemandStore.h |    2 +-
 .../coordinator/CoordinationServer.h          |   30 +
 .../{ => core}/AccumulativeChecksumUtil.cpp   |    6 +-
 .../BackupProgress.cpp}                       |   45 +-
 .../BulkDumpUtil.cpp}                         |  165 +-
 .../BulkLoadUtil.cpp}                         |  455 +-
 fdbserver/core/CMakeLists.txt                 |   34 +
 .../CoordinatedState.cpp}                     |  199 +-
 fdbserver/core/CoordinationInterface.cpp      |   55 +
 fdbserver/core/DataMovement.cpp               |   82 +
 fdbserver/core/FDBSimulationPolicy.cpp        |  401 ++
 fdbserver/{ => core}/LatencyBandConfig.cpp    |   10 +-
 fdbserver/{ => core}/LeaderElection.actor.cpp |   68 +-
 fdbserver/{ => core}/LogSystemConfig.cpp      |    6 +-
 fdbserver/core/MasterInterface.cpp            |    4 +
 .../{MoveKeys.actor.cpp => core/MoveKeys.cpp} | 1705 ++---
 fdbserver/{ => core}/MutationTracking.cpp     |   24 +-
 fdbserver/core/OpenDatabase.cpp               |   56 +
 .../QuietDatabase.cpp}                        |  638 +-
 fdbserver/core/RatekeeperLimitReasons.cpp     |   55 +
 .../RocksDBCheckpointUtils.cpp}               |  338 +-
 fdbserver/core/SeedShardServers.cpp           |  101 +
 .../ServerCheckpoint.cpp}                     |   44 +-
 {fdbclient => fdbserver/core}/ServerKnobs.cpp |  261 +-
 fdbserver/core/ShardSizing.cpp                |  102 +
 fdbserver/core/SimulatorSupport.cpp           |    7 +
 .../StorageMetrics.cpp}                       |  196 +-
 fdbserver/core/TSSMappingUtil.cpp             |   48 +
 .../WaitFailure.cpp}                          |   91 +-
 fdbserver/core/WorkerEvents.cpp               |   55 +
 fdbserver/core/WorkerInterface.cpp            |   33 +
 fdbserver/core/WorkerSupport.cpp              |  243 +
 fdbserver/core/WorkloadKeys.cpp               |   23 +
 .../core}/AccumulativeChecksumUtil.h          |    2 +-
 .../include/fdbserver/core}/BackupInterface.h |    0
 .../include/fdbserver/core/BackupProgress.h}  |   22 +-
 .../fdbserver/core/BackupProgressTypes.h      |   31 +
 .../include/fdbserver/core/BulkDumpUtil.h}    |   55 +-
 .../include/fdbserver/core/BulkLoadUtil.h     |   80 +
 .../include/fdbserver/core/ConflictBatch.h    |   37 +
 .../fdbserver/core}/CoordinatedState.h        |    4 +-
 .../fdbserver/core}/CoordinationInterface.h   |   16 +-
 .../include/fdbserver/core}/DBCoreState.h     |   11 +-
 .../core}/DataDistributorInterface.h          |   23 +-
 .../include/fdbserver/core/DataMovement.h     |   26 +
 .../fdbserver/core/FDBSimulationPolicy.h      |  117 +
 .../include/fdbserver/core}/IDiskQueue.h      |   10 +-
 .../core/include/fdbserver/core/Knobs.h       |  180 +-
 .../fdbserver/core}/LatencyBandConfig.h       |    0
 .../include/fdbserver/core}/LeaderElection.h  |    0
 .../fdbserver/core}/LogProtocolMessage.h      |    0
 .../include/fdbserver/core}/LogSystemConfig.h |   11 +-
 .../include/fdbserver/core}/MasterInterface.h |    7 +-
 .../include/fdbserver/core/MoveKeys.h}        |   48 +-
 .../fdbserver/core}/MutationTracking.h        |    0
 .../fdbserver/core}/OTELSpanContextMessage.h  |    2 +-
 .../include/fdbserver/core/QuietDatabase.h    |   61 +
 .../fdbserver/core}/RatekeeperInterface.h     |   11 +-
 .../fdbserver/core/RatekeeperLimitReasons.h   |   44 +
 .../include/fdbserver/core}/RecoveryState.h   |    2 +-
 .../fdbserver/core}/ResolverInterface.h       |    0
 .../fdbserver/core/RocksDBCheckpointUtils.h}  |   27 +-
 .../include/fdbserver/core/SeedShardServers.h |   32 +
 .../fdbserver/core/ServerCheckpoint.h}        |   31 +-
 .../include/fdbserver/core/ServerDBInfo.h}    |   38 +-
 .../include/fdbserver/core/ShardMetrics.h     |   49 +
 .../core/include/fdbserver/core/ShardSizing.h |   41 +
 .../fdbserver/core}/SpanContextMessage.h      |    2 +-
 .../include/fdbserver/core/StorageMetrics.h}  |  105 +-
 .../include/fdbserver/core}/TLogInterface.h   |    8 +-
 .../include/fdbserver/core/TSSMappingUtil.h   |   35 +
 .../include/fdbserver/core/TesterInterface.h} |   38 +-
 .../core/include/fdbserver/core/WaitFailure.h |   42 +
 .../include/fdbserver/core/WorkerEvents.h     |   34 +
 .../fdbserver/core}/WorkerInterface.actor.h   |  192 +-
 .../include/fdbserver/core/WorkloadKeys.h     |   32 +
 fdbserver/coroimpl/CoroFlow.actor.cpp         |    2 +-
 fdbserver/coroimpl/CoroFlowCoro.actor.cpp     |    4 +-
 fdbserver/datadistributor/AuditUtilsTests.cpp |  407 ++
 fdbserver/datadistributor/CMakeLists.txt      |   14 +
 .../DDRelocationQueue.actor.cpp               |  725 +--
 .../DDRelocationQueue.h                       |   16 +-
 .../DDShardTracker.cpp}                       |  574 +-
 fdbserver/datadistributor/DDSharedContext.cpp |   34 +
 .../DDTeamCollection.actor.cpp                | 1075 ++--
 .../DDTxnProcessor.cpp}                       |  480 +-
 .../DataDistribution.cpp}                     | 2490 ++++----
 .../ExclusionTracker.h}                       |   80 +-
 .../MockGlobalState.cpp}                      |  125 +-
 .../MovingWindow.h                            |    4 +-
 .../ShardsAffectedByTeamFailure.cpp           |    6 +-
 .../datadistributor/SimulatedCluster.cpp      |   96 +
 .../TCInfo.cpp}                               |  112 +-
 .../datadistributor}/DDShardTracker.h         |    4 +-
 .../datadistributor}/DDSharedContext.h        |   23 +-
 .../datadistributor}/DDTeamCollection.h       |   37 +-
 .../datadistributor}/DDTxnProcessor.h         |    8 +-
 .../datadistributor/DataDistribution.h}       |  126 +-
 .../datadistributor}/DataDistributionTeam.h   |    4 +-
 .../datadistributor/DataDistributor.h         |   30 +
 .../datadistributor}/MockDataDistributor.h    |    6 +-
 .../datadistributor}/MockGlobalState.h        |   15 +-
 .../ShardsAffectedByTeamFailure.h             |    0
 .../datadistributor}/SimulatedCluster.h       |    0
 .../fdbserver/datadistributor}/TCInfo.h       |    4 +-
 .../{fdbserver.actor.cpp => fdbserver.cpp}    |  384 +-
 fdbserver/grvproxy/CMakeLists.txt             |   15 +
 .../GrvProxyServer.cpp}                       |  640 +-
 fdbserver/grvproxy/GrvQueueDelay.cpp          |   91 +
 fdbserver/grvproxy/GrvQueueDelay.h            |   54 +
 fdbserver/grvproxy/GrvQueueDelayTests.cpp     |  181 +
 .../GrvTransactionRateInfo.cpp}               |   34 +-
 .../GrvTransactionRateInfo.h                  |    4 +
 .../fdbserver/grvproxy/GrvProxyServer.h       |   31 +
 .../include/fdbserver/BulkLoadUtil.actor.h    |   91 -
 .../include/fdbserver/FDBExecHelper.actor.h   |   91 -
 .../include/fdbserver/GrvProxyTagThrottler.h  |  104 -
 fdbserver/include/fdbserver/IKeyValueStore.h  |   82 -
 fdbserver/include/fdbserver/KmsConnector.h    |   48 -
 fdbserver/include/fdbserver/LatencyBandsMap.h |   60 -
 fdbserver/include/fdbserver/LogSystem.h       |  943 ---
 fdbserver/include/fdbserver/NetworkTest.h     |    6 +-
 fdbserver/include/fdbserver/QuietDatabase.h   |   65 -
 .../include/fdbserver/RESTKmsConnectorUtils.h |  116 -
 fdbserver/include/fdbserver/RESTSimKmsVault.h |   41 -
 .../fdbserver/RemoteIKeyValueStore.actor.h    |  498 --
 .../include/fdbserver/RestoreApplier.actor.h  |  417 --
 .../include/fdbserver/RestoreCommon.actor.h   |  391 --
 .../fdbserver/RestoreController.actor.h       |  464 --
 .../include/fdbserver/RestoreLoader.actor.h   |  244 -
 .../fdbserver/RestoreRoleCommon.actor.h       |  130 -
 fdbserver/include/fdbserver/RestoreUtil.h     |  113 -
 .../include/fdbserver/RestoreWorker.actor.h   |   74 -
 .../fdbserver/RestoreWorkerInterface.actor.h  |  739 ---
 .../fdbserver/ServerThroughputTracker.h       |   78 -
 .../fdbserver/SimEncryptKmsProxy.actor.h      |  152 -
 .../include/fdbserver/TSSMappingUtil.actor.h  |   50 -
 .../fdbserver/TagPartitionedLogSystem.actor.h |  451 --
 fdbserver/include/fdbserver/WaitFailure.h     |   50 -
 .../fdbserver => kvstore}/ArtMutationBuffer.h |   10 +-
 fdbserver/kvstore/CMakeLists.txt              |   29 +
 .../fdbserver => kvstore}/DeltaTree.h         |   10 +-
 .../DiskQueue.cpp}                            |  384 +-
 .../FDBExecHelper.cpp}                        |  237 +-
 .../IKeyValueContainer.h                      |    4 +-
 fdbserver/{ => kvstore}/IKeyValueStore.cpp    |   20 +-
 fdbserver/{ => kvstore}/IPager.cpp            |    6 +-
 .../{include/fdbserver => kvstore}/IPager.h   |   51 +
 .../KeyValueStoreCompressTestData.cpp}        |   49 +-
 .../KeyValueStoreMemory.cpp}                  |  183 +-
 .../KeyValueStoreRocksDB.actor.cpp            |  186 +-
 .../KeyValueStoreSQLite.cpp}                  |  123 +-
 .../KeyValueStoreShardedRocksDB.actor.cpp     |  159 +-
 .../fdbserver => kvstore}/RadixTree.h         |    8 +-
 fdbserver/{ => kvstore}/RocksDBCommon.cpp     |    4 +-
 .../fdbserver => kvstore}/RocksDBCommon.h     |    0
 .../RocksDBLogForwarder.cpp}                  |   15 +-
 .../RocksDBLogForwarder.h                     |    0
 .../TransactionStoreMutationTracking.cpp      |    8 +-
 .../TransactionStoreMutationTracking.h        |    0
 fdbserver/{ => kvstore}/VFSAsync.cpp          |   34 +-
 .../{include/fdbserver => kvstore}/VFSAsync.h |    0
 .../{ => kvstore}/VersionedBTree.actor.cpp    | 1512 ++---
 .../{ => kvstore}/VersionedBTreeDebug.cpp     |    4 +-
 .../VersionedBTreeDebug.h                     |    0
 .../{include/fdbserver => kvstore}/art.h      |    6 +-
 .../{include/fdbserver => kvstore}/art_impl.h |    0
 .../include/fdbserver/kvstore/FDBExecHelper.h |   57 +
 .../include/fdbserver/kvstore/IDiskQueue.h    |   33 +
 .../fdbserver/kvstore/IKeyValueStore.h        |   97 +-
 .../include/fdbserver/kvstore/KVFileUtils.h}  |   18 +-
 .../fdbserver => kvstore}/template_fdb.h      |    0
 fdbserver/logrouter/CMakeLists.txt            |   13 +
 fdbserver/{ => logrouter}/LogRouter.cpp       |   44 +-
 .../include/fdbserver/logrouter/LogRouter.h   |   31 +
 .../{ => logsystem}/ApplyMetadataMutation.cpp |   60 +-
 fdbserver/logsystem/CMakeLists.txt            |   16 +
 .../{LogSystem.cpp => logsystem/LogSet.cpp}   |  159 +-
 .../LogSystem.cpp}                            | 1455 +++--
 .../LogSystemDiskQueueAdapter.cpp}            |  106 +-
 fdbserver/logsystem/LogSystemFactory.cpp      |   44 +
 .../LogSystemPeekCursor.cpp}                  |  762 +--
 .../LogSystemRecoveryTests.cpp}               |   30 +-
 .../logsystem}/ApplyMetadataMutation.h        |   57 +-
 .../include/fdbserver/logsystem/LogSystem.h   |  692 ++
 .../logsystem}/LogSystemDiskQueueAdapter.h    |   19 +-
 .../fdbserver/logsystem/LogSystemFactory.h    |   52 +
 .../fdbserver/logsystem/LogSystemTypes.h      |  358 ++
 fdbserver/masterserver.swift                  |  494 --
 fdbserver/mocks3/CMakeLists.txt               |   11 +
 .../MockS3Server.cpp}                         |  481 +-
 .../MockS3ServerChaos.cpp}                    |   86 +-
 .../include/fdbserver/mocks3}/MockS3Server.h  |    2 +-
 .../fdbserver/mocks3}/MockS3ServerChaos.h     |    2 +-
 ...{networktest.actor.cpp => networktest.cpp} |  397 +-
 fdbserver/ratekeeper/CMakeLists.txt           |   14 +
 .../Ratekeeper.cpp}                           | 1071 ++--
 .../fdbserver => ratekeeper}/Ratekeeper.h     |   64 +-
 fdbserver/ratekeeper/RatekeeperLimits.cpp     |   41 +
 .../RkTagThrottleCollection.cpp               |    4 +-
 .../RkTagThrottleCollection.h                 |    9 +-
 .../TagThrottler.cpp}                         |   49 +-
 .../fdbserver => ratekeeper}/TagThrottler.h   |   52 +-
 .../fdbserver/ratekeeper/Ratekeeper.h}        |   17 +-
 fdbserver/resolver/CMakeLists.txt             |   15 +
 .../ConflictSet.cpp}                          |   53 +-
 .../fdbserver => resolver}/ConflictSet.h      |   13 +-
 .../Resolver.cpp}                             |  258 +-
 fdbserver/{ => resolver}/ResolverBug.cpp      |    2 +-
 .../include/fdbserver/resolver/Resolver.h     |   31 +
 .../include/fdbserver/resolver}/ResolverBug.h |    6 +-
 fdbserver/sequencer/CMakeLists.txt            |   14 +
 .../MasterData.h}                             |   92 +-
 .../ResolutionBalancer.cpp}                   |   50 +-
 .../ResolutionBalancer.h}                     |   25 +-
 .../fdbserver/sequencer/MasterServer.h        |   35 +
 .../masterserver.cpp}                         |  467 +-
 fdbserver/storageserver/CMakeLists.txt        |   17 +
 .../ReadLatencySamples.cpp                    |    4 +-
 .../ReadLatencySamples.h                      |    0
 .../StorageServerUtils.cpp                    |    4 +-
 .../StorageServerUtils.h                      |    2 +-
 .../TransactionTagCounter.cpp}                |   64 +-
 .../TransactionTagCounter.h                   |    2 +-
 .../storageserver}/StorageCorruptionBug.h     |    8 +-
 .../fdbserver/storageserver/StorageServer.h   |   49 +
 .../storageserver.actor.cpp                   | 5217 ++++++++-------
 fdbserver/test.cpp                            | 3324 ----------
 fdbserver/tester/CMakeLists.txt               |   17 +
 fdbserver/tester/ConsistencyChecker.cpp       |  760 +++
 fdbserver/tester/ConsistencyChecker.h         |   47 +
 .../CustomShardConfigWorkload.cpp}            |   57 +-
 fdbserver/tester/DatabaseMaintenance.cpp      |  264 +
 fdbserver/tester/DatabaseMaintenance.h        |   37 +
 .../{ => tester}/KnobProtectiveGroups.cpp     |    9 +-
 fdbserver/tester/TestSpecParser.cpp           |  413 ++
 fdbserver/tester/TestSpecParser.h             |   35 +
 fdbserver/tester/TesterServer.cpp             |  623 ++
 fdbserver/tester/TesterServer.h               |   28 +
 fdbserver/tester/WorkloadUtils.cpp            |  439 ++
 .../fdbserver/tester}/KnobProtectiveGroups.h  |   10 +-
 .../fdbserver/tester/TestEncryptionUtils.h    |   29 +
 .../include/fdbserver/tester/WorkloadUtils.h  |  129 +
 .../tester/include/fdbserver/tester/tester.h  |   51 +
 .../include/fdbserver/tester/workloads.h}     |  194 +-
 fdbserver/tester/test.cpp                     | 1085 ++++
 fdbserver/tlog/CMakeLists.txt                 |   15 +
 .../TLogServer.cpp}                           | 1451 +++--
 .../TestTLogServer.cpp}                       |  194 +-
 .../TestTLogServer.h}                         |   35 +-
 .../tlog/include/fdbserver/tlog/TLogServer.h  |   49 +
 fdbserver/worker/CMakeLists.txt               |   54 +
 .../MetricClient.cpp}                         |   25 +-
 .../fdbserver => worker}/MetricClient.h       |   11 +-
 fdbserver/{ => worker}/MetricLogger.actor.cpp |   20 +-
 .../fdbserver => worker}/MetricLogger.actor.h |   18 +-
 .../RoleLineage.cpp}                          |    2 +-
 .../RoleLineage.h}                            |   31 +-
 .../worker/include/fdbserver/worker/Worker.h  |   50 +
 fdbserver/{ => worker}/worker.actor.cpp       |  923 ++-
 ...rrectness.actor.cpp => ApiCorrectness.cpp} |  314 +-
 ...{ApiWorkload.actor.cpp => ApiWorkload.cpp} |  132 +-
 .../fdbserver => }/workloads/ApiWorkload.h    |   14 +-
 fdbserver/workloads/AsyncFile.cpp             |    4 +-
 .../AsyncFile.h}                              |   52 +-
 ...ess.actor.cpp => AsyncFileCorrectness.cpp} |   77 +-
 ...ncFileRead.actor.cpp => AsyncFileRead.cpp} |   95 +-
 ...FileWrite.actor.cpp => AsyncFileWrite.cpp} |   43 +-
 .../{AtomicOps.actor.cpp => AtomicOps.cpp}    |  175 +-
 ....actor.cpp => AtomicOpsApiCorrectness.cpp} |  398 +-
 ...micRestore.actor.cpp => AtomicRestore.cpp} |  105 +-
 ...tchover.actor.cpp => AtomicSwitchover.cpp} |  107 +-
 ...r.cpp => AutomaticIdempotencyWorkload.cpp} |  247 +-
 ...tors.actor.cpp => BackgroundSelectors.cpp} |  101 +-
 .../{Backup.actor.cpp => Backup.cpp}          |  186 +-
 ...kupAndParallelRestoreCorrectness.actor.cpp |  807 ---
 .../BackupAndRestoreValidation.actor.cpp      |  246 -
 .../workloads/BackupAndRestoreValidation.cpp  |  238 +
 ...ctness.actor.cpp => BackupCorrectness.cpp} |  557 +-
 ...r.cpp => BackupCorrectnessPartitioned.cpp} |  398 +-
 ....actor.cpp => BackupS3BlobCorrectness.cpp} |  475 +-
 ...oDBAbort.actor.cpp => BackupToDBAbort.cpp} |   68 +-
 ...ss.actor.cpp => BackupToDBCorrectness.cpp} |  421 +-
 ...pgrade.actor.cpp => BackupToDBUpgrade.cpp} |  284 +-
 ...{BulkDumping.actor.cpp => BulkDumping.cpp} |  279 +-
 .../{BulkLoad.actor.cpp => BulkLoad.cpp}      |   36 +-
 ...{BulkLoading.actor.cpp => BulkLoading.cpp} |  347 +-
 .../BulkSetup.h}                              |  201 +-
 fdbserver/workloads/CMakeLists.txt            |   17 +
 fdbserver/workloads/ChangeConfig.actor.cpp    |  231 -
 fdbserver/workloads/ChangeConfig.cpp          |  229 +
 ...leRange.actor.cpp => ClearSingleRange.cpp} |   36 +-
 ...lientMetric.actor.cpp => ClientMetric.cpp} |  121 +-
 ...> ClientTransactionProfileCorrectness.cpp} |   79 +-
 ...tWorkload.actor.cpp => ClientWorkload.cpp} |  155 +-
 ...emoteTLog.actor.cpp => ClogRemoteTLog.cpp} |  125 +-
 ...ion.actor.cpp => ClogSingleConnection.cpp} |    9 +-
 .../{ClogTlog.actor.cpp => ClogTlog.cpp}      |   82 +-
 ...tBugCheck.actor.cpp => CommitBugCheck.cpp} |  116 +-
 ...tabase.actor.cpp => ConfigureDatabase.cpp} |  159 +-
 ...flictRange.actor.cpp => ConflictRange.cpp} |  196 +-
 ...cyCheck.actor.cpp => ConsistencyCheck.cpp} |  454 +-
 ...t.actor.cpp => ConsistencyCheckUrgent.cpp} |  313 +-
 ...{CpuProfiler.actor.cpp => CpuProfiler.cpp} |   57 +-
 .../workloads/{Cycle.actor.cpp => Cycle.cpp}  |  104 +-
 .../{DDBalance.actor.cpp => DDBalance.cpp}    |  127 +-
 .../{DDMetrics.actor.cpp => DDMetrics.cpp}    |   37 +-
 ...Exclude.actor.cpp => DDMetricsExclude.cpp} |   47 +-
 ....actor.cpp => DataDistributionMetrics.cpp} |  151 +-
 ...ecovery.actor.cpp => DataLossRecovery.cpp} |  170 +-
 .../workloads/{DcLag.actor.cpp => DcLag.cpp}  |   65 +-
 ....actor.cpp => DifferentClustersSameRV.cpp} |  243 +-
 ...urability.actor.cpp => DiskDurability.cpp} |  109 +-
 ...yTest.actor.cpp => DiskDurabilityTest.cpp} |   92 +-
 ...ion.actor.cpp => DiskFailureInjection.cpp} |  152 +-
 ... ExcludeIncludeStorageServersWorkload.cpp} |   71 +-
 .../ExpectStableThroughput.actor.cpp          |  133 -
 ...orkload.actor.cpp => ExternalWorkload.cpp} |   33 +-
 ...hSSLag.actor.cpp => FailoverWithSSLag.cpp} |  128 +-
 ...hes.actor.cpp => FastTriggeredWatches.cpp} |  101 +-
 .../{FileSystem.actor.cpp => FileSystem.cpp}  |  160 +-
 ...tness.actor.cpp => FuzzApiCorrectness.cpp} |  262 +-
 fdbserver/workloads/GcGenerations.actor.cpp   |  221 -
 fdbserver/workloads/GcGenerations.cpp         |  267 +
 ...ze.actor.cpp => GetEstimatedRangeSize.cpp} |   49 +-
 ...ppedRange.actor.cpp => GetMappedRange.cpp} |  427 +-
 ...ngeStream.actor.cpp => GetRangeStream.cpp} |   98 +-
 ...eStore.actor.cpp => HTTPKeyValueStore.cpp} |  162 +-
 ...ricsApi.actor.cpp => HealthMetricsApi.cpp} |   71 +-
 ...HighContentionPrefixAllocatorWorkload.cpp} |   89 +-
 ....cpp => IDDTxnProcessorApiCorrectness.cpp} |  273 +-
 .../{Increment.actor.cpp => Increment.cpp}    |   65 +-
 ...Backup.actor.cpp => IncrementalBackup.cpp} |  251 +-
 .../{IndexScan.actor.cpp => IndexScan.cpp}    |   88 +-
 .../{Inventory.actor.cpp => Inventory.cpp}    |   72 +-
 ...{KVStoreTest.actor.cpp => KVStoreTest.cpp} |  120 +-
 .../{KillRegion.actor.cpp => KillRegion.cpp}  |   69 +-
 ...tekeeper.actor.cpp => LocalRatekeeper.cpp} |   80 +-
 ...ockDatabase.actor.cpp => LockDatabase.cpp} |   91 +-
 ...y.actor.cpp => LockDatabaseFrequently.cpp} |   40 +-
 .../{LogMetrics.actor.cpp => LogMetrics.cpp}  |   50 +-
 .../{LowLatency.actor.cpp => LowLatency.cpp}  |   62 +-
 ...trition.actor.cpp => MachineAttrition.cpp} |  228 +-
 .../workloads/{Mako.actor.cpp => Mako.cpp}    |  482 +-
 fdbserver/workloads/MaxGrvQueueDelay.cpp      |  208 +
 fdbserver/workloads/MemoryKeyValueStore.cpp   |   14 +-
 .../workloads/MemoryKeyValueStore.h           |    2 +-
 ...yLifetime.actor.cpp => MemoryLifetime.cpp} |   99 +-
 ...ricLogging.actor.cpp => MetricLogging.cpp} |   33 +-
 .../{MiniCycle.actor.cpp => MiniCycle.cpp}    |  127 +-
 ...eadWrite.actor.cpp => MockDDReadWrite.cpp} |   11 +-
 .../{MockDDTest.actor.cpp => MockDDTest.cpp}  |    7 +-
 .../fdbserver => }/workloads/MockDDTest.h     |    8 +-
 ...or.cpp => MockDDTrackerShardEvaluator.cpp} |   41 +-
 ...r.cpp => MutationLogReaderCorrectness.cpp} |   46 +-
 fdbserver/workloads/ParallelRestore.actor.cpp |   61 -
 ...{Performance.actor.cpp => Performance.cpp} |   78 +-
 .../PerpetualWiggleStatsWorkload.actor.cpp    |  241 -
 .../PerpetualWiggleStatsWorkload.cpp          |  233 +
 ...rpetualWiggleStorageMigrationWorkload.cpp} |  117 +-
 ...rdMove.actor.cpp => PhysicalShardMove.cpp} |  448 +-
 .../workloads/{Ping.actor.cpp => Ping.cpp}    |  111 +-
 ...dpoints.actor.cpp => PrivateEndpoints.cpp} |   48 +-
 ...lVersion.actor.cpp => ProtocolVersion.cpp} |   18 +-
 ...ultiples.actor.cpp => PubSubMultiples.cpp} |   54 +-
 .../{QueuePush.actor.cpp => QueuePush.cpp}    |   50 +-
 .../{RYWDisable.actor.cpp => RYWDisable.cpp}  |   44 +-
 fdbserver/workloads/RYWPerformance.actor.cpp  |  314 -
 fdbserver/workloads/RYWPerformance.cpp        |  318 +
 ...mClogging.actor.cpp => RandomClogging.cpp} |   74 +-
 ...mMoveKeys.actor.cpp => RandomMoveKeys.cpp} |  109 +-
 ...angeLock.actor.cpp => RandomRangeLock.cpp} |   57 +-
 ...mSelector.actor.cpp => RandomSelector.cpp} |  436 +-
 .../{RangeLock.actor.cpp => RangeLock.cpp}    |  258 +-
 ...fterWrite.actor.cpp => ReadAfterWrite.cpp} |   72 +-
 ...tection.actor.cpp => ReadHotDetection.cpp} |  108 +-
 .../{ReadWrite.actor.cpp => ReadWrite.cpp}    |  613 +-
 .../ReadWriteWorkload.h}                      |   62 +-
 ...fely.actor.cpp => RemoveServersSafely.cpp} |  247 +-
 ...ys.actor.cpp => ReportConflictingKeys.cpp} |   80 +-
 ...{ResolverBug.actor.cpp => ResolverBug.cpp} |   78 +-
 .../{Restore.actor.cpp => Restore.cpp}        |  212 +-
 ...toreBackup.actor.cpp => RestoreBackup.cpp} |  109 +-
 ...anges.actor.cpp => RestoreMultiRanges.cpp} |  145 +-
 ...dation.actor.cpp => RestoreValidation.cpp} |  178 +-
 .../{Rollback.actor.cpp => Rollback.cpp}      |   47 +-
 ...rectness.actor.cpp => RyowCorrectness.cpp} |   91 +-
 ...orkload.actor.cpp => S3ClientWorkload.cpp} |   96 +-
 ...{SaveAndKill.actor.cpp => SaveAndKill.cpp} |   43 +-
 ...ness.actor.cpp => SelectorCorrectness.cpp} |  101 +-
 ...zability.actor.cpp => Serializability.cpp} |  343 +-
 .../{Sideband.actor.cpp => Sideband.cpp}      |   88 +-
 ...andSingle.actor.cpp => SidebandSingle.cpp} |   90 +-
 ...tomicAdd.actor.cpp => SimpleAtomicAdd.cpp} |   88 +-
 ...eadWrite.actor.cpp => SkewedReadWrite.cpp} |  172 +-
 ...orkload.actor.cpp => SlowTaskWorkload.cpp} |   13 +-
 .../{SnapTest.actor.cpp => SnapTest.cpp}      |  157 +-
 ...tor.cpp => SpecialKeySpaceCorrectness.cpp} |  564 +-
 ...ctor.cpp => SpecialKeySpaceRobustness.cpp} |  446 +-
 ...sWorkload.actor.cpp => StatusWorkload.cpp} |   73 +-
 ...uption.actor.cpp => StorageCorruption.cpp} |   33 +-
 ...=> StorageServerCheckpointRestoreTest.cpp} |  160 +-
 .../{Storefront.actor.cpp => Storefront.cpp}  |   90 +-
 ...eRead.actor.cpp => StreamingRangeRead.cpp} |   92 +-
 ...eamingRead.actor.cpp => StreamingRead.cpp} |   52 +-
 ...ubmitBackup.actor.cpp => SubmitBackup.cpp} |   52 +-
 ...ocesses.actor.cpp => SuspendProcesses.cpp} |   39 +-
 ...rottleApi.actor.cpp => TagThrottleApi.cpp} |  119 +-
 ...argetedKill.actor.cpp => TargetedKill.cpp} |   43 +-
 ...ss.actor.cpp => TaskBucketCorrectness.cpp} |  193 +-
 ...hreadSafety.actor.cpp => ThreadSafety.cpp} |   56 +-
 .../{Throttling.actor.cpp => Throttling.cpp}  |  109 +-
 .../{Throughput.actor.cpp => Throughput.cpp}  |  146 +-
 fdbserver/workloads/ThroughputQuota.actor.cpp |   69 -
 ...ss.actor.cpp => TimeKeeperCorrectness.cpp} |   66 +-
 ...tionCost.actor.cpp => TransactionCost.cpp} |  129 +-
 ...Recovery.actor.cpp => TriggerRecovery.cpp} |   93 +-
 .../{TxnTimeout.actor.cpp => TxnTimeout.cpp}  |  205 +-
 fdbserver/workloads/UDPWorkload.actor.cpp     |  269 -
 fdbserver/workloads/UDPWorkload.cpp           |  266 +
 .../{UnitPerf.actor.cpp => UnitPerf.cpp}      |   24 +-
 .../{UnitTests.actor.cpp => UnitTests.cpp}    |   89 +-
 .../{Unreadable.actor.cpp => Unreadable.cpp}  |   59 +-
 ...eStorage.actor.cpp => ValidateStorage.cpp} |  483 +-
 ...ersionStamp.actor.cpp => VersionStamp.cpp} |  276 +-
 ...atchAndWait.actor.cpp => WatchAndWait.cpp} |   51 +-
 .../{Watches.actor.cpp => Watches.cpp}        |   42 +-
 ...ctor.cpp => WatchesSameKeyCorrectness.cpp} |  190 +-
 ...orkerErrors.actor.cpp => WorkerErrors.cpp} |   33 +-
 ...Bandwidth.actor.cpp => WriteBandwidth.cpp} |   60 +-
 ...ringRead.actor.cpp => WriteDuringRead.cpp} |  809 +--
 ...tling.actor.cpp => WriteTagThrottling.cpp} |  111 +-
 .../pubsub.cpp}                               |  312 +-
 .../{include/fdbserver => workloads}/pubsub.h |    2 +-
 flow/ActorCollection.actor.cpp                |    8 +-
 flow/Arena.cpp                                |   93 +-
 flow/CMakeLists.txt                           |   20 +-
 flow/CodeProbe.cpp                            |   57 +-
 ...pressedInt.actor.cpp => CompressedInt.cpp} |   10 +-
 flow/CompressionUtils.cpp                     |    2 +-
 flow/{include/flow => }/CompressionUtils.h    |    0
 {fdbrpc => flow}/CoroTests.cpp                |  785 ++-
 flow/Deque.cpp                                |    2 +-
 flow/DeterministicRandom.cpp                  |   24 +-
 flow/Error.cpp                                |    3 -
 .../flow/EventTypes.actor.h => EventTypes.h}  |   23 +-
 flow/FileTraceLogWriter.cpp                   |    6 +-
 flow/{include/flow => }/FileTraceLogWriter.h  |    0
 flow/FlowTest.cpp                             |  362 ++
 flow/Histogram.cpp                            |    1 -
 flow/{Hostname.actor.cpp => Hostname.cpp}     |   60 +-
 flow/IAsyncFile.actor.cpp                     |  203 -
 flow/IAsyncFile.cpp                           |   81 +
 flow/IThreadPool.cpp                          |   11 +-
 ...PoolTest.actor.cpp => IThreadPoolTest.cpp} |  103 +-
 flow/IndexedSet.cpp                           |    8 +-
 flow/JsonTraceLogFormatter.cpp                |    3 +-
 .../flow => }/JsonTraceLogFormatter.h         |    0
 flow/Knobs.cpp                                |   47 +-
 flow/{include/flow => }/MetricSample.h        |    0
 flow/MkCert.cpp                               |    1 -
 flow/MkCertCli.cpp                            |    2 +-
 flow/{Net2.actor.cpp => Net2.cpp}             |  482 +-
 flow/Net2Packet.cpp                           |    2 +-
 flow/{Platform.actor.cpp => Platform.cpp}     |  161 +-
 flow/ProcessEvents.cpp                        |    2 +-
 flow/{Profiler.actor.cpp => Profiler.cpp}     |   35 +-
 flow/SimBugInjector.cpp                       |    9 +-
 flow/SimpleCounter.cpp                        |    6 +-
 flow/StreamCipher.cpp                         |   10 +-
 flow/SystemMonitor.cpp                        |   48 +-
 flow/TDMetric.cpp                             |    8 +-
 flow/{TLSConfig.actor.cpp => TLSConfig.cpp}   |   74 +-
 ...hreadHelper.actor.cpp => ThreadHelper.cpp} |   21 +-
 flow/ThreadPrimitives.cpp                     |    6 -
 flow/Trace.cpp                                |   75 +-
 flow/WipedString.cpp                          |    1 -
 ...riteOnlySet.actor.cpp => WriteOnlySet.cpp} |   17 +-
 flow/XmlTraceLogFormatter.cpp                 |   59 +-
 .../{include/flow => }/XmlTraceLogFormatter.h |    0
 flow/actorcompiler/ActorCompiler.cs           |   73 +-
 flow/actorcompiler/ActorParser.cs             |   84 -
 flow/actorcompiler/ParseTree.cs               |   14 -
 flow/actorcompiler/usertype.dat               |    1 -
 flow/actorcompiler_py/actor_compiler.py       |   74 +-
 flow/actorcompiler_py/actor_parser.py         |   89 +-
 flow/actorcompiler_py/parse_tree.py           |   14 -
 flow/bench/BenchAsyncResult.cpp               |   96 +
 {flowbench => flow/bench}/BenchCallback.cpp   |    0
 .../bench/BenchCallbackActor.cpp              |   21 +-
 .../bench}/BenchConflictSet.cpp               |    0
 flow/bench/BenchCoroChooseRace.cpp            |  278 +
 flow/bench/BenchCoroutineOverhead.cpp         |   96 +
 {flowbench => flow/bench}/BenchHash.cpp       |   13 +-
 flow/bench/BenchMain.cpp                      |   25 +
 {flowbench => flow/bench}/BenchMem.cpp        |    0
 flow/bench/BenchMemcpy.cpp                    |  242 +
 {flowbench => flow/bench}/BenchNet2.cpp       |    2 +-
 .../bench/BenchNet2Actor.cpp                  |   53 +-
 flow/bench/BenchNoThrowOnCancel.cpp           |  158 +
 .../bench/BenchPriorityMultiLock.cpp          |   29 +-
 {flowbench => flow/bench}/BenchRandom.cpp     |    0
 {flowbench => flow/bench}/BenchRef.cpp        |    0
 .../bench/BenchStream.cpp                     |   25 +-
 .../bench/BenchSupport.h                      |   36 +-
 flow/bench/BenchTimeout.cpp                   |   95 +
 {flowbench => flow/bench}/BenchTimer.cpp      |    0
 flow/bench/BenchWaitForAllReady.cpp           |  100 +
 {flowbench => flow/bench}/BenchZstd.cpp       |    2 +-
 flow/bench/CMakeLists.txt                     |   18 +
 {flowbench => flow/bench}/README.md           |   21 +-
 flow/flat_buffers.cpp                         |   10 +-
 flow/flow.cpp                                 |   83 +-
 flow/genericactors.actor.cpp                  |  140 +-
 flow/genericcoros.h                           |  215 +
 flow/include/flow/ActorContext.h              |    4 +-
 flow/include/flow/Arena.h                     |  137 +-
 flow/include/flow/AsioReactor.h               |    2 +-
 .../include/flow/BenchMain.h                  |   36 +-
 flow/include/flow/CodeProbe.h                 |   18 +-
 flow/include/flow/CompressedInt.h             |    2 +-
 flow/include/flow/CoroUtils.h                 |  208 +-
 flow/include/flow/Coroutines.h                |  248 +-
 flow/include/flow/CoroutinesImpl.h            |  996 ++-
 flow/include/flow/Deque.h                     |    4 +-
 flow/include/flow/DeterministicRandom.h       |   12 +-
 flow/include/flow/Error.h                     |    4 +-
 flow/include/flow/FastRef.h                   |   12 +-
 flow/include/flow/FlowThread.h                |    8 +-
 flow/include/flow/Histogram.h                 |   12 +-
 flow/include/flow/IConnection.h               |   17 +-
 flow/include/flow/IPAddress.h                 |    2 +-
 flow/include/flow/IRandom.h                   |   10 +-
 flow/include/flow/IThreadPool.h               |    2 +-
 flow/include/flow/IndexedSet.actor.h          |   75 -
 flow/include/flow/IndexedSet.h                |   78 +-
 flow/include/flow/Knobs.h                     |    7 +-
 flow/include/flow/MkCert.h                    |    2 +-
 flow/include/flow/OTELMetrics.h               |    9 +-
 flow/include/flow/ObjectSerializer.h          |    4 +-
 flow/include/flow/ObjectSerializerTraits.h    |   10 +-
 flow/include/flow/Optional.h                  |   37 +-
 flow/include/flow/OwningResource.h            |   18 +-
 flow/include/flow/PKey.h                      |    4 +-
 flow/include/flow/Platform.actor.h            |   51 -
 flow/include/flow/Platform.h                  |   10 +-
 ...yMultiLock.actor.h => PriorityMultiLock.h} |   80 +-
 flow/include/flow/ScopeExit.h                 |    2 +-
 flow/include/flow/SendBufferIterator.h        |    2 +-
 flow/include/flow/SimpleCounter.h             |   13 +-
 flow/include/flow/StreamCipher.h              |    4 +-
 flow/include/flow/SystemMonitor.h             |   45 +-
 .../flow/{TDMetric.actor.h => TDMetric.h}     |  164 +-
 .../flow/{TLSConfig.actor.h => TLSConfig.h}   |   15 +-
 flow/include/flow/TaskQueue.h                 |    4 +-
 flow/include/flow/ThreadHelper.actor.h        |   55 +-
 flow/include/flow/ThreadPrimitives.h          |    6 +-
 flow/include/flow/ThreadSafeQueue.h           |    4 +-
 flow/include/flow/Trace.h                     |   44 +-
 flow/include/flow/Traceable.h                 |    6 +-
 flow/include/flow/TreeBenchmark.h             |    2 +-
 flow/include/flow/TxnCounters.h               |   42 +
 flow/include/flow/Util.h                      |  119 +-
 .../flow/{WatchFile.actor.h => WatchFile.h}   |   39 +-
 flow/include/flow/WipedString.h               |    4 +-
 flow/include/flow/actorcompiler.h             |    1 -
 flow/include/flow/error_definitions.h         |    9 +-
 flow/include/flow/flat_buffers.h              |   79 +-
 flow/include/flow/flow.h                      |   94 +-
 flow/include/flow/genericactors.actor.h       |  812 +--
 flow/include/flow/network.h                   |    2 +-
 flow/include/flow/serialize.h                 |   20 +-
 flow/include/flow/singleton.h                 |    4 +-
 flow/include/flow/swift/ABI/MetadataValues.h  |    2 +-
 flow/include/flow/swift/Basic/FlagSet.h       |    2 +-
 flow/include/flow/swift_concurrency_hooks.h   |    2 +-
 flow/include/flow/unactorcompiler.h           |    1 -
 flow/network.cpp                              |   56 +-
 flow/{include/flow => }/rte_memcpy.h          |    0
 flow/swift_concurrency_hooks.cpp              |    2 +-
 flow/swift_task_priority.cpp                  |    2 +-
 flow/test_memcpy.cpp                          |    2 +-
 flow/test_memcpy_perf.cpp                     |  341 -
 flowbench/CMakeLists.txt                      |   66 -
 packaging/docker/Dockerfile                   |   29 +-
 packaging/docker/build-images.sh              |    7 +
 packaging/docker/kubernetes/test_config.yaml  |   78 +-
 .../samples/python/app/requirements.txt       |    2 +-
 packaging/docker/sidecar_test.py              |   16 +-
 packaging/multiversion/server/postinst        |    2 +
 packaging/osx/buildpkg.sh                     |    6 +-
 pull_request_template.md                      |   28 +-
 recipes/java-recipes/MicroSpatial.java        |    2 +-
 tests/CMakeLists.txt                          |   26 +-
 ...llelRestoreApiCorrectnessAtomicRestore.txt |   36 -
 ...eOldBackupApiCorrectnessAtomicRestore.toml |   40 -
 .../TestRunner/fdb_test_runner/fdb_version.py |    4 +
 tests/TestRunner/requirements.txt             |    5 +-
 tests/fast/BulkDumping.toml                   |    2 +-
 tests/fast/BulkLoading.toml                   |    2 +-
 tests/fast/EncryptionUnitTests.toml           |   17 -
 tests/fast/MaxGrvQueueDelay.toml              |   19 +
 tests/fast/RESTUnit.toml                      |   21 -
 tests/fast/StatusDuringOutage.toml            |   26 +
 tests/fast/TxnStateStoreCycleTest.toml        |    2 +
 tests/loopback_cluster/run_custom_cluster.sh  |    1 +
 tests/rare/ExpectStableThroughput.toml        |   20 -
 tests/rare/ThroughputQuota.toml               |   13 -
 tests/slow/ApiCorrectnessAtomicRestore.toml   |    2 +-
 tests/slow/BackupAndRestore.toml              |    2 +-
 tests/slow/BackupNewAndOldRestore.toml        |    4 +-
 tests/slow/BackupOldAndNewRestore.toml        |    4 +-
 tests/slow/BackupS3BlobBulkLoadRestore.toml   |    2 +-
 ...BackupS3BlobBulkLoadRestoreMultiRange.toml |    2 +-
 .../BackupS3BlobBulkLoadRestoreWithChaos.toml |    2 +-
 tests/slow/BulkDumpingS3.toml                 |    2 +-
 tests/slow/BulkDumpingS3WithChaos.toml        |    2 +-
 tests/slow/DiskFailureCycle.toml              |    3 +-
 ...elRestoreNewBackupCorrectnessAtomicOp.toml |   62 -
 ...allelRestoreNewBackupCorrectnessCycle.toml |   57 -
 ...estoreNewBackupCorrectnessMultiCycles.toml |   80 -
 ...NewBackupWriteDuringReadAtomicRestore.toml |   52 -
 ...elRestoreOldBackupCorrectnessAtomicOp.toml |   60 -
 ...allelRestoreOldBackupCorrectnessCycle.toml |   57 -
 ...estoreOldBackupCorrectnessMultiCycles.toml |   80 -
 ...OldBackupWriteDuringReadAtomicRestore.toml |   52 -
 tests/slow/WriteDuringReadAtomicRestore.toml  |    2 +-
 1151 files changed, 86684 insertions(+), 77932 deletions(-)
 create mode 100644 .clang-tidy
 create mode 100644 .github/workflows/stale.yml
 rename bindings/flow/{DirectoryLayer.actor.cpp => DirectoryLayer.cpp} (56%)
 rename bindings/flow/{HighContentionAllocator.actor.cpp => HighContentionAllocator.cpp} (73%)
 rename bindings/flow/{Node.actor.cpp => Node.cpp} (89%)
 rename bindings/flow/{fdb_flow.actor.cpp => fdb_flow.cpp} (95%)
 rename bindings/flow/tester/{DirectoryTester.actor.cpp => DirectoryTester.cpp} (64%)
 rename bindings/flow/tester/{Tester.actor.cpp => Tester.cpp} (66%)
 rename bindings/flow/tester/{Tester.actor.h => Tester.h} (82%)
 create mode 100644 bindings/go/src/fdb/error_codes_generated.go
 create mode 100644 bindings/go/src/internal/gen_errors/main.go
 rename bindings/go/src/{_util => internal}/translate_fdb_options.go (92%)
 create mode 100644 cmake/AddFdbServerLinkTest.cmake
 create mode 100644 cmake/FDBBenchmark.cmake
 rename flowbench/benchmark.cmake => cmake/benchmark-download.cmake (79%)
 delete mode 100644 contrib/mockkms/CMakeLists.txt
 delete mode 100644 contrib/mockkms/fault_injection.go
 delete mode 100644 contrib/mockkms/get_encryption_keys.go
 delete mode 100644 contrib/mockkms/mock_kms.go
 delete mode 100644 contrib/mockkms/mockkms_test.go
 create mode 100644 design/AI-generated/FDB_NETWORK_PROTOCOL.md
 create mode 100644 design/AI-generated/README.md
 create mode 100644 design/AI-generated/diagram_01_flow_runtime.md
 create mode 100644 design/AI-generated/diagram_02_rpc_transport.md
 create mode 100644 design/AI-generated/diagram_03_client_library.md
 create mode 100644 design/AI-generated/diagram_04_cluster_controller.md
 create mode 100644 design/AI-generated/diagram_05_commit_pipeline.md
 create mode 100644 design/AI-generated/diagram_06_tlog_logsystem.md
 create mode 100644 design/AI-generated/diagram_07_storage_server.md
 create mode 100644 design/AI-generated/diagram_08_data_distribution.md
 create mode 100644 design/AI-generated/diagram_09_cluster_recovery.md
 create mode 100644 design/AI-generated/diagram_10_ratekeeper.md
 create mode 100644 design/AI-generated/diagram_11_backup_dr.md
 create mode 100644 design/AI-generated/diagram_12_simulation_testing.md
 create mode 100644 design/AI-generated/diagram_overview.md
 create mode 100644 design/AI-generated/foundationdb_subsystem_map.md
 create mode 100644 design/AI-generated/subsystem_01_flow_runtime.md
 create mode 100644 design/AI-generated/subsystem_02_rpc_transport.md
 create mode 100644 design/AI-generated/subsystem_03_client_library.md
 create mode 100644 design/AI-generated/subsystem_04_cluster_controller.md
 create mode 100644 design/AI-generated/subsystem_05_commit_pipeline.md
 create mode 100644 design/AI-generated/subsystem_06_tlog_logsystem.md
 create mode 100644 design/AI-generated/subsystem_07_storage_server.md
 create mode 100644 design/AI-generated/subsystem_08_data_distribution.md
 create mode 100644 design/AI-generated/subsystem_09_cluster_recovery.md
 create mode 100644 design/AI-generated/subsystem_10_ratekeeper.md
 create mode 100644 design/AI-generated/subsystem_11_backup_dr.md
 create mode 100644 design/AI-generated/subsystem_12_simulation_testing.md
 create mode 100644 design/cluster_health_metric.md
 create mode 100644 design/fdb-coroutines-internals.md
 create mode 100644 design/file-level-encryption-backup.md
 delete mode 100644 design/global-tag-throttling.md
 create mode 100644 documentation/sphinx/source/clang-format.rst
 create mode 100644 documentation/sphinx/source/clang-tidy.rst
 create mode 100644 documentation/sphinx/source/developer-tools.rst
 rename fdbbackup/{FileConverter.actor.cpp => FileConverter.cpp} (85%)
 rename fdbbackup/{FileDecoder.actor.cpp => FileDecoder.cpp} (86%)
 rename fdbbackup/{backup.actor.cpp => backup.cpp} (82%)
 rename fdbbackup/tests/{s3_backup_test.sh => blob_backup_restore_test.sh} (85%)
 rename fdbcli/{AdvanceVersionCommand.actor.cpp => AdvanceVersionCommand.cpp} (77%)
 rename fdbcli/{AuditStorageCommand.actor.cpp => AuditStorageCommand.cpp} (89%)
 rename fdbcli/{BulkDumpCommand.actor.cpp => BulkDumpCommand.cpp} (60%)
 rename fdbcli/{BulkLoadCommand.actor.cpp => BulkLoadCommand.cpp} (68%)
 rename fdbcli/{ConfigureCommand.actor.cpp => ConfigureCommand.cpp} (90%)
 rename fdbcli/{ConsistencyCheckCommand.actor.cpp => ConsistencyCheckCommand.cpp} (72%)
 rename fdbcli/{ConsistencyScanCommand.actor.cpp => ConsistencyScanCommand.cpp} (77%)
 rename fdbcli/{CoordinatorsCommand.actor.cpp => CoordinatorsCommand.cpp} (70%)
 rename fdbcli/{DataDistributionCommand.actor.cpp => DataDistributionCommand.cpp} (72%)
 rename fdbcli/{DebugCommands.actor.cpp => DebugCommands.cpp} (77%)
 rename fdbcli/{ExcludeCommand.actor.cpp => ExcludeCommand.cpp} (69%)
 rename fdbcli/{ExpensiveDataCheckCommand.actor.cpp => ExpensiveDataCheckCommand.cpp} (84%)
 rename fdbcli/{FileConfigureCommand.actor.cpp => FileConfigureCommand.cpp} (87%)
 rename fdbcli/{FlowLineNoise.actor.cpp => FlowLineNoise.cpp} (93%)
 rename fdbcli/{ForceRecoveryWithDataLossCommand.actor.cpp => ForceRecoveryWithDataLossCommand.cpp} (82%)
 rename fdbcli/{GetAuditStatusCommand.actor.cpp => GetAuditStatusCommand.cpp} (66%)
 rename fdbcli/{HotRangeCommand.actor.cpp => HotRangeCommand.cpp} (83%)
 rename fdbcli/{IdempotencyIdsCommand.actor.cpp => IdempotencyIdsCommand.cpp} (80%)
 rename fdbcli/{IncludeCommand.actor.cpp => IncludeCommand.cpp} (72%)
 rename fdbcli/{KillCommand.actor.cpp => KillCommand.cpp} (80%)
 rename fdbcli/{LocationMetadataCommand.actor.cpp => LocationMetadataCommand.cpp} (63%)
 rename fdbcli/{LockCommand.actor.cpp => LockCommand.cpp} (59%)
 rename fdbcli/{MaintenanceCommand.actor.cpp => MaintenanceCommand.cpp} (76%)
 rename fdbcli/{ProfileCommand.actor.cpp => ProfileCommand.cpp} (85%)
 rename fdbcli/{RangeConfigCommand.actor.cpp => RangeConfigCommand.cpp} (78%)
 rename fdbcli/{SetClassCommand.actor.cpp => SetClassCommand.cpp} (70%)
 rename fdbcli/{SnapshotCommand.actor.cpp => SnapshotCommand.cpp} (79%)
 rename fdbcli/{StatusCommand.actor.cpp => StatusCommand.cpp} (92%)
 rename fdbcli/{SuspendCommand.actor.cpp => SuspendCommand.cpp} (78%)
 rename fdbcli/{ThrottleCommand.actor.cpp => ThrottleCommand.cpp} (90%)
 rename fdbcli/{TriggerDDTeamInfoLogCommand.actor.cpp => TriggerDDTeamInfoLogCommand.cpp} (79%)
 rename fdbcli/{TssqCommand.actor.cpp => TssqCommand.cpp} (67%)
 delete mode 100644 fdbcli/Util.actor.cpp
 create mode 100644 fdbcli/Util.cpp
 rename fdbcli/{VersionEpochCommand.actor.cpp => VersionEpochCommand.cpp} (68%)
 rename fdbcli/{fdbcli.actor.cpp => fdbcli.cpp} (87%)
 delete mode 100644 fdbcli/include/fdbcli/fdbcli.actor.h
 create mode 100644 fdbcli/include/fdbcli/fdbcli.h
 rename fdbclient/{AsyncFileS3BlobStore.actor.cpp => AsyncFileBlobStore.cpp} (66%)
 delete mode 100644 fdbclient/AsyncTaskThread.actor.cpp
 rename fdbclient/{AuditUtils.actor.cpp => AuditUtils.cpp} (64%)
 rename fdbclient/{BackupAgentBase.actor.cpp => BackupAgentBase.cpp} (74%)
 rename fdbclient/{BackupContainer.actor.cpp => BackupContainer.cpp} (78%)
 rename fdbclient/{BackupContainerS3BlobStore.actor.cpp => BackupContainerBlobStore.cpp} (54%)
 rename fdbclient/{include/fdbclient/BackupContainerS3BlobStore.h => BackupContainerBlobStore.h} (64%)
 rename fdbclient/{BackupContainerFileSystem.actor.cpp => BackupContainerFileSystem.cpp} (77%)
 rename fdbclient/{BackupContainerLocalDirectory.actor.cpp => BackupContainerLocalDirectory.cpp} (86%)
 rename fdbclient/{include/fdbclient => }/BackupContainerLocalDirectory.h (90%)
 create mode 100644 fdbclient/BlobStoreCommon.cpp
 delete mode 100644 fdbclient/ClientKnobCollection.cpp
 rename fdbclient/{ClusterConnectionFile.actor.cpp => ClusterConnectionFile.cpp} (97%)
 rename fdbclient/{ClusterConnectionKey.actor.cpp => ClusterConnectionKey.cpp} (70%)
 rename fdbclient/{include/fdbclient/ClusterConnectionKey.actor.h => ClusterConnectionKey.h} (76%)
 rename fdbclient/{ClusterConnectionMemoryRecord.actor.cpp => ClusterConnectionMemoryRecord.cpp} (94%)
 rename fdbclient/{DataDistributionConfig.actor.cpp => DataDistributionConfig.cpp} (91%)
 rename fdbclient/{DatabaseBackupAgent.actor.cpp => DatabaseBackupAgent.cpp} (59%)
 rename fdbclient/{DatabaseContext.actor.cpp => DatabaseContext.cpp} (87%)
 rename fdbclient/{include/fdbclient => }/FDBAWSCredentialsProvider.h (100%)
 rename fdbclient/{FileBackupAgent.actor.cpp => FileBackupAgent.cpp} (61%)
 create mode 100644 fdbclient/GCSBlobStore.cpp
 create mode 100644 fdbclient/GCSBlobStore.h
 rename fdbclient/{GlobalConfig.actor.cpp => GlobalConfig.cpp} (62%)
 delete mode 100644 fdbclient/IKnobCollection.cpp
 rename fdbclient/{IdempotencyId.actor.cpp => IdempotencyId.cpp} (81%)
 rename fdbclient/{KeyRangeMap.actor.cpp => KeyRangeMap.cpp} (80%)
 rename fdbclient/{include/fdbclient => }/LocalClientAPI.h (99%)
 rename fdbclient/{ManagementAPI.actor.cpp => ManagementAPI.cpp} (68%)
 rename fdbclient/{MonitorLeader.actor.cpp => MonitorLeader.cpp} (80%)
 rename fdbclient/{MultiVersionTransaction.actor.cpp => MultiVersionTransaction.cpp} (94%)
 rename fdbclient/{MutationLogReader.actor.cpp => MutationLogReader.cpp} (73%)
 rename fdbclient/{include/fdbclient => }/NameLineage.h (100%)
 delete mode 100644 fdbclient/ParallelStream.actor.cpp
 rename fdbclient/{include/fdbclient => }/PartitionedLogIterator.h (99%)
 create mode 100644 fdbclient/ProxyLoadBalance.h
 rename fdbclient/{RESTClient.actor.cpp => RESTClient.cpp} (88%)
 rename fdbclient/{include/fdbclient => }/RESTClient.h (99%)
 rename fdbclient/{RESTUtils.actor.cpp => RESTUtils.cpp} (93%)
 rename fdbclient/{include/fdbclient => }/RESTUtils.h (97%)
 rename fdbclient/{include/fdbclient => }/RestoreInterface.h (100%)
 delete mode 100644 fdbclient/S3BlobStore.actor.cpp
 create mode 100644 fdbclient/S3BlobStore.cpp
 rename fdbclient/{S3Client.actor.cpp => S3Client.cpp} (62%)
 rename fdbclient/{S3Client_cli.actor.cpp => S3Client_cli.cpp} (91%)
 delete mode 100644 fdbclient/ServerKnobCollection.cpp
 rename fdbclient/{SpecialKeySpace.actor.cpp => SpecialKeySpace.cpp} (85%)
 rename fdbclient/{StatusClient.actor.cpp => StatusClient.cpp} (80%)
 rename fdbclient/{TagThrottle.actor.cpp => TagThrottle.cpp} (73%)
 rename fdbclient/{TaskBucket.actor.cpp => TaskBucket.cpp} (68%)
 delete mode 100644 fdbclient/TestKnobCollection.cpp
 rename fdbclient/{Tracing.actor.cpp => Tracing.cpp} (91%)
 delete mode 100644 fdbclient/VersionedMap.cpp
 rename {flowbench => fdbclient/bench}/BenchIdempotencyIds.cpp (100%)
 rename {flowbench => fdbclient/bench}/BenchIterate.cpp (98%)
 rename fdbserver/include/fdbserver/ServerDBInfo.h => fdbclient/bench/BenchMain.cpp (82%)
 rename {flowbench => fdbclient/bench}/BenchMetadataCheck.cpp (100%)
 rename {flowbench => fdbclient/bench}/BenchPopulate.cpp (98%)
 rename {flowbench => fdbclient/bench}/BenchTempTagMessages.cpp (100%)
 rename {flowbench => fdbclient/bench}/BenchVersionVector.cpp (100%)
 rename {flowbench => fdbclient/bench}/BenchVersionVectorSerialization.cpp (100%)
 create mode 100644 fdbclient/bench/BenchVersionedMap.cpp
 create mode 100644 fdbclient/bench/CMakeLists.txt
 rename {flowbench => fdbclient/bench}/GlobalData.cpp (98%)
 rename {flowbench/include/flowbench => fdbclient/bench}/GlobalData.h (88%)
 rename fdbclient/include/fdbclient/{AsyncFileS3BlobStore.actor.h => AsyncFileBlobStore.h} (62%)
 delete mode 100644 fdbclient/include/fdbclient/AsyncTaskThread.h
 delete mode 100644 fdbclient/include/fdbclient/AuditUtils.actor.h
 create mode 100644 fdbclient/include/fdbclient/AuditUtils.h
 rename fdbclient/include/fdbclient/{BackupAgent.actor.h => BackupAgent.h} (86%)
 delete mode 100644 fdbclient/include/fdbclient/ClientKnobCollection.h
 delete mode 100644 fdbclient/include/fdbclient/ClientKnobs.h
 rename fdbclient/include/fdbclient/{ConsistencyScanInterface.actor.h => ConsistencyScanInterface.h} (76%)
 rename fdbclient/include/fdbclient/{DataDistributionConfig.actor.h => DataDistributionConfig.h} (81%)
 delete mode 100644 fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
 create mode 100644 fdbclient/include/fdbclient/EventTypes.h
 rename fdbclient/include/fdbclient/{GenericManagementAPI.actor.h => GenericManagementAPI.h} (75%)
 delete mode 100644 fdbclient/include/fdbclient/GlobalConfig.actor.h
 rename fdbclient/include/fdbclient/{HighContentionPrefixAllocator.actor.h => HighContentionPrefixAllocator.h} (66%)
 create mode 100644 fdbclient/include/fdbclient/IBlobStore.h
 delete mode 100644 fdbclient/include/fdbclient/IKnobCollection.h
 rename fdbclient/include/fdbclient/{IdempotencyId.actor.h => IdempotencyId.h} (89%)
 rename fdbclient/include/fdbclient/{KeyBackedRangeMap.actor.h => KeyBackedRangeMap.h} (86%)
 rename fdbclient/include/fdbclient/{KeyBackedTypes.actor.h => KeyBackedTypes.h} (91%)
 delete mode 100644 fdbclient/include/fdbclient/ManagementAPI.actor.h
 create mode 100644 fdbclient/include/fdbclient/ManagementAPI.h
 rename fdbclient/include/fdbclient/{MutationLogReader.actor.h => MutationLogReader.h} (78%)
 delete mode 100644 fdbclient/include/fdbclient/ParallelStream.actor.h
 delete mode 100644 fdbclient/include/fdbclient/RunRYWTransaction.actor.h
 create mode 100644 fdbclient/include/fdbclient/RunRYWTransaction.h
 rename fdbclient/include/fdbclient/{RunTransaction.actor.h => RunTransaction.h} (69%)
 rename fdbclient/include/fdbclient/{S3Client.actor.h => S3Client.h} (81%)
 delete mode 100644 fdbclient/include/fdbclient/ServerKnobCollection.h
 rename fdbclient/include/fdbclient/{SpecialKeySpace.actor.h => SpecialKeySpace.h} (90%)
 rename fdbclient/include/fdbclient/{StorageWiggleMetrics.actor.h => StorageWiggleMetrics.h} (90%)
 rename fdbclient/include/fdbclient/{TagThrottle.actor.h => TagThrottle.h} (77%)
 delete mode 100644 fdbclient/include/fdbclient/TestKnobCollection.h
 delete mode 100644 fdbclient/include/fdbclient/VersionedMap.actor.h
 rename fdbclient/{include/fdbclient => }/sha1/SHA1.h (100%)
 create mode 100644 fdbclient/tests/gcp_fixture.sh
 create mode 100644 fdbkubernetesmonitor/CMakeLists.txt
 rename fdbrpc/{AsyncFileCached.actor.cpp => AsyncFileCached.cpp} (99%)
 rename fdbrpc/{include/fdbrpc => }/AsyncFileChaos.h (100%)
 rename fdbrpc/{include/fdbrpc/AsyncFileEIO.actor.h => AsyncFileEIO.h} (71%)
 rename fdbrpc/{AsyncFileEncrypted.actor.cpp => AsyncFileEncrypted.cpp} (57%)
 rename fdbrpc/{include/fdbrpc/AsyncFileKAIO.actor.h => AsyncFileKAIO.h} (86%)
 delete mode 100644 fdbrpc/AsyncFileNonDurable.actor.cpp
 create mode 100644 fdbrpc/AsyncFileNonDurable.cpp
 rename fdbrpc/{include/fdbrpc/AsyncFileWinASIO.actor.h => AsyncFileWinASIO.h} (93%)
 rename fdbrpc/{include/fdbrpc/AsyncFileWriteChecker.actor.h => AsyncFileWriteChecker.h} (92%)
 rename fdbrpc/{include/fdbrpc => }/Base64Decode.h (100%)
 rename fdbrpc/{include/fdbrpc => }/Base64Encode.h (100%)
 rename fdbrpc/{include/fdbrpc => }/ContinuousSample.h (100%)
 rename fdbrpc/{DDSketchTest.actor.cpp => DDSketchTest.cpp} (97%)
 rename fdbrpc/{FailureMonitor.actor.cpp => FailureMonitor.cpp} (89%)
 rename fdbrpc/{include/fdbrpc => }/FileTransfer.h (96%)
 rename fdbrpc/{include/fdbrpc => }/FlowGrpcTests.h (96%)
 rename fdbrpc/{FlowTransport.actor.cpp => FlowTransport.cpp} (87%)
 rename fdbrpc/{HTTP.actor.cpp => HTTP.cpp} (80%)
 rename fdbrpc/{HTTPServer.actor.cpp => HTTPServer.cpp} (67%)
 rename fdbrpc/{HealthMonitor.actor.cpp => HealthMonitor.cpp} (95%)
 rename fdbrpc/{include/fdbrpc => }/JsonWebKeySet.h (92%)
 create mode 100644 fdbrpc/Net2FileSystemTests.cpp
 rename fdbrpc/{SimExternalConnection.actor.cpp => SimExternalConnection.cpp} (74%)
 rename fdbrpc/{include/fdbrpc => }/SimExternalConnection.h (94%)
 rename fdbrpc/{Stats.actor.cpp => Stats.cpp} (92%)
 rename {flowbench => fdbrpc/bench}/BenchAsyncFileWriteCheckerLRU.cpp (86%)
 rename {flowbench => fdbrpc/bench}/BenchIONet2.actor.cpp (98%)
 create mode 100644 fdbrpc/bench/BenchMain.cpp
 rename {flowbench => fdbrpc/bench}/BenchSamples.cpp (98%)
 rename {flowbench => fdbrpc/bench}/BenchSelectReplicas.cpp (91%)
 rename contrib/mockkms/utils.go => fdbrpc/bench/BenchSupport.h (57%)
 create mode 100644 fdbrpc/bench/CMakeLists.txt
 rename fdbrpc/include/fdbrpc/{AsyncFileCached.actor.h => AsyncFileCached.h} (88%)
 rename fdbrpc/include/fdbrpc/{AsyncFileNonDurable.actor.h => AsyncFileNonDurable.h} (81%)
 rename fdbrpc/include/fdbrpc/{AsyncFileReadAhead.actor.h => AsyncFileReadAhead.h} (82%)
 delete mode 100644 fdbrpc/include/fdbrpc/FlowProcess.actor.h
 rename fdbrpc/include/fdbrpc/{genericactors.actor.h => genericactors.h} (53%)
 rename fdbrpc/include/fdbrpc/{networksender.actor.h => networksender.h} (62%)
 rename fdbrpc/{include/fdbrpc => }/linux_kaio.h (100%)
 rename fdbrpc/{sim2.actor.cpp => sim2.cpp} (80%)
 rename fdbrpc/tests/{AuthzTlsTest.actor.cpp => AuthzTlsTest.cpp} (98%)
 rename fdbrpc/tests/{fdbrpc_bench.actor.cpp => fdbrpc_bench.cpp} (77%)
 delete mode 100644 fdbserver/BackupWorkerRangePartitioned.actor.cpp
 delete mode 100644 fdbserver/GlobalTagThrottler.actor.cpp
 delete mode 100644 fdbserver/GrvProxyTagThrottler.actor.cpp
 delete mode 100644 fdbserver/LatencyBandsMap.actor.cpp
 delete mode 100644 fdbserver/RemoteIKeyValueStore.actor.cpp
 delete mode 100644 fdbserver/RestoreApplier.actor.cpp
 delete mode 100644 fdbserver/RestoreCommon.actor.cpp
 delete mode 100644 fdbserver/RestoreController.actor.cpp
 delete mode 100644 fdbserver/RestoreLoader.actor.cpp
 delete mode 100644 fdbserver/RestoreRoleCommon.actor.cpp
 delete mode 100644 fdbserver/RestoreUtil.actor.cpp
 delete mode 100644 fdbserver/RestoreWorker.actor.cpp
 delete mode 100644 fdbserver/RestoreWorkerInterface.actor.cpp
 delete mode 100644 fdbserver/ServerThroughputTracker.cpp
 rename fdbserver/{SimulatedCluster.actor.cpp => SimulatedCluster.cpp} (84%)
 delete mode 100644 fdbserver/TSSMappingUtil.actor.cpp
 rename fdbserver/{BackupPartitionMap.actor.cpp => backupworker/BackupPartitionMap.cpp} (69%)
 create mode 100644 fdbserver/backupworker/BackupPartitionMap.h
 create mode 100644 fdbserver/backupworker/BackupRangePartitionedProgress.cpp
 create mode 100644 fdbserver/backupworker/BackupRangePartitionedProgress.h
 rename fdbserver/{BackupWorker.actor.cpp => backupworker/BackupWorker.cpp} (63%)
 create mode 100644 fdbserver/backupworker/BackupWorkerRangePartitioned.cpp
 create mode 100644 fdbserver/backupworker/CMakeLists.txt
 create mode 100644 fdbserver/backupworker/include/fdbserver/backupworker/BackupWorker.h
 create mode 100644 fdbserver/clustercontroller/CMakeLists.txt
 rename fdbserver/{ => clustercontroller}/ClusterController.actor.cpp (87%)
 rename fdbserver/{include/fdbserver/ClusterController.actor.h => clustercontroller/ClusterController.h} (96%)
 create mode 100644 fdbserver/clustercontroller/ClusterHealthIFactor.cpp
 create mode 100644 fdbserver/clustercontroller/ClusterHealthIFactor.h
 create mode 100644 fdbserver/clustercontroller/ClusterHealthMonitor.cpp
 create mode 100644 fdbserver/clustercontroller/ClusterHealthMonitor.h
 create mode 100644 fdbserver/clustercontroller/ClusterHealthMonitorTesting.cpp
 rename fdbserver/{ClusterRecovery.actor.cpp => clustercontroller/ClusterRecovery.cpp} (75%)
 rename fdbserver/{include/fdbserver/ClusterRecovery.actor.h => clustercontroller/ClusterRecovery.h} (84%)
 create mode 100644 fdbserver/clustercontroller/RatekeeperMonitor.cpp
 create mode 100644 fdbserver/clustercontroller/RatekeeperMonitor.h
 rename fdbserver/{include/fdbserver => clustercontroller}/SingletonRoles.h (90%)
 rename fdbserver/{Status.actor.cpp => clustercontroller/Status.cpp} (77%)
 rename fdbserver/{include/fdbserver/Status.actor.h => clustercontroller/Status.h} (50%)
 create mode 100644 fdbserver/clustercontroller/include/fdbserver/clustercontroller/ClusterController.h
 create mode 100644 fdbserver/commitproxy/CMakeLists.txt
 rename fdbserver/{CommitProxyServer.actor.cpp => commitproxy/CommitProxyServer.cpp} (73%)
 rename fdbserver/{include/fdbserver/ProxyCommitData.actor.h => commitproxy/ProxyCommitData.h} (87%)
 create mode 100644 fdbserver/commitproxy/include/fdbserver/commitproxy/CommitProxyServer.h
 create mode 100644 fdbserver/consistencyscan/CMakeLists.txt
 rename fdbserver/{ConsistencyScan.actor.cpp => consistencyscan/ConsistencyScan.cpp} (75%)
 create mode 100644 fdbserver/consistencyscan/include/fdbserver/consistencyscan/ConsistencyScan.h
 create mode 100644 fdbserver/coordinator/CMakeLists.txt
 rename fdbserver/{Coordination.actor.cpp => coordinator/Coordination.cpp} (57%)
 rename fdbserver/{OnDemandStore.actor.cpp => coordinator/OnDemandStore.cpp} (88%)
 rename fdbserver/{include/fdbserver => coordinator}/OnDemandStore.h (96%)
 create mode 100644 fdbserver/coordinator/include/fdbserver/coordinator/CoordinationServer.h
 rename fdbserver/{ => core}/AccumulativeChecksumUtil.cpp (99%)
 rename fdbserver/{BackupProgress.actor.cpp => core/BackupProgress.cpp} (87%)
 rename fdbserver/{BulkDumpUtil.actor.cpp => core/BulkDumpUtil.cpp} (67%)
 rename fdbserver/{BulkLoadUtil.actor.cpp => core/BulkLoadUtil.cpp} (56%)
 create mode 100644 fdbserver/core/CMakeLists.txt
 rename fdbserver/{CoordinatedState.actor.cpp => core/CoordinatedState.cpp} (64%)
 create mode 100644 fdbserver/core/CoordinationInterface.cpp
 create mode 100644 fdbserver/core/DataMovement.cpp
 create mode 100644 fdbserver/core/FDBSimulationPolicy.cpp
 rename fdbserver/{ => core}/LatencyBandConfig.cpp (94%)
 rename fdbserver/{ => core}/LeaderElection.actor.cpp (87%)
 rename fdbserver/{ => core}/LogSystemConfig.cpp (98%)
 create mode 100644 fdbserver/core/MasterInterface.cpp
 rename fdbserver/{MoveKeys.actor.cpp => core/MoveKeys.cpp} (72%)
 rename fdbserver/{ => core}/MutationTracking.cpp (90%)
 create mode 100644 fdbserver/core/OpenDatabase.cpp
 rename fdbserver/{QuietDatabase.actor.cpp => core/QuietDatabase.cpp} (64%)
 create mode 100644 fdbserver/core/RatekeeperLimitReasons.cpp
 rename fdbserver/{RocksDBCheckpointUtils.actor.cpp => core/RocksDBCheckpointUtils.cpp} (84%)
 create mode 100644 fdbserver/core/SeedShardServers.cpp
 rename fdbserver/{ServerCheckpoint.actor.cpp => core/ServerCheckpoint.cpp} (76%)
 rename {fdbclient => fdbserver/core}/ServerKnobs.cpp (89%)
 create mode 100644 fdbserver/core/ShardSizing.cpp
 create mode 100644 fdbserver/core/SimulatorSupport.cpp
 rename fdbserver/{StorageMetrics.actor.cpp => core/StorageMetrics.cpp} (85%)
 create mode 100644 fdbserver/core/TSSMappingUtil.cpp
 rename fdbserver/{WaitFailure.actor.cpp => core/WaitFailure.cpp} (52%)
 create mode 100644 fdbserver/core/WorkerEvents.cpp
 create mode 100644 fdbserver/core/WorkerInterface.cpp
 create mode 100644 fdbserver/core/WorkerSupport.cpp
 create mode 100644 fdbserver/core/WorkloadKeys.cpp
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/AccumulativeChecksumUtil.h (98%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/BackupInterface.h (100%)
 rename fdbserver/{include/fdbserver/BackupProgress.actor.h => core/include/fdbserver/core/BackupProgress.h} (81%)
 create mode 100644 fdbserver/core/include/fdbserver/core/BackupProgressTypes.h
 rename fdbserver/{include/fdbserver/BulkDumpUtil.actor.h => core/include/fdbserver/core/BulkDumpUtil.h} (70%)
 create mode 100644 fdbserver/core/include/fdbserver/core/BulkLoadUtil.h
 create mode 100644 fdbserver/core/include/fdbserver/core/ConflictBatch.h
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/CoordinatedState.h (96%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/CoordinationInterface.h (92%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/DBCoreState.h (97%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/DataDistributorInterface.h (93%)
 create mode 100644 fdbserver/core/include/fdbserver/core/DataMovement.h
 create mode 100644 fdbserver/core/include/fdbserver/core/FDBSimulationPolicy.h
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/IDiskQueue.h (93%)
 rename fdbclient/include/fdbclient/ServerKnobs.h => fdbserver/core/include/fdbserver/core/Knobs.h (89%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/LatencyBandConfig.h (100%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/LeaderElection.h (100%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/LogProtocolMessage.h (100%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/LogSystemConfig.h (97%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/MasterInterface.h (98%)
 rename fdbserver/{include/fdbserver/MoveKeys.actor.h => core/include/fdbserver/core/MoveKeys.h} (83%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/MutationTracking.h (100%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/OTELSpanContextMessage.h (95%)
 create mode 100644 fdbserver/core/include/fdbserver/core/QuietDatabase.h
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/RatekeeperInterface.h (93%)
 create mode 100644 fdbserver/core/include/fdbserver/core/RatekeeperLimitReasons.h
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/RecoveryState.h (98%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/ResolverInterface.h (100%)
 rename fdbserver/{include/fdbserver/RocksDBCheckpointUtils.actor.h => core/include/fdbserver/core/RocksDBCheckpointUtils.h} (93%)
 create mode 100644 fdbserver/core/include/fdbserver/core/SeedShardServers.h
 rename fdbserver/{include/fdbserver/ServerCheckpoint.actor.h => core/include/fdbserver/core/ServerCheckpoint.h} (79%)
 rename fdbserver/{include/fdbserver/ServerDBInfo.actor.h => core/include/fdbserver/core/ServerDBInfo.h} (78%)
 create mode 100644 fdbserver/core/include/fdbserver/core/ShardMetrics.h
 create mode 100644 fdbserver/core/include/fdbserver/core/ShardSizing.h
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/SpanContextMessage.h (96%)
 rename fdbserver/{include/fdbserver/StorageMetrics.actor.h => core/include/fdbserver/core/StorageMetrics.h} (80%)
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/TLogInterface.h (98%)
 create mode 100644 fdbserver/core/include/fdbserver/core/TSSMappingUtil.h
 rename fdbserver/{include/fdbserver/TesterInterface.actor.h => core/include/fdbserver/core/TesterInterface.h} (66%)
 create mode 100644 fdbserver/core/include/fdbserver/core/WaitFailure.h
 create mode 100644 fdbserver/core/include/fdbserver/core/WorkerEvents.h
 rename fdbserver/{include/fdbserver => core/include/fdbserver/core}/WorkerInterface.actor.h (82%)
 create mode 100644 fdbserver/core/include/fdbserver/core/WorkloadKeys.h
 create mode 100644 fdbserver/datadistributor/AuditUtilsTests.cpp
 create mode 100644 fdbserver/datadistributor/CMakeLists.txt
 rename fdbserver/{ => datadistributor}/DDRelocationQueue.actor.cpp (85%)
 rename fdbserver/{include/fdbserver => datadistributor}/DDRelocationQueue.h (96%)
 rename fdbserver/{DDShardTracker.actor.cpp => datadistributor/DDShardTracker.cpp} (85%)
 create mode 100644 fdbserver/datadistributor/DDSharedContext.cpp
 rename fdbserver/{ => datadistributor}/DDTeamCollection.actor.cpp (91%)
 rename fdbserver/{DDTxnProcessor.actor.cpp => datadistributor/DDTxnProcessor.cpp} (74%)
 rename fdbserver/{DataDistribution.actor.cpp => datadistributor/DataDistribution.cpp} (74%)
 rename fdbserver/{include/fdbserver/ExclusionTracker.actor.h => datadistributor/ExclusionTracker.h} (68%)
 rename fdbserver/{MockGlobalState.actor.cpp => datadistributor/MockGlobalState.cpp} (93%)
 rename fdbserver/{include/fdbserver => datadistributor}/MovingWindow.h (97%)
 rename fdbserver/{ => datadistributor}/ShardsAffectedByTeamFailure.cpp (97%)
 create mode 100644 fdbserver/datadistributor/SimulatedCluster.cpp
 rename fdbserver/{TCInfo.actor.cpp => datadistributor/TCInfo.cpp} (86%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/DDShardTracker.h (97%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/DDSharedContext.h (80%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/DDTeamCollection.h (97%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/DDTxnProcessor.h (98%)
 rename fdbserver/{include/fdbserver/DataDistribution.actor.h => datadistributor/include/fdbserver/datadistributor/DataDistribution.h} (90%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/DataDistributionTeam.h (98%)
 create mode 100644 fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributor.h
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/MockDataDistributor.h (85%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/MockGlobalState.h (96%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/ShardsAffectedByTeamFailure.h (100%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/SimulatedCluster.h (100%)
 rename fdbserver/{include/fdbserver => datadistributor/include/fdbserver/datadistributor}/TCInfo.h (98%)
 rename fdbserver/{fdbserver.actor.cpp => fdbserver.cpp} (88%)
 create mode 100644 fdbserver/grvproxy/CMakeLists.txt
 rename fdbserver/{GrvProxyServer.actor.cpp => grvproxy/GrvProxyServer.cpp} (69%)
 create mode 100644 fdbserver/grvproxy/GrvQueueDelay.cpp
 create mode 100644 fdbserver/grvproxy/GrvQueueDelay.h
 create mode 100644 fdbserver/grvproxy/GrvQueueDelayTests.cpp
 rename fdbserver/{GrvTransactionRateInfo.actor.cpp => grvproxy/GrvTransactionRateInfo.cpp} (83%)
 rename fdbserver/{include/fdbserver => grvproxy}/GrvTransactionRateInfo.h (92%)
 create mode 100644 fdbserver/grvproxy/include/fdbserver/grvproxy/GrvProxyServer.h
 delete mode 100644 fdbserver/include/fdbserver/BulkLoadUtil.actor.h
 delete mode 100644 fdbserver/include/fdbserver/FDBExecHelper.actor.h
 delete mode 100644 fdbserver/include/fdbserver/GrvProxyTagThrottler.h
 delete mode 100644 fdbserver/include/fdbserver/IKeyValueStore.h
 delete mode 100644 fdbserver/include/fdbserver/KmsConnector.h
 delete mode 100644 fdbserver/include/fdbserver/LatencyBandsMap.h
 delete mode 100644 fdbserver/include/fdbserver/LogSystem.h
 delete mode 100644 fdbserver/include/fdbserver/QuietDatabase.h
 delete mode 100644 fdbserver/include/fdbserver/RESTKmsConnectorUtils.h
 delete mode 100644 fdbserver/include/fdbserver/RESTSimKmsVault.h
 delete mode 100644 fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreApplier.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreCommon.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreController.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreLoader.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreRoleCommon.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreUtil.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreWorker.actor.h
 delete mode 100644 fdbserver/include/fdbserver/RestoreWorkerInterface.actor.h
 delete mode 100644 fdbserver/include/fdbserver/ServerThroughputTracker.h
 delete mode 100644 fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h
 delete mode 100644 fdbserver/include/fdbserver/TSSMappingUtil.actor.h
 delete mode 100644 fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h
 delete mode 100644 fdbserver/include/fdbserver/WaitFailure.h
 rename fdbserver/{include/fdbserver => kvstore}/ArtMutationBuffer.h (92%)
 create mode 100644 fdbserver/kvstore/CMakeLists.txt
 rename fdbserver/{include/fdbserver => kvstore}/DeltaTree.h (99%)
 rename fdbserver/{DiskQueue.actor.cpp => kvstore/DiskQueue.cpp} (84%)
 rename fdbserver/{FDBExecHelper.actor.cpp => kvstore/FDBExecHelper.cpp} (58%)
 rename fdbserver/{include/fdbserver => kvstore}/IKeyValueContainer.h (95%)
 rename fdbserver/{ => kvstore}/IKeyValueStore.cpp (76%)
 rename fdbserver/{ => kvstore}/IPager.cpp (93%)
 rename fdbserver/{include/fdbserver => kvstore}/IPager.h (92%)
 rename fdbserver/{KeyValueStoreCompressTestData.actor.cpp => kvstore/KeyValueStoreCompressTestData.cpp} (78%)
 rename fdbserver/{KeyValueStoreMemory.actor.cpp => kvstore/KeyValueStoreMemory.cpp} (86%)
 rename fdbserver/{ => kvstore}/KeyValueStoreRocksDB.actor.cpp (95%)
 rename fdbserver/{KeyValueStoreSQLite.actor.cpp => kvstore/KeyValueStoreSQLite.cpp} (96%)
 rename fdbserver/{ => kvstore}/KeyValueStoreShardedRocksDB.actor.cpp (97%)
 rename fdbserver/{include/fdbserver => kvstore}/RadixTree.h (99%)
 rename fdbserver/{ => kvstore}/RocksDBCommon.cpp (98%)
 rename fdbserver/{include/fdbserver => kvstore}/RocksDBCommon.h (100%)
 rename fdbserver/{RocksDBLogForwarder.actor.cpp => kvstore/RocksDBLogForwarder.cpp} (93%)
 rename fdbserver/{include/fdbserver => kvstore}/RocksDBLogForwarder.h (100%)
 rename fdbserver/{ => kvstore}/TransactionStoreMutationTracking.cpp (94%)
 rename fdbserver/{include/fdbserver => kvstore}/TransactionStoreMutationTracking.h (100%)
 rename fdbserver/{ => kvstore}/VFSAsync.cpp (97%)
 rename fdbserver/{include/fdbserver => kvstore}/VFSAsync.h (100%)
 rename fdbserver/{ => kvstore}/VersionedBTree.actor.cpp (90%)
 rename fdbserver/{ => kvstore}/VersionedBTreeDebug.cpp (96%)
 rename fdbserver/{include/fdbserver => kvstore}/VersionedBTreeDebug.h (100%)
 rename fdbserver/{include/fdbserver => kvstore}/art.h (98%)
 rename fdbserver/{include/fdbserver => kvstore}/art_impl.h (100%)
 create mode 100644 fdbserver/kvstore/include/fdbserver/kvstore/FDBExecHelper.h
 create mode 100644 fdbserver/kvstore/include/fdbserver/kvstore/IDiskQueue.h
 rename fdbclient/include/fdbclient/IKeyValueStore.actor.h => fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h (67%)
 rename fdbserver/{include/fdbserver/RESTKmsConnector.h => kvstore/include/fdbserver/kvstore/KVFileUtils.h} (70%)
 rename fdbserver/{include/fdbserver => kvstore}/template_fdb.h (100%)
 create mode 100644 fdbserver/logrouter/CMakeLists.txt
 rename fdbserver/{ => logrouter}/LogRouter.cpp (96%)
 create mode 100644 fdbserver/logrouter/include/fdbserver/logrouter/LogRouter.h
 rename fdbserver/{ => logsystem}/ApplyMetadataMutation.cpp (95%)
 create mode 100644 fdbserver/logsystem/CMakeLists.txt
 rename fdbserver/{LogSystem.cpp => logsystem/LogSet.cpp} (54%)
 rename fdbserver/{TagPartitionedLogSystem.actor.cpp => logsystem/LogSystem.cpp} (73%)
 rename fdbserver/{LogSystemDiskQueueAdapter.actor.cpp => logsystem/LogSystemDiskQueueAdapter.cpp} (74%)
 create mode 100644 fdbserver/logsystem/LogSystemFactory.cpp
 rename fdbserver/{LogSystemPeekCursor.actor.cpp => logsystem/LogSystemPeekCursor.cpp} (67%)
 rename fdbserver/{TagPartitionedLogSystemRecoveryTests.actor.cpp => logsystem/LogSystemRecoveryTests.cpp} (94%)
 rename fdbserver/{include/fdbserver => logsystem/include/fdbserver/logsystem}/ApplyMetadataMutation.h (69%)
 create mode 100644 fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h
 rename fdbserver/{include/fdbserver => logsystem/include/fdbserver/logsystem}/LogSystemDiskQueueAdapter.h (89%)
 create mode 100644 fdbserver/logsystem/include/fdbserver/logsystem/LogSystemFactory.h
 create mode 100644 fdbserver/logsystem/include/fdbserver/logsystem/LogSystemTypes.h
 delete mode 100644 fdbserver/masterserver.swift
 create mode 100644 fdbserver/mocks3/CMakeLists.txt
 rename fdbserver/{MockS3Server.actor.cpp => mocks3/MockS3Server.cpp} (81%)
 rename fdbserver/{MockS3ServerChaos.actor.cpp => mocks3/MockS3ServerChaos.cpp} (83%)
 rename fdbserver/{include/fdbserver => mocks3/include/fdbserver/mocks3}/MockS3Server.h (98%)
 rename fdbserver/{include/fdbserver => mocks3/include/fdbserver/mocks3}/MockS3ServerChaos.h (98%)
 rename fdbserver/{networktest.actor.cpp => networktest.cpp} (65%)
 create mode 100644 fdbserver/ratekeeper/CMakeLists.txt
 rename fdbserver/{Ratekeeper.actor.cpp => ratekeeper/Ratekeeper.cpp} (54%)
 rename fdbserver/{include/fdbserver => ratekeeper}/Ratekeeper.h (81%)
 create mode 100644 fdbserver/ratekeeper/RatekeeperLimits.cpp
 rename fdbserver/{ => ratekeeper}/RkTagThrottleCollection.cpp (99%)
 rename fdbserver/{include/fdbserver => ratekeeper}/RkTagThrottleCollection.h (89%)
 rename fdbserver/{TagThrottler.actor.cpp => ratekeeper/TagThrottler.cpp} (88%)
 rename fdbserver/{include/fdbserver => ratekeeper}/TagThrottler.h (50%)
 rename fdbserver/{include/fdbserver/Knobs.h => ratekeeper/include/fdbserver/ratekeeper/Ratekeeper.h} (71%)
 create mode 100644 fdbserver/resolver/CMakeLists.txt
 rename fdbserver/{SkipList.cpp => resolver/ConflictSet.cpp} (96%)
 rename fdbserver/{include/fdbserver => resolver}/ConflictSet.h (92%)
 rename fdbserver/{Resolver.actor.cpp => resolver/Resolver.cpp} (81%)
 rename fdbserver/{ => resolver}/ResolverBug.cpp (94%)
 create mode 100644 fdbserver/resolver/include/fdbserver/resolver/Resolver.h
 rename fdbserver/{include/fdbserver => resolver/include/fdbserver/resolver}/ResolverBug.h (90%)
 create mode 100644 fdbserver/sequencer/CMakeLists.txt
 rename fdbserver/{include/fdbserver/MasterData.actor.h => sequencer/MasterData.h} (50%)
 rename fdbserver/{ResolutionBalancer.actor.cpp => sequencer/ResolutionBalancer.cpp} (81%)
 rename fdbserver/{include/fdbserver/ResolutionBalancer.actor.h => sequencer/ResolutionBalancer.h} (68%)
 create mode 100644 fdbserver/sequencer/include/fdbserver/sequencer/MasterServer.h
 rename fdbserver/{masterserver.actor.cpp => sequencer/masterserver.cpp} (52%)
 create mode 100644 fdbserver/storageserver/CMakeLists.txt
 rename fdbserver/{ => storageserver}/ReadLatencySamples.cpp (97%)
 rename fdbserver/{include/fdbserver => storageserver}/ReadLatencySamples.h (100%)
 rename fdbserver/{ => storageserver}/StorageServerUtils.cpp (98%)
 rename fdbserver/{include/fdbserver => storageserver}/StorageServerUtils.h (99%)
 rename fdbserver/{TransactionTagCounter.actor.cpp => storageserver/TransactionTagCounter.cpp} (76%)
 rename fdbserver/{include/fdbserver => storageserver}/TransactionTagCounter.h (97%)
 rename fdbserver/{include/fdbserver => storageserver/include/fdbserver/storageserver}/StorageCorruptionBug.h (85%)
 create mode 100644 fdbserver/storageserver/include/fdbserver/storageserver/StorageServer.h
 rename fdbserver/{ => storageserver}/storageserver.actor.cpp (78%)
 delete mode 100644 fdbserver/test.cpp
 create mode 100644 fdbserver/tester/CMakeLists.txt
 create mode 100644 fdbserver/tester/ConsistencyChecker.cpp
 create mode 100644 fdbserver/tester/ConsistencyChecker.h
 rename fdbserver/{workloads/workloads.actor.cpp => tester/CustomShardConfigWorkload.cpp} (67%)
 create mode 100644 fdbserver/tester/DatabaseMaintenance.cpp
 create mode 100644 fdbserver/tester/DatabaseMaintenance.h
 rename fdbserver/{ => tester}/KnobProtectiveGroups.cpp (88%)
 create mode 100644 fdbserver/tester/TestSpecParser.cpp
 create mode 100644 fdbserver/tester/TestSpecParser.h
 create mode 100644 fdbserver/tester/TesterServer.cpp
 create mode 100644 fdbserver/tester/TesterServer.h
 create mode 100644 fdbserver/tester/WorkloadUtils.cpp
 rename fdbserver/{include/fdbserver => tester/include/fdbserver/tester}/KnobProtectiveGroups.h (86%)
 create mode 100644 fdbserver/tester/include/fdbserver/tester/TestEncryptionUtils.h
 create mode 100644 fdbserver/tester/include/fdbserver/tester/WorkloadUtils.h
 create mode 100644 fdbserver/tester/include/fdbserver/tester/tester.h
 rename fdbserver/{include/fdbserver/workloads/workloads.actor.h => tester/include/fdbserver/tester/workloads.h} (61%)
 create mode 100644 fdbserver/tester/test.cpp
 create mode 100644 fdbserver/tlog/CMakeLists.txt
 rename fdbserver/{TLogServer.actor.cpp => tlog/TLogServer.cpp} (77%)
 rename fdbserver/{TestTLogServer.actor.cpp => tlog/TestTLogServer.cpp} (75%)
 rename fdbserver/{include/fdbserver/TestTLogServer.actor.h => tlog/TestTLogServer.h} (78%)
 create mode 100644 fdbserver/tlog/include/fdbserver/tlog/TLogServer.h
 create mode 100644 fdbserver/worker/CMakeLists.txt
 rename fdbserver/{MetricClient.actor.cpp => worker/MetricClient.cpp} (89%)
 rename fdbserver/{include/fdbserver => worker}/MetricClient.h (92%)
 rename fdbserver/{ => worker}/MetricLogger.actor.cpp (97%)
 rename fdbserver/{include/fdbserver => worker}/MetricLogger.actor.h (71%)
 rename fdbserver/{RoleLineage.actor.cpp => worker/RoleLineage.cpp} (95%)
 rename fdbserver/{include/fdbserver/RoleLineage.actor.h => worker/RoleLineage.h} (62%)
 create mode 100644 fdbserver/worker/include/fdbserver/worker/Worker.h
 rename fdbserver/{ => worker}/worker.actor.cpp (85%)
 rename fdbserver/workloads/{ApiCorrectness.actor.cpp => ApiCorrectness.cpp} (72%)
 rename fdbserver/workloads/{ApiWorkload.actor.cpp => ApiWorkload.cpp} (73%)
 rename fdbserver/{include/fdbserver => }/workloads/ApiWorkload.h (97%)
 rename fdbserver/{include/fdbserver/workloads/AsyncFile.actor.h => workloads/AsyncFile.h} (69%)
 rename fdbserver/workloads/{AsyncFileCorrectness.actor.cpp => AsyncFileCorrectness.cpp} (87%)
 rename fdbserver/workloads/{AsyncFileRead.actor.cpp => AsyncFileRead.cpp} (76%)
 rename fdbserver/workloads/{AsyncFileWrite.actor.cpp => AsyncFileWrite.cpp} (81%)
 rename fdbserver/workloads/{AtomicOps.actor.cpp => AtomicOps.cpp} (74%)
 rename fdbserver/workloads/{AtomicOpsApiCorrectness.actor.cpp => AtomicOpsApiCorrectness.cpp} (58%)
 rename fdbserver/workloads/{AtomicRestore.actor.cpp => AtomicRestore.cpp} (50%)
 rename fdbserver/workloads/{AtomicSwitchover.actor.cpp => AtomicSwitchover.cpp} (59%)
 rename fdbserver/workloads/{AutomaticIdempotencyWorkload.actor.cpp => AutomaticIdempotencyWorkload.cpp} (64%)
 rename fdbserver/workloads/{BackgroundSelectors.actor.cpp => BackgroundSelectors.cpp} (68%)
 rename fdbserver/workloads/{Backup.actor.cpp => Backup.cpp} (66%)
 delete mode 100644 fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
 delete mode 100644 fdbserver/workloads/BackupAndRestoreValidation.actor.cpp
 create mode 100644 fdbserver/workloads/BackupAndRestoreValidation.cpp
 rename fdbserver/workloads/{BackupCorrectness.actor.cpp => BackupCorrectness.cpp} (65%)
 rename fdbserver/workloads/{BackupCorrectnessPartitioned.actor.cpp => BackupCorrectnessPartitioned.cpp} (66%)
 rename fdbserver/workloads/{BackupS3BlobCorrectness.actor.cpp => BackupS3BlobCorrectness.cpp} (67%)
 rename fdbserver/workloads/{BackupToDBAbort.actor.cpp => BackupToDBAbort.cpp} (51%)
 rename fdbserver/workloads/{BackupToDBCorrectness.actor.cpp => BackupToDBCorrectness.cpp} (65%)
 rename fdbserver/workloads/{BackupToDBUpgrade.actor.cpp => BackupToDBUpgrade.cpp} (64%)
 rename fdbserver/workloads/{BulkDumping.actor.cpp => BulkDumping.cpp} (76%)
 rename fdbserver/workloads/{BulkLoad.actor.cpp => BulkLoad.cpp} (86%)
 rename fdbserver/workloads/{BulkLoading.actor.cpp => BulkLoading.cpp} (72%)
 rename fdbserver/{include/fdbserver/workloads/BulkSetup.actor.h => workloads/BulkSetup.h} (71%)
 create mode 100644 fdbserver/workloads/CMakeLists.txt
 delete mode 100644 fdbserver/workloads/ChangeConfig.actor.cpp
 create mode 100644 fdbserver/workloads/ChangeConfig.cpp
 rename fdbserver/workloads/{ClearSingleRange.actor.cpp => ClearSingleRange.cpp} (65%)
 rename fdbserver/workloads/{ClientMetric.actor.cpp => ClientMetric.cpp} (67%)
 rename fdbserver/workloads/{ClientTransactionProfileCorrectness.actor.cpp => ClientTransactionProfileCorrectness.cpp} (85%)
 rename fdbserver/workloads/{ClientWorkload.actor.cpp => ClientWorkload.cpp} (57%)
 rename fdbserver/workloads/{ClogRemoteTLog.actor.cpp => ClogRemoteTLog.cpp} (79%)
 rename fdbserver/workloads/{ClogSingleConnection.actor.cpp => ClogSingleConnection.cpp} (87%)
 rename fdbserver/workloads/{ClogTlog.actor.cpp => ClogTlog.cpp} (77%)
 rename fdbserver/workloads/{CommitBugCheck.actor.cpp => CommitBugCheck.cpp} (56%)
 rename fdbserver/workloads/{ConfigureDatabase.actor.cpp => ConfigureDatabase.cpp} (79%)
 rename fdbserver/workloads/{ConflictRange.actor.cpp => ConflictRange.cpp} (74%)
 rename fdbserver/workloads/{ConsistencyCheck.actor.cpp => ConsistencyCheck.cpp} (76%)
 rename fdbserver/workloads/{ConsistencyCheckUrgent.actor.cpp => ConsistencyCheckUrgent.cpp} (77%)
 rename fdbserver/workloads/{CpuProfiler.actor.cpp => CpuProfiler.cpp} (72%)
 rename fdbserver/workloads/{Cycle.actor.cpp => Cycle.cpp} (79%)
 rename fdbserver/workloads/{DDBalance.actor.cpp => DDBalance.cpp} (72%)
 rename fdbserver/workloads/{DDMetrics.actor.cpp => DDMetrics.cpp} (65%)
 rename fdbserver/workloads/{DDMetricsExclude.actor.cpp => DDMetricsExclude.cpp} (69%)
 rename fdbserver/workloads/{DataDistributionMetrics.actor.cpp => DataDistributionMetrics.cpp} (61%)
 rename fdbserver/workloads/{DataLossRecovery.actor.cpp => DataLossRecovery.cpp} (71%)
 rename fdbserver/workloads/{DcLag.actor.cpp => DcLag.cpp} (79%)
 rename fdbserver/workloads/{DifferentClustersSameRV.actor.cpp => DifferentClustersSameRV.cpp} (51%)
 rename fdbserver/workloads/{DiskDurability.actor.cpp => DiskDurability.cpp} (54%)
 rename fdbserver/workloads/{DiskDurabilityTest.actor.cpp => DiskDurabilityTest.cpp} (66%)
 rename fdbserver/workloads/{DiskFailureInjection.actor.cpp => DiskFailureInjection.cpp} (64%)
 rename fdbserver/workloads/{ExcludeIncludeStorageServersWorkload.actor.cpp => ExcludeIncludeStorageServersWorkload.cpp} (75%)
 delete mode 100644 fdbserver/workloads/ExpectStableThroughput.actor.cpp
 rename fdbserver/workloads/{ExternalWorkload.actor.cpp => ExternalWorkload.cpp} (93%)
 rename fdbserver/workloads/{FailoverWithSSLag.actor.cpp => FailoverWithSSLag.cpp} (69%)
 rename fdbserver/workloads/{FastTriggeredWatches.actor.cpp => FastTriggeredWatches.cpp} (71%)
 rename fdbserver/workloads/{FileSystem.actor.cpp => FileSystem.cpp} (70%)
 rename fdbserver/workloads/{FuzzApiCorrectness.actor.cpp => FuzzApiCorrectness.cpp} (89%)
 delete mode 100644 fdbserver/workloads/GcGenerations.actor.cpp
 create mode 100644 fdbserver/workloads/GcGenerations.cpp
 rename fdbserver/workloads/{GetEstimatedRangeSize.actor.cpp => GetEstimatedRangeSize.cpp} (70%)
 rename fdbserver/workloads/{GetMappedRange.actor.cpp => GetMappedRange.cpp} (60%)
 rename fdbserver/workloads/{GetRangeStream.actor.cpp => GetRangeStream.cpp} (50%)
 rename fdbserver/workloads/{HTTPKeyValueStore.actor.cpp => HTTPKeyValueStore.cpp} (74%)
 rename fdbserver/workloads/{HealthMetricsApi.actor.cpp => HealthMetricsApi.cpp} (74%)
 rename fdbserver/workloads/{HighContentionPrefixAllocatorWorkload.actor.cpp => HighContentionPrefixAllocatorWorkload.cpp} (58%)
 rename fdbserver/workloads/{IDDTxnProcessorApiCorrectness.actor.cpp => IDDTxnProcessorApiCorrectness.cpp} (61%)
 rename fdbserver/workloads/{Increment.actor.cpp => Increment.cpp} (80%)
 rename fdbserver/workloads/{IncrementalBackup.actor.cpp => IncrementalBackup.cpp} (52%)
 rename fdbserver/workloads/{IndexScan.actor.cpp => IndexScan.cpp} (62%)
 rename fdbserver/workloads/{Inventory.actor.cpp => Inventory.cpp} (82%)
 rename fdbserver/workloads/{KVStoreTest.actor.cpp => KVStoreTest.cpp} (81%)
 rename fdbserver/workloads/{KillRegion.actor.cpp => KillRegion.cpp} (65%)
 rename fdbserver/workloads/{LocalRatekeeper.actor.cpp => LocalRatekeeper.cpp} (69%)
 rename fdbserver/workloads/{LockDatabase.actor.cpp => LockDatabase.cpp} (57%)
 rename fdbserver/workloads/{LockDatabaseFrequently.actor.cpp => LockDatabaseFrequently.cpp} (55%)
 rename fdbserver/workloads/{LogMetrics.actor.cpp => LogMetrics.cpp} (67%)
 rename fdbserver/workloads/{LowLatency.actor.cpp => LowLatency.cpp} (68%)
 rename fdbserver/workloads/{MachineAttrition.actor.cpp => MachineAttrition.cpp} (74%)
 rename fdbserver/workloads/{Mako.actor.cpp => Mako.cpp} (63%)
 create mode 100644 fdbserver/workloads/MaxGrvQueueDelay.cpp
 rename fdbserver/{include/fdbserver => }/workloads/MemoryKeyValueStore.h (97%)
 rename fdbserver/workloads/{MemoryLifetime.actor.cpp => MemoryLifetime.cpp} (62%)
 rename fdbserver/workloads/{MetricLogging.actor.cpp => MetricLogging.cpp} (74%)
 rename fdbserver/workloads/{MiniCycle.actor.cpp => MiniCycle.cpp} (81%)
 rename fdbserver/workloads/{MockDDReadWrite.actor.cpp => MockDDReadWrite.cpp} (90%)
 rename fdbserver/workloads/{MockDDTest.actor.cpp => MockDDTest.cpp} (97%)
 rename fdbserver/{include/fdbserver => }/workloads/MockDDTest.h (93%)
 rename fdbserver/workloads/{MockDDTrackerShardEvaluator.actor.cpp => MockDDTrackerShardEvaluator.cpp} (83%)
 rename fdbserver/workloads/{MutationLogReaderCorrectness.actor.cpp => MutationLogReaderCorrectness.cpp} (80%)
 delete mode 100644 fdbserver/workloads/ParallelRestore.actor.cpp
 rename fdbserver/workloads/{Performance.actor.cpp => Performance.cpp} (79%)
 delete mode 100644 fdbserver/workloads/PerpetualWiggleStatsWorkload.actor.cpp
 create mode 100644 fdbserver/workloads/PerpetualWiggleStatsWorkload.cpp
 rename fdbserver/workloads/{PerpetualWiggleStorageMigrationWorkload.actor.cpp => PerpetualWiggleStorageMigrationWorkload.cpp} (67%)
 rename fdbserver/workloads/{PhysicalShardMove.actor.cpp => PhysicalShardMove.cpp} (58%)
 rename fdbserver/workloads/{Ping.actor.cpp => Ping.cpp} (76%)
 rename fdbserver/workloads/{PrivateEndpoints.actor.cpp => PrivateEndpoints.cpp} (83%)
 rename fdbserver/workloads/{ProtocolVersion.actor.cpp => ProtocolVersion.cpp} (70%)
 rename fdbserver/workloads/{PubSubMultiples.actor.cpp => PubSubMultiples.cpp} (69%)
 rename fdbserver/workloads/{QueuePush.actor.cpp => QueuePush.cpp} (80%)
 rename fdbserver/workloads/{RYWDisable.actor.cpp => RYWDisable.cpp} (72%)
 delete mode 100644 fdbserver/workloads/RYWPerformance.actor.cpp
 create mode 100644 fdbserver/workloads/RYWPerformance.cpp
 rename fdbserver/workloads/{RandomClogging.actor.cpp => RandomClogging.cpp} (67%)
 rename fdbserver/workloads/{RandomMoveKeys.actor.cpp => RandomMoveKeys.cpp} (80%)
 rename fdbserver/workloads/{RandomRangeLock.actor.cpp => RandomRangeLock.cpp} (79%)
 rename fdbserver/workloads/{RandomSelector.actor.cpp => RandomSelector.cpp} (65%)
 rename fdbserver/workloads/{RangeLock.actor.cpp => RangeLock.cpp} (72%)
 rename fdbserver/workloads/{ReadAfterWrite.actor.cpp => ReadAfterWrite.cpp} (60%)
 rename fdbserver/workloads/{ReadHotDetection.actor.cpp => ReadHotDetection.cpp} (70%)
 rename fdbserver/workloads/{ReadWrite.actor.cpp => ReadWrite.cpp} (53%)
 rename fdbserver/{include/fdbserver/workloads/ReadWriteWorkload.actor.h => workloads/ReadWriteWorkload.h} (76%)
 rename fdbserver/workloads/{RemoveServersSafely.actor.cpp => RemoveServersSafely.cpp} (81%)
 rename fdbserver/workloads/{ReportConflictingKeys.actor.cpp => ReportConflictingKeys.cpp} (85%)
 rename fdbserver/workloads/{ResolverBug.actor.cpp => ResolverBug.cpp} (66%)
 rename fdbserver/workloads/{Restore.actor.cpp => Restore.cpp} (63%)
 rename fdbserver/workloads/{RestoreBackup.actor.cpp => RestoreBackup.cpp} (59%)
 rename fdbserver/workloads/{RestoreMultiRanges.actor.cpp => RestoreMultiRanges.cpp} (52%)
 rename fdbserver/workloads/{RestoreValidation.actor.cpp => RestoreValidation.cpp} (71%)
 rename fdbserver/workloads/{Rollback.actor.cpp => Rollback.cpp} (78%)
 rename fdbserver/workloads/{RyowCorrectness.actor.cpp => RyowCorrectness.cpp} (79%)
 rename fdbserver/workloads/{S3ClientWorkload.actor.cpp => S3ClientWorkload.cpp} (83%)
 rename fdbserver/workloads/{SaveAndKill.actor.cpp => SaveAndKill.cpp} (83%)
 rename fdbserver/workloads/{SelectorCorrectness.actor.cpp => SelectorCorrectness.cpp} (79%)
 rename fdbserver/workloads/{Serializability.actor.cpp => Serializability.cpp} (57%)
 rename fdbserver/workloads/{Sideband.actor.cpp => Sideband.cpp} (70%)
 rename fdbserver/workloads/{SidebandSingle.actor.cpp => SidebandSingle.cpp} (75%)
 rename fdbserver/workloads/{SimpleAtomicAdd.actor.cpp => SimpleAtomicAdd.cpp} (53%)
 rename fdbserver/workloads/{SkewedReadWrite.actor.cpp => SkewedReadWrite.cpp} (72%)
 rename fdbserver/workloads/{SlowTaskWorkload.actor.cpp => SlowTaskWorkload.cpp} (89%)
 rename fdbserver/workloads/{SnapTest.actor.cpp => SnapTest.cpp} (69%)
 rename fdbserver/workloads/{SpecialKeySpaceCorrectness.actor.cpp => SpecialKeySpaceCorrectness.cpp} (73%)
 rename fdbserver/workloads/{SpecialKeySpaceRobustness.actor.cpp => SpecialKeySpaceRobustness.cpp} (61%)
 rename fdbserver/workloads/{StatusWorkload.actor.cpp => StatusWorkload.cpp} (79%)
 rename fdbserver/workloads/{StorageCorruption.actor.cpp => StorageCorruption.cpp} (74%)
 rename fdbserver/workloads/{StorageServerCheckpointRestoreTest.actor.cpp => StorageServerCheckpointRestoreTest.cpp} (60%)
 rename fdbserver/workloads/{Storefront.actor.cpp => Storefront.cpp} (78%)
 rename fdbserver/workloads/{StreamingRangeRead.actor.cpp => StreamingRangeRead.cpp} (57%)
 rename fdbserver/workloads/{StreamingRead.actor.cpp => StreamingRead.cpp} (78%)
 rename fdbserver/workloads/{SubmitBackup.actor.cpp => SubmitBackup.cpp} (61%)
 rename fdbserver/workloads/{SuspendProcesses.actor.cpp => SuspendProcesses.cpp} (73%)
 rename fdbserver/workloads/{TagThrottleApi.actor.cpp => TagThrottleApi.cpp} (65%)
 rename fdbserver/workloads/{TargetedKill.actor.cpp => TargetedKill.cpp} (85%)
 rename fdbserver/workloads/{TaskBucketCorrectness.actor.cpp => TaskBucketCorrectness.cpp} (72%)
 rename fdbserver/workloads/{ThreadSafety.actor.cpp => ThreadSafety.cpp} (83%)
 rename fdbserver/workloads/{Throttling.actor.cpp => Throttling.cpp} (63%)
 rename fdbserver/workloads/{Throughput.actor.cpp => Throughput.cpp} (76%)
 delete mode 100644 fdbserver/workloads/ThroughputQuota.actor.cpp
 rename fdbserver/workloads/{TimeKeeperCorrectness.actor.cpp => TimeKeeperCorrectness.cpp} (61%)
 rename fdbserver/workloads/{TransactionCost.actor.cpp => TransactionCost.cpp} (74%)
 rename fdbserver/workloads/{TriggerRecovery.actor.cpp => TriggerRecovery.cpp} (67%)
 rename fdbserver/workloads/{TxnTimeout.actor.cpp => TxnTimeout.cpp} (62%)
 delete mode 100644 fdbserver/workloads/UDPWorkload.actor.cpp
 create mode 100644 fdbserver/workloads/UDPWorkload.cpp
 rename fdbserver/workloads/{UnitPerf.actor.cpp => UnitPerf.cpp} (74%)
 rename fdbserver/workloads/{UnitTests.actor.cpp => UnitTests.cpp} (79%)
 rename fdbserver/workloads/{Unreadable.actor.cpp => Unreadable.cpp} (92%)
 rename fdbserver/workloads/{ValidateStorage.actor.cpp => ValidateStorage.cpp} (62%)
 rename fdbserver/workloads/{VersionStamp.actor.cpp => VersionStamp.cpp} (66%)
 rename fdbserver/workloads/{WatchAndWait.actor.cpp => WatchAndWait.cpp} (69%)
 rename fdbserver/workloads/{Watches.actor.cpp => Watches.cpp} (89%)
 rename fdbserver/workloads/{WatchesSameKeyCorrectness.actor.cpp => WatchesSameKeyCorrectness.cpp} (51%)
 rename fdbserver/workloads/{WorkerErrors.actor.cpp => WorkerErrors.cpp} (61%)
 rename fdbserver/workloads/{WriteBandwidth.actor.cpp => WriteBandwidth.cpp} (72%)
 rename fdbserver/workloads/{WriteDuringRead.actor.cpp => WriteDuringRead.cpp} (57%)
 rename fdbserver/workloads/{WriteTagThrottling.actor.cpp => WriteTagThrottling.cpp} (80%)
 rename fdbserver/{pubsub.actor.cpp => workloads/pubsub.cpp} (62%)
 rename fdbserver/{include/fdbserver => workloads}/pubsub.h (98%)
 rename flow/{CompressedInt.actor.cpp => CompressedInt.cpp} (93%)
 rename flow/{include/flow => }/CompressionUtils.h (100%)
 rename {fdbrpc => flow}/CoroTests.cpp (67%)
 rename flow/{include/flow/EventTypes.actor.h => EventTypes.h} (59%)
 rename flow/{include/flow => }/FileTraceLogWriter.h (100%)
 create mode 100644 flow/FlowTest.cpp
 rename flow/{Hostname.actor.cpp => Hostname.cpp} (75%)
 delete mode 100644 flow/IAsyncFile.actor.cpp
 create mode 100644 flow/IAsyncFile.cpp
 rename flow/{IThreadPoolTest.actor.cpp => IThreadPoolTest.cpp} (68%)
 rename flow/{include/flow => }/JsonTraceLogFormatter.h (100%)
 rename flow/{include/flow => }/MetricSample.h (100%)
 rename flow/{Net2.actor.cpp => Net2.cpp} (87%)
 rename flow/{Platform.actor.cpp => Platform.cpp} (97%)
 rename flow/{Profiler.actor.cpp => Profiler.cpp} (90%)
 rename flow/{TLSConfig.actor.cpp => TLSConfig.cpp} (95%)
 rename flow/{ThreadHelper.actor.cpp => ThreadHelper.cpp} (81%)
 rename flow/{WriteOnlySet.actor.cpp => WriteOnlySet.cpp} (96%)
 rename flow/{include/flow => }/XmlTraceLogFormatter.h (100%)
 create mode 100644 flow/bench/BenchAsyncResult.cpp
 rename {flowbench => flow/bench}/BenchCallback.cpp (100%)
 rename flowbench/BenchCallback.actor.cpp => flow/bench/BenchCallbackActor.cpp (84%)
 rename {flowbench => flow/bench}/BenchConflictSet.cpp (100%)
 create mode 100644 flow/bench/BenchCoroChooseRace.cpp
 create mode 100644 flow/bench/BenchCoroutineOverhead.cpp
 rename {flowbench => flow/bench}/BenchHash.cpp (84%)
 create mode 100644 flow/bench/BenchMain.cpp
 rename {flowbench => flow/bench}/BenchMem.cpp (100%)
 create mode 100644 flow/bench/BenchMemcpy.cpp
 rename {flowbench => flow/bench}/BenchNet2.cpp (98%)
 rename flowbench/BenchNet2.actor.cpp => flow/bench/BenchNet2Actor.cpp (67%)
 create mode 100644 flow/bench/BenchNoThrowOnCancel.cpp
 rename flowbench/BenchPriorityMultiLock.actor.cpp => flow/bench/BenchPriorityMultiLock.cpp (76%)
 rename {flowbench => flow/bench}/BenchRandom.cpp (100%)
 rename {flowbench => flow/bench}/BenchRef.cpp (100%)
 rename flowbench/BenchStream.actor.cpp => flow/bench/BenchStream.cpp (71%)
 rename fdbclient/include/fdbclient/EventTypes.actor.h => flow/bench/BenchSupport.h (56%)
 create mode 100644 flow/bench/BenchTimeout.cpp
 rename {flowbench => flow/bench}/BenchTimer.cpp (100%)
 create mode 100644 flow/bench/BenchWaitForAllReady.cpp
 rename {flowbench => flow/bench}/BenchZstd.cpp (99%)
 create mode 100644 flow/bench/CMakeLists.txt
 rename {flowbench => flow/bench}/README.md (70%)
 create mode 100644 flow/genericcoros.h
 rename flowbench/flowbench.actor.cpp => flow/include/flow/BenchMain.h (71%)
 delete mode 100644 flow/include/flow/IndexedSet.actor.h
 delete mode 100644 flow/include/flow/Platform.actor.h
 rename flow/include/flow/{PriorityMultiLock.actor.h => PriorityMultiLock.h} (87%)
 rename flow/include/flow/{TDMetric.actor.h => TDMetric.h} (91%)
 rename flow/include/flow/{TLSConfig.actor.h => TLSConfig.h} (95%)
 create mode 100644 flow/include/flow/TxnCounters.h
 rename flow/include/flow/{WatchFile.actor.h => WatchFile.h} (60%)
 rename flow/{include/flow => }/rte_memcpy.h (100%)
 delete mode 100644 flow/test_memcpy_perf.cpp
 delete mode 100644 flowbench/CMakeLists.txt
 delete mode 100644 tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
 delete mode 100644 tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
 create mode 100644 tests/TestRunner/fdb_test_runner/fdb_version.py
 delete mode 100644 tests/fast/EncryptionUnitTests.toml
 create mode 100644 tests/fast/MaxGrvQueueDelay.toml
 create mode 100644 tests/fast/StatusDuringOutage.toml
 delete mode 100644 tests/rare/ExpectStableThroughput.toml
 delete mode 100644 tests/rare/ThroughputQuota.toml
 delete mode 100644 tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
 delete mode 100644 tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
 delete mode 100644 tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
 delete mode 100644 tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
 delete mode 100644 tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
 delete mode 100644 tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
 delete mode 100644 tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
 delete mode 100644 tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml

diff --git a/.clang-tidy b/.clang-tidy
new file mode 100644
index 00000000000..5db9e83d08c
--- /dev/null
+++ b/.clang-tidy
@@ -0,0 +1,21 @@
+---
+Checks: >
+  -*,
+  bugprone-sizeof-expression,
+  bugprone-suspicious-string-compare,
+  bugprone-use-after-move,
+  modernize-use-auto,
+  modernize-use-equals-default,
+  modernize-use-override,
+  modernize-use-using,
+  performance-for-range-copy,
+  readability-container-contains,
+  readability-const-return-type,
+  readability-container-size-empty,
+  readability-delete-null-pointer,
+  readability-duplicate-include,
+  readability-inconsistent-ifelse-braces
+CheckOptions:
+  - key: modernize-use-auto.MinTypeNameLength
+    value: '11'
+...
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
new file mode 100644
index 00000000000..7b55968c90d
--- /dev/null
+++ b/.github/workflows/stale.yml
@@ -0,0 +1,24 @@
+name: 'Close stale pull requests'
+on:
+  schedule:
+    - cron: '15 20 * * *' # 20:15 UTC
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
+        with:
+          stale-pr-message: >
+            This PR is stale because it has been open 150 days with no activity.
+            Remove stale label or add a comment to prevent automatic closure in 14 days.
+          days-before-pr-stale: 150
+          days-before-close: 14
+          days-before-issue-stale: 5475 # 15 years, basically disabled for issues
+          # throttled to 20 per day
+          operations-per-run: 20
+          ascending: true
diff --git a/.github/workflows/windows-boost-test.yml b/.github/workflows/windows-boost-test.yml
index 8bf7f551a05..e19b8d5e59c 100644
--- a/.github/workflows/windows-boost-test.yml
+++ b/.github/workflows/windows-boost-test.yml
@@ -24,7 +24,7 @@ jobs:
         run: |
           vcpkg install boost-filesystem:x64-windows boost-iostreams:x64-windows boost-serialization:x64-windows boost-system:x64-windows boost-program-options:x64-windows boost-url:x64-windows boost-context:x64-windows lz4:x64-windows
         shell: cmd
-      
+        
       - name: Configure and Build FoundationDB (without Swift)
         run: |
           mkdir build
diff --git a/.gitignore b/.gitignore
index b608ae140fd..fae11ee2e97 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,7 @@ cmake-build-*/
 *.g.S
 *.g.asm
 *.pom
+fdbserver/include/fdbserver/FDBRocksDBVersion.h
 bindings/java/pom*.xml
 bindings/java/src*/main/overview.html
 bindings/java/src*/main/com/apple/foundationdb/NetworkOptions.java
@@ -113,4 +114,5 @@ temp/
 
 # AI assistants
 *.aider*
-*.fdq
\ No newline at end of file
+*.fdq
+.claude/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 01314db22ce..33bd957d068 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -314,14 +314,11 @@ if(NOT WIN32)
   if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
                                        # x-compiling.
     add_subdirectory(fdbmonitor)
+    add_subdirectory(fdbkubernetesmonitor)
   endif()
 endif()
 add_subdirectory(fdbbackup)
 add_subdirectory(tests)
-if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
-                                     # x-compiling.
-  add_subdirectory(flowbench EXCLUDE_FROM_ALL)
-endif()
 if(WITH_PYTHON AND WITH_C_BINDING)
   if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
                                        # x-compiling.
diff --git a/README.md b/README.md
index 3a74c410803..dfa32e585f2 100755
--- a/README.md
+++ b/README.md
@@ -140,17 +140,18 @@ Building FoundationDB requires at least 8GB of memory. More memory is needed whe
 
 ### macOS
 
-The build under macOS will work the same way as on Linux. [Homebrew](https://brew.sh/) can be used to install the `boost` library and the `ninja` build tool.
+The build under macOS will work the same way as on Linux. [Homebrew](https://brew.sh/) can be used to install the `boost` library and the `ninja` build tool. Be carefull, curent main branch use boost 1.86, do install this version or just let cmake download one. Also, if swift binding is not interest, use -DBUILD_SWIFT_BINDING=OFF.
 
 ```sh
-cmake -G Ninja <FDB_SOURCE_DIR>
+cmake -G Ninja <FDB_SOURCE_DIR> -B <BUILD_DIR>
+cd <BUILD_DIR>
 ninja
 ```
 
 To generate an installable package,
 
 ```sh
-<FDB_SOURCE_DIR>/packaging/osx/buildpkg.sh . <FDB_SOURCE_DIR>
+<FDB_SOURCE_DIR>/packaging/osx/buildpkg.sh <BUILD_DIR> <FDB_SOURCE_DIR>
 ```
 
 ### Windows
@@ -185,6 +186,10 @@ CMake will not produce a `compile_commands.json` by default; you must pass `-DCM
 
 Note that if the building is done inside the `foundationdb/build` Docker image, the resulting paths will still be incorrect and require manual fixing. One will wish to re-run `cmake` with `-DCMAKE_EXPORT_COMPILE_COMMANDS=OFF` to prevent it from reverting the manual changes.
 
+### Code Formatting and Static Analysis
+
+`clang-format` and `clang-tidy` run as part of CI on every pull request. See the [clang-format](https://apple.github.io/foundationdb/clang-format.html) and [clang-tidy](https://apple.github.io/foundationdb/clang-tidy.html) guides for how to run them locally before pushing.
+
 ### Using IDEs
 
 CMake provides built-in support for several popular IDEs. However, most FoundationDB files are written in the `flow` language, which is an extension of the C++ programming language,  for coroutine support (Note that when FoundationDB was being developed, C++20 was not available). The `flow` language will be transpiled into C++ code using `actorcompiler`, while preventing most IDEs from recognizing `flow`-specific syntax.
diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index a6405a1ccf9..422bbe6e0a4 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -211,7 +211,7 @@ if(NOT WIN32)
   target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
 
   target_include_directories(fdb_c_api_tester_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include ${CMAKE_BINARY_DIR}/flow/include)
-  target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
+  target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11::toml11 Threads::Threads fmt::fmt boost_target)
   if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     target_link_libraries(fdb_c_api_tester_impl PRIVATE stdc++fs)
   endif()
@@ -230,6 +230,9 @@ if(NOT WIN32)
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
   target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
+  if(NOT OPEN_FOR_IDE)
+    strip_debug_symbols(mako)
+  endif()
 
   if(NOT OPEN_FOR_IDE)
     # Make sure that fdb_c.h is compatible with c90
diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp
index bdb2f90f84b..d904e50f760 100644
--- a/bindings/c/fdb_c.cpp
+++ b/bindings/c/fdb_c.cpp
@@ -86,7 +86,7 @@ extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_erro
 		       code == error_code_commit_proxy_memory_limit_exceeded ||
 		       code == error_code_transaction_throttled_hot_shard || code == error_code_batch_transaction_throttled ||
 		       code == error_code_process_behind || code == error_code_tag_throttled ||
-		       code == error_code_proxy_tag_throttled || code == error_code_transaction_rejected_range_locked;
+		       code == error_code_transaction_rejected_range_locked;
 	}
 	return false;
 }
@@ -222,7 +222,7 @@ class CAPICallback final : public ThreadCallback {
 extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f,
                                                          void (*callbackf)(FDBFuture*, void*),
                                                          void* userdata) {
-	CAPICallback* cb = new CAPICallback(callbackf, f, userdata);
+	auto* cb = new CAPICallback(callbackf, f, userdata);
 	int ignore;
 	CATCH_AND_RETURN(TSAVB(f)->callOrSetAsCallback(cb, ignore, 0););
 }
diff --git a/bindings/c/test/apitester/TesterApiWorkload.cpp b/bindings/c/test/apitester/TesterApiWorkload.cpp
index 7a607da357f..b295aeb7a84 100644
--- a/bindings/c/test/apitester/TesterApiWorkload.cpp
+++ b/bindings/c/test/apitester/TesterApiWorkload.cpp
@@ -171,7 +171,7 @@ fdb::KeyRange ApiWorkload::randomNonEmptyKeyRange() {
 }
 
 std::optional<int> ApiWorkload::randomTenant() {
-	if (tenants.size() > 0) {
+	if (!tenants.empty()) {
 		return Random::get().randomInt(0, tenants.size() - 1);
 	} else {
 		return {};
@@ -244,7 +244,7 @@ void ApiWorkload::populateTenantData(TTaskFct cont, std::optional<int> tenantId)
 }
 
 void ApiWorkload::createTenantsIfNecessary(TTaskFct cont) {
-	if (tenants.size() > 0) {
+	if (!tenants.empty()) {
 		ASSERT(false);
 	} else {
 		schedule(cont);
@@ -252,7 +252,7 @@ void ApiWorkload::createTenantsIfNecessary(TTaskFct cont) {
 }
 
 void ApiWorkload::populateData(TTaskFct cont) {
-	if (tenants.size() > 0) {
+	if (!tenants.empty()) {
 		populateTenantData(cont, std::make_optional(0));
 	} else {
 		populateTenantData(cont, {});
diff --git a/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp
index 3743bbab882..fa619e55536 100644
--- a/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp
+++ b/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp
@@ -37,8 +37,8 @@ class AtomicOpsCorrectnessWorkload : public ApiWorkload {
 	AtomicOpsCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {}
 
 private:
-	typedef std::function<uint64_t(uint64_t, uint64_t)> IntAtomicOpFunction;
-	typedef std::function<Value(ValueRef, ValueRef)> AtomicOpFunction;
+	using IntAtomicOpFunction = std::function<uint64_t(uint64_t, uint64_t)>;
+	using AtomicOpFunction = std::function<Value(ValueRef, ValueRef)>;
 
 	enum OpType {
 		OP_ATOMIC_ADD,
@@ -57,7 +57,7 @@ class AtomicOpsCorrectnessWorkload : public ApiWorkload {
 	};
 
 	void randomOperation(TTaskFct cont) override {
-		OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
+		auto txType = OpType(Random::get().randomInt(0, OP_LAST));
 
 		switch (txType) {
 		case OP_ATOMIC_ADD:
diff --git a/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
index deb77e16e47..c58b75a874d 100644
--- a/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
+++ b/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
@@ -248,7 +248,7 @@ class ApiCorrectnessWorkload : public ApiWorkload {
 				                      results->size()));
 			    } else {
 				    auto expected_kv = expected.begin();
-				    for (auto actual_kv : *results) {
+				    for (const auto& actual_kv : *results) {
 					    if (actual_kv.key != expected_kv->key || actual_kv.value != expected_kv->value) {
 						    error(fmt::format(
 						        "randomGetRangeOp mismatch. expected key: {} actual key: {} expected value: "
@@ -266,7 +266,7 @@ class ApiCorrectnessWorkload : public ApiWorkload {
 		    getTenant(tenantId));
 	}
 
-	void randomOperation(TTaskFct cont) {
+	void randomOperation(TTaskFct cont) override {
 		std::optional<int> tenantId = randomTenant();
 		OpType txType = (stores[tenantId].size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
 
diff --git a/bindings/c/test/apitester/TesterKeyValueStore.cpp b/bindings/c/test/apitester/TesterKeyValueStore.cpp
index af572d628bf..2a4e910b707 100644
--- a/bindings/c/test/apitester/TesterKeyValueStore.cpp
+++ b/bindings/c/test/apitester/TesterKeyValueStore.cpp
@@ -42,7 +42,7 @@ bool KeyValueStore::exists(fdb::KeyRef key) {
 fdb::Key KeyValueStore::getKey(fdb::KeyRef keyName, bool orEqual, int offset) const {
 	std::unique_lock<std::mutex> lock(mutex);
 	// Begin by getting the start key referenced by the key selector
-	std::map<fdb::Key, fdb::Value>::const_iterator mapItr = store.lower_bound(keyName);
+	auto mapItr = store.lower_bound(keyName);
 
 	// Update the iterator position if necessary based on the value of orEqual
 	int count = 0;
@@ -92,16 +92,16 @@ std::vector<fdb::KeyValue> KeyValueStore::getRange(fdb::KeyRef begin, fdb::KeyRe
 	std::unique_lock<std::mutex> lock(mutex);
 	std::vector<fdb::KeyValue> results;
 	if (!reverse) {
-		std::map<fdb::Key, fdb::Value>::const_iterator mapItr = store.lower_bound(begin);
-
-		for (; mapItr != store.end() && mapItr->first < end && results.size() < limit; mapItr++)
+		for (auto mapItr = store.lower_bound(begin);
+		     mapItr != store.end() && mapItr->first < end && results.size() < limit;
+		     mapItr++)
 			results.push_back(fdb::KeyValue{ mapItr->first, mapItr->second });
 	}
 
 	// Support for reverse getRange queries is supported, but not tested at this time.  This is because reverse range
 	// queries have been disallowed by the database at the API level
 	else {
-		std::map<fdb::Key, fdb::Value>::const_iterator mapItr = store.lower_bound(end);
+		auto mapItr = store.lower_bound(end);
 		if (mapItr == store.begin())
 			return results;
 
diff --git a/bindings/c/test/apitester/TesterTestSpec.cpp b/bindings/c/test/apitester/TesterTestSpec.cpp
index 2a8ab161691..f65784a7d29 100644
--- a/bindings/c/test/apitester/TesterTestSpec.cpp
+++ b/bindings/c/test/apitester/TesterTestSpec.cpp
@@ -137,7 +137,7 @@ TestSpec readTomlTestSpec(std::string fileName) {
 
 	// Then parse each test
 	const toml::array& tests = toml::find(conf, "test").as_array();
-	if (tests.size() == 0) {
+	if (tests.empty()) {
 		throw TesterError("Invalid test file. No [test] section found");
 	} else if (tests.size() > 1) {
 		throw TesterError("Invalid test file. More than one [test] section found");
diff --git a/bindings/c/test/apitester/TesterTransactionExecutor.cpp b/bindings/c/test/apitester/TesterTransactionExecutor.cpp
index 6bd4cf30d62..19cec9fe8fc 100644
--- a/bindings/c/test/apitester/TesterTransactionExecutor.cpp
+++ b/bindings/c/test/apitester/TesterTransactionExecutor.cpp
@@ -97,7 +97,7 @@ class TransactionContextBase : public ITransactionContext {
 		}
 	}
 
-	virtual ~TransactionContextBase() { ASSERT(txState == TxState::DONE); }
+	~TransactionContextBase() override { ASSERT(txState == TxState::DONE); }
 
 	// A state machine:
 	// IN_PROGRESS -> (ON_ERROR -> IN_PROGRESS)* [-> ON_ERROR] -> DONE
@@ -165,7 +165,7 @@ class TransactionContextBase : public ITransactionContext {
 		}
 	}
 
-	virtual void onError(fdb::Error err) override {
+	void onError(fdb::Error err) override {
 		std::unique_lock<std::mutex> lock(mutex);
 		if (txState != TxState::IN_PROGRESS) {
 			// Ignore further errors, if the transaction is in the error handing mode or completed
@@ -438,7 +438,7 @@ class BlockingTransactionContext : public TransactionContextBase {
 		onError(err);
 	}
 
-	virtual void handleOnErrorFuture() override {
+	void handleOnErrorFuture() override {
 		ASSERT(txState == TxState::ON_ERROR);
 
 		auto start = timeNow();
@@ -503,7 +503,7 @@ class AsyncTransactionContext : public TransactionContextBase {
 
 	static void futureReadyCallback(fdb::Future f, void* param) {
 		try {
-			AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
+			auto* txCtx = (AsyncTransactionContext*)param;
 			txCtx->onFutureReady(f);
 		} catch (std::exception& err) {
 			fmt::print("Unexpected exception in callback {}\n", err.what());
@@ -550,7 +550,7 @@ class AsyncTransactionContext : public TransactionContextBase {
 		onError(err);
 	}
 
-	virtual void handleOnErrorFuture() override {
+	void handleOnErrorFuture() override {
 		ASSERT(txState == TxState::ON_ERROR);
 
 		onErrorCallTimePoint = timeNow();
@@ -565,7 +565,7 @@ class AsyncTransactionContext : public TransactionContextBase {
 
 	static void onErrorReadyCallback(fdb::Future f, void* param) {
 		try {
-			AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
+			auto* txCtx = (AsyncTransactionContext*)param;
 			txCtx->onErrorReady(f);
 		} catch (std::exception& err) {
 			fmt::print("Unexpected exception in callback {}\n", err.what());
@@ -642,7 +642,7 @@ class TransactionExecutorBase : public ITransactionExecutor {
 public:
 	TransactionExecutorBase(const TransactionExecutorOptions& options) : options(options), scheduler(nullptr) {}
 
-	~TransactionExecutorBase() {
+	~TransactionExecutorBase() override {
 		if (tamperClusterFileThread.joinable()) {
 			tamperClusterFileThread.join();
 		}
diff --git a/bindings/c/test/apitester/TesterUtil.h b/bindings/c/test/apitester/TesterUtil.h
index 4b5e6d41926..f8c96f63892 100644
--- a/bindings/c/test/apitester/TesterUtil.h
+++ b/bindings/c/test/apitester/TesterUtil.h
@@ -136,7 +136,8 @@ KeyRangeArray copyKeyRangeArray(fdb::future_var::KeyRangeRefArray::Type array);
 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "Do not support non-little-endian systems");
 
 // Converts a little-endian encoded number into an integral type.
-template <class T, typename = std::enable_if_t<std::is_integral<T>::value>>
+template <class T>
+    requires(std::is_integral_v<T>)
 static T toInteger(fdb::BytesRef value) {
 	ASSERT(value.size() == sizeof(T));
 	T output;
@@ -145,7 +146,8 @@ static T toInteger(fdb::BytesRef value) {
 }
 
 // Converts an integral type to a little-endian encoded byte string.
-template <class T, typename = std::enable_if_t<std::is_integral<T>::value>>
+template <class T>
+    requires(std::is_integral_v<T>)
 static fdb::ByteString toByteString(T value) {
 	fdb::ByteString output(sizeof(T), 0);
 	memcpy(output.data(), (const uint8_t*)&value, sizeof(value));
diff --git a/bindings/c/test/apitester/TesterWorkload.cpp b/bindings/c/test/apitester/TesterWorkload.cpp
index ec8ca8487d7..6faf51fc769 100644
--- a/bindings/c/test/apitester/TesterWorkload.cpp
+++ b/bindings/c/test/apitester/TesterWorkload.cpp
@@ -204,14 +204,14 @@ void WorkloadManager::run() {
 	std::vector<std::shared_ptr<IWorkload>> initialWorkloads;
 	{
 		std::unique_lock<std::mutex> lock(mutex);
-		for (auto iter : workloads) {
+		for (const auto& iter : workloads) {
 			initialWorkloads.push_back(iter.second.ref);
 		}
 	}
-	for (auto iter : initialWorkloads) {
+	for (const auto& iter : initialWorkloads) {
 		iter->init(this);
 	}
-	for (auto iter : initialWorkloads) {
+	for (const auto& iter : initialWorkloads) {
 		iter->start();
 	}
 	scheduler->join();
@@ -265,7 +265,7 @@ void WorkloadManager::readControlInput(std::string pipeName) {
 	// Open in binary mode and read char-by-char to avoid
 	// any kind of buffering
 	FILE* f = fopen(pipeName.c_str(), "rb");
-	setbuf(f, NULL);
+	setbuf(f, nullptr);
 	std::string line;
 	while (true) {
 		int ch = fgetc(f);
@@ -291,7 +291,7 @@ void WorkloadManager::readControlInput(std::string pipeName) {
 
 void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) {
 	statsTimer = scheduler->scheduleWithDelay(timeIntervalMs, [this, timeIntervalMs]() {
-		for (auto workload : getActiveWorkloads()) {
+		for (const auto& workload : getActiveWorkloads()) {
 			workload->printStats();
 		}
 		this->schedulePrintStatistics(timeIntervalMs);
@@ -301,7 +301,7 @@ void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) {
 std::vector<std::shared_ptr<IWorkload>> WorkloadManager::getActiveWorkloads() {
 	std::unique_lock<std::mutex> lock(mutex);
 	std::vector<std::shared_ptr<IWorkload>> res;
-	for (auto iter : workloads) {
+	for (const auto& iter : workloads) {
 		res.push_back(iter.second.ref);
 	}
 	return res;
diff --git a/bindings/c/test/apitester/fdb_c_api_tester.cpp b/bindings/c/test/apitester/fdb_c_api_tester.cpp
index 4ae10f8fc00..35f38b93d8a 100644
--- a/bindings/c/test/apitester/fdb_c_api_tester.cpp
+++ b/bindings/c/test/apitester/fdb_c_api_tester.cpp
@@ -325,7 +325,7 @@ void applyNetworkOptions(TesterOptions& options) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_RETAIN_CLIENT_LIBRARY_COPIES);
 	}
 
-	for (auto knob : options.testSpec.knobs) {
+	for (const auto& knob : options.testSpec.knobs) {
 		fmt::print(stderr, "Setting knob {}={}\n", knob.first.c_str(), knob.second.c_str());
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB,
 		                        fmt::format("{}={}", knob.first.c_str(), knob.second.c_str()));
diff --git a/bindings/c/test/mako/async.cpp b/bindings/c/test/mako/async.cpp
index 5f9c1b058f5..25bd8640254 100644
--- a/bindings/c/test/mako/async.cpp
+++ b/bindings/c/test/mako/async.cpp
@@ -74,7 +74,7 @@ void ResumableStateForPopulate::runOneTick() {
 					stats.incrOpCount(OP_COMMIT);
 					stats.incrOpCount(OP_TRANSACTION);
 					tx.reset();
-					setTransactionTimeoutIfEnabled(args, tx);
+					setTransactionOptionsIfEnabled(args, tx);
 					watch_tx.startFromStop();
 					key_checkpoint = i + 1;
 					if (i != key_end) {
@@ -165,7 +165,7 @@ void ResumableStateForRunWorkload::updateStepStats() {
 			stats.addLatency(OP_COMMIT, step_latency);
 		}
 		tx.reset();
-		setTransactionTimeoutIfEnabled(args, tx);
+		setTransactionOptionsIfEnabled(args, tx);
 		stats.incrOpCount(OP_COMMIT);
 		needs_commit = false;
 	}
@@ -211,7 +211,7 @@ void ResumableStateForRunWorkload::onTransactionSuccess() {
 				stats.incrOpCount(OP_COMMIT);
 				stats.incrOpCount(OP_TRANSACTION);
 				tx.reset();
-				setTransactionTimeoutIfEnabled(args, tx);
+				setTransactionOptionsIfEnabled(args, tx);
 				watch_tx.startFromStop();
 				onIterationEnd(FutureRC::OK);
 			}
@@ -226,7 +226,7 @@ void ResumableStateForRunWorkload::onTransactionSuccess() {
 		stats.incrOpCount(OP_TRANSACTION);
 		watch_tx.startFromStop();
 		tx.reset();
-		setTransactionTimeoutIfEnabled(args, tx);
+		setTransactionOptionsIfEnabled(args, tx);
 		onIterationEnd(FutureRC::OK);
 	}
 }
@@ -238,6 +238,9 @@ void ResumableStateForRunWorkload::onIterationEnd(FutureRC rc) {
 	if (ended()) {
 		signalEnd();
 	} else {
+		if (rc == FutureRC::RETRY) {
+			setTransactionOptionsIfEnabled(args, tx);
+		}
 		iter = getOpBegin(args);
 		needs_commit = false;
 		postNextTick();
diff --git a/bindings/c/test/mako/async.hpp b/bindings/c/test/mako/async.hpp
index 26271b7aac7..66867f739d1 100644
--- a/bindings/c/test/mako/async.hpp
+++ b/bindings/c/test/mako/async.hpp
@@ -109,7 +109,7 @@ struct ResumableStateForRunWorkload : std::enable_shared_from_this<ResumableStat
 		key1.resize(args.key_length);
 		key2.resize(args.key_length);
 		val.resize(args.value_length);
-		setTransactionTimeoutIfEnabled(args, tx);
+		setTransactionOptionsIfEnabled(args, tx);
 	}
 	void signalEnd() noexcept { stopcount.fetch_add(1); }
 	bool ended() noexcept { return (max_iters != -1 && total_xacts >= max_iters) || signal.load() == SIGNAL_RED; }
diff --git a/bindings/c/test/mako/mako.cpp b/bindings/c/test/mako/mako.cpp
index b628980bd48..01859e5300f 100644
--- a/bindings/c/test/mako/mako.cpp
+++ b/bindings/c/test/mako/mako.cpp
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include <algorithm>
 #include <array>
 #include <cassert>
 #include <cmath>
@@ -38,6 +39,8 @@
 #include <string_view>
 #include <thread>
 
+#include "flow/BooleanParam.h"
+
 #include <fcntl.h>
 #include <getopt.h>
 #include <sys/mman.h>
@@ -45,8 +48,6 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <boost/asio.hpp>
 #include <fmt/format.h>
 #include <fmt/printf.h>
@@ -291,6 +292,9 @@ int runOneTransaction(Transaction& tx,
 			if (future_rc == FutureRC::ABORT) {
 				return -1;
 			}
+			setTransactionOptionsIfEnabled(args, tx);
+			if (token)
+				tx.setOption(FDB_TR_OPTION_AUTHORIZATION_TOKEN, *token);
 			// retry from first op
 			op_iter = getOpBegin(args);
 			needs_commit = false;
@@ -304,6 +308,7 @@ int runOneTransaction(Transaction& tx,
 				stats.addLatency(OP_COMMIT, step_latency);
 			}
 			tx.reset();
+			setTransactionOptionsIfEnabled(args, tx);
 			if (token)
 				tx.setOption(FDB_TR_OPTION_AUTHORIZATION_TOKEN, *token);
 			stats.incrOpCount(OP_COMMIT);
@@ -331,8 +336,9 @@ int runOneTransaction(Transaction& tx,
 		const auto rc = waitAndHandleError(tx, f, "COMMIT_AT_TX_END", args.isAnyTimeoutEnabled());
 		updateErrorStatsRunMode(stats, f.error(), OP_COMMIT);
 		watch_commit.stop();
-		auto tx_resetter = ExitGuard([&tx, &token]() {
+		auto tx_resetter = ExitGuard([&tx, &token, &args]() {
 			tx.reset();
+			setTransactionOptionsIfEnabled(args, tx);
 			if (token)
 				tx.setOption(FDB_TR_OPTION_AUTHORIZATION_TOKEN, *token);
 		});
@@ -418,7 +424,7 @@ int runWorkload(Database db,
 
 		if (current_tps > 0 || thread_tps == 0 /* throttling off */) {
 			auto [tx, token] = createNewTransaction(db, args, -1);
-			setTransactionTimeoutIfEnabled(args, tx);
+			setTransactionOptionsIfEnabled(args, tx);
 
 			/* enable transaction trace */
 			if (dotrace) {
@@ -541,7 +547,7 @@ void runAsyncWorkload(Arguments const& args,
 		while (shm.headerConst().signal.load() != SIGNAL_GREEN)
 			usleep(1000);
 		// launch [async_xacts] concurrent transactions
-		for (auto state : states)
+		for (const auto& state : states)
 			state->postNextTick();
 		while (stopcount.load() != args.async_xacts)
 			usleep(1000);
@@ -573,7 +579,7 @@ void runAsyncWorkload(Arguments const& args,
 		}
 		while (shm.headerConst().signal.load() != SIGNAL_GREEN)
 			usleep(1000);
-		for (auto state : states)
+		for (const auto& state : states)
 			state->postNextTick();
 		logr.debug("Launched {} concurrent transactions", states.size());
 		while (stopcount.load() != args.async_xacts)
@@ -787,6 +793,7 @@ Arguments::Arguments() {
 	load_factor = 1.0;
 	row_digits = digits(rows);
 	seconds = 0;
+	warmup_seconds = 0;
 	iteration = 0;
 	tpsmax = 0;
 	tpsmin = -1;
@@ -823,6 +830,7 @@ Arguments::Arguments() {
 	distributed_tracer_client = 0;
 	transaction_timeout_db = 0;
 	transaction_timeout_tx = 0;
+	max_grv_queue_delay_ms = -1;
 	num_report_files = 0;
 }
 
@@ -968,6 +976,9 @@ int parseTransaction(Arguments& args, char const* optarg) {
 		} else if (strncmp(ptr, "g", 1) == 0) {
 			op = OP_GET;
 			ptr++;
+		} else if (strncmp(ptr, "sj", 2) == 0) {
+			op = OP_STATUSJSON;
+			ptr += 2;
 		} else if (strncmp(ptr, "sgr", 3) == 0) {
 			op = OP_SGETRANGE;
 			rangeop = 1;
@@ -1084,6 +1095,9 @@ void usage() {
 	printf("%-24s %s\n", "-l, --load_factor=LOAD_FACTOR", "Specify load factor");
 	printf("%-24s %s\n", "-s, --seconds=SECONDS", "Specify the test duration in seconds\n");
 	printf("%-24s %s\n", "", "This option cannot be specified with --iteration.");
+	printf("%-24s %s\n",
+	       "    --warmup_seconds=SECONDS",
+	       "Ignore the initial SECONDS of a run when computing aggregate throughput and count-based totals");
 	printf("%-24s %s\n", "-i, --iteration=ITERS", "Specify the number of iterations.\n");
 	printf("%-24s %s\n", "", "This option cannot be specified with --seconds.");
 	printf("%-24s %s\n", "    --keylen=LENGTH", "Specify the key lengths");
@@ -1135,6 +1149,9 @@ void usage() {
 	printf("%-24s %s\n",
 	       "    --transaction_timeout_tx=DURATION",
 	       "Duration in milliseconds after which a transaction times out in run mode. Set as transaction option");
+	printf("%-24s %s\n",
+	       "    --max_grv_queue_delay=DURATION",
+	       "Maximum estimated GRV proxy queue delay in milliseconds. Set as transaction option in run mode.");
 }
 
 /* parse benchmark parameters */
@@ -1147,59 +1164,64 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
 		static struct option long_options[] = {
 			/* name, has_arg, flag, val */
 			/* options requiring an argument */
-			{ "api_version", required_argument, NULL, 'a' },
-			{ "cluster", required_argument, NULL, 'c' },
-			{ "num_databases", required_argument, NULL, 'd' },
-			{ "procs", required_argument, NULL, 'p' },
-			{ "threads", required_argument, NULL, 't' },
-			{ "async_xacts", required_argument, NULL, ARG_ASYNC },
-			{ "rows", required_argument, NULL, 'r' },
-			{ "load_factor", required_argument, NULL, 'l' },
-			{ "seconds", required_argument, NULL, 's' },
-			{ "iteration", required_argument, NULL, 'i' },
-			{ "keylen", required_argument, NULL, ARG_KEYLEN },
-			{ "vallen", required_argument, NULL, ARG_VALLEN },
-			{ "transaction", required_argument, NULL, 'x' },
-			{ "tps", required_argument, NULL, ARG_TPS },
-			{ "tpsmax", required_argument, NULL, ARG_TPSMAX },
-			{ "tpsmin", required_argument, NULL, ARG_TPSMIN },
-			{ "tpsinterval", required_argument, NULL, ARG_TPSINTERVAL },
-			{ "tpschange", required_argument, NULL, ARG_TPSCHANGE },
-			{ "sampling", required_argument, NULL, ARG_SAMPLING },
-			{ "verbose", required_argument, NULL, 'v' },
-			{ "mode", required_argument, NULL, 'm' },
-			{ "knobs", required_argument, NULL, ARG_KNOBS },
-			{ "loggroup", required_argument, NULL, ARG_LOGGROUP },
-			{ "tracepath", required_argument, NULL, ARG_TRACEPATH },
-			{ "trace_format", required_argument, NULL, ARG_TRACEFORMAT },
-			{ "streaming", required_argument, NULL, ARG_STREAMING_MODE },
-			{ "txntrace", required_argument, NULL, ARG_TXNTRACE },
-			{ "txntagging", required_argument, NULL, ARG_TXNTAGGING },
-			{ "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX },
-			{ "client_threads_per_version", required_argument, NULL, ARG_CLIENT_THREADS_PER_VERSION },
-			{ "bg_file_path", required_argument, NULL, ARG_BG_FILE_PATH },
-			{ "distributed_tracer_client", required_argument, NULL, ARG_DISTRIBUTED_TRACER_CLIENT },
-			{ "tls_certificate_file", required_argument, NULL, ARG_TLS_CERTIFICATE_FILE },
-			{ "tls_key_file", required_argument, NULL, ARG_TLS_KEY_FILE },
-			{ "tls_ca_file", required_argument, NULL, ARG_TLS_CA_FILE },
-			{ "authorization_keypair_id", required_argument, NULL, ARG_AUTHORIZATION_KEYPAIR_ID },
-			{ "authorization_private_key_pem_file", required_argument, NULL, ARG_AUTHORIZATION_PRIVATE_KEY_PEM_FILE },
-			{ "transaction_timeout_tx", required_argument, NULL, ARG_TRANSACTION_TIMEOUT_TX },
-			{ "transaction_timeout_db", required_argument, NULL, ARG_TRANSACTION_TIMEOUT_DB },
+			{ "api_version", required_argument, nullptr, 'a' },
+			{ "cluster", required_argument, nullptr, 'c' },
+			{ "num_databases", required_argument, nullptr, 'd' },
+			{ "procs", required_argument, nullptr, 'p' },
+			{ "threads", required_argument, nullptr, 't' },
+			{ "async_xacts", required_argument, nullptr, ARG_ASYNC },
+			{ "rows", required_argument, nullptr, 'r' },
+			{ "load_factor", required_argument, nullptr, 'l' },
+			{ "seconds", required_argument, nullptr, 's' },
+			{ "warmup_seconds", required_argument, nullptr, ARG_WARMUP_SECONDS },
+			{ "iteration", required_argument, nullptr, 'i' },
+			{ "keylen", required_argument, nullptr, ARG_KEYLEN },
+			{ "vallen", required_argument, nullptr, ARG_VALLEN },
+			{ "transaction", required_argument, nullptr, 'x' },
+			{ "tps", required_argument, nullptr, ARG_TPS },
+			{ "tpsmax", required_argument, nullptr, ARG_TPSMAX },
+			{ "tpsmin", required_argument, nullptr, ARG_TPSMIN },
+			{ "tpsinterval", required_argument, nullptr, ARG_TPSINTERVAL },
+			{ "tpschange", required_argument, nullptr, ARG_TPSCHANGE },
+			{ "sampling", required_argument, nullptr, ARG_SAMPLING },
+			{ "verbose", required_argument, nullptr, 'v' },
+			{ "mode", required_argument, nullptr, 'm' },
+			{ "knobs", required_argument, nullptr, ARG_KNOBS },
+			{ "loggroup", required_argument, nullptr, ARG_LOGGROUP },
+			{ "tracepath", required_argument, nullptr, ARG_TRACEPATH },
+			{ "trace_format", required_argument, nullptr, ARG_TRACEFORMAT },
+			{ "streaming", required_argument, nullptr, ARG_STREAMING_MODE },
+			{ "txntrace", required_argument, nullptr, ARG_TXNTRACE },
+			{ "txntagging", required_argument, nullptr, ARG_TXNTAGGING },
+			{ "txntagging_prefix", required_argument, nullptr, ARG_TXNTAGGINGPREFIX },
+			{ "client_threads_per_version", required_argument, nullptr, ARG_CLIENT_THREADS_PER_VERSION },
+			{ "bg_file_path", required_argument, nullptr, ARG_BG_FILE_PATH },
+			{ "distributed_tracer_client", required_argument, nullptr, ARG_DISTRIBUTED_TRACER_CLIENT },
+			{ "tls_certificate_file", required_argument, nullptr, ARG_TLS_CERTIFICATE_FILE },
+			{ "tls_key_file", required_argument, nullptr, ARG_TLS_KEY_FILE },
+			{ "tls_ca_file", required_argument, nullptr, ARG_TLS_CA_FILE },
+			{ "authorization_keypair_id", required_argument, nullptr, ARG_AUTHORIZATION_KEYPAIR_ID },
+			{ "authorization_private_key_pem_file",
+			  required_argument,
+			  nullptr,
+			  ARG_AUTHORIZATION_PRIVATE_KEY_PEM_FILE },
+			{ "transaction_timeout_tx", required_argument, nullptr, ARG_TRANSACTION_TIMEOUT_TX },
+			{ "transaction_timeout_db", required_argument, nullptr, ARG_TRANSACTION_TIMEOUT_DB },
+			{ "max_grv_queue_delay", required_argument, nullptr, ARG_MAX_GRV_QUEUE_DELAY },
 			/* options which may or may not have an argument */
-			{ "json_report", optional_argument, NULL, ARG_JSON_REPORT },
-			{ "stats_export_path", optional_argument, NULL, ARG_EXPORT_PATH },
+			{ "json_report", optional_argument, nullptr, ARG_JSON_REPORT },
+			{ "stats_export_path", optional_argument, nullptr, ARG_EXPORT_PATH },
 			/* options without an argument */
-			{ "help", no_argument, NULL, 'h' },
-			{ "zipf", no_argument, NULL, 'z' },
-			{ "commitget", no_argument, NULL, ARG_COMMITGET },
-			{ "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS },
-			{ "prefix_padding", no_argument, NULL, ARG_PREFIXPADDING },
-			{ "trace", no_argument, NULL, ARG_TRACE },
-			{ "version", no_argument, NULL, ARG_VERSION },
-			{ "disable_client_bypass", no_argument, NULL, ARG_DISABLE_CLIENT_BYPASS },
-			{ "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW },
-			{ NULL, 0, NULL, 0 }
+			{ "help", no_argument, nullptr, 'h' },
+			{ "zipf", no_argument, nullptr, 'z' },
+			{ "commitget", no_argument, nullptr, ARG_COMMITGET },
+			{ "flatbuffers", no_argument, nullptr, ARG_FLATBUFFERS },
+			{ "prefix_padding", no_argument, nullptr, ARG_PREFIXPADDING },
+			{ "trace", no_argument, nullptr, ARG_TRACE },
+			{ "version", no_argument, nullptr, ARG_VERSION },
+			{ "disable_client_bypass", no_argument, nullptr, ARG_DISABLE_CLIENT_BYPASS },
+			{ "disable_ryw", no_argument, nullptr, ARG_DISABLE_RYW },
+			{ nullptr, 0, nullptr, 0 }
 		};
 
 /* For optional arguments, optarg is only set when the argument is passed as "--option=[ARGUMENT]" but not as
@@ -1228,9 +1250,9 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
 		case 'c': {
 			const char delim[] = ",";
 			char* cluster_file = strtok(optarg, delim);
-			while (cluster_file != NULL) {
+			while (cluster_file != nullptr) {
 				strcpy(args.cluster_files[args.num_fdb_clusters++], cluster_file);
-				cluster_file = strtok(NULL, delim);
+				cluster_file = strtok(nullptr, delim);
 			}
 			break;
 		}
@@ -1276,8 +1298,7 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
 				args.mode = MODE_RUN;
 			} else if (strcmp(optarg, "report") == 0) {
 				args.mode = MODE_REPORT;
-				int i = optind;
-				for (; i < argc; i++) {
+				for (int i = optind; i < argc; i++) {
 					if (argv[i][0] != '-') {
 						const std::string report_file = argv[i];
 						strncpy(args.report_files[args.num_report_files], report_file.c_str(), report_file.size());
@@ -1461,6 +1482,12 @@ int parseArguments(int argc, char* argv[], Arguments& args) {
 		case ARG_TRANSACTION_TIMEOUT_DB:
 			args.transaction_timeout_db = atoi(optarg);
 			break;
+		case ARG_MAX_GRV_QUEUE_DELAY:
+			args.max_grv_queue_delay_ms = atoi(optarg);
+			break;
+		case ARG_WARMUP_SECONDS:
+			args.warmup_seconds = atoi(optarg);
+			break;
 		}
 	}
 
@@ -1555,12 +1582,36 @@ int Arguments::validate() {
 			logr.error("--transaction_timeout_[tx|db] must be a non-negative integer");
 			return -1;
 		}
+		if (max_grv_queue_delay_ms < -1 || max_grv_queue_delay_ms == 0) {
+			logr.error("--max_grv_queue_delay must be a positive integer");
+			return -1;
+		}
+		if (warmup_seconds < 0) {
+			logr.error("--warmup_seconds must be a non-negative integer");
+			return -1;
+		}
+		if (warmup_seconds > 0 && iteration > 0) {
+			logr.error("--warmup_seconds is only supported with --seconds");
+			return -1;
+		}
+		if (seconds > 0 && warmup_seconds >= seconds) {
+			logr.error("--warmup_seconds must be smaller than --seconds");
+			return -1;
+		}
 	}
 
 	if (mode != MODE_RUN && (transaction_timeout_db != 0 || transaction_timeout_tx != 0)) {
 		logr.error("--transaction_timeout_[tx|db] only supported in run mode");
 		return -1;
 	}
+	if (mode != MODE_RUN && max_grv_queue_delay_ms != -1) {
+		logr.error("--max_grv_queue_delay only supported in run mode");
+		return -1;
+	}
+	if (mode != MODE_RUN && warmup_seconds != 0) {
+		logr.error("--warmup_seconds only supported in run mode");
+		return -1;
+	}
 
 	if (mode == MODE_RUN || mode == MODE_BUILD) {
 		if (tpsmax > 0) {
@@ -1654,7 +1705,37 @@ void printStats(Arguments const& args, WorkflowStatistics const* stats, double c
 	prev = current;
 }
 
-void printStatsHeader(Arguments const& args, bool show_commit, bool is_first_header_empty, bool show_op_stats) {
+WorkflowStatistics aggregateWorkerStats(Arguments const& args, WorkflowStatistics const* worker_stats) {
+	auto aggregate = WorkflowStatistics{};
+	const auto num_workers = args.async_xacts > 0 ? args.async_xacts : args.num_threads;
+	for (auto i = 0; i < args.num_processes * num_workers; i++) {
+		aggregate.combine(worker_stats[i]);
+	}
+	return aggregate;
+}
+
+struct WarmupSnapshot {
+	WorkflowStatistics worker_stats;
+	double duration_sec;
+};
+
+std::optional<WarmupSnapshot> maybeCaptureWarmupSnapshot(Arguments const& args,
+                                                         WorkflowStatistics const* worker_stats,
+                                                         double elapsed_sec) {
+	if (args.warmup_seconds == 0 || elapsed_sec < args.warmup_seconds) {
+		return std::nullopt;
+	}
+	return WarmupSnapshot{ aggregateWorkerStats(args, worker_stats), elapsed_sec };
+}
+
+FDB_BOOLEAN_PARAM(ShowCommit);
+FDB_BOOLEAN_PARAM(IsFirstHeaderEmpty);
+FDB_BOOLEAN_PARAM(ShowOpStats);
+
+void printStatsHeader(Arguments const& args,
+                      ShowCommit show_commit,
+                      IsFirstHeaderEmpty is_first_header_empty,
+                      ShowOpStats show_op_stats) {
 	/* header */
 	if (is_first_header_empty)
 		putTitle("");
@@ -1709,7 +1790,7 @@ void printWorkerStats(WorkflowStatistics& final_stats, Arguments args, FILE* fp,
 	}
 
 	fmt::print("Latency (us)");
-	printStatsHeader(args, true, false, true);
+	printStatsHeader(args, ShowCommit::True, IsFirstHeaderEmpty::False, ShowOpStats::True);
 
 	/* Total Samples */
 	putTitle("Samples");
@@ -1944,16 +2025,18 @@ void printReport(Arguments const& args,
                  WorkflowStatistics const* worker_stats,
                  ThreadStatistics const* thread_stats,
                  ProcessStatistics const* process_stats,
-                 double const duration_sec,
+                 double const run_duration_sec,
+                 std::optional<WarmupSnapshot> const& warmup_snapshot,
                  pid_t pid_main,
                  FILE* fp) {
 
-	auto final_worker_stats = WorkflowStatistics{};
-	const auto num_workers = args.async_xacts > 0 ? args.async_xacts : args.num_threads;
-
-	for (auto i = 0; i < args.num_processes * num_workers; i++) {
-		final_worker_stats.combine(worker_stats[i]);
+	auto final_worker_stats = aggregateWorkerStats(args, worker_stats);
+	auto measured_worker_stats = final_worker_stats;
+	if (warmup_snapshot.has_value()) {
+		measured_worker_stats.subtractCounters(warmup_snapshot->worker_stats);
 	}
+	const auto warmup_duration_sec = warmup_snapshot.has_value() ? warmup_snapshot->duration_sec : 0.0;
+	const auto measurement_duration_sec = std::max(run_duration_sec - warmup_duration_sec, 1e-9);
 
 	double cpu_time_worker_threads =
 	    std::accumulate(thread_stats,
@@ -2002,7 +2085,9 @@ void printReport(Arguments const& args,
 	    (total_duration_local_fdb_networks); // assume that external networks have same total duration as local networks
 
 	/* overall stats */
-	fmt::printf("\n====== Total Duration %6.3f sec ======\n\n", duration_sec);
+	fmt::printf("\n====== Measured Duration %6.3f sec ======\n\n", measurement_duration_sec);
+	fmt::printf("Run Duration:      %8.3f\n", run_duration_sec);
+	fmt::printf("Warmup Duration:   %8.3f\n", warmup_duration_sec);
 	fmt::printf("Total Processes:   %8d\n", args.num_processes);
 	fmt::printf("Total Threads:     %8d\n", args.num_threads);
 	fmt::printf("Total Async Xacts: %8d\n", args.async_xacts);
@@ -2025,30 +2110,36 @@ void printReport(Arguments const& args,
 			break;
 		}
 	}
-	const auto tps_f = final_worker_stats.getOpCount(OP_TRANSACTION) / duration_sec;
+	const auto tps_f = measured_worker_stats.getOpCount(OP_TRANSACTION) / measurement_duration_sec;
 	const auto tps_i = static_cast<uint64_t>(tps_f);
 
-	fmt::printf("Total Xacts:       %8lu\n", final_worker_stats.getOpCount(OP_TRANSACTION));
-	fmt::printf("Total Conflicts:   %8lu\n", final_worker_stats.getConflictCount());
-	fmt::printf("Total Errors:      %8lu\n", final_worker_stats.getTotalErrorCount());
-	fmt::printf("Total Timeouts:    %8lu\n", final_worker_stats.getTotalTimeoutCount());
+	fmt::printf("Total Xacts:       %8lu\n", measured_worker_stats.getOpCount(OP_TRANSACTION));
+	fmt::printf("Total Conflicts:   %8lu\n", measured_worker_stats.getConflictCount());
+	fmt::printf("Total Errors:      %8lu\n", measured_worker_stats.getTotalErrorCount());
+	fmt::printf("Total Timeouts:    %8lu\n", measured_worker_stats.getTotalTimeoutCount());
 	fmt::printf("Overall TPS:       %8lu\n\n", tps_i);
 	fmt::printf("%%CPU Worker Processes:         %6.2f \n", cpu_util_worker_processes);
 	fmt::printf("%%CPU Worker Threads:           %6.2f \n", cpu_util_worker_threads);
 	fmt::printf("%%CPU Local Network Threads:    %6.2f \n", cpu_util_local_fdb_networks);
 	fmt::printf("%%CPU External Network Threads: %6.2f \n\n", cpu_util_external_fdb_networks);
+	if (warmup_duration_sec > 0) {
+		fmt::printf(
+		    "Note: aggregate throughput and count-based totals exclude warmup; latency stats still include it.\n\n");
+	}
 
 	if (fp) {
 		fmt::fprintf(fp, "\"results\": {");
-		fmt::fprintf(fp, "\"totalDuration\": %6.3f,", duration_sec);
+		fmt::fprintf(fp, "\"runDuration\": %6.3f,", run_duration_sec);
+		fmt::fprintf(fp, "\"warmupDuration\": %6.3f,", warmup_duration_sec);
+		fmt::fprintf(fp, "\"totalDuration\": %6.3f,", measurement_duration_sec);
 		fmt::fprintf(fp, "\"totalProcesses\": %d,", args.num_processes);
 		fmt::fprintf(fp, "\"totalThreads\": %d,", args.num_threads);
 		fmt::fprintf(fp, "\"totalAsyncXacts\": %d,", args.async_xacts);
 		fmt::fprintf(fp, "\"targetTPS\": %d,", args.tpsmax);
-		fmt::fprintf(fp, "\"totalXacts\": %lu,", final_worker_stats.getOpCount(OP_TRANSACTION));
-		fmt::fprintf(fp, "\"totalConflicts\": %lu,", final_worker_stats.getConflictCount());
-		fmt::fprintf(fp, "\"totalErrors\": %lu,", final_worker_stats.getTotalErrorCount());
-		fmt::fprintf(fp, "\"totalTimeouts\": %lu,", final_worker_stats.getTotalTimeoutCount());
+		fmt::fprintf(fp, "\"totalXacts\": %lu,", measured_worker_stats.getOpCount(OP_TRANSACTION));
+		fmt::fprintf(fp, "\"totalConflicts\": %lu,", measured_worker_stats.getConflictCount());
+		fmt::fprintf(fp, "\"totalErrors\": %lu,", measured_worker_stats.getTotalErrorCount());
+		fmt::fprintf(fp, "\"totalTimeouts\": %lu,", measured_worker_stats.getTotalTimeoutCount());
 		fmt::fprintf(fp, "\"overallTPS\": %lu,", tps_i);
 		fmt::fprintf(fp, "\"workerProcesseCPU\": %.8f,", cpu_util_worker_processes);
 		fmt::fprintf(fp, "\"workerThreadCPU\": %.8f,", cpu_util_worker_threads);
@@ -2057,7 +2148,7 @@ void printReport(Arguments const& args,
 	}
 
 	/* per-op stats */
-	printStatsHeader(args, true, true, false);
+	printStatsHeader(args, ShowCommit::True, IsFirstHeaderEmpty::True, ShowOpStats::False);
 
 	/* OPS */
 	putTitle("Total OPS");
@@ -2067,24 +2158,24 @@ void printReport(Arguments const& args,
 	auto first_op = true;
 	for (auto op = 0; op < MAX_OP; op++) {
 		if ((args.txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) {
-			putField(final_worker_stats.getOpCount(op));
+			putField(measured_worker_stats.getOpCount(op));
 			if (fp) {
 				if (first_op) {
 					first_op = false;
 				} else {
 					fmt::fprintf(fp, ",");
 				}
-				fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), final_worker_stats.getOpCount(op));
+				fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), measured_worker_stats.getOpCount(op));
 			}
 		}
 	}
 
 	/* TPS */
-	const auto tps = final_worker_stats.getOpCount(OP_TRANSACTION) / duration_sec;
+	const auto tps = measured_worker_stats.getOpCount(OP_TRANSACTION) / measurement_duration_sec;
 	putFieldFloat(tps, 2);
 
 	/* Conflicts */
-	const auto conflicts_rate = final_worker_stats.getConflictCount() / duration_sec;
+	const auto conflicts_rate = measured_worker_stats.getConflictCount() / measurement_duration_sec;
 	putFieldFloat(conflicts_rate, 2);
 	fmt::print("\n");
 
@@ -2097,14 +2188,14 @@ void printReport(Arguments const& args,
 	first_op = true;
 	for (auto op = 0; op < MAX_OP; op++) {
 		if ((args.txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) {
-			putField(final_worker_stats.getErrorCount(op));
+			putField(measured_worker_stats.getErrorCount(op));
 			if (fp) {
 				if (first_op) {
 					first_op = false;
 				} else {
 					fmt::fprintf(fp, ",");
 				}
-				fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), final_worker_stats.getErrorCount(op));
+				fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), measured_worker_stats.getErrorCount(op));
 			}
 		}
 	}
@@ -2118,14 +2209,14 @@ void printReport(Arguments const& args,
 	first_op = true;
 	for (auto op = 0; op < MAX_OP; op++) {
 		if ((args.txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) {
-			putField(final_worker_stats.getTimeoutCount(op));
+			putField(measured_worker_stats.getTimeoutCount(op));
 			if (fp) {
 				if (first_op) {
 					first_op = false;
 				} else {
 					fmt::fprintf(fp, ",");
 				}
-				fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), final_worker_stats.getTimeoutCount(op));
+				fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), measured_worker_stats.getTimeoutCount(op));
 			}
 		}
 	}
@@ -2176,6 +2267,7 @@ int statsProcessMain(Arguments const& args,
                      std::atomic<int> const& stopcount,
                      pid_t pid_main) {
 	bool first_stats = true;
+	auto warmup_snapshot = std::optional<WarmupSnapshot>{};
 
 	/* wait until the signal turn on */
 	while (signal.load() == SIGNAL_OFF) {
@@ -2183,9 +2275,9 @@ int statsProcessMain(Arguments const& args,
 	}
 
 	if (args.verbose >= VERBOSE_DEFAULT)
-		printStatsHeader(args, false, true, false);
+		printStatsHeader(args, ShowCommit::False, IsFirstHeaderEmpty::True, ShowOpStats::False);
 
-	FILE* fp = NULL;
+	FILE* fp = nullptr;
 	if (args.json_output_path[0] != '\0') {
 		fp = fopen(args.json_output_path, "w");
 		fmt::fprintf(fp, "{\"makoArgs\": {");
@@ -2198,6 +2290,7 @@ int statsProcessMain(Arguments const& args,
 		fmt::fprintf(fp, "\"rows\": %d,", args.rows);
 		fmt::fprintf(fp, "\"load_factor\": %lf,", args.load_factor);
 		fmt::fprintf(fp, "\"seconds\": %d,", args.seconds);
+		fmt::fprintf(fp, "\"warmup_seconds\": %d,", args.warmup_seconds);
 		fmt::fprintf(fp, "\"iteration\": %d,", args.iteration);
 		fmt::fprintf(fp, "\"tpsmax\": %d,", args.tpsmax);
 		fmt::fprintf(fp, "\"tpsmin\": %d,", args.tpsmin);
@@ -2223,6 +2316,7 @@ int statsProcessMain(Arguments const& args,
 		fmt::fprintf(fp, "\"disable_ryw\": %d,", args.disable_ryw);
 		fmt::fprintf(fp, "\"transaction_timeout_db\": %d,", args.transaction_timeout_db);
 		fmt::fprintf(fp, "\"transaction_timeout_tx\": %d,", args.transaction_timeout_tx);
+		fmt::fprintf(fp, "\"max_grv_queue_delay_ms\": %d,", args.max_grv_queue_delay_ms);
 		fmt::fprintf(fp, "\"json_output_path\": \"%s\"", args.json_output_path);
 		fmt::fprintf(fp, "},\"samples\": [");
 	}
@@ -2235,6 +2329,10 @@ int statsProcessMain(Arguments const& args,
 
 		/* print stats every (roughly) 1 sec */
 		if (toDoubleSeconds(time_now - time_prev) >= 1.0) {
+			if (!warmup_snapshot.has_value()) {
+				warmup_snapshot =
+				    maybeCaptureWarmupSnapshot(args, worker_stats, toDoubleSeconds(time_now - time_start));
+			}
 
 			/* adjust throttle rate if needed */
 			if (args.tpsmax != args.tpsmin) {
@@ -2290,11 +2388,14 @@ int statsProcessMain(Arguments const& args,
 	/* print report */
 	if (args.verbose >= VERBOSE_DEFAULT) {
 		auto time_now = steady_clock::now();
+		const auto run_duration_sec = toDoubleSeconds(time_now - time_start);
+		if (!warmup_snapshot.has_value()) {
+			warmup_snapshot = maybeCaptureWarmupSnapshot(args, worker_stats, run_duration_sec);
+		}
 		while (stopcount.load() < args.num_threads * args.num_processes) {
 			usleep(10000); /* 10ms */
 		}
-		printReport(
-		    args, worker_stats, thread_stats, process_stats, toDoubleSeconds(time_now - time_start), pid_main, fp);
+		printReport(args, worker_stats, thread_stats, process_stats, run_duration_sec, warmup_snapshot, pid_main, fp);
 	}
 
 	if (fp) {
@@ -2360,7 +2461,7 @@ int main(int argc, char* argv[]) {
 
 	if (args.mode == MODE_REPORT) {
 		WorkflowStatistics stats = mergeSketchReport(args);
-		printWorkerStats(stats, args, NULL, true);
+		printWorkerStats(stats, args, nullptr, true);
 		return 0;
 	}
 
@@ -2391,7 +2492,7 @@ int main(int argc, char* argv[]) {
 	}
 
 	/* map it */
-	shm = mmap(NULL, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0);
+	shm = mmap(nullptr, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0);
 	if (shm == MAP_FAILED) {
 		logr.error("mmap (fd:{} size:{}) failed", shmfd, shmsize);
 		return -1;
diff --git a/bindings/c/test/mako/mako.hpp b/bindings/c/test/mako/mako.hpp
index f02e7b6adf3..e4a7bf8708f 100644
--- a/bindings/c/test/mako/mako.hpp
+++ b/bindings/c/test/mako/mako.hpp
@@ -88,6 +88,8 @@ enum ArgKind {
 	ARG_ENABLE_TOKEN_BASED_AUTHORIZATION,
 	ARG_TRANSACTION_TIMEOUT_TX,
 	ARG_TRANSACTION_TIMEOUT_DB,
+	ARG_WARMUP_SECONDS,
+	ARG_MAX_GRV_QUEUE_DELAY,
 };
 
 constexpr const int OP_COUNT = 0;
@@ -101,6 +103,7 @@ enum OpKind {
 	OP_GETRANGE,
 	OP_SGET,
 	OP_SGETRANGE,
+	OP_STATUSJSON,
 	OP_UPDATE,
 	OP_INSERT,
 	OP_INSERTRANGE,
@@ -159,6 +162,7 @@ struct Arguments {
 	double load_factor;
 	int row_digits;
 	int seconds;
+	int warmup_seconds;
 	int iteration;
 	int tpsmax;
 	int tpsmin;
@@ -203,13 +207,17 @@ struct Arguments {
 	std::optional<std::string> private_key_pem;
 	int transaction_timeout_db;
 	int transaction_timeout_tx;
+	int max_grv_queue_delay_ms;
 };
 
 // helper functions
-inline void setTransactionTimeoutIfEnabled(const Arguments& args, fdb::Transaction& tx) {
+inline void setTransactionOptionsIfEnabled(const Arguments& args, fdb::Transaction& tx) {
 	if (args.transaction_timeout_tx > 0) {
 		tx.setOption(FDB_TR_OPTION_TIMEOUT, args.transaction_timeout_tx);
 	}
+	if (args.max_grv_queue_delay_ms > 0) {
+		tx.setOption(FDB_TR_OPTION_MAX_GRV_QUEUE_DELAY, args.max_grv_queue_delay_ms);
+	}
 }
 
 } // namespace mako
diff --git a/bindings/c/test/mako/mako.rst b/bindings/c/test/mako/mako.rst
index c978ec1b85d..99781bfb274 100644
--- a/bindings/c/test/mako/mako.rst
+++ b/bindings/c/test/mako/mako.rst
@@ -68,6 +68,11 @@ Arguments
   | Test duration in seconds (Default: 30)
   | This option cannot be set with ``--iteration``.
 
+- | ``--warmup_seconds <seconds>``
+  | Ignore the initial ``<seconds>`` of a run when computing aggregate throughput and count-based totals.
+  | Raw per-second samples are still emitted for the full run.
+  | This option is only supported with ``--seconds`` and must be smaller than ``--seconds``.
+
 - | ``-i | --iteration <iters>``
   | Specify the number of operations to be executed.
   | This option cannot be set with ``--seconds``.
@@ -143,6 +148,9 @@ Arguments
 - | ``--transaction_timeout_db <duration>``
   | Duration in milliseconds after which a transaction times out in run mode. Set as database option.
 
+- | ``--max_grv_queue_delay <duration>``
+  | Maximum estimated GRV proxy queue delay in milliseconds. Set as transaction option in run mode.
+
 Transaction Specification
 =========================
 | A transaction may contain multiple operations of various types.
@@ -156,6 +164,7 @@ Operation Types
 - ``gr`` – GET RANGE
 - ``sg`` – Snapshot GET
 - ``sgr`` – Snapshot GET RANGE
+- ``sj`` – GET ``\xff\xff/status/json`` from the special key space
 - ``u`` – Update (= GET followed by SET)
 - ``i`` – Insert (= SET with a new key)
 - ``ir`` – Insert Range (Sequential)
diff --git a/bindings/c/test/mako/operations.cpp b/bindings/c/test/mako/operations.cpp
index 7f7cf6a6b79..5c75fa8a599 100644
--- a/bindings/c/test/mako/operations.cpp
+++ b/bindings/c/test/mako/operations.cpp
@@ -111,6 +111,20 @@ const std::array<Operation, MAX_OP> opTable{ {
 	      } } },
 	  1,
 	  false },
+	{ "STATUSJSON",
+	  { { StepKind::READ,
+	      [](Transaction& tx, Arguments const&, ByteString&, ByteString&, ByteString&) {
+	          static constexpr auto status_json_key =
+	              std::string_view{ "\xff\xff/status/json", sizeof("\xff\xff/status/json") - 1 };
+	          return tx.get(toBytesRef(status_json_key), false /*snapshot*/).eraseType();
+	      },
+	      [](Future& f, Transaction&, Arguments const&, ByteString&, ByteString&, ByteString&) {
+	          if (f && !f.error()) {
+		          f.get<future_var::ValueRef>();
+	          }
+	      } } },
+	  1,
+	  false },
 	{ "UPDATE",
 	  { { StepKind::READ,
 	      [](Transaction& tx, Arguments const& args, ByteString& key, ByteString&, ByteString&) {
diff --git a/bindings/c/test/mako/stats.hpp b/bindings/c/test/mako/stats.hpp
index f2498047737..6348da9515b 100644
--- a/bindings/c/test/mako/stats.hpp
+++ b/bindings/c/test/mako/stats.hpp
@@ -22,6 +22,7 @@
 #define MAKO_STATS_HPP
 
 #include <array>
+#include <cassert>
 #include <cstdint>
 #include <cstring>
 #include <fstream>
@@ -174,6 +175,23 @@ class alignas(64) WorkflowStatistics {
 		latency_us_total[op] += latency_us;
 	}
 
+	void subtractCounters(const WorkflowStatistics& baseline) noexcept {
+		assert(conflicts >= baseline.conflicts);
+		assert(total_errors >= baseline.total_errors);
+		assert(total_timeouts >= baseline.total_timeouts);
+		conflicts -= baseline.conflicts;
+		total_errors -= baseline.total_errors;
+		total_timeouts -= baseline.total_timeouts;
+		for (auto op = 0; op < MAX_OP; op++) {
+			assert(ops[op] >= baseline.ops[op]);
+			assert(errors[op] >= baseline.errors[op]);
+			assert(timeouts[op] >= baseline.timeouts[op]);
+			ops[op] -= baseline.ops[op];
+			errors[op] -= baseline.errors[op];
+			timeouts[op] -= baseline.timeouts[op];
+		}
+	}
+
 	void writeToFile(const std::string& filename, int op) const {
 		rapidjson::StringBuffer ss;
 		rapidjson::Writer<rapidjson::StringBuffer> writer(ss);
diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c
index 736f74c75d6..ba4e78d0d0f 100644
--- a/bindings/c/test/performance_test.c
+++ b/bindings/c/test/performance_test.c
@@ -81,8 +81,7 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
             const char* kpiName) {
 	int numRuns = 25;
 	int* results = malloc(sizeof(int) * numRuns);
-	int i = 0;
-	for (; i < numRuns; ++i) {
+	for (int i = 0; i < numRuns; ++i) {
 		struct RunResult res = run(rs, db, testFxn);
 		if (res.e) {
 			logError(res.e, kpiName, rs);
@@ -110,8 +109,7 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
               const char* kpiName) {
 	int numRuns = 25;
 	int* results = malloc(sizeof(int) * numRuns);
-	int i = 0;
-	for (; i < numRuns; ++i) {
+	for (int i = 0; i < numRuns; ++i) {
 		struct RunResult res = testFxn(rs, db);
 		if (res.e) {
 			logError(res.e, kpiName, rs);
@@ -465,8 +463,9 @@ struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
 	}
 
 	if (totalOut != GET_RANGE_COUNT) {
-		char* msg = (char*)malloc((sizeof(char)) * 200);
-		sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
+		const size_t nbytes = 200;
+		char* msg = (char*)malloc(nbytes);
+		snprintf(msg, nbytes, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
 		logError(4100, msg, rs);
 		free(msg);
 		fdb_future_destroy(f);
diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c
index f729f8949ff..2a20bcd46a8 100644
--- a/bindings/c/test/ryw_benchmark.c
+++ b/bindings/c/test/ryw_benchmark.c
@@ -50,8 +50,7 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
             const char* kpiName) {
 	int numRuns = 25;
 	int* results = malloc(sizeof(int) * numRuns);
-	int i = 0;
-	for (; i < numRuns; ++i) {
+	for (int i = 0; i < numRuns; ++i) {
 		results[i] = testFxn(tr, rs);
 		if (results[i] < 0) {
 			free(results);
@@ -231,24 +230,24 @@ int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) {
 	int length;
 	int i;
 
-	uint8_t* k = (uint8_t*)"foo";
-	uint8_t v[10];
+	char* k = "foo";
+	char v[12];
 	int num = 1;
 
 	double start = getTime();
-	sprintf((char*)v, "%d", num);
-	fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
+	snprintf(v, sizeof(v), "%d", num);
+	fdb_transaction_set(tr, (uint8_t*)k, 3, (uint8_t*)v, strlen(v));
 
 	for (i = 0; i < 10000; ++i) {
-		FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
+		FDBFuture* f = fdb_transaction_get(tr, (uint8_t*)k, 3, 0);
 		if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
 			return -1;
 		if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
 			return -1;
 		fdb_future_destroy(f);
 
-		sprintf((char*)v, "%d", ++num);
-		fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
+		snprintf(v, sizeof(v), "%d", ++num);
+		fdb_transaction_set(tr, (uint8_t*)k, 3, (uint8_t*)v, strlen(v));
 	}
 	double end = getTime();
 
diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h
index afb6e5010ac..3f314b885c5 100644
--- a/bindings/c/test/test.h
+++ b/bindings/c/test/test.h
@@ -42,8 +42,9 @@ double getTime() {
 }
 
 void writeKey(uint8_t** dest, int key, int keySize) {
-	*dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize);
-	sprintf((char*)*dest, "%0*d", keySize, key);
+	size_t bufsize = keySize + 1;
+	*dest = (uint8_t*)malloc(bufsize);
+	snprintf((char*)*dest, bufsize, "%0*d", keySize, key);
 }
 
 uint8_t** generateKeys(int numKeys, int keySize) {
@@ -130,7 +131,7 @@ void addError(struct ResultSet* rs, const char* message) {
 void writeResultSet(struct ResultSet* rs) {
 	uint64_t id = ((uint64_t)rand() << 32) + rand();
 	char name[100];
-	sprintf(name, "fdb-c_result-%" SCNu64 ".json", id);
+	snprintf(name, sizeof(name), "fdb-c_result-%" SCNu64 ".json", id);
 	FILE* fp = fopen(name, "w");
 	if (!fp) {
 		fprintf(stderr, "Could not open results file %s\n", name);
@@ -190,8 +191,9 @@ void freeResultSet(struct ResultSet* rs) {
 
 fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) {
 	if (err) {
-		char* msg = (char*)malloc(strlen(context) + 100);
-		sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
+		size_t nbytes = strlen(context) + 100;
+		char* msg = (char*)malloc(nbytes);
+		snprintf(msg, nbytes, "Error in %s: %s", context, fdb_get_error(err));
 		fprintf(stderr, "%s\n", msg);
 		if (rs != NULL) {
 			addError(rs, msg);
@@ -214,8 +216,9 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) {
 }
 
 fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) {
-	char* msg = (char*)malloc(strlen(context) + 100);
-	sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
+	size_t nbytes = strlen(context) + 100;
+	char* msg = (char*)malloc(nbytes);
+	snprintf(msg, nbytes, "Error in %s: %s", context, fdb_get_error(err));
 	fprintf(stderr, "%s\n", msg);
 	if (rs != NULL) {
 		addError(rs, msg);
diff --git a/bindings/c/test/workloads/workloads.cpp b/bindings/c/test/workloads/workloads.cpp
index 5f53d87039d..5e49326fafc 100644
--- a/bindings/c/test/workloads/workloads.cpp
+++ b/bindings/c/test/workloads/workloads.cpp
@@ -20,7 +20,7 @@
 
 #include "workloads.h"
 
-FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {}
+FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() = default;
 
 std::map<std::string, IFDBWorkloadFactory*>& FDBWorkloadFactoryImpl::factories() {
 	static std::map<std::string, IFDBWorkloadFactory*> _factories;
diff --git a/bindings/flow/CMakeLists.txt b/bindings/flow/CMakeLists.txt
index 5e5a31c6bd2..1a7bb07ce54 100644
--- a/bindings/flow/CMakeLists.txt
+++ b/bindings/flow/CMakeLists.txt
@@ -1,19 +1,19 @@
 set(SRCS
-  DirectoryLayer.actor.cpp
+  DirectoryLayer.cpp
   DirectoryLayer.h
   DirectoryPartition.h
   DirectorySubspace.cpp
   DirectorySubspace.h
   FDBLoanerTypes.h
-  HighContentionAllocator.actor.cpp
+  HighContentionAllocator.cpp
   HighContentionAllocator.h
   IDirectory.h
-  Node.actor.cpp
+  Node.cpp
   Subspace.cpp
   Subspace.h
   Tuple.cpp
   Tuple.h
-  fdb_flow.actor.cpp
+  fdb_flow.cpp
   fdb_flow.h)
 
 add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS})
@@ -21,10 +21,8 @@ target_link_libraries(fdb_flow PUBLIC fdb_c)
 target_link_libraries(fdb_flow PUBLIC fdbclient)
 target_include_directories(fdb_flow PUBLIC
   "${CMAKE_SOURCE_DIR}"
-  "${CMAKE_CURRENT_BINARY_DIR}"
   "${CMAKE_CURRENT_SOURCE_DIR}"
   "${CMAKE_CURRENT_SOURCE_DIR}/tester"
-  "${CMAKE_CURRENT_BINARY_DIR}/tester"
   )
 
 add_subdirectory(tester)
diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.cpp
similarity index 56%
rename from bindings/flow/DirectoryLayer.actor.cpp
rename to bindings/flow/DirectoryLayer.cpp
index c9b8ef3d488..86e5378b8f6 100644
--- a/bindings/flow/DirectoryLayer.actor.cpp
+++ b/bindings/flow/DirectoryLayer.cpp
@@ -1,5 +1,5 @@
 /*
- * DirectoryLayer.actor.cpp
+ * DirectoryLayer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -52,34 +52,31 @@ Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) con
 	return nodeWithPrefix(prefix.get());
 }
 
-ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
-                                        Reference<Transaction> tr,
-                                        IDirectory::Path path) {
-	state int pathIndex = 0;
-	state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
+Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
+                                  Reference<Transaction> tr,
+                                  IDirectory::Path path) {
+	DirectoryLayer::Node node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
 
-	for (; pathIndex != path.size(); ++pathIndex) {
+	for (int pathIndex = 0; pathIndex != path.size(); ++pathIndex) {
 		ASSERT(node.subspace.present());
 		Optional<FDBStandalone<ValueRef>> val =
-		    wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
+		    co_await tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key());
 
 		node.path.push_back(path[pathIndex]);
 		node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
 
-		DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
-		node = _node;
+		node = co_await node.loadMetadata(tr);
 
 		if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
-			return node;
+			co_return node;
 		}
 	}
 
 	if (!node.loadedMetadata) {
-		DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
-		node = _node;
+		node = co_await node.loadMetadata(tr);
 	}
 
-	return node;
+	co_return node;
 }
 
 IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
@@ -98,11 +95,10 @@ Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node
 	Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
 
 	if (layer == PARTITION_LAYER) {
-		return Reference<DirectorySubspace>(
-		    new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
+		return makeReference<DirectoryPartition>(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this));
 	} else {
-		return Reference<DirectorySubspace>(
-		    new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
+		return makeReference<DirectorySubspace>(
+		    toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer);
 	}
 }
 
@@ -112,7 +108,7 @@ Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef>
 	if (!allowOpen) {
 		throw directory_already_exists();
 	}
-	if (layer.size() > 0 && layer != existingNode.layer) {
+	if (!layer.empty() && layer != existingNode.layer) {
 		throw mismatched_layer();
 	}
 
@@ -129,15 +125,15 @@ void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const
 	tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
 }
 
-ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
+Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
 	Optional<FDBStandalone<ValueRef>> versionBytes =
-	    wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
+	    co_await tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY));
 
 	if (!versionBytes.present()) {
 		if (writeAccess) {
 			dirLayer->initializeDirectory(tr);
 		}
-		return Void();
+		co_return;
 	} else {
 		if (versionBytes.get().size() != 12) {
 			throw invalid_directory_layer_metadata();
@@ -148,151 +144,147 @@ ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Referenc
 			throw incompatible_directory_version();
 		}
 	}
-
-	return Void();
 }
 
 Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
 	return checkVersionInternal(this, tr, writeAccess);
 }
 
-ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
-                                              Reference<Transaction> tr,
-                                              Optional<Standalone<StringRef>> prefix) {
+Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
+                                        Reference<Transaction> tr,
+                                        Optional<Standalone<StringRef>> prefix) {
 	if (!prefix.present()) {
-		Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
-		state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
+		Standalone<StringRef> allocated = co_await dirLayer->allocator.allocate(tr);
+		Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
 
-		FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
+		FDBStandalone<RangeResultRef> result = co_await tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1);
 
-		if (result.size() > 0) {
+		if (!result.empty()) {
 			throw directory_prefix_not_empty();
 		}
 
-		return finalPrefix;
+		co_return finalPrefix;
 	}
 
-	return prefix.get();
+	co_return prefix.get();
 }
 
-ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
-                                                   Reference<Transaction> tr,
-                                                   Standalone<StringRef> key,
-                                                   bool snapshot) {
+Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
+                                             Reference<Transaction> tr,
+                                             Standalone<StringRef> key,
+                                             bool snapshot) {
 	if (key.startsWith(dirLayer->nodeSubspace.key())) {
-		return dirLayer->rootNode;
+		co_return dirLayer->rootNode;
 	}
 
 	KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
-	FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
+	FDBStandalone<RangeResultRef> result = co_await tr->getRange(range, 1, snapshot, true);
 
-	if (result.size() > 0) {
+	if (!result.empty()) {
 		Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
 		if (key.startsWith(prevPrefix)) {
-			return dirLayer->nodeWithPrefix(prevPrefix);
+			co_return dirLayer->nodeWithPrefix(prevPrefix);
 		}
 	}
 
-	return Optional<Subspace>();
+	co_return Optional<Subspace>();
 }
 
-ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
-                                Reference<Transaction> tr,
-                                Standalone<StringRef> prefix,
-                                bool snapshot) {
-	if (!prefix.size()) {
-		return false;
+Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
+                          Reference<Transaction> tr,
+                          Standalone<StringRef> prefix,
+                          bool snapshot) {
+	if (prefix.empty()) {
+		co_return false;
 	}
 
-	Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
+	Optional<Subspace> node = co_await nodeContainingKey(dirLayer, tr, prefix, snapshot);
 	if (node.present()) {
-		return false;
+		co_return false;
 	}
 
-	FDBStandalone<RangeResultRef> result = wait(tr->getRange(
-	    KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
-	return !result.size();
+	FDBStandalone<RangeResultRef> result = co_await tr->getRange(
+	    KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot);
+	co_return result.empty();
 }
 
-ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
-                                     Reference<Transaction> tr,
-                                     IDirectory::Path path) {
+Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
 	if (path.size() > 1) {
 		Reference<DirectorySubspace> parent =
-		    wait(dirLayer->createOrOpenInternal(tr,
-		                                        IDirectory::Path(path.begin(), path.end() - 1),
-		                                        StringRef(),
-		                                        Optional<Standalone<StringRef>>(),
-		                                        true,
-		                                        true));
-		return dirLayer->nodeWithPrefix(parent->key());
+		    co_await dirLayer->createOrOpenInternal(tr,
+		                                            IDirectory::Path(path.begin(), path.end() - 1),
+		                                            StringRef(),
+		                                            Optional<Standalone<StringRef>>(),
+		                                            true,
+		                                            true);
+		co_return dirLayer->nodeWithPrefix(parent->key());
 	} else {
-		return dirLayer->rootNode;
+		co_return dirLayer->rootNode;
 	}
 }
 
-ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
-                                                          Reference<Transaction> tr,
-                                                          IDirectory::Path path,
-                                                          Standalone<StringRef> layer,
-                                                          Optional<Standalone<StringRef>> prefix,
-                                                          bool allowCreate) {
+Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
+                                                    Reference<Transaction> tr,
+                                                    IDirectory::Path path,
+                                                    Standalone<StringRef> layer,
+                                                    Optional<Standalone<StringRef>> prefix,
+                                                    bool allowCreate) {
 	if (!allowCreate) {
 		throw directory_does_not_exist();
 	}
 
-	wait(dirLayer->checkVersion(tr, true));
+	co_await dirLayer->checkVersion(tr, true);
 
-	state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
-	bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
+	Standalone<StringRef> newPrefix = co_await getPrefix(dirLayer, tr, prefix);
+	bool isFree = co_await isPrefixFree(dirLayer, tr, newPrefix, !prefix.present());
 
 	if (!isFree) {
 		throw directory_prefix_in_use();
 	}
 
-	Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
+	Subspace parentNode = co_await getParentNode(dirLayer, tr, path);
 	Subspace node = dirLayer->nodeWithPrefix(newPrefix);
 
 	tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
 	tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
-	return dirLayer->contentsOfNode(node, path, layer);
+	co_return dirLayer->contentsOfNode(node, path, layer);
 }
 
-ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
-                                                                 Reference<Transaction> tr,
-                                                                 IDirectory::Path path,
-                                                                 Standalone<StringRef> layer,
-                                                                 Optional<Standalone<StringRef>> prefix,
-                                                                 bool allowCreate,
-                                                                 bool allowOpen) {
+Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
+                                                           Reference<Transaction> tr,
+                                                           IDirectory::Path path,
+                                                           Standalone<StringRef> layer,
+                                                           Optional<Standalone<StringRef>> prefix,
+                                                           bool allowCreate,
+                                                           bool allowOpen) {
 	ASSERT(!prefix.present() || allowCreate);
-	wait(dirLayer->checkVersion(tr, false));
+	co_await dirLayer->checkVersion(tr, false);
 
 	if (prefix.present() && !dirLayer->allowManualPrefixes) {
-		if (!dirLayer->getPath().size()) {
+		if (dirLayer->getPath().empty()) {
 			throw manual_prefixes_not_enabled();
 		} else {
 			throw prefix_in_partition();
 		}
 	}
 
-	if (!path.size()) {
+	if (path.empty()) {
 		throw cannot_open_root_directory();
 	}
 
-	state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
+	DirectoryLayer::Node existingNode = co_await find(dirLayer, tr, path);
 	if (existingNode.exists()) {
 		if (existingNode.isInPartition()) {
 			IDirectory::Path subpath = existingNode.getPartitionSubpath();
 			Reference<DirectorySubspace> dirSpace =
-			    wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
-			        tr, subpath, layer, prefix, allowCreate, allowOpen));
-			return dirSpace;
+			    co_await existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
+			        tr, subpath, layer, prefix, allowCreate, allowOpen);
+			co_return dirSpace;
 		}
-		return dirLayer->openInternal(layer, existingNode, allowOpen);
+		co_return dirLayer->openInternal(layer, existingNode, allowOpen);
 	} else {
-		Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
-		return dirSpace;
+		Reference<DirectorySubspace> dirSpace = co_await createInternal(dirLayer, tr, path, layer, prefix, allowCreate);
+		co_return dirSpace;
 	}
 }
 
@@ -319,35 +311,35 @@ Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Tran
 	return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
 }
 
-ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
-                                                            Reference<Transaction> tr,
-                                                            IDirectory::Path path) {
-	wait(dirLayer->checkVersion(tr, false));
+Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
+                                                      Reference<Transaction> tr,
+                                                      IDirectory::Path path) {
+	co_await dirLayer->checkVersion(tr, false);
 
-	state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
+	DirectoryLayer::Node node = co_await find(dirLayer, tr, path);
 
 	if (!node.exists()) {
 		throw directory_does_not_exist();
 	}
 	if (node.isInPartition(true)) {
 		Standalone<VectorRef<StringRef>> partitionList =
-		    wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
-		return partitionList;
+		    co_await node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath());
+		co_return partitionList;
 	}
 
-	state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
-	state Key begin = subdir.range().begin;
-	state Standalone<VectorRef<StringRef>> subdirectories;
+	Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
+	Key begin = subdir.range().begin;
+	Standalone<VectorRef<StringRef>> subdirectories;
 
-	loop {
-		FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
+	while (true) {
+		FDBStandalone<RangeResultRef> subdirRange = co_await tr->getRange(KeyRangeRef(begin, subdir.range().end));
 
 		for (int i = 0; i < subdirRange.size(); ++i) {
 			subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
 		}
 
 		if (!subdirRange.more) {
-			return subdirectories;
+			co_return subdirectories;
 		}
 
 		begin = keyAfter(subdirRange.back().key);
@@ -373,23 +365,19 @@ bool pathsEqual(IDirectory::Path const& path1,
 	return true;
 }
 
-ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
-                                    Reference<Transaction> tr,
-                                    IDirectory::Path path) {
-	ASSERT(path.size() >= 1);
-	DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
+Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
+	ASSERT(!path.empty());
+	DirectoryLayer::Node parentNode = co_await find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1));
 	if (parentNode.subspace.present()) {
 		tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
 	}
-
-	return Void();
 }
 
-ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
-                                                        Reference<Transaction> tr,
-                                                        IDirectory::Path oldPath,
-                                                        IDirectory::Path newPath) {
-	wait(dirLayer->checkVersion(tr, true));
+Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
+                                                  Reference<Transaction> tr,
+                                                  IDirectory::Path oldPath,
+                                                  IDirectory::Path newPath) {
+	co_await dirLayer->checkVersion(tr, true);
 
 	if (oldPath.size() <= newPath.size()) {
 		if (pathsEqual(oldPath, newPath, oldPath.size())) {
@@ -401,10 +389,10 @@ ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer
 	futures.push_back(find(dirLayer, tr, oldPath));
 	futures.push_back(find(dirLayer, tr, newPath));
 
-	std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
+	std::vector<DirectoryLayer::Node> nodes = co_await getAll(futures);
 
-	state DirectoryLayer::Node oldNode = nodes[0];
-	state DirectoryLayer::Node newNode = nodes[1];
+	DirectoryLayer::Node oldNode = nodes[0];
+	DirectoryLayer::Node newNode = nodes[1];
 
 	if (!oldNode.exists()) {
 		throw directory_does_not_exist();
@@ -416,24 +404,24 @@ ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer
 		}
 
 		Reference<DirectorySubspace> partitionMove =
-		    wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
-		return partitionMove;
+		    co_await newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath());
+		co_return partitionMove;
 	}
 
 	if (newNode.exists() || newPath.empty()) {
 		throw directory_already_exists();
 	}
 
-	DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
+	DirectoryLayer::Node parentNode = co_await find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1));
 	if (!parentNode.exists()) {
 		throw parent_directory_does_not_exist();
 	}
 
 	tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
 	        dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
-	wait(removeFromParent(dirLayer, tr, oldPath));
+	co_await removeFromParent(dirLayer, tr, oldPath);
 
-	return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
+	co_return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
 }
 
 Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
@@ -447,14 +435,13 @@ Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transactio
 	throw cannot_modify_root_directory();
 }
 
-Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
-ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
-	state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
-	state Key begin = subdir.range().begin;
-	state std::vector<Future<Void>> futures;
+Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
+	Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
+	Key begin = subdir.range().begin;
+	std::vector<Future<Void>> futures;
 
-	loop {
-		FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
+	while (true) {
+		FDBStandalone<RangeResultRef> range = co_await tr->getRange(KeyRangeRef(begin, subdir.range().end));
 		for (int i = 0; i < range.size(); ++i) {
 			Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
 			futures.push_back(removeRecursive(dirLayer, tr, subNode));
@@ -468,53 +455,47 @@ ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference
 	}
 
 	// waits are done concurrently
-	wait(waitForAll(futures));
+	co_await waitForAll(futures);
 
 	Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
 
 	tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
 	tr->clear(nodeSub.range());
-
-	return Void();
 }
 
-Future<bool> removeInternal(Reference<DirectoryLayer> const&,
-                            Reference<Transaction> const&,
-                            IDirectory::Path const&,
-                            bool const&);
-ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
-                                  Reference<Transaction> tr,
-                                  IDirectory::Path path,
-                                  bool failOnNonexistent) {
-	wait(dirLayer->checkVersion(tr, true));
+Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
+                            Reference<Transaction> tr,
+                            IDirectory::Path path,
+                            bool failOnNonexistent) {
+	co_await dirLayer->checkVersion(tr, true);
 
 	if (path.empty()) {
 		throw cannot_modify_root_directory();
 	}
 
-	state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
+	DirectoryLayer::Node node = co_await find(dirLayer, tr, path);
 
 	if (!node.exists()) {
 		if (failOnNonexistent) {
 			throw directory_does_not_exist();
 		} else {
-			return false;
+			co_return false;
 		}
 	}
 
 	if (node.isInPartition()) {
-		bool recurse = wait(
-		    removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
-		return recurse;
+		bool recurse = co_await removeInternal(
+		    node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent);
+		co_return recurse;
 	}
 
-	state std::vector<Future<Void>> futures;
+	std::vector<Future<Void>> futures;
 	futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
 	futures.push_back(removeFromParent(dirLayer, tr, path));
 
-	wait(waitForAll(futures));
+	co_await waitForAll(futures);
 
-	return true;
+	co_return true;
 }
 
 Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
@@ -525,23 +506,21 @@ Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Pa
 	return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
 }
 
-ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
-                                  Reference<Transaction> tr,
-                                  IDirectory::Path path) {
-	wait(dirLayer->checkVersion(tr, false));
+Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
+	co_await dirLayer->checkVersion(tr, false);
 
-	DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
+	DirectoryLayer::Node node = co_await find(dirLayer, tr, path);
 
 	if (!node.exists()) {
-		return false;
+		co_return false;
 	}
 
 	if (node.isInPartition()) {
-		bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
-		return exists;
+		bool exists = co_await node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath());
+		co_return exists;
 	}
 
-	return true;
+	co_return true;
 }
 
 Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
diff --git a/bindings/flow/DirectoryPartition.h b/bindings/flow/DirectoryPartition.h
index df4073290f4..f046ef754f3 100644
--- a/bindings/flow/DirectoryPartition.h
+++ b/bindings/flow/DirectoryPartition.h
@@ -32,12 +32,12 @@ class DirectoryPartition : public DirectorySubspace {
 
 public:
 	DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
-	  : DirectorySubspace(path,
-	                      prefix,
-	                      Reference<DirectoryLayer>(new DirectoryLayer(
-	                          Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
-	                          Subspace(prefix))),
-	                      DirectoryLayer::PARTITION_LAYER),
+	  : DirectorySubspace(
+	        path,
+	        prefix,
+	        makeReference<DirectoryLayer>(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
+	                                      Subspace(prefix)),
+	        DirectoryLayer::PARTITION_LAYER),
 	    parentDirectoryLayer(parentDirectoryLayer) {
 		this->directoryLayer->path = path;
 	}
diff --git a/bindings/flow/HighContentionAllocator.actor.cpp b/bindings/flow/HighContentionAllocator.cpp
similarity index 73%
rename from bindings/flow/HighContentionAllocator.actor.cpp
rename to bindings/flow/HighContentionAllocator.cpp
index eebbada0ae2..92f63a48c65 100644
--- a/bindings/flow/HighContentionAllocator.actor.cpp
+++ b/bindings/flow/HighContentionAllocator.cpp
@@ -1,5 +1,5 @@
 /*
- * HighContentionAllocator.actor.cpp
+ * HighContentionAllocator.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,19 +21,19 @@
 #include "HighContentionAllocator.h"
 
 namespace FDB {
-ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent) {
-	state int64_t start = 0;
-	state int64_t window = 0;
+Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent) {
+	int64_t start = 0;
+	int64_t window = 0;
 
-	loop {
-		FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
+	while (true) {
+		FDBStandalone<RangeResultRef> range = co_await tr->getRange(counters.range(), 1, true, true);
 
-		if (range.size() > 0) {
+		if (!range.empty()) {
 			start = counters.unpack(range[0].key).getInt(0);
 		}
 
-		state bool windowAdvanced = false;
-		loop {
+		bool windowAdvanced = false;
+		while (true) {
 			// if thread safety is needed, this should be locked {
 			if (windowAdvanced) {
 				tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
@@ -46,7 +46,7 @@ ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspac
 			Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
 			// }
 
-			Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
+			Optional<FDBStandalone<ValueRef>> countValue = co_await countFuture;
 
 			int64_t count = 0;
 			if (countValue.present()) {
@@ -65,19 +65,19 @@ ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspac
 			windowAdvanced = true;
 		}
 
-		loop {
-			state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
+		while (true) {
+			int64_t candidate = deterministicRandom()->randomInt(start, start + window);
 
 			// if thread safety is needed, this should be locked {
-			state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
-			state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
+			Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
+			Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
 			tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 			tr->set(recent.get(candidate).key(), ValueRef());
 			// }
 
-			wait(success(latestCounter) && success(candidateValue));
+			co_await (success(latestCounter) && success(candidateValue));
 			int64_t currentWindowStart = 0;
-			if (latestCounter.get().size() > 0) {
+			if (!latestCounter.get().empty()) {
 				currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
 			}
 
@@ -87,7 +87,7 @@ ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspac
 
 			if (!candidateValue.get().present()) {
 				tr->addWriteConflictKey(recent.get(candidate).key());
-				return Tuple().append(candidate).pack();
+				co_return Tuple().append(candidate).pack();
 			}
 		}
 	}
diff --git a/bindings/flow/Node.actor.cpp b/bindings/flow/Node.cpp
similarity index 89%
rename from bindings/flow/Node.actor.cpp
rename to bindings/flow/Node.cpp
index 0be50d25532..55e13bd51bc 100644
--- a/bindings/flow/Node.actor.cpp
+++ b/bindings/flow/Node.cpp
@@ -1,5 +1,5 @@
 /*
- * Node.actor.cpp
+ * Node.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -31,18 +31,18 @@ bool DirectoryLayer::Node::exists() const {
 	return subspace.present();
 }
 
-ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node* n, Reference<Transaction> tr) {
+Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node* n, Reference<Transaction> tr) {
 	if (!n->exists()) {
 		n->loadedMetadata = true;
-		return *n;
+		co_return *n;
 	}
 
-	Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
+	Optional<FDBStandalone<ValueRef>> layer = co_await tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY));
 
 	n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
 	n->loadedMetadata = true;
 
-	return *n;
+	co_return *n;
 }
 
 // Calls to loadMetadata must keep the Node alive while the future is outstanding
diff --git a/bindings/flow/Subspace.cpp b/bindings/flow/Subspace.cpp
index 4f09408b43a..1605fd7a2fd 100644
--- a/bindings/flow/Subspace.cpp
+++ b/bindings/flow/Subspace.cpp
@@ -39,7 +39,7 @@ Subspace::Subspace(StringRef const& rawPrefix) {
 	this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
 }
 
-Subspace::~Subspace() {}
+Subspace::~Subspace() = default;
 
 Key Subspace::key() const {
 	return StringRef(rawPrefix.begin(), rawPrefix.size());
@@ -89,4 +89,4 @@ Subspace Subspace::subspace(Tuple const& tuple) const {
 Subspace Subspace::get(Tuple const& tuple) const {
 	return subspace(tuple);
 }
-} // namespace FDB
\ No newline at end of file
+} // namespace FDB
diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.cpp
similarity index 95%
rename from bindings/flow/fdb_flow.actor.cpp
rename to bindings/flow/fdb_flow.cpp
index f9df8f5266e..9599cbece25 100644
--- a/bindings/flow/fdb_flow.actor.cpp
+++ b/bindings/flow/fdb_flow.cpp
@@ -1,5 +1,5 @@
 /*
- * fdb_flow.actor.cpp
+ * fdb_flow.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,14 +21,14 @@
 #include "fdb_flow.h"
 
 #include <cstdint>
-#include <stdio.h>
 #include <cinttypes>
+#include <stdio.h>
+#include <type_traits>
 
 #include "fmt/format.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/SystemMonitor.h"
-#include "flow/TLSConfig.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TLSConfig.h"
 
 using namespace FDB;
 
@@ -37,47 +37,45 @@ THREAD_FUNC networkThread(void* fdb) {
 	THREAD_RETURN;
 }
 
-ACTOR Future<Void> _test() {
+Future<Void> _test() {
 	API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION);
 	auto db = fdb->createDatabase();
-	state Reference<Transaction> tr = db->createTransaction();
+	Reference<Transaction> tr = db->createTransaction();
 
 	// tr->setVersion(1);
 
-	Version ver = wait(tr->getReadVersion());
+	Version ver = co_await tr->getReadVersion();
 	fmt::print("{}\n", ver);
 
-	state std::vector<Future<Version>> versions;
+	std::vector<Future<Version>> versions;
 
-	state double starttime = timer_monotonic();
-	state int i;
-	// for (i = 0; i < 100000; i++) {
+	double starttime = timer_monotonic();
+	// for (int i = 0; i < 100000; i++) {
 	// 	Version v = wait( tr->getReadVersion() );
 	// }
-	for (i = 0; i < 100000; i++) {
+	for (int i = 0; i < 100000; i++) {
 		versions.push_back(tr->getReadVersion());
 	}
-	for (i = 0; i < 100000; i++) {
-		Version v = wait(versions[i]);
+	for (int i = 0; i < 100000; i++) {
+		co_await versions[i];
 	}
 	// wait( waitForAllReady( versions ) );
 	printf("Elapsed: %lf\n", timer_monotonic() - starttime);
 
 	tr->set("foo"_sr, "bar"_sr);
 
-	Optional<FDBStandalone<ValueRef>> v = wait(tr->get("foo"_sr));
+	Optional<FDBStandalone<ValueRef>> v = co_await tr->get("foo"_sr);
 	if (v.present()) {
 		printf("%s\n", v.get().toString().c_str());
 	}
 
-	FDBStandalone<RangeResultRef> r = wait(tr->getRange(KeyRangeRef("a"_sr, "z"_sr), 100));
+	FDBStandalone<RangeResultRef> r = co_await tr->getRange(KeyRangeRef("a"_sr, "z"_sr), 100);
 
 	for (auto kv : r) {
 		printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
 	}
 
 	g_network->stop();
-	return Void();
 }
 
 void fdb_flow_test() {
@@ -100,7 +98,7 @@ void fdb_flow_test() {
 namespace FDB {
 class DatabaseImpl : public Database, NonCopyable {
 public:
-	virtual ~DatabaseImpl() { fdb_database_destroy(db); }
+	~DatabaseImpl() override { fdb_database_destroy(db); }
 
 	Reference<Transaction> createTransaction() override;
 	void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
@@ -119,7 +117,7 @@ class TransactionImpl : public Transaction, private NonCopyable, public FastAllo
 	friend class DatabaseImpl;
 
 public:
-	virtual ~TransactionImpl() {
+	~TransactionImpl() override {
 		if (tr) {
 			fdb_transaction_destroy(tr);
 		}
@@ -201,17 +199,22 @@ void backToFutureCallback(FDBFuture* f, void* data) {
 // Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
 //   and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
 // You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
-ACTOR template <class T, class Function>
+template <class T, class Function>
 static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
-	state Reference<CFuture> f(new CFuture(_f));
+	Reference<CFuture> f(new CFuture(_f));
 
 	Promise<Void> ready;
 	Future<Void> onReady = ready.getFuture();
 
 	throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer()));
-	wait(onReady);
+	co_await onReady;
 
-	return convertValue(f);
+	if constexpr (std::is_same_v<T, Void>) {
+		convertValue(f);
+		co_return;
+	} else {
+		co_return convertValue(f);
+	}
 }
 
 void API::setNetworkOption(FDBNetworkOption option, Optional<StringRef> value) {
diff --git a/bindings/flow/tester/CMakeLists.txt b/bindings/flow/tester/CMakeLists.txt
index dd7f8988de4..c1e67628fdf 100644
--- a/bindings/flow/tester/CMakeLists.txt
+++ b/bindings/flow/tester/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(TEST_SRCS
-  DirectoryTester.actor.cpp
-  Tester.actor.cpp
-  Tester.actor.h)
+  DirectoryTester.cpp
+  Tester.cpp
+  Tester.h)
 add_flow_target(EXECUTABLE NAME fdb_flow_tester SRCS ${TEST_SRCS})
 target_link_libraries(fdb_flow_tester fdb_flow)
diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.cpp
similarity index 64%
rename from bindings/flow/tester/DirectoryTester.actor.cpp
rename to bindings/flow/tester/DirectoryTester.cpp
index a915b6b4266..e2c065c3592 100644
--- a/bindings/flow/tester/DirectoryTester.actor.cpp
+++ b/bindings/flow/tester/DirectoryTester.cpp
@@ -1,5 +1,5 @@
 /*
- * DirectoryTester.actor.cpp
+ * DirectoryTester.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,40 +18,38 @@
  * limitations under the License.
  */
 
-#include "Tester.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "Tester.h"
 
 using namespace FDB;
 
-ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) {
-	state std::vector<Tuple> tuples;
+Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) {
+	std::vector<Tuple> tuples;
 
 	while (tuples.size() < count) {
-		Standalone<StringRef> sizeStr = wait(data->stack.pop()[0].value);
+		Standalone<StringRef> sizeStr = co_await data->stack.pop()[0].value;
 		int size = Tuple::unpack(sizeStr).getInt(0);
 
-		state std::vector<StackItem> tupleItems = data->stack.pop(size);
-		state Tuple tuple;
+		std::vector<StackItem> tupleItems = data->stack.pop(size);
+		Tuple tuple;
 
-		state int index;
-		for (index = 0; index < tupleItems.size(); ++index) {
-			Standalone<StringRef> itemStr = wait(tupleItems[index].value);
+		for (int index = 0; index < tupleItems.size(); ++index) {
+			Standalone<StringRef> itemStr = co_await tupleItems[index].value;
 			tuple.append(Tuple::unpack(itemStr));
 		}
 
 		tuples.push_back(tuple);
 	}
 
-	return tuples;
+	co_return tuples;
 }
 
-ACTOR Future<Tuple> popTuple(Reference<FlowTesterData> data) {
-	std::vector<Tuple> tuples = wait(popTuples(data));
-	return tuples[0];
+Future<Tuple> popTuple(Reference<FlowTesterData> data) {
+	std::vector<Tuple> tuples = co_await popTuples(data);
+	co_return tuples[0];
 }
 
-ACTOR Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> data, int count = 1) {
-	std::vector<Tuple> tuples = wait(popTuples(data, count));
+Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> data, int count = 1) {
+	std::vector<Tuple> tuples = co_await popTuples(data, count);
 
 	std::vector<IDirectory::Path> paths;
 	for (auto& tuple : tuples) {
@@ -63,12 +61,12 @@ ACTOR Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> d
 		paths.push_back(path);
 	}
 
-	return paths;
+	co_return paths;
 }
 
-ACTOR Future<IDirectory::Path> popPath(Reference<FlowTesterData> data) {
-	std::vector<IDirectory::Path> paths = wait(popPaths(data));
-	return paths[0];
+Future<IDirectory::Path> popPath(Reference<FlowTesterData> data) {
+	std::vector<IDirectory::Path> paths = co_await popPaths(data);
+	co_return paths[0];
 }
 
 std::string pathToString(IDirectory::Path const& path) {
@@ -86,7 +84,7 @@ std::string pathToString(IDirectory::Path const& path) {
 
 IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) {
 	IDirectory::Path outPath(path1.begin(), path1.end());
-	for (auto p : path2) {
+	for (const auto& p : path2) {
 		outPath.push_back(p);
 	}
 
@@ -104,14 +102,13 @@ void logOp(std::string message, bool force = false) {
 struct DirectoryCreateSubspaceFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state Tuple path = wait(popTuple(data));
-		Tuple rawPrefix = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple path = co_await popTuple(data);
+		Tuple rawPrefix = co_await data->stack.waitAndPop();
 
 		logOp(format(
 		    "Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
 		data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
-		return Void();
 	}
 };
 const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE";
@@ -121,8 +118,8 @@ REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc);
 struct DirectoryCreateLayerFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		std::vector<Tuple> args = wait(data->stack.waitAndPop(3));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<Tuple> args = co_await data->stack.waitAndPop(3);
 
 		int index1 = args[0].getInt(0);
 		int index2 = args[1].getInt(0);
@@ -141,11 +138,9 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
 			             index2,
 			             nodeSubspace->key().printable().c_str(),
 			             allowManualPrefixes));
-			data->directoryData.push(
-			    Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
+			data->directoryData.push(Reference<IDirectory>(
+			    makeReference<DirectoryLayer>(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
 		}
-
-		return Void();
 	}
 };
 const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER";
@@ -155,8 +150,8 @@ REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc);
 struct DirectoryChangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple index = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple index = co_await data->stack.waitAndPop();
 		data->directoryData.directoryListIndex = index.getInt(0);
 		ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size());
 
@@ -173,8 +168,6 @@ struct DirectoryChangeFunc : InstructionFunc {
 			                             : d.subspace.get()->key().printable().c_str());
 			fflush(stdout);
 		}
-
-		return Void();
 	}
 };
 const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE";
@@ -184,11 +177,9 @@ REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc);
 struct DirectorySetErrorIndexFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple index = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple index = co_await data->stack.waitAndPop();
 		data->directoryData.directoryErrorIndex = index.getInt(0);
-
-		return Void();
 	}
 };
 const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX";
@@ -198,9 +189,9 @@ REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc);
 struct DirectoryCreateOrOpenFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state IDirectory::Path path = wait(popPath(data));
-		Tuple layerTuple = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		IDirectory::Path path = co_await popPath(data);
+		Tuple layerTuple = co_await data->stack.waitAndPop();
 		Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
 
 		Reference<IDirectory> directory = data->directoryData.directory();
@@ -208,12 +199,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
 		             pathToString(combinePaths(directory->getPath(), path)).c_str(),
 		             layer.printable().c_str()));
 
-		Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
-		    instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
+		Reference<DirectorySubspace> dirSubspace =
+		    co_await executeMutation(instruction, [instruction, directory, path, layer]() {
+			    return directory->createOrOpen(instruction->tr, path, layer);
+		    });
 
 		data->directoryData.push(dirSubspace);
-
-		return Void();
 	}
 };
 const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN";
@@ -223,9 +214,9 @@ REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc);
 struct DirectoryCreateFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state IDirectory::Path path = wait(popPath(data));
-		std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		IDirectory::Path path = co_await popPath(data);
+		std::vector<Tuple> args = co_await data->stack.waitAndPop(2);
 		Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
 		Optional<Standalone<StringRef>> prefix =
 		    args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
@@ -237,13 +228,11 @@ struct DirectoryCreateFunc : InstructionFunc {
 		             prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
 
 		Reference<DirectorySubspace> dirSubspace =
-		    wait(executeMutation(instruction, [this, directory, layer, prefix]() {
+		    co_await executeMutation(instruction, [instruction, directory, path, layer, prefix]() {
 			    return directory->create(instruction->tr, path, layer, prefix);
-		    }));
+		    });
 
 		data->directoryData.push(dirSubspace);
-
-		return Void();
 	}
 };
 const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE";
@@ -253,19 +242,17 @@ REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc);
 struct DirectoryOpenFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state IDirectory::Path path = wait(popPath(data));
-		Tuple layerTuple = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		IDirectory::Path path = co_await popPath(data);
+		Tuple layerTuple = co_await data->stack.waitAndPop();
 		Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
 
 		Reference<IDirectory> directory = data->directoryData.directory();
 		logOp(format("open %s: layer=%s",
 		             pathToString(combinePaths(directory->getPath(), path)).c_str(),
 		             layer.printable().c_str()));
-		Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
+		Reference<DirectorySubspace> dirSubspace = co_await directory->open(instruction->tr, path, layer);
 		data->directoryData.push(dirSubspace);
-
-		return Void();
 	}
 };
 const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN";
@@ -275,20 +262,20 @@ REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc);
 struct DirectoryMoveFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<IDirectory::Path> paths = co_await popPaths(data, 2);
 
 		Reference<IDirectory> directory = data->directoryData.directory();
 		logOp(format("move %s to %s",
 		             pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
 		             pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
 
-		Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
-		    instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
+		Reference<DirectorySubspace> dirSubspace =
+		    co_await executeMutation(instruction, [instruction, directory, paths]() {
+			    return directory->move(instruction->tr, paths[0], paths[1]);
+		    });
 
 		data->directoryData.push(dirSubspace);
-
-		return Void();
 	}
 };
 const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE";
@@ -298,18 +285,16 @@ REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc);
 struct DirectoryMoveToFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		IDirectory::Path path = wait(popPath(data));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		IDirectory::Path path = co_await popPath(data);
 
 		Reference<IDirectory> directory = data->directoryData.directory();
 		logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
 
-		Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
-		    instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
+		Reference<DirectorySubspace> dirSubspace = co_await executeMutation(
+		    instruction, [instruction, directory, path]() { return directory->moveTo(instruction->tr, path); });
 
 		data->directoryData.push(dirSubspace);
-
-		return Void();
 	}
 };
 const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO";
@@ -319,21 +304,21 @@ REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc);
 struct DirectoryRemoveFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple count = wait(data->stack.waitAndPop());
-		state Reference<IDirectory> directory = data->directoryData.directory();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple count = co_await data->stack.waitAndPop();
+		Reference<IDirectory> directory = data->directoryData.directory();
 		if (count.getInt(0) == 0) {
 			logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
 
-			wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
+			co_await executeMutation(instruction,
+			                         [instruction, directory]() { return directory->remove(instruction->tr); });
 		} else {
-			IDirectory::Path path = wait(popPath(data));
+			IDirectory::Path path = co_await popPath(data);
 			logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
 
-			wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
+			co_await executeMutation(
+			    instruction, [instruction, directory, path]() { return directory->remove(instruction->tr, path); });
 		}
-
-		return Void();
 	}
 };
 const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE";
@@ -343,23 +328,22 @@ REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc);
 struct DirectoryRemoveIfExistsFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple count = wait(data->stack.waitAndPop());
-		state Reference<IDirectory> directory = data->directoryData.directory();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple count = co_await data->stack.waitAndPop();
+		Reference<IDirectory> directory = data->directoryData.directory();
 		if (count.getInt(0) == 0) {
 			logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
 
-			wait(
-			    success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
+			co_await executeMutation(instruction,
+			                         [instruction, directory]() { return directory->removeIfExists(instruction->tr); });
 		} else {
-			IDirectory::Path path = wait(popPath(data));
+			IDirectory::Path path = co_await popPath(data);
 			logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
 
-			wait(success(executeMutation(instruction,
-			                             [this, path]() { return directory->removeIfExists(instruction->tr, path); })));
+			co_await executeMutation(instruction, [instruction, directory, path]() {
+				return directory->removeIfExists(instruction->tr, path);
+			});
 		}
-
-		return Void();
 	}
 };
 const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS";
@@ -369,18 +353,18 @@ REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc);
 struct DirectoryListFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple count = wait(data->stack.waitAndPop());
-		state Reference<IDirectory> directory = data->directoryData.directory();
-		state Standalone<VectorRef<StringRef>> subdirs;
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple count = co_await data->stack.waitAndPop();
+		Reference<IDirectory> directory = data->directoryData.directory();
+		Standalone<VectorRef<StringRef>> subdirs;
 		if (count.getInt(0) == 0) {
 			logOp(format("list %s", pathToString(directory->getPath()).c_str()));
-			Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
+			Standalone<VectorRef<StringRef>> _subdirs = co_await directory->list(instruction->tr);
 			subdirs = _subdirs;
 		} else {
-			IDirectory::Path path = wait(popPath(data));
+			IDirectory::Path path = co_await popPath(data);
 			logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
-			Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
+			Standalone<VectorRef<StringRef>> _subdirs = co_await directory->list(instruction->tr, path);
 			subdirs = _subdirs;
 		}
 
@@ -390,7 +374,6 @@ struct DirectoryListFunc : InstructionFunc {
 		}
 
 		data->stack.pushTuple(subdirTuple.pack());
-		return Void();
 	}
 };
 const char* DirectoryListFunc::name = "DIRECTORY_LIST";
@@ -400,23 +383,22 @@ REGISTER_INSTRUCTION_FUNC(DirectoryListFunc);
 struct DirectoryExistsFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple count = wait(data->stack.waitAndPop());
-		state Reference<IDirectory> directory = data->directoryData.directory();
-		state bool result;
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple count = co_await data->stack.waitAndPop();
+		Reference<IDirectory> directory = data->directoryData.directory();
+		bool result{ false };
 		if (count.getInt(0) == 0) {
-			bool _result = wait(directory->exists(instruction->tr));
+			bool _result = co_await directory->exists(instruction->tr);
 			result = _result;
 			logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
 		} else {
-			state IDirectory::Path path = wait(popPath(data));
-			bool _result = wait(directory->exists(instruction->tr, path));
+			IDirectory::Path path = co_await popPath(data);
+			bool _result = co_await directory->exists(instruction->tr, path);
 			result = _result;
 			logOp(format("exists %s: %d", pathToString(combinePaths(directory->getPath(), path)).c_str(), result));
 		}
 
 		data->stack.push(Tuple().append(result ? 1 : 0).pack());
-		return Void();
 	}
 };
 const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS";
@@ -426,11 +408,9 @@ REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc);
 struct DirectoryPackKeyFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple tuple = wait(popTuple(data));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple tuple = co_await popTuple(data);
 		data->stack.pushTuple(data->directoryData.subspace()->pack(tuple));
-
-		return Void();
 	}
 };
 const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY";
@@ -440,8 +420,8 @@ REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc);
 struct DirectoryUnpackKeyFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple key = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple key = co_await data->stack.waitAndPop();
 		Subspace* subspace = data->directoryData.subspace();
 		logOp(format("Unpack %s in subspace with prefix %s",
 		             key.getString(0).printable().c_str(),
@@ -450,8 +430,6 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
 		for (int i = 0; i < tuple.size(); ++i) {
 			data->stack.push(tuple.subTuple(i, i + 1).pack());
 		}
-
-		return Void();
 	}
 };
 const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY";
@@ -461,13 +439,11 @@ REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc);
 struct DirectoryRangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple tuple = wait(popTuple(data));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple tuple = co_await popTuple(data);
 		KeyRange range = data->directoryData.subspace()->range(tuple);
 		data->stack.pushTuple(range.begin);
 		data->stack.pushTuple(range.end);
-
-		return Void();
 	}
 };
 const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE";
@@ -477,12 +453,10 @@ REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc);
 struct DirectoryContainsFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple key = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple key = co_await data->stack.waitAndPop();
 		bool result = data->directoryData.subspace()->contains(key.getString(0));
 		data->stack.push(Tuple().append(result ? 1 : 0).pack());
-
-		return Void();
 	}
 };
 const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS";
@@ -492,14 +466,12 @@ REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc);
 struct DirectoryOpenSubspaceFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple tuple = wait(popTuple(data));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple tuple = co_await popTuple(data);
 		Subspace* subspace = data->directoryData.subspace();
 		logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str()));
 		Subspace* child = new Subspace(subspace->subspace(tuple));
 		data->directoryData.push(child);
-
-		return Void();
 	}
 };
 const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE";
@@ -509,13 +481,11 @@ REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc);
 struct DirectoryLogSubspaceFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple prefix = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple prefix = co_await data->stack.waitAndPop();
 		Tuple tuple;
 		tuple.append(data->directoryData.directoryListIndex);
 		instruction->tr->set(Subspace(tuple, prefix.getString(0)).key(), data->directoryData.subspace()->key());
-
-		return Void();
 	}
 };
 const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE";
@@ -525,14 +495,14 @@ REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc);
 struct DirectoryLogDirectoryFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state Reference<IDirectory> directory = data->directoryData.directory();
-		state Tuple prefix = wait(data->stack.waitAndPop());
-		state bool exists = wait(directory->exists(instruction->tr));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Reference<IDirectory> directory = data->directoryData.directory();
+		Tuple prefix = co_await data->stack.waitAndPop();
+		bool exists = co_await directory->exists(instruction->tr);
 
-		state Tuple childrenTuple;
+		Tuple childrenTuple;
 		if (exists) {
-			Standalone<VectorRef<StringRef>> children = wait(directory->list(instruction->tr));
+			Standalone<VectorRef<StringRef>> children = co_await directory->list(instruction->tr);
 			for (auto& c : children) {
 				childrenTuple.append(c, true);
 			}
@@ -549,8 +519,6 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
 		instruction->tr->set(logSubspace.pack("layer"_sr, true), Tuple().append(directory->getLayer()).pack());
 		instruction->tr->set(logSubspace.pack("exists"_sr, true), Tuple().append(exists ? 1 : 0).pack());
 		instruction->tr->set(logSubspace.pack("children"_sr, true), childrenTuple.pack());
-
-		return Void();
 	}
 };
 const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY";
@@ -560,12 +528,11 @@ REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc);
 struct DirectoryStripPrefixFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Tuple str = wait(data->stack.waitAndPop());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Tuple str = co_await data->stack.waitAndPop();
 		Subspace* subspace = data->directoryData.subspace();
 		ASSERT(str.getString(0).startsWith(subspace->key()));
 		data->stack.pushTuple(str.getString(0).substr(subspace->key().size()));
-		return Void();
 	}
 };
 const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.cpp
similarity index 66%
rename from bindings/flow/tester/Tester.actor.cpp
rename to bindings/flow/tester/Tester.cpp
index f1aaf401cf6..331f4f3fc09 100644
--- a/bindings/flow/tester/Tester.actor.cpp
+++ b/bindings/flow/tester/Tester.cpp
@@ -1,5 +1,5 @@
 /*
- * Tester.actor.cpp
+ * Tester.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "tester/Tester.actor.h"
+#include "tester/Tester.h"
 #include <cinttypes>
 #ifdef __linux__
 #include <string.h>
@@ -28,8 +28,7 @@
 #include "bindings/flow/FDBLoanerTypes.h"
 #include "fdbrpc/fdbrpc.h"
 #include "flow/DeterministicRandom.h"
-#include "flow/TLSConfig.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TLSConfig.h"
 
 // Otherwise we have to type setupNetwork(), FDB::open(), etc.
 using namespace FDB;
@@ -62,26 +61,25 @@ bool hasEnding(std::string const& fullString, std::string const& ending) {
 	}
 }
 
-ACTOR Future<std::vector<Tuple>> waitAndPop(FlowTesterStack* self, int count) {
-	state std::vector<Tuple> tuples;
-	state std::vector<StackItem> items = self->pop(count);
+Future<std::vector<Tuple>> waitAndPop(FlowTesterStack* self, int count) {
+	std::vector<Tuple> tuples;
+	std::vector<StackItem> items = self->pop(count);
 
-	state int index;
-	for (index = 0; index < items.size(); ++index) {
-		Standalone<StringRef> itemStr = wait(items[index].value);
+	for (int index = 0; index < items.size(); ++index) {
+		Standalone<StringRef> itemStr = co_await items[index].value;
 		tuples.push_back(Tuple::unpack(itemStr));
 	}
 
-	return tuples;
+	co_return tuples;
 }
 
 Future<std::vector<Tuple>> FlowTesterStack::waitAndPop(int count) {
 	return ::waitAndPop(this, count);
 }
 
-ACTOR Future<Tuple> waitAndPop(FlowTesterStack* self) {
-	std::vector<Tuple> tuples = wait(waitAndPop(self, 1));
-	return tuples[0];
+Future<Tuple> waitAndPop(FlowTesterStack* self) {
+	std::vector<Tuple> tuples = co_await waitAndPop(self, 1);
+	co_return tuples[0];
 }
 
 Future<Tuple> FlowTesterStack::waitAndPop() {
@@ -131,34 +129,34 @@ std::string tupleToString(Tuple const& tuple) {
 	return str;
 }
 
-ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
-                                                  KeySelectorRef begin,
-                                                  KeySelectorRef end,
-                                                  int limits = 0,
-                                                  bool snapshot = false,
-                                                  bool reverse = false,
-                                                  FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
-	state KeySelector ks_begin(begin);
-	state KeySelector ks_end(end);
-	state Standalone<RangeResultRef> results;
-	state int iteration = 1;
-	loop {
+Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
+                                            KeySelectorRef begin,
+                                            KeySelectorRef end,
+                                            int limits = 0,
+                                            bool snapshot = false,
+                                            bool reverse = false,
+                                            FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
+	KeySelector ks_begin(begin);
+	KeySelector ks_end(end);
+	Standalone<RangeResultRef> results;
+	int iteration = 1;
+	while (true) {
 		// printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(),
 		// limits);
-		state FDBStandalone<RangeResultRef> r;
+		FDBStandalone<RangeResultRef> r;
 		if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) {
 			int effective_iteration = std::min(iteration, MAX_ITERATION);
 			int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1];
-			FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(ks_begin,
-			                                                        ks_end,
-			                                                        GetRangeLimits(limits, bytes_limit),
-			                                                        snapshot,
-			                                                        reverse,
-			                                                        (FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
+			FDBStandalone<RangeResultRef> rTemp = co_await tr->getRange(ks_begin,
+			                                                            ks_end,
+			                                                            GetRangeLimits(limits, bytes_limit),
+			                                                            snapshot,
+			                                                            reverse,
+			                                                            (FDBStreamingMode)FDB_STREAMING_MODE_EXACT);
 			r = rTemp;
 		} else {
 			FDBStandalone<RangeResultRef> rTemp =
-			    wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode));
+			    co_await tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode);
 			r = rTemp;
 		}
 		iteration += 1;
@@ -177,7 +175,7 @@ ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
 		ASSERT(limits == 0 || limits >= r.size());
 
 		if (!r.more || (limits > 0 && limits == r.size())) {
-			return results;
+			co_return results;
 		}
 
 		if (limits > 0) {
@@ -186,32 +184,32 @@ ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
 	}
 }
 
-ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
-                                                  KeyRange keys,
-                                                  int limits = 0,
-                                                  bool snapshot = false,
-                                                  bool reverse = false,
-                                                  FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
-	state Key begin(keys.begin);
-	state Key end(keys.end);
-	state Standalone<RangeResultRef> results;
-	state int iteration = 1;
-	loop {
+Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
+                                            KeyRange keys,
+                                            int limits = 0,
+                                            bool snapshot = false,
+                                            bool reverse = false,
+                                            FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
+	Key begin(keys.begin);
+	Key end(keys.end);
+	Standalone<RangeResultRef> results;
+	int iteration = 1;
+	while (true) {
 		// printf("=====DB: begin:%s, limits:%d\n", printable(begin).c_str(), limits);
 		KeyRange keyRange(KeyRangeRef(begin, end > begin ? end : begin));
-		state FDBStandalone<RangeResultRef> r;
+		FDBStandalone<RangeResultRef> r;
 		if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) {
 			int effective_iteration = std::min(iteration, MAX_ITERATION);
 			int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1];
-			FDBStandalone<RangeResultRef> rTemp = wait(tr->getRange(keyRange,
-			                                                        GetRangeLimits(limits, bytes_limit),
-			                                                        snapshot,
-			                                                        reverse,
-			                                                        (FDBStreamingMode)FDB_STREAMING_MODE_EXACT));
+			FDBStandalone<RangeResultRef> rTemp = co_await tr->getRange(keyRange,
+			                                                            GetRangeLimits(limits, bytes_limit),
+			                                                            snapshot,
+			                                                            reverse,
+			                                                            (FDBStreamingMode)FDB_STREAMING_MODE_EXACT);
 			r = rTemp;
 		} else {
 			FDBStandalone<RangeResultRef> rTemp =
-			    wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode));
+			    co_await tr->getRange(keyRange, limits, snapshot, reverse, streamingMode);
 			r = rTemp;
 		}
 		iteration += 1;
@@ -230,7 +228,7 @@ ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
 		ASSERT(limits == 0 || limits >= r.size());
 
 		if (!r.more || (limits > 0 && limits == r.size())) {
-			return results;
+			co_return results;
 		}
 
 		if (limits > 0) {
@@ -253,54 +251,50 @@ ACTOR Future<Standalone<RangeResultRef>> getRange(Reference<Transaction> tr,
 //	return Void();
 //}
 
-ACTOR Future<Void> stackSub(FlowTesterStack* stack) {
+Future<Void> stackSub(FlowTesterStack* stack) {
 	if (stack->data.size() < 2)
-		return Void();
+		co_return;
 
 	StackItem a = stack->data.back();
 	stack->data.pop_back();
-	state Standalone<StringRef> sa = wait(a.value);
+	Standalone<StringRef> sa = co_await a.value;
 
 	StackItem b = stack->data.back();
 	stack->data.pop_back();
-	Standalone<StringRef> sb = wait(b.value);
+	Standalone<StringRef> sb = co_await b.value;
 
 	int64_t c = Tuple::unpack(sa).getInt(0) - Tuple::unpack(sb).getInt(0);
 	Tuple f;
 	f.append(c);
 	stack->push(f.pack());
-
-	return Void();
 }
 
-ACTOR Future<Void> stackConcat(FlowTesterStack* stack) {
+Future<Void> stackConcat(FlowTesterStack* stack) {
 	if (stack->data.size() < 2)
-		return Void();
+		co_return;
 
 	StackItem a = stack->data.back();
 	stack->data.pop_back();
-	state Standalone<StringRef> sa = wait(a.value);
-	state Tuple ta = Tuple::unpack(sa);
+	Standalone<StringRef> sa = co_await a.value;
+	Tuple ta = Tuple::unpack(sa);
 
 	StackItem b = stack->data.back();
 	stack->data.pop_back();
-	Standalone<StringRef> sb = wait(b.value);
-	state Tuple tb = Tuple::unpack(sb);
+	Standalone<StringRef> sb = co_await b.value;
+	Tuple tb = Tuple::unpack(sb);
 
 	ASSERT(ta.getType(0) == tb.getType(0));
 	stack->pushTuple(tb.getString(0).withPrefix(ta.getString(0)), ta.getType(0) == Tuple::ElementType::UTF8);
-
-	return Void();
 }
 
-ACTOR Future<Void> stackSwap(FlowTesterStack* stack) {
+Future<Void> stackSwap(FlowTesterStack* stack) {
 	if (stack->data.size() < 3)
-		return Void();
+		co_return;
 
 	StackItem pop = stack->data.back();
 	stack->data.pop_back();
 
-	Standalone<StringRef> sv = wait(pop.value);
+	Standalone<StringRef> sv = co_await pop.value;
 	int64_t idx = stack->data.size() - 1;
 	int64_t idx1 = idx - Tuple::unpack(sv).getInt(0);
 	if (idx1 < idx) {
@@ -309,18 +303,15 @@ ACTOR Future<Void> stackSwap(FlowTesterStack* stack) {
 		stack->data[idx] = stack->data[idx1];
 		stack->data[idx1] = item;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> printFlowTesterStack(FlowTesterStack* stack) {
+Future<Void> printFlowTesterStack(FlowTesterStack* stack) {
 	// printf("====================stack item count:%ld\n", stack->data.size());
-	state int idx;
-	for (idx = stack->data.size() - 1; idx >= 0; --idx) {
-		Standalone<StringRef> value = wait(stack->data[idx].value);
+	for (int idx = stack->data.size() - 1; idx >= 0; --idx) {
+		Standalone<StringRef> value = co_await stack->data[idx].value;
 		// printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index,
 		// value.printable().c_str());
 	}
-	return Void();
 }
 
 //
@@ -367,9 +358,8 @@ REGISTER_INSTRUCTION_FUNC(EmptyStackFunc);
 struct SwapFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		wait(stackSwap(&(data->stack)));
-		return Void();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		co_await stackSwap(&(data->stack));
 	}
 };
 const char* SwapFunc::name = "SWAP";
@@ -378,12 +368,11 @@ REGISTER_INSTRUCTION_FUNC(SwapFunc);
 struct PopFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
-		for (StackItem item : items) {
-			wait(success(item.value));
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
+		for (const StackItem& item : items) {
+			co_await item.value;
 		}
-		return Void();
 	}
 };
 const char* PopFunc::name = "POP";
@@ -392,9 +381,8 @@ REGISTER_INSTRUCTION_FUNC(PopFunc);
 struct SubFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		wait(stackSub(&(data->stack)));
-		return Void();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		co_await stackSub(&(data->stack));
 	}
 };
 const char* SubFunc::name = "SUB";
@@ -403,9 +391,8 @@ REGISTER_INSTRUCTION_FUNC(SubFunc);
 struct ConcatFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		wait(stackConcat(&(data->stack)));
-		return Void();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		co_await stackConcat(&(data->stack));
 	}
 };
 const char* ConcatFunc::name = "CONCAT";
@@ -414,50 +401,50 @@ REGISTER_INSTRUCTION_FUNC(ConcatFunc);
 struct LogStackFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> logStack(Reference<FlowTesterData> data,
-	                                   std::map<int, StackItem> entries,
-	                                   Standalone<StringRef> prefix) {
-		loop {
-			state Reference<Transaction> tr = data->db->createTransaction();
+	static Future<Void> logStack(Reference<FlowTesterData> data,
+	                             std::map<int, StackItem> entries,
+	                             Standalone<StringRef> prefix) {
+		while (true) {
+			Reference<Transaction> tr = data->db->createTransaction();
+			Error err;
 			try {
-				for (auto it : entries) {
+				for (const auto& it : entries) {
 					Tuple tk;
 					tk.append(it.first);
 					tk.append((int64_t)it.second.index);
-					state Standalone<StringRef> pk = tk.pack().withPrefix(prefix);
-					Standalone<StringRef> pv = wait(it.second.value);
+					Standalone<StringRef> pk = tk.pack().withPrefix(prefix);
+					Standalone<StringRef> pv = co_await it.second.value;
 					tr->set(pk, pv.substr(0, std::min(pv.size(), 40000)));
 				}
 
-				wait(tr->commit());
-				return Void();
+				co_await tr->commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.empty())
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
 
-		state std::map<int, StackItem> entries;
-		while (data->stack.data.size() > 0) {
-			state std::vector<StackItem> it = data->stack.pop();
+		std::map<int, StackItem> entries;
+		while (!data->stack.data.empty()) {
+			std::vector<StackItem> it = data->stack.pop();
 			ASSERT(it.size() == 1);
 			entries[data->stack.data.size()] = it.front();
 			if (entries.size() == 100) {
-				wait(logStack(data, entries, prefix));
+				co_await logStack(data, entries, prefix);
 				entries.clear();
 			}
 		}
-		wait(logStack(data, entries, prefix));
-
-		return Void();
+		co_await logStack(data, entries, prefix);
 	}
 };
 const char* LogStackFunc::name = "LOG_STACK";
@@ -466,12 +453,12 @@ REGISTER_INSTRUCTION_FUNC(LogStackFunc);
 //
 // FoundationDB Operations
 //
-ACTOR Future<Standalone<StringRef>> waitForVoid(Future<Void> f) {
+Future<Standalone<StringRef>> waitForVoid(Future<Void> f) {
 	try {
-		wait(f);
+		co_await f;
 		Tuple t;
 		t.append("RESULT_NOT_PRESENT"_sr);
-		return t.pack();
+		co_return t.pack();
 	} catch (Error& e) {
 		// printf("FDBError1:%d\n", e.code());
 		Tuple t;
@@ -480,16 +467,16 @@ ACTOR Future<Standalone<StringRef>> waitForVoid(Future<Void> f) {
 		// pack above as error string into another tuple
 		Tuple ret;
 		ret.append(t.pack());
-		return ret.pack();
+		co_return ret.pack();
 	}
 }
 
-ACTOR Future<Standalone<StringRef>> waitForValue(Future<FDBStandalone<KeyRef>> f) {
+Future<Standalone<StringRef>> waitForValue(Future<FDBStandalone<KeyRef>> f) {
 	try {
-		FDBStandalone<KeyRef> value = wait(f);
+		FDBStandalone<KeyRef> value = co_await f;
 		Tuple t;
 		t.append(value);
-		return t.pack();
+		co_return t.pack();
 	} catch (Error& e) {
 		// printf("FDBError2:%d\n", e.code());
 		Tuple t;
@@ -498,13 +485,13 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<FDBStandalone<KeyRef>> f
 		// pack above as error string into another tuple
 		Tuple ret;
 		ret.append(t.pack());
-		return ret.pack();
+		co_return ret.pack();
 	}
 }
 
-ACTOR Future<Standalone<StringRef>> waitForValue(Future<Optional<FDBStandalone<ValueRef>>> f) {
+Future<Standalone<StringRef>> waitForValue(Future<Optional<FDBStandalone<ValueRef>>> f) {
 	try {
-		Optional<FDBStandalone<ValueRef>> value = wait(f);
+		Optional<FDBStandalone<ValueRef>> value = co_await f;
 		Standalone<StringRef> str;
 		if (value.present())
 			str = value.get();
@@ -513,7 +500,7 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<Optional<FDBStandalone<V
 
 		Tuple t;
 		t.append(str);
-		return t.pack();
+		co_return t.pack();
 	} catch (Error& e) {
 		// printf("FDBError3:%d\n", e.code());
 		Tuple t;
@@ -522,13 +509,13 @@ ACTOR Future<Standalone<StringRef>> waitForValue(Future<Optional<FDBStandalone<V
 		// pack above as error string into another tuple
 		Tuple ret;
 		ret.append(t.pack());
-		return ret.pack();
+		co_return ret.pack();
 	}
 }
 
-ACTOR Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Standalone<StringRef> prefixFilter) {
+Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Standalone<StringRef> prefixFilter) {
 	try {
-		FDBStandalone<KeyRef> key = wait(f);
+		FDBStandalone<KeyRef> key = co_await f;
 		Tuple t;
 
 		if (key.startsWith(prefixFilter)) {
@@ -539,7 +526,7 @@ ACTOR Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Stan
 			t.append(strinc(prefixFilter));
 		}
 
-		return t.pack();
+		co_return t.pack();
 	} catch (Error& e) {
 		// printf("FDBError4:%d\n", e.code());
 		Tuple t;
@@ -548,7 +535,7 @@ ACTOR Future<Standalone<StringRef>> getKey(Future<FDBStandalone<KeyRef>> f, Stan
 		// pack above as error string into another tuple
 		Tuple ret;
 		ret.append(t.pack());
-		return ret.pack();
+		co_return ret.pack();
 	}
 }
 
@@ -566,15 +553,14 @@ REGISTER_INSTRUCTION_FUNC(NewTransactionFunc);
 struct UseTransactionFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
-		Standalone<StringRef> name = wait(items[0].value);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
+		Standalone<StringRef> name = co_await items[0].value;
 		data->trName = name;
 
-		if (trMap.count(data->trName) == 0) {
+		if (!trMap.contains(data->trName)) {
 			trMap[data->trName] = data->db->createTransaction();
 		}
-		return Void();
 	}
 };
 const char* UseTransactionFunc::name = "USE_TRANSACTION";
@@ -583,17 +569,16 @@ REGISTER_INSTRUCTION_FUNC(UseTransactionFunc);
 struct OnErrorFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.empty())
-			return Void();
+			co_return;
 
-		Standalone<StringRef> value = wait(items[0].value);
+		Standalone<StringRef> value = co_await items[0].value;
 		int err_code = Tuple::unpack(value).getInt(0);
 		// printf("OnError:%d:%d:%s\n", err_code, items[0].index, printable(value).c_str());
 
 		data->stack.push(waitForVoid(instruction->tr->onError(Error(err_code))));
-		return Void();
 	}
 };
 const char* OnErrorFunc::name = "ON_ERROR";
@@ -602,16 +587,16 @@ REGISTER_INSTRUCTION_FUNC(OnErrorFunc);
 struct SetFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(2);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(2);
 		if (items.size() != 2)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> sk = wait(items[0].value);
-		state Standalone<StringRef> key = Tuple::unpack(sk).getString(0);
+		Standalone<StringRef> sk = co_await items[0].value;
+		Standalone<StringRef> key = Tuple::unpack(sk).getString(0);
 		// if (instruction->isDatabase)
 		// printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase);
-		Standalone<StringRef> sv = wait(items[1].value);
+		Standalone<StringRef> sv = co_await items[1].value;
 		Standalone<StringRef> value = Tuple::unpack(sv).getString(0);
 		// printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str());
 
@@ -626,10 +611,8 @@ struct SetFunc : InstructionFunc {
 		if (instruction->isDatabase) {
 			data->stack.push(waitForVoid(mutation));
 		} else {
-			wait(mutation);
+			co_await mutation;
 		}
-
-		return Void();
 	}
 };
 const char* SetFunc::name = "SET";
@@ -638,18 +621,16 @@ REGISTER_INSTRUCTION_FUNC(SetFunc);
 struct GetFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> sk = wait(items[0].value);
-		state Standalone<StringRef> key = Tuple::unpack(sk).getString(0);
+		Standalone<StringRef> sk = co_await items[0].value;
+		Standalone<StringRef> key = Tuple::unpack(sk).getString(0);
 
 		Future<Optional<FDBStandalone<ValueRef>>> fk = instruction->tr->get(StringRef(key), instruction->isSnapshot);
 		data->stack.push(waitForValue(holdWhile(instruction->tr, fk)));
-
-		return Void();
 	}
 };
 const char* GetFunc::name = "GET";
@@ -658,21 +639,19 @@ REGISTER_INSTRUCTION_FUNC(GetFunc);
 struct GetEstimatedRangeSize : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(2);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(2);
 		if (items.size() != 2)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> beginKey = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> beginKey = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
 		Future<int64_t> fsize = instruction->tr->getEstimatedRangeSizeBytes(KeyRangeRef(beginKey, endKey));
-		int64_t size = wait(fsize);
+		co_await fsize;
 		data->stack.pushTuple("GOT_ESTIMATED_RANGE_SIZE"_sr);
-
-		return Void();
 	}
 };
 const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE";
@@ -681,26 +660,24 @@ REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize);
 struct GetRangeSplitPoints : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(3);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(3);
 		if (items.size() != 3)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> beginKey = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> beginKey = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
 
-		Standalone<StringRef> s3 = wait(items[2].value);
-		state int64_t chunkSize = Tuple::unpack(s3).getInt(0);
+		Standalone<StringRef> s3 = co_await items[2].value;
+		int64_t chunkSize = Tuple::unpack(s3).getInt(0);
 
 		Future<FDBStandalone<VectorRef<KeyRef>>> fsplitPoints =
 		    instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize);
-		FDBStandalone<VectorRef<KeyRef>> splitPoints = wait(fsplitPoints);
+		FDBStandalone<VectorRef<KeyRef>> splitPoints = co_await fsplitPoints;
 		data->stack.pushTuple("GOT_RANGE_SPLIT_POINTS"_sr);
-
-		return Void();
 	}
 };
 const char* GetRangeSplitPoints::name = "GET_RANGE_SPLIT_POINTS";
@@ -709,29 +686,27 @@ REGISTER_INSTRUCTION_FUNC(GetRangeSplitPoints);
 struct GetKeyFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(4);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(4);
 		if (items.size() != 4)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> key = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> key = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state int64_t or_equal = Tuple::unpack(s2).getInt(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		int64_t or_equal = Tuple::unpack(s2).getInt(0);
 
-		Standalone<StringRef> s3 = wait(items[2].value);
-		state int64_t offset = Tuple::unpack(s3).getInt(0);
+		Standalone<StringRef> s3 = co_await items[2].value;
+		int64_t offset = Tuple::unpack(s3).getInt(0);
 
-		Standalone<StringRef> s4 = wait(items[3].value);
+		Standalone<StringRef> s4 = co_await items[3].value;
 		Standalone<StringRef> prefix = Tuple::unpack(s4).getString(0);
 
 		// printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset);
 		Future<FDBStandalone<KeyRef>> fk =
 		    instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot);
 		data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix));
-
-		return Void();
 	}
 };
 const char* GetKeyFunc::name = "GET_KEY";
@@ -740,11 +715,10 @@ REGISTER_INSTRUCTION_FUNC(GetKeyFunc);
 struct GetReadVersionFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		Version v = wait(instruction->tr->getReadVersion());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		Version v = co_await instruction->tr->getReadVersion();
 		data->lastVersion = v;
 		data->stack.pushTuple("GOT_READ_VERSION"_sr);
-		return Void();
 	}
 };
 const char* GetReadVersionFunc::name = "GET_READ_VERSION";
@@ -778,11 +752,10 @@ REGISTER_INSTRUCTION_FUNC(GetCommittedVersionFunc);
 struct GetApproximateSizeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		int64_t _ = wait(instruction->tr->getApproximateSize());
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		int64_t _ = co_await instruction->tr->getApproximateSize();
 		(void)_; // disable unused variable warning
 		data->stack.pushTuple("GOT_APPROXIMATE_SIZE"_sr);
-		return Void();
 	}
 };
 const char* GetApproximateSizeFunc::name = "GET_APPROXIMATE_SIZE";
@@ -816,14 +789,13 @@ REGISTER_INSTRUCTION_FUNC(CommitFunc);
 struct WaitFutureFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> sk = wait(items[0].value);
+		Standalone<StringRef> sk = co_await items[0].value;
 		data->stack.push(StackItem(items[0].index, sk));
-		return Void();
 	}
 };
 const char* WaitFutureFunc::name = "WAIT_FUTURE";
@@ -833,12 +805,12 @@ REGISTER_INSTRUCTION_FUNC(WaitFutureFunc);
 struct ClearFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> sk = wait(items[0].value);
+		Standalone<StringRef> sk = co_await items[0].value;
 		Standalone<StringRef> key = Tuple::unpack(sk).getString(0);
 
 		Reference<InstructionData> instructionCopy = instruction;
@@ -851,10 +823,8 @@ struct ClearFunc : InstructionFunc {
 		if (instruction->isDatabase) {
 			data->stack.push(waitForVoid(mutation));
 		} else {
-			wait(mutation);
+			co_await mutation;
 		}
-
-		return Void();
 	}
 };
 const char* ClearFunc::name = "CLEAR";
@@ -887,35 +857,35 @@ REGISTER_INSTRUCTION_FUNC(CancelFunc);
 struct GetRangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(5);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(5);
 		if (items.size() != 5)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
 
-		Standalone<StringRef> s3 = wait(items[2].value);
-		state int limit = Tuple::unpack(s3).getInt(0);
+		Standalone<StringRef> s3 = co_await items[2].value;
+		int limit = Tuple::unpack(s3).getInt(0);
 
-		Standalone<StringRef> s4 = wait(items[3].value);
-		state int reverse = Tuple::unpack(s4).getInt(0);
+		Standalone<StringRef> s4 = co_await items[3].value;
+		int reverse = Tuple::unpack(s4).getInt(0);
 
-		Standalone<StringRef> s5 = wait(items[4].value);
+		Standalone<StringRef> s5 = co_await items[4].value;
 		FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0);
 
 		// printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(),
 		// printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot);
 
-		Standalone<RangeResultRef> results = wait(getRange(instruction->tr,
-		                                                   KeyRange(KeyRangeRef(begin, end > begin ? end : begin)),
-		                                                   limit,
-		                                                   instruction->isSnapshot,
-		                                                   reverse,
-		                                                   mode));
+		Standalone<RangeResultRef> results = co_await getRange(instruction->tr,
+		                                                       KeyRange(KeyRangeRef(begin, end > begin ? end : begin)),
+		                                                       limit,
+		                                                       instruction->isSnapshot,
+		                                                       reverse,
+		                                                       mode);
 		Tuple t;
 		for (auto& s : results) {
 			t.append(s.key);
@@ -926,7 +896,6 @@ struct GetRangeFunc : InstructionFunc {
 		// printf("=====Results Count:%d, size:%d\n", results.size(), str.size());
 
 		data->stack.push(Tuple().append(t.pack()).pack());
-		return Void();
 	}
 };
 const char* GetRangeFunc::name = "GET_RANGE";
@@ -935,31 +904,31 @@ REGISTER_INSTRUCTION_FUNC(GetRangeFunc);
 struct GetRangeStartsWithFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(4);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(4);
 		if (items.size() != 4)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state int limit = Tuple::unpack(s2).getInt(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		int limit = Tuple::unpack(s2).getInt(0);
 
-		Standalone<StringRef> s3 = wait(items[2].value);
-		state int reverse = Tuple::unpack(s3).getInt(0);
+		Standalone<StringRef> s3 = co_await items[2].value;
+		int reverse = Tuple::unpack(s3).getInt(0);
 
-		Standalone<StringRef> s4 = wait(items[3].value);
+		Standalone<StringRef> s4 = co_await items[3].value;
 		FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0);
 
 		// printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit,
 		// reverse, mode, isSnapshot);
-		Standalone<RangeResultRef> results = wait(getRange(instruction->tr,
-		                                                   KeyRange(KeyRangeRef(prefix, strinc(prefix))),
-		                                                   limit,
-		                                                   instruction->isSnapshot,
-		                                                   reverse,
-		                                                   mode));
+		Standalone<RangeResultRef> results = co_await getRange(instruction->tr,
+		                                                       KeyRange(KeyRangeRef(prefix, strinc(prefix))),
+		                                                       limit,
+		                                                       instruction->isSnapshot,
+		                                                       reverse,
+		                                                       mode);
 		Tuple t;
 		// printf("=====Results Count:%d\n", results.size());
 		for (auto& s : results) {
@@ -970,7 +939,6 @@ struct GetRangeStartsWithFunc : InstructionFunc {
 		}
 
 		data->stack.push(Tuple().append(t.pack()).pack());
-		return Void();
 	}
 };
 const char* GetRangeStartsWithFunc::name = "GET_RANGE_STARTS_WITH";
@@ -979,15 +947,15 @@ REGISTER_INSTRUCTION_FUNC(GetRangeStartsWithFunc);
 struct ClearRangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(2);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(2);
 		if (items.size() != 2)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
+		Standalone<StringRef> s2 = co_await items[1].value;
 		Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
 
 		Reference<InstructionData> instructionCopy = instruction;
@@ -1001,10 +969,8 @@ struct ClearRangeFunc : InstructionFunc {
 		if (instruction->isDatabase) {
 			data->stack.push(waitForVoid(mutation));
 		} else {
-			wait(mutation);
+			co_await mutation;
 		}
-
-		return Void();
 	}
 };
 const char* ClearRangeFunc::name = "CLEAR_RANGE";
@@ -1013,12 +979,12 @@ REGISTER_INSTRUCTION_FUNC(ClearRangeFunc);
 struct ClearRangeStartWithFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
 
 		Reference<InstructionData> instructionCopy = instruction;
@@ -1031,10 +997,8 @@ struct ClearRangeStartWithFunc : InstructionFunc {
 		if (instruction->isDatabase) {
 			data->stack.push(waitForVoid(mutation));
 		} else {
-			wait(mutation);
+			co_await mutation;
 		}
-
-		return Void();
 	}
 };
 const char* ClearRangeStartWithFunc::name = "CLEAR_RANGE_STARTS_WITH";
@@ -1043,40 +1007,40 @@ REGISTER_INSTRUCTION_FUNC(ClearRangeStartWithFunc);
 struct GetRangeSelectorFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(10);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(10);
 		if (items.size() != 10)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
 
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state bool begin_or_equal = Tuple::unpack(s2).getInt(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		bool begin_or_equal = Tuple::unpack(s2).getInt(0);
 
-		Standalone<StringRef> s3 = wait(items[2].value);
-		state int64_t begin_offset = Tuple::unpack(s3).getInt(0);
+		Standalone<StringRef> s3 = co_await items[2].value;
+		int64_t begin_offset = Tuple::unpack(s3).getInt(0);
 
-		Standalone<StringRef> s4 = wait(items[3].value);
-		state Standalone<StringRef> end = Tuple::unpack(s4).getString(0);
+		Standalone<StringRef> s4 = co_await items[3].value;
+		Standalone<StringRef> end = Tuple::unpack(s4).getString(0);
 
-		Standalone<StringRef> s5 = wait(items[4].value);
-		state bool end_or_equal = Tuple::unpack(s5).getInt(0);
+		Standalone<StringRef> s5 = co_await items[4].value;
+		bool end_or_equal = Tuple::unpack(s5).getInt(0);
 
-		Standalone<StringRef> s6 = wait(items[5].value);
-		state int64_t end_offset = Tuple::unpack(s6).getInt(0);
+		Standalone<StringRef> s6 = co_await items[5].value;
+		int64_t end_offset = Tuple::unpack(s6).getInt(0);
 
-		Standalone<StringRef> s7 = wait(items[6].value);
-		state int limit = Tuple::unpack(s7).getInt(0);
+		Standalone<StringRef> s7 = co_await items[6].value;
+		int limit = Tuple::unpack(s7).getInt(0);
 
-		Standalone<StringRef> s8 = wait(items[7].value);
-		state int reverse = Tuple::unpack(s8).getInt(0);
+		Standalone<StringRef> s8 = co_await items[7].value;
+		int reverse = Tuple::unpack(s8).getInt(0);
 
-		Standalone<StringRef> s9 = wait(items[8].value);
-		state FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s9).getInt(0);
+		Standalone<StringRef> s9 = co_await items[8].value;
+		FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s9).getInt(0);
 
-		Standalone<StringRef> s10 = wait(items[9].value);
-		state Optional<Standalone<StringRef>> prefix;
+		Standalone<StringRef> s10 = co_await items[9].value;
+		Optional<Standalone<StringRef>> prefix;
 		Tuple t10 = Tuple::unpack(s10);
 		if (t10.getType(0) != Tuple::ElementType::NULL_TYPE) {
 			prefix = t10.getString(0);
@@ -1092,7 +1056,7 @@ struct GetRangeSelectorFunc : InstructionFunc {
 		                                                instruction->isSnapshot,
 		                                                reverse,
 		                                                mode);
-		Standalone<RangeResultRef> results = wait(holdWhile(instruction->tr, f));
+		Standalone<RangeResultRef> results = co_await holdWhile(instruction->tr, f);
 		Tuple t;
 		// printf("=====Results Count:%d\n", results.size());
 		for (auto& s : results) {
@@ -1105,7 +1069,6 @@ struct GetRangeSelectorFunc : InstructionFunc {
 		}
 
 		data->stack.push(Tuple().append(t.pack()).pack());
-		return Void();
 	}
 };
 const char* GetRangeSelectorFunc::name = "GET_RANGE_SELECTOR";
@@ -1116,22 +1079,21 @@ REGISTER_INSTRUCTION_FUNC(GetRangeSelectorFunc);
 struct TuplePackFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state int64_t count = Tuple::unpack(s1).getInt(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		int64_t count = Tuple::unpack(s1).getInt(0);
 
-		state std::vector<StackItem> items1 = data->stack.pop(count);
+		std::vector<StackItem> items1 = data->stack.pop(count);
 		if (items1.size() != count)
-			return Void();
+			co_return;
 
-		state Tuple tuple;
-		state int i = 0;
-		for (; i < items1.size(); ++i) {
-			Standalone<StringRef> str = wait(items1[i].value);
+		Tuple tuple;
+		for (int i = 0; i < items1.size(); ++i) {
+			Standalone<StringRef> str = co_await items1[i].value;
 			Tuple itemTuple = Tuple::unpack(str);
 			if (deterministicRandom()->coinflip()) {
 				Tuple::ElementType type = itemTuple.getType(0);
@@ -1164,7 +1126,6 @@ struct TuplePackFunc : InstructionFunc {
 		}
 
 		data->stack.pushTuple(tuple.pack());
-		return Void();
 	}
 };
 const char* TuplePackFunc::name = "TUPLE_PACK";
@@ -1174,12 +1135,12 @@ REGISTER_INSTRUCTION_FUNC(TuplePackFunc);
 struct TupleUnpackFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		Tuple t = Tuple::unpack(Tuple::unpack(s1).getString(0));
 
 		for (int i = 0; i < t.size(); ++i) {
@@ -1187,7 +1148,6 @@ struct TupleUnpackFunc : InstructionFunc {
 			// printf("=====value:%s\n", printable(str).c_str());
 			data->stack.pushTuple(str);
 		}
-		return Void();
 	}
 };
 const char* TupleUnpackFunc::name = "TUPLE_UNPACK";
@@ -1197,22 +1157,21 @@ REGISTER_INSTRUCTION_FUNC(TupleUnpackFunc);
 struct TupleRangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state int64_t count = Tuple::unpack(s1).getInt(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		int64_t count = Tuple::unpack(s1).getInt(0);
 
-		state std::vector<StackItem> items1 = data->stack.pop(count);
+		std::vector<StackItem> items1 = data->stack.pop(count);
 		if (items1.size() != count)
-			return Void();
+			co_return;
 
-		state Tuple tuple;
-		state size_t i = 0;
-		for (; i < items1.size(); ++i) {
-			Standalone<StringRef> str = wait(items1[i].value);
+		Tuple tuple;
+		for (size_t i = 0; i < items1.size(); ++i) {
+			Standalone<StringRef> str = co_await items1[i].value;
 			Tuple itemTuple = Tuple::unpack(str);
 			if (deterministicRandom()->coinflip()) {
 				Tuple::ElementType type = itemTuple.getType(0);
@@ -1248,7 +1207,6 @@ struct TupleRangeFunc : InstructionFunc {
 
 		data->stack.pushTuple(range.begin);
 		data->stack.pushTuple(range.end);
-		return Void();
 	}
 };
 const char* TupleRangeFunc::name = "TUPLE_RANGE";
@@ -1258,22 +1216,21 @@ REGISTER_INSTRUCTION_FUNC(TupleRangeFunc);
 struct TupleSortFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state int64_t count = Tuple::unpack(s1).getInt(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		int64_t count = Tuple::unpack(s1).getInt(0);
 
-		state std::vector<StackItem> items1 = data->stack.pop(count);
+		std::vector<StackItem> items1 = data->stack.pop(count);
 		if (items1.size() != count)
-			return Void();
+			co_return;
 
-		state std::vector<Tuple> tuples;
-		state size_t i = 0;
-		for (; i < items1.size(); i++) {
-			Standalone<StringRef> value = wait(items1[i].value);
+		std::vector<Tuple> tuples;
+		for (size_t i = 0; i < items1.size(); i++) {
+			Standalone<StringRef> value = co_await items1[i].value;
 			tuples.push_back(Tuple::unpack(value));
 		}
 
@@ -1281,8 +1238,6 @@ struct TupleSortFunc : InstructionFunc {
 		for (Tuple const& t : tuples) {
 			data->stack.push(t.pack());
 		}
-
-		return Void();
 	}
 };
 const char* TupleSortFunc::name = "TUPLE_SORT";
@@ -1292,12 +1247,12 @@ REGISTER_INSTRUCTION_FUNC(TupleSortFunc);
 struct EncodeFloatFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		Standalone<StringRef> fBytes = Tuple::unpack(s1).getString(0);
 		ASSERT(fBytes.size() == 4);
 
@@ -1308,8 +1263,6 @@ struct EncodeFloatFunc : InstructionFunc {
 		Tuple t;
 		t.append(fVal);
 		data->stack.push(t.pack());
-
-		return Void();
 	}
 };
 const char* EncodeFloatFunc::name = "ENCODE_FLOAT";
@@ -1319,12 +1272,12 @@ REGISTER_INSTRUCTION_FUNC(EncodeFloatFunc);
 struct EncodeDoubleFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		Standalone<StringRef> dBytes = Tuple::unpack(s1).getString(0);
 		ASSERT(dBytes.size() == 8);
 
@@ -1335,8 +1288,6 @@ struct EncodeDoubleFunc : InstructionFunc {
 		Tuple t;
 		t.append(dVal);
 		data->stack.push(t.pack());
-
-		return Void();
 	}
 };
 const char* EncodeDoubleFunc::name = "ENCODE_DOUBLE";
@@ -1346,12 +1297,12 @@ REGISTER_INSTRUCTION_FUNC(EncodeDoubleFunc);
 struct DecodeFloatFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		float fVal = Tuple::unpack(s1).getFloat(0);
 		int32_t intVal = *(int32_t*)&fVal;
 		intVal = bigEndian32(intVal);
@@ -1359,8 +1310,6 @@ struct DecodeFloatFunc : InstructionFunc {
 		Tuple t;
 		t.append(StringRef((uint8_t*)&intVal, 4), false);
 		data->stack.push(t.pack());
-
-		return Void();
 	}
 };
 const char* DecodeFloatFunc::name = "DECODE_FLOAT";
@@ -1370,12 +1319,12 @@ REGISTER_INSTRUCTION_FUNC(DecodeFloatFunc);
 struct DecodeDoubleFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		double dVal = Tuple::unpack(s1).getDouble(0);
 		int64_t intVal = *(int64_t*)&dVal;
 		intVal = bigEndian64(intVal);
@@ -1383,7 +1332,6 @@ struct DecodeDoubleFunc : InstructionFunc {
 		Tuple t;
 		t.append(StringRef((uint8_t*)&intVal, 8), false);
 		data->stack.push(t.pack());
-		return Void();
 	}
 };
 const char* DecodeDoubleFunc::name = "DECODE_DOUBLE";
@@ -1394,35 +1342,34 @@ REGISTER_INSTRUCTION_FUNC(DecodeDoubleFunc);
 struct StartThreadFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
 		// printf("=========START_THREAD:%s\n", printable(prefix).c_str());
 
-		Reference<FlowTesterData> newData = Reference<FlowTesterData>(new FlowTesterData(data->api));
+		auto newData = makeReference<FlowTesterData>(data->api);
 		data->subThreads.push_back(runTest(newData, data->db, prefix));
-
-		return Void();
 	}
 };
 const char* StartThreadFunc::name = "START_THREAD";
 REGISTER_INSTRUCTION_FUNC(StartThreadFunc);
 
-ACTOR template <class Function>
+template <class Function>
 Future<decltype(std::declval<Function>()(Reference<ReadTransaction>()).getValue())> read(Reference<Database> db,
                                                                                          Function func) {
-	state Reference<ReadTransaction> tr = db->createTransaction();
-	loop {
+	Reference<ReadTransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
-			state decltype(std::declval<Function>()(Reference<ReadTransaction>()).getValue()) result = wait(func(tr));
-			return result;
+			co_return co_await func(tr);
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
@@ -1430,28 +1377,25 @@ Future<decltype(std::declval<Function>()(Reference<ReadTransaction>()).getValue(
 struct WaitEmptyFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
+		Standalone<StringRef> s1 = co_await items[0].value;
 		Standalone<StringRef> prefix = Tuple::unpack(s1).getString(0);
 		// printf("=========WAIT_EMPTY:%s\n", printable(prefix).c_str());
 
-		wait(read(data->db,
-		          [=](Reference<ReadTransaction> tr) -> Future<Void> { return checkEmptyPrefix(tr, prefix); }));
-
-		return Void();
+		co_await read(data->db,
+		              [=](Reference<ReadTransaction> tr) -> Future<Void> { return checkEmptyPrefix(tr, prefix); });
 	}
 
 private:
-	ACTOR static Future<Void> checkEmptyPrefix(Reference<ReadTransaction> tr, Standalone<StringRef> prefix) {
-		FDBStandalone<RangeResultRef> results = wait(tr->getRange(KeyRangeRef(prefix, strinc(prefix)), 1));
-		if (results.size() > 0) {
+	static Future<Void> checkEmptyPrefix(Reference<ReadTransaction> tr, Standalone<StringRef> prefix) {
+		FDBStandalone<RangeResultRef> results = co_await tr->getRange(KeyRangeRef(prefix, strinc(prefix)), 1);
+		if (!results.empty()) {
 			throw not_committed();
 		}
-		return Void();
 	}
 };
 const char* WaitEmptyFunc::name = "WAIT_EMPTY";
@@ -1475,18 +1419,17 @@ REGISTER_INSTRUCTION_FUNC(DisableWriteConflictFunc);
 struct ReadConflictKeyFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> key = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> key = Tuple::unpack(s1).getString(0);
 		// printf("=========READ_CONFLICT_KEY:%s\n", printable(key).c_str());
 		instruction->tr->addReadConflictKey(key);
 
 		data->stack.pushTuple("SET_CONFLICT_KEY"_sr);
-		return Void();
 	}
 };
 const char* ReadConflictKeyFunc::name = "READ_CONFLICT_KEY";
@@ -1496,18 +1439,17 @@ REGISTER_INSTRUCTION_FUNC(ReadConflictKeyFunc);
 struct WriteConflictKeyFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop();
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop();
 		if (items.size() != 1)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> key = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> key = Tuple::unpack(s1).getString(0);
 		// printf("=========WRITE_CONFLICT_KEY:%s\n", printable(key).c_str());
 		instruction->tr->addWriteConflictKey(key);
 
 		data->stack.pushTuple("SET_CONFLICT_KEY"_sr);
-		return Void();
 	}
 };
 const char* WriteConflictKeyFunc::name = "WRITE_CONFLICT_KEY";
@@ -1517,20 +1459,19 @@ REGISTER_INSTRUCTION_FUNC(WriteConflictKeyFunc);
 struct ReadConflictRangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(2);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(2);
 		if (items.size() != 2)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
 
 		// printf("=========READ_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str());
 		instruction->tr->addReadConflictRange(KeyRange(KeyRangeRef(begin, end)));
 		data->stack.pushTuple("SET_CONFLICT_RANGE"_sr);
-		return Void();
 	}
 };
 const char* ReadConflictRangeFunc::name = "READ_CONFLICT_RANGE";
@@ -1540,21 +1481,20 @@ REGISTER_INSTRUCTION_FUNC(ReadConflictRangeFunc);
 struct WriteConflictRangeFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(2);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(2);
 		if (items.size() != 2)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> begin = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		Standalone<StringRef> end = Tuple::unpack(s2).getString(0);
 
 		// printf("=========WRITE_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str());
 		instruction->tr->addWriteConflictRange(KeyRange(KeyRangeRef(begin, end)));
 
 		data->stack.pushTuple("SET_CONFLICT_RANGE"_sr);
-		return Void();
 	}
 };
 const char* WriteConflictRangeFunc::name = "WRITE_CONFLICT_RANGE";
@@ -1564,17 +1504,17 @@ REGISTER_INSTRUCTION_FUNC(WriteConflictRangeFunc);
 struct AtomicOPFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
-		state std::vector<StackItem> items = data->stack.pop(3);
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		std::vector<StackItem> items = data->stack.pop(3);
 		if (items.size() != 3)
-			return Void();
+			co_return;
 
-		Standalone<StringRef> s1 = wait(items[0].value);
-		state Standalone<StringRef> op = Tuple::unpack(s1).getString(0);
-		Standalone<StringRef> s2 = wait(items[1].value);
-		state Standalone<StringRef> key = Tuple::unpack(s2).getString(0);
-		Standalone<StringRef> s3 = wait(items[2].value);
-		state Standalone<StringRef> value = Tuple::unpack(s3).getString(0);
+		Standalone<StringRef> s1 = co_await items[0].value;
+		Standalone<StringRef> op = Tuple::unpack(s1).getString(0);
+		Standalone<StringRef> s2 = co_await items[1].value;
+		Standalone<StringRef> key = Tuple::unpack(s2).getString(0);
+		Standalone<StringRef> s3 = co_await items[2].value;
+		Standalone<StringRef> value = Tuple::unpack(s3).getString(0);
 
 		ASSERT(optionInfo.find(op.toString()) != optionInfo.end());
 
@@ -1595,10 +1535,8 @@ struct AtomicOPFunc : InstructionFunc {
 		if (instruction->isDatabase) {
 			data->stack.push(waitForVoid(mutation));
 		} else {
-			wait(mutation);
+			co_await mutation;
 		}
-
-		return Void();
 	}
 };
 const char* AtomicOPFunc::name = "ATOMIC_OP";
@@ -1608,12 +1546,12 @@ REGISTER_INSTRUCTION_FUNC(AtomicOPFunc);
 struct UnitTestsFunc : InstructionFunc {
 	static const char* name;
 
-	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+	static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
 		ASSERT(data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(1020)));
 		ASSERT(!data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(10)));
 
 		ASSERT(API::isAPIVersionSelected());
-		state API* fdb = API::getInstance();
+		API* fdb = API::getInstance();
 		ASSERT(fdb->getAPIVersion() <= FDB_API_VERSION);
 		try {
 			API::selectAPIVersion(fdb->getAPIVersion() + 1);
@@ -1664,7 +1602,7 @@ struct UnitTestsFunc : InstructionFunc {
 		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY);
 		data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS);
 
-		state Reference<Transaction> tr = data->db->createTransaction();
+		Reference<Transaction> tr = data->db->createTransaction();
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_BATCH);
@@ -1689,42 +1627,41 @@ struct UnitTestsFunc : InstructionFunc {
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS);
 		tr->setOption(FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS);
 
-		Optional<FDBStandalone<ValueRef>> _ = wait(tr->get("\xff"_sr));
+		Optional<FDBStandalone<ValueRef>> _ = co_await tr->get("\xff"_sr);
 		tr->cancel();
-
-		return Void();
 	}
 };
 const char* UnitTestsFunc::name = "UNIT_TESTS";
 REGISTER_INSTRUCTION_FUNC(UnitTestsFunc);
 
-ACTOR static Future<Void> getInstructions(Reference<FlowTesterData> data, StringRef prefix) {
-	state Reference<Transaction> tr = data->db->createTransaction();
+static Future<Void> getInstructions(Reference<FlowTesterData> data, StringRef prefix) {
+	Reference<Transaction> tr = data->db->createTransaction();
 
 	// get test instructions
-	state Tuple testSpec;
+	Tuple testSpec;
 	testSpec.append(prefix);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			Standalone<RangeResultRef> results = wait(getRange(tr, testSpec.range()));
+			Standalone<RangeResultRef> results = co_await getRange(tr, testSpec.range());
 			data->instructions = results;
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
+static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 	// printf("Total num instructions:%d\n", data->instructions.size());
-	state size_t idx = 0;
-	for (; idx < data->instructions.size(); ++idx) {
+	for (size_t idx = 0; idx < data->instructions.size(); ++idx) {
 		Tuple opTuple = Tuple::unpack(data->instructions[idx].value);
-		state Standalone<StringRef> op = opTuple.getString(0);
+		Standalone<StringRef> op = opTuple.getString(0);
 
-		state bool isDatabase = op.endsWith("_DATABASE"_sr);
-		state bool isSnapshot = op.endsWith("_SNAPSHOT"_sr);
-		state bool isDirectory = op.startsWith("DIRECTORY_"_sr);
+		bool isDatabase = op.endsWith("_DATABASE"_sr);
+		bool isSnapshot = op.endsWith("_SNAPSHOT"_sr);
+		bool isDirectory = op.startsWith("DIRECTORY_"_sr);
 
 		try {
 			if (LOG_INSTRUCTIONS) {
@@ -1747,10 +1684,10 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 			// wait(printFlowTesterStack(&(data->stack)));
 			// wait(debugPrintRange(instruction->tr, "\x01test_results", ""));
 
-			state Reference<InstructionData> instruction = Reference<InstructionData>(
-			    new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference<Transaction>()));
+			auto instruction = makeReference<InstructionData>(
+			    isDatabase, isSnapshot, data->instructions[idx].value, Reference<Transaction>());
 			if (isDatabase) {
-				state Reference<Transaction> tr = data->db->createTransaction();
+				Reference<Transaction> tr = data->db->createTransaction();
 				instruction->tr = tr;
 			} else {
 				instruction->tr = trMap[data->trName];
@@ -1760,7 +1697,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 			ASSERT(!isDirectory || !isSnapshot);
 
 			data->stack.index = idx;
-			wait(InstructionFunc::call(op.toString(), data, instruction));
+			co_await InstructionFunc::call(op.toString(), data, instruction);
 		} catch (Error& e) {
 			if (LOG_ERRORS) {
 				printf("Error: %s (%d)\n", e.name(), e.code());
@@ -1768,7 +1705,7 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 			}
 
 			if (isDirectory) {
-				if (opsThatCreateDirectories.count(op.toString())) {
+				if (opsThatCreateDirectories.contains(op.toString())) {
 					data->directoryData.directoryList.push_back(DirectoryOrSubspace());
 				}
 				data->stack.pushTuple("DIRECTORY_ERROR"_sr);
@@ -1778,21 +1715,20 @@ ACTOR static Future<Void> doInstructions(Reference<FlowTesterData> data) {
 		}
 	}
 	// printf("Total num instructions:%d\n", data->instructions.size());
-	return Void();
 }
 
-ACTOR static Future<Void> runTest(Reference<FlowTesterData> data, Reference<Database> db, StringRef prefix) {
+static Future<Void> runTest(Reference<FlowTesterData> const& data,
+                            Reference<Database> const& db,
+                            StringRef const& prefix) {
 	ASSERT(data);
 	try {
 		data->db = db;
-		wait(getInstructions(data, prefix));
-		wait(doInstructions(data));
-		wait(waitForAll(data->subThreads));
+		co_await getInstructions(data, prefix);
+		co_await doInstructions(data);
+		co_await waitForAll(data->subThreads);
 	} catch (Error& e) {
 		TraceEvent(SevError, "FlowTesterDataRunError").error(e);
 	}
-
-	return Void();
 }
 
 void populateAtomicOpMap() {
@@ -1823,7 +1759,7 @@ void populateOpsThatCreateDirectories() {
 	opsThatCreateDirectories.insert("DIRECTORY_OPEN_SUBSPACE");
 }
 
-ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersion) {
+Future<Void> startTest(Uncancellable, std::string clusterFilename, StringRef prefix, int apiVersion) {
 	try {
 		populateAtomicOpMap(); // FIXME: NOOOOOO!
 		populateOpsThatCreateDirectories(); // FIXME
@@ -1851,8 +1787,8 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
 		// Connect to the default cluster/database, and create a transaction
 		auto db = fdb->createDatabase(clusterFilename);
 
-		Reference<FlowTesterData> data = Reference<FlowTesterData>(new FlowTesterData(fdb));
-		wait(runTest(data, db, prefix));
+		auto data = makeReference<FlowTesterData>(fdb);
+		co_await runTest(data, db, prefix);
 
 		// Stopping the network returns from g_network->run() and allows
 		// the program to terminate
@@ -1867,7 +1803,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
 	}
 }
 
-ACTOR void _test_versionstamp() {
+Future<Void> _test_versionstamp(Uncancellable) {
 	try {
 		g_network = newNet2(TLSConfig());
 
@@ -1877,20 +1813,20 @@ ACTOR void _test_versionstamp() {
 		startThread(networkThread, fdb);
 
 		auto db = fdb->createDatabase();
-		state Reference<Transaction> tr = db->createTransaction();
+		Reference<Transaction> tr = db->createTransaction();
 
-		state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();
+		Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();
 
 		tr->atomicOp(
 		    "foo"_sr, "blahblahbl\x00\x00\x00\x00"_sr, FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
 
-		wait(tr->commit()); // should use retry loop
+		co_await tr->commit(); // should use retry loop
 
 		tr->reset();
 
-		Optional<FDBStandalone<StringRef>> optionalDbVersion = wait(tr->get("foo"_sr));
-		state FDBStandalone<StringRef> dbVersion = optionalDbVersion.get();
-		FDBStandalone<StringRef> trVersion = wait(ftrVersion);
+		Optional<FDBStandalone<StringRef>> optionalDbVersion = co_await tr->get("foo"_sr);
+		FDBStandalone<StringRef> dbVersion = optionalDbVersion.get();
+		FDBStandalone<StringRef> trVersion = co_await ftrVersion;
 
 		ASSERT(trVersion.compare(dbVersion) == 0);
 
@@ -1932,7 +1868,7 @@ int main(int argc, char** argv) {
 		}
 
 		// start test
-		startTest(clusterFilename, prefix, apiVersion);
+		startTest(Uncancellable(), clusterFilename, prefix, apiVersion);
 
 		// Run the network until someone tells us to stop
 		g_network->run();
diff --git a/bindings/flow/tester/Tester.actor.h b/bindings/flow/tester/Tester.h
similarity index 82%
rename from bindings/flow/tester/Tester.actor.h
rename to bindings/flow/tester/Tester.h
index 96890bcfdcc..89bef41bafb 100644
--- a/bindings/flow/tester/Tester.actor.h
+++ b/bindings/flow/tester/Tester.h
@@ -1,5 +1,5 @@
 /*
- * Tester.actor.h
+ * Tester.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,16 +18,9 @@
  * limitations under the License.
  */
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
-#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
-#include "tester/Tester.actor.g.h"
-#elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H)
-#define FDB_FLOW_TESTER_TESTER_ACTOR_H
-
 #pragma once
 
+#include <type_traits>
 #include <utility>
 
 #include "flow/IDispatched.h"
@@ -35,7 +28,6 @@
 #include "bindings/flow/IDirectory.h"
 #include "bindings/flow/Subspace.h"
 #include "bindings/flow/DirectoryLayer.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 constexpr bool LOG_ALL = false;
 constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false;
@@ -135,9 +127,9 @@ struct DirectoryOrSubspace {
 	Optional<FDB::Subspace*> subspace;
 
 	DirectoryOrSubspace() {}
-	DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
-	DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
-	DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
+	explicit DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
+	explicit DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
+	explicit DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
 	  : directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
 
 	bool valid() { return directory.present() || subspace.present(); }
@@ -173,7 +165,7 @@ struct DirectoryTesterData {
 	}
 
 	DirectoryTesterData() : directoryListIndex(0), directoryErrorIndex(0) {
-		directoryList.push_back(Reference<FDB::IDirectory>(new FDB::DirectoryLayer()));
+		directoryList.push_back(DirectoryOrSubspace(Reference<FDB::IDirectory>(makeReference<FDB::DirectoryLayer>())));
 	}
 
 	template <class T>
@@ -204,29 +196,43 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
 		    instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
 	}
 
-	FlowTesterData(FDB::API* api) { this->api = api; }
+	explicit FlowTesterData(FDB::API* api) { this->api = api; }
 };
 
 std::string tupleToString(FDB::Tuple const& tuple);
 
-ACTOR template <class F>
-Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<InstructionData> instruction, F func) {
-	loop {
+template <class F>
+using ExecuteMutationResult = decltype(std::declval<F>()().getValue());
+
+template <class F>
+Future<ExecuteMutationResult<F>> executeMutation(Reference<InstructionData> instruction, F func) {
+	while (true) {
+		Error err;
+		bool hasErr = false;
 		try {
-			state decltype(std::declval<F>()().getValue()) result = wait(func());
-			if (instruction->isDatabase) {
-				wait(instruction->tr->commit());
+			if constexpr (std::is_same_v<ExecuteMutationResult<F>, Void>) {
+				co_await func();
+				if (instruction->isDatabase) {
+					co_await instruction->tr->commit();
+				}
+				co_return;
+			} else {
+				ExecuteMutationResult<F> result = co_await func();
+				if (instruction->isDatabase) {
+					co_await instruction->tr->commit();
+				}
+				co_return result;
 			}
-			return result;
 		} catch (Error& e) {
+			err = e;
+			hasErr = true;
+		}
+		if (hasErr) {
 			if (instruction->isDatabase) {
-				wait(instruction->tr->onError(e));
+				co_await instruction->tr->onError(err);
 			} else {
-				throw;
+				throw err;
 			}
 		}
 	}
 }
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/bindings/go/CMakeLists.txt b/bindings/go/CMakeLists.txt
index cae3ca3dc03..d4670e94967 100644
--- a/bindings/go/CMakeLists.txt
+++ b/bindings/go/CMakeLists.txt
@@ -22,7 +22,6 @@ set(SRCS
   src/fdb/directory/directory_subspace.go
   src/fdb/fdb_test.go
   src/fdb/snapshot.go
-
   go.mod)
 
 set(GOPATH ${CMAKE_CURRENT_BINARY_DIR})
@@ -43,6 +42,7 @@ set(GO_PACKAGE_OUTDIR ${GOPATH}/pkg/${GOPLATFORM}/${GO_IMPORT_PATH})
 file(MAKE_DIRECTORY ${GOPATH}
                     ${GO_DEST})
 set(go_options_file ${GO_DEST}/src/fdb/generated.go)
+set(go_error_file ${GO_DEST}/src/fdb/error_codes_generated.go)
 
 set(go_env GOPATH=${GOPATH}
   CGO_CFLAGS="-I${CMAKE_BINARY_DIR}/bindings/c/foundationdb;-I${CMAKE_SOURCE_DIR}/bindings/c"
@@ -62,14 +62,23 @@ foreach(src_file IN LISTS SRCS)
 endforeach()
 add_custom_target(copy_go_sources DEPENDS ${SRCS_OUT})
 add_custom_command(OUTPUT ${go_options_file}
-  COMMAND ${GO_EXECUTABLE} run ${CMAKE_CURRENT_SOURCE_DIR}/src/_util/translate_fdb_options.go
+  COMMAND ${GO_EXECUTABLE} run ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/translate_fdb_options.go
           -in ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options
           -out ${go_options_file}
-  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/_util/translate_fdb_options.go
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/translate_fdb_options.go
           ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options
   COMMENT "Generate FDBOptions for GO")
 add_custom_target(go_options_file DEPENDS ${go_options_file})
 add_dependencies(go_options_file copy_go_sources)
+add_custom_command(OUTPUT ${go_error_file}
+  COMMAND ${GO_EXECUTABLE} run ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/gen_errors/main.go
+          -in ${CMAKE_SOURCE_DIR}/flow/include/flow/error_definitions.h
+          -out ${go_error_file}
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/gen_errors/main.go
+          ${CMAKE_SOURCE_DIR}/flow/include/flow/error_definitions.h
+  COMMENT "Generate FDB errors for GO")
+add_custom_target(go_error_file DEPENDS ${go_error_file})
+add_dependencies(go_error_file copy_go_sources)
 
 function(build_go_package)
   set(options LIBRARY EXECUTABLE INCLUDE_TEST)
@@ -128,7 +137,7 @@ function(build_go_package)
 endfunction()
 
 build_go_package(LIBRARY NAME fdb_go PATH fdb INCLUDE_TEST)
-add_dependencies(fdb_go fdb_c go_options_file)
+add_dependencies(fdb_go fdb_c go_options_file go_error_file)
 
 build_go_package(LIBRARY NAME tuple_go PATH fdb/tuple INCLUDE_TEST)
 add_dependencies(tuple_go fdb_go)
@@ -149,3 +158,23 @@ add_test(
   NAME update_bindings_go_src_fdb_generated_go
   COMMAND ${CMAKE_COMMAND} -E compare_files ${go_options_file} ${CMAKE_CURRENT_SOURCE_DIR}/src/fdb/generated.go
 )
+
+# If this fails, then you need to update bindings/go/src/fdb/error_codes_generated.go
+# Ideally this wouldn't be necessary, but it looks like we distribute the go
+# bindings directly from the source on github.
+add_test(
+  NAME update_generated_errors
+  COMMAND ${CMAKE_COMMAND} -E compare_files ${go_error_file} ${CMAKE_CURRENT_SOURCE_DIR}/src/fdb/error_codes_generated.go
+)
+
+# gofmt should be installed with go, but it's better to check if the binary is available.
+find_program(GOFMT_EXECUTABLE gofmt HINTS /usr/local/go/bin/)
+if(GOFMT_EXECUTABLE)
+  add_test(
+    NAME fdb-go-fmt
+    # gofmt -d doesn't return a non-zero exit code, so we have to check if the output is empty.
+    COMMAND bash -c "out=$(\"${GOFMT_EXECUTABLE}\" -d \"${CMAKE_CURRENT_SOURCE_DIR}\"); test -z \"$out\" || { echo \"$out\"; exit 1; }"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+else()
+  message(WARNING "gofmt not found. The 'fdb-go-fmt' test will not be available.")
+endif()
diff --git a/bindings/go/README.md b/bindings/go/README.md
index 884b5f1ffae..13e35307c04 100644
--- a/bindings/go/README.md
+++ b/bindings/go/README.md
@@ -45,3 +45,13 @@ make fdb_go
 ```
 
 This will create binary packages for the appropriate platform within the "build" subdirectory of this folder.
+
+Generating options file
+----------
+
+The [generated.go](./src/fdb/generated.go) is generated based on the [fdb.options](../../fdbclient/vexillographer/fdb.options) file.
+If you change the `fdb.options` file you can update the `generated.go` by running the following command from the root of this reporsitory:
+
+```bash
+go run bindings/go/src/_util/translate_fdb_options.go < fdbclient/vexillographer/fdb.options > bindings/go/src/fdb/generated.go
+```
diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go
index 610d51e9db3..47e8cdc21f4 100644
--- a/bindings/go/src/fdb/database.go
+++ b/bindings/go/src/fdb/database.go
@@ -115,6 +115,9 @@ func (d Database) CreateTransaction() (Transaction, error) {
 // process address is the form of IP:Port pair without the :tls suffix if the cluster is running
 // with TLS enabled. The address can also be multiple processes addresses concated by a comma, e.g.
 // "IP1:Port,IP2:port", in this case the RebootWorker will reboot all provided addresses concurrently.
+// It's not recommened to close the Database directly after calling the RebootWorker method as the
+// reboot commands will be queue and executed asynchronous. If the Database context is closed immediately
+// some of the commands might not be delievered, even if the RebootWorker method returned successfully.
 func (d Database) RebootWorker(address string, checkFile bool, suspendDuration int) error {
 	t := &futureInt64{
 		future: newFutureWithDb(
@@ -144,7 +147,7 @@ func (d Database) RebootWorker(address string, checkFile bool, suspendDuration i
 // NOTE: ErrMultiVersionClientUnavailable will be returned if the Multi-Version client API was not enabled.
 func (d Database) GetClientStatus() ([]byte, error) {
 	if apiVersion == 0 {
-		return nil, errAPIVersionUnset
+		return nil, ErrAPIVersionUnset
 	}
 
 	st := &futureByteSlice{
diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go
index 2ca154cea75..6601573df31 100644
--- a/bindings/go/src/fdb/doc.go
+++ b/bindings/go/src/fdb/doc.go
@@ -39,31 +39,33 @@ A basic interaction with the FoundationDB API is demonstrated below:
 package main
 
 import (
-    "github.com/apple/foundationdb/bindings/go/src/fdb"
-    "log"
-    "fmt"
+
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"log"
+	"fmt"
+
 )
 
-func main() {
-    // Different API versions may expose different runtime behaviors.
-    fdb.MustAPIVersion(800)
-
-    // Open the default database from the system cluster
-    db := fdb.MustOpenDefault()
-
-    // Database reads and writes happen inside transactions
-    ret, err := db.Transact(func(tr fdb.Transaction) (interface{}, error) {
-	tr.Set(fdb.Key("hello"), []byte("world"))
-	return tr.Get(fdb.Key("foo")).MustGet(), nil
-	// db.Transact automatically commits (and if necessary,
-	// retries) the transaction
-    })
-    if err != nil {
-	log.Fatalf("Unable to perform FDB transaction (%v)", err)
-    }
-
-    fmt.Printf("hello is now world, foo was: %s\n", string(ret.([]byte)))
-}
+	func main() {
+	    // Different API versions may expose different runtime behaviors.
+	    fdb.MustAPIVersion(800)
+
+	    // Open the default database from the system cluster
+	    db := fdb.MustOpenDefault()
+
+	    // Database reads and writes happen inside transactions
+	    ret, err := db.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.Set(fdb.Key("hello"), []byte("world"))
+		return tr.Get(fdb.Key("foo")).MustGet(), nil
+		// db.Transact automatically commits (and if necessary,
+		// retries) the transaction
+	    })
+	    if err != nil {
+		log.Fatalf("Unable to perform FDB transaction (%v)", err)
+	    }
+
+	    fmt.Printf("hello is now world, foo was: %s\n", string(ret.([]byte)))
+	}
 
 # Futures
 
diff --git a/bindings/go/src/fdb/error_codes_generated.go b/bindings/go/src/fdb/error_codes_generated.go
new file mode 100644
index 00000000000..69546fe6178
--- /dev/null
+++ b/bindings/go/src/fdb/error_codes_generated.go
@@ -0,0 +1,677 @@
+// Code generated by gen_errors. DO NOT EDIT.
+
+/*
+ * error_codes_generated.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fdb
+
+// To regenerate this file run this command:
+// go run ./bindings/go/src/internal/gen_errors/main.go -in ./flow/include/flow/error_definitions.h -out ./bindings/go/src/fdb/error_codes_generated.go
+//
+// Error sentinel variables derived from flow/include/flow/error_definitions.h.
+// Use with errors.Is to match a specific FoundationDB error code:
+//
+//	errors.Is(err, fdb.ErrTransactionTooOld)
+var (
+	// Success
+	ErrSuccess = Error{Code: 0}
+	// End of stream
+	ErrEndOfStream = Error{Code: 1}
+	// Operation failed
+	ErrOperationFailed = Error{Code: 1000}
+	// Shard is not available from this server
+	ErrWrongShardServer = Error{Code: 1001}
+	// Operation result no longer necessary
+	ErrOperationObsolete = Error{Code: 1002}
+	// Cache server is not warm for this range
+	ErrColdCacheServer = Error{Code: 1003}
+	// Operation timed out
+	ErrTimedOut = Error{Code: 1004}
+	// Conflict occurred while changing coordination information
+	ErrCoordinatedStateConflict = Error{Code: 1005}
+	// All alternatives failed
+	ErrAllAlternativesFailed = Error{Code: 1006}
+	// Transaction is too old to perform reads or be committed
+	ErrTransactionTooOld = Error{Code: 1007}
+	// Not enough physical servers available
+	ErrNoMoreServers = Error{Code: 1008}
+	// Request for future version
+	ErrFutureVersion = Error{Code: 1009}
+	// Conflicting attempts to change data distribution
+	ErrMovekeysConflict = Error{Code: 1010}
+	// TLog stopped
+	ErrTLogStopped = Error{Code: 1011}
+	// Server request queue is full
+	ErrServerRequestQueueFull = Error{Code: 1012}
+	// Transaction not committed due to conflict with another transaction
+	ErrNotCommitted = Error{Code: 1020}
+	// Transaction may or may not have committed
+	ErrCommitUnknownResult = Error{Code: 1021}
+	// Idempotency id for transaction may have expired, so the commit status of the transaction cannot be determined
+	ErrCommitUnknownResultFatal = Error{Code: 1022}
+	// Operation aborted because the transaction was cancelled
+	ErrTransactionCancelled = Error{Code: 1025}
+	// Network connection failed
+	ErrConnectionFailed = Error{Code: 1026}
+	// Coordination servers have changed
+	ErrCoordinatorsChanged = Error{Code: 1027}
+	// New coordination servers did not respond in a timely way
+	ErrNewCoordinatorsTimedOut = Error{Code: 1028}
+	// Watch cancelled because storage server watch limit exceeded
+	ErrWatchCancelled = Error{Code: 1029}
+	// Request may or may not have been delivered
+	ErrRequestMaybeDelivered = Error{Code: 1030}
+	// Operation aborted because the transaction timed out
+	ErrTransactionTimedOut = Error{Code: 1031}
+	// Too many watches currently set
+	ErrTooManyWatches = Error{Code: 1032}
+	// Locality information not available
+	ErrLocalityInformationUnavailable = Error{Code: 1033}
+	// Watches cannot be set if read your writes is disabled
+	ErrWatchesDisabled = Error{Code: 1034}
+	// Default error for an ErrorOr object
+	ErrDefaultErrorOr = Error{Code: 1035}
+	// Read or wrote an unreadable key
+	ErrAccessedUnreadable = Error{Code: 1036}
+	// Storage process does not have recent mutations
+	ErrProcessBehind = Error{Code: 1037}
+	// Database is locked
+	ErrDatabaseLocked = Error{Code: 1038}
+	// The protocol version of the cluster has changed
+	ErrClusterVersionChanged = Error{Code: 1039}
+	// External client has already been loaded
+	ErrExternalClientAlreadyLoaded = Error{Code: 1040}
+	// DNS lookup failed
+	ErrLookupFailed = Error{Code: 1041}
+	// CommitProxy commit memory limit exceeded
+	ErrCommitProxyMemoryLimitExceeded = Error{Code: 1042}
+	// Operation no longer supported due to shutdown
+	ErrShutdownInProgress = Error{Code: 1043}
+	// Failed to deserialize an object
+	ErrSerializationFailed = Error{Code: 1044}
+	// No peer references for connection
+	ErrConnectionUnreferenced = Error{Code: 1048}
+	// Connection closed after idle timeout
+	ErrConnectionIdle = Error{Code: 1049}
+	// The disk queue adapter reset
+	ErrDiskAdapterReset = Error{Code: 1050}
+	// Batch GRV request rate limit exceeded
+	ErrBatchTransactionThrottled = Error{Code: 1051}
+	// Data distribution components cancelled
+	ErrDdCancelled = Error{Code: 1052}
+	// Data distributor not found
+	ErrDdNotFound = Error{Code: 1053}
+	// Connection file mismatch
+	ErrWrongConnectionFile = Error{Code: 1054}
+	// The requested changes have been compacted away
+	ErrVersionAlreadyCompacted = Error{Code: 1055}
+	// Local configuration file has changed. Restart and apply these changes
+	ErrLocalConfigChanged = Error{Code: 1056}
+	// Format version not supported
+	ErrUnsupportedFormatVersion = Error{Code: 1058}
+	// Change feed not found
+	ErrUnknownChangeFeed = Error{Code: 1059}
+	// Change feed not registered
+	ErrChangeFeedNotRegistered = Error{Code: 1060}
+	// Change feed was cancelled
+	ErrChangeFeedCancelled = Error{Code: 1062}
+	// Tried to read a version older than what has been popped from the change feed
+	ErrChangeFeedPopped = Error{Code: 1066}
+	// Page header does not match location on disk
+	ErrPageHeaderWrongPageID = Error{Code: 1068}
+	// Page header checksum failed
+	ErrPageHeaderChecksumFailed = Error{Code: 1069}
+	// Page header version is not supported
+	ErrPageHeaderVersionNotSupported = Error{Code: 1070}
+	// Page encoding type is not supported or not valid
+	ErrPageEncodingNotSupported = Error{Code: 1071}
+	// Page content decoding failed
+	ErrPageDecodingFailed = Error{Code: 1072}
+	// Page content decoding failed
+	ErrUnexpectedEncodingType = Error{Code: 1073}
+	// Encryption key not found
+	ErrEncryptionKeyNotFound = Error{Code: 1074}
+	// Data move was cancelled
+	ErrDataMoveCancelled = Error{Code: 1075}
+	// Dest team was not found for data move
+	ErrDataMoveDestTeamNotFound = Error{Code: 1076}
+	// GetReadVersion proxy memory limit exceeded
+	ErrGRVProxyMemoryLimitExceeded = Error{Code: 1078}
+	// Too many feed streams to a single storage server
+	ErrStorageTooManyFeedStreams = Error{Code: 1080}
+	// Storage engine was never successfully initialized.
+	ErrStorageEngineNotInitialized = Error{Code: 1081}
+	// Storage engine type is not recognized.
+	ErrUnknownStorageEngine = Error{Code: 1082}
+	// A duplicate snapshot request has been sent, the old request is discarded.
+	ErrDuplicateSnapshotRequest = Error{Code: 1083}
+	// DataDistribution configuration changed.
+	ErrDdConfigChanged = Error{Code: 1084}
+	// Consistency check urgent task is failed
+	ErrConsistencyCheckUrgentTaskFailed = Error{Code: 1085}
+	// Data move conflict in SS
+	ErrDataMoveConflict = Error{Code: 1086}
+	// Consistency check urgent got a duplicate request
+	ErrConsistencyCheckUrgentDuplicateRequest = Error{Code: 1087}
+	// Broken promise
+	ErrBrokenPromise = Error{Code: 1100}
+	// Asynchronous operation cancelled
+	ErrOperationCancelled = Error{Code: 1101}
+	// Future has been released
+	ErrFutureReleased = Error{Code: 1102}
+	// Connection object leaked
+	ErrConnectionLeaked = Error{Code: 1103}
+	// Never reply to the request
+	ErrNeverReply = Error{Code: 1104}
+	// Retry operation
+	ErrRetry = Error{Code: 1105}
+	// Recruitment of a server failed
+	ErrRecruitmentFailed = Error{Code: 1200}
+	// Attempt to move keys to a storage server that was removed
+	ErrMoveToRemovedServer = Error{Code: 1201}
+	// Normal worker shut down
+	ErrWorkerRemoved = Error{Code: 1202}
+	// Cluster recovery failed
+	ErrClusterRecoveryFailed = Error{Code: 1203}
+	// Master hit maximum number of versions in flight
+	ErrMasterMaxVersionsInFlight = Error{Code: 1204}
+	// Cluster recovery terminating because a TLog failed
+	ErrTLogFailed = Error{Code: 1205}
+	// Recovery of a worker process failed
+	ErrWorkerRecoveryFailed = Error{Code: 1206}
+	// Reboot of server process requested
+	ErrPleaseReboot = Error{Code: 1207}
+	// Reboot of server process requested, with deletion of state
+	ErrPleaseRebootDelete = Error{Code: 1208}
+	// Master terminating because a CommitProxy failed
+	ErrCommitProxyFailed = Error{Code: 1209}
+	// Cluster recovery terminating because a Resolver failed
+	ErrResolverFailed = Error{Code: 1210}
+	// Server is under too much load and cannot respond
+	ErrServerOverloaded = Error{Code: 1211}
+	// Cluster recovery terminating because a backup worker failed
+	ErrBackupWorkerFailed = Error{Code: 1212}
+	// Transaction tag is being throttled
+	ErrTagThrottled = Error{Code: 1213}
+	// Cluster recovery terminating because a GRVProxy failed
+	ErrGRVProxyFailed = Error{Code: 1214}
+	// The data distribution tracker has been cancelled
+	ErrDdTrackerCancelled = Error{Code: 1215}
+	// Process has failed to make sufficient progress
+	ErrFailedToProgress = Error{Code: 1216}
+	// Attempted to join cluster with a different cluster ID
+	ErrInvalidClusterID = Error{Code: 1217}
+	// Restart cluster controller process
+	ErrRestartClusterController = Error{Code: 1218}
+	// Need to reboot the storage engine
+	ErrPleaseRebootKVStore = Error{Code: 1219}
+	// Current software does not support database format
+	ErrIncompatibleSoftwareVersion = Error{Code: 1220}
+	// Validate storage consistency operation failed
+	ErrAuditStorageFailed = Error{Code: 1221}
+	// Exceeded the max number of allowed concurrent audit storage requests
+	ErrAuditStorageExceededRequestLimit = Error{Code: 1222}
+	// Exceeded maximum time allowed to read or write.
+	ErrKeyValueStoreDeadlineExceeded = Error{Code: 1224}
+	// Found data corruption
+	ErrAuditStorageError = Error{Code: 1226}
+	// Cluster recovery terminating because master has failed
+	ErrMasterFailed = Error{Code: 1227}
+	// Test failed
+	ErrTestFailed = Error{Code: 1228}
+	// Need background datamove cleanup
+	ErrRetryCleanUpDatamoveTombstoneAdded = Error{Code: 1229}
+	// Persist new audit metadata error
+	ErrPersistNewAuditMetadataError = Error{Code: 1230}
+	// Failed to cancel an audit
+	ErrCancelAuditStorageFailed = Error{Code: 1231}
+	// Audit has been cancelled
+	ErrAuditStorageCancelled = Error{Code: 1232}
+	// Found location metadata corruption
+	ErrLocationMetadataCorruption = Error{Code: 1233}
+	// Audit task is scheduled by an outdated DD
+	ErrAuditStorageTaskOutdated = Error{Code: 1234}
+	// Transaction throttled due to hot shard
+	ErrTransactionThrottledHotShard = Error{Code: 1235}
+	// Storage replicas not consistent
+	ErrStorageReplicaComparisonError = Error{Code: 1236}
+	// Storage replica cannot be reached
+	ErrUnreachableStorageReplica = Error{Code: 1237}
+	// Bulk loading task failed
+	ErrBulkloadTaskFailed = Error{Code: 1238}
+	// Bulk loading task outdated
+	ErrBulkloadTaskOutdated = Error{Code: 1239}
+	// Lock range failed
+	ErrRangeLockFailed = Error{Code: 1241}
+	// Transaction rejected due to range lock
+	ErrTransactionRejectedRangeLocked = Error{Code: 1242}
+	// Bulk dumping task failed
+	ErrBulkdumpTaskFailed = Error{Code: 1243}
+	// Bulk dumping task outdated
+	ErrBulkdumpTaskOutdated = Error{Code: 1244}
+	// Bulkload fileset provides invalid filepath
+	ErrBulkloadFilesetInvalidFilepath = Error{Code: 1245}
+	// Bulkload manifest string is failed to decode
+	ErrBulkloadManifestDecodeError = Error{Code: 1246}
+	// Range lock is rejected
+	ErrRangeLockReject = Error{Code: 1247}
+	// Range unlock is rejected
+	ErrRangeUnlockReject = Error{Code: 1248}
+	// Bulkload dataset does not cover the required range
+	ErrBulkloadDatasetNotCoverRequiredRange = Error{Code: 1249}
+	// BulkLoad requires cluster configuration with both shard_encode_location_metadata=1 and enable_read_lock_on_range=1
+	ErrBulkloadInvalidConfiguration = Error{Code: 1250}
+	// GRV request rejected because estimated queue wait exceeds transaction limit
+	ErrTransactionGRVQueueRejected = Error{Code: 1251}
+	// finishMoveKeys exceeded retry limit
+	ErrFinishMoveKeysTooManyRetries = Error{Code: 1252}
+	// Platform error
+	ErrPlatformError = Error{Code: 1500}
+	// Large block allocation failed
+	ErrLargeAllocFailed = Error{Code: 1501}
+	// QueryPerformanceCounter error
+	ErrPerformanceCounterError = Error{Code: 1502}
+	// Null allocator was used to allocate memory
+	ErrBadAllocator = Error{Code: 1503}
+	// Disk i/o operation failed
+	ErrIOError = Error{Code: 1510}
+	// File not found
+	ErrFileNotFound = Error{Code: 1511}
+	// Unable to bind to network
+	ErrBindFailed = Error{Code: 1512}
+	// File could not be read
+	ErrFileNotReadable = Error{Code: 1513}
+	// File could not be written
+	ErrFileNotWritable = Error{Code: 1514}
+	// No cluster file found in current directory or default location
+	ErrNoClusterFileFound = Error{Code: 1515}
+	// File too large to be read
+	ErrFileTooLarge = Error{Code: 1516}
+	// Non sequential file operation not allowed
+	ErrNonSequentialOp = Error{Code: 1517}
+	// HTTP response was badly formed
+	ErrHTTPBadResponse = Error{Code: 1518}
+	// HTTP request not accepted
+	ErrHTTPNotAccepted = Error{Code: 1519}
+	// A data checksum failed
+	ErrChecksumFailed = Error{Code: 1520}
+	// A disk IO operation failed to complete in a timely manner
+	ErrIOTimeout = Error{Code: 1521}
+	// A structurally corrupt data file was detected
+	ErrFileCorrupt = Error{Code: 1522}
+	// HTTP response code not received or indicated failure
+	ErrHTTPRequestFailed = Error{Code: 1523}
+	// HTTP request failed due to bad credentials
+	ErrHTTPAuthFailed = Error{Code: 1524}
+	// HTTP response contained an unexpected X-Request-ID header
+	ErrHTTPBadRequestID = Error{Code: 1525}
+	// Invalid REST URI
+	ErrRestInvalidURI = Error{Code: 1526}
+	// Invalid RESTClient knob
+	ErrRestInvalidRestClientKnob = Error{Code: 1527}
+	// ConnectKey not found in connection pool
+	ErrRestConnectpoolKeyNotFound = Error{Code: 1528}
+	// Unable to lock the file
+	ErrLockFileFailure = Error{Code: 1529}
+	// Unsupported REST protocol
+	ErrRestUnsupportedProtocol = Error{Code: 1530}
+	// Malformed REST response
+	ErrRestMalformedResponse = Error{Code: 1531}
+	// Max BaseCipher length violation
+	ErrRestMaxBaseCipherLen = Error{Code: 1532}
+	// Requested resource was not found
+	ErrResourceNotFound = Error{Code: 1533}
+	// Invalid API call
+	ErrClientInvalidOperation = Error{Code: 2000}
+	// Commit with incomplete read
+	ErrCommitReadIncomplete = Error{Code: 2002}
+	// Invalid test specification
+	ErrTestSpecificationInvalid = Error{Code: 2003}
+	// Key outside legal range
+	ErrKeyOutsideLegalRange = Error{Code: 2004}
+	// Range begin key larger than end key
+	ErrInvertedRange = Error{Code: 2005}
+	// Option set with an invalid value
+	ErrInvalidOptionValue = Error{Code: 2006}
+	// Option not valid in this context
+	ErrInvalidOption = Error{Code: 2007}
+	// Action not possible before the network is configured
+	ErrNetworkNotSetup = Error{Code: 2008}
+	// Network can be configured only once
+	ErrNetworkAlreadySetup = Error{Code: 2009}
+	// Transaction already has a read version set
+	ErrReadVersionAlreadySet = Error{Code: 2010}
+	// Version not valid
+	ErrVersionInvalid = Error{Code: 2011}
+	// Range limits not valid
+	ErrRangeLimitsInvalid = Error{Code: 2012}
+	// Database name must be 'DB'
+	ErrInvalidDatabaseName = Error{Code: 2013}
+	// Attribute not found
+	ErrAttributeNotFound = Error{Code: 2014}
+	// Future not ready
+	ErrFutureNotSet = Error{Code: 2015}
+	// Future not an error
+	ErrFutureNotError = Error{Code: 2016}
+	// Operation issued while a commit was outstanding
+	ErrUsedDuringCommit = Error{Code: 2017}
+	// Unrecognized atomic mutation type
+	ErrInvalidMutationType = Error{Code: 2018}
+	// Attribute too large for type int
+	ErrAttributeTooLarge = Error{Code: 2019}
+	// Transaction does not have a valid commit version
+	ErrTransactionInvalidVersion = Error{Code: 2020}
+	// Transaction is read-only and therefore does not have a commit version
+	ErrNoCommitVersion = Error{Code: 2021}
+	// Environment variable network option could not be set
+	ErrEnvironmentVariableNetworkOptionFailed = Error{Code: 2022}
+	// Attempted to commit a transaction specified as read-only
+	ErrTransactionReadOnly = Error{Code: 2023}
+	// Invalid cache eviction policy, only random and lru are supported
+	ErrInvalidCacheEvictionPolicy = Error{Code: 2024}
+	// Network can only be started once
+	ErrNetworkCannotBeRestarted = Error{Code: 2025}
+	// Detected a deadlock in a callback called from the network thread
+	ErrBlockedFromNetworkThread = Error{Code: 2026}
+	// The index in K[] or V[] is not a valid number or out of range
+	ErrMapperBadIndex = Error{Code: 2030}
+	// A mapped key is not set in database
+	ErrMapperNoSuchKey = Error{Code: 2031}
+	// One of the mapped range queries is too large
+	ErrQuickGetKeyValuesHasMore = Error{Code: 2033}
+	// Found a mapped key that is not served in the same SS
+	ErrQuickGetValueMiss = Error{Code: 2034}
+	// Found a mapped range that is not served in the same SS
+	ErrQuickGetKeyValuesMiss = Error{Code: 2035}
+	// getMappedRange does not support continuation for now
+	ErrGetMappedKeyValuesHasMore = Error{Code: 2038}
+	// getMappedRange tries to read data that were previously written in the transaction
+	ErrGetMappedRangeReadsYourWrites = Error{Code: 2039}
+	// Checkpoint not found
+	ErrCheckpointNotFound = Error{Code: 2040}
+	// The key cannot be parsed as a tuple
+	ErrKeyNotTuple = Error{Code: 2041}
+	// The value cannot be parsed as a tuple
+	ErrValueNotTuple = Error{Code: 2042}
+	// The mapper cannot be parsed as a tuple
+	ErrMapperNotTuple = Error{Code: 2043}
+	// Invalid checkpoint format
+	ErrInvalidCheckpointFormat = Error{Code: 2044}
+	// Failed to create a checkpoint
+	ErrFailedToCreateCheckpoint = Error{Code: 2046}
+	// Failed to restore a checkpoint
+	ErrFailedToRestoreCheckpoint = Error{Code: 2047}
+	// Failed to dump shard metadata for a checkpoint to a sst file
+	ErrFailedToCreateCheckpointShardMetadata = Error{Code: 2048}
+	// Failed to parse address
+	ErrAddressParseError = Error{Code: 2049}
+	// Incompatible protocol version
+	ErrIncompatibleProtocolVersion = Error{Code: 2100}
+	// Transaction exceeds byte limit
+	ErrTransactionTooLarge = Error{Code: 2101}
+	// Key length exceeds limit
+	ErrKeyTooLarge = Error{Code: 2102}
+	// Value length exceeds limit
+	ErrValueTooLarge = Error{Code: 2103}
+	// Connection string invalid
+	ErrConnectionStringInvalid = Error{Code: 2104}
+	// Local address in use
+	ErrAddressInUse = Error{Code: 2105}
+	// Invalid local address
+	ErrInvalidLocalAddress = Error{Code: 2106}
+	// TLS error
+	ErrTLSError = Error{Code: 2107}
+	// Operation is not supported
+	ErrUnsupportedOperation = Error{Code: 2108}
+	// Too many tags set on transaction
+	ErrTooManyTags = Error{Code: 2109}
+	// Tag set on transaction is too long
+	ErrTagTooLong = Error{Code: 2110}
+	// Too many tag throttles have been created
+	ErrTooManyTagThrottles = Error{Code: 2111}
+	// Special key space range read crosses modules. Refer to the `special_key_space_relaxed' transaction option for more details.
+	ErrSpecialKeysCrossModuleRead = Error{Code: 2112}
+	// Special key space range read does not intersect a module. Refer to the `special_key_space_relaxed' transaction option for more details.
+	ErrSpecialKeysNoModuleFound = Error{Code: 2113}
+	// Special Key space is not allowed to write by default. Refer to the `special_key_space_enable_writes` transaction option for more details.
+	ErrSpecialKeysWriteDisabled = Error{Code: 2114}
+	// Special key space key or keyrange in set or clear does not intersect a module
+	ErrSpecialKeysNoWriteModuleFound = Error{Code: 2115}
+	// Special key space clear crosses modules
+	ErrSpecialKeysCrossModuleClear = Error{Code: 2116}
+	// Api call through special keys failed. For more information, call get on special key 0xff0xff/error_message to get a json string of the error message.
+	ErrSpecialKeysAPIFailure = Error{Code: 2117}
+	// Invalid client library metadata.
+	ErrClientLibInvalidMetadata = Error{Code: 2118}
+	// Client library with same identifier already exists on the cluster.
+	ErrClientLibAlreadyExists = Error{Code: 2119}
+	// Client library for the given identifier not found.
+	ErrClientLibNotFound = Error{Code: 2120}
+	// Client library exists, but is not available for download.
+	ErrClientLibNotAvailable = Error{Code: 2121}
+	// Invalid client library binary.
+	ErrClientLibInvalidBinary = Error{Code: 2122}
+	// No external client library provided.
+	ErrNoExternalClientProvided = Error{Code: 2123}
+	// All external clients have failed.
+	ErrAllExternalClientsFailed = Error{Code: 2124}
+	// None of the available clients match the protocol version of the cluster.
+	ErrIncompatibleClient = Error{Code: 2125}
+	// API version is not set
+	ErrAPIVersionUnset = Error{Code: 2200}
+	// API version may be set only once
+	ErrAPIVersionAlreadySet = Error{Code: 2201}
+	// API version not valid
+	ErrAPIVersionInvalid = Error{Code: 2202}
+	// API version not supported
+	ErrAPIVersionNotSupported = Error{Code: 2203}
+	// Failed to load a required FDB API function.
+	ErrAPIFunctionMissing = Error{Code: 2204}
+	// EXACT streaming mode requires limits, but none were given
+	ErrExactModeWithoutLimits = Error{Code: 2210}
+	// Unrecognized data type in packed tuple
+	ErrInvalidTupleDataType = Error{Code: 2250}
+	// Tuple does not have element at specified index
+	ErrInvalidTupleIndex = Error{Code: 2251}
+	// Cannot unpack key that is not in subspace
+	ErrKeyNotInSubspace = Error{Code: 2252}
+	// Cannot specify a prefix unless manual prefixes are enabled
+	ErrManualPrefixesNotEnabled = Error{Code: 2253}
+	// Cannot specify a prefix in a partition
+	ErrPrefixInPartition = Error{Code: 2254}
+	// Root directory cannot be opened
+	ErrCannotOpenRootDirectory = Error{Code: 2255}
+	// Directory already exists
+	ErrDirectoryAlreadyExists = Error{Code: 2256}
+	// Directory does not exist
+	ErrDirectoryDoesNotExist = Error{Code: 2257}
+	// Directory's parent does not exist
+	ErrParentDirectoryDoesNotExist = Error{Code: 2258}
+	// Directory has already been created with a different layer string
+	ErrMismatchedLayer = Error{Code: 2259}
+	// Invalid directory layer metadata
+	ErrInvalidDirectoryLayerMetadata = Error{Code: 2260}
+	// Directory cannot be moved between partitions
+	ErrCannotMoveDirectoryBetweenPartitions = Error{Code: 2261}
+	// Directory partition cannot be used as subspace
+	ErrCannotUsePartitionAsSubspace = Error{Code: 2262}
+	// Directory layer was created with an incompatible version
+	ErrIncompatibleDirectoryVersion = Error{Code: 2263}
+	// Database has keys stored at the prefix chosen by the automatic prefix allocator
+	ErrDirectoryPrefixNotEmpty = Error{Code: 2264}
+	// Directory layer already has a conflicting prefix
+	ErrDirectoryPrefixInUse = Error{Code: 2265}
+	// Target directory is invalid
+	ErrInvalidDestinationDirectory = Error{Code: 2266}
+	// Root directory cannot be modified
+	ErrCannotModifyRootDirectory = Error{Code: 2267}
+	// UUID is not sixteen bytes
+	ErrInvalidUUIDSize = Error{Code: 2268}
+	// Versionstamp is not exactly twelve bytes
+	ErrInvalidVersionstampSize = Error{Code: 2269}
+	// Backup error
+	ErrBackupError = Error{Code: 2300}
+	// Restore error
+	ErrRestoreError = Error{Code: 2301}
+	// Backup duplicate request
+	ErrBackupDuplicate = Error{Code: 2311}
+	// Backup unneeded request
+	ErrBackupUnneeded = Error{Code: 2312}
+	// Backup file block size too small
+	ErrBackupBadBlockSize = Error{Code: 2313}
+	// Backup Container URL invalid
+	ErrBackupInvalidURL = Error{Code: 2314}
+	// Backup Container info invalid
+	ErrBackupInvalidInfo = Error{Code: 2315}
+	// Cannot expire requested data from backup without violating minimum restorability
+	ErrBackupCannotExpire = Error{Code: 2316}
+	// Cannot find authentication details (such as a password or secret key) for the specified Backup Container URL
+	ErrBackupAuthMissing = Error{Code: 2317}
+	// Cannot read or parse one or more sources of authentication information for Backup Container URLs
+	ErrBackupAuthUnreadable = Error{Code: 2318}
+	// Backup does not exist
+	ErrBackupDoesNotExist = Error{Code: 2319}
+	// Backup before 6.3 cannot be filtered with key ranges
+	ErrBackupNotFilterableWithKeyRanges = Error{Code: 2320}
+	// Backup key ranges doesn't overlap with key ranges filter
+	ErrBackupNotOverlappedWithKeysFilter = Error{Code: 2321}
+	// bucket is not in the URL for backup
+	ErrBucketNotInURL = Error{Code: 2322}
+	// Invalid restore version
+	ErrRestoreInvalidVersion = Error{Code: 2361}
+	// Corrupted backup data
+	ErrRestoreCorruptedData = Error{Code: 2362}
+	// Missing backup data
+	ErrRestoreMissingData = Error{Code: 2363}
+	// Restore duplicate request
+	ErrRestoreDuplicateTag = Error{Code: 2364}
+	// Restore tag does not exist
+	ErrRestoreUnknownTag = Error{Code: 2365}
+	// Unknown backup/restore file type
+	ErrRestoreUnknownFileType = Error{Code: 2366}
+	// Unsupported backup file version
+	ErrRestoreUnsupportedFileVersion = Error{Code: 2367}
+	// Unexpected number of bytes read
+	ErrRestoreBadRead = Error{Code: 2368}
+	// Backup file has unexpected padding bytes
+	ErrRestoreCorruptedDataPadding = Error{Code: 2369}
+	// Attempted to restore into a non-empty destination database
+	ErrRestoreDestinationNotEmpty = Error{Code: 2370}
+	// Attempted to restore using a UID that had been used for an aborted restore
+	ErrRestoreDuplicateUID = Error{Code: 2371}
+	// Invalid task version
+	ErrTaskInvalidVersion = Error{Code: 2381}
+	// Task execution stopped due to timeout, abort, or completion by another worker
+	ErrTaskInterrupted = Error{Code: 2382}
+	// The provided encryption key file has invalid contents
+	ErrInvalidEncryptionKeyFile = Error{Code: 2383}
+	// Missing mutation logs
+	ErrBlobRestoreMissingLogs = Error{Code: 2384}
+	// Corrupted mutation logs
+	ErrBlobRestoreCorruptedLogs = Error{Code: 2385}
+	// Invalid manifest URL
+	ErrBlobRestoreInvalidManifestURL = Error{Code: 2386}
+	// Corrupted manifest
+	ErrBlobRestoreCorruptedManifest = Error{Code: 2387}
+	// Missing manifest
+	ErrBlobRestoreMissingManifest = Error{Code: 2388}
+	// Blob migrator is replaced
+	ErrBlobMigratorReplaced = Error{Code: 2389}
+	// BulkDump dataset incomplete. Use --rangefile flag
+	ErrRestoreBulkloadDatasetIncomplete = Error{Code: 2390}
+	// BulkLoad operation failed
+	ErrRestoreBulkloadFailed = Error{Code: 2391}
+	// BulkDump operation timed out
+	ErrBackupBulkdumpTimeout = Error{Code: 2392}
+	// BulkDump operation failed
+	ErrBackupBulkdumpFailed = Error{Code: 2393}
+	// Expected key is missing
+	ErrKeyNotFound = Error{Code: 2400}
+	// JSON string was malformed
+	ErrJSONMalformed = Error{Code: 2401}
+	// JSON string did not terminate where expected
+	ErrJSONEofExpected = Error{Code: 2402}
+	// Failed to disable tlog pops
+	ErrSnapDisableTLogPopFailed = Error{Code: 2500}
+	// Failed to snapshot storage nodes
+	ErrSnapStorageFailed = Error{Code: 2501}
+	// Failed to snapshot TLog nodes
+	ErrSnapTLogFailed = Error{Code: 2502}
+	// Failed to snapshot coordinator nodes
+	ErrSnapCoordFailed = Error{Code: 2503}
+	// Failed to enable tlog pops
+	ErrSnapEnableTLogPopFailed = Error{Code: 2504}
+	// Snapshot create binary path not whitelisted
+	ErrSnapPathNotWhitelisted = Error{Code: 2505}
+	// Unsupported when the cluster is not fully recovered
+	ErrSnapNotFullyRecoveredUnsupported = Error{Code: 2506}
+	// Unsupported when log anti quorum is configured
+	ErrSnapLogAntiQuorumUnsupported = Error{Code: 2507}
+	// Cluster recovery during snapshot operation not supported
+	ErrSnapWithRecoveryUnsupported = Error{Code: 2508}
+	// The given uid string is not a 32-length hex string
+	ErrSnapInvalidUIDString = Error{Code: 2509}
+	// Encryption operation error
+	ErrEncryptOpsError = Error{Code: 2700}
+	// Encryption header metadata mismatch
+	ErrEncryptHeaderMetadataMismatch = Error{Code: 2701}
+	// Expected encryption key is missing
+	ErrEncryptKeyNotFound = Error{Code: 2702}
+	// Expected encryption key TTL has expired
+	ErrEncryptKeyTTLExpired = Error{Code: 2703}
+	// Encryption header authentication token mismatch
+	ErrEncryptHeaderAuthtokenMismatch = Error{Code: 2704}
+	// Attempt to update encryption cipher key
+	ErrEncryptUpdateCipher = Error{Code: 2705}
+	// Invalid encryption cipher details
+	ErrEncryptInvalidID = Error{Code: 2706}
+	// Encryption keys fetch from external KMS failed
+	ErrEncryptKeysFetchFailed = Error{Code: 2707}
+	// Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc.
+	ErrEncryptInvalidKMSConfig = Error{Code: 2708}
+	// Encryption not supported
+	ErrEncryptUnsupported = Error{Code: 2709}
+	// Encryption mode mismatch with configuration
+	ErrEncryptModeMismatch = Error{Code: 2710}
+	// Encryption key-check-value mismatch
+	ErrEncryptKeyCheckValueMismatch = Error{Code: 2711}
+	// Max BaseCipher buffer length violation
+	ErrEncryptMaxBaseCipherLen = Error{Code: 2712}
+	// An unknown error occurred
+	ErrUnknownError = Error{Code: 4000}
+	// An internal error occurred
+	ErrInternalError = Error{Code: 4100}
+	// Not implemented yet
+	ErrNotImplemented = Error{Code: 4200}
+	// Client tried to access unauthorized data
+	ErrPermissionDenied = Error{Code: 6000}
+	// A untrusted client tried to send a message to a private endpoint
+	ErrUnauthorizedAttempt = Error{Code: 6001}
+	// Digital signature operation error
+	ErrDigitalSignatureOpsError = Error{Code: 6002}
+	// Failed to verify authorization token
+	ErrAuthorizationTokenVerifyFailed = Error{Code: 6003}
+	// Failed to decode public/private key
+	ErrPkeyDecodeError = Error{Code: 6004}
+	// Failed to encode public/private key
+	ErrPkeyEncodeError = Error{Code: 6005}
+	// gRPC Error
+	ErrGRPCError = Error{Code: 7000}
+)
diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go
index c40063f2f42..b72bc901e59 100644
--- a/bindings/go/src/fdb/errors.go
+++ b/bindings/go/src/fdb/errors.go
@@ -20,6 +20,8 @@
 
 // FoundationDB Go API
 
+//go:generate go run ./internal/gen_errors/main.go -in ../../../flow/include/flow/error_definitions.h -out fdb/error_codes_generated.go
+
 package fdb
 
 // #define FDB_API_VERSION 800
@@ -34,10 +36,12 @@ import (
 // Error may be returned by any FoundationDB API function that returns error, or
 // as a panic from any FoundationDB API function whose name ends with OrPanic.
 //
-// You may compare the Code field of an Error against the list of FoundationDB
-// error codes at https://apple.github.io/foundationdb/api-error-codes.html,
-// but generally an Error should be passed to (Transaction).OnError. When using
-// (Database).Transact, non-fatal errors will be retried automatically.
+// Use errors.Is with the sentinel variables defined in error_codes_generated.go
+// to check for specific error codes:
+//
+//	errors.Is(err, fdb.ErrTransactionTooOld)
+//
+// When using (Database).Transact, non-fatal errors will be retried automatically.
 type Error struct {
 	Code int
 }
@@ -46,14 +50,15 @@ func (e Error) Error() string {
 	return fmt.Sprintf("FoundationDB error code %d (%s)", e.Code, C.GoString(C.fdb_get_error(C.fdb_error_t(e.Code))))
 }
 
-// SOMEDAY: these (along with others) should be coming from fdb.options?
-
-var (
-	errNetworkNotSetup          = Error{2008}
-	errNetworkAlreadySetup      = Error{2009} // currently unused
-	errNetworkCannotBeRestarted = Error{2025} // currently unused
-
-	errAPIVersionUnset        = Error{2200}
-	errAPIVersionAlreadySet   = Error{2201}
-	errAPIVersionNotSupported = Error{2203}
-)
+// Is implements the errors.Is contract. It matches target if target is an Error
+// or *Error with the same Code, enabling errors.Is(err, fdb.ErrTransactionTooOld).
+func (e Error) Is(target error) bool {
+	switch t := target.(type) {
+	case Error:
+		return e.Code == t.Code
+	case *Error:
+		return t != nil && e.Code == t.Code
+	default:
+		return false
+	}
+}
diff --git a/bindings/go/src/fdb/errors_test.go b/bindings/go/src/fdb/errors_test.go
index 15babf7f0d6..a6e054a7271 100644
--- a/bindings/go/src/fdb/errors_test.go
+++ b/bindings/go/src/fdb/errors_test.go
@@ -60,3 +60,30 @@ func TestErrorWrapping(t *testing.T) {
 		}
 	}
 }
+
+func TestErrorIs(t *testing.T) {
+	// value sentinel — direct match
+	if !errors.Is(Error{Code: 1007}, ErrTransactionTooOld) {
+		t.Error("expected match: Error{1007} vs ErrTransactionTooOld")
+	}
+	// pointer target (user takes address of sentinel var)
+	if !errors.Is(Error{Code: 1007}, &ErrTransactionTooOld) {
+		t.Error("expected match: Error{1007} vs &ErrTransactionTooOld")
+	}
+	// wrapped value
+	if !errors.Is(fmt.Errorf("wrap: %w", Error{Code: 1007}), ErrTransactionTooOld) {
+		t.Error("expected match through wrapping")
+	}
+	// wrapped pointer
+	if !errors.Is(fmt.Errorf("wrap: %w", &Error{Code: 1007}), ErrTransactionTooOld) {
+		t.Error("expected match through wrapped *Error")
+	}
+	// no false positive: wrong code
+	if errors.Is(Error{Code: 1008}, ErrTransactionTooOld) {
+		t.Error("expected no match: Error{1008} vs ErrTransactionTooOld")
+	}
+	// no false positive: unrelated error
+	if errors.Is(errors.New("other"), ErrTransactionTooOld) {
+		t.Error("expected no match: non-fdb error vs ErrTransactionTooOld")
+	}
+}
diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go
index 340a1874105..da86bc3c0aa 100644
--- a/bindings/go/src/fdb/fdb.go
+++ b/bindings/go/src/fdb/fdb.go
@@ -43,7 +43,7 @@ var (
 	// with the network thread while the network thread is no more running.
 	ErrNetworkIsStopped = errors.New("network is stopped")
 
-	// ErrNetworkAlreadyStopped for a too early call to StopNetwork().
+	// ErrNetworkNotStarted for a too early call to StopNetwork().
 	ErrNetworkNotStarted = errors.New("network has not been started")
 )
 
@@ -107,7 +107,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error {
 	defer networkMutex.Unlock()
 
 	if apiVersion == 0 {
-		return errAPIVersionUnset
+		return ErrAPIVersionUnset
 	}
 
 	return setOpt(func(p *C.uint8_t, pl C.int) C.fdb_error_t {
@@ -138,11 +138,11 @@ func APIVersion(version int) error {
 		if apiVersion == version {
 			return nil
 		}
-		return errAPIVersionAlreadySet
+		return ErrAPIVersionAlreadySet
 	}
 
 	if version < 200 || version > headerVersion {
-		return errAPIVersionNotSupported
+		return ErrAPIVersionNotSupported
 	}
 
 	if e := C.fdb_select_api_version_impl(C.int(version), C.int(headerVersion)); e != 0 {
@@ -181,7 +181,7 @@ func GetAPIVersion() (int, error) {
 	if IsAPIVersionSelected() {
 		return apiVersion, nil
 	}
-	return 0, errAPIVersionUnset
+	return 0, ErrAPIVersionUnset
 }
 
 // MustAPIVersion is like APIVersion but panics if the API version is not
@@ -266,7 +266,7 @@ func executeWithRunningNetworkThread(f func()) error {
 // StartNetwork does nothing, but it will ensure that the API version is set and return an error otherwise.
 func StartNetwork() error {
 	if apiVersion == 0 {
-		return errAPIVersionUnset
+		return ErrAPIVersionUnset
 	}
 
 	return nil
@@ -378,7 +378,7 @@ func MustOpen(clusterFile string, dbName []byte) Database {
 // Caller must call Close() to release resources.
 func createDatabase(clusterFile string) (Database, error) {
 	if apiVersion == 0 {
-		return Database{}, errAPIVersionUnset
+		return Database{}, ErrAPIVersionUnset
 	}
 
 	var cf *C.char
@@ -412,7 +412,7 @@ func createDatabase(clusterFile string) (Database, error) {
 // Caller must call Close() to release resources.
 func OpenWithConnectionString(connectionString string) (Database, error) {
 	if apiVersion == 0 {
-		return Database{}, errAPIVersionUnset
+		return Database{}, ErrAPIVersionUnset
 	}
 
 	var cf *C.char
@@ -451,11 +451,11 @@ func CreateCluster(clusterFile string) (Cluster, error) {
 	defer networkMutex.Unlock()
 
 	if apiVersion == 0 {
-		return Cluster{}, errAPIVersionUnset
+		return Cluster{}, ErrAPIVersionUnset
 	}
 
 	if !networkStarted {
-		return Cluster{}, errNetworkNotSetup
+		return Cluster{}, ErrNetworkNotSetup
 	}
 
 	if networkStopped {
diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index c33ffc7feaf..76e39cca157 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -60,14 +60,14 @@ func (o NetworkOptions) SetTraceEnable(param string) error {
 	return o.setOpt(30, []byte(param))
 }
 
-// Sets the maximum size in bytes of a single trace output file. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on individual file size. The default is a maximum size of 10,485,760 bytes.
+// Sets the maximum size in bytes of a single trace output file. This value should be in the range "[0, INT64_MAX]". If the value is set to 0, there is no limit on individual file size. The default is a maximum size of 10,485,760 bytes.
 //
 // Parameter: max size of a single trace output file
 func (o NetworkOptions) SetTraceRollSize(param int64) error {
 	return o.setOpt(31, int64ToBytes(param))
 }
 
-// Sets the maximum size of all the trace output files put together. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time.
+// Sets the maximum size of all the trace output files put together. This value should be in the range "[0, INT64_MAX]". If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time.
 //
 // Parameter: max total size of trace files
 func (o NetworkOptions) SetTraceMaxLogsSize(param int64) error {
@@ -336,7 +336,7 @@ func (o NetworkOptions) SetDistributedClientTracer(param string) error {
 
 // Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp
 //
-// Parameter: Client directory for temporary files. 
+// Parameter: Client directory for temporary files.
 func (o NetworkOptions) SetClientTmpDir(param string) error {
 	return o.setOpt(91, []byte(param))
 }
@@ -379,35 +379,35 @@ func (o DatabaseOptions) SetSnapshotRywDisable() error {
 	return o.setOpt(27, nil)
 }
 
-// Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option. This sets the ``transaction_logging_max_field_length`` option of each transaction created by this database. See the transaction option description for more information.
+// Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option. This sets the "transaction_logging_max_field_length" option of each transaction created by this database. See the transaction option description for more information.
 //
 // Parameter: Maximum length of escaped key and value fields.
 func (o DatabaseOptions) SetTransactionLoggingMaxFieldLength(param int64) error {
 	return o.setOpt(405, int64ToBytes(param))
 }
 
-// Set a timeout in milliseconds which, when elapsed, will cause each transaction automatically to be cancelled. This sets the ``timeout`` option of each transaction created by this database. See the transaction option description for more information. Using this option requires that the API version is 610 or higher.
+// Set a timeout in milliseconds which, when elapsed, will cause each transaction automatically to be cancelled. This sets the "timeout" option of each transaction created by this database. See the transaction option description for more information. Using this option requires that the API version is 610 or higher.
 //
 // Parameter: value in milliseconds of timeout
 func (o DatabaseOptions) SetTransactionTimeout(param int64) error {
 	return o.setOpt(500, int64ToBytes(param))
 }
 
-// Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information.
+// Set a maximum number of retries after which additional calls to "onError" will throw the most recently seen error code. This sets the "retry_limit" option of each transaction created by this database. See the transaction option description for more information.
 //
 // Parameter: number of times to retry
 func (o DatabaseOptions) SetTransactionRetryLimit(param int64) error {
 	return o.setOpt(501, int64ToBytes(param))
 }
 
-// Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. This sets the ``max_retry_delay`` option of each transaction created by this database. See the transaction option description for more information.
+// Set the maximum amount of backoff delay incurred in the call to "onError" if the error is retryable. This sets the "max_retry_delay" option of each transaction created by this database. See the transaction option description for more information.
 //
 // Parameter: value in milliseconds of maximum delay
 func (o DatabaseOptions) SetTransactionMaxRetryDelay(param int64) error {
 	return o.setOpt(502, int64ToBytes(param))
 }
 
-// Set the maximum transaction size in bytes. This sets the ``size_limit`` option on each transaction created by this database. See the transaction option description for more information.
+// Set the maximum transaction size in bytes. This sets the "size_limit" option on each transaction created by this database. See the transaction option description for more information.
 //
 // Parameter: value in bytes
 func (o DatabaseOptions) SetTransactionSizeLimit(param int64) error {
@@ -429,7 +429,7 @@ func (o DatabaseOptions) SetTransactionAutomaticIdempotency() error {
 	return o.setOpt(506, nil)
 }
 
-// Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. This sets the ``bypass_unreadable`` option of each transaction created by this database. See the transaction option description for more information.
+// Allows "get" operations to read from sections of keyspace that have become unreadable because of versionstamp operations. This sets the "bypass_unreadable" option of each transaction created by this database. See the transaction option description for more information.
 func (o DatabaseOptions) SetTransactionBypassUnreadable() error {
 	return o.setOpt(700, nil)
 }
@@ -597,21 +597,21 @@ func (o TransactionOptions) SetServerRequestTracing() error {
 	return o.setOpt(406, nil)
 }
 
-// Set a timeout in milliseconds which, when elapsed, will cause the transaction automatically to be cancelled. Valid parameter values are ``[0, INT_MAX]``. If set to 0, will disable all timeouts. All pending and any future uses of the transaction will throw an exception. The transaction can be used again after it is reset. Prior to API version 610, like all other transaction options, the timeout must be reset after a call to ``onError``. If the API version is 610 or greater, the timeout is not reset after an ``onError`` call. This allows the user to specify a longer timeout on specific transactions than the default timeout specified through the ``transaction_timeout`` database option without the shorter database timeout cancelling transactions that encounter a retryable error. Note that at all API versions, it is safe and legal to set the timeout each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
+// Set a timeout in milliseconds which, when elapsed, will cause the transaction automatically to be cancelled. Valid parameter values are "[0, INT_MAX]". If set to 0, will disable all timeouts. All pending and any future uses of the transaction will throw an exception. The transaction can be used again after it is reset. Prior to API version 610, like all other transaction options, the timeout must be reset after a call to "onError". If the API version is 610 or greater, the timeout is not reset after an "onError" call. This allows the user to specify a longer timeout on specific transactions than the default timeout specified through the "transaction_timeout" database option without the shorter database timeout cancelling transactions that encounter a retryable error. Note that at all API versions, it is safe and legal to set the timeout each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
 //
 // Parameter: value in milliseconds of timeout
 func (o TransactionOptions) SetTimeout(param int64) error {
 	return o.setOpt(500, int64ToBytes(param))
 }
 
-// Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. Valid parameter values are ``[-1, INT_MAX]``. If set to -1, will disable the retry limit. Prior to API version 610, like all other transaction options, the retry limit must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the retry limit each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
+// Set a maximum number of retries after which additional calls to "onError" will throw the most recently seen error code. Valid parameter values are "[-1, INT_MAX]". If set to -1, will disable the retry limit. Prior to API version 610, like all other transaction options, the retry limit must be reset after a call to "onError". If the API version is 610 or greater, the retry limit is not reset after an "onError" call. Note that at all API versions, it is safe and legal to set the retry limit each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
 //
 // Parameter: number of times to retry
 func (o TransactionOptions) SetRetryLimit(param int64) error {
 	return o.setOpt(501, int64ToBytes(param))
 }
 
-// Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. Defaults to 1000 ms. Valid parameter values are ``[0, INT_MAX]``. If the maximum retry delay is less than the current retry delay of the transaction, then the current retry delay will be clamped to the maximum retry delay. Prior to API version 610, like all other transaction options, the maximum retry delay must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the maximum retry delay each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
+// Set the maximum amount of backoff delay incurred in the call to "onError" if the error is retryable. Defaults to 1000 ms. Valid parameter values are "[0, INT_MAX]". If the maximum retry delay is less than the current retry delay of the transaction, then the current retry delay will be clamped to the maximum retry delay. Prior to API version 610, like all other transaction options, the maximum retry delay must be reset after a call to "onError". If the API version is 610 or greater, the retry limit is not reset after an "onError" call. Note that at all API versions, it is safe and legal to set the maximum retry delay each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
 //
 // Parameter: value in milliseconds of maximum delay
 func (o TransactionOptions) SetMaxRetryDelay(param int64) error {
@@ -625,7 +625,7 @@ func (o TransactionOptions) SetSizeLimit(param int64) error {
 	return o.setOpt(503, int64ToBytes(param))
 }
 
-// Automatically assign a random 16 byte idempotency id for this transaction. Prevents commits from failing with ``commit_unknown_result``. WARNING: If you are also using the multiversion client or transaction timeouts, if either cluster_version_changed or transaction_timed_out was thrown during a commit, then that commit may have already succeeded or may succeed in the future. This feature is in development and not ready for general use.
+// Automatically assign a random 16 byte idempotency id for this transaction. Prevents commits from failing with "commit_unknown_result". WARNING: If you are also using the multiversion client or transaction timeouts, if either cluster_version_changed or transaction_timed_out was thrown during a commit, then that commit may have already succeeded or may succeed in the future. This feature is in development and not ready for general use.
 func (o TransactionOptions) SetAutomaticIdempotency() error {
 	return o.setOpt(505, nil)
 }
@@ -650,7 +650,7 @@ func (o TransactionOptions) SetUsedDuringCommitProtectionDisable() error {
 	return o.setOpt(701, nil)
 }
 
-// The transaction can read from locked databases.
+// The transaction can read from locked databases. This option will make the transaction read-only if the lock_aware option is not set.
 func (o TransactionOptions) SetReadLockAware() error {
 	return o.setOpt(702, nil)
 }
@@ -689,7 +689,7 @@ func (o TransactionOptions) SetAutoThrottleTag(param string) error {
 	return o.setOpt(801, []byte(param))
 }
 
-// Adds a parent to the Span of this transaction. Used for transaction tracing. A span can be identified with a 33 bytes serialized binary format which consists of: 8 bytes protocol version, e.g. ``0x0FDB00B073000000LL`` in little-endian format, 16 bytes trace id, 8 bytes span id, 1 byte set to 1 if sampling is enabled
+// Adds a parent to the Span of this transaction. Used for transaction tracing. A span can be identified with a 33 bytes serialized binary format which consists of: 8 bytes protocol version, e.g. "0x0FDB00B073000000LL" in little-endian format, 16 bytes trace id, 8 bytes span id, 1 byte set to 1 if sampling is enabled
 //
 // Parameter: A serialized binary byte string of length 33 used to associate the span of this transaction with a parent
 func (o TransactionOptions) SetSpanParent(param []byte) error {
@@ -701,7 +701,7 @@ func (o TransactionOptions) SetExpensiveClearCostEstimationEnable() error {
 	return o.setOpt(1000, nil)
 }
 
-// Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp.
+// Allows "get" operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp.
 func (o TransactionOptions) SetBypassUnreadable() error {
 	return o.setOpt(1100, nil)
 }
@@ -711,6 +711,13 @@ func (o TransactionOptions) SetUseGrvCache() error {
 	return o.setOpt(1101, nil)
 }
 
+// Rejects a GRV request with "transaction_grv_queue_rejected" if the GRV proxy estimates that ratekeeper throttling will keep the request queued longer than this limit. Valid parameter values are "[0, INT_MAX]". The estimate is advisory and only accounts for time spent queued at the GRV proxy due to ratekeeper throttling.
+//
+// Parameter: value in milliseconds of maximum estimated GRV queue delay
+func (o TransactionOptions) SetMaxGrvQueueDelay(param int64) error {
+	return o.setOpt(1103, int64ToBytes(param))
+}
+
 // Attach given authorization token to the transaction such that subsequent tenant-aware requests are authorized
 //
 // Parameter: A JSON Web Token authorized to access data belonging to one or more tenants, indicated by 'tenants' claim of the token's payload.
@@ -744,13 +751,13 @@ const (
 	// order to minimize costs if the client doesn't read the entire range), and
 	// as the caller iterates over more items in the range larger batches will
 	// be transferred in order to minimize latency. After enough iterations,
-	// the iterator mode will eventually reach the same byte limit as “WANT_ALL“
+	// the iterator mode will eventually reach the same byte limit as "WANT_ALL"
 	StreamingModeIterator StreamingMode = 0
 
 	// Infrequently used. The client has passed a specific row limit and wants
 	// that many rows delivered in a single batch. Because of iterator operation
 	// in client drivers make request batches transparent to the user, consider
-	// “WANT_ALL“ StreamingMode instead. A row limit must be specified if this
+	// "WANT_ALL" StreamingMode instead. A row limit must be specified if this
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
@@ -777,7 +784,7 @@ const (
 	StreamingModeSerial StreamingMode = 5
 )
 
-// Performs an addition of little-endian integers. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The integers to be added must be stored in a little-endian representation.  They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to ``param`` and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow.
+// Performs an addition of little-endian integers. If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes.  If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". The integers to be added must be stored in a little-endian representation.  They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to "param" and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow.
 func (t Transaction) Add(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 2)
 }
@@ -787,7 +794,7 @@ func (t Transaction) And(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 6)
 }
 
-// Performs a bitwise ``and`` operation.  If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
+// Performs a bitwise "and" operation.  If the existing value in the database is not present, then "param" is stored in the database. If the existing value in the database is shorter than "param", it is first extended to the length of "param" with zero bytes.  If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param".
 func (t Transaction) BitAnd(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 6)
 }
@@ -797,7 +804,7 @@ func (t Transaction) Or(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 7)
 }
 
-// Performs a bitwise ``or`` operation.  If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
+// Performs a bitwise "or" operation.  If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes.  If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param".
 func (t Transaction) BitOr(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 7)
 }
@@ -807,47 +814,47 @@ func (t Transaction) Xor(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 8)
 }
 
-// Performs a bitwise ``xor`` operation.  If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
+// Performs a bitwise "xor" operation.  If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes.  If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param".
 func (t Transaction) BitXor(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 8)
 }
 
-// Appends ``param`` to the end of the existing value already in the database at the given key (or creates the key and sets the value to ``param`` if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size.
+// Appends "param" to the end of the existing value already in the database at the given key (or creates the key and sets the value to "param" if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size.
 func (t Transaction) AppendIfFits(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 9)
 }
 
-// Performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The larger of the two values is then stored in the database.
+// Performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes.  If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". The larger of the two values is then stored in the database.
 func (t Transaction) Max(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 12)
 }
 
-// Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database.
+// Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then "param" is stored in the database. If the existing value in the database is shorter than "param", it is first extended to the length of "param" with zero bytes.  If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". The smaller of the two values is then stored in the database.
 func (t Transaction) Min(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 13)
 }
 
-// Transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes.
+// Transforms "key" using a versionstamp for the transaction. Sets the transformed key in the database to "param". The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position "pos". The 10 bytes of the key from "pos" to "pos + 10" are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes.
 func (t Transaction) SetVersionstampedKey(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 14)
 }
 
-// Transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset.
+// Transforms "param" using a versionstamp for the transaction. Sets the "key" given to the transformed "param". The parameter is transformed by removing the final four bytes from "param" and reading those as a little-Endian 32-bit integer to get a position "pos". The 10 bytes of the parameter from "pos" to "pos + 10" are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset.
 func (t Transaction) SetVersionstampedValue(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 15)
 }
 
-// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the smaller of the two values is then stored in the database.
+// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then "param" is stored. Otherwise the smaller of the two values is then stored in the database.
 func (t Transaction) ByteMin(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 16)
 }
 
-// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database.
+// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then "param" is stored. Otherwise the larger of the two values is then stored in the database.
 func (t Transaction) ByteMax(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 17)
 }
 
-// Performs an atomic ``compare and clear`` operation. If the existing value in the database is equal to the given value, then given key is cleared.
+// Performs an atomic "compare and clear" operation. If the existing value in the database is equal to the given value, then given key is cleared.
 func (t Transaction) CompareAndClear(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 20)
 }
@@ -867,15 +874,15 @@ type ErrorPredicate int
 
 const (
 
-	// Returns “true“ if the error indicates the operations in the transactions
+	// Returns "true" if the error indicates the operations in the transactions
 	// should be retried because of transient error.
 	ErrorPredicateRetryable ErrorPredicate = 50000
 
-	// Returns “true“ if the error indicates the transaction may have succeeded,
+	// Returns "true" if the error indicates the transaction may have succeeded,
 	// though not in a way the system can verify.
 	ErrorPredicateMaybeCommitted ErrorPredicate = 50001
 
-	// Returns “true“ if the error indicates the transaction has not committed,
+	// Returns "true" if the error indicates the transaction has not committed,
 	// though in a way that can be retried.
 	ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002
 )
diff --git a/bindings/go/src/internal/gen_errors/main.go b/bindings/go/src/internal/gen_errors/main.go
new file mode 100644
index 00000000000..429403a52b6
--- /dev/null
+++ b/bindings/go/src/internal/gen_errors/main.go
@@ -0,0 +1,176 @@
+//go:build ignore
+// +build ignore
+
+/*
+ * main.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// gen_errors reads flow/include/flow/error_definitions.h and generates a Go
+// source file containing integer constants for every FDB error code.
+//
+// Usage:
+//
+//	go run internal/gen_errors/main.go -in <input.h> -out <output.go>
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"text/template"
+	"unicode"
+)
+
+var errorRe = regexp.MustCompile(`^\s*ERROR\(\s*(\w+)\s*,\s*(\d+)\s*,\s*"([^"]+)"\s*\)`)
+
+// initialism maps lower-case words to their expected go words. Most of them will be upper-case with TLog being an exception.
+var initialism = map[string]string{
+	"api":  "API",
+	"db":   "DB",
+	"dns":  "DNS",
+	"grpc": "GRPC",
+	"grv":  "GRV",
+	"http": "HTTP",
+	"id":   "ID",
+	"ids":  "IDs",
+	"io":   "IO",
+	"ip":   "IP",
+	"json": "JSON",
+	"kms":  "KMS",
+	"kv":   "KV",
+	"ssl":  "SSL",
+	"tlog": "TLog",
+	"tls":  "TLS",
+	"ttl":  "TTL",
+	"uid":  "UID",
+	"uri":  "URI",
+	"url":  "URL",
+	"uuid": "UUID",
+}
+
+type errorDef struct {
+	Name        string // exported Go constant name, e.g. ErrCodeTransactionTooOld
+	Code        string // numeric string, e.g. "1007"
+	Description string // human-readable description from the macro
+}
+
+func toPascalCase(s string) string {
+	parts := strings.Split(s, "_")
+	var b strings.Builder
+	for _, p := range parts {
+		if len(p) == 0 {
+			continue
+		}
+		if upper, ok := initialism[strings.ToLower(p)]; ok {
+			b.WriteString(upper)
+			continue
+		}
+		runes := []rune(p)
+		b.WriteRune(unicode.ToUpper(runes[0]))
+		b.WriteString(string(runes[1:]))
+	}
+	return b.String()
+}
+
+func main() {
+	var inFile string
+	var outFile string
+	flag.StringVar(&inFile, "in", "stdin", "Input file")
+	flag.StringVar(&outFile, "out", "stdout", "Output file")
+	flag.Parse()
+
+	f, err := os.Open(inFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "open %s: %v\n", inFile, err)
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	var defs []errorDef
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		m := errorRe.FindStringSubmatch(scanner.Text())
+		if m == nil {
+			continue
+		}
+		defs = append(defs, errorDef{
+			Name:        "Err" + toPascalCase(m[1]),
+			Code:        m[2],
+			Description: m[3],
+		})
+	}
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "scan: %v\n", err)
+		os.Exit(1)
+	}
+
+	out, err := os.Create(outFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "create %s: %v\n", outFile, err)
+		os.Exit(1)
+	}
+	defer out.Close()
+
+	if err := tmpl.Execute(out, defs); err != nil {
+		fmt.Fprintf(os.Stderr, "template: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+var tmpl = template.Must(template.New("").Parse(`// Code generated by gen_errors. DO NOT EDIT.
+
+/*
+ * error_codes_generated.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fdb
+
+// To regenerate this file run this command:
+// go run ./bindings/go/src/internal/gen_errors/main.go -in ./flow/include/flow/error_definitions.h -out ./bindings/go/src/fdb/error_codes_generated.go
+//
+// Error sentinel variables derived from flow/include/flow/error_definitions.h.
+// Use with errors.Is to match a specific FoundationDB error code:
+//
+//	errors.Is(err, fdb.ErrTransactionTooOld)
+var (
+{{- range .}}
+	// {{.Description}}
+	{{.Name}} = Error{Code: {{.Code}}}
+{{- end}}
+)
+`))
diff --git a/bindings/go/src/_util/translate_fdb_options.go b/bindings/go/src/internal/translate_fdb_options.go
similarity index 92%
rename from bindings/go/src/_util/translate_fdb_options.go
rename to bindings/go/src/internal/translate_fdb_options.go
index f62d9493aec..58b6687837c 100644
--- a/bindings/go/src/_util/translate_fdb_options.go
+++ b/bindings/go/src/internal/translate_fdb_options.go
@@ -42,14 +42,23 @@ type Option struct {
 	Description string `xml:"description,attr"`
 	Hidden      bool   `xml:"hidden,attr"`
 }
+
 type Scope struct {
 	Name   string `xml:"name,attr"`
 	Option []Option
 }
+
 type Options struct {
 	Scope []Scope
 }
 
+// sanitize normalizes description text to be gofmt-compliant: replaces RST
+// double-backtick spans with Go-style double-quoted strings and removes
+// trailing whitespace.
+func sanitize(s string) string {
+	return strings.TrimRight(strings.ReplaceAll(s, "``", "\""), " \t")
+}
+
 func writeOptString(w io.Writer, receiver string, function string, opt Option) {
 	fmt.Fprintf(w, `func (o %s) %s(param string) error {
 	return o.setOpt(%d, []byte(param))
@@ -84,9 +93,9 @@ func writeOpt(w io.Writer, receiver string, opt Option) {
 	fmt.Fprintln(w)
 
 	if opt.Description != "" {
-		fmt.Fprintf(w, "// %s\n", opt.Description)
+		fmt.Fprintf(w, "// %s\n", sanitize(opt.Description))
 		if opt.ParamDesc != "" {
-			fmt.Fprintf(w, "//\n// Parameter: %s\n", opt.ParamDesc)
+			fmt.Fprintf(w, "//\n// Parameter: %s\n", sanitize(opt.ParamDesc))
 		}
 	} else {
 		fmt.Fprintf(w, "// Not yet implemented.\n")
@@ -117,14 +126,13 @@ func writeMutation(w io.Writer, opt Option) {
 func (t Transaction) %s(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, %d)
 }
-`, opt.Description, tname, opt.Code)
+`, sanitize(opt.Description), tname, opt.Code)
 }
 
 func writeEnum(w io.Writer, scope Scope, opt Option, delta int) {
 	fmt.Fprintln(w)
 	if opt.Description != "" {
-		doc.ToText(w, opt.Description, "\t// ", "", 73)
-		// fmt.Printf("	// %s\n", opt.Description)
+		doc.ToText(w, sanitize(opt.Description), "\t// ", "", 73)
 	}
 	fmt.Fprintf(w, "	%s %s = %d\n", scope.Name+translateName(opt.Name), scope.Name, opt.Code+delta)
 }
diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp
index c14d6e2ee66..a10a8d0c0bc 100644
--- a/bindings/java/JavaWorkload.cpp
+++ b/bindings/java/JavaWorkload.cpp
@@ -90,17 +90,17 @@ void printTrace(JNIEnv* env, jclass, jlong logger, jint severity, jstring messag
 }
 
 jlong getProcessID(JNIEnv* env, jclass, jlong self) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	return jlong(context->getProcessID());
 }
 
 void setProcessID(JNIEnv* env, jclass, jlong self, jlong processID) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	context->setProcessID(processID);
 }
 
 jboolean getOptionBool(JNIEnv* env, jclass, jlong self, jstring name, jboolean defaultValue) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	jboolean isCopy = true;
 	const char* utf = env->GetStringUTFChars(name, &isCopy);
 	auto res = jboolean(context->getOption(utf, bool(defaultValue)));
@@ -111,7 +111,7 @@ jboolean getOptionBool(JNIEnv* env, jclass, jlong self, jstring name, jboolean d
 }
 
 jlong getOptionLong(JNIEnv* env, jclass, jlong self, jstring name, jlong defaultValue) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	jboolean isCopy = true;
 	const char* utf = env->GetStringUTFChars(name, &isCopy);
 	auto res = jlong(context->getOption(utf, long(defaultValue)));
@@ -122,7 +122,7 @@ jlong getOptionLong(JNIEnv* env, jclass, jlong self, jstring name, jlong default
 }
 
 jdouble getOptionDouble(JNIEnv* env, jclass, jlong self, jstring name, jdouble defaultValue) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	jboolean isCopy = true;
 	const char* utf = env->GetStringUTFChars(name, &isCopy);
 	auto res = jdouble(context->getOption(utf, double(defaultValue)));
@@ -133,7 +133,7 @@ jdouble getOptionDouble(JNIEnv* env, jclass, jlong self, jstring name, jdouble d
 }
 
 jstring getOptionString(JNIEnv* env, jclass, jlong self, jstring name, jstring defaultValue) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	jboolean isCopy;
 	jboolean defIsCopy;
 	const char* nameStr = env->GetStringUTFChars(name, &isCopy);
@@ -149,17 +149,17 @@ jstring getOptionString(JNIEnv* env, jclass, jlong self, jstring name, jstring d
 }
 
 jint getClientID(JNIEnv* env, jclass, jlong self) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	return jint(context->clientId());
 }
 
 jint getClientCount(JNIEnv* env, jclass, jlong self) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	return jint(context->clientCount());
 }
 
 jlong getSharedRandomNumber(JNIEnv* env, jclass, jlong self) {
-	FDBWorkloadContext* context = reinterpret_cast<FDBWorkloadContext*>(self);
+	auto* context = reinterpret_cast<FDBWorkloadContext*>(self);
 	return jlong(context->sharedRandomNumber());
 }
 
@@ -324,7 +324,7 @@ struct JVM {
 		if (!env) {
 			throw JNIError{};
 		}
-		if (classPath.count(path) > 0) {
+		if (classPath.contains(path)) {
 			// already added
 			return;
 		}
diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp
index 81e4f391161..451cf8d1555 100644
--- a/bindings/java/fdbJNI.cpp
+++ b/bindings/java/fdbJNI.cpp
@@ -448,7 +448,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyRangeArray_Future
 		return JNI_NULL;
 	}
 
-	jobjectArray kr_values = jenv->NewObjectArray(count, keyrange_class, NULL);
+	jobjectArray kr_values = jenv->NewObjectArray(count, keyrange_class, nullptr);
 	if (!kr_values) {
 		if (!jenv->ExceptionOccurred())
 			throwOutOfMem(jenv);
@@ -600,7 +600,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 		return JNI_NULL;
 	}
 
-	jobjectArray mrr_values = jenv->NewObjectArray(count, mapped_key_value_class, NULL);
+	jobjectArray mrr_values = jenv->NewObjectArray(count, mapped_key_value_class, nullptr);
 	if (!mrr_values) {
 		if (!jenv->ExceptionOccurred())
 			throwOutOfMem(jenv);
@@ -753,7 +753,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1create
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBDatabase* database = (FDBDatabase*)dbPtr;
+	auto* database = (FDBDatabase*)dbPtr;
 	FDBTransaction* tr;
 	fdb_error_t err = fdb_database_create_transaction(database, &tr);
 	if (err) {
@@ -780,12 +780,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOpti
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBDatabase* c = (FDBDatabase*)dPtr;
+	auto* c = (FDBDatabase*)dPtr;
 	uint8_t* barr = nullptr;
 	int size = 0;
 
 	if (value != JNI_NULL) {
-		barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL);
+		barr = (decltype(barr))jenv->GetByteArrayElements(value, JNI_NULL);
 		if (!barr) {
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 			return;
@@ -810,7 +810,7 @@ JNIEXPORT jdouble JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getM
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBDatabase* database = (FDBDatabase*)dbPtr;
+	auto* database = (FDBDatabase*)dbPtr;
 	return (jdouble)fdb_database_get_main_thread_busyness(database);
 }
 
@@ -822,7 +822,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getCli
 		return 0;
 	}
 
-	FDBDatabase* database = (FDBDatabase*)dbPtr;
+	auto* database = (FDBDatabase*)dbPtr;
 
 	FDBFuture* f = fdb_database_get_client_status(database);
 	return (jlong)f;
@@ -869,7 +869,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	fdb_transaction_set_read_version(tr, version);
 }
 
@@ -880,7 +880,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	FDBFuture* f = fdb_transaction_get_read_version(tr);
 	return (jlong)f;
 }
@@ -894,9 +894,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
+	auto* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
 	if (!barr) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -919,9 +919,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
+	auto* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
 	if (!barr) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -953,16 +953,16 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL);
+	auto* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL);
 	if (!barrBegin) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return 0;
 	}
 
-	uint8_t* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL);
+	auto* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL);
 	if (!barrEnd) {
 		jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1010,16 +1010,16 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL);
+	auto* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL);
 	if (!barrBegin) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return 0;
 	}
 
-	uint8_t* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL);
+	auto* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL);
 	if (!barrEnd) {
 		jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1027,7 +1027,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		return 0;
 	}
 
-	uint8_t* barrMapper = (uint8_t*)jenv->GetByteArrayElements(mapperBytes, JNI_NULL);
+	auto* barrMapper = (uint8_t*)jenv->GetByteArrayElements(mapperBytes, JNI_NULL);
 	if (!barrMapper) {
 		jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT);
 		jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT);
@@ -1076,7 +1076,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1
 		return;
 	}
 
-	FDBFuture* f = (FDBFuture*)future;
+	auto* f = (FDBFuture*)future;
 	const FDBKeyValue* kvs;
 	int count;
 	fdb_bool_t more;
@@ -1153,7 +1153,7 @@ Java_com_apple_foundationdb_FutureMappedResults_FutureMappedResults_1getDirect(J
 		return;
 	}
 
-	FDBFuture* f = (FDBFuture*)future;
+	auto* f = (FDBFuture*)future;
 	const FDBMappedKeyValue* kvms;
 	int count;
 	fdb_bool_t more;
@@ -1219,16 +1219,16 @@ Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeByt
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	auto* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
 	if (!startKey) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return 0;
 	}
 
-	uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	auto* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
 	if (!endKey) {
 		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1254,16 +1254,16 @@ Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIE
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	auto* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
 	if (!startKey) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return 0;
 	}
 
-	uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	auto* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
 	if (!endKey) {
 		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1287,16 +1287,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
+	auto* barrKey = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
 	if (!barrKey) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return;
 	}
 
-	uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(valueBytes, JNI_NULL);
+	auto* barrValue = (uint8_t*)jenv->GetByteArrayElements(valueBytes, JNI_NULL);
 	if (!barrValue) {
 		jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1317,9 +1317,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
+	auto* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL);
 	if (!barr) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -1339,16 +1339,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barrKeyBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL);
+	auto* barrKeyBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL);
 	if (!barrKeyBegin) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return;
 	}
 
-	uint8_t* barrKeyEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL);
+	auto* barrKeyEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL);
 	if (!barrKeyEnd) {
 		jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1372,16 +1372,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1m
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL);
+	auto* barrKey = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL);
 	if (!barrKey) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
 		return;
 	}
 
-	uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL);
+	auto* barrValue = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL);
 	if (!barrValue) {
 		jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
@@ -1403,7 +1403,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	FDBFuture* f = fdb_transaction_commit(tr);
 	return (jlong)f;
 }
@@ -1417,12 +1417,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	uint8_t* barr = nullptr;
 	int size = 0;
 
 	if (value != JNI_NULL) {
-		barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL);
+		barr = (decltype(barr))jenv->GetByteArrayElements(value, JNI_NULL);
 		if (!barr) {
 			if (!jenv->ExceptionOccurred())
 				throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -1445,7 +1445,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	int64_t version;
 	fdb_error_t err = fdb_transaction_get_committed_version(tr, &version);
 	if (err) {
@@ -1462,7 +1462,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBFuture* f = fdb_transaction_get_approximate_size((FDBTransaction*)tPtr);
+	auto* f = fdb_transaction_get_approximate_size((FDBTransaction*)tPtr);
 	return (jlong)f;
 }
 
@@ -1473,7 +1473,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	FDBFuture* f = fdb_transaction_get_versionstamp(tr);
 	return (jlong)f;
 }
@@ -1486,9 +1486,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL);
+	auto* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL);
 	if (!barr) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -1510,7 +1510,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 	FDBFuture* f = fdb_transaction_on_error(tr, (fdb_error_t)errorCode);
 	return (jlong)f;
 }
@@ -1543,9 +1543,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1
 		throwParamNotNull(jenv);
 		return 0;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL);
+	auto* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL);
 	if (!barr) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -1578,9 +1578,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a
 		throwParamNotNull(jenv);
 		return;
 	}
-	FDBTransaction* tr = (FDBTransaction*)tPtr;
+	auto* tr = (FDBTransaction*)tPtr;
 
-	uint8_t* begin_barr = (uint8_t*)jenv->GetByteArrayElements(keyBegin, JNI_NULL);
+	auto* begin_barr = (uint8_t*)jenv->GetByteArrayElements(keyBegin, JNI_NULL);
 	if (!begin_barr) {
 		if (!jenv->ExceptionOccurred())
 			throwRuntimeEx(jenv, "Error getting handle to native resources");
@@ -1588,7 +1588,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a
 	}
 	int begin_size = jenv->GetArrayLength(keyBegin);
 
-	uint8_t* end_barr = (uint8_t*)jenv->GetByteArrayElements(keyEnd, JNI_NULL);
+	auto* end_barr = (uint8_t*)jenv->GetByteArrayElements(keyEnd, JNI_NULL);
 	if (!end_barr) {
 		jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT);
 		if (!jenv->ExceptionOccurred())
diff --git a/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
index 437a194ce13..e9881719e3f 100644
--- a/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
+++ b/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java
@@ -152,74 +152,67 @@ public interface RangeQueryWithIndex {
 	}
 
 	RangeQueryWithIndex rangeQueryAndThenRangeQueries = (int begin, int end, Database db) -> db.run(tr -> {
-		try {
-			List<KeyValue> kvs = tr.getRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
-			                                 KeySelector.firstGreaterOrEqual(indexEntryKey(end)),
-			                                 ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL)
-			                         .asList()
-			                         .get();
-			Assertions.assertEquals(end - begin, kvs.size());
-
-			// Get the records of each index entry IN PARALLEL.
-			List<CompletableFuture<List<KeyValue>>> resultFutures = new ArrayList<>();
-			// In reality, we need to get the record key by parsing the index entry key. But considering this is a
-			// performance test, we just ignore the returned key and simply generate it from recordKey.
+		// Let Database.run retry transient FDB errors instead of turning them into assertion failures.
+		List<KeyValue> kvs = tr.getRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
+		                                 KeySelector.firstGreaterOrEqual(indexEntryKey(end)),
+		                                 ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL)
+		                         .asList()
+		                         .join();
+		Assertions.assertEquals(end - begin, kvs.size());
+
+		// Get the records of each index entry IN PARALLEL.
+		List<CompletableFuture<List<KeyValue>>> resultFutures = new ArrayList<>();
+		// In reality, we need to get the record key by parsing the index entry key. But considering this is a
+		// performance test, we just ignore the returned key and simply generate it from recordKey.
+		for (int id = begin; id < end; id++) {
+			resultFutures.add(tr.getRange(Range.startsWith(recordKeyPrefix(id)),
+			                              ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL).asList());
+		}
+		AsyncUtil.whenAll(resultFutures).join();
+
+		if (validate) {
+			final Iterator<KeyValue> indexes = kvs.iterator();
+			final Iterator<CompletableFuture<List<KeyValue>>> records = resultFutures.iterator();
 			for (int id = begin; id < end; id++) {
-				resultFutures.add(tr.getRange(Range.startsWith(recordKeyPrefix(id)),
-				                              ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL).asList());
-			}
-			AsyncUtil.whenAll(resultFutures).get();
-
-			if (validate) {
-				final Iterator<KeyValue> indexes = kvs.iterator();
-				final Iterator<CompletableFuture<List<KeyValue>>> records = resultFutures.iterator();
-				for (int id = begin; id < end; id++) {
-					Assertions.assertTrue(indexes.hasNext());
-					assertByteArrayEquals(indexEntryKey(id), indexes.next().getKey());
-
-					Assertions.assertTrue(records.hasNext());
-					List<KeyValue> rangeResult = records.next().get();
-					validateRangeResult(id, rangeResult);
-				}
-				Assertions.assertFalse(indexes.hasNext());
-				Assertions.assertFalse(records.hasNext());
+				Assertions.assertTrue(indexes.hasNext());
+				assertByteArrayEquals(indexEntryKey(id), indexes.next().getKey());
+
+				Assertions.assertTrue(records.hasNext());
+				List<KeyValue> rangeResult = records.next().join();
+				validateRangeResult(id, rangeResult);
 			}
-		} catch (Exception e) {
-			Assertions.fail("Unexpected exception", e);
+			Assertions.assertFalse(indexes.hasNext());
+			Assertions.assertFalse(records.hasNext());
 		}
 		return null;
 	});
 
 	RangeQueryWithIndex mappedRangeQuery = (int begin, int end, Database db) -> db.run(tr -> {
-		try {
-			List<MappedKeyValue> kvs =
-			    tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
-			                      KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER,
-			                      ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL)
-			        .asList()
-			        .get();
-			Assertions.assertEquals(end - begin, kvs.size());
-
-			if (validate) {
-				final Iterator<MappedKeyValue> results = kvs.iterator();
-				for (int id = begin; id < end; id++) {
-					Assertions.assertTrue(results.hasNext());
-					MappedKeyValue mappedKeyValue = results.next();
-					assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
-					assertByteArrayEquals(EMPTY, mappedKeyValue.getValue());
-					assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
-					byte[] prefix = recordKeyPrefix(id);
-					assertByteArrayEquals(prefix, mappedKeyValue.getRangeBegin());
-					prefix[prefix.length - 1] = (byte)0x01;
-					assertByteArrayEquals(prefix, mappedKeyValue.getRangeEnd());
-
-					List<KeyValue> rangeResult = mappedKeyValue.getRangeResult();
-					validateRangeResult(id, rangeResult);
-				}
-				Assertions.assertFalse(results.hasNext());
+		List<MappedKeyValue> kvs =
+		    tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)),
+		                      KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER,
+		                      ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL)
+		        .asList()
+		        .join();
+		Assertions.assertEquals(end - begin, kvs.size());
+
+		if (validate) {
+			final Iterator<MappedKeyValue> results = kvs.iterator();
+			for (int id = begin; id < end; id++) {
+				Assertions.assertTrue(results.hasNext());
+				MappedKeyValue mappedKeyValue = results.next();
+				assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
+				assertByteArrayEquals(EMPTY, mappedKeyValue.getValue());
+				assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey());
+				byte[] prefix = recordKeyPrefix(id);
+				assertByteArrayEquals(prefix, mappedKeyValue.getRangeBegin());
+				prefix[prefix.length - 1] = (byte)0x01;
+				assertByteArrayEquals(prefix, mappedKeyValue.getRangeEnd());
+
+				List<KeyValue> rangeResult = mappedKeyValue.getRangeResult();
+				validateRangeResult(id, rangeResult);
 			}
-		} catch (Exception e) {
-			Assertions.fail("Unexpected exception", e);
+			Assertions.assertFalse(results.hasNext());
 		}
 		return null;
 	});
diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
index 8cdecd172c2..28068d9c4de 100644
--- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
+++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
@@ -640,6 +640,11 @@ public CompletableFuture<Void> commit() {
 
 	@Override
 	public Long getCommittedVersion() {
+		return getCommittedVersionAsPrimitive();
+	}
+
+	@Override
+	public long getCommittedVersionAsPrimitive() {
 		if (eventKeeper != null) {
 			eventKeeper.increment(Events.JNI_CALL);
 		}
diff --git a/bindings/java/src/main/com/apple/foundationdb/Transaction.java b/bindings/java/src/main/com/apple/foundationdb/Transaction.java
index 3e07779cf99..a194e521fea 100644
--- a/bindings/java/src/main/com/apple/foundationdb/Transaction.java
+++ b/bindings/java/src/main/com/apple/foundationdb/Transaction.java
@@ -253,6 +253,24 @@ public interface Transaction extends AutoCloseable, ReadTransaction, Transaction
 	 */
 	Long getCommittedVersion();
 
+	/**
+	 * Gets the version number at which a successful commit modified the database,
+	 * as a primitive {@code long} to avoid the overhead of autoboxing.
+	 * This must be called only after the successful (non-error) completion of a call
+	 * to {@link #commit()} on this {@code Transaction}, or the behavior is undefined.
+	 * Read-only transactions do not modify the database when committed and will have
+	 * a committed version of -1. Keep in mind that a transaction which reads keys and
+	 * then sets them to their current values may be optimized to a read-only transaction.
+	 *
+	 * <p>This method is equivalent to {@link #getCommittedVersion()} but avoids the
+	 * allocation of a {@link Long} object on every call.</p>
+	 *
+	 * @return the database version at which the commit succeeded, as a primitive {@code long}
+	 */
+	default long getCommittedVersionAsPrimitive() {
+		return getCommittedVersion();
+	}
+
 	/**
 	 * Returns a future which will contain the versionstamp which was used by any versionstamp 
 	 * operations in this transaction. The future will be ready only after the successful 
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java
index 2eb9d674f98..438a26cb474 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java
@@ -65,7 +65,6 @@ public void apply(Database db) {
 				System.out.println(" sem: " + semaphore);
 				System.out.println();
 			} catch (InterruptedException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java
index 6a53f871b9b..20f7890add7 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java
@@ -62,7 +62,6 @@ private static double runThreadedTest(Database database, int threadCount) {
 				t.join();
 				rowsPerSecond += t.getAverageRowsPerSecond();
 			} catch (InterruptedException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}
@@ -90,7 +89,6 @@ public void run() {
 			try {
 				Thread.sleep(r.nextInt(1000));
 			} catch (InterruptedException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 			long totalTime = 0;
diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java
index a28b78a86fa..23dcca2c0af 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java
@@ -104,7 +104,6 @@ public static void raceTest(Database db) {
 						Thread.sleep(1);
 					}
 					catch(InterruptedException e1) {
-						// TODO Auto-generated catch block
 						e1.printStackTrace();
 					}
 				}
diff --git a/cmake/AddFdbServerLinkTest.cmake b/cmake/AddFdbServerLinkTest.cmake
new file mode 100644
index 00000000000..71ab3b81558
--- /dev/null
+++ b/cmake/AddFdbServerLinkTest.cmake
@@ -0,0 +1,8 @@
+function(add_fdbserver_link_test target_name)
+  set(link_libs ${ARGN})
+  list(GET link_libs 0 primary_lib)
+  list(REMOVE_AT link_libs 0)
+  file(RELATIVE_PATH link_test_src "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/fdbclient/LinkTest.cpp")
+  add_flow_target(LINK_TEST NAME ${target_name} SRCS ${link_test_src})
+  target_link_libraries(${target_name} PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,${primary_lib}>" ${link_libs} rapidxml)
+endfunction()
diff --git a/cmake/CompileBoost.cmake b/cmake/CompileBoost.cmake
index 925bae405f8..98b3b3084e2 100644
--- a/cmake/CompileBoost.cmake
+++ b/cmake/CompileBoost.cmake
@@ -38,10 +38,6 @@ function(compile_boost)
   set(BOOST_LINK_FLAGS "")
   if(APPLE OR ICX OR USE_LIBCXX)
     list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
-    if (APPLE)
-      # Remove this after boost 1.81 or above is used
-      list(APPEND BOOST_COMPILER_FLAGS -D_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION)
-    endif()
     list(APPEND BOOST_LINK_FLAGS -lc++ -lc++abi)
     if (NOT APPLE)
       list(APPEND BOOST_LINK_FLAGS -static-libgcc)
@@ -164,11 +160,10 @@ set(Boost_USE_STATIC_LIBS ON)
 if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$" AND USE_LIBCXX)
   list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_86_0_clang)
   set(BOOST_HINT_PATHS /opt/boost_1_86_0_clang)
-  message(STATUS "Using Clang version of boost")
+  message(STATUS "Preferring _clang build of boost ...")
 else ()
   list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_86_0)
   set(BOOST_HINT_PATHS /opt/boost_1_86_0)
-  message(STATUS "Using g++ version of boost")
 endif ()
 
 if(BOOST_ROOT)
@@ -178,10 +173,18 @@ endif()
 if(WIN32)
   # Use CONFIG mode to prefer Boost's BoostConfig.cmake over deprecated FindBoost module
   # This is required for CMake 3.30+ compatibility (policy CMP0167)
-  find_package(Boost 1.86.0 QUIET COMPONENTS filesystem iostreams serialization system program_options url CONFIG PATHS ${BOOST_HINT_PATHS})
+  find_package(Boost 1.86.0 QUIET
+               COMPONENTS filesystem iostreams serialization system program_options url
+               CONFIG PATHS ${BOOST_HINT_PATHS})
   add_library(boost_target INTERFACE)
-  target_link_libraries(boost_target INTERFACE Boost::boost Boost::filesystem Boost::iostreams Boost::serialization Boost::system Boost::url)
-  
+  target_link_libraries(boost_target
+                        INTERFACE Boost::boost
+                                  Boost::filesystem
+                                  Boost::iostreams
+                                  Boost::serialization
+                                  Boost::system
+                                  Boost::url
+  )
   # Add zstd library since boost::iostreams may depend on it
   find_package(PkgConfig QUIET)
   if(PkgConfig_FOUND)
@@ -191,15 +194,17 @@ if(WIN32)
       target_link_directories(boost_target INTERFACE ${ZSTD_LIBRARY_DIRS})
     endif()
   endif()
-
   add_library(boost_target_program_options INTERFACE)
   target_link_libraries(boost_target_program_options INTERFACE Boost::boost Boost::program_options)
   return()
 endif()
 
-find_package(Boost 1.86.0 EXACT QUIET COMPONENTS context filesystem iostreams program_options serialization system url CONFIG PATHS ${BOOST_HINT_PATHS})
-set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost")
 
+find_package(Boost 1.86.0 EXACT QUIET
+             COMPONENTS context filesystem iostreams program_options serialization system url
+             CONFIG PATHS ${BOOST_HINT_PATHS})
+
+set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost")
 # The precompiled boost silently broke in CI.  While investigating, I considered extending
 # the old check with something like this, so that it would fail loudly if it found a bad
 # pre-existing boost.  It turns out the error messages we get from CMake explain what is
@@ -214,9 +219,17 @@ set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores an
 #      message(FATAL_ERROR "Unacceptable precompiled boost found")
 #
 if(Boost_FOUND AND NOT FORCE_BOOST_BUILD)
+  message(STATUS "Found Boost: ${Boost_DIR}")
   add_library(boost_target INTERFACE)
-  target_link_libraries(boost_target INTERFACE Boost::boost Boost::context Boost::filesystem Boost::iostreams Boost::serialization Boost::system Boost::url)
-  
+  target_link_libraries(boost_target
+                        INTERFACE Boost::boost
+                                  Boost::context
+                                  Boost::filesystem
+                                  Boost::iostreams
+                                  Boost::serialization
+                                  Boost::system
+                                  Boost::url
+  )
   # Add zstd library since boost::iostreams may depend on it when compiled with homebrew on macOS
   if(APPLE)
     find_package(PkgConfig QUIET)
@@ -228,7 +241,6 @@ if(Boost_FOUND AND NOT FORCE_BOOST_BUILD)
       endif()
     endif()
   endif()
-
   add_library(boost_target_program_options INTERFACE)
   target_link_libraries(boost_target_program_options INTERFACE Boost::boost Boost::program_options)
 elseif(WIN32)
diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 99ae2581a3c..c50b1dacef6 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -64,18 +64,25 @@ else()
   set(FDB_ROCKSDB_GIT_HASH "")
 endif()
 
-# Generate FDBRocksDBVersion.h from template into build directory
+# Generate FDBRocksDBVersion.h from template into the core build include tree
 # This file is NOT source-controlled - it's regenerated at cmake configure time
-file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/fdbserver/include/fdbserver)
+file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/fdbserver/core/include/fdbserver/core)
 configure_file(
   ${CMAKE_CURRENT_LIST_DIR}/FDBRocksDBVersion.h.in
-  ${CMAKE_BINARY_DIR}/fdbserver/include/fdbserver/FDBRocksDBVersion.h
+  ${CMAKE_BINARY_DIR}/fdbserver/core/include/fdbserver/core/FDBRocksDBVersion.h
   @ONLY)
 message(STATUS "Generated FDBRocksDBVersion.h with version ${FDB_ROCKSDB_MAJOR}.${FDB_ROCKSDB_MINOR}.${FDB_ROCKSDB_PATCH}")
 
 # Only try to find system RocksDB if not using a specific commit hash
 if(NOT ROCKSDB_GIT_HASH)
+  # Save the configured version - FindRocksDB.cmake may overwrite ROCKSDB_VERSION
+  # with the system version even when find_package fails
+  set(_FDB_ROCKSDB_VERSION_CONFIGURED ${ROCKSDB_VERSION})
   find_package(RocksDB ${ROCKSDB_VERSION})
+  if(NOT ROCKSDB_FOUND)
+    # Restore configured version since FindRocksDB overwrote it
+    set(ROCKSDB_VERSION ${_FDB_ROCKSDB_VERSION_CONFIGURED})
+  endif()
 endif()
 
 include(ExternalProject)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 321ac4a22a9..11a1d0b3062 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -13,6 +13,9 @@ env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer")
 env_set(FDB_RELEASE_CANDIDATE OFF BOOL "This is a building of a release candidate")
 env_set(FDB_RELEASE OFF BOOL "This is a building of a final release")
 env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available")
+env_set(USE_CLANG_TIDY OFF BOOL "Run clang-tidy during C/C++ compilation")
+env_set(CLANG_TIDY "" STRING "Path to clang-tidy executable (empty to auto-detect)")
+env_set(CLANG_TIDY_EXTRA_ARGS "" STRING "Additional clang-tidy arguments (space-separated)")
 env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info")
 env_set(USE_WERROR OFF BOOL "Compile with -Werror. Recommended for local development and CI.")
 default_linker(_use_ld)
@@ -102,6 +105,33 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 
+if(USE_CLANG_TIDY)
+  if(CLANG_TIDY)
+    set(_clang_tidy_program "${CLANG_TIDY}")
+  else()
+    find_program(_clang_tidy_program NAMES clang-tidy)
+  endif()
+
+  if(NOT _clang_tidy_program)
+    message(
+      FATAL_ERROR
+      "USE_CLANG_TIDY is enabled, but clang-tidy was not found. Install clang-tidy or set CLANG_TIDY=/path/to/clang-tidy.")
+  endif()
+
+  set(_clang_tidy_command "${_clang_tidy_program}")
+  list(APPEND _clang_tidy_command "--config-file=${CMAKE_SOURCE_DIR}/.clang-tidy")
+  if(CLANG_TIDY_EXTRA_ARGS)
+    separate_arguments(_clang_tidy_extra_args NATIVE_COMMAND "${CLANG_TIDY_EXTRA_ARGS}")
+    list(APPEND _clang_tidy_command ${_clang_tidy_extra_args})
+  endif()
+
+  set(CMAKE_C_CLANG_TIDY ${_clang_tidy_command})
+  set(CMAKE_CXX_CLANG_TIDY ${_clang_tidy_command})
+
+  string(REPLACE ";" " " _clang_tidy_command_display "${_clang_tidy_command}")
+  message(STATUS "clang-tidy enabled for C/C++ compilation: ${_clang_tidy_command_display}")
+endif()
+
 if(NOT OPEN_FOR_IDE)
   add_compile_definitions(NO_INTELLISENSE)
 endif()
diff --git a/cmake/FDBBenchmark.cmake b/cmake/FDBBenchmark.cmake
new file mode 100644
index 00000000000..505ca64892b
--- /dev/null
+++ b/cmake/FDBBenchmark.cmake
@@ -0,0 +1,61 @@
+function(fdb_setup_googlebenchmark)
+  if(TARGET fdb_google_benchmark)
+    return()
+  endif()
+
+  if(NOT benchmark_ROOT)
+    if(EXISTS /opt/googlebenchmark-f91b6b AND CLANG AND USE_LIBCXX)
+      set(benchmark_ROOT /opt/googlebenchmark-f91b6b)
+    elseif(EXISTS /opt/googlebenchmark-f91b6b-g++ AND NOT USE_LIBCXX)
+      set(benchmark_ROOT /opt/googlebenchmark-f91b6b-g++)
+    endif()
+  endif()
+
+  find_package(benchmark)
+
+  add_library(fdb_google_benchmark INTERFACE)
+
+  if(benchmark_FOUND)
+    target_link_libraries(fdb_google_benchmark INTERFACE benchmark::benchmark)
+    target_include_directories(
+      fdb_google_benchmark
+      INTERFACE $<TARGET_PROPERTY:benchmark::benchmark,INTERFACE_INCLUDE_DIRECTORIES>)
+    return()
+  endif()
+
+  if(NOT TARGET benchmark)
+    set(googlebenchmark_root ${CMAKE_BINARY_DIR}/googlebenchmark-download)
+
+    configure_file(
+      ${CMAKE_SOURCE_DIR}/cmake/benchmark-download.cmake
+      ${googlebenchmark_root}/CMakeLists.txt
+      COPYONLY)
+
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+      RESULT_VARIABLE results
+      WORKING_DIRECTORY ${googlebenchmark_root})
+    if(results)
+      message(FATAL_ERROR "Configuration step for Benchmark has Failed. ${results}")
+    endif()
+
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} --build . --config Release
+      RESULT_VARIABLE results
+      WORKING_DIRECTORY ${googlebenchmark_root})
+    if(results)
+      message(FATAL_ERROR "Build step for Benchmark has Failed. ${results}")
+    endif()
+
+    set(BENCHMARK_ENABLE_TESTING OFF)
+    add_subdirectory(
+      ${googlebenchmark_root}/googlebenchmark-src
+      ${googlebenchmark_root}/googlebenchmark-build
+      EXCLUDE_FROM_ALL)
+  endif()
+
+  target_include_directories(
+    fdb_google_benchmark
+    INTERFACE ${CMAKE_BINARY_DIR}/googlebenchmark-download/googlebenchmark-src/include)
+  target_link_libraries(fdb_google_benchmark INTERFACE benchmark)
+endfunction()
diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake
index 59023fc03f4..8a2c7ea1bc1 100644
--- a/cmake/FDBComponents.cmake
+++ b/cmake/FDBComponents.cmake
@@ -159,7 +159,7 @@ if(BUILD_SWIFT_BINDING AND NOT WITH_C_BINDING)
   message(WARNING "Swift binding depends on C binding, but C binding is not enabled")
 endif()
 
-if(NOT BUILD_SWIFT_BINDING OR NOT BUILD_C_BINDING OR OPEN_FOR_IDE)
+if(NOT BUILD_SWIFT_BINDING OR NOT BUILD_C_BINDING OR OPEN_FOR_IDE OR NOT WITH_SWIFT)
   set(WITH_SWIFT_BINDING OFF)
 else()
   find_program(SWIFT_EXECUTABLE swift)
@@ -262,28 +262,15 @@ set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON
 # TOML11
 ################################################################################
 
-# TOML can download and install itself into the binary directory, so it should
-# always be available.
-find_package(toml11 3.4.0)
-if(toml11_FOUND)
-  message(STATUS "Using TOML11 from system")
-  add_library(toml11_target INTERFACE)
-  target_link_libraries(toml11_target INTERFACE toml11)
-else()
-  include(ExternalProject)
-  ExternalProject_add(toml11Project
-    URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz"
-    URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d
-    CMAKE_CACHE_ARGS
-      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
-      -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
-      -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
-      -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11
-      -Dtoml11_BUILD_TEST:BOOL=OFF
-    BUILD_ALWAYS ON)
-  add_library(toml11_target INTERFACE)
-  add_dependencies(toml11_target toml11Project)
-  target_include_directories(toml11_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/toml11/include)
+find_package(toml11 3.8.1 EXACT CONFIG)
+if(NOT toml11_FOUND)
+  include(FetchContent)
+  FetchContent_Declare(
+    toml11
+    URL "https://github.com/ToruNiina/toml11/archive/v3.8.1.tar.gz"
+    URL_HASH SHA256=6a3d20080ecca5ea42102c078d3415bef80920f6c4ea2258e87572876af77849
+  )
+  FetchContent_MakeAvailable(toml11)
 endif()
 
 ################################################################################
diff --git a/cmake/FindCoroutines.cmake b/cmake/FindCoroutines.cmake
index 9391ddcaed9..b7411ed2f79 100644
--- a/cmake/FindCoroutines.cmake
+++ b/cmake/FindCoroutines.cmake
@@ -249,7 +249,7 @@ if(CXX_COROUTINES_HAVE_COROUTINES)
 
   # Try to compile a simple coroutines program without any compiler flags
   set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
-  
+
   check_cxx_source_compiles("${code}" CXX_COROUTINES_NO_AWAIT_NEEDED)
 
   set(can_link ${CXX_COROUTINES_NO_AWAIT_NEEDED})
diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake
index ecb890896af..d197dd8765d 100644
--- a/cmake/GetFmt.cmake
+++ b/cmake/GetFmt.cmake
@@ -1,30 +1,11 @@
-# Try to find fmt using pkg-config first (for homebrew installations)
-find_package(PkgConfig QUIET)
-if(PkgConfig_FOUND)
-  pkg_check_modules(FMT QUIET fmt)
-  if(FMT_FOUND)
-    # Create an imported target for fmt
-    add_library(fmt::fmt INTERFACE IMPORTED)
-    target_link_libraries(fmt::fmt INTERFACE ${FMT_LIBRARIES})
-    target_link_directories(fmt::fmt INTERFACE ${FMT_LIBRARY_DIRS})
-    target_include_directories(fmt::fmt INTERFACE ${FMT_INCLUDE_DIRS})
-    target_compile_options(fmt::fmt INTERFACE ${FMT_CFLAGS_OTHER})
-    set(fmt_FOUND TRUE)
-  endif()
-endif()
-
-# Fall back to standard find_package
-if(NOT fmt_FOUND)
-  find_package(fmt 11.0.2 EXACT QUIET CONFIG)
-endif()
+find_package(fmt 11.1.4 EXACT CONFIG)
 
-# Fall back to FetchContent if still not found
 if(NOT fmt_FOUND)
   include(FetchContent)
   FetchContent_Declare(
     fmt
     GIT_REPOSITORY https://github.com/fmtlib/fmt
-    GIT_TAG 11.0.2
+    GIT_TAG 11.1.4
   )
   FetchContent_MakeAvailable(fmt)
 endif()
diff --git a/cmake/RocksDBVersion.cmake b/cmake/RocksDBVersion.cmake
index 3b0c349640f..51f5e5d3512 100644
--- a/cmake/RocksDBVersion.cmake
+++ b/cmake/RocksDBVersion.cmake
@@ -5,7 +5,7 @@
 # Choose ONE of the two options below by uncommenting the appropriate section.
 # Do NOT set both - CMake will error if both are configured.
 #
-# CMake will automatically generate fdbserver/include/fdbserver/FDBRocksDBVersion.h
+# CMake will automatically generate fdbserver/core/include/fdbserver/core/FDBRocksDBVersion.h
 # based on the version specified here. Do NOT edit that file manually.
 #
 # To get SHA256:
diff --git a/flowbench/benchmark.cmake b/cmake/benchmark-download.cmake
similarity index 79%
rename from flowbench/benchmark.cmake
rename to cmake/benchmark-download.cmake
index 3fa69e8066b..410842e2cfd 100644
--- a/flowbench/benchmark.cmake
+++ b/cmake/benchmark-download.cmake
@@ -4,8 +4,8 @@ project(googlebenchmark-download NONE)
 include(ExternalProject)
 ExternalProject_Add(googlebenchmark
   GIT_REPOSITORY    https://github.com/google/benchmark.git
-  # If you change this, then be sure to also update the directory name (which contains this SHA) in flowbench/CMakeLists.txt,
-  # and the fdb-build-support Dockerfile (which downloads + compiles the prebuilt version)
+  # If you change this, then be sure to also update the directory name (which contains this SHA)
+  # in FDB's build environment and the prebuilt googlebenchmark package.
   GIT_TAG           f91b6b42b1b9854772a90ae9501464a161707d1e # v1.6.0
   GIT_SHALLOW       ON
   GIT_CONFIG        advice.detachedHead=false
@@ -21,8 +21,7 @@ ExternalProject_Add(googlebenchmark
 include(ExternalProject)
 ExternalProject_Add(googletest DEPENDS googlebenchmark
   GIT_REPOSITORY    https://github.com/google/googletest.git
-  # If you change this, be sure to update the perl one-liner in the part of fdb-build-support Dockerfile that downloads
-  # googlebenchmark.  That one-liner determines which SHA will be used in the pre-built version of googlebenchmark.
+  # If you change this, be sure to update the prebuilt googlebenchmark package as well.
   GIT_TAG           2fe3bd994b3189899d93f1d5a881e725e046fdc2 # release-1.8.1
   GIT_SHALLOW       ON
   GIT_CONFIG        advice.detachedHead=false
diff --git a/cmake/utils.cmake b/cmake/utils.cmake
index 6a907285028..e90811ec16b 100644
--- a/cmake/utils.cmake
+++ b/cmake/utils.cmake
@@ -110,4 +110,9 @@ function(generate_grpc_protobuf pkg_name)
       APPEND_PATH ${out_rel_path}
       PROTOS ${proto_files}
   )
+
+  get_target_property(_generated_proto_sources ${target_name} SOURCES)
+  if(_generated_proto_sources)
+    set_source_files_properties(${_generated_proto_sources} PROPERTIES SKIP_LINTING ON)
+  endif()
 endfunction()
diff --git a/contrib/Joshua/scripts/correctnessTest.sh b/contrib/Joshua/scripts/correctnessTest.sh
index 32f5cd8b3c1..54c9ad7be58 100755
--- a/contrib/Joshua/scripts/correctnessTest.sh
+++ b/contrib/Joshua/scripts/correctnessTest.sh
@@ -99,10 +99,21 @@
 #                                Maps to: --long-running
 #                                Values: true/false (default: false)
 #
+# TH_DISABLE_CODE_PROBES        Disable code probe collection and ensemble coverage checks
+#                                Maps to: --disable-code-probes
+#                                Values: true/false (default: false)
+#
 # TH_RANDOM_SEED                Force specific random seed for debugging
 #                                Maps to: --random-seed
 #                                Default: auto-generated
 #
+# TH_FDBSERVER_MEMORY           Pass --memory SIZE to fdbserver
+#                                Maps to: --fdbserver-memory
+#                                Example: 12288MiB
+#                                Default: unset
+#                                Note: if unset, TestHarness2 automatically uses
+#                                12288MiB for ASAN/UBSAN-instrumented binaries
+#
 # TH_OUTPUT_FORMAT              Output format for test results
 #                                Maps to: --output-format
 #                                Values: xml/json (default: xml)
diff --git a/contrib/TestHarness2/README.md b/contrib/TestHarness2/README.md
index f9a180558e7..5e82f8bb624 100644
--- a/contrib/TestHarness2/README.md
+++ b/contrib/TestHarness2/README.md
@@ -76,11 +76,17 @@ TestHarness2 supports environment variables for configuration. Most variables fo
 - **`TH_BUGGIFY`**: Buggify mode (`on`, `off`, or `random`, default: `random`)
 - **`TH_USE_VALGRIND`**: Run tests under valgrind (`true`/`false`, default: `false`)
 - **`TH_LONG_RUNNING`**: Enable long-running test mode (`true`/`false`, default: `false`)
+- **`TH_DISABLE_CODE_PROBES`**: Disable code probe collection and ensemble coverage checks (`true`/`false`, default: `false`)
+- **`TH_FDBSERVER_MEMORY`**: Pass `--memory SIZE` to `fdbserver` (for example `12288MiB`)
 
 **Optional Configuration:**
 - **`TH_RANDOM_SEED`**: Force specific random seed for debugging
 - **`TH_OUTPUT_FORMAT`**: Output format (`xml` or `json`, default: `xml`)
 
+When `TH_FDBSERVER_MEMORY` is unset, TestHarness2 automatically runs sanitizer-instrumented
+`fdbserver` binaries with `--memory 12288MiB` so ASAN/UBSAN correctness jobs do not stay
+pinned to the default 8 GiB simulator limit.
+
 For a complete list of all variables, run: `python3 -m test_harness.app --help`
 
 ## Output Format
diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py
index 54c80c0cbb7..7f92d0ce6fe 100644
--- a/contrib/TestHarness2/test_harness/config.py
+++ b/contrib/TestHarness2/test_harness/config.py
@@ -216,10 +216,21 @@ def __init__(self):
         }
         self.print_coverage = False
         self.print_coverage_args = {"action": "store_true"}
+        self.disable_code_probes: bool = False
+        self.disable_code_probes_args = {
+            "action": "store_true",
+            "help": "Disable code probe collection and ensemble coverage checks",
+        }
         self.binary = Path("bin") / (
             "fdbserver.exe" if os.name == "nt" else "fdbserver"
         )
         self.binary_args = {"help": "Path to executable"}
+        self.fdbserver_memory: str | None = None
+        self.fdbserver_memory_args = {
+            "type": str,
+            "required": False,
+            "help": "Pass --memory SIZE to fdbserver (for example 12288MiB)",
+        }
         self.hit_per_runs_ratio: int = 20000
         self.hit_per_runs_ratio_args = {
             "help": "Maximum test runs before each code probe hit at least once"
@@ -352,7 +363,23 @@ def _read_env(self):
                 # Use the env var to supply the default value, so that if the
                 # environment variable is set and the corresponding command line
                 # flag is not, the environment variable has an effect.
-                self._config_map[attr].kwargs["default"] = attr_type(e)
+                self._config_map[attr].kwargs["default"] = self._parse_env_value(
+                    attr_type, env_name, e
+                )
+
+    def _parse_env_value(self, attr_type: type, env_name: str, value: str):
+        if attr_type is bool:
+            normalized = value.lower()
+            if normalized in ("true", "1", "yes", "on"):
+                return True
+            if normalized in ("false", "0", "no", "off"):
+                return False
+            raise ValueError(
+                "Invalid boolean value {} for {} -- use true or false".format(
+                    value, env_name
+                )
+            )
+        return attr_type(value)
 
     def build_arguments(self, parser: argparse.ArgumentParser):
         for val in self._config_map.values():
diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py
index c0124ce080c..4d31ddbafae 100644
--- a/contrib/TestHarness2/test_harness/results.py
+++ b/contrib/TestHarness2/test_harness/results.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import collections
 import io
 import json
 import re
@@ -28,9 +29,14 @@ def __init__(self, cluster_file: str | None, ensemble_id: str):
         self.fdb_path = ("joshua", "ensembles", "results", "application", ensemble_id)
         self.coverage_path = self.fdb_path + ("coverage",)
         self.statistics = test_harness.fdb.Statistics(cluster_file, self.fdb_path)
-        coverage_dict: OrderedDict[Coverage, int] = test_harness.fdb.read_coverage(
-            cluster_file, self.coverage_path
-        )
+        self.code_probe_tracking_enabled = not config.disable_code_probes
+        coverage_dict: OrderedDict[Coverage, int]
+        if self.code_probe_tracking_enabled:
+            coverage_dict = test_harness.fdb.read_coverage(
+                cluster_file, self.coverage_path
+            )
+        else:
+            coverage_dict = collections.OrderedDict()
         self.coverage: List[Tuple[Coverage, int]] = []
         self.min_coverage_hit: int | None = None
         self.ratio = self.global_statistics.total_test_runs / config.hit_per_runs_ratio
@@ -54,7 +60,9 @@ def __init__(self, cluster_file: str | None, ensemble_id: str):
             self.global_statistics.total_cpu_time += v.runtime
             self.stats.append((k, v.runtime, v.run_count))
         self.stats.sort(key=lambda x: x[1], reverse=True)
-        if self.min_coverage_hit is not None:
+        if not self.code_probe_tracking_enabled:
+            self.coverage_ok = True
+        elif self.min_coverage_hit is not None:
             self.coverage_ok = self.min_coverage_hit > self.ratio
         else:
             self.coverage_ok = False
@@ -64,6 +72,9 @@ def dump(self, prefix: str):
         out = SummaryTree("EnsembleResults")
         out.attributes["TotalRuntime"] = str(self.global_statistics.total_cpu_time)
         out.attributes["TotalTestRuns"] = str(self.global_statistics.total_test_runs)
+        out.attributes["CodeProbeTrackingEnabled"] = (
+            "1" if self.code_probe_tracking_enabled else "0"
+        )
         out.attributes["TotalProbesHit"] = str(self.global_statistics.total_probes_hit)
         out.attributes["MinProbeHit"] = str(self.min_coverage_hit)
         out.attributes["TotalProbes"] = str(len(self.coverage))
diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py
index d0133e588ab..90b5eed9dd1 100644
--- a/contrib/TestHarness2/test_harness/run.py
+++ b/contrib/TestHarness2/test_harness/run.py
@@ -22,6 +22,10 @@
 
 from test_harness.summarize import Summary, SummaryTree
 
+DEFAULT_SANITIZER_MEMORY = "12288MiB"
+SANITIZER_MARKERS = (b"__asan_init", b"__ubsan_handle_")
+_SANITIZER_BINARY_CACHE: Dict[Path, bool] = {}
+
 
 def parse_test_args_file(args_file: Path) -> tuple[Path, int, bool, List[str]]:
     """
@@ -85,6 +89,44 @@ def parse_test_args_file(args_file: Path) -> tuple[Path, int, bool, List[str]]:
     return test_file, random_seed, buggify_enabled, extra_args
 
 
+def extra_args_include_memory(extra_args: List[str]) -> bool:
+    return any(arg in ("-m", "--memory") for arg in extra_args)
+
+
+def binary_uses_sanitizers(binary: Path) -> bool:
+    cached = _SANITIZER_BINARY_CACHE.get(binary)
+    if cached is not None:
+        return cached
+
+    found = False
+    max_marker_len = max(len(marker) for marker in SANITIZER_MARKERS)
+    overlap = max_marker_len - 1
+    with binary.open("rb") as infile:
+        tail = b""
+        while True:
+            chunk = infile.read(1024 * 1024)
+            if not chunk:
+                break
+            haystack = tail + chunk
+            if any(marker in haystack for marker in SANITIZER_MARKERS):
+                found = True
+                break
+            tail = haystack[-overlap:] if overlap else b""
+
+    _SANITIZER_BINARY_CACHE[binary] = found
+    return found
+
+
+def resolve_fdbserver_memory(binary: Path, extra_args: List[str]) -> str | None:
+    if extra_args_include_memory(extra_args):
+        return None
+    if config.fdbserver_memory is not None:
+        return config.fdbserver_memory
+    if binary_uses_sanitizers(binary):
+        return DEFAULT_SANITIZER_MEMORY
+    return None
+
+
 @total_ordering
 class TestDescription:
     def __init__(self, path: Path, name: str, priority: float):
@@ -432,6 +474,7 @@ def __init__(
         extra_args: List[str] | None = None,
     ):
         self.binary = binary
+        self.is_old_binary = False
         self.test_file = test_file
         self.random_seed = random_seed
         self.uid = uid
@@ -553,6 +596,9 @@ def run(self):
             "-s",
             str(self.random_seed),
         ]
+        memory_limit = resolve_fdbserver_memory(self.binary, self.extra_args)
+        if memory_limit is not None:
+            command += ["--memory", memory_limit]
         if self.trace_format is not None:
             command += ["--trace_format", self.trace_format]
         if self.use_tls_plugin:
diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py
index 94526018637..d8e11329134 100644
--- a/contrib/TestHarness2/test_harness/summarize.py
+++ b/contrib/TestHarness2/test_harness/summarize.py
@@ -390,7 +390,11 @@ def summarize(self, trace_dir: Path, command: str):
             return
         self.summarize_files(trace_files[0])
         # Skip write_coverage for old binaries in restarting tests
-        if config.joshua_dir is not None and not self.is_old_binary:
+        if (
+            config.joshua_dir is not None
+            and not self.is_old_binary
+            and not config.disable_code_probes
+        ):
             import test_harness.fdb
 
             test_harness.fdb.write_coverage(
@@ -424,7 +428,7 @@ def ok(self):
         return (not self.error) != self.is_negative_test
 
     def done(self):
-        if config.print_coverage:
+        if config.print_coverage and not config.disable_code_probes:
             for k, v in self.coverage.items():
                 child = SummaryTree("CodeCoverage")
                 child.attributes["File"] = k.file
@@ -500,6 +504,13 @@ def done(self):
                 ):
                     # When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives.
                     continue
+                if "WARNING: ASan is ignoring requested __asan_handle_no_return: stack type:" in line:
+                    # ASAN emits this with makecontext/swapcontext coroutine stacks.
+                    continue
+                if line == "False positive error reports may follow":
+                    continue
+                if line == "For details see https://github.com/google/sanitizers/issues/189":
+                    continue
                 if line.endswith("Warning: unimplemented fcntl command: 1036"):
                     # Valgrind produces this warning when F_SET_RW_HINT is used
                     continue
@@ -695,6 +706,8 @@ def parse_error(attrs: Dict[str, str]):
         self.handler.add_handler(("Severity", "40"), parse_error)
 
         def coverage(attrs: Dict[str, str]):
+            if config.disable_code_probes:
+                return
             covered = True
             if "Covered" in attrs:
                 covered = int(attrs["Covered"]) != 0
diff --git a/contrib/benchmark_comparison.py b/contrib/benchmark_comparison.py
index ada345f691c..b5776ad558e 100644
--- a/contrib/benchmark_comparison.py
+++ b/contrib/benchmark_comparison.py
@@ -3,11 +3,11 @@
 Generate comprehensive actor vs coroutine benchmark comparison report.
 
 USAGE:
-    cd build_output  # or your build directory with bin/flowbench
+    cd build_output  # or your build directory with bin/flow_bench
     python3 ../contrib/benchmark_comparison.py
 
 REQUIREMENTS:
-    - Working bin/flowbench executable
+    - Working bin/flow_bench executable
     - Both actor and coroutine benchmarks available:
       * bench_net2, coroutine_net2
       * bench_delay.*DELAY.*, coroutine_delay_bench
@@ -33,7 +33,7 @@ def geomean(values):
 
 def get_benchmark_data(filter_name):
     """Get benchmark data and parse results"""
-    result = subprocess.run(['./bin/flowbench', f'--benchmark_filter={filter_name}', '--benchmark_format=json'], 
+    result = subprocess.run(['./bin/flow_bench', f'--benchmark_filter={filter_name}', '--benchmark_format=json'],
                           capture_output=True, text=True, cwd='/root/build_output')
     if result.returncode == 0 and result.stdout.strip():
         return json.loads(result.stdout)['benchmarks']
@@ -87,7 +87,7 @@ def main():
         print(f"OVERALL_GEOMEAN                                               {geom:+7.4f}         {geom:+7.4f}             0             0             0             0")
     
     print("")
-    print("Comparing bench_net2 to coroutine_net2 (from ./build_output/bin/flowbench)")
+    print("Comparing bench_net2 to coroutine_net2 (from ./build_output/bin/flow_bench)")
     print("Benchmark                                               Time             CPU      Time Old      Time New       CPU Old       CPU New")
     print("----------------------------------------------------------------------------------------------------------------------------")
     
@@ -114,7 +114,7 @@ def main():
         print(f"OVERALL_GEOMEAN                                      {geom:+7.4f}         {geom:+7.4f}             0             0             0             0")
     
     print("")
-    print("Comparing bench_callback to coroutine_callback (from ./build_output/bin/flowbench)")
+    print("Comparing bench_callback to coroutine_callback (from ./build_output/bin/flow_bench)")
     print("Benchmark                                                        Time             CPU      Time Old      Time New       CPU Old       CPU New")
     print("---------------------------------------------------------------------------------------------------------------------------------------------")
     
@@ -147,4 +147,4 @@ def main():
         print(f"OVERALL_GEOMEAN                                               {geom:+7.4f}         {geom:+7.4f}             0             0             0             0")
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()
diff --git a/contrib/joshua_logtool.py b/contrib/joshua_logtool.py
index 87d579ab3f5..8f296de9998 100755
--- a/contrib/joshua_logtool.py
+++ b/contrib/joshua_logtool.py
@@ -20,7 +20,7 @@
 
 from typing import List, Union
 
-# Defined in SimulatedCluster.actor.cpp:SimulationConfig::setStorageEngine
+# Defined in SimulatedCluster.cpp:SimulationConfig::setStorageEngine
 ROCKSDB_TRACEEVENT_STRING = ["RocksDBNonDeterminism", "ShardedRocksDBNonDeterminism"]
 
 # e.g. /var/joshua/ensembles/20230221-051349-xiaogesu-c9fc5b230dcd91cf
diff --git a/contrib/linenoise/CMakeLists.txt b/contrib/linenoise/CMakeLists.txt
index 52487a6f228..9191ea44432 100644
--- a/contrib/linenoise/CMakeLists.txt
+++ b/contrib/linenoise/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_library(linenoise STATIC linenoise.c)
 target_include_directories(linenoise PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
+set_target_properties(linenoise PROPERTIES C_CLANG_TIDY "")
diff --git a/contrib/lsan.suppressions b/contrib/lsan.suppressions
index 3d3d40e59eb..70fd84f298d 100644
--- a/contrib/lsan.suppressions
+++ b/contrib/lsan.suppressions
@@ -3,3 +3,14 @@
 
 # Not all incoming connections are cleanly shut down in client API tests
 leak:ConnectionReaderActorState
+
+# Peer objects in TransportData::peers are intentionally not freed at shutdown.
+# FlowTransport is allocated into a global slot and never deleted because its
+# destructor triggers actor cancellation cascades that access freed state.
+# Peer::Peer covers allocations during Peer construction (DDSketch, AsyncTrigger).
+# connectionKeeper and connectionMonitor cover Promise allocations made during
+# the Peer's lifetime that are leaked along with the Peer.
+leak:TransportData::getOrOpenPeer
+leak:Peer::Peer
+leak:connectionKeeper
+leak:connectionMonitor
diff --git a/contrib/mockkms/CMakeLists.txt b/contrib/mockkms/CMakeLists.txt
deleted file mode 100644
index 6a0b75639c2..00000000000
--- a/contrib/mockkms/CMakeLists.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-if(WITH_GO_BINDING)
-  set(MOCK_KMS_SRC fault_injection.go get_encryption_keys.go mock_kms.go utils.go)
-  set(MOCK_KMS_TEST_SRC ${MOCK_KMS_SRC} mockkms_test.go)
-  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms
-    COMMAND ${GO_EXECUTABLE} build -o ${CMAKE_BINARY_DIR}/bin/mockkms ${MOCK_KMS_SRC}
-    DEPENDS ${MOCK_KMS_SRC}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-  add_custom_target(mockkms ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms)
-  fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/mockkms DESTINATION bin COMPONENT server)
-
-  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms_test
-    COMMAND ${GO_EXECUTABLE} test -c -o ${CMAKE_BINARY_DIR}/bin/mockkms_test ${MOCK_KMS_TEST_SRC}
-    DEPENDS ${MOCK_KMS_TEST_SRC}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-  add_custom_target(mockkms_test ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms_test)
-  add_test(NAME mockkms COMMAND ${CMAKE_BINARY_DIR}/bin/mockkms_test)
-
-endif()
diff --git a/contrib/mockkms/fault_injection.go b/contrib/mockkms/fault_injection.go
deleted file mode 100644
index 8b5ff01b194..00000000000
--- a/contrib/mockkms/fault_injection.go
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * fault_injection.go
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Interface supports client to inject fault(s)
-// Module enables a client to update { FaultLocation -> FaultStatus } mapping in a
-// thread-safe manner, however, client is responsible to synchronize fault status
-// updates across 'getEncryptionKeys' REST requests to obtain predictable results.
-
-package main
-
-import (
-    "encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"sync"
-)
-
-type Fault struct {
-	Location int		 `json:"fault_location"`
-	Enable bool 		 `json:"enable_fault"`
-}
-
-type FaultInjectionRequest struct {
-	Faults []Fault   	 `json:"faults"`
-}
-
-type FaultInjectionResponse struct {
-	Faults []Fault   	 `json:"faults"`
-}
-
-type faultLocMap struct {
-	locMap map[int]bool
-	rwLock sync.RWMutex
-}
-
-var (
-	faultLocMapInstance *faultLocMap		// Singleton mapping of { FaultLocation -> FaultStatus }
-)
-
-// Caller is responsible for thread synchronization. Recommended to be invoked during package::init()
-func NewFaultLocMap() *faultLocMap {
-	if faultLocMapInstance == nil {
-		faultLocMapInstance = &faultLocMap{}
-
-		faultLocMapInstance.rwLock = sync.RWMutex{}
-		faultLocMapInstance.locMap = map[int]bool {
-			READ_HTTP_REQUEST_BODY      : false,
-			UNMARSHAL_REQUEST_BODY_JSON : false,
-			UNSUPPORTED_QUERY_MODE      : false,
-			PARSE_HTTP_REQUEST          : false,
-			MARSHAL_RESPONSE            : false,
-		}
-	}
-	return faultLocMapInstance
-}
-
-func getLocFaultStatus(loc int) (val bool, found bool) {
-	if faultLocMapInstance == nil {
-		panic("FaultLocMap not intialized")
-		os.Exit(1)
-	}
-
-	faultLocMapInstance.rwLock.RLock()
-	defer faultLocMapInstance.rwLock.RUnlock()
-	val, found = faultLocMapInstance.locMap[loc]
-	if !found {
-		return
-	}
-	return
-}
-
-func updateLocFaultStatuses(faults []Fault) (updated []Fault, err error) {
-	if faultLocMapInstance == nil {
-		panic("FaultLocMap not intialized")
-		os.Exit(1)
-	}
-
-	updated = []Fault{}
-	err = nil
-
-	faultLocMapInstance.rwLock.Lock()
-	defer faultLocMapInstance.rwLock.Unlock()
-	for i := 0; i < len(faults); i++ {
-		fault := faults[i]
-
-		oldVal, found := faultLocMapInstance.locMap[fault.Location]
-		if !found {
-			err = fmt.Errorf("Unknown fault_location '%d'", fault.Location)
-			return
-		}
-		faultLocMapInstance.locMap[fault.Location] = fault.Enable
-		log.Printf("Update Location '%d' oldVal '%t' newVal '%t'", fault.Location, oldVal, fault.Enable)
-	}
-
-	// return the updated faultLocMap
-	for loc, enable := range faultLocMapInstance.locMap {
-		var f Fault
-		f.Location = loc
-		f.Enable = enable
-		updated = append(updated, f)
-	}
-	return
-}
-
-func jsonifyFaultArr(w http.ResponseWriter, faults []Fault) (jResp string) {
-	resp := FaultInjectionResponse{
-		Faults:	faults,
-	}
-
-	mResp, err := json.Marshal(resp)
-	if err != nil {
-		log.Printf("Error marshaling response '%s'", err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-	jResp = string(mResp)
-	return
-}
-
-func updateFaultLocMap(w http.ResponseWriter, faults []Fault) {
-	updated , err := updateLocFaultStatuses(faults)
-	if err != nil {
-		sendErrorResponse(w, err)
-		return
-	}
-
-	fmt.Fprintf(w, jsonifyFaultArr(w, updated))
-}
-
-func shouldInjectFault(loc int) bool {
-	status, found := getLocFaultStatus(loc)
-	if !found {
-		log.Printf("Unknown fault_location '%d'", loc)
-		return false
-	}
-	return status
-}
-
-func handleUpdateFaultInjection(w http.ResponseWriter, r *http.Request) {
-	byteArr, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		log.Printf("Http request body read error '%s'", err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	req := FaultInjectionRequest{}
-	err = json.Unmarshal(byteArr, &req)
-	if err != nil {
-		log.Printf("Error parsing FaultInjectionRequest '%s'", string(byteArr))
-		sendErrorResponse(w, err)
-	}
-	updateFaultLocMap(w, req.Faults)
-}
-
-func initFaultLocMap() {
-	faultLocMapInstance = NewFaultLocMap()
-    log.Printf("FaultLocMap int done")
-}
\ No newline at end of file
diff --git a/contrib/mockkms/get_encryption_keys.go b/contrib/mockkms/get_encryption_keys.go
deleted file mode 100644
index 1f48e0f371d..00000000000
--- a/contrib/mockkms/get_encryption_keys.go
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * get_encryption_keys.go
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// GetEncryptionKeys handler
-// Handler is resposible for the following:
-// 1. Parse the incoming HttpRequest and validate JSON request structural sanity
-// 2. Ability to handle getEncryptionKeys by 'KeyId' or 'DomainId' as requested
-// 3. Ability to inject faults if requested
-
-package main
-
-import (
-    "encoding/json"
-    "fmt"
-    "io/ioutil"
-    "log"
-    "math/rand"
-    "net/http"
-)
-
-type CipherDetailRes struct {
-    BaseCipherId    uint64 `json:"base_cipher_id"`
-    EncryptDomainId int64  `json:"encrypt_domain_id"`
-    BaseCipher      string `json:"base_cipher"`
-}
-
-type ValidationToken struct {
-    TokenName  string `json:"token_name"`
-    TokenValue string `json:"token_value"`
-}
-
-type CipherDetailReq struct {
-    BaseCipherId    uint64 `json:"base_cipher_id"`
-    EncryptDomainId int64  `json:"encrypt_domain_id"`
-}
-
-type GetEncryptKeysResponse struct {
-    CipherDetails []CipherDetailRes `json:"cipher_key_details"`
-    KmsUrls       []string          `json:"kms_urls"`
-}
-
-type GetEncryptKeysRequest struct {
-    QueryMode        string            `json:"query_mode"`
-    CipherDetails    []CipherDetailReq `json:"cipher_key_details"`
-    ValidationTokens []ValidationToken `json:"validation_tokens"`
-    RefreshKmsUrls   bool              `json:"refresh_kms_urls"`
-}
-
-type cipherMapInstanceSingleton map[uint64][]byte
-
-const (
-    READ_HTTP_REQUEST_BODY = iota
-    UNMARSHAL_REQUEST_BODY_JSON
-    UNSUPPORTED_QUERY_MODE
-    PARSE_HTTP_REQUEST
-    MARSHAL_RESPONSE
-)
-
-const (
-    maxCipherKeys = uint64(1024 * 1024) // Max cipher keys
-    maxCipherSize = 16                  // Max cipher buffer size
-)
-
-var (
-    cipherMapInstance cipherMapInstanceSingleton // Singleton mapping of { baseCipherId -> baseCipher }
-)
-
-// const mapping of { Location -> errorString }
-func errStrMap() func(int) string {
-	_errStrMap := map[int]string{
-		READ_HTTP_REQUEST_BODY:      "Http request body read error",
-		UNMARSHAL_REQUEST_BODY_JSON: "Http request body unmarshal error",
-		UNSUPPORTED_QUERY_MODE:      "Unsupported query_mode",
-		PARSE_HTTP_REQUEST:          "Error parsing GetEncryptionKeys request",
-		MARSHAL_RESPONSE:            "Error marshaling response",
-	}
-
-	return func(key int) string {
-		return _errStrMap[key]
-	}
-}
-
-// Caller is responsible for thread synchronization. Recommended to be invoked during package::init()
-func NewCipherMap(maxKeys uint64, cipherSize int) cipherMapInstanceSingleton {
-	if cipherMapInstance == nil {
-		cipherMapInstance = make(map[uint64][]byte)
-
-		for i := uint64(1); i <= maxKeys; i++ {
-			cipher := make([]byte, cipherSize)
-			rand.Read(cipher)
-			cipherMapInstance[i] = cipher
-		}
-		log.Printf("KMS cipher map populate done, maxCiphers '%d'", maxCipherKeys)
-	}
-	return cipherMapInstance
-}
-
-func getKmsUrls() (urls []string) {
-	urlCount := rand.Intn(5) + 1
-	for i := 1; i <= urlCount; i++ {
-		url := fmt.Sprintf("https://KMS/%d:%d:%d:%d", i, i, i, i)
-		urls = append(urls, url)
-	}
-	return
-}
-
-func isEncryptDomainIdValid(id int64) bool {
-	if id > 0 || id == -1 || id == -2 {
-		return true
-	}
-	return false
-}
-
-func abs(x int64) int64 {
-	if x < 0 {
-		return -x
-	}
-	return x
-}
-
-func getBaseCipherIdFromDomainId(domainId int64) (baseCipherId uint64) {
-	baseCipherId = uint64(1) + uint64(abs(domainId))%maxCipherKeys
-	return
-}
-
-func getEncryptionKeysByKeyIds(w http.ResponseWriter, byteArr []byte) {
-	req := GetEncryptKeysRequest{}
-	err := json.Unmarshal(byteArr, &req)
-	if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) {
-		var err error
-		if shouldInjectFault(PARSE_HTTP_REQUEST) {
-			err = fmt.Errorf("[FAULT] %s %s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr))
-		} else {
-			err = fmt.Errorf("%s %s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err)
-		}
-		log.Println(err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	var details []CipherDetailRes
-	for i := 0; i < len(req.CipherDetails); i++ {
-		var baseCipherId = uint64(req.CipherDetails[i].BaseCipherId)
-
-		var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId)
-		if !isEncryptDomainIdValid(encryptDomainId) {
-			err := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId)
-			sendErrorResponse(w, err)
-			return
-		}
-
-		cipher, found := cipherMapInstance[baseCipherId]
-		if !found {
-			err := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId)
-			sendErrorResponse(w, err)
-			return
-		}
-
-		var detail = CipherDetailRes{
-			BaseCipherId:    baseCipherId,
-			EncryptDomainId: encryptDomainId,
-			BaseCipher:      string(cipher),
-		}
-		details = append(details, detail)
-	}
-
-	var urls []string
-	if req.RefreshKmsUrls {
-		urls = getKmsUrls()
-	}
-
-	resp := GetEncryptKeysResponse{
-		CipherDetails: details,
-		KmsUrls:       urls,
-	}
-
-	mResp, err := json.Marshal(resp)
-	if err != nil || shouldInjectFault(MARSHAL_RESPONSE) {
-		var err error
-		if shouldInjectFault(MARSHAL_RESPONSE) {
-			err = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE))
-		} else {
-			err = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err)
-		}
-		log.Println(err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	fmt.Fprintf(w, string(mResp))
-}
-
-func getEncryptionKeysByDomainIds(w http.ResponseWriter, byteArr []byte) {
-	req := GetEncryptKeysRequest{}
-	err := json.Unmarshal(byteArr, &req)
-	if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) {
-		var err error
-		if shouldInjectFault(PARSE_HTTP_REQUEST) {
-			err = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr))
-		} else {
-			err = fmt.Errorf("%s '%s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err)
-		}
-		log.Println(err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	var details []CipherDetailRes
-	for i := 0; i < len(req.CipherDetails); i++ {
-		var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId)
-		if !isEncryptDomainIdValid(encryptDomainId) {
-			err := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId)
-			sendErrorResponse(w, err)
-			return
-		}
-
-		var baseCipherId = getBaseCipherIdFromDomainId(encryptDomainId)
-		cipher, found := cipherMapInstance[baseCipherId]
-		if !found {
-			err := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId)
-			sendErrorResponse(w, err)
-			return
-		}
-
-		var detail = CipherDetailRes{
-			BaseCipherId:    baseCipherId,
-			EncryptDomainId: encryptDomainId,
-			BaseCipher:      string(cipher),
-		}
-		details = append(details, detail)
-	}
-
-	var urls []string
-	if req.RefreshKmsUrls {
-		urls = getKmsUrls()
-	}
-
-	resp := GetEncryptKeysResponse{
-		CipherDetails: details,
-		KmsUrls:       urls,
-	}
-
-	mResp, err := json.Marshal(resp)
-	if err != nil || shouldInjectFault(MARSHAL_RESPONSE) {
-		var err error
-		if shouldInjectFault(MARSHAL_RESPONSE) {
-			err = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE))
-		} else {
-			err = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err)
-		}
-		log.Println(err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	fmt.Fprintf(w, string(mResp))
-}
-
-func handleGetEncryptionKeys(w http.ResponseWriter, r *http.Request) {
-	byteArr, err := ioutil.ReadAll(r.Body)
-	if err != nil || shouldInjectFault(READ_HTTP_REQUEST_BODY) {
-		var err error
-		if shouldInjectFault(READ_HTTP_REQUEST_BODY) {
-			err = fmt.Errorf("[FAULT] %s", errStrMap()(READ_HTTP_REQUEST_BODY))
-		} else {
-			err = fmt.Errorf("%s err '%v'", errStrMap()(READ_HTTP_REQUEST_BODY), err)
-		}
-		log.Println(err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	var arbitrary_json map[string]interface{}
-	err = json.Unmarshal(byteArr, &arbitrary_json)
-	if err != nil || shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) {
-		var err error
-		if shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) {
-			err = fmt.Errorf("[FAULT] %s", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON))
-		} else {
-			err = fmt.Errorf("%s err '%v'", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON), err)
-		}
-		log.Println(err.Error())
-		sendErrorResponse(w, err)
-		return
-	}
-
-	if shouldInjectFault(UNSUPPORTED_QUERY_MODE) {
-		err = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"])
-		sendErrorResponse(w, err)
-		return
-	} else if arbitrary_json["query_mode"] == "lookupByKeyId" {
-		getEncryptionKeysByKeyIds(w, byteArr)
-	} else if arbitrary_json["query_mode"] == "lookupByDomainId" {
-		getEncryptionKeysByDomainIds(w, byteArr)
-	} else {
-		err = fmt.Errorf("%s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"])
-		sendErrorResponse(w, err)
-		return
-	}
-}
-
-func initEncryptCipherMap() {
-	cipherMapInstance = NewCipherMap(maxCipherKeys, maxCipherSize)
-}
diff --git a/contrib/mockkms/mock_kms.go b/contrib/mockkms/mock_kms.go
deleted file mode 100644
index 52d72539f72..00000000000
--- a/contrib/mockkms/mock_kms.go
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * mock_kms.go
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// FoundationDB Mock KMS (Key Management Solution/Service) interface
-// Interface runs an HTTP server handling REST calls simulating FDB communications
-// with an external KMS.
-
-package main
-
-// Importing module enable .note.gnu.build-id insertion in the ELF executable.
-// Few drawbacks of the scheme are:
-// 1. Potentially slower builds
-// 2. No cross-compilation support
-// 3. Limited `go tools` availability.
-// TODO: Replace with a better scheme if possible.
-
-import "C"
-
-import (
-    "log"
-    "math/rand"
-    "net/http"
-    "sync"
-    "time"
-)
-
-// KMS supported endpoints
-const (
-    getEncryptionKeysEndpoint    = "/getEncryptionKeys"
-    updateFaultInjectionEndpoint = "/updateFaultInjection"
-)
-
-// Routine is responsible to instantiate data-structures necessary for MockKMS functioning
-func init () {
-    var wg sync.WaitGroup
-
-    wg.Add(2)
-    go func(){
-        initEncryptCipherMap()
-        wg.Done()
-    }()
-    go func(){
-        initFaultLocMap()
-        wg.Done()
-    }()
-
-    wg.Wait()
-
-	rand.Seed(time.Now().UTC().UnixNano())
-}
-
-func main() {
-    http.NewServeMux()
-    http.HandleFunc(getEncryptionKeysEndpoint, handleGetEncryptionKeys)
-    http.HandleFunc(updateFaultInjectionEndpoint, handleUpdateFaultInjection)
-
-    log.Fatal(http.ListenAndServe(":5001", nil))
-}
\ No newline at end of file
diff --git a/contrib/mockkms/mockkms_test.go b/contrib/mockkms/mockkms_test.go
deleted file mode 100644
index 38b901bd8cc..00000000000
--- a/contrib/mockkms/mockkms_test.go
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * mockkms_test.go
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// MockKMS unit tests, the coverage includes:
-// 1. Mock HttpRequest creation and HttpResponse writer.
-// 2. Construct fake request to validate the following scenarios:
-//  2.1. Request with "unsupported query mode"
-//  2.2. Get encryption keys by KeyIds; with and without 'RefreshKmsUrls' flag.
-//  2.2. Get encryption keys by DomainIds; with and without 'RefreshKmsUrls' flag.
-//  2.3. Random fault injection and response validation
-
-package main
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-)
-
-const (
-	ByKeyIdReqWithRefreshUrls = `{
-		"query_mode": "lookupByKeyId",
-		"cipher_key_details": [
-			{
-				"base_cipher_id": 77,
-				"encrypt_domain_id": 76
-			},
-			{
-				"base_cipher_id": 2,
-				"encrypt_domain_id": -1
-			}
-		],
-		"validation_tokens": [
-			{
-				"token_name": "1",
-				"token_value":"12344"
-			},
-			{
-				"token_name": "2",
-				"token_value":"12334"
-			}
-		],
-		"refresh_kms_urls": true
-	}`
-	ByKeyIdReqWithoutRefreshUrls = `{
-		"query_mode": "lookupByKeyId",
-		"cipher_key_details": [
-			{
-				"base_cipher_id": 77,
-				"encrypt_domain_id": 76
-			},
-			{
-				"base_cipher_id": 2,
-				"encrypt_domain_id": -1
-			}
-		],
-		"validation_tokens": [
-			{
-				"token_name": "1",
-				"token_value":"12344"
-			},
-			{
-				"token_name": "2",
-				"token_value":"12334"
-			}
-		],
-		"refresh_kms_urls": false
-	}`
-	ByDomainIdReqWithRefreshUrls = `{
-		"query_mode": "lookupByDomainId",
-		"cipher_key_details": [
-			{
-				"encrypt_domain_id": 76
-			},
-			{
-				"encrypt_domain_id": -1
-			}
-		],
-		"validation_tokens": [
-			{
-				"token_name": "1",
-				"token_value":"12344"
-			},
-			{
-				"token_name": "2",
-				"token_value":"12334"
-			}
-		],
-		"refresh_kms_urls": true
-	}`
-	ByDomainIdReqWithoutRefreshUrls = `{
-		"query_mode": "lookupByDomainId",
-		"cipher_key_details": [
-			{
-				"encrypt_domain_id": 76
-			},
-			{
-				"encrypt_domain_id": -1
-			}
-		],
-		"validation_tokens": [
-			{
-				"token_name": "1",
-				"token_value":"12344"
-			},
-			{
-				"token_name": "2",
-				"token_value":"12334"
-			}
-		],
-		"refresh_kms_urls": false
-	}`
-	UnsupportedQueryMode= `{
-		"query_mode": "foo_mode",
-		"cipher_key_details": [
-			{
-				"encrypt_domain_id": 76
-			},
-			{
-				"encrypt_domain_id": -1
-			}
-		],
-		"validation_tokens": [
-			{
-				"token_name": "1",
-				"token_value":"12344"
-			},
-			{
-				"token_name": "2",
-				"token_value":"12334"
-			}
-		],
-		"refresh_kms_urls": false
-	}`
-)
-
-func unmarshalValidResponse(data []byte, t *testing.T) (resp GetEncryptKeysResponse) {
-	resp = GetEncryptKeysResponse{}
-	err := json.Unmarshal(data, &resp)
-	if err != nil {
-		t.Errorf("Error unmarshaling valid response '%s' error '%v'", string(data), err)
-		t.Fail()
-	}
-	return
-}
-
-func unmarshalErrorResponse(data []byte, t *testing.T) (resp ErrorResponse) {
-	resp = ErrorResponse{}
-	err := json.Unmarshal(data, &resp)
-	if err != nil {
-		t.Errorf("Error unmarshaling error response resp '%s' error '%v'", string(data), err)
-		t.Fail()
-	}
-	return
-}
-
-func checkGetEncyptKeysResponseValidity(resp GetEncryptKeysResponse, t *testing.T) {
-	if len(resp.CipherDetails) != 2 {
-		t.Errorf("Unexpected CipherDetails count, expected '%d' actual '%d'", 2, len(resp.CipherDetails))
-		t.Fail()
-	}
-
-	baseCipherIds := [...]uint64 {uint64(77), uint64(2)}
-	encryptDomainIds := [...]int64 {int64(76), int64(-1)}
-
-	for i := 0; i < len(resp.CipherDetails); i++ {
-		if resp.CipherDetails[i].BaseCipherId != baseCipherIds[i] {
-			t.Errorf("Mismatch BaseCipherId, expected '%d' actual '%d'", baseCipherIds[i], resp.CipherDetails[i].BaseCipherId)
-			t.Fail()
-		}
-		if resp.CipherDetails[i].EncryptDomainId != encryptDomainIds[i] {
-			t.Errorf("Mismatch EncryptDomainId, expected '%d' actual '%d'", encryptDomainIds[i], resp.CipherDetails[i].EncryptDomainId)
-			t.Fail()
-		}
-		if len(resp.CipherDetails[i].BaseCipher) == 0 {
-			t.Error("Empty BaseCipher")
-			t.Fail()
-		}
-	}
-}
-
-func runQueryExpectingErrorResponse(payload string, url string, errSubStr string, t *testing.T) {
-	body := strings.NewReader(payload)
-	req := httptest.NewRequest(http.MethodPost, url, body)
-	w := httptest.NewRecorder()
-	handleGetEncryptionKeys(w, req)
-	res := w.Result()
-	defer res.Body.Close()
-	data, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		t.Errorf("Error %v", err)
-	}
-
-	resp := unmarshalErrorResponse(data, t)
-	if !strings.Contains(resp.Err.Detail, errSubStr) {
-		t.Errorf("Unexpected error response '%s'", resp.Err.Detail)
-		t.Fail()
-	}
-}
-
-func runQueryExpectingValidResponse(payload string, url string, t *testing.T) {
-	body := strings.NewReader(payload)
-	req := httptest.NewRequest(http.MethodPost, url, body)
-	w := httptest.NewRecorder()
-	handleGetEncryptionKeys(w, req)
-	res := w.Result()
-	defer res.Body.Close()
-	data, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		t.Errorf("Error %v", err)
-	}
-
-	resp := unmarshalValidResponse(data, t)
-	checkGetEncyptKeysResponseValidity(resp, t)
-}
-
-func TestUnsupportedQueryMode(t *testing.T) {
-	runQueryExpectingErrorResponse(UnsupportedQueryMode, getEncryptionKeysEndpoint, errStrMap()(UNSUPPORTED_QUERY_MODE), t)
-}
-
-func TestGetEncryptionKeysByKeyIdsWithRefreshUrls(t *testing.T) {
-	runQueryExpectingValidResponse(ByKeyIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t)
-}
-
-func TestGetEncryptionKeysByKeyIdsWithoutRefreshUrls(t *testing.T) {
-	runQueryExpectingValidResponse(ByKeyIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t)
-}
-
-func TestGetEncryptionKeysByDomainIdsWithRefreshUrls(t *testing.T) {
-	runQueryExpectingValidResponse(ByDomainIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t)
-}
-
-func TestGetEncryptionKeysByDomainIdsWithoutRefreshUrls(t *testing.T) {
-	runQueryExpectingValidResponse(ByDomainIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t)
-}
-
-func TestFaultInjection(t *testing.T) {
-	numIterations := rand.Intn(701) + 86
-
-	for i := 0; i < numIterations; i++ {
-		loc := rand.Intn(MARSHAL_RESPONSE + 1)
-		f := Fault{}
-		f.Location = loc
-		f.Enable = true
-
-		var faults []Fault
-		faults = append(faults, f)
-		fW := httptest.NewRecorder()
-		body := strings.NewReader(jsonifyFaultArr(fW, faults))
-		fReq := httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body)
-		handleUpdateFaultInjection(fW, fReq)
-		if !shouldInjectFault(loc) {
-			t.Errorf("Expected fault enabled for loc '%d'", loc)
-			t.Fail()
-		}
-
-		var payload string
-		lottery := rand.Intn(100)
-		if lottery < 25 {
-			payload = ByKeyIdReqWithRefreshUrls
-		} else if lottery >= 25 && lottery < 50 {
-			payload = ByKeyIdReqWithoutRefreshUrls
-		} else if lottery >= 50 && lottery < 75 {
-			payload = ByDomainIdReqWithRefreshUrls
-		} else {
-			payload = ByDomainIdReqWithoutRefreshUrls
-		}
-		runQueryExpectingErrorResponse(payload, getEncryptionKeysEndpoint, errStrMap()(loc), t)
-
-		// reset Fault
-		faults[0].Enable = false
-		fW = httptest.NewRecorder()
-		body = strings.NewReader(jsonifyFaultArr(fW, faults))
-		fReq = httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body)
-		handleUpdateFaultInjection(fW, fReq)
-		if shouldInjectFault(loc) {
-			t.Errorf("Expected fault disabled for loc '%d'", loc)
-			t.Fail()
-		}
-	}
-}
\ No newline at end of file
diff --git a/contrib/monitoring/actor_flamegraph.cpp b/contrib/monitoring/actor_flamegraph.cpp
index 58bd0d2c862..b97aaa360b8 100644
--- a/contrib/monitoring/actor_flamegraph.cpp
+++ b/contrib/monitoring/actor_flamegraph.cpp
@@ -113,11 +113,11 @@ class Traces {
 					actor->stack.push_back(currentStack.top()->name);
 				}
 			} else if (op == OP_DESTROY) {
-				if (actors.count(id)) {
+				if (actors.contains(id)) {
 					actors.erase(id);
 				}
 			} else if (op == OP_ENTER) {
-				if (actors.count(id) == 0) {
+				if (!actors.contains(id)) {
 					actors[id] = std::make_shared<Actor>(results, id, name);
 				}
 				currentStack.push(actors[id]);
diff --git a/contrib/pkg_tester/requirements.txt b/contrib/pkg_tester/requirements.txt
index af94b20328e..d56e3045ff4 100644
--- a/contrib/pkg_tester/requirements.txt
+++ b/contrib/pkg_tester/requirements.txt
@@ -6,7 +6,7 @@ packaging==20.9
 pluggy==0.13.1
 py==1.10.0
 pyparsing==2.4.7
-pytest==6.2.4
+pytest==9.0.3
 syrupy==1.2.3
 toml==0.10.2
 typing-extensions==3.10.0.0
diff --git a/contrib/replay/CMakeLists.txt b/contrib/replay/CMakeLists.txt
index 85e393c707d..7a113685ced 100644
--- a/contrib/replay/CMakeLists.txt
+++ b/contrib/replay/CMakeLists.txt
@@ -25,27 +25,23 @@ file(MAKE_DIRECTORY ${REPLAY_OUTPUT_DIR})
 
 # Get all Go source files for dependency tracking
 file(GLOB REPLAY_GO_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.go")
+set(REPLAY_GO_MOD "${CMAKE_CURRENT_SOURCE_DIR}/go.mod")
+set(REPLAY_GO_SUM "${CMAKE_CURRENT_SOURCE_DIR}/go.sum")
 
-# Custom target to build replay
-# This handles all Go dependencies automatically - users just need Go installed
-# ALL means it builds by default (but only if Go is available - see check above)
-add_custom_target(replay ALL
-    COMMAND ${CMAKE_COMMAND} -E echo "Downloading Go dependencies..."
-    COMMAND ${GO_EXECUTABLE} mod download
-    COMMAND ${CMAKE_COMMAND} -E echo "Tidying Go modules..."
-    COMMAND ${GO_EXECUTABLE} mod tidy
-    COMMAND ${CMAKE_COMMAND} -E echo "Building replay..."
+# Custom command to build replay - only rebuilds when source files change
+# go build handles dependencies automatically (downloads if needed)
+add_custom_command(
+    OUTPUT ${REPLAY_BINARY}
     COMMAND ${GO_EXECUTABLE} build -o ${REPLAY_BINARY} .
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMENT "Building FDB trace replay tool (all dependencies handled automatically)"
-    BYPRODUCTS ${REPLAY_BINARY}
-    SOURCES ${REPLAY_GO_SOURCES}
+    DEPENDS ${REPLAY_GO_SOURCES} ${REPLAY_GO_MOD} ${REPLAY_GO_SUM}
+    COMMENT "Building FDB trace replay tool"
 )
 
-# Make sure the binary is executable
-add_custom_command(TARGET replay POST_BUILD
-    COMMAND chmod +x ${REPLAY_BINARY}
-    COMMENT "Making replay executable"
+# Target that depends on the binary - only triggers rebuild when deps change
+add_custom_target(replay ALL
+    DEPENDS ${REPLAY_BINARY}
+    SOURCES ${REPLAY_GO_SOURCES}
 )
 
 message(STATUS "replay will be built by default (Go found)")
diff --git a/contrib/sqlite/CMakeLists.txt b/contrib/sqlite/CMakeLists.txt
index 71ca7e6666c..41ae5a42781 100644
--- a/contrib/sqlite/CMakeLists.txt
+++ b/contrib/sqlite/CMakeLists.txt
@@ -7,6 +7,8 @@ add_library(sqlite STATIC
   sqliteLimit.h
   sqlite3.amalgamation.c)
 
+set_target_properties(sqlite PROPERTIES C_CLANG_TIDY "")
+
 
 target_include_directories(sqlite PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 # Suppress warnings in sqlite since it's third party
@@ -18,4 +20,4 @@ endif()
 # SQLite won't build with -ffast-math
 if(ICX)
   target_compile_options(sqlite PRIVATE -fno-fast-math)
-endif()
\ No newline at end of file
+endif()
diff --git a/contrib/stacktrace/CMakeLists.txt b/contrib/stacktrace/CMakeLists.txt
index f82b4b6a6c3..6d958db451f 100644
--- a/contrib/stacktrace/CMakeLists.txt
+++ b/contrib/stacktrace/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_library(stacktrace STATIC stacktrace.amalgamation.cpp)
+set_target_properties(stacktrace PROPERTIES CXX_CLANG_TIDY "")
 target_include_directories(stacktrace PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
 if (USE_ASAN)
   target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER)
diff --git a/design/AI-generated/FDB_NETWORK_PROTOCOL.md b/design/AI-generated/FDB_NETWORK_PROTOCOL.md
new file mode 100644
index 00000000000..6e4ca2c82bd
--- /dev/null
+++ b/design/AI-generated/FDB_NETWORK_PROTOCOL.md
@@ -0,0 +1,2666 @@
+# FoundationDB Network Protocol Message Reference
+
+This document is a comprehensive enumeration of every serialized message type in the
+FoundationDB network protocol. These types are not documented in any single place in
+the source; they are defined implicitly by `serialize`/`serializer` methods on C++
+structs throughout the codebase.
+
+*NOTE: By design FDB protocols can change between minor versions. Information in
+this file is very detailed and is subject to being out of date. However it is
+probably straightforward to ask state of the art models to update the documentation
+for you if needed.*
+
+**Conventions in this document:**
+- Every struct referenced in any message is fully defined with all serialized fields.
+- Types are resolved recursively down to C/C++ base types.
+- "→ base" annotations after a type show what it resolves to at the wire level.
+- Fields marked "(not serialized)" are present in the C++ struct but do not appear on the wire.
+- `Optional<T>` is serialized as: `bool present` + (if true) the value of type `T`.
+- `std::vector<T>` / `VectorRef<T>` are serialized as: `int32_t count` + count × T.
+- `Arena` fields provide memory backing for `Ref` types; they are serialized last.
+
+---
+
+## Table of Contents
+
+- [1. Base Types and Aliases](#1-base-types-and-aliases)
+- [2. Foundational Struct Definitions](#2-foundational-struct-definitions)
+- [3. Coordination Protocol](#3-coordination-protocol)
+- [4. Cluster Controller Protocol](#4-cluster-controller-protocol)
+- [5. Master Protocol](#5-master-protocol)
+- [6. GRV Proxy Protocol](#6-grv-proxy-protocol)
+- [7. Commit Proxy Protocol](#7-commit-proxy-protocol)
+- [8. Resolver Protocol](#8-resolver-protocol)
+- [9. Transaction Log (TLog) Protocol](#9-transaction-log-tlog-protocol)
+- [10. Storage Server Protocol](#10-storage-server-protocol)
+- [11. Data Distributor Protocol](#11-data-distributor-protocol)
+- [12. Ratekeeper Protocol](#12-ratekeeper-protocol)
+- [13. Worker Protocol](#13-worker-protocol)
+- [14. Backup / Consistency Scan / Test Protocols](#14-backup--consistency-scan--test-protocols)
+- [15. Client Worker / Debug / Process Protocols](#15-client-worker--debug--process-protocols)
+- [Appendix A: Log System Configuration Types](#appendix-a-log-system-configuration-types)
+- [Appendix B: In-Band Log Messages](#appendix-b-in-band-log-messages)
+- [Appendix C: Backpressure Protocol](#appendix-c-backpressure-protocol)
+- [Appendix D: Client Library Protocol Flows](#appendix-d-client-library-protocol-flows)
+- [Appendix E: fdbcli Protocol Usage](#appendix-e-fdbcli-protocol-usage)
+- [Appendix F: Server-Side Protocol Flows](#appendix-f-server-side-protocol-flows)
+
+---
+
+## Architecture Overview
+
+FDB uses an RPC framework built on its "Flow" actor model. Each server role exposes
+an **interface** struct containing `RequestStream<T>` fields — typed endpoints.
+Request messages carry a `ReplyPromise<R>` (or `ReplyPromiseStream<R>` for streaming)
+that the receiver fulfills with the reply.
+
+- **Endpoint multiplexing**: many interfaces serialize only one `RequestStream` and
+  reconstruct the others via `getAdjustedEndpoint(offset)` during deserialization.
+- **Wire format**: `ProtocolVersion` (uint64) header, then fields serialized
+  sequentially via `serializer(ar, field1, field2, ...)` with no inter-field framing.
+
+---
+
+## 1. Base Types and Aliases
+
+These are the leaf types that all struct fields ultimately resolve to.
+
+| Wire Type | C++ Type | Size | Notes |
+|-----------|----------|------|-------|
+| `bool` | `bool` | 1 byte | |
+| `int8_t` / `uint8_t` | integer | 1 byte | |
+| `int16_t` / `uint16_t` | integer | 2 bytes | little-endian |
+| `int32_t` / `uint32_t` / `int` | integer | 4 bytes | little-endian |
+| `int64_t` / `uint64_t` | integer | 8 bytes | little-endian |
+| `double` | IEEE 754 | 8 bytes | little-endian |
+| `StringRef` | length-prefixed bytes | `int32_t` len + bytes | |
+| `std::string` | length-prefixed bytes | `int32_t` len + bytes | |
+| `ProtocolVersion` | `uint64_t` | 8 bytes | version with feature flags |
+| enums | underlying integer type | varies | typically `uint8_t` or `uint32_t` |
+
+**Type aliases** used throughout the protocol (all are simple typedefs):
+
+| Alias | Resolves To | Purpose |
+|-------|-------------|---------|
+| `Version` | `int64_t` | MVCC version number |
+| `LogEpoch` | `uint64_t` | TLog generation epoch |
+| `Generation` | `int64_t` | Cluster controller generation |
+| `Sequence` | `uint64_t` | Sequence number |
+| `DBRecoveryCount` | `uint64_t` | Sequential recovery counter |
+| `Key` | `Standalone<KeyRef>` → `StringRef` | Key bytes (arena-owned) |
+| `KeyRef` | `StringRef` | Key bytes (ref) |
+| `Value` | `Standalone<ValueRef>` → `StringRef` | Value bytes (arena-owned) |
+| `ValueRef` | `StringRef` | Value bytes (ref) |
+| `TransactionTag` | `Standalone<TransactionTagRef>` → `StringRef` | Transaction tag string |
+| `TransactionTagRef` | `StringRef` | Transaction tag ref |
+| `SpanID` | `UID` | Tracing span identifier |
+
+**Generic containers** (serialization is count + elements):
+
+| Container | Wire Format |
+|-----------|-------------|
+| `std::vector<T>` | `int32_t` count, then count × T |
+| `VectorRef<T>` | `int32_t` count, then count × T |
+| `std::set<T>` | `int32_t` count, then count × T |
+| `std::map<K,V>` | `int32_t` count, then count × (K, V) pairs |
+| `std::unordered_map<K,V>` | `int32_t` count, then count × (K, V) pairs |
+| `std::deque<T>` | `int32_t` count, then count × T |
+| `Optional<T>` | `bool` present, then (if true) T |
+| `std::variant<Ts...>` | `uint8_t` index, then the active variant |
+| `Standalone<T>` | same as T (Arena is implicit) |
+
+---
+
+## 2. Foundational Struct Definitions
+
+Every struct used as a field in protocol messages is defined here, in dependency
+order. Each definition shows all serialized fields and their fully-resolved types.
+
+### UID
+Universally unique identifier. Serialized as two `uint64_t` values.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| part[0] | `uint64_t` | First 8 bytes |
+| part[1] | `uint64_t` | Second 8 bytes |
+
+### IPAddress
+IP address supporting both IPv4 and IPv6.
+
+Serialized as: `bool isV6`, then either `uint32_t` (v4) or `std::array<uint8_t, 16>` (v6).
+
+### NetworkAddress
+A network endpoint address.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| ip | `IPAddress` | IP address (see above) |
+| port | `uint16_t` | Port number |
+| flags | `uint16_t` | Flags (TLS, public, etc.) |
+| fromHostname | `bool` | Whether resolved from hostname (conditional on protocol version) |
+
+### NetworkAddressList
+A primary + optional secondary address for multi-homed processes.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| address | `NetworkAddress` | Primary address |
+| secondaryAddress | `Optional<NetworkAddress>` | Secondary address |
+
+### Hostname
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| host | `std::string` | Hostname string |
+| service | `std::string` | Service/port string |
+| isTLS | `bool` | Whether TLS is enabled |
+
+### AddressExclusion
+An IP address + optional port for excluding servers.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| ip | `IPAddress` | IP address |
+| port | `int` (int32_t) | Port (0 = all ports on this IP) |
+
+### Endpoint
+A network-addressable RPC endpoint.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| addresses | `NetworkAddressList` | Network addresses |
+| token | `UID` | Endpoint token (two `uint64_t`) |
+
+### Tag
+Identifies which storage server(s) should receive a mutation.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| locality | `int8_t` | Locality group (-1 = special, 0+ = DC index) |
+| id | `uint16_t` | Tag ID within the locality |
+
+### KeyRangeRef
+A contiguous range of keys [begin, end).
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| begin | `KeyRef` → `StringRef` | Start key (inclusive) |
+| end | `KeyRef` → `StringRef` | End key (exclusive) |
+
+**Note:** `KeyRange` is `Standalone<KeyRangeRef>` (same wire format, arena-owned).
+
+### KeyValueRef
+A single key-value pair.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `KeyRef` → `StringRef` | Key bytes |
+| value | `ValueRef` → `StringRef` | Value bytes |
+
+**Note:** `KeyValue` is `Standalone<KeyValueRef>`.
+
+### KeySelectorRef
+A relative key reference: "the first/last key ≥/≤ `key`, plus `offset`."
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `KeyRef` → `StringRef` | Anchor key |
+| orEqual | `bool` | Whether to include the anchor key |
+| offset | `int` (int32_t) | Offset from anchor |
+
+### RangeResultRef
+Result of a range read. Inherits from `VectorRef<KeyValueRef>` (vector of KV pairs).
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| (inherited) | `VectorRef<KeyValueRef>` | Key-value pairs |
+| more | `bool` | Whether more results exist |
+| readThrough | `Optional<KeyRef>` | Read-through key |
+| readToBegin | `bool` | Read reached beginning |
+| readThroughEnd | `bool` | Read reached end |
+
+### MutationRef
+A single database mutation.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| type | `uint8_t` | Mutation type (SetValue=0, ClearRange=1, AddValue=2, ...) |
+| param1 | `StringRef` | First parameter (key or range start) |
+| param2 | `StringRef` | Second parameter (value or range end) |
+| checksum | `Optional<uint32_t>` | Optional checksum (conditional on protocol version) |
+| accumulativeChecksumIndex | `Optional<uint16_t>` | Checksum index (conditional on protocol version) |
+
+### CommitTransactionRef
+A complete transaction to be committed.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| read_conflict_ranges | `VectorRef<KeyRangeRef>` | Read conflict ranges |
+| write_conflict_ranges | `VectorRef<KeyRangeRef>` | Write conflict ranges |
+| mutations | `VectorRef<MutationRef>` | Mutations to apply |
+| read_snapshot | `Version` → `int64_t` | Read snapshot version |
+| report_conflicting_keys | `bool` | Whether to report conflicts |
+| lock_aware | `bool` | Whether lock-aware |
+| spanContext | `Optional<SpanContext>` | Tracing context |
+| tenantIds | `Optional<VectorRef<int64_t>>` | Tenant IDs (conditional on protocol version) |
+
+### MutationsAndVersionRef
+A set of mutations at a particular version.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| mutations | `VectorRef<MutationRef>` | Mutations |
+| version | `Version` → `int64_t` | Version |
+| knownCommittedVersion | `Version` → `int64_t` | Known committed version |
+
+### SpanContext
+OpenTelemetry-compatible distributed tracing context.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| traceID | `UID` (two `uint64_t`) | Trace identifier |
+| spanID | `uint64_t` | Span identifier |
+| m_Flags | `uint8_t` | Trace flags |
+
+### FailureStatus
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| failed | `bool` | Whether the process is failed |
+
+### SystemFailureStatus
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| addresses | `NetworkAddressList` | Process addresses |
+| status | `FailureStatus` | Failure status (→ `bool failed`) |
+
+### LocalityData
+Locality key-value map for a process. Serialized as a `uint64_t` map size followed
+by (key, value) pairs.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| _data | `std::map<Standalone<StringRef>, Optional<Standalone<StringRef>>>` | Locality key-value pairs |
+
+### ProcessClass
+Classification of a process (role fitness).
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| _class | `int16_t` | Class type (unset, storage, transaction, resolution, ...) |
+| _source | `int16_t` | Source (command line, configure, ...) |
+
+### ClusterControllerPriorityInfo
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| processClassFitness | `uint8_t` | Fitness for cluster controller role |
+| isExcluded | `bool` | Whether the process is excluded |
+| dcFitness | `uint8_t` | Data center fitness |
+
+### StorageMetrics
+Load metrics for a storage range.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| bytes | `int64_t` | Bytes stored |
+| bytesWrittenPerKSecond | `int64_t` | Write rate |
+| iosPerKSecond | `int64_t` | I/O rate |
+| bytesReadPerKSecond | `int64_t` | Read rate |
+| opsReadPerKSecond | `int64_t` | Read operation rate |
+
+### StorageBytes
+Disk space information.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| free | `int64_t` | Free bytes |
+| total | `int64_t` | Total bytes |
+| used | `int64_t` | Used bytes |
+| available | `int64_t` | Available bytes |
+
+### HealthMetrics
+Cluster-wide health metrics.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| worstStorageQueue | `int64_t` | Worst SS queue size |
+| limitingStorageQueue | `int64_t` | Limiting SS queue size |
+| worstStorageDurabilityLag | `int64_t` | Worst SS durability lag |
+| limitingStorageDurabilityLag | `int64_t` | Limiting SS durability lag |
+| worstTLogQueue | `int64_t` | Worst TLog queue size |
+| tpsLimit | `double` | Current TPS limit |
+| batchLimited | `bool` | Whether batch is limited |
+| storageStats | `std::map<UID, StorageStats>` | Per-SS stats |
+| tLogQueue | `std::map<UID, int64_t>` | Per-TLog queue sizes |
+
+#### HealthMetrics::StorageStats
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| storageQueue | `int64_t` | Queue size |
+| storageDurabilityLag | `int64_t` | Durability lag |
+| diskUsage | `double` | Disk usage fraction |
+| cpuUsage | `double` | CPU usage fraction |
+
+### BusyTagInfo
+Information about a busy transaction tag on a storage server.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tag | `TransactionTag` → `StringRef` | Tag string |
+| rate | `double` | Tag rate |
+| fractionalBusyness | `double` | Fraction of busyness |
+
+### DDMetricsRef
+Data distribution shard metrics.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| shardBytes | `int64_t` | Bytes in shard |
+| shardBytesPerKSecond | `int64_t` | Write rate |
+| beginKey | `KeyRef` → `StringRef` | Shard begin key |
+
+### LifetimeToken
+Identifies a master's lifetime for fencing.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| ccID | `UID` (two `uint64_t`) | Cluster controller ID |
+| count | `int64_t` | Monotonic counter |
+
+### UniqueGeneration
+Paxos generation identifier.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| generation | `uint64_t` | Generation number |
+| uid | `UID` (two `uint64_t`) | Unique ID for tie-breaking |
+
+### LeaderInfo
+Information about the current cluster controller leader.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| changeID | `UID` (two `uint64_t`) | Unique ID for this leader epoch |
+| serializedInfo | `Value` → `StringRef` | Serialized `ClusterControllerClientInterface` |
+| forward | `bool` | If true, serializedInfo is a forwarding address |
+
+### ClusterConnectionString
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| coords | `std::vector<NetworkAddress>` | Coordinator addresses |
+| hostnames | `std::vector<Hostname>` | Coordinator hostnames |
+| key | `Key` → `StringRef` | Cluster description key |
+| keyDesc | `Key` → `StringRef` | Cluster description |
+
+### ClientVersionRef
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| clientVersion | `StringRef` | Client version string |
+| sourceVersion | `StringRef` | Source version hash |
+| protocolVersion | `StringRef` | Protocol version string |
+
+### VersionHistory
+Global configuration change record.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `Version` → `int64_t` | Version of this config change |
+| mutations | `Standalone<VectorRef<MutationRef>>` | Config mutations |
+
+### ReadOptions
+Options for read operations.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| type | `ReadType` → `uint8_t` | Read type (NORMAL, EAGER, FETCH) |
+| cacheResult | `bool` | Whether to cache the result |
+| debugID | `Optional<UID>` | Debug trace ID |
+| consistencyCheckStartVersion | `Optional<Version>` | Consistency check version |
+| lockAware | `bool` | Whether lock-aware |
+
+### TagSet
+A set of transaction tags. Custom serialization.
+
+Serialized as a vector of `TransactionTagRef` (`StringRef`) entries.
+
+### ClientTagThrottleLimits
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tpsRate | `double` | Transactions per second rate limit |
+| duration | `double` | Duration of throttle |
+
+### TransactionCommitCostEstimation
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| opsSum | `int` (int32_t) | Total operation count |
+| costSum | `uint64_t` | Total cost |
+
+### IdempotencyIdRef
+A 16-byte idempotency identifier. Serialized as raw bytes with custom encoding.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| first | `uint64_t` | First 8 bytes (or size indicator) |
+| second | `uint64_t` | Second 8 bytes (or pointer to data) |
+
+### KeyValueStoreType
+Storage engine type enum.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| type | `uint32_t` | Engine type (SSD=1, MEMORY=2, SSD_ROCKSDB=4, ...) |
+
+### TLogVersion
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `uint32_t` | TLog version (V2=2, ..., V8=8) |
+
+### TLogSpillType
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| type | `uint32_t` | Spill type (DEFAULT=0, VALUE=1, REFERENCE=2) |
+
+### EncryptionAtRestModeDeprecated
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| mode | `uint32_t` | Encryption mode (DISABLED=0, DOMAIN_AWARE=1, CLUSTER_AWARE=2) |
+
+### MoveKeysLock
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| prevOwner | `UID` (two `uint64_t`) | Previous lock owner |
+| myOwner | `UID` (two `uint64_t`) | Current lock owner |
+| prevWrite | `UID` (two `uint64_t`) | Previous write |
+
+### LogMessageVersion
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `Version` → `int64_t` | Version |
+| sub | `uint32_t` | Sub-version within the version |
+
+### Versionstamp
+Serialized in big-endian for ordered comparison.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `int64_t` | Version (big-endian on wire) |
+| batchNumber | `int16_t` | Batch number (big-endian on wire) |
+
+### StorageMetadataType
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| createdTime | `double` | Creation timestamp |
+| storeType | `KeyValueStoreType` → `uint32_t` | Storage engine type |
+
+### RecoveryState (enum)
+Serialized as its underlying integer type.
+
+| Value | Name | Meaning |
+|-------|------|---------|
+| 0 | UNINITIALIZED | Not started |
+| 1 | READING_CSTATE | Reading coordinated state |
+| 2 | LOCKING_CSTATE | Locking coordinated state |
+| 3 | RECRUITING | Recruiting transaction system |
+| 4 | RECOVERY_TRANSACTION | Writing recovery transaction |
+| 5 | WRITING_CSTATE | Writing coordinated state |
+| 6 | ACCEPTING_COMMITS | Accepting commits |
+| 7 | ALL_LOGS_RECRUITED | All logs recruited |
+| 8 | STORAGE_RECOVERED | Storage servers recovered |
+| 9 | FULLY_RECOVERED | Fully recovered |
+
+### ClusterType (enum)
+
+| Value | Name |
+|-------|------|
+| 0 | STANDALONE |
+| 1 | LEGACY_UNUSED_METACLUSTER_MANAGEMENT |
+| 2 | LEGACY_UNUSED_METACLUSTER_DATA |
+
+### LogSystemType (enum)
+
+| Value | Name |
+|-------|------|
+| -1 | unset |
+| 0 | empty |
+| 2 | tagPartitioned |
+
+### ResolverMoveRef
+Describes a shard reassignment between resolvers.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| range | `KeyRangeRef` | Key range being moved |
+| dest | `int` (int32_t) | Destination resolver index |
+
+### StateTransactionRef
+A state transaction with its commit status.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| committed | `bool` | Whether committed |
+| mutations | `VectorRef<MutationRef>` | State mutations |
+
+### UnknownCommittedVersions
+Tracks version ranges whose commit status is unknown during recovery.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `Version` → `int64_t` | Version |
+| prev | `Version` → `int64_t` | Previous version |
+| tLogLocIds | `std::vector<uint16_t>` | TLog locality IDs |
+
+### StorageServerShard
+Physical shard metadata on a storage server.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| range | `KeyRange` | Shard key range |
+| version | `Version` → `int64_t` | Shard version |
+| id | `uint64_t` | Shard ID |
+| desiredId | `uint64_t` | Desired shard ID |
+| shardState | `int8_t` | Shard state (ReadWrite, Adding, etc.) |
+| moveInShardId | `Optional<UID>` | Move-in shard ID if being moved |
+
+### CheckpointMetaData
+Metadata describing a storage checkpoint.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `Version` → `int64_t` | Checkpoint version |
+| ranges | `std::vector<KeyRange>` | Key ranges in checkpoint |
+| format | `int16_t` | Checkpoint format |
+| state | `int16_t` | Checkpoint state |
+| checkpointID | `UID` (two `uint64_t`) | Checkpoint identifier |
+| src | `std::vector<UID>` | Source servers |
+| serializedCheckpoint | `Standalone<StringRef>` | Serialized checkpoint data |
+| actionId | `Optional<UID>` | Action ID |
+| bytesSampleFile | `Optional<std::string>` | Sample file path |
+| dir | `std::string` | Directory path |
+
+### OverlappingChangeFeedEntry
+An entry describing a change feed that overlaps a key range.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| feedId | `KeyRef` → `StringRef` | Change feed identifier |
+| range | `KeyRangeRef` | Feed's key range |
+| emptyVersion | `Version` → `int64_t` | Version at which feed was empty |
+| stopVersion | `Version` → `int64_t` | Version at which feed was stopped |
+| feedMetadataVersion | `Version` → `int64_t` | Feed metadata version |
+
+### ReadHotRangeWithMetrics
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| keys | `KeyRangeRef` | Hot key range |
+| density | `double` | Read density |
+| readBandwidthSec | `double` | Read bandwidth (bytes/sec) |
+| bytes | `int64_t` | Bytes in range |
+| readOpsSec | `double` | Read operations per second |
+
+### CheckSumMetaData
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| range | `KeyRange` | Key range |
+| version | `Version` → `int64_t` | Version |
+| checkSumValue | `StringRef` | Checksum bytes |
+
+### BulkDumpState
+State of a bulk dump job.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| jobId | `UID` (two `uint64_t`) | Job identifier |
+| jobRange | `KeyRange` | Key range being dumped |
+| phase | `uint8_t` | Dump phase |
+| taskId | `Optional<UID>` | Task identifier |
+| manifest | `BulkLoadManifest` | Manifest data (complex, serialized as bytes) |
+
+### AuditStorageState
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| id | `UID` (two `uint64_t`) | Audit ID |
+| auditServerId | `UID` (two `uint64_t`) | Server performing audit |
+| range | `KeyRange` | Audited key range |
+| type | `uint8_t` | Audit type |
+| phase | `uint8_t` | Audit phase |
+| error | `std::string` | Error message |
+| ddId | `UID` (two `uint64_t`) | Data distributor ID |
+| engineType | `KeyValueStoreType` → `uint32_t` | Storage engine type |
+
+### MappedKeyValueRef
+A key-value pair with a secondary index lookup result.
+Inherits from `KeyValueRef`.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `KeyRef` → `StringRef` | Primary key (inherited) |
+| value | `ValueRef` → `StringRef` | Primary value (inherited) |
+| reqAndResult | `std::variant<GetValueReqAndResultRef, GetRangeReqAndResultRef>` | Mapped result |
+
+#### GetValueReqAndResultRef
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `KeyRef` → `StringRef` | Looked-up key |
+| result | `Optional<ValueRef>` | Looked-up value |
+
+#### GetRangeReqAndResultRef
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| begin | `KeySelectorRef` | Range begin selector |
+| end | `KeySelectorRef` | Range end selector |
+| result | `RangeResultRef` | Range result |
+
+### DatabaseConfiguration
+Database configuration. Serialized as a key-value list.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| rawConfiguration | `Standalone<VectorRef<KeyValueRef>>` | Configuration key-value pairs |
+
+The key-value pairs encode redundancy mode, storage engine, proxy/resolver/log counts,
+region configuration, and other tuning parameters.
+
+### OpenDatabaseRequest::Samples
+Sampling data for client issue/version tracking.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| count | `int` (int32_t) | Total count |
+| samples | `std::set<std::pair<NetworkAddress, Key>>` | Sampled (address, key) pairs |
+
+### DiskFailureCommand (nested in SetFailureInjection)
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| stallInterval | `double` | Interval between stalls |
+| stallPeriod | `double` | Duration of each stall |
+| throttlePeriod | `double` | Throttle period |
+
+### FlipBitsCommand (nested in SetFailureInjection)
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| percentBitFlips | `double` | Percentage of bits to flip |
+
+### SerializedSample
+An actor lineage sample.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| time | `double` | Sample timestamp |
+| data | `std::unordered_map<WaitState, std::string>` | Wait state → data map |
+
+`WaitState` is serialized as its underlying integer type.
+
+### PerfMetric
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| m_name | `std::string` | Metric name |
+| m_format_code | `std::string` | Format code |
+| m_value | `double` | Metric value |
+| m_averaged | `bool` | Whether averaged |
+
+---
+
+## 3. Coordination Protocol
+
+**Source:** `fdbclient/include/fdbclient/CoordinationInterface.h`,
+`fdbserver/core/include/fdbserver/core/CoordinationInterface.h`
+
+Coordinators run Paxos-based leader election and store the `DBCoreState`.
+
+### ProtocolInfoRequest
+Handshake message to discover a peer's protocol version.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| reply | `ReplyPromise<ProtocolInfoReply>` → `UID` token | Where to send reply |
+
+**Reply: ProtocolInfoReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `ProtocolVersion` → `uint64_t` | Protocol version of the responder |
+
+### GetLeaderRequest
+Ask a coordinator who the current leader (cluster controller) is.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Cluster description key |
+| knownLeader | `UID` | Last known leader change ID (long-poll) |
+| reply | `ReplyPromise<Optional<LeaderInfo>>` | Where to send reply |
+
+**Reply:** `Optional<LeaderInfo>` — see [LeaderInfo](#leaderinfo) definition.
+
+### OpenDatabaseCoordRequest
+Client requests current `ClientDBInfo` from coordinators.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| traceLogGroup | `Key` → `StringRef` | Client's trace log group |
+| issues | `Standalone<VectorRef<StringRef>>` | Client-reported issues |
+| supportedVersions | `Standalone<VectorRef<ClientVersionRef>>` | Client library versions |
+| knownClientInfoID | `UID` | Last known DB info ID (long-poll) |
+| clusterKey | `Key` → `StringRef` | Cluster connection key |
+| hostnames | `std::vector<Hostname>` | Coordinator hostnames |
+| coordinators | `std::vector<NetworkAddress>` | Coordinator addresses |
+| internal | `bool` | Whether this is an internal request |
+| reply | `ReplyPromise<CachedSerialization<ClientDBInfo>>` | Where to send reply |
+
+**Reply:** `ClientDBInfo` — see [Section 4](#4-cluster-controller-protocol).
+
+### CheckDescriptorMutableRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| reply | `ReplyPromise<CheckDescriptorMutableReply>` | Where to send reply |
+
+**Reply: CheckDescriptorMutableReply** — `bool isMutable`.
+
+### CandidacyRequest
+Propose a candidate for cluster controller during leader election.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Cluster description key |
+| myInfo | `LeaderInfo` | Candidate's leader info |
+| knownLeader | `UID` | Currently known leader |
+| prevChangeID | `UID` | Previous change ID |
+| reply | `ReplyPromise<Optional<LeaderInfo>>` | Election result |
+
+### ElectionResultRequest
+Wait for the result of an election.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Cluster description key |
+| hostnames | `std::vector<Hostname>` | Coordinator hostnames |
+| coordinators | `std::vector<NetworkAddress>` | Coordinator addresses |
+| knownLeader | `UID` | Currently known leader |
+| reply | `ReplyPromise<Optional<LeaderInfo>>` | Election result |
+
+### LeaderHeartbeatRequest
+Leader sends heartbeats to maintain leadership.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Cluster description key |
+| myInfo | `LeaderInfo` | Leader's info |
+| prevChangeID | `UID` | Previous change ID |
+| reply | `ReplyPromise<LeaderHeartbeatReply>` | Heartbeat ack |
+
+**Reply: LeaderHeartbeatReply** — `bool value` (accepted or not).
+
+### ForwardRequest
+Tell a coordinator to forward clients to a new connection string.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Cluster description key |
+| conn | `Value` → `StringRef` | New cluster connection string |
+| reply | `ReplyPromise<Void>` | Acknowledgment |
+
+### GenerationRegReadRequest
+Read from the generation register (Paxos storage layer).
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Register key |
+| gen | `UniqueGeneration` | Generation to read at |
+| reply | `ReplyPromise<GenerationRegReadReply>` | Register value |
+
+**Reply: GenerationRegReadReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| value | `Optional<Value>` | Stored value |
+| gen | `UniqueGeneration` | Write generation |
+| rgen | `UniqueGeneration` | Read generation |
+
+### GenerationRegWriteRequest
+Write to the generation register.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| kv | `KeyValue` (→ `StringRef` key + `StringRef` value) | Key-value to write |
+| gen | `UniqueGeneration` | Generation for this write |
+| reply | `ReplyPromise<UniqueGeneration>` | Actual generation after write |
+
+---
+
+## 4. Cluster Controller Protocol
+
+**Source:** `fdbclient/include/fdbclient/ClusterInterface.h`
+
+The cluster controller manages failure detection, role recruiting, and distributes
+`ClientDBInfo` to clients.
+
+### Interface: ClusterInterface
+
+| Endpoint | Request Type |
+|----------|-------------|
+| openDatabase | `OpenDatabaseRequest` |
+| failureMonitoring | `FailureMonitoringRequest` |
+| databaseStatus | `StatusRequest` |
+| ping | `ReplyPromise<Void>` |
+| getClientWorkers | `GetClientWorkersRequest` |
+| forceRecovery | `ForceRecoveryRequest` |
+| moveShard | `MoveShardRequest` |
+| repairSystemData | `RepairSystemDataRequest` |
+| splitShard | `SplitShardRequest` |
+| triggerAudit | `TriggerAuditRequest` |
+
+### OpenDatabaseRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| clientCount | `int` (int32_t) | Number of clients |
+| issues | `std::map<Key, Samples>` | Client-reported issues |
+| supportedVersions | `std::map<Standalone<ClientVersionRef>, Samples>` | Client versions |
+| maxProtocolSupported | `std::map<Key, Samples>` | Max protocol per client |
+| knownClientInfoID | `UID` | Last known info ID (long-poll) |
+| reply | `ReplyPromise<ClientDBInfo>` | Current database info |
+
+### ClientDBInfo
+Returned to clients; contains everything needed to reach proxies.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| grvProxies | `std::vector<GrvProxyInterface>` | GRV proxy interfaces |
+| commitProxies | `std::vector<CommitProxyInterface>` | Commit proxy interfaces |
+| id | `UID` | Unique ID for this info snapshot |
+| forward | `Optional<Value>` → `Optional<StringRef>` | Forwarding connection string |
+| history | `std::vector<VersionHistory>` | Configuration version history |
+| clusterId | `UID` | Cluster ID |
+| clusterType | `ClusterType` → enum (int) | Cluster type |
+
+Note: the struct also contains `firstCommitProxy` (`Optional<CommitProxyInterface>`)
+but this field is **not serialized** — it is reconstructed locally from `commitProxies`.
+
+### FailureMonitoringRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| senderStatus | `Optional<FailureStatus>` | Sender's own status |
+| failureInformationVersion | `Version` → `int64_t` | Last known failure info version |
+| addresses | `NetworkAddressList` | Sender's addresses |
+| reply | `ReplyPromise<FailureMonitoringReply>` | Failure updates |
+
+**Reply: FailureMonitoringReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| changes | `VectorRef<SystemFailureStatus>` | Changed failure statuses |
+| failureInformationVersion | `Version` → `int64_t` | Updated version |
+| allOthersFailed | `bool` | Whether all other processes are failed |
+| clientRequestIntervalMS | `int` (int32_t) | Suggested poll interval |
+| considerServerFailedTimeoutMS | `int` (int32_t) | Failure timeout threshold |
+| arena | `Arena` | Memory arena |
+
+### StatusRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| statusField | `std::string` | Optional field filter |
+| reply | `ReplyPromise<StatusReply>` | JSON status |
+
+**Reply: StatusReply** — serializes `statusStr` (`std::string`, JSON); `statusObj` is reconstructed on deserialization.
+
+### TriggerAuditRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| type | `uint8_t` | Audit type |
+| range | `KeyRange` | Range to audit |
+| id | `UID` | Audit ID |
+| cancel | `bool` | Whether to cancel |
+| engineType | `KeyValueStoreType` → `uint32_t` | Storage engine type |
+| reply | `ReplyPromise<UID>` | Audit ID |
+
+### Other Cluster Controller Requests
+- **GetClientWorkersRequest**: `reply` → `std::vector<ClientWorkerInterface>`
+- **ForceRecoveryRequest**: `Key dcId`, `reply` → `Void`
+- **MoveShardRequest**: `KeyRange shard`, `std::vector<NetworkAddress> addresses`, `reply` → `Void`
+- **RepairSystemDataRequest**: `reply` → `Void`
+- **SplitShardRequest**: `KeyRange shard`, `int num`, `reply` → **SplitShardReply** {`std::vector<KeyRange> shards`}
+
+---
+
+## 5. Master Protocol
+
+**Source:** `fdbserver/core/include/fdbserver/core/MasterInterface.h`
+
+The master coordinates recovery and hands out commit versions to proxies.
+
+### Interface: MasterInterface
+Serializes `locality` and `waitFailure`; other endpoints reconstructed via offsets.
+
+| Endpoint | Request Type |
+|----------|-------------|
+| waitFailure | `ReplyPromise<Void>` |
+| getCommitVersion | `GetCommitVersionRequest` |
+| getLiveCommittedVersion | `GetRawCommittedVersionRequest` |
+| reportLiveCommittedVersion | `ReportRawCommittedVersionRequest` |
+| updateRecoveryData | `UpdateRecoveryDataRequest` |
+
+### GetCommitVersionRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| spanContext | `SpanContext` | Tracing context |
+| requestNum | `uint64_t` | Monotonic request number |
+| mostRecentProcessedRequestNum | `uint64_t` | Most recent processed request |
+| requestingProxy | `UID` | Proxy ID |
+| reply | `ReplyPromise<GetCommitVersionReply>` | Assigned version |
+
+**Reply: GetCommitVersionReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| resolverChanges | `Standalone<VectorRef<ResolverMoveRef>>` | Resolver shard reassignments |
+| resolverChangesVersion | `Version` → `int64_t` | Version of changes |
+| version | `Version` → `int64_t` | Assigned commit version |
+| prevVersion | `Version` → `int64_t` | Previous commit version |
+| requestNum | `uint64_t` | Echoed request number |
+
+### ReportRawCommittedVersionRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `Version` → `int64_t` | Committed version |
+| locked | `bool` | Database locked |
+| metadataVersion | `Optional<Value>` | Metadata version |
+| minKnownCommittedVersion | `Version` → `int64_t` | Min known committed |
+| prevVersion | `Optional<Version>` | Previous version |
+| writtenTags | `Optional<std::set<Tag>>` | Tags written |
+| reply | `ReplyPromise<Void>` | Acknowledgment |
+
+### UpdateRecoveryDataRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| recoveryTransactionVersion | `Version` → `int64_t` | Recovery txn version |
+| lastEpochEnd | `Version` → `int64_t` | End of last epoch |
+| commitProxies | `std::vector<CommitProxyInterface>` | Proxy interfaces |
+| resolvers | `std::vector<ResolverInterface>` | Resolver interfaces |
+| versionEpoch | `Optional<int64_t>` | Version epoch |
+| primaryLocality | `int8_t` | Primary DC locality |
+| reply | `ReplyPromise<Void>` | Acknowledgment |
+
+### ChangeCoordinatorsRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| newConnectionString | `Standalone<StringRef>` | New connection string |
+| masterId | `UID` | Master ID |
+| reply | `ReplyPromise<Void>` | Acknowledgment |
+
+---
+
+## 6. GRV Proxy Protocol
+
+**Source:** `fdbclient/include/fdbclient/GrvProxyInterface.h`
+
+### Interface: GrvProxyInterface
+Serializes `processId` (`Optional<Key>`), `provisional` (`bool`), and
+`getConsistentReadVersion`; other endpoints reconstructed via offsets.
+
+### GetReadVersionRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| spanContext | `SpanContext` | Tracing context |
+| transactionCount | `uint32_t` | Transactions in batch |
+| flags | `uint32_t` | Priority flags |
+| tags | `TransactionTagMap<uint32_t>` → `std::unordered_map<TransactionTag, uint32_t>` | Tag counts |
+| debugID | `Optional<UID>` | Debug trace ID |
+| maxVersion | `Version` → `int64_t` | Max acceptable version |
+| reply | `ReplyPromise<GetReadVersionReply>` | Read version |
+
+**Reply: GetReadVersionReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| processBusyTime | `double` | Proxy busy time |
+| version | `Version` → `int64_t` | Assigned read version |
+| locked | `bool` | Database locked |
+| metadataVersion | `Optional<Value>` | Metadata version |
+| tagThrottleInfo | `TransactionTagMap<ClientTagThrottleLimits>` | Per-tag throttle limits |
+| midShardSize | `int64_t` | Median shard size |
+| rkDefaultThrottled | `bool` | Ratekeeper throttling default |
+| rkBatchThrottled | `bool` | Ratekeeper throttling batch |
+| ssVersionVectorDelta | `VersionVector` | SS version vector delta (custom serialization) |
+| proxyId | `UID` | Proxy ID |
+
+### GlobalConfigRefreshRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| lastKnown | `Version` → `int64_t` | Last known config version |
+| reply | `ReplyPromise<GlobalConfigRefreshReply>` | Updated config |
+
+**Reply: GlobalConfigRefreshReply**: `Version version`, `RangeResultRef result`, `Arena arena`.
+
+---
+
+## 7. Commit Proxy Protocol
+
+**Source:** `fdbclient/include/fdbclient/CommitProxyInterface.h`
+
+### Interface: CommitProxyInterface
+Serializes `processId` (`Optional<Key>`), `provisional` (`bool`), and `commit`;
+11 other endpoints reconstructed via offsets 1–10 and 13 (offsets 11–12 are
+reserved/unused).
+
+### CommitTransactionRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| spanContext | `SpanContext` | Tracing context |
+| transaction | `CommitTransactionRef` | Transaction to commit |
+| flags | `uint32_t` | Commit flags |
+| debugID | `Optional<UID>` | Debug trace ID |
+| commitCostEstimation | `Optional<ClientTrCommitCostEstimation>` | Cost estimate (see below) |
+| tagSet | `Optional<TagSet>` | Transaction tags |
+| idempotencyId | `IdempotencyIdRef` | Idempotency identifier |
+| arena | `Arena` | Memory arena |
+| reply | `ReplyPromise<CommitID>` | Commit result |
+
+#### ClientTrCommitCostEstimation
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| opsCount | `int` (int32_t) | Operation count |
+| writeCosts | `uint64_t` | Write cost estimate |
+| clearIdxCosts | `std::deque<std::pair<int, uint64_t>>` | Clear index costs |
+| expensiveCostEstCount | `uint32_t` | Expensive estimate count |
+
+**Reply: CommitID**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| version | `Version` → `int64_t` | Committed version |
+| txnBatchId | `uint16_t` | Batch identifier |
+| metadataVersion | `Optional<Value>` | Metadata version |
+| conflictingKRIndices | `Optional<Standalone<VectorRef<int>>>` | Conflicting key range indices |
+
+### GetKeyServerLocationsRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| spanContext | `SpanContext` | Tracing context |
+| begin | `KeyRef` → `StringRef` | Range begin key |
+| end | `Optional<KeyRef>` | Range end key |
+| limit | `int` (int32_t) | Max results |
+| reverse | `bool` | Reverse iteration |
+| legacyVersion | `Version` → `int64_t` | Legacy version |
+| arena | `Arena` | Memory arena |
+| reply | `ReplyPromise<GetKeyServerLocationsReply>` | Locations |
+
+**Reply: GetKeyServerLocationsReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| results | `std::vector<std::pair<KeyRangeRef, std::vector<StorageServerInterface>>>` | Range→SS mapping |
+| resultsTssMapping | `std::vector<std::pair<UID, StorageServerInterface>>` | TSS pairs |
+| resultsTagMapping | `std::vector<std::pair<UID, Tag>>` | SS→Tag mapping |
+| arena | `Arena` | Memory arena |
+
+### Other Commit Proxy Messages
+- **GetRawCommittedVersionRequest**: `SpanContext`, `Optional<UID> debugID`, `Version maxVersion`, `reply` → **GetRawCommittedVersionReply** {`Optional<UID> debugID`, `Version version`, `bool locked`, `Optional<Value> metadataVersion`, `Version minKnownCommittedVersion`, `VersionVector ssVersionVectorDelta`}
+- **GetStorageServerRejoinInfoRequest**: `UID id`, `Optional<Value> dcId`, `reply` → **GetStorageServerRejoinInfoReply** {`Version version`, `Tag tag`, `Optional<Tag> newTag`, `bool newLocality`, `std::vector<std::pair<Version, Tag>> history`, `EncryptionAtRestModeDeprecated encryptMode`}
+- **TxnStateRequest**: `VectorRef<KeyValueRef> data`, `Sequence sequence` (uint64_t), `bool last`, `std::vector<Endpoint> broadcastInfo`, `Arena arena`, `reply` → `Void`
+- **GetHealthMetricsRequest**: `bool detailed`, `reply` → **GetHealthMetricsReply** {`Standalone<StringRef> serialized`}
+- **GetDDMetricsRequest**: `KeyRange keys`, `int shardLimit`, `reply` → **GetDDMetricsReply** {`Standalone<VectorRef<DDMetricsRef>> storageMetricsList`}
+- **ProxySnapRequest**: `StringRef snapPayload`, `UID snapUID`, `Optional<UID> debugID`, `Arena arena`, `reply` → `Void`
+- **ExclusionSafetyCheckRequest**: `std::vector<AddressExclusion> exclusions`, `reply` → **ExclusionSafetyCheckReply** {`bool safe`}
+- **SetThrottledShardRequest**: `std::vector<KeyRange> throttledShards`, `double expirationTime`, `reply` → **SetThrottledShardReply** {}
+- **ExpireIdempotencyIdRequest** (fire-and-forget): `Version commitVersion`, `uint8_t batchIndexHighByte`
+- **GlobalConfigRefreshRequest**: `Version lastKnown`, `reply` → **GlobalConfigRefreshReply** {`Version version`, `RangeResultRef result`, `Arena arena`}
+
+---
+
+## 8. Resolver Protocol
+
+**Source:** `fdbserver/core/include/fdbserver/core/ResolverInterface.h`
+
+### Interface: ResolverInterface
+All endpoints serialized directly: `uniqueID` (`UID`), `locality` (`LocalityData`),
+`resolve`, `metrics`, `split`, `waitFailure`, `txnState`.
+
+### ResolveTransactionBatchRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| spanContext | `SpanContext` | Tracing context |
+| prevVersion | `Version` → `int64_t` | Previous batch version |
+| version | `Version` → `int64_t` | This batch's version |
+| lastReceivedVersion | `Version` → `int64_t` | Last received by this resolver |
+| transactions | `VectorRef<CommitTransactionRef>` | Transactions to check |
+| txnStateTransactions | `VectorRef<int>` | Indices of state transactions |
+| debugID | `Optional<UID>` | Debug trace ID |
+| writtenTags | `std::set<Tag>` | Tags written |
+| lastShardMove | `Version` → `int64_t` | Last shard move version |
+| arena | `Arena` | Memory arena |
+| reply | `ReplyPromise<ResolveTransactionBatchReply>` | Results |
+
+**Reply: ResolveTransactionBatchReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| committed | `VectorRef<uint8_t>` | Per-txn status (0=ok, 1=conflict, ...) |
+| stateMutations | `VectorRef<VectorRef<StateTransactionRef>>` | State mutations |
+| debugID | `Optional<UID>` | Debug trace ID |
+| conflictingKeyRangeMap | `std::map<int, VectorRef<int>>` | Conflict details |
+| privateMutations | `VectorRef<StringRef>` | Private mutations |
+| privateMutationCount | `uint32_t` | Count |
+| tpcvMap | `std::unordered_map<uint16_t, Version>` | Per-proxy committed versions |
+| writtenTags | `std::set<Tag>` | Tags written |
+| lastShardMove | `Version` → `int64_t` | Last shard move |
+| arena | `Arena` | Memory arena |
+
+### Other Resolver Messages
+- **ResolutionMetricsRequest**: `reply` → **ResolutionMetricsReply** {`int64_t value`}
+- **ResolutionSplitRequest**: `KeyRange range`, `int64_t offset`, `bool front`, `reply` → **ResolutionSplitReply** {`Key key`, `int64_t used`}
+
+---
+
+## 9. Transaction Log (TLog) Protocol
+
+**Source:** `fdbserver/core/include/fdbserver/core/TLogInterface.h`
+
+### Interface: TLogInterface
+Serializes `uniqueID` (`UID`), `sharedTLogID` (`UID`), `filteredLocality` (`LocalityData`),
+and `peekMessages`; 12 other endpoints reconstructed via offsets.
+
+### TLogCommitRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| spanContext | `SpanContext` | Tracing context |
+| prevVersion | `Version` → `int64_t` | Previous version |
+| version | `Version` → `int64_t` | Commit version |
+| knownCommittedVersion | `Version` → `int64_t` | Known committed version |
+| minKnownCommittedVersion | `Version` → `int64_t` | Min known committed |
+| seqPrevVersion | `Version` → `int64_t` | Sequential previous version |
+| messages | `StringRef` | Serialized mutation messages |
+| tLogCount | `uint16_t` | Number of TLogs |
+| tLogLocIds | `std::vector<uint16_t>` | TLog locality IDs |
+| debugID | `Optional<UID>` | Debug trace ID |
+| arena | `Arena` | Memory arena |
+| reply | `ReplyPromise<TLogCommitReply>` | Acknowledgment |
+
+**Reply: TLogCommitReply** — `Version version` (int64_t).
+
+### TLogPeekRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| begin | `Version` → `int64_t` | Start version |
+| tag | `Tag` | Tag to peek |
+| returnIfBlocked | `bool` | Return immediately if blocked |
+| onlySpilled | `bool` | Only spilled data |
+| sequence | `Optional<std::pair<UID, int>>` | Ordering sequence |
+| end | `Optional<Version>` | End version |
+| returnEmptyIfStopped | `Optional<bool>` | Return empty if stopped |
+| reply | `ReplyPromise<TLogPeekReply>` | Mutations |
+
+**Reply: TLogPeekReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| messages | `StringRef` | Serialized mutations |
+| end | `Version` → `int64_t` | End version |
+| popped | `Optional<Version>` | Popped version |
+| maxKnownVersion | `Version` → `int64_t` | Max known version |
+| minKnownCommittedVersion | `Version` → `int64_t` | Min known committed |
+| begin | `Optional<Version>` | Actual begin |
+| onlySpilled | `bool` | Only from spill |
+| arena | `Arena` | Memory arena |
+
+### TLogPeekStreamRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| begin | `Version` → `int64_t` | Start version |
+| tag | `Tag` | Tag to peek |
+| returnIfBlocked | `bool` | Return immediately |
+| limitBytes | `int` (int32_t) | Byte limit per reply |
+| end | `Optional<Version>` | End version |
+| returnEmptyIfStopped | `Optional<bool>` | Return empty if stopped |
+| reply | `ReplyPromiseStream<TLogPeekStreamReply>` | Streaming reply |
+
+**Reply: TLogPeekStreamReply** (extends ReplyPromiseStreamReply)
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| acknowledgeToken | `Optional<UID>` | Backpressure token |
+| sequence | `uint16_t` | Sequence number |
+| rep | `TLogPeekReply` | Peek data (see above) |
+
+### Other TLog Messages
+- **TLogPopRequest**: `Version to`, `Version durableKnownCommittedVersion`, `Tag tag`, `reply` → `Void`
+- **TLogQueuingMetricsRequest**: `reply` → **TLogQueuingMetricsReply** {`double localTime`, `int64_t instanceID`, `int64_t bytesDurable`, `int64_t bytesInput`, `StorageBytes storageBytes`, `Version v`}
+- **TLogConfirmRunningRequest**: `Optional<UID> debugID`, `reply` → `Void`
+- **TLogRecoveryFinishedRequest**: `reply` → `Void`
+- **TLogDisablePopRequest**: `UID snapUID`, `Optional<UID> debugID`, `reply` → `Void`
+- **TLogEnablePopRequest**: `UID snapUID`, `Optional<UID> debugID`, `reply` → `Void`
+- **TLogSnapRequest**: `StringRef snapPayload`, `UID snapUID`, `StringRef role`, `Arena arena`, `reply` → `Void`
+- **TrackTLogRecoveryRequest**: `Version oldestGenRecoverAtVersion`, `reply` → **TrackTLogRecoveryReply** {`Version oldestUnrecoveredStartVersion`}
+- **TLogLockResult** (reply to `lock` endpoint): `Version end`, `Version knownCommittedVersion`, `std::deque<UnknownCommittedVersions> unknownCommittedVersions`, `UID id`, `UID logId`
+
+---
+
+## 10. Storage Server Protocol
+
+**Source:** `fdbclient/include/fdbclient/StorageServerInterface.h`
+
+### Interface: StorageServerInterface
+Serializes `uniqueID` (`UID`), `locality` (`LocalityData`), `getValue`,
+`tssPairID` (`Optional<UID>`), and `acceptingRequests` (`bool`); 30+ other
+endpoints reconstructed via offsets.
+
+### GetValueRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Key to read |
+| version | `Version` → `int64_t` | Read version |
+| tags | `Optional<TagSet>` | Transaction tags |
+| reply | `ReplyPromise<GetValueReply>` | Value |
+| spanContext | `SpanContext` | Tracing context |
+| options | `Optional<ReadOptions>` | Read options |
+| ssLatestCommitVersions | `VersionVector` | SS version vector |
+
+**Reply: GetValueReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| penalty | `double` | Load balancing penalty |
+| error | `Optional<Error>` | Error (serialized as error code) |
+| value | `Optional<Value>` → `Optional<StringRef>` | The value |
+| cached | `bool` | From cache |
+
+### GetKeyRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| sel | `KeySelectorRef` | Key selector |
+| version | `Version` → `int64_t` | Read version |
+| tags | `Optional<TagSet>` | Transaction tags |
+| reply | `ReplyPromise<GetKeyReply>` | Resolved key |
+| spanContext | `SpanContext` | Tracing context |
+| options | `Optional<ReadOptions>` | Read options |
+| ssLatestCommitVersions | `VersionVector` | SS version vector |
+| arena | `Arena` | Memory arena |
+
+**Reply: GetKeyReply** — `double penalty`, `Optional<Error> error`, `KeySelector sel`, `bool cached`.
+
+### GetKeyValuesRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| begin | `KeySelectorRef` | Range begin |
+| end | `KeySelectorRef` | Range end |
+| version | `Version` → `int64_t` | Read version |
+| limit | `int` (int32_t) | Row limit |
+| limitBytes | `int` (int32_t) | Byte limit |
+| tags | `Optional<TagSet>` | Transaction tags |
+| reply | `ReplyPromise<GetKeyValuesReply>` | Results |
+| spanContext | `SpanContext` | Tracing context |
+| options | `Optional<ReadOptions>` | Read options |
+| ssLatestCommitVersions | `VersionVector` | SS version vector |
+| taskID | `Optional<TaskPriority>` → `Optional<int>` | Task priority |
+| arena | `Arena` | Memory arena |
+
+**Reply: GetKeyValuesReply** — `double penalty`, `Optional<Error> error`, `VectorRef<KeyValueRef> data`, `Version version`, `bool more`, `bool cached`, `Arena arena`.
+
+### GetMappedKeyValuesRequest
+Same as `GetKeyValuesRequest` plus `KeyRef mapper`. Reply: `VectorRef<MappedKeyValueRef> data`, `Version version`, `bool more`, `bool cached`, `Arena arena`.
+
+### GetKeyValuesStreamRequest
+Same fields as `GetKeyValuesRequest` (without `taskID`) but with `ReplyPromiseStream<GetKeyValuesStreamReply> reply`.
+
+**Reply: GetKeyValuesStreamReply** — `Optional<UID> acknowledgeToken`, `uint16_t sequence`, `VectorRef<KeyValueRef> data`, `Version version`, `bool more`, `bool cached`, `Arena arena`.
+
+### WatchValueRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| key | `Key` → `StringRef` | Key to watch |
+| value | `Optional<Value>` | Expected current value |
+| version | `Version` → `int64_t` | Version of expected value |
+| tags | `Optional<TagSet>` | Transaction tags |
+| debugID | `Optional<UID>` | Debug trace ID |
+| reply | `ReplyPromise<WatchValueReply>` | Watch trigger |
+| spanContext | `SpanContext` | Tracing context |
+
+**Reply: WatchValueReply** — `Version version`, `bool cached`.
+
+### ChangeFeedStreamRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| rangeID | `Key` → `StringRef` | Change feed ID |
+| begin | `Version` → `int64_t` | Start version |
+| end | `Version` → `int64_t` | End version |
+| range | `KeyRange` | Key range |
+| reply | `ReplyPromiseStream<ChangeFeedStreamReply>` | Streaming reply |
+| spanContext | `SpanContext` | Tracing context |
+| replyBufferSize | `int` (int32_t) | Buffer size |
+| canReadPopped | `bool` | Can read popped data |
+| id | `UID` | Request ID |
+| options | `Optional<ReadOptions>` | Read options |
+| encrypted | `bool` | Whether to encrypt |
+| arena | `Arena` | Memory arena |
+
+**Reply: ChangeFeedStreamReply** — `Optional<UID> acknowledgeToken`, `uint16_t sequence`, `VectorRef<MutationsAndVersionRef> mutations`, `bool atLatestVersion`, `Version minStreamVersion`, `Version popVersion`, `Arena arena`.
+
+### Other Storage Server Messages
+- **GetShardStateRequest**: `KeyRange keys`, `int32_t mode`, `bool includePhysicalShard`, `reply` → **GetShardStateReply** {`Version first`, `Version second`, `std::vector<StorageServerShard> shards`}
+- **WaitMetricsRequest**: `KeyRangeRef keys`, `StorageMetrics min`, `StorageMetrics max`, `Version legacyVersion`, `Arena arena`, `reply` → `StorageMetrics`
+- **SplitMetricsRequest**: `KeyRangeRef keys`, `StorageMetrics limits/used/estimated`, `bool isLastShard`, `Optional<int> minSplitBytes`, `Arena arena`, `reply` → **SplitMetricsReply** {`Standalone<VectorRef<KeyRef>> splits`, `StorageMetrics used`, `bool more`}
+- **GetStorageMetricsRequest**: `reply` → **GetStorageMetricsReply** {`StorageMetrics load/available/capacity`, `double bytesInputRate`, `int64_t versionLag`, `double lastUpdate`, `int64_t bytesDurable`, `int64_t bytesInput`, `int ongoingBulkLoadTaskCount`}
+- **StorageQueuingMetricsRequest**: `reply` → **StorageQueuingMetricsReply** {`double localTime`, `int64_t instanceID/bytesDurable/bytesInput`, `StorageBytes storageBytes`, `Version version/durableVersion`, `double cpuUsage/diskUsage/localRateLimit`, `std::vector<BusyTagInfo> busiestTags`}
+- **OverlappingChangeFeedsRequest**: `KeyRange range`, `Version minVersion`, `reply` → **OverlappingChangeFeedsReply** {`VectorRef<OverlappingChangeFeedEntry> feeds`, `Version feedMetadataVersion`, `Arena arena`}
+- **ChangeFeedPopRequest**: `Key rangeID`, `Version version`, `KeyRange range`, `reply` → `Void`
+- **ChangeFeedVersionUpdateRequest**: `Version minVersion`, `reply` → **ChangeFeedVersionUpdateReply** {`Version version`}
+- **GetCheckpointRequest**: `Version version`, `std::vector<KeyRange> ranges`, `int16_t format`, `Optional<UID> actionId`, `reply` → `CheckpointMetaData`
+- **FetchCheckpointRequest**: `UID checkpointID`, `Standalone<StringRef> token`, `reply` (stream) → **FetchCheckpointReply** {`Optional<UID> acknowledgeToken`, `uint16_t sequence`, `Standalone<StringRef> token`, `Standalone<StringRef> data`}
+- **FetchCheckpointKeyValuesRequest**: `UID checkpointID`, `KeyRange range`, `reply` (stream) → **FetchCheckpointKeyValuesStreamReply** {`Optional<UID> acknowledgeToken`, `uint16_t sequence`, `VectorRef<KeyValueRef> data`, `Arena arena`}
+- **ReadHotSubRangeRequest**: `KeyRangeRef keys`, `uint8_t type`, `int chunkCount`, `Arena arena`, `reply` → **ReadHotSubRangeReply** {`Standalone<VectorRef<ReadHotRangeWithMetrics>> readHotRanges`}
+- **SplitRangeRequest**: `KeyRangeRef keys`, `int64_t chunkSize`, `Arena arena`, `reply` → **SplitRangeReply** {`Standalone<VectorRef<KeyRef>> splitPoints`}
+- **UpdateCommitCostRequest**: `UID ratekeeperID`, `double postTime/elapsed`, `TransactionTag busiestTag`, `int opsSum`, `uint64_t costSum/totalWriteCosts`, `bool reported`, `reply` → `Void`
+- **GetHotShardsRequest**: `reply` → **GetHotShardsReply** {`std::vector<KeyRange> hotShards`}
+- **GetStorageCheckSumRequest**: `std::vector<std::pair<KeyRange, Optional<Version>>> ranges`, `Optional<UID> actionId`, `uint8_t checkSumMethod`, `reply` → **GetStorageCheckSumReply** {`std::vector<CheckSumMetaData> checkSums`, `uint8_t checkSumMethod`}
+- **BulkDumpRequest**: `std::vector<UID> checksumServers`, `BulkDumpState bulkDumpState`, `reply` → `BulkDumpState`
+- **AuditStorageRequest**: `UID id/ddId`, `KeyRange range`, `uint8_t type`, `std::vector<UID> targetServers`, `reply` → `Void`
+
+---
+
+## 11. Data Distributor Protocol
+
+**Source:** `fdbserver/core/include/fdbserver/core/DataDistributorInterface.h`
+
+### Interface: DataDistributorInterface
+Serializes all endpoints directly: `waitFailure`, `haltDataDistributor`,
+`locality` (`LocalityData`), `myId` (`UID`), and 7 additional `RequestStream`s.
+
+- **HaltDataDistributorRequest**: `UID requesterID`, `reply` → `Void`
+- **GetDataDistributorMetricsRequest**: `KeyRange keys`, `int shardLimit`, `bool midOnly`, `reply` → **GetDataDistributorMetricsReply** {`Standalone<VectorRef<DDMetricsRef>> storageMetricsList`, `Optional<int64_t> midShardSize`}
+- **DistributorSnapRequest**: `StringRef snapPayload`, `UID snapUID`, `Optional<UID> debugID`, `Arena arena`, `reply` → `Void`
+- **DistributorExclusionSafetyCheckRequest**: `std::vector<AddressExclusion> exclusions`, `reply` → **DistributorExclusionSafetyCheckReply** {`bool safe`}
+- **DistributorSplitRangeRequest**: `std::vector<Key> splitPoints`, `reply` → **SplitShardReply** {`std::vector<KeyRange> shards`}
+- **GetStorageWigglerStateRequest**: `reply` → **GetStorageWigglerStateReply** {`uint8_t primary`, `uint8_t remote`}
+- **PrepareBlobRestoreRequest**: `UID requesterID`, `StorageServerInterface ssi`, `KeyRange keys`, `reply` → **PrepareBlobRestoreReply** {`int8_t res`}
+
+---
+
+## 12. Ratekeeper Protocol
+
+**Source:** `fdbserver/core/include/fdbserver/core/RatekeeperInterface.h`
+
+### Interface: RatekeeperInterface
+Serializes all endpoints directly: `waitFailure`, `getRateInfo`, `haltRatekeeper`,
+`reportCommitCostEstimation`, `getSSVersionLag`, `locality` (`LocalityData`), `myId` (`UID`).
+
+### GetRateInfoRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| requesterID | `UID` | Proxy ID |
+| totalReleasedTransactions | `int64_t` | Total released |
+| batchReleasedTransactions | `int64_t` | Batch released |
+| version | `Version` → `int64_t` | Current version |
+| throttledTagCounts | `TransactionTagMap<uint64_t>` | Throttled tag counts |
+| detailed | `bool` | Include detailed metrics |
+| reply | `ReplyPromise<GetRateInfoReply>` | Rate limits |
+
+**Reply: GetRateInfoReply**
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| transactionRate | `double` | Default TPS limit |
+| batchTransactionRate | `double` | Batch TPS limit |
+| leaseDuration | `double` | Rate lease duration |
+| healthMetrics | `HealthMetrics` | Cluster health |
+| clientThrottledTags | `Optional<PrioritizedTransactionTagMap<ClientTagThrottleLimits>>` | Client throttles |
+
+`PrioritizedTransactionTagMap<T>` is `std::map<TransactionPriority, TransactionTagMap<T>>` where `TransactionPriority` is a `uint8_t` enum.
+
+### Other Ratekeeper Messages
+- **HaltRatekeeperRequest**: `UID requesterID`, `reply` → `Void`
+- **ReportCommitCostEstimationRequest**: `UIDTransactionTagMap<TransactionCommitCostEstimation> ssTrTagCommitCost`, `reply` → `Void`
+- **GetSSVersionLagRequest**: `reply` → **GetSSVersionLagReply** {`Version maxPrimarySSVersion`, `Version maxRemoteSSVersion`}
+
+---
+
+## 13. Worker Protocol
+
+**Source:** `fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h`
+
+Workers host server roles. The cluster controller sends initialization requests.
+
+### Interface: WorkerInterface
+Serializes all 24 `RequestStream` endpoints and `TesterInterface` directly.
+
+### RegisterWorkerRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| wi | `WorkerInterface` | Worker's interface |
+| initialClass | `ProcessClass` | Initial class |
+| processClass | `ProcessClass` | Current class |
+| priorityInfo | `ClusterControllerPriorityInfo` | Priority info |
+| generation | `Generation` → `int64_t` | Cluster generation |
+| distributorInterf | `Optional<DataDistributorInterface>` | DD interface |
+| ratekeeperInterf | `Optional<RatekeeperInterface>` | RK interface |
+| consistencyScanInterf | `Optional<ConsistencyScanInterface>` | CS interface |
+| issues | `Standalone<VectorRef<StringRef>>` | Worker issues |
+| incompatiblePeers | `std::vector<NetworkAddress>` | Incompatible peers |
+| degraded | `bool` | Worker degraded |
+| recoveredDiskFiles | `bool` | Disk files recovered |
+| clusterId | `Optional<UID>` | Cluster ID |
+| reply | `ReplyPromise<RegisterWorkerReply>` | Registration result |
+
+**Reply: RegisterWorkerReply** — `ProcessClass processClass`, `ClusterControllerPriorityInfo priorityInfo`.
+
+### Role Initialization Requests
+
+All follow the pattern: fields describing the role configuration + `ReplyPromise<RoleInterface>`.
+
+- **RecruitMasterRequest**: `LifetimeToken lifetime`, `bool forceRecovery`, `reply` → `MasterInterface`
+- **InitializeTLogRequest**: `UID recruitmentID`, `LogSystemConfig recoverFrom`, `Version recoverAt/knownCommittedVersion`, `LogEpoch epoch`, `std::vector<Tag> recoverTags/allTags`, `TLogVersion logVersion`, `KeyValueStoreType storeType`, `TLogSpillType spillType`, `Tag remoteTag`, `int8_t locality`, `bool isPrimary`, `Version startVersion`, `int logRouterTags/txsTags`, `Version recoveryTransactionVersion`, `std::vector<Version> oldGenerationRecoverAtVersions`, `reply` → `TLogInterface`
+- **InitializeCommitProxyRequest**: `MasterInterface master`, `LifetimeToken masterLifetime`, `uint64_t recoveryCount`, `Version recoveryTransactionVersion`, `bool firstProxy`, `EncryptionAtRestModeDeprecated encryptMode`, `uint16_t commitProxyIndex`, `reply` → `CommitProxyInterface`
+- **InitializeGrvProxyRequest**: `MasterInterface master`, `LifetimeToken masterLifetime`, `uint64_t recoveryCount`, `reply` → `GrvProxyInterface`
+- **InitializeResolverRequest**: `LifetimeToken masterLifetime`, `uint64_t recoveryCount`, `int commitProxyCount/resolverCount`, `UID masterId`, `EncryptionAtRestModeDeprecated encryptMode`, `reply` → `ResolverInterface`
+- **InitializeStorageRequest**: `Tag seedTag`, `UID reqId/interfaceId`, `KeyValueStoreType storeType`, `Optional<std::pair<UID, Version>> tssPairIDAndVersion`, `Version initialClusterVersion`, `EncryptionAtRestModeDeprecated encryptMode`, `reply` → **InitializeStorageReply** {`StorageServerInterface interf`, `Version addedVersion`}
+- **InitializeDataDistributorRequest**: `UID reqId`, `reply` → `DataDistributorInterface`
+- **InitializeRatekeeperRequest**: `UID reqId`, `reply` → `RatekeeperInterface`
+- **InitializeConsistencyScanRequest**: `UID reqId`, `reply` → `ConsistencyScanInterface`
+- **InitializeLogRouterRequest**: `uint64_t recoveryCount`, `Tag routerTag`, `Version startVersion`, `std::vector<LocalityData> tLogLocalities`, `Reference<IReplicationPolicy> tLogPolicy`, `int8_t locality`, `Optional<Version> recoverAt`, `Optional<std::map<uint8_t, std::vector<uint16_t>>> knownLockedTLogIds`, `bool allowDropInSim/isReplacement`, `UID reqId`, `reply` → `TLogInterface`
+- **InitializeBackupRequest**: `UID reqId`, `LogEpoch recruitedEpoch/backupEpoch`, `Tag routerTag`, `int totalTags`, `Version startVersion`, `Optional<Version> endVersion`, `reply` → **InitializeBackupReply** {`BackupInterface interf`, `LogEpoch backupEpoch`}
+
+### Recruitment Requests
+- **RecruitFromConfigurationRequest**: `DatabaseConfiguration configuration`, `bool recruitSeedServers`, `int maxOldLogRouters`, `reply` → **RecruitFromConfigurationReply** {`std::vector<WorkerInterface>` for: tLogs, satelliteTLogs, commitProxies, grvProxies, resolvers, storageServers, oldLogRouters, backupWorkers; `Optional<Key> dcId`, `bool satelliteFallback`}
+- **RecruitRemoteFromConfigurationRequest**: `DatabaseConfiguration configuration`, `Optional<Key> dcId`, `int logRouterCount`, `std::vector<UID> exclusionWorkerIds`, `Optional<UID> dbgId`, `reply` → **RecruitRemoteFromConfigurationReply** {`std::vector<WorkerInterface> remoteTLogs/logRouters`, `Optional<UID> dbgId`}
+- **RecruitStorageRequest**: `std::vector<Optional<Standalone<StringRef>>> excludeMachines/includeDCs`, `std::vector<AddressExclusion> excludeAddresses`, `bool criticalRecruitment`, `reply` → **RecruitStorageReply** {`WorkerInterface worker`, `ProcessClass processClass`}
+
+### Other Worker Messages
+- **RegisterMasterRequest**: `UID id`, `LocalityData mi`, `LogSystemConfig logSystemConfig`, `std::vector<CommitProxyInterface> commitProxies`, `std::vector<GrvProxyInterface> grvProxies`, `std::vector<ResolverInterface> resolvers`, `DBRecoveryCount recoveryCount`, `int64_t registrationCount`, `Optional<DatabaseConfiguration> configuration`, `std::vector<UID> priorCommittedLogServers`, `RecoveryState recoveryState`, `bool recoveryStalled` (no reply)
+- **GetWorkersRequest**: `int flags`, `reply` → `std::vector<WorkerDetails>` where **WorkerDetails** = {`WorkerInterface interf`, `ProcessClass processClass`, `bool degraded`, `bool recoveredDiskFiles`}
+- **TLogRejoinRequest**: `TLogInterface myInterface`, `reply` → **TLogRejoinReply** {`bool masterIsRecovered`}
+- **BackupWorkerDoneRequest**: `UID workerUID`, `LogEpoch backupEpoch`, `reply` → `Void`
+- **GetEncryptionAtRestModeRequest**: `UID tlogId`, `reply` → **GetEncryptionAtRestModeResponse** {`uint32_t mode`}
+- **CoordinationPingMessage** (fire-and-forget): `UID clusterControllerId`, `int64_t timeStep`
+- **SetMetricsLogRateRequest** (fire-and-forget): `uint32_t metricsLogsPerSecond`
+- **UpdateWorkerHealthRequest** (fire-and-forget): `NetworkAddress address`, `std::vector<NetworkAddress> degradedPeers/disconnectedPeers/recoveredPeers`
+- **LoadedPingRequest**: `UID id`, `bool loadReply`, `Standalone<StringRef> payload`, `reply` → **LoadedReply** {`Standalone<StringRef> payload`, `UID id`}
+- **EventLogRequest**: `bool getLastError`, `Standalone<StringRef> eventName`, `reply` → `TraceEventFields`
+- **TraceBatchDumpRequest**: `reply` → `Void`
+- **ExecuteRequest**: `StringRef execPayload`, `Arena arena`, `reply` → `Void`
+- **WorkerSnapRequest**: `StringRef snapPayload`, `UID snapUID`, `StringRef role`, `Arena arena`, `reply` → `Void`
+- **DiskStoreRequest**: `bool includePartialStores`, `reply` → `Standalone<VectorRef<UID>>`
+
+---
+
+## 14. Backup / Consistency Scan / Test Protocols
+
+### BackupInterface
+`RequestStream<ReplyPromise<Void>> waitFailure`, `LocalityData locality`.
+
+### ConsistencyScanInterface
+`RequestStream<ReplyPromise<Void>> waitFailure`, `RequestStream<HaltConsistencyScanRequest> haltConsistencyScan`, `LocalityData locality`, `UID myId`.
+
+**HaltConsistencyScanRequest**: `UID requesterID`, `reply` → `Void`.
+
+### TesterInterface
+`RequestStream<WorkloadRequest> recruitments`.
+
+**WorkloadRequest**: `StringRef title`, `int timeout`, `double databasePingDelay`, `int64_t sharedRandomNumber`, `bool useDatabase/runFailureWorkloads`, `VectorRef<VectorRef<KeyValueRef>> options`, `std::vector<KeyRange> rangesToCheck`, `int clientId/clientCount`, `std::vector<std::string> disabledFailureInjectionWorkloads`, `Arena arena`, `reply` → **WorkloadInterface** {`RequestStream` endpoints for setup/start/check/metrics/stop}.
+
+### NetworkTestRequest
+`Key key`, `uint32_t replySize`, `reply` → **NetworkTestReply** {`Value value`}.
+
+### NetworkTestStreamingRequest
+`reply` (stream) → **NetworkTestStreamingReply** {`Optional<UID> acknowledgeToken`, `uint16_t sequence`, `int index`}.
+
+---
+
+## 15. Client Worker / Debug / Process Protocols
+
+### ClientWorkerInterface
+`RequestStream<RebootRequest> reboot`, `RequestStream<ProfilerRequest> profiler`, `RequestStream<SetFailureInjection> setFailureInjection`, `Optional<NetworkAddress> grpcAddress`.
+
+- **RebootRequest** (fire-and-forget): `bool deleteData`, `bool checkData`, `uint32_t waitForDuration`
+- **ProfilerRequest**: `Type` (enum/int), `Action` (enum/int), `int duration`, `Standalone<StringRef> outputFile`, `reply` → `Void`
+- **SetFailureInjection**: `Optional<DiskFailureCommand> diskFailure`, `Optional<FlipBitsCommand> flipBits`, `reply` → `Void`
+
+### ProcessInterface
+`RequestStream<GetProcessInterfaceRequest> getInterface`, `RequestStream<ActorLineageRequest> actorLineage`.
+
+- **GetProcessInterfaceRequest**: `reply` → `ProcessInterface`
+- **ActorLineageRequest**: `WaitState waitStateStart/waitStateEnd` (enum/int), `time_t timeStart/timeEnd` (int64_t), `reply` → **ActorLineageReply** {`std::vector<SerializedSample> samples`}
+
+---
+
+## Appendix A: Log System Configuration Types
+
+**Source:** `fdbserver/core/include/fdbserver/core/LogSystemConfig.h`,
+`fdbserver/core/include/fdbserver/core/DBCoreState.h`,
+`fdbserver/core/include/fdbserver/core/ServerDBInfo.h`
+
+### OptionalInterface\<T\>
+Conditionally serializes a full interface or just its UID.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| iface | `Optional<T>` | Full interface (if present) |
+| ident | `UID` | Interface ID (serialized only if iface is absent) |
+
+### TLogSet
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tLogs | `std::vector<OptionalInterface<TLogInterface>>` | TLog interfaces |
+| logRouters | `std::vector<OptionalInterface<TLogInterface>>` | Log router interfaces |
+| backupWorkers | `std::vector<OptionalInterface<BackupInterface>>` | Backup worker interfaces |
+| tLogWriteAntiQuorum | `int32_t` | Write anti-quorum |
+| tLogReplicationFactor | `int32_t` | Replication factor |
+| tLogPolicy | `Reference<IReplicationPolicy>` | Replication policy (custom serialization) |
+| tLogLocalities | `std::vector<LocalityData>` | TLog localities |
+| isLocal | `bool` | Whether this is the local DC |
+| locality | `int8_t` | DC locality |
+| startVersion | `Version` → `int64_t` | Start version |
+| satelliteTagLocations | `std::vector<std::vector<int>>` | Satellite tag locations |
+| tLogVersion | `TLogVersion` → `uint32_t` | TLog version |
+
+### OldTLogConf
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tLogs | `std::vector<TLogSet>` | TLog sets |
+| epochBegin | `Version` → `int64_t` | Epoch begin version |
+| epochEnd | `Version` → `int64_t` | Epoch end version |
+| recoverAt | `Version` → `int64_t` | Recovery version |
+| logRouterTags | `int32_t` | Number of log router tags |
+| txsTags | `int32_t` | Number of txs tags |
+| pseudoLocalities | `std::set<int8_t>` | Pseudo-localities |
+| epoch | `LogEpoch` → `uint64_t` | Epoch number |
+
+### LogSystemConfig
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| logSystemType | `LogSystemType` → enum (int) | Always `tagPartitioned` (2) |
+| tLogs | `std::vector<TLogSet>` | Current TLog sets |
+| logRouterTags | `int32_t` | Log router tag count |
+| txsTags | `int32_t` | Txs tag count |
+| oldTLogs | `std::vector<OldTLogConf>` | Previous generation configs |
+| expectedLogSets | `int32_t` | Expected log set count |
+| recruitmentID | `UID` | Recruitment ID |
+| stopped | `bool` | Whether stopped |
+| recoveredAt | `Optional<Version>` | Recovery version |
+| pseudoLocalities | `std::set<int8_t>` | Pseudo-localities |
+| epoch | `LogEpoch` → `uint64_t` | Current epoch |
+| oldestBackupEpoch | `LogEpoch` → `uint64_t` | Oldest backup epoch |
+| knownLockedTLogIds | `std::map<uint8_t, std::vector<uint16_t>>` | Known locked TLog IDs |
+
+### CoreTLogSet
+Persisted on coordinators (UIDs only, no full interfaces).
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tLogs | `std::vector<UID>` | TLog UIDs |
+| tLogWriteAntiQuorum | `int32_t` | Write anti-quorum |
+| tLogReplicationFactor | `int32_t` | Replication factor |
+| tLogPolicy | `Reference<IReplicationPolicy>` | Replication policy |
+| tLogLocalities | `std::vector<LocalityData>` | TLog localities |
+| isLocal | `bool` | Local DC |
+| locality | `int8_t` | DC locality |
+| startVersion | `Version` → `int64_t` | Start version |
+| satelliteTagLocations | `std::vector<std::vector<int>>` | Satellite locations |
+| tLogVersion | `TLogVersion` → `uint32_t` | TLog version |
+
+### OldTLogCoreData
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tLogs | `std::vector<CoreTLogSet>` | Core TLog sets |
+| logRouterTags | `int32_t` | Log router tags |
+| txsTags | `int32_t` | Txs tags |
+| epochBegin | `Version` → `int64_t` | Epoch begin |
+| epochEnd | `Version` → `int64_t` | Epoch end |
+| recoverAt | `Version` → `int64_t` | Recovery version (conditional on protocol version) |
+| pseudoLocalities | `std::set<int8_t>` | Pseudo-localities |
+| epoch | `LogEpoch` → `uint64_t` | Epoch |
+
+### DBCoreState
+Persisted on coordinators — the ground truth for recovery.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| tLogs | `std::vector<CoreTLogSet>` | Current TLog sets |
+| logRouterTags | `int32_t` | Log router tags |
+| txsTags | `int32_t` | Txs tags |
+| oldTLogData | `std::vector<OldTLogCoreData>` | Previous generations |
+| recoveryCount | `DBRecoveryCount` → `uint64_t` | Recovery counter |
+| logSystemType | `LogSystemType` → enum (int) | Log system type |
+| pseudoLocalities | `std::set<int8_t>` | Pseudo-localities |
+| newestProtocolVersion | `ProtocolVersion` → `uint64_t` | Newest protocol (conditional) |
+| lowestCompatibleProtocolVersion | `ProtocolVersion` → `uint64_t` | Lowest compatible (conditional) |
+| encryptionAtRestModeDeprecated | `EncryptionAtRestModeDeprecated` → `uint32_t` | Encryption mode (conditional) |
+
+### ServerDBInfo
+Distributed to all server processes.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| id | `UID` | Info snapshot ID |
+| clusterInterface | `ClusterControllerFullInterface` | CC interface |
+| client | `ClientDBInfo` | Client-facing info |
+| distributor | `Optional<DataDistributorInterface>` | DD interface |
+| master | `MasterInterface` | Master interface |
+| ratekeeper | `Optional<RatekeeperInterface>` | RK interface |
+| consistencyScan | `Optional<ConsistencyScanInterface>` | CS interface |
+| resolvers | `std::vector<ResolverInterface>` | Resolver interfaces |
+| recoveryCount | `DBRecoveryCount` → `uint64_t` | Recovery counter |
+| recoveryState | `RecoveryState` → enum (int) | Recovery state |
+| masterLifetime | `LifetimeToken` | Master lifetime token |
+| logSystemConfig | `LogSystemConfig` | Log system config |
+| priorCommittedLogServers | `std::vector<UID>` | Prior committed log servers |
+| latencyBandConfig | `Optional<LatencyBandConfig>` | Latency band config |
+| infoGeneration | `int64_t` | Info generation counter |
+
+### LatencyBandConfig
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| grvConfig | `GrvConfig` | GRV latency bands |
+| readConfig | `ReadConfig` | Read latency bands |
+| commitConfig | `CommitConfig` | Commit latency bands |
+
+Each config inherits from **RequestConfig** {`std::set<double> bands`} and adds:
+- **ReadConfig**: `Optional<int> maxReadBytes`, `Optional<int> maxKeySelectorOffset`
+- **CommitConfig**: `Optional<int> maxCommitBytes`
+
+### UpdateServerDBInfoRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| serializedDbInfo | `Standalone<StringRef>` | Serialized `ServerDBInfo` |
+| broadcastInfo | `std::vector<Endpoint>` | Endpoints to broadcast to |
+| reply | `ReplyPromise<std::vector<Endpoint>>` | Updated endpoints |
+
+### GetServerDBInfoRequest
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| knownServerInfoID | `UID` | Last known info ID (long-poll) |
+| reply | `ReplyPromise<ServerDBInfo>` | Server DB info |
+
+---
+
+## Appendix B: In-Band Log Messages
+
+**Source:** `fdbserver/core/include/fdbserver/core/LogProtocolMessage.h`,
+`fdbserver/core/include/fdbserver/core/SpanContextMessage.h`,
+`fdbserver/core/include/fdbserver/core/OTELSpanContextMessage.h`
+
+These messages are embedded in the TLog mutation stream (`messages` field of
+`TLogCommitRequest` / `TLogPeekReply`). They use reserved `MutationRef` type codes.
+
+### LogProtocolMessage
+Marks a protocol version change in the log stream.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| (type code) | `uint8_t` | `MutationRef::Reserved_For_LogProtocolMessage` |
+| (version) | `ProtocolVersion` → `uint64_t` | New protocol version |
+
+### SpanContextMessage
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| (type code) | `uint8_t` | `MutationRef::Reserved_For_SpanContextMessage` |
+| spanContext | `SpanID` → `UID` (two `uint64_t`) | Tracing span context |
+
+### OTELSpanContextMessage
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| (type code) | `uint8_t` | `MutationRef::Reserved_For_OTELSpanContextMessage` |
+| spanContext | `SpanContext` | OpenTelemetry span context |
+
+---
+
+## Appendix C: Backpressure Protocol
+
+**Source:** `fdbrpc/include/fdbrpc/fdbrpc.h`
+
+For streaming replies (`ReplyPromiseStream`), FDB implements flow control via
+acknowledgments:
+
+### AcknowledgementReply
+Sent from client to server to acknowledge received bytes.
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| bytes | `int64_t` | Total bytes acknowledged |
+
+The server tracks `bytesSent` vs `bytesAcknowledged` and pauses when the
+difference exceeds `bytesLimit`. On cancellation, the client sends
+`operation_obsolete` on the acknowledgment stream.
+
+### ReplyPromiseStreamReply (base for all streaming replies)
+
+| Field | Type | Purpose |
+|-------|------|---------|
+| acknowledgeToken | `Optional<UID>` | Ack endpoint token (sent in first reply) |
+| sequence | `uint16_t` | Monotonic sequence number |
+
+---
+
+## Appendix D: Client Library Protocol Flows
+
+**Source:** `fdbclient/NativeAPI.actor.cpp`, `fdbclient/MonitorLeader.cpp`,
+`fdbclient/GlobalConfig.cpp`
+
+This appendix describes the exact sequence of protocol messages the FDB client
+library sends during each operation, as implemented in the NativeAPI layer.
+
+### D.1 Cluster Discovery and Connection
+
+The client's first task is to discover the cluster controller and obtain the
+current proxy list. This happens continuously in the background.
+
+**Step 1: Parse connection file.**
+The client reads the cluster file (e.g., `fdb.cluster`) containing
+`description:id@coordinator1:port,coordinator2:port,...`. This yields a
+`ClusterConnectionString`.
+
+**Step 2: Contact coordinators (monitorProxies loop).**
+The client enters an infinite loop that monitors proxy changes. For each
+coordinator, it creates a `ClientLeaderRegInterface` and sends:
+
+```
+Client → Coordinator:  OpenDatabaseCoordRequest
+    clusterKey:          Key (from connection string)
+    hostnames:           std::vector<Hostname>
+    coordinators:        std::vector<NetworkAddress>
+    knownClientInfoID:   UID (last known ClientDBInfo.id, for long-poll)
+    supportedVersions:   std::vector<ClientVersionRef>
+    traceLogGroup:       Key
+    internal:            bool
+```
+
+The coordinator responds with `CachedSerialization<ClientDBInfo>`, which the
+client deserializes to obtain:
+
+```
+Coordinator → Client:  ClientDBInfo  (serialization order)
+    grvProxies:     std::vector<GrvProxyInterface>
+    commitProxies:  std::vector<CommitProxyInterface>
+    id:             UID
+    forward:        Optional<Value>   (if set, reconnect to new coordinators)
+    history:        std::vector<VersionHistory>
+    clusterId:      UID
+    clusterType:    ClusterType
+```
+
+Note: `firstCommitProxy` is a struct member but is NOT serialized — it is
+reconstructed locally.
+
+If `forward` is set, the client switches to the new connection string and
+restarts discovery. Otherwise, it stores the proxy lists and long-polls
+(blocks until `id` changes).
+
+**Step 3: Proxy list management.**
+The client randomly shuffles the proxy lists and truncates them to
+`MAX_COMMIT_PROXY_CONNECTIONS` and `MAX_GRV_PROXY_CONNECTIONS` to limit
+connection fan-out. It selects `firstCommitProxy` for operations that must
+go to a single proxy.
+
+### D.2 Get Read Version (GRV)
+
+Every transaction begins by acquiring a read version.
+
+**Step 1: Check GRV cache.**
+If GRV caching is enabled and a cached version is recent enough (within
+`MAX_VERSION_CACHE_LAG`), the client returns the cached version immediately
+with no network round-trip.
+
+**Step 2: Batch GRV requests.**
+Multiple concurrent transactions' GRV needs are batched together by the
+`readVersionBatcher` actor. It collects requests until `MAX_BATCH_SIZE` is
+reached or `GRV_BATCH_TIMEOUT` expires, then sends a single request.
+
+**Step 3: Send batched request to GRV proxy.**
+
+```
+Client → GrvProxy:  GetReadVersionRequest
+    spanContext:       SpanContext
+    transactionCount:  uint32_t  (number of transactions in batch)
+    flags:             uint32_t  (PRIORITY_SYSTEM_IMMEDIATE | PRIORITY_DEFAULT | PRIORITY_BATCH)
+    tags:              TransactionTagMap<uint32_t>  (tag counts for throttling)
+    debugID:           Optional<UID>
+    maxVersion:        Version  (max version in client's version vector cache)
+```
+
+Proxy selection: `basicLoadBalance()` across `grvProxies[]` with round-robin.
+
+**Step 4: Process reply.**
+
+```
+GrvProxy → Client:  GetReadVersionReply
+    processBusyTime:           double
+    version:                   Version    (the assigned read version)
+    locked:                    bool
+    metadataVersion:           Optional<Value>
+    midShardSize:              int64_t
+    rkDefaultThrottled:        bool
+    rkBatchThrottled:          bool
+    tagThrottleInfo:           TransactionTagMap<ClientTagThrottleLimits>
+    ssVersionVectorDelta:      VersionVector
+    proxyId:                   UID
+```
+
+The client caches the version, updates its tag throttle table, and applies
+the storage server version vector delta for read-your-writes consistency.
+
+### D.3 Key Location Discovery
+
+Before reading, the client must know which storage servers hold the target keys.
+
+**Step 1: Check location cache.**
+The client maintains a `KeyRangeMap` mapping key ranges → storage server
+interfaces. If the target range is cached, skip to the read.
+
+**Step 2: Request locations from commit proxy.**
+
+```
+Client → CommitProxy:  GetKeyServerLocationsRequest
+    spanContext:    SpanContext
+    begin:          KeyRef       (start of range)
+    end:            Optional<KeyRef>  (end of range, if range query)
+    limit:          int          (max shard locations, typically 100)
+    reverse:        bool
+    legacyVersion:  Version
+    arena:          Arena
+```
+
+Proxy selection: `commitProxyLoadBalance()` across `commitProxies[]`.
+
+**Step 3: Cache response.**
+
+```
+CommitProxy → Client:  GetKeyServerLocationsReply
+    results:            std::vector<pair<KeyRangeRef, std::vector<StorageServerInterface>>>
+    resultsTssMapping:  std::vector<pair<UID, StorageServerInterface>>
+    resultsTagMapping:  std::vector<pair<UID, Tag>>
+    arena:              Arena
+```
+
+Each result maps a key range to the set of storage server replicas hosting it.
+The client caches these mappings. TSS (Testing Storage Server) pairs are stored
+separately for read validation.
+
+### D.4 Reading Data
+
+#### Single Key Read (getValue)
+
+```
+Client → StorageServer:  GetValueRequest  (serialization order)
+    key:                     Key
+    version:                 Version       (transaction's read version)
+    tags:                    Optional<TagSet>
+    reply:                   ReplyPromise<GetValueReply>
+    spanContext:             SpanContext
+    options:                 Optional<ReadOptions>
+    ssLatestCommitVersions:  VersionVector
+```
+
+Server selection: `loadBalance()` across replicas from the location cache,
+with locality-aware preference for same-DC servers.
+
+```
+StorageServer → Client:  GetValueReply
+    penalty:  double          (load balancing hint)
+    error:    Optional<Error>
+    value:    Optional<Value> (None if key not found)
+    cached:   bool
+```
+
+#### Range Read (getRange / getKeyValues)
+
+```
+Client → StorageServer:  GetKeyValuesRequest  (serialization order)
+    begin:                   KeySelectorRef
+    end:                     KeySelectorRef
+    version:                 Version
+    limit:                   int     (row limit; negative for reverse)
+    limitBytes:              int     (byte limit)
+    tags:                    Optional<TagSet>
+    reply:                   ReplyPromise<GetKeyValuesReply>
+    spanContext:             SpanContext
+    options:                 Optional<ReadOptions>
+    ssLatestCommitVersions:  VersionVector
+    taskID:                  Optional<TaskPriority>
+    arena:                   Arena
+```
+
+```
+StorageServer → Client:  GetKeyValuesReply
+    penalty:  double
+    error:    Optional<Error>
+    data:     VectorRef<KeyValueRef>  (key-value pairs)
+    version:  Version
+    more:     bool    (true if more data available)
+    cached:   bool
+    arena:    Arena
+```
+
+If `more` is true, the client adjusts the key selectors and sends another
+request. If the range spans multiple shards, the client sends parallel
+requests to different storage servers and merges results.
+
+**Error handling during reads:**
+- `wrong_shard_server`: Shard moved; invalidate location cache, re-lookup, retry.
+- `all_alternatives_failed`: All replicas failed; same as wrong_shard_server.
+- `future_version`: Requested version not yet available; wait and retry.
+- `transaction_too_old`: Version expired; restart transaction with new GRV.
+
+#### Streaming Range Read (getKeyValuesStream)
+
+For large range reads, the client can use the streaming variant:
+
+```
+Client → StorageServer:  GetKeyValuesStreamRequest
+    (same fields as GetKeyValuesRequest, without taskID)
+    reply:  ReplyPromiseStream<GetKeyValuesStreamReply>
+```
+
+This returns a stream of `GetKeyValuesStreamReply` messages with backpressure
+(see Appendix C). Each reply carries `acknowledgeToken` and `sequence` for
+flow control.
+
+### D.5 Committing a Transaction
+
+**Step 1: Build commit request.**
+The client accumulates mutations (set, clear, atomic operations) and conflict
+ranges during the transaction. When `commit()` is called:
+
+```
+Client → CommitProxy:  CommitTransactionRequest
+    spanContext:           SpanContext
+    transaction:           CommitTransactionRef
+        read_conflict_ranges:   VectorRef<KeyRangeRef>
+        write_conflict_ranges:  VectorRef<KeyRangeRef>
+        mutations:              VectorRef<MutationRef>
+        read_snapshot:          Version  (the read version)
+        report_conflicting_keys: bool
+        lock_aware:             bool
+        spanContext:            Optional<SpanContext>
+        tenantIds:              Optional<VectorRef<int64_t>>
+    flags:                 uint32_t  (LOCK_AWARE, FIRST_IN_BATCH, etc.)
+    debugID:               Optional<UID>
+    commitCostEstimation:  Optional<ClientTrCommitCostEstimation>
+    tagSet:                Optional<TagSet>
+    idempotencyId:         IdempotencyIdRef
+    arena:                 Arena
+```
+
+Proxy selection: `basicLoadBalance()` across `commitProxies[]`, or
+`firstCommitProxy` if `COMMIT_ON_FIRST_PROXY` option is set.
+
+**Step 2: Wait for reply.**
+
+```
+CommitProxy → Client:  CommitID
+    version:                Version    (committed version, or invalidVersion on failure)
+    txnBatchId:             uint16_t   (for constructing versionstamps)
+    metadataVersion:        Optional<Value>
+    conflictingKRIndices:   Optional<Standalone<VectorRef<int>>>
+```
+
+**Step 3: Handle result.**
+- **Success** (`version != invalidVersion`): Transaction is committed. The
+  client constructs a `Versionstamp` from `version` and `txnBatchId`, updates
+  its committed version, and sends `ExpireIdempotencyIdRequest` if using
+  idempotency.
+- **Conflict** (`not_committed`): Transaction conflicted; throw to caller.
+- **Unknown** (`commit_unknown_result`): Proxy may have crashed after committing.
+  The client can retry with idempotency to determine the outcome.
+
+### D.6 Watching a Key
+
+```
+Client → StorageServer:  WatchValueRequest
+    spanContext:  SpanContext
+    key:          Key
+    value:        Optional<Value>  (expected current value)
+    version:      Version          (version at which value was read)
+    tags:         Optional<TagSet>
+    debugID:      Optional<UID>
+```
+
+This is a long-poll: the storage server holds the request until the key's value
+changes, then replies:
+
+```
+StorageServer → Client:  WatchValueReply
+    version:  Version  (version at which value changed)
+    cached:   bool
+```
+
+The client then waits for `waitForCommittedVersion(version)` to ensure the
+change is durable before triggering the watch future. If the watch is cancelled
+by a `watch_cancelled` error (too many concurrent watches), the client retries
+with backoff.
+
+### D.7 Change Feed Streaming
+
+```
+Client → StorageServer:  ChangeFeedStreamRequest
+    spanContext:      SpanContext
+    rangeID:          Key           (change feed identifier)
+    begin:            Version       (start version)
+    end:              Version       (end version)
+    range:            KeyRange
+    replyBufferSize:  int
+    canReadPopped:    bool
+    id:               UID
+    options:          Optional<ReadOptions>
+    encrypted:        bool
+    arena:            Arena
+    reply:            ReplyPromiseStream<ChangeFeedStreamReply>
+```
+
+Returns a stream of `ChangeFeedStreamReply` messages, each containing:
+```
+    mutations:        VectorRef<MutationsAndVersionRef>
+    atLatestVersion:  bool
+    minStreamVersion: Version
+    popVersion:       Version
+```
+
+### D.8 Global Configuration Refresh
+
+Runs periodically in the background:
+
+```
+Client → GrvProxy:  GlobalConfigRefreshRequest
+    lastKnown:  Version  (last known config version)
+```
+
+```
+GrvProxy → Client:  GlobalConfigRefreshReply
+    version:  Version
+    result:   RangeResultRef  (config key-value pairs)
+    arena:    Arena
+```
+
+### D.9 Error Handling and Retry Strategy
+
+| Error | Meaning | Client Action |
+|-------|---------|---------------|
+| `wrong_shard_server` | Key range relocated | Invalidate cache, retry |
+| `all_alternatives_failed` | All replicas unreachable | Same as wrong_shard_server |
+| `transaction_too_old` | Read version expired | Get new GRV, restart transaction |
+| `future_version` | Version not yet available | Wait `FUTURE_VERSION_RETRY_DELAY`, retry |
+| `not_committed` | Transaction conflicted | Throw conflict error to caller |
+| `commit_unknown_result` | Proxy crash mid-commit | Verify via idempotency or retry |
+| `commit_proxy_memory_limit_exceeded` | Proxy overloaded | Exponential backoff |
+| `batch_transaction_throttled` | Batch priority throttled | Retry after delay |
+
+Backoff: starts at `CLIENT_KNOBS->BACKOFF_DELAY`, multiplied by
+`BACKOFF_GROWTH_RATE` on each failure, capped at
+`RESOURCE_CONSTRAINED_MAX_BACKOFF`.
+
+---
+
+## Appendix E: fdbcli Protocol Usage
+
+**Source:** `fdbcli/*.actor.cpp`, `fdbclient/include/fdbclient/ManagementAPI.h`
+
+The `fdbcli` command-line tool uses three protocol layers:
+1. **Normal transactions** via the NativeAPI (same as any client).
+2. **Special Key Space writes** — transactions that write to `\xff\xff/management/`
+   keys, which the commit proxy intercepts and translates into cluster operations.
+3. **Direct RPC** to ClusterInterface or worker endpoints.
+
+### E.1 Direct RPC Commands
+
+These commands send protocol messages directly to cluster endpoints, bypassing
+the transaction layer.
+
+#### `status` Command
+```
+fdbcli → ClusterController:  StatusRequest
+    statusField:  std::string  (optional filter like "json")
+    reply:        ReplyPromise<StatusReply>
+```
+Reply: `StatusReply` containing `statusStr` (JSON string).
+
+#### `force_recovery_with_data_loss <DCID>`
+```
+fdbcli → ClusterController:  ForceRecoveryRequest
+    dcId:   Key  (datacenter ID)
+    reply:  ReplyPromise<Void>
+```
+
+#### `kill` / `suspend` / `expensive_data_check`
+First discovers workers via:
+```
+fdbcli → ClusterController:  GetClientWorkersRequest
+    reply:  ReplyPromise<std::vector<ClientWorkerInterface>>
+```
+
+Then sends to each target worker:
+```
+fdbcli → Worker:  RebootRequest  (fire-and-forget)
+    deleteData:       bool      (false for kill/suspend)
+    checkData:        bool      (true for expensive_data_check)
+    waitForDuration:  uint32_t  (0 for kill, >0 for suspend seconds)
+```
+
+#### `profile` Command (flow/heap profiling)
+```
+fdbcli → Worker:  ProfilerRequest
+    type:       Type enum    (GPROF, FLOW, GPROF_HEAP)
+    action:     Action enum  (ENABLE, DISABLE, RUN)
+    duration:   int
+    outputFile: Standalone<StringRef>
+    reply:      ReplyPromise<Void>
+```
+
+#### `audit_storage` Command
+```
+fdbcli → ClusterController:  TriggerAuditRequest
+    type:       uint8_t  (ValidateHA, ValidateReplica, ValidateLocationMetadata, ...)
+    range:      KeyRange
+    id:         UID
+    cancel:     bool
+    engineType: KeyValueStoreType (uint32_t)
+    reply:      ReplyPromise<UID>
+```
+
+#### `snapshot` Command
+Uses `IDatabase::createSnapshot()` which sends `ProxySnapRequest` to a commit
+proxy. The proxy then forwards a `DistributorSnapRequest` to the data distributor,
+which orchestrates the actual snapshot across TLogs and storage servers.
+
+### E.2 Special Key Space Commands
+
+These commands write to `\xff\xff/management/` keys within a transaction with
+`SPECIAL_KEY_SPACE_ENABLE_WRITES` option. The commit proxy intercepts these
+writes and performs the corresponding cluster operations.
+
+#### `exclude` / `include` Commands
+```
+Write to: \xff\xff/management/excluded/<ADDRESS> = ""
+Write to: \xff\xff/management/failed/<ADDRESS> = ""
+Write to: \xff\xff/management/excluded_locality/<LOCALITY> = ""
+Write to: \xff\xff/management/failed_locality/<LOCALITY> = ""
+
+Options:
+Write to: \xff\xff/management/options/excluded/force = ""   (skip safety check)
+
+Read from: \xff\xff/management/in_progress_exclusion/<ADDRESS>  (check progress)
+```
+
+#### `configure` Command
+Uses `ManagementAPI::changeConfig()` which:
+1. Reads current configuration from `\xff/conf/` system keys.
+2. Writes updated configuration via a normal transaction.
+3. The commit proxy detects system key changes and triggers reconfiguration.
+
+#### `coordinators` Command
+```
+Write to: \xff\xff/configuration/coordinators/processes = "<addr1>,<addr2>,..."
+Write to: \xff\xff/configuration/coordinators/cluster_description = "<name>"
+
+Read from: \xff\xff/management/auto_coordinators  (get auto-selected coordinators)
+```
+
+#### `lock` / `unlock` Commands
+```
+Write to: \xff\xff/management/db_locked = <UID>   (lock)
+Clear:    \xff\xff/management/db_locked             (unlock, requires matching UID)
+```
+
+#### `maintenance` Command
+```
+Write to: \xff\xff/management/maintenance/<ZONEID> = <duration_seconds>
+Write to: \xff\xff/management/maintenance/IgnoreSSFailures = ""  (disable DD for SS failures)
+Clear:    \xff\xff/management/maintenance/<ZONEID>  (end maintenance)
+```
+
+#### `datadistribution` Command
+```
+Write to: \xff\xff/management/data_distribution/mode = "0" or "1"  (off/on)
+Write to: \xff\xff/management/data_distribution/rebalance_ignored = <flags>
+```
+
+#### `advanceversion` Command
+```
+Write to: \xff\xff/management/min_required_commit_version = <version>
+```
+
+#### `throttle` Command
+Uses `ThrottleApi` functions which read/write:
+```
+\xff\x02/throttledTags/tag/<tag>/<type>/<priority> = TagThrottleValue
+\xff\x02/throttledTags/autoThrottledTags/<tag>/<type>/<priority> = TagThrottleValue
+```
+
+#### `setclass` Command
+Uses `ManagementAPI::setClass()` which writes to system keys to change a
+worker's process class assignment.
+
+### E.3 Backup / Restore CLI Commands
+
+Backup and restore commands use the `FileBackupAgent` and `DatabaseBackupAgent`
+classes, which operate through normal transactions reading/writing system keys:
+
+```
+\xff\x02/fdbbackup/        — backup task state
+\xff\x02/fdbrestore/       — restore task state
+\xff\x02/backupProgress/   — backup progress markers
+\xff\x02/backupStarted/    — backup start markers
+```
+
+These are standard key-value operations — the backup agents are implemented as
+Taskbucket workflows that run as normal transactions within the cluster.
+
+### E.4 Transaction Options Used by fdbcli
+
+| Option | Purpose |
+|--------|---------|
+| `SPECIAL_KEY_SPACE_ENABLE_WRITES` | Enable writes to `\xff\xff/` keys |
+| `PRIORITY_SYSTEM_IMMEDIATE` | Highest priority (bypass throttling) |
+| `READ_SYSTEM_KEYS` | Allow reading `\xff/` system keys |
+| `LOCK_AWARE` | Operate while database is locked |
+| `ACCESS_SYSTEM_KEYS` | Full access to system keyspace |
+
+---
+
+## Appendix F: Server-Side Protocol Flows
+
+**Source:** `fdbserver/commitproxy/CommitProxyServer.actor.cpp`,
+`fdbserver/clustercontroller/ClusterRecovery.cpp`,
+`fdbserver/storageserver/storageserver.actor.cpp`,
+`fdbserver/sequencer/masterserver.cpp`
+
+### F.1 Transaction Commit Path
+
+This is the critical write path: how a client's `CommitTransactionRequest`
+becomes durable. The commit proxy orchestrates a multi-phase protocol.
+
+```
+┌────────┐     ┌──────────────┐     ┌────────┐     ┌──────────┐     ┌───────┐
+│ Client │────▶│ Commit Proxy │────▶│ Master │     │ Resolver │     │ TLog  │
+└────────┘     └──────────────┘     └────────┘     └──────────┘     └───────┘
+                      │                  │               │               │
+                      │ GetCommitVersion │               │               │
+                      │─────────────────▶│               │               │
+                      │   Reply(version) │               │               │
+                      │◀─────────────────│               │               │
+                      │                                  │               │
+                      │  ResolveTransactionBatchRequest   │               │
+                      │─────────────────────────────────▶│               │
+                      │  Reply(committed[], conflicts)   │               │
+                      │◀─────────────────────────────────│               │
+                      │                                                  │
+                      │              TLogCommitRequest                   │
+                      │─────────────────────────────────────────────────▶│
+                      │              TLogCommitReply                     │
+                      │◀─────────────────────────────────────────────────│
+                      │                  │                               │
+                      │ ReportRawCommittedVersion                        │
+                      │─────────────────▶│                               │
+```
+
+**Phase 1: Get Commit Version**
+
+The commit proxy batches incoming `CommitTransactionRequest` messages and
+requests a commit version from the master:
+
+```
+CommitProxy → Master:  GetCommitVersionRequest
+    spanContext:                      SpanContext
+    requestNum:                      uint64_t  (monotonic per-proxy)
+    mostRecentProcessedRequestNum:   uint64_t
+    requestingProxy:                 UID
+```
+
+```
+Master → CommitProxy:  GetCommitVersionReply
+    resolverChanges:         Standalone<VectorRef<ResolverMoveRef>>
+    resolverChangesVersion:  Version
+    version:                 Version  (assigned commit version)
+    prevVersion:             Version
+    requestNum:              uint64_t
+```
+
+The master assigns monotonically increasing versions. If resolver topology has
+changed, it includes `resolverChanges` so the proxy knows which resolver
+handles which key ranges.
+
+**Phase 2: Conflict Resolution**
+
+The proxy sends the transaction batch to each resolver. Each resolver is
+responsible for a range of keys. Transactions whose read/write conflict ranges
+span multiple resolvers are sent to all relevant resolvers.
+
+```
+CommitProxy → Resolver[i]:  ResolveTransactionBatchRequest
+    spanContext:           SpanContext
+    prevVersion:           Version
+    version:               Version
+    lastReceivedVersion:   Version
+    transactions:          VectorRef<CommitTransactionRef>
+    txnStateTransactions:  VectorRef<int>
+    debugID:               Optional<UID>
+    writtenTags:           std::set<Tag>
+    lastShardMove:         Version
+    arena:                 Arena
+```
+
+```
+Resolver[i] → CommitProxy:  ResolveTransactionBatchReply
+    committed:               VectorRef<uint8_t>  (per-txn: 0=ok, 1=conflict)
+    stateMutations:          VectorRef<VectorRef<StateTransactionRef>>
+    conflictingKeyRangeMap:  std::map<int, VectorRef<int>>
+    privateMutations:        VectorRef<StringRef>
+    privateMutationCount:    uint32_t
+    tpcvMap:                 std::unordered_map<uint16_t, Version>
+    writtenTags:             std::set<Tag>
+    lastShardMove:           Version
+    arena:                   Arena
+```
+
+The proxy merges results from all resolvers. A transaction is committed only
+if all resolvers agree it has no conflicts.
+
+**Phase 3: TLog Push**
+
+For committed transactions, the proxy serializes mutations into a `LogPushData`
+object. Each mutation is tagged with the `Tag` of the storage server(s) that
+should receive it. The proxy then pushes to all TLog replicas:
+
+```
+CommitProxy → TLog[j]:  TLogCommitRequest
+    spanContext:              SpanContext
+    prevVersion:              Version
+    version:                  Version
+    knownCommittedVersion:    Version
+    minKnownCommittedVersion: Version
+    seqPrevVersion:           Version
+    messages:                 StringRef  (packed, tagged mutations)
+    tLogCount:                uint16_t
+    tLogLocIds:               std::vector<uint16_t>
+    debugID:                  Optional<UID>
+    arena:                    Arena
+```
+
+The commit is durable when a quorum of TLog replicas acknowledge:
+
+```
+TLog[j] → CommitProxy:  TLogCommitReply
+    version:  Version  (durable version)
+```
+
+**Phase 4: Report Committed Version**
+
+```
+CommitProxy → Master:  ReportRawCommittedVersionRequest
+    version:                  Version
+    locked:                   bool
+    metadataVersion:          Optional<Value>
+    minKnownCommittedVersion: Version
+    prevVersion:              Optional<Version>
+    writtenTags:              Optional<std::set<Tag>>
+```
+
+The master updates its committed version. The proxy then replies to the
+original client with `CommitID`.
+
+### F.2 Recovery Protocol
+
+Recovery happens when the cluster controller detects that the master has
+failed, or during initial cluster startup. It is a complex multi-phase
+protocol that establishes a new transaction system generation.
+
+```
+Phase 0:  CC recruits new roles
+Phase 1:  Lock old TLogs to freeze previous generation
+Phase 2:  Determine recovery point (last committed version)
+Phase 3:  Recruit and initialize new TLogs
+Phase 4:  Initialize resolvers and proxies with transaction state
+Phase 5:  Write new DBCoreState to coordinators
+Phase 6:  Begin accepting commits
+```
+
+**Phase 0: Role Recruitment**
+
+```
+ClusterController → Workers:  RecruitFromConfigurationRequest
+    configuration:      DatabaseConfiguration
+    recruitSeedServers: bool
+    maxOldLogRouters:   int
+```
+
+```
+Workers → ClusterController:  RecruitFromConfigurationReply
+    tLogs:           std::vector<WorkerInterface>
+    satelliteTLogs:  std::vector<WorkerInterface>
+    commitProxies:   std::vector<WorkerInterface>
+    grvProxies:      std::vector<WorkerInterface>
+    resolvers:       std::vector<WorkerInterface>
+    storageServers:  std::vector<WorkerInterface>
+    oldLogRouters:   std::vector<WorkerInterface>
+    backupWorkers:   std::vector<WorkerInterface>
+    dcId:            Optional<Key>
+    satelliteFallback: bool
+```
+
+**Phase 1: Lock Old TLogs**
+
+The master sends a lock request to every TLog from the previous generation:
+
+```
+Master → OldTLog[i]:  TLogInterface::lock  (ReplyPromise<TLogLockResult>)
+```
+
+```
+OldTLog[i] → Master:  TLogLockResult
+    end:                       Version    (last version on this TLog)
+    knownCommittedVersion:     Version
+    unknownCommittedVersions:  std::deque<UnknownCommittedVersions>
+    id:                        UID
+    logId:                     UID
+```
+
+The master gathers all lock results to determine the recovery point: the
+highest version that reached a quorum of TLogs.
+
+**Phase 2: Initialize New TLogs**
+
+```
+ClusterController → NewTLog[j]:  InitializeTLogRequest
+    recruitmentID:    UID
+    recoverFrom:      LogSystemConfig  (previous generation's config)
+    recoverAt:        Version
+    knownCommittedVersion: Version
+    epoch:            LogEpoch
+    recoverTags:      std::vector<Tag>
+    allTags:          std::vector<Tag>
+    logVersion:       TLogVersion
+    storeType:        KeyValueStoreType
+    spillType:        TLogSpillType
+    remoteTag:        Tag
+    locality:         int8_t
+    isPrimary:        bool
+    startVersion:     Version
+    logRouterTags:    int
+    txsTags:          int
+    recoveryTransactionVersion: Version
+    oldGenerationRecoverAtVersions: std::vector<Version>
+```
+
+```
+NewTLog[j] → ClusterController:  TLogInterface
+    (the new TLog's interface, with all its RequestStream endpoints)
+```
+
+**Phase 3: Initialize Proxies and Resolvers**
+
+```
+CC → NewProxy:  InitializeCommitProxyRequest
+    master:                     MasterInterface
+    masterLifetime:             LifetimeToken
+    recoveryCount:              uint64_t
+    recoveryTransactionVersion: Version
+    firstProxy:                 bool
+    encryptMode:                EncryptionAtRestModeDeprecated
+    commitProxyIndex:           uint16_t
+```
+
+```
+CC → NewResolver:  InitializeResolverRequest
+    masterLifetime:  LifetimeToken
+    recoveryCount:   uint64_t
+    commitProxyCount: int
+    resolverCount:   int
+    masterId:        UID
+    encryptMode:     EncryptionAtRestModeDeprecated
+```
+
+**Phase 4: Broadcast Transaction State**
+
+The master broadcasts the transaction system state (metadata about shards,
+server assignments, etc.) to all resolvers:
+
+```
+Master → Resolver[i]:  TxnStateRequest  (sent in chunks)
+    data:           VectorRef<KeyValueRef>  (transaction state KV pairs)
+    sequence:       Sequence  (chunk sequence number)
+    last:           bool      (true for final chunk)
+    broadcastInfo:  std::vector<Endpoint>
+```
+
+**Phase 5: Write to Coordinators**
+
+The master writes the new `DBCoreState` to the coordination service using
+`GenerationRegWriteRequest`. This is the atomic commit point — once this
+write succeeds, the new generation is official.
+
+**Phase 6: Accept Commits**
+
+The master updates `ServerDBInfo` and broadcasts it to all workers:
+
+```
+CC → Workers:  UpdateServerDBInfoRequest
+    serializedDbInfo:  Standalone<StringRef>  (serialized ServerDBInfo)
+    broadcastInfo:     std::vector<Endpoint>
+```
+
+The master then sets `recoveryReadyForCommits`, allowing commit proxies to
+begin processing client transactions.
+
+### F.3 Storage Server Data Flow
+
+Storage servers pull mutations from TLogs using a continuous peek cursor.
+The storage server calls `logSystem->peekSingle()` which creates a cursor
+for its assigned tag. By default (knob `PEEK_USING_STREAMING = false`), this
+uses regular `TLogPeekRequest` messages. If `PEEK_USING_STREAMING` is enabled,
+it uses the streaming `TLogPeekStreamRequest` variant instead.
+
+**Steady-State Mutation Pull (default, non-streaming):**
+
+```
+StorageServer → TLog:  TLogPeekRequest
+    begin:              Version
+    tag:                Tag     (this SS's tag)
+    returnIfBlocked:    bool
+    onlySpilled:        bool
+    sequence:           Optional<std::pair<UID, int>>
+    end:                Optional<Version>
+    returnEmptyIfStopped: Optional<bool>
+```
+
+```
+TLog → StorageServer:  TLogPeekReply
+    messages:                  StringRef  (packed mutations)
+    end:                       Version
+    popped:                    Optional<Version>
+    maxKnownVersion:           Version
+    minKnownCommittedVersion:  Version
+    begin:                     Optional<Version>
+    onlySpilled:               bool
+    arena:                     Arena
+```
+
+**Streaming variant (when `PEEK_USING_STREAMING = true`):**
+
+```
+StorageServer → TLog:  TLogPeekStreamRequest
+    begin:              Version
+    tag:                Tag
+    returnIfBlocked:    bool
+    limitBytes:         int
+    end:                Optional<Version>
+    returnEmptyIfStopped: Optional<bool>
+    reply:              ReplyPromiseStream<TLogPeekStreamReply>
+```
+
+The storage server deserializes mutations from `messages`, applies them to its
+in-memory versioned data structure, and periodically makes them durable.
+
+**Pop (Garbage Collection):**
+
+Once a storage server has durably written mutations, it tells the TLog to
+discard them:
+
+```
+StorageServer → TLog:  TLogPopRequest
+    to:                           Version  (pop up to this version)
+    durableKnownCommittedVersion: Version
+    tag:                          Tag
+```
+
+**Shard Movement (Data Distribution):**
+
+When data distribution moves a shard to a new storage server:
+
+1. The new SS begins peeking the TLog for the shard's tag at the transfer version.
+2. The new SS uses `Transaction::getRange()` (or `getRangeStream()` if
+   `FETCH_USING_STREAMING` is enabled) to read the shard's data. This goes through
+   the normal client path: commit proxy for location lookup, then storage server
+   reads via `GetKeyValuesRequest` (or `GetKeyValuesStreamRequest`). The reads
+   target the old storage servers that still hold the shard.
+3. Once caught up, the new SS transitions the shard to `ReadWrite` state.
+4. Data distribution updates the shard map (via system key transactions through
+   the commit proxy), and future `GetKeyServerLocationsReply` messages will
+   direct clients to the new SS.
diff --git a/design/AI-generated/README.md b/design/AI-generated/README.md
new file mode 100644
index 00000000000..d7e35407c0a
--- /dev/null
+++ b/design/AI-generated/README.md
@@ -0,0 +1,63 @@
+# AI-generated FDB code overview
+
+This directory contains an AI-generated code overview of the current `main` code base as of April, 2026.
+
+This was generated using a commercially available coding assistant in widespread use.  We are not disclosing
+the specific model used.  It took about an hour and cost about $20 in API usage.
+
+Prompts:
+
+*Ok I want you to study the foundationdb code in the current
+  directory.  It should be unmodified main.  I am not interested in
+  specific diffs or PRs.  I am interested in understanding the entire
+  system.  What are the major modules?  Do not rely too much on
+  filenames.  Instead try to figure out the flow of data through the
+  system and the dynamic relationships more than just the static
+  arrangements.  I would like some assessment of the code in terms of
+  a partition of subsystems that covers materially the entire code
+  base, but not too granular.  I am thinking of 10-15 subsystems at
+  most (or fewer if that is justified by the structure of the code).
+  Tell me the major subsystems, the principle files in each, what they
+  are responsible for, and so on.  Then we can do deep dives on each
+  one.  But for now I want an overview of the whole system.*
+
+This generated the file [`foundationdb_subsystem_map.md`](foundationdb_subsystem_map.md).
+
+Subsequently:
+
+*for each of the 12 subsystems identified above, please study the code
+ in depth and generate a module-specific
+ architecture/design/implementation overview, focusing on the key data
+ structures, methods, and data flow in the system.  Please save each
+ subsystem-specific overview to its own .md file*
+
+This generated the 12 files whose names have a `subsystem_` prefix.
+
+## Rationale and relationship to existing documentation
+
+Basically we did this on a Friday afternoon to see what the AI would
+come up with.  On cursory examination it seems pretty reasonable.  We
+were pleased to note that the first thing mentioned in the first file
+we reviewed is `SAV`, which is of central importance to the system but
+is buried in `flow.h` and is hardly mentioned in ../*.md.
+
+In any case, we hope this is usable as a starting point for people new
+to the system or new to specific areas.  It is meant to supplement,
+not replace, the existing documentation in the parent directory.
+
+Note that the industry standard for developer-maintained documentation
+of software systems (not just FDB) is generally considered to be
+*widely uneven*.  Some areas have in-depth, well-written deep dives.
+Some areas are out of date.  Some areas are not covered at all.  This
+is just the nature of things.  AI-generated documentation, by
+contrast, will be more comprehensive in coverage, but not quite as
+in-depth.  It might have random errors and omissions, but
+human-maintained documentation certainly suffers from that too.
+
+## Caveats
+
+We have not reviewed this in detail. Errors/omissions will be
+incrementally addressed.  Feel free to send PRs to make corrections.
+At some point in a few quarters it may be appropriate to completely
+regenerate this documentation with more advanced AI models or
+more creative prompts.
diff --git a/design/AI-generated/diagram_01_flow_runtime.md b/design/AI-generated/diagram_01_flow_runtime.md
new file mode 100644
index 00000000000..3fc47498cc6
--- /dev/null
+++ b/design/AI-generated/diagram_01_flow_runtime.md
@@ -0,0 +1,76 @@
+# Flow Runtime — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph ActorSystem["Actor System"]
+        Actor["ACTOR function\n(.actor.cpp source)"]
+        Compiler["Actor Compiler\n(Python preprocessor)"]
+        StateMachine["Generated State Machine\n(.actor.g.h)"]
+        Coro["C++20 Coroutine\n(co_await migration)"]
+    end
+
+    subgraph AsyncPrimitives["Async Primitives"]
+        Promise["Promise&lt;T&gt;"]
+        Future["Future&lt;T&gt;"]
+        SAV["SAV\n(SingleAssignmentVar)"]
+        PS["PromiseStream&lt;T&gt;"]
+        FS["FutureStream&lt;T&gt;"]
+    end
+
+    subgraph EventLoop["Event Loop (Net2)"]
+        Scheduler["Priority Task Scheduler"]
+        ASIO["Boost.ASIO\n(I/O multiplex)"]
+        Timers["Timer Queue"]
+        Yields["Yield Points"]
+    end
+
+    subgraph Memory["Memory Management"]
+        Arena["Arena"]
+        StringRef["StringRef / KeyRef\n(non-owning view)"]
+        Standalone["Standalone&lt;T&gt;\n(owning)"]
+        RefCounted["Reference&lt;T&gt;\n(intrusive refcount)"]
+    end
+
+    subgraph Diagnostics["Diagnostics"]
+        Trace["TraceEvent\n(.detail() chains)"]
+        DRandom["deterministicRandom()\n(seedable PRNG)"]
+        CodeProbe["CODE_PROBE\n(coverage marks)"]
+        Buggify["BUGGIFY\n(fault injection)"]
+    end
+
+    Actor --> Compiler --> StateMachine
+    Actor -.->|migrating to| Coro
+    StateMachine --> Future
+    Promise --> SAV --> Future
+    PS --> FS
+
+    Future --> Scheduler
+    Timers --> Scheduler
+    Scheduler --> ASIO
+
+    Arena --> StringRef
+    Arena --> Standalone
+
+    style ActorSystem fill:#e1f0ff,stroke:#4a90d9
+    style AsyncPrimitives fill:#fff3e0,stroke:#f5a623
+    style EventLoop fill:#e8f5e9,stroke:#4caf50
+    style Memory fill:#fce4ec,stroke:#e91e63
+    style Diagnostics fill:#f3e5f5,stroke:#9c27b0
+```
+## Future/Promise Lifecycle
+
+```mermaid
+sequenceDiagram
+    participant Producer as Producer Actor
+    participant SAV as SAV (SharedState)
+    participant Consumer as Consumer Actor
+    participant Scheduler as Net2 Scheduler
+
+    Producer->>SAV: Promise created (SAV allocated)
+    Consumer->>SAV: Future obtained (refcount++)
+    Consumer->>SAV: wait(future) — register callback
+    Note over Consumer: Suspended
+    Producer->>SAV: promise.send(value)
+    SAV->>Scheduler: enqueue callback
+    Scheduler->>Consumer: Resume with value
+```
diff --git a/design/AI-generated/diagram_02_rpc_transport.md b/design/AI-generated/diagram_02_rpc_transport.md
new file mode 100644
index 00000000000..17371aa7fcc
--- /dev/null
+++ b/design/AI-generated/diagram_02_rpc_transport.md
@@ -0,0 +1,109 @@
+# RPC & Transport — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph Transport["FlowTransport (singleton)"]
+        SendR["sendReliable()"]
+        SendU["sendUnreliable()"]
+        Listen["listen()"]
+    end
+
+    subgraph Peers["Peer Management"]
+        Peer["Peer\n(per-destination)"]
+        UnsentQ["Unsent Queue"]
+        ReliableQ["Reliable Packet List"]
+        ConnFuture["Connection Future\n(lazy connect)"]
+    end
+
+    subgraph Addressing["Addressing"]
+        Endpoint["Endpoint\n= (NetworkAddressList, Token)"]
+        EndpointMap["EndpointMap\n(Token → Receiver)"]
+        WellKnown["Well-Known Tokens\n(system services)"]
+    end
+
+    subgraph RequestReply["Request/Reply Pattern"]
+        ReqStream["RequestStream&lt;Req&gt;"]
+        RP["ReplyPromise&lt;Rep&gt;\n(self-addressed envelope)"]
+        Interface["Server Interface\n(struct of RequestStreams)"]
+    end
+
+    subgraph Monitoring["Failure Monitoring"]
+        FM["FailureMonitor"]
+        OnFailed["onFailed(Endpoint)\n→ Future&lt;Void&gt;"]
+        OnStateChange["onStateChanged(Endpoint)"]
+    end
+
+    subgraph SimNetwork["Simulation Network (Sim2)"]
+        VirtProcess["Virtual Processes\n(in-thread)"]
+        VirtTime["Deterministic Time"]
+        Clogging["Network Clogging"]
+        Partitions["Network Partitions"]
+        FaultInj["Fault Injection\n(kill process/machine/DC)"]
+    end
+
+    Interface --> ReqStream
+    ReqStream --> SendR
+    SendR --> Peer --> UnsentQ
+    Peer --> ReliableQ
+    Peer --> ConnFuture
+
+    Listen --> EndpointMap --> Endpoint
+    RP --> SendR
+
+    FM --> OnFailed
+    FM --> OnStateChange
+
+    SimNetwork -.->|replaces| Transport
+
+    style Transport fill:#e1f0ff,stroke:#4a90d9
+    style Peers fill:#fff3e0,stroke:#f5a623
+    style Addressing fill:#e8f5e9,stroke:#4caf50
+    style RequestReply fill:#fce4ec,stroke:#e91e63
+    style Monitoring fill:#f3e5f5,stroke:#9c27b0
+    style SimNetwork fill:#fffde7,stroke:#fbc02d
+```
+
+## Message Flow Between Processes
+
+```mermaid
+sequenceDiagram
+    participant A as Actor A (Process 1)
+    participant FT1 as FlowTransport (P1)
+    participant Net as Network / Sim2
+    participant FT2 as FlowTransport (P2)
+    participant EM as EndpointMap (P2)
+    participant B as Actor B (Process 2)
+
+    A->>FT1: send(Endpoint, Request{ReplyPromise})
+    FT1->>FT1: serialize(Request)
+    FT1->>Net: TCP send / sim route
+    Net->>FT2: deliver packet
+    FT2->>EM: lookup(Token)
+    EM->>B: receive(Request)
+    B->>B: process request
+    B->>FT2: ReplyPromise.send(Reply)
+    FT2->>Net: TCP send / sim route
+    Net->>FT1: deliver reply
+    FT1->>A: Future resolved with Reply
+```
+
+## Locality & Placement
+
+```mermaid
+graph LR
+    subgraph LocalityData
+        Zone["zoneId"]
+        Machine["machineId"]
+        DC["dcId"]
+        DataHall["data_hall"]
+    end
+
+    subgraph ProcessClass
+        Fitness["Fitness for Role"]
+        ClassType["ClassType\n(storage, transaction,\nresolution, stateless)"]
+    end
+
+    LocalityData --> ReplicationPolicy["Replication Policy\n(e.g. 3 zones)"]
+    ProcessClass --> Recruitment["CC Role Recruitment"]
+    ReplicationPolicy --> TeamBuilding["DD Team Building"]
+```
diff --git a/design/AI-generated/diagram_03_client_library.md b/design/AI-generated/diagram_03_client_library.md
new file mode 100644
index 00000000000..944e6f80b51
--- /dev/null
+++ b/design/AI-generated/diagram_03_client_library.md
@@ -0,0 +1,104 @@
+# Client Library — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph Layers["Transaction Layers (outside → in)"]
+        App["Application Code"]
+        CAPI["C API (fdb_c.h)\n/ Language Bindings"]
+        MV["MultiVersionTransaction\n(protocol negotiation)"]
+        RYW["ReadYourWritesTransaction\n(local write map + cache)"]
+        Native["NativeAPI / Transaction\n(raw operations)"]
+    end
+
+    subgraph ReadPath["Read Operations"]
+        Get["get(key)"]
+        GetRange["getRange(begin, end)"]
+        LocCache["Location Cache\n(KeyRange → SS list)"]
+        SS["Storage Server"]
+    end
+
+    subgraph WritePath["Write Operations"]
+        Set["set(key, value)"]
+        Clear["clear(key/range)"]
+        WriteMap["Local Write Map"]
+        Commit["commit()"]
+        CP["Commit Proxy"]
+    end
+
+    subgraph VersionMgmt["Version Management"]
+        GRV["getReadVersion()"]
+        GRVProxy["GRV Proxy"]
+        Watch["watch(key)"]
+    end
+
+    subgraph Context["DatabaseContext"]
+        ProxyLB["Proxy Load Balancer\n(round-robin + failover)"]
+        RetryLoop["Retry Loop\n(onError → backoff)"]
+        MonLeader["monitorLeader()\n(cluster file → CC)"]
+    end
+
+    App --> CAPI --> MV --> RYW --> Native
+
+    Native --> Get
+    Native --> GetRange
+    Get --> LocCache
+    GetRange --> LocCache
+    LocCache -->|cache hit| SS
+    LocCache -->|cache miss| CP
+
+    Native --> Set --> WriteMap
+    Native --> Clear --> WriteMap
+    Native --> Commit --> CP
+
+    RYW -->|reads check| WriteMap
+    RYW -->|then merges| SS
+
+    Native --> GRV --> GRVProxy
+    Native --> Watch --> SS
+
+    ProxyLB --> CP
+    ProxyLB --> GRVProxy
+    RetryLoop --> Native
+
+    style Layers fill:#e1f0ff,stroke:#4a90d9
+    style ReadPath fill:#e8f5e9,stroke:#4caf50
+    style WritePath fill:#fff3e0,stroke:#f5a623
+    style VersionMgmt fill:#f3e5f5,stroke:#9c27b0
+    style Context fill:#fffde7,stroke:#fbc02d
+```
+
+## Read-Your-Writes Merge
+
+```mermaid
+graph LR
+    subgraph RYW["ReadYourWritesTransaction"]
+        Read["read(key)"]
+        WM["Write Map\n(local mutations)"]
+        SC["Snapshot Cache\n(previous reads)"]
+    end
+
+    Read --> WM
+    WM -->|hit: locally set| Result["Return local value"]
+    WM -->|hit: locally cleared| Cleared["Return not found"]
+    WM -->|miss| SC
+    SC -->|hit| Result2["Return cached"]
+    SC -->|miss| SS["Storage Server read"]
+    SS --> Merge["Merge SS result\nwith write map"]
+    Merge --> Result3["Return merged"]
+```
+
+## Transaction Retry Loop
+
+```mermaid
+stateDiagram-v2
+    [*] --> GetReadVersion
+    GetReadVersion --> Execute: version V
+    Execute --> Commit: mutations ready
+    Commit --> Success: committed
+    Commit --> OnError: not_committed / too_old
+    Execute --> OnError: error during read
+    OnError --> GetReadVersion: retryable (reset + backoff)
+    OnError --> Failed: non-retryable
+    Success --> [*]
+    Failed --> [*]
+```
diff --git a/design/AI-generated/diagram_04_cluster_controller.md b/design/AI-generated/diagram_04_cluster_controller.md
new file mode 100644
index 00000000000..bbaef2f44e7
--- /dev/null
+++ b/design/AI-generated/diagram_04_cluster_controller.md
@@ -0,0 +1,107 @@
+# Cluster Controller & Coordination — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph Election["Leader Election"]
+        Coord1["Coordinator 1"]
+        Coord2["Coordinator 2"]
+        Coord3["Coordinator 3"]
+        GenReg["Generation Register\n(monotonic gen numbers)"]
+        Cand["CC Candidates\n(CandidacyRequest)"]
+    end
+
+    subgraph CC["Cluster Controller"]
+        WorkerPool["Worker Registry\n(ProcessClass + Locality)"]
+        Recruit["Role Recruitment"]
+        Monitor["Health Monitoring"]
+        ServerDBInfo["ServerDBInfo\n(cluster-wide broadcast)"]
+    end
+
+    subgraph Roles["Recruited Roles"]
+        Master["Master / Sequencer"]
+        CPr["Commit Proxies"]
+        GRVPr["GRV Proxies"]
+        Resolvers["Resolvers"]
+        TLogs["TLogs"]
+        DDist["Data Distributor"]
+        RK["Ratekeeper"]
+        CS["Consistency Scan"]
+    end
+
+    subgraph Workers["Worker Processes"]
+        W1["Worker 1\n(transaction class)"]
+        W2["Worker 2\n(storage class)"]
+        W3["Worker 3\n(stateless class)"]
+        Wn["Worker N"]
+    end
+
+    Cand --> Coord1
+    Cand --> Coord2
+    Cand --> Coord3
+    Coord1 --> GenReg
+    Coord2 --> GenReg
+    Coord3 --> GenReg
+    GenReg -->|majority agrees| CC
+
+    W1 -->|registrationClient()| WorkerPool
+    W2 -->|registrationClient()| WorkerPool
+    W3 -->|registrationClient()| WorkerPool
+    Wn -->|registrationClient()| WorkerPool
+    WorkerPool --> Recruit
+    Recruit -->|fitness + locality| Roles
+
+    Monitor -->|process failure| Recruit
+    CC --> ServerDBInfo -->|broadcast| Workers
+
+    style Election fill:#f3e5f5,stroke:#9c27b0
+    style CC fill:#e1f0ff,stroke:#4a90d9
+    style Roles fill:#fff3e0,stroke:#f5a623
+    style Workers fill:#e8f5e9,stroke:#4caf50
+```
+
+## Leader Election Protocol
+
+```mermaid
+sequenceDiagram
+    participant C1 as CC Candidate 1 (better fitness)
+    participant C2 as CC Candidate 2
+    participant Coord as Coordinators (majority)
+
+    C1->>Coord: CandidacyRequest (fitness in changeID)
+    C2->>Coord: CandidacyRequest (fitness in changeID)
+    Coord->>Coord: Compare fitness (top bits of changeID)
+    Coord-->>C1: Nominated as leader
+    Coord-->>C2: Not nominated
+    C1->>C1: Become CC
+    C2->>C2: monitorLeader() — defer to C1
+    Note over C1: Begins recruiting roles
+```
+
+## Role Recruitment Decision
+
+```mermaid
+graph TD
+    Need["Need to fill role\n(e.g., CommitProxy)"]
+    Need --> Filter["Filter workers by\nProcessClass fitness"]
+    Filter --> Locality["Apply locality constraints\n(DC, zone, machine)"]
+    Locality --> Exclude["Exclude failed /\nexcluded processes"]
+    Exclude --> Best["Select best candidate"]
+    Best --> Recruit["Send recruitment request"]
+    Recruit --> Running["Role running on worker"]
+```
+
+## ServerDBInfo Broadcast
+
+```mermaid
+graph LR
+    CC["Cluster Controller"] -->|update| Info["ServerDBInfo"]
+
+    Info --> |contains| M["Master interface"]
+    Info --> |contains| PL["Proxy list"]
+    Info --> |contains| LSC["Log system config"]
+    Info --> |contains| RS["Recovery state"]
+    Info --> |contains| LB["Latency band config"]
+
+    Info -->|broadcast to all| W["All Workers"]
+    W -->|react to changes| Roles["Update connections\nto new proxies, etc."]
+```
diff --git a/design/AI-generated/diagram_05_commit_pipeline.md b/design/AI-generated/diagram_05_commit_pipeline.md
new file mode 100644
index 00000000000..b962555c469
--- /dev/null
+++ b/design/AI-generated/diagram_05_commit_pipeline.md
@@ -0,0 +1,107 @@
+# Transaction Commit Pipeline — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph GRVPath["GRV Path (Read Versions)"]
+        Client1["Client"]
+        GRVProxy["GRV Proxy"]
+        RK["Ratekeeper"]
+    end
+
+    subgraph CommitPath["Commit Path (4 Phases)"]
+        Client2["Client"]
+        Batcher["commitBatcher\n(batches by byte size)"]
+        CP["Commit Proxy"]
+        Seq["Master / Sequencer"]
+        Res["Resolver(s)\n(sharded by key range)"]
+        TLog["TLog (quorum write)"]
+    end
+
+    Client1 -- "GetReadVersionRequest" --> GRVProxy
+    RK -- "rate limit" --> GRVProxy
+    GRVProxy -- "read version V" --> Client1
+
+    Client2 -- "CommitTransactionRequest" --> Batcher --> CP
+    CP -- "① GetCommitVersion" --> Seq
+    Seq -- "version V + prevVersion" --> CP
+    CP -- "② ResolveTransactionBatch" --> Res
+    Res -- "conflict/no-conflict per txn" --> CP
+    CP -- "③ LogSystem::push()" --> TLog
+    TLog -- "quorum ack" --> CP
+    CP -- "④ CommitTransactionReply" --> Client2
+
+    style GRVPath fill:#e1f0ff,stroke:#4a90d9
+    style CommitPath fill:#fff3e0,stroke:#f5a623
+```
+
+## Commit Proxy: Batch Processing Pipeline
+
+```mermaid
+graph LR
+    subgraph Intake["Intake"]
+        Requests["Incoming\nCommitRequests"]
+        Batcher["Batcher\n(group by size/count)"]
+    end
+
+    subgraph Phase1["Phase 1: Version"]
+        GetVer["GetCommitVersion\nfrom Sequencer"]
+    end
+
+    subgraph Phase2["Phase 2: Resolve"]
+        Split["Split batch by\nresolver key range"]
+        R1["Resolver 1"]
+        R2["Resolver 2"]
+        Merge["Merge conflict\nresults"]
+    end
+
+    subgraph Phase3["Phase 3: Log"]
+        TagMut["Tag mutations\n(key → shard → SS tag)"]
+        Push["LogSystem::push()\nto TLog quorum"]
+    end
+
+    subgraph Phase4["Phase 4: Reply"]
+        Reply["Send replies\n(committed / conflicted)"]
+    end
+
+    Requests --> Batcher --> GetVer --> Split
+    Split --> R1 --> Merge
+    Split --> R2 --> Merge
+    Merge --> TagMut --> Push --> Reply
+```
+
+## Resolver: Conflict Detection
+
+```mermaid
+graph TB
+    subgraph Window["Sliding Conflict Window"]
+        Committed["Committed write ranges\n(version-ordered)"]
+        Incoming["Incoming txn batch\n(read version + read ranges)"]
+    end
+
+    Incoming --> Check{"Any read range overlaps\nwith writes at versions\n> txn's read version?"}
+    Check -->|yes| Conflict["CONFLICT\n(txn rejected)"]
+    Check -->|no| NoConflict["NO CONFLICT\n(txn proceeds)"]
+    NoConflict --> Update["Add txn's write ranges\nto committed window"]
+    Update --> Expire["Expire old entries\nbeyond window"]
+
+    style Conflict fill:#fce4ec,stroke:#e91e63
+    style NoConflict fill:#e8f5e9,stroke:#4caf50
+```
+
+## Master/Sequencer: Version Assignment
+
+```mermaid
+sequenceDiagram
+    participant CP1 as Commit Proxy 1
+    participant CP2 as Commit Proxy 2
+    participant Seq as Sequencer
+
+    Note over Seq: Maintains: nextVersion, prevVersion
+    CP1->>Seq: GetCommitVersion(lastVersion=100)
+    Seq->>Seq: nextVersion = max(wall_clock, prev+1)
+    Seq-->>CP1: version=150, prevVersion=100
+    CP2->>Seq: GetCommitVersion(lastVersion=120)
+    Seq->>Seq: nextVersion = max(wall_clock, prev+1)
+    Seq-->>CP2: version=200, prevVersion=150
+    Note over Seq: Monotonic, roughly tracks real time
+```
diff --git a/design/AI-generated/diagram_06_tlog_logsystem.md b/design/AI-generated/diagram_06_tlog_logsystem.md
new file mode 100644
index 00000000000..1e6939b1351
--- /dev/null
+++ b/design/AI-generated/diagram_06_tlog_logsystem.md
@@ -0,0 +1,139 @@
+# TLog & Log System — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph LogSystem["Log System Abstraction"]
+        LogSystemType["LogSystem"]
+        Push["push(mutations, version)\n→ quorum write"]
+        Peek["peek(tag, fromVersion)\n→ IPeekCursor"]
+    end
+
+    subgraph TLogInstance["TLog Server Instance"]
+        Receive["Receive push\nfrom Commit Proxy"]
+        DiskQ["DiskQueue\n(append-only log)"]
+        MemIdx["In-Memory Index\n(tag → version → data)"]
+        PeekServe["Serve peek requests\nfrom Storage Servers"]
+    end
+
+    subgraph Replication["Replication"]
+        TLog1["TLog 1"]
+        TLog2["TLog 2"]
+        TLog3["TLog 3"]
+        Quorum["Quorum: f+1 of 2f+1"]
+    end
+
+    subgraph MultiRegion["Multi-Region"]
+        PrimaryTLogs["Primary TLogs"]
+        LogRouter["Log Router"]
+        RemoteTLogs["Remote TLogs"]
+        SatTLogs["Satellite TLogs"]
+    end
+
+    subgraph Consumers["Log Consumers"]
+        SS["Storage Servers\n(peek by tag)"]
+        BW["Backup Worker\n(peek mutations)"]
+    end
+
+    LogSystemType --> Push
+    LogSystemType --> Peek
+
+    Push --> TLog1
+    Push --> TLog2
+    Push --> TLog3
+    TLog1 --> Quorum
+    TLog2 --> Quorum
+    TLog3 --> Quorum
+
+    Receive --> DiskQ
+    Receive --> MemIdx
+    MemIdx --> PeekServe
+
+    PrimaryTLogs --> LogRouter --> RemoteTLogs
+    PrimaryTLogs --> SatTLogs
+
+    SS --> Peek
+    BW --> Peek
+
+    style LogSystem fill:#e1f0ff,stroke:#4a90d9
+    style TLogInstance fill:#fff3e0,stroke:#f5a623
+    style Replication fill:#e8f5e9,stroke:#4caf50
+    style MultiRegion fill:#f3e5f5,stroke:#9c27b0
+    style Consumers fill:#fffde7,stroke:#fbc02d
+```
+
+## Tag-Partitioned Mutation Flow
+
+```mermaid
+sequenceDiagram
+    participant CP as Commit Proxy
+    participant LS as LogSystem
+    participant T1 as TLog 1
+    participant T2 as TLog 2
+    participant T3 as TLog 3
+    participant SS_A as Storage Server A (tag=2)
+    participant SS_B as Storage Server B (tag=5)
+
+    CP->>CP: Tag mutations by destination SS
+    Note over CP: set("apple","1") → tag=2 (SS_A's shard)<br/>set("zebra","2") → tag=5 (SS_B's shard)
+    CP->>LS: push(tagged mutations, version V)
+    LS->>T1: replicate (all tags)
+    LS->>T2: replicate (all tags)
+    LS->>T3: replicate (all tags)
+    T1-->>LS: ack
+    T2-->>LS: ack
+    Note over LS: Quorum (2 of 3) — committed
+
+    SS_A->>T1: peek(tag=2, fromVersion)
+    T1-->>SS_A: set("apple","1") at V
+    SS_B->>T2: peek(tag=5, fromVersion)
+    T2-->>SS_B: set("zebra","2") at V
+```
+
+## Log Epochs and Recovery
+
+```mermaid
+graph LR
+    subgraph Epoch1["Epoch 1 (old)"]
+        OT1["TLog A"]
+        OT2["TLog B"]
+        OT3["TLog C"]
+    end
+
+    subgraph Epoch2["Epoch 2 (old)"]
+        OT4["TLog D"]
+        OT5["TLog E"]
+        OT6["TLog F"]
+    end
+
+    subgraph Epoch3["Epoch 3 (current)"]
+        NT1["TLog G"]
+        NT2["TLog H"]
+        NT3["TLog I"]
+    end
+
+    Epoch1 -->|"recovery: lock + peek"| Epoch2
+    Epoch2 -->|"recovery: lock + peek"| Epoch3
+
+    SS["Storage Server"] -->|"peek across epochs\n(merged cursor)"| Epoch1
+    SS -->|"peek across epochs\n(merged cursor)"| Epoch2
+    SS -->|"peek across epochs\n(merged cursor)"| Epoch3
+
+    Note1["Old epochs kept until\nall SS have consumed\ntheir data, then GC'd"]
+
+    style Epoch1 fill:#fffde7,stroke:#fbc02d
+    style Epoch2 fill:#fff3e0,stroke:#f5a623
+    style Epoch3 fill:#e8f5e9,stroke:#4caf50
+```
+
+## TLog Internal Write Path
+
+```mermaid
+graph TD
+    Push["push(mutations, version)"]
+    Push --> Deser["Deserialize mutations"]
+    Deser --> TagIndex["Index by tag\nin memory"]
+    Deser --> DiskWrite["Append to DiskQueue\n(sequential I/O)"]
+    DiskWrite --> Fsync["fsync / fdatasync"]
+    Fsync --> Ack["Acknowledge to\nCommit Proxy"]
+    TagIndex --> ServePeek["Serve peek(tag)\nfrom memory or disk"]
+```
diff --git a/design/AI-generated/diagram_07_storage_server.md b/design/AI-generated/diagram_07_storage_server.md
new file mode 100644
index 00000000000..7a789480fe9
--- /dev/null
+++ b/design/AI-generated/diagram_07_storage_server.md
@@ -0,0 +1,119 @@
+# Storage Server & Engines — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph UpdateLoop["Update Loop (pull from log)"]
+        Peek["peek(tag, fromVersion)\nfrom TLog"]
+        Deser["Deserialize mutations"]
+        Apply["Apply to KVStore"]
+        AdvVer["Advance durable version"]
+    end
+
+    subgraph ReadServing["Read Serving"]
+        GetVal["GetValueRequest"]
+        GetKey["GetKeyRequest"]
+        GetRange["GetKeyValuesRequest"]
+        VerCheck["Version check\n(oldest ≤ V ≤ current)"]
+        ShardCheck["Shard ownership check"]
+    end
+
+    subgraph Versions["Version Management"]
+        StorageVer["storageVersion\n(latest applied)"]
+        DurableVer["durableVersion\n(latest fsynced)"]
+        OldestVer["desiredOldestVersion\n(GC boundary)"]
+    end
+
+    subgraph Engine["KV Engine (pluggable)"]
+        IKV["IKeyValueStore interface"]
+        RocksDB["RocksDB\n(primary production)"]
+        ShardedRocks["Sharded RocksDB\n(per-shard column families)"]
+        SQLite["SQLite\n(legacy)"]
+        MemEngine["Memory Engine\n(testing / txnStateStore)"]
+    end
+
+    subgraph ShardMgmt["Shard Management"]
+        Shards["Assigned Shards\n(key ranges)"]
+        AddShard["addShard\n(from data movement)"]
+        RemShard["removeShard"]
+        FetchKeys["fetchKeys\n(bulk copy from source SS)"]
+    end
+
+    Peek --> Deser --> Apply --> AdvVer
+    Apply --> IKV
+
+    GetVal --> VerCheck
+    GetKey --> VerCheck
+    GetRange --> VerCheck
+    VerCheck --> ShardCheck --> IKV
+    IKV --> RocksDB
+    IKV --> ShardedRocks
+    IKV --> SQLite
+    IKV --> MemEngine
+
+    StorageVer --> VerCheck
+    DurableVer --> VerCheck
+    OldestVer --> VerCheck
+
+    AddShard --> FetchKeys --> Shards
+    RemShard --> Shards
+
+    style UpdateLoop fill:#e1f0ff,stroke:#4a90d9
+    style ReadServing fill:#e8f5e9,stroke:#4caf50
+    style Versions fill:#fff3e0,stroke:#f5a623
+    style Engine fill:#fce4ec,stroke:#e91e63
+    style ShardMgmt fill:#f3e5f5,stroke:#9c27b0
+```
+
+## Storage Server Lifecycle
+
+```mermaid
+stateDiagram-v2
+    [*] --> Recruited: DD assigns shard
+    Recruited --> FetchingKeys: fetchKeys from source SS
+    FetchingKeys --> WaitingForLog: data fetched, start peeking log
+    WaitingForLog --> Serving: caught up to current version
+    Serving --> Serving: continuous peek + apply + serve reads
+    Serving --> Removed: DD moves shard away
+    Removed --> [*]
+```
+
+## Read Request Processing
+
+```mermaid
+sequenceDiagram
+    participant Client as Client
+    participant SS as Storage Server
+    participant KV as KV Engine
+
+    Client->>SS: GetKeyValuesRequest(range, version V)
+    SS->>SS: Check: oldest ≤ V ≤ storageVersion?
+    alt Version too old
+        SS-->>Client: Error: transaction_too_old
+    else Version too new
+        SS->>SS: Wait until storageVersion ≥ V
+    end
+    SS->>SS: Check: owns shard for range?
+    alt Shard not owned
+        SS-->>Client: Error: wrong_shard_server
+    end
+    SS->>KV: readRange(range, version V)
+    KV-->>SS: results
+    SS-->>Client: GetKeyValuesReply(results)
+```
+
+## Storage Engine Comparison
+
+```mermaid
+graph LR
+    subgraph Engines
+        R["RocksDB"]
+        SR["Sharded RocksDB"]
+        SQ["SQLite"]
+        M["Memory"]
+    end
+
+    R --- R_desc["LSM-tree, production default\nGood compression, range perf"]
+    SR --- SR_desc["Per-shard column families\nBetter isolation between shards"]
+    SQ --- SQ_desc["B-tree, legacy\nStill supported"]
+    M --- M_desc["In-memory only\nFor txnStateStore, testing"]
+```
diff --git a/design/AI-generated/diagram_08_data_distribution.md b/design/AI-generated/diagram_08_data_distribution.md
new file mode 100644
index 00000000000..776ee0aeb39
--- /dev/null
+++ b/design/AI-generated/diagram_08_data_distribution.md
@@ -0,0 +1,129 @@
+# Data Distribution — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph DD["Data Distributor (singleton)"]
+        DDMain["DD Main Loop"]
+    end
+
+    subgraph Tracking["Shard Tracking"]
+        ShardTracker["DDShardTracker\n(monitors size + load)"]
+        Split["Split decision\n(shard too large)"]
+        Merge["Merge decision\n(adjacent shards too small)"]
+    end
+
+    subgraph Teams["Team Management"]
+        TeamColl["DDTeamCollection"]
+        BuildTeams["buildTeams()\n(satisfy replication policy)"]
+        HealthyTeams["Healthy Teams"]
+        UnhealthyTeams["Unhealthy Teams\n(missing replicas)"]
+        SATF["ShardsAffectedByTeamFailure\n(tracks which shards need moves)"]
+    end
+
+    subgraph Relocation["Relocation Queue"]
+        RelocQ["DDRelocationQueue\n(prioritized)"]
+        Priority["Priorities:\n• Emergency (replica lost)\n• High (rebalance unhealthy)\n• Normal (load balance)\n• Low (merge)"]
+        Parallel["Concurrent moves\n(parallelism limit)"]
+    end
+
+    subgraph MoveKeys["MoveKeys Protocol"]
+        Lock["moveKeysLock\n(serialize concurrent moves)"]
+        Fetch["Dest SS: fetchKeys\nfrom source SS"]
+        LogPeek["Dest SS: start\npeeking TLog"]
+        AtomicXfer["Atomic ownership\ntransfer (system key txn)"]
+    end
+
+    DDMain --> ShardTracker
+    DDMain --> TeamColl
+    DDMain --> RelocQ
+
+    ShardTracker --> Split
+    ShardTracker --> Merge
+    Split --> RelocQ
+    Merge --> RelocQ
+    TeamColl --> BuildTeams --> HealthyTeams
+    BuildTeams --> UnhealthyTeams
+    UnhealthyTeams --> SATF --> RelocQ
+
+    RelocQ --> Priority --> Parallel --> MoveKeys
+    Lock --> Fetch --> LogPeek --> AtomicXfer
+
+    style DD fill:#e1f0ff,stroke:#4a90d9
+    style Tracking fill:#e8f5e9,stroke:#4caf50
+    style Teams fill:#fff3e0,stroke:#f5a623
+    style Relocation fill:#fce4ec,stroke:#e91e63
+    style MoveKeys fill:#f3e5f5,stroke:#9c27b0
+```
+
+## Shard Move Sequence
+
+```mermaid
+sequenceDiagram
+    participant DD as Data Distributor
+    participant RQ as Relocation Queue
+    participant MK as MoveKeys
+    participant Src as Source SS
+    participant Dst as Destination SS
+    participant TLog as TLog
+    participant SysKey as System Key Space
+
+    DD->>RQ: RelocateShard(range, priority)
+    RQ->>MK: execute(range, srcTeam, dstTeam)
+    MK->>MK: Acquire moveKeysLock
+
+    MK->>Dst: Assign shard (begin fetch)
+    Dst->>Src: fetchKeys(range)
+    Src-->>Dst: key-value data (bulk)
+
+    Dst->>TLog: peek(tag, version) — start consuming
+    TLog-->>Dst: mutations (catch up)
+    Note over Dst: Catching up to current version...
+
+    Dst-->>MK: Ready (caught up)
+
+    MK->>SysKey: Transaction: update keyServers + serverKeys
+    Note over MK: Atomic ownership transfer
+    SysKey-->>MK: committed
+
+    MK->>Src: Remove shard assignment
+    MK-->>RQ: Move complete
+    MK->>MK: Release moveKeysLock
+```
+
+## Team Building
+
+```mermaid
+graph TD
+    Policy["Replication Policy\ne.g., triple redundancy\nacross 3 zones"]
+    Policy --> Candidates["Available Storage Servers\n(with LocalityData)"]
+    Candidates --> Build["Build teams of N servers\nsatisfying zone/DC/rack diversity"]
+    Build --> Team1["Team {SS1-zoneA, SS2-zoneB, SS3-zoneC}"]
+    Build --> Team2["Team {SS1-zoneA, SS4-zoneD, SS5-zoneE}"]
+    Build --> TeamN["..."]
+
+    Team1 --> Health{"All members\nhealthy?"}
+    Health -->|yes| Healthy["Healthy Team List"]
+    Health -->|no| Unhealthy["Unhealthy Team List\n→ trigger re-replication"]
+```
+
+## Reasons for Data Movement
+
+```mermaid
+graph LR
+    subgraph Triggers
+        T1["Shard too large\n→ split"]
+        T2["SS failure\n→ re-replicate"]
+        T3["Load imbalance\n→ rebalance"]
+        T4["Config change\n(replication factor)"]
+        T5["Server exclusion\n(maintenance)"]
+        T6["Storage wiggle\n(proactive)"]
+    end
+
+    T1 --> RQ["Relocation Queue"]
+    T2 --> RQ
+    T3 --> RQ
+    T4 --> RQ
+    T5 --> RQ
+    T6 --> RQ
+    RQ --> Move["Execute MoveKeys"]
+```
diff --git a/design/AI-generated/diagram_09_cluster_recovery.md b/design/AI-generated/diagram_09_cluster_recovery.md
new file mode 100644
index 00000000000..6b86b6e7cae
--- /dev/null
+++ b/design/AI-generated/diagram_09_cluster_recovery.md
@@ -0,0 +1,149 @@
+# Cluster Recovery — Internal Architecture
+
+```mermaid
+stateDiagram-v2
+    [*] --> READING_CSTATE: Master failure detected
+
+    state READING_CSTATE {
+        [*] --> ReadCoord: Read from coordinators
+        ReadCoord --> ParseDBState: Parse DBCoreState
+        ParseDBState --> LearnOldEpoch: Learn previous epoch's TLogs
+    }
+
+    READING_CSTATE --> LOCKING_CSTATE
+
+    state LOCKING_CSTATE {
+        [*] --> LockOldTLogs: Prevent split-brain
+        LockOldTLogs --> EndOldEpoch: Old TLogs stop accepting writes
+    }
+
+    LOCKING_CSTATE --> RECRUITING
+
+    state RECRUITING {
+        [*] --> RecruitTLogs: New TLog set
+        RecruitTLogs --> RecruitProxies: CommitProxies + GrvProxies
+        RecruitProxies --> RecruitResolvers: Resolvers
+    }
+
+    RECRUITING --> RECOVERY_TRANSACTION
+
+    state RECOVERY_TRANSACTION {
+        [*] --> PeekOldLogs: Read uncommitted mutations
+        PeekOldLogs --> ReplayTxnState: Reconstruct txnStateStore
+        ReplayTxnState --> SetRecoveryVer: Establish recovery version
+    }
+
+    RECOVERY_TRANSACTION --> WRITING_CSTATE
+
+    state WRITING_CSTATE {
+        [*] --> WriteCoord: Write new epoch to coordinators
+        WriteCoord --> [*]: New epoch official
+    }
+
+    WRITING_CSTATE --> ACCEPTING_COMMITS: Cluster is live
+
+    ACCEPTING_COMMITS --> ALL_LOGS_RECRUITED: Remote TLogs recruited
+
+    ALL_LOGS_RECRUITED --> FULLY_RECOVERED: Old generations consumed + GC'd
+```
+
+## Recovery Data Flow
+
+```mermaid
+graph TB
+    subgraph OldEpoch["Old Epoch"]
+        OldTLogs["Old TLogs\n(locked)"]
+        OldCState["Old Coordinated State\n(stored on coordinators)"]
+    end
+
+    subgraph Recovery["Recovery Process"]
+        ReadCState["① Read coordinated state"]
+        LockTLogs["② Lock old TLogs"]
+        PeekReplay["③ Peek + replay\ntxnStateStore"]
+        RecVer["④ Set recovery version"]
+        WriteCState["⑤ Write new coordinated\nstate to coordinators"]
+    end
+
+    subgraph NewEpoch["New Epoch"]
+        NewTLogs["New TLogs"]
+        NewProxies["New Proxies"]
+        NewResolvers["New Resolvers"]
+        NewMaster["New Master/Sequencer"]
+    end
+
+    OldCState --> ReadCState
+    ReadCState --> LockTLogs --> OldTLogs
+    OldTLogs --> PeekReplay
+    PeekReplay --> RecVer
+    RecVer --> WriteCState
+
+    WriteCState --> NewTLogs
+    WriteCState --> NewProxies
+    WriteCState --> NewResolvers
+    WriteCState --> NewMaster
+
+    NewTLogs -.->|"SS peek across epochs"| OldTLogs
+
+    style OldEpoch fill:#fffde7,stroke:#fbc02d
+    style Recovery fill:#e1f0ff,stroke:#4a90d9
+    style NewEpoch fill:#e8f5e9,stroke:#4caf50
+```
+
+## Generation GC (trackTlogRecovery)
+
+```mermaid
+sequenceDiagram
+    participant TR as trackTlogRecovery
+    participant TLogs as TLog replicas
+    participant CState as Coordinated State
+
+    loop Poll TLog recovery progress
+        TR->>TLogs: trackRecovery.getReply()
+        TLogs-->>TR: oldestUnrecoveredStartVersion
+        TR->>TR: recoveredVersion = min(all TLogs)
+    end
+
+    TR->>TR: For each old generation:<br/>if recoverAt < recoveredVersion<br/>AND epoch < oldestBackupEpoch
+    TR->>CState: Purge old generation data
+    Note over CState: oldTLogData.resize(i)<br/>removes consumed generations
+    TR->>CState: Write updated state
+```
+
+## Multi-Region Recovery
+
+```mermaid
+graph TB
+    subgraph Primary["Primary DC"]
+        PCC["Cluster Controller"]
+        PTLogs["Primary TLogs"]
+        PProxies["Proxies + Resolvers"]
+    end
+
+    subgraph Satellite["Satellite DCs"]
+        STLogs1["Satellite TLogs (DC2)"]
+        STLogs2["Satellite TLogs (DC4)"]
+    end
+
+    subgraph Remote["Remote DC"]
+        RTLogs["Remote TLogs"]
+        RLogRouter["Log Router"]
+    end
+
+    PCC -->|recruit| PTLogs
+    PCC -->|recruit| PProxies
+    PCC -->|recruit| STLogs1
+    PCC -->|recruit| STLogs2
+    PCC -->|recruit| RTLogs
+    PCC -->|recruit| RLogRouter
+
+    PProxies -->|push| PTLogs
+    PTLogs -->|replicate| STLogs1
+    PTLogs -->|replicate| STLogs2
+    PTLogs -->|route via| RLogRouter -->|push| RTLogs
+
+    Note1["Recovery states:\nACCEPTING_COMMITS = primary ready\nALL_LOGS_RECRUITED = remote ready\nFULLY_RECOVERED = all generations GC'd"]
+
+    style Primary fill:#e8f5e9,stroke:#4caf50
+    style Satellite fill:#fff3e0,stroke:#f5a623
+    style Remote fill:#e1f0ff,stroke:#4a90d9
+```
diff --git a/design/AI-generated/diagram_10_ratekeeper.md b/design/AI-generated/diagram_10_ratekeeper.md
new file mode 100644
index 00000000000..6e8189d4533
--- /dev/null
+++ b/design/AI-generated/diagram_10_ratekeeper.md
@@ -0,0 +1,91 @@
+# Rate Keeping & Throttling — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph Monitoring["What Ratekeeper Monitors"]
+        SSQueue["Storage Server\nqueue depth"]
+        TLogQueue["TLog\nqueue depth"]
+        DiskIO["Storage Server\ndisk throughput / IOPS"]
+        TagRates["Per-tag\ntransaction rates"]
+    end
+
+    subgraph RK["Ratekeeper"]
+        Calc["Rate Calculator"]
+        TT["TagThrottler\n(per-tag limits)"]
+    end
+
+    subgraph Control["What Ratekeeper Controls"]
+        TxnRate["Transaction rate limit\n(to GRV Proxies)"]
+        BatchRate["Batch priority rate\n(lower priority)"]
+        TagThrottle["Per-tag throttle\n(slow/reject specific tags)"]
+    end
+
+    subgraph Enforcement["Enforcement"]
+        GRVProxy["GRV Proxy"]
+        Delay["Delay GetReadVersion\nresponses"]
+        Reject["Reject when\noverthrottled"]
+    end
+
+    SSQueue --> Calc
+    TLogQueue --> Calc
+    DiskIO --> Calc
+    TagRates --> Calc
+    Calc --> TxnRate
+    Calc --> BatchRate
+    TagRates --> TT --> TagThrottle
+
+    TxnRate --> GRVProxy
+    BatchRate --> GRVProxy
+    TagThrottle --> GRVProxy
+    GRVProxy --> Delay
+    GRVProxy --> Reject
+
+    style Monitoring fill:#e1f0ff,stroke:#4a90d9
+    style RK fill:#fff3e0,stroke:#f5a623
+    style Control fill:#e8f5e9,stroke:#4caf50
+    style Enforcement fill:#fce4ec,stroke:#e91e63
+```
+
+## Back-Pressure Feedback Loop
+
+```mermaid
+sequenceDiagram
+    participant SS as Storage Servers
+    participant TLog as TLogs
+    participant RK as Ratekeeper
+    participant GRV as GRV Proxy
+    participant Client as Clients
+
+    SS->>RK: Report queue depth, disk stats
+    TLog->>RK: Report queue depth
+    RK->>RK: Calculate safe transaction rate
+    RK->>GRV: UpdateRateInfo(txnRate, batchRate)
+
+    Client->>GRV: GetReadVersion
+    alt Under limit
+        GRV-->>Client: version (immediately)
+    else Over limit
+        GRV->>GRV: Delay response
+        GRV-->>Client: version (delayed)
+    end
+    Note over RK: Continuous feedback loop:<br/>SS falls behind → lower rate →<br/>fewer commits → SS catches up →<br/>raise rate
+```
+
+## Tag-Based Throttling
+
+```mermaid
+graph TD
+    Txn["Transaction with tag='team-A'"]
+    Txn --> GRV["GRV Proxy"]
+    GRV --> CheckTag{"Tag throttled?"}
+    CheckTag -->|no| Proceed["Assign read version"]
+    CheckTag -->|yes| CheckRate{"Under tag's\nrate limit?"}
+    CheckRate -->|yes| Proceed
+    CheckRate -->|no| ThrottleResp["Reject or delay\n(tag_throttled error)"]
+
+    TT["TagThrottler"] -->|"per-tag limits"| GRV
+    RK["Ratekeeper"] -->|"tag usage stats"| TT
+
+    style ThrottleResp fill:#fce4ec,stroke:#e91e63
+    style Proceed fill:#e8f5e9,stroke:#4caf50
+```
diff --git a/design/AI-generated/diagram_11_backup_dr.md b/design/AI-generated/diagram_11_backup_dr.md
new file mode 100644
index 00000000000..b98a49a0df1
--- /dev/null
+++ b/design/AI-generated/diagram_11_backup_dr.md
@@ -0,0 +1,114 @@
+# Backup, Restore & DR — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph BackupPipeline["Backup Pipeline"]
+        BW["Backup Worker\n(server-side role)"]
+        TLog["TLog"]
+        FBA["FileBackupAgent\n(orchestrator)"]
+        TaskBucket["TaskBucket\n(reliable task queue)"]
+    end
+
+    subgraph FileTypes["Backup File Types"]
+        RangeFile["Range Files\n(KV snapshot at version)"]
+        LogFile["Log Files\n(mutation stream\nbetween versions)"]
+    end
+
+    subgraph Storage["Backup Storage"]
+        BC["BackupContainer\n(abstract interface)"]
+        LocalDir["Local Directory"]
+        S3["S3 Blob Store"]
+    end
+
+    subgraph Restore["Restore"]
+        RA["Restore Agent"]
+        ReadRange["Read range files\n(base state)"]
+        ApplyLog["Apply log files\n(to target version)"]
+        PrefixXform["Key prefix\ntransformation"]
+    end
+
+    subgraph DR["Disaster Recovery"]
+        DBA["DatabaseBackupAgent"]
+        SrcCluster["Source Cluster"]
+        DstCluster["Standby Cluster"]
+    end
+
+    BW -->|"peek mutations"| TLog
+    FBA --> TaskBucket
+    BW --> RangeFile
+    BW --> LogFile
+    RangeFile --> BC
+    LogFile --> BC
+    BC --> LocalDir
+    BC --> S3
+
+    RA --> ReadRange --> BC
+    RA --> ApplyLog --> BC
+    RA --> PrefixXform
+
+    DBA -->|"stream mutations"| SrcCluster
+    DBA -->|"apply mutations"| DstCluster
+
+    style BackupPipeline fill:#e1f0ff,stroke:#4a90d9
+    style FileTypes fill:#fff3e0,stroke:#f5a623
+    style Storage fill:#e8f5e9,stroke:#4caf50
+    style Restore fill:#fce4ec,stroke:#e91e63
+    style DR fill:#f3e5f5,stroke:#9c27b0
+```
+
+## Backup Lifecycle
+
+```mermaid
+sequenceDiagram
+    participant User as User / fdbbackup
+    participant FBA as FileBackupAgent
+    participant TB as TaskBucket
+    participant BW as Backup Worker
+    participant TLog as TLogs
+    participant BC as BackupContainer (S3)
+
+    User->>FBA: Start backup
+    FBA->>TB: Create backup tasks
+
+    loop Continuous
+        TB->>BW: Snapshot task
+        BW->>BW: Read key ranges from SS
+        BW->>BC: Write range files
+
+        TB->>BW: Log task
+        BW->>TLog: peek(logRouterTag)
+        TLog-->>BW: mutation stream
+        BW->>BC: Write log files
+    end
+
+    User->>FBA: Stop backup
+    FBA->>TB: Finalize
+    Note over BC: Backup complete:<br/>range files + log files<br/>= point-in-time restorable
+```
+
+## Point-in-Time Restore
+
+```mermaid
+graph LR
+    subgraph BackupData["Backup Data"]
+        R1["Range File @ V100\n(full snapshot)"]
+        L1["Log File V100→V200"]
+        L2["Log File V200→V300"]
+        L3["Log File V300→V400"]
+    end
+
+    subgraph Restore["Restore to V350"]
+        Step1["① Load range file @ V100\n(base state)"]
+        Step2["② Apply logs V100→V200"]
+        Step3["③ Apply logs V200→V300"]
+        Step4["④ Apply logs V300→V350\n(partial)"]
+    end
+
+    R1 --> Step1
+    L1 --> Step2
+    L2 --> Step3
+    L3 --> Step4
+    Step4 --> Result["Restored cluster\nat version 350"]
+
+    style Result fill:#e8f5e9,stroke:#4caf50
+```
diff --git a/design/AI-generated/diagram_12_simulation_testing.md b/design/AI-generated/diagram_12_simulation_testing.md
new file mode 100644
index 00000000000..b40a6ef7e18
--- /dev/null
+++ b/design/AI-generated/diagram_12_simulation_testing.md
@@ -0,0 +1,155 @@
+# Simulation & Testing — Internal Architecture
+
+```mermaid
+graph TB
+    subgraph Sim2["Sim2 (replaces Net2)"]
+        VirtProcs["Virtual Processes\n(many in one OS thread)"]
+        DetTime["Deterministic Time\n(virtual clock)"]
+        DetRand["deterministicRandom()\n(seedable PRNG)"]
+        SimNet["Simulated Network"]
+    end
+
+    subgraph FaultInjection["Fault Injection"]
+        Buggify["BUGGIFY\n(probabilistic code paths)"]
+        NetFaults["Network Faults"]
+        ProcFaults["Process Faults"]
+        DiskFaults["Disk Faults"]
+    end
+
+    subgraph NetFaultTypes["Network Fault Types"]
+        Clog["Clogging\n(delay packets)"]
+        Partition["Partitions\n(drop packets)"]
+        Latency["Latency injection"]
+    end
+
+    subgraph ProcFaultTypes["Process Fault Types"]
+        KillProc["Kill process"]
+        KillMachine["Kill machine\n(all processes)"]
+        KillZone["Kill zone"]
+        KillDC["Kill datacenter"]
+        Reboot["Reboot process"]
+    end
+
+    subgraph TestFramework["Test Framework"]
+        TOML["Test Config\n(.toml files)"]
+        SimCluster["SimulatedCluster\n(virtual topology)"]
+        Tester["Test Runner\n(fdbserver -r simulation)"]
+    end
+
+    subgraph Workloads["Workloads (~160)"]
+        WBase["TestWorkload base class"]
+        Setup["setup()"]
+        Start["start()"]
+        Check["check()"]
+        Metrics["getMetrics()"]
+    end
+
+    Tester --> TOML --> SimCluster --> VirtProcs
+    VirtProcs --> DetTime
+    VirtProcs --> DetRand
+    VirtProcs --> SimNet
+    SimNet --> NetFaults --> NetFaultTypes
+    ProcFaults --> ProcFaultTypes
+    DiskFaults --> SimNet
+
+    Buggify --> VirtProcs
+
+    Tester --> WBase
+    WBase --> Setup --> Start --> Check --> Metrics
+
+    style Sim2 fill:#e1f0ff,stroke:#4a90d9
+    style FaultInjection fill:#fce4ec,stroke:#e91e63
+    style NetFaultTypes fill:#fff3e0,stroke:#f5a623
+    style ProcFaultTypes fill:#fff3e0,stroke:#f5a623
+    style TestFramework fill:#e8f5e9,stroke:#4caf50
+    style Workloads fill:#f3e5f5,stroke:#9c27b0
+```
+
+## Deterministic Simulation Flow
+
+```mermaid
+sequenceDiagram
+    participant User as User
+    participant Tester as fdbserver -r simulation
+    participant Sim as Sim2
+    participant Cluster as Virtual Cluster
+
+    User->>Tester: seed=12345, test=Fast/Cycle.toml
+    Tester->>Sim: Initialize with seed
+    Sim->>Sim: deterministicRandom(12345)
+    Sim->>Cluster: Create virtual topology<br/>(machines, DCs, processes)
+
+    loop Simulation Loop
+        Sim->>Sim: Advance virtual clock<br/>to next event
+        Sim->>Cluster: Deliver network messages
+        Sim->>Cluster: Fire timers
+        Sim->>Cluster: Inject faults (BUGGIFY)
+        Cluster->>Cluster: Execute actors
+    end
+
+    Tester->>Tester: Run workload check()
+    Tester->>User: Pass/Fail + Unseed
+    Note over User: Same seed → same unseed<br/>(on identical binary)
+```
+
+## BUGGIFY System
+
+```mermaid
+graph TD
+    subgraph BuggifyDecision["At Startup (per BUGGIFY site)"]
+        Site["BUGGIFY at file:line"]
+        Coin["deterministicRandom()\ncoin flip"]
+        Coin -->|"~10% chance"| Enabled["Site ENABLED\nfor this run"]
+        Coin -->|"~90% chance"| Disabled["Site DISABLED\nfor this run"]
+    end
+
+    subgraph Runtime["At Runtime"]
+        Enabled --> Hit{"Code reaches\nBUGGIFY site"}
+        Hit -->|"second coin flip\n(~10%)"| Fire["Fault injected"]
+        Hit -->|"~90%"| Skip["Normal path"]
+    end
+
+    subgraph Examples["Example BUGGIFY Effects"]
+        E1["Short timeouts\n(trigger timeouts)"]
+        E2["Extra delays\n(expose races)"]
+        E3["Small buffer sizes\n(exercise edge cases)"]
+        E4["Wrong shard responses\n(test retry logic)"]
+        E5["Knob value changes\n(test parameter sensitivity)"]
+    end
+
+    Fire --> E1
+    Fire --> E2
+    Fire --> E3
+    Fire --> E4
+    Fire --> E5
+
+    style BuggifyDecision fill:#e1f0ff,stroke:#4a90d9
+    style Runtime fill:#fff3e0,stroke:#f5a623
+    style Examples fill:#fce4ec,stroke:#e91e63
+```
+
+## Test Organization
+
+```mermaid
+graph LR
+    subgraph Tests["tests/"]
+        Fast["fast/\n(quick correctness)"]
+        Rare["rare/\n(failure scenarios:\nrollbacks, clogging,\nattrition)"]
+        Slow["slow/\n(longer, complex\nscenarios)"]
+        Negative["negative/\n(error conditions)"]
+        NoSim["noSim/\n(live cluster\nintegration tests)"]
+    end
+
+    subgraph WorkloadTypes["Workload Categories"]
+        Correctness["Correctness\n(Cycle, AtomicOps,\nWriteDuringRead)"]
+        Config["Configuration\n(ConfigureDatabase,\nChangeConfig)"]
+        Failure["Failure\n(Attrition, Rollback,\nMachineAttrition)"]
+        Perf["Performance\n(ReadWrite,\nThroughput)"]
+        Feature["Feature-specific\n(Backup, BulkLoad,\nGcGenerations)"]
+    end
+
+    Fast --> TOML[".toml config\n(workload list +\nparameters)"]
+    Rare --> TOML
+    Slow --> TOML
+    TOML --> WorkloadTypes
+```
diff --git a/design/AI-generated/diagram_overview.md b/design/AI-generated/diagram_overview.md
new file mode 100644
index 00000000000..b2118616e83
--- /dev/null
+++ b/design/AI-generated/diagram_overview.md
@@ -0,0 +1,163 @@
+# FoundationDB Architecture Overview
+
+## System-Level Architecture
+
+```mermaid
+graph TB
+    subgraph Client["Client Layer"]
+        App["Application"]
+        CAPI["C API / Bindings"]
+        MV["MultiVersionTransaction"]
+        RYW["ReadYourWritesTransaction"]
+        Native["NativeAPI / Transaction"]
+    end
+
+    subgraph Coordination["Coordination"]
+        Coord["Coordinators"]
+        CC["Cluster Controller"]
+    end
+
+    subgraph CommitPipeline["Transaction Commit Pipeline"]
+        GRV["GRV Proxy"]
+        CP["Commit Proxy"]
+        Seq["Master / Sequencer"]
+        Res["Resolver"]
+    end
+
+    subgraph LogSystem["Log System"]
+        TLog["TLog Replicas"]
+        LR["Log Router"]
+        TLogRemote["Remote TLogs"]
+    end
+
+    subgraph Storage["Storage Layer"]
+        SS["Storage Server"]
+        KV["KV Engine\n(RocksDB / SQLite / Memory)"]
+    end
+
+    subgraph Background["Background Services"]
+        DD["Data Distributor"]
+        RK["Ratekeeper"]
+        BW["Backup Worker"]
+    end
+
+    App --> CAPI --> MV --> RYW --> Native
+
+    Native -->|GetReadVersion| GRV
+    Native -->|"get / getRange"| SS
+    Native -->|commit| CP
+
+    GRV -.->|rate limit check| RK
+    CP -->|GetCommitVersion| Seq
+    CP -->|resolve conflicts| Res
+    CP -->|push mutations| TLog
+
+    SS -->|"peek (pull)"| TLog
+    SS --> KV
+
+    TLog -->|route to remote| LR --> TLogRemote
+
+    CC -->|recruit| CP
+    CC -->|recruit| GRV
+    CC -->|recruit| Res
+    CC -->|recruit| Seq
+    CC -->|recruit| TLog
+    CC -->|recruit| SS
+    CC -->|recruit| DD
+    CC -->|recruit| RK
+    CC -->|leader election| Coord
+
+    DD -->|move shards| SS
+    RK -->|throttle signals| GRV
+    BW -->|peek mutations| TLog
+
+    style Client fill:#e1f0ff,stroke:#4a90d9
+    style CommitPipeline fill:#fff3e0,stroke:#f5a623
+    style LogSystem fill:#e8f5e9,stroke:#4caf50
+    style Storage fill:#fce4ec,stroke:#e91e63
+    style Coordination fill:#f3e5f5,stroke:#9c27b0
+    style Background fill:#fffde7,stroke:#fbc02d
+```
+
+## Write Path
+
+```mermaid
+sequenceDiagram
+    participant App as Application
+    participant CP as Commit Proxy
+    participant Seq as Master/Sequencer
+    participant Res as Resolver
+    participant TLog as TLog (quorum)
+    participant SS as Storage Server
+
+    App->>CP: CommitTransactionRequest
+    CP->>Seq: GetCommitVersion
+    Seq-->>CP: version V (monotonic)
+    CP->>Res: ResolveTransactionBatch
+    Res-->>CP: conflict/no-conflict
+    Note over CP: Drop conflicting txns
+    CP->>TLog: push(mutations, version V)
+    TLog-->>CP: ack (quorum)
+    CP-->>App: committed at version V
+    Note over TLog,SS: Async (decoupled)
+    SS->>TLog: peek(tag, fromVersion)
+    TLog-->>SS: mutations
+    SS->>SS: apply to KVStore
+```
+
+## Read Path
+
+```mermaid
+sequenceDiagram
+    participant App as Application
+    participant GRV as GRV Proxy
+    participant RK as Ratekeeper
+    participant SS as Storage Server
+    participant KV as KV Engine
+
+    App->>GRV: GetReadVersion
+    Note over GRV,RK: Rate-limited
+    GRV-->>App: read version V
+    App->>SS: get/getRange at version V
+    SS->>KV: read(key, version V)
+    KV-->>SS: value
+    SS-->>App: result
+```
+
+## Recovery Path
+
+```mermaid
+stateDiagram-v2
+    [*] --> READING_CSTATE: CC detects failure
+    READING_CSTATE --> LOCKING_CSTATE: Read coordinated state
+    LOCKING_CSTATE --> RECRUITING: Lock old TLogs
+    RECRUITING --> RECOVERY_TRANSACTION: Recruit new roles
+    RECOVERY_TRANSACTION --> WRITING_CSTATE: Replay last epoch
+    WRITING_CSTATE --> ACCEPTING_COMMITS: Write new coordinated state
+    ACCEPTING_COMMITS --> ALL_LOGS_RECRUITED: Primary + remote TLogs ready
+    ALL_LOGS_RECRUITED --> FULLY_RECOVERED: Old TLog data consumed
+    FULLY_RECOVERED --> [*]
+```
+
+## Data Movement Path
+
+```mermaid
+sequenceDiagram
+    participant DDT as DD Shard Tracker
+    participant DDQ as DD Relocation Queue
+    participant MK as MoveKeys Protocol
+    participant Src as Source SS
+    participant Dst as Destination SS
+    participant TLog as TLog
+
+    DDT->>DDQ: RelocateShard (priority)
+    DDQ->>MK: execute move
+    MK->>Dst: fetch data from source
+    Dst->>Src: bulk fetch key range
+    Src-->>Dst: data
+    Dst->>TLog: start peeking tag
+    TLog-->>Dst: mutations (catch up)
+    Note over MK: Atomic ownership transfer
+    MK->>MK: update keyServers system key
+    MK-->>DDQ: move complete
+```
diff --git a/design/AI-generated/foundationdb_subsystem_map.md b/design/AI-generated/foundationdb_subsystem_map.md
new file mode 100644
index 00000000000..e714626692e
--- /dev/null
+++ b/design/AI-generated/foundationdb_subsystem_map.md
@@ -0,0 +1,359 @@
+# FoundationDB Codebase: Major Subsystem Map
+
+This is a research document, not an implementation plan. It partitions the entire FoundationDB codebase into 12 major subsystems based on data flow and runtime relationships.
+
+---
+
+## The 12 Subsystems at a Glance
+
+| # | Subsystem | Primary Location | ~LOC | One-line summary |
+|---|-----------|-----------------|------|------------------|
+| 1 | [**Flow Runtime**](subsystem_01_flow_runtime.md) | [`flow/`](https://github.com/apple/foundationdb/tree/main/flow) | 31K impl + headers | Async actor framework, event loop, deterministic time, memory arenas |
+| 2 | [**RPC & Transport**](subsystem_02_rpc_transport.md) | [`fdbrpc/`](https://github.com/apple/foundationdb/tree/main/fdbrpc) | 28K impl + headers | Endpoint-addressed messaging, peer management, failure monitoring, simulation network |
+| 3 | [**Client Library**](subsystem_03_client_library.md) | [`fdbclient/`](https://github.com/apple/foundationdb/tree/main/fdbclient) | 75K impl, 34K headers | Transaction API, read-your-writes, location caching, multi-version client |
+| 4 | [**Cluster Controller & Coordination**](subsystem_04_cluster_controller.md) | [`fdbserver/clustercontroller/`](https://github.com/apple/foundationdb/tree/main/fdbserver/clustercontroller), [`fdbserver/coordinator/`](https://github.com/apple/foundationdb/tree/main/fdbserver/coordinator), [`fdbserver/core/LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp) | 13K + coordination | Leader election, role recruitment, process registration, cluster-wide decision making |
+| 5 | [**Transaction Commit Pipeline**](subsystem_05_commit_pipeline.md) | [`fdbserver/commitproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/commitproxy), [`fdbserver/grvproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/grvproxy), [`fdbserver/resolver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/resolver), [`fdbserver/sequencer/`](https://github.com/apple/foundationdb/tree/main/fdbserver/sequencer) | ~12K | Version assignment, conflict detection, commit batching — the write-path orchestration |
+| 6 | [**Transaction Log (TLog) & Log System**](subsystem_06_tlog_logsystem.md) | [`fdbserver/tlog/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tlog), [`fdbserver/logsystem/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logsystem), [`fdbserver/logrouter/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logrouter) | 17K | Durable mutation logging, tag-partitioned replication, peek cursors for storage servers |
+| 7 | [**Storage Server & Engines**](subsystem_07_storage_server.md) | [`fdbserver/storageserver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/storageserver), [`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore) | 63K | Serves reads, applies mutations from log, pluggable storage backends (RocksDB, SQLite, memory) |
+| 8 | [**Data Distribution**](subsystem_08_data_distribution.md) | [`fdbserver/datadistributor/`](https://github.com/apple/foundationdb/tree/main/fdbserver/datadistributor) | 22K | Shard management, team building, rebalancing, MoveKeys protocol |
+| 9 | [**Cluster Recovery**](subsystem_09_cluster_recovery.md) | [`fdbserver/clustercontroller/ClusterRecovery*`](https://github.com/apple/foundationdb/tree/main/fdbserver/clustercontroller), [`fdbserver/core/RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h) | (part of #4 files) | 10-state recovery state machine to reconstitute the transaction system after failures |
+| 10 | [**Rate Keeping & Throttling**](subsystem_10_ratekeeper.md) | [`fdbserver/ratekeeper/`](https://github.com/apple/foundationdb/tree/main/fdbserver/ratekeeper) | 4K | Back-pressure on commits and reads, tag-based throttling, throughput tracking |
+| 11 | [**Backup, Restore & DR**](subsystem_11_backup_dr.md) | [`fdbserver/backupworker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/backupworker), `fdbclient/FileBackupAgent*`, `fdbclient/BackupContainer*`, [`fdbbackup/`](https://github.com/apple/foundationdb/tree/main/fdbbackup) | ~10K server, ~20K client | Continuous backup to external storage, point-in-time restore, cross-cluster DR |
+| 12 | [**Simulation & Testing**](subsystem_12_simulation_testing.md) | [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), [`fdbserver/tester/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tester), [`tests/`](https://github.com/apple/foundationdb/tree/main/tests) | 51K workloads + sim | Deterministic simulation (Sim2), fault injection (Buggify), workload-based integration tests |
+
+Plus supporting code: [`fdbserver/worker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/worker) (process startup, ~4K), [`fdbserver/core/`](https://github.com/apple/foundationdb/tree/main/fdbserver/core) (shared interfaces/utilities, ~14K), [`fdbcli/`](https://github.com/apple/foundationdb/tree/main/fdbcli) (CLI, ~8K), [`bindings/`](https://github.com/apple/foundationdb/tree/main/bindings) (language FFI), [`fdbserver/consistencyscan/`](https://github.com/apple/foundationdb/tree/main/fdbserver/consistencyscan) (~2K).
+
+---
+
+## Detailed Subsystem Descriptions
+
+### 1. [Flow Runtime](subsystem_01_flow_runtime.md) ([`flow/`](https://github.com/apple/foundationdb/tree/main/flow))
+
+**What it is:** A custom async programming framework that underpins every line of FDB code. It is *not* a library you can opt out of — it *is* the execution model.
+
+**Key abstractions:**
+- **Future<T> / Promise<T>** — single-assignment async values. A Promise delivers once; a Future receives. Connected via reference-counted `SingleAssignmentVar` (SAV).
+- **FutureStream<T> / PromiseStream<T>** — multi-value async channels. The primary request/reply pattern: send a struct containing a `ReplyPromise<T>` into a `RequestStream`, and the receiver sends the reply back through it (self-addressed envelope).
+- **ACTOR functions** — compiled by a custom C# actor compiler (`actorcompiler.exe`) into state machines. `wait(expr)` suspends; `state` variables persist across suspensions. Being migrated to C++20 coroutines (`co_await`).
+- **Arena / StringRef / Standalone<T>** — zero-copy memory management. `StringRef` is a non-owning pointer+length; `Arena` owns the backing memory; `Standalone<T>` bundles a T with its Arena.
+- **Net2** ([`Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp)) — the real event loop. Single-threaded, priority-based task scheduling over Boost.ASIO. Handles timers, yields, I/O multiplexing.
+- **Deterministic random** ([`IRandom.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/IRandom.h)) — `deterministicRandom()` provides a seedable PRNG used everywhere, enabling reproducible simulation.
+- **Tracing** ([`Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp)) — structured event logging with `.detail()` chains, used for diagnostics and simulation analysis.
+
+**Key dynamic behavior:** Every server-side component is a collection of ACTOR coroutines scheduled on Net2's run loop. There is no threading within a process (except for disk I/O thread pools). All concurrency is cooperative, driven by Future resolution.
+
+**Principal files:** [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h), [`Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h), [`Error.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Error.h), [`genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h), [`Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp), [`Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp), [`serialize.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/serialize.h), [`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h)
+
+---
+
+### 2. [RPC & Transport](subsystem_02_rpc_transport.md) ([`fdbrpc/`](https://github.com/apple/foundationdb/tree/main/fdbrpc))
+
+**What it is:** The layer that makes Flow's Future/Promise work *across process boundaries*. It maps the actor model onto a network.
+
+**Key abstractions:**
+- **Endpoint** = `(NetworkAddressList, Token)` — a globally-unique address for a message receiver. Tokens are 128-bit UIDs. Well-known tokens exist for system services.
+- **FlowTransport** — singleton that owns all peer connections. `sendReliable()` guarantees delivery (retransmit on reconnect); `sendUnreliable()` is fire-and-forget.
+- **Peer** — per-destination state: unsent queue, reliable packet list, connection future, latency histogram. Connection is lazy (established on first send).
+- **EndpointMap** — maps Token → `NetworkMessageReceiver*`. When a packet arrives, its token is looked up and the receiver's `receive()` method is called to deserialize and deliver.
+- **FailureMonitor** — tracks endpoint/process liveness. Components register interest via `onFailed(Endpoint)` futures.
+- **Locality** ([`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h)) — `LocalityData` (zone, machine, datacenter, data hall) and `ProcessClass` (fitness for roles). Drives replication policy enforcement.
+- **Sim2** ([`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)) — *replaces* Net2 during simulation. Runs many virtual processes in one real thread with deterministic time, simulated network (clogging, latency, partitions), and fault injection (kill process/machine/zone/datacenter).
+
+**Key dynamic behavior:** When Actor A on Process 1 sends a request to Actor B on Process 2, FlowTransport serializes the message, TCP delivers it, the receiver's EndpointMap dispatches it, and the reply flows back the same way. In simulation, this all happens in-process with virtual routing. The abstraction is seamless — actors don't know whether they're simulated or real.
+
+**Principal files:** [`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h), [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h), [`FlowTransport.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/FlowTransport.cpp), [`FailureMonitor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FailureMonitor.h), [`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h), [`ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h), [`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h)
+
+---
+
+### 3. [Client Library](subsystem_03_client_library.md) ([`fdbclient/`](https://github.com/apple/foundationdb/tree/main/fdbclient))
+
+**What it is:** Everything an application (or internal FDB component acting as a client) uses to read and write data.
+
+**Transaction layers (outside-in):**
+1. **C API / Language Bindings** — FFI boundary. All external languages go through [`fdb_c.h`](https://github.com/apple/foundationdb/blob/main/bindings/c/foundationdb/fdb_c.h). Multi-version support loads different client `.so` files for protocol compatibility.
+2. **MultiVersionTransaction** — version-negotiating wrapper.
+3. **ReadYourWritesTransaction** (RYW) — the layer most internal code uses. Maintains a local write map and snapshot read cache. Reads check the write map first, then merge with storage server results. This provides read-your-own-writes semantics within a transaction.
+4. **NativeAPI / Transaction** — the raw transaction. `getReadVersion()` contacts a GRV proxy; `get()`/`getRange()` contact storage servers; `commit()` sends mutations + conflict ranges to a commit proxy.
+
+**Key dynamic behavior:**
+- **Location cache**: `DatabaseContext` caches key-range → storage-server mappings. On cache miss or stale entry, re-resolves via a commit proxy's `getKeyServerLocations`.
+- **Proxy load balancing**: `commitProxyLoadBalance()` / `grvProxyLoadBalance()` — round-robin with failover and automatic retry on proxy set changes.
+- **Retry loop**: `Transaction::onError()` handles `not_committed`, `transaction_too_old`, etc. by resetting and retrying with backoff. The canonical usage pattern is `loop { try { ... tr.commit(); break; } catch(Error& e) { wait(tr.onError(e)); } }`.
+- **Watches**: `tr.watch(key)` registers interest; the storage server notifies when the key changes.
+
+**Principal files:** [`NativeAPI.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/NativeAPI.actor.cpp), [`ReadYourWrites.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/ReadYourWrites.actor.cpp), [`DatabaseContext.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseContext.cpp), [`MultiVersionTransaction.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/MultiVersionTransaction.cpp), [`CommitTransaction.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/CommitTransaction.h), [`SystemData.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/SystemData.h), [`MonitorLeader.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/MonitorLeader.h)
+
+---
+
+### 4. [Cluster Controller & Coordination](subsystem_04_cluster_controller.md)
+
+**What it is:** The "brain" that decides who does what. One process is elected Cluster Controller (CC); it recruits and monitors all server roles.
+
+**Leader election:**
+- Coordinators (typically 3 or 5, addresses in the cluster file) run a `leaderRegister` actor.
+- CC candidates send `CandidacyRequest`s. Coordinators track candidates and nominate based on process class fitness (encoded in top bits of `LeaderInfo.changeID`).
+- The winner becomes CC; losers detect this via `monitorLeader()` and defer.
+- Uses a generation register (`localGenerationReg`) for safe state transitions — reads and writes carry generation numbers to prevent stale updates. This protocol is isomorphic to single-decree Paxos (generation = ballot, read = prepare, write = accept), with quorum voting in `CoordinatedState`.
+- Coordinator state is persisted via `KeyValueStoreMemory` backed by a `DiskQueue` (`.fdq` files) -- the same engine used by TLogs.
+
+**Role recruitment:**
+- Worker processes register with CC via `registrationClient()` ([`worker.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/worker/worker.actor.cpp)).
+- CC maintains a pool of available workers with their `ProcessClass` and `LocalityData`.
+- When the transaction system needs to be (re)constituted, CC recruits: Master/Sequencer, CommitProxies, GrvProxies, Resolvers, TLogs.
+- Recruitment considers fitness (class match), locality (datacenter placement), and excludes failed/excluded processes.
+- CC also recruits singleton roles: DataDistributor, Ratekeeper, ConsistencyScan.
+
+**Cluster-wide state:**
+- `ServerDBInfo` is the cluster-wide configuration broadcast. Contains: master interface, proxy lists, log system config, recovery state, latency band config.
+- Updated by CC and distributed to all workers. Workers react to changes (e.g., new proxy set).
+
+**Principal files:** [`ClusterController.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterController.actor.cpp), `ClusterController.h`, [`Coordination.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/Coordination.cpp), [`LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp), [`CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp), [`WorkerInterface.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h)
+
+---
+
+### 5. [Transaction Commit Pipeline](subsystem_05_commit_pipeline.md)
+
+**What it is:** The assembly line that takes a client's commit request and makes it durable. Four server roles collaborate in a strict pipeline.
+
+**The pipeline (a single commit's journey):**
+
+```
+Client ──CommitTransactionRequest──▶ CommitProxy
+                                        │
+Phase 1: GET VERSION                    │ GetCommitVersionRequest
+                                        ▼
+                                    Master/Sequencer
+                                        │ assigns monotonic version
+                                        │ (tracks prevVersion for ordering)
+                                        ▼
+Phase 2: RESOLVE                    CommitProxy
+                                        │ ResolveTransactionBatchRequest
+                                        ▼
+                                    Resolver(s)  [parallel, sharded by key range]
+                                        │ checks read-ranges vs committed write-ranges
+                                        │ returns conflict/no-conflict per txn
+                                        ▼
+Phase 3: LOG                        CommitProxy
+                                        │ pushes non-conflicting mutations
+                                        │ via LogSystem::push()
+                                        ▼
+                                    TLog replicas (quorum write)
+                                        │
+Phase 4: REPLY                      CommitProxy
+                                        │ CommitTransactionReply
+                                        ▼
+                                    Client (committed!)
+```
+
+**Master/Sequencer** ([`masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp)): Assigns commit versions via `getVersion()`. Ensures versions are monotonically increasing and roughly track wall-clock time. Also tracks the "live committed version" for GRV.
+
+**GRV Proxy** ([`GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp)): Handles `GetReadVersionRequest` from clients. Assigns read versions that are guaranteed to see all previously committed transactions. Enforces cluster-wide flow control: receives TPS limits from Ratekeeper and gates transaction starts using a smoothed token bucket (`GrvTransactionRateInfo`). Batch-priority transactions are throttled more aggressively and dropped first under pressure.
+
+**Commit Proxy** ([`CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp)): The workhorse. Batches commits (`commitBatcher`), drives the 4-phase pipeline, handles metadata mutations (system key writes), and updates the key-server location map.
+
+**Resolver** ([`Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp), [`ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp)): Maintains a sliding window of committed write ranges. For each incoming batch, checks if any transaction's read ranges overlap with committed writes at versions newer than the transaction's read version. Pure conflict detection — no side effects.
+
+**Principal files:** [`CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp), [`GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp), [`masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp), [`Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp), [`ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp), [`LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h)
+
+---
+
+### 6. [Transaction Log (TLog) & Log System](subsystem_06_tlog_logsystem.md)
+
+**What it is:** The durable write-ahead log. All committed mutations flow through TLogs before being applied to storage servers. This is FDB's durability guarantee.
+
+**TLog** ([`TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp)):
+- Receives mutation batches from commit proxies via `push()`.
+- Writes to a `DiskQueue` (append-only on-disk log) and an in-memory index.
+- Mutations are **tagged** — each mutation carries one or more `Tag`s indicating which storage server(s) it's relevant to. Tags are assigned by the commit proxy based on the key-to-shard mapping.
+- Provides `peek()` interface: storage servers pull mutations by tag and version range.
+- Quorum replication: a mutation is committed when `f+1` of `2f+1` TLog replicas acknowledge.
+
+**Log System** ([`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp)):
+- Abstraction over the set of TLog replicas.
+- `LogSystem::push()` — sends mutations to all relevant TLog replicas, waits for quorum.
+- `LogSystem::peek()` → `IPeekCursor` — merges results from multiple TLogs for a given tag, handling multi-generation log sets during recovery.
+- Manages log epochs: each recovery creates a new epoch of TLogs. Old epochs are kept until storage servers have consumed all their data.
+
+**Log Router** ([`LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp)):
+- In multi-region configurations, routes mutation data from the primary's TLogs to the remote region's TLogs.
+- Bridges the tag-partitioned log system across datacenters.
+
+**Key dynamic behavior:** Storage servers are *consumers* of the log. They continuously peek their assigned tag, pulling committed mutations and applying them to local storage. This decoupling means commits don't wait for storage servers — only for TLog quorum.
+
+**Principal files:** [`TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp), [`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp), [`LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp), [`LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h), [`LogSystemConfig.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/LogSystemConfig.h), [`IDiskQueue.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/IDiskQueue.h)
+
+---
+
+### 7. [Storage Server & Engines](subsystem_07_storage_server.md)
+
+**What it is:** The read path and durable key-value store. Storage servers serve client reads and maintain the materialized state of their assigned key ranges (shards).
+
+**Storage Server** ([`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)):
+- Each SS owns a set of key ranges (shards) and a version.
+- **Update loop**: continuously peeks its tag from the log system, deserializes mutations, applies them to the local KVStore, and advances its durable version.
+- **Read serving**: handles `GetKeyValuesRequest`, `GetKeyRequest`, `GetValueRequest`. Checks that the requested version is available (between oldest readable version and current version), that the shard is owned, and reads from the KVStore.
+- **Version management**: tracks `storageVersion` (latest durable), `durableVersion`, and `desiredOldestVersion`. Advertises its version to the cluster for GRV decisions.
+
+**Storage Engines** ([`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore)):
+- **IKeyValueStore** interface: `readValue()`, `readRange()`, `set()`, `clear()`, `commit()`. All operations are on a versioned key-value store.
+- **RocksDB** ([`KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp)) — primary production engine. Also a sharded variant ([`KeyValueStoreShardedRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp)) that uses per-shard column families.
+- **SQLite** ([`KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp)) — legacy engine, still supported.
+- **Memory** ([`KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp)) — for testing and special uses (e.g., txnStateStore during recovery).
+
+**Key dynamic behavior:** The storage server is *pull-based*. It pulls from the log system at its own pace. If a storage server falls behind, it catches up by reading more from TLogs. If it falls too far behind, it may be removed from its team and re-replicated. Reads at a given version are served from a snapshot — the SS maintains enough history to serve reads at any version between its oldest and current.
+
+**Principal files:** [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp), [`KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp), [`KeyValueStoreShardedRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp), [`KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp), [`KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp), [`IKeyValueStore.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h), [`StorageMetrics.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/StorageMetrics.cpp)
+
+---
+
+### 8. [Data Distribution](subsystem_08_data_distribution.md)
+
+**What it is:** The subsystem that decides which storage servers hold which key ranges, and moves data when the assignment needs to change.
+
+**Why data moves:**
+- A shard grows too large → split
+- A storage server fails → re-replicate its shards to healthy servers
+- Load imbalance → move shards from hot to cold servers
+- Configuration change (e.g., replication factor increase)
+- Exclusion of a server/machine/datacenter
+
+**Components:**
+- **DDShardTracker** ([`DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp)) — monitors shard sizes and read/write rates. Triggers splits, merges, and moves.
+- **DDTeamCollection** ([`DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp)) — builds "teams" of storage servers that satisfy the replication policy (e.g., 3 servers in 3 different zones). Maintains healthy/unhealthy team lists.
+- **DDRelocationQueue** ([`DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp)) — prioritized queue of shard movements. Executes moves concurrently up to a parallelism limit.
+- **MoveKeys** ([`fdbserver/core/MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp)) — the protocol for atomically transferring shard ownership. Uses a "move keys lock" to serialize concurrent moves. Updates the `keyServers` and `serverKeys` system key spaces.
+- **DataMovement** ([`fdbserver/core/DataMovement.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/DataMovement.cpp)) — data move metadata and physical shard ID tracking.
+
+**Key dynamic behavior:** Data distribution is a continuous background process. The DD singleton (recruited by CC) watches for changes in shard metrics, server health, and configuration. It produces `RelocateShard` requests that flow through the relocation queue, which executes the MoveKeys protocol. During a move, the destination SS fetches data from the source SS and begins consuming from the log system. When caught up, ownership transfers atomically via a system key transaction.
+
+**Principal files:** [`DataDistribution.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DataDistribution.cpp), [`DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp), [`DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp), [`DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp), [`MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp), [`DataMovement.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/DataMovement.cpp), [`ShardsAffectedByTeamFailure.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp)
+
+---
+
+### 9. [Cluster Recovery](subsystem_09_cluster_recovery.md)
+
+**What it is:** The state machine that reconstitutes the entire transaction system after a failure (master crash, network partition, coordinator change, etc.).
+
+**The 10 recovery states** (from [`RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h)):
+0. **UNINITIALIZED** — starting
+1. **READING_CSTATE** — read coordinated state from coordinators (learn about previous epoch)
+2. **LOCKING_CSTATE** — lock old TLogs to prevent split-brain
+3. **RECRUITING** — recruit new TLogs, CommitProxies, GrvProxies, Resolvers
+4. **RECOVERY_TRANSACTION** — replay the last epoch's committed-but-unapplied mutations; establish the recovery version
+5. **WRITING_CSTATE** — write the new epoch's coordinated state to coordinators
+6. **ACCEPTING_COMMITS** — new transaction system is live, accepting commits
+7. **ALL_LOGS_RECRUITED** — all TLog sets (including remote region) are populated
+8. **STORAGE_RECOVERED** — storage servers have recovered their data from the new TLogs
+9. **FULLY_RECOVERED** — old TLog data fully consumed; old generations can be discarded
+
+**Key operations during recovery:**
+- `readTransactionSystemState()` — peeks old TLogs to reconstruct the txnStateStore (key-server mappings, configuration, etc.)
+- `recruitEverything()` — recruits the full transaction system in parallel
+- `trackTlogRecovery()` — monitors TLog set completeness and writes coordinated state updates
+- `provisionalMaster()` — handles the gap between old and new transaction systems
+
+**Principal files:** [`ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp), [`ClusterRecovery.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.h), [`RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h), [`DBCoreState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/DBCoreState.h), [`CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp), [`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) (newEpoch)
+
+---
+
+### 10. [Rate Keeping & Throttling](subsystem_10_ratekeeper.md) ([`fdbserver/ratekeeper/`](https://github.com/apple/foundationdb/tree/main/fdbserver/ratekeeper))
+
+**What it is:** Back-pressure mechanism that prevents the cluster from being overwhelmed.
+
+**What it monitors:**
+- Storage server queue depths (how far behind are they from the log?)
+- TLog queue depths
+- Storage server disk throughput and IOPS
+- Per-tag transaction rates (for tenant/workload isolation)
+
+**What it controls:**
+- Transaction rate limits (`tpsLimit`) computed separately for normal (SYSTEM+DEFAULT) and batch priorities, sent to GRV proxies via `GetRateInfoReply`. Each proxy receives `tpsLimit / numProxies`. See [Subsystem 5](subsystem_05_commit_pipeline.md) for how GRV proxies enforce these limits.
+- Per-tag throttling via `TagThrottler` — can slow down or reject transactions from specific tags
+- Batch priority vs. default priority differentiation (batch uses more aggressive queue depth thresholds)
+
+**Principal files:** [`Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp), [`Ratekeeper.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.h), [`TagThrottler.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/TagThrottler.cpp), [`RkTagThrottleCollection.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/RkTagThrottleCollection.cpp)
+
+---
+
+### 11. [Backup, Restore & DR](subsystem_11_backup_dr.md)
+
+**What it is:** Continuous backup of the mutation stream to external storage, with point-in-time restore capability.
+
+**Backup architecture:**
+- **BackupWorker** ([`fdbserver/backupworker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/backupworker)) — server-side role that reads mutations from TLogs (like a storage server, but writes to backup storage instead of a local KVStore). Tag: `tagLocalityLogRouter`.
+- **FileBackupAgent** ([`FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp)) — orchestrates backup lifecycle. Uses a TaskBucket pattern for reliable, resumable tasks.
+- **BackupContainer** (`fdbclient/BackupContainer*`) — abstraction over backup storage destinations. Implementations: local directory, S3 blob store.
+- **Two file types**: Range files (snapshot of key-value pairs at a version) and Log files (mutation stream between versions). Together they enable point-in-time restore.
+
+**Restore:**
+- Restore agent reads range files to reconstruct base state, then applies log files to reach target version.
+- Supports key prefix transformation during restore (add/remove prefix).
+
+**DR (Disaster Recovery):**
+- `DatabaseBackupAgent` — streams mutations from one cluster to another in near-real-time.
+- Enables warm standby clusters.
+
+**Principal files:** [`BackupWorker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/backupworker/BackupWorker.cpp), [`FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp), [`DatabaseBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseBackupAgent.cpp), [`BackupContainer.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupContainer.h), [`BackupContainerFileSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/BackupContainerFileSystem.cpp), [`BackupAgent.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupAgent.h), [`backup.cpp`](https://github.com/apple/foundationdb/blob/main/fdbbackup/backup.cpp)
+
+---
+
+### 12. [Simulation & Testing](subsystem_12_simulation_testing.md)
+
+**What it is:** FDB's most distinctive engineering practice — deterministic simulation testing that can find bugs that would take years to manifest in production.
+
+**Sim2** ([`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)):
+- Replaces Net2 as the network implementation during tests.
+- Runs the entire cluster (multiple "processes") in a single OS thread with virtual time.
+- Deterministic: given the same random seed, produces identical execution.
+- Simulates: network latency/clogging/partitions, disk I/O errors, process crashes, machine failures, datacenter failures.
+
+**Buggify** (throughout the codebase):
+- `BUGGIFY` / `BUGGIFY_WITH_PROB(p)` — randomly enables rare code paths in simulation.
+- Examples: short timeouts, extra delays, buffer size changes, early returns.
+- Dramatically increases code path coverage during testing.
+- Disabled in production builds.
+
+**Workloads** ([`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), ~51K LOC):
+- `TestWorkload` base class with `setup()`, `start()`, `check()`, `getMetrics()`.
+- ~60+ workload types covering: API correctness, backup/restore, bulk operations, configuration changes, failure injection, performance benchmarks.
+- Configured via TOML files in [`tests/`](https://github.com/apple/foundationdb/tree/main/tests).
+- Multiple workloads can run concurrently in a single simulation.
+
+**Test runner** ([`fdbserver/tester/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tester)):
+- [`SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp) — sets up simulated cluster topology (machines, processes, datacenter layout).
+- `TestConfig` — reads TOML test configuration.
+- Orchestrates workload execution and result checking.
+
+**Principal files:** [`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h), [`SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp), [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), [`tests/`](https://github.com/apple/foundationdb/tree/main/tests)
+
+---
+
+## Data Flow Summary
+
+### Write Path (end to end)
+```
+App → C API → NativeAPI.commit() → CommitProxy → Master (version) → Resolver (conflicts)
+  → CommitProxy → LogSystem.push() → TLog quorum (durable) → reply to client
+  ~~async~~> StorageServer.peek() → apply mutations to KVStore
+```
+
+### Read Path (end to end)
+```
+App → C API → NativeAPI.getReadVersion() → GrvProxy (rate-limited by Ratekeeper TPS budget)
+App → C API → NativeAPI.get()/getRange() → [location cache lookup] → StorageServer
+  → KVStore.read() at requested version → reply to client
+```
+
+### Recovery Path
+```
+CC detects master failure → elect new CC (if needed) → read coordinated state
+  → lock old TLogs → recruit new {Master, Proxies, Resolvers, TLogs}
+  → replay old epoch → write new coordinated state → accept commits
+```
+
+### Data Movement Path
+```
+DDShardTracker detects imbalance → RelocateShard request → DDRelocationQueue
+  → MoveKeys protocol: dest SS fetches data + starts consuming log
+  → atomic ownership transfer via system key transaction
+```
diff --git a/design/AI-generated/subsystem_01_flow_runtime.md b/design/AI-generated/subsystem_01_flow_runtime.md
new file mode 100644
index 00000000000..60688cee2e1
--- /dev/null
+++ b/design/AI-generated/subsystem_01_flow_runtime.md
@@ -0,0 +1,378 @@
+# Subsystem 1: Flow Runtime
+
+**[Diagrams](diagram_01_flow_runtime.md)**
+
+**Location:** [`flow/`](https://github.com/apple/foundationdb/tree/main/flow)
+**Size:** ~31K implementation + headers  
+**Role:** The custom async programming framework that is the execution model for all FoundationDB code.
+
+---
+
+## Overview
+
+Flow is not a library you can opt out of -- it **is** the execution model. Every server component, every client operation, every test runs as a collection of cooperatively-scheduled coroutines (called "actors") on a single-threaded event loop. There are no application-level threads; all concurrency comes from Future resolution and callback dispatch.
+
+---
+
+## Key Data Structures
+
+### SAV (SingleAssignmentVar) -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:730-914`
+
+The core primitive underlying all async communication. A reference-counted cell that holds either a value or an error, set exactly once.
+
+```
+struct SAV<T> : Callback<T>, FastAllocated<SAV<T>> {
+    int promises;       // Promise reference count
+    int futures;        // Future reference count
+    aligned_storage<T>  value_storage;
+    Error error_state;  // UNSET_ERROR_CODE (-3), SET_ERROR_CODE, NEVER_ERROR_CODE, or actual error
+};
+```
+
+**Key methods:**
+- `send(U&& value)` -- sets value, fires all callbacks in the chain
+- `sendError(Error)` -- sets error, fires error callbacks
+- `isSet()` / `canBeSet()` / `isError()` -- state queries
+- `addCallbackAndDelFutureRef()` -- registers a continuation
+- `delPromiseRef()` / `delFutureRef()` -- reference counting; destroys when both reach 0
+
+**Callback chain:** SAV inherits from `Callback<T>`, which forms a **doubly-linked circular list**. Multiple waiters can register on one SAV. When the SAV fires, it walks the chain calling `fire(value)` or `error(err)` on each callback, then unlinks them.
+
+### Future<T> -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:973-1068`
+
+Read-only handle to a SAV. Cannot set the value, only observe it.
+
+```
+Future<T> f;           // default: points to nothing
+Future<T> f(value);    // immediately ready
+Future<T> f(Never());  // never completes
+Future<T> f(error);    // immediately errored
+```
+
+**Key methods:**
+- `get()` -- returns value or throws error (must be ready)
+- `isReady()` -- true if set (value or error)
+- `canGet()` -- ready and not error
+- `cancel()` -- calls `sav->cancel()`, used by actor system
+- `addCallbackAndClear(cb)` -- transfers ownership of callback to SAV
+
+### Promise<T> -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:1091-1157`
+
+Write-only handle to a SAV. Delivers the value exactly once.
+
+**Key methods:**
+- `send(value)` -- delivers value
+- `sendError(error)` -- delivers error
+- `getFuture()` -- returns the associated Future<T>
+- `isSet()` / `canBeSet()` -- state queries
+
+**Lifecycle:** Creating a Promise creates a SAV with `promises=1, futures=0`. Calling `getFuture()` increments `futures`. When the Promise is destroyed without sending, it sends `broken_promise()` error.
+
+### FutureStream<T> / PromiseStream<T> -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:1159-1456`
+
+Multi-value async channels for request/reply patterns.
+
+**NotifiedQueue<T>** (backing structure):
+- `std::queue<T, Deque<T>> queue` -- buffered values
+- Single callback chain (not doubly-linked)
+- `send(value)` -- if callback waiting, delivers directly; else queues
+- `pop()` -- dequeues or throws error
+
+**PromiseStream<T>** key pattern -- self-addressed envelope:
+```cpp
+struct MyRequest {
+    int param;
+    ReplyPromise<MyReply> reply;  // sender embeds a reply channel
+};
+PromiseStream<MyRequest> stream;
+stream.send(MyRequest{42, ReplyPromise<MyReply>()});
+// receiver calls req.reply.send(result)
+```
+
+No explicit backpressure -- values accumulate in the queue. Implicit pressure: if the consumer doesn't drain, memory grows.
+
+### Error / ErrorOr<T> -- [`Error.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Error.h)`:44-81`, [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:124-317`
+
+**Error class:**
+- `uint16_t error_code` + `uint16_t flags` (includes `FLAG_INJECTED_FAULT`)
+- Propagates through Futures: `SAV::sendError()` stores the error; `Future::get()` throws it
+- Defined via macros in `error_definitions.h`: `ERROR(name, number, description)`
+
+**ErrorOr<T>:**
+- `std::variant<Error, T>` -- holds either an error or a value
+- `present()` / `get()` / `getError()` -- access methods
+- Used by `tryGetReply()` to return errors without throwing
+
+---
+
+## The Actor System
+
+### ACTOR Compiler
+
+FDB uses a custom **C# actor compiler** (`actorcompiler.exe`) that transforms `.actor.cpp` files into standard C++. The `ACTOR` keyword marks a function as a coroutine:
+
+```cpp
+ACTOR Future<int> myActor(Future<int> input) {
+    state int x = 42;           // persists across waits
+    int val = wait(input);      // suspension point
+    return val + x;
+}
+```
+
+The compiler:
+1. Scans for `wait()` and `waitNext()` calls
+2. Breaks the function into callback states at each suspension point
+3. Generates a C++ class inheriting from `Actor<ReturnType>` with a SAV
+4. Each `wait()` becomes a callback registration + return
+5. `state` variables are promoted to class members (persist across suspensions)
+
+**Key macros** (`actorcompiler.h`):
+- `ACTOR` -- marks coroutine function
+- `state` -- marks variables that survive across `wait()`
+- `wait(Future<T>)` -- suspend until Future ready, return T
+- `waitNext(FutureStream<T>)` -- suspend until next stream value
+- `choose { when(wait(a)) {...} when(wait(b)) {...} }` -- race two futures
+- `loop` -- `while(true)` alias
+
+### C++20 Coroutine Integration -- [`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h)
+
+The codebase is being migrated from the custom actor compiler to C++20 native coroutines:
+
+**CoroPromise<T>** ([`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h)`:737-844`):
+- The `promise_type` for `Future<T>` coroutines
+- Embeds a `CoroActor<T>` which inherits from `Actor<T>` (which inherits from SAV)
+- `get_return_object()` returns `Future<T>` backed by the coroutine's SAV
+- `initial_suspend()` returns `suspend_never` (eager start)
+- `await_transform()` overloads convert `Future<U>` into `AwaitableFuture`
+
+**AwaitableFuture** ([`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h)`:453-534`):
+- The awaiter type for `co_await Future<T>`
+- `await_ready()` -- checks if future is ready or coroutine cancelled
+- `await_suspend()` -- registers callback with the future's SAV
+- `await_resume()` -- extracts value or throws error
+- Inherits from `Callback<T>` so it can be inserted into SAV's callback chain
+
+**Cancellation:** `CoroActor::cancel()` sets `ACTOR_WAIT_STATE_CANCELLED` and resumes the coroutine, which checks the flag in `await_ready()` and throws `actor_cancelled()`.
+
+---
+
+## Net2 Event Loop -- [`Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp)
+
+The real (non-simulated) event loop. Single-threaded, priority-based task scheduling over Boost.ASIO.
+
+### Net2 Class (line 139)
+
+```
+class Net2 : public INetwork, public INetworkConnections {
+    ASIOReactor reactor;              // Boost.ASIO wrapper
+    TaskQueue<PromiseTask> taskQueue; // Priority queue of pending tasks
+    std::atomic<double> currentTime;
+    std::atomic<bool> stopped;
+};
+```
+
+### Run Loop
+
+The `run()` method:
+1. Polls ASIO for I/O events (socket readable/writable, timers)
+2. Processes ready tasks from the priority queue (highest priority first)
+3. Tracks starvation metrics per priority level
+4. Yields back to ASIO periodically
+
+### Task Priority
+
+`TaskPriority` enum defines ~30 priority levels from `Min` to `Max`. Examples:
+- `RunLoop` (highest) -- event loop housekeeping
+- `ReadSocket` -- incoming network data
+- `DefaultPromiseEndpoint` -- RPC message delivery
+- `DefaultYield` -- cooperative yielding
+- `DefaultDelay` -- timer-based delays
+- `DiskWrite` / `DiskRead` -- I/O completions
+
+### Key Methods
+
+- `delay(seconds, taskId)` -- returns `Future<Void>` that fires after `seconds` at priority `taskId`
+- `yield(taskId)` -- suspends current actor, re-enqueues at `taskId` priority
+- `check_yield(taskId)` -- returns true if the current task has been running too long
+- `onMainThread(promise, taskId)` -- schedules work from a thread pool thread back onto the main thread
+
+### Boost.ASIO Integration
+
+- TCP/TLS connections via `boost::asio::ip::tcp::socket`
+- UDP sockets for connectionless messaging
+- Timer operations for `delay()` implementation
+- Thread pool integration for async file I/O
+
+---
+
+## Arena Memory System -- [`Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h)
+
+### Arena
+
+A bump-pointer allocator. Memory is allocated in blocks (`ArenaBlock`), never freed individually -- the entire Arena is freed at once.
+
+```
+class Arena {
+    Reference<ArenaBlock> impl;   // first block in chain
+    
+    Arena();                       // empty
+    Arena(size_t reservedSize);    // pre-allocate
+    void dependsOn(const Arena&); // link arenas (for reference counting)
+    void* operator new(size_t, Arena&);  // allocate in arena
+};
+```
+
+**ArenaBlock** ([`Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h)`:171-212`):
+- Fixed block sizes: `SMALL=64`, `LARGE=8193`
+- Bump pointer allocation: `bigUsed` tracks next free offset
+- Blocks chain together; all freed when Arena destroyed
+- `secure` flag for zero-on-dealloc (sensitive data)
+
+### StringRef / KeyRef / ValueRef
+
+Non-owning pointer+length references. Do not own memory.
+
+```
+struct StringRef {
+    const uint8_t* data;
+    int length;
+    
+    StringRef(Arena& a, const StringRef& toCopy);  // copy into arena
+    StringRef substr(int start, int size);
+    int compare(StringRef const& other);           // lexicographic
+};
+
+using KeyRef = StringRef;
+using ValueRef = StringRef;
+```
+
+### Standalone<T>
+
+Bundles a T with its own Arena, creating an owning wrapper:
+
+```
+template<class T>
+class Standalone : private Arena, public T {
+    Arena& arena();     // access the arena
+    T& contents();      // access the value
+};
+// Example: Standalone<StringRef> owns the bytes it points to
+```
+
+### VectorRef<T>
+
+Non-owning reference to an array of T. Like StringRef but for structured data.
+
+---
+
+## FastAlloc -- `FastAlloc.h`
+
+Thread-local pool allocator for common sizes (16, 32, 64, 96, 128, 256 bytes).
+
+- `FastAllocator<Size>::allocate()` -- O(1) from thread-local free list
+- `FastAllocator<Size>::release()` -- O(1) return to free list
+- Magazine system: when free list empty, swap with a full magazine from global pool
+- `FastAllocated<Object>` trait: overloads `operator new/delete` to use FastAllocator
+- No locks for allocation when free list has items
+- 128KB magazines exchanged between thread-local and global pools
+
+---
+
+## Serialization -- [`serialize.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/serialize.h)
+
+Unified framework for binary serialization:
+
+- `is_binary_serializable<T>` trait for fixed-size types (int, double, UID, etc.)
+- `Serializer<Archive, T>` template: calls `t.serialize(ar)` by default
+- Specializations for containers: `std::vector`, `std::map`, `std::variant`, etc.
+- `BinaryWriter` / `BinaryReader` archives
+- Protocol version support: `_IncludeVersion`, `_AssumeVersion`, `_Unversioned`
+- Flatbuffers integration via `ObjectSerializer` for network messages
+
+---
+
+## Tracing -- `Trace.h`, [`Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp)
+
+Structured event logging used everywhere:
+
+```cpp
+TraceEvent("MyEvent", self->dbgid)
+    .detail("Key", key)
+    .detail("Version", version)
+    .detail("Duration", timer() - startTime);
+```
+
+- Fluent `.detail()` chain builds key-value fields
+- Severity levels: `SevVerbose(0)`, `SevDebug(5)`, `SevInfo(10)`, `SevWarn(20)`, `SevError(40)`
+- `AuditedEvent` for security-critical events that bypass suppression
+- `.suppressFor(duration)` -- rate-limit repeated events
+- `.trackLatest(key)` -- keep latest value of a recurring event
+
+---
+
+## Generic Actors -- [`genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h)
+
+Key utility actors:
+
+| Actor | Purpose |
+|-------|---------|
+| `timeout(Future<T>, seconds, defaultValue)` | Race future against timer |
+| `timeoutError(Future<T>, seconds)` | Timeout throws `timed_out()` |
+| `delayed(Future<T>, seconds)` | Add delay after completion |
+| `errorOr(Future<T>)` | Catch error into `ErrorOr<T>` |
+| `store(X& out, Future<T>)` | Store result in variable |
+| `waitForAll(vector<Future<T>>)` | Wait for all futures |
+| `waitForAllReady(vector<Future<T>>)` | Wait ignoring errors |
+| `map(Future<T>, func)` | Transform value |
+| `mapAsync(Future<T>, actorFunc)` | Async transform |
+| `holdWhile(object, Future<T>)` | Keep object alive until future resolves |
+| `uncancellable(Future<T>)` | Prevent cancellation |
+
+**AsyncVar<T>** ([`genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h)`:753-785`):
+- Reactive variable: holds a value, notifies on change
+- `get()` -- current value
+- `onChange()` -- returns `Future<Void>` that fires on next change
+- `set(value)` -- updates value and triggers onChange if different
+- Used extensively for cluster state (e.g., `AsyncVar<ServerDBInfo>`)
+
+---
+
+## Data Flow Summary
+
+### Actor Execution Cycle
+
+```
+1. Actor created → SAV allocated, initial code runs until first wait()
+2. wait(future) → registers callback on future's SAV, returns to event loop
+3. Event fires (network data, timer, another actor sends) → SAV::send(value)
+4. Callback chain walked → actor's callback called
+5. Actor resumes from wait point, runs until next wait() or return
+6. return value → SAV::send(value) on actor's own SAV → caller's callback fires
+```
+
+### Memory Flow
+
+```
+1. Arena allocated (bump pointer in ArenaBlock)
+2. StringRef/KeyRef/ValueRef created referencing arena memory
+3. Standalone<T> bundles T + Arena for ownership transfer
+4. When Standalone destroyed, Arena freed, all StringRefs invalidated
+5. No per-object deallocation -- whole Arena freed at once
+```
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`flow/include/flow/flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h) | Master include: Future, Promise, SAV, Callback, ErrorOr |
+| [`flow/include/flow/Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h) | Arena, StringRef, Standalone, VectorRef |
+| [`flow/include/flow/Error.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Error.h) | Error class and error code definitions |
+| [`flow/include/flow/CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h) | C++20 coroutine integration (CoroPromise, AwaitableFuture) |
+| [`flow/include/flow/genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h) | Utility actors (delay, timeout, waitForAll, AsyncVar) |
+| [`flow/include/flow/serialize.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/serialize.h) | Serialization framework |
+| `flow/include/flow/FastAlloc.h` | Thread-local pool allocator |
+| [`flow/Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp) | Real event loop (Boost.ASIO based) |
+| [`flow/Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp) | Structured event tracing |
+| `flow/include/flow/actorcompiler.h` | ACTOR/wait/state macro definitions |
diff --git a/design/AI-generated/subsystem_02_rpc_transport.md b/design/AI-generated/subsystem_02_rpc_transport.md
new file mode 100644
index 00000000000..f00fcf71315
--- /dev/null
+++ b/design/AI-generated/subsystem_02_rpc_transport.md
@@ -0,0 +1,340 @@
+# Subsystem 2: RPC & Transport
+
+**[Diagrams](diagram_02_rpc_transport.md)**
+
+**Location:** [`fdbrpc/`](https://github.com/apple/foundationdb/tree/main/fdbrpc)
+**Size:** ~28K implementation + headers  
+**Role:** Maps Flow's actor model onto a network -- makes Future/Promise work across process boundaries.
+
+---
+
+## Overview
+
+FlowTransport is the layer that lets actors on different processes (or simulated processes) communicate transparently. It provides endpoint-addressed messaging with both reliable (at-least-once) and unreliable (at-most-once) delivery semantics. In simulation, the same code runs in-process with virtual routing via Sim2.
+
+---
+
+## Key Data Structures
+
+### Endpoint -- [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h)`:43-120`
+
+A globally unique address for a message receiver:
+
+```
+class Endpoint {
+    NetworkAddressList addresses;  // primary + optional secondary (TLS/non-TLS)
+    Token token;                   // UID: 128-bit unique identifier
+};
+```
+
+- **Token** = `UID` (pair of `uint64_t`). Lower 32 bits encode an index into the EndpointMap; upper 32 bits encode task priority.
+- **Well-known tokens**: `wellKnownToken(int id)` returns `UID(-1, id)`. Reserved IDs: `WLTOKEN_ENDPOINT_NOT_FOUND(0)`, `WLTOKEN_PING_PACKET`, `WLTOKEN_UNAUTHORIZED_ENDPOINT`, plus system services (leader election, config transactions, etc.)
+- **Address selection**: `choosePrimaryAddress()` swaps primary/secondary based on local TLS preference.
+
+### Peer -- [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h)`:149-193`
+
+Per-destination connection state:
+
+```
+struct Peer : ReferenceCounted<Peer> {
+    NetworkAddress destination;
+    UnsentPacketQueue unsent;          // buffered outgoing packets
+    ReliablePacketList reliable;       // retransmit list (circular linked)
+    AsyncTrigger dataToSend;           // signal: unsent queue non-empty
+    Future<Void> connect;              // connectionKeeper actor
+    bool compatible, connected;
+    double reconnectionDelay;          // exponential backoff
+    int peerReferences;                // stream reference count (-1 = no streams)
+    int outstandingReplies;
+    DDSketch<double> pingLatencies;    // latency distribution
+    AsyncVar<Optional<ProtocolVersion>> protocolVersion;
+    Promise<Void> disconnect;          // notified on connection loss
+};
+```
+
+**Connection lifecycle:**
+1. First send to address creates Peer and starts `connectionKeeper()` actor
+2. `connectionKeeper()` calls `INetworkConnections::connect()`, performs TLS handshake
+3. Sends `ConnectPacket` (protocol version, local address, connection ID)
+4. Spawns `connectionWriter()` (async write loop) and `connectionReader()` (async read loop)
+5. `connectionMonitor()` sends periodic pings, detects timeouts
+6. On failure: exponential backoff (INITIAL_RECONNECTION_TIME to MAX_RECONNECTION_TIME, growth factor 1.5x)
+7. `discardUnreliablePackets()` on disconnect; reliable packets resent after reconnect
+
+### EndpointMap -- [`FlowTransport.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/FlowTransport.cpp)`:90-230`
+
+Maps tokens to local message receivers:
+
+```
+struct Entry {
+    union { uint64_t uid[2]; uint32_t nextFree; };
+    NetworkMessageReceiver* receiver;
+};
+```
+
+- Pre-allocates slots for well-known endpoints (indices 0 to wellKnownEndpointCount-1)
+- Dynamic endpoints allocated from free list; doubles table size when full
+- `get(token)` -- O(1) lookup by token's lower 32 bits
+- `insert()` -- allocates from free list, encodes priority in upper 32 bits
+- `remove()` -- returns slot to free list
+
+### FlowTransport -- [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h)`:199-315`
+
+Singleton managing all network communication:
+
+**Key methods:**
+- `bind(publicAddress, listenAddress)` -- start listening
+- `sendReliable(data, endpoint)` -- guaranteed delivery (retransmit on reconnect)
+- `sendUnreliable(data, endpoint, openConnection)` -- fire-and-forget
+- `addEndpoint(endpoint, receiver, priority)` -- register local endpoint
+- `addWellKnownEndpoint(endpoint, receiver, priority)` -- register system service
+- `removeEndpoint(endpoint, receiver)` -- unregister
+- `resetConnection(address)` -- force reconnect
+- `getDegraded()` -- `AsyncVar<bool>` for network degradation status
+
+---
+
+## Request/Reply Pattern -- [`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)
+
+The fundamental communication pattern in FDB:
+
+### RequestStream<T> ([`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)`:728-949`)
+
+A typed channel for sending requests to a service. Wraps a `NetNotifiedQueue<T>`:
+
+```cpp
+RequestStream<GetValueRequest> getValue;  // on StorageServerInterface
+
+// Sending a request:
+GetValueRequest req;
+req.key = myKey;
+req.reply = ReplyPromise<GetValueReply>();  // self-addressed envelope
+stream.send(req);
+GetValueReply reply = wait(req.reply.getFuture());
+```
+
+### ReplyPromise<T> ([`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)`:131-204`)
+
+Single-value reply channel embedded in requests:
+
+- Wraps a `NetSAV<T>` (SAV + FlowReceiver + FastAllocated)
+- Serialization only stores the endpoint token (16 bytes on wire)
+- On deserialize at receiver: creates local endpoint to receive reply
+
+### Complete Request-Reply Lifecycle
+
+```
+CLIENT:
+1. Create request with embedded ReplyPromise<T>
+2. stream.send(request) → serialize → FlowTransport.sendReliable()
+3. Transport finds/creates Peer → queues in unsent buffer
+4. connectionWriter() sends packet: [length][checksum][token][serialized data]
+5. Client waits on reply.getFuture()
+
+SERVER:
+6. connectionReader() receives packet → scanPackets()
+7. Extract token → EndpointMap lookup → find receiver
+8. deliver() actor: yield to correct TaskPriority, call receiver.receive()
+9. NetNotifiedQueue deserializes request, queues for application
+10. Application calls request.reply.send(response)
+11. sendUnreliable() back to client's reply endpoint
+
+CLIENT:
+12. connectionReader() receives reply packet
+13. Token lookup → NetSAV::receive() → deserialize → SAV::send(value)
+14. Client's wait() resolves
+```
+
+### ReplyPromiseStream<T> ([`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)`:460-640`)
+
+Stream of replies with flow control:
+- Server sends multiple responses
+- Client acknowledges consumed bytes
+- `setByteLimit()` configures backpressure threshold
+- `onReady()` blocks server when too much data in-flight
+
+---
+
+## Wire Protocol
+
+### Packet Format
+
+```
+┌─────────────────┬────────────┬───────────────┬──────────────┐
+│ Packet Length    │ Checksum   │ Token (UID)   │ Message Data │
+│ (4 bytes)       │ (8 bytes)* │ (16 bytes)    │ (variable)   │
+└─────────────────┴────────────┴───────────────┴──────────────┘
+* Checksum only for non-TLS connections (XXH3_64bits)
+```
+
+### ConnectPacket (sent at connection start)
+
+```cpp
+struct ConnectPacket {
+    uint32_t connectPacketLength;
+    ProtocolVersion protocolVersion;
+    uint16_t canonicalRemotePort;
+    uint64_t connectionId;           // multi-version client ID
+    uint32_t canonicalRemoteIp4;     // or 0 for IPv6
+    uint16_t flags;                  // FLAG_IPV6
+    uint8_t canonicalRemoteIp6[16];
+};
+```
+
+---
+
+## Failure Monitoring -- [`FailureMonitor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FailureMonitor.h)
+
+Tracks endpoint/process availability:
+
+```
+struct FailureStatus { bool failed; };
+
+class SimpleFailureMonitor : public IFailureMonitor {
+    std::unordered_map<NetworkAddress, FailureStatus> addressStatus;
+    YieldedAsyncMap<Endpoint, bool> endpointKnownFailed;
+    AsyncMap<NetworkAddress, bool> disconnectTriggers;
+};
+```
+
+**Key methods:**
+- `getState(endpoint)` -- current failure status
+- `onStateChanged(endpoint)` -- future that fires on status change
+- `onDisconnectOrFailure(endpoint)` -- wait for either event
+- `onlyEndpointFailed(endpoint)` -- endpoint failed but address still up
+- `setStatus(address, status)` -- update status
+
+**Detection mechanism** (`connectionMonitor()` actor):
+1. Sends periodic ping to `WLTOKEN_PING_PACKET`
+2. Waits for reply with `CONNECTION_MONITOR_TIMEOUT` (1s default)
+3. On timeout: increment count, eventually throw `connection_failed()`
+4. Records ping latency in `DDSketch` histogram
+5. After `FAILURE_DETECTION_DELAY` (5s): mark address as failed
+
+---
+
+## Locality & Replication -- [`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h), [`ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h)
+
+### LocalityData
+
+Key-value map of location attributes for each process:
+
+| Key | Example | Purpose |
+|-----|---------|---------|
+| `keyProcessId` | UUID | Unique process identifier |
+| `keyZoneId` | "zone-a" | Failure zone |
+| `keyMachineId` | "machine-1" | Physical machine |
+| `keyDcId` | "dc-east" | Datacenter |
+| `keyDataHallId` | "hall-1" | Data hall within DC |
+
+### ProcessClass ([`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h)`:27-154`)
+
+Classifies process suitability for roles:
+- **ClassType**: `StorageClass`, `TransactionClass`, `LogClass`, `CommitProxyClass`, `GrvProxyClass`, `MasterClass`, `ResolverClass`, etc.
+- **Fitness**: `BestFit` > `GoodFit` > `UnsetFit` > `OkayFit` > `WorstFit` > `ExcludeFit` > `NeverAssign`
+
+### Replication Policies ([`ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h))
+
+Composable constraint system:
+- **PolicyOne** -- select 1 replica anywhere
+- **PolicyAcross(count, attribute, subpolicy)** -- "N replicas across distinct values of attribute"
+  - Example: `PolicyAcross(3, "dcId", PolicyOne())` = 3 datacenters
+- **PolicyAnd(policies...)** -- intersection of multiple policies
+
+Used by team building (Data Distribution) to ensure failure domain separation.
+
+### Load Balance Distance
+
+```
+enum LBDistance { SAME_MACHINE=0, SAME_DC=1, DISTANT=2 };
+```
+
+Computed from locality attributes; used for preferring nearby replicas.
+
+---
+
+## Sim2 -- Deterministic Simulator -- [`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h)
+
+Replaces Net2 during testing. Runs the entire cluster in a single OS thread with virtual time.
+
+### Key Properties
+
+- **Deterministic**: Given same random seed → identical execution
+- **Virtual time**: No wall-clock dependency; time advances via task scheduling
+- **Multi-process**: Many virtual processes in one real thread via `ProcessInfo`
+- **Fault injection**: Kill processes, machines, zones, datacenters; inject network clogging, disconnection, disk errors
+
+### ProcessInfo (`SimulatorProcessInfo.h:42-170`)
+
+State per simulated process:
+- `name`, `dataFolder`, `coordinationFolder`
+- `machine` -- `MachineInfo*` for shared machine state
+- `addresses` -- virtual network addresses
+- `locality` -- zone/machine/DC placement
+- `failed`, `excluded`, `rebooting`, `failedDisk` -- fault state
+- `fault_injection_p1`, `fault_injection_p2` -- injection probabilities
+- `globals` -- per-process global variables
+- `shutdownSignal` -- Promise<KillType> for clean shutdown
+
+### SimClogging ([`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)`:198-297`)
+
+Simulates network impairment:
+- `getSendDelay()` / `getRecvDelay()` -- adds latency between addresses
+- `clogPairFor(addr1, addr2, seconds)` -- delay packets between pair
+- `disconnectPairFor(addr1, addr2, seconds)` -- sever connection between pair
+- Latency model: 99.9% fast (`FAST_NETWORK_LATENCY`), 0.1% slow (`SLOW_NETWORK_LATENCY`)
+
+### Kill Operations
+
+| Method | Scope | Effect |
+|--------|-------|--------|
+| `killProcess(pid, kt)` | Single process | Fail/fault/reboot process |
+| `killMachine(machineId, kt)` | All processes on machine | Checks replication safety first |
+| `killZone(zoneId, kt)` | All machines in zone | |
+| `killDataCenter(dcId, kt)` | All machines in DC | |
+
+**KillTypes**: `KillInstantly`, `InjectFaults`, `FailDisk`, `Reboot`, `RebootProcess`, `RebootAndDelete`, `RebootProcessAndDelete`
+
+---
+
+## Load Balancing -- `LoadBalance.actor.h`
+
+Client-side request distribution:
+
+- **QueueModel**: Estimates response latency based on pending requests per server
+- **Round-robin**: Distributes across replicas
+- **Failure routing**: Routes around failed/degraded servers
+- **Locality-aware**: Prefers geographically proximate servers (SAME_MACHINE > SAME_DC > DISTANT)
+- **TSS comparison**: Can shadow requests to Testing Storage Servers for correctness validation
+
+---
+
+## Key Constants
+
+| Knob | Default | Purpose |
+|------|---------|---------|
+| `INITIAL_RECONNECTION_TIME` | 0.2s | First reconnect delay |
+| `MAX_RECONNECTION_TIME` | 30s | Max backoff |
+| `RECONNECTION_TIME_GROWTH_RATE` | 1.5x | Backoff multiplier |
+| `CONNECTION_MONITOR_TIMEOUT` | 1s | Ping timeout |
+| `FAILURE_DETECTION_DELAY` | 5s | Before marking failed |
+| `PACKET_LIMIT` | 512MB | Max packet size |
+| `MIN_COALESCE_DELAY` | 0.5ms | Min buffer wait |
+| `MAX_COALESCE_DELAY` | 2ms | Max buffer wait |
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbrpc/include/fdbrpc/fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h) | RequestStream, ReplyPromise, NetSAV, request/reply pattern |
+| [`fdbrpc/include/fdbrpc/FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h) | Endpoint, Peer, FlowTransport API |
+| [`fdbrpc/FlowTransport.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/FlowTransport.cpp) | Transport implementation, EndpointMap, connection actors |
+| [`fdbrpc/include/fdbrpc/FailureMonitor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FailureMonitor.h) | Failure detection interface and implementation |
+| [`fdbrpc/include/fdbrpc/Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h) | LocalityData, ProcessClass, LBDistance |
+| [`fdbrpc/include/fdbrpc/ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h) | PolicyOne, PolicyAcross, PolicyAnd |
+| `fdbrpc/include/fdbrpc/LoadBalance.actor.h` | Client-side load balancing |
+| [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp) | Sim2 deterministic simulator |
+| [`fdbrpc/include/fdbrpc/simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h) | ISimulator interface, ProcessInfo |
+| `flow/include/flow/IConnection.h` | IConnection, IListener interfaces |
diff --git a/design/AI-generated/subsystem_03_client_library.md b/design/AI-generated/subsystem_03_client_library.md
new file mode 100644
index 00000000000..e3bb2071b82
--- /dev/null
+++ b/design/AI-generated/subsystem_03_client_library.md
@@ -0,0 +1,288 @@
+# Subsystem 3: Client Library
+
+**[Diagrams](diagram_03_client_library.md)**
+
+**Location:** [`fdbclient/`](https://github.com/apple/foundationdb/tree/main/fdbclient)
+**Size:** ~75K implementation, ~34K headers  
+**Role:** Everything an application (or internal FDB component) uses to read and write data.
+
+---
+
+## Overview
+
+The client library provides a layered transaction API. From outside in: C API / language bindings -> MultiVersionTransaction -> ReadYourWritesTransaction -> NativeAPI Transaction. Most internal FDB code uses ReadYourWritesTransaction. External applications go through the C API.
+
+---
+
+## Transaction Layers
+
+### Layer 1: NativeAPI / Transaction -- `NativeAPI.actor.h:311-525`
+
+The raw transaction. Directly communicates with proxies and storage servers.
+
+**TransactionState** (`NativeAPI.actor.h:255-309`):
+```
+- Database cx                    // DatabaseContext reference
+- Future<Version> readVersionFuture
+- TransactionOptions options
+- int64_t totalCost             // accumulated operation cost for throttling
+- Version committedVersion      // set after successful commit
+```
+
+**Key methods:**
+- `getReadVersion()` -- fetches version from GRV proxy (batched via versionBatcher)
+- `get(key, Snapshot)` -- point read from storage server, adds read conflict range if !snapshot
+- `getRange(begin, end, limits, Snapshot, Reverse)` -- range scan from storage server(s)
+- `set(key, value)` -- appends `MutationRef(SetValue, key, value)` to mutations vector
+- `clear(range)` -- appends `MutationRef(ClearRange, begin, end)`
+- `commit()` -- sends `CommitTransactionRequest` to commit proxy via `tryCommit()`
+- `onError(error)` -- handles retryable errors, resets transaction, implements backoff
+
+### Layer 2: ReadYourWritesTransaction -- `ReadYourWrites.h:69-268`
+
+Adds read-your-own-writes semantics on top of NativeAPI:
+
+```
+class ReadYourWritesTransaction {
+    Arena arena;
+    Transaction tr;              // underlying native transaction
+    SnapshotCache cache;         // cached reads from storage servers
+    WriteMap writes;             // pending local writes
+    CoalescedKeyRefRangeMap<bool> readConflicts;
+};
+```
+
+**RYW read flow:**
+1. Check `writes` (local write map) for the key
+2. If found: return local value (or "cleared" indicator)
+3. If not found: check `cache` (snapshot cache) for previously-read value
+4. If not cached: issue read to storage server, cache result
+5. Merge results using `RYWIterator` which walks both maps in parallel
+
+**RYW write flow:**
+- `set()`/`clear()` buffer in local `WriteMap`
+- On `commit()`: flush all writes to underlying `Transaction`, then commit
+
+### Layer 3: MultiVersionTransaction -- [`MultiVersionTransaction.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/MultiVersionTransaction.cpp)
+
+Supports running against clusters with different protocol versions:
+
+- **DLTransaction**: Wraps C API function pointers from dynamically-loaded client library
+- **MultiVersionApi**: Loads multiple client `.so` files for protocol compatibility
+- Converts between `ThreadFuture<T>` (cross-thread) and internal `Future<T>`
+- Enables rolling upgrades with mixed-version clusters
+
+### Layer 4: C API / Language Bindings -- [`fdb_c.h`](https://github.com/apple/foundationdb/blob/main/bindings/c/foundationdb/fdb_c.h)
+
+FFI boundary for external languages:
+- `fdb_setup_network()` / `fdb_run_network()` / `fdb_stop_network()` -- lifecycle
+- `fdb_database_create_transaction()` -- create transaction
+- `fdb_transaction_get()` / `fdb_transaction_set()` / `fdb_transaction_commit()` -- operations
+- All async operations return `FDBFuture*` handles
+
+---
+
+## DatabaseContext -- `DatabaseContext.h`, [`DatabaseContext.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseContext.cpp)
+
+Central state for a database connection. Shared by all transactions on this connection.
+
+### Location Cache (`DatabaseContext.h:410`)
+
+```
+CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache;
+```
+
+Maps key ranges to storage server locations. Used to route reads directly to the right SS without asking a proxy every time.
+
+- `getCachedLocation(key)` -- O(log n) lookup in range map
+- `setCachedLocation(range, servers)` -- insert/update, evicts old entries when cache full
+- `invalidateCache(key)` / `invalidateCache(range)` -- clear on shard movement
+
+### Proxy Management
+
+```
+Reference<CommitProxyInfo> commitProxies;   // for commits and location queries
+Reference<GrvProxyInfo> grvProxies;         // for read version assignment
+AsyncTrigger onProxiesChanged;              // fires when proxy set updates
+```
+
+Updated by `monitorProxies()` actor which watches for cluster configuration changes.
+
+### GRV Cache
+
+```
+double lastGrvTime;
+Version cachedReadVersion;
+```
+
+Caches recent read versions to reduce GRV proxy load. Background updater via `backgroundGrvUpdater()`.
+
+---
+
+## Commit Flow -- [`NativeAPI.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/NativeAPI.actor.cpp)`:4369+`
+
+### tryCommit() Actor
+
+1. **Estimate cost**: If tagged transaction, calculate cost for throttling
+2. **Build CommitTransactionRequest**:
+   - `read_snapshot` = read version
+   - `mutations[]` = all set/clear/atomic operations
+   - `read_conflict_ranges[]` = keys that were read
+   - `write_conflict_ranges[]` = keys that were written
+3. **Select proxy**: `basicLoadBalance()` across commit proxies, or first proxy if `commitOnFirstProxy`
+4. **Send and wait**: Retry on proxy set change
+5. **Handle reply**:
+   - **Success** (version != invalidVersion): update `committedVersion`, send versionstamp, update metadata cache
+   - **Conflict** (version == invalidVersion): extract conflicting key info, throw `not_committed()`
+   - **Unknown result**: use idempotency ID to determine final status
+
+### CommitTransactionRequest structure
+
+```
+struct CommitTransactionRequest {
+    SpanContext spanContext;
+    TransactionRef transaction {
+        VectorRef<MutationRef> mutations;
+        VectorRef<KeyRangeRef> read_conflict_ranges;
+        VectorRef<KeyRangeRef> write_conflict_ranges;
+        Version read_snapshot;
+    };
+    bool report_conflicting_keys;
+    // ...
+};
+```
+
+### MutationRef -- [`CommitTransaction.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/CommitTransaction.h)`:64-150`
+
+```
+struct MutationRef {
+    enum Type : uint8_t {
+        SetValue=0, ClearRange, AddValue, CompareAndClear,
+        SetVersionstampedKey, SetVersionstampedValue,
+        // atomic ops: And, Or, Xor, AppendIfFits, Max, Min, ByteMin, ByteMax, ...
+    };
+    uint8_t type;
+    StringRef param1;  // key (or begin key for ClearRange)
+    StringRef param2;  // value (or end key for ClearRange)
+};
+```
+
+---
+
+## Read Flow
+
+### Point Read -- `get(key)`
+
+```
+1. getReadVersion()  →  GRV proxy (batched)
+2. getKeyLocation(key)  →  location cache or commit proxy
+3. getValue(key, version)  →  storage server via loadBalance()
+4. Return value (or Optional<Value>() if not found)
+```
+
+### Range Read -- `getRange(begin, end, limits)`
+
+```
+1. getReadVersion()
+2. Resolve KeySelectors to concrete keys
+3. For each shard in range:
+   a. getKeyLocation()  →  find storage server
+   b. Send GetKeyValuesRequest to SS
+   c. Accumulate results respecting row/byte limits
+   d. If more data and limits not reached, continue to next shard
+4. Return RangeResult
+```
+
+### KeySelector Resolution
+
+```
+struct KeySelectorRef {
+    KeyRef key;
+    bool orEqual;
+    int offset;
+};
+
+firstGreaterOrEqual(k) = KeySelector(k, false, +1)
+lastLessOrEqual(k)     = KeySelector(k, true, 0)
+lastLessThan(k)        = KeySelector(k, false, 0)
+firstGreaterThan(k)    = KeySelector(k, true, +1)
+```
+
+### Proxy Load Balancing -- `ProxyLoadBalance.h`
+
+```cpp
+// Pattern: send request, race against proxy set change
+while (true) {
+    auto reply = basicLoadBalance(cx->getCommitProxies(), channel, req);
+    auto result = co_await race(reply, cx->onProxiesChanged());
+    if (result.index() == 0) co_return std::get<0>(result);
+    // proxy set changed, retry with new proxies
+}
+```
+
+---
+
+## Cluster Discovery -- [`MonitorLeader.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/MonitorLeader.h)
+
+### Connection Flow
+
+1. Parse cluster file: `description:id@coord1,coord2,coord3`
+2. `monitorLeader()` contacts coordinators, tracks current cluster controller
+3. `monitorProxies()` fetches commit/GRV proxy lists from cluster controller
+4. `DatabaseContext` updated with new proxy interfaces
+5. Transactions use proxies for all operations
+
+### Watch Operations
+
+```cpp
+Future<Void> watch = tr.watch(key);  // register interest
+wait(tr.commit());                    // commit first
+wait(watch);                          // then wait for change
+```
+
+- Watch metadata tracked in `DatabaseContext::watchMap`
+- Storage server monitors key, notifies on value change
+- Watch survives transaction commit
+
+---
+
+## System Keys -- [`SystemData.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/SystemData.h)
+
+Special key ranges for cluster metadata (prefix `\xff`):
+
+| Range | Purpose |
+|-------|---------|
+| `keyServersKeys` (`\xff/keyServers/`) | key range → storage server mapping |
+| `serverKeysRange` (`\xff/serverKeys/`) | storage server → owned key ranges (reverse map) |
+| `serverListKeys` (`\xff/serverList/`) | storage server interfaces |
+| `configKeys` (`\xff/conf/`) | database configuration |
+| `tagLocalityListKeys` | datacenter locality assignments |
+| `serverTagKeys` | storage server tag assignments |
+| `moveKeysLockOwnerKey` | data distribution lock |
+| `backupStartedKey` | active backup state |
+
+---
+
+## Special Key Space (`\xff\xff` prefix)
+
+Management API exposed as key-value operations:
+
+- Read `\xff\xff/status/json` for cluster status
+- Write to `\xff\xff/management/` for configuration changes
+- `SpecialKeyRangeReadImpl` / `SpecialKeyRangeRWImpl` abstract handlers
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbclient/NativeAPI.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/NativeAPI.actor.cpp) | Core transaction implementation, tryCommit, getValue, getRange |
+| `fdbclient/include/fdbclient/NativeAPI.actor.h` | Transaction class, TransactionState |
+| [`fdbclient/ReadYourWrites.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/ReadYourWrites.actor.cpp) | RYW layer, write map, snapshot cache merging |
+| `fdbclient/include/fdbclient/DatabaseContext.h` | DatabaseContext: location cache, proxy tracking, watches |
+| [`fdbclient/include/fdbclient/CommitTransaction.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/CommitTransaction.h) | MutationRef, CommitTransactionRef |
+| [`fdbclient/MultiVersionTransaction.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/MultiVersionTransaction.cpp) | Multi-version client, DLTransaction |
+| `fdbclient/MonitorLeader.cpp` | Cluster discovery, leader tracking |
+| [`fdbclient/include/fdbclient/SystemData.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/SystemData.h) | System key range definitions |
+| `fdbclient/ProxyLoadBalance.h` | Proxy load balancing templates |
diff --git a/design/AI-generated/subsystem_04_cluster_controller.md b/design/AI-generated/subsystem_04_cluster_controller.md
new file mode 100644
index 00000000000..8d29c2e8d5b
--- /dev/null
+++ b/design/AI-generated/subsystem_04_cluster_controller.md
@@ -0,0 +1,319 @@
+# Subsystem 4: Cluster Controller & Coordination
+
+**[Diagrams](diagram_04_cluster_controller.md)**
+
+**Location:** [`fdbserver/clustercontroller/`](https://github.com/apple/foundationdb/tree/main/fdbserver/clustercontroller), [`fdbserver/coordinator/`](https://github.com/apple/foundationdb/tree/main/fdbserver/coordinator), [`fdbserver/core/LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp)
+**Size:** ~13K + coordination code  
+**Role:** Leader election, role recruitment, process registration, cluster-wide decision making.
+
+---
+
+## Overview
+
+One process is elected Cluster Controller (CC). It is the "brain" of the cluster: it recruits all server roles, monitors health, broadcasts configuration, and initiates recovery. Leadership is determined by a coordination quorum running on designated coordinator processes.
+
+---
+
+## Leader Election
+
+### Coordinators
+
+Typically 3 or 5 processes (addresses hardcoded in cluster file) that run a `leaderRegister` actor. They don't store user data -- only coordination metadata. Coordinators are not involved in committing transactions; they hold ~1KB of state that changes only during recovery.
+
+### Generation Register (Paxos Acceptor) -- [`Coordination.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/Coordination.cpp)`:121-173`
+
+The fundamental primitive: a register with generation numbers that prevents stale writes. Each coordinator runs a `localGenerationReg()` actor that implements a single Paxos acceptor. The generation register protocol is isomorphic to single-decree Paxos:
+
+| Paxos concept | FDB equivalent |
+|---|---|
+| Ballot number | `UniqueGeneration` (uint64 generation + UID tiebreaker) |
+| Prepare(n) | `GenerationRegReadRequest(key, gen)` |
+| Promise(n, v) | `GenerationRegReadReply(value, writeGen, readGen)` |
+| Accept(n, v) | `GenerationRegWriteRequest(kv, gen)` |
+| Quorum of acceptors | Majority of coordinators (n/2 + 1) |
+
+```
+struct GenerationRegVal {
+    UniqueGeneration readGen;   // highest generation promised (Paxos: highest prepare seen)
+    UniqueGeneration writeGen;  // highest generation accepted (Paxos: highest accept seen)
+    Optional<Value> val;        // the accepted value
+};
+```
+
+**Read protocol** (`GenerationRegReadRequest`) -- equivalent to Paxos **prepare**:
+- If client's generation > stored readGen: update readGen (promise not to accept lower)
+- Persist to disk (`store->commit()`)
+- Return (value, writeGen, readGen)
+
+**Write protocol** (`GenerationRegWriteRequest`) -- equivalent to Paxos **accept**:
+- Check: `readGen <= requestGen AND writeGen < requestGen`
+- If valid: update writeGen and value, persist, return writeGen
+- If invalid: return `max(readGen, writeGen)` so client can retry with a higher generation
+
+The core consensus logic in `localGenerationReg()` is ~50 lines of code.
+
+### Election Algorithm -- [`LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp)
+
+**`tryBecomeLeaderInternal()`** (lines 107-300):
+
+1. **Nomination phase** (lines 136-227):
+   - Generate `changeID` and `LeaderInfo` (encodes process class fitness in top bits)
+   - Submit `CandidacyRequest` to all coordinators
+   - Each coordinator nominates the candidate with best fitness
+   - Loop: check if quorum nominates us
+   - If quorum: become leader. If not: retry with updated info
+
+2. **Heartbeat phase** (lines 233-294):
+   - Send `LeaderHeartbeatRequest` to all coordinators at `HEARTBEAT_FREQUENCY`
+   - Expect quorum to respond `true`
+   - If quorum responds `false`: leadership lost, restart nomination
+   - Update `changeID` when priority info changes
+
+**LeaderInfo structure:**
+- `serializedInfo` -- serialized cluster interface
+- `changeID` -- unique ID for this leadership claim (top 7 bits = process class fitness)
+- `forward` flag -- signals connection string update
+
+### Coordinated State (Paxos Proposer) -- [`CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp)`:71-210`
+
+Higher-level abstraction that implements the Paxos proposer role over generation registers. This is where the quorum logic lives -- individual coordinators are acceptors that know nothing about quorums; `CoordinatedStateImpl` drives the two-phase protocol across the quorum.
+
+```
+struct CoordinatedStateImpl {
+    ServerCoordinators coordinators;
+    UniqueGeneration gen;
+    uint64_t conflictGen;
+    bool doomed;
+    bool initial;        // true if no prior value has been written
+};
+```
+
+**`read()`** -- Paxos prepare phase (two rounds):
+1. **Round 1**: Send `GenerationRegReadRequest(key, UniqueGeneration())` to all coordinators. Wait for majority. Learns the highest existing generation.
+2. **Compute ballot**: `conflictGen = max(all seen generations) + 1`. Create `gen = UniqueGeneration(conflictGen, randomUID)`.
+3. **Round 2**: Send `GenerationRegReadRequest(key, gen)` to all coordinators. Wait for majority. Each acceptor promises not to accept writes below `gen`. Returns the value with the highest `writeGen`.
+
+**`setExclusive()`** -- Paxos accept phase:
+- Send `GenerationRegWriteRequest(key, value, gen)` to all coordinators.
+- For initial state (no prior writes): wait for **all** replicas. Otherwise: wait for **majority** (n/2 + 1).
+- If any coordinator returns a generation > `gen`, throw `coordinated_state_conflict` (another proposer won).
+
+**Quorum sizes** (from `replicatedRead` / `replicatedWrite`):
+- Read quorum: `replicas.size() / 2 + 1` (majority)
+- Write quorum: `replicas.size() / 2 + 1` (majority), except initial writes require all replicas
+
+**`replicatedRead()`** has a subtlety: it races two sets of futures -- replies with a value vs. replies without. If a majority report empty (no value), it returns the empty reply with the highest `rgen`. Otherwise it returns the non-empty reply with the highest `writeGen`. This handles the bootstrap case where some coordinators have been written and others haven't.
+
+Used by recovery to store/update `DBCoreState` (log system configuration). This is the only mutable state managed by coordinators.
+
+### Coordinator Storage -- [`OnDemandStore`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/OnDemandStore.cpp), [`DiskQueue`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DiskQueue.cpp)
+
+Each coordinator's Paxos state (generation numbers and the coordinated value) is persisted via `KeyValueStoreMemory` -- an in-memory key-value store backed by a `DiskQueue` for durability.
+
+**On-disk format:**
+- Two files per coordinator: `coordination-0.fdq` and `coordination-1.fdq` in the process's data directory
+- `.fdq` = **F**oundation**D**B **Q**ueue -- an append-only circular log that alternates between the two files
+- Entries are checksummed (xxhash3 in DiskQueueVersion::V2)
+- `KeyValueStoreMemory` periodically snapshots the full in-memory state into the log, after which older entries are reclaimed
+
+**Access pattern:**
+- `OnDemandStore` is a lazy wrapper: opens the `KeyValueStoreMemory` only on first access (coordinator processes that are never contacted don't create files)
+- `localGenerationReg()` calls `store->readValue(key)` on each request and `store->set()` + `store->commit()` to persist generation updates
+- `store->commit()` flushes to the `.fdq` log, making Paxos promises and accepts crash-safe
+- The memory budget is 500MB (`keyValueStoreMemory(path, id, 500e6)`) -- vastly more than needed for the ~1KB of coordination state, but it's the standard `KeyValueStoreMemory` interface
+
+This is the same storage engine used by transaction logs for their mutation queues and by `txnStateStore` during recovery.
+
+---
+
+## ClusterControllerData -- `ClusterController.h:122-3412`
+
+The state maintained by the elected CC.
+
+### DBInfo (lines 124-215)
+
+```
+struct DBInfo {
+    Reference<AsyncVar<ClientDBInfo>> clientInfo;    // proxy list for clients
+    Reference<AsyncVar<ServerDBInfo>> serverInfo;    // full cluster config for servers
+    AsyncTrigger forceMasterFailure;
+    DatabaseConfiguration config, fullyRecoveredConfig;
+    int unfinishedRecoveries;
+    Database db;
+};
+```
+
+### Worker Pool
+
+```
+std::map<Optional<Standalone<StringRef>>, WorkerInfo> id_worker;  // processId → WorkerInfo
+std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class; // processId → configured class
+```
+
+### Recruitment State
+
+```
+std::vector<Reference<RecruitWorkersInfo>> outstandingRecruitmentRequests;
+std::vector<Reference<RecruitRemoteWorkersInfo>> outstandingRemoteRecruitmentRequests;
+std::vector<std::pair<RecruitStorageRequest, double>> outstandingStorageRequests;
+
+// Singleton role tracking
+AsyncVar<bool> recruitDistributor, recruitRatekeeper, recruitConsistencyScan;
+Optional<UID> recruitingDistributorID, recruitingRatekeeperID, recruitingConsistencyScanID;
+```
+
+### Health Monitoring
+
+```
+std::unordered_map<NetworkAddress, WorkerHealth> workerHealth;
+struct WorkerHealth {
+    std::unordered_map<NetworkAddress, DegradedTimes> degradedPeers;
+    std::unordered_map<NetworkAddress, DegradedTimes> disconnectedPeers;
+};
+```
+
+---
+
+## Worker Registration -- [`ClusterController.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterController.actor.cpp)`:1252-1434`
+
+### RegisterWorkerRequest
+
+```
+struct RegisterWorkerRequest {
+    WorkerInterface wi;
+    ProcessClass initialClass, processClass;
+    ClusterControllerPriorityInfo priorityInfo;
+    Generation generation;
+    Optional<DataDistributorInterface> distributorInterf;
+    Optional<RatekeeperInterface> ratekeeperInterf;
+    Optional<ConsistencyScanInterface> consistencyScanInterf;
+    bool degraded, recoveredDiskFiles;
+};
+```
+
+### Registration Flow
+
+1. **Cluster ID validation** -- reject workers from different clusters
+2. **Priority calculation** -- compute CC fitness and DC fitness for this worker
+3. **Process class resolution** -- use DB-configured class or initial class
+4. **Worker add/update** -- create `WorkerInfo`, start watcher actor, add to `id_worker`
+5. **Singleton handling** -- if worker reports DD/Ratekeeper/ConsistencyScan interface, update DBInfo
+6. **Trigger recruitment check** -- call `checkOutstandingRequests()` to re-evaluate pending recruitment
+
+---
+
+## Role Recruitment
+
+### Fitness Calculation -- `ClusterController.h:1443-1643`
+
+**RoleFitness structure:**
+```
+struct RoleFitness {
+    ProcessClass::Fitness bestFit, worstFit;
+    ProcessClass::ClusterRole role;
+    int count, worstUsed;
+    bool degraded;
+};
+```
+
+**Comparison priority** (for `operator<`):
+1. Worst fitness (lower is better)
+2. Worst used count (fewer concurrent roles is better)
+3. Total count (more is better)
+4. Prefer non-degraded
+5. For non-TLog: compare best fitness
+
+**Worker selection methods:**
+- `getWorkerForRoleInDatacenter()` -- single best worker for role in DC
+- `getWorkersForRoleInDatacenter()` -- multiple workers, prioritized by fitness
+- `getWorkersForTlogsSimple()` -- TLog recruitment respecting zone distribution
+- `getWorkersForTlogsComplex()` -- TLog recruitment for cross-zone policies
+
+### Recruitment Functions -- `ClusterRecovery.cpp`
+
+| Function | Role Recruited |
+|----------|---------------|
+| `recruitNewMaster()` | Master/Sequencer (must be same DC as CC) |
+| `newCommitProxies()` | Commit proxies (via `InitializeCommitProxyRequest`) |
+| `newGrvProxies()` | GRV proxies (via `InitializeGrvProxyRequest`) |
+| `newResolvers()` | Resolvers (via `InitializeResolverRequest`) |
+| `newTLogServers()` | TLogs, including remote region and log routers |
+| `newSeedServers()` | Initial storage servers (version 0 only) |
+
+---
+
+## ServerDBInfo Broadcasting -- `ServerDBInfo.h:34-88`
+
+### ServerDBInfo Contents
+
+```
+struct ServerDBInfo {
+    UID id;
+    ClusterControllerFullInterface clusterInterface;
+    ClientDBInfo client;                          // proxy lists
+    MasterInterface master;
+    Optional<DataDistributorInterface> distributor;
+    Optional<RatekeeperInterface> ratekeeper;
+    std::vector<ResolverInterface> resolvers;
+    DBRecoveryCount recoveryCount;
+    RecoveryState recoveryState;
+    LifetimeToken masterLifetime;
+    LogSystemConfig logSystemConfig;
+    int64_t infoGeneration;                       // increment on each change
+};
+```
+
+### Broadcasting
+
+`dbInfoUpdater()` actor:
+1. Waits for `updateDBInfo` trigger or `serverInfo` change
+2. Batches changes with `DBINFO_BATCH_DELAY`
+3. Serializes and sends `UpdateServerDBInfoRequest` to all workers
+4. Workers store in their local `AsyncVar<ServerDBInfo>` and react to changes
+
+---
+
+## ClusterController Main Loop -- `clusterControllerCore()`
+
+**Core actors spawned:**
+- `clusterWatchDatabase()` -- monitors master, triggers recovery
+- `statusServer()` -- handles status requests from fdbcli
+- `timeKeeper()` -- periodic version tracking
+- `monitorProcessClasses()` -- watches DB for class changes
+- `monitorDataDistributor()` / `monitorRatekeeper()` / `monitorConsistencyScan()` -- singleton lifecycles
+- `dbInfoUpdater()` -- broadcasts ServerDBInfo
+- `workerHealthMonitor()` -- optional health tracking
+
+**Event loop handles:**
+- `RegisterWorkerRequest` -- worker self-registration
+- `RecruitStorageRequest` -- on-demand storage server recruitment
+- `OpenDatabaseRequest` -- client database access
+- `GetWorkersRequest` / `GetClientWorkersRequest` -- worker queries
+- Coordination pings at `WORKER_COORDINATION_PING_DELAY` intervals
+- Leader failure detection
+
+---
+
+## WorkerInterface -- [`WorkerInterface.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h)`:45-126`
+
+RPCs exposed by every worker process:
+
+**Initialization:**
+`tLog`, `master`, `commitProxy`, `grvProxy`, `dataDistributor`, `ratekeeper`, `consistencyScan`, `resolver`, `storage`, `logRouter`, `backup`
+
+**Monitoring:**
+`debugPing`, `coordinationPing`, `waitFailure`, `setMetricsRate`, `eventLogRequest`, `updateServerDBInfo`
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/clustercontroller/ClusterController.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterController.actor.cpp) | CC main loop, worker registration, event handling |
+| `fdbserver/clustercontroller/ClusterController.h` | ClusterControllerData, fitness calculation, recruitment |
+| [`fdbserver/coordinator/Coordination.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/Coordination.cpp) | leaderRegister, generation register, coordination |
+| [`fdbserver/core/LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp) | tryBecomeLeaderInternal, candidacy, heartbeat |
+| [`fdbserver/core/CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp) | Replicated read/write over generation registers |
+| [`fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h) | WorkerInterface, ClusterControllerFullInterface |
+| `fdbserver/core/include/fdbserver/core/ServerDBInfo.h` | ServerDBInfo structure and broadcasting |
diff --git a/design/AI-generated/subsystem_05_commit_pipeline.md b/design/AI-generated/subsystem_05_commit_pipeline.md
new file mode 100644
index 00000000000..93d62016b7c
--- /dev/null
+++ b/design/AI-generated/subsystem_05_commit_pipeline.md
@@ -0,0 +1,305 @@
+# Subsystem 5: Transaction Commit Pipeline
+
+**[Diagrams](diagram_05_commit_pipeline.md)**
+
+**Location:** [`fdbserver/commitproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/commitproxy), [`fdbserver/grvproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/grvproxy), [`fdbserver/resolver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/resolver), [`fdbserver/sequencer/`](https://github.com/apple/foundationdb/tree/main/fdbserver/sequencer)
+**Size:** ~12K  
+**Role:** Version assignment, conflict detection, commit batching -- the write-path orchestration.
+
+---
+
+## Overview
+
+Four server roles collaborate in a strict pipeline to commit transactions:
+
+```
+Client ──CommitTransactionRequest──▶ CommitProxy
+                                        │
+Phase 1: GET VERSION                    ▼
+                                    Master/Sequencer  (monotonic version)
+Phase 2: RESOLVE                        ▼
+                                    Resolver(s)  (conflict detection)
+Phase 3: LOG                            ▼
+                                    TLog replicas  (durable write)
+Phase 4: REPLY                          ▼
+                                    Client (committed!)
+```
+
+---
+
+## Commit Proxy -- [`CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp)
+
+The workhorse of the commit pipeline. Batches client commits and drives them through all phases.
+
+### ProxyCommitData -- `ProxyCommitData.h:183`
+
+```
+struct ProxyCommitData {
+    MasterInterface master;                           // for version assignment
+    std::vector<ResolverInterface> resolvers;         // for conflict detection
+    Reference<LogSystem> logSystem;                  // for TLog push
+    IKeyValueStore* txnStateStore;                    // persistent metadata
+    
+    NotifiedVersion committedVersion;                 // largest durable version
+    NotifiedVersion version;                          // current proxy version
+    
+    KeyRangeMap<ServerCacheInfo> keyInfo;             // key range → storage servers + tags
+    
+    const std::vector<Tag>& tagsForKey(StringRef key); // tag lookup for mutations
+};
+```
+
+### commitBatch() -- 5-Phase Pipeline
+
+**CommitBatchContext** (`CommitProxyServer.actor.cpp:491`):
+```
+struct CommitBatchContext {
+    std::vector<CommitTransactionRequest> trs;        // batch of transactions
+    LogPushData toCommit;                             // serialized mutations for TLogs
+    Version commitVersion, prevVersion;
+    std::vector<ResolveTransactionBatchReply> resolution;  // conflict results
+    std::set<Tag> writtenTags;
+};
+```
+
+**Phase 1: Pre-Resolution** (`preresolutionProcessing`):
+- Validate transaction sizes
+- Compute read/write conflict ranges
+- Determine storage server tags via `tagsForKey()` for each mutation
+- Check hot shard throttling
+
+**Phase 2: Resolution** (`getResolution`):
+- Send `ResolveTransactionBatchRequest` to all resolvers in parallel
+- Each resolver handles a partition of the key space
+- Wait for all responses
+
+**Phase 3: Post-Resolution** (`postResolution`):
+- Process conflict results: mark conflicting transactions
+- Apply metadata mutations (`applyMetadataToCommittedTransactions()`)
+- Update storage server mappings
+- Process idempotency keys
+
+**Phase 4: Transaction Logging** (`transactionLogging`):
+- Call `logSystem->push()` with tagged mutations
+- Block until mutations are durable (TLog quorum ack)
+- Update `queueCommittedVersion`
+
+**Phase 5: Reply** (`reply`):
+- Send `CommitTransactionReply` to each client
+- Conflicting transactions get `not_committed` error
+- Successful transactions get their commit version
+
+---
+
+## GRV Proxy -- [`GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp)
+
+Assigns read versions to client transactions. The GRV proxy is the primary enforcement point for cluster-wide flow control: it is where ratekeeper's back-pressure decisions become concrete delays and rejections applied to client requests.
+
+### GrvProxyData (line 202)
+
+```
+struct GrvProxyData {
+    MasterInterface master;                    // version source
+    VersionVector ssVersionVectorCache;        // storage server version tracking
+    Version version;
+};
+```
+
+### End-to-End Flow Control: Ratekeeper → GRV Proxy → Client
+
+Flow control is the mechanism that prevents the cluster from being overwhelmed when storage servers or TLogs fall behind. It works as a feedback loop:
+
+**1. Ratekeeper computes global TPS limits** (`Ratekeeper::updateRate()`, `Ratekeeper.cpp:642`):
+- Runs periodically (every `METRIC_UPDATE_RATE` seconds, default 0.1s).
+- Polls every storage server and TLog for queue depths, durable bytes rates, and free disk space.
+- Computes a `tpsLimit` for both `normalLimits` (DEFAULT + SYSTEM priority) and `batchLimits` (BATCH priority) independently.
+- The core formula scales `actualTps` (smoothed observed transaction rate) by the ratio of durable-bytes throughput to input-bytes rate, further modulated by a `targetRateRatio` derived from how full the storage/TLog queue is relative to target and spring thresholds. The result is: if queues are within target, `tpsLimit ≈ infinity`; as queues grow past the target, the limit ramps down toward zero.
+- Batch limits use more aggressive thresholds (larger `storageTargetBytes`, `logTargetBytes`) so batch transactions are throttled first.
+
+**2. Ratekeeper sends per-proxy rate limits** (`handleGetRateInfoReqs()`, `Ratekeeper.cpp:347`):
+- Each GRV proxy periodically sends a `GetRateInfoRequest` to ratekeeper (every `leaseDuration/2` seconds, jittered).
+- Ratekeeper replies with `transactionRate = normalLimits.tpsLimit / numProxies` and `batchTransactionRate = batchLimits.tpsLimit / numProxies` — the global limit divided evenly across all GRV proxies.
+- The reply also includes a `leaseDuration` (default `METRIC_UPDATE_RATE` = 0.1s). If the proxy doesn't hear back within the lease, it disables rate limiting (sets rate to 0 via `GrvTransactionRateInfo::disable()`), which smoothly ramps the allowed rate to zero and stops releasing transactions.
+
+**3. GRV proxy enforces the rate via a token bucket** (`GrvTransactionRateInfo`, `GrvTransactionRateInfo.h`):
+- Each proxy maintains two `GrvTransactionRateInfo` objects: `normalRateInfo` (for SYSTEM + DEFAULT) and `batchRateInfo` (for BATCH).
+- Each object is a smoothed token bucket: `setRate()` feeds the ratekeeper-provided rate through a `Smoother`, and `startReleaseWindow()` computes a `limit` from the smoothed difference between the allowed rate and the actual release rate, times the rate window.
+- `canStart(numAlreadyStarted, count)` checks if `numAlreadyStarted + count <= limit + budget`. The `budget` accumulates unused capacity across release windows, bounded by `maxEmptyQueueBudget` when queues are empty (to avoid stale budget causing bursts).
+- The smoothing prevents sudden rate changes from causing oscillation.
+
+**4. The `transactionStarter` loop releases batches** (`transactionStarter()`, line 877):
+- On each GRVTimer tick, it drains the three priority queues in strict order: SYSTEM first, then DEFAULT, then BATCH.
+- SYSTEM transactions bypass rate limiting entirely (`canStart` is never checked for them).
+- DEFAULT transactions are gated by `normalRateInfo.canStart()`.
+- BATCH transactions are gated by `batchRateInfo.canStart()`, which uses the more aggressive batch rate limit.
+- Transactions that pass rate-limiting are batched into a single `getLiveCommittedVersion()` call to the master, and replies are sent to all clients in the batch simultaneously.
+
+### Dynamic Batching Based on Reply Latency
+
+The GRV batch interval is adaptively tuned based on observed round-trip latency (`queueGetReadVersionRequests`, line 518):
+
+- After each batch of non-risky-read requests completes, `timeReply()` measures the round-trip time and feeds it into a `normalGRVLatency` stream.
+- The `queueGetReadVersionRequests` actor receives this latency and computes:
+  ```
+  target_latency = reply_latency * START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION
+  GRVBatchTime = α * target_latency + (1 - α) * GRVBatchTime
+  ```
+  clamped to `[START_TRANSACTION_BATCH_INTERVAL_MIN, START_TRANSACTION_BATCH_INTERVAL_MAX]`.
+- When the first request arrives into empty queues, a GRVTimer is scheduled with delay `max(0, GRVBatchTime - timeSinceLastGRV)`. This means: when the system is fast, batching intervals shrink (lower latency for clients); when the system is slow, batches grow larger (better amortization of the master round-trip).
+
+### Lower Priorities Dropped Under Pressure
+
+When the total GRV queue depth exceeds `START_TRANSACTION_MAX_QUEUE_SIZE`, the proxy drops lower-priority requests to make room (`queueGetReadVersionRequests`, line 541):
+
+- A new **BATCH** request is rejected immediately with `grv_proxy_memory_limit_exceeded`.
+- A new **DEFAULT** request evicts one BATCH request from the front of its queue (if any); if none exist, the DEFAULT request itself is rejected.
+- A new **SYSTEM** request evicts one BATCH request, or failing that one DEFAULT request; only if both queues are empty is the SYSTEM request itself rejected.
+- Additionally, when the batch rate from ratekeeper drops to effectively zero (`batchRateInfo.getRate() <= 1/numProxies`), BATCH requests are immediately rejected with `batch_transaction_throttled` before even being queued (line 605).
+
+### GetReadVersion Flow
+
+1. **Queuing** (`queueGetReadVersionRequests`, line 518):
+   - Three priority queues: SYSTEM, DEFAULT, BATCH
+   - Dynamic batching as described above
+
+2. **Version fetch** (`getLiveCommittedVersion`, line 670):
+   - Request to master: `master.getLiveCommittedVersion.getReply()`
+   - Causal read check (unless `CAUSAL_READ_RISKY`): `updateLastCommit()` confirms epoch is still live by calling `logSystem->confirmEpochLive()`. This ensures the version returned actually represents committed state.
+   - Version vector integration if enabled: applies delta from master to local SS version cache
+
+3. **Reply** (`sendGrvReplies`, line 747):
+   - Sends the version to all requests in the batch
+   - Attaches per-tag throttle information so clients can self-throttle
+   - Sets `rkBatchThrottled` / `rkDefaultThrottled` flags if sustained throttling detected (sustained for `GRV_SUSTAINED_THROTTLING_THRESHOLD` seconds), signaling clients to back off
+
+---
+
+## Master/Sequencer -- [`masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp)
+
+Assigns monotonically increasing commit versions.
+
+### MasterData -- `MasterData.h:50`
+
+```
+struct MasterData {
+    Version lastEpochEnd;                      // last version from prior epoch
+    Version recoveryTransactionVersion;        // first version in this epoch
+    NotifiedVersionValue liveCommittedVersion;  // largest live-committed version
+    Version version;                           // last assigned version
+    double lastVersionTime;                    // timestamp of last version
+    Optional<Version> referenceVersion;        // for wall-clock alignment
+    
+    std::map<UID, CommitProxyVersionReplies> lastCommitProxyVersionReplies;
+    VersionVector ssVersionVector;             // per-SS commit versions
+    ResolutionBalancer resolutionBalancer;
+};
+```
+
+### Version Assignment -- `getVersion()` (lines 74-178)
+
+1. **Proxy validation**: Check proxy is known
+2. **Request ordering**: Wait for prior request (`latestRequestNum.whenAtLeast(requestNum - 1)`)
+3. **Deduplication**: Return cached reply if already processed
+4. **Version calculation**:
+   ```
+   toAdd = max(1, min(MAX_READ_TRANSACTION_LIFE_VERSIONS,
+                      VERSIONS_PER_SECOND * (now - lastVersionTime)))
+   ```
+   If `referenceVersion` set, align to wall clock via `figureVersion()`
+5. **Reply**: `GetCommitVersionReply` with `prevVersion` and `version`
+
+### figureVersion() -- Wall-Clock Alignment (lines 43-72)
+
+```
+expectedVersion = now * VERSIONS_PER_SECOND - referenceVersion
+version = clamp(version + toAdd, expectedVersion ± scaled_bounds)
+```
+
+Keeps versions roughly aligned with wall-clock microseconds while maintaining monotonicity.
+
+### Live Committed Version Tracking
+
+- `serveLiveCommittedVersion()`: Responds to GRV proxy and proxy reports
+- `updateLiveCommittedVersion()`: Updates when proxy reports new committed version
+- If Version Vectors enabled: tracks per-tag commit info
+
+---
+
+## Resolver -- [`Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp), [`ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp)
+
+Pure conflict detection. Maintains a sliding window of committed write ranges.
+
+### Resolver Structure (line 128)
+
+```
+struct Resolver {
+    int commitProxyCount, resolverCount;
+    NotifiedVersion version;
+    ConflictSet* conflictSet;                         // SkipList-based conflict tracking
+    IKeyValueStore* txnStateStore;                    // metadata KVS
+    std::map<NetworkAddress, ProxyRequestsInfo> proxyInfoMap;
+    KeyRangeMap<ServerCacheInfo> keyInfo;
+};
+```
+
+### ConflictSet
+
+```
+struct ConflictSet {
+    SkipList versionHistory;  // version → conflict data
+    Version oldestVersion;
+};
+```
+
+### resolveBatch() (line 261)
+
+1. **Memory check**: Block if `totalStateBytes > RESOLVER_STATE_MEMORY_LIMIT`
+2. **Version ordering** (`versionReady()`): Wait for resolver to reach `prevVersion`
+3. **Conflict detection**:
+   ```cpp
+   ConflictBatch conflictBatch(self->conflictSet, &reply.conflictingKeyRangeMap);
+   for (auto& txn : req.transactions)
+       conflictBatch.addTransaction(txn, newOldestVersion);
+   conflictBatch.detectConflicts(req.version, newOldestVersion, commitList, &tooOldList);
+   ```
+   - For each transaction: check if read ranges overlap with committed writes at newer versions
+   - Populates `reply.committed[]` with per-transaction conflict status
+4. **State transaction processing**: Apply metadata mutations if enabled
+5. **Version cleanup**: Erase old state transactions to bound memory
+
+### ConflictBatch::detectConflicts()
+
+```
+void detectConflicts(Version now, Version newOldestVersion,
+                    std::vector<int>& nonConflicting,
+                    std::vector<int>* tooOldTransactions)
+```
+
+Algorithm: For each transaction's read ranges, check overlap with committed write ranges in the SkipList at versions newer than the transaction's read version.
+
+---
+
+## Tag Assignment
+
+Tags connect mutations to storage servers:
+
+1. **During pre-resolution**: For each mutation, `tagsForKey(mutation.key)` returns tags
+2. **Tag source**: `KeyRangeMap<ServerCacheInfo> keyInfo` maps key ranges → storage servers → tags
+3. **Tags added to LogPushData**: `toCommit.addTags(tags)` for each mutation
+4. **In TLog**: Messages routed to `TagData[tag.locality][tag.id]` queues
+5. **Storage servers**: Peek their assigned tag to pull relevant mutations
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/commitproxy/CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp) | 5-phase commit batch pipeline |
+| `fdbserver/commitproxy/ProxyCommitData.h` | ProxyCommitData, tag lookup |
+| [`fdbserver/grvproxy/GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp) | Read version assignment, rate limiting enforcement |
+| `fdbserver/grvproxy/GrvTransactionRateInfo.h` | Token-bucket rate limiter driven by ratekeeper |
+| [`fdbserver/sequencer/masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp) | Version assignment, wall-clock alignment |
+| `fdbserver/sequencer/MasterData.h` | MasterData, version tracking |
+| [`fdbserver/resolver/Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp) | Conflict detection, batch resolution |
+| [`fdbserver/resolver/ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp) | SkipList-based conflict range tracking |
diff --git a/design/AI-generated/subsystem_06_tlog_logsystem.md b/design/AI-generated/subsystem_06_tlog_logsystem.md
new file mode 100644
index 00000000000..429c613b510
--- /dev/null
+++ b/design/AI-generated/subsystem_06_tlog_logsystem.md
@@ -0,0 +1,243 @@
+# Subsystem 6: Transaction Log (TLog) & Log System
+
+**[Diagrams](diagram_06_tlog_logsystem.md)**
+
+**Location:** [`fdbserver/tlog/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tlog), [`fdbserver/logsystem/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logsystem), [`fdbserver/logrouter/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logrouter)
+**Size:** ~17K  
+**Role:** Durable mutation logging, tag-partitioned replication, peek cursors for storage servers.
+
+---
+
+## Overview
+
+The TLog subsystem is FDB's durability guarantee. All committed mutations flow through TLogs before being applied to storage servers. A mutation is committed when a quorum of TLog replicas acknowledge. Storage servers then pull (peek) mutations asynchronously.
+
+---
+
+## TLog Server -- [`TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp)
+
+### TLogData (lines 295-420)
+
+Top-level container for all TLog data in a process:
+
+```
+struct TLogData {
+    Deque<UID> popOrder;                     // pop ordering across generations
+    Deque<UID> spillOrder;                   // spill ordering
+    std::map<UID, Reference<LogData>> id_data;  // all log generations by UID
+    
+    IKeyValueStore* persistentData;          // spilled data on disk (KVStore)
+    IDiskQueue* rawPersistentQueue;          // physical disk queue
+    TLogQueue* persistentQueue;              // logical queue wrapper
+    
+    int64_t diskQueueCommitBytes;
+    int64_t bytesInput, bytesDurable;
+    int64_t targetVolatileBytes;             // spill threshold
+    
+    NotifiedVersion queueCommitEnd;          // durable commit location
+    Version queueCommitBegin;
+};
+```
+
+### LogData (lines 422-810)
+
+Per-epoch/generation log container:
+
+```
+struct LogData {
+    struct TagData : ReferenceCounted<TagData> {
+        std::deque<std::pair<Version, LengthPrefixedStringRef>> versionMessages;
+        bool nothingPersistent;
+        Version popped;                      // consumed up to this version
+        Version persistentPopped;            // durable pop point
+        Tag tag;
+    };
+    
+    Map<Version, std::pair<IDiskQueue::location, IDiskQueue::location>> versionLocation;
+    Deque<std::pair<Version, Standalone<VectorRef<uint8_t>>>> messageBlocks;
+    std::vector<std::vector<Reference<TagData>>> tag_data;  // [locality][id]
+    
+    VersionMetricHandle persistentDataVersion;
+    VersionMetricHandle persistentDataDurableVersion;
+    NotifiedVersion version;                 // current log version
+    NotifiedVersion queueCommittedVersion;   // disk queue durability point
+    Version knownCommittedVersion;
+};
+```
+
+### Push Flow (`tLogPush`)
+
+1. Receive mutation batch from commit proxy
+2. Deserialize `TLogQueueEntry` containing tagged mutations
+3. Route each message by tag to appropriate `TagData` queue
+4. Append to `versionMessages` deque (in-memory)
+5. Write to `persistentQueue` (DiskQueue) for durability
+6. Update `versionLocation` map
+7. Reply when disk write confirmed
+
+### Peek Flow (`tLogPeekStream`)
+
+1. Storage server requests mutations for tag starting at version
+2. Iterate `TagData::versionMessages` from requested version
+3. For versions not in memory: read from `persistentData` (KVStore)
+4. Send results in `TLogPeekStreamReply` chunks
+5. Respect `limitBytes` and `peekMemoryLimiter` for backpressure
+6. Track popped version and cursor position
+
+### DiskQueue -- `TLogQueue` (lines 112-285)
+
+Append-only durable queue:
+- `push(entry)` -- add entry with tag metadata
+- `pop(location)` -- remove committed entries up to location
+- Entries prefixed with length + protocol version
+- Recovery-safe: CRC checks for partial writes
+- Linked to version boundaries for efficient cleanup
+
+### Spilling
+
+When in-memory data exceeds `targetVolatileBytes`:
+- Oldest data spilled from memory to `persistentData` (KVStore on disk)
+- `spillOrder` determines which generation spills first
+- Peek reads transparently merge in-memory and spilled data
+
+---
+
+## Log System -- [`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp)
+
+Abstraction over the set of TLog replicas.
+
+### LogSystem Interface -- [`LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h)`:260-601`
+
+**Core methods:**
+
+| Method | Purpose |
+|--------|---------|
+| `push(versionSet, data, spanContext)` | Send mutations to TLog replicas, wait for quorum |
+| `peek(dbgid, begin, end, tag)` | Create cursor for reading mutations by tag |
+| `peekSingle(dbgid, begin, tag, history)` | Peek from preferred TLog |
+| `peekLogRouter(dbgid, begin, tag)` | Peek from same-generation logs only |
+| `peekTxs(dbgid, begin, tag, locality, isLong)` | Peek transaction state mutations |
+| `pop(upTo, tag, knownCommittedVersion)` | Allow TLog to discard messages before version |
+| `onCoreStateChanged()` | Future that fires when log set changes |
+| `newEpoch(recr, fRemoteWorkers, config, ...)` | Create new log epoch during recovery |
+
+### IPeekCursor Interface (lines 60-141)
+
+Lazy iterator over log messages:
+
+```
+struct IPeekCursor {
+    bool hasMessage() const;               // current message exists?
+    VectorRef<Tag> getTags() const;        // tags for current message
+    Arena& arena();                        // arena for message data
+    ArenaReader* reader();                 // deserialize current message
+    void nextMessage();                    // advance to next
+    Future<Void> getMore(TaskPriority);    // wait for more messages
+    LogMessageVersion version();           // current message version
+    Version popped();                      // earliest available version
+    bool isActive() / isExhausted();       // cursor status
+};
+```
+
+### Push Implementation
+
+1. **Tag distribution**: Partition mutations by tag (from commit proxy's `tagsForKey()`)
+2. **TLog selection**: `getPushLocations()` determines which TLogs handle each tag
+3. **Message formatting**: Build `LogPushData` with serialized mutations
+4. **Parallel push**: Send to all relevant TLog replicas
+5. **Quorum wait**: Block until `f+1` of `2f+1` TLogs acknowledge
+6. **Return**: Promise resolves to next pushable version
+
+### Epoch Management
+
+Each recovery creates a new "epoch" of TLogs:
+- `newEpoch()` recruits new TLogs and creates new log generation
+- Old epochs kept until all storage servers have consumed their data
+- `purgeOldRecoveredGenerations()` cleans up fully-consumed epochs
+- Peek cursors can span multiple epochs during recovery
+
+---
+
+## Log Router -- [`LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp)
+
+Routes mutations across regions in multi-datacenter deployments:
+
+- Reads mutations from primary TLogs via peek cursor
+- Forwards to remote region's TLogs
+- Reduces cross-region traffic by batching
+- Uses `tagLocalityLogRouter` tags
+- Bridges tag-partitioned log system across datacenters
+
+---
+
+## Tag System
+
+### Tag Structure
+
+```
+struct Tag {
+    int8_t locality;   // identifies the shard location
+    uint16_t id;       // identifies the specific storage server
+};
+```
+
+### Special Tag Localities
+
+| Locality | Purpose |
+|----------|---------|
+| `tagLocalitySpecial` | Special/system tags |
+| `tagLocalityLogRouter` | Log router tags for cross-region |
+| `tagLocalityRemoteLog` | Remote region log tags |
+| `tagLocalityUpgraded` | Tags from older versions |
+| `tagLocalitySS` | Standard storage server tags |
+
+### Tag Assignment Flow
+
+1. **Cluster controller** assigns tags to storage servers during recruitment
+2. **Tags stored** in `serverTagKeys` system key range
+3. **Commit proxy** maintains `keyInfo` map: key range → storage servers → tags
+4. **During commit**: `tagsForKey(key)` looks up tags for each mutation's key
+5. **TLog routing**: Tags attached to mutations; TLog stores per-tag queues
+6. **Storage server**: Peeks its assigned tag to pull only relevant mutations
+
+### Version Vector
+
+Optional per-storage-server version tracking:
+- Master maintains `VersionVector ssVersionVector`
+- GRV proxy caches and sends delta in `GetReadVersionReply`
+- Enables per-shard causal consistency without global ordering
+
+---
+
+## Data Flow: Mutation Lifecycle
+
+```
+1. Client commits transaction with mutations
+2. CommitProxy assigns tags: mutation.key → tagsForKey() → [Tag1, Tag2, ...]
+3. CommitProxy calls logSystem->push(mutations_with_tags)
+4. LogSystem sends to all relevant TLog replicas
+5. Each TLog:
+   a. Routes mutation to TagData[tag.locality][tag.id].versionMessages
+   b. Writes to DiskQueue for durability
+   c. Acks to LogSystem
+6. LogSystem waits for quorum → commit confirmed
+7. Storage server (background):
+   a. logSystem->peek(myTag, myVersion) → IPeekCursor
+   b. cursor.getMore() → new mutations
+   c. Apply mutations to local KVStore
+   d. Advance version
+   e. logSystem->pop(myVersion, myTag) → allow TLog cleanup
+```
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/tlog/TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp) | TLog server: push, peek, spill, DiskQueue |
+| [`fdbserver/logsystem/LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) | LogSystem: push/peek/pop across TLog replicas |
+| [`fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h) | LogSystem, IPeekCursor interfaces |
+| [`fdbserver/core/include/fdbserver/core/LogSystemConfig.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/LogSystemConfig.h) | Log system configuration and epoch tracking |
+| [`fdbserver/core/include/fdbserver/core/IDiskQueue.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/IDiskQueue.h) | IDiskQueue interface for TLog persistence |
+| [`fdbserver/logrouter/LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp) | Cross-region mutation routing |
diff --git a/design/AI-generated/subsystem_07_storage_server.md b/design/AI-generated/subsystem_07_storage_server.md
new file mode 100644
index 00000000000..8df3bc998ff
--- /dev/null
+++ b/design/AI-generated/subsystem_07_storage_server.md
@@ -0,0 +1,766 @@
+# Subsystem 7: Storage Server & Engines
+
+**[Diagrams](diagram_07_storage_server.md)**
+
+**Location:** [`fdbserver/storageserver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/storageserver), [`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore)
+**Size:** ~63K  
+**Role:** Serves reads, applies mutations from log, pluggable storage backends.
+
+---
+
+## Overview
+
+Storage servers are the read path and the materialized state of the database. Each SS owns a set of key ranges (shards), continuously pulls committed mutations from the TLog via tagged peek cursors, applies them to a local key-value store, and serves client reads at specific versions.
+
+---
+
+## StorageServer Structure -- [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)`:850-1528`
+
+```
+struct StorageServer : IStorageMetricsService {
+    // MVCC data
+    VersionedData versionedData;                         // versioned sets/clears
+    std::map<Version, Standalone<VerUpdateRef>> mutationLog;  // (durableVersion, version]
+    
+    // Shard ownership
+    KeyRangeMap<Reference<ShardInfo>> shards;            // key range → shard state
+    KeyRangeMap<Version> newestAvailableVersion;         // key availability per version
+    
+    // Version tracking
+    NotifiedVersion version;                             // current SS version
+    NotifiedVersion desiredOldestVersion;                // target for advancing durable
+    NotifiedVersion oldestVersion;                       // minimum readable version
+    NotifiedVersion durableVersion;                      // committed to disk
+    
+    // Log system connection
+    Reference<LogSystem> logSystem;
+    Reference<IPeekCursor> logCursor;
+    
+    // Storage engine
+    StorageServerDisk storage;                           // wraps IKeyValueStore
+    
+    // Move-in tracking
+    std::unordered_map<UID, std::shared_ptr<MoveInShard>> moveInShards;
+};
+```
+
+---
+
+## StorageServerDisk Wrapper -- [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)`:602-719`
+
+`StorageServerDisk` is the thin layer between the storage server logic and the `IKeyValueStore` interface. It delegates all I/O to the underlying engine and adds metrics counters (`kvGets`, `kvScans`, `kvCommits`, `kvCommitLogicalBytes`, `kvClearRanges`, `kvClearSingleKey`).
+
+Key methods:
+- **`readValue(key)`** / **`readValuePrefix(key, maxLength)`** -- point reads delegated to `storage->readValue()`, incrementing `kvGets`
+- **`readRange(keys, rowLimit, byteLimit)`** -- range reads delegated to `storage->readRange()`, incrementing `kvScans`
+- **`readNextKeyInclusive(key)`** -- helper that does a `readRange(key, allKeys.end, rowLimit=1)` and returns the first key found (or `allKeys.end`)
+- **`makeVersionMutationsDurable(prevVersion, newVersion, bytesLeft, ...)`** -- iterates the in-memory `mutationLog` and writes `set()`/`clear()` calls to the storage engine for each mutation
+- **`commit()`** -- calls `storage->commit()` to make buffered writes durable
+- **Checkpoint operations** -- `checkpoint()`, `restore()`, `deleteCheckpoint()` -- forwarded for physical shard movement
+
+---
+
+## Update Loop -- [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)`:9562+`
+
+The `update()` function (a coroutine) pulls mutations from the log system and applies them locally.
+
+### Flow
+
+1. **Emergency brake** (lines 9576-9610):
+   - If `queueSize >= STORAGE_HARD_LIMIT_BYTES` and `durableVersion < desiredOldestVersion`: block
+   - Prevents OOM from unbounded mutation accumulation
+   - Tracks `lastBytesInputEBrake` and `lastDurableVersionEBrake` to allow progress between braking periods
+
+2. **Log cursor read** (lines 9636-9651):
+   - `cursor->getMore()` -- wait for new mutations from TLog
+   - Validate cursor hasn't been popped (would mean SS removed via `worker_removed()`)
+
+3. **Eager read collection** (lines 9674+):
+   - Clone cursor, extract mutations to identify keys needing eager reads
+   - `UpdateEagerReadInfo` collects keys that need pre-reading (for atomic ops like `AppendIfFits`, `ByteMin`/`ByteMax`, `CompareAndClear`, and `ClearRange` endpoint resolution)
+   - `doEagerReads()` prefetches values from the storage engine
+   - Handles shard changes during read, retries if needed
+
+4. **Mutation application** (version loop):
+   - Create `StorageUpdater` for version transitions
+   - For each mutation at each version:
+     - `applyMutation(data, mutation, version)` -- update MVCC data
+     - Advance `data->version` on new version boundary
+     - Yield if accumulated bytes exceed `DESIRED_UPDATE_BYTES`
+
+### Version Management
+
+| Version | Meaning |
+|---------|---------|
+| `version` | Latest version SS knows about (from TLog stream) |
+| `durableVersion` | Latest version committed to disk |
+| `oldestVersion` | Minimum version readable from storage |
+| `desiredOldestVersion` | Target set by data distributor for MVCC cleanup |
+
+**Gap `(durableVersion, version]`**: Held in MVCC `mutationLog` (memory).  
+**Gap `(oldestVersion, durableVersion]`**: On disk but not yet MVCC-freeable.
+
+---
+
+## Durability Loop -- `updateStorage()` (line 10274)
+
+The `updateStorage` actor runs continuously and flushes in-memory mutations to the storage engine.
+
+### Flow
+
+1. **Wait** until `desiredOldestVersion > storageVersion` (i.e., the data distributor has indicated it's safe to advance)
+2. **Check pending operations** -- inspect `pendingCheckpoints` and clamp `desiredVersion` accordingly
+3. **Write mutations** via `storage.makeVersionMutationsDurable()` which iterates the `mutationLog` from `storageVersion` to `desiredVersion`, calling `storage.set()` and `storage.clear()` for each mutation. This is bounded by `STORAGE_COMMIT_BYTES` (bytes limit) and for RocksDB also by `ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT`
+4. **Advance `oldestVersion`** and asynchronously free MVCC data for versions before `newOldestVersion`
+5. **Checkpoint handling** -- if a pending checkpoint version was reached, create the checkpoint via `storage.checkpoint()`
+6. **`storage.commit()`** -- atomically make all the above mutations durable
+7. **Advance `durableVersion`** and signal `durableInProgress` to unblock the update loop
+8. **Pop** old TLog data via `data->logSystem->pop()` now that mutations are durable
+
+### Persistent Version Tracking
+
+Each storage engine stores a special key `\xff\xffVersion` (`persistVersion`) that records the durable version. This is written by `makeVersionDurable()` on every commit and read during `restoreDurableState()` on startup. On recovery, the storage server reads `persistVersion` from the engine to learn how far it had committed, then resumes pulling mutations from the TLog starting from that version. If `persistVersion` is absent, the database was never initialized and the storage server self-destructs.
+
+### RocksDB-Specific Tuning
+
+For RocksDB backends, `updateStorage()` has additional controls:
+- **`ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT`** -- limits the number of `DeleteRange` operations per commit to avoid excessive compaction pressure
+- **`ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS`** -- enables pre-reading clear range endpoints for optimization
+- **`canCommit()` backpressure** -- before committing, the RocksDB engine checks pending compaction bytes and immutable memtable count; if overloaded, it delays the commit
+
+---
+
+## Read Serving
+
+### Point Read -- `getValueQ()` (line 2148)
+
+1. **Priority management**: Acquire `PriorityMultiLock` with request priority
+2. **Version wait**: `waitForVersion(data, commitVersion, req.version)` -- block until SS version >= requested
+3. **Shard check**: `data->shards[key]->isReadable()` -- throw `wrong_shard_server()` if not owned
+4. **Value read**:
+   - Try MVCC data at version: `data->data().at(version).lastLessOrEqual(key)`
+   - If not in MVCC: read from storage engine: `data->storage.readValue(key)`
+   - Validate version hasn't become too old, shard ownership hasn't changed
+5. **Response**: Return value with penalty metric
+
+### Range Read -- `getKeyValuesQ()` (line 3314)
+
+1. **Shard validation**: Ensure begin and end resolve within single shard
+2. **Key selector resolution**: Resolve begin/end `KeySelector` to concrete keys via `findKey()`
+3. **Range read**: `readRange(data, version, range, limit, &remainingBytes)`
+4. **Shard change detection**: Double-check ownership via change counters before returning
+
+---
+
+## Shard Management
+
+### ShardInfo (lines 448-600)
+
+Each shard is in one of four states:
+
+| State | Meaning |
+|-------|---------|
+| `readWrite` | Assigned, readable at storageVersion and later |
+| `adding` | Being fetched from another SS (fetchKeys in progress) |
+| `moveInShard` | Physical shard being moved via checkpoint |
+| `notAssigned` | SS has no data for this range |
+
+### AddingShard (lines 386-446)
+
+Active data transfer state:
+```
+struct AddingShard {
+    KeyRange keys;
+    Future<Void> fetchClient;                  // fetchKeys() actor
+    Promise<Void> fetchComplete, readWrite;
+    std::deque<Standalone<VerUpdateRef>> updates;  // mutations during fetch
+    enum Phase { WaitPrevious, Fetching, FetchingCF, Waiting };
+    Version transferredVersion, fetchVersion;
+};
+```
+
+### fetchKeys() Actor (line 6850)
+
+Transfers shard data from source to destination SS:
+
+1. **Setup**: Wait for `coreStarted`, version advancement, overlapping ranges durable
+2. **Fetch preparation**: Acquire `fetchKeysParallelismLock`, set phase to `Fetching`
+3. **Data fetch loop**:
+   - Create transaction at `fetchVersion`
+   - Read key-value blocks from source SS via `tryGetRange()`
+   - Apply deferred mutations from `updates` deque
+   - Write to storage: `data->storage.set()` for each KV pair
+4. **Completion**: Set phase to `Waiting`, fire `fetchComplete`, update `newestAvailableVersion`
+
+### changeServerKeys
+
+Modifies shard map to move ranges between states:
+- Updates `serverKeys` system namespace reflecting ownership
+- Contexts: `CSK_UPDATE` (normal), `CSK_RESTORE` (recovery), `CSK_ASSIGN_EMPTY` (new), `CSK_FALL_BACK` (fallback)
+
+---
+
+## Storage Engines -- [`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore)
+
+### IKeyValueStore Interface -- [`IKeyValueStore.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h)
+
+All storage engines implement the `IKeyValueStore` interface (which extends `IClosable`). This is the contract between the storage server and the underlying storage mechanism.
+
+```
+class IKeyValueStore : public IClosable {
+public:
+    // === Core mutation operations (buffered, not immediately durable) ===
+    virtual void set(KeyValueRef keyValue, const Arena* arena = nullptr) = 0;
+    virtual void clear(KeyRangeRef range, const Arena* arena = nullptr) = 0;
+    
+    // === Durability ===
+    virtual Future<Void> canCommit() { return Void(); }  // backpressure hook
+    virtual Future<Void> commit(bool sequential = false) = 0;  // atomically make prior set/clear durable
+    
+    // === Read operations (async) ===
+    virtual Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions>) = 0;
+    virtual Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions>) = 0;
+    virtual Future<RangeResult> readRange(KeyRangeRef keys, int rowLimit, int byteLimit, Optional<ReadOptions>) = 0;
+    
+    // === Metadata ===
+    virtual KeyValueStoreType getType() const = 0;
+    virtual StorageBytes getStorageBytes() const = 0;
+    
+    // === Checkpoints (physical shard movement) ===
+    virtual Future<CheckpointMetaData> checkpoint(const CheckpointRequest& request) { throw not_implemented(); }
+    virtual Future<Void> restore(const std::vector<CheckpointMetaData>& checkpoints) { throw not_implemented(); }
+    virtual Future<Void> deleteCheckpoint(const CheckpointMetaData& checkpoint) { throw not_implemented(); }
+    
+    // === Bulk operations ===
+    virtual Future<Void> compactRange(KeyRangeRef range) { throw not_implemented(); }
+    virtual Future<Void> replaceRange(KeyRange range, Standalone<VectorRef<KeyValueRef>> data);
+    virtual Future<Void> ingestSSTFiles(std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) { throw not_implemented(); }
+    
+    // === Lifecycle ===
+    virtual Future<Void> init() { return Void(); }  // MUST be idempotent (may be called again on rollback)
+};
+```
+
+#### Concurrency Contract
+
+From the header comment:
+
+> **Causal consistency**: A read which begins after a commit ends sees the effects of the commit. A read which ends before a commit begins does not see the effects of the commit.
+
+In practice: reads return data as of a version established by the most recent `commit()` that completed before the read ends, such that no subsequent commit also completed before the read began. This means reads and writes can overlap, but reads always see a consistent committed snapshot.
+
+#### ReadOptions
+
+```
+struct ReadOptions {
+    ReadType type = ReadType::NORMAL;  // NORMAL or FETCH
+    Optional<UID> debugID;
+};
+```
+
+`ReadType::FETCH` is used during shard data movement (fetchKeys). The engine may apply different throttling or priority to fetch reads vs. normal client reads.
+
+#### Factory -- [`IKeyValueStore.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/IKeyValueStore.cpp)
+
+```
+IKeyValueStore* openKVStore(KeyValueStoreType storeType, std::string filename, UID logID, ...) {
+    switch (storeType) {
+        case SSD_BTREE_V1:       return keyValueStoreSQLite(...);
+        case SSD_BTREE_V2:       return keyValueStoreSQLite(...);
+        case MEMORY:             return keyValueStoreMemory(...);
+        case SSD_REDWOOD_V1:     return keyValueStoreRedwoodV1(...);
+        case SSD_ROCKSDB_V1:     return keyValueStoreRocksDB(...);
+        case MEMORY_RADIXTREE:   return keyValueStoreMemory(..., "fdr", MEMORY_RADIXTREE);
+    }
+}
+```
+
+---
+
+### RocksDB Engine -- [`KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp) (~3K lines)
+
+The primary production storage engine. Wraps Facebook's RocksDB LSM-tree engine, adapting it to the `IKeyValueStore` interface via thread pools for non-blocking I/O.
+
+#### Architecture
+
+```
+RocksDBKeyValueStore (IKeyValueStore)
+├── Writer (IThreadPoolReceiver, single thread)
+│   ├── OpenAction        -- opens/creates the RocksDB database
+│   ├── CommitAction      -- writes accumulated WriteBatch to RocksDB
+│   ├── CloseAction       -- closes the database
+│   ├── CheckpointAction  -- creates a RocksDB checkpoint
+│   ├── RestoreAction     -- restores from checkpoint
+│   ├── CompactRangeAction
+│   └── IngestSSTFilesAction
+├── Reader (IThreadPoolReceiver, N parallel threads)
+│   ├── ReadValueAction
+│   ├── ReadValuePrefixAction
+│   └── ReadRangeAction
+├── SharedRocksDBState    -- holds RocksDB options configured from knobs
+├── ReadIteratorPool      -- recycles rocksdb::Iterator objects across reads
+└── FlowLock semaphores   -- readSemaphore, fetchSemaphore for backpressure
+```
+
+In simulation, the reader/writer threads use `CoroThreadPool` (running on the network thread) to avoid time-advancement issues. In production, they use real OS threads.
+
+#### Write Path
+
+**`set(KeyValueRef kv)`** (line 2183):
+1. Lazy-initializes a `rocksdb::WriteBatch` (with configurable per-key protection bytes)
+2. Calls `writeBatch->Put(columnFamily, key, value)`
+3. If `ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE` is enabled, tracks the key in `keysSet` for later delete optimization
+
+**`clear(KeyRangeRef range)`** (line 2202):
+Three strategies depending on knobs and range shape:
+1. **Single-key delete** -- if `range.singleKeyRange()` and not `ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE`: `writeBatch->Delete(key)`. This is preferred because RocksDB handles point tombstones more efficiently than range tombstones.
+2. **Converted deletes** -- if `ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE` is enabled and the delete budget (`maxDeletes`) hasn't been exhausted: iterate the range using a RocksDB iterator and emit individual `Delete()` calls for each key found. Also deletes any keys in `keysSet` and `previousCommitKeysSet` that fall within the range. Falls back to `DeleteRange()` if iteration fails or budget exhausted.
+3. **Native `DeleteRange()`** -- `writeBatch->DeleteRange(begin, end)`. Used as the default when conversion is disabled.
+
+**`commit()`** (line 2308, via `commitInRocksDB`):
+1. If `writeBatch` is null (no mutations), returns immediately
+2. Creates a `Writer::CommitAction` and transfers ownership of the `WriteBatch`
+3. Records delete histograms, moves `keysSet` to `previousCommitKeysSet`
+4. Posts the action to `writeThread`
+5. The writer thread calls `rocksdb::DB::Write(writeOptions, writeBatch)` -- this atomically persists all buffered mutations
+
+**`canCommit()` backpressure** (line 2282):
+Before each commit, the storage server calls `canCommit()` which invokes `checkRocksdbState()`. This checks:
+- `EstimatePendingCompactionBytes` > `ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT`
+- `NumImmutableMemTable` >= `ROCKSDB_CAN_COMMIT_IMMUTABLE_MEMTABLES_LIMIT`
+
+If overloaded, it delays up to `ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD * ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD` seconds, rechecking each iteration.
+
+#### Read Path
+
+**`readValue(KeyRef key, ReadOptions)`** (line 2365):
+1. Extract `ReadType` (NORMAL or FETCH) from options
+2. If the key is a system key or an eager read, skip throttling and post directly to `readThreads`
+3. Otherwise, check `FlowLock` waiter count; throw `server_overloaded()` if too many waiters
+4. Acquire `FlowLock` semaphore with timeout (`ROCKSDB_READ_QUEUE_WAIT`)
+5. Post `ReadValueAction` to `readThreads`; the reader thread calls `rocksdb::DB::Get()`
+6. Return the result future
+
+**`readRange(KeyRangeRef keys, rowLimit, byteLimit, ReadOptions)`** (line 2433):
+Similar throttling path. The reader thread:
+1. Gets an iterator from `ReadIteratorPool` (reuse if configured, or create new with bounded key range)
+2. For forward reads: `Seek(begin)`, iterate with `Next()` until `end`/limit
+3. For reverse reads: `SeekForPrev(end)`, iterate with `Prev()` until `begin`/limit
+4. Enforces read timeouts (`ROCKSDB_SET_READ_TIMEOUT`) and returns `transaction_too_old` if exceeded
+5. Returns the iterator to the pool for potential reuse
+
+#### ReadIteratorPool (line 539)
+
+Manages a pool of `rocksdb::Iterator` objects to avoid the cost of creating/destroying them for every read:
+- **On read**: provides an unused iterator if available (matching key range for bounded iterators), or creates a new one
+- **On commit**: marks all iterators as stale (increments `deletedUptoIndex`), since the underlying data has changed
+- **Periodic refresh**: a background actor (`refreshReadIteratorPool`) periodically removes stale and expired iterators
+- **Bounded iterators**: when `ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS` is enabled, iterators are created with `iterate_lower_bound`/`iterate_upper_bound` set, and can only be reused for sub-ranges
+- **Pool size limit**: `ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT` caps the number of cached iterators to avoid memory pressure
+
+#### Error Handling
+
+`RocksDBErrorListener` (an `rocksdb::EventListener`) catches background errors from compaction and flush:
+- IO errors → `io_error()`
+- Corruption → `file_corrupt()`
+- Other → `unknown_error()`
+
+These errors are forwarded to the storage server via a promise, causing the SS to shut down and be re-recruited.
+
+#### Configuration (Knobs)
+
+`SharedRocksDBState` constructs RocksDB options from server knobs. Key settings:
+
+| Category | Knobs |
+|----------|-------|
+| **Write buffers** | `ROCKSDB_MEMTABLE_BYTES`, `ROCKSDB_WRITE_BUFFER_SIZE`, `ROCKSDB_MAX_WRITE_BUFFER_NUMBER`, `ROCKSDB_MIN_WRITE_BUFFER_NUMBER_TO_MERGE` |
+| **Compaction** | `ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER`, `ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER`, `ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER`, `ROCKSDB_PERIODIC_COMPACTION_SECONDS` (with random 2-day jitter), `ROCKSDB_MAX_COMPACTION_BYTES`, `ROCKSDB_MAX_BYTES_FOR_LEVEL_MULTIPLIER`, `ROCKSDB_COMPACTION_PRI` |
+| **Block cache** | `ROCKSDB_BLOCK_CACHE_SIZE` (LRU cache), `ROCKSDB_CACHE_HIGH_PRI_POOL_RATIO`, `ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS` |
+| **Bloom filter** | `ROCKSDB_BLOOM_BITS_PER_KEY` (typically 10, ~1% false positive rate), `ROCKSDB_PREFIX_LEN` (fixed prefix for prefix bloom), `ROCKSDB_BLOOM_WHOLE_KEY_FILTERING` |
+| **I/O** | `ROCKSDB_USE_DIRECT_READS`, `ROCKSDB_USE_DIRECT_IO_FLUSH_COMPACTION`, `ROCKSDB_BACKGROUND_PARALLELISM`, `ROCKSDB_MAX_SUBCOMPACTIONS`, `ROCKSDB_MAX_OPEN_FILES` |
+| **Protection** | `ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY`, `ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY`, `ROCKSDB_PARANOID_FILE_CHECKS`, `ROCKSDB_FULLFILE_CHECKSUM` (CRC32c) |
+| **Write rate** | `ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC`, `ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS`, `ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE` |
+| **Deletion optimization** | `ROCKSDB_ENABLE_COMPACT_ON_DELETION` (triggers compaction when tombstone density is high in a sliding window: `ROCKSDB_CDCF_SLIDING_WINDOW_SIZE`, `ROCKSDB_CDCF_DELETION_TRIGGER`, `ROCKSDB_CDCF_DELETION_RATIO`), `ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE`, `ROCKSDB_SINGLEKEY_DELETES_MAX` |
+| **Read throttling** | `ROCKSDB_READ_QUEUE_SOFT_MAX`, `ROCKSDB_READ_QUEUE_HARD_MAX`, `ROCKSDB_FETCH_QUEUE_SOFT_MAX`, `ROCKSDB_FETCH_QUEUE_HARD_MAX`, `ROCKSDB_READ_QUEUE_WAIT` |
+| **Read parallelism** | `ROCKSDB_READ_PARALLELISM` (number of reader threads), `ROCKSDB_WRITER_THREAD_PRIORITY`, `ROCKSDB_READER_THREAD_PRIORITY` |
+
+#### Observability
+
+Comprehensive metrics are emitted via `rocksDBMetricLogger` (line 1008):
+- **RocksDB ticker stats**: block cache hits/misses, bloom filter stats, bytes read/written, compaction bytes, WAL syncs, memtable hits, iterator counts
+- **RocksDB histogram stats** (when `ROCKSDB_STATS_LEVEL > kExceptHistogramOrTimers`): compaction time, compression time, write stall duration
+- **RocksDB property stats**: live SST size, pending compaction bytes, memtable counts, snapshot counts, estimated key count
+- **FDB-side histograms**: commit latency, read value/range latency, queue wait times, deletes per commit
+- **PerfContext metrics** (when `ROCKSDB_PERFCONTEXT_ENABLE`): per-thread breakdown of block reads, seeks, memtable operations, environment operations
+
+---
+
+### Redwood (VersionedBTree) Engine -- [`VersionedBTree.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/VersionedBTree.actor.cpp) (~11K lines)
+
+A custom versioned copy-on-write B-tree storage engine built from scratch for FoundationDB. Named "Redwood," it is designed for high space efficiency through delta compression and for tight integration with FDB's versioned storage model.
+
+#### Architecture
+
+```
+KeyValueStoreRedwood (IKeyValueStore)
+└── VersionedBTree
+    ├── IPager2* m_pager              -- DWALPager instance (physical page I/O)
+    ├── MutationBuffer* m_pBuffer     -- in-memory pending mutations (ART-based)
+    ├── BTreeCommitHeader m_header    -- persistent root pointer + metadata
+    ├── LazyClearQueue                -- deferred page deletions (FIFO)
+    └── BTreeCursor                   -- read cursor for tree traversal
+
+DWALPager (IPager2)
+├── PageCacheT pageCache              -- LRU page cache (ObjectCache<LogicalPageID, PageCacheEntry>)
+├── PageToVersionedMapT remappedPages -- logical→physical page remapping for COW
+├── DelayedFreePageQueueT             -- pages freed at a version (deferred reclamation)
+├── RemapQueueT                       -- remap cleanup queue
+├── PriorityMultiLock ioLock          -- I/O prioritization
+└── IAsyncFile* pageFile              -- the underlying data file
+
+BTreePage (in-memory representation of a disk page)
+├── height (1 = leaf, >1 = internal)
+├── kvBytes (total uncompressed KV bytes)
+└── DeltaTree2<RedwoodRecordRef> tree -- delta-compressed sorted record storage
+```
+
+#### KeyValueStoreRedwood (line 7609)
+
+The `IKeyValueStore` adapter. Constructor creates a `DWALPager` and wraps it in a `VersionedBTree`.
+
+Note: Although `VersionedBTree` supports multi-version reads via the pager's snapshot mechanism, `KeyValueStoreRedwood` currently only keeps one version -- each `commit()` immediately sets `oldestReadableVersion` to the commit version, discarding all prior versions. All reads go against `getLastCommittedVersion()`. This means the storage server's MVCC layer (in-memory `versionedData`) provides version history, not the storage engine itself.
+
+```cpp
+KeyValueStoreRedwood(filename, logID, db, encodingType, pageCacheBytes) {
+    IPager2* pager = new DWALPager(pageSize, extentSize, filename, pageCacheBytes,
+                                   remapCleanupWindowBytes, concurrentExtentReads, false);
+    m_tree = new VersionedBTree(pager, filename, logID, db);
+}
+```
+
+Key parameters:
+- **`pageSize`**: `REDWOOD_DEFAULT_PAGE_SIZE` (default ~4096), BUGGIFied in simulation to random values 1000-16384
+- **`extentSize`**: `REDWOOD_DEFAULT_EXTENT_SIZE` (multiple pages per I/O for large sequential reads)
+- **`pageCacheBytes`**: configurable via `PAGE_CACHE_4K` knob (or `SIM_PAGE_CACHE_4K` in simulation)
+- **`remapCleanupWindowBytes`**: `REDWOOD_REMAP_CLEANUP_WINDOW_BYTES` (100MB default), controls how much remap history to keep before cleaning up; BUGGIFied in simulation
+
+#### Write Path
+
+**`set(KeyValueRef kv)`** / **`clear(KeyRangeRef range)`** -- delegated directly to `VersionedBTree`:
+
+```cpp
+// VersionedBTree::set() (line 4916)
+void set(KeyValueRef keyValue) {
+    ++m_mutationCount;
+    m_pBuffer->insert(keyValue.key)
+        .mutation().setBoundaryValue(m_pBuffer->copyToArena(keyValue.value));
+}
+
+// VersionedBTree::clear() (line 4924)
+void clear(KeyRangeRef clearedRange) {
+    ++m_mutationCount;
+    if (clearedRange.singleKeyRange()) {
+        m_pBuffer->insert(clearedRange.begin).mutation().clearBoundary();
+    } else {
+        auto iBegin = m_pBuffer->insert(clearedRange.begin);
+        auto iEnd = m_pBuffer->insert(clearedRange.end);
+        iBegin.mutation().clearAll();
+        ++iBegin;
+        m_pBuffer->erase(iBegin, iEnd);
+    }
+}
+```
+
+Mutations are buffered in `MutationBuffer` (either an ART (Adaptive Radix Tree) implementation for performance, or `std::map`-based fallback). The buffer represents pending mutations as a sorted map from key boundaries to `RangeMutation` structures, which track whether the boundary is set/cleared and whether the range after it is cleared.
+
+**`commit(Version v)`** (line 5168):
+1. Swaps the current `MutationBuffer` with a fresh one (so new mutations can arrive during commit)
+2. Creates a `CommitBatch` with the mutations, read version (from pager), and write version
+3. Takes a snapshot for reading the current tree state
+4. Walks the B-tree, merging buffered mutations with existing pages:
+   - Reads existing leaf pages
+   - Applies mutations (sets, clears) to produce updated records
+   - Rebuilds modified pages using `PageToBuild` to calculate optimal page splits
+   - Writes modified pages via `m_pager->updatePage()` or `atomicUpdatePage()` (copy-on-write)
+   - Propagates changes up through internal pages as needed
+5. Runs lazy clear to free pages from deleted subtrees
+6. Updates `BTreeCommitHeader` with new root pointer and lazy delete queue state
+7. Calls `m_pager->commit(version, commitRecord)` to make everything durable
+
+#### Read Path
+
+**`readValue(key)`** (line 7879):
+1. Initialize a `BTreeCursor` at the last committed version
+2. `cursor.seekGTE(key)` -- traverse the B-tree from root to leaf
+3. If the cursor lands on the exact key, return the value (with arena dependency on the page buffer)
+4. Otherwise return `Optional<Value>()`
+
+**`readRange(keys, rowLimit, byteLimit)`** (line 7738):
+1. Initialize `BTreeCursor` at last committed version
+2. For forward reads: `seekGTE(keys.begin)`, optionally `prefetch(keys.end, ...)`
+3. Walk through leaf pages using the `BTreePage::BinaryTree::Cursor` directly (no async waits needed within a leaf page)
+4. For each leaf: check whether the entire leaf is within the query range to skip per-key bounds checking
+5. Accumulate results until `rowLimit` or `byteLimit` is reached
+6. When a leaf is exhausted, `popPath()` and `moveNext()` to traverse to the next leaf (this may require async page reads)
+7. For reverse reads: `seekLT(keys.end)`, traverse with `movePrev()`
+
+The `prefetch()` mechanism pre-reads pages along the expected traversal path, reducing latency for sequential scans.
+
+#### DWALPager (Double Write-Ahead Log Pager) (line 1934)
+
+The physical page storage layer for Redwood. Manages pages on disk with copy-on-write versioning.
+
+**Key concepts:**
+- **Logical vs. Physical pages**: The B-tree operates on logical page IDs. The pager maintains a mapping (`PageToVersionedMapT`) from logical to physical page IDs, which changes when a page is updated (copy-on-write). Old physical pages are freed at the version they were superseded.
+- **Page cache**: An LRU `ObjectCache<LogicalPageID, PageCacheEntry>` that caches pages in memory. Each entry tracks read and write futures and is evictable when both are complete.
+- **Header pages**: Two header pages (primary at physical page 0, backup at page 1) store the pager's metadata. They are alternately updated for crash safety.
+- **Recovery**: On startup, `recover()` reads header pages, validates checksums, and reconstructs the page mapping from the remap queue.
+
+**Key operations:**
+- `updatePage(logicalPageIDs, data)` -- writes a page, potentially to a new physical location (copy-on-write)
+- `atomicUpdatePage(logicalPageID, data, version)` -- creates a new physical page and remaps the logical ID to it at the given version
+- `readPage(physicalPageID)` -- reads a page from cache or disk
+- `newPageID()` -- allocates a new logical page ID from the free list
+- `freePage(logicalPageID, version)` -- schedules a page for deferred deletion at the given version
+- `commit(version, commitRecord)` -- flushes all pending writes, updates the header, and syncs to disk
+
+**Queues:**
+- `DelayedFreePageQueueT` -- pages whose physical storage can be reclaimed after `oldestReadableVersion` advances past them
+- `RemapQueueT` -- tracks logical→physical remappings that need cleanup; entries have types: `REMAP` (logical moved to new physical), `FREE` (logical page freed), `DETACH` (logical page detached from remap tracking)
+- `LazyClearQueueT` (in VersionedBTree) -- subtrees that need their pages freed; processed incrementally between commits by `incrementalLazyClear()`
+
+#### BTreePage and DeltaTree Compression (line 4566)
+
+Each B-tree page stores its records in a `DeltaTree2<RedwoodRecordRef>` -- a sorted, delta-compressed structure optimized for space efficiency.
+
+**RedwoodRecordRef** (line 4106):
+```
+struct RedwoodRecordRef {
+    KeyRef key;
+    Optional<ValueRef> value;  // absent = tombstone (cleared key)
+    // For internal pages, value = child page IDs (BTreeNodeLinkRef)
+};
+```
+
+**Delta encoding**: Each record is stored as a delta relative to a reference record (typically the previous sibling in the tree). The `Delta` structure uses a flags byte and variable-length fields:
+- **Prefix borrowing**: Keys share a common prefix with a reference record (left or right neighbor). The delta stores only the differing suffix.
+- **4 length formats** (encoded in 2 bits of the flags byte): 3, 4, 6, or 7 bytes of length fields, supporting keys and values from small (<256 byte) to large (multi-GB)
+- **Flags**: `PREFIX_SOURCE_PREV` (borrow from left vs. right ancestor), `IS_DELETED` (tombstone), `HAS_VALUE` (value present)
+
+This typically achieves 50-70% space savings over raw key-value storage for workloads with key locality.
+
+**Page structure:**
+- `height`: 1 = leaf (stores user key-value pairs), >1 = internal (stores separator keys → child page ID pointers)
+- `kvBytes`: total uncompressed KV bytes on the page (for metrics)
+- The rest of the page is the `DeltaTree2` binary data
+
+**Page building** (`PageToBuild`, line 5469): During commits, records are packed into pages with awareness of delta sizes, block boundaries, and page utilization. The `shiftItem()` method rebalances records between adjacent pages to improve space efficiency.
+
+#### Lazy Clear Queue
+
+When a subtree is deleted (e.g., via a range clear that removes an entire internal page's worth of data), the internal page is pushed onto the `LazyClearQueue` rather than immediately freeing all descendant pages. The `incrementalLazyClear()` actor (line 4974) processes this queue between commits:
+1. Pop entries from the queue in batches of `REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES`
+2. For each entry, read the page and iterate its children:
+   - Height 2 pages: free leaf children directly (`freeBTreePage`)
+   - Height >2 pages: re-queue children for later processing
+3. Free the page itself
+4. Stop when the queue is exhausted or `REDWOOD_LAZY_CLEAR_MAX_PAGES` have been freed
+
+This amortizes the cost of large deletions across multiple commits.
+
+#### BTreeCommitHeader (line 4862)
+
+Persistent metadata stored in the pager's commit record:
+```
+struct BTreeCommitHeader {
+    uint32_t formatVersion;      // currently FORMAT_VERSION = 17
+    EncodingType encodingType;   // page encoding (XXHash64, etc.)
+    uint8_t height;              // tree height
+    LazyClearQueueT::QueueState lazyDeleteQueue;  // lazy clear queue state
+    BTreeNodeLink root;          // root page ID(s) -- may be multi-page for oversized root
+};
+```
+
+---
+
+### SQLite Engine -- [`KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp) (~2.4K lines)
+
+The original FDB storage engine, using a modified embedded SQLite B-tree. Available as `SSD_BTREE_V1` (`.fdb` files) and `SSD_BTREE_V2` (`.sqlite` files, with checksums and an improved page format). This is a legacy engine; RocksDB and Redwood are the active production engines.
+
+#### Architecture
+
+```
+KeyValueStoreSQLite (IKeyValueStore)
+├── Writer (IThreadPoolReceiver, single CoroThreadPool thread)
+│   ├── InitAction       -- opens SQLite DB, runs checkpoint, optional integrity check
+│   ├── SetAction         -- cursor->set(kv)
+│   ├── ClearAction       -- cursor->fastClear(range) + cursor->clear(range)
+│   ├── CommitAction      -- cursor->commit(), fullCheckpoint()
+│   └── SpringCleaningAction -- lazy page deletion + vacuuming
+├── Reader (IThreadPoolReceiver, SQLITE_READER_THREADS CoroThreadPool threads)
+│   ├── ReadValueAction
+│   ├── ReadValuePrefixAction
+│   └── ReadRangeAction
+├── ReadCursor[]          -- per-reader cursor, recycled up to SQLITE_CURSOR_MAX_LIFETIME_BYTES
+├── PageChecksumCodec     -- per-page CRC32c checksums (V2 only)
+└── VFSAsync              -- async VFS layer bridging SQLite to FDB's IAsyncFile
+```
+
+Like RocksDB, all I/O runs on `CoroThreadPool` threads (coroutine-based, running on the network thread). The writer thread handles all mutations and commits; multiple reader threads handle reads in parallel.
+
+#### Write Path
+
+**`set(KeyValueRef kv)`** (line 2231):
+Posts a `Writer::SetAction` to the write thread. The writer calls `cursor->set(kv)` on the SQLite B-tree cursor. Before each set, `checkFreePages()` runs lazy deletion if the free list is below `CHECK_FREE_PAGE_AMOUNT`.
+
+**`clear(KeyRangeRef range)`** (line 2235):
+Posts a `Writer::ClearAction`. The writer calls both:
+1. `cursor->fastClear(range)` -- marks B-tree pages as pending lazy deletion (added to the "free table") without immediately reclaiming them
+2. `cursor->clear(range)` -- removes the key range from the B-tree index
+
+**`commit()`** (line 2239):
+Posts a `Writer::CommitAction`. The writer:
+1. Calls `cursor->commit()` to finalize the SQLite transaction
+2. Destroys the cursor (releasing the read transaction)
+3. Calls `fullCheckpoint()` -- resets all reader cursors, then runs `conn.checkpoint()` twice (once to flush WAL to the main DB file, once to reset the WAL). This ensures the WAL file is truncated after every commit.
+4. Creates a new write cursor for the next batch of mutations
+
+#### Read Path
+
+Reads are posted directly to the reader thread pool without throttling (unlike RocksDB):
+- **`readValue(key)`** -- `getCursor()->get(key)` on the SQLite B-tree
+- **`readValuePrefix(key, maxLength)`** -- `getCursor()->getPrefix(key, maxLength)`
+- **`readRange(keys, rowLimit, byteLimit)`** -- `getCursor()->getRange(keys, rowLimit, byteLimit)`
+
+Each reader thread holds a `ReadCursor` that is recycled after `SQLITE_CURSOR_MAX_LIFETIME_BYTES` of data read. All reader cursors are invalidated on each commit via `resetReaders()` in `fullCheckpoint()`.
+
+#### Spring Cleaning
+
+A background `cleanPeriodically()` actor runs `SpringCleaningAction` on the writer thread to reclaim space:
+1. **Lazy deletion** -- processes the "free table" (B-tree pages marked for deletion by `fastClear`), freeing them in batches of `SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE`. Bounded by time (`SPRING_CLEANING_LAZY_DELETE_TIME_ESTIMATE`) and page count (`SPRING_CLEANING_MIN/MAX_LAZY_DELETE_PAGES`).
+2. **Vacuuming** -- calls `conn.vacuum()` to compact the database file by moving pages from the end to free slots, enabling file truncation. Bounded by `SPRING_CLEANING_MIN/MAX_VACUUM_PAGES` and `SPRING_CLEANING_VACUUM_TIME_ESTIMATE`.
+3. The relative frequency of lazy deletion vs. vacuuming is controlled by `SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE`.
+
+The cleaning interval adapts: `SPRING_CLEANING_LAZY_DELETE_INTERVAL` if there's more work to do, `SPRING_CLEANING_NO_ACTION_INTERVAL` if idle.
+
+#### Page Checksums (V2)
+
+`PageChecksumCodec` computes an 8-byte checksum (two 32-bit halves: `part1` and `part2`) stored in the reserved bytes at the end of each SQLite page.
+
+**Writes** always use xxHash3 (with `part1`'s top 8 bits zeroed as a format marker).
+
+**Reads** try three algorithms in order to handle pages written by older code:
+1. **CRC32c** -- tried first if `part1 == 0` (legacy format)
+2. **xxHash3** -- tried next if `part1`'s top 8 bits are zero (current format)
+3. **hashlittle2** -- fallback for the oldest format (seeded with page number)
+
+If all three fail, the page is corrupt. The V1 format (`SSD_BTREE_V1`) does not use page checksums.
+
+#### Integrity Checking
+
+On open, optional integrity checks can be performed:
+- **`checkAllChecksumsOnOpen`** -- scans every page and validates checksums; throws `file_corrupt()` on mismatch
+- **`checkIntegrityOnOpen`** -- runs a full B-tree structural integrity check via `conn.check()`
+
+A standalone `KVFileCheck()` utility function (line 2311) can validate `.fdb`/`.sqlite` files offline, and `KVFileDump()` can dump all key-value pairs.
+
+---
+
+### Memory Engine -- [`KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp) (~960 lines)
+
+In-memory key-value store backed by a [`DiskQueue`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DiskQueue.cpp) (append-only circular log) for durability:
+- All mutations are logged to `.fdq` files (a pair of alternating files, checksummed with xxhash3)
+- On recovery, the DiskQueue log is replayed to reconstruct the in-memory state
+- Periodic snapshots write the full state to the log so older entries can be reclaimed
+- Two backing containers: `IKeyValueContainer` (std::map-like, default) or `radix_tree` (`MEMORY_RADIXTREE` type)
+
+**Users:**
+- **Coordinator Paxos state** -- `OnDemandStore` creates a `KeyValueStoreMemory` per coordinator to persist generation register state (see [Cluster Controller & Coordination](subsystem_04_cluster_controller.md#coordinator-storage))
+- **Transaction log mutation queues** -- TLogs use `KeyValueStoreMemory` with a DiskQueue for their durable mutation log
+- **`txnStateStore`** during recovery -- ephemeral metadata reconstructed from TLog data
+- **Testing** -- in-memory storage engine for simulation tests
+
+---
+
+## Data Flow Summary
+
+```
+TLog stream ──peek(myTag)──▶ StorageServer update() loop
+                                │
+                                ▼
+                            Parse mutations from cursor
+                                │
+                                ▼
+                          Eager reads (prefetch for atomic ops)
+                                │
+                                ▼
+                            applyMutation() → MVCC versionedData
+                                │
+                                ▼
+                            Advance version
+                                │
+                                ▼
+                        updateStorage() → makeVersionMutationsDurable()
+                                │
+                                ▼
+                        storage.set()/clear() → engine-specific write batch
+                                │
+                                ▼
+                        canCommit() (RocksDB backpressure check)
+                                │
+                                ▼
+                        storage.commit() → durableVersion advances
+                                │
+                                ▼
+                        pop(durableVersion, myTag) → TLog can discard old data
+
+
+Client ──GetValueRequest──▶ StorageServer getValueQ()
+                                │
+                                ▼
+                            waitForVersion(requestedVersion)
+                                │
+                                ▼
+                            Check shard ownership
+                                │
+                                ▼
+                            Read from MVCC or KVStore at version
+                                │
+                                ▼
+                            Return value to client
+```
+
+---
+
+## Engine Comparison
+
+| Aspect | RocksDB | Redwood | SQLite | Memory |
+|--------|---------|---------|--------|--------|
+| **Type** | LSM-tree | Copy-on-write B-tree | Modified SQLite B-tree | In-memory + WAL |
+| **Store types** | `SSD_ROCKSDB_V1` | `SSD_REDWOOD_V1` | `SSD_BTREE_V1`, `SSD_BTREE_V2` | `MEMORY`, `MEMORY_RADIXTREE` |
+| **Write amplification** | Higher (compaction) | Lower (in-place COW) | Low (WAL + checkpoint) | None (append-only log) |
+| **Space amplification** | Moderate (dead data in LSM levels) | Low (delta compression) | Moderate (free-list fragmentation) | N/A (RAM-bound) |
+| **Read amplification** | Moderate (bloom filter helps) | Low (direct B-tree traversal) | Low (direct B-tree lookup) | None (direct map lookup) |
+| **Compression** | RocksDB block compression | DeltaTree prefix/suffix sharing | None | None |
+| **Checkpoints** | Native RocksDB checkpoints | Not implemented | Not implemented | Not implemented |
+| **Page checksums** | Block protection bytes | XXHash64 page encoding | xxHash3 per-page (V2 only; reads also accept CRC32c and hashlittle2 legacy formats) | N/A |
+| **Threading** | Dedicated read/write thread pools | Flow event loop (async page I/O) | CoroThreadPool (read + write) | Single-threaded |
+| **Space reclamation** | Background compaction | Lazy clear queue + remap cleanup | Spring cleaning (lazy delete + vacuum) | DiskQueue log reclamation |
+| **Primary use** | Production storage servers | Alternative storage engine | Legacy (original engine) | Coordinators, TLogs, testing |
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/storageserver/storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp) | SS main loop, update, read serving, shard management |
+| [`fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h) | IKeyValueStore interface definition |
+| [`fdbserver/kvstore/IKeyValueStore.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/IKeyValueStore.cpp) | Factory function `openKVStore()` |
+| [`fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp) | RocksDB engine implementation |
+| [`fdbserver/kvstore/VersionedBTree.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/VersionedBTree.actor.cpp) | Redwood engine (VersionedBTree + DWALPager) |
+| [`fdbserver/kvstore/IPager.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/IPager.h) | Pager interface (IPager2) for Redwood |
+| [`fdbserver/kvstore/DeltaTree.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DeltaTree.h) | DeltaTree2 delta-compressed sorted structure |
+| [`fdbserver/kvstore/KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp) | SQLite engine (V1/V2) |
+| [`fdbserver/kvstore/KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp) | Memory engine |
+| [`fdbserver/kvstore/DiskQueue.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DiskQueue.cpp) | Append-only circular log (for Memory engine durability) |
diff --git a/design/AI-generated/subsystem_08_data_distribution.md b/design/AI-generated/subsystem_08_data_distribution.md
new file mode 100644
index 00000000000..8e29afad6bb
--- /dev/null
+++ b/design/AI-generated/subsystem_08_data_distribution.md
@@ -0,0 +1,238 @@
+# Subsystem 8: Data Distribution
+
+**[Diagrams](diagram_08_data_distribution.md)**
+
+**Location:** [`fdbserver/datadistributor/`](https://github.com/apple/foundationdb/tree/main/fdbserver/datadistributor)
+**Size:** ~22K  
+**Role:** Shard management, team building, rebalancing, MoveKeys protocol.
+
+---
+
+## Overview
+
+Data Distribution (DD) is a continuous background process that decides which storage servers hold which key ranges, and moves data when the assignment needs to change. The DD singleton is recruited by the Cluster Controller and runs for the lifetime of the cluster (re-recruited on failure).
+
+---
+
+## Main Actor -- [`DataDistribution.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DataDistribution.cpp)`:2602-2888`
+
+The `dataDistribution()` actor spawns four major components:
+
+1. **DataDistributionTracker** -- monitors shard metrics, triggers splits/merges/moves
+2. **DDQueue** -- prioritized queue of shard relocations, executes moves
+3. **DDTeamCollection (Primary)** -- builds teams, monitors health, recruits storage servers
+4. **DDTeamCollection (Remote)** -- same for remote datacenter (if `usableRegions > 1`)
+
+Plus optional bulk load/dump cores.
+
+### Initialization Sequence
+
+1. `initDDConfigWatch()` -- watch database configuration for changes
+2. `DataDistributor::init()` -- load metadata from system keys
+3. Resume relocations from previous run
+4. Spawn tracker, queue, and team collections
+5. `waitForAll(actors)` -- run until fatal error
+
+---
+
+## DDTeamCollection -- [`DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp)
+
+### What is a Team?
+
+A "team" is a set of storage servers that together hold replicas of a shard. For `triple` replication, a team has 3 servers in 3 different zones.
+
+### Key Members (`DDTeamCollection.h:209+`)
+
+```
+std::vector<Reference<TCTeamInfo>> badTeams;
+std::map<UID, Reference<TCServerInfo>> server_and_tss_info;
+std::map<Key, int> lagging_zones;
+Reference<StorageWiggler> storageWiggler;
+
+int healthyTeamCount, optimalTeamCount;
+AsyncVar<bool> zeroHealthyTeams;
+Future<Void> teamBuilder;
+PromiseStream<RelocateShard> output;          // relocation requests
+```
+
+### Team Building
+
+`buildTeams()` creates new teams from healthy servers:
+- Each team satisfies the replication policy (e.g., 3 replicas across 3 zones)
+- All teams have exactly `teamSize` members
+- Respects excluded/failed server lists
+- **Machine teams** enforce datacenter locality constraints
+
+### Health Monitoring
+
+**`teamTracker()`** -- per-team actor:
+- Watches for server failures (team < replication factor)
+- Watches for server exclusion
+- Emits `RelocateShard` when team health degrades
+
+**`storageServerTracker()`** -- per-server actor:
+- Monitors interface responsiveness, disk space, excluded status
+- Marks failed on timeout or storage errors
+
+**`storageRecruiter()`** -- continuous recruitment:
+- Recruits new servers when existing ones fail
+- Sends `RecruitStorageRequest` to cluster controller
+
+---
+
+## DDShardTracker -- [`DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp)
+
+Monitors shard sizes and operation rates, triggers splits/merges.
+
+### Bandwidth Status (lines 42-58)
+
+| Status | Condition |
+|--------|-----------|
+| `BandwidthStatusLow` | `bytesWrittenPerKSec < SHARD_MIN_BYTES_PER_KSEC` |
+| `BandwidthStatusNormal` | Between min and max |
+| `BandwidthStatusHigh` | `bytesWrittenPerKSec > SHARD_MAX_BYTES_PER_KSEC` |
+
+### Shard Size Bounds (lines 85-151)
+
+`calculateShardSizeBounds()` returns bounds that trigger splits/merges:
+
+- **Byte bounds**: `max.bytes = max(currentBytes * 1.1, MIN_SHARD_BYTES)`, permits 10% deviation
+- **Write bandwidth bounds**: Based on `SHARD_MIN/MAX_BYTES_PER_KSEC`
+- **Read bandwidth bounds**: Based on `SHARD_MAX_READ_DENSITY_RATIO * bytes`
+- **Read operation bounds**: Track `opsReadPerKSecond`, detect hot shards
+
+### Tracking Logic (`trackShardMetrics`)
+
+Continuously monitors each shard. When metrics exceed bounds:
+- **Split** (high bytes/ops): Create two smaller shards, emit `RelocateShard` with `SPLIT_SHARD`
+- **Merge** (low bytes/ops): Merge with adjacent shard, emit `RelocateShard` with `MERGE_SHARD`
+
+---
+
+## DDQueue / DDRelocationQueue -- [`DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp)
+
+### Key Members (`DDRelocationQueue.h:125+`)
+
+```
+std::set<RelocateData, std::greater<>> fetchingSourcesQueue;
+std::map<UID, std::set<RelocateData>> queue;      // per-server queue
+KeyRangeMap<RelocateData> inFlight;                // currently executing
+KeyRangeActorMap inFlightActors;                   // actor per in-flight move
+KeyRangeMap<DDDataMove> dataMoves;                 // move metadata
+std::map<UID, Busyness> busymap, destBusymap;     // server utilization
+```
+
+### RelocateData (lines 74-122)
+
+```
+struct RelocateData {
+    int priority;                   // urgency level
+    RelocateReason reason;
+    DataMovementReason dmReason;
+    KeyRange keys;                  // range to move
+    std::vector<UID> src;           // source server IDs
+    int workFactor;                 // estimated cost
+    UID dataMoveId;
+    double startTime;
+};
+```
+
+### Relocation Priorities
+
+| Priority | Trigger |
+|----------|---------|
+| `PRIORITY_TEAM_REDUNDANT` | Team replicated on same machine |
+| `PRIORITY_POPULATE_REGION` | Populate new region |
+| `PRIORITY_TEAM_UNHEALTHY` | Unhealthy team |
+| `PRIORITY_REBALANCE_*` | Disk/read balancing |
+| `PRIORITY_SPLIT_SHARD` | Shard too large/hot |
+| `PRIORITY_MERGE_SHARD` | Shard too small/cold |
+
+Ordering: `(priority desc) → (startTime asc) → (randomId desc)`
+
+### Core Operations
+
+- **`queueRelocation()`**: Enqueue shard relocation, search for source team
+- **`launchQueuedWork()`**: Check for overlapping in-flight moves, cancel lower priority, start MoveKeys
+- **`completeSourceFetch()`**: Source servers identified, move to ready-to-launch
+
+---
+
+## MoveKeys Protocol -- [`fdbserver/core/MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp)
+
+Atomic shard ownership transfer using system key transactions.
+
+### Overall Flow
+
+1. Destination SS starts `fetchKeys()` -- pulls data from source SS
+2. During fetch, mutations forwarded to dest via TLog stream (same tag assignment)
+3. Once dest reaches `fetchVersion`: ownership transfers atomically
+4. Source removes shard
+
+### System Keys Modified
+
+| Key Range | Change |
+|-----------|--------|
+| `serverKeysPrefixFor(sourceId)` | Remove key range |
+| `serverKeysPrefixFor(destId)` | Add key range |
+| `keyServersKey(range)` | Update server list for range |
+| `moveKeysLockOwnerKey` | DD process ownership |
+
+### Key Functions
+
+- **`takeMoveKeysLock()`**: Acquire exclusive lock (prevents concurrent moves by different DD instances)
+- **`readMoveKeysLock()`**: Validate current DD still holds lock
+- **`unassignServerKeys()`**: Remove server from key range in system keys, with coalescing
+- **`deleteCheckpoints()`**: Clean up checkpoint metadata after physical shard move
+
+### Conflict Resolution
+
+- Read conflicts added at range boundaries to detect concurrent assignments
+- Ensures no mutations lost during ownership transfer
+- Transactions retry if conflicts detected
+
+---
+
+## Why Data Moves
+
+| Trigger | Action |
+|---------|--------|
+| Shard grows too large | Split into smaller shards |
+| Storage server fails | Re-replicate shards to healthy servers |
+| Load imbalance | Move shards from hot to cold servers |
+| Configuration change | Adjust replication factor |
+| Server exclusion | Move all shards off excluded server |
+| Shard too small | Merge with adjacent shard |
+
+---
+
+## Data Flow
+
+```
+DDShardTracker ──metrics exceed bounds──▶ RelocateShard request
+                                              │
+                                              ▼
+DDQueue ──prioritize, dedup, find dest team──▶ launchQueuedWork()
+                                              │
+                                              ▼
+MoveKeys protocol:
+  1. Dest SS: fetchKeys() from source SS
+  2. Dest SS: start consuming TLog for shard's tag
+  3. Dest SS: catches up to current version
+  4. Atomic system key transaction: transfer ownership
+  5. Source SS: remove shard
+```
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/datadistributor/DataDistribution.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DataDistribution.cpp) | Main DD actor, initialization |
+| [`fdbserver/datadistributor/DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp) | Team building, health monitoring, SS recruitment |
+| [`fdbserver/datadistributor/DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp) | Shard metrics, split/merge decisions |
+| [`fdbserver/datadistributor/DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp) | Relocation queue, prioritization, execution |
+| [`fdbserver/core/MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp) | Atomic shard transfer protocol |
+| [`fdbserver/core/DataMovement.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/DataMovement.cpp) | Data move metadata |
+| [`fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp) | Impact analysis for team failures |
diff --git a/design/AI-generated/subsystem_09_cluster_recovery.md b/design/AI-generated/subsystem_09_cluster_recovery.md
new file mode 100644
index 00000000000..2316caa1703
--- /dev/null
+++ b/design/AI-generated/subsystem_09_cluster_recovery.md
@@ -0,0 +1,215 @@
+# Subsystem 9: Cluster Recovery
+
+**[Diagrams](diagram_09_cluster_recovery.md)**
+
+**Location:** [`fdbserver/clustercontroller/ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp), [`fdbserver/core/RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h)
+**Size:** Part of ClusterController files  
+**Role:** 10-state recovery state machine to reconstitute the transaction system after failures.
+
+---
+
+## Overview
+
+Recovery is triggered when the master (sequencer) fails, a network partition heals, or coordinators change. The Cluster Controller drives a 10-state state machine that locks old TLogs, recruits a new transaction system, replays uncommitted data, and resumes accepting commits.
+
+---
+
+## ClusterRecoveryData -- [`ClusterRecovery.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.h)`:178-318`
+
+```
+struct ClusterRecoveryData {
+    UID dbgid;
+    Version lastEpochEnd;                        // last committed version in old epoch
+    Version recoveryTransactionVersion;          // first version in new epoch
+    Optional<int64_t> versionEpoch;
+    
+    DatabaseConfiguration configuration;
+    std::vector<Optional<Key>> primaryDcId, remoteDcIds;
+    
+    Reference<LogSystem> logSystem;
+    IKeyValueStore* txnStateStore;               // in-memory metadata store
+    LogSystemDiskQueueAdapter* txnStateLogAdapter;
+    
+    std::vector<CommitProxyInterface> commitProxies;
+    std::vector<GrvProxyInterface> grvProxies;
+    std::vector<ResolverInterface> resolvers;
+    
+    ReusableCoordinatedState cstate;             // coordinated cluster state
+    MasterInterface masterInterface;
+    
+    // Recovery signaling
+    Promise<Void> recoveryReadyForCommits;
+    Promise<Void> cstateUpdated;
+};
+```
+
+---
+
+## The 10 Recovery States -- [`RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h)`:31-42`
+
+```
+enum class RecoveryState {
+    UNINITIALIZED = 0,
+    READING_CSTATE = 1,
+    LOCKING_CSTATE = 2,
+    RECRUITING = 3,
+    RECOVERY_TRANSACTION = 4,
+    WRITING_CSTATE = 5,
+    ACCEPTING_COMMITS = 6,
+    ALL_LOGS_RECRUITED = 7,
+    STORAGE_RECOVERED = 8,
+    FULLY_RECOVERED = 9
+};
+```
+
+---
+
+## clusterRecoveryCore() Walk-Through
+
+### State 1: READING_CSTATE (line ~1504)
+
+- Read coordinated state from coordinators
+- Learn about previous epoch: `DBCoreState` with TLog configuration
+- Validate protocol versions
+
+### State 2: LOCKING_CSTATE (line ~1517)
+
+- Lock coordinated state (increment recovery count)
+- Prevents split-brain: old master cannot write new state
+- Uses generation register for safe state transitions
+
+### State 3: RECRUITING (line ~1584)
+
+**`recruitEverything()`** (lines 999-1117):
+1. Send `RecruitFromConfigurationRequest` to cluster controller
+2. CC selects workers based on fitness, locality, exclusions
+3. Initialize in parallel:
+   - `newCommitProxies()` -- `InitializeCommitProxyRequest` to workers
+   - `newGrvProxies()` -- `InitializeGrvProxyRequest`
+   - `newResolvers()` -- `InitializeResolverRequest`
+   - `newTLogServers()` -- creates new log epoch via `oldLogSystem->newEpoch()`
+   - `newSeedServers()` -- seed storage servers (only at version 0)
+4. `monitorInitializingTxnSystem()` provides timeout with exponential backoff
+
+**`provisionalMaster()`** (line ~858):
+- Provides minimal proxy implementation during recovery
+- Serves clients with provisional (limited) responses
+- Accepts emergency transactions (configuration mutations)
+
+### State 4: RECOVERY_TRANSACTION (line ~1635)
+
+**`readTransactionSystemState()`** (lines 1139-1270):
+Peeks old TLogs to reconstruct metadata:
+- `versionEpochKey` -- version epoch
+- `minRequiredCommitVersionKey` -- for versionstamped operations
+- Configuration keys -- database config
+- Tag locality mappings -- datacenter localities
+- Server tags and tag history
+
+Calculates:
+- `lastEpochEnd` -- last committed version in old epoch
+- `recoveryTransactionVersion` -- first version in new epoch
+
+Executes the recovery transaction: writes initial state to the new transaction system.
+
+### State 5: WRITING_CSTATE (line ~1767)
+
+- Write new epoch's `DBCoreState` to coordinators via coordinated state
+- Contains: new TLog configuration, recovery count, log system state
+- Must succeed at quorum of coordinators
+
+### State 6: ACCEPTING_COMMITS (line ~1802)
+
+- New transaction system is live
+- CommitProxies, GrvProxies, Resolvers, TLogs all operational
+- Client commits now flow through the new system
+- Old TLog data still being consumed by storage servers
+
+### State 7: ALL_LOGS_RECRUITED
+
+Transition when all TLog sets (including remote region) are fully populated.
+
+### State 8: STORAGE_RECOVERED
+
+All old generation TLogs have been fully recruited into the system.
+
+### State 9: FULLY_RECOVERED
+
+- Old TLog data fully consumed by storage servers
+- Old generations can be discarded
+- `oldLogSystems->get()->stopRejoins()`
+- Normal operation
+
+---
+
+## trackTlogRecovery() -- [`ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp)`:449-533`
+
+Monitors TLog set completeness and drives state transitions:
+
+```
+loop {
+    // Convert current logSystem to DBCoreState
+    self->logSystem->toCoreState(newState);
+    
+    // Check completeness
+    allLogs = newState.tLogs.size() == configuration.expectedLogSets(primaryDcId);
+    finalUpdate = !newState.oldTLogData.size() && allLogs;
+    
+    // Write state durably
+    co_await self->cstate.write(newState, finalUpdate);
+    
+    // Signal readiness
+    if (self->recoveryReadyForCommits.canBeSet())
+        self->recoveryReadyForCommits.send(Void());
+    
+    // Broadcast updated recovery state
+    self->logSystem->coreStateWritten(newState);
+    
+    if (finalUpdate) {
+        // FULLY_RECOVERED: stop rejoins, start rejoin handler
+        co_return;
+    }
+    
+    co_await changed;  // wait for next logSystem state change
+}
+```
+
+---
+
+## CoordinatedState Protocol
+
+### Read
+
+1. Replicated read from quorum of coordinators
+2. Returns highest-generation value
+3. If client's generation > stored readGen: updates readGen
+
+### Write
+
+1. Replicated write with generation check
+2. `readGen <= requestGen AND writeGen < requestGen` must hold
+3. If valid: update value and writeGen
+4. If conflict: throws `coordinated_state_conflict`
+
+---
+
+## Recovery Timing
+
+| Knob | Default | Purpose |
+|------|---------|---------|
+| `CC_RECOVERY_INIT_REQ_TIMEOUT` | 30s | Base timeout for recruitment |
+| `CC_RECOVERY_INIT_REQ_TIMEOUT_GROWTH_FACTOR` | 2x | Exponential backoff per unfinished recovery |
+| `CC_RECOVERY_INIT_REQ_MAX_TIMEOUT` | 300s | Maximum timeout cap |
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/clustercontroller/ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp) | clusterRecoveryCore, recruitment, state machine |
+| [`fdbserver/clustercontroller/ClusterRecovery.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.h) | ClusterRecoveryData struct |
+| [`fdbserver/core/include/fdbserver/core/RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h) | RecoveryState enum |
+| [`fdbserver/core/include/fdbserver/core/DBCoreState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/DBCoreState.h) | DBCoreState (coordinated TLog config) |
+| [`fdbserver/core/CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp) | Replicated read/write over generation registers |
+| [`fdbserver/logsystem/LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) | newEpoch() for creating new TLog generations |
diff --git a/design/AI-generated/subsystem_10_ratekeeper.md b/design/AI-generated/subsystem_10_ratekeeper.md
new file mode 100644
index 00000000000..a55ea688194
--- /dev/null
+++ b/design/AI-generated/subsystem_10_ratekeeper.md
@@ -0,0 +1,203 @@
+# Subsystem 10: Rate Keeping & Throttling
+
+**[Diagrams](diagram_10_ratekeeper.md)**
+
+**Location:** [`fdbserver/ratekeeper/`](https://github.com/apple/foundationdb/tree/main/fdbserver/ratekeeper)
+**Size:** ~4K  
+**Role:** Back-pressure on commits and reads, tag-based throttling, throughput tracking.
+
+---
+
+## Overview
+
+The Ratekeeper is a singleton role (recruited by Cluster Controller) that monitors cluster health metrics and calculates transaction rate limits. These limits are sent to GRV proxies, which enforce them by delaying `GetReadVersion` responses. This prevents the cluster from being overwhelmed when storage servers or TLogs fall behind.
+
+---
+
+## Ratekeeper Structure -- [`Ratekeeper.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.h)`:129-232`
+
+```
+class Ratekeeper {
+    Map<UID, StorageQueueInfo> storageQueueInfo;     // per-SS metrics
+    Map<UID, TLogQueueInfo> tlogQueueInfo;           // per-TLog metrics
+    std::map<UID, GrvProxyInfo> grvProxyInfo;
+    
+    Smoother smoothReleasedTransactions;              // actual TPS
+    Smoother smoothBatchReleasedTransactions;
+    Smoother smoothTotalDurableBytes;
+    
+    std::unique_ptr<ITagThrottler> tagThrottler;
+    
+    RatekeeperLimits normalLimits;                    // default priority limits
+    RatekeeperLimits batchLimits;                     // batch priority limits
+};
+```
+
+### StorageQueueInfo (lines 37-73)
+
+```
+class StorageQueueInfo {
+    Smoother smoothFreeSpace, smoothTotalSpace;
+    Smoother smoothDurableBytes, smoothInputBytes;
+    Smoother smoothDurableVersion, smoothLatestVersion;
+    StorageQueuingMetricsReply lastReply;
+    limitReason_t limitReason;
+    std::vector<BusyTagInfo> busiestReadTags, busiestWriteTags;
+};
+```
+
+### TLogQueueInfo (lines 75-94)
+
+```
+class TLogQueueInfo {
+    Smoother smoothDurableBytes, smoothInputBytes;
+    Smoother smoothFreeSpace, smoothTotalSpace;
+    TLogQueuingMetricsReply lastReply;
+};
+```
+
+### RatekeeperLimits (lines 96-127)
+
+```
+struct RatekeeperLimits {
+    double tpsLimit;
+    int64_t storageTargetBytes, storageSpringBytes;
+    int64_t logTargetBytes, logSpringBytes;
+    double maxVersionDifference;
+    int64_t durabilityLagTargetVersions;
+    double durabilityLagLimit;
+    TransactionPriority priority;
+};
+```
+
+---
+
+## Rate Calculation -- [`Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp)`:642-850`
+
+### `updateRate(RatekeeperLimits* limits)`
+
+Evaluates multiple metrics and takes the minimum rate:
+
+### 1. Storage Queue Depth
+
+```
+storageQueue = ss.getStorageQueueBytes()
+```
+
+If queue between `springBytes` and `targetBytes`:
+```
+targetRateRatio = (queue - target + spring) / spring
+tpsLimit = min(actualTps * maxBytesPerSecond / inputRate,
+               maxBytesPerSecond * MAX_TRANSACTIONS_PER_BYTE)
+```
+
+### 2. Durability Lag
+
+```
+storageDurabilityLag = ss.getDurabilityLag()  // version gap between latest and durable
+```
+
+If lag exceeds `durabilityLagTargetVersions`: reduce rate proportionally.
+
+### 3. Free Space
+
+- Minimum free space enforcement with ratio factor
+- Hard limit: `STORAGE_HARD_LIMIT_BYTES`
+- Below hard limit: rate drops to 0
+
+### 4. TLog Queue Depth
+
+Similar to storage queue depth but for TLog queues:
+- `logTargetBytes` / `logSpringBytes` thresholds
+- Limits rate when TLogs fall behind
+
+### 5. Write Bandwidth MVCC
+
+```
+maxBytesPerSecond = (targetBytes - springBytes) / mvcc_window
+```
+
+Prevents storage server from filling its MVCC queue faster than it can drain.
+
+### Rate Selection
+
+The final rate is the **minimum** across all metrics, with the `limitReason` tracking which factor is most constraining.
+
+---
+
+## Communication with GRV Proxies -- [`Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp)`:345-410`
+
+### GetRateInfoRequest
+
+```
+struct GetRateInfoRequest {
+    UID requesterID;
+    int64_t totalReleasedTransactions;
+    int64_t batchReleasedTransactions;
+    Version version;
+    TransactionTagMap<uint64_t> throttledTagCounts;
+    bool detailed;
+};
+```
+
+### GetRateInfoReply
+
+```
+struct GetRateInfoReply {
+    double transactionRate;              // normal priority TPS limit
+    double batchTransactionRate;         // batch priority TPS limit
+    double leaseDuration;               // METRIC_UPDATE_RATE
+    HealthMetrics healthMetrics;
+    Optional<PrioritizedTransactionTagMap<ClientTagThrottleLimits>> clientThrottledTags;
+};
+```
+
+### Flow
+
+1. GRV proxy sends `GetRateInfoRequest` periodically
+2. Ratekeeper updates released transaction counts
+3. Calculates per-proxy limits (total limit / num proxies)
+4. Includes per-tag throttle info if tags changed
+5. Returns rate limits + health metrics
+
+GRV proxy enforces limits by delaying `GetReadVersion` responses when rate exceeded.
+
+---
+
+## Tag Throttling -- [`TagThrottler.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/TagThrottler.cpp)
+
+Per-tag (per-tenant/workload) transaction throttling.
+
+### Concepts
+
+| Term | Meaning |
+|------|---------|
+| **Manual throttle** | Operator-defined tag rate limit |
+| **Automatic throttle** | Ratekeeper-generated tag limit for a busy tag |
+| **Priority** | Maximum transaction priority affected by a throttle |
+| **Expiration** | Time at which a throttle is removed |
+
+### Key Functions
+
+- `tryUpdateAutoThrottling()` -- update automatic throttles from busy-tag signals
+- `getClientRates()` -- collect active limits to send to clients
+- `monitorThrottlingChanges()` -- watch persisted throttle state
+
+### Throttle Enforcement
+
+1. Ratekeeper calculates per-tag rate limits
+2. Sends to GRV proxies in `GetRateInfoReply.clientThrottledTags`
+3. Clients enforce tag-specific limits locally
+4. Clients see `tag_throttled` if the rate is exceeded
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/ratekeeper/Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp) | Rate calculation, proxy communication |
+| [`fdbserver/ratekeeper/Ratekeeper.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.h) | Ratekeeper, StorageQueueInfo, TLogQueueInfo, RatekeeperLimits |
+| [`fdbserver/ratekeeper/TagThrottler.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/TagThrottler.cpp) | Per-tag throttling implementation |
+| [`fdbserver/ratekeeper/RkTagThrottleCollection.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/RkTagThrottleCollection.cpp) | Tag throttle collection management |
+| `fdbserver/ratekeeper/RatekeeperLimits.cpp` | Limit calculation utilities |
diff --git a/design/AI-generated/subsystem_11_backup_dr.md b/design/AI-generated/subsystem_11_backup_dr.md
new file mode 100644
index 00000000000..94e3404ef03
--- /dev/null
+++ b/design/AI-generated/subsystem_11_backup_dr.md
@@ -0,0 +1,290 @@
+# Subsystem 11: Backup, Restore & DR
+
+**[Diagrams](diagram_11_backup_dr.md)**
+
+**Location:** [`fdbserver/backupworker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/backupworker), `fdbclient/FileBackupAgent*`, `fdbclient/BackupContainer*`, [`fdbbackup/`](https://github.com/apple/foundationdb/tree/main/fdbbackup)
+**Size:** ~10K server, ~20K client  
+**Role:** Continuous backup to external storage, point-in-time restore, cross-cluster DR.
+
+---
+
+## Overview
+
+FDB supports continuous backup of the mutation stream to external storage, enabling point-in-time restore. The backup system has two main components: server-side BackupWorkers that pull mutations from TLogs, and client-side BackupAgents that orchestrate the backup lifecycle and manage backup containers.
+
+---
+
+## Backup Architecture
+
+### Two Backup File Types
+
+**Log files** -- continuous mutation stream:
+```
+struct LogFile {
+    Version beginVersion, endVersion;
+    uint32_t blockSize;
+    std::string fileName;
+    int64_t fileSize;
+    int tagId, totalTags;      // for partitioned logs
+};
+```
+
+**Range files** -- snapshot of key-value pairs at a version:
+```
+struct RangeFile {
+    Version version;
+    uint32_t blockSize;
+    std::string fileName;
+    int64_t fileSize;
+};
+```
+
+Together, range files + log files enable point-in-time restore: restore the range snapshot, then replay log files to reach any target version.
+
+---
+
+## BackupWorker -- [`fdbserver/backupworker/BackupWorker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/backupworker/BackupWorker.cpp)
+
+Server-side role that pulls mutations from TLogs (like a storage server, but writes to backup storage).
+
+### BackupData (lines 105-287)
+
+```
+struct BackupData {
+    const UID myId;
+    const Tag tag;                          // log router tag (-2, i)
+    const int totalTags;
+    const Version startVersion;
+    const Optional<Version> endVersion;     // old epoch end
+    const LogEpoch recruitedEpoch, backupEpoch;
+    
+    Version savedVersion;                   // largest version saved to storage
+    AsyncVar<Reference<LogSystem>> logSystem;
+    std::vector<VersionedMessage> messages; // in-memory mutation buffer
+    std::map<UID, PerBackupInfo> backups;   // per-backup configurations
+};
+```
+
+### PerBackupInfo (lines 125-258)
+
+```
+struct PerBackupInfo {
+    Version startVersion, lastSavedVersion;
+    Future<Optional<Reference<IBackupContainer>>> container;
+    Future<Optional<std::vector<KeyRange>>> ranges;
+    bool stopped;
+};
+```
+
+### Mutation Pulling
+
+1. BackupWorker peeks TLog via its assigned `tagLocalityLogRouter` tag
+2. Filters mutations based on backup key ranges
+3. Buffers in `messages` vector
+4. Writes to backup container when buffer reaches threshold
+5. Tracks progress in system keys
+
+---
+
+## BackupAgent -- [`fdbclient/BackupAgent.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupAgent.h), [`fdbclient/FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp)
+
+Client-side orchestration of backup lifecycle.
+
+### Backup States
+
+```
+enum class EnumState {
+    STATE_ERRORED = 0,
+    STATE_SUBMITTED = 1,
+    STATE_RUNNING = 2,
+    STATE_RUNNING_DIFFERENTIAL = 3,
+    STATE_COMPLETED = 4,
+    STATE_NEVERRAN = 5,
+    STATE_ABORTED = 6
+};
+```
+
+### FileBackupAgent Methods
+
+| Method | Purpose |
+|--------|---------|
+| `submitBackup()` | Queue a new backup task |
+| `restore()` | Start point-in-time restore |
+| `atomicRestore()` | Atomic restore (single transaction) |
+| `abortRestore()` | Cancel in-progress restore |
+| `waitRestore()` | Wait for restore completion |
+| `restoreStatus()` | Query restore progress |
+
+### TaskBucket Pattern
+
+Backup/restore operations use a distributed task execution pattern:
+- Tasks stored as key-value entries in the database
+- `TaskBucket` polls for ready tasks at `pollDelay` intervals
+- Tasks can spawn sub-tasks for parallelism
+- `FutureBucket` tracks task completion
+- Provides reliability: tasks survive process failures
+
+```
+Future<Void> run(Database cx, double pollDelay, int maxConcurrentTasks);
+```
+
+---
+
+## Backup Container -- [`fdbclient/BackupContainer.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupContainer.h)
+
+### IBackupContainer Interface (lines 249-366)
+
+```
+class IBackupContainer {
+    // Write operations
+    Future<Reference<IBackupFile>> writeLogFile(beginVersion, endVersion, blockSize);
+    Future<Reference<IBackupFile>> writeRangeFile(version, snapshotFileCount, ...);
+    Future<Reference<IBackupFile>> writeTaggedLogFile(...);
+    
+    // Read/describe
+    Future<Reference<IAsyncFile>> readFile(name);
+    Future<BackupDescription> describeBackup(bool deepScan);
+    Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion);
+    
+    // Lifecycle
+    Future<Void> create();
+    Future<bool> exists();
+    Future<Void> expireData(Version expireEndVersion, bool force);
+    Future<Void> deleteContainer(int* pNumDeleted);
+};
+```
+
+### Implementations
+
+- **BackupContainerLocalDirectory** -- local filesystem
+- **BackupContainerS3BlobStore** -- S3-compatible object storage
+
+### File Format Versions
+
+| Version | Format |
+|---------|--------|
+| `BACKUP_AGENT_MLOG_VERSION = 2001` | Old FileBackupAgent mutation logs |
+| `PARTITIONED_MLOG_VERSION = 4110` | BackupWorker partitioned logs |
+| `RANGE_PARTITIONED_MLOG_VERSION = 5001` | Range-partitioned logs |
+| `BACKUP_AGENT_SNAPSHOT_FILE_VERSION = 1001` | Range file snapshots |
+| `BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION = 1002` | Encrypted snapshots |
+
+### RestorableFileSet (BackupContainer.h:221-234)
+
+```
+struct RestorableFileSet {
+    Version targetVersion;
+    std::vector<RangeFile> ranges;           // snapshot files
+    std::vector<LogFile> logs;               // mutation log files
+    Version continuousBeginVersion;
+    Version continuousEndVersion;
+    std::vector<KeyspaceSnapshotFile> snapshots;
+};
+```
+
+---
+
+## Restore Flow
+
+### Restore Modes
+
+| Mode | Description |
+|------|-------------|
+| `RANGEFILE` | Traditional: read range files + apply log files |
+| `BULKLOAD` | Efficient: SST file ingestion via BulkLoad |
+
+### Traditional Restore
+
+1. Read range files to reconstruct base key-value state at snapshot version
+2. Apply log files from snapshot version to target version
+3. Each log file contains mutations that are replayed in order
+4. Supports key prefix transformation (add/remove prefix during restore)
+
+### RestoreConfig
+
+```
+class RestoreConfig : public KeyBackedTaskConfig {
+    KeyBackedProperty<ERestoreState> stateEnum();
+    KeyBackedProperty<Key> addPrefix(), removePrefix();
+    KeyBackedProperty<bool> onlyApplyMutationLogs();
+    KeyBackedProperty<std::string> bulkDumpJobId();
+    KeyBackedProperty<Version> firstConsistentVersion();
+};
+```
+
+---
+
+## Disaster Recovery (DR) -- [`fdbclient/DatabaseBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseBackupAgent.cpp)
+
+Streams mutations from one cluster to another in near-real-time.
+
+### DatabaseBackupAgent
+
+```
+class DatabaseBackupAgent {
+    static const Key keyAddPrefix, keyRemovePrefix;
+    static const Key keyRangeVersions;
+    static const Key keyCopyStop, keyDatabasesInSync;
+};
+```
+
+### DR Flow
+
+1. Source cluster runs backup agent that reads committed mutations
+2. Mutations streamed to destination cluster in near-real-time
+3. Destination cluster applies mutations with optional prefix transformation
+4. Enables warm standby cluster for disaster recovery
+
+### BackupRangeTaskFunc
+
+- Executes range-based copying for DR
+- Finds shard boundaries, creates sub-tasks for parallelism
+- Uses FlowLock for memory management (`CLIENT_KNOBS->BACKUP_LOCK_BYTES`)
+
+---
+
+## Data Flow
+
+### Backup
+```
+TLog ──peek(logRouterTag)──▶ BackupWorker
+                                │
+                                ▼
+                            Filter by backup key ranges
+                                │
+                                ▼
+                            Buffer mutations
+                                │
+                                ▼
+                            Write to IBackupContainer (log files)
+                                │
+                            + periodic range file snapshots
+```
+
+### Restore
+```
+IBackupContainer ──read range files──▶ Restore Agent
+                                          │
+                                          ▼
+                                      Write base state to empty cluster
+                                          │
+                                          ▼
+                  ──read log files──▶ Replay mutations to target version
+                                          │
+                                          ▼
+                                      Cluster at target version state
+```
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbserver/backupworker/BackupWorker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/backupworker/BackupWorker.cpp) | Server-side mutation pulling and writing |
+| [`fdbclient/include/fdbclient/BackupAgent.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupAgent.h) | BackupAgentBase, FileBackupAgent API |
+| [`fdbclient/FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp) | Backup/restore orchestration, TaskBucket |
+| [`fdbclient/DatabaseBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseBackupAgent.cpp) | DR agent, cross-cluster streaming |
+| [`fdbclient/include/fdbclient/BackupContainer.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupContainer.h) | IBackupContainer, file format definitions |
+| [`fdbclient/BackupContainerFileSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/BackupContainerFileSystem.cpp) | Filesystem-based container implementation |
+| [`fdbbackup/backup.cpp`](https://github.com/apple/foundationdb/blob/main/fdbbackup/backup.cpp) | Backup CLI tool |
diff --git a/design/AI-generated/subsystem_12_simulation_testing.md b/design/AI-generated/subsystem_12_simulation_testing.md
new file mode 100644
index 00000000000..179b63198fb
--- /dev/null
+++ b/design/AI-generated/subsystem_12_simulation_testing.md
@@ -0,0 +1,345 @@
+# Subsystem 12: Simulation & Testing
+
+**[Diagrams](diagram_12_simulation_testing.md)**
+
+**Location:** [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), `fdbserver/tester/`, `tests/`
+**Size:** ~51K workloads + simulator  
+**Role:** Deterministic simulation (Sim2), fault injection (Buggify), workload-based integration tests.
+
+---
+
+## Overview
+
+FDB's most distinctive engineering practice. The entire cluster -- multiple processes, network, disk -- runs deterministically in a single OS thread with virtual time. Combined with aggressive fault injection (Buggify), this finds bugs that would take years to manifest in production.
+
+---
+
+## Sim2 -- Deterministic Simulator -- [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)
+
+### Sim2 Class (line 1025)
+
+```
+class Sim2 final : public ISimulator, public INetworkConnections
+```
+
+Implements `INetwork` (the event loop interface) and `ISimulator` (fault injection).
+
+### Key Properties
+
+- **Deterministic**: Same random seed → identical execution
+- **Virtual time**: No wall-clock dependency; time advances via task scheduling
+- **Single-threaded**: All virtual processes share one OS thread
+- **Full cluster**: Multiple processes with separate globals, endpoints, listeners
+
+### Virtual Event Loop
+
+- `TaskQueue<PromiseTask> taskQueue` -- priority queue of pending tasks
+- Time advances through task completion, not wall-clock
+- `delay(seconds, taskId)` enqueues task at `currentTime + seconds`
+- `timer()` can be up to 0.1s ahead of `now()` for timer jitter
+- Buggified delays added via `FLOW_KNOBS->MAX_BUGGIFIED_DELAY`
+
+---
+
+## ProcessInfo -- `SimulatorProcessInfo.h:42-170`
+
+State per simulated process:
+
+```
+struct ProcessInfo : NonCopyable {
+    std::string name, dataFolder, coordinationFolder;
+    MachineInfo* machine;
+    NetworkAddressList addresses;
+    LocalityData locality;
+    ProcessClass startingClass;
+    
+    bool failed, excluded, cleared, rebooting, failedDisk;
+    double fault_injection_p1, fault_injection_p2;
+    
+    std::vector<flowGlobalType> globals;     // per-process globals
+    INetworkConnections* network;
+    UID uid;
+    ProtocolVersion protocolVersion;
+    std::vector<ProcessInfo*> childs;
+    Promise<KillType> shutdownSignal;
+    
+    bool isReliable();     // no faults injected
+    bool isAvailable();    // !excluded && isReliable()
+};
+```
+
+Each process has its own:
+- Global variables (via `globals` vector)
+- Endpoint registrations
+- Listener map
+- Fault injection parameters
+
+---
+
+## Fault Injection
+
+### Kill Types -- `SimulatorKillType.h`
+
+| KillType | Effect |
+|----------|--------|
+| `KillInstantly` | Immediate failure, process gone |
+| `InjectFaults` | Enable probabilistic fault injection |
+| `FailDisk` | Simulate disk failure |
+| `Reboot` | Machine reboot, keep data |
+| `RebootProcess` | Process reboot, keep data |
+| `RebootAndDelete` | Machine reboot, delete data |
+| `RebootProcessAndDelete` | Process reboot, delete data |
+| `RebootProcessAndSwitch` | Reboot with different cluster file |
+
+### Kill Hierarchy
+
+| Method | Scope |
+|--------|-------|
+| `killProcess(pid, kt)` | Single process |
+| `killMachine(machineId, kt)` | All processes on machine |
+| `killZone(zoneId, kt)` | All machines in zone |
+| `killDataCenter(dcId, kt)` | All machines in datacenter |
+| `killDataHall(hallId, kt)` | All machines in data hall |
+
+`killMachine()` checks replication policies before killing -- may upgrade kill type if processes are protected.
+
+### Network Clogging -- `SimClogging` (sim2.cpp:198-297)
+
+```
+struct SimClogging {
+    std::map<IPAddress, double> clogSendUntil, clogRecvUntil;
+    std::map<pair<IP,IP>, double> clogPairUntil, clogPairLatency;
+    std::map<pair<IP,IP>, double> disconnectPairUntil;
+    
+    double getSendDelay(from, to);
+    double getRecvDelay(from, to);
+    bool disconnected(from, to);
+    void clogPairFor(from, to, seconds);
+    void disconnectPairFor(from, to, seconds);
+};
+```
+
+Latency model:
+- 99.9% of packets: `MIN_NETWORK_LATENCY + FAST_NETWORK_LATENCY`
+- 0.1% tail: `MIN_NETWORK_LATENCY + SLOW_NETWORK_LATENCY`
+- Additional per-pair clogging latency
+- Disconnection: throws `connection_failed()`
+
+### Disk Fault Injection -- `AsyncFileNonDurable.h`
+
+Simulates unreliable disk writes:
+
+```
+class AsyncFileNonDurable {
+    Reference<IAsyncFile> file;           // real underlying file
+    RangeMap<uint64_t, Future<Void>> pendingModifications;
+    enum KillMode { NO_CORRUPTION, DROP_ONLY, FULL_CORRUPTION };
+};
+```
+
+- Delays writes randomly (up to `NON_DURABLE_MAX_WRITE_DELAY`)
+- On `kill()`: drops or corrupts pending writes based on `KillMode`
+- `sync()` makes writes durable
+- Simulates power failure: un-synced writes are lost
+
+### I/O Error Injection
+
+`simulator_should_inject_fault()` (sim2.cpp) -- called during I/O operations:
+- Checks per-process `fault_injection_p1` and `fault_injection_p2`
+- Probabilistically returns true → caller throws `io_timeout`, `io_error`, or `platform_error`
+
+---
+
+## Buggify System -- `flow/include/flow/Buggify.h`
+
+Randomly enables rare code paths throughout the codebase.
+
+### Mechanism
+
+```cpp
+#define BUGGIFY (getGeneralSBVar(__FILE__, __LINE__) && deterministicRandom()->random01() < P_FIRES)
+#define BUGGIFY_WITH_PROB(x) (getGeneralSBVar(__FILE__, __LINE__) && deterministicRandom()->random01() < (x))
+```
+
+Two-level randomization:
+1. **Activation** (once per code location): `P_BUGGIFIED_SECTION_ACTIVATED = 0.25` -- 25% of BUGGIFY sites are active
+2. **Firing** (each execution): `P_BUGGIFIED_SECTION_FIRES = 0.25` -- active sites fire 25% of the time
+
+Decisions cached in `SBVars` map keyed by `__FILE__ + __LINE__` for deterministic replay.
+
+### Variants
+
+| Macro | Scope |
+|-------|-------|
+| `BUGGIFY` | General server code |
+| `BUGGIFY_WITH_PROB(x)` | Custom probability |
+| `CLIENT_BUGGIFY` | Client-only code |
+| `CLIENT_BUGGIFY_WITH_PROB(x)` | Client custom probability |
+
+### Examples in Codebase
+
+```cpp
+if (BUGGIFY) delay(deterministicRandom()->random01());     // random extra delay
+if (BUGGIFY) bufferSize = 1;                                // tiny buffer
+if (BUGGIFY_WITH_PROB(0.001)) throw io_error();            // rare I/O error
+```
+
+---
+
+## CODE_PROBE System -- `flow/include/flow/CodeProbe.h`
+
+Tracks code path coverage during testing:
+
+```cpp
+CODE_PROBE(condition, "description of what happened");
+```
+
+### Implementation
+
+```
+struct ICodeProbe {
+    const char* filePath(), line(), comment(), condition();
+    bool wasHit() const;
+    unsigned hitCount() const;
+    void trace(bool condition) const;
+};
+```
+
+- Static singleton per source location
+- Atomic hit counter
+- Trace event logged on first hit
+- Annotations: `context::Sim2`, `decoration::Rare`, `assert::SimOnly`
+- Used to verify rare code paths are exercised during simulation
+
+---
+
+## Workload System -- [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads)
+
+### TestWorkload Base Class (`workloads.h:65-98`)
+
+```
+struct TestWorkload : NonCopyable, WorkloadContext, ReferenceCounted<TestWorkload> {
+    int phases;  // SETUP | EXECUTION | CHECK | METRICS
+    
+    virtual std::string description() const = 0;
+    virtual Future<Void> setup(Database const& cx);    // optional
+    virtual Future<Void> start(Database const& cx) = 0;  // main logic
+    virtual Future<bool> check(Database const& cx) = 0;  // verify results
+    virtual Future<std::vector<PerfMetric>> getMetrics();
+};
+```
+
+### WorkloadContext (lines 38-50)
+
+- `clientId`, `clientCount` -- for distributed coordination
+- `sharedRandomNumber` -- coordinated randomness across clients
+- Reference to `ServerDBInfo`
+
+### Representative Workloads
+
+| Workload | Purpose |
+|----------|---------|
+| `ApiCorrectness` | Tests get/set/getRange/clear correctness |
+| `ReadWrite` | Mixed read/write load with configurable patterns |
+| `ConsistencyCheck` | Validates data consistency across replicas |
+| `Cycle` | Tests cluster configuration changes |
+| `BackupCorrectness` | Validates backup/restore integrity |
+| `BulkLoad` / `BulkDumping` | Tests bulk operations |
+| `AtomicOps` | Validates atomic operation correctness |
+| `ChangeConfig` | Tests configuration changes under load |
+
+~60+ workload types covering all aspects of the system.
+
+---
+
+## Test Configuration -- `tests/` directory
+
+### TOML Format
+
+```toml
+[[knobs]]
+enable_replica_consistency_check_on_reads = true
+
+[[test]]
+testTitle = 'ApiCorrectnessWithConsistencyCheck'
+clearAfterTest = true
+timeout = 2100
+
+    [[test.workload]]
+    testName = 'ApiCorrectness'
+    numKeys = 5000
+    numGets = 1000
+    numGetRanges = 100
+```
+
+### Parameters
+
+- `testTitle` -- test name
+- `timeout` -- seconds before test times out
+- `clearAfterTest` -- clear database after test
+- Multiple `[[test.workload]]` sections for concurrent workloads
+- `[[knobs]]` sections for server knob overrides
+
+---
+
+## SimulatedCluster -- [`fdbserver/SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp)
+
+### TestConfig (lines 131-429)
+
+```
+class TestConfig : public BasicTestConfig {
+    int minimumReplication, minimumRegions;
+    Optional<int> datacenters, processesPerMachine;
+    std::set<SimulationStorageEngine> storageEngineExcludeTypes;
+    Optional<bool> generateFearless, buggify, faultInjection;
+    bool simHTTPServerEnabled;
+    bool injectTargetedSSRestart, injectSSDelay;
+};
+```
+
+### Cluster Setup
+
+1. Load test configuration from TOML/INI file
+2. Create machines with virtual IP addresses
+3. Create processes with locality data (zone, machine, DC)
+4. Configure replication policies
+5. Start workload execution
+
+---
+
+## Test Execution -- `fdbserver/tester/test.cpp`
+
+### `runWorkload()` (lines 67-187)
+
+Four phases:
+
+1. **Setup** -- call `setup()` on each tester, initialize state
+2. **Execution** -- call `start()` on each tester, run workload logic
+3. **Check** -- call `check()` on each tester, verify results
+4. **Metrics** -- call `getMetrics()`, aggregate performance data
+
+### Distributed Coordination
+
+Each tester receives `WorkloadRequest`:
+- `clientId` -- unique client identifier
+- `clientCount` -- total clients
+- `sharedRandomNumber` -- coordinated RNG seed
+- Results aggregated via `aggregateMetrics()`
+
+---
+
+## Principal Files
+
+| File | Purpose |
+|------|---------|
+| [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp) | Sim2 deterministic simulator implementation |
+| [`fdbrpc/include/fdbrpc/simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h) | ISimulator interface |
+| `fdbrpc/include/fdbrpc/SimulatorProcessInfo.h` | ProcessInfo, MachineInfo |
+| `fdbrpc/include/fdbrpc/AsyncFileNonDurable.h` | Simulated unreliable disk |
+| `flow/include/flow/Buggify.h` | BUGGIFY macros |
+| `flow/include/flow/CodeProbe.h` | CODE_PROBE coverage tracking |
+| [`fdbserver/SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp) | Simulated cluster setup, TestConfig |
+| `fdbserver/tester/include/fdbserver/tester/workloads.h` | TestWorkload base class |
+| `fdbserver/tester/test.cpp` | Test execution orchestration |
+| `fdbserver/workloads/*.cpp` | 60+ workload implementations |
+| `tests/**/*.toml` | Test configuration files |
diff --git a/design/AcAC.md b/design/AcAC.md
index 3e2360e6200..00183100d46 100644
--- a/design/AcAC.md
+++ b/design/AcAC.md
@@ -54,9 +54,9 @@ The output is:
        15624 /root/src/fdbserver/worker.actor.cpp:storageServerRollbackRebooter
         5407 /root/src/fdbserver/worker.actor.cpp:workerServer
         5140 /root/src/fdbserver/worker.actor.cpp:fdbd
-          16 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedFDBDRebooter
-          15 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedMachine
-          10 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulationSetupAndRun
+          16 /root/src/fdbserver/SimulatedCluster.cpp:simulatedFDBDRebooter
+          15 /root/src/fdbserver/SimulatedCluster.cpp:simulatedMachine
+          10 /root/src/fdbserver/SimulatedCluster.cpp:simulationSetupAndRun
 ```
 
 which provides the actor ID, an analog of process ID in the actor context; the path of the source file, which contains the actor; the name of the actor; and a `<ACTIVE>` mark for the current running actor.
@@ -102,15 +102,15 @@ AAAAAAAAAADwo2kloY1WAKDXp1f+BrgAAAAAAAAAAA==
 \ qDHU7CAAAAAAAADsIAAAAAAAAAD/+N1Ch/X3AEwORpyq68Y8AAAAAAAAADwAAAAAAAAAAEqD
 \ W3c/M9kAHCBUzBTlojsAAAAAAAAAOwAAAAAAAAAA+uRVDFla0gA5DXKp8yQbCgAAAAAAAAAK
 \ AAAAAAAAAADwo2kloY1WAKDXp1f+BrgAAAAAAAAAAA=="|bin/acac
-     1304263 /root/src/fdbserver/workloads/FuzzApiCorrectness.actor.cpp:loadAndRun  <ACTIVE>
+     1304263 /root/src/fdbserver/workloads/FuzzApiCorrectness.cpp:loadAndRun  <ACTIVE>
      1269030 /root/src/fdbserver/tester.actor.cpp:runWorkloadAsync
      1254091 /root/src/fdbserver/tester.actor.cpp:testerServerWorkload
         8760 /root/src/fdbserver/tester.actor.cpp:testerServerCore
         8746 /root/src/fdbserver/worker.actor.cpp:workerServer
         8428 /root/src/fdbserver/worker.actor.cpp:fdbd
-          60 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedFDBDRebooter
-          59 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedMachine
-          10 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulationSetupAndRun
+          60 /root/src/fdbserver/SimulatedCluster.cpp:simulatedFDBDRebooter
+          59 /root/src/fdbserver/SimulatedCluster.cpp:simulatedMachine
+          10 /root/src/fdbserver/SimulatedCluster.cpp:simulationSetupAndRun
 ```
 
 The reason for not being able to generate the actor call backtrace inside the debugger directly is that the actor names are internally represented as UUIDs, and the mapping requires external files generated by the actor compiler.
@@ -126,4 +126,3 @@ The reason for not being able to generate the actor call backtrace inside the de
   * All binding tests fail
   * Random failure on the `fdbserver`
 * It is possible for AcAC to report the line number of the source code where the actor is created.
-
diff --git a/design/Commit/commit.sequence b/design/Commit/commit.sequence
index 502a572c222..a03727967d8 100644
--- a/design/Commit/commit.sequence
+++ b/design/Commit/commit.sequence
@@ -10,7 +10,7 @@ end
 participantgroup **CommitProxy** (CommitProxyServer.actor.cpp)
 	participant "commitBatcher" as cB
     participant "commitBatch" as Batch
-    participant "TagPartitionedLogSystem" as TPLS
+    participant "LogSystem" as TPLS
 end
 
 participantgroup **Master** 
diff --git a/design/backup_v2_partitioned_logs.md b/design/backup_v2_partitioned_logs.md
index 3e85e7e3363..1905601d7c7 100644
--- a/design/backup_v2_partitioned_logs.md
+++ b/design/backup_v2_partitioned_logs.md
@@ -70,7 +70,7 @@ A command line tool `fdbconvert` has been written to convert new backup logs int
 * How to start a new type backup: e.g.,
 
   ```
-  fdbbackup start -C fdb.cluster --partitioned-log-experimental -d blob_url
+  fdbbackup start -C fdb.cluster --mutation-log-type partitioned-log-experimental -d blob_url
   ```
 
 ### KPI's and Health
@@ -224,7 +224,7 @@ We strive to keep the operational interface the same as the old backup system. T
 
 By default, backup workers are not enabled in the system. When operators submit a new backup request for the first time, the database performs a configuration change (`backup_worker_enabled:=1`) that enables backup workers.
 
-The operator’s backup request can indicate if an old backup or a new backup is used. This is a command line option (i.e., `--partitioned-log-experimental`) in the `fdbbackup` command. A backup request of the new type is started in the following steps:
+The operator’s backup request can indicate if an old backup or a new backup is used. This is a command line option (i.e., `--mutation-log-type partitioned-log-experimental`) in the `fdbbackup` command. A backup request of the new type is started in the following steps:
 
 1. Operators use `fdbbackup` tool to write the backup range to a system key, i.e., `\xff\x02/backupStarted`.
 2. All backup workers monitor the key `\xff\x02/backupStarted`, see the change, and start logging mutations.
diff --git a/design/cluster_health_metric.md b/design/cluster_health_metric.md
new file mode 100644
index 00000000000..5a15a9f1d4e
--- /dev/null
+++ b/design/cluster_health_metric.md
@@ -0,0 +1,171 @@
+# Cluster Health Metric
+
+`cluster_health::Monitor` is a cluster controller background task that periodically evaluates a fixed set of health factors and emits a `ClusterHealthMetric` trace event.
+
+The monitor is controlled by these server knobs:
+
+| Knob | Default | Meaning |
+| --- | --- | --- |
+| `CLUSTER_HEALTH_METRIC_ENABLE` | `false` | Enables the cluster controller monitor. When disabled, the monitor exits immediately and does not emit `ClusterHealthMetric`. Simulation buggification can set this to `true`. |
+| `CLUSTER_HEALTH_METRIC_POLL_INTERVAL` | `5.0` seconds | Time between monitor evaluations after the first evaluation. |
+| `CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD` | `0.20` | Storage server free-space warning threshold. This is a ratio, not a percentage: `0.20` means 20% free space. |
+| `CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD` | `0.10` | Storage server free-space critical threshold. This should be less than or equal to the storage intervention threshold. |
+| `CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD` | `0.20` | TLog queue-disk free-space warning threshold. This is a ratio, not a percentage: `0.20` means 20% free space. |
+| `CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD` | `0.10` | TLog queue-disk free-space critical threshold. This should be less than or equal to the TLog intervention threshold. |
+| `CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD` | `1.2` | Ratekeeper critical throttling threshold for `TPSLimit / ReleasedTPS`. With the default, ratekeeper is critical when the current TPS limit is less than 120% of recently released TPS. |
+
+Threshold comparisons are strict: a value equal to a threshold does not trigger that threshold's health level.
+
+## Health Levels
+
+Each factor uses the following levels:
+
+- `HEALTHY`: No issue is currently indicated by the factor.
+- `SELF_HEALING`: The cluster is degraded, but automatic recovery or repair is already in progress.
+- `INTERVENTION_REQUIRED`: The cluster is still functioning, but an operator likely needs to act to restore full health.
+- `CRITICAL_INTERVENTION_REQUIRED`: The cluster is functioning, but in a severe state that likely requires immediate operator action.
+- `OUTAGE`: The factor indicates loss of availability or loss of a required safety property.
+- `METRICS_MISSING`: The factor could not evaluate because the required trace-event data was unavailable or malformed.
+
+For aggregation, the implementation assigns an internal score to each level and keeps the lowest-scoring factor as the aggregate result:
+
+- `HEALTHY` = 100
+- `SELF_HEALING` = 80
+- `INTERVENTION_REQUIRED` = 60
+- `CRITICAL_INTERVENTION_REQUIRED` = 40
+- `METRICS_MISSING` = 20
+- `OUTAGE` = 0
+
+These numeric values provide a scalar metric that can be emitted, and leave room for additional intermediate values to be added in the future.
+
+## Factors
+
+The monitor currently evaluates these factors:
+
+### `StorageSpace`
+
+Source event:
+- `"<StorageServerInterface.id()>/StorageMetrics"`
+
+Fields used:
+- `KvstoreBytesAvailable`
+- `KvstoreBytesTotal`
+
+Behavior:
+- Computes the minimum `KvstoreBytesAvailable / KvstoreBytesTotal` ratio across storage servers.
+- Returns `CRITICAL_INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD`.
+- Returns `INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD` and at or above `CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD`.
+- Returns `HEALTHY` otherwise.
+
+### `TLogSpace`
+
+Source event:
+- `"<TLogInterface.id()>/TLogMetrics"`
+
+Fields used:
+- `QueueDiskBytesAvailable`
+- `QueueDiskBytesTotal`
+
+Behavior:
+- Computes the minimum `QueueDiskBytesAvailable / QueueDiskBytesTotal` ratio across tlogs.
+- Returns `CRITICAL_INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD`.
+- Returns `INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD` and at or above `CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD`.
+- Returns `HEALTHY` otherwise.
+
+### `StorageReplication`
+
+Source event:
+- `MovingData`
+
+Fields used:
+- `InQueue`
+- `InFlight`
+- `PriorityTeamUnhealthy`
+- `PriorityTeam2Left`
+- `PriorityTeam1Left`
+- `PriorityTeam0Left`
+
+Behavior:
+- Returns `OUTAGE` if any `PriorityTeam0Left > 0`.
+- Returns `CRITICAL_INTERVENTION_REQUIRED` if any `PriorityTeam1Left > 0` and the configured storage team size is 3.
+- Returns `SELF_HEALING` if data movement is queued or in flight to restore replication.
+- Returns `HEALTHY` otherwise.
+
+### `RecoveryState`
+
+Source:
+- `ServerDBInfo::recoveryState` from the cluster controller
+
+Fields used:
+- none; this factor reads the cluster controller's in-memory `RecoveryState`
+
+Behavior:
+- Returns `OUTAGE` if recovery state is below `RecoveryState::ACCEPTING_COMMITS`.
+- Returns `SELF_HEALING` if recovery is at or above `ACCEPTING_COMMITS` but below `FULLY_RECOVERED`.
+- Returns `HEALTHY` at `FULLY_RECOVERED`.
+
+### `ProcessErrors`
+
+Source event:
+- latest worker error event, fetched with `EventLogRequest()`
+
+Fields used:
+- none are parsed directly; the factor only checks whether a non-empty latest-error trace event exists.
+
+Behavior:
+- Returns `CRITICAL_INTERVENTION_REQUIRED` if any worker reports a non-empty latest error.
+- Returns `HEALTHY` if all non-failed latest-error records are empty and at least one latest-error request succeeded.
+- Returns `METRICS_MISSING` only if every latest-error request fails.
+
+### `RkThrottling`
+
+Source event:
+- `RkUpdate`
+
+Fields used:
+- `ReleasedTPS`: Ratekeeper's smoothed rate of transactions that GRV proxies have recently released from their start-transaction queues. A released transaction has been allowed to get a read version and begin; this is not commit throughput.
+- `TPSLimit`: Ratekeeper's current transaction-per-second limit for normal transaction starts, computed from cluster pressure signals and sent back to GRV proxies to pace future releases. A value of `0` means ratekeeper is not allowing normal transaction starts.
+
+Behavior:
+- Returns `OUTAGE` if any `TPSLimit == 0`.
+- Computes the minimum `TPSLimit / ReleasedTPS` ratio across samples with nonzero `ReleasedTPS`.
+- Ignores samples with `ReleasedTPS == 0` for the ratio calculation to avoid dividing by zero.
+- Returns `CRITICAL_INTERVENTION_REQUIRED` when the minimum ratio is below `CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD`.
+- Returns `HEALTHY` otherwise, including the case where every non-outage sample has `ReleasedTPS == 0`.
+
+For `CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD`, lower ratios are worse:
+
+- `TPSLimit / ReleasedTPS == 2.0` means ratekeeper is allowing twice the recently released transaction rate, so this factor is healthy with the default threshold.
+- `TPSLimit / ReleasedTPS == 1.0` means ratekeeper's current limit is equal to the recently released transaction rate, so this factor is critical with the default threshold of `1.2`.
+- `TPSLimit / ReleasedTPS < 1.0` means ratekeeper's current limit is below the recently released transaction rate, which is also critical.
+
+## Missing Metrics Semantics
+
+Most factors treat missing or malformed inputs as `METRICS_MISSING`.
+
+This can happen when:
+
+- the underlying latest-event RPC fails
+- no relevant worker emitted the requested trace event
+- the trace event exists but does not contain the required fields
+- a field cannot be parsed to the expected type
+
+The implementation filters out empty `TraceEventFields` before interpreting a factor, so workers that never emit a role-specific event do not automatically degrade the whole metric.
+
+## `ClusterHealthMetric` Trace Event Schema
+
+The monitor emits a periodic trace event named `ClusterHealthMetric`.
+
+Fields:
+
+- `FactorStorageSpace`: string enum, one of `Outage`, `CriticalInterventionRequired`, `InterventionRequired`, `SelfHealing`, `MetricsMissing`, `Healthy`
+- `FactorTLogSpace`: same enum
+- `FactorStorageReplication`: same enum
+- `FactorRecoveryState`: same enum
+- `FactorProcessErrors`: same enum
+- `FactorRkThrottling`: same enum
+- `Aggregate`: same enum, computed from the most limiting factor
+- `AggregateValue`: numeric aggregate score, currently one of `0`, `20`, `40`, `60`, `80`, or `100`
+- `LimitingFactor`: factor name string, present when not healthy
+
+The aggregate is determined by taking the lowest-scoring level across all factors.
diff --git a/design/coroutines.md b/design/coroutines.md
index 230408a7144..b35524043ef 100644
--- a/design/coroutines.md
+++ b/design/coroutines.md
@@ -6,6 +6,7 @@
 * [Choose-When](#choose-when)
     * [Execution in when-expressions](#execution-in-when-expressions)
     * [Waiting in When-Blocks](#waiting-in-when-blocks)
+    * [Migrating `choose`-`when`](#migrating-choose-when)
 * [Generators](#generators)
     * [Generators and ranges](#generators-and-ranges)
     * [Eager vs Lazy Execution](#eager-vs-lazy-execution)
@@ -18,6 +19,7 @@
     * [Don't Wait in Error-Handlers](#dont-wait-in-error-handlers)
     * [Make Static Functions Class Members](#make-static-functions-class-members)
     * [Initialization of Locals](#initialization-of-locals)
+* [Internals Reference](fdb-coroutines-internals.md) — deep dive into the coroutine runtime (SAV, Actor, awaiters, cancellation)
 
 ## Introduction
 
@@ -98,9 +100,36 @@ choose {
 }
 ```
 
-Since this is a compiler functionality, we can't use this with C++ coroutines. We could
-keep only this feature around, but only using standard C++ is desirable. So instead, we
-introduce a new `class` called `Choose` to achieve something very similar:
+Since this is a compiler functionality, we can't use this with C++ coroutines. For most
+coroutine conversions, prefer `race(...)` and then branch on the returned `std::variant`.
+This keeps the control flow explicit and usually maps more directly to what the coroutine
+will do after the wait. `Choose` is still available for cases where the old `choose`-`when`
+shape is the clearest fit, or where you need its ordered evaluation behavior described below.
+
+For example, this actor pattern:
+
+```c++
+choose {
+    when(R res = wait(f1)) {
+        return res;
+    }
+    when(wait(timeout(...))) {
+        throw io_timeout();
+    }
+}
+```
+
+should usually become:
+
+```c++
+auto result = co_await race(f1, timeout(...));
+if (result.index() == 0) {
+    co_return std::get<0>(result);
+}
+throw io_timeout();
+```
+
+`Choose` remains useful when you specifically want callback-style handling of the winner:
 
 ```c++
 co_await Choose()
@@ -130,7 +159,7 @@ co_await Choose()
     })
     .When([](){ return foo() }, [](Foo const& f) {
         // do something else
-    }).Run();
+    }).run();
 ```
 
 The implementation of `When` will guarantee that this lambda will only be executed if all previous
@@ -195,7 +224,40 @@ loop {
 }
 ```
 
-However, often using `choose`-`when` (or `Choose().When`) is overkill and other facilities like `quorum` and
+### Migrating `choose`-`when`
+
+When porting actor code, use this rule of thumb:
+
+* Prefer `race(...)` when the `choose` picks one winner and then the code branches on which future completed.
+* Use `Choose()` when you need ordered `when` evaluation, lazy creation of later futures, or the callback style is
+  genuinely clearer than branching on a `std::variant`.
+* Use more specific helpers like `quorum`, `waitForAll`, `waitForAllReady`, or `operator||` when they express the
+  intent better than either `race` or `Choose`.
+
+`race(...)` is usually the best direct replacement for patterns like:
+
+```c++
+choose {
+    when(R res = wait(f1)) {
+        return res;
+    }
+    when(S res = wait(f2)) {
+        return use(res);
+    }
+}
+```
+
+which becomes:
+
+```c++
+auto result = co_await race(f1, f2);
+if (result.index() == 0) {
+    co_return std::get<0>(result);
+}
+co_return use(std::get<1>(result));
+```
+
+However, often using `choose`-`when` is overkill and other facilities like `quorum` and
 `operator||` should be used instead. For example this:
 
 ```c++
@@ -490,6 +552,22 @@ But this one is:
 co_await bar(Uncancellable());
 ```
 
+## NoThrowOnCancel
+
+`NoThrowOnCancel` is a marker for coroutines that should still cancel, but should not run cancellation through an
+`actor_cancelled` exception inside the coroutine:
+
+```c++
+Future<Void> foo(NoThrowOnCancel = {}) {
+    Resource r;
+    co_await something();
+}
+```
+
+When a coroutine with this marker is cancelled, the coroutine frame is destroyed and normal C++ RAII cleanup runs for
+locals in scope. `catch` blocks inside the coroutine do not observe `actor_cancelled` for cancellation. This differs from
+`Uncancellable`, where cancelling the returned future is a no-op and the coroutine continues until completion.
+
 ## Porting `ACTOR`'s to C++ Coroutines
 
 If you have an existing `ACTOR`, you can port it to a C++ coroutine by following these steps:
@@ -500,7 +578,8 @@ If you have an existing `ACTOR`, you can port it to a C++ coroutine by following
 3. Remove all `state` modifiers from local variables.
 4. Replace all `wait(expr)` with `co_await expr`.
 5. Remove all `waitNext(expr)` with `co_await expr`.
-6. Rewrite existing `choose-when` statements using the `Choose` class.
+6. Rewrite existing `choose-when` statements by preferring `race(...)` for first-ready branching; use `Choose`
+   only when you need ordered `when` semantics or callback-style handling.
 
 In addition, the following things should be looked out for:
 
@@ -789,17 +868,26 @@ When a function is converted from `ACTOR` to a coroutine, any forward declaratio
 If you also write `const&` explicitly, the generated code will contain `const& const&`, which is a compile error.
 
 ```c++
-// workloads.actor.h — WRONG: ACTOR + const& = double const&
+// workloads.h — WRONG: ACTOR + const& = double const&
 ACTOR Future<Void> foo(Database const& cx);
 
-// workloads.actor.h — CORRECT: remove ACTOR since foo() is now a coroutine
+// workloads.h — CORRECT: remove ACTOR since foo() is now a coroutine
 Future<Void> foo(Database const& cx);
 ```
 
+### `DESCR` Deprecation
+
+The older TDMetric `DESCR` shorthand is deprecated. When a coroutine conversion touches metric event types, do not add
+new `DESCR(...)`-style declarations. Instead, define an explicit payload type with a
+`...Descriptor` suffix and specialize `Descriptor<T>` with `DescribeType<...>` and `DescribeField<...>` next to it.
+
+This keeps descriptor types unambiguous in coroutine-converted code and matches the old TDMetric pattern in files
+such as `flow/EventTypes.h`, `fdbclient/EventTypes.h`, and the workload metric definitions.
+
 ### File Naming
 
 Converted files should be renamed from `.actor.cpp` to `.cpp` (or `.actor.h` to `.h`) since they no longer need the
-actor compiler. Both `fdbserver` and `flowbench` use `fdb_find_sources()` in their `CMakeLists.txt`, which
+actor compiler. Both `fdbserver` and `flow_bench` use `fdb_find_sources()` in their `CMakeLists.txt`, which
 automatically picks up files by glob, so the rename is usually sufficient without any CMake changes.
 
 ### Conversion Checklist
@@ -811,7 +899,8 @@ automatically picks up files by glob, so the rename is usually sufficient withou
    - **Check**: is the variable inside a block (`if`/`else`/`for`/`try`)? If so, move it to function scope.
 5. Replace `wait(expr)` with `co_await expr`. Replace `waitNext(expr)` with `co_await expr`.
 6. Replace `return expr` with `co_return expr`. Replace `return Void()` with `co_return`.
-7. Rewrite `choose`/`when` using the `Choose` class.
+7. Rewrite `choose`/`when` by preferring `race(...)`; use `Choose` only for patterns that do not map cleanly to
+   `race`.
 8. Simplify: `wait(success(f))` → `co_await f`; `wait(store(v, f))` → `v = co_await f`.
 9. For `const&` parameters: copy to a local before the first `co_await`.
 10. Remove `ACTOR` from any forward declarations of the converted functions in `.actor.h` files.
@@ -931,7 +1020,7 @@ python3 ../contrib/benchmark_comparison.py
 - OVERALL_GEOMEAN calculations for statistical analysis
 
 **Requirements**:
-- Working flowbench binary with both actor and coroutine benchmarks
+- Working flow_bench binary with both actor and coroutine benchmarks
 - Benchmark infrastructure must include: bench_net2, coroutine_net2, bench_delay, coroutine_delay_bench, coroutine_yield_bench, bench_callback, coroutine_callback
 
 **Usage**: Tool automatically runs benchmarks and generates comparison report in the format matching historical coroutine optimization reports.
diff --git a/design/data-distributor-internals.md b/design/data-distributor-internals.md
index 55ec8da3068..c18423a4a7a 100644
--- a/design/data-distributor-internals.md
+++ b/design/data-distributor-internals.md
@@ -88,7 +88,7 @@ There are roughly 4 class of movement priorities
 * Boundary change priority. The movement will change current shard boundaries.
 * Others. Like resuming a in-flight movement.
 
-Each shard movement has a priority associating with the move attempt,  The explanation of each priority knob (`PRIORITY_<XXX>`) is in `ServerKnobs.h`.
+Each shard movement has a priority associating with the move attempt,  The explanation of each priority knob (`PRIORITY_<XXX>`) is in `Knobs.h`.
 
 In `status json` output, please look at field `.data.team_tracker.state` for team priority state.
 
@@ -186,3 +186,807 @@ CPU utilization. This metric is in a positive relationship with “FinishedQueri
   * First find the TraceId fields in `BgDDMountainChopper*`, `BgDDValleyFiller*`, which indicates a relocation is triggered.
   * (Only when enabled) Find the `QueuedRelocation` event with the same `BeginPair` and `EndPair` as the original `TraceId`. This means the relocation request is queued.
   * Find the `RelocateShard` event whose `BeginPair`, `EndPair` field is the same as `TraceId`. This event means the relocation is ongoing.
+
+
+# Alternative View -- AI-generated for FDB 7.3
+
+The following was generated by an AI assistant with access to the FDB source repository, including the documentation above in this file.
+
+## 1. Role in the Cluster
+
+The Data Distributor (DD) is a singleton role recruited by the Cluster Controller. It is
+the only component authorized to mutate the shard-to-server mapping stored in the system
+keyspace. Its responsibilities:
+
+1. **Shard lifecycle management** -- splitting shards that grow too large or too hot, merging
+   shards that shrink below thresholds.
+2. **Team formation** -- building and maintaining server teams (groups of `k` storage servers,
+   where `k` = replication factor) and machine teams that satisfy the replication policy.
+3. **Data movement** -- relocating shards to repair replication (after server failures), to
+   rebalance disk/read/write load, or to carry out operational directives (exclusions, wiggle).
+4. **Audit & bulk operations** -- storage audits and bulk load/dump coordination (7.3 additions).
+
+DD is a single point of coordination -- if it crashes, the Cluster Controller recruits a new
+one which recovers state from the system keyspace. During the gap, no new data movement occurs,
+but existing storage servers continue to serve reads/writes normally.
+
+### Singleton Guarantee: MoveKeysLock
+
+Only one DD instance can modify the shard map at a time, enforced by `MoveKeysLock`:
+
+```
+moveKeysLockOwnerKey  (\xff/moveKeysLock/Owner)  -- which DD owns the lock
+moveKeysLockWriteKey  (\xff/moveKeysLock/Write)   -- which DD is actively mutating
+```
+
+When a new DD starts, it sets its UID as the owner. If during a `moveKeys` operation it
+discovers another DD has taken ownership, it throws `movekeys_conflict` and terminates. This
+prevents split-brain corruption of the shard map.
+
+Source: [`MoveKeys.actor.h:37`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/MoveKeys.actor.h#L37) (`MoveKeysLock`),
+[`DataDistribution.actor.cpp:511`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L511) (`takeMoveKeysLock`).
+
+
+## 2. Top-Level Actor Topology
+
+```
+dataDistribution()                               [DataDistribution.actor.cpp:889]
+  ├─ DataDistributor::init()                     -- recover state from system keyspace
+  ├─ resumeRelocations()                         -- resume in-flight data moves
+  │
+  ├─ DataDistributionTracker::run()              -- shard health monitoring
+  │    ├─ shardTracker (per shard)               -- monitors size/bandwidth/read metrics
+  │    ├─ shardSplitter                          -- splits oversized/overloaded shards
+  │    ├─ shardMerger                            -- merges underutilized neighbors
+  │    └─ readHotDetection                       -- identifies read-hot shards
+  │
+  ├─ DDQueue::run()                              -- relocation scheduling
+  │    ├─ dataDistributionRelocator              -- per-relocation actor
+  │    ├─ BgDDLoadRebalance (MC disk)            -- MountainChopper for disk
+  │    ├─ BgDDLoadRebalance (VF disk)            -- ValleyFiller for disk
+  │    ├─ BgDDLoadRebalance (MC read)            -- MountainChopper for read
+  │    ├─ BgDDLoadRebalance (VF read)            -- ValleyFiller for read
+  │    └─ periodicalRefreshCounter               -- refresh DDQueueServerCounter metrics
+  │
+  ├─ DDTeamCollection::run() [primary]           -- primary DC team management
+  │    ├─ buildTeams                             -- create server/machine teams
+  │    ├─ storageServerTracker (per SS)          -- monitor SS health
+  │    ├─ teamTracker (per team)                 -- monitor team health, emit RelocateShards
+  │    ├─ serverTeamRemover                      -- prune excess server teams
+  │    ├─ machineTeamRemover                     -- prune excess machine teams
+  │    ├─ storageRecruiter                       -- recruit new storage servers
+  │    ├─ monitorPerpetualStorageWiggle          -- perpetual wiggle orchestration
+  │    │    ├─ perpetualStorageWiggleIterator
+  │    │    └─ perpetualStorageWiggler
+  │    ├─ trackExcludedServers                   -- process exclusion list
+  │    └─ monitorHealthyTeams                    -- track healthy team count
+  │
+  └─ DDTeamCollection::run() [remote]            -- remote DC (if usableRegions > 1)
+       └─ (same actors as primary)
+```
+
+Source: [`dataDistribution()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L889).
+
+
+## 3. Core Data Structures
+
+### 3.1 Storage Server Info (`TCServerInfo`)
+
+Defined in [`TCInfo.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/TCInfo.h). One per storage server tracked by DD. Contains:
+- **Locality**: processID (unique to ip:port), zoneId (rack), dcId (datacenter)
+- **Teams**: list of `TCTeamInfo` references this server belongs to
+- **Metrics**: disk usage, CPU, storage queue length
+- **Status**: [`ServerStatus`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDTeamCollection.h#L139) in `DDTeamCollection.h` tracks `isFailed`, `isUndesired`,
+  `isWiggling`, `isWrongConfiguration`
+
+A server is "healthy" if its storage engine matches the configured engine AND it is marked
+as desired by DD. See `ServerStatus::isUnhealthy()` at [`DDTeamCollection.h:152`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDTeamCollection.h#L152).
+
+### 3.2 Machine Info (`TCMachineInfo`)
+
+A "machine" in FDB represents a **rack** -- the assumption is one physical host per rack.
+All servers sharing a `zoneId` locality belong to the same machine. Defined in [`TCInfo.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/TCInfo.h).
+
+### 3.3 Server Teams (`TCTeamInfo`)
+
+A group of `k` servers (where `k` = `storageTeamSize`, typically 3) that replicate the same
+shards. Implements `IDataDistributionTeam`. A team is healthy iff every server is healthy
+AND the team satisfies the replication policy.
+
+### 3.4 Machine Teams (`TCMachineTeamInfo`)
+
+A group of `k` machines. Every server team must map onto a machine team (one server per
+machine in the team). This ensures that server teams span distinct failure domains.
+
+### 3.5 Team Collection (`DDTeamCollection`)
+
+Defined in [`DDTeamCollection.h:205`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDTeamCollection.h#L205). The global view of all servers, server teams, machines,
+and machine teams for one datacenter. Key data members:
+
+```cpp
+std::map<UID, Reference<TCServerInfo>> server_info;        // all servers
+std::vector<Reference<TCTeamInfo>> teams;                  // all server teams
+std::map<Standalone<StringRef>, Reference<TCMachineInfo>> machine_info;
+std::vector<Reference<TCMachineTeamInfo>> machineTeams;
+```
+
+There is one `DDTeamCollection` per DC (primary + optionally remote).
+
+### 3.6 Shards (`DDShardInfo`)
+
+A shard is a contiguous key range assigned to a server team. Defined in
+[`DataDistribution.actor.h:483`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L483):
+
+```cpp
+struct DDShardInfo {
+    Key key;                      // start key of this shard
+    std::vector<UID> primarySrc;  // current source servers (primary DC)
+    std::vector<UID> remoteSrc;   // current source servers (remote DC)
+    std::vector<UID> primaryDest; // destination servers during move
+    std::vector<UID> remoteDest;
+    bool hasDest;                 // true if a move is in progress
+    UID srcId, destId;            // data move IDs
+};
+```
+
+Shard boundaries are stored in the system keyspace:
+- `\xff/keyServers/[start_key]` → source and destination server IDs
+- `\xff/serverKeys/[serverID]/[start_key]` → which shards a server owns
+
+### 3.7 Shard Size Bounds
+
+Shard size is **workload-driven**, not fixed. The maximum shard size scales with database size:
+
+```cpp
+int64_t getMaxShardSize(double dbSizeEstimate) {
+    return min((MIN_SHARD_BYTES + sqrt(max(dbSizeEstimate, 0)) * SHARD_BYTES_PER_SQRT_BYTES)
+               * SHARD_BYTES_RATIO, MAX_SHARD_BYTES);
+}
+```
+
+| DB Size Estimate | Max Shard Size |
+|------------------|----------------|
+| 0                | ~40 MB         |
+| 1 GB             | ~46 MB         |
+| 100 GB           | ~97 MB         |
+| 1 TB             | ~220 MB        |
+| >=6.5 TB         | 500 MB (cap)   |
+
+Source: [`DataDistribution.actor.h:518`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L518) (`ShardSizeBounds`), documented in [`design/hotshard.md`](https://github.com/apple/foundationdb/blob/release-7.3/design/hotshard.md).
+
+### 3.8 RelocateShard
+
+The fundamental unit of work for data movement. Defined in [`DataDistribution.actor.h:147`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L147):
+
+```cpp
+struct RelocateShard {
+    KeyRange keys;
+    int priority;
+    bool cancelled;
+    std::shared_ptr<DataMove> dataMove;  // non-null if restoring a prior move
+    UID dataMoveId;
+    RelocateReason reason;
+    DataMovementReason moveReason;
+    UID traceId;                         // for lifecycle tracking
+};
+```
+
+### 3.9 DDSharedContext
+
+The shared state container connecting all DD sub-components. Defined in [`DDSharedContext.h:31`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDSharedContext.h#L31):
+
+```cpp
+class DDSharedContext {
+    std::unique_ptr<DDEnabledState> ddEnabledState;
+    DataDistributorInterface interface;
+    UID ddId;
+    MoveKeysLock lock;
+    DatabaseConfiguration configuration;
+    Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure;
+    Reference<DataDistributionTracker> tracker;
+    Reference<DDQueue> ddQueue;
+    Reference<DDTeamCollection> primaryTeamCollection, remoteTeamCollection;
+};
+```
+
+
+## 4. Initialization Flow
+
+When a new DD is recruited ([`DataDistribution.actor.cpp:889`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L889)):
+
+1. **Open database connection** -- `openDBOnServer()` with `TaskPriority::DataDistributionLaunch`
+2. **Initialize config watcher** -- `initDDConfigWatch()` establishes baseline
+3. **`DataDistributor::init()`** ([line 931](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L931)) -- reads system keyspace to recover:
+   - Current shard boundaries from `\xff/keyServers/`
+   - Server-shard assignments from `\xff/serverKeys/`
+   - Active storage servers and their interfaces from `\xff/serverList/`
+   - Cluster configuration (replication mode, storage engine, etc.)
+   - Takes ownership of `moveKeysLock`
+4. **Resume relocations** -- [`resumeRelocations()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L806) reads persisted `DataMoveMetaData` to
+   resume any in-flight data moves from the previous DD
+5. **Start sub-components** in parallel:
+   - `DataDistributionTracker::run()` -- begins shard monitoring
+   - `DDQueue::run()` -- begins processing relocation requests
+   - `DDTeamCollection::run()` (primary) -- team management for primary DC
+   - `DDTeamCollection::run()` (remote) -- if `usableRegions > 1`
+   - `pollMoveKeysLock()` -- periodic lock validation
+   - Physical shard monitoring (if enabled)
+
+
+## 5. Data Movement Priority System
+
+Every relocation request has a priority that determines scheduling order. Higher values
+take precedence. The full priority table from [`ServerKnobs.cpp:155-173`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/ServerKnobs.cpp#L155):
+
+| Priority | Value | DataMovementReason                    | Category      |
+|----------|-------|---------------------------------------|---------------|
+| `PRIORITY_RECOVER_MOVE`                | 110  | `RECOVER_MOVE`                 | Recovery      |
+| `PRIORITY_REBALANCE_UNDERUTILIZED_TEAM`| 120  | `REBALANCE_UNDERUTILIZED_TEAM` | Load Balance  |
+| `PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM`| 121 | `REBALANCE_READ_UNDERUTIL_TEAM`| Load Balance  |
+| `PRIORITY_REBALANCE_OVERUTILIZED_TEAM` | 122  | `REBALANCE_OVERUTILIZED_TEAM`  | Load Balance  |
+| `PRIORITY_REBALANCE_READ_OVERUTIL_TEAM`| 123  | `REBALANCE_READ_OVERUTIL_TEAM` | Load Balance  |
+| `PRIORITY_REBALANCE_STORAGE_QUEUE`     | 124  | `REBALANCE_STORAGE_QUEUE`      | Load Balance  |
+| `PRIORITY_TEAM_HEALTHY`                | 140  | `TEAM_HEALTHY`                 | Health        |
+| `PRIORITY_PERPETUAL_STORAGE_WIGGLE`    | 141  | `PERPETUAL_STORAGE_WIGGLE`     | Maintenance   |
+| `PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER`| 150 | `TEAM_CONTAINS_UNDESIRED_SERVER`| Health       |
+| `PRIORITY_TEAM_REDUNDANT`              | 200  | `TEAM_REDUNDANT`               | Health        |
+| `PRIORITY_MERGE_SHARD`                 | 340  | `MERGE_SHARD`                  | Boundary      |
+| `PRIORITY_POPULATE_REGION`             | 600  | `POPULATE_REGION`              | Health        |
+| `PRIORITY_TEAM_UNHEALTHY`             | 700  | `TEAM_UNHEALTHY`               | Health        |
+| `PRIORITY_TEAM_2_LEFT`                 | 709  | `TEAM_2_LEFT`                  | Health        |
+| `PRIORITY_TEAM_1_LEFT`                 | 800  | `TEAM_1_LEFT`                  | Health        |
+| `PRIORITY_TEAM_FAILED`                 | 805  | `TEAM_FAILED`                  | Health        |
+| `PRIORITY_TEAM_0_LEFT`                 | 809  | `TEAM_0_LEFT`                  | Health        |
+| `PRIORITY_SPLIT_SHARD`                 | 950  | `SPLIT_SHARD`                  | Boundary      |
+| `PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD` | 960 | Physical shard enforcement | Boundary |
+
+Priority categories from [`DDRelocationQueue.h:41-86`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L41) (`RelocateData` class):
+- **Health priorities**: repairs to replication factor, team health
+- **Boundary priorities**: shard splits and merges
+- **Load balance priorities**: disk/read/write rebalancing (lowest urgency)
+
+Source: [`DDRelocationQueue.actor.cpp:77-101`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L77) for the complete mapping.
+
+
+## 6. Shard Tracker
+
+The [`DataDistributionTracker`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDShardTracker.h#L54) (defined in `DDShardTracker.h`) monitors every shard's
+metrics and initiates splits, merges, and relocations.
+
+### 6.1 Shard Splitting
+
+Triggered when a shard exceeds size or write bandwidth thresholds:
+- **Size split**: `shard.bytes > maxShardSize` → `PRIORITY_SPLIT_SHARD (950)`
+- **Write split**: `bytesWrittenPerKSecond > SHARD_MAX_BYTES_PER_KSEC (1 GB/s)`
+
+The split process:
+1. Tracker calls `splitStorageMetrics` on the storage servers hosting the shard
+2. Storage servers use their sampled byte/write histograms to compute split points
+3. `getSplitKey()` adds jitter to avoid picking the same boundary repeatedly
+4. Tracker calls `executeShardSplit` to update the shard map and issue relocations
+
+Split constraints:
+- Each resulting piece must have bandwidth < `SHARD_SPLIT_BYTES_PER_KSEC (250 MB/s)`
+- Minimum shard size enforced: `MIN_SHARD_BYTES (10 MB by default)`
+- If the shard is too small to split (e.g., single hot key), no split occurs
+
+Source: [`shardSplitter()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDShardTracker.actor.cpp#L852), [`StorageMetrics.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/StorageMetrics.actor.cpp).
+
+### 6.2 Shard Merging
+
+Triggered when a shard's write bandwidth drops below `SHARD_MIN_BYTES_PER_KSEC (100 MB/s)`
+and its size is small enough that merging with a neighbor stays under the max shard size.
+Priority: `PRIORITY_MERGE_SHARD (340)`.
+
+### 6.3 Read-Hot Shard Detection
+
+When `READ_SAMPLING_ENABLED=true`, storage servers sample read operations. Shards exceeding
+`SHARD_MAX_READ_OPS_PER_KSEC (45k ops/s)` or having high read density ratios are flagged.
+The tracker emits `ReadHotRangeLog` trace events. Read-hot shards trigger **rebalance moves**
+(not splits) -- DD moves the shard to a less-loaded team.
+
+Source: [`StorageMetrics.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/StorageMetrics.actor.cpp), [`design/hotshard.md`](https://github.com/apple/foundationdb/blob/release-7.3/design/hotshard.md).
+
+
+## 7. Relocation Queue (DDQueue)
+
+[`DDQueue`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L113) (defined in `DDRelocationQueue.h`) is the scheduler for all data movements.
+It receives `RelocateShard` requests from the tracker and team collection, prioritizes them,
+selects destination teams, and launches the actual data transfers.
+
+### 7.1 Queue Data Structures
+
+```cpp
+KeyRangeMap<RelocateData> queueMap;                               // all queued relocations
+std::set<RelocateData, greater> fetchingSourcesQueue;             // fetching source info
+std::set<RelocateData, greater> fetchKeysComplete;                // ready to launch
+std::map<UID, std::set<RelocateData, greater>> queue;             // per-server queue
+KeyRangeMap<RelocateData> inFlight;                               // actively moving
+std::map<UID, Busyness> busymap;                                  // source server busyness
+std::map<UID, Busyness> destBusymap;                              // dest server busyness
+std::map<int, int> priority_relocations;                          // count by priority
+```
+
+Source: [`DDRelocationQueue.h:113-307`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L113).
+
+### 7.2 Throttling
+
+The [`Busyness`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L86) struct tracks per-server load across 10 priority
+buckets. Before launching a relocation, DDQueue checks:
+- Source server busyness: `RELOCATION_PARALLELISM_PER_SOURCE_SERVER`
+- Dest server busyness: `RELOCATION_PARALLELISM_PER_DEST_SERVER`
+- Total move keys parallelism: `DD_MOVE_KEYS_PARALLELISM`
+
+Inflight penalties increase apparent busyness based on relocation health status:
+- `INFLIGHT_PENALTY_HEALTHY`, `INFLIGHT_PENALTY_REDUNDANT`, `INFLIGHT_PENALTY_UNHEALTHY`,
+  `INFLIGHT_PENALTY_ONE_LEFT`
+
+### 7.3 Relocation Lifecycle
+
+Each relocation goes through:
+1. **Queued** → `queueRelocation()` -- overlapping in-flight moves are cancelled
+2. **Fetch sources** → `fetchSourceServersComplete` -- identify current shard owners
+3. **Team selection** → `getSrcDestTeams()` → `GetTeamRequest` to `DDTeamCollection`
+4. **Launch** → [`dataDistributionRelocator()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L1302)
+5. **Move keys** → `moveKeys()` in [`MoveKeys.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3180) -- the actual data transfer:
+   - Add destination as new shard owner (private mutation to `keyServers`/`serverKeys`)
+   - Destination SS reads data from sources via FDB transactions
+   - Remove source from shard ownership
+   - Update `ShardsAffectedByTeamFailure`
+6. **Complete** → `relocationComplete` stream
+
+
+## 8. Load Rebalancing: MountainChopper & ValleyFiller
+
+DD continuously rebalances load using two complementary algorithms implemented in a single
+actor, [`BgDDLoadRebalance`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L2281):
+
+### MountainChopper (Overutilized → Underutilized)
+
+- **Source selection**: `TeamSelect::WANT_TRUE_BEST` -- picks the most loaded team
+- **Dest selection**: `TeamSelect::ANY` with `PreferLowerDiskUtil::True`
+- **DataMovementReason**: `REBALANCE_OVERUTILIZED_TEAM` (disk) or `REBALANCE_READ_OVERUTIL_TEAM` (read)
+- For read rebalancing, calls `rebalanceReadLoad()` which selects top-K shards by read density
+
+### ValleyFiller (Fill Underutilized Teams)
+
+- **Source selection**: `TeamSelect::ANY` -- picks a random loaded team
+- **Dest selection**: `TeamSelect::WANT_TRUE_BEST` with `PreferLowerDiskUtil::True` -- picks
+  the least loaded team
+- **DataMovementReason**: `REBALANCE_UNDERUTILIZED_TEAM` (disk) or `REBALANCE_READ_UNDERUTIL_TEAM` (read)
+
+### Rebalancing Controls
+
+- Polling interval: `BG_REBALANCE_POLLING_INTERVAL` (adaptive, backed off on repeated no-ops)
+- Parallelism cap: `DD_REBALANCE_PARALLELISM` -- max concurrent rebalance moves per priority
+- Can be disabled at runtime via special database key (checked via `getSkipRebalanceValue()`)
+- Read rebalance CPU threshold: `READ_REBALANCE_CPU_THRESHOLD = 0.15` (source team worst CPU)
+- Read rebalance shard selection: top-K by `READ_DENSITY = READ_BYTES_PER_KSEC / SHARD_BYTES`
+  with constraint `LOAD(shard) < (LOAD(src) - LOAD(dest)) * READ_REBALANCE_MAX_SHARD_FRAC`
+
+Source: [`BgDDLoadRebalance`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L2281),
+[`isDataMovementForMountainChopper`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L58),
+[`isDataMovementForValleyFiller`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L69).
+
+
+## 9. Team Building
+
+The team builder ([`buildTeams`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDTeamCollection.actor.cpp#L706) in `DDTeamCollection`) is invoked when:
+- A new server is added
+- A server is removed
+- Zero teams exist
+
+The algorithm:
+1. **Build machine teams** using `addBestMachineTeams()`:
+   - Uses `ReplicationPolicy::selectReplicas()` to find machine combinations
+   - Target count: `DESIRED_TEAMS_PER_SERVER * serverCount / teamSize`
+2. **Build server teams** using `addTeamsBestOf()`:
+   - For each machine team, pick one server from each machine
+   - Prefer the server with the fewest existing teams (`findOneLeastUsedServer`)
+   - Verify the team satisfies the replication policy
+
+### Team Health Monitoring
+
+**[`teamTracker`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDTeamCollection.actor.cpp#L860)** (per team): When a team's health changes, it evaluates:
+- If all servers healthy → no action
+- If a server is undesired → `PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER (150)`
+- If team is redundant → `PRIORITY_TEAM_REDUNDANT (200)`
+- If team is unhealthy → `PRIORITY_TEAM_UNHEALTHY (700)`
+- If only 2 replicas left → `PRIORITY_TEAM_2_LEFT (709)`
+- If only 1 replica left → `PRIORITY_TEAM_1_LEFT (800)`
+- If team failed → `PRIORITY_TEAM_FAILED (805)`
+- If 0 replicas left → `PRIORITY_TEAM_0_LEFT (809)`
+
+The priority escalation drives DDQueue to process the most critical relocations first.
+
+**`storageServerTracker`** (per SS): Monitors each server's health, storage engine type,
+locality validity. When a server fails, it triggers data moves off all teams containing
+that server.
+
+### Team Pruning
+
+- `serverTeamRemover`: periodically removes the server team whose members are on the most
+  teams when total team count > desired. Threshold: `TR_REDUNDANT_TEAM_PERCENTAGE_THRESHOLD`.
+- `machineTeamRemover`: removes machine teams with the most overlapping members when count
+  exceeds desired.
+
+
+## 10. Perpetual Storage Wiggle
+
+The "wiggle" gradually cycles data off each storage server in turn, enabling rolling
+maintenance and ensuring no server holds stale data indefinitely.
+
+### Components
+
+- **[`StorageWiggler`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L563)**: Maintains a min-heap priority queue
+  of servers ordered by `StorageMetadataType` (creation time). Servers with the oldest metadata
+  are wiggled first.
+- **`perpetualStorageWiggleIterator`**: Selects the next server to wiggle and writes its ID to
+  `perpetualStorageWiggleIDPrefix` in the system keyspace.
+- **`perpetualStorageWiggler`**: Watches the wiggle ID key, excludes the target server (causing
+  DD to move data off it), waits for completion, then includes it again and signals the iterator.
+- **`clusterHealthCheckForPerpetualWiggle`**: Pauses wiggling if the cluster becomes unhealthy.
+
+### Configuration
+
+- Enable: `configure perpetual_storage_wiggle=1`
+- Target specific locality: `configure perpetual_storage_wiggle_locality=<KEY>:<VALUE>`
+- Wiggle pause threshold: `DD_STORAGE_WIGGLE_PAUSE_THRESHOLD`
+- Wiggle stuck threshold: `DD_STORAGE_WIGGLE_STUCK_THRESHOLD`
+- Min bytes balance ratio: `PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO`
+- Min SS age before wiggle: `DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC`
+
+### Wiggle States
+
+From `StorageWiggler::State`:
+- `INVALID (0)`: Not initialized
+- `RUN (1)`: Actively wiggling
+- `PAUSE (2)`: Paused due to cluster health
+
+The wiggler state and timestamp are queryable via `GetStorageWigglerStateRequest` RPC.
+
+
+## 11. Physical Shard Collection (Experimental)
+
+[`PhysicalShardCollection`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L277) introduces a higher-level grouping
+where multiple logical shards (key ranges) share the same server team. This reduces metadata
+overhead and enables more efficient bulk moves.
+
+Key concepts:
+- A **physical shard** contains one or more key ranges (logical shards) on the same team
+- Two-step team selection: (1) select a physical shard for the primary team, (2) find the
+  remote team sharing that physical shard
+- Physical shards track aggregate metrics and can be oversized, triggering
+  `PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD (960)` moves
+
+Enabled via: `SHARD_ENCODE_LOCATION_METADATA=true` + `ENABLE_DD_PHYSICAL_SHARD=true`
+
+When DD restarts, all key ranges start in the "anonymous" physical shard and are gradually
+transitioned out as data moves occur.
+
+
+## 12. Multi-Datacenter Architecture
+
+With `usableRegions > 1`, DD maintains two `DDTeamCollection` instances:
+
+1. **Primary Team Collection** -- manages teams in `primaryDcId`
+2. **Remote Team Collection** -- manages teams in `remoteDcIds`
+
+The remote collection only starts after `remoteRecovered()` signals that the remote region
+has completed recovery.
+
+Every shard requires a team in both DCs. The DDQueue coordinates
+relocations across both DCs -- when selecting destination teams, it requests teams from
+both collections and ensures consistency.
+
+`ShardsAffectedByTeamFailure` tracks teams with a `primary` flag to distinguish:
+```cpp
+struct Team {
+    std::vector<UID> servers;
+    bool primary;
+};
+```
+
+
+## 13. Storage Server Recruitment
+
+`storageRecruiter` in `DDTeamCollection` handles recruiting new storage servers:
+- Sends `RecruitStorageRequest` to the Cluster Controller
+- Validates the candidate's locality against the replication policy
+- Optionally pairs with a Testing Storage Server (TSS) via `TSSPairState`
+- Calls `initializeStorage()` to register the new SS
+
+The recruiter is debounced via `restartRecruiting` to avoid thundering herd during
+rapid server changes.
+
+
+## 14. Shard Map Metadata Lifecycle: `keyServers` and `serverKeys`
+
+The shard-to-server mapping is the most critical metadata in the cluster. It is stored as
+two complementary indexes in the system keyspace and flows through every major component.
+
+### 14.1 Schema
+
+**`keyServers`** (`\xff/keyServers/` prefix, defined in [`SystemData.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/include/fdbclient/SystemData.h)):
+
+```
+\xff/keyServers/[begin_key] → encoded(src_UIDs[], dest_UIDs[])
+```
+
+Maps each shard's start key to its current source servers (who own the data) and destination
+servers (who are receiving the data during a move). When no move is in progress, `dest` is
+empty. The encoding supports both UID-based and Tag-based formats, plus optional shard IDs
+when `SHARD_ENCODE_LOCATION_METADATA` is enabled.
+
+Key encoding/decoding functions in [`SystemData.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/SystemData.cpp):
+- `keyServersKey(k)` -- constructs the key
+- `keyServersValue(UIDtoTagMap, src, dest)` -- encodes src/dest with tags
+- `decodeKeyServersValue(serverTagMap, value, src, dest)` -- decodes
+
+**`serverKeys`** (`\xff/serverKeys/` prefix, defined in [`SystemData.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/include/fdbclient/SystemData.h)):
+
+```
+\xff/serverKeys/[serverID]/[begin_key] → serverKeysTrue | serverKeysFalse
+```
+
+The inverse index: for each storage server, records which shard ranges it owns.
+`serverKeysTrue` means the server owns that range; `serverKeysFalse` means it doesn't.
+When `SHARD_ENCODE_LOCATION_METADATA` is enabled, the value also encodes the `DataMoveType`,
+`DataMoveId`, and `DataMovementReason`.
+
+Key functions in [`SystemData.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/SystemData.cpp):
+- `serverKeysPrefixFor(serverID)` -- prefix for all shards on a server
+- `serverKeysKey(serverID, key)` -- specific shard assignment key
+- `decodeServerKeysValue(value, nowAssigned, emptyRange, type, id, reason)` -- decodes
+
+### 14.2 Bootstrap: `seedShardServers`
+
+At initial database creation, the master calls [`seedShardServers()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3202)
+to write the very first shard map:
+
+```cpp
+// 1. Set keyServers: all initial servers own the entire keyspace
+auto ksValue = keyServersValue(serverTags, serverSrcUID);
+tr.set(arena, keyServersKey(allKeys.begin), ksValue);     // \xff/keyServers/\x00
+tr.set(arena, keyServersKey(allKeys.end), ...);            // \xff/keyServers/\xff
+
+// 2. Set serverKeys: each server owns everything
+for (auto& s : servers) {
+    tr.set(arena, serverKeysKey(s.id(), allKeys.begin), serverKeysTrue);
+}
+```
+
+This is committed as the very first transaction (`read_snapshot = 0`), establishing the
+invariant that every key in the database is assigned to a team.
+
+### 14.3 The moveKeys Protocol in Detail
+
+The data transfer protocol is implemented in [`MoveKeys.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp) and consists of three
+coordinated phases. The [`moveKeys()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3180) function orchestrates them:
+
+```cpp
+wait(rawStartMovement(occ, params, tssMapping));   // Phase 1: startMoveKeys
+wait(rawCheckFetchingState(occ, params, ...));      // Phase 2: monitor data copy
+wait(rawFinishMovement(occ, params, tssMapping));   // Phase 3: finishMoveKeys
+```
+
+#### Phase 1: `startMoveKeys` ([`MoveKeys.actor.cpp:924`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L924))
+
+This phase sets up the destination servers and tells them to begin fetching data.
+
+1. **Acquire lock**: `checkMoveKeysLock()` verifies this DD still owns the lock
+2. **Read existing shard map**: `krmGetRanges(tr, keyServersPrefix, keys)` fetches all
+   current shard assignments overlapping the move range
+3. **Validate destinations**: confirms each destination server exists in `serverList`;
+   throws `move_to_removed_server` if any are gone
+4. **For each overlapping sub-shard**:
+   - Decode existing `src` and `dest` from `keyServers`
+   - **Write new `keyServers`**: [`krmSetPreviouslyEmptyRange`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1044) sets `dest` to the new destination team while
+     preserving `src`
+   - Track old destinations that need cleanup
+5. **Remove stale destination entries**: for old `dest` servers no longer needed,
+   `removeOldDestinations()` sets their `serverKeys` to `serverKeysFalse`
+6. **Write new `serverKeys` for destinations**: [`krmSetRangeCoalescing`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1082) for each destination server
+7. **Commit transaction**
+
+This transaction may be split into multiple batches if there are many overlapping sub-shards
+(`MOVE_KEYS_KRM_LIMIT` controls batch size). Each batch advances the `begin` cursor.
+
+**Important**: at this point, `keyServers[range]` has both `src` (old team) and `dest`
+(new team). Both teams serve reads for this shard. The commit of this transaction triggers
+the private mutation propagation described in section 14.4.
+
+#### Phase 2: Data Copy (Storage Server Fetch)
+
+After the `startMoveKeys` transaction commits, it propagates as a private mutation through
+the transaction system (see 14.4). The destination storage servers receive the mutation and
+begin `fetchKeys` -- reading the shard data from source servers via standard FDB read
+transactions. DD monitors this via `rawCheckFetchingState()` which polls the destination
+servers until they report the shard is ready.
+
+#### Phase 3: `finishMoveKeys` ([`MoveKeys.actor.cpp:1229`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1229))
+
+Once all destinations have the data, DD atomically swaps ownership:
+
+1. **Read and validate**: reads `keyServers` for the range, decodes `src` and `dest`,
+   verifies `dest` matches expected servers
+2. **Check readiness**: for each destination server, checks if the shard is fetched and ready
+3. **Atomic swap** ([line 1519-1524](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1519)):
+   ```cpp
+   // Update keyServers: src = old_dest, dest = [] (move complete)
+   krmSetRangeCoalescing(&tr, keyServersPrefix, currentKeys, keys,
+                         keyServersValue(UIDtoTagMap, dest));  // dest becomes new src
+
+   // Update serverKeys for ALL involved servers
+   for (server in allServers) {
+       bool destHasServer = (server in dest);
+       krmSetRangeCoalescing(&tr, serverKeysPrefixFor(server), currentKeys,
+                             allKeys, destHasServer ? serverKeysTrue : serverKeysFalse);
+   }
+   ```
+4. **Commit**
+
+After this commit, the old source servers are no longer owners. The private mutation
+propagation tells them to drop the shard data.
+
+**Retry handling**: on transaction conflict, `finishMoveKeys` retries with
+exponential backoff. After every 10 retries ([line 1551](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1551)) it emits a `RelocateShard_FinishMoveKeysRetrying`
+warning; at 20 retries it escalates to `SevWarnAlways`.
+
+### 14.4 Private Mutation Propagation
+
+When `keyServers` or `serverKeys` mutations commit, they are classified as **private
+mutations** and receive special treatment at every layer of the transaction system.
+
+#### At the Commit Proxy ([`ApplyMetadataMutation.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp))
+
+**[`checkSetKeyServersPrefix`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L207)** (line 207): When the proxy sees a mutation to
+`\xff/keyServers/`:
+1. Stores the mutation in the local `txnStateStore` -- this is the proxy's
+   in-memory replica of the shard map
+2. Decodes `src` and `dest` UIDs
+3. Looks up each UID's `Tag` from the `storageCache`
+4. Updates `keyInfo` (the `ServerCacheInfo` map) with the combined tag set
+5. This updated `keyInfo` is used immediately for **routing subsequent mutations** -- any
+   user transaction touching this key range will now be tagged for both `src` and `dest`
+   servers
+
+**[`checkSetServerKeysPrefix`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L258)** (line 258): When the proxy sees a mutation to
+`\xff/serverKeys/`:
+1. Looks up the target server's `Tag` (line 263: [`if (toCommit)`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L263) guard)
+2. **Privatizes** the mutation by prepending `systemKeys.begin` to `param1`
+3. Adds the server's tag to the commit batch: `toCommit->addTag(tag)` ([line 277](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L277))
+4. Writes the privatized mutation
+
+This ensures the `serverKeys` change is streamed only to the specific storage server it
+affects (via the tag-based routing through tLogs).
+
+#### At the tLog
+
+The tLog receives the tagged mutations and stores them in the appropriate tag-specific
+queue. Storage servers pull mutations by their tag, so each SS only sees `serverKeys`
+mutations addressed to it.
+
+#### At the Storage Server ([`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp))
+
+[`applyPrivateData()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp#L10435) processes incoming `serverKeys` mutations in pairs (begin, end):
+
+**First mutation**: Decodes the start key and metadata:
+```cpp
+decodeServerKeysValue(m.param2, nowAssigned, emptyRange, dataMoveType, dataMoveId, dataMoveReason);
+```
+This extracts: whether the server is now assigned this range, whether it's an empty-range
+optimization, the `DataMoveType` (LOGICAL, PHYSICAL, PHYSICAL_BULKLOAD), and the move reason.
+
+**Second mutation**: Forms the key range and applies the change:
+- `setAssignedStatus(data, keys, nowAssigned)` -- updates the SS's in-memory shard map
+- For shard-aware SS: [`changeServerKeysWithPhysicalShards()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp#L8852) -- handles physical shard moves
+- For non-shard-aware SS: [`changeServerKeys()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp#L9803) -- triggers `fetchKeys` actor if newly assigned,
+  or drops data if unassigned
+
+If the SS is **newly assigned** this range (`nowAssigned = true`), `changeServerKeys` spawns
+a `fetchKeys` actor that reads all data from the source servers and writes it into local
+storage. Once complete, the shard transitions from FETCHING to READABLE.
+
+If the SS is **unassigned** from this range (`nowAssigned = false`), the data is removed
+from the SS's versioned data.
+
+### 14.5 Recovery
+
+During cluster recovery, the `txnStateStore` is reconstructed from tLog data before any
+new transactions can proceed. This means `keyServers` and `serverKeys` are recovered to
+their last committed state. The new commit proxies replay all metadata mutations, rebuilding
+their `keyInfo` routing tables. When DD starts on the new cluster controller, `init()` reads
+the recovered `keyServers` and `serverKeys` to reconstruct its in-memory shard map, teams,
+and any in-flight data moves.
+
+### 14.6 Consistency Model
+
+The use of FDB's own transaction system for shard map mutations provides strong guarantees:
+
+1. **Serializability**: all shard map changes are serialized through the resolver, so
+   concurrent moves to overlapping ranges are detected and one is retried
+2. **Atomicity**: `finishMoveKeys` atomically updates both `keyServers` and all affected
+   `serverKeys` entries in a single transaction
+3. **Consistency across proxies**: because metadata mutations update `txnStateStore` on
+   every commit proxy in the same total order, all proxies have an identical view of which
+   servers own which shards
+4. **Durability**: mutations are persisted in tLogs before acknowledgment
+
+The cost of this elegance is coupling: shard map mutations compete with user transactions
+for commit proxy bandwidth and resolver cycles. During heavy data movement, this can
+measurably impact user commit latencies.
+
+### 14.7 Commented-Out Trace Events in moveKeys
+
+Notably, `startMoveKeys` contains several **commented-out** trace events ([lines 1009, 1031, 1054, 1060](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1009)) that would log per-sub-shard details during the move:
+
+```cpp
+// TraceEvent("StartMoveKeysOldRange", relocationIntervalId)
+//     .detail("KeyBegin", rangeIntersectKeys.begin.toString())
+//     .detail("KeyEnd", rangeIntersectKeys.end.toString())
+//     .detail("OldSrc", describe(src))
+//     .detail("OldDest", describe(dest))
+//     .detail("ReadVersion", tr->getReadVersion().get());
+```
+
+These were likely disabled due to volume concerns, but their absence creates an observability
+gap that we address in the recommendations document.
+
+
+## 15. The High-Level moveKeys Orchestration
+
+Wrapping the protocol above, [`moveKeys()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3180) orchestrates
+the three phases:
+
+```cpp
+ACTOR Future<Void> moveKeys(Database occ, MoveKeysParams params) {
+    wait(rawStartMovement(occ, params, tssMapping));       // startMoveKeys
+    state Future<Void> completionSignaller =
+        rawCheckFetchingState(occ, params, tssMapping);    // monitor fetch progress
+    wait(rawFinishMovement(occ, params, tssMapping));      // finishMoveKeys
+    completionSignaller.cancel();
+    if (!params.dataMovementComplete.isSet())
+        params.dataMovementComplete.send(Void());
+    return Void();
+}
+```
+
+The `dataMovementComplete` promise signals the relocation queue that the move has finished,
+allowing it to update `ShardsAffectedByTeamFailure` and complete the relocation lifecycle.
+
+
+## 16. DD RPC Interface
+
+[`DataDistributorInterface`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistributorInterface.h#L30) exposes these RPCs:
+
+| RPC | Purpose |
+|-----|---------|
+| `waitFailure` | Cluster Controller liveness check |
+| `haltDataDistributor` | Graceful shutdown |
+| `distributorSnapReq` | Consistent snapshot coordination |
+| `distributorExclCheckReq` | Safety check before server exclusion |
+| `dataDistributorMetrics` | Query shard metrics |
+| `distributorSplitRange` | External split point insertion |
+| `storageWigglerState` | Query wiggle status |
+| `triggerAudit` | Trigger/cancel storage audit |
+
+
+## 17. Error Handling & Recovery
+
+Normal DD errors that trigger re-recruitment (not cluster-wide failure):
+```cpp
+{error_code_movekeys_conflict, error_code_broken_promise,
+ error_code_data_move_cancelled, error_code_data_move_dest_team_not_found}
+```
+
+Source: [`DataDistribution.actor.cpp:305`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L305).
+
+When DD encounters a `movekeys_conflict` (another DD took the lock), it terminates and the
+Cluster Controller recruits a fresh instance. The new DD re-reads all state from the system
+keyspace, so no data is lost -- only in-flight moves may need to be re-evaluated.
+
+`DDEnabledState` tracks the overall mode:
+- `ENABLED` -- normal operation
+- `SNAPSHOT` -- DD paused for consistent snapshot
+- `BLOB_RESTORE_PREPARING` -- DD paused for blob restore preparation
diff --git a/design/fdb-coroutines-internals.md b/design/fdb-coroutines-internals.md
new file mode 100644
index 00000000000..83f1da78287
--- /dev/null
+++ b/design/fdb-coroutines-internals.md
@@ -0,0 +1,727 @@
+# How C++ Coroutines Work in FoundationDB
+
+*Caveat: work in progress. This mentions code not yet submitted. Accuracy is believed to be good,
+but ongoing work to increase clarity is needed.*
+
+This document explains the coroutine infrastructure in FoundationDB, from the
+C++20 standard machinery down to FDB-specific implementation details.
+
+## C++20 Coroutine Fundamentals
+
+Any function whose body contains `co_await`, `co_return`, or `co_yield` is a
+coroutine. The compiler rewrites it into a state machine. The programmer does
+**not** call the coroutine machinery directly — the compiler generates all the
+calls. The programmer's job is to implement the **policy** types that the
+compiler calls into.
+
+### What the compiler generates
+
+Given:
+
+```cpp
+Future<int> foo(int x) {
+    int y = co_await bar();
+    co_return y + 1;
+}
+```
+
+The compiler emits roughly:
+
+```cpp
+Future<int> foo(int x) {
+    // 1. Look up the promise type via coroutine_traits<Future<int>, int>
+    using promise_type = std::coroutine_traits<Future<int>, int>::promise_type;
+
+    // 2. Allocate the coroutine frame on the heap
+    //    (calls promise_type::operator new if defined)
+    //    The frame holds: the promise object, all local variables,
+    //    bookkeeping for each suspend point, and the function arguments.
+
+    // 3. Construct the promise
+    promise_type promise;
+
+    // 4. Get the return object — this is what the caller receives.
+    //    Called BEFORE the body executes.
+    Future<int> retval = promise.get_return_object();
+
+    // 5. co_await promise.initial_suspend()
+    //    FDB returns suspend_never → body starts immediately.
+
+    try {
+        // === user's body ===
+
+        // co_await bar() expands to:
+        auto&& awaitable = promise.await_transform(bar());
+        if (!awaitable.await_ready()) {
+            awaitable.await_suspend(handle_to_this_coroutine);
+            // <-- coroutine suspends here, control returns to caller -->
+            // <-- later: something calls handle.resume() -->
+        }
+        int y = awaitable.await_resume();
+
+        // co_return y + 1 expands to:
+        promise.return_value(y + 1);
+        goto final_suspend;
+
+    } catch (...) {
+        promise.unhandled_exception();
+    }
+
+final_suspend:
+    // 6. co_await promise.final_suspend()
+    //    The compiler generates the same await_ready / await_suspend /
+    //    await_resume sequence here.
+
+    return retval;  // Already returned to caller at step 4/5
+}
+```
+
+Key insight: `retval` is returned to the caller when the coroutine first
+suspends (or completes without suspending). The caller never sees the
+`co_return` value directly — it goes through `promise.return_value()` into
+whatever result cell the promise manages.
+
+### The coroutine frame
+
+The compiler heap-allocates a "coroutine frame" containing:
+
+- The promise object
+- Copies of all function parameters
+- All local variables (live across any suspend point)
+- Internal state for tracking which suspend point the coroutine is at
+- Awaiter objects for each `co_await` (since they must survive suspension)
+
+FDB overrides `operator new`/`operator delete` on promise types to use its
+`allocateFast`/`freeFast` allocator.
+
+A typical Clang-generated frame layout:
+
+```
++0    void (*__resume)(void*)          // resume function pointer
++8    void (*__destroy)(void*)         // destroy function pointer
++16   promise_type promise             // in FDB, contains the CoroActor / SAV
++??   uint32_t __suspend_index          // which suspend point we're at
++??   bool __initial_await_resume_called
++??   <parameters>                     // copies of function arguments
++??   <locals>                         // variables live across suspend points
++??   <awaiters>                       // one per co_await, alive during suspension
++??   <padding/alignment>
+```
+
+The layout is compiler-generated and not standardized. Everything that is live
+across a suspend point must be stored in the frame. Typical FDB coroutines
+range from ~200 bytes to over 1KB depending on the number of locals and
+`co_await` points.
+
+**Parameter storage caveat:** The frame stores copies of all function
+parameters. For `const&` parameters, the frame stores the *reference*, not a
+copy of the referent. If the caller passed a temporary, it will be destroyed
+before the coroutine resumes — leaving a dangling reference in the frame.
+Always copy `const&` parameters to a local before the first `co_await`.
+
+The compiler splits the original function into three pieces:
+
+1. **Ramp function** — the entry point the caller invokes. Allocates the frame,
+   constructs the promise, calls `get_return_object()` and
+   `initial_suspend()`, starts the body, and returns `retval` to the caller
+   when the coroutine first suspends.
+2. **Resume function** — called via `handle.resume()`. Jumps to the current
+   suspend point (via a switch on the saved index) and continues execution.
+3. **Destroy function** — called via `handle.destroy()`. Jumps to the current
+   suspend point and runs destructors for in-scope locals, then deallocates.
+
+### `coroutine_handle<>`
+
+A single pointer to the coroutine frame — **8 bytes**. The resume and destroy
+function pointers are stored in the frame itself (see layout above), not in
+the handle. Calling `handle.resume()` dereferences the frame pointer to find
+the resume function pointer, then makes an indirect call.
+
+`coroutine_handle<P>` (typed) and `coroutine_handle<void>` (erased) are the
+same size. The typed version adds a `promise()` accessor that returns a
+reference to the promise at a compiler-known offset in the frame. They are
+freely interconvertible.
+
+The compiler passes the handle to `await_suspend`. Key operations:
+
+- `handle.resume()` — resume the coroutine at its current suspend point
+- `handle.destroy()` — destroy the coroutine frame (runs destructors for
+  in-scope locals, deallocates)
+- `handle.done()` — true if the coroutine is at its final suspend point
+
+### `await_suspend` return types
+
+| Return type | Meaning |
+|---|---|
+| `void` | Always suspend |
+| `bool` | `true` → suspend; `false` → don't suspend (resume immediately) |
+| `coroutine_handle<U>` | Symmetric transfer: suspend this coroutine, resume that one |
+
+### `final_suspend` rules
+
+- The expression `co_await promise.final_suspend()` **must not throw**
+  (compiler-enforced). Mark `final_suspend()` and the awaiter's methods
+  `noexcept`.
+- If `await_suspend` returns `void` or `true`: the coroutine is suspended at
+  the final suspend point. The frame stays alive until someone calls
+  `handle.destroy()`. Calling `handle.resume()` on a coroutine suspended at
+  its final suspend point is **undefined behavior**.
+- If `await_suspend` returns `false`: the coroutine was never suspended. The
+  runtime auto-destroys the frame after `await_resume()` completes. This is
+  well-defined.
+
+### `promise_type` required members
+
+| Method | Required? | Purpose |
+|---|---|---|
+| `get_return_object()` | Yes | Create the value returned to the caller |
+| `initial_suspend()` | Yes | Controls whether body starts immediately |
+| `final_suspend()` | Yes, `noexcept` | Controls frame lifetime after body completes |
+| `unhandled_exception()` | Yes | Called if an exception escapes the body |
+| `return_value(T)` or `return_void()` | Exactly one | Stores the co_return value |
+| `await_transform(U)` | Optional | Intercepts every `co_await` expression |
+| `yield_value(T)` | Optional | Enables `co_yield` |
+| `operator new` / `operator delete` | Optional | Custom frame allocation |
+
+---
+
+## FDB's Type Hierarchy
+
+### `Callback<T>` — intrusive linked list node
+
+```
+flow/include/flow/flow.h:468
+```
+
+A doubly-linked list node with virtual `fire(T const&)`, `fire(T&&)`,
+`error(Error)`, and `unwait()`. When a `SAV<T>` has no value yet, it serves as
+the sentinel of a circular list of waiting callbacks.
+
+When a callback is `remove()`d and it was the last one, `unwait()` is called on
+the sentinel (the SAV), which decrements the future refcount.
+
+### `SAV<T>` — Single Assignment Variable
+
+```
+flow/include/flow/flow.h:732
+```
+
+The core result cell. Inherits privately from `Callback<T>` (to serve as the
+list sentinel) and `FastAllocated<SAV<T>>`.
+
+**Two independent refcounts:**
+
+- `promises` — one per `Promise<T>` pointing at it, plus one for an active
+  actor (the producer side)
+- `futures` — one per `Future<T>` pointing at it, plus one if any callbacks
+  are registered
+
+**State machine** via `error_state.code()`:
+
+| Code | Constant | Meaning |
+|---|---|---|
+| -3 | `UNSET_ERROR_CODE` | Not yet assigned. `canBeSet()` is true. |
+| -2 | `NEVER_ERROR_CODE` | Will never complete (set to `Never`). |
+| -1 | `SET_ERROR_CODE` | Successfully set with a value. |
+| > 0 | (error code) | Set with an error. |
+
+**Lifecycle rules:**
+
+- `delPromiseRef()`: when the last promise ref is released (promises is 1) and
+  the SAV is still unset, sends `broken_promise()` to any waiting callbacks.
+  Then sets promises to 0. If futures is also 0, calls `destroy()`.
+- `delFutureRef()`: if futures drops to 0 and promises > 0, calls `cancel()`
+  (this is how dropping the last Future cancels an actor). If promises is also
+  0, calls `destroy()`.
+- `destroy()` and `cancel()` are virtual — subclasses override them.
+
+**Callback management:** `addCallbackAndDelFutureRef(cb)` transfers a future
+reference into the callback list. If this is the first callback, the future
+count stays the same (the "+1 for having callbacks" replaces the future ref).
+If callbacks already exist, the future count is decremented.
+
+### `Actor<ReturnValue>` — actor base class
+
+```
+flow/include/flow/flow.h:1500
+```
+
+Inherits from `SAV<ReturnValue>`. Adds one field:
+
+```cpp
+int8_t actor_wait_state;
+```
+
+| Value | Constant | Meaning |
+|---|---|---|
+| > 0 | `ACTOR_WAIT_STATE_WAITING` (1) | Coroutine is suspended at a `co_await`; a callback is registered on some Future's SAV that will call `handle.resume()` when fired |
+| 0 | `ACTOR_WAIT_STATE_NOT_WAITING` | Coroutine is running (between `co_await` points) or has completed |
+| -1 | `ACTOR_WAIT_STATE_CANCELLED` | Cancellation requested |
+| -2 | `ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK` | Cancellation detected during `await_ready()` before the awaiter registered a callback |
+
+In actor-compiler-generated code, values >1 identify *which* callback group
+fired (the generated state machine uses the value to jump to the right
+continuation). For C++ coroutines this distinction is unnecessary — the
+`coroutine_handle` already encodes the suspend point — so the value is
+always 1.
+
+Constructed with `SAV(1, 1)`: starts with futures=1 and promises=1. The
+futures=1 is consumed by the `Future<T>` returned from `get_return_object()`.
+The promises=1 represents the actor itself as the producer.
+
+`Actor<void>` is a separate specialization with no SAV base — for
+fire-and-forget actors.
+
+### `Promise<T>` and `Future<T>`
+
+```
+flow/include/flow/flow.h:1091, 982
+```
+
+`Promise<T>` wraps a `SAV<T>*`. Constructor creates `SAV(0, 1)` — zero
+futures, one promise. `getFuture()` calls `addFutureRef()` then returns
+`Future<T>(sav)`.
+
+`Future<T>` wraps a `SAV<T>*` — **8 bytes**. Copy constructor calls
+`addFutureRef()`. Destructor calls `delFutureRef()`. The explicit constructor
+`Future(SAV<T>* sav)` does **not** increment the refcount — it assumes
+the count is already correct (used by `get_return_object()`).
+
+---
+
+## FDB's Coroutine Promise: `CoroPromise`
+
+```
+flow/include/flow/CoroutinesImpl.h
+```
+
+### `coroutine_traits` specialization
+
+```
+flow/include/flow/Coroutines.h
+```
+
+```cpp
+template <typename ReturnValue, typename... Args>
+struct coroutine_traits<Future<ReturnValue>, Args...> {
+    using promise_type = coro::CoroPromise<ReturnValue,
+                                           !coro::hasUncancellable<Args...>,
+                                           coro::hasExplicitVoid<Args...>,
+                                           coro::hasNoThrowOnCancel<Args...>>;
+};
+```
+
+The `Args...` includes all function parameter types. Marker types like
+`Uncancellable`, `NoThrowOnCancel`, and `ExplicitVoid` are detected via
+template metaprogramming on the parameter list. They are always defaulted
+parameters (`NoThrowOnCancel = {}`) so callers don't pass them.
+
+### `CoroActor<T, IsCancellable>`
+
+```
+flow/include/flow/CoroutinesImpl.h:124
+```
+
+Inherits from `Actor<T>`. Embedded **inside** the coroutine frame (the promise
+holds it as a member). This means the frame allocation also allocates the
+SAV — a single allocation for both.
+
+Adds a `coroutine_handle<>` and overrides:
+
+- `cancel()`: sets `actor_wait_state = CANCELLED`, then resumes the coroutine
+  if it was waiting. The resumed coroutine sees the cancelled state in
+  `resumeImpl()` and throws `actor_cancelled()`, which unwinds through the
+  coroutine body (running destructors and catch blocks), reaches
+  `unhandled_exception()`, and the error is stored on the SAV.
+- `destroy()`: calls `handle.destroy()` to free the coroutine frame (which
+  also frees the embedded CoroActor, since it's part of the frame).
+
+### `get_return_object()`
+
+```cpp
+ReturnFutureType get_return_object() noexcept {
+    coroActor.handle = handle();
+    return ReturnFutureType(&coroActor);
+}
+```
+
+`ReturnFutureType` is `Future<ReturnValue>` — an 8-byte wrapper around a
+`SAV<T>*` pointer. This constructs a `Future<T>` pointing at the actor's SAV.
+The SAV was initialized with `futures=1` by the `Actor` constructor, and the
+explicit `Future(SAV*)` constructor does not increment the refcount — so the
+returned Future consumes that initial reference.
+
+The compiler calls `get_return_object()` before the body executes. The caller
+receives this Future immediately (when the coroutine first suspends or
+completes without suspending).
+
+### `initial_suspend()`
+
+Returns `suspend_never` — the coroutine body starts executing immediately when
+called. The `Future<T>` has already been returned to the caller via
+`get_return_object()`.
+
+### `return_value()` / `return_void()`
+
+Implemented via the `CoroReturn<T, Promise>` CRTP mixin. Calls
+`actor()->set(value)` which placement-news the value into the SAV's storage
+and sets `error_state = SET_ERROR_CODE`.
+
+### `final_suspend()`
+
+Returns a custom `FinalAwaitable`:
+
+- `await_ready()` returns `false` — always enter `await_suspend`
+- `await_suspend()` calls `finishSendAndDelPromiseRef()` (or the error
+  variant), which fires all waiting callbacks and decrements the promise count.
+  Returns `void` — the coroutine is left suspended. The frame stays alive
+  until the SAV's `destroy()` virtual is called (when both refcounts hit 0),
+  which calls `handle.destroy()`.
+
+### `unhandled_exception()`
+
+Catches the exception, extracts the `Error`, and stores it on the SAV via
+`setError()`. If the exception isn't an `Error`, stores `unknown_error()`.
+
+### `await_transform()`
+
+Intercepts every `co_await` expression and wraps it in an FDB-specific awaiter
+type:
+
+| Awaited type | Awaiter created |
+|---|---|
+| `Future<U>` | `AwaitableFuture<..., false>` |
+| `FutureStream<U>` | `AwaitableFuture<..., true>` |
+| `AsyncResult<U>` | `AwaitableAsyncResult` |
+| `ThreadFutureStream<U>` | `ThreadAwaitableFutureStream` |
+| `coro::FutureIgnore<U>` | `AwaitableFutureIgnore` |
+| `coro::FutureErrorOr<U>` | `AwaitableFutureErrorOr` |
+
+---
+
+## Awaiters: How `co_await` Suspends and Resumes
+
+All three awaiter methods — `await_ready()`, `await_suspend()`, and
+`await_resume()` — are called by the **compiler**, never by FDB code directly.
+The programmer implements them on the awaiter types; the compiler generates
+calls to them in the correct order at each `co_await` expression.
+
+### `AwaitableFuture<PromiseType, ValueType, IsStream>`
+
+The most common awaiter. Used for `co_await someFuture` and
+`co_await someStream`.
+
+Inherits from `Callback<T>` (or `SingleCallback<T>` for streams), which means
+the awaiter itself is the callback node that gets inserted into the Future's
+callback list.
+
+**`await_ready()`**: Returns `true` if the Future is already resolved (skip
+suspension). Also checks if the actor has been cancelled — if so, sets
+`CANCELLED_DURING_READY_CHECK` and returns `true` to proceed to
+`await_resume()` where the cancellation is handled.
+
+**`await_suspend(handle)`**:
+1. Stores the coroutine handle via `pt->setHandle(h)`
+2. Sets `actor_wait_state = ACTOR_WAIT_STATE_WAITING`
+3. Registers this awaiter as a callback on the Future via
+   `addCallbackAndClear(this)` — this transfers the Future's reference to
+   the callback list
+
+**`fire(value)` / `error(err)`** (inherited from Callback): Called when the
+awaited Future is resolved — someone called `promise.send(value)` or
+`promise.sendError(err)`, and `SAV::send()` walked its callback list calling
+`fire()` on each node. This happens synchronously on whatever call stack
+resolved the Future (FDB is cooperative single-threaded — there is no
+scheduler). For streams, `fire()` first stores the value in a local
+`AwaitableFutureStore` (since the stream may yield further values). The
+awaiter's `fire()` then calls `pt->resume()` → `handle.resume()`, which
+resumes the coroutine at its suspend point. From the coroutine's perspective,
+control returns from the `co_await` expression.
+
+**`resumeImpl()`**: Called at the start of `await_resume()`. Checks
+`actor_wait_state`:
+- If cancelled: detaches from the callback list and throws
+  `actor_cancelled()`
+- If still waiting (future wasn't ready yet when we entered await_resume via
+  the non-ready-check path): detaches from callback list, resets wait state
+- Returns whether the future was already ready (affects how the value is
+  retrieved for streams)
+
+**`await_resume()`**: Implemented via the `AwaitableResume` CRTP mixin.
+Calls `resumeImpl()`, then returns the value from the resolved Future.
+Throws if the Future completed with an error.
+
+### The full `co_await` sequence, step by step
+
+```cpp
+Future<Void> example() {
+    Promise<int> p;
+    Future<int> f = p.getFuture();
+
+    // ... elsewhere, someone will call p.send(42)
+
+    int val = co_await f;
+    //        ^^^^^^^^^
+    // 1. Compiler calls promise.await_transform(f)
+    //    → creates AwaitableFuture<..., int, false>{f, &promise}
+    //
+    // 2. Compiler calls awaiter.await_ready()
+    //    → f is not resolved yet → returns false
+    //
+    // 3. Compiler calls awaiter.await_suspend(handle)
+    //    → stores handle in actor
+    //    → sets actor_wait_state = WAITING
+    //    → registers awaiter as callback on f's SAV
+    //    → coroutine suspends, control returns to caller
+    //
+    // ... time passes, someone calls p.send(42) ...
+    //     (The coroutine is suspended. Execution has returned to whoever
+    //     called example(). That caller — or another coroutine that runs
+    //     later on the same thread — eventually calls p.send(42).
+    //     The send fires callbacks synchronously on that call stack.)
+    //
+    // 4. SAV::send() fires all callbacks → awaiter.fire(42)
+    //    → calls promise.resume() → handle.resume()
+    //    → coroutine resumes
+    //
+    // 5. Compiler calls awaiter.await_resume()
+    //    → resumeImpl() checks wait state, detaches callback
+    //    → returns f.get() → 42
+    //
+    // 6. val = 42
+
+    co_return val;
+}
+```
+
+---
+
+## Cancellation
+
+### How cancellation is triggered
+
+Cancellation flows through the SAV refcount system:
+
+1. The caller drops the last `Future<T>` reference to an actor
+2. `Future::~Future()` calls `sav->delFutureRef()`
+3. `delFutureRef()` sees futures hit 0 and promises > 0 → calls `cancel()`
+4. `cancel()` is virtual, dispatched to the actor subclass
+
+Or explicitly: the caller calls `f.cancel()` which calls `sav->cancel()`.
+
+### Normal cancellation (CoroActor)
+
+```cpp
+void cancel() override {
+    auto prev_wait_state = actor_wait_state;
+    actor_wait_state = ACTOR_WAIT_STATE_CANCELLED;
+    if (actorWaitStateIsWaiting(prev_wait_state)) {
+        handle.resume();  // resume the coroutine
+    }
+}
+```
+
+If the coroutine is suspended at a `co_await`, it is resumed. The awaiter's
+`resumeImpl()` sees `CANCELLED` and throws `actor_cancelled()`. The exception
+unwinds through the coroutine body — running destructors, entering catch
+blocks — until it reaches `unhandled_exception()`, which stores the error on
+the SAV.
+
+If the coroutine is not waiting (between co_await points, which shouldn't
+happen in cooperative scheduling), the flag is just set. The next
+`await_ready()` will detect it.
+
+### `Uncancellable` coroutines
+
+`CoroActor<T, false>` — the `cancel()` body is compiled away
+(`if constexpr (IsCancellable)` is false). The coroutine runs to completion
+regardless of whether futures are dropped.
+
+### `NoThrowOnCancel` coroutines
+
+Uses `NoThrowOnCancelCoroActor<T>` instead of `CoroActor`. Key differences:
+
+1. **The actor is heap-allocated separately** from the coroutine frame
+   (via `new ActorType()` in the promise). This is necessary because cancel
+   destroys the frame while the SAV must survive for Future observers.
+
+2. **`cancel()` destroys the frame directly** instead of resuming:
+
+```cpp
+void cancel() override {
+    if (!canBeSet()) return;
+
+    if (cancelHandler) {
+        // Detach the awaiter's callback before destroying the frame
+        cancelHandler->cancelWait();
+    }
+
+    destroyFrame();  // handle.destroy() — RAII destructors run
+
+    if (futures) {
+        sendErrorAndDelPromiseRef(actor_cancelled());
+    } else {
+        promises = 0;
+        delete this;
+    }
+}
+```
+
+3. **`final_suspend` returns `false`** from `await_suspend`. After completing
+   the SAV and clearing the handle, the frame auto-destructs (the coroutine was
+   never suspended at the final suspend point). This is well-defined per the
+   C++ standard: returning `false` means the coroutine resumes, control flows
+   off the end, and the frame is destroyed.
+
+4. **`AwaitCancelHandler`**: A virtual interface that awaiters implement. Each
+   awaiter registers itself via `pt->setCancelHandler(this)` in
+   `await_suspend` and clears itself in `resumeImpl`. When `cancel()` fires,
+   it calls `cancelHandler->cancelWait()` to detach the awaiter's callback
+   from whatever Future it's waiting on, *before* destroying the frame. Without
+   this, the destroyed awaiter would be a dangling node in the Future's
+   callback list.
+
+**Behavioral difference from normal cancellation:**
+
+- RAII destructors run (frame destruction invokes ~locals)
+- `catch` blocks inside the coroutine are **not** entered for cancellation
+- The coroutine never "sees" the cancellation — it simply ceases to exist
+
+---
+
+## `AsyncResult<T>`
+
+```
+flow/include/flow/Coroutines.h, CoroutinesImpl.h
+```
+
+An alternative to `Future<T>` that transfers ownership of the result value
+through `co_await`. Unlike `Future<T>` where `co_await` returns `T const&`,
+`co_await AsyncResult<T>` returns `T` by value (moved out). This avoids copies
+for expensive payloads.
+
+`AsyncResult<T>` is backed by `AsyncResultState<T>` (separate from SAV) with
+its own reference counting. The coroutine promise is `AsyncResultPromise`,
+which manages `producerHandle` and `producerWaitState` directly rather than
+inheriting from Actor.
+
+Cancellation for AsyncResult coroutines goes through
+`AsyncResultState::cancelProducer()`, which mirrors the Actor cancel logic:
+either resumes the producer to throw `actor_cancelled()` or (with
+`noThrowOnCancel`) destroys the producer frame directly.
+
+---
+
+## `choose` / `when` for Coroutines
+
+```
+flow/include/flow/CoroUtils.h
+```
+
+The coroutine equivalent of the actor compiler's `choose { when(...) { ... } }`
+pattern. Two forms:
+
+### `Choose().When(future, handler).When(...).run()`
+
+Callback-based. Creates a `ChooseImplActor` that registers callbacks on all
+futures simultaneously. When the first one fires, it removes all other
+callbacks and runs the handler. Returns `Future<Void>`.
+
+**Limitations:** handlers are synchronous functions — you cannot `co_await`
+inside a `When` handler.
+
+### `co_await race(future1, future2, ...)`
+
+Template-based alternative that supports heterogeneous future types. Returns
+`Future<std::variant<T1, T2, ...>>` where each `Ti` is the value type of the
+corresponding future. When the first future completes, `race` cancels the
+others and constructs the variant with `std::in_place_index<N>` for the
+winning future's position.
+
+Use `result.index()` to determine which future won, and `std::get<N>(result)`
+or `std::visit(...)` to extract the value:
+
+```cpp
+auto result = co_await race(futureInt, futureString);
+if (result.index() == 0) {
+    int val = std::get<0>(result);
+} else {
+    std::string val = std::get<1>(result);
+}
+```
+
+---
+
+## Marker Parameters
+
+FDB uses defaulted function parameters as compile-time markers:
+
+```cpp
+Future<Void> foo(int x, NoThrowOnCancel = {}) {
+    co_await bar();
+    co_return;
+}
+```
+
+The compiler sees the parameter types `(int, NoThrowOnCancel)` and passes them
+through `coroutine_traits<Future<Void>, int, NoThrowOnCancel>`, which selects
+the appropriate promise type.
+
+| Marker | Effect |
+|---|---|
+| (none) | Normal cancellable coroutine |
+| `Uncancellable` | `cancel()` is a no-op; coroutine runs to completion |
+| `NoThrowOnCancel` | `cancel()` destroys the frame directly; no `actor_cancelled` thrown |
+| `ExplicitVoid` | `co_await Future<Void>` produces `Void` instead of `void` |
+
+`Uncancellable` and `NoThrowOnCancel` are mutually exclusive (enforced by
+`static_assert`). Neither is compatible with `AsyncGenerator`.
+
+---
+
+## Lifecycle Summary
+
+### Normal completion
+
+1. Coroutine is called → frame allocated, `get_return_object()` returns
+   `Future<T>` to caller
+2. `initial_suspend()` returns `suspend_never` → body runs immediately
+3. Body executes, suspending/resuming at `co_await` points
+4. `co_return value` → `promise.return_value(value)` stores result in SAV
+5. `final_suspend()` → `await_suspend` fires callbacks via
+   `finishSendAndDelPromiseRef()`, decrements promise count
+6. Frame is either left suspended (normal) or auto-destroyed
+   (NoThrowOnCancel)
+7. When both refcounts hit 0, `destroy()` frees the frame (if still alive)
+   and the SAV
+
+### Cancellation (normal)
+
+1. Last `Future<T>` dropped → `delFutureRef()` → `cancel()` on the actor
+2. `cancel()` sets `actor_wait_state = CANCELLED`, resumes the coroutine
+3. Awaiter's `resumeImpl()` throws `actor_cancelled()`
+4. Exception unwinds through the body (running destructors and catch blocks)
+5. `unhandled_exception()` stores the error on the SAV (sets `error_state`)
+6. `final_suspend()` → `await_suspend` fires waiting callbacks via
+   `finishSendErrorAndDelPromiseRef()` and decrements the promise count
+7. Frame cleanup proceeds as in normal completion
+
+### Cancellation (NoThrowOnCancel)
+
+1. Last `Future<T>` dropped → `delFutureRef()` → `cancel()` on the actor
+2. `cancel()` calls `cancelHandler->cancelWait()` to detach the awaiter
+3. `cancel()` calls `handle.destroy()` — frame is destroyed, RAII runs
+4. `cancel()` sends `actor_cancelled()` to any remaining Futures via SAV
+5. Actor (heap-allocated) survives until all Future references are dropped
+
+### Refcount flow
+
+```
+Actor constructor:     promises=1  futures=1
+get_return_object():   Future<T> takes the futures=1 ref (no increment)
+Caller copies Future:  futures=2
+co_await registers cb: futures stays same (cb replaces future ref)
+Future fires cb:       coroutine resumes
+final_suspend:         promises-- (via finishSendAndDelPromiseRef)
+Caller drops Future:   futures-- → if both 0, destroy()
+```
diff --git a/design/file-level-encryption-backup.md b/design/file-level-encryption-backup.md
new file mode 100644
index 00000000000..265af6c00ea
--- /dev/null
+++ b/design/file-level-encryption-backup.md
@@ -0,0 +1,370 @@
+# File-Level Encryption in FoundationDB Backups
+
+This document describes the design and usage of file-level encryption for
+FoundationDB backups and restores.
+
+---
+
+## 1. Overview
+
+File-level encryption protects backup data files written to local
+directories or object stores (e.g., S3) by encrypting each file with a
+user-supplied key before it is uploaded. Restore requires the same key to
+be available to all participating processes.
+
+**What is encrypted:**
+
+- All **snapshot**, **range**, and **log** files.
+
+**What is *not* encrypted:**
+
+- The metadata files in the `properties/` folder of the backup container,
+  so tools like `fdbbackup describe` and `fdbbackup expire` can work
+  without holding the key.
+- Anything inside the live cluster — mutations, keys, and values are
+  unchanged on the wire and in storage.
+
+**Cipher:** AES-256-GCM. The same key must be supplied to backup, restore,
+modify (when re-encrypting), and decode operations.
+
+---
+
+## 2. Encryption Key
+
+The encryption key is a **32-byte (256-bit)** value in raw binary format —
+the exact key length AES-256 requires.
+
+The key value itself is **never** passed on the command line. The user
+provides the **path** to a file holding the 32 raw bytes via
+`--encryption-key-file <PATH>`, and the CLI reads the file to load the key.
+The file must contain exactly 32 bytes of raw binary data — no headers, no
+encoding, no trailing newline.
+
+For example, a key file can be generated using OpenSSL:
+
+```
+openssl rand 32 > key_file
+```
+
+---
+
+## 3. CLI Arguments
+
+Two command-line arguments drive the encrypted-backup workflow:
+
+### `--encryption-key-file <PATH>`
+
+Path to the file holding the AES-256 key. Encryption is opt-in: a backup
+is encrypted only when this argument is supplied to `fdbbackup start`. The
+same argument must be supplied to `fdbrestore` and `fdbdecode` so they can
+read the ciphertext.
+
+### `--encryption-block-size <BYTES>`
+
+Block size used to encrypt each data file. Optional on `fdbbackup start`,
+defaults to `1048576` (1 MB). The chosen value is recorded in the backup's
+metadata so that restore and decode pick it up automatically — those tools
+do not accept the flag themselves.
+
+Validation rules on `fdbbackup start`:
+
+- Must be a positive integer; non-numeric or non-positive values are
+  rejected with `ERROR: Invalid encryption block size`.
+- Must be combined with `--encryption-key-file`. Supplying it without a
+  key file is rejected with
+  `ERROR: --encryption-block-size option requires --encryption-key-file to be set`.
+
+### Per-command summary
+
+| Command            | `--encryption-key-file` | `--encryption-block-size` | Notes                                                                                  |
+| ------------------ | :---------------------: | :-----------------------: | -------------------------------------------------------------------------------------- |
+| `fdbbackup start`  | required for encryption | optional, default 1 MB    | Starts a new encrypted backup.                                                         |
+| `fdbbackup modify` | conditional             | n/a                       | Required only when the destination URL changes and future files must be re-encrypted. |
+| `fdbrestore start` | required if encrypted   | n/a                       | Block size is read from the backup's metadata file, not the CLI.                       |
+| `fdbdecode`        | required if encrypted   | n/a                       | Decodes encrypted log files with the supplied key.                                     |
+
+---
+
+## 4. Usage
+
+The same workflow applies whether the destination is S3 or a local file
+directory.
+
+### 4.1 Start a backup
+
+`--encryption-block-size` is optional; it defaults to 1 MB (`1048576`)
+when omitted. The example below sets it explicitly to half a megabyte
+(`524288`):
+
+```
+fdbbackup start \
+    -C   $CLUSTER_FILE \
+    -t   test_backup \
+    -w \
+    -d   $BACKUP_URL \
+    -k   '"" \xff' \
+    --log --logdir=$LOG_DIR \
+    --blob-credentials $BLOB_CREDENTIALS_FILE \
+    --knob_blobstore_encryption_type=aws:kms \
+    --encryption-key-file   /root/backup_testing/test_encryption_key_file \
+    --encryption-block-size 524288
+```
+
+### 4.2 Stop a backup
+
+```
+fdbbackup discontinue -C $CLUSTER_FILE -t test_backup
+```
+
+### 4.3 Restore a backup
+
+Restore must use the **same key** that was used at backup time:
+
+```
+fdbrestore start \
+    --dest-cluster-file $CLUSTER_FILE \
+    -t   test_backup \
+    -w \
+    -r   $BACKUP_URL \
+    --log --logdir=$LOG_DIR \
+    --blob-credentials $BLOB_CREDENTIALS_FILE \
+    --knob_blobstore_encryption_type=aws:kms \
+    --encryption-key-file /root/backup_testing/test_encryption_key_file
+```
+
+`fdbrestore` does **not** take `--encryption-block-size`; the block size
+is read from the backup's `encryption_metadata` file.
+
+### 4.4 Modify a backup destination
+
+```
+fdbbackup modify \
+    -C   $CLUSTER_FILE \
+    -t   mybackup \
+    -d   $BACKUP_URL \
+    --encryption-key-file /root/local_testing/key_file \
+    --log --logdir=$LOG_DIR
+```
+
+Caveats:
+
+- If the **same URL** is provided with a **different** key, `modify`
+  rejects the change to avoid mixing differently-encrypted files at the
+  same URL.
+- If a **different URL** is provided, the encryption key may be different,
+  the same, or omitted entirely.
+
+### 4.5 Decode encrypted log files
+
+```
+fdbdecode \
+    -r   $BACKUP_URL \
+    --encryption-key-file /root/local_testing/key_file
+```
+
+`--encryption-key-file` is required only when the backup at `BACKUP_URL`
+is encrypted.
+
+---
+
+## 5. Verifying Encryption Status
+
+### 5.1 `fdbbackup describe`
+
+`fdbbackup describe` reports whether a backup is encrypted and the block
+size used. **No encryption key is required.**
+
+```
+fdbbackup describe \
+    -C   $CLUSTER_FILE \
+    -d   $BACKUP_URL \
+    --blob-credentials $BLOB_CREDENTIALS_FILE
+```
+
+Text output:
+
+```
+URL: ...
+Restorable: true
+Partitioned logs: false
+File-level encryption: true
+Encryption block size: 1048576
+Snapshot: ...
+```
+
+JSON output (`--json`) adds `FileLevelEncryption` and
+`EncryptionBlockSize`:
+
+```json
+{
+  "SchemaVersion": "1.0.0",
+  "URL": "file:///root/local_testing/backup_before/backup-2026-01-06-04-03-08.320838/",
+  "Restorable": true,
+  "Partitioned": false,
+  "FileLevelEncryption": true,
+  "EncryptionBlockSize": 1048576,
+  "Snapshots": [ ... ]
+}
+```
+
+### 5.2 `fdbcli` status JSON
+
+Each backup agent reports an array of encryption key file paths along
+with read success status, so distribution failures are visible at a
+glance:
+
+```json
+"encryption_keys": [
+  { "path": "/root/local_testing/key1",     "success": false },
+  { "path": "/root/local_testing/key_file", "success": true  }
+]
+```
+
+Each backup tag reports its key file path and encryption status:
+
+```json
+"encryption_key_file": "/root/local_testing/key1",
+"file_level_encryption": true
+```
+
+A complete fragment:
+
+```json
+"instances": {
+  "4747c54caca01d682a2780d27b7dc9f6": {
+    "blob_stats": { ... },
+    "configured_workers": 10,
+    "encryption_keys": [
+      { "path": "/root/local_testing/key1",     "success": false },
+      { "path": "/root/local_testing/key_file", "success": true  }
+    ],
+    "id": "4747c54caca01d682a2780d27b7dc9f6"
+  }
+},
+"tags": {
+  "test_backup5": {
+    "current_container": "file:///root/backup_testing/backupfolder/backup-2026-01-21-22-47-47.952405",
+    "current_status": "has been completed",
+    "encryption_key_file": "/root/local_testing/key1",
+    "file_level_encryption": true,
+    ...
+  },
+  "test_backup8": {
+    "current_container": "file:///root/backup_testing/backupfolder/backup-2026-01-21-22-57-08.832311",
+    "current_status": "has been completed",
+    "file_level_encryption": false,
+    ...
+  }
+}
+```
+
+---
+
+## 6. Errors and Conflicting Scenarios
+
+| Scenario                                                  | Error                                                                                                                            |
+| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
+| Restoring with a key when backup is **not** encrypted     | `ERROR: Backup is not encrypted, please remove the encryption key file path.`                                                |
+| Restoring **without** a key when backup **is** encrypted  | `ERROR: Backup is encrypted, please provide the encryption key file path.`                                                   |
+| Restoring with the **wrong** key                          | `ERROR: Failed to read data. Verify that backup and restore encryption keys match (if provided) or the data is corrupted.` |
+| `modify` with same URL and different key                  | `Destination URL matches the existing backup URL for tag '...', but the encryption key file does not match. Fatal Error: Backup error` |
+| `--encryption-block-size` without `--encryption-key-file` | `ERROR: --encryption-block-size option requires --encryption-key-file to be set`                                                 |
+| Invalid `--encryption-block-size` value                   | `ERROR: Invalid encryption block size '<value>'`                                                                                 |
+
+---
+
+## 7. Key Distribution Requirements
+
+The encryption key file **path** is added to the backup container and
+persisted in `BackupConfig`. That same path is then propagated to every
+process that touches the encrypted files:
+
+- Every **`backup_agent`** host participating in the backup.
+- Every **`backup_agent`** host participating in a restore (restore
+  workers read the encrypted files and must decrypt them).
+- The **`fdbrestore`** client process that drives the restore — it opens
+  the backup container directly to read metadata and validate the
+  configuration.
+
+Each of these hosts **must** have the **same** encryption key file
+stored locally **at the exact same path**. Otherwise individual agents or
+the `fdbrestore` process will fail to read or write encrypted files and
+the backup/restore will be unable to make progress.
+
+`fdbcli status json` (Section 5.2) is the operational signal for whether
+the key file is present and readable on every agent.
+
+---
+
+## 8. Design Internals
+
+### 8.1 What "file-level" means
+
+"File-level" refers to the **scope** of encryption: each whole backup
+file (snapshot, range, or log) is the unit that gets encrypted, as
+opposed to encrypting individual mutations, keys/values, or live cluster
+storage. Mutations and the live cluster are unchanged; only the on-disk
+backup artifacts are ciphertext. Filenames, directory layout, and files
+under `properties/` stay plaintext so `fdbbackup describe` and
+`fdbbackup expire` can work without the key.
+
+### 8.2 Block layout inside a file
+
+Inside each file, the data is split into fixed-size blocks (the
+**encryption block size**) and each block is encrypted independently with
+AES-256-GCM using the 32-byte key from `--encryption-key-file`. The
+blocks are an implementation detail of the AES-GCM stream cipher — not a
+change in scope — and they exist so restore can read a small byte range
+without decrypting the entire file.
+
+Per-block IVs are derived deterministically: the first 12 bytes come from
+a hash of the filename, and the last 4 bytes are the block index. This
+gives every block a unique IV while keeping reads seekable. On restore,
+the `encryption_metadata` file is read first to recover the
+enable flag and the block size, so the reader splits each file into the
+exact same blocks that were produced at backup time.
+
+### 8.3 Metadata files
+
+Metadata lives in the `properties/` folder of the backup container and
+stays unencrypted.
+
+| File                       | Contents                            | Purpose                                                                                |
+| -------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------- |
+| `encryption_metadata`      | JSON object (see below)             | Indicates whether the backup is encrypted and the encryption block size used.          |
+| `mutation_log_type`        | `"1"` or `"0"` (1 byte)             | Whether the backup uses partitioned mutation logs.                                     |
+| `log_begin_version`        | Version number as text (10 bytes)   | Earliest mutation-log version present in the backup.                                   |
+| `log_end_version`          | Version number as text (10 bytes)   | Latest mutation-log version present in the backup.                                     |
+
+`encryption_metadata` is a JSON object containing both the on/off flag
+and the block size used to encrypt the data files:
+
+```json
+{
+  "is_encryption_enabled": true,
+  "encryption_block_size": 1048576
+}
+```
+
+- `is_encryption_enabled` — `true` if all snapshot/range/log files are
+  encrypted, `false` otherwise.
+- `encryption_block_size` — block size in bytes used to encrypt the data
+  files, or `0` if the backup is not encrypted. The block size must match
+  what restore/decode tools use for the same backup, so it is persisted
+  with the backup itself.
+
+The `encryption_metadata` metadata file is created immediately when a
+backup starts. The other metadata files are generated only when
+`fdbrestore` or `fdbbackup describe` runs. Those operations scan
+filenames only, not file contents, when reconstructing the on-disk view.
+
+Example container layout:
+
+```
+ls -ltrh properties/
+total 16K
+-rw-r--r-- 1 root root  74 May 5 21:04 encryption_metadata
+-rw-r--r-- 1 root root   1 May 5 21:06 mutation_log_type
+-rw-r--r-- 1 root root  10 May 5 21:41 log_end_version
+-rw-r--r-- 1 root root  10 May 5 21:41 log_begin_version
+```
diff --git a/design/global-tag-throttling.md b/design/global-tag-throttling.md
deleted file mode 100644
index 1ea0db9a354..00000000000
--- a/design/global-tag-throttling.md
+++ /dev/null
@@ -1,131 +0,0 @@
-## Global Tag Throttling
-
-When the `GLOBAL_TAG_THROTTLING` knob is enabled, the ratekeeper will use the [transaction tagging feature](https://apple.github.io/foundationdb/transaction-tagging.html) to throttle tags according to the global tag throttling algorithm. This page describes the implementation of this algorithm.
-
-### Tag Quotas
-The global tag throttler bases throttling decisions on "quotas" provided by clients through the tag quota API. Each tag quota has two components:
-
-* Reserved quota
-* Total quota
-
-The global tag throttler cannot throttle tags to a throughput below the reserved quota, and it cannot allow throughput to exceed the total quota.
-
-### Cost
-Internally, the units for these quotas are bytes. The cost of an operation is rounded up to the nearest page size. The cost of a read operation is computed as:
-
-```
-readCost = ceiling(bytesRead / CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE) * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-```
-
-The cost of a write operation is computed as:
-
-```
-writeCost = CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * ceiling(bytesWritten / CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE) * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-```
-
-Here `bytesWritten` includes cleared bytes. The size of range clears is estimated at commit time.
-
-### Tuple Layer
-Tag quotas are stored inside of the system keyspace (with prefix `\xff/tagQuota/`). They are stored using the tuple layer, in a tuple of form: `(reservedQuota, totalQuota)`. There is currently no custom code in the bindings for manipulating these system keys. However, in any language for which bindings are available, it is possible to use the tuple layer to manipulate tag quotas.
-
-### fdbcli
-The easiest way for an external client to interact with tag quotas is through `fdbcli`. To get the quota (in bytes/second) of a particular tag, run the following command:
-
-```
-fdbcli> quota get <tag> [reserved_throughput|total_throughput]
-```
-
-To set the quota through `fdbcli`, run:
-
-```
-fdbcli> quota set <tag> [reserved_throughput|total_throughput] <bytes_per_second>
-```
-
-To clear a both reserved and total throughput quotas for a tag, run:
-
-```
-fdbcli> quota clear <tag>
-```
-
-### Limit Calculation
-The transaction budget that ratekeeper calculates and distributes to clients (via GRV proxies) for each tag is calculated based on several intermediate rate calculations, outlined in this section.
-
-* Reserved Rate: Based on reserved quota and the average transaction cost, a reserved TPS rate is computed for each tag.
-
-* Desired Rate: Based on total quota and the average transaction cost, a desired TPS rate is computed for each tag.
-
-* Limiting Rate: When a storage server is near saturation, tags contributing notably to the workload on this storage server will receive a limiting TPS rate, computed to relieve the workload on the storage server.
-
-* Target Rate: The target rate is the cluster-wide rate enforced by the global tag throttler. This rate is computed as:
-
-```
-targetTps = max(reservedTps, min(desiredTps, limitingTps));
-```
-
-* Per-Client Rate: While the target rate represents the cluster-wide desired throughput according to the global tag throttler, this budget must be shared across potentially many clients. Therefore, based on observed throughput from various clients, each client will receive an equal budget based on a per-client, per-tag rate computed by the global tag throttler. This rate is in the end what will be sent to clients to enforce throttling.
-
-## Implementation
-
-### Stat collection
-The transaction rates and costs of all transactions must be visible to the global tag throttler. Whenever a client tags a transaction, sampling is performed to determine whether to attach the tag to messages sent to storage servers and commit proxies.
-
-For read operations that are sampled (with probability `CLIENT_KNOBS->READ_TAG_SAMPLE_RATE`), read costs are aggregated on storage servers using the `TransactionTagCounter` class. This class tracks the busyness of the top-k tags affecting the storage server with read load (here `k` is determined by `SERVER_KNOBS->SS_THROTTLE_TAGS_TRACKED`). Storage servers periodically send per-tag read cost statistics to the ratekeeper through `StorageQueuingMetricsReply` messages.
-
-For write operations that are sampled (with probability `COMMIT_SAMPLE_COST`), write costs are aggregated on commit proxies in the `ProxyCommitData::ssTrTagCommitCost` object. Per-storage, per-tag write cost statistics are periodically sent from commit proxies to the ratekeeper through `ReportCommitCostEstimationRequest` messages.
-
-The ratekeeper tracks per-storage, per-tag cost statistics in the `GlobalTagThrottlerImpl::throughput` object.
-
-The ratekeeper must also track the rate of transactions performed with each tag. Each GRV proxy agreggates a per-tag counter of transactions started (without sampling). These are sent to the ratekeeper through `GetRateInfoRequest` messages. The global tag throttler then tracks per tag transaction rates in the `GlobalTagThrottlerImpl::tagStatistics` object.
-
-### Average Cost Calculation
-Quotas are expressed in terms of cost, but because throttling is enforced at the beginning of transactions, budgets need to be calculated in terms of transactions per second. To make this conversion, it is necessary to track the average cost of transactions (per-tag, and per-tag on a particular storage server).
-
-Both cost and transaction counters are exponentially smoothed over time, with knob-configurable smoothing intervals.
-
-### Reserved Rate Calculation
-The global tag throttler periodically reads reserved quotas from the system keyspace. Using these reserved quotas and the average cost of transactions with the given tag, a reserved TPS rate is computed. Read and write rates are aggregated as follows:
-
-```
-reservedTps = max(reservedReadTps, reservedWriteTps);
-```
-
-### Desired Rate Calculation
-Similar to reserved rate calculation, the total quota is read from the system key space. Then, using the average cost of transactions with the given tag, a desired TPS rate is computed. Read and write rates are aggregated as follows:
-
-```
-desiredTps = min(desiredReadTps, desiredWriteTps);
-```
-
-### Limiting Rate Calculation
-In addition to tag busyness statistics, the `StorageQueuingMetricsReply` messages sent from storage servers to the ratekeeper also contain metrics on the health of storage servers. The ratekeeper uses these metrics as part of its calculation of a global transaction rate (independent of tag throttling).
-
-The global tag throttler also makes use of these metrics to compute a "throttling ratio" for each storage server. This throttling ratio is computed in `StorageQueueInfo::getThrottlingRatio`. The global tag throttler uses the throttling ratio for each tracked storage server to compute a "limiting transaction rate" for each combination of storage server and tag.
-
-In the "healthy" case where no metrics are near saturation, the throttling ratio will be an empty `Optional<double>`, indicating that the storage server is not near saturation. If, on the other hand, the metrics indicate approaching saturation, the throttling ratio will be a number between 0 and 2 indicating the ratio of current throughput the storage server can serve. In this case, the global tag throttler looks at the current cost being served by the storage server, multiplies it by the throttling ratio, and computes a limiting cost for the storage server. Among all tags using significant resources on this storage server, this limiting cost is divided up according to the relative total quotas allocated to these tags. Next, a transaction limit is determined for each tag, based on how much the average transaction for the given tag affects the given storage server.
-
-These per-tag, per-storage limiting transaction rates are aggregated to compute per-tag limiting transaction rates:
-
-```
-limitingTps(tag) = min{limitingTps(tag, storage) : all storage servers}
-```
-
-If the throttling ratio is empty for all storage servers affected by a tag, then the per-tag, per-storage limiting TPS rate is also empty. In this case the target rate for this tag is simply the desired rate.
-
-If an individual zone is unhealthy, it may cause the throttling ratio for storage servers in that zone to shoot up. This should not be misinterpreted as a workload issue that requires active throttling. Therefore, the zone with the worst throttling ratios is ignored when computing the limiting transaction rate for a tag (similar to the calculation of the global transaction limit in `Ratekeeper::updateRate`).
-
-### Client Rate Calculation
-The smoothed per-client rate for each tag is tracked within `GlobalTagThrottlerImpl::PerTagStatistics`. Once a target rate has been computed, this is passed to `GlobalTagThrottlerImpl::PerTagStatistics::updateAndGetPerClientRate` which adjusts the per-client rate. The per-client rate is meant to limit the busiest clients, so that at equilibrium, the per-client rate will remain constant and the sum of throughput from all clients will match the target rate.
-
-## Simulation Testing
-The `ThroughputQuota.toml` test provides a simple end-to-end test using the global tag throttler. Quotas are set using the internal tag quota API in the `ThroughputQuota` workload. This is run with the `Cycle` workload, which randomly tags transactions. 
-
-In addition to this end-to-end test, there is a suite of unit tests with the `/GlobalTagThrottler/` prefix. These tests run in a mock environment, with mock storage servers providing simulated storage queue statistics and tag busyness reports. Mock clients simulate workload on these mock storage servers, and get throttling feedback directly from a global tag throttler which is monitoring the mock storage servers.
-
-In each unit test, the `GlobalTagThrottlerTesting::monitor` function is used to periodically check whether or not a desired equilibrium state has been reached. If the desired state is reached and maintained for a sufficient period of time, the test passes. If the unit test is unable to reach this desired equilibrium state before a timeout, the test will fail. Commonly, the desired state is for the global tag throttler to report a client rate sufficiently close to the desired rate specified as an input to the `GlobalTagThrottlerTesting::rateIsNear` function.
-
-## Visibility
-
-### Tracing
-On the ratekeeper, every `SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL` seconds, the ratekeeper will call `GlobalTagThrottler::getClientRates`. At the end of the rate calculation for each tag, a trace event of type `GlobalTagThrottler_GotClientRate` is produced. This trace event reports the relevant inputs that went in to the rate calculation, and can be used for debugging.
-
-On storage servers, every `SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL` seconds, there are `BusyReadTag` events for every tag that has sufficient read cost to be reported to the ratekeeper. Both cost and fractional busyness are reported.
diff --git a/design/mocks3server_chaos_design.md b/design/mocks3server_chaos_design.md
index 25c03ce0f7c..0e41be8cbab 100644
--- a/design/mocks3server_chaos_design.md
+++ b/design/mocks3server_chaos_design.md
@@ -53,7 +53,7 @@ if (BUGGIFY) {
 
 * **HTTP errors**: 429 (throttling), 503 (service unavailable), 500/502 (server errors)
 * **Auth errors**: 401 (unauthorized), 406 (not acceptable) 
-* **S3-specific errors**: InvalidToken, ExpiredToken (matching [`S3BlobStore.actor.cpp`](https://github.com/apple/foundationdb/tree/main/fdbclient/S3BlobStore.actor.cpp#L1241) patterns)
+* **S3-specific errors**: InvalidToken, ExpiredToken (matching [`S3BlobStore.cpp`](https://github.com/apple/foundationdb/tree/main/fdbclient/S3BlobStore.cpp#L1241) patterns)
 * **Connection issues**: Connection drops, timeouts
 * **Data corruption**: Malformed responses, bit flips
 
@@ -119,4 +119,3 @@ wait(startMockS3ServerChaos(listenAddress));
 ```
 
 Chaos behavior is controlled by **S3FaultInjector rates** (0.0-1.0), with **BUGGIFY providing occasional extra chaos** - no master boolean switch.
-
diff --git a/design/recovery-internals.md b/design/recovery-internals.md
index 4b4ac34ebb6..9f77bc60e15 100644
--- a/design/recovery-internals.md
+++ b/design/recovery-internals.md
@@ -71,7 +71,7 @@ This phase locks the coordinated state (cstate) to make sure there is only one C
 Recall that `ServerDBInfo` has CC's interface and is propagated by CC to every process in a cluster. The current running tLogs can use the CC interface in its `ServerDBInfo` to send its interface to CC.
 CC simply waits on receiving the `TLogRejoinRequest` streams: for each tLog’s interface received, the CC compares the interface ID with the tLog ID read from cstate. Once the CC collects enough old tLog interfaces, it will use the interfaces to lock those tLogs.
 The logic of collecting tLogs’ interfaces is implemented in `trackRejoins()` function.
-The logic of locking the tLogs is implemented in `epochEnd()` function in [TagPartitionedLogSystems.actor.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/TagPartitionedLogSystem.actor.cpp).
+The logic of locking the tLogs is implemented in the `epochEnd()` function in [LogSystem.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp).
 
 Once we lock the cstate, we bump the `recoveryCount` by 1 and write the `recoveryCount` to the cstate. Each tLog in a recovery attempt records the `recoveryCount` and monitors the change of the variable. If the `recoveryCount` increases, becoming larger than the recorded value, the tLog will terminate itself. This mechanism makes sure that when multiple recovery attempts happen concurrently, only tLogs in the most recent recovery will be running. tLogs in other recovery attempts can release their memory earlier, reducing the memory pressure during recovery. This is an important memory optimization before shared tLogs, which allows tLogs in different generations to share the same memory, is introduced.
 
@@ -100,7 +100,7 @@ Once the CC gets enough tLogs, it calculates the known committed version (i.e.,
 
 The CC's finds the maximum `knownCommittedVersion` of all the tLogs. This defines the lower bound of what version range of mutations needs to be copied to the new generation. That is, any versions larger than the CC's `knownCommittedVersion` are not guaranteed to persist on all replicas. The CC chooses a *recovery version*, which is the minimum of durable versions on all tLogs of the old generation, and recruits a new set of tLogs that copy all data between `knownCommittedVersion + 1` and `recoveryVersion` from old tLogs. This copy makes sure data within the range has enough replicas to satisfy the replication policy.
 
-Later, the CC will use the recruited tLogs and `recoveryVersion` to create a new `TagPartitionedLogSystem` for the new generation.
+Later, the CC will use the recruited tLogs and `recoveryVersion` to create a new `LogSystem` for the new generation.
 Note that tLogs can continue to rejoin the cluster after the `recoveryVersion` has been calculated but before the log system has been created.
 
 **An example of `knownCommittedVersion` and `recoveryVersion`:**
@@ -117,7 +117,7 @@ Consider an old generation with three TLogs: `A, B, C`. Their durable versions a
 
 * Situation 1: Too many tLogs in the previous generation permanently died, say due to hardware failure. If force recovery is allowed by system administrator, the CC can choose to force recovery, which can cause data loss; otherwise, to unblock the recovery, system administrator has to bring up those died tLogs, for example by copying their files onto new hardware.
 
-* Situation 2: A tLog may die after it reports alive to the CC in the RECRUITING phase. This may cause the `RecoveryVersion` calculated by the CC in this phase to no longer be valid in the next phases (see [getDurableVersion()](https://github.com/apple/foundationdb/blob/cf7c8f41b22612c4a8a632ea6f173d891b00380d/fdbserver/TagPartitionedLogSystem.actor.cpp#L1940)). When this happens, the CC will detect it (that the previous computed recovery version is no longer viable, because the tail of versions are no longer retrievable from current set of live TLogs), terminate the current recovery, and start a new recovery. See https://github.com/apple/foundationdb/blob/cf7c8f41b22612c4a8a632ea6f173d891b00380d/fdbserver/TagPartitionedLogSystem.actor.cpp#L2468-L2525, where changes to TLogs can cause changes to old log system (`outLogSystem->set(logSystem);`), which in turn causes the recovery loop https://github.com/apple/foundationdb/blob/cf7c8f41b22612c4a8a632ea6f173d891b00380d/fdbserver/ClusterRecovery.actor.cpp#L1582-L1610 to abandon progress and restart.
+* Situation 2: A tLog may die after it reports alive to the CC in the RECRUITING phase. This may cause the `RecoveryVersion` calculated by the CC in this phase to no longer be valid in the next phases (see `getDurableVersion()` in [LogSystem.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp)). When this happens, the CC will detect it (that the previous computed recovery version is no longer viable, because the tail of versions are no longer retrievable from current set of live TLogs), terminate the current recovery, and start a new recovery. Changes to TLogs can cause changes to old log system (`outLogSystem->set(logSystem);`), which in turn causes the recovery loop in [ClusterRecovery.actor.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.actor.cpp) to abandon progress and restart.
 
 Once we have a `knownCommittedVersion`, the CC will reconstruct the [transaction state store](https://github.com/apple/foundationdb/blob/main/design/transaction-state-store.md) by peeking the txnStateTag in oldLogSystem.
 Recall that the txnStateStore includes the transaction system’s configuration, such as the assignment of shards to SS and to tLogs and that the txnStateStore was durable on disk in the oldLogSystem.
diff --git a/design/restore_v1.md b/design/restore_v1.md
index 95145597042..0a64b60bf05 100644
--- a/design/restore_v1.md
+++ b/design/restore_v1.md
@@ -106,7 +106,7 @@ In the restore mechanism, we define five core tasks:
 
         2. `RESTORE_DISPATCH_ADDTASK_SIZE` specifies how many works are dispatched by a single transaction, where we do not want this value too large to make the transaction too large to commit successfully. We do not want this value too small to have too much overhead of transactions to dispatch tasks.
 
-    - When `usePartitionedLog` is set, the `StartFullRestoreTaskFunc` spawns `RestoreDispatchPartitionedTaskFunc` instead, as the core of Restore V2. We omit it here since we focus on the Restore V1.
+    - When `mutation-log-type partitioned-log-experimental` is set, the `StartFullRestoreTaskFunc` spawns `RestoreDispatchPartitionedTaskFunc` instead, as the core of Restore V2. We omit it here since we focus on the Restore V1.
 
 3. `RestoreRangeTaskFunc` (aka. snapshot restore task)
 
diff --git a/design/transaction-state-store.md b/design/transaction-state-store.md
index 5cb126bea4b..13a5bed33fc 100644
--- a/design/transaction-state-store.md
+++ b/design/transaction-state-store.md
@@ -114,4 +114,4 @@ during recovery, `txnStateStore` is recovered by reading all data from this vers
 
 * Commit proxies only write metadata mutations in its own transaction batch to TLogs: https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L772-L774 adds mutations to `storeCommits`. Later in `postResolution()`, https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L1162-L1176, only the last one in `storeCommits` are send to TLogs.
 
-* Commit proxies clear the buffered data, and advance the transaction state store's pop version `poppedUpTo`, in `LogSystemDiskQueueAdapter` after TLog push: https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L1283-L1287 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/KeyValueStoreMemory.actor.cpp#L1033-L1034 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/LogSystemDiskQueueAdapter.actor.cpp#L213.
+* Commit proxies clear the buffered data, and advance the transaction state store's pop version `poppedUpTo`, in `LogSystemDiskQueueAdapter` after TLog push: https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L1283-L1287 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/KeyValueStoreMemory.cpp#L1033-L1034 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/LogSystemDiskQueueAdapter.actor.cpp#L213.
diff --git a/design/validating_restored_data_using_one_cluster.md b/design/validating_restored_data_using_one_cluster.md
index 34cac712288..ebb16a379dd 100644
--- a/design/validating_restored_data_using_one_cluster.md
+++ b/design/validating_restored_data_using_one_cluster.md
@@ -83,5 +83,3 @@ Alternative Design Considerations
     * Add new AuditType **ValidateRestore.**
     * New audit actor auditRestoreQ() in StorageServer similar to auditStorageShardReplicaQ()
         * Change the code to read the range appended with the prefix restored_data_prefix. Can compare, update metadata and error mechanism in the same way. [Range](https://github.com/apple/foundationdb/blob/release-7.4/fdbserver/storageserver.actor.cpp#L5671)
-
-
diff --git a/documentation/coro_tutorial/tutorial.cpp b/documentation/coro_tutorial/tutorial.cpp
index 91686bec934..fe52bb1d90d 100644
--- a/documentation/coro_tutorial/tutorial.cpp
+++ b/documentation/coro_tutorial/tutorial.cpp
@@ -24,7 +24,7 @@
 #include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include <functional>
 #include <unordered_map>
@@ -194,8 +194,7 @@ Future<Void> echoServer() {
 			          [&requests](StreamRequest const& req) {
 				          requests.add([](StreamRequest req) -> Future<Void> {
 					          req.reply.setByteLimit(1024);
-					          int i = 0;
-					          for (; i < 100; ++i) {
+					          for (int i = 0; i < 100; ++i) {
 						          co_await req.reply.onReady();
 						          std::cout << "Send " << i << std::endl;
 						          req.reply.send(StreamReply{ i });
@@ -439,7 +438,7 @@ Future<Void> fdbClientStream() {
 			                                        GetRangeLimits());
 			loop {
 				Standalone<RangeResultRef> range = co_await results.getFuture();
-				if (range.size()) {
+				if (!range.empty()) {
 					bytes += range.expectedSize();
 					next = keyAfter(range.back().key);
 				}
@@ -555,17 +554,6 @@ Future<Void> fdbClient() {
 	}
 }
 
-Future<Void> fdbStatusStresser() {
-	Database db = Database::createDatabase(clusterFile, 300);
-	Key statusJson(std::string("\xff\xff/status/json"));
-	loop {
-		co_await runRYWTransaction(db, [&statusJson](ReadYourWritesTransaction* tr) -> Future<Void> {
-			co_await tr->get(statusJson);
-			co_return;
-		});
-	}
-}
-
 AsyncGenerator<Optional<StringRef>> readBlocks(Reference<IAsyncFile> file, int64_t blockSize) {
 	auto sz = co_await file->size();
 	decltype(sz) offset = 0;
@@ -646,9 +634,8 @@ std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
 	{ "fdbClientStream", &fdbClientStream }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientStream
 	{ "fdbClientGetRange", &fdbClientGetRange }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientGetRange
 	{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
-	{ "fdbStatusStresser", &fdbStatusStresser },
 	{ "testReadLines", &testReadLines }
-}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
+};
 
 int main(int argc, char* argv[]) {
 	bool isServer = false;
diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst
index a97fa5900cc..4a807d6f7e8 100644
--- a/documentation/sphinx/source/api-c.rst
+++ b/documentation/sphinx/source/api-c.rst
@@ -870,11 +870,11 @@ Applications must provide error handling and an appropriate retry loop around th
 
 .. function:: FDBFuture* fdb_transaction_get_tag_throttled_duration(FDBTransaction* transaction)
 
-  |future-return0| the time (in seconds) that the transaction was throttled by the tag throttler in the returned future. |future-return1| call :func:`fdb_future_get_double()` to extract the duration, |future-return2|
+  |future-return0| ``0``. This compatibility API used to report proxy-side tag throttling time, but proxy-side tag throttling has been removed. |future-return1| call :func:`fdb_future_get_double()` to extract the duration, |future-return2|
 
 .. function:: FDBFuture* fdb_transaction_get_total_cost(FDBTransaction* transaction)
 
-  |future-return0| the cost of the transaction so far (in bytes) in the returned future, as computed by the tag throttler, and used for tag throttling if throughput quotas are specified. |future-return1| call :func:`fdb_future_get_int64()` to extract the cost, |future-return2|
+  |future-return0| the cost of the transaction so far (in bytes) in the returned future, as computed by the tag throttler. |future-return1| call :func:`fdb_future_get_int64()` to extract the cost, |future-return2|
 
 .. function:: FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* transaction)
 
diff --git a/documentation/sphinx/source/backups.rst b/documentation/sphinx/source/backups.rst
index 77323530ee4..459dee628c1 100644
--- a/documentation/sphinx/source/backups.rst
+++ b/documentation/sphinx/source/backups.rst
@@ -284,7 +284,7 @@ The ``start`` subcommand is used to start a backup.  If there is already a backu
 
 ::
 
-   user@host$ fdbbackup start [-t <TAG>] -d <BACKUP_URL> [-z] [-s <DURATION>] [--partitioned-log-experimental] [-w] [-k '<BEGIN>[ <END>]']...
+   user@host$ fdbbackup start [-t <TAG>] -d <BACKUP_URL> [-z] [-s <DURATION>] [--mutation-log-type partitioned-log-experimental] [-w] [-k '<BEGIN>[ <END>]']...
 
 ``-z``
   Perform the backup continuously rather than terminating once a restorable backup is achieved.  Database mutations within the backup's target key ranges will be continuously written to the backup as well as repeated inconsistent snapshots at the configured snapshot rate.
@@ -295,7 +295,7 @@ The ``start`` subcommand is used to start a backup.  If there is already a backu
 ``--initial-snapshot-interval <DURATION>``  
   Specifies the duration, in seconds, of the first inconsistent snapshot written to the backup.  The default is 0, which means as fast as possible.
 
-``--partitioned-log-experimental``
+``--mutation-log-type partitioned-log-experimental``
   Specifies the backup uses the partitioned mutation logs generated by backup workers. Since FDB version 6.3, this option is experimental and requires using fast restore for restoring the database from the generated files. The default is to use non-partitioned mutation logs generated by backup agents.
 
 ``-w``
diff --git a/documentation/sphinx/source/clang-format.rst b/documentation/sphinx/source/clang-format.rst
new file mode 100644
index 00000000000..52ecbf41ce4
--- /dev/null
+++ b/documentation/sphinx/source/clang-format.rst
@@ -0,0 +1,68 @@
+############
+Clang-Format
+############
+
+``clang-format`` enforces consistent code style across the FoundationDB codebase. It runs as part of CI on every pull request. If your changes introduce formatting violations, the PR build will fail.
+
+Configuration is in the ``.clang-format`` file at the repository root, based on the Mozilla style with FDB-specific adjustments (120 column limit, tab width 4, etc.).
+
+Running clang-format locally
+============================
+
+Format only your changed lines
+-------------------------------
+
+Use ``git clang-format`` to format only the lines you changed, avoiding reformatting untouched code:
+
+.. code-block:: shell
+
+   # Format staged changes (before committing)
+   git clang-format
+
+   # Format changes between your branch and main
+   git clang-format origin/main
+
+   # Preview what would change (dry run)
+   git clang-format --diff origin/main
+
+Format specific files
+---------------------
+
+If ``git clang-format`` complains about unstaged changes, or you want to format an entire file (not just your changed lines), use ``clang-format`` directly:
+
+.. code-block:: shell
+
+   clang-format -i path/to/file.cpp
+
+   # Format multiple files
+   clang-format -i fdbserver/DataDistribution.actor.cpp fdbclient/SystemData.cpp
+
+.. note::
+
+   This reformats the entire file, not just your changes. This may produce a large diff if the file was not previously formatted.
+
+IDE integration
+===============
+
+**VS Code** (with clangd extension):
+
+1. Open Settings (Cmd+,)
+2. Set **Editor: Default Formatter** to ``llvm-vs-code-extensions.vscode-clangd``
+3. Check **Editor: Format On Save**
+4. Set **Editor: Format On Save Mode** to ``file``
+
+Files will be automatically formatted on save using the ``.clang-format`` configuration.
+
+Key style rules
+===============
+
+The ``.clang-format`` configuration enforces:
+
+* **Column limit**: 120 characters
+* **Indentation**: Tabs with width 4
+* **Braces**: Attach style (opening brace on same line)
+* **Arguments**: One per line when they don't fit
+* **Pointer alignment**: Left (``int* p``, not ``int *p``)
+* **Short functions**: Inline functions may be on a single line
+
+See the ``.clang-format`` file in the repository root for the full configuration.
diff --git a/documentation/sphinx/source/clang-tidy.rst b/documentation/sphinx/source/clang-tidy.rst
new file mode 100644
index 00000000000..145a7209bb2
--- /dev/null
+++ b/documentation/sphinx/source/clang-tidy.rst
@@ -0,0 +1,186 @@
+##########
+Clang-Tidy
+##########
+
+``clang-tidy`` is a static analysis tool that detects common programming errors, enforces coding standards, and suggests modern C++ improvements.
+It runs as part of CI on every pull request targeting a branch that has a ``.clang-tidy`` file (currently only ``main``). Findings are reported in the build log but do not currently fail the build. To enforce failures, add ``WarningsAsErrors: '*'`` to ``.clang-tidy``.
+
+This guide explains how to run ``clang-tidy`` locally so you can fix issues before pushing.
+
+What clang-tidy checks
+======================
+
+FoundationDB enables 13 checks configured in the ``.clang-tidy`` file at the repository root. The
+intent is to enable more as we go forward. Here are some example rules:
+
+* **3 Bugprone rules** -- catch potential runtime errors (e.g., ``bugprone-use-after-move``)
+* **4 Modernize rules** -- encourage modern C++ practices (e.g., ``modernize-use-auto``, ``modernize-use-override``)
+* **1 Performance rule** -- avoid unnecessary copies (``performance-for-range-copy``)
+* **5 Readability rules** -- improve code clarity (e.g., ``readability-container-contains``, ``readability-container-size-empty``)
+
+Basic examples of ``clang-tidy`` style and performance improvement changes:
+
+.. code-block:: cpp
+
+   // Use empty() instead of size() == 0
+   -if (str.size() == 0) {
+   +if (str.empty()) {
+
+   // Use contains() instead of count() > 0
+   -if (classPath.count(path) > 0) {
+   +if (classPath.contains(path)) {
+
+   // Avoid unnecessary copies in range-for loops
+   -for (auto state : states)
+   +for (const auto& state : states)
+
+   // Remove unnecessary const on return types
+   -const Key keyServersKey(const KeyRef& k) {
+   +Key keyServersKey(const KeyRef& k) {
+
+clang-tidy vs :doc:`clang-format`
+=================================
+
+Both run in CI but serve different purposes:
+
+.. list-table::
+   :header-rows: 1
+   :widths: 15 40 45
+
+   * - Feature
+     - ``clang-format`` (Code Style)
+     - ``clang-tidy`` (Code Quality)
+   * - **Focus**
+     - Whitespace, indentation, braces
+     - Bugs, performance, modern practices
+   * - **Speed**
+     - Very fast (syntax-only)
+     - Slower (requires compilation database)
+   * - **Action**
+     - Modifies files in-place (``-i``)
+     - Reports violations (read-only)
+   * - **Example**
+     - ``if(x){...}`` to ``if (x) { ... }``
+     - ``vec.size() == 0`` to ``vec.empty()``
+
+Running clang-tidy locally
+==========================
+
+Step 1: Generate the compilation database
+------------------------------------------
+
+``clang-tidy`` needs a ``compile_commands.json`` file to understand include paths and macros. Add ``-DCMAKE_EXPORT_COMPILE_COMMANDS=ON`` when you configure your build:
+
+.. code-block:: shell
+
+   cmake -B ~/build_output -S . -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+
+Then symlink it to your source root:
+
+.. code-block:: shell
+
+   ln -sf ~/build_output/compile_commands.json .
+
+.. note::
+
+   A full build is recommended so that generated files (e.g., ``.actor.g.cpp`` headers) exist and include paths resolve correctly. Without a build, ``clang-tidy`` may report false errors on files that depend on generated code. If your ``compile_commands.json`` was generated on a different machine (e.g., Okteto), fix the paths with:
+
+   .. code-block:: shell
+
+      sed -i '' 's|/root/src/foundationdb/|/Users/YOU/checkouts/foundationdb/|g' compile_commands.json
+
+Step 2: Locate the helper script
+--------------------------------
+
+``clang-tidy-diff.py`` runs ``clang-tidy`` only on the lines changed in your diff, avoiding noise from existing code you did not touch.
+
+On macOS (with Homebrew LLVM):
+
+.. code-block:: shell
+
+   export TIDY_DIFF=$(find $(brew --prefix llvm)/share/clang -name "clang-tidy-diff.py")
+
+On Linux:
+
+.. code-block:: shell
+
+   export TIDY_DIFF=$(find /usr/lib/llvm-*/share/clang -name "clang-tidy-diff.py" | head -n 1)
+
+To make this permanent, add to your ``~/.bashrc`` or ``~/.zshrc``:
+
+.. code-block:: shell
+
+   alias fdb-tidy='python3 $(find $(brew --prefix llvm 2>/dev/null || echo "/usr/lib/llvm-*") -name "clang-tidy-diff.py" | head -n 1) -p 1 -path .'
+
+Step 3: Run against your changes
+---------------------------------
+
+Check all changes between your branch and ``main``:
+
+.. code-block:: shell
+
+   git diff -U0 origin/main...HEAD | grep -v -E '\.actor\.cpp' | python3 "$TIDY_DIFF" -p 1 -path .
+
+   # Or with the alias:
+   git diff -U0 origin/main...HEAD | grep -v -E '\.actor\.cpp' | fdb-tidy
+
+Check a specific commit:
+
+.. code-block:: shell
+
+   git show -U0 <commit_hash> | python3 "$TIDY_DIFF" -p 1 -path .
+
+.. important::
+
+   Use ``-U0`` (no context lines). Without it, ``clang-tidy`` may lint surrounding lines that you did not change, reporting pre-existing issues.
+
+Understanding the flags
+-----------------------
+
+The ``clang-tidy-diff.py`` script has different flag conventions from ``clang-tidy`` itself:
+
+* ``-p 1`` -- strip one directory prefix from git diff paths (``a/src/main.cpp`` becomes ``src/main.cpp``)
+* ``-path .`` -- directory containing ``compile_commands.json``
+* ``-quiet`` -- suppress per-file warning count summaries
+* ``-j N`` -- run N clang-tidy instances in parallel
+
+.. warning::
+
+   In the standard ``clang-tidy`` command, ``-p`` points to the build directory. In ``clang-tidy-diff.py``, ``-path`` points to the build directory and ``-p`` is for path stripping.
+
+Running clang-tidy during builds (CMake integration)
+=====================================================
+
+FoundationDB's CMake build supports running ``clang-tidy`` automatically during C/C++ compilation:
+
+.. code-block:: shell
+
+   cmake -S . -B build -G Ninja -DUSE_CLANG_TIDY=ON
+   ninja -C build fdbserver
+
+This runs ``clang-tidy`` on every file as it compiles. It is slower than using ``clang-tidy-diff.py`` on your diff but catches issues in all compiled code.
+
+Optional CMake variables:
+
+* ``CLANG_TIDY`` -- path to the ``clang-tidy`` executable (auto-detected by default)
+* ``CLANG_TIDY_EXTRA_ARGS`` -- additional space-separated arguments passed to ``clang-tidy``
+
+Known limitations
+-----------------
+
+**``.actor.cpp`` files cannot be analyzed.** These files use FoundationDB's custom actor compiler syntax (``ACTOR``, ``wait()``, ``state``) that ``clang-tidy`` cannot parse. Exclude them from your diff when running locally:
+
+Quick reference
+===============
+
+.. code-block:: shell
+
+   git diff -U0 origin/main...HEAD | grep -v -E '\.actor\.cpp' | python3 "$TIDY_DIFF" -p 1 -path .
+
+**GCC-built compile_commands.json with clang-tidy.** If your ``compile_commands.json`` was generated with GCC, it may contain GCC-specific flags that ``clang-tidy`` (which uses the clang frontend) does not recognize. Add ``-extra-arg=-Wno-unknown-warning-option`` to suppress these errors:
+
+.. code-block:: shell
+
+   git diff -U0 origin/main...HEAD | python3 "$TIDY_DIFF" -p 1 -path . -extra-arg=-Wno-unknown-warning-option
+
+Quick reference
diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst
index 779a1975c55..390c271e33a 100644
--- a/documentation/sphinx/source/command-line-interface.rst
+++ b/documentation/sphinx/source/command-line-interface.rst
@@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu
 configure
 ---------
 
-The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>] [count=<TSS_COUNT>] [perpetual_storage_wiggle=<WIGGLE_SPEED>] [perpetual_storage_wiggle_locality=<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>] [storage_migration_type={disabled|aggressive|gradual}]``.
+The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=<N>] [commit_proxies=<N>] [resolvers=<N>] [logs=<N>] [count=<TSS_COUNT>] [perpetual_storage_wiggle=<WIGGLE_SPEED>] [perpetual_storage_wiggle_locality=<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>] [perpetual_storage_wiggle_engine=<ENGINE>] [storage_migration_type={disabled|aggressive|gradual}]``.
 
 The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified.
 
diff --git a/documentation/sphinx/source/developer-tools.rst b/documentation/sphinx/source/developer-tools.rst
new file mode 100644
index 00000000000..15a100c6382
--- /dev/null
+++ b/documentation/sphinx/source/developer-tools.rst
@@ -0,0 +1,13 @@
+###############
+Developer Tools
+###############
+
+Tools and guides for FoundationDB contributors.
+
+.. toctree::
+   :maxdepth: 1
+
+   local-dev
+   clang-format
+   clang-tidy
+   internal-dev-tools
diff --git a/documentation/sphinx/source/global-configuration.rst b/documentation/sphinx/source/global-configuration.rst
index 663ad26eb4e..56316e7c52a 100644
--- a/documentation/sphinx/source/global-configuration.rst
+++ b/documentation/sphinx/source/global-configuration.rst
@@ -79,9 +79,9 @@ Values must always be encoded according to the :ref:`api-python-tuple-layer`.
 
 .. code-block:: cpp
 
-   // In GlobalConfig.actor.h
+   // In GlobalConfig.h
    extern const KeyRef myGlobalConfigKey;
-   // In GlobalConfig.actor.cpp
+   // In GlobalConfig.cpp
    const KeyRef myGlobalConfigKey = "config/key"_sr;
    
    // When you want to set the value..
diff --git a/documentation/sphinx/source/ha-write-path.rst b/documentation/sphinx/source/ha-write-path.rst
index deea5d0b931..92d86b363e9 100644
--- a/documentation/sphinx/source/ha-write-path.rst
+++ b/documentation/sphinx/source/ha-write-path.rst
@@ -118,9 +118,9 @@ tLogs also maintain two properties:
 At primary SS
 -------------
 
-**Primary tLog of a SS.** Since a SS’s tag is identically mapped to one tLog. The tLog has all mutations for the SS and is the primary tLog for the SS. When the SS peeks data from tLogs, it will prefer to peek data from its primary tLog. If the primary tLog crashes, it will contact the rest of tLogs, ask for mutations with the SS’s tag, and merge them together. This complex merge operation is abstracted in the TagPartitionedLogSystem interface.
+**Primary tLog of a SS.** Since a SS’s tag is identically mapped to one tLog. The tLog has all mutations for the SS and is the primary tLog for the SS. When the SS peeks data from tLogs, it will prefer to peek data from its primary tLog. If the primary tLog crashes, it will contact the rest of tLogs, ask for mutations with the SS’s tag, and merge them together. This complex merge operation is abstracted in the LogSystem interface.
 
-**Pulling data from tLogs.** Each SS in the primary DC keeps pulling mutations, whose tag is the SS’s tag, from tLogs. Once mutations before a version V1 are made durable on a SS, the SS pops the tag upto the version V1 from *all* tLogs. The pop operation is an RPC to tLogs through the TagPartitionedLogSystem interface.
+**Pulling data from tLogs.** Each SS in the primary DC keeps pulling mutations, whose tag is the SS’s tag, from tLogs. Once mutations before a version V1 are made durable on a SS, the SS pops the tag upto the version V1 from *all* tLogs. The pop operation is an RPC to tLogs through the LogSystem interface.
 
 Since the mutation m1 has three tags for primary SSes, the mutation will be made durable on three primary SSes. This marks the end of the mutation’s journey in the primary DC. 
 
@@ -164,7 +164,7 @@ https://github.com/apple/foundationdb/blob/7eabdf784a21bca102f84e7eaf14bafc54605
 Mutation Serialization (WiP)
 ============================
 
-This section will go into detail on how mutations are serialized as preparation for ingestion into the TagPartitionedLogSystem. This has also been covered at:
+This section will go into detail on how mutations are serialized as preparation for ingestion into the LogSystem. This has also been covered at:
 
 https://drive.google.com/file/d/1OaP5bqH2kst1VxD6RWj8h2cdr9rhhBHy/view
 
diff --git a/documentation/sphinx/source/index.rst b/documentation/sphinx/source/index.rst
index 167bebda43f..55f3d733a95 100644
--- a/documentation/sphinx/source/index.rst
+++ b/documentation/sphinx/source/index.rst
@@ -44,13 +44,14 @@ The latest changes are detailed in :ref:`release-notes`. The documentation has t
 
 * :doc:`visibility` contains documentation related to Visibility into FoundationDB.
 
+* :doc:`developer-tools` contains documentation for FoundationDB contributors, including local development setup, static analysis, and internal testing tools.
+
 .. toctree::
    :maxdepth: 1
    :titlesonly:
    :hidden:
 
-   local-dev
-   internal-dev-tools
+   developer-tools
    why-foundationdb
    technical-overview
    client-design
diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index 060ff759ec0..a6df96e873e 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -348,6 +348,7 @@
                         "trace_log_file_write_error",
                         "trace_log_could_not_create_file",
                         "trace_log_writer_thread_unresponsive",
+                        "exclude_from_tlog_recruitment_low_disk",
                         "process_error",
                         "io_error",
                         "io_timeout",
@@ -625,7 +626,8 @@
                         "incorrect_cluster_file_contents",
                         "trace_log_file_write_error",
                         "trace_log_could_not_create_file",
-                        "trace_log_writer_thread_unresponsive"
+                        "trace_log_writer_thread_unresponsive",
+                        "exclude_from_tlog_recruitment_low_disk"
                      ]
                   },
                   "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."
diff --git a/documentation/sphinx/source/read-write-path.rst b/documentation/sphinx/source/read-write-path.rst
index 8257f8905f9..277950b832f 100644
--- a/documentation/sphinx/source/read-write-path.rst
+++ b/documentation/sphinx/source/read-write-path.rst
@@ -179,7 +179,7 @@ Implementation of FDB read path
     * Proxy confirm queuing system is alive:
       https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1199
     * How is confirmEpochLive(..) implemented for the above item:
-      https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/TagPartitionedLogSystem.actor.cpp#L1216-L1225
+      https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp
 
 * **Step 4 (Locality request)**:
   https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1312-L1313
diff --git a/documentation/sphinx/source/release-notes/release-notes-700.rst b/documentation/sphinx/source/release-notes/release-notes-700.rst
index de56447519c..e350341fb0a 100644
--- a/documentation/sphinx/source/release-notes/release-notes-700.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-700.rst
@@ -32,7 +32,7 @@ Features
 Performance
 -----------
 * Improved Deque copy performance. `(PR #3197) <https://github.com/apple/foundationdb/pull/3197>`_
-* Increased performance of dr_agent when copying the mutation log. The ``COPY_LOG_BLOCK_SIZE``, ``COPY_LOG_BLOCKS_PER_TASK``, ``COPY_LOG_PREFETCH_BLOCKS``, ``COPY_LOG_READ_AHEAD_BYTES`` and ``COPY_LOG_TASK_DURATION_NANOS`` knobs can be set. `(PR #3436) <https://github.com/apple/foundationdb/pull/3436>`_
+* Increased performance of dr_agent when copying the mutation log. The ``COPY_LOG_BLOCK_SIZE``, ``COPY_LOG_BLOCKS_PER_TASK``, ``COPY_LOG_PREFETCH_BLOCKS``, ``COPY_LOG_READ_AHEAD_BYTES`` and ``COPY_LOG_TASK_DURATION_SECONDS`` knobs can be set. `(PR #3436) <https://github.com/apple/foundationdb/pull/3436>`_ `(PR #12733) <https://github.com/apple/foundationdb/pull/12733>`_
 * Added multiple new microbenchmarks for PromiseStream, Reference, IRandom, and timer, as well as support for benchmarking actors. `(PR #3590) <https://github.com/apple/foundationdb/pull/3590>`_
 * Use xxhash3 for SQLite page checksums. `(PR #4075) <https://github.com/apple/foundationdb/pull/4075>`_
 * fdbserver now uses jemalloc on Linux instead of the system malloc. `(PR #4222) <https://github.com/apple/foundationdb/pull/4222>`_
diff --git a/documentation/sphinx/source/release-notes/release-notes-730.rst b/documentation/sphinx/source/release-notes/release-notes-730.rst
index f00ea980918..72935f9148c 100644
--- a/documentation/sphinx/source/release-notes/release-notes-730.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-730.rst
@@ -2,6 +2,16 @@
 Release Notes
 #############
 
+7.3.77
+======
+* Same as 7.3.76 release with AVX enabled.
+
+7.3.76
+======
+* Fixed peer disconnect detection in waitValueOrSignal so dead connections are detected immediately instead of hanging indefinitely. `(PR #12935) <https://github.com/apple/foundationdb/pull/12935>`_
+* Made getTeamByServers O(1) in time which was sometimes causing Data Distributor (DD) to get stuck during initialization when shard_encode_location_metadata was enabled. `(PR #12938) <https://github.com/apple/foundationdb/pull/12938>`_
+* Added miscellaneous observability improvements for CPU and memory usage, DD startup, and S3 backup subsystems. `(PR #12937) <https://github.com/apple/foundationdb/pull/12937>`_, `(PR #12913) <https://github.com/apple/foundationdb/pull/12913>`_, and `(PR #12997) <https://github.com/apple/foundationdb/pull/12997>`_
+
 7.3.75
 ======
 * Same as 7.3.74 release with AVX enabled.
diff --git a/documentation/tutorial/dining_philosophers.actor.cpp b/documentation/tutorial/dining_philosophers.actor.cpp
index b2825fd57cd..8f6bc7065f2 100644
--- a/documentation/tutorial/dining_philosophers.actor.cpp
+++ b/documentation/tutorial/dining_philosophers.actor.cpp
@@ -24,7 +24,7 @@
 #include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include <functional>
 #include <unordered_map>
 #include <memory>
@@ -102,8 +102,8 @@ struct GetForkRequest {
 	ForkState forkState;
 	ReplyPromise<ForkState> reply;
 
-	GetForkRequest(ForkState fork_state) : forkState(fork_state) {}
-	GetForkRequest() {}
+	explicit GetForkRequest(ForkState fork_state) : forkState(fork_state) {}
+	GetForkRequest() = default;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -116,8 +116,8 @@ struct ReleaseForkRequest {
 	ForkState forkState;
 	ReplyPromise<ForkState> reply;
 
-	ReleaseForkRequest(ForkState fork_state) : forkState(fork_state) {}
-	ReleaseForkRequest() {}
+	explicit ReleaseForkRequest(ForkState fork_state) : forkState(fork_state) {}
+	ReleaseForkRequest() = default;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
diff --git a/documentation/tutorial/make_h2o.actor.cpp b/documentation/tutorial/make_h2o.actor.cpp
index d3f0977e96a..d3739750450 100644
--- a/documentation/tutorial/make_h2o.actor.cpp
+++ b/documentation/tutorial/make_h2o.actor.cpp
@@ -24,7 +24,7 @@
 #include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/actorcompiler.h"
 
 #include <functional>
diff --git a/documentation/tutorial/play.actor.cpp b/documentation/tutorial/play.actor.cpp
index 12eef72324a..a8ee1db16b8 100644
--- a/documentation/tutorial/play.actor.cpp
+++ b/documentation/tutorial/play.actor.cpp
@@ -22,7 +22,7 @@
 #include "flow/Arena.h"
 #include "flow/flow.h"
 #include "flow/Platform.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/actorcompiler.h"
 
 #include <iostream>
diff --git a/documentation/tutorial/play_network.actor.cpp b/documentation/tutorial/play_network.actor.cpp
index 707ccde612c..3eafaf64f31 100644
--- a/documentation/tutorial/play_network.actor.cpp
+++ b/documentation/tutorial/play_network.actor.cpp
@@ -22,7 +22,7 @@
 #include "flow/Arena.h"
 #include "flow/flow.h"
 #include "flow/Platform.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/actorcompiler.h"
 
 #include <iostream>
diff --git a/documentation/tutorial/print_in_order.actor.cpp b/documentation/tutorial/print_in_order.actor.cpp
index 97adf841ebc..174f202bfe4 100644
--- a/documentation/tutorial/print_in_order.actor.cpp
+++ b/documentation/tutorial/print_in_order.actor.cpp
@@ -24,7 +24,7 @@
 #include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/actorcompiler.h"
 
 #include <functional>
diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp
index 29888d0b69e..5a706a6f837 100644
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@@ -25,7 +25,7 @@
 #include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include <functional>
 #include <unordered_map>
 #include <memory>
@@ -518,20 +518,6 @@ ACTOR Future<Void> fdbClient() {
 	}
 }
 
-ACTOR Future<Void> fdbStatusStresser() {
-	state Database db = Database::createDatabase(clusterFile, 300);
-	state ReadYourWritesTransaction tx(db);
-	state Key statusJson(std::string("\xff\xff/status/json"));
-	loop {
-		try {
-			tx.reset();
-			Optional<Value> _ = wait(tx.get(statusJson));
-		} catch (Error& e) {
-			wait(tx.onError(e));
-		}
-	}
-}
-
 std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
 	{ "timer", &simpleTimer }, // ./tutorial timer
 	{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
@@ -543,9 +529,8 @@ std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
 	{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
 	{ "fdbClientStream", &fdbClientStream }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientStream
 	{ "fdbClientGetRange", &fdbClientGetRange }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientGetRange
-	{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
-	{ "fdbStatusStresser", &fdbStatusStresser }
-}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
+	{ "fdbClient", &fdbClient } // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
+};
 
 int main(int argc, char* argv[]) {
 	bool isServer = false;
diff --git a/fdbbackup/CMakeLists.txt b/fdbbackup/CMakeLists.txt
index ff0b0c2d559..63e5b394fba 100644
--- a/fdbbackup/CMakeLists.txt
+++ b/fdbbackup/CMakeLists.txt
@@ -1,22 +1,22 @@
 set(FDBBACKUP_SRCS
 	Decode.cpp
-	backup.actor.cpp)
+	backup.cpp)
 
 add_flow_target(EXECUTABLE NAME fdbbackup SRCS ${FDBBACKUP_SRCS})
-target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
 target_link_libraries(fdbbackup PRIVATE fdbclient)
 
 set(FDBCONVERT_SRCS
-	FileConverter.actor.cpp)
+	FileConverter.cpp)
 add_flow_target(EXECUTABLE NAME fdbconvert SRCS ${FDBCONVERT_SRCS})
-target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
 target_link_libraries(fdbconvert PRIVATE fdbclient)
 
 set(FDBDECODE_SRCS
 	Decode.cpp
-	FileDecoder.actor.cpp)
+	FileDecoder.cpp)
 add_flow_target(EXECUTABLE NAME fdbdecode SRCS ${FDBDECODE_SRCS})
-target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
 target_link_libraries(fdbdecode PRIVATE fdbclient)
 
 if(NOT OPEN_FOR_IDE)
@@ -39,19 +39,25 @@ if(NOT OPEN_FOR_IDE)
   symlink_files(
     LOCATION packages/bin
     SOURCE fdbbackup
-    TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_tool)
+    TARGETS fdbdr dr_agent backup_agent fdbrestore)
   symlink_files(
     LOCATION bin
     SOURCE fdbbackup
-    TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_tool)
+    TARGETS fdbdr dr_agent backup_agent fdbrestore)
 
-  # Test version of backup.actor.cpp without main() function
-  add_flow_target(EXECUTABLE NAME backup_tests SRCS backup.actor.cpp)
-  target_include_directories(backup_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+  # Test version of backup.cpp without main() function
+  add_flow_target(EXECUTABLE NAME backup_tests SRCS backup.cpp)
+  target_include_directories(backup_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
   target_link_libraries(backup_tests PRIVATE fdbclient)
   target_compile_definitions(backup_tests PRIVATE EXCLUDE_MAIN_FUNCTION=1)
   add_test(NAME BackupTests COMMAND backup_tests)
 
+  # Test version of FileDecoder.cpp without main() function
+  add_flow_target(EXECUTABLE NAME fdbdecode_tests SRCS Decode.cpp FileDecoder.cpp)
+  target_include_directories(fdbdecode_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
+  target_link_libraries(fdbdecode_tests PRIVATE fdbclient)
+  target_compile_definitions(fdbdecode_tests PRIVATE EXCLUDE_MAIN_FUNCTION=1)
+  add_test(NAME FileDecoderTests COMMAND fdbdecode_tests)
 endif()
 
 if (GPERFTOOLS_FOUND)
@@ -62,7 +68,7 @@ endif()
 if (NOT WIN32 AND NOT OPEN_FOR_IDE)
   enable_testing()
   add_test(NAME dir_backup_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/dir_backup_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
-  add_test(NAME s3_backup_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/s3_backup_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} --encrypt-at-random)
+  add_test(NAME blob_backup_restore_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/blob_backup_restore_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
   # Note: No --encrypt flag - BulkLoad doesn't support encryption yet
   add_test(NAME s3_backup_bulkdump_bulkload_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/s3_backup_bulkdump_bulkload.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
 endif()
diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.cpp
similarity index 85%
rename from fdbbackup/FileConverter.actor.cpp
rename to fdbbackup/FileConverter.cpp
index 7ba2ae04e7c..27cf69243c1 100644
--- a/fdbbackup/FileConverter.actor.cpp
+++ b/fdbbackup/FileConverter.cpp
@@ -1,5 +1,5 @@
 /*
- * FileConverter.actor.cpp
+ * FileConverter.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,13 +26,12 @@
 #include <cstdio>
 #include <vector>
 
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/MutationList.h"
 #include "flow/flow.h"
 #include "flow/serialize.h"
 #include "fdbclient/BuildFlags.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 namespace file_converter {
 
@@ -254,7 +253,7 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 		std::cout << msg << "\n  ";
 		for (const auto& fp : fileProgress) {
 			std::cout << fp->fd->getFilename() << " " << fp->mutations.size() << " mutations";
-			if (fp->mutations.size() > 0) {
+			if (!fp->mutations.empty()) {
 				std::cout << ", range " << fp->mutations[0].version.toString() << " "
 				          << fp->mutations.back().version.toString() << "\n";
 			} else {
@@ -277,15 +276,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 	// The caller must hold on the the arena associated with the mutation.
 	Future<VersionedData> getNextMutation() { return getMutationImpl(this); }
 
-	ACTOR static Future<VersionedData> getMutationImpl(MutationFilesReadProgress* self) {
+	static Future<VersionedData> getMutationImpl(MutationFilesReadProgress* self) {
 		ASSERT(!self->fileProgress.empty() && !self->fileProgress[0]->mutations.empty());
 
-		state Reference<FileProgress> fp = self->fileProgress[0];
-		state VersionedData data = fp->mutations[0];
+		Reference<FileProgress> fp = self->fileProgress[0];
+		VersionedData data = fp->mutations[0];
 		fp->mutations.erase(fp->mutations.begin());
 		if (fp->mutations.empty()) {
 			// decode one more block
-			wait(decodeToVersion(fp, /*version=*/0, self->endVersion, self->getLogFile(fp->idx)));
+			co_await decodeToVersion(fp, /*version=*/0, self->endVersion, self->getLogFile(fp->idx));
 		}
 
 		if (fp->empty()) {
@@ -299,7 +298,7 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 				std::swap(self->fileProgress[i - 1], self->fileProgress[i]);
 			}
 		}
-		return data;
+		co_return data;
 	}
 
 	LogFile& getLogFile(int index) { return files[index]; }
@@ -307,13 +306,12 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 	Future<Void> openLogFiles(Reference<IBackupContainer> container) { return openLogFilesImpl(this, container); }
 
 	// Opens log files in the progress and starts decoding until the beginVersion is seen.
-	ACTOR static Future<Void> openLogFilesImpl(MutationFilesReadProgress* progress,
-	                                           Reference<IBackupContainer> container) {
-		state std::vector<Future<Reference<IAsyncFile>>> asyncFiles;
+	static Future<Void> openLogFilesImpl(MutationFilesReadProgress* progress, Reference<IBackupContainer> container) {
+		std::vector<Future<Reference<IAsyncFile>>> asyncFiles;
 		for (const auto& file : progress->files) {
 			asyncFiles.push_back(container->readFile(file.fileName));
 		}
-		wait(waitForAll(asyncFiles)); // open all files
+		co_await waitForAll(asyncFiles); // open all files
 
 		// Attempt decode the first few blocks of log files until beginVersion is consumed
 		std::vector<Future<Void>> fileDecodes;
@@ -324,37 +322,37 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 			    decodeToVersion(fp, progress->beginVersion, progress->endVersion, progress->getLogFile(i)));
 		}
 
-		wait(waitForAll(fileDecodes));
+		co_await waitForAll(fileDecodes);
 
 		progress->sortAndRemoveEmpty();
-
-		return Void();
 	}
 
 	// Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
 	// Skip mutations >= maxVersion.
-	ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
-	                                          Version minVersion,
-	                                          Version maxVersion,
-	                                          LogFile file) {
-		if (fp->empty())
-			return Void();
+	static Future<Void> decodeToVersion(Reference<FileProgress> fp,
+	                                    Version minVersion,
+	                                    Version maxVersion,
+	                                    LogFile file) {
+		if (fp->empty()) {
+			co_return;
+		}
 
-		if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
-			return Void();
+		if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) {
+			co_return;
+		}
 
-		state int64_t len;
+		int64_t len = 0;
 		try {
 			// Read block by block until we see the minVersion
-			loop {
+			while (true) {
 				len = std::min<int64_t>(file.blockSize, file.fileSize - fp->offset);
 				if (len == 0) {
 					fp->eof = true;
-					return Void();
+					co_return;
 				}
 
-				state Standalone<StringRef> buf = makeString(len);
-				int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
+				Standalone<StringRef> buf = makeString(len);
+				int rLen = co_await fp->fd->read(mutateString(buf), len, fp->offset);
 				if (len != rLen)
 					throw restore_bad_read();
 
@@ -365,7 +363,6 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 				if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
 					break;
 			}
-			return Void();
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "CorruptedLogFileBlock")
 			    .error(e)
@@ -382,7 +379,7 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
 };
 
 // Writes a log file in the old backup format, described in backup-dataFormat.md.
-// This is similar to the LogFileWriter in FileBackupAgent.actor.cpp.
+// This is similar to the LogFileWriter in FileBackupAgent.cpp.
 struct LogFileWriter {
 	LogFileWriter() : blockSize(-1) {}
 	LogFileWriter(Reference<IBackupFile> f, int bsize) : file(f), blockSize(bsize) {}
@@ -400,50 +397,47 @@ struct LogFileWriter {
 	}
 
 	// Start a new block if needed, then write the key and value
-	ACTOR static Future<Void> writeKV_impl(LogFileWriter* self, Key k, Value v) {
+	static Future<Void> writeKV_impl(LogFileWriter* self, Key k, Value v) {
 		// If key and value do not fit in this block, end it and start a new one
 		int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size();
 		if (self->file->size() + toWrite > self->blockEnd) {
 			// Write padding if needed
 			int bytesLeft = self->blockEnd - self->file->size();
 			if (bytesLeft > 0) {
-				state Value paddingFFs = fileBackup::makePadding(bytesLeft);
-				wait(self->file->append(paddingFFs.begin(), bytesLeft));
+				Value paddingFFs = fileBackup::makePadding(bytesLeft);
+				co_await self->file->append(paddingFFs.begin(), bytesLeft);
 			}
 
 			// Set new blockEnd
 			self->blockEnd += self->blockSize;
 
 			// write Header
-			wait(self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION)));
+			co_await self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION));
 		}
 
-		wait(self->file->appendStringRefWithLen(k));
-		wait(self->file->appendStringRefWithLen(v));
+		co_await self->file->appendStringRefWithLen(k);
+		co_await self->file->appendStringRefWithLen(v);
 
 		// At this point we should be in whatever the current block is or the block size is too small
 		if (self->file->size() > self->blockEnd)
 			throw backup_bad_block_size();
-
-		return Void();
 	}
 
 	Future<Void> writeKV(Key k, Value v) { return writeKV_impl(this, k, v); }
 
 	// Adds a new mutation to an internal buffer and writes out when encountering
 	// a new commitVersion or exceeding the block size.
-	ACTOR static Future<Void> addMutation(LogFileWriter* self, Version commitVersion, MutationListRef mutations) {
-		state Standalone<StringRef> value = BinaryWriter::toValue(mutations, IncludeVersion());
+	static Future<Void> addMutation(LogFileWriter* self, Version commitVersion, MutationListRef mutations) {
+		Standalone<StringRef> value = BinaryWriter::toValue(mutations, IncludeVersion());
 
-		state int part = 0;
+		int part = 0;
 		for (; part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE < value.size(); part++) {
 			StringRef partBuf = value.substr(
 			    part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE,
 			    std::min(value.size() - part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, CLIENT_KNOBS->MUTATION_BLOCK_SIZE));
 			Standalone<StringRef> key = getBlockKey(commitVersion, part);
-			wait(writeKV_impl(self, key, partBuf));
+			co_await writeKV_impl(self, key, partBuf);
 		}
-		return Void();
 	}
 
 private:
@@ -452,13 +446,12 @@ struct LogFileWriter {
 	int64_t blockEnd = 0;
 };
 
-ACTOR Future<Void> convert(ConvertParams params) {
-	state Reference<IBackupContainer> container =
-	    IBackupContainer::openContainer(params.container_url, params.proxy, {});
-	state BackupFileList listing = wait(container->dumpFileList());
+Future<Void> convert(ConvertParams params) {
+	Reference<IBackupContainer> container = IBackupContainer::openContainer(params.container_url, params.proxy, {}, 0);
+	BackupFileList listing = co_await container->dumpFileList();
 	std::sort(listing.logs.begin(), listing.logs.end());
 	TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size());
-	state BackupDescription desc = wait(container->describeBackup());
+	BackupDescription desc = co_await container->describeBackup();
 	std::cout << "\n" << desc.toString() << "\n";
 
 	// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec <<
@@ -467,24 +460,24 @@ ACTOR Future<Void> convert(ConvertParams params) {
 	std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end);
 	printLogFiles("Range has", logs);
 
-	state Reference<MutationFilesReadProgress> progress(new MutationFilesReadProgress(logs, params.begin, params.end));
+	Reference<MutationFilesReadProgress> progress(new MutationFilesReadProgress(logs, params.begin, params.end));
 
-	wait(progress->openLogFiles(container));
+	co_await progress->openLogFiles(container);
 
-	state int blockSize = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
-	state Reference<IBackupFile> outFile = wait(container->writeLogFile(params.begin, params.end, blockSize));
-	state LogFileWriter logFile(outFile, blockSize);
+	int blockSize = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
+	Reference<IBackupFile> outFile = co_await container->writeLogFile(params.begin, params.end, blockSize);
+	LogFileWriter logFile(outFile, blockSize);
 	std::cout << "Output file: " << outFile->getFileName() << "\n";
 
-	state MutationList list;
-	state Arena arena;
-	state Version version = invalidVersion;
+	MutationList list;
+	Arena arena;
+	Version version = invalidVersion;
 	while (progress->hasMutations()) {
-		state VersionedData data = wait(progress->getNextMutation());
+		VersionedData data = co_await progress->getNextMutation();
 
 		// emit a mutation batch to file when encounter a new version
 		if (list.totalSize() > 0 && version != data.version.version) {
-			wait(LogFileWriter::addMutation(&logFile, version, list));
+			co_await LogFileWriter::addMutation(&logFile, version, list);
 			list = MutationList();
 			arena = Arena();
 		}
@@ -499,12 +492,10 @@ ACTOR Future<Void> convert(ConvertParams params) {
 		version = data.version.version;
 	}
 	if (list.totalSize() > 0) {
-		wait(LogFileWriter::addMutation(&logFile, version, list));
+		co_await LogFileWriter::addMutation(&logFile, version, list);
 	}
 
-	wait(outFile->finish());
-
-	return Void();
+	co_await outFile->finish();
 }
 
 int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
@@ -623,4 +614,4 @@ int main(int argc, char** argv) {
 		TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
 		return FDB_EXIT_MAIN_EXCEPTION;
 	}
-}
\ No newline at end of file
+}
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.cpp
similarity index 86%
rename from fdbbackup/FileDecoder.actor.cpp
rename to fdbbackup/FileDecoder.cpp
index 71872010481..bc192a80b64 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.cpp
@@ -1,5 +1,5 @@
 /*
- * FileDecoder.actor.cpp
+ * FileDecoder.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -35,13 +35,11 @@
 #include "fdbclient/BuildFlags.h"
 #include "fdbbackup/FileConverter.h"
 #include "fdbbackup/Decode.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbclient/BuildFlags.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/MutationList.h"
@@ -55,8 +53,6 @@
 #include "flow/flow.h"
 #include "flow/serialize.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 #define SevDecodeInfo SevVerbose
 
 extern bool g_crashOnError;
@@ -103,6 +99,8 @@ void printDecodeUsage() {
 	             "  --knob-KNOBNAME KNOBVALUE\n"
 	             "                 Changes a knob value. KNOBNAME should be lowercase.\n"
 	             "  -s, --save     Save a copy of downloaded files (default: not saving).\n"
+	             "  --encryption-key-file FILE\n"
+	             "                 AES-128-GCM encryption key file for encrypted backups.\n"
 	             "\n";
 	return;
 }
@@ -130,6 +128,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	Version endVersionFilter = std::numeric_limits<Version>::max();
 
 	std::vector<std::pair<std::string, std::string>> knobs;
+	Optional<std::string> encryptionKeyFileName;
 
 	// Returns if [begin, end) overlap with the filter range
 	bool overlap(Version begin, Version end) const {
@@ -139,6 +138,8 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 
 	bool overlap(Version version) const { return version >= beginVersionFilter && version < endVersionFilter; }
 
+	bool validVersionFilters() { return beginVersionFilter < endVersionFilter; }
+
 	void updateRangeMap() { filters.updateFilters(prefixes); }
 
 	bool matchFilters(const MutationRef& m) const {
@@ -237,14 +238,17 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 			s.append(", KNOB-").append(knob).append(" = ").append(value);
 		}
 		s.append(", SaveFile: ").append(save_file_locally ? "true" : "false");
+		if (encryptionKeyFileName.present()) {
+			s.append(", EncryptionKeyFile: ").append(encryptionKeyFileName.get());
+		}
 		return s;
 	}
 
 	void updateKnobs() {
-		IKnobCollection::setupKnobs(knobs);
+		setupClientKnobs(knobs);
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
+		initializeClientKnobs(Randomize::False, IsSimulated::False);
 	}
 };
 
@@ -305,7 +309,7 @@ int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 				param->decode_range = false;
 			} else if (ftype == "range") {
 				param->decode_logs = false;
-			} else if (ftype != "both" && ftype != "") {
+			} else if (ftype != "both" && !ftype.empty()) {
 				err = true;
 				std::cerr << "ERROR: Unrecognized backup file type option: " << args->OptionArg() << "\n";
 				return FDB_EXIT_ERROR;
@@ -394,6 +398,10 @@ int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 			param->save_file_locally = true;
 			break;
 
+		case OPT_ENCRYPTION_KEY_FILE:
+			param->encryptionKeyFileName = args->OptionArg();
+			break;
+
 		case TLSConfig::OPT_TLS_PLUGIN:
 			args->OptionArg();
 			break;
@@ -468,9 +476,9 @@ struct VersionedMutations {
  * Model a decoding progress for a mutation file. Usage is:
  *
  *    DecodeProgress progress(logfile);
- *    wait(progress->openFile(container));
+ *    co_await progress.openFile(container);
  *    while (1) {
- *        Optional<VersionedMutations> batch = wait(progress->getNextBatch());
+ *        Optional<VersionedMutations> batch = progress.getNextBatch();
  *        if (!batch.present()) break;
  *        ... // process the batch mutations
  *    }
@@ -523,11 +531,10 @@ class DecodeProgress {
 		return Optional<VersionedMutations>();
 	}
 
-	ACTOR static Future<Void> openFileImpl(DecodeProgress* self, Reference<IBackupContainer> container) {
-		Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
-		self->fd = fd;
-		state Standalone<StringRef> buf = makeString(self->file.fileSize);
-		int rLen = wait(self->fd->read(mutateString(buf), self->file.fileSize, 0));
+	static Future<Void> openFileImpl(DecodeProgress* self, Reference<IBackupContainer> container) {
+		self->fd = co_await container->readFile(self->file.fileName);
+		Standalone<StringRef> buf = makeString(self->file.fileSize);
+		int rLen = co_await self->fd->read(mutateString(buf), self->file.fileSize, 0);
 		if (rLen != self->file.fileSize) {
 			throw restore_bad_read();
 		}
@@ -557,7 +564,6 @@ class DecodeProgress {
 		}
 
 		self->decodeFile(buf);
-		return Void();
 	}
 
 	// Add chunks to mutationBlocksByVersion
@@ -571,7 +577,7 @@ class DecodeProgress {
 	// Reads a file a file content in the buffer, decodes it into key/value pairs, and stores these pairs.
 	void decodeFile(const Standalone<StringRef>& buf) {
 		try {
-			loop {
+			while (true) {
 				int64_t len = std::min<int64_t>(file.blockSize, file.fileSize - offset);
 				if (len == 0) {
 					return;
@@ -617,13 +623,12 @@ class DecodeRangeProgress {
 	// Open and loads file into memory
 	Future<Void> openFile(Reference<IBackupContainer> container) { return openFileImpl(this, container); }
 
-	ACTOR static Future<Void> openFileImpl(DecodeRangeProgress* self, Reference<IBackupContainer> container) {
+	static Future<Void> openFileImpl(DecodeRangeProgress* self, Reference<IBackupContainer> container) {
 		TraceEvent("ReadFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize);
 
-		Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
-		self->fd = fd;
-		state Standalone<StringRef> buf = makeString(self->file.fileSize);
-		int rLen = wait(self->fd->read(mutateString(buf), self->file.fileSize, 0));
+		self->fd = co_await container->readFile(self->file.fileName);
+		Standalone<StringRef> buf = makeString(self->file.fileSize);
+		int rLen = co_await self->fd->read(mutateString(buf), self->file.fileSize, 0);
 		if (rLen != self->file.fileSize) {
 			throw restore_bad_read();
 		}
@@ -654,13 +659,12 @@ class DecodeRangeProgress {
 		}
 
 		self->decodeFile(buf);
-		return Void();
 	}
 
 	// Reads a file content in the buffer, decodes it into key/value pairs, and stores these pairs.
 	void decodeFile(const Standalone<StringRef>& buf) {
 		try {
-			loop {
+			while (true) {
 				// process one block at a time
 				int64_t len = std::min<int64_t>(file.blockSize, file.fileSize - offset);
 				if (len == 0) {
@@ -698,18 +702,18 @@ std::string hexStringRef(const StringRef& s) {
 	return result;
 }
 
-ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
-                                      RangeFile file,
-                                      UID uid,
-                                      Reference<DecodeParams> params) {
+Future<Void> process_range_file(Reference<IBackupContainer> container,
+                                RangeFile file,
+                                UID uid,
+                                Reference<DecodeParams> params) {
 
 	if (file.fileSize == 0) {
 		TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName);
-		return Void();
+		co_return;
 	}
 
-	state DecodeRangeProgress progress(file, params->save_file_locally);
-	wait(progress.openFile(container));
+	DecodeRangeProgress progress(file, params->save_file_locally);
+	co_await progress.openFile(container);
 
 	for (auto& block : progress.blocks) {
 		for (const auto& kv : block) {
@@ -730,21 +734,19 @@ ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
 		}
 	}
 	TraceEvent("ProcessRangeFileDone", uid).detail("File", file.fileName);
-
-	return Void();
 }
 
-ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
-                                LogFile file,
-                                UID uid,
-                                Reference<DecodeParams> params) {
+Future<Void> process_file(Reference<IBackupContainer> container,
+                          LogFile file,
+                          UID uid,
+                          Reference<DecodeParams> params) {
 	if (file.fileSize == 0) {
 		TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName);
-		return Void();
+		co_return;
 	}
 
-	state DecodeProgress progress(file, params->save_file_locally);
-	wait(progress.openFile(container));
+	DecodeProgress progress(file, params->save_file_locally);
+	co_await progress.openFile(container);
 	while (true) {
 		auto batch = progress.getNextBatch();
 		if (!batch.present())
@@ -775,21 +777,19 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
 		}
 	}
 	TraceEvent("ProcessFileDone", uid).detail("File", file.fileName);
-	return Void();
 }
 
 // Use the snapshot metadata to quickly identify relevant range files and
 // then filter by versions.
-ACTOR Future<std::vector<RangeFile>> getRangeFiles(Reference<IBackupContainer> bc, Reference<DecodeParams> params) {
-	state std::vector<KeyspaceSnapshotFile> snapshots =
-	    wait((dynamic_cast<BackupContainerFileSystem*>(bc.getPtr()))->listKeyspaceSnapshots());
-	state std::vector<RangeFile> files;
+Future<std::vector<RangeFile>> getRangeFiles(Reference<IBackupContainer> bc, Reference<DecodeParams> params) {
+	std::vector<KeyspaceSnapshotFile> snapshots =
+	    co_await (dynamic_cast<BackupContainerFileSystem*>(bc.getPtr()))->listKeyspaceSnapshots();
+	std::vector<RangeFile> files;
 
-	state int i = 0;
-	for (; i < snapshots.size(); i++) {
+	for (int i = 0; i < snapshots.size(); i++) {
 		try {
 			std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>> results =
-			    wait((dynamic_cast<BackupContainerFileSystem*>(bc.getPtr()))->readKeyspaceSnapshot(snapshots[i]));
+			    co_await (dynamic_cast<BackupContainerFileSystem*>(bc.getPtr()))->readKeyspaceSnapshot(snapshots[i]);
 			for (const auto& rangeFile : results.first) {
 				const auto& keyRange = results.second.at(rangeFile.fileName);
 				if (params->matchFilters(keyRange)) {
@@ -803,14 +803,16 @@ ACTOR Future<std::vector<RangeFile>> getRangeFiles(Reference<IBackupContainer> b
 			}
 		}
 	}
-	return getRelevantRangeFiles(files, params);
+	co_return getRelevantRangeFiles(files, params);
 }
 
-ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
-	state Reference<IBackupContainer> container =
-	    IBackupContainer::openContainer(params->container_url, params->proxy, {});
-	state UID uid = deterministicRandom()->randomUniqueID();
-	state BackupFileList listing = wait(container->dumpFileList());
+Future<Void> decode_logs(Reference<DecodeParams> params) {
+	Reference<IBackupContainer> container =
+	    IBackupContainer::openContainer(params->container_url, params->proxy, params->encryptionKeyFileName, 0);
+	UID uid = deterministicRandom()->randomUniqueID();
+
+	BackupFileList listing = co_await container->dumpFileList();
+
 	// remove partitioned logs
 	listing.logs.erase(std::remove_if(listing.logs.begin(),
 	                                  listing.logs.end(),
@@ -823,11 +825,14 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	TraceEvent("Container", uid).detail("URL", params->container_url).detail("Logs", listing.logs.size());
 	TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params->toString());
 
-	BackupDescription desc = wait(container->describeBackup());
+	BackupDescription desc = co_await container->describeBackup();
+
+	container->setEncryptionBlockSize(desc.encryptionBlockSize);
+
 	std::cout << "\n" << desc.toString() << "\n";
 
-	state std::vector<LogFile> logFiles;
-	state std::vector<RangeFile> rangeFiles;
+	std::vector<LogFile> logFiles;
+	std::vector<RangeFile> rangeFiles;
 
 	if (params->decode_logs) {
 		logFiles = getRelevantLogFiles(listing.logs, params);
@@ -836,7 +841,7 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 
 	if (params->decode_range) {
 		// rangeFiles = getRelevantRangeFiles(filteredRangeFiles, params);
-		std::vector<RangeFile> files = wait(getRangeFiles(container, params));
+		std::vector<RangeFile> files = co_await getRangeFiles(container, params);
 		rangeFiles = files;
 		printLogFiles("Relevant range files are: ", rangeFiles);
 	}
@@ -844,14 +849,14 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	TraceEvent("TotalFiles", uid).detail("LogFiles", logFiles.size()).detail("RangeFiles", rangeFiles.size());
 
 	if (params->list_only)
-		return Void();
+		co_return;
 
 	// Decode log files.
-	state int idx = 0;
+	int idx = 0;
 	if (params->decode_logs) {
 		while (idx < logFiles.size()) {
 			TraceEvent("ProcessFile").detail("Name", logFiles[idx].fileName).detail("I", idx);
-			wait(process_file(container, logFiles[idx], uid, params));
+			co_await process_file(container, logFiles[idx], uid, params);
 			idx++;
 		}
 		TraceEvent("DecodeLogsDone", uid).log();
@@ -862,17 +867,16 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 		idx = 0;
 		while (idx < rangeFiles.size()) {
 			TraceEvent("ProcessFile").detail("Name", rangeFiles[idx].fileName).detail("I", idx);
-			wait(process_range_file(container, rangeFiles[idx], uid, params));
+			co_await process_range_file(container, rangeFiles[idx], uid, params);
 			idx++;
 		}
 		TraceEvent("DecodeRangeFileDone", uid).log();
 	}
-
-	return Void();
 }
 
 } // namespace file_converter
 
+#ifndef EXCLUDE_MAIN_FUNCTION
 int main(int argc, char** argv) {
 	std::string commandLine;
 	for (int a = 0; a < argc; a++) {
@@ -893,6 +897,15 @@ int main(int argc, char** argv) {
 			return status;
 		}
 
+		// Check if the beginVersionFilter is greater than the endVersionFilter, otherwise the filtering will be
+		// invalid.
+		if (!param->validVersionFilters()) {
+			std::cerr << "--begin-version-filter " << param->beginVersionFilter
+			          << " cannot be equal or greater than --end-version-filter " << param->endVersionFilter << "\n";
+			file_converter::printDecodeUsage();
+			return FDB_EXIT_ERROR;
+		}
+
 		if (param->log_enabled) {
 			if (param->log_dir.empty()) {
 				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE);
@@ -928,7 +941,7 @@ int main(int argc, char** argv) {
 		    .detail("SourceVersion", getSourceVersion())
 		    .detail("Version", FDB_VT_VERSION)
 		    .detail("PackageName", FDB_VT_PACKAGE_NAME)
-		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
+		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 		    .setMaxFieldLength(10000)
 		    .detail("CommandLine", commandLine)
 		    .setMaxFieldLength(0)
@@ -955,3 +968,36 @@ int main(int argc, char** argv) {
 		return FDB_EXIT_MAIN_EXCEPTION;
 	}
 }
+#else // EXCLUDE_MAIN_FUNCTION
+
+int main() {
+	auto assertValid = [](file_converter::DecodeParams& p, bool expected, const char* label) {
+		bool result = p.validVersionFilters();
+		if (result != expected) {
+			fprintf(stderr, "FAIL [%s]: expected %s\n", label, expected ? "valid" : "invalid");
+			return false;
+		}
+		printf("PASS [%s]\n", label);
+		return true;
+	};
+
+	bool ok = true;
+	file_converter::DecodeParams p;
+
+	ok &= assertValid(p, true, "defaults");
+
+	p.beginVersionFilter = 100;
+	p.endVersionFilter = 200;
+	ok &= assertValid(p, true, "begin < end");
+
+	p.beginVersionFilter = 200;
+	p.endVersionFilter = 200;
+	ok &= assertValid(p, false, "begin == end");
+
+	p.beginVersionFilter = 300;
+	p.endVersionFilter = 200;
+	ok &= assertValid(p, false, "begin > end");
+
+	return ok ? 0 : 1;
+}
+#endif // EXCLUDE_MAIN_FUNCTION
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.cpp
similarity index 82%
rename from fdbbackup/backup.actor.cpp
rename to fdbbackup/backup.cpp
index 168bd2ccb62..0c24028f33a 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.cpp
@@ -1,5 +1,5 @@
 /*
- * backup.actor.cpp
+ * backup.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -28,6 +28,7 @@
 #include "flow/Error.h"
 #include "flow/SystemMonitor.h"
 #include "flow/Trace.h"
+#include "flow/CoroUtils.h"
 #define BOOST_DATE_TIME_NO_LIB
 #include <boost/interprocess/managed_shared_memory.hpp>
 
@@ -36,23 +37,22 @@
 #include "flow/serialize.h"
 #include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/ClusterConnectionFile.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/IKnobCollection.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbclient/S3BlobStore.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/RunRYWTransaction.h"
+#include "fdbclient/IBlobStore.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/json_spirit/json_spirit_writer_template.h"
 #include "fdbclient/BulkLoading.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/BackupContainer.h"
+#include "fdbclient/ManagementAPI.h"
 
 #include "flow/Platform.h"
 
@@ -82,10 +82,9 @@
 #include "fdbclient/BuildFlags.h"
 
 #include "SimpleOpt/SimpleOpt.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // Type of program being executed
-enum class ProgramExe { AGENT, BACKUP, RESTORE, FASTRESTORE_TOOL, DR_AGENT, DB_BACKUP, UNDEFINED };
+enum class ProgramExe { AGENT, BACKUP, RESTORE, DR_AGENT, DB_BACKUP, UNDEFINED };
 
 enum class BackupType {
 	UNDEFINED = 0,
@@ -135,7 +134,7 @@ enum {
 	OPT_JSON,
 	OPT_DELETE_DATA,
 	OPT_MIN_CLEANUP_SECONDS,
-	OPT_USE_PARTITIONED_LOG,
+	OPT_MUTATION_LOG_TYPE,
 	OPT_MODE,
 
 	// Backup and Restore constants
@@ -147,6 +146,7 @@ enum {
 	OPT_BACKUPKEYS_FILTER,
 	OPT_INCREMENTALONLY,
 	OPT_ENCRYPTION_KEY_FILE,
+	OPT_ENCRYPTION_BLOCK_SIZE,
 
 	// Backup Modify
 	OPT_MOD_ACTIVE_INTERVAL,
@@ -253,9 +253,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
 	{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
 	{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
 	{ OPT_PROXY, "--proxy", SO_REQ_SEP },
-	// Enable "-p" option after GA
-	// { OPT_USE_PARTITIONED_LOG, "-p",                 SO_NONE },
-	{ OPT_USE_PARTITIONED_LOG, "--partitioned-log-experimental", SO_NONE },
+	{ OPT_MUTATION_LOG_TYPE, "--mutation-log-type", SO_REQ_SEP },
 	{ OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP },
 	{ OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP },
 	{ OPT_INITIAL_SNAPSHOT_INTERVAL, "--initial-snapshot-interval", SO_REQ_SEP },
@@ -284,6 +282,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
 	{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
 	{ OPT_INCREMENTALONLY, "--incremental", SO_NONE },
 	{ OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP },
+	{ OPT_ENCRYPTION_BLOCK_SIZE, "--encryption-block-size", SO_REQ_SEP },
 	{ OPT_MODE, "--mode", SO_REQ_SEP },
 	TLS_OPTION_FLAGS,
 	SO_END_OF_OPTIONS
@@ -696,7 +695,7 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
 	SO_END_OF_OPTIONS
 };
 
-// g_rgRestoreOptions is used by fdbrestore and fastrestore_tool
+// g_rgRestoreOptions is used by fdbrestore
 CSimpleOpt::SOption g_rgRestoreOptions[] = {
 #ifdef _WIN32
 	{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
@@ -932,7 +931,6 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
 const KeyRef exeAgent = "backup_agent"_sr;
 const KeyRef exeBackup = "fdbbackup"_sr;
 const KeyRef exeRestore = "fdbrestore"_sr;
-const KeyRef exeFastRestoreTool = "fastrestore_tool"_sr; // must be lower case
 const KeyRef exeDatabaseAgent = "dr_agent"_sr;
 const KeyRef exeDatabaseBackup = "fdbdr"_sr;
 
@@ -1115,7 +1113,10 @@ static void printBackupUsage(bool devhelp) {
 	       "                 If not specified, the entire database will be backed up or no filter will be applied.\n");
 	printf("  --keys-file FILE\n"
 	       "                 Same as -k option, except keys are specified in the input file.\n");
-	printf("  --partitioned-log-experimental  Starts with new type of backup system using partitioned logs.\n");
+	printf("  --mutation-log-type TYPE\n"
+	       "                 Specifies the mutation log type. Valid values are: "
+	       "partitioned-log-experimental, range-partitioned-log-experimental.\n"
+	       "If not specified, default log type is used.\n");
 	printf("  -n, --dryrun   For backup start or restore start, performs a trial run with no actual changes made.\n");
 	printf("  --log          Enables trace file logging for the CLI session.\n"
 	       "  --logdir PATH  Specifies the output directory for trace files. If\n"
@@ -1141,6 +1142,9 @@ static void printBackupUsage(bool devhelp) {
 	       "                 For modify operations, need to pass encryption key file only if Backup container URL is "
 	       "changed to "
 	       "re-encrypt all future backup files. \n");
+	printf("  --encryption-block-size"
+	       "                 Block size in bytes for file encryption. Only used with fdbbackup start command. Default "
+	       "is 1048576 (1MB).\n");
 
 	printf(TLS_HELP);
 	printf("  -w, --wait     Wait for the backup to complete (allowed with `start' and `discontinue').\n");
@@ -1257,14 +1261,6 @@ static void printRestoreUsage(bool devhelp) {
 	return;
 }
 
-static void printFastRestoreUsage(bool devhelp) {
-	printf(" NOTE: Fast restore aims to support the same fdbrestore option list.\n");
-	printf("       But fast restore is still under development. The options may not be fully supported.\n");
-	printf(" Supported options are: --dest-cluster-file, -r, --waitfordone, --logdir\n");
-	printRestoreUsage(devhelp);
-	return;
-}
-
 static void printDBAgentUsage(bool devhelp) {
 	printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
 	printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str());
@@ -1367,9 +1363,6 @@ static void printUsage(ProgramExe programExe, bool devhelp) {
 	case ProgramExe::RESTORE:
 		printRestoreUsage(devhelp);
 		break;
-	case ProgramExe::FASTRESTORE_TOOL:
-		printFastRestoreUsage(devhelp);
-		break;
 	case ProgramExe::DR_AGENT:
 		printDBAgentUsage(devhelp);
 		break;
@@ -1429,14 +1422,6 @@ ProgramExe getProgramType(std::string programExe) {
 		enProgramExe = ProgramExe::RESTORE;
 	}
 
-	// Check if restore
-	else if ((programExe.length() >= exeFastRestoreTool.size()) &&
-	         (programExe.compare(programExe.length() - exeFastRestoreTool.size(),
-	                             exeFastRestoreTool.size(),
-	                             (const char*)exeFastRestoreTool.begin()) == 0)) {
-		enProgramExe = ProgramExe::FASTRESTORE_TOOL;
-	}
-
 	// Check if db agent
 	else if ((programExe.length() >= exeDatabaseAgent.size()) &&
 	         (programExe.compare(programExe.length() - exeDatabaseAgent.size(),
@@ -1510,6 +1495,16 @@ Optional<RestoreMode> getRestoreMode(std::string mode) {
 	return Optional<RestoreMode>();
 }
 
+Optional<MutationLogType> getMutationLogType(std::string type) {
+	std::transform(type.begin(), type.end(), type.begin(), ::tolower);
+
+	if (type == "partitioned-log-experimental")
+		return MutationLogType::PARTITIONED_LOG;
+	if (type == "range-partitioned-log-experimental")
+		return MutationLogType::RANGE_PARTITIONED_LOG;
+	return Optional<MutationLogType>();
+}
+
 RestoreType getRestoreType(std::string name) {
 	if (name == "start")
 		return RestoreType::START;
@@ -1544,34 +1539,34 @@ DBType getDBType(std::string dbType) {
 	return enBackupType;
 }
 
-ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr,
-                                         IPAddress localIP,
-                                         std::string name,
-                                         std::string id,
-                                         ProgramExe exe,
-                                         Database dest,
-                                         Snapshot snapshot = Snapshot::False) {
+AsyncResult<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr,
+                                        IPAddress localIP,
+                                        std::string name,
+                                        std::string id,
+                                        ProgramExe exe,
+                                        Database dest,
+                                        Snapshot snapshot = Snapshot::False) {
 	// This process will write a document that looks like this:
 	// { backup : { $expires : {<subdoc>}, version: <version from approximately 30 seconds from now> }
 	// so that the value under 'backup' will eventually expire to null and thus be ignored by
 	// readers of status.  This is because if all agents die then they can no longer clean up old
 	// status docs from other dead agents.
 
-	state Version readVer = wait(tr->getReadVersion());
+	Version readVer = co_await tr->getReadVersion();
 
-	state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root
+	json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root
 	JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root
 	JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation
 	// Create the $expires key which is where the rest of the status output will go
 
-	state JSONDoc layerRoot = op.subDoc("$expires");
+	JSONDoc layerRoot = op.subDoc("$expires");
 	// Set the version argument in the $expires operator object.
 	op.create("version") = readVer + 120 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
 
 	layerRoot.create("instances_running.$sum") = 1;
 	layerRoot.create("last_updated.$max") = now();
 
-	state JSONDoc o = layerRoot.subDoc("instances." + id);
+	JSONDoc o = layerRoot.subDoc("instances." + id);
 
 	o.create("version") = FDB_VT_VERSION;
 	o.create("id") = id;
@@ -1586,12 +1581,12 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 	o.create("networkAddress") = localIP.toString();
 
 	if (exe == ProgramExe::AGENT) {
-		static S3BlobStoreEndpoint::Stats last_stats;
+		static IBlobStoreEndpoint::Stats last_stats;
 		static double last_ts = 0;
-		S3BlobStoreEndpoint::Stats current_stats = S3BlobStoreEndpoint::s_stats;
+		IBlobStoreEndpoint::Stats current_stats = IBlobStoreEndpoint::s_stats;
 		JSONDoc blobstats = o.create("blob_stats");
 		blobstats.create("total") = current_stats.getJSON();
-		S3BlobStoreEndpoint::Stats diff = current_stats - last_stats;
+		IBlobStoreEndpoint::Stats diff = current_stats - last_stats;
 		json_spirit::mObject diffObj = diff.getJSON();
 		if (last_ts > 0)
 			diffObj["bytes_per_second"] = double(current_stats.bytes_sent - last_stats.bytes_sent) / (now() - last_ts);
@@ -1603,21 +1598,21 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 		for (auto& p : diffObj)
 			totalBlobStats.create(p.first + ".$sum") = p.second;
 
-		state FileBackupAgent fba;
-		state std::vector<KeyBackedTag> backupTags = wait(getAllBackupTags(tr, snapshot));
-		state std::vector<Future<Optional<Version>>> tagLastRestorableVersions;
-		state std::vector<Future<EBackupState>> tagStates;
-		state std::vector<Future<Reference<IBackupContainer>>> tagContainers;
-		state std::vector<Future<int64_t>> tagRangeBytes;
-		state std::vector<Future<int64_t>> tagLogBytes;
-		state Future<Optional<Value>> fBackupPaused = tr->get(fba.taskBucket->getPauseKey(), snapshot);
+		FileBackupAgent fba;
+		std::vector<KeyBackedTag> backupTags = co_await getAllBackupTags(tr, snapshot);
+		std::vector<Future<Optional<Version>>> tagLastRestorableVersions;
+		std::vector<Future<EBackupState>> tagStates;
+		std::vector<Future<Reference<IBackupContainer>>> tagContainers;
+		std::vector<Future<int64_t>> tagRangeBytes;
+		std::vector<Future<int64_t>> tagLogBytes;
+		Future<Optional<Value>> fBackupPaused = tr->get(fba.taskBucket->getPauseKey(), snapshot);
 
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state std::vector<KeyBackedTag>::iterator tag;
-		state std::vector<UID> backupTagUids;
+		std::vector<KeyBackedTag>::iterator tag;
+		std::vector<UID> backupTagUids;
 		for (tag = backupTags.begin(); tag != backupTags.end(); tag++) {
-			UidAndAbortedFlagT uidAndAbortedFlag = wait(tag->getOrThrow(tr, snapshot));
+			UidAndAbortedFlagT uidAndAbortedFlag = co_await tag->getOrThrow(tr, snapshot);
 			BackupConfig config(uidAndAbortedFlag.first);
 			backupTagUids.push_back(config.getUid());
 
@@ -1628,11 +1623,11 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 			tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName), snapshot));
 		}
 
-		wait(waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) &&
-		     waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused));
+		co_await (waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) &&
+		          waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused));
 
-		state std::vector<Future<Void>> encryptionSetupResults;
-		state std::vector<int> encryptionContainerIndices;
+		std::vector<Future<Void>> encryptionSetupResults;
+		std::vector<int> encryptionContainerIndices;
 
 		for (int i = 0; i < tagContainers.size(); i++) {
 			if (tagContainers[i].get()->getEncryptionKeyFileName().present()) {
@@ -1640,14 +1635,14 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 				encryptionContainerIndices.push_back(i);
 			}
 		}
-		wait(waitForAllReady(encryptionSetupResults));
+		co_await waitForAllReady(encryptionSetupResults);
 		json_spirit::mArray keysArr;
 		std::unordered_set<std::string> seenKeyPaths;
 		for (int j = 0; j < encryptionContainerIndices.size() && j < 1e6; j++) {
 			int i = encryptionContainerIndices[j];
 			std::string keyPath = tagContainers[i].get()->getEncryptionKeyFileName().get();
 
-			if (seenKeyPaths.find(keyPath) == seenKeyPaths.end()) {
+			if (!seenKeyPaths.contains(keyPath)) {
 				seenKeyPaths.insert(keyPath);
 				json_spirit::mObject keyObj;
 				keyObj["path"] = tagContainers[i].get()->getEncryptionKeyFileName().get();
@@ -1657,19 +1652,19 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 		}
 		o.create("encryption_keys") = keysArr;
 
-		JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest");
+		JSONDoc tagsRoot = layerRoot.subDoc("tags");
 		layerRoot.create("tags.timestamp") = now();
 		layerRoot.create("total_workers.$sum") =
 		    fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT;
 		layerRoot.create("paused.$latest") = fBackupPaused.get().present();
 
 		int j = 0;
-		for (KeyBackedTag eachTag : backupTags) {
+		for (const KeyBackedTag& eachTag : backupTags) {
 			EBackupState status = tagStates[j].get();
 			const char* statusText = fba.getStateText(status);
 
 			// The object for this backup tag inside this instance's subdocument
-			JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName);
+			JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName).subDoc("$latest");
 			tagRoot.create("current_container") = tagContainers[j].get()->getURL();
 			tagRoot.create("current_status") = statusText;
 			if (tagLastRestorableVersions[j].get().present()) {
@@ -1693,18 +1688,18 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 			j++;
 		}
 	} else if (exe == ProgramExe::DR_AGENT) {
-		state DatabaseBackupAgent dba;
-		state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(dest));
+		DatabaseBackupAgent dba;
+		Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(dest));
 		tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr2->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state RangeResult tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000, snapshot));
-		state std::vector<Future<Optional<Key>>> backupVersion;
-		state std::vector<Future<EBackupState>> backupStatus;
-		state std::vector<Future<int64_t>> tagRangeBytesDR;
-		state std::vector<Future<int64_t>> tagLogBytesDR;
-		state Future<Optional<Value>> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot);
-
-		state std::vector<UID> drTagUids;
+		RangeResult tagNames = co_await tr2->getRange(dba.tagNames.range(), 10000, snapshot);
+		std::vector<Future<Optional<Key>>> backupVersion;
+		std::vector<Future<EBackupState>> backupStatus;
+		std::vector<Future<int64_t>> tagRangeBytesDR;
+		std::vector<Future<int64_t>> tagLogBytesDR;
+		Future<Optional<Value>> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot);
+
+		std::vector<UID> drTagUids;
 		for (int i = 0; i < tagNames.size(); i++) {
 			backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin), snapshot));
 			UID tagUID = BinaryReader::fromStringRef<UID>(tagNames[i].value, Unversioned());
@@ -1714,10 +1709,10 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 			tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID, snapshot));
 		}
 
-		wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) &&
-		     waitForAll(tagLogBytesDR) && success(fDRPaused));
+		co_await (waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) &&
+		          waitForAll(tagLogBytesDR) && success(fDRPaused));
 
-		JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest");
+		JSONDoc tagsRoot = layerRoot.subDoc("tags");
 		layerRoot.create("tags.timestamp") = now();
 		layerRoot.create("total_workers.$sum") = fDRPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT;
 		layerRoot.create("paused.$latest") = fDRPaused.get().present();
@@ -1727,7 +1722,7 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 
 			auto status = backupStatus[i].get();
 
-			JSONDoc tagRoot = tagsRoot.create(tagName);
+			JSONDoc tagRoot = tagsRoot.subDoc(tagName).subDoc("$latest");
 			tagRoot.create("running_backup") =
 			    (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING);
 			tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL);
@@ -1748,21 +1743,20 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 	}
 
 	std::string json = json_spirit::write_string(layersRootValue);
-	return json;
+	co_return json;
 }
 
 // Check for unparsable or expired statuses and delete them.
 // First checks the first doc in the key range, and if it is valid, alive and not "me" then
 // returns.  Otherwise, checks the rest of the range as well.
-ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
-                                 std::string rootKey,
-                                 std::string name,
-                                 std::string id,
-                                 int limit = 1) {
-	state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True));
-	state bool readMore = false;
-	state int i;
-	for (i = 0; i < docs.size(); ++i) {
+Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
+                           std::string rootKey,
+                           std::string name,
+                           std::string id,
+                           int limit = 1) {
+	RangeResult docs = co_await tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True);
+	bool readMore = false;
+	for (int i = 0; i < docs.size(); ++i) {
 		json_spirit::mValue docValue;
 		try {
 			json_spirit::read_string(docs[i].value.toString(), docValue);
@@ -1788,52 +1782,52 @@ ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr,
 		}
 		if (readMore) {
 			limit = 10000;
-			RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True));
+			RangeResult docs2 = co_await tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True);
 			docs = std::move(docs2);
 			readMore = false;
 		}
 	}
-
-	return Void();
 }
 
 // Get layer status document for just this layer
-ACTOR Future<json_spirit::mObject> getLayerStatus(Database src, std::string rootKey) {
-	state Transaction tr(src);
+AsyncResult<json_spirit::mObject> getLayerStatus(Database src, std::string rootKey) {
+	Transaction tr(src);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			state RangeResult kvPairs =
-			    wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), GetRangeLimits::ROW_LIMIT_UNLIMITED));
-			state json_spirit::mObject statusDoc;
-			state JSONDoc modifier(statusDoc);
+			RangeResult kvPairs =
+			    co_await tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), GetRangeLimits::ROW_LIMIT_UNLIMITED);
+			json_spirit::mObject statusDoc;
+			JSONDoc modifier(statusDoc);
 			for (auto& kv : kvPairs) {
-				state json_spirit::mValue docValue;
+				json_spirit::mValue docValue;
 				json_spirit::read_string(kv.value.toString(), docValue);
-				wait(yield());
+				co_await yield();
 				modifier.absorb(docValue);
-				wait(yield());
+				co_await yield();
 			}
 			JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get();
 			modifier.cleanOps();
-			return statusDoc;
+			co_return statusDoc;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
 // Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay
 // based on that and BACKUP_AGGREGATE_POLL_RATE
-ACTOR Future<Void> updateAgentPollRate(Database src,
-                                       std::string rootKey,
-                                       std::string name,
-                                       std::shared_ptr<double> pollDelay) {
-	loop {
+Future<Void> updateAgentPollRate(Database src,
+                                 std::string rootKey,
+                                 std::string name,
+                                 std::shared_ptr<double> pollDelay) {
+	while (true) {
 		try {
-			json_spirit::mObject status = wait(getLayerStatus(src, rootKey));
+			json_spirit::mObject status = co_await getLayerStatus(src, rootKey);
 			int64_t processes = 0;
 			// If instances count is present and greater than 0 then update pollDelay
 			if (JSONDoc(status).tryGet<int64_t>(name + ".instances_running", processes) && processes > 0) {
@@ -1845,161 +1839,171 @@ ACTOR Future<Void> updateAgentPollRate(Database src,
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "BackupAgentPollRateUpdateError").error(e);
 		}
-		wait(delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL));
+		co_await delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL);
 	}
 }
 
-ACTOR Future<Void> statusUpdateActor(Database statusUpdateDest,
-                                     std::string name,
-                                     ProgramExe exe,
-                                     std::shared_ptr<double> pollDelay,
-                                     Database taskDest = Database(),
-                                     std::string id = nondeterministicRandom()->randomUniqueID().toString()) {
-	state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name;
-	state std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json";
-	state std::string instanceKey = rootKey + "/" + "agent-" + id;
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(statusUpdateDest));
-	state Future<Void> pollRateUpdater;
+Future<Void> statusUpdateActor(Database statusUpdateDest,
+                               std::string name,
+                               ProgramExe exe,
+                               std::shared_ptr<double> pollDelay,
+                               Database taskDest = Database(),
+                               std::string id = nondeterministicRandom()->randomUniqueID().toString()) {
+	std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name;
+	std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json";
+	std::string instanceKey = rootKey + "/" + "agent-" + id;
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(statusUpdateDest));
+	Future<Void> pollRateUpdater;
 
 	// In order to report a useful networkAddress to the cluster's layer status JSON object, determine which local
 	// network interface IP will be used to talk to the cluster.  This is a blocking call, so it is only done once,
 	// and in a retry loop because if we can't connect to the cluster we can't do any work anyway.
-	state IPAddress localIP;
+	IPAddress localIP;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			localIP = statusUpdateDest->getConnectionRecord()->getConnectionString().determineLocalSourceIP();
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "AgentCouldNotDetermineLocalIP").error(e);
-			wait(delay(1.0));
+			err = e;
 		}
+		TraceEvent(SevWarn, "AgentCouldNotDetermineLocalIP").error(err);
+		co_await delay(1.0);
 	}
 
 	// Register the existence of this layer in the meta key space
-	loop {
+	while (true) {
 		tr->reset();
+		Error err;
 		try {
-			loop {
+			while (true) {
+				Error innerErr;
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 					tr->set(metaKey, rootKey);
-					wait(tr->commit());
+					co_await tr->commit();
 					break;
 				} catch (Error& e) {
-					TraceEvent(SevWarnAlways, "LayerStatusMetaKeyUpdateError").errorUnsuppressed(e);
-					wait(tr->onError(e)); // Non-retryable txns throws back the error.
+					innerErr = e;
 				}
+				TraceEvent(SevWarnAlways, "LayerStatusMetaKeyUpdateError").errorUnsuppressed(innerErr);
+				co_await tr->onError(innerErr); // Non-retryable txns throws back the error.
 			}
 			break;
 		} catch (Error& e) {
-			// For non-retryable txns, do delay, reset txn and retry
-			TraceEvent(SevWarnAlways, "UnableToWriteLayerStatusMetaKey").errorUnsuppressed(e);
-			wait(delay(5.0));
+			err = e;
 		}
+		// For non-retryable txns, do delay, reset txn and retry
+		TraceEvent(SevWarnAlways, "UnableToWriteLayerStatusMetaKey").errorUnsuppressed(err);
+		co_await delay(5.0);
 	}
 
 	// Write status periodically
-	loop {
+	while (true) {
 		tr->reset();
+		Error err;
 		try {
-			loop {
+			while (true) {
+				Error innerErr;
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-					state Future<std::string> futureStatusDoc =
+					AsyncResult<std::string> asyncResultStatusDoc =
 					    getLayerStatus(tr, localIP, name, id, exe, taskDest, Snapshot::True);
-					wait(cleanupStatus(tr, rootKey, name, id));
-					std::string statusdoc = wait(futureStatusDoc);
+					co_await cleanupStatus(tr, rootKey, name, id);
+					std::string statusdoc = co_await std::move(asyncResultStatusDoc);
 					tr->set(instanceKey, statusdoc);
-					wait(tr->commit());
+					co_await tr->commit();
 					break;
 				} catch (Error& e) {
-					TraceEvent(SevWarnAlways, "LayerBackupStatusUpdateError").errorUnsuppressed(e);
-					wait(tr->onError(e));
+					innerErr = e;
 				}
+				TraceEvent(SevWarnAlways, "LayerBackupStatusUpdateError").errorUnsuppressed(innerErr);
+				co_await tr->onError(innerErr);
 			}
 
-			wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY *
-			           ((1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER) +
-			            2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER)));
+			co_await delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY *
+			               ((1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER) +
+			                2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER));
 
 			// Now that status was written at least once by this process (and hopefully others), start the poll rate
 			// control updater if it wasn't started yet
 			if (!pollRateUpdater.isValid())
 				pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay);
+			continue;
 		} catch (Error& e) {
-			TraceEvent(SevWarnAlways, "UnableToWriteBackupStatus").error(e);
-			wait(delay(10.0));
+			err = e;
 		}
+		TraceEvent(SevWarnAlways, "UnableToWriteBackupStatus").error(err);
+		co_await delay(10.0);
 	}
 }
 
-ACTOR Future<Void> runDBAgent(Database src, Database dest) {
-	state std::shared_ptr<double> pollDelay = std::make_shared<double>(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE);
+Future<Void> runDBAgent(Database src, Database dest) {
+	std::shared_ptr<double> pollDelay = std::make_shared<double>(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE);
 	std::string id = nondeterministicRandom()->randomUniqueID().toString();
-	state Future<Void> status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id);
-	state Future<Void> status_other =
-	    statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id);
+	Future<Void> status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id);
+	Future<Void> status_other = statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id);
 
-	state DatabaseBackupAgent backupAgent(src);
+	DatabaseBackupAgent backupAgent(src);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			wait(backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
+			co_await backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT);
 			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_operation_cancelled)
-				throw;
+			err = e;
+		}
+		if (err.code() == error_code_operation_cancelled)
+			throw err;
 
-			TraceEvent(SevError, "DA_runAgent").error(e);
-			fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", e.what());
+		TraceEvent(SevError, "DA_runAgent").error(err);
+		fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", err.what());
 
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-		}
+		co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> runAgent(Database db) {
-	state std::shared_ptr<double> pollDelay = std::make_shared<double>(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE);
-	state Future<Void> status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay);
+Future<Void> runAgent(Database db) {
+	std::shared_ptr<double> pollDelay = std::make_shared<double>(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE);
+	Future<Void> status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay);
 
-	state FileBackupAgent backupAgent;
+	FileBackupAgent backupAgent;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			wait(backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
+			co_await backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT);
 			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_operation_cancelled)
-				throw;
+			err = e;
+		}
+		if (err.code() == error_code_operation_cancelled)
+			throw err;
 
-			TraceEvent(SevError, "BA_runAgent").error(e);
-			fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", e.what());
+		TraceEvent(SevError, "BA_runAgent").error(err);
+		fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", err.what());
 
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-		}
+		co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> submitDBBackup(Database src,
-                                  Database dest,
-                                  Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                  std::string tagName) {
+Future<Void> submitDBBackup(Database src,
+                            Database dest,
+                            Standalone<VectorRef<KeyRangeRef>> backupRanges,
+                            std::string tagName) {
 	try {
-		state DatabaseBackupAgent backupAgent(src);
+		DatabaseBackupAgent backupAgent(src);
 		ASSERT(!backupRanges.empty());
 
-		wait(backupAgent.submitBackup(
-		    dest, KeyRef(tagName), backupRanges, StopWhenDone::False, StringRef(), StringRef(), LockDB::True));
+		co_await backupAgent.submitBackup(
+		    dest, KeyRef(tagName), backupRanges, StopWhenDone::False, StringRef(), StringRef(), LockDB::True);
 
 		// Check if a backup agent is running
-		bool agentRunning = wait(backupAgent.checkActive(dest));
+		bool agentRunning = co_await backupAgent.checkActive(dest);
 
 		if (!agentRunning) {
 			printf("The DR on tag `%s' was successfully submitted but no DR agents are responding.\n",
@@ -2029,35 +2033,34 @@ ACTOR Future<Void> submitDBBackup(Database src,
 
 		throw backup_error();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> submitBackup(Database db,
-                                std::string url,
-                                Optional<std::string> proxy,
-                                int initialSnapshotIntervalSeconds,
-                                int snapshotIntervalSeconds,
-                                Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                std::string tagName,
-                                bool dryRun,
-                                WaitForComplete waitForCompletion,
-                                StopWhenDone stopWhenDone,
-                                UsePartitionedLog usePartitionedLog,
-                                IncrementalBackupOnly incrementalBackupOnly,
-                                Optional<std::string> encryptionKeyFile,
-                                SnapshotMode snapshotMode = SnapshotMode::RANGEFILE) {
+Future<Void> submitBackup(Database db,
+                          std::string url,
+                          Optional<std::string> proxy,
+                          int initialSnapshotIntervalSeconds,
+                          int snapshotIntervalSeconds,
+                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
+                          std::string tagName,
+                          bool dryRun,
+                          WaitForComplete waitForCompletion,
+                          StopWhenDone stopWhenDone,
+                          MutationLogType mutationLogType,
+                          IncrementalBackupOnly incrementalBackupOnly,
+                          Optional<std::string> encryptionKeyFile,
+                          int encryptionBlockSize,
+                          SnapshotMode snapshotMode = SnapshotMode::RANGEFILE) {
 	try {
-		state FileBackupAgent backupAgent;
+		FileBackupAgent backupAgent;
 		ASSERT(!backupRanges.empty());
 
 		if (dryRun) {
-			state KeyBackedTag tag = makeBackupTag(tagName);
-			Optional<UidAndAbortedFlagT> uidFlag = wait(tag.get(db.getReference()));
+			KeyBackedTag tag = makeBackupTag(tagName);
+			Optional<UidAndAbortedFlagT> uidFlag = co_await tag.get(db.getReference());
 
 			if (uidFlag.present()) {
 				BackupConfig config(uidFlag.get().first);
-				EBackupState backupStatus = wait(config.stateEnum().getOrThrow(db.getReference()));
+				EBackupState backupStatus = co_await config.stateEnum().getOrThrow(db.getReference());
 
 				// Throw error if a backup is currently running until we support parallel backups
 				if (BackupAgentBase::isRunnable(backupStatus)) {
@@ -2072,7 +2075,7 @@ ACTOR Future<Void> submitBackup(Database db,
 
 			else {
 				// Check if a backup agent is running
-				bool agentRunning = wait(backupAgent.checkActive(db));
+				bool agentRunning = co_await backupAgent.checkActive(db);
 
 				if (!agentRunning) {
 					printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. "
@@ -2089,27 +2092,28 @@ ACTOR Future<Void> submitBackup(Database db,
 		}
 
 		else {
-			wait(backupAgent.submitBackup(db,
-			                              KeyRef(url),
-			                              proxy,
-			                              initialSnapshotIntervalSeconds,
-			                              snapshotIntervalSeconds,
-			                              tagName,
-			                              backupRanges,
-			                              stopWhenDone,
-			                              usePartitionedLog,
-			                              incrementalBackupOnly,
-			                              encryptionKeyFile,
-			                              static_cast<int>(snapshotMode)));
+			co_await backupAgent.submitBackup(db,
+			                                  KeyRef(url),
+			                                  proxy,
+			                                  initialSnapshotIntervalSeconds,
+			                                  snapshotIntervalSeconds,
+			                                  tagName,
+			                                  backupRanges,
+			                                  stopWhenDone,
+			                                  mutationLogType,
+			                                  incrementalBackupOnly,
+			                                  encryptionKeyFile,
+			                                  encryptionBlockSize,
+			                                  static_cast<int>(snapshotMode));
 
 			// Wait for the backup to complete, if requested
 			if (waitForCompletion) {
 				printf("Submitted and now waiting for the backup on tag `%s' to complete.\n",
 				       printable(StringRef(tagName)).c_str());
-				wait(success(backupAgent.waitBackup(db, tagName)));
+				co_await backupAgent.waitBackup(db, tagName);
 			} else {
 				// Check if a backup agent is running
-				bool agentRunning = wait(backupAgent.checkActive(db));
+				bool agentRunning = co_await backupAgent.checkActive(db);
 
 				if (!agentRunning) {
 					printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n",
@@ -2140,20 +2144,19 @@ ACTOR Future<Void> submitBackup(Database db,
 
 		throw backup_error();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> switchDBBackup(Database src,
-                                  Database dest,
-                                  Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                  std::string tagName,
-                                  ForceAction forceAction) {
+Future<Void> switchDBBackup(Database src,
+                            Database dest,
+                            Standalone<VectorRef<KeyRangeRef>> backupRanges,
+                            std::string tagName,
+                            ForceAction forceAction) {
 	try {
-		state DatabaseBackupAgent backupAgent(src);
+		DatabaseBackupAgent backupAgent(src);
 		ASSERT(!backupRanges.empty());
 
-		wait(backupAgent.atomicSwitchover(dest, KeyRef(tagName), backupRanges, StringRef(), StringRef(), forceAction));
+		co_await backupAgent.atomicSwitchover(
+		    dest, KeyRef(tagName), backupRanges, StringRef(), StringRef(), forceAction);
 		printf("The DR on tag `%s' was successfully switched.\n", printable(StringRef(tagName)).c_str());
 	}
 
@@ -2174,15 +2177,13 @@ ACTOR Future<Void> switchDBBackup(Database src,
 
 		throw backup_error();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) {
+Future<Void> statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) {
 	try {
-		state DatabaseBackupAgent backupAgent(src);
+		DatabaseBackupAgent backupAgent(src);
 
-		std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName)));
+		std::string statusText = co_await backupAgent.getStatus(dest, errorLimit, StringRef(tagName));
 		printf("%s\n", statusText.c_str());
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
@@ -2190,16 +2191,14 @@ ACTOR Future<Void> statusDBBackup(Database src, Database dest, std::string tagNa
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) {
+Future<Void> statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) {
 	try {
-		state FileBackupAgent backupAgent;
+		FileBackupAgent backupAgent;
 
 		std::string statusText =
-		    wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName));
+		    co_await (json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName));
 		printf("%s\n", statusText.c_str());
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
@@ -2207,20 +2206,14 @@ ACTOR Future<Void> statusBackup(Database db, std::string tagName, ShowErrors sho
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> abortDBBackup(Database src,
-                                 Database dest,
-                                 std::string tagName,
-                                 PartialBackup partial,
-                                 DstOnly dstOnly) {
+Future<Void> abortDBBackup(Database src, Database dest, std::string tagName, PartialBackup partial, DstOnly dstOnly) {
 	try {
-		state DatabaseBackupAgent backupAgent(src);
+		DatabaseBackupAgent backupAgent(src);
 
-		wait(backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::False, dstOnly));
-		wait(backupAgent.unlockBackup(dest, Key(tagName)));
+		co_await backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::False, dstOnly);
+		co_await backupAgent.unlockBackup(dest, Key(tagName));
 
 		printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str());
 	} catch (Error& e) {
@@ -2239,15 +2232,13 @@ ACTOR Future<Void> abortDBBackup(Database src,
 		}
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> abortBackup(Database db, std::string tagName) {
+Future<Void> abortBackup(Database db, std::string tagName) {
 	try {
-		state FileBackupAgent backupAgent;
+		FileBackupAgent backupAgent;
 
-		wait(backupAgent.abortBackup(db, tagName));
+		co_await backupAgent.abortBackup(db, tagName);
 
 		printf("The backup on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str());
 	} catch (Error& e) {
@@ -2266,28 +2257,24 @@ ACTOR Future<Void> abortBackup(Database db, std::string tagName) {
 		}
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> cleanupMutations(Database db, DeleteData deleteData) {
+Future<Void> cleanupMutations(Database db, DeleteData deleteData) {
 	try {
-		wait(cleanupBackup(db, deleteData));
+		co_await cleanupBackup(db, deleteData);
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
 			throw;
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) {
+Future<Void> waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) {
 	try {
-		state FileBackupAgent backupAgent;
+		FileBackupAgent backupAgent;
 
-		EBackupState status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone));
+		EBackupState status = co_await backupAgent.waitBackup(db, tagName, stopWhenDone);
 
 		printf("The backup on tag `%s' %s.\n",
 		       printable(StringRef(tagName)).c_str(),
@@ -2298,21 +2285,19 @@ ACTOR Future<Void> waitBackup(Database db, std::string tagName, StopWhenDone sto
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) {
+Future<Void> discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) {
 	try {
-		state FileBackupAgent backupAgent;
+		FileBackupAgent backupAgent;
 
-		wait(backupAgent.discontinueBackup(db, StringRef(tagName)));
+		co_await backupAgent.discontinueBackup(db, StringRef(tagName));
 
 		// Wait for the backup to complete, if requested
 		if (waitForCompletion) {
 			printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n",
 			       printable(StringRef(tagName)).c_str());
-			wait(success(backupAgent.waitBackup(db, tagName)));
+			co_await backupAgent.waitBackup(db, tagName);
 		} else {
 			printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str());
 		}
@@ -2338,14 +2323,12 @@ ACTOR Future<Void> discontinueBackup(Database db, std::string tagName, WaitForCo
 		}
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> changeBackupResumed(Database db, bool pause) {
+Future<Void> changeBackupResumed(Database db, bool pause) {
 	try {
 		FileBackupAgent backupAgent;
-		wait(backupAgent.changePause(db, pause));
+		co_await backupAgent.changePause(db, pause);
 		printf("All backup agents have been %s.\n", pause ? "paused" : "resumed");
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
@@ -2353,14 +2336,12 @@ ACTOR Future<Void> changeBackupResumed(Database db, bool pause) {
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> changeDBBackupResumed(Database src, Database dest, bool pause) {
+Future<Void> changeDBBackupResumed(Database src, Database dest, bool pause) {
 	try {
-		state DatabaseBackupAgent backupAgent(src);
-		wait(backupAgent.taskBucket->changePause(dest, pause));
+		DatabaseBackupAgent backupAgent(src);
+		co_await backupAgent.taskBucket->changePause(dest, pause);
 		printf("All DR agents have been %s.\n", pause ? "paused" : "resumed");
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
@@ -2368,14 +2349,13 @@ ACTOR Future<Void> changeDBBackupResumed(Database src, Database dest, bool pause
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
 Reference<IBackupContainer> openBackupContainer(const char* name,
                                                 const std::string& destinationContainer,
                                                 const Optional<std::string>& proxy,
-                                                const Optional<std::string>& encryptionKeyFile) {
+                                                const Optional<std::string>& encryptionKeyFile,
+                                                int encryptionBlockSize) {
 	// Error, if no dest container was specified
 	if (destinationContainer.empty()) {
 		fprintf(stderr, "ERROR: No backup destination was specified.\n");
@@ -2392,7 +2372,7 @@ Reference<IBackupContainer> openBackupContainer(const char* name,
 
 	Reference<IBackupContainer> c;
 	try {
-		c = IBackupContainer::openContainer(destinationContainer, proxy, encryptionKeyFile);
+		c = IBackupContainer::openContainer(destinationContainer, proxy, encryptionKeyFile, encryptionBlockSize);
 	} catch (Error& e) {
 		std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str());
 		if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) {
@@ -2408,24 +2388,24 @@ Reference<IBackupContainer> openBackupContainer(const char* name,
 
 // Submit the restore request to the database if "performRestore" is true. Otherwise,
 // check if the restore can be performed.
-ACTOR Future<Void> runRestore(Database db,
-                              std::string originalClusterFile,
-                              std::string tagName,
-                              std::string container,
-                              Optional<std::string> proxy,
-                              Standalone<VectorRef<KeyRangeRef>> ranges,
-                              Version beginVersion,
-                              Version targetVersion,
-                              std::string targetTimestamp,
-                              bool performRestore,
-                              Verbose verbose,
-                              WaitForComplete waitForDone,
-                              std::string addPrefix,
-                              std::string removePrefix,
-                              OnlyApplyMutationLogs onlyApplyMutationLogs,
-                              InconsistentSnapshotOnly inconsistentSnapshotOnly,
-                              Optional<std::string> encryptionKeyFile,
-                              RestoreMode restoreMode = RestoreMode::RANGEFILE) {
+Future<Void> runRestore(Database db,
+                        std::string originalClusterFile,
+                        std::string tagName,
+                        std::string container,
+                        Optional<std::string> proxy,
+                        Standalone<VectorRef<KeyRangeRef>> ranges,
+                        Version beginVersion,
+                        Version targetVersion,
+                        std::string targetTimestamp,
+                        bool performRestore,
+                        Verbose verbose,
+                        WaitForComplete waitForDone,
+                        std::string addPrefix,
+                        std::string removePrefix,
+                        OnlyApplyMutationLogs onlyApplyMutationLogs,
+                        InconsistentSnapshotOnly inconsistentSnapshotOnly,
+                        Optional<std::string> encryptionKeyFile,
+                        RestoreMode restoreMode = RestoreMode::RANGEFILE) {
 	ASSERT(!ranges.empty());
 
 	if (targetVersion != invalidVersion && !targetTimestamp.empty()) {
@@ -2433,7 +2413,7 @@ ACTOR Future<Void> runRestore(Database db,
 		throw restore_error();
 	}
 
-	state Optional<Database> origDb;
+	Optional<Database> origDb;
 
 	// Resolve targetTimestamp if given
 	if (!targetTimestamp.empty()) {
@@ -2451,17 +2431,18 @@ ACTOR Future<Void> runRestore(Database db,
 		}
 
 		origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION);
-		Version v = wait(timeKeeperVersionFromDatetime(targetTimestamp, origDb.get()));
+		Version v = co_await timeKeeperVersionFromDatetime(targetTimestamp, origDb.get());
 		fmt::print("Timestamp '{0}' resolves to version {1}\n", targetTimestamp, v);
 		targetVersion = v;
 	}
 
 	try {
-		state FileBackupAgent backupAgent;
-
-		state Reference<IBackupContainer> bc =
-		    openBackupContainer(exeRestore.toString().c_str(), container, proxy, encryptionKeyFile);
+		FileBackupAgent backupAgent;
 
+		// encryptionBlockSize is passed 0 because we don't know about the block size yet and it will be read in the
+		// describeBackup call after this.
+		Reference<IBackupContainer> bc = openBackupContainer(
+		    exeRestore.toString().c_str(), container, proxy, encryptionKeyFile, /*encryptionBlockSize=*/0);
 		// If targetVersion is unset then use the maximum restorable version from the backup description
 		if (targetVersion == invalidVersion) {
 			if (verbose)
@@ -2469,7 +2450,7 @@ ACTOR Future<Void> runRestore(Database db,
 				    "No restore target version given, will use maximum restorable version from backup description.\n");
 
 			// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
-			BackupDescription desc = wait(bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0));
+			BackupDescription desc = co_await bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0);
 
 			if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) {
 				targetVersion = desc.contiguousLogEnd.get() - 1;
@@ -2486,32 +2467,32 @@ ACTOR Future<Void> runRestore(Database db,
 		}
 
 		if (performRestore) {
-			Version restoredVersion = wait(backupAgent.restore(db,
-			                                                   origDb,
-			                                                   KeyRef(tagName),
-			                                                   KeyRef(container),
-			                                                   proxy,
-			                                                   ranges,
-			                                                   waitForDone,
-			                                                   targetVersion,
-			                                                   verbose,
-			                                                   KeyRef(addPrefix),
-			                                                   KeyRef(removePrefix),
-			                                                   LockDB::True,
-			                                                   UnlockDB::True,
-			                                                   onlyApplyMutationLogs,
-			                                                   inconsistentSnapshotOnly,
-			                                                   beginVersion,
-			                                                   encryptionKeyFile,
-			                                                   {},
-			                                                   restoreMode == RestoreMode::RANGEFILE));
+			Version restoredVersion = co_await backupAgent.restore(db,
+			                                                       origDb,
+			                                                       KeyRef(tagName),
+			                                                       KeyRef(container),
+			                                                       proxy,
+			                                                       ranges,
+			                                                       waitForDone,
+			                                                       targetVersion,
+			                                                       verbose,
+			                                                       KeyRef(addPrefix),
+			                                                       KeyRef(removePrefix),
+			                                                       LockDB::True,
+			                                                       UnlockDB::True,
+			                                                       onlyApplyMutationLogs,
+			                                                       inconsistentSnapshotOnly,
+			                                                       beginVersion,
+			                                                       encryptionKeyFile,
+			                                                       {},
+			                                                       restoreMode == RestoreMode::RANGEFILE);
 
 			if (waitForDone && verbose) {
 				// If restore is now complete then report version restored
 				fmt::print("Restored to version {}\n", restoredVersion);
 			}
 		} else {
-			state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(targetVersion, ranges));
+			Optional<RestorableFileSet> rset = co_await bc->getRestoreSet(targetVersion, ranges);
 
 			if (!rset.present()) {
 				fmt::print(stderr,
@@ -2529,139 +2510,17 @@ ACTOR Future<Void> runRestore(Database db,
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
-}
-
-// Fast restore agent that kicks off the restore: send restore requests to restore workers.
-ACTOR Future<Void> runFastRestoreTool(Database db,
-                                      std::string tagName,
-                                      std::string container,
-                                      Optional<std::string> proxy,
-                                      Standalone<VectorRef<KeyRangeRef>> ranges,
-                                      Version dbVersion,
-                                      bool performRestore,
-                                      Verbose verbose,
-                                      WaitForComplete waitForDone) {
-	try {
-		state FileBackupAgent backupAgent;
-		state Version restoreVersion = invalidVersion;
-
-		if (ranges.size() > 1) {
-			fprintf(stdout, "[WARNING] Currently only a single restore range is tested!\n");
-		}
-
-		if (ranges.size() == 0) {
-			ranges.push_back(ranges.arena(), normalKeys);
-		}
-
-		printf("[INFO] runFastRestoreTool: restore_ranges:%d first range:%s\n",
-		       ranges.size(),
-		       ranges.front().toString().c_str());
-		TraceEvent ev("FastRestoreTool");
-		ev.detail("RestoreRanges", ranges.size());
-		for (int i = 0; i < ranges.size(); ++i) {
-			ev.detail(format("Range%d", i), ranges[i]);
-		}
-
-		if (performRestore) {
-			if (dbVersion == invalidVersion) {
-				TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", "Largest restorable version");
-				// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
-				BackupDescription desc =
-				    wait(IBackupContainer::openContainer(container, proxy, {})
-				             ->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0));
-				if (!desc.maxRestorableVersion.present()) {
-					fprintf(stderr, "The specified backup is not restorable to any version.\n");
-					throw restore_error();
-				}
-
-				dbVersion = desc.maxRestorableVersion.get();
-				TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", dbVersion);
-			}
-			state UID randomUID = deterministicRandom()->randomUniqueID();
-			TraceEvent("FastRestoreTool")
-			    .detail("SubmitRestoreRequests", ranges.size())
-			    .detail("RestoreUID", randomUID);
-			wait(backupAgent.submitParallelRestore(db,
-			                                       KeyRef(tagName),
-			                                       ranges,
-			                                       KeyRef(container),
-			                                       proxy,
-			                                       dbVersion,
-			                                       LockDB::True,
-			                                       randomUID,
-			                                       ""_sr,
-			                                       ""_sr));
-			// TODO: Support addPrefix and removePrefix
-			if (waitForDone) {
-				// Wait for parallel restore to finish and unlock DB after that
-				TraceEvent("FastRestoreTool").detail("BackupAndParallelRestore", "WaitForRestoreToFinish");
-				wait(backupAgent.parallelRestoreFinish(db, randomUID));
-				TraceEvent("FastRestoreTool").detail("BackupAndParallelRestore", "RestoreFinished");
-			} else {
-				TraceEvent("FastRestoreTool")
-				    .detail("RestoreUID", randomUID)
-				    .detail("OperationGuide", "Manually unlock DB when restore finishes");
-				printf("WARNING: DB will be in locked state after restore. Need UID:%s to unlock DB\n",
-				       randomUID.toString().c_str());
-			}
-
-			restoreVersion = dbVersion;
-		} else {
-			state Reference<IBackupContainer> bc = IBackupContainer::openContainer(container, proxy, {});
-			// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
-			state BackupDescription description =
-			    wait(bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0));
-
-			if (dbVersion <= 0) {
-				wait(description.resolveVersionTimes(db));
-				if (description.maxRestorableVersion.present())
-					restoreVersion = description.maxRestorableVersion.get();
-				else {
-					fprintf(stderr, "Backup is not restorable\n");
-					throw restore_invalid_version();
-				}
-			} else {
-				restoreVersion = dbVersion;
-			}
-
-			state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(restoreVersion));
-			if (!rset.present()) {
-				fmt::print(stderr, "Insufficient data to restore to version {}\n", restoreVersion);
-				throw restore_invalid_version();
-			}
-
-			// Display the restore information, if requested
-			if (verbose) {
-				fmt::print("[DRY RUN] Restoring backup to version: {}\n", restoreVersion);
-				fmt::print("{}\n", description.toString());
-			}
-		}
-
-		if (waitForDone && verbose) {
-			// If restore completed then report version restored
-			fmt::print("Restored to version {0}{1}\n", restoreVersion, (performRestore) ? "" : " (DRY RUN)");
-		}
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled)
-			throw;
-		fprintf(stderr, "ERROR: %s\n", e.what());
-		throw;
-	}
-
-	return Void();
 }
 
-ACTOR Future<Void> dumpBackupData(const char* name,
-                                  std::string destinationContainer,
-                                  Optional<std::string> proxy,
-                                  Version beginVersion,
-                                  Version endVersion) {
-	state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, {});
+Future<Void> dumpBackupData(const char* name,
+                            std::string destinationContainer,
+                            Optional<std::string> proxy,
+                            Version beginVersion,
+                            Version endVersion) {
+	Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, {}, 0);
 
 	if (beginVersion < 0 || endVersion < 0) {
-		BackupDescription desc = wait(c->describeBackup());
+		BackupDescription desc = co_await c->describeBackup();
 
 		if (!desc.maxLogEnd.present()) {
 			fprintf(stderr, "ERROR: Backup must have log data in order to use relative begin/end versions.\n");
@@ -2678,29 +2537,26 @@ ACTOR Future<Void> dumpBackupData(const char* name,
 	}
 
 	fmt::print("Scanning version range {0} to {1}\n", beginVersion, endVersion);
-	BackupFileList files = wait(c->dumpFileList(beginVersion, endVersion));
+	BackupFileList files = co_await c->dumpFileList(beginVersion, endVersion);
 	files.toStream(stdout);
-
-	return Void();
 }
 
-ACTOR Future<Void> expireBackupData(const char* name,
-                                    std::string destinationContainer,
-                                    Optional<std::string> proxy,
-                                    Version endVersion,
-                                    std::string endDatetime,
-                                    Database db,
-                                    bool force,
-                                    Version restorableAfterVersion,
-                                    std::string restorableAfterDatetime,
-                                    Optional<std::string> encryptionKeyFile) {
+Future<Void> expireBackupData(const char* name,
+                              std::string destinationContainer,
+                              Optional<std::string> proxy,
+                              Version endVersion,
+                              std::string endDatetime,
+                              Database db,
+                              bool force,
+                              Version restorableAfterVersion,
+                              std::string restorableAfterDatetime) {
 	if (!endDatetime.empty()) {
-		Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db));
+		Version v = co_await timeKeeperVersionFromDatetime(endDatetime, db);
 		endVersion = v;
 	}
 
 	if (!restorableAfterDatetime.empty()) {
-		Version v = wait(timeKeeperVersionFromDatetime(restorableAfterDatetime, db));
+		Version v = co_await timeKeeperVersionFromDatetime(restorableAfterDatetime, db);
 		restorableAfterVersion = v;
 	}
 
@@ -2708,29 +2564,25 @@ ACTOR Future<Void> expireBackupData(const char* name,
 		fprintf(stderr, "ERROR: No version or date/time is specified.\n");
 		printHelpTeaser(name);
 		throw backup_error();
-		;
 	}
 
 	try {
-		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, encryptionKeyFile);
-
-		state IBackupContainer::ExpireProgress progress;
-		state std::string lastProgress;
-		state Future<Void> expire = c->expireData(endVersion, force, &progress, restorableAfterVersion);
-
-		loop {
-			choose {
-				when(wait(delay(5))) {
-					std::string p = progress.toString();
-					if (p != lastProgress) {
-						int spaces = lastProgress.size() - p.size();
-						printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : ""));
-						lastProgress = p;
-					}
-				}
-				when(wait(expire)) {
-					break;
-				}
+		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, {}, 0);
+
+		IBackupContainer::ExpireProgress progress;
+		std::string lastProgress;
+		Future<Void> expire = c->expireData(endVersion, force, &progress, restorableAfterVersion);
+
+		while (true) {
+			if (auto const res = co_await timeout(expire, 5); res.present()) {
+				break;
+			}
+
+			std::string p = progress.toString();
+			if (p != lastProgress) {
+				int spaces = lastProgress.size() - p.size();
+				printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : ""));
+				lastProgress = p;
 			}
 		}
 
@@ -2756,32 +2608,25 @@ ACTOR Future<Void> expireBackupData(const char* name,
 			fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> deleteBackupContainer(const char* name,
-                                         std::string destinationContainer,
-                                         Optional<std::string> proxy) {
+Future<Void> deleteBackupContainer(const char* name, std::string destinationContainer, Optional<std::string> proxy) {
 	try {
-		state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, {});
-		state int numDeleted = 0;
-		state Future<Void> done = c->deleteContainer(&numDeleted);
+		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, {}, 0);
+		int numDeleted = 0;
+		Future<Void> done = c->deleteContainer(&numDeleted);
 
-		state int lastUpdate = -1;
+		int lastUpdate = -1;
 		printf("Deleting %s...\n", destinationContainer.c_str());
 
-		loop {
-			choose {
-				when(wait(done)) {
-					break;
-				}
-				when(wait(delay(5))) {
-					if (numDeleted != lastUpdate) {
-						printf("\r%d...", numDeleted);
-						lastUpdate = numDeleted;
-					}
-				}
+		while (true) {
+			if (auto const res = co_await timeout(done, 5); res.present()) {
+				break;
+			}
+
+			if (numDeleted != lastUpdate) {
+				printf("\r%d...", numDeleted);
+				lastUpdate = numDeleted;
 			}
 		}
 		printf("\r%d objects deleted\n", numDeleted);
@@ -2792,22 +2637,19 @@ ACTOR Future<Void> deleteBackupContainer(const char* name,
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> describeBackup(const char* name,
-                                  std::string destinationContainer,
-                                  Optional<std::string> proxy,
-                                  bool deep,
-                                  Optional<Database> cx,
-                                  bool json,
-                                  Optional<std::string> encryptionKeyFile) {
+Future<Void> describeBackup(const char* name,
+                            std::string destinationContainer,
+                            Optional<std::string> proxy,
+                            bool deep,
+                            Optional<Database> cx,
+                            bool json) {
 	try {
-		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, encryptionKeyFile);
-		state BackupDescription desc = wait(c->describeBackup(deep));
+		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, proxy, {}, 0);
+		BackupDescription desc = co_await c->describeBackup(deep);
 		if (cx.present())
-			wait(desc.resolveVersionTimes(cx.get()));
+			co_await desc.resolveVersionTimes(cx.get());
 		printf("%s\n", (json ? desc.toJSON() : desc.toString()).c_str());
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
@@ -2815,8 +2657,6 @@ ACTOR Future<Void> describeBackup(const char* name,
 		fprintf(stderr, "ERROR: %s\n", e.what());
 		throw;
 	}
-
-	return Void();
 }
 
 static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) {
@@ -2846,19 +2686,19 @@ std::pair<Version, Version> getMaxMinRestorableVersions(const BackupDescription&
 // If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for
 // selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overridden to the version
 // resolved to that timestamp.
-ACTOR Future<Void> queryBackup(const char* name,
-                               std::string destinationContainer,
-                               Optional<std::string> proxy,
-                               Standalone<VectorRef<KeyRangeRef>> keyRangesFilter,
-                               Version restoreVersion,
-                               Version snapshotVersion,
-                               std::string originalClusterFile,
-                               std::string restoreTimestamp,
-                               Verbose verbose,
-                               Optional<Database> cx) {
-	state UID operationId = deterministicRandom()->randomUniqueID();
-	state JsonBuilderObject result;
-	state std::string errorMessage;
+Future<Void> queryBackup(const char* name,
+                         std::string destinationContainer,
+                         Optional<std::string> proxy,
+                         Standalone<VectorRef<KeyRangeRef>> keyRangesFilter,
+                         Version restoreVersion,
+                         Version snapshotVersion,
+                         std::string originalClusterFile,
+                         std::string restoreTimestamp,
+                         Verbose verbose,
+                         Optional<Database> cx) {
+	UID operationId = deterministicRandom()->randomUniqueID();
+	JsonBuilderObject result;
+	std::string errorMessage;
 	result["key_ranges_filter"] = printable(keyRangesFilter);
 	result["destination_container"] = destinationContainer;
 
@@ -2879,7 +2719,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 			    result,
 			    format("an original cluster file must be given in order to resolve restore target timestamp '%s'",
 			           restoreTimestamp.c_str()));
-			return Void();
+			co_return;
 		}
 
 		if (!fileExists(originalClusterFile)) {
@@ -2887,23 +2727,23 @@ ACTOR Future<Void> queryBackup(const char* name,
 			                       result,
 			                       format("The specified original source database cluster file '%s' does not exist\n",
 			                              originalClusterFile.c_str()));
-			return Void();
+			co_return;
 		}
 
 		Database origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION);
-		Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb));
+		Version v = co_await timeKeeperVersionFromDatetime(restoreTimestamp, origDb);
 		result["restore_timestamp"] = restoreTimestamp;
 		result["restore_timestamp_resolved_version"] = v;
 		restoreVersion = v;
 	}
 
-	state int64_t totalRangeFilesSize = 0;
-	state int64_t totalLogFilesSize = 0;
-	state JsonBuilderArray rangeFilesJson;
-	state JsonBuilderArray logFilesJson;
+	int64_t totalRangeFilesSize = 0;
+	int64_t totalLogFilesSize = 0;
+	JsonBuilderArray rangeFilesJson;
+	JsonBuilderArray logFilesJson;
 	try {
-		state Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer, proxy, {});
-		BackupDescription desc = wait(bc->describeBackup());
+		Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer, proxy, {}, 0);
+		BackupDescription desc = co_await bc->describeBackup();
 		// Use continuous log end version for the maximum restorable version for the key ranges when a restorable
 		// version doesn't exist.
 		auto [maxRestorableVersion, minRestorableVersion] = getMaxMinRestorableVersions(desc, !keyRangesFilter.empty());
@@ -2924,14 +2764,14 @@ ACTOR Future<Void> queryBackup(const char* name,
 			                       result,
 			                       errorMessage =
 			                           format("the specified restorable version %lld is not valid", restoreVersion));
-			return Void();
+			co_return;
 		}
 
-		state Optional<RestorableFileSet> fileSet;
+		Optional<RestorableFileSet> fileSet;
 		if (snapshotVersion != invalidVersion) {
 			// When a snapshot version is specified, we will first get a restore set using the latest snapshot file to
 			// restore to the snapshot version. After snapshot version, we will only use mutation logs to restore.
-			wait(store(fileSet, bc->getRestoreSet(snapshotVersion, keyRangesFilter)));
+			fileSet = co_await bc->getRestoreSet(snapshotVersion, keyRangesFilter);
 			if (fileSet.present()) {
 				result["snapshot_version"] = fileSet.get().targetVersion;
 				for (const auto& rangeFile : fileSet.get().ranges) {
@@ -2939,7 +2779,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 					object["file_name"] = rangeFile.fileName;
 					object["file_size"] = rangeFile.fileSize;
 					object["version"] = rangeFile.version;
-					object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0
+					object["key_range"] = !fileSet.get().keyRanges.contains(rangeFile.fileName)
 					                          ? "none"
 					                          : fileSet.get().keyRanges.at(rangeFile.fileName).toString();
 					rangeFilesJson.push_back(object);
@@ -2972,14 +2812,14 @@ ACTOR Future<Void> queryBackup(const char* name,
 				    result,
 				    format("no restorable files set found for specified key ranges from snapshotVersion %lld",
 				           snapshotVersion));
-				return Void();
+				co_return;
 			}
 
 			// We only need to know all the mutation logs from `snapshotVersion` to `restoreVersion`.
-			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter, /*logOnly=*/true, snapshotVersion)));
+			fileSet = co_await bc->getRestoreSet(restoreVersion, keyRangesFilter, /*logOnly=*/true, snapshotVersion);
 		} else {
 			// When a snapshot version is not specified, we use the latest snapshot to restore to the `restoreVersion`.
-			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter)));
+			fileSet = co_await bc->getRestoreSet(restoreVersion, keyRangesFilter);
 		}
 
 		if (fileSet.present()) {
@@ -2989,7 +2829,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 				object["file_name"] = rangeFile.fileName;
 				object["file_size"] = rangeFile.fileSize;
 				object["version"] = rangeFile.version;
-				object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0
+				object["key_range"] = !fileSet.get().keyRanges.contains(rangeFile.fileName)
 				                          ? "none"
 				                          : fileSet.get().keyRanges.at(rangeFile.fileName).toString();
 				rangeFilesJson.push_back(object);
@@ -3015,12 +2855,12 @@ ACTOR Future<Void> queryBackup(const char* name,
 			    .detail("LogFilesBytes", totalLogFilesSize);
 		} else if (snapshotVersion == invalidVersion) {
 			reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges");
-			return Void();
+			co_return;
 		}
 
 	} catch (Error& e) {
 		reportBackupQueryError(operationId, result, e.what());
-		return Void();
+		co_return;
 	}
 
 	result["total_range_files_size"] = totalRangeFilesSize;
@@ -3032,13 +2872,12 @@ ACTOR Future<Void> queryBackup(const char* name,
 	}
 
 	printf("%s\n", result.getJson().c_str());
-	return Void();
 }
 
-ACTOR Future<Void> listBackup(std::string baseUrl, Optional<std::string> proxy) {
+Future<Void> listBackup(std::string baseUrl, Optional<std::string> proxy) {
 	try {
-		std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl, proxy));
-		for (std::string container : containers) {
+		std::vector<std::string> containers = co_await IBackupContainer::listContainers(baseUrl, proxy);
+		for (const std::string& container : containers) {
 			printf("%s\n", container.c_str());
 		}
 	} catch (Error& e) {
@@ -3049,24 +2888,24 @@ ACTOR Future<Void> listBackup(std::string baseUrl, Optional<std::string> proxy)
 		fprintf(stderr, "%s\n", msg.c_str());
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> listBackupTags(Database cx) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-	loop {
+Future<Void> listBackupTags(Database cx) {
+	auto tr = makeReference<ReadYourWritesTransaction>(cx);
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::vector<KeyBackedTag> tags = wait(getAllBackupTags(tr));
+			std::vector<KeyBackedTag> tags = co_await getAllBackupTags(tr);
 			for (const auto& tag : tags) {
 				printf("%s\n", tag.tagName.c_str());
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
@@ -3082,21 +2921,22 @@ struct BackupModifyOptions {
 	}
 };
 
-ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOptions options) {
+Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOptions options) {
 	if (!options.hasChanges()) {
 		fprintf(stderr, "No changes were specified, nothing to do!\n");
 		throw backup_error();
 	}
 
-	state KeyBackedTag tag = makeBackupTag(tagName);
+	KeyBackedTag tag = makeBackupTag(tagName);
 
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
-	loop {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			state Optional<UidAndAbortedFlagT> uidFlag = wait(tag.get(db.getReference()));
+			Optional<UidAndAbortedFlagT> uidFlag = co_await tag.get(db.getReference());
 
 			if (!uidFlag.present()) {
 				fprintf(stderr, "No backup exists on tag '%s'\n", tagName.c_str());
@@ -3108,8 +2948,8 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 				throw backup_error();
 			}
 
-			state BackupConfig config(uidFlag.get().first);
-			EBackupState s = wait(config.stateEnum().getOrThrow(tr, Snapshot::False, backup_invalid_info()));
+			BackupConfig config(uidFlag.get().first);
+			EBackupState s = co_await config.stateEnum().getOrThrow(tr, Snapshot::False, backup_invalid_info());
 			if (!FileBackupAgent::isRunnable(s)) {
 				fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str());
 				throw backup_error();
@@ -3125,8 +2965,8 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 			}
 
 			if (options.destURL.present()) {
-				state Reference<IBackupContainer> prevContainer =
-				    wait(config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info()));
+				Reference<IBackupContainer> prevContainer =
+				    co_await config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info());
 				std::string prevURL = prevContainer->getURL();
 				std::string newURL = options.destURL.get();
 				if (!prevURL.empty() && prevURL.back() == '/') {
@@ -3149,16 +2989,19 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 					}
 				}
 
-				state Reference<IBackupContainer> bc;
+				Reference<IBackupContainer> bc;
 				TraceEvent("ModifyBackupSetNewContainer")
 				    .detail("TagName", tagName)
 				    .detail("DestURL", options.destURL.get())
 				    .detail("EncryptionKeyFile",
 				            options.encryptionKeyFile.present() ? options.encryptionKeyFile.get() : "None");
-				bc = openBackupContainer(
-				    exeBackup.toString().c_str(), options.destURL.get(), options.proxy, options.encryptionKeyFile);
+				bc = openBackupContainer(exeBackup.toString().c_str(),
+				                         options.destURL.get(),
+				                         options.proxy,
+				                         options.encryptionKeyFile,
+				                         prevContainer->getEncryptionBlockSize());
 				try {
-					wait(timeoutError(bc->create(), 30));
+					co_await timeoutError(bc->create(), 30);
 				} catch (Error& e) {
 					if (e.code() == error_code_actor_cancelled)
 						throw;
@@ -3168,9 +3011,8 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 					        e.what());
 					throw backup_error();
 				}
-
 				config.backupContainer().set(tr, bc);
-				wait(bc->writeEncryptionMetadata());
+				co_await bc->writeEncryptionMetadata(bc->getEncryptionBlockSize());
 			} else if (options.encryptionKeyFile.present()) {
 				fprintf(stdout,
 				        " Encryption key file specified without a new destination URL."
@@ -3182,22 +3024,22 @@ ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOp
 			}
 
 			if (options.activeSnapshotIntervalSeconds.present()) {
-				Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, Snapshot::False, backup_error()));
+				Version begin = co_await config.snapshotBeginVersion().getOrThrow(tr, Snapshot::False, backup_error());
 				config.snapshotTargetEndVersion().set(tr,
 				                                      begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() *
 				                                               CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
 			}
 
-			wait(tr->commit());
+			co_await tr->commit();
 			break;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
-
-	return Void();
 }
 
+// NOLINTBEGIN(bugprone-use-after-move): ignore clang-tidy false positives in parseLine's token buffer handling.
 static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& err, bool& partial) {
 	err = false;
 	partial = false;
@@ -3219,8 +3061,9 @@ static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& er
 					buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset));
 				ret.push_back(std::move(buf));
 				offset = i = line.find_first_not_of(' ', i + 1);
-			} else
+			} else {
 				i++;
+			}
 			break;
 		case '"':
 			quoted = !quoted;
@@ -3233,8 +3076,9 @@ static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& er
 				buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset));
 				offset = i = line.find_first_not_of(' ', i);
 				forcetoken = false;
-			} else
+			} else {
 				i++;
+			}
 			break;
 		case '\\':
 			if (i + 2 > line.length()) {
@@ -3289,6 +3133,7 @@ static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& er
 
 	return ret;
 }
+// NOLINTEND(bugprone-use-after-move)
 
 static void addKeyRange(std::string optionValue, Standalone<VectorRef<KeyRangeRef>>& keyRanges) {
 	bool err = false, partial = false;
@@ -3539,8 +3384,8 @@ int main(int argc, char* argv[]) {
 		registerCrashHandler();
 
 		// Set default of line buffering standard out and error
-		setvbuf(stdout, NULL, _IONBF, 0);
-		setvbuf(stderr, NULL, _IONBF, 0);
+		setvbuf(stdout, nullptr, _IONBF, 0);
+		setvbuf(stderr, nullptr, _IONBF, 0);
 
 		ProgramExe programExe = getProgramType(argv[0]);
 		BackupType backupType = BackupType::UNDEFINED;
@@ -3700,21 +3545,6 @@ int main(int argc, char* argv[]) {
 				    newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
 			}
 			break;
-		case ProgramExe::FASTRESTORE_TOOL:
-			if (newArgC < 2) {
-				printFastRestoreUsage(false);
-				return FDB_EXIT_ERROR;
-			}
-			// Get the restore operation type
-			restoreType = getRestoreType(newArgV[1]);
-			if (restoreType == RestoreType::UNKNOWN) {
-				args =
-				    std::make_unique<CSimpleOpt>(newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
-			} else {
-				args = std::make_unique<CSimpleOpt>(
-				    newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
-			}
-			break;
 		case ProgramExe::UNDEFINED:
 		default:
 			fprintf(stderr, "FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
@@ -3756,7 +3586,7 @@ int main(int argc, char* argv[]) {
 		std::string restoreTimestamp;
 		WaitForComplete waitForDone{ false };
 		StopWhenDone stopWhenDone{ true };
-		UsePartitionedLog usePartitionedLog{ false }; // Set to true to use new backup system
+		MutationLogType mutationLogType{ MutationLogType::DEFAULT };
 		IncrementalBackupOnly incrementalBackupOnly{ false };
 		OnlyApplyMutationLogs onlyApplyMutationLogs{ false };
 		InconsistentSnapshotOnly inconsistentSnapshotOnly{ false };
@@ -3787,6 +3617,7 @@ int main(int argc, char* argv[]) {
 		bool jsonOutput = false;
 		DeleteData deleteData{ false };
 		Optional<std::string> encryptionKeyFile;
+		int encryptionBlockSize = 0;
 		Optional<std::string> blobManifestUrl;
 		SnapshotMode snapshotMode = SnapshotMode::RANGEFILE; // Default to legacy rangefile mode
 
@@ -3858,12 +3689,12 @@ int main(int argc, char* argv[]) {
 				return FDB_EXIT_SUCCESS;
 				break;
 			case OPT_NOBUFSTDOUT:
-				setvbuf(stdout, NULL, _IONBF, 0);
-				setvbuf(stderr, NULL, _IONBF, 0);
+				setvbuf(stdout, nullptr, _IONBF, 0);
+				setvbuf(stderr, nullptr, _IONBF, 0);
 				break;
 			case OPT_BUFSTDOUTERR:
-				setvbuf(stdout, NULL, _IOFBF, BUFSIZ);
-				setvbuf(stderr, NULL, _IOFBF, BUFSIZ);
+				setvbuf(stdout, nullptr, _IOFBF, BUFSIZ);
+				setvbuf(stderr, nullptr, _IOFBF, BUFSIZ);
 				break;
 			case OPT_QUIET:
 				quietDisplay = true;
@@ -4039,9 +3870,18 @@ int main(int argc, char* argv[]) {
 			case OPT_NOSTOPWHENDONE:
 				stopWhenDone.set(false);
 				break;
-			case OPT_USE_PARTITIONED_LOG:
-				usePartitionedLog.set(true);
+			case OPT_MUTATION_LOG_TYPE: {
+				auto parsedType = getMutationLogType(args->OptionArg());
+				if (!parsedType.present()) {
+					fprintf(stderr,
+					        "ERROR: Unknown mutation log type '%s'. Valid modes are: partitioned-log-experimental, "
+					        "range-partitioned-log-experimental\n",
+					        args->OptionArg());
+					return FDB_EXIT_ERROR;
+				}
+				mutationLogType = parsedType.get();
 				break;
+			}
 			case OPT_INCREMENTALONLY:
 				incrementalBackupOnly.set(true);
 				onlyApplyMutationLogs.set(true);
@@ -4050,6 +3890,20 @@ int main(int argc, char* argv[]) {
 				encryptionKeyFile = args->OptionArg();
 				modifyOptions.encryptionKeyFile = encryptionKeyFile;
 				break;
+			case OPT_ENCRYPTION_BLOCK_SIZE:
+				try {
+					encryptionBlockSize = std::stoi(args->OptionArg());
+				} catch (std::exception&) {
+					fprintf(stderr, "ERROR: Invalid encryption block size `%s'\n", args->OptionArg());
+					printHelpTeaser(newArgV[0]);
+					return FDB_EXIT_ERROR;
+				}
+				if (encryptionBlockSize <= 0) {
+					fprintf(stderr, "ERROR: Invalid encryption block size `%s'\n", args->OptionArg());
+					printHelpTeaser(newArgV[0]);
+					return FDB_EXIT_ERROR;
+				}
+				break;
 			case OPT_RESTORECONTAINER:
 				restoreContainer = args->OptionArg();
 				// If the url starts with '/' then prepend "file://" for backwards compatibility
@@ -4217,7 +4071,7 @@ int main(int argc, char* argv[]) {
 						return FDB_EXIT_ERROR;
 					}
 					snapshotMode = parsedMode.get();
-				} else if (programExe == ProgramExe::RESTORE || programExe == ProgramExe::FASTRESTORE_TOOL) {
+				} else if (programExe == ProgramExe::RESTORE) {
 					// Validate and store mode parameter for restore mechanism
 					auto parsedMode = getRestoreMode(args->OptionArg());
 					if (!parsedMode.present()) {
@@ -4244,7 +4098,7 @@ int main(int argc, char* argv[]) {
 				// Add the backup key range
 			case ProgramExe::BACKUP:
 				// Error, if the keys option was not specified
-				if (backupKeys.size() == 0) {
+				if (backupKeys.empty()) {
 					fprintf(stderr, "ERROR: Unknown backup option value `%s'\n", args->File(argLoop));
 					printHelpTeaser(newArgV[0]);
 					return FDB_EXIT_ERROR;
@@ -4266,13 +4120,6 @@ int main(int argc, char* argv[]) {
 				return FDB_EXIT_ERROR;
 				break;
 
-			case ProgramExe::FASTRESTORE_TOOL:
-				fprintf(
-				    stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n", args->File(argLoop));
-				printHelpTeaser(newArgV[0]);
-				return FDB_EXIT_ERROR;
-				break;
-
 			case ProgramExe::DR_AGENT:
 				fprintf(stderr, "ERROR: DR Agent does not support argument value `%s'\n", args->File(argLoop));
 				printHelpTeaser(newArgV[0]);
@@ -4281,7 +4128,7 @@ int main(int argc, char* argv[]) {
 
 			case ProgramExe::DB_BACKUP:
 				// Error, if the keys option was not specified
-				if (backupKeys.size() == 0) {
+				if (backupKeys.empty()) {
 					fprintf(stderr, "ERROR: Unknown DR option value `%s'\n", args->File(argLoop));
 					printHelpTeaser(newArgV[0]);
 					return FDB_EXIT_ERROR;
@@ -4350,16 +4197,16 @@ int main(int argc, char* argv[]) {
 
 		Future<Void> memoryUsageMonitor = startMemoryUsageMonitor(memLimit);
 
-		IKnobCollection::setupKnobs(knobs);
+		setupClientKnobs(knobs);
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
+		initializeClientKnobs(Randomize::False, IsSimulated::False);
 
 		TraceEvent("ProgramStart")
 		    .setMaxEventLength(12000)
 		    .detail("SourceVersion", getSourceVersion())
 		    .detail("Version", FDB_VT_VERSION)
 		    .detail("PackageName", FDB_VT_PACKAGE_NAME)
-		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
+		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 		    .setMaxFieldLength(10000)
 		    .detail("CommandLine", commandLine)
 		    .setMaxFieldLength(0)
@@ -4402,7 +4249,7 @@ int main(int argc, char* argv[]) {
 		};
 
 		auto initSourceCluster = [&](bool required, bool quiet = false) {
-			if (!sourceClusterFile.size() && required) {
+			if (sourceClusterFile.empty() && required) {
 				if (!quiet) {
 					fprintf(stderr, "ERROR: source cluster file is required\n");
 				}
@@ -4427,18 +4274,19 @@ int main(int argc, char* argv[]) {
 			return result.present();
 		};
 
-		// The fastrestore tool does not yet support multiple ranges and is incompatible with
-		// features that back up data in the system keys.
-		if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty() &&
-		    programExe != ProgramExe::FASTRESTORE_TOOL) {
-			addDefaultBackupRanges(backupKeys);
+		if (encryptionKeyFile.present() && encryptionBlockSize == 0) {
+			encryptionBlockSize = DEFAULT_ENCRYPTION_BLOCK_SIZE;
 		}
 
-		if ((restoreSystemKeys || restoreUserKeys) && programExe == ProgramExe::FASTRESTORE_TOOL) {
-			fprintf(stderr, "ERROR: Options: --user-data and --system-metadata are not supported with fastrestore\n");
+		if (encryptionBlockSize > 0 && !encryptionKeyFile.present()) {
+			fprintf(stderr, "ERROR: --encryption-block-size option requires --encryption-key-file to be set\n");
 			return FDB_EXIT_ERROR;
 		}
 
+		if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty()) {
+			addDefaultBackupRanges(backupKeys);
+		}
+
 		if ((restoreUserKeys || restoreSystemKeys) && !backupKeys.empty()) {
 			fprintf(stderr,
 			        "ERROR: Cannot specify additional ranges when using --user-data or --system-metadata "
@@ -4467,7 +4315,7 @@ int main(int argc, char* argv[]) {
 					return FDB_EXIT_ERROR;
 				// Test out the backup url to make sure it parses.  Doesn't test to make sure it's actually
 				// writeable.
-				openBackupContainer(newArgV[0], destinationContainer, proxy, encryptionKeyFile);
+				openBackupContainer(newArgV[0], destinationContainer, proxy, encryptionKeyFile, encryptionBlockSize);
 				f = stopAfter(submitBackup(db,
 				                           destinationContainer,
 				                           proxy,
@@ -4478,9 +4326,10 @@ int main(int argc, char* argv[]) {
 				                           dryRun,
 				                           waitForDone,
 				                           stopWhenDone,
-				                           usePartitionedLog,
+				                           mutationLogType,
 				                           incrementalBackupOnly,
 				                           encryptionKeyFile,
+				                           encryptionBlockSize,
 				                           snapshotMode));
 				break;
 			}
@@ -4550,8 +4399,7 @@ int main(int argc, char* argv[]) {
 				                               db,
 				                               forceAction,
 				                               expireRestorableAfterVersion,
-				                               expireRestorableAfterDatetime,
-				                               encryptionKeyFile));
+				                               expireRestorableAfterDatetime));
 				break;
 
 			case BackupType::DELETE_BACKUP:
@@ -4572,8 +4420,7 @@ int main(int argc, char* argv[]) {
 				                             proxy,
 				                             describeDeep,
 				                             describeTimestamps ? Optional<Database>(db) : Optional<Database>(),
-				                             jsonOutput,
-				                             encryptionKeyFile));
+				                             jsonOutput));
 				break;
 
 			case BackupType::LIST:
@@ -4694,80 +4541,6 @@ int main(int argc, char* argv[]) {
 				throw restore_error();
 			}
 			break;
-		case ProgramExe::FASTRESTORE_TOOL:
-			// Support --dest-cluster-file option as fdbrestore does
-			if (dryRun) {
-				if (restoreType != RestoreType::START) {
-					fprintf(stderr, "Restore dry run only works for 'start' command\n");
-					return FDB_EXIT_ERROR;
-				}
-
-				// Must explicitly call trace file options handling if not calling Database::createDatabase()
-				initTraceFile();
-			} else {
-				if (restoreClusterFileDest.empty()) {
-					fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n");
-					return FDB_EXIT_ERROR;
-				}
-
-				if (!fileExists(restoreClusterFileDest)) {
-					fprintf(stderr,
-					        "Restore destination cluster file '%s' does not exist.\n",
-					        restoreClusterFileDest.c_str());
-					return FDB_EXIT_ERROR;
-				}
-
-				try {
-					db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION);
-				} catch (Error& e) {
-					fprintf(stderr,
-					        "Restore destination cluster file '%s' invalid: %s\n",
-					        restoreClusterFileDest.c_str(),
-					        e.what());
-					return FDB_EXIT_ERROR;
-				}
-			}
-			// TODO: We have not implemented the code commented out in this case
-			switch (restoreType) {
-			case RestoreType::START:
-				f = stopAfter(runFastRestoreTool(db,
-				                                 tagName,
-				                                 restoreContainer,
-				                                 proxy,
-				                                 backupKeys,
-				                                 restoreVersion,
-				                                 !dryRun,
-				                                 Verbose{ !quietDisplay },
-				                                 waitForDone));
-				break;
-			case RestoreType::WAIT:
-				printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");
-				throw restore_error();
-				//					f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
-				break;
-			case RestoreType::ABORT:
-				printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n");
-				throw restore_error();
-				//					f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)),
-				//[tagName](FileBackupAgent::ERestoreState s) -> Void { 						printf("Tag: %s
-				// State: %s\n", tagName.c_str(),
-				// FileBackupAgent::restoreStateText(s).toString().c_str()); 						return Void();
-				//					}) );
-				break;
-			case RestoreType::STATUS:
-				printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n");
-				throw restore_error();
-				// If no tag is specifically provided then print all tag status, don't just use "default"
-				if (tagProvided)
-					tag = tagName;
-				//					f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void
-				//{ 						printf("%s\n", s.c_str()); 						return Void();
-				//					}) );
-				break;
-			default:
-				throw restore_error();
-			}
-			break;
 		case ProgramExe::DR_AGENT:
 			if (!initCluster() || !initSourceCluster(true)) {
 				return FDB_EXIT_ERROR;
@@ -4834,7 +4607,7 @@ int main(int argc, char* argv[]) {
 				std::string s;
 
 #ifdef __linux__
-				char* demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL);
+				char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr);
 				if (demangled) {
 					s = demangled;
 					if (StringRef(s).startsWith("(anonymous namespace)::"_sr))
diff --git a/fdbbackup/include/fdbbackup/FileConverter.h b/fdbbackup/include/fdbbackup/FileConverter.h
index 62be298d96c..f858c8d3101 100644
--- a/fdbbackup/include/fdbbackup/FileConverter.h
+++ b/fdbbackup/include/fdbbackup/FileConverter.h
@@ -24,7 +24,7 @@
 
 #include <cinttypes>
 #include "SimpleOpt/SimpleOpt.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 
 namespace file_converter {
 
@@ -52,6 +52,7 @@ enum {
 	OPT_END_VERSION_FILTER,
 	OPT_KNOB,
 	OPT_SAVE_FILE,
+	OPT_ENCRYPTION_KEY_FILE,
 	OPT_HELP
 };
 
@@ -84,6 +85,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_KNOB, "--knob-", SO_REQ_SEP },
 	                                        { OPT_SAVE_FILE, "-s", SO_NONE },
 	                                        { OPT_SAVE_FILE, "--save", SO_NONE },
+	                                        { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP },
 	                                        { OPT_HELP, "-?", SO_NONE },
 	                                        { OPT_HELP, "-h", SO_NONE },
 	                                        { OPT_HELP, "--help", SO_NONE },
diff --git a/fdbbackup/tests/backup_tests_common.sh b/fdbbackup/tests/backup_tests_common.sh
index 81dac39fce6..1ab34b62503 100644
--- a/fdbbackup/tests/backup_tests_common.sh
+++ b/fdbbackup/tests/backup_tests_common.sh
@@ -19,9 +19,15 @@
 # limitations under the License.
 #
 # Common backup test functions
-# Shared between s3_backup_test.sh, s3_backup_bulkdump_bulkload.sh, dir_backup_test.sh, etc.
+# Shared between blob_backup_restore_test.sh, s3_backup_bulkdump_bulkload.sh, dir_backup_test.sh, etc.
 # These functions work with both S3/blobstore and file-based backup testing
 
+# Source shared test utilities (output_contains, output_matches, etc.)
+_BACKUP_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [[ -f "${_BACKUP_COMMON_DIR}/../../fdbclient/tests/tests_common.sh" ]]; then
+  source "${_BACKUP_COMMON_DIR}/../../fdbclient/tests/tests_common.sh"
+fi
+
 # Helper function to add base arguments (cluster file and logging)
 # Uses bash nameref (requires bash 4.3+) to modify the array in place
 # $1 name of the array variable to modify (passed by name, not value)
@@ -106,7 +112,7 @@ function s3_cleanup_url {
   local local_scratch_dir="${2}"
   local local_url="${3}"
   local credentials="${4}"
-  
+
   local cmd=("${local_build_dir}/bin/s3client")
   cmd+=("${KNOBS[@]}")
   
@@ -151,7 +157,11 @@ function run_backup {
   add_common_optional_args cmd_args "${blob_credentials}" "${backup_mode}" "${local_encryption_key_file}"
 
   if [[ "${USE_PARTITIONED_LOG:-false}" == "true" ]]; then
-    cmd_args+=("--partitioned-log-experimental")
+    cmd_args+=("--mutation-log-type" "partitioned-log-experimental")
+  fi
+
+  if [[ "${USE_ENCRYPTION_BLOCK_SIZE:-false}" == "true" ]]; then
+    cmd_args+=("--encryption-block-size" "4096")
   fi
 
   # Start backup without -w flag to avoid hanging
@@ -182,12 +192,12 @@ function run_backup {
     set -e
     
     # Check if backup is restorable (differential state) or completed
-    if echo "${status_output}" | grep -q "is restorable"; then
+    if output_contains "${status_output}" "is restorable"; then
       log "Backup is now restorable after ${elapsed}s"
       break
     fi
     
-    if echo "${status_output}" | grep -q "completed"; then
+    if output_contains "${status_output}" "completed"; then
       log "Backup completed after ${elapsed}s"
       break
     fi
@@ -196,17 +206,17 @@ function run_backup {
     if [[ $((elapsed % 30)) -eq 0 ]]; then
       log "Still waiting for backup to become restorable (${elapsed}s elapsed)..."
       # Show current state for debugging
-      if echo "${status_output}" | grep -q "is restorable"; then
+      if output_contains "${status_output}" "is restorable"; then
         log "  Status: backup is restorable (should have exited loop)"
-      elif echo "${status_output}" | grep -q "in progress to"; then
+      elif output_contains "${status_output}" "in progress to"; then
         log "  Status: backup running, waiting for snapshot to complete"
-      elif echo "${status_output}" | grep -q "just started"; then
+      elif output_contains "${status_output}" "just started"; then
         log "  Status: backup submitted, tasks starting up"
       fi
       # Check snapshot mode for debugging
-      if echo "${status_output}" | grep -q "Snapshot Mode: bulkdump"; then
+      if output_contains "${status_output}" "Snapshot Mode: bulkdump"; then
         log "  Snapshot Mode: bulkdump (using BulkDump for snapshots)"
-      elif echo "${status_output}" | grep -q "Snapshot Mode: both"; then
+      elif output_contains "${status_output}" "Snapshot Mode: both"; then
         log "  Snapshot Mode: both (generating both formats)"
       fi
     fi
@@ -218,13 +228,48 @@ function run_backup {
     echo "${status_output}"
     return 1
   fi
-  
+
+  # For 'both' or 'bulkdump' mode, wait for BulkDump snapshot to be written
+  # BulkDump writes the snapshot file AFTER DD job completes, which is async from rangefile snapshot
+  if [[ "${backup_mode}" == "both" || "${backup_mode}" == "bulkdump" ]]; then
+    log "Waiting for BulkDump snapshot to be written (mode: ${backup_mode})..."
+    local bulkdump_timeout=120  # Additional timeout for BulkDump
+    local bulkdump_elapsed=0
+    local bulkdump_poll=5
+
+    while [[ $bulkdump_elapsed -lt $bulkdump_timeout ]]; do
+      sleep $bulkdump_poll
+      bulkdump_elapsed=$((bulkdump_elapsed + bulkdump_poll))
+
+      # Check for BulkDump snapshot file with ",bulk" suffix (used in 'both' mode)
+      # or check if any snapshot has bulkDumpJobId field
+      set +e
+      snapshot_list=$("${local_build_dir}"/bin/fdbbackup describe -d "${local_url}" -C "${local_scratch_dir}/loopback_cluster/fdb.cluster" --log --logdir="${local_scratch_dir}" 2>&1)
+      set -e
+
+      # Check if BulkDump snapshot exists (look for bulkDumpJobId in describe output)
+      if output_matches "${snapshot_list}" "bulkDumpJobId\|,bulk"; then
+        log "BulkDump snapshot found after ${bulkdump_elapsed}s"
+        break
+      fi
+
+      if [[ $((bulkdump_elapsed % 30)) -eq 0 ]]; then
+        log "Still waiting for BulkDump snapshot (${bulkdump_elapsed}s elapsed)..."
+      fi
+    done
+
+    if [[ $bulkdump_elapsed -ge $bulkdump_timeout ]]; then
+      log "Warning: Timeout waiting for BulkDump snapshot after ${bulkdump_timeout}s"
+      log "BulkDump may still be running - proceeding anyway"
+    fi
+  fi
+
   # Check if backup already completed (no need to discontinue)
-  if echo "${status_output}" | grep -q "completed"; then
+  if output_contains "${status_output}" "completed"; then
     log "Backup already completed - no need to discontinue"
     return 0
   fi
-  
+
   # Stop the backup to finalize it (only if still running)
   log "Stopping backup to finalize restorable state"
   set +e
@@ -233,7 +278,7 @@ function run_backup {
   set -e
   
   if [[ $stop_exit_code -ne 0 ]]; then
-    if echo "${stop_output}" | grep -q "already discontinued\|not running\|unneeded"; then
+    if output_matches "${stop_output}" "already discontinued\|not running\|unneeded"; then
       log "Backup already completed and finalized - this is success!"
     else
       err "Failed to stop backup: ${stop_output}"
@@ -299,13 +344,13 @@ function run_restore {
     # Check if restore completed
     # Status output contains "State: completed" or "Phase: Complete" when done
     # Also check "No restore" for when restore tag doesn't exist (completed and cleaned up)
-    if echo "${status_output}" | grep -qi "State:.*completed\|Phase:.*Complete\|No restore"; then
+    if output_matches_i "${status_output}" "State:.*completed\|Phase:.*Complete\|No restore"; then
       log "Restore completed after ${elapsed}s"
       return 0
     fi
     
     # Check if restore failed (be specific - "LastError: None" contains "Error" so avoid false positives)
-    if echo "${status_output}" | grep -qi "State:.*aborted"; then
+    if output_matches_i "${status_output}" "State:.*aborted"; then
       err "Restore aborted after ${elapsed}s"
       log "Status output:"
       echo "${status_output}"
@@ -324,7 +369,7 @@ function run_restore {
     if [[ $((elapsed % 30)) -eq 0 ]]; then
       log "Still waiting for restore to complete (${elapsed}s elapsed)..."
       # Show phase info for debugging
-      if echo "${status_output}" | grep -qi "Phase:"; then
+      if output_matches_i "${status_output}" "Phase:"; then
         phase_info=$(echo "${status_output}" | grep -i "Phase:" | head -1)
         log "  ${phase_info}"
       fi
@@ -380,7 +425,9 @@ function test_encryption_mismatches {
 
     if [[ ${exit_code1} -eq 0 ]]; then
       err "Restore without encryption on encrypted backup succeeded when it should have failed!"
-      rm -rf "${mismatch_logdir}"
+      if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then
+        rm -rf "${mismatch_logdir}"
+      fi
       return 1
     fi
     log "SUCCESS: Restore without encryption on encrypted backup failed as expected (exit code: ${exit_code1})"
@@ -402,7 +449,9 @@ function test_encryption_mismatches {
 
     if [[ ${exit_code2} -eq 0 ]]; then
       err "Restore with wrong encryption key succeeded when it should have failed!"
-      rm -rf "${mismatch_logdir}"
+      if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then
+        rm -rf "${mismatch_logdir}"
+      fi
       return 1
     fi
     log "SUCCESS: Restore with wrong encryption key failed as expected (exit code: ${exit_code2})"
@@ -428,14 +477,18 @@ function test_encryption_mismatches {
 
     if [[ ${exit_code} -eq 0 ]]; then
       err "Restore with encryption on unencrypted backup succeeded when it should have failed!"
-      rm -rf "${mismatch_logdir}"
+      if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then
+        rm -rf "${mismatch_logdir}"
+      fi
       return 1
     fi
     log "SUCCESS: Restore with encryption on unencrypted backup failed as expected (exit code: ${exit_code})"
   fi
 
   # Clean up separate log directory
-  rm -rf "${mismatch_logdir}"
+  if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then
+    rm -rf "${mismatch_logdir}"
+  fi
 
   log "All encryption mismatch tests completed successfully"
   return 0
@@ -461,11 +514,11 @@ function run_restore_wait {
     status_output=$("${local_build_dir}"/bin/fdbrestore status -t "${local_tag}" --dest-cluster-file "${local_scratch_dir}/loopback_cluster/fdb.cluster" --log --logdir="${local_scratch_dir}" 2>&1)
     set -e
     
-    if echo "${status_output}" | grep -qi "State:.*completed\|Phase:.*Complete\|No restore"; then
+    if output_matches_i "${status_output}" "State:.*completed\|Phase:.*Complete\|No restore"; then
       return 0
     fi
     
-    if echo "${status_output}" | grep -qi "State:.*aborted"; then
+    if output_matches_i "${status_output}" "State:.*aborted"; then
       return 1
     fi
     
@@ -485,15 +538,18 @@ function run_restore_wait {
 function setup_backup_test_environment {
   local http_verbose_level="${1}"
   local additional_knobs=("${@:2}")
-  
+
   # Clear proxy environment variables
   unset HTTP_PROXY
   unset HTTPS_PROXY
-  
+
   # Set USE_S3 based on environment
   readonly USE_S3="${USE_S3:-$( if [[ -n "${OKTETO_NAMESPACE+x}" ]]; then echo "true" ; else echo "false"; fi )}"
-  
-  # Set KNOBS based on whether we're using real S3 or MockS3Server
+
+  # Detect GCS from environment variables
+  detect_blobstore_provider
+
+  # Set KNOBS based on which provider we're using
   if [[ "${USE_S3}" == "true" ]]; then
     # Use AWS KMS encryption for real S3
     KNOBS=("--knob_blobstore_encryption_type=aws:kms" "--knob_http_verbose_level=${http_verbose_level}")
@@ -501,13 +557,13 @@ function setup_backup_test_environment {
     # No encryption for MockS3Server
     KNOBS=("--knob_http_verbose_level=${http_verbose_level}")
   fi
-  
+
   # Add any additional knobs (handle empty array when set -u is enabled)
   if [[ ${#additional_knobs[@]} -gt 0 ]]; then
     KNOBS+=("${additional_knobs[@]}")
   fi
   readonly KNOBS
-  
+
   setup_tls_ca_file
 }
 
diff --git a/fdbbackup/tests/s3_backup_test.sh b/fdbbackup/tests/blob_backup_restore_test.sh
similarity index 85%
rename from fdbbackup/tests/s3_backup_test.sh
rename to fdbbackup/tests/blob_backup_restore_test.sh
index 2664c55c0d5..b2dbc3dc003 100755
--- a/fdbbackup/tests/s3_backup_test.sh
+++ b/fdbbackup/tests/blob_backup_restore_test.sh
@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
 #
-# Test backup and restore from s3.
+# Test backup and restore from blob storage (S3, GCS, Azure, or MockS3Server).
 #
 # In the below we start a small FDB cluster, populate it with
-# some data and then start up MockS3Server or use S3
-# if it is available. We then run a backup to 'S3' and then
+# some data and then start up MockS3Server or use S3/GCS/Azure
+# if configured. We then run a backup to blob storage and then
 # a restore. We verify the restore is the same as the original.
 #
 # Debugging:
-#   - Run with -x flag: bash -x s3_backup_test.sh...
-#   - Preserve test data: PRESERVE_TEST_DATA=1 ./s3_backup_test.sh ...
+#   - Run with -x flag: bash -x blob_backup_restore_test.sh...
+#   - Preserve test data: PRESERVE_TEST_DATA=1 ./blob_backup_restore_test.sh ...
 #     This will leave all test data including MockS3 persistence files
 #     in the test scratch directory for analysis after the test completes.
 #
 # Usage:
-#   s3_backup_unified.sh <source_dir> <build_dir> [scratch_dir] [--encrypt]
+#   blob_backup_restore_test.sh <source_dir> <build_dir> [scratch_dir] [--encrypt]
 #
 # See https://apple.github.io/foundationdb/backups.html
 
@@ -104,7 +104,7 @@ function test_s3_backup_and_restore {
     err "Failed loading data into fdb"
     return 1
   fi
-  log "Run s3 backup"
+  log "Run blob storage backup"
   if ! run_backup "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "" "${credentials}"; then
     err "Failed backup"
     return 1
@@ -121,7 +121,7 @@ function test_s3_backup_and_restore {
   log "Testing encryption mismatches"
   test_encryption_mismatches "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "${credentials}"
 
-  log "Restore from s3"
+  log "Restore from blob storage"
   if ! run_restore "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "" "${credentials}"; then
     err "Failed restore"
     return 1
@@ -131,10 +131,12 @@ function test_s3_backup_and_restore {
     err "Failed verification of data in fdb"
     return 1
   fi
-  
-  # Cleanup test data.
-  if ! s3_cleanup_url "${local_build_dir}" "${local_scratch_dir}" "${edited_url}" "${credentials}"; then
-    return 1
+
+  # Cleanup test data (skip if preserving test data for debugging).
+  if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then
+    if ! s3_cleanup_url "${local_build_dir}" "${local_scratch_dir}" "${edited_url}" "${credentials}"; then
+      return 1
+    fi
   fi
   log "Check for Severity=40 errors"
   if ! grep_for_severity40 "${local_scratch_dir}"; then
@@ -150,42 +152,13 @@ set -o pipefail
 set -o noclobber
 
 # Parse command line arguments
-USE_ENCRYPTION=false
+USE_ENCRYPTION=$(((RANDOM % 2)) && echo true || echo false )
 USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false )
-PARAMS=()
 
-while (( "$#" )); do
-  case "$1" in
-    --encrypt)
-      USE_ENCRYPTION=true
-      shift
-      ;;
-    --encrypt-at-random)
-      USE_ENCRYPTION=$(((RANDOM % 2)) && echo true || echo false )
-      shift
-      ;;
-    --partitioned-log-experimental)
-      USE_PARTITIONED_LOG=true
-      shift
-      ;;
-    --partitioned-log-experimental-at-random)
-      USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false )
-      shift
-      ;;
-    -*|--*=) # unsupported flags
-      err "Error: Unsupported flag $1" >&2
-      exit 1
-      ;;
-    *) # preserve positional arguments
-      PARAMS+=("$1")
-      shift
-      ;;
-  esac
-done
-
-# Set positional arguments in their proper place
-if [ ${#PARAMS[@]} -ne 0 ]; then
-  set -- "${PARAMS[@]}"
+# Set USE_ENCRYPTION_BLOCK_SIZE only if encryption is enabled
+USE_ENCRYPTION_BLOCK_SIZE=false
+if [[ "${USE_ENCRYPTION}" == "true" ]]; then
+  USE_ENCRYPTION_BLOCK_SIZE=$(((RANDOM % 2)) && echo true || echo false)
 fi
 
 # Get the working directory for this script.
@@ -257,6 +230,7 @@ else
 fi
 readonly ENCRYPTION_KEY_FILE
 readonly USE_PARTITIONED_LOG
+readonly USE_ENCRYPTION_BLOCK_SIZE
 
 # Setup S3/MockS3 environment using common function
 readonly temp_dir_prefix="mocks3_backup_test"
diff --git a/fdbbackup/tests/dir_backup_test.sh b/fdbbackup/tests/dir_backup_test.sh
index 27523b56c53..a4e3b65f17f 100755
--- a/fdbbackup/tests/dir_backup_test.sh
+++ b/fdbbackup/tests/dir_backup_test.sh
@@ -19,6 +19,10 @@ SCRATCH_DIR=
 readonly TAG="test_backup"
 USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false )
 USE_ENCRYPTION=$(((RANDOM % 2)) && echo true || echo false )
+USE_ENCRYPTION_BLOCK_SIZE=false
+if [[ "${USE_ENCRYPTION}" == "true" ]]; then
+  USE_ENCRYPTION_BLOCK_SIZE=$(((RANDOM % 2)) && echo true || echo false)
+fi
 ENCRYPTION_KEY_FILE=
 
 # Install signal traps. Calls the cleanup function.
@@ -30,21 +34,27 @@ trap cleanup  EXIT
 function cleanup {
   echo "$(date -Iseconds) cleanup: starting (with 30s hard timeout)"
   start_cleanup_watchdog 30
-  
+
+  if cleanup_with_preserve_check; then
+    echo "$(date -Iseconds) cleanup: preserving test data, skipping cleanup"
+    cancel_cleanup_watchdog
+    return 0
+  fi
+
   echo "$(date -Iseconds) cleanup: shutting down FDB cluster"
   shutdown_fdb_cluster
-  
+
   if [[ -d "${SCRATCH_DIR}" ]]; then
     echo "$(date -Iseconds) cleanup: removing scratch dir: ${SCRATCH_DIR}"
     rm -rf "${SCRATCH_DIR}"
   fi
-  
+
   # Clean up encryption key file
   if [[ -n "${ENCRYPTION_KEY_FILE:-}" ]] && [[ -f "${ENCRYPTION_KEY_FILE}" ]]; then
     echo "$(date -Iseconds) cleanup: removing encryption key file: ${ENCRYPTION_KEY_FILE}"
     rm -f "${ENCRYPTION_KEY_FILE}"
   fi
-  
+
   echo "$(date -Iseconds) cleanup: complete"
   cancel_cleanup_watchdog
 }
@@ -82,8 +92,12 @@ function backup {
     cmd_args+=("--encryption-key-file" "${local_encryption_key_file}")
   fi
 
+  if [[ "${USE_ENCRYPTION_BLOCK_SIZE}" == "true" ]]; then
+    cmd_args+=("--encryption-block-size" "4096")
+  fi
+
   if [[ "${USE_PARTITIONED_LOG}" == "true" ]]; then
-    cmd_args+=("--partitioned-log-experimental")
+    cmd_args+=("--mutation-log-type" "partitioned-log-experimental")
   fi
 
   if ! "${local_build_dir}"/bin/fdbbackup start "${cmd_args[@]}"; then
@@ -157,6 +171,17 @@ function test_dir_backup_and_restore {
     err "Failed clear data in fdb"
     return 1
   fi
+
+  log "Testing encryption mismatches"
+  local backup
+  local backup_name
+  if ! backup=$(ls -dt "${scratch_dir}"/backups/backup-* | head -1); then
+    err "Failed to list backups under ${scratch_dir}/backups/"
+    return 1
+  fi
+  backup_name=$(basename "${backup}")
+  test_encryption_mismatches "${local_build_dir}" "${scratch_dir}" "file://${scratch_dir}/backups/${backup_name}" "${TAG}" "${local_encryption_key_file}"
+
   log "Restore"
   if ! restore "${local_build_dir}" "${scratch_dir}" "${local_encryption_key_file}"; then
     err "Failed restore"
@@ -197,6 +222,11 @@ if ! source "${cwd}/../../fdbclient/tests/tests_common.sh"; then
   err "Failed to source tests_common.sh"
   exit 1
 fi
+# shellcheck source=/dev/null
+if ! source "${cwd}/backup_tests_common.sh"; then
+  err "Failed to source backup_tests_common.sh"
+  exit 1
+fi
 
 # Process command-line options.
 if (( $# < 2 )) || (( $# > 3 )); then
@@ -238,6 +268,7 @@ fi
 
 readonly USE_PARTITIONED_LOG
 readonly USE_ENCRYPTION
+readonly USE_ENCRYPTION_BLOCK_SIZE
 readonly ENCRYPTION_KEY_FILE
 
 # Startup fdb cluster and backup agent.
diff --git a/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh b/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh
index f460e903eb3..334d200e070 100755
--- a/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh
+++ b/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh
@@ -164,7 +164,7 @@ function run_validate_restore_audit {
       break
     fi
     
-    if echo "${audit_output}" | grep -qE "1221|1230|1010"; then
+    if output_matches_E "${audit_output}" "1221|1230|1010"; then
       log "Transient error detected, retrying in ${retry_delay}s..."
       sleep $retry_delay
       continue
@@ -193,12 +193,12 @@ function run_validate_restore_audit {
     local status_output
     status_output=$("${fdbcli}" -C "${cluster_file}" --exec "get_audit_status validate_restore id ${audit_id}" 2>&1)
     
-    if echo "${status_output}" | grep -q "Phase.*2"; then
+    if output_matches "${status_output}" "Phase.*2"; then
       log "Audit completed successfully after ${elapsed}s"
       return 0
     fi
     
-    if echo "${status_output}" | grep -q "Phase.*[34]"; then
+    if output_matches "${status_output}" "Phase.*[34]"; then
       err "Audit failed with status: ${status_output}"
       return 1
     fi
diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.cpp
similarity index 77%
rename from fdbcli/AdvanceVersionCommand.actor.cpp
rename to fdbcli/AdvanceVersionCommand.cpp
index 656f06daeb6..7be9acc3916 100644
--- a/fdbcli/AdvanceVersionCommand.actor.cpp
+++ b/fdbcli/AdvanceVersionCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * AdvanceVersionCommand.actor.cpp
+ * AdvanceVersionCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,45 +20,47 @@
 
 #include "boost/lexical_cast.hpp"
 #include "fmt/format.h"
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/IClientApi.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
 const KeyRef advanceVersionSpecialKey = "\xff\xff/management/min_required_commit_version"_sr;
 
-ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() != 2) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	} else {
-		state Version v;
+		Version v;
 		int n = 0;
 		if (sscanf(tokens[1].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[1].size()) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		} else {
-			state Reference<ITransaction> tr = db->createTransaction();
-			loop {
+			Reference<ITransaction> tr = db->createTransaction();
+			while (true) {
 				tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+				Error err;
 				try {
-					Version rv = wait(safeThreadFutureToFuture(tr->getReadVersion()));
+					Version rv = co_await safeThreadFutureToFuture(tr->getReadVersion());
 					if (rv <= v) {
 						tr->set(advanceVersionSpecialKey, boost::lexical_cast<std::string>(v));
-						wait(safeThreadFutureToFuture(tr->commit()));
+						co_await safeThreadFutureToFuture(tr->commit());
+						continue;
 					} else {
 						fmt::print("Current read version is {}\n", rv);
-						return true;
+						co_return true;
 					}
 				} catch (Error& e) {
-					wait(safeThreadFutureToFuture(tr->onError(e)));
+					err = e;
 				}
+				co_await safeThreadFutureToFuture(tr->onError(err));
 			}
 		}
 	}
diff --git a/fdbcli/AuditStorageCommand.actor.cpp b/fdbcli/AuditStorageCommand.cpp
similarity index 89%
rename from fdbcli/AuditStorageCommand.actor.cpp
rename to fdbcli/AuditStorageCommand.cpp
index 3d63a58912c..b2f18ebc374 100644
--- a/fdbcli/AuditStorageCommand.actor.cpp
+++ b/fdbcli/AuditStorageCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * AuditStorageCommand.actor.cpp
+ * AuditStorageCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -58,32 +58,30 @@
  * ============================================================================
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/IClientApi.h"
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/Audit.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
-ACTOR Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> clusterFile,
-                                           std::vector<StringRef> tokens) {
+Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> clusterFile, std::vector<StringRef> tokens) {
 	if (tokens.size() < 2) {
 		printUsage(tokens[0]);
-		return UID();
+		co_return UID();
 	}
 
-	state UID resAuditId;
+	UID resAuditId;
 	if (tokencmp(tokens[1], "cancel")) {
 		if (tokens.size() != 4) {
 			printUsage(tokens[0]);
-			return UID();
+			co_return UID();
 		}
 		AuditType type = AuditType::Invalid;
 		if (tokencmp(tokens[2], "ha")) {
@@ -98,10 +96,10 @@ ACTOR Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> c
 			type = AuditType::ValidateRestore;
 		} else {
 			printUsage(tokens[0]);
-			return UID();
+			co_return UID();
 		}
 		const UID auditId = UID::fromString(tokens[3].toString());
-		UID cancelledAuditId = wait(cancelAuditStorage(clusterFile, type, auditId, /*timeoutSeconds=*/60));
+		UID cancelledAuditId = co_await cancelAuditStorage(clusterFile, type, auditId, /*timeoutSeconds=*/60);
 		resAuditId = cancelledAuditId;
 
 	} else {
@@ -118,7 +116,7 @@ ACTOR Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> c
 			type = AuditType::ValidateRestore;
 		} else {
 			printUsage(tokens[0]);
-			return UID();
+			co_return UID();
 		}
 
 		Key begin = allKeys.begin, end = allKeys.end;
@@ -129,15 +127,15 @@ ACTOR Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> c
 			end = tokens[3];
 		} else {
 			printUsage(tokens[0]);
-			return UID();
+			co_return UID();
 		}
 		if (end > allKeys.end) {
 			printUsage(tokens[0]);
-			return UID();
+			co_return UID();
 		}
 		if (begin >= end) {
 			printUsage(tokens[0]);
-			return UID();
+			co_return UID();
 		}
 
 		KeyValueStoreType engineType = KeyValueStoreType::END;
@@ -146,16 +144,16 @@ ACTOR Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> c
 			if (engineType != KeyValueStoreType::SSD_BTREE_V2 && engineType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
 			    engineType != KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
 				printUsage(tokens[0]);
-				return UID();
+				co_return UID();
 			}
 		}
 		// For KeyValueStoreType::END: do not specify any storage engine
 		// Every storage engine will be audited
 		UID startedAuditId =
-		    wait(auditStorage(clusterFile, KeyRangeRef(begin, end), type, engineType, /*timeoutSeconds=*/60));
+		    co_await auditStorage(clusterFile, KeyRangeRef(begin, end), type, engineType, /*timeoutSeconds=*/60);
 		resAuditId = startedAuditId;
 	}
-	return resAuditId;
+	co_return resAuditId;
 }
 
 CommandFactory auditStorageFactory(
diff --git a/fdbcli/BulkDumpCommand.actor.cpp b/fdbcli/BulkDumpCommand.cpp
similarity index 60%
rename from fdbcli/BulkDumpCommand.actor.cpp
rename to fdbcli/BulkDumpCommand.cpp
index fa95a1e1ea1..3776a7394ad 100644
--- a/fdbcli/BulkDumpCommand.actor.cpp
+++ b/fdbcli/BulkDumpCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkDumpCommand.actor.cpp
+ * BulkDumpCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,13 @@
 
 #include <cstddef>
 #include <fmt/core.h>
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 #include "fdbclient/BulkDumping.h"
 #include "fdbclient/BulkLoading.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "flow/Arena.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/Util.h"
 
 namespace fdb_cli {
 
@@ -41,45 +41,46 @@ static const std::string BULK_DUMP_HELP_MESSAGE =
     std::string(BULK_DUMP_MODE_USAGE) + std::string(BULK_DUMP_DUMP_USAGE) + std::string(BULK_DUMP_STATUS_USAGE) +
     std::string(BULK_DUMP_CANCEL_USAGE);
 
-ACTOR Future<bool> getOngoingBulkDumpJob(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<bool> getOngoingBulkDumpJob(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			Optional<BulkDumpState> job = wait(getSubmittedBulkDumpJob(&tr));
+			Optional<BulkDumpState> job = co_await getSubmittedBulkDumpJob(&tr);
 			if (job.present()) {
 				fmt::println("Running bulk dumping job: {}", job.get().getJobId().toString());
-				return true;
+				co_return true;
 			} else {
 				fmt::println("No bulk dumping job is running");
-				return false;
+				co_return false;
 			}
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> getBulkDumpCompleteRanges(Database cx, KeyRange rangeToRead) {
+Future<Void> getBulkDumpCompleteRanges(Database cx, KeyRange rangeToRead) {
 	try {
-		size_t finishCount = wait(getBulkDumpCompleteTaskCount(cx, rangeToRead));
+		size_t finishCount = co_await getBulkDumpCompleteTaskCount(cx, rangeToRead);
 		fmt::println("Finished {} tasks", finishCount);
 	} catch (Error& e) {
 		if (e.code() == error_code_timed_out) {
 			fmt::println("timed out");
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> tokens) {
-	state BulkDumpState bulkDumpJob;
+Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> tokens) {
+	BulkDumpState bulkDumpJob;
 	if (tokencmp(tokens[1], "mode")) {
 		if (tokens.size() != 2 && tokens.size() != 3) {
 			fmt::println("{}", BULK_DUMP_MODE_USAGE);
-			return UID();
+			co_return UID();
 		}
 		if (tokens.size() == 2) {
-			int mode = wait(getBulkDumpMode(cx));
+			int mode = co_await getBulkDumpMode(cx);
 			if (mode == 0) {
 				fmt::println("Bulkdump mode is disabled");
 			} else if (mode == 1) {
@@ -87,32 +88,32 @@ ACTOR Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> token
 			} else {
 				fmt::println("Invalid bulkload mode value {}", mode);
 			}
-			return UID();
+			co_return UID();
 		}
 		ASSERT(tokens.size() == 3);
 		if (tokencmp(tokens[2], "on")) {
-			int old = wait(setBulkDumpMode(cx, 1));
+			int old = co_await setBulkDumpMode(cx, 1);
 			TraceEvent("SetBulkDumpModeCommand").detail("OldValue", old).detail("NewValue", 1);
-			return UID();
+			co_return UID();
 		} else if (tokencmp(tokens[2], "off")) {
-			int old = wait(setBulkDumpMode(cx, 0));
+			int old = co_await setBulkDumpMode(cx, 0);
 			TraceEvent("SetBulkDumpModeCommand").detail("OldValue", old).detail("NewValue", 0);
-			return UID();
+			co_return UID();
 		} else {
 			fmt::println("ERROR: Invalid bulkdump mode value {}", tokens[2].toString());
 			fmt::println("{}", BULK_DUMP_MODE_USAGE);
-			return UID();
+			co_return UID();
 		}
 
 	} else if (tokencmp(tokens[1], "dump")) {
-		int mode = wait(getBulkDumpMode(cx));
+		int mode = co_await getBulkDumpMode(cx);
 		if (mode == 0) {
 			fmt::println("ERROR: Bulkdump mode must be enabled to dump data");
-			return UID();
+			co_return UID();
 		}
 		if (tokens.size() != 5) {
 			fmt::println("{}", BULK_DUMP_DUMP_USAGE);
-			return UID();
+			co_return UID();
 		}
 		Key rangeBegin = tokens[2];
 		Key rangeEnd = tokens[3];
@@ -121,7 +122,7 @@ ACTOR Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> token
 			fmt::println(
 			    "ERROR: Invalid range: {} to {}, normal key space only", rangeBegin.toString(), rangeEnd.toString());
 			fmt::println("{}", BULK_DUMP_DUMP_USAGE);
-			return UID();
+			co_return UID();
 		}
 		KeyRange range = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
 		std::string jobRoot = tokens[4].toString();
@@ -130,35 +131,83 @@ ACTOR Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> token
 		                                BulkLoadType::SST,
 		                                jobRoot.find("blobstore://") == 0 ? BulkLoadTransportMethod::BLOBSTORE
 		                                                                  : BulkLoadTransportMethod::CP);
-		wait(submitBulkDumpJob(cx, bulkDumpJob));
-		return bulkDumpJob.getJobId();
+		co_await submitBulkDumpJob(cx, bulkDumpJob);
+		co_return bulkDumpJob.getJobId();
 	} else if (tokencmp(tokens[1], "cancel")) {
 		if (tokens.size() != 3) {
 			fmt::println("{}", BULK_DUMP_CANCEL_USAGE);
-			return UID();
+			co_return UID();
 		}
-		state UID jobId = UID::fromString(tokens[2].toString());
-		wait(cancelBulkDumpJob(cx, jobId));
+		UID jobId = validateBulkJobId(tokens[2], BULK_DUMP_CANCEL_USAGE);
+		co_await cancelBulkDumpJob(cx, jobId);
 		fmt::println("Job {} has been cancelled. No new tasks will be spawned.", jobId.toString());
-		return UID();
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "status")) {
 		if (tokens.size() != 2) {
 			fmt::println("{}", BULK_DUMP_STATUS_USAGE);
-			return UID();
+			co_return UID();
 		}
-		bool anyJob = wait(getOngoingBulkDumpJob(cx));
-		if (!anyJob) {
-			return UID();
+
+		// Get aggregated progress
+		Optional<BulkDumpProgress> progressOpt = co_await getBulkDumpProgress(cx);
+		if (!progressOpt.present()) {
+			fmt::println("No bulk dumping job is running");
+			co_return UID();
+		}
+
+		BulkDumpProgress progress = progressOpt.get();
+
+		// ctest is dependent on this output; don't change it.
+		fmt::println("Running bulk dumping job: {}", progress.jobId.toString());
+
+		// Check if this bulkdump is owned by another operation
+		Transaction tr(cx);
+		Optional<BulkDumpState> jobState;
+		while (true) {
+			Error err;
+			try {
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				Optional<BulkDumpState> job = co_await getSubmittedBulkDumpJob(&tr);
+				jobState = job;
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+
+		std::string ownerSuffix = co_await getBulkOwnerSuffix(cx, progress.jobId, true);
+
+		fmt::println(" Range: {}{}", progress.jobRange.toString(), ownerSuffix);
+		fmt::println("");
+		fmt::println("Progress:");
+		int runningTasks = progress.totalTasks - progress.completeTasks - progress.errorTasks;
+		if (progress.totalTasks > 0) {
+			fmt::println(" Tasks: {}/{} complete ({:.1f}%)  |  {} running{}",
+			             progress.completeTasks,
+			             progress.totalTasks,
+			             progress.progressPercent(),
+			             runningTasks > 0 ? runningTasks : 0,
+			             progress.errorTasks > 0 ? fmt::format(", {} error", progress.errorTasks) : "");
 		}
-		KeyRange range = Standalone(KeyRangeRef(normalKeys.begin, normalKeys.end));
-		wait(getBulkDumpCompleteRanges(cx, range));
-		return UID();
+
+		fmt::println(" Bytes: {}", formatBytesProgress(progress.completedBytes, progress.totalBytes));
+
+		printProgressMetrics(progress.avgBytesPerSecond(), progress.etaSeconds(), progress.elapsedSeconds);
+
+		if (progress.errorTasks > 0) {
+			fmt::println("");
+			fmt::println("WARNING: {} tasks in error state", progress.errorTasks);
+		}
+
+		co_return UID();
 
 	} else {
 		printUsage(tokens[0]);
 		printLongDesc(tokens[0]);
-		return UID();
+		co_return UID();
 	}
 }
 
diff --git a/fdbcli/BulkLoadCommand.actor.cpp b/fdbcli/BulkLoadCommand.cpp
similarity index 68%
rename from fdbcli/BulkLoadCommand.actor.cpp
rename to fdbcli/BulkLoadCommand.cpp
index 63aa9af9b37..3bfb1a64f8a 100644
--- a/fdbcli/BulkLoadCommand.actor.cpp
+++ b/fdbcli/BulkLoadCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkLoadCommand.actor.cpp
+ * BulkLoadCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbcli/fdbcli.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/BulkLoading.h"
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/Util.h"
 
 namespace fdb_cli {
 
@@ -50,32 +50,32 @@ static const std::string BULK_LOAD_HELP_MESSAGE =
     BULK_LOAD_HISTORY_USAGE + BULK_LOAD_HISTORY_CLEAR_USAGE + BULKLOAD_ADD_LOCK_OWNER_USAGE +
     BULKLOAD_PRINT_LOCK_USAGE + BULKLOAD_PRINT_LOCK_OWNER_USAGE + BULKLOAD_CLEAR_LOCK_USAGE;
 
-ACTOR Future<Void> printPastBulkLoadJob(Database cx) {
-	std::vector<BulkLoadJobState> jobs = wait(getBulkLoadJobFromHistory(cx));
+Future<Void> printPastBulkLoadJob(Database cx) {
+	std::vector<BulkLoadJobState> jobs = co_await getBulkLoadJobFromHistory(cx);
 	if (jobs.empty()) {
 		fmt::println("No bulk loading job in the history");
-		return Void();
+		co_return;
 	}
 	for (const auto& job : jobs) {
 		ASSERT(job.getPhase() == BulkLoadJobPhase::Complete || job.getPhase() == BulkLoadJobPhase::Error ||
 		       job.getPhase() == BulkLoadJobPhase::Cancelled);
 		if (!job.getTaskCount().present()) {
-			fmt::println("Job {} submitted at {} for range {}. The job has not initialized for {} mins and exited "
+			fmt::println("Job {} submitted at {} for range {}. The job has not initialized for {:.1f} mins and exited "
 			             "with status {}.",
 			             job.getJobId().toString(),
-			             std::to_string(job.getSubmitTime()),
+			             formatTimeISO8601(job.getSubmitTime()),
 			             job.getJobRange().toString(),
-			             std::to_string((job.getEndTime() - job.getSubmitTime()) / 60.0),
+			             (job.getEndTime() - job.getSubmitTime()) / 60.0,
 			             convertBulkLoadJobPhaseToString(job.getPhase()));
 		} else {
 			fmt::println(
-			    "Job {} submitted at {} for range {}. The job has {} tasks. The job ran for {} mins and exited "
+			    "Job {} submitted at {} for range {}. The job has {} tasks. The job ran for {:.1f} mins and exited "
 			    "with status {}.",
 			    job.getJobId().toString(),
-			    std::to_string(job.getSubmitTime()),
+			    formatTimeISO8601(job.getSubmitTime()),
 			    job.getJobRange().toString(),
 			    job.getTaskCount().get(),
-			    std::to_string((job.getEndTime() - job.getSubmitTime()) / 60.0),
+			    (job.getEndTime() - job.getSubmitTime()) / 60.0,
 			    convertBulkLoadJobPhaseToString(job.getPhase()));
 		}
 		if (job.getPhase() == BulkLoadJobPhase::Error) {
@@ -83,7 +83,6 @@ ACTOR Future<Void> printPastBulkLoadJob(Database cx) {
 			fmt::println("Error message: {}", errorMessage.present() ? errorMessage.get() : "Not provided.");
 		}
 	}
-	return Void();
 }
 
 void printBulkLoadJobTotalTaskCount(Optional<uint64_t> count) {
@@ -95,22 +94,24 @@ void printBulkLoadJobTotalTaskCount(Optional<uint64_t> count) {
 	return;
 }
 
-ACTOR Future<Void> printBulkLoadJobProgress(Database cx, BulkLoadJobState job) {
-	state Transaction tr(cx);
-	state Key readBegin = job.getJobRange().begin;
-	state Key readEnd = job.getJobRange().end;
-	state UID jobId = job.getJobId();
-	state RangeResult rangeResult;
-	state size_t completeTaskCount = 0;
-	state size_t submitTaskCount = 0;
-	state size_t errorTaskCount = 0;
-	state Optional<uint64_t> totalTaskCount = job.getTaskCount();
+Future<Void> printBulkLoadJobProgress(Database cx, BulkLoadJobState job) {
+	Transaction tr(cx);
+	Key readBegin = job.getJobRange().begin;
+	Key readEnd = job.getJobRange().end;
+	UID jobId = job.getJobId();
+	RangeResult rangeResult;
+	size_t completeTaskCount = 0;
+	size_t submitTaskCount = 0;
+	size_t errorTaskCount = 0;
+	Optional<uint64_t> totalTaskCount = job.getTaskCount();
 	while (readBegin < readEnd) {
+		Error err;
+		bool hasErr = false;
 		try {
 			rangeResult.clear();
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			wait(store(rangeResult, krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(readBegin, readEnd))));
+			rangeResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(readBegin, readEnd));
 			for (int i = 0; i < rangeResult.size() - 1; ++i) {
 				if (rangeResult[i].value.empty()) {
 					continue;
@@ -121,11 +122,10 @@ ACTOR Future<Void> printBulkLoadJobProgress(Database cx, BulkLoadJobState job) {
 					fmt::println("Finished {} tasks", completeTaskCount);
 					fmt::println("Error {} tasks", errorTaskCount);
 					printBulkLoadJobTotalTaskCount(totalTaskCount);
-					if (bulkLoadTask.phase == BulkLoadPhase::Submitted &&
-					    bulkLoadTask.getJobId() != UID::fromString("00000000-0000-0000-0000-000000000000")) {
+					if (bulkLoadTask.phase == BulkLoadPhase::Submitted && bulkLoadTask.getJobId().isValid()) {
 						fmt::println("Job {} has been cancelled or has completed", jobId.toString());
 					}
-					return Void();
+					co_return;
 				}
 				if (bulkLoadTask.phase == BulkLoadPhase::Complete) {
 					completeTaskCount = completeTaskCount + bulkLoadTask.getManifests().size();
@@ -136,20 +136,23 @@ ACTOR Future<Void> printBulkLoadJobProgress(Database cx, BulkLoadJobState job) {
 			}
 			readBegin = rangeResult.back().key;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
+			hasErr = true;
+		}
+		if (hasErr) {
+			co_await tr.onError(err);
 		}
 	}
 	fmt::println("Submitted {} tasks", submitTaskCount);
 	fmt::println("Finished {} tasks", completeTaskCount);
 	fmt::println("Error {} tasks", errorTaskCount);
 	printBulkLoadJobTotalTaskCount(totalTaskCount);
-	return Void();
 }
 
-ACTOR Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> tokens) {
+Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> tokens) {
 	if (tokencmp(tokens[1], "mode")) {
 		if (tokens.size() == 2) {
-			int mode = wait(getBulkLoadMode(cx));
+			int mode = co_await getBulkLoadMode(cx);
 			if (mode == 0) {
 				fmt::println("Bulkload mode is disabled");
 			} else if (mode == 1) {
@@ -157,42 +160,37 @@ ACTOR Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> token
 			} else {
 				fmt::println("Invalid bulkload mode value {}", mode);
 			}
-			return UID();
+			co_return UID();
 		}
 		// Set bulk loading mode
 		if (tokens.size() != 3) {
 			fmt::println("{}", BULK_LOAD_MODE_USAGE);
-			return UID();
+			co_return UID();
 		}
 		if (tokencmp(tokens[2], "on")) {
-			int old = wait(setBulkLoadMode(cx, 1));
+			int old = co_await setBulkLoadMode(cx, 1);
 			TraceEvent("SetBulkLoadModeCommand").detail("OldValue", old).detail("NewValue", 1);
-			return UID();
+			co_return UID();
 		} else if (tokencmp(tokens[2], "off")) {
-			int old = wait(setBulkLoadMode(cx, 0));
+			int old = co_await setBulkLoadMode(cx, 0);
 			TraceEvent("SetBulkLoadModeCommand").detail("OldValue", old).detail("NewValue", 0);
-			return UID();
+			co_return UID();
 		} else {
 			fmt::println("ERROR: Invalid bulkload mode value {}", tokens[2].toString());
 			fmt::println("{}", BULK_LOAD_MODE_USAGE);
-			return UID();
+			co_return UID();
 		}
 	} else if (tokencmp(tokens[1], "load")) {
-		int mode = wait(getBulkLoadMode(cx));
+		int mode = co_await getBulkLoadMode(cx);
 		if (mode == 0) {
 			fmt::println("ERROR: Bulkload mode must be enabled to load data");
-			return UID();
+			co_return UID();
 		}
 		if (tokens.size() != 6) {
 			fmt::println("{}", BULK_LOAD_LOAD_USAGE);
-			return UID();
-		}
-		UID jobId = UID::fromString(tokens[2].toString());
-		if (!jobId.isValid()) {
-			fmt::println("ERROR: Invalid job id {}", tokens[2].toString());
-			fmt::println("{}", BULK_LOAD_LOAD_USAGE);
-			return UID();
+			co_return UID();
 		}
+		UID jobId = validateBulkJobId(tokens[2], BULK_LOAD_LOAD_USAGE.c_str());
 		Key rangeBegin = tokens[3];
 		Key rangeEnd = tokens[4];
 		// Bulk load can only inject data to normal key space, aka "" ~ \xff
@@ -200,118 +198,142 @@ ACTOR Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> token
 			fmt::println(
 			    "ERROR: Invalid range: {} to {}, normal key space only", rangeBegin.toString(), rangeEnd.toString());
 			fmt::println("{}", BULK_LOAD_LOAD_USAGE);
-			return UID();
+			co_return UID();
 		}
 		std::string jobRoot = tokens[5].toString();
 		KeyRange range = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
-		state BulkLoadJobState bulkLoadJob = createBulkLoadJob(
+		BulkLoadJobState bulkLoadJob = createBulkLoadJob(
 		    jobId,
 		    range,
 		    jobRoot,
 		    jobRoot.find("blobstore://") == 0 ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP);
-		wait(submitBulkLoadJob(cx, bulkLoadJob));
-		return bulkLoadJob.getJobId();
+		co_await submitBulkLoadJob(cx, bulkLoadJob);
+		co_return bulkLoadJob.getJobId();
 	} else if (tokencmp(tokens[1], "cancel")) {
 		if (tokens.size() != 3) {
 			fmt::println("{}", BULK_LOAD_CANCEL_USAGE);
-			return UID();
-		}
-		state UID jobId = UID::fromString(tokens[2].toString());
-		if (!jobId.isValid()) {
-			fmt::println("ERROR: Invalid job id {}", tokens[2].toString());
-			fmt::println("{}", BULK_LOAD_CANCEL_USAGE);
-			return UID();
+			co_return UID();
 		}
-		wait(cancelBulkLoadJob(cx, jobId));
-		fmt::println("Job {} has been cancelled. The job range lock has been cleared", jobId.toString());
-		return UID();
+		UID cancelJobId = validateBulkJobId(tokens[2], BULK_LOAD_CANCEL_USAGE.c_str());
+		co_await cancelBulkLoadJob(cx, cancelJobId);
+		fmt::println("Job {} has been cancelled. The job range lock has been cleared", cancelJobId.toString());
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "status")) {
 		if (tokens.size() != 2) {
 			fmt::println("{}", BULK_LOAD_STATUS_USAGE);
-			return UID();
+			co_return UID();
 		}
-		Optional<BulkLoadJobState> job = wait(getRunningBulkLoadJob(cx));
-		if (!job.present()) {
+
+		// Get aggregated progress
+		Optional<BulkLoadProgress> progressOpt = co_await getBulkLoadProgress(cx);
+		if (!progressOpt.present()) {
 			fmt::println("No bulk loading job is running");
-			return UID();
+			co_return UID();
 		}
-		fmt::println("Running bulk loading job: {}", job.get().getJobId().toString());
-		fmt::println("Job information: {}", job.get().toString());
-		wait(printBulkLoadJobProgress(cx, job.get()));
-		return UID();
+
+		BulkLoadProgress progress = progressOpt.get();
+
+		fmt::println("BulkLoad job {} is in progress.", progress.jobId.toString());
+
+		std::string ownerSuffix = co_await getBulkOwnerSuffix(cx, progress.jobId, false);
+
+		fmt::println(" Range: {}{}", progress.jobRange.toString(), ownerSuffix);
+		fmt::println("");
+		fmt::println("Progress:");
+
+		int totalTasks = progress.submittedTasks + progress.triggeredTasks + progress.runningTasks +
+		                 progress.completeTasks + progress.errorTasks;
+		if (totalTasks > 0) {
+			fmt::println(" Tasks: {}/{} complete ({:.1f}%)  |  {} submitted, {} triggered, {} running{}",
+			             progress.completeTasks,
+			             totalTasks,
+			             100.0 * progress.completeTasks / totalTasks,
+			             progress.submittedTasks,
+			             progress.triggeredTasks,
+			             progress.runningTasks,
+			             progress.errorTasks > 0 ? fmt::format(", {} error", progress.errorTasks) : "");
+		}
+
+		fmt::println(" Bytes: {}", formatBytesProgress(progress.completedBytes, progress.totalBytes));
+
+		printProgressMetrics(progress.avgBytesPerSecond(), progress.etaSeconds(), progress.elapsedSeconds);
+
+		// Only show warnings for actual problems (stalled tasks, errors)
+		printStalledTasks(progress.stalledTasks);
+		if (progress.errorTasks > 0) {
+			fmt::println("");
+			fmt::println("WARNING: {} tasks in error state", progress.errorTasks);
+		}
+
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "history")) {
 		if (tokens.size() == 2) {
-			wait(printPastBulkLoadJob(cx));
-			return UID();
+			co_await printPastBulkLoadJob(cx);
+			co_return UID();
 		}
 		if (tokens.size() == 3) {
 			if (tokencmp(tokens[2], "clear")) {
 				fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE);
-				return UID();
+				co_return UID();
 			} else {
 				fmt::println("ERROR: Invalid history option {}", tokens[2].toString());
 				fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE);
-				return UID();
+				co_return UID();
 			}
 		}
 		if (tokens.size() == 4) {
 			if (tokencmp(tokens[2], "clear")) {
 				if (tokencmp(tokens[3], "all")) {
-					wait(clearBulkLoadJobHistory(cx));
+					co_await clearBulkLoadJobHistory(cx);
 					fmt::println("All bulkload job history has been cleared");
-					return UID();
+					co_return UID();
 				} else {
 					fmt::println("ERROR: Invalid history clear option {}", tokens[3].toString());
 					fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE);
-					return UID();
+					co_return UID();
 				}
 			} else {
 				fmt::println("ERROR: Invalid history clear option {}", tokens[2].toString());
 				fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE);
-				return UID();
+				co_return UID();
 			}
 		}
 		if (tokens.size() == 5) {
 			if (tokencmp(tokens[2], "clear") && tokencmp(tokens[3], "id")) {
-				UID jobId = UID::fromString(tokens[4].toString());
-				if (!jobId.isValid()) {
-					fmt::println("ERROR: Invalid job id {}", tokens[4].toString());
-					fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE);
-					return UID();
-				}
-				wait(clearBulkLoadJobHistory(cx, jobId));
+				UID jobId = validateBulkJobId(tokens[4], BULK_LOAD_HISTORY_CLEAR_USAGE.c_str());
+				co_await clearBulkLoadJobHistory(cx, jobId);
 				fmt::println("Bulkload job {} has been cleared from history", jobId.toString());
-				return jobId;
+				co_return jobId;
 			}
 		}
 		printLongDesc(tokens[0]);
-		return UID();
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "addlockowner")) {
 		// For debugging purposes and invisible to users.
 		if (tokens.size() != 3) {
 			fmt::println("{}", BULK_LOAD_STATUS_USAGE);
-			return UID();
+			co_return UID();
 		}
 		std::string ownerUniqueID = tokens[2].toString();
 		if (ownerUniqueID.empty()) {
 			fmt::println("ERROR: Owner unique id cannot be empty");
 			fmt::println("{}", BULKLOAD_ADD_LOCK_OWNER_USAGE);
-			return UID();
+			co_return UID();
 		}
-		wait(registerRangeLockOwner(cx, ownerUniqueID, ownerUniqueID));
-		return UID();
+		co_await registerRangeLockOwner(cx, ownerUniqueID, ownerUniqueID);
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "printlock")) {
 		// For debugging purposes and invisible to users.
 		if (tokens.size() != 2) {
 			fmt::println("{}", BULKLOAD_PRINT_LOCK_USAGE);
-			return UID();
+			co_return UID();
 		}
 		std::vector<std::pair<KeyRange, RangeLockState>> lockedRanges =
-		    wait(findExclusiveReadLockOnRange(cx, normalKeys));
+		    co_await findExclusiveReadLockOnRange(cx, normalKeys);
 		fmt::println("Total {} locked ranges", lockedRanges.size());
 		if (lockedRanges.size() > 10) {
 			fmt::println("First 10 locks are:");
@@ -324,34 +346,34 @@ ACTOR Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> token
 			fmt::println("Lock {} on {} for {}", count, lock.first.toString(), lock.second.toString());
 			count++;
 		}
-		return UID();
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "printlockowner")) {
 		// For debugging purposes and invisible to users.
 		if (tokens.size() != 2) {
 			fmt::println("{}", BULKLOAD_PRINT_LOCK_OWNER_USAGE);
-			return UID();
+			co_return UID();
 		}
-		std::vector<RangeLockOwner> owners = wait(getAllRangeLockOwners(cx));
-		for (const auto owner : owners) {
+		std::vector<RangeLockOwner> owners = co_await getAllRangeLockOwners(cx);
+		for (const auto& owner : owners) {
 			fmt::println("{}", owner.toString());
 		}
-		return UID();
+		co_return UID();
 
 	} else if (tokencmp(tokens[1], "clearlock")) {
 		// For debugging purposes and invisible to users.
 		if (tokens.size() != 3) {
 			fmt::println("{}", BULKLOAD_CLEAR_LOCK_USAGE);
-			return UID();
+			co_return UID();
 		}
 		std::string ownerUniqueID = tokens[2].toString();
-		wait(releaseExclusiveReadLockByUser(cx, ownerUniqueID));
-		return UID();
+		co_await releaseExclusiveReadLockByUser(cx, ownerUniqueID);
+		co_return UID();
 
 	} else {
 		printUsage(tokens[0]);
 		printLongDesc(tokens[0]);
-		return UID();
+		co_return UID();
 	}
 }
 
diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt
index 090649d080f..ad28477312c 100644
--- a/fdbcli/CMakeLists.txt
+++ b/fdbcli/CMakeLists.txt
@@ -2,7 +2,7 @@ include(AddFdbTest)
 fdb_find_sources(FDBCLI_SRCS)
 
 add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
-target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
 target_link_libraries(fdbcli PRIVATE fdbctl fdbclient SimpleOpt)
 
 if (USE_UBSAN)
diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.cpp
similarity index 90%
rename from fdbcli/ConfigureCommand.actor.cpp
rename to fdbcli/ConfigureCommand.cpp
index 3f0a4164abe..13566965a28 100644
--- a/fdbcli/ConfigureCommand.actor.cpp
+++ b/fdbcli/ConfigureCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ConfigureCommand.actor.cpp
+ * ConfigureCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,28 +19,28 @@
  */
 
 #include "fdbcli/FlowLineNoise.h"
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
-ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
-                                         Database localDb,
-                                         std::vector<StringRef> tokens,
-                                         LineNoise* linenoise,
-                                         Future<Void> warn) {
-	state ConfigurationResult result;
-	state StatusObject s;
-	state int startToken = 1;
-	state bool force = false;
+Future<bool> configureCommandActor(Reference<IDatabase> db,
+                                   Database localDb,
+                                   std::vector<StringRef> tokens,
+                                   LineNoise* linenoise,
+                                   Future<Void> _warn) {
+	auto warn = std::move(_warn);
+	ConfigurationResult result = ConfigurationResult::SUCCESS;
+	StatusObject s;
+	int startToken = 1;
+	bool force = false;
 	if (tokens.size() < 2)
 		result = ConfigurationResult::NO_OPTIONS_PROVIDED;
 	else {
@@ -49,19 +49,19 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 			startToken = 2;
 		}
 
-		state Optional<ConfigureAutoResult> conf;
+		Optional<ConfigureAutoResult> conf;
 		if (tokens[startToken] == "auto"_sr) {
 			// get cluster status
-			state Reference<ITransaction> tr = db->createTransaction();
+			Reference<ITransaction> tr = db->createTransaction();
 			if (!tr->isValid()) {
-				StatusObject _s = wait(StatusClient::statusFetcher(localDb));
+				StatusObject _s = co_await StatusClient::statusFetcher(localDb);
 				s = _s;
 			} else {
-				state ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
-				Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
+				ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
+				Optional<Value> statusValue = co_await safeThreadFutureToFuture(statusValueF);
 				if (!statusValue.present()) {
 					fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
-					return false;
+					co_return false;
 				}
 				json_spirit::mValue mv;
 				json_spirit::read_string(statusValue.get().toString(), mv);
@@ -75,7 +75,7 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 
 			if (!conf.get().isValid()) {
 				printf("Unable to provide advice for the current configuration.\n");
-				return false;
+				co_return false;
 			}
 
 			bool noChanges = conf.get().old_replication == conf.get().auto_replication &&
@@ -143,13 +143,13 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 			std::printf("%s", outputString.c_str());
 
 			if (noChanges)
-				return true;
+				co_return true;
 
 			// TODO: disable completion
-			Optional<std::string> line = wait(linenoise->read("Would you like to make these changes? [y/n]> "));
+			Optional<std::string> line = co_await linenoise->read("Would you like to make these changes? [y/n]> ");
 
 			if (!line.present() || (line.get() != "y" && line.get() != "Y")) {
-				return true;
+				co_return true;
 			}
 		}
 
@@ -160,11 +160,16 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 				result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED;
 				break;
 			}
+			if (it->startsWith("range_backup_worker_enabled:="_sr)) {
+				result = ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED;
+				break;
+			}
 		}
 
-		if (result != ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED) {
-			ConfigurationResult r = wait(ManagementAPI::changeConfig(
-			    db, std::vector<StringRef>(tokens.begin() + startToken, tokens.end()), conf, force));
+		if (result != ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED &&
+		    result != ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED) {
+			ConfigurationResult r = co_await ManagementAPI::changeConfig(
+			    db, std::vector<StringRef>(tokens.begin() + startToken, tokens.end()), conf, force);
 			result = r;
 		}
 	}
@@ -287,11 +292,17 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 		        "backup system.\n");
 		ret = false;
 		break;
+	case ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED:
+		fprintf(stderr,
+		        "ERROR: range_backup_worker_enabled configuration is restricted in fdbcli and managed "
+		        "automatically by the backup system.\n");
+		ret = false;
+		break;
 	default:
 		ASSERT(false);
 		ret = false;
 	};
-	return ret;
+	co_return ret;
 }
 
 void configureGenerator(const char* text,
@@ -317,7 +328,6 @@ void configureGenerator(const char* text,
 		                   "resolvers=",
 		                   "perpetual_storage_wiggle=",
 		                   "perpetual_storage_wiggle_locality=",
-		                   // TODO(zhewu): update fdbcli command documentation.
 		                   "perpetual_storage_wiggle_engine=",
 		                   "storage_migration_type=",
 		                   nullptr };
@@ -331,7 +341,8 @@ CommandFactory configureFactory(
         "<single|double|triple|three_data_hall|three_datacenter|ssd|memory|memory-radixtree|proxies=<PROXIES>|"
         "commit_proxies=<COMMIT_PROXIES>|grv_proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*|"
         "count=<TSS_COUNT>|perpetual_storage_wiggle=<WIGGLE_SPEED>|perpetual_storage_wiggle_locality="
-        "<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>|storage_migration_type={disabled|gradual|aggressive}"
+        "<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>|perpetual_storage_wiggle_engine=<ENGINE>|"
+        "storage_migration_type={disabled|gradual|aggressive}"
         "|exclude=<ADDRESS...>",
         "change the database configuration",
         "The `new' option, if present, initializes a new database with the given configuration rather than changing "
diff --git a/fdbcli/ConsistencyCheckCommand.actor.cpp b/fdbcli/ConsistencyCheckCommand.cpp
similarity index 72%
rename from fdbcli/ConsistencyCheckCommand.actor.cpp
rename to fdbcli/ConsistencyCheckCommand.cpp
index 6745e3b830b..3540fe6c181 100644
--- a/fdbcli/ConsistencyCheckCommand.actor.cpp
+++ b/fdbcli/ConsistencyCheckCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ConsistencyCheckCommand.actor.cpp
+ * ConsistencyCheckCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -26,36 +26,34 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
 const KeyRef consistencyCheckSpecialKey = "\xff\xff/management/consistency_check_suspended"_sr;
 
-ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
-                                                std::vector<StringRef> tokens,
-                                                bool intrans) {
+Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
+                                          std::vector<StringRef> const& tokens,
+                                          bool intrans) {
 	// Here we do not proceed in a try-catch loop since the transaction is always supposed to succeed.
-	// If not, the outer loop catch block(fdbcli.actor.cpp) will handle the error and print out the error message
+	// If not, the outer loop catch block(fdbcli.cpp) will handle the error and print out the error message
 	tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 	if (tokens.size() == 1) {
 		// hold the returned standalone object's memory
-		state ThreadFuture<Optional<Value>> suspendedF = tr->get(consistencyCheckSpecialKey);
-		Optional<Value> suspended = wait(safeThreadFutureToFuture(suspendedF));
+		ThreadFuture<Optional<Value>> suspendedF = tr->get(consistencyCheckSpecialKey);
+		Optional<Value> suspended = co_await safeThreadFutureToFuture(suspendedF);
 		printf("ConsistencyCheck is %s\n", suspended.present() ? "off" : "on");
 	} else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) {
 		tr->set(consistencyCheckSpecialKey, Value());
 		if (!intrans)
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 	} else if (tokens.size() == 2 && tokencmp(tokens[1], "on")) {
 		tr->clear(consistencyCheckSpecialKey);
 		if (!intrans)
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
-	return true;
+	co_return true;
 }
 
 CommandFactory consistencyCheckFactory(
diff --git a/fdbcli/ConsistencyScanCommand.actor.cpp b/fdbcli/ConsistencyScanCommand.cpp
similarity index 77%
rename from fdbcli/ConsistencyScanCommand.actor.cpp
rename to fdbcli/ConsistencyScanCommand.cpp
index 1b215dd8d1d..0d56b7a0c35 100644
--- a/fdbcli/ConsistencyScanCommand.actor.cpp
+++ b/fdbcli/ConsistencyScanCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ConsistencyScanCommand.actor.cpp
+ * ConsistencyScanCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,42 +20,41 @@
 
 #include <boost/lexical_cast.hpp>
 #include <list>
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/RunTransaction.actor.h"
-#include "fdbclient/ConsistencyScanInterface.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/RunTransaction.h"
+#include "fdbclient/ConsistencyScanInterface.h"
 
 namespace fdb_cli {
 
-ACTOR Future<Void> dumpStats(ConsistencyScanState* cs, Reference<ReadYourWritesTransaction> tr) {
-	state ConsistencyScanState::LifetimeStats statsLifetime;
-	state ConsistencyScanState::RoundStats statsCurrentRound;
-	wait(store(statsLifetime, cs->lifetimeStats().getD(tr)) &&
-	     store(statsCurrentRound, cs->currentRoundStats().getD(tr)));
+Future<Void> dumpStats(ConsistencyScanState* cs, Reference<ReadYourWritesTransaction> tr) {
+	ConsistencyScanState::LifetimeStats statsLifetime;
+	ConsistencyScanState::RoundStats statsCurrentRound;
+	co_await (store(statsLifetime, cs->lifetimeStats().getD(tr)) &&
+	          store(statsCurrentRound, cs->currentRoundStats().getD(tr)));
 	printf(
 	    "Current Round:\n%s\n",
 	    json_spirit::write_string(json_spirit::mValue(statsCurrentRound.toJSON()), json_spirit::pretty_print).c_str());
 	printf("Lifetime:\n%s\n",
 	       json_spirit::write_string(json_spirit::mValue(statsLifetime.toJSON()), json_spirit::pretty_print).c_str());
-	return Void();
+	co_return;
 }
 
-ACTOR Future<bool> consistencyScanCommandActor(Database db, std::vector<StringRef> tokens) {
+Future<bool> consistencyScanCommandActor(Database db, std::vector<StringRef> const& tokens) {
 	// Skip the command token so start at begin+1
-	state std::list<StringRef> args(tokens.begin() + 1, tokens.end());
+	std::list<StringRef> args(tokens.begin() + 1, tokens.end());
 
-	state ConsistencyScanState cs = ConsistencyScanState();
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	state bool error = false;
+	ConsistencyScanState cs = ConsistencyScanState();
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
+	bool error = false;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			SystemDBWriteLockedNow(db.getReference())->setOptions(tr);
 
-			state ConsistencyScanState::Config config = wait(ConsistencyScanState().config().getD(tr));
+			ConsistencyScanState::Config config = co_await ConsistencyScanState().config().getD(tr);
 
 			if (args.empty()) {
 				printf(
@@ -76,9 +75,9 @@ ACTOR Future<bool> consistencyScanCommandActor(Database db, std::vector<StringRe
 				} else if (next == "restart") {
 					config.minStartVersion = tr->getReadVersion().get();
 				} else if (next == "stats") {
-					wait(dumpStats(&cs, tr));
+					co_await dumpStats(&cs, tr);
 				} else if (next == "clearstats") {
-					wait(cs.clearStats(tr));
+					co_await cs.clearStats(tr);
 				} else if (next == "maxRate") {
 					error = args.empty();
 					if (!error) {
@@ -98,19 +97,20 @@ ACTOR Future<bool> consistencyScanCommandActor(Database db, std::vector<StringRe
 				break;
 			}
 			cs.config().set(tr, config);
-			wait(tr->commit());
+			co_await tr->commit();
 			break;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 
 	if (error) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
-	return true;
+	co_return true;
 }
 
 CommandFactory consistencyScanFactory(
@@ -137,4 +137,4 @@ CommandFactory consistencyScanFactory(
         // TODO:  Syntax hint generator
         ));
 
-} // namespace fdb_cli
\ No newline at end of file
+} // namespace fdb_cli
diff --git a/fdbcli/CoordinatorsCommand.actor.cpp b/fdbcli/CoordinatorsCommand.cpp
similarity index 70%
rename from fdbcli/CoordinatorsCommand.actor.cpp
rename to fdbcli/CoordinatorsCommand.cpp
index d7b6ef947cc..319f5144c93 100644
--- a/fdbcli/CoordinatorsCommand.actor.cpp
+++ b/fdbcli/CoordinatorsCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * CoordinatorsCommand.actor.cpp
+ * CoordinatorsCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,50 +20,51 @@
 
 #include "boost/algorithm/string.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/Schemas.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
-ACTOR Future<Void> printCoordinatorsInfo(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> printCoordinatorsInfo(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
 			// Hold the reference to the standalone's memory
-			state ThreadFuture<Optional<Value>> descriptionF = tr->get(fdb_cli::clusterDescriptionSpecialKey);
-			Optional<Value> description = wait(safeThreadFutureToFuture(descriptionF));
+			ThreadFuture<Optional<Value>> descriptionF = tr->get(fdb_cli::clusterDescriptionSpecialKey);
+			Optional<Value> description = co_await safeThreadFutureToFuture(descriptionF);
 			ASSERT(description.present());
 			printf("Cluster description: %s\n", description.get().toString().c_str());
 			// Hold the reference to the standalone's memory
-			state ThreadFuture<Optional<Value>> processesF = tr->get(fdb_cli::coordinatorsProcessSpecialKey);
-			Optional<Value> processes = wait(safeThreadFutureToFuture(processesF));
+			ThreadFuture<Optional<Value>> processesF = tr->get(fdb_cli::coordinatorsProcessSpecialKey);
+			Optional<Value> processes = co_await safeThreadFutureToFuture(processesF);
 			ASSERT(processes.present());
 			std::vector<std::string> process_addresses;
 			boost::split(process_addresses, processes.get().toString(), [](char c) { return c == ','; });
 			printf("Cluster coordinators (%zu): %s\n", process_addresses.size(), processes.get().toString().c_str());
 			printf("Type `help coordinators' to learn how to change this information.\n");
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	state int retries = 0;
-	state int notEnoughMachineResults = 0;
-	state StringRef new_cluster_description;
-	state std::string auto_coordinators_str;
+Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	int retries = 0;
+	int notEnoughMachineResults = 0;
+	StringRef new_cluster_description;
+	std::string auto_coordinators_str;
 	StringRef nameTokenBegin = "description="_sr;
 	for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok) {
 		if (tok->startsWith(nameTokenBegin) && new_cluster_description.empty()) {
@@ -75,51 +76,52 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 		}
 	}
 
-	state bool automatic = tokens.size() == 2 && tokens[1] == "auto"_sr;
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+	bool automatic = tokens.size() == 2 && tokens[1] == "auto"_sr;
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error caughtErr;
+		bool hasCaughtErr = false;
 		try {
 			// update cluster description
-			if (new_cluster_description.size()) {
+			if (!new_cluster_description.empty()) {
 				tr->set(fdb_cli::clusterDescriptionSpecialKey, new_cluster_description);
 			}
 			// if auto change, read the special key to retrieve the recommended config
 			if (automatic) {
 				// if previous read failed, retry, otherwise, use the same recommended config
-				if (!auto_coordinators_str.size()) {
+				if (auto_coordinators_str.empty()) {
 					// Hold the reference to the standalone's memory
-					state ThreadFuture<Optional<Value>> auto_coordinatorsF =
-					    tr->get(fdb_cli::coordinatorsAutoSpecialKey);
-					Optional<Value> auto_coordinators = wait(safeThreadFutureToFuture(auto_coordinatorsF));
+					ThreadFuture<Optional<Value>> auto_coordinatorsF = tr->get(fdb_cli::coordinatorsAutoSpecialKey);
+					Optional<Value> auto_coordinators = co_await safeThreadFutureToFuture(auto_coordinatorsF);
 					ASSERT(auto_coordinators.present());
 					auto_coordinators_str = auto_coordinators.get().toString();
 				}
 				tr->set(fdb_cli::coordinatorsProcessSpecialKey, auto_coordinators_str);
 			} else if (tokens.size() > 1) {
-				state std::set<NetworkAddress> new_coordinators_addresses;
-				state std::set<Hostname> new_coordinators_hostnames;
-				state std::vector<std::string> newCoordinatorslist;
-				state std::vector<StringRef>::iterator t;
+				std::set<NetworkAddress> new_coordinators_addresses;
+				std::set<Hostname> new_coordinators_hostnames;
+				std::vector<std::string> newCoordinatorslist;
+				std::vector<StringRef>::iterator t;
 				for (t = tokens.begin() + 1; t != tokens.end(); ++t) {
 					try {
 						if (Hostname::isHostname(t->toString())) {
 							// We do not resolve hostnames here. We commit them as is.
 							const auto& hostname = Hostname::parse(t->toString());
-							if (new_coordinators_hostnames.count(hostname)) {
+							if (new_coordinators_hostnames.contains(hostname)) {
 								fprintf(stderr,
 								        "ERROR: passed redundant coordinators: `%s'\n",
 								        hostname.toString().c_str());
-								return true;
+								co_return true;
 							}
 							new_coordinators_hostnames.insert(hostname);
 							newCoordinatorslist.push_back(hostname.toString());
 						} else {
 							const auto& addr = NetworkAddress::parse(t->toString());
-							if (new_coordinators_addresses.count(addr)) {
+							if (new_coordinators_addresses.contains(addr)) {
 								fprintf(
 								    stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str());
-								return true;
+								co_return true;
 							}
 							new_coordinators_addresses.insert(addr);
 							newCoordinatorslist.push_back(addr.toString());
@@ -128,7 +130,7 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 						if (e.code() == error_code_connection_string_invalid) {
 							fprintf(
 							    stderr, "ERROR: '%s' is not a valid network endpoint address\n", t->toString().c_str());
-							return true;
+							co_return true;
 						}
 						throw;
 					}
@@ -136,20 +138,26 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 				std::string new_coordinators_str = boost::algorithm::join(newCoordinatorslist, ",");
 				tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_coordinators_str);
 			}
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 			// commit should always fail here
-			// If the commit succeeds, the coordinators change and the commit will fail with commit_unknown_result().
+			// If the commit succeeds, the coordinators change and the commit will fail with
+			// commit_unknown_result().
 			ASSERT(false);
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
+			caughtErr = e;
+			hasCaughtErr = true;
+		}
+		if (hasCaughtErr) {
+			Error err(caughtErr);
+			if (caughtErr.code() == error_code_special_keys_api_failure) {
+				std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr);
 				if (errorMsgStr == ManagementAPI::generateErrorMessage(CoordinatorsResult::NOT_ENOUGH_MACHINES) &&
 				    notEnoughMachineResults < 1) {
-					// we could get not_enough_machines if we happen to see the database while the cluster controller is
-					// updating the worker list, so make sure it happens twice before returning a failure
+					// we could get not_enough_machines if we happen to see the database while the cluster
+					// controller is updating the worker list, so make sure it happens twice before returning a
+					// failure
 					notEnoughMachineResults++;
-					wait(delay(1.0));
+					co_await delay(1.0);
 					tr->reset();
 					continue;
 				} else if (errorMsgStr ==
@@ -158,13 +166,13 @@ ACTOR Future<bool> changeCoordinators(Reference<IDatabase> db, std::vector<Strin
 						printf("Coordination state changed\n");
 					else
 						printf("No change (existing configuration satisfies request)\n");
-					return true;
+					co_return true;
 				} else {
 					fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
-					return false;
+					co_return false;
 				}
 			}
-			wait(safeThreadFutureToFuture(tr->onError(err)));
+			co_await safeThreadFutureToFuture(tr->onError(err));
 			++retries;
 		}
 	}
@@ -178,13 +186,13 @@ const KeyRef clusterDescriptionSpecialKey = "\xff\xff/configuration/coordinators
 const KeyRef coordinatorsAutoSpecialKey = "\xff\xff/management/auto_coordinators"_sr;
 const KeyRef coordinatorsProcessSpecialKey = "\xff\xff/configuration/coordinators/processes"_sr;
 
-ACTOR Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() < 2) {
-		wait(printCoordinatorsInfo(db));
-		return true;
+		co_await printCoordinatorsInfo(db);
+		co_return true;
 	} else {
-		bool result = wait(changeCoordinators(db, tokens));
-		return result;
+		bool result = co_await changeCoordinators(db, tokens);
+		co_return result;
 	}
 }
 
diff --git a/fdbcli/DataDistributionCommand.actor.cpp b/fdbcli/DataDistributionCommand.cpp
similarity index 72%
rename from fdbcli/DataDistributionCommand.actor.cpp
rename to fdbcli/DataDistributionCommand.cpp
index 498cd1a580c..acc0457d72e 100644
--- a/fdbcli/DataDistributionCommand.actor.cpp
+++ b/fdbcli/DataDistributionCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * DataDistributionCommand.actor.cpp
+ * DataDistributionCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,7 @@
 
 #include "boost/lexical_cast.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IClientApi.h"
@@ -29,22 +29,22 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
-ACTOR Future<Void> setDDMode(Reference<IDatabase> db, int mode) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> setDDMode(Reference<IDatabase> db, int mode) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			tr->set(fdb_cli::ddModeSpecialKey, boost::lexical_cast<std::string>(mode));
 			if (mode) {
 				// set DDMode to 1 will enable all disabled parts, for instance the SS failure monitors.
 				// hold the returned standalone object's memory
-				state ThreadFuture<RangeResult> resultFuture =
+				ThreadFuture<RangeResult> resultFuture =
 				    tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-				RangeResult res = wait(safeThreadFutureToFuture(resultFuture));
+				RangeResult res = co_await safeThreadFutureToFuture(resultFuture);
 				ASSERT(res.size() <= 1);
 				if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) {
 					// only clear the key if it is currently being used to disable all SS failure data movement
@@ -52,30 +52,32 @@ ACTOR Future<Void> setDDMode(Reference<IDatabase> db, int mode) {
 				}
 				tr->clear(fdb_cli::ddIgnoreRebalanceSpecialKey);
 			}
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return Void();
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return;
 		} catch (Error& e) {
-			TraceEvent("SetDDModeRetrying").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		TraceEvent("SetDDModeRetrying").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Reference<IDatabase> db, uint8_t DDIgnoreOptionMask, bool setMaskedBit) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> setDDIgnoreRebalanceSwitch(Reference<IDatabase> db, uint8_t DDIgnoreOptionMask, bool setMaskedBit) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+		Error err;
 		try {
-			state ThreadFuture<Optional<Value>> resultFuture = tr->get(rebalanceDDIgnoreKey);
-			Optional<Value> v = wait(safeThreadFutureToFuture(resultFuture));
+			ThreadFuture<Optional<Value>> resultFuture = tr->get(rebalanceDDIgnoreKey);
+			Optional<Value> v = co_await safeThreadFutureToFuture(resultFuture);
 			uint8_t oldValue = DDIgnore::NONE; // nothing is disabled
 			if (v.present()) {
-				if (v.get().size() > 0) {
+				if (!v.get().empty()) {
 					oldValue = BinaryReader::fromStringRef<uint8_t>(v.get(), Unversioned());
 				} else {
-					// In old version (<= 7.1), the value is an empty string, which means all DD rebalance functions are
-					// disabled
+					// In old version (<= 7.1), the value is an empty string, which means all DD rebalance functions
+					// are disabled
 					oldValue = DDIgnore::ALL;
 				}
 				// printf("oldValue: %d Mask: %d V:%d\n", oldValue, DDIgnoreOptionMask, v.get().size());
@@ -86,11 +88,12 @@ ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Reference<IDatabase> db, uint8_t D
 			} else {
 				tr->clear(fdb_cli::ddIgnoreRebalanceSpecialKey);
 			}
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return Void();
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -113,30 +116,30 @@ const KeyRef ddIgnoreRebalanceSpecialKey = "\xff\xff/management/data_distributio
 constexpr auto usage =
     "Usage: datadistribution <on|off|disable <ssfailure|rebalance|rebalance_disk|rebalance_read>|enable "
     "<ssfailure|rebalance|rebalance_disk|rebalance_read>>\n";
-ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	state bool result = true;
+Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	bool result = true;
 	if (tokens.size() != 2 && tokens.size() != 3) {
 		printf(usage);
 		result = false;
 	} else {
 		if (tokencmp(tokens[1], "on")) {
-			wait(success(setDDMode(db, 1)));
+			co_await setDDMode(db, 1);
 			printf("Data distribution is turned on.\n");
 		} else if (tokencmp(tokens[1], "off")) {
-			wait(success(setDDMode(db, 0)));
+			co_await setDDMode(db, 0);
 			printf("Data distribution is turned off.\n");
 		} else if (tokencmp(tokens[1], "disable")) {
 			if (tokencmp(tokens[2], "ssfailure")) {
-				wait(success((setHealthyZone(db, "IgnoreSSFailures"_sr, 0))));
+				co_await (setHealthyZone(db, "IgnoreSSFailures"_sr, 0));
 				printf("Data distribution is disabled for storage server failures.\n");
 			} else if (tokencmp(tokens[2], "rebalance")) {
-				wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ));
+				co_await setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ);
 				printf("Data distribution is disabled for rebalance.\n");
 			} else if (tokencmp(tokens[2], "rebalance_disk")) {
-				wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK));
+				co_await setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK);
 				printf("Data distribution is disabled for rebalance_disk.\n");
 			} else if (tokencmp(tokens[2], "rebalance_read")) {
-				wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_READ));
+				co_await setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_READ);
 				printf("Data distribution is disabled for rebalance_read.\n");
 			} else {
 				printf(usage);
@@ -144,16 +147,16 @@ ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::ve
 			}
 		} else if (tokencmp(tokens[1], "enable")) {
 			if (tokencmp(tokens[2], "ssfailure")) {
-				wait(success((clearHealthyZone(db, false, true))));
+				co_await (clearHealthyZone(db, false, true));
 				printf("Data distribution is enabled for storage server failures.\n");
 			} else if (tokencmp(tokens[2], "rebalance")) {
-				wait(setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ));
+				co_await setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ);
 				printf("Data distribution is enabled for rebalance.\n");
 			} else if (tokencmp(tokens[2], "rebalance_disk")) {
-				wait(setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK));
+				co_await setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK);
 				printf("Data distribution is enabled for rebalance_disk.\n");
 			} else if (tokencmp(tokens[2], "rebalance_read")) {
-				wait(setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_READ));
+				co_await setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_READ);
 				printf("Data distribution is enabled for rebalance_read.\n");
 			} else {
 				printf(usage);
@@ -164,7 +167,7 @@ ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::ve
 			result = false;
 		}
 	}
-	return result;
+	co_return result;
 }
 
 // hidden commands, no help text for now
diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.cpp
similarity index 77%
rename from fdbcli/DebugCommands.actor.cpp
rename to fdbcli/DebugCommands.cpp
index 2403a650ab4..571a221427c 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.cpp
@@ -1,5 +1,5 @@
 /*
- * DebugCommands.actor.cpp
+ * DebugCommands.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,12 +20,12 @@
 
 #include <fmt/core.h>
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 namespace fdb_cli {
 
@@ -39,16 +39,18 @@ std::string toHex(StringRef v) {
 }
 
 // Gets a version at which to read from the storage servers
-ACTOR Future<Version> getVersion(Database cx) {
-	loop {
-		state Transaction tr(cx);
+Future<Version> getVersion(Database cx) {
+	while (true) {
+		Transaction tr(cx);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		Error err;
 		try {
-			Version version = wait(tr.getReadVersion());
-			return version;
+			Version version = co_await tr.getReadVersion();
+			co_return version;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -57,78 +59,77 @@ ACTOR Future<Version> getVersion(Database cx) {
 // keyServersPromise will never be set).
 // If dcid is set, only return the storage servers in the given datacenter.
 // Ignore the input dcid if the cluster has not set dcid.
-ACTOR Future<bool> getKeyServers(
+Future<bool> getKeyServers(
     Database cx,
     Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
     KeyRangeRef kr,
     Optional<StringRef> dcid) {
-	state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
+	std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
 
 	// Try getting key server locations from the first commit proxy
-	state Future<ErrorOr<GetKeyServerLocationsReply>> keyServerLocationFuture;
-	state Key begin = kr.begin;
-	state Key end = kr.end;
-	state int limitKeyServers = 100;
+	Future<ErrorOr<GetKeyServerLocationsReply>> keyServerLocationFuture;
+	Key begin = kr.begin;
+	Key end = kr.end;
+	int limitKeyServers = 100;
 
 	while (begin < end) {
-		state Reference<CommitProxyInfo> commitProxyInfo =
-		    wait(cx->getCommitProxiesFuture(UseProvisionalProxies::False));
+		Reference<CommitProxyInfo> commitProxyInfo = co_await cx->getCommitProxiesFuture(UseProvisionalProxies::False);
 		keyServerLocationFuture =
 		    commitProxyInfo->get(0, &CommitProxyInterface::getKeyServersLocations)
 		        .getReplyUnlessFailedFor(
 		            GetKeyServerLocationsRequest({}, begin, end, limitKeyServers, false, latestVersion, Arena()), 2, 0);
 
-		state bool keyServersInsertedForThisIteration = false;
-		choose {
-			when(ErrorOr<GetKeyServerLocationsReply> shards = wait(keyServerLocationFuture)) {
-				// Get the list of shards if one was returned.
-				if (shards.present() && !keyServersInsertedForThisIteration) {
-					std::vector<std::pair<KeyRangeRef, std::vector<StorageServerInterface>>> shardResultList;
-					for (auto& result : shards.get().results) {
-						std::vector<StorageServerInterface> servers;
-						for (auto& server : result.second) {
-							// Filter out storage servers that are not in the given datacenter
-							Optional<Standalone<StringRef>> serverDcId = server.locality.dcId();
-							if (dcid.present() && serverDcId.present() && serverDcId.get() != dcid.get()) {
-								continue;
-							}
-							servers.push_back(server);
+		bool keyServersInsertedForThisIteration = false;
+		auto choice = co_await race(keyServerLocationFuture, cx->onProxiesChanged());
+		if (choice.index() == 0) {
+			ErrorOr<GetKeyServerLocationsReply> shards = std::get<0>(std::move(choice));
+			// Get the list of shards if one was returned.
+			if (shards.present() && !keyServersInsertedForThisIteration) {
+				std::vector<std::pair<KeyRangeRef, std::vector<StorageServerInterface>>> shardResultList;
+				for (auto& result : shards.get().results) {
+					std::vector<StorageServerInterface> servers;
+					for (auto& server : result.second) {
+						// Filter out storage servers that are not in the given datacenter
+						Optional<Standalone<StringRef>> serverDcId = server.locality.dcId();
+						if (dcid.present() && serverDcId.present() && serverDcId.get() != dcid.get()) {
+							continue;
 						}
-						shardResultList.push_back({ result.first, servers });
+						servers.push_back(server);
 					}
-					keyServers.insert(keyServers.end(), shardResultList.begin(), shardResultList.end());
-					keyServersInsertedForThisIteration = true;
-					begin = shards.get().results.back().first.end;
-					break;
+					shardResultList.push_back({ result.first, servers });
 				}
+				keyServers.insert(keyServers.end(), shardResultList.begin(), shardResultList.end());
+				keyServersInsertedForThisIteration = true;
+				begin = shards.get().results.back().first.end;
 			}
-			when(wait(cx->onProxiesChanged())) {}
+		} else if (choice.index() != 1) {
+			UNREACHABLE();
 		}
 
 		if (!keyServersInsertedForThisIteration) // Retry the entire workflow
-			wait(delay(1.0));
+			co_await delay(1.0);
 	}
 
 	keyServersPromise.send(keyServers);
-	return true;
+	co_return true;
 }
 
 // The command is used to get all storage server addresses for a given key.
-ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens) {
+Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens) {
 	if (tokens.size() != 2 && tokens.size() != 3) {
 		fmt::println("getlocation <KEY> [<KEY2>]\n"
 		             "fetch the storage server address for a given key or range.\n"
 		             "Displays the addresses of storage servers, or `not found' if location is not found.");
-		return false;
+		co_return false;
 	}
 
-	state KeyRange kr = KeyRangeRef(tokens[1], tokens.size() == 3 ? tokens[2] : keyAfter(tokens[1]));
+	KeyRange kr = KeyRangeRef(tokens[1], tokens.size() == 3 ? tokens[2] : keyAfter(tokens[1]));
 	// find key range locations without GRV
-	state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise;
-	bool found = wait(getKeyServers(cx, keyServersPromise, kr, Optional<StringRef>()));
+	Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise;
+	bool found = co_await getKeyServers(cx, keyServersPromise, kr, Optional<StringRef>());
 	if (!found) {
 		fmt::println("{} locations not found", printable(kr));
-		return false;
+		co_return false;
 	}
 	std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 	    keyServersPromise.getFuture().get();
@@ -138,33 +139,33 @@ ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> t
 			fmt::println("  {}", server.address().toString());
 		}
 	}
-	return true;
+	co_return true;
 }
 // hidden commands, no help text for now
 CommandFactory getLocationCommandFactory("getlocation");
 
 // The command is used to get values from all storage servers that have the given key.
-ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
+Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
 	if (tokens.size() != 2) {
 		fmt::println("getall <KEY>\n"
 		             "fetch values from all storage servers that have the given key.\n"
 		             "Displays the value and the addresses of storage servers, or `not found' if key is not found.");
-		return false;
+		co_return false;
 	}
 
-	KeyRangeLocationInfo loc = wait(getKeyLocation_internal(
-	    cx, tokens[1], SpanContext(), Optional<UID>(), UseProvisionalProxies::False, Reverse::False, version));
+	KeyRangeLocationInfo loc = co_await getKeyLocation_internal(
+	    cx, tokens[1], SpanContext(), Optional<UID>(), UseProvisionalProxies::False, Reverse::False, version);
 
 	if (loc.locations) {
 		fmt::println("version is {}", version);
 		fmt::println("`{}' is at:", printable(tokens[1]));
-		state Reference<LocationInfo::Locations> locations = loc.locations->locations();
-		state std::vector<Future<GetValueReply>> replies;
+		Reference<LocationInfo::Locations> locations = loc.locations->locations();
+		std::vector<Future<GetValueReply>> replies;
 		for (int i = 0; locations && i < locations->size(); i++) {
 			GetValueRequest req(/*spanContext=*/{}, tokens[1], version, {}, {}, {});
 			replies.push_back(locations->get(i, &StorageServerInterface::getValue).getReply(req));
 		}
-		wait(waitForAll(replies));
+		co_await waitForAll(replies);
 		for (int i = 0; i < replies.size(); i++) {
 			std::string ssi = locations->getInterface(i).address().toString();
 			if (replies[i].isError()) {
@@ -177,7 +178,7 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 	} else {
 		fmt::println("`{}': location not found", printable(tokens[1]));
 	}
-	return true;
+	co_return true;
 }
 // hidden commands, no help text for now
 CommandFactory getallCommandFactory("getall");
@@ -309,45 +310,46 @@ bool checkResults(Version version,
 	return allSame;
 }
 
-ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringRef> dcid, bool checkAll);
+Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringRef> dcid, bool checkAll);
 // Return whether inconsistency is detected in the inputRange
-ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringRef> dcid, bool checkAll) {
-	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
-	state bool consistent = true;
-	loop {
+Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringRef> dcid, bool checkAll) {
+	Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+	bool consistent = true;
+	while (true) {
+		Error err;
 		try {
 			fmt::println("Start checking for range: {}", printable(inputRange));
 			// Get SS interface for each shard of the inputRange
-			state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-			bool foundKeyServers = wait(getKeyServers(cx, keyServerPromise, inputRange, dcid));
+			Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
+			bool foundKeyServers = co_await getKeyServers(cx, keyServerPromise, inputRange, dcid);
 			if (!foundKeyServers) {
 				fmt::println("key server locations for {} not found, retrying in 1s...", printable(inputRange));
-				wait(delay(1.0));
+				co_await delay(1.0);
 				continue;
 			}
-			state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+			std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 			    keyServerPromise.getFuture().get();
 			// We partition the entire input range into shards
 			// and we conduct comparison shard by shard
-			state int i = 0;
+			int i = 0;
 			for (; i < keyServers.size(); i++) { // for each shard
-				state KeyRange rangeToCheck = keyServers[i].first;
+				KeyRange rangeToCheck = keyServers[i].first;
 				rangeToCheck = rangeToCheck & inputRange; // Only check the shard part within the inputRange
 				if (rangeToCheck.empty()) {
 					continue; // Skip the shard if it is outside of the inputRange
 				}
 				const auto& servers = keyServers[i].second;
-				state Key beginKeyToCheck = rangeToCheck.begin;
+				Key beginKeyToCheck = rangeToCheck.begin;
 				fmt::println("Key range to check: {}", printable(rangeToCheck));
 				for (const auto& server : servers) {
 					fmt::println("\t{}", server.address().toString());
 				}
-				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> replies;
-				state bool hasMore = true;
-				state int round = 0;
-				state Version version;
+				std::vector<Future<ErrorOr<GetKeyValuesReply>>> replies;
+				bool hasMore = true;
+				int round = 0;
+				Version version{ 0 };
 				while (hasMore) {
-					wait(store(version, getVersion(cx)));
+					version = co_await getVersion(cx);
 					replies.clear();
 					fmt::println("Round {}: {} - {}", round, toHex(beginKeyToCheck), toHex(rangeToCheck.end));
 					for (const auto& s : keyServers[i].second) { // for each storage server
@@ -360,7 +362,7 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringR
 						req.tags = TagSet();
 						replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 					}
-					wait(waitForAll(replies));
+					co_await waitForAll(replies);
 
 					// Decide comparison scope
 					Key claimEndKey; // used for the next round if hasMore == true
@@ -376,7 +378,7 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringR
 							throw reply.get().error.get();
 						}
 						GetKeyValuesReply current = reply.get();
-						if (current.data.size() == 0) {
+						if (current.data.empty()) {
 							continue; // Ignore if no data has replied
 						}
 						if (claimEndKey.empty() || current.data[current.data.size() - 1].key < claimEndKey) {
@@ -404,12 +406,12 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringR
 						// In the third code path, if hasMore == true and beginKeyToCheck == claimEndKey,
 						// The next round of beginKeyToCheck (aka claimEndKey) will always be beginKeyToCheck of the
 						// current round. To avoid this issue, we spawn a child checkall on a smaller range
-						// (beginKeyToCheck ~ maxEndKey). This smaller range guarantees that the hasMore is always false
-						// and the child checkall will complete and the global progress will move forward. Once the
-						// child checkall is done, we move to the range: maxEndKey ~ rangeToCheck.end
-						state KeyRange spawnedRangeToCheck = Standalone(KeyRangeRef(beginKeyToCheck, maxEndKey));
+						// (beginKeyToCheck ~ maxEndKey). This smaller range guarantees that the hasMore is always
+						// false and the child checkall will complete and the global progress will move forward.
+						// Once the child checkall is done, we move to the range: maxEndKey ~ rangeToCheck.end
+						KeyRange spawnedRangeToCheck = Standalone(KeyRangeRef(beginKeyToCheck, maxEndKey));
 						fmt::println("Spawn new checkall for range {}", printable(spawnedRangeToCheck));
-						bool allSame = wait(doCheckAll(cx, spawnedRangeToCheck, dcid, checkAll));
+						bool allSame = co_await doCheckAll(cx, spawnedRangeToCheck, dcid, checkAll);
 						beginKeyToCheck = spawnedRangeToCheck.end;
 						consistent = consistent && allSame; // !allSame of any subrange results in !consistent
 					} else {
@@ -431,7 +433,7 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringR
 						consistent = consistent && allSame; // !allSame of any subrange results in !consistent
 					}
 					if (!consistent && !checkAll) {
-						return false;
+						co_return false;
 					}
 					round++;
 				}
@@ -439,20 +441,21 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, Optional<StringR
 			break;
 
 		} catch (Error& e) {
-			fmt::print("Error: {}", e.what());
-			wait(onErrorTr.onError(e));
-			fmt::println(", retrying in 1s...");
+			err = e;
 		}
-		wait(delay(1.0));
+		fmt::print("Error: {}", err.what());
+		co_await onErrorTr.onError(err);
+		fmt::println(", retrying in 1s...");
+		co_await delay(1.0);
 	}
-	return consistent;
+	co_return consistent;
 }
 
 // The command is used to check the data inconsistency of the user input range
-ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
-	state bool checkAll = false; // If set, do not return on first error, continue checking all keys
-	state Optional<StringRef> dcid;
-	state KeyRange inputRange;
+Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
+	bool checkAll = false; // If set, do not return on first error, continue checking all keys
+	Optional<StringRef> dcid;
+	KeyRange inputRange;
 	if (tokens.size() == 3) {
 		inputRange = KeyRangeRef(tokens[1], tokens[2]);
 	} else if (tokens.size() == 4 && tokens[3] == "all"_sr) {
@@ -475,16 +478,16 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 		    "`all` is optional. When `all` is appended, the checker does not stop until all subranges have checked.\n"
 		    "Note this is intended to check a small range of keys, not the entire database (consider consistencycheck "
 		    "for that purpose).");
-		return false;
+		co_return false;
 	}
 	if (inputRange.empty()) {
 		fmt::println("Input empty range: {}.\nImmediately exit.", printable(inputRange));
-		return false;
+		co_return false;
 	}
 	// At this point, we have a non-empty inputRange to check
-	bool res = wait(doCheckAll(cx, inputRange, dcid, checkAll));
+	bool res = co_await doCheckAll(cx, inputRange, dcid, checkAll);
 	fmt::println("Checking complete. AllSame: {}", res);
-	return true;
+	co_return true;
 }
 
 CommandFactory checkallCommandFactory("checkall");
diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.cpp
similarity index 69%
rename from fdbcli/ExcludeCommand.actor.cpp
rename to fdbcli/ExcludeCommand.cpp
index 1fde6c61de8..2590a802945 100644
--- a/fdbcli/ExcludeCommand.actor.cpp
+++ b/fdbcli/ExcludeCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ExcludeCommand.actor.cpp
+ * ExcludeCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,33 +18,33 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/Knobs.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/Schemas.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
 // Exclude the given servers and localities
-ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
-                                               std::vector<AddressExclusion> servers,
-                                               std::unordered_set<std::string> localities,
-                                               bool failed,
-                                               bool force) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
+                                         std::vector<AddressExclusion> servers,
+                                         std::unordered_set<std::string> localities,
+                                         bool failed,
+                                         bool force) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		Error err;
 		try {
-			if (force && servers.size())
+			if (force && !servers.empty())
 				tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey,
 				        ValueRef());
 			for (const auto& s : servers) {
@@ -52,7 +52,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 				                  : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString());
 				tr->set(addr, ValueRef());
 			}
-			if (force && localities.size())
+			if (force && !localities.empty())
 				tr->set(failed ? fdb_cli::failedLocalityForceOptionSpecialKey
 				               : fdb_cli::excludedLocalityForceOptionSpecialKey,
 				        ValueRef());
@@ -61,37 +61,38 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 				                  : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l);
 				tr->set(addr, ValueRef());
 			}
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return true;
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return true;
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				// last character is \n
-				auto pos = errorMsgStr.find_last_of("\n", errorMsgStr.size() - 2);
-				auto last_line = errorMsgStr.substr(pos + 1);
-				// customized the error message for fdbcli
-				fprintf(stderr,
-				        "%s\n%s\n",
-				        errorMsgStr.substr(0, pos).c_str(),
-				        last_line.find("free space") != std::string::npos
-				            ? "Type `exclude FORCE <ADDRESS...>' to exclude without checking free space."
-				            : "Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.");
-				return false;
-			}
-			TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err);
-			wait(safeThreadFutureToFuture(tr->onError(err)));
+			err = e;
+		}
+		if (err.code() == error_code_special_keys_api_failure) {
+			std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr);
+			// last character is \n
+			auto pos = errorMsgStr.find_last_of("\n", errorMsgStr.size() - 2);
+			auto last_line = errorMsgStr.substr(pos + 1);
+			// customized the error message for fdbcli
+			fprintf(stderr,
+			        "%s\n%s\n",
+			        errorMsgStr.substr(0, pos).c_str(),
+			        last_line.find("free space") != std::string::npos
+			            ? "Type `exclude FORCE <ADDRESS...>' to exclude without checking free space."
+			            : "Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.");
+			co_return false;
 		}
+		TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+AsyncResult<std::vector<std::string>> getExcludedServers(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			state RangeResult r = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 			std::vector<std::string> exclusions;
@@ -99,22 +100,24 @@ ACTOR Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> d
 				auto addr = i.key.removePrefix(fdb_cli::excludedServersSpecialKeyRange.begin).toString();
 				exclusions.push_back(addr);
 			}
-			return exclusions;
+			co_return exclusions;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "GetExcludedServersError").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		TraceEvent(SevWarn, "GetExcludedServersError").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
 // Get the list of excluded localities by reading the keys.
-ACTOR Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+AsyncResult<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			state RangeResult r = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 			std::vector<std::string> excludedLocalities;
@@ -122,20 +125,22 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase
 				auto locality = i.key.removePrefix(fdb_cli::excludedLocalitySpecialKeyRange.begin).toString();
 				excludedLocalities.push_back(locality);
 			}
-			return excludedLocalities;
+			co_return excludedLocalities;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<std::vector<std::string>> getFailedServers(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+AsyncResult<std::vector<std::string>> getFailedServers(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			state RangeResult r = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 			std::vector<std::string> exclusions;
@@ -143,21 +148,23 @@ ACTOR Future<std::vector<std::string>> getFailedServers(Reference<IDatabase> db)
 				auto addr = i.key.removePrefix(fdb_cli::failedServersSpecialKeyRange.begin).toString();
 				exclusions.push_back(addr);
 			}
-			return exclusions;
+			co_return exclusions;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
 // Get the list of failed localities by reading the keys.
-ACTOR Future<std::vector<std::string>> getFailedLocalities(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+AsyncResult<std::vector<std::string>> getFailedLocalities(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			state RangeResult r = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 			std::vector<std::string> excludedLocalities;
@@ -165,40 +172,43 @@ ACTOR Future<std::vector<std::string>> getFailedLocalities(Reference<IDatabase>
 				auto locality = i.key.removePrefix(fdb_cli::failedLocalitySpecialKeyRange.begin).toString();
 				excludedLocalities.push_back(locality);
 			}
-			return excludedLocalities;
+			co_return excludedLocalities;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<std::set<NetworkAddress>> getInProgressExclusion(Reference<ITransaction> tr) {
+Future<std::set<NetworkAddress>> getInProgressExclusion(Reference<ITransaction> tr) {
 	ThreadFuture<RangeResult> resultFuture =
 	    tr->getRange(fdb_cli::exclusionInProgressSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-	RangeResult result = wait(safeThreadFutureToFuture(resultFuture));
+	RangeResult result = co_await safeThreadFutureToFuture(resultFuture);
 	ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY);
 	std::set<NetworkAddress> inProgressExclusion;
 	for (const auto& addr : result) {
 		inProgressExclusion.insert(
 		    NetworkAddress::parse(addr.key.removePrefix(fdb_cli::exclusionInProgressSpecialKeyRange.begin).toString()));
 	}
-	return inProgressExclusion;
+	co_return inProgressExclusion;
 }
 
-ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatabase> db,
-                                                                std::set<AddressExclusion> exclusions,
-                                                                bool waitForAllExcluded) {
-	state std::set<NetworkAddress> inProgressExclusion;
-	state Reference<ITransaction> tr = db->createTransaction();
+Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatabase> db,
+                                                          std::set<AddressExclusion> exclusions,
+                                                          bool waitForAllExcluded) {
+	std::set<NetworkAddress> inProgressExclusion;
+	Reference<ITransaction> tr = db->createTransaction();
 	tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	loop {
+	while (true) {
 		inProgressExclusion.clear();
+		Error err;
+		bool hasErr = false;
 		try {
-			std::set<NetworkAddress> result = wait(getInProgressExclusion(tr));
+			std::set<NetworkAddress> result = co_await getInProgressExclusion(tr);
 			if (result.empty())
-				return inProgressExclusion;
+				co_return inProgressExclusion;
 			inProgressExclusion = result;
 
 			// Check if all of the specified exclusions are done.
@@ -219,38 +229,44 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatab
 
 			if (allExcluded) {
 				inProgressExclusion.clear();
-				return inProgressExclusion;
+				co_return inProgressExclusion;
 			}
 
 			if (!waitForAllExcluded)
 				break;
 
-			wait(delayJittered(1.0)); // SOMEDAY: watches!
+			co_await delayJittered(1.0); // SOMEDAY: watches!
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "CheckForExcludingServersError").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
+			hasErr = true;
+		}
+		if (hasErr) {
+			TraceEvent(SevWarn, "CheckForExcludingServersError").error(err);
+			co_await safeThreadFutureToFuture(tr->onError(err));
 		}
 	}
-	return inProgressExclusion;
+	co_return inProgressExclusion;
 }
 
-ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<AddressExclusion> exclusions) {
+Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<AddressExclusion> exclusions) {
 
-	state bool foundCoordinator = false;
-	state std::vector<NetworkAddress> coordinatorList;
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+	bool foundCoordinator = false;
+	std::vector<NetworkAddress> coordinatorList;
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
 			// Hold the reference to the standalone's memory
-			state ThreadFuture<Optional<Value>> coordinatorsF = tr->get(fdb_cli::coordinatorsProcessSpecialKey);
-			Optional<Value> coordinators = wait(safeThreadFutureToFuture(coordinatorsF));
+			ThreadFuture<Optional<Value>> coordinatorsF = tr->get(fdb_cli::coordinatorsProcessSpecialKey);
+			Optional<Value> coordinators = co_await safeThreadFutureToFuture(coordinatorsF);
 			ASSERT(coordinators.present());
 			coordinatorList = NetworkAddress::parseList(coordinators.get().toString());
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "CheckForCoordinatorsError").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		TraceEvent(SevWarn, "CheckForCoordinatorsError").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 
 	for (const auto& c : coordinatorList) {
@@ -263,7 +279,6 @@ ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<Addres
 	if (foundCoordinator)
 		printf("Type `help coordinators' for information on how to change the\n"
 		       "cluster's coordination servers before removing them.\n");
-	return Void();
 }
 
 } // namespace
@@ -284,18 +299,19 @@ const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/
 const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr,
                                                      "\xff\xff/management/in_progress_exclusion0"_sr);
 
-ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> warn) {
+Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> _warn) {
+	auto warn = std::move(_warn);
 	if (tokens.size() <= 1) {
-		state std::vector<std::string> excludedAddresses = wait(getExcludedServers(db));
-		state std::vector<std::string> excludedLocalities = wait(getExcludedLocalities(db));
-		state std::vector<std::string> failedAddresses = wait(getFailedServers(db));
-		state std::vector<std::string> failedLocalities = wait(getFailedLocalities(db));
+		std::vector<std::string> excludedAddresses = co_await getExcludedServers(db);
+		std::vector<std::string> excludedLocalities = co_await getExcludedLocalities(db);
+		std::vector<std::string> failedAddresses = co_await getFailedServers(db);
+		std::vector<std::string> failedLocalities = co_await getFailedLocalities(db);
 
-		if (!excludedAddresses.size() && !excludedLocalities.size() && !failedAddresses.size() &&
-		    !failedLocalities.size()) {
+		if (excludedAddresses.empty() && excludedLocalities.empty() && failedAddresses.empty() &&
+		    failedLocalities.empty()) {
 			printf("There are currently no servers or localities excluded from the database.\n"
 			       "To learn how to exclude a server, type `help exclude'.\n");
-			return true;
+			co_return true;
 		}
 
 		printf("There are currently %zu servers or localities being excluded from the database:\n",
@@ -305,7 +321,7 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 		for (const auto& e : excludedLocalities)
 			printf("  %s\n", e.c_str());
 
-		if (excludedAddresses.size() || excludedLocalities.size()) {
+		if (!excludedAddresses.empty() || !excludedLocalities.empty()) {
 			printf("To find out whether it is safe to remove one or more of these\n"
 			       "servers from the cluster, type `exclude <addresses>'.\n"
 			       "To return one of these servers to the cluster, type `include <addresses>'.\n");
@@ -320,33 +336,33 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 		for (const auto& f : failedLocalities)
 			printf("  %s\n", f.c_str());
 
-		if (failedAddresses.size() || failedLocalities.size()) {
+		if (!failedAddresses.empty() || !failedLocalities.empty()) {
 			printf("To return one of these servers to the cluster, type `include failed <addresses>'.\n");
 		}
 
 		printf("\n");
 
 		Reference<ITransaction> tr = db->createTransaction();
-		std::set<NetworkAddress> inProgressExclusion = wait(getInProgressExclusion(tr));
+		std::set<NetworkAddress> inProgressExclusion = co_await getInProgressExclusion(tr);
 		printf("There are currently %zu processes for which exclusion is in progress:\n", inProgressExclusion.size());
 		for (const auto& addr : inProgressExclusion) {
 			printf("%s\n", addr.toString().c_str());
 		}
 
-		return true;
+		co_return true;
 	} else {
-		state std::set<AddressExclusion> exclusionSet;
-		state std::vector<AddressExclusion> exclusionAddresses;
-		state std::unordered_set<std::string> exclusionLocalities;
-		state std::vector<std::string> noMatchLocalities;
-		state bool force = false;
-		state bool waitForAllExcluded = true;
-		state bool markFailed = false;
-		state std::vector<ProcessData> workers;
-		state std::map<std::string, StorageServerInterface> server_interfaces;
-		state Future<bool> future_workers = fdb_cli::getWorkers(db, &workers);
-		state Future<Void> future_server_interfaces = fdb_cli::getStorageServerInterfaces(db, &server_interfaces);
-		wait(success(future_workers) && success(future_server_interfaces));
+		std::set<AddressExclusion> exclusionSet;
+		std::vector<AddressExclusion> exclusionAddresses;
+		std::unordered_set<std::string> exclusionLocalities;
+		std::vector<std::string> noMatchLocalities;
+		bool force = false;
+		bool waitForAllExcluded = true;
+		bool markFailed = false;
+		std::vector<ProcessData> workers;
+		std::map<std::string, StorageServerInterface> server_interfaces;
+		Future<bool> future_workers = fdb_cli::getWorkers(db, &workers);
+		Future<Void> future_server_interfaces = fdb_cli::getStorageServerInterfaces(db, &server_interfaces);
+		co_await (success(future_workers) && success(future_server_interfaces));
 
 		for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
 			if (*t == "FORCE"_sr) {
@@ -380,7 +396,7 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 					        t->toString().c_str());
 					if (t->toString().find(":tls") != std::string::npos)
 						printf("        Do not include the `:tls' suffix when naming a process\n");
-					return true;
+					co_return true;
 				}
 				exclusionSet.insert(a);
 				exclusionAddresses.push_back(a);
@@ -391,12 +407,12 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 		// here the provided locality and/or address will not be excluded.
 		if (exclusionAddresses.empty() && exclusionLocalities.empty()) {
 			fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality must be provided\n");
-			return false;
+			co_return false;
 		}
 
-		bool res = wait(excludeServersAndLocalities(db, exclusionAddresses, exclusionLocalities, markFailed, force));
+		bool res = co_await excludeServersAndLocalities(db, exclusionAddresses, exclusionLocalities, markFailed, force);
 		if (!res)
-			return false;
+			co_return false;
 
 		if (waitForAllExcluded) {
 			printf("Waiting for state to be removed from all excluded servers. This may take a while.\n");
@@ -406,10 +422,10 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 		if (warn.isValid())
 			warn.cancel();
 
-		state std::set<NetworkAddress> notExcludedServers =
-		    wait(checkForExcludingServers(db, exclusionSet, waitForAllExcluded));
+		std::set<NetworkAddress> notExcludedServers =
+		    co_await checkForExcludingServers(db, exclusionSet, waitForAllExcluded);
 		std::map<IPAddress, std::set<uint16_t>> workerPorts;
-		for (auto addr : workers)
+		for (const auto& addr : workers)
 			workerPorts[addr.address.ip].insert(addr.address.port);
 
 		// Print a list of all excluded addresses that don't have a corresponding worker
@@ -418,7 +434,7 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 			auto worker = workerPorts.find(addr.ip);
 			if (worker == workerPorts.end())
 				absentExclusions.insert(addr);
-			else if (addr.port > 0 && worker->second.count(addr.port) == 0)
+			else if (addr.port > 0 && !worker->second.contains(addr.port))
 				absentExclusions.insert(addr);
 		}
 
@@ -470,8 +486,8 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 			    locality.c_str());
 		}
 
-		wait(checkForCoordinators(db, exclusionSet));
-		return true;
+		co_await checkForCoordinators(db, exclusionSet);
+		co_return true;
 	}
 }
 
diff --git a/fdbcli/ExpensiveDataCheckCommand.actor.cpp b/fdbcli/ExpensiveDataCheckCommand.cpp
similarity index 84%
rename from fdbcli/ExpensiveDataCheckCommand.actor.cpp
rename to fdbcli/ExpensiveDataCheckCommand.cpp
index a9b02c144b4..0b56e63b687 100644
--- a/fdbcli/ExpensiveDataCheckCommand.actor.cpp
+++ b/fdbcli/ExpensiveDataCheckCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ExpensiveDataCheckCommand.actor.cpp
+ * ExpensiveDataCheckCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,7 @@
 
 #include "boost/algorithm/string.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -29,39 +29,38 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
 // The command is used to send a data check request to the specified process
 // The check request is accomplished by rebooting the process
 
-ACTOR Future<bool> expensiveDataCheckCommandActor(
+Future<bool> expensiveDataCheckCommandActor(
     Reference<IDatabase> db,
     Reference<ITransaction> tr,
     std::vector<StringRef> tokens,
     std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
-	state bool result = true;
-	state std::string addressesStr;
+	bool result = true;
+	std::string addressesStr;
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
 		address_interface->clear();
-		wait(getWorkerInterfaces(tr, address_interface, true));
+		co_await getWorkerInterfaces(tr, address_interface, true);
 	}
 	if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
-		if (address_interface->size() == 0) {
+		if (address_interface->empty()) {
 			printf("\nNo addresses can be checked.\n");
 		} else if (address_interface->size() == 1) {
 			printf("\nThe following address can be checked:\n");
 		} else {
 			printf("\nThe following %zu addresses can be checked:\n", address_interface->size());
 		}
-		for (auto it : *address_interface) {
+		for (const auto& it : *address_interface) {
 			printf("%s\n", printable(it.first).c_str());
 		}
 		printf("\n");
 	} else if (tokencmp(tokens[1], "all")) {
-		if (address_interface->size() == 0) {
+		if (address_interface->empty()) {
 			fprintf(stderr,
 			        "ERROR: no processes to check. You must run the `expensive_data_check’ "
 			        "command before running `expensive_data_check all’.\n");
@@ -72,7 +71,7 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 			}
 			addressesStr = boost::algorithm::join(addressesVec, ",");
 			// make sure we only call the interface once to send requests in parallel
-			int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)));
+			int64_t checkRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0));
 			if (!checkRequestsSent) {
 				result = false;
 				fprintf(stderr,
@@ -83,7 +82,7 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 			}
 		}
 	} else {
-		state int i;
+		int i{ 0 };
 		for (i = 1; i < tokens.size(); i++) {
 			if (!address_interface->count(tokens[i])) {
 				fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
@@ -98,7 +97,7 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 				addressesVec.push_back(tokens[i].toString());
 			}
 			addressesStr = boost::algorithm::join(addressesVec, ",");
-			int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)));
+			int64_t checkRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0));
 			if (!checkRequestsSent) {
 				result = false;
 				fprintf(stderr,
@@ -110,7 +109,7 @@ ACTOR Future<bool> expensiveDataCheckCommandActor(
 			}
 		}
 	}
-	return result;
+	co_return result;
 }
 // hidden commands, no help text for now
 CommandFactory expensiveDataCheckFactory("expensive_data_check");
diff --git a/fdbcli/FileConfigureCommand.actor.cpp b/fdbcli/FileConfigureCommand.cpp
similarity index 87%
rename from fdbcli/FileConfigureCommand.actor.cpp
rename to fdbcli/FileConfigureCommand.cpp
index a521e70aa92..b4535e53686 100644
--- a/fdbcli/FileConfigureCommand.actor.cpp
+++ b/fdbcli/FileConfigureCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * FileConfigureCommand.actor.cpp
+ * FileConfigureCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,35 +19,33 @@
  */
 
 #include "fdbcli/FlowLineNoise.h"
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/Schemas.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
-ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
-                                             std::string filePath,
-                                             bool isNewDatabase,
-                                             bool force) {
-	state ConfigurationResult result;
+Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
+                                       std::string const& filePath,
+                                       bool isNewDatabase,
+                                       bool force) {
+	ConfigurationResult result;
 	std::string contents(readFileBytes(filePath, 100000));
 	json_spirit::mValue config;
 	if (!json_spirit::read_string(contents, config)) {
 		fprintf(stderr, "ERROR: Invalid JSON\n");
-		return false;
+		co_return false;
 	}
 	if (config.type() != json_spirit::obj_type) {
 		fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n");
-		return false;
+		co_return false;
 	}
 	StatusObject configJSON = config.get_obj();
 
@@ -59,17 +57,17 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 	std::string errorStr;
 	if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) {
 		printf("%s", errorStr.c_str());
-		return false;
+		co_return false;
 	}
 
-	state std::string configString;
+	std::string configString;
 
 	try {
 		configString += DatabaseConfiguration::configureStringFromJSON(configJSON);
 	} catch (Error& e) {
 		fmt::print("ERROR: {}", e.what());
 		printUsage("fileconfigure"_sr);
-		return false;
+		co_return false;
 	}
 
 	if (isNewDatabase) {
@@ -82,8 +80,10 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 	// This setting is now managed automatically by the backup system.
 	if (configString.find(" backup_worker_enabled:=") != std::string::npos) {
 		result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED;
+	} else if (configString.find(" range_backup_worker_enabled:=") != std::string::npos) {
+		result = ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED;
 	} else {
-		ConfigurationResult r = wait(ManagementAPI::changeConfig(db, configString, force));
+		ConfigurationResult r = co_await ManagementAPI::changeConfig(db, configString, force);
 		result = r;
 	}
 	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
@@ -170,11 +170,17 @@ ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 		        "backup system\n");
 		ret = false;
 		break;
+	case ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED:
+		fprintf(stderr,
+		        "ERROR: range_backup_worker_enabled configuration is restricted in fdbcli and managed "
+		        "automatically by the backup system\n");
+		ret = false;
+		break;
 	default:
 		ASSERT(false);
 		ret = false;
 	};
-	return ret;
+	co_return ret;
 }
 
 CommandFactory fileconfigureFactory(
diff --git a/fdbcli/FlowLineNoise.actor.cpp b/fdbcli/FlowLineNoise.cpp
similarity index 93%
rename from fdbcli/FlowLineNoise.actor.cpp
rename to fdbcli/FlowLineNoise.cpp
index 91757e39267..05f52f57b4b 100644
--- a/fdbcli/FlowLineNoise.actor.cpp
+++ b/fdbcli/FlowLineNoise.cpp
@@ -1,5 +1,5 @@
 /*
- * FlowLineNoise.actor.cpp
+ * FlowLineNoise.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -40,7 +40,6 @@
 #else
 #define HAVE_LINENOISE 0
 #endif
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct LineNoiseReader final : IThreadPoolReceiver {
 	void init() override {}
@@ -143,8 +142,8 @@ Future<Optional<std::string>> LineNoise::read(std::string const& prompt) {
 	return f;
 }
 
-ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
-	state boost::asio::signal_set signals(*ios, SIGINT);
+Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
+	boost::asio::signal_set signals(*ios, SIGINT);
 	Promise<Void> result;
 	signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
 		if (error) {
@@ -154,12 +153,11 @@ ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
 		}
 	});
 
-	wait(result.getFuture());
-	return Void();
+	co_await result.getFuture();
 }
 
 Future<Void> LineNoise::onKeyboardInterrupt() {
-	boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
+	auto* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
 	if (!ios)
 		return Never();
 	return waitKeyboardInterrupt(ios);
diff --git a/fdbcli/ForceRecoveryWithDataLossCommand.actor.cpp b/fdbcli/ForceRecoveryWithDataLossCommand.cpp
similarity index 82%
rename from fdbcli/ForceRecoveryWithDataLossCommand.actor.cpp
rename to fdbcli/ForceRecoveryWithDataLossCommand.cpp
index 7c5e35c152d..37e1d4a1ee6 100644
--- a/fdbcli/ForceRecoveryWithDataLossCommand.actor.cpp
+++ b/fdbcli/ForceRecoveryWithDataLossCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ForceRecoveryWithDataLossCommand.actor.cpp
+ * ForceRecoveryWithDataLossCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,24 +18,22 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/IClientApi.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
-ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> const& tokens) {
 	if (tokens.size() != 2) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
-	wait(safeThreadFutureToFuture(db->forceRecoveryWithDataLoss(tokens[1])));
-	return true;
+	co_await safeThreadFutureToFuture(db->forceRecoveryWithDataLoss(tokens[1]));
+	co_return true;
 }
 
 CommandFactory forceRecoveryWithDataLossFactory(
diff --git a/fdbcli/GetAuditStatusCommand.actor.cpp b/fdbcli/GetAuditStatusCommand.cpp
similarity index 66%
rename from fdbcli/GetAuditStatusCommand.actor.cpp
rename to fdbcli/GetAuditStatusCommand.cpp
index e66c610c3e4..0a9cf1d5983 100644
--- a/fdbcli/GetAuditStatusCommand.actor.cpp
+++ b/fdbcli/GetAuditStatusCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * GetAuditStatusCommand.actor.cpp
+ * GetAuditStatusCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,29 +20,28 @@
 
 #include <fmt/core.h>
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 #include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/IClientApi.h"
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
-ACTOR Future<Void> getAuditProgressByRange(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) {
-	state KeyRange rangeToRead = auditRange;
-	state Key rangeToReadBegin = auditRange.begin;
-	state int retryCount = 0;
-	state int64_t finishCount = 0;
+Future<Void> getAuditProgressByRange(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) {
+	KeyRange rangeToRead = auditRange;
+	Key rangeToReadBegin = auditRange.begin;
+	int retryCount = 0;
+	int64_t finishCount = 0;
 	while (rangeToReadBegin < auditRange.end) {
-		loop {
+		while (true) {
+			Error err;
 			try {
 				rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end);
-				state std::vector<AuditStorageState> auditStates =
-				    wait(getAuditStateByRange(cx, auditType, auditId, rangeToRead));
+				std::vector<AuditStorageState> auditStates =
+				    co_await getAuditStateByRange(cx, auditType, auditId, rangeToRead);
 				for (int i = 0; i < auditStates.size(); i++) {
 					AuditPhase phase = auditStates[i].getPhase();
 					if (phase == AuditPhase::Invalid) {
@@ -58,99 +57,104 @@ ACTOR Future<Void> getAuditProgressByRange(Database cx, AuditType auditType, UID
 				rangeToReadBegin = auditStates.back().range.end;
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				if (retryCount > 30) {
-					fmt::println("Incomplete check");
-					return Void();
-				}
-				wait(delay(0.5));
-				retryCount++;
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			}
+			if (retryCount > 30) {
+				fmt::println("Incomplete check");
+				co_return;
 			}
+			co_await delay(0.5);
+			retryCount++;
 		}
 	}
 	fmt::println("Finished range count: {}", finishCount);
-	return Void();
 }
 
-ACTOR Future<std::vector<StorageServerInterface>> getStorageServers(Database cx) {
-	state Transaction tr(cx);
-	loop {
+AsyncResult<std::vector<StorageServerInterface>> getStorageServers(Database cx) {
+	Transaction tr(cx);
+	while (true) {
 		tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		Error err;
 		try {
-			RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 			std::vector<StorageServerInterface> servers;
 			servers.reserve(serverList.size());
 			for (int i = 0; i < serverList.size(); i++)
 				servers.push_back(decodeServerListValue(serverList[i].value));
-			return servers;
+			co_return servers;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<AuditPhase> getAuditProgressByServer(Database cx,
-                                                  AuditType auditType,
-                                                  UID auditId,
-                                                  KeyRange auditRange,
-                                                  UID serverId) {
-	state KeyRange rangeToRead = auditRange;
-	state Key rangeToReadBegin = auditRange.begin;
-	state int retryCount = 0;
+Future<AuditPhase> getAuditProgressByServer(Database cx,
+                                            AuditType auditType,
+                                            UID auditId,
+                                            KeyRange auditRange,
+                                            UID serverId) {
+	KeyRange rangeToRead = auditRange;
+	Key rangeToReadBegin = auditRange.begin;
+	int retryCount = 0;
 	while (rangeToReadBegin < auditRange.end) {
-		loop {
+		while (true) {
+			Error err;
 			try {
 				rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end);
-				state std::vector<AuditStorageState> auditStates =
-				    wait(getAuditStateByServer(cx, auditType, auditId, serverId, rangeToRead));
+				std::vector<AuditStorageState> auditStates =
+				    co_await getAuditStateByServer(cx, auditType, auditId, serverId, rangeToRead);
 				for (int i = 0; i < auditStates.size(); i++) {
 					AuditPhase phase = auditStates[i].getPhase();
 					if (phase == AuditPhase::Invalid) {
-						return AuditPhase::Running;
+						co_return AuditPhase::Running;
 					} else if (phase == AuditPhase::Error) {
-						return AuditPhase::Error;
+						co_return AuditPhase::Error;
 					}
 				}
 				rangeToReadBegin = auditStates.back().range.end;
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				if (retryCount > 30) {
-					return AuditPhase::Invalid;
-				}
-				wait(delay(0.5));
-				retryCount++;
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			}
+			if (retryCount > 30) {
+				co_return AuditPhase::Invalid;
 			}
+			co_await delay(0.5);
+			retryCount++;
 		}
 	}
-	return AuditPhase::Complete;
+	co_return AuditPhase::Complete;
 }
 
-ACTOR Future<Void> getAuditProgress(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) {
+Future<Void> getAuditProgress(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) {
 	if (auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica ||
 	    auditType == AuditType::ValidateLocationMetadata || auditType == AuditType::ValidateRestore) {
-		wait(getAuditProgressByRange(cx, auditType, auditId, auditRange));
+		co_await getAuditProgressByRange(cx, auditType, auditId, auditRange);
 	} else if (auditType == AuditType::ValidateStorageServerShard) {
-		state std::vector<Future<Void>> fs;
-		state std::unordered_map<UID, bool> res;
-		state std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
-		state int i = 0;
-		state int numCompleteServers = 0;
-		state int numOngoingServers = 0;
-		state int numErrorServers = 0;
-		state int numTSSes = 0;
+		std::vector<Future<Void>> fs;
+		std::unordered_map<UID, bool> res;
+		std::vector<StorageServerInterface> interfs = co_await getStorageServers(cx);
+		int i = 0;
+		int numCompleteServers = 0;
+		int numOngoingServers = 0;
+		int numErrorServers = 0;
+		int numTSSes = 0;
 		for (; i < interfs.size(); i++) {
 			if (interfs[i].isTss()) {
 				numTSSes++;
 				continue; // SSShard audit does not test TSS
 			}
-			AuditPhase serverPhase = wait(getAuditProgressByServer(cx, auditType, auditId, allKeys, interfs[i].id()));
+			AuditPhase serverPhase =
+			    co_await getAuditProgressByServer(cx, auditType, auditId, allKeys, interfs[i].id());
 			if (serverPhase == AuditPhase::Running) {
 				numOngoingServers++;
 			} else if (serverPhase == AuditPhase::Complete) {
@@ -168,13 +172,12 @@ ACTOR Future<Void> getAuditProgress(Database cx, AuditType auditType, UID auditI
 	} else {
 		fmt::println("AuditType not implemented");
 	}
-	return Void();
 }
 
-ACTOR Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef> tokens) {
+Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef> tokens) {
 	if (tokens.size() < 2 || tokens.size() > 5) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
 	AuditType type = AuditType::Invalid;
@@ -190,26 +193,26 @@ ACTOR Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef
 		type = AuditType::ValidateRestore;
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
 	if (tokencmp(tokens[2], "id")) {
 		if (tokens.size() != 4) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
 		const UID id = UID::fromString(tokens[3].toString());
-		AuditStorageState res = wait(getAuditState(cx, type, id));
+		AuditStorageState res = co_await getAuditState(cx, type, id);
 		fmt::println("Audit result is:\n{}", res.toString());
 	} else if (tokencmp(tokens[2], "progress")) {
 		if (tokens.size() != 4) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
 		const UID id = UID::fromString(tokens[3].toString());
-		state AuditStorageState res = wait(getAuditState(cx, type, id));
+		AuditStorageState res = co_await getAuditState(cx, type, id);
 		if (res.getPhase() == AuditPhase::Running) {
-			wait(getAuditProgress(cx, res.getType(), res.id, res.range));
+			co_await getAuditProgress(cx, res.getType(), res.id, res.range);
 		} else {
 			fmt::println("Already complete");
 		}
@@ -218,7 +221,7 @@ ACTOR Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef
 		if (tokens.size() == 4) {
 			count = std::stoi(tokens[3].toString());
 		}
-		std::vector<AuditStorageState> res = wait(getAuditStates(cx, type, /*newFirst=*/true, count));
+		std::vector<AuditStorageState> res = co_await getAuditStates(cx, type, /*newFirst=*/true, count);
 		for (const auto& it : res) {
 			fmt::println("Audit result is:\n{}", it.toString());
 		}
@@ -226,22 +229,22 @@ ACTOR Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef
 		AuditPhase phase = stringToAuditPhase(tokens[3].toString());
 		if (phase == AuditPhase::Invalid) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
 		int count = CLIENT_KNOBS->TOO_MANY;
 		if (tokens.size() == 5) {
 			count = std::stoi(tokens[4].toString());
 		}
-		std::vector<AuditStorageState> res = wait(getAuditStates(cx, type, /*newFirst=*/true, count, phase));
+		std::vector<AuditStorageState> res = co_await getAuditStates(cx, type, /*newFirst=*/true, count, phase);
 		for (const auto& it : res) {
 			fmt::println("Audit result is:\n{}", it.toString());
 		}
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
-	return true;
+	co_return true;
 }
 
 CommandFactory getAuditStatusFactory(
diff --git a/fdbcli/HotRangeCommand.actor.cpp b/fdbcli/HotRangeCommand.cpp
similarity index 83%
rename from fdbcli/HotRangeCommand.actor.cpp
rename to fdbcli/HotRangeCommand.cpp
index b3d6788fffc..2913ced41e8 100644
--- a/fdbcli/HotRangeCommand.actor.cpp
+++ b/fdbcli/HotRangeCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * HotRangeCommand.actor.cpp
+ * HotRangeCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,7 @@
 
 #include <boost/lexical_cast.hpp>
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -31,8 +31,6 @@
 #include "flow/FastRef.h"
 #include "flow/NetworkAddress.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
 
 ReadHotSubRangeRequest::SplitType parseSplitType(const std::string& typeStr) {
@@ -53,43 +51,43 @@ ReadHotSubRangeRequest::SplitType parseSplitType(const std::string& typeStr) {
 
 namespace fdb_cli {
 
-ACTOR Future<bool> hotRangeCommandActor(Database localdb,
-                                        Reference<IDatabase> db,
-                                        std::vector<StringRef> tokens,
-                                        std::map<std::string, StorageServerInterface>* storage_interface) {
+Future<bool> hotRangeCommandActor(Database localdb,
+                                  Reference<IDatabase> db,
+                                  std::vector<StringRef> const& tokens,
+                                  std::map<std::string, StorageServerInterface>* const& storage_interface) {
 
 	if (tokens.size() == 1) {
 		// initialize storage interfaces
 		storage_interface->clear();
-		wait(getStorageServerInterfaces(db, storage_interface));
+		co_await getStorageServerInterfaces(db, storage_interface);
 		fmt::print("\nThe following {} storage servers can be queried:\n", storage_interface->size());
 		for (const auto& [addr, _] : *storage_interface) {
 			fmt::print("{}\n", addr);
 		}
 		fmt::print("\n");
 	} else if (tokens.size() == 6) {
-		if (storage_interface->size() == 0) {
+		if (storage_interface->empty()) {
 			fprintf(stderr, "ERROR: no storage processes to query. You must run the `hotrange’ command first.\n");
-			return false;
+			co_return false;
 		}
-		state Key address = tokens[1];
+		Key address = tokens[1];
 		// At present we only support one process(IP:Port) at a time
 		if (!storage_interface->count(address.toString())) {
 			fprintf(stderr, "ERROR: storage process `%s' not recognized.\n", printable(address).c_str());
-			return false;
+			co_return false;
 		}
-		state int splitCount;
+		int splitCount{ 0 };
 		try {
 			splitCount = boost::lexical_cast<int>(tokens[5].toString());
 		} catch (...) {
 			fmt::print("Error: splitCount value: '{}', cannot be parsed to an Integer\n", tokens[5].toString());
-			return false;
+			co_return false;
 		}
 		ReadHotSubRangeRequest::SplitType splitType = parseSplitType(tokens[2].toString());
 		KeyRangeRef range(tokens[3], tokens[4]);
 		// TODO: refactor this to support multiversion fdbcli in the future
-		Standalone<VectorRef<ReadHotRangeWithMetrics>> metrics =
-		    wait(localdb->getHotRangeMetrics((*storage_interface)[address.toString()], range, splitType, splitCount));
+		Standalone<VectorRef<ReadHotRangeWithMetrics>> metrics = co_await localdb->getHotRangeMetrics(
+		    (*storage_interface)[address.toString()], range, splitType, splitCount);
 		// next parse the result and form a json array for each object
 		json_spirit::mArray resultArray;
 		for (const auto& metric : metrics) {
@@ -107,10 +105,10 @@ ACTOR Future<bool> hotRangeCommandActor(Database localdb,
 		fmt::print("\n{}\n", result);
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
-	return true;
+	co_return true;
 }
 
 CommandFactory hotRangeFactory(
diff --git a/fdbcli/IdempotencyIdsCommand.actor.cpp b/fdbcli/IdempotencyIdsCommand.cpp
similarity index 80%
rename from fdbcli/IdempotencyIdsCommand.actor.cpp
rename to fdbcli/IdempotencyIdsCommand.cpp
index 56457027235..1984b769cbc 100644
--- a/fdbcli/IdempotencyIdsCommand.actor.cpp
+++ b/fdbcli/IdempotencyIdsCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * IdempotencyIdsCommand.actor.cpp
+ * IdempotencyIdsCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,12 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
-#include "fdbclient/IdempotencyId.actor.h"
+#include "fdbcli/fdbcli.h"
+#include "fdbclient/IdempotencyId.h"
 #include "fdbclient/JsonBuilder.h"
 #include "fdbclient/json_spirit/json_spirit_reader_template.h"
-#include "flow/actorcompiler.h" // This must be the last include
-
 namespace {
 
 Optional<double> parseAgeValue(StringRef token) {
@@ -38,36 +36,36 @@ Optional<double> parseAgeValue(StringRef token) {
 
 namespace fdb_cli {
 
-ACTOR Future<bool> idempotencyIdsCommandActor(Database db, std::vector<StringRef> tokens) {
+Future<bool> idempotencyIdsCommandActor(Database db, std::vector<StringRef> const& tokens) {
 	if (tokens.size() < 2 || tokens.size() > 3) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	} else {
 		auto const action = tokens[1];
 		if (action == "status"_sr) {
 			if (tokens.size() != 2) {
 				printUsage(tokens[0]);
-				return false;
+				co_return false;
 			}
-			JsonBuilderObject status = wait(getIdmpKeyStatus(db));
+			JsonBuilderObject status = co_await getIdmpKeyStatus(db);
 			fmt::print("{}\n", status.getJson());
-			return true;
+			co_return true;
 		} else if (action == "clear"_sr) {
 			if (tokens.size() != 3) {
 				printUsage(tokens[0]);
-				return false;
+				co_return false;
 			}
 			auto const age = parseAgeValue(tokens[2]);
 			if (!age.present()) {
 				printUsage(tokens[0]);
-				return false;
+				co_return false;
 			}
-			wait(cleanIdempotencyIds(db, age.get()));
+			co_await cleanIdempotencyIds(db, age.get());
 			fmt::print("Successfully cleared idempotency IDs.\n");
-			return true;
+			co_return true;
 		} else {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
 	}
 }
diff --git a/fdbcli/IncludeCommand.actor.cpp b/fdbcli/IncludeCommand.cpp
similarity index 72%
rename from fdbcli/IncludeCommand.actor.cpp
rename to fdbcli/IncludeCommand.cpp
index 40a42e435e6..2691915bc45 100644
--- a/fdbcli/IncludeCommand.actor.cpp
+++ b/fdbcli/IncludeCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * IncludeCommand.actor.cpp
+ * IncludeCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -27,22 +27,21 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
 // Remove the given localities from the exclusion list.
 // include localities by clearing the keys.
-ACTOR Future<Void> includeLocalities(Reference<IDatabase> db,
-                                     std::vector<std::string> localities,
-                                     bool failed,
-                                     bool includeAll) {
-	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> includeLocalities(Reference<IDatabase> db,
+                               std::vector<std::string> localities,
+                               bool failed,
+                               bool includeAll) {
+	std::string versionKey = deterministicRandom()->randomUniqueID().toString();
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
-
 			if (includeAll) {
 				if (failed) {
 					tr->clear(fdb_cli::failedLocalitySpecialKeyRange);
@@ -56,20 +55,22 @@ ACTOR Future<Void> includeLocalities(Reference<IDatabase> db,
 					tr->clear(locality);
 				}
 			}
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return Void();
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return;
 		} catch (Error& e) {
-			TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		TraceEvent("IncludeLocalitiesError").errorUnsuppressed(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<Void> includeServers(Reference<IDatabase> db, std::vector<AddressExclusion> servers, bool failed) {
-	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> includeServers(Reference<IDatabase> db, std::vector<AddressExclusion> servers, bool failed) {
+	std::string versionKey = deterministicRandom()->randomUniqueID().toString();
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			for (auto& s : servers) {
 				// include all, just clear the whole key range
@@ -95,21 +96,22 @@ ACTOR Future<Void> includeServers(Reference<IDatabase> db, std::vector<AddressEx
 						tr->clear(KeyRangeRef(addr.withSuffix(":"_sr), addr.withSuffix(";"_sr)));
 				}
 			}
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return Void();
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return;
 		} catch (Error& e) {
-			TraceEvent("IncludeServersError").errorUnsuppressed(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		TraceEvent("IncludeServersError").errorUnsuppressed(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
 // Includes the servers that could be IP addresses or localities back to the cluster.
-ACTOR Future<bool> include(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	state std::vector<AddressExclusion> addresses;
-	state std::vector<std::string> localities;
-	state bool failed = false;
-	state bool all = false;
+Future<bool> include(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	std::vector<AddressExclusion> addresses;
+	std::vector<std::string> localities;
+	bool failed = false;
+	bool all = false;
 	for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
 		if (*t == "all"_sr) {
 			all = true;
@@ -126,7 +128,7 @@ ACTOR Future<bool> include(Reference<IDatabase> db, std::vector<StringRef> token
 				        t->toString().c_str());
 				if (t->toString().find(":tls") != std::string::npos)
 					printf("        Do not include the `:tls' suffix when naming a process\n");
-				return false;
+				co_return false;
 			}
 			addresses.push_back(a);
 		}
@@ -134,31 +136,31 @@ ACTOR Future<bool> include(Reference<IDatabase> db, std::vector<StringRef> token
 	if (all) {
 		std::vector<AddressExclusion> includeAll;
 		includeAll.push_back(AddressExclusion());
-		wait(includeServers(db, includeAll, failed));
-		wait(includeLocalities(db, localities, failed, all));
+		co_await includeServers(db, includeAll, failed);
+		co_await includeLocalities(db, localities, failed, all);
 	} else {
 		if (!addresses.empty()) {
-			wait(includeServers(db, addresses, failed));
+			co_await includeServers(db, addresses, failed);
 		}
 		if (!localities.empty()) {
 			// include the servers that belong to given localities.
-			wait(includeLocalities(db, localities, failed, all));
+			co_await includeLocalities(db, localities, failed, all);
 		}
 	}
-	return true;
+	co_return true;
 };
 
 } // namespace
 
 namespace fdb_cli {
 
-ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() < 2) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	} else {
-		bool result = wait(include(db, tokens));
-		return result;
+		bool result = co_await include(db, tokens);
+		co_return result;
 	}
 }
 
diff --git a/fdbcli/KillCommand.actor.cpp b/fdbcli/KillCommand.cpp
similarity index 80%
rename from fdbcli/KillCommand.actor.cpp
rename to fdbcli/KillCommand.cpp
index c520cd15c34..baa12ec976c 100644
--- a/fdbcli/KillCommand.actor.cpp
+++ b/fdbcli/KillCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * KillCommand.actor.cpp
+ * KillCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,7 @@
 
 #include "boost/algorithm/string.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -30,36 +30,35 @@
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/CodeProbe.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
-ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
-                                    Reference<ITransaction> tr,
-                                    std::vector<StringRef> tokens,
-                                    std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
-	ASSERT(tokens.size() >= 1);
-	state bool result = true;
-	state std::string addressesStr;
+Future<bool> killCommandActor(Reference<IDatabase> db,
+                              Reference<ITransaction> tr,
+                              std::vector<StringRef> tokens,
+                              std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
+	ASSERT(!tokens.empty());
+	bool result = true;
+	std::string addressesStr;
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
 		address_interface->clear();
-		wait(getWorkerInterfaces(tr, address_interface, true));
+		co_await getWorkerInterfaces(tr, address_interface, true);
 	}
 	if (tokens.size() == 1 || tokencmp(tokens[1], "list")) {
-		if (address_interface->size() == 0) {
+		if (address_interface->empty()) {
 			printf("\nNo addresses can be killed.\n");
 		} else if (address_interface->size() == 1) {
 			printf("\nThe following address can be killed:\n");
 		} else {
 			printf("\nThe following %zu addresses can be killed:\n", address_interface->size());
 		}
-		for (auto it : *address_interface) {
+		for (const auto& it : *address_interface) {
 			printf("%s\n", printable(it.first).c_str());
 		}
 		printf("\n");
 	} else if (tokencmp(tokens[1], "all")) {
-		if (address_interface->size() == 0) {
+		if (address_interface->empty()) {
 			result = false;
 			fprintf(stderr,
 			        "ERROR: no processes to kill. You must run the `kill’ command before "
@@ -71,7 +70,7 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 			}
 			addressesStr = boost::algorithm::join(addressesVec, ",");
 			// make sure we only call the interface once to send requests in parallel
-			int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)));
+			int64_t killRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0));
 			if (!killRequestsSent) {
 				result = false;
 				fprintf(stderr,
@@ -82,7 +81,7 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 			}
 		}
 	} else {
-		state int i;
+		int i{ 0 };
 		for (i = 1; i < tokens.size(); i++) {
 			if (!address_interface->count(tokens[i])) {
 				fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
@@ -97,7 +96,7 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 				addressesVec.push_back(tokens[i].toString());
 			}
 			addressesStr = boost::algorithm::join(addressesVec, ",");
-			int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)));
+			int64_t killRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0));
 			if (!killRequestsSent) {
 				result = false;
 				fprintf(stderr,
@@ -106,12 +105,12 @@ ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
 				        addressesStr.c_str());
 			} else {
 				// delay in case the network queue is not flush before the client exits
-				wait(delay(3.0));
+				co_await delay(3.0);
 				printf("Attempted to kill %zu processes\n", tokens.size() - 1);
 			}
 		}
 	}
-	return result;
+	co_return result;
 }
 
 void killGenerator(const char* text,
diff --git a/fdbcli/LocationMetadataCommand.actor.cpp b/fdbcli/LocationMetadataCommand.cpp
similarity index 63%
rename from fdbcli/LocationMetadataCommand.actor.cpp
rename to fdbcli/LocationMetadataCommand.cpp
index c1923242b82..94dfae46a1f 100644
--- a/fdbcli/LocationMetadataCommand.actor.cpp
+++ b/fdbcli/LocationMetadataCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * LocationMetadataCommand.actor.cpp
+ * LocationMetadataCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,23 +18,21 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 #include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/IClientApi.h"
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
-ACTOR Future<std::string> describeServers(Reference<ReadYourWritesTransaction> tr, std::vector<UID> ids) {
+Future<std::string> describeServers(Reference<ReadYourWritesTransaction> tr, std::vector<UID> ids) {
 	std::vector<Future<Optional<Value>>> serverListEntries;
 	for (const UID& id : ids) {
 		serverListEntries.push_back(tr->get(serverListKeyFor(id)));
 	}
-	std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+	std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 	std::string res;
 	for (auto& v : serverListValues) {
 		StorageServerInterface ssi = decodeServerListValue(v.get());
@@ -44,50 +42,50 @@ ACTOR Future<std::string> describeServers(Reference<ReadYourWritesTransaction> t
 		res += fmt::format("ServerID: {}, Addr: {}", ssi.uniqueID.toString(), ssi.stableAddress().toString());
 	}
 
-	return res;
+	co_return res;
 }
 
-ACTOR Future<Void> printKeyServersEntry(Reference<ReadYourWritesTransaction> tr,
-                                        RangeResultRef UIDtoTagMap,
-                                        Value entry,
-                                        KeyRangeRef range) {
-	state std::vector<UID> src;
-	state std::vector<UID> dest;
-	state UID srcId;
-	state UID destId;
+Future<Void> printKeyServersEntry(Reference<ReadYourWritesTransaction> tr,
+                                  RangeResultRef UIDtoTagMap,
+                                  Value entry,
+                                  KeyRangeRef range) {
+	std::vector<UID> src;
+	std::vector<UID> dest;
+	UID srcId;
+	UID destId;
 	decodeKeyServersValue(UIDtoTagMap, entry, src, dest, srcId, destId);
-	state std::string srcDesc = wait(describeServers(tr, src));
-	std::string destDesc = wait(describeServers(tr, dest));
+	std::string srcDesc = co_await describeServers(tr, src);
+	std::string destDesc = co_await describeServers(tr, dest);
 	printf("Range: %s, ShardID: %s, Src Servers: %s, Dest Servers: %s\n",
 	       Traceable<KeyRangeRef>::toString(range).c_str(),
 	       srcId.toString().c_str(),
 	       srcDesc.c_str(),
 	       destDesc.c_str());
-	return Void();
 }
 
-ACTOR Future<Void> printRandomShards(Database cx, int n, bool physicalShard) {
-	state Key begin = allKeys.begin;
-	state int numShards = 0;
+Future<Void> printRandomShards(Database cx, int n, bool physicalShard) {
+	Key begin = allKeys.begin;
+	int numShards = 0;
 
 	while (begin < allKeys.end && numShards < n) {
 		// RYW to optimize re-reading the same key ranges
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state KeyRangeRef currentKeys(begin, allKeys.end);
-				state RangeResult shards =
-				    wait(krmGetRanges(tr, keyServersPrefix, currentKeys, n, CLIENT_KNOBS->TOO_MANY));
+				KeyRangeRef currentKeys(begin, allKeys.end);
+				RangeResult shards =
+				    co_await krmGetRanges(tr, keyServersPrefix, currentKeys, n, CLIENT_KNOBS->TOO_MANY);
 
-				state int i = 0;
+				int i = 0;
 				for (; i < shards.size() - 1 && numShards < n; ++i) {
 					KeyRangeRef currentRange(shards[i].key, shards[i + 1].key);
 					std::vector<UID> src;
@@ -96,7 +94,7 @@ ACTOR Future<Void> printRandomShards(Database cx, int n, bool physicalShard) {
 					UID destId;
 					decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId);
 					if (physicalShard == (srcId != anonymousShardId)) {
-						wait(printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange));
+						co_await printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange);
 						++numShards;
 					}
 				}
@@ -104,37 +102,37 @@ ACTOR Future<Void> printRandomShards(Database cx, int n, bool physicalShard) {
 				begin = shards.back().key;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
 	printf("Found %d %s shards\n", numShards, physicalShard ? "Physical" : "Non-physical");
-
-	return Void();
 }
 
-ACTOR Future<Void> printPhysicalShardCount(Database cx) {
-	state Key begin = allKeys.begin;
-	state int numShards = 0;
-	state int numPhysicalShards = 0;
+Future<Void> printPhysicalShardCount(Database cx) {
+	Key begin = allKeys.begin;
+	int numShards = 0;
+	int numPhysicalShards = 0;
 
 	while (begin < allKeys.end) {
 		// RYW to optimize re-reading the same key ranges
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
 				KeyRangeRef currentKeys(begin, allKeys.end);
-				RangeResult shards = wait(
-				    krmGetRanges(tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY));
+				RangeResult shards = co_await krmGetRanges(
+				    tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY);
 
 				for (int i = 0; i < shards.size() - 1; ++i) {
 					std::vector<UID> src;
@@ -151,31 +149,31 @@ ACTOR Future<Void> printPhysicalShardCount(Database cx) {
 				numShards += shards.size() - 1;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
 	printf("Total number of shards: %d, number of physical shards: %d\n", numShards, numPhysicalShards);
-
-	return Void();
 }
 
-ACTOR Future<Void> printServerShards(Database cx, UID serverId) {
-	state Key begin = allKeys.begin;
+Future<Void> printServerShards(Database cx, UID serverId) {
+	Key begin = allKeys.begin;
 
 	while (begin < allKeys.end) {
 		// RYW to optimize re-reading the same key ranges
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				RangeResult serverShards =
-				    wait(krmGetRanges(tr, serverKeysPrefixFor(serverId), KeyRangeRef(begin, allKeys.end)));
+				    co_await krmGetRanges(tr, serverKeysPrefixFor(serverId), KeyRangeRef(begin, allKeys.end));
 
 				for (int i = 0; i < serverShards.size() - 1; ++i) {
 					KeyRangeRef currentRange(serverShards[i].key, serverShards[i + 1].key);
@@ -194,92 +192,91 @@ ACTOR Future<Void> printServerShards(Database cx, UID serverId) {
 				begin = serverShards.back().key;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> resolveRange(Database cx, KeyRange range) {
-	state Key begin = range.begin;
+Future<Void> resolveRange(Database cx, KeyRange range) {
+	Key begin = range.begin;
 
 	while (begin < range.end) {
 		// RYW to optimize re-reading the same key ranges
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state KeyRangeRef currentKeys(begin, range.end);
-				state RangeResult shards = wait(
-				    krmGetRanges(tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY));
+				KeyRangeRef currentKeys(begin, range.end);
+				RangeResult shards = co_await krmGetRanges(
+				    tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY);
 
-				state int i = 0;
+				int i = 0;
 				for (; i < shards.size() - 1; ++i) {
-					state KeyRangeRef currentRange(shards[i].key, shards[i + 1].key);
-					wait(printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange));
+					KeyRangeRef currentRange(shards[i].key, shards[i + 1].key);
+					co_await printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange);
 				}
 
 				begin = shards.back().key;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
-
-	return Void();
 }
 
 } // namespace
 
 namespace fdb_cli {
 
-ACTOR Future<bool> locationMetadataCommandActor(Database cx, std::vector<StringRef> tokens) {
+Future<bool> locationMetadataCommandActor(Database cx, std::vector<StringRef> tokens) {
 	if (tokens.size() < 2 || tokens.size() > 4) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
 	if (tokens.size() == 2) {
 		if (!tokencmp(tokens[1], "physicalshards")) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
-		wait(printPhysicalShardCount(cx));
+		co_await printPhysicalShardCount(cx);
 	} else if (tokencmp(tokens[1], "resolve")) {
 		if (tokens.size() == 3) {
-			wait(resolveRange(cx, singleKeyRange(tokens[2])));
+			co_await resolveRange(cx, singleKeyRange(tokens[2]));
 		} else {
-			wait(resolveRange(cx, KeyRangeRef(tokens[2], tokens[3])));
+			co_await resolveRange(cx, KeyRangeRef(tokens[2], tokens[3]));
 		}
 	} else if (tokencmp(tokens[1], "servershards")) {
 		if (tokens.size() != 3) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
-		wait(printServerShards(cx, UID::fromString(tokens[2].toString())));
+		co_await printServerShards(cx, UID::fromString(tokens[2].toString()));
 	} else if (tokencmp(tokens[1], "listshards")) {
 		if (tokens.size() == 4 && !tokencmp(tokens[3], "physical")) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		}
 		const bool physical = tokens.size() == 4;
-		wait(printRandomShards(cx, std::stoi(tokens[2].toString()), physical));
+		co_await printRandomShards(cx, std::stoi(tokens[2].toString()), physical);
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
-	return true;
+	co_return true;
 }
 
 CommandFactory locationMetadataFactory(
diff --git a/fdbcli/LockCommand.actor.cpp b/fdbcli/LockCommand.cpp
similarity index 59%
rename from fdbcli/LockCommand.actor.cpp
rename to fdbcli/LockCommand.cpp
index 1ed959ae183..5570e7be763 100644
--- a/fdbcli/LockCommand.actor.cpp
+++ b/fdbcli/LockCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * LockCommand.actor.cpp
+ * LockCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -28,30 +28,29 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
 
-ACTOR Future<bool> lockDatabase(Reference<IDatabase> db, UID id) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<bool> lockDatabase(Reference<IDatabase> db, UID id) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			tr->set(fdb_cli::lockSpecialKey, id.toString());
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 			printf("Database locked.\n");
-			return true;
+			co_return true;
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_database_locked)
-				throw e;
-			else if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				fprintf(stderr, "%s\n", errorMsgStr.c_str());
-				return false;
-			}
-			wait(safeThreadFutureToFuture(tr->onError(err)));
+			err = e;
+		}
+		if (err.code() == error_code_database_locked)
+			throw err;
+		if (err.code() == error_code_special_keys_api_failure) {
+			std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr);
+			fprintf(stderr, "%s\n", errorMsgStr.c_str());
+			co_return false;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -61,47 +60,48 @@ namespace fdb_cli {
 
 const KeyRef lockSpecialKey = "\xff\xff/management/db_locked"_sr;
 
-ACTOR Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> const& tokens) {
 	if (tokens.size() != 1) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	} else {
-		state UID lockUID = deterministicRandom()->randomUniqueID();
+		UID lockUID = deterministicRandom()->randomUniqueID();
 		printf("Locking database with lockUID: %s\n", lockUID.toString().c_str());
-		bool result = wait((lockDatabase(db, lockUID)));
-		return result;
+		bool result = co_await lockDatabase(db, lockUID);
+		co_return result;
 	}
 }
 
-ACTOR Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
-			state ThreadFuture<Optional<Value>> valF = tr->get(fdb_cli::lockSpecialKey);
-			Optional<Value> val = wait(safeThreadFutureToFuture(valF));
+			ThreadFuture<Optional<Value>> valF = tr->get(fdb_cli::lockSpecialKey);
+			Optional<Value> val = co_await safeThreadFutureToFuture(valF);
 
 			if (!val.present())
-				return true;
+				co_return true;
 
 			if (val.present() && UID::fromString(val.get().toString()) != uid) {
 				printf("Unable to unlock database. Make sure to unlock with the correct lock UID.\n");
-				return false;
+				co_return false;
 			}
 
 			tr->clear(fdb_cli::lockSpecialKey);
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 			printf("Database unlocked.\n");
-			return true;
+			co_return true;
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				fprintf(stderr, "%s\n", errorMsgStr.c_str());
-				return false;
-			}
-			wait(safeThreadFutureToFuture(tr->onError(err)));
+			err = e;
+		}
+		if (err.code() == error_code_special_keys_api_failure) {
+			std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr);
+			fprintf(stderr, "%s\n", errorMsgStr.c_str());
+			co_return false;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
diff --git a/fdbcli/MaintenanceCommand.actor.cpp b/fdbcli/MaintenanceCommand.cpp
similarity index 76%
rename from fdbcli/MaintenanceCommand.actor.cpp
rename to fdbcli/MaintenanceCommand.cpp
index ca4da82a803..bcd7cf99e93 100644
--- a/fdbcli/MaintenanceCommand.actor.cpp
+++ b/fdbcli/MaintenanceCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * MaintenanceCommand.actor.cpp
+ * MaintenanceCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,7 +23,7 @@
 #include "boost/lexical_cast.hpp"
 #include "fmt/format.h"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IClientApi.h"
@@ -32,36 +32,37 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
 // print zoneId under maintenance, only one is possible at the same time
-ACTOR Future<Void> printHealthyZone(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> printHealthyZone(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			// We need to keep the future as the returned standalone is not guaranteed to manage its memory when
 			// using an external client, but the ThreadFuture holds a reference to the memory
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			RangeResult res = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult res = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(res.size() <= 1);
 			if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) {
 				printf("Data distribution has been disabled for all storage server failures in this cluster and thus "
 				       "maintenance mode is not active.\n");
-			} else if (!res.size() || boost::lexical_cast<double>(res[0].value.toString()) <= 0) {
+			} else if (res.empty() || boost::lexical_cast<double>(res[0].value.toString()) <= 0) {
 				printf("No ongoing maintenance.\n");
 			} else {
 				std::string zoneId = res[0].key.removePrefix(fdb_cli::maintenanceSpecialKeyRange.begin).toString();
 				int64_t seconds = static_cast<int64_t>(boost::lexical_cast<double>(res[0].value.toString()));
 				fmt::print("Maintenance for zone {0} will continue for {1} seconds.\n", zoneId, seconds);
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -75,16 +76,17 @@ const KeyRangeRef maintenanceSpecialKeyRange =
 const KeyRef ignoreSSFailureSpecialKey = "\xff\xff/management/maintenance/IgnoreSSFailures"_sr;
 
 // add a zone to maintenance and specify the maintenance duration
-ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning) {
-	state Reference<ITransaction> tr = db->createTransaction();
+Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning) {
+	Reference<ITransaction> tr = db->createTransaction();
 	TraceEvent("SetHealthyZone").detail("Zone", zoneId).detail("DurationSeconds", seconds);
-	loop {
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			// hold the returned standalone object's memory
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			RangeResult res = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult res = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(res.size() <= 1);
 			if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) {
 				if (printWarning) {
@@ -92,29 +94,31 @@ ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, dou
 					        "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage "
 					        "server failures. Use 'datadistribution on' to reenable data distribution.\n");
 				}
-				return false;
+				co_return false;
 			}
 			tr->set(fdb_cli::maintenanceSpecialKeyRange.begin.withSuffix(zoneId),
 			        boost::lexical_cast<std::string>(seconds));
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return true;
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return true;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
 // clear ongoing maintenance, let clearSSFailureZoneString = true to enable data distribution for storage
-ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db, bool printWarning, bool clearSSFailureZoneString) {
-	state Reference<ITransaction> tr = db->createTransaction();
+Future<bool> clearHealthyZone(Reference<IDatabase> db, bool printWarning, bool clearSSFailureZoneString) {
+	Reference<ITransaction> tr = db->createTransaction();
 	TraceEvent("ClearHealthyZone").detail("ClearSSFailureZoneString", clearSSFailureZoneString);
-	loop {
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			// hold the returned standalone object's memory
-			state ThreadFuture<RangeResult> resultFuture =
+			ThreadFuture<RangeResult> resultFuture =
 			    tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			RangeResult res = wait(safeThreadFutureToFuture(resultFuture));
+			RangeResult res = co_await safeThreadFutureToFuture(resultFuture);
 			ASSERT(res.size() <= 1);
 			if (!clearSSFailureZoneString && res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) {
 				if (printWarning) {
@@ -122,24 +126,25 @@ ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db, bool printWarning,
 					        "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage "
 					        "server failures. Use 'datadistribution on' to reenable data distribution.\n");
 				}
-				return false;
+				co_return false;
 			}
 
 			tr->clear(fdb_cli::maintenanceSpecialKeyRange);
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return true;
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return true;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	state bool result = true;
+Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+	bool result = true;
 	if (tokens.size() == 1) {
-		wait(printHealthyZone(db));
+		co_await printHealthyZone(db);
 	} else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) {
-		bool clearResult = wait(clearHealthyZone(db, true));
+		bool clearResult = co_await clearHealthyZone(db, true);
 		result = clearResult;
 	} else if (tokens.size() == 4 && tokencmp(tokens[1], "on")) {
 		double seconds;
@@ -149,14 +154,14 @@ ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<
 			printUsage(tokens[0]);
 			result = false;
 		} else {
-			bool setResult = wait(setHealthyZone(db, tokens[2], seconds, true));
+			bool setResult = co_await setHealthyZone(db, tokens[2], seconds, true);
 			result = setResult;
 		}
 	} else {
 		printUsage(tokens[0]);
 		result = false;
 	}
-	return result;
+	co_return result;
 }
 
 CommandFactory maintenanceFactory(
diff --git a/fdbcli/ProfileCommand.actor.cpp b/fdbcli/ProfileCommand.cpp
similarity index 85%
rename from fdbcli/ProfileCommand.actor.cpp
rename to fdbcli/ProfileCommand.cpp
index a997e7f1a39..a08392701ad 100644
--- a/fdbcli/ProfileCommand.actor.cpp
+++ b/fdbcli/ProfileCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ProfileCommand.actor.cpp
+ * ProfileCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,9 +20,9 @@
 
 #include "boost/lexical_cast.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
-#include "fdbclient/GlobalConfig.actor.h"
+#include "fdbclient/GlobalConfig.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/Knobs.h"
@@ -31,28 +31,24 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
-ACTOR Future<bool> profileCommandActor(Database db,
-                                       Reference<ITransaction> tr,
-                                       std::vector<StringRef> tokens,
-                                       bool intrans) {
-	state bool result = true;
+Future<bool> profileCommandActor(Database db, Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans) {
+	bool result = true;
 	if (tokens.size() == 1) {
 		printUsage(tokens[0]);
 		result = false;
 	} else if (tokencmp(tokens[1], "client")) {
 		if (tokens.size() == 2) {
 			fprintf(stderr, "ERROR: Usage: profile client <get|set>\n");
-			return false;
+			co_return false;
 		}
-		wait(db->globalConfig->onInitialized());
+		co_await db->globalConfig->onInitialized();
 		if (tokencmp(tokens[2], "get")) {
 			if (tokens.size() != 3) {
 				fprintf(stderr, "ERROR: Additional arguments to `get` are not supported.\n");
-				return false;
+				co_return false;
 			}
 			std::string sampleRateStr = "default";
 			std::string sizeLimitStr = "default";
@@ -71,7 +67,7 @@ ACTOR Future<bool> profileCommandActor(Database db,
 		} else if (tokencmp(tokens[2], "set")) {
 			if (tokens.size() != 5) {
 				fprintf(stderr, "ERROR: Usage: profile client set <RATE|default> <SIZE|default>\n");
-				return false;
+				co_return false;
 			}
 			double sampleRate;
 			if (tokencmp(tokens[3], "default")) {
@@ -81,7 +77,7 @@ ACTOR Future<bool> profileCommandActor(Database db,
 				sampleRate = std::strtod((const char*)tokens[3].begin(), &end);
 				if (!std::isspace(*end)) {
 					fprintf(stderr, "ERROR: %s failed to parse.\n", printable(tokens[3]).c_str());
-					return false;
+					co_return false;
 				}
 			}
 			int64_t sizeLimit;
@@ -93,7 +89,7 @@ ACTOR Future<bool> profileCommandActor(Database db,
 					sizeLimit = parsed.get();
 				} else {
 					fprintf(stderr, "ERROR: `%s` failed to parse.\n", printable(tokens[4]).c_str());
-					return false;
+					co_return false;
 				}
 			}
 
@@ -103,7 +99,7 @@ ACTOR Future<bool> profileCommandActor(Database db,
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
 			if (!intrans) {
-				wait(safeThreadFutureToFuture(tr->commit()));
+				co_await safeThreadFutureToFuture(tr->commit());
 			}
 		} else {
 			fprintf(stderr, "ERROR: Unknown action: %s\n", printable(tokens[2]).c_str());
@@ -112,12 +108,12 @@ ACTOR Future<bool> profileCommandActor(Database db,
 	} else if (tokencmp(tokens[1], "list")) {
 		if (tokens.size() != 2) {
 			fprintf(stderr, "ERROR: Usage: profile list\n");
-			return false;
+			co_return false;
 		}
 		// Hold the reference to the standalone's memory
-		state ThreadFuture<RangeResult> kvsFuture = tr->getRange(
+		ThreadFuture<RangeResult> kvsFuture = tr->getRange(
 		    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY);
-		RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture));
+		RangeResult kvs = co_await safeThreadFutureToFuture(kvsFuture);
 		ASSERT(!kvs.more);
 		for (const auto& pair : kvs) {
 			auto ip_port = (pair.key.endsWith(":tls"_sr) ? pair.key.removeSuffix(":tls"_sr) : pair.key)
@@ -128,7 +124,7 @@ ACTOR Future<bool> profileCommandActor(Database db,
 		fprintf(stderr, "ERROR: Unknown type: %s\n", printable(tokens[1]).c_str());
 		result = false;
 	}
-	return result;
+	co_return result;
 }
 
 CommandFactory profileFactory("profile",
diff --git a/fdbcli/RangeConfigCommand.actor.cpp b/fdbcli/RangeConfigCommand.cpp
similarity index 78%
rename from fdbcli/RangeConfigCommand.actor.cpp
rename to fdbcli/RangeConfigCommand.cpp
index bf83ddecfa8..f8b4e0e3d93 100644
--- a/fdbcli/RangeConfigCommand.actor.cpp
+++ b/fdbcli/RangeConfigCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * RangeConfigCommand.actor.cpp
+ * RangeConfigCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,16 +23,14 @@
 #include <boost/lexical_cast/bad_lexical_cast.hpp>
 #include <list>
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/DataDistributionConfig.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/DataDistributionConfig.h"
 
 namespace fdb_cli {
 
-ACTOR Future<bool> rangeConfigCommandActor(Database cx, std::vector<StringRef> tokens) {
-	state std::function<bool(std::string)> fail = [&](std::string msg) {
+Future<bool> rangeConfigCommandActor(Database cx, std::vector<StringRef> tokens) {
+	std::function<bool(std::string)> fail = [&](std::string msg) {
 		if (!msg.empty()) {
 			fmt::print(stderr, "ERROR: {}\n", msg);
 		}
@@ -40,60 +38,60 @@ ACTOR Future<bool> rangeConfigCommandActor(Database cx, std::vector<StringRef> t
 		return false;
 	};
 
-	state std::list<StringRef> args(tokens.begin() + 1, tokens.end());
-	state std::function<StringRef()> nextArg = [&]() {
+	std::list<StringRef> args(tokens.begin() + 1, tokens.end());
+	std::function<StringRef()> nextArg = [&]() {
 		ASSERT(!args.empty());
 		auto s = args.front();
 		args.pop_front();
 		return s;
 	};
-	state std::function<int()> nextArgInt = [&]() {
+	std::function<int()> nextArgInt = [&]() {
 		if (args.empty()) {
 			throw boost::bad_lexical_cast();
 		}
 		return boost::lexical_cast<int>(nextArg().toString());
 	};
 
-	if (args.size() < 1) {
-		return fail("No subcommand given.");
+	if (args.empty()) {
+		co_return fail("No subcommand given.");
 	}
 
-	state StringRef cmd = nextArg();
+	StringRef cmd = nextArg();
 
 	if (cmd == "show"_sr) {
-		state bool includeDefault = false;
+		bool includeDefault = false;
 
 		while (!args.empty()) {
 			auto arg = nextArg();
 			if (arg == "includeDefault"_sr) {
 				includeDefault = true;
 			} else {
-				return fail(fmt::format("Unknown argument: '{}'", arg.printable()));
+				co_return fail(fmt::format("Unknown argument: '{}'", arg.printable()));
 			}
 		}
 
 		Reference<DDConfiguration::RangeConfigMapSnapshot> config =
-		    wait(DDConfiguration().userRangeConfig().getSnapshot(
-		        SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end));
+		    co_await DDConfiguration().userRangeConfig().getSnapshot(
+		        SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end);
 		fmt::print(
 		    "{}\n",
 		    json_spirit::write_string(DDConfiguration::toJSON(*config, includeDefault), json_spirit::pretty_print));
 
 	} else if (cmd == "update"_sr || cmd == "set"_sr) {
 		if (args.size() < 3) {
-			return fail("Begin, end, and at least one configuration option are required.");
+			co_return fail("Begin, end, and at least one configuration option are required.");
 		}
 
-		state KeyRef begin = nextArg();
-		state KeyRef end = nextArg();
+		KeyRef begin = nextArg();
+		KeyRef end = nextArg();
 		if (end <= begin) {
-			return fail("Range end must be > range begin.");
+			co_return fail("Range end must be > range begin.");
 		}
 
-		state DDRangeConfig rangeConfig;
+		DDRangeConfig rangeConfig;
 
 		while (!args.empty()) {
-			state StringRef option = nextArg();
+			StringRef option = nextArg();
 
 			try {
 				if (option == "replication"_sr) {
@@ -103,21 +101,21 @@ ACTOR Future<bool> rangeConfigCommandActor(Database cx, std::vector<StringRef> t
 				} else if (option == "default"_sr) {
 					rangeConfig = DDRangeConfig();
 				} else {
-					return fail(fmt::format("Unknown range option: '{}'", option.printable()));
+					co_return fail(fmt::format("Unknown range option: '{}'", option.printable()));
 				}
 			} catch (...) {
-				return fail(
+				co_return fail(
 				    fmt::format("Required argument for range option '{}' missing or invalid.", option.toString()));
 			}
 
-			wait(DDConfiguration().userRangeConfig().updateRange(
-			    SystemDBWriteLockedNow(cx.getReference()), begin, end, rangeConfig, cmd == "set"_sr));
+			co_await DDConfiguration().userRangeConfig().updateRange(
+			    SystemDBWriteLockedNow(cx.getReference()), begin, end, rangeConfig, cmd == "set"_sr);
 		}
 	} else {
-		return fail(fmt::format("Unknown command: '{}'", cmd.printable()));
+		co_return fail(fmt::format("Unknown command: '{}'", cmd.printable()));
 	}
 
-	return true;
+	co_return true;
 }
 
 std::vector<const char*> rangeConfigGenerator(std::vector<StringRef> const& tokens, bool inArgument) {
diff --git a/fdbcli/SetClassCommand.actor.cpp b/fdbcli/SetClassCommand.cpp
similarity index 70%
rename from fdbcli/SetClassCommand.actor.cpp
rename to fdbcli/SetClassCommand.cpp
index 80f8e4a2333..d5155d3ab75 100644
--- a/fdbcli/SetClassCommand.actor.cpp
+++ b/fdbcli/SetClassCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * SetClassCommand.actor.cpp
+ * SetClassCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,7 @@
 
 #include "fmt/format.h"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -29,26 +29,26 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
-ACTOR Future<Void> printProcessClass(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> printProcessClass(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
 			// Hold the reference to the memory
-			state ThreadFuture<RangeResult> classTypeFuture =
+			ThreadFuture<RangeResult> classTypeFuture =
 			    tr->getRange(fdb_cli::processClassTypeSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			state ThreadFuture<RangeResult> classSourceFuture =
+			ThreadFuture<RangeResult> classSourceFuture =
 			    tr->getRange(fdb_cli::processClassSourceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
-			wait(success(safeThreadFutureToFuture(classSourceFuture)) &&
-			     success(safeThreadFutureToFuture(classTypeFuture)));
+			co_await (success(safeThreadFutureToFuture(classSourceFuture)) &&
+			          success(safeThreadFutureToFuture(classTypeFuture)));
 			RangeResult processTypeList = classTypeFuture.get();
 			RangeResult processSourceList = classSourceFuture.get();
 			ASSERT(processSourceList.size() == processTypeList.size());
-			if (!processTypeList.size())
+			if (processTypeList.empty())
 				printf("No processes are registered in the database.\n");
 			fmt::print("There are currently {} processes in the database:\n", processTypeList.size());
 			for (int index = 0; index < processTypeList.size(); index++) {
@@ -65,38 +65,40 @@ ACTOR Future<Void> printProcessClass(Reference<IDatabase> db) {
 				       processTypeList[index].value.toString().c_str(),
 				       processSourceList[index].value.toString().c_str());
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 };
 
-ACTOR Future<bool> setProcessClass(Reference<IDatabase> db, KeyRef network_address, KeyRef class_type) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<bool> setProcessClass(Reference<IDatabase> db, KeyRef network_address, KeyRef class_type) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		Error err;
 		try {
-			state ThreadFuture<Optional<Value>> result =
+			ThreadFuture<Optional<Value>> result =
 			    tr->get(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin));
-			Optional<Value> val = wait(safeThreadFutureToFuture(result));
+			Optional<Value> val = co_await safeThreadFutureToFuture(result);
 			if (!val.present()) {
 				printf("No matching addresses found\n");
-				return false;
+				co_return false;
 			}
 			tr->set(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin), class_type);
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return true;
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return true;
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				// error message already has \n at the end
-				fprintf(stderr, "%s", errorMsgStr.c_str());
-				return false;
-			}
-			wait(safeThreadFutureToFuture(tr->onError(err)));
+			err = e;
+		}
+		if (err.code() == error_code_special_keys_api_failure) {
+			std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr);
+			// error message already has \n at the end
+			fprintf(stderr, "%s", errorMsgStr.c_str());
+			co_return false;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -110,17 +112,17 @@ const KeyRangeRef processClassSourceSpecialKeyRange =
 const KeyRangeRef processClassTypeSpecialKeyRange =
     KeyRangeRef("\xff\xff/configuration/process/class_type/"_sr, "\xff\xff/configuration/process/class_type0"_sr);
 
-ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() != 3 && tokens.size() != 1) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	} else if (tokens.size() == 1) {
-		wait(printProcessClass(db));
+		co_await printProcessClass(db);
 	} else {
-		bool successful = wait(setProcessClass(db, tokens[1], tokens[2]));
-		return successful;
+		bool successful = co_await setProcessClass(db, tokens[1], tokens[2]);
+		co_return successful;
 	}
-	return true;
+	co_return true;
 }
 
 CommandFactory setClassFactory(
@@ -131,7 +133,7 @@ CommandFactory setClassFactory(
                 "`default' resets the process class to the class specified on the command line. The available "
                 "classes are `unset', `storage', `transaction', `resolution', `commit_proxy', `grv_proxy', "
                 "`master', `test', "
-                "`stateless', `log', `router', `cluster_controller', `fast_restore', `data_distributor', "
+                "`stateless', `log', `router', `cluster_controller', `data_distributor', "
                 "`coordinator', `ratekeeper', `backup', and `default'."));
 
 } // namespace fdb_cli
diff --git a/fdbcli/SnapshotCommand.actor.cpp b/fdbcli/SnapshotCommand.cpp
similarity index 79%
rename from fdbcli/SnapshotCommand.actor.cpp
rename to fdbcli/SnapshotCommand.cpp
index 844ae88d588..d02a25c52c3 100644
--- a/fdbcli/SnapshotCommand.actor.cpp
+++ b/fdbcli/SnapshotCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * SnapshotCommand.actor.cpp
+ * SnapshotCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,25 +18,23 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/IClientApi.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
-ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	state bool result = true;
+Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<StringRef> const& tokens) {
+	bool result = true;
 	if (tokens.size() < 2) {
 		printUsage(tokens[0]);
 		result = false;
 	} else {
 		Standalone<StringRef> snap_cmd;
-		state Key uid(deterministicRandom()->randomUniqueID().toString());
+		Key uid(deterministicRandom()->randomUniqueID().toString());
 		for (int i = 1; i < tokens.size(); i++) {
 			snap_cmd = snap_cmd.withSuffix(tokens[i]);
 			if (i != tokens.size() - 1) {
@@ -44,7 +42,7 @@ ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<Str
 			}
 		}
 		try {
-			wait(safeThreadFutureToFuture(db->createSnapshot(uid, snap_cmd)));
+			co_await safeThreadFutureToFuture(db->createSnapshot(uid, snap_cmd));
 			printf("Snapshot command succeeded with UID %s\n", uid.toString().c_str());
 		} catch (Error& e) {
 			fprintf(stderr,
@@ -56,7 +54,7 @@ ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<Str
 			result = false;
 		}
 	}
-	return result;
+	co_return result;
 }
 
 // hidden commands, no help text for now
diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.cpp
similarity index 92%
rename from fdbcli/StatusCommand.actor.cpp
rename to fdbcli/StatusCommand.cpp
index 0a144d33fc1..ff53f12e487 100644
--- a/fdbcli/StatusCommand.actor.cpp
+++ b/fdbcli/StatusCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * StatusCommand.actor.cpp
+ * StatusCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 #include "fmt/chrono.h"
 #include "fmt/core.h"
 #include "fmt/format.h"
@@ -30,7 +30,6 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
@@ -103,6 +102,23 @@ int getNumofNonExcludedMachines(StatusObjectReader statusObjCluster) {
 	return numOfNonExcludedMachines;
 }
 
+bool logEpochsMayBeLosingData(StatusObjectReader statusObjCluster) {
+	if (!statusObjCluster.has("logs")) {
+		return true;
+	}
+
+	bool sawLogEpoch = false;
+	for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) {
+		sawLogEpoch = true;
+		bool possiblyLosingData = true;
+		if (!logEpoch.get("possibly_losing_data", possiblyLosingData) || possiblyLosingData) {
+			return true;
+		}
+	}
+
+	return !sawLogEpoch;
+}
+
 std::string getDateInfoString(StatusObjectReader statusObj, std::string key) {
 	time_t curTime;
 	if (!statusObj.has(key)) {
@@ -117,7 +133,7 @@ std::string getDateInfoString(StatusObjectReader statusObj, std::string key) {
 }
 
 std::string getProcessAddressByServerID(StatusObjectReader processesMap, std::string serverID) {
-	if (serverID == "")
+	if (serverID.empty())
 		return "unknown";
 
 	for (auto proc : processesMap.obj()) {
@@ -185,11 +201,11 @@ void getBackupDRTags(StatusObjectReader& statusObjCluster,
 
 std::string logBackupDR(const char* context, std::map<std::string, std::string> const& tagMap) {
 	std::string outputString = "";
-	if (tagMap.size() > 0) {
+	if (!tagMap.empty()) {
 		outputString += format("\n\n%s:", context);
-		for (auto itr : tagMap) {
+		for (const auto& itr : tagMap) {
 			outputString += format("\n  %-22s", itr.first.c_str());
-			if (itr.second.size() > 0) {
+			if (!itr.second.empty()) {
 				outputString += format(" - %s", itr.second.c_str());
 			}
 		}
@@ -316,7 +332,7 @@ void printStatus(StatusObjectReader statusObj,
 									           numOfNonExcludedProcessesAndZones.second);
 								}
 							} else if (name == "locking_old_transaction_servers" &&
-							           recoveryState["missing_logs"].get_str().size()) {
+							           !recoveryState["missing_logs"].get_str().empty()) {
 								description += format("\nNeed one or more of the following log servers: %s",
 								                      recoveryState["missing_logs"].get_str().c_str());
 							}
@@ -339,7 +355,7 @@ void printStatus(StatusObjectReader statusObj,
 			// Check if cluster controllable is reachable
 			try {
 				// print any cluster messages
-				if (statusObjCluster.has("messages") && statusObjCluster.last().get_array().size()) {
+				if (statusObjCluster.has("messages") && !statusObjCluster.last().get_array().empty()) {
 
 					// any messages we don't want to display
 					std::set<std::string> skipMsgs = { "unreachable_process", "" };
@@ -358,7 +374,7 @@ void printStatus(StatusObjectReader statusObj,
 						if (!msgObj.get("name", messageName)) {
 							continue;
 						}
-						if (skipMsgs.count(messageName)) {
+						if (skipMsgs.contains(messageName)) {
 							continue;
 						} else if (messageName == "client_issues") {
 							if (msgObj.has("issues")) {
@@ -457,7 +473,7 @@ void printStatus(StatusObjectReader statusObj,
 				} else
 					outputString += "unknown";
 
-				if (excludedServersArr.size()) {
+				if (!excludedServersArr.empty()) {
 					outputString += format("\n  Exclusions             - %d (type `exclude' for details)",
 					                       excludedServersArr.size());
 				}
@@ -526,7 +542,7 @@ void printStatus(StatusObjectReader statusObj,
 							outputString += "\n    Region -";
 						}
 						outputString += format("\n        Datacenter                    - %s", regionDC.c_str());
-						if (regionSatelliteDCs.size() > 0) {
+						if (!regionSatelliteDCs.empty()) {
 							outputString += "\n        Satellite datacenters         - ";
 							for (int i = 0; i < regionSatelliteDCs.size(); i++) {
 								if (i != regionSatelliteDCs.size() - 1) {
@@ -585,7 +601,7 @@ void printStatus(StatusObjectReader statusObj,
 						if (excluded) {
 							processExclusions++;
 						}
-						if (process.has("messages") && process.last().get_array().size()) {
+						if (process.has("messages") && !process.last().get_array().empty()) {
 							errors++;
 						}
 
@@ -609,10 +625,10 @@ void printStatus(StatusObjectReader statusObj,
 				} else
 					outputString += "unknown";
 
-				if (zones.size() > 0) {
+				if (!zones.empty()) {
 					outputString += format("\n  Zones                  - %d", zones.size());
 					int zoneExclusions = 0;
-					for (auto itr : zones) {
+					for (const auto& itr : zones) {
 						if (itr.second == 0) {
 							++zoneExclusions;
 						}
@@ -690,26 +706,31 @@ void printStatus(StatusObjectReader statusObj,
 
 						if (dataLoss == -1) {
 							ASSERT_WE_THINK(availLoss == -1);
-							StatusObjectReader logs;
-							std::string epochData;
-							int32_t minFaultTolerance = 0;
+							const bool possiblyLosingData = logEpochsMayBeLosingData(statusObjCluster);
+							if (possiblyLosingData) {
+								outputString += format(
+								    "\n\n  Warning: the database may have data loss and availability loss. Please "
+								    "restart following tlog interfaces, otherwise storage servers may never be able "
+								    "to catch up.\n");
+							} else {
+								outputString += format(
+								    "\n\n  Warning: the database may have availability loss. The current log state "
+								    "does not indicate data loss.\n");
+							}
 							if (statusObjCluster.has("logs")) {
 								for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) {
-									bool possiblyLosingData;
-									if (logEpoch.get("possibly_losing_data", possiblyLosingData) &&
-									    !possiblyLosingData) {
+									bool logEpochPossiblyLosingData;
+									if (logEpoch.get("possibly_losing_data", logEpochPossiblyLosingData) &&
+									    !logEpochPossiblyLosingData) {
 										continue;
 									}
 									// Current epoch doesn't have an end version.
 									int64_t epoch, beginVersion, endVersion = invalidVersion;
 									bool current;
-									int32_t faultTolerance;
 									logEpoch.get("epoch", epoch);
 									logEpoch.get("begin_version", beginVersion);
 									logEpoch.get("end_version", endVersion);
 									logEpoch.get("current", current);
-									logEpoch.get("remote_log_fault_tolerance", faultTolerance);
-									minFaultTolerance = std::min(minFaultTolerance, faultTolerance);
 									std::string missing_log_interfaces;
 									if (logEpoch.has("log_interfaces")) {
 										for (StatusObjectReader logInterface : logEpoch.last().get_array()) {
@@ -718,14 +739,11 @@ void printStatus(StatusObjectReader statusObj,
 											if (logInterface.get("healthy", healthy) && !healthy) {
 												logInterface.get("id", id);
 												logInterface.get("address", address);
-												missing_log_interfaces +=
-												    format("%s,%s ",
-												           id.c_str(),
-												           address.empty() ? "unknown" : address.c_str());
+												missing_log_interfaces += format("%s,%s ", id.c_str(), address.c_str());
 											}
 										}
 									}
-									epochData += format(
+									outputString += format(
 									    "  %s log epoch: %lld begin: %lld end: %s, missing "
 									    "log interfaces(id,address): %s\n",
 									    current ? "Current" : "Old",
@@ -735,20 +753,12 @@ void printStatus(StatusObjectReader statusObj,
 									    missing_log_interfaces.c_str());
 								}
 							}
-							outputString += format("\n\n  ");
-							if (minFaultTolerance < 0) {
-								outputString += format("Warning: the database may have data loss and availability loss. ");
-							}
-							outputString += format(
-							    "Please restart "
-							    "following tlog interfaces, otherwise storage servers may never be able to catch "
-							    "up.\n") + epochData;
 						}
 					}
 				}
 
 				std::string serverTime = getDateInfoString(statusObjCluster, "cluster_controller_timestamp");
-				if (serverTime != "") {
+				if (!serverTime.empty()) {
 					outputString += "\n  Server time            - " + serverTime;
 				}
 			} catch (std::runtime_error&) {
@@ -776,12 +786,12 @@ void printStatus(StatusObjectReader statusObj,
 
 				bool healthy;
 				if (statusObjDataState.get("healthy", healthy) && healthy) {
-					outputString += "Healthy" + (description != "" ? " (" + description + ")" : "");
+					outputString += "Healthy" + (!description.empty() ? " (" + description + ")" : "");
 				} else if (dataState == "missing_data") {
-					outputString += "UNHEALTHY" + (description != "" ? ": " + description : "");
+					outputString += "UNHEALTHY" + (!description.empty() ? ": " + description : "");
 				} else if (dataState == "healing") {
-					outputString += "HEALING" + (description != "" ? ": " + description : "");
-				} else if (description != "") {
+					outputString += "HEALING" + (!description.empty() ? ": " + description : "");
+				} else if (!description.empty()) {
 					outputString += description;
 				} else {
 					outputString += "unknown";
@@ -848,7 +858,7 @@ void printStatus(StatusObjectReader statusObj,
 			} catch (std::runtime_error&) {
 				storageWigglerString += "\n  Unable to retrieve storage wiggler status";
 			}
-			if (storageWigglerString.size()) {
+			if (!storageWigglerString.empty()) {
 				outputString += "\n\nStorage wiggle:";
 				outputString += storageWigglerString;
 			}
@@ -897,11 +907,11 @@ void printStatus(StatusObjectReader statusObj,
 						std::string serverID;
 						limit.get("reason_server_id", serverID);
 						std::string procAddr = getProcessAddressByServerID(processesMap, serverID);
-						performanceLimited = format("\n  Performance limited by %s: %s",
-						                            (procAddr == "unknown")
-						                                ? ("server" + (serverID == "" ? "" : (" " + serverID))).c_str()
-						                                : "process",
-						                            desc.c_str());
+						performanceLimited = format(
+						    "\n  Performance limited by %s: %s",
+						    (procAddr == "unknown") ? ("server" + (serverID.empty() ? "" : (" " + serverID))).c_str()
+						                            : "process",
+						    desc.c_str());
 						if (procAddr != "unknown")
 							performanceLimited += format("\n  Most limiting process: %s", procAddr.c_str());
 					}
@@ -951,7 +961,7 @@ void printStatus(StatusObjectReader statusObj,
 					}
 					if (process.has("messages")) {
 						StatusArray processMessagesArr = process.last().get_array();
-						if (processMessagesArr.size()) {
+						if (!processMessagesArr.empty()) {
 							for (StatusObjectReader msg : processMessagesArr) {
 								std::string desc;
 								std::string addr;
@@ -963,10 +973,10 @@ void printStatus(StatusObjectReader statusObj,
 						}
 					}
 				}
-				if (messagesAddrs.size()) {
+				if (!messagesAddrs.empty()) {
 					outputString += format("\n\n%d FoundationDB processes reported unable to update cluster file:",
 					                       messagesAddrs.size());
-					for (auto msg : messagesAddrs) {
+					for (const auto& msg : messagesAddrs) {
 						outputString += "\n  " + msg;
 					}
 				}
@@ -990,16 +1000,16 @@ void printStatus(StatusObjectReader statusObj,
 			outputString += format("\n  Running backups        - %d", backupTags.size());
 			outputString += format("\n  Running DRs            - ");
 
-			if (drPrimaryTags.size() == 0 && drSecondaryTags.size() == 0) {
+			if (drPrimaryTags.empty() && drSecondaryTags.empty()) {
 				outputString += format("%d", 0);
 			} else {
-				if (drPrimaryTags.size() > 0) {
+				if (!drPrimaryTags.empty()) {
 					outputString += format("%d as primary", drPrimaryTags.size());
-					if (drSecondaryTags.size() > 0) {
+					if (!drSecondaryTags.empty()) {
 						outputString += ", ";
 					}
 				}
-				if (drSecondaryTags.size() > 0) {
+				if (!drSecondaryTags.empty()) {
 					outputString += format("%d as secondary", drSecondaryTags.size());
 				}
 			}
@@ -1108,7 +1118,7 @@ void printStatus(StatusObjectReader statusObj,
 							workerDetails[parsedAddress] = noMetrics;
 						}
 					}
-					for (auto w : workerDetails)
+					for (const auto& w : workerDetails)
 						outputString += "\n" + format("%s", w.second.c_str());
 				} catch (std::runtime_error&) {
 					outputString = outputStringCache;
@@ -1124,7 +1134,7 @@ void printStatus(StatusObjectReader statusObj,
 
 			// client time
 			std::string clientTime = getDateInfoString(statusObjClient, "timestamp");
-			if (clientTime != "") {
+			if (!clientTime.empty()) {
 				outputString += "\n\nClient time: " + clientTime;
 			}
 
@@ -1225,12 +1235,12 @@ void printStatus(StatusObjectReader statusObj,
 // localDb is the native Database object
 // localDb is rarely needed except the "db" has not established a connection to the cluster where the operation will
 // return Never as we expect status command to always return, we use "localDb" to return the default result
-ACTOR Future<bool> statusCommandActor(Reference<IDatabase> db,
-                                      Database localDb,
-                                      std::vector<StringRef> tokens,
-                                      bool isExecMode) {
+Future<bool> statusCommandActor(Reference<IDatabase> db,
+                                Database localDb,
+                                std::vector<StringRef> tokens,
+                                bool isExecMode) {
 
-	state StatusClient::StatusLevel level;
+	StatusClient::StatusLevel level;
 	if (tokens.size() == 1)
 		level = StatusClient::NORMAL;
 	else if (tokens.size() == 2 && tokencmp(tokens[1], "details"))
@@ -1241,17 +1251,17 @@ ACTOR Future<bool> statusCommandActor(Reference<IDatabase> db,
 		level = StatusClient::JSON;
 	else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
-	state StatusObject s;
-	state Reference<ITransaction> tr = db->createTransaction();
+	StatusObject s;
+	Reference<ITransaction> tr = db->createTransaction();
 	if (!tr->isValid()) {
-		StatusObject _s = wait(StatusClient::statusFetcher(localDb));
+		StatusObject _s = co_await StatusClient::statusFetcher(localDb);
 		s = _s;
 	} else {
-		state ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
-		Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
+		ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
+		Optional<Value> statusValue = co_await safeThreadFutureToFuture(statusValueF);
 		if (!statusValue.present()) {
 			fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
 		}
@@ -1265,7 +1275,7 @@ ACTOR Future<bool> statusCommandActor(Reference<IDatabase> db,
 	printStatus(s, level);
 	if (!isExecMode)
 		printf("\n");
-	return true;
+	co_return true;
 }
 
 void statusGenerator(const char* text,
diff --git a/fdbcli/SuspendCommand.actor.cpp b/fdbcli/SuspendCommand.cpp
similarity index 78%
rename from fdbcli/SuspendCommand.actor.cpp
rename to fdbcli/SuspendCommand.cpp
index e4bc2a46534..9176aaf48d1 100644
--- a/fdbcli/SuspendCommand.actor.cpp
+++ b/fdbcli/SuspendCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * SuspendCommand.actor.cpp
+ * SuspendCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,7 @@
 
 #include "boost/algorithm/string.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -29,29 +29,27 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
-ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
-                                       Reference<ITransaction> tr,
-                                       std::vector<StringRef> tokens,
-                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
-	ASSERT(tokens.size() >= 1);
-	state bool result = true;
-	state std::string addressesStr;
+Future<bool> suspendCommandActor(Reference<IDatabase> db,
+                                 Reference<ITransaction> tr,
+                                 std::vector<StringRef> const& tokens,
+                                 std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
+	ASSERT(!tokens.empty());
+	bool result = true;
+	std::string addressesStr;
 	if (tokens.size() == 1) {
 		// initialize worker interfaces
 		address_interface->clear();
-		wait(getWorkerInterfaces(tr, address_interface, true));
-		if (address_interface->size() == 0) {
+		co_await getWorkerInterfaces(tr, address_interface, true);
+		if (address_interface->empty()) {
 			printf("\nNo addresses can be suspended.\n");
 		} else if (address_interface->size() == 1) {
 			printf("\nThe following address can be suspended:\n");
 		} else {
 			printf("\nThe following %zu addresses can be suspended:\n", address_interface->size());
 		}
-		for (auto it : *address_interface) {
+		for (const auto& it : *address_interface) {
 			printf("%s\n", printable(it.first).c_str());
 		}
 		printf("\n");
@@ -68,9 +66,9 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
 		}
 
 		if (result) {
-			state double seconds;
+			double seconds{ 0 };
 			int n = 0;
-			state int i;
+			int i{ 0 };
 			auto secondsStr = tokens[1].toString();
 			if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) {
 				printUsage(tokens[0]);
@@ -82,7 +80,7 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
 				}
 				addressesStr = boost::algorithm::join(addressesVec, ",");
 				int64_t suspendRequestSent =
-				    wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, static_cast<int>(seconds))));
+				    co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, static_cast<int>(seconds)));
 				if (!suspendRequestSent) {
 					result = false;
 					fprintf(
@@ -96,7 +94,7 @@ ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
 			}
 		}
 	}
-	return result;
+	co_return result;
 }
 
 CommandFactory suspendFactory(
diff --git a/fdbcli/ThrottleCommand.actor.cpp b/fdbcli/ThrottleCommand.cpp
similarity index 90%
rename from fdbcli/ThrottleCommand.actor.cpp
rename to fdbcli/ThrottleCommand.cpp
index 02a3cd40633..82531090a4b 100644
--- a/fdbcli/ThrottleCommand.actor.cpp
+++ b/fdbcli/ThrottleCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * ThrottleCommand.actor.cpp
+ * ThrottleCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/IClientApi.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TagThrottle.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/CommitTransaction.h"
@@ -29,27 +29,26 @@
 #include "flow/Arena.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
 static constexpr int defaultThrottleListLimit = 100;
 
-ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 
 	if (tokens.size() == 1) {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	} else if (tokencmp(tokens[1], "list")) {
 		if (tokens.size() > 4) {
 			fmt::print("Usage: throttle list [throttled|recommended|all] [LIMIT]\n\n");
 			fmt::print("Lists tags that are currently throttled.\n");
 			fmt::print("The default LIMIT is {} tags.\n", defaultThrottleListLimit);
-			return false;
+			co_return false;
 		}
 
-		state bool reportThrottled = true;
-		state bool reportRecommended = false;
+		bool reportThrottled = true;
+		bool reportRecommended = false;
 		if (tokens.size() >= 3) {
 			if (tokencmp(tokens[2], "recommended")) {
 				reportThrottled = false;
@@ -59,27 +58,27 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 				reportRecommended = true;
 			} else if (!tokencmp(tokens[2], "throttled")) {
 				printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str());
-				return false;
+				co_return false;
 			}
 		}
 
-		state int throttleListLimit = defaultThrottleListLimit;
+		int throttleListLimit = defaultThrottleListLimit;
 		if (tokens.size() >= 4) {
 			char* end;
 			throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10);
 			if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) {
 				fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str());
-				return false;
+				co_return false;
 			}
 		}
 
-		state std::vector<TagThrottleInfo> tags;
+		std::vector<TagThrottleInfo> tags;
 		if (reportThrottled && reportRecommended) {
-			wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit, ContainsRecommended::True)));
+			tags = co_await ThrottleApi::getThrottledTags(db, throttleListLimit, ContainsRecommended::True);
 		} else if (reportThrottled) {
-			wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit)));
+			tags = co_await ThrottleApi::getThrottledTags(db, throttleListLimit);
 		} else if (reportRecommended) {
-			wait(store(tags, ThrottleApi::getRecommendedTags(db, throttleListLimit)));
+			tags = co_await ThrottleApi::getRecommendedTags(db, throttleListLimit);
 		}
 
 		bool anyLogged = false;
@@ -130,7 +129,7 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 			       "d) (default 1h).\n");
 			printf("An optional priority can be specified. Choices are `default', `immediate', and "
 			       "`batch' (default `default').\n");
-			return false;
+			co_return false;
 		}
 
 		double tpsRate = 0.0;
@@ -142,24 +141,24 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 			tpsRate = std::strtod((const char*)tokens[4].begin(), &end);
 			if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) {
 				fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str());
-				return false;
+				co_return false;
 			}
 			if (tpsRate < 0) {
 				fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate);
-				return false;
+				co_return false;
 			}
 		}
 		if (tokens.size() == 6) {
 			Optional<uint64_t> parsedDuration = parseDuration(tokens[5].toString());
 			if (!parsedDuration.present()) {
 				fprintf(stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str());
-				return false;
+				co_return false;
 			}
 			duration = parsedDuration.get();
 
 			if (duration == 0) {
 				fprintf(stderr, "ERROR: throttle duration cannot be 0\n");
-				return false;
+				co_return false;
 			}
 		}
 		if (tokens.size() == 7) {
@@ -174,18 +173,18 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 				        "ERROR: unrecognized priority `%s'. Must be one of `default',\n  `immediate', "
 				        "or `batch'.\n",
 				        tokens[6].toString().c_str());
-				return false;
+				co_return false;
 			}
 		}
 
 		TagSet tagSet;
 		tagSet.addTag(tokens[3]);
 
-		wait(ThrottleApi::throttleTags(db, tagSet, tpsRate, duration, TagThrottleType::MANUAL, priority));
+		co_await ThrottleApi::throttleTags(db, tagSet, tpsRate, duration, TagThrottleType::MANUAL, priority);
 		printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str());
 	} else if (tokencmp(tokens[1], "off")) {
 		int nextIndex = 2;
-		state TagSet tagSet;
+		TagSet tagSet;
 		bool throttleTypeSpecified = false;
 		bool is_error = false;
 		Optional<TagThrottleType> throttleType = TagThrottleType::MANUAL;
@@ -254,13 +253,13 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 		}
 
 		if (!is_error) {
-			state const char* throttleTypeString =
+			const char* throttleTypeString =
 			    !throttleType.present() ? "" : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually ");
-			state std::string priorityString =
+			std::string priorityString =
 			    priority.present() ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) : "";
 
 			if (tagSet.size() > 0) {
-				bool success = wait(ThrottleApi::unthrottleTags(db, tagSet, throttleType, priority));
+				bool success = co_await ThrottleApi::unthrottleTags(db, tagSet, throttleType, priority);
 				if (success) {
 					fmt::print("Unthrottled {0}{1}\n", tagSet.toString(), priorityString);
 				} else {
@@ -270,7 +269,7 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 					           priorityString);
 				}
 			} else {
-				bool unthrottled = wait(ThrottleApi::unthrottleAll(db, throttleType, priority));
+				bool unthrottled = co_await ThrottleApi::unthrottleAll(db, throttleType, priority);
 				if (unthrottled) {
 					printf("Unthrottled all %sthrottled tags%s\n", throttleTypeString, priorityString.c_str());
 				} else {
@@ -297,17 +296,17 @@ ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<Str
 			printf("Usage: throttle <enable|disable> auto\n");
 			printf("\n");
 			printf("Enables or disable automatic tag throttling.\n");
-			return false;
+			co_return false;
 		}
-		state bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable");
-		wait(ThrottleApi::enableAuto(db, autoTagThrottlingEnabled));
+		bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable");
+		co_await ThrottleApi::enableAuto(db, autoTagThrottlingEnabled);
 		printf("Automatic tag throttling has been %s\n", autoTagThrottlingEnabled ? "enabled" : "disabled");
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
 
-	return true;
+	co_return true;
 }
 
 void throttleGenerator(const char* text,
diff --git a/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp b/fdbcli/TriggerDDTeamInfoLogCommand.cpp
similarity index 79%
rename from fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp
rename to fdbcli/TriggerDDTeamInfoLogCommand.cpp
index 25785fcdedb..00bfa7231a3 100644
--- a/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp
+++ b/fdbcli/TriggerDDTeamInfoLogCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * TriggerDDTeamInfoLogCommand.actor.cpp
+ * TriggerDDTeamInfoLogCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
@@ -27,24 +27,24 @@
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace fdb_cli {
 
-ACTOR Future<bool> triggerddteaminfologCommandActor(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<bool> triggerddteaminfologCommandActor(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			std::string v = deterministicRandom()->randomUniqueID().toString();
 			tr->set(triggerDDTeamInfoPrintKey, v);
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 			printf("Triggered team info logging in data distribution.\n");
-			return true;
+			co_return true;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
diff --git a/fdbcli/TssqCommand.actor.cpp b/fdbcli/TssqCommand.cpp
similarity index 67%
rename from fdbcli/TssqCommand.actor.cpp
rename to fdbcli/TssqCommand.cpp
index a81df3df9e7..96453001503 100644
--- a/fdbcli/TssqCommand.actor.cpp
+++ b/fdbcli/TssqCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * TssqCommand.actor.cpp
+ * TssqCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,74 +18,76 @@
  * limitations under the License.
  */
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/IClientApi.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/SystemData.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
-ACTOR Future<Void> tssQuarantineList(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+Future<Void> tssQuarantineList(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			// Hold the reference to the standalone's memory
-			state ThreadFuture<RangeResult> resultFuture = tr->getRange(tssQuarantineKeys, CLIENT_KNOBS->TOO_MANY);
-			RangeResult result = wait(safeThreadFutureToFuture(resultFuture));
+			ThreadFuture<RangeResult> resultFuture = tr->getRange(tssQuarantineKeys, CLIENT_KNOBS->TOO_MANY);
+			RangeResult result = co_await safeThreadFutureToFuture(resultFuture);
 			// shouldn't have many quarantined TSSes
 			ASSERT(!result.more);
-			printf("Found %d quarantined TSS processes%s\n", result.size(), result.size() == 0 ? "." : ":");
+			printf("Found %d quarantined TSS processes%s\n", result.size(), result.empty() ? "." : ":");
 			for (auto& it : result) {
 				printf("  %s\n", decodeTssQuarantineKey(it.key).toString().c_str());
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<bool> tssQuarantine(Reference<IDatabase> db, bool enable, UID tssId) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
+Future<bool> tssQuarantine(Reference<IDatabase> db, bool enable, UID tssId) {
+	Reference<ITransaction> tr = db->createTransaction();
+	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 			// Do some validation first to make sure the command is valid
 			// hold the returned standalone object's memory
-			state ThreadFuture<Optional<Value>> serverListValueF = tr->get(serverListKeyFor(tssId));
-			Optional<Value> serverListValue = wait(safeThreadFutureToFuture(serverListValueF));
+			ThreadFuture<Optional<Value>> serverListValueF = tr->get(serverListKeyFor(tssId));
+			Optional<Value> serverListValue = co_await safeThreadFutureToFuture(serverListValueF);
 			if (!serverListValue.present()) {
 				printf("No TSS %s found in cluster!\n", tssId.toString().c_str());
-				return false;
+				co_return false;
 			}
-			state StorageServerInterface ssi = decodeServerListValue(serverListValue.get());
+			StorageServerInterface ssi = decodeServerListValue(serverListValue.get());
 			if (!ssi.isTss()) {
 				printf("Cannot quarantine Non-TSS storage ID %s!\n", tssId.toString().c_str());
-				return false;
+				co_return false;
 			}
 
 			// hold the returned standalone object's memory
-			state ThreadFuture<Optional<Value>> currentQuarantineValueF = tr->get(tssQuarantineKeyFor(tssId));
-			Optional<Value> currentQuarantineValue = wait(safeThreadFutureToFuture(currentQuarantineValueF));
+			ThreadFuture<Optional<Value>> currentQuarantineValueF = tr->get(tssQuarantineKeyFor(tssId));
+			Optional<Value> currentQuarantineValue = co_await safeThreadFutureToFuture(currentQuarantineValueF);
 			if (enable && currentQuarantineValue.present()) {
 				printf("TSS %s already in quarantine, doing nothing.\n", tssId.toString().c_str());
-				return false;
+				co_return false;
 			} else if (!enable && !currentQuarantineValue.present()) {
 				printf("TSS %s is not in quarantine, cannot remove from quarantine!.\n", tssId.toString().c_str());
-				return false;
+				co_return false;
 			}
 
 			if (enable) {
@@ -96,44 +98,45 @@ ACTOR Future<bool> tssQuarantine(Reference<IDatabase> db, bool enable, UID tssId
 				tr->clear(tssQuarantineKeyFor(tssId));
 			}
 
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 			break;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 	printf("Successfully %s TSS %s\n", enable ? "quarantined" : "removed", tssId.toString().c_str());
-	return true;
+	co_return true;
 }
 
 } // namespace
 
 namespace fdb_cli {
 
-ACTOR Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
+Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
 	if (tokens.size() == 2) {
 		if (tokens[1] != "list"_sr) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		} else {
-			wait(tssQuarantineList(db));
+			co_await tssQuarantineList(db);
 		}
 	} else if (tokens.size() == 3) {
 		if ((tokens[1] != "start"_sr && tokens[1] != "stop"_sr) || (tokens[2].size() != 32) ||
 		    !std::all_of(tokens[2].begin(), tokens[2].end(), &isxdigit)) {
 			printUsage(tokens[0]);
-			return false;
+			co_return false;
 		} else {
 			bool enable = tokens[1] == "start"_sr;
 			UID tssId = UID::fromString(tokens[2].toString());
-			bool success = wait(tssQuarantine(db, enable, tssId));
-			return success;
+			bool success = co_await tssQuarantine(db, enable, tssId);
+			co_return success;
 		}
 	} else {
 		printUsage(tokens[0]);
-		return false;
+		co_return false;
 	}
-	return true;
+	co_return true;
 }
 
 CommandFactory tssqFactory(
diff --git a/fdbcli/Util.actor.cpp b/fdbcli/Util.actor.cpp
deleted file mode 100644
index de5556ab8d3..00000000000
--- a/fdbcli/Util.actor.cpp
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Util.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbcli/fdbcli.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/Schemas.h"
-#include "fdbclient/Status.h"
-
-#include "flow/Arena.h"
-
-#include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-namespace fdb_cli {
-
-bool tokencmp(StringRef token, const char* command) {
-	if (token.size() != strlen(command))
-		return false;
-
-	return !memcmp(token.begin(), command, token.size());
-}
-
-void printUsage(StringRef command) {
-	const auto& helpMap = CommandFactory::commands();
-	auto i = helpMap.find(command.toString());
-	if (i != helpMap.end())
-		printf("Usage: %s\n", i->second.usage.c_str());
-	else
-		fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str());
-}
-
-void printLongDesc(StringRef command) {
-	const auto& helpMap = CommandFactory::commands();
-	auto i = helpMap.find(command.toString());
-	if (i != helpMap.end())
-		printf("%s\n", i->second.long_desc.c_str());
-	else
-		fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str());
-}
-
-ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr) {
-	// hold the returned standalone object's memory
-	state ThreadFuture<Optional<Value>> errorMsgF = tr->get(fdb_cli::errorMsgSpecialKey);
-	Optional<Value> errorMsg = wait(safeThreadFutureToFuture(errorMsgF));
-	// Error message should be present
-	ASSERT(errorMsg.present());
-	// Read the json string
-	auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-	// verify schema
-	auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-	std::string errorStr;
-	ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-	// return the error message
-	return valueObj["message"].get_str();
-}
-
-void addInterfacesFromKVs(RangeResult& kvs,
-                          std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
-	for (const auto& kv : kvs) {
-		ClientWorkerInterface workerInterf;
-		try {
-			// the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version
-			workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
-		} catch (Error& e) {
-			fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what());
-			return;
-		}
-		ClientLeaderRegInterface leaderInterf(workerInterf.address());
-		StringRef ip_port = (kv.key.endsWith(":tls"_sr) ? kv.key.removeSuffix(":tls"_sr) : kv.key)
-		                        .removePrefix("\xff\xff/worker_interfaces/"_sr);
-		(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
-
-		if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
-			Key full_ip_port2 =
-			    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
-			StringRef ip_port2 =
-			    full_ip_port2.endsWith(":tls"_sr) ? full_ip_port2.removeSuffix(":tls"_sr) : full_ip_port2;
-			(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
-		}
-	}
-}
-
-ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
-                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
-                                       bool verify) {
-	if (verify) {
-		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-		tr->set(workerInterfacesVerifyOptionSpecialKey, ValueRef());
-	}
-	// Hold the reference to the standalone's memory
-	state ThreadFuture<RangeResult> kvsFuture = tr->getRange(
-	    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY);
-	state RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture));
-	ASSERT(!kvs.more);
-	if (verify) {
-		// remove the option if set
-		tr->clear(workerInterfacesVerifyOptionSpecialKey);
-	}
-	addInterfacesFromKVs(kvs, address_interface);
-	return Void();
-}
-
-ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
-		try {
-			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state ThreadFuture<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
-			state ThreadFuture<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
-
-			wait(success(safeThreadFutureToFuture(processClasses)) && success(safeThreadFutureToFuture(processData)));
-			ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
-			ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
-
-			state std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class;
-			state int i;
-			for (i = 0; i < processClasses.get().size(); i++) {
-				try {
-					id_class[decodeProcessClassKey(processClasses.get()[i].key)] =
-					    decodeProcessClassValue(processClasses.get()[i].value);
-				} catch (Error& e) {
-					fprintf(stderr, "Error: %s; Client version is too old, please use a newer version\n", e.what());
-					return false;
-				}
-			}
-
-			for (i = 0; i < processData.get().size(); i++) {
-				ProcessData data = decodeWorkerListValue(processData.get()[i].value);
-				ProcessClass processClass = id_class[data.locality.processId()];
-
-				if (processClass.classSource() == ProcessClass::DBSource ||
-				    data.processClass.classType() == ProcessClass::UnsetClass)
-					data.processClass = processClass;
-
-				if (data.processClass.classType() != ProcessClass::TesterClass)
-					workers->push_back(data);
-			}
-
-			return true;
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "GetWorkersError").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
-		}
-	}
-}
-
-ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
-                                              std::map<std::string, StorageServerInterface>* interfaces) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
-		interfaces->clear();
-		try {
-			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state ThreadFuture<RangeResult> serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-			wait(success(safeThreadFutureToFuture(serverListF)));
-			ASSERT(!serverListF.get().more);
-			ASSERT_LT(serverListF.get().size(), CLIENT_KNOBS->TOO_MANY);
-			RangeResult serverList = serverListF.get();
-			// decode server interfaces
-			for (int i = 0; i < serverList.size(); i++) {
-				auto ssi = decodeServerListValue(serverList[i].value);
-				(*interfaces)[ssi.address().toString()] = ssi;
-			}
-			return Void();
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(e);
-			wait(safeThreadFutureToFuture(tr->onError(e)));
-		}
-	}
-}
-
-} // namespace fdb_cli
diff --git a/fdbcli/Util.cpp b/fdbcli/Util.cpp
new file mode 100644
index 00000000000..39da058925d
--- /dev/null
+++ b/fdbcli/Util.cpp
@@ -0,0 +1,465 @@
+/*
+ * Util.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/Schemas.h"
+#include "fdbclient/Status.h"
+#include "fdbclient/BulkDumping.h"
+#include "fdbclient/BulkLoading.h"
+#include "flow/Arena.h"
+#include "flow/ThreadHelper.actor.h"
+#include <fmt/core.h>
+
+namespace fdb_cli {
+
+bool tokencmp(StringRef token, const char* command) {
+	if (token.size() != strlen(command))
+		return false;
+
+	return !memcmp(token.begin(), command, token.size());
+}
+
+void printUsage(StringRef command) {
+	const auto& helpMap = CommandFactory::commands();
+	auto i = helpMap.find(command.toString());
+	if (i != helpMap.end())
+		printf("Usage: %s\n", i->second.usage.c_str());
+	else
+		fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str());
+}
+
+void printLongDesc(StringRef command) {
+	const auto& helpMap = CommandFactory::commands();
+	auto i = helpMap.find(command.toString());
+	if (i != helpMap.end())
+		printf("%s\n", i->second.long_desc.c_str());
+	else
+		fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str());
+}
+
+Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr) {
+	// hold the returned standalone object's memory
+	ThreadFuture<Optional<Value>> errorMsgF = tr->get(fdb_cli::errorMsgSpecialKey);
+	Optional<Value> errorMsg = co_await safeThreadFutureToFuture(errorMsgF);
+	// Error message should be present
+	ASSERT(errorMsg.present());
+	// Read the json string
+	auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+	// verify schema
+	auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+	std::string errorStr;
+	ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+	// return the error message
+	co_return valueObj["message"].get_str();
+}
+
+void addInterfacesFromKVs(RangeResult& kvs,
+                          std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
+	for (const auto& kv : kvs) {
+		ClientWorkerInterface workerInterf;
+		try {
+			// the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version
+			workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
+		} catch (Error& e) {
+			fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what());
+			return;
+		}
+		ClientLeaderRegInterface leaderInterf(workerInterf.address());
+		StringRef ip_port = (kv.key.endsWith(":tls"_sr) ? kv.key.removeSuffix(":tls"_sr) : kv.key)
+		                        .removePrefix("\xff\xff/worker_interfaces/"_sr);
+		(*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf);
+
+		if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
+			Key full_ip_port2 =
+			    StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
+			StringRef ip_port2 =
+			    full_ip_port2.endsWith(":tls"_sr) ? full_ip_port2.removeSuffix(":tls"_sr) : full_ip_port2;
+			(*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf);
+		}
+	}
+}
+
+Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
+                                 std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
+                                 bool verify) {
+	if (verify) {
+		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		tr->set(workerInterfacesVerifyOptionSpecialKey, ValueRef());
+	}
+	// Hold the reference to the standalone's memory
+	ThreadFuture<RangeResult> kvsFuture = tr->getRange(
+	    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY);
+	RangeResult kvs = co_await safeThreadFutureToFuture(kvsFuture);
+	ASSERT(!kvs.more);
+	if (verify) {
+		// remove the option if set
+		tr->clear(workerInterfacesVerifyOptionSpecialKey);
+	}
+	addInterfacesFromKVs(kvs, address_interface);
+}
+
+Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			ThreadFuture<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
+			ThreadFuture<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
+
+			co_await (success(safeThreadFutureToFuture(processClasses)) &&
+			          success(safeThreadFutureToFuture(processData)));
+			ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
+			ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
+
+			std::map<Optional<Standalone<StringRef>>, ProcessClass> id_class;
+			int i{ 0 };
+			for (i = 0; i < processClasses.get().size(); i++) {
+				try {
+					id_class[decodeProcessClassKey(processClasses.get()[i].key)] =
+					    decodeProcessClassValue(processClasses.get()[i].value);
+				} catch (Error& e) {
+					fprintf(stderr, "Error: %s; Client version is too old, please use a newer version\n", e.what());
+					co_return false;
+				}
+			}
+
+			for (i = 0; i < processData.get().size(); i++) {
+				ProcessData data = decodeWorkerListValue(processData.get()[i].value);
+				ProcessClass processClass = id_class[data.locality.processId()];
+
+				if (processClass.classSource() == ProcessClass::DBSource ||
+				    data.processClass.classType() == ProcessClass::UnsetClass)
+					data.processClass = processClass;
+
+				if (data.processClass.classType() != ProcessClass::TesterClass)
+					workers->push_back(data);
+			}
+
+			co_return true;
+		} catch (Error& e) {
+			err = e;
+		}
+		TraceEvent(SevWarn, "GetWorkersError").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
+	}
+}
+
+Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
+                                        std::map<std::string, StorageServerInterface>* interfaces) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		interfaces->clear();
+		Error err;
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			ThreadFuture<RangeResult> serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+			co_await safeThreadFutureToFuture(serverListF);
+			ASSERT(!serverListF.get().more);
+			ASSERT_LT(serverListF.get().size(), CLIENT_KNOBS->TOO_MANY);
+			RangeResult serverList = serverListF.get();
+			// decode server interfaces
+			for (int i = 0; i < serverList.size(); i++) {
+				auto ssi = decodeServerListValue(serverList[i].value);
+				(*interfaces)[ssi.address().toString()] = ssi;
+			}
+			co_return;
+		} catch (Error& e) {
+			err = e;
+		}
+		TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(err);
+		co_await safeThreadFutureToFuture(tr->onError(err));
+	}
+}
+
+// Shared UID validation for bulk operations (BulkDump/BulkLoad)
+UID validateBulkJobId(StringRef token, const char* usage) {
+	UID jobId;
+	try {
+		jobId = UID::fromStringThrowsOnFailure(token.toString());
+	} catch (Error&) {
+		fmt::println("ERROR: Invalid job id '{}' (expected 32 hex characters)", token.toString());
+		fmt::println("{}", usage);
+		throw operation_failed();
+	}
+
+	if (!jobId.isValid()) {
+		fmt::println("ERROR: Invalid job id {}", token.toString());
+		fmt::println("{}", usage);
+		throw operation_failed();
+	}
+
+	return jobId;
+}
+
+Future<std::string> getBulkOwnerSuffix(Database cx, UID jobId, bool isDumpJob) {
+	if (isDumpJob) {
+		Optional<BulkDumpOwnerInfo> ownerInfo = co_await getBulkDumpOwner(cx, jobId);
+		if (ownerInfo.present()) {
+			co_return fmt::format(" (owned by {} '{}')", ownerInfo.get().ownerType, ownerInfo.get().ownerName);
+		}
+	} else {
+		Optional<BulkDumpOwnerInfo> ownerInfo = co_await getBulkLoadOwner(cx, jobId);
+		if (ownerInfo.present()) {
+			co_return fmt::format(" (owned by {} '{}')", ownerInfo.get().ownerType, ownerInfo.get().ownerName);
+		}
+	}
+	co_return std::string("");
+}
+
+// Format common progress metrics (throughput, ETA, elapsed time)
+void printProgressMetrics(double avgBytesPerSecond, Optional<double> etaSeconds, double elapsedSeconds) {
+	// Throughput
+	if (avgBytesPerSecond > 0) {
+		fmt::println(" Throughput - {:.1f} MB/s", avgBytesPerSecond / 1048576.0);
+	}
+
+	// ETA
+	if (etaSeconds.present()) {
+		fmt::println(" Estimated time remaining - {}", formatDurationHumanReadable((int)etaSeconds.get()));
+	}
+
+	// Elapsed time
+	if (elapsedSeconds > 0) {
+		fmt::println(" Elapsed time - {}", formatDurationHumanReadable((int)elapsedSeconds));
+	}
+}
+
+// Print detailed task breakdown for bulk operations
+void printTaskBreakdown(int submittedTasks, int triggeredTasks, int runningTasks, int completeTasks, int errorTasks) {
+	int totalTasks = submittedTasks + triggeredTasks + runningTasks + completeTasks + errorTasks;
+	if (totalTasks > 0) {
+		fmt::println(" Task Status Breakdown:");
+		if (submittedTasks > 0)
+			fmt::println("   Submitted: {}", submittedTasks);
+		if (triggeredTasks > 0)
+			fmt::println("   Triggered: {}", triggeredTasks);
+		if (runningTasks > 0)
+			fmt::println("   Running: {}", runningTasks);
+		if (completeTasks > 0)
+			fmt::println("   Complete: {}", completeTasks);
+		if (errorTasks > 0)
+			fmt::println("   Error: {}", errorTasks);
+	}
+}
+
+// Advanced performance monitoring for bulk operations
+BulkHealthMetrics BulkHealthMetrics::analyze(double throughputMBps,
+                                             double efficiencyPercent,
+                                             int stalledTasks,
+                                             int errorTasks,
+                                             double elapsedMinutes) {
+	BulkHealthMetrics metrics;
+
+	// Calculate health score (weighted average)
+	double throughputScore = std::min(100.0, throughputMBps * 2); // 50MB/s = 100 points
+	double efficiencyScore = efficiencyPercent;
+	double reliabilityScore = std::max(0.0, 100.0 - (stalledTasks * 10) - (errorTasks * 20));
+
+	metrics.healthScore = (throughputScore * 0.4) + (efficiencyScore * 0.4) + (reliabilityScore * 0.2);
+
+	// Determine status
+	if (metrics.healthScore >= 90) {
+		metrics.healthStatus = "Excellent";
+	} else if (metrics.healthScore >= 75) {
+		metrics.healthStatus = "Good";
+	} else if (metrics.healthScore >= 50) {
+		metrics.healthStatus = "Fair";
+	} else {
+		metrics.healthStatus = "Poor";
+	}
+
+	// Generate recommendations
+	if (throughputMBps < 5) {
+		metrics.recommendations.push_back("Consider increasing cluster resources for better throughput");
+	}
+	if (errorTasks > 0) {
+		metrics.recommendations.push_back("Investigate error tasks for potential issues");
+	}
+	if (stalledTasks > 0) {
+		metrics.recommendations.push_back("Monitor stalled tasks - may indicate network or storage issues");
+	}
+	if (elapsedMinutes > 60 && efficiencyPercent < 50) {
+		metrics.recommendations.push_back("Long-running job with low efficiency - consider optimization");
+	}
+
+	return metrics;
+}
+
+void printBulkHealthAnalysis(const BulkHealthMetrics& health) {
+	fmt::println("");
+	fmt::println("Health Analysis:");
+	fmt::println(" Overall Health: {:.1f}/100 ({})", health.healthScore, health.healthStatus);
+
+	if (!health.recommendations.empty()) {
+		fmt::println(" Recommendations");
+		for (const auto& rec : health.recommendations) {
+			fmt::println("   - {}", rec);
+		}
+	}
+}
+
+BulkErrorAnalysis BulkErrorAnalysis::analyze(int errorTasks,
+                                             int stalledTasks,
+                                             const std::vector<std::string>& recentErrors) {
+	BulkErrorAnalysis analysis;
+	analysis.totalErrors = errorTasks;
+
+	// Categorize errors
+	for (const auto& error : recentErrors) {
+		if (error.find("timeout") != std::string::npos) {
+			analysis.errorCategories["Timeout Issues"]++;
+		} else if (error.find("network") != std::string::npos || error.find("connection") != std::string::npos) {
+			analysis.errorCategories["Network Issues"]++;
+		} else if (error.find("permission") != std::string::npos || error.find("access") != std::string::npos) {
+			analysis.errorCategories["Access Issues"]++;
+		} else if (error.find("disk") != std::string::npos || error.find("storage") != std::string::npos) {
+			analysis.errorCategories["Storage Issues"]++;
+		} else {
+			analysis.errorCategories["Other Issues"]++;
+		}
+	}
+
+	return analysis;
+}
+
+void printErrorDiagnostics(const BulkErrorAnalysis& analysis) {
+	if (analysis.totalErrors > 0) {
+		fmt::println("");
+		fmt::println("Error Diagnostics:");
+
+		if (!analysis.errorCategories.empty()) {
+			fmt::println(" Error Categories:");
+			for (const auto& [category, count] : analysis.errorCategories) {
+				fmt::println("   {} - {} occurrences", category, count);
+			}
+		}
+	}
+}
+
+BulkOptimizationRecommendations BulkOptimizationRecommendations::generate(double throughputMBps,
+                                                                          double efficiency,
+                                                                          int stalledTasks,
+                                                                          int errorTasks,
+                                                                          double elapsedMinutes,
+                                                                          int totalTasks) {
+	BulkOptimizationRecommendations recs;
+
+	// Performance recommendations
+	if (throughputMBps < 10 && totalTasks > 100) {
+		recs.performanceRecommendations.push_back(
+		    "Low throughput detected. Consider increasing storage server resources or parallelism");
+	}
+	if (elapsedMinutes > 30 && efficiency < 80) {
+		recs.performanceRecommendations.push_back(
+		    "Long-running job with low efficiency. Consider optimizing shard sizes or task distribution");
+	}
+
+	return recs;
+}
+
+void printOptimizationRecommendations(const BulkOptimizationRecommendations& recs) {
+	if (!recs.performanceRecommendations.empty()) {
+		fmt::println("");
+		fmt::println("Optimization Recommendations:");
+		fmt::println(" Performance:");
+		for (const auto& rec : recs.performanceRecommendations) {
+			fmt::println("   - {}", rec);
+		}
+	}
+}
+
+// Format bytes with progress percentage for bulk operations
+std::string formatBytesProgress(int64_t completedBytes, Optional<int64_t> totalBytes) {
+	std::string result = format("%.2f MB", completedBytes / 1048576.0);
+	if (totalBytes.present()) {
+		result += fmt::format(" / {:.2f} MB ({:.1f}%)",
+		                      totalBytes.get() / 1048576.0,
+		                      totalBytes.get() > 0 ? 100.0 * completedBytes / totalBytes.get() : 0.0);
+	}
+	return result;
+}
+
+// Print progress summary for bulk operations
+void printProgressSummary(const char* operationName,
+                          int completeTasks,
+                          int totalTasks,
+                          int64_t completedBytes,
+                          int64_t totalBytes) {
+	fmt::println("");
+	fmt::println("{} Progress Summary:", operationName);
+	fmt::println(" Tasks: {} / {} ({:.1f}%)",
+	             completeTasks,
+	             totalTasks,
+	             totalTasks > 0 ? 100.0 * completeTasks / totalTasks : 0.0);
+	fmt::println(" Data: {}", formatBytesProgress(completedBytes, totalBytes));
+}
+
+void printStalledTasks(const std::vector<BulkLoadStalledTask>& stalledTasks) {
+	if (!stalledTasks.empty()) {
+		fmt::println("");
+		fmt::println("WARNING: {} stalled tasks (no progress > 60s):", stalledTasks.size());
+		for (const auto& stalled : stalledTasks) {
+			fmt::println(" Task {}: {}, stalled {:.0f}s, {} restarts",
+			             stalled.taskId.shortString(),
+			             stalled.range.toString(),
+			             stalled.stalledSeconds,
+			             stalled.restartCount);
+			if (!stalled.lastError.empty()) {
+				fmt::println("           Last error: {}", stalled.lastError);
+			}
+		}
+	}
+}
+
+void printBulkAnalysis(double avgBytesPerSecond,
+                       double elapsedSeconds,
+                       int completeTasks,
+                       int totalTasks,
+                       int errorTasks,
+                       const std::vector<BulkLoadStalledTask>& stalledTasks) {
+	double efficiency = totalTasks > 0 ? 100.0 * completeTasks / totalTasks : 0.0;
+
+	auto healthMetrics = BulkHealthMetrics::analyze(
+	    avgBytesPerSecond / 1048576.0, efficiency, stalledTasks.size(), errorTasks, elapsedSeconds / 60.0);
+	printBulkHealthAnalysis(healthMetrics);
+
+	printStalledTasks(stalledTasks);
+
+	std::vector<std::string> recentErrors;
+	auto errorAnalysis = BulkErrorAnalysis::analyze(errorTasks, stalledTasks.size(), recentErrors);
+	printErrorDiagnostics(errorAnalysis);
+
+	auto recommendations = BulkOptimizationRecommendations::generate(
+	    avgBytesPerSecond / 1048576.0, efficiency, stalledTasks.size(), errorTasks, elapsedSeconds / 60.0, totalTasks);
+	printOptimizationRecommendations(recommendations);
+
+	if (errorTasks > 0) {
+		fmt::println("");
+		fmt::println("WARNING: {} tasks in error state", errorTasks);
+	}
+}
+
+} // namespace fdb_cli
diff --git a/fdbcli/VersionEpochCommand.actor.cpp b/fdbcli/VersionEpochCommand.cpp
similarity index 68%
rename from fdbcli/VersionEpochCommand.actor.cpp
rename to fdbcli/VersionEpochCommand.cpp
index 41e4e268572..c10eefcdef5 100644
--- a/fdbcli/VersionEpochCommand.actor.cpp
+++ b/fdbcli/VersionEpochCommand.cpp
@@ -1,5 +1,5 @@
 /*
- * VersionEpochCommand.actor.cpp
+ * VersionEpochCommand.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,15 +20,14 @@
 
 #include "boost/lexical_cast.hpp"
 
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include "fdbclient/IClientApi.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
@@ -39,44 +38,48 @@ struct VersionInfo {
 	int64_t expectedVersion;
 };
 
-ACTOR static Future<Optional<VersionInfo>> getVersionInfo(Reference<IDatabase> db) {
-	state Reference<ITransaction> tr = db->createTransaction();
-	loop {
+static Future<Optional<VersionInfo>> getVersionInfo(Reference<IDatabase> db) {
+	Reference<ITransaction> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			state Version rv = wait(safeThreadFutureToFuture(tr->getReadVersion()));
-			state ThreadFuture<Optional<Value>> versionEpochValFuture = tr->get(versionEpochKey);
-			Optional<Value> versionEpochVal = wait(safeThreadFutureToFuture(versionEpochValFuture));
+			Version rv = co_await safeThreadFutureToFuture(tr->getReadVersion());
+			ThreadFuture<Optional<Value>> versionEpochValFuture = tr->get(versionEpochKey);
+			Optional<Value> versionEpochVal = co_await safeThreadFutureToFuture(versionEpochValFuture);
 			if (!versionEpochVal.present()) {
-				return Optional<VersionInfo>();
+				co_return Optional<VersionInfo>();
 			}
 			int64_t versionEpoch = BinaryReader::fromStringRef<int64_t>(versionEpochVal.get(), Unversioned());
 			int64_t expected = g_network->timer() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND - versionEpoch;
-			return VersionInfo{ rv, expected };
+			co_return VersionInfo{ rv, expected };
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR static Future<Optional<int64_t>> getVersionEpoch(Reference<ITransaction> tr) {
-	loop {
+static Future<Optional<int64_t>> getVersionEpoch(Reference<ITransaction> tr) {
+	while (true) {
+		Error err;
 		try {
-			state ThreadFuture<Optional<Value>> versionEpochValFuture = tr->get(versionEpochSpecialKey);
-			Optional<Value> versionEpochVal = wait(safeThreadFutureToFuture(versionEpochValFuture));
-			return versionEpochVal.present() ? boost::lexical_cast<int64_t>(versionEpochVal.get().toString())
-			                                 : Optional<int64_t>();
+			ThreadFuture<Optional<Value>> versionEpochValFuture = tr->get(versionEpochSpecialKey);
+			Optional<Value> versionEpochVal = co_await safeThreadFutureToFuture(versionEpochValFuture);
+			co_return versionEpochVal.present() ? boost::lexical_cast<int64_t>(versionEpochVal.get().toString())
+			                                    : Optional<int64_t>();
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens) {
+Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens) {
 	if (tokens.size() <= 3) {
-		state Reference<ITransaction> tr = db->createTransaction();
+		Reference<ITransaction> tr = db->createTransaction();
 		if (tokens.size() == 1) {
-			Optional<VersionInfo> versionInfo = wait(getVersionInfo(db));
+			Optional<VersionInfo> versionInfo = co_await getVersionInfo(db);
 			if (versionInfo.present()) {
 				int64_t diff = versionInfo.get().expectedVersion - versionInfo.get().version;
 				printf("Version:    %" PRId64 "\n", versionInfo.get().version);
@@ -85,75 +88,79 @@ ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx
 			} else {
 				printf("Version epoch is unset\n");
 			}
-			return true;
+			co_return true;
 		} else if (tokens.size() == 2 && tokencmp(tokens[1], "get")) {
-			Optional<int64_t> versionEpoch = wait(getVersionEpoch(db->createTransaction()));
+			Optional<int64_t> versionEpoch = co_await getVersionEpoch(db->createTransaction());
 			if (versionEpoch.present()) {
 				printf("Current version epoch is %" PRId64 "\n", versionEpoch.get());
 			} else {
 				printf("Version epoch is unset\n");
 			}
-			return true;
+			co_return true;
 		} else if (tokens.size() == 2 && tokencmp(tokens[1], "disable")) {
 			// Clearing the version epoch means versions will no longer attempt
 			// to advance at the same rate as the clock. The current version
 			// will remain unchanged.
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					Optional<int64_t> versionEpoch = wait(getVersionEpoch(db->createTransaction()));
+					Optional<int64_t> versionEpoch = co_await getVersionEpoch(db->createTransaction());
 					if (!versionEpoch.present()) {
-						return true;
+						co_return true;
 					} else {
 						tr->clear(versionEpochSpecialKey);
-						wait(safeThreadFutureToFuture(tr->commit()));
+						co_await safeThreadFutureToFuture(tr->commit());
 					}
 				} catch (Error& e) {
-					wait(safeThreadFutureToFuture(tr->onError(e)));
+					err = e;
 				}
+				co_await safeThreadFutureToFuture(tr->onError(err));
 			}
 		} else if ((tokens.size() == 2 && tokencmp(tokens[1], "enable")) ||
 		           (tokens.size() == 3 && tokencmp(tokens[1], "set"))) {
-			state int64_t v;
+			int64_t v;
 			if (tokens.size() == 3) {
 				int n = 0;
 				if (sscanf(tokens[2].toString().c_str(), "%" SCNd64 "%n", &v, &n) != 1 || n != tokens[2].size()) {
 					printUsage(tokens[0]);
-					return false;
+					co_return false;
 				}
 			} else {
 				v = 0; // default version epoch
 			}
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					Optional<int64_t> versionEpoch = wait(getVersionEpoch(tr));
+					Optional<int64_t> versionEpoch = co_await getVersionEpoch(tr);
 					if (!versionEpoch.present() || (versionEpoch.get() != v && tokens.size() == 3)) {
 						tr->set(versionEpochSpecialKey, BinaryWriter::toValue(v, Unversioned()));
-						wait(safeThreadFutureToFuture(tr->commit()));
+						co_await safeThreadFutureToFuture(tr->commit());
 					} else {
 						printf("Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target "
 						       "version\n");
-						return true;
+						co_return true;
 					}
 				} catch (Error& e) {
-					wait(safeThreadFutureToFuture(tr->onError(e)));
+					err = e;
 				}
+				co_await safeThreadFutureToFuture(tr->onError(err));
 			}
 		} else if (tokens.size() == 2 && tokencmp(tokens[1], "commit")) {
-			Optional<VersionInfo> versionInfo = wait(getVersionInfo(db));
+			Optional<VersionInfo> versionInfo = co_await getVersionInfo(db);
 			if (versionInfo.present()) {
-				wait(advanceVersion(cx, versionInfo.get().expectedVersion));
+				co_await advanceVersion(cx, versionInfo.get().expectedVersion);
 			} else {
 				printf("Must set the version epoch before committing it (see `versionepoch enable`)\n");
 			}
-			return true;
+			co_return true;
 		}
 	}
 
 	printUsage(tokens[0]);
-	return false;
+	co_return false;
 }
 
 CommandFactory versionEpochFactory(
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.cpp
similarity index 87%
rename from fdbcli/fdbcli.actor.cpp
rename to fdbcli/fdbcli.cpp
index f6184f13ca1..ff8312e430a 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.cpp
@@ -1,5 +1,5 @@
 /*
- * fdbcli.actor.cpp
+ * fdbcli.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,20 +26,19 @@
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/MultiVersionTransaction.h"
 #include "fdbclient/Status.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/StatusClient.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/IKnobCollection.h"
-#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/Knobs.h"
 #include "fdbclient/ClusterInterface.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/Schemas.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TagThrottle.h"
 #include "fdbclient/Tuple.h"
 
 #include "fdbclient/ThreadSafeTransaction.h"
@@ -51,13 +50,14 @@
 #include "flow/Platform.h"
 #include "flow/SystemMonitor.h"
 #include "flow/CodeProbe.h"
+#include "flow/CoroUtils.h"
 
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/ThreadHelper.actor.h"
 #include "SimpleOpt/SimpleOpt.h"
 
 #include "fdbcli/FlowLineNoise.h"
-#include "fdbcli/fdbcli.actor.h"
+#include "fdbcli/fdbcli.h"
 
 #include <cinttypes>
 #include <cstdio>
@@ -72,8 +72,6 @@
 #include "fdbclient/versions.h"
 #include "fdbclient/BuildFlags.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 /*
  * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi
  * instance (e.g. from ThreadSafeApi)
@@ -248,8 +246,8 @@ class FdbOptions {
 		std::map<typename T::Option, Optional<Standalone<StringRef>>> options;
 		std::map<std::string, typename T::Option> legalOptions;
 
-		OptionGroup() {}
-		OptionGroup(OptionGroup<T>& base)
+		OptionGroup() = default;
+		explicit(false) OptionGroup(OptionGroup<T>& base)
 		  : options(base.options.begin(), base.options.end()), legalOptions(base.legalOptions) {}
 
 		// Enable or disable an option. Returns true if option value changed
@@ -331,7 +329,7 @@ class FdbOptions {
 			transactionOptions.legalOptions[itr->second.name] = itr->first;
 	}
 
-	FdbOptions(FdbOptions& base) : transactionOptions(base.transactionOptions) {}
+	explicit(false) FdbOptions(FdbOptions& base) = default;
 };
 
 static std::string formatStringRef(StringRef item, bool fullEscaping = false) {
@@ -353,6 +351,7 @@ static std::string formatStringRef(StringRef item, bool fullEscaping = false) {
 	return ret;
 }
 
+// NOLINTBEGIN(bugprone-use-after-move): ignore clang-tidy false positives in parseLine's token buffer handling.
 static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& err, bool& partial) {
 	err = false;
 	partial = false;
@@ -470,6 +469,7 @@ static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& er
 
 	return ret;
 }
+// NOLINTEND(bugprone-use-after-move)
 
 static void printProgramUsage(const char* name) {
 	printf("FoundationDB CLI " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"
@@ -610,7 +610,7 @@ void printBuildInformation() {
 void printHelpOverview() {
 	printf("\nList of commands:\n\n");
 	for (const auto& [command, help] : helpMap) {
-		if (help.short_desc.size())
+		if (!help.short_desc.empty())
 			printf(" %s:\n      %s\n", command.c_str(), help.short_desc.c_str());
 	}
 	printf("\nFor information on a specific command, type `help <command>'.");
@@ -620,7 +620,7 @@ void printHelpOverview() {
 
 void printHelp(StringRef command) {
 	auto i = helpMap.find(command.toString());
-	if (i != helpMap.end() && i->second.short_desc.size()) {
+	if (i != helpMap.end() && !i->second.short_desc.empty()) {
 		printf("\n%s\n\n", i->second.usage.c_str());
 		auto cstr = i->second.short_desc.c_str();
 		printf("%c%s.\n", toupper(cstr[0]), cstr + 1);
@@ -653,29 +653,28 @@ int printStatusFromJSON(std::string const& jsonFileName) {
 	}
 }
 
-ACTOR Future<Void> timeWarning(double when, const char* msg) {
-	wait(delay(when));
+Future<Void> timeWarning(double when, const char* msg) {
+	co_await delay(when);
 	fputs(msg, stderr);
-
-	return Void();
 }
 
-ACTOR Future<Void> checkStatus(Future<Void> f,
-                               Reference<IDatabase> db,
-                               Database localDb,
-                               bool displayDatabaseAvailable = true) {
-	wait(f);
-	state Reference<ITransaction> tr = db->createTransaction();
-	state StatusObject s;
+Future<Void> checkStatus(Future<Void> _f,
+                         Reference<IDatabase> db,
+                         Database localDb,
+                         bool displayDatabaseAvailable = true) {
+	auto f = std::move(_f);
+	co_await f;
+	Reference<ITransaction> tr = db->createTransaction();
+	StatusObject s;
 	if (!tr->isValid()) {
-		StatusObject _s = wait(StatusClient::statusFetcher(localDb));
+		StatusObject _s = co_await StatusClient::statusFetcher(localDb);
 		s = _s;
 	} else {
-		state ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
-		Optional<Value> statusValue = wait(safeThreadFutureToFuture(statusValueF));
+		ThreadFuture<Optional<Value>> statusValueF = tr->get("\xff\xff/status/json"_sr);
+		Optional<Value> statusValue = co_await safeThreadFutureToFuture(statusValueF);
 		if (!statusValue.present()) {
 			fprintf(stderr, "ERROR: Failed to get status json from the cluster\n");
-			return Void();
+			co_return;
 		}
 		json_spirit::mValue mv;
 		json_spirit::read_string(statusValue.get().toString(), mv);
@@ -684,36 +683,32 @@ ACTOR Future<Void> checkStatus(Future<Void> f,
 	printf("\n");
 	printStatus(s, StatusClient::MINIMAL, displayDatabaseAvailable);
 	printf("\n");
-	return Void();
 }
 
-ACTOR template <class T>
-Future<T> makeInterruptable(Future<T> f) {
+template <class T>
+Future<T> makeInterruptable(Future<T> f, ExplicitVoid = {}) {
 	Future<Void> interrupt = LineNoise::onKeyboardInterrupt();
-	choose {
-		when(T t = wait(f)) {
-			return t;
-		}
-		when(wait(interrupt)) {
-			f.cancel();
-			throw operation_cancelled();
-		}
+	auto choice = co_await race(f, interrupt);
+	if (choice.index() == 0) {
+		co_return std::get<0>(std::move(choice));
 	}
+	ASSERT_EQ(choice.index(), 1);
+	f.cancel();
+	throw operation_cancelled();
 }
 
-ACTOR Future<Void> commitTransaction(Reference<ITransaction> tr) {
-	wait(makeInterruptable(safeThreadFutureToFuture(tr->commit())));
+Future<Void> commitTransaction(Reference<ITransaction> tr) {
+	co_await makeInterruptable(safeThreadFutureToFuture(tr->commit()));
 	auto ver = tr->getCommittedVersion();
 	if (ver != invalidVersion)
 		fmt::print("Committed ({})\n", ver);
 	else
 		fmt::print("Nothing to commit\n");
-	return Void();
 }
 
-ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
-	state Standalone<StringRef> snapCmd;
-	state UID snapUID = deterministicRandom()->randomUniqueID();
+Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
+	Standalone<StringRef> snapCmd;
+	UID snapUID = deterministicRandom()->randomUniqueID();
 	for (int i = 1; i < tokens.size(); i++) {
 		snapCmd = snapCmd.withSuffix(tokens[i]);
 		if (i != tokens.size() - 1) {
@@ -721,7 +716,7 @@ ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
 		}
 	}
 	try {
-		wait(makeInterruptable(mgmtSnapCreate(db, snapCmd, snapUID)));
+		co_await makeInterruptable(mgmtSnapCreate(db, snapCmd, snapUID));
 		printf("Snapshot command succeeded with UID %s\n", snapUID.toString().c_str());
 	} catch (Error& e) {
 		fprintf(stderr,
@@ -730,9 +725,9 @@ ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
 		        e.code(),
 		        e.what(),
 		        snapUID.toString().c_str());
-		return true;
+		co_return true;
 	}
-	return false;
+	co_return false;
 }
 
 // TODO: Update the function to get rid of the Database after refactoring
@@ -836,7 +831,7 @@ void fdbcliCompCmd(std::string const& text, std::vector<std::string>& lc) {
 	std::string base_input = text;
 
 	// If there is a token and the input does not end in a space
-	if (count && text.size() > 0 && text[text.size() - 1] != ' ') {
+	if (count && !text.empty() && text[text.size() - 1] != ' ') {
 		count--; // Ignore the last token for purposes of later code
 		ntext = tokens.back().toString();
 		base_input = whole_line.substr(0, whole_line.rfind(ntext));
@@ -877,6 +872,8 @@ void LogCommand(std::string line, UID randomID, std::string errMsg) {
 }
 
 struct CLIOptions {
+	static constexpr int DEFERRED_EXIT_CODE = -2;
+
 	std::string program_name;
 	int exit_code = -1;
 
@@ -889,6 +886,7 @@ struct CLIOptions {
 	std::string logGroup;
 	int exit_timeout = 0;
 	Optional<std::string> exec;
+	Optional<std::string> statusJsonFile;
 	bool initialStatusCheck = true;
 	bool cliHints = true;
 	bool cliHistory = true;
@@ -920,12 +918,15 @@ struct CLIOptions {
 
 		while (args.Next()) {
 			int ec = processArg(args);
+			if (ec == DEFERRED_EXIT_CODE) {
+				return;
+			}
 			if (ec != -1) {
 				exit_code = ec;
 				return;
 			}
 		}
-		if (exit_timeout && !exec.present()) {
+		if (exit_timeout && !exec.present() && !statusJsonFile.present()) {
 			fprintf(stderr, "ERROR: --timeout may only be specified with --exec\n");
 			exit_code = FDB_EXIT_ERROR;
 			return;
@@ -933,10 +934,10 @@ struct CLIOptions {
 	}
 
 	void setupKnobs() {
-		IKnobCollection::setupKnobs(knobs);
+		setupClientKnobs(knobs);
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
+		initializeClientKnobs(Randomize::False, IsSimulated::False);
 	}
 
 	int processArg(CSimpleOpt& args) {
@@ -1031,7 +1032,8 @@ struct CLIOptions {
 			printProgramUsage(program_name.c_str());
 			return 0;
 		case OPT_STATUS_FROM_JSON:
-			return printStatusFromJSON(args.OptionArg());
+			statusJsonFile = args.OptionArg();
+			return DEFERRED_EXIT_CODE;
 		case OPT_TRACE_FORMAT:
 			if (!validateTraceFormat(args.OptionArg())) {
 				fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg());
@@ -1064,36 +1066,36 @@ struct CLIOptions {
 	}
 };
 
-ACTOR template <class T>
-Future<T> stopNetworkAfter(Future<T> what) {
+template <class T>
+Future<T> stopNetworkAfter(Future<T> what, ExplicitVoid = {}) {
 	try {
-		T t = wait(what);
+		T t = co_await what;
 		API->stopNetwork();
-		return t;
+		co_return t;
 	} catch (...) {
 		API->stopNetwork();
 		throw;
 	}
 }
 
-ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterConnectionFile> ccf) {
-	state LineNoise& linenoise = *plinenoise;
-	state bool intrans = false;
+Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterConnectionFile> ccf) {
+	LineNoise& linenoise = *plinenoise;
+	bool intrans = false;
 
-	state Database localDb;
-	state Reference<IDatabase> db;
-	state Reference<ITransaction> tr;
-	state Transaction trx;
+	Database localDb;
+	Reference<IDatabase> db;
+	Reference<ITransaction> tr;
+	Transaction trx;
 
-	state bool writeMode = false;
+	bool writeMode = false;
 
-	state std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
-	state std::map<std::string, StorageServerInterface> storage_interface;
+	std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
+	std::map<std::string, StorageServerInterface> storage_interface;
 
-	state FdbOptions globalOptions;
-	state FdbOptions activeOptions;
+	FdbOptions globalOptions;
+	FdbOptions activeOptions;
 
-	state FdbOptions* options = &globalOptions;
+	FdbOptions* options = &globalOptions;
 
 	// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are
 	// logged. This thread will eventually run the network, so call it now.
@@ -1108,7 +1110,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 	} catch (Error& e) {
 		fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code());
 		printf("Unable to connect to cluster from `%s'\n", ccf->getLocation().c_str());
-		return 1;
+		co_return 1;
 	}
 
 	if (opt.trace) {
@@ -1129,18 +1131,19 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 	// when using external clients, it will throw cluster_version_changed for the first time establish the connection to
 	// the cluster. Thus, we catch it by doing a get version request to establish the connection
 	// The 3.0 timeout is a guard to avoid waiting forever when the cli cannot talk to any coordinators
-	loop {
+	while (true) {
+		Optional<Error> err;
 		try {
 			getTransaction(db, tr, options, intrans);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			wait(delay(3.0) || success(safeThreadFutureToFuture(tr->getReadVersion())));
+			co_await (delay(3.0) || success(safeThreadFutureToFuture(tr->getReadVersion())));
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_operation_cancelled) {
 				throw;
 			}
 			if (e.code() == error_code_cluster_version_changed) {
-				wait(safeThreadFutureToFuture(tr->onError(e)));
+				err = e;
 			} else {
 				// unexpected errors
 				fprintf(stderr, "ERROR: unexpected error %d while initializing the multiversion database\n", e.code());
@@ -1148,12 +1151,15 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				break;
 			}
 		}
+		if (err.present()) {
+			co_await safeThreadFutureToFuture(tr->onError(err.get()));
+		}
 	}
 
 	if (!opt.exec.present()) {
 		if (opt.initialStatusCheck) {
 			Future<Void> checkStatusF = checkStatus(Void(), db, localDb);
-			wait(makeInterruptable(success(checkStatusF)));
+			co_await makeInterruptable(success(checkStatusF));
 		} else {
 			printf("\n");
 		}
@@ -1162,47 +1168,47 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 		validOptions = options->getValidOptions();
 	}
 
-	state bool is_error = false;
-	state Future<Void> warn;
-	loop {
+	bool is_error = false;
+	Future<Void> warn;
+	while (true) {
 		if (warn.isValid())
 			warn.cancel();
 
-		state std::string line;
+		std::string line;
 
 		if (opt.exec.present()) {
 			line = opt.exec.get();
 		} else {
-			Optional<std::string> rawline = wait(linenoise.read("fdb> "));
+			Optional<std::string> rawline = co_await linenoise.read("fdb> ");
 			if (!rawline.present()) {
 				printf("\n");
-				return 0;
+				co_return 0;
 			}
 			line = rawline.get();
 
-			if (!line.size())
+			if (line.empty())
 				continue;
 
 			// Don't put dangerous commands in the command history
 			if (line.find("writemode") == std::string::npos && line.find("expensive_data_check") == std::string::npos &&
-			    line.find("unlock") == std::string::npos && line.find("blobrange") == std::string::npos)
+			    line.find("unlock") == std::string::npos)
 				linenoise.historyAdd(line);
 		}
 
 		warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db, localDb);
 
 		try {
-			state UID randomID = deterministicRandom()->randomUniqueID();
+			UID randomID = deterministicRandom()->randomUniqueID();
 			TraceEvent(SevInfo, "CLICommandLog", randomID).detail("Command", line);
 
 			bool malformed, partial;
-			state std::vector<std::vector<StringRef>> parsed = parseLine(line, malformed, partial);
+			std::vector<std::vector<StringRef>> parsed = parseLine(line, malformed, partial);
 			if (malformed)
 				LogCommand(line, randomID, "ERROR: malformed escape sequence");
 			if (partial)
 				LogCommand(line, randomID, "ERROR: unterminated quote");
 			if (malformed || partial) {
-				if (parsed.size() > 0) {
+				if (!parsed.empty()) {
 					// Denote via a special token that the command was a parse failure.
 					auto& last_command = parsed.back();
 					last_command.insert(last_command.begin(),
@@ -1210,19 +1216,19 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 			}
 
-			state bool multi = parsed.size() > 1;
+			bool multi = parsed.size() > 1;
 			is_error = false;
 
-			state std::vector<std::vector<StringRef>>::iterator iter;
+			std::vector<std::vector<StringRef>>::iterator iter;
 			for (iter = parsed.begin(); iter != parsed.end(); ++iter) {
-				state std::vector<StringRef> tokens = *iter;
+				std::vector<StringRef> tokens = *iter;
 
 				if (is_error) {
 					printf("WARNING: the previous command failed, the remaining commands will not be executed.\n");
 					break;
 				}
 
-				if (!tokens.size())
+				if (tokens.empty())
 					continue;
 
 				if (tokencmp(tokens[0], "parse_error")) {
@@ -1244,14 +1250,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 					printf("\n");
 				}
 
-				if (!helpMap.count(tokens[0].toString()) && !hiddenCommands.count(tokens[0].toString())) {
+				if (!helpMap.contains(tokens[0].toString()) && !hiddenCommands.contains(tokens[0].toString())) {
 					fprintf(stderr, "ERROR: Unknown command `%s'. Try `help'?\n", formatStringRef(tokens[0]).c_str());
 					is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "exit") || tokencmp(tokens[0], "quit")) {
-					return 0;
+					co_return 0;
 				}
 
 				if (tokencmp(tokens[0], "help")) {
@@ -1293,13 +1299,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "waitconnected")) {
-					wait(makeInterruptable(localDb->onConnected()));
+					co_await makeInterruptable(localDb->onConnected());
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "waitopen")) {
-					wait(makeInterruptable(
-					    success(safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion()))));
+					co_await makeInterruptable(
+					    success(safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())));
 					continue;
 				}
 
@@ -1314,7 +1320,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 							printUsage(tokens[0]);
 							is_error = true;
 						} else {
-							wait(delay(v));
+							co_await delay(v);
 						}
 					}
 					continue;
@@ -1324,19 +1330,20 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 					// Warn at 7 seconds since status will spend as long as 5 seconds trying to read/write from the
 					// database
 					warn = timeWarning(7.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n");
-					bool _result = wait(makeInterruptable(statusCommandActor(db, localDb, tokens, opt.exec.present())));
+					bool _result =
+					    co_await makeInterruptable(statusCommandActor(db, localDb, tokens, opt.exec.present()));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "triggerddteaminfolog")) {
-					wait(success(makeInterruptable(triggerddteaminfologCommandActor(db))));
+					co_await makeInterruptable(triggerddteaminfologCommandActor(db));
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "tssq")) {
-					bool _result = wait(makeInterruptable(tssqCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(tssqCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1344,7 +1351,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 
 				if (tokencmp(tokens[0], "configure")) {
 					bool _result =
-					    wait(makeInterruptable(configureCommandActor(db, localDb, tokens, &linenoise, warn)));
+					    co_await makeInterruptable(configureCommandActor(db, localDb, tokens, &linenoise, warn));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1353,8 +1360,8 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				if (tokencmp(tokens[0], "fileconfigure")) {
 					if (tokens.size() == 2 ||
 					    (tokens.size() == 3 && (tokens[1] == "new"_sr || tokens[1] == "FORCE"_sr))) {
-						bool _result = wait(makeInterruptable(fileConfigureCommandActor(
-						    db, tokens.back().toString(), tokens[1] == "new"_sr, tokens[1] == "FORCE"_sr)));
+						bool _result = co_await makeInterruptable(fileConfigureCommandActor(
+						    db, tokens.back().toString(), tokens[1] == "new"_sr, tokens[1] == "FORCE"_sr));
 						if (!_result)
 							is_error = true;
 					} else {
@@ -1365,35 +1372,35 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "coordinators")) {
-					bool _result = wait(makeInterruptable(coordinatorsCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(coordinatorsCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "exclude")) {
-					bool _result = wait(makeInterruptable(excludeCommandActor(db, tokens, warn)));
+					bool _result = co_await makeInterruptable(excludeCommandActor(db, tokens, warn));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "include")) {
-					bool _result = wait(makeInterruptable(includeCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(includeCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "snapshot")) {
-					bool _result = wait(makeInterruptable(snapshotCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(snapshotCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "lock")) {
-					bool _result = wait(makeInterruptable(lockCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(lockCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1405,18 +1412,18 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						printUsage(tokens[0]);
 						is_error = true;
 					} else {
-						state std::string passPhrase = deterministicRandom()->randomAlphaNumeric(10);
+						std::string passPhrase = deterministicRandom()->randomAlphaNumeric(10);
 						warn.cancel(); // don't warn while waiting on user input
 						printf("Unlocking the database is a potentially dangerous operation.\n");
 						printf("%s\n", passPhrase.c_str());
 						fflush(stdout);
-						Optional<std::string> input =
-						    wait(linenoise.read(format("Repeat the above passphrase if you would like to proceed:")));
+						Optional<std::string> input = co_await linenoise.read(
+						    format("Repeat the above passphrase if you would like to proceed:"));
 						warn =
 						    checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db, localDb);
 						if (input.present() && input.get() == passPhrase) {
 							UID unlockUID = UID::fromString(tokens[1].toString());
-							bool _result = wait(makeInterruptable(unlockDatabaseActor(db, unlockUID)));
+							bool _result = co_await makeInterruptable(unlockDatabaseActor(db, unlockUID));
 							if (!_result)
 								is_error = true;
 						} else {
@@ -1428,7 +1435,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "setclass")) {
-					bool _result = wait(makeInterruptable(setClassCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(setClassCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1461,7 +1468,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 					} else {
 						warn =
 						    checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db, localDb);
-						wait(commitTransaction(tr));
+						co_await commitTransaction(tr);
 						intrans = false;
 						options = &globalOptions;
 					}
@@ -1506,9 +1513,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						printUsage(tokens[0]);
 						is_error = true;
 					} else {
-						state ThreadFuture<Optional<Value>> valueF =
-						    getTransaction(db, tr, options, intrans)->get(tokens[1]);
-						Optional<Standalone<StringRef>> v = wait(makeInterruptable(safeThreadFutureToFuture(valueF)));
+						ThreadFuture<Optional<Value>> valueF = getTransaction(db, tr, options, intrans)->get(tokens[1]);
+						Optional<Standalone<StringRef>> v =
+						    co_await makeInterruptable(safeThreadFutureToFuture(valueF));
 
 						if (v.present())
 							printf("`%s' is `%s'\n", printable(tokens[1]).c_str(), printable(v.get()).c_str());
@@ -1519,23 +1526,23 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "getlocation")) {
-					bool _result = wait(makeInterruptable(getLocationCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(getLocationCommandActor(localDb, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "getall")) {
-					Version _v = wait(makeInterruptable(
-					    safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())));
-					bool _result = wait(makeInterruptable(getallCommandActor(localDb, tokens, _v)));
+					Version _v = co_await makeInterruptable(
+					    safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion()));
+					bool _result = co_await makeInterruptable(getallCommandActor(localDb, tokens, _v));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "checkall")) {
-					bool _result = wait(makeInterruptable(checkallCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(checkallCommandActor(localDb, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1546,22 +1553,22 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						printUsage(tokens[0]);
 						is_error = true;
 					} else {
-						Version v = wait(makeInterruptable(
-						    safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion())));
+						Version v = co_await makeInterruptable(
+						    safeThreadFutureToFuture(getTransaction(db, tr, options, intrans)->getReadVersion()));
 						fmt::print("{}\n", v);
 					}
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "advanceversion")) {
-					bool _result = wait(makeInterruptable(advanceVersionCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(advanceVersionCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "versionepoch")) {
-					bool _result = wait(makeInterruptable(versionEpochCommandActor(db, localDb, tokens)));
+					bool _result = co_await makeInterruptable(versionEpochCommandActor(db, localDb, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1569,7 +1576,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 
 				if (tokencmp(tokens[0], "kill")) {
 					getTransaction(db, tr, options, intrans);
-					bool _result = wait(makeInterruptable(killCommandActor(db, tr, tokens, &address_interface)));
+					bool _result = co_await makeInterruptable(killCommandActor(db, tr, tokens, &address_interface));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1577,14 +1584,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 
 				if (tokencmp(tokens[0], "suspend")) {
 					getTransaction(db, tr, options, intrans);
-					bool _result = wait(makeInterruptable(suspendCommandActor(db, tr, tokens, &address_interface)));
+					bool _result = co_await makeInterruptable(suspendCommandActor(db, tr, tokens, &address_interface));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "audit_storage")) {
-					UID auditId = wait(makeInterruptable(auditStorageCommandActor(ccf, tokens)));
+					UID auditId = co_await makeInterruptable(auditStorageCommandActor(ccf, tokens));
 					if (!auditId.isValid()) {
 						is_error = true;
 					} else {
@@ -1596,7 +1603,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "get_audit_status")) {
-					bool _result = wait(makeInterruptable(getAuditStatusCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(getAuditStatusCommandActor(localDb, tokens));
 					if (!_result) {
 						is_error = true;
 					}
@@ -1604,7 +1611,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "location_metadata")) {
-					bool _result = wait(makeInterruptable(locationMetadataCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(locationMetadataCommandActor(localDb, tokens));
 					if (!_result) {
 						is_error = true;
 					}
@@ -1612,7 +1619,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "bulkload")) {
-					UID taskId = wait(makeInterruptable(bulkLoadCommandActor(localDb, tokens)));
+					UID taskId = co_await makeInterruptable(bulkLoadCommandActor(localDb, tokens));
 					if (taskId.isValid()) {
 						printf("Received Job ID: %s\n", taskId.toString().c_str());
 					}
@@ -1620,7 +1627,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "bulkdump")) {
-					UID jobId = wait(makeInterruptable(bulkDumpCommandActor(localDb, tokens)));
+					UID jobId = co_await makeInterruptable(bulkDumpCommandActor(localDb, tokens));
 					if (jobId.isValid()) {
 						printf("Received Job ID: %s\n", jobId.toString().c_str());
 					}
@@ -1628,14 +1635,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "force_recovery_with_data_loss")) {
-					bool _result = wait(makeInterruptable(forceRecoveryWithDataLossCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(forceRecoveryWithDataLossCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "maintenance")) {
-					bool _result = wait(makeInterruptable(maintenanceCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(maintenanceCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1643,14 +1650,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 
 				if (tokencmp(tokens[0], "consistencycheck")) {
 					getTransaction(db, tr, options, intrans);
-					bool _result = wait(makeInterruptable(consistencyCheckCommandActor(tr, tokens, intrans)));
+					bool _result = co_await makeInterruptable(consistencyCheckCommandActor(tr, tokens, intrans));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "consistencyscan")) {
-					bool _result = wait(makeInterruptable(consistencyScanCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(consistencyScanCommandActor(localDb, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1658,7 +1665,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 
 				if (tokencmp(tokens[0], "profile")) {
 					getTransaction(db, tr, options, intrans);
-					bool _result = wait(makeInterruptable(profileCommandActor(localDb, tr, tokens, intrans)));
+					bool _result = co_await makeInterruptable(profileCommandActor(localDb, tr, tokens, intrans));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1667,7 +1674,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				if (tokencmp(tokens[0], "expensive_data_check")) {
 					getTransaction(db, tr, options, intrans);
 					bool _result =
-					    wait(makeInterruptable(expensiveDataCheckCommandActor(db, tr, tokens, &address_interface)));
+					    co_await makeInterruptable(expensiveDataCheckCommandActor(db, tr, tokens, &address_interface));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1679,7 +1686,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						printUsage(tokens[0]);
 						is_error = true;
 					} else {
-						state int limit;
+						int limit;
 						bool valid = true;
 						if (tokens.size() == 4) {
 							// INT_MAX is 10 digits; rather than
@@ -1714,7 +1721,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						Standalone<StringRef> endKey;
 						if (tokens.size() >= 3) {
 							endKey = tokens[2];
-						} else if (tokens[1].size() == 0) {
+						} else if (tokens[1].empty()) {
 							endKey = normalKeys.end;
 						} else if (tokens[1] == systemKeys.begin) {
 							endKey = systemKeys.end;
@@ -1725,8 +1732,8 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						}
 
 						getTransaction(db, tr, options, intrans);
-						state ThreadFuture<RangeResult> kvsF = tr->getRange(KeyRangeRef(tokens[1], endKey), limit);
-						RangeResult kvs = wait(makeInterruptable(safeThreadFutureToFuture(kvsF)));
+						ThreadFuture<RangeResult> kvsF = tr->getRange(KeyRangeRef(tokens[1], endKey), limit);
+						RangeResult kvs = co_await makeInterruptable(safeThreadFutureToFuture(kvsF));
 
 						printf("\nRange limited to %d keys\n", limit);
 						for (auto iter = kvs.begin(); iter < kvs.end(); iter++) {
@@ -1773,7 +1780,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						tr->set(tokens[1], tokens[2]);
 
 						if (!intrans) {
-							wait(commitTransaction(tr));
+							co_await commitTransaction(tr);
 						}
 					}
 					continue;
@@ -1794,7 +1801,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						tr->clear(tokens[1]);
 
 						if (!intrans) {
-							wait(commitTransaction(tr));
+							co_await commitTransaction(tr);
 						}
 					}
 					continue;
@@ -1815,14 +1822,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 						tr->clear(KeyRangeRef(tokens[1], tokens[2]));
 
 						if (!intrans) {
-							wait(commitTransaction(tr));
+							co_await commitTransaction(tr);
 						}
 					}
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "datadistribution")) {
-					bool _result = wait(makeInterruptable(dataDistributionCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(dataDistributionCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1887,14 +1894,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "throttle")) {
-					bool _result = wait(makeInterruptable(throttleCommandActor(db, tokens)));
+					bool _result = co_await makeInterruptable(throttleCommandActor(db, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
 				}
 
 				if (tokencmp(tokens[0], "idempotencyids")) {
-					bool _result = wait(makeInterruptable(idempotencyIdsCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(idempotencyIdsCommandActor(localDb, tokens));
 					if (!_result) {
 						is_error = true;
 					}
@@ -1903,7 +1910,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 
 				if (tokencmp(tokens[0], "hotrange")) {
 					bool _result =
-					    wait(makeInterruptable(hotRangeCommandActor(localDb, db, tokens, &storage_interface)));
+					    co_await makeInterruptable(hotRangeCommandActor(localDb, db, tokens, &storage_interface));
 					if (!_result) {
 						is_error = true;
 					}
@@ -1911,7 +1918,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 				}
 
 				if (tokencmp(tokens[0], "rangeconfig")) {
-					bool _result = wait(makeInterruptable(rangeConfigCommandActor(localDb, tokens)));
+					bool _result = co_await makeInterruptable(rangeConfigCommandActor(localDb, tokens));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1938,13 +1945,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
 		}
 
 		if (opt.exec.present()) {
-			return is_error ? 1 : 0;
+			co_return is_error ? 1 : 0;
 		}
 	}
 }
 
-ACTOR Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
-	state LineNoise linenoise(
+Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
+	LineNoise linenoise(
 	    [](std::string const& line, std::vector<std::string>& completions) { fdbcliCompCmd(line, completions); },
 	    [enabled = opt.cliHints](std::string const& line) -> LineNoise::Hint {
 		    if (!enabled) {
@@ -1955,7 +1962,7 @@ ACTOR Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
 		    bool partial = false;
 		    std::string linecopy = line;
 		    std::vector<std::vector<StringRef>> parsed = parseLine(linecopy, error, partial);
-		    if (parsed.size() == 0 || parsed.back().size() == 0)
+		    if (parsed.empty() || parsed.back().empty())
 			    return LineNoise::Hint();
 		    StringRef command = parsed.back().front();
 		    int finishedParameters = parsed.back().size() + error;
@@ -1996,7 +2003,7 @@ ACTOR Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
 	    1000,
 	    false);
 
-	state std::string historyFilename;
+	std::string historyFilename;
 	if (opt.cliHistory) {
 		try {
 			historyFilename = joinPath(getUserHomeDirectory(), ".fdbcli_history");
@@ -2008,7 +2015,7 @@ ACTOR Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
 			    .GetLastError();
 		}
 	}
-	state int result = wait(cli(opt, &linenoise, ccf));
+	int result = co_await cli(opt, &linenoise, ccf);
 
 	if (opt.cliHistory && !historyFilename.empty()) {
 		try {
@@ -2021,13 +2028,12 @@ ACTOR Future<int> runCli(CLIOptions opt, Reference<ClusterConnectionFile> ccf) {
 		}
 	}
 
-	return result;
+	co_return result;
 }
 
-ACTOR Future<Void> timeExit(double duration) {
-	wait(delay(duration));
+Future<Void> timeExit(double duration) {
+	co_await delay(duration);
 	fprintf(stderr, "Specified timeout reached -- exiting...\n");
-	return Void();
 }
 
 const char* checkTlsConfigAgainstCoordAddrs(const ClusterConnectionString& ccs) {
@@ -2104,7 +2110,7 @@ int main(int argc, char** argv) {
 	initHelp();
 
 	// deferred TLS options
-	if (opt.tlsCertPath.size()) {
+	if (!opt.tlsCertPath.empty()) {
 		try {
 			setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, opt.tlsCertPath);
 		} catch (Error& e) {
@@ -2113,7 +2119,7 @@ int main(int argc, char** argv) {
 		}
 	}
 
-	if (opt.tlsCAPath.size()) {
+	if (!opt.tlsCAPath.empty()) {
 		try {
 			setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, opt.tlsCAPath);
 		} catch (Error& e) {
@@ -2121,9 +2127,9 @@ int main(int argc, char** argv) {
 			return 1;
 		}
 	}
-	if (opt.tlsKeyPath.size()) {
+	if (!opt.tlsKeyPath.empty()) {
 		try {
-			if (opt.tlsPassword.size())
+			if (!opt.tlsPassword.empty())
 				setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, opt.tlsPassword);
 
 			setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, opt.tlsKeyPath);
@@ -2132,7 +2138,7 @@ int main(int argc, char** argv) {
 			return 1;
 		}
 	}
-	if (opt.tlsVerifyPeers.size()) {
+	if (!opt.tlsVerifyPeers.empty()) {
 		try {
 			setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, opt.tlsVerifyPeers);
 		} catch (Error& e) {
@@ -2158,6 +2164,21 @@ int main(int argc, char** argv) {
 		return 1;
 	}
 
+	if (opt.statusJsonFile.present()) {
+		try {
+			API->selectApiVersion(opt.apiVersion);
+			if (opt.useFutureProtocolVersion) {
+				API->useFutureProtocolVersion();
+			}
+			API->setupNetwork();
+			opt.setupKnobs();
+			return printStatusFromJSON(opt.statusJsonFile.get());
+		} catch (Error& e) {
+			fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code());
+			return 1;
+		}
+	}
+
 	if (opt.debugTLS) {
 		// Backdoor into NativeAPI's tlsConfig, which is where the above network option settings ended up.
 		extern TLSConfig tlsConfig;
diff --git a/fdbcli/include/fdbcli/fdbcli.actor.h b/fdbcli/include/fdbcli/fdbcli.actor.h
deleted file mode 100644
index 0627ef14920..00000000000
--- a/fdbcli/include/fdbcli/fdbcli.actor.h
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * fdbcli.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated
-// version of this file.  In intellisense use the source version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLI_FDBCLI_ACTOR_G_H)
-#define FDBCLI_FDBCLI_ACTOR_G_H
-#include "fdbcli/fdbcli.actor.g.h"
-#elif !defined(FDBCLI_FDBCLI_ACTOR_H)
-#define FDBCLI_FDBCLI_ACTOR_H
-
-#include "fdbcli/FlowLineNoise.h"
-
-#include "fdbclient/CoordinationInterface.h"
-#include "fdbclient/IClientApi.h"
-#include "fdbclient/StatusClient.h"
-#include "fdbclient/StorageServerInterface.h"
-#include "flow/Arena.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-namespace fdb_cli {
-
-constexpr char msgTypeKey[] = "type";
-constexpr char msgClusterKey[] = "cluster";
-constexpr char msgClusterTypeKey[] = "cluster_type";
-constexpr char msgDataClustersKey[] = "data_clusters";
-constexpr char msgCapacityKey[] = "capacity";
-constexpr char msgAllocatedKey[] = "allocated";
-constexpr char msgErrorKey[] = "error";
-
-struct CommandHelp {
-	std::string usage;
-	std::string short_desc;
-	std::string long_desc;
-	CommandHelp() {}
-	CommandHelp(const char* usage, const char* short_desc, const char* long_desc)
-	  : usage(usage), short_desc(short_desc), long_desc(long_desc) {}
-};
-
-void arrayGenerator(const char* text, const char* line, const char** options, std::vector<std::string>& lc);
-
-struct CommandFactory {
-	typedef void (*CompletionGeneratorFunc)(const char* text,
-	                                        const char* line,
-	                                        std::vector<std::string>& lc,
-	                                        std::vector<StringRef> const& tokens);
-
-	typedef std::vector<const char*> (*HintGeneratorFunc)(std::vector<StringRef> const& tokens, bool inArgument);
-
-	CommandFactory(const char* name,
-	               CommandHelp help,
-	               CompletionGeneratorFunc completionFunc = nullptr,
-	               HintGeneratorFunc hintFunc = nullptr) {
-		commands()[name] = help;
-		if (completionFunc) {
-			completionGenerators()[name] = completionFunc;
-		}
-		if (hintFunc) {
-			hintGenerators()[name] = hintFunc;
-		}
-	}
-	CommandFactory(const char* name) { hiddenCommands().insert(name); }
-	static std::map<std::string, CommandHelp>& commands() {
-		static std::map<std::string, CommandHelp> helpMap;
-		return helpMap;
-	}
-	static std::set<std::string>& hiddenCommands() {
-		static std::set<std::string> commands;
-		return commands;
-	}
-	static std::map<std::string, CompletionGeneratorFunc>& completionGenerators() {
-		static std::map<std::string, CompletionGeneratorFunc> completionMap;
-		return completionMap;
-	}
-	static std::map<std::string, HintGeneratorFunc>& hintGenerators() {
-		static std::map<std::string, HintGeneratorFunc> hintMap;
-		return hintMap;
-	}
-};
-
-// Special keys used by fdbcli commands
-
-extern const KeyRef advanceVersionSpecialKey;
-extern const KeyRef consistencyCheckSpecialKey;
-extern const KeyRef clusterDescriptionSpecialKey;
-extern const KeyRef coordinatorsAutoSpecialKey;
-extern const KeyRef coordinatorsProcessSpecialKey;
-extern const KeyRef ddModeSpecialKey;
-extern const KeyRef ddIgnoreRebalanceSpecialKey;
-
-extern const KeyRangeRef excludedServersSpecialKeyRange;
-extern const KeyRangeRef failedServersSpecialKeyRange;
-extern const KeyRangeRef excludedLocalitySpecialKeyRange;
-extern const KeyRangeRef failedLocalitySpecialKeyRange;
-extern const KeyRef excludedForceOptionSpecialKey;
-extern const KeyRef failedForceOptionSpecialKey;
-extern const KeyRef excludedLocalityForceOptionSpecialKey;
-extern const KeyRef failedLocalityForceOptionSpecialKey;
-extern const KeyRangeRef exclusionInProgressSpecialKeyRange;
-
-extern const KeyRef lockSpecialKey;
-
-extern const KeyRangeRef maintenanceSpecialKeyRange;
-extern const KeyRef ignoreSSFailureSpecialKey;
-
-extern const KeyRangeRef processClassSourceSpecialKeyRange;
-extern const KeyRangeRef processClassTypeSpecialKeyRange;
-
-inline const KeyRef errorMsgSpecialKey = "\xff\xff/error_message"_sr;
-inline const KeyRef workerInterfacesVerifyOptionSpecialKey = "\xff\xff/management/options/worker_interfaces/verify"_sr;
-
-ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers);
-ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
-                                              std::map<std::string, StorageServerInterface>* interfaces);
-
-bool tokencmp(StringRef token, const char* command);
-void printUsage(StringRef command);
-void printLongDesc(StringRef command);
-
-// Pre: tr failed with special_keys_api_failure error
-// Read the error message special key and return the message
-ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr);
-// Using \xff\xff/worker_interfaces/ special key, get all worker interfaces.
-// A worker list will be returned from CC.
-// If verify, we will try to establish connections to all workers returned.
-// In particular, it will deserialize \xff\xff/worker_interfaces/<address>:=<ClientInterface> kv pairs and issue RPC
-// calls, then only return interfaces(kv pairs) the client can talk to
-ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
-                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
-                                       bool verify = false);
-// print cluster status info
-void printStatus(StatusObjectReader statusObj,
-                 StatusClient::StatusLevel level,
-                 bool displayDatabaseAvailable = true,
-                 bool hideErrorMessages = false);
-
-// All fdbcli commands (alphabetically)
-// All below actors return true if the command is executed successfully
-// advanceversion command
-ACTOR Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// configure command
-ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
-                                         Database localDb,
-                                         std::vector<StringRef> tokens,
-                                         LineNoise* linenoise,
-                                         Future<Void> warn);
-// consistency command
-ACTOR Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
-                                                std::vector<StringRef> tokens,
-                                                bool intrans);
-// consistency scan command
-ACTOR Future<bool> consistencyScanCommandActor(Database localDb, std::vector<StringRef> tokens);
-// coordinators command
-ACTOR Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// datadistribution command
-ACTOR Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// exclude command
-ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> warn);
-// expensive_data_check command
-ACTOR Future<bool> expensiveDataCheckCommandActor(
-    Reference<IDatabase> db,
-    Reference<ITransaction> tr,
-    std::vector<StringRef> tokens,
-    std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
-// fileconfigure command
-ACTOR Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
-                                             std::string filePath,
-                                             bool isNewDatabase,
-                                             bool force);
-// Trigger audit storage
-ACTOR Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> clusterFile,
-                                           std::vector<StringRef> tokens);
-// Retrieve audit storage status
-ACTOR Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef> tokens);
-// Retrieve shard information command
-ACTOR Future<bool> locationMetadataCommandActor(Database cx, std::vector<StringRef> tokens);
-// Bulk loading command
-ACTOR Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> tokens);
-// Bulk dumping command
-ACTOR Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> tokens);
-// force_recovery_with_data_loss command
-ACTOR Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// include command
-ACTOR Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// kill command
-ACTOR Future<bool> killCommandActor(Reference<IDatabase> db,
-                                    Reference<ITransaction> tr,
-                                    std::vector<StringRef> tokens,
-                                    std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
-// lock/unlock command
-ACTOR Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-ACTOR Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid);
-
-// blobrestore command
-ACTOR Future<bool> blobRestoreCommandActor(Database localDb, std::vector<StringRef> tokens);
-// hotrange command
-ACTOR Future<bool> hotRangeCommandActor(Database localDb,
-                                        Reference<IDatabase> db,
-                                        std::vector<StringRef> tokens,
-                                        std::map<std::string, StorageServerInterface>* storage_interface);
-
-// maintenance command
-ACTOR Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning = false);
-ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
-                                    bool printWarning = false,
-                                    bool clearSSFailureZoneString = false);
-ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// profile command
-ACTOR Future<bool> profileCommandActor(Database db,
-                                       Reference<ITransaction> tr,
-                                       std::vector<StringRef> tokens,
-                                       bool intrans);
-// quota command
-ACTOR Future<bool> quotaCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// setclass command
-ACTOR Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// snapshot command
-ACTOR Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// status command
-ACTOR Future<bool> statusCommandActor(Reference<IDatabase> db,
-                                      Database localDb,
-                                      std::vector<StringRef> tokens,
-                                      bool isExecMode = false);
-// suspend command
-ACTOR Future<bool> suspendCommandActor(Reference<IDatabase> db,
-                                       Reference<ITransaction> tr,
-                                       std::vector<StringRef> tokens,
-                                       std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
-// throttle command
-ACTOR Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// triggerteaminfolog command
-ACTOR Future<bool> triggerddteaminfologCommandActor(Reference<IDatabase> db);
-// tssq command
-ACTOR Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// versionepoch command
-ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens);
-// targetversion command
-ACTOR Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
-// idempotencyids command
-ACTOR Future<bool> idempotencyIdsCommandActor(Database cx, std::vector<StringRef> tokens);
-
-// rangeconfig command
-ACTOR Future<bool> rangeConfigCommandActor(Database cx, std::vector<StringRef> tokens);
-
-// debug commands: getlocation, getall
-ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens);
-ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version);
-ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens);
-} // namespace fdb_cli
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbcli/include/fdbcli/fdbcli.h b/fdbcli/include/fdbcli/fdbcli.h
new file mode 100644
index 00000000000..cdc3ce7975d
--- /dev/null
+++ b/fdbcli/include/fdbcli/fdbcli.h
@@ -0,0 +1,342 @@
+/*
+ * fdbcli.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbcli/FlowLineNoise.h"
+
+#include "fdbclient/BulkLoading.h"
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/IClientApi.h"
+#include "fdbclient/StatusClient.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "flow/Arena.h"
+
+namespace fdb_cli {
+
+constexpr char msgTypeKey[] = "type";
+constexpr char msgClusterKey[] = "cluster";
+constexpr char msgCapacityKey[] = "capacity";
+constexpr char msgAllocatedKey[] = "allocated";
+constexpr char msgErrorKey[] = "error";
+
+struct CommandHelp {
+	std::string usage;
+	std::string short_desc;
+	std::string long_desc;
+	CommandHelp() {}
+	CommandHelp(const char* usage, const char* short_desc, const char* long_desc)
+	  : usage(usage), short_desc(short_desc), long_desc(long_desc) {}
+};
+
+void arrayGenerator(const char* text, const char* line, const char** options, std::vector<std::string>& lc);
+
+struct CommandFactory {
+	typedef void (*CompletionGeneratorFunc)(const char* text,
+	                                        const char* line,
+	                                        std::vector<std::string>& lc,
+	                                        std::vector<StringRef> const& tokens);
+
+	typedef std::vector<const char*> (*HintGeneratorFunc)(std::vector<StringRef> const& tokens, bool inArgument);
+
+	CommandFactory(const char* name,
+	               CommandHelp help,
+	               CompletionGeneratorFunc completionFunc = nullptr,
+	               HintGeneratorFunc hintFunc = nullptr) {
+		commands()[name] = help;
+		if (completionFunc) {
+			completionGenerators()[name] = completionFunc;
+		}
+		if (hintFunc) {
+			hintGenerators()[name] = hintFunc;
+		}
+	}
+	explicit CommandFactory(const char* name) { hiddenCommands().insert(name); }
+	static std::map<std::string, CommandHelp>& commands() {
+		static std::map<std::string, CommandHelp> helpMap;
+		return helpMap;
+	}
+	static std::set<std::string>& hiddenCommands() {
+		static std::set<std::string> commands;
+		return commands;
+	}
+	static std::map<std::string, CompletionGeneratorFunc>& completionGenerators() {
+		static std::map<std::string, CompletionGeneratorFunc> completionMap;
+		return completionMap;
+	}
+	static std::map<std::string, HintGeneratorFunc>& hintGenerators() {
+		static std::map<std::string, HintGeneratorFunc> hintMap;
+		return hintMap;
+	}
+};
+
+// Special keys used by fdbcli commands
+
+extern const KeyRef advanceVersionSpecialKey;
+extern const KeyRef consistencyCheckSpecialKey;
+extern const KeyRef clusterDescriptionSpecialKey;
+extern const KeyRef coordinatorsAutoSpecialKey;
+extern const KeyRef coordinatorsProcessSpecialKey;
+extern const KeyRef ddModeSpecialKey;
+extern const KeyRef ddIgnoreRebalanceSpecialKey;
+
+extern const KeyRangeRef excludedServersSpecialKeyRange;
+extern const KeyRangeRef failedServersSpecialKeyRange;
+extern const KeyRangeRef excludedLocalitySpecialKeyRange;
+extern const KeyRangeRef failedLocalitySpecialKeyRange;
+extern const KeyRef excludedForceOptionSpecialKey;
+extern const KeyRef failedForceOptionSpecialKey;
+extern const KeyRef excludedLocalityForceOptionSpecialKey;
+extern const KeyRef failedLocalityForceOptionSpecialKey;
+extern const KeyRangeRef exclusionInProgressSpecialKeyRange;
+
+extern const KeyRef lockSpecialKey;
+
+extern const KeyRangeRef maintenanceSpecialKeyRange;
+extern const KeyRef ignoreSSFailureSpecialKey;
+
+extern const KeyRangeRef processClassSourceSpecialKeyRange;
+extern const KeyRangeRef processClassTypeSpecialKeyRange;
+
+inline const KeyRef errorMsgSpecialKey = "\xff\xff/error_message"_sr;
+inline const KeyRef workerInterfacesVerifyOptionSpecialKey = "\xff\xff/management/options/worker_interfaces/verify"_sr;
+
+Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>* workers);
+Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
+                                        std::map<std::string, StorageServerInterface>* interfaces);
+
+bool tokencmp(StringRef token, const char* command);
+void printUsage(StringRef command);
+void printLongDesc(StringRef command);
+
+// Pre: tr failed with special_keys_api_failure error
+// Read the error message special key and return the message
+Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr);
+
+// ============================================================================
+// BulkDump/BulkLoad CLI Utilities
+// ============================================================================
+
+// Parse and validate a job ID token from CLI input. Prints usage and throws on invalid input.
+UID validateBulkJobId(StringRef token, const char* usage);
+
+// Get owner information suffix for display (e.g., " (owned by: backup 'daily-backup')").
+// Returns empty string if no owner is set.
+Future<std::string> getBulkOwnerSuffix(Database cx, UID jobId, bool isDumpJob);
+
+// Format bytes with optional total and percentage.
+std::string formatBytesProgress(int64_t completedBytes, Optional<int64_t> totalBytes);
+
+// Print throughput, ETA, and elapsed time metrics to stdout.
+void printProgressMetrics(double avgBytesPerSecond, Optional<double> etaSeconds, double elapsedSeconds);
+
+// Print task and byte completion summary (e.g., "Tasks: 5/10, Bytes: 1.2 GB / 2.4 GB").
+void printProgressSummary(const char* operationName,
+                          int completeTasks,
+                          int totalTasks,
+                          int64_t completedBytes,
+                          int64_t totalBytes);
+
+// Print detailed task state breakdown for bulkload operations.
+void printTaskBreakdown(int submittedTasks, int triggeredTasks, int runningTasks, int completeTasks, int errorTasks);
+
+// ============================================================================
+// BulkDump/BulkLoad Health Analysis
+// Used by "bulkdump status --analyze" and "bulkload status --analyze"
+// ============================================================================
+
+// Health assessment based on throughput, efficiency, and error rates.
+// healthScore: 0-100 rating, healthStatus: "healthy"/"degraded"/"critical"
+struct BulkHealthMetrics {
+	double healthScore;
+	std::string healthStatus;
+	std::vector<std::string> recommendations;
+
+	static BulkHealthMetrics analyze(double throughputMBps,
+	                                 double efficiencyPercent,
+	                                 int stalledTasks,
+	                                 int errorTasks,
+	                                 double elapsedMinutes);
+};
+
+// Error categorization and actionable diagnostics.
+struct BulkErrorAnalysis {
+	int totalErrors;
+	std::map<std::string, int> errorCategories;
+	std::vector<std::string> criticalIssues;
+	std::vector<std::string> actionableAdvice;
+
+	static BulkErrorAnalysis analyze(int errorTasks, int stalledTasks, const std::vector<std::string>& recentErrors);
+};
+
+// Performance and operational recommendations based on current metrics.
+struct BulkOptimizationRecommendations {
+	std::vector<std::string> performanceRecommendations;
+	std::vector<std::string> reliabilityRecommendations;
+	std::vector<std::string> operationalRecommendations;
+
+	static BulkOptimizationRecommendations generate(double throughputMBps,
+	                                                double efficiency,
+	                                                int stalledTasks,
+	                                                int errorTasks,
+	                                                double elapsedMinutes,
+	                                                int totalTasks);
+};
+
+// Print formatted health analysis to stdout.
+void printBulkHealthAnalysis(const BulkHealthMetrics& health);
+
+// Print error diagnostics and advice to stdout.
+void printErrorDiagnostics(const BulkErrorAnalysis& analysis);
+
+// Print optimization recommendations to stdout.
+void printOptimizationRecommendations(const BulkOptimizationRecommendations& recs);
+
+// Print warnings for tasks that have stalled (no progress for extended period).
+void printStalledTasks(const std::vector<BulkLoadStalledTask>& stalledTasks);
+
+// Run full analysis and print results: health metrics, stalled tasks, error diagnostics, recommendations.
+void printBulkAnalysis(double avgBytesPerSecond,
+                       double elapsedSeconds,
+                       int completeTasks,
+                       int totalTasks,
+                       int errorTasks,
+                       const std::vector<BulkLoadStalledTask>& stalledTasks);
+
+// Using \xff\xff/worker_interfaces/ special key, get all worker interfaces.
+// A worker list will be returned from CC.
+// If verify, we will try to establish connections to all workers returned.
+// In particular, it will deserialize \xff\xff/worker_interfaces/<address>:=<ClientInterface> kv pairs and issue RPC
+// calls, then only return interfaces(kv pairs) the client can talk to
+Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
+                                 std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface,
+                                 bool verify = false);
+// print cluster status info
+void printStatus(StatusObjectReader statusObj,
+                 StatusClient::StatusLevel level,
+                 bool displayDatabaseAvailable = true,
+                 bool hideErrorMessages = false);
+
+// All fdbcli commands (alphabetically)
+// All below actors return true if the command is executed successfully
+// advanceversion command
+Future<bool> advanceVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// configure command
+Future<bool> configureCommandActor(Reference<IDatabase> db,
+                                   Database localDb,
+                                   std::vector<StringRef> tokens,
+                                   LineNoise* linenoise,
+                                   Future<Void> _warn);
+// consistency command
+Future<bool> consistencyCheckCommandActor(Reference<ITransaction> tr,
+                                          std::vector<StringRef> const& tokens,
+                                          bool intrans);
+// consistency scan command
+Future<bool> consistencyScanCommandActor(Database localDb, std::vector<StringRef> const& tokens);
+// coordinators command
+Future<bool> coordinatorsCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// datadistribution command
+Future<bool> dataDistributionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// exclude command
+Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> _warn);
+// expensive_data_check command
+Future<bool> expensiveDataCheckCommandActor(
+    Reference<IDatabase> db,
+    Reference<ITransaction> tr,
+    std::vector<StringRef> tokens,
+    std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
+// fileconfigure command
+Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
+                                       std::string const& filePath,
+                                       bool isNewDatabase,
+                                       bool force);
+// Trigger audit storage
+Future<UID> auditStorageCommandActor(Reference<IClusterConnectionRecord> clusterFile, std::vector<StringRef> tokens);
+// Retrieve audit storage status
+Future<bool> getAuditStatusCommandActor(Database cx, std::vector<StringRef> tokens);
+// Retrieve shard information command
+Future<bool> locationMetadataCommandActor(Database cx, std::vector<StringRef> tokens);
+// Bulk loading command
+Future<UID> bulkLoadCommandActor(Database cx, std::vector<StringRef> tokens);
+// Bulk dumping command
+Future<UID> bulkDumpCommandActor(Database cx, std::vector<StringRef> tokens);
+// force_recovery_with_data_loss command
+Future<bool> forceRecoveryWithDataLossCommandActor(Reference<IDatabase> db, std::vector<StringRef> const& tokens);
+// include command
+Future<bool> includeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// kill command
+Future<bool> killCommandActor(Reference<IDatabase> db,
+                              Reference<ITransaction> tr,
+                              std::vector<StringRef> tokens,
+                              std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
+// lock/unlock command
+Future<bool> lockCommandActor(Reference<IDatabase> db, std::vector<StringRef> const& tokens);
+Future<bool> unlockDatabaseActor(Reference<IDatabase> db, UID uid);
+
+// hotrange command
+Future<bool> hotRangeCommandActor(Database localDb,
+                                  Reference<IDatabase> db,
+                                  std::vector<StringRef> const& tokens,
+                                  std::map<std::string, StorageServerInterface>* const& storage_interface);
+
+// maintenance command
+Future<bool> setHealthyZone(Reference<IDatabase> db, StringRef zoneId, double seconds, bool printWarning = false);
+Future<bool> clearHealthyZone(Reference<IDatabase> db,
+                              bool printWarning = false,
+                              bool clearSSFailureZoneString = false);
+Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// profile command
+Future<bool> profileCommandActor(Database db, Reference<ITransaction> tr, std::vector<StringRef> tokens, bool intrans);
+// setclass command
+Future<bool> setClassCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// snapshot command
+Future<bool> snapshotCommandActor(Reference<IDatabase> db, std::vector<StringRef> const& tokens);
+// status command
+Future<bool> statusCommandActor(Reference<IDatabase> db,
+                                Database localDb,
+                                std::vector<StringRef> tokens,
+                                bool isExecMode = false);
+// suspend command
+Future<bool> suspendCommandActor(Reference<IDatabase> db,
+                                 Reference<ITransaction> tr,
+                                 std::vector<StringRef> const& tokens,
+                                 std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);
+// throttle command
+Future<bool> throttleCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// triggerteaminfolog command
+Future<bool> triggerddteaminfologCommandActor(Reference<IDatabase> db);
+// tssq command
+Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// versionepoch command
+Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens);
+// targetversion command
+Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// idempotencyids command
+Future<bool> idempotencyIdsCommandActor(Database cx, std::vector<StringRef> const& tokens);
+
+// rangeconfig command
+Future<bool> rangeConfigCommandActor(Database cx, std::vector<StringRef> tokens);
+
+// debug commands: getlocation, getall
+Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens);
+Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version);
+Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens);
+} // namespace fdb_cli
diff --git a/fdbcli/tests/fdbcli_tests.py b/fdbcli/tests/fdbcli_tests.py
index 57722e8bd33..22f9d80cb39 100755
--- a/fdbcli/tests/fdbcli_tests.py
+++ b/fdbcli/tests/fdbcli_tests.py
@@ -119,76 +119,6 @@ def maintenance(logger):
     assert output3 == no_maintenance_output
 
 
-@enable_logging()
-def quota(logger):
-    # Should be a noop
-    command = "quota clear green"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "Successfully cleared quota."
-
-    command = "quota get green total_throughput"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "<empty>"
-
-    # Ignored update
-    command = "quota set red total_throughput 49152"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "Successfully updated quota."
-
-    command = "quota set green total_throughput 32768"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "Successfully updated quota."
-
-    command = "quota set green reserved_throughput 16384"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "Successfully updated quota."
-
-    command = "quota set green storage 98765"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "Successfully updated quota."
-
-    command = "quota get green total_throughput"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "32768"
-
-    command = "quota get green reserved_throughput"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "16384"
-
-    command = "quota get green storage"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "98765"
-
-    command = "quota clear green"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "Successfully cleared quota."
-
-    command = "quota get green total_throughput"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "<empty>"
-
-    command = "quota get green storage"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-    assert output == "<empty>"
-
-    # Too few arguments, should log help message
-    command = "quota get green"
-    output = run_fdbcli_command(command)
-    logger.debug(command + " : " + output)
-
-
 @enable_logging()
 def setclass(logger):
     # get all processes' network addresses
@@ -379,7 +309,7 @@ def suspend(logger):
     port = address.split(":")[1]
     logger.debug("Port: {}".format(port))
     # use the port number to find the exact fdb process we are connecting to
-    # child process like fdbserver -r flowprocess does not provide `datadir` in the command line
+    # auxiliary processes may not provide `datadir` in the command line
     pinfo = list(filter(lambda x: port in x and "datadir" in x, pinfos))
     assert len(pinfo) == 1
     pid = pinfo[0].split(" ")[0]
@@ -471,6 +401,64 @@ def get_value_from_status_json(retry, *args):
     return result
 
 
+def status_json_file_region_failover_message():
+    status_json = {
+        "client": {
+            "cluster_file": {"path": "fdb.cluster", "up_to_date": True},
+            "coordinators": {"coordinators": [], "quorum_reachable": True},
+            "database_status": {"available": True, "healthy": False},
+            "messages": [],
+            "timestamp": 1417807090,
+        },
+        "cluster": {
+            "configuration": {
+                "redundancy_mode": "three_data_hall",
+                "storage_engine": "ssd-2",
+                "coordinators_count": 3,
+                "excluded_servers": [],
+            },
+            "data": {"state": {"name": "healthy", "healthy": True}},
+            "fault_tolerance": {
+                "max_zone_failures_without_losing_availability": -1,
+                "max_zone_failures_without_losing_data": -1,
+            },
+            "logs": [
+                {
+                    "epoch": 1,
+                    "current": True,
+                    "begin_version": 1,
+                    "possibly_losing_data": False,
+                    "log_interfaces": [
+                        {
+                            "id": "aaaaaaaaaaaaaaaa",
+                            "healthy": False,
+                            "address": "1.1.1.1:4500",
+                        }
+                    ],
+                }
+            ],
+            "machines": {},
+            "processes": {},
+        },
+    }
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".json") as status_file:
+        json.dump(status_json, status_file)
+        status_file.flush()
+        result = subprocess.run(
+            [command_template[0], "--status-from-json", status_file.name],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=fdbcli_env,
+        )
+
+    stdout = result.stdout.decode("utf-8")
+    stderr = result.stderr.decode("utf-8")
+    assert result.returncode == 0, stderr
+    assert "Warning: the database may have availability loss." in stdout
+    assert "may have data loss" not in stdout
+
+
 @enable_logging()
 def consistencycheck(logger):
     consistency_check_on_output = "ConsistencyCheck is on"
@@ -939,17 +927,16 @@ def tls_address_suffix():
         lockAndUnlock()
         maintenance()
         profile()
-        # TODO: re-enable it until it's stable
+        # TODO: re-enable once stable
         # suspend()
         transaction()
-        # this is replaced by the "quota" command
+        # TODO: re-enable once stable
         # throttle()
         triggerddteaminfolog()
         versionepoch()
         integer_options()
         tls_address_suffix()
-        # TODO: fix the issue when running through the external client
-        # quota()
+        status_json_file_region_failover_message()
         idempotency_ids()
     else:
         assert args.process_number > 1, "Process number should be positive"
diff --git a/fdbclient/ActorLineageProfiler.cpp b/fdbclient/ActorLineageProfiler.cpp
index 50c6cfa0602..17180df6ddc 100644
--- a/fdbclient/ActorLineageProfiler.cpp
+++ b/fdbclient/ActorLineageProfiler.cpp
@@ -22,7 +22,7 @@
 #include "flow/singleton.h"
 #include "flow/IAsyncFile.h"
 #include "fdbclient/ActorLineageProfiler.h"
-#include "fdbclient/NameLineage.h"
+#include "NameLineage.h"
 #include <msgpack.hpp>
 #include <memory>
 #include <typeindex>
@@ -313,7 +313,7 @@ boost::asio::io_context& ActorLineageProfilerT::context() {
 	return impl->context;
 }
 
-SampleIngestor::~SampleIngestor() {}
+SampleIngestor::~SampleIngestor() = default;
 
 void ProfilerConfigT::reset(std::map<std::string, std::string> const& config) {
 	bool expectNoMore = false, useFluentD = false, useTCP = false;
diff --git a/fdbclient/AsyncFileS3BlobStore.actor.cpp b/fdbclient/AsyncFileBlobStore.cpp
similarity index 66%
rename from fdbclient/AsyncFileS3BlobStore.actor.cpp
rename to fdbclient/AsyncFileBlobStore.cpp
index 3e542794bfc..0952fec0752 100644
--- a/fdbclient/AsyncFileS3BlobStore.actor.cpp
+++ b/fdbclient/AsyncFileBlobStore.cpp
@@ -1,5 +1,5 @@
 /*
- * AsyncFileS3BlobStore.actor.cpp
+ * AsyncFileBlobStore.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,50 +18,45 @@
  * limitations under the License.
  */
 
-#include "fdbclient/AsyncFileS3BlobStore.actor.h"
-#include "fdbrpc/AsyncFileReadAhead.actor.h"
+#include "fdbclient/AsyncFileBlobStore.h"
 #include "flow/UnitTest.h"
-#include "flow/IConnection.h"
-#include "flow/actorcompiler.h" // has to be last include
 
-Future<int64_t> AsyncFileS3BlobStoreRead::size() const {
+Future<int64_t> AsyncFileBlobStoreRead::size() const {
 	if (!m_size.isValid())
 		m_size = m_bstore->objectSize(m_bucket, m_object);
 	return m_size;
 }
 
-Future<int> AsyncFileS3BlobStoreRead::read(void* data, int length, int64_t offset) {
+Future<int> AsyncFileBlobStoreRead::read(void* data, int length, int64_t offset) {
 	return m_bstore->readObject(m_bucket, m_object, data, length, offset);
 }
 
-ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
+Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
 	printf("Starting fake sender %d which will send send %d bytes.\n", id, bytes);
-	state double ts = timer();
-	state int total = 0;
+	double ts = timer();
+	int total = 0;
 	while (total < bytes) {
-		state int r = std::min<int>(deterministicRandom()->randomInt(0, 1000), bytes - total);
-		wait(t->getAllowance(r));
+		int r = std::min<int>(deterministicRandom()->randomInt(0, 1000), bytes - total);
+		co_await t->getAllowance(r);
 		total += r;
 	}
 	double dur = timer() - ts;
 	printf("Sender %d: Sent %d in %fs, %f/s\n", id, total, dur, total / dur);
-	return Void();
 }
 
 TEST_CASE("/backup/throttling") {
 	// Test will not work in simulation.
 	if (g_network->isSimulated())
-		return Void();
+		co_return;
 
-	state int limit = 100000;
-	state Reference<IRateControl> t(new SpeedLimit(limit, 1));
+	int limit = 100000;
+	Reference<IRateControl> t(new SpeedLimit(limit, 1));
 
-	state int id = 1;
+	int id = 1;
 	std::vector<Future<Void>> f;
-	state double ts = timer();
-	state int total = 0;
-	int s;
-	s = 500000;
+	double ts = timer();
+	int total = 0;
+	int s = 500000;
 	f.push_back(sendStuff(id++, t, s));
 	total += s;
 	f.push_back(sendStuff(id++, t, s));
@@ -75,11 +70,9 @@ TEST_CASE("/backup/throttling") {
 	f.push_back(sendStuff(id++, t, s));
 	total += s;
 
-	wait(waitForAll(f));
+	co_await waitForAll(f);
 	double dur = timer() - ts;
 	int speed = int(total / dur);
 	printf("Speed limit was %d, measured speed was %d\n", limit, speed);
 	ASSERT(abs(speed - limit) / limit < .01);
-
-	return Void();
 }
diff --git a/fdbclient/AsyncTaskThread.actor.cpp b/fdbclient/AsyncTaskThread.actor.cpp
deleted file mode 100644
index adf9321355f..00000000000
--- a/fdbclient/AsyncTaskThread.actor.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * AsyncTaskThread.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <atomic>
-
-#include "fdbclient/AsyncTaskThread.h"
-#include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-namespace {
-
-class TerminateTask final : public IAsyncTask {
-public:
-	void operator()() override { ASSERT(false); }
-	bool isTerminate() const override { return true; }
-};
-
-ACTOR Future<Void> asyncTaskThreadClient(AsyncTaskThread* asyncTaskThread,
-                                         std::atomic<int>* sum,
-                                         int count,
-                                         int clientId,
-                                         double meanSleep) {
-	state int i = 0;
-	state double randomSleep = 0.0;
-	for (; i < count; ++i) {
-		randomSleep = deterministicRandom()->random01() * 2 * meanSleep;
-		wait(delay(randomSleep));
-		wait(asyncTaskThread->execAsync([sum = sum] {
-			sum->fetch_add(1);
-			return Void();
-		}));
-		TraceEvent("AsyncTaskThreadIncrementedSum")
-		    .detail("Index", i)
-		    .detail("Sum", sum->load())
-		    .detail("ClientId", clientId)
-		    .detail("RandomSleep", randomSleep)
-		    .detail("MeanSleep", meanSleep);
-	}
-	return Void();
-}
-
-} // namespace
-
-const double AsyncTaskThread::meanDelay = 0.01;
-
-AsyncTaskThread::AsyncTaskThread() : thread([this] { run(this); }) {}
-
-AsyncTaskThread::~AsyncTaskThread() {
-	bool wakeUp = false;
-	{
-		std::lock_guard<std::mutex> g(m);
-		wakeUp = queue.push(std::make_unique<TerminateTask>());
-	}
-	if (wakeUp) {
-		cv.notify_one();
-	}
-	thread.join();
-}
-
-void AsyncTaskThread::run(AsyncTaskThread* self) {
-	while (true) {
-		std::unique_ptr<IAsyncTask> task;
-		{
-			std::unique_lock<std::mutex> lk(self->m);
-			self->cv.wait(lk, [self] { return !self->queue.canSleep(); });
-			task = self->queue.pop().get();
-			if (task->isTerminate()) {
-				return;
-			}
-		}
-		(*task)();
-	}
-}
-
-TEST_CASE("/asynctaskthread/add") {
-	state std::atomic<int> sum = 0;
-	state AsyncTaskThread asyncTaskThread;
-	state int numClients = 10;
-	state int incrementsPerClient = 100;
-	std::vector<Future<Void>> clients;
-	clients.reserve(numClients);
-	for (int clientId = 0; clientId < numClients; ++clientId) {
-		clients.push_back(asyncTaskThreadClient(
-		    &asyncTaskThread, &sum, incrementsPerClient, clientId, deterministicRandom()->random01() * 0.01));
-	}
-	wait(waitForAll(clients));
-	ASSERT_EQ(sum.load(), numClients * incrementsPerClient);
-	return Void();
-}
-
-TEST_CASE("/asynctaskthread/error") {
-	state AsyncTaskThread asyncTaskThread;
-	try {
-		wait(asyncTaskThread.execAsync([] {
-			throw operation_failed();
-			return Void();
-		}));
-		ASSERT(false);
-	} catch (Error& e) {
-		ASSERT_EQ(e.code(), error_code_operation_failed);
-	}
-	return Void();
-}
diff --git a/fdbclient/AuditUtils.actor.cpp b/fdbclient/AuditUtils.cpp
similarity index 64%
rename from fdbclient/AuditUtils.actor.cpp
rename to fdbclient/AuditUtils.cpp
index d3fd08cc294..0bb5b3a72a5 100644
--- a/fdbclient/AuditUtils.actor.cpp
+++ b/fdbclient/AuditUtils.cpp
@@ -1,5 +1,5 @@
 /*
- * AuditUtils.actor.cpp
+ * AuditUtils.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,17 +18,16 @@
  * limitations under the License.
  */
 
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 
 #include "fdbclient/Audit.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/ClientKnobs.h"
+#include "fdbclient/Knobs.h"
+#include "flow/CoroUtils.h"
 #include <fmt/format.h>
 
-#include "flow/actorcompiler.h" // has to be last include
-
 void clearAuditProgressMetadata(Transaction* tr, AuditType auditType, UID auditId) {
 	// There are two possible places to store AuditProgressMetadata:
 	// (1) auditServerBasedProgressRangeFor or (2) auditRangeBasedProgressRangeFor
@@ -53,67 +52,68 @@ void clearAuditProgressMetadata(Transaction* tr, AuditType auditType, UID auditI
 	return;
 }
 
-ACTOR Future<bool> checkStorageServerRemoved(Database cx, UID ssid) {
-	state bool res = false;
-	state Transaction tr(cx);
+Future<bool> checkStorageServerRemoved(Database cx, UID ssid) {
+	bool res = false;
+	Transaction tr(cx);
 	TraceEvent(SevDebug, "AuditUtilStorageServerRemovedStart").detail("StorageServer", ssid);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> serverListValue = wait(tr.get(serverListKeyFor(ssid)));
+			Optional<Value> serverListValue = co_await tr.get(serverListKeyFor(ssid));
 			if (!serverListValue.present()) {
 				res = true; // SS is removed
 			}
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "AuditUtilStorageServerRemovedError")
-			    .errorUnsuppressed(e)
-			    .detail("StorageServer", ssid);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevDebug, "AuditUtilStorageServerRemovedError").errorUnsuppressed(err).detail("StorageServer", ssid);
+		co_await tr.onError(err);
 	}
 
 	TraceEvent(SevDebug, "AuditUtilStorageServerRemovedEnd").detail("StorageServer", ssid).detail("Removed", res);
-	return res;
+	co_return res;
 }
 
-ACTOR Future<Void> cancelAuditMetadata(Database cx, AuditType auditType, UID auditId) {
+Future<Void> cancelAuditMetadata(Database cx, AuditType auditType, UID auditId) {
 	try {
-		state Transaction tr(cx);
+		Transaction tr(cx);
 		TraceEvent(SevInfo, "AuditUtilCancelAuditMetadataStart", auditId)
 		    .detail("AuditKey", auditKey(auditType, auditId));
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> res_ = wait(tr.get(auditKey(auditType, auditId)));
+				Optional<Value> res_ = co_await tr.get(auditKey(auditType, auditId));
 				if (!res_.present()) { // has been cancelled
 					break; // Nothing to cancel
 				}
-				state AuditStorageState toCancelState = decodeAuditStorageState(res_.get());
+				AuditStorageState toCancelState = decodeAuditStorageState(res_.get());
 				// For a zombie audit, it is in running state
 				ASSERT(toCancelState.id == auditId && toCancelState.getType() == auditType);
 				toCancelState.setPhase(AuditPhase::Failed);
 				tr.set(auditKey(toCancelState.getType(), toCancelState.id), auditStorageStateValue(toCancelState));
 				clearAuditProgressMetadata(&tr, toCancelState.getType(), toCancelState.id);
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent(SevInfo, "AuditUtilCancelAuditMetadataEnd", auditId)
 				    .detail("AuditKey", auditKey(auditType, auditId));
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevWarn, "AuditUtilCancelAuditMetadataError", auditId)
-				    .detail("AuditKey", auditKey(auditType, auditId));
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent(SevWarn, "AuditUtilCancelAuditMetadataError", auditId)
+			    .detail("AuditKey", auditKey(auditType, auditId));
+			co_await tr.onError(err);
 		}
-	} catch (Error& e) {
+	} catch (Error&) {
 		throw cancel_audit_storage_failed();
 	}
-	return Void();
 }
 
 AuditPhase stringToAuditPhase(std::string auditPhaseStr) {
@@ -135,20 +135,21 @@ AuditPhase stringToAuditPhase(std::string auditPhaseStr) {
 }
 
 // This is not transactional
-ACTOR Future<std::vector<AuditStorageState>> getAuditStates(Database cx,
-                                                            AuditType auditType,
-                                                            bool newFirst,
-                                                            Optional<int> num,
-                                                            Optional<AuditPhase> phase) {
-	state Transaction tr(cx);
-	state std::vector<AuditStorageState> auditStates;
-	state Key readBegin;
-	state Key readEnd;
-	state Reverse reverse = newFirst ? Reverse::True : Reverse::False;
+AsyncResult<std::vector<AuditStorageState>> getAuditStates(Database cx,
+                                                           AuditType auditType,
+                                                           bool newFirst,
+                                                           Optional<int> num,
+                                                           Optional<AuditPhase> phase) {
+	Transaction tr(cx);
+	std::vector<AuditStorageState> auditStates;
+	Key readBegin;
+	Key readEnd;
+	Reverse reverse = newFirst ? Reverse::True : Reverse::False;
 	if (num.present() && num.get() == 0) {
-		return auditStates;
+		co_return auditStates;
 	}
-	loop {
+	while (true) {
+		Error err;
 		try {
 			readBegin = auditKeyRange(auditType).begin;
 			readEnd = auditKeyRange(auditType).end;
@@ -158,18 +159,18 @@ ACTOR Future<std::vector<AuditStorageState>> getAuditStates(Database cx,
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				KeyRangeRef rangeToRead(readBegin, readEnd);
-				state RangeResult res = wait(tr.getRange(rangeToRead,
-				                                         num.present() ? GetRangeLimits(num.get()) : GetRangeLimits(),
-				                                         Snapshot::False,
-				                                         reverse));
-				for (int i = 0; i < res.size(); ++i) {
-					const AuditStorageState auditState = decodeAuditStorageState(res[i].value);
+				RangeResult res = co_await tr.getRange(rangeToRead,
+				                                       num.present() ? GetRangeLimits(num.get()) : GetRangeLimits(),
+				                                       Snapshot::False,
+				                                       reverse);
+				for (const auto& auditKeyValue : res) {
+					const AuditStorageState auditState = decodeAuditStorageState(auditKeyValue.value);
 					if (phase.present() && auditState.getPhase() != phase.get()) {
 						continue;
 					}
 					auditStates.push_back(auditState);
 					if (num.present() && auditStates.size() == num.get()) {
-						return auditStates; // since res.more is not reliable when GetRangeLimits is set to 1
+						co_return auditStates; // since res.more is not reliable when GetRangeLimits is set to 1
 					}
 				}
 				if (!res.more) {
@@ -184,27 +185,28 @@ ACTOR Future<std::vector<AuditStorageState>> getAuditStates(Database cx,
 			}
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return auditStates;
+	co_return auditStates;
 }
 
-ACTOR Future<Void> clearAuditMetadataForType(Database cx,
-                                             AuditType auditType,
-                                             UID maxAuditIdToClear,
-                                             int numFinishAuditToKeep) {
-	state Transaction tr(cx);
-	state int numFinishAuditCleaned = 0; // We regard "Complete" and "Failed" audits as finish audits
+Future<Void> clearAuditMetadataForType(Database cx,
+                                       AuditType auditType,
+                                       UID maxAuditIdToClear,
+                                       int numFinishAuditToKeep) {
+	Transaction tr(cx);
+	int numFinishAuditCleaned = 0; // We regard "Complete" and "Failed" audits as finish audits
 	TraceEvent(SevDebug, "AuditUtilClearAuditMetadataForTypeStart")
 	    .detail("AuditType", auditType)
 	    .detail("MaxAuditIdToClear", maxAuditIdToClear);
 
 	try {
-		loop { // Cleanup until succeed or facing unretriable error
+		while (true) { // Cleanup until succeed or facing unretriable error
+			Error err;
 			try {
-				state std::vector<AuditStorageState> auditStates =
-				    wait(getAuditStates(cx, auditType, /*newFirst=*/false));
+				std::vector<AuditStorageState> auditStates = co_await getAuditStates(cx, auditType, /*newFirst=*/false);
 				// auditStates has ascending order of auditIds
 
 				// Read and clear are not atomic
@@ -244,15 +246,15 @@ ACTOR Future<Void> clearAuditMetadataForType(Database cx,
 					}
 					// For a zombie audit, it is in running state
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent(SevDebug, "AuditUtilClearAuditMetadataForTypeEnd")
 				    .detail("AuditType", auditType)
 				    .detail("NumCleanedFinishAudits", numFinishAuditCleaned);
 				break;
-
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		TraceEvent(SevInfo, "AuditUtilClearAuditMetadataForTypeError")
@@ -260,25 +262,23 @@ ACTOR Future<Void> clearAuditMetadataForType(Database cx,
 		    .errorUnsuppressed(e);
 		// We do not want audit cleanup effects DD
 	}
-
-	return Void();
 }
 
-ACTOR static Future<Void> checkMoveKeysLockForAudit(Transaction* tr,
-                                                    MoveKeyLockInfo lock,
-                                                    bool isDDEnabled,
-                                                    bool isWrite = true) {
+static Future<Void> checkMoveKeysLockForAudit(Transaction* tr,
+                                              MoveKeyLockInfo lock,
+                                              bool isDDEnabled,
+                                              bool isWrite = true) {
 	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	if (!isDDEnabled) {
 		TraceEvent(SevDebug, "AuditUtilDisabledByInMemoryCheck").log();
 		throw movekeys_conflict(); // need a new name
 	}
-	Optional<Value> readVal = wait(tr->get(moveKeysLockOwnerKey));
+	Optional<Value> readVal = co_await tr->get(moveKeysLockOwnerKey);
 	UID currentOwner = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 
 	if (currentOwner == lock.prevOwner) {
 		// Check that the previous owner hasn't touched the lock since we took it
-		Optional<Value> readVal = wait(tr->get(moveKeysLockWriteKey));
+		Optional<Value> readVal = co_await tr->get(moveKeysLockWriteKey);
 		UID lastWrite = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 		if (lastWrite != lock.prevWrite) {
 			TraceEvent(SevDebug, "ConflictWithPreviousOwner");
@@ -299,7 +299,7 @@ ACTOR static Future<Void> checkMoveKeysLockForAudit(Transaction* tr,
 			    .detail("MyOwner", lock.myOwner.toString())
 			    .detail("Writer", lastWriter.toString());
 		}
-		return Void();
+		co_return;
 	} else if (currentOwner == lock.myOwner) {
 		if (isWrite) {
 			// Touch the lock, preventing overlapping attempts to take it
@@ -309,7 +309,7 @@ ACTOR static Future<Void> checkMoveKeysLockForAudit(Transaction* tr,
 			// Make this transaction self-conflicting so the database will not execute it twice with the same write key
 			tr->makeSelfConflicting();
 		}
-		return Void();
+		co_return;
 	} else {
 		TraceEvent(SevDebug, "AuditUtilConflictWithNewOwner")
 		    .detail("CurrentOwner", currentOwner.toString())
@@ -320,25 +320,23 @@ ACTOR static Future<Void> checkMoveKeysLockForAudit(Transaction* tr,
 	}
 }
 
-ACTOR Future<UID> persistNewAuditState(Database cx,
-                                       AuditStorageState auditState,
-                                       MoveKeyLockInfo lock,
-                                       bool ddEnabled) {
+Future<UID> persistNewAuditState(Database cx, AuditStorageState auditState, MoveKeyLockInfo lock, bool ddEnabled) {
 	ASSERT(!auditState.id.isValid());
-	state Transaction tr(cx);
-	state UID auditId;
-	state AuditStorageState latestExistingAuditState;
+	Transaction tr(cx);
+	UID auditId;
+	AuditStorageState latestExistingAuditState;
 	TraceEvent(SevDebug, "AuditUtilPersistedNewAuditStateStart", auditId);
 	try {
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(checkMoveKeysLockForAudit(&tr, lock, ddEnabled, true));
+				co_await checkMoveKeysLockForAudit(&tr, lock, ddEnabled, true);
 				RangeResult res =
-				    wait(tr.getRange(auditKeyRange(auditState.getType()), 1, Snapshot::False, Reverse::True));
-				ASSERT(res.size() == 0 || res.size() == 1);
+				    co_await tr.getRange(auditKeyRange(auditState.getType()), 1, Snapshot::False, Reverse::True);
+				ASSERT(res.empty() || res.size() == 1);
 				uint64_t nextId = 1;
 				if (!res.empty()) {
 					latestExistingAuditState = decodeAuditStorageState(res[0].value);
@@ -348,13 +346,12 @@ ACTOR Future<UID> persistNewAuditState(Database cx,
 						if (latestExistingAuditState.id.first() == auditId.first()) {
 							// The new audit Id has been successfully persisted
 							// No more action needed
-							return auditId;
-						} else {
-							// When latestExistingAuditState.id.first() < auditId
-							// The new audit Id is failed to persist
-							// Check to confirm no other actor can persist new audit state
-							ASSERT(auditId.first() == latestExistingAuditState.id.first() + 1);
+							co_return auditId;
 						}
+						// When latestExistingAuditState.id.first() < auditId
+						// The new audit Id is failed to persist
+						// Check to confirm no other actor can persist new audit state
+						ASSERT(auditId.first() == latestExistingAuditState.id.first() + 1);
 					}
 					nextId = latestExistingAuditState.id.first() + 1;
 				}
@@ -363,16 +360,17 @@ ACTOR Future<UID> persistNewAuditState(Database cx,
 				TraceEvent(SevVerbose, "AuditUtilPersistedNewAuditStateIdSelected", auditId)
 				    .detail("AuditKey", auditKey(auditState.getType(), auditId));
 				tr.set(auditKey(auditState.getType(), auditId), auditStorageStateValue(auditState));
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent(SevDebug, "AuditUtilPersistedNewAuditState", auditId)
 				    .detail("AuditKey", auditKey(auditState.getType(), auditId));
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevDebug, "AuditUtilPersistedNewAuditStateError", auditId)
-				    .errorUnsuppressed(e)
-				    .detail("AuditKey", auditKey(auditState.getType(), auditId));
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent(SevDebug, "AuditUtilPersistedNewAuditStateError", auditId)
+			    .errorUnsuppressed(err)
+			    .detail("AuditKey", auditKey(auditState.getType(), auditId));
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "AuditUtilPersistedNewAuditStateUnretriableError", auditId)
@@ -381,35 +379,35 @@ ACTOR Future<UID> persistNewAuditState(Database cx,
 		ASSERT_WE_THINK(e.code() == error_code_actor_cancelled || e.code() == error_code_movekeys_conflict);
 		if (e.code() == error_code_actor_cancelled) {
 			throw e;
-		} else {
-			throw persist_new_audit_metadata_error();
 		}
+		throw persist_new_audit_metadata_error();
 	}
 
-	return auditId;
+	co_return auditId;
 }
 
-ACTOR Future<Void> persistAuditState(Database cx,
-                                     AuditStorageState auditState,
-                                     std::string context,
-                                     MoveKeyLockInfo lock,
-                                     bool ddEnabled) {
-	state Transaction tr(cx);
-	state AuditPhase auditPhase = auditState.getPhase();
+Future<Void> persistAuditState(Database cx,
+                               AuditStorageState auditState,
+                               std::string context,
+                               MoveKeyLockInfo lock,
+                               bool ddEnabled) {
+	Transaction tr(cx);
+	AuditPhase auditPhase = auditState.getPhase();
 	ASSERT(auditPhase == AuditPhase::Complete || auditPhase == AuditPhase::Failed || auditPhase == AuditPhase::Error);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			wait(checkMoveKeysLockForAudit(&tr, lock, ddEnabled, true));
+			co_await checkMoveKeysLockForAudit(&tr, lock, ddEnabled, true);
 			// Clear persistent progress data of the new audit if complete
 			if (auditPhase == AuditPhase::Complete) {
 				clearAuditProgressMetadata(&tr, auditState.getType(), auditState.id);
 			} // We keep the progress metadata of Failed and Error audits for further investigations
 			// Check existing state
-			Optional<Value> res_ = wait(tr.get(auditKey(auditState.getType(), auditState.id)));
+			Optional<Value> res_ = co_await tr.get(auditKey(auditState.getType(), auditState.id));
 			if (!res_.present()) { // has been cancelled
 				throw audit_storage_cancelled();
 			} else {
@@ -421,7 +419,7 @@ ACTOR Future<Void> persistAuditState(Database cx,
 			}
 			// Persist audit result
 			tr.set(auditKey(auditState.getType(), auditState.id), auditStorageStateValue(auditState));
-			wait(tr.commit());
+			co_await tr.commit();
 			TraceEvent(SevInfo, "AuditUtilPersistAuditState", auditState.id)
 			    .detail("AuditID", auditState.id)
 			    .detail("AuditType", auditState.getType())
@@ -430,62 +428,63 @@ ACTOR Future<Void> persistAuditState(Database cx,
 			    .detail("Context", context);
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "AuditUtilPersistAuditStateError", auditState.id)
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", auditState.id)
-			    .detail("AuditType", auditState.getType())
-			    .detail("AuditPhase", auditPhase)
-			    .detail("AuditKey", auditKey(auditState.getType(), auditState.id))
-			    .detail("Context", context);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevWarn, "AuditUtilPersistAuditStateError", auditState.id)
+		    .errorUnsuppressed(err)
+		    .detail("AuditID", auditState.id)
+		    .detail("AuditType", auditState.getType())
+		    .detail("AuditPhase", auditPhase)
+		    .detail("AuditKey", auditKey(auditState.getType(), auditState.id))
+		    .detail("Context", context);
+		co_await tr.onError(err);
 	}
-
-	return Void();
 }
 
-ACTOR Future<AuditStorageState> getAuditState(Database cx, AuditType type, UID id) {
-	state Transaction tr(cx);
-	state Optional<Value> res;
+Future<AuditStorageState> getAuditState(Database cx, AuditType type, UID id) {
+	Transaction tr(cx);
+	Optional<Value> res;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> res_ = wait(tr.get(auditKey(type, id)));
-			res = res_;
+			res = co_await tr.get(auditKey(type, id));
 			TraceEvent(SevDebug, "AuditUtilReadAuditState", id)
 			    .detail("AuditID", id)
 			    .detail("AuditType", type)
 			    .detail("AuditKey", auditKey(type, id));
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "AuditUtilReadAuditStateError", id)
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", id)
-			    .detail("AuditType", type)
-			    .detail("AuditKey", auditKey(type, id));
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevDebug, "AuditUtilReadAuditStateError", id)
+		    .errorUnsuppressed(err)
+		    .detail("AuditID", id)
+		    .detail("AuditType", type)
+		    .detail("AuditKey", auditKey(type, id));
+		co_await tr.onError(err);
 	}
 
 	if (!res.present()) {
 		throw key_not_found();
 	}
 
-	return decodeAuditStorageState(res.get());
+	co_return decodeAuditStorageState(res.get());
 }
 
-ACTOR Future<Void> persistAuditStateByRange(Database cx, AuditStorageState auditState) {
-	state Transaction tr(cx);
+Future<Void> persistAuditStateByRange(Database cx, AuditStorageState auditState) {
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> ddAuditState_ = wait(tr.get(auditKey(auditState.getType(), auditState.id)));
+			Optional<Value> ddAuditState_ = co_await tr.get(auditKey(auditState.getType(), auditState.id));
 			if (!ddAuditState_.present()) {
 				throw audit_storage_cancelled();
 			}
@@ -507,48 +506,45 @@ ACTOR Future<Void> persistAuditStateByRange(Database cx, AuditStorageState audit
 			if (ddAuditState.getPhase() == AuditPhase::Failed) {
 				throw audit_storage_cancelled();
 			}
-			wait(krmSetRange(&tr,
-			                 auditRangeBasedProgressPrefixFor(auditState.getType(), auditState.id),
-			                 auditState.range,
-			                 auditStorageStateValue(auditState)));
-			wait(tr.commit());
+			co_await krmSetRange(&tr,
+			                     auditRangeBasedProgressPrefixFor(auditState.getType(), auditState.id),
+			                     auditState.range,
+			                     auditStorageStateValue(auditState));
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "AuditUtilPersistAuditStateByRangeError")
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", auditState.id)
-			    .detail("AuditType", auditState.getType())
-			    .detail("AuditPhase", auditState.getPhase());
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevDebug, "AuditUtilPersistAuditStateByRangeError")
+		    .errorUnsuppressed(err)
+		    .detail("AuditID", auditState.id)
+		    .detail("AuditType", auditState.getType())
+		    .detail("AuditPhase", auditState.getPhase());
+		co_await tr.onError(err);
 	}
-
-	return Void();
 }
 
-ACTOR Future<std::vector<AuditStorageState>> getAuditStateByRange(Database cx,
-                                                                  AuditType type,
-                                                                  UID auditId,
-                                                                  KeyRange range) {
-	state RangeResult auditStates;
-	state Transaction tr(cx);
+Future<std::vector<AuditStorageState>> getAuditStateByRange(Database cx, AuditType type, UID auditId, KeyRange range) {
+	RangeResult auditStates;
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			RangeResult res_ = wait(krmGetRanges(&tr,
-			                                     auditRangeBasedProgressPrefixFor(type, auditId),
-			                                     range,
-			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES));
-			auditStates = res_;
+			auditStates = co_await krmGetRanges(&tr,
+			                                    auditRangeBasedProgressPrefixFor(type, auditId),
+			                                    range,
+			                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+			                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "AuditUtilGetAuditStateForRangeError").errorUnsuppressed(e).detail("AuditID", auditId);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevDebug, "AuditUtilGetAuditStateForRangeError").errorUnsuppressed(err).detail("AuditID", auditId);
+		co_await tr.onError(err);
 	}
 
 	// For a range of state that have value read from auditRangeBasedProgressPrefixFor
@@ -567,18 +563,19 @@ ACTOR Future<std::vector<AuditStorageState>> getAuditStateByRange(Database cx,
 		res.push_back(auditState);
 	}
 
-	return res;
+	co_return res;
 }
 
-ACTOR Future<Void> persistAuditStateByServer(Database cx, AuditStorageState auditState) {
-	state Transaction tr(cx);
+Future<Void> persistAuditStateByServer(Database cx, AuditStorageState auditState) {
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> ddAuditState_ = wait(tr.get(auditKey(auditState.getType(), auditState.id)));
+			Optional<Value> ddAuditState_ = co_await tr.get(auditKey(auditState.getType(), auditState.id));
 			if (!ddAuditState_.present()) {
 				throw audit_storage_cancelled();
 			}
@@ -600,55 +597,55 @@ ACTOR Future<Void> persistAuditStateByServer(Database cx, AuditStorageState audi
 			if (ddAuditState.getPhase() == AuditPhase::Failed) {
 				throw audit_storage_cancelled();
 			}
-			wait(krmSetRange(
+			co_await krmSetRange(
 			    &tr,
 			    auditServerBasedProgressPrefixFor(auditState.getType(), auditState.id, auditState.auditServerId),
 			    auditState.range,
-			    auditStorageStateValue(auditState)));
-			wait(tr.commit());
+			    auditStorageStateValue(auditState));
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "AuditUtilPersistAuditStateByRangeError")
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", auditState.id)
-			    .detail("AuditType", auditState.getType())
-			    .detail("AuditPhase", auditState.getPhase())
-			    .detail("AuditServerID", auditState.auditServerId);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevDebug, "AuditUtilPersistAuditStateByRangeError")
+		    .errorUnsuppressed(err)
+		    .detail("AuditID", auditState.id)
+		    .detail("AuditType", auditState.getType())
+		    .detail("AuditPhase", auditState.getPhase())
+		    .detail("AuditServerID", auditState.auditServerId);
+		co_await tr.onError(err);
 	}
-
-	return Void();
 }
 
-ACTOR Future<std::vector<AuditStorageState>> getAuditStateByServer(Database cx,
-                                                                   AuditType type,
-                                                                   UID auditId,
-                                                                   UID auditServerId,
-                                                                   KeyRange range) {
-	state RangeResult auditStates;
-	state Transaction tr(cx);
+Future<std::vector<AuditStorageState>> getAuditStateByServer(Database cx,
+                                                             AuditType type,
+                                                             UID auditId,
+                                                             UID auditServerId,
+                                                             KeyRange range) {
+	RangeResult auditStates;
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			RangeResult res_ = wait(krmGetRanges(&tr,
-			                                     auditServerBasedProgressPrefixFor(type, auditId, auditServerId),
-			                                     range,
-			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES));
-			auditStates = res_;
+			auditStates = co_await krmGetRanges(&tr,
+			                                    auditServerBasedProgressPrefixFor(type, auditId, auditServerId),
+			                                    range,
+			                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+			                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "AuditUtilGetAuditStateForRangeError")
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", auditId)
-			    .detail("AuditType", type)
-			    .detail("AuditServerID", auditServerId);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent(SevDebug, "AuditUtilGetAuditStateForRangeError")
+		    .errorUnsuppressed(err)
+		    .detail("AuditID", auditId)
+		    .detail("AuditType", type)
+		    .detail("AuditServerID", auditServerId);
+		co_await tr.onError(err);
 	}
 
 	// For a range of state that have value read from auditServerBasedProgressPrefixFor
@@ -667,24 +664,22 @@ ACTOR Future<std::vector<AuditStorageState>> getAuditStateByServer(Database cx,
 		res.push_back(auditState);
 	}
 
-	return res;
+	co_return res;
 }
 
-ACTOR Future<bool> checkAuditProgressCompleteByRange(Database cx,
-                                                     AuditType auditType,
-                                                     UID auditId,
-                                                     KeyRange auditRange) {
+Future<bool> checkAuditProgressCompleteByRange(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) {
 	ASSERT(auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica ||
 	       auditType == AuditType::ValidateLocationMetadata || auditType == AuditType::ValidateRestore);
-	state KeyRange rangeToRead = auditRange;
-	state Key rangeToReadBegin = auditRange.begin;
-	state int retryCount = 0;
+	KeyRange rangeToRead = auditRange;
+	Key rangeToReadBegin = auditRange.begin;
+	int retryCount = 0;
 	while (rangeToReadBegin < auditRange.end) {
-		loop {
+		while (true) {
+			Error err;
 			try {
 				rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end);
-				state std::vector<AuditStorageState> auditStates =
-				    wait(getAuditStateByRange(cx, auditType, auditId, rangeToRead));
+				std::vector<AuditStorageState> auditStates =
+				    co_await getAuditStateByRange(cx, auditType, auditId, rangeToRead);
 				for (int i = 0; i < auditStates.size(); i++) {
 					AuditPhase phase = auditStates[i].getPhase();
 					if (phase == AuditPhase::Invalid) {
@@ -693,50 +688,52 @@ ACTOR Future<bool> checkAuditProgressCompleteByRange(Database cx,
 						    .detail("AuditRange", auditRange)
 						    .detail("AuditType", auditType)
 						    .detail("UnfinishedRange", auditStates[i].range);
-						return false;
+						co_return false;
 					}
 				}
 				rangeToReadBegin = auditStates.back().range.end;
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				if (retryCount > 30) {
-					TraceEvent(SevWarn, "AuditUtilCheckAuditProgressFailed")
-					    .detail("AuditID", auditId)
-					    .detail("AuditRange", auditRange)
-					    .detail("AuditType", auditType);
-					throw audit_storage_failed();
-				}
-				wait(delay(0.5));
-				retryCount++;
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
 			}
+			if (retryCount > 30) {
+				TraceEvent(SevWarn, "AuditUtilCheckAuditProgressFailed")
+				    .detail("AuditID", auditId)
+				    .detail("AuditRange", auditRange)
+				    .detail("AuditType", auditType);
+				throw audit_storage_failed();
+			}
+			co_await delay(0.5);
+			retryCount++;
 		}
 	}
 	TraceEvent(SevInfo, "AuditUtilCheckAuditProgressFinish")
 	    .detail("AuditID", auditId)
 	    .detail("AuditRange", auditRange)
 	    .detail("AuditType", auditType);
-	return true;
+	co_return true;
 }
 
-ACTOR Future<bool> checkAuditProgressCompleteByServer(Database cx,
-                                                      AuditType auditType,
-                                                      UID auditId,
-                                                      KeyRange auditRange,
-                                                      UID serverId,
-                                                      std::shared_ptr<AsyncVar<int>> checkProgressBudget) {
+Future<bool> checkAuditProgressCompleteByServer(Database cx,
+                                                AuditType auditType,
+                                                UID auditId,
+                                                KeyRange auditRange,
+                                                UID serverId,
+                                                std::shared_ptr<AsyncVar<int>> checkProgressBudget) {
 	ASSERT(auditType == AuditType::ValidateStorageServerShard);
-	state KeyRange rangeToRead = auditRange;
-	state Key rangeToReadBegin = auditRange.begin;
-	state int retryCount = 0;
+	KeyRange rangeToRead = auditRange;
+	Key rangeToReadBegin = auditRange.begin;
+	int retryCount = 0;
 	while (rangeToReadBegin < auditRange.end) {
-		loop {
+		while (true) {
+			Error err;
 			try {
 				rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end);
-				state std::vector<AuditStorageState> auditStates =
-				    wait(getAuditStateByServer(cx, auditType, auditId, serverId, rangeToRead));
+				std::vector<AuditStorageState> auditStates =
+				    co_await getAuditStateByServer(cx, auditType, auditId, serverId, rangeToRead);
 				for (int i = 0; i < auditStates.size(); i++) {
 					AuditPhase phase = auditStates[i].getPhase();
 					if (phase == AuditPhase::Invalid) {
@@ -747,27 +744,28 @@ ACTOR Future<bool> checkAuditProgressCompleteByServer(Database cx,
 						    .detail("AuditType", auditType)
 						    .detail("UnfinishedRange", auditStates[i].range);
 						checkProgressBudget->set(checkProgressBudget->get() + 1);
-						return false;
+						co_return false;
 					}
 				}
 				rangeToReadBegin = auditStates.back().range.end;
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				if (retryCount > 30) {
-					TraceEvent(SevWarn, "AuditUtilCheckAuditProgressFailed")
-					    .detail("ServerID", serverId)
-					    .detail("AuditID", auditId)
-					    .detail("AuditRange", auditRange)
-					    .detail("AuditType", auditType);
-					checkProgressBudget->set(checkProgressBudget->get() + 1);
-					throw audit_storage_failed();
-				}
-				wait(delay(0.5));
-				retryCount++;
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			}
+			if (retryCount > 30) {
+				TraceEvent(SevWarn, "AuditUtilCheckAuditProgressFailed")
+				    .detail("ServerID", serverId)
+				    .detail("AuditID", auditId)
+				    .detail("AuditRange", auditRange)
+				    .detail("AuditType", auditType);
+				checkProgressBudget->set(checkProgressBudget->get() + 1);
+				throw audit_storage_failed();
 			}
+			co_await delay(0.5);
+			retryCount++;
 		}
 	}
 	checkProgressBudget->set(checkProgressBudget->get() + 1);
@@ -776,21 +774,22 @@ ACTOR Future<bool> checkAuditProgressCompleteByServer(Database cx,
 	    .detail("AuditID", auditId)
 	    .detail("AuditRange", auditRange)
 	    .detail("AuditType", auditType);
-	return true;
+	co_return true;
 }
 
 // Load RUNNING audit states to resume, clean up COMPLETE and FAILED audit states
 // Update ddId for RUNNING audit states
-ACTOR Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
-                                                               MoveKeyLockInfo lock,
-                                                               bool ddEnabled,
-                                                               UID dataDistributorId,
-                                                               int persistFinishAuditCount) {
-	state std::unordered_map<AuditType, std::vector<AuditStorageState>> existingAuditStates;
-	state std::vector<AuditStorageState> auditStatesToResume;
-	state Transaction tr(cx);
-	state int retryCount = 0;
-	loop {
+Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
+                                                         MoveKeyLockInfo lock,
+                                                         bool ddEnabled,
+                                                         UID dataDistributorId,
+                                                         int persistFinishAuditCount) {
+	std::unordered_map<AuditType, std::vector<AuditStorageState>> existingAuditStates;
+	std::vector<AuditStorageState> auditStatesToResume;
+	Transaction tr(cx);
+	int retryCount = 0;
+	while (true) {
+		Error err;
 		try {
 			// Load existing audit states and update ddId in audit states
 			existingAuditStates.clear();
@@ -798,8 +797,8 @@ ACTOR Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			wait(checkMoveKeysLockForAudit(&tr, lock, ddEnabled, true));
-			RangeResult result = wait(tr.getRange(auditKeys, CLIENT_KNOBS->TOO_MANY));
+			co_await checkMoveKeysLockForAudit(&tr, lock, ddEnabled, true);
+			RangeResult result = co_await tr.getRange(auditKeys, CLIENT_KNOBS->TOO_MANY);
 			if (result.more || result.size() >= CLIENT_KNOBS->TOO_MANY) {
 				TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
 				           "AuditUtilLoadMetadataIncomplete",
@@ -807,8 +806,8 @@ ACTOR Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
 				    .detail("ResMore", result.more)
 				    .detail("ResSize", result.size());
 			}
-			for (int i = 0; i < result.size(); ++i) {
-				auto auditState = decodeAuditStorageState(result[i].value);
+			for (const auto& auditKeyValue : result) {
+				auto auditState = decodeAuditStorageState(auditKeyValue.value);
 				TraceEvent(SevVerbose, "AuditUtilLoadMetadataEach", dataDistributorId)
 				    .detail("CurrentDDID", dataDistributorId)
 				    .detail("AuditDDID", auditState.ddId)
@@ -870,26 +869,27 @@ ACTOR Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
 					}
 				}
 			}
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled || e.code() == error_code_movekeys_conflict) {
-				throw e;
-			}
-			if (retryCount > 50) {
-				TraceEvent(SevWarnAlways, "AuditUtilInitAuditMetadataExceedRetryMax", dataDistributorId)
-				    .errorUnsuppressed(e);
-				break;
-			}
-			try {
-				wait(tr.onError(e));
-			} catch (Error& e) {
-				retryCount++;
-				tr.reset();
-			}
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled || err.code() == error_code_movekeys_conflict) {
+			throw err;
+		}
+		if (retryCount > 50) {
+			TraceEvent(SevWarnAlways, "AuditUtilInitAuditMetadataExceedRetryMax", dataDistributorId)
+			    .errorUnsuppressed(err);
+			break;
+		}
+		try {
+			co_await tr.onError(err);
+		} catch (Error&) {
+			retryCount++;
+			tr.reset();
 		}
 	}
-	return auditStatesToResume;
+	co_return auditStatesToResume;
 }
 
 // Check if any pair of ranges are exclusive with each other
@@ -1007,22 +1007,146 @@ Optional<std::pair<KeyRange, KeyRange>> rangesSame(std::vector<KeyRange> rangesA
 	return Optional<std::pair<KeyRange, KeyRange>>();
 }
 
+std::vector<LocationMetadataError> checkLocationMetadataConsistency(
+    const std::unordered_map<UID, std::vector<KeyRange>>& mapFromKeyServers,
+    const std::unordered_map<UID, std::vector<KeyRange>>& mapFromServerKeys,
+    KeyRange claimRange) {
+	std::vector<LocationMetadataError> errors;
+	// Check mapFromKeyServers => mapFromServerKeys
+	for (const auto& [ssid, keyServerRanges] : mapFromKeyServers) {
+		auto it = mapFromServerKeys.find(ssid);
+		if (it == mapFromServerKeys.end()) {
+			LocationMetadataError err;
+			err.serverId = ssid;
+			err.message = format("KeyServers and serverKeys mismatch: Some key in range(%s, %s) exists "
+			                     "on Server(%s) in KeyServers but not ServerKeys",
+			                     claimRange.begin.printable().c_str(),
+			                     claimRange.end.printable().c_str(),
+			                     ssid.toString().c_str());
+			errors.push_back(std::move(err));
+			continue;
+		}
+		Optional<std::pair<KeyRange, KeyRange>> anyMismatch = rangesSame(keyServerRanges, it->second);
+		if (anyMismatch.present()) {
+			LocationMetadataError err;
+			err.serverId = ssid;
+			err.mismatchedRangeByKeyServer = anyMismatch.get().first;
+			err.mismatchedRangeByServerKey = anyMismatch.get().second;
+			err.message = format("KeyServers and serverKeys mismatch on Server(%s): KeyServer: %s; ServerKey: %s",
+			                     ssid.toString().c_str(),
+			                     anyMismatch.get().first.toString().c_str(),
+			                     anyMismatch.get().second.toString().c_str());
+			errors.push_back(std::move(err));
+		}
+	}
+	// Check mapFromServerKeys => mapFromKeyServers
+	for (const auto& [ssid, serverKeyRanges] : mapFromServerKeys) {
+		if (!mapFromKeyServers.contains(ssid)) {
+			LocationMetadataError err;
+			err.serverId = ssid;
+			err.message = format("KeyServers and serverKeys mismatch: Some key of range(%s, %s) exists "
+			                     "on Server(%s) in ServerKeys but not KeyServers",
+			                     claimRange.begin.printable().c_str(),
+			                     claimRange.end.printable().c_str(),
+			                     ssid.toString().c_str());
+			errors.push_back(std::move(err));
+		}
+	}
+	return errors;
+}
+
+LocationMetadataMaps buildLocationMetadataMaps(
+    const std::unordered_map<UID, std::vector<KeyRange>>& mapFromKeyServersRaw,
+    const std::unordered_map<UID, std::vector<KeyRange>>& serverOwnRangesMap,
+    KeyRange claimRange,
+    Optional<UID> traceId) {
+	LocationMetadataMaps result;
+	for (const auto& [ssid, ownRanges] : serverOwnRangesMap) {
+		for (const auto& range : ownRanges) {
+			KeyRange overlappingRange = range & claimRange;
+			if (overlappingRange.empty()) {
+				continue;
+			}
+			if (traceId.present()) {
+				TraceEvent(SevVerbose, "DDAuditLocationMetadataAddToServerKeyMap", traceId.get())
+				    .detail("RawRange", range)
+				    .detail("ClaimRange", claimRange)
+				    .detail("Range", overlappingRange)
+				    .detail("SSID", ssid);
+			}
+			result.fromServerKeys[ssid].push_back(overlappingRange);
+			result.numValidatedServerKeys++;
+		}
+	}
+	for (const auto& [ssid, ranges] : mapFromKeyServersRaw) {
+		std::vector mergedRanges = coalesceRangeList(ranges);
+		for (const auto& range : mergedRanges) {
+			KeyRange overlappingRange = range & claimRange;
+			if (overlappingRange.empty()) {
+				continue;
+			}
+			if (traceId.present()) {
+				TraceEvent(SevVerbose, "DDAuditLocationMetadataAddToKeyServerMap", traceId.get())
+				    .detail("RawRange", range)
+				    .detail("ClaimRange", claimRange)
+				    .detail("Range", overlappingRange)
+				    .detail("SSID", ssid);
+			}
+			result.fromKeyServers[ssid].push_back(overlappingRange);
+			result.numValidatedKeyServers++;
+		}
+	}
+	return result;
+}
+
+std::vector<KeyRange> buildOwnRangesFromServerKeysResult(const RangeResult& krmResult) {
+	std::vector<KeyRange> ownRanges;
+	for (int i = 0; i < krmResult.size() - 1; ++i) {
+		if (serverHasKey(krmResult[i].value)) {
+			ownRanges.push_back(Standalone(KeyRangeRef(krmResult[i].key, krmResult[i + 1].key)));
+		}
+	}
+	return ownRanges;
+}
+
+// src and dest UIDs from decodeKeyServersValue are expected to be sorted by the encoder
+KeyServersOwnershipMap buildOwnershipMapFromKeyServersResult(const RangeResult& krmResult,
+                                                             const RangeResult& uidToTagMap) {
+	KeyServersOwnershipMap result;
+	for (int i = 0; i < krmResult.size() - 1; ++i) {
+		std::vector<UID> src;
+		std::vector<UID> dest;
+		UID srcID;
+		UID destID;
+		decodeKeyServersValue(uidToTagMap, krmResult[i].value, src, dest, srcID, destID);
+		if (srcID == anonymousShardId) {
+			result.shardsInAnonymousPhysicalShardCount++;
+		}
+		result.totalShardsCount++;
+		std::vector<UID> servers(src.size() + dest.size());
+		std::merge(src.begin(), src.end(), dest.begin(), dest.end(), servers.begin());
+		for (auto& ssid : servers) {
+			result.rangeOwnershipMap[ssid].push_back(Standalone(KeyRangeRef(krmResult[i].key, krmResult[i + 1].key)));
+		}
+	}
+	return result;
+}
+
 // Given an input server, get ranges within the input range via the input transaction
 // from the perspective of ServerKeys system key space
 // Input: (1) SS id; (2) transaction tr; (3) within range
 // Return AuditGetServerKeysRes, including: (1) complete range by a single read range;
 // (2) version of the read; (3) ranges of the input SS
-ACTOR Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID, Transaction* tr, KeyRange range) {
-	state RangeResult readResult;
-	state AuditGetServerKeysRes res;
+Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID, Transaction* tr, KeyRange range) {
+	RangeResult readResult;
+	AuditGetServerKeysRes res;
 
 	try {
-		wait(store(readResult,
-		           krmGetRanges(tr,
-		                        serverKeysPrefixFor(serverID),
-		                        range,
-		                        CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-		                        CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES)));
+		readResult = co_await krmGetRanges(tr,
+		                                   serverKeysPrefixFor(serverID),
+		                                   range,
+		                                   CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+		                                   CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 		Future<Version> grvF = tr->getReadVersion();
 		if (!grvF.isReady()) {
 			TraceEvent(SevWarnAlways, "AuditUtilReadServerKeysGRVError", serverID);
@@ -1036,17 +1160,13 @@ ACTOR Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID
 		    .detail("ResultSize", readResult.size())
 		    .detail("AuditServerID", serverID);
 
-		std::vector<KeyRange> ownRanges;
+		std::vector<KeyRange> ownRanges = buildOwnRangesFromServerKeysResult(readResult);
 		for (int i = 0; i < readResult.size() - 1; ++i) {
 			TraceEvent(SevVerbose, "AuditUtilGetThisServerKeysFromServerKeysAddToResult", serverID)
 			    .detail("ValueIsServerKeysFalse", readResult[i].value == serverKeysFalse)
 			    .detail("ServerHasKey", serverHasKey(readResult[i].value))
 			    .detail("Range", KeyRangeRef(readResult[i].key, readResult[i + 1].key))
 			    .detail("AuditServerID", serverID);
-			if (serverHasKey(readResult[i].value)) {
-				KeyRange shardRange;
-				ownRanges.push_back(Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key)));
-			}
 		}
 		const KeyRange completeRange = Standalone(KeyRangeRef(range.begin, readResult.back().key));
 		TraceEvent(SevVerbose, "AuditUtilGetThisServerKeysFromServerKeysEnd", serverID)
@@ -1065,7 +1185,7 @@ ACTOR Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID
 		throw e;
 	}
 
-	return res;
+	co_return res;
 }
 
 // Given an input server, get ranges within the input range via the input transaction
@@ -1073,13 +1193,11 @@ ACTOR Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID
 // Input: (1) Audit Server ID (for logging); (2) transaction tr; (3) within range
 // Return AuditGetKeyServersRes, including : (1) complete range by a single read range; (2) version of the read;
 // (3) map between SSes and their ranges --- in KeyServers space, a range corresponds to multiple SSes
-ACTOR Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId, Transaction* tr, KeyRange range) {
-	state AuditGetKeyServersRes res;
-	state std::vector<Future<Void>> actors;
-	state RangeResult readResult;
-	state RangeResult UIDtoTagMap;
-	state int64_t totalShardsCount = 0;
-	state int64_t shardsInAnonymousPhysicalShardCount = 0;
+Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId, Transaction* tr, KeyRange range) {
+	AuditGetKeyServersRes res;
+	std::vector<Future<Void>> actors;
+	RangeResult readResult;
+	RangeResult UIDtoTagMap;
 
 	try {
 		// read
@@ -1090,7 +1208,7 @@ ACTOR Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId,
 		                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
 		                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES)));
 		actors.push_back(store(UIDtoTagMap, tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY)));
-		wait(waitForAll(actors));
+		co_await waitForAll(actors);
 		if (UIDtoTagMap.more || UIDtoTagMap.size() >= CLIENT_KNOBS->TOO_MANY) {
 			TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
 			           "AuditUtilReadKeyServersReadTagError",
@@ -1110,31 +1228,16 @@ ACTOR Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId,
 		    .detail("AuditServerID", auditServerId);
 
 		// produce result
-		std::unordered_map<UID, std::vector<KeyRange>> serverOwnRanges;
-		for (int i = 0; i < readResult.size() - 1; ++i) {
-			std::vector<UID> src;
-			std::vector<UID> dest;
-			UID srcID;
-			UID destID;
-			decodeKeyServersValue(UIDtoTagMap, readResult[i].value, src, dest, srcID, destID);
-			if (srcID == anonymousShardId) {
-				shardsInAnonymousPhysicalShardCount++;
-			}
-			totalShardsCount++;
-			std::vector<UID> servers(src.size() + dest.size());
-			std::merge(src.begin(), src.end(), dest.begin(), dest.end(), servers.begin());
-			for (auto& ssid : servers) {
-				serverOwnRanges[ssid].push_back(Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key)));
-			}
-		}
+		KeyServersOwnershipMap ownershipResult = buildOwnershipMapFromKeyServersResult(readResult, UIDtoTagMap);
 		const KeyRange completeRange = Standalone(KeyRangeRef(range.begin, readResult.back().key));
 		TraceEvent(SevInfo, "AuditUtilGetThisServerKeysFromKeyServersEnd", auditServerId)
 		    .detail("Range", range)
 		    .detail("CompleteRange", completeRange)
 		    .detail("AtVersion", readAtVersion)
-		    .detail("ShardsInAnonymousPhysicalShardCount", shardsInAnonymousPhysicalShardCount)
-		    .detail("TotalShardsCount", totalShardsCount);
-		res = AuditGetKeyServersRes(completeRange, readAtVersion, serverOwnRanges, readResult.logicalSize());
+		    .detail("ShardsInAnonymousPhysicalShardCount", ownershipResult.shardsInAnonymousPhysicalShardCount)
+		    .detail("TotalShardsCount", ownershipResult.totalShardsCount);
+		res = AuditGetKeyServersRes(
+		    completeRange, readAtVersion, std::move(ownershipResult.rangeOwnershipMap), readResult.logicalSize());
 
 	} catch (Error& e) {
 		TraceEvent(SevDebug, "AuditUtilGetThisServerKeysFromKeyServersError", auditServerId)
@@ -1143,5 +1246,5 @@ ACTOR Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId,
 		throw e;
 	}
 
-	return res;
+	co_return res;
 }
diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.cpp
similarity index 74%
rename from fdbclient/BackupAgentBase.actor.cpp
rename to fdbclient/BackupAgentBase.cpp
index 3b9a832194e..e5018393db8 100644
--- a/fdbclient/BackupAgentBase.actor.cpp
+++ b/fdbclient/BackupAgentBase.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupAgentBase.actor.cpp
+ * BackupAgentBase.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,20 +21,18 @@
 #include <iomanip>
 #include <time.h>
 
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/SystemData.h"
 #include "fdbrpc/simulator.h"
 #include "flow/ActorCollection.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/network.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 std::string BackupAgentBase::formatTime(int64_t epochs) {
 	time_t curTime = (time_t)epochs;
 	char buffer[30];
@@ -279,19 +277,19 @@ void _addResult(VectorRef<MutationRef>* result, int* mutationSize, Arena* arena,
  "result" vector.
  Each `value` is a param2
 */
-ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
-                                               VectorRef<MutationRef>* result,
-                                               int* mutationSize,
-                                               Standalone<StringRef> value,
-                                               Key addPrefix,
-                                               Key removePrefix,
-                                               Version version,
-                                               Reference<KeyRangeMap<Version>> key_version,
-                                               Database cx,
-                                               bool provisionalProxy,
-                                               std::shared_ptr<DatabaseConfiguration> dbConfig) {
+static Future<Void> decodeBackupLogValue(Arena* arena,
+                                         VectorRef<MutationRef>* result,
+                                         int* mutationSize,
+                                         Standalone<StringRef> value,
+                                         Key addPrefix,
+                                         Key removePrefix,
+                                         Version version,
+                                         Reference<KeyRangeMap<Version>> key_version,
+                                         Database cx,
+                                         bool provisionalProxy,
+                                         std::shared_ptr<DatabaseConfiguration> dbConfig) {
 	try {
-		state uint64_t offset(0);
+		uint64_t offset(0);
 		uint64_t protocolVersion = 0;
 		memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
 		offset += sizeof(uint64_t);
@@ -303,10 +301,10 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
 			throw incompatible_protocol_version();
 		}
 
-		state uint32_t totalBytes = 0;
+		uint32_t totalBytes = 0;
 		memcpy(&totalBytes, value.begin() + offset, sizeof(uint32_t));
 		offset += sizeof(uint32_t);
-		state uint32_t consumed = 0;
+		uint32_t consumed = 0;
 
 		if (totalBytes + offset > value.size()) {
 			TraceEvent(SevError, "OffsetOutOfBoundary")
@@ -317,25 +315,25 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
 			throw restore_missing_data();
 		}
 
-		state int originalOffset = offset;
+		int originalOffset = offset;
 
 		while (consumed < totalBytes) {
 			uint32_t type = 0;
 			// encoding format: type|kLen|vLen|Key|Value
 			memcpy(&type, value.begin() + offset, sizeof(uint32_t));
 			offset += sizeof(uint32_t);
-			state uint32_t len1 = 0;
+			uint32_t len1 = 0;
 			memcpy(&len1, value.begin() + offset, sizeof(uint32_t));
 			offset += sizeof(uint32_t);
-			state uint32_t len2 = 0;
+			uint32_t len2 = 0;
 			memcpy(&len2, value.begin() + offset, sizeof(uint32_t));
 			offset += sizeof(uint32_t);
 
 			ASSERT(offset + len1 + len2 <= value.size() && isValidMutationType(type));
 
 			// Construct MutationRef from StringRef
-			state MutationRef logValue;
-			state Arena tempArena;
+			MutationRef logValue;
+			Arena tempArena;
 			logValue.type = type;
 			logValue.param1 = value.substr(offset, len1);
 			offset += len1;
@@ -348,24 +346,24 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
 				for (auto r : ranges) {
 					if (version > r.value() && r.value() != invalidVersion) {
 						KeyRef minKey = std::min(r.range().end, range.end);
-						if (minKey == (removePrefix == StringRef() ? allKeys.end : strinc(removePrefix))) {
+						if (minKey == (removePrefix.empty() ? allKeys.end : strinc(removePrefix))) {
 							logValue.param1 = std::max(r.range().begin, range.begin);
-							if (removePrefix.size()) {
+							if (!removePrefix.empty()) {
 								logValue.param1 = logValue.param1.removePrefix(removePrefix);
 							}
-							if (addPrefix.size()) {
+							if (!addPrefix.empty()) {
 								logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
 							}
-							logValue.param2 = addPrefix == StringRef() ? allKeys.end : strinc(addPrefix, tempArena);
+							logValue.param2 = addPrefix.empty() ? allKeys.end : strinc(addPrefix, tempArena);
 							_addResult(result, mutationSize, arena, logValue);
 						} else {
 							logValue.param1 = std::max(r.range().begin, range.begin);
 							logValue.param2 = minKey;
-							if (removePrefix.size()) {
+							if (!removePrefix.empty()) {
 								logValue.param1 = logValue.param1.removePrefix(removePrefix);
 								logValue.param2 = logValue.param2.removePrefix(removePrefix);
 							}
-							if (addPrefix.size()) {
+							if (!addPrefix.empty()) {
 								logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
 								logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena);
 							}
@@ -380,10 +378,10 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
 				// ver: the old version stored in keyVersionMap
 				// as a result, only add this mutation in log when the version is larger(to work with range file)
 				if (version > ver && ver != invalidVersion) {
-					if (removePrefix.size()) {
+					if (!removePrefix.empty()) {
 						logValue.param1 = logValue.param1.removePrefix(removePrefix);
 					}
-					if (addPrefix.size()) {
+					if (!addPrefix.empty()) {
 						logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
 					}
 					_addResult(result, mutationSize, arena, logValue);
@@ -438,33 +436,35 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
 	return logError(tr->getDatabase(), keyErrors, message);
 }
 
-ACTOR Future<Void> readCommitted(Database cx,
-                                 PromiseStream<RangeResultWithVersion> results,
-                                 Reference<FlowLock> lock,
-                                 KeyRangeRef range,
-                                 Terminator terminator,
-                                 AccessSystemKeys systemAccess,
-                                 LockAware lockAware,
-                                 ReadLowPriority readLowPriority) {
-	state KeySelector begin = firstGreaterOrEqual(range.begin);
-	state KeySelector end = firstGreaterOrEqual(range.end);
-	state Transaction tr(cx);
-	state FlowLock::Releaser releaser;
-
-	loop {
+Future<Void> readCommitted(Database cx,
+                           PromiseStream<RangeResultWithVersion> results,
+                           Reference<FlowLock> lock,
+                           KeyRangeRef range,
+                           Terminator terminator,
+                           AccessSystemKeys systemAccess,
+                           LockAware lockAware,
+                           ReadLowPriority readLowPriority) {
+	KeySelector begin = firstGreaterOrEqual(range.begin);
+	KeySelector end = firstGreaterOrEqual(range.end);
+	Transaction tr(cx);
+	FlowLock::Releaser releaser;
+
+	while (true) {
+		Error err;
 		try {
-			state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
-			                            (g_network->isSimulated() && !g_simulator->speedUpSimulation)
-			                                ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
-			                                : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
+			GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
+			                      (g_network->isSimulated() && !g_simulator->speedUpSimulation)
+			                          ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
+			                          : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
 
 			if (systemAccess)
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			if (lockAware)
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			if (readLowPriority) {
 				tr.setOption(FDBTransactionOptions::READ_PRIORITY_LOW);
 			}
+			tr.setOption(FDBTransactionOptions::READ_SERVER_SIDE_CACHE_DISABLE);
 			if (CLIENT_KNOBS->ENABLE_REPLICA_CONSISTENCY_CHECK_ON_BACKUP_READS) {
 				tr.setOption(FDBTransactionOptions::ENABLE_REPLICA_CONSISTENCY_CHECK);
 				int64_t requiredReplicas = CLIENT_KNOBS->BACKUP_CONSISTENCY_CHECK_REQUIRED_REPLICAS;
@@ -474,12 +474,12 @@ ACTOR Future<Void> readCommitted(Database cx,
 
 			// add lock
 			releaser.release();
-			wait(lock->take(TaskPriority::DefaultYield,
-			                limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
+			co_await lock->take(TaskPriority::DefaultYield,
+			                    limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
 			releaser = FlowLock::Releaser(
 			    *lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
 
-			state RangeResult values = wait(tr.getRange(begin, end, limits));
+			RangeResult values = co_await tr.getRange(begin, end, limits);
 
 			// When this buggify line is enabled, if there are more than 1 result then use half of the results
 			// Copy the data instead of messing with the results directly to avoid TSS issues.
@@ -493,7 +493,7 @@ ACTOR Future<Void> readCommitted(Database cx,
 				values.more = true;
 				// Half of the time wait for this tr to expire so that the next read is at a different version
 				if (deterministicRandom()->random01() < 0.5)
-					wait(delay(6.0));
+					co_await delay(6.0);
 			}
 
 			releaser.remaining -=
@@ -502,51 +502,56 @@ ACTOR Future<Void> readCommitted(Database cx,
 
 			results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
 
-			if (values.size() > 0)
+			if (!values.empty())
 				begin = firstGreaterThan(values.end()[-1].key);
 
 			if (!values.more && !limits.isReached()) {
 				if (terminator)
 					results.sendError(end_of_stream());
-				return Void();
+				co_return;
 			}
+
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_transaction_too_old) {
-				// We are using this transaction until it's too old and then resetting to a fresh one,
-				// so we don't need to delay.
-				tr.fullReset();
-			} else {
-				wait(tr.onError(e));
-			}
+			err = e;
+		}
+
+		if (err.code() == error_code_transaction_too_old) {
+			// We are using this transaction until it's too old and then resetting to a fresh one,
+			// so we don't need to delay.
+			tr.fullReset();
+		} else {
+			co_await tr.onError(err);
 		}
 	}
 }
 
 // read each version, potentially multiple part within the same version
-ACTOR Future<Void> readCommitted(Database cx,
-                                 PromiseStream<RCGroup> results,
-                                 Future<Void> active,
-                                 Reference<FlowLock> lock,
-                                 KeyRangeRef range,
-                                 std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
-                                 Terminator terminator,
-                                 AccessSystemKeys systemAccess,
-                                 LockAware lockAware,
-                                 ReadLowPriority readLowPriority) {
-	state KeySelector nextKey = firstGreaterOrEqual(range.begin);
-	state KeySelector end = firstGreaterOrEqual(range.end);
-
-	state RCGroup rcGroup = RCGroup();
-	state uint64_t skipGroup(ULLONG_MAX);
-	state Transaction tr(cx);
-	state FlowLock::Releaser releaser;
-
-	loop {
+Future<Void> readCommitted(Database cx,
+                           PromiseStream<RCGroup> results,
+                           Future<Void> active,
+                           Reference<FlowLock> lock,
+                           KeyRangeRef range,
+                           std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
+                           Terminator terminator,
+                           AccessSystemKeys systemAccess,
+                           LockAware lockAware,
+                           ReadLowPriority readLowPriority) {
+	KeySelector nextKey = firstGreaterOrEqual(range.begin);
+	KeySelector end = firstGreaterOrEqual(range.end);
+
+	RCGroup rcGroup = RCGroup();
+	uint64_t skipGroup(ULLONG_MAX);
+	Transaction tr(cx);
+	FlowLock::Releaser releaser;
+
+	while (true) {
+		Error err;
 		try {
-			state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
-			                            (g_network->isSimulated() && !g_simulator->speedUpSimulation)
-			                                ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
-			                                : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
+			GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
+			                      (g_network->isSimulated() && !g_simulator->speedUpSimulation)
+			                          ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
+			                          : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
 
 			if (systemAccess)
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -555,8 +560,9 @@ ACTOR Future<Void> readCommitted(Database cx,
 			if (readLowPriority) {
 				tr.setOption(FDBTransactionOptions::READ_PRIORITY_LOW);
 			}
+			tr.setOption(FDBTransactionOptions::READ_SERVER_SIDE_CACHE_DISABLE);
 
-			state RangeResult rangevalue = wait(tr.getRange(nextKey, end, limits));
+			RangeResult rangevalue = co_await tr.getRange(nextKey, end, limits);
 
 			// When this buggify line is enabled, if there are more than 1 result then use half of the results.
 			// Copy the data instead of messing with the results directly to avoid TSS issues.
@@ -570,13 +576,13 @@ ACTOR Future<Void> readCommitted(Database cx,
 				rangevalue.more = true;
 				// Some of the time wait for this tr to expire so that the next read is at a different version
 				if (deterministicRandom()->random01() < 0.01)
-					wait(delay(6.0));
+					co_await delay(6.0);
 			}
 
 			// add lock
-			wait(active);
+			co_await active;
 			releaser.release();
-			wait(lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize()));
+			co_await lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize());
 			releaser = FlowLock::Releaser(*lock, rangevalue.expectedSize() + rcGroup.items.expectedSize());
 
 			// iterate on a version range, each key-value pair is (version, part)
@@ -632,18 +638,21 @@ ACTOR Future<Void> readCommitted(Database cx,
 
 				if (terminator)
 					results.sendError(end_of_stream());
-				return Void();
+				co_return;
 			}
 
 			nextKey = firstGreaterThan(rangevalue.end()[-1].key);
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_transaction_too_old) {
-				// We are using this transaction until it's too old and then resetting to a fresh one,
-				// so we don't need to delay.
-				tr.fullReset();
-			} else {
-				wait(tr.onError(e));
-			}
+			err = e;
+		}
+
+		if (err.code() == error_code_transaction_too_old) {
+			// We are using this transaction until it's too old and then resetting to a fresh one,
+			// so we don't need to delay.
+			tr.fullReset();
+		} else {
+			co_await tr.onError(err);
 		}
 	}
 }
@@ -668,15 +677,15 @@ Future<Void> readCommitted(Database cx,
 
 // restore transaction has to be first in the batch, or it is the only txn in batch to make sure it never conflicts with
 // others.
-ACTOR Future<Void> sendCommitTransactionRequest(CommitTransactionRequest req,
-                                                Key uid,
-                                                Version newBeginVersion,
-                                                Key rangeBegin,
-                                                NotifiedVersion* committedVersion,
-                                                int* totalBytes,
-                                                int* mutationSize,
-                                                FlowLock* commitLock,
-                                                PublicRequestStream<CommitTransactionRequest> commit) {
+Future<Void> sendCommitTransactionRequest(CommitTransactionRequest req,
+                                          Key uid,
+                                          Version newBeginVersion,
+                                          Key rangeBegin,
+                                          NotifiedVersion* committedVersion,
+                                          int* totalBytes,
+                                          int* mutationSize,
+                                          FlowLock* commitLock,
+                                          PublicRequestStream<CommitTransactionRequest> commit) {
 	Key applyBegin = uid.withPrefix(applyMutationsBeginRange.begin);
 	Key versionKey = BinaryWriter::toValue(newBeginVersion, Unversioned());
 	Key rangeEnd = getApplyKey(newBeginVersion, uid);
@@ -694,7 +703,7 @@ ACTOR Future<Void> sendCommitTransactionRequest(CommitTransactionRequest req,
 	req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
 
 	*totalBytes += *mutationSize;
-	wait(commitLock->take(TaskPriority::DefaultYield, *mutationSize));
+	co_await commitLock->take(TaskPriority::DefaultYield, *mutationSize);
 	Future<Void> commitAndUnlock = commitLock->releaseWhen(success(commit.getReply(req)), *mutationSize);
 
 	// NOTE: tenant has been removed, so revisit the following comment and possibly
@@ -704,41 +713,40 @@ ACTOR Future<Void> sendCommitTransactionRequest(CommitTransactionRequest req,
 	// Because we are bumping applyBegin version, we need to wait for the commit to be done.
 	// Otherwise, an update to the applyEnd key will trigger another applyMutation() which can
 	// have an overlapping range with the current applyMutation() and cause conflicts.
-	wait(commitAndUnlock);
-	return Void();
+	co_await commitAndUnlock;
 }
 
 // Decodes the backup mutation log and send the mutations to the CommitProxy.
 // The mutation logs are grouped by version and passed in as a stream of RCGroup from readCommitted().
 // The mutations are then decoded and sent to the CommitProxy in a batch.
-ACTOR Future<int> kvMutationLogToTransactions(Database cx,
-                                              PromiseStream<RCGroup> results,
-                                              Reference<FlowLock> lock,
-                                              Key uid,
-                                              Key addPrefix,
-                                              Key removePrefix,
-                                              PublicRequestStream<CommitTransactionRequest> commit,
-                                              NotifiedVersion* committedVersion,
-                                              Optional<Version> endVersion,
-                                              Key rangeBegin,
-                                              FlowLock* commitLock,
-                                              Reference<KeyRangeMap<Version>> keyVersion,
-                                              bool provisionalProxy,
-                                              std::shared_ptr<DatabaseConfiguration> dbConfig) {
-	state Version lastVersion = invalidVersion;
-	state bool endOfStream = false;
-	state int totalBytes = 0;
+Future<int> kvMutationLogToTransactions(Database cx,
+                                        PromiseStream<RCGroup> results,
+                                        Reference<FlowLock> lock,
+                                        Key uid,
+                                        Key addPrefix,
+                                        Key removePrefix,
+                                        PublicRequestStream<CommitTransactionRequest> commit,
+                                        NotifiedVersion* committedVersion,
+                                        Optional<Version> endVersion,
+                                        Key rangeBegin,
+                                        FlowLock* commitLock,
+                                        Reference<KeyRangeMap<Version>> keyVersion,
+                                        bool provisionalProxy,
+                                        std::shared_ptr<DatabaseConfiguration> dbConfig) {
+	Version lastVersion = invalidVersion;
+	bool endOfStream = false;
+	int totalBytes = 0;
 	// outer loop to batch multiple versions while inner loop for each version
-	loop {
-		state CommitTransactionRequest req;
-		state Version newBeginVersion = invalidVersion;
-		state int mutationSize = 0;
-		loop {
+	while (true) {
+		CommitTransactionRequest req;
+		Version newBeginVersion = invalidVersion;
+		int mutationSize = 0;
+		while (true) {
 			try {
-				state RCGroup group = waitNext(results.getFuture());
-				state CommitTransactionRequest curReq;
+				RCGroup group = co_await results.getFuture();
+				CommitTransactionRequest curReq;
 				lock->release(group.items.expectedSize());
-				state int curBatchMutationSize = 0;
+				int curBatchMutationSize = 0;
 
 				BinaryWriter bw(Unversioned());
 
@@ -749,17 +757,17 @@ ACTOR Future<int> kvMutationLogToTransactions(Database cx,
 				}
 				// Parse a single transaction from the backup mutation log
 				Standalone<StringRef> value = bw.toValue();
-				wait(decodeBackupLogValue(&curReq.arena,
-				                          &curReq.transaction.mutations,
-				                          &curBatchMutationSize,
-				                          value,
-				                          addPrefix,
-				                          removePrefix,
-				                          group.groupKey,
-				                          keyVersion,
-				                          cx,
-				                          provisionalProxy,
-				                          dbConfig));
+				co_await decodeBackupLogValue(&curReq.arena,
+				                              &curReq.transaction.mutations,
+				                              &curBatchMutationSize,
+				                              value,
+				                              addPrefix,
+				                              removePrefix,
+				                              group.groupKey,
+				                              keyVersion,
+				                              cx,
+				                              provisionalProxy,
+				                              dbConfig);
 
 				for (int i = 0; i < curReq.transaction.mutations.size(); i++) {
 					req.transaction.mutations.push_back_deep(req.arena, curReq.transaction.mutations[i]);
@@ -772,7 +780,7 @@ ACTOR Future<int> kvMutationLogToTransactions(Database cx,
 						newBeginVersion = endVersion.get();
 					}
 					if (newBeginVersion == invalidVersion)
-						return totalBytes;
+						co_return totalBytes;
 					endOfStream = true;
 					break;
 				}
@@ -780,25 +788,25 @@ ACTOR Future<int> kvMutationLogToTransactions(Database cx,
 			}
 		}
 		// TraceEvent("MutationLogRestore").detail("BeginVersion", newBeginVersion);
-		wait(sendCommitTransactionRequest(
-		    req, uid, newBeginVersion, rangeBegin, committedVersion, &totalBytes, &mutationSize, commitLock, commit));
+		co_await sendCommitTransactionRequest(
+		    req, uid, newBeginVersion, rangeBegin, committedVersion, &totalBytes, &mutationSize, commitLock, commit);
 		if (endOfStream) {
-			return totalBytes;
+			co_return totalBytes;
 		}
 	}
 }
 
-ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
-                                           Version endVersion,
-                                           Reference<KeyRangeMap<Version>> keyVersion,
-                                           PublicRequestStream<CommitTransactionRequest> commit,
-                                           NotifiedVersion* committedVersion,
-                                           PromiseStream<Future<Void>> addActor,
-                                           FlowLock* commitLock) {
+Future<Void> coalesceKeyVersionCache(Key uid,
+                                     Version endVersion,
+                                     Reference<KeyRangeMap<Version>> keyVersion,
+                                     PublicRequestStream<CommitTransactionRequest> commit,
+                                     NotifiedVersion* committedVersion,
+                                     PromiseStream<Future<Void>> addActor,
+                                     FlowLock* commitLock) {
 	Version lastVersion = -1000;
 	int64_t removed = 0;
-	state CommitTransactionRequest req;
-	state int64_t mutationSize = 0;
+	CommitTransactionRequest req;
+	int64_t mutationSize = 0;
 	Key mapPrefix = uid.withPrefix(applyMutationsKeyVersionMapRange.begin);
 
 	for (auto it : keyVersion->ranges()) {
@@ -834,58 +842,56 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
 		req.transaction.read_snapshot = committedVersion->get();
 		req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
 
-		wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
+		co_await commitLock->take(TaskPriority::DefaultYield, mutationSize);
 		addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> applyMutations(Database cx,
-                                  Key uid,
-                                  Key addPrefix,
-                                  Key removePrefix,
-                                  Version beginVersion,
-                                  Version* endVersion,
-                                  PublicRequestStream<CommitTransactionRequest> commit,
-                                  NotifiedVersion* committedVersion,
-                                  Reference<KeyRangeMap<Version>> keyVersion,
-                                  bool provisionalProxy) {
-	state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
-	state PromiseStream<Future<Void>> addActor;
-	state Future<Void> error = actorCollection(addActor.getFuture());
-	state int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES;
-	state std::shared_ptr<DatabaseConfiguration> dbConfig = std::make_shared<DatabaseConfiguration>();
+Future<Void> applyMutations(Database cx,
+                            Key uid,
+                            Key addPrefix,
+                            Key removePrefix,
+                            Version beginVersion,
+                            Version* endVersion,
+                            PublicRequestStream<CommitTransactionRequest> commit,
+                            NotifiedVersion* committedVersion,
+                            Reference<KeyRangeMap<Version>> keyVersion,
+                            bool provisionalProxy) {
+	FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
+	PromiseStream<Future<Void>> addActor;
+	Future<Void> error = actorCollection(addActor.getFuture());
+	int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES;
+	std::shared_ptr<DatabaseConfiguration> dbConfig = std::make_shared<DatabaseConfiguration>();
 
 	keyVersion->insert(metadataVersionKey, 0);
 
 	try {
-		wait(store(*dbConfig, getDatabaseConfiguration(cx)));
+		*dbConfig = co_await getDatabaseConfiguration(cx);
 
-		loop {
+		while (true) {
 			if (beginVersion >= *endVersion) {
 				// Why do we need to take a lock here?
-				wait(commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES));
+				co_await commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES);
 				commitLock.release(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
 				if (beginVersion >= *endVersion) {
-					return Void();
+					co_return;
 				}
 			}
 
 			int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
 			// newEndVersion can only be at most of size APPLY_BLOCK_SIZE
-			state Version newEndVersion = std::min(*endVersion,
-			                                       ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
-			                                           CLIENT_KNOBS->APPLY_BLOCK_SIZE);
+			Version newEndVersion = std::min(*endVersion,
+			                                 ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
+			                                     CLIENT_KNOBS->APPLY_BLOCK_SIZE);
 
 			// ranges each represent a partition of version, e.g. [100, 200], [201, 300], [301, 400]
 			// (64, 200) -> [(64, 128), (128, 192), (192, 200)] assuming block size is 64
 			// ranges have format: applyLogKeys.begin/uid/hash(uint8)/version(64bites)/part
-			state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
-			state size_t idx;
-			state std::vector<PromiseStream<RCGroup>> results;
-			state std::vector<Future<Void>> rc;
-			state std::vector<Reference<FlowLock>> locks;
+			Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
+			size_t idx{ 0 };
+			std::vector<PromiseStream<RCGroup>> results;
+			std::vector<Future<Void>> rc;
+			std::vector<Reference<FlowLock>> locks;
 
 			// each RCGroup is for a single version, each results[i] is for a single range who can have multiple
 			// versions
@@ -900,30 +906,30 @@ ACTOR Future<Void> applyMutations(Database cx,
 			maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
 			for (idx = 0; idx < ranges.size(); ++idx) {
 				int bytes =
-				    wait(kvMutationLogToTransactions(cx,
-				                                     results[idx],
-				                                     locks[idx],
-				                                     uid,
-				                                     addPrefix,
-				                                     removePrefix,
-				                                     commit,
-				                                     committedVersion,
-				                                     idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
-				                                     ranges[idx].begin,
-				                                     &commitLock,
-				                                     keyVersion,
-				                                     provisionalProxy,
-				                                     dbConfig));
+				    co_await kvMutationLogToTransactions(cx,
+				                                         results[idx],
+				                                         locks[idx],
+				                                         uid,
+				                                         addPrefix,
+				                                         removePrefix,
+				                                         commit,
+				                                         committedVersion,
+				                                         idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
+				                                         ranges[idx].begin,
+				                                         &commitLock,
+				                                         keyVersion,
+				                                         provisionalProxy,
+				                                         dbConfig);
 				maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
 				if (error.isError())
 					throw error.getError();
 			}
 
-			wait(coalesceKeyVersionCache(
-			    uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
+			co_await coalesceKeyVersionCache(
+			    uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock);
 			beginVersion = newEndVersion;
 			if (BUGGIFY) {
-				wait(delay(2.0));
+				co_await delay(2.0);
 			}
 		}
 	} catch (Error& e) {
@@ -937,17 +943,17 @@ ACTOR Future<Void> applyMutations(Database cx,
 	}
 }
 
-ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
-                                        Key logUidValue,
-                                        Key destUidValue,
-                                        Optional<Version> endVersion,
-                                        CheckBackupUID checkBackupUid,
-                                        Version backupUid) {
-	state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
-	state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
-
-	if (!destUidValue.size()) {
-		return Void();
+static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
+                                  Key logUidValue,
+                                  Key destUidValue,
+                                  Optional<Version> endVersion,
+                                  CheckBackupUID checkBackupUid,
+                                  Version backupUid) {
+	Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
+	Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
+
+	if (destUidValue.empty()) {
+		co_return;
 	}
 
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -956,16 +962,16 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
 	if (checkBackupUid) {
 		Subspace sourceStates =
 		    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
-		Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
+		Optional<Value> v = co_await tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId));
 		if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
-			return Void();
+			co_return;
 	}
 
-	state RangeResult backupVersions = wait(
-	    tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
+	RangeResult backupVersions = co_await tr->getRange(
+	    KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY);
 
 	// Make sure version history key does exist and lower the beginVersion if needed
-	state Version currBeginVersion = invalidVersion;
+	Version currBeginVersion = invalidVersion;
 	for (auto backupVersion : backupVersions) {
 		Key currLogUidValue = backupVersion.key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
 
@@ -977,15 +983,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
 
 	// Do not clear anything if version history key cannot be found
 	if (currBeginVersion == invalidVersion) {
-		return Void();
+		co_return;
 	}
 
-	state Version currEndVersion = std::numeric_limits<Version>::max();
+	Version currEndVersion = std::numeric_limits<Version>::max();
 	if (endVersion.present()) {
 		currEndVersion = std::min(currEndVersion, endVersion.get());
 	}
 
-	state Version nextSmallestVersion = currEndVersion;
+	Version nextSmallestVersion = currEndVersion;
 	bool clearLogRangesRequired = true;
 
 	// More than one backup/DR with the same range
@@ -1053,16 +1059,14 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
 	}
 
 	if (!endVersion.present() && backupVersions.size() == 1) {
-		RangeResult existingDestUidValues =
-		    wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
+		RangeResult existingDestUidValues = co_await tr->getRange(
+		    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY);
 		for (auto it : existingDestUidValues) {
 			if (it.value == destUidValue) {
 				tr->clear(it.key);
 			}
 		}
 	}
-
-	return Void();
 }
 
 Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
@@ -1074,45 +1078,45 @@ Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
 	return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
 }
 
-ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool deleteData) {
-	state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
+Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool deleteData) {
+	Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
 
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state Optional<Key> removingLogUid;
-	state std::set<Key> loggedLogUids;
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	Optional<Key> removingLogUid;
+	std::set<Key> loggedLogUids;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-			state RangeResult backupVersions = wait(tr->getRange(
-			    KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
-			state Version readVer = tr->getReadVersion().get();
+			RangeResult backupVersions = co_await tr->getRange(
+			    KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY);
+			Version readVer = tr->getReadVersion().get();
 
-			state Version minVersion = std::numeric_limits<Version>::max();
-			state Key minVersionLogUid;
+			Version minVersion = std::numeric_limits<Version>::max();
+			Key minVersionLogUid;
 
-			state int backupIdx = 0;
-			for (; backupIdx < backupVersions.size(); backupIdx++) {
-				state Version currVersion =
+			for (int backupIdx = 0; backupIdx < backupVersions.size(); backupIdx++) {
+				Version currVersion =
 				    BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
-				state Key currLogUid =
+				Key currLogUid =
 				    backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
 				if (currVersion < minVersion) {
 					minVersionLogUid = currLogUid;
 					minVersion = currVersion;
 				}
 
-				if (!loggedLogUids.count(currLogUid)) {
-					state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
-					                                                       .get(BackupAgentBase::keySourceStates)
-					                                                       .get(currLogUid)
-					                                                       .pack(DatabaseBackupAgent::keyStateStatus));
-					state Future<Optional<Value>> foundBackupKey = tr->get(
+				if (!loggedLogUids.contains(currLogUid)) {
+					Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
+					                                                 .get(BackupAgentBase::keySourceStates)
+					                                                 .get(currLogUid)
+					                                                 .pack(DatabaseBackupAgent::keyStateStatus));
+					Future<Optional<Value>> foundBackupKey = tr->get(
 					    Subspace(currLogUid.withPrefix("uid->config/"_sr).withPrefix(fileBackupPrefixRange.begin))
 					        .pack("stateEnum"_sr));
-					wait(success(foundDRKey) && success(foundBackupKey));
+					co_await (success(foundDRKey) && success(foundBackupKey));
 
 					if (foundDRKey.get().present() && foundBackupKey.get().present()) {
 						printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
@@ -1136,8 +1140,8 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
 				if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
 				    (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
 					removingLogUid = minVersionLogUid;
-					wait(eraseLogData(tr, minVersionLogUid, destUidValue));
-					wait(tr->commit());
+					co_await eraseLogData(tr, minVersionLogUid, destUidValue);
+					co_await tr->commit();
 					printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
 					       (readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
 				} else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
@@ -1158,30 +1162,35 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
 				       (readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
 			}
 
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR Future<Void> cleanupBackup(Database cx, DeleteData deleteData) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	loop {
+Future<Void> cleanupBackup(Database cx, DeleteData deleteData) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-			state RangeResult destUids = wait(
-			    tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
+			RangeResult destUids = co_await tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)),
+			                                             CLIENT_KNOBS->TOO_MANY);
 
 			for (auto destUid : destUids) {
-				wait(cleanupLogMutations(cx, destUid.value, deleteData));
+				co_await cleanupLogMutations(cx, destUid.value, deleteData);
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+
+		co_await tr->onError(err);
 	}
 }
 
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.cpp
similarity index 78%
rename from fdbclient/BackupContainer.actor.cpp
rename to fdbclient/BackupContainer.cpp
index 37e5e5f61e1..520eb271787 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupContainer.actor.cpp
+ * BackupContainer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,48 +18,31 @@
  * limitations under the License.
  */
 
-#include <cstdlib>
-#include <ostream>
-
-// FIXME: Trim this down
-#include "fmt/format.h"
-#include "flow/Platform.actor.h"
-#include "fdbclient/AsyncTaskThread.h"
 #include "fdbclient/BackupContainer.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/JsonBuilder.h"
 #include "flow/Arena.h"
 #include "flow/Trace.h"
-#include "flow/UnitTest.h"
-#include "flow/Hash3.h"
-#include "fdbrpc/AsyncFileReadAhead.actor.h"
-#include "fdbrpc/simulator.h"
 #include "flow/Platform.h"
-#include "fdbclient/AsyncFileS3BlobStore.actor.h"
 #ifdef BUILD_AZURE_BACKUP
 #include "fdbclient/BackupContainerAzureBlobStore.h"
 #endif
-#include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbclient/BackupContainerLocalDirectory.h"
-#include "fdbclient/BackupContainerS3BlobStore.h"
-#include "fdbclient/Status.h"
+#include "BackupContainerLocalDirectory.h"
+#include "BackupContainerBlobStore.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include <algorithm>
 #include <cinttypes>
-#include <time.h>
-#include "flow/actorcompiler.h" // has to be last include
 
 namespace IBackupFile_impl {
 
-ACTOR Future<Void> appendStringRefWithLen(Reference<IBackupFile> file, Standalone<StringRef> s) {
-	state uint32_t lenBuf = bigEndian32((uint32_t)s.size());
-	wait(file->append(&lenBuf, sizeof(lenBuf)));
-	wait(file->append(s.begin(), s.size()));
-	return Void();
+Future<Void> appendStringRefWithLen(Reference<IBackupFile> file, Standalone<StringRef> s) {
+	uint32_t lenBuf = bigEndian32((uint32_t)s.size());
+	co_await file->append(&lenBuf, sizeof(lenBuf));
+	co_await file->append(s.begin(), s.size());
 }
 
 } // namespace IBackupFile_impl
@@ -137,8 +120,9 @@ std::string BackupDescription::toString() const {
 
 	info.append(format("URL: %s\n", url.c_str()));
 	info.append(format("Restorable: %s\n", maxRestorableVersion.present() ? "true" : "false"));
-	info.append(format("Partitioned logs: %s\n", partitioned ? "true" : "false"));
+	info.append(format("Mutation Log Type: %s\n", mutationLogTypeToString(mutationLogType).c_str()));
 	info.append(format("File-level encryption: %s\n", fileLevelEncryption ? "true" : "false"));
+	info.append(format("Encryption block size: %d\n", encryptionBlockSize));
 
 	auto formatVersion = [&](Version v) {
 		std::string s;
@@ -158,13 +142,15 @@ std::string BackupDescription::toString() const {
 	};
 
 	for (const KeyspaceSnapshotFile& m : snapshots) {
-		info.append(
-		    format("Snapshot:  startVersion=%s  endVersion=%s  totalBytes=%lld  restorable=%s  expiredPct=%.2f\n",
-		           formatVersion(m.beginVersion).c_str(),
-		           formatVersion(m.endVersion).c_str(),
-		           m.totalSize,
-		           m.restorable.orDefault(false) ? "true" : "false",
-		           m.expiredPct(expiredEndVersion)));
+		const char* snapshotType = m.isBulkDump() ? "bulkdump" : "rangefile";
+		info.append(format(
+		    "Snapshot:  type=%s  startVersion=%s  endVersion=%s  totalBytes=%lld  restorable=%s  expiredPct=%.2f\n",
+		    snapshotType,
+		    formatVersion(m.beginVersion).c_str(),
+		    formatVersion(m.endVersion).c_str(),
+		    m.totalSize,
+		    m.restorable.orDefault(false) ? "true" : "false",
+		    m.expiredPct(expiredEndVersion)));
 	}
 
 	info.append(format("SnapshotBytes: %lld\n", snapshotBytes));
@@ -196,8 +182,9 @@ std::string BackupDescription::toJSON() const {
 	doc.setKey("SchemaVersion", "1.0.0");
 	doc.setKey("URL", url.c_str());
 	doc.setKey("Restorable", maxRestorableVersion.present());
-	doc.setKey("Partitioned", partitioned);
+	doc.setKey("MutationLogType", mutationLogTypeToString(mutationLogType).c_str());
 	doc.setKey("FileLevelEncryption", fileLevelEncryption);
+	doc.setKey("EncryptionBlockSize", encryptionBlockSize);
 
 	auto formatVersion = [&](Version v) {
 		JsonBuilderObject doc;
@@ -258,14 +245,15 @@ std::vector<std::string> IBackupContainer::getURLFormats() {
 		BackupContainerAzureBlobStore::getURLFormat(),
 #endif
 		BackupContainerLocalDirectory::getURLFormat(),
-		BackupContainerS3BlobStore::getURLFormat(),
+		BackupContainerBlobStore::getURLFormat(),
 	};
 }
 
 // Get an IBackupContainer based on a container URL string
 Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& url,
                                                             const Optional<std::string>& proxy,
-                                                            const Optional<std::string>& encryptionKeyFileName) {
+                                                            const Optional<std::string>& encryptionKeyFileName,
+                                                            int encryptionBlockSize) {
 	static std::map<std::string, Reference<IBackupContainer>> m_cache;
 
 	// In simulation, disable caching for blobstore:// URLs to prevent cross-process connection issues.
@@ -275,24 +263,26 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 	// when containers are cached globally and accessed by different simulated processes, the connection
 	// pools are shared across processes. This violates Sim2Conn's requirement that connections stay on
 	// their originating process, leading to assertion failures:
-	//   "g_simulator->getCurrentProcess() == self->peerProcess" at sim2.actor.cpp:500
+	//   "g_simulator->getCurrentProcess() == self->peerProcess" at sim2.cpp:500
 	//
 	// Fix: Disable caching for blobstore:// URLs in simulation so each process creates its own
-	// container with its own connection pool. File-based backups don't have this issue because they
-	// don't use process-bound network connections.
+	// container with its own connection pool. Also disable caching for file:// URLs in simulation:
+	// multiple concurrent backups to the same base URL can have different encryption settings.
 	//
 	// Note: This only affects simulation; production always uses the cache for performance.
-	bool skipCache = g_network && g_network->isSimulated() && isBlobstoreUrl(url);
+	bool skipCache = g_network && g_network->isSimulated() && (isBlobstoreUrl(url) || url.find("file://") == 0);
 
 	// Use a reference to the cache entry (for automatic cache population) unless we're skipping cache
 	Reference<IBackupContainer> r_local;
 	Reference<IBackupContainer>& r = skipCache ? r_local : m_cache[url];
-	if (r)
+	if (r) {
 		return r;
+	}
+
 	try {
 		StringRef u(url);
 		if (u.startsWith("file://"_sr)) {
-			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName);
+			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName, encryptionBlockSize);
 		} else if (u.startsWith("blobstore://"_sr)) {
 			std::string resource;
 			Optional<std::string> blobstoreProxy;
@@ -306,16 +296,13 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 			}
 
 			// The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options.
-			S3BlobStoreEndpoint::ParametersT backupParams;
-			Reference<S3BlobStoreEndpoint> bstore =
-			    S3BlobStoreEndpoint::fromString(url, blobstoreProxy, &resource, &lastOpenError, &backupParams);
+			IBlobStoreEndpoint::ParametersT backupParams;
+			Reference<IBlobStoreEndpoint> bstore =
+			    IBlobStoreEndpoint::fromString(url, blobstoreProxy, &resource, &lastOpenError, &backupParams);
 
-			if (resource.empty())
-				throw backup_invalid_url();
-			for (auto c : resource)
-				if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
-					throw backup_invalid_url();
-			r = makeReference<BackupContainerS3BlobStore>(bstore, resource, backupParams, encryptionKeyFileName, true);
+			BackupContainerBlobStore::validateBackupUrl(resource);
+			r = makeReference<BackupContainerBlobStore>(
+			    bstore, resource, backupParams, encryptionKeyFileName, encryptionBlockSize, /*isBackup=*/true);
 		}
 #ifdef BUILD_AZURE_BACKUP
 		else if (u.startsWith("azure://"_sr)) {
@@ -387,17 +374,17 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 
 // Get a list of URLS to backup containers based on some a shorter URL.  This function knows about some set of supported
 // URL types which support this sort of backup discovery.
-ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL, Optional<std::string> proxy) {
+Future<std::vector<std::string>> listContainers_impl(std::string baseURL, Optional<std::string> proxy) {
 	try {
 		StringRef u(baseURL);
 		if (u.startsWith("file://"_sr)) {
-			std::vector<std::string> results = wait(BackupContainerLocalDirectory::listURLs(baseURL));
-			return results;
+			std::vector<std::string> results = co_await BackupContainerLocalDirectory::listURLs(baseURL);
+			co_return results;
 		} else if (u.startsWith("blobstore://"_sr)) {
 			std::string resource;
 
-			S3BlobStoreEndpoint::ParametersT backupParams;
-			Reference<S3BlobStoreEndpoint> bstore = S3BlobStoreEndpoint::fromString(
+			IBlobStoreEndpoint::ParametersT backupParams;
+			Reference<IBlobStoreEndpoint> bstore = IBlobStoreEndpoint::fromString(
 			    baseURL, proxy, &resource, &IBackupContainer::lastOpenError, &backupParams);
 
 			if (!resource.empty()) {
@@ -408,10 +395,15 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL,
 			}
 
 			// Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name
-			BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {}, true);
-
-			std::vector<std::string> results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket()));
-			return results;
+			BackupContainerBlobStore dummy(bstore,
+			                               "dummy",
+			                               backupParams,
+			                               /*encryptionKeyFileName=*/{},
+			                               /*isBackup=*/true,
+			                               /*encryptionBlockSize=*/0);
+
+			std::vector<std::string> results = co_await BackupContainerBlobStore::listURLs(bstore, dummy.getBucket());
+			co_return results;
 		}
 		// TODO: Enable this when Azure backups are ready
 		/*
@@ -424,7 +416,6 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL,
 			IBackupContainer::lastOpenError = "invalid URL prefix";
 			throw backup_invalid_url();
 		}
-
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
 			throw;
@@ -445,27 +436,28 @@ Future<std::vector<std::string>> IBackupContainer::listContainers(const std::str
 	return listContainers_impl(baseURL, proxy);
 }
 
-ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Database db) {
-	state KeyBackedMap<int64_t, Version> versionMap(timeKeeperPrefixRange.begin);
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
+Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Database db) {
+	KeyBackedMap<int64_t, Version> versionMap(timeKeeperPrefixRange.begin);
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
 
-	state int64_t time = BackupAgentBase::parseTime(datetime);
+	int64_t time = BackupAgentBase::parseTime(datetime);
 	if (time < 0) {
 		fprintf(
 		    stderr, "ERROR: Incorrect date/time or format.  Format is %s.\n", BackupAgentBase::timeFormat().c_str());
 		throw backup_error();
 	}
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state KeyBackedRangeResult<std::pair<int64_t, Version>> rangeResult =
-			    wait(versionMap.getRange(tr, 0, time, 1, Snapshot::False, Reverse::True));
+			KeyBackedRangeResult<std::pair<int64_t, Version>> rangeResult =
+			    co_await versionMap.getRange(tr, 0, time, 1, Snapshot::False, Reverse::True);
 			if (rangeResult.results.size() != 1) {
 				// No key less than time was found in the database
 				// Look for a key >= time.
-				wait(store(rangeResult, versionMap.getRange(tr, time, std::numeric_limits<int64_t>::max(), 1)));
+				rangeResult = co_await versionMap.getRange(tr, time, std::numeric_limits<int64_t>::max(), 1);
 
 				if (rangeResult.results.size() != 1) {
 					fprintf(stderr, "ERROR: Unable to calculate a version for given date/time.\n");
@@ -475,42 +467,43 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
 
 			// Adjust version found by the delta between time and the time found and min with 0.
 			auto& result = rangeResult.results[0];
-			return std::max<Version>(0, result.second + (time - result.first) * CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
-
+			co_return std::max<Version>(0,
+			                            result.second + (time - result.first) * CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference<ReadYourWritesTransaction> tr) {
-	state KeyBackedMap<int64_t, Version> versionMap(timeKeeperPrefixRange.begin);
+Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference<ReadYourWritesTransaction> tr) {
+	KeyBackedMap<int64_t, Version> versionMap(timeKeeperPrefixRange.begin);
 
 	// Binary search to find the closest date with a version <= v
-	state int64_t min = 0;
-	state int64_t max = (int64_t)now();
-	state int64_t mid;
-	state std::pair<int64_t, Version> found;
+	int64_t min = 0;
+	int64_t max = (int64_t)now();
+	int64_t mid{ 0 };
+	std::pair<int64_t, Version> found;
 
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	loop {
+	while (true) {
 		mid = (min + max + 1) / 2; // ceiling
 
 		// Find the highest time < mid
-		state KeyBackedRangeResult<std::pair<int64_t, Version>> rangeResult =
-		    wait(versionMap.getRange(tr, min, mid, 1, Snapshot::False, Reverse::True));
+		KeyBackedRangeResult<std::pair<int64_t, Version>> rangeResult =
+		    co_await versionMap.getRange(tr, min, mid, 1, Snapshot::False, Reverse::True);
 
 		if (rangeResult.results.size() != 1) {
 			if (mid == min) {
 				// There aren't any records having a version < v, so just look for any record having a time < now
 				// and base a result on it
-				wait(store(rangeResult, versionMap.getRange(tr, 0, (int64_t)now(), 1)));
+				rangeResult = co_await versionMap.getRange(tr, 0, (int64_t)now(), 1);
 
 				if (rangeResult.results.size() != 1) {
 					// There aren't any timekeeper records to base a result on so return nothing
-					return Optional<int64_t>();
+					co_return Optional<int64_t>();
 				}
 
 				found = rangeResult.results[0];
@@ -533,5 +526,5 @@ ACTOR Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference
 		}
 	}
 
-	return found.first + (v - found.second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
+	co_return found.first + (v - found.second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
 }
diff --git a/fdbclient/BackupContainerS3BlobStore.actor.cpp b/fdbclient/BackupContainerBlobStore.cpp
similarity index 54%
rename from fdbclient/BackupContainerS3BlobStore.actor.cpp
rename to fdbclient/BackupContainerBlobStore.cpp
index f57bf53a929..505dea70a41 100644
--- a/fdbclient/BackupContainerS3BlobStore.actor.cpp
+++ b/fdbclient/BackupContainerBlobStore.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupContainerS3BlobStore.actor.cpp
+ * BackupContainerBlobStore.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,14 +18,14 @@
  * limitations under the License.
  */
 
-#include "fdbclient/AsyncFileS3BlobStore.actor.h"
-#include "fdbclient/BackupContainerS3BlobStore.h"
+#include "fdbclient/AsyncFileBlobStore.h"
+#include "BackupContainerBlobStore.h"
+#include "fdbclient/IBlobStore.h"
 #include "fdbrpc/AsyncFileEncrypted.h"
-#include "fdbrpc/AsyncFileReadAhead.actor.h"
+#include "fdbrpc/AsyncFileReadAhead.h"
 #include "fdbrpc/HTTP.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
-class BackupContainerS3BlobStoreImpl {
+class BackupContainerBlobStoreImpl {
 public:
 	// Backup files to under a single folder prefix with subfolders for each named backup
 	static const std::string DATAFOLDER;
@@ -34,9 +34,9 @@ class BackupContainerS3BlobStoreImpl {
 	// number of slashes so the backup names are kept in a separate folder tree from their actual data.
 	static const std::string INDEXFOLDER;
 
-	ACTOR static Future<std::vector<std::string>> listURLs(Reference<S3BlobStoreEndpoint> bstore, std::string bucket) {
-		state std::string basePath = INDEXFOLDER + '/';
-		S3BlobStoreEndpoint::ListResult contents = wait(bstore->listObjects(bucket, basePath));
+	static Future<std::vector<std::string>> listURLs(Reference<IBlobStoreEndpoint> bstore, std::string bucket) {
+		std::string basePath = INDEXFOLDER + '/';
+		IBlobStoreEndpoint::ListResult contents = co_await bstore->listObjects(bucket, basePath);
 		std::vector<std::string> results;
 		for (const auto& f : contents.objects) {
 			// URL decode the object name since S3 XML responses contain URL-encoded names
@@ -44,7 +44,7 @@ class BackupContainerS3BlobStoreImpl {
 			results.push_back(
 			    bstore->getResourceURL(decodedName.substr(basePath.size()), format("bucket=%s", bucket.c_str())));
 		}
-		return results;
+		co_return results;
 	}
 
 	class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
@@ -76,21 +76,21 @@ class BackupContainerS3BlobStoreImpl {
 		int64_t m_offset;
 	};
 
-	ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles(
-	    Reference<BackupContainerS3BlobStore> bc,
+	static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles(
+	    Reference<BackupContainerBlobStore> bc,
 	    std::string path,
 	    std::function<bool(std::string const&)> pathFilter) {
 		// pathFilter expects container based paths, so create a wrapper which converts a raw path
 		// to a container path by removing the known backup name prefix.
-		state int prefixTrim = bc->dataPath("").size();
+		int prefixTrim = bc->dataPath("").size();
 		std::function<bool(std::string const&)> rawPathFilter = [=](const std::string& folderPath) {
 			ASSERT(folderPath.size() >= prefixTrim);
 			return pathFilter(folderPath.substr(prefixTrim));
 		};
 
 		// Use flat listing for backup files to ensure all files are found regardless of directory structure
-		state S3BlobStoreEndpoint::ListResult result =
-		    wait(bc->m_bstore->listObjects(bc->m_bucket, bc->dataPath(path), Optional<char>(), 0, rawPathFilter));
+		IBlobStoreEndpoint::ListResult result =
+		    co_await bc->m_bstore->listObjects(bc->m_bucket, bc->dataPath(path), Optional<char>(), 0, rawPathFilter);
 		BackupContainerFileSystem::FilesAndSizesT files;
 		for (const auto& o : result.objects) {
 			ASSERT(o.name.size() >= prefixTrim);
@@ -98,51 +98,47 @@ class BackupContainerS3BlobStoreImpl {
 			std::string decodedName = HTTP::urlDecode(o.name);
 			files.push_back({ decodedName.substr(prefixTrim), o.size });
 		}
-		return files;
+		co_return files;
 	}
 
-	ACTOR static Future<Void> create(Reference<BackupContainerS3BlobStore> bc) {
-		wait(bc->m_bstore->createBucket(bc->m_bucket));
+	static Future<Void> create(Reference<BackupContainerBlobStore> bc) {
+		co_await bc->m_bstore->createBucket(bc->m_bucket);
 
 		// Check/create the index entry
-		bool exists = wait(bc->m_bstore->objectExists(bc->m_bucket, bc->indexEntry()));
+		bool exists = co_await bc->m_bstore->objectExists(bc->m_bucket, bc->indexEntry());
 		if (!exists) {
-			wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), ""));
+			co_await bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), "");
 		}
 
 		if (bc->usesEncryption()) {
-			wait(bc->encryptionSetupComplete());
+			co_await bc->encryptionSetupComplete();
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> deleteContainer(Reference<BackupContainerS3BlobStore> bc, int* pNumDeleted) {
-		bool e = wait(bc->exists());
+	static Future<Void> deleteContainer(Reference<BackupContainerBlobStore> bc, int* pNumDeleted) {
+		bool e = co_await bc->exists();
 		if (!e) {
 			TraceEvent(SevWarnAlways, "BackupContainerDoesNotExist").detail("URL", bc->getURL());
 			throw backup_does_not_exist();
 		}
 
 		// First delete everything under the data prefix in the bucket
-		wait(bc->m_bstore->deleteRecursively(bc->m_bucket, bc->dataPath(""), pNumDeleted));
+		co_await bc->m_bstore->deleteRecursively(bc->m_bucket, bc->dataPath(""), pNumDeleted);
 
 		// Now that all files are deleted, delete the index entry
-		wait(bc->m_bstore->deleteObject(bc->m_bucket, bc->indexEntry()));
-
-		return Void();
+		co_await bc->m_bstore->deleteObject(bc->m_bucket, bc->indexEntry());
 	}
 };
 
-const std::string BackupContainerS3BlobStoreImpl::DATAFOLDER = "data";
-const std::string BackupContainerS3BlobStoreImpl::INDEXFOLDER = "backups";
+const std::string BackupContainerBlobStoreImpl::DATAFOLDER = "data";
+const std::string BackupContainerBlobStoreImpl::INDEXFOLDER = "backups";
 
-std::string BackupContainerS3BlobStore::dataPath(const std::string& path) {
+std::string BackupContainerBlobStore::dataPath(const std::string& path) {
 	// if backup, include the backup data prefix.
 	// if m_name ends in a trailing slash, don't add another
 	std::string dataPath = "";
 	if (isBackup) {
-		dataPath = BackupContainerS3BlobStoreImpl::DATAFOLDER + "/";
+		dataPath = BackupContainerBlobStoreImpl::DATAFOLDER + "/";
 	}
 	if (!m_name.empty() && m_name.back() == '/') {
 		dataPath += m_name + path;
@@ -153,46 +149,57 @@ std::string BackupContainerS3BlobStore::dataPath(const std::string& path) {
 }
 
 // Get the path of the backups's index entry
-std::string BackupContainerS3BlobStore::indexEntry() {
+std::string BackupContainerBlobStore::indexEntry() {
 	ASSERT(isBackup);
-	return BackupContainerS3BlobStoreImpl::INDEXFOLDER + "/" + m_name;
+	return BackupContainerBlobStoreImpl::INDEXFOLDER + "/" + m_name;
 }
 
-BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
-                                                       const std::string& name,
-                                                       const S3BlobStoreEndpoint::ParametersT& params,
-                                                       const Optional<std::string>& encryptionKeyFileName,
-                                                       bool isBackup)
+BackupContainerBlobStore::BackupContainerBlobStore(Reference<IBlobStoreEndpoint> bstore,
+                                                   const std::string& name,
+                                                   const IBlobStoreEndpoint::ParametersT& params,
+                                                   const Optional<std::string>& encryptionKeyFileName,
+                                                   int encryptionBlockSize,
+                                                   bool isBackup)
   : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2"), isBackup(isBackup) {
 	setEncryptionKey(encryptionKeyFileName);
+	this->encryptionBlockSize = encryptionBlockSize;
 	// Currently only one parameter is supported, "bucket"
 	for (const auto& [name, value] : params) {
 		if (name == "bucket") {
 			m_bucket = value;
 			continue;
 		}
-		TraceEvent(SevWarn, "BackupContainerS3BlobStoreInvalidParameter").detail("Name", name).detail("Value", value);
+		TraceEvent(SevWarn, "BackupContainerBlobStoreInvalidParameter").detail("Name", name).detail("Value", value);
 		IBackupContainer::lastOpenError = format("Unknown URL parameter: '%s'", name.c_str());
 		throw backup_invalid_url();
 	}
 }
 
-void BackupContainerS3BlobStore::addref() {
-	return ReferenceCounted<BackupContainerS3BlobStore>::addref();
+void BackupContainerBlobStore::addref() {
+	return ReferenceCounted<BackupContainerBlobStore>::addref();
+}
+void BackupContainerBlobStore::delref() {
+	return ReferenceCounted<BackupContainerBlobStore>::delref();
 }
-void BackupContainerS3BlobStore::delref() {
-	return ReferenceCounted<BackupContainerS3BlobStore>::delref();
+
+std::string BackupContainerBlobStore::getURLFormat(bool withResource) {
+	return IBlobStoreEndpoint::getURLFormat(withResource) + " (Note: The 'bucket' parameter is required.)";
 }
 
-std::string BackupContainerS3BlobStore::getURLFormat() {
-	return S3BlobStoreEndpoint::getURLFormat(true) + " (Note: The 'bucket' parameter is required.)";
+void BackupContainerBlobStore::validateBackupUrl(const std::string& resource) {
+	if (resource.empty())
+		throw backup_invalid_url();
+
+	for (auto c : resource)
+		if (!isalnum(static_cast<unsigned char>(c)) && c != '_' && c != '-' && c != '.' && c != '/')
+			throw backup_invalid_url();
 }
 
-Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
-	Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreRead>(m_bstore, m_bucket, dataPath(path));
+Future<Reference<IAsyncFile>> BackupContainerBlobStore::readFile(const std::string& path) {
+	Reference<IAsyncFile> f = makeReference<AsyncFileBlobStoreRead>(m_bstore, m_bucket, dataPath(path));
 
 	if (usesEncryption() && !StringRef(path).startsWith("properties/"_sr)) {
-		f = makeReference<AsyncFileEncrypted>(f, AsyncFileEncrypted::Mode::READ_ONLY);
+		f = makeReference<AsyncFileEncrypted>(f, AsyncFileEncrypted::Mode::READ_ONLY, encryptionBlockSize);
 	}
 	if (m_bstore->knobs.enable_read_cache) {
 		f = makeReference<AsyncFileReadAheadCache>(f,
@@ -204,47 +211,46 @@ Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::st
 	return f;
 }
 
-Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<S3BlobStoreEndpoint> bstore,
-                                                                      const std::string& bucket) {
-	return BackupContainerS3BlobStoreImpl::listURLs(bstore, bucket);
+Future<std::vector<std::string>> BackupContainerBlobStore::listURLs(Reference<IBlobStoreEndpoint> bstore,
+                                                                    const std::string& bucket) {
+	return BackupContainerBlobStoreImpl::listURLs(bstore, bucket);
 }
 
-Future<Reference<IBackupFile>> BackupContainerS3BlobStore::writeFile(const std::string& path) {
-	Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreWrite>(m_bstore, m_bucket, dataPath(path));
+Future<Reference<IBackupFile>> BackupContainerBlobStore::writeFile(const std::string& path) {
+	Reference<IAsyncFile> f = makeReference<AsyncFileBlobStoreWrite>(m_bstore, m_bucket, dataPath(path));
 	if (usesEncryption() && !StringRef(path).startsWith("properties/"_sr)) {
-		f = makeReference<AsyncFileEncrypted>(f, AsyncFileEncrypted::Mode::APPEND_ONLY);
+		f = makeReference<AsyncFileEncrypted>(f, AsyncFileEncrypted::Mode::APPEND_ONLY, encryptionBlockSize);
 	}
-	return Future<Reference<IBackupFile>>(makeReference<BackupContainerS3BlobStoreImpl::BackupFile>(path, f));
+	return Future<Reference<IBackupFile>>(makeReference<BackupContainerBlobStoreImpl::BackupFile>(path, f));
 }
 
-Future<Void> BackupContainerS3BlobStore::writeEntireFile(const std::string& path, const std::string& fileContents) {
+Future<Void> BackupContainerBlobStore::writeEntireFile(const std::string& path, const std::string& fileContents) {
 	return m_bstore->writeEntireFile(m_bucket, dataPath(path), fileContents);
 }
 
-Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {
+Future<Void> BackupContainerBlobStore::deleteFile(const std::string& path) {
 	return m_bstore->deleteObject(m_bucket, dataPath(path));
 }
 
-Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerS3BlobStore::listFiles(
+Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerBlobStore::listFiles(
     const std::string& path,
     std::function<bool(std::string const&)> pathFilter) {
-	return BackupContainerS3BlobStoreImpl::listFiles(
-	    Reference<BackupContainerS3BlobStore>::addRef(this), path, pathFilter);
+	return BackupContainerBlobStoreImpl::listFiles(Reference<BackupContainerBlobStore>::addRef(this), path, pathFilter);
 }
 
-Future<Void> BackupContainerS3BlobStore::create() {
-	return BackupContainerS3BlobStoreImpl::create(Reference<BackupContainerS3BlobStore>::addRef(this));
+Future<Void> BackupContainerBlobStore::create() {
+	return BackupContainerBlobStoreImpl::create(Reference<BackupContainerBlobStore>::addRef(this));
 }
 
-Future<bool> BackupContainerS3BlobStore::exists() {
+Future<bool> BackupContainerBlobStore::exists() {
 	return m_bstore->objectExists(m_bucket, indexEntry());
 }
 
-Future<Void> BackupContainerS3BlobStore::deleteContainer(int* pNumDeleted) {
-	return BackupContainerS3BlobStoreImpl::deleteContainer(Reference<BackupContainerS3BlobStore>::addRef(this),
-	                                                       pNumDeleted);
+Future<Void> BackupContainerBlobStore::deleteContainer(int* pNumDeleted) {
+	return BackupContainerBlobStoreImpl::deleteContainer(Reference<BackupContainerBlobStore>::addRef(this),
+	                                                     pNumDeleted);
 }
 
-std::string BackupContainerS3BlobStore::getBucket() const {
+std::string BackupContainerBlobStore::getBucket() const {
 	return m_bucket;
 }
diff --git a/fdbclient/include/fdbclient/BackupContainerS3BlobStore.h b/fdbclient/BackupContainerBlobStore.h
similarity index 64%
rename from fdbclient/include/fdbclient/BackupContainerS3BlobStore.h
rename to fdbclient/BackupContainerBlobStore.h
index 9aeca0fb0cf..546430b0554 100644
--- a/fdbclient/include/fdbclient/BackupContainerS3BlobStore.h
+++ b/fdbclient/BackupContainerBlobStore.h
@@ -1,5 +1,5 @@
 /*
- * BackupContainerS3BlobStore.h
+ * BackupContainerBlobStore.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,16 +18,16 @@
  * limitations under the License.
  */
 
-#ifndef FDBCLIENT_BACKUP_CONTAINER_S3_BLOBSTORE_H
-#define FDBCLIENT_BACKUP_CONTAINER_S3_BLOBSTORE_H
+#ifndef FDBCLIENT_BACKUP_CONTAINER_BLOBSTORE_H
+#define FDBCLIENT_BACKUP_CONTAINER_BLOBSTORE_H
 #pragma once
 
-#include "fdbclient/AsyncFileS3BlobStore.actor.h"
+#include "fdbclient/AsyncFileBlobStore.h"
 #include "fdbclient/BackupContainerFileSystem.h"
+#include "fdbclient/IBlobStore.h"
 
-class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
-                                         ReferenceCounted<BackupContainerS3BlobStore> {
-	Reference<S3BlobStoreEndpoint> m_bstore;
+class BackupContainerBlobStore final : public BackupContainerFileSystem, ReferenceCounted<BackupContainerBlobStore> {
+	Reference<IBlobStoreEndpoint> m_bstore;
 	std::string m_name;
 
 	// All backup data goes into a single bucket
@@ -41,23 +41,26 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
 	// Get the path of the backups's index entry
 	std::string indexEntry();
 
-	friend class BackupContainerS3BlobStoreImpl;
+	friend class BackupContainerBlobStoreImpl;
 
 public:
-	BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
-	                           const std::string& name,
-	                           const S3BlobStoreEndpoint::ParametersT& params,
-	                           const Optional<std::string>& encryptionKeyFileName,
-	                           bool isBackup);
+	BackupContainerBlobStore(Reference<IBlobStoreEndpoint> bstore,
+	                         const std::string& name,
+	                         const IBlobStoreEndpoint::ParametersT& params,
+	                         const Optional<std::string>& encryptionKeyFileName,
+	                         int encryptionBlockSize,
+	                         bool isBackup);
 
 	void addref() override;
 	void delref() override;
 
-	static std::string getURLFormat();
+	static std::string getURLFormat(bool withResource = true);
+
+	static void validateBackupUrl(const std::string& resource);
 
 	Future<Reference<IAsyncFile>> readFile(const std::string& path) final;
 
-	static Future<std::vector<std::string>> listURLs(Reference<S3BlobStoreEndpoint> bstore, const std::string& bucket);
+	static Future<std::vector<std::string>> listURLs(Reference<IBlobStoreEndpoint> bstore, const std::string& bucket);
 
 	Future<Reference<IBackupFile>> writeFile(const std::string& path) final;
 
diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.cpp
similarity index 77%
rename from fdbclient/BackupContainerFileSystem.actor.cpp
rename to fdbclient/BackupContainerFileSystem.cpp
index e881a9e583e..69f38f76182 100644
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupContainerFileSystem.actor.cpp
+ * BackupContainerFileSystem.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,15 @@
  * limitations under the License.
  */
 
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "flow/BooleanParam.h"
 #ifdef BUILD_AZURE_BACKUP
 #include "fdbclient/BackupContainerAzureBlobStore.h"
 #endif
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbclient/BackupContainerLocalDirectory.h"
-#include "fdbclient/BackupContainerS3BlobStore.h"
+#include "BackupContainerLocalDirectory.h"
+#include "BackupContainerBlobStore.h"
 #include "fdbclient/JsonBuilder.h"
 #include "flow/StreamCipher.h"
 #include "flow/UnitTest.h"
@@ -34,29 +34,27 @@
 #include <algorithm>
 #include <cinttypes>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 class BackupContainerFileSystemImpl {
 public:
 	// TODO:  Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of
 	// megabytes.
-	ACTOR static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
+	static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
 	    Reference<BackupContainerFileSystem> bc,
 	    KeyspaceSnapshotFile snapshot) {
 		// Read the range file list for the specified version range, and then index them by fileName.
 		// This is so we can verify that each of the files listed in the manifest file are also in the container at this
 		// time.
-		std::vector<RangeFile> files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion));
-		state std::map<std::string, RangeFile> rangeIndex;
+		std::vector<RangeFile> files = co_await bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion);
+		std::map<std::string, RangeFile> rangeIndex;
 		for (auto& f : files)
 			rangeIndex[f.fileName] = std::move(f);
 
 		// Read the snapshot file, verify the version range, then find each of the range files by name in the index and
 		// return them.
-		state Reference<IAsyncFile> f = wait(bc->readFile(snapshot.fileName));
-		int64_t size = wait(f->size());
-		state Standalone<StringRef> buf = makeString(size);
-		wait(success(f->read(mutateString(buf), buf.size(), 0)));
+		Reference<IAsyncFile> f = co_await bc->readFile(snapshot.fileName);
+		int64_t size = co_await f->size();
+		Standalone<StringRef> buf = makeString(size);
+		co_await f->read(mutateString(buf), buf.size(), 0);
 		json_spirit::mValue json;
 		if (!json_spirit::read_string(buf.toString(), json)) {
 			fprintf(stderr,
@@ -126,13 +124,9 @@ class BackupContainerFileSystemImpl {
 			}
 		}
 
-		return std::make_pair(results, fileKeyRanges);
+		co_return std::make_pair(results, fileKeyRanges);
 	}
 
-	// Backup log types
-	static constexpr Version NON_PARTITIONED_MUTATION_LOG = 0;
-	static constexpr Version PARTITIONED_MUTATION_LOG = 1;
-
 	// Find what should be the filename of a path by finding whatever is after the last forward or backward slash, or
 	// failing to find those, the whole string.
 	static std::string fileNameOnly(const std::string& path) {
@@ -163,17 +157,17 @@ class BackupContainerFileSystemImpl {
 		return false;
 	}
 
-	ACTOR static Future<Void> writeKeyspaceSnapshotFile(Reference<BackupContainerFileSystem> bc,
-	                                                    std::vector<std::string> fileNames,
-	                                                    std::vector<std::pair<Key, Key>> beginEndKeys,
-	                                                    int64_t totalBytes,
-	                                                    IncludeKeyRangeMap includeKeyRangeMap,
-	                                                    Optional<SnapshotMetadata> metadata) {
-		state Version minVer = std::numeric_limits<Version>::max();
-		state Version maxVer = 0;
-		state RangeFile rf;
-		state json_spirit::mArray fileArray;
-		state bool isBulkDump = metadata.present() && metadata.get().isBulkDump();
+	static Future<Void> writeKeyspaceSnapshotFile(Reference<BackupContainerFileSystem> bc,
+	                                              std::vector<std::string> fileNames,
+	                                              std::vector<std::pair<Key, Key>> beginEndKeys,
+	                                              int64_t totalBytes,
+	                                              IncludeKeyRangeMap includeKeyRangeMap,
+	                                              Optional<SnapshotMetadata> metadata) {
+		Version minVer = std::numeric_limits<Version>::max();
+		Version maxVer = 0;
+		RangeFile rf;
+		json_spirit::mArray fileArray;
+		bool isBulkDump = metadata.present() && metadata.get().isBulkDump();
 
 		if (isBulkDump) {
 			// For BulkDump snapshots, version comes from metadata
@@ -192,14 +186,15 @@ class BackupContainerFileSystemImpl {
 						minVer = rf.version;
 					if (rf.version > maxVer)
 						maxVer = rf.version;
-				} else
+				} else {
 					throw restore_unknown_file_type();
-				wait(yield());
+				}
+				co_await yield();
 			}
 		}
 
-		state json_spirit::mValue json;
-		state JSONDoc doc(json);
+		json_spirit::mValue json;
+		JSONDoc doc(json);
 
 		doc.create("files") = std::move(fileArray);
 		doc.create("totalBytes") = totalBytes;
@@ -230,17 +225,17 @@ class BackupContainerFileSystemImpl {
 			}
 		}
 
-		wait(yield());
-		state std::string docString = json_spirit::write_string(json);
+		co_await yield();
+		std::string docString = json_spirit::write_string(json);
 
 		// Generate filename - add suffixes only when 'both' mode is active to prevent collision
 		// Single modes use original format for backward compatibility
-		state std::string fileName;
-		state std::string baseFileName = format("snapshots/snapshot,%lld,%lld,%lld", minVer, maxVer, totalBytes);
+		std::string fileName;
+		std::string baseFileName = format("snapshots/snapshot,%lld,%lld,%lld", minVer, maxVer, totalBytes);
 
 		if (isBulkDump) {
 			// For BulkDump: check if we're in 'both' mode by looking for potential rangefile collision
-			std::vector<KeyspaceSnapshotFile> existingSnapshots = wait(bc->listKeyspaceSnapshots());
+			std::vector<KeyspaceSnapshotFile> existingSnapshots = co_await bc->listKeyspaceSnapshots();
 			bool hasRangefileSnapshot = false;
 
 			for (const auto& snapshot : existingSnapshots) {
@@ -259,7 +254,7 @@ class BackupContainerFileSystemImpl {
 			}
 		} else {
 			// For Rangefile: check if we're in 'both' mode by looking for BulkDump collision
-			std::vector<KeyspaceSnapshotFile> existingSnapshots = wait(bc->listKeyspaceSnapshots());
+			std::vector<KeyspaceSnapshotFile> existingSnapshots = co_await bc->listKeyspaceSnapshots();
 			bool hasBulkDumpSnapshot = false;
 
 			for (const auto& snapshot : existingSnapshots) {
@@ -278,25 +273,23 @@ class BackupContainerFileSystemImpl {
 			}
 		}
 
-		state Reference<IBackupFile> f = wait(bc->writeFile(fileName));
-		wait(f->append(docString.data(), docString.size()));
-		wait(f->finish());
-		return Void();
+		Reference<IBackupFile> f = co_await bc->writeFile(fileName);
+		co_await f->append(docString.data(), docString.size());
+		co_await f->finish();
 	}
 
-	ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc,
-	                                                 Version begin,
-	                                                 Version end) {
-		state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
-		state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
-		state std::vector<LogFile> logs;
-		state std::vector<LogFile> pLogs;
+	static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc, Version begin, Version end) {
+		Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
+		Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
+		std::vector<LogFile> logs;
+		std::vector<LogFile> pLogs;
 
-		wait(success(fRanges) && success(fSnapshots) && store(logs, bc->listLogFiles(begin, end, false)) &&
-		     store(pLogs, bc->listLogFiles(begin, end, true)));
+		co_await (success(fRanges) && success(fSnapshots) &&
+		          store(logs, bc->listLogFiles(begin, end, MutationLogType::DEFAULT)) &&
+		          store(pLogs, bc->listLogFiles(begin, end, MutationLogType::PARTITIONED_LOG)));
 		logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
-		return BackupFileList({ fRanges.get(), std::move(logs), fSnapshots.get() });
+		co_return BackupFileList({ fRanges.get(), std::move(logs), fSnapshots.get() });
 	}
 
 	static Version resolveRelativeVersion(Optional<Version> max, Version v, const char* name, Error e) {
@@ -434,7 +427,7 @@ class BackupContainerFileSystemImpl {
 		if (logs.empty())
 			return;
 
-		Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion;
+		Version snapshotBeginVersion = !desc->snapshots.empty() ? desc->snapshots[0].beginVersion : invalidVersion;
 		Version begin = std::max(scanBegin, desc->minLogBegin.get());
 		TraceEvent("ContinuousLogEnd")
 		    .detail("ScanBegin", scanBegin)
@@ -561,10 +554,80 @@ class BackupContainerFileSystemImpl {
 		return prevEnd;
 	}
 
-	ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc,
-	                                                      bool deepScan,
-	                                                      Version logStartVersionOverride) {
-		state BackupDescription desc;
+	// Read encryption metadata from JSON file
+	static Future<std::pair<bool, int>> readEncryptionMetadata(Reference<BackupContainerFileSystem> bc) {
+		try {
+			Reference<IAsyncFile> f = co_await bc->readFile(BackupContainerFileSystem::encryptionMetadataFileName());
+			int64_t size = co_await f->size();
+			std::string content;
+			content.resize(size);
+			co_await f->read((uint8_t*)content.data(), size, 0);
+
+			json_spirit::mValue json;
+			if (json_spirit::read_string(content, json) && json.type() == json_spirit::obj_type) {
+				auto& obj = json.get_obj();
+
+				// Both fields must be present with correct types.
+				if (!obj.count("is_encryption_enabled") ||
+				    obj.at("is_encryption_enabled").type() != json_spirit::bool_type ||
+				    !obj.count("encryption_block_size") ||
+				    obj.at("encryption_block_size").type() != json_spirit::int_type) {
+					fprintf(stderr, "ERROR: Encryption metadata file is missing required fields or has wrong types.\n");
+					TraceEvent(SevError, "BackupContainerReadEncryptionMetadataMalformed")
+					    .detail("URL", bc->getURL())
+					    .detail("File", BackupContainerFileSystem::encryptionMetadataFileName());
+					throw file_corrupt();
+				}
+
+				bool enabled = obj.at("is_encryption_enabled").get_bool();
+				int blockSize = obj.at("encryption_block_size").get_int();
+
+				if ((enabled && blockSize <= 0) || (!enabled && blockSize != 0)) {
+					fprintf(stderr,
+					        "ERROR: Encryption metadata is inconsistent (enabled=%d, blockSize=%d).\n",
+					        enabled,
+					        blockSize);
+					TraceEvent(SevError, "BackupContainerReadEncryptionMetadataInconsistent")
+					    .detail("URL", bc->getURL())
+					    .detail("IsEncryptionEnabled", enabled)
+					    .detail("EncryptionBlockSize", blockSize);
+					throw file_corrupt();
+				}
+
+				TraceEvent("BackupContainerReadEncryptionMetadata")
+				    .detail("URL", bc->getURL())
+				    .detail("IsEncryptionEnabled", enabled)
+				    .detail("EncryptionBlockSize", blockSize);
+				co_return std::make_pair(enabled, blockSize);
+			} else {
+				// If the file is malformed, throw an error.
+				fprintf(stderr, "ERROR: Failed to read encryption_metadata file due to incorrect format\n");
+				TraceEvent(SevError, "BackupContainerReadEncryptionMetadataMalformed")
+				    .detail("URL", bc->getURL())
+				    .detail("File", BackupContainerFileSystem::encryptionMetadataFileName());
+				throw file_corrupt();
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_file_not_found) {
+				// File may not be present for older backups, return encryption disabled.
+				TraceEvent(SevWarn, "BackupContainerEncryptionMetadataNotFound")
+				    .detail("URL", bc->getURL())
+				    .detail("File", BackupContainerFileSystem::encryptionMetadataFileName());
+				co_return std::make_pair(false, 0);
+			}
+			fprintf(stderr, "ERROR: Failed to read encryption_metadata file due to an error.\n");
+			TraceEvent(SevError, "BackupContainerReadEncryptionMetadataError")
+			    .error(e)
+			    .detail("URL", bc->getURL())
+			    .detail("File", BackupContainerFileSystem::encryptionMetadataFileName());
+			throw file_not_readable();
+		}
+	}
+
+	static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc,
+	                                                bool deepScan,
+	                                                Version logStartVersionOverride) {
+		BackupDescription desc;
 		desc.url = bc->getURL();
 		desc.proxy = bc->getProxy();
 
@@ -573,7 +636,7 @@ class BackupContainerFileSystemImpl {
 		    .detail("LogStartVersionOverride", logStartVersionOverride)
 		    .detail("DeepScan", deepScan);
 
-		bool e = wait(bc->exists());
+		bool e = co_await bc->exists();
 		if (!e) {
 			TraceEvent(SevWarnAlways, "BackupContainerDoesNotExist").detail("URL", bc->getURL());
 			throw backup_does_not_exist();
@@ -583,24 +646,24 @@ class BackupContainerFileSystemImpl {
 		// from which to resolve the relative version.
 		// This could be handled more efficiently without recursion but it's tricky, this will do for now.
 		if (logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) {
-			BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion));
+			BackupDescription tmp = co_await bc->describeBackup(false, invalidVersion);
 			logStartVersionOverride = resolveRelativeVersion(
 			    tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value());
 		}
 
 		// Get metadata versions
-		state Optional<Version> metaLogBegin;
-		state Optional<Version> metaLogEnd;
-		state Optional<Version> metaExpiredEnd;
-		state Optional<Version> metaUnreliableEnd;
-		state Optional<Version> metaLogType;
-		state Optional<Version> fileLevelEncryption;
+		Optional<Version> metaLogBegin;
+		Optional<Version> metaLogEnd;
+		Optional<Version> metaExpiredEnd;
+		Optional<Version> metaUnreliableEnd;
+		Optional<Version> metaLogType;
+		bool fileLevelEncryptionEnabled = false;
+		int encryptionBlockSize = 0;
 
 		std::vector<Future<Void>> metaReads;
 		metaReads.push_back(store(metaExpiredEnd, bc->expiredEndVersion().get()));
 		metaReads.push_back(store(metaUnreliableEnd, bc->unreliableEndVersion().get()));
 		metaReads.push_back(store(metaLogType, bc->logType().get()));
-		metaReads.push_back(store(fileLevelEncryption, bc->fileLevelEncryption().get()));
 
 		// Only read log begin/end versions if not doing a deep scan, otherwise scan files and recalculate them.
 		if (!deepScan) {
@@ -608,7 +671,11 @@ class BackupContainerFileSystemImpl {
 			metaReads.push_back(store(metaLogEnd, bc->logEndVersion().get()));
 		}
 
-		wait(waitForAll(metaReads));
+		co_await waitForAll(metaReads);
+
+		std::pair<bool, int> encryptionMeta = co_await readEncryptionMetadata(bc);
+		fileLevelEncryptionEnabled = encryptionMeta.first;
+		encryptionBlockSize = encryptionMeta.second;
 
 		TraceEvent("BackupContainerDescribe2")
 		    .detail("URL", bc->getURL())
@@ -617,7 +684,9 @@ class BackupContainerFileSystemImpl {
 		    .detail("UnreliableEndVersion", metaUnreliableEnd.orDefault(invalidVersion))
 		    .detail("LogBeginVersion", metaLogBegin.orDefault(invalidVersion))
 		    .detail("LogEndVersion", metaLogEnd.orDefault(invalidVersion))
-		    .detail("LogType", metaLogType.orDefault(-1));
+		    .detail("LogType", metaLogType.orDefault(-1))
+		    .detail("FileLevelEncryption", fileLevelEncryptionEnabled)
+		    .detail("EncryptionBlockSize", encryptionBlockSize);
 
 		// If the logStartVersionOverride is positive (not relative) then ensure that unreliableEndVersion is equal or
 		// greater
@@ -657,8 +726,8 @@ class BackupContainerFileSystemImpl {
 
 		// Start scanning at the end of the unreliable version range, which is the version before which data is likely
 		// missing because an expire process has operated on that range.
-		state Version scanBegin = desc.unreliableEndVersion.orDefault(0);
-		state Version scanEnd = std::numeric_limits<Version>::max();
+		Version scanBegin = desc.unreliableEndVersion.orDefault(0);
+		Version scanEnd = std::numeric_limits<Version>::max();
 
 		// Use the known log range if present
 		// Logs are assumed to be contiguous between metaLogBegin and metaLogEnd, so initialize desc accordingly
@@ -676,13 +745,13 @@ class BackupContainerFileSystemImpl {
 			scanBegin = desc.contiguousLogEnd.get();
 		}
 
-		state std::vector<LogFile> logs;
-		state std::vector<LogFile> plogs;
+		std::vector<LogFile> logs;
+		std::vector<LogFile> plogs;
 		TraceEvent("BackupContainerListFiles").detail("URL", bc->getURL());
 
-		wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
-		     store(plogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
-		     store(desc.snapshots, bc->listKeyspaceSnapshots()));
+		co_await (store(logs, bc->listLogFiles(scanBegin, scanEnd, MutationLogType::DEFAULT)) &&
+		          store(plogs, bc->listLogFiles(scanBegin, scanEnd, MutationLogType::PARTITIONED_LOG)) &&
+		          store(desc.snapshots, bc->listKeyspaceSnapshots()));
 
 		TraceEvent("BackupContainerListFiles")
 		    .detail("URL", bc->getURL())
@@ -690,19 +759,16 @@ class BackupContainerFileSystemImpl {
 		    .detail("PLogsFiles", plogs.size())
 		    .detail("Snapshots", desc.snapshots.size());
 
-		if (plogs.size() > 0) {
-			desc.partitioned = true;
+		if (!plogs.empty()) {
+			desc.mutationLogType = MutationLogType::PARTITIONED_LOG;
 			logs.swap(plogs);
 		} else {
-			desc.partitioned =
-			    metaLogType.present() && metaLogType.get() == BackupContainerFileSystemImpl::PARTITIONED_MUTATION_LOG;
+			desc.mutationLogType =
+			    metaLogType.present() ? static_cast<MutationLogType>(metaLogType.get()) : MutationLogType::DEFAULT;
 		}
 
-		if (fileLevelEncryption.present() && fileLevelEncryption.get() != 0) {
-			desc.fileLevelEncryption = true;
-		} else {
-			desc.fileLevelEncryption = false;
-		}
+		desc.fileLevelEncryption = fileLevelEncryptionEnabled;
+		desc.encryptionBlockSize = encryptionBlockSize;
 
 		// List logs in version order so log continuity can be analyzed
 		std::sort(logs.begin(), logs.end());
@@ -713,7 +779,7 @@ class BackupContainerFileSystemImpl {
 			// If we didn't get log versions above then seed them using the first log file
 			if (!desc.contiguousLogEnd.present()) {
 				desc.minLogBegin = logs.begin()->beginVersion;
-				if (desc.partitioned) {
+				if (desc.mutationLogType == MutationLogType::PARTITIONED_LOG) {
 					// Cannot use the first file's end version, which may not be contiguous
 					// for other partitions. Set to its beginVersion to be safe.
 					desc.contiguousLogEnd = logs.begin()->beginVersion;
@@ -722,7 +788,7 @@ class BackupContainerFileSystemImpl {
 				}
 			}
 
-			if (desc.partitioned) {
+			if (desc.mutationLogType == MutationLogType::PARTITIONED_LOG) {
 				updatePartitionedLogsContinuousEnd(&desc, logs, scanBegin, scanEnd);
 			} else {
 				Version& end = desc.contiguousLogEnd.get();
@@ -738,7 +804,7 @@ class BackupContainerFileSystemImpl {
 			// then attempt to update the metadata in the backup container but ignore errors in case the
 			// container is not writeable.
 			try {
-				state Future<Void> updates = Void();
+				Future<Void> updates = Void();
 
 				if (desc.minLogBegin.present() && metaLogBegin != desc.minLogBegin) {
 					updates = updates && bc->logBeginVersion().set(desc.minLogBegin.get());
@@ -749,13 +815,10 @@ class BackupContainerFileSystemImpl {
 				}
 
 				if (!metaLogType.present()) {
-					updates =
-					    updates && bc->logType().set(desc.partitioned
-					                                     ? BackupContainerFileSystemImpl::PARTITIONED_MUTATION_LOG
-					                                     : BackupContainerFileSystemImpl::NON_PARTITIONED_MUTATION_LOG);
+					updates = updates && bc->logType().set(static_cast<int>(desc.mutationLogType));
 				}
 
-				wait(updates);
+				co_await updates;
 			} catch (Error& e) {
 				if (e.code() == error_code_actor_cancelled)
 					throw;
@@ -775,7 +838,7 @@ class BackupContainerFileSystemImpl {
 				// If there is logs gap after contiguousLogEnd, then check whether the current snapshot
 				// can be restored from the logs available after contiguousLogEnd.
 				if (desc.contiguousLogEnd.present() && desc.contiguousLogEnd.get() <= s.beginVersion) {
-					if (desc.partitioned)
+					if (desc.mutationLogType == MutationLogType::PARTITIONED_LOG)
 						s.restorable = isPartitionedLogsContinuous(logs, s.beginVersion, s.endVersion);
 					else
 						s.restorable = hasContinuousLogsForSnapshot(logs, s.beginVersion, s.endVersion);
@@ -821,7 +884,7 @@ class BackupContainerFileSystemImpl {
 				if (desc.minRestorableVersion.present() && desc.maxRestorableVersion.present()) {
 					// check if we have contiguous logs from minRestorableVersion to current snapshot endVersion
 					bool contiguousLogs = false;
-					if (desc.partitioned)
+					if (desc.mutationLogType == MutationLogType::PARTITIONED_LOG)
 						contiguousLogs =
 						    isPartitionedLogsContinuous(logs, desc.minRestorableVersion.get(), s.endVersion);
 					else
@@ -846,7 +909,7 @@ class BackupContainerFileSystemImpl {
 
 				// Find the continuousLogEnd after snapshotEndVersion and set it as
 				// maxRestorableVersion.
-				if (desc.partitioned) {
+				if (desc.mutationLogType == MutationLogType::PARTITIONED_LOG) {
 					// TO DO: Yet to implement similar function findContinuousLogEnd for partitioned logs.
 					desc.maxRestorableVersion = s.endVersion;
 				} else {
@@ -857,14 +920,14 @@ class BackupContainerFileSystemImpl {
 			}
 		}
 
-		return desc;
+		co_return desc;
 	}
 
-	ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc,
-	                                     Version expireEndVersion,
-	                                     bool force,
-	                                     IBackupContainer::ExpireProgress* progress,
-	                                     Version restorableBeginVersion) {
+	static Future<Void> expireData(Reference<BackupContainerFileSystem> bc,
+	                               Version expireEndVersion,
+	                               bool force,
+	                               IBackupContainer::ExpireProgress* progress,
+	                               Version restorableBeginVersion) {
 		if (progress != nullptr) {
 			progress->step = "Describing backup";
 			progress->total = 0;
@@ -876,7 +939,7 @@ class BackupContainerFileSystemImpl {
 		    .detail("RestorableBeginVersion", restorableBeginVersion);
 
 		// Get the backup description.
-		state BackupDescription desc = wait(bc->describeBackup(false, expireEndVersion));
+		BackupDescription desc = co_await bc->describeBackup(false, expireEndVersion);
 
 		// Resolve relative versions using max log version
 		expireEndVersion =
@@ -891,14 +954,14 @@ class BackupContainerFileSystemImpl {
 		// If the expire request is to a version at or before the previous version to which data was already deleted
 		// then do nothing and just return
 		if (expireEndVersion <= desc.expiredEndVersion.orDefault(invalidVersion)) {
-			return Void();
+			co_return;
 		}
 
 		// Assume force is needed, then try to prove otherwise.
 		// Force is required if there is not a restorable snapshot which both
 		//   - begins at or after expireEndVersion
 		//   - ends at or before restorableBeginVersion
-		state bool forceNeeded = true;
+		bool forceNeeded = true;
 		for (KeyspaceSnapshotFile& s : desc.snapshots) {
 			if (s.restorable.orDefault(false) && s.beginVersion >= expireEndVersion &&
 			    s.endVersion <= restorableBeginVersion) {
@@ -915,7 +978,7 @@ class BackupContainerFileSystemImpl {
 			throw backup_cannot_expire();
 
 		// Start scan for files to delete at the last completed expire operation's end or 0.
-		state Version scanBegin = desc.expiredEndVersion.orDefault(0);
+		Version scanBegin = desc.expiredEndVersion.orDefault(0);
 
 		TraceEvent("BackupContainerFileSystemExpire2")
 		    .detail("URL", bc->getURL())
@@ -923,21 +986,21 @@ class BackupContainerFileSystemImpl {
 		    .detail("RestorableBeginVersion", restorableBeginVersion)
 		    .detail("ScanBeginVersion", scanBegin);
 
-		state std::vector<LogFile> logs;
-		state std::vector<LogFile> pLogs; // partitioned mutation logs
-		state std::vector<RangeFile> ranges;
+		std::vector<LogFile> logs;
+		std::vector<LogFile> pLogs; // partitioned mutation logs
+		std::vector<RangeFile> ranges;
 
 		if (progress != nullptr) {
 			progress->step = "Listing files";
 		}
 		// Get log files or range files that contain any data at or before expireEndVersion
-		wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1, false)) &&
-		     store(pLogs, bc->listLogFiles(scanBegin, expireEndVersion - 1, true)) &&
-		     store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
+		co_await (store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1, MutationLogType::DEFAULT)) &&
+		          store(pLogs, bc->listLogFiles(scanBegin, expireEndVersion - 1, MutationLogType::PARTITIONED_LOG)) &&
+		          store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1)));
 		logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end()));
 
 		// The new logBeginVersion will be taken from the last log file, if there is one
-		state Optional<Version> newLogBeginVersion;
+		Optional<Version> newLogBeginVersion;
 		if (!logs.empty()) {
 			// Linear scan the unsorted logs to find the latest one in sorted order
 			LogFile& last = *std::max_element(logs.begin(), logs.end());
@@ -960,7 +1023,7 @@ class BackupContainerFileSystemImpl {
 		}
 
 		// Make a list of files to delete
-		state std::vector<std::string> toDelete;
+		std::vector<std::string> toDelete;
 
 		// Move filenames out of vector then destroy it to save memory
 		for (auto const& f : logs) {
@@ -995,9 +1058,9 @@ class BackupContainerFileSystemImpl {
 		if (progress != nullptr) {
 			progress->step = "Initial metadata update";
 		}
-		Optional<Version> metaUnreliableEnd = wait(bc->unreliableEndVersion().get());
+		Optional<Version> metaUnreliableEnd = co_await bc->unreliableEndVersion().get();
 		if (metaUnreliableEnd.orDefault(0) < expireEndVersion) {
-			wait(bc->unreliableEndVersion().set(expireEndVersion));
+			co_await bc->unreliableEndVersion().set(expireEndVersion);
 		}
 
 		if (progress != nullptr) {
@@ -1008,7 +1071,7 @@ class BackupContainerFileSystemImpl {
 
 		// Delete files, but limit parallelism because the file list could use a lot of memory and the corresponding
 		// delete actor states would use even more if they all existed at the same time.
-		state std::list<Future<Void>> deleteFutures;
+		std::list<Future<Void>> deleteFutures;
 
 		while (!toDelete.empty() || !deleteFutures.empty()) {
 
@@ -1021,10 +1084,10 @@ class BackupContainerFileSystemImpl {
 			// Wait for deletes to finish until there are only targetDeletesInFlight remaining.
 			// If there are no files left to start then this value is 0, otherwise it is one less
 			// than the delete concurrency limit.
-			state int targetFuturesSize = toDelete.empty() ? 0 : (CLIENT_KNOBS->BACKUP_CONCURRENT_DELETES - 1);
+			int targetFuturesSize = toDelete.empty() ? 0 : (CLIENT_KNOBS->BACKUP_CONCURRENT_DELETES - 1);
 
 			while (deleteFutures.size() > targetFuturesSize) {
-				wait(deleteFutures.front());
+				co_await deleteFutures.front();
 				if (progress != nullptr) {
 					++progress->done;
 				}
@@ -1038,12 +1101,10 @@ class BackupContainerFileSystemImpl {
 		}
 		// Update the expiredEndVersion metadata to indicate that everything prior to that version has been
 		// successfully deleted if the current version is lower or missing
-		Optional<Version> metaExpiredEnd = wait(bc->expiredEndVersion().get());
+		Optional<Version> metaExpiredEnd = co_await bc->expiredEndVersion().get();
 		if (metaExpiredEnd.orDefault(0) < expireEndVersion) {
-			wait(bc->expiredEndVersion().set(expireEndVersion));
+			co_await bc->expiredEndVersion().set(expireEndVersion);
 		}
-
-		return Void();
 	}
 
 	// Returns true if logs are continuous in the range [begin, end].
@@ -1125,43 +1186,42 @@ class BackupContainerFileSystemImpl {
 	// If "logsOnly" is true, then only log files are returned and "keyRangesFilter" is ignored,
 	// because the log can contain mutations of the whole key space, unlike range files that each
 	// is limited to a smaller key range.
-	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet(Reference<BackupContainerFileSystem> bc,
-	                                                               Version targetVersion,
-	                                                               VectorRef<KeyRangeRef> keyRangesFilter,
-	                                                               bool logsOnly = false,
-	                                                               Version beginVersion = invalidVersion) {
+	static Future<Optional<RestorableFileSet>> getRestoreSet(Reference<BackupContainerFileSystem> bc,
+	                                                         Version targetVersion,
+	                                                         VectorRef<KeyRangeRef> keyRangesFilter,
+	                                                         bool logsOnly = false,
+	                                                         Version beginVersion = invalidVersion) {
 		for (const auto& range : keyRangesFilter) {
 			TraceEvent("BackupContainerGetRestoreSet").detail("RangeFilter", printable(range));
 		}
 
 		if (logsOnly) {
-			state RestorableFileSet restorableSet;
+			RestorableFileSet restorableSet;
 			restorableSet.targetVersion = targetVersion;
-			state std::vector<LogFile> logFiles;
+			std::vector<LogFile> logFiles;
 			Version begin = beginVersion == invalidVersion ? 0 : beginVersion;
-			wait(store(logFiles, bc->listLogFiles(begin, targetVersion, false)));
+			logFiles = co_await bc->listLogFiles(begin, targetVersion, MutationLogType::DEFAULT);
 			// List logs in version order so log continuity can be analyzed
 			std::sort(logFiles.begin(), logFiles.end());
 			if (!logFiles.empty()) {
-				return getRestoreSetFromLogs(logFiles, targetVersion, restorableSet);
+				co_return getRestoreSetFromLogs(logFiles, targetVersion, restorableSet);
 			}
 		}
 
 		// Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion.
-		state std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
-		state int i = snapshots.size() - 1;
-		for (; i >= 0; i--) {
+		std::vector<KeyspaceSnapshotFile> snapshots = co_await bc->listKeyspaceSnapshots();
+		for (int i = snapshots.size() - 1; i >= 0; i--) {
 			// The smallest version of filtered range files >= snapshot beginVersion > targetVersion
 			if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) {
 				continue;
 			}
 
-			state RestorableFileSet restorable;
-			state Version minKeyRangeVersion = MAX_VERSION;
-			state Version maxKeyRangeVersion = -1;
+			RestorableFileSet restorable;
+			Version minKeyRangeVersion = MAX_VERSION;
+			Version maxKeyRangeVersion = -1;
 
 			std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>> results =
-			    wait(bc->readKeyspaceSnapshot(snapshots[i]));
+			    co_await bc->readKeyspaceSnapshot(snapshots[i]);
 
 			// If there is no key ranges filter for the restore OR if the snapshot contains no per-file key range info
 			// then return all of the range files
@@ -1201,16 +1261,19 @@ class BackupContainerFileSystemImpl {
 				    .detail("KeyRangeVersion", restorable.targetVersion)
 				    .detail("NumberOfRangeFiles", restorable.ranges.size())
 				    .detail("KeyRangesFilter", printable(keyRangesFilter));
-				return Optional<RestorableFileSet>(restorable);
+				co_return Optional<RestorableFileSet>(restorable);
 			}
 
 			// FIXME: check if there are tagged logs. for each tag, there is no version gap.
-			state std::vector<LogFile> logs;
-			state std::vector<LogFile> plogs;
-			wait(store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, false)) &&
-			     store(plogs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, true)));
-
-			if (plogs.size() > 0) {
+			std::vector<LogFile> logs;
+			std::vector<LogFile> plogs;
+			co_await (
+			    store(logs, bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, MutationLogType::DEFAULT)) &&
+			    store(
+			        plogs,
+			        bc->listLogFiles(minKeyRangeVersion, restorable.targetVersion, MutationLogType::PARTITIONED_LOG)));
+
+			if (!plogs.empty()) {
 				logs.swap(plogs);
 				// sort by tag ID so that filterDuplicates works.
 				std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) {
@@ -1226,9 +1289,9 @@ class BackupContainerFileSystemImpl {
 				if (isPartitionedLogsContinuous(restorable.logs, minKeyRangeVersion, restorable.targetVersion)) {
 					restorable.continuousBeginVersion = minKeyRangeVersion;
 					restorable.continuousEndVersion = restorable.targetVersion + 1; // not inclusive
-					return Optional<RestorableFileSet>(restorable);
+					co_return Optional<RestorableFileSet>(restorable);
 				}
-				return Optional<RestorableFileSet>();
+				co_return Optional<RestorableFileSet>();
 			}
 
 			// List logs in version order so log continuity can be analyzed
@@ -1236,10 +1299,10 @@ class BackupContainerFileSystemImpl {
 			// If there are logs and the first one starts at or before the keyrange's snapshot begin version, then
 			// it is valid restore set and proceed
 			if (!logs.empty() && logs.front().beginVersion <= minKeyRangeVersion) {
-				return getRestoreSetFromLogs(logs, targetVersion, restorable);
+				co_return getRestoreSetFromLogs(logs, targetVersion, restorable);
 			}
 		}
-		return Optional<RestorableFileSet>();
+		co_return Optional<RestorableFileSet>();
 	}
 
 	static std::string versionFolderString(Version v, int smallestBucket) {
@@ -1285,8 +1348,18 @@ class BackupContainerFileSystemImpl {
 
 	// The innermost folder covers 100,000 seconds (1e11 versions) which is 5,000 mutation log files at current
 	// settings.
-	static std::string logVersionFolderString(Version v, bool partitioned) {
-		return format("%s/%s/", (partitioned ? "plogs" : "logs"), versionFolderString(v, 11).c_str());
+	static std::string logVersionFolderString(Version v, MutationLogType mutationLogType) {
+		return format("%s/%s/",
+		              (mutationLogType == MutationLogType::PARTITIONED_LOG ? "plogs" : "logs"),
+		              versionFolderString(v, 11).c_str());
+	}
+
+	static std::string logVersionFolderStringForRangePartitioned(Version v, Version baseVersion) {
+		Version directoryVersion =
+		    baseVersion + ((v - baseVersion) / CLIENT_KNOBS->BACKUP_RANGE_PARTITIONED_VDIR_INTERVAL) *
+		                      CLIENT_KNOBS->BACKUP_RANGE_PARTITIONED_VDIR_INTERVAL;
+		std::string vFixed = format("%019lld", directoryVersion);
+		return format("rlogs/%s/", vFixed.c_str());
 	}
 
 	static bool pathToLogFile(LogFile& out, const std::string& path, int64_t size) {
@@ -1324,15 +1397,18 @@ class BackupContainerFileSystemImpl {
 		KeyspaceSnapshotFile f;
 		f.fileName = path;
 		int len;
+		char typeBuf[64] = {};
 
 		// Try new format with type suffix: snapshot,beginVersion,endVersion,totalSize,type
 		if (sscanf(name.c_str(),
-		           "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 ",%*[^,]%n",
+		           "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 ",%63[^,]%n",
 		           &f.beginVersion,
 		           &f.endVersion,
 		           &f.totalSize,
-		           &len) == 3 &&
+		           typeBuf,
+		           &len) == 4 &&
 		    len == name.size()) {
+			f.snapshotType = typeBuf;
 			out = f;
 			return true;
 		}
@@ -1352,47 +1428,45 @@ class BackupContainerFileSystemImpl {
 	}
 
 	// fallback for using existing write api if the underlying blob store doesn't support efficient writeEntireFile
-	ACTOR static Future<Void> writeEntireFileFallback(Reference<BackupContainerFileSystem> bc,
-	                                                  std::string fileName,
-	                                                  std::string fileContents) {
-		state Reference<IBackupFile> objectFile = wait(bc->writeFile(fileName));
-		wait(objectFile->append(&fileContents[0], fileContents.size()));
-		wait(objectFile->finish());
-		return Void();
+	static Future<Void> writeEntireFileFallback(Reference<BackupContainerFileSystem> bc,
+	                                            std::string fileName,
+	                                            std::string fileContents) {
+		Reference<IBackupFile> objectFile = co_await bc->writeFile(fileName);
+		co_await objectFile->append(&fileContents[0], fileContents.size());
+		co_await objectFile->finish();
 	}
 
-	ACTOR static Future<Void> createTestEncryptionKeyFile(std::string filename) {
+	static Future<Void> createTestEncryptionKeyFile(std::string filename) {
 		if (fileExists(filename)) {
 			// Key file already exists, don't overwrite it -> only for testing between backup and restore workloads to
 			// share the key.
 			TraceEvent("EncryptionKeyFileExists").detail("FileName", filename);
-			return Void();
+			co_return;
 		}
-		state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(
+		Reference<IAsyncFile> keyFile = co_await IAsyncFileSystem::filesystem()->open(
 		    filename,
 		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,
-		    0600));
+		    0600);
 		StreamCipherKey testKey(AES_256_KEY_LENGTH);
 		testKey.initializeRandomTestKey();
 		keyFile->write(testKey.data(), testKey.size(), 0);
-		wait(keyFile->sync());
-		return Void();
+		co_await keyFile->sync();
 	}
 
-	ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
-		state Reference<IAsyncFile> keyFile;
-		state StreamCipherKey const* cipherKey = StreamCipherKey::getGlobalCipherKey();
+	static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
+		Reference<IAsyncFile> keyFile;
+		StreamCipherKey const* cipherKey = StreamCipherKey::getGlobalCipherKey();
 		try {
-			Reference<IAsyncFile> _keyFile = wait(IAsyncFileSystem::filesystem()->open(
+			Reference<IAsyncFile> _keyFile = co_await IAsyncFileSystem::filesystem()->open(
 			    encryptionKeyFileName,
 			    IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED,
-			    0400));
+			    0400);
 			keyFile = _keyFile;
 		} catch (Error& e) {
 			TraceEvent(SevError, "FailedToOpenEncryptionKeyFile").error(e).detail("FileName", encryptionKeyFileName);
 			throw e;
 		}
-		int bytesRead = wait(uncancellable(keyFile->read(cipherKey->data(), cipherKey->size(), 0)));
+		int bytesRead = co_await uncancellable(keyFile->read(cipherKey->data(), cipherKey->size(), 0));
 		if (bytesRead != cipherKey->size()) {
 			TraceEvent(SevError, "InvalidEncryptionKeyFileSize")
 			    .detail("ExpectedSize", cipherKey->size())
@@ -1401,20 +1475,38 @@ class BackupContainerFileSystemImpl {
 			throw invalid_encryption_key_file();
 		}
 		ASSERT_EQ(bytesRead, cipherKey->size());
-		return Void();
 	}
 
-	ACTOR static Future<Void> writeEncryptionMetadataIfNotExists(Reference<BackupContainerFileSystem> bc) {
-		Optional<Version> existingEncryptionMetadata = wait(bc->fileLevelEncryption().get());
-
-		if (!existingEncryptionMetadata.present()) {
-			bool exists = wait(bc->exists());
-			if (!exists) {
-				wait(bc->create());
+	static Future<Void> writeEncryptionMetadataIfNotExists(Reference<BackupContainerFileSystem> bc,
+	                                                       int encryptionBlockSize) {
+		try {
+			Reference<IAsyncFile> f = co_await bc->readFile(BackupContainerFileSystem::encryptionMetadataFileName());
+			int64_t size = co_await f->size();
+			TraceEvent("WriteEncryptionMetadataAlreadyExists").detail("URL", bc->getURL()).detail("FileSize", size);
+			co_return;
+		} catch (Error& e) {
+			if (e.code() != error_code_file_not_found) {
+				TraceEvent(SevWarn, "WriteEncryptionMetadataReadError").error(e).detail("URL", bc->getURL());
+				throw e;
 			}
-			wait(bc->fileLevelEncryption().set(bc->encryptionKeyFileName.present() ? 1 : 0));
 		}
-		return Void();
+
+		bool exists = co_await bc->exists();
+		if (!exists) {
+			TraceEvent("WriteEncryptionMetadataCreatingContainer").detail("URL", bc->getURL());
+			co_await bc->create();
+		}
+
+		// Write JSON with encryption metadata
+		bool enabled = bc->encryptionKeyFileName.present();
+		JsonBuilderObject doc;
+		doc.setKey("is_encryption_enabled", enabled);
+		doc.setKey("encryption_block_size", encryptionBlockSize);
+
+		std::string jsonStr = doc.getJson();
+		TraceEvent("WriteEncryptionMetadataCompleted").detail("URL", bc->getURL()).detail("JSON", jsonStr);
+
+		co_await bc->writeEntireFile(BackupContainerFileSystem::encryptionMetadataFileName(), jsonStr);
 	}
 
 }; // class BackupContainerFileSystemImpl
@@ -1422,7 +1514,7 @@ class BackupContainerFileSystemImpl {
 Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
                                                                        Version endVersion,
                                                                        int blockSize) {
-	return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, false) +
+	return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, MutationLogType::DEFAULT) +
 	                 format("log,%lld,%lld,%s,%d",
 	                        beginVersion,
 	                        endVersion,
@@ -1435,14 +1527,25 @@ Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Ver
                                                                              int blockSize,
                                                                              uint16_t tagId,
                                                                              int totalTags) {
-	return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, true) +
-	                 format("log,%lld,%lld,%s,%d-of-%d,%d",
-	                        beginVersion,
-	                        endVersion,
-	                        deterministicRandom()->randomUniqueID().toString().c_str(),
-	                        tagId,
-	                        totalTags,
-	                        blockSize));
+	return writeFile(
+	    BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, MutationLogType::PARTITIONED_LOG) +
+	    format("log,%lld,%lld,%s,%d-of-%d,%d",
+	           beginVersion,
+	           endVersion,
+	           deterministicRandom()->randomUniqueID().toString().c_str(),
+	           tagId,
+	           totalTags,
+	           blockSize));
+}
+
+Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangePartitionedLogFile(Version beginVersion,
+                                                                                       Version endVersion,
+                                                                                       Version baseVersion,
+                                                                                       int32_t partitionId,
+                                                                                       int blockSize) {
+	return writeFile(
+	    BackupContainerFileSystemImpl::logVersionFolderStringForRangePartitioned(beginVersion, baseVersion) +
+	    format("log,%lld,%lld,%d,%d", beginVersion, endVersion, partitionId, blockSize));
 }
 
 Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version snapshotBeginVersion,
@@ -1461,6 +1564,12 @@ Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version
 	                 format("/%d/", snapshotFileCount / (BUGGIFY ? 1 : 5000)) + fileName);
 }
 
+Future<Void> BackupContainerFileSystem::writePartitionListFile(Version v, std::string contents) {
+	return writeEntireFile(BackupContainerFileSystemImpl::logVersionFolderStringForRangePartitioned(v, v) +
+	                           "partitionId_keyRange_map",
+	                       contents);
+}
+
 Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>>
 BackupContainerFileSystem::readKeyspaceSnapshot(KeyspaceSnapshotFile snapshot) {
 	return BackupContainerFileSystemImpl::readKeyspaceSnapshot(Reference<BackupContainerFileSystem>::addRef(this),
@@ -1482,7 +1591,7 @@ Future<Void> BackupContainerFileSystem::writeKeyspaceSnapshotFile(const std::vec
 
 Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion,
                                                                      Version targetVersion,
-                                                                     bool partitioned) {
+                                                                     MutationLogType mutationLogType) {
 	// The first relevant log file could have a begin version less than beginVersion based on the knobs which
 	// determine log file range size, so start at an earlier version adjusted by how many versions a file could
 	// contain.
@@ -1492,9 +1601,9 @@ Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beg
 	    BackupContainerFileSystemImpl::cleanFolderString(BackupContainerFileSystemImpl::logVersionFolderString(
 	        std::max<Version>(0,
 	                          beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE),
-	        partitioned));
+	        mutationLogType));
 	std::string lastPath = BackupContainerFileSystemImpl::cleanFolderString(
-	    BackupContainerFileSystemImpl::logVersionFolderString(targetVersion, partitioned));
+	    BackupContainerFileSystemImpl::logVersionFolderString(targetVersion, mutationLogType));
 
 	std::function<bool(std::string const&)> pathFilter = [=](const std::string& folderPath) {
 		// Remove slashes in the given folder path so that the '/' positions in the version folder string do not
@@ -1505,16 +1614,17 @@ Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beg
 		       (cleaned > firstPath && cleaned < lastPath);
 	};
 
-	return map(listFiles((partitioned ? "plogs/" : "logs/"), pathFilter), [=](const FilesAndSizesT& files) {
-		std::vector<LogFile> results;
-		LogFile lf;
-		for (auto& f : files) {
-			if (BackupContainerFileSystemImpl::pathToLogFile(lf, f.first, f.second) && lf.endVersion > beginVersion &&
-			    lf.beginVersion <= targetVersion)
-				results.push_back(lf);
-		}
-		return results;
-	});
+	return map(listFiles((mutationLogType == MutationLogType::PARTITIONED_LOG ? "plogs/" : "logs/"), pathFilter),
+	           [=](const FilesAndSizesT& files) {
+		           std::vector<LogFile> results;
+		           LogFile lf;
+		           for (auto& f : files) {
+			           if (BackupContainerFileSystemImpl::pathToLogFile(lf, f.first, f.second) &&
+			               lf.endVersion > beginVersion && lf.beginVersion <= targetVersion)
+				           results.push_back(lf);
+		           }
+		           return results;
+	           });
 }
 
 Future<std::vector<RangeFile>> BackupContainerFileSystem::old_listRangeFiles(Version beginVersion, Version endVersion) {
@@ -1606,27 +1716,26 @@ Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion,
 	    Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 }
 
-Future<Void> BackupContainerFileSystem::writeEncryptionMetadata() {
+Future<Void> BackupContainerFileSystem::writeEncryptionMetadata(int encryptionBlockSize) {
 	return BackupContainerFileSystemImpl::writeEncryptionMetadataIfNotExists(
-	    Reference<BackupContainerFileSystem>::addRef(this));
+	    Reference<BackupContainerFileSystem>::addRef(this), encryptionBlockSize);
 }
 
-ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc,
-                                                           RangeFile file,
-                                                           Database cx) {
-	state int readFileRetries = 0;
-	state bool beginKeySet = false;
-	state Key beginKey;
-	state Key endKey;
-	loop {
+static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc,
+                                                     RangeFile file,
+                                                     Database cx) {
+	int readFileRetries = 0;
+	Key beginKey;
+	Key endKey;
+	while (true) {
+		Error err;
 		try {
-			state Reference<IAsyncFile> inFile = wait(bc->readFile(file.fileName));
-			beginKeySet = false;
-			state int64_t j = 0;
-			for (; j < file.fileSize; j += file.blockSize) {
+			Reference<IAsyncFile> inFile = co_await bc->readFile(file.fileName);
+			bool beginKeySet = false;
+			for (int64_t j = 0; j < file.fileSize; j += file.blockSize) {
 				int64_t len = std::min<int64_t>(file.blockSize, file.fileSize - j);
 				Standalone<VectorRef<KeyValueRef>> blockData =
-				    wait(fileBackup::decodeRangeFileBlock(inFile, j, len, cx));
+				    co_await fileBackup::decodeRangeFileBlock(inFile, j, len, cx);
 				if (!beginKeySet) {
 					beginKey = blockData.front().key;
 					beginKeySet = true;
@@ -1635,33 +1744,33 @@ ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupConta
 			}
 			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_restore_bad_read || e.code() == error_code_restore_unsupported_file_version ||
-			    e.code() == error_code_restore_corrupted_data_padding) { // no retriable error
-				TraceEvent(SevError, "BackupContainerGetSnapshotFileKeyRange").error(e);
-				throw;
-			} else if (e.code() == error_code_http_request_failed || e.code() == error_code_connection_failed ||
-			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
-				// blob http request failure, retry
-				TraceEvent(SevWarnAlways, "BackupContainerGetSnapshotFileKeyRangeConnectionFailure")
-				    .error(e)
-				    .detail("Retries", ++readFileRetries);
-				wait(delayJittered(0.1));
-			} else {
-				TraceEvent(SevError, "BackupContainerGetSnapshotFileKeyRangeUnexpectedError").error(e);
-				throw;
-			}
+			err = e;
+		}
+		if (err.code() == error_code_restore_bad_read || err.code() == error_code_restore_unsupported_file_version ||
+		    err.code() == error_code_restore_corrupted_data_padding) { // no retriable error
+			TraceEvent(SevError, "BackupContainerGetSnapshotFileKeyRange").error(err);
+			throw err;
+		} else if (err.code() == error_code_http_request_failed || err.code() == error_code_connection_failed ||
+		           err.code() == error_code_timed_out || err.code() == error_code_lookup_failed) {
+			// blob http request failure, retry
+			TraceEvent(SevWarnAlways, "BackupContainerGetSnapshotFileKeyRangeConnectionFailure")
+			    .error(err)
+			    .detail("Retries", ++readFileRetries);
+			co_await delayJittered(0.1);
+		} else {
+			TraceEvent(SevError, "BackupContainerGetSnapshotFileKeyRangeUnexpectedError").error(err);
+			throw err;
 		}
 	}
-	return KeyRange(KeyRangeRef(beginKey, endKey));
+	co_return KeyRange(KeyRangeRef(beginKey, endKey));
 }
 
-ACTOR static Future<Void> writeVersionProperty(Reference<BackupContainerFileSystem> bc, std::string path, Version v) {
+static Future<Void> writeVersionProperty(Reference<BackupContainerFileSystem> bc, std::string path, Version v) {
 	try {
-		state Reference<IBackupFile> f = wait(bc->writeFile(path));
+		Reference<IBackupFile> f = co_await bc->writeFile(path);
 		std::string s = format("%lld", v);
-		wait(f->append(s.data(), s.size()));
-		wait(f->finish());
-		return Void();
+		co_await f->append(s.data(), s.size());
+		co_await f->finish();
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "BackupContainerWritePropertyFailed")
 		    .error(e)
@@ -1671,24 +1780,24 @@ ACTOR static Future<Void> writeVersionProperty(Reference<BackupContainerFileSyst
 	}
 }
 
-ACTOR static Future<Optional<Version>> readVersionProperty(Reference<BackupContainerFileSystem> bc, std::string path) {
+static Future<Optional<Version>> readVersionProperty(Reference<BackupContainerFileSystem> bc, std::string path) {
 	try {
-		state Reference<IAsyncFile> f = wait(bc->readFile(path));
-		state int64_t size = wait(f->size());
-		state std::string s;
+		Reference<IAsyncFile> f = co_await bc->readFile(path);
+		int64_t size = co_await f->size();
+		std::string s;
 		s.resize(size);
-		int rs = wait(f->read((uint8_t*)s.data(), size, 0));
+		int rs = co_await f->read((uint8_t*)s.data(), size, 0);
 		Version v;
 		int len;
 		if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size)
-			return v;
+			co_return v;
 
 		TraceEvent(SevWarn, "BackupContainerInvalidProperty").detail("URL", bc->getURL()).detail("Path", path);
 
 		throw backup_invalid_info();
 	} catch (Error& e) {
 		if (e.code() == error_code_file_not_found)
-			return Optional<Version>();
+			co_return Optional<Version>();
 
 		TraceEvent(SevWarn, "BackupContainerReadPropertyFailed")
 		    .error(e)
@@ -1737,8 +1846,9 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable
 BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() {
 	return { Reference<BackupContainerFileSystem>::addRef(this), "mutation_log_type" };
 }
-BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::fileLevelEncryption() {
-	return { Reference<BackupContainerFileSystem>::addRef(this), "file_level_encryption" };
+
+std::string BackupContainerFileSystem::encryptionMetadataFileName() {
+	return "properties/encryption_metadata";
 }
 
 bool BackupContainerFileSystem::usesEncryption() const {
@@ -1759,121 +1869,11 @@ void BackupContainerFileSystem::setEncryptionKey(Optional<std::string> const& en
 		encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get());
 	}
 }
+
 Future<Void> BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const& filename) {
 	return BackupContainerFileSystemImpl::createTestEncryptionKeyFile(filename);
 }
 
-// Get a BackupContainerFileSystem based on a container URL string
-// TODO: refactor to not duplicate IBackupContainer::openContainer. It's the exact same
-// code but returning a different template type because you can't cast between them
-Reference<BackupContainerFileSystem> BackupContainerFileSystem::openContainerFS(
-    const std::string& url,
-    const Optional<std::string>& proxy,
-    const Optional<std::string>& encryptionKeyFileName,
-    bool isBackup) {
-	static std::map<std::string, Reference<BackupContainerFileSystem>> m_cache;
-
-	Reference<BackupContainerFileSystem>& r = m_cache[url];
-	if (r)
-		return r;
-
-	try {
-		StringRef u(url);
-		if (u.startsWith("file://"_sr)) {
-			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName);
-		} else if (u.startsWith("blobstore://"_sr)) {
-			std::string resource;
-			Optional<std::string> blobstoreProxy;
-
-			// If no proxy is passed down to the openContainer method, try to fallback to the
-			// fileBackupAgentProxy which is a global variable and will be set for the backup_agent.
-			if (proxy.present()) {
-				blobstoreProxy = proxy.get();
-			} else if (fileBackupAgentProxy.present()) {
-				blobstoreProxy = fileBackupAgentProxy.get();
-			}
-
-			// The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options.
-			S3BlobStoreEndpoint::ParametersT backupParams;
-			Reference<S3BlobStoreEndpoint> bstore =
-			    S3BlobStoreEndpoint::fromString(url, blobstoreProxy, &resource, &lastOpenError, &backupParams);
-
-			if (resource.empty())
-				throw backup_invalid_url();
-			for (auto c : resource)
-				if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
-					throw backup_invalid_url();
-			r = makeReference<BackupContainerS3BlobStore>(
-			    bstore, resource, backupParams, encryptionKeyFileName, isBackup);
-		}
-#ifdef BUILD_AZURE_BACKUP
-		else if (u.startsWith("azure://"_sr)) {
-			u.eat("azure://"_sr);
-			auto address = u.eat("/"_sr);
-			if (address.endsWith(std::string(azure::storage_lite::constants::default_endpoint_suffix))) {
-				CODE_PROBE(true, "Azure backup url with standard azure storage account endpoint");
-				// <account>.<service>.core.windows.net/<resource_path>
-				auto endPoint = address.toString();
-				auto accountName = address.eat("."_sr).toString();
-				auto containerName = u.eat("/"_sr).toString();
-				r = makeReference<BackupContainerAzureBlobStore>(
-				    endPoint, accountName, containerName, encryptionKeyFileName);
-			} else {
-				// resolve the network address if necessary
-				std::string endpoint(address.toString());
-				Optional<NetworkAddress> parsedAddress = NetworkAddress::parseOptional(endpoint);
-				if (!parsedAddress.present()) {
-					try {
-						auto hostname = Hostname::parse(endpoint);
-						auto resolvedAddress = hostname.resolveBlocking();
-						if (resolvedAddress.present()) {
-							CODE_PROBE(true, "Azure backup url with hostname in the endpoint");
-							parsedAddress = resolvedAddress.get();
-						}
-					} catch (Error& e) {
-						TraceEvent(SevError, "InvalidAzureBackupUrl").error(e).detail("Endpoint", endpoint);
-						throw backup_invalid_url();
-					}
-				}
-				if (!parsedAddress.present()) {
-					TraceEvent(SevError, "InvalidAzureBackupUrl").detail("Endpoint", endpoint);
-					throw backup_invalid_url();
-				}
-				auto accountName = u.eat("/"_sr).toString();
-				// Avoid including ":tls" and "(fromHostname)"
-				// note: the endpoint needs to contain the account name
-				// so either "<account_name>.blob.core.windows.net" or "<ip>:<port>/<account_name>"
-				endpoint =
-				    fmt::format("{}/{}", formatIpPort(parsedAddress.get().ip, parsedAddress.get().port), accountName);
-				auto containerName = u.eat("/"_sr).toString();
-				r = makeReference<BackupContainerAzureBlobStore>(
-				    endpoint, accountName, containerName, encryptionKeyFileName);
-			}
-		}
-#endif
-		else {
-			lastOpenError = "invalid URL prefix";
-			throw backup_invalid_url();
-		}
-
-		r->encryptionKeyFileName = encryptionKeyFileName;
-		r->URL = url;
-		return r;
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled)
-			throw;
-
-		TraceEvent m(SevWarn, "BackupContainer");
-		m.error(e);
-		m.detail("Description", "Invalid container specification.  See help.");
-		m.detail("URL", url);
-		if (e.code() == error_code_backup_invalid_url)
-			m.detail("LastOpenError", lastOpenError);
-
-		throw;
-	}
-}
-
 namespace backup_test {
 
 int chooseFileSize(std::vector<int>& sizes) {
@@ -1885,40 +1885,36 @@ int chooseFileSize(std::vector<int>& sizes) {
 	return deterministicRandom()->randomInt(0, 2e6);
 }
 
-ACTOR Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c,
-                                      Reference<IBackupFile> f,
-                                      int size,
-                                      FlowLock* lock) {
-	state Standalone<VectorRef<uint8_t>> content;
+Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c, Reference<IBackupFile> f, int size, FlowLock* lock) {
+	Standalone<VectorRef<uint8_t>> content;
 
-	wait(lock->take(TaskPriority::DefaultYield, size));
-	state FlowLock::Releaser releaser(*lock, size);
+	co_await lock->take(TaskPriority::DefaultYield, size);
+	FlowLock::Releaser releaser(*lock, size);
 
 	printf("writeAndVerify size=%d file=%s\n", size, f->getFileName().c_str());
 	content.resize(content.arena(), size);
-	for (int i = 0; i < content.size(); ++i) {
-		content[i] = (uint8_t)deterministicRandom()->randomInt(0, 256);
+	for (auto& contentByte : content) {
+		contentByte = (uint8_t)deterministicRandom()->randomInt(0, 256);
 	}
 
-	state VectorRef<uint8_t> sendBuf = content;
-	while (sendBuf.size() > 0) {
-		state int n = std::min(sendBuf.size(), deterministicRandom()->randomInt(1, 16384));
-		wait(f->append(sendBuf.begin(), n));
+	VectorRef<uint8_t> sendBuf = content;
+	while (!sendBuf.empty()) {
+		int n = std::min(sendBuf.size(), deterministicRandom()->randomInt(1, 16384));
+		co_await f->append(sendBuf.begin(), n);
 		sendBuf.pop_front(n);
 	}
-	wait(f->finish());
+	co_await f->finish();
 
-	state Reference<IAsyncFile> inputFile = wait(c->readFile(f->getFileName()));
-	int64_t fileSize = wait(inputFile->size());
+	Reference<IAsyncFile> inputFile = co_await c->readFile(f->getFileName());
+	int64_t fileSize = co_await inputFile->size();
 	ASSERT_EQ(size, fileSize);
 	if (size > 0) {
-		state Standalone<VectorRef<uint8_t>> buf;
+		Standalone<VectorRef<uint8_t>> buf;
 		buf.resize(buf.arena(), fileSize);
-		int b = wait(inputFile->read(buf.begin(), buf.size(), 0));
+		int b = co_await inputFile->read(buf.begin(), buf.size(), 0);
 		ASSERT_EQ(b, buf.size());
 		ASSERT(buf == content);
 	}
-	return Void();
 }
 
 // Randomly advance version by up to 1 second of versions
@@ -1928,72 +1924,73 @@ Version nextVersion(Version v) {
 }
 
 // Write a snapshot file with only begin & end key
-ACTOR static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key begin, Key end, uint32_t blockSize) {
+static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key begin, Key end, uint32_t blockSize) {
 	ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size());
 
 	uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION;
 	// write Header
-	wait(file->append((uint8_t*)&fileVersion, sizeof(fileVersion)));
+	co_await file->append((uint8_t*)&fileVersion, sizeof(fileVersion));
 
 	// write begin key length and key
-	wait(file->appendStringRefWithLen(begin));
+	co_await file->appendStringRefWithLen(begin);
 
 	// write end key length and key
-	wait(file->appendStringRefWithLen(end));
+	co_await file->appendStringRefWithLen(end);
 
 	int bytesLeft = blockSize - file->size();
 	if (bytesLeft > 0) {
 		Value paddings = fileBackup::makePadding(bytesLeft);
-		wait(file->append(paddings.begin(), bytesLeft));
+		co_await file->append(paddings.begin(), bytesLeft);
 	}
-	wait(file->finish());
-	return Void();
+	co_await file->finish();
 }
 
-ACTOR Future<Void> testBackupContainer(std::string url,
-                                       Optional<std::string> proxy,
-                                       Optional<std::string> encryptionKeyFileName) {
-	state FlowLock lock(100e6);
+Future<Void> testBackupContainer(std::string url,
+                                 Optional<std::string> proxy,
+                                 Optional<std::string> encryptionKeyFileName) {
+	FlowLock lock(100e6);
 
 	if (encryptionKeyFileName.present()) {
-		wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get()));
+		co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 	}
 
 	printf("BackupContainerTest URL %s\n", url.c_str());
 
-	state Reference<IBackupContainer> c = IBackupContainer::openContainer(url, proxy, encryptionKeyFileName);
+	int encryptionBlockSize = encryptionKeyFileName.present() ? 4096 : 0;
+	Reference<IBackupContainer> c =
+	    IBackupContainer::openContainer(url, proxy, encryptionKeyFileName, encryptionBlockSize);
 
 	// Make sure container doesn't exist, then create it.
 	try {
-		wait(c->deleteContainer());
+		co_await c->deleteContainer();
 	} catch (Error& e) {
 		if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist)
 			throw;
 	}
 
-	wait(c->create());
+	co_await c->create();
 
-	state std::vector<Future<Void>> writes;
-	state std::map<Version, std::vector<std::string>> snapshots;
-	state std::map<Version, int64_t> snapshotSizes;
-	state std::map<Version, std::vector<std::pair<Key, Key>>> snapshotBeginEndKeys;
-	state int nRangeFiles = 0;
-	state std::map<Version, std::string> logs;
-	state Version v = deterministicRandom()->randomInt64(0, std::numeric_limits<Version>::max() / 2);
+	std::vector<Future<Void>> writes;
+	std::map<Version, std::vector<std::string>> snapshots;
+	std::map<Version, int64_t> snapshotSizes;
+	std::map<Version, std::vector<std::pair<Key, Key>>> snapshotBeginEndKeys;
+	int nRangeFiles = 0;
+	std::map<Version, std::string> logs;
+	Version v = deterministicRandom()->randomInt64(0, std::numeric_limits<Version>::max() / 2);
 
 	// List of sizes to use to test edge cases on underlying file implementations
-	state std::vector<int> fileSizes = { 0 };
+	std::vector<int> fileSizes = { 0 };
 	if (StringRef(url).startsWith("blob"_sr)) {
 		fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE);
 		fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10);
 	}
 
-	loop {
-		state Version logStart = v;
-		state int kvfiles = deterministicRandom()->randomInt(0, 3);
-		state Key begin = ""_sr;
-		state Key end = ""_sr;
-		state int blockSize = 3 * sizeof(uint32_t) + begin.size() + end.size() + 8;
+	while (true) {
+		Version logStart = v;
+		int kvfiles = deterministicRandom()->randomInt(0, 3);
+		Key begin = ""_sr;
+		Key end = ""_sr;
+		int blockSize = 3 * sizeof(uint32_t) + begin.size() + end.size() + 8;
 
 		while (kvfiles > 0) {
 			if (snapshots.empty()) {
@@ -2004,7 +2001,7 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 					v = nextVersion(v);
 				}
 			}
-			Reference<IBackupFile> range = wait(c->writeRangeFile(snapshots.rbegin()->first, 0, v, blockSize));
+			Reference<IBackupFile> range = co_await c->writeRangeFile(snapshots.rbegin()->first, 0, v, blockSize);
 			++nRangeFiles;
 			v = nextVersion(v);
 			snapshots.rbegin()->second.push_back(range->getFileName());
@@ -2014,7 +2011,7 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 			snapshotSizes.rbegin()->second += size;
 			// Write in actual range file format, instead of random data.
 			// writes.push_back(writeAndVerifyFile(c, range, size, &lock));
-			wait(testWriteSnapshotFile(range, begin, end, blockSize));
+			co_await testWriteSnapshotFile(range, begin, end, blockSize);
 
 			if (deterministicRandom()->random01() < .2) {
 				writes.push_back(c->writeKeyspaceSnapshotFile(snapshots.rbegin()->second,
@@ -2033,7 +2030,7 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 		if (logStart == v || deterministicRandom()->coinflip()) {
 			v = nextVersion(v);
 		}
-		state Reference<IBackupFile> log = wait(c->writeLogFile(logStart, v, 10));
+		Reference<IBackupFile> log = co_await c->writeLogFile(logStart, v, 10);
 		logs[logStart] = log->getFileName();
 		int size = chooseFileSize(fileSizes);
 		writes.push_back(writeAndVerifyFile(c, log, size, &lock));
@@ -2046,99 +2043,95 @@ ACTOR Future<Void> testBackupContainer(std::string url,
 		}
 	}
 
-	wait(waitForAll(writes));
+	co_await waitForAll(writes);
 
-	state BackupFileList listing = wait(c->dumpFileList());
+	BackupFileList listing = co_await c->dumpFileList();
 	ASSERT_EQ(listing.ranges.size(), nRangeFiles);
 	ASSERT_EQ(listing.logs.size(), logs.size());
 	ASSERT_EQ(listing.snapshots.size(), snapshots.size());
 
-	state BackupDescription desc = wait(c->describeBackup());
+	BackupDescription desc = co_await c->describeBackup();
 	printf("\n%s\n", desc.toString().c_str());
 
 	// Do a series of expirations and verify resulting state
-	state int i = 0;
+	int i = 0;
 	for (; i < listing.snapshots.size(); ++i) {
 		{
 			// Ensure we can still restore to the latest version
-			Optional<RestorableFileSet> rest = wait(c->getRestoreSet(desc.maxRestorableVersion.get()));
+			Optional<RestorableFileSet> rest = co_await c->getRestoreSet(desc.maxRestorableVersion.get());
 			ASSERT(rest.present());
 		}
 
 		{
 			// Ensure we can restore to the end version of snapshot i
-			Optional<RestorableFileSet> rest = wait(c->getRestoreSet(listing.snapshots[i].endVersion));
+			Optional<RestorableFileSet> rest = co_await c->getRestoreSet(listing.snapshots[i].endVersion);
 			ASSERT(rest.present());
 		}
 
 		// Test expiring to the end of this snapshot
-		state Version expireVersion = listing.snapshots[i].endVersion;
+		Version expireVersion = listing.snapshots[i].endVersion;
 
 		// Expire everything up to but not including the snapshot end version
 		fmt::print("EXPIRE TO {}\n", expireVersion);
-		state Future<Void> f = c->expireData(expireVersion);
-		wait(ready(f));
+		Future<Void> f = c->expireData(expireVersion);
+		co_await ready(f);
 
 		// If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot
 		if (f.isError()) {
 			ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire);
 			ASSERT_EQ(i, listing.snapshots.size() - 1);
-			wait(c->expireData(expireVersion, true));
+			co_await c->expireData(expireVersion, true);
 		}
 
-		BackupDescription d = wait(c->describeBackup());
+		BackupDescription d = co_await c->describeBackup();
 		printf("\n%s\n", d.toString().c_str());
 	}
 
 	printf("DELETING\n");
-	wait(c->deleteContainer());
+	co_await c->deleteContainer();
 
-	state Future<BackupDescription> d = c->describeBackup();
-	wait(ready(d));
+	Future<BackupDescription> d = c->describeBackup();
+	co_await ready(d);
 	ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist);
 
-	BackupFileList empty = wait(c->dumpFileList());
+	BackupFileList empty = co_await c->dumpFileList();
 	ASSERT_EQ(empty.ranges.size(), 0);
 	ASSERT_EQ(empty.logs.size(), 0);
 	ASSERT_EQ(empty.snapshots.size(), 0);
 
 	printf("BackupContainerTest URL=%s PASSED.\n", url.c_str());
-
-	return Void();
 }
 
 TEST_CASE("/backup/containers/localdir/unencrypted") {
-	wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {}, {}));
-	return Void();
+	co_await testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()),
+	                             Optional<std::string>(),
+	                             Optional<std::string>());
 }
 
 TEST_CASE("/backup/containers/localdir/encrypted") {
-	wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()),
-	                         {},
-	                         format("%s/test_encryption_key", params.getDataDir().c_str())));
-	return Void();
+	co_await testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()),
+	                             Optional<std::string>(),
+	                             format("%s/test_encryption_key", params.getDataDir().c_str()));
 }
 
 TEST_CASE("/backup/containers/url") {
 	if (!g_network->isSimulated()) {
 		const char* url = getenv("FDB_TEST_BACKUP_URL");
 		ASSERT(url != nullptr);
-		wait(testBackupContainer(url, {}, {}));
+		co_await testBackupContainer(url, Optional<std::string>(), Optional<std::string>());
 	}
-	return Void();
 }
 
 TEST_CASE("/backup/containers_list") {
 	if (!g_network->isSimulated()) {
-		state const char* url = getenv("FDB_TEST_BACKUP_URL");
+		const char* url = getenv("FDB_TEST_BACKUP_URL");
 		ASSERT(url != nullptr);
 		printf("Listing %s\n", url);
-		std::vector<std::string> urls = wait(IBackupContainer::listContainers(url, {}));
+		std::vector<std::string> urls = co_await IBackupContainer::listContainers(url, {});
 		for (auto& u : urls) {
 			printf("%s\n", u.c_str());
 		}
 	}
-	return Void();
 }
 
 TEST_CASE("/backup/time") {
@@ -2376,15 +2369,15 @@ TEST_CASE("/backup/logs_continuous") {
 
 void printFileList(BackupFileList& backupFileList) {
 	printf("\nRangeFiles count:%lu", backupFileList.ranges.size());
-	for (auto r : backupFileList.ranges)
+	for (const auto& r : backupFileList.ranges)
 		printf("\n%s", r.toString().c_str());
 
 	printf("\nLogFiles count:%lu", backupFileList.logs.size());
-	for (auto l : backupFileList.logs)
+	for (const auto& l : backupFileList.logs)
 		printf("\n%s", l.toString().c_str());
 
 	printf("\nSnapshotFiles count:%lu", backupFileList.snapshots.size());
-	for (auto s : backupFileList.snapshots)
+	for (const auto& s : backupFileList.snapshots)
 		printf("\n%" PRId64 ", %" PRId64 ", %s, %" PRId64 "\n",
 		       s.beginVersion,
 		       s.endVersion,
@@ -2393,45 +2386,46 @@ void printFileList(BackupFileList& backupFileList) {
 }
 
 // Intentionally missing some log range files and checking if the snapshot can be restored.
-ACTOR Future<Void> testBackupContainerWithMissingLogRanges(std::string url, Optional<std::string> proxy) {
-	state FlowLock lock(100e6);
+Future<Void> testBackupContainerWithMissingLogRanges(std::string url, Optional<std::string> proxy) {
+	FlowLock lock(100e6);
 	printf("BackupContainerTest URL %s\n", url.c_str());
 
-	state Reference<IBackupContainer> c = IBackupContainer::openContainer(url, proxy, {});
+	Reference<IBackupContainer> c =
+	    IBackupContainer::openContainer(url, proxy, /*encryptionKeyFileName=*/{}, /*encryptionBlockSize=*/0);
 	// Make sure container doesn't exist, then create it.
 	try {
-		wait(c->deleteContainer());
+		co_await c->deleteContainer();
 	} catch (Error& e) {
 		if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist)
 			throw;
 	}
-	wait(c->create());
-
-	state Key begin = randomKeyBetween(normalKeys);
-	state Key end = randomKeyBetween(KeyRangeRef(begin, normalKeys.end));
-	state int blockSize = 3 * sizeof(uint32_t) + begin.size() + end.size() + 8;
-	state std::vector<Future<Void>> writes;
-	state std::pair<Key, Key> beginEndKeys = std::make_pair(begin, end);
-	state std::vector<bool> snapshotsMissingLogs;
-	state Version v = deterministicRandom()->randomInt64(0, std::numeric_limits<Version>::max() / 2);
-	state Version tempLogEnd = 0;
-	state Version logStart = v;
-	state Version logEnd = v;
-	state Version snapshotBeginVersion = v;
-	state Version snapshotEndVersion = v;
-	state Version lastMissedLogFileEnd = 0;
+	co_await c->create();
+
+	Key begin = randomKeyBetween(normalKeys);
+	Key end = randomKeyBetween(KeyRangeRef(begin, normalKeys.end));
+	int blockSize = 3 * sizeof(uint32_t) + begin.size() + end.size() + 8;
+	std::vector<Future<Void>> writes;
+	std::pair<Key, Key> beginEndKeys = std::make_pair(begin, end);
+	std::vector<bool> snapshotsMissingLogs;
+	Version v = deterministicRandom()->randomInt64(0, std::numeric_limits<Version>::max() / 2);
+	Version tempLogEnd = 0;
+	Version logStart = v;
+	Version logEnd = v;
+	Version snapshotBeginVersion = v;
+	Version snapshotEndVersion = v;
+	Version lastMissedLogFileEnd = 0;
 
 	// create a random number of snapshots
-	state int numSnapshots = deterministicRandom()->randomInt(1, 10);
+	int numSnapshots = deterministicRandom()->randomInt(1, 10);
 	while (numSnapshots) {
-		state std::vector<std::string> rangeFileNames;
-		state std::vector<std::pair<Key, Key>> snapshotBeginEndKeys;
+		std::vector<std::string> rangeFileNames;
+		std::vector<std::pair<Key, Key>> snapshotBeginEndKeys;
 
 		// create a random number of range files per snapshot
-		state int numRangeFiles = deterministicRandom()->randomInt(2, 5);
+		int numRangeFiles = deterministicRandom()->randomInt(2, 5);
 		snapshotBeginVersion = v;
 		while (numRangeFiles) {
-			state Reference<IBackupFile> range = wait(c->writeRangeFile(v, 0, v, blockSize));
+			Reference<IBackupFile> range = co_await c->writeRangeFile(v, 0, v, blockSize);
 			writes.push_back(writeAndVerifyFile(c, range, deterministicRandom()->randomInt(0, 2e6), &lock));
 			rangeFileNames.push_back(range->getFileName());
 			snapshotBeginEndKeys.push_back(beginEndKeys);
@@ -2456,7 +2450,7 @@ ACTOR Future<Void> testBackupContainerWithMissingLogRanges(std::string url, Opti
 		while (logStart < logEnd) {
 			tempLogEnd = nextVersion(logStart);
 			if (deterministicRandom()->random01() < 0.5) {
-				state Reference<IBackupFile> log = wait(c->writeLogFile(logStart, tempLogEnd, blockSize));
+				Reference<IBackupFile> log = co_await c->writeLogFile(logStart, tempLogEnd, blockSize);
 				writes.push_back(writeAndVerifyFile(c, log, deterministicRandom()->randomInt(0, 2e6), &lock));
 			} else { // intentionally missing writing of some log files.
 				// If the missing log range falls in the current snapshot range, mark it.
@@ -2470,21 +2464,21 @@ ACTOR Future<Void> testBackupContainerWithMissingLogRanges(std::string url, Opti
 		--numSnapshots;
 	}
 
-	wait(waitForAll(writes));
-	state BackupFileList listing = wait(c->dumpFileList());
+	co_await waitForAll(writes);
+	BackupFileList listing = co_await c->dumpFileList();
 	printFileList(listing);
 
 	printf("\n\nSnapshots missing logs:");
-	state int i = 0;
+	int i = 0;
 	for (; i < snapshotsMissingLogs.size(); ++i)
 		printf("\nSnapshot%d: %s", i, snapshotsMissingLogs[i] ? "true" : "false");
 
-	state BackupDescription desc = wait(c->describeBackup());
+	BackupDescription desc = co_await c->describeBackup();
 	printf("\n\n%s\n", desc.toString().c_str());
 
 	for (i = 0; i < listing.snapshots.size(); ++i) {
 		// Ensure we can restore to the end version of snapshot i
-		Optional<RestorableFileSet> rest = wait(c->getRestoreSet(listing.snapshots[i].endVersion));
+		Optional<RestorableFileSet> rest = co_await c->getRestoreSet(listing.snapshots[i].endVersion);
 		if (snapshotsMissingLogs[i])
 			ASSERT(!rest.present());
 		else
@@ -2509,59 +2503,57 @@ ACTOR Future<Void> testBackupContainerWithMissingLogRanges(std::string url, Opti
 	}
 
 	printf("DELETING\n");
-	wait(c->deleteContainer());
+	co_await c->deleteContainer();
 
-	state Future<BackupDescription> d = c->describeBackup();
-	wait(ready(d));
+	Future<BackupDescription> d = c->describeBackup();
+	co_await ready(d);
 	ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist);
 
-	BackupFileList empty = wait(c->dumpFileList());
+	BackupFileList empty = co_await c->dumpFileList();
 	ASSERT_EQ(empty.ranges.size(), 0);
 	ASSERT_EQ(empty.logs.size(), 0);
 	ASSERT_EQ(empty.snapshots.size(), 0);
 
 	printf("BackupContainerTest URL=%s PASSED.\n", url.c_str());
-
-	return Void();
 }
 
 TEST_CASE("/backup/containers/localdir/missingLogRangesRestorability") {
-	wait(testBackupContainerWithMissingLogRanges(
-	    format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {}));
-	return Void();
+	co_await testBackupContainerWithMissingLogRanges(
+	    format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), Optional<std::string>());
 }
 
-ACTOR Future<Void> testBackupContinuousLogEndVer(std::string url, Optional<std::string> proxy) {
-	state FlowLock lock(100e6);
+Future<Void> testBackupContinuousLogEndVer(std::string url, Optional<std::string> proxy) {
+	FlowLock lock(100e6);
 	printf("BackupContainerTest URL %s\n", url.c_str());
-	state Reference<IBackupContainer> c = IBackupContainer::openContainer(url, proxy, {});
+	Reference<IBackupContainer> c =
+	    IBackupContainer::openContainer(url, proxy, /*encryptionKeyFileName=*/{}, /*encryptionBlockSize=*/0);
 
 	// Make sure container doesn't exist, then create it.
 	try {
-		wait(c->deleteContainer());
+		co_await c->deleteContainer();
 	} catch (Error& e) {
 		if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist)
 			throw;
 	}
 
-	wait(c->create());
+	co_await c->create();
 
-	state int blockSize = 1024;
-	state std::vector<std::string> rangeFileNames;
-	state Key begin = randomKeyBetween(normalKeys);
-	state Key end = randomKeyBetween(KeyRangeRef(begin, normalKeys.end));
-	state std::pair<Key, Key> beginEndKeys = std::make_pair(begin, end);
-	state std::vector<std::pair<Key, Key>> snapshotBeginEndKeys;
+	int blockSize = 1024;
+	std::vector<std::string> rangeFileNames;
+	Key begin = randomKeyBetween(normalKeys);
+	Key end = randomKeyBetween(KeyRangeRef(begin, normalKeys.end));
+	std::pair<Key, Key> beginEndKeys = std::make_pair(begin, end);
+	std::vector<std::pair<Key, Key>> snapshotBeginEndKeys;
 
 	// writing random number of range files with rangeSize 100
-	state std::vector<Future<Void>> writes;
-	state Version snapshotBeginVersion = 10;
-	state Version snapshotEndVersion = deterministicRandom()->randomInt(500, 1000);
-	state Version rangeSize = 100;
-	state Version v = snapshotBeginVersion;
-	state int numRangeFiles = 0;
+	std::vector<Future<Void>> writes;
+	Version snapshotBeginVersion = 10;
+	Version snapshotEndVersion = deterministicRandom()->randomInt(500, 1000);
+	Version rangeSize = 100;
+	Version v = snapshotBeginVersion;
+	int numRangeFiles = 0;
 	while (v <= snapshotEndVersion) {
-		state Reference<IBackupFile> range = wait(c->writeRangeFile(v, 0, v, blockSize));
+		Reference<IBackupFile> range = co_await c->writeRangeFile(v, 0, v, blockSize);
 		writes.push_back(writeAndVerifyFile(c, range, 100, &lock));
 		rangeFileNames.push_back(range->getFileName());
 		snapshotBeginEndKeys.push_back(beginEndKeys);
@@ -2571,11 +2563,11 @@ ACTOR Future<Void> testBackupContinuousLogEndVer(std::string url, Optional<std::
 	snapshotEndVersion = v - rangeSize;
 
 	// writing random number of log files with logSize 70, covering the entire snapshot
-	state Version logSize = 70;
+	Version logSize = 70;
 	v = snapshotBeginVersion;
-	state int numLogFiles = 0;
+	int numLogFiles = 0;
 	while (v <= snapshotEndVersion) {
-		Reference<IBackupFile> log = wait(c->writeLogFile(v, v + logSize, blockSize));
+		Reference<IBackupFile> log = co_await c->writeLogFile(v, v + logSize, blockSize);
 		writes.push_back(writeAndVerifyFile(c, log, 100, &lock));
 		++numLogFiles;
 		v += logSize;
@@ -2584,15 +2576,15 @@ ACTOR Future<Void> testBackupContinuousLogEndVer(std::string url, Optional<std::
 	// writing snapshot file
 	writes.push_back(c->writeKeyspaceSnapshotFile(
 	    rangeFileNames, snapshotBeginEndKeys, deterministicRandom()->randomInt(0, 2e6), IncludeKeyRangeMap(BUGGIFY)));
-	wait(waitForAll(writes));
+	co_await waitForAll(writes);
 
-	state BackupFileList fileList = wait(c->dumpFileList());
+	BackupFileList fileList = co_await c->dumpFileList();
 	printFileList(fileList);
 	ASSERT_EQ(fileList.ranges.size(), numRangeFiles);
 	ASSERT_EQ(fileList.logs.size(), numLogFiles);
 	ASSERT_EQ(fileList.snapshots.size(), 1);
 
-	state BackupDescription desc = wait(c->describeBackup());
+	BackupDescription desc = co_await c->describeBackup();
 	printf("\n%s\n", desc.toString().c_str());
 	ASSERT_EQ(desc.minLogBegin, snapshotBeginVersion);
 	ASSERT_EQ(desc.maxLogEnd, v);
@@ -2602,24 +2594,24 @@ ACTOR Future<Void> testBackupContinuousLogEndVer(std::string url, Optional<std::
 	ASSERT_EQ(desc.contiguousLogEnd, v);
 
 	// writing random number of more continuous log files
-	state int newNumLogFiles = deterministicRandom()->randomInt(2, 8);
+	int newNumLogFiles = deterministicRandom()->randomInt(2, 8);
 	numLogFiles += newNumLogFiles;
 	writes.clear();
 	while (newNumLogFiles) {
-		Reference<IBackupFile> log = wait(c->writeLogFile(v, v + logSize, blockSize));
+		Reference<IBackupFile> log = co_await c->writeLogFile(v, v + logSize, blockSize);
 		writes.push_back(writeAndVerifyFile(c, log, 100, &lock));
 		--newNumLogFiles;
 		v += logSize;
 	}
-	wait(waitForAll(writes));
+	co_await waitForAll(writes);
 
-	state BackupFileList fileList1 = wait(c->dumpFileList());
+	BackupFileList fileList1 = co_await c->dumpFileList();
 	printFileList(fileList1);
 	ASSERT_EQ(fileList1.ranges.size(), numRangeFiles);
 	ASSERT_EQ(fileList1.logs.size(), numLogFiles);
 	ASSERT_EQ(fileList1.snapshots.size(), 1);
 
-	state BackupDescription desc1 = wait(c->describeBackup());
+	BackupDescription desc1 = co_await c->describeBackup();
 	printf("\n%s\n", desc1.toString().c_str());
 	ASSERT_EQ(desc1.minLogBegin, snapshotBeginVersion);
 	ASSERT_EQ(desc1.maxLogEnd, v);
@@ -2627,14 +2619,11 @@ ACTOR Future<Void> testBackupContinuousLogEndVer(std::string url, Optional<std::
 	ASSERT_EQ(desc1.maxRestorableVersion, v - 1);
 	ASSERT_EQ(desc1.snapshots[0].restorable, true);
 	ASSERT_EQ(desc1.contiguousLogEnd, v);
-
-	return Void();
 }
 
 TEST_CASE("/backup/containers/localdir/continuousLogEndVersion") {
-	wait(testBackupContinuousLogEndVer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()),
-	                                   {}));
-	return Void();
+	co_await testBackupContinuousLogEndVer(
+	    format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), Optional<std::string>());
 }
 
 } // namespace backup_test
diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.cpp
similarity index 86%
rename from fdbclient/BackupContainerLocalDirectory.actor.cpp
rename to fdbclient/BackupContainerLocalDirectory.cpp
index a7e63c39fed..00b83a79b0b 100644
--- a/fdbclient/BackupContainerLocalDirectory.actor.cpp
+++ b/fdbclient/BackupContainerLocalDirectory.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupContainerLocalDirectory.actor.cpp
+ * BackupContainerLocalDirectory.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,14 @@
  * limitations under the License.
  */
 
-#include "fdbclient/BackupContainerLocalDirectory.h"
-#include "fdbrpc/AsyncFileReadAhead.actor.h"
+#include "BackupContainerLocalDirectory.h"
+#include "fdbrpc/AsyncFileEncrypted.h"
+#include "fdbrpc/AsyncFileReadAhead.h"
 #include "flow/IAsyncFile.h"
 #include "flow/FaultInjection.h"
-#include "flow/Platform.actor.h"
 #include "flow/Platform.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
@@ -73,17 +72,15 @@ class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
 		return r;
 	}
 
-	ACTOR static Future<Void> finish_impl(Reference<BackupFile> f) {
-		wait(f->flush(f->m_buffer.size()));
-		wait(f->m_file->truncate(f->size())); // Some IAsyncFile implementations extend in whole block sizes.
-		wait(f->m_file->sync());
+	static Future<Void> finish_impl(Reference<BackupFile> f) {
+		co_await f->flush(f->m_buffer.size());
+		co_await f->m_file->truncate(f->size()); // Some IAsyncFile implementations extend in whole block sizes.
+		co_await f->m_file->sync();
 		std::string name = f->m_file->getFilename();
 		f->m_file.clear();
-		wait(IAsyncFileSystem::filesystem()->renameFile(name, f->m_finalFullPath));
+		co_await IAsyncFileSystem::filesystem()->renameFile(name, f->m_finalFullPath);
 
 		INJECT_BLOB_FAULT(http_request_failed, "BackupContainerLocalDirectory::finish");
-
-		return Void();
 	}
 
 	int64_t size() const override { return m_buffer.size() + m_writeOffset; }
@@ -101,9 +98,9 @@ class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
 	int m_blockSize;
 };
 
-ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(std::string path, std::string m_path) {
-	state std::vector<std::string> files;
-	wait(platform::findFilesRecursivelyAsync(joinPath(m_path, path), &files));
+static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(std::string path, std::string m_path) {
+	std::vector<std::string> files;
+	co_await platform::findFilesRecursivelyAsync(joinPath(m_path, path), &files);
 
 	BackupContainerFileSystem::FilesAndSizesT results;
 
@@ -124,7 +121,7 @@ ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(st
 
 	INJECT_BLOB_FAULT(http_request_failed, "BackupContainerLocalDirectory::listFiles");
 
-	return results;
+	co_return results;
 }
 
 } // namespace
@@ -141,8 +138,10 @@ std::string BackupContainerLocalDirectory::getURLFormat() {
 }
 
 BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url,
-                                                             const Optional<std::string>& encryptionKeyFileName) {
+                                                             const Optional<std::string>& encryptionKeyFileName,
+                                                             int encryptionBlockSize) {
 	setEncryptionKey(encryptionKeyFileName);
+	this->encryptionBlockSize = encryptionBlockSize;
 
 	std::string path;
 	if (url.find("file://") != 0) {
@@ -233,10 +232,6 @@ Future<bool> BackupContainerLocalDirectory::exists() {
 
 Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std::string& path) {
 	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED;
-	// Skip encryption for properties/ folder
-	if (usesEncryption() && !StringRef(path).startsWith("properties/"_sr)) {
-		flags |= IAsyncFile::OPEN_ENCRYPTED;
-	}
 	INJECT_BLOB_FAULT(http_request_failed, "BackupContainerLocalDirectory::readFile");
 	// Simulation does not properly handle opening the same file from multiple machines using a shared filesystem,
 	// so create a symbolic link to make each file opening appear to be unique.  This could also work in production
@@ -261,6 +256,14 @@ Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std:
 // by the same process that failed the first task execution anyway, which is a very rare case.
 #endif
 	Future<Reference<IAsyncFile>> f = IAsyncFileSystem::filesystem()->open(fullPath, flags, 0644);
+	// Skip encryption for properties/ folder
+	if (usesEncryption() && !StringRef(path).startsWith("properties/"_sr)) {
+		int encBlockSize = encryptionBlockSize;
+		f = map(f, [encBlockSize](Reference<IAsyncFile> r) {
+			return Reference<IAsyncFile>(
+			    makeReference<AsyncFileEncrypted>(r, AsyncFileEncrypted::Mode::READ_ONLY, encBlockSize));
+		});
+	}
 
 	if (g_network->isSimulated()) {
 		int blockSize = 0;
@@ -277,11 +280,11 @@ Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std:
 		}
 		ASSERT(blockSize > 0);
 
-		return map(f, [=](Reference<IAsyncFile> fr) {
+		return map(f, [=](Reference<IAsyncFile> fr) -> Reference<IAsyncFile> {
 			int readAhead = deterministicRandom()->randomInt(0, 3);
 			int reads = deterministicRandom()->randomInt(1, 3);
 			int cacheSize = deterministicRandom()->randomInt(0, 3);
-			return Reference<IAsyncFile>(new AsyncFileReadAheadCache(fr, blockSize, readAhead, reads, cacheSize));
+			return makeReference<AsyncFileReadAheadCache>(fr, blockSize, readAhead, reads, cacheSize);
 		});
 	}
 
@@ -292,15 +295,21 @@ Future<Reference<IBackupFile>> BackupContainerLocalDirectory::writeFile(const st
 	INJECT_BLOB_FAULT(http_request_failed, "BackupContainerLocalDirectory::writeFile");
 	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE |
 	            IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE;
-	// Skip encryption for properties/ folder
-	if (usesEncryption() && !StringRef(path).startsWith("properties/"_sr)) {
-		flags |= IAsyncFile::OPEN_ENCRYPTED;
-	}
 	std::string fullPath = joinPath(m_path, path);
 	platform::createDirectory(parentDirectory(fullPath));
 	std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp";
 	Future<Reference<IAsyncFile>> f = IAsyncFileSystem::filesystem()->open(temp, flags, 0644);
-	return map(f, [=](Reference<IAsyncFile> f) { return Reference<IBackupFile>(new BackupFile(path, f, fullPath)); });
+	// Skip encryption for properties/ folder
+	if (usesEncryption() && !StringRef(path).startsWith("properties/"_sr)) {
+		int encBlockSize = encryptionBlockSize;
+		f = map(f, [encBlockSize](Reference<IAsyncFile> r) {
+			return Reference<IAsyncFile>(
+			    makeReference<AsyncFileEncrypted>(r, AsyncFileEncrypted::Mode::APPEND_ONLY, encBlockSize));
+		});
+	}
+	return map(f, [=](Reference<IAsyncFile> file) -> Reference<IBackupFile> {
+		return makeReference<BackupFile>(path, file, fullPath);
+	});
 }
 
 Future<Void> BackupContainerLocalDirectory::writeEntireFile(const std::string& path, const std::string& contents) {
@@ -324,7 +333,7 @@ Future<Void> BackupContainerLocalDirectory::deleteContainer(int* pNumDeleted) {
 	// and make sure it has something in it.
 	return map(describeBackup(false, invalidVersion), [=](BackupDescription const& desc) {
 		// If the backup has no snapshots and no logs then it's probably not a valid backup
-		if (desc.snapshots.size() == 0 && !desc.minLogBegin.present())
+		if (desc.snapshots.empty() && !desc.minLogBegin.present())
 			throw backup_invalid_url();
 
 		int count = platform::eraseDirectoryRecursive(m_path);
diff --git a/fdbclient/include/fdbclient/BackupContainerLocalDirectory.h b/fdbclient/BackupContainerLocalDirectory.h
similarity index 90%
rename from fdbclient/include/fdbclient/BackupContainerLocalDirectory.h
rename to fdbclient/BackupContainerLocalDirectory.h
index 47d34d87699..40ed369f41f 100644
--- a/fdbclient/include/fdbclient/BackupContainerLocalDirectory.h
+++ b/fdbclient/BackupContainerLocalDirectory.h
@@ -33,7 +33,9 @@ class BackupContainerLocalDirectory : public BackupContainerFileSystem,
 
 	static std::string getURLFormat();
 
-	BackupContainerLocalDirectory(const std::string& url, Optional<std::string> const& encryptionKeyFileName);
+	BackupContainerLocalDirectory(const std::string& url,
+	                              Optional<std::string> const& encryptionKeyFileName,
+	                              int encryptionBlockSize);
 
 	static Future<std::vector<std::string>> listURLs(const std::string& url);
 
diff --git a/fdbclient/BackupTLSConfig.cpp b/fdbclient/BackupTLSConfig.cpp
index b866bee9236..a8e1ed79092 100644
--- a/fdbclient/BackupTLSConfig.cpp
+++ b/fdbclient/BackupTLSConfig.cpp
@@ -34,13 +34,13 @@ void BackupTLSConfig::setupBlobCredentials() {
 		StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV));
 		do {
 			StringRef file = t.eat(":");
-			if (file.size() != 0)
+			if (!file.empty())
 				blobCredentials.push_back(file.toString());
-		} while (t.size() != 0);
+		} while (!t.empty());
 	}
 
 	// Update the global blob credential files list
-	std::vector<std::string>* pFiles = (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
+	auto* pFiles = (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
 	if (pFiles != nullptr) {
 		for (auto& f : blobCredentials) {
 			pFiles->push_back(f);
@@ -49,7 +49,7 @@ void BackupTLSConfig::setupBlobCredentials() {
 }
 
 bool BackupTLSConfig::setupTLS() {
-	if (tlsCertPath.size()) {
+	if (!tlsCertPath.empty()) {
 		try {
 			setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath);
 		} catch (Error& e) {
@@ -58,7 +58,7 @@ bool BackupTLSConfig::setupTLS() {
 		}
 	}
 
-	if (tlsCAPath.size()) {
+	if (!tlsCAPath.empty()) {
 		try {
 			setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath);
 		} catch (Error& e) {
@@ -66,9 +66,9 @@ bool BackupTLSConfig::setupTLS() {
 			return false;
 		}
 	}
-	if (tlsKeyPath.size()) {
+	if (!tlsKeyPath.empty()) {
 		try {
-			if (tlsPassword.size())
+			if (!tlsPassword.empty())
 				setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword);
 
 			setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath);
@@ -77,7 +77,7 @@ bool BackupTLSConfig::setupTLS() {
 			return false;
 		}
 	}
-	if (tlsVerifyPeers.size()) {
+	if (!tlsVerifyPeers.empty()) {
 		try {
 			setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers);
 		} catch (Error& e) {
diff --git a/fdbclient/BlobStoreCommon.cpp b/fdbclient/BlobStoreCommon.cpp
new file mode 100644
index 00000000000..693f28bce88
--- /dev/null
+++ b/fdbclient/BlobStoreCommon.cpp
@@ -0,0 +1,1015 @@
+/*
+ * BlobStoreCommon.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/IBlobStore.h"
+#include "fdbclient/S3BlobStore.h"
+#include "GCSBlobStore.h"
+#include "fdbclient/Knobs.h"
+#include "flow/Hostname.h"
+#include "flow/IAsyncFile.h"
+#include "flow/IConnection.h"
+#include "flow/Trace.h"
+#include "flow/CoroUtils.h"
+#include "flow/genericactors.actor.h"
+
+#include "md5/md5.h"
+#include "libb64/encode.h"
+#include <openssl/sha.h>
+
+#include <boost/algorithm/string.hpp>
+#include <climits>
+#include <limits>
+
+IBlobStoreEndpoint::ConnectionPoolData::~ConnectionPoolData() {
+	// In simulation, explicitly close all pooled connections before destruction.
+	// This satisfies Sim2Conn's assertion: !opened || closedByCaller
+	// Without this, connections would be destroyed without being closed, causing assertion failures.
+	if (g_network && g_network->isSimulated()) {
+		while (!pool.empty()) {
+			ReusableConnection& rconn = pool.front();
+			if (rconn.conn.isValid()) {
+				rconn.conn->close();
+			}
+			pool.pop();
+		}
+	}
+}
+
+json_spirit::mObject IBlobStoreEndpoint::Stats::getJSON() {
+	json_spirit::mObject o;
+
+	o["requests_failed"] = requests_failed;
+	o["requests_successful"] = requests_successful;
+	o["bytes_sent"] = bytes_sent;
+
+	return o;
+}
+
+IBlobStoreEndpoint::Stats IBlobStoreEndpoint::Stats::operator-(const Stats& rhs) {
+	Stats r;
+	r.requests_failed = requests_failed - rhs.requests_failed;
+	r.requests_successful = requests_successful - rhs.requests_successful;
+	r.bytes_sent = bytes_sent - rhs.bytes_sent;
+	return r;
+}
+
+IBlobStoreEndpoint::Stats IBlobStoreEndpoint::s_stats;
+
+BlobKnobs::BlobKnobs() {
+	secure_connection = 1;
+	connect_tries = CLIENT_KNOBS->BLOBSTORE_CONNECT_TRIES;
+	connect_timeout = CLIENT_KNOBS->BLOBSTORE_CONNECT_TIMEOUT;
+	max_connection_life = CLIENT_KNOBS->BLOBSTORE_MAX_CONNECTION_LIFE;
+	request_tries = CLIENT_KNOBS->BLOBSTORE_REQUEST_TRIES;
+	request_timeout_min = CLIENT_KNOBS->BLOBSTORE_REQUEST_TIMEOUT_MIN;
+	requests_per_second = CLIENT_KNOBS->BLOBSTORE_REQUESTS_PER_SECOND;
+	concurrent_requests = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_REQUESTS;
+	list_requests_per_second = CLIENT_KNOBS->BLOBSTORE_LIST_REQUESTS_PER_SECOND;
+	write_requests_per_second = CLIENT_KNOBS->BLOBSTORE_WRITE_REQUESTS_PER_SECOND;
+	read_requests_per_second = CLIENT_KNOBS->BLOBSTORE_READ_REQUESTS_PER_SECOND;
+	delete_requests_per_second = CLIENT_KNOBS->BLOBSTORE_DELETE_REQUESTS_PER_SECOND;
+	multipart_max_part_size = CLIENT_KNOBS->BLOBSTORE_MULTIPART_MAX_PART_SIZE;
+	multipart_min_part_size = CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE;
+	concurrent_uploads = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_UPLOADS;
+	concurrent_lists = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_LISTS;
+	concurrent_reads_per_file = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_READS_PER_FILE;
+	concurrent_writes_per_file = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
+	enable_read_cache = CLIENT_KNOBS->BLOBSTORE_ENABLE_READ_CACHE;
+	read_block_size = CLIENT_KNOBS->BLOBSTORE_READ_BLOCK_SIZE;
+	read_ahead_blocks = CLIENT_KNOBS->BLOBSTORE_READ_AHEAD_BLOCKS;
+	read_cache_blocks_per_file = CLIENT_KNOBS->BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
+	max_send_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
+	max_recv_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
+	max_delay_retryable_error = CLIENT_KNOBS->BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
+	max_delay_connection_failed = CLIENT_KNOBS->BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
+	multipart_retry_delay_ms = CLIENT_KNOBS->BLOBSTORE_MULTIPART_RETRY_DELAY_MS;
+	sdk_auth = false;
+	enable_object_integrity_check = CLIENT_KNOBS->BLOBSTORE_ENABLE_OBJECT_INTEGRITY_CHECK;
+	global_connection_pool = CLIENT_KNOBS->BLOBSTORE_GLOBAL_CONNECTION_POOL;
+}
+
+bool BlobKnobs::set(StringRef name, int value) {
+#define TRY_PARAM(n, sn)                                                                                               \
+	if (name == #n || name == #sn) {                                                                                   \
+		n = value;                                                                                                     \
+		return true;                                                                                                   \
+	}
+	TRY_PARAM(secure_connection, sc)
+	TRY_PARAM(connect_tries, ct);
+	TRY_PARAM(connect_timeout, cto);
+	TRY_PARAM(max_connection_life, mcl);
+	TRY_PARAM(request_tries, rt);
+	TRY_PARAM(request_timeout_min, rtom);
+	// TODO: For backward compatibility because request_timeout was renamed to request_timeout_min
+	if (name == "request_timeout"_sr || name == "rto"_sr) {
+		request_timeout_min = value;
+		return true;
+	}
+	TRY_PARAM(requests_per_second, rps);
+	TRY_PARAM(list_requests_per_second, lrps);
+	TRY_PARAM(write_requests_per_second, wrps);
+	TRY_PARAM(read_requests_per_second, rrps);
+	TRY_PARAM(delete_requests_per_second, drps);
+	TRY_PARAM(concurrent_requests, cr);
+	TRY_PARAM(multipart_max_part_size, maxps);
+	TRY_PARAM(multipart_min_part_size, minps);
+	TRY_PARAM(multipart_retry_delay_ms, mrd);
+	TRY_PARAM(concurrent_uploads, cu);
+	TRY_PARAM(concurrent_lists, cl);
+	TRY_PARAM(concurrent_reads_per_file, crpf);
+	TRY_PARAM(concurrent_writes_per_file, cwpf);
+	TRY_PARAM(enable_read_cache, erc);
+	TRY_PARAM(read_block_size, rbs);
+	TRY_PARAM(read_ahead_blocks, rab);
+	TRY_PARAM(read_cache_blocks_per_file, rcb);
+	TRY_PARAM(max_send_bytes_per_second, sbps);
+	TRY_PARAM(max_recv_bytes_per_second, rbps);
+	TRY_PARAM(max_delay_retryable_error, dre);
+	TRY_PARAM(max_delay_connection_failed, dcf);
+	TRY_PARAM(sdk_auth, sa);
+	TRY_PARAM(enable_object_integrity_check, eoic);
+	TRY_PARAM(global_connection_pool, gcp);
+#undef TRY_PARAM
+	return false;
+}
+
+// Returns a Blob URL parameter string that specifies all of the non-default options using short names.
+std::string BlobKnobs::getURLParameters() const {
+	static BlobKnobs defaults;
+	std::string r;
+#define _CHECK_PARAM(n, sn)                                                                                            \
+	if (n != defaults.n) {                                                                                             \
+		r += format("%s%s=%d", r.empty() ? "" : "&", #sn, n);                                                          \
+	}
+	_CHECK_PARAM(secure_connection, sc);
+	_CHECK_PARAM(connect_tries, ct);
+	_CHECK_PARAM(connect_timeout, cto);
+	_CHECK_PARAM(max_connection_life, mcl);
+	_CHECK_PARAM(request_tries, rt);
+	_CHECK_PARAM(request_timeout_min, rto);
+	_CHECK_PARAM(requests_per_second, rps);
+	_CHECK_PARAM(list_requests_per_second, lrps);
+	_CHECK_PARAM(write_requests_per_second, wrps);
+	_CHECK_PARAM(read_requests_per_second, rrps);
+	_CHECK_PARAM(delete_requests_per_second, drps);
+	_CHECK_PARAM(concurrent_requests, cr);
+	_CHECK_PARAM(multipart_max_part_size, maxps);
+	_CHECK_PARAM(multipart_min_part_size, minps);
+	_CHECK_PARAM(concurrent_uploads, cu);
+	_CHECK_PARAM(concurrent_lists, cl);
+	_CHECK_PARAM(concurrent_reads_per_file, crpf);
+	_CHECK_PARAM(concurrent_writes_per_file, cwpf);
+	_CHECK_PARAM(enable_read_cache, erc);
+	_CHECK_PARAM(read_block_size, rbs);
+	_CHECK_PARAM(read_ahead_blocks, rab);
+	_CHECK_PARAM(read_cache_blocks_per_file, rcb);
+	_CHECK_PARAM(max_send_bytes_per_second, sbps);
+	_CHECK_PARAM(max_recv_bytes_per_second, rbps);
+	_CHECK_PARAM(sdk_auth, sa);
+	_CHECK_PARAM(enable_object_integrity_check, eoic);
+	_CHECK_PARAM(global_connection_pool, gcp);
+	_CHECK_PARAM(max_delay_retryable_error, dre);
+	_CHECK_PARAM(max_delay_connection_failed, dcf);
+#undef _CHECK_PARAM
+	return r;
+}
+
+Future<Optional<json_spirit::mObject>> IBlobStoreEndpoint::tryReadJSONFile(std::string path) {
+	std::string content;
+
+	// Event type to be logged in the event of an exception
+	const char* errorEventType = "BlobCredentialFileError";
+
+	try {
+		Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+		    path, IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0);
+		int64_t size = co_await f->size();
+		Standalone<StringRef> buf = makeString(size);
+		int r = co_await f->read(mutateString(buf), size, 0);
+		ASSERT(r == size);
+		content = buf.toString();
+
+		// Any exceptions from here forward are parse failures
+		errorEventType = "BlobCredentialFileParseFailed";
+		json_spirit::mValue json;
+		json_spirit::read_string(content, json);
+		if (json.type() == json_spirit::obj_type)
+			co_return json.get_obj();
+		else
+			TraceEvent(SevWarn, "BlobCredentialFileNotJSONObject").suppressFor(60).detail("File", path);
+
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled)
+			TraceEvent(SevWarn, errorEventType).errorUnsuppressed(e).suppressFor(60).detail("File", path);
+	}
+
+	co_return Optional<json_spirit::mObject>();
+}
+
+std::string IBlobStoreEndpoint::getResourceURL(std::string resource, std::string params) const {
+	std::string hostPort = host;
+	if (!service.empty()) {
+		hostPort.append(":");
+		hostPort.append(service);
+	}
+
+	std::string r = format("blobstore://@%s/%s", hostPort.c_str(), resource.c_str());
+
+	// Get params that are deviations from knob defaults
+	std::string knobParams = knobs.getURLParameters();
+	if (!knobParams.empty()) {
+		if (!params.empty()) {
+			params.append("&");
+		}
+		params.append(knobParams);
+	}
+
+	for (const auto& [k, v] : extraHeaders) {
+		if (!params.empty()) {
+			params.append("&");
+		}
+		params.append("header=");
+		params.append(k);
+		params.append(":");
+		params.append(v);
+	}
+
+	if (!params.empty())
+		r.append("?").append(params);
+
+	return r;
+}
+
+Reference<IBlobStoreEndpoint> IBlobStoreEndpoint::fromString(const std::string& url,
+                                                             const Optional<std::string>& proxy,
+                                                             std::string* resourceFromURL,
+                                                             std::string* error,
+                                                             ParametersT* ignored_parameters) {
+	if (resourceFromURL)
+		resourceFromURL->clear();
+
+	try {
+		// Replace HTML-encoded ampersands with raw ampersands
+		std::string decoded_url = url;
+		size_t pos = 0;
+		while ((pos = decoded_url.find("&amp;", pos)) != std::string::npos) {
+			decoded_url.replace(pos, 5, "&");
+			pos += 1;
+		}
+
+		// Also handle double-encoded ampersands
+		pos = 0;
+		while ((pos = decoded_url.find("&amp;amp;", pos)) != std::string::npos) {
+			decoded_url.replace(pos, 9, "&");
+			pos += 1;
+		}
+
+		StringRef t(decoded_url);
+		StringRef prefix = t.eat("://");
+		if (prefix != "blobstore"_sr)
+			throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str());
+
+		Optional<std::string> proxyHost, proxyPort;
+		if (proxy.present()) {
+			StringRef proxyRef(proxy.get());
+			if (proxy.get().find("://") != std::string::npos) {
+				StringRef proxyPrefix = proxyRef.eat("://");
+				if (proxyPrefix != "http"_sr) {
+					throw format("Invalid proxy URL prefix '%s'. Either don't use a prefix, or use http://",
+					             proxyPrefix.toString().c_str());
+				}
+			}
+			std::string proxyBody = proxyRef.eat().toString();
+			if (!Hostname::isHostname(proxyBody) && !NetworkAddress::parseOptional(proxyBody).present()) {
+				throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.",
+				             proxyBody.c_str());
+			}
+			StringRef p(proxyBody);
+			proxyHost = p.eat(":").toString();
+			proxyPort = p.eat().toString();
+		}
+
+		Optional<StringRef> cred;
+		if (url.find("@") != std::string::npos) {
+			cred = t.eat("@");
+		}
+		uint8_t foundSeparator = 0;
+		StringRef hostPort = t.eatAny("/?", &foundSeparator);
+		StringRef resource;
+		if (foundSeparator == '/') {
+			resource = t.eat("?");
+		}
+
+		// hostPort is at least a host or IP address, optionally followed by :portNumber or :serviceName
+		StringRef h(hostPort);
+		StringRef host = h.eat(":");
+		if (host.size() == 0)
+			throw std::string("host cannot be empty");
+
+		StringRef service = h.eat();
+
+		std::string region;
+		std::string gcsProjectId;
+		std::string provider = "s3";
+
+		BlobKnobs knobs;
+		HTTP::Headers extraHeaders;
+		while (1) {
+			StringRef name = t.eat("=");
+			if (name.size() == 0)
+				break;
+			StringRef value = t.eat("&");
+
+			// Special case for header
+			if (name == "header"_sr) {
+				StringRef originalValue = value;
+				StringRef headerFieldName = value.eat(":");
+				StringRef headerFieldValue = value;
+				if (headerFieldName.size() == 0 || headerFieldValue.size() == 0) {
+					throw format("'%s' is not a valid value for '%s' parameter.  Format is <FieldName>:<FieldValue> "
+					             "where strings are not empty.",
+					             originalValue.toString().c_str(),
+					             name.toString().c_str());
+				}
+				std::string& fieldValue = extraHeaders[headerFieldName.toString()];
+				// RFC 2616 section 4.2 says header field names can repeat but only if it is valid to concatenate
+				// their values with comma separation
+				if (!fieldValue.empty()) {
+					fieldValue.append(",");
+				}
+				fieldValue.append(headerFieldValue.toString());
+				continue;
+			}
+
+			// overwrite region from parameter
+			if (name == "region"_sr) {
+				region = value.toString();
+				continue;
+			}
+
+			if (name == "gcs_project_id"_sr || name == "gcspid"_sr) {
+				gcsProjectId = value.toString();
+				continue;
+			}
+
+			if (name == "provider"_sr || name == "p"_sr) {
+				provider = value.toString();
+				continue;
+			}
+
+			// See if the parameter is a knob
+			// First try setting a dummy value (all knobs are currently numeric) just to see if this parameter is
+			// known. If it is, then we will set it to a good value or throw below, so the dummy set has no bad
+			// side effects.
+			bool known = knobs.set(name, 0);
+
+			// If the parameter is not known then throw unless there is an ignored_parameters set to add it to
+			if (!known) {
+				if (ignored_parameters == nullptr) {
+					throw format("%s is not a valid parameter name", name.toString().c_str());
+				}
+				(*ignored_parameters)[name.toString()] = value.toString();
+				continue;
+			}
+
+			// The parameter is known so it must be numeric and valid.
+			char* valueEnd = nullptr;
+			std::string s = value.toString();
+			long int ivalue = strtol(s.c_str(), &valueEnd, 10);
+			if (*valueEnd || (ivalue == 0 && s != "0") ||
+			    (((ivalue == LONG_MAX) || (ivalue == LONG_MIN)) && errno == ERANGE))
+				throw format("%s is not a valid value for %s", s.c_str(), name.toString().c_str());
+
+			// It should not be possible for this set to fail now since the dummy set above had to have worked.
+			ASSERT(knobs.set(name, ivalue));
+		}
+
+		if (resourceFromURL != nullptr)
+			*resourceFromURL = resource.toString();
+
+		if (provider == "gcs") {
+			return makeReference<GCSBlobStoreEndpoint>(
+			    host.toString(), service.toString(), proxyHost, proxyPort, cred, gcsProjectId, knobs, extraHeaders);
+		}
+
+		return makeReference<S3BlobStoreEndpoint>(
+		    host.toString(), service.toString(), region, proxyHost, proxyPort, cred, knobs, extraHeaders);
+
+	} catch (std::string& err) {
+		if (error != nullptr)
+			*error = err;
+		TraceEvent(SevWarnAlways, "BlobStoreEndpointBadURL")
+		    .suppressFor(60)
+		    .detail("Description", err)
+		    .detail("Format", getURLFormat())
+		    .detail("URL", url);
+		throw backup_invalid_url();
+	}
+}
+
+Future<Void> updateSecret_impl(Reference<IBlobStoreEndpoint> b) {
+	auto* pFiles = (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
+	if (pFiles == nullptr)
+		co_return;
+
+	std::vector<Future<Optional<json_spirit::mObject>>> reads;
+	for (auto& f : *pFiles)
+		reads.push_back(IBlobStoreEndpoint::tryReadJSONFile(f));
+
+	co_await waitForAll(reads);
+
+	std::string credentialsFileKey = b->credentialFileKey();
+
+	int invalid = 0;
+
+	for (auto& f : reads) {
+		// If value not present then the credentials file wasn't readable or valid. Continue to check other results.
+		if (!f.get().present()) {
+			TraceEvent(SevWarn, "BlobStoreCredentialFileNotReadable")
+			    .detail("Endpoint", b->host)
+			    .detail("Service", b->service);
+			++invalid;
+			continue;
+		}
+
+		JSONDoc doc(f.get().get());
+		if (doc.has("accounts") && doc.last().type() == json_spirit::obj_type) {
+			JSONDoc accounts(doc.last().get_obj());
+			if (accounts.has(credentialsFileKey, false) && accounts.last().type() == json_spirit::obj_type) {
+				JSONDoc account(accounts.last());
+				if (b->extractCredentialFields(account)) {
+					co_return;
+				}
+			}
+		}
+	}
+
+	// If any sources were invalid
+	if (invalid > 0) {
+		TraceEvent(SevWarn, "BlobStoreCredentialFileInvalid")
+		    .detail("Endpoint", b->host)
+		    .detail("Service", b->service)
+		    .detail("Invalid", invalid);
+		throw backup_auth_unreadable();
+	}
+
+	// All sources were valid but didn't contain the desired info
+	TraceEvent(SevWarn, "BlobStoreCredentialsMissing")
+	    .detail("Endpoint", b->host)
+	    .detail("Service", b->service)
+	    .detail("Reason", "No valid credentials found");
+	throw backup_auth_missing();
+}
+
+Future<Void> IBlobStoreEndpoint::updateSecret() {
+	return updateSecret_impl(Reference<IBlobStoreEndpoint>::addRef(this));
+}
+
+AsyncResult<IBlobStoreEndpoint::ListResult> listObjects_impl(
+    Reference<IBlobStoreEndpoint> bstore,
+    std::string bucket,
+    Optional<std::string> prefix,
+    Optional<char> delimiter,
+    int maxDepth,
+    std::function<bool(std::string const&)> recurseFilterUnsafe) {
+	// C++20 coroutine safety: copy const& params to survive across suspension.
+	auto recurseFilter = recurseFilterUnsafe;
+
+	IBlobStoreEndpoint::ListResult results;
+	PromiseStream<IBlobStoreEndpoint::ListResult> resultStream;
+	Future<Void> done = bstore->listObjectsStream(bucket, resultStream, prefix, delimiter, maxDepth, recurseFilter);
+	// Wrap done in an actor which sends end_of_stream because list does not so that many lists can write to the same
+	// stream
+	done = map(done, [=](Void) mutable {
+		resultStream.sendError(end_of_stream());
+		return Void();
+	});
+
+	try {
+		while (true) {
+			// Preserve actor choose{} semantics: if the stream result and completion are both ready,
+			// drain the stream item before honoring done. Using race(done, stream) drops the final
+			// stream item because race() breaks ties by argument order.
+			auto res = co_await race(resultStream.getFuture(), done);
+			if (res.index() == 0) {
+				IBlobStoreEndpoint::ListResult info = std::get<0>(std::move(res));
+
+				results.commonPrefixes.insert(
+				    results.commonPrefixes.end(), info.commonPrefixes.begin(), info.commonPrefixes.end());
+				results.objects.insert(results.objects.end(), info.objects.begin(), info.objects.end());
+			} else if (res.index() == 1) {
+				done = Never();
+			} else {
+				UNREACHABLE();
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_end_of_stream)
+			throw;
+	}
+
+	co_return results;
+}
+
+AsyncResult<IBlobStoreEndpoint::ListResult> IBlobStoreEndpoint::listObjects(
+    std::string const& bucket,
+    Optional<std::string> prefix,
+    Optional<char> delimiter,
+    int maxDepth,
+    std::function<bool(std::string const&)> recurseFilter) {
+	return listObjects_impl(
+	    Reference<IBlobStoreEndpoint>::addRef(this), bucket, prefix, delimiter, maxDepth, recurseFilter);
+}
+
+Future<Void> deleteRecursively_impl(Reference<IBlobStoreEndpoint> b,
+                                    std::string bucket,
+                                    std::string prefix,
+                                    int* pNumDeleted,
+                                    int64_t* pBytesDeleted) {
+	PromiseStream<IBlobStoreEndpoint::ListResult> resultStream;
+	// Start a recursive parallel listing which will send results to resultStream as they are received
+	Future<Void> done = b->listObjectsStream(bucket, resultStream, prefix, '/', std::numeric_limits<int>::max());
+	// Wrap done in an actor which will send end_of_stream since listObjectsStream() does not (so that many calls can
+	// write to the same stream)
+	done = map(done, [=](Void) mutable {
+		resultStream.sendError(end_of_stream());
+		return Void();
+	});
+
+	std::list<Future<Void>> deleteFutures;
+	try {
+		while (true) {
+			// Preserve actor choose{} semantics: if the stream result and completion are both ready,
+			// drain the stream item before honoring done. Using race(done, stream) drops the final
+			// stream item because race() breaks ties by argument order.
+			auto res = co_await race(resultStream.getFuture(), done);
+			if (res.index() == 0) {
+				IBlobStoreEndpoint::ListResult list = std::get<0>(std::move(res));
+
+				for (auto& object : list.objects) {
+					deleteFutures.push_back(map(b->deleteObject(bucket, object.name), [=](Void) -> Void {
+						if (pNumDeleted != nullptr) {
+							++*pNumDeleted;
+						}
+						if (pBytesDeleted != nullptr) {
+							*pBytesDeleted += object.size;
+						}
+						return Void();
+					}));
+				}
+			} else if (res.index() == 1) {
+				done = Never();
+			} else {
+				UNREACHABLE();
+			}
+
+			// This is just a precaution to avoid having too many outstanding delete actors waiting to run
+			while (deleteFutures.size() > CLIENT_KNOBS->BLOBSTORE_CONCURRENT_REQUESTS) {
+				co_await deleteFutures.front();
+				deleteFutures.pop_front();
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_end_of_stream)
+			throw;
+	}
+
+	while (!deleteFutures.empty()) {
+		co_await deleteFutures.front();
+		deleteFutures.pop_front();
+	}
+}
+
+Future<Void> IBlobStoreEndpoint::deleteRecursively(std::string const& bucket,
+                                                   std::string prefix,
+                                                   int* pNumDeleted,
+                                                   int64_t* pBytesDeleted) {
+	return deleteRecursively_impl(
+	    Reference<IBlobStoreEndpoint>::addRef(this), bucket, prefix, pNumDeleted, pBytesDeleted);
+}
+
+Future<Void> writeEntireFile_impl(Reference<IBlobStoreEndpoint> bstore,
+                                  std::string bucket,
+                                  std::string object,
+                                  std::string content) {
+	UnsentPacketQueue packets;
+	if (content.size() > bstore->knobs.multipart_max_part_size)
+		throw file_too_large();
+
+	PacketWriter pw(packets.getWriteBuffer(content.size()), nullptr, Unversioned());
+	pw.serializeBytes(content);
+
+	// Yield because we may have just had to copy several MB's into packet buffer chain and next we have to calculate a
+	// hash of it.
+	// TODO: If this actor is used to send large files then combine the summing and packetization into a loop
+	// with a yield() every 20k or so.
+	co_await yield();
+
+	// If enable_object_integrity_check is true, calculate the sha256 sum of the content.
+	// Otherwise, do md5. The hash type must match what the provider's writeEntireFileFromBuffer expects.
+	std::string contentHash;
+	if (bstore->knobs.enable_object_integrity_check) {
+		unsigned char hash[SHA256_DIGEST_LENGTH];
+		SHA256_CTX sha256;
+		SHA256_Init(&sha256);
+		SHA256_Update(&sha256, content.data(), content.size());
+		SHA256_Final(hash, &sha256);
+		contentHash = base64::encoder::from_string(std::string(reinterpret_cast<char*>(hash), SHA256_DIGEST_LENGTH));
+		contentHash.resize(contentHash.size() - 1);
+	} else {
+		MD5_CTX sum;
+		::MD5_Init(&sum);
+		::MD5_Update(&sum, content.data(), content.size());
+		std::string sumBytes;
+		sumBytes.resize(16);
+		::MD5_Final((unsigned char*)sumBytes.data(), &sum);
+		contentHash = base64::encoder::from_string(sumBytes);
+		contentHash.resize(contentHash.size() - 1);
+	}
+
+	co_await bstore->writeEntireFileFromBuffer(bucket, object, &packets, content.size(), contentHash);
+}
+
+Future<Void> IBlobStoreEndpoint::writeEntireFile(std::string const& bucket,
+                                                 std::string const& object,
+                                                 std::string const& content) {
+	return writeEntireFile_impl(Reference<IBlobStoreEndpoint>::addRef(this), bucket, object, content);
+}
+
+Future<IBlobStoreEndpoint::ReusableConnection> connect_impl(Reference<IBlobStoreEndpoint> b, bool* reusingConn) {
+	// First try to get a connection from the pool
+	*reusingConn = false;
+	while (!b->connectionPool->pool.empty()) {
+		IBlobStoreEndpoint::ReusableConnection rconn = b->connectionPool->pool.front();
+		b->connectionPool->pool.pop();
+
+		// If the connection expires in the future then return it
+		if (rconn.expirationTime > now()) {
+			*reusingConn = true;
+			++b->blobStats->reusedConnections;
+			TraceEvent("IBlobStoreEndpointReusingConnected")
+			    .suppressFor(60)
+			    .detail("RemoteEndpoint", rconn.conn->getPeerAddress())
+			    .detail("ExpiresIn", rconn.expirationTime - now())
+			    .detail("Proxy", b->proxyHost.orDefault(""));
+			co_return rconn;
+		}
+		++b->blobStats->expiredConnections;
+	}
+	++b->blobStats->newConnections;
+	std::string host = b->host, service = b->service;
+	TraceEvent(SevDebug, "IBlobStoreEndpointBuildingNewConnection")
+	    .detail("UseProxy", b->useProxy)
+	    .detail("TLS", b->knobs.secure_connection == 1)
+	    .detail("Host", host)
+	    .detail("Service", service)
+	    .log();
+	if (service.empty()) {
+		if (b->useProxy) {
+			fprintf(stderr, "ERROR: Port can't be empty when using HTTP proxy.\n");
+			throw connection_failed();
+		}
+		service = b->knobs.secure_connection ? "https" : "http";
+	}
+	bool isTLS = b->knobs.isTLS();
+	Reference<IConnection> conn;
+	if (b->useProxy) {
+		if (isTLS) {
+			conn = co_await HTTP::proxyConnect(host, service, b->proxyHost.get(), b->proxyPort.get());
+		} else {
+			host = b->proxyHost.get();
+			service = b->proxyPort.get();
+			conn = co_await INetworkConnections::net()->connect(host, service, false);
+		}
+	} else {
+		conn = co_await INetworkConnections::net()->connect(host, service, isTLS);
+	}
+	ASSERT(conn.isValid());
+	co_await conn->connectHandshake();
+
+	TraceEvent("IBlobStoreEndpointNewConnectionSuccess")
+	    .suppressFor(60)
+	    .detail("RemoteEndpoint", conn->getPeerAddress())
+	    .detail("ExpiresIn", b->knobs.max_connection_life)
+	    .detail("Proxy", b->proxyHost.orDefault(""));
+
+	if (b->lookupSecretOnEachRequest()) {
+		co_await b->updateSecret();
+	}
+
+	co_return IBlobStoreEndpoint::ReusableConnection({ conn, now() + b->knobs.max_connection_life });
+}
+
+Future<IBlobStoreEndpoint::ReusableConnection> IBlobStoreEndpoint::connect(bool* reusingConn) {
+	return connect_impl(Reference<IBlobStoreEndpoint>::addRef(this), reusingConn);
+}
+
+void IBlobStoreEndpoint::returnConnection(ReusableConnection& rconn) {
+	// If it expires in the future then add it to the pool in the front
+	if (rconn.expirationTime > now()) {
+		connectionPool->pool.push(rconn);
+	} else {
+		++blobStats->expiredConnections;
+	}
+	rconn.conn = Reference<IConnection>();
+}
+
+// Do a request, get a Response.
+// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
+// itself must live for the life of this actor and be destroyed by the caller.
+Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<IBlobStoreEndpoint> bstore,
+                                                         std::string verb,
+                                                         std::string resource,
+                                                         HTTP::Headers headers,
+                                                         UnsentPacketQueue* pContent,
+                                                         int contentLen,
+                                                         std::set<unsigned int> successCodes) {
+	UnsentPacketQueue contentCopy;
+	auto req = makeReference<HTTP::OutgoingRequest>();
+	req->verb = verb;
+	req->data.content = &contentCopy;
+	req->data.contentLen = contentLen;
+
+	if (resource.empty()) {
+		resource = "/";
+	}
+
+	// For requests with content to upload, the request timeout should be at least twice the amount of time
+	// it would take to upload the content given the upload bandwidth and concurrency limits.
+	int bandwidthThisRequest = 1 + bstore->knobs.max_send_bytes_per_second / bstore->knobs.concurrent_uploads;
+	int contentUploadSeconds = contentLen / bandwidthThisRequest;
+	int requestTimeout = std::max(bstore->knobs.request_timeout_min, 3 * contentUploadSeconds);
+
+	co_await bstore->concurrentRequests.take();
+	FlowLock::Releaser globalReleaser(bstore->concurrentRequests, 1);
+
+	int maxTries = std::min(bstore->knobs.request_tries, bstore->knobs.connect_tries);
+	int thisTry = 1;
+	double nextRetryDelay = 2.0;
+	bool retryExtended = false;
+
+	while (true) {
+		Optional<Error> err;
+		Optional<NetworkAddress> remoteAddress;
+		bool connectionEstablished = false;
+		Reference<HTTP::IncomingResponse> r;
+		std::string canonicalURI = resource;
+		UID connID = UID();
+		double reqStartTimer{ 0 };
+		double connectStartTimer = g_network->timer();
+		bool reusingConn = false;
+		bool fastRetry = false;
+		IBlobStoreEndpoint::ReusableConnection rconn;
+
+		// Reset headers to initial state for this retry attempt to prevent header accumulation
+		req->data.headers = headers;
+		req->data.headers["Host"] = bstore->host;
+
+		// Merge extraHeaders into headers
+		for (const auto& [k, v] : bstore->extraHeaders) {
+			std::string& fieldValue = req->data.headers[k];
+			if (!fieldValue.empty()) {
+				fieldValue.append(",");
+			}
+			fieldValue.append(v);
+		}
+
+		try {
+			Future<IBlobStoreEndpoint::ReusableConnection> frconn = bstore->connect(&reusingConn);
+
+			// Make a shallow copy of the queue
+			req->data.content->discardAll();
+			if (pContent != nullptr) {
+				PacketBuffer* pFirst = pContent->getUnsent();
+				PacketBuffer* pLast = nullptr;
+				for (PacketBuffer* p = pFirst; p != nullptr; p = p->nextPacketBuffer()) {
+					p->addref();
+					p->bytes_sent = 0; // Reset sent count on each buffer
+					pLast = p;
+				}
+				req->data.content->prependWriteBuffer(pFirst, pLast);
+			}
+
+			rconn = co_await timeoutError(frconn, bstore->knobs.connect_timeout);
+			connectionEstablished = true;
+			connID = rconn.conn->getDebugID();
+			reqStartTimer = g_network->timer();
+
+			// Provider hook: pre-retry check
+			if (retryExtended) {
+				bool proceed = co_await bstore->preRetryCheck(verb, resource, rconn, requestTimeout, retryExtended);
+				if (!proceed) {
+					bstore->returnConnection(rconn);
+					continue;
+				}
+			}
+
+			// Finish/update the request headers (which includes auth headers).
+			// This must be done AFTER the connection is ready because credentials may be
+			// refreshed when a new connection is established.
+			bstore->setRequestHeaders(verb, resource, req->data.headers);
+			canonicalURI = bstore->normalizeResourceForRequest(resource);
+
+			// Has to be in absolute-form.
+			if (bstore->useProxy && bstore->knobs.secure_connection == 0) {
+				canonicalURI = "http://" + bstore->host + ":" + bstore->service + canonicalURI;
+			}
+
+			req->resource = canonicalURI;
+
+			remoteAddress = rconn.conn->getPeerAddress();
+			co_await bstore->requestRate->getAllowance(1);
+
+			Future<Reference<HTTP::IncomingResponse>> reqF =
+			    HTTP::doRequest(rconn.conn, req, bstore->sendRate, &bstore->s_stats.bytes_sent, bstore->recvRate);
+
+			if (reqF.isReady() && reusingConn) {
+				fastRetry = true;
+			}
+
+			Reference<HTTP::IncomingResponse> _r = co_await timeoutError(reqF, requestTimeout);
+			r = _r;
+
+			bstore->simulateRequestFailure(verb, resource, r);
+
+			// Since the response was parsed successfully reuse the connection unless we
+			// received the "Connection: close" header.
+			if (r->data.headers["Connection"] != "close") {
+				bstore->returnConnection(rconn);
+			} else {
+				++bstore->blobStats->expiredConnections;
+			}
+			rconn.conn.clear();
+
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "BlobStoreDoRequestError")
+			    .errorUnsuppressed(e)
+			    .detail("Verb", verb)
+			    .detail("Resource", resource);
+			// Close the connection on error to satisfy Sim2Conn's assertion in simulation
+			if (connectionEstablished && rconn.conn.isValid()) {
+				rconn.conn->close();
+				rconn.conn.clear();
+			}
+			if (e.code() == error_code_actor_cancelled)
+				throw;
+			err = e;
+		}
+
+		double end = g_network->timer();
+		double connectDuration = reqStartTimer - connectStartTimer;
+		double reqDuration = end - reqStartTimer;
+		bstore->blobStats->requestLatency.addMeasurement(reqDuration);
+
+		// If err is not present then r is valid. If r->code is in successCodes then record the successful request and
+		// return r.
+		if (!err.present() && successCodes.contains(r->code)) {
+			bstore->s_stats.requests_successful++;
+			++bstore->blobStats->requestsSuccessful;
+			co_return r;
+		}
+
+		bstore->s_stats.requests_failed++;
+		++bstore->blobStats->requestsFailed;
+
+		// All errors in err are potentially retryable as well as certain HTTP response codes...
+		// But only if our previous attempt was not the last allowable try.
+		bool retryable = err.present() || r->code == 500 || r->code == 502 || r->code == 503 || r->code == 429;
+		retryable = retryable && (thisTry < maxTries);
+
+		if (!retryable || !err.present()) {
+			fastRetry = false;
+		}
+
+		TraceEvent event(SevWarn,
+		                 (retryable || retryExtended) ? (fastRetry ? "BlobStoreEndpointRequestFailedFastRetryable"
+		                                                           : "BlobStoreEndpointRequestFailedRetryable")
+		                                              : "BlobStoreEndpointRequestFailed");
+
+		bool connectionFailed = false;
+		if (err.present()) {
+			event.errorUnsuppressed(err.get());
+			if (err.get().code() == error_code_connection_failed) {
+				connectionFailed = true;
+			}
+		}
+		event.suppressFor(1);
+		if (!err.present()) {
+			event.detail("ResponseCode", r->code);
+		}
+
+		// Provider hook: process failure
+		if (!err.present()) {
+			bstore->processRequestFailure(r, event, retryExtended);
+		}
+
+		event.detail("ConnectionEstablished", connectionEstablished);
+		event.detail("ReusingConn", reusingConn);
+		if (connectionEstablished) {
+			event.detail("ConnID", connID);
+			event.detail("ConnectDuration", connectDuration);
+			event.detail("ReqDuration", reqDuration);
+		}
+
+		if (remoteAddress.present())
+			event.detail("RemoteEndpoint", remoteAddress.get());
+		else
+			event.detail("RemoteHost", bstore->host);
+
+		event.detail("Verb", verb)
+		    .detail("Resource", resource)
+		    .detail("ThisTry", thisTry)
+		    .detail("URI", canonicalURI)
+		    .detail("Proxy", bstore->proxyHost.orDefault(""));
+
+		// If r is not valid or not code 429 then increment the try count.
+		// 429's will not count against the attempt limit. Also skip incrementing the retry count for fast retries.
+		if (!fastRetry && (!r || r->code != 429))
+			++thisTry;
+
+		if (fastRetry) {
+			++bstore->blobStats->fastRetries;
+			co_await delay(0);
+		} else if (retryable || retryExtended) {
+			// We will wait delay seconds before the next retry, start with nextRetryDelay.
+			double delaySeconds = nextRetryDelay;
+			// connectionFailed is treated specially
+			double limit =
+			    connectionFailed ? bstore->knobs.max_delay_connection_failed : bstore->knobs.max_delay_retryable_error;
+			// Double but limit the *next* nextRetryDelay.
+			nextRetryDelay = std::min(nextRetryDelay * 2, limit);
+			// If r is valid then obey the Retry-After response header if present.
+			if (r) {
+				auto iRetryAfter = r->data.headers.find("Retry-After");
+				if (iRetryAfter != r->data.headers.end()) {
+					event.detail("RetryAfterHeader", iRetryAfter->second);
+					char* pEnd;
+					double retryAfter = strtod(iRetryAfter->second.c_str(), &pEnd);
+					if (*pEnd)
+						retryAfter = 300;
+					delaySeconds = std::max(delaySeconds, retryAfter);
+				}
+			}
+
+			event.detail("RetryDelay", delaySeconds);
+			co_await ::delay(delaySeconds);
+		} else {
+			// We can't retry, so throw something.
+			if (r && r->code == 406)
+				throw http_not_accepted();
+
+			// This error code means the authentication header was not accepted, likely the account or key is wrong.
+			if (r && r->code == 401)
+				throw http_auth_failed();
+
+			// Recognize and throw specific errors
+			if (err.present()) {
+				int code = err.get().code();
+				// If we get a timed_out error during the connect() phase, we'll call that connection_failed
+				// despite the fact that there was technically never a 'connection' to begin with. It differentiates
+				// between an active connection timing out vs a connection timing out, though not between an active
+				// connection failing vs connection attempt failing.
+				if (code == error_code_timed_out && !connectionEstablished) {
+					TraceEvent(SevWarn, "BlobStoreEndpointConnectTimeout")
+					    .suppressFor(60)
+					    .detail("Timeout", requestTimeout);
+					throw connection_failed();
+				}
+
+				// TODO: Add more error types?
+				if (code == error_code_timed_out || code == error_code_connection_failed ||
+				    code == error_code_lookup_failed)
+					throw err.get();
+			}
+
+			throw http_request_failed();
+		}
+	}
+}
+
+Future<Reference<HTTP::IncomingResponse>> IBlobStoreEndpoint::doRequest(std::string const& verb,
+                                                                        std::string const& resource,
+                                                                        const HTTP::Headers& headers,
+                                                                        UnsentPacketQueue* pContent,
+                                                                        int contentLen,
+                                                                        std::set<unsigned int> successCodes) {
+	return doRequest_impl(
+	    Reference<IBlobStoreEndpoint>::addRef(this), verb, resource, headers, pContent, contentLen, successCodes);
+}
diff --git a/fdbclient/BulkLoading.cpp b/fdbclient/BulkLoading.cpp
index eba11f76f8e..ed31f040cc3 100644
--- a/fdbclient/BulkLoading.cpp
+++ b/fdbclient/BulkLoading.cpp
@@ -23,8 +23,6 @@
 
 #include <boost/url/url.hpp>
 #include <boost/url/parse.hpp>
-#include <boost/url/error_types.hpp>
-#include <boost/url/string_view.hpp>
 
 bool getConductBulkLoadFromDataMoveId(const UID& dataMoveId) {
 	bool nowAssigned = false;
@@ -119,7 +117,13 @@ std::string getPath(const std::string& path) {
 	// We want boost::url to parse out the path but it cannot digest credentials. Strip them out
 	// before passing to boost::url.
 	try {
-		return boost::urls::parse_uri(matches[1].str() + matches[3].str()).value().path();
+		std::string result = boost::urls::parse_uri(matches[1].str() + matches[3].str()).value().path();
+		// Remove leading slash if present - S3 object keys don't have leading slashes,
+		// and BackupContainer's dataPath() expects paths without them
+		if (!result.empty() && result[0] == '/') {
+			result = result.substr(1);
+		}
+		return result;
 	} catch (std::system_error& e) {
 		TraceEvent(SevError, "BulkLoadGetPathError")
 		    .detail("Path", path)
diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt
index 5aefcc219bf..b8a43bffbd1 100644
--- a/fdbclient/CMakeLists.txt
+++ b/fdbclient/CMakeLists.txt
@@ -1,4 +1,6 @@
 fdb_find_sources(FDBCLIENT_SRCS)
+list(REMOVE_ITEM FDBCLIENT_SRCS LinkTest.cpp)
+list(REMOVE_ITEM FDBCLIENT_SRCS S3Client_cli.cpp)
 list(APPEND FDBCLIENT_SRCS sha1/SHA1.cpp)
 
 message(STATUS "FDB version is ${FDB_VERSION}")
@@ -90,7 +92,12 @@ if(WITH_AWS_BACKUP)
 endif()
 
 add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
-target_include_directories(fdbclient PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbclient
+  PUBLIC
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    "${CMAKE_CURRENT_BINARY_DIR}/include"
+  PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/versions.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/versions.h)
 add_dependencies(fdbclient fdboptions)
 target_link_libraries(fdbclient PUBLIC fdbrpc msgpack PRIVATE rapidxml)
@@ -98,10 +105,9 @@ if(WITH_TLS)
   target_link_libraries(fdbclient PRIVATE OpenSSL::Crypto)
 endif()
 
-add_flow_target(EXECUTABLE NAME s3client SRCS S3Client_cli.actor.cpp)
+add_flow_target(EXECUTABLE NAME s3client SRCS S3Client_cli.cpp)
 target_include_directories(s3client PRIVATE
     "${CMAKE_CURRENT_SOURCE_DIR}/include"
-    "${CMAKE_CURRENT_BINARY_DIR}/include"
     "${CMAKE_SOURCE_DIR}/flow/include"
     "${CMAKE_BINARY_DIR}/flow/include"
     "${CMAKE_SOURCE_DIR}/fdbrpc/include"
@@ -180,7 +186,12 @@ endif()
 # fdbserver retain sampling functionality in client code while disabling
 # sampling for pure clients.
 add_flow_target(STATIC_LIBRARY NAME fdbclient_sampling SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
-target_include_directories(fdbclient_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbclient_sampling
+  PUBLIC
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    "${CMAKE_CURRENT_BINARY_DIR}/include"
+  PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}")
 add_dependencies(fdbclient_sampling fdboptions)
 target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack PRIVATE rapidxml)
 target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING)
@@ -189,7 +200,12 @@ if(WIN32)
 endif()
 
 add_flow_target(LINK_TEST NAME fdbclientlinktest SRCS LinkTest.cpp)
-target_link_libraries(fdbclientlinktest PRIVATE fdbclient rapidxml) # re-link rapidxml due to private link interface
+target_link_libraries(fdbclientlinktest PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,fdbclient>" rapidxml) # re-link rapidxml due to private link interface
+
+if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
+                                     # x-compiling.
+  add_subdirectory(bench EXCLUDE_FROM_ALL)
+endif()
 
 if(BUILD_AZURE_BACKUP)
   target_link_libraries(fdbclient PRIVATE curl azure-storage-lite)
@@ -197,12 +213,13 @@ if(BUILD_AZURE_BACKUP)
 endif()
 
 if(WITH_AWS_BACKUP)
-  target_link_libraries(fdbclient PUBLIC awssdk_target)
-  target_link_libraries(fdbclient_sampling PUBLIC awssdk_target)
+  target_link_libraries(fdbclient PRIVATE awssdk_target)
+  target_link_libraries(fdbclient_sampling PRIVATE awssdk_target)
 endif()
 
 if (NOT WIN32 AND NOT OPEN_FOR_IDE)
   enable_testing()
   add_test(NAME s3client_test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/s3client_test.sh ${CMAKE_BINARY_DIR})
+  add_test(NAME gcs_client_test COMMAND ${CMAKE_COMMAND} -E env USE_MOCK_GCS=true ${CMAKE_CURRENT_SOURCE_DIR}/tests/s3client_test.sh ${CMAKE_BINARY_DIR})
   add_test(NAME bulkload_test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/bulkload_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
 endif()
diff --git a/fdbclient/ClientKnobCollection.cpp b/fdbclient/ClientKnobCollection.cpp
deleted file mode 100644
index 81826218478..00000000000
--- a/fdbclient/ClientKnobCollection.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * ClientKnobCollection.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/ClientKnobCollection.h"
-
-ClientKnobCollection::ClientKnobCollection(Randomize randomize, IsSimulated isSimulated)
-  : flowKnobs(randomize, isSimulated), clientKnobs(randomize, isSimulated) {}
-
-void ClientKnobCollection::initialize(Randomize randomize, IsSimulated isSimulated) {
-	flowKnobs.initialize(randomize, isSimulated);
-	clientKnobs.initialize(randomize, isSimulated);
-}
-
-void ClientKnobCollection::reset(Randomize randomize, IsSimulated isSimulated) {
-	flowKnobs.reset(randomize, isSimulated);
-	clientKnobs.reset(randomize, isSimulated);
-}
-
-Optional<KnobValue> ClientKnobCollection::tryParseKnobValue(std::string const& knobName,
-                                                            std::string const& knobValue) const {
-	auto parsedKnobValue = flowKnobs.parseKnobValue(knobName, knobValue);
-	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
-		return KnobValueRef::create(parsedKnobValue);
-	}
-	parsedKnobValue = clientKnobs.parseKnobValue(knobName, knobValue);
-	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
-		return KnobValueRef::create(parsedKnobValue);
-	}
-	return {};
-}
-
-bool ClientKnobCollection::trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) {
-	// Do not short circuit by directly returning:
-	//     knobValue.visitSetKnob(knobName, flowKnobs) || knobValue.visitSetKnob(knobName, clientKnobs)
-	// This is because if a knob with name FOO exists in both fdbclient/ClientKnobs.h and flow/Knobs.h, then
-	// only flow knob FOO will be updated but client knob FOO will not.
-	// Although today there are no knobs that share their name between flow and client, this was done to be
-	// consistent with ServerKnobCollection, see:
-	// https://github.com/apple/foundationdb/blob/596f4cba6dcac00a14080c57d5ea037272240a08/fdbclient/ServerKnobCollection.cpp#L51-L57
-	const bool setFlowKnob = knobValue.visitSetKnob(knobName, flowKnobs);
-	const bool setClientKnob = knobValue.visitSetKnob(knobName, clientKnobs);
-	return setFlowKnob || setClientKnob;
-}
diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 2c6f9b8ba0a..8e82ec2dc43 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -24,6 +24,8 @@
 #include "flow/UnitTest.h"
 #include "flow/flow.h"
 
+#include <iostream>
+
 int getSimulatedTxnTimeoutSeconds() {
 	if (deterministicRandom()->truePercent(90)) {
 		return 5;
@@ -38,6 +40,103 @@ ClientKnobs::ClientKnobs(Randomize randomize, IsSimulated isSimulated) {
 	initialize(randomize, isSimulated);
 }
 
+namespace {
+FlowKnobs globalFlowKnobs(Randomize::False, IsSimulated::False);
+ClientKnobs globalClientKnobs(Randomize::False, IsSimulated::False);
+ClientKnobs bootstrapGlobalClientKnobs(Randomize::False, IsSimulated::False);
+
+FlowKnobs& mutableFlowKnobs() {
+	return *const_cast<FlowKnobs*>(FLOW_KNOBS);
+}
+
+ClientKnobs& mutableClientKnobs() {
+	return *const_cast<ClientKnobs*>(CLIENT_KNOBS);
+}
+
+Optional<KnobValue> tryParseKnobValueImpl(std::string const& knobName, std::string const& knobValue) {
+	auto parsedKnobValue = FLOW_KNOBS->parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+
+	parsedKnobValue = CLIENT_KNOBS->parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+
+	return {};
+}
+} // namespace
+
+ClientKnobs const* CLIENT_KNOBS = &bootstrapGlobalClientKnobs;
+
+void resetClientKnobs(Randomize randomize, IsSimulated isSimulated) {
+	FLOW_KNOBS = &globalFlowKnobs;
+	CLIENT_KNOBS = &globalClientKnobs;
+	globalFlowKnobs.reset(randomize, isSimulated);
+	globalClientKnobs.reset(randomize, isSimulated);
+}
+
+void initializeClientKnobs(Randomize randomize, IsSimulated isSimulated) {
+	mutableFlowKnobs().initialize(randomize, isSimulated);
+	mutableClientKnobs().initialize(randomize, isSimulated);
+}
+
+Optional<KnobValue> tryParseClientKnobValue(std::string const& knobName, std::string const& knobValue) {
+	return tryParseKnobValueImpl(knobName, knobValue);
+}
+
+KnobValue parseClientKnobValue(std::string const& knobName, std::string const& knobValue) {
+	auto result = tryParseClientKnobValue(knobName, knobValue);
+	if (!result.present()) {
+		throw invalid_option();
+	}
+	return result.get();
+}
+
+bool trySetClientKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	const bool setFlowKnob = knobValue.visitSetKnob(knobName, mutableFlowKnobs());
+	const bool setClientKnob = knobValue.visitSetKnob(knobName, mutableClientKnobs());
+	return setFlowKnob || setClientKnob;
+}
+
+void setClientKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	if (trySetClientKnob(knobName, knobValue)) {
+		return;
+	}
+
+	TraceEvent(SevWarnAlways, "FailedToSetKnob").detail("KnobName", knobName).detail("KnobValue", knobValue.toString());
+	throw invalid_option_value();
+}
+
+void setupClientKnobs(std::vector<std::pair<std::string, std::string>> const& knobs) {
+	for (const auto& [knobName, knobValueString] : knobs) {
+		try {
+			setClientKnob(knobName, parseClientKnobValue(knobName, knobValueString));
+		} catch (Error& e) {
+			if (e.code() == error_code_invalid_option_value) {
+				std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName
+				          << "'\n";
+				TraceEvent(SevWarnAlways, "InvalidKnobValue")
+				    .detail("Knob", printable(knobName))
+				    .detail("Value", printable(knobValueString));
+			} else if (e.code() == error_code_invalid_option) {
+				std::cerr << "WARNING: Invalid knob option '" << knobName << "'\n";
+				TraceEvent(SevWarnAlways, "InvalidKnobName")
+				    .detail("Knob", printable(knobName))
+				    .detail("Value", printable(knobValueString));
+			} else {
+				std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n";
+				TraceEvent(SevError, "FailedToSetKnob")
+				    .errorUnsuppressed(e)
+				    .detail("Knob", printable(knobName))
+				    .detail("Value", printable(knobValueString));
+				throw e;
+			}
+		}
+	}
+}
+
 void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	// clang-format off
 
@@ -61,6 +160,7 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( MAX_COMMIT_PROXY_CONNECTIONS,              5 ); if( randomize && BUGGIFY ) MAX_COMMIT_PROXY_CONNECTIONS = 1;
 	init( MAX_GRV_PROXY_CONNECTIONS,                 3 ); if( randomize && BUGGIFY ) MAX_GRV_PROXY_CONNECTIONS = 1;
 	init( STATUS_IDLE_TIMEOUT,                   120.0 );
+	init( STATUS_TIMEOUT,                         30.0 );
 	init( GRPC_CTL_SERVICE_DEFAULT_TIMEOUT,        5.0 );
 	init( SEND_ENTIRE_VERSION_VECTOR,            false );
 
@@ -114,8 +214,6 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( DETAILED_HEALTH_METRICS_MAX_STALENESS,   5.0 );
 	init( MID_SHARD_SIZE_MAX_STALENESS,           10.0 );
 	init( TAG_ENCODE_KEY_SERVERS,                false ); if( randomize && BUGGIFY ) TAG_ENCODE_KEY_SERVERS = true;
-	init( RANGESTREAM_FRAGMENT_SIZE,               1e6 );
-	init( RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT,     20 );
 	init( QUARANTINE_TSS_ON_MISMATCH,             true ); if( randomize && BUGGIFY ) QUARANTINE_TSS_ON_MISMATCH = false; // if true, a tss mismatch will put the offending tss in quarantine. If false, it will just be killed
 	init( CHANGE_FEED_EMPTY_BATCH_TIME,          0.005 );
 
@@ -185,7 +283,7 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( COPY_LOG_BLOCKS_PER_TASK,               1000 );
 	init( COPY_LOG_PREFETCH_BLOCKS,                  3 );
 	init( COPY_LOG_READ_AHEAD_BYTES,        BACKUP_LOCK_BYTES / COPY_LOG_PREFETCH_BLOCKS); // each task will use up to COPY_LOG_PREFETCH_BLOCKS * COPY_LOG_READ_AHEAD_BYTES memory
-	init( COPY_LOG_TASK_DURATION_NANOS,	      1e10 ); // 10 seconds
+	init( COPY_LOG_TASK_DURATION_SECONDS,	        10 );
 	init( BACKUP_TASKS_PER_AGENT,                   10 );
 	init( BACKUP_POLL_PROGRESS_SECONDS,             10 );
 	init( SIM_BACKUP_TASKS_PER_AGENT,               10 );
@@ -205,9 +303,9 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( BACKUP_STATUS_DELAY,                    40.0 );
 	init( BACKUP_STATUS_JITTER,                   0.05 );
 	init( MIN_CLEANUP_SECONDS,                  3600.0 );
-	init( FASTRESTORE_ATOMICOP_WEIGHT,               1 ); if( randomize && BUGGIFY ) { FASTRESTORE_ATOMICOP_WEIGHT = deterministicRandom()->random01() * 200 + 1; }
 	init( RESTORE_RANGES_READ_BATCH,             10000 );
 
+	init( BACKUP_RANGE_PARTITIONED_VDIR_INTERVAL, 100000 * 1000000LL );
 	init( BACKUP_CONTAINER_LOCAL_ALLOW_RELATIVE_PATH, false );
 	init( ENABLE_REPLICA_CONSISTENCY_CHECK_ON_BACKUP_READS, false ); if( randomize && BUGGIFY ) { ENABLE_REPLICA_CONSISTENCY_CHECK_ON_BACKUP_READS = true; }
 	init( BACKUP_CONSISTENCY_CHECK_REQUIRED_REPLICAS, -2 ); // Do consistency check based on all available storage replicas
@@ -321,8 +419,10 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( TAG_THROTTLE_RECHECK_INTERVAL,            5.0 ); if( randomize && BUGGIFY ) TAG_THROTTLE_RECHECK_INTERVAL = 0.0;
 	init( TAG_THROTTLE_EXPIRATION_INTERVAL,        60.0 ); if( randomize && BUGGIFY ) TAG_THROTTLE_EXPIRATION_INTERVAL = 1.0;
 	init( TAG_THROTTLING_PAGE_SIZE,                4096 ); if( randomize && BUGGIFY ) TAG_THROTTLING_PAGE_SIZE = 4096;
-	init( GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO,            4.0 );
-	init( PROXY_MAX_TAG_THROTTLE_DURATION,          5.0 ); if( randomize && BUGGIFY ) PROXY_MAX_TAG_THROTTLE_DURATION = 0.5;
+	init( TAG_THROTTLING_RW_FUNGIBILITY_RATIO,      4.0 );
+	// Preserve the old public knob spelling while moving new code to the non-global name.
+	double_knobs["global_tag_throttling_rw_fungibility_ratio"] =
+	    KnobValue<double>{ &TAG_THROTTLING_RW_FUNGIBILITY_RATIO };
 	init( TRANSACTION_LOCK_REJECTION_RETRIABLE,    true );
 
 	// busyness reporting
diff --git a/fdbclient/ClientStatusReport.cpp b/fdbclient/ClientStatusReport.cpp
index be1fa9e475a..a4a7c4d742c 100644
--- a/fdbclient/ClientStatusReport.cpp
+++ b/fdbclient/ClientStatusReport.cpp
@@ -30,7 +30,7 @@ namespace {
 
 class ClientReportGenerator {
 public:
-	ClientReportGenerator(DatabaseContext& cx) : cx(cx), healthy(true), numConnectionsFailed(0) {}
+	explicit ClientReportGenerator(DatabaseContext& cx) : cx(cx), healthy(true), numConnectionsFailed(0) {}
 
 	Standalone<StringRef> generateReport() {
 		if (cx.isError()) {
@@ -89,7 +89,7 @@ class ClientReportGenerator {
 		statusObj["CommitProxies"] = commitProxyArr;
 
 		// Update health status
-		if (clientInfo.grvProxies.size() == 0 || clientInfo.commitProxies.size() == 0) {
+		if (clientInfo.grvProxies.empty() || clientInfo.commitProxies.empty()) {
 			healthy = false;
 		}
 	}
diff --git a/fdbclient/ClusterConnectionFile.actor.cpp b/fdbclient/ClusterConnectionFile.cpp
similarity index 97%
rename from fdbclient/ClusterConnectionFile.actor.cpp
rename to fdbclient/ClusterConnectionFile.cpp
index 4dff5e3181f..52883104718 100644
--- a/fdbclient/ClusterConnectionFile.actor.cpp
+++ b/fdbclient/ClusterConnectionFile.cpp
@@ -1,5 +1,5 @@
 /*
- * ClusterConnectionFile.actor.cpp
+ * ClusterConnectionFile.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,7 +20,6 @@
 
 #include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/MonitorLeader.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 // Loads and parses the file at 'filename', throwing errors if the file cannot be read or the format is invalid.
 ClusterConnectionFile::ClusterConnectionFile(std::string const& filename)
@@ -52,7 +51,7 @@ Reference<ClusterConnectionFile> ClusterConnectionFile::openOrDefault(const char
 
 // Sets the connections string held by this object and persists it.
 Future<Void> ClusterConnectionFile::setAndPersistConnectionString(ClusterConnectionString const& conn) {
-	ASSERT(filename.size());
+	ASSERT(!filename.empty());
 	cs = conn;
 	return success(persist());
 }
@@ -105,7 +104,7 @@ std::string ClusterConnectionFile::toString() const {
 
 // returns <resolved name, was default file>
 std::pair<std::string, bool> ClusterConnectionFile::lookupClusterFileName(std::string const& filename) {
-	if (filename.length())
+	if (!filename.empty())
 		return std::make_pair(filename, false);
 
 	std::string f;
@@ -148,7 +147,7 @@ std::string ClusterConnectionFile::getErrorString(std::pair<std::string, bool> c
 Future<bool> ClusterConnectionFile::persist() {
 	setPersisted();
 
-	if (filename.size()) {
+	if (!filename.empty()) {
 		try {
 			atomicReplace(filename,
 			              "# DO NOT EDIT!\n# This file is auto-generated, it is not to be edited by hand\n" +
diff --git a/fdbclient/ClusterConnectionKey.actor.cpp b/fdbclient/ClusterConnectionKey.cpp
similarity index 70%
rename from fdbclient/ClusterConnectionKey.actor.cpp
rename to fdbclient/ClusterConnectionKey.cpp
index a6c00dda09e..cb0dcd966d5 100644
--- a/fdbclient/ClusterConnectionKey.actor.cpp
+++ b/fdbclient/ClusterConnectionKey.cpp
@@ -1,5 +1,5 @@
 /*
- * ClusterConnectionKey.actor.cpp
+ * ClusterConnectionKey.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,8 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ClusterConnectionKey.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "ClusterConnectionKey.h"
 
 // Creates a cluster connection record with a given connection string and saves it to the specified key. Needs to be
 // persisted should be set to true unless this ClusterConnectionKey is being created with the value read from the
@@ -37,22 +36,24 @@ ClusterConnectionKey::ClusterConnectionKey(Database db,
 
 // Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or the
 // format is invalid.
-ACTOR Future<Reference<ClusterConnectionKey>> ClusterConnectionKey::loadClusterConnectionKey(Database db,
-                                                                                             Key connectionStringKey) {
-	state Transaction tr(db);
-	loop {
+Future<Reference<ClusterConnectionKey>> ClusterConnectionKey::loadClusterConnectionKey(Database db,
+                                                                                       Key connectionStringKey) {
+	Transaction tr(db);
+	while (true) {
+		Error err;
 		try {
-			Optional<Value> v = wait(tr.get(connectionStringKey));
+			Optional<Value> v = co_await tr.get(connectionStringKey);
 			if (!v.present()) {
 				throw connection_string_invalid();
 			}
-			return makeReference<ClusterConnectionKey>(db,
-			                                           connectionStringKey,
-			                                           ClusterConnectionString(v.get().toString()),
-			                                           ConnectionStringNeedsPersisted::False);
+			co_return makeReference<ClusterConnectionKey>(db,
+			                                              connectionStringKey,
+			                                              ClusterConnectionString(v.get().toString()),
+			                                              ConnectionStringNeedsPersisted::False);
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -63,31 +64,31 @@ Future<Void> ClusterConnectionKey::setAndPersistConnectionString(ClusterConnecti
 }
 
 // Get the connection string stored in the database.
-ACTOR Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionStringImpl(
+Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionStringImpl(
     Reference<ClusterConnectionKey> self) {
 	Reference<ClusterConnectionKey> cck =
-	    wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
-	return cck->cs;
+	    co_await ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey);
+	co_return cck->cs;
 }
 
 Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionString() {
 	return getStoredConnectionStringImpl(Reference<ClusterConnectionKey>::addRef(this));
 }
 
-ACTOR Future<bool> ClusterConnectionKey::upToDateImpl(Reference<ClusterConnectionKey> self,
-                                                      ClusterConnectionString* connectionString) {
+Future<bool> ClusterConnectionKey::upToDateImpl(Reference<ClusterConnectionKey> self,
+                                                ClusterConnectionString* connectionString) {
 	try {
 		// the cluster file hasn't been created yet so there's nothing to check
 		if (self->needsToBePersisted())
-			return true;
+			co_return true;
 
 		Reference<ClusterConnectionKey> temp =
-		    wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
+		    co_await ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey);
 		*connectionString = temp->getConnectionString();
-		return connectionString->toString() == self->cs.toString();
+		co_return connectionString->toString() == self->cs.toString();
 	} catch (Error& e) {
 		TraceEvent(SevWarnAlways, "ClusterKeyError").error(e).detail("Key", self->connectionStringKey);
-		return false; // Swallow the error and report that the file is out of date
+		co_return false; // Swallow the error and report that the file is out of date
 	}
 }
 
@@ -114,25 +115,26 @@ std::string ClusterConnectionKey::toString() const {
 	return "fdbkey://" + printable(connectionStringKey);
 }
 
-ACTOR Future<bool> ClusterConnectionKey::persistImpl(Reference<ClusterConnectionKey> self) {
+Future<bool> ClusterConnectionKey::persistImpl(Reference<ClusterConnectionKey> self) {
 	self->setPersisted();
-	state Value newConnectionString = ValueRef(self->cs.toString());
+	Value newConnectionString = ValueRef(self->cs.toString());
 
 	try {
-		state Transaction tr(self->db);
-		loop {
+		Transaction tr(self->db);
+		while (true) {
+			Error err;
 			try {
-				Optional<Value> existingConnectionString = wait(tr.get(self->connectionStringKey));
+				Optional<Value> existingConnectionString = co_await tr.get(self->connectionStringKey);
 				// Someone has already updated the connection string to what we want
 				if (existingConnectionString.present() && existingConnectionString.get() == newConnectionString) {
 					self->lastPersistedConnectionString = newConnectionString;
-					return true;
+					co_return true;
 				}
-				// Someone has updated the connection string to something we didn't expect, in which case we leave it
-				// alone. It's possible this could result in the stored string getting stuck if the connection string
-				// changes twice and only the first change is recorded. If the process that wrote the first change dies
-				// and no other process attempts to write the intermediate state, then only a newly opened connection
-				// key would be able to update the state.
+				// Someone has updated the connection string to something we didn't expect, in which case we leave
+				// it alone. It's possible this could result in the stored string getting stuck if the connection
+				// string changes twice and only the first change is recorded. If the process that wrote the first
+				// change dies and no other process attempts to write the intermediate state, then only a newly
+				// opened connection key would be able to update the state.
 				else if (existingConnectionString.present() &&
 				         existingConnectionString != self->lastPersistedConnectionString) {
 					TraceEvent(SevWarnAlways, "UnableToChangeConnectionKeyDueToMismatch")
@@ -140,16 +142,17 @@ ACTOR Future<bool> ClusterConnectionKey::persistImpl(Reference<ClusterConnection
 					    .detail("NewConnectionString", newConnectionString)
 					    .detail("ExpectedStoredConnectionString", self->lastPersistedConnectionString)
 					    .detail("ActualStoredConnectionString", existingConnectionString);
-					return false;
+					co_return false;
 				}
 				tr.set(self->connectionStringKey, newConnectionString);
-				wait(tr.commit());
+				co_await tr.commit();
 
 				self->lastPersistedConnectionString = newConnectionString;
-				return true;
+				co_return true;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		TraceEvent(SevWarnAlways, "UnableToChangeConnectionKey")
@@ -158,10 +161,10 @@ ACTOR Future<bool> ClusterConnectionKey::persistImpl(Reference<ClusterConnection
 		    .detail("ConnectionString", self->cs.toString());
 	}
 
-	return false;
+	co_return false;
 };
 
 // Writes the connection string to the database
 Future<bool> ClusterConnectionKey::persist() {
 	return persistImpl(Reference<ClusterConnectionKey>::addRef(this));
-}
\ No newline at end of file
+}
diff --git a/fdbclient/include/fdbclient/ClusterConnectionKey.actor.h b/fdbclient/ClusterConnectionKey.h
similarity index 76%
rename from fdbclient/include/fdbclient/ClusterConnectionKey.actor.h
rename to fdbclient/ClusterConnectionKey.h
index e728f61a265..ce8be18b1a4 100644
--- a/fdbclient/include/fdbclient/ClusterConnectionKey.actor.h
+++ b/fdbclient/ClusterConnectionKey.h
@@ -1,5 +1,5 @@
 /*
- * ClusterConnectionKey.actor.h
+ * ClusterConnectionKey.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,17 +20,8 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_G_H)
-#define FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_G_H
-#include "fdbclient/ClusterConnectionKey.actor.g.h"
-#elif !defined(FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_H)
-#define FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_H
-
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 // An implementation of IClusterConnectionRecord backed by a key in a FoundationDB database.
 class ClusterConnectionKey : public IClusterConnectionRecord, ReferenceCounted<ClusterConnectionKey>, NonCopyable {
@@ -45,7 +36,7 @@ class ClusterConnectionKey : public IClusterConnectionRecord, ReferenceCounted<C
 
 	// Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or the
 	// format is invalid.
-	ACTOR static Future<Reference<ClusterConnectionKey>> loadClusterConnectionKey(Database db, Key connectionStringKey);
+	static Future<Reference<ClusterConnectionKey>> loadClusterConnectionKey(Database db, Key connectionStringKey);
 
 	// Sets the connections string held by this object and persists it.
 	Future<Void> setAndPersistConnectionString(ClusterConnectionString const&) override;
@@ -76,10 +67,9 @@ class ClusterConnectionKey : public IClusterConnectionRecord, ReferenceCounted<C
 	Future<bool> persist() override;
 
 private:
-	ACTOR static Future<ClusterConnectionString> getStoredConnectionStringImpl(Reference<ClusterConnectionKey> self);
-	ACTOR static Future<bool> upToDateImpl(Reference<ClusterConnectionKey> self,
-	                                       ClusterConnectionString* connectionString);
-	ACTOR static Future<bool> persistImpl(Reference<ClusterConnectionKey> self);
+	static Future<ClusterConnectionString> getStoredConnectionStringImpl(Reference<ClusterConnectionKey> self);
+	static Future<bool> upToDateImpl(Reference<ClusterConnectionKey> self, ClusterConnectionString* connectionString);
+	static Future<bool> persistImpl(Reference<ClusterConnectionKey> self);
 
 	// The database where the connection key is stored. Note that this does not need to be the same database as the one
 	// that the connection string would connect to.
@@ -87,6 +77,3 @@ class ClusterConnectionKey : public IClusterConnectionRecord, ReferenceCounted<C
 	Key connectionStringKey;
 	Optional<Value> lastPersistedConnectionString;
 };
-
-#include "flow/unactorcompiler.h"
-#endif
\ No newline at end of file
diff --git a/fdbclient/ClusterConnectionMemoryRecord.actor.cpp b/fdbclient/ClusterConnectionMemoryRecord.cpp
similarity index 94%
rename from fdbclient/ClusterConnectionMemoryRecord.actor.cpp
rename to fdbclient/ClusterConnectionMemoryRecord.cpp
index ec7d2a9afb3..fbc0f86c752 100644
--- a/fdbclient/ClusterConnectionMemoryRecord.actor.cpp
+++ b/fdbclient/ClusterConnectionMemoryRecord.cpp
@@ -1,5 +1,5 @@
 /*
- * ClusterConnectionMemoryRecord.actor.cpp
+ * ClusterConnectionMemoryRecord.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,8 +19,6 @@
  */
 
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbclient/MonitorLeader.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 // Sets the connections string held by this object.
 Future<Void> ClusterConnectionMemoryRecord::setAndPersistConnectionString(ClusterConnectionString const& conn) {
@@ -60,4 +58,4 @@ std::string ClusterConnectionMemoryRecord::toString() const {
 // This is a no-op for memory records. Returns true to indicate success.
 Future<bool> ClusterConnectionMemoryRecord::persist() {
 	return true;
-}
\ No newline at end of file
+}
diff --git a/fdbclient/CoordinationInterface.cpp b/fdbclient/CoordinationInterface.cpp
index 4a58fe9e8cd..d8ee138fb7c 100644
--- a/fdbclient/CoordinationInterface.cpp
+++ b/fdbclient/CoordinationInterface.cpp
@@ -18,9 +18,6 @@
  * limitations under the License.
  */
 
-#include "flow/Platform.h"
-#include <algorithm>
-
 #ifndef BOOST_SYSTEM_NO_LIB
 #define BOOST_SYSTEM_NO_LIB
 #endif
diff --git a/fdbclient/DataDistributionConfig.actor.cpp b/fdbclient/DataDistributionConfig.cpp
similarity index 91%
rename from fdbclient/DataDistributionConfig.actor.cpp
rename to fdbclient/DataDistributionConfig.cpp
index 649efd88387..43d3302daa2 100644
--- a/fdbclient/DataDistributionConfig.actor.cpp
+++ b/fdbclient/DataDistributionConfig.cpp
@@ -1,5 +1,5 @@
 /*
- * DataDistributionConfig.actor.cpp
+ * DataDistributionConfig.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,9 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbclient/DataDistributionConfig.actor.h"
+#include "fdbclient/DataDistributionConfig.h"
 #include "fdbclient/json_spirit/json_spirit_value.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 json_spirit::mValue DDConfiguration::toJSON(RangeConfigMapSnapshot const& config, bool includeDefaultRanges) {
 	json_spirit::mObject doc;
diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.cpp
similarity index 59%
rename from fdbclient/DatabaseBackupAgent.actor.cpp
rename to fdbclient/DatabaseBackupAgent.cpp
index 6bdcaa154e8..a094fec7ece 100644
--- a/fdbclient/DatabaseBackupAgent.actor.cpp
+++ b/fdbclient/DatabaseBackupAgent.cpp
@@ -1,5 +1,5 @@
 /*
- * DatabaseBackupAgent.actor.cpp
+ * DatabaseBackupAgent.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,7 +19,7 @@
  */
 
 #include <iterator>
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/StatusClient.h"
 #include "fdbclient/DatabaseContext.h"
@@ -32,13 +32,11 @@
 #include "flow/genericactors.actor.h"
 #include "flow/Hash3.h"
 #include <numeric>
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include <inttypes.h>
 #include <map>
 
-#include "flow/actorcompiler.h" // has to be last include
-
 const Key DatabaseBackupAgent::keyAddPrefix = "add_prefix"_sr;
 const Key DatabaseBackupAgent::keyRemovePrefix = "remove_prefix"_sr;
 const Key DatabaseBackupAgent::keyRangeVersions = "range_versions"_sr;
@@ -76,9 +74,9 @@ DatabaseBackupAgent::DatabaseBackupAgent(Database src)
 // Any new per-DR properties should go here.
 class DRConfig {
 public:
-	DRConfig(UID uid = UID())
+	explicit DRConfig(UID uid = UID())
 	  : uid(uid), configSpace(uidPrefixKey("uid->config/"_sr.withPrefix(databaseBackupPrefixRange.begin), uid)) {}
-	DRConfig(Reference<Task> task)
+	explicit DRConfig(Reference<Task> task)
 	  : DRConfig(BinaryReader::fromStringRef<UID>(task->params[BackupAgentBase::keyConfigLogUid], Unversioned())) {}
 
 	KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(__FUNCTION__sr); }
@@ -110,7 +108,7 @@ bool copyDefaultParameters(Reference<Task> source, Reference<Task> dest) {
 	return false;
 }
 
-ACTOR template <class Tr>
+template <class Tr>
 Future<Void> checkTaskVersion(Tr tr, Reference<Task> task, StringRef name, uint32_t version) {
 	uint32_t taskVersion = task->getVersion();
 	if (taskVersion > version) {
@@ -118,19 +116,17 @@ Future<Void> checkTaskVersion(Tr tr, Reference<Task> task, StringRef name, uint3
 		    .detail("TaskVersion", taskVersion)
 		    .detail("Name", name)
 		    .detail("Version", version);
-		wait(logError(tr,
-		              Subspace(databaseBackupPrefixRange.begin)
-		                  .get(BackupAgentBase::keyErrors)
-		                  .pack(task->params[BackupAgentBase::keyConfigLogUid]),
-		              format("ERROR: %s task version `%lu' is greater than supported version `%lu'",
-		                     task->params[Task::reservedTaskParamKeyType].toString().c_str(),
-		                     (unsigned long)taskVersion,
-		                     (unsigned long)version)));
+		co_await logError(tr,
+		                  Subspace(databaseBackupPrefixRange.begin)
+		                      .get(BackupAgentBase::keyErrors)
+		                      .pack(task->params[BackupAgentBase::keyConfigLogUid]),
+		                  format("ERROR: %s task version `%lu' is greater than supported version `%lu'",
+		                         task->params[Task::reservedTaskParamKeyType].toString().c_str(),
+		                         (unsigned long)taskVersion,
+		                         (unsigned long)version));
 
 		throw task_invalid_version();
 	}
-
-	return Void();
 }
 
 struct BackupRangeTaskFunc : TaskFuncBase {
@@ -159,32 +155,32 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr,
-	                                                                    Key beginKey,
-	                                                                    Key endKey,
-	                                                                    int limit) {
+	static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr,
+	                                                              Key beginKey,
+	                                                              Key endKey,
+	                                                              int limit) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Standalone<VectorRef<KeyRef>> results;
-		RangeResult values = wait(tr->getRange(
-		    KeyRangeRef(keyAfter(beginKey.withPrefix(keyServersPrefix)), endKey.withPrefix(keyServersPrefix)), limit));
+		Standalone<VectorRef<KeyRef>> results;
+		RangeResult values = co_await tr->getRange(
+		    KeyRangeRef(keyAfter(beginKey.withPrefix(keyServersPrefix)), endKey.withPrefix(keyServersPrefix)), limit);
 
 		for (auto& s : values) {
 			KeyRef k = s.key.removePrefix(keyServersPrefix);
 			results.push_back_deep(results.arena(), k);
 		}
 
-		return results;
+		co_return results;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Key begin,
-	                                 Key end,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Key begin,
+	                           Key end,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(BackupRangeTaskFunc::name, BackupRangeTaskFunc::version, doneKey);
 
 		copyDefaultParameters(parentTask, task);
@@ -193,84 +189,85 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		task->params[BackupAgentBase::keyEndKey] = end;
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
 
-		wait(checkTaskVersion(cx, task, BackupRangeTaskFunc::name, BackupRangeTaskFunc::version));
+		co_await checkTaskVersion(cx, task, BackupRangeTaskFunc::name, BackupRangeTaskFunc::version);
 		// Find out if there is a shard boundary in(beginKey, endKey)
 		Standalone<VectorRef<KeyRef>> keys =
-		    wait(runRYWTransaction(taskBucket->src, [=](Reference<ReadYourWritesTransaction> tr) {
+		    co_await runRYWTransaction(taskBucket->src, [=](Reference<ReadYourWritesTransaction> tr) {
 			    return getBlockOfShards(tr,
 			                            task->params[DatabaseBackupAgent::keyBeginKey],
 			                            task->params[DatabaseBackupAgent::keyEndKey],
 			                            CLIENT_KNOBS->BACKUP_SHARD_TASK_LIMIT);
-		    }));
-		if (keys.size() > 0) {
+		    });
+		if (!keys.empty()) {
 			task->params[BackupRangeTaskFunc::keyAddBackupRangeTasks] = BinaryWriter::toValue(keys, IncludeVersion());
-			return Void();
+			co_return;
 		}
 
 		// Read everything from beginKey to endKey, write it to an output file, run the output file processor, and
 		// then set on_done.If we are still writing after X seconds, end the output file and insert a new backup_range
 		// task for the remainder.
-		state double timeout = now() + CLIENT_KNOBS->BACKUP_RANGE_TIMEOUT;
-		state Key addPrefix = task->params[DatabaseBackupAgent::keyAddPrefix];
-		state Key removePrefix = task->params[DatabaseBackupAgent::keyRemovePrefix];
+		double timeout = now() + CLIENT_KNOBS->BACKUP_RANGE_TIMEOUT;
+		Key addPrefix = task->params[DatabaseBackupAgent::keyAddPrefix];
+		Key removePrefix = task->params[DatabaseBackupAgent::keyRemovePrefix];
 
-		state KeyRange range(
+		KeyRange range(
 		    KeyRangeRef(task->params[BackupAgentBase::keyBeginKey], task->params[BackupAgentBase::keyEndKey]));
 
 		// retrieve kvData
-		state PromiseStream<RangeResultWithVersion> results;
-
-		state Future<Void> rc = readCommitted(taskBucket->src,
-		                                      results,
-		                                      lock,
-		                                      range,
-		                                      Terminator::True,
-		                                      AccessSystemKeys::True,
-		                                      LockAware::True,
-		                                      ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY));
-		state Key rangeBegin = range.begin;
-		state Key rangeEnd;
-		state bool endOfStream = false;
-		state RangeResultWithVersion nextValues;
-		state int64_t nextValuesSize = 0;
+		PromiseStream<RangeResultWithVersion> results;
+
+		Future<Void> rc = readCommitted(taskBucket->src,
+		                                results,
+		                                lock,
+		                                range,
+		                                Terminator::True,
+		                                AccessSystemKeys::True,
+		                                LockAware::True,
+		                                ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY));
+		Key rangeBegin = range.begin;
+		Key rangeEnd;
+		bool endOfStream = false;
+		RangeResultWithVersion nextValues;
+		int64_t nextValuesSize = 0;
 		nextValues.second = invalidVersion;
-		loop {
+		while (true) {
 			if (endOfStream && nextValues.second == invalidVersion) {
-				return Void();
+				co_return;
 			}
-			state RangeResultWithVersion values = std::move(nextValues);
-			state int64_t valuesSize = nextValuesSize;
+			RangeResultWithVersion values = std::move(nextValues);
+			int64_t valuesSize = nextValuesSize;
 			nextValues = RangeResultWithVersion();
 			nextValues.second = invalidVersion;
 			nextValuesSize = 0;
 
 			if (!endOfStream) {
-				loop {
+				while (true) {
+					Error err;
 					try {
-						RangeResultWithVersion v = waitNext(results.getFuture());
+						RangeResultWithVersion v = co_await results.getFuture();
 						int64_t resultSize = v.first.expectedSize();
 						lock->release(resultSize);
 
@@ -288,71 +285,74 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 						}
 
 						valuesSize += resultSize;
+						continue;
 					} catch (Error& e) {
-						state Error err = e;
-						if (err.code() == error_code_actor_cancelled)
-							throw err;
+						err = e;
+					}
+					if (err.code() == error_code_actor_cancelled)
+						throw err;
 
-						if (err.code() == error_code_end_of_stream) {
-							endOfStream = true;
-							if (values.second != invalidVersion)
-								break;
-							return Void();
-						}
+					if (err.code() == error_code_end_of_stream) {
+						endOfStream = true;
+						if (values.second != invalidVersion)
+							break;
+						co_return;
+					}
 
-						wait(logError(cx,
-						              Subspace(databaseBackupPrefixRange.begin)
-						                  .get(BackupAgentBase::keyErrors)
-						                  .pack(task->params[BackupAgentBase::keyConfigLogUid]),
-						              format("ERROR: %s", err.what())));
+					co_await logError(cx,
+					                  Subspace(databaseBackupPrefixRange.begin)
+					                      .get(BackupAgentBase::keyErrors)
+					                      .pack(task->params[BackupAgentBase::keyConfigLogUid]),
+					                  format("ERROR: %s", err.what()));
 
-						throw err;
-					}
+					throw err;
 				}
 			}
 
 			if (now() >= timeout) {
 				task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin;
-				return Void();
+				co_return;
 			}
 
 			rangeEnd = values.first.more ? keyAfter(values.first.end()[-1].key) : range.end;
 
-			state int valueLoc = 0;
-			state int committedValueLoc = 0;
-			state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-			loop {
+			int valueLoc = 0;
+			int committedValueLoc = 0;
+			auto tr = makeReference<ReadYourWritesTransaction>(cx);
+			while (true) {
+				Error err;
+				bool hasErr = false;
 				try {
 					tr->reset();
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-					state Key prefix = task->params[BackupAgentBase::keyConfigLogUid].withPrefix(
+					Key prefix = task->params[BackupAgentBase::keyConfigLogUid].withPrefix(
 					    applyMutationsKeyVersionMapRange.begin);
-					state Key rangeCountKey = task->params[BackupAgentBase::keyConfigLogUid].withPrefix(
+					Key rangeCountKey = task->params[BackupAgentBase::keyConfigLogUid].withPrefix(
 					    applyMutationsKeyVersionCountRange.begin);
-					state Future<RangeResult> backupVersions =
+					Future<RangeResult> backupVersions =
 					    krmGetRanges(tr, prefix, KeyRangeRef(rangeBegin, rangeEnd), BUGGIFY ? 2 : 2000, 1e5);
-					state Future<Optional<Value>> logVersionValue =
+					Future<Optional<Value>> logVersionValue =
 					    tr->get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin),
 					            Snapshot::True);
-					state Future<Optional<Value>> rangeCountValue = tr->get(rangeCountKey, Snapshot::True);
-					state Future<RangeResult> prevRange = tr->getRange(firstGreaterOrEqual(prefix),
-					                                                   lastLessOrEqual(rangeBegin.withPrefix(prefix)),
-					                                                   1,
-					                                                   Snapshot::True,
-					                                                   Reverse::True);
-					state Future<RangeResult> nextRange = tr->getRange(firstGreaterOrEqual(rangeEnd.withPrefix(prefix)),
-					                                                   firstGreaterOrEqual(strinc(prefix)),
-					                                                   1,
-					                                                   Snapshot::True,
-					                                                   Reverse::False);
-					state Future<Void> verified = taskBucket->keepRunning(tr, task);
-
-					wait(checkDatabaseLock(tr,
-					                       BinaryReader::fromStringRef<UID>(
-					                           task->params[BackupAgentBase::keyConfigLogUid], Unversioned())));
-					wait(success(backupVersions) && success(logVersionValue) && success(rangeCountValue) &&
-					     success(prevRange) && success(nextRange) && success(verified));
+					Future<Optional<Value>> rangeCountValue = tr->get(rangeCountKey, Snapshot::True);
+					Future<RangeResult> prevRange = tr->getRange(firstGreaterOrEqual(prefix),
+					                                             lastLessOrEqual(rangeBegin.withPrefix(prefix)),
+					                                             1,
+					                                             Snapshot::True,
+					                                             Reverse::True);
+					Future<RangeResult> nextRange = tr->getRange(firstGreaterOrEqual(rangeEnd.withPrefix(prefix)),
+					                                             firstGreaterOrEqual(strinc(prefix)),
+					                                             1,
+					                                             Snapshot::True,
+					                                             Reverse::False);
+					Future<Void> verified = taskBucket->keepRunning(tr, task);
+
+					co_await checkDatabaseLock(tr,
+					                           BinaryReader::fromStringRef<UID>(
+					                               task->params[BackupAgentBase::keyConfigLogUid], Unversioned()));
+					co_await (success(backupVersions) && success(logVersionValue) && success(rangeCountValue) &&
+					          success(prevRange) && success(nextRange) && success(verified));
 
 					int64_t rangeCount = 0;
 					if (rangeCountValue.get().present()) {
@@ -361,10 +361,10 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 					}
 
 					bool prevAdjacent =
-					    prevRange.get().size() && prevRange.get()[0].value.size() &&
+					    !prevRange.get().empty() && !prevRange.get()[0].value.empty() &&
 					    BinaryReader::fromStringRef<Version>(prevRange.get()[0].value, Unversioned()) != invalidVersion;
 					bool nextAdjacent =
-					    nextRange.get().size() && nextRange.get()[0].value.size() &&
+					    !nextRange.get().empty() && !nextRange.get()[0].value.empty() &&
 					    BinaryReader::fromStringRef<Version>(nextRange.get()[0].value, Unversioned()) != invalidVersion;
 
 					if ((!prevAdjacent || !nextAdjacent) &&
@@ -376,9 +376,9 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 						if (rangeCount > CLIENT_KNOBS->BACKUP_MAP_KEY_UPPER_LIMIT)
 							TraceEvent(SevWarnAlways, "DBA_KeyRangeMapTooLarge").log();
 
-						wait(delay(1));
+						co_await delay(1);
 						task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin;
-						return Void();
+						co_return;
 					}
 
 					Version logVersion =
@@ -387,21 +387,21 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 					        : ::invalidVersion;
 					if (logVersion >= values.second) {
 						task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin;
-						return Void();
+						co_return;
 					}
 
 					//TraceEvent("DBA_Range").detail("Range", KeyRangeRef(rangeBegin, rangeEnd)).detail("Version", values.second).detail("Size", values.first.size()).detail("LogUID", task->params[BackupAgentBase::keyConfigLogUid]).detail("AddPrefix", addPrefix).detail("RemovePrefix", removePrefix);
 
 					Subspace krv(conf.get(DatabaseBackupAgent::keyRangeVersions));
-					state KeyRange versionRange = singleKeyRange(krv.pack(values.second));
+					KeyRange versionRange = singleKeyRange(krv.pack(values.second));
 					tr->addReadConflictRange(versionRange);
 					tr->addWriteConflictRange(versionRange);
 
 					int versionLoc = 0;
 					std::vector<Future<Void>> setRanges;
-					state int64_t bytesSet = 0;
+					int64_t bytesSet = 0;
 
-					loop {
+					while (true) {
 						while (versionLoc < backupVersions.get().size() - 1 &&
 						       (backupVersions.get()[versionLoc].value.size() < sizeof(Version) ||
 						        BinaryReader::fromStringRef<Version>(backupVersions.get()[versionLoc].value,
@@ -413,10 +413,10 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 							break;
 
 						if (backupVersions.get()[versionLoc + 1].key ==
-						    (removePrefix == StringRef() ? allKeys.end : strinc(removePrefix))) {
+						    (removePrefix.empty() ? allKeys.end : strinc(removePrefix))) {
 							tr->clear(KeyRangeRef(
 							    backupVersions.get()[versionLoc].key.removePrefix(removePrefix).withPrefix(addPrefix),
-							    addPrefix == StringRef() ? allKeys.end : strinc(addPrefix)));
+							    addPrefix.empty() ? allKeys.end : strinc(addPrefix)));
 						} else {
 							tr->clear(KeyRangeRef(backupVersions.get()[versionLoc].key,
 							                      backupVersions.get()[versionLoc + 1].key)
@@ -448,9 +448,9 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 						versionLoc++;
 					}
 
-					wait(waitForAll(setRanges));
+					co_await waitForAll(setRanges);
 
-					wait(tr->commit());
+					co_await tr->commit();
 					Params.bytesWritten().set(task, Params.bytesWritten().getOrDefault(task) + bytesSet);
 					//TraceEvent("DBA_SetComplete", debugID).detail("Ver", values.second).detail("LogVersion", logVersion).detail("ReadVersion", readVer).detail("CommitVer", tr.getCommittedVersion()).detail("Range", versionRange);
 
@@ -462,7 +462,11 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 						break;
 					}
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
+					hasErr = true;
+				}
+				if (hasErr) {
+					co_await tr->onError(err);
 					valueLoc = committedValueLoc;
 				}
 			}
@@ -471,23 +475,23 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		}
 	}
 
-	ACTOR static Future<Void> startBackupRangeInternal(Reference<ReadYourWritesTransaction> tr,
-	                                                   Standalone<VectorRef<KeyRef>> keys,
-	                                                   Reference<TaskBucket> taskBucket,
-	                                                   Reference<FutureBucket> futureBucket,
-	                                                   Reference<Task> task,
-	                                                   Reference<TaskFuture> onDone) {
+	static Future<Void> startBackupRangeInternal(Reference<ReadYourWritesTransaction> tr,
+	                                             Standalone<VectorRef<KeyRef>> keys,
+	                                             Reference<TaskBucket> taskBucket,
+	                                             Reference<FutureBucket> futureBucket,
+	                                             Reference<Task> task,
+	                                             Reference<TaskFuture> onDone) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Key nextKey = task->params[BackupAgentBase::keyBeginKey];
+		Key nextKey = task->params[BackupAgentBase::keyBeginKey];
 
 		std::vector<Future<Key>> addTaskVector;
-		for (int idx = 0; idx < keys.size(); ++idx) {
-			if (nextKey != keys[idx]) {
+		for (const auto& splitKey : keys) {
+			if (nextKey != splitKey) {
 				addTaskVector.push_back(
-				    addTask(tr, taskBucket, task, nextKey, keys[idx], TaskCompletionKey::joinWith(onDone)));
+				    addTask(tr, taskBucket, task, nextKey, splitKey, TaskCompletionKey::joinWith(onDone)));
 			}
-			nextKey = keys[idx];
+			nextKey = splitKey;
 		}
 
 		if (nextKey != task->params[BackupAgentBase::keyEndKey]) {
@@ -499,16 +503,14 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 			                                TaskCompletionKey::joinWith(onDone)));
 		}
 
-		wait(waitForAll(addTaskVector));
-
-		return Void();
+		co_await waitForAll(addTaskVector);
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
 		// Get the bytesWritten parameter from task and atomically add it to the rangeBytesWritten() property of the DR
 		// config.
@@ -517,31 +519,29 @@ struct BackupRangeTaskFunc : TaskFuncBase {
 		config.rangeBytesWritten().atomicOp(tr, bytesWritten, MutationRef::AddValue);
 
 		if (task->params.find(BackupRangeTaskFunc::keyAddBackupRangeTasks) != task->params.end()) {
-			wait(startBackupRangeInternal(
-			         tr,
-			         BinaryReader::fromStringRef<Standalone<VectorRef<KeyRef>>>(
-			             task->params[BackupRangeTaskFunc::keyAddBackupRangeTasks], IncludeVersion()),
-			         taskBucket,
-			         futureBucket,
-			         task,
-			         taskFuture) &&
-			     taskBucket->finish(tr, task));
+			co_await (startBackupRangeInternal(
+			              tr,
+			              BinaryReader::fromStringRef<Standalone<VectorRef<KeyRef>>>(
+			                  task->params[BackupRangeTaskFunc::keyAddBackupRangeTasks], IncludeVersion()),
+			              taskBucket,
+			              futureBucket,
+			              task,
+			              taskFuture) &&
+			          taskBucket->finish(tr, task));
 		} else if (task->params.find(BackupRangeTaskFunc::keyBackupRangeBeginKey) != task->params.end() &&
 		           task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] <
 		               task->params[BackupAgentBase::keyEndKey]) {
-			ASSERT(taskFuture->key.size() > 0);
-			wait(success(BackupRangeTaskFunc::addTask(tr,
-			                                          taskBucket,
-			                                          task,
-			                                          task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey],
-			                                          task->params[BackupAgentBase::keyEndKey],
-			                                          TaskCompletionKey::signal(taskFuture->key))) &&
-			     taskBucket->finish(tr, task));
+			ASSERT(!taskFuture->key.empty());
+			co_await (success(BackupRangeTaskFunc::addTask(tr,
+			                                               taskBucket,
+			                                               task,
+			                                               task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey],
+			                                               task->params[BackupAgentBase::keyEndKey],
+			                                               TaskCompletionKey::signal(taskFuture->key))) &&
+			          taskBucket->finish(tr, task));
 		} else {
-			wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
+			co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 		}
-
-		return Void();
 	}
 };
 StringRef BackupRangeTaskFunc::name = "dr_backup_range"_sr;
@@ -553,50 +553,48 @@ struct FinishFullBackupTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Subspace states = Subspace(databaseBackupPrefixRange.begin)
-		                            .get(BackupAgentBase::keyStates)
-		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		wait(checkTaskVersion(tr, task, FinishFullBackupTaskFunc::name, FinishFullBackupTaskFunc::version));
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Subspace states = Subspace(databaseBackupPrefixRange.begin)
+		                      .get(BackupAgentBase::keyStates)
+		                      .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		co_await checkTaskVersion(tr, task, FinishFullBackupTaskFunc::name, FinishFullBackupTaskFunc::version);
 
 		// Enable the stop key
 		Transaction srcTr(taskBucket->src);
 		srcTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		Version readVersion = wait(srcTr.getReadVersion());
+		Version readVersion = co_await srcTr.getReadVersion();
 		tr->set(states.pack(DatabaseBackupAgent::keyCopyStop), BinaryWriter::toValue(readVersion, Unversioned()));
 		TraceEvent("DBA_FinishFullBackup").detail("CopyStop", readVersion);
-		wait(taskBucket->finish(tr, task));
-
-		return Void();
+		co_await taskBucket->finish(tr, task);
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
 		// After the BackupRangeTask completes, set the stop key which will stop the BackupLogsTask
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(FinishFullBackupTaskFunc::name, FinishFullBackupTaskFunc::version, doneKey);
 
 		copyDefaultParameters(parentTask, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -636,41 +634,47 @@ struct EraseLogRangeTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state FlowLock lock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		FlowLock lock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
 
-		wait(checkTaskVersion(cx, task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version));
+		co_await checkTaskVersion(cx, task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version);
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(taskBucket->src));
-		loop {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(taskBucket->src));
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				Version endVersion = BinaryReader::fromStringRef<Version>(
 				    task->params[DatabaseBackupAgent::keyEndVersion], Unversioned());
-				wait(eraseLogData(
+				co_await eraseLogData(
 				    tr,
 				    task->params[BackupAgentBase::keyConfigLogUid],
 				    task->params[BackupAgentBase::destUid],
 				    Optional<Version>(endVersion),
 				    CheckBackupUID::True,
-				    BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned())));
-				wait(tr->commit());
-				return Void();
+				    BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned()));
+				co_await tr->commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version endVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version endVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version, doneKey, 1);
 
 		copyDefaultParameters(parentTask, task);
@@ -680,29 +684,28 @@ struct EraseLogRangeTaskFunc : TaskFuncBase {
 		task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
-		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
-		return Void();
+		co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 	}
 };
 StringRef EraseLogRangeTaskFunc::name = "dr_erase_log_range"_sr;
@@ -735,36 +738,37 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 
 	// store mutation data from results until the end of stream or the timeout. If breaks on timeout returns the first
 	// uncopied version
-	ACTOR static Future<Optional<Version>> dumpData(Database cx,
-	                                                Reference<Task> task,
-	                                                PromiseStream<RCGroup> results,
-	                                                FlowLock* lock,
-	                                                Reference<TaskBucket> tb,
-	                                                double breakTime) {
-		state bool endOfStream = false;
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		state std::vector<RangeResult> nextMutations;
-		state bool isTimeoutOccurred = false;
-		state Optional<KeyRef> lastKey;
-		state Version lastVersion;
-		state int64_t nextMutationSize = 0;
-		loop {
+	static Future<Optional<Version>> dumpData(Database cx,
+	                                          Reference<Task> task,
+	                                          PromiseStream<RCGroup> results,
+	                                          FlowLock* lock,
+	                                          Reference<TaskBucket> tb,
+	                                          double breakTime) {
+		bool endOfStream = false;
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		std::vector<RangeResult> nextMutations;
+		bool isTimeoutOccurred = false;
+		Optional<KeyRef> lastKey;
+		Version lastVersion{ 0 };
+		int64_t nextMutationSize = 0;
+		while (true) {
+			Error err;
 			try {
 				if (endOfStream && !nextMutationSize) {
-					return Optional<Version>();
+					co_return Optional<Version>();
 				}
 
-				state std::vector<RangeResult> mutations = std::move(nextMutations);
-				state int64_t mutationSize = nextMutationSize;
+				std::vector<RangeResult> mutations = std::move(nextMutations);
+				int64_t mutationSize = nextMutationSize;
 				nextMutations = std::vector<RangeResult>();
 				nextMutationSize = 0;
 
 				if (!endOfStream) {
-					loop {
+					while (true) {
 						try {
-							RCGroup group = waitNext(results.getFuture());
+							RCGroup group = co_await results.getFuture();
 							lock->release(group.items.expectedSize());
 
 							int vecSize = group.items.expectedSize();
@@ -778,7 +782,7 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 							mutations.push_back(group.items);
 							mutationSize += vecSize;
 						} catch (Error& e) {
-							state Error error = e;
+							Error error = e;
 							if (e.code() == error_code_end_of_stream) {
 								endOfStream = true;
 								break;
@@ -789,20 +793,22 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 					}
 				}
 
-				state Optional<Version> nextVersionAfterBreak;
-				state Transaction tr(cx);
+				Optional<Version> nextVersionAfterBreak;
+				Transaction tr(cx);
 
-				loop {
+				while (true) {
+					Error err;
+					bool hasErr = false;
 					try {
 						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 						tr.trState->options.sizeLimit = 2 * CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
-						wait(checkDatabaseLock(&tr,
-						                       BinaryReader::fromStringRef<UID>(
-						                           task->params[BackupAgentBase::keyConfigLogUid], Unversioned())));
-						state int64_t bytesSet = 0;
+						co_await checkDatabaseLock(&tr,
+						                           BinaryReader::fromStringRef<UID>(
+						                               task->params[BackupAgentBase::keyConfigLogUid], Unversioned()));
+						int64_t bytesSet = 0;
 
 						bool first = true;
-						for (auto m : mutations) {
+						for (const auto& m : mutations) {
 							for (auto kv : m) {
 								if (isTimeoutOccurred) {
 									Version newVersion = getLogKeyVersion(kv.key);
@@ -827,15 +833,19 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 							}
 						}
 
-						wait(tr.commit());
+						co_await tr.commit();
 						Params.bytesWritten().set(task, Params.bytesWritten().getOrDefault(task) + bytesSet);
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
+						hasErr = true;
+					}
+					if (hasErr) {
+						co_await tr.onError(err);
 					}
 				}
 				if (nextVersionAfterBreak.present()) {
-					return nextVersionAfterBreak;
+					co_return nextVersionAfterBreak;
 				}
 				if (!isTimeoutOccurred && timer_monotonic() >= breakTime && lastKey.present()) {
 					// timeout occurred
@@ -845,33 +855,34 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 					lastVersion = getLogKeyVersion(lastKey.get());
 					isTimeoutOccurred = true;
 				}
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled || e.code() == error_code_backup_error)
-					throw e;
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled || err.code() == error_code_backup_error)
+				throw err;
 
-				state Error err = e;
-				wait(logError(cx,
-				              Subspace(databaseBackupPrefixRange.begin)
-				                  .get(BackupAgentBase::keyErrors)
-				                  .pack(task->params[BackupAgentBase::keyConfigLogUid]),
-				              format("ERROR: Failed to dump mutations because of error %s", err.what())));
+			co_await logError(cx,
+			                  Subspace(databaseBackupPrefixRange.begin)
+			                      .get(BackupAgentBase::keyErrors)
+			                      .pack(task->params[BackupAgentBase::keyConfigLogUid]),
+			                  format("ERROR: Failed to dump mutations because of error %s", err.what()));
 
-				throw err;
-			}
+			throw err;
 		}
 	}
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
 		// state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
 
-		wait(checkTaskVersion(cx, task, CopyLogRangeTaskFunc::name, CopyLogRangeTaskFunc::version));
+		co_await checkTaskVersion(cx, task, CopyLogRangeTaskFunc::name, CopyLogRangeTaskFunc::version);
 
-		state Version beginVersion =
+		Version beginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
-		state Version endVersion =
+		Version endVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyEndVersion], Unversioned());
 
 		Version newEndVersion = std::min(endVersion,
@@ -880,18 +891,18 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 		                                  (g_network->isSimulated() ? CLIENT_KNOBS->BACKUP_SIM_COPY_LOG_RANGES : 0)) *
 		                                     CLIENT_KNOBS->COPY_LOG_BLOCK_SIZE);
 
-		state Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(
+		Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(
 		    beginVersion, newEndVersion, task->params[BackupAgentBase::destUid], CLIENT_KNOBS->COPY_LOG_BLOCK_SIZE);
-		state int nRanges = ranges.size();
+		int nRanges = ranges.size();
 
-		state std::vector<PromiseStream<RCGroup>> results;
-		state std::vector<Future<Void>> rc;
-		state std::vector<Reference<FlowLock>> locks;
-		state Version nextVersion = beginVersion;
-		state double breakTime = timer_monotonic() + CLIENT_KNOBS->COPY_LOG_TASK_DURATION_NANOS;
-		state int rangeN = 0;
+		std::vector<PromiseStream<RCGroup>> results;
+		std::vector<Future<Void>> rc;
+		std::vector<Reference<FlowLock>> locks;
+		Version nextVersion = beginVersion;
+		double breakTime = timer_monotonic() + CLIENT_KNOBS->COPY_LOG_TASK_DURATION_SECONDS;
+		int rangeN = 0;
 
-		loop {
+		while (true) {
 			if (rangeN >= nRanges)
 				break;
 
@@ -915,14 +926,14 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 
 			// copy the range
 			Optional<Version> nextVersionBr =
-			    wait(dumpData(cx, task, results[rangeN], locks[rangeN].getPtr(), taskBucket, breakTime));
+			    co_await dumpData(cx, task, results[rangeN], locks[rangeN].getPtr(), taskBucket, breakTime);
 
 			// exit from the task if a timeout occurs
 			if (nextVersionBr.present()) {
 				nextVersion = nextVersionBr.get();
 				// cancel prefetch
 				TraceEvent(SevInfo, "CopyLogRangeTaskFuncAborted")
-				    .detail("DurationNanos", CLIENT_KNOBS->COPY_LOG_TASK_DURATION_NANOS)
+				    .detail("DurationSeconds", CLIENT_KNOBS->COPY_LOG_TASK_DURATION_SECONDS)
 				    .detail("RangeN", rangeN)
 				    .detail("BytesWritten", Params.bytesWritten().getOrDefault(task));
 				for (int j = results.size(); --j >= rangeN;)
@@ -937,17 +948,16 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 		if (nextVersion < endVersion) {
 			task->params[CopyLogRangeTaskFunc::keyNextBeginVersion] = BinaryWriter::toValue(nextVersion, Unversioned());
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version beginVersion,
-	                                 Version endVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version beginVersion,
+	                           Version endVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(CopyLogRangeTaskFunc::name, CopyLogRangeTaskFunc::version, doneKey, 1);
 
 		copyDefaultParameters(parentTask, task);
@@ -956,28 +966,28 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 		task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 
-		state Version endVersion =
+		Version endVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyEndVersion], Unversioned());
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
 		// Get the bytesWritten parameter from task and atomically add it to the logBytesWritten() property of the DR
 		// config.
@@ -986,16 +996,14 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
 		config.logBytesWritten().atomicOp(tr, bytesWritten, MutationRef::AddValue);
 
 		if (task->params.find(CopyLogRangeTaskFunc::keyNextBeginVersion) != task->params.end()) {
-			state Version nextVersion = BinaryReader::fromStringRef<Version>(
+			Version nextVersion = BinaryReader::fromStringRef<Version>(
 			    task->params[CopyLogRangeTaskFunc::keyNextBeginVersion], Unversioned());
-			wait(success(CopyLogRangeTaskFunc::addTask(
-			         tr, taskBucket, task, nextVersion, endVersion, TaskCompletionKey::signal(taskFuture->key))) &&
-			     taskBucket->finish(tr, task));
+			co_await (success(CopyLogRangeTaskFunc::addTask(
+			              tr, taskBucket, task, nextVersion, endVersion, TaskCompletionKey::signal(taskFuture->key))) &&
+			          taskBucket->finish(tr, task));
 		} else {
-			wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
+			co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 		}
-
-		return Void();
 	}
 };
 StringRef CopyLogRangeTaskFunc::name = "dr_copy_log_range"_sr;
@@ -1006,46 +1014,45 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		state Subspace states = Subspace(databaseBackupPrefixRange.begin)
-		                            .get(BackupAgentBase::keyStates)
-		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		wait(checkTaskVersion(tr, task, CopyLogsTaskFunc::name, CopyLogsTaskFunc::version));
-
-		state Version beginVersion =
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		Subspace states = Subspace(databaseBackupPrefixRange.begin)
+		                      .get(BackupAgentBase::keyStates)
+		                      .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		co_await checkTaskVersion(tr, task, CopyLogsTaskFunc::name, CopyLogsTaskFunc::version);
+
+		Version beginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
-		state Version prevBeginVersion =
+		Version prevBeginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyPrevBeginVersion], Unversioned());
-		state Future<Optional<Value>> fStopValue = tr->get(states.pack(DatabaseBackupAgent::keyCopyStop));
-		state Future<Optional<Value>> fAppliedValue =
+		Future<Optional<Value>> fStopValue = tr->get(states.pack(DatabaseBackupAgent::keyCopyStop));
+		Future<Optional<Value>> fAppliedValue =
 		    tr->get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsBeginRange.begin));
 
 		Transaction srcTr(taskBucket->src);
 		srcTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Version endVersion = wait(srcTr.getReadVersion());
+		Version endVersion = co_await srcTr.getReadVersion();
 
-		state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
 		if (endVersion <= beginVersion) {
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			wait(success(CopyLogsTaskFunc::addTask(
-			    tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone))));
-			wait(taskBucket->finish(tr, task));
-			return Void();
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			co_await CopyLogsTaskFunc::addTask(
+			    tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone));
+			co_await taskBucket->finish(tr, task);
+			co_return;
 		}
 
-		Optional<Value> appliedValue = wait(fAppliedValue);
-		state Version appliedVersion =
+		Optional<Value> appliedValue = co_await fAppliedValue;
+		Version appliedVersion =
 		    appliedValue.present() ? BinaryReader::fromStringRef<Version>(appliedValue.get(), Unversioned()) : 100;
 
-		state Version applyVersion =
-		    std::max<Version>(appliedVersion, beginVersion - CLIENT_KNOBS->BACKUP_VERSION_DELAY);
+		Version applyVersion = std::max<Version>(appliedVersion, beginVersion - CLIENT_KNOBS->BACKUP_VERSION_DELAY);
 		Subspace krv = conf.get(DatabaseBackupAgent::keyRangeVersions);
 		KeyRange versionRange = KeyRangeRef(krv.pack(0), krv.pack(applyVersion + 1));
 		tr->addReadConflictRange(versionRange);
@@ -1053,8 +1060,8 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 		tr->set(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin),
 		        BinaryWriter::toValue(applyVersion, Unversioned()));
 
-		Optional<Value> stopValue = wait(fStopValue);
-		state Version stopVersionData =
+		Optional<Value> stopValue = co_await fStopValue;
+		Version stopVersionData =
 		    stopValue.present() ? BinaryReader::fromStringRef<Version>(stopValue.get(), Unversioned()) : -1;
 
 		if (endVersion - beginVersion > deterministicRandom()->randomInt64(0, CLIENT_KNOBS->BACKUP_VERSION_DELAY)) {
@@ -1067,7 +1074,7 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 		}
 
 		if ((stopVersionData == -1) || (stopVersionData >= applyVersion)) {
-			state Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
+			Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
 			std::vector<Future<Key>> addTaskVector;
 			addTaskVector.push_back(CopyLogsTaskFunc::addTask(
 			    tr, taskBucket, task, beginVersion, endVersion, TaskCompletionKey::signal(onDone), allPartsDone));
@@ -1091,31 +1098,29 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 				    tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(allPartsDone)));
 			}
 
-			wait(waitForAll(addTaskVector) && taskBucket->finish(tr, task));
+			co_await (waitForAll(addTaskVector) && taskBucket->finish(tr, task));
 		} else {
 			if (appliedVersion < applyVersion) {
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				wait(success(CopyLogsTaskFunc::addTask(
-				    tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone))));
-				wait(taskBucket->finish(tr, task));
-				return Void();
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+				co_await CopyLogsTaskFunc::addTask(
+				    tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone));
+				co_await taskBucket->finish(tr, task);
+				co_return;
 			}
 
-			wait(onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
+			co_await (onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
 			tr->set(states.pack(DatabaseBackupAgent::keyStateStop), BinaryWriter::toValue(beginVersion, Unversioned()));
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version prevBeginVersion,
-	                                 Version beginVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version prevBeginVersion,
+	                           Version beginVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(CopyLogsTaskFunc::name, CopyLogsTaskFunc::version, doneKey, 1);
 
 		copyDefaultParameters(parentTask, task);
@@ -1123,18 +1128,18 @@ struct CopyLogsTaskFunc : TaskFuncBase {
 		task->params[DatabaseBackupAgent::keyPrevBeginVersion] = BinaryWriter::toValue(prevBeginVersion, Unversioned());
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -1162,107 +1167,120 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase {
 
 	StringRef getName() const override { return name; };
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin)
-		                                  .get(BackupAgentBase::keySourceStates)
-		                                  .get(task->params[BackupAgentBase::keyConfigLogUid]);
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin)
+		                            .get(BackupAgentBase::keySourceStates)
+		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
 
-		wait(checkTaskVersion(cx, task, FinishedFullBackupTaskFunc::name, FinishedFullBackupTaskFunc::version));
+		co_await checkTaskVersion(cx, task, FinishedFullBackupTaskFunc::name, FinishedFullBackupTaskFunc::version);
 
-		state Transaction tr2(cx);
-		loop {
+		Transaction tr2(cx);
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr2.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> beginValue = wait(
-				    tr2.get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsBeginRange.begin)));
-				state Version appliedVersion =
+				Optional<Value> beginValue = co_await tr2.get(
+				    task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsBeginRange.begin));
+				Version appliedVersion =
 				    beginValue.present() ? BinaryReader::fromStringRef<Version>(beginValue.get(), Unversioned()) : -1;
-				Optional<Value> endValue = wait(
-				    tr2.get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin)));
+				Optional<Value> endValue = co_await tr2.get(
+				    task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin));
 				Version endVersion =
 				    endValue.present() ? BinaryReader::fromStringRef<Version>(endValue.get(), Unversioned()) : -1;
 
 				//TraceEvent("DBA_FinishedFullBackup").detail("Applied", appliedVersion).detail("EndVer", endVersion);
 				if (appliedVersion < endVersion) {
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 					task->params[FinishedFullBackupTaskFunc::keyInsertTask] = StringRef();
-					return Void();
+					co_return;
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr2.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr2.onError(err);
 			}
 		}
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(taskBucket->src));
-		state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
-		state Key destUidValue = task->params[BackupAgentBase::destUid];
-		state Version backupUid =
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(taskBucket->src));
+		Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
+		Key destUidValue = task->params[BackupAgentBase::destUid];
+		Version backupUid =
 		    BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned());
 
-		loop {
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
+				Optional<Value> v = co_await tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId));
 				if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >
 				                       BinaryReader::fromStringRef<Version>(
 				                           task->params[DatabaseBackupAgent::keyFolderId], Unversioned()))
-					return Void();
+					co_return;
 
-				wait(eraseLogData(tr, logUidValue, destUidValue, Optional<Version>(), CheckBackupUID::True, backupUid));
-				wait(tr->commit());
-				return Void();
+				co_await eraseLogData(
+				    tr, logUidValue, destUidValue, Optional<Version>(), CheckBackupUID::True, backupUid);
+				co_await tr->commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(FinishedFullBackupTaskFunc::name, FinishedFullBackupTaskFunc::version, doneKey);
 
 		copyDefaultParameters(parentTask, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		state Subspace states = Subspace(databaseBackupPrefixRange.begin)
-		                            .get(BackupAgentBase::keyStates)
-		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		Subspace states = Subspace(databaseBackupPrefixRange.begin)
+		                      .get(BackupAgentBase::keyStates)
+		                      .get(task->params[BackupAgentBase::keyConfigLogUid]);
 
 		if (task->params.find(FinishedFullBackupTaskFunc::keyInsertTask) != task->params.end()) {
-			state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-			wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone))));
-			wait(taskBucket->finish(tr, task));
-			return Void();
+			Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+			co_await FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone));
+			co_await taskBucket->finish(tr, task);
+			co_return;
 		}
 
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
@@ -1275,8 +1293,7 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase {
 		tr->set(states.pack(DatabaseBackupAgent::keyStateStatus),
 		        StringRef(BackupAgentBase::getStateText(EBackupState::STATE_COMPLETED)));
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
 	}
 
 	Future<Void> execute(Database cx,
@@ -1300,41 +1317,41 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		state Subspace states = Subspace(databaseBackupPrefixRange.begin)
-		                            .get(BackupAgentBase::keyStates)
-		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		wait(checkTaskVersion(tr, task, CopyDiffLogsTaskFunc::name, CopyDiffLogsTaskFunc::version));
-
-		state Version beginVersion =
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		Subspace states = Subspace(databaseBackupPrefixRange.begin)
+		                      .get(BackupAgentBase::keyStates)
+		                      .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		co_await checkTaskVersion(tr, task, CopyDiffLogsTaskFunc::name, CopyDiffLogsTaskFunc::version);
+
+		Version beginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
-		state Version prevBeginVersion =
+		Version prevBeginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyPrevBeginVersion], Unversioned());
-		state Future<Optional<Value>> fStopWhenDone = tr->get(conf.pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey));
+		Future<Optional<Value>> fStopWhenDone = tr->get(conf.pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey));
 
 		Transaction srcTr(taskBucket->src);
 		srcTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Version endVersion = wait(srcTr.getReadVersion());
+		Version endVersion = co_await srcTr.getReadVersion();
 
-		state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
 		if (endVersion <= beginVersion) {
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			wait(success(CopyDiffLogsTaskFunc::addTask(
-			    tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone))));
-			wait(taskBucket->finish(tr, task));
-			return Void();
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			co_await CopyDiffLogsTaskFunc::addTask(
+			    tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::signal(onDone));
+			co_await taskBucket->finish(tr, task);
+			co_return;
 		}
 
 		tr->set(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin),
 		        BinaryWriter::toValue(beginVersion, Unversioned()));
-		Optional<Value> stopWhenDone = wait(fStopWhenDone);
+		Optional<Value> stopWhenDone = co_await fStopWhenDone;
 
 		if (endVersion - beginVersion > deterministicRandom()->randomInt64(0, CLIENT_KNOBS->BACKUP_VERSION_DELAY)) {
 			TraceEvent("DBA_CopyDiffLogs")
@@ -1348,7 +1365,7 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 		        BinaryWriter::toValue(beginVersion, Unversioned()));
 
 		if (!stopWhenDone.present()) {
-			state Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
+			Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
 			std::vector<Future<Key>> addTaskVector;
 			addTaskVector.push_back(CopyDiffLogsTaskFunc::addTask(
 			    tr, taskBucket, task, beginVersion, endVersion, TaskCompletionKey::signal(onDone), allPartsDone));
@@ -1371,21 +1388,20 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 				    tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(allPartsDone)));
 			}
 
-			wait(waitForAll(addTaskVector) && taskBucket->finish(tr, task));
+			co_await (waitForAll(addTaskVector) && taskBucket->finish(tr, task));
 		} else {
-			wait(onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
+			co_await (onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version prevBeginVersion,
-	                                 Version beginVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version prevBeginVersion,
+	                           Version beginVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(CopyDiffLogsTaskFunc::name, CopyDiffLogsTaskFunc::version, doneKey, 1);
 
 		copyDefaultParameters(parentTask, task);
@@ -1394,18 +1410,18 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
 		task->params[DatabaseBackupAgent::keyPrevBeginVersion] = BinaryWriter::toValue(prevBeginVersion, Unversioned());
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -1431,13 +1447,12 @@ struct SkipOldEraseLogRangeTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
-		return Void();
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 	}
 
 	StringRef getName() const override { return name; };
@@ -1485,33 +1500,34 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> dumpData(Database cx,
-	                                   Reference<Task> task,
-	                                   PromiseStream<RCGroup> results,
-	                                   FlowLock* lock,
-	                                   Reference<TaskBucket> tb) {
-		state bool endOfStream = false;
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
-
-		state std::vector<RangeResult> nextMutations;
-		state int64_t nextMutationSize = 0;
-		loop {
+	static Future<Void> dumpData(Database cx,
+	                             Reference<Task> task,
+	                             PromiseStream<RCGroup> results,
+	                             FlowLock* lock,
+	                             Reference<TaskBucket> tb) {
+		bool endOfStream = false;
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
+
+		std::vector<RangeResult> nextMutations;
+		int64_t nextMutationSize = 0;
+		while (true) {
+			Error err;
 			try {
 				if (endOfStream && !nextMutationSize) {
-					return Void();
+					co_return;
 				}
 
-				state std::vector<RangeResult> mutations = std::move(nextMutations);
-				state int64_t mutationSize = nextMutationSize;
+				std::vector<RangeResult> mutations = std::move(nextMutations);
+				int64_t mutationSize = nextMutationSize;
 				nextMutations = std::vector<RangeResult>();
 				nextMutationSize = 0;
 
 				if (!endOfStream) {
-					loop {
+					while (true) {
 						try {
-							RCGroup group = waitNext(results.getFuture());
+							RCGroup group = co_await results.getFuture();
 							lock->release(group.items.expectedSize());
 
 							int vecSize = group.items.expectedSize();
@@ -1525,7 +1541,7 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 							mutations.push_back(group.items);
 							mutationSize += vecSize;
 						} catch (Error& e) {
-							state Error error = e;
+							Error error = e;
 							if (e.code() == error_code_end_of_stream) {
 								endOfStream = true;
 								break;
@@ -1536,19 +1552,21 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 					}
 				}
 
-				state Transaction tr(cx);
+				Transaction tr(cx);
 
-				loop {
+				while (true) {
+					Error err;
+					bool hasErr = false;
 					try {
 						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 						tr.trState->options.sizeLimit = 2 * CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
-						wait(checkDatabaseLock(&tr,
-						                       BinaryReader::fromStringRef<UID>(
-						                           task->params[BackupAgentBase::keyConfigLogUid], Unversioned())));
-						state int64_t bytesSet = 0;
+						co_await checkDatabaseLock(&tr,
+						                           BinaryReader::fromStringRef<UID>(
+						                               task->params[BackupAgentBase::keyConfigLogUid], Unversioned()));
+						int64_t bytesSet = 0;
 
 						bool first = true;
-						for (auto m : mutations) {
+						for (const auto& m : mutations) {
 							for (auto kv : m) {
 								if (first) {
 									tr.addReadConflictRange(singleKeyRange(kv.key));
@@ -1561,54 +1579,58 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 							}
 						}
 
-						wait(tr.commit());
+						co_await tr.commit();
 						Params.bytesWritten().set(task, Params.bytesWritten().getOrDefault(task) + bytesSet);
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
+						hasErr = true;
+					}
+					if (hasErr) {
+						co_await tr.onError(err);
 					}
 				}
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled || e.code() == error_code_backup_error)
-					throw e;
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled || err.code() == error_code_backup_error)
+				throw err;
 
-				state Error err = e;
-				wait(logError(cx,
-				              Subspace(databaseBackupPrefixRange.begin)
-				                  .get(BackupAgentBase::keyErrors)
-				                  .pack(task->params[BackupAgentBase::keyConfigLogUid]),
-				              format("ERROR: Failed to dump mutations because of error %s", err.what())));
+			co_await logError(cx,
+			                  Subspace(databaseBackupPrefixRange.begin)
+			                      .get(BackupAgentBase::keyErrors)
+			                      .pack(task->params[BackupAgentBase::keyConfigLogUid]),
+			                  format("ERROR: Failed to dump mutations because of error %s", err.what()));
 
-				throw err;
-			}
+			throw err;
 		}
 	}
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
 
-		wait(checkTaskVersion(cx, task, OldCopyLogRangeTaskFunc::name, OldCopyLogRangeTaskFunc::version));
+		co_await checkTaskVersion(cx, task, OldCopyLogRangeTaskFunc::name, OldCopyLogRangeTaskFunc::version);
 
-		state Version beginVersion =
+		Version beginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
-		state Version endVersion =
+		Version endVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyEndVersion], Unversioned());
-		state Version newEndVersion =
-		    std::min(endVersion,
-		             (((beginVersion - 1) / CLIENT_KNOBS->BACKUP_BLOCK_SIZE) + 2 +
-		              (g_network->isSimulated() ? CLIENT_KNOBS->BACKUP_SIM_COPY_LOG_RANGES : 0)) *
-		                 CLIENT_KNOBS->BACKUP_BLOCK_SIZE);
-
-		state Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion,
-		                                                               newEndVersion,
-		                                                               task->params[BackupAgentBase::keyConfigLogUid],
-		                                                               CLIENT_KNOBS->BACKUP_BLOCK_SIZE);
-		state std::vector<PromiseStream<RCGroup>> results;
-		state std::vector<Future<Void>> rc;
-		state std::vector<Future<Void>> dump;
+		Version newEndVersion = std::min(endVersion,
+		                                 (((beginVersion - 1) / CLIENT_KNOBS->BACKUP_BLOCK_SIZE) + 2 +
+		                                  (g_network->isSimulated() ? CLIENT_KNOBS->BACKUP_SIM_COPY_LOG_RANGES : 0)) *
+		                                     CLIENT_KNOBS->BACKUP_BLOCK_SIZE);
+
+		Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion,
+		                                                         newEndVersion,
+		                                                         task->params[BackupAgentBase::keyConfigLogUid],
+		                                                         CLIENT_KNOBS->BACKUP_BLOCK_SIZE);
+		std::vector<PromiseStream<RCGroup>> results;
+		std::vector<Future<Void>> rc;
+		std::vector<Future<Void>> dump;
 
 		for (int i = 0; i < ranges.size(); ++i) {
 			results.push_back(PromiseStream<RCGroup>());
@@ -1625,24 +1647,22 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 			dump.push_back(dumpData(cx, task, results[i], lock.getPtr(), taskBucket));
 		}
 
-		wait(waitForAll(dump));
+		co_await waitForAll(dump);
 
 		if (newEndVersion < endVersion) {
 			task->params[OldCopyLogRangeTaskFunc::keyNextBeginVersion] =
 			    BinaryWriter::toValue(newEndVersion, Unversioned());
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version beginVersion,
-	                                 Version endVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version beginVersion,
+	                           Version endVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(OldCopyLogRangeTaskFunc::name, OldCopyLogRangeTaskFunc::version, doneKey, 1);
 
 		copyDefaultParameters(parentTask, task);
@@ -1651,28 +1671,28 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 		task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 
-		state Version endVersion =
+		Version endVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyEndVersion], Unversioned());
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
 		// Get the bytesWritten parameter from task and atomically add it to the logBytesWritten() property of the DR
 		// config.
@@ -1681,16 +1701,14 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
 		config.logBytesWritten().atomicOp(tr, bytesWritten, MutationRef::AddValue);
 
 		if (task->params.find(OldCopyLogRangeTaskFunc::keyNextBeginVersion) != task->params.end()) {
-			state Version nextVersion = BinaryReader::fromStringRef<Version>(
+			Version nextVersion = BinaryReader::fromStringRef<Version>(
 			    task->params[OldCopyLogRangeTaskFunc::keyNextBeginVersion], Unversioned());
-			wait(success(OldCopyLogRangeTaskFunc::addTask(
-			         tr, taskBucket, task, nextVersion, endVersion, TaskCompletionKey::signal(taskFuture->key))) &&
-			     taskBucket->finish(tr, task));
+			co_await (success(OldCopyLogRangeTaskFunc::addTask(
+			              tr, taskBucket, task, nextVersion, endVersion, TaskCompletionKey::signal(taskFuture->key))) &&
+			          taskBucket->finish(tr, task));
 		} else {
-			wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
+			co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 		}
-
-		return Void();
 	}
 };
 StringRef OldCopyLogRangeTaskFunc::name = "db_copy_log_range"_sr;
@@ -1701,69 +1719,72 @@ struct AbortOldBackupTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state DatabaseBackupAgent srcDrAgent(taskBucket->src);
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Key tagNameKey;
-
-		loop {
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		DatabaseBackupAgent srcDrAgent(taskBucket->src);
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Key tagNameKey;
+
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				Key tagPath = srcDrAgent.states.get(task->params[DatabaseBackupAgent::keyConfigLogUid])
 				                  .pack(BackupAgentBase::keyConfigBackupTag);
-				Optional<Key> tagName = wait(tr->get(tagPath));
+				Optional<Key> tagName = co_await tr->get(tagPath);
 				if (!tagName.present()) {
-					return Void();
+					co_return;
 				}
 
 				tagNameKey = tagName.get();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 
 		TraceEvent("DBA_AbortOldBackup").detail("TagName", tagNameKey.printable());
-		wait(srcDrAgent.abortBackup(cx, tagNameKey, PartialBackup::False, AbortOldBackup::True));
-
-		return Void();
+		co_await srcDrAgent.abortBackup(cx, tagNameKey, PartialBackup::False, AbortOldBackup::True);
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		wait(taskBucket->finish(tr, task));
-		return Void();
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		co_await taskBucket->finish(tr, task);
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(AbortOldBackupTaskFunc::name, AbortOldBackupTaskFunc::version, doneKey, 1);
 
 		copyDefaultParameters(parentTask, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -1795,58 +1816,65 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
-		state Subspace sourceStates =
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
+		Subspace sourceStates =
 		    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
-		state Subspace config =
-		    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue);
-		wait(checkTaskVersion(cx, task, CopyDiffLogsUpgradeTaskFunc::name, CopyDiffLogsUpgradeTaskFunc::version));
+		Subspace config = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue);
+		co_await checkTaskVersion(cx, task, CopyDiffLogsUpgradeTaskFunc::name, CopyDiffLogsUpgradeTaskFunc::version);
 
 		// Retrieve backupRanges
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		loop {
+		Standalone<VectorRef<KeyRangeRef>> backupRanges;
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				Future<Void> verified = taskBucket->keepRunning(tr, task);
-				wait(verified);
+				co_await verified;
 
-				Optional<Key> backupKeysPacked = wait(tr->get(config.pack(BackupAgentBase::keyConfigBackupRanges)));
+				Optional<Key> backupKeysPacked = co_await tr->get(config.pack(BackupAgentBase::keyConfigBackupRanges));
 				if (!backupKeysPacked.present()) {
-					return Void();
+					co_return;
 				}
 
 				BinaryReader br(backupKeysPacked.get(), IncludeVersion());
 				br >> backupRanges;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 
 		// Set destUidValue and versionKey on src side
-		state Key destUidValue(logUidValue);
-		state Reference<ReadYourWritesTransaction> srcTr(new ReadYourWritesTransaction(taskBucket->src));
-		loop {
+		Key destUidValue(logUidValue);
+		Reference<ReadYourWritesTransaction> srcTr(new ReadYourWritesTransaction(taskBucket->src));
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				srcTr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				srcTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-				state Optional<Value> v = wait(srcTr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
+				Optional<Value> v = co_await srcTr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId));
 				if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >
 				                       BinaryReader::fromStringRef<Version>(
 				                           task->params[DatabaseBackupAgent::keyFolderId], Unversioned())) {
-					return Void();
+					co_return;
 				}
 
 				if (backupRanges.size() == 1 || isDefaultBackup(backupRanges)) {
-					RangeResult existingDestUidValues = wait(srcTr->getRange(
-					    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
+					RangeResult existingDestUidValues = co_await srcTr->getRange(
+					    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY);
 					bool found = false;
 					KeyRangeRef targetRange =
 					    (backupRanges.size() == 1) ? backupRanges[0] : getDefaultBackupSharedRange();
@@ -1856,7 +1884,7 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 						if (uidRange == targetRange) {
 							if (destUidValue != it.value) {
 								// existing backup/DR is running
-								return Void();
+								co_return;
 							} else {
 								// due to unknown commit result
 								found = true;
@@ -1876,29 +1904,31 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 
 				Key versionKey = logUidValue.withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix);
 				srcTr->set(versionKey, task->params[DatabaseBackupAgent::keyBeginVersion]);
-				wait(srcTr->commit());
+				co_await srcTr->commit();
 				break;
 			} catch (Error& e) {
-				wait(srcTr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await srcTr->onError(err);
 			}
 		}
 
 		task->params[BackupAgentBase::destUid] = destUidValue;
 		ASSERT(destUidValue == logUidValue);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		wait(checkTaskVersion(tr, task, CopyDiffLogsUpgradeTaskFunc::name, CopyDiffLogsUpgradeTaskFunc::version));
-		state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		co_await checkTaskVersion(tr, task, CopyDiffLogsUpgradeTaskFunc::name, CopyDiffLogsUpgradeTaskFunc::version);
+		Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
-		if (task->params[BackupAgentBase::destUid].size() == 0) {
+		if (task->params[BackupAgentBase::destUid].empty()) {
 			TraceEvent("DBA_CopyDiffLogsUpgradeTaskFuncAbortInUpgrade").log();
-			wait(success(AbortOldBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone))));
+			co_await AbortOldBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::signal(onDone));
 		} else {
 			Version beginVersion =
 			    BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
@@ -1908,12 +1938,11 @@ struct CopyDiffLogsUpgradeTaskFunc : TaskFuncBase {
 			tr->set(config.pack(BackupAgentBase::destUid), task->params[BackupAgentBase::destUid]);
 			tr->set(config.pack(BackupAgentBase::keyDrVersion),
 			        BinaryWriter::toValue(DatabaseBackupAgent::LATEST_DR_VERSION, Unversioned()));
-			wait(success(CopyDiffLogsTaskFunc::addTask(
-			    tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::signal(onDone))));
+			co_await CopyDiffLogsTaskFunc::addTask(
+			    tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::signal(onDone));
 		}
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
 	}
 
 	StringRef getName() const override { return name; };
@@ -1938,16 +1967,18 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin)
-		                                  .get(BackupAgentBase::keySourceStates)
-		                                  .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		wait(checkTaskVersion(cx, task, BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version));
-		state Transaction tr(taskBucket->src);
-		loop {
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin)
+		                            .get(BackupAgentBase::keySourceStates)
+		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		co_await checkTaskVersion(cx, task, BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version);
+		Transaction tr(taskBucket->src);
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.addReadConflictRange(singleKeyRange(sourceStates.pack(DatabaseBackupAgent::keyStateStatus)));
@@ -1957,41 +1988,45 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
 				Key versionKey = task->params[DatabaseBackupAgent::keyConfigLogUid]
 				                     .withPrefix(task->params[BackupAgentBase::destUid])
 				                     .withPrefix(backupLatestVersionsPrefix);
-				Optional<Key> prevBeginVersion = wait(tr.get(versionKey));
+				Optional<Key> prevBeginVersion = co_await tr.get(versionKey);
 				if (!prevBeginVersion.present()) {
-					return Void();
+					co_return;
 				}
 
 				task->params[DatabaseBackupAgent::keyPrevBeginVersion] = prevBeginVersion.get();
 
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Subspace conf = Subspace(databaseBackupPrefixRange.begin)
-		                          .get(BackupAgentBase::keyConfig)
-		                          .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		state Subspace states = Subspace(databaseBackupPrefixRange.begin)
-		                            .get(BackupAgentBase::keyStates)
-		                            .get(task->params[BackupAgentBase::keyConfigLogUid]);
-		wait(checkTaskVersion(tr, task, BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version));
-
-		state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-
-		state Optional<Value> stopValue = wait(tr->get(states.pack(DatabaseBackupAgent::keyStateStop)));
-		state Version restoreVersion =
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Subspace conf = Subspace(databaseBackupPrefixRange.begin)
+		                    .get(BackupAgentBase::keyConfig)
+		                    .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		Subspace states = Subspace(databaseBackupPrefixRange.begin)
+		                      .get(BackupAgentBase::keyStates)
+		                      .get(task->params[BackupAgentBase::keyConfigLogUid]);
+		co_await checkTaskVersion(tr, task, BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version);
+
+		Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+
+		Optional<Value> stopValue = co_await tr->get(states.pack(DatabaseBackupAgent::keyStateStop));
+		Version restoreVersion =
 		    stopValue.present() ? BinaryReader::fromStringRef<Version>(stopValue.get(), Unversioned()) : -1;
 
-		state Optional<Value> stopWhenDone = wait(tr->get(conf.pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey)));
-		state Reference<TaskFuture> allPartsDone;
+		Optional<Value> stopWhenDone = co_await tr->get(conf.pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey));
+		Reference<TaskFuture> allPartsDone;
 
 		TraceEvent("DBA_Complete")
 		    .detail("RestoreVersion", restoreVersion)
@@ -2001,7 +2036,7 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
 		// Start the complete task, if differential is not enabled
 		if (stopWhenDone.present()) {
 			// After the Backup completes, clear the backup subspace and update the status
-			wait(success(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal())));
+			co_await FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal());
 		} else { // Start the writing of logs, if differential
 			tr->set(states.pack(DatabaseBackupAgent::keyStateStatus),
 			        StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING_DIFFERENTIAL)));
@@ -2010,41 +2045,40 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
 
 			Version prevBeginVersion = BinaryReader::fromStringRef<Version>(
 			    task->params[DatabaseBackupAgent::keyPrevBeginVersion], Unversioned());
-			wait(success(CopyDiffLogsTaskFunc::addTask(
-			    tr, taskBucket, task, prevBeginVersion, restoreVersion, TaskCompletionKey::joinWith(allPartsDone))));
+			co_await CopyDiffLogsTaskFunc::addTask(
+			    tr, taskBucket, task, prevBeginVersion, restoreVersion, TaskCompletionKey::joinWith(allPartsDone));
 
 			// After the Backup completes, clear the backup subspace and update the status
-			wait(success(FinishedFullBackupTaskFunc::addTask(
-			    tr, taskBucket, task, TaskCompletionKey::noSignal(), allPartsDone)));
+			co_await FinishedFullBackupTaskFunc::addTask(
+			    tr, taskBucket, task, TaskCompletionKey::noSignal(), allPartsDone);
 		}
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(BackupRestorableTaskFunc::name, BackupRestorableTaskFunc::version, doneKey);
 
 		copyDefaultParameters(parentTask, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           parentTask->params[Task::reservedTaskParamValidKey],
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           parentTask->params[Task::reservedTaskParamValidKey],
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              parentTask->params[Task::reservedTaskParamValidKey],
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               parentTask->params[Task::reservedTaskParamValidKey],
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -2069,31 +2103,32 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 	static StringRef name;
 	static constexpr uint32_t version = 1;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
-		state Subspace sourceStates =
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Key logUidValue = task->params[DatabaseBackupAgent::keyConfigLogUid];
+		Subspace sourceStates =
 		    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
-		wait(checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version));
-		state Key destUidValue(logUidValue);
+		co_await checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version);
+		Key destUidValue(logUidValue);
 
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges =
-		    BinaryReader::fromStringRef<Standalone<VectorRef<KeyRangeRef>>>(
-		        task->params[DatabaseBackupAgent::keyConfigBackupRanges], IncludeVersion());
-		state Key beginVersionKey;
+		auto backupRanges = BinaryReader::fromStringRef<Standalone<VectorRef<KeyRangeRef>>>(
+		    task->params[DatabaseBackupAgent::keyConfigBackupRanges], IncludeVersion());
+		Key beginVersionKey;
 
-		state Reference<ReadYourWritesTransaction> srcTr(new ReadYourWritesTransaction(taskBucket->src));
-		loop {
+		Reference<ReadYourWritesTransaction> srcTr(new ReadYourWritesTransaction(taskBucket->src));
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				srcTr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				srcTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 				// Initialize destUid
 				if (backupRanges.size() == 1 || isDefaultBackup(backupRanges)) {
-					RangeResult existingDestUidValues = wait(srcTr->getRange(
-					    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
+					RangeResult existingDestUidValues = co_await srcTr->getRange(
+					    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY);
 					KeyRangeRef targetRange =
 					    (backupRanges.size() == 1) ? backupRanges[0] : getDefaultBackupSharedRange();
 					bool found = false;
@@ -2117,39 +2152,45 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 					}
 				}
 
-				Version bVersion = wait(srcTr->getReadVersion());
+				Version bVersion = co_await srcTr->getReadVersion();
 				beginVersionKey = BinaryWriter::toValue(bVersion, Unversioned());
 
-				state Key versionKey = logUidValue.withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix);
-				Optional<Key> versionRecord = wait(srcTr->get(versionKey));
+				Key versionKey = logUidValue.withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix);
+				Optional<Key> versionRecord = co_await srcTr->get(versionKey);
 				if (!versionRecord.present()) {
 					srcTr->set(versionKey, beginVersionKey);
 				}
 
 				task->params[BackupAgentBase::destUid] = destUidValue;
 
-				wait(srcTr->commit());
+				co_await srcTr->commit();
 				break;
 			} catch (Error& e) {
-				wait(srcTr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await srcTr->onError(err);
 			}
 		}
 
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				state Future<Void> verified = taskBucket->keepRunning(tr, task);
-				wait(verified);
+				Future<Void> verified = taskBucket->keepRunning(tr, task);
+				co_await verified;
 
 				// Set destUid at destination side
-				state Subspace config =
+				Subspace config =
 				    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue);
 				tr->set(config.pack(BackupAgentBase::destUid), task->params[BackupAgentBase::destUid]);
 
 				// Use existing beginVersion if we already have one
-				Optional<Key> backupStartVersion = wait(tr->get(config.pack(BackupAgentBase::backupStartVersion)));
+				Optional<Key> backupStartVersion = co_await tr->get(config.pack(BackupAgentBase::backupStartVersion));
 				if (backupStartVersion.present()) {
 					beginVersionKey = backupStartVersion.get();
 				} else {
@@ -2158,26 +2199,32 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 
 				task->params[BackupAgentBase::keyBeginVersion] = beginVersionKey;
 
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				TraceEvent("SetDestUidOrBeginVersionError").errorUnsuppressed(e);
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				TraceEvent("SetDestUidOrBeginVersionError").errorUnsuppressed(err);
+				co_await tr->onError(err);
 			}
 		}
 
-		state Reference<ReadYourWritesTransaction> srcTr2(new ReadYourWritesTransaction(taskBucket->src));
-		loop {
+		Reference<ReadYourWritesTransaction> srcTr2(new ReadYourWritesTransaction(taskBucket->src));
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				srcTr2->setOption(FDBTransactionOptions::LOCK_AWARE);
 				srcTr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-				state Optional<Value> v = wait(srcTr2->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
+				Optional<Value> v = co_await srcTr2->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId));
 
 				if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >=
 				                       BinaryReader::fromStringRef<Version>(
 				                           task->params[DatabaseBackupAgent::keyFolderId], Unversioned()))
-					return Void();
+					co_return;
 
 				srcTr2->set(Subspace(databaseBackupPrefixRange.begin)
 				                .get(BackupAgentBase::keySourceTagName)
@@ -2188,7 +2235,7 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 				srcTr2->set(sourceStates.pack(DatabaseBackupAgent::keyStateStatus),
 				            StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING)));
 
-				state Key destPath = destUidValue.withPrefix(backupLogKeys.begin);
+				Key destPath = destUidValue.withPrefix(backupLogKeys.begin);
 				// Start logging the mutations for the specified ranges of the tag
 				for (auto& backupRange : backupRanges) {
 					srcTr2->set(logRangesEncodeKey(backupRange.begin,
@@ -2196,46 +2243,51 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 					            logRangesEncodeValue(backupRange.end, destPath));
 				}
 
-				wait(srcTr2->commit());
+				co_await srcTr2->commit();
 				break;
 			} catch (Error& e) {
-				wait(srcTr2->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await srcTr2->onError(err);
 			}
 		}
 
-		state Reference<ReadYourWritesTransaction> srcTr3(new ReadYourWritesTransaction(taskBucket->src));
-		loop {
+		Reference<ReadYourWritesTransaction> srcTr3(new ReadYourWritesTransaction(taskBucket->src));
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				srcTr3->setOption(FDBTransactionOptions::LOCK_AWARE);
 				srcTr3->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 				srcTr3->atomicOp(metadataVersionKey, metadataVersionRequiredValue, MutationRef::SetVersionstampedValue);
 
-				wait(srcTr3->commit());
+				co_await srcTr3->commit();
 				break;
 			} catch (Error& e) {
-				wait(srcTr3->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await srcTr3->onError(err);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Key logUidValue = task->params[BackupAgentBase::keyConfigLogUid];
-		state Subspace states =
-		    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(logUidValue);
-		state Subspace config =
-		    Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue);
-
-		state Version beginVersion =
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Key logUidValue = task->params[BackupAgentBase::keyConfigLogUid];
+		Subspace states = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyStates).get(logUidValue);
+		Subspace config = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyConfig).get(logUidValue);
+
+		Version beginVersion =
 		    BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyBeginVersion], Unversioned());
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges =
-		    BinaryReader::fromStringRef<Standalone<VectorRef<KeyRangeRef>>>(
-		        task->params[DatabaseBackupAgent::keyConfigBackupRanges], IncludeVersion());
+		auto backupRanges = BinaryReader::fromStringRef<Standalone<VectorRef<KeyRangeRef>>>(
+		    task->params[DatabaseBackupAgent::keyConfigBackupRanges], IncludeVersion());
 
 		tr->set(logUidValue.withPrefix(applyMutationsBeginRange.begin),
 		        BinaryWriter::toValue(beginVersion, Unversioned()));
@@ -2244,40 +2296,39 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 		tr->set(states.pack(DatabaseBackupAgent::keyStateStatus),
 		        StringRef(BackupAgentBase::getStateText(EBackupState::STATE_RUNNING)));
 
-		state Reference<TaskFuture> kvBackupRangeComplete = futureBucket->future(tr);
-		state Reference<TaskFuture> kvBackupComplete = futureBucket->future(tr);
-		state int rangeCount = 0;
+		Reference<TaskFuture> kvBackupRangeComplete = futureBucket->future(tr);
+		Reference<TaskFuture> kvBackupComplete = futureBucket->future(tr);
+		int rangeCount = 0;
 
 		if (task->params[DatabaseBackupAgent::keyDatabasesInSync] != std::string("t")) {
 			for (; rangeCount < backupRanges.size(); ++rangeCount) {
-				wait(success(BackupRangeTaskFunc::addTask(tr,
-				                                          taskBucket,
-				                                          task,
-				                                          backupRanges[rangeCount].begin,
-				                                          backupRanges[rangeCount].end,
-				                                          TaskCompletionKey::joinWith(kvBackupRangeComplete))));
+				co_await BackupRangeTaskFunc::addTask(tr,
+				                                      taskBucket,
+				                                      task,
+				                                      backupRanges[rangeCount].begin,
+				                                      backupRanges[rangeCount].end,
+				                                      TaskCompletionKey::joinWith(kvBackupRangeComplete));
 			}
 		} else {
 			kvBackupRangeComplete->set(tr, taskBucket);
 		}
 
 		// After the BackupRangeTask completes, set the stop key which will stop the BackupLogsTask
-		wait(success(FinishFullBackupTaskFunc::addTask(
-		    tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupRangeComplete)));
+		co_await FinishFullBackupTaskFunc::addTask(
+		    tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupRangeComplete);
 
 		// Backup the logs which will create BackupLogRange tasks
-		wait(success(CopyLogsTaskFunc::addTask(
-		    tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::joinWith(kvBackupComplete))));
+		co_await CopyLogsTaskFunc::addTask(
+		    tr, taskBucket, task, 0, beginVersion, TaskCompletionKey::joinWith(kvBackupComplete));
 
 		// After the Backup completes, clear the backup subspace and update the status
-		wait(success(
-		    BackupRestorableTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupComplete)));
+		co_await BackupRestorableTaskFunc::addTask(
+		    tr, taskBucket, task, TaskCompletionKey::noSignal(), kvBackupComplete);
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
 	}
 
-	ACTOR static Future<Key> addTask(
+	static Future<Key> addTask(
 	    Reference<ReadYourWritesTransaction> tr,
 	    Reference<TaskBucket> taskBucket,
 	    Key logUid,
@@ -2289,7 +2340,7 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 	    TaskCompletionKey completionKey,
 	    Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
 	    DatabaseBackupAgent::PreBackupAction backupAction = DatabaseBackupAgent::PreBackupAction::VERIFY) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 		auto task = makeReference<Task>(StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version, doneKey);
 
 		task->params[BackupAgentBase::keyFolderId] = backupUid;
@@ -2302,24 +2353,24 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
 		    backupAction == DatabaseBackupAgent::PreBackupAction::NONE ? "t"_sr : "f"_sr;
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr,
-			                           task,
-			                           Subspace(databaseBackupPrefixRange.begin)
-			                               .get(BackupAgentBase::keyConfig)
-			                               .get(logUid)
-			                               .pack(BackupAgentBase::keyFolderId),
-			                           task->params[BackupAgentBase::keyFolderId]);
-		}
-
-		wait(waitFor->onSetAddTask(tr,
-		                           taskBucket,
-		                           task,
-		                           Subspace(databaseBackupPrefixRange.begin)
-		                               .get(BackupAgentBase::keyConfig)
-		                               .get(logUid)
-		                               .pack(BackupAgentBase::keyFolderId),
-		                           task->params[BackupAgentBase::keyFolderId]));
-		return "OnSetAddTask"_sr;
+			co_return taskBucket->addTask(tr,
+			                              task,
+			                              Subspace(databaseBackupPrefixRange.begin)
+			                                  .get(BackupAgentBase::keyConfig)
+			                                  .get(logUid)
+			                                  .pack(BackupAgentBase::keyFolderId),
+			                              task->params[BackupAgentBase::keyFolderId]);
+		}
+
+		co_await waitFor->onSetAddTask(tr,
+		                               taskBucket,
+		                               task,
+		                               Subspace(databaseBackupPrefixRange.begin)
+		                                   .get(BackupAgentBase::keyConfig)
+		                                   .get(logUid)
+		                                   .pack(BackupAgentBase::keyFolderId),
+		                               task->params[BackupAgentBase::keyFolderId]);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -2349,7 +2400,7 @@ std::set<std::string> getDRAgentsIds(StatusObjectReader statusObj, const char* c
 		StatusObjectReader instances;
 		std::string path = format("%s.instances", context);
 		if (statusObjLayers.tryGet(path, instances)) {
-			for (auto itr : instances.obj()) {
+			for (const auto& itr : instances.obj()) {
 				drBackupAgents.insert(itr.first);
 			}
 		}
@@ -2434,23 +2485,25 @@ void checkAtomicSwitchOverConfig(StatusObjectReader srcStatus, StatusObjectReade
 
 class DatabaseBackupAgentImpl {
 public:
-	ACTOR static Future<Void> waitUpgradeToLatestDrVersion(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
-		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
-		state Key drVersionKey = backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
-		                             .pack(DatabaseBackupAgent::keyDrVersion);
+	static Future<Void> waitUpgradeToLatestDrVersion(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
+		UID logUid = co_await backupAgent->getLogUid(cx, tagName);
+		Key drVersionKey = backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
+		                       .pack(DatabaseBackupAgent::keyDrVersion);
 
 		TraceEvent("DRU_WatchLatestDrVersion")
 		    .detail("DrVersionKey", drVersionKey.printable())
 		    .detail("LogUid", BinaryWriter::toValue(logUid, Unversioned()).printable());
 
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-			loop {
+			while (true) {
+				Error err;
+				bool hasErr = false;
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-					Optional<Value> drVersion = wait(tr->get(drVersionKey));
+					Optional<Value> drVersion = co_await tr->get(drVersionKey);
 
 					TraceEvent("DRU_VersionCheck")
 					    .detail("Current",
@@ -2460,97 +2513,113 @@ class DatabaseBackupAgentImpl {
 					    .detail("LogUid", BinaryWriter::toValue(logUid, Unversioned()).printable());
 					if (drVersion.present() && BinaryReader::fromStringRef<int>(drVersion.get(), Unversioned()) ==
 					                               DatabaseBackupAgent::LATEST_DR_VERSION) {
-						return Void();
+						co_return;
 					}
 
-					state Future<Void> watchDrVersionFuture = tr->watch(drVersionKey);
-					wait(tr->commit());
-					wait(watchDrVersionFuture);
+					Future<Void> watchDrVersionFuture = tr->watch(drVersionKey);
+					co_await tr->commit();
+					co_await watchDrVersionFuture;
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
+					hasErr = true;
+				}
+				if (hasErr) {
+					co_await tr->onError(err);
 				}
 			}
 		}
 	}
 
 	// This method will return the final status of the backup
-	ACTOR static Future<EBackupState> waitBackup(DatabaseBackupAgent* backupAgent,
-	                                             Database cx,
-	                                             Key tagName,
-	                                             StopWhenDone stopWhenDone) {
-		state std::string backTrace;
-		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
-		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
-		                          .pack(DatabaseBackupAgent::keyStateStatus);
-
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	static Future<EBackupState> waitBackup(DatabaseBackupAgent* backupAgent,
+	                                       Database cx,
+	                                       Key tagName,
+	                                       StopWhenDone stopWhenDone) {
+		std::string backTrace;
+		UID logUid = co_await backupAgent->getLogUid(cx, tagName);
+		Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
+		                    .pack(DatabaseBackupAgent::keyStateStatus);
+
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
+			Error err;
+			bool hasErr = false;
 			try {
-				state EBackupState status = wait(backupAgent->getStateValue(tr, logUid));
+				EBackupState status = co_await backupAgent->getStateValue(tr, logUid);
 
 				// Break, if no longer runnable
 				if (!DatabaseBackupAgent::isRunnable(status) || EBackupState::STATE_PARTIALLY_ABORTED == status) {
-					return status;
+					co_return status;
 				}
 
 				// Break, if in differential mode (restorable) and stopWhenDone is not enabled
 				if ((!stopWhenDone) && (EBackupState::STATE_RUNNING_DIFFERENTIAL == status)) {
-					return status;
+					co_return status;
 				}
 
-				state Future<Void> watchFuture = tr->watch(statusKey);
-				wait(tr->commit());
-				wait(watchFuture);
+				Future<Void> watchFuture = tr->watch(statusKey);
+				co_await tr->commit();
+				co_await watchFuture;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 	}
 
 	// This method will return the final status of the backup
-	ACTOR static Future<EBackupState> waitSubmitted(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
-		state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
-		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
-		                          .pack(DatabaseBackupAgent::keyStateStatus);
+	static Future<EBackupState> waitSubmitted(DatabaseBackupAgent* backupAgent, Database cx, Key tagName) {
+		UID logUid = co_await backupAgent->getLogUid(cx, tagName);
+		Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
+		                    .pack(DatabaseBackupAgent::keyStateStatus);
 
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
+			Error err;
+			bool hasErr = false;
 			try {
-				state EBackupState status = wait(backupAgent->getStateValue(tr, logUid));
+				EBackupState status = co_await backupAgent->getStateValue(tr, logUid);
 
 				// Break, if no longer runnable
 				if (EBackupState::STATE_SUBMITTED != status) {
-					return status;
+					co_return status;
 				}
 
-				state Future<Void> watchFuture = tr->watch(statusKey);
-				wait(tr->commit());
-				wait(watchFuture);
+				Future<Void> watchFuture = tr->watch(statusKey);
+				co_await tr->commit();
+				co_await watchFuture;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Void> submitBackup(DatabaseBackupAgent* backupAgent,
-	                                       Reference<ReadYourWritesTransaction> tr,
-	                                       Key tagName,
-	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                       StopWhenDone stopWhenDone,
-	                                       Key addPrefix,
-	                                       Key removePrefix,
-	                                       LockDB lockDB,
-	                                       DatabaseBackupAgent::PreBackupAction backupAction) {
-		state UID logUid = deterministicRandom()->randomUniqueID();
-		state Key logUidValue = BinaryWriter::toValue(logUid, Unversioned());
-		state UID logUidCurrent = wait(backupAgent->getLogUid(tr, tagName));
+	static Future<Void> submitBackup(DatabaseBackupAgent* backupAgent,
+	                                 Reference<ReadYourWritesTransaction> tr,
+	                                 Key tagName,
+	                                 Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                                 StopWhenDone stopWhenDone,
+	                                 Key addPrefix,
+	                                 Key removePrefix,
+	                                 LockDB lockDB,
+	                                 DatabaseBackupAgent::PreBackupAction backupAction) {
+		UID logUid = deterministicRandom()->randomUniqueID();
+		Key logUidValue = BinaryWriter::toValue(logUid, Unversioned());
+		UID logUidCurrent = co_await backupAgent->getLogUid(tr, tagName);
 
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -2560,7 +2629,7 @@ class DatabaseBackupAgentImpl {
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 
 		// We will use the global status for now to ensure that multiple backups do not start place with different tags
-		state EBackupState status = wait(backupAgent->getStateValue(tr, logUidCurrent));
+		EBackupState status = co_await backupAgent->getStateValue(tr, logUidCurrent);
 
 		if (DatabaseBackupAgent::isRunnable(status)) {
 			throw backup_duplicate();
@@ -2571,11 +2640,11 @@ class DatabaseBackupAgentImpl {
 			logUidValue = BinaryWriter::toValue(logUid, Unversioned());
 		}
 
-		Optional<Key> v = wait(tr->get(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId)));
+		Optional<Key> v = co_await tr->get(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId));
 		Version uidVersion = 0;
 		if (v.present())
 			uidVersion = BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) + 1;
-		state Standalone<StringRef> backupUid = BinaryWriter::toValue(uidVersion, Unversioned());
+		Standalone<StringRef> backupUid = BinaryWriter::toValue(uidVersion, Unversioned());
 
 		KeyRangeMap<int> backupRangeSet;
 		for (auto& backupRange : backupRanges) {
@@ -2593,14 +2662,14 @@ class DatabaseBackupAgentImpl {
 
 		if (backupAction == DatabaseBackupAgent::PreBackupAction::VERIFY) {
 			// Make sure all of the ranges are empty before we backup into them.
-			state std::vector<Future<RangeResult>> backupIntoResults;
+			std::vector<Future<RangeResult>> backupIntoResults;
 			for (auto& backupRange : backupRanges) {
 				backupIntoResults.push_back(
 				    tr->getRange(backupRange.removePrefix(removePrefix).withPrefix(addPrefix), 1));
 			}
-			wait(waitForAll(backupIntoResults));
-			for (auto result : backupIntoResults) {
-				if (result.get().size() > 0) {
+			co_await waitForAll(backupIntoResults);
+			for (const auto& result : backupIntoResults) {
+				if (!result.get().empty()) {
 					// One of the ranges we will be backing up into has pre-existing data.
 					throw restore_destination_not_empty();
 				}
@@ -2640,23 +2709,23 @@ class DatabaseBackupAgentImpl {
 		}
 
 		int64_t startCount = 0;
-		state Key mapPrefix = logUidValue.withPrefix(applyMutationsKeyVersionMapRange.begin);
+		Key mapPrefix = logUidValue.withPrefix(applyMutationsKeyVersionMapRange.begin);
 		Key mapEnd = allKeys.end.withPrefix(mapPrefix);
 		tr->set(logUidValue.withPrefix(applyMutationsAddPrefixRange.begin), addPrefix);
 		tr->set(logUidValue.withPrefix(applyMutationsRemovePrefixRange.begin), removePrefix);
 		tr->set(logUidValue.withPrefix(applyMutationsKeyVersionCountRange.begin), StringRef((uint8_t*)&startCount, 8));
 		tr->clear(KeyRangeRef(mapPrefix, mapEnd));
 
-		state Version readVersion = invalidVersion;
+		Version readVersion = invalidVersion;
 		if (backupAction == DatabaseBackupAgent::PreBackupAction::NONE) {
 			Transaction readTransaction(backupAgent->taskBucket->src);
 			readTransaction.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Version _ = wait(readTransaction.getReadVersion());
+			Version _ = co_await readTransaction.getReadVersion();
 			readVersion = _;
 		}
 		tr->set(mapPrefix, BinaryWriter::toValue<Version>(readVersion, Unversioned()));
 
-		Key taskKey = wait(dbBackup::StartFullBackupTaskFunc::addTask(
+		Key taskKey = co_await dbBackup::StartFullBackupTaskFunc::addTask(
 		    tr,
 		    backupAgent->taskBucket,
 		    logUidValue,
@@ -2667,12 +2736,12 @@ class DatabaseBackupAgentImpl {
 		    tagName,
 		    TaskCompletionKey::noSignal(),
 		    Reference<TaskFuture>(),
-		    backupAction));
+		    backupAction);
 
 		if (lockDB)
-			wait(lockDatabase(tr, logUid));
+			co_await lockDatabase(tr, logUid);
 		else
-			wait(checkDatabaseLock(tr, logUid));
+			co_await checkDatabaseLock(tr, logUid);
 
 		TraceEvent("DBA_Submit")
 		    .detail("LogUid", logUid)
@@ -2681,29 +2750,26 @@ class DatabaseBackupAgentImpl {
 		    .detail("Tag", tagName)
 		    .detail("Key", backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId))
 		    .detail("MapPrefix", mapPrefix);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> unlockBackup(DatabaseBackupAgent* backupAgent,
-	                                       Reference<ReadYourWritesTransaction> tr,
-	                                       Key tagName) {
-		UID logUid = wait(backupAgent->getLogUid(tr, tagName));
-		wait(unlockDatabase(tr, logUid));
+	static Future<Void> unlockBackup(DatabaseBackupAgent* backupAgent,
+	                                 Reference<ReadYourWritesTransaction> tr,
+	                                 Key tagName) {
+		UID logUid = co_await backupAgent->getLogUid(tr, tagName);
+		co_await unlockDatabase(tr, logUid);
 		TraceEvent("DBA_Unlock").detail("Tag", tagName);
-		return Void();
 	}
 
-	ACTOR static Future<Void> atomicSwitchover(DatabaseBackupAgent* backupAgent,
-	                                           Database dest,
-	                                           Key tagName,
-	                                           Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                           Key addPrefix,
-	                                           Key removePrefix,
-	                                           ForceAction forceAction) {
-		state DatabaseBackupAgent drAgent(dest);
-		state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName));
-		state EBackupState status = wait(backupAgent->getStateValue(dest, destlogUid));
+	static Future<Void> atomicSwitchover(DatabaseBackupAgent* backupAgent,
+	                                     Database dest,
+	                                     Key tagName,
+	                                     Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                                     Key addPrefix,
+	                                     Key removePrefix,
+	                                     ForceAction forceAction) {
+		DatabaseBackupAgent drAgent(dest);
+		UID destlogUid = co_await backupAgent->getLogUid(dest, tagName);
+		EBackupState status = co_await backupAgent->getStateValue(dest, destlogUid);
 
 		TraceEvent("DBA_SwitchoverStart").detail("Status", status);
 		if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL && status != EBackupState::STATE_COMPLETED) {
@@ -2711,14 +2777,14 @@ class DatabaseBackupAgentImpl {
 		}
 
 		if (!g_network->isSimulated() && !forceAction) {
-			state StatusObject srcStatus = wait(StatusClient::statusFetcher(backupAgent->taskBucket->src));
-			StatusObject destStatus = wait(StatusClient::statusFetcher(dest));
+			StatusObject srcStatus = co_await StatusClient::statusFetcher(backupAgent->taskBucket->src);
+			StatusObject destStatus = co_await StatusClient::statusFetcher(dest);
 			checkAtomicSwitchOverConfig(srcStatus, destStatus, tagName);
 		}
 
-		state UID logUid = deterministicRandom()->randomUniqueID();
-		state Key logUidValue = BinaryWriter::toValue(logUid, Unversioned());
-		state UID logUidCurrent = wait(drAgent.getLogUid(backupAgent->taskBucket->src, tagName));
+		UID logUid = deterministicRandom()->randomUniqueID();
+		Key logUidValue = BinaryWriter::toValue(logUid, Unversioned());
+		UID logUidCurrent = co_await drAgent.getLogUid(backupAgent->taskBucket->src, tagName);
 
 		if (logUidCurrent.isValid()) {
 			logUid = logUidCurrent;
@@ -2726,31 +2792,39 @@ class DatabaseBackupAgentImpl {
 		}
 
 		// Lock src, record commit version
-		state Transaction tr(backupAgent->taskBucket->src);
-		state Version commitVersion;
-		loop {
+		Transaction tr(backupAgent->taskBucket->src);
+		Version commitVersion{ 0 };
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
-				wait(lockDatabase(&tr, logUid));
+				co_await lockDatabase(&tr, logUid);
 				tr.set(backupAgent->tagNames.pack(tagName), logUidValue);
-				wait(tr.commit());
+				co_await tr.commit();
 				commitVersion = tr.getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr.onError(err);
 			}
 		}
 
 		TraceEvent("DBA_SwitchoverLocked").detail("Version", commitVersion);
 
 		// Wait for the destination to apply mutations up to the lock commit before switching over.
-		state ReadYourWritesTransaction tr2(dest);
-		loop {
+		ReadYourWritesTransaction tr2(dest);
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr2.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr2.setOption(FDBTransactionOptions::LOCK_AWARE);
-				state Optional<Value> backupUid =
-				    wait(tr2.get(backupAgent->states.get(BinaryWriter::toValue(destlogUid, Unversioned()))
-				                     .pack(DatabaseBackupAgent::keyFolderId)));
+				Optional<Value> backupUid =
+				    co_await tr2.get(backupAgent->states.get(BinaryWriter::toValue(destlogUid, Unversioned()))
+				                         .pack(DatabaseBackupAgent::keyFolderId));
 				TraceEvent("DBA_SwitchoverBackupUID")
 				    .detail("Uid", backupUid)
 				    .detail("Key",
@@ -2758,66 +2832,76 @@ class DatabaseBackupAgentImpl {
 				                .pack(DatabaseBackupAgent::keyFolderId));
 				if (!backupUid.present())
 					throw backup_duplicate();
-				Optional<Value> v = wait(tr2.get(
-				    BinaryWriter::toValue(destlogUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin)));
+				Optional<Value> v = co_await tr2.get(
+				    BinaryWriter::toValue(destlogUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin));
 				TraceEvent("DBA_SwitchoverVersion")
 				    .detail("Version", v.present() ? BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) : 0);
 				if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >= commitVersion)
 					break;
 
-				state Future<Void> versionWatch = tr2.watch(
+				Future<Void> versionWatch = tr2.watch(
 				    BinaryWriter::toValue(destlogUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin));
-				wait(tr2.commit());
-				wait(versionWatch);
+				co_await tr2.commit();
+				co_await versionWatch;
 				tr2.reset();
 			} catch (Error& e) {
-				wait(tr2.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr2.onError(err);
 			}
 		}
 
 		TraceEvent("DBA_SwitchoverReady").log();
 
 		try {
-			wait(backupAgent->discontinueBackup(dest, tagName));
+			co_await backupAgent->discontinueBackup(dest, tagName);
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_duplicate && e.code() != error_code_backup_unneeded)
 				throw;
 		}
 
-		wait(success(backupAgent->waitBackup(dest, tagName, StopWhenDone::True)));
+		co_await backupAgent->waitBackup(dest, tagName, StopWhenDone::True);
 
 		TraceEvent("DBA_SwitchoverStopped").log();
 
-		state ReadYourWritesTransaction tr3(dest);
-		loop {
+		ReadYourWritesTransaction tr3(dest);
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr3.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr3.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Version destVersion = wait(tr3.getReadVersion());
+				Version destVersion = co_await tr3.getReadVersion();
 				TraceEvent("DBA_SwitchoverVersionUpgrade").detail("Src", commitVersion).detail("Dest", destVersion);
 				if (destVersion <= commitVersion) {
 					CODE_PROBE(true, "Forcing dest backup cluster to higher version");
 					tr3.set(minRequiredCommitVersionKey, BinaryWriter::toValue(commitVersion + 1, Unversioned()));
-					wait(tr3.commit());
+					co_await tr3.commit();
 				} else {
 					break;
 				}
 			} catch (Error& e) {
-				wait(tr3.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr3.onError(err);
 			}
 		}
 
 		TraceEvent("DBA_SwitchoverVersionUpgraded").log();
 
 		try {
-			wait(drAgent.submitBackup(backupAgent->taskBucket->src,
-			                          tagName,
-			                          backupRanges,
-			                          StopWhenDone::False,
-			                          addPrefix,
-			                          removePrefix,
-			                          LockDB::True,
-			                          DatabaseBackupAgent::PreBackupAction::NONE));
+			co_await drAgent.submitBackup(backupAgent->taskBucket->src,
+			                              tagName,
+			                              backupRanges,
+			                              StopWhenDone::False,
+			                              addPrefix,
+			                              removePrefix,
+			                              LockDB::True,
+			                              DatabaseBackupAgent::PreBackupAction::NONE);
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_duplicate)
 				throw;
@@ -2825,32 +2909,30 @@ class DatabaseBackupAgentImpl {
 
 		TraceEvent("DBA_SwitchoverSubmitted").log();
 
-		wait(success(drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName)));
+		co_await drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName);
 
 		TraceEvent("DBA_SwitchoverStarted").log();
 
-		wait(backupAgent->unlockBackup(dest, tagName));
+		co_await backupAgent->unlockBackup(dest, tagName);
 
 		TraceEvent("DBA_SwitchoverUnlocked").log();
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> discontinueBackup(DatabaseBackupAgent* backupAgent,
-	                                            Reference<ReadYourWritesTransaction> tr,
-	                                            Key tagName) {
+	static Future<Void> discontinueBackup(DatabaseBackupAgent* backupAgent,
+	                                      Reference<ReadYourWritesTransaction> tr,
+	                                      Key tagName) {
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state UID logUid = wait(backupAgent->getLogUid(tr, tagName));
-		state EBackupState status = wait(backupAgent->getStateValue(tr, logUid));
+		UID logUid = co_await backupAgent->getLogUid(tr, tagName);
+		EBackupState status = co_await backupAgent->getStateValue(tr, logUid);
 
 		TraceEvent("DBA_Discontinue").detail("Status", status);
 		if (!DatabaseBackupAgent::isRunnable(status)) {
 			throw backup_unneeded();
 		}
 
-		state Optional<Value> stopWhenDoneValue =
-		    wait(tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
-		                     .pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey)));
+		Optional<Value> stopWhenDoneValue =
+		    co_await tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
+		                         .pack(DatabaseBackupAgent::keyConfigStopWhenDoneKey));
 
 		if (stopWhenDoneValue.present()) {
 			throw backup_duplicate();
@@ -2861,35 +2943,37 @@ class DatabaseBackupAgentImpl {
 		tr->set(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
 		            .pack(BackupAgentBase::keyConfigStopWhenDoneKey),
 		        StringRef());
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> abortBackup(DatabaseBackupAgent* backupAgent,
-	                                      Database cx,
-	                                      Key tagName,
-	                                      PartialBackup partial,
-	                                      AbortOldBackup abortOldBackup,
-	                                      DstOnly dstOnly,
-	                                      WaitForDestUID waitForDestUID) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Key logUidValue, destUidValue;
-		state UID logUid, destUid;
-		state Value backupUid;
-
-		loop {
+	static Future<Void> abortBackup(DatabaseBackupAgent* backupAgent,
+	                                Database cx,
+	                                Key tagName,
+	                                PartialBackup partial,
+	                                AbortOldBackup abortOldBackup,
+	                                DstOnly dstOnly,
+	                                WaitForDestUID waitForDestUID) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Key logUidValue;
+		Key destUidValue;
+		UID logUid;
+		UID destUid;
+		Value backupUid;
+
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 
-				UID _logUid = wait(backupAgent->getLogUid(tr, tagName));
+				UID _logUid = co_await backupAgent->getLogUid(tr, tagName);
 				logUid = _logUid;
 				logUidValue = BinaryWriter::toValue(logUid, Unversioned());
 
-				state Future<EBackupState> statusFuture = backupAgent->getStateValue(tr, logUid);
-				state Future<UID> destUidFuture = backupAgent->getDestUid(tr, logUid);
-				wait(success(statusFuture) && success(destUidFuture));
+				Future<EBackupState> statusFuture = backupAgent->getStateValue(tr, logUid);
+				Future<UID> destUidFuture = backupAgent->getDestUid(tr, logUid);
+				co_await (success(statusFuture) && success(destUidFuture));
 
 				EBackupState status = statusFuture.get();
 				if (!backupAgent->isRunnable(status)) {
@@ -2898,7 +2982,7 @@ class DatabaseBackupAgentImpl {
 				UID destUid = destUidFuture.get();
 				if (destUid.isValid()) {
 					destUidValue = BinaryWriter::toValue(destUid, Unversioned());
-				} else if (destUidValue.size() == 0 && waitForDestUID) {
+				} else if (destUidValue.empty() && waitForDestUID) {
 					// Give DR task a chance to update destUid to avoid the problem of
 					// leftover version key. If we got an commit_unknown_result before,
 					// reuse the previous destUidValue.
@@ -2906,7 +2990,7 @@ class DatabaseBackupAgentImpl {
 				}
 
 				Optional<Value> _backupUid =
-				    wait(tr->get(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId)));
+				    co_await tr->get(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId));
 				backupUid = _backupUid.get();
 
 				// Clearing the folder id will prevent future tasks from executing
@@ -2920,17 +3004,21 @@ class DatabaseBackupAgentImpl {
 				tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)),
 				        StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_PARTIALLY_ABORTED)));
 
-				wait(tr->commit());
+				co_await tr->commit();
 				TraceEvent("DBA_Abort").detail("CommitVersion", tr->getCommittedVersion());
 				break;
 			} catch (Error& e) {
-				TraceEvent("DBA_AbortError").errorUnsuppressed(e);
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				TraceEvent("DBA_AbortError").errorUnsuppressed(err);
+				co_await tr->onError(err);
 			}
 		}
 
 		tr = makeReference<ReadYourWritesTransaction>(cx);
-		loop {
+		while (true) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			// dumpData's commits are unstoppable, and we need to make sure that no dumpData commits
@@ -2941,9 +3029,11 @@ class DatabaseBackupAgentImpl {
 			// transactions to the proxy's commit promise stream, so our commit will come after all
 			// dumpData transactions.
 			tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
+			Error err;
+			bool hasErr = false;
 			try {
 				// Ensure that we're at a version higher than the data that we've written.
-				Optional<Value> lastApplied = wait(tr->get(logUidValue.withPrefix(applyMutationsBeginRange.begin)));
+				Optional<Value> lastApplied = co_await tr->get(logUidValue.withPrefix(applyMutationsBeginRange.begin));
 				if (lastApplied.present()) {
 					Version current = tr->getReadVersion().get();
 					Version applied = BinaryReader::fromStringRef<Version>(lastApplied.get(), Unversioned());
@@ -2963,27 +3053,32 @@ class DatabaseBackupAgentImpl {
 						tr->addWriteConflictRange(singleKeyRange(minRequiredCommitVersionKey));
 					}
 				}
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 
 		if (!dstOnly) {
-			state Future<Void> partialTimeout = partial ? delay(30.0) : Never();
-			state Reference<ReadYourWritesTransaction> srcTr(
-			    new ReadYourWritesTransaction(backupAgent->taskBucket->src));
+			Future<Void> partialTimeout = partial ? delay(30.0) : Never();
+			Reference<ReadYourWritesTransaction> srcTr(new ReadYourWritesTransaction(backupAgent->taskBucket->src));
 
-			loop {
+			while (true) {
+				Error err;
+				bool hasErr = false;
 				try {
 					srcTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					srcTr->setOption(FDBTransactionOptions::LOCK_AWARE);
-					state Future<Optional<Value>> backupVersionF =
+					Future<Optional<Value>> backupVersionF =
 					    srcTr->get(backupAgent->sourceStates.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId));
-					wait(success(backupVersionF) || partialTimeout);
+					co_await (success(backupVersionF) || partialTimeout);
 					if (partialTimeout.isReady()) {
-						return Void();
+						co_return;
 					}
 
 					if (backupVersionF.get().present() &&
@@ -3004,10 +3099,10 @@ class DatabaseBackupAgentImpl {
 
 					Key latestVersionKey = logUidValue.withPrefix(destUidValue.withPrefix(backupLatestVersionsPrefix));
 
-					state Future<Optional<Key>> bVersionF = srcTr->get(latestVersionKey);
-					wait(success(bVersionF) || partialTimeout);
+					Future<Optional<Key>> bVersionF = srcTr->get(latestVersionKey);
+					co_await (success(bVersionF) || partialTimeout);
 					if (partialTimeout.isReady()) {
-						return Void();
+						co_return;
 					}
 
 					if (!bVersionF.get().present()) {
@@ -3019,121 +3114,130 @@ class DatabaseBackupAgentImpl {
 					srcTr->set(backupAgent->sourceStates.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId),
 					           backupUid);
 
-					wait(eraseLogData(srcTr, logUidValue, destUidValue) || partialTimeout);
+					co_await (eraseLogData(srcTr, logUidValue, destUidValue) || partialTimeout);
 					if (partialTimeout.isReady()) {
-						return Void();
+						co_return;
 					}
 
-					wait(srcTr->commit() || partialTimeout);
+					co_await (srcTr->commit() || partialTimeout);
 					if (partialTimeout.isReady()) {
-						return Void();
+						co_return;
 					}
 
 					break;
 				} catch (Error& e) {
-					wait(srcTr->onError(e));
+					err = e;
+					hasErr = true;
+				}
+				if (hasErr) {
+					co_await srcTr->onError(err);
 				}
 			}
 		}
 
 		tr = makeReference<ReadYourWritesTransaction>(cx);
-		loop {
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				Optional<Value> v = wait(
-				    tr->get(StringRef(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId))));
+				Optional<Value> v = co_await tr->get(
+				    StringRef(backupAgent->config.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId)));
 				if (v.present()) {
-					return Void();
+					co_return;
 				}
 
 				tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)),
 				        StringRef(DatabaseBackupAgent::getStateText(EBackupState::STATE_ABORTED)));
 
-				wait(tr->commit());
+				co_await tr->commit();
 
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<std::string> getStatus(DatabaseBackupAgent* backupAgent,
-	                                           Database cx,
-	                                           int errorLimit,
-	                                           Key tagName) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	static Future<std::string> getStatus(DatabaseBackupAgent* backupAgent, Database cx, int errorLimit, Key tagName) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state std::string statusText;
-		state int retries = 0;
+		std::string statusText;
+		int retries = 0;
 
-		loop {
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(success(tr->getReadVersion())); // get the read version before getting a version from the source
-				                                     // database to prevent the time differential from going negative
+				co_await tr->getReadVersion(); // get the read version before getting a version from the source
+				                               // database to prevent the time differential from going negative
 
-				state Transaction scrTr(backupAgent->taskBucket->src);
+				Transaction scrTr(backupAgent->taskBucket->src);
 				scrTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				state Future<Version> srcReadVersion = scrTr.getReadVersion();
+				Future<Version> srcReadVersion = scrTr.getReadVersion();
 
 				statusText = "";
 
-				state UID logUid = wait(backupAgent->getLogUid(tr, tagName));
+				UID logUid = co_await backupAgent->getLogUid(tr, tagName);
 
-				state Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
-				state Future<RangeResult> fErrorValues =
+				Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
+				Future<RangeResult> fErrorValues =
 				    errorLimit > 0
 				        ? tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(),
 				                       errorLimit,
 				                       Snapshot::False,
 				                       Reverse::True)
 				        : Future<RangeResult>();
-				state Future<Optional<Value>> fBackupUid =
+				Future<Optional<Value>> fBackupUid =
 				    tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
 				                .pack(DatabaseBackupAgent::keyFolderId));
-				state Future<Optional<Value>> fBackupVerison =
+				Future<Optional<Value>> fBackupVerison =
 				    tr->get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin));
-				state Future<Optional<Key>> fTagName =
+				Future<Optional<Key>> fTagName =
 				    tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
 				                .pack(BackupAgentBase::keyConfigBackupTag));
-				state Future<Optional<Value>> fStopVersionKey =
+				Future<Optional<Value>> fStopVersionKey =
 				    tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
 				                .pack(BackupAgentBase::keyStateStop));
-				state Future<Optional<Key>> fBackupKeysPacked =
+				Future<Optional<Key>> fBackupKeysPacked =
 				    tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned()))
 				                .pack(BackupAgentBase::keyConfigBackupRanges));
-				state Future<Optional<Value>> flogVersionKey =
+				Future<Optional<Value>> flogVersionKey =
 				    tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
 				                .pack(BackupAgentBase::keyStateLogBeginVersion));
 
-				state EBackupState backupState = wait(backupAgent->getStateValue(tr, logUid));
+				EBackupState backupState = co_await backupAgent->getStateValue(tr, logUid);
 
 				if (backupState == EBackupState::STATE_NEVERRAN) {
 					statusText += "No previous backups found.\n";
 				} else {
-					state std::string tagNameDisplay;
-					Optional<Key> tagName = wait(fTagName);
+					std::string tagNameDisplay;
+					Optional<Key> tagName = co_await fTagName;
 
 					// Define the display tag name
 					if (tagName.present()) {
 						tagNameDisplay = tagName.get().toString();
 					}
 
-					state Optional<Value> stopVersionKey = wait(fStopVersionKey);
-					Optional<Value> logVersionKey = wait(flogVersionKey);
-					state std::string logVersionText =
+					Optional<Value> stopVersionKey = co_await fStopVersionKey;
+					Optional<Value> logVersionKey = co_await flogVersionKey;
+					std::string logVersionText =
 					    ". Last log version is " +
 					    (logVersionKey.present()
 					         ? format("%lld", BinaryReader::fromStringRef<Version>(logVersionKey.get(), Unversioned()))
 					         : "unset");
-					Optional<Key> backupKeysPacked = wait(fBackupKeysPacked);
+					Optional<Key> backupKeysPacked = co_await fBackupKeysPacked;
 
-					state Standalone<VectorRef<KeyRangeRef>> backupRanges;
+					Standalone<VectorRef<KeyRangeRef>> backupRanges;
 					if (backupKeysPacked.present()) {
 						BinaryReader br(backupKeysPacked.get(), IncludeVersion());
 						br >> backupRanges;
@@ -3175,10 +3279,10 @@ class DatabaseBackupAgentImpl {
 
 				// Append the errors, if requested
 				if (errorLimit > 0) {
-					RangeResult values = wait(fErrorValues);
+					RangeResult values = co_await fErrorValues;
 
 					// Display the errors, if any
-					if (values.size() > 0) {
+					if (!values.empty()) {
 						// Inform the user that the list of errors is complete or partial
 						statusText += (values.size() < errorLimit)
 						                  ? "WARNING: Some DR agents have reported issues:\n"
@@ -3192,72 +3296,77 @@ class DatabaseBackupAgentImpl {
 				}
 
 				// calculate time differential
-				Optional<Value> backupUid = wait(fBackupUid);
+				Optional<Value> backupUid = co_await fBackupUid;
 				if (backupUid.present()) {
-					Optional<Value> v = wait(fBackupVerison);
+					Optional<Value> v = co_await fBackupVerison;
 					if (v.present()) {
-						state Version destApplyBegin = BinaryReader::fromStringRef<Version>(v.get(), Unversioned());
-						Version sourceVersion = wait(srcReadVersion);
+						Version destApplyBegin = BinaryReader::fromStringRef<Version>(v.get(), Unversioned());
+						Version sourceVersion = co_await srcReadVersion;
 						double secondsBehind =
 						    ((double)(sourceVersion - destApplyBegin)) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
 						statusText += format("\nThe DR is %.6f seconds behind.\n", secondsBehind);
 					}
 				}
 
-				Optional<Value> paused = wait(fPaused);
+				Optional<Value> paused = co_await fPaused;
 				if (paused.present()) {
 					statusText += format("\nAll DR agents have been paused.\n");
 				}
 
 				break;
 			} catch (Error& e) {
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
 				retries++;
 				if (retries > 5) {
-					statusText += format("\nWARNING: Could not fetch full DR status: %s\n", e.name());
-					return statusText;
+					statusText += format("\nWARNING: Could not fetch full DR status: %s\n", err.name());
+					co_return statusText;
 				}
-				wait(tr->onError(e));
+				co_await tr->onError(err);
 			}
 		}
 
-		return statusText;
+		co_return statusText;
 	}
 
-	ACTOR static Future<EBackupState> getStateValue(DatabaseBackupAgent* backupAgent,
-	                                                Reference<ReadYourWritesTransaction> tr,
-	                                                UID logUid,
-	                                                Snapshot snapshot) {
+	static Future<EBackupState> getStateValue(DatabaseBackupAgent* backupAgent,
+	                                          Reference<ReadYourWritesTransaction> tr,
+	                                          UID logUid,
+	                                          Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
-		                          .pack(DatabaseBackupAgent::keyStateStatus);
-		Optional<Value> status = wait(tr->get(statusKey, snapshot));
+		Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned()))
+		                    .pack(DatabaseBackupAgent::keyStateStatus);
+		Optional<Value> status = co_await tr->get(statusKey, snapshot);
 
-		return (!status.present()) ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(status.get().toString());
+		co_return (!status.present()) ? EBackupState::STATE_NEVERRAN
+		                              : BackupAgentBase::getState(status.get().toString());
 	}
 
-	ACTOR static Future<UID> getDestUid(DatabaseBackupAgent* backupAgent,
-	                                    Reference<ReadYourWritesTransaction> tr,
-	                                    UID logUid,
-	                                    Snapshot snapshot) {
+	static Future<UID> getDestUid(DatabaseBackupAgent* backupAgent,
+	                              Reference<ReadYourWritesTransaction> tr,
+	                              UID logUid,
+	                              Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Key destUidKey =
+		Key destUidKey =
 		    backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::destUid);
-		Optional<Value> destUid = wait(tr->get(destUidKey, snapshot));
+		Optional<Value> destUid = co_await tr->get(destUidKey, snapshot);
 
-		return (destUid.present()) ? BinaryReader::fromStringRef<UID>(destUid.get(), Unversioned()) : UID();
+		co_return (destUid.present()) ? BinaryReader::fromStringRef<UID>(destUid.get(), Unversioned()) : UID();
 	}
 
-	ACTOR static Future<UID> getLogUid(DatabaseBackupAgent* backupAgent,
-	                                   Reference<ReadYourWritesTransaction> tr,
-	                                   Key tagName,
-	                                   Snapshot snapshot) {
+	static Future<UID> getLogUid(DatabaseBackupAgent* backupAgent,
+	                             Reference<ReadYourWritesTransaction> tr,
+	                             Key tagName,
+	                             Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Optional<Value> logUid = wait(tr->get(backupAgent->tagNames.pack(tagName), snapshot));
+		Optional<Value> logUid = co_await tr->get(backupAgent->tagNames.pack(tagName), snapshot);
 
-		return (logUid.present()) ? BinaryReader::fromStringRef<UID>(logUid.get(), Unversioned()) : UID();
+		co_return (logUid.present()) ? BinaryReader::fromStringRef<UID>(logUid.get(), Unversioned()) : UID();
 	}
 };
 
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index 2e05ea18f43..5dcc24d7d93 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -52,6 +52,7 @@ void DatabaseConfiguration::resetInternal() {
 	remoteDesiredTLogCount = -1;
 	remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0;
 	backupWorkerEnabled = false;
+	rangeBackupWorkerEnabled = false;
 	perpetualStorageWiggleSpeed = 0;
 	perpetualStorageWiggleLocality = "0";
 	storageMigrationType = StorageMigrationType::DEFAULT;
@@ -83,7 +84,7 @@ void parseReplicationPolicy(Reference<IReplicationPolicy>* policy, ValueRef cons
 
 void parse(std::vector<RegionInfo>* regions, ValueRef const& v) {
 	try {
-		StatusObject statusObj = BinaryReader::fromStringRef<StatusObject>(v, IncludeVersion());
+		auto statusObj = BinaryReader::fromStringRef<StatusObject>(v, IncludeVersion());
 		regions->clear();
 		if (statusObj["regions"].type() != json_spirit::array_type) {
 			return;
@@ -122,47 +123,39 @@ void parse(std::vector<RegionInfo>* regions, ValueRef const& v) {
 					info.satelliteTLogReplicationFactor = 1;
 					info.satelliteTLogUsableDcs = 1;
 					info.satelliteTLogWriteAntiQuorum = 0;
-					info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
+					info.satelliteTLogPolicy = makeReference<PolicyOne>();
 				} else if (satelliteReplication == "one_satellite_double") {
 					info.satelliteTLogReplicationFactor = 2;
 					info.satelliteTLogUsableDcs = 1;
 					info.satelliteTLogWriteAntiQuorum = 0;
-					info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
-					    new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+					info.satelliteTLogPolicy = makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>());
 				} else if (satelliteReplication == "one_satellite_triple") {
 					info.satelliteTLogReplicationFactor = 3;
 					info.satelliteTLogUsableDcs = 1;
 					info.satelliteTLogWriteAntiQuorum = 0;
-					info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
-					    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+					info.satelliteTLogPolicy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
 				} else if (satelliteReplication == "two_satellite_safe") {
 					info.satelliteTLogReplicationFactor = 4;
 					info.satelliteTLogUsableDcs = 2;
 					info.satelliteTLogWriteAntiQuorum = 0;
-					info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
-					    new PolicyAcross(2,
-					                     "dcid",
-					                     Reference<IReplicationPolicy>(new PolicyAcross(
-					                         2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+					info.satelliteTLogPolicy = makeReference<PolicyAcross>(
+					    2, "dcid", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 					info.satelliteTLogReplicationFactorFallback = 2;
 					info.satelliteTLogUsableDcsFallback = 1;
 					info.satelliteTLogWriteAntiQuorumFallback = 0;
-					info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
-					    new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+					info.satelliteTLogPolicyFallback =
+					    makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>());
 				} else if (satelliteReplication == "two_satellite_fast") {
 					info.satelliteTLogReplicationFactor = 4;
 					info.satelliteTLogUsableDcs = 2;
 					info.satelliteTLogWriteAntiQuorum = 2;
-					info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
-					    new PolicyAcross(2,
-					                     "dcid",
-					                     Reference<IReplicationPolicy>(new PolicyAcross(
-					                         2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+					info.satelliteTLogPolicy = makeReference<PolicyAcross>(
+					    2, "dcid", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 					info.satelliteTLogReplicationFactorFallback = 2;
 					info.satelliteTLogUsableDcsFallback = 1;
 					info.satelliteTLogWriteAntiQuorumFallback = 0;
-					info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
-					    new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+					info.satelliteTLogPolicyFallback =
+					    makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>());
 				} else {
 					throw invalid_option();
 				}
@@ -184,29 +177,63 @@ void parse(std::vector<RegionInfo>* regions, ValueRef const& v) {
 
 void DatabaseConfiguration::setDefaultReplicationPolicy() {
 	if (!storagePolicy) {
-		storagePolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		storagePolicy = makeReference<PolicyAcross>(storageTeamSize, "zoneid", makeReference<PolicyOne>());
 	}
 	if (!tLogPolicy) {
-		tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		tLogPolicy = makeReference<PolicyAcross>(tLogReplicationFactor, "zoneid", makeReference<PolicyOne>());
 	}
 	if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
-		remoteTLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		remoteTLogPolicy =
+		    makeReference<PolicyAcross>(remoteTLogReplicationFactor, "zoneid", makeReference<PolicyOne>());
 	}
 	for (auto& r : regions) {
 		if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
-			r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(
-			    r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+			r.satelliteTLogPolicy =
+			    makeReference<PolicyAcross>(r.satelliteTLogReplicationFactor, "zoneid", makeReference<PolicyOne>());
 		}
 		if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
-			r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(
-			    r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+			r.satelliteTLogPolicyFallback = makeReference<PolicyAcross>(
+			    r.satelliteTLogReplicationFactorFallback, "zoneid", makeReference<PolicyOne>());
 		}
 	}
 }
 
+int32_t DatabaseConfiguration::maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const {
+	int worstSatelliteTLogReplicationFactor = !regions.empty() ? std::numeric_limits<int>::max() : 0;
+	int regionsWithNonNegativePriority = 0;
+	for (auto& r : regions) {
+		if (r.priority >= 0) {
+			regionsWithNonNegativePriority++;
+		}
+		worstSatelliteTLogReplicationFactor = std::min(
+		    worstSatelliteTLogReplicationFactor, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
+		if (r.satelliteTLogUsableDcsFallback > 0) {
+			worstSatelliteTLogReplicationFactor =
+			    std::min(worstSatelliteTLogReplicationFactor,
+			             r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
+		}
+	}
+
+	if (worstSatelliteTLogReplicationFactor <= 0) {
+		// HA is not enabled in this database. Return single cluster zone failures to tolerate.
+		return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1);
+	}
+
+	// Compute HA enabled database zone failure tolerance.
+	auto isGeoReplicatedData = [this, &fullyReplicatedRegions]() {
+		return usableRegions > 1 && fullyReplicatedRegions > 1;
+	};
+
+	if (isGeoReplicatedData() && (!forAvailability || regionsWithNonNegativePriority > 1)) {
+		return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum,
+		                             worstSatelliteTLogReplicationFactor - 1),
+		                    storageTeamSize - 1);
+	}
+	// Primary and Satellite tLogs are synchronously replicated, hence we can lose all but 1.
+	return std::min(tLogReplicationFactor + worstSatelliteTLogReplicationFactor - 1 - tLogWriteAntiQuorum,
+	                storageTeamSize - 1);
+}
+
 bool DatabaseConfiguration::isValid() const {
 	// enable this via `fdbcli --knob_cli_print_invalid_configuration=1` command line parameter
 	auto log_test = [](const char* text, bool val) {
@@ -233,8 +260,8 @@ bool DatabaseConfiguration::isValid() const {
 	      LOG_TEST(remoteTLogReplicationFactor >= 0) && LOG_TEST(repopulateRegionAntiQuorum >= 0) &&
 	      LOG_TEST(repopulateRegionAntiQuorum <= 1) && LOG_TEST(usableRegions >= 1) && LOG_TEST(usableRegions <= 2) &&
 	      LOG_TEST(regions.size() <= 2) && LOG_TEST((usableRegions == 1 || regions.size() == 2)) &&
-	      LOG_TEST((regions.size() == 0 || regions[0].priority >= 0)) &&
-	      LOG_TEST((regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1")) &&
+	      LOG_TEST((regions.empty() || regions[0].priority >= 0)) &&
+	      LOG_TEST((regions.empty() || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1")) &&
 	      // We cannot specify regions with three_datacenter replication
 	      LOG_TEST((perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1)) &&
 	      LOG_TEST(isValidPerpetualStorageWiggleLocality(perpetualStorageWiggleLocality)) &&
@@ -245,9 +272,9 @@ bool DatabaseConfiguration::isValid() const {
 	std::set<Key> dcIds;
 	dcIds.insert(Key());
 	for (auto& r : regions) {
-		if (!(!dcIds.count(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
+		if (!(!dcIds.contains(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
 		      r.satelliteTLogUsableDcs >= 1 &&
-		      (r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) &&
+		      (r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && !r.satellites.empty())) &&
 		      (r.satelliteTLogUsableDcsFallback == 0 ||
 		       (r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0)))) {
 			return false;
@@ -257,7 +284,7 @@ bool DatabaseConfiguration::isValid() const {
 		satelliteDcIds.insert(Key());
 		satelliteDcIds.insert(r.dcId);
 		for (auto& s : r.satellites) {
-			if (satelliteDcIds.count(s.dcId)) {
+			if (satelliteDcIds.contains(s.dcId)) {
 				return false;
 			}
 			satelliteDcIds.insert(s.dcId);
@@ -340,7 +367,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	}
 	result["usable_regions"] = usableRegions;
 
-	if (regions.size()) {
+	if (!regions.empty()) {
 		result["regions"] = getRegionJSON();
 	}
 
@@ -394,6 +421,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	}
 
 	result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
+	result["range_backup_worker_enabled"] = (int32_t)rangeBackupWorkerEnabled;
 	result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed;
 	result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality;
 	if (perpetualStoreType.storeType() != KeyValueStoreType::END) {
@@ -506,7 +534,7 @@ StatusArray DatabaseConfiguration::getRegionJSON() const {
 			regionObj["satellite_logs"] = r.satelliteDesiredTLogCount;
 		}
 
-		if (r.satellites.size()) {
+		if (!r.satellites.empty()) {
 			for (auto& s : r.satellites) {
 				StatusObject satObj;
 				satObj["id"] = s.dcId.toString();
@@ -664,6 +692,9 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	} else if (ck == "backup_worker_enabled"_sr) {
 		parse((&type), value);
 		backupWorkerEnabled = (type != 0);
+	} else if (ck == "range_backup_worker_enabled"_sr) {
+		parse((&type), value);
+		rangeBackupWorkerEnabled = (type != 0);
 	} else if (ck == "usable_regions"_sr) {
 		parse(&usableRegions, value);
 	} else if (ck == "repopulate_anti_quorum"_sr) {
diff --git a/fdbclient/DatabaseContext.actor.cpp b/fdbclient/DatabaseContext.cpp
similarity index 87%
rename from fdbclient/DatabaseContext.actor.cpp
rename to fdbclient/DatabaseContext.cpp
index fb1fbaad1e2..9870f382939 100644
--- a/fdbclient/DatabaseContext.actor.cpp
+++ b/fdbclient/DatabaseContext.cpp
@@ -18,80 +18,47 @@
  * limitations under the License.
  */
 
-// TODO: prune down the list of includes. This was copied from NativeAPI.actor.cpp.
 #include "fdbclient/NativeAPI.actor.h"
 
-#include <algorithm>
-#include <cstdio>
+#include <functional>
 #include <iterator>
 #include <limits>
 #include <memory>
-#include <regex>
 #include <string>
-#include <unordered_set>
-#include <tuple>
 #include <utility>
 #include <vector>
 
-#include "boost/algorithm/string.hpp"
-
 #include "fdbclient/Knobs.h"
 #include "flow/CodeProbe.h"
-#include "fmt/format.h"
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/MultiInterface.h"
 
-#include "fdbclient/ActorLineageProfiler.h"
 #include "fdbclient/AnnotateActor.h"
-#include "fdbclient/Atomic.h"
 #include "fdbclient/ClusterInterface.h"
-#include "fdbclient/ClusterConnectionFile.h"
-#include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/IKnobCollection.h"
-#include "fdbclient/JsonBuilder.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/NameLineage.h"
 #include "fdbclient/CommitProxyInterface.h"
-#include "fdbclient/MonitorLeader.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/ParallelStream.actor.h"
+#include "ProxyLoadBalance.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/SpecialKeySpace.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/TransactionLineage.h"
-#include "fdbclient/versions.h"
-#include "fdbrpc/WellKnownEndpoints.h"
-#include "fdbrpc/LoadBalance.h"
-#include "fdbrpc/Net2FileSystem.h"
-#include "fdbrpc/simulator.h"
-#include "fdbrpc/sim_validation.h"
 #include "flow/Arena.h"
 #include "flow/ActorCollection.h"
-#include "flow/DeterministicRandom.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
-#include "flow/GetSourceVersion.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
-#include "flow/ProtocolVersion.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
-#include "flow/Knobs.h"
 #include "flow/Platform.h"
-#include "flow/SystemMonitor.h"
-#include "flow/TLSConfig.actor.h"
 #include "fdbclient/Tracing.h"
-#include "flow/UnitTest.h"
 #include "flow/network.h"
 #include "flow/serialize.h"
 
@@ -105,9 +72,8 @@
 #undef min
 #undef max
 #else
-#include <time.h>
 #endif
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 Reference<WatchMetadata> DatabaseContext::getWatchMetadata(KeyRef key) const {
 	const auto it = watchMap.find(WatchMapKey(key));
@@ -456,22 +422,17 @@ inline HealthMetrics populateHealthMetrics(const HealthMetrics& detailedMetrics,
 	}
 }
 
-ACTOR static Future<HealthMetrics> getHealthMetricsActor(DatabaseContext* cx, bool detailed, bool sendDetailedRequest) {
-	loop {
-		choose {
-			when(wait(cx->onProxiesChanged())) {}
-			when(GetHealthMetricsReply rep = wait(basicLoadBalance(cx->getGrvProxies(UseProvisionalProxies::False),
-			                                                       &GrvProxyInterface::getHealthMetrics,
-			                                                       GetHealthMetricsRequest(sendDetailedRequest)))) {
-				cx->healthMetrics.update(rep.healthMetrics, sendDetailedRequest, true);
-				cx->healthMetricsLastUpdated = now();
-				if (sendDetailedRequest) {
-					cx->detailedHealthMetricsLastUpdated = now();
-				}
-				return populateHealthMetrics(cx->healthMetrics, detailed);
-			}
-		}
+static Future<HealthMetrics> getHealthMetricsActor(DatabaseContext* cx, bool detailed, bool sendDetailedRequest) {
+	GetHealthMetricsReply rep =
+	    co_await grvProxyLoadBalance(Database(Reference<DatabaseContext>::addRef(cx)),
+	                                 makeReqBuilder<GetHealthMetricsRequest>(sendDetailedRequest),
+	                                 &GrvProxyInterface::getHealthMetrics);
+	cx->healthMetrics.update(rep.healthMetrics, sendDetailedRequest, true);
+	cx->healthMetricsLastUpdated = now();
+	if (sendDetailedRequest) {
+		cx->detailedHealthMetricsLastUpdated = now();
 	}
+	co_return populateHealthMetrics(cx->healthMetrics, detailed);
 }
 
 Future<HealthMetrics> DatabaseContext::getHealthMetrics(bool detailed = false) {
@@ -483,13 +444,13 @@ Future<HealthMetrics> DatabaseContext::getHealthMetrics(bool detailed = false) {
 	return getHealthMetricsActor(this, detailed, sendDetailedRequest);
 }
 
-Future<Optional<HealthMetrics::StorageStats>> DatabaseContext::getStorageStats(const UID& id, double maxStaleness) {
+Future<Optional<HealthMetrics::StorageStats>> DatabaseContext::getStorageStats(UID id, double maxStaleness) {
 	if (now() - detailedHealthMetricsLastUpdated < maxStaleness) {
 		auto it = healthMetrics.storageStats.find(id);
 		return it == healthMetrics.storageStats.end() ? Optional<HealthMetrics::StorageStats>() : it->second;
 	}
 
-	return map(getHealthMetricsActor(this, true, true), [&id](auto metrics) -> Optional<HealthMetrics::StorageStats> {
+	return map(getHealthMetricsActor(this, true, true), [id](auto metrics) -> Optional<HealthMetrics::StorageStats> {
 		auto it = metrics.storageStats.find(id);
 		return it == metrics.storageStats.end() ? Optional<HealthMetrics::StorageStats>() : it->second;
 	});
@@ -548,10 +509,10 @@ void traceTSSPercentiles(TraceEvent& ev,
 	}
 }
 
-ACTOR Future<Void> tssLogger(DatabaseContext* cx) {
-	state double lastLogged = 0;
-	loop {
-		wait(delay(CLIENT_KNOBS->TSS_METRICS_LOGGING_INTERVAL, TaskPriority::FlushTrace));
+Future<Void> tssLogger(DatabaseContext* cx) {
+	double lastLogged = 0;
+	while (true) {
+		co_await delay(CLIENT_KNOBS->TSS_METRICS_LOGGING_INTERVAL, TaskPriority::FlushTrace);
 
 		// Log each TSS pair separately
 		for (const auto& it : cx->tssMetrics) {
@@ -592,10 +553,10 @@ ACTOR Future<Void> tssLogger(DatabaseContext* cx) {
 	}
 }
 
-ACTOR Future<Void> databaseLogger(DatabaseContext* cx) {
-	state double lastLogged = 0;
-	loop {
-		wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace));
+Future<Void> databaseLogger(DatabaseContext* cx) {
+	double lastLogged = 0;
+	while (true) {
+		co_await delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace);
 
 		bool logMetrics = !g_network->isSimulated() || BUGGIFY_WITH_PROB(0.01);
 		if (logMetrics) {
@@ -653,15 +614,16 @@ struct TrInfoChunk {
 static const Key CLIENT_LATENCY_INFO_PREFIX = "client_latency/"_sr;
 static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = "client_latency_counter/"_sr;
 
-ACTOR static Future<Void> transactionInfoCommitActor(Transaction* tr, std::vector<TrInfoChunk>* chunks) {
-	state const Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
-	state int retryCount = 0;
-	loop {
+static Future<Void> transactionInfoCommitActor(Transaction* tr, std::vector<TrInfoChunk>* chunks) {
+	const Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
+	int retryCount = 0;
+	while (true) {
+		Error err;
 		try {
 			tr->reset();
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state Future<Standalone<StringRef>> vstamp = tr->getVersionstamp();
+			[[maybe_unused]] Future<Standalone<StringRef>> vstamp = tr->getVersionstamp();
 			int64_t numCommitBytes = 0;
 			for (auto& chunk : *chunks) {
 				tr->atomicOp(chunk.key, chunk.value, MutationRef::SetVersionstampedKey);
@@ -669,47 +631,49 @@ ACTOR static Future<Void> transactionInfoCommitActor(Transaction* tr, std::vecto
 				                  4; // subtract number of bytes of key that denotes version stamp index
 			}
 			tr->atomicOp(clientLatencyAtomicCtr, StringRef((uint8_t*)&numCommitBytes, 8), MutationRef::AddValue);
-			wait(tr->commit());
-			return Void();
+			co_await tr->commit();
+			co_return;
 		} catch (Error& e) {
-			retryCount++;
-			if (retryCount == 10)
-				throw;
-			wait(tr->onError(e));
+			err = e;
 		}
+		retryCount++;
+		if (retryCount == 10)
+			throw err;
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR static Future<Void> delExcessClntTxnEntriesActor(Transaction* tr, int64_t clientTxInfoSizeLimit) {
-	state const Key clientLatencyName = CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
-	state const Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
+static Future<Void> delExcessClntTxnEntriesActor(Transaction* tr, int64_t clientTxInfoSizeLimit) {
+	const Key clientLatencyName = CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
+	const Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
 	TraceEvent(SevInfo, "DelExcessClntTxnEntriesCalled").log();
 
 	// If we don't limit it with retries, the DatabaseContext will never cleanup as Transaction
 	// object will be alive and hold reference to DatabaseContext.
-	state int retries = 0;
-	loop {
+	int retries = 0;
+	while (true) {
+		Error err;
 		try {
 			tr->reset();
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> ctrValue = wait(tr->get(KeyRef(clientLatencyAtomicCtr), Snapshot::True));
+			Optional<Value> ctrValue = co_await tr->get(KeyRef(clientLatencyAtomicCtr), Snapshot::True);
 			if (!ctrValue.present()) {
 				TraceEvent(SevInfo, "NumClntTxnEntriesNotFound").log();
-				return Void();
+				co_return;
 			}
-			state int64_t txInfoSize = 0;
+			int64_t txInfoSize = 0;
 			ASSERT(ctrValue.get().size() == sizeof(int64_t));
 			memcpy(&txInfoSize, ctrValue.get().begin(), ctrValue.get().size());
 			if (txInfoSize < clientTxInfoSizeLimit)
-				return Void();
+				co_return;
 			int getRangeByteLimit = (txInfoSize - clientTxInfoSizeLimit) < CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT
 			                            ? (txInfoSize - clientTxInfoSizeLimit)
 			                            : CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
 			GetRangeLimits limit(GetRangeLimits::ROW_LIMIT_UNLIMITED, getRangeByteLimit);
 			RangeResult txEntries =
-			    wait(tr->getRange(KeyRangeRef(clientLatencyName, strinc(clientLatencyName)), limit));
-			state int64_t numBytesToDel = 0;
+			    co_await tr->getRange(KeyRangeRef(clientLatencyName, strinc(clientLatencyName)), limit);
+			int64_t numBytesToDel = 0;
 			KeyRef endKey;
 			for (auto& kv : txEntries) {
 				endKey = kv.key;
@@ -723,44 +687,48 @@ ACTOR static Future<Void> delExcessClntTxnEntriesActor(Transaction* tr, int64_t
 				int64_t bytesDel = -numBytesToDel;
 
 				tr->atomicOp(clientLatencyAtomicCtr, StringRef((uint8_t*)&bytesDel, 8), MutationRef::AddValue);
-				wait(tr->commit());
+				co_await tr->commit();
 			}
 			if (txInfoSize - numBytesToDel <= clientTxInfoSizeLimit)
-				return Void();
+				co_return;
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled || retries++ >= 10) {
-				throw;
-			}
-
-			wait(tr->onError(e));
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled || retries++ >= 10) {
+			throw err;
 		}
+
+		co_await tr->onError(err);
 	}
 }
 
-// FIXME: explain what "client status" is
+// Client status is the client-side transaction profiling data persisted under
+// fdbClientInfoPrefixRange for status reporting.
 // The reason for getting a pointer to DatabaseContext instead of a reference counted object is because reference
 // counting will increment reference count for DatabaseContext which holds the future of this actor. This creates a
 // cyclic reference and hence this actor and Database object will not be destroyed at all.
-ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
-	state const std::string clientLatencyName =
+static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
+	const std::string clientLatencyName =
 	    CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin).toString();
-	state Transaction tr;
-	state std::vector<TrInfoChunk> commitQ;
-	state int txBytes = 0;
+	Transaction tr;
+	std::vector<TrInfoChunk> commitQ;
+	int txBytes = 0;
 
-	loop {
+	while (true) {
 		// Make sure we are connected to the server. Otherwise we may just try to keep reconnecting
 		// with incompatible clusters.
-		wait(cx->onConnected());
+		co_await cx->onConnected();
 
 		// Need to make sure that we eventually destroy tr. We can't rely on getting cancelled to do
 		// this because of the cyclic reference to self.
-		wait(refreshTransaction(cx, &tr));
+		co_await refreshTransaction(cx, &tr);
+		Error err;
 		try {
 			ASSERT(cx->clientStatusUpdater.outStatusQ.empty());
 			cx->clientStatusUpdater.inStatusQ.swap(cx->clientStatusUpdater.outStatusQ);
 			// Split Transaction Info into chunks
-			state std::vector<TrInfoChunk> trChunksQ;
+			std::vector<TrInfoChunk> trChunksQ;
 			for (auto& entry : cx->clientStatusUpdater.outStatusQ) {
 				auto& bw = entry.second;
 				int64_t value_size_limit = BUGGIFY
@@ -789,19 +757,19 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
 			}
 
 			// Commit the chunks splitting into different transactions if needed
-			state int64_t dataSizeLimit =
+			int64_t dataSizeLimit =
 			    BUGGIFY ? deterministicRandom()->randomInt(200e3, 1.5 * CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT)
 			            : 0.8 * CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
-			state std::vector<TrInfoChunk>::iterator tracking_iter = trChunksQ.begin();
+			std::vector<TrInfoChunk>::iterator tracking_iter = trChunksQ.begin();
 			ASSERT(commitQ.empty() && (txBytes == 0));
-			loop {
-				state std::vector<TrInfoChunk>::iterator iter = tracking_iter;
+			while (true) {
+				std::vector<TrInfoChunk>::iterator iter = tracking_iter;
 				txBytes = 0;
 				commitQ.clear();
 				try {
 					while (iter != trChunksQ.end()) {
 						if (iter->value.size() + iter->key.size() + txBytes > dataSizeLimit) {
-							wait(transactionInfoCommitActor(&tr, &commitQ));
+							co_await transactionInfoCommitActor(&tr, &commitQ);
 							tracking_iter = iter;
 							commitQ.clear();
 							txBytes = 0;
@@ -811,7 +779,7 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
 						++iter;
 					}
 					if (!commitQ.empty()) {
-						wait(transactionInfoCommitActor(&tr, &commitQ));
+						co_await transactionInfoCommitActor(&tr, &commitQ);
 						commitQ.clear();
 						txBytes = 0;
 					}
@@ -829,7 +797,7 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
 				}
 			}
 			cx->clientStatusUpdater.outStatusQ.clear();
-			wait(cx->globalConfig->onInitialized());
+			co_await cx->globalConfig->onInitialized();
 			double sampleRate =
 			    cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
 			double clientSamplingProbability =
@@ -837,28 +805,30 @@ ACTOR static Future<Void> clientStatusUpdateActor(DatabaseContext* cx) {
 			int64_t sizeLimit = cx->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
 			int64_t clientTxnInfoSizeLimit = sizeLimit == -1 ? CLIENT_KNOBS->CSI_SIZE_LIMIT : sizeLimit;
 			if (!trChunksQ.empty() && deterministicRandom()->random01() < clientSamplingProbability)
-				wait(delExcessClntTxnEntriesActor(&tr, clientTxnInfoSizeLimit));
+				co_await delExcessClntTxnEntriesActor(&tr, clientTxnInfoSizeLimit);
 
 			// Cleanup Transaction sooner than later, so that we don't hold reference to context.
 			tr = Transaction();
-			wait(delay(CLIENT_KNOBS->CSI_STATUS_DELAY));
+			co_await delay(CLIENT_KNOBS->CSI_STATUS_DELAY);
+			continue;
 		} catch (Error& e) {
-			TraceEvent(SevWarnAlways, "UnableToWriteClientStatus").error(e);
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
-			}
-			cx->clientStatusUpdater.outStatusQ.clear();
-
-			// Cleanup Transaction sooner than later, so that we don't hold reference to context.
-			tr = Transaction();
-			wait(delay(10.0));
+			err = e;
 		}
+		TraceEvent(SevWarnAlways, "UnableToWriteClientStatus").error(err);
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		cx->clientStatusUpdater.outStatusQ.clear();
+
+		// Cleanup Transaction sooner than later, so that we don't hold reference to context.
+		tr = Transaction();
+		co_await delay(10.0);
 	}
 }
 
-ACTOR Future<Void> assertFailure(GrvProxyInterface remote, Future<ErrorOr<GetReadVersionReply>> reply) {
+Future<Void> assertFailure(GrvProxyInterface remote, Future<ErrorOr<GetReadVersionReply>> reply) {
 	try {
-		ErrorOr<GetReadVersionReply> res = wait(reply);
+		ErrorOr<GetReadVersionReply> res = co_await reply;
 		if (!res.isError()) {
 			TraceEvent(SevError, "GotStaleReadVersion")
 			    .detail("Remote", remote.getConsistentReadVersion.getEndpoint().addresses.address.toString())
@@ -872,7 +842,6 @@ ACTOR Future<Void> assertFailure(GrvProxyInterface remote, Future<ErrorOr<GetRea
 		}
 		// we want this to fail -- so getting here is good, we'll just ignore the error.
 	}
-	return Void();
 }
 
 Future<Void> attemptGRVFromOldProxies(std::vector<GrvProxyInterface> oldProxies,
@@ -883,7 +852,7 @@ Future<Void> attemptGRVFromOldProxies(std::vector<GrvProxyInterface> oldProxies,
 	std::vector<Future<Void>> replies;
 	replies.reserve(oldProxies.size());
 	GetReadVersionRequest req(
-	    span.context, 1, TransactionPriority::IMMEDIATE, GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY);
+	    span.context, 1, TransactionPriority::IMMEDIATE, invalidVersion, GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY);
 	TraceEvent evt("AttemptGRVFromOldProxies");
 	evt.detail("NumOldProxies", oldProxies.size()).detail("NumNewProxies", newProxies.size());
 	auto traceProxies = [&](std::vector<GrvProxyInterface>& proxies, std::string const& key) {
@@ -902,45 +871,45 @@ Future<Void> attemptGRVFromOldProxies(std::vector<GrvProxyInterface> oldProxies,
 	return waitForAll(replies);
 }
 
-ACTOR static Future<Void> monitorClientDBInfoChange(DatabaseContext* cx,
-                                                    Reference<AsyncVar<ClientDBInfo> const> clientDBInfo,
-                                                    AsyncTrigger* proxiesChangeTrigger) {
-	state std::vector<CommitProxyInterface> curCommitProxies;
-	state std::vector<GrvProxyInterface> curGrvProxies;
-	state ActorCollection actors(false);
-	state Future<Void> clientDBInfoOnChange = clientDBInfo->onChange();
+static Future<Void> monitorClientDBInfoChange(DatabaseContext* cx,
+                                              Reference<AsyncVar<ClientDBInfo> const> clientDBInfo,
+                                              AsyncTrigger* proxiesChangeTrigger) {
+	std::vector<CommitProxyInterface> curCommitProxies;
+	std::vector<GrvProxyInterface> curGrvProxies;
+	ActorCollection actors(false);
+	Future<Void> clientDBInfoOnChange = clientDBInfo->onChange();
 	curCommitProxies = clientDBInfo->get().commitProxies;
 	curGrvProxies = clientDBInfo->get().grvProxies;
 
-	loop {
-		choose {
-			when(wait(clientDBInfoOnChange)) {
-				clientDBInfoOnChange = clientDBInfo->onChange();
-				if (clientDBInfo->get().commitProxies != curCommitProxies ||
-				    clientDBInfo->get().grvProxies != curGrvProxies) {
-					// This condition is a bit complicated. Here we want to verify that we're unable to receive a read
-					// version from a proxy of an old generation after a successful recovery. The conditions are:
-					// 1. We only do this with a configured probability.
-					// 2. If the old set of Grv proxies is empty, there's nothing to do
-					// 3. If the new set of Grv proxies is empty, it means the recovery is not complete. So if an old
-					//    Grv proxy still gives out read versions, this would be correct behavior.
-					// 4. If we see a provisional proxy, it means the recovery didn't complete yet, so the same as (3)
-					//    applies.
-					if (deterministicRandom()->random01() < cx->verifyCausalReadsProp && !curGrvProxies.empty() &&
-					    !clientDBInfo->get().grvProxies.empty() && !clientDBInfo->get().grvProxies[0].provisional) {
-						actors.add(attemptGRVFromOldProxies(curGrvProxies, clientDBInfo->get().grvProxies));
-					}
-					curCommitProxies = clientDBInfo->get().commitProxies;
-					curGrvProxies = clientDBInfo->get().grvProxies;
-					// Commits in the previous epoch may have been recovered but not included in the version vector.
-					// Clear the version vector to ensure the latest commit versions are received.
-					cx->ssVersionVectorCache.clear();
-					proxiesChangeTrigger->trigger();
+	while (true) {
+		auto res = co_await race(clientDBInfoOnChange, actors.getResult());
+		if (res.index() == 0) {
+			clientDBInfoOnChange = clientDBInfo->onChange();
+			if (clientDBInfo->get().commitProxies != curCommitProxies ||
+			    clientDBInfo->get().grvProxies != curGrvProxies) {
+				// This condition is a bit complicated. Here we want to verify that we're unable to receive a read
+				// version from a proxy of an old generation after a successful recovery. The conditions are:
+				// 1. We only do this with a configured probability.
+				// 2. If the old set of Grv proxies is empty, there's nothing to do
+				// 3. If the new set of Grv proxies is empty, it means the recovery is not complete. So if an old
+				//    Grv proxy still gives out read versions, this would be correct behavior.
+				// 4. If we see a provisional proxy, it means the recovery didn't complete yet, so the same as (3)
+				//    applies.
+				if (deterministicRandom()->random01() < cx->verifyCausalReadsProp && !curGrvProxies.empty() &&
+				    !clientDBInfo->get().grvProxies.empty() && !clientDBInfo->get().grvProxies[0].provisional) {
+					actors.add(attemptGRVFromOldProxies(curGrvProxies, clientDBInfo->get().grvProxies));
 				}
+				curCommitProxies = clientDBInfo->get().commitProxies;
+				curGrvProxies = clientDBInfo->get().grvProxies;
+				// Commits in the previous epoch may have been recovered but not included in the version vector.
+				// Clear the version vector to ensure the latest commit versions are received.
+				cx->ssVersionVectorCache.clear();
+				proxiesChangeTrigger->trigger();
 			}
-			when(wait(actors.getResult())) {
-				UNSTOPPABLE_ASSERT(false);
-			}
+		} else if (res.index() == 1) {
+			UNSTOPPABLE_ASSERT(false);
+		} else {
+			UNREACHABLE();
 		}
 	}
 }
@@ -980,19 +949,19 @@ Reference<LocationInfo> addCaches(const Reference<LocationInfo>& loc,
 	return makeReference<LocationInfo>(interfaces, true);
 }
 
-ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
-	state Reference<ReadYourWritesTransaction> tr;
-	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
-	state KeyBackedMap<Tuple, std::string> tssMismatchDB = KeyBackedMap<Tuple, std::string>(tssMismatchKeys.begin);
-	loop {
+static Future<Void> handleTssMismatches(DatabaseContext* cx) {
+	Reference<ReadYourWritesTransaction> tr;
+	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
+	KeyBackedMap<Tuple, std::string> tssMismatchDB = KeyBackedMap<Tuple, std::string>(tssMismatchKeys.begin);
+	while (true) {
 		// return to calling actor, cx might be destroyed already with the tr reset below.
 		// This gives ~DatabaseContext a chance to cancel this actor.
-		wait(delay(0));
+		co_await delay(0);
 
 		// <tssid, list of detailed mismatch data>
-		state std::pair<UID, std::vector<DetailedTSSMismatch>> data = waitNext(cx->tssMismatchStream.getFuture());
+		std::pair<UID, std::vector<DetailedTSSMismatch>> data = co_await cx->tssMismatchStream.getFuture();
 		// find ss pair id so we can remove it from the mapping
-		state UID tssPairID;
+		UID tssPairID;
 		bool found = false;
 		for (const auto& it : cx->tssMapping) {
 			if (it.second.id() == data.first) {
@@ -1002,15 +971,16 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 			}
 		}
 		if (found) {
-			state bool quarantine = CLIENT_KNOBS->QUARANTINE_TSS_ON_MISMATCH;
+			bool quarantine = CLIENT_KNOBS->QUARANTINE_TSS_ON_MISMATCH;
 			TraceEvent(SevWarnAlways, quarantine ? "TSS_QuarantineMismatch" : "TSS_KillMismatch")
 			    .detail("TSSID", data.first.toString());
 			CODE_PROBE(quarantine, "Quarantining TSS because it got mismatch");
 			CODE_PROBE(!quarantine, "Killing TSS because it got mismatch");
 
 			tr = makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(cx)));
-			state int tries = 0;
-			loop {
+			int tries = 0;
+			while (true) {
+				Error err;
 				try {
 					tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -1028,12 +998,13 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 						                  d.traceString);
 					}
 
-					wait(tr->commit());
+					co_await tr->commit();
 
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 				}
+				co_await tr->onError(err);
 				tries++;
 				if (tries > 10) {
 					// Give up, it'll get another mismatch or a human will investigate eventually
@@ -1049,8 +1020,6 @@ ACTOR static Future<Void> handleTssMismatches(DatabaseContext* cx) {
 	}
 }
 
-ACTOR Future<Optional<Value>> getJSON(Database db, std::string jsonField = "");
-
 struct SingleSpecialKeyImpl : SpecialKeyRangeReadImpl {
 	Future<RangeResult> getRange(ReadYourWritesTransaction* ryw,
 	                             KeyRangeRef kr,
@@ -1149,12 +1118,12 @@ static RangeResult healthMetricsToKVPairs(const HealthMetrics& metrics, KeyRange
 	return result;
 }
 
-ACTOR static Future<RangeResult> healthMetricsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
-	HealthMetrics metrics = wait(ryw->getDatabase()->getHealthMetrics(
+static Future<RangeResult> healthMetricsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+	HealthMetrics metrics = co_await ryw->getDatabase()->getHealthMetrics(
 	    /*detailed ("per process")*/ kr.intersects(
 	        KeyRangeRef("\xff\xff/metrics/health/storage/"_sr, "\xff\xff/metrics/health/storage0"_sr)) ||
-	    kr.intersects(KeyRangeRef("\xff\xff/metrics/health/log/"_sr, "\xff\xff/metrics/health/log0"_sr))));
-	return healthMetricsToKVPairs(metrics, kr);
+	    kr.intersects(KeyRangeRef("\xff\xff/metrics/health/log/"_sr, "\xff\xff/metrics/health/log0"_sr)));
+	co_return healthMetricsToKVPairs(metrics, kr);
 }
 
 HealthMetricsRangeImpl::HealthMetricsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
@@ -1165,11 +1134,11 @@ Future<RangeResult> HealthMetricsRangeImpl::getRange(ReadYourWritesTransaction*
 	return healthMetricsGetRangeActor(ryw, kr);
 }
 
-ACTOR Future<UID> getClusterId(Database db) {
+Future<UID> getClusterId(Database db) {
 	while (!db->clientInfo->get().clusterId.isValid()) {
-		wait(db->clientInfo->onChange());
+		co_await db->clientInfo->onChange();
 	}
-	return db->clientInfo->get().clusterId;
+	co_return db->clientInfo->get().clusterId;
 }
 
 void DatabaseContext::initializeSpecialCounters() {
@@ -1471,7 +1440,8 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
 			        return Optional<Value>();
 		        }));
 	}
-	throttleExpirer = recurring([this]() { expireThrottles(); }, CLIENT_KNOBS->TAG_THROTTLE_EXPIRATION_INTERVAL);
+	throttleExpirer = recurring(std::bind_front(&DatabaseContext::expireThrottles, this),
+	                            CLIENT_KNOBS->TAG_THROTTLE_EXPIRATION_INTERVAL);
 
 	if (BUGGIFY) {
 		DatabaseContext::debugUseTags = true;
@@ -1546,7 +1516,6 @@ DatabaseContext::~DatabaseContext() {
 	clientDBInfoMonitor.cancel();
 	monitorTssInfoChange.cancel();
 	tssMismatchHandler.cancel();
-	storage = nullptr;
 
 	if (grvUpdateHandler.isValid()) {
 		grvUpdateHandler.cancel();
diff --git a/fdbclient/FDBAWSCredentialsProvider.cpp b/fdbclient/FDBAWSCredentialsProvider.cpp
index d0324bb11c3..7cde0de401a 100644
--- a/fdbclient/FDBAWSCredentialsProvider.cpp
+++ b/fdbclient/FDBAWSCredentialsProvider.cpp
@@ -19,7 +19,7 @@
  * limitations under the License.
  */
 
-#include "fdbclient/FDBAWSCredentialsProvider.h"
+#include "FDBAWSCredentialsProvider.h"
 #include "fdbclient/Tracing.h"
 
 #ifdef WITH_AWS_BACKUP
diff --git a/fdbclient/include/fdbclient/FDBAWSCredentialsProvider.h b/fdbclient/FDBAWSCredentialsProvider.h
similarity index 100%
rename from fdbclient/include/fdbclient/FDBAWSCredentialsProvider.h
rename to fdbclient/FDBAWSCredentialsProvider.h
diff --git a/fdbclient/FDBTypes.cpp b/fdbclient/FDBTypes.cpp
index 6e8a1bf749b..c86acd86b08 100644
--- a/fdbclient/FDBTypes.cpp
+++ b/fdbclient/FDBTypes.cpp
@@ -75,8 +75,7 @@ Key randomKeyBetween(const KeyRangeRef& keys) {
 	ASSERT_LT(pos, end.size()); // otherwise, begin >= end
 
 	// find the lowest char in range begin[pos+1, begin.size()) that is not \xff (255)
-	int lowest = begin.size() - 1;
-	for (; lowest > pos; lowest--) {
+	for (int lowest = begin.size() - 1; lowest > pos; lowest--) {
 		if (begin[lowest] < 255) {
 			Key res = begin;
 			uint8_t* ptr = mutateString(res);
@@ -206,7 +205,7 @@ TEST_CASE("/KeyRangeUtil/KeyRangeComplement") {
 		Key b = "a"_sr;
 		Key e = "z"_sr;
 		std::vector<KeyRangeRef> result = range - KeyRangeRef(b, e);
-		ASSERT(result.size() == 0);
+		ASSERT(result.empty());
 	}
 
 	return Void();
@@ -396,4 +395,4 @@ TEST_CASE("/PerpetualStorageWiggleLocality/ParsePerpetualStorageWiggleLocality")
 		}
 	}
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.cpp
similarity index 61%
rename from fdbclient/FileBackupAgent.actor.cpp
rename to fdbclient/FileBackupAgent.cpp
index b0637db904c..99423c0ba56 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.cpp
@@ -1,5 +1,5 @@
 /*
- * FileBackupAgent.actor.cpp
+ * FileBackupAgent.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,7 +25,7 @@
 #include "flow/FastRef.h"
 #include "flow/flow.h"
 #include "fmt/format.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/BulkDumping.h"
@@ -35,17 +35,18 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/JsonBuilder.h"
 #include "fdbclient/JSONDoc.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/Knobs.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/PartitionedLogIterator.h"
-#include "fdbclient/RestoreInterface.h"
+#include "fdbclient/ManagementAPI.h"
+#include "PartitionedLogIterator.h"
+#include "RestoreInterface.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TaskBucket.h"
 #include "flow/network.h"
 #include "flow/Trace.h"
+#include "flow/Util.h"
 
 #include <cinttypes>
 #include <cstdint>
@@ -64,91 +65,67 @@
 #include <unordered_map>
 #include <utility>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 // Counters to verify BulkDump/BulkLoad were actually used (for test assertions)
 std::atomic<int> g_bulkDumpTaskCompleteCount(0);
 std::atomic<int> g_bulkLoadRestoreTaskCompleteCount(0);
 
 // Helper function to monitor BulkDump job completion
 // Returns true if job completed successfully, false if timed out
-ACTOR Future<bool> monitorBulkDumpJobCompletion(Database cx, UID jobId, double timeoutDuration, double pollInterval) {
-	state double timeoutStart = now();
-	state Transaction tr(cx);
+Future<bool> monitorBulkDumpJobCompletion(Database cx, UID jobId, double timeoutDuration, double pollInterval) {
+	double timeoutStart = now();
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			Optional<BulkDumpState> currentJob = wait(getSubmittedBulkDumpJob(&tr));
+			Optional<BulkDumpState> currentJob = co_await getSubmittedBulkDumpJob(&tr);
 			bool stillRunning = currentJob.present() && currentJob.get().getJobId() == jobId;
 
 			if (!stillRunning) {
-				return true; // Job completed successfully
+				co_return true; // Job completed successfully
 			}
 
 			if (now() - timeoutStart > timeoutDuration) {
-				return false; // Timed out
+				co_return false; // Timed out
 			}
 
-			wait(delay(pollInterval));
+			co_await delay(pollInterval);
 			tr.reset();
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-}
-
-// Helper function to monitor BulkLoad job completion
-// Returns true if job completed successfully, false if timed out
-// lockAware must be true when DB is locked (e.g., during restore)
-ACTOR Future<bool> monitorBulkLoadJobCompletion(Database cx,
-                                                UID jobId,
-                                                double timeoutDuration,
-                                                double pollInterval,
-                                                bool lockAware) {
-	state double timeoutStart = now();
-
-	loop {
-		Optional<BulkLoadJobState> currentJob = wait(getRunningBulkLoadJob(cx, lockAware));
-		bool stillRunning = currentJob.present() && currentJob.get().getJobId() == jobId;
-
-		if (!stillRunning) {
-			return true; // Job completed successfully
-		}
-
-		if (now() - timeoutStart > timeoutDuration) {
-			return false; // Timed out
+			err = e;
 		}
-
-		wait(delay(pollInterval));
+		co_await tr.onError(err);
 	}
 }
 
 // Verify that a complete BulkDump dataset exists for BulkLoad restore
 // Returns true if dataset is complete, false if incomplete
 // JobId corresponds to a single BulkDump job which represents one snapshot at a specific version
-ACTOR Future<bool> verifyBulkDumpDatasetCompleteness(Reference<IBackupContainer> bc, std::string bulkDumpJobId) {
+Future<bool> verifyBulkDumpDatasetCompleteness(Reference<IBackupContainer> bc, std::string bulkDumpJobId) {
 	try {
 		if (bulkDumpJobId.empty()) {
 			TraceEvent(SevWarn, "BulkLoadVerifyDatasetEmptyJobId");
-			return false;
+			co_return false;
 		}
 
 		// Check if job-specific directory exists: bulkdump_data/<job-uuid>/
 		// BulkDump stores data under data/<container>/bulkdump_data/ via getBackupDataPath(),
 		// which is consistent with where BackupContainer stores other files (logs, ranges, etc.)
-		state std::string jobDirectoryPath = "bulkdump_data/" + bulkDumpJobId + "/";
+		std::string jobDirectoryPath = "bulkdump_data/" + bulkDumpJobId + "/";
 
+		Error err;
 		try {
 			// Try to list files in the job directory to verify it exists and has content
 			Reference<BackupContainerFileSystem> bcfs = bc.castTo<BackupContainerFileSystem>();
 			if (bcfs) {
 				// Standard listFiles works because BulkDump now writes under data/<container>/
-				BackupContainerFileSystem::FilesAndSizesT files = wait(bcfs->listFiles(jobDirectoryPath));
+				BackupContainerFileSystem::FilesAndSizesT files = co_await bcfs->listFiles(jobDirectoryPath);
 				if (files.empty()) {
 					TraceEvent(SevWarn, "BulkLoadVerifyDatasetJobDirectoryEmpty")
 					    .detail("JobDirectoryPath", jobDirectoryPath)
 					    .detail("BulkDumpJobId", bulkDumpJobId);
-					return false;
+					co_return false;
 				}
 
 				TraceEvent("BulkLoadVerifyDatasetJobDirectoryFound")
@@ -158,28 +135,28 @@ ACTOR Future<bool> verifyBulkDumpDatasetCompleteness(Reference<IBackupContainer>
 
 				// Basic verification: job directory exists and contains files
 				// More detailed verification (parsing shard manifests) is delegated to BulkLoad system
-				return true;
+				co_return true;
 			} else {
 				// Cannot verify - backup container doesn't support file listing
 				// Use SevWarn (not SevError) to avoid abort in simulation
 				TraceEvent(SevWarn, "BulkLoadVerifyDatasetCannotCast")
 				    .detail("BulkDumpJobId", bulkDumpJobId)
 				    .detail("Note", "BackupContainer does not support file listing required for verification");
-				return false;
+				co_return false;
 			}
 		} catch (Error& e) {
 			if (e.code() == error_code_file_not_found) {
 				TraceEvent(SevWarn, "BulkLoadVerifyDatasetJobDirectoryNotFound")
 				    .detail("JobDirectoryPath", jobDirectoryPath)
 				    .detail("BulkDumpJobId", bulkDumpJobId);
-				return false;
+				co_return false;
 			}
 			throw;
 		}
 
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "BulkLoadVerifyDatasetError").error(e).detail("BulkDumpJobId", bulkDumpJobId);
-		return false;
+		co_return false;
 	}
 }
 
@@ -236,7 +213,7 @@ std::string secondsToTimeFormat(int64_t seconds) {
 const Key FileBackupAgent::keyLastRestorable = "last_restorable"_sr;
 
 // For convenience
-typedef FileBackupAgent::ERestoreState ERestoreState;
+using ERestoreState = FileBackupAgent::ERestoreState;
 
 StringRef FileBackupAgent::restoreStateText(ERestoreState id) {
 	switch (id) {
@@ -262,15 +239,15 @@ Key FileBackupAgent::getPauseKey() {
 	return backupAgent.taskBucket->getPauseKey();
 }
 
-ACTOR Future<std::vector<KeyBackedTag>> TagUidMap::getAll_impl(TagUidMap* tagsMap,
-                                                               Reference<ReadYourWritesTransaction> tr,
-                                                               Snapshot snapshot) {
-	state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor
-	TagMap::RangeResultType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot));
+Future<std::vector<KeyBackedTag>> TagUidMap::getAll_impl(TagUidMap* tagsMap,
+                                                         Reference<ReadYourWritesTransaction> tr,
+                                                         Snapshot snapshot) {
+	Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor
+	TagMap::RangeResultType tagPairs = co_await tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot);
 	std::vector<KeyBackedTag> results;
 	for (auto& p : tagPairs.results)
 		results.push_back(KeyBackedTag(p.first, prefix));
-	return results;
+	co_return results;
 }
 
 KeyBackedTag::KeyBackedTag(std::string tagName, StringRef tagMapPrefix)
@@ -278,38 +255,69 @@ KeyBackedTag::KeyBackedTag(std::string tagName, StringRef tagMapPrefix)
     tagMapPrefix(tagMapPrefix) {}
 
 // Lists all backups and find if any partitioned backup is running.
-ACTOR Future<bool> anyPartitionedBackupRunning(Reference<ReadYourWritesTransaction> tr) {
+Future<bool> anyPartitionedBackupRunning(Reference<ReadYourWritesTransaction> tr) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	std::vector<KeyBackedTag> tags = wait(getAllBackupTags(tr));
+	std::vector<KeyBackedTag> tags = co_await getAllBackupTags(tr);
 
-	state std::vector<Future<Optional<UidAndAbortedFlagT>>> futures;
+	std::vector<Future<Optional<UidAndAbortedFlagT>>> futures;
 	for (const auto& tag : tags) {
 		futures.push_back(tag.get(tr));
 	}
 
-	wait(waitForAll(futures));
-	state int i = 0;
+	co_await waitForAll(futures);
+	int i = 0;
 	for (i = 0; i < futures.size(); i++) {
 		if (futures[i].get().present()) {
-			state Optional<bool> partitionedLog;
-			state EBackupState eState;
+			Optional<MutationLogType> mutationLogType;
+			EBackupState eState;
 			BackupConfig config(futures[i].get().get().first);
 
-			wait(store(eState, config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)) &&
-			     store(partitionedLog, config.partitionedLogEnabled().get(tr)));
-			if (FileBackupAgent::isRunnable(eState) && partitionedLog.present() && partitionedLog.get()) {
-				return true;
+			co_await (store(eState, config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)) &&
+			          store(mutationLogType, config.mutationLogType().get(tr)));
+			if (FileBackupAgent::isRunnable(eState) &&
+			    mutationLogType.orDefault(MutationLogType::DEFAULT) == MutationLogType::PARTITIONED_LOG) {
+				co_return true;
 			}
 		}
 	}
-	return false;
+	co_return false;
+}
+
+// Lists all backups and find if any range-partitioned backup is running.
+Future<bool> anyRangePartitionedBackupRunning(Reference<ReadYourWritesTransaction> tr) {
+	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+	std::vector<KeyBackedTag> tags = co_await getAllBackupTags(tr);
+
+	std::vector<Future<Optional<UidAndAbortedFlagT>>> futures;
+	for (const auto& tag : tags) {
+		futures.push_back(tag.get(tr));
+	}
+
+	co_await waitForAll(futures);
+	int i = 0;
+	for (i = 0; i < futures.size(); i++) {
+		if (futures[i].get().present()) {
+			Optional<MutationLogType> mutationLogType;
+			EBackupState eState;
+			BackupConfig config(futures[i].get().get().first);
+
+			co_await (store(eState, config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN)) &&
+			          store(mutationLogType, config.mutationLogType().get(tr)));
+			if (FileBackupAgent::isRunnable(eState) &&
+			    mutationLogType.orDefault(MutationLogType::DEFAULT) == MutationLogType::RANGE_PARTITIONED_LOG) {
+				co_return true;
+			}
+		}
+	}
+	co_return false;
 }
 
 class RestoreConfig : public KeyBackedTaskConfig {
 public:
-	RestoreConfig(UID uid = UID()) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, uid) {}
-	RestoreConfig(Reference<Task> task) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, task) {}
+	explicit RestoreConfig(UID uid = UID()) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, uid) {}
+	explicit RestoreConfig(Reference<Task> task) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, task) {}
 
 	KeyBackedProperty<ERestoreState> stateEnum() { return configSpace.pack(__FUNCTION__sr); }
 	Future<StringRef> stateText(Reference<ReadYourWritesTransaction> tr) {
@@ -321,7 +329,7 @@ class RestoreConfig : public KeyBackedTaskConfig {
 	KeyBackedProperty<bool> onlyApplyMutationLogs() { return configSpace.pack(__FUNCTION__sr); }
 	KeyBackedProperty<bool> inconsistentSnapshotOnly() { return configSpace.pack(__FUNCTION__sr); }
 	KeyBackedProperty<bool> unlockDBAfterRestore() { return configSpace.pack(__FUNCTION__sr); }
-	KeyBackedProperty<bool> transformPartitionedLog() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedProperty<MutationLogType> mutationLogType() { return configSpace.pack(__FUNCTION__sr); }
 	// BulkLoad integration properties
 	KeyBackedProperty<bool> useRangeFileRestore() { return configSpace.pack(__FUNCTION__sr); }
 	KeyBackedProperty<std::string> bulkDumpJobId() { return configSpace.pack(__FUNCTION__sr); }
@@ -353,20 +361,25 @@ class RestoreConfig : public KeyBackedTaskConfig {
 	KeyBackedBinaryValue<int64_t> fileCount() { return configSpace.pack(__FUNCTION__sr); }
 	// Total number of file blocks in the fileMap
 	KeyBackedBinaryValue<int64_t> fileBlockCount() { return configSpace.pack(__FUNCTION__sr); }
+	// BulkLoad sub-phase task counts for detailed progress tracking
+	KeyBackedBinaryValue<int64_t> bulkLoadSubmittedTasks() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedBinaryValue<int64_t> bulkLoadTriggeredTasks() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedBinaryValue<int64_t> bulkLoadRunningTasks() { return configSpace.pack(__FUNCTION__sr); }
+	KeyBackedBinaryValue<int64_t> bulkLoadTotalTasks() { return configSpace.pack(__FUNCTION__sr); }
 
 	Future<std::vector<KeyRange>> getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr) {
 		return getRestoreRangesOrDefault_impl(this, tr);
 	}
 
-	ACTOR static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfig* self,
-	                                                                          Reference<ReadYourWritesTransaction> tr) {
-		state std::vector<KeyRange> ranges;
-		state int batchSize = BUGGIFY ? 1 : CLIENT_KNOBS->RESTORE_RANGES_READ_BATCH;
-		state Optional<KeyRange> begin;
-		state Arena arena;
-		loop {
+	static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfig* self,
+	                                                                    Reference<ReadYourWritesTransaction> tr) {
+		std::vector<KeyRange> ranges;
+		int batchSize = BUGGIFY ? 1 : CLIENT_KNOBS->RESTORE_RANGES_READ_BATCH;
+		Optional<KeyRange> begin;
+		Arena arena;
+		while (true) {
 			KeyBackedSet<KeyRange>::RangeResultType rangeResult =
-			    wait(self->restoreRangeSet().getRange(tr, begin, {}, batchSize));
+			    co_await self->restoreRangeSet().getRange(tr, begin, {}, batchSize);
 			ranges.insert(ranges.end(), rangeResult.results.begin(), rangeResult.results.end());
 			if (!rangeResult.more) {
 				break;
@@ -377,14 +390,14 @@ class RestoreConfig : public KeyBackedTaskConfig {
 
 		// fall back to original fields if the new field is empty
 		if (ranges.empty()) {
-			std::vector<KeyRange> _ranges = wait(self->restoreRanges().getD(tr));
+			std::vector<KeyRange> _ranges = co_await self->restoreRanges().getD(tr);
 			ranges = _ranges;
 			if (ranges.empty()) {
-				KeyRange range = wait(self->restoreRange().getD(tr));
+				KeyRange range = co_await self->restoreRange().getD(tr);
 				ranges.push_back(range);
 			}
 		}
-		return ranges;
+		co_return ranges;
 	}
 
 	// Describes a file to load blocks from during restore.  Ordered by version and then fileName to enable
@@ -418,7 +431,7 @@ class RestoreConfig : public KeyBackedTaskConfig {
 		}
 	};
 
-	typedef KeyBackedSet<RestoreFile> FileSetT;
+	using FileSetT = KeyBackedSet<RestoreFile>;
 	FileSetT fileSet() { return configSpace.pack(__FUNCTION__sr); }
 
 	FileSetT logFileSet() { return configSpace.pack(__FUNCTION__sr); }
@@ -451,18 +464,17 @@ class RestoreConfig : public KeyBackedTaskConfig {
 
 	Key applyMutationsMapPrefix() { return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid); }
 
-	ACTOR static Future<int64_t> getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
-		state Future<Optional<Value>> beginVal =
-		    tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::True);
-		state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::True);
-		wait(success(beginVal) && success(endVal));
+	static Future<int64_t> getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
+		Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::True);
+		Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::True);
+		co_await (success(beginVal) && success(endVal));
 
 		if (!beginVal.get().present() || !endVal.get().present())
-			return 0;
+			co_return 0;
 
 		Version beginVersion = BinaryReader::fromStringRef<Version>(beginVal.get().get(), Unversioned());
 		Version endVersion = BinaryReader::fromStringRef<Version>(endVal.get().get(), Unversioned());
-		return endVersion - beginVersion;
+		co_return endVersion - beginVersion;
 	}
 
 	Future<int64_t> getApplyVersionLag(Reference<ReadYourWritesTransaction> tr) {
@@ -532,52 +544,199 @@ class RestoreConfig : public KeyBackedTaskConfig {
 		           });
 	}
 
-	ACTOR static Future<Version> getCurrentVersion_impl(RestoreConfig* self, Reference<ReadYourWritesTransaction> tr) {
-		state ERestoreState status = wait(self->stateEnum().getD(tr));
-		state Version version = -1;
+	static Future<Version> getCurrentVersion_impl(RestoreConfig* self, Reference<ReadYourWritesTransaction> tr) {
+		ERestoreState status = co_await self->stateEnum().getD(tr);
+		Version version = -1;
 		if (status == ERestoreState::RUNNING) {
-			wait(store(version, self->getApplyBeginVersion(tr)));
+			version = co_await self->getApplyBeginVersion(tr);
 		} else if (status == ERestoreState::COMPLETED) {
-			wait(store(version, self->restoreVersion().getD(tr)));
+			version = co_await self->restoreVersion().getD(tr);
 		}
-		return version;
+		co_return version;
 	}
 
 	Future<Version> getCurrentVersion(Reference<ReadYourWritesTransaction> tr) {
 		return getCurrentVersion_impl(this, tr);
 	}
 
-	ACTOR static Future<std::string> getProgress_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr);
+	static Future<std::string> getProgress_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr);
 	Future<std::string> getProgress(Reference<ReadYourWritesTransaction> tr) { return getProgress_impl(*this, tr); }
 
-	ACTOR static Future<std::string> getFullStatus_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr);
+	static Future<std::string> getFullStatus_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr);
 	Future<std::string> getFullStatus(Reference<ReadYourWritesTransaction> tr) { return getFullStatus_impl(*this, tr); }
 };
 
-typedef RestoreConfig::RestoreFile RestoreFile;
+using RestoreFile = RestoreConfig::RestoreFile;
+
+// Helper to count bulkload task progress for a job
+// Returns: <completedTasks, submittedTasks, triggeredTasks, runningTasks, totalTasks, completedBytes>
+// Sub-phases help track progress during the long "running" period:
+//   - Submitted: task created, waiting to be picked up by DD
+//   - Triggered: assigned to storage server, waiting to start
+//   - Running: storage server actively downloading/ingesting SST files
+Future<std::tuple<int64_t, int64_t, int64_t, int64_t, int64_t, int64_t>> getBulkLoadTaskProgress(Database cx,
+                                                                                                 UID jobId) {
+	Transaction tr(cx);
+	Key readBegin = normalKeys.begin;
+	Key readEnd = normalKeys.end;
+	int64_t completedTasks = 0;
+	int64_t submittedTasks = 0;
+	int64_t triggeredTasks = 0;
+	int64_t runningTasks = 0;
+	int64_t totalTasks = 0;
+	int64_t completedBytes = 0;
+
+	while (readBegin < readEnd) {
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			RangeResult rangeResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(readBegin, readEnd));
+			if (rangeResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
+				if (rangeResult[i].value.empty()) {
+					continue;
+				}
+				BulkLoadTaskState task = decodeBulkLoadTaskState(rangeResult[i].value);
+				if (task.getJobId() != jobId) {
+					// Different job, stop counting
+					co_return std::make_tuple(
+					    completedTasks, submittedTasks, triggeredTasks, runningTasks, totalTasks, completedBytes);
+				}
+				int manifestCount = task.getManifests().size();
+				totalTasks += manifestCount;
+				if (task.phase == BulkLoadPhase::Complete) {
+					completedTasks += manifestCount;
+					// Sum bytes from manifest data sizes
+					for (const auto& manifest : task.getManifests()) {
+						completedBytes += manifest.getTotalBytes();
+					}
+				} else if (task.phase == BulkLoadPhase::Submitted) {
+					submittedTasks += manifestCount;
+				} else if (task.phase == BulkLoadPhase::Triggered) {
+					triggeredTasks += manifestCount;
+				} else if (task.phase == BulkLoadPhase::Running) {
+					runningTasks += manifestCount;
+				}
+			}
+			readBegin = rangeResult.back().key;
+		} catch (Error& e) {
+			err = e;
+		}
+		if (err.isValid() && err.code() != error_code_success) {
+			TraceEvent(SevWarn, "BulkLoadTaskProgressRetry").error(err).detail("JobId", jobId);
+			co_await tr.onError(err);
+		}
+	}
+	co_return std::make_tuple(completedTasks, submittedTasks, triggeredTasks, runningTasks, totalTasks, completedBytes);
+}
+
+// Monitor BulkLoad job completion and update restore progress counters
+// restoreUid is used to update the RestoreConfig progress
+Future<bool> monitorBulkLoadJobCompletionWithProgress(Database cx,
+                                                      UID jobId,
+                                                      UID restoreUid,
+                                                      int64_t totalBlocks,
+                                                      double timeoutDuration,
+                                                      double pollInterval,
+                                                      bool lockAware) {
+	double timeoutStart = now();
+	RestoreConfig restore(restoreUid);
+
+	while (true) {
+		Optional<BulkLoadJobState> currentJob = co_await getRunningBulkLoadJob(cx, lockAware);
+		bool stillRunning = currentJob.present() && currentJob.get().getJobId() == jobId;
+
+		if (!stillRunning) {
+			co_return true;
+		}
+
+		// Update progress based on completed bulkload tasks
+		try {
+			auto [completed, submitted, triggered, running, total, bytes] = co_await getBulkLoadTaskProgress(cx, jobId);
+			if (total > 0) {
+				// For bulkload restores, fileBlockCount is 0, so use task count as "blocks"
+				// This provides meaningful progress tracking for the restore status display
+				// Include all in-progress tasks in dispatched count to show scheduling progress
+				int64_t inProgress = submitted + triggered + running;
+				int64_t effectiveTotalBlocks = totalBlocks > 0 ? totalBlocks : total;
+				int64_t blocksFinished = totalBlocks > 0 ? (totalBlocks * completed) / total : completed;
+				int64_t blocksDispatched =
+				    totalBlocks > 0 ? (totalBlocks * (completed + inProgress)) / total : (completed + inProgress);
+
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				if (lockAware) {
+					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				}
+				restore.fileBlocksFinished().set(tr, blocksFinished);
+				restore.filesBlocksDispatched().set(tr, blocksDispatched);
+				restore.fileBlockCount().set(tr, effectiveTotalBlocks);
+				restore.bytesWritten().set(tr, bytes);
+				// Store sub-phase counts for detailed progress display
+				restore.bulkLoadSubmittedTasks().set(tr, submitted);
+				restore.bulkLoadTriggeredTasks().set(tr, triggered);
+				restore.bulkLoadRunningTasks().set(tr, running);
+				restore.bulkLoadTotalTasks().set(tr, total);
+				co_await tr->commit();
+
+				TraceEvent("BulkLoadRestoreProgress")
+				    .detail("RestoreUID", restoreUid)
+				    .detail("JobId", jobId)
+				    .detail("CompletedTasks", completed)
+				    .detail("SubmittedTasks", submitted)
+				    .detail("TriggeredTasks", triggered)
+				    .detail("RunningTasks", running)
+				    .detail("TotalTasks", total)
+				    .detail("BlocksFinished", blocksFinished)
+				    .detail("BlocksDispatched", blocksDispatched)
+				    .detail("EffectiveTotalBlocks", effectiveTotalBlocks)
+				    .detail("BytesWritten", bytes);
+			}
+		} catch (Error& e) {
+			// Log but don't fail - progress updates are best-effort
+			TraceEvent(SevWarn, "BulkLoadRestoreProgressError").error(e).detail("JobId", jobId);
+		}
+
+		if (now() - timeoutStart > timeoutDuration) {
+			co_return false; // Timed out
+		}
+
+		co_await delay(pollInterval);
+	}
+}
 
-ACTOR Future<std::string> RestoreConfig::getProgress_impl(RestoreConfig restore,
-                                                          Reference<ReadYourWritesTransaction> tr) {
+Future<std::string> RestoreConfig::getProgress_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state Future<int64_t> fileCount = restore.fileCount().getD(tr);
-	state Future<int64_t> fileBlockCount = restore.fileBlockCount().getD(tr);
-	state Future<int64_t> fileBlocksDispatched = restore.filesBlocksDispatched().getD(tr);
-	state Future<int64_t> fileBlocksFinished = restore.fileBlocksFinished().getD(tr);
-	state Future<int64_t> bytesWritten = restore.bytesWritten().getD(tr);
-	state Future<StringRef> status = restore.stateText(tr);
-	state Future<Version> currentVersion = restore.getCurrentVersion(tr);
-	state Future<Version> lag = restore.getApplyVersionLag(tr);
-	state Future<Version> firstConsistentVersion = restore.firstConsistentVersion().getD(tr);
-	state Future<std::string> tag = restore.tag().getD(tr);
-	state Future<std::pair<std::string, Version>> lastError = restore.lastError().getD(tr);
-
-	// restore might no longer be valid after the first wait so make sure it is not needed anymore.
-	state UID uid = restore.getUid();
-	wait(success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) &&
-	     success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(currentVersion) &&
-	     success(lag) && success(firstConsistentVersion) && success(tag) && success(lastError));
+	Future<int64_t> fileCount = restore.fileCount().getD(tr);
+	Future<int64_t> fileBlockCount = restore.fileBlockCount().getD(tr);
+	Future<int64_t> fileBlocksDispatched = restore.filesBlocksDispatched().getD(tr);
+	Future<int64_t> fileBlocksFinished = restore.fileBlocksFinished().getD(tr);
+	Future<int64_t> bytesWritten = restore.bytesWritten().getD(tr);
+	Future<StringRef> status = restore.stateText(tr);
+	Future<Version> currentVersion = restore.getCurrentVersion(tr);
+	Future<Version> lag = restore.getApplyVersionLag(tr);
+	Future<Version> firstConsistentVersion = restore.firstConsistentVersion().getD(tr);
+	Future<std::string> tag = restore.tag().getD(tr);
+	Future<std::pair<std::string, Version>> lastError = restore.lastError().getD(tr);
+	Future<int64_t> submittedTasks = restore.bulkLoadSubmittedTasks().getD(tr);
+	Future<int64_t> triggeredTasks = restore.bulkLoadTriggeredTasks().getD(tr);
+	Future<int64_t> runningTasks = restore.bulkLoadRunningTasks().getD(tr);
+	Future<int64_t> totalTasks = restore.bulkLoadTotalTasks().getD(tr);
+	Future<Optional<bool>> useRangeFileRestore = restore.useRangeFileRestore().get(tr);
+
+	UID uid = restore.getUid();
+	co_await (success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) &&
+	          success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(currentVersion) &&
+	          success(lag) && success(firstConsistentVersion) && success(tag) && success(lastError) &&
+	          success(submittedTasks) && success(triggeredTasks) && success(runningTasks) && success(totalTasks) &&
+	          success(useRangeFileRestore));
+
+	bool useRangeFile = !useRangeFileRestore.get().present() || useRangeFileRestore.get().get();
 
 	std::string errstr = "None";
 	if (lastError.get().second != 0)
@@ -593,44 +752,63 @@ ACTOR Future<std::string> RestoreConfig::getProgress_impl(RestoreConfig restore,
 	    .detail("FileBlocksFinished", fileBlocksFinished.get())
 	    .detail("FileBlocksTotal", fileBlockCount.get())
 	    .detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
+	    .detail("SubmittedTasks", submittedTasks.get())
+	    .detail("TriggeredTasks", triggeredTasks.get())
+	    .detail("RunningTasks", runningTasks.get())
+	    .detail("TotalTasks", totalTasks.get())
 	    .detail("BytesWritten", bytesWritten.get())
 	    .detail("CurrentVersion", currentVersion.get())
 	    .detail("FirstConsistentVersion", firstConsistentVersion.get())
-	    .detail("ApplyLag", lag.get())
-	    .detail("TaskInstance", THIS_ADDR);
-
-	return format("Tag: %s  UID: %s  State: %s  Blocks: %lld/%lld  BlocksInProgress: %lld  Files: %lld  BytesWritten: "
-	              "%lld  ApplyVersionLag: %lld  LastError: %s",
-	              tag.get().c_str(),
-	              uid.toString().c_str(),
-	              status.get().toString().c_str(),
-	              fileBlocksFinished.get(),
-	              fileBlockCount.get(),
-	              fileBlocksDispatched.get() - fileBlocksFinished.get(),
-	              fileCount.get(),
-	              bytesWritten.get(),
-	              lag.get(),
-	              errstr.c_str());
+	    .detail("ApplyLag", lag.get());
+
+	std::string progressStr;
+	if (useRangeFile) {
+		progressStr = format("Tag: %s  UID: %s  State: %s\n",
+		                     tag.get().c_str(),
+		                     uid.toString().c_str(),
+		                     status.get().toString().c_str());
+		progressStr += format(" Blocks: %lld/%lld complete\n", fileBlocksFinished.get(), fileBlockCount.get());
+		progressStr += format(" Files: %lld\n", fileCount.get());
+		progressStr += format(" Bytes written: %s\n", formatBytesHumanReadable(bytesWritten.get()).c_str());
+		progressStr += format(" Apply version lag: %s\n", versionToString(lag.get()).c_str());
+	} else {
+		progressStr = format("Tag: %s  UID: %s  State: %s\n",
+		                     tag.get().c_str(),
+		                     uid.toString().c_str(),
+		                     status.get().toString().c_str());
+		progressStr += format(" Tasks submitted: %lld  triggered: %lld  running: %lld\n",
+		                      submittedTasks.get(),
+		                      triggeredTasks.get(),
+		                      runningTasks.get());
+		progressStr += format(" Tasks triggered: %lld / %lld total\n", triggeredTasks.get(), totalTasks.get());
+		progressStr += format(" Bytes written: %s\n", formatBytesHumanReadable(bytesWritten.get()).c_str());
+		double avgBytesPerTask = triggeredTasks.get() > 0 ? (double)bytesWritten.get() / triggeredTasks.get() : 0;
+		if (avgBytesPerTask > 0) {
+			progressStr += format(" Avg bytes/task: %s\n", formatBytesHumanReadable((int64_t)avgBytesPerTask).c_str());
+		}
+	}
+
+	co_return progressStr;
 }
 
-ACTOR Future<std::string> RestoreConfig::getFullStatus_impl(RestoreConfig restore,
-                                                            Reference<ReadYourWritesTransaction> tr) {
+Future<std::string> RestoreConfig::getFullStatus_impl(RestoreConfig restore, Reference<ReadYourWritesTransaction> tr) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state Future<std::vector<KeyRange>> ranges = restore.getRestoreRangesOrDefault(tr);
-	state Future<Key> addPrefix = restore.addPrefix().getD(tr);
-	state Future<Key> removePrefix = restore.removePrefix().getD(tr);
-	state Future<Key> url = restore.sourceContainerURL().getD(tr);
-	state Future<Version> restoreVersion = restore.restoreVersion().getD(tr);
-	state Future<std::string> progress = restore.getProgress(tr);
-	state Future<ERestoreState> restoreState = restore.stateEnum().getD(tr);
-	state Future<Optional<bool>> useRangeFileRestore = restore.useRangeFileRestore().get(tr);
-	state Future<Optional<bool>> bulkLoadComplete = restore.bulkLoadComplete().get(tr);
+	Future<std::vector<KeyRange>> ranges = restore.getRestoreRangesOrDefault(tr);
+	Future<Key> addPrefix = restore.addPrefix().getD(tr);
+	Future<Key> removePrefix = restore.removePrefix().getD(tr);
+	Future<Key> url = restore.sourceContainerURL().getD(tr);
+	Future<Version> restoreVersion = restore.restoreVersion().getD(tr);
+	Future<std::string> progress = restore.getProgress(tr);
+	Future<ERestoreState> restoreState = restore.stateEnum().getD(tr);
+	Future<Optional<bool>> useRangeFileRestore = restore.useRangeFileRestore().get(tr);
+	Future<Optional<bool>> bulkLoadComplete = restore.bulkLoadComplete().get(tr);
 
 	// restore might no longer be valid after the first wait so make sure it is not needed anymore.
-	wait(success(ranges) && success(addPrefix) && success(removePrefix) && success(url) && success(restoreVersion) &&
-	     success(progress) && success(restoreState) && success(useRangeFileRestore) && success(bulkLoadComplete));
+	co_await (success(ranges) && success(addPrefix) && success(removePrefix) && success(url) &&
+	          success(restoreVersion) && success(progress) && success(restoreState) && success(useRangeFileRestore) &&
+	          success(bulkLoadComplete));
 
 	std::string returnStr;
 	returnStr = format("%s  URL: %s", progress.get().c_str(), url.get().toString().c_str());
@@ -643,15 +821,15 @@ ACTOR Future<std::string> RestoreConfig::getFullStatus_impl(RestoreConfig restor
 	                    restoreVersion.get());
 
 	// Add enhanced status fields for BulkLoad integration
-	// useRangeFileRestore is Future<Optional<bool>>, after wait() we call .get() to get Optional<bool>
 	bool usingBulkLoad = useRangeFileRestore.get().present() && !useRangeFileRestore.get().get();
 	std::string snapshotMethod = usingBulkLoad ? "bulkload" : "rangefile";
 	returnStr += format("  Snapshot Method: %s", snapshotMethod.c_str());
 
 	// Add phase status information
 	ERestoreState currentState = restoreState.get();
+	bool bulkLoadDone = bulkLoadComplete.get().present() && bulkLoadComplete.get().get();
+
 	if (currentState == ERestoreState::RUNNING) {
-		bool bulkLoadDone = bulkLoadComplete.get().present() && bulkLoadComplete.get().get();
 		if (usingBulkLoad) {
 			std::string snapshotPhase = bulkLoadDone ? "complete" : "in_progress";
 			returnStr += format("  Snapshot Phase: %s", snapshotPhase.c_str());
@@ -664,7 +842,7 @@ ACTOR Future<std::string> RestoreConfig::getFullStatus_impl(RestoreConfig restor
 		returnStr += "  Snapshot Phase: complete  Mutation Log Phase: complete";
 	}
 
-	return returnStr;
+	co_return returnStr;
 }
 
 // two buffers are alternatively serving data and reading data from file
@@ -681,7 +859,7 @@ class TwoBuffers : public ReferenceCounted<TwoBuffers>, NonCopyable {
 		size_t size;
 		int index;
 		int capacity;
-		IteratorBuffer(int _capacity) {
+		explicit IteratorBuffer(int _capacity) {
 			capacity = _capacity;
 			data = std::shared_ptr<char[]>(new char[capacity]());
 			fetchingData.reset();
@@ -699,10 +877,10 @@ class TwoBuffers : public ReferenceCounted<TwoBuffers>, NonCopyable {
 	// because a shared_ptr cannot be wrapped by a Future
 	// this method ensures the current buffer has available data
 	Future<Void> ready();
-	ACTOR static Future<Void> ready(Reference<TwoBuffers> self);
+	static Future<Void> ready(Reference<TwoBuffers> self);
 	// fill buffer[index] with the next block of file
 	// it has side effects to change currentFileIndex and currentFilePosition
-	ACTOR static Future<Void> readNextBlock(Reference<TwoBuffers> self, int index);
+	static Future<Void> readNextBlock(Reference<TwoBuffers> self, int index);
 	// peek can only be called after ready is called
 	// it returns the pointer to the active buffer
 	std::shared_ptr<char[]> peek();
@@ -762,19 +940,18 @@ Future<Void> TwoBuffers::ready() {
 	return ready(Reference<TwoBuffers>::addRef(this));
 }
 
-ACTOR Future<Void> TwoBuffers::ready(Reference<TwoBuffers> self) {
+Future<Void> TwoBuffers::ready(Reference<TwoBuffers> self) {
 	// if cur is not ready, then wait
 	if (!self->hasNext()) {
-		return Void();
+		co_return;
 	}
 	// try to fill the current buffer, and wait before it is filled
 	self->fillBufferIfAbsent(self->cur);
-	wait(self->buffers[self->cur]->fetchingData.value());
+	co_await self->buffers[self->cur]->fetchingData.value();
 	// try to fill the next buffer, do not wait for the filling
 	if (self->hasNext()) {
 		self->fillBufferIfAbsent(1 - self->cur);
 	}
-	return Void();
 }
 
 std::shared_ptr<char[]> TwoBuffers::peek() {
@@ -815,8 +992,7 @@ size_t TwoBuffers::getBufferSize() {
 }
 
 // only one readNextBlock can be run at a single time, otherwie the same block might be loaded twice
-ACTOR Future<Void> TwoBuffers::readNextBlock(Reference<TwoBuffers> self, int index) {
-	state Reference<IAsyncFile> asyncFile;
+Future<Void> TwoBuffers::readNextBlock(Reference<TwoBuffers> self, int index) {
 	if (self->currentFileIndex >= self->files.size()) {
 		TraceEvent(SevError, "ReadNextBlockOutOfBound")
 		    .detail("FileIndex", self->currentFileIndex)
@@ -825,22 +1001,19 @@ ACTOR Future<Void> TwoBuffers::readNextBlock(Reference<TwoBuffers> self, int ind
 		    .detail("FileSize", self->files[self->currentFileIndex].fileSize)
 		    .detail("FilesCount", self->files.size())
 		    .log();
-		return Void();
+		co_return;
 	}
-	Reference<IAsyncFile> asyncFileTmp = wait(self->bc->readFile(self->files[self->currentFileIndex].fileName));
-	asyncFile = asyncFileTmp;
-	state size_t fileSize = self->files[self->currentFileIndex].fileSize;
+	Reference<IAsyncFile> asyncFile = co_await self->bc->readFile(self->files[self->currentFileIndex].fileName);
+	size_t fileSize = self->files[self->currentFileIndex].fileSize;
 	size_t remaining = fileSize - self->currentFilePosition;
-	state size_t bytesToRead = std::min(self->bufferCapacity, remaining);
-	state int bytesRead = wait(
-	    asyncFile->read(static_cast<void*>(self->buffers[index]->data.get()), bytesToRead, self->currentFilePosition));
+	size_t bytesToRead = std::min(self->bufferCapacity, remaining);
+	int bytesRead = co_await asyncFile->read(
+	    static_cast<void*>(self->buffers[index]->data.get()), bytesToRead, self->currentFilePosition);
 	if (bytesRead != bytesToRead)
 		throw restore_bad_read();
 	self->buffers[index]->index = self->currentFileIndex;
 	self->buffers[index]->size = bytesRead; // Set to actual bytes read
 	self->currentFilePosition += bytesRead;
-
-	return Void();
 }
 
 void TwoBuffers::fillBufferIfAbsent(int index) {
@@ -883,22 +1056,21 @@ class PartitionedLogIteratorSimple : public PartitionedLogIterator {
 	                             std::vector<RestoreConfig::RestoreFile> _files,
 	                             std::vector<Version> _endVersions);
 
-	bool hasNext();
+	bool hasNext() override;
 	Future<Void> loadNextBlock();
-	ACTOR static Future<Void> loadNextBlock(Reference<PartitionedLogIteratorSimple> self);
+	static Future<Void> loadNextBlock(Reference<PartitionedLogIteratorSimple> self);
 	void removeBlockHeader();
 
 	Standalone<VectorRef<VersionedMutation>> consumeData(Version firstVersion);
 
 	// find the next version without advanding the iterator
-	Future<Version> peekNextVersion();
-	ACTOR static Future<Version> peekNextVersion(Reference<PartitionedLogIteratorSimple> iterator);
+	Future<Version> peekNextVersion() override;
+	static Future<Version> peekNextVersion(Reference<PartitionedLogIteratorSimple> iterator);
 
 	// get all the mutations of next version and advance the iterator
 	// this might issue multiple consumeData() if the data of a version cross buffer boundary
-	Future<Standalone<VectorRef<VersionedMutation>>> getNext();
-	ACTOR static Future<Standalone<VectorRef<VersionedMutation>>> getNext(
-	    Reference<PartitionedLogIteratorSimple> iterator);
+	Future<Standalone<VectorRef<VersionedMutation>>> getNext() override;
+	static Future<Standalone<VectorRef<VersionedMutation>>> getNext(Reference<PartitionedLogIteratorSimple> iterator);
 };
 
 PartitionedLogIteratorSimple::PartitionedLogIteratorSimple(Reference<IBackupContainer> _bc,
@@ -1005,42 +1177,38 @@ Future<Void> PartitionedLogIteratorSimple::loadNextBlock() {
 	return loadNextBlock(Reference<PartitionedLogIteratorSimple>::addRef(this));
 }
 
-ACTOR Future<Void> PartitionedLogIteratorSimple::loadNextBlock(Reference<PartitionedLogIteratorSimple> self) {
+Future<Void> PartitionedLogIteratorSimple::loadNextBlock(Reference<PartitionedLogIteratorSimple> self) {
 	if (self->bufferOffset < self->bufferSize) {
 		// do nothing
-		return Void();
+		co_return;
 	}
 	if (!self->hasNext()) {
-		return Void();
+		co_return;
 	}
-	state Reference<IAsyncFile> asyncFile;
-	Reference<IAsyncFile> asyncFileTmp = wait(self->bc->readFile(self->files[self->fileIndex].fileName));
-	asyncFile = asyncFileTmp;
-	state size_t fileSize = self->files[self->fileIndex].fileSize;
+	Reference<IAsyncFile> asyncFile = co_await self->bc->readFile(self->files[self->fileIndex].fileName);
+	size_t fileSize = self->files[self->fileIndex].fileSize;
 	size_t remaining = fileSize - self->fileOffset;
-	state size_t bytesToRead = std::min(self->bufferCapacity, remaining);
-	state int bytesRead =
-	    wait(asyncFile->read(static_cast<void*>((self->buffer.get())), bytesToRead, self->fileOffset));
+	size_t bytesToRead = std::min(self->bufferCapacity, remaining);
+	int bytesRead = co_await asyncFile->read(static_cast<void*>((self->buffer.get())), bytesToRead, self->fileOffset);
 	if (bytesRead != bytesToRead)
 		throw restore_bad_read();
 	self->bufferSize = bytesRead; // Set to actual bytes read
 	self->bufferOffset = 0; // Reset bufferOffset for the new data
 	self->fileOffset += bytesRead;
-	return Void();
 }
 
 Future<Version> PartitionedLogIteratorSimple::peekNextVersion() {
 	return peekNextVersion(Reference<PartitionedLogIteratorSimple>::addRef(this));
 }
 
-ACTOR Future<Version> PartitionedLogIteratorSimple::peekNextVersion(Reference<PartitionedLogIteratorSimple> self) {
+Future<Version> PartitionedLogIteratorSimple::peekNextVersion(Reference<PartitionedLogIteratorSimple> self) {
 	// Read the first mutation's version
 	if (!self->hasNext()) {
-		return Version(0);
+		co_return Version(0);
 	}
-	wait(self->loadNextBlock());
+	co_await self->loadNextBlock();
 	self->removeBlockHeader();
-	state Version version;
+	Version version{ 0 };
 	std::memcpy(&version, self->buffer.get() + self->bufferOffset, sizeof(Version));
 	version = bigEndian64(version);
 
@@ -1053,26 +1221,26 @@ ACTOR Future<Version> PartitionedLogIteratorSimple::peekNextVersion(Reference<Pa
 		self->bufferSize = 0;
 		self->fileOffset = 0;
 		self->fileIndex += 1;
-		wait(self->loadNextBlock());
+		co_await self->loadNextBlock();
 		self->removeBlockHeader();
 		std::memcpy(&version, self->buffer.get() + self->bufferOffset, sizeof(Version));
 		version = bigEndian64(version);
 	}
-	return version;
+	co_return version;
 }
 
-ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSimple::getNext(
+Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSimple::getNext(
     Reference<PartitionedLogIteratorSimple> self) {
-	state Standalone<VectorRef<VersionedMutation>> mutations;
+	Standalone<VectorRef<VersionedMutation>> mutations;
 	if (!self->hasNext()) {
 		TraceEvent(SevWarn, "SimpleIteratorExhausted")
 		    .detail("BufferOffset", self->bufferOffset)
 		    .detail("BufferSize", self->bufferSize)
 		    .detail("Tag", self->tag)
 		    .log();
-		return mutations;
+		co_return mutations;
 	}
-	state Version firstVersion = wait(self->peekNextVersion());
+	Version firstVersion = co_await self->peekNextVersion();
 	Standalone<VectorRef<VersionedMutation>> firstBatch = self->consumeData(firstVersion);
 	mutations = firstBatch;
 	// If the current buffer is fully consumed, then we need to check the next buffer in case
@@ -1084,7 +1252,7 @@ ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSim
 		if (self->hasNext()) {
 			// now this is run for each block, but it is not necessary if it is the last block of a file
 			// cannot check hasMoreData here because other buffer might have the last piece
-			wait(self->loadNextBlock());
+			co_await self->loadNextBlock();
 			Standalone<VectorRef<VersionedMutation>> batch = self->consumeData(firstVersion);
 			for (const VersionedMutation& vm : batch) {
 				mutations.push_back_deep(mutations.arena(), vm);
@@ -1093,7 +1261,7 @@ ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSim
 			break;
 		}
 	}
-	return mutations;
+	co_return mutations;
 }
 
 Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorSimple::getNext() {
@@ -1108,7 +1276,7 @@ class PartitionedLogIteratorTwoBuffers : public PartitionedLogIterator {
 	// stop if seeing a different version from the parameter.
 	// it has side effects to update bufferOffset after reading the data
 	Future<Standalone<VectorRef<VersionedMutation>>> consumeData(Version firstVersion);
-	ACTOR static Future<Standalone<VectorRef<VersionedMutation>>> consumeData(
+	static Future<Standalone<VectorRef<VersionedMutation>>> consumeData(
 	    Reference<PartitionedLogIteratorTwoBuffers> self,
 	    Version v);
 
@@ -1137,16 +1305,16 @@ class PartitionedLogIteratorTwoBuffers : public PartitionedLogIterator {
 	                                 std::vector<Version> _endVersions);
 
 	// whether there are more contents for this tag in all files specified
-	bool hasNext();
+	bool hasNext() override;
 
 	// find the next version without advanding the iterator
-	Future<Version> peekNextVersion();
-	ACTOR static Future<Version> peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers> iterator);
+	Future<Version> peekNextVersion() override;
+	static Future<Version> peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers> iterator);
 
 	// get all the mutations of next version and advance the iterator
 	// this might issue multiple consumeData() if the data of a version cross buffer boundary
-	Future<Standalone<VectorRef<VersionedMutation>>> getNext();
-	ACTOR static Future<Standalone<VectorRef<VersionedMutation>>> getNext(
+	Future<Standalone<VectorRef<VersionedMutation>>> getNext() override;
+	static Future<Standalone<VectorRef<VersionedMutation>>> getNext(
 	    Reference<PartitionedLogIteratorTwoBuffers> iterator);
 };
 
@@ -1154,11 +1322,11 @@ Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffer
 	return consumeData(Reference<PartitionedLogIteratorTwoBuffers>::addRef(this), firstVersion);
 }
 
-ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::consumeData(
+Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::consumeData(
     Reference<PartitionedLogIteratorTwoBuffers> self,
     Version firstVersion) {
-	state Standalone<VectorRef<VersionedMutation>> mutations = Standalone<VectorRef<VersionedMutation>>();
-	wait(self->twobuffer->ready());
+	Standalone<VectorRef<VersionedMutation>> mutations = Standalone<VectorRef<VersionedMutation>>();
+	co_await self->twobuffer->ready();
 	std::shared_ptr<char[]> start = self->twobuffer->peek();
 	int size = self->twobuffer->getBufferSize();
 	bool foundNewVersion = false;
@@ -1217,7 +1385,7 @@ ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwo
 			break;
 		}
 	}
-	return mutations;
+	co_return mutations;
 }
 
 void PartitionedLogIteratorTwoBuffers::removeBlockHeader() {
@@ -1242,16 +1410,15 @@ bool PartitionedLogIteratorTwoBuffers::hasNext() {
 Future<Version> PartitionedLogIteratorTwoBuffers::peekNextVersion() {
 	return peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers>::addRef(this));
 }
-ACTOR Future<Version> PartitionedLogIteratorTwoBuffers::peekNextVersion(
-    Reference<PartitionedLogIteratorTwoBuffers> self) {
+Future<Version> PartitionedLogIteratorTwoBuffers::peekNextVersion(Reference<PartitionedLogIteratorTwoBuffers> self) {
 	// Read the first mutation's version
-	state std::shared_ptr<char[]> start;
-	state Version version;
-	state int fileIndex;
+	std::shared_ptr<char[]> start;
+	Version version{ 0 };
+	int fileIndex{ 0 };
 	if (!self->hasNext()) {
-		return Version(0);
+		co_return Version(0);
 	}
-	wait(self->twobuffer->ready());
+	co_await self->twobuffer->ready();
 	start = self->twobuffer->peek();
 	self->removeBlockHeader();
 	std::memcpy(&version, start.get() + self->bufferOffset, sizeof(Version));
@@ -1266,26 +1433,26 @@ ACTOR Future<Version> PartitionedLogIteratorTwoBuffers::peekNextVersion(
 		self->twobuffer->reset();
 		self->bufferOffset = 0;
 		self->twobuffer->setFileIndex(fileIndex + 1);
-		wait(self->twobuffer->ready());
+		co_await self->twobuffer->ready();
 		start = self->twobuffer->peek();
 		self->removeBlockHeader();
 		std::memcpy(&version, start.get() + self->bufferOffset, sizeof(Version));
 		version = bigEndian64(version);
 		fileIndex = self->twobuffer->getFileIndex();
 	}
-	return version;
+	co_return version;
 }
 
-ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::getNext(
+Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::getNext(
     Reference<PartitionedLogIteratorTwoBuffers> self) {
-	state Standalone<VectorRef<VersionedMutation>> mutations;
+	Standalone<VectorRef<VersionedMutation>> mutations;
 	if (!self->hasNext()) {
 		TraceEvent(SevWarn, "IteratorExhausted").log();
-		return mutations;
+		co_return mutations;
 	}
-	state Version firstVersion = wait(self->peekNextVersion());
+	Version firstVersion = co_await self->peekNextVersion();
 
-	Standalone<VectorRef<VersionedMutation>> firstBatch = wait(self->consumeData(firstVersion));
+	Standalone<VectorRef<VersionedMutation>> firstBatch = co_await self->consumeData(firstVersion);
 	mutations = firstBatch;
 	// If the current buffer is fully consumed, then we need to check the next buffer in case
 	// the version is sliced across this buffer boundary
@@ -1297,11 +1464,11 @@ ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwo
 		if (self->twobuffer->hasNext()) {
 			// now this is run for each block, but it is not necessary if it is the last block of a file
 			// cannot check hasMoreData here because other buffer might have the last piece
-			Version nextVersion = wait(self->peekNextVersion());
+			Version nextVersion = co_await self->peekNextVersion();
 			if (nextVersion != firstVersion) {
 				break;
 			}
-			Standalone<VectorRef<VersionedMutation>> batch = wait(self->consumeData(firstVersion));
+			Standalone<VectorRef<VersionedMutation>> batch = co_await self->consumeData(firstVersion);
 			for (const VersionedMutation& vm : batch) {
 				mutations.push_back_deep(mutations.arena(), vm);
 			}
@@ -1309,7 +1476,7 @@ ACTOR Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwo
 			break;
 		}
 	}
-	return mutations;
+	co_return mutations;
 }
 
 Future<Standalone<VectorRef<VersionedMutation>>> PartitionedLogIteratorTwoBuffers::getNext() {
@@ -1351,7 +1518,7 @@ struct IRangeFileWriter {
 
 	virtual Future<Void> finish() = 0;
 
-	virtual ~IRangeFileWriter() {}
+	virtual ~IRangeFileWriter() = default;
 };
 
 // File Format handlers.
@@ -1388,46 +1555,46 @@ struct IRangeFileWriter {
 //   then the space after the final key to the next 1MB boundary would
 //   just be padding anyway.
 struct RangeFileWriter : public IRangeFileWriter {
-	RangeFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
+	explicit RangeFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
 	  : file(file), blockSize(blockSize), blockEnd(0), fileVersion(BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {}
 
 	// Handles the first block and internal blocks.  Ends current block if needed.
 	// The final flag is used in simulation to pad the file's final block to a whole block size
-	ACTOR static Future<Void> newBlock(RangeFileWriter* self, int bytesNeeded, bool final = false) {
+	static Future<Void> newBlock(RangeFileWriter* self, int bytesNeeded, bool final = false) {
 		// Write padding to finish current block if needed
 		int bytesLeft = self->blockEnd - self->file->size();
 		if (bytesLeft > 0) {
-			state Value paddingFFs = makePadding(bytesLeft);
-			wait(self->file->append(paddingFFs.begin(), bytesLeft));
+			Value paddingFFs = makePadding(bytesLeft);
+			co_await self->file->append(paddingFFs.begin(), bytesLeft);
 		}
 
 		if (final) {
 			ASSERT(g_network->isSimulated());
-			return Void();
+			co_return;
 		}
 
 		// Set new blockEnd
 		self->blockEnd += self->blockSize;
 
 		// write Header
-		wait(self->file->append((uint8_t*)&self->fileVersion, sizeof(self->fileVersion)));
+		co_await self->file->append((uint8_t*)&self->fileVersion, sizeof(self->fileVersion));
 
 		// If this is NOT the first block then write duplicate stuff needed from last block
 		if (self->blockEnd > self->blockSize) {
-			wait(self->file->appendStringRefWithLen(self->lastKey));
-			wait(self->file->appendStringRefWithLen(self->lastKey));
-			wait(self->file->appendStringRefWithLen(self->lastValue));
+			co_await self->file->appendStringRefWithLen(self->lastKey);
+			co_await self->file->appendStringRefWithLen(self->lastKey);
+			co_await self->file->appendStringRefWithLen(self->lastValue);
 		}
 
 		// There must now be room in the current block for bytesNeeded or the block size is too small
 		if (self->file->size() + bytesNeeded > self->blockEnd)
 			throw backup_bad_block_size();
 
-		return Void();
+		co_return;
 	}
 
 	// Used in simulation only to create backup file sizes which are an integer multiple of the block size
-	Future<Void> padEnd(bool final) {
+	Future<Void> padEnd(bool final) override {
 		ASSERT(g_network->isSimulated());
 		if (file->size() > 0) {
 			return newBlock(this, 0, final);
@@ -1443,29 +1610,29 @@ struct RangeFileWriter : public IRangeFileWriter {
 	}
 
 	// Start a new block if needed, then write the key and value
-	ACTOR static Future<Void> writeKV_impl(RangeFileWriter* self, Key k, Value v) {
+	static Future<Void> writeKV_impl(RangeFileWriter* self, Key k, Value v) {
 		int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size();
-		wait(self->newBlockIfNeeded(toWrite));
-		wait(self->file->appendStringRefWithLen(k));
-		wait(self->file->appendStringRefWithLen(v));
+		co_await self->newBlockIfNeeded(toWrite);
+		co_await self->file->appendStringRefWithLen(k);
+		co_await self->file->appendStringRefWithLen(v);
 		self->lastKey = k;
 		self->lastValue = v;
-		return Void();
+		co_return;
 	}
 
-	Future<Void> writeKV(Key k, Value v) { return writeKV_impl(this, k, v); }
+	Future<Void> writeKV(Key k, Value v) override { return writeKV_impl(this, k, v); }
 
 	// Write begin key or end key.
-	ACTOR static Future<Void> writeKey_impl(RangeFileWriter* self, Key k) {
+	static Future<Void> writeKey_impl(RangeFileWriter* self, Key k) {
 		int toWrite = sizeof(uint32_t) + k.size();
-		wait(self->newBlockIfNeeded(toWrite));
-		wait(self->file->appendStringRefWithLen(k));
-		return Void();
+		co_await self->newBlockIfNeeded(toWrite);
+		co_await self->file->appendStringRefWithLen(k);
+		co_return;
 	}
 
-	Future<Void> writeKey(Key k) { return writeKey_impl(this, k); }
+	Future<Void> writeKey(Key k) override { return writeKey_impl(this, k); }
 
-	Future<Void> finish() { return Void(); }
+	Future<Void> finish() override { return Void(); }
 
 	Reference<IBackupFile> file;
 	int blockSize;
@@ -1513,7 +1680,8 @@ void decodeKVPairs(StringRefReader* reader, Standalone<VectorRef<KeyValueRef>>*
 }
 
 static Reference<IBackupContainer> getBackupContainerWithProxy(Reference<IBackupContainer> _bc) {
-	Reference<IBackupContainer> bc = IBackupContainer::openContainer(_bc->getURL(), fileBackupAgentProxy, {});
+	Reference<IBackupContainer> bc = IBackupContainer::openContainer(
+	    _bc->getURL(), fileBackupAgentProxy, _bc->getEncryptionKeyFileName(), _bc->getEncryptionBlockSize());
 	return bc;
 }
 
@@ -1562,29 +1730,27 @@ Standalone<VectorRef<KeyValueRef>> decodeRangeFileBlock(const Standalone<StringR
 	return results;
 }
 
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
-                                                                      int64_t offset,
-                                                                      int len,
-                                                                      Database cx) {
-	state Standalone<StringRef> buf = makeString(len);
-	int rLen = wait(uncancellable(holdWhile(buf, file->read(mutateString(buf), len, offset))));
+Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
+                                                                int64_t offset,
+                                                                int len,
+                                                                Database cx) {
+	Standalone<StringRef> buf = makeString(len);
+	int rLen = co_await uncancellable(holdWhile(buf, file->read(mutateString(buf), len, offset)));
 	if (rLen != len)
 		throw restore_bad_read();
 
 	simulateBlobFailure();
 
-	state Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state StringRefReader reader(buf, restore_corrupted_data());
-	state Arena arena;
-	state int64_t blockDomainId = -1;
-
+	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
+	StringRefReader reader(buf, restore_corrupted_data());
+	Arena arena;
 	try {
 		int32_t file_version = reader.consume<int32_t>();
 		if (file_version != BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {
 			throw restore_unsupported_file_version();
 		}
 		decodeKVPairs(&reader, &results);
-		return results;
+		co_return results;
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")
 		    .error(e)
@@ -1600,36 +1766,36 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 // Very simple format compared to KeyRange files.
 // Header, [Key, Value]... Key len
 struct LogFileWriter {
-	LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
+	explicit LogFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0)
 	  : file(file), blockSize(blockSize), blockEnd(0) {}
 
 	// Start a new block if needed, then write the key and value
-	ACTOR static Future<Void> writeKV_impl(LogFileWriter* self, Key k, Value v) {
+	static Future<Void> writeKV_impl(LogFileWriter* self, Key k, Value v) {
 		// If key and value do not fit in this block, end it and start a new one
 		int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size();
 		if (self->file->size() + toWrite > self->blockEnd) {
 			// Write padding if needed
 			int bytesLeft = self->blockEnd - self->file->size();
 			if (bytesLeft > 0) {
-				state Value paddingFFs = makePadding(bytesLeft);
-				wait(self->file->append(paddingFFs.begin(), bytesLeft));
+				Value paddingFFs = makePadding(bytesLeft);
+				co_await self->file->append(paddingFFs.begin(), bytesLeft);
 			}
 
 			// Set new blockEnd
 			self->blockEnd += self->blockSize;
 
 			// write the block header
-			wait(self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION)));
+			co_await self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION));
 		}
 
-		wait(self->file->appendStringRefWithLen(k));
-		wait(self->file->appendStringRefWithLen(v));
+		co_await self->file->appendStringRefWithLen(k);
+		co_await self->file->appendStringRefWithLen(v);
 
 		// At this point we should be in whatever the current block is or the block size is too small
 		if (self->file->size() > self->blockEnd)
 			throw backup_bad_block_size();
 
-		return Void();
+		co_return;
 	}
 
 	Future<Void> writeKV(Key k, Value v) { return writeKV_impl(this, k, v); }
@@ -1674,16 +1840,16 @@ Standalone<VectorRef<KeyValueRef>> decodeMutationLogFileBlock(const Standalone<S
 	return results;
 }
 
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
-                                                                            int64_t offset,
-                                                                            int len) {
-	state Standalone<StringRef> buf = makeString(len);
-	int rLen = wait(file->read(mutateString(buf), len, offset));
+Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
+                                                                      int64_t offset,
+                                                                      int len) {
+	Standalone<StringRef> buf = makeString(len);
+	int rLen = co_await file->read(mutateString(buf), len, offset);
 	if (rLen != len)
 		throw restore_bad_read();
 
 	try {
-		return decodeMutationLogFileBlock(buf);
+		co_return decodeMutationLogFileBlock(buf);
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
 		    .error(e)
@@ -1694,10 +1860,10 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Refe
 	}
 }
 
-ACTOR Future<Void> checkTaskVersion(Database cx, Reference<Task> task, StringRef name, uint32_t version) {
+Future<Void> checkTaskVersion(Database cx, Reference<Task> task, StringRef name, uint32_t version) {
 	uint32_t taskVersion = task->getVersion();
 	if (taskVersion > version) {
-		state Error err = task_invalid_version();
+		Error err = task_invalid_version();
 
 		TraceEvent(SevWarn, "BA_BackupRangeTaskFuncExecute")
 		    .detail("TaskVersion", taskVersion)
@@ -1708,35 +1874,33 @@ ACTOR Future<Void> checkTaskVersion(Database cx, Reference<Task> task, StringRef
 			                         task->params[Task::reservedTaskParamKeyType].toString().c_str(),
 			                         (unsigned long)taskVersion,
 			                         (unsigned long)version);
-			wait(BackupConfig(task).logError(cx, err, msg));
+			co_await BackupConfig(task).logError(cx, err, msg);
 		}
 
 		throw err;
 	}
-
-	return Void();
 }
 
-ACTOR static Future<Void> abortFiveZeroBackup(FileBackupAgent* backupAgent,
-                                              Reference<ReadYourWritesTransaction> tr,
-                                              std::string tagName) {
+static Future<Void> abortFiveZeroBackup(FileBackupAgent* backupAgent,
+                                        Reference<ReadYourWritesTransaction> tr,
+                                        std::string tagName) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state Subspace tagNames = backupAgent->subspace.get(BackupAgentBase::keyTagName);
-	Optional<Value> uidStr = wait(tr->get(tagNames.pack(Key(tagName))));
+	Subspace tagNames = backupAgent->subspace.get(BackupAgentBase::keyTagName);
+	Optional<Value> uidStr = co_await tr->get(tagNames.pack(Key(tagName)));
 	if (!uidStr.present()) {
 		TraceEvent(SevWarn, "FileBackupAbortIncompatibleBackup_TagNotFound").detail("TagName", tagName.c_str());
-		return Void();
+		co_return;
 	}
-	state UID uid = BinaryReader::fromStringRef<UID>(uidStr.get(), Unversioned());
+	UID uid = BinaryReader::fromStringRef<UID>(uidStr.get(), Unversioned());
 
-	state Subspace statusSpace = backupAgent->subspace.get(BackupAgentBase::keyStates).get(uid.toString());
-	state Subspace globalConfig = backupAgent->subspace.get(BackupAgentBase::keyConfig).get(uid.toString());
-	state Subspace newConfigSpace = uidPrefixKey("uid->config/"_sr.withPrefix(fileBackupPrefixRange.begin), uid);
+	Subspace statusSpace = backupAgent->subspace.get(BackupAgentBase::keyStates).get(uid.toString());
+	Subspace globalConfig = backupAgent->subspace.get(BackupAgentBase::keyConfig).get(uid.toString());
+	Subspace newConfigSpace(uidPrefixKey("uid->config/"_sr.withPrefix(fileBackupPrefixRange.begin), uid));
 
-	Optional<Value> statusStr = wait(tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus)));
-	state EBackupState status =
+	Optional<Value> statusStr = co_await tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus));
+	EBackupState status =
 	    !statusStr.present() ? EBackupState::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString());
 
 	TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup")
@@ -1760,26 +1924,24 @@ ACTOR static Future<Void> abortFiveZeroBackup(FileBackupAgent* backupAgent,
 	// Set old style state key to Aborted if it was Runnable
 	if (backupAgent->isRunnable(status))
 		tr->set(statusKey, StringRef(FileBackupAgent::getStateText(EBackupState::STATE_ABORTED)));
-
-	return Void();
 }
 
 struct AbortFiveZeroBackupTask : TaskFuncBase {
 	static StringRef name;
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state FileBackupAgent backupAgent;
-		state std::string tagName = task->params[BackupAgentBase::keyConfigBackupTag].toString();
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		FileBackupAgent backupAgent;
+		std::string tagName = task->params[BackupAgentBase::keyConfigBackupTag].toString();
 
 		TraceEvent(SevInfo, "FileBackupCancelOldTask")
 		    .detail("Task", task->params[Task::reservedTaskParamKeyType])
 		    .detail("TagName", tagName);
-		wait(abortFiveZeroBackup(&backupAgent, tr, tagName));
+		co_await abortFiveZeroBackup(&backupAgent, tr, tagName);
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
+		co_return;
 	}
 
 	StringRef getName() const override {
@@ -1813,17 +1975,17 @@ REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_finish_full_backup);
 REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_finished_full_backup);
 REGISTER_TASKFUNC_ALIAS(AbortFiveZeroBackupTask, file_start_full_backup);
 
-ACTOR static Future<Void> abortFiveOneBackup(FileBackupAgent* backupAgent,
-                                             Reference<ReadYourWritesTransaction> tr,
-                                             std::string tagName) {
+static Future<Void> abortFiveOneBackup(FileBackupAgent* backupAgent,
+                                       Reference<ReadYourWritesTransaction> tr,
+                                       std::string tagName) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state KeyBackedTag tag = makeBackupTag(tagName);
-	state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::False, backup_unneeded()));
+	KeyBackedTag tag = makeBackupTag(tagName);
+	UidAndAbortedFlagT current = co_await tag.getOrThrow(tr, Snapshot::False, backup_unneeded());
 
-	state BackupConfig config(current.first);
-	EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+	BackupConfig config(current.first);
+	EBackupState status = co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 
 	if (!backupAgent->isRunnable(status)) {
 		throw backup_unneeded();
@@ -1834,7 +1996,7 @@ ACTOR static Future<Void> abortFiveOneBackup(FileBackupAgent* backupAgent,
 	    .detail("Status", BackupAgentBase::getStateText(status));
 
 	// Cancel backup task through tag
-	wait(tag.cancel(tr));
+	co_await tag.cancel(tr);
 
 	Key configPath = uidPrefixKey(logRangesRange.begin, config.getUid());
 	Key logsPath = uidPrefixKey(backupLogKeys.begin, config.getUid());
@@ -1843,27 +2005,24 @@ ACTOR static Future<Void> abortFiveOneBackup(FileBackupAgent* backupAgent,
 	tr->clear(KeyRangeRef(logsPath, strinc(logsPath)));
 
 	config.stateEnum().set(tr, EBackupState::STATE_ABORTED);
-
-	return Void();
 }
 
 struct AbortFiveOneBackupTask : TaskFuncBase {
 	static StringRef name;
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state FileBackupAgent backupAgent;
-		state BackupConfig config(task);
-		state std::string tagName = wait(config.tag().getOrThrow(tr));
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		FileBackupAgent backupAgent;
+		BackupConfig config(task);
+		std::string tagName = co_await config.tag().getOrThrow(tr);
 
 		TraceEvent(SevInfo, "FileBackupCancelFiveOneTask")
 		    .detail("Task", task->params[Task::reservedTaskParamKeyType])
 		    .detail("TagName", tagName);
-		wait(abortFiveOneBackup(&backupAgent, tr, tagName));
+		co_await abortFiveOneBackup(&backupAgent, tr, tagName);
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
 	}
 
 	StringRef getName() const override {
@@ -1898,41 +2057,41 @@ REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_write_snapshot_manif
 REGISTER_TASKFUNC_ALIAS(AbortFiveOneBackupTask, file_backup_start);
 
 std::function<void(Reference<Task>)> NOP_SETUP_TASK_FN = [](Reference<Task> task) { /* NOP */ };
-ACTOR static Future<Key> addBackupTask(StringRef name,
-                                       uint32_t version,
-                                       Reference<ReadYourWritesTransaction> tr,
-                                       Reference<TaskBucket> taskBucket,
-                                       TaskCompletionKey completionKey,
-                                       BackupConfig config,
-                                       Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
-                                       std::function<void(Reference<Task>)> setupTaskFn = NOP_SETUP_TASK_FN,
-                                       int priority = 0,
-                                       SetValidation setValidation = SetValidation::True) {
+static Future<Key> addBackupTask(StringRef name,
+                                 uint32_t version,
+                                 Reference<ReadYourWritesTransaction> tr,
+                                 Reference<TaskBucket> taskBucket,
+                                 TaskCompletionKey completionKey,
+                                 BackupConfig config,
+                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
+                                 std::function<void(Reference<Task>)> setupTaskFn = NOP_SETUP_TASK_FN,
+                                 int priority = 0,
+                                 SetValidation setValidation = SetValidation::True) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	Key doneKey = wait(completionKey.get(tr, taskBucket));
-	state Reference<Task> task(new Task(name, version, doneKey, priority));
+	Key doneKey = co_await completionKey.get(tr, taskBucket);
+	Reference<Task> task(new Task(name, version, doneKey, priority));
 
 	// Bind backup config to new task
 	// allow this new task to find the config(keyspace) of the parent task
-	wait(config.toTask(tr, task, setValidation));
+	co_await config.toTask(tr, task, setValidation);
 
 	// Set task specific params
 	setupTaskFn(task);
 
 	if (!waitFor) {
-		return taskBucket->addTask(tr, task);
+		co_return taskBucket->addTask(tr, task);
 	}
-	wait(waitFor->onSetAddTask(tr, taskBucket, task));
+	co_await waitFor->onSetAddTask(tr, taskBucket, task);
 
-	return "OnSetAddTask"_sr;
+	co_return "OnSetAddTask"_sr;
 }
 
 // Clears the backup ID from "backupStartedKey" to pause backup workers.
-ACTOR static Future<Void> clearBackupStartID(Reference<ReadYourWritesTransaction> tr, UID backupUid) {
+static Future<Void> clearBackupStartID(Reference<ReadYourWritesTransaction> tr, UID backupUid) {
 	// If backup worker is not enabled, exit early.
-	Optional<Value> started = wait(tr->get(backupStartedKey));
+	Optional<Value> started = co_await tr->get(backupStartedKey);
 	std::vector<std::pair<UID, Version>> ids;
 	if (started.present()) {
 		ids = decodeBackupStartedValue(started.get());
@@ -1949,7 +2108,6 @@ ACTOR static Future<Void> clearBackupStartID(Reference<ReadYourWritesTransaction
 	} else {
 		tr->set(backupStartedKey, encodeBackupStartedValue(ids));
 	}
-	return Void();
 }
 
 // Backup and Restore taskFunc definitions will inherit from one of the following classes which
@@ -1974,23 +2132,23 @@ struct BackupTaskFuncBase : TaskFuncBase {
 	virtual std::string toString(Reference<Task> task) const { return ""; }
 };
 
-ACTOR static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr,
-                                                                    Key beginKey,
-                                                                    Key endKey,
-                                                                    int limit) {
+static Future<Standalone<VectorRef<KeyRef>>> getBlockOfShards(Reference<ReadYourWritesTransaction> tr,
+                                                              Key beginKey,
+                                                              Key endKey,
+                                                              int limit) {
 
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	state Standalone<VectorRef<KeyRef>> results;
-	RangeResult values = wait(tr->getRange(
-	    KeyRangeRef(keyAfter(beginKey.withPrefix(keyServersPrefix)), endKey.withPrefix(keyServersPrefix)), limit));
+	Standalone<VectorRef<KeyRef>> results;
+	RangeResult values = co_await tr->getRange(
+	    KeyRangeRef(keyAfter(beginKey.withPrefix(keyServersPrefix)), endKey.withPrefix(keyServersPrefix)), limit);
 
 	for (auto& s : values) {
 		KeyRef k = s.key.removePrefix(keyServersPrefix);
 		results.push_back_deep(results.arena(), k);
 	}
 
-	return results;
+	co_return results;
 }
 
 struct BackupRangeTaskFunc : BackupTaskFuncBase {
@@ -2032,28 +2190,38 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 	//  - update the task begin key
 	//  - save/extend the task with the new params
 	// Returns whether or not the caller should continue executing the task.
-	ACTOR static Future<bool> finishRangeFile(Reference<IBackupFile> file,
-	                                          Database cx,
-	                                          Reference<Task> task,
-	                                          Reference<TaskBucket> taskBucket,
-	                                          KeyRange range,
-	                                          Version version) {
-		wait(file->finish());
+	static Future<bool> finishRangeFile(Reference<IBackupFile> file,
+	                                    Database cx,
+	                                    Reference<Task> task,
+	                                    Reference<TaskBucket> taskBucket,
+	                                    KeyRange range,
+	                                    Version version) {
+		co_await file->finish();
+
+		TraceEvent("BackupRangeFileFinished")
+		    .detail("BackupUID", BackupConfig(task).getUid())
+		    .detail("FileName", file->getFileName())
+		    .detail("FileSize", file->size())
+		    .detail("RangeBegin", range.begin.printable())
+		    .detail("RangeEnd", range.end.printable())
+		    .detail("RangeEmpty", range.empty())
+		    .detail("Version", version);
 
 		// Ignore empty ranges.
 		if (range.empty())
-			return false;
+			co_return false;
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state BackupConfig backup(task);
-		state bool usedFile = false;
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		BackupConfig backup(task);
+		bool usedFile = false;
 
 		// Avoid unnecessary conflict by prevent taskbucket's automatic timeout extension
 		// because the following transaction loop extends and updates the task.
-		wait(task->extendMutex.take());
-		state FlowLock::Releaser releaser(task->extendMutex, 1);
+		co_await task->extendMutex.take();
+		FlowLock::Releaser releaser(task->extendMutex, 1);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -2063,41 +2231,42 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 				Params.beginKey().set(task, range.end);
 
 				// Save and extend the task with the new begin parameter
-				state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::True));
+				Version newTimeout = co_await taskBucket->extendTimeout(tr, task, UpdateParams::True);
 
 				// Update the range bytes written in the backup config
 				backup.rangeBytesWritten().atomicOp(tr, file->size(), MutationRef::AddValue);
 				backup.snapshotRangeFileCount().atomicOp(tr, 1, MutationRef::AddValue);
 
 				// See if there is already a file for this key which has an earlier begin, update the map if not.
-				Optional<BackupConfig::RangeSlice> s = wait(backup.snapshotRangeFileMap().get(tr, range.end));
+				Optional<BackupConfig::RangeSlice> s = co_await backup.snapshotRangeFileMap().get(tr, range.end);
 				if (!s.present() || s.get().begin >= range.begin) {
 					backup.snapshotRangeFileMap().set(
 					    tr, range.end, { range.begin, version, file->getFileName(), file->size() });
 					usedFile = true;
 				}
 
-				wait(tr->commit());
+				co_await tr->commit();
 				task->timeoutVersion = newTimeout;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		return usedFile;
+		co_return usedFile;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 int priority,
-	                                 Key begin,
-	                                 Key end,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
-	                                 Version scheduledVersion = invalidVersion) {
-		Key key = wait(addBackupTask(
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           int priority,
+	                           Key begin,
+	                           Key end,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
+	                           Version scheduledVersion = invalidVersion) {
+		Key key = co_await addBackupTask(
 		    BackupRangeTaskFunc::name,
 		    BackupRangeTaskFunc::version,
 		    tr,
@@ -2112,20 +2281,20 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 			    if (scheduledVersion != invalidVersion)
 				    ReservedTaskParams::scheduledVersion().set(task, scheduledVersion);
 		    },
-		    priority));
-		return key;
+		    priority);
+		co_return key;
 	}
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
 
-		wait(checkTaskVersion(cx, task, BackupRangeTaskFunc::name, BackupRangeTaskFunc::version));
+		co_await checkTaskVersion(cx, task, BackupRangeTaskFunc::name, BackupRangeTaskFunc::version);
 
-		state Key beginKey = Params.beginKey().get(task);
-		state Key endKey = Params.endKey().get(task);
+		Key beginKey = Params.beginKey().get(task);
+		Key endKey = Params.endKey().get(task);
 
 		TraceEvent("FileBackupRangeStart")
 		    .suppressFor(60)
@@ -2137,54 +2306,55 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		// When a key range task saves the last chunk of progress and then the executor dies, when the task
 		// continues its beginKey and endKey will be equal but there is no work to be done.
 		if (beginKey == endKey)
-			return Void();
+			co_return;
 
 		// Find out if there is a shard boundary in(beginKey, endKey)
-		Standalone<VectorRef<KeyRef>> keys = wait(runRYWTransaction(
-		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return getBlockOfShards(tr, beginKey, endKey, 1); }));
-		if (keys.size() > 0) {
+		Standalone<VectorRef<KeyRef>> keys = co_await runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return getBlockOfShards(tr, beginKey, endKey, 1); });
+		if (!keys.empty()) {
 			Params.addBackupRangeTasks().set(task, true);
-			return Void();
+			co_return;
 		}
 
 		// Read everything from beginKey to endKey, write it to an output file, run the output file processor, and
 		// then set on_done. If we are still writing after X seconds, end the output file and insert a new
 		// backup_range task for the remainder.
-		state Reference<IBackupFile> outFile;
-		state Version outVersion = invalidVersion;
-		state Key lastKey;
+		Reference<IBackupFile> outFile;
+		Version outVersion = invalidVersion;
+		Key lastKey;
 
 		// retrieve kvData
-		state PromiseStream<RangeResultWithVersion> results;
-
-		state Future<Void> rc = readCommitted(cx,
-		                                      results,
-		                                      lock,
-		                                      KeyRangeRef(beginKey, endKey),
-		                                      Terminator::True,
-		                                      AccessSystemKeys::True,
-		                                      LockAware::True,
-		                                      ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY));
-		state std::unique_ptr<IRangeFileWriter> rangeFile;
-		state BackupConfig backup(task);
-		state Arena arena;
-
-		DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+		PromiseStream<RangeResultWithVersion> results;
+
+		Future<Void> rc = readCommitted(cx,
+		                                results,
+		                                lock,
+		                                KeyRangeRef(beginKey, endKey),
+		                                Terminator::True,
+		                                AccessSystemKeys::True,
+		                                LockAware::True,
+		                                ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY));
+		std::unique_ptr<IRangeFileWriter> rangeFile;
+		BackupConfig backup(task);
+		Arena arena;
+
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 
 		// Don't need to check keepRunning(task) here because we will do that while finishing each output file, but
 		// if bc is false then clearly the backup is no longer in progress
-		Reference<IBackupContainer> _bc = wait(backup.backupContainer().getD(cx.getReference()));
+		Reference<IBackupContainer> _bc = co_await backup.backupContainer().getD(cx.getReference());
 		if (!_bc) {
-			return Void();
+			co_return;
 		}
-		state Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
-		state bool done = false;
-		state int64_t nrKeys = 0;
 
-		loop {
-			state RangeResultWithVersion values;
+		Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
+		bool done = false;
+		int64_t nrKeys = 0;
+
+		while (true) {
+			RangeResultWithVersion values;
 			try {
-				RangeResultWithVersion _values = waitNext(results.getFuture());
+				RangeResultWithVersion _values = co_await results.getFuture();
 				values = _values;
 				lock->release(values.first.expectedSize());
 			} catch (Error& e) {
@@ -2199,17 +2369,17 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 			if (values.second != outVersion || done) {
 				if (outFile) {
 					CODE_PROBE(outVersion != invalidVersion, "Backup range task wrote multiple versions");
-					state Key nextKey = done ? endKey : keyAfter(lastKey);
-					wait(rangeFile->writeKey(nextKey));
+					Key nextKey = done ? endKey : keyAfter(lastKey);
+					co_await rangeFile->writeKey(nextKey);
 
 					if (BUGGIFY) {
-						wait(rangeFile->padEnd(true));
+						co_await rangeFile->padEnd(true);
 					}
 
-					wait(rangeFile->finish());
+					co_await rangeFile->finish();
 
-					bool usedFile = wait(
-					    finishRangeFile(outFile, cx, task, taskBucket, KeyRangeRef(beginKey, nextKey), outVersion));
+					bool usedFile = co_await finishRangeFile(
+					    outFile, cx, task, taskBucket, KeyRangeRef(beginKey, nextKey), outVersion);
 					TraceEvent("FileBackupWroteRangeFile")
 					    .suppressFor(60)
 					    .detail("BackupUID", backup.getUid())
@@ -2225,48 +2395,49 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 				}
 
 				if (done)
-					return Void();
+					co_return;
 
 				// Start writing a new file after verifying this task should keep running as of a new read version
 				// (which must be >= outVersion)
 				outVersion = values.second;
 				// block size must be at least large enough for 3 max size keys and 2 max size values + overhead so
 				// 250k conservatively.
-				state int blockSize =
+				int blockSize =
 				    BUGGIFY ? deterministicRandom()->randomInt(250e3, 4e6) : CLIENT_KNOBS->BACKUP_RANGEFILE_BLOCK_SIZE;
-				state Version snapshotBeginVersion;
-				state int64_t snapshotRangeFileCount;
+				Version snapshotBeginVersion{ 0 };
+				int64_t snapshotRangeFileCount{ 0 };
 
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-				loop {
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+				while (true) {
+					Error err;
 					try {
 						tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 						tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-						wait(taskBucket->keepRunning(tr, task) &&
-						     storeOrThrow(snapshotBeginVersion, backup.snapshotBeginVersion().get(tr)) &&
-						     store(snapshotRangeFileCount, backup.snapshotRangeFileCount().getD(tr)));
+						co_await (taskBucket->keepRunning(tr, task) &&
+						          storeOrThrow(snapshotBeginVersion, backup.snapshotBeginVersion().get(tr)) &&
+						          store(snapshotRangeFileCount, backup.snapshotRangeFileCount().getD(tr)));
 
 						break;
 					} catch (Error& e) {
-						wait(tr->onError(e));
+						err = e;
 					}
+					co_await tr->onError(err);
 				}
 
 				Reference<IBackupFile> f =
-				    wait(bc->writeRangeFile(snapshotBeginVersion, snapshotRangeFileCount, outVersion, blockSize));
+				    co_await bc->writeRangeFile(snapshotBeginVersion, snapshotRangeFileCount, outVersion, blockSize);
 				outFile = f;
 
 				// Initialize range file writer and write begin key
 				rangeFile = std::make_unique<RangeFileWriter>(outFile, blockSize);
-				wait(rangeFile->writeKey(beginKey));
+				co_await rangeFile->writeKey(beginKey);
 			}
 
 			// write kvData to file, update lastKey and key count
-			if (values.first.size() != 0) {
-				state size_t i = 0;
-				for (; i < values.first.size(); ++i) {
-					wait(rangeFile->writeKV(values.first[i].key, values.first[i].value));
+			if (!values.first.empty()) {
+				for (size_t i = 0; i < values.first.size(); ++i) {
+					co_await rangeFile->writeKV(values.first[i].key, values.first[i].value);
 				}
 				lastKey = values.first.back().key;
 				nrKeys += values.first.size();
@@ -2274,70 +2445,65 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		}
 	}
 
-	ACTOR static Future<Void> startBackupRangeInternal(Reference<ReadYourWritesTransaction> tr,
-	                                                   Reference<TaskBucket> taskBucket,
-	                                                   Reference<FutureBucket> futureBucket,
-	                                                   Reference<Task> task,
-	                                                   Reference<TaskFuture> onDone) {
+	static Future<Void> startBackupRangeInternal(Reference<ReadYourWritesTransaction> tr,
+	                                             Reference<TaskBucket> taskBucket,
+	                                             Reference<FutureBucket> futureBucket,
+	                                             Reference<Task> task,
+	                                             Reference<TaskFuture> onDone) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Key nextKey = Params.beginKey().get(task);
-		state Key endKey = Params.endKey().get(task);
+		Key nextKey = Params.beginKey().get(task);
+		Key endKey = Params.endKey().get(task);
 
-		state Standalone<VectorRef<KeyRef>> keys =
-		    wait(getBlockOfShards(tr, nextKey, endKey, CLIENT_KNOBS->BACKUP_SHARD_TASK_LIMIT));
+		Standalone<VectorRef<KeyRef>> keys =
+		    co_await getBlockOfShards(tr, nextKey, endKey, CLIENT_KNOBS->BACKUP_SHARD_TASK_LIMIT);
 
 		std::vector<Future<Key>> addTaskVector;
-		for (int idx = 0; idx < keys.size(); ++idx) {
-			if (nextKey != keys[idx]) {
-				addTaskVector.push_back(addTask(tr,
-				                                taskBucket,
-				                                task,
-				                                task->getPriority(),
-				                                nextKey,
-				                                keys[idx],
-				                                TaskCompletionKey::joinWith(onDone)));
+		for (const auto& splitKey : keys) {
+			if (nextKey != splitKey) {
+				addTaskVector.push_back(addTask(
+				    tr, taskBucket, task, task->getPriority(), nextKey, splitKey, TaskCompletionKey::joinWith(onDone)));
 				TraceEvent("FileBackupRangeSplit")
 				    .suppressFor(60)
 				    .detail("BackupUID", BackupConfig(task).getUid())
 				    .detail("BeginKey", Params.beginKey().get(task).printable())
 				    .detail("EndKey", Params.endKey().get(task).printable())
 				    .detail("SliceBeginKey", nextKey.printable())
-				    .detail("SliceEndKey", keys[idx].printable());
+				    .detail("SliceEndKey", splitKey.printable());
 			}
-			nextKey = keys[idx];
+			nextKey = splitKey;
 		}
 
-		wait(waitForAll(addTaskVector));
+		co_await waitForAll(addTaskVector);
 
 		if (nextKey != endKey) {
 			// Add task to cover nextKey to the end, using the priority of the current task
-			wait(success(addTask(tr,
-			                     taskBucket,
-			                     task,
-			                     task->getPriority(),
-			                     nextKey,
-			                     endKey,
-			                     TaskCompletionKey::joinWith(onDone),
-			                     Reference<TaskFuture>(),
-			                     task->getPriority())));
+			co_await addTask(tr,
+			                 taskBucket,
+			                 task,
+			                 task->getPriority(),
+			                 nextKey,
+			                 endKey,
+			                 TaskCompletionKey::joinWith(onDone),
+			                 Reference<TaskFuture>(),
+			                 task->getPriority());
 		}
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 		if (Params.addBackupRangeTasks().get(task)) {
-			wait(startBackupRangeInternal(tr, taskBucket, futureBucket, task, taskFuture));
+			co_await startBackupRangeInternal(tr, taskBucket, futureBucket, task, taskFuture);
 		} else {
-			wait(taskFuture->set(tr, taskBucket));
+			co_await taskFuture->set(tr, taskBucket);
 		}
 
-		wait(taskBucket->finish(tr, task));
+		co_await taskBucket->finish(tr, task);
 
 		TraceEvent("FileBackupRangeFinish")
 		    .suppressFor(60)
@@ -2346,7 +2512,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 		    .detail("EndKey", Params.endKey().get(task).printable())
 		    .detail("TaskKey", task->key.printable());
 
-		return Void();
+		co_return;
 	}
 };
 StringRef BackupRangeTaskFunc::name = "file_backup_write_range_5.2"_sr;
@@ -2380,14 +2546,14 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 int priority,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
-	                                 Version scheduledVersion = invalidVersion) {
-		Key key = wait(addBackupTask(
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           int priority,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
+	                           Version scheduledVersion = invalidVersion) {
+		Key key = co_await addBackupTask(
 		    name,
 		    version,
 		    tr,
@@ -2399,40 +2565,41 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 			    if (scheduledVersion != invalidVersion)
 				    ReservedTaskParams::scheduledVersion().set(task, scheduledVersion);
 		    },
-		    priority));
-		return key;
+		    priority);
+		co_return key;
 	}
 
 	enum DispatchState { SKIP = 0, DONE = 1, NOT_DONE_MIN = 2 };
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
-		wait(checkTaskVersion(cx, task, name, version));
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
+		co_await checkTaskVersion(cx, task, name, version);
 
-		state double startTime = timer();
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		double startTime = timer();
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
 		// The shard map will use 3 values classes.  Exactly SKIP, exactly DONE, then any number >= NOT_DONE_MIN
 		// which will mean not done. This is to enable an efficient coalesce() call to squash adjacent ranges which
 		// are not yet finished to enable efficiently finding random database shards which are not done.
-		state int notDoneSequence = NOT_DONE_MIN;
-		state KeyRangeMap<int> shardMap(notDoneSequence++);
-		state Key beginKey = allKeys.begin;
+		int notDoneSequence = NOT_DONE_MIN;
+		KeyRangeMap<int> shardMap(notDoneSequence++);
+		Key beginKey = allKeys.begin;
 
 		// Read all shard boundaries and add them to the map
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state Future<Standalone<VectorRef<KeyRef>>> shardBoundaries =
+				Future<Standalone<VectorRef<KeyRef>>> shardBoundaries =
 				    getBlockOfShards(tr, beginKey, allKeys.end, CLIENT_KNOBS->TOO_MANY);
-				wait(success(shardBoundaries) && taskBucket->keepRunning(tr, task));
+				co_await (success(shardBoundaries) && taskBucket->keepRunning(tr, task));
 
-				if (shardBoundaries.get().size() == 0)
+				if (shardBoundaries.get().empty())
 					break;
 
 				for (auto& boundary : shardBoundaries.get()) {
@@ -2441,38 +2608,41 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 
 				beginKey = keyAfter(shardBoundaries.get().back());
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		// Read required stuff from backup config
-		state BackupConfig config(task);
-		state Version recentReadVersion;
-		state Version snapshotBeginVersion;
-		state Version snapshotTargetEndVersion;
-		state int64_t snapshotIntervalSeconds;
-		state Optional<Version> latestSnapshotEndVersion;
-		state std::vector<KeyRange> backupRanges;
-		state Optional<Key> snapshotBatchFutureKey;
-		state Reference<TaskFuture> snapshotBatchFuture;
-		state Optional<int64_t> snapshotBatchSize;
+		BackupConfig config(task);
+		Version recentReadVersion{ 0 };
+		Version snapshotBeginVersion{ 0 };
+		Version snapshotTargetEndVersion{ 0 };
+		int64_t snapshotIntervalSeconds{ 0 };
+		Optional<Version> latestSnapshotEndVersion;
+		std::vector<KeyRange> backupRanges;
+		Optional<Key> snapshotBatchFutureKey;
+		Reference<TaskFuture> snapshotBatchFuture;
+		Optional<int64_t> snapshotBatchSize;
 
 		tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr)) &&
-				     store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr)) &&
-				     store(backupRanges, config.backupRanges().getOrThrow(tr)) &&
-				     store(snapshotIntervalSeconds, config.snapshotIntervalSeconds().getOrThrow(tr))
-				     // The next two parameters are optional
-				     && store(snapshotBatchFutureKey, config.snapshotBatchFuture().get(tr)) &&
-				     store(snapshotBatchSize, config.snapshotBatchSize().get(tr)) &&
-				     store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
-				     store(recentReadVersion, tr->getReadVersion()) && taskBucket->keepRunning(tr, task));
+				co_await (store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr)) &&
+				          store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr)) &&
+				          store(backupRanges, config.backupRanges().getOrThrow(tr)) &&
+				          store(snapshotIntervalSeconds, config.snapshotIntervalSeconds().getOrThrow(tr))
+				          // The next two parameters are optional
+				          && store(snapshotBatchFutureKey, config.snapshotBatchFuture().get(tr)) &&
+				          store(snapshotBatchSize, config.snapshotBatchSize().get(tr)) &&
+				          store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
+				          store(recentReadVersion, tr->getReadVersion()) && taskBucket->keepRunning(tr, task));
 
 				// If the snapshot batch future key does not exist, this is the first execution of this dispatch
 				// task so
@@ -2489,16 +2659,16 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 					// The dispatch of this batch can take multiple separate executions if the executor fails
 					// so store a completion key for the dispatch finish() to set when dispatching the batch is
 					// done.
-					state TaskCompletionKey dispatchCompletionKey = TaskCompletionKey::joinWith(snapshotBatchFuture);
+					TaskCompletionKey dispatchCompletionKey = TaskCompletionKey::joinWith(snapshotBatchFuture);
 					// this is a bad hack - but flow doesn't work well with lambda functions and capturing
 					// state variables...
 					auto cfg = &config;
 					auto tx = &tr;
-					wait(map(dispatchCompletionKey.get(tr, taskBucket), [cfg, tx](Key const& k) {
+					co_await map(dispatchCompletionKey.get(tr, taskBucket), [cfg, tx](Key const& k) {
 						cfg->snapshotBatchDispatchDoneKey().set(*tx, k);
 						return Void();
-					}));
-					wait(tr->commit());
+					});
+					co_await tr->commit();
 				} else {
 					ASSERT(snapshotBatchSize.present());
 					// Batch future key exists in the config so create future from it
@@ -2507,24 +2677,26 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		// Read all dispatched ranges
-		state std::vector<std::pair<Key, bool>> dispatchBoundaries;
+		std::vector<std::pair<Key, bool>> dispatchBoundaries;
 		tr->reset();
 		beginKey = allKeys.begin;
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state Future<BackupConfig::RangeDispatchMapT::RangeResultType> bounds =
+				Future<BackupConfig::RangeDispatchMapT::RangeResultType> bounds =
 				    config.snapshotRangeDispatchMap().getRange(
 				        tr, beginKey, keyAfter(allKeys.end), CLIENT_KNOBS->TOO_MANY);
-				wait(success(bounds) && taskBucket->keepRunning(tr, task) &&
-				     store(recentReadVersion, tr->getReadVersion()));
+				co_await (success(bounds) && taskBucket->keepRunning(tr, task) &&
+				          store(recentReadVersion, tr->getReadVersion()));
 
 				if (!bounds.get().results.empty()) {
 					dispatchBoundaries.reserve(dispatchBoundaries.size() + bounds.get().results.size());
@@ -2532,29 +2704,32 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 					    dispatchBoundaries.end(), bounds.get().results.begin(), bounds.get().results.end());
 				}
 
-				if (!bounds.get().more)
+				if (!bounds.get().more) {
 					break;
+				}
 
 				beginKey = keyAfter(bounds.get().results.back().first);
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		// The next few sections involve combining the results above.  Yields are used after operations
 		// that could have operated on many thousands of things and in loops which could have many
 		// thousands of iterations.
 		// Declare some common iterators which must be state vars and will be used multiple times.
-		state int i;
-		state RangeMap<Key, int, KeyRangeRef>::iterator iShard;
-		state RangeMap<Key, int, KeyRangeRef>::iterator iShardEnd;
+		int i{ 0 };
+		RangeMap<Key, int, KeyRangeRef>::iterator iShard;
+		RangeMap<Key, int, KeyRangeRef>::iterator iShardEnd;
 
 		// Set anything inside a dispatched range to DONE.
 		// Also ensure that the boundary value are true, false, [true, false]...
-		if (dispatchBoundaries.size() > 0) {
-			state bool lastValue = false;
-			state Key lastKey;
+		if (!dispatchBoundaries.empty()) {
+			bool lastValue = false;
+			Key lastKey;
 			for (i = 0; i < dispatchBoundaries.size(); ++i) {
 				const std::pair<Key, bool>& boundary = dispatchBoundaries[i];
 
@@ -2571,34 +2746,34 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 					iShardEnd = shardRanges.end();
 					for (; iShard != iShardEnd; ++iShard) {
 						iShard->value() = DONE;
-						wait(yield());
+						co_await yield();
 					}
 				}
 				lastValue = dispatchBoundaries[i].second;
 				lastKey = dispatchBoundaries[i].first;
 
-				wait(yield());
+				co_await yield();
 			}
 			ASSERT(lastValue == false);
 		}
 
 		// Set anything outside the backup ranges to SKIP.  We can use insert() here instead of modify()
 		// because it's OK to delete shard boundaries in the skipped ranges.
-		if (backupRanges.size() > 0) {
+		if (!backupRanges.empty()) {
 			shardMap.insert(KeyRangeRef(allKeys.begin, backupRanges.front().begin), SKIP);
-			wait(yield());
+			co_await yield();
 
 			for (i = 0; i < backupRanges.size() - 1; ++i) {
 				shardMap.insert(KeyRangeRef(backupRanges[i].end, backupRanges[i + 1].begin), SKIP);
-				wait(yield());
+				co_await yield();
 			}
 
 			shardMap.insert(KeyRangeRef(backupRanges.back().end, allKeys.end), SKIP);
-			wait(yield());
+			co_await yield();
 		}
 
-		state int countShardsDone = 0;
-		state int countShardsNotDone = 0;
+		int countShardsDone = 0;
+		int countShardsNotDone = 0;
 
 		// Scan through the shard map, counting the DONE and NOT_DONE shards.
 		RangeMap<Key, int, KeyRangeRef>::Ranges shardRanges = shardMap.ranges();
@@ -2610,15 +2785,26 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 			} else if (iShard->value() >= NOT_DONE_MIN)
 				++countShardsNotDone;
 
-			wait(yield());
+			co_await yield();
 		}
 
 		// Coalesce the shard map to make random selection below more efficient.
 		shardMap.coalesce(allKeys);
-		wait(yield());
+		co_await yield();
 
 		// In this context "all" refers to all of the shards relevant for this particular backup
-		state int countAllShards = countShardsDone + countShardsNotDone;
+		int countAllShards = countShardsDone + countShardsNotDone;
+
+		// Log backup ranges and shard counts for debugging mode=BOTH issues
+		TraceEvent("FileBackupSnapshotDispatchShardCount")
+		    .detail("BackupUID", config.getUid())
+		    .detail("BackupRangesCount", backupRanges.size())
+		    .detail("FirstRangeBegin", backupRanges.empty() ? ""_sr : backupRanges.front().begin.printable())
+		    .detail("FirstRangeEnd", backupRanges.empty() ? ""_sr : backupRanges.front().end.printable())
+		    .detail("CountAllShards", countAllShards)
+		    .detail("CountShardsDone", countShardsDone)
+		    .detail("CountShardsNotDone", countShardsNotDone)
+		    .detail("LatestSnapshotEndVersion", latestSnapshotEndVersion.orDefault(-1));
 
 		// NOTE: Don't finish here even if countShardsNotDone == 0. We need to dispatch tasks first.
 		// The completion check after dispatch (with dispatchedInThisIteration guard) prevents
@@ -2630,7 +2816,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		}
 
 		// Decide when the next snapshot dispatch should run.
-		state Version nextDispatchVersion;
+		Version nextDispatchVersion{ 0 };
 
 		// In simulation, use snapshot interval / 5 to ensure multiple dispatches run
 		// Otherwise, use the knob for the number of seconds between snapshot dispatch tasks.
@@ -2660,8 +2846,8 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		else
 			timeElapsed = 1.0;
 
-		state int countExpectedShardsDone = countAllShards * timeElapsed;
-		state int countShardsToDispatch = std::max<int>(0, countExpectedShardsDone - countShardsDone);
+		int countExpectedShardsDone = countAllShards * timeElapsed;
+		int countShardsToDispatch = std::max<int>(0, countExpectedShardsDone - countShardsDone);
 
 		// Calculate the number of shards that would have been dispatched by a normal (on-schedule)
 		// BackupSnapshotDispatchTask given the dispatch window and the start and expected-end versions of the
@@ -2702,12 +2888,12 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		    .detail("SnapshotIntervalSeconds", snapshotIntervalSeconds);
 
 		// Track whether we dispatched any tasks in this iteration
-		state bool dispatchedInThisIteration = false;
+		bool dispatchedInThisIteration = false;
 
 		// Dispatch random shards to catch up to the expected progress
 		while (countShardsToDispatch > 0) {
 			// First select ranges to add
-			state std::vector<KeyRange> rangesToAdd;
+			std::vector<KeyRange> rangesToAdd;
 
 			// Limit number of tasks added per transaction
 			int taskBatchSize = BUGGIFY ? deterministicRandom()->randomInt(1, countShardsToDispatch + 1)
@@ -2735,12 +2921,13 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 				}
 			}
 
-			state int64_t oldBatchSize = snapshotBatchSize.get();
-			state int64_t newBatchSize = oldBatchSize + rangesToAdd.size();
+			int64_t oldBatchSize = snapshotBatchSize.get();
+			int64_t newBatchSize = oldBatchSize + rangesToAdd.size();
 
 			// Now add the selected ranges in a single transaction.
 			tr->reset();
-			loop {
+			while (true) {
+				Error err;
 				try {
 					TraceEvent("FileBackupSnapshotDispatchAddingTasks")
 					    .suppressFor(2)
@@ -2751,16 +2938,16 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 					// For each range, make sure it isn't set in the dispatched range map.
-					state std::vector<Future<Optional<bool>>> beginReads;
-					state std::vector<Future<Optional<bool>>> endReads;
+					std::vector<Future<Optional<bool>>> beginReads;
+					std::vector<Future<Optional<bool>>> endReads;
 
 					for (auto& range : rangesToAdd) {
 						beginReads.push_back(config.snapshotRangeDispatchMap().get(tr, range.begin));
 						endReads.push_back(config.snapshotRangeDispatchMap().get(tr, range.end));
 					}
 
-					wait(store(snapshotBatchSize.get(), config.snapshotBatchSize().getOrThrow(tr)) &&
-					     waitForAll(beginReads) && waitForAll(endReads) && taskBucket->keepRunning(tr, task));
+					co_await (store(snapshotBatchSize.get(), config.snapshotBatchSize().getOrThrow(tr)) &&
+					          waitForAll(beginReads) && waitForAll(endReads) && taskBucket->keepRunning(tr, task));
 
 					// Snapshot batch size should be either oldBatchSize or newBatchSize. If new, this transaction
 					// is already done.
@@ -2774,7 +2961,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 						config.snapshotDispatchLastVersion().set(tr, tr->getReadVersion().get());
 					}
 
-					state std::vector<Future<Void>> addTaskFutures;
+					std::vector<Future<Void>> addTaskFutures;
 
 					for (i = 0; i < beginReads.size(); ++i) {
 						KeyRange& range = rangesToAdd[i];
@@ -2834,13 +3021,14 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 						}
 					}
 
-					wait(waitForAll(addTaskFutures));
-					wait(tr->commit());
+					co_await waitForAll(addTaskFutures);
+					co_await tr->commit();
 					dispatchedInThisIteration = true;
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 				}
+				co_await tr->onError(err);
 			}
 		}
 
@@ -2861,7 +3049,7 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 			Params.snapshotFinished().set(task, true);
 		}
 
-		return Void();
+		co_return;
 	}
 
 	// This function is just a wrapper for BackupSnapshotManifest::addTask() which is defined below.
@@ -2874,22 +3062,21 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 	                                           TaskCompletionKey completionKey,
 	                                           Reference<TaskFuture> waitFor = Reference<TaskFuture>());
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state BackupConfig config(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		BackupConfig config(task);
 
 		// Get the batch future and dispatch done keys, then clear them.
-		state Key snapshotBatchFutureKey;
-		state Key snapshotBatchDispatchDoneKey;
+		Key snapshotBatchFutureKey;
+		Key snapshotBatchDispatchDoneKey;
 
-		wait(store(snapshotBatchFutureKey, config.snapshotBatchFuture().getOrThrow(tr)) &&
-		     store(snapshotBatchDispatchDoneKey, config.snapshotBatchDispatchDoneKey().getOrThrow(tr)));
+		co_await (store(snapshotBatchFutureKey, config.snapshotBatchFuture().getOrThrow(tr)) &&
+		          store(snapshotBatchDispatchDoneKey, config.snapshotBatchDispatchDoneKey().getOrThrow(tr)));
 
-		state Reference<TaskFuture> snapshotBatchFuture = futureBucket->unpack(snapshotBatchFutureKey);
-		state Reference<TaskFuture> snapshotBatchDispatchDoneFuture =
-		    futureBucket->unpack(snapshotBatchDispatchDoneKey);
+		Reference<TaskFuture> snapshotBatchFuture = futureBucket->unpack(snapshotBatchFutureKey);
+		Reference<TaskFuture> snapshotBatchDispatchDoneFuture = futureBucket->unpack(snapshotBatchDispatchDoneKey);
 		config.snapshotBatchFuture().clear(tr);
 		config.snapshotBatchDispatchDoneKey().clear(tr);
 		config.snapshotBatchSize().clear(tr);
@@ -2898,30 +3085,38 @@ struct BackupSnapshotDispatchTask : BackupTaskFuncBase {
 		config.snapshotDispatchLastShardsBehind().set(tr, Params.shardsBehind().getOrDefault(task, 0));
 		config.snapshotDispatchLastVersion().set(tr, tr->getReadVersion().get());
 
-		state Reference<TaskFuture> snapshotFinishedFuture = task->getDoneFuture(futureBucket);
+		Reference<TaskFuture> snapshotFinishedFuture = task->getDoneFuture(futureBucket);
+
+		bool snapshotFinished = Params.snapshotFinished().getOrDefault(task, false);
+		TraceEvent("FileBackupSnapshotDispatchFinish")
+		    .detail("BackupUID", config.getUid())
+		    .detail("SnapshotFinished", snapshotFinished)
+		    .detail("ShardsBehind", Params.shardsBehind().getOrDefault(task, 0))
+		    .detail("NextDispatchVersion", Params.nextDispatchVersion().getOrDefault(task, -1));
 
 		// If the snapshot is finished, the next task is to write a snapshot manifest, otherwise it's another
 		// snapshot dispatch task. In either case, the task should wait for snapshotBatchFuture. The snapshot done
 		// key, passed to the current task, is also passed on.
-		if (Params.snapshotFinished().getOrDefault(task, false)) {
-			wait(success(addSnapshotManifestTask(
-			    tr, taskBucket, task, TaskCompletionKey::signal(snapshotFinishedFuture), snapshotBatchFuture)));
+		if (snapshotFinished) {
+			TraceEvent("FileBackupSnapshotDispatchAddingManifestTask").detail("BackupUID", config.getUid());
+			co_await addSnapshotManifestTask(
+			    tr, taskBucket, task, TaskCompletionKey::signal(snapshotFinishedFuture), snapshotBatchFuture);
 		} else {
-			wait(success(addTask(tr,
-			                     taskBucket,
-			                     task,
-			                     1,
-			                     TaskCompletionKey::signal(snapshotFinishedFuture),
-			                     snapshotBatchFuture,
-			                     Params.nextDispatchVersion().get(task))));
+			co_await addTask(tr,
+			                 taskBucket,
+			                 task,
+			                 1,
+			                 TaskCompletionKey::signal(snapshotFinishedFuture),
+			                 snapshotBatchFuture,
+			                 Params.nextDispatchVersion().get(task));
 		}
 
 		// This snapshot batch is finished, so set the batch done future.
-		wait(snapshotBatchDispatchDoneFuture->set(tr, taskBucket));
+		co_await snapshotBatchDispatchDoneFuture->set(tr, taskBucket);
 
-		wait(taskBucket->finish(tr, task));
+		co_await taskBucket->finish(tr, task);
 
-		return Void();
+		co_return;
 	}
 };
 StringRef BackupSnapshotDispatchTask::name = "file_backup_dispatch_ranges_5.2"_sr;
@@ -2953,71 +3148,77 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
 
-		wait(checkTaskVersion(cx, task, BackupLogRangeTaskFunc::name, BackupLogRangeTaskFunc::version));
+		co_await checkTaskVersion(cx, task, BackupLogRangeTaskFunc::name, BackupLogRangeTaskFunc::version);
 
-		state Version beginVersion = Params.beginVersion().get(task);
-		state Version endVersion = Params.endVersion().get(task);
+		Version beginVersion = Params.beginVersion().get(task);
+		Version endVersion = Params.endVersion().get(task);
 
-		state BackupConfig config(task);
-		state Reference<IBackupContainer> bc;
+		BackupConfig config(task);
+		Reference<IBackupContainer> bc;
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		loop {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			// Wait for the read version to pass endVersion
-			try {
-				wait(taskBucket->keepRunning(tr, task));
+			{
+				Error err;
+				try {
+					co_await taskBucket->keepRunning(tr, task);
 
-				if (!bc) {
-					// Backup container must be present if we're still here
-					Reference<IBackupContainer> _bc = wait(config.backupContainer().getOrThrow(tr));
-					bc = getBackupContainerWithProxy(_bc);
-				}
+					if (!bc) {
+						// Backup container must be present if we're still here
+						Reference<IBackupContainer> _bc = co_await config.backupContainer().getOrThrow(tr);
+						bc = getBackupContainerWithProxy(_bc);
+					}
 
-				Version currentVersion = tr->getReadVersion().get();
-				if (endVersion < currentVersion)
-					break;
+					Version currentVersion = tr->getReadVersion().get();
+					if (endVersion < currentVersion)
+						break;
 
-				wait(delay(std::max(CLIENT_KNOBS->BACKUP_RANGE_MINWAIT,
-				                    (double)(endVersion - currentVersion) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND)));
-				tr->reset();
-			} catch (Error& e) {
-				wait(tr->onError(e));
+					co_await delay(
+					    std::max(CLIENT_KNOBS->BACKUP_RANGE_MINWAIT,
+					             (double)(endVersion - currentVersion) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
+					tr->reset();
+					continue;
+				} catch (Error& e) {
+					err = e;
+				}
+				co_await tr->onError(err);
 			}
 		}
 
-		Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
+		Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
 
 		// Get the set of key ranges that hold mutations for (beginVersion, endVersion).  They will be queried in
 		// parallel below and there is a limit on how many we want to process in a single BackupLogRangeTask so if
 		// that limit is exceeded then set the addBackupLogRangeTasks boolean in Params and stop, signalling the
 		// finish() step to break up the (beginVersion, endVersion) range into smaller intervals which are then
 		// processed by individual BackupLogRangeTasks.
-		state Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion, endVersion, destUidValue);
+		Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion, endVersion, destUidValue);
 		if (ranges.size() > CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES) {
 			Params.addBackupLogRangeTasks().set(task, true);
-			return Void();
+			co_return;
 		}
 
 		// Block size must be at least large enough for 1 max size key, 1 max size value, and overhead, so
 		// conservatively 125k.
-		state int blockSize =
+		int blockSize =
 		    BUGGIFY ? deterministicRandom()->randomInt(125e3, 4e6) : CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE;
-		state Reference<IBackupFile> outFile = wait(bc->writeLogFile(beginVersion, endVersion, blockSize));
-		state LogFileWriter logFile(outFile, blockSize);
+		Reference<IBackupFile> outFile = co_await bc->writeLogFile(beginVersion, endVersion, blockSize);
+		LogFileWriter logFile(outFile, blockSize);
 
 		// Query all key ranges covering (beginVersion, endVersion) in parallel, writing their results to the
 		// results promise stream as they are received.  Note that this means the records read from the results
 		// stream are not likely to be in increasing Version order.
-		state PromiseStream<RangeResultWithVersion> results;
-		state std::vector<Future<Void>> rc;
+		PromiseStream<RangeResultWithVersion> results;
+		std::vector<Future<Void>> rc;
 
 		for (auto& range : ranges) {
 			rc.push_back(readCommitted(cx,
@@ -3030,7 +3231,7 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 			                           ReadLowPriority(CLIENT_KNOBS->BACKUP_READS_USE_LOW_PRIORITY)));
 		}
 
-		state Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const& result) mutable {
+		Future<Void> sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr<Void> const& result) mutable {
 			if (result.isError())
 				results.sendError(result.getError());
 			else
@@ -3038,34 +3239,42 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 			return Void();
 		});
 
-		state Version lastVersion;
-		try {
-			loop {
-				state RangeResultWithVersion r = waitNext(results.getFuture());
-				lock->release(r.first.expectedSize());
-
-				state int i = 0;
-				for (; i < r.first.size(); ++i) {
-					// Remove the backupLogPrefix + UID bytes from the key
-					wait(logFile.writeKV(r.first[i].key.substr(backupLogPrefixBytes + 16), r.first[i].value));
-					lastVersion = r.second;
+		Version lastVersion{ 0 };
+		{
+			Error caughtErr;
+			bool hasCaughtErr = false;
+			try {
+				while (true) {
+					RangeResultWithVersion r = co_await results.getFuture();
+					lock->release(r.first.expectedSize());
+
+					for (int i = 0; i < r.first.size(); ++i) {
+						// Remove the backupLogPrefix + UID bytes from the key
+						co_await logFile.writeKV(r.first[i].key.substr(backupLogPrefixBytes + 16), r.first[i].value);
+						lastVersion = r.second;
+					}
 				}
+			} catch (Error& e) {
+				caughtErr = e;
+				hasCaughtErr = true;
 			}
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled)
-				throw;
-
-			if (e.code() != error_code_end_of_stream) {
-				state Error err = e;
-				wait(config.logError(cx, err, format("Failed to write to file `%s'", outFile->getFileName().c_str())));
-				throw err;
+			if (hasCaughtErr) {
+				if (caughtErr.code() == error_code_actor_cancelled)
+					throw caughtErr;
+
+				if (caughtErr.code() != error_code_end_of_stream) {
+					Error err = caughtErr;
+					co_await config.logError(
+					    cx, err, format("Failed to write to file `%s'", outFile->getFileName().c_str()));
+					throw err;
+				}
 			}
 		}
 
 		// Make sure this task is still alive, if it's not then the data read above could be incomplete.
-		wait(taskBucket->keepRunning(cx, task));
+		co_await taskBucket->keepRunning(cx, task);
 
-		wait(outFile->finish());
+		co_await outFile->finish();
 
 		TraceEvent("FileBackupWroteLogFile")
 		    .suppressFor(60)
@@ -3077,18 +3286,18 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 
 		Params.fileSize().set(task, outFile->size());
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 int priority,
-	                                 Version beginVersion,
-	                                 Version endVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key key = wait(addBackupTask(
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           int priority,
+	                           Version beginVersion,
+	                           Version endVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key key = co_await addBackupTask(
 		    BackupLogRangeTaskFunc::name,
 		    BackupLogRangeTaskFunc::version,
 		    tr,
@@ -3101,17 +3310,17 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 			    Params.endVersion().set(task, endVersion);
 			    Params.addBackupLogRangeTasks().set(task, false);
 		    },
-		    priority));
-		return key;
+		    priority);
+		co_return key;
 	}
 
-	ACTOR static Future<Void> startBackupLogRangeInternal(Reference<ReadYourWritesTransaction> tr,
-	                                                      Reference<TaskBucket> taskBucket,
-	                                                      Reference<FutureBucket> futureBucket,
-	                                                      Reference<Task> task,
-	                                                      Reference<TaskFuture> taskFuture,
-	                                                      Version beginVersion,
-	                                                      Version endVersion) {
+	static Future<Void> startBackupLogRangeInternal(Reference<ReadYourWritesTransaction> tr,
+	                                                Reference<TaskBucket> taskBucket,
+	                                                Reference<FutureBucket> futureBucket,
+	                                                Reference<Task> task,
+	                                                Reference<TaskFuture> taskFuture,
+	                                                Version beginVersion,
+	                                                Version endVersion) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
@@ -3140,32 +3349,33 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 			tasks++;
 		}
 
-		wait(waitForAll(addTaskVector));
+		co_await waitForAll(addTaskVector);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Version beginVersion = Params.beginVersion().get(task);
-		state Version endVersion = Params.endVersion().get(task);
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		state BackupConfig config(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Version beginVersion = Params.beginVersion().get(task);
+		Version endVersion = Params.endVersion().get(task);
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		BackupConfig config(task);
 
 		if (Params.fileSize().exists(task)) {
 			config.logBytesWritten().atomicOp(tr, Params.fileSize().get(task), MutationRef::AddValue);
 		}
 
 		if (Params.addBackupLogRangeTasks().get(task)) {
-			wait(startBackupLogRangeInternal(tr, taskBucket, futureBucket, task, taskFuture, beginVersion, endVersion));
+			co_await startBackupLogRangeInternal(
+			    tr, taskBucket, futureBucket, task, taskFuture, beginVersion, endVersion);
 		} else {
-			wait(taskFuture->set(tr, taskBucket));
+			co_await taskFuture->set(tr, taskBucket);
 		}
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
+		co_return;
 	}
 };
 
@@ -3184,14 +3394,14 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 		static TaskParam<Key> destUidValue() { return __FUNCTION__sr; }
 	} Params;
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 UID logUid,
-	                                 TaskCompletionKey completionKey,
-	                                 Key destUidValue,
-	                                 Version endVersion = 0,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key key = wait(addBackupTask(
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           UID logUid,
+	                           TaskCompletionKey completionKey,
+	                           Key destUidValue,
+	                           Version endVersion = 0,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key key = co_await addBackupTask(
 		    EraseLogRangeTaskFunc::name,
 		    EraseLogRangeTaskFunc::version,
 		    tr,
@@ -3206,30 +3416,31 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase {
 			    Params.destUidValue().set(task, destUidValue);
 		    },
 		    0,
-		    SetValidation::False));
+		    SetValidation::False);
 
-		return key;
+		co_return key;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
-		wait(checkTaskVersion(tr->getDatabase(), task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version));
+		co_await checkTaskVersion(tr->getDatabase(), task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version);
 
-		state Version endVersion = Params.endVersion().get(task);
-		state Key destUidValue = Params.destUidValue().get(task);
+		Version endVersion = Params.endVersion().get(task);
+		Key destUidValue = Params.destUidValue().get(task);
 
-		state BackupConfig config(task);
-		state Key logUidValue = config.getUidAsKey();
+		BackupConfig config(task);
+		Key logUidValue = config.getUidAsKey();
 
-		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task) &&
-		     eraseLogData(
-		         tr, logUidValue, destUidValue, endVersion != 0 ? Optional<Version>(endVersion) : Optional<Version>()));
+		co_await (
+		    taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task) &&
+		    eraseLogData(
+		        tr, logUidValue, destUidValue, endVersion != 0 ? Optional<Version>(endVersion) : Optional<Version>()));
 
-		return Void();
+		co_return;
 	}
 
 	Future<Void> execute(Database cx,
@@ -3257,33 +3468,34 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
 	} Params;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		wait(checkTaskVersion(tr->getDatabase(), task, BackupLogsDispatchTask::name, BackupLogsDispatchTask::version));
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		co_await checkTaskVersion(
+		    tr->getDatabase(), task, BackupLogsDispatchTask::name, BackupLogsDispatchTask::version);
 
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-		state Reference<TaskFuture> onDone = task->getDoneFuture(futureBucket);
-		state Version prevBeginVersion = Params.prevBeginVersion().get(task);
-		state Version beginVersion = Params.beginVersion().get(task);
-		state BackupConfig config(task);
+		Reference<TaskFuture> onDone = task->getDoneFuture(futureBucket);
+		Version prevBeginVersion = Params.prevBeginVersion().get(task);
+		Version beginVersion = Params.beginVersion().get(task);
+		BackupConfig config(task);
 		config.latestLogEndVersion().set(tr, beginVersion);
 
-		state bool stopWhenDone;
-		state Optional<Version> restorableVersion;
-		state EBackupState backupState;
-		state Optional<std::string> tag;
-		state Optional<Version> latestSnapshotEndVersion;
-		state Optional<bool> partitionedLog;
+		bool stopWhenDone{ false };
+		Optional<Version> restorableVersion;
+		EBackupState backupState;
+		Optional<std::string> tag;
+		Optional<Version> latestSnapshotEndVersion;
+		Optional<MutationLogType> mutationLogType;
 
-		wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
-		     store(restorableVersion, config.getLatestRestorableVersion(tr)) &&
-		     store(backupState, config.stateEnum().getOrThrow(tr)) && store(tag, config.tag().get(tr)) &&
-		     store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
-		     store(partitionedLog, config.partitionedLogEnabled().get(tr)));
+		co_await (store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
+		          store(restorableVersion, config.getLatestRestorableVersion(tr)) &&
+		          store(backupState, config.stateEnum().getOrThrow(tr)) && store(tag, config.tag().get(tr)) &&
+		          store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
+		          store(mutationLogType, config.mutationLogType().get(tr)));
 
 		// If restorable, update the last restorable version for this tag
 		if (restorableVersion.present() && tag.present()) {
@@ -3297,19 +3509,19 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		// If stopWhenDone is set and there is a restorable version, set the done future and do not create further
 		// tasks.
 		if (stopWhenDone && restorableVersion.present()) {
-			wait(onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
+			co_await (onDone->set(tr, taskBucket) && taskBucket->finish(tr, task));
 
 			TraceEvent("FileBackupLogsDispatchDone")
 			    .detail("BackupUID", config.getUid())
 			    .detail("BeginVersion", beginVersion)
 			    .detail("RestorableVersion", restorableVersion.orDefault(-1));
 
-			return Void();
+			co_return;
 		}
 
-		state Version endVersion = std::max<Version>(tr->getReadVersion().get() + 1,
-		                                             beginVersion + (CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES - 1) *
-		                                                                CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
+		Version endVersion = std::max<Version>(tr->getReadVersion().get() + 1,
+		                                       beginVersion + (CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES - 1) *
+		                                                          CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
 
 		TraceEvent("FileBackupLogDispatch")
 		    .suppressFor(60)
@@ -3317,53 +3529,53 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		    .detail("EndVersion", endVersion)
 		    .detail("RestorableVersion", restorableVersion.orDefault(-1));
 
-		state Reference<TaskFuture> logDispatchBatchFuture = futureBucket->future(tr);
+		Reference<TaskFuture> logDispatchBatchFuture = futureBucket->future(tr);
 
 		// If a snapshot has ended for this backup then mutations are higher priority to reduce backup lag
-		state int priority = latestSnapshotEndVersion.present() ? 1 : 0;
+		int priority = latestSnapshotEndVersion.present() ? 1 : 0;
 
-		if (!partitionedLog.present() || !partitionedLog.get()) {
+		if (!mutationLogType.present() || mutationLogType.get() == MutationLogType::DEFAULT) {
 			// Add the initial log range task to read/copy the mutations and the next logs dispatch task which will
 			// run after this batch is done
 			// read blog/ prefix and write those (param1, param2) into files
-			wait(success(BackupLogRangeTaskFunc::addTask(tr,
-			                                             taskBucket,
-			                                             task,
-			                                             priority,
-			                                             beginVersion,
-			                                             endVersion,
-			                                             TaskCompletionKey::joinWith(logDispatchBatchFuture))));
+			co_await BackupLogRangeTaskFunc::addTask(tr,
+			                                         taskBucket,
+			                                         task,
+			                                         priority,
+			                                         beginVersion,
+			                                         endVersion,
+			                                         TaskCompletionKey::joinWith(logDispatchBatchFuture));
 			// issue the next key range
-			wait(success(BackupLogsDispatchTask::addTask(tr,
-			                                             taskBucket,
-			                                             task,
-			                                             priority,
-			                                             beginVersion,
-			                                             endVersion,
-			                                             TaskCompletionKey::signal(onDone),
-			                                             logDispatchBatchFuture)));
+			co_await BackupLogsDispatchTask::addTask(tr,
+			                                         taskBucket,
+			                                         task,
+			                                         priority,
+			                                         beginVersion,
+			                                         endVersion,
+			                                         TaskCompletionKey::signal(onDone),
+			                                         logDispatchBatchFuture);
 
 			// Do not erase at the first time
 			if (prevBeginVersion > 0) {
-				state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
-				wait(eraseLogData(tr, config.getUidAsKey(), destUidValue, Optional<Version>(beginVersion)));
+				Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
+				co_await eraseLogData(tr, config.getUidAsKey(), destUidValue, Optional<Version>(beginVersion));
 			}
 		} else {
 			// Skip mutation copy and erase backup mutations. Just check back periodically.
 			Version scheduledVersion = tr->getReadVersion().get() +
 			                           CLIENT_KNOBS->BACKUP_POLL_PROGRESS_SECONDS * CLIENT_KNOBS->VERSIONS_PER_SECOND;
-			wait(success(BackupLogsDispatchTask::addTask(tr,
-			                                             taskBucket,
-			                                             task,
-			                                             1,
-			                                             beginVersion,
-			                                             endVersion,
-			                                             TaskCompletionKey::signal(onDone),
-			                                             Reference<TaskFuture>(),
-			                                             scheduledVersion)));
+			co_await BackupLogsDispatchTask::addTask(tr,
+			                                         taskBucket,
+			                                         task,
+			                                         1,
+			                                         beginVersion,
+			                                         endVersion,
+			                                         TaskCompletionKey::signal(onDone),
+			                                         Reference<TaskFuture>(),
+			                                         scheduledVersion);
 		}
 
-		wait(taskBucket->finish(tr, task));
+		co_await taskBucket->finish(tr, task);
 
 		TraceEvent("FileBackupLogsDispatchContinuing")
 		    .suppressFor(60)
@@ -3371,19 +3583,19 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 		    .detail("BeginVersion", beginVersion)
 		    .detail("EndVersion", endVersion);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 int priority,
-	                                 Version prevBeginVersion,
-	                                 Version beginVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
-	                                 Version scheduledVersion = invalidVersion) {
-		Key key = wait(addBackupTask(
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           int priority,
+	                           Version prevBeginVersion,
+	                           Version beginVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>(),
+	                           Version scheduledVersion = invalidVersion) {
+		Key key = co_await addBackupTask(
 		    BackupLogsDispatchTask::name,
 		    BackupLogsDispatchTask::version,
 		    tr,
@@ -3398,8 +3610,8 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 				    ReservedTaskParams::scheduledVersion().set(task, scheduledVersion);
 			    }
 		    },
-		    priority));
-		return key;
+		    priority);
+		co_return key;
 	}
 
 	StringRef getName() const override { return name; };
@@ -3426,42 +3638,43 @@ struct FileBackupFinishedTask : BackupTaskFuncBase {
 
 	StringRef getName() const override { return name; };
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		wait(checkTaskVersion(tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version));
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		co_await checkTaskVersion(
+		    tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version);
 
-		state BackupConfig backup(task);
-		state UID uid = backup.getUid();
+		BackupConfig backup(task);
+		UID uid = backup.getUid();
 
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
-		state Key destUidValue = wait(backup.destUidValue().getOrThrow(tr));
+		Key destUidValue = co_await backup.destUidValue().getOrThrow(tr);
 
-		wait(eraseLogData(tr, backup.getUidAsKey(), destUidValue) && clearBackupStartID(tr, uid));
+		co_await (eraseLogData(tr, backup.getUidAsKey(), destUidValue) && clearBackupStartID(tr, uid));
 
 		backup.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
 
-		wait(taskBucket->finish(tr, task));
+		co_await taskBucket->finish(tr, task);
 
 		TraceEvent("FileBackupFinished").detail("BackupUID", uid);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key key = wait(addBackupTask(FileBackupFinishedTask::name,
-		                             FileBackupFinishedTask::version,
-		                             tr,
-		                             taskBucket,
-		                             completionKey,
-		                             BackupConfig(parentTask),
-		                             waitFor));
-		return key;
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key key = co_await addBackupTask(FileBackupFinishedTask::name,
+		                                 FileBackupFinishedTask::version,
+		                                 tr,
+		                                 taskBucket,
+		                                 completionKey,
+		                                 BackupConfig(parentTask),
+		                                 waitFor);
+		co_return key;
 	}
 
 	Future<Void> execute(Database cx,
@@ -3485,60 +3698,64 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 	static constexpr uint32_t version = 1;
 	static struct {
 		static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
+		static TaskParam<int64_t> totalBytes() { return __FUNCTION__sr; }
 	} Params;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state BackupConfig config(task);
-		state Reference<IBackupContainer> bc;
-		state DatabaseConfiguration dbConfig;
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		BackupConfig config(task);
+		Reference<IBackupContainer> bc;
+		DatabaseConfiguration dbConfig;
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
 		// Read the entire range file map into memory, then walk it backwards from its last entry to produce a list
 		// of non overlapping key range files
-		state std::map<Key, BackupConfig::RangeSlice> localmap;
-		state Key startKey;
-		state int batchSize = BUGGIFY ? 1 : 1000000;
+		std::map<Key, BackupConfig::RangeSlice> localmap;
+		Key startKey;
+		int batchSize = BUGGIFY ? 1 : 1000000;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(taskBucket->keepRunning(tr, task));
-				wait(store(dbConfig, getDatabaseConfiguration(cx)));
+				co_await taskBucket->keepRunning(tr, task);
+				dbConfig = co_await getDatabaseConfiguration(cx);
 
 				if (!bc) {
 					// Backup container must be present if we're still here
-					Reference<IBackupContainer> _bc = wait(config.backupContainer().getOrThrow(tr));
+					Reference<IBackupContainer> _bc = co_await config.backupContainer().getOrThrow(tr);
 					bc = getBackupContainerWithProxy(_bc);
 				}
-
 				BackupConfig::RangeFileMapT::RangeResultType rangeresults =
-				    wait(config.snapshotRangeFileMap().getRange(tr, startKey, {}, batchSize));
+				    co_await config.snapshotRangeFileMap().getRange(tr, startKey, {}, batchSize);
 
 				for (auto& p : rangeresults.results) {
 					localmap.insert(p);
 				}
 
-				if (!rangeresults.more)
+				if (!rangeresults.more) {
 					break;
+				}
 
 				startKey = keyAfter(rangeresults.results.back().first);
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		state std::vector<std::string> files;
-		state std::vector<std::pair<Key, Key>> beginEndKeys;
-		state Version maxVer = 0;
-		state Version minVer = std::numeric_limits<Version>::max();
-		state int64_t totalBytes = 0;
+		std::vector<std::string> files;
+		std::vector<std::pair<Key, Key>> beginEndKeys;
+		Version maxVer = 0;
+		Version minVer = std::numeric_limits<Version>::max();
+		int64_t totalBytes = 0;
 
 		if (!localmap.empty()) {
 			// Get iterator that points to greatest key, start there.
@@ -3575,10 +3792,20 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 			}
 		}
 
+		// Log what range files were found for debugging mode=BOTH issues
+		TraceEvent("BackupSnapshotManifestRangeFileSummary")
+		    .detail("BackupUID", config.getUid())
+		    .detail("LocalMapSize", localmap.size())
+		    .detail("FilesFound", files.size())
+		    .detail("TotalBytes", totalBytes)
+		    .detail("MinVersion", minVer == std::numeric_limits<Version>::max() ? -1 : minVer)
+		    .detail("MaxVersion", maxVer);
+
 		Params.endVersion().set(task, maxVer);
+		Params.totalBytes().set(task, totalBytes);
 
 		// Avoid keyRange filtering optimization for 'manifest' files
-		wait(bc->writeKeyspaceSnapshotFile(files, beginEndKeys, totalBytes, IncludeKeyRangeMap::True));
+		co_await bc->writeKeyspaceSnapshotFile(files, beginEndKeys, totalBytes, IncludeKeyRangeMap::True);
 
 		TraceEvent(SevInfo, "FileBackupWroteSnapshotManifest")
 		    .detail("BackupUID", config.getUid())
@@ -3586,37 +3813,40 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 		    .detail("EndVersion", maxVer)
 		    .detail("TotalBytes", totalBytes);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		wait(checkTaskVersion(tr->getDatabase(), task, BackupSnapshotManifest::name, BackupSnapshotManifest::version));
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		co_await checkTaskVersion(
+		    tr->getDatabase(), task, BackupSnapshotManifest::name, BackupSnapshotManifest::version);
 
-		state BackupConfig config(task);
+		BackupConfig config(task);
 
 		// Set the latest snapshot end version, which was set during the execute phase
 		config.latestSnapshotEndVersion().set(tr, Params.endVersion().get(task));
 
-		state bool stopWhenDone;
-		state EBackupState backupState;
-		state Optional<Version> restorableVersion;
-		state Optional<Version> firstSnapshotEndVersion;
-		state Optional<std::string> tag;
+		bool stopWhenDone{ false };
+		EBackupState backupState;
+		Optional<Version> restorableVersion;
+		Optional<Version> firstSnapshotEndVersion;
+		Optional<std::string> tag;
 
-		wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
-		     store(backupState, config.stateEnum().getOrThrow(tr)) &&
-		     store(restorableVersion, config.getLatestRestorableVersion(tr)) &&
-		     store(firstSnapshotEndVersion, config.firstSnapshotEndVersion().get(tr)) &&
-		     store(tag, config.tag().get(tr)));
+		co_await (store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) &&
+		          store(backupState, config.stateEnum().getOrThrow(tr)) &&
+		          store(restorableVersion, config.getLatestRestorableVersion(tr)) &&
+		          store(firstSnapshotEndVersion, config.firstSnapshotEndVersion().get(tr)) &&
+		          store(tag, config.tag().get(tr)));
 
 		// If restorable, update the last restorable version for this tag
 		if (restorableVersion.present() && tag.present()) {
 			FileBackupAgent().setLastRestorable(tr, StringRef(tag.get()), restorableVersion.get());
 		}
 
+		// Always set firstSnapshotEndVersion if not already set
+		// This is required for getLatestRestorableVersion() to work correctly
 		if (!firstSnapshotEndVersion.present()) {
 			config.firstSnapshotEndVersion().set(tr, Params.endVersion().get(task));
 		}
@@ -3628,33 +3858,33 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 		// Unless we are to stop, start the next snapshot using the default interval
 		Reference<TaskFuture> snapshotDoneFuture = task->getDoneFuture(futureBucket);
 		if (!stopWhenDone) {
-			wait(config.initNewSnapshot(tr) &&
-			     success(BackupSnapshotDispatchTask::addTask(
-			         tr, taskBucket, task, 1, TaskCompletionKey::signal(snapshotDoneFuture))));
+			co_await (config.initNewSnapshot(tr) &&
+			          success(BackupSnapshotDispatchTask::addTask(
+			              tr, taskBucket, task, 1, TaskCompletionKey::signal(snapshotDoneFuture))));
 		} else {
 			// Set the done future as the snapshot is now complete.
-			wait(snapshotDoneFuture->set(tr, taskBucket));
+			co_await snapshotDoneFuture->set(tr, taskBucket);
 		}
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key key = wait(addBackupTask(BackupSnapshotManifest::name,
-		                             BackupSnapshotManifest::version,
-		                             tr,
-		                             taskBucket,
-		                             completionKey,
-		                             BackupConfig(parentTask),
-		                             waitFor,
-		                             NOP_SETUP_TASK_FN,
-		                             1));
-		return key;
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key key = co_await addBackupTask(BackupSnapshotManifest::name,
+		                                 BackupSnapshotManifest::version,
+		                                 tr,
+		                                 taskBucket,
+		                                 completionKey,
+		                                 BackupConfig(parentTask),
+		                                 waitFor,
+		                                 NOP_SETUP_TASK_FN,
+		                                 1);
+		co_return key;
 	}
 
 	StringRef getName() const override { return name; };
@@ -3710,13 +3940,13 @@ struct BulkDumpTaskFunc : BackupTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state BackupConfig config(task);
-		state Version snapshotVersion = Params.snapshotVersion().get(task);
-		state std::string jobId = Params.bulkDumpJobId().getOrDefault(task, "");
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		BackupConfig config(task);
+		Version snapshotVersion = Params.snapshotVersion().get(task);
+		std::string jobId = Params.bulkDumpJobId().getOrDefault(task, "");
 
 		TraceEvent("BulkDumpTaskStart")
 		    .detail("BackupUID", config.getUid())
@@ -3724,213 +3954,252 @@ struct BulkDumpTaskFunc : BackupTaskFuncBase {
 		    .detail("BulkDumpJobId", jobId);
 
 		// Declare state variables before try block so they're accessible in catch block
-		state std::vector<KeyRange> backupRanges;
-		state Reference<IBackupContainer> bc;
-		state int originalBulkDumpMode = 0;
-		state BulkLoadTransportMethod transportMethod = BulkLoadTransportMethod::CP;
-		state BulkDumpState bulkDumpJob;
-		state bool jobAlreadyRunning = false;
-
-		try {
-			// Submit BulkDump job via ManagementAPI
-			// This is a black box delegation to the existing BulkDump system
-
-			// Get backup ranges from config
-			std::vector<KeyRange> ranges = wait(config.backupRanges().getOrThrow(cx.getReference()));
-			backupRanges = ranges;
-			Reference<IBackupContainer> container = wait(config.backupContainer().getOrThrow(cx.getReference()));
-			bc = container;
-
-			// Read the original BulkDump mode from config (saved by StartFullBackupTaskFunc before task creation).
-			// This is persisted in the database so we can restore the correct mode even after a crash.
-			int mode = wait(config.originalBulkDumpMode().getD(cx.getReference(), Snapshot::False, 0));
-			originalBulkDumpMode = mode;
-
-			// Determine transport method from backup URL
-			transportMethod =
-			    isBlobstoreUrl(bc->getURL()) ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP;
-
-			// Check if there's already a running BulkDump job (e.g., from a previous task attempt or another agent).
-			// If so, we'll monitor that job instead of submitting a new one. This prevents the "Conflict to a
-			// running BulkDump job" error that occurs when multiple backup agents try to run the same task.
-			state Transaction checkTr(cx);
-			loop {
-				try {
-					checkTr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-					checkTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-					Optional<BulkDumpState> existingJob = wait(getSubmittedBulkDumpJob(&checkTr));
-					if (existingJob.present()) {
-						bulkDumpJob = existingJob.get();
-						jobAlreadyRunning = true;
-						TraceEvent("BulkDumpTaskUsingExistingJob")
-						    .detail("BackupUID", config.getUid())
-						    .detail("ExistingJobId", bulkDumpJob.getJobId());
+		std::vector<KeyRange> backupRanges;
+		Reference<IBackupContainer> bc;
+		int originalBulkDumpMode = 0;
+		BulkLoadTransportMethod transportMethod = BulkLoadTransportMethod::CP;
+		BulkDumpState bulkDumpJob;
+		bool jobAlreadyRunning = false;
+
+		{
+			Error savedError;
+			try {
+				// Submit BulkDump job via ManagementAPI
+				// This is a black box delegation to the existing BulkDump system
+
+				// Get backup ranges from config
+				std::vector<KeyRange> ranges = co_await config.backupRanges().getOrThrow(cx.getReference());
+				backupRanges = ranges;
+				Reference<IBackupContainer> container = co_await config.backupContainer().getOrThrow(cx.getReference());
+				bc = container;
+
+				// Read the original BulkDump mode from config (saved by StartFullBackupTaskFunc before task creation).
+				// This is persisted in the database so we can restore the correct mode even after a crash.
+				int mode = co_await config.originalBulkDumpMode().getD(cx.getReference(), Snapshot::False, 0);
+				originalBulkDumpMode = mode;
+
+				// Determine transport method from backup URL
+				transportMethod =
+				    isBlobstoreUrl(bc->getURL()) ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP;
+
+				// Check if there's already a running BulkDump job (e.g., from a previous task attempt or another
+				// agent). If so, we'll monitor that job instead of submitting a new one. This prevents the "Conflict to
+				// a running BulkDump job" error that occurs when multiple backup agents try to run the same task.
+				Transaction checkTr(cx);
+				while (true) {
+					Error err;
+					try {
+						checkTr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+						checkTr.setOption(FDBTransactionOptions::LOCK_AWARE);
+						Optional<BulkDumpState> existingJob = co_await getSubmittedBulkDumpJob(&checkTr);
+						if (existingJob.present()) {
+							bulkDumpJob = existingJob.get();
+							jobAlreadyRunning = true;
+							TraceEvent("BulkDumpTaskUsingExistingJob")
+							    .detail("BackupUID", config.getUid())
+							    .detail("ExistingJobId", bulkDumpJob.getJobId());
+						}
+						break;
+					} catch (Error& e) {
+						err = e;
 					}
-					break;
-				} catch (Error& e) {
-					wait(checkTr.onError(e));
+					co_await checkTr.onError(err);
 				}
-			}
 
-			if (!jobAlreadyRunning) {
-				// Enable BulkDump mode at the DD level before submitting the job
-				wait(success(setBulkDumpMode(cx, 1)));
+				if (!jobAlreadyRunning) {
+					// Enable BulkDump mode at the DD level before submitting the job
+					co_await setBulkDumpMode(cx, 1);
 
-				// Configure BulkDump job for the full keyspace
-				// BulkDump/BulkLoad requires the load range to be a subset of the dump range.
-				// Using normalKeys for both ensures compatibility regardless of user-specified ranges.
-				// Store data under data/<container>/bulkdump_data/ to be consistent with backup container layout
-				std::string bulkDumpRoot = getBackupDataPath(bc->getURL(), "bulkdump_data");
-				bulkDumpJob = createBulkDumpJob(normalKeys, bulkDumpRoot, BulkLoadType::SST, transportMethod);
+					// Configure BulkDump job for the full keyspace
+					// BulkDump/BulkLoad requires the load range to be a subset of the dump range.
+					// Using normalKeys for both ensures compatibility regardless of user-specified ranges.
+					// Store data under data/<container>/bulkdump_data/ to be consistent with backup container layout
+					std::string bulkDumpRoot = getBackupDataPath(bc->getURL(), "bulkdump_data");
+					bulkDumpJob = createBulkDumpJob(normalKeys, bulkDumpRoot, BulkLoadType::SST, transportMethod);
 
-				// Submit the BulkDump job
-				wait(submitBulkDumpJob(cx, bulkDumpJob));
-			}
+					// Submit the BulkDump job
+					co_await submitBulkDumpJob(cx, bulkDumpJob);
 
-			// Store job ID for monitoring
-			Params.bulkDumpJobId().set(task, bulkDumpJob.getJobId().toString());
+					// Set ownership so bulkdump status shows it belongs to this backup
+					std::string backupTag = co_await config.tag().getOrThrow(cx.getReference());
+					BulkDumpOwnerInfo ownerInfo(config.getUid(), "backup", backupTag, now());
+					co_await setBulkDumpOwner(cx, bulkDumpJob.getJobId(), ownerInfo);
+				}
 
-			// Monitor BulkDump progress - timeout is configurable for large datasets
-			state bool completed = wait(monitorBulkDumpJobCompletion(cx,
-			                                                         bulkDumpJob.getJobId(),
-			                                                         CLIENT_KNOBS->BULKDUMP_JOB_TIMEOUT,
-			                                                         5.0)); // Poll every 5 seconds
+				// Store job ID for monitoring
+				Params.bulkDumpJobId().set(task, bulkDumpJob.getJobId().toString());
+
+				// Monitor BulkDump progress - timeout is configurable for large datasets
+				bool completed = co_await monitorBulkDumpJobCompletion(cx,
+				                                                       bulkDumpJob.getJobId(),
+				                                                       CLIENT_KNOBS->BULKDUMP_JOB_TIMEOUT,
+				                                                       5.0); // Poll every 5 seconds
+
+				if (!completed) {
+					TraceEvent(SevWarn, "BulkDumpTaskTimeout")
+					    .detail("BackupUID", config.getUid())
+					    .detail("BulkDumpJobId", bulkDumpJob.getJobId())
+					    .detail("TimeoutDuration", 300.0);
+					Params.timeoutOccurred().set(task, true);
+				}
 
-			if (!completed) {
-				TraceEvent(SevWarn, "BulkDumpTaskTimeout")
+				TraceEvent("BulkDumpTaskComplete")
 				    .detail("BackupUID", config.getUid())
 				    .detail("BulkDumpJobId", bulkDumpJob.getJobId())
-				    .detail("TimeoutDuration", 300.0);
-				Params.timeoutOccurred().set(task, true);
-			}
+				    .detail("TimeoutOccurred", Params.timeoutOccurred().getOrDefault(task, false))
+				    .detail("JobAlreadyRunning", jobAlreadyRunning);
 
-			TraceEvent("BulkDumpTaskComplete")
-			    .detail("BackupUID", config.getUid())
-			    .detail("BulkDumpJobId", bulkDumpJob.getJobId())
-			    .detail("TimeoutOccurred", Params.timeoutOccurred().getOrDefault(task, false))
-			    .detail("JobAlreadyRunning", jobAlreadyRunning);
+				// Restore original BulkDump mode after job completes, but only if:
+				// 1. We actually enabled the mode (not if we just monitored an existing job)
+				// 2. The original mode was different from what we set
+				if (!jobAlreadyRunning && originalBulkDumpMode != 1) {
+					co_await setBulkDumpMode(cx, originalBulkDumpMode);
+				}
 
-			// Restore original BulkDump mode after job completes, but only if:
-			// 1. We actually enabled the mode (not if we just monitored an existing job)
-			// 2. The original mode was different from what we set
-			if (!jobAlreadyRunning && originalBulkDumpMode != 1) {
-				wait(success(setBulkDumpMode(cx, originalBulkDumpMode)));
-			}
+				// Write the keyspace snapshot file to mark the backup as complete
+				// This is essential for the restore process to find the snapshot
+				if (completed) {
+					// Verify that BulkDump data was actually written before writing snapshot
+					bool datasetComplete =
+					    co_await verifyBulkDumpDatasetCompleteness(bc, bulkDumpJob.getJobId().toString());
+					if (!datasetComplete) {
+						TraceEvent(SevError, "BulkDumpTaskDatasetIncomplete")
+						    .detail("BackupUID", config.getUid())
+						    .detail("BulkDumpJobId", bulkDumpJob.getJobId());
+						throw backup_error();
+					}
+
+					// Build beginEndKeys from backup ranges
+					std::vector<std::pair<Key, Key>> beginEndKeys;
+					for (const auto& range : backupRanges) {
+						beginEndKeys.emplace_back(range.begin, range.end);
+					}
 
-			// Write the keyspace snapshot file to mark the backup as complete
-			// This is essential for the restore process to find the snapshot
-			if (completed) {
-				// Build beginEndKeys from backup ranges
-				std::vector<std::pair<Key, Key>> beginEndKeys;
-				for (const auto& range : backupRanges) {
-					beginEndKeys.emplace_back(range.begin, range.end);
+					// Create BulkDump snapshot metadata
+					// Note: totalBytes and totalKeys are set to 0 here. Unlike traditional range file backups
+					// which accumulate fileSize from each RangeSlice, BulkDump writes its data to manifest files
+					// in the backup container (one per shard). To get accurate byte/key counts, we would need to:
+					//   1. List all manifest files under bc->getURL()/bulkDumpJobId/
+					//   2. Read each BulkLoadManifest and call getTotalBytes()/getKeyCount()
+					//   3. Aggregate the totals
+					// For now, zeros are acceptable since the critical field is bulkDumpJobId which BulkLoad
+					// uses to locate the data. TODO: Fix. The byte counts are informational for status display.
+					SnapshotMetadata metadata =
+					    SnapshotMetadata::bulkDump(bulkDumpJob.getJobId().toString(), snapshotVersion, 0, 0);
+
+					// Write the snapshot file - empty file list since BulkDump uses job manifests, not range files
+					co_await bc->writeKeyspaceSnapshotFile({}, // No individual range files for BulkDump
+					                                       beginEndKeys,
+					                                       0, // See comment above about totalBytes
+					                                       IncludeKeyRangeMap::False,
+					                                       metadata);
+
+					TraceEvent("BulkDumpTaskWroteSnapshot")
+					    .detail("BackupUID", config.getUid())
+					    .detail("BulkDumpJobId", bulkDumpJob.getJobId())
+					    .detail("SnapshotVersion", snapshotVersion)
+					    .detail("RangeCount", backupRanges.size());
 				}
 
-				// Create BulkDump snapshot metadata
-				// Note: totalBytes and totalKeys are set to 0 here. Unlike traditional range file backups
-				// which accumulate fileSize from each RangeSlice, BulkDump writes its data to manifest files
-				// in the backup container (one per shard). To get accurate byte/key counts, we would need to:
-				//   1. List all manifest files under bc->getURL()/bulkDumpJobId/
-				//   2. Read each BulkLoadManifest and call getTotalBytes()/getKeyCount()
-				//   3. Aggregate the totals
-				// For now, zeros are acceptable since the critical field is bulkDumpJobId which BulkLoad
-				// uses to locate the data. TODO: Fix. The byte counts are informational for status display.
-				SnapshotMetadata metadata =
-				    SnapshotMetadata::bulkDump(bulkDumpJob.getJobId().toString(), snapshotVersion, 0, 0);
-
-				// Write the snapshot file - empty file list since BulkDump uses job manifests, not range files
-				wait(bc->writeKeyspaceSnapshotFile({}, // No individual range files for BulkDump
-				                                   beginEndKeys,
-				                                   0, // See comment above about totalBytes
-				                                   IncludeKeyRangeMap::False,
-				                                   metadata));
-
-				TraceEvent("BulkDumpTaskWroteSnapshot")
-				    .detail("BackupUID", config.getUid())
-				    .detail("BulkDumpJobId", bulkDumpJob.getJobId())
-				    .detail("SnapshotVersion", snapshotVersion)
-				    .detail("RangeCount", backupRanges.size());
+				// Increment counter for test assertions
+				g_bulkDumpTaskCompleteCount.fetch_add(1);
+				co_return;
+			} catch (Error& e) {
+				savedError = e;
 			}
-
-			// Increment counter for test assertions
-			g_bulkDumpTaskCompleteCount.fetch_add(1);
-
-		} catch (Error& e) {
-			state Error savedError = e;
 			TraceEvent(SevWarn, "BulkDumpTaskError")
-			    .error(e)
+			    .error(savedError)
 			    .detail("BackupUID", config.getUid())
 			    .detail("SnapshotVersion", snapshotVersion)
 			    .detail("JobAlreadyRunning", jobAlreadyRunning);
 			// Restore original BulkDump mode on error, but only if we were the ones who enabled it
 			try {
 				if (!jobAlreadyRunning && originalBulkDumpMode != 1) {
-					wait(success(setBulkDumpMode(cx, originalBulkDumpMode)));
+					co_await setBulkDumpMode(cx, originalBulkDumpMode);
 				}
 			} catch (Error& e2) {
 				TraceEvent(SevWarn, "BulkDumpTaskRestoreModeError").error(e2);
 			}
 			throw savedError;
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state BackupConfig config(task);
-		state Version snapshotVersion = Params.snapshotVersion().get(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		BackupConfig config(task);
+		Version snapshotVersion = Params.snapshotVersion().get(task);
+		std::string jobId = Params.bulkDumpJobId().getOrDefault(task, "");
 
 		TraceEvent("BulkDumpTaskFinishStart")
 		    .detail("BackupUID", config.getUid())
-		    .detail("SnapshotVersion", snapshotVersion);
+		    .detail("SnapshotVersion", snapshotVersion)
+		    .detail("BulkDumpJobId", jobId);
 
 		// Set latestSnapshotEndVersion so BackupLogsDispatchTask knows we're restorable
 		// This is critical for the backup to complete when using BulkDump
 		config.latestSnapshotEndVersion().set(tr, snapshotVersion);
 
-		// Also set firstSnapshotEndVersion if not already set (first snapshot)
-		state Optional<Version> firstSnapshotEnd = wait(config.firstSnapshotEndVersion().get(tr));
-		if (!firstSnapshotEnd.present()) {
-			config.firstSnapshotEndVersion().set(tr, snapshotVersion);
+		// Set bulkDumpSnapshotEndVersion to track that BulkDump data is available
+		// This is used by getLatestRestorableVersion() for mode=BOTH to ensure both
+		// rangefile and bulkdump data exist before marking backup as restorable
+		config.bulkDumpSnapshotEndVersion().set(tr, snapshotVersion);
+
+		// CRITICAL: Set bulkDumpJobId on the backup CONFIG so that status checks
+		// and restore operations can find it. Without this, the backup status shows
+		// "BulkLoad Compatible: no" and bulkload restore fails with "Missing backup data".
+		if (!jobId.empty()) {
+			config.bulkDumpJobId().set(tr, jobId);
+		}
+
+		// Set firstSnapshotEndVersion if not already set, BUT only if we're NOT in
+		// mode=BOTH. In mode=BOTH, firstSnapshotEndVersion tracks the rangefile snapshot
+		// completion and must only be set by BackupSnapshotManifest::_finish().
+		// Setting it here would make getLatestRestorableVersion() think both snapshots
+		// are complete when only bulkdump is done.
+		Optional<int> mode = co_await config.snapshotMode().get(tr);
+		if (!mode.present() || mode.get() != 2) {
+			Optional<Version> firstSnapshotEnd = co_await config.firstSnapshotEndVersion().get(tr);
+			if (!firstSnapshotEnd.present()) {
+				config.firstSnapshotEndVersion().set(tr, snapshotVersion);
+			}
 		}
 
 		TraceEvent("BulkDumpTaskFinishSetVersions")
 		    .detail("BackupUID", config.getUid())
 		    .detail("SnapshotVersion", snapshotVersion)
-		    .detail("FirstSnapshotWasSet", !firstSnapshotEnd.present());
+		    .detail("Mode", mode.present() ? mode.get() : 0);
 
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		wait(taskFuture->set(tr, taskBucket));
-		wait(taskBucket->finish(tr, task));
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		co_await taskFuture->set(tr, taskBucket);
+		co_await taskBucket->finish(tr, task);
 
 		TraceEvent("BulkDumpTaskFinish")
 		    .detail("BackupUID", config.getUid())
 		    .detail("SnapshotVersion", snapshotVersion);
 
-		return Void();
-	}
-
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version snapshotVersion,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key key = wait(addBackupTask(BulkDumpTaskFunc::name,
-		                             BulkDumpTaskFunc::version,
-		                             tr,
-		                             taskBucket,
-		                             completionKey,
-		                             BackupConfig(parentTask),
-		                             waitFor,
-		                             [=](Reference<Task> task) {
-			                             Params.snapshotVersion().set(task, snapshotVersion);
-			                             Params.timeoutOccurred().set(task, false);
-		                             }));
-		return key;
+		co_return;
+	}
+
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version snapshotVersion,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key key = co_await addBackupTask(BulkDumpTaskFunc::name,
+		                                 BulkDumpTaskFunc::version,
+		                                 tr,
+		                                 taskBucket,
+		                                 completionKey,
+		                                 BackupConfig(parentTask),
+		                                 waitFor,
+		                                 [=](Reference<Task> task) {
+			                                 Params.snapshotVersion().set(task, snapshotVersion);
+			                                 Params.timeoutOccurred().set(task, false);
+		                                 });
+		co_return key;
 	}
 };
 StringRef BulkDumpTaskFunc::name = "bulk_dump_snapshot_5.2"_sr;
@@ -3944,70 +4213,73 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 		static TaskParam<Version> beginVersion() { return __FUNCTION__sr; }
 	} Params;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		wait(checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version));
-
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state BackupConfig config(task);
-		state Future<Optional<bool>> partitionedLog;
-		loop {
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		co_await checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version);
+
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		BackupConfig config(task);
+		Future<Optional<MutationLogType>> mutationLogType;
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				partitionedLog = config.partitionedLogEnabled().get(tr);
-				wait(success(partitionedLog));
+				mutationLogType = config.mutationLogType().get(tr);
+				co_await mutationLogType;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		// Check if backup worker is enabled
-		DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(cx));
-		state bool backupWorkerEnabled = dbConfig.backupWorkerEnabled;
-		if (!backupWorkerEnabled && partitionedLog.get().present() && partitionedLog.get().get()) {
-			// Change configuration only when we set to use partitioned logs and
-			// the flag was not set before.
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=1", true)));
-			backupWorkerEnabled = true;
-			// the user is responsible for manually disabling backup worker after the last backup is done
+		// Enable the appropriate backup worker type if not already enabled.
+		if (mutationLogType.get().present()) {
+			if (mutationLogType.get().get() == MutationLogType::PARTITIONED_LOG) {
+				co_await enableBackupWorker(cx);
+			} else if (mutationLogType.get().get() == MutationLogType::RANGE_PARTITIONED_LOG) {
+				co_await enableRangeBackupWorker(cx);
+			}
 		}
 
 		// Get start version after backup worker are enabled
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->reset();
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state Future<Version> startVersionFuture = tr->getReadVersion();
-				wait(success(startVersionFuture));
+				Future<Version> startVersionFuture = tr->getReadVersion();
+				co_await startVersionFuture;
 				Params.beginVersion().set(task, startVersionFuture.get());
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		// Set the "backupStartedKey" and wait for all backup worker started
 		tr->reset();
-		loop {
-			state Future<Void> watchFuture;
+		while (true) {
+			Future<Void> watchFuture;
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				state Future<Void> keepRunning = taskBucket->keepRunning(tr, task);
+				Future<Void> keepRunning = taskBucket->keepRunning(tr, task);
 
-				state Future<Optional<Value>> started = tr->get(backupStartedKey);
-				state Future<Optional<Value>> taskStarted = tr->get(config.allWorkerStarted().key);
-				partitionedLog = config.partitionedLogEnabled().get(tr);
-				wait(success(started) && success(taskStarted) && success(partitionedLog));
+				Future<Optional<Value>> started = tr->get(backupStartedKey);
+				Future<Optional<Value>> taskStarted = tr->get(config.allWorkerStarted().key);
+				mutationLogType = config.mutationLogType().get(tr);
+				co_await (success(started) && success(taskStarted) && success(mutationLogType));
 
-				if (!partitionedLog.get().present() || !partitionedLog.get().get()) {
-					return Void(); // Skip if not using partitioned logs
+				if (!mutationLogType.get().present() || mutationLogType.get().get() == MutationLogType::DEFAULT) {
+					co_return; // Skip if not using partitioned logs
 				}
 
 				std::vector<std::pair<UID, Version>> ids;
@@ -4030,57 +4302,59 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 					watchFuture = tr->watch(config.allWorkerStarted().key);
 				}
 
-				wait(keepRunning);
-				wait(tr->commit());
+				co_await keepRunning;
+				co_await tr->commit();
 				if (!taskStarted.get().present()) {
-					wait(watchFuture);
+					co_await watchFuture;
 				}
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state BackupConfig config(task);
-		state Version beginVersion = Params.beginVersion().get(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		BackupConfig config(task);
+		Version beginVersion = Params.beginVersion().get(task);
 
-		state Future<std::vector<KeyRange>> backupRangesFuture = config.backupRanges().getOrThrow(tr);
-		state Future<Key> destUidValueFuture = config.destUidValue().getOrThrow(tr);
-		state Future<Optional<bool>> partitionedLog = config.partitionedLogEnabled().get(tr);
-		state Future<Optional<bool>> incrementalBackupOnly = config.incrementalBackupOnly().get(tr);
-		wait(success(backupRangesFuture) && success(destUidValueFuture) && success(partitionedLog) &&
-		     success(incrementalBackupOnly));
+		Future<std::vector<KeyRange>> backupRangesFuture = config.backupRanges().getOrThrow(tr);
+		Future<Key> destUidValueFuture = config.destUidValue().getOrThrow(tr);
+		Future<Optional<MutationLogType>> mutationLogType = config.mutationLogType().get(tr);
+		Future<Optional<bool>> incrementalBackupOnly = config.incrementalBackupOnly().get(tr);
+		co_await (success(backupRangesFuture) && success(destUidValueFuture) && success(mutationLogType) &&
+		          success(incrementalBackupOnly));
 		std::vector<KeyRange> backupRanges = backupRangesFuture.get();
 		Key destUidValue = destUidValueFuture.get();
 
 		// Start logging the mutations for the specified ranges of the tag if needed
-		if (!partitionedLog.get().present() || !partitionedLog.get().get()) {
+		if (!mutationLogType.get().present() || mutationLogType.get().get() == MutationLogType::DEFAULT) {
 			for (auto& backupRange : backupRanges) {
 				config.startMutationLogs(tr, backupRange, destUidValue);
 			}
 		}
 
-		Reference<IBackupContainer> bc = wait(config.backupContainer().getOrThrow(tr));
-		wait(bc->writeEncryptionMetadata());
-
+		// Properties folder is created here to ensure backup container exists because localDirectory doesn't create
+		// container during in constructor.
+		Reference<IBackupContainer> bc = co_await config.backupContainer().getOrThrow(tr);
+		co_await bc->writeEncryptionMetadata(bc->getEncryptionBlockSize());
 		config.stateEnum().set(tr, EBackupState::STATE_RUNNING);
 
-		state Reference<TaskFuture> backupFinished = futureBucket->future(tr);
+		Reference<TaskFuture> backupFinished = futureBucket->future(tr);
 
 		// Initialize the initial snapshot and create tasks to continually write logs and snapshots.
-		state Optional<int64_t> initialSnapshotIntervalSeconds = wait(config.initialSnapshotIntervalSeconds().get(tr));
-		wait(config.initNewSnapshot(tr, initialSnapshotIntervalSeconds.orDefault(0)));
+		Optional<int64_t> initialSnapshotIntervalSeconds = co_await config.initialSnapshotIntervalSeconds().get(tr);
+		co_await config.initNewSnapshot(tr, initialSnapshotIntervalSeconds.orDefault(0));
 
 		// Using priority 1 for both of these to at least start both tasks soon
 		// Do not add snapshot task if we only want the incremental backup
 		if (!incrementalBackupOnly.get().present() || !incrementalBackupOnly.get().get()) {
 			// Check snapshot mode: 0=RANGEFILE, 1=BULKDUMP, 2=BOTH
-			state int snapshotModeValue = wait(config.snapshotMode().getD(tr, Snapshot::False, 0));
+			int snapshotModeValue = co_await config.snapshotMode().getD(tr, Snapshot::False, 0);
 
 			TraceEvent("StartFullBackupTaskSnapshotMode")
 			    .detail("BackupUID", config.getUid())
@@ -4092,46 +4366,45 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 			if (snapshotModeValue == 1 || snapshotModeValue == 2) {
 				// Save original BulkDump mode BEFORE creating the task, so it's persisted in the database.
 				// This allows crash recovery to restore the correct mode even if the task restarts.
-				int currentBulkDumpMode = wait(getBulkDumpMode(tr->getDatabase()));
+				int currentBulkDumpMode = co_await getBulkDumpMode(tr->getDatabase());
 				config.originalBulkDumpMode().set(tr, currentBulkDumpMode);
 
-				wait(success(BulkDumpTaskFunc::addTask(
-				    tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(backupFinished))));
+				co_await BulkDumpTaskFunc::addTask(
+				    tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(backupFinished));
 			}
 
 			// Add traditional range file snapshot if mode is RANGEFILE(0) or BOTH(2)
 			if (snapshotModeValue == 0 || snapshotModeValue == 2) {
-				wait(success(BackupSnapshotDispatchTask::addTask(
-				    tr, taskBucket, task, 1, TaskCompletionKey::joinWith(backupFinished))));
+				co_await BackupSnapshotDispatchTask::addTask(
+				    tr, taskBucket, task, 1, TaskCompletionKey::joinWith(backupFinished));
 			}
 		}
 
-		wait(success(BackupLogsDispatchTask::addTask(
-		    tr, taskBucket, task, 1, 0, beginVersion, TaskCompletionKey::joinWith(backupFinished))));
+		co_await BackupLogsDispatchTask::addTask(
+		    tr, taskBucket, task, 1, 0, beginVersion, TaskCompletionKey::joinWith(backupFinished));
 
 		// If a clean stop is requested, the log and snapshot tasks will quit after the backup is restorable, then
 		// the following task will clean up and set the completed state.
-		wait(success(
-		    FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished)));
+		co_await FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished);
 
-		wait(taskBucket->finish(tr, task));
+		co_await taskBucket->finish(tr, task);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 UID uid,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key key = wait(addBackupTask(StartFullBackupTaskFunc::name,
-		                             StartFullBackupTaskFunc::version,
-		                             tr,
-		                             taskBucket,
-		                             completionKey,
-		                             BackupConfig(uid),
-		                             waitFor));
-		return key;
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           UID uid,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key key = co_await addBackupTask(StartFullBackupTaskFunc::name,
+		                                 StartFullBackupTaskFunc::version,
+		                                 tr,
+		                                 taskBucket,
+		                                 completionKey,
+		                                 BackupConfig(uid),
+		                                 waitFor);
+		co_return key;
 	}
 
 	StringRef getName() const override { return name; };
@@ -4179,14 +4452,14 @@ struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state RestoreConfig restore(task);
-		state Version restoreVersion = Params.restoreVersion().get(task);
-		state std::string backupUrl = Params.backupUrl().get(task);
-		state std::string bulkDumpJobId = Params.bulkDumpJobId().getOrDefault(task, "");
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		RestoreConfig restore(task);
+		Version restoreVersion = Params.restoreVersion().get(task);
+		std::string backupUrl = Params.backupUrl().get(task);
+		std::string bulkDumpJobId = Params.bulkDumpJobId().getOrDefault(task, "");
 
 		TraceEvent("BulkLoadRestoreTaskStart")
 		    .detail("RestoreUID", restore.getUid())
@@ -4195,186 +4468,212 @@ struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
 		    .detail("BulkDumpJobId", bulkDumpJobId);
 
 		// Declare state variable before try block so it's accessible in catch block
-		state int originalBulkLoadMode = 0;
+		int originalBulkLoadMode = 0;
 
-		try {
-			// Open backup container for metadata access
-			state Reference<IBackupContainer> bcRef = IBackupContainer::openContainer(backupUrl, {}, {});
+		{
+			Error savedError;
+			try {
+				// Open backup container for metadata access
+				Reference<IBackupContainer> bcRef = IBackupContainer::openContainer(backupUrl, {}, {}, 0);
 
-			// Get restore ranges using a transaction
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			state std::vector<KeyRange> restoreRanges = wait(restore.getRestoreRangesOrDefault(tr));
-
-			// If bulkDumpJobId is empty, read it from the snapshot file metadata
-			if (bulkDumpJobId.empty()) {
-				// Use the backup container we already opened to find the snapshot file for our restore version
-				BackupContainerFileSystem* bcfsPtr = dynamic_cast<BackupContainerFileSystem*>(bcRef.getPtr());
-				if (bcfsPtr != nullptr) {
-					state Reference<BackupContainerFileSystem> bcfs =
-					    Reference<BackupContainerFileSystem>::addRef(bcfsPtr);
-					state std::vector<KeyspaceSnapshotFile> snapshots = wait(bcfs->listKeyspaceSnapshots());
-					state int snapIdx = snapshots.size() - 1;
-					while (snapIdx >= 0 && bulkDumpJobId.empty()) {
-						// For BulkDump snapshots: beginVersion == endVersion == snapshotVersion
-						// Accept any snapshot that is at or before our restore version
-						if (snapshots[snapIdx].endVersion <= restoreVersion) {
-							// Read the snapshot file to get bulkDumpJobId
-							state std::string snapFileName = snapshots[snapIdx].fileName;
-							state Reference<IAsyncFile> snapFile = wait(bcfs->readFile(snapFileName));
-							state int64_t snapSize = wait(snapFile->size());
-							state Standalone<StringRef> snapBuf = makeString(snapSize);
-							int bytesRead = wait(snapFile->read(mutateString(snapBuf), snapSize, 0));
-							if (bytesRead == snapSize) {
-								json_spirit::mValue json;
-								if (json_spirit::read_string(snapBuf.toString(), json)) {
-									JSONDoc doc(json);
-									std::string jobId;
-									if (doc.tryGet("bulkDumpJobId", jobId) && !jobId.empty()) {
-										bulkDumpJobId = jobId;
-										TraceEvent("BulkLoadRestoreFoundJobId")
-										    .detail("RestoreUID", restore.getUid())
-										    .detail("BulkDumpJobId", bulkDumpJobId)
-										    .detail("SnapshotFile", snapFileName);
+				// Get restore ranges using a transaction
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				std::vector<KeyRange> restoreRanges = co_await restore.getRestoreRangesOrDefault(tr);
+
+				// If bulkDumpJobId is empty, read it from the snapshot file metadata
+				if (bulkDumpJobId.empty()) {
+					// Use the backup container we already opened to find the snapshot file for our restore version
+					auto* bcfsPtr = dynamic_cast<BackupContainerFileSystem*>(bcRef.getPtr());
+					if (bcfsPtr != nullptr) {
+						Reference<BackupContainerFileSystem> bcfs =
+						    Reference<BackupContainerFileSystem>::addRef(bcfsPtr);
+						std::vector<KeyspaceSnapshotFile> snapshots = co_await bcfs->listKeyspaceSnapshots();
+						int snapIdx = snapshots.size() - 1;
+						while (snapIdx >= 0 && bulkDumpJobId.empty()) {
+							// For BulkDump snapshots: beginVersion == endVersion == snapshotVersion
+							// Accept any snapshot that is at or before our restore version
+							if (snapshots[snapIdx].endVersion <= restoreVersion) {
+								// Read the snapshot file to get bulkDumpJobId
+								std::string snapFileName = snapshots[snapIdx].fileName;
+								Reference<IAsyncFile> snapFile = co_await bcfs->readFile(snapFileName);
+								int64_t snapSize = co_await snapFile->size();
+								Standalone<StringRef> snapBuf = makeString(snapSize);
+								int bytesRead = co_await snapFile->read(mutateString(snapBuf), snapSize, 0);
+								if (bytesRead == snapSize) {
+									json_spirit::mValue json;
+									if (json_spirit::read_string(snapBuf.toString(), json)) {
+										JSONDoc doc(json);
+										std::string jobId;
+										if (doc.tryGet("bulkDumpJobId", jobId) && !jobId.empty()) {
+											bulkDumpJobId = jobId;
+											TraceEvent("BulkLoadRestoreFoundJobId")
+											    .detail("RestoreUID", restore.getUid())
+											    .detail("BulkDumpJobId", bulkDumpJobId)
+											    .detail("SnapshotFile", snapFileName);
+										}
 									}
 								}
 							}
+							snapIdx--;
 						}
-						snapIdx--;
 					}
 				}
-			}
 
-			// Create BulkLoad job from the BulkDump data
-			// BulkDump stores data under data/<container>/bulkdump_data/ subdirectory.
-			// DD appends the jobId via getBulkLoadJobRoot(jobRoot, jobId) when accessing files.
-			state std::string jobRoot = getBackupDataPath(backupUrl, "bulkdump_data");
-			state UID dumpJobUid;
-			if (!bulkDumpJobId.empty()) {
-				dumpJobUid = UID::fromString(bulkDumpJobId);
-			} else {
-				// No BulkDump job ID found - this is an error for BulkLoad restore
-				TraceEvent(SevError, "BulkLoadRestoreNoBulkDumpJobId")
-				    .detail("RestoreUID", restore.getUid())
-				    .detail("BackupUrl", backupUrl);
-				throw restore_missing_data();
-			}
+				// Create BulkLoad job from the BulkDump data
+				// BulkDump stores data under data/<container>/bulkdump_data/ subdirectory.
+				// DD appends the jobId via getBulkLoadJobRoot(jobRoot, jobId) when accessing files.
+				std::string jobRoot = getBackupDataPath(backupUrl, "bulkdump_data");
+				UID dumpJobUid;
+				if (!bulkDumpJobId.empty()) {
+					dumpJobUid = UID::fromString(bulkDumpJobId);
+				} else {
+					// No BulkDump job ID found - this is a permanent error for BulkLoad restore.
+					// Abort the restore immediately instead of retrying forever.
+					TraceEvent(SevError, "BulkLoadRestoreNoBulkDumpJobId")
+					    .detail("RestoreUID", restore.getUid())
+					    .detail("BackupUrl", backupUrl)
+					    .detail("Action", "Aborting restore - backup is not BulkLoad compatible");
+					co_await restore.logError(
+					    cx, restore_missing_data(), "BulkLoad restore failed: backup has no bulkdump data", nullptr);
+					// Abort the restore by setting state to ABORTED
+					Reference<ReadYourWritesTransaction> abortTr(new ReadYourWritesTransaction(cx));
+					abortTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					abortTr->setOption(FDBTransactionOptions::LOCK_AWARE);
+					restore.stateEnum().set(abortTr, ERestoreState::ABORTED);
+					co_await abortTr->commit();
+					throw restore_missing_data();
+				}
 
-			// Verify BulkDump dataset completeness before proceeding
-			bool datasetComplete = wait(verifyBulkDumpDatasetCompleteness(bcRef, bulkDumpJobId));
-			if (!datasetComplete) {
-				TraceEvent(SevWarn, "BulkLoadRestoreDatasetIncomplete")
+				// Verify BulkDump dataset completeness before proceeding
+				bool datasetComplete = co_await verifyBulkDumpDatasetCompleteness(bcRef, bulkDumpJobId);
+				if (!datasetComplete) {
+					// Dataset is incomplete - abort the restore permanently
+					TraceEvent(SevError, "BulkLoadRestoreDatasetIncomplete")
+					    .detail("RestoreUID", restore.getUid())
+					    .detail("BulkDumpJobId", bulkDumpJobId)
+					    .detail("BackupUrl", backupUrl)
+					    .detail("Action", "Aborting restore - bulkdump dataset is incomplete");
+					co_await restore.logError(
+					    cx, restore_missing_data(), "BulkLoad restore failed: bulkdump dataset incomplete", nullptr);
+					// Abort the restore by setting state to ABORTED
+					Reference<ReadYourWritesTransaction> abortTr(new ReadYourWritesTransaction(cx));
+					abortTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					abortTr->setOption(FDBTransactionOptions::LOCK_AWARE);
+					restore.stateEnum().set(abortTr, ERestoreState::ABORTED);
+					co_await abortTr->commit();
+					throw restore_missing_data();
+				}
+				TraceEvent("BulkLoadRestoreDatasetVerified")
 				    .detail("RestoreUID", restore.getUid())
-				    .detail("BulkDumpJobId", bulkDumpJobId)
-				    .detail("BackupUrl", backupUrl);
-				throw restore_missing_data();
-			}
-			TraceEvent("BulkLoadRestoreDatasetVerified")
-			    .detail("RestoreUID", restore.getUid())
-			    .detail("BulkDumpJobId", bulkDumpJobId);
+				    .detail("BulkDumpJobId", bulkDumpJobId);
 
-			// Determine transport method from backup URL
-			state BulkLoadTransportMethod loadTransportMethod =
-			    isBlobstoreUrl(backupUrl) ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP;
+				// Determine transport method from backup URL
+				BulkLoadTransportMethod loadTransportMethod =
+				    isBlobstoreUrl(backupUrl) ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP;
 
-			// BulkLoad range must match the BulkDump range (normalKeys)
-			// The actual restore ranges will be applied via mutation log replay
-			state BulkLoadJobState bulkLoadJob =
-			    createBulkLoadJob(dumpJobUid, normalKeys, jobRoot, loadTransportMethod);
+				// BulkLoad range must match the BulkDump range (normalKeys)
+				// The actual restore ranges will be applied via mutation log replay
+				BulkLoadJobState bulkLoadJob = createBulkLoadJob(dumpJobUid, normalKeys, jobRoot, loadTransportMethod);
 
-			TraceEvent("BulkLoadRestoreJobCreated")
-			    .detail("RestoreUID", restore.getUid())
-			    .detail("BulkLoadJobId", bulkLoadJob.getJobId())
-			    .detail("JobRoot", jobRoot);
+				TraceEvent("BulkLoadRestoreJobCreated")
+				    .detail("RestoreUID", restore.getUid())
+				    .detail("BulkLoadJobId", bulkLoadJob.getJobId())
+				    .detail("JobRoot", jobRoot);
 
-			// Register the BulkLoad range lock owner (required for range locking)
-			wait(registerRangeLockOwner(cx, "BulkLoad", "BulkLoad restore operation"));
+				// Register the BulkLoad range lock owner (required for range locking)
+				co_await registerRangeLockOwner(cx, "BulkLoad", "BulkLoad restore operation");
 
-			TraceEvent("BulkLoadRestoreRegisteredLockOwner")
-			    .detail("RestoreUID", restore.getUid())
-			    .detail("Owner", "BulkLoad");
+				TraceEvent("BulkLoadRestoreRegisteredLockOwner")
+				    .detail("RestoreUID", restore.getUid())
+				    .detail("Owner", "BulkLoad");
 
-			// Note: BulkLoad configuration validation (shard_encode_location_metadata, enable_read_lock_on_range)
-			// is performed by the BulkLoad system on the server side when the job is submitted.
+				// Note: BulkLoad configuration validation (shard_encode_location_metadata, enable_read_lock_on_range)
+				// is performed by the BulkLoad system on the server side when the job is submitted.
 
-			// Read the original BulkLoad mode from config (saved by StartFullRestoreTaskFunc before task creation).
-			// This is persisted in the database so we can restore the correct mode even after a crash.
-			int mode = wait(restore.originalBulkLoadMode().getD(cx.getReference(), Snapshot::False, 0));
-			originalBulkLoadMode = mode;
+				// Read the original BulkLoad mode from config (saved by StartFullRestoreTaskFunc before task creation).
+				// This is persisted in the database so we can restore the correct mode even after a crash.
+				int mode = co_await restore.originalBulkLoadMode().getD(cx.getReference(), Snapshot::False, 0);
+				originalBulkLoadMode = mode;
 
-			// Enable BulkLoad mode at DD level so the job will be processed
-			wait(success(setBulkLoadMode(cx, 1)));
+				// Enable BulkLoad mode at DD level so the job will be processed
+				co_await setBulkLoadMode(cx, 1);
 
-			TraceEvent("BulkLoadRestoreEnabledMode")
-			    .detail("RestoreUID", restore.getUid())
-			    .detail("BulkLoadJobId", bulkLoadJob.getJobId());
+				TraceEvent("BulkLoadRestoreEnabledMode")
+				    .detail("RestoreUID", restore.getUid())
+				    .detail("BulkLoadJobId", bulkLoadJob.getJobId());
 
-			// Submit BulkLoad job (must be lockAware since DB is locked during restore)
-			wait(submitBulkLoadJob(cx, bulkLoadJob, true /* lockAware */));
+				// Submit BulkLoad job (must be lockAware since DB is locked during restore)
+				co_await submitBulkLoadJob(cx, bulkLoadJob, true /* lockAware */);
 
-			TraceEvent("BulkLoadRestoreJobSubmitted")
-			    .detail("RestoreUID", restore.getUid())
-			    .detail("BulkLoadJobId", bulkLoadJob.getJobId());
-
-			// Monitor BulkLoad progress - timeout is configurable for large datasets
-			// Must be lockAware since DB is locked during restore
-			bool completed = wait(monitorBulkLoadJobCompletion(cx,
-			                                                   bulkLoadJob.getJobId(),
-			                                                   CLIENT_KNOBS->BULKLOAD_JOB_TIMEOUT,
-			                                                   5.0, // Poll every 5 seconds
-			                                                   true)); // lockAware
-
-			if (!completed) {
-				TraceEvent(SevWarn, "BulkLoadRestoreTimeout")
+				TraceEvent("BulkLoadRestoreJobSubmitted")
 				    .detail("RestoreUID", restore.getUid())
-				    .detail("BulkLoadJobId", bulkLoadJob.getJobId())
-				    .detail("TimeoutDuration", CLIENT_KNOBS->BULKLOAD_JOB_TIMEOUT);
-				// Restore original BulkLoad mode before throwing
-				if (originalBulkLoadMode != 1) {
-					wait(success(setBulkLoadMode(cx, originalBulkLoadMode)));
+				    .detail("BulkLoadJobId", bulkLoadJob.getJobId());
+
+				// Get total block count for progress tracking
+				int64_t totalBlocks = co_await restore.fileBlockCount().getD(cx.getReference(), Snapshot::False, 0);
+
+				// Monitor BulkLoad progress - timeout is configurable for large datasets
+				// Must be lockAware since DB is locked during restore
+				// Use the progress-tracking version to update restore counters
+				bool completed = co_await monitorBulkLoadJobCompletionWithProgress(cx,
+				                                                                   bulkLoadJob.getJobId(),
+				                                                                   restore.getUid(),
+				                                                                   totalBlocks,
+				                                                                   CLIENT_KNOBS->BULKLOAD_JOB_TIMEOUT,
+				                                                                   5.0, // Poll every 5 seconds
+				                                                                   true); // lockAware
+
+				if (!completed) {
+					TraceEvent(SevWarn, "BulkLoadRestoreTimeout")
+					    .detail("RestoreUID", restore.getUid())
+					    .detail("BulkLoadJobId", bulkLoadJob.getJobId())
+					    .detail("TimeoutDuration", CLIENT_KNOBS->BULKLOAD_JOB_TIMEOUT);
+					// Restore original BulkLoad mode before throwing
+					if (originalBulkLoadMode != 1) {
+						co_await setBulkLoadMode(cx, originalBulkLoadMode);
+					}
+					throw timed_out();
 				}
-				throw timed_out();
-			}
-
-			// Restore original BulkLoad mode now that the job is complete
-			if (originalBulkLoadMode != 1) {
-				wait(success(setBulkLoadMode(cx, originalBulkLoadMode)));
-			}
 
-			TraceEvent("BulkLoadRestoreTaskComplete")
-			    .detail("RestoreUID", restore.getUid())
-			    .detail("BulkLoadJobId", bulkLoadJob.getJobId())
-			    .detail("RestoreVersion", restoreVersion);
+				// Restore original BulkLoad mode now that the job is complete
+				if (originalBulkLoadMode != 1) {
+					co_await setBulkLoadMode(cx, originalBulkLoadMode);
+				}
 
-			// Increment counter for test assertions
-			g_bulkLoadRestoreTaskCompleteCount.fetch_add(1);
+				TraceEvent("BulkLoadRestoreTaskComplete")
+				    .detail("RestoreUID", restore.getUid())
+				    .detail("BulkLoadJobId", bulkLoadJob.getJobId())
+				    .detail("RestoreVersion", restoreVersion);
 
-		} catch (Error& e) {
-			state Error savedError = e;
+				// Increment counter for test assertions
+				g_bulkLoadRestoreTaskCompleteCount.fetch_add(1);
+				co_return;
+			} catch (Error& e) {
+				savedError = e;
+			}
 			TraceEvent(SevWarn, "BulkLoadRestoreTaskError")
-			    .error(e)
+			    .error(savedError)
 			    .detail("RestoreUID", restore.getUid())
 			    .detail("BackupUrl", backupUrl);
 			// Restore original BulkLoad mode on error
 			try {
 				if (originalBulkLoadMode != 1) {
-					wait(success(setBulkLoadMode(cx, originalBulkLoadMode)));
+					co_await setBulkLoadMode(cx, originalBulkLoadMode);
 				}
 			} catch (Error& e2) {
 				TraceEvent(SevWarn, "BulkLoadRestoreRestoreModeError").error(e2);
 			}
 			throw savedError;
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		RestoreConfig restore(task);
 
 		// Mark BulkLoad phase as complete
 		restore.bulkLoadComplete().set(tr, true);
@@ -4382,8 +4681,8 @@ struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
 		// Update fileBlocksFinished and filesBlocksDispatched to match fileBlockCount
 		// so the restore progress tracker knows all range data has been loaded.
 		// BulkLoad doesn't use the traditional block dispatching mechanism.
-		state int64_t totalBlocks = wait(restore.fileBlockCount().getD(tr, Snapshot::False, 0));
-		state int64_t finishedBlocks = wait(restore.fileBlocksFinished().getD(tr, Snapshot::False, 0));
+		int64_t totalBlocks = co_await restore.fileBlockCount().getD(tr, Snapshot::False, 0);
+		int64_t finishedBlocks = co_await restore.fileBlocksFinished().getD(tr, Snapshot::False, 0);
 		if (finishedBlocks < totalBlocks) {
 			// Set both dispatched and finished to match total to indicate all range data is done
 			restore.filesBlocksDispatched().set(tr, totalBlocks);
@@ -4392,11 +4691,11 @@ struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
 
 		// Set firstConsistentVersion if not already set
 		// For BulkLoad restore, this is the snapshot version from the BulkDump
-		state Version firstConsistentVer =
-		    wait(restore.firstConsistentVersion().getD(tr, Snapshot::False, invalidVersion));
+		Version firstConsistentVer =
+		    co_await restore.firstConsistentVersion().getD(tr, Snapshot::False, invalidVersion);
 		if (firstConsistentVer == invalidVersion) {
 			// Get the restore version as the first consistent version
-			state Version restoreVer = wait(restore.restoreVersion().getD(tr, Snapshot::False, invalidVersion));
+			Version restoreVer = co_await restore.restoreVersion().getD(tr, Snapshot::False, invalidVersion);
 			if (restoreVer != invalidVersion) {
 				restore.firstConsistentVersion().set(tr, restoreVer);
 			}
@@ -4407,22 +4706,22 @@ struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
 		    .detail("TotalBlocks", totalBlocks)
 		    .detail("FinishedBlocks", finishedBlocks);
 
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		wait(taskFuture->set(tr, taskBucket));
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		co_await taskFuture->set(tr, taskBucket);
+		co_await taskBucket->finish(tr, task);
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version restoreVersion,
-	                                 std::string backupUrl,
-	                                 std::string bulkDumpJobId,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
-		state Reference<Task> task(new Task(BulkLoadRestoreTaskFunc::name, BulkLoadRestoreTaskFunc::version, doneKey));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version restoreVersion,
+	                           std::string backupUrl,
+	                           std::string bulkDumpJobId,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
+		Reference<Task> task(new Task(BulkLoadRestoreTaskFunc::name, BulkLoadRestoreTaskFunc::version, doneKey));
 
 		// Set task parameters
 		Params.restoreVersion().set(task, restoreVersion);
@@ -4430,30 +4729,30 @@ struct BulkLoadRestoreTaskFunc : RestoreTaskFuncBase {
 		Params.bulkDumpJobId().set(task, bulkDumpJobId);
 
 		// Get restore config from parent task and bind it to new task
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 };
 StringRef BulkLoadRestoreTaskFunc::name = "bulk_load_restore_5.2"_sr;
 REGISTER_TASKFUNC(BulkLoadRestoreTaskFunc);
 
 struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 
-		wait(checkTaskVersion(tr->getDatabase(), task, name, version));
+		co_await checkTaskVersion(tr->getDatabase(), task, name, version);
 
-		state RestoreConfig restore(task);
+		RestoreConfig restore(task);
 		restore.stateEnum().set(tr, ERestoreState::COMPLETED);
-		state bool unlockDB = wait(restore.unlockDBAfterRestore().getD(tr, Snapshot::False, true));
+		bool unlockDB = co_await restore.unlockDBAfterRestore().getD(tr, Snapshot::False, true);
 
 		tr->atomicOp(metadataVersionKey, metadataVersionRequiredValue, MutationRef::SetVersionstampedValue);
 		// Clear the file map now since it could be huge.
@@ -4469,32 +4768,32 @@ struct RestoreCompleteTaskFunc : RestoreTaskFuncBase {
 		// version.
 		restore.clearApplyMutationsKeys(tr);
 
-		wait(taskBucket->finish(tr, task));
+		co_await taskBucket->finish(tr, task);
 
 		if (unlockDB) {
-			wait(unlockDatabase(tr, restore.getUid()));
+			co_await unlockDatabase(tr, restore.getUid());
 		}
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
-		state Reference<Task> task(new Task(RestoreCompleteTaskFunc::name, RestoreCompleteTaskFunc::version, doneKey));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
+		Reference<Task> task(new Task(RestoreCompleteTaskFunc::name, RestoreCompleteTaskFunc::version, doneKey));
 
 		// Get restore config from parent task and bind it to new task
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	static StringRef name;
@@ -4558,15 +4857,15 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		return returnStr;
 	}
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		RestoreConfig restore(task);
 
-		state RestoreFile rangeFile = Params.inputFile().get(task);
-		state int64_t readOffset = Params.readOffset().get(task);
-		state int64_t readLen = Params.readLen().get(task);
+		RestoreFile rangeFile = Params.inputFile().get(task);
+		int64_t readOffset = Params.readOffset().get(task);
+		int64_t readLen = Params.readLen().get(task);
 
 		TraceEvent("FileRestoreRangeStart")
 		    .suppressFor(60)
@@ -4575,57 +4874,58 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		    .detail("FileVersion", rangeFile.version)
 		    .detail("FileSize", rangeFile.fileSize)
 		    .detail("ReadOffset", readOffset)
-		    .detail("ReadLen", readLen)
-		    .detail("TaskInstance", THIS_ADDR);
+		    .detail("ReadLen", readLen);
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Future<Reference<IBackupContainer>> bc;
-		state Future<std::vector<KeyRange>> restoreRanges;
-		state Future<Key> addPrefix;
-		state Future<Key> removePrefix;
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Future<Reference<IBackupContainer>> bc;
+		Future<std::vector<KeyRange>> restoreRanges;
+		Future<Key> addPrefix;
+		Future<Key> removePrefix;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
+				Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
 				bc = getBackupContainerWithProxy(_bc);
 				restoreRanges = restore.getRestoreRangesOrDefault(tr);
 				addPrefix = restore.addPrefix().getD(tr);
 				removePrefix = restore.removePrefix().getD(tr);
 
-				wait(taskBucket->keepRunning(tr, task));
+				co_await taskBucket->keepRunning(tr, task);
 
-				wait(success(bc) && success(restoreRanges) && success(addPrefix) && success(removePrefix) &&
-				     checkTaskVersion(tr->getDatabase(), task, name, version));
+				co_await (success(bc) && success(restoreRanges) && success(addPrefix) && success(removePrefix) &&
+				          checkTaskVersion(tr->getDatabase(), task, name, version));
 				break;
 
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		state Reference<IAsyncFile> inFile = wait(bc.get()->readFile(rangeFile.fileName));
-		state Standalone<VectorRef<KeyValueRef>> blockData;
+		Reference<IAsyncFile> inFile = co_await bc.get()->readFile(rangeFile.fileName);
+		Standalone<VectorRef<KeyValueRef>> blockData;
 		try {
 			// data is each real KV, not encoded mutations
-			Standalone<VectorRef<KeyValueRef>> data = wait(decodeRangeFileBlock(inFile, readOffset, readLen, cx));
+			Standalone<VectorRef<KeyValueRef>> data = co_await decodeRangeFileBlock(inFile, readOffset, readLen, cx);
 			blockData = data;
 		} catch (Error& e) {
 			if (e.code() == error_code_encrypt_keys_fetch_failed || e.code() == error_code_encrypt_key_not_found) {
-				return Void();
+				co_return;
 			}
 			throw;
 		}
-		state Arena arena;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+		Arena arena;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 
 		// First and last key are the range for this file
-		state KeyRange fileRange;
-		state std::vector<KeyRange> originalFileRanges;
+		KeyRange fileRange;
+		std::vector<KeyRange> originalFileRanges;
 		// If fileRange doesn't intersect restore range then we're done.
-		state int index;
+		int index{ 0 };
 		for (index = 0; index < restoreRanges.get().size(); index++) {
 			auto& restoreRange = restoreRanges.get()[index];
 			fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
@@ -4644,19 +4944,19 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 			while (rangeEnd > rangeStart && !restoreRange.contains(blockData[rangeEnd - 1].key))
 				--rangeEnd;
 
-			state VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
+			VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
 
 			// Shrink file range to be entirely within restoreRange and translate it to the new prefix
 			// First, use the untranslated file range to create the shrunk original file range which must be used in
 			// the kv range version map for applying mutations
-			state KeyRange originalFileRange =
+			KeyRange originalFileRange =
 			    KeyRangeRef(std::max(fileRange.begin, restoreRange.begin), std::min(fileRange.end, restoreRange.end));
 			originalFileRanges.push_back(originalFileRange);
 
 			// Now shrink and translate fileRange
 			Key fileEnd = std::min(fileRange.end, restoreRange.end);
-			if (fileEnd == (removePrefix.get() == StringRef() ? allKeys.end : strinc(removePrefix.get()))) {
-				fileEnd = addPrefix.get() == StringRef() ? allKeys.end : strinc(addPrefix.get());
+			if (fileEnd == (removePrefix.get().empty() ? allKeys.end : strinc(removePrefix.get()))) {
+				fileEnd = addPrefix.get().empty() ? allKeys.end : strinc(addPrefix.get());
 			} else {
 				fileEnd = fileEnd.removePrefix(removePrefix.get()).withPrefix(addPrefix.get());
 			}
@@ -4665,20 +4965,21 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 			                            .withPrefix(addPrefix.get()),
 			                        fileEnd);
 
-			state int start = 0;
-			state int end = data.size();
-			state int dataSizeLimit =
+			int start = 0;
+			int end = data.size();
+			int dataSizeLimit =
 			    BUGGIFY ? deterministicRandom()->randomInt(256 * 1024, 10e6) : CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
 
 			tr->reset();
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-					state int i = start;
-					state int txBytes = 0;
-					state int iend = start;
+					int i = start;
+					int txBytes = 0;
+					int iend = start;
 
 					// find iend that results in the desired transaction size
 					for (; iend < end && txBytes < dataSizeLimit; ++iend) {
@@ -4690,7 +4991,7 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 					// If start == 0 then use fileBegin for the start of the range, else data[start]
 					// If iend == end then use fileEnd for the end of the range, else data[iend]
 					// clear the raw key(without alog prefix)
-					state KeyRange trRange = KeyRangeRef(
+					KeyRange trRange = KeyRangeRef(
 					    (start == 0) ? fileRange.begin
 					                 : data[start].key.removePrefix(removePrefix.get()).withPrefix(addPrefix.get()),
 					    (iend == end) ? fileRange.end
@@ -4706,13 +5007,13 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 					// Add to bytes written count
 					restore.bytesWritten().atomicOp(tr, txBytes, MutationRef::Type::AddValue);
 
-					state Future<Void> checkLock = checkDatabaseLock(tr, restore.getUid());
+					Future<Void> checkLock = checkDatabaseLock(tr, restore.getUid());
 
-					wait(taskBucket->keepRunning(tr, task));
+					co_await taskBucket->keepRunning(tr, task);
 
-					wait(checkLock);
+					co_await checkLock;
 
-					wait(tr->commit());
+					co_await tr->commit();
 
 					TraceEvent("FileRestoreCommittedRange")
 					    .suppressFor(60)
@@ -4729,8 +5030,7 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 					    .detail("EndIndex", i)
 					    .detail("DataSize", data.size())
 					    .detail("Bytes", txBytes)
-					    .detail("OriginalFileRange", originalFileRange)
-					    .detail("TaskInstance", THIS_ADDR);
+					    .detail("OriginalFileRange", originalFileRange);
 
 					// Commit succeeded, so advance starting point
 					start = i;
@@ -4738,12 +5038,14 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 					if (start == end)
 						break;
 					tr->reset();
+					continue;
 				} catch (Error& e) {
-					if (e.code() == error_code_transaction_too_large)
-						dataSizeLimit /= 2;
-					else
-						wait(tr->onError(e));
+					err = e;
 				}
+				if (err.code() == error_code_transaction_too_large)
+					dataSizeLimit /= 2;
+				else
+					co_await tr->onError(err);
 			}
 		}
 		if (!originalFileRanges.empty()) {
@@ -4753,14 +5055,14 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 				Params.originalFileRanges().set(task, originalFileRanges);
 			}
 		}
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		RestoreConfig restore(task);
 		restore.fileBlocksFinished().atomicOp(tr, 1, MutationRef::Type::AddValue);
 
 		// Update the KV range map if originalFileRange is set
@@ -4772,36 +5074,36 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 			updateMap.push_back(krmSetRange(tr, restore.applyMutationsMapPrefix(), range, versionEncoded));
 		}
 
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task) && waitForAll(updateMap));
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task) && waitForAll(updateMap));
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 RestoreFile rf,
-	                                 int64_t offset,
-	                                 int64_t len,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
-		state Reference<Task> task(new Task(RestoreRangeTaskFunc::name, RestoreRangeTaskFunc::version, doneKey));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           RestoreFile rf,
+	                           int64_t offset,
+	                           int64_t len,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
+		Reference<Task> task(new Task(RestoreRangeTaskFunc::name, RestoreRangeTaskFunc::version, doneKey));
 
 		// Create a restore config from the current task and bind it to the new task.
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 
 		Params.inputFile().set(task, rf);
 		Params.readOffset().set(task, offset);
 		Params.readLen().set(task, len);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	static StringRef name;
@@ -4990,15 +5292,15 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 	static struct : InputParams {
 	} Params;
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		RestoreConfig restore(task);
 
-		state RestoreFile logFile = Params.inputFile().get(task);
-		state int64_t readOffset = Params.readOffset().get(task);
-		state int64_t readLen = Params.readLen().get(task);
+		RestoreFile logFile = Params.inputFile().get(task);
+		int64_t readOffset = Params.readOffset().get(task);
+		int64_t readLen = Params.readLen().get(task);
 
 		TraceEvent("FileRestoreLogStart")
 		    .suppressFor(60)
@@ -5008,57 +5310,59 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 		    .detail("FileEndVersion", logFile.endVersion)
 		    .detail("FileSize", logFile.fileSize)
 		    .detail("ReadOffset", readOffset)
-		    .detail("ReadLen", readLen)
-		    .detail("TaskInstance", THIS_ADDR);
+		    .detail("ReadLen", readLen);
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Reference<IBackupContainer> bc;
-		state std::vector<KeyRange> ranges;
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Reference<IBackupContainer> bc;
+		std::vector<KeyRange> ranges;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
+				Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
 				bc = getBackupContainerWithProxy(_bc);
 
-				wait(store(ranges, restore.getRestoreRangesOrDefault(tr)));
+				ranges = co_await restore.getRestoreRangesOrDefault(tr);
 
-				wait(checkTaskVersion(tr->getDatabase(), task, name, version));
-				wait(taskBucket->keepRunning(tr, task));
+				co_await checkTaskVersion(tr->getDatabase(), task, name, version);
+				co_await taskBucket->keepRunning(tr, task);
 
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		state Key mutationLogPrefix = restore.mutationLogPrefix();
-		state Reference<IAsyncFile> inFile = wait(bc->readFile(logFile.fileName));
-		state Standalone<VectorRef<KeyValueRef>> dataOriginal =
-		    wait(decodeMutationLogFileBlock(inFile, readOffset, readLen));
+		Key mutationLogPrefix = restore.mutationLogPrefix();
+		Reference<IAsyncFile> inFile = co_await bc->readFile(logFile.fileName);
+		Standalone<VectorRef<KeyValueRef>> dataOriginal =
+		    co_await decodeMutationLogFileBlock(inFile, readOffset, readLen);
 
 		// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for
 		// this restore based on the restore range set.
 		RangeMapFilters filters(ranges);
-		state std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, filters);
+		std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, filters);
 
-		state int start = 0;
-		state int end = dataFiltered.size();
-		state int dataSizeLimit =
+		int start = 0;
+		int end = dataFiltered.size();
+		int dataSizeLimit =
 		    BUGGIFY ? deterministicRandom()->randomInt(256 * 1024, 10e6) : CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
 
 		tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				if (start == end)
-					return Void();
+					co_return;
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state int i = start;
-				state int txBytes = 0;
+				int i = start;
+				int txBytes = 0;
 				for (; i < end && txBytes < dataSizeLimit; ++i) {
 					Key k = dataFiltered[i].key.withPrefix(mutationLogPrefix);
 					ValueRef v = dataFiltered[i].value; // each KV is a [param1 with added prefix -> param2]
@@ -5067,15 +5371,15 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 					txBytes += v.expectedSize();
 				}
 
-				state Future<Void> checkLock = checkDatabaseLock(tr, restore.getUid());
+				Future<Void> checkLock = checkDatabaseLock(tr, restore.getUid());
 
-				wait(taskBucket->keepRunning(tr, task));
-				wait(checkLock);
+				co_await taskBucket->keepRunning(tr, task);
+				co_await checkLock;
 
 				// Add to bytes written count
 				restore.bytesWritten().atomicOp(tr, txBytes, MutationRef::Type::AddValue);
 
-				wait(tr->commit());
+				co_await tr->commit();
 
 				TraceEvent("FileRestoreCommittedLog")
 				    .suppressFor(60)
@@ -5091,60 +5395,61 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 				    .detail("EndIndex", i)
 				    .detail("RecordCountOriginal", dataOriginal.size())
 				    .detail("RecordCountFiltered", dataFiltered.size())
-				    .detail("Bytes", txBytes)
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Bytes", txBytes);
 
 				// Commit succeeded, so advance starting point
 				start = i;
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_transaction_too_large)
-					dataSizeLimit /= 2;
-				else
-					wait(tr->onError(e));
+				err = e;
 			}
+			if (err.code() == error_code_transaction_too_large)
+				dataSizeLimit /= 2;
+			else
+				co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 		RestoreConfig(task).fileBlocksFinished().atomicOp(tr, 1, MutationRef::Type::AddValue);
 
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
 		// TODO:  Check to see if there is a leak in the FutureBucket since an invalid task (validation key fails)
 		// will never set its taskFuture.
-		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
+		co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 RestoreFile lf,
-	                                 int64_t offset,
-	                                 int64_t len,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
-		state Reference<Task> task(new Task(RestoreLogDataTaskFunc::name, RestoreLogDataTaskFunc::version, doneKey));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           RestoreFile lf,
+	                           int64_t offset,
+	                           int64_t len,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
+		Reference<Task> task(new Task(RestoreLogDataTaskFunc::name, RestoreLogDataTaskFunc::version, doneKey));
 
 		// Create a restore config from the current task and bind it to the new task.
 		// RestoreConfig(parentTask) creates prefix of : fileRestorePrefixRange.begin/uid->config/[uid]
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 		Params.inputFile().set(task, lf);
 		Params.readOffset().set(task, offset);
 		Params.readLen().set(task, len);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	Future<Void> execute(Database cx,
@@ -5270,40 +5575,37 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 	} Params;
 
 	// From all iterators, find the minimum version and return it together with if this version has data.
-	ACTOR static Future<std::pair<Version, bool>> findNextVersion(
-	    std::vector<Reference<PartitionedLogIterator>> iterators) {
-		state Version minVersion = std::numeric_limits<int64_t>::max();
-		state bool atLeastOneIteratorHasNext = false;
-		state std::vector<Version> minVs(iterators.size(), 0); // can be removed or leave for debugging
-
-		state int k = 0;
-		for (; k < iterators.size(); k++) {
+	static Future<std::pair<Version, bool>> findNextVersion(std::vector<Reference<PartitionedLogIterator>> iterators) {
+		Version minVersion = std::numeric_limits<int64_t>::max();
+		bool atLeastOneIteratorHasNext = false;
+
+		for (int k = 0; k < iterators.size(); k++) {
 			if (!iterators[k]->hasNext()) {
 				continue;
 			}
 			atLeastOneIteratorHasNext = true;
-			Version v = wait(iterators[k]->peekNextVersion());
-			minVs[k] = v;
+			Version v = co_await iterators[k]->peekNextVersion();
 			minVersion = std::min(minVersion, v);
 		}
-		return std::make_pair(minVersion, atLeastOneIteratorHasNext);
+		co_return std::make_pair(minVersion, atLeastOneIteratorHasNext);
 	}
 
 	// Reads from all iterators and sends a list of mutations for the given version
 	// to the mutationStream for consumption. This actor reads as fast as possible,
 	// which is fine since the task is working on the version range of size
 	// RESTORE_PARTITIONED_BATCH_VERSION_SIZE, thus limiting the amount of memory used.
-	ACTOR static Future<Void> readLogData(PromiseStream<Standalone<VectorRef<KeyValueRef>>> mutationStream,
-	                                      std::vector<Reference<PartitionedLogIterator>> iterators,
-	                                      Version begin,
-	                                      Version end) {
+	static Future<Void> readLogData(PromiseStream<Standalone<VectorRef<KeyValueRef>>> mutationStream,
+	                                std::vector<Reference<PartitionedLogIterator>> iterators,
+	                                Version begin,
+	                                Version end) {
 		// mergeSort all iterator until all are exhausted
 		// it stores all mutations for the next min version, in new format
 		try {
-			state bool atLeastOneIteratorHasNext = true;
-			state Version minVersion, lastMinVersion = invalidVersion;
+			bool atLeastOneIteratorHasNext = true;
+			Version minVersion{ 0 };
+			Version lastMinVersion = invalidVersion;
 			while (atLeastOneIteratorHasNext) {
-				std::pair<Version, bool> minVersionAndHasNext = wait(findNextVersion(iterators));
+				std::pair<Version, bool> minVersionAndHasNext = co_await findNextVersion(iterators);
 				minVersion = minVersionAndHasNext.first;
 				atLeastOneIteratorHasNext = minVersionAndHasNext.second;
 				ASSERT_LT(lastMinVersion, minVersion);
@@ -5311,7 +5613,7 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 
 				if (atLeastOneIteratorHasNext) {
 					std::vector<Standalone<VectorRef<VersionedMutation>>> mutationsSingleVersion =
-					    wait(getMutationsForVersion(iterators, minVersion));
+					    co_await getMutationsForVersion(iterators, minVersion);
 
 					if (minVersion < begin) {
 						// skip generating mutations, because this is not within desired range
@@ -5334,52 +5636,52 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 			TraceEvent(SevWarn, "FileRestoreLogReadError").error(e);
 			mutationStream.sendError(e);
 		}
-		return Void();
+		co_return;
 	}
 
 	// Reads from all iterators and returns a list of mutations for the given version.
-	ACTOR static Future<std::vector<Standalone<VectorRef<VersionedMutation>>>> getMutationsForVersion(
+	static Future<std::vector<Standalone<VectorRef<VersionedMutation>>>> getMutationsForVersion(
 	    std::vector<Reference<PartitionedLogIterator>> iterators,
 	    Version minVersion) {
-		state std::vector<Standalone<VectorRef<VersionedMutation>>> mutationsSingleVersion;
+		std::vector<Standalone<VectorRef<VersionedMutation>>> mutationsSingleVersion;
 
-		state int k = 0;
-		for (; k < iterators.size(); k++) {
+		for (int k = 0; k < iterators.size(); k++) {
 			if (!iterators[k]->hasNext()) {
 				continue;
 			}
-			Version v = wait(iterators[k]->peekNextVersion());
+			Version v = co_await iterators[k]->peekNextVersion();
 			if (v == minVersion) {
-				Standalone<VectorRef<VersionedMutation>> tmp = wait(iterators[k]->getNext());
+				Standalone<VectorRef<VersionedMutation>> tmp = co_await iterators[k]->getNext();
 				mutationsSingleVersion.push_back(tmp);
 			}
 		}
-		return mutationsSingleVersion;
+		co_return mutationsSingleVersion;
 	}
 
 	// Writes backup mutations to the database
-	ACTOR static Future<Void> writeMutations(Database cx,
-	                                         std::vector<Standalone<VectorRef<KeyValueRef>>> mutations,
-	                                         Key mutationLogPrefix,
-	                                         Reference<Task> task,
-	                                         Reference<TaskBucket> taskBucket) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Standalone<VectorRef<KeyValueRef>> oldFormatMutations;
-		state int mutationIndex = 0;
-		state int mutationCount = 0;
-		state int txBytes = 0;
-		state int txBytesLimit = CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
+	static Future<Void> writeMutations(Database cx,
+	                                   std::vector<Standalone<VectorRef<KeyValueRef>>> mutations,
+	                                   Key mutationLogPrefix,
+	                                   Reference<Task> task,
+	                                   Reference<TaskBucket> taskBucket) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Standalone<VectorRef<KeyValueRef>> oldFormatMutations;
+		int mutationIndex = 0;
+		int mutationCount = 0;
+		int txBytes = 0;
+		int txBytesLimit = CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE;
 
 		for (const auto& data : mutations) {
 			oldFormatMutations.append(oldFormatMutations.arena(), data.begin(), data.size());
 		}
-		state int totalMutation = oldFormatMutations.size();
+		int totalMutation = oldFormatMutations.size();
 
 		// multiple transactions are needed, so this has to be in a _execute method rather than
 		// a _finish method
 		// this transaction does blind writes, so they are idempotent operations even if multiple instances of
 		// the same tasks are running these transactions.
-		loop {
+		while (true) {
+			Error err;
 			try {
 				if (mutationIndex == totalMutation) {
 					break;
@@ -5398,60 +5700,63 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 					txBytes += v.expectedSize();
 					++mutationCount;
 				}
-				wait(taskBucket->keepRunning(tr, task));
-				wait(tr->commit());
+				co_await taskBucket->keepRunning(tr, task);
+				co_await tr->commit();
 
 				int64_t oldBytes = Params.bytesWritten().get(task);
 				Params.bytesWritten().set(task, oldBytes + txBytes);
 				DisabledTraceEvent("FileRestorePartitionedLogCommittData")
-				    .detail("TaskInstance", THIS_ADDR)
 				    .detail("MutationIndex", mutationIndex)
 				    .detail("MutationCount", mutationCount)
 				    .detail("TotalMutation", totalMutation)
 				    .detail("Bytes", txBytes);
 				mutationIndex += mutationCount; // update mutationIndex after the commit succeeds
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_transaction_too_large) {
-					txBytesLimit /= 2;
-				} else {
-					wait(tr->onError(e));
-				}
+				err = e;
+			}
+			if (err.code() == error_code_transaction_too_large) {
+				txBytesLimit /= 2;
+			} else {
+				co_await tr->onError(err);
 			}
 		}
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		RestoreConfig restore(task);
 
-		state int64_t maxTagID = Params.maxTagID().get(task);
-		state std::vector<RestoreConfig::RestoreFile> logs = Params.logs().get(task);
-		state Version begin = Params.beginVersion().get(task);
-		state Version end = Params.endVersion().get(task);
+		int64_t maxTagID = Params.maxTagID().get(task);
+		std::vector<RestoreConfig::RestoreFile> logs = Params.logs().get(task);
+		Version begin = Params.beginVersion().get(task);
+		Version end = Params.endVersion().get(task);
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Reference<IBackupContainer> bc;
-		state std::vector<KeyRange> ranges; // this is the actual KV, not version
-		loop {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Reference<IBackupContainer> bc;
+		std::vector<KeyRange> ranges; // this is the actual KV, not version
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
+				Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
 				bc = getBackupContainerWithProxy(_bc);
 
-				wait(store(ranges, restore.getRestoreRangesOrDefault(tr)));
+				ranges = co_await restore.getRestoreRangesOrDefault(tr);
 
-				wait(checkTaskVersion(tr->getDatabase(), task, name, version));
-				wait(taskBucket->keepRunning(tr, task));
+				co_await checkTaskVersion(tr->getDatabase(), task, name, version);
+				co_await taskBucket->keepRunning(tr, task);
 
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		std::vector<std::vector<RestoreConfig::RestoreFile>> filesByTag(maxTagID + 1);
@@ -5483,7 +5788,7 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 			}
 		}
 
-		state std::vector<Reference<PartitionedLogIterator>> iterators(maxTagID + 1);
+		std::vector<Reference<PartitionedLogIterator>> iterators(maxTagID + 1);
 		// for each tag, create an iterator
 		for (int k = 0; k < filesByTag.size(); k++) {
 			iterators[k] =
@@ -5493,55 +5798,56 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 		DisabledTraceEvent("FileRestorePartitionedLogDataExeStart")
 		    .detail("BeginVersion", begin)
 		    .detail("EndVersion", end)
-		    .detail("Files", logs.size())
-		    .detail("TaskInstance", THIS_ADDR);
+		    .detail("Files", logs.size());
 
 		// Converted backup mutations
-		state PromiseStream<Standalone<VectorRef<KeyValueRef>>> mutationStream;
-		state Future<Void> reader = readLogData(mutationStream, iterators, begin, end);
-
-		state std::vector<Standalone<VectorRef<KeyValueRef>>> mutations;
-		state int64_t totalBytes = 0;
-		state Standalone<VectorRef<KeyValueRef>> oneVersionData;
-		loop {
+		PromiseStream<Standalone<VectorRef<KeyValueRef>>> mutationStream;
+		Future<Void> reader = readLogData(mutationStream, iterators, begin, end);
+
+		std::vector<Standalone<VectorRef<KeyValueRef>>> mutations;
+		int64_t totalBytes = 0;
+		Standalone<VectorRef<KeyValueRef>> oneVersionData;
+		while (true) {
+			Error err;
 			try {
-				Standalone<VectorRef<KeyValueRef>> _data = waitNext(mutationStream.getFuture());
+				Standalone<VectorRef<KeyValueRef>> _data = co_await mutationStream.getFuture();
 				oneVersionData = _data;
 
 				// batching mutations from multiple versions together before writing to the database
-				state int64_t bytes = oneVersionData.expectedSize();
+				int64_t bytes = oneVersionData.expectedSize();
 				if (totalBytes + bytes > CLIENT_KNOBS->RESTORE_WRITE_TX_SIZE) {
-					wait(writeMutations(cx, mutations, restore.mutationLogPrefix(), task, taskBucket));
+					co_await writeMutations(cx, mutations, restore.mutationLogPrefix(), task, taskBucket);
 					mutations.clear();
 					totalBytes = 0;
 				}
 				mutations.push_back(oneVersionData);
 				totalBytes += bytes;
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_end_of_stream) {
-					if (mutations.size() > 0) {
-						wait(writeMutations(cx, mutations, restore.mutationLogPrefix(), task, taskBucket));
-					}
-					break;
-				} else {
-					throw;
+				err = e;
+			}
+			if (err.code() == error_code_end_of_stream) {
+				if (!mutations.empty()) {
+					co_await writeMutations(cx, mutations, restore.mutationLogPrefix(), task, taskBucket);
 				}
+				break;
+			} else {
+				throw err;
 			}
 		}
 		DisabledTraceEvent("FileRestorePartitionedLogDataExeDone")
 		    .detail("BeginVersion", begin)
 		    .detail("EndVersion", end)
-		    .detail("Files", logs.size())
-		    .detail("TaskInstance", THIS_ADDR);
+		    .detail("Files", logs.size());
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state int64_t logBytesWritten = Params.bytesWritten().get(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		int64_t logBytesWritten = Params.bytesWritten().get(task);
 		RestoreConfig(task).bytesWritten().atomicOp(tr, logBytesWritten, MutationRef::Type::AddValue);
 
 		int64_t blocks =
@@ -5551,32 +5857,31 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 		RestoreConfig(task).fileBlocksFinished().atomicOp(tr, blocks, MutationRef::Type::AddValue);
 
 		DisabledTraceEvent("FileRestorePartitionedLogCommittedData")
-		    .detail("TaskInstance", THIS_ADDR)
 		    .detail("Blocks", blocks)
 		    .detail("LogBytes", logBytesWritten);
 
-		state Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		wait(taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
+		Reference<TaskFuture> taskFuture = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		co_await (taskFuture->set(tr, taskBucket) && taskBucket->finish(tr, task));
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 int64_t maxTagID,
-	                                 std::vector<RestoreConfig::RestoreFile> logs,
-	                                 Version begin,
-	                                 Version end,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
-		state Reference<Task> task(
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           int64_t maxTagID,
+	                           std::vector<RestoreConfig::RestoreFile> logs,
+	                           Version begin,
+	                           Version end,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
+		Reference<Task> task(
 		    new Task(RestoreLogDataPartitionedTaskFunc::name, RestoreLogDataPartitionedTaskFunc::version, doneKey));
 
 		// Create a restore config from the current task and bind it to the new task.
 		// RestoreConfig(parentTask) createsa prefix of : fileRestorePrefixRange.begin/uid->config/[uid]
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 		Params.maxTagID().set(task, maxTagID);
 		Params.beginVersion().set(task, begin);
 		Params.endVersion().set(task, end);
@@ -5584,11 +5889,11 @@ struct RestoreLogDataPartitionedTaskFunc : RestoreFileTaskFuncBase {
 		Params.bytesWritten().set(task, 0);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	Future<Void> execute(Database cx,
@@ -5621,27 +5926,27 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 		static TaskParam<Version> endVersion() { return __FUNCTION__sr; }
 	} Params;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		RestoreConfig restore(task);
 
-		state Version beginVersion = Params.beginVersion().get(task);
-		state Version firstVersion = Params.firstVersion().get(task);
-		state Version endVersion = Params.endVersion().get(task);
-		Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
-		state Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
-		state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Version beginVersion = Params.beginVersion().get(task);
+		Version firstVersion = Params.firstVersion().get(task);
+		Version endVersion = Params.endVersion().get(task);
+		Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
+		Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
+		Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
-		state Version restoreVersion;
-		state int fileLimit = 1000;
+		Version restoreVersion{ 0 };
+		int fileLimit = 1000;
 
-		wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) &&
-		     checkTaskVersion(tr->getDatabase(), task, name, version));
+		co_await (store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) &&
+		          checkTaskVersion(tr->getDatabase(), task, name, version));
 
 		// if current is [40, 50] and restore version is 50, we need another [50, 51] task to process data at version 50
-		state Version nextEndVersion =
+		Version nextEndVersion =
 		    std::min(restoreVersion + 1, endVersion + CLIENT_KNOBS->RESTORE_PARTITIONED_BATCH_VERSION_SIZE);
 		// update the apply mutations end version so the mutations from the previous batch can be applied.
 		// Only do this once beginVersion is > 0 (it will be 0 for the initial dispatch).
@@ -5652,25 +5957,24 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 		}
 
 		// The applyLag must be retrieved AFTER potentially updating the apply end version.
-		state int64_t applyLag = wait(restore.getApplyVersionLag(tr));
+		int64_t applyLag = co_await restore.getApplyVersionLag(tr);
 		// this is to guarantee commit proxy is catching up doing apply alog -> normal key
 		// with this  backupFile -> alog process
 		// If starting a new batch and the apply lag is too large then re-queue and wait
 		if (applyLag > (BUGGIFY ? 1 : CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 300)) {
 			// Wait a small amount of time and then re-add this same task.
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			wait(success(RestoreDispatchPartitionedTaskFunc::addTask(
-			    tr, taskBucket, task, firstVersion, beginVersion, endVersion)));
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			co_await RestoreDispatchPartitionedTaskFunc::addTask(
+			    tr, taskBucket, task, firstVersion, beginVersion, endVersion);
 
 			TraceEvent("FileRestorePartitionDispatch")
 			    .detail("RestoreUID", restore.getUid())
 			    .detail("BeginVersion", beginVersion)
 			    .detail("ApplyLag", applyLag)
-			    .detail("Decision", "too_far_behind")
-			    .detail("TaskInstance", THIS_ADDR);
+			    .detail("Decision", "too_far_behind");
 
-			wait(taskBucket->finish(tr, task));
-			return Void();
+			co_await taskBucket->finish(tr, task);
+			co_return;
 		}
 
 		// Get a batch of files.  We're targeting batchSize blocks(30k) being dispatched so query for batchSize(150)
@@ -5684,34 +5988,34 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 		// begin needs to start from the beginning --
 		// if we have a 3 files, (log,100,200), (range, 180), (log, 200, 300), then if we want to restore[190, 250]
 		// it would stop at (range, 180) and not looking at (log, 100, 200)
-		state Optional<RestoreConfig::RestoreFile> beginLogInclude = Optional<RestoreConfig::RestoreFile>{};
+		Optional<RestoreConfig::RestoreFile> beginLogInclude = Optional<RestoreConfig::RestoreFile>{};
 		// because of the encoding of RestoreFile, we use greaterThanOrEqual(end + 1) instead of greaterThan(end)
 		// because RestoreFile::pack has the version at the most significant position, and keyAfter(end) does not result
 		// in a end+1
-		state Optional<RestoreConfig::RestoreFile> endLogExclude = wait(
-		    restore.logFileSet().seekGreaterOrEqual(tr, RestoreConfig::RestoreFile({ endVersion + 1, "", false })));
-		state RestoreConfig::FileSetT::RangeResultType logFiles =
-		    wait(restore.logFileSet().getRange(tr, beginLogInclude, endLogExclude, fileLimit));
-		state Optional<RestoreConfig::RestoreFile> beginRangeInclude =
-		    wait(restore.rangeFileSet().seekGreaterOrEqual(tr, RestoreConfig::RestoreFile({ beginVersion, "", true })));
+		Optional<RestoreConfig::RestoreFile> endLogExclude = co_await restore.logFileSet().seekGreaterOrEqual(
+		    tr, RestoreConfig::RestoreFile({ endVersion + 1, "", false }));
+		RestoreConfig::FileSetT::RangeResultType logFiles =
+		    co_await restore.logFileSet().getRange(tr, beginLogInclude, endLogExclude, fileLimit);
+		Optional<RestoreConfig::RestoreFile> beginRangeInclude = co_await restore.rangeFileSet().seekGreaterOrEqual(
+		    tr, RestoreConfig::RestoreFile({ beginVersion, "", true }));
 		// greaterThanOrEqual(end + 1) instead of greaterThan(end)
 		// because RestoreFile::pack has the version at the most significant position, and keyAfter(end) does not result
 		// in a end+1
-		state Optional<RestoreConfig::RestoreFile> endRangeExclude = wait(
-		    restore.rangeFileSet().seekGreaterOrEqual(tr, RestoreConfig::RestoreFile({ endVersion + 1, "", true })));
-		state RestoreConfig::FileSetT::RangeResultType rangeFiles =
-		    wait(restore.rangeFileSet().getRange(tr, beginRangeInclude, endRangeExclude, fileLimit));
-		state int64_t maxTagID = 0;
-		state std::vector<RestoreConfig::RestoreFile> logs;
-		state std::vector<RestoreConfig::RestoreFile> ranges;
-		for (auto f : logFiles.results) {
+		Optional<RestoreConfig::RestoreFile> endRangeExclude = co_await restore.rangeFileSet().seekGreaterOrEqual(
+		    tr, RestoreConfig::RestoreFile({ endVersion + 1, "", true }));
+		RestoreConfig::FileSetT::RangeResultType rangeFiles =
+		    co_await restore.rangeFileSet().getRange(tr, beginRangeInclude, endRangeExclude, fileLimit);
+		int64_t maxTagID = 0;
+		std::vector<RestoreConfig::RestoreFile> logs;
+		std::vector<RestoreConfig::RestoreFile> ranges;
+		for (const auto& f : logFiles.results) {
 			if (f.endVersion > beginVersion) {
 				// skip all files whose endVersion is smaller or equal to beginVersion
 				logs.push_back(f);
 				maxTagID = std::max(maxTagID, f.tagId);
 			}
 		}
-		for (auto f : rangeFiles.results) {
+		for (const auto& f : rangeFiles.results) {
 			// the getRange might get out-of-bound range file because log files need them to work
 			if (f.version >= beginVersion && f.version < endVersion) {
 				ranges.push_back(f);
@@ -5719,7 +6023,7 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 		}
 		// allPartsDone will be set once all block tasks in the current batch are finished.
 		// create a new future for the new batch
-		state Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
+		Reference<TaskFuture> allPartsDone = futureBucket->future(tr);
 		restore.batchFuture().set(tr, allPartsDone->pack());
 
 		// if there are no files, if i am not the last batch, then on to the next batch
@@ -5729,41 +6033,39 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 			if (applyLag == 0) {
 				// i am the last batch
 				// If apply lag is 0 then we are done so create the completion task
-				wait(success(RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal())));
+				co_await RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal());
 
 				TraceEvent("FileRestorePartitionDispatch")
 				    .detail("RestoreUID", restore.getUid())
 				    .detail("BeginVersion", beginVersion)
 				    .detail("ApplyLag", applyLag)
-				    .detail("Decision", "restore_complete")
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Decision", "restore_complete");
 			} else {
 				// i am the last batch, and applyLag is not zero, then I will create another dummy task to wait
 				// for apply log to be zero, then it will go into the branch above.
 				// Applying of mutations is not yet finished so wait a small amount of time and then re-add this
 				// same task.
 				// this is only to create a dummy one wait for it to finish
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				wait(success(RestoreDispatchPartitionedTaskFunc::addTask(
-				    tr, taskBucket, task, firstVersion, beginVersion, endVersion)));
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+				co_await RestoreDispatchPartitionedTaskFunc::addTask(
+				    tr, taskBucket, task, firstVersion, beginVersion, endVersion);
 
 				TraceEvent("FileRestorePartitionDispatch")
 				    .detail("RestoreUID", restore.getUid())
 				    .detail("BeginVersion", beginVersion)
 				    .detail("ApplyLag", applyLag)
-				    .detail("Decision", "apply_still_behind")
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Decision", "apply_still_behind");
 			}
-			wait(taskBucket->finish(tr, task));
-			return Void();
+			co_await taskBucket->finish(tr, task);
+			co_return;
 		}
 
 		// if we reach here, this batch is not empty(i.e. we have range and/or mutation files in this)
 		// Start moving through the file list and queuing up blocks.  Only queue up to RESTORE_DISPATCH_ADDTASK_SIZE
 		// blocks per Dispatch task and target batchSize total per batch but a batch must end on a complete version
 		// boundary so exceed the limit if necessary to reach the end of a version of files.
-		state std::vector<Future<Key>> addTaskFutures;
-		state int i = 0;
+		std::vector<Future<Key>> addTaskFutures;
+		int i = 0;
 		// need to process all range files, keep using the same RestoreRangeTaskFunc as non-partitioned restore.
 		// this can be done first, because they are not overlap within a restore uid
 		// each task will read the file, restore those key to their original keys after clear that range
@@ -5798,8 +6100,8 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 		                                                                     TaskCompletionKey::noSignal(),
 		                                                                     allPartsDone));
 
-		wait(waitForAll(addTaskFutures));
-		wait(taskBucket->finish(tr, task));
+		co_await waitForAll(addTaskFutures);
+		co_await taskBucket->finish(tr, task);
 
 		TraceEvent("FileRestorePartitionDispatch")
 		    .detail("RestoreUID", restore.getUid())
@@ -5807,39 +6109,38 @@ struct RestoreDispatchPartitionedTaskFunc : RestoreTaskFuncBase {
 		    .detail("EndVersion", endVersion)
 		    .detail("NextEndVersion", nextEndVersion)
 		    .detail("ApplyLag", applyLag)
-		    .detail("Decision", "dispatch_batch_complete")
-		    .detail("TaskInstance", THIS_ADDR);
+		    .detail("Decision", "dispatch_batch_complete");
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version firstVersion,
-	                                 Version beginVersion,
-	                                 Version endVersion,
-	                                 TaskCompletionKey completionKey = TaskCompletionKey::noSignal(),
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version firstVersion,
+	                           Version beginVersion,
+	                           Version endVersion,
+	                           TaskCompletionKey completionKey = TaskCompletionKey::noSignal(),
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 
 		// Use high priority for dispatch tasks that have to queue more blocks for the current batch
 		unsigned int priority = 0;
-		state Reference<Task> task(new Task(
+		Reference<Task> task(new Task(
 		    RestoreDispatchPartitionedTaskFunc::name, RestoreDispatchPartitionedTaskFunc::version, doneKey, priority));
 
 		// Create a config from the parent task and bind it to the new task
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 		Params.firstVersion().set(task, firstVersion);
 		Params.beginVersion().set(task, beginVersion);
 		Params.endVersion().set(task, endVersion);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	Future<Void> execute(Database cx,
@@ -5871,21 +6172,21 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		static TaskParam<int64_t> remainingInBatch() { return __FUNCTION__sr; }
 	} Params;
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state RestoreConfig restore(task);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		RestoreConfig restore(task);
 
-		state Version beginVersion = Params.beginVersion().get(task);
-		state Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		Version beginVersion = Params.beginVersion().get(task);
+		Reference<TaskFuture> onDone = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
 
-		state int64_t remainingInBatch = Params.remainingInBatch().get(task);
-		state bool addingToExistingBatch = remainingInBatch > 0;
-		state Version restoreVersion;
+		int64_t remainingInBatch = Params.remainingInBatch().get(task);
+		bool addingToExistingBatch = remainingInBatch > 0;
+		Version restoreVersion{ 0 };
 
-		wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) &&
-		     checkTaskVersion(tr->getDatabase(), task, name, version));
+		co_await (store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) &&
+		          checkTaskVersion(tr->getDatabase(), task, name, version));
 
 		// If not adding to an existing batch then update the apply mutations end version so the mutations from the
 		// previous batch can be applied.  Only do this once beginVersion is > 0 (it will be 0 for the initial
@@ -5898,43 +6199,42 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		}
 
 		// The applyLag must be retrieved AFTER potentially updating the apply end version.
-		state int64_t applyLag = wait(restore.getApplyVersionLag(tr));
-		state int64_t batchSize = Params.batchSize().get(task);
+		int64_t applyLag = co_await restore.getApplyVersionLag(tr);
+		int64_t batchSize = Params.batchSize().get(task);
 
 		// If starting a new batch and the apply lag is too large then re-queue and wait
 		if (!addingToExistingBatch && applyLag > (BUGGIFY ? 1 : CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 300)) {
 			// Wait a small amount of time and then re-add this same task.
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			wait(success(RestoreDispatchTaskFunc::addTask(
-			    tr, taskBucket, task, beginVersion, "", 0, batchSize, remainingInBatch)));
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			co_await RestoreDispatchTaskFunc::addTask(
+			    tr, taskBucket, task, beginVersion, "", 0, batchSize, remainingInBatch);
 
 			TraceEvent("FileRestoreDispatch")
 			    .detail("RestoreUID", restore.getUid())
 			    .detail("BeginVersion", beginVersion)
 			    .detail("ApplyLag", applyLag)
 			    .detail("BatchSize", batchSize)
-			    .detail("Decision", "too_far_behind")
-			    .detail("TaskInstance", THIS_ADDR);
+			    .detail("Decision", "too_far_behind");
 
-			wait(taskBucket->finish(tr, task));
-			return Void();
+			co_await taskBucket->finish(tr, task);
+			co_return;
 		}
 
 		// need beginFile to handle stop in the middle of version case
-		state std::string beginFile = Params.beginFile().getOrDefault(task);
+		std::string beginFile = Params.beginFile().getOrDefault(task);
 		// Get a batch of files.  We're targeting batchSize blocks being dispatched so query for batchSize files
 		// (each of which is 0 or more blocks).
-		state int taskBatchSize = BUGGIFY ? 1 : CLIENT_KNOBS->RESTORE_DISPATCH_ADDTASK_SIZE;
-		state RestoreConfig::FileSetT::RangeResultType files = wait(restore.fileSet().getRange(
-		    tr, Optional<RestoreConfig::RestoreFile>({ beginVersion, beginFile }), {}, taskBatchSize));
+		int taskBatchSize = BUGGIFY ? 1 : CLIENT_KNOBS->RESTORE_DISPATCH_ADDTASK_SIZE;
+		RestoreConfig::FileSetT::RangeResultType files = co_await restore.fileSet().getRange(
+		    tr, Optional<RestoreConfig::RestoreFile>({ beginVersion, beginFile }), {}, taskBatchSize);
 
 		// allPartsDone will be set once all block tasks in the current batch are finished.
-		state Reference<TaskFuture> allPartsDone;
+		Reference<TaskFuture> allPartsDone;
 
 		// If adding to existing batch then join the new block tasks to the existing batch future
 		if (addingToExistingBatch) {
-			Key fKey = wait(restore.batchFuture().getD(tr));
-			allPartsDone = Reference<TaskFuture>(new TaskFuture(futureBucket, fKey));
+			Key fKey = co_await restore.batchFuture().getD(tr);
+			allPartsDone = makeReference<TaskFuture>(futureBucket, fKey);
 		} else {
 			// Otherwise create a new future for the new batch
 			allPartsDone = futureBucket->future(tr);
@@ -5944,22 +6244,22 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		}
 
 		// If there were no files to load then this batch is done and restore is almost done.
-		if (files.results.size() == 0) {
+		if (files.results.empty()) {
 			// If adding to existing batch then blocks could be in progress so create a new Dispatch task that waits
 			// for them to finish
 			if (addingToExistingBatch) {
 				// Setting next begin to restoreVersion + 1 so that any files in the file map at the restore version
 				// won't be dispatched again.
-				wait(success(RestoreDispatchTaskFunc::addTask(tr,
-				                                              taskBucket,
-				                                              task,
-				                                              restoreVersion + 1,
-				                                              "",
-				                                              0,
-				                                              batchSize,
-				                                              0,
-				                                              TaskCompletionKey::noSignal(),
-				                                              allPartsDone)));
+				co_await RestoreDispatchTaskFunc::addTask(tr,
+				                                          taskBucket,
+				                                          task,
+				                                          restoreVersion + 1,
+				                                          "",
+				                                          0,
+				                                          batchSize,
+				                                          0,
+				                                          TaskCompletionKey::noSignal(),
+				                                          allPartsDone);
 
 				TraceEvent("FileRestoreDispatch")
 				    .detail("RestoreUID", restore.getUid())
@@ -5968,12 +6268,11 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 				    .detail("BeginBlock", Params.beginBlock().get(task))
 				    .detail("RestoreVersion", restoreVersion)
 				    .detail("ApplyLag", applyLag)
-				    .detail("Decision", "end_of_final_batch")
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Decision", "end_of_final_batch");
 			} else if (beginVersion < restoreVersion) {
 				// If beginVersion is less than restoreVersion then do one more dispatch task to get there
 				// there are no more files between beginVersion and restoreVersion
-				wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion, "", 0, batchSize)));
+				co_await RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, restoreVersion, "", 0, batchSize);
 
 				TraceEvent("FileRestoreDispatch")
 				    .detail("RestoreUID", restore.getUid())
@@ -5982,11 +6281,10 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 				    .detail("BeginBlock", Params.beginBlock().get(task))
 				    .detail("RestoreVersion", restoreVersion)
 				    .detail("ApplyLag", applyLag)
-				    .detail("Decision", "apply_to_restore_version")
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Decision", "apply_to_restore_version");
 			} else if (applyLag == 0) {
 				// If apply lag is 0 then we are done so create the completion task
-				wait(success(RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal())));
+				co_await RestoreCompleteTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal());
 
 				TraceEvent("FileRestoreDispatch")
 				    .detail("RestoreUID", restore.getUid())
@@ -5994,21 +6292,19 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 				    .detail("BeginFile", Params.beginFile().get(task))
 				    .detail("BeginBlock", Params.beginBlock().get(task))
 				    .detail("ApplyLag", applyLag)
-				    .detail("Decision", "restore_complete")
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Decision", "restore_complete");
 			} else {
 				// Applying of mutations is not yet finished so wait a small amount of time and then re-add this
 				// same task.
 				// this is only to create a dummy one wait for it to finish
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize)));
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+				co_await RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, beginVersion, "", 0, batchSize);
 
 				TraceEvent("FileRestoreDispatch")
 				    .detail("RestoreUID", restore.getUid())
 				    .detail("BeginVersion", beginVersion)
 				    .detail("ApplyLag", applyLag)
-				    .detail("Decision", "apply_still_behind")
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("Decision", "apply_still_behind");
 			}
 
 			// If adding to existing batch then task is joined with a batch future so set done future
@@ -6016,18 +6312,18 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 			// blockers already finished.
 			Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
 
-			wait(taskBucket->finish(tr, task) && setDone);
-			return Void();
+			co_await (taskBucket->finish(tr, task) && setDone);
+			co_return;
 		}
 
 		// Start moving through the file list and queuing up blocks.  Only queue up to RESTORE_DISPATCH_ADDTASK_SIZE
 		// blocks per Dispatch task and target batchSize total per batch but a batch must end on a complete version
 		// boundary so exceed the limit if necessary to reach the end of a version of files.
-		state std::vector<Future<Key>> addTaskFutures;
-		state Version endVersion = files.results[0].version;
-		state int blocksDispatched = 0;
-		state int64_t beginBlock = Params.beginBlock().getOrDefault(task);
-		state int i = 0;
+		std::vector<Future<Key>> addTaskFutures;
+		Version endVersion = files.results[0].version;
+		int blocksDispatched = 0;
+		int64_t beginBlock = Params.beginBlock().getOrDefault(task);
+		int i = 0;
 
 		// for each file
 		// not creating a new task at this level because restore files are read back together -- both range and log
@@ -6053,7 +6349,7 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 			endVersion = f.version;
 			beginFile = f.fileName;
 
-			state int64_t j = beginBlock * f.blockSize;
+			int64_t j = beginBlock * f.blockSize;
 			// For each block of the file
 			for (; j < f.fileSize; j += f.blockSize) {
 				// Stop if we've reached the addtask limit
@@ -6098,8 +6394,7 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 			TraceEvent("FileRestoreDispatchedFile")
 			    .suppressFor(60)
 			    .detail("RestoreUID", restore.getUid())
-			    .detail("FileName", f.fileName)
-			    .detail("TaskInstance", THIS_ADDR);
+			    .detail("FileName", f.fileName);
 		}
 
 		// If no blocks were dispatched then the next dispatch task should run now and be joined with the
@@ -6124,27 +6419,26 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 			    .detail("ApplyLag", applyLag)
 			    .detail("BatchSize", batchSize)
 			    .detail("Decision", decision)
-			    .detail("TaskInstance", THIS_ADDR)
 			    .detail("RemainingInBatch", remainingInBatch);
 
-			wait(success(RestoreDispatchTaskFunc::addTask(tr,
-			                                              taskBucket,
-			                                              task,
-			                                              endVersion,
-			                                              beginFile,
-			                                              beginBlock,
-			                                              batchSize,
-			                                              remainingInBatch,
-			                                              TaskCompletionKey::joinWith((allPartsDone)))));
+			co_await RestoreDispatchTaskFunc::addTask(tr,
+			                                          taskBucket,
+			                                          task,
+			                                          endVersion,
+			                                          beginFile,
+			                                          beginBlock,
+			                                          batchSize,
+			                                          remainingInBatch,
+			                                          TaskCompletionKey::joinWith((allPartsDone)));
 
 			// If adding to existing batch then task is joined with a batch future so set done future.
 			// Note that this must be done after joining at least one task with the batch future in case all other
 			// blockers already finished.
 			Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
 
-			wait(setDone && taskBucket->finish(tr, task));
+			co_await (setDone && taskBucket->finish(tr, task));
 
-			return Void();
+			co_return;
 		}
 
 		// Increment the number of blocks dispatched in the restore config
@@ -6184,12 +6478,12 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 			                                                          TaskCompletionKey::noSignal(),
 			                                                          allPartsDone));
 
-		wait(waitForAll(addTaskFutures));
+		co_await waitForAll(addTaskFutures);
 
 		// If adding to existing batch then task is joined with a batch future so set done future.
 		Future<Void> setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void();
 
-		wait(setDone && taskBucket->finish(tr, task));
+		co_await (setDone && taskBucket->finish(tr, task));
 
 		TraceEvent("FileRestoreDispatch")
 		    .detail("RestoreUID", restore.getUid())
@@ -6202,31 +6496,30 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		    .detail("Decision", "dispatched_files")
 		    .detail("FilesDispatched", i)
 		    .detail("BlocksDispatched", blocksDispatched)
-		    .detail("TaskInstance", THIS_ADDR)
 		    .detail("RemainingInBatch", remainingInBatch);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<Task> parentTask,
-	                                 Version beginVersion,
-	                                 std::string beginFile,
-	                                 int64_t beginBlock,
-	                                 int64_t batchSize,
-	                                 int64_t remainingInBatch = 0,
-	                                 TaskCompletionKey completionKey = TaskCompletionKey::noSignal(),
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<Task> parentTask,
+	                           Version beginVersion,
+	                           std::string beginFile,
+	                           int64_t beginBlock,
+	                           int64_t batchSize,
+	                           int64_t remainingInBatch = 0,
+	                           TaskCompletionKey completionKey = TaskCompletionKey::noSignal(),
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
 
 		// Use high priority for dispatch tasks that have to queue more blocks for the current batch
-		unsigned int priority = (remainingInBatch > 0) ? 1 : 0;
-		state Reference<Task> task(
+		auto priority = (remainingInBatch > 0) ? 1u : 0u;
+		Reference<Task> task(
 		    new Task(RestoreDispatchTaskFunc::name, RestoreDispatchTaskFunc::version, doneKey, priority));
 
 		// Create a config from the parent task and bind it to the new task
-		wait(RestoreConfig(parentTask).toTask(tr, task));
+		co_await RestoreConfig(parentTask).toTask(tr, task);
 		Params.beginVersion().set(task, beginVersion);
 		Params.batchSize().set(task, batchSize);
 		Params.remainingInBatch().set(task, remainingInBatch);
@@ -6234,11 +6527,11 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 		Params.beginFile().set(task, beginFile);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	Future<Void> execute(Database cx,
@@ -6257,48 +6550,53 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase {
 StringRef RestoreDispatchTaskFunc::name = "restore_dispatch"_sr;
 REGISTER_TASKFUNC(RestoreDispatchTaskFunc);
 
-ACTOR Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName) {
+Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName) {
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state std::vector<KeyBackedTag> tags;
-	if (tagName.size() == 0) {
-		std::vector<KeyBackedTag> t = wait(getAllRestoreTags(tr));
+	std::vector<KeyBackedTag> tags;
+	if (tagName.empty()) {
+		std::vector<KeyBackedTag> t = co_await getAllRestoreTags(tr);
 		tags = t;
 	} else
 		tags.push_back(makeRestoreTag(tagName.toString()));
 
-	state std::string result;
-	state int i = 0;
+	// If no tags found, return helpful message
+	if (tags.empty()) {
+		co_return "No restores found.\n\n"
+		          "To start a restore:\n"
+		          "  fdbrestore start -r <BACKUP_URL> [-t <TAG>]\n";
+	}
 
-	for (; i < tags.size(); ++i) {
-		UidAndAbortedFlagT u = wait(tags[i].getD(tr));
-		std::string s = wait(RestoreConfig(u.first).getFullStatus(tr));
+	std::string result;
+	for (int i = 0; i < tags.size(); ++i) {
+		UidAndAbortedFlagT u = co_await tags[i].getD(tr);
+		std::string s = co_await RestoreConfig(u.first).getFullStatus(tr);
 		result.append(s);
 		result.append("\n\n");
 	}
 
-	return result;
+	co_return result;
 }
 
-ACTOR Future<ERestoreState> abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
+Future<ERestoreState> abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 
-	state KeyBackedTag tag = makeRestoreTag(tagName.toString());
-	state Optional<UidAndAbortedFlagT> current = wait(tag.get(tr));
+	KeyBackedTag tag = makeRestoreTag(tagName.toString());
+	Optional<UidAndAbortedFlagT> current = co_await tag.get(tr);
 	if (!current.present())
-		return ERestoreState::UNINITIALIZED;
+		co_return ERestoreState::UNINITIALIZED;
 
-	state RestoreConfig restore(current.get().first);
+	RestoreConfig restore(current.get().first);
 
-	state ERestoreState status = wait(restore.stateEnum().getD(tr));
-	state bool runnable = wait(restore.isRunnable(tr));
+	ERestoreState status = co_await restore.stateEnum().getD(tr);
+	bool runnable = co_await restore.isRunnable(tr);
 
 	if (!runnable)
-		return status;
+		co_return status;
 
 	restore.stateEnum().set(tr, ERestoreState::ABORTED);
 
@@ -6306,45 +6604,48 @@ ACTOR Future<ERestoreState> abortRestore(Reference<ReadYourWritesTransaction> tr
 	restore.clearApplyMutationsKeys(tr);
 
 	// Cancel the backup tasks on this tag
-	wait(tag.cancel(tr));
+	co_await tag.cancel(tr);
 
-	wait(unlockDatabase(tr, current.get().first));
-	return ERestoreState::ABORTED;
+	co_await unlockDatabase(tr, current.get().first);
+	co_return ERestoreState::ABORTED;
 }
 
-ACTOR Future<ERestoreState> abortRestore(Database cx, Key tagName) {
-	state Reference<ReadYourWritesTransaction> tr =
-	    Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
+Future<ERestoreState> abortRestore(Database cx, Key tagName) {
+	auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			ERestoreState estate = wait(abortRestore(tr, tagName));
+			ERestoreState estate = co_await abortRestore(tr, tagName);
 			if (estate != ERestoreState::ABORTED) {
-				return estate;
+				co_return estate;
 			}
-			wait(tr->commit());
+			co_await tr->commit();
 			break;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 
-	tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
+	tr = makeReference<ReadYourWritesTransaction>(cx);
 
 	// Commit a dummy transaction before returning success, to ensure the mutation applier has stopped submitting
 	// mutations
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 			tr->addReadConflictRange(singleKeyRange(KeyRef()));
 			tr->addWriteConflictRange(singleKeyRange(KeyRef()));
-			wait(tr->commit());
-			return ERestoreState::ABORTED;
+			co_await tr->commit();
+			co_return ERestoreState::ABORTED;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
@@ -6358,95 +6659,100 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 
 	// Find all files needed for the restore and save them in the RestoreConfig for the task.
 	// Update the total number of files and blocks and change state to starting.
-	ACTOR static Future<Void> _execute(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   Reference<Task> task) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state RestoreConfig restore(task);
-		state Version restoreVersion;
-		state Version beginVersion;
-		state Reference<IBackupContainer> bc;
-		state std::vector<KeyRange> ranges;
-		state bool logsOnly;
-		state bool inconsistentSnapshotOnly;
-
-		loop {
+	static Future<Void> _execute(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             Reference<Task> task) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		RestoreConfig restore(task);
+		Version restoreVersion{ 0 };
+		Version beginVersion{ 0 };
+		Reference<IBackupContainer> bc;
+		std::vector<KeyRange> ranges;
+		bool logsOnly{ false };
+		bool inconsistentSnapshotOnly{ false };
+
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(checkTaskVersion(tr->getDatabase(), task, name, version));
-				wait(store(beginVersion, restore.beginVersion().getD(tr, Snapshot::False, ::invalidVersion)));
+				co_await checkTaskVersion(tr->getDatabase(), task, name, version);
+				beginVersion = co_await restore.beginVersion().getD(tr, Snapshot::False, ::invalidVersion);
 
-				wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)));
-				wait(store(ranges, restore.getRestoreRangesOrDefault(tr)));
-				wait(store(logsOnly, restore.onlyApplyMutationLogs().getD(tr, Snapshot::False, false)));
-				wait(store(inconsistentSnapshotOnly,
-				           restore.inconsistentSnapshotOnly().getD(tr, Snapshot::False, false)));
-				wait(taskBucket->keepRunning(tr, task));
+				restoreVersion = co_await restore.restoreVersion().getOrThrow(tr);
+				ranges = co_await restore.getRestoreRangesOrDefault(tr);
+				logsOnly = co_await restore.onlyApplyMutationLogs().getD(tr, Snapshot::False, false);
+				inconsistentSnapshotOnly = co_await restore.inconsistentSnapshotOnly().getD(tr, Snapshot::False, false);
+				co_await taskBucket->keepRunning(tr, task);
 
-				ERestoreState oldState = wait(restore.stateEnum().getD(tr));
+				ERestoreState oldState = co_await restore.stateEnum().getD(tr);
 				if (oldState != ERestoreState::QUEUED && oldState != ERestoreState::STARTING) {
-					wait(restore.logError(cx,
-					                      restore_error(),
-					                      format("StartFullRestore: Encountered unexpected state(%d)", oldState),
-					                      THIS));
-					return Void();
+					co_await restore.logError(cx,
+					                          restore_error(),
+					                          format("StartFullRestore: Encountered unexpected state(%d)", oldState),
+					                          nullptr);
+					co_return;
 				}
 				restore.stateEnum().set(tr, ERestoreState::STARTING);
 				restore.fileSet().clear(tr);
 				restore.fileBlockCount().clear(tr);
 				restore.fileCount().clear(tr);
-				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
+				Reference<IBackupContainer> _bc = co_await restore.sourceContainer().getOrThrow(tr);
 				bc = getBackupContainerWithProxy(_bc);
 
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				Version destVersion = wait(tr->getReadVersion());
+				Version destVersion = co_await tr->getReadVersion();
 				TraceEvent("FileRestoreVersionUpgrade")
 				    .detail("RestoreVersion", restoreVersion)
 				    .detail("Dest", destVersion);
 				if (destVersion <= restoreVersion) {
 					CODE_PROBE(true, "Forcing restored cluster to higher version");
 					tr->set(minRequiredCommitVersionKey, BinaryWriter::toValue(restoreVersion + 1, Unversioned()));
-					wait(tr->commit());
+					co_await tr->commit();
+					tr->reset();
+					continue;
 				} else {
 					break;
 				}
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		state Version firstConsistentVersion = invalidVersion;
+		Version firstConsistentVersion = invalidVersion;
 		if (beginVersion == invalidVersion) {
 			beginVersion = 0;
 		}
-		state Standalone<VectorRef<KeyRangeRef>> keyRangesFilter;
+		Standalone<VectorRef<KeyRangeRef>> keyRangesFilter;
 		for (auto const& r : ranges) {
 			keyRangesFilter.push_back_deep(keyRangesFilter.arena(), KeyRangeRef(r));
 		}
-		state Optional<RestorableFileSet> restorable =
-		    wait(bc->getRestoreSet(restoreVersion, keyRangesFilter, logsOnly, beginVersion));
+		Optional<RestorableFileSet> restorable =
+		    co_await bc->getRestoreSet(restoreVersion, keyRangesFilter, logsOnly, beginVersion);
 		if (!restorable.present())
 			throw restore_missing_data();
 
 		// Convert the two lists in restorable (logs and ranges) to a single list of RestoreFiles.
 		// Order does not matter, they will be put in order when written to the restoreFileMap below.
-		state std::vector<RestoreConfig::RestoreFile> files;
-		state std::vector<RestoreConfig::RestoreFile> logFiles;
-		state std::vector<RestoreConfig::RestoreFile> rangeFiles;
+		std::vector<RestoreConfig::RestoreFile> files;
+		std::vector<RestoreConfig::RestoreFile> logFiles;
+		std::vector<RestoreConfig::RestoreFile> rangeFiles;
 		if (!logsOnly) {
 			beginVersion = restorable.get().snapshot.beginVersion;
 
@@ -6488,35 +6794,38 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 		Params.firstVersion().set(task, beginVersion);
 
 		tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				restore.firstConsistentVersion().set(tr, firstConsistentVersion);
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		// add log files
-		state std::vector<RestoreConfig::RestoreFile>::iterator logStart = logFiles.begin();
-		state std::vector<RestoreConfig::RestoreFile>::iterator logEnd = logFiles.end();
-		state int txBytes = 0;
+		auto logStart = logFiles.begin();
+		auto logEnd = logFiles.end();
+		int txBytes = 0;
 
 		tr->reset();
 		while (logStart != logEnd) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(taskBucket->keepRunning(tr, task));
+				co_await taskBucket->keepRunning(tr, task);
 
-				state std::vector<RestoreConfig::RestoreFile>::iterator logIt = logStart;
+				auto logIt = logStart;
 
 				txBytes = 0;
-				state int logFileCount = 0;
+				int logFileCount = 0;
 				auto fileSet = restore.logFileSet();
 				// TODO: split files into multiple keys, because files can be in the order of 10k or 100k, which
 				// probably can't fit due to size limit for a value in FDB. as a result, fileSet has everything,
@@ -6525,36 +6834,38 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 					txBytes += fileSet.insert(tr, *logIt);
 					++logFileCount;
 				}
-				wait(tr->commit());
+				co_await tr->commit();
 
 				TraceEvent("FileRestoreLoadedLogFiles")
 				    .detail("RestoreUID", restore.getUid())
 				    .detail("FileCount", logFileCount)
-				    .detail("TransactionBytes", txBytes)
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("TransactionBytes", txBytes);
 
 				logStart = logIt;
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		state std::vector<RestoreConfig::RestoreFile>::iterator rangeStart = rangeFiles.begin();
-		state std::vector<RestoreConfig::RestoreFile>::iterator rangeEnd = rangeFiles.end();
+		auto rangeStart = rangeFiles.begin();
+		auto rangeEnd = rangeFiles.end();
 
 		tr->reset();
 		while (rangeStart != rangeEnd) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(taskBucket->keepRunning(tr, task));
+				co_await taskBucket->keepRunning(tr, task);
 
-				state std::vector<RestoreConfig::RestoreFile>::iterator rangeIt = rangeStart;
+				auto rangeIt = rangeStart;
 
 				txBytes = 0;
-				state int rangeFileCount = 0;
+				int rangeFileCount = 0;
 				auto fileSet = restore.rangeFileSet();
 				// as a result, fileSet has everything, including [beginVersion, endVersion] for each tag
 				for (; rangeIt != rangeEnd && txBytes < 1e6; ++rangeIt) {
@@ -6562,38 +6873,40 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 					// handle the remaining
 					++rangeFileCount;
 				}
-				wait(tr->commit());
+				co_await tr->commit();
 
 				TraceEvent("FileRestoreLoadedRangeFiles")
 				    .detail("RestoreUID", restore.getUid())
 				    .detail("FileCount", rangeFileCount)
-				    .detail("TransactionBytes", txBytes)
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("TransactionBytes", txBytes);
 
 				rangeStart = rangeIt;
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		// add files
-		state std::vector<RestoreConfig::RestoreFile>::iterator start = files.begin();
-		state std::vector<RestoreConfig::RestoreFile>::iterator end = files.end();
+		auto start = files.begin();
+		auto end = files.end();
 
 		tr->reset();
 		while (start != end) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(taskBucket->keepRunning(tr, task));
+				co_await taskBucket->keepRunning(tr, task);
 
-				state std::vector<RestoreConfig::RestoreFile>::iterator it = start;
+				auto it = start;
 
 				txBytes = 0;
-				state int nFileBlocks = 0;
-				state int nFiles = 0;
+				int nFileBlocks = 0;
+				int nFiles = 0;
 				auto fileSet = restore.fileSet();
 				// as a result, fileSet has everything, including [beginVersion, endVersion] for each tag
 				for (; it != end && txBytes < 1e6; ++it) {
@@ -6606,48 +6919,49 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 				restore.fileCount().atomicOp(tr, nFiles, MutationRef::Type::AddValue);
 				restore.fileBlockCount().atomicOp(tr, nFileBlocks, MutationRef::Type::AddValue);
 
-				wait(tr->commit());
+				co_await tr->commit();
 
 				TraceEvent("FileRestoreLoadedFiles")
 				    .detail("RestoreUID", restore.getUid())
 				    .detail("FileCount", nFiles)
 				    .detail("FileBlockCount", nFileBlocks)
-				    .detail("TransactionBytes", txBytes)
-				    .detail("TaskInstance", THIS_ADDR);
+				    .detail("TransactionBytes", txBytes);
 
 				start = it;
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state RestoreConfig restore(task);
-		state bool transformPartitionedLog;
-		state Version restoreVersion;
-		state Version firstVersion = Params.firstVersion().getOrDefault(task, invalidVersion);
-		state bool useRangeFileRestore = false;
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		RestoreConfig restore(task);
+		MutationLogType mutationLogType{ MutationLogType::DEFAULT };
+		Version restoreVersion{ 0 };
+		Version firstVersion = Params.firstVersion().getOrDefault(task, invalidVersion);
+		bool useRangeFileRestore = false;
 
 		if (firstVersion == invalidVersion) {
-			wait(restore.logError(
-			    tr->getDatabase(), restore_missing_data(), "StartFullRestore: The backup had no data.", THIS));
-			std::string tag = wait(restore.tag().getD(tr));
-			wait(success(abortRestore(tr, StringRef(tag))));
-			return Void();
+			co_await restore.logError(
+			    tr->getDatabase(), restore_missing_data(), "StartFullRestore: The backup had no data.", nullptr);
+			std::string tag = co_await restore.tag().getD(tr);
+			co_await abortRestore(tr, StringRef(tag));
+			co_return;
 		}
 
 		restore.stateEnum().set(tr, ERestoreState::RUNNING);
 
 		// Check if using traditional rangefile restore instead of BulkLoad
 		// Default to true (traditional rangefile restore) for backward compatibility
-		state Optional<bool> rangeFileRestore = wait(restore.useRangeFileRestore().get(tr));
+		Optional<bool> rangeFileRestore = co_await restore.useRangeFileRestore().get(tr);
 		useRangeFileRestore = !rangeFileRestore.present() || rangeFileRestore.get();
 
 		// Set applyMutation versions
@@ -6655,21 +6969,21 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 		restore.setApplyEndVersion(tr, firstVersion);
 
 		// Apply range data using either BulkLoad or traditional range file restore
-		wait(store(transformPartitionedLog, restore.transformPartitionedLog().getD(tr, Snapshot::False, false)));
-		wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)));
+		mutationLogType = co_await restore.mutationLogType().getD(tr, Snapshot::False, MutationLogType::DEFAULT);
+		restoreVersion = co_await restore.restoreVersion().getOrThrow(tr);
 
 		if (!useRangeFileRestore) {
 			// Use BulkLoad for range data restoration, then apply logs
-			state Reference<IBackupContainer> bc = wait(restore.sourceContainer().getOrThrow(tr));
+			Reference<IBackupContainer> bc = co_await restore.sourceContainer().getOrThrow(tr);
 
 			// Get the BulkDump job ID - this should have been stored by the backup
 			// when it completed the BulkDump phase
-			state Optional<std::string> bulkDumpJobIdOpt = wait(restore.bulkDumpJobId().get(tr));
-			state std::string bulkDumpJobId = bulkDumpJobIdOpt.present() ? bulkDumpJobIdOpt.get() : "";
+			Optional<std::string> bulkDumpJobIdOpt = co_await restore.bulkDumpJobId().get(tr);
+			std::string bulkDumpJobId = bulkDumpJobIdOpt.present() ? bulkDumpJobIdOpt.get() : "";
 
 			// Save original BulkLoad mode BEFORE creating the task, so it's persisted in the database.
 			// This allows crash recovery to restore the correct mode even if the task restarts.
-			int currentBulkLoadMode = wait(getBulkLoadMode(tr->getDatabase()));
+			int currentBulkLoadMode = co_await getBulkLoadMode(tr->getDatabase());
 			restore.originalBulkLoadMode().set(tr, currentBulkLoadMode);
 
 			TraceEvent("StartFullRestoreUsingBulkLoad")
@@ -6679,80 +6993,79 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 			    .detail("RestoreVersion", restoreVersion);
 
 			// Create a future that BulkLoad will signal when done
-			state Reference<TaskFuture> bulkLoadDone = futureBucket->future(tr);
+			Reference<TaskFuture> bulkLoadDone = futureBucket->future(tr);
 
 			// Add BulkLoad task for range data
-			wait(success(BulkLoadRestoreTaskFunc::addTask(tr,
-			                                              taskBucket,
-			                                              task,
-			                                              restoreVersion,
-			                                              bc->getURL(),
-			                                              bulkDumpJobId,
-			                                              TaskCompletionKey::signal(bulkLoadDone))));
+			co_await BulkLoadRestoreTaskFunc::addTask(tr,
+			                                          taskBucket,
+			                                          task,
+			                                          restoreVersion,
+			                                          bc->getURL(),
+			                                          bulkDumpJobId,
+			                                          TaskCompletionKey::signal(bulkLoadDone));
 
 			// After BulkLoad completes, run RestoreDispatch to apply mutation logs
 			// Set onlyApplyMutationLogs so it only processes logs, not range files
 			restore.onlyApplyMutationLogs().set(tr, true);
 
 			// Add RestoreDispatch task that waits for BulkLoad to complete
-			wait(success(RestoreDispatchTaskFunc::addTask(tr,
-			                                              taskBucket,
-			                                              task,
-			                                              0,
-			                                              "",
-			                                              0,
-			                                              CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE,
-			                                              0,
-			                                              TaskCompletionKey::noSignal(),
-			                                              bulkLoadDone)));
-		} else if (transformPartitionedLog) {
+			co_await RestoreDispatchTaskFunc::addTask(tr,
+			                                          taskBucket,
+			                                          task,
+			                                          0,
+			                                          "",
+			                                          0,
+			                                          CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE,
+			                                          0,
+			                                          TaskCompletionKey::noSignal(),
+			                                          bulkLoadDone);
+		} else if (mutationLogType == MutationLogType::PARTITIONED_LOG) {
 			// Traditional restore with partitioned logs
 			Version endVersion =
 			    std::min(firstVersion + CLIENT_KNOBS->RESTORE_PARTITIONED_BATCH_VERSION_SIZE, restoreVersion);
-			wait(success(RestoreDispatchPartitionedTaskFunc::addTask(
-			    tr, taskBucket, task, firstVersion, firstVersion, endVersion)));
+			co_await RestoreDispatchPartitionedTaskFunc::addTask(
+			    tr, taskBucket, task, firstVersion, firstVersion, endVersion);
 		} else {
 			// Traditional restore with non-partitioned logs
-			wait(success(RestoreDispatchTaskFunc::addTask(
-			    tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE)));
+			co_await RestoreDispatchTaskFunc::addTask(
+			    tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE);
 		}
 
 		// Initialize apply mutations map.
-		state Future<Optional<bool>> logsOnly = restore.onlyApplyMutationLogs().get(tr);
-		wait(success(logsOnly));
+		Future<Optional<bool>> logsOnly = restore.onlyApplyMutationLogs().get(tr);
+		co_await logsOnly;
 		if (logsOnly.get().present() && logsOnly.get().get()) {
 			//  If this is an incremental restore, we need to set the applyMutationsMapPrefix
 			//  to the earliest log version so no mutations are missed
 			Value versionEncoded = BinaryWriter::toValue(Params.firstVersion().get(task), Unversioned());
-			wait(krmSetRange(tr, restore.applyMutationsMapPrefix(), normalKeys, versionEncoded));
+			co_await krmSetRange(tr, restore.applyMutationsMapPrefix(), normalKeys, versionEncoded);
 		}
 
-		wait(taskBucket->finish(tr, task));
-		return Void();
+		co_await taskBucket->finish(tr, task);
+		co_return;
 	}
 
-	ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 UID uid,
-	                                 TaskCompletionKey completionKey,
-	                                 Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
+	static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr,
+	                           Reference<TaskBucket> taskBucket,
+	                           UID uid,
+	                           TaskCompletionKey completionKey,
+	                           Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-		Key doneKey = wait(completionKey.get(tr, taskBucket));
-		state Reference<Task> task(
-		    new Task(StartFullRestoreTaskFunc::name, StartFullRestoreTaskFunc::version, doneKey));
+		Key doneKey = co_await completionKey.get(tr, taskBucket);
+		Reference<Task> task(new Task(StartFullRestoreTaskFunc::name, StartFullRestoreTaskFunc::version, doneKey));
 
-		state RestoreConfig restore(uid);
+		RestoreConfig restore(uid);
 		// Bind the restore config to the new task
-		wait(restore.toTask(tr, task));
+		co_await restore.toTask(tr, task);
 
 		if (!waitFor) {
-			return taskBucket->addTask(tr, task);
+			co_return taskBucket->addTask(tr, task);
 		}
 
-		wait(waitFor->onSetAddTask(tr, taskBucket, task));
-		return "OnSetAddTask"_sr;
+		co_await waitFor->onSetAddTask(tr, taskBucket, task);
+		co_return "OnSetAddTask"_sr;
 	}
 
 	StringRef getName() const override { return name; };
@@ -6786,176 +7099,32 @@ struct LogInfo : public ReferenceCounted<LogInfo> {
 
 class FileBackupAgentImpl {
 public:
-	// Parallel restore
-	ACTOR static Future<Void> parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB = UnlockDB::True) {
-		state ReadYourWritesTransaction tr(cx);
-		state Optional<Value> restoreRequestDoneKeyValue;
-		TraceEvent("FastRestoreToolWaitForRestoreToFinish").detail("DBLock", randomUID);
-		// TODO: register watch first and then check if the key exist
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> _restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey));
-				restoreRequestDoneKeyValue = _restoreRequestDoneKeyValue;
-				// Restore may finish before restoreTool waits on the restore finish event.
-				if (restoreRequestDoneKeyValue.present()) {
-					break;
-				} else {
-					state Future<Void> watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey);
-					wait(tr.commit());
-					wait(watchForRestoreRequestDone);
-					break;
-				}
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-
-		TraceEvent("FastRestoreToolRestoreFinished")
-		    .detail("ClearRestoreRequestDoneKey", restoreRequestDoneKeyValue.present());
-		// Only this agent can clear the restoreRequestDoneKey
-		wait(runRYWTransaction(cx, [](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr->clear(restoreRequestDoneKey);
-			return Void();
-		}));
-
-		if (unlockDB) {
-			TraceEvent("FastRestoreToolRestoreFinished").detail("UnlockDBStart", randomUID);
-			wait(unlockDatabase(cx, randomUID));
-			TraceEvent("FastRestoreToolRestoreFinished").detail("UnlockDBFinish", randomUID);
-		} else {
-			TraceEvent("FastRestoreToolRestoreFinished").detail("DBLeftLockedAfterRestore", randomUID);
-		}
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> submitParallelRestore(Database cx,
-	                                                Key backupTag,
-	                                                Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                                Key bcUrl,
-	                                                Optional<std::string> proxy,
-	                                                Version targetVersion,
-	                                                LockDB lockDB,
-	                                                UID randomUID,
-	                                                Key addPrefix,
-	                                                Key removePrefix) {
-		// Sanity check backup is valid
-		state std::string bcUrlStr = bcUrl.toString();
-		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(bcUrlStr, proxy, {});
-		// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
-		// This is needed for S3 where metadata may not be immediately consistent
-		state BackupDescription desc = wait(bc->describeBackup(false, isBlobstoreUrl(bcUrlStr) ? invalidVersion : 0));
-		wait(desc.resolveVersionTimes(cx));
-
-		if (targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
-			targetVersion = desc.maxRestorableVersion.get();
-			TraceEvent(SevWarn, "FastRestoreSubmitRestoreRequestWithInvalidTargetVersion")
-			    .detail("OverrideTargetVersion", targetVersion);
-		}
-
-		Optional<RestorableFileSet> restoreSet = wait(bc->getRestoreSet(targetVersion));
-
-		if (!restoreSet.present()) {
-			TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
-			    .detail("BackupContainer", bc->getURL())
-			    .detail("TargetVersion", targetVersion);
-			throw restore_invalid_version();
-		}
-
-		TraceEvent("FastRestoreSubmitRestoreRequest")
-		    .detail("BackupDesc", desc.toString())
-		    .detail("TargetVersion", targetVersion);
-
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state int restoreIndex = 0;
-		state int numTries = 0;
-		// lock DB for restore
-		loop {
-			try {
-				if (lockDB) {
-					wait(lockDatabase(cx, randomUID));
-				}
-				wait(checkDatabaseLock(tr, randomUID));
-
-				TraceEvent("FastRestoreToolSubmitRestoreRequests").detail("DBIsLocked", randomUID);
-				break;
-			} catch (Error& e) {
-				TraceEvent(numTries > 50 ? SevError : SevInfo, "FastRestoreToolSubmitRestoreRequestsMayFail")
-				    .error(e)
-				    .detail("Reason", "DB is not properly locked")
-				    .detail("ExpectedLockID", randomUID);
-				numTries++;
-				wait(tr->onError(e));
-			}
-		}
-
-		// set up restore request
-		tr->reset();
-		numTries = 0;
-		loop {
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			try {
-				// Note: we always lock DB here in case DB is modified at the bacupRanges boundary.
-				for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) {
-					auto range = backupRanges[restoreIndex];
-					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
-					// Register the request request in DB, which will be picked up by restore worker leader
-					struct RestoreRequest restoreRequest(restoreIndex,
-					                                     restoreTag,
-					                                     bcUrl,
-					                                     proxy,
-					                                     targetVersion,
-					                                     range,
-					                                     deterministicRandom()->randomUniqueID(),
-					                                     addPrefix,
-					                                     removePrefix);
-					tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest));
-				}
-				tr->set(restoreRequestTriggerKey,
-				        restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size()));
-				wait(tr->commit()); // Trigger restore
-				break;
-			} catch (Error& e) {
-				TraceEvent(numTries > 50 ? SevError : SevInfo, "FastRestoreToolSubmitRestoreRequestsRetry")
-				    .error(e)
-				    .detail("RestoreIndex", restoreIndex);
-				numTries++;
-				wait(tr->onError(e));
-			}
-		}
-		return Void();
-	}
-
 	// This method will return the final status of the backup at tag, and return the URL that was used on the tag
 	// when that status value was read.
-	ACTOR static Future<EBackupState> waitBackup(FileBackupAgent* backupAgent,
-	                                             Database cx,
-	                                             std::string tagName,
-	                                             StopWhenDone stopWhenDone,
-	                                             Reference<IBackupContainer>* pContainer = nullptr,
-	                                             UID* pUID = nullptr) {
-		state std::string backTrace;
-		state KeyBackedTag tag = makeBackupTag(tagName);
-
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	static Future<EBackupState> waitBackup(FileBackupAgent* backupAgent,
+	                                       Database cx,
+	                                       std::string tagName,
+	                                       StopWhenDone stopWhenDone,
+	                                       Reference<IBackupContainer>* pContainer = nullptr,
+	                                       UID* pUID = nullptr) {
+		std::string backTrace;
+		KeyBackedTag tag = makeBackupTag(tagName);
+
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
+			Error err;
 			try {
-				state Optional<UidAndAbortedFlagT> oldUidAndAborted = wait(tag.get(tr));
+				Optional<UidAndAbortedFlagT> oldUidAndAborted = co_await tag.get(tr);
 				if (!oldUidAndAborted.present()) {
-					return EBackupState::STATE_NEVERRAN;
+					co_return EBackupState::STATE_NEVERRAN;
 				}
 
-				state BackupConfig config(oldUidAndAborted.get().first);
-				state EBackupState status =
-				    wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+				BackupConfig config(oldUidAndAborted.get().first);
+				EBackupState status =
+				    co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 
 				// Break, if one of the following is true
 				//  - no longer runnable
@@ -6965,7 +7134,7 @@ class FileBackupAgentImpl {
 
 					if (pContainer != nullptr) {
 						Reference<IBackupContainer> c =
-						    wait(config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info()));
+						    co_await config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info());
 						*pContainer = fileBackup::getBackupContainerWithProxy(c);
 					}
 
@@ -6973,31 +7142,34 @@ class FileBackupAgentImpl {
 						*pUID = oldUidAndAborted.get().first;
 					}
 
-					return status;
+					co_return status;
 				}
 
-				state Future<Void> watchFuture = tr->watch(config.stateEnum().key);
-				wait(tr->commit());
-				wait(watchFuture);
+				Future<Void> watchFuture = tr->watch(config.stateEnum().key);
+				co_await tr->commit();
+				co_await watchFuture;
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
-		}
-	}
-
-	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent,
-	                                       Reference<ReadYourWritesTransaction> tr,
-	                                       Key outContainer,
-	                                       Optional<std::string> proxy,
-	                                       int initialSnapshotIntervalSeconds,
-	                                       int snapshotIntervalSeconds,
-	                                       std::string tagName,
-	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                       StopWhenDone stopWhenDone,
-	                                       UsePartitionedLog partitionedLog,
-	                                       IncrementalBackupOnly incrementalBackupOnly,
-	                                       Optional<std::string> encryptionKeyFileName,
-	                                       int snapshotMode) {
+			co_await tr->onError(err);
+		}
+	}
+
+	static Future<Void> submitBackup(FileBackupAgent* backupAgent,
+	                                 Reference<ReadYourWritesTransaction> tr,
+	                                 Key outContainer,
+	                                 Optional<std::string> proxy,
+	                                 int initialSnapshotIntervalSeconds,
+	                                 int snapshotIntervalSeconds,
+	                                 std::string tagName,
+	                                 Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                                 StopWhenDone stopWhenDone,
+	                                 MutationLogType mutationLogType,
+	                                 IncrementalBackupOnly incrementalBackupOnly,
+	                                 Optional<std::string> encryptionKeyFileName,
+	                                 int encryptionBlockSize,
+	                                 int snapshotMode) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
@@ -7005,15 +7177,17 @@ class FileBackupAgentImpl {
 		TraceEvent(SevInfo, "FBA_SubmitBackup")
 		    .detail("TagName", tagName.c_str())
 		    .detail("StopWhenDone", stopWhenDone)
-		    .detail("UsePartitionedLog", partitionedLog)
-		    .detail("OutContainer", outContainer.toString());
+		    .detail("MutationLogType", mutationLogType)
+		    .detail("OutContainer", outContainer.toString())
+		    .detail("EncryptionKeyFileName", encryptionKeyFileName.present() ? encryptionKeyFileName.get() : "None")
+		    .detail("EncryptionBlockSize", encryptionBlockSize);
 
-		state KeyBackedTag tag = makeBackupTag(tagName);
-		Optional<UidAndAbortedFlagT> uidAndAbortedFlag = wait(tag.get(tr));
+		KeyBackedTag tag = makeBackupTag(tagName);
+		Optional<UidAndAbortedFlagT> uidAndAbortedFlag = co_await tag.get(tr);
 		if (uidAndAbortedFlag.present()) {
-			state BackupConfig prevConfig(uidAndAbortedFlag.get().first);
-			state EBackupState prevBackupStatus =
-			    wait(prevConfig.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+			BackupConfig prevConfig(uidAndAbortedFlag.get().first);
+			EBackupState prevBackupStatus =
+			    co_await prevConfig.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 			if (FileBackupAgent::isRunnable(prevBackupStatus)) {
 				throw backup_duplicate();
 			}
@@ -7022,12 +7196,38 @@ class FileBackupAgentImpl {
 			prevConfig.clear(tr);
 		}
 
-		state BackupConfig config(deterministicRandom()->randomUniqueID());
-		state UID uid = config.getUid();
+		// PARTITIONED_LOG and RANGE_PARTITIONED_LOG backups are mutually exclusive: backup worker
+		// recruitment is determined by the active non-default type, so both cannot run concurrently.
+		if (mutationLogType == MutationLogType::PARTITIONED_LOG) {
+			if (co_await anyRangePartitionedBackupRunning(tr)) {
+				TraceEvent(SevError, "FBA_SubmitBackupMutationLogTypeConflict")
+				    .detail("TagName", tagName)
+				    .detail("RequestedType", mutationLogTypeToString(MutationLogType::PARTITIONED_LOG))
+				    .detail("ConflictingType", mutationLogTypeToString(MutationLogType::RANGE_PARTITIONED_LOG));
+				fprintf(stderr,
+				        "ERROR: Cannot start a backup with mutation-log-type `partitioned-log' while a "
+				        "range-partitioned-log backup is running.\n");
+				throw backup_error();
+			}
+		} else if (mutationLogType == MutationLogType::RANGE_PARTITIONED_LOG) {
+			if (co_await anyPartitionedBackupRunning(tr)) {
+				TraceEvent(SevError, "FBA_SubmitBackupMutationLogTypeConflict")
+				    .detail("TagName", tagName)
+				    .detail("RequestedType", mutationLogTypeToString(MutationLogType::RANGE_PARTITIONED_LOG))
+				    .detail("ConflictingType", mutationLogTypeToString(MutationLogType::PARTITIONED_LOG));
+				fprintf(stderr,
+				        "ERROR: Cannot start a backup with mutation-log-type `range-partitioned-log' while a "
+				        "partitioned-log backup is running.\n");
+				throw backup_error();
+			}
+		}
+
+		BackupConfig config(deterministicRandom()->randomUniqueID());
+		UID uid = config.getUid();
 
 		// This check will ensure that current backupUid is later than the last backup Uid
-		state Standalone<StringRef> nowStr = BackupAgentBase::getCurrentTime();
-		state std::string backupContainer = outContainer.toString();
+		Standalone<StringRef> nowStr = BackupAgentBase::getCurrentTime();
+		std::string backupContainer = outContainer.toString();
 
 		// To be consistent with directory handling behavior since FDB backup was first released, if the container
 		// string describes a local directory then "/backup-<timestamp>" will be added to it.
@@ -7035,12 +7235,12 @@ class FileBackupAgentImpl {
 			backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString());
 		}
 
-		state Reference<IBackupContainer> bc =
-		    IBackupContainer::openContainer(backupContainer, proxy, encryptionKeyFileName);
+		Reference<IBackupContainer> bc =
+		    IBackupContainer::openContainer(backupContainer, proxy, encryptionKeyFileName, encryptionBlockSize);
 		try {
 			// Use longer timeout for blobstore:// URLs in simulation to handle slow S3 mock operations
-			state double createTimeout = (g_network->isSimulated() && isBlobstoreUrl(backupContainer)) ? 300.0 : 30.0;
-			wait(timeoutError(bc->create(), createTimeout));
+			double createTimeout = (g_network->isSimulated() && isBlobstoreUrl(backupContainer)) ? 300.0 : 30.0;
+			co_await timeoutError(bc->create(), createTimeout);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
 				throw;
@@ -7048,7 +7248,7 @@ class FileBackupAgentImpl {
 			throw backup_error();
 		}
 
-		Optional<Value> lastBackupTimestamp = wait(backupAgent->lastBackupTimestamp().get(tr));
+		Optional<Value> lastBackupTimestamp = co_await backupAgent->lastBackupTimestamp().get(tr);
 
 		if ((lastBackupTimestamp.present()) && (lastBackupTimestamp.get() >= nowStr)) {
 			fprintf(stderr,
@@ -7063,7 +7263,7 @@ class FileBackupAgentImpl {
 		}
 
 		backupRangeSet.coalesce(allKeys);
-		state std::vector<KeyRange> normalizedRanges;
+		std::vector<KeyRange> normalizedRanges;
 
 		for (auto& backupRange : backupRangeSet.ranges()) {
 			if (backupRange.value()) {
@@ -7073,10 +7273,10 @@ class FileBackupAgentImpl {
 
 		config.clear(tr);
 
-		state Key destUidValue(BinaryWriter::toValue(uid, Unversioned()));
+		Key destUidValue(BinaryWriter::toValue(uid, Unversioned()));
 		if (normalizedRanges.size() == 1 || isDefaultBackup(normalizedRanges)) {
-			RangeResult existingDestUidValues = wait(
-			    tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
+			RangeResult existingDestUidValues = co_await tr->getRange(
+			    KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY);
 			bool found = false;
 			KeyRangeRef targetRange =
 			    normalizedRanges.size() == 1 ? normalizedRanges[0] : getDefaultBackupSharedRange();
@@ -7116,39 +7316,40 @@ class FileBackupAgentImpl {
 		config.backupRanges().set(tr, normalizedRanges);
 		config.initialSnapshotIntervalSeconds().set(tr, initialSnapshotIntervalSeconds);
 		config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds);
-		config.partitionedLogEnabled().set(tr, partitionedLog);
+		config.mutationLogType().set(tr, mutationLogType);
 		config.incrementalBackupOnly().set(tr, incrementalBackupOnly);
 		config.snapshotMode().set(tr, snapshotMode);
-
-		Key taskKey = wait(fileBackup::StartFullBackupTaskFunc::addTask(
-		    tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal()));
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> submitRestore(FileBackupAgent* backupAgent,
-	                                        Reference<ReadYourWritesTransaction> tr,
-	                                        Key tagName,
-	                                        Key backupURL,
-	                                        Optional<std::string> proxy,
-	                                        Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                        Version restoreVersion,
-	                                        Key addPrefix,
-	                                        Key removePrefix,
-	                                        LockDB lockDB,
-	                                        UnlockDB unlockDB,
-	                                        OnlyApplyMutationLogs onlyApplyMutationLogs,
-	                                        InconsistentSnapshotOnly inconsistentSnapshotOnly,
-	                                        Version beginVersion,
-	                                        UID uid,
-	                                        TransformPartitionedLog transformPartitionedLog,
-	                                        bool useRangeFileRestore = true) {
+		Key taskKey = co_await fileBackup::StartFullBackupTaskFunc::addTask(
+		    tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal());
+
+		co_return;
+	}
+
+	static Future<Void> submitRestore(FileBackupAgent* backupAgent,
+	                                  Reference<ReadYourWritesTransaction> tr,
+	                                  Key tagName,
+	                                  Key backupURL,
+	                                  Optional<std::string> proxy,
+	                                  Standalone<VectorRef<KeyRangeRef>> ranges,
+	                                  Version restoreVersion,
+	                                  Key addPrefix,
+	                                  Key removePrefix,
+	                                  LockDB lockDB,
+	                                  UnlockDB unlockDB,
+	                                  OnlyApplyMutationLogs onlyApplyMutationLogs,
+	                                  InconsistentSnapshotOnly inconsistentSnapshotOnly,
+	                                  Version beginVersion,
+	                                  UID uid,
+	                                  MutationLogType mutationLogType,
+	                                  Optional<std::string> encryptionKeyFileName,
+	                                  int encryptionBlockSize,
+	                                  bool useRangeFileRestore = true) {
 		KeyRangeMap<int> restoreRangeSet;
 		for (auto& range : ranges) {
 			restoreRangeSet.insert(range, 1);
 		}
 		restoreRangeSet.coalesce(allKeys);
-		state std::vector<KeyRange> restoreRanges;
+		std::vector<KeyRange> restoreRanges;
 		for (auto& restoreRange : restoreRangeSet.ranges()) {
 			if (restoreRange.value()) {
 				restoreRanges.push_back(KeyRange(KeyRangeRef(restoreRange.range().begin, restoreRange.range().end)));
@@ -7162,21 +7363,21 @@ class FileBackupAgentImpl {
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 		// Get old restore config for this tag
-		state KeyBackedTag tag = makeRestoreTag(tagName.toString());
-		state Optional<UidAndAbortedFlagT> oldUidAndAborted = wait(tag.get(tr));
+		KeyBackedTag tag = makeRestoreTag(tagName.toString());
+		Optional<UidAndAbortedFlagT> oldUidAndAborted = co_await tag.get(tr);
 		if (oldUidAndAborted.present()) {
 			if (oldUidAndAborted.get().first == uid) {
 				if (oldUidAndAborted.get().second) {
 					throw restore_duplicate_uid();
 				} else {
-					return Void();
+					co_return;
 				}
 			}
 
-			state RestoreConfig oldRestore(oldUidAndAborted.get().first);
+			RestoreConfig oldRestore(oldUidAndAborted.get().first);
 
 			// Make sure old restore for this tag is not runnable
-			bool runnable = wait(oldRestore.isRunnable(tr));
+			bool runnable = co_await oldRestore.isRunnable(tr);
 
 			if (runnable) {
 				throw restore_duplicate_tag();
@@ -7187,27 +7388,28 @@ class FileBackupAgentImpl {
 		}
 
 		if (!onlyApplyMutationLogs) {
-			state int index;
+			int index{ 0 };
 			for (index = 0; index < restoreRanges.size(); index++) {
 				KeyRange restoreIntoRange = KeyRangeRef(restoreRanges[index].begin, restoreRanges[index].end)
 				                                .removePrefix(removePrefix)
 				                                .withPrefix(addPrefix);
-				RangeResult existingRows = wait(tr->getRange(restoreIntoRange, 1));
+				RangeResult existingRows = co_await tr->getRange(restoreIntoRange, 1);
 				// Allow restoring over existing data only when using the validation restore prefix.
 				// validateRestoreLogKeys.begin (\xff\x02/rlog/) is the designated prefix for validation restores.
 				// Using any other prefix with existing data could corrupt user data.
-				if (existingRows.size() > 0 && addPrefix != validateRestoreLogKeys.begin) {
+				if (!existingRows.empty() && addPrefix != validateRestoreLogKeys.begin) {
 					throw restore_destination_not_empty();
 				}
 			}
 		}
 		// Make new restore config
-		state RestoreConfig restore(uid);
+		RestoreConfig restore(uid);
 
 		// Point the tag to the new uid
 		tag.set(tr, { uid, false });
 
-		Reference<IBackupContainer> bc = IBackupContainer::openContainer(backupURL.toString(), proxy, {});
+		Reference<IBackupContainer> bc =
+		    IBackupContainer::openContainer(backupURL.toString(), proxy, encryptionKeyFileName, encryptionBlockSize);
 
 		// Configure the new restore
 		restore.tag().set(tr, tagName.toString());
@@ -7218,7 +7420,7 @@ class FileBackupAgentImpl {
 		restore.inconsistentSnapshotOnly().set(tr, inconsistentSnapshotOnly);
 		restore.beginVersion().set(tr, beginVersion);
 		restore.unlockDBAfterRestore().set(tr, unlockDB);
-		restore.transformPartitionedLog().set(tr, transformPartitionedLog);
+		restore.mutationLogType().set(tr, mutationLogType);
 		restore.useRangeFileRestore().set(tr, useRangeFileRestore);
 		if (BUGGIFY && restoreRanges.size() == 1) {
 			restore.restoreRange().set(tr, restoreRanges[0]);
@@ -7229,78 +7431,80 @@ class FileBackupAgentImpl {
 		}
 		// this also sets restore.add/removePrefix.
 		restore.initApplyMutations(tr, addPrefix, removePrefix, onlyApplyMutationLogs);
-
-		Key taskKey = wait(fileBackup::StartFullRestoreTaskFunc::addTask(
-		    tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal()));
+		Key taskKey = co_await fileBackup::StartFullRestoreTaskFunc::addTask(
+		    tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal());
 
 		if (lockDB)
-			wait(lockDatabase(tr, uid));
+			co_await lockDatabase(tr, uid);
 		else
-			wait(checkDatabaseLock(tr, uid));
+			co_await checkDatabaseLock(tr, uid);
 
-		return Void();
+		co_return;
 	}
 
 	// This method will return the final status of the backup
-	ACTOR static Future<ERestoreState> waitRestore(Database cx, Key tagName, Verbose verbose) {
-		state ERestoreState status;
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	static Future<ERestoreState> waitRestore(Database cx, Key tagName, Verbose verbose) {
+		ERestoreState status;
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state KeyBackedTag tag = makeRestoreTag(tagName.toString());
-				Optional<UidAndAbortedFlagT> current = wait(tag.get(tr));
+				KeyBackedTag tag = makeRestoreTag(tagName.toString());
+				Optional<UidAndAbortedFlagT> current = co_await tag.get(tr);
 				if (!current.present()) {
 					if (verbose)
 						printf("waitRestore: Tag: %s  State: %s\n",
 						       tagName.toString().c_str(),
 						       FileBackupAgent::restoreStateText(ERestoreState::UNINITIALIZED).toString().c_str());
-					return ERestoreState::UNINITIALIZED;
+					co_return ERestoreState::UNINITIALIZED;
 				}
 
-				state RestoreConfig restore(current.get().first);
+				RestoreConfig restore(current.get().first);
 
 				if (verbose) {
-					state std::string details = wait(restore.getProgress(tr));
+					std::string details = co_await restore.getProgress(tr);
 					printf("%s\n", details.c_str());
 				}
 
-				ERestoreState status_ = wait(restore.stateEnum().getD(tr));
+				ERestoreState status_ = co_await restore.stateEnum().getD(tr);
 				status = status_;
-				state bool runnable = wait(restore.isRunnable(tr));
+				bool runnable = co_await restore.isRunnable(tr);
 
 				// State won't change from here
 				if (!runnable)
 					break;
 
 				// Wait for a change
-				state Future<Void> watchFuture = tr->watch(restore.stateEnum().key);
-				wait(tr->commit());
+				Future<Void> watchFuture = tr->watch(restore.stateEnum().key);
+				co_await tr->commit();
 				if (verbose)
-					wait(watchFuture || delay(1));
+					co_await (watchFuture || delay(1));
 				else
-					wait(watchFuture);
+					co_await watchFuture;
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		return status;
+		co_return status;
 	}
 
-	ACTOR static Future<Void> discontinueBackup(FileBackupAgent* backupAgent,
-	                                            Reference<ReadYourWritesTransaction> tr,
-	                                            Key tagName) {
+	static Future<Void> discontinueBackup(FileBackupAgent* backupAgent,
+	                                      Reference<ReadYourWritesTransaction> tr,
+	                                      Key tagName) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-		state KeyBackedTag tag = makeBackupTag(tagName.toString());
-		state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::False, backup_unneeded()));
-		state BackupConfig config(current.first);
-		state EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+		KeyBackedTag tag = makeBackupTag(tagName.toString());
+		UidAndAbortedFlagT current = co_await tag.getOrThrow(tr, Snapshot::False, backup_unneeded());
+		BackupConfig config(current.first);
+		EBackupState status = co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 
 		if (!FileBackupAgent::isRunnable(status)) {
 			throw backup_unneeded();
@@ -7308,7 +7512,7 @@ class FileBackupAgentImpl {
 
 		// If the backup is already restorable then 'mostly' abort it - cancel all tasks via the tag
 		// and clear the mutation logging config and data - but set its state as COMPLETED instead of ABORTED.
-		state Optional<Version> latestRestorableVersion = wait(config.getLatestRestorableVersion(tr));
+		Optional<Version> latestRestorableVersion = co_await config.getLatestRestorableVersion(tr);
 
 		TraceEvent(SevInfo, "FBA_DiscontinueBackup")
 		    .detail("AlreadyRestorable", latestRestorableVersion.present() ? "Yes" : "No")
@@ -7317,21 +7521,21 @@ class FileBackupAgentImpl {
 
 		if (latestRestorableVersion.present()) {
 			// Cancel all backup tasks through tag
-			wait(tag.cancel(tr));
+			co_await tag.cancel(tr);
 
 			tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
 
-			state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
-			wait(success(tr->getReadVersion()));
-			wait(eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
-			     fileBackup::clearBackupStartID(tr, config.getUid()));
+			Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
+			co_await tr->getReadVersion();
+			co_await (eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
+			          fileBackup::clearBackupStartID(tr, config.getUid()));
 
 			config.stateEnum().set(tr, EBackupState::STATE_COMPLETED);
 
-			return Void();
+			co_return;
 		}
 
-		state bool stopWhenDone = wait(config.stopWhenDone().getOrThrow(tr));
+		bool stopWhenDone = co_await config.stopWhenDone().getOrThrow(tr);
 
 		if (stopWhenDone) {
 			throw backup_duplicate();
@@ -7339,21 +7543,21 @@ class FileBackupAgentImpl {
 
 		config.stopWhenDone().set(tr, true);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> abortBackup(FileBackupAgent* backupAgent,
-	                                      Reference<ReadYourWritesTransaction> tr,
-	                                      std::string tagName) {
+	static Future<Void> abortBackup(FileBackupAgent* backupAgent,
+	                                Reference<ReadYourWritesTransaction> tr,
+	                                std::string tagName) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-		state KeyBackedTag tag = makeBackupTag(tagName);
-		state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::False, backup_unneeded()));
+		KeyBackedTag tag = makeBackupTag(tagName);
+		UidAndAbortedFlagT current = co_await tag.getOrThrow(tr, Snapshot::False, backup_unneeded());
 
-		state BackupConfig config(current.first);
-		state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
-		EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+		BackupConfig config(current.first);
+		Key destUidValue = co_await config.destUidValue().getOrThrow(tr);
+		EBackupState status = co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 
 		if (!backupAgent->isRunnable(status)) {
 			throw backup_unneeded();
@@ -7363,29 +7567,39 @@ class FileBackupAgentImpl {
 		    .detail("TagName", tagName.c_str())
 		    .detail("Status", BackupAgentBase::getStateText(status));
 
-		wait(tag.cancel(tr));
+		co_await tag.cancel(tr);
 
-		wait(eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
-		     fileBackup::clearBackupStartID(tr, config.getUid()));
+		co_await (eraseLogData(tr, config.getUidAsKey(), destUidValue) &&
+		          fileBackup::clearBackupStartID(tr, config.getUid()));
 
 		config.stateEnum().set(tr, EBackupState::STATE_ABORTED);
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> checkAndDisableBackupWorkers(Database cx) {
-		state bool running = wait(runRYWTransaction(
-		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return anyPartitionedBackupRunning(tr); }));
+	static Future<Void> checkAndDisableBackupWorkers(Database cx) {
+		bool running = co_await runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return anyPartitionedBackupRunning(tr); });
 		if (!running) {
-			wait(disableBackupWorker(cx));
+			co_await disableBackupWorker(cx);
 		}
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> changePause(FileBackupAgent* backupAgent, Database db, bool pause) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
+	static Future<Void> checkAndDisableRangeBackupWorkers(Database cx) {
+		bool running = co_await runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return anyRangePartitionedBackupRunning(tr); });
+		if (!running) {
+			co_await disableRangeBackupWorker(cx);
+		}
+		co_return;
+	}
+
+	static Future<Void> changePause(FileBackupAgent* backupAgent, Database db, bool pause) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
 
-		loop {
+		while (true) {
+			Error err;
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -7395,14 +7609,15 @@ class FileBackupAgentImpl {
 				backupAgent->taskBucket->changePause(tr, pause);
 				// This is backup workers' pause key.
 				tr->set(backupPausedKey, pause ? "1"_sr : "0"_sr);
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 		TraceEvent("FileBackupAgentChangePaused").detail("Action", pause ? "Paused" : "Resumed");
-		return Void();
+		co_return;
 	}
 
 	struct TimestampedVersion {
@@ -7426,34 +7641,35 @@ class FileBackupAgentImpl {
 
 	// Helper actor for generating status
 	// If f is present, lookup epochs using timekeeper and tr, return TimestampedVersion
-	ACTOR static Future<TimestampedVersion> getTimestampedVersion(Reference<ReadYourWritesTransaction> tr,
-	                                                              Future<Optional<Version>> f) {
-		state TimestampedVersion tv;
-		wait(store(tv.version, f));
+	static Future<TimestampedVersion> getTimestampedVersion(Reference<ReadYourWritesTransaction> tr,
+	                                                        Future<Optional<Version>> f) {
+		TimestampedVersion tv;
+		tv.version = co_await f;
 		if (tv.version.present()) {
-			wait(store(tv.epochs, timeKeeperEpochsFromVersion(tv.version.get(), tr)));
+			tv.epochs = co_await timeKeeperEpochsFromVersion(tv.version.get(), tr);
 		}
-		return tv;
+		co_return tv;
 	}
 
-	ACTOR static Future<std::string> getStatusJSON(FileBackupAgent* backupAgent, Database cx, std::string tagName) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	static Future<std::string> getStatusJSON(FileBackupAgent* backupAgent, Database cx, std::string tagName) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state JsonBuilderObject doc;
+				JsonBuilderObject doc;
 				doc.setKey("SchemaVersion", "1.0.0");
 
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state KeyBackedTag tag = makeBackupTag(tagName);
-				state Optional<UidAndAbortedFlagT> uidAndAbortedFlag;
-				state Optional<Value> paused;
-				state Version recentReadVersion;
+				KeyBackedTag tag = makeBackupTag(tagName);
+				Optional<UidAndAbortedFlagT> uidAndAbortedFlag;
+				Optional<Value> paused;
+				Version recentReadVersion{ 0 };
 
-				wait(store(paused, tr->get(backupAgent->taskBucket->getPauseKey())) &&
-				     store(uidAndAbortedFlag, tag.get(tr)) && store(recentReadVersion, tr->getReadVersion()));
+				co_await (store(paused, tr->get(backupAgent->taskBucket->getPauseKey())) &&
+				          store(uidAndAbortedFlag, tag.get(tr)) && store(recentReadVersion, tr->getReadVersion()));
 
 				doc.setKey("BackupAgentsPaused", paused.present());
 				doc.setKey("Tag", tag.tagName);
@@ -7461,10 +7677,10 @@ class FileBackupAgentImpl {
 				if (uidAndAbortedFlag.present()) {
 					doc.setKey("UID", uidAndAbortedFlag.get().first.toString());
 
-					state BackupConfig config(uidAndAbortedFlag.get().first);
+					BackupConfig config(uidAndAbortedFlag.get().first);
 
-					state EBackupState backupState =
-					    wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+					EBackupState backupState =
+					    co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 					JsonBuilderObject statusDoc;
 					statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState));
 					statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState));
@@ -7472,13 +7688,13 @@ class FileBackupAgentImpl {
 					statusDoc.setKey("Running", BackupAgentBase::isRunnable(backupState));
 					doc.setKey("Status", statusDoc);
 
-					state Future<Void> done = Void();
+					Future<Void> done = Void();
 
 					if (backupState != EBackupState::STATE_NEVERRAN) {
-						state Reference<IBackupContainer> bc;
-						state TimestampedVersion latestRestorable;
+						Reference<IBackupContainer> bc;
+						TimestampedVersion latestRestorable;
 
-						wait(
+						co_await (
 						    store(latestRestorable, getTimestampedVersion(tr, config.getLatestRestorableVersion(tr))) &&
 						    store(bc, config.backupContainer().getOrThrow(tr)));
 						bc = fileBackup::getBackupContainerWithProxy(bc);
@@ -7499,18 +7715,18 @@ class FileBackupAgentImpl {
 
 					if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL ||
 					    backupState == EBackupState::STATE_RUNNING) {
-						state int64_t snapshotInterval;
-						state int64_t logBytesWritten;
-						state int64_t rangeBytesWritten;
-						state bool stopWhenDone;
-						state TimestampedVersion snapshotBegin;
-						state TimestampedVersion snapshotTargetEnd;
-						state TimestampedVersion latestLogEnd;
-						state TimestampedVersion latestSnapshotEnd;
-						state TimestampedVersion snapshotLastDispatch;
-						state Optional<int64_t> snapshotLastDispatchShardsBehind;
-
-						wait(
+						int64_t snapshotInterval{ 0 };
+						int64_t logBytesWritten{ 0 };
+						int64_t rangeBytesWritten{ 0 };
+						bool stopWhenDone{ false };
+						TimestampedVersion snapshotBegin;
+						TimestampedVersion snapshotTargetEnd;
+						TimestampedVersion latestLogEnd;
+						TimestampedVersion latestSnapshotEnd;
+						TimestampedVersion snapshotLastDispatch;
+						Optional<int64_t> snapshotLastDispatchShardsBehind;
+
+						co_await (
 						    store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr)) &&
 						    store(logBytesWritten, config.logBytesWritten().getD(tr)) &&
 						    store(rangeBytesWritten, config.rangeBytesWritten().getD(tr)) &&
@@ -7564,9 +7780,77 @@ class FileBackupAgentImpl {
 						doc.setKey("CurrentSnapshot", snapshot);
 					}
 
+					// Add snapshot mode information
+					Optional<int> snapshotModeOpt = co_await config.snapshotMode().get(tr);
+					int snapshotModeValue = snapshotModeOpt.present() ? snapshotModeOpt.get() : 0;
+					std::string snapshotModeText;
+					switch (snapshotModeValue) {
+					case 0:
+						snapshotModeText = "rangefile";
+						break;
+					case 1:
+						snapshotModeText = "bulkdump";
+						break;
+					case 2:
+						snapshotModeText = "both";
+						break;
+					default:
+						TraceEvent(SevError, "BackupInvalidSnapshotMode")
+						    .detail("BackupUID", config.getUid())
+						    .detail("SnapshotModeValue", snapshotModeValue)
+						    .detail("ValidValues", "0=rangefile, 1=bulkdump, 2=both");
+						throw backup_error();
+					}
+					doc.setKey("SnapshotMode", snapshotModeText);
+
+					// Check if backup is BulkLoad compatible
+					Optional<std::string> bulkDumpJobIdOpt = co_await config.bulkDumpJobId().get(tr);
+					doc.setKey("BulkLoadCompatible", bulkDumpJobIdOpt.present() && !bulkDumpJobIdOpt.get().empty());
+
+					// If using bulkdump mode, get bulkdump progress
+					if (snapshotModeValue == 1 || snapshotModeValue == 2) {
+						Optional<BulkDumpProgress> bulkDumpProgressOpt = co_await getBulkDumpProgress(cx);
+						if (bulkDumpProgressOpt.present()) {
+							BulkDumpProgress bdProgress = bulkDumpProgressOpt.get();
+							JsonBuilderObject bdDoc;
+							bdDoc.setKey("JobId", bdProgress.jobId.toString());
+							bdDoc.setKey("TotalTasks", bdProgress.totalTasks);
+							bdDoc.setKey("CompleteTasks", bdProgress.completeTasks);
+							bdDoc.setKey("RunningTasks", bdProgress.runningTasks);
+							bdDoc.setKey("ErrorTasks", bdProgress.errorTasks);
+							bdDoc.setKey("ProgressPercent", bdProgress.progressPercent());
+							bdDoc.setKey("TotalBytes", bdProgress.totalBytes);
+							bdDoc.setKey("CompletedBytes", bdProgress.completedBytes);
+							bdDoc.setKey("AvgBytesPerSecond", bdProgress.avgBytesPerSecond());
+							if (bdProgress.etaSeconds().present()) {
+								bdDoc.setKey("EstimatedSecondsRemaining", bdProgress.etaSeconds().get());
+							}
+							bdDoc.setKey("ElapsedSeconds", bdProgress.elapsedSeconds);
+
+							// Add stalled tasks if any
+							if (!bdProgress.stalledTasks.empty()) {
+								JsonBuilderArray stalledArray;
+								for (const auto& stalled : bdProgress.stalledTasks) {
+									JsonBuilderObject stalledDoc;
+									stalledDoc.setKey("TaskId", stalled.taskId.toString());
+									stalledDoc.setKey("Range", stalled.range.toString());
+									stalledDoc.setKey("StalledSeconds", stalled.stalledSeconds);
+									stalledDoc.setKey("RestartCount", stalled.restartCount);
+									if (!stalled.lastError.empty()) {
+										stalledDoc.setKey("LastError", stalled.lastError);
+									}
+									stalledArray.push_back(stalledDoc);
+								}
+								bdDoc.setKey("StalledTasks", stalledArray);
+							}
+
+							doc.setKey("BulkDumpProgress", bdDoc);
+						}
+					}
+
 					KeyBackedMap<int64_t, std::pair<std::string, Version>>::RangeResultType errors =
-					    wait(config.lastErrorPerType().getRange(
-					        tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY));
+					    co_await config.lastErrorPerType().getRange(
+					        tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY);
 					JsonBuilderArray errorList;
 					for (auto& e : errors.results) {
 						std::string msg = e.second.first;
@@ -7580,54 +7864,57 @@ class FileBackupAgentImpl {
 					doc.setKey("Errors", errorList);
 				}
 
-				return doc.getJson();
+				co_return doc.getJson();
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<std::string> getStatus(FileBackupAgent* backupAgent,
-	                                           Database cx,
-	                                           ShowErrors showErrors,
-	                                           std::string tagName) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state std::string statusText;
+	static Future<std::string> getStatus(FileBackupAgent* backupAgent,
+	                                     Database cx,
+	                                     ShowErrors showErrors,
+	                                     std::string tagName) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		std::string statusText;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				state KeyBackedTag tag;
-				state BackupConfig config;
-				state EBackupState backupState;
+				KeyBackedTag tag;
+				BackupConfig config;
+				EBackupState backupState;
 
 				statusText = "";
 				tag = makeBackupTag(tagName);
-				state Optional<UidAndAbortedFlagT> uidAndAbortedFlag = wait(tag.get(tr));
-				state Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
+				Optional<UidAndAbortedFlagT> uidAndAbortedFlag = co_await tag.get(tr);
+				Future<Optional<Value>> fPaused = tr->get(backupAgent->taskBucket->getPauseKey());
 				if (uidAndAbortedFlag.present()) {
 					config = BackupConfig(uidAndAbortedFlag.get().first);
 					EBackupState status =
-					    wait(config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN));
+					    co_await config.stateEnum().getD(tr, Snapshot::False, EBackupState::STATE_NEVERRAN);
 					backupState = status;
 				}
 
 				if (!uidAndAbortedFlag.present() || backupState == EBackupState::STATE_NEVERRAN) {
-					statusText += "No previous backups found.\n";
+					statusText += "No previous backups found on tag '" + tagName + "'.\n";
+					statusText += "Use 'fdbbackup tags' to list all backup tags.\n";
 				} else {
-					state std::string backupStatus(BackupAgentBase::getStateText(backupState));
-					state Reference<IBackupContainer> bc;
-					state Optional<Version> latestRestorableVersion;
-					state Version recentReadVersion;
-
-					wait(store(latestRestorableVersion, config.getLatestRestorableVersion(tr)) &&
-					     store(bc, config.backupContainer().getOrThrow(tr)) &&
-					     store(recentReadVersion, tr->getReadVersion()));
+					std::string backupStatus(BackupAgentBase::getStateText(backupState));
+					Reference<IBackupContainer> bc;
+					Optional<Version> latestRestorableVersion;
+					Version recentReadVersion{ 0 };
+
+					co_await (store(latestRestorableVersion, config.getLatestRestorableVersion(tr)) &&
+					          store(bc, config.backupContainer().getOrThrow(tr)) &&
+					          store(recentReadVersion, tr->getReadVersion()));
 					bc = fileBackup::getBackupContainerWithProxy(bc);
 
-					state bool snapshotProgress = false;
+					bool snapshotProgress = false;
 
 					switch (backupState) {
 					case EBackupState::STATE_SUBMITTED:
@@ -7657,7 +7944,7 @@ class FileBackupAgentImpl {
 					statusText += format("BackupURL: %s\n", bc->getURL().c_str());
 
 					// Add enhanced status fields for BulkLoad integration
-					state Optional<int> snapshotModeOpt = wait(config.snapshotMode().get(tr));
+					Optional<int> snapshotModeOpt = co_await config.snapshotMode().get(tr);
 					int snapshotModeValue = snapshotModeOpt.present() ? snapshotModeOpt.get() : 0;
 					std::string snapshotModeText;
 					switch (snapshotModeValue) {
@@ -7671,80 +7958,135 @@ class FileBackupAgentImpl {
 						snapshotModeText = "both";
 						break;
 					default:
-						snapshotModeText = "unknown";
-						break;
+						TraceEvent(SevError, "BackupInvalidSnapshotMode")
+						    .detail("BackupUID", config.getUid())
+						    .detail("SnapshotModeValue", snapshotModeValue)
+						    .detail("ValidValues", "0=rangefile, 1=bulkdump, 2=both");
+						throw backup_error();
 					}
 					statusText += format("Snapshot Mode: %s\n", snapshotModeText.c_str());
 
 					// Check if backup is BulkLoad compatible (has bulkdump_data/)
-					state Optional<std::string> bulkDumpJobIdOpt = wait(config.bulkDumpJobId().get(tr));
+					Optional<std::string> bulkDumpJobIdOpt = co_await config.bulkDumpJobId().get(tr);
 					bool bulkLoadCompatible = bulkDumpJobIdOpt.present() && !bulkDumpJobIdOpt.get().empty();
 					statusText += format("BulkLoad Compatible: %s\n", bulkLoadCompatible ? "yes" : "no");
 
+					bool showBulkDump = (snapshotModeValue == 1 || snapshotModeValue == 2);
+					bool showRangeFile = (snapshotModeValue == 0 || snapshotModeValue == 2);
+
 					if (snapshotProgress) {
-						state int64_t snapshotInterval;
-						state Version snapshotBeginVersion;
-						state Version snapshotTargetEndVersion;
-						state Optional<Version> latestSnapshotEndVersion;
-						state Optional<Version> latestLogEndVersion;
-						state Optional<int64_t> logBytesWritten;
-						state Optional<int64_t> rangeBytesWritten;
-						state Optional<int64_t> latestSnapshotEndVersionTimestamp;
-						state Optional<int64_t> latestLogEndVersionTimestamp;
-						state Optional<int64_t> snapshotBeginVersionTimestamp;
-						state Optional<int64_t> snapshotTargetEndVersionTimestamp;
-						state bool stopWhenDone;
-
-						wait(store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr)) &&
-						     store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr)) &&
-						     store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr)) &&
-						     store(logBytesWritten, config.logBytesWritten().get(tr)) &&
-						     store(rangeBytesWritten, config.rangeBytesWritten().get(tr)) &&
-						     store(latestLogEndVersion, config.latestLogEndVersion().get(tr)) &&
-						     store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
-						     store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)));
-
-						wait(store(latestSnapshotEndVersionTimestamp,
-						           getTimestampFromVersion(latestSnapshotEndVersion, tr)) &&
-						     store(latestLogEndVersionTimestamp, getTimestampFromVersion(latestLogEndVersion, tr)) &&
-						     store(snapshotBeginVersionTimestamp,
-						           timeKeeperEpochsFromVersion(snapshotBeginVersion, tr)) &&
-						     store(snapshotTargetEndVersionTimestamp,
-						           timeKeeperEpochsFromVersion(snapshotTargetEndVersion, tr)));
-
-						statusText += format("Snapshot interval is %lld seconds.  ", snapshotInterval);
-						if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL)
-							statusText += format("Current snapshot progress target is %3.2f%% (>100%% means the "
-							                     "snapshot is supposed to be done)\n",
-							                     100.0 * (recentReadVersion - snapshotBeginVersion) /
-							                         (snapshotTargetEndVersion - snapshotBeginVersion));
-						else
-							statusText += "The initial snapshot is still running.\n";
-
-						statusText += format("\nDetails:\n LogBytes written - %lld\n RangeBytes written - %lld\n "
-						                     "Last complete log version and timestamp        - %s, %s\n "
-						                     "Last complete snapshot version and timestamp   - %s, %s\n "
-						                     "Current Snapshot start version and timestamp   - %s, %s\n "
-						                     "Expected snapshot end version and timestamp    - %s, %s\n "
-						                     "Backup supposed to stop at next snapshot completion - %s\n",
-						                     logBytesWritten.orDefault(0),
-						                     rangeBytesWritten.orDefault(0),
+						int64_t snapshotInterval{ 0 };
+						Version snapshotBeginVersion{ 0 };
+						Version snapshotTargetEndVersion{ 0 };
+						Optional<Version> latestSnapshotEndVersion;
+						Optional<Version> latestLogEndVersion;
+						Optional<int64_t> logBytesWritten;
+						Optional<int64_t> rangeBytesWritten;
+						Optional<int64_t> latestSnapshotEndVersionTimestamp;
+						Optional<int64_t> latestLogEndVersionTimestamp;
+						Optional<int64_t> snapshotBeginVersionTimestamp;
+						Optional<int64_t> snapshotTargetEndVersionTimestamp;
+						bool stopWhenDone{ false };
+
+						co_await (store(snapshotBeginVersion, config.snapshotBeginVersion().getOrThrow(tr)) &&
+						          store(snapshotTargetEndVersion, config.snapshotTargetEndVersion().getOrThrow(tr)) &&
+						          store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr)) &&
+						          store(logBytesWritten, config.logBytesWritten().get(tr)) &&
+						          store(rangeBytesWritten, config.rangeBytesWritten().get(tr)) &&
+						          store(latestLogEndVersion, config.latestLogEndVersion().get(tr)) &&
+						          store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) &&
+						          store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)));
+
+						co_await (
+						    store(latestSnapshotEndVersionTimestamp,
+						          getTimestampFromVersion(latestSnapshotEndVersion, tr)) &&
+						    store(latestLogEndVersionTimestamp, getTimestampFromVersion(latestLogEndVersion, tr)) &&
+						    store(snapshotBeginVersionTimestamp,
+						          timeKeeperEpochsFromVersion(snapshotBeginVersion, tr)) &&
+						    store(snapshotTargetEndVersionTimestamp,
+						          timeKeeperEpochsFromVersion(snapshotTargetEndVersion, tr)));
+
+						if (showBulkDump) {
+							Optional<BulkDumpProgress> bulkDumpProgressOpt = co_await getBulkDumpProgress(cx);
+							statusText += "\nBulkDump Snapshot:\n";
+							if (bulkDumpProgressOpt.present()) {
+								BulkDumpProgress bdProgress = bulkDumpProgressOpt.get();
+								statusText += format(" Tasks: %d/%d complete (%.1f%%)\n",
+								                     bdProgress.completeTasks,
+								                     bdProgress.totalTasks,
+								                     bdProgress.progressPercent());
+								statusText += format(
+								    " Bytes: %s\n",
+								    formatBytesProgress(bdProgress.completedBytes, bdProgress.totalBytes).c_str());
+
+								double throughput = bdProgress.avgBytesPerSecond();
+								if (throughput > 0) {
+									statusText += format(" Throughput: %.1f MB/s\n", throughput / 1048576.0);
+								}
+
+								if (bdProgress.elapsedSeconds > 0) {
+									statusText +=
+									    format(" Elapsed: %s\n",
+									           formatDurationHumanReadable((int)bdProgress.elapsedSeconds).c_str());
+								}
+
+								if (bdProgress.etaSeconds().present()) {
+									statusText +=
+									    format(" ETA: %s\n",
+									           formatDurationHumanReadable((int)bdProgress.etaSeconds().get()).c_str());
+								}
+
+								if (!bdProgress.stalledTasks.empty()) {
+									statusText += format("\nWARNING: %zu stalled tasks (no progress > 60s):\n",
+									                     bdProgress.stalledTasks.size());
+									for (const auto& stalled : bdProgress.stalledTasks) {
+										statusText += format(" Task %s: %s, stalled %.0fs, %d restarts\n",
+										                     stalled.taskId.shortString().c_str(),
+										                     stalled.range.toString().c_str(),
+										                     stalled.stalledSeconds,
+										                     stalled.restartCount);
+										if (!stalled.lastError.empty()) {
+											statusText +=
+											    format("           Last error: %s\n", stalled.lastError.c_str());
+										}
+									}
+								}
+							} else {
+								statusText += " Status: pending\n";
+							}
+						}
+
+						if (showRangeFile) {
+							statusText += "\nRangefile Snapshot:\n";
+							statusText += format(" Bytes written: %s\n",
+							                     formatBytesHumanReadable(rangeBytesWritten.orDefault(0)).c_str());
+
+							if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL) {
+								double pct = 100.0 * (recentReadVersion - snapshotBeginVersion) /
+								             (snapshotTargetEndVersion - snapshotBeginVersion);
+								statusText += format(" Progress: %.2f%%\n", pct);
+							} else {
+								statusText += " Status: Initial snapshot still running\n";
+							}
+							statusText +=
+							    format(" Started: %s\n", timeStampToString(snapshotBeginVersionTimestamp).c_str());
+						}
+
+						statusText += format("\nMutation Logs:\n");
+						statusText += format(" Bytes written: %s\n",
+						                     formatBytesHumanReadable(logBytesWritten.orDefault(0)).c_str());
+						statusText += format(" Last complete version: %s (%s)\n",
 						                     versionToString(latestLogEndVersion).c_str(),
-						                     timeStampToString(latestLogEndVersionTimestamp).c_str(),
-						                     versionToString(latestSnapshotEndVersion).c_str(),
-						                     timeStampToString(latestSnapshotEndVersionTimestamp).c_str(),
-						                     versionToString(snapshotBeginVersion).c_str(),
-						                     timeStampToString(snapshotBeginVersionTimestamp).c_str(),
-						                     versionToString(snapshotTargetEndVersion).c_str(),
-						                     timeStampToString(snapshotTargetEndVersionTimestamp).c_str(),
-						                     boolToYesOrNo(stopWhenDone).c_str());
+						                     timeStampToString(latestLogEndVersionTimestamp).c_str());
+
+						statusText += format("\nSnapshot interval is %lld seconds.\n", snapshotInterval);
 					}
 
 					// Append the errors, if requested
 					if (showErrors) {
 						KeyBackedMap<int64_t, std::pair<std::string, Version>>::RangeResultType errors =
-						    wait(config.lastErrorPerType().getRange(
-						        tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY));
+						    co_await config.lastErrorPerType().getRange(
+						        tr, 0, std::numeric_limits<int>::max(), CLIENT_KNOBS->TOO_MANY);
 						std::string recentErrors;
 						std::string pastErrors;
 
@@ -7781,31 +8123,32 @@ class FileBackupAgentImpl {
 					}
 				}
 
-				Optional<Value> paused = wait(fPaused);
+				Optional<Value> paused = co_await fPaused;
 				if (paused.present()) {
 					statusText += format("\nAll backup agents have been paused.\n");
 				}
 
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		return statusText;
+		co_return statusText;
 	}
 
-	ACTOR static Future<Optional<Version>> getLastRestorable(FileBackupAgent* backupAgent,
-	                                                         Reference<ReadYourWritesTransaction> tr,
-	                                                         Key tagName,
-	                                                         Snapshot snapshot) {
+	static Future<Optional<Version>> getLastRestorable(FileBackupAgent* backupAgent,
+	                                                   Reference<ReadYourWritesTransaction> tr,
+	                                                   Key tagName,
+	                                                   Snapshot snapshot) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		state Optional<Value> version = wait(tr->get(backupAgent->lastRestorable.pack(tagName), snapshot));
+		Optional<Value> version = co_await tr->get(backupAgent->lastRestorable.pack(tagName), snapshot);
 
-		return (version.present())
-		           ? Optional<Version>(BinaryReader::fromStringRef<Version>(version.get(), Unversioned()))
-		           : Optional<Version>();
+		co_return (version.present())
+		    ? Optional<Version>(BinaryReader::fromStringRef<Version>(version.get(), Unversioned()))
+		    : Optional<Version>();
 	}
 
 	static StringRef read(StringRef& data, int bytes) {
@@ -7836,37 +8179,38 @@ class FileBackupAgentImpl {
 	//                             When set to true, gives an inconsistent snapshot, thus not recommended
 	//   beginVersions: restore's begin version for each range
 	//   randomUid: the UID for lock the database
-	ACTOR static Future<Version> restore(FileBackupAgent* backupAgent,
-	                                     Database cx,
-	                                     Optional<Database> cxOrig,
-	                                     Key tagName,
-	                                     Key url,
-	                                     Optional<std::string> proxy,
-	                                     Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                     Standalone<VectorRef<Version>> beginVersions,
-	                                     WaitForComplete waitForComplete,
-	                                     Version targetVersion,
-	                                     Verbose verbose,
-	                                     Key addPrefix,
-	                                     Key removePrefix,
-	                                     LockDB lockDB,
-	                                     UnlockDB unlockDB,
-	                                     OnlyApplyMutationLogs onlyApplyMutationLogs,
-	                                     InconsistentSnapshotOnly inconsistentSnapshotOnly,
-	                                     Optional<std::string> encryptionKeyFileName,
-	                                     UID randomUid,
-	                                     bool useRangeFileRestore = true) {
+	static Future<Version> restore(FileBackupAgent* backupAgent,
+	                               Database cx,
+	                               Optional<Database> cxOrig,
+	                               Key tagName,
+	                               Key url,
+	                               Optional<std::string> proxy,
+	                               Standalone<VectorRef<KeyRangeRef>> ranges,
+	                               Standalone<VectorRef<Version>> beginVersions,
+	                               WaitForComplete waitForComplete,
+	                               Version targetVersion,
+	                               Verbose verbose,
+	                               Key addPrefix,
+	                               Key removePrefix,
+	                               LockDB lockDB,
+	                               UnlockDB unlockDB,
+	                               OnlyApplyMutationLogs onlyApplyMutationLogs,
+	                               InconsistentSnapshotOnly inconsistentSnapshotOnly,
+	                               Optional<std::string> encryptionKeyFileName,
+	                               UID randomUid,
+	                               bool useRangeFileRestore = true) {
 		// The restore command line tool won't allow ranges to be empty, but correctness workloads somehow might.
 		if (ranges.empty()) {
 			throw restore_error();
 		}
 
-		state std::string urlStr = url.toString();
-		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(urlStr, proxy, encryptionKeyFileName);
+		std::string urlStr = url.toString();
+		Reference<IBackupContainer> bc =
+		    IBackupContainer::openContainer(urlStr, proxy, /*encryptionKeyFileName=*/{}, /*encryptionBlockSize=*/0);
 
 		// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
 		// This is needed for S3 where metadata may not be immediately consistent
-		state BackupDescription desc = wait(bc->describeBackup(true, isBlobstoreUrl(urlStr) ? invalidVersion : 0));
+		BackupDescription desc = co_await bc->describeBackup(true, isBlobstoreUrl(urlStr) ? invalidVersion : 0);
 
 		if (desc.fileLevelEncryption && !encryptionKeyFileName.present()) {
 			fprintf(stderr, "ERROR: Backup is encrypted, please provide the encryption key file path.\n");
@@ -7876,8 +8220,15 @@ class FileBackupAgentImpl {
 			throw restore_error();
 		}
 
+		if (desc.fileLevelEncryption) {
+			bc = IBackupContainer::openContainer(urlStr, proxy, encryptionKeyFileName, desc.encryptionBlockSize);
+			// openContainer may return a cached container that has blockSize=0 (seeded earlier without blockSize).
+			// Set it explicitly to ensure the correct value is used.
+			bc->setEncryptionBlockSize(desc.encryptionBlockSize);
+		}
+
 		if (cxOrig.present()) {
-			wait(desc.resolveVersionTimes(cxOrig.get()));
+			co_await desc.resolveVersionTimes(cxOrig.get());
 		}
 
 		printf("Backup Description\n%s", desc.toString().c_str());
@@ -7888,12 +8239,12 @@ class FileBackupAgentImpl {
 			targetVersion = desc.contiguousLogEnd.get() - 1;
 		}
 
-		state Version beginVersion = invalidVersion; // min begin version for all ranges
+		Version beginVersion = invalidVersion; // min begin version for all ranges
 		if (!beginVersions.empty()) {
 			beginVersion = *std::min_element(beginVersions.begin(), beginVersions.end());
 		}
 		Optional<RestorableFileSet> restoreSet =
-		    wait(bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion));
+		    co_await bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion);
 
 		if (!restoreSet.present()) {
 			TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
@@ -7907,68 +8258,71 @@ class FileBackupAgentImpl {
 			printf("Restoring backup to version: %lld\n", (long long)targetVersion);
 		}
 
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		loop {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(submitRestore(backupAgent,
-				                   tr,
-				                   tagName,
-				                   url,
-				                   proxy,
-				                   ranges,
-				                   targetVersion,
-				                   addPrefix,
-				                   removePrefix,
-				                   lockDB,
-				                   unlockDB,
-				                   onlyApplyMutationLogs,
-				                   inconsistentSnapshotOnly,
-				                   beginVersion,
-				                   randomUid,
-				                   TransformPartitionedLog(desc.partitioned),
-				                   useRangeFileRestore));
-				wait(tr->commit());
+				co_await submitRestore(backupAgent,
+				                       tr,
+				                       tagName,
+				                       url,
+				                       proxy,
+				                       ranges,
+				                       targetVersion,
+				                       addPrefix,
+				                       removePrefix,
+				                       lockDB,
+				                       unlockDB,
+				                       onlyApplyMutationLogs,
+				                       inconsistentSnapshotOnly,
+				                       beginVersion,
+				                       randomUid,
+				                       desc.mutationLogType,
+				                       encryptionKeyFileName,
+				                       desc.encryptionBlockSize,
+				                       useRangeFileRestore);
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_restore_duplicate_tag) {
-					throw;
-				}
-				wait(tr->onError(e));
+				err = e;
 			}
+			if (err.code() == error_code_restore_duplicate_tag) {
+				throw err;
+			}
+			co_await tr->onError(err);
 		}
 
 		if (waitForComplete) {
-			ERestoreState finalState = wait(waitRestore(cx, tagName, verbose));
+			ERestoreState finalState = co_await waitRestore(cx, tagName, verbose);
 			if (finalState != ERestoreState::COMPLETED)
 				throw restore_error();
 		}
 
-		return targetVersion;
+		co_return targetVersion;
 	}
 
 	// used for correctness only, locks the database before discontinuing the backup and that same lock is then used
 	// while doing the restore. the tagname of the backup must be the same as the restore.
-	ACTOR static Future<Version> atomicRestore(FileBackupAgent* backupAgent,
-	                                           Database cx,
-	                                           Key tagName,
-	                                           Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                           Key addPrefix,
-	                                           Key removePrefix,
-	                                           UsePartitionedLog fastRestore) {
-		state Reference<ReadYourWritesTransaction> ryw_tr =
-		    Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-		state BackupConfig backupConfig;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
-		loop {
+	static Future<Version> atomicRestore(FileBackupAgent* backupAgent,
+	                                     Database cx,
+	                                     Key tagName,
+	                                     Standalone<VectorRef<KeyRangeRef>> ranges,
+	                                     Key addPrefix,
+	                                     Key removePrefix) {
+		auto ryw_tr = makeReference<ReadYourWritesTransaction>(cx);
+		BackupConfig backupConfig;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
+		while (true) {
+			Error err;
 			try {
 				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				state KeyBackedTag tag = makeBackupTag(tagName.toString());
-				UidAndAbortedFlagT uidFlag = wait(tag.getOrThrow(ryw_tr));
+				KeyBackedTag tag = makeBackupTag(tagName.toString());
+				UidAndAbortedFlagT uidFlag = co_await tag.getOrThrow(ryw_tr);
 				backupConfig = BackupConfig(uidFlag.first);
-				state EBackupState status = wait(backupConfig.stateEnum().getOrThrow(ryw_tr));
+				EBackupState status = co_await backupConfig.stateEnum().getOrThrow(ryw_tr);
 
 				if (status != EBackupState::STATE_RUNNING_DIFFERENTIAL) {
 					throw backup_duplicate();
@@ -7976,66 +8330,74 @@ class FileBackupAgentImpl {
 
 				break;
 			} catch (Error& e) {
-				wait(ryw_tr->onError(e));
+				err = e;
 			}
+			co_await ryw_tr->onError(err);
 		}
 
 		// Lock src, record commit version
-		state Transaction tr(cx);
-		state Version commitVersion;
-		state UID randomUid = deterministicRandom()->randomUniqueID();
-		loop {
+		Transaction tr(cx);
+		Version commitVersion{ 0 };
+		UID randomUid = deterministicRandom()->randomUniqueID();
+		while (true) {
+			Error err;
 			try {
 				// We must get a commit version so add a conflict range that won't likely cause conflicts
 				// but will ensure that the transaction is actually submitted.
 				tr.addWriteConflictRange(backupConfig.snapshotRangeDispatchMap().subspace);
-				wait(lockDatabase(&tr, randomUid));
-				wait(tr.commit());
+				co_await lockDatabase(&tr, randomUid);
+				co_await tr.commit();
 				commitVersion = tr.getCommittedVersion();
 				TraceEvent("AS_Locked").detail("CommitVer", commitVersion);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		ryw_tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
-				Optional<Version> restoreVersion = wait(backupConfig.getLatestRestorableVersion(ryw_tr));
+				Optional<Version> restoreVersion = co_await backupConfig.getLatestRestorableVersion(ryw_tr);
 				if (restoreVersion.present() && restoreVersion.get() >= commitVersion) {
 					TraceEvent("AS_RestoreVersion").detail("RestoreVer", restoreVersion.get());
 					break;
 				} else {
 					ryw_tr->reset();
-					wait(delay(0.2));
+					co_await delay(0.2);
+					continue;
 				}
 			} catch (Error& e) {
-				wait(ryw_tr->onError(e));
+				err = e;
 			}
+			co_await ryw_tr->onError(err);
 		}
 
 		ryw_tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(discontinueBackup(backupAgent, ryw_tr, tagName));
-				wait(ryw_tr->commit());
+				co_await discontinueBackup(backupAgent, ryw_tr, tagName);
+				co_await ryw_tr->commit();
 				TraceEvent("AS_DiscontinuedBackup").log();
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate) {
-					break;
-				}
-				wait(ryw_tr->onError(e));
+				err = e;
 			}
+			if (err.code() == error_code_backup_unneeded || err.code() == error_code_backup_duplicate) {
+				break;
+			}
+			co_await ryw_tr->onError(err);
 		}
 
-		wait(success(waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::True)));
+		co_await waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::True);
 		TraceEvent("AS_BackupStopped").log();
 
 		ryw_tr->reset();
-		loop {
-
+		while (true) {
+			Error err;
 			try {
 				ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -8043,151 +8405,89 @@ class FileBackupAgentImpl {
 					ryw_tr->addReadConflictRange(range);
 					ryw_tr->clear(range);
 				}
-				wait(ryw_tr->commit());
+				co_await ryw_tr->commit();
 				TraceEvent("AS_ClearedRange").log();
 				break;
 			} catch (Error& e) {
-				wait(ryw_tr->onError(e));
+				err = e;
 			}
+			co_await ryw_tr->onError(err);
 		}
 
-		state Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx.getReference()));
+		Reference<IBackupContainer> bc = co_await backupConfig.backupContainer().getOrThrow(cx.getReference());
 
 		bc = fileBackup::getBackupContainerWithProxy(bc);
 
-		if (fastRestore) {
-			TraceEvent("AtomicParallelRestoreStartRestore").log();
-			Version targetVersion = ::invalidVersion;
-			wait(submitParallelRestore(cx,
-			                           tagName,
-			                           ranges,
-			                           KeyRef(bc->getURL()),
-			                           bc->getProxy(),
-			                           targetVersion,
-			                           LockDB::True,
-			                           randomUid,
-			                           addPrefix,
-			                           removePrefix));
-			state bool hasPrefix = (addPrefix.size() > 0 || removePrefix.size() > 0);
-			TraceEvent("AtomicParallelRestoreWaitForRestoreFinish").detail("HasPrefix", hasPrefix);
-			wait(parallelRestoreFinish(cx, randomUid, UnlockDB{ !hasPrefix }));
-			// If addPrefix or removePrefix set, we want to transform the effect by copying data
-			if (hasPrefix) {
-				wait(transformRestoredDatabase(cx, ranges, addPrefix, removePrefix));
-				wait(unlockDatabase(cx, randomUid));
-			}
-			return -1;
-		} else {
-			TraceEvent("AS_StartRestore").log();
-			state Standalone<VectorRef<KeyRangeRef>> restoreRange;
-			state Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
-			for (auto r : ranges) {
-				restoreRange.push_back_deep(restoreRange.arena(), r);
-			}
-			if (!systemRestoreRange.empty()) {
-				// restore system keys
-				wait(success(restore(backupAgent,
-				                     cx,
-				                     cx,
-				                     "system_restore"_sr,
-				                     KeyRef(bc->getURL()),
-				                     bc->getProxy(),
-				                     systemRestoreRange,
-				                     {},
-				                     WaitForComplete::True,
-				                     ::invalidVersion,
-				                     Verbose::True,
-				                     addPrefix,
-				                     removePrefix,
-				                     LockDB::True,
-				                     UnlockDB::False,
-				                     OnlyApplyMutationLogs::False,
-				                     InconsistentSnapshotOnly::False,
-				                     {},
-				                     randomUid)));
-				state Reference<ReadYourWritesTransaction> rywTransaction =
-				    Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-				// clear old restore config associated with system keys
-				loop {
-					try {
-						rywTransaction->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-						rywTransaction->setOption(FDBTransactionOptions::LOCK_AWARE);
-						state RestoreConfig oldRestore(randomUid);
-						oldRestore.clear(rywTransaction);
-						wait(rywTransaction->commit());
-						break;
-					} catch (Error& e) {
-						wait(rywTransaction->onError(e));
-					}
+		TraceEvent("AS_StartRestore").log();
+		Standalone<VectorRef<KeyRangeRef>> restoreRange;
+		Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
+		for (auto r : ranges) {
+			restoreRange.push_back_deep(restoreRange.arena(), r);
+		}
+		if (!systemRestoreRange.empty()) {
+			// restore system keys
+			co_await restore(backupAgent,
+			                 cx,
+			                 cx,
+			                 "system_restore"_sr,
+			                 KeyRef(bc->getURL()),
+			                 bc->getProxy(),
+			                 systemRestoreRange,
+			                 {},
+			                 WaitForComplete::True,
+			                 ::invalidVersion,
+			                 Verbose::True,
+			                 addPrefix,
+			                 removePrefix,
+			                 LockDB::True,
+			                 UnlockDB::False,
+			                 OnlyApplyMutationLogs::False,
+			                 InconsistentSnapshotOnly::False,
+			                 {},
+			                 randomUid);
+			auto rywTransaction = makeReference<ReadYourWritesTransaction>(cx);
+			// clear old restore config associated with system keys
+			while (true) {
+				Error err;
+				try {
+					rywTransaction->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					rywTransaction->setOption(FDBTransactionOptions::LOCK_AWARE);
+					RestoreConfig oldRestore(randomUid);
+					oldRestore.clear(rywTransaction);
+					co_await rywTransaction->commit();
+					break;
+				} catch (Error& e) {
+					err = e;
 				}
+				co_await rywTransaction->onError(err);
 			}
-			// restore user data
-			state Version ver = wait(restore(backupAgent,
-			                                 cx,
-			                                 cx,
-			                                 tagName,
-			                                 KeyRef(bc->getURL()),
-			                                 bc->getProxy(),
-			                                 restoreRange,
-			                                 {},
-			                                 WaitForComplete::True,
-			                                 ::invalidVersion,
-			                                 Verbose::True,
-			                                 addPrefix,
-			                                 removePrefix,
-			                                 LockDB::True,
-			                                 UnlockDB::True,
-			                                 OnlyApplyMutationLogs::False,
-			                                 InconsistentSnapshotOnly::False,
-			                                 {},
-			                                 randomUid));
-			return ver;
-		}
-	}
-
-	// Similar to atomicRestore, only used in simulation test.
-	// locks the database before discontinuing the backup and that same lock is then used while doing the restore.
-	// the tagname of the backup must be the same as the restore.
-	static Future<Void> atomicParallelRestore(FileBackupAgent* backupAgent,
-	                                          Database cx,
-	                                          Key tagName,
-	                                          Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                          Key addPrefix,
-	                                          Key removePrefix) {
-		return success(
-		    atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::True));
+		}
+		// restore user data
+		Version ver = co_await restore(backupAgent,
+		                               cx,
+		                               cx,
+		                               tagName,
+		                               KeyRef(bc->getURL()),
+		                               bc->getProxy(),
+		                               restoreRange,
+		                               {},
+		                               WaitForComplete::True,
+		                               ::invalidVersion,
+		                               Verbose::True,
+		                               addPrefix,
+		                               removePrefix,
+		                               LockDB::True,
+		                               UnlockDB::True,
+		                               OnlyApplyMutationLogs::False,
+		                               InconsistentSnapshotOnly::False,
+		                               {},
+		                               randomUid);
+		co_return ver;
 	}
 };
 
 const int FileBackupAgent::dataFooterSize = 20;
 
-// Return if parallel restore has finished
-Future<Void> FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB) {
-	return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID, unlockDB);
-}
-
-Future<Void> FileBackupAgent::submitParallelRestore(Database cx,
-                                                    Key backupTag,
-                                                    Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                                    Key bcUrl,
-                                                    Optional<std::string> proxy,
-                                                    Version targetVersion,
-                                                    LockDB lockDB,
-                                                    UID randomUID,
-                                                    Key addPrefix,
-                                                    Key removePrefix) {
-	return FileBackupAgentImpl::submitParallelRestore(
-	    cx, backupTag, backupRanges, bcUrl, proxy, targetVersion, lockDB, randomUID, addPrefix, removePrefix);
-}
-
-Future<Void> FileBackupAgent::atomicParallelRestore(Database cx,
-                                                    Key tagName,
-                                                    Standalone<VectorRef<KeyRangeRef>> ranges,
-                                                    Key addPrefix,
-                                                    Key removePrefix) {
-	return FileBackupAgentImpl::atomicParallelRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
-}
-
 Future<Version> FileBackupAgent::restore(Database cx,
                                          Optional<Database> cxOrig,
                                          Key tagName,
@@ -8339,8 +8639,7 @@ Future<Version> FileBackupAgent::atomicRestore(Database cx,
                                                Standalone<VectorRef<KeyRangeRef>> ranges,
                                                Key addPrefix,
                                                Key removePrefix) {
-	return FileBackupAgentImpl::atomicRestore(
-	    this, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::False);
+	return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix);
 }
 
 Future<ERestoreState> FileBackupAgent::abortRestore(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@@ -8367,9 +8666,10 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
                                            std::string const& tagName,
                                            Standalone<VectorRef<KeyRangeRef>> backupRanges,
                                            StopWhenDone stopWhenDone,
-                                           UsePartitionedLog partitionedLog,
+                                           MutationLogType mutationLogType,
                                            IncrementalBackupOnly incrementalBackupOnly,
                                            Optional<std::string> const& encryptionKeyFileName,
+                                           int encryptionBlockSize,
                                            int snapshotMode) {
 	return FileBackupAgentImpl::submitBackup(this,
 	                                         tr,
@@ -8380,9 +8680,10 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
 	                                         tagName,
 	                                         backupRanges,
 	                                         stopWhenDone,
-	                                         partitionedLog,
+	                                         mutationLogType,
 	                                         incrementalBackupOnly,
 	                                         encryptionKeyFileName,
+	                                         encryptionBlockSize,
 	                                         snapshotMode);
 }
 
@@ -8398,6 +8699,10 @@ Future<Void> FileBackupAgent::checkAndDisableBackupWorkers(Database cx) {
 	return FileBackupAgentImpl::checkAndDisableBackupWorkers(cx);
 }
 
+Future<Void> FileBackupAgent::checkAndDisableRangeBackupWorkers(Database cx) {
+	return FileBackupAgentImpl::checkAndDisableRangeBackupWorkers(cx);
+}
+
 Future<std::string> FileBackupAgent::getStatus(Database cx, ShowErrors showErrors, std::string tagName) {
 	return FileBackupAgentImpl::getStatus(this, cx, showErrors, tagName);
 }
@@ -8460,8 +8765,8 @@ static std::pair<bool, bool> insideValidRange(KeyValueRef kv,
 }
 
 // Write [begin, end) in kvs to DB
-ACTOR static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef>> kvs, int begin, int end) {
-	wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef>> kvs, int begin, int end) {
+	co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		int index = begin;
@@ -8475,11 +8780,12 @@ ACTOR static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef
 			++index;
 		}
 		return Void();
-	}));
+	});
 
 	// Sanity check data has been written to DB
-	state ReadYourWritesTransaction tr(cx);
-	loop {
+	ReadYourWritesTransaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
@@ -8489,192 +8795,17 @@ ACTOR static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef
 			    .detail("Range", KeyRangeRef(k1, k2))
 			    .detail("Begin", begin)
 			    .detail("End", end);
-			RangeResult readKVs = wait(tr.getRange(KeyRangeRef(k1, k2), CLIENT_KNOBS->TOO_MANY));
-			ASSERT(readKVs.size() > 0 || begin == end);
+			RangeResult readKVs = co_await tr.getRange(KeyRangeRef(k1, k2), CLIENT_KNOBS->TOO_MANY);
+			ASSERT(!readKVs.empty() || begin == end);
 			break;
 		} catch (Error& e) {
-			TraceEvent("TransformDatabaseContentsWriteKVReadBackError").error(e);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("TransformDatabaseContentsWriteKVReadBackError").error(err);
+		co_await tr.onError(err);
 	}
 
 	TraceEvent(SevFRTestInfo, "TransformDatabaseContentsWriteKVDone").detail("Begin", begin).detail("End", end);
-
-	return Void();
-}
-
-// restoreRanges is the actual range that has applied removePrefix and addPrefix processed by restore system
-// Assume: restoreRanges do not overlap which is achieved by ensuring backup ranges do not overlap
-ACTOR static Future<Void> transformDatabaseContents(Database cx,
-                                                    Key addPrefix,
-                                                    Key removePrefix,
-                                                    Standalone<VectorRef<KeyRangeRef>> restoreRanges) {
-	state ReadYourWritesTransaction tr(cx);
-	state Standalone<VectorRef<KeyValueRef>> oldData;
-
-	TraceEvent("FastRestoreWorkloadTransformDatabaseContents")
-	    .detail("AddPrefix", addPrefix)
-	    .detail("RemovePrefix", removePrefix);
-	state int i = 0;
-	loop { // Read all data from DB
-		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			for (i = 0; i < restoreRanges.size(); ++i) {
-				RangeResult kvs = wait(tr.getRange(restoreRanges[i], CLIENT_KNOBS->TOO_MANY));
-				ASSERT(!kvs.more);
-				for (auto kv : kvs) {
-					oldData.push_back_deep(oldData.arena(), KeyValueRef(kv.key, kv.value));
-				}
-			}
-			break;
-		} catch (Error& e) {
-			TraceEvent("FastRestoreWorkloadTransformDatabaseContentsGetAllKeys")
-			    .error(e)
-			    .detail("Index", i)
-			    .detail("RestoreRange", restoreRanges[i]);
-			oldData = Standalone<VectorRef<KeyValueRef>>(); // clear the vector
-			wait(tr.onError(e));
-		}
-	}
-
-	// Convert data by removePrefix and addPrefix in memory
-	state Standalone<VectorRef<KeyValueRef>> newKVs;
-	for (int i = 0; i < oldData.size(); ++i) {
-		Key newKey(oldData[i].key);
-		TraceEvent(SevFRTestInfo, "TransformDatabaseContents")
-		    .detail("Keys", oldData.size())
-		    .detail("Index", i)
-		    .detail("GetKey", oldData[i].key)
-		    .detail("GetValue", oldData[i].value);
-		if (newKey.size() < removePrefix.size()) { // If true, must check why.
-			TraceEvent(SevError, "TransformDatabaseContents")
-			    .detail("Key", newKey)
-			    .detail("RemovePrefix", removePrefix);
-			continue;
-		}
-		newKey = newKey.removePrefix(removePrefix).withPrefix(addPrefix);
-		newKVs.push_back_deep(newKVs.arena(), KeyValueRef(newKey.contents(), oldData[i].value));
-		TraceEvent(SevFRTestInfo, "TransformDatabaseContents")
-		    .detail("Keys", newKVs.size())
-		    .detail("Index", i)
-		    .detail("NewKey", newKVs.back().key)
-		    .detail("NewValue", newKVs.back().value);
-	}
-
-	state Standalone<VectorRef<KeyRangeRef>> backupRanges; // dest. ranges
-	for (auto& range : restoreRanges) {
-		KeyRange tmpRange = range;
-		backupRanges.push_back_deep(backupRanges.arena(), tmpRange.removePrefix(removePrefix).withPrefix(addPrefix));
-	}
-
-	// Clear the transformed data (original data with removePrefix and addPrefix) in restoreRanges
-	wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		for (int i = 0; i < restoreRanges.size(); i++) {
-			TraceEvent(SevFRTestInfo, "TransformDatabaseContents")
-			    .detail("ClearRestoreRange", restoreRanges[i])
-			    .detail("ClearBackupRange", backupRanges[i]);
-			tr->clear(restoreRanges[i]); // Clear the range.removePrefix().withPrefix()
-			tr->clear(backupRanges[i]);
-		}
-		return Void();
-	}));
-
-	// Sanity check to ensure no data in the ranges
-	tr.reset();
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			RangeResult emptyData = wait(tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY));
-			for (int i = 0; i < emptyData.size(); ++i) {
-				TraceEvent(SevError, "ExpectEmptyData")
-				    .detail("Index", i)
-				    .detail("Key", emptyData[i].key)
-				    .detail("Value", emptyData[i].value);
-			}
-			break;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-
-	// Write transformed KVs (i.e., kv backup took) back to DB
-	state std::vector<Future<Void>> fwrites;
-	loop {
-		try {
-			state int begin = 0;
-			state int len = 0;
-			while (begin < newKVs.size()) {
-				len = std::min(100, newKVs.size() - begin);
-				fwrites.push_back(writeKVs(cx, newKVs, begin, begin + len));
-				begin = begin + len;
-			}
-			wait(waitForAll(fwrites));
-			break;
-		} catch (Error& e) {
-			TraceEvent(SevError, "FastRestoreWorkloadTransformDatabaseContentsUnexpectedErrorOnWriteKVs").error(e);
-			wait(tr.onError(e));
-		}
-	}
-
-	// Sanity check
-	tr.reset();
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			RangeResult allData = wait(tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY));
-			TraceEvent(SevFRTestInfo, "SanityCheckData").detail("Size", allData.size());
-			for (int i = 0; i < allData.size(); ++i) {
-				std::pair<bool, bool> backupRestoreValid = insideValidRange(allData[i], restoreRanges, backupRanges);
-				TraceEvent(backupRestoreValid.first ? SevFRTestInfo : SevError, "SanityCheckData")
-				    .detail("Index", i)
-				    .detail("Key", allData[i].key)
-				    .detail("Value", allData[i].value)
-				    .detail("InsideBackupRange", backupRestoreValid.first)
-				    .detail("InsideRestoreRange", backupRestoreValid.second);
-			}
-			break;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-
-	TraceEvent("FastRestoreWorkloadTransformDatabaseContentsFinish")
-	    .detail("AddPrefix", addPrefix)
-	    .detail("RemovePrefix", removePrefix);
-
-	return Void();
-}
-
-// addPrefix and removePrefix are the options used in the restore request:
-// every backup key applied removePrefix and addPrefix in restore;
-// transformRestoredDatabase actor will revert it by remove addPrefix and add removePrefix.
-ACTOR Future<Void> transformRestoredDatabase(Database cx,
-                                             Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                             Key addPrefix,
-                                             Key removePrefix) {
-	try {
-		Standalone<VectorRef<KeyRangeRef>> restoreRanges;
-		for (int i = 0; i < backupRanges.size(); ++i) {
-			KeyRange range(backupRanges[i]);
-			Key begin = range.begin.removePrefix(removePrefix).withPrefix(addPrefix);
-			Key end = range.end.removePrefix(removePrefix).withPrefix(addPrefix);
-			TraceEvent("FastRestoreTransformRestoredDatabase")
-			    .detail("From", KeyRangeRef(begin.contents(), end.contents()))
-			    .detail("To", range);
-			restoreRanges.push_back_deep(restoreRanges.arena(), KeyRangeRef(begin.contents(), end.contents()));
-		}
-		wait(transformDatabaseContents(cx, removePrefix, addPrefix, restoreRanges));
-	} catch (Error& e) {
-		TraceEvent(SevError, "FastRestoreTransformRestoredDatabaseUnexpectedError").error(e);
-		throw;
-	}
-
-	return Void();
 }
 
 void simulateBlobFailure() {
diff --git a/fdbclient/FluentDSampleIngestor.cpp b/fdbclient/FluentDSampleIngestor.cpp
index f440a1f2967..de9261909e7 100644
--- a/fdbclient/FluentDSampleIngestor.cpp
+++ b/fdbclient/FluentDSampleIngestor.cpp
@@ -39,7 +39,7 @@ boost::asio::ip::basic_endpoint<Protocol> toEndpoint(NetworkAddress const n) {
 }
 
 struct FluentDSocket {
-	virtual ~FluentDSocket() {}
+	virtual ~FluentDSocket() = default;
 	virtual void connect(NetworkAddress const& endpoint) = 0;
 	virtual void send(std::shared_ptr<Sample> const& sample) = 0;
 	virtual const boost::system::error_code& failed() const = 0;
@@ -137,7 +137,7 @@ struct FluentDSocketImpl : FluentDSocket, std::enable_shared_from_this<FluentDSo
 	static constexpr unsigned MAX_QUEUE_SIZE = 100;
 	boost::asio::io_context& context;
 	typename Protocol::socket socket;
-	FluentDSocketImpl(boost::asio::io_context& context) : context(context), socket(context) {}
+	explicit FluentDSocketImpl(boost::asio::io_context& context) : context(context), socket(context) {}
 	bool ready = false;
 	std::deque<std::shared_ptr<Sample>> queue;
 	boost::system::error_code _failed;
diff --git a/fdbclient/GCSBlobStore.cpp b/fdbclient/GCSBlobStore.cpp
new file mode 100644
index 00000000000..5621de59d3e
--- /dev/null
+++ b/fdbclient/GCSBlobStore.cpp
@@ -0,0 +1,289 @@
+/*
+ * GCSBlobStore.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "GCSBlobStore.h"
+#include "fdbclient/JSONDoc.h"
+#include "flow/Trace.h"
+
+GCSBlobStoreEndpoint::GCSBlobStoreEndpoint(std::string const& host,
+                                           std::string const& service,
+                                           Optional<std::string> const& proxyHost,
+                                           Optional<std::string> const& proxyPort,
+                                           Optional<StringRef> const& creds,
+                                           std::string const& projectId,
+                                           BlobKnobs const& knobs,
+                                           HTTP::Headers extraHeaders)
+  : S3BlobStoreEndpoint(host,
+                        service,
+                        "auto",
+                        proxyHost,
+                        proxyPort,
+                        Optional<StringRef>(), // Don't pass creds to S3 — we handle auth ourselves
+                        knobs,
+                        extraHeaders),
+    projectId(projectId) {
+	if (creds.present()) {
+		token = creds.get().toString();
+	}
+	// Mirror S3's lookupKey/lookupSecret pattern: if the URL didn't supply a token, we must load it from
+	// credential files on each new connection and omit it from the serialized URL.
+	lookupToken = token.empty();
+}
+
+void GCSBlobStoreEndpoint::setRequestHeaders(std::string const& verb,
+                                             std::string const& resource,
+                                             HTTP::Headers& headers) {
+	headers["Accept"] = "application/xml";
+	if (!token.empty()) {
+		headers["Authorization"] = "Bearer " + token;
+	}
+	if (!projectId.empty()) {
+		headers["x-goog-project-id"] = projectId;
+	}
+}
+
+Future<Void> GCSBlobStoreEndpoint::updateSecret() {
+	// S3BlobStoreEndpoint::updateSecret() short-circuits when credentials are absent, which is always the case for
+	// GCS (we carry the bearer token in `token`, not in S3's `credentials`). Skip the S3 override and go straight to
+	// the generic credential-file loop in the base class.
+	return IBlobStoreEndpoint::updateSecret();
+}
+
+bool GCSBlobStoreEndpoint::extractCredentialFields(JSONDoc& account) {
+	std::string newToken;
+	if (!account.tryGet("token", newToken)) {
+		return false;
+	}
+	token = newToken;
+	return true;
+}
+
+bool GCSBlobStoreEndpoint::lookupSecretOnEachRequest() {
+	return lookupToken;
+}
+
+std::string GCSBlobStoreEndpoint::credentialFileKey() const {
+	return "@" + host;
+}
+
+std::string GCSBlobStoreEndpoint::getResourceURL(std::string resource, std::string params) const {
+	if (!params.empty())
+		params.append("&");
+	params.append("p=gcs");
+	if (!projectId.empty()) {
+		params.append("&gcspid=").append(projectId);
+	}
+	std::string url = S3BlobStoreEndpoint::getResourceURL(resource, params);
+
+	// Preserve an inline token in the URL so it round-trips through fromString. Tokens loaded from credential
+	// files (lookupToken == true) are omitted to avoid baking a refreshed value into a serialized URL.
+	if (!lookupToken && !token.empty()) {
+		const std::string placeholder = "blobstore://@";
+		size_t pos = url.find(placeholder);
+		if (pos != std::string::npos) {
+			url.replace(pos, placeholder.size(), "blobstore://" + token + "@");
+		}
+	}
+	return url;
+}
+
+Future<Void> createBucket_gcs_impl(Reference<GCSBlobStoreEndpoint> b, std::string bucket) {
+	co_await b->requestRateWrite->getAllowance(1);
+
+	bool exists = co_await b->bucketExists(bucket);
+	if (exists) {
+		co_return;
+	}
+
+	if (b->projectId.empty()) {
+		TraceEvent(SevError, "GCSBucketCreateMissingProject")
+		    .detail("Bucket", bucket)
+		    .detail("Hint", "Set gcs_project_id (or gcspid) URL parameter");
+		throw backup_invalid_url();
+	}
+
+	// GCS XML API accepts the same PUT request as S3 for bucket creation.
+	// x-goog-project-id is added by setRequestHeaders on every request.
+	std::string resource = b->constructResourcePath(bucket, "");
+	HTTP::Headers headers;
+	co_await b->doRequest("PUT", resource, headers, nullptr, 0, { 200, 409 });
+}
+
+Future<Void> GCSBlobStoreEndpoint::createBucket(std::string const& bucket) {
+	return createBucket_gcs_impl(Reference<GCSBlobStoreEndpoint>::addRef(this), bucket);
+}
+
+TEST_CASE("/backup/gcs/fromString") {
+	// URL with p=gcs should produce a GCSBlobStoreEndpoint
+	{
+		std::string url =
+		    "blobstore://my-token@storage.googleapis.com/resource?bucket=my-bucket&p=gcs&gcspid=my-project";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+		ASSERT(endpoint.isValid());
+		ASSERT(resource == "resource");
+
+		auto* gcs = dynamic_cast<GCSBlobStoreEndpoint*>(endpoint.getPtr());
+		ASSERT(gcs != nullptr);
+		ASSERT(gcs->projectId == "my-project");
+		ASSERT(gcs->token == "my-token");
+	}
+
+	// provider=gcs long form should also work
+	{
+		std::string url = "blobstore://tok@storage.googleapis.com/r?bucket=b&provider=gcs&gcs_project_id=proj";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+		ASSERT(endpoint.isValid());
+
+		auto* gcs = dynamic_cast<GCSBlobStoreEndpoint*>(endpoint.getPtr());
+		ASSERT(gcs != nullptr);
+		ASSERT(gcs->projectId == "proj");
+		ASSERT(gcs->token == "tok");
+	}
+
+	// URL without p=gcs should NOT produce a GCSBlobStoreEndpoint
+	{
+		std::string url = "blobstore://s3.us-west-2.amazonaws.com/resource?bucket=b&sa=1";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+		ASSERT(endpoint.isValid());
+		ASSERT(dynamic_cast<GCSBlobStoreEndpoint*>(endpoint.getPtr()) == nullptr);
+	}
+
+	return Void();
+}
+
+TEST_CASE("/backup/gcs/setRequestHeaders") {
+	// With token and projectId — all headers set
+	{
+		GCSBlobStoreEndpoint gcs("storage.googleapis.com", "443", {}, {}, "my-bearer-token"_sr, "my-project");
+		HTTP::Headers headers;
+		gcs.setRequestHeaders("GET", "/bucket/object", headers);
+
+		ASSERT(headers["Authorization"] == "Bearer my-bearer-token");
+		ASSERT(headers["Accept"] == "application/xml");
+		ASSERT(headers["x-goog-project-id"] == "my-project");
+	}
+
+	// Empty token — no Authorization header
+	{
+		GCSBlobStoreEndpoint gcs("storage.googleapis.com", "443", {}, {}, Optional<StringRef>(), "my-project");
+		HTTP::Headers headers;
+		gcs.setRequestHeaders("PUT", "/bucket", headers);
+
+		ASSERT(!headers.contains("Authorization"));
+		ASSERT(headers["Accept"] == "application/xml");
+		ASSERT(headers["x-goog-project-id"] == "my-project");
+	}
+
+	// Empty projectId — no x-goog-project-id header
+	{
+		GCSBlobStoreEndpoint gcs("storage.googleapis.com", "443", {}, {}, "tok"_sr, "");
+		HTTP::Headers headers;
+		gcs.setRequestHeaders("GET", "/bucket/object", headers);
+
+		ASSERT(headers["Authorization"] == "Bearer tok");
+		ASSERT(!headers.contains("x-goog-project-id"));
+	}
+
+	return Void();
+}
+
+TEST_CASE("/backup/gcs/getResourceURL") {
+	// getResourceURL should include p=gcs and gcspid
+	{
+		std::string url = "blobstore://tok@storage.googleapis.com/resource?bucket=b&p=gcs&gcspid=my-project";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+		ASSERT(endpoint.isValid());
+
+		std::string resURL = endpoint->getResourceURL("test-resource", "");
+		ASSERT(resURL.find("p=gcs") != std::string::npos);
+		ASSERT(resURL.find("gcspid=my-project") != std::string::npos);
+	}
+
+	// Without projectId — no gcspid in URL
+	{
+		std::string url = "blobstore://tok@storage.googleapis.com/resource?bucket=b&p=gcs";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+		ASSERT(endpoint.isValid());
+
+		std::string resURL = endpoint->getResourceURL("test-resource", "");
+		ASSERT(resURL.find("p=gcs") != std::string::npos);
+		ASSERT(resURL.find("gcspid") == std::string::npos);
+	}
+
+	// Round-trip: an inline token survives parse → serialize → parse.
+	{
+		std::string url = "blobstore://tok@storage.googleapis.com/resource?bucket=b&p=gcs&gcspid=proj";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+
+		std::string serialized = endpoint->getResourceURL("new-resource", "");
+		ASSERT(serialized.find("tok@") != std::string::npos);
+
+		std::string resource2;
+		std::string error2;
+		S3BlobStoreEndpoint::ParametersT params2;
+		Reference<S3BlobStoreEndpoint> endpoint2 =
+		    S3BlobStoreEndpoint::fromString(serialized, {}, &resource2, &error2, &params2);
+		ASSERT(endpoint2.isValid());
+
+		auto* gcs2 = dynamic_cast<GCSBlobStoreEndpoint*>(endpoint2.getPtr());
+		ASSERT(gcs2 != nullptr);
+		ASSERT(gcs2->projectId == "proj");
+		ASSERT(gcs2->token == "tok");
+		ASSERT(!gcs2->lookupToken);
+	}
+
+	// Tokens loaded from credential files are not embedded in the serialized URL.
+	{
+		std::string url = "blobstore://@storage.googleapis.com/resource?bucket=b&p=gcs&gcspid=proj";
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT params;
+		Reference<S3BlobStoreEndpoint> endpoint = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &params);
+
+		auto* gcs = dynamic_cast<GCSBlobStoreEndpoint*>(endpoint.getPtr());
+		ASSERT(gcs != nullptr);
+		ASSERT(gcs->lookupToken);
+
+		// Simulate a credential-file refresh populating the token.
+		gcs->token = "refreshed-token";
+		std::string serialized = gcs->getResourceURL("r", "");
+		ASSERT(serialized.find("refreshed-token") == std::string::npos);
+	}
+
+	return Void();
+}
diff --git a/fdbclient/GCSBlobStore.h b/fdbclient/GCSBlobStore.h
new file mode 100644
index 00000000000..3dfc49c497a
--- /dev/null
+++ b/fdbclient/GCSBlobStore.h
@@ -0,0 +1,59 @@
+/*
+ * GCSBlobStore.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/S3BlobStore.h"
+
+// GCS blob store endpoint using the S3-compatible XML API with OAuth2 Bearer token authentication.
+// Inherits all blob operations (list, read, write, multipart) from S3BlobStoreEndpoint since the
+// GCS XML API uses the same protocol. Only auth headers and credential handling differ.
+class GCSBlobStoreEndpoint : public S3BlobStoreEndpoint {
+public:
+	GCSBlobStoreEndpoint(std::string const& host,
+	                     std::string const& service,
+	                     Optional<std::string> const& proxyHost,
+	                     Optional<std::string> const& proxyPort,
+	                     Optional<StringRef> const& creds,
+	                     std::string const& projectId,
+	                     BlobKnobs const& knobs = BlobKnobs(),
+	                     HTTP::Headers extraHeaders = HTTP::Headers());
+
+	// OAuth2 Bearer token instead of SigV4
+	void setRequestHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) override;
+
+	Future<Void> updateSecret() override;
+	bool extractCredentialFields(JSONDoc& account) override;
+
+	// Only consult credential files when the URL didn't supply an inline token.
+	bool lookupSecretOnEachRequest() override;
+
+	// GCS credential files are keyed by "@host" — no access-key prefix like S3's "accessKey@host".
+	std::string credentialFileKey() const override;
+
+	std::string getResourceURL(std::string resource, std::string params) const override;
+
+	// GCS bucket creation needs x-goog-project-id header
+	Future<Void> createBucket(std::string const& bucket) override;
+
+	std::string projectId;
+	std::string token;
+	bool lookupToken;
+};
diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.cpp
similarity index 62%
rename from fdbclient/GlobalConfig.actor.cpp
rename to fdbclient/GlobalConfig.cpp
index 4588e63e912..a95ff40ab1b 100644
--- a/fdbclient/GlobalConfig.actor.cpp
+++ b/fdbclient/GlobalConfig.cpp
@@ -1,5 +1,5 @@
 /*
- * GlobalConfig.actor.cpp
+ * GlobalConfig.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,8 +19,8 @@
  */
 
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/SpecialKeySpace.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/Tuple.h"
 #include "flow/Error.h"
@@ -71,7 +71,7 @@ Key GlobalConfig::prefixedKey(KeyRef key) {
 	return key.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin);
 }
 
-const Reference<ConfigValue> GlobalConfig::get(KeyRef name) {
+Reference<ConfigValue> GlobalConfig::get(KeyRef name) {
 	auto it = data.find(name);
 	if (it == data.end()) {
 		return Reference<ConfigValue>();
@@ -79,7 +79,7 @@ const Reference<ConfigValue> GlobalConfig::get(KeyRef name) {
 	return it->second;
 }
 
-const std::map<KeyRef, Reference<ConfigValue>> GlobalConfig::get(KeyRangeRef range) {
+std::map<KeyRef, Reference<ConfigValue>> GlobalConfig::get(KeyRangeRef range) {
 	std::map<KeyRef, Reference<ConfigValue>> results;
 	for (const auto& [key, value] : data) {
 		if (range.contains(key)) {
@@ -154,113 +154,109 @@ void GlobalConfig::erase(KeyRangeRef range) {
 	}
 }
 
-ACTOR Future<Version> GlobalConfig::refresh(GlobalConfig* self, Version lastKnown, Version largestSeen) {
+Future<Version> GlobalConfig::refresh(Version lastKnown, Version largestSeen) {
 	// TraceEvent trace(SevInfo, "GlobalConfigRefresh");
-	self->erase(KeyRangeRef(""_sr, "\xff"_sr));
+	erase(KeyRangeRef(""_sr, "\xff"_sr));
 
-	state Backoff backoff(CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_BACKOFF, CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_MAX_BACKOFF);
+	Backoff backoff(CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_BACKOFF, CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_MAX_BACKOFF);
 	loop {
+		Error err;
 		try {
 			GlobalConfigRefreshReply reply =
-			    wait(timeoutError(basicLoadBalance(self->cx->getGrvProxies(UseProvisionalProxies::False),
-			                                       &GrvProxyInterface::refreshGlobalConfig,
-			                                       GlobalConfigRefreshRequest{ lastKnown }),
-			                      CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT));
+			    co_await timeoutError(basicLoadBalance(cx->getGrvProxies(UseProvisionalProxies::False),
+			                                           &GrvProxyInterface::refreshGlobalConfig,
+			                                           GlobalConfigRefreshRequest{ lastKnown }),
+			                          CLIENT_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT);
 			for (const auto& kv : reply.result) {
 				KeyRef systemKey = kv.key.removePrefix(globalConfigKeysPrefix);
-				self->insert(systemKey, kv.value);
+				insert(systemKey, kv.value);
 			}
 			if (reply.version >= largestSeen || largestSeen == std::numeric_limits<Version>::max()) {
-				return reply.version;
-			} else {
-				wait(delay(0.25));
+				co_return reply.version;
 			}
+			co_await delay(0.25);
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
-			}
-
-			wait(backoff.onError());
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
 		}
+		co_await backoff.onError();
 	}
 }
 
 // Applies updates to the local copy of the global configuration when this
 // process receives an updated history.
-ACTOR Future<Void> GlobalConfig::updater(GlobalConfig* self, const ClientDBInfo* dbInfo) {
+Future<Void> GlobalConfig::updater(const ClientDBInfo* dbInfo) {
 	loop {
+		Error err;
 		try {
-			if (self->initialized.canBeSet()) {
-				wait(self->cx->onConnected());
+			if (initialized.canBeSet()) {
+				co_await cx->onConnected();
 
-				Version version = wait(self->refresh(self, -1, 0));
-				self->lastUpdate = version;
+				lastUpdate = co_await refresh(-1, 0);
 
-				self->cx->addref();
-				self->initialized.send(Void());
-				self->cx->delref();
+				cx->addref();
+				initialized.send(Void());
+				cx->delref();
 			}
 
 			loop {
-				try {
-					// run one iteration at the beginning
-					wait(delay(0));
-					if (dbInfo->history.size() > 0) {
-						if (self->lastUpdate < dbInfo->history[0].version) {
-							// This process missed too many global configuration
-							// history updates or the protocol version changed, so it
-							// must re-read the entire configuration range.
-							Version version =
-							    wait(self->refresh(self, self->lastUpdate, dbInfo->history.back().version));
-							self->lastUpdate = version;
-							// DBInfo could have changed after the wait. If
-							// changes are present, re-run the loop to make
-							// sure they are applied.
-							if (dbInfo->history.size() > 0 &&
-							    dbInfo->history[0].version != std::numeric_limits<Version>::max()) {
-								continue;
+				// run one iteration at the beginning
+				co_await delay(0);
+				if (!dbInfo->history.empty()) {
+					if (lastUpdate < dbInfo->history[0].version) {
+						// This process missed too many global configuration
+						// history updates or the protocol version changed, so it
+						// must re-read the entire configuration range.
+						lastUpdate = co_await refresh(lastUpdate, dbInfo->history.back().version);
+						// DBInfo could have changed after the wait. If
+						// changes are present, re-run the loop to make
+						// sure they are applied.
+						if (!dbInfo->history.empty() &&
+						    dbInfo->history[0].version != std::numeric_limits<Version>::max()) {
+							continue;
+						}
+					} else {
+						// Apply history in order, from lowest version to highest
+						// version. Mutation history should already be stored in
+						// ascending version order.
+						for (const auto& vh : dbInfo->history) {
+							if (vh.version <= lastUpdate) {
+								continue; // already applied this mutation
 							}
-						} else {
-							// Apply history in order, from lowest version to highest
-							// version. Mutation history should already be stored in
-							// ascending version order.
-							for (const auto& vh : dbInfo->history) {
-								if (vh.version <= self->lastUpdate) {
-									continue; // already applied this mutation
-								}
 
-								for (const auto& mutation : vh.mutations.contents()) {
-									if (mutation.type == MutationRef::SetValue) {
-										self->insert(mutation.param1, mutation.param2);
-									} else if (mutation.type == MutationRef::ClearRange) {
-										self->erase(KeyRangeRef(mutation.param1, mutation.param2));
-									} else {
-										UNREACHABLE();
-									}
+							for (const auto& mutation : vh.mutations.contents()) {
+								if (mutation.type == MutationRef::SetValue) {
+									insert(mutation.param1, mutation.param2);
+								} else if (mutation.type == MutationRef::ClearRange) {
+									erase(KeyRangeRef(mutation.param1, mutation.param2));
+								} else {
+									UNREACHABLE();
 								}
-
-								ASSERT(vh.version > self->lastUpdate);
-								self->lastUpdate = vh.version;
 							}
+
+							ASSERT(vh.version > lastUpdate);
+							lastUpdate = vh.version;
 						}
-						self->configChanged.trigger();
 					}
-					// In case this actor is canceled in the d'tor of GlobalConfig we can exit here.
-					wait(delay(0));
-					wait(self->dbInfoChanged.onTrigger());
-				} catch (Error& e) {
-					throw;
+					configChanged.trigger();
 				}
+				// In case this actor is canceled in the d'tor of GlobalConfig we can exit here.
+				co_await delay(0);
+				co_await dbInfoChanged.onTrigger();
 			}
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
-			}
-
-			// There shouldn't be any uncaught errors that fall to this point,
-			// but in case there are, catch them and restart the updater.
-			TraceEvent("GlobalConfigUpdaterError").error(e);
-			wait(delay(1.0));
+			err = e;
 		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+
+		// There shouldn't be any uncaught errors that fall to this point,
+		// but in case there are, catch them and restart the updater.
+		TraceEvent("GlobalConfigUpdaterError").error(err);
+		co_await delay(1.0);
 	}
 }
diff --git a/fdbclient/IKnobCollection.cpp b/fdbclient/IKnobCollection.cpp
deleted file mode 100644
index 1531991b626..00000000000
--- a/fdbclient/IKnobCollection.cpp
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * IKnobCollection.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/IKnobCollection.h"
-#include "fdbclient/ClientKnobCollection.h"
-#include "fdbclient/ServerKnobCollection.h"
-#include "fdbclient/TestKnobCollection.h"
-
-std::unique_ptr<IKnobCollection> IKnobCollection::create(Type type, Randomize randomize, IsSimulated isSimulated) {
-	if (type == Type::CLIENT) {
-		return std::make_unique<ClientKnobCollection>(randomize, isSimulated);
-	} else if (type == Type::SERVER) {
-		return std::make_unique<ServerKnobCollection>(randomize, isSimulated);
-	} else if (type == Type::TEST) {
-		return std::make_unique<TestKnobCollection>(randomize, isSimulated);
-	}
-	UNSTOPPABLE_ASSERT(false);
-}
-
-KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::string const& knobValue) const {
-	auto result = tryParseKnobValue(knobName, knobValue);
-	if (!result.present()) {
-		throw invalid_option();
-	}
-	return result.get();
-}
-
-void IKnobCollection::setKnob(std::string const& knobName, KnobValueRef const& knobValue) {
-	if (!trySetKnob(knobName, knobValue)) {
-		TraceEvent(SevWarnAlways, "FailedToSetKnob")
-		    .detail("KnobName", knobName)
-		    .detail("KnobValue", knobValue.toString());
-		throw invalid_option_value();
-	}
-}
-
-KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::string const& knobValue, Type type) {
-	// TODO: Ideally it should not be necessary to create a template object to parse knobs
-	static std::unique_ptr<IKnobCollection> clientKnobCollection, serverKnobCollection, testKnobCollection;
-	if (type == Type::CLIENT) {
-		if (!clientKnobCollection) {
-			clientKnobCollection = create(type, Randomize::False, IsSimulated::False);
-		}
-		return clientKnobCollection->parseKnobValue(knobName, knobValue);
-	} else if (type == Type::SERVER) {
-		if (!serverKnobCollection) {
-			serverKnobCollection = create(type, Randomize::False, IsSimulated::False);
-		}
-		return serverKnobCollection->parseKnobValue(knobName, knobValue);
-	} else if (type == Type::TEST) {
-		if (!testKnobCollection) {
-			testKnobCollection = create(type, Randomize::False, IsSimulated::False);
-		}
-		return testKnobCollection->parseKnobValue(knobName, knobValue);
-	}
-	UNSTOPPABLE_ASSERT(false);
-}
-
-std::unique_ptr<IKnobCollection>& IKnobCollection::globalKnobCollection() {
-	static std::unique_ptr<IKnobCollection> res;
-	if (!res) {
-		res = IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::False, IsSimulated::False);
-	}
-	return res;
-}
-
-void IKnobCollection::setGlobalKnobCollection(Type type, Randomize randomize, IsSimulated isSimulated) {
-	globalKnobCollection() = create(type, randomize, isSimulated);
-	ASSERT(FLOW_KNOBS == &bootstrapGlobalFlowKnobs);
-	FLOW_KNOBS = &globalKnobCollection()->getFlowKnobs();
-}
-
-IKnobCollection const& IKnobCollection::getGlobalKnobCollection() {
-	return *globalKnobCollection();
-}
-
-IKnobCollection& IKnobCollection::getMutableGlobalKnobCollection() {
-	return *globalKnobCollection();
-}
-
-void IKnobCollection::setupKnobs(const std::vector<std::pair<std::string, std::string>>& knobs) {
-	auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
-	for (const auto& [knobName, knobValueString] : knobs) {
-		try {
-			auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
-			g_knobs.setKnob(knobName, knobValue);
-		} catch (Error& e) {
-			if (e.code() == error_code_invalid_option_value) {
-				std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName
-				          << "'\n";
-				TraceEvent(SevWarnAlways, "InvalidKnobValue")
-				    .detail("Knob", printable(knobName))
-				    .detail("Value", printable(knobValueString));
-			} else if (e.code() == error_code_invalid_option) {
-				std::cerr << "WARNING: Invalid knob option '" << knobName << "'\n";
-				TraceEvent(SevWarnAlways, "InvalidKnobName")
-				    .detail("Knob", printable(knobName))
-				    .detail("Value", printable(knobValueString));
-			} else {
-				std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n";
-				TraceEvent(SevError, "FailedToSetKnob")
-				    .errorUnsuppressed(e)
-				    .detail("Knob", printable(knobName))
-				    .detail("Value", printable(knobValueString));
-				throw e;
-			}
-		}
-	}
-}
diff --git a/fdbclient/IdempotencyId.actor.cpp b/fdbclient/IdempotencyId.cpp
similarity index 81%
rename from fdbclient/IdempotencyId.actor.cpp
rename to fdbclient/IdempotencyId.cpp
index f9110f9181b..c52099c9806 100644
--- a/fdbclient/IdempotencyId.actor.cpp
+++ b/fdbclient/IdempotencyId.cpp
@@ -1,5 +1,5 @@
 /*
- * IdempotencyId.actor.cpp
+ * IdempotencyId.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbclient/IdempotencyId.actor.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/IdempotencyId.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
 #include "flow/BooleanParam.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // this has to be the last include
 
 struct IdempotencyIdKVBuilderImpl {
 	Optional<Version> commitVersion;
@@ -153,7 +152,7 @@ TEST_CASE("/fdbclient/IdempotencyId/basic") {
 		idSet.erase(id);
 		ASSERT(idSet.find(id) == idSet.end());
 	}
-	ASSERT(idSet.size() == 0);
+	ASSERT(idSet.empty());
 
 	ASSERT(!kvContainsIdempotencyId(kv, generate(arena)).present());
 
@@ -210,15 +209,15 @@ FDB_BOOLEAN_PARAM(Oldest);
 
 // Find the youngest or oldest idempotency id key in `range` (depending on `oldest`)
 // Write the timestamp to `*time` and the version to `*version` when non-null.
-ACTOR static Future<Optional<Key>> getBoundary(Reference<ReadYourWritesTransaction> tr,
-                                               KeyRange range,
-                                               Oldest oldest,
-                                               Version* version,
-                                               int64_t* time) {
+static Future<Optional<Key>> getBoundary(Reference<ReadYourWritesTransaction> tr,
+                                         KeyRange range,
+                                         Oldest oldest,
+                                         Version* version,
+                                         int64_t* time) {
 	RangeResult result =
-	    wait(tr->getRange(range, /*limit*/ 1, Snapshot::False, oldest ? Reverse::False : Reverse::True));
-	if (!result.size()) {
-		return Optional<Key>();
+	    co_await tr->getRange(range, /*limit*/ 1, Snapshot::False, oldest ? Reverse::False : Reverse::True);
+	if (result.empty()) {
+		co_return Optional<Key>();
 	}
 	if (version != nullptr) {
 		BinaryReader rd(result.front().key, Unversioned());
@@ -230,25 +229,26 @@ ACTOR static Future<Optional<Key>> getBoundary(Reference<ReadYourWritesTransacti
 		BinaryReader rd(result.front().value, IncludeVersion());
 		rd >> *time;
 	}
-	return result.front().key;
+	co_return result.front().key;
 }
 
-ACTOR Future<JsonBuilderObject> getIdmpKeyStatus(Database db) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	state int64_t size;
-	state IdempotencyIdsExpiredVersion expired;
-	state KeyBackedObjectProperty<IdempotencyIdsExpiredVersion, _Unversioned> expiredKey(idempotencyIdsExpiredVersion,
-	                                                                                     Unversioned());
-	state int64_t oldestIdVersion = 0;
-	state int64_t oldestIdTime = 0;
-	loop {
+AsyncResult<JsonBuilderObject> getIdmpKeyStatus(Database db) {
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
+	int64_t size{ 0 };
+	IdempotencyIdsExpiredVersion expired;
+	KeyBackedObjectProperty<IdempotencyIdsExpiredVersion, _Unversioned> expiredKey(idempotencyIdsExpiredVersion,
+	                                                                               Unversioned());
+	int64_t oldestIdVersion = 0;
+	int64_t oldestIdTime = 0;
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 
-			wait(store(size, tr->getEstimatedRangeSizeBytes(idempotencyIdKeys)) &&
-			     store(expired, expiredKey.getD(tr)) &&
-			     success(getBoundary(tr, idempotencyIdKeys, Oldest::True, &oldestIdVersion, &oldestIdTime)));
+			co_await (store(size, tr->getEstimatedRangeSizeBytes(idempotencyIdKeys)) &&
+			          store(expired, expiredKey.getD(tr)) &&
+			          success(getBoundary(tr, idempotencyIdKeys, Oldest::True, &oldestIdVersion, &oldestIdTime)));
 			JsonBuilderObject result;
 			result["size_bytes"] = size;
 			if (expired.expired != 0) {
@@ -263,38 +263,40 @@ ACTOR Future<JsonBuilderObject> getIdmpKeyStatus(Database db) {
 			if (oldestIdTime != 0) {
 				result["oldest_id_age"] = int64_t(now()) - oldestIdTime;
 			}
-			return result;
+			co_return result;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds) {
-	state int64_t idmpKeySize;
-	state int64_t candidateDeleteSize;
-	state KeyRange finalRange;
-	state Reference<ReadYourWritesTransaction> tr;
+Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds) {
+	int64_t idmpKeySize{ 0 };
+	int64_t candidateDeleteSize{ 0 };
+	KeyRange finalRange;
+	Reference<ReadYourWritesTransaction> tr;
 
 	// Only assigned to once
-	state Key oldestKey;
-	state Version oldestVersion;
-	state int64_t oldestTime;
+	Key oldestKey;
+	Version oldestVersion{ 0 };
+	int64_t oldestTime{ 0 };
 
 	// Assigned to multiple times looking for a suitable range
-	state Version candidateDeleteVersion;
-	state int64_t candidateDeleteTime;
-	state KeyRange candidateRangeToClean;
+	Version candidateDeleteVersion{ 0 };
+	int64_t candidateDeleteTime{ 0 };
+	KeyRange candidateRangeToClean;
 
 	tr = makeReference<ReadYourWritesTransaction>(db);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			// Check if any keys are older than minAgeSeconds
 			Optional<Key> oldestKey_ =
-			    wait(getBoundary(tr, idempotencyIdKeys, Oldest::True, &oldestVersion, &oldestTime));
+			    co_await getBoundary(tr, idempotencyIdKeys, Oldest::True, &oldestVersion, &oldestTime);
 			if (!oldestKey_.present()) {
 				break;
 			}
@@ -304,15 +306,14 @@ ACTOR Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds) {
 			}
 
 			// Only used for a trace event
-			wait(store(idmpKeySize, tr->getEstimatedRangeSizeBytes(idempotencyIdKeys)));
+			idmpKeySize = co_await tr->getEstimatedRangeSizeBytes(idempotencyIdKeys);
 
 			// Get the version of the most recent idempotency ID
-			wait(success(
-			    getBoundary(tr, idempotencyIdKeys, Oldest::False, &candidateDeleteVersion, &candidateDeleteTime)));
+			co_await getBoundary(tr, idempotencyIdKeys, Oldest::False, &candidateDeleteVersion, &candidateDeleteTime);
 
 			// Keep dividing the candidate range until clearing it would not delete something younger than
 			// minAgeSeconds
-			loop {
+			while (true) {
 
 				candidateRangeToClean =
 				    KeyRangeRef(oldestKey,
@@ -325,8 +326,8 @@ ACTOR Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds) {
 				}
 
 				// Find the youngest key in candidate range
-				wait(success(getBoundary(
-				    tr, candidateRangeToClean, Oldest::False, &candidateDeleteVersion, &candidateDeleteTime)));
+				co_await getBoundary(
+				    tr, candidateRangeToClean, Oldest::False, &candidateDeleteVersion, &candidateDeleteTime);
 
 				// Update the range so that it ends at an idempotency id key. Since we're binary searching, the
 				// candidate range was probably too large before.
@@ -335,7 +336,7 @@ ACTOR Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds) {
 				                BinaryWriter::toValue(bigEndian64(candidateDeleteVersion + 1), Unversioned())
 				                    .withPrefix(idempotencyIdKeys.begin));
 
-				wait(store(candidateDeleteSize, tr->getEstimatedRangeSizeBytes(candidateRangeToClean)));
+				candidateDeleteSize = co_await tr->getEstimatedRangeSizeBytes(candidateRangeToClean);
 
 				int64_t youngestAge = int64_t(now()) - candidateDeleteTime;
 				TraceEvent("IdempotencyIdsCleanerCandidateDelete")
@@ -363,13 +364,13 @@ ACTOR Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds) {
 				    .detail("ClearRangeSizeEstimate", candidateDeleteSize)
 				    .detail("ExpiredVersion", candidateDeleteVersion)
 				    .detail("ExpiredVersionAgeEstimate", static_cast<int64_t>(now()) - candidateDeleteTime);
-				wait(tr->commit());
+				co_await tr->commit();
 			}
 			break;
 		} catch (Error& e) {
-			TraceEvent("IdempotencyIdsCleanerError").error(e);
-			wait(tr->onError(e));
+			err = e;
 		}
+		TraceEvent("IdempotencyIdsCleanerError").error(err);
+		co_await tr->onError(err);
 	}
-	return Void();
 }
diff --git a/fdbclient/JsonBuilder.cpp b/fdbclient/JsonBuilder.cpp
index cc3073fe699..2eb458cdca7 100644
--- a/fdbclient/JsonBuilder.cpp
+++ b/fdbclient/JsonBuilder.cpp
@@ -19,7 +19,6 @@
  */
 
 #include "fdbclient/JsonBuilder.h"
-#include <iostream>
 
 JsonBuilderObject JsonBuilder::makeMessage(const char* name, const char* description) {
 	JsonBuilderObject out;
diff --git a/fdbclient/KeyRangeMap.actor.cpp b/fdbclient/KeyRangeMap.cpp
similarity index 80%
rename from fdbclient/KeyRangeMap.actor.cpp
rename to fdbclient/KeyRangeMap.cpp
index 94fb3c33ea8..a74c2c70276 100644
--- a/fdbclient/KeyRangeMap.actor.cpp
+++ b/fdbclient/KeyRangeMap.cpp
@@ -1,5 +1,5 @@
 /*
- * KeyRangeMap.actor.cpp
+ * KeyRangeMap.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,7 +24,6 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, std::vector<KeyRange>& affectedRanges) {
 	auto s = map.rangeContaining(keys.begin);
@@ -90,67 +89,63 @@ RangeResult krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, RangeResult kv, boo
 }
 
 // Returns keys.begin, all transitional points in keys, and keys.end, and their values
-ACTOR Future<RangeResult> krmGetRanges(Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes) {
+Future<RangeResult> krmGetRanges(Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes) {
 	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
 
-	state GetRangeLimits limits(limit, limitBytes);
+	GetRangeLimits limits(limit, limitBytes);
 	limits.minRows = 2;
-	RangeResult kv = wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
+	RangeResult kv = co_await tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits);
 
-	return krmDecodeRanges(mapPrefix, keys, kv);
+	co_return krmDecodeRanges(mapPrefix, keys, kv);
 }
 
-ACTOR Future<RangeResult> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
-                                       Key mapPrefix,
-                                       KeyRange keys,
-                                       int limit,
-                                       int limitBytes) {
+Future<RangeResult> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
+                                 Key mapPrefix,
+                                 KeyRange keys,
+                                 int limit,
+                                 int limitBytes) {
 	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
 
-	state GetRangeLimits limits(limit, limitBytes);
+	GetRangeLimits limits(limit, limitBytes);
 	limits.minRows = 2;
-	RangeResult kv = wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
+	RangeResult kv = co_await tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits);
 
-	return krmDecodeRanges(mapPrefix, keys, kv);
+	co_return krmDecodeRanges(mapPrefix, keys, kv);
 }
 
 // Returns keys.begin, all transitional points in keys, and keys.end, and their values
-ACTOR Future<RangeResult> krmGetRangesUnaligned(Transaction* tr,
-                                                Key mapPrefix,
-                                                KeyRange keys,
-                                                int limit,
-                                                int limitBytes) {
+Future<RangeResult> krmGetRangesUnaligned(Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes) {
 	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
 
-	state GetRangeLimits limits(limit, limitBytes);
+	GetRangeLimits limits(limit, limitBytes);
 	limits.minRows = 2;
 	// wait to include the next highest row >= keys.end in the result, so since end is exclusive, we need +2 and
 	// !orEqual
 	RangeResult kv =
-	    wait(tr->getRange(lastLessOrEqual(withPrefix.begin), KeySelectorRef(withPrefix.end, false, +2), limits));
+	    co_await tr->getRange(lastLessOrEqual(withPrefix.begin), KeySelectorRef(withPrefix.end, false, +2), limits);
 
-	return krmDecodeRanges(mapPrefix, keys, kv, false);
+	co_return krmDecodeRanges(mapPrefix, keys, kv, false);
 }
 
-ACTOR Future<RangeResult> krmGetRangesUnaligned(Reference<ReadYourWritesTransaction> tr,
-                                                Key mapPrefix,
-                                                KeyRange keys,
-                                                int limit,
-                                                int limitBytes) {
+Future<RangeResult> krmGetRangesUnaligned(Reference<ReadYourWritesTransaction> tr,
+                                          Key mapPrefix,
+                                          KeyRange keys,
+                                          int limit,
+                                          int limitBytes) {
 	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
 
-	state GetRangeLimits limits(limit, limitBytes);
+	GetRangeLimits limits(limit, limitBytes);
 	limits.minRows = 2;
 	// wait to include the next highest row >= keys.end in the result, so since end is exclusive, we need +2 and
 	// !orEqual
 	RangeResult kv =
-	    wait(tr->getRange(lastLessOrEqual(withPrefix.begin), KeySelectorRef(withPrefix.end, false, +2), limits));
+	    co_await tr->getRange(lastLessOrEqual(withPrefix.begin), KeySelectorRef(withPrefix.end, false, +2), limits);
 
-	return krmDecodeRanges(mapPrefix, keys, kv, false);
+	co_return krmDecodeRanges(mapPrefix, keys, kv, false);
 }
 
 void krmSetPreviouslyEmptyRange(Transaction* tr,
@@ -176,14 +171,14 @@ void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
 	tr.set(trArena, withPrefix.end, oldEndValue);
 }
 
-ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
-	state KeyRange withPrefix =
+Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
+	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
 	RangeResult old =
-	    wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::True));
+	    co_await tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::True);
 
 	Value oldValue;
-	bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
+	bool hasResult = !old.empty() && old[0].key.startsWith(mapPrefix);
 	if (hasResult)
 		oldValue = old[0].value;
 
@@ -194,18 +189,16 @@ ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, V
 	tr->clear(withPrefix);
 	tr->set(withPrefix.begin, value);
 	tr->set(withPrefix.end, oldValue);
-
-	return Void();
 }
 
-ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
-	state KeyRange withPrefix =
+Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
+	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
 	RangeResult old =
-	    wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::True));
+	    co_await tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::True);
 
 	Value oldValue;
-	bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
+	bool hasResult = !old.empty() && old[0].key.startsWith(mapPrefix);
 	if (hasResult)
 		oldValue = old[0].value;
 
@@ -218,14 +211,12 @@ ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapP
 	// set keyVersionMap/end to oldValue: because end is exclusive here,
 	// but starting from end it might be covered by another range file, so set it to old value
 	tr->set(withPrefix.end, oldValue);
-
-	return Void();
 }
 
 // Sets a range of keys in a key range map, coalescing with adjacent regions if the values match
 // Ranges outside of maxRange will not be coalesced
 // CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
-ACTOR template <class Transaction>
+template <class Transaction>
 static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
                                            Key mapPrefix,
                                            KeyRange range,
@@ -233,24 +224,24 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
                                            Value value) {
 	ASSERT(maxRange.contains(range));
 
-	state KeyRange withPrefix =
+	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
-	state KeyRange maxWithPrefix =
+	KeyRange maxWithPrefix =
 	    KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
 
-	state std::vector<Future<RangeResult>> keys;
+	std::vector<Future<RangeResult>> keys;
 	keys.push_back(
 	    tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::True));
 	keys.push_back(
 	    tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, Snapshot::True));
-	wait(waitForAll(keys));
+	co_await waitForAll(keys);
 
 	// Determine how far to extend this range at the beginning
 	auto beginRange = keys[0].get();
-	bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix);
+	bool hasBegin = !beginRange.empty() && beginRange[0].key.startsWith(mapPrefix);
 	Value beginValue = hasBegin ? beginRange[0].value : ""_sr;
 
-	state Key beginKey = withPrefix.begin;
+	Key beginKey = withPrefix.begin;
 	if (beginValue == value) {
 		bool outsideRange = !hasBegin || beginRange[0].key < maxWithPrefix.begin;
 		beginKey = outsideRange ? maxWithPrefix.begin : beginRange[0].key;
@@ -258,7 +249,7 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 
 	// Determine how far to extend this range at the end
 	auto endRange = keys[1].get();
-	bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
+	bool hasEnd = !endRange.empty() && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
 	bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
 	               (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
 	Value existingValue = hasEnd ? endRange[0].value : ""_sr;
@@ -273,8 +264,8 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 	if (!conflictRange.empty())
 		tr->addReadConflictRange(conflictRange);
 
-	state Key endKey;
-	state Value endValue;
+	Key endKey;
+	Value endValue;
 
 	// Case 1: Coalesce completely with the following range
 	if (hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) {
@@ -299,8 +290,6 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
 	ASSERT(value != endValue || endKey == maxWithPrefix.end);
 	tr->set(beginKey, value);
 	tr->set(endKey, endValue);
-
-	return Void();
 }
 Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
                                    Key const& mapPrefix,
@@ -361,8 +350,8 @@ TEST_CASE("/keyrangemap/decoderange/aligned") {
 	// [""(start), A, B, C, D, E(end)]
 	decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(StringRef(), keyE), kv);
 	ASSERT(decodedRanges.size() == 6);
-	ASSERT(decodedRanges.front().key == StringRef());
-	ASSERT(decodedRanges.front().value == StringRef());
+	ASSERT(decodedRanges.front().key.empty());
+	ASSERT(decodedRanges.front().value.empty());
 	ASSERT(decodedRanges.back().key == keyE);
 	ASSERT(decodedRanges.back().value == keyD);
 
@@ -413,10 +402,10 @@ TEST_CASE("/keyrangemap/decoderange/unaligned") {
 	// [""(start), A, B, C, D, E(end)]
 	decodedRanges = krmDecodeRanges(prefix, KeyRangeRef(StringRef(), keyE), kv, false);
 	ASSERT(decodedRanges.size() == 6);
-	ASSERT(decodedRanges.front().key == StringRef());
-	ASSERT(decodedRanges.front().value == StringRef());
+	ASSERT(decodedRanges.front().key.empty());
+	ASSERT(decodedRanges.front().value.empty());
 	ASSERT(decodedRanges.back().key == keyE);
 	ASSERT(decodedRanges.back().value == keyD);
 
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbclient/KnobValue.cpp b/fdbclient/KnobValue.cpp
index c63473af1af..bb466855ed2 100644
--- a/fdbclient/KnobValue.cpp
+++ b/fdbclient/KnobValue.cpp
@@ -21,7 +21,6 @@
 #include "fdbclient/KnobValue.h"
 #include "fdbclient/Tuple.h"
 #include "flow/flow.h"
-#include "flow/Trace.h"
 
 namespace {
 
diff --git a/fdbclient/LocalClientAPI.cpp b/fdbclient/LocalClientAPI.cpp
index d404ede86d2..1e6597a9e4b 100644
--- a/fdbclient/LocalClientAPI.cpp
+++ b/fdbclient/LocalClientAPI.cpp
@@ -18,10 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbclient/LocalClientAPI.h"
+#include "LocalClientAPI.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 
 IClientApi* getLocalClientAPI() {
 	static IClientApi* api = new ThreadSafeApi();
 	return api;
-}
\ No newline at end of file
+}
diff --git a/fdbclient/include/fdbclient/LocalClientAPI.h b/fdbclient/LocalClientAPI.h
similarity index 99%
rename from fdbclient/include/fdbclient/LocalClientAPI.h
rename to fdbclient/LocalClientAPI.h
index 8569cd40485..4d2a17cd73f 100644
--- a/fdbclient/include/fdbclient/LocalClientAPI.h
+++ b/fdbclient/LocalClientAPI.h
@@ -25,4 +25,4 @@
 #include "fdbclient/IClientApi.h"
 
 IClientApi* getLocalClientAPI();
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.cpp
similarity index 68%
rename from fdbclient/ManagementAPI.actor.cpp
rename to fdbclient/ManagementAPI.cpp
index ae7b713d126..dfec09d5d13 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.cpp
@@ -1,5 +1,5 @@
 /*
- * ManagementAPI.actor.cpp
+ * ManagementAPI.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,7 +25,7 @@
 
 #include "fdbclient/BulkDumping.h"
 #include "fdbclient/BulkLoading.h"
-#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/GenericManagementAPI.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/RangeLock.h"
 #include "flow/Error.h"
@@ -36,7 +36,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 
 #include "fdbclient/SystemData.h"
 #include "fdbclient/NativeAPI.actor.h"
@@ -51,7 +51,7 @@
 #include "fdbclient/Schemas.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 bool isInteger(const std::string& s) {
 	if (s.empty())
@@ -270,59 +270,40 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 	if (mode == "single") {
 		redundancy = "1";
 		log_replicas = "1";
-		storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
+		storagePolicy = tLogPolicy = makeReference<PolicyOne>();
 
 	} else if (mode == "double" || mode == "fast_recovery_double") {
 		redundancy = "2";
 		log_replicas = "2";
-		storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		storagePolicy = tLogPolicy = makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>());
 	} else if (mode == "triple" || mode == "fast_recovery_triple") {
 		redundancy = "3";
 		log_replicas = "3";
-		storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		storagePolicy = tLogPolicy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
 	} else if (mode == "three_datacenter" || mode == "multi_dc") {
 		redundancy = "6";
 		log_replicas = "4";
-		storagePolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(3,
-		                     "dcid",
-		                     Reference<IReplicationPolicy>(
-		                         new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
-		tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2,
-		                     "dcid",
-		                     Reference<IReplicationPolicy>(
-		                         new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+		storagePolicy = makeReference<PolicyAcross>(
+		    3, "dcid", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
+		tLogPolicy = makeReference<PolicyAcross>(
+		    2, "dcid", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 	} else if (mode == "three_datacenter_fallback") {
 		redundancy = "4";
 		log_replicas = "4";
-		storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2,
-		                     "dcid",
-		                     Reference<IReplicationPolicy>(
-		                         new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+		storagePolicy = tLogPolicy = makeReference<PolicyAcross>(
+		    2, "dcid", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 	} else if (mode == "three_data_hall") {
 		redundancy = "3";
 		log_replicas = "4";
-		storagePolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(3, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
-		tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2,
-		                     "data_hall",
-		                     Reference<IReplicationPolicy>(
-		                         new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+		storagePolicy = makeReference<PolicyAcross>(3, "data_hall", makeReference<PolicyOne>());
+		tLogPolicy = makeReference<PolicyAcross>(
+		    2, "data_hall", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 	} else if (mode == "three_data_hall_fallback") {
 		redundancy = "2";
 		log_replicas = "4";
-		storagePolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
-		tLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2,
-		                     "data_hall",
-		                     Reference<IReplicationPolicy>(
-		                         new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+		storagePolicy = makeReference<PolicyAcross>(2, "data_hall", makeReference<PolicyOne>());
+		tLogPolicy = makeReference<PolicyAcross>(
+		    2, "data_hall", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 	} else
 		redundancySpecified = false;
 	if (redundancySpecified) {
@@ -350,25 +331,20 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 	} else if (mode == "remote_single") {
 		remote_redundancy = "1";
 		remote_log_replicas = "1";
-		remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
+		remoteTLogPolicy = makeReference<PolicyOne>();
 	} else if (mode == "remote_double") {
 		remote_redundancy = "2";
 		remote_log_replicas = "2";
-		remoteTLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		remoteTLogPolicy = makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>());
 	} else if (mode == "remote_triple") {
 		remote_redundancy = "3";
 		remote_log_replicas = "3";
-		remoteTLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		remoteTLogPolicy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
 	} else if (mode == "remote_three_data_hall") { // FIXME: not tested in simulation
 		remote_redundancy = "3";
 		remote_log_replicas = "4";
-		remoteTLogPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(2,
-		                     "data_hall",
-		                     Reference<IReplicationPolicy>(
-		                         new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
+		remoteTLogPolicy = makeReference<PolicyAcross>(
+		    2, "data_hall", makeReference<PolicyAcross>(2, "zoneid", makeReference<PolicyOne>()));
 	} else
 		remoteRedundancySpecified = false;
 	if (remoteRedundancySpecified) {
@@ -388,13 +364,13 @@ ConfigurationResult buildConfiguration(std::vector<StringRef> const& modeTokens,
 	for (auto it : modeTokens) {
 		std::string mode = it.toString();
 		auto m = configForToken(mode);
-		if (!m.size()) {
+		if (m.empty()) {
 			TraceEvent(SevWarnAlways, "UnknownOption").detail("Option", mode);
 			return ConfigurationResult::UNKNOWN_OPTION;
 		}
 
 		for (auto t = m.begin(); t != m.end(); ++t) {
-			if (outConf.count(t->first)) {
+			if (outConf.contains(t->first)) {
 				TraceEvent(SevWarnAlways, "ConflictingOption")
 				    .detail("Option", t->first)
 				    .detail("Value", t->second)
@@ -405,34 +381,34 @@ ConfigurationResult buildConfiguration(std::vector<StringRef> const& modeTokens,
 		}
 	}
 	auto p = configKeysPrefix.toString();
-	if (!outConf.count(p + "storage_replication_policy") && outConf.count(p + "storage_replicas")) {
+	if (!outConf.contains(p + "storage_replication_policy") && outConf.contains(p + "storage_replicas")) {
 		int storageCount = stoi(outConf[p + "storage_replicas"]);
-		Reference<IReplicationPolicy> storagePolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(storageCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		Reference<IReplicationPolicy> storagePolicy =
+		    makeReference<PolicyAcross>(storageCount, "zoneid", makeReference<PolicyOne>());
 		BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy()));
 		serializeReplicationPolicy(policyWriter, storagePolicy);
 		outConf[p + "storage_replication_policy"] = policyWriter.toValue().toString();
 	}
 
-	if (!outConf.count(p + "log_replication_policy") && outConf.count(p + "log_replicas")) {
+	if (!outConf.contains(p + "log_replication_policy") && outConf.contains(p + "log_replicas")) {
 		int logCount = stoi(outConf[p + "log_replicas"]);
-		Reference<IReplicationPolicy> logPolicy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(logCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
+		Reference<IReplicationPolicy> logPolicy =
+		    makeReference<PolicyAcross>(logCount, "zoneid", makeReference<PolicyOne>());
 		BinaryWriter policyWriter(IncludeVersion(ProtocolVersion::withReplicationPolicy()));
 		serializeReplicationPolicy(policyWriter, logPolicy);
 		outConf[p + "log_replication_policy"] = policyWriter.toValue().toString();
 	}
-	if (outConf.count(p + "istss")) {
+	if (outConf.contains(p + "istss")) {
 		// redo config parameters to be tss config instead of normal config
 
 		// save param values from parsing as a normal config
-		bool isNew = outConf.count(p + "initialized");
+		bool isNew = outConf.contains(p + "initialized");
 		Optional<std::string> count;
 		Optional<std::string> storageEngine;
-		if (outConf.count(p + "count")) {
+		if (outConf.contains(p + "count")) {
 			count = Optional<std::string>(outConf[p + "count"]);
 		}
-		if (outConf.count(p + "storage_engine")) {
+		if (outConf.contains(p + "storage_engine")) {
 			storageEngine = Optional<std::string>(outConf[p + "storage_engine"]);
 		}
 
@@ -477,56 +453,84 @@ bool isCompleteConfiguration(std::map<std::string, std::string> const& options)
 	       options.count(p + "storage_engine") == 1;
 }
 
-ACTOR Future<Void> disableBackupWorker(Database cx) {
-	DatabaseConfiguration configuration = wait(getDatabaseConfiguration(cx));
+Future<Void> disableBackupWorker(Database cx) {
+	DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
 	if (!configuration.backupWorkerEnabled) {
 		TraceEvent("BackupWorkerAlreadyDisabled");
-		return Void();
+		co_return;
 	}
-	ConfigurationResult res = wait(ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=0", true));
+	ConfigurationResult res = co_await ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=0", true);
 	if (res != ConfigurationResult::SUCCESS) {
 		TraceEvent("BackupWorkerDisableFailed").detail("Result", res);
 		throw operation_failed();
 	}
-	return Void();
 }
 
-ACTOR Future<Void> enableBackupWorker(Database cx) {
-	DatabaseConfiguration configuration = wait(getDatabaseConfiguration(cx));
+Future<Void> enableBackupWorker(Database cx) {
+	DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
 	if (configuration.backupWorkerEnabled) {
 		TraceEvent("BackupWorkerAlreadyEnabled");
-		return Void();
+		co_return;
 	}
-	ConfigurationResult res = wait(ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=1", true));
+	ConfigurationResult res = co_await ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=1", true);
 	if (res != ConfigurationResult::SUCCESS) {
 		TraceEvent("BackupWorkerEnableFailed").detail("Result", res);
 		throw operation_failed();
 	}
-	return Void();
 }
 
-ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Transaction* tr, bool useSystemPriority) {
+Future<Void> enableRangeBackupWorker(Database cx) {
+	DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
+	if (configuration.rangeBackupWorkerEnabled) {
+		TraceEvent("RangeBackupWorkerAlreadyEnabled");
+		co_return;
+	}
+	ConfigurationResult res =
+	    co_await ManagementAPI::changeConfig(cx.getReference(), "range_backup_worker_enabled:=1", true);
+	if (res != ConfigurationResult::SUCCESS) {
+		TraceEvent("RangeBackupWorkerEnableFailed").detail("Result", res);
+		throw operation_failed();
+	}
+}
+
+Future<Void> disableRangeBackupWorker(Database cx) {
+	DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
+	if (!configuration.rangeBackupWorkerEnabled) {
+		TraceEvent("RangeBackupWorkerAlreadyDisabled");
+		co_return;
+	}
+	ConfigurationResult res =
+	    co_await ManagementAPI::changeConfig(cx.getReference(), "range_backup_worker_enabled:=0", true);
+	if (res != ConfigurationResult::SUCCESS) {
+		TraceEvent("RangeBackupWorkerDisableFailed").detail("Result", res);
+		throw operation_failed();
+	}
+}
+
+Future<DatabaseConfiguration> getDatabaseConfiguration(Transaction* tr, bool useSystemPriority) {
 	if (useSystemPriority) {
 		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	}
 	tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-	RangeResult res = wait(tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
+	RangeResult res = co_await tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(res.size() < CLIENT_KNOBS->TOO_MANY);
 	DatabaseConfiguration config;
 	config.fromKeyValues((VectorRef<KeyValueRef>)res);
-	return config;
+	co_return config;
 }
 
-ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx, bool useSystemPriority) {
-	state Transaction tr(cx);
-	loop {
+Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx, bool useSystemPriority) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			DatabaseConfiguration config = wait(getDatabaseConfiguration(&tr, useSystemPriority));
-			return config;
+			DatabaseConfiguration config = co_await getDatabaseConfiguration(&tr, useSystemPriority);
+			co_return config;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -754,7 +758,7 @@ ConfigureAutoResult parseConfig(StatusObject const& status) {
 
 	// if one process on a machine is transaction class, make them all transaction class
 	for (auto& it : count_processes) {
-		if (machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) {
+		if (machinesWithTransaction.contains(it.first.second) && !machinesWithStorage.contains(it.first.second)) {
 			for (auto& proc : it.second) {
 				if (proc.second == ProcessClass::UnsetClass &&
 				    proc.second.classSource() == ProcessClass::CommandLineSource) {
@@ -773,7 +777,7 @@ ConfigureAutoResult parseConfig(StatusObject const& status) {
 		if (machinesWithTransaction.size() >= logCount && totalTransactionProcesses >= desiredTotalTransactionProcesses)
 			break;
 
-		if (!machinesWithTransaction.count(it.first.second) && !machinesWithStorage.count(it.first.second)) {
+		if (!machinesWithTransaction.contains(it.first.second) && !machinesWithStorage.contains(it.first.second)) {
 			for (auto& proc : it.second) {
 				if (proc.second == ProcessClass::UnsetClass &&
 				    proc.second.classSource() == ProcessClass::CommandLineSource) {
@@ -799,11 +803,11 @@ ConfigureAutoResult parseConfig(StatusObject const& status) {
 	return result;
 }
 
-ACTOR Future<std::vector<ProcessData>> getWorkers(Transaction* tr) {
-	state Future<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
-	state Future<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
+Future<std::vector<ProcessData>> getWorkers(Transaction* tr) {
+	Future<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
+	Future<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
 
-	wait(success(processClasses) && success(processData));
+	co_await (success(processClasses) && success(processData));
 	ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
 
@@ -827,37 +831,41 @@ ACTOR Future<std::vector<ProcessData>> getWorkers(Transaction* tr) {
 			results.push_back(data);
 	}
 
-	return results;
+	co_return results;
 }
 
-ACTOR Future<std::vector<ProcessData>> getWorkers(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<std::vector<ProcessData>> getWorkers(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::vector<ProcessData> workers = wait(getWorkers(&tr));
-			return workers;
+			std::vector<ProcessData> workers = co_await getWorkers(&tr);
+			co_return workers;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Optional<ClusterConnectionString>> getConnectionString(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<Optional<ClusterConnectionString>> getConnectionString(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			Optional<Value> currentKey = wait(tr.get(coordinatorsKey));
+			Optional<Value> currentKey = co_await tr.get(coordinatorsKey);
 			if (!currentKey.present())
-				return Optional<ClusterConnectionString>();
-			return ClusterConnectionString(currentKey.get().toString());
+				co_return Optional<ClusterConnectionString>();
+			co_return ClusterConnectionString(currentKey.get().toString());
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -865,20 +873,20 @@ static std::vector<std::string> connectionStrings;
 
 namespace {
 
-ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromStorageServer(Transaction* tr) {
+Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromStorageServer(Transaction* tr) {
 
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-	state int retryTimes = 0;
-	loop {
+	int retryTimes = 0;
+	while (true) {
 		if (retryTimes >= CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_TIMES) {
-			return Optional<ClusterConnectionString>();
+			co_return Optional<ClusterConnectionString>();
 		}
 
-		state Optional<Value> currentKey = wait(tr->get(coordinatorsKey));
+		Optional<Value> currentKey = co_await tr->get(coordinatorsKey);
 		if (g_network->isSimulated() && currentKey.present()) {
 			// If the change coordinators request succeeded, the coordinators
 			// should have changed to the connection string of the most
@@ -887,7 +895,7 @@ ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromSt
 			// and we are breaking the promises we make with
 			// commit_unknown_result (the transaction must no longer be in
 			// progress when receiving commit_unknown_result).
-			int n = connectionStrings.size() > 0 ? connectionStrings.size() - 1 : 0; // avoid underflow
+			int n = !connectionStrings.empty() ? connectionStrings.size() - 1 : 0; // avoid underflow
 			for (int i = 0; i < n; ++i) {
 				ASSERT(currentKey.get() != connectionStrings.at(i));
 			}
@@ -896,69 +904,71 @@ ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromSt
 		if (!currentKey.present()) {
 			// Someone deleted this key entirely?
 			++retryTimes;
-			wait(delay(CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_DELAY));
+			co_await delay(CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_DELAY);
 			continue;
 		}
 
-		state ClusterConnectionString clusterConnectionString(currentKey.get().toString());
+		ClusterConnectionString clusterConnectionString(currentKey.get().toString());
 		if (tr->getDatabase()->getConnectionRecord() &&
 		    clusterConnectionString.clusterKeyName().toString() !=
 		        tr->getDatabase()->getConnectionRecord()->getConnectionString().clusterKeyName()) {
 			// Someone changed the "name" of the database??
 			++retryTimes;
-			wait(delay(CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_DELAY));
+			co_await delay(CLIENT_KNOBS->CHANGE_QUORUM_BAD_STATE_RETRY_DELAY);
 			continue;
 		}
 
-		return clusterConnectionString;
+		co_return clusterConnectionString;
 	}
 }
 
-ACTOR Future<Void> resetPreviousCoordinatorsKey(Database cx) {
-	loop {
+Future<Void> resetPreviousCoordinatorsKey(Database cx) {
+	while (true) {
 		// When the change coordinators transaction succeeds, it uses the
 		// special key space error message to return a message to the client.
 		// This causes the underlying transaction to not be committed. In order
 		// to make sure we clear the previous coordinators key, we have to use
 		// a new transaction here.
-		state Reference<ReadYourWritesTransaction> clearTr = makeReference<ReadYourWritesTransaction>(cx);
+		auto clearTr = makeReference<ReadYourWritesTransaction>(cx);
+		Error err;
 		try {
 			clearTr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			clearTr->clear(previousCoordinatorsKey);
-			wait(clearTr->commit());
-			return Void();
+			co_await clearTr->commit();
+			co_return;
 		} catch (Error& e2) {
-			wait(clearTr->onError(e2));
+			err = e2;
 		}
+		co_await clearTr->onError(err);
 	}
 }
 
 } // namespace
 
-ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
-                                                               ClusterConnectionString* conn,
-                                                               std::string newName) {
+Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
+                                                         ClusterConnectionString* conn,
+                                                         std::string newName) {
 	TraceEvent("ChangeQuorumCheckerStart").detail("NewConnectionString", conn->toString());
-	state Optional<ClusterConnectionString> clusterConnectionStringOptional =
-	    wait(getClusterConnectionStringFromStorageServer(tr));
+	Optional<ClusterConnectionString> clusterConnectionStringOptional =
+	    co_await getClusterConnectionStringFromStorageServer(tr);
 
 	if (!clusterConnectionStringOptional.present()) {
-		return CoordinatorsResult::BAD_DATABASE_STATE;
+		co_return CoordinatorsResult::BAD_DATABASE_STATE;
 	}
 
 	// The cluster connection string stored in the storage server
-	state ClusterConnectionString old = clusterConnectionStringOptional.get();
+	ClusterConnectionString old = clusterConnectionStringOptional.get();
 
 	if (conn->hostnames.size() + conn->coords.size() == 0) {
 		conn->hostnames = old.hostnames;
 		conn->coords = old.coords;
 	}
-	state std::vector<NetworkAddress> desiredCoordinators = wait(conn->tryResolveHostnames());
+	std::vector<NetworkAddress> desiredCoordinators = co_await conn->tryResolveHostnames();
 	if (desiredCoordinators.size() != conn->hostnames.size() + conn->coords.size()) {
 		TraceEvent("ChangeQuorumCheckerEarlyTermination")
 		    .detail("Reason", "One or more hostnames are unresolvable")
 		    .backtrace();
-		return CoordinatorsResult::COORDINATOR_UNREACHABLE;
+		co_return CoordinatorsResult::COORDINATOR_UNREACHABLE;
 	}
 
 	if (newName.empty()) {
@@ -974,10 +984,10 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 			// Introduce a random delay in simulation to allow processes to be
 			// killed before previousCoordinatorKeys has been reset. This helps
 			// exercise coordinator change edge cases around key cleanup.
-			wait(delay(deterministicRandom()->random01() * 10));
+			co_await delay(deterministicRandom()->random01() * 10);
 		}
-		wait(resetPreviousCoordinatorsKey(tr->getDatabase()));
-		return CoordinatorsResult::SAME_NETWORK_ADDRESSES;
+		co_await resetPreviousCoordinatorsKey(tr->getDatabase());
+		co_return CoordinatorsResult::SAME_NETWORK_ADDRESSES;
 	}
 
 	conn->parseKey(newName + ':' + deterministicRandom()->randomAlphaNumeric(32));
@@ -996,9 +1006,9 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 				continue;
 			}
 
-			g_simulator->protectedAddresses.insert(process->addresses.address);
+			g_simulator->protectAddress(process->addresses.address);
 			if (addresses.secondaryAddress.present()) {
-				g_simulator->protectedAddresses.insert(process->addresses.secondaryAddress.get());
+				g_simulator->protectAddress(process->addresses.secondaryAddress.get());
 			}
 			TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace();
 			protectedCount++;
@@ -1011,12 +1021,12 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 			    .detail("MinimumRequired", minimumCoordinators)
 			    .detail("ConnectionString", conn->toString());
 
-			return CoordinatorsResult::COORDINATOR_UNREACHABLE;
+			co_return CoordinatorsResult::COORDINATOR_UNREACHABLE;
 		}
 	}
 
 	std::vector<Future<Optional<LeaderInfo>>> leaderServers;
-	ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(*conn)));
+	ClientCoordinators coord(makeReference<ClusterConnectionMemoryRecord>(*conn));
 
 	leaderServers.reserve(coord.clientLeaderServers.size());
 	for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
@@ -1032,47 +1042,45 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 		}
 	}
 
-	choose {
-		when(wait(waitForAll(leaderServers))) {}
-		when(wait(delay(5.0))) {
-			return CoordinatorsResult::COORDINATOR_UNREACHABLE;
-		}
+	auto leaderServersResult = co_await timeout(waitForAll(leaderServers), 5.0);
+	if (!leaderServersResult.present()) {
+		co_return CoordinatorsResult::COORDINATOR_UNREACHABLE;
 	}
 	TraceEvent("ChangeQuorumCheckerSetCoordinatorsKey")
 	    .detail("CurrentCoordinators", old.toString())
 	    .detail("NewCoordinators", conn->toString());
 	tr->set(coordinatorsKey, conn->toString());
-	return Optional<CoordinatorsResult>();
+	co_return Optional<CoordinatorsResult>();
 }
 
-ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change) {
-	state Transaction tr(cx);
-	state int retries = 0;
-	state std::vector<NetworkAddress> desiredCoordinators;
-	state int notEnoughMachineResults = 0;
+Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change) {
+	Transaction tr(cx);
+	int retries = 0;
+	std::vector<NetworkAddress> desiredCoordinators;
+	int notEnoughMachineResults = 0;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			state Optional<ClusterConnectionString> clusterConnectionStringOptional =
-			    wait(getClusterConnectionStringFromStorageServer(&tr));
+			Optional<ClusterConnectionString> clusterConnectionStringOptional =
+			    co_await getClusterConnectionStringFromStorageServer(&tr);
 
 			if (!clusterConnectionStringOptional.present()) {
-				return CoordinatorsResult::BAD_DATABASE_STATE;
+				co_return CoordinatorsResult::BAD_DATABASE_STATE;
 			}
 
 			// The cluster connection string stored in the storage server
-			state ClusterConnectionString oldClusterConnectionString = clusterConnectionStringOptional.get();
-			state Key oldClusterKeyName = oldClusterConnectionString.clusterKeyName();
-
-			state std::vector<NetworkAddress> oldCoordinators = wait(oldClusterConnectionString.tryResolveHostnames());
-			state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
-			if (!desiredCoordinators.size()) {
-				std::vector<NetworkAddress> _desiredCoordinators = wait(
-				    change->getDesiredCoordinators(&tr,
-				                                   oldCoordinators,
-				                                   Reference<ClusterConnectionMemoryRecord>(
-				                                       new ClusterConnectionMemoryRecord(oldClusterConnectionString)),
-				                                   result));
+			ClusterConnectionString oldClusterConnectionString = clusterConnectionStringOptional.get();
+			Key oldClusterKeyName = oldClusterConnectionString.clusterKeyName();
+
+			std::vector<NetworkAddress> oldCoordinators = co_await oldClusterConnectionString.tryResolveHostnames();
+			CoordinatorsResult result = CoordinatorsResult::SUCCESS;
+			if (desiredCoordinators.empty()) {
+				std::vector<NetworkAddress> _desiredCoordinators = co_await change->getDesiredCoordinators(
+				    &tr,
+				    oldCoordinators,
+				    makeReference<ClusterConnectionMemoryRecord>(oldClusterConnectionString),
+				    result);
 				desiredCoordinators = _desiredCoordinators;
 			}
 
@@ -1080,14 +1088,14 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				// we could get not_enough_machines if we happen to see the database while the cluster controller is
 				// updating the worker list, so make sure it happens twice before returning a failure
 				notEnoughMachineResults++;
-				wait(delay(1.0));
+				co_await delay(1.0);
 				tr.reset();
 				continue;
 			}
 			if (result != CoordinatorsResult::SUCCESS)
-				return result;
-			if (!desiredCoordinators.size())
-				return CoordinatorsResult::INVALID_NETWORK_ADDRESSES;
+				co_return result;
+			if (desiredCoordinators.empty())
+				co_return CoordinatorsResult::INVALID_NETWORK_ADDRESSES;
 			std::sort(desiredCoordinators.begin(), desiredCoordinators.end());
 
 			std::string newName = change->getDesiredClusterKeyName();
@@ -1095,20 +1103,20 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				newName = oldClusterKeyName.toString();
 
 			if (oldCoordinators == desiredCoordinators && oldClusterKeyName == newName)
-				return retries ? CoordinatorsResult::SUCCESS : CoordinatorsResult::SAME_NETWORK_ADDRESSES;
+				co_return retries ? CoordinatorsResult::SUCCESS : CoordinatorsResult::SAME_NETWORK_ADDRESSES;
 
-			state ClusterConnectionString newClusterConnectionString(
+			ClusterConnectionString newClusterConnectionString(
 			    desiredCoordinators, StringRef(newName + ':' + deterministicRandom()->randomAlphaNumeric(32)));
-			state Key newClusterKeyName = newClusterConnectionString.clusterKeyName();
+			Key newClusterKeyName = newClusterConnectionString.clusterKeyName();
 
 			if (g_network->isSimulated()) {
 				for (int i = 0; i < (desiredCoordinators.size() / 2) + 1; i++) {
 					auto process = g_simulator->getProcessByAddress(desiredCoordinators[i]);
 					ASSERT(process->isReliable() || process->rebooting);
 
-					g_simulator->protectedAddresses.insert(process->addresses.address);
+					g_simulator->protectAddress(process->addresses.address);
 					if (process->addresses.secondaryAddress.present()) {
-						g_simulator->protectedAddresses.insert(process->addresses.secondaryAddress.get());
+						g_simulator->protectAddress(process->addresses.secondaryAddress.get());
 					}
 					TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace();
 				}
@@ -1120,15 +1128,16 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 			CODE_PROBE(oldClusterKeyName != newClusterKeyName, "Quorum change with new name");
 			CODE_PROBE(oldClusterKeyName == newClusterKeyName, "Quorum change with unchanged name");
 
-			state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
-			state ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(
+			std::vector<Future<Optional<LeaderInfo>>> leaderServers;
+			ClientCoordinators coord(Reference<ClusterConnectionMemoryRecord>(
 			    new ClusterConnectionMemoryRecord(newClusterConnectionString)));
 			// check if allowed to modify the cluster descriptor
 			if (!change->getDesiredClusterKeyName().empty()) {
 				CheckDescriptorMutableReply mutabilityReply =
-				    wait(coord.clientLeaderServers[0].checkDescriptorMutable.getReply(CheckDescriptorMutableRequest()));
+				    co_await coord.clientLeaderServers[0].checkDescriptorMutable.getReply(
+				        CheckDescriptorMutableRequest());
 				if (!mutabilityReply.isMutable) {
-					return CoordinatorsResult::BAD_DATABASE_STATE;
+					co_return CoordinatorsResult::BAD_DATABASE_STATE;
 				}
 			}
 			leaderServers.reserve(coord.clientLeaderServers.size());
@@ -1136,22 +1145,21 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				leaderServers.push_back(retryBrokenPromise(coord.clientLeaderServers[i].getLeader,
 				                                           GetLeaderRequest(coord.clusterKey, UID()),
 				                                           TaskPriority::CoordinationReply));
-			choose {
-				when(wait(waitForAll(leaderServers))) {}
-				when(wait(delay(5.0))) {
-					return CoordinatorsResult::COORDINATOR_UNREACHABLE;
-				}
+			auto leaderServersResult = co_await timeout(waitForAll(leaderServers), 5.0);
+			if (!leaderServersResult.present()) {
+				co_return CoordinatorsResult::COORDINATOR_UNREACHABLE;
 			}
 
 			tr.set(coordinatorsKey, newClusterConnectionString.toString());
 
-			wait(tr.commit());
+			co_await tr.commit();
 			ASSERT(false); // commit should fail, but the value has changed
 		} catch (Error& e) {
-			TraceEvent("RetryQuorumChange").error(e).detail("Retries", retries);
-			wait(tr.onError(e));
-			++retries;
+			err = e;
 		}
+		TraceEvent("RetryQuorumChange").error(err).detail("Retries", retries);
+		co_await tr.onError(err);
+		++retries;
 	}
 }
 
@@ -1169,7 +1177,7 @@ struct NameQuorumChange final : IQuorumChange {
 	std::string getDesiredClusterKeyName() const override { return newName; }
 };
 Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other) {
-	return Reference<IQuorumChange>(new NameQuorumChange(name, other));
+	return makeReference<NameQuorumChange>(name, other);
 }
 
 struct AutoQuorumChange final : IQuorumChange {
@@ -1183,37 +1191,37 @@ struct AutoQuorumChange final : IQuorumChange {
 		return getDesired(Reference<AutoQuorumChange>::addRef(this), tr, oldCoordinators, ccr, &err);
 	}
 
-	ACTOR static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) {
-		state Future<Optional<Value>> fStorageReplicas = tr->get("storage_replicas"_sr.withPrefix(configKeysPrefix));
-		state Future<Optional<Value>> fLogReplicas = tr->get("log_replicas"_sr.withPrefix(configKeysPrefix));
-		wait(success(fStorageReplicas) && success(fLogReplicas));
+	static Future<int> getRedundancy(AutoQuorumChange* self, Transaction* tr) {
+		Future<Optional<Value>> fStorageReplicas = tr->get("storage_replicas"_sr.withPrefix(configKeysPrefix));
+		Future<Optional<Value>> fLogReplicas = tr->get("log_replicas"_sr.withPrefix(configKeysPrefix));
+		co_await (success(fStorageReplicas) && success(fLogReplicas));
 		int redundancy = std::min(atoi(fStorageReplicas.get().get().toString().c_str()),
 		                          atoi(fLogReplicas.get().get().toString().c_str()));
 
-		return redundancy;
+		co_return redundancy;
 	}
 
-	ACTOR static Future<bool> isAcceptable(AutoQuorumChange* self,
-	                                       Transaction* tr,
-	                                       std::vector<NetworkAddress> oldCoordinators,
-	                                       Reference<IClusterConnectionRecord> ccr,
-	                                       int desiredCount,
-	                                       std::set<AddressExclusion>* excluded) {
+	static Future<bool> isAcceptable(AutoQuorumChange* self,
+	                                 Transaction* tr,
+	                                 std::vector<NetworkAddress> oldCoordinators,
+	                                 Reference<IClusterConnectionRecord> ccr,
+	                                 int desiredCount,
+	                                 std::set<AddressExclusion>* excluded) {
 		ClusterConnectionString cs = ccr->getConnectionString();
 		if (oldCoordinators.size() != cs.hostnames.size() + cs.coords.size()) {
-			return false;
+			co_return false;
 		}
 
 		// Are there enough coordinators for the redundancy level?
 		if (oldCoordinators.size() < desiredCount)
-			return false;
+			co_return false;
 		if (oldCoordinators.size() % 2 != 1)
-			return false;
+			co_return false;
 
 		// Check exclusions
 		for (auto& c : oldCoordinators) {
 			if (addressExcluded(*excluded, c))
-				return false;
+				co_return false;
 		}
 
 		// Check locality
@@ -1221,7 +1229,7 @@ struct AutoQuorumChange final : IQuorumChange {
 		std::sort(oldCoordinators.begin(), oldCoordinators.end());
 		for (int i = 1; i < oldCoordinators.size(); i++)
 			if (oldCoordinators[i - 1].ip == oldCoordinators[i].ip)
-				return false; // Multiple coordinators share an IP
+				co_return false; // Multiple coordinators share an IP
 
 		// Check availability
 		ClientCoordinators coord(ccr);
@@ -1240,39 +1248,39 @@ struct AutoQuorumChange final : IQuorumChange {
 			}
 		}
 		Optional<std::vector<Optional<LeaderInfo>>> results =
-		    wait(timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY));
+		    co_await timeout(getAll(leaderServers), CLIENT_KNOBS->IS_ACCEPTABLE_DELAY);
 		if (!results.present()) {
-			return false;
+			co_return false;
 		} // Not all responded
 		for (auto& r : results.get()) {
 			if (!r.present()) {
-				return false; // Coordinator doesn't know about this database?
+				co_return false; // Coordinator doesn't know about this database?
 			}
 		}
 
-		return true; // The status quo seems fine
+		co_return true; // The status quo seems fine
 	}
 
-	ACTOR static Future<std::vector<NetworkAddress>> getDesired(Reference<AutoQuorumChange> self,
-	                                                            Transaction* tr,
-	                                                            std::vector<NetworkAddress> oldCoordinators,
-	                                                            Reference<IClusterConnectionRecord> ccr,
-	                                                            CoordinatorsResult* err) {
-		state int desiredCount = self->desired;
+	static Future<std::vector<NetworkAddress>> getDesired(Reference<AutoQuorumChange> self,
+	                                                      Transaction* tr,
+	                                                      std::vector<NetworkAddress> oldCoordinators,
+	                                                      Reference<IClusterConnectionRecord> ccr,
+	                                                      CoordinatorsResult* err) {
+		int desiredCount = self->desired;
 
 		if (desiredCount == -1) {
-			int redundancy = wait(getRedundancy(self.getPtr(), tr));
+			int redundancy = co_await getRedundancy(self.getPtr(), tr);
 			desiredCount = redundancy * 2 - 1;
 		}
 
-		std::vector<AddressExclusion> excl = wait(getAllExcludedServers(tr));
-		state std::set<AddressExclusion> excluded(excl.begin(), excl.end());
+		std::vector<AddressExclusion> excl = co_await getAllExcludedServers(tr);
+		std::set<AddressExclusion> excluded(excl.begin(), excl.end());
 
-		std::vector<ProcessData> _workers = wait(getWorkers(tr));
-		state std::vector<ProcessData> workers = _workers;
+		std::vector<ProcessData> _workers = co_await getWorkers(tr);
+		std::vector<ProcessData> workers = _workers;
 
 		std::map<NetworkAddress, LocalityData> addr_locality;
-		for (auto w : workers)
+		for (const auto& w : workers)
 			addr_locality[w.address] = w.locality;
 
 		// since we don't have the locality data for oldCoordinators:
@@ -1282,7 +1290,7 @@ struct AutoQuorumChange final : IQuorumChange {
 		std::set<Optional<Standalone<StringRef>>> checkDuplicates;
 		for (auto addr : oldCoordinators) {
 			auto findResult = addr_locality.find(addr);
-			if (findResult == addr_locality.end() || checkDuplicates.count(findResult->second.zoneId())) {
+			if (findResult == addr_locality.end() || checkDuplicates.contains(findResult->second.zoneId())) {
 				checkAcceptable = false;
 				break;
 			}
@@ -1290,10 +1298,10 @@ struct AutoQuorumChange final : IQuorumChange {
 		}
 
 		if (checkAcceptable) {
-			bool ok = wait(isAcceptable(self.getPtr(), tr, oldCoordinators, ccr, desiredCount, &excluded));
+			bool ok = co_await isAcceptable(self.getPtr(), tr, oldCoordinators, ccr, desiredCount, &excluded);
 			if (ok) {
 				*err = CoordinatorsResult::SAME_NETWORK_ADDRESSES;
-				return oldCoordinators;
+				co_return oldCoordinators;
 			}
 		}
 
@@ -1308,12 +1316,12 @@ struct AutoQuorumChange final : IQuorumChange {
 				    .detail("DesiredCoordinators", desiredCount)
 				    .detail("CurrentCoordinators", oldCoordinators.size());
 				*err = CoordinatorsResult::NOT_ENOUGH_MACHINES;
-				return std::vector<NetworkAddress>();
+				co_return std::vector<NetworkAddress>();
 			}
 			chosen.resize((chosen.size() - 1) | 1);
 		}
 
-		return chosen;
+		co_return chosen;
 	}
 
 	// Select a desired set of workers such that
@@ -1408,15 +1416,16 @@ struct AutoQuorumChange final : IQuorumChange {
 	}
 };
 Reference<IQuorumChange> autoQuorumChange(int desired) {
-	return Reference<IQuorumChange>(new AutoQuorumChange(desired));
+	return makeReference<AutoQuorumChange>(desired);
 }
 
-ACTOR Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed) {
+Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-	std::vector<AddressExclusion> excl = wait(failed ? getExcludedFailedServerList(tr) : getExcludedServerList(tr));
+	std::vector<AddressExclusion> excl =
+	    co_await (failed ? getExcludedFailedServerList(tr) : getExcludedServerList(tr));
 	std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
 	bool containNewExclusion = false;
 	for (auto& s : servers) {
@@ -1441,13 +1450,13 @@ ACTOR Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion>
 	    .detail("Servers", describe(servers))
 	    .detail("ExcludeFailed", failed)
 	    .detail("ExclusionUpdated", containNewExclusion);
-	return Void();
 }
 
-ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
+Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
 	if (cx->apiVersionAtLeast(700)) {
-		state ReadYourWritesTransaction ryw(cx);
-		loop {
+		ReadYourWritesTransaction ryw(cx);
+		while (true) {
+			Error err;
 			try {
 				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -1463,35 +1472,38 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 				TraceEvent("ExcludeServersSpecialKeySpaceCommit")
 				    .detail("Servers", describe(servers))
 				    .detail("ExcludeFailed", failed);
-				wait(ryw.commit());
-				return Void();
+				co_await ryw.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("ExcludeServersError").errorUnsuppressed(e);
-				wait(ryw.onError(e));
+				err = e;
 			}
+			TraceEvent("ExcludeServersError").errorUnsuppressed(err);
+			co_await ryw.onError(err);
 		}
 	} else {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				wait(excludeServers(&tr, servers, failed));
-				wait(tr.commit());
-				return Void();
+				co_await excludeServers(&tr, servers, failed);
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("ExcludeServersError").errorUnsuppressed(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("ExcludeServersError").errorUnsuppressed(err);
+			co_await tr.onError(err);
 		}
 	}
 }
 
 // excludes localities by setting the keys in api version below 7.0
-ACTOR Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string> localities, bool failed) {
+Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string> localities, bool failed) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-	std::vector<std::string> excl = wait(failed ? getExcludedFailedLocalityList(tr) : getExcludedLocalityList(tr));
+	std::vector<std::string> excl = co_await (failed ? getExcludedFailedLocalityList(tr) : getExcludedLocalityList(tr));
 	std::set<std::string> exclusion(excl.begin(), excl.end());
 	bool containNewExclusion = false;
 	for (const auto& l : localities) {
@@ -1515,15 +1527,15 @@ ACTOR Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::st
 	    .detail("Localities", describe(localities))
 	    .detail("ExcludeFailed", failed)
 	    .detail("ExclusionUpdated", containNewExclusion);
-	return Void();
 }
 
 // Exclude the servers matching the given set of localities from use as state servers.
 // excludes localities by setting the keys.
-ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> localities, bool failed) {
+Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> localities, bool failed) {
 	if (cx->apiVersionAtLeast(700)) {
-		state ReadYourWritesTransaction ryw(cx);
-		loop {
+		ReadYourWritesTransaction ryw(cx);
+		while (true) {
+			Error err;
 			try {
 				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -1540,33 +1552,37 @@ ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string
 				    .detail("Localities", describe(localities))
 				    .detail("ExcludeFailed", failed);
 
-				wait(ryw.commit());
-				return Void();
+				co_await ryw.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(e);
-				wait(ryw.onError(e));
+				err = e;
 			}
+			TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(err);
+			co_await ryw.onError(err);
 		}
 	} else {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				wait(excludeLocalities(&tr, localities, failed));
-				wait(tr.commit());
-				return Void();
+				co_await excludeLocalities(&tr, localities, failed);
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(err);
+			co_await tr.onError(err);
 		}
 	}
 }
 
-ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
-	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
+Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
+	std::string versionKey = deterministicRandom()->randomUniqueID().toString();
 	if (cx->apiVersionAtLeast(700)) {
-		state ReadYourWritesTransaction ryw(cx);
-		loop {
+		ReadYourWritesTransaction ryw(cx);
+		while (true) {
+			Error err;
 			try {
 				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -1596,24 +1612,26 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 				}
 				TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
 
-				wait(ryw.commit());
-				return Void();
+				co_await ryw.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("IncludeServersError").errorUnsuppressed(e);
-				wait(ryw.onError(e));
+				err = e;
 			}
+			TraceEvent("IncludeServersError").errorUnsuppressed(err);
+			co_await ryw.onError(err);
 		}
 	} else {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 
-				// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting
-				// and CAUSAL_WRITE_RISKY
+				// includeServers might be used in an emergency transaction, so make sure it is
+				// retry-self-conflicting and CAUSAL_WRITE_RISKY
 				tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
 				if (failed) {
 					tr.addReadConflictRange(singleKeyRange(failedServersVersionKey));
@@ -1653,23 +1671,25 @@ ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> ser
 
 				TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
 
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("IncludeServersError").errorUnsuppressed(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("IncludeServersError").errorUnsuppressed(err);
+			co_await tr.onError(err);
 		}
 	}
 }
 
 // Remove the given localities from the exclusion list.
 // include localities by clearing the keys.
-ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> localities, bool failed, bool includeAll) {
-	state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
+Future<Void> includeLocalities(Database cx, std::vector<std::string> localities, bool failed, bool includeAll) {
+	std::string versionKey = deterministicRandom()->randomUniqueID().toString();
 	if (cx->apiVersionAtLeast(700)) {
-		state ReadYourWritesTransaction ryw(cx);
-		loop {
+		ReadYourWritesTransaction ryw(cx);
+		while (true) {
+			Error err;
 			try {
 				ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
 				ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -1692,16 +1712,18 @@ ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> local
 				    .detail("Failed", failed)
 				    .detail("IncludeAll", includeAll);
 
-				wait(ryw.commit());
-				return Void();
+				co_await ryw.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e);
-				wait(ryw.onError(e));
+				err = e;
 			}
+			TraceEvent("IncludeLocalitiesError").errorUnsuppressed(err);
+			co_await ryw.onError(err);
 		}
 	} else {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -1740,27 +1762,29 @@ ACTOR Future<Void> includeLocalities(Database cx, std::vector<std::string> local
 				    .detail("Failed", failed)
 				    .detail("IncludeAll", includeAll);
 
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("IncludeLocalitiesError").errorUnsuppressed(err);
+			co_await tr.onError(err);
 		}
 	}
 }
 
-ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass) {
-	state Transaction tr(cx);
+Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass) {
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 
-			std::vector<ProcessData> workers = wait(getWorkers(&tr));
+			std::vector<ProcessData> workers = co_await getWorkers(&tr);
 
 			bool foundChange = false;
 			for (int i = 0; i < workers.size(); i++) {
@@ -1777,16 +1801,17 @@ ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass p
 			if (foundChange)
 				tr.set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString());
 
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* tr) {
-	state RangeResult r = wait(tr->getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY));
+Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* tr) {
+	RangeResult r = co_await tr->getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::vector<AddressExclusion> exclusions;
@@ -1796,11 +1821,11 @@ ACTOR Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* t
 			exclusions.push_back(a);
 	}
 	uniquify(exclusions);
-	return exclusions;
+	co_return exclusions;
 }
 
-ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr) {
-	state RangeResult r = wait(tr->getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY));
+Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr) {
+	RangeResult r = co_await tr->getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::vector<AddressExclusion> exclusions;
@@ -1810,20 +1835,20 @@ ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transact
 			exclusions.push_back(a);
 	}
 	uniquify(exclusions);
-	return exclusions;
+	co_return exclusions;
 }
 
-ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr) {
-	state std::vector<AddressExclusion> exclusions;
+Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr) {
+	std::vector<AddressExclusion> exclusions;
 	// Request all exclusion based information concurrently.
-	state Future<std::vector<AddressExclusion>> fExcludedServers = getExcludedServerList(tr);
-	state Future<std::vector<AddressExclusion>> fExcludedFailed = getExcludedFailedServerList(tr);
-	state Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(tr);
-	state Future<std::vector<ProcessData>> fWorkers = getWorkers(tr);
+	Future<std::vector<AddressExclusion>> fExcludedServers = getExcludedServerList(tr);
+	Future<std::vector<AddressExclusion>> fExcludedFailed = getExcludedFailedServerList(tr);
+	Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(tr);
+	Future<std::vector<ProcessData>> fWorkers = getWorkers(tr);
 
 	// Wait until all data is gathered, we are not waiting here for the workers future to return
 	// instead we wait for the worker future only if we need the data.
-	wait(success(fExcludedServers) && success(fExcludedFailed) && success(fExcludedLocalities));
+	co_await (success(fExcludedServers) && success(fExcludedFailed) && success(fExcludedLocalities));
 	// Update the exclusions vector with all excluded servers.
 	auto excludedServers = fExcludedServers.get();
 	exclusions.insert(exclusions.end(), excludedServers.begin(), excludedServers.end());
@@ -1832,13 +1857,13 @@ ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* t
 
 	// We have to return all servers that are excluded, this includes servers that are excluded
 	// based on the locality. Otherwise those excluded servers might be used, even if they shouldn't.
-	state std::vector<std::string> excludedLocalities = fExcludedLocalities.get();
+	std::vector<std::string> excludedLocalities = fExcludedLocalities.get();
 
 	// Only if at least one locality was found we have to perform this check.
 	if (!excludedLocalities.empty()) {
 		// First we have to fetch all workers to match the localities of each worker against the excluded localities.
-		wait(success(fWorkers));
-		state std::vector<ProcessData> workers = fWorkers.get();
+		co_await fWorkers;
+		std::vector<ProcessData> workers = fWorkers.get();
 
 		for (const auto& locality : excludedLocalities) {
 			std::set<AddressExclusion> localityAddresses = getAddressesByLocality(workers, locality);
@@ -1850,26 +1875,28 @@ ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* t
 	}
 
 	uniquify(exclusions);
-	return exclusions;
+	co_return exclusions;
 }
 
-ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::vector<AddressExclusion> exclusions = wait(getAllExcludedServers(&tr));
-			return exclusions;
+			std::vector<AddressExclusion> exclusions = co_await getAllExcludedServers(&tr);
+			co_return exclusions;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr) {
-	state RangeResult r = wait(tr->getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
+Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr) {
+	RangeResult r = co_await tr->getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::vector<std::string> excludedLocalities;
@@ -1878,11 +1905,11 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr)
 		excludedLocalities.push_back(a);
 	}
 	uniquify(excludedLocalities);
-	return excludedLocalities;
+	co_return excludedLocalities;
 }
 
-ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr) {
-	state RangeResult r = wait(tr->getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
+Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr) {
+	RangeResult r = co_await tr->getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::vector<std::string> excludedLocalities;
@@ -1891,16 +1918,16 @@ ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction
 		excludedLocalities.push_back(a);
 	}
 	uniquify(excludedLocalities);
-	return excludedLocalities;
+	co_return excludedLocalities;
 }
 
-ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr) {
-	state std::vector<std::string> exclusions;
-	state Future<std::vector<std::string>> fExcludedLocalities = getExcludedLocalityList(tr);
-	state Future<std::vector<std::string>> fFailedLocalities = getExcludedFailedLocalityList(tr);
+Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr) {
+	std::vector<std::string> exclusions;
+	Future<std::vector<std::string>> fExcludedLocalities = getExcludedLocalityList(tr);
+	Future<std::vector<std::string>> fFailedLocalities = getExcludedFailedLocalityList(tr);
 
 	// Wait until all data is gathered.
-	wait(success(fExcludedLocalities) && success(fFailedLocalities));
+	co_await (success(fExcludedLocalities) && success(fFailedLocalities));
 
 	auto excludedLocalities = fExcludedLocalities.get();
 	exclusions.insert(exclusions.end(), excludedLocalities.begin(), excludedLocalities.end());
@@ -1908,22 +1935,24 @@ ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr)
 	exclusions.insert(exclusions.end(), failedLocalities.begin(), failedLocalities.end());
 
 	uniquify(exclusions);
-	return exclusions;
+	co_return exclusions;
 }
 
 // Get the list of excluded localities by reading the keys.
-ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<std::vector<std::string>> getAllExcludedLocalities(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::vector<std::string> exclusions = wait(getAllExcludedLocalities(&tr));
-			return exclusions;
+			std::vector<std::string> exclusions = co_await getAllExcludedLocalities(&tr);
+			co_return exclusions;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -1993,13 +2022,14 @@ std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>
 	return locality_addresses;
 }
 
-ACTOR Future<Void> printHealthyZone(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> printHealthyZone(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			Optional<Value> val = wait(tr.get(healthyZoneKey));
+			Optional<Value> val = co_await tr.get(healthyZoneKey);
 			if (val.present() && decodeHealthyZoneValue(val.get()).first == ignoreSSFailuresZoneString) {
 				printf("Data distribution has been disabled for all storage server failures in this cluster and thus "
 				       "maintenance mode is not active.\n");
@@ -2011,79 +2041,85 @@ ACTOR Future<Void> printHealthyZone(Database cx) {
 				           healthyZone.first.toString(),
 				           (healthyZone.second - tr.getReadVersion().get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<bool> clearHealthyZone(Database cx, bool printWarning, bool clearSSFailureZoneString) {
-	state Transaction tr(cx);
+Future<bool> clearHealthyZone(Database cx, bool printWarning, bool clearSSFailureZoneString) {
+	Transaction tr(cx);
 	TraceEvent("ClearHealthyZone").detail("ClearSSFailureZoneString", clearSSFailureZoneString);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			Optional<Value> val = wait(tr.get(healthyZoneKey));
+			Optional<Value> val = co_await tr.get(healthyZoneKey);
 			if (!clearSSFailureZoneString && val.present() &&
 			    decodeHealthyZoneValue(val.get()).first == ignoreSSFailuresZoneString) {
 				if (printWarning) {
 					printf("ERROR: Maintenance mode cannot be used while data distribution is disabled for storage "
 					       "server failures. Use 'datadistribution on' to reenable data distribution.\n");
 				}
-				return false;
+				co_return false;
 			}
 
 			tr.clear(healthyZoneKey);
-			wait(tr.commit());
-			return true;
+			co_await tr.commit();
+			co_return true;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds, bool printWarning) {
-	state Transaction tr(cx);
+Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds, bool printWarning) {
+	Transaction tr(cx);
 	TraceEvent("SetHealthyZone").detail("Zone", zoneId).detail("DurationSeconds", seconds);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			Optional<Value> val = wait(tr.get(healthyZoneKey));
+			Optional<Value> val = co_await tr.get(healthyZoneKey);
 			if (val.present() && decodeHealthyZoneValue(val.get()).first == ignoreSSFailuresZoneString) {
 				if (printWarning) {
 					printf("ERROR: Maintenance mode cannot be used while data distribution is disabled for storage "
 					       "server failures. Use 'datadistribution on' to reenable data distribution.\n");
 				}
-				return false;
+				co_return false;
 			}
-			Version readVersion = wait(tr.getReadVersion());
+			Version readVersion = co_await tr.getReadVersion();
 			tr.set(healthyZoneKey,
 			       healthyZoneValue(zoneId, readVersion + (seconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)));
-			wait(tr.commit());
-			return true;
+			co_await tr.commit();
+			co_return true;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<int> setDDMode(Database cx, int mode) {
-	state Transaction tr(cx);
-	state int oldMode = -1;
-	state BinaryWriter wr(Unversioned());
+Future<int> setDDMode(Database cx, int mode) {
+	Transaction tr(cx);
+	int oldMode = -1;
+	BinaryWriter wr(Unversioned());
 	wr << mode;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			Optional<Value> old = wait(tr.get(dataDistributionModeKey));
+			Optional<Value> old = co_await tr.get(dataDistributionModeKey);
 			if (oldMode < 0) {
 				oldMode = 1;
 				if (old.present()) {
@@ -2103,7 +2139,7 @@ ACTOR Future<int> setDDMode(Database cx, int mode) {
 				// set DDMode to 1 will enable all disabled parts, for instance the SS failure monitors.
 				// set DDMode to 2 is a security mode which disables data moves but allows auditStorage part
 				// DDMode=2 is set when shard location metadata inconsistency is detected
-				Optional<Value> currentHealthyZoneValue = wait(tr.get(healthyZoneKey));
+				Optional<Value> currentHealthyZoneValue = co_await tr.get(healthyZoneKey);
 				if (currentHealthyZoneValue.present() &&
 				    decodeHealthyZoneValue(currentHealthyZoneValue.get()).first == ignoreSSFailuresZoneString) {
 					// only clear the key if it is currently being used to disable all SS failure data movement
@@ -2111,22 +2147,23 @@ ACTOR Future<int> setDDMode(Database cx, int mode) {
 				}
 				tr.clear(rebalanceDDIgnoreKey);
 			}
-			wait(tr.commit());
-			return oldMode;
+			co_await tr.commit();
+			co_return oldMode;
 		} catch (Error& e) {
-			TraceEvent("SetDDModeRetrying").error(e);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("SetDDModeRetrying").error(err);
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
-                                                   std::set<AddressExclusion>* exclusions,
-                                                   std::set<NetworkAddress>* inProgressExclusion) {
+Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
+                                             std::set<AddressExclusion>* exclusions,
+                                             std::set<NetworkAddress>* inProgressExclusion) {
 	// TODO : replace using ExclusionInProgressRangeImpl in special key space
-	ASSERT(inProgressExclusion->size() == 0); //  Make sure every time it is cleared beforehand
-	if (!exclusions->size())
-		return true;
+	ASSERT(inProgressExclusion->empty()); //  Make sure every time it is cleared beforehand
+	if (exclusions->empty())
+		co_return true;
 
 	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
@@ -2136,10 +2173,10 @@ ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr
 	// recovery
 
 	// Check that there aren't any storage servers with addresses violating the exclusions
-	RangeResult serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+	RangeResult serverList = co_await tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
-	state bool ok = true;
+	bool ok = true;
 	for (auto& s : serverList) {
 		auto addresses = decodeServerListValue(s.value).getKeyValues.getEndpoint().addresses;
 		if (addressExcluded(*exclusions, addresses.address)) {
@@ -2153,82 +2190,85 @@ ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr
 	}
 
 	if (ok) {
-		Optional<Standalone<StringRef>> value = wait(tr->get(logsKey));
+		Optional<Standalone<StringRef>> value = co_await tr->get(logsKey);
 		ASSERT(value.present());
 		auto logs = decodeLogsValue(value.get());
-		for (auto const& log : logs.first) {
-			if (log.second == NetworkAddress() || addressExcluded(*exclusions, log.second)) {
+		for (const auto& [_logId, logAddress] : logs.first) {
+			if (logAddress == NetworkAddress() || addressExcluded(*exclusions, logAddress)) {
 				ok = false;
-				inProgressExclusion->insert(log.second);
+				inProgressExclusion->insert(logAddress);
 			}
 		}
-		for (auto const& log : logs.second) {
-			if (log.second == NetworkAddress() || addressExcluded(*exclusions, log.second)) {
+		for (const auto& [_logId, logAddress] : logs.second) {
+			if (logAddress == NetworkAddress() || addressExcluded(*exclusions, logAddress)) {
 				ok = false;
-				inProgressExclusion->insert(log.second);
+				inProgressExclusion->insert(logAddress);
 			}
 		}
 	}
 
-	return ok;
+	co_return ok;
 }
 
-ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
-                                                                std::vector<AddressExclusion> excl,
-                                                                bool waitForAllExcluded) {
-	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
-	state std::set<NetworkAddress> inProgressExclusion;
+Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
+                                                          std::vector<AddressExclusion> excl,
+                                                          bool waitForAllExcluded) {
+	std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
+	std::set<NetworkAddress> inProgressExclusion;
 
-	loop {
-		state ReadYourWritesTransaction tr(cx);
+	while (true) {
+		ReadYourWritesTransaction tr(cx);
 		inProgressExclusion.clear();
+		Error err;
 		try {
-			bool ok = wait(checkForExcludingServersTxActor(&tr, &exclusions, &inProgressExclusion));
+			bool ok = co_await checkForExcludingServersTxActor(&tr, &exclusions, &inProgressExclusion);
 			if (ok)
-				return inProgressExclusion;
+				co_return inProgressExclusion;
 			if (!waitForAllExcluded)
 				break;
 
-			wait(delayJittered(1.0)); // SOMEDAY: watches!
+			co_await delayJittered(1.0); // SOMEDAY: watches!
+			continue;
 		} catch (Error& e) {
-			TraceEvent("CheckForExcludingServersError").error(e);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("CheckForExcludingServersError").error(err);
+		co_await tr.onError(err);
 	}
-	return inProgressExclusion;
+	co_return inProgressExclusion;
 }
 
-ACTOR Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID) {
+Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID) {
 	try {
-		wait(snapCreate(cx, snapCmd, snapUID));
+		co_await snapCreate(cx, snapCmd, snapUID);
 		TraceEvent("SnapCreateSucceeded").detail("snapUID", snapUID);
-		return Void();
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "SnapCreateFailed").error(e).detail("snapUID", snapUID);
 		throw;
 	}
 }
 
-ACTOR Future<Void> waitForFullReplication(Database cx) {
-	state ReadYourWritesTransaction tr(cx);
-	loop {
+Future<Void> waitForFullReplication(Database cx) {
+	ReadYourWritesTransaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			RangeResult confResults = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult confResults = co_await tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!confResults.more && confResults.size() < CLIENT_KNOBS->TOO_MANY);
-			state DatabaseConfiguration config;
+			DatabaseConfiguration config;
 			config.fromKeyValues((VectorRef<KeyValueRef>)confResults);
 
-			state std::vector<Future<Optional<Value>>> replicasFutures;
+			std::vector<Future<Optional<Value>>> replicasFutures;
 			for (auto& region : config.regions) {
 				replicasFutures.push_back(tr.get(datacenterReplicasKeyFor(region.dcId)));
 			}
-			wait(waitForAll(replicasFutures));
+			co_await waitForAll(replicasFutures);
 
-			state std::vector<Future<Void>> watchFutures;
+			std::vector<Future<Void>> watchFutures;
 			for (int i = 0; i < config.regions.size(); i++) {
 				if (!replicasFutures[i].get().present() ||
 				    decodeDatacenterReplicasValue(replicasFutures[i].get().get()) < config.storageTeamSize) {
@@ -2236,42 +2276,46 @@ ACTOR Future<Void> waitForFullReplication(Database cx) {
 				}
 			}
 
-			if (!watchFutures.size() || (config.usableRegions == 1 && watchFutures.size() < config.regions.size())) {
-				return Void();
+			if (watchFutures.empty() || (config.usableRegions == 1 && watchFutures.size() < config.regions.size())) {
+				co_return;
 			}
 
-			wait(tr.commit());
-			wait(waitForAny(watchFutures));
+			co_await tr.commit();
+			co_await waitForAny(watchFutures);
 			tr.reset();
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> timeKeeperSetDisable(Database cx) {
-	loop {
-		state Transaction tr(cx);
+Future<Void> timeKeeperSetDisable(Database cx) {
+	while (true) {
+		Transaction tr(cx);
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.set(timeKeeperDisableKey, StringRef());
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> lockDatabase(Transaction* tr, UID id) {
+Future<Void> lockDatabase(Transaction* tr, UID id) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	Optional<Value> val = wait(tr->get(databaseLockedKey));
+	Optional<Value> val = co_await tr->get(databaseLockedKey);
 
 	if (val.present()) {
 		if (BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) == id) {
-			return Void();
+			co_return;
 		} else {
 			//TraceEvent("DBA_LockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
 			throw database_locked();
@@ -2282,17 +2326,16 @@ ACTOR Future<Void> lockDatabase(Transaction* tr, UID id) {
 	             BinaryWriter::toValue(id, Unversioned()).withPrefix("0123456789"_sr).withSuffix("\x00\x00\x00\x00"_sr),
 	             MutationRef::SetVersionstampedValue);
 	tr->addWriteConflictRange(normalKeys);
-	return Void();
 }
 
-ACTOR Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id) {
+Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	Optional<Value> val = wait(tr->get(databaseLockedKey));
+	Optional<Value> val = co_await tr->get(databaseLockedKey);
 
 	if (val.present()) {
 		if (BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) == id) {
-			return Void();
+			co_return;
 		} else {
 			//TraceEvent("DBA_LockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
 			throw database_locked();
@@ -2303,34 +2346,35 @@ ACTOR Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id)
 	             BinaryWriter::toValue(id, Unversioned()).withPrefix("0123456789"_sr).withSuffix("\x00\x00\x00\x00"_sr),
 	             MutationRef::SetVersionstampedValue);
 	tr->addWriteConflictRange(normalKeys);
-	return Void();
 }
 
-ACTOR Future<Void> lockDatabase(Database cx, UID id) {
-	state Transaction tr(cx);
+Future<Void> lockDatabase(Database cx, UID id) {
+	Transaction tr(cx);
 	UID debugID = deterministicRandom()->randomUniqueID();
 	TraceEvent("LockDatabaseTransaction", debugID).log();
 	tr.debugTransaction(debugID);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			wait(lockDatabase(&tr, id));
-			wait(tr.commit());
-			return Void();
+			co_await lockDatabase(&tr, id);
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			if (e.code() == error_code_database_locked)
-				throw e;
-			wait(tr.onError(e));
+			err = e;
 		}
+		if (err.code() == error_code_database_locked)
+			throw err;
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> unlockDatabase(Transaction* tr, UID id) {
+Future<Void> unlockDatabase(Transaction* tr, UID id) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	Optional<Value> val = wait(tr->get(databaseLockedKey));
+	Optional<Value> val = co_await tr->get(databaseLockedKey);
 
 	if (!val.present())
-		return Void();
+		co_return;
 
 	if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
 		//TraceEvent("DBA_UnlockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
@@ -2338,16 +2382,15 @@ ACTOR Future<Void> unlockDatabase(Transaction* tr, UID id) {
 	}
 
 	tr->clear(singleKeyRange(databaseLockedKey));
-	return Void();
 }
 
-ACTOR Future<Void> unlockDatabase(Reference<ReadYourWritesTransaction> tr, UID id) {
+Future<Void> unlockDatabase(Reference<ReadYourWritesTransaction> tr, UID id) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	Optional<Value> val = wait(tr->get(databaseLockedKey));
+	Optional<Value> val = co_await tr->get(databaseLockedKey);
 
 	if (!val.present())
-		return Void();
+		co_return;
 
 	if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
 		//TraceEvent("DBA_UnlockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
@@ -2355,103 +2398,102 @@ ACTOR Future<Void> unlockDatabase(Reference<ReadYourWritesTransaction> tr, UID i
 	}
 
 	tr->clear(singleKeyRange(databaseLockedKey));
-	return Void();
 }
 
-ACTOR Future<Void> unlockDatabase(Database cx, UID id) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> unlockDatabase(Database cx, UID id) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			wait(unlockDatabase(&tr, id));
-			wait(tr.commit());
-			return Void();
+			co_await unlockDatabase(&tr, id);
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			if (e.code() == error_code_database_locked)
-				throw e;
-			wait(tr.onError(e));
+			err = e;
 		}
+		if (err.code() == error_code_database_locked)
+			throw err;
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> checkDatabaseLock(Transaction* tr, UID id) {
+Future<Void> checkDatabaseLock(Transaction* tr, UID id) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	Optional<Value> val = wait(tr->get(databaseLockedKey));
+	Optional<Value> val = co_await tr->get(databaseLockedKey);
 
 	if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
 		//TraceEvent("DBA_CheckLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())).backtrace();
 		throw database_locked();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> checkDatabaseLock(Reference<ReadYourWritesTransaction> tr, UID id) {
+Future<Void> checkDatabaseLock(Reference<ReadYourWritesTransaction> tr, UID id) {
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-	Optional<Value> val = wait(tr->get(databaseLockedKey));
+	Optional<Value> val = co_await tr->get(databaseLockedKey);
 
 	if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
 		//TraceEvent("DBA_CheckLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())).backtrace();
 		throw database_locked();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> advanceVersion(Database cx, Version v) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> advanceVersion(Database cx, Version v) {
+	Transaction tr(cx);
+	while (true) {
 		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		Error err;
 		try {
-			Version rv = wait(tr.getReadVersion());
+			Version rv = co_await tr.getReadVersion();
 			if (rv <= v) {
 				tr.set(minRequiredCommitVersionKey, BinaryWriter::toValue(v + 1, Unversioned()));
-				wait(tr.commit());
+				co_await tr.commit();
+				continue;
 			} else {
 				fmt::print("Current read version is {}\n", rv);
-				return Void();
+				co_return;
 			}
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Key dcId) {
-	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
-	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
+Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Key dcId) {
+	Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
+	Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
 
-	loop {
-		choose {
-			when(wait(clusterInterface->get().present()
-			              ? brokenPromiseToNever(
-			                    clusterInterface->get().get().forceRecovery.getReply(ForceRecoveryRequest(dcId)))
-			              : Never())) {
-				return Void();
-			}
-			when(wait(clusterInterface->onChange())) {}
+	while (true) {
+		Future<Void> forceRecoveryFuture = Never();
+		if (clusterInterface->get().present()) {
+			forceRecoveryFuture =
+			    brokenPromiseToNever(clusterInterface->get().get().forceRecovery.getReply(ForceRecoveryRequest(dcId)));
+		}
+		if (auto const res = co_await race(forceRecoveryFuture, clusterInterface->onChange()); res.index() == 0) {
+			co_return;
 		}
 	}
 }
 
-ACTOR Future<UID> auditStorage(Reference<IClusterConnectionRecord> clusterFile,
-                               KeyRange range,
-                               AuditType type,
-                               KeyValueStoreType engineType,
-                               double timeoutSeconds) {
-	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
-	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
+Future<UID> auditStorage(Reference<IClusterConnectionRecord> clusterFile,
+                         KeyRange range,
+                         AuditType type,
+                         KeyValueStoreType engineType,
+                         double timeoutSeconds) {
+	Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
+	Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
 	TraceEvent(SevVerbose, "ManagementAPIAuditStorageTrigger").detail("AuditType", type).detail("Range", range);
-	state UID auditId;
+	UID auditId;
 	try {
 		while (!clusterInterface->get().present()) {
-			wait(clusterInterface->onChange());
+			co_await clusterInterface->onChange();
 		}
 		TraceEvent(SevVerbose, "ManagementAPIAuditStorageBegin").detail("AuditType", type).detail("Range", range);
 		TriggerAuditRequest req(type, range, engineType);
-		UID auditId_ = wait(timeoutError(clusterInterface->get().get().triggerAudit.getReply(req), timeoutSeconds));
+		UID auditId_ = co_await timeoutError(clusterInterface->get().get().triggerAudit.getReply(req), timeoutSeconds);
 		auditId = auditId_;
 		TraceEvent(SevVerbose, "ManagementAPIAuditStorageEnd")
 		    .detail("AuditType", type)
@@ -2466,27 +2508,27 @@ ACTOR Future<UID> auditStorage(Reference<IClusterConnectionRecord> clusterFile,
 		throw e;
 	}
 
-	return auditId;
+	co_return auditId;
 }
 
-ACTOR Future<UID> cancelAuditStorage(Reference<IClusterConnectionRecord> clusterFile,
-                                     AuditType type,
-                                     UID auditId,
-                                     double timeoutSeconds) {
-	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
-	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
+Future<UID> cancelAuditStorage(Reference<IClusterConnectionRecord> clusterFile,
+                               AuditType type,
+                               UID auditId,
+                               double timeoutSeconds) {
+	Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
+	Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
 	TraceEvent(SevVerbose, "ManagementAPICancelAuditStorageTrigger")
 	    .detail("AuditType", type)
 	    .detail("AuditId", auditId);
 	try {
 		while (!clusterInterface->get().present()) {
-			wait(clusterInterface->onChange());
+			co_await clusterInterface->onChange();
 		}
 		TraceEvent(SevVerbose, "ManagementAPICancelAuditStorageBegin")
 		    .detail("AuditType", type)
 		    .detail("AuditId", auditId);
 		TriggerAuditRequest req(type, auditId);
-		UID auditId_ = wait(timeoutError(clusterInterface->get().get().triggerAudit.getReply(req), timeoutSeconds));
+		UID auditId_ = co_await timeoutError(clusterInterface->get().get().triggerAudit.getReply(req), timeoutSeconds);
 		ASSERT(auditId_ == auditId);
 		TraceEvent(SevVerbose, "ManagementAPICancelAuditStorageEnd")
 		    .detail("AuditType", type)
@@ -2499,20 +2541,21 @@ ACTOR Future<UID> cancelAuditStorage(Reference<IClusterConnectionRecord> cluster
 		throw e;
 	}
 
-	return auditId;
+	co_return auditId;
 }
 
-ACTOR Future<int> setBulkLoadMode(Database cx, int mode) {
-	state Transaction tr(cx);
-	state BinaryWriter wr(Unversioned());
+Future<int> setBulkLoadMode(Database cx, int mode) {
+	Transaction tr(cx);
+	BinaryWriter wr(Unversioned());
 	wr << mode;
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			state int oldMode = 0;
-			Optional<Value> oldModeValue = wait(tr.get(bulkLoadModeKey));
+			int oldMode = 0;
+			Optional<Value> oldModeValue = co_await tr.get(bulkLoadModeKey);
 			if (oldModeValue.present()) {
 				BinaryReader rd(oldModeValue.get(), Unversioned());
 				rd >> oldMode;
@@ -2525,59 +2568,61 @@ ACTOR Future<int> setBulkLoadMode(Database cx, int mode) {
 				wrLastWrite << deterministicRandom()->randomUniqueID(); // triger DD restarts
 				tr.set(moveKeysLockWriteKey, wrLastWrite.toValue());
 				tr.set(bulkLoadModeKey, wr.toValue());
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadEngineModeKeyChanged")
 				    .detail("NewMode", mode)
 				    .detail("OldMode", oldMode);
 			}
-			return oldMode;
+			co_return oldMode;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<int> getBulkLoadMode(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<int> getBulkLoadMode(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			state int oldMode = 0;
-			Optional<Value> oldModeValue = wait(tr.get(bulkLoadModeKey));
+			int oldMode = 0;
+			Optional<Value> oldModeValue = co_await tr.get(bulkLoadModeKey);
 			if (oldModeValue.present()) {
 				BinaryReader rd(oldModeValue.get(), Unversioned());
 				rd >> oldMode;
 			}
-			return oldMode;
+			co_return oldMode;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> setBulkLoadSubmissionTransaction(Transaction* tr, BulkLoadTaskState bulkLoadTask) {
+Future<Void> setBulkLoadSubmissionTransaction(Transaction* tr, BulkLoadTaskState bulkLoadTask) {
 	ASSERT(normalKeys.contains(bulkLoadTask.getRange()) &&
 	       (bulkLoadTask.phase == BulkLoadPhase::Submitted ||
 	        (bulkLoadTask.phase == BulkLoadPhase::Complete && bulkLoadTask.hasEmptyData())));
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	bulkLoadTask.submitTime = now();
-	wait(krmSetRange(tr, bulkLoadTaskPrefix, bulkLoadTask.getRange(), bulkLoadTaskStateValue(bulkLoadTask)));
-	return Void();
+	co_await krmSetRange(tr, bulkLoadTaskPrefix, bulkLoadTask.getRange(), bulkLoadTaskStateValue(bulkLoadTask));
 }
 
 // Get bulk load task metadata with range and taskId and phase selector
 // Throw error if the task is outdated or the task is not in any input phase at the tr read version
 // TODO: check jobId
-ACTOR Future<BulkLoadTaskState> getBulkLoadTask(Transaction* tr,
-                                                KeyRange range,
-                                                UID taskId,
-                                                std::vector<BulkLoadPhase> phases) {
-	state BulkLoadTaskState bulkLoadTaskState;
+Future<BulkLoadTaskState> getBulkLoadTask(Transaction* tr,
+                                          KeyRange range,
+                                          UID taskId,
+                                          std::vector<BulkLoadPhase> phases) {
+	BulkLoadTaskState bulkLoadTaskState;
 	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-	RangeResult result = wait(krmGetRanges(tr, bulkLoadTaskPrefix, range));
+	RangeResult result = co_await krmGetRanges(tr, bulkLoadTaskPrefix, range);
 	if (result.size() > 2) {
 		TraceEvent(SevWarn, "GetBulkLoadTaskError")
 		    .detail("Reason", "TooManyRanges")
@@ -2628,7 +2673,7 @@ ACTOR Future<BulkLoadTaskState> getBulkLoadTask(Transaction* tr,
 		    .backtrace();
 		throw bulkload_task_outdated();
 	}
-	if (phases.size() > 0 && !bulkLoadTaskState.onAnyPhase(phases)) {
+	if (!phases.empty() && !bulkLoadTaskState.onAnyPhase(phases)) {
 		TraceEvent(SevWarn, "GetBulkLoadTaskError")
 		    .detail("Reason", "PhaseMismatch")
 		    .detail("Range", printable(range))
@@ -2637,16 +2682,15 @@ ACTOR Future<BulkLoadTaskState> getBulkLoadTask(Transaction* tr,
 		    .backtrace();
 		throw bulkload_task_outdated();
 	}
-	return bulkLoadTaskState;
+	co_return bulkLoadTaskState;
 }
 
-ACTOR Future<Void> setBulkLoadFinalizeTransaction(Transaction* tr, KeyRange range, UID taskId) {
-	state BulkLoadTaskState bulkLoadTaskState;
+Future<Void> setBulkLoadFinalizeTransaction(Transaction* tr, KeyRange range, UID taskId) {
+	BulkLoadTaskState bulkLoadTaskState;
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	wait(store(bulkLoadTaskState,
-	           getBulkLoadTask(
-	               tr, range, taskId, { BulkLoadPhase::Complete, BulkLoadPhase::Acknowledged, BulkLoadPhase::Error })));
+	bulkLoadTaskState = co_await getBulkLoadTask(
+	    tr, range, taskId, { BulkLoadPhase::Complete, BulkLoadPhase::Acknowledged, BulkLoadPhase::Error });
 	if (bulkLoadTaskState.phase == BulkLoadPhase::Error) {
 		TraceEvent(SevWarnAlways, "ManagementAPIAcknowledgeErrorTask")
 		    .detail("TaskId", taskId.toString())
@@ -2655,22 +2699,22 @@ ACTOR Future<Void> setBulkLoadFinalizeTransaction(Transaction* tr, KeyRange rang
 	bulkLoadTaskState.phase = BulkLoadPhase::Acknowledged;
 	ASSERT(range == bulkLoadTaskState.getRange() && taskId == bulkLoadTaskState.getTaskId());
 	ASSERT(normalKeys.contains(range));
-	wait(krmSetRange(tr, bulkLoadTaskPrefix, bulkLoadTaskState.getRange(), bulkLoadTaskStateValue(bulkLoadTaskState)));
-	return Void();
+	co_await krmSetRange(
+	    tr, bulkLoadTaskPrefix, bulkLoadTaskState.getRange(), bulkLoadTaskStateValue(bulkLoadTaskState));
 }
 
 // This is the only place to update job history map. So, we check the number of job history entries here is sufficient
 // to maintain that the number of jobs in the history is no more than BULKLOAD_JOB_HISTORY_COUNT_MAX.
-ACTOR Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState jobState) {
-	state Key newJobKey = bulkLoadJobHistoryKeyFor(jobState.getJobId());
-	state RangeResult jobHistoryResult;
-	state Optional<BulkLoadJobState> oldestJobState; // Set to remove when the job history is full.
-	state Key beginKey = bulkLoadJobHistoryKeys.begin;
-	state Key endKey = bulkLoadJobHistoryKeys.end;
-	loop {
+Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState jobState) {
+	Key newJobKey = bulkLoadJobHistoryKeyFor(jobState.getJobId());
+	RangeResult jobHistoryResult;
+	Optional<BulkLoadJobState> oldestJobState; // Set to remove when the job history is full.
+	Key beginKey = bulkLoadJobHistoryKeys.begin;
+	Key endKey = bulkLoadJobHistoryKeys.end;
+	while (true) {
 		jobHistoryResult.clear();
-		wait(store(jobHistoryResult,
-		           tr->getRange(KeyRangeRef(beginKey, endKey), CLIENT_KNOBS->BULKLOAD_JOB_HISTORY_COUNT_MAX * 2)));
+		jobHistoryResult =
+		    co_await tr->getRange(KeyRangeRef(beginKey, endKey), CLIENT_KNOBS->BULKLOAD_JOB_HISTORY_COUNT_MAX * 2);
 		// Set limit twice the max count to check the number of jobs in the history is no more than the max
 		// count.
 		for (int i = 0; i < jobHistoryResult.size(); i++) {
@@ -2690,7 +2734,7 @@ ACTOR Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState job
 				tr->set(newJobKey, bulkLoadJobValue(jobState));
 				// BulkLoad job with the same jobId can run for multiple times, we only keep the latest one
 				// in the history.
-				return Void();
+				co_return;
 			}
 			if (jobHistoryResult.size() > CLIENT_KNOBS->BULKLOAD_JOB_HISTORY_COUNT_MAX) {
 				TraceEvent(SevError, "DDBulkLoadJobHistoryCountExceed", jobState.getJobId())
@@ -2704,6 +2748,7 @@ ACTOR Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState job
 		}
 		if (jobHistoryResult.more) {
 			beginKey = keyAfter(jobHistoryResult.back().key);
+			continue;
 		} else {
 			break;
 		}
@@ -2712,20 +2757,20 @@ ACTOR Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState job
 		tr->clear(bulkLoadJobHistoryKeyFor(oldestJobState.get().getJobId()));
 	}
 	tr->set(newJobKey, bulkLoadJobValue(jobState));
-	return Void();
 }
 
-ACTOR Future<std::vector<BulkLoadJobState>> getBulkLoadJobFromHistory(Database cx) {
-	state RangeResult jobHistoryResult;
-	state Key beginKey = bulkLoadJobHistoryKeys.begin;
-	state Key endKey = bulkLoadJobHistoryKeys.end;
-	state Transaction tr(cx);
-	state std::vector<BulkLoadJobState> res;
-	loop {
+AsyncResult<std::vector<BulkLoadJobState>> getBulkLoadJobFromHistory(Database cx) {
+	RangeResult jobHistoryResult;
+	Key beginKey = bulkLoadJobHistoryKeys.begin;
+	Key endKey = bulkLoadJobHistoryKeys.end;
+	Transaction tr(cx);
+	std::vector<BulkLoadJobState> res;
+	while (true) {
+		Error err;
 		try {
 			jobHistoryResult.clear();
-			wait(store(jobHistoryResult,
-			           tr.getRange(KeyRangeRef(beginKey, endKey), CLIENT_KNOBS->BULKLOAD_JOB_HISTORY_COUNT_MAX)));
+			jobHistoryResult =
+			    co_await tr.getRange(KeyRangeRef(beginKey, endKey), CLIENT_KNOBS->BULKLOAD_JOB_HISTORY_COUNT_MAX);
 			for (int i = 0; i < jobHistoryResult.size(); i++) {
 				BulkLoadJobState jobState = decodeBulkLoadJobState(jobHistoryResult[i].value);
 				ASSERT_WE_THINK(jobState.isValid());
@@ -2737,39 +2782,45 @@ ACTOR Future<std::vector<BulkLoadJobState>> getBulkLoadJobFromHistory(Database c
 			}
 			if (jobHistoryResult.more) {
 				beginKey = keyAfter(jobHistoryResult.back().key);
+				continue;
 			} else {
 				break;
 			}
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return res;
+	co_return res;
 }
 
-ACTOR Future<Void> clearBulkLoadJobHistory(Database cx, Optional<UID> jobId) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> clearBulkLoadJobHistory(Database cx, Optional<UID> jobId) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			if (jobId.present()) {
 				tr.clear(bulkLoadJobHistoryKeyFor(jobId.get()));
 			} else {
 				tr.clear(bulkLoadJobHistoryKeys);
 			}
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<Optional<BulkLoadJobState>> getSubmittedBulkLoadJob(Transaction* tr) {
-	state RangeResult rangeResult;
+Future<Optional<BulkLoadJobState>> getSubmittedBulkLoadJob(Transaction* tr) {
+	RangeResult rangeResult;
 	// At most one job at a time, so looking at the first returned range is sufficient
-	wait(store(rangeResult, krmGetRanges(tr, bulkLoadJobPrefix, normalKeys)));
-	for (int i = 0; i < rangeResult.size() - 1; ++i) {
+	rangeResult = co_await krmGetRanges(tr, bulkLoadJobPrefix, normalKeys);
+	if (rangeResult.empty()) {
+		co_return Optional<BulkLoadJobState>();
+	}
+	for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
 		if (rangeResult[i].value.empty()) {
 			continue;
 		}
@@ -2777,19 +2828,20 @@ ACTOR Future<Optional<BulkLoadJobState>> getSubmittedBulkLoadJob(Transaction* tr
 		if (!jobState.isValid()) {
 			continue;
 		}
-		return jobState;
+		co_return jobState;
 	}
-	return Optional<BulkLoadJobState>();
+	co_return Optional<BulkLoadJobState>();
 }
 
-ACTOR Future<Void> cancelBulkLoadJob(Database cx, UID jobId) {
-	state Transaction tr(cx);
-	state Optional<BulkLoadJobState> aliveJob;
-	loop {
+Future<Void> cancelBulkLoadJob(Database cx, UID jobId) {
+	Transaction tr(cx);
+	Optional<BulkLoadJobState> aliveJob;
+	while (true) {
+		Error err;
 		try {
-			wait(store(aliveJob, getSubmittedBulkLoadJob(&tr)));
+			aliveJob = co_await getSubmittedBulkLoadJob(&tr);
 			if (!aliveJob.present()) {
-				return Void(); // Has been cancelled
+				co_return; // Has been cancelled
 			}
 			if (aliveJob.get().getJobId() != jobId) {
 				throw bulkload_task_outdated(); // jobId is outdated
@@ -2803,46 +2855,46 @@ ACTOR Future<Void> cancelBulkLoadJob(Database cx, UID jobId) {
 			tr.set(moveKeysLockWriteKey, wrLastWrite.toValue());
 			// Clear all metadata of the job
 			ASSERT(!aliveJob.get().getJobRange().empty());
-			wait(krmSetRangeCoalescing(&tr,
-			                           bulkLoadJobPrefix,
-			                           aliveJob.get().getJobRange(),
-			                           normalKeys,
-			                           bulkLoadJobValue(BulkLoadJobState())));
+			co_await krmSetRangeCoalescing(
+			    &tr, bulkLoadJobPrefix, aliveJob.get().getJobRange(), normalKeys, bulkLoadJobValue(BulkLoadJobState()));
 			// Clear all metadata of the task. The task and the job is guaranteed to be consistent.
-			wait(krmSetRangeCoalescing(&tr,
-			                           bulkLoadTaskPrefix,
-			                           aliveJob.get().getJobRange(),
-			                           normalKeys,
-			                           bulkLoadTaskStateValue(BulkLoadTaskState())));
+			co_await krmSetRangeCoalescing(&tr,
+			                               bulkLoadTaskPrefix,
+			                               aliveJob.get().getJobRange(),
+			                               normalKeys,
+			                               bulkLoadTaskStateValue(BulkLoadTaskState()));
 			// Add cancelled job to history
 			aliveJob.get().setEndTime(now());
 			aliveJob.get().setCancelledPhase();
-			wait(addBulkLoadJobToHistory(&tr, aliveJob.get()));
-			wait(releaseExclusiveReadLockOnRange(&tr, aliveJob.get().getJobRange(), rangeLockNameForBulkLoad));
-			wait(tr.commit());
+			co_await addBulkLoadJobToHistory(&tr, aliveJob.get());
+			co_await releaseExclusiveReadLockOnRange(&tr, aliveJob.get().getJobRange(), rangeLockNameForBulkLoad);
+			// Clean up BulkLoad owner info when clearing job metadata
+			tr.clear(bulkLoadOwnerKeyFor(jobId));
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			// Currently, only bulkload job uses the range lock, and one job exists at a time.
-			// TODO(BulkLoad): support multiple jobs at a time
-			ASSERT(e.code() != error_code_range_unlock_reject);
-			wait(tr.onError(e));
+			err = e;
 		}
+		// Currently, only bulkload job uses the range lock, and one job exists at a time.
+		// TODO(BulkLoad): support multiple jobs at a time
+		ASSERT(err.code() != error_code_range_unlock_reject);
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
 // TODO(Zhe): clear bulkload task metadata within the input range
-ACTOR Future<Void> submitBulkLoadJob(Database cx, BulkLoadJobState jobState, bool lockAware) {
+Future<Void> submitBulkLoadJob(Database cx, BulkLoadJobState jobState, bool lockAware) {
 	ASSERT(jobState.getPhase() == BulkLoadJobPhase::Submitted);
 
-	state Transaction tr(cx);
-	loop {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			if (lockAware) {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			}
 			// There is at most one bulkLoad job or bulkDump job at a time globally
-			Optional<BulkDumpState> aliveBulkDumpJob = wait(getSubmittedBulkDumpJob(&tr));
+			Optional<BulkDumpState> aliveBulkDumpJob = co_await getSubmittedBulkDumpJob(&tr);
 			if (aliveBulkDumpJob.present()) {
 				TraceEvent(SevWarn, "SubmitBulkLoadJobFailed")
 				    .setMaxEventLength(-1)
@@ -2852,10 +2904,10 @@ ACTOR Future<Void> submitBulkLoadJob(Database cx, BulkLoadJobState jobState, boo
 				    .detail("ExistBulkDumpJob", aliveBulkDumpJob.get().toString());
 				throw bulkload_task_failed();
 			}
-			Optional<BulkLoadJobState> aliveJob = wait(getSubmittedBulkLoadJob(&tr));
+			Optional<BulkLoadJobState> aliveJob = co_await getSubmittedBulkLoadJob(&tr);
 			if (aliveJob.present()) {
 				if (aliveJob.get().getJobId() == jobState.getJobId()) {
-					return Void(); // The job has been submitted.
+					co_return; // The job has been submitted.
 				}
 				TraceEvent(SevWarn, "SubmitBulkLoadJobFailed")
 				    .setMaxEventLength(-1)
@@ -2883,41 +2935,45 @@ ACTOR Future<Void> submitBulkLoadJob(Database cx, BulkLoadJobState jobState, boo
 			}
 			ASSERT(!jobState.getJobRange().empty());
 			// Init the map of task states
-			wait(krmSetRange(
-			    &tr, bulkLoadTaskPrefix, jobState.getJobRange(), bulkLoadTaskStateValue(BulkLoadTaskState())));
+			co_await krmSetRange(
+			    &tr, bulkLoadTaskPrefix, jobState.getJobRange(), bulkLoadTaskStateValue(BulkLoadTaskState()));
 			// Persist job metadata
-			wait(krmSetRange(&tr, bulkLoadJobPrefix, jobState.getJobRange(), bulkLoadJobValue(jobState)));
+			co_await krmSetRange(&tr, bulkLoadJobPrefix, jobState.getJobRange(), bulkLoadJobValue(jobState));
 			// Take lock on the job range
-			wait(takeExclusiveReadLockOnRange(&tr, jobState.getJobRange(), rangeLockNameForBulkLoad));
-			wait(tr.commit());
+			co_await takeExclusiveReadLockOnRange(&tr, jobState.getJobRange(), rangeLockNameForBulkLoad);
+			co_await tr.commit();
 			TraceEvent(SevInfo, "BulkLoadJobSubmitted")
 			    .setMaxEventLength(-1)
 			    .setMaxFieldLength(-1)
 			    .detail("SubmitBulkLoadJob", jobState.toString());
 			break;
 		} catch (Error& e) {
-			// Currently, only bulkload job uses the range lock, and one job exists at a time.
-			// TODO(BulkLoad): support multiple jobs at a time
-			ASSERT(e.code() != error_code_range_lock_reject);
-			wait(tr.onError(e));
+			err = e;
 		}
+		// Currently, only bulkload job uses the range lock, and one job exists at a time.
+		// TODO(BulkLoad): support multiple jobs at a time
+		ASSERT(err.code() != error_code_range_lock_reject);
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<Optional<BulkLoadJobState>> getRunningBulkLoadJob(Database cx, bool lockAware) {
-	state RangeResult rangeResult;
-	state Transaction tr(cx);
-	state Key beginKey = normalKeys.begin;
-	state Key endKey = normalKeys.end;
+Future<Optional<BulkLoadJobState>> getRunningBulkLoadJob(Database cx, bool lockAware) {
+	RangeResult rangeResult;
+	Transaction tr(cx);
+	Key beginKey = normalKeys.begin;
+	Key endKey = normalKeys.end;
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			if (lockAware) {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			}
 			rangeResult.clear();
-			wait(store(rangeResult, krmGetRanges(&tr, bulkLoadJobPrefix, KeyRangeRef(beginKey, endKey))));
-			for (int i = 0; i < rangeResult.size() - 1; i++) {
+			rangeResult = co_await krmGetRanges(&tr, bulkLoadJobPrefix, KeyRangeRef(beginKey, endKey));
+			if (rangeResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; i++) {
 				if (rangeResult[i].value.empty()) {
 					continue;
 				}
@@ -2927,31 +2983,37 @@ ACTOR Future<Optional<BulkLoadJobState>> getRunningBulkLoadJob(Database cx, bool
 				}
 				// If a job is fully completed, the metadata should be removed from bulkLoadJobKeys
 				// The metadata is added to bulkload history.
-				return jobState;
+				co_return jobState;
 			}
 			beginKey = rangeResult.back().key;
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Optional<BulkLoadJobState>();
+	co_return Optional<BulkLoadJobState>();
 }
 
-ACTOR Future<Void> acknowledgeAllErrorBulkLoadTasks(Database cx, UID jobId, KeyRange jobRange) {
-	state Transaction tr(cx);
-	state Key beginKey = jobRange.begin;
-	state Key endKey = jobRange.end;
-	state Optional<Key> lastKey;
-	state BulkLoadTaskState existTask;
-	state RangeResult bulkLoadTaskResult;
-	state int i = 0;
+Future<Void> acknowledgeAllErrorBulkLoadTasks(Database cx, UID jobId, KeyRange jobRange) {
+	Transaction tr(cx);
+	Key beginKey = jobRange.begin;
+	Key endKey = jobRange.end;
+	Optional<Key> lastKey;
+	BulkLoadTaskState existTask;
+	RangeResult bulkLoadTaskResult;
+	int i = 0;
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			tr.reset();
 			bulkLoadTaskResult.clear();
-			wait(store(bulkLoadTaskResult, krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey))));
+			bulkLoadTaskResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey));
+			if (bulkLoadTaskResult.empty()) {
+				break;
+			}
 			i = 0;
-			for (; i < bulkLoadTaskResult.size() - 1; i++) {
+			for (; i < static_cast<int>(bulkLoadTaskResult.size()) - 1; i++) {
 				if (bulkLoadTaskResult[i].value.empty()) {
 					lastKey = bulkLoadTaskResult[i + 1].key;
 					continue;
@@ -2974,32 +3036,34 @@ ACTOR Future<Void> acknowledgeAllErrorBulkLoadTasks(Database cx, UID jobId, KeyR
 					    .detail("ExistTaskID", existTask.getTaskId())
 					    .detail("ExistTaskRange", existTask.getRange())
 					    .detail("ExistTaskJobId", existTask.getJobId());
-					wait(setBulkLoadFinalizeTransaction(&tr, existTask.getRange(), existTask.getTaskId()));
+					co_await setBulkLoadFinalizeTransaction(&tr, existTask.getRange(), existTask.getTaskId());
 				}
 				lastKey = bulkLoadTaskResult[i + 1].key;
 				break; // We actively break because we do not want transaction large
 			}
-			wait(tr.commit());
+			co_await tr.commit();
 			ASSERT(lastKey.present());
 			beginKey = lastKey.get();
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<int> setBulkDumpMode(Database cx, int mode) {
-	state Transaction tr(cx);
-	state BinaryWriter wr(Unversioned());
+Future<int> setBulkDumpMode(Database cx, int mode) {
+	Transaction tr(cx);
+	BinaryWriter wr(Unversioned());
 	wr << mode;
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			state int oldMode = 0;
-			Optional<Value> oldModeValue = wait(tr.get(bulkDumpModeKey));
+			int oldMode = 0;
+			Optional<Value> oldModeValue = co_await tr.get(bulkDumpModeKey);
 			if (oldModeValue.present()) {
 				BinaryReader rd(oldModeValue.get(), Unversioned());
 				rd >> oldMode;
@@ -3012,33 +3076,36 @@ ACTOR Future<int> setBulkDumpMode(Database cx, int mode) {
 				wrLastWrite << deterministicRandom()->randomUniqueID(); // triger DD restarts
 				tr.set(moveKeysLockWriteKey, wrLastWrite.toValue());
 				tr.set(bulkDumpModeKey, wr.toValue());
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent(SevInfo, "DDBulkDumpModeKeyChanged").detail("NewMode", mode).detail("OldMode", oldMode);
 			}
-			return oldMode;
+			co_return oldMode;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<int> getBulkDumpMode(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<int> getBulkDumpMode(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			state int oldMode = 0;
-			Optional<Value> oldModeValue = wait(tr.get(bulkDumpModeKey));
+			int oldMode = 0;
+			Optional<Value> oldModeValue = co_await tr.get(bulkDumpModeKey);
 			if (oldModeValue.present()) {
 				BinaryReader rd(oldModeValue.get(), Unversioned());
 				rd >> oldMode;
 			}
-			return oldMode;
+			co_return oldMode;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -3046,22 +3113,25 @@ ACTOR Future<int> getBulkDumpMode(Database cx) {
 // There is at most one bulk dump job at any time on the entire key space.
 // A job of a range can spawn multiple tasks according to the shard boundary.
 // Those tasks share the same job Id (aka belonging to the same job).
-ACTOR Future<Optional<BulkDumpState>> getSubmittedBulkDumpJob(Transaction* tr) {
-	state RangeResult rangeResult;
-	state KeyRange rangeToRead = normalKeys;
-	state Key beginKey = normalKeys.begin;
+Future<Optional<BulkDumpState>> getSubmittedBulkDumpJob(Transaction* tr) {
+	RangeResult rangeResult;
+	KeyRange rangeToRead = normalKeys;
+	Key beginKey = normalKeys.begin;
 	while (beginKey < normalKeys.end) {
+		Error err;
 		try {
 			rangeResult.clear();
-			wait(store(rangeResult,
-			           krmGetRanges(tr,
-			                        bulkDumpPrefix,
-			                        KeyRangeRef(beginKey, normalKeys.end),
-			                        CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-			                        CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES)));
+			rangeResult = co_await krmGetRanges(tr,
+			                                    bulkDumpPrefix,
+			                                    KeyRangeRef(beginKey, normalKeys.end),
+			                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+			                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 			// krmGetRanges splits the result into batches.
 			// Check first batch is enough since we only check if any task exists
-			for (int i = 0; i < rangeResult.size() - 1; ++i) {
+			if (rangeResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
 				if (rangeResult[i].value.empty()) {
 					continue;
 				}
@@ -3069,24 +3139,27 @@ ACTOR Future<Optional<BulkDumpState>> getSubmittedBulkDumpJob(Transaction* tr) {
 				if (!bulkDumpState.isValid()) {
 					continue;
 				}
-				return bulkDumpState;
+				co_return bulkDumpState;
 			}
 			beginKey = rangeResult.back().key;
+			continue;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
-	return Optional<BulkDumpState>();
+	co_return Optional<BulkDumpState>();
 }
 
-ACTOR Future<Void> submitBulkDumpJob(Database cx, BulkDumpState bulkDumpJob) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> submitBulkDumpJob(Database cx, BulkDumpState bulkDumpJob) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			// There is at most one bulkLoad job or bulkDump job at a time globally
-			Optional<BulkLoadJobState> aliveBulkLoadJob = wait(getSubmittedBulkLoadJob(&tr));
+			Optional<BulkLoadJobState> aliveBulkLoadJob = co_await getSubmittedBulkLoadJob(&tr);
 			if (aliveBulkLoadJob.present()) {
 				TraceEvent(SevWarn, "SubmitBulkDumpJobFailed")
 				    .setMaxEventLength(-1)
@@ -3096,10 +3169,10 @@ ACTOR Future<Void> submitBulkDumpJob(Database cx, BulkDumpState bulkDumpJob) {
 				    .detail("NewJob", bulkDumpJob.toString());
 				throw bulkdump_task_failed();
 			}
-			Optional<BulkDumpState> aliveJob = wait(getSubmittedBulkDumpJob(&tr));
+			Optional<BulkDumpState> aliveJob = co_await getSubmittedBulkDumpJob(&tr);
 			if (aliveJob.present()) {
 				if (aliveJob.get().getJobId() == bulkDumpJob.getJobId()) {
-					return Void(); // The job has been persisted
+					co_return; // The job has been persisted
 				}
 				TraceEvent(SevWarn, "SubmitBulkDumpJobFailed")
 				    .setMaxEventLength(-1)
@@ -3125,29 +3198,30 @@ ACTOR Future<Void> submitBulkDumpJob(Database cx, BulkDumpState bulkDumpJob) {
 				    .detail("NewJob", bulkDumpJob.toString());
 				throw bulkdump_task_failed();
 			}
-			wait(krmSetRange(&tr, bulkDumpPrefix, bulkDumpJob.getJobRange(), bulkDumpStateValue(bulkDumpJob)));
-			wait(tr.commit());
+			co_await krmSetRange(&tr, bulkDumpPrefix, bulkDumpJob.getJobRange(), bulkDumpStateValue(bulkDumpJob));
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> cancelBulkDumpJob(Database cx, UID jobId) {
-	state Transaction tr(cx);
-	state Key beginKey = normalKeys.begin;
-	state Key endKey = normalKeys.end;
-	state BulkDumpState existJob;
-	state KeyRange rangeToRead;
-	state RangeResult bulkDumpResult;
+Future<Void> cancelBulkDumpJob(Database cx, UID jobId) {
+	Transaction tr(cx);
+	Key beginKey = normalKeys.begin;
+	Key endKey = normalKeys.end;
+	BulkDumpState existJob;
+	KeyRange rangeToRead;
+	RangeResult bulkDumpResult;
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			bulkDumpResult.clear();
 			rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
-			wait(store(bulkDumpResult, krmGetRanges(&tr, bulkDumpPrefix, rangeToRead)));
-			for (int i = 0; i < bulkDumpResult.size() - 1; i++) {
+			bulkDumpResult = co_await krmGetRanges(&tr, bulkDumpPrefix, rangeToRead);
+			for (int i = 0; i < static_cast<int>(bulkDumpResult.size()) - 1; i++) {
 				if (bulkDumpResult[i].value.empty()) {
 					continue;
 				}
@@ -3157,8 +3231,9 @@ ACTOR Future<Void> cancelBulkDumpJob(Database cx, UID jobId) {
 				}
 				// We only clear the metadata if it has the same jobId as the input Id.
 				// When there is a new jobId persisted different than the input Id,
-				// a new job has been submitted successfully. Since a new job can be submitted successfully if and only
-				// if no old metadata exists (the old job metadata has been cleared). So, we can stop at this point.
+				// a new job has been submitted successfully. Since a new job can be submitted successfully if and
+				// only if no old metadata exists (the old job metadata has been cleared). So, we can stop at this
+				// point.
 				if (existJob.getJobId() != jobId) {
 					TraceEvent(SevWarn, "DDBulkDumpJobHasChanged")
 					    .detail("InputJobID", jobId.toString())
@@ -3166,48 +3241,117 @@ ACTOR Future<Void> cancelBulkDumpJob(Database cx, UID jobId) {
 					throw bulkload_task_outdated();
 				}
 			}
-			wait(krmSetRangeCoalescing(
-			    &tr, bulkDumpPrefix, rangeToRead, normalKeys, bulkDumpStateValue(BulkDumpState())));
-			wait(tr.commit());
+			co_await krmSetRangeCoalescing(
+			    &tr, bulkDumpPrefix, rangeToRead, normalKeys, bulkDumpStateValue(BulkDumpState()));
+			// Clean up owner info when clearing job metadata
+			tr.clear(bulkDumpOwnerKeyFor(jobId));
+			co_await tr.commit();
 			tr.reset();
 
 			beginKey = bulkDumpResult.back().key;
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
+	}
+}
+
+// Generic owner tracking implementation for bulk operations
+Future<Void> setBulkOwner(Database cx, UID jobId, BulkDumpOwnerInfo ownerInfo, bool isBulkDump) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			Key ownerKey = isBulkDump ? bulkDumpOwnerKeyFor(jobId) : bulkLoadOwnerKeyFor(jobId);
+			tr.set(ownerKey, ObjectWriter::toValue(ownerInfo, IncludeVersion()));
+			co_await tr.commit();
+			co_return;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<size_t> getBulkDumpCompleteTaskCount(Database cx, KeyRange rangeToRead) {
-	state Transaction tr(cx);
-	state Key readBegin = rangeToRead.begin;
-	state Key readEnd = rangeToRead.end;
-	state RangeResult rangeResult;
-	state size_t completeTaskCount = 0;
+Future<Optional<BulkDumpOwnerInfo>> getBulkOwner(Database cx, UID jobId, bool isBulkDump) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			Key ownerKey = isBulkDump ? bulkDumpOwnerKeyFor(jobId) : bulkLoadOwnerKeyFor(jobId);
+			Optional<Value> value = co_await tr.get(ownerKey);
+			if (!value.present()) {
+				co_return Optional<BulkDumpOwnerInfo>();
+			}
+			BulkDumpOwnerInfo info;
+			ObjectReader reader(value.get().begin(), IncludeVersion());
+			reader.deserialize(info);
+			co_return info;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr.onError(err);
+	}
+}
+
+// Public API wrappers for backward compatibility
+Future<Void> setBulkDumpOwner(Database cx, UID jobId, BulkDumpOwnerInfo ownerInfo) {
+	co_await setBulkOwner(cx, jobId, ownerInfo, true);
+}
+
+Future<Optional<BulkDumpOwnerInfo>> getBulkDumpOwner(Database cx, UID jobId) {
+	Optional<BulkDumpOwnerInfo> result = co_await getBulkOwner(cx, jobId, true);
+	co_return result;
+}
+
+Future<Void> setBulkLoadOwner(Database cx, UID jobId, BulkDumpOwnerInfo ownerInfo) {
+	co_await setBulkOwner(cx, jobId, ownerInfo, false);
+}
+
+Future<Optional<BulkDumpOwnerInfo>> getBulkLoadOwner(Database cx, UID jobId) {
+	Optional<BulkDumpOwnerInfo> result = co_await getBulkOwner(cx, jobId, false);
+	co_return result;
+}
+
+Future<size_t> getBulkDumpCompleteTaskCount(Database cx, KeyRange rangeToRead) {
+	Transaction tr(cx);
+	Key readBegin = rangeToRead.begin;
+	Key readEnd = rangeToRead.end;
+	RangeResult rangeResult;
+	size_t completeTaskCount = 0;
 	while (readBegin < readEnd) {
-		state int retryCount = 0;
-		loop {
+		int retryCount = 0;
+		while (true) {
+			Error err;
 			try {
 				rangeResult.clear();
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(store(rangeResult,
-				           krmGetRanges(&tr,
-				                        bulkDumpPrefix,
-				                        KeyRangeRef(readBegin, readEnd),
-				                        CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-				                        CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES)));
+				rangeResult = co_await krmGetRanges(&tr,
+				                                    bulkDumpPrefix,
+				                                    KeyRangeRef(readBegin, readEnd),
+				                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+				                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 				break;
 			} catch (Error& e) {
-				if (retryCount > 30) {
-					throw timed_out();
-				}
-				wait(tr.onError(e));
-				retryCount++;
+				err = e;
+			}
+			if (retryCount > 30) {
+				throw timed_out();
 			}
+			co_await tr.onError(err);
+			retryCount++;
 		}
-		for (int i = 0; i < rangeResult.size() - 1; ++i) {
+		// Guard against empty results (can happen during cluster instability)
+		if (rangeResult.empty()) {
+			break;
+		}
+		for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
 			if (rangeResult[i].value.empty()) {
 				continue;
 			}
@@ -3218,94 +3362,296 @@ ACTOR Future<size_t> getBulkDumpCompleteTaskCount(Database cx, KeyRange rangeToR
 		}
 		readBegin = rangeResult.back().key;
 	}
-	return completeTaskCount;
+	co_return completeTaskCount;
+}
+
+Future<Optional<BulkDumpProgress>> getBulkDumpProgress(Database cx) {
+	Transaction tr(cx);
+	BulkDumpProgress progress;
+	double currentTime = now();
+
+	Optional<BulkDumpState> submittedJob;
+	while (true) {
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<BulkDumpState> job = co_await getSubmittedBulkDumpJob(&tr);
+			submittedJob = job;
+			if (!submittedJob.present()) {
+				co_return Optional<BulkDumpProgress>();
+			}
+			progress.jobId = submittedJob.get().getJobId();
+			progress.jobRange = submittedJob.get().getJobRange();
+			break;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr.onError(err);
+	}
+
+	// Get start time from owner info (stored separately from BulkDumpState)
+	Optional<BulkDumpOwnerInfo> ownerInfo = co_await getBulkDumpOwner(cx, progress.jobId);
+	if (ownerInfo.present()) {
+		progress.startTime = ownerInfo.get().submitTime;
+	} else {
+		// Fallback if no owner info (standalone bulkdump): use current time (elapsed will be ~0)
+		progress.startTime = currentTime;
+	}
+
+	Key readBegin = progress.jobRange.begin;
+	Key readEnd = progress.jobRange.end;
+	RangeResult rangeResult;
+
+	while (readBegin < readEnd) {
+		int retryCount = 0;
+		while (true) {
+			Error err;
+			try {
+				rangeResult.clear();
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				rangeResult = co_await krmGetRanges(&tr,
+				                                    bulkDumpPrefix,
+				                                    KeyRangeRef(readBegin, readEnd),
+				                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+				                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			if (retryCount > 30) {
+				throw timed_out();
+			}
+			co_await tr.onError(err);
+			retryCount++;
+		}
+
+		// Guard against empty results (can happen during cluster instability)
+		if (rangeResult.empty()) {
+			break;
+		}
+		for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
+			if (rangeResult[i].value.empty()) {
+				continue;
+			}
+			BulkDumpState taskState = decodeBulkDumpState(rangeResult[i].value);
+			progress.totalTasks++;
+
+			if (taskState.getPhase() == BulkDumpPhase::Complete) {
+				progress.completeTasks++;
+				progress.completedBytes += taskState.getManifest().getTotalBytes();
+			} else if (taskState.getPhase() == BulkDumpPhase::Submitted) {
+				progress.runningTasks++;
+			}
+
+			progress.totalBytes += taskState.getManifest().getTotalBytes();
+		}
+		readBegin = rangeResult.back().key;
+	}
+
+	progress.elapsedSeconds = currentTime - progress.startTime;
+
+	co_return progress;
+}
+
+Future<Optional<BulkLoadProgress>> getBulkLoadProgress(Database cx) {
+	Transaction tr(cx);
+	BulkLoadProgress progress;
+	double currentTime = now();
+
+	while (true) {
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<BulkLoadJobState> runningJob = co_await getRunningBulkLoadJob(cx);
+			if (!runningJob.present()) {
+				co_return Optional<BulkLoadProgress>();
+			}
+			progress.jobId = runningJob.get().getJobId();
+			progress.jobRange = runningJob.get().getJobRange();
+			Optional<uint64_t> taskCount = runningJob.get().getTaskCount();
+			progress.totalTasks = taskCount.present() ? (int)taskCount.get() : 0;
+			progress.startTime = runningJob.get().getSubmitTime();
+			progress.elapsedSeconds = currentTime - progress.startTime;
+			break;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr.onError(err);
+	}
+
+	Key readBegin = progress.jobRange.begin;
+	Key readEnd = progress.jobRange.end;
+	RangeResult rangeResult;
+
+	while (readBegin < readEnd) {
+		int retryCount = 0;
+		while (true) {
+			Error err;
+			try {
+				rangeResult.clear();
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				rangeResult = co_await krmGetRanges(&tr,
+				                                    bulkLoadTaskPrefix,
+				                                    KeyRangeRef(readBegin, readEnd),
+				                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+				                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			if (retryCount > 30) {
+				throw timed_out();
+			}
+			co_await tr.onError(err);
+			retryCount++;
+		}
+
+		// Guard against empty results (can happen during cluster instability)
+		if (rangeResult.empty()) {
+			break;
+		}
+		for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
+			if (rangeResult[i].value.empty()) {
+				continue;
+			}
+			BulkLoadTaskState taskState = decodeBulkLoadTaskState(rangeResult[i].value);
+
+			switch (taskState.phase) {
+			case BulkLoadPhase::Submitted:
+				progress.submittedTasks++;
+				break;
+			case BulkLoadPhase::Triggered:
+				progress.triggeredTasks++;
+				break;
+			case BulkLoadPhase::Running:
+				progress.runningTasks++;
+				if (taskState.startTime > 0 &&
+				    (currentTime - taskState.startTime) > BULK_TASK_STALL_THRESHOLD_SECONDS) {
+					BulkLoadStalledTask stalledTask;
+					stalledTask.taskId = taskState.getTaskId();
+					stalledTask.range = taskState.getRange();
+					stalledTask.stalledSeconds = currentTime - taskState.startTime;
+					stalledTask.restartCount = taskState.restartCount;
+					// TODO: Get storage server ID and last error when available
+					progress.stalledTasks.push_back(stalledTask);
+				}
+				break;
+			case BulkLoadPhase::Complete:
+			case BulkLoadPhase::Acknowledged:
+				progress.completeTasks++;
+				progress.completedBytes += taskState.getTotalBytes();
+				break;
+			case BulkLoadPhase::Error:
+				progress.errorTasks++;
+				break;
+			default:
+				break;
+			}
+
+			progress.totalBytes += taskState.getTotalBytes();
+		}
+		readBegin = rangeResult.back().key;
+	}
+
+	co_return progress;
 }
 
 // Persist a new owner if input ownerUniqueID is not existing; Update description if input ownerUniqueID exists
-ACTOR Future<Void> registerRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID, std::string description) {
+Future<Void> registerRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID, std::string description) {
 	if (ownerUniqueID.empty() || description.empty()) {
 		throw range_lock_failed();
 	}
-	state Transaction tr(cx);
-	loop {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			Optional<Value> res = wait(tr.get(rangeLockOwnerKeyFor(ownerUniqueID)));
+			Optional<Value> res = co_await tr.get(rangeLockOwnerKeyFor(ownerUniqueID));
 			RangeLockOwner owner;
 			if (res.present()) {
 				owner = decodeRangeLockOwner(res.get());
 				ASSERT(owner.isValid());
 				if (owner.getDescription() == description) {
-					return Void();
+					co_return;
 				}
 				owner.setDescription(description);
 			} else {
 				owner = RangeLockOwner(ownerUniqueID, description);
 			}
 			tr.set(rangeLockOwnerKeyFor(ownerUniqueID), rangeLockOwnerValue(owner));
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> removeRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID) {
+Future<Void> removeRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID) {
 	if (ownerUniqueID.empty()) {
 		throw range_lock_failed();
 	}
-	state Transaction tr(cx);
-	loop {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			Optional<Value> res = wait(tr.get(rangeLockOwnerKeyFor(ownerUniqueID)));
+			Optional<Value> res = co_await tr.get(rangeLockOwnerKeyFor(ownerUniqueID));
 			if (!res.present()) {
-				return Void();
+				co_return;
 			}
 			RangeLockOwner owner = decodeRangeLockOwner(res.get());
 			ASSERT(owner.isValid());
 			tr.clear(rangeLockOwnerKeyFor(ownerUniqueID));
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Optional<RangeLockOwner>> getRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID) {
-	state Transaction tr(cx);
-	loop {
+Future<Optional<RangeLockOwner>> getRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			Optional<Value> res = wait(tr.get(rangeLockOwnerKeyFor(ownerUniqueID)));
+			Optional<Value> res = co_await tr.get(rangeLockOwnerKeyFor(ownerUniqueID));
 			if (!res.present()) {
-				return Optional<RangeLockOwner>();
+				co_return Optional<RangeLockOwner>();
 			}
 			RangeLockOwner owner = decodeRangeLockOwner(res.get());
 			ASSERT(owner.isValid());
-			return owner;
+			co_return owner;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<std::vector<RangeLockOwner>> getAllRangeLockOwners(Database cx) {
-	state std::vector<RangeLockOwner> res;
-	state Key beginKey = rangeLockOwnerKeys.begin;
-	state Key endKey = rangeLockOwnerKeys.end;
-	state Transaction tr(cx);
+AsyncResult<std::vector<RangeLockOwner>> getAllRangeLockOwners(Database cx) {
+	std::vector<RangeLockOwner> res;
+	Key beginKey = rangeLockOwnerKeys.begin;
+	Key endKey = rangeLockOwnerKeys.end;
+	Transaction tr(cx);
 	while (beginKey < endKey) {
-		state KeyRange rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
+		KeyRange rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			RangeResult result = wait(tr.getRange(rangeToRead, CLIENT_KNOBS->TOO_MANY));
+			RangeResult result = co_await tr.getRange(rangeToRead, CLIENT_KNOBS->TOO_MANY);
 			for (const auto& kv : result) {
 				RangeLockOwner owner = decodeRangeLockOwner(kv.value);
 				ASSERT(owner.isValid());
@@ -3315,30 +3661,36 @@ ACTOR Future<std::vector<RangeLockOwner>> getAllRangeLockOwners(Database cx) {
 			if (!result.more) {
 				break;
 			}
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return res;
+	co_return res;
 }
 
 // Not transactional
-ACTOR Future<std::vector<std::pair<KeyRange, RangeLockState>>>
+Future<std::vector<std::pair<KeyRange, RangeLockState>>>
 findExclusiveReadLockOnRange(Database cx, KeyRange range, Optional<RangeLockOwnerName> ownerName) {
 	if (range.end > normalKeys.end) {
 		throw range_lock_failed();
 	}
-	state std::vector<std::pair<KeyRange, RangeLockState>> lockedRanges;
-	state Key beginKey = range.begin;
-	state Key endKey = range.end;
-	state Transaction tr(cx);
+	std::vector<std::pair<KeyRange, RangeLockState>> lockedRanges;
+	Key beginKey = range.begin;
+	Key endKey = range.end;
+	Transaction tr(cx);
 	while (beginKey < endKey) {
-		state KeyRange rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
+		KeyRange rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			RangeResult result = wait(krmGetRanges(&tr, rangeLockPrefix, rangeToRead));
-			for (int i = 0; i < result.size() - 1; i++) {
+			RangeResult result = co_await krmGetRanges(&tr, rangeLockPrefix, rangeToRead);
+			if (result.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(result.size()) - 1; i++) {
 				if (result[i].value.empty()) {
 					continue;
 				}
@@ -3353,19 +3705,19 @@ findExclusiveReadLockOnRange(Database cx, KeyRange range, Optional<RangeLockOwne
 				}
 			}
 			beginKey = result.back().key;
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return lockedRanges;
+	co_return lockedRanges;
 }
 
 // Validate the input range and owner.
 // If invalid, reject the request by throwing range_lock_failed error.
 // If the range has been locked, reject the request by throwing range_lock_reject error.
-ACTOR Future<Void> prepareExclusiveRangeLockOperation(Transaction* tr,
-                                                      KeyRange range,
-                                                      RangeLockOwnerName ownerUniqueID) {
+Future<Void> prepareExclusiveRangeLockOperation(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID) {
 	// Check input range
 	if (range.end > normalKeys.end) {
 		TraceEvent(SevDebug, "PrepareExclusiveRangeLockOperationFailed")
@@ -3374,7 +3726,7 @@ ACTOR Future<Void> prepareExclusiveRangeLockOperation(Transaction* tr,
 		throw range_lock_failed();
 	}
 	// Check owner
-	state Optional<Value> ownerValue = wait(tr->get(rangeLockOwnerKeyFor(ownerUniqueID)));
+	Optional<Value> ownerValue = co_await tr->get(rangeLockOwnerKeyFor(ownerUniqueID));
 	if (!ownerValue.present()) {
 		TraceEvent(SevDebug, "PrepareExclusiveRangeLockOperationFailed")
 		    .detail("Reason", "Owner not found")
@@ -3382,16 +3734,19 @@ ACTOR Future<Void> prepareExclusiveRangeLockOperation(Transaction* tr,
 		    .detail("Range", range);
 		throw range_lock_failed();
 	}
-	state RangeLockOwner owner = decodeRangeLockOwner(ownerValue.get());
+	RangeLockOwner owner = decodeRangeLockOwner(ownerValue.get());
 	ASSERT(owner.isValid());
 	// Check lock state on the entire input range. Throw exception if the range has been locked by a different owner.
-	state Key beginKey = range.begin;
-	state Key endKey = range.end;
-	state KeyRange rangeToRead;
+	Key beginKey = range.begin;
+	Key endKey = range.end;
+	KeyRange rangeToRead;
 	while (beginKey < endKey) {
 		rangeToRead = KeyRangeRef(beginKey, endKey);
-		RangeResult res = wait(krmGetRanges(tr, rangeLockPrefix, rangeToRead));
-		for (int i = 0; i < res.size() - 1; i++) {
+		RangeResult res = co_await krmGetRanges(tr, rangeLockPrefix, rangeToRead);
+		if (res.empty()) {
+			break;
+		}
+		for (int i = 0; i < static_cast<int>(res.size()) - 1; i++) {
 			if (res[i].value.empty()) {
 				continue;
 			}
@@ -3410,14 +3765,11 @@ ACTOR Future<Void> prepareExclusiveRangeLockOperation(Transaction* tr,
 				throw range_lock_reject(); // Has been locked
 			}
 		}
-		beginKey = res[res.size() - 1].key;
+		beginKey = res.back().key;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> prepareExclusiveRangeUnlockOperation(Transaction* tr,
-                                                        KeyRange range,
-                                                        RangeLockOwnerName ownerUniqueID) {
+Future<Void> prepareExclusiveRangeUnlockOperation(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID) {
 	// Check input range
 	if (range.end > normalKeys.end) {
 		TraceEvent(SevDebug, "PrepareExclusiveRangeUnlockOperationFailed")
@@ -3426,7 +3778,7 @@ ACTOR Future<Void> prepareExclusiveRangeUnlockOperation(Transaction* tr,
 		throw range_lock_failed();
 	}
 	// Check owner
-	state Optional<Value> ownerValue = wait(tr->get(rangeLockOwnerKeyFor(ownerUniqueID)));
+	Optional<Value> ownerValue = co_await tr->get(rangeLockOwnerKeyFor(ownerUniqueID));
 	if (!ownerValue.present()) {
 		TraceEvent(SevDebug, "PrepareExclusiveRangeUnlockOperationFailed")
 		    .detail("Reason", "Owner not found")
@@ -3434,17 +3786,20 @@ ACTOR Future<Void> prepareExclusiveRangeUnlockOperation(Transaction* tr,
 		    .detail("Range", range);
 		throw range_lock_failed();
 	}
-	state RangeLockOwner owner = decodeRangeLockOwner(ownerValue.get());
+	RangeLockOwner owner = decodeRangeLockOwner(ownerValue.get());
 	ASSERT(owner.isValid());
 
 	// Check lock state on the entire input range. Throw exception if the range has been locked by a different owner.
-	state Key beginKey = range.begin;
-	state Key endKey = range.end;
-	state KeyRange rangeToRead;
+	Key beginKey = range.begin;
+	Key endKey = range.end;
+	KeyRange rangeToRead;
 	while (beginKey < endKey) {
 		rangeToRead = KeyRangeRef(beginKey, endKey);
-		RangeResult res = wait(krmGetRanges(tr, rangeLockPrefix, rangeToRead));
-		for (int i = 0; i < res.size() - 1; i++) {
+		RangeResult res = co_await krmGetRanges(tr, rangeLockPrefix, rangeToRead);
+		if (res.empty()) {
+			break;
+		}
+		for (int i = 0; i < static_cast<int>(res.size()) - 1; i++) {
 			if (res[i].value.empty()) {
 				continue;
 			}
@@ -3462,64 +3817,65 @@ ACTOR Future<Void> prepareExclusiveRangeUnlockOperation(Transaction* tr,
 				throw range_unlock_reject();
 			}
 		}
-		beginKey = res[res.size() - 1].key;
+		beginKey = res.back().key;
 	}
-	return Void();
 }
 
 // Transactional. One transaction can call takeExclusiveReadLockOnRange at most for one time.
 // This is the limitation of the krmSetRangeCoalescing.
-ACTOR Future<Void> takeExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID) {
+Future<Void> takeExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID) {
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	// Add conflict range
 	tr->addWriteConflictRange(range);
-	wait(prepareExclusiveRangeLockOperation(tr, range, ownerUniqueID));
+	co_await prepareExclusiveRangeLockOperation(tr, range, ownerUniqueID);
 	// At this point, no lock presents on the range.
 	// Lock range by writting the range.
 	RangeLockStateSet rangeLockStateSet;
 	rangeLockStateSet.insertIfNotExist(RangeLockState(RangeLockType::ExclusiveReadLock, ownerUniqueID, range));
-	wait(krmSetRange(tr, rangeLockPrefix, range, rangeLockStateSetValue(rangeLockStateSet)));
+	co_await krmSetRange(tr, rangeLockPrefix, range, rangeLockStateSetValue(rangeLockStateSet));
 	TraceEvent(SevInfo, "TakeExclusiveReadLockTransactionOnRange").detail("Range", range);
-	return Void();
 }
 
 // Transactional. One transaction can call releaseExclusiveReadLockOnRange at most for one time.
 // This is the limitation of the krmSetRangeCoalescing.
-ACTOR Future<Void> releaseExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID) {
+Future<Void> releaseExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID) {
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	wait(prepareExclusiveRangeUnlockOperation(tr, range, ownerUniqueID));
+	co_await prepareExclusiveRangeUnlockOperation(tr, range, ownerUniqueID);
 	// At this point, no lock presents on the range.
 	// Unlock by overwiting the range.
-	wait(krmSetRangeCoalescing(tr, rangeLockPrefix, range, normalKeys, rangeLockStateSetValue(RangeLockStateSet())));
+	co_await krmSetRangeCoalescing(tr, rangeLockPrefix, range, normalKeys, rangeLockStateSetValue(RangeLockStateSet()));
 	TraceEvent(SevInfo, "ReleaseExclusiveReadLockTransactionOnRange").detail("Range", range);
-	return Void();
 }
 
-ACTOR Future<Void> releaseExclusiveReadLockByUser(Database cx, RangeLockOwnerName ownerUniqueID) {
-	state Key beginKey = normalKeys.begin;
-	state Key endKey = normalKeys.end;
-	state Transaction tr(cx);
-	state int i = 0;
-	state RangeResult result;
-	state KeyRange rangeToRead;
-	state RangeLockStateSet currentRangeLockStateSet;
-	state KeyRange currentRange;
-	state Key beginKeyToClear;
-	state Key endKeyToClear;
+Future<Void> releaseExclusiveReadLockByUser(Database cx, RangeLockOwnerName ownerUniqueID) {
+	Key beginKey = normalKeys.begin;
+	Key endKey = normalKeys.end;
+	Transaction tr(cx);
+	int i = 0;
+	RangeResult result;
+	KeyRange rangeToRead;
+	RangeLockStateSet currentRangeLockStateSet;
+	KeyRange currentRange;
+	Key beginKeyToClear;
+	Key endKeyToClear;
 	while (beginKey < endKey) {
 		rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
+		Error err;
 		try {
 			tr.reset();
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			result.clear();
-			wait(store(result, krmGetRanges(&tr, rangeLockPrefix, rangeToRead)));
+			result = co_await krmGetRanges(&tr, rangeLockPrefix, rangeToRead);
+			if (result.empty()) {
+				break;
+			}
 			i = 0;
 			beginKeyToClear = result[0].key;
 			endKeyToClear = result[0].key; // Expanding when currentRange is valid to clear
-			for (; i < result.size() - 1; i++) {
+			for (; i < static_cast<int>(result.size()) - 1; i++) {
 				currentRange = KeyRangeRef(result[i].key, result[i + 1].key);
 				if (result[i].value.empty()) {
 					endKeyToClear = currentRange.end;
@@ -3537,71 +3893,77 @@ ACTOR Future<Void> releaseExclusiveReadLockByUser(Database cx, RangeLockOwnerNam
 			}
 			if (beginKeyToClear != endKeyToClear) {
 				ASSERT(endKeyToClear > beginKeyToClear);
-				wait(krmSetRangeCoalescing(&tr,
-				                           rangeLockPrefix,
-				                           KeyRangeRef(beginKeyToClear, endKeyToClear),
-				                           normalKeys,
-				                           rangeLockStateSetValue(RangeLockStateSet())));
-				wait(tr.commit());
+				co_await krmSetRangeCoalescing(&tr,
+				                               rangeLockPrefix,
+				                               KeyRangeRef(beginKeyToClear, endKeyToClear),
+				                               normalKeys,
+				                               rangeLockStateSetValue(RangeLockStateSet()));
+				co_await tr.commit();
 			}
 			beginKey = currentRange.end; // We skip the currentRange if it is not locked by the input owner.
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
 // Transactional
-ACTOR Future<Void> takeExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> takeExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			wait(takeExclusiveReadLockOnRange(&tr, range, ownerUniqueID));
-			wait(tr.commit());
+			co_await takeExclusiveReadLockOnRange(&tr, range, ownerUniqueID);
+			co_await tr.commit();
 			TraceEvent(SevInfo, "TakeExclusiveReadLockOnRange").detail("Range", range);
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
 // Transactional
-ACTOR Future<Void> releaseExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> releaseExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			wait(releaseExclusiveReadLockOnRange(&tr, range, ownerUniqueID));
-			wait(tr.commit());
+			co_await releaseExclusiveReadLockOnRange(&tr, range, ownerUniqueID);
+			co_await tr.commit();
 			TraceEvent(SevInfo, "ReleaseExclusiveReadLockOnRange").detail("Range", range);
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> waitForPrimaryDC(Database cx, StringRef dcId) {
-	state ReadYourWritesTransaction tr(cx);
+Future<Void> waitForPrimaryDC(Database cx, StringRef dcId) {
+	ReadYourWritesTransaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			Optional<Value> res = wait(tr.get(primaryDatacenterKey));
+			Optional<Value> res = co_await tr.get(primaryDatacenterKey);
 			if (res.present() && res.get() == dcId) {
-				return Void();
+				co_return;
 			}
 
-			state Future<Void> watchFuture = tr.watch(primaryDatacenterKey);
-			wait(tr.commit());
-			wait(watchFuture);
+			Future<Void> watchFuture = tr.watch(primaryDatacenterKey);
+			co_await tr.commit();
+			co_await watchFuture;
 			tr.reset();
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -3656,7 +4018,7 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
 					schemaCoverage(spath);
 				}
 
-				if (!schema.count(key)) {
+				if (!schema.contains(key)) {
 					errorStr += format("ERROR: Unknown key `%s'\n", kpath.c_str());
 					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaPath", spath);
 					ok = false;
@@ -3664,7 +4026,7 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
 				}
 				auto& sv = schema.at(key);
 
-				if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$enum")) {
+				if (sv.type() == json_spirit::obj_type && sv.get_obj().contains("$enum")) {
 					auto& enum_values = sv.get_obj().at("$enum").get_array();
 
 					bool any_match = false;
@@ -3689,7 +4051,7 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
 						}
 						ok = false;
 					}
-				} else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) {
+				} else if (sv.type() == json_spirit::obj_type && sv.get_obj().contains("$map")) {
 					if (rv.type() != json_spirit::obj_type) {
 						errorStr += format("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str());
 						TraceEvent(sev, "SchemaMismatch")
@@ -3733,9 +4095,9 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
 		} else if (resultValue.type() == json_spirit::array_type) {
 			auto& valueArray = resultValue.get_array();
 			auto& schemaArray = schemaValue.get_array();
-			if (!schemaArray.size()) {
+			if (schemaArray.empty()) {
 				// An empty schema array means that the value array is required to be empty
-				if (valueArray.size()) {
+				if (!valueArray.empty()) {
 					errorStr += format("ERROR: Expected an empty array for key `%s'\n", path.c_str());
 					TraceEvent(sev, "SchemaMismatch")
 					    .detail("Path", path)
@@ -3807,8 +4169,6 @@ std::string ManagementAPI::generateErrorMessage(const CoordinatorsResult& res) {
 }
 
 TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") {
-	wait(Future<Void>(Void()));
-
 	std::vector<ProcessData> workers;
 	std::vector<NetworkAddress> chosen;
 	std::set<AddressExclusion> excluded;
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.cpp
similarity index 80%
rename from fdbclient/MonitorLeader.actor.cpp
rename to fdbclient/MonitorLeader.cpp
index 83c2d434050..86e58a2d26f 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.cpp
@@ -1,5 +1,5 @@
 /*
- * MonitorLeader.actor.cpp
+ * MonitorLeader.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,10 +24,10 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "flow/ActorCollection.h"
 #include "flow/UnitTest.h"
-#include "fdbrpc/genericactors.actor.h"
+#include "fdbrpc/genericactors.h"
 #include "flow/Platform.h"
 #include "flow/IConnection.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "flow/CoroUtils.h"
 
 namespace {
 
@@ -126,7 +126,7 @@ ClusterConnectionString::ClusterConnectionString(const std::string& connectionSt
 }
 
 TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/addresses") {
-	state std::string input;
+	std::string input;
 
 	{
 		input = "asdf:2345@1.1.1.1:345";
@@ -186,7 +186,7 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/addresses") {
 }
 
 TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/hostnames") {
-	state std::string input;
+	std::string input;
 
 	{
 		input = "asdf:2345@localhost:1234";
@@ -267,8 +267,8 @@ TEST_CASE("/fdbclient/MonitorLeader/ConnectionString/hostname") {
 	return Void();
 }
 
-ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnectionString* self) {
-	state std::set<NetworkAddress> allCoordinatorsSet;
+Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnectionString* self) {
+	std::set<NetworkAddress> allCoordinatorsSet;
 	for (const auto& coord : self->coords) {
 		allCoordinatorsSet.insert(coord);
 	}
@@ -281,10 +281,10 @@ ACTOR Future<std::vector<NetworkAddress>> tryResolveHostnamesImpl(ClusterConnect
 			return Void();
 		}));
 	}
-	wait(waitForAll(fs));
+	co_await waitForAll(fs);
 	std::vector<NetworkAddress> allCoordinators(allCoordinatorsSet.begin(), allCoordinatorsSet.end());
 	std::sort(allCoordinators.begin(), allCoordinators.end());
-	return allCoordinators;
+	co_return allCoordinators;
 }
 
 Future<std::vector<NetworkAddress>> ClusterConnectionString::tryResolveHostnames() {
@@ -295,16 +295,14 @@ TEST_CASE("/fdbclient/MonitorLeader/PartialResolve") {
 	std::string connectionString = "TestCluster:0@host.name:1234,host-name:5678";
 	std::string hn = "host-name", port = "5678";
 
-	state NetworkAddress address = NetworkAddress::parse("1.0.0.0:5678");
+	NetworkAddress address = NetworkAddress::parse("1.0.0.0:5678");
 
 	INetworkConnections::net()->addMockTCPEndpoint(hn, port, { address });
 
 	ClusterConnectionString cs(connectionString);
-	std::vector<NetworkAddress> allCoordinators = wait(cs.tryResolveHostnames());
+	std::vector<NetworkAddress> allCoordinators = co_await cs.tryResolveHostnames();
 	ASSERT(allCoordinators.size() == 1 &&
 	       std::find(allCoordinators.begin(), allCoordinators.end(), address) != allCoordinators.end());
-
-	return Void();
 }
 
 TEST_CASE("/flow/FlatBuffers/LeaderInfo") {
@@ -494,27 +492,26 @@ std::string ClientLeaderRegInterface::getAddressString() const {
 
 // Nominee is the worker among all workers that are considered as leader by one coordinator
 // This function contacts a coordinator coord to ask who is its nominee.
-ACTOR Future<Void> monitorNominee(Key key,
-                                  ClientLeaderRegInterface coord,
-                                  AsyncTrigger* nomineeChange,
-                                  Optional<LeaderInfo>* info) {
-	loop {
-		state Optional<LeaderInfo> li;
-		wait(Future<Void>(Void())); // Make sure we weren't cancelled
+Future<Void> monitorNominee(Key key,
+                            ClientLeaderRegInterface coord,
+                            AsyncTrigger* nomineeChange,
+                            Optional<LeaderInfo>* info) {
+	while (true) {
+		Optional<LeaderInfo> li;
+		co_await Future<Void>(Void()); // Make sure we weren't cancelled
 		if (coord.hostname.present()) {
-			wait(store(li,
-			           retryGetReplyFromHostname(GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
-			                                     coord.hostname.get(),
-			                                     WLTOKEN_CLIENTLEADERREG_GETLEADER,
-			                                     TaskPriority::CoordinationReply)));
+			li = co_await retryGetReplyFromHostname(
+			    GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
+			    coord.hostname.get(),
+			    WLTOKEN_CLIENTLEADERREG_GETLEADER,
+			    TaskPriority::CoordinationReply);
 		} else {
-			wait(store(li,
-			           retryBrokenPromise(coord.getLeader,
-			                              GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
-			                              TaskPriority::CoordinationReply)));
+			li = co_await retryBrokenPromise(coord.getLeader,
+			                                 GetLeaderRequest(key, info->present() ? info->get().changeID : UID()),
+			                                 TaskPriority::CoordinationReply);
 		}
 
-		wait(Future<Void>(Void())); // Make sure we weren't cancelled
+		co_await Future<Void>(Void()); // Make sure we weren't cancelled
 
 		TraceEvent("GetLeaderReply")
 		    .suppressFor(1.0)
@@ -527,7 +524,7 @@ ACTOR Future<Void> monitorNominee(Key key,
 			nomineeChange->trigger();
 
 			if (li.present() && li.get().forward)
-				wait(Future<Void>(Never()));
+				co_await Future<Void>(Never());
 		}
 	}
 }
@@ -580,18 +577,18 @@ Optional<std::pair<LeaderInfo, bool>> getLeader(const std::vector<Optional<Leade
 }
 
 // Leader is the process that will be elected by coordinators as the cluster controller
-ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterConnectionRecord> connRecord,
-                                                           Reference<AsyncVar<Value>> outSerializedLeaderInfo,
-                                                           MonitorLeaderInfo info) {
-	state ClientCoordinators coordinators(info.intermediateConnRecord);
-	state AsyncTrigger nomineeChange;
-	state std::vector<Optional<LeaderInfo>> nominees;
-	state Future<Void> allActors;
-	state Optional<std::pair<LeaderInfo, bool>> leader;
+Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterConnectionRecord> connRecord,
+                                                     Reference<AsyncVar<Value>> outSerializedLeaderInfo,
+                                                     MonitorLeaderInfo info) {
+	ClientCoordinators coordinators(info.intermediateConnRecord);
+	AsyncTrigger nomineeChange;
+	std::vector<Optional<LeaderInfo>> nominees;
+	Future<Void> allActors;
+	Optional<std::pair<LeaderInfo, bool>> leader;
 
 	nominees.resize(coordinators.clientLeaderServers.size());
 
-	state std::vector<Future<Void>> actors;
+	std::vector<Future<Void>> actors;
 	// Ask all coordinators if the worker is considered as a leader (leader nominee) by the coordinator.
 	actors.reserve(coordinators.clientLeaderServers.size());
 	for (int i = 0; i < coordinators.clientLeaderServers.size(); i++) {
@@ -600,7 +597,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 	}
 	allActors = waitForAll(actors);
 
-	loop {
+	while (true) {
 		leader = getLeader(nominees);
 		TraceEvent("MonitorLeaderChange")
 		    .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1));
@@ -612,7 +609,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 				    .trackLatest("MonitorLeaderForwarding");
 				info.intermediateConnRecord = connRecord->makeIntermediateRecord(
 				    ClusterConnectionString(leader.get().first.serializedInfo.toString()));
-				return info;
+				co_return info;
 			}
 			if (connRecord != info.intermediateConnRecord) {
 				if (!info.hasConnected) {
@@ -622,7 +619,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 					    .detail("CurrentConnectionString",
 					            info.intermediateConnRecord->getConnectionString().toString());
 				}
-				wait(connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString()));
+				co_await connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString());
 				info.intermediateConnRecord = connRecord;
 			}
 
@@ -631,36 +628,36 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration(Reference<IClusterCon
 
 			outSerializedLeaderInfo->set(leader.get().first.serializedInfo);
 		}
-		wait(nomineeChange.onTrigger() || allActors);
+		co_await (nomineeChange.onTrigger() || allActors);
 	}
 }
 
-ACTOR Future<Void> monitorLeaderInternal(Reference<IClusterConnectionRecord> connRecord,
-                                         Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
-	state MonitorLeaderInfo info(connRecord);
-	loop {
-		MonitorLeaderInfo _info = wait(monitorLeaderOneGeneration(connRecord, outSerializedLeaderInfo, info));
+Future<Void> monitorLeaderInternal(Reference<IClusterConnectionRecord> connRecord,
+                                   Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
+	MonitorLeaderInfo info(connRecord);
+	while (true) {
+		MonitorLeaderInfo _info = co_await monitorLeaderOneGeneration(connRecord, outSerializedLeaderInfo, info);
 		info = _info;
 	}
 }
 
-ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> serializedInfo,
-                                                    Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
-	state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
+Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> serializedInfo,
+                                              Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
+	Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
 	    new AsyncVar<Optional<ClusterControllerClientInterface>>{});
-	state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
-	loop {
-		choose {
-			when(wait(deserializer)) {
-				UNSTOPPABLE_ASSERT(false);
-			}
-			when(wait(knownLeader->onChange())) {
-				if (knownLeader->get().present()) {
-					outKnownLeader->set(knownLeader->get().get().clientInterface);
-				} else {
-					outKnownLeader->set(Optional<ClusterInterface>{});
-				}
+	Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
+	while (true) {
+		auto res = co_await race(deserializer, knownLeader->onChange());
+		if (res.index() == 0) {
+			UNSTOPPABLE_ASSERT(false);
+		} else if (res.index() == 1) {
+			if (knownLeader->get().present()) {
+				outKnownLeader->set(knownLeader->get().get().clientInterface);
+			} else {
+				outKnownLeader->set(Optional<ClusterInterface>{});
 			}
+		} else {
+			UNREACHABLE();
 		}
 	}
 }
@@ -706,16 +703,16 @@ OpenDatabaseRequest ClientData::getRequest() {
 	return req;
 }
 
-ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader,
-                                           ClientData* clientData) {
+Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader,
+                                     ClientData* clientData) {
 	while (!knownLeader->get().present()) {
-		wait(knownLeader->onChange());
+		co_await knownLeader->onChange();
 	}
 
-	state double lastRequestTime = now();
-	state OpenDatabaseRequest req = clientData->getRequest();
+	double lastRequestTime = now();
+	OpenDatabaseRequest req = clientData->getRequest();
 
-	loop {
+	while (true) {
 		if (now() - lastRequestTime > CLIENT_KNOBS->MAX_CLIENT_STATUS_AGE) {
 			lastRequestTime = now();
 			req = clientData->getRequest();
@@ -723,30 +720,30 @@ ACTOR Future<Void> getClientInfoFromLeader(Reference<AsyncVar<Optional<ClusterCo
 			resetReply(req);
 		}
 		req.knownClientInfoID = clientData->clientInfo->get().read().id;
-		choose {
-			when(ClientDBInfo ni =
-			         wait(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)))) {
-				TraceEvent("GetClientInfoFromLeaderGotClientInfo", knownLeader->get().get().clientInterface.id())
-				    .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].address().toString() : "")
-				    .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].address().toString() : "")
-				    .detail("ClientID", ni.id);
-				clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(ni));
-			}
-			when(wait(knownLeader->onChange())) {}
-		}
-	}
-}
-
-ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
-                                                 std::vector<Hostname> hostnames,
-                                                 std::vector<NetworkAddress> coordinators,
-                                                 ClientData* clientData,
-                                                 Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
-	state std::vector<ClientLeaderRegInterface> clientLeaderServers;
-	state AsyncTrigger nomineeChange;
-	state std::vector<Optional<LeaderInfo>> nominees;
-	state Future<Void> allActors;
-	state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
+		auto res =
+		    co_await race(brokenPromiseToNever(knownLeader->get().get().clientInterface.openDatabase.getReply(req)),
+		                  knownLeader->onChange());
+		if (res.index() == 0) {
+			ClientDBInfo ni = std::get<0>(std::move(res));
+			TraceEvent("GetClientInfoFromLeaderGotClientInfo", knownLeader->get().get().clientInterface.id())
+			    .detail("CommitProxy0", ni.commitProxies.size() ? ni.commitProxies[0].address().toString() : "")
+			    .detail("GrvProxy0", ni.grvProxies.size() ? ni.grvProxies[0].address().toString() : "")
+			    .detail("ClientID", ni.id);
+			clientData->clientInfo->set(CachedSerialization<ClientDBInfo>(ni));
+		}
+	}
+}
+
+Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
+                                           std::vector<Hostname> hostnames,
+                                           std::vector<NetworkAddress> coordinators,
+                                           ClientData* clientData,
+                                           Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo) {
+	std::vector<ClientLeaderRegInterface> clientLeaderServers;
+	AsyncTrigger nomineeChange;
+	std::vector<Optional<LeaderInfo>> nominees;
+	Future<Void> allActors;
+	Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
 	    new AsyncVar<Optional<ClusterControllerClientInterface>>{});
 
 	clientLeaderServers.reserve(hostnames.size() + coordinators.size());
@@ -768,7 +765,7 @@ ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
 	actors.push_back(getClientInfoFromLeader(knownLeader, clientData));
 	allActors = waitForAll(actors);
 
-	loop {
+	while (true) {
 		Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
 		TraceEvent("MonitorLeaderAndGetClientInfoLeaderChange")
 		    .detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1, 1))
@@ -782,7 +779,7 @@ ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
 				leaderInfo->set(leader.get().first);
 				TraceEvent("MonitorLeaderAndGetClientInfoForwarding")
 				    .detail("NewConnStr", leader.get().first.serializedInfo.toString());
-				return Void();
+				co_return;
 			}
 
 			if (leader.get().first.serializedInfo.size()) {
@@ -793,7 +790,7 @@ ACTOR Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
 				leaderInfo->set(leader.get().first);
 			}
 		}
-		wait(nomineeChange.onTrigger() || allActors);
+		co_await (nomineeChange.onTrigger() || allActors);
 	}
 }
 
@@ -837,7 +834,7 @@ void shrinkProxyList(ClientDBInfo& ni,
 	}
 }
 
-ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
+Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
     Reference<IClusterConnectionRecord> connRecord,
     Reference<AsyncVar<ClientDBInfo>> clientInfo,
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
@@ -845,17 +842,17 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
     Key traceLogGroup,
     IsInternal internal) {
-	state ClusterConnectionString cs = info.intermediateConnRecord->getConnectionString();
-	state int coordinatorsSize = cs.hostnames.size() + cs.coords.size();
-	state int index = 0;
-	state int successIndex = 0;
-	state Optional<double> incorrectTime;
-	state std::vector<UID> lastCommitProxyUIDs;
-	state std::vector<CommitProxyInterface> lastCommitProxies;
-	state std::vector<UID> lastGrvProxyUIDs;
-	state std::vector<GrvProxyInterface> lastGrvProxies;
-	state std::vector<ClientLeaderRegInterface> clientLeaderServers;
-	state bool allConnectionsFailed = false;
+	ClusterConnectionString cs = info.intermediateConnRecord->getConnectionString();
+	int coordinatorsSize = cs.hostnames.size() + cs.coords.size();
+	int index = 0;
+	int successIndex = 0;
+	Optional<double> incorrectTime;
+	std::vector<UID> lastCommitProxyUIDs;
+	std::vector<CommitProxyInterface> lastCommitProxies;
+	std::vector<UID> lastGrvProxyUIDs;
+	std::vector<GrvProxyInterface> lastGrvProxies;
+	std::vector<ClientLeaderRegInterface> clientLeaderServers;
+	bool allConnectionsFailed = false;
 
 	clientLeaderServers.reserve(coordinatorsSize);
 	for (const auto& h : cs.hostnames) {
@@ -868,9 +865,9 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 
 	deterministicRandom()->randomShuffle(clientLeaderServers);
 
-	loop {
-		state ClientLeaderRegInterface clientLeaderServer = clientLeaderServers[index];
-		state OpenDatabaseCoordRequest req;
+	while (true) {
+		ClientLeaderRegInterface clientLeaderServer = clientLeaderServers[index];
+		OpenDatabaseCoordRequest req;
 
 		req.clusterKey = cs.clusterKey();
 		req.hostnames = cs.hostnames;
@@ -880,9 +877,9 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 		req.traceLogGroup = traceLogGroup;
 		req.internal = internal;
 
-		state ClusterConnectionString storedConnectionString;
+		ClusterConnectionString storedConnectionString;
 		if (connRecord) {
-			bool upToDate = wait(connRecord->upToDate(storedConnectionString));
+			bool upToDate = co_await connRecord->upToDate(storedConnectionString);
 			if (upToDate) {
 				incorrectTime = Optional<double>();
 			} else if (allConnectionsFailed && storedConnectionString.getNumberOfCoordinators() > 0) {
@@ -895,9 +892,9 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 				    .detail("ClusterFile", connRecord->toString())
 				    .detail("StoredConnectionString", storedConnectionString.toString())
 				    .detail("CurrentConnectionString", connRecord->getConnectionString().toString());
-				wait(connRecord->setAndPersistConnectionString(storedConnectionString));
+				co_await connRecord->setAndPersistConnectionString(storedConnectionString);
 				info.intermediateConnRecord = connRecord;
-				return info;
+				co_return info;
 			} else {
 				req.issues.push_back_deep(req.issues.arena(), "incorrect_cluster_file_contents"_sr);
 				std::string connectionString = connRecord->getConnectionString().toString();
@@ -916,7 +913,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			incorrectTime = Optional<double>();
 		}
 
-		state Future<ErrorOr<CachedSerialization<ClientDBInfo>>> repFuture;
+		Future<ErrorOr<CachedSerialization<ClientDBInfo>>> repFuture;
 		if (clientLeaderServer.hostname.present()) {
 			repFuture = tryGetReplyFromHostname(req,
 			                                    clientLeaderServer.hostname.get(),
@@ -934,7 +931,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 		// object being created in FlowTransport. Having this peer is a prerequisite to us signaling the AsyncVar.
 		coordinator->setUnconditional(clientLeaderServer);
 
-		state ErrorOr<CachedSerialization<ClientDBInfo>> rep = wait(repFuture);
+		ErrorOr<CachedSerialization<ClientDBInfo>> rep = co_await repFuture;
 
 		if (rep.present()) {
 			if (rep.get().read().forward.present()) {
@@ -944,7 +941,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 				info.intermediateConnRecord = connRecord->makeIntermediateRecord(
 				    ClusterConnectionString(rep.get().read().forward.get().toString()));
 				ASSERT(info.intermediateConnRecord->getConnectionString().getNumberOfCoordinators() > 0);
-				return info;
+				co_return info;
 			}
 			if (connRecord != info.intermediateConnRecord) {
 				if (!info.hasConnected) {
@@ -954,7 +951,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 					    .detail("CurrentConnectionString",
 					            info.intermediateConnRecord->getConnectionString().toString());
 				}
-				wait(connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString()));
+				co_await connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString());
 				info.intermediateConnRecord = connRecord;
 			}
 
@@ -975,31 +972,30 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 			index = (index + 1) % coordinatorsSize;
 			if (index == successIndex) {
 				allConnectionsFailed = true;
-				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
+				co_await delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY);
 			}
 		}
 	}
 }
 
-ACTOR Future<Void> monitorProxies(
-    Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connRecord,
-    Reference<AsyncVar<ClientDBInfo>> clientInfo,
-    Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
-    Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
-    Key traceLogGroup,
-    IsInternal internal) {
-	state MonitorLeaderInfo info(connRecord->get());
-	loop {
+Future<Void> monitorProxies(Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connRecord,
+                            Reference<AsyncVar<ClientDBInfo>> clientInfo,
+                            Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
+                            Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
+                            Key traceLogGroup,
+                            IsInternal internal) {
+	MonitorLeaderInfo info(connRecord->get());
+	while (true) {
 		ASSERT(connRecord->get().isValid());
-		choose {
-			when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
-			         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup, internal))) {
-				info = _info;
-			}
-			when(wait(connRecord->onChange())) {
-				info.hasConnected = false;
-				info.intermediateConnRecord = connRecord->get();
-			}
+		auto res = co_await race(
+		    monitorProxiesOneGeneration(
+		        connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup, internal),
+		    connRecord->onChange());
+		if (res.index() == 0) {
+			info = std::get<0>(std::move(res));
+		} else if (res.index() == 1) {
+			info.hasConnected = false;
+			info.intermediateConnRecord = connRecord->get();
 		}
 	}
 }
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.cpp
similarity index 94%
rename from fdbclient/MultiVersionTransaction.actor.cpp
rename to fdbclient/MultiVersionTransaction.cpp
index 14c9fdd3bd1..e93bc507d4e 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.cpp
@@ -1,5 +1,5 @@
 /*
- * MultiVersionTransaction.actor.cpp
+ * MultiVersionTransaction.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -34,11 +34,11 @@
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/GenericManagementAPI.h"
 #include "fdbclient/MultiVersionTransaction.h"
 #include "fdbclient/MultiVersionAssignmentVars.h"
 #include "fdbclient/ClientVersion.h"
-#include "fdbclient/LocalClientAPI.h"
+#include "LocalClientAPI.h"
 #include "fdbclient/VersionVector.h"
 
 #include "flow/ThreadPrimitives.h"
@@ -46,14 +46,11 @@
 #include "flow/Platform.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/UnitTest.h"
-#include "flow/Trace.h"
 
 #ifdef __unixish__
 #include <fcntl.h>
 #endif // __unixish__
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 #ifdef FDBCLIENT_NATIVEAPI_ACTOR_H
 #error "MVC should not depend on the Native API"
 #endif
@@ -412,7 +409,7 @@ ThreadFuture<Void> DLDatabase::onReady() {
 Reference<ITransaction> DLDatabase::createTransaction() {
 	FdbCApi::FDBTransaction* tr;
 	throwIfError(api->databaseCreateTransaction(db, &tr));
-	return Reference<ITransaction>(new DLTransaction(api, tr));
+	return makeReference<DLTransaction>(api, tr);
 }
 
 void DLDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
@@ -808,7 +805,7 @@ Reference<IDatabase> DLApi::createDatabase(const char* clusterFilePath) {
 	if (headerVersion >= 610) {
 		FdbCApi::FDBDatabase* db;
 		throwIfError(api->createDatabase(clusterFilePath, &db));
-		return Reference<IDatabase>(new DLDatabase(api, db));
+		return makeReference<DLDatabase>(api, db);
 	} else {
 		return DLApi::createDatabase609(clusterFilePath);
 	}
@@ -821,7 +818,7 @@ Reference<IDatabase> DLApi::createDatabaseFromConnectionString(const char* conne
 
 	FdbCApi::FDBDatabase* db;
 	throwIfError(api->createDatabaseFromConnectionString(connectionString, &db));
-	return Reference<IDatabase>(new DLDatabase(api, db));
+	return makeReference<DLDatabase>(api, db);
 }
 
 void DLApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) {
@@ -856,16 +853,16 @@ void MultiVersionTransaction::updateTransaction(bool setPersistentOptions) {
 	// need to set it.
 	if (setPersistentOptions || !newTr.transaction) {
 		Optional<StringRef> timeout;
-		for (auto const& option : persistentOptions) {
-			if (option.first == FDBTransactionOptions::TIMEOUT) {
-				timeout = option.second.castTo<StringRef>();
+		for (auto const& [option, value] : persistentOptions) {
+			if (option == FDBTransactionOptions::TIMEOUT) {
+				timeout = value.castTo<StringRef>();
 			} else if (newTr.transaction) {
-				newTr.transaction->setOption(option.first, option.second.castTo<StringRef>());
+				newTr.transaction->setOption(option, value.castTo<StringRef>());
 			}
 		}
 		if (newTr.transaction) {
-			for (auto const& option : sensitivePersistentOptions) {
-				newTr.transaction->setOption(option.first, option.second.castTo<StringRef>());
+			for (auto const& [option, value] : sensitivePersistentOptions) {
+				newTr.transaction->setOption(option, value.castTo<StringRef>());
 			}
 		}
 
@@ -1163,14 +1160,13 @@ ThreadFuture<Void> MultiVersionTransaction::onError(Error const& e) {
 
 // Waits for the specified duration and signals the assignment variable with a timed out error
 // This will be canceled if a new timeout is set, in which case the tsav will not be signaled.
-ACTOR Future<Void> timeoutImpl(Reference<ThreadSingleAssignmentVar<Void>> tsav, double duration) {
-	state double endTime = now() + duration;
+Future<Void> timeoutImpl(Reference<ThreadSingleAssignmentVar<Void>> tsav, double duration) {
+	double endTime = now() + duration;
 	while (now() < endTime) {
-		wait(delayUntil(std::min(endTime + 0.0001, now() + CLIENT_KNOBS->TRANSACTION_TIMEOUT_DELAY_INTERVAL)));
+		co_await delayUntil(std::min(endTime + 0.0001, now() + CLIENT_KNOBS->TRANSACTION_TIMEOUT_DELAY_INTERVAL));
 	}
 
 	tsav->trySendError(transaction_timed_out());
-	return Void();
 }
 
 namespace {
@@ -1375,13 +1371,13 @@ MultiVersionDatabase::~MultiVersionDatabase() {
 // Create a MultiVersionDatabase that wraps an already created IDatabase object
 // For internal use in testing
 Reference<IDatabase> MultiVersionDatabase::debugCreateFromExistingDatabase(Reference<IDatabase> db) {
-	return Reference<IDatabase>(new MultiVersionDatabase(
-	    MultiVersionApi::api, 0, ClusterConnectionRecord::fromConnectionString(""), db, db, false));
+	return makeReference<MultiVersionDatabase>(
+	    MultiVersionApi::api, 0, ClusterConnectionRecord::fromConnectionString(""), db, db, false);
 }
 
 Reference<ITransaction> MultiVersionDatabase::createTransaction() {
-	return Reference<ITransaction>(
-	    new MultiVersionTransaction(Reference<MultiVersionDatabase>::addRef(this), dbState->transactionDefaultOptions));
+	return makeReference<MultiVersionTransaction>(Reference<MultiVersionDatabase>::addRef(this),
+	                                              dbState->transactionDefaultOptions);
 }
 
 void MultiVersionDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
@@ -1663,19 +1659,19 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
 	// Reapply database options on the new database
 	if (newDb) {
 		optionLock.enter();
-		for (auto option : options) {
+		for (const auto& [option, value] : options) {
 			try {
 				// In practice, this will set a deferred error instead of throwing. If that happens, the database
 				// will be unusable (attempts to use it will throw errors).
-				newDb->setOption(option.first, option.second.castTo<StringRef>());
+				newDb->setOption(option, value.castTo<StringRef>());
 			} catch (Error& e) {
 				optionLock.leave();
 
 				// If we can't set all of the options on a cluster, we abandon the client
 				TraceEvent(SevError, "ClusterVersionChangeOptionError")
 				    .error(e)
-				    .detail("Option", option.first)
-				    .detail("OptionValue", option.second)
+				    .detail("Option", option)
+				    .detail("OptionValue", value)
 				    .detail("LibPath", client->libPath);
 				client->failed = true;
 				MultiVersionApi::api->updateSupportedVersions();
@@ -1797,7 +1793,7 @@ Standalone<StringRef> MultiVersionDatabase::DatabaseState::getClientStatus(
 		statusObj["InitializationError"] = initializationError.code();
 	}
 	json_spirit::mArray clientArr;
-	for (auto [protocolVersion, client] : this->clients) {
+	for (const auto& [protocolVersion, client] : this->clients) {
 		json_spirit::mObject clientDesc;
 		clientDesc["ProtocolVersion"] = format("%llx", client->protocolVersion.version());
 		clientDesc["ReleaseVersion"] = client->releaseVersion;
@@ -1912,13 +1908,21 @@ void MultiVersionApi::useFutureProtocolVersion() {
 }
 
 namespace {
-void validateOption(Optional<StringRef> value, bool canBePresent, bool canBeAbsent, bool canBeEmpty = true) {
+
+FDB_BOOLEAN_PARAM(CanBePresent);
+FDB_BOOLEAN_PARAM(CanBeAbsent);
+FDB_BOOLEAN_PARAM(CanBeEmpty);
+
+void validateOption(Optional<StringRef> value,
+                    CanBePresent canBePresent,
+                    CanBeAbsent canBeAbsent,
+                    CanBeEmpty canBeEmpty = CanBeEmpty::True) {
 	ASSERT(canBePresent || canBeAbsent);
 
-	if (!canBePresent && value.present() && (!canBeEmpty || value.get().size() > 0)) {
+	if (!canBePresent && value.present() && (!canBeEmpty || !value.get().empty())) {
 		throw invalid_option_value();
 	}
-	if (!canBeAbsent && (!value.present() || (!canBeEmpty && value.get().size() == 0))) {
+	if (!canBeAbsent && (!value.present() || (!canBeEmpty && value.get().empty()))) {
 		throw invalid_option_value();
 	}
 }
@@ -1959,7 +1963,7 @@ void MultiVersionApi::addExternalLibrary(std::string path, bool useFutureVersion
 	// library.
 	threadCount = std::max(threadCount, 1);
 
-	if (externalClientDescriptions.count(filename) == 0) {
+	if (!externalClientDescriptions.contains(filename)) {
 		TraceEvent("AddingExternalClient").detail("LibraryPath", filename).detail("UseFutureVersion", useFutureVersion);
 		externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(path, true, useFutureVersion)));
 	}
@@ -1978,9 +1982,9 @@ void MultiVersionApi::addExternalLibraryDirectory(std::string path) {
 	// library.
 	threadCount = std::max(threadCount, 1);
 
-	for (auto filename : files) {
+	for (const auto& filename : files) {
 		std::string lib = abspath(joinPath(path, filename));
-		if (externalClientDescriptions.count(filename) == 0) {
+		if (!externalClientDescriptions.contains(filename)) {
 			TraceEvent("AddingExternalClient").detail("LibraryPath", filename);
 			externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(lib, true, false)));
 		}
@@ -2113,19 +2117,19 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 	}
 
 	if (option == FDBNetworkOptions::DISABLE_MULTI_VERSION_CLIENT_API) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		disableMultiVersionClientApi();
 	} else if (option == FDBNetworkOptions::CALLBACKS_ON_EXTERNAL_THREADS) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		setCallbacksOnExternalThreads();
 	} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT_LIBRARY) {
-		validateOption(value, true, false, false);
+		validateOption(value, CanBePresent::True, CanBeAbsent::False, CanBeEmpty::False);
 		addExternalLibrary(abspath(value.get().toString()), false);
 	} else if (option == FDBNetworkOptions::EXTERNAL_CLIENT_DIRECTORY) {
-		validateOption(value, true, false, false);
+		validateOption(value, CanBePresent::True, CanBeAbsent::False, CanBeEmpty::False);
 		addExternalLibraryDirectory(value.get().toString());
 	} else if (option == FDBNetworkOptions::DISABLE_LOCAL_CLIENT) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		disableLocalClient();
 	} else if (option == FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS) {
 		ASSERT(value.present());
@@ -2145,7 +2149,7 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 		disableBypass = true;
 	} else if (option == FDBNetworkOptions::CLIENT_THREADS_PER_VERSION) {
 		MutexHolder holder(lock);
-		validateOption(value, true, false, false);
+		validateOption(value, CanBePresent::True, CanBeAbsent::False, CanBeEmpty::False);
 		if (networkStartSetup) {
 			throw invalid_option();
 		}
@@ -2156,13 +2160,13 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 		threadCount = extractIntOption(value, 1, 1);
 #endif
 	} else if (option == FDBNetworkOptions::CLIENT_TMP_DIR) {
-		validateOption(value, true, false, false);
+		validateOption(value, CanBePresent::True, CanBeAbsent::False, CanBeEmpty::False);
 		tmpDir = abspath(value.get().toString());
 	} else if (option == FDBNetworkOptions::FUTURE_VERSION_CLIENT_LIBRARY) {
-		validateOption(value, true, false, false);
+		validateOption(value, CanBePresent::True, CanBeAbsent::False, CanBeEmpty::False);
 		addExternalLibrary(abspath(value.get().toString()), true);
 	} else if (option == FDBNetworkOptions::TRACE_FILE_IDENTIFIER) {
-		validateOption(value, true, false, true);
+		validateOption(value, CanBePresent::True, CanBeAbsent::False, CanBeEmpty::True);
 		traceFileIdentifier = value.get().toString();
 		{
 			MutexHolder holder(lock);
@@ -2171,16 +2175,16 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 			localClient->api->setNetworkOption(option, value);
 		}
 	} else if (option == FDBNetworkOptions::TRACE_SHARE_AMONG_CLIENT_THREADS) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		traceShareBaseNameAmongThreads = true;
 	} else if (option == FDBNetworkOptions::IGNORE_EXTERNAL_CLIENT_FAILURES) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		ignoreExternalClientFailures = true;
 	} else if (option == FDBNetworkOptions::FAIL_INCOMPATIBLE_CLIENT) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		failIncompatibleClient = true;
 	} else if (option == FDBNetworkOptions::RETAIN_CLIENT_LIBRARY_COPIES) {
-		validateOption(value, false, true);
+		validateOption(value, CanBePresent::False, CanBeAbsent::True);
 		retainClientLibCopies = true;
 	} else {
 		forwardOption = true;
@@ -2233,8 +2237,7 @@ void MultiVersionApi::setupNetwork() {
 			}
 
 			if (!bypassMultiClientApi) {
-				transportId =
-				    (uint64_t(uint32_t(platform::getRandomSeed())) << 32) ^ uint32_t(platform::getRandomSeed());
+				transportId = platform::getRandomSeed();
 				if (transportId <= 1)
 					transportId += 2;
 				localClient->api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID,
@@ -2246,22 +2249,22 @@ void MultiVersionApi::setupNetwork() {
 				ignoreExternalClientFailures = true;
 			}
 
-			for (auto i : externalClientDescriptions) {
+			for (const auto& i : externalClientDescriptions) {
 				std::string path = i.second.libPath;
 				std::string filename = basename(path);
 				bool useFutureVersion = i.second.useFutureVersion;
 
 				// Copy external lib for each thread
-				if (externalClients.count(filename) == 0) {
+				if (!externalClients.contains(filename)) {
 					externalClients[filename] = {};
 					auto libCopies = copyExternalLibraryPerThread(path);
 					for (int idx = 0; idx < libCopies.size(); ++idx) {
 						bool unlinkOnLoad = libCopies[idx].second && !retainClientLibCopies;
-						externalClients[filename].push_back(Reference<ClientInfo>(
-						    new ClientInfo(new DLApi(libCopies[idx].first, unlinkOnLoad /*unlink on load*/),
-						                   path,
-						                   useFutureVersion,
-						                   idx)));
+						externalClients[filename].push_back(
+						    makeReference<ClientInfo>(new DLApi(libCopies[idx].first, unlinkOnLoad /*unlink on load*/),
+						                              path,
+						                              useFutureVersion,
+						                              idx));
 					}
 				}
 			}
@@ -2293,8 +2296,8 @@ void MultiVersionApi::setupNetwork() {
 			MutexHolder holder(lock);
 			runOnExternalClientsAllThreads(
 			    [this, transportId, baseTraceFileId](Reference<ClientInfo> client) {
-				    for (auto option : options) {
-					    client->api->setNetworkOption(option.first, option.second.castTo<StringRef>());
+				    for (const auto& [option, value] : options) {
+					    client->api->setNetworkOption(option, value.castTo<StringRef>());
 				    }
 				    client->api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID,
 				                                  std::to_string(transportId));
@@ -2429,8 +2432,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(ClusterConnectionRecord con
 		lock.leave();
 
 		Reference<IDatabase> localDb = connectionRecord.createDatabase(localClient->api);
-		return Reference<IDatabase>(
-		    new MultiVersionDatabase(this, threadIdx, connectionRecord, Reference<IDatabase>(), localDb));
+		return makeReference<MultiVersionDatabase>(this, threadIdx, connectionRecord, Reference<IDatabase>(), localDb);
 	}
 
 	lock.leave();
@@ -2441,8 +2443,7 @@ Reference<IDatabase> MultiVersionApi::createDatabase(ClusterConnectionRecord con
 	if (bypassMultiClientApi) {
 		return localDb;
 	} else {
-		return Reference<IDatabase>(
-		    new MultiVersionDatabase(this, 0, connectionRecord, Reference<IDatabase>(), localDb));
+		return makeReference<MultiVersionDatabase>(this, 0, connectionRecord, Reference<IDatabase>(), localDb);
 	}
 }
 
@@ -2479,13 +2480,13 @@ void MultiVersionApi::updateSupportedVersions() {
 }
 
 // Must be called from the main thread
-ACTOR Future<std::string> updateClusterSharedStateMapImpl(MultiVersionApi* self,
-                                                          ClusterConnectionRecord connectionRecord,
-                                                          ProtocolVersion dbProtocolVersion,
-                                                          Reference<IDatabase> db) {
+Future<std::string> updateClusterSharedStateMapImpl(MultiVersionApi* self,
+                                                    ClusterConnectionRecord connectionRecord,
+                                                    ProtocolVersion dbProtocolVersion,
+                                                    Reference<IDatabase> db) {
 	// The cluster ID will be the connection record string (either a filename or the connection string itself)
 	// in versions before we could read the cluster ID.
-	state std::string clusterId = connectionRecord.toString();
+	std::string clusterId = connectionRecord.toString();
 
 	if (self->clusterSharedStateMap.find(clusterId) == self->clusterSharedStateMap.end()) {
 		TraceEvent("CreatingClusterSharedState")
@@ -2501,19 +2502,19 @@ ACTOR Future<std::string> updateClusterSharedStateMapImpl(MultiVersionApi* self,
 			    .detail("ClusterId", clusterId)
 			    .detail("ProtocolVersionExpected", dbProtocolVersion)
 			    .detail("ProtocolVersionFound", sharedStateInfo.protocolVersion);
-			return clusterId;
+			co_return clusterId;
 		}
 
 		TraceEvent("SettingClusterSharedState")
 		    .detail("ClusterId", clusterId)
 		    .detail("ProtocolVersion", dbProtocolVersion);
 
-		state ThreadFuture<DatabaseSharedState*> entry = sharedStateInfo.sharedStateFuture;
-		DatabaseSharedState* sharedState = wait(safeThreadFutureToFuture(entry));
+		ThreadFuture<DatabaseSharedState*> entry = sharedStateInfo.sharedStateFuture;
+		DatabaseSharedState* sharedState = co_await safeThreadFutureToFuture(entry);
 		db->setSharedState(sharedState);
 	}
 
-	return clusterId;
+	co_return clusterId;
 }
 
 // Must be called from the main thread
@@ -2593,13 +2594,13 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
 		return;
 	}
 
-	for (auto option : FDBNetworkOptions::optionInfo) {
+	for (const auto& option : FDBNetworkOptions::optionInfo) {
 		if (!option.second.hidden) {
 			std::string valueStr;
 			try {
 				if (platform::getEnvironmentVar(("FDB_NETWORK_OPTION_" + option.second.name).c_str(), valueStr)) {
 					FDBOptionInfo::ParamType curParamType = option.second.paramType;
-					for (auto value : parseOptionValues(valueStr)) {
+					for (const auto& value : parseOptionValues(valueStr)) {
 						Standalone<StringRef> currentValue;
 						int64_t intParamVal;
 						if (curParamType == FDBOptionInfo::ParamType::Int) {
@@ -2622,7 +2623,7 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
 						}
 						{ // lock scope
 							MutexHolder holder(lock);
-							if (setEnvOptions[option.first].count(currentValue) == 0) {
+							if (!setEnvOptions[option.first].contains(currentValue)) {
 								setNetworkOptionInternal(option.first, currentValue);
 								setEnvOptions[option.first].insert(currentValue);
 							}
@@ -2699,7 +2700,7 @@ TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing") {
 	ASSERT(vals.size() == 1 && vals[0] == "abcde");
 
 	vals = parseOptionValues("");
-	ASSERT(vals.size() == 1 && vals[0] == "");
+	ASSERT(vals.size() == 1 && vals[0].empty());
 
 	vals = parseOptionValues("a:b:c:d:e");
 	ASSERT(vals.size() == 5 && vals[0] == "a" && vals[1] == "b" && vals[2] == "c" && vals[3] == "d" && vals[4] == "e");
@@ -2708,13 +2709,13 @@ TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing") {
 	ASSERT(vals.size() == 3 && vals[0] == "\\a:" && vals[1] == ":b" && vals[2] == "\\");
 
 	vals = parseOptionValues("abcd:");
-	ASSERT(vals.size() == 2 && vals[0] == "abcd" && vals[1] == "");
+	ASSERT(vals.size() == 2 && vals[0] == "abcd" && vals[1].empty());
 
 	vals = parseOptionValues(":abcd");
-	ASSERT(vals.size() == 2 && vals[0] == "" && vals[1] == "abcd");
+	ASSERT(vals.size() == 2 && vals[0].empty() && vals[1] == "abcd");
 
 	vals = parseOptionValues(":");
-	ASSERT(vals.size() == 2 && vals[0] == "" && vals[1] == "");
+	ASSERT(vals.size() == 2 && vals[0].empty() && vals[1].empty());
 
 	try {
 		vals = parseOptionValues("\\x");
@@ -2739,7 +2740,7 @@ class ValidateFuture final : public ThreadCallback {
 	}
 
 	void error(const Error& e, int& userParam) override {
-		ASSERT(legalErrors.count(e.code()) > 0 ||
+		ASSERT(legalErrors.contains(e.code()) ||
 		       (f.isError() && expectedValue.isError() && f.getError().code() == expectedValue.getError().code()));
 		delete this;
 	}
@@ -2852,33 +2853,25 @@ THREAD_FUNC cancel(void* arg) {
 	THREAD_RETURN;
 }
 
-ACTOR Future<Void> checkUndestroyedFutures(std::vector<ThreadSingleAssignmentVar<int>*> undestroyed) {
-	state int fNum;
-	state ThreadSingleAssignmentVar<int>* f;
-	state double start = now();
+Future<Void> checkUndestroyedFutures(std::vector<ThreadSingleAssignmentVar<int>*> undestroyed) {
+	double start = now();
 
-	for (fNum = 0; fNum < undestroyed.size(); ++fNum) {
-		f = undestroyed[fNum];
-
-		while (!f->isReady() && start + 5 >= now()) {
-			wait(delay(1.0));
+	for (auto* undestroyedFuture : undestroyed) {
+		while (!undestroyedFuture->isReady() && start + 5 >= now()) {
+			co_await delay(1.0);
 		}
 
-		ASSERT(f->isReady());
+		ASSERT(undestroyedFuture->isReady());
 	}
 
-	wait(delay(1.0));
-
-	for (fNum = 0; fNum < undestroyed.size(); ++fNum) {
-		f = undestroyed[fNum];
+	co_await delay(1.0);
 
-		ASSERT_EQ(f->debugGetReferenceCount(), 1);
-		ASSERT(f->isReady());
+	for (auto* undestroyedFuture : undestroyed) {
+		ASSERT_EQ(undestroyedFuture->debugGetReferenceCount(), 1);
+		ASSERT(undestroyedFuture->isReady());
 
-		f->cancel();
+		undestroyedFuture->cancel();
 	}
-
-	return Void();
 }
 
 // Common code for tests of single assignment vars. Tests both correctness and thread safety.
@@ -2959,7 +2952,7 @@ THREAD_FUNC runSingleAssignmentVarTest(void* arg) {
 
 struct AbortableTest {
 	static FutureInfo createThreadFuture(FutureInfo f) {
-		ThreadSingleAssignmentVar<Void>* abort = new ThreadSingleAssignmentVar<Void>();
+		auto* abort = new ThreadSingleAssignmentVar<Void>();
 		abort->addref(); // this leaks if abort is never set
 
 		auto newFuture =
@@ -2976,16 +2969,14 @@ struct AbortableTest {
 };
 
 TEST_CASE("fdbclient/multiversionclient/AbortableSingleAssignmentVar") {
-	state volatile bool done = false;
-	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<AbortableTest>, (void*)&done);
+	volatile bool done = false;
+	THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<AbortableTest>, (void*)&done);
 
 	while (!done) {
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 
 	waitThread(thread);
-
-	return Void();
 }
 
 class CAPICallback final : public ThreadCallback {
@@ -3030,7 +3021,7 @@ struct DLTest {
 			// Functions needed for DLSingleAssignmentVar
 			api->futureSetCallback = [](FdbCApi::FDBFuture* f, FdbCApi::FDBCallback callback, void* callbackParameter) {
 				try {
-					CAPICallback* cb = new CAPICallback(callback, f, callbackParameter);
+					auto* cb = new CAPICallback(callback, f, callbackParameter);
 					int ignore;
 					((ThreadSingleAssignmentVarBase*)f)->callOrSetAsCallback(cb, ignore, 0);
 					return FdbCApi::fdb_error_t(error_code_success);
@@ -3053,13 +3044,13 @@ struct DLTest {
 };
 
 TEST_CASE("fdbclient/multiversionclient/DLSingleAssignmentVar") {
-	state volatile bool done = false;
+	volatile bool done = false;
 
 	MultiVersionApi::api->callbackOnMainThread = true;
-	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<DLTest>, (void*)&done);
+	THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<DLTest>, (void*)&done);
 
 	while (!done) {
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 
 	waitThread(thread);
@@ -3069,12 +3060,10 @@ TEST_CASE("fdbclient/multiversionclient/DLSingleAssignmentVar") {
 	thread = g_network->startThread(runSingleAssignmentVarTest<DLTest>, (void*)&done);
 
 	while (!done) {
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 
 	waitThread(thread);
-
-	return Void();
 }
 
 struct MapTest {
@@ -3083,7 +3072,7 @@ struct MapTest {
 		newFuture.legalErrors = f.legalErrors;
 		newFuture.future = mapThreadFuture<int, int>(f.future, [f, newFuture](ErrorOr<int> v) {
 			if (v.isError()) {
-				ASSERT(f.legalErrors.count(v.getError().code()) > 0 ||
+				ASSERT(f.legalErrors.contains(v.getError().code()) ||
 				       (f.expectedValue.isError() && f.expectedValue.getError().code() == v.getError().code()));
 			} else {
 				ASSERT(!f.expectedValue.isError() && f.expectedValue.get() == v.get());
@@ -3097,16 +3086,14 @@ struct MapTest {
 };
 
 TEST_CASE("fdbclient/multiversionclient/MapSingleAssignmentVar") {
-	state volatile bool done = false;
-	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<MapTest>, (void*)&done);
+	volatile bool done = false;
+	THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<MapTest>, (void*)&done);
 
 	while (!done) {
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 
 	waitThread(thread);
-
-	return Void();
 }
 
 struct FlatMapTest {
@@ -3118,7 +3105,7 @@ struct FlatMapTest {
 		        f.future,
 		        [f, mapFuture](ErrorOr<int> v) {
 			        if (v.isError()) {
-				        ASSERT(f.legalErrors.count(v.getError().code()) > 0 ||
+				        ASSERT(f.legalErrors.contains(v.getError().code()) ||
 				               (f.expectedValue.isError() && f.expectedValue.getError().code() == v.getError().code()));
 			        } else {
 				        ASSERT(!f.expectedValue.isError() && f.expectedValue.get() == v.get());
@@ -3136,14 +3123,12 @@ struct FlatMapTest {
 };
 
 TEST_CASE("fdbclient/multiversionclient/FlatMapSingleAssignmentVar") {
-	state volatile bool done = false;
-	state THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<FlatMapTest>, (void*)&done);
+	volatile bool done = false;
+	THREAD_HANDLE thread = g_network->startThread(runSingleAssignmentVarTest<FlatMapTest>, (void*)&done);
 
 	while (!done) {
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 
 	waitThread(thread);
-
-	return Void();
 }
diff --git a/fdbclient/MutationLogReader.actor.cpp b/fdbclient/MutationLogReader.cpp
similarity index 73%
rename from fdbclient/MutationLogReader.actor.cpp
rename to fdbclient/MutationLogReader.cpp
index ce9b077f269..73f260aa46c 100644
--- a/fdbclient/MutationLogReader.actor.cpp
+++ b/fdbclient/MutationLogReader.cpp
@@ -1,5 +1,5 @@
 /*
- * MutationLogReader.actor.cpp
+ * MutationLogReader.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbclient/MutationLogReader.actor.h"
+#include "fdbclient/MutationLogReader.h"
 #include "fdbrpc/simulator.h"
 #include "flow/UnitTest.h"
 #include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 
@@ -63,28 +62,29 @@ Future<Void> PipelinedReader::getNext(Database cx) {
 	return getNext_impl(this, cx);
 }
 
-ACTOR Future<Void> PipelinedReader::getNext_impl(PipelinedReader* self, Database cx) {
-	state Transaction tr(cx);
+Future<Void> PipelinedReader::getNext_impl(PipelinedReader* self, Database cx) {
+	Transaction tr(cx);
 
-	state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
-	                            (g_network->isSimulated() && !g_simulator->speedUpSimulation)
-	                                ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
-	                                : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
+	GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
+	                      (g_network->isSimulated() && !g_simulator->speedUpSimulation)
+	                          ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
+	                          : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
 
-	state Key begin = versionToKey(self->currentBeginVersion, self->prefix);
-	state Key end = versionToKey(self->endVersion, self->prefix);
+	Key begin = versionToKey(self->currentBeginVersion, self->prefix);
+	Key end = versionToKey(self->endVersion, self->prefix);
 
-	loop {
+	while (true) {
 		// Get the lock
-		wait(self->readerLimit.take());
+		co_await self->readerLimit.take();
 
 		// Read begin to end forever until successful
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				RangeResult kvs = wait(tr.getRange(KeyRangeRef(begin, end), limits));
+				RangeResult kvs = co_await tr.getRange(KeyRangeRef(begin, end), limits);
 
 				// No more results, send end of stream
 				if (!kvs.empty()) {
@@ -100,20 +100,21 @@ ACTOR Future<Void> PipelinedReader::getNext_impl(PipelinedReader* self, Database
 
 				if (!kvs.more) {
 					self->reads.sendError(end_of_stream());
-					return Void();
+					co_return;
 				}
 
 				begin = kvs.getReadThrough();
 
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_transaction_too_old) {
-					// We are using this transaction until it's too old and then resetting to a fresh one,
-					// so we don't need to delay.
-					tr.fullReset();
-				} else {
-					wait(tr.onError(e));
-				}
+				err = e;
+			}
+			if (err.code() == error_code_transaction_too_old) {
+				// We are using this transaction until it's too old and then resetting to a fresh one,
+				// so we don't need to delay.
+				tr.fullReset();
+			} else {
+				co_await tr.onError(err);
 			}
 		}
 	}
@@ -121,11 +122,11 @@ ACTOR Future<Void> PipelinedReader::getNext_impl(PipelinedReader* self, Database
 
 } // namespace mutation_log_reader
 
-ACTOR Future<Void> MutationLogReader::initializePQ(MutationLogReader* self) {
-	state int h;
+Future<Void> MutationLogReader::initializePQ(MutationLogReader* self) {
+	int h{ 0 };
 	for (h = 0; h < 256; ++h) {
 		try {
-			mutation_log_reader::RangeResultBlock front = waitNext(self->pipelinedReaders[h]->reads.getFuture());
+			mutation_log_reader::RangeResultBlock front = co_await self->pipelinedReaders[h]->reads.getFuture();
 			self->priorityQueue.push(front);
 		} catch (Error& e) {
 			if (e.code() != error_code_end_of_stream) {
@@ -134,31 +135,30 @@ ACTOR Future<Void> MutationLogReader::initializePQ(MutationLogReader* self) {
 			++self->finished;
 		}
 	}
-	return Void();
 }
 
 Future<Standalone<RangeResultRef>> MutationLogReader::getNext() {
 	return getNext_impl(this);
 }
 
-ACTOR Future<Standalone<RangeResultRef>> MutationLogReader::getNext_impl(MutationLogReader* self) {
-	loop {
+Future<Standalone<RangeResultRef>> MutationLogReader::getNext_impl(MutationLogReader* self) {
+	while (true) {
 		if (self->finished == 256) {
-			state int i;
+			int i{ 0 };
 			for (i = 0; i < self->pipelinedReaders.size(); ++i) {
-				wait(self->pipelinedReaders[i]->done());
+				co_await self->pipelinedReaders[i]->done();
 			}
 			throw end_of_stream();
 		}
 		mutation_log_reader::RangeResultBlock top = self->priorityQueue.top();
 		self->priorityQueue.pop();
 		uint8_t hash = top.hash;
-		state Standalone<RangeResultRef> ret = top.consume();
+		Standalone<RangeResultRef> ret = top.consume();
 		if (top.empty()) {
 			self->pipelinedReaders[(int)hash]->release();
 			try {
 				mutation_log_reader::RangeResultBlock next =
-				    waitNext(self->pipelinedReaders[(int)hash]->reads.getFuture());
+				    co_await self->pipelinedReaders[(int)hash]->reads.getFuture();
 				self->priorityQueue.push(next);
 			} catch (Error& e) {
 				if (e.code() == error_code_end_of_stream) {
@@ -170,8 +170,8 @@ ACTOR Future<Standalone<RangeResultRef>> MutationLogReader::getNext_impl(Mutatio
 		} else {
 			self->priorityQueue.push(top);
 		}
-		if (ret.size() != 0) {
-			return ret;
+		if (!ret.empty()) {
+			co_return ret;
 		}
 	}
 }
diff --git a/fdbclient/NameLineage.cpp b/fdbclient/NameLineage.cpp
index 466b5ecb95b..ae68d7360ca 100644
--- a/fdbclient/NameLineage.cpp
+++ b/fdbclient/NameLineage.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbclient/NameLineage.h"
+#include "NameLineage.h"
 
 namespace {
 NameLineageCollector nameLineageCollector;
diff --git a/fdbclient/include/fdbclient/NameLineage.h b/fdbclient/NameLineage.h
similarity index 100%
rename from fdbclient/include/fdbclient/NameLineage.h
rename to fdbclient/NameLineage.h
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 7a0b24a56b6..58b23c4d625 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -25,6 +25,7 @@
 #include <iterator>
 #include <limits>
 #include <memory>
+#include <random>
 #include <regex>
 #include <string>
 #include <unordered_set>
@@ -52,19 +53,17 @@
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/IKnobCollection.h"
+#include "fdbclient/GlobalConfig.h"
 #include "fdbclient/JsonBuilder.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/NameLineage.h"
+#include "fdbclient/ManagementAPI.h"
+#include "NameLineage.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/MonitorLeader.h"
 #include "fdbclient/MutationList.h"
-#include "fdbclient/ParallelStream.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/SpecialKeySpace.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TransactionLineage.h"
@@ -76,6 +75,7 @@
 #include "fdbrpc/sim_validation.h"
 #include "flow/Arena.h"
 #include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
@@ -88,12 +88,14 @@
 #include "flow/Knobs.h"
 #include "flow/Platform.h"
 #include "flow/SystemMonitor.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbclient/Tracing.h"
 #include "flow/UnitTest.h"
 #include "flow/network.h"
 #include "flow/serialize.h"
 
+#include "ProxyLoadBalance.h"
+
 #ifdef ADDRESS_SANITIZER
 #include <sanitizer/lsan_interface.h>
 #endif
@@ -139,16 +141,12 @@ void delref(DatabaseContext* ptr) {
 	ptr->delref();
 }
 
-ACTOR Future<Void> refreshTransaction(DatabaseContext* self, Transaction* tr) {
+Future<Void> refreshTransaction(DatabaseContext* self, Transaction* tr) {
 	*tr = Transaction();
-	wait(delay(0)); // Give ourselves the chance to get cancelled if self was destroyed
+	co_await delay(0); // Give ourselves the chance to get cancelled if self was destroyed
 	*tr = Transaction(Database(Reference<DatabaseContext>::addRef(self)));
-	return Void();
 }
 
-// FIXME: get a prototype in a header file, ugh.
-ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
-
 Optional<KeyRangeLocationInfo> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) {
 	Arena arena;
 
@@ -384,8 +382,7 @@ Future<Void> DatabaseContext::onConnected() const {
 	return connected;
 }
 
-ACTOR static Future<Void> switchConnectionRecordImpl(Reference<IClusterConnectionRecord> connRecord,
-                                                     DatabaseContext* self) {
+static Future<Void> switchConnectionRecordImpl(Reference<IClusterConnectionRecord> connRecord, DatabaseContext* self) {
 	CODE_PROBE(true, "Switch connection file");
 	TraceEvent("SwitchConnectionRecord")
 	    .detail("ClusterFile", connRecord->toString())
@@ -406,23 +403,25 @@ ACTOR static Future<Void> switchConnectionRecordImpl(Reference<IClusterConnectio
 	self->clientInfo->set(clearedClientInfo);
 	self->connectionRecord->set(connRecord);
 
-	state Database db(Reference<DatabaseContext>::addRef(self));
-	state Transaction tr(db);
-	loop {
+	Database db(Reference<DatabaseContext>::addRef(self));
+	Transaction tr(db);
+	while (true) {
 		tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+		Error err;
 		try {
 			TraceEvent("SwitchConnectionRecordAttemptingGRV").log();
-			Version v = wait(tr.getReadVersion());
+			Version v = co_await tr.getReadVersion();
 			TraceEvent("SwitchConnectionRecordGotRV")
 			    .detail("ReadVersion", v)
 			    .detail("MinAcceptableReadVersion", self->minAcceptableReadVersion);
 			ASSERT(self->minAcceptableReadVersion != std::numeric_limits<Version>::max());
 			self->connectionFileChangedTrigger.trigger();
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			TraceEvent("SwitchConnectionRecordError").detail("Error", e.what());
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("SwitchConnectionRecordError").detail("Error", err.what());
+		co_await tr.onError(err);
 	}
 }
 
@@ -733,9 +732,9 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 		std::string knobValueString = optionValue.substr(eq + 1);
 
 		try {
-			auto knobValue = IKnobCollection::parseKnobValue(knobName, knobValueString, IKnobCollection::Type::CLIENT);
+			auto knobValue = parseClientKnobValue(knobName, knobValueString);
 			if (g_network) {
-				IKnobCollection::getMutableGlobalKnobCollection().setKnob(knobName, knobValue);
+				setClientKnob(knobName, knobValue);
 			} else {
 				networkOptions.knobs[knobName] = knobValue;
 			}
@@ -860,10 +859,10 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 }
 
 // update the network busyness on a 1s cadence
-ACTOR Future<Void> monitorNetworkBusyness() {
-	state double prevTime = now();
-	loop {
-		wait(delay(CLIENT_KNOBS->NETWORK_BUSYNESS_MONITOR_INTERVAL, TaskPriority::FlushTrace));
+Future<Void> monitorNetworkBusyness() {
+	double prevTime = now();
+	while (true) {
+		co_await delay(CLIENT_KNOBS->NETWORK_BUSYNESS_MONITOR_INTERVAL, TaskPriority::FlushTrace);
 		double elapsed = now() - prevTime; // get elapsed time from last execution
 		prevTime = now();
 		struct NetworkMetrics::PriorityStats& tracker = g_network->networkInfo.metrics.starvationTrackerNetworkBusyness;
@@ -900,9 +899,9 @@ ACTOR Future<Void> monitorNetworkBusyness() {
 }
 
 static void setupGlobalKnobs() {
-	IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::False, IsSimulated::False);
+	resetClientKnobs(Randomize::False, IsSimulated::False);
 	for (const auto& [knobName, knobValue] : networkOptions.knobs) {
-		IKnobCollection::getMutableGlobalKnobCollection().setKnob(knobName, knobValue);
+		setClientKnob(knobName, knobValue);
 	}
 }
 
@@ -1007,13 +1006,13 @@ bool DatabaseContext::isCurrentGrvProxy(UID proxyId) const {
 
 // Actor which will wait until the MultiInterface<CommitProxyInterface> returned by the DatabaseContext cx is not
 // nullptr
-ACTOR Future<Reference<CommitProxyInfo>> getCommitProxiesFuture(DatabaseContext* cx,
-                                                                UseProvisionalProxies useProvisionalProxies) {
-	loop {
+Future<Reference<CommitProxyInfo>> getCommitProxiesFuture(DatabaseContext* cx,
+                                                          UseProvisionalProxies useProvisionalProxies) {
+	while (true) {
 		Reference<CommitProxyInfo> commitProxies = cx->getCommitProxies(useProvisionalProxies);
 		if (commitProxies)
-			return commitProxies;
-		wait(cx->onProxiesChanged());
+			co_return commitProxies;
+		co_await cx->onProxiesChanged();
 	}
 }
 
@@ -1124,36 +1123,36 @@ Future<RangeResult> getRange(Reference<TransactionState> const& trState,
                              GetRangeLimits const& limits,
                              Reverse const& reverse);
 
-ACTOR Future<Optional<StorageServerInterface>> fetchServerInterface(Reference<TransactionState> trState, UID id) {
-	Optional<Value> val = wait(getValue(trState, serverListKeyFor(id), TransactionRecordLogInfo::False));
+Future<Optional<StorageServerInterface>> fetchServerInterface(Reference<TransactionState> trState, UID id) {
+	Optional<Value> val = co_await getValue(trState, serverListKeyFor(id), TransactionRecordLogInfo::False);
 
 	if (!val.present()) {
 		// A storage server has been removed from serverList since we read keyServers
-		return Optional<StorageServerInterface>();
+		co_return Optional<StorageServerInterface>();
 	}
 
-	return decodeServerListValue(val.get());
+	co_return decodeServerListValue(val.get());
 }
 
-ACTOR Future<Optional<std::vector<StorageServerInterface>>> transactionalGetServerInterfaces(
+Future<Optional<std::vector<StorageServerInterface>>> transactionalGetServerInterfaces(
     Reference<TransactionState> trState,
     std::vector<UID> ids) {
-	state std::vector<Future<Optional<StorageServerInterface>>> serverListEntries;
+	std::vector<Future<Optional<StorageServerInterface>>> serverListEntries;
 	serverListEntries.reserve(ids.size());
-	for (int s = 0; s < ids.size(); s++) {
-		serverListEntries.push_back(fetchServerInterface(trState, ids[s]));
+	for (const auto& id : ids) {
+		serverListEntries.push_back(fetchServerInterface(trState, id));
 	}
 
-	std::vector<Optional<StorageServerInterface>> serverListValues = wait(getAll(serverListEntries));
+	std::vector<Optional<StorageServerInterface>> serverListValues = co_await getAll(serverListEntries);
 	std::vector<StorageServerInterface> serverInterfaces;
-	for (int s = 0; s < serverListValues.size(); s++) {
-		if (!serverListValues[s].present()) {
+	for (const auto& serverListValue : serverListValues) {
+		if (!serverListValue.present()) {
 			// A storage server has been removed from ServerList since we read keyServers
-			return Optional<std::vector<StorageServerInterface>>();
+			co_return Optional<std::vector<StorageServerInterface>>();
 		}
-		serverInterfaces.push_back(serverListValues[s].get());
+		serverInterfaces.push_back(serverListValue.get());
 	}
-	return serverInterfaces;
+	co_return serverInterfaces;
 }
 
 void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) {
@@ -1166,11 +1165,11 @@ void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) {
 		}
 	}
 
-	for (const auto& mapping : reply.resultsTssMapping) {
-		auto ssi = ssiById.find(mapping.first);
+	for (const auto& [storageServerId, tss] : reply.resultsTssMapping) {
+		auto ssi = ssiById.find(storageServerId);
 		ASSERT(ssi != ssiById.end());
-		cx->addTssMapping(*ssi->second, mapping.second);
-		ssiById.erase(mapping.first);
+		cx->addTssMapping(*ssi->second, tss);
+		ssiById.erase(storageServerId);
 	}
 
 	// if SS didn't have a mapping above, it's still in the ssiById map, so remove its tss mapping
@@ -1180,22 +1179,22 @@ void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) {
 }
 
 void updateTagMappings(Database cx, const GetKeyServerLocationsReply& reply) {
-	for (const auto& mapping : reply.resultsTagMapping) {
-		cx->addSSIdTagMapping(mapping.first, mapping.second);
+	for (const auto& [storageServerId, tag] : reply.resultsTagMapping) {
+		cx->addSSIdTagMapping(storageServerId, tag);
 	}
 }
 
 // If isBackward == true, returns the shard containing the key before 'key' (an infinitely long, inexpressible key).
 // Otherwise returns the shard containing key
-ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
-                                                           Key key,
-                                                           SpanContext spanContext,
-                                                           Optional<UID> debugID,
-                                                           UseProvisionalProxies useProvisionalProxies,
-                                                           Reverse isBackward,
-                                                           Version version) {
-
-	state Span span("NAPI:getKeyLocation"_loc, spanContext);
+Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
+                                                     Key key,
+                                                     SpanContext spanContext,
+                                                     Optional<UID> debugID,
+                                                     UseProvisionalProxies useProvisionalProxies,
+                                                     Reverse isBackward,
+                                                     Version version) {
+
+	Span span("NAPI:getKeyLocation"_loc, spanContext);
 	if (isBackward) {
 		ASSERT(key != allKeys.begin && key <= allKeys.end);
 	} else {
@@ -1205,32 +1204,28 @@ ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
 	if (debugID.present())
 		g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.Before");
 
-	loop {
+	while (true) {
 		try {
-			wait(cx->getBackoff());
+			co_await cx->getBackoff();
 			++cx->transactionKeyServerLocationRequests;
-			choose {
-				when(wait(cx->onProxiesChanged())) {}
-				when(GetKeyServerLocationsReply rep = wait(basicLoadBalance(
-				         cx->getCommitProxies(useProvisionalProxies),
-				         &CommitProxyInterface::getKeyServersLocations,
-				         GetKeyServerLocationsRequest(
-				             span.context, key, Optional<KeyRef>(), /*limit=*/100, isBackward, version, key.arena()),
-				         TaskPriority::DefaultPromiseEndpoint))) {
-					++cx->transactionKeyServerLocationRequestsCompleted;
-					if (debugID.present())
-						g_traceBatch.addEvent(
-						    "TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.After");
-					ASSERT(rep.results.size() == 1);
-
-					auto locationInfo = cx->setCachedLocation(rep.results[0].first, rep.results[0].second);
-					updateTssMappings(cx, rep);
-					updateTagMappings(cx, rep);
-
-					cx->updateBackoff(success());
-					return KeyRangeLocationInfo(KeyRange(rep.results[0].first, rep.arena), locationInfo);
-				}
-			}
+			GetKeyServerLocationsReply rep = co_await commitProxyLoadBalance(
+			    cx,
+			    makeReqBuilder<GetKeyServerLocationsRequest>(
+			        span.context, key, Optional<KeyRef>(), /*limit=*/100, isBackward, version, key.arena()),
+			    &CommitProxyInterface::getKeyServersLocations,
+			    useProvisionalProxies,
+			    TaskPriority::DefaultPromiseEndpoint);
+			++cx->transactionKeyServerLocationRequestsCompleted;
+			if (debugID.present())
+				g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.After");
+			ASSERT(rep.results.size() == 1);
+
+			auto locationInfo = cx->setCachedLocation(rep.results[0].first, rep.results[0].second);
+			updateTssMappings(cx, rep);
+			updateTagMappings(cx, rep);
+
+			cx->updateBackoff(success());
+			co_return KeyRangeLocationInfo(KeyRange(rep.results[0].first, rep.arena), locationInfo);
 		} catch (Error& e) {
 			if (e.code() == error_code_commit_proxy_memory_limit_exceeded) {
 				// Eats commit_proxy_memory_limit_exceeded error from commit proxies
@@ -1346,55 +1341,47 @@ void DatabaseContext::updateBackoff(const Error& err) {
 	}
 }
 
-ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
-    Database cx,
-    KeyRange keys,
-    int limit,
-    Reverse reverse,
-    SpanContext spanContext,
-    Optional<UID> debugID,
-    UseProvisionalProxies useProvisionalProxies,
-    Version version) {
-	state Span span("NAPI:getKeyRangeLocations"_loc, spanContext);
+Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(Database cx,
+                                                                        KeyRange keys,
+                                                                        int limit,
+                                                                        Reverse reverse,
+                                                                        SpanContext spanContext,
+                                                                        Optional<UID> debugID,
+                                                                        UseProvisionalProxies useProvisionalProxies,
+                                                                        Version version) {
+	Span span("NAPI:getKeyRangeLocations"_loc, spanContext);
 	if (debugID.present())
 		g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.Before");
 
-	loop {
+	while (true) {
 		try {
-			wait(cx->getBackoff());
+			co_await cx->getBackoff();
 			++cx->transactionKeyServerLocationRequests;
-			choose {
-				when(wait(cx->onProxiesChanged())) {}
-				when(GetKeyServerLocationsReply _rep = wait(basicLoadBalance(
-				         cx->getCommitProxies(useProvisionalProxies),
-				         &CommitProxyInterface::getKeyServersLocations,
-				         GetKeyServerLocationsRequest(
-				             span.context, keys.begin, keys.end, limit, reverse, version, keys.arena()),
-				         TaskPriority::DefaultPromiseEndpoint))) {
-					++cx->transactionKeyServerLocationRequestsCompleted;
-					state GetKeyServerLocationsReply rep = _rep;
-					if (debugID.present())
-						g_traceBatch.addEvent(
-						    "TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.After");
-					ASSERT(rep.results.size());
-
-					state std::vector<KeyRangeLocationInfo> results;
-					state int shard = 0;
-					for (; shard < rep.results.size(); shard++) {
-						// FIXME: these shards are being inserted into the map sequentially, it would be much more CPU
-						// efficient to save the map pairs and insert them all at once.
-						results.emplace_back(
-						    (rep.results[shard].first & keys),
-						    cx->setCachedLocation(rep.results[shard].first, rep.results[shard].second));
-						wait(yield());
-					}
-					updateTssMappings(cx, rep);
-					updateTagMappings(cx, rep);
-
-					cx->updateBackoff(success());
-					return results;
-				}
+			GetKeyServerLocationsReply rep = co_await commitProxyLoadBalance(
+			    cx,
+			    makeReqBuilder<GetKeyServerLocationsRequest>(
+			        span.context, keys.begin, keys.end, limit, reverse, version, keys.arena()),
+			    &CommitProxyInterface::getKeyServersLocations,
+			    useProvisionalProxies,
+			    TaskPriority::DefaultPromiseEndpoint);
+			++cx->transactionKeyServerLocationRequestsCompleted;
+			if (debugID.present())
+				g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.After");
+			ASSERT(rep.results.size());
+
+			std::vector<KeyRangeLocationInfo> results;
+			for (int shard = 0; shard < rep.results.size(); ++shard) {
+				// FIXME: these shards are being inserted into the map sequentially, it would be much more CPU
+				// efficient to save the map pairs and insert them all at once.
+				results.emplace_back((rep.results[shard].first & keys),
+				                     cx->setCachedLocation(rep.results[shard].first, rep.results[shard].second));
+				co_await yield();
 			}
+			updateTssMappings(cx, rep);
+			updateTagMappings(cx, rep);
+
+			cx->updateBackoff(success());
+			co_return results;
 		} catch (Error& e) {
 			if (e.code() == error_code_commit_proxy_memory_limit_exceeded) {
 				// Eats commit_proxy_memory_limit_exceeded error from commit proxies
@@ -1476,14 +1463,14 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<Transac
 	                                : latestVersion);
 }
 
-ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange keys) {
-	state int totalRanges = 0;
-	state int totalRequests = 0;
+Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange keys) {
+	int totalRanges = 0;
+	int totalRequests = 0;
 
-	wait(trState->startTransaction());
+	co_await trState->startTransaction();
 
-	loop {
-		std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations_internal(
+	while (true) {
+		std::vector<KeyRangeLocationInfo> locations = co_await getKeyRangeLocations_internal(
 		    trState->cx,
 		    keys,
 		    CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT,
@@ -1491,7 +1478,7 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
 		    trState->spanContext,
 		    trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
 		    trState->useProvisionalProxies,
-		    trState->readVersion()));
+		    trState->readVersion());
 		totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT;
 		totalRequests++;
 		if (locations.size() == 0 || totalRanges >= trState->cx->locationCacheSize ||
@@ -1502,21 +1489,21 @@ ACTOR Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange
 
 		if (totalRequests % 20 == 0) {
 			// To avoid blocking the proxies from starting other transactions, occasionally get a read version.
-			state Transaction tr(trState->cx);
-			loop {
+			Transaction tr(trState->cx);
+			while (true) {
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 					tr.setOption(FDBTransactionOptions::CAUSAL_READ_RISKY);
-					wait(success(tr.getReadVersion()));
+					co_await tr.getReadVersion();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 	}
-
-	return Void();
 }
 
 SpanContext generateSpanID(bool transactionTracingSample, SpanContext parentContext = SpanContext()) {
@@ -1545,8 +1532,7 @@ Reference<TransactionState> TransactionState::cloneAndReset(Reference<Transactio
                                                             bool generateNewSpan) const {
 
 	SpanContext newSpanContext = generateNewSpan ? generateSpanID(cx->transactionTracingSample) : spanContext;
-	Reference<TransactionState> newState =
-	    makeReference<TransactionState>(cx, cx->taskID, newSpanContext, newTrLogInfo);
+	auto newState = makeReference<TransactionState>(cx, cx->taskID, newSpanContext, newTrLogInfo);
 
 	if (!cx->apiVersionAtLeast(16)) {
 		newState->options = options;
@@ -2004,19 +1990,19 @@ ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> p
 	}
 }
 
-ACTOR Future<Void> watchStorageServerResp(Key key, Database cx) {
-	loop {
+Future<Void> watchStorageServerResp(Key key, Database cx) {
+	while (true) {
 		try {
-			state Reference<WatchMetadata> metadata = cx->getWatchMetadata(key);
+			Reference<WatchMetadata> metadata = cx->getWatchMetadata(key);
 			if (!metadata.isValid()) {
-				return Void();
+				co_return;
 			}
 
-			Version watchVersion = wait(watchValue(cx, metadata->parameters));
+			Version watchVersion = co_await watchValue(cx, metadata->parameters);
 
 			metadata = cx->getWatchMetadata(key);
 			if (!metadata.isValid())
-				return Void();
+				co_return;
 
 			// case 1: version_1 (SS) >= version_2 (map)
 			if (watchVersion >= metadata->parameters->version) {
@@ -2041,10 +2027,10 @@ ACTOR Future<Void> watchStorageServerResp(Key key, Database cx) {
 
 			Reference<WatchMetadata> metadata = cx->getWatchMetadata(key);
 			if (!metadata.isValid()) {
-				return Void();
+				co_return;
 			} else if (metadata->watchPromise.getFutureReferenceCount() == 1) {
 				cx->deleteWatchMetadata(key);
-				return Void();
+				co_return;
 			} else if (e.code() == error_code_future_version) {
 				continue;
 			}
@@ -2055,13 +2041,14 @@ ACTOR Future<Void> watchStorageServerResp(Key key, Database cx) {
 	}
 }
 
-ACTOR Future<Void> sameVersionDiffValue(Database cx, Reference<WatchParameters> parameters) {
-	state ReadYourWritesTransaction tr(cx);
+Future<Void> sameVersionDiffValue(Database cx, Reference<WatchParameters> parameters) {
+	ReadYourWritesTransaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			state Optional<Value> valSS = wait(tr.get(parameters->key));
+			Optional<Value> valSS = co_await tr.get(parameters->key);
 			Reference<WatchMetadata> metadata = cx->getWatchMetadata(parameters->key);
 
 			// val_3 != val_1 (storage server value doesn't match value in map)
@@ -2078,13 +2065,14 @@ ACTOR Future<Void> sameVersionDiffValue(Database cx, Reference<WatchParameters>
 				cx->setWatchMetadata(metadata);
 
 				metadata->watchFutureSS = watchStorageServerResp(parameters->key, cx);
-				wait(success(metadata->watchPromise.getFuture()));
+				co_await metadata->watchPromise.getFuture();
 			}
 
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -2856,9 +2844,9 @@ struct TSSDuplicateStreamData {
 	Promise<Void> tssComparisonDone;
 
 	// empty constructor for optional?
-	TSSDuplicateStreamData() {}
+	TSSDuplicateStreamData() = default;
 
-	TSSDuplicateStreamData(PromiseStream<StreamReply> stream) : stream(stream) {}
+	explicit TSSDuplicateStreamData(PromiseStream<StreamReply> stream) : stream(stream) {}
 
 	bool done() { return tssComparisonDone.getFuture().isReady(); }
 
@@ -2868,7 +2856,7 @@ struct TSSDuplicateStreamData {
 		}
 	}
 
-	~TSSDuplicateStreamData() {}
+	~TSSDuplicateStreamData() = default;
 };
 
 // Error tracking here is weird, and latency doesn't really mean the same thing here as it does with normal tss
@@ -2950,7 +2938,7 @@ static Future<Void> tssStreamComparison(Request request,
 			if ((!ssEndOfStream || !tssEndOfStream) && !TSS_doCompare(ssReply.get(), tssReply.get())) {
 				CODE_PROBE(true, "TSS mismatch in stream comparison");
 				TraceEvent mismatchEvent(
-				    (g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
+				    (simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch))
 				        ? SevWarnAlways
 				        : SevError,
 				    LB_mismatchTraceName(request, TSS_COMPARISON));
@@ -2972,11 +2960,12 @@ static Future<Void> tssStreamComparison(Request request,
 						tssData.metrics->recordDetailedMismatchData(mismatchUID, mismatchEvent.getFields().toString());
 
 						// record a summarized trace event instead
-						TraceEvent summaryEvent((g_network->isSimulated() &&
-						                         g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
-						                            ? SevWarnAlways
-						                            : SevError,
-						                        LB_mismatchTraceName(request, TSS_COMPARISON));
+						TraceEvent summaryEvent(
+						    (g_network->isSimulated() &&
+						     simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch))
+						        ? SevWarnAlways
+						        : SevError,
+						    LB_mismatchTraceName(request, TSS_COMPARISON));
 						summaryEvent.detail("TSSID", tssData.tssId).detail("MismatchId", mismatchUID);
 					}
 				} else {
@@ -3018,14 +3007,14 @@ maybeDuplicateTSSStreamFragment(Request& req, QueueModel* model, RequestStream<R
 	return Optional<TSSDuplicateStreamData<REPLYSTREAM_TYPE(Request)>>();
 }
 
-// Streams all of the KV pairs in a target key range into a ParallelStream fragment
-ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
-                                          ParallelStream<RangeResult>::Fragment* results,
-                                          KeyRange keys,
-                                          GetRangeLimits limits,
-                                          Snapshot snapshot,
-                                          Reverse reverse,
-                                          SpanContext spanContext) {
+// Streams all of the KV pairs in a target key range directly to the client in order.
+ACTOR Future<Void> getRangeStreamImpl(Reference<TransactionState> trState,
+                                      PromiseStream<RangeResult> results,
+                                      KeyRange keys,
+                                      GetRangeLimits limits,
+                                      Snapshot snapshot,
+                                      Reverse reverse,
+                                      SpanContext spanContext) {
 	loop {
 		state std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(
 		    trState, keys, CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT, reverse, &StorageServerInterface::getKeyValuesStream));
@@ -3065,7 +3054,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 
 				if (locations[shard].locations->size() == 0) {
 					wait(trState->cx->connectionFileChanged());
-					results->sendError(transaction_too_old());
+					results.sendError(transaction_too_old());
 					return Void();
 				}
 
@@ -3122,11 +3111,11 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 
 				state bool breakAgain = false;
 				loop {
-					wait(results->onEmpty());
+					wait(results.onEmpty());
 					try {
 						choose {
 							when(wait(trState->cx->connectionFileChanged())) {
-								results->sendError(transaction_too_old());
+								results.sendError(transaction_too_old());
 								if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
 									tssDuplicateStream.get().stream.sendError(transaction_too_old());
 								}
@@ -3225,12 +3214,11 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 									output.readThroughEnd = true;
 								}
 								output.arena().dependsOn(keys.arena());
-								// for getRangeStreamFragment, one fragment end doesn't mean it's the end of getRange
-								// so set 'more' to true
+								// getRangeStream() uses end_of_stream to indicate exhaustion, so keep 'more' true.
 								output.more = true;
 								output.setReadThrough(reverse ? keys.begin : keys.end);
-								results->send(std::move(output));
-								results->finish();
+								results.send(std::move(output));
+								results.sendError(end_of_stream());
 								if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
 									tssDuplicateStream.get().stream.sendError(end_of_stream());
 								}
@@ -3242,10 +3230,10 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 							++shard;
 						}
 						output.arena().dependsOn(range.arena());
-						// if it's not the last shard, set more to true and readThrough to the shard boundary
+						// If it's not the last shard, set more to true and readThrough to the shard boundary.
 						output.more = true;
 						output.setReadThrough(reverse ? range.begin : range.end);
-						results->send(std::move(output));
+						results.send(std::move(output));
 						break;
 					}
 
@@ -3256,7 +3244,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 					if (keys.end == allKeys.end && reverse) {
 						output.readThroughEnd = true;
 					}
-					results->send(std::move(output));
+					results.send(std::move(output));
 				}
 				if (breakAgain) {
 					break;
@@ -3283,7 +3271,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 					wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
 					break;
 				} else {
-					results->sendError(e);
+					results.sendError(e);
 					return Void();
 				}
 			}
@@ -3294,13 +3282,7 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<TransactionState> trState,
                                                                 KeyRange keys,
                                                                 int64_t chunkSize);
-
-static KeyRange intersect(KeyRangeRef lhs, KeyRangeRef rhs) {
-	return KeyRange(KeyRangeRef(std::max(lhs.begin, rhs.begin), std::min(lhs.end, rhs.end)));
-}
-
-// Divides the requested key range into 1MB fragments, create range streams for each fragment, and merges the results so
-// the client get them in order
+// Streams the requested key range directly from storage servers without fragment-level parallelism.
 ACTOR Future<Void> getRangeStream(Reference<TransactionState> trState,
                                   PromiseStream<RangeResult> _results,
                                   KeySelector begin,
@@ -3309,8 +3291,6 @@ ACTOR Future<Void> getRangeStream(Reference<TransactionState> trState,
                                   Promise<std::pair<Key, Key>> conflictRange,
                                   Snapshot snapshot,
                                   Reverse reverse) {
-	state ParallelStream<RangeResult> results(_results, CLIENT_KNOBS->RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT);
-
 	// FIXME: better handling to disable row limits
 	ASSERT(!limits.hasRowLimit());
 	state Span span("NAPI:getRangeStream"_loc, trState->spanContext);
@@ -3332,52 +3312,11 @@ ACTOR Future<Void> getRangeStream(Reference<TransactionState> trState,
 	}
 
 	if (b >= e) {
-		wait(results.finish());
+		_results.sendError(end_of_stream());
 		return Void();
 	}
 
-	// if e is allKeys.end, we have read through the end of the database
-	// if b is allKeys.begin, we have either read through the beginning of the database,
-	// or allKeys.begin exists in the database and will be part of the conflict range anyways
-
-	state std::vector<Future<Void>> outstandingRequests;
-	while (b < e) {
-		state KeyRangeLocationInfo locationInfo =
-		    wait(getKeyLocation(trState, reverse ? e : b, &StorageServerInterface::getKeyValuesStream, reverse));
-		state KeyRange shardIntersection = intersect(locationInfo.range, KeyRangeRef(b, e));
-		state Standalone<VectorRef<KeyRef>> splitPoints =
-		    wait(getRangeSplitPoints(trState, shardIntersection, CLIENT_KNOBS->RANGESTREAM_FRAGMENT_SIZE));
-		state std::vector<KeyRange> toSend;
-		// state std::vector<Future<std::list<KeyRangeRef>::iterator>> outstandingRequests;
-
-		if (!splitPoints.empty()) {
-			toSend.push_back(KeyRange(KeyRangeRef(shardIntersection.begin, splitPoints.front()), splitPoints.arena()));
-			for (int i = 0; i < splitPoints.size() - 1; ++i) {
-				toSend.push_back(KeyRange(KeyRangeRef(splitPoints[i], splitPoints[i + 1]), splitPoints.arena()));
-			}
-			toSend.push_back(KeyRange(KeyRangeRef(splitPoints.back(), shardIntersection.end), splitPoints.arena()));
-		} else {
-			toSend.push_back(KeyRange(KeyRangeRef(shardIntersection.begin, shardIntersection.end)));
-		}
-
-		state int idx = 0;
-		state int useIdx = 0;
-		for (; idx < toSend.size(); ++idx) {
-			useIdx = reverse ? toSend.size() - idx - 1 : idx;
-			if (toSend[useIdx].empty()) {
-				continue;
-			}
-			ParallelStream<RangeResult>::Fragment* fragment = wait(results.createFragment());
-			outstandingRequests.push_back(
-			    getRangeStreamFragment(trState, fragment, toSend[useIdx], limits, snapshot, reverse, span.context));
-		}
-		if (reverse) {
-			e = shardIntersection.begin;
-		} else {
-			b = shardIntersection.end;
-		}
-	}
-	wait(waitForAll(outstandingRequests) && results.finish());
+	wait(getRangeStreamImpl(trState, _results, KeyRange(KeyRangeRef(b, e)), limits, snapshot, reverse, span.context));
 	return Void();
 }
 
@@ -3536,21 +3475,19 @@ void Watch::setWatch(Future<Void> watchFuture) {
 }
 
 // Restarts a watch after a database switch
-ACTOR Future<Void> restartWatch(Database cx,
-                                Key key,
-                                Optional<Value> value,
-                                TagSet tags,
-                                SpanContext spanContext,
-                                TaskPriority taskID,
-                                Optional<UID> debugID,
-                                UseProvisionalProxies useProvisionalProxies) {
+Future<Void> restartWatch(Database cx,
+                          Key key,
+                          Optional<Value> value,
+                          TagSet tags,
+                          SpanContext spanContext,
+                          TaskPriority taskID,
+                          Optional<UID> debugID,
+                          UseProvisionalProxies useProvisionalProxies) {
 	// Remove the reference count as the old watches should be all dropped when switching connectionFile.
 	cx->deleteWatchMetadata(key, /* removeReferenceCount */ true);
 
-	wait(watchValueMap(
-	    cx->minAcceptableReadVersion, key, value, cx, tags, spanContext, taskID, debugID, useProvisionalProxies));
-
-	return Void();
+	co_await watchValueMap(
+	    cx->minAcceptableReadVersion, key, value, cx, tags, spanContext, taskID, debugID, useProvisionalProxies);
 }
 
 // FIXME: This seems pretty horrible. Now a Database can't die until all of its watches do...
@@ -3620,25 +3557,25 @@ Future<Void> Transaction::watch(Reference<Watch> watch) {
 	               trState->useProvisionalProxies);
 }
 
-ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Reference<TransactionState> trState, Key key) {
-	state std::vector<StorageServerInterface> ssi;
+Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Reference<TransactionState> trState, Key key) {
+	std::vector<StorageServerInterface> ssi;
 
-	wait(trState->startTransaction());
+	co_await trState->startTransaction();
 
-	state Key resolvedKey = key;
+	Key resolvedKey = key;
 
 	// If key >= allKeys.end, then getRange will return a kv-pair with an empty value. This will result in our
 	// serverInterfaces vector being empty, which will cause us to return an empty addresses list.
-	state Key ksKey = keyServersKey(resolvedKey);
-	state RangeResult serverTagResult = wait(getRange(trState,
-	                                                  lastLessOrEqual(serverTagKeys.begin),
-	                                                  firstGreaterThan(serverTagKeys.end),
-	                                                  GetRangeLimits(CLIENT_KNOBS->TOO_MANY),
-	                                                  Reverse::False));
+	Key ksKey = keyServersKey(resolvedKey);
+	RangeResult serverTagResult = co_await getRange(trState,
+	                                                lastLessOrEqual(serverTagKeys.begin),
+	                                                firstGreaterThan(serverTagKeys.end),
+	                                                GetRangeLimits(CLIENT_KNOBS->TOO_MANY),
+	                                                Reverse::False);
 	ASSERT(!serverTagResult.more && serverTagResult.size() < CLIENT_KNOBS->TOO_MANY);
 	Future<RangeResult> futureServerUids =
 	    getRange(trState, lastLessOrEqual(ksKey), firstGreaterThan(ksKey), GetRangeLimits(1), Reverse::False);
-	RangeResult serverUids = wait(futureServerUids);
+	RangeResult serverUids = co_await futureServerUids;
 
 	ASSERT(serverUids.size()); // every shard needs to have a team
 
@@ -3648,7 +3585,7 @@ ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Referen
 	                         // the move is finished, because it could be cancelled at any time.
 	decodeKeyServersValue(serverTagResult, serverUids[0].value, src, ignore);
 	Optional<std::vector<StorageServerInterface>> serverInterfaces =
-	    wait(transactionalGetServerInterfaces(trState, src));
+	    co_await transactionalGetServerInterfaces(trState, src);
 
 	ASSERT(serverInterfaces.present()); // since this is happening transactionally, /FF/keyServers and /FF/serverList
 	                                    // need to be consistent with one another
@@ -3661,7 +3598,7 @@ ACTOR Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Referen
 		strcpy(c_string, ipString.c_str());
 		addresses.push_back(addresses.arena(), c_string);
 	}
-	return addresses;
+	co_return addresses;
 }
 
 Future<Standalone<VectorRef<const char*>>> Transaction::getAddressesForKey(const Key& key) {
@@ -4069,6 +4006,7 @@ void TransactionOptions::clear() {
 	rawAccess = false;
 	bypassStorageQuota = false;
 	enableReplicaConsistencyCheck = false;
+	maxGrvQueueDelayMS = Optional<int64_t>();
 	requiredReplicas = 0;
 }
 
@@ -4119,8 +4057,8 @@ class MutationBlock {
 	ValueRef setValue;
 
 	MutationBlock() : mutated(false) {}
-	MutationBlock(bool _cleared) : mutated(true), cleared(_cleared) {}
-	MutationBlock(ValueRef value) : mutated(true), cleared(false), setValue(value) {}
+	explicit MutationBlock(bool _cleared) : mutated(true), cleared(_cleared) {}
+	explicit MutationBlock(ValueRef value) : mutated(true), cleared(false), setValue(value) {}
 };
 
 bool compareBegin(KeyRangeRef lhs, KeyRangeRef rhs) {
@@ -4148,57 +4086,57 @@ Optional<KeyRangeRef> intersects(VectorRef<KeyRangeRef> lhs, VectorRef<KeyRangeR
 	return Optional<KeyRangeRef>();
 }
 
-ACTOR void checkWrites(Reference<TransactionState> trState,
-                       Future<Void> committed,
-                       Promise<Void> outCommitted,
-                       CommitTransactionRequest req) {
-	state Version version;
+Future<Void> checkWrites(Uncancellable,
+                         Reference<TransactionState> trState,
+                         Future<Void> committed,
+                         Promise<Void> outCommitted,
+                         CommitTransactionRequest req) {
+	Version version{ 0 };
 	try {
-		wait(committed);
+		co_await committed;
 		// If the commit is successful, by definition the transaction still exists for now.  Grab the version, and don't
 		// use it again.
 		version = trState->committedVersion;
 		outCommitted.send(Void());
 	} catch (Error& e) {
 		outCommitted.sendError(e);
-		return;
+		co_return;
 	}
 
-	wait(delay(deterministicRandom()->random01())); // delay between 0 and 1 seconds
+	co_await delay(deterministicRandom()->random01()); // delay between 0 and 1 seconds
 
-	state KeyRangeMap<MutationBlock> expectedValues;
+	KeyRangeMap<MutationBlock> expectedValues;
 
 	auto& mutations = req.transaction.mutations;
-	state int mCount = mutations.size(); // debugging info for traceEvent
+	const int mCount = mutations.size(); // debugging info for traceEvent
 
-	for (int idx = 0; idx < mutations.size(); idx++) {
-		if (mutations[idx].type == MutationRef::SetValue)
-			expectedValues.insert(singleKeyRange(mutations[idx].param1), MutationBlock(mutations[idx].param2));
-		else if (mutations[idx].type == MutationRef::ClearRange)
-			expectedValues.insert(KeyRangeRef(mutations[idx].param1, mutations[idx].param2), MutationBlock(true));
+	for (const auto& mutation : mutations) {
+		if (mutation.type == MutationRef::SetValue)
+			expectedValues.insert(singleKeyRange(mutation.param1), MutationBlock(mutation.param2));
+		else if (mutation.type == MutationRef::ClearRange)
+			expectedValues.insert(KeyRangeRef(mutation.param1, mutation.param2), MutationBlock(true));
 	}
 
 	try {
-		state Transaction tr(trState->cx);
+		Transaction tr(trState->cx);
 		tr.setVersion(version);
-		state int checkedRanges = 0;
-		state KeyRangeMap<MutationBlock>::Ranges ranges = expectedValues.ranges();
-		state KeyRangeMap<MutationBlock>::iterator it = ranges.begin();
-		for (; it != ranges.end(); ++it) {
-			state MutationBlock m = it->value();
+		int checkedRanges = 0;
+		auto ranges = expectedValues.ranges();
+		for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+			MutationBlock m = it->value();
 			if (m.mutated) {
 				checkedRanges++;
 				if (m.cleared) {
-					RangeResult shouldBeEmpty = wait(tr.getRange(it->range(), 1));
+					RangeResult shouldBeEmpty = co_await tr.getRange(it->range(), 1);
 					if (shouldBeEmpty.size()) {
 						TraceEvent(SevError, "CheckWritesFailed")
 						    .detail("Class", "Clear")
 						    .detail("KeyBegin", it->range().begin)
 						    .detail("KeyEnd", it->range().end);
-						return;
+						co_return;
 					}
 				} else {
-					Optional<Value> val = wait(tr.get(it->range().begin));
+					Optional<Value> val = co_await tr.get(it->range().begin);
 					if (!val.present() || val.get() != m.setValue) {
 						TraceEvent evt(SevError, "CheckWritesFailed");
 						evt.detail("Class", "Set").detail("Key", it->range().begin).detail("Expected", m.setValue);
@@ -4206,7 +4144,7 @@ ACTOR void checkWrites(Reference<TransactionState> trState,
 							evt.detail("Actual", "_Value Missing_");
 						else
 							evt.detail("Actual", val.get());
-						return;
+						co_return;
 					}
 				}
 			}
@@ -4222,12 +4160,13 @@ ACTOR void checkWrites(Reference<TransactionState> trState,
 	}
 }
 
-ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trState, KeyRange range) {
-	state Transaction tr(trState->cx);
-	state int retries = 0;
-	state Span span("NAPI:dummyTransaction"_loc, trState->spanContext);
+static Future<Void> commitDummyTransaction(Reference<TransactionState> trState, KeyRange range) {
+	Transaction tr(trState->cx);
+	int retries = 0;
+	Span span("NAPI:dummyTransaction"_loc, trState->spanContext);
 	tr.span.setParent(span.context);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			TraceEvent("CommitDummyTransaction").detail("Key", range.begin).detail("Retries", retries);
 			tr.trState->options = trState->options;
@@ -4238,29 +4177,31 @@ ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trS
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.addReadConflictRange(range);
 			tr.addWriteConflictRange(range);
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			TraceEvent("CommitDummyTransactionError")
-			    .errorUnsuppressed(e)
-			    .detail("Key", range.begin)
-			    .detail("Retries", retries);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("CommitDummyTransactionError")
+		    .errorUnsuppressed(err)
+		    .detail("Key", range.begin)
+		    .detail("Retries", retries);
+		co_await tr.onError(err);
 		++retries;
 	}
 }
 
-ACTOR static Future<Optional<CommitResult>> determineCommitStatus(Reference<TransactionState> trState,
-                                                                  Version minPossibleCommitVersion,
-                                                                  Version maxPossibleCommitVersion,
-                                                                  IdempotencyIdRef idempotencyId) {
-	state Transaction tr(trState->cx);
-	state int retries = 0;
-	state Version expiredVersion;
-	state Span span("NAPI:determineCommitStatus"_loc, trState->spanContext);
+static Future<Optional<CommitResult>> determineCommitStatus(Reference<TransactionState> trState,
+                                                            Version minPossibleCommitVersion,
+                                                            Version maxPossibleCommitVersion,
+                                                            IdempotencyIdRef idempotencyId) {
+	Transaction tr(trState->cx);
+	int retries = 0;
+	Version expiredVersion{ 0 };
+	Span span("NAPI:determineCommitStatus"_loc, trState->spanContext);
 	tr.span.setParent(span.context);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.trState->options = trState->options;
 			tr.trState->taskID = trState->taskID;
@@ -4269,12 +4210,12 @@ ACTOR static Future<Optional<CommitResult>> determineCommitStatus(Reference<Tran
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			KeyBackedObjectProperty<IdempotencyIdsExpiredVersion, _Unversioned> expiredKey(idempotencyIdsExpiredVersion,
 			                                                                               Unversioned());
-			IdempotencyIdsExpiredVersion expiredVal = wait(expiredKey.getD(&tr));
+			IdempotencyIdsExpiredVersion expiredVal = co_await expiredKey.getD(&tr);
 			expiredVersion = expiredVal.expired;
 			if (expiredVersion >= minPossibleCommitVersion) {
 				throw commit_unknown_result_fatal();
 			}
-			Version rv = wait(tr.getReadVersion());
+			Version rv = co_await tr.getReadVersion();
 			TraceEvent("DetermineCommitStatusAttempt")
 			    .detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
 			    .detail("Retries", retries)
@@ -4287,7 +4228,7 @@ ACTOR static Future<Optional<CommitResult>> determineCommitStatus(Reference<Tran
 			                    .withPrefix(idempotencyIdKeys.begin),
 			                BinaryWriter::toValue(bigEndian64(maxPossibleCommitVersion + 1), Unversioned())
 			                    .withPrefix(idempotencyIdKeys.begin));
-			RangeResult range = wait(tr.getRange(possibleRange, CLIENT_KNOBS->TOO_MANY));
+			RangeResult range = co_await tr.getRange(possibleRange, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!range.more);
 			for (const auto& kv : range) {
 				auto commitResult = kvContainsIdempotencyId(kv, idempotencyId);
@@ -4296,29 +4237,30 @@ ACTOR static Future<Optional<CommitResult>> determineCommitStatus(Reference<Tran
 					    .detail("Committed", 1)
 					    .detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
 					    .detail("Retries", retries);
-					return commitResult;
+					co_return commitResult;
 				}
 			}
 			TraceEvent("DetermineCommitStatus")
 			    .detail("Committed", 0)
 			    .detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
 			    .detail("Retries", retries);
-			return Optional<CommitResult>();
+			co_return Optional<CommitResult>();
 		} catch (Error& e) {
-			TraceEvent("DetermineCommitStatusError")
-			    .errorUnsuppressed(e)
-			    .detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
-			    .detail("Retries", retries);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("DetermineCommitStatusError")
+		    .errorUnsuppressed(err)
+		    .detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
+		    .detail("Retries", retries);
+		co_await tr.onError(err);
 		++retries;
 	}
 }
 
 void Transaction::cancelWatches(Error const& e) {
-	for (int i = 0; i < watches.size(); ++i)
-		if (!watches[i]->onChangeTrigger.isSet())
-			watches[i]->onChangeTrigger.sendError(e);
+	for (auto& watch : watches)
+		if (!watch->onChangeTrigger.isSet())
+			watch->onChangeTrigger.sendError(e);
 
 	watches.clear();
 }
@@ -4327,11 +4269,11 @@ void Transaction::setupWatches() {
 	try {
 		Future<Version> watchVersion = getCommittedVersion() > 0 ? getCommittedVersion() : getReadVersion();
 
-		for (int i = 0; i < watches.size(); ++i)
-			watches[i]->setWatch(
+		for (auto& watch : watches)
+			watch->setWatch(
 			    watchValueMap(watchVersion,
-			                  watches[i]->key,
-			                  watches[i]->value,
+			                  watch->key,
+			                  watch->value,
 			                  trState->cx,
 			                  trState->options.readTags,
 			                  trState->spanContext,
@@ -4569,7 +4511,8 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState, CommitT
 			}
 		}
 	} catch (Error& e) {
-		if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result) {
+		if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result ||
+		    e.code() == error_code_never_reply) {
 			// We don't know if the commit happened, and it might even still be in flight.
 
 			if (!trState->options.causalWriteRisky || req.idempotencyId.valid()) {
@@ -4595,6 +4538,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState, CommitT
 					    req.transaction.read_snapshot + CLIENT_KNOBS->MAX_WRITE_TRANSACTION_LIFE_VERSIONS,
 					    req.idempotencyId));
 					if (commitResult.present()) {
+						trState->committedVersion = commitResult.get().commitVersion;
 						Standalone<StringRef> ret = makeString(10);
 						placeVersionstamp(
 						    mutateString(ret), commitResult.get().commitVersion, commitResult.get().batchIndex);
@@ -4617,7 +4561,7 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState, CommitT
 			    e.code() != error_code_grv_proxy_memory_limit_exceeded &&
 			    e.code() != error_code_batch_transaction_throttled && e.code() != error_code_tag_throttled &&
 			    e.code() != error_code_process_behind && e.code() != error_code_future_version &&
-			    e.code() != error_code_proxy_tag_throttled && e.code() != error_code_transaction_throttled_hot_shard &&
+			    e.code() != error_code_transaction_throttled_hot_shard &&
 			    e.code() != error_code_transaction_rejected_range_locked) {
 				TraceEvent(SevError, "TryCommitError").error(e);
 			}
@@ -4672,11 +4616,10 @@ Future<Void> Transaction::commitMutations() {
 		}
 
 		bool isCheckingWrites = trState->options.checkWritesEnabled && deterministicRandom()->random01() < 0.01;
-		for (int i = 0; i < extraConflictRanges.size(); i++)
-			if (extraConflictRanges[i].isReady() &&
-			    extraConflictRanges[i].get().first < extraConflictRanges[i].get().second)
+		for (const auto& extraConflictRange : extraConflictRanges)
+			if (extraConflictRange.isReady() && extraConflictRange.get().first < extraConflictRange.get().second)
 				tr.transaction.read_conflict_ranges.emplace_back(
-				    tr.arena, extraConflictRanges[i].get().first, extraConflictRanges[i].get().second);
+				    tr.arena, extraConflictRange.get().first, extraConflictRange.get().second);
 
 		if (tr.idempotencyId.valid()) {
 			// We need to be able confirm that this transaction is no longer in
@@ -4727,7 +4670,7 @@ Future<Void> Transaction::commitMutations() {
 
 		if (isCheckingWrites) {
 			Promise<Void> committed;
-			checkWrites(trState, commitResult, committed, tr);
+			checkWrites(Uncancellable(), trState, commitResult, committed, tr);
 			return committed.getFuture();
 		}
 		return commitResult;
@@ -4785,6 +4728,18 @@ Future<Void> Transaction::commit() {
 	return committing;
 }
 
+// Returns a thread-local mt19937_64 seeded once with 32 bytes of OS entropy.
+// Used for AUTOMATIC_IDEMPOTENCY ID generation in non-simulation runs.
+static std::mt19937_64& getIdempotencyRng() {
+	static thread_local std::mt19937_64 rng = []() {
+		uint32_t seed_data[8];
+		platform::getRandomBytes(seed_data, sizeof(seed_data));
+		std::seed_seq seq(seed_data, seed_data + 8);
+		return std::mt19937_64(seq);
+	}();
+	return rng;
+}
+
 void Transaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
 	switch (option) {
 	case FDBTransactionOptions::INITIALIZE_NEW_DATABASE:
@@ -4904,6 +4859,11 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 		trState->options.maxBackoff = extractIntOption(value, 0, std::numeric_limits<int32_t>::max()) / 1000.0;
 		break;
 
+	case FDBTransactionOptions::MAX_GRV_QUEUE_DELAY:
+		validateOptionValuePresent(value);
+		trState->options.maxGrvQueueDelayMS = extractIntOption(value, 0, std::numeric_limits<int32_t>::max());
+		break;
+
 	case FDBTransactionOptions::SIZE_LIMIT:
 		validateOptionValuePresent(value);
 		trState->options.sizeLimit = extractIntOption(value, 32, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
@@ -5026,9 +4986,15 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional<Strin
 	case FDBTransactionOptions::AUTOMATIC_IDEMPOTENCY:
 		validateOptionValueNotPresent(value);
 		if (!tr.idempotencyId.valid()) {
-			tr.idempotencyId = IdempotencyIdRef(
-			    tr.arena,
-			    IdempotencyIdRef(BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned())));
+			StringRef id = makeString(16, tr.arena);
+			if (g_network->isSimulated()) {
+				deterministicRandom()->randomBytes(mutateString(id), 16);
+			} else {
+				auto& rng = getIdempotencyRng();
+				uint64_t buf[2] = { rng(), rng() };
+				memcpy(mutateString(id), buf, 16);
+			}
+			tr.idempotencyId = IdempotencyIdRef(id);
 		}
 		trState->automaticIdempotency = true;
 		break;
@@ -5073,7 +5039,8 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
                                                            TransactionPriority priority,
                                                            uint32_t flags,
                                                            TransactionTagMap<uint32_t> tags,
-                                                           Optional<UID> debugID) {
+                                                           Optional<UID> debugID,
+                                                           Optional<int64_t> maxGrvQueueDelayMS) {
 	state Span span("NAPI:getConsistentReadVersion"_loc, parentSpan);
 
 	++cx->transactionReadVersionBatches;
@@ -5087,7 +5054,8 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
 			                                cx->ssVersionVectorCache.getMaxVersion(),
 			                                flags,
 			                                tags,
-			                                debugID);
+			                                debugID,
+			                                maxGrvQueueDelayMS);
 			state Future<Void> onProxiesChanged = cx->onProxiesChanged();
 
 			choose {
@@ -5100,8 +5068,6 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
 				                               &GrvProxyInterface::getConsistentReadVersion,
 				                               req,
 				                               cx->taskID))) {
-					CODE_PROBE(v.proxyTagThrottledDuration > 0.0,
-					           "getConsistentReadVersion received GetReadVersionReply delayed by proxy tag throttling");
 					if (tags.size() != 0) {
 						auto& priorityThrottledTags = cx->throttledTags[priority];
 						for (auto& tag : tags) {
@@ -5136,7 +5102,8 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
 			}
 		} catch (Error& e) {
 			if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled &&
-			    e.code() != error_code_grv_proxy_memory_limit_exceeded && e.code() != error_code_proxy_tag_throttled)
+			    e.code() != error_code_grv_proxy_memory_limit_exceeded &&
+			    e.code() != error_code_transaction_grv_queue_rejected)
 				TraceEvent(SevError, "GetConsistentReadVersionError").error(e);
 			throw;
 		}
@@ -5146,7 +5113,8 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpa
 ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
                                       FutureStream<DatabaseContext::VersionRequest> versionStream,
                                       TransactionPriority priority,
-                                      uint32_t flags) {
+                                      uint32_t flags,
+                                      Optional<int64_t> maxGrvQueueDelayMS) {
 	state std::vector<Promise<GetReadVersionReply>> requests;
 	state PromiseStream<Future<Void>> addActor;
 	state Future<Void> collection = actorCollection(addActor.getFuture());
@@ -5221,7 +5189,8 @@ ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
 			addActor.send(ready(timeReply(GRVReply.getFuture(), replyTimes)));
 
 			Future<Void> batch = incrementalBroadcastWithError(
-			    getConsistentReadVersion(span.context, cx, count, priority, flags, std::move(tags), std::move(debugID)),
+			    getConsistentReadVersion(
+			        span.context, cx, count, priority, flags, std::move(tags), std::move(debugID), maxGrvQueueDelayMS),
 			    std::move(requests),
 			    CLIENT_KNOBS->BROADCAST_BATCH_SIZE);
 
@@ -5246,7 +5215,6 @@ ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
 	double latency = replyTime - trState->startTime;
 	trState->cx->lastProxyRequestTime = trState->startTime;
 	trState->cx->updateCachedReadVersion(trState->startTime, rep.version);
-	trState->proxyTagThrottledDuration += rep.proxyTagThrottledDuration;
 	if (rep.rkBatchThrottled) {
 		trState->cx->lastRkBatchThrottleTime = replyTime;
 	}
@@ -5453,14 +5421,18 @@ Future<Version> TransactionState::getReadVersion(uint32_t flags) {
 		}
 	}
 
-	auto& batcher = cx->versionBatcher[flags];
-	if (!batcher.actor.isValid()) {
-		batcher.actor = readVersionBatcher(cx.getPtr(), batcher.stream.getFuture(), options.priority, flags);
-	}
-
 	Location location = "NAPI:getReadVersion"_loc;
 	SpanContext derivedSpanContext = generateSpanID(cx->transactionTracingSample, spanContext);
 	Optional<UID> versionDebugID = readOptions.present() ? readOptions.get().debugID : Optional<UID>();
+
+	// Include the max GRV queue delay in the batcher key so coalesced requests
+	// share the same proxy-side admission threshold.
+	auto& batcher = cx->versionBatcher[DatabaseContext::VersionBatcherKey(flags, options.maxGrvQueueDelayMS)];
+	if (!batcher.actor.isValid()) {
+		batcher.actor = readVersionBatcher(
+		    cx.getPtr(), batcher.stream.getFuture(), options.priority, flags, options.maxGrvQueueDelayMS);
+	}
+
 	auto const req = DatabaseContext::VersionRequest(derivedSpanContext, options.tags, versionDebugID);
 	batcher.stream.send(req);
 	startTime = now();
@@ -5477,7 +5449,7 @@ Optional<Version> Transaction::getCachedReadVersion() const {
 }
 
 double Transaction::getTagThrottledDuration() const {
-	return trState->proxyTagThrottledDuration;
+	return 0.0;
 }
 
 Future<Standalone<StringRef>> Transaction::getVersionstamp() {
@@ -5488,32 +5460,31 @@ Future<Standalone<StringRef>> Transaction::getVersionstamp() {
 }
 
 // Gets the protocol version reported by a coordinator via the protocol info interface
-ACTOR Future<ProtocolVersion> getCoordinatorProtocol(NetworkAddress coordinatorAddress) {
+Future<ProtocolVersion> getCoordinatorProtocol(NetworkAddress coordinatorAddress) {
 	RequestStream<ProtocolInfoRequest> requestStream(
 	    Endpoint::wellKnown({ coordinatorAddress }, WLTOKEN_PROTOCOL_INFO));
-	ProtocolInfoReply reply = wait(retryBrokenPromise(requestStream, ProtocolInfoRequest{}));
-	return reply.version;
+	ProtocolInfoReply reply = co_await retryBrokenPromise(requestStream, ProtocolInfoRequest{});
+	co_return reply.version;
 }
 
 // Gets the protocol version reported by a coordinator in its connect packet
 // If we are unable to get a version from the connect packet (e.g. because we lost connection with the peer), then this
 // function will return with an unset result.
 // If an expected version is given, this future won't return if the actual protocol version matches the expected version
-ACTOR Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(
-    NetworkAddress coordinatorAddress,
-    Optional<ProtocolVersion> expectedVersion) {
-	state Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> protocolVersion =
+Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(NetworkAddress coordinatorAddress,
+                                                                          Optional<ProtocolVersion> expectedVersion) {
+	Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> protocolVersion =
 	    FlowTransport::transport().getPeerProtocolAsyncVar(coordinatorAddress);
 
 	if (!protocolVersion.present()) {
 		TraceEvent(SevWarnAlways, "GetCoordinatorProtocolPeerMissing").detail("Address", coordinatorAddress);
-		wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
-		return Optional<ProtocolVersion>();
+		co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
+		co_return Optional<ProtocolVersion>();
 	}
 
-	loop {
+	while (true) {
 		if (protocolVersion.get()->get().present() && protocolVersion.get()->get() != expectedVersion) {
-			return protocolVersion.get()->get();
+			co_return protocolVersion.get()->get();
 		}
 
 		Future<Void> change = protocolVersion.get()->onChange();
@@ -5522,10 +5493,10 @@ ACTOR Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(
 			change = timeout(change, FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT, Void());
 		}
 
-		wait(change);
+		co_await change;
 
 		if (!protocolVersion.get()->get().present()) {
-			return protocolVersion.get()->get();
+			co_return protocolVersion.get()->get();
 		}
 	}
 }
@@ -5629,7 +5600,7 @@ Future<Void> Transaction::onError(Error const& e) {
 	    e.code() == error_code_database_locked || e.code() == error_code_commit_proxy_memory_limit_exceeded ||
 	    e.code() == error_code_grv_proxy_memory_limit_exceeded || e.code() == error_code_process_behind ||
 	    e.code() == error_code_batch_transaction_throttled || e.code() == error_code_tag_throttled ||
-	    e.code() == error_code_proxy_tag_throttled || e.code() == error_code_transaction_throttled_hot_shard ||
+	    e.code() == error_code_transaction_throttled_hot_shard ||
 	    (e.code() == error_code_transaction_rejected_range_locked &&
 	     CLIENT_KNOBS->TRANSACTION_LOCK_REJECTION_RETRIABLE)) {
 		if (e.code() == error_code_not_committed)
@@ -5644,9 +5615,6 @@ Future<Void> Transaction::onError(Error const& e) {
 		else if (e.code() == error_code_batch_transaction_throttled || e.code() == error_code_tag_throttled ||
 		         e.code() == error_code_transaction_throttled_hot_shard) {
 			++trState->cx->transactionsThrottled;
-		} else if (e.code() == error_code_proxy_tag_throttled) {
-			++trState->cx->transactionsThrottled;
-			trState->proxyTagThrottledDuration += CLIENT_KNOBS->PROXY_MAX_TAG_THROTTLE_DURATION;
 		} else if (e.code() == error_code_transaction_rejected_range_locked) {
 			++trState->cx->transactionsLockRejected;
 		}
@@ -5671,83 +5639,85 @@ Future<Void> Transaction::onError(Error const& e) {
 
 	return e;
 }
-ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx,
-                                                            KeyRange keys,
-                                                            Optional<Reference<TransactionState>> trState);
+Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx,
+                                                      KeyRange keys,
+                                                      Optional<Reference<TransactionState>> trState);
 
-ACTOR Future<StorageMetrics> doGetStorageMetrics(Database cx,
-                                                 Version version,
-                                                 KeyRange keys,
-                                                 Reference<LocationInfo> locationInfo,
-                                                 Optional<Reference<TransactionState>> trState) {
+Future<StorageMetrics> doGetStorageMetrics(Database cx,
+                                           Version version,
+                                           KeyRange keys,
+                                           Reference<LocationInfo> locationInfo,
+                                           Optional<Reference<TransactionState>> trState) {
+	Error err;
 	try {
 		WaitMetricsRequest req(version, keys, StorageMetrics(), StorageMetrics());
 		req.min.bytes = 0;
 		req.max.bytes = -1;
-		StorageMetrics m = wait(loadBalance(
-		    locationInfo->locations(), &StorageServerInterface::waitMetrics, req, TaskPriority::DataDistribution));
-		return m;
+		StorageMetrics m = co_await loadBalance(
+		    locationInfo->locations(), &StorageServerInterface::waitMetrics, req, TaskPriority::DataDistribution);
+		co_return m;
 	} catch (Error& e) {
-		if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
-			cx->invalidateCache(keys);
-			wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
-		} else if (e.code() == error_code_future_version) {
-			wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution));
-		} else {
-			TraceEvent(SevError, "DoGetStorageMetricsError").error(e);
-			throw;
-		}
-
-		StorageMetrics m = wait(getStorageMetricsLargeKeyRange(cx, keys, trState));
-		return m;
+		err = e;
+	}
+	if (err.code() == error_code_wrong_shard_server || err.code() == error_code_all_alternatives_failed) {
+		cx->invalidateCache(keys);
+		co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
+	} else if (err.code() == error_code_future_version) {
+		co_await delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution);
+	} else {
+		TraceEvent(SevError, "DoGetStorageMetricsError").error(err);
+		throw err;
 	}
+
+	co_return co_await getStorageMetricsLargeKeyRange(cx, keys, trState);
 }
 
-ACTOR Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx,
-                                                            KeyRange keys,
-                                                            Optional<Reference<TransactionState>> trState) {
-	state Span span("NAPI:GetStorageMetricsLargeKeyRange"_loc);
+Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx,
+                                                      KeyRange keys,
+                                                      Optional<Reference<TransactionState>> trState) {
+	Span span("NAPI:GetStorageMetricsLargeKeyRange"_loc);
 	if (trState.present()) {
-		wait(trState.get()->startTransaction());
-	}
-
-	state Version version = trState.present() ? trState.get()->readVersion() : latestVersion;
-	std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(cx,
-	                                                                        keys,
-	                                                                        std::numeric_limits<int>::max(),
-	                                                                        Reverse::False,
-	                                                                        &StorageServerInterface::waitMetrics,
-	                                                                        span.context,
-	                                                                        Optional<UID>(),
-	                                                                        UseProvisionalProxies::False,
-	                                                                        version));
-	state int nLocs = locations.size();
-	state std::vector<Future<StorageMetrics>> fx(nLocs);
-	state StorageMetrics total;
+		co_await trState.get()->startTransaction();
+	}
+
+	Version version = trState.present() ? trState.get()->readVersion() : latestVersion;
+	std::vector<KeyRangeLocationInfo> locations = co_await getKeyRangeLocations(cx,
+	                                                                            keys,
+	                                                                            std::numeric_limits<int>::max(),
+	                                                                            Reverse::False,
+	                                                                            &StorageServerInterface::waitMetrics,
+	                                                                            span.context,
+	                                                                            Optional<UID>(),
+	                                                                            UseProvisionalProxies::False,
+	                                                                            version);
+	int nLocs = locations.size();
+	std::vector<Future<StorageMetrics>> fx(nLocs);
+	StorageMetrics total;
 	KeyRef partBegin, partEnd;
 	for (int i = 0; i < nLocs; i++) {
 		partBegin = (i == 0) ? keys.begin : locations[i].range.begin;
 		partEnd = (i == nLocs - 1) ? keys.end : locations[i].range.end;
 		fx[i] = doGetStorageMetrics(cx, version, KeyRangeRef(partBegin, partEnd), locations[i].locations, trState);
 	}
-	wait(waitForAll(fx));
+	co_await waitForAll(fx);
 	for (int i = 0; i < nLocs; i++) {
 		total += fx[i].get();
 	}
-	return total;
+	co_return total;
 }
 
-ACTOR Future<Void> trackBoundedStorageMetrics(Version version,
-                                              KeyRange keys,
-                                              Reference<LocationInfo> location,
-                                              StorageMetrics x,
-                                              StorageMetrics halfError,
-                                              PromiseStream<StorageMetrics> deltaStream) {
+Future<Void> trackBoundedStorageMetrics(Version version,
+                                        KeyRange keys,
+                                        Reference<LocationInfo> location,
+                                        StorageMetrics x,
+                                        StorageMetrics halfError,
+                                        PromiseStream<StorageMetrics> deltaStream) {
 
 	try {
-		loop {
+		while (true) {
 			WaitMetricsRequest req(version, keys, x - halfError, x + halfError);
-			StorageMetrics nextX = wait(loadBalance(location->locations(), &StorageServerInterface::waitMetrics, req));
+			StorageMetrics nextX =
+			    co_await loadBalance(location->locations(), &StorageServerInterface::waitMetrics, req);
 			deltaStream.send(nextX - x);
 			x = nextX;
 		}
@@ -5757,19 +5727,19 @@ ACTOR Future<Void> trackBoundedStorageMetrics(Version version,
 	}
 }
 
-ACTOR Future<StorageMetrics> waitStorageMetricsMultipleLocations(Version version,
-                                                                 std::vector<KeyRangeLocationInfo> locations,
-                                                                 StorageMetrics min,
-                                                                 StorageMetrics max,
-                                                                 StorageMetrics permittedError) {
-	state int nLocs = locations.size();
-	state std::vector<Future<StorageMetrics>> fx(nLocs);
-	state StorageMetrics total;
-	state PromiseStream<StorageMetrics> deltas;
-	state std::vector<Future<Void>> wx(fx.size());
-	state StorageMetrics halfErrorPerMachine = permittedError * (0.5 / nLocs);
-	state StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs - 1);
-	state StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs - 1);
+Future<StorageMetrics> waitStorageMetricsMultipleLocations(Version version,
+                                                           std::vector<KeyRangeLocationInfo> locations,
+                                                           StorageMetrics min,
+                                                           StorageMetrics max,
+                                                           StorageMetrics permittedError) {
+	int nLocs = locations.size();
+	std::vector<Future<StorageMetrics>> fx(nLocs);
+	StorageMetrics total;
+	PromiseStream<StorageMetrics> deltas;
+	std::vector<Future<Void>> wx(fx.size());
+	StorageMetrics halfErrorPerMachine = permittedError * (0.5 / nLocs);
+	StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs - 1);
+	StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs - 1);
 
 	for (int i = 0; i < nLocs; i++) {
 		WaitMetricsRequest req(version, locations[i].range, StorageMetrics(), StorageMetrics());
@@ -5780,64 +5750,65 @@ ACTOR Future<StorageMetrics> waitStorageMetricsMultipleLocations(Version version
 		                    req,
 		                    TaskPriority::DataDistribution);
 	}
-	wait(waitForAll(fx));
+	co_await waitForAll(fx);
 
 	// invariant: true total is between (total-permittedError/2, total+permittedError/2)
 	for (int i = 0; i < nLocs; i++)
 		total += fx[i].get();
 
 	if (!total.allLessOrEqual(maxPlus))
-		return total;
+		co_return total;
 	if (!minMinus.allLessOrEqual(total))
-		return total;
+		co_return total;
 
 	for (int i = 0; i < nLocs; i++)
 		wx[i] = trackBoundedStorageMetrics(
 		    version, locations[i].range, locations[i].locations, fx[i].get(), halfErrorPerMachine, deltas);
 
-	loop {
-		StorageMetrics delta = waitNext(deltas.getFuture());
+	while (true) {
+		StorageMetrics delta = co_await deltas.getFuture();
 		total += delta;
 		if (!total.allLessOrEqual(maxPlus))
-			return total;
+			co_return total;
 		if (!minMinus.allLessOrEqual(total))
-			return total;
+			co_return total;
 	}
 }
 
-ACTOR Future<StorageMetrics> extractMetrics(Future<std::pair<Optional<StorageMetrics>, int>> fMetrics) {
-	std::pair<Optional<StorageMetrics>, int> x = wait(fMetrics);
-	return x.first.get();
+Future<StorageMetrics> extractMetrics(Future<std::pair<Optional<StorageMetrics>, int>> fMetrics) {
+	std::pair<Optional<StorageMetrics>, int> x = co_await fMetrics;
+	co_return x.first.get();
 }
 
-ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Database cx, KeyRange keys) {
-	state Span span("NAPI:GetReadHotRanges"_loc);
-	loop {
+Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Database cx, KeyRange keys) {
+	Span span("NAPI:GetReadHotRanges"_loc);
+	while (true) {
 		int64_t shardLimit = 100; // Shard limit here does not really matter since this function is currently only used
 		                          // to find the read-hot sub ranges within a read-hot shard.
 		std::vector<KeyRangeLocationInfo> locations =
-		    wait(getKeyRangeLocations(cx,
-		                              keys,
-		                              shardLimit,
-		                              Reverse::False,
-		                              &StorageServerInterface::getReadHotRanges,
-		                              span.context,
-		                              Optional<UID>(),
-		                              UseProvisionalProxies::False,
-		                              latestVersion));
+		    co_await getKeyRangeLocations(cx,
+		                                  keys,
+		                                  shardLimit,
+		                                  Reverse::False,
+		                                  &StorageServerInterface::getReadHotRanges,
+		                                  span.context,
+		                                  Optional<UID>(),
+		                                  UseProvisionalProxies::False,
+		                                  latestVersion);
+		Error err;
 		try {
 			// TODO: how to handle this?
 			// This function is called whenever a shard becomes read-hot. But somehow the shard was split across more
 			// than one storage server after becoming read-hot and before this function is called, i.e. a race
 			// condition. Should we abort and wait for the newly split shards to be hot again?
-			state int nLocs = locations.size();
+			int nLocs = locations.size();
 			// if (nLocs > 1) {
 			//	TraceEvent("RHDDebug")
 			//	    .detail("NumSSIs", nLocs)
 			//	    .detail("KeysBegin", keys.begin.printable().c_str())
 			//	    .detail("KeysEnd", keys.end.printable().c_str());
 			// }
-			state std::vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
+			std::vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
 			KeyRef partBegin, partEnd;
 			for (int i = 0; i < nLocs; i++) {
 				partBegin = (i == 0) ? keys.begin : locations[i].range.begin;
@@ -5849,11 +5820,11 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 				                          TaskPriority::DataDistribution);
 			}
 
-			wait(waitForAll(fReplies));
+			co_await waitForAll(fReplies);
 
 			if (nLocs == 1) {
 				CODE_PROBE(true, "Single-shard read hot range request");
-				return fReplies[0].get().readHotRanges;
+				co_return fReplies[0].get().readHotRanges;
 			} else {
 				CODE_PROBE(true, "Multi-shard read hot range request");
 				Standalone<VectorRef<ReadHotRangeWithMetrics>> results;
@@ -5864,25 +5835,26 @@ ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Da
 					results.arena().dependsOn(fReplies[i].get().readHotRanges.arena());
 				}
 
-				return results;
+				co_return results;
 			}
 		} catch (Error& e) {
-			if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) {
-				TraceEvent(SevError, "GetReadHotSubRangesError").error(e);
-				throw;
-			}
-			cx->invalidateCache(keys);
-			wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
+			err = e;
+		}
+		if (err.code() != error_code_wrong_shard_server && err.code() != error_code_all_alternatives_failed) {
+			TraceEvent(SevError, "GetReadHotSubRangesError").error(err);
+			throw err;
 		}
+		cx->invalidateCache(keys);
+		co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
 	}
 }
 
-ACTOR Future<Optional<StorageMetrics>> waitStorageMetricsWithLocation(Version version,
-                                                                      KeyRange keys,
-                                                                      std::vector<KeyRangeLocationInfo> locations,
-                                                                      StorageMetrics min,
-                                                                      StorageMetrics max,
-                                                                      StorageMetrics permittedError) {
+Future<Optional<StorageMetrics>> waitStorageMetricsWithLocation(Version version,
+                                                                KeyRange keys,
+                                                                std::vector<KeyRangeLocationInfo> locations,
+                                                                StorageMetrics min,
+                                                                StorageMetrics max,
+                                                                StorageMetrics permittedError) {
 	Future<StorageMetrics> fx;
 	if (locations.size() > 1) {
 		fx = waitStorageMetricsMultipleLocations(version, locations, min, max, permittedError);
@@ -5893,39 +5865,40 @@ ACTOR Future<Optional<StorageMetrics>> waitStorageMetricsWithLocation(Version ve
 		                 req,
 		                 TaskPriority::DataDistribution);
 	}
-	StorageMetrics x = wait(fx);
-	return x;
-}
-
-ACTOR Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(
-    Database cx,
-    KeyRange keys,
-    StorageMetrics min,
-    StorageMetrics max,
-    StorageMetrics permittedError,
-    int shardLimit,
-    int expectedShardCount,
-    Optional<Reference<TransactionState>> trState) {
-	state Span span("NAPI:WaitStorageMetrics"_loc, generateSpanID(cx->transactionTracingSample));
-	loop {
+	StorageMetrics x = co_await fx;
+	co_return x;
+}
+
+Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(Database cx,
+                                                                    KeyRange keys,
+                                                                    StorageMetrics min,
+                                                                    StorageMetrics max,
+                                                                    StorageMetrics permittedError,
+                                                                    int shardLimit,
+                                                                    int expectedShardCount,
+                                                                    Optional<Reference<TransactionState>> trState) {
+	Span span("NAPI:WaitStorageMetrics"_loc, generateSpanID(cx->transactionTracingSample));
+	double startTime = now();
+	int retryCount = 0;
+	while (true) {
 		if (trState.present()) {
-			wait(trState.get()->startTransaction());
+			co_await trState.get()->startTransaction();
 		}
-		state Version version = trState.present() ? trState.get()->readVersion() : latestVersion;
-		state std::vector<KeyRangeLocationInfo> locations =
-		    wait(getKeyRangeLocations(cx,
-		                              keys,
-		                              shardLimit,
-		                              Reverse::False,
-		                              &StorageServerInterface::waitMetrics,
-		                              span.context,
-		                              Optional<UID>(),
-		                              UseProvisionalProxies::False,
-		                              version));
+		Version version = trState.present() ? trState.get()->readVersion() : latestVersion;
+		std::vector<KeyRangeLocationInfo> locations =
+		    co_await getKeyRangeLocations(cx,
+		                                  keys,
+		                                  shardLimit,
+		                                  Reverse::False,
+		                                  &StorageServerInterface::waitMetrics,
+		                                  span.context,
+		                                  Optional<UID>(),
+		                                  UseProvisionalProxies::False,
+		                                  version);
 		if (expectedShardCount >= 0 && locations.size() != expectedShardCount) {
 			// NOTE(xwang): This happens only when a split shard haven't been moved to another location. We may need to
 			// change this if we allow split shard stay the same location.
-			return std::make_pair(Optional<StorageMetrics>(), locations.size());
+			co_return std::make_pair(Optional<StorageMetrics>(), locations.size());
 		}
 
 		// SOMEDAY: Right now, if there are too many shards we delay and check again later. There may be a better
@@ -5936,29 +5909,39 @@ ACTOR Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(
 			    .detail("Limit", shardLimit)
 			    .detail("LocationSize", locations.size())
 			    .detail("JitteredSecondsOfPenitence", CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY);
-			wait(delayJittered(CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, TaskPriority::DataDistribution));
+			co_await delayJittered(CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, TaskPriority::DataDistribution);
 			// make sure that the next getKeyRangeLocations() call will actually re-fetch the range
 			cx->invalidateCache(keys);
 			continue;
 		}
 
+		Error err;
 		try {
 			Optional<StorageMetrics> res =
-			    wait(waitStorageMetricsWithLocation(version, keys, locations, min, max, permittedError));
+			    co_await waitStorageMetricsWithLocation(version, keys, locations, min, max, permittedError);
 			if (res.present()) {
-				return std::make_pair(res, -1);
+				co_return std::make_pair(res, -1);
 			}
+			continue;
 		} catch (Error& e) {
-			TraceEvent(SevDebug, "WaitStorageMetricsHandleError").error(e);
-			if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
-				cx->invalidateCache(keys);
-				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
-			} else if (e.code() == error_code_future_version) {
-				wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution));
-			} else {
-				TraceEvent(SevError, "WaitStorageMetricsError").error(e);
-				throw;
-			}
+			err = e;
+		}
+		retryCount++;
+		// Upgrade from SevDebug to SevWarn after 60 seconds of retrying
+		Severity sev = (now() - startTime > 60.0) ? SevWarn : SevDebug;
+		TraceEvent(sev, "WaitStorageMetricsHandleError")
+		    .error(err)
+		    .detail("Keys", keys)
+		    .detail("Elapsed", now() - startTime)
+		    .detail("Retries", retryCount);
+		if (err.code() == error_code_wrong_shard_server || err.code() == error_code_all_alternatives_failed) {
+			cx->invalidateCache(keys);
+			co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
+		} else if (err.code() == error_code_future_version) {
+			co_await delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution);
+		} else {
+			TraceEvent(SevError, "WaitStorageMetricsError").error(err);
+			throw err;
 		}
 	}
 }
@@ -6000,23 +5983,12 @@ Future<StorageMetrics> DatabaseContext::getStorageMetrics(KeyRange const& keys,
 	}
 }
 
-ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
-                                                                                  KeyRange keys,
-                                                                                  int shardLimit) {
-	loop {
-		choose {
-			when(wait(cx->onProxiesChanged())) {}
-			when(ErrorOr<GetDDMetricsReply> rep =
-			         wait(errorOr(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False),
-			                                       &CommitProxyInterface::getDDMetrics,
-			                                       GetDDMetricsRequest(keys, shardLimit))))) {
-				if (rep.isError()) {
-					throw rep.getError();
-				}
-				return rep.get().storageMetricsList;
-			}
-		}
-	}
+Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
+                                                                            KeyRange keys,
+                                                                            int shardLimit) {
+	GetDDMetricsReply rep = co_await commitProxyLoadBalance(
+	    cx, makeReqBuilder<GetDDMetricsRequest>(keys, shardLimit), &CommitProxyInterface::getDDMetrics);
+	co_return rep.storageMetricsList;
 }
 
 Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> DatabaseContext::getReadHotRanges(KeyRange const& keys) {
@@ -6081,10 +6053,10 @@ Future<Standalone<VectorRef<KeyRef>>> Transaction::getRangeSplitPoints(KeyRange
 	return ::getRangeSplitPoints(trState, keys, chunkSize);
 }
 
-ACTOR Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) {
-	state ReadYourWritesTransaction tr(cx);
-	state Version version = invalidVersion;
-	loop {
+Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) {
+	ReadYourWritesTransaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			if (lockAware) {
@@ -6092,42 +6064,39 @@ ACTOR Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAw
 			}
 
 			tr.set(perpetualStorageWiggleKey, enable ? "1"_sr : "0"_sr);
-			wait(tr.commit());
-			version = tr.getCommittedVersion();
-			break;
+			co_await tr.commit();
+			co_return tr.getCommittedVersion();
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return version;
 }
 
-ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
-                                                                                      bool primary,
-                                                                                      bool use_system_priority) {
-	state StorageWiggleData wiggleState;
-	state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap =
-	    wiggleState.wigglingStorageServer(PrimaryRegion(primary));
+Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
+                                                                                bool primary,
+                                                                                bool use_system_priority) {
+	StorageWiggleData wiggleState;
+	auto metadataMap = wiggleState.wigglingStorageServer(PrimaryRegion(primary));
+	auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state KeyBackedRangeResult<std::pair<UID, StorageWiggleValue>> res;
-
-	// read the wiggling pairs
-	loop {
+	while (true) {
+		Error err;
 		try {
+			KeyBackedRangeResult<std::pair<UID, StorageWiggleValue>> res;
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			if (use_system_priority) {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			}
-			wait(store(res, metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY)));
-			wait(tr->commit());
-			break;
+			res = co_await metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY);
+			co_await tr->commit();
+			co_return res.results;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
-	return res.results;
 }
 
 ACTOR Future<Void> splitStorageMetricsStream(PromiseStream<Key> resultStream,
@@ -6383,22 +6352,14 @@ void enableClientInfoLogging() {
 	TraceEvent(SevInfo, "ClientInfoLoggingEnabled").log();
 }
 
-ACTOR Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID) {
+Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID) {
 	TraceEvent("SnapCreateEnter").detail("SnapCmd", snapCmd).detail("UID", snapUID);
 	try {
-		loop {
-			choose {
-				when(wait(cx->onProxiesChanged())) {}
-				when(wait(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False),
-				                           &CommitProxyInterface::proxySnapReq,
-				                           ProxySnapRequest(snapCmd, snapUID, snapUID),
-				                           cx->taskID,
-				                           AtMostOnce::True))) {
-					TraceEvent("SnapCreateExit").detail("SnapCmd", snapCmd).detail("UID", snapUID);
-					return Void();
-				}
-			}
-		}
+		co_await commitProxyLoadBalance(cx,
+		                                makeReqBuilder<ProxySnapRequest>(snapCmd, snapUID, snapUID),
+		                                &CommitProxyInterface::proxySnapReq,
+		                                AtMostOnce::True);
+		TraceEvent("SnapCreateExit").detail("SnapCmd", snapCmd).detail("UID", snapUID);
 	} catch (Error& e) {
 		TraceEvent("SnapCreateError").error(e).detail("SnapCmd", snapCmd.toString()).detail("UID", snapUID);
 		throw;
@@ -6470,20 +6431,20 @@ Future<Void> createCheckpoint(Transaction* tr,
 
 // Gets CheckpointMetaData of the specific keyrange, version and format from one of the storage servers, if none of the
 // servers have the checkpoint, a checkpoint_not_found error is returned.
-ACTOR static Future<CheckpointMetaData> getCheckpointMetaDataInternal(KeyRange range,
-                                                                      Version version,
-                                                                      CheckpointFormat format,
-                                                                      Optional<UID> actionId,
-                                                                      Reference<LocationInfo> alternatives,
-                                                                      double timeout) {
+static Future<CheckpointMetaData> getCheckpointMetaDataInternal(KeyRange range,
+                                                                Version version,
+                                                                CheckpointFormat format,
+                                                                Optional<UID> actionId,
+                                                                Reference<LocationInfo> alternatives,
+                                                                double timeout) {
 	TraceEvent(SevDebug, "GetCheckpointMetaDataInternalBegin")
 	    .detail("Range", range)
 	    .detail("Version", version)
 	    .detail("Format", static_cast<int>(format))
 	    .detail("Locations", alternatives->description());
 
-	state std::vector<Future<ErrorOr<CheckpointMetaData>>> futures;
-	state int index = 0;
+	std::vector<Future<ErrorOr<CheckpointMetaData>>> futures;
+	int index = 0;
 	for (index = 0; index < alternatives->size(); ++index) {
 		// For each shard, all storage servers are checked, only one is required.
 		futures.push_back(errorOr(timeoutError(alternatives->getInterface(index).checkpoint.getReply(
@@ -6491,8 +6452,8 @@ ACTOR static Future<CheckpointMetaData> getCheckpointMetaDataInternal(KeyRange r
 		                                       timeout)));
 	}
 
-	state Optional<Error> error;
-	wait(waitForAll(futures));
+	Optional<Error> error;
+	co_await waitForAll(futures);
 	TraceEvent(SevDebug, "GetCheckpointMetaDataInternalWaitEnd").detail("Range", range).detail("Version", version);
 
 	for (index = 0; index < futures.size(); ++index) {
@@ -6516,7 +6477,7 @@ ACTOR static Future<CheckpointMetaData> getCheckpointMetaDataInternal(KeyRange r
 				error = e;
 			}
 		} else {
-			return futures[index].get().get();
+			co_return futures[index].get().get();
 		}
 	}
 
@@ -6598,20 +6559,20 @@ ACTOR static Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getChe
 	return res;
 }
 
-ACTOR Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpointMetaData(Database cx,
-                                                                                         std::vector<KeyRange> ranges,
-                                                                                         Version version,
-                                                                                         CheckpointFormat format,
-                                                                                         Optional<UID> actionId,
-                                                                                         double timeout) {
-	state std::vector<Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>>> futures;
+Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpointMetaData(Database cx,
+                                                                                   std::vector<KeyRange> ranges,
+                                                                                   Version version,
+                                                                                   CheckpointFormat format,
+                                                                                   Optional<UID> actionId,
+                                                                                   double timeout) {
+	std::vector<Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>>> futures;
 
 	// TODO(heliu): Avoid send requests to the same shard.
 	for (const auto& range : ranges) {
 		futures.push_back(getCheckpointMetaDataForRange(cx, range, version, format, actionId, timeout));
 	}
 
-	std::vector<std::vector<std::pair<KeyRange, CheckpointMetaData>>> results = wait(getAll(futures));
+	std::vector<std::vector<std::pair<KeyRange, CheckpointMetaData>>> results = co_await getAll(futures);
 
 	std::vector<std::pair<KeyRange, CheckpointMetaData>> res;
 
@@ -6620,28 +6581,20 @@ ACTOR Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpoint
 		res.insert(res.end(), r.begin(), r.end());
 	}
 
-	return res;
+	co_return res;
 }
 
-ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions) {
+Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions) {
 	TraceEvent("ExclusionSafetyCheckBegin")
 	    .detail("NumExclusion", exclusions.size())
 	    .detail("Exclusions", describe(exclusions));
-	state bool ddCheck;
+	bool ddCheck{ false };
 	try {
-		loop {
-			choose {
-				when(wait(cx->onProxiesChanged())) {}
-				when(ExclusionSafetyCheckReply _ddCheck =
-				         wait(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False),
-				                               &CommitProxyInterface::exclusionSafetyCheckReq,
-				                               ExclusionSafetyCheckRequest(exclusions),
-				                               cx->taskID))) {
-					ddCheck = _ddCheck.safe;
-					break;
-				}
-			}
-		}
+		ExclusionSafetyCheckReply _ddCheck =
+		    co_await commitProxyLoadBalance(cx,
+		                                    makeReqBuilder<ExclusionSafetyCheckRequest>(exclusions),
+		                                    &CommitProxyInterface::exclusionSafetyCheckReq);
+		ddCheck = _ddCheck.safe;
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled) {
 			TraceEvent("ExclusionSafetyCheckError")
@@ -6652,32 +6605,30 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 		throw;
 	}
 	TraceEvent("ExclusionSafetyCheckCoordinators").log();
-	state ClientCoordinators coordinatorList(cx->getConnectionRecord());
-	state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
+	ClientCoordinators coordinatorList(cx->getConnectionRecord());
+	std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 	leaderServers.reserve(coordinatorList.clientLeaderServers.size());
-	for (int i = 0; i < coordinatorList.clientLeaderServers.size(); i++) {
-		if (coordinatorList.clientLeaderServers[i].hostname.present()) {
+	for (const auto& clientLeaderServer : coordinatorList.clientLeaderServers) {
+		if (clientLeaderServer.hostname.present()) {
 			leaderServers.push_back(retryGetReplyFromHostname(GetLeaderRequest(coordinatorList.clusterKey, UID()),
-			                                                  coordinatorList.clientLeaderServers[i].hostname.get(),
+			                                                  clientLeaderServer.hostname.get(),
 			                                                  WLTOKEN_CLIENTLEADERREG_GETLEADER,
 			                                                  TaskPriority::CoordinationReply));
 		} else {
-			leaderServers.push_back(retryBrokenPromise(coordinatorList.clientLeaderServers[i].getLeader,
+			leaderServers.push_back(retryBrokenPromise(clientLeaderServer.getLeader,
 			                                           GetLeaderRequest(coordinatorList.clusterKey, UID()),
 			                                           TaskPriority::CoordinationReply));
 		}
 	}
 	// Wait for quorum so we don't dismiss live coordinators as unreachable by acting too fast
-	choose {
-		when(wait(smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.0))) {}
-		when(wait(delay(3.0))) {
-			TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum").log();
-			return false;
-		}
+	auto res = co_await race(smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.0), delay(3.0));
+	if (res.index() == 1) {
+		TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum").log();
+		co_return false;
 	}
 	int attemptCoordinatorExclude = 0;
 	int coordinatorsUnavailable = 0;
-	for (int i = 0; i < leaderServers.size(); i++) {
+	for (int i = 0; i < leaderServers.size(); ++i) {
 		NetworkAddress leaderAddress =
 		    coordinatorList.clientLeaderServers[i].getLeader.getEndpoint().getPrimaryAddress();
 		if (leaderServers[i].isReady()) {
@@ -6700,7 +6651,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 	    .detail("CoordinatorCheck", coordinatorCheck)
 	    .detail("DataDistributorCheck", ddCheck);
 
-	return (ddCheck && coordinatorCheck);
+	co_return ddCheck&& coordinatorCheck;
 }
 
 // returns true if we can connect to the given worker interface
@@ -6720,20 +6671,19 @@ ACTOR Future<bool> verifyInterfaceActor(Reference<FlowLock> connectLock, ClientW
 	}
 }
 
-ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
+static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
 	// ignore negative value
 	if (duration < 0)
 		duration = 0;
 	if (!cx->getConnectionRecord())
-		return 0;
+		co_return 0;
 	// fetch all workers' addresses and interfaces from CC
-	RangeResult kvs = wait(getWorkerInterfaces(cx->getConnectionRecord()));
+	RangeResult kvs = co_await getWorkerInterfaces(cx->getConnectionRecord());
 	ASSERT(!kvs.more);
 	// map worker network address to its interface
-	state std::map<Key, ClientWorkerInterface> workerInterfaces;
+	std::map<Key, ClientWorkerInterface> workerInterfaces;
 	for (const auto& it : kvs) {
-		ClientWorkerInterface workerInterf =
-		    BinaryReader::fromStringRef<ClientWorkerInterface>(it.value, IncludeVersion());
+		auto workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(it.value, IncludeVersion());
 		Key primaryAddress = it.key.endsWith(":tls"_sr) ? it.key.removeSuffix(":tls"_sr) : it.key;
 		workerInterfaces[primaryAddress] = workerInterf;
 		// Also add mapping from a worker's second address(if present) to its interface
@@ -6745,30 +6695,30 @@ ACTOR static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef add
 		}
 	}
 	// split and get all the requested addresses to send reboot requests
-	state std::vector<std::string> addressesVec;
+	std::vector<std::string> addressesVec;
 	boost::algorithm::split(addressesVec, addr.toString(), boost::is_any_of(","));
 	// Note: reuse this knob from fdbcli, change it if necessary
 	Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
-	state std::vector<Future<bool>> verifyInterfs;
+	std::vector<Future<bool>> verifyInterfs;
 	for (const auto& requestedAddress : addressesVec) {
 		// step 1: check that the requested address is in the worker list provided by CC
 		if (!workerInterfaces.count(Key(requestedAddress)))
-			return 0;
+			co_return 0;
 		// step 2: try to establish connections to the requested worker
 		verifyInterfs.push_back(verifyInterfaceActor(connectLock, workerInterfaces[Key(requestedAddress)]));
 	}
 	// step 3: check if we can establish connections to all requested workers, return if not
-	wait(waitForAll(verifyInterfs));
+	co_await waitForAll(verifyInterfs);
 	for (const auto& f : verifyInterfs) {
 		if (!f.get())
-			return 0;
+			co_return 0;
 	}
 	// step 4: After verifying we can connect to all requested workers, send reboot requests together
 	for (const auto& address : addressesVec) {
 		// Note: We want to make sure these requests are sent in parallel
 		workerInterfaces[Key(address)].reboot.send(RebootRequest(false, check, duration));
 	}
-	return 1;
+	co_return 1;
 }
 
 Future<int64_t> DatabaseContext::rebootWorker(StringRef addr, bool check, int duration) {
@@ -6779,9 +6729,8 @@ Future<Void> DatabaseContext::forceRecoveryWithDataLoss(StringRef dcId) {
 	return forceRecovery(getConnectionRecord(), dcId);
 }
 
-ACTOR static Future<Void> createSnapshotActor(DatabaseContext* cx, UID snapUID, StringRef snapCmd) {
-	wait(mgmtSnapCreate(cx->clone(), snapCmd, snapUID));
-	return Void();
+static Future<Void> createSnapshotActor(DatabaseContext* cx, UID snapUID, StringRef snapCmd) {
+	co_await mgmtSnapCreate(cx->clone(), snapCmd, snapUID);
 }
 
 Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_command) {
@@ -6802,7 +6751,7 @@ void sharedStateDelRef(DatabaseSharedState* ssPtr) {
 
 Future<DatabaseSharedState*> DatabaseContext::initSharedState() {
 	ASSERT(!sharedStatePtr); // Don't re-initialize shared state if a pointer already exists
-	DatabaseSharedState* newState = new DatabaseSharedState();
+	auto* newState = new DatabaseSharedState();
 	// Increment refcount by 1 on creation to account for the one held in MultiVersionApi map
 	// Therefore, on initialization, refCount should be 2 (after also going to setSharedState)
 	newState->refCount++;
@@ -6821,17 +6770,18 @@ Reference<DatabaseContext::TransactionT> DatabaseContext::createTransaction() {
 	return makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(this)));
 }
 
-ACTOR static Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>>
-getHotRangeMetricsActor(Reference<DatabaseContext> db, StorageServerInterface ssi, ReadHotSubRangeRequest req) {
+static Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getHotRangeMetricsActor(Reference<DatabaseContext> db,
+                                                                                      StorageServerInterface ssi,
+                                                                                      ReadHotSubRangeRequest req) {
 
-	ErrorOr<ReadHotSubRangeReply> fs = wait(ssi.getReadHotRanges.tryGetReply(req));
+	ErrorOr<ReadHotSubRangeReply> fs = co_await ssi.getReadHotRanges.tryGetReply(req);
 	if (fs.isError()) {
 		fmt::print("Error({}): cannot get read hot metrics from storage server {}.\n",
 		           fs.getError().what(),
 		           ssi.address().toString());
-		return Standalone<VectorRef<ReadHotRangeWithMetrics>>();
+		co_return Standalone<VectorRef<ReadHotRangeWithMetrics>>();
 	} else {
-		return fs.get().readHotRanges;
+		co_return fs.get().readHotRanges;
 	}
 }
 
@@ -6863,24 +6813,23 @@ int64_t getMaxClearKeySize(KeyRef const& key) {
 
 namespace NativeAPI {
 
-ACTOR Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(
-    Transaction* tr) {
-	state Future<std::vector<ProcessData>> workers = getWorkers(tr);
-	state Future<RangeResult> serverList = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-	wait(success(workers) && success(serverList));
+Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(Transaction* tr) {
+	Future<std::vector<ProcessData>> workers = getWorkers(tr);
+	Future<RangeResult> serverList = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+	co_await (success(workers) && success(serverList));
 	ASSERT(!serverList.get().more && serverList.get().size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::map<Optional<Standalone<StringRef>>, ProcessData> id_data;
-	for (int i = 0; i < workers.get().size(); i++)
-		id_data[workers.get()[i].locality.processId()] = workers.get()[i];
+	for (const auto& worker : workers.get())
+		id_data[worker.locality.processId()] = worker;
 
 	std::vector<std::pair<StorageServerInterface, ProcessClass>> results;
-	for (int i = 0; i < serverList.get().size(); i++) {
-		auto ssi = decodeServerListValue(serverList.get()[i].value);
+	for (const auto& server : serverList.get()) {
+		auto ssi = decodeServerListValue(server.value);
 		results.emplace_back(ssi, id_data[ssi.locality.processId()].processClass);
 	}
 
-	return results;
+	co_return results;
 }
 
 } // namespace NativeAPI
diff --git a/fdbclient/ParallelStream.actor.cpp b/fdbclient/ParallelStream.actor.cpp
deleted file mode 100644
index 147c66f0161..00000000000
--- a/fdbclient/ParallelStream.actor.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * ParallelStreamCorrectness.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vector>
-
-#include "fdbclient/ParallelStream.actor.h"
-#include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-namespace ParallelStreamTest {
-
-struct TestValue {
-	int x;
-	TestValue(int x) : x(x) {}
-	int expectedSize() const { return sizeof(int); }
-};
-
-ACTOR static Future<Void> produce(ParallelStream<ParallelStreamTest::TestValue>::Fragment* fragment,
-                                  ParallelStreamTest::TestValue value) {
-	wait(delay(deterministicRandom()->random01()));
-	fragment->send(value);
-	wait(delay(deterministicRandom()->random01()));
-	fragment->finish();
-	return Void();
-}
-
-ACTOR static Future<Void> consume(FutureStream<ParallelStreamTest::TestValue> stream, int expected) {
-	state int next = 0;
-	try {
-		loop {
-			ParallelStreamTest::TestValue value = waitNext(stream);
-			ASSERT(value.x == next++);
-		}
-	} catch (Error& e) {
-		ASSERT(e.code() == error_code_end_of_stream);
-		ASSERT(next == expected);
-		return Void();
-	}
-}
-
-} // namespace ParallelStreamTest
-
-TEST_CASE("/fdbclient/ParallelStream") {
-	state PromiseStream<ParallelStreamTest::TestValue> results;
-	state size_t bufferLimit = deterministicRandom()->randomInt(0, 21);
-	state size_t numProducers = deterministicRandom()->randomInt(1, 1001);
-	state ParallelStream<ParallelStreamTest::TestValue> parallelStream(results, bufferLimit);
-	state Future<Void> consumer = ParallelStreamTest::consume(results.getFuture(), numProducers);
-	state std::vector<Future<Void>> producers;
-	TraceEvent("StartingParallelStreamTest").detail("BufferLimit", bufferLimit).detail("NumProducers", numProducers);
-	state int i = 0;
-	for (; i < numProducers; ++i) {
-		ParallelStream<ParallelStreamTest::TestValue>::Fragment* fragment = wait(parallelStream.createFragment());
-		producers.push_back(ParallelStreamTest::produce(fragment, ParallelStreamTest::TestValue(i)));
-	}
-	wait(parallelStream.finish());
-	wait(consumer);
-	return Void();
-}
-
-void forceLinkParallelStreamTests() {}
diff --git a/fdbclient/include/fdbclient/PartitionedLogIterator.h b/fdbclient/PartitionedLogIterator.h
similarity index 99%
rename from fdbclient/include/fdbclient/PartitionedLogIterator.h
rename to fdbclient/PartitionedLogIterator.h
index 03c6f6d194a..67e76f1698c 100644
--- a/fdbclient/include/fdbclient/PartitionedLogIterator.h
+++ b/fdbclient/PartitionedLogIterator.h
@@ -25,4 +25,4 @@ class PartitionedLogIterator : public ReferenceCounted<PartitionedLogIterator> {
 
 	virtual ~PartitionedLogIterator() = default;
 };
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbclient/ProxyLoadBalance.h b/fdbclient/ProxyLoadBalance.h
new file mode 100644
index 00000000000..895a5635eb2
--- /dev/null
+++ b/fdbclient/ProxyLoadBalance.h
@@ -0,0 +1,95 @@
+/*
+ * ProxyLoadBalance.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+#include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/GrvProxyInterface.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbrpc/LoadBalance.actor.h"
+
+// Stores constructor arguments so a request can be rebuilt on each retry.
+template <class Req, class... Args>
+class ReqBuilder {
+	std::tuple<Args...> args;
+
+public:
+	explicit ReqBuilder(Args... args) : args(std::move(args)...) {}
+
+	Req build() const {
+		return std::apply([](auto const&... unpackedArgs) { return Req(unpackedArgs...); }, args);
+	}
+};
+
+// Infers the stored argument types while ensuring the request is constructible.
+template <class Req, class... Args>
+ReqBuilder<Req, std::decay_t<Args>...> makeReqBuilder(Args&&... args) {
+	static_assert(std::is_constructible_v<Req, std::decay_t<Args>...>);
+	return ReqBuilder<Req, std::decay_t<Args>...>(std::forward<Args>(args)...);
+}
+
+// Retries a commit-proxy request whenever the proxy set changes before a reply arrives.
+template <class Req, class Builder, bool IsPublicStream>
+AsyncResult<REPLY_TYPE(Req)> commitProxyLoadBalance(Database cx,
+                                                    Builder reqBuilder,
+                                                    RequestStream<Req, IsPublicStream> CommitProxyInterface::* channel,
+                                                    UseProvisionalProxies useProvisionalProxies,
+                                                    TaskPriority taskID,
+                                                    AtMostOnce atMostOnce = AtMostOnce::False,
+                                                    ExplicitVoid = {}) {
+	while (true) {
+		Future<REPLY_TYPE(Req)> replyFuture = basicLoadBalance(
+		    cx->getCommitProxies(useProvisionalProxies), channel, reqBuilder.build(), taskID, atMostOnce);
+		auto res = co_await race(replyFuture, cx->onProxiesChanged());
+		if (res.index() == 0) {
+			co_return std::get<0>(std::move(res));
+		}
+	}
+}
+
+template <class Req, class Builder, bool IsPublicStream>
+AsyncResult<REPLY_TYPE(Req)> commitProxyLoadBalance(Database cx,
+                                                    Builder reqBuilder,
+                                                    RequestStream<Req, IsPublicStream> CommitProxyInterface::* channel,
+                                                    AtMostOnce atMostOnce = AtMostOnce::False,
+                                                    ExplicitVoid = {}) {
+	return commitProxyLoadBalance(cx, reqBuilder, channel, UseProvisionalProxies::False, cx->taskID, atMostOnce);
+}
+
+// Retries a GRV-proxy request whenever the proxy set changes before a reply arrives.
+template <class Req, class Builder, bool IsPublicStream>
+AsyncResult<REPLY_TYPE(Req)> grvProxyLoadBalance(Database cx,
+                                                 Builder reqBuilder,
+                                                 RequestStream<Req, IsPublicStream> GrvProxyInterface::* channel,
+                                                 AtMostOnce atMostOnce = AtMostOnce::False,
+                                                 ExplicitVoid = {}) {
+	while (true) {
+		Future<REPLY_TYPE(Req)> replyFuture = basicLoadBalance(
+		    cx->getGrvProxies(UseProvisionalProxies::False), channel, reqBuilder.build(), cx->taskID, atMostOnce);
+		auto res = co_await race(replyFuture, cx->onProxiesChanged());
+		if (res.index() == 0) {
+			co_return std::get<0>(std::move(res));
+		}
+	}
+}
diff --git a/fdbclient/RESTClient.actor.cpp b/fdbclient/RESTClient.cpp
similarity index 88%
rename from fdbclient/RESTClient.actor.cpp
rename to fdbclient/RESTClient.cpp
index 4fa5ffe9614..e46ae6ec101 100644
--- a/fdbclient/RESTClient.actor.cpp
+++ b/fdbclient/RESTClient.cpp
@@ -1,5 +1,5 @@
 /*
- * RESTClient.actor.cpp
+ * RESTClient.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbclient/RESTClient.h"
+#include "RESTClient.h"
 
 #include "fdbrpc/HTTP.h"
 #include "flow/IRateControl.h"
-#include "fdbclient/RESTUtils.h"
+#include "RESTUtils.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
@@ -38,8 +38,6 @@
 #include <memory>
 #include <unordered_map>
 
-#include "flow/actorcompiler.h" // always the last include
-
 #define TRACE_REST_OP(opName, url)                                                                                     \
 	do {                                                                                                               \
 		if (FLOW_KNOBS->REST_LOG_LEVEL >= RESTLogSeverity::DEBUG) {                                                    \
@@ -94,14 +92,14 @@ bool isErrorRetryable(const Error& e) {
 	return e.code() != error_code_timed_out && e.code() != error_code_connection_failed;
 }
 
-ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTClient> client,
-                                                               std::string verb,
-                                                               HTTP::Headers headers,
-                                                               RESTUrl url,
-                                                               std::set<unsigned int> successCodes) {
+Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTClient> client,
+                                                         std::string verb,
+                                                         HTTP::Headers headers,
+                                                         RESTUrl url,
+                                                         std::set<unsigned int> successCodes) {
 
-	state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
-	state UnsentPacketQueue content;
+	auto req = makeReference<HTTP::OutgoingRequest>();
+	UnsentPacketQueue content;
 	req->data.content = &content;
 	req->data.contentLen = url.body.size();
 	req->data.headers = headers;
@@ -113,7 +111,7 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTCli
 		TraceEvent("RESTDoRequestImpl").detail("Url", url.toString());
 	}
 
-	if (url.body.size() > 0) {
+	if (!url.body.empty()) {
 		PacketWriter pw(req->data.content->getWriteBuffer(url.body.size()), nullptr, Unversioned());
 		pw.serializeBytes(url.body);
 	}
@@ -124,21 +122,21 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTCli
 		client->statsMap.emplace(statsKey, std::make_unique<RESTClient::Stats>(statsKey));
 	}
 
-	state int maxTries = std::min(client->knobs.request_tries, client->knobs.connect_tries);
-	state int thisTry = 1;
-	state double nextRetryDelay = 2.0;
-	state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
-	state double reqTimeout = (client->knobs.request_timeout_secs * 1.0);
-	state RESTConnectionPoolKey connectPoolKey = RESTConnectionPool::getConnectionPoolKey(url.host, url.service);
-	state RESTClient::Stats* statsPtr = client->statsMap[statsKey].get();
+	int maxTries = std::min(client->knobs.request_tries, client->knobs.connect_tries);
+	int thisTry = 1;
+	double nextRetryDelay = 2.0;
+	Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+	double reqTimeout = (client->knobs.request_timeout_secs * 1.0);
+	RESTConnectionPoolKey connectPoolKey = RESTConnectionPool::getConnectionPoolKey(url.host, url.service);
+	RESTClient::Stats* statsPtr = client->statsMap[statsKey].get();
 
-	loop {
-		state Optional<Error> err;
-		state Optional<NetworkAddress> remoteAddress;
-		state bool connectionEstablished = false;
+	while (true) {
+		Optional<Error> err;
+		Optional<NetworkAddress> remoteAddress;
+		bool connectionEstablished = false;
 
-		state Reference<HTTP::IncomingResponse> r;
-		state RESTConnectionPool::ReusableConnection rconn;
+		Reference<HTTP::IncomingResponse> r;
+		RESTConnectionPool::ReusableConnection rconn;
 
 		try {
 			// Start connecting
@@ -146,14 +144,12 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTCli
 			    client->conectionPool->connect(connectPoolKey, url.connType.secure, client->knobs.max_connection_life);
 
 			// Finish connecting, do request
-			wait(store(rconn, timeoutError(frconn, client->knobs.connect_timeout)));
+			rconn = co_await timeoutError(frconn, client->knobs.connect_timeout);
 			connectionEstablished = true;
 
 			remoteAddress = rconn.conn->getPeerAddress();
-			wait(store(
-			    r,
-			    timeoutError(HTTP::doRequest(rconn.conn, req, sendReceiveRate, &statsPtr->bytes_sent, sendReceiveRate),
-			                 reqTimeout)));
+			r = co_await timeoutError(
+			    HTTP::doRequest(rconn.conn, req, sendReceiveRate, &statsPtr->bytes_sent, sendReceiveRate), reqTimeout);
 
 			// Since the response was parsed successfully (which is why we are here) reuse the connection unless we
 			// received the "Connection: close" header.
@@ -178,9 +174,9 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTCli
 
 		// If err is not present then r is valid.
 		// If r->code is in successCodes then record the successful request and return r.
-		if (!err.present() && successCodes.count(r->code) != 0) {
+		if (!err.present() && successCodes.contains(r->code)) {
 			statsPtr->requests_successful++;
-			return r;
+			co_return r;
 		}
 
 		// Otherwise, this request is considered failed.  Update failure count.
@@ -249,7 +245,7 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<RESTCli
 
 			// Log the delay then wait.
 			event.detail("RetryDelay", delay);
-			wait(::delay(delay));
+			co_await ::delay(delay);
 		} else {
 			// We can't retry, so throw something.
 
diff --git a/fdbclient/include/fdbclient/RESTClient.h b/fdbclient/RESTClient.h
similarity index 99%
rename from fdbclient/include/fdbclient/RESTClient.h
rename to fdbclient/RESTClient.h
index 7434da16474..011c86470c6 100644
--- a/fdbclient/include/fdbclient/RESTClient.h
+++ b/fdbclient/RESTClient.h
@@ -25,7 +25,7 @@
 
 #include "fdbclient/JSONDoc.h"
 #include "fdbrpc/HTTP.h"
-#include "fdbclient/RESTUtils.h"
+#include "RESTUtils.h"
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 #include "flow/flow.h"
diff --git a/fdbclient/RESTUtils.actor.cpp b/fdbclient/RESTUtils.cpp
similarity index 93%
rename from fdbclient/RESTUtils.actor.cpp
rename to fdbclient/RESTUtils.cpp
index 4e632b1ff6d..db0517a0d09 100644
--- a/fdbclient/RESTUtils.actor.cpp
+++ b/fdbclient/RESTUtils.cpp
@@ -1,5 +1,5 @@
 /*
- * RESTUtils.actor.cpp
+ * RESTUtils.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbclient/RESTUtils.h"
+#include "RESTUtils.h"
 #include "fdbclient/Knobs.h"
 
 #include "flow/flat_buffers.h"
@@ -28,8 +28,6 @@
 #include <boost/algorithm/string.hpp>
 #include <queue>
 
-#include "flow/actorcompiler.h" // always the last include
-
 // RESTConnectionPool destructor implementation
 RESTConnectionPool::~RESTConnectionPool() {
 	// In simulation, explicitly close all pooled connections before destruction.
@@ -125,10 +123,10 @@ std::unordered_map<std::string, int> RESTClientKnobs::get() const {
 	return details;
 }
 
-ACTOR Future<RESTConnectionPool::ReusableConnection> connect_impl(Reference<RESTConnectionPool> connectionPool,
-                                                                  RESTConnectionPoolKey connectKey,
-                                                                  bool isSecure,
-                                                                  int maxConnLife) {
+Future<RESTConnectionPool::ReusableConnection> connect_impl(Reference<RESTConnectionPool> connectionPool,
+                                                            RESTConnectionPoolKey connectKey,
+                                                            bool isSecure,
+                                                            int maxConnLife) {
 
 	if (FLOW_KNOBS->REST_LOG_LEVEL >= RESTLogSeverity::VERBOSE) {
 		TraceEvent("RESTUtilConnectStart")
@@ -152,16 +150,16 @@ ACTOR Future<RESTConnectionPool::ReusableConnection> connect_impl(Reference<REST
 				    .detail("ExpireIn", rconn.expirationTime - now())
 				    .detail("NumConnsInPool", poolItr->second.size());
 			}
-			return rconn;
+			co_return rconn;
 		}
 	}
 
 	ASSERT(poolItr == connectionPool->connectionPoolMap.end() || poolItr->second.empty());
 
 	// No valid connection exists, create a new one
-	state Reference<IConnection> conn =
-	    wait(INetworkConnections::net()->connect(connectKey.first, connectKey.second, isSecure));
-	wait(conn->connectHandshake());
+	Reference<IConnection> conn =
+	    co_await INetworkConnections::net()->connect(connectKey.first, connectKey.second, isSecure);
+	co_await conn->connectHandshake();
 
 	TraceEvent("RESTTUilCreateNewConn")
 	    .suppressFor(60)
@@ -170,7 +168,7 @@ ACTOR Future<RESTConnectionPool::ReusableConnection> connect_impl(Reference<REST
 	    .detail("RemoteEndpoint", conn->getPeerAddress())
 	    .detail("ConnPoolSize", connectionPool->connectionPoolMap.size());
 
-	return RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife });
+	co_return RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife });
 }
 
 Future<RESTConnectionPool::ReusableConnection> RESTConnectionPool::connect(RESTConnectionPoolKey connectKey,
@@ -252,7 +250,7 @@ void RESTUrl::parseUrl(const std::string& fullUrl) {
 		// hostPort is at least a host or IP address, optionally followed by :portNumber or :serviceName
 		StringRef hRef(hostPort);
 		StringRef h = hRef.eat(":");
-		if (h.size() == 0) {
+		if (h.empty()) {
 			CODE_PROBE(true, "REST URI empty host");
 			throw std::string("host cannot be empty");
 		}
diff --git a/fdbclient/include/fdbclient/RESTUtils.h b/fdbclient/RESTUtils.h
similarity index 97%
rename from fdbclient/include/fdbclient/RESTUtils.h
rename to fdbclient/RESTUtils.h
index 08ab804cfba..af8c65f4b7a 100644
--- a/fdbclient/include/fdbclient/RESTUtils.h
+++ b/fdbclient/RESTUtils.h
@@ -59,9 +59,9 @@ class RESTConnectionPool : public ReferenceCounted<RESTConnectionPool> {
 	std::unordered_map<RESTConnectionPoolKey, std::queue<ReusableConnection>, boost::hash<RESTConnectionPoolKey>>
 	    connectionPoolMap;
 
-	RESTConnectionPool(const int maxConnsPerKey) : maxConnPerConnectKey(maxConnsPerKey) {}
+	explicit RESTConnectionPool(const int maxConnsPerKey) : maxConnPerConnectKey(maxConnsPerKey) {}
 
-	// Destructor implementation in RESTUtils.actor.cpp
+	// Destructor implementation in RESTUtils.cpp
 	// In simulation, explicitly closes all pooled connections before destruction
 	~RESTConnectionPool();
 
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index 196a22d5427..8483b287766 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -22,9 +22,10 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/Atomic.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/SpecialKeySpace.h"
 #include "fdbclient/StatusClient.h"
 #include "fdbclient/MonitorLeader.h"
+#include "flow/CoroUtils.h"
 #include "flow/Util.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -57,13 +58,13 @@ class RYWImpl {
 	struct GetValueReq {
 		explicit GetValueReq(Key key) : key(key) {}
 		Key key;
-		typedef Optional<Value> Result;
+		using Result = Optional<Value>;
 	};
 
 	struct GetKeyReq {
 		explicit GetKeyReq(KeySelector key) : key(key) {}
 		KeySelector key;
-		typedef Key Result;
+		using Result = Key;
 	};
 
 	template <bool reverse>
@@ -91,7 +92,7 @@ class RYWImpl {
 	// to update the given *it to point to the key that was read, so that the corresponding overload of
 	// addConflictRange() can make use of it.
 
-	ACTOR template <class Iter>
+	template <class Iter>
 	static Future<Optional<Value>> read(ReadYourWritesTransaction* ryw, GetValueReq read, Iter* it) {
 		// This overload is required to provide postcondition: it->extractWriteMapIterator().segmentContains(read.key)
 
@@ -99,29 +100,29 @@ class RYWImpl {
 			it->bypassUnreadableProtection();
 		}
 		it->skip(read.key);
-		state bool dependent = it->is_dependent();
+		bool dependent = it->is_dependent();
 		if (it->is_kv()) {
 			const KeyValueRef* result = it->kv(ryw->arena);
 			if (result != nullptr) {
-				return result->value;
+				co_return result->value;
 			} else {
-				return Optional<Value>();
+				co_return Optional<Value>();
 			}
 		} else if (it->is_empty_range()) {
-			return Optional<Value>();
+			co_return Optional<Value>();
 		} else {
-			Optional<Value> res = wait(ryw->tr.get(read.key, Snapshot::True));
+			Optional<Value> res = co_await ryw->tr.get(read.key, Snapshot::True);
 			KeyRef k(ryw->arena, read.key);
 
 			if (res.present()) {
 				if (ryw->cache.insert(k, res.get()))
 					ryw->arena.dependsOn(res.get().arena());
 				if (!dependent)
-					return res;
+					co_return res;
 			} else {
 				ryw->cache.insert(k, Optional<ValueRef>());
 				if (!dependent)
-					return Optional<Value>();
+					co_return Optional<Value>();
 			}
 
 			// There was a dependent write at the key, so we need to lookup the iterator again
@@ -130,32 +131,32 @@ class RYWImpl {
 			ASSERT(it->is_kv());
 			const KeyValueRef* result = it->kv(ryw->arena);
 			if (result != nullptr) {
-				return result->value;
+				co_return result->value;
 			} else {
-				return Optional<Value>();
+				co_return Optional<Value>();
 			}
 		}
 	}
 
-	ACTOR template <class Iter>
+	template <class Iter>
 	static Future<Key> read(ReadYourWritesTransaction* ryw, GetKeyReq read, Iter* it) {
 		if (read.key.offset > 0) {
 			RangeResult result =
-			    wait(getRangeValue(ryw, read.key, firstGreaterOrEqual(ryw->getMaxReadKey()), GetRangeLimits(1), it));
+			    co_await getRangeValue(ryw, read.key, firstGreaterOrEqual(ryw->getMaxReadKey()), GetRangeLimits(1), it);
 			if (result.readToBegin)
-				return allKeys.begin;
+				co_return allKeys.begin;
 			if (result.readThroughEnd || !result.size())
-				return ryw->getMaxReadKey();
-			return result[0].key;
+				co_return ryw->getMaxReadKey();
+			co_return result[0].key;
 		} else {
 			read.key.offset++;
 			RangeResult result =
-			    wait(getRangeValueBack(ryw, firstGreaterOrEqual(allKeys.begin), read.key, GetRangeLimits(1), it));
+			    co_await getRangeValueBack(ryw, firstGreaterOrEqual(allKeys.begin), read.key, GetRangeLimits(1), it);
 			if (result.readThroughEnd)
-				return ryw->getMaxReadKey();
+				co_return ryw->getMaxReadKey();
 			if (result.readToBegin || !result.size())
-				return allKeys.begin;
-			return result[0].key;
+				co_return allKeys.begin;
+			co_return result[0].key;
 		}
 	};
 
@@ -177,41 +178,41 @@ class RYWImpl {
 		return ryw->tr.get(read.key, snapshot);
 	}
 
-	ACTOR static Future<Key> readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, Snapshot snapshot) {
-		Key key = wait(ryw->tr.getKey(read.key, snapshot));
+	static Future<Key> readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, Snapshot snapshot) {
+		Key key = co_await ryw->tr.getKey(read.key, snapshot);
 		if (ryw->getMaxReadKey() < key)
-			return ryw->getMaxReadKey(); // Filter out results in the system keys if they are not accessible
-		return key;
+			co_return ryw->getMaxReadKey(); // Filter out results in the system keys if they are not accessible
+		co_return key;
 	}
 
-	ACTOR template <bool backwards>
+	template <bool backwards>
 	static Future<RangeResult> readThrough(ReadYourWritesTransaction* ryw,
 	                                       GetRangeReq<backwards> read,
 	                                       Snapshot snapshot) {
 		if (backwards && read.end.offset > 1) {
 			// FIXME: Optimistically assume that this will not run into the system keys, and only reissue if the result
 			// actually does.
-			Key key = wait(ryw->tr.getKey(read.end, snapshot));
+			Key key = co_await ryw->tr.getKey(read.end, snapshot);
 			if (key > ryw->getMaxReadKey())
 				read.end = firstGreaterOrEqual(ryw->getMaxReadKey());
 			else
 				read.end = KeySelector(firstGreaterOrEqual(key), key.arena());
 		}
 
-		RangeResult v = wait(
-		    ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, backwards ? Reverse::True : Reverse::False));
+		RangeResult v = co_await ryw->tr.getRange(
+		    read.begin, read.end, read.limits, snapshot, backwards ? Reverse::True : Reverse::False);
 		KeyRef maxKey = ryw->getMaxReadKey();
 		if (v.size() > 0) {
 			if (!backwards && v[v.size() - 1].key >= maxKey) {
-				state RangeResult _v = v;
+				RangeResult _v = v;
 				int i = _v.size() - 2;
 				for (; i >= 0 && _v[i].key >= maxKey; --i) {
 				}
-				return RangeResult(RangeResultRef(VectorRef<KeyValueRef>(&_v[0], i + 1), false), _v.arena());
+				co_return RangeResult(RangeResultRef(VectorRef<KeyValueRef>(&_v[0], i + 1), false), _v.arena());
 			}
 		}
 
-		return v;
+		co_return v;
 	}
 
 	// addConflictRange(ryw,read,result) is called after a serializable read and is responsible for adding the relevant
@@ -590,45 +591,45 @@ class RYWImpl {
 	}
 
 	// TODO: read to begin, read through end flags for result
-	ACTOR template <class Iter>
+	template <class Iter>
 	static Future<RangeResult> getRangeValue(ReadYourWritesTransaction* ryw,
 	                                         KeySelector begin,
 	                                         KeySelector end,
 	                                         GetRangeLimits limits,
 	                                         Iter* pit) {
-		state Iter& it(*pit);
-		state Iter itEnd(*pit);
-		state RangeResult result;
-		state int64_t additionalRows = 0;
-		state int itemsPastEnd = 0;
-		state int requestCount = 0;
-		state bool readToBegin = false;
-		state bool readThroughEnd = false;
-		state int actualBeginOffset = begin.offset;
-		state int actualEndOffset = end.offset;
+		Iter& it(*pit);
+		Iter itEnd(*pit);
+		RangeResult result;
+		int64_t additionalRows = 0;
+		int itemsPastEnd = 0;
+		int requestCount = 0;
+		bool readToBegin = false;
+		bool readThroughEnd = false;
+		int actualBeginOffset = begin.offset;
+		int actualEndOffset = end.offset;
 		// state UID randomID = nondeterministicRandom()->randomUniqueID();
 
 		resolveKeySelectorFromCache(begin, it, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualBeginOffset);
 		resolveKeySelectorFromCache(end, itEnd, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualEndOffset);
 
 		if (actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
-			return RangeResultRef(false, false);
+			co_return RangeResultRef(false, false);
 		} else if ((begin.isFirstGreaterOrEqual() && begin.getKey() == ryw->getMaxReadKey()) ||
 		           (end.isFirstGreaterOrEqual() && end.getKey() == allKeys.begin)) {
-			return RangeResultRef(readToBegin, readThroughEnd);
+			co_return RangeResultRef(readToBegin, readThroughEnd);
 		}
 
 		if (!end.isFirstGreaterOrEqual() && begin.getKey() > end.getKey()) {
-			Key resolvedEnd = wait(read(ryw, GetKeyReq(end), pit));
+			Key resolvedEnd = co_await read(ryw, GetKeyReq(end), pit);
 			if (resolvedEnd == allKeys.begin)
 				readToBegin = true;
 			if (resolvedEnd == ryw->getMaxReadKey())
 				readThroughEnd = true;
 
 			if (begin.getKey() >= resolvedEnd && !begin.isBackward()) {
-				return RangeResultRef(false, false);
+				co_return RangeResultRef(false, false);
 			} else if (resolvedEnd == allKeys.begin) {
-				return RangeResultRef(readToBegin, readThroughEnd);
+				co_return RangeResultRef(readToBegin, readThroughEnd);
 			}
 
 			resolveKeySelectorFromCache(
@@ -639,7 +640,7 @@ class RYWImpl {
 
 		//TraceEvent("RYWSelectorsStartForward", randomID).detail("ByteLimit", limits.bytes).detail("RowLimit", limits.rows);
 
-		loop {
+		while (true) {
 			/*TraceEvent("RYWSelectors", randomID).detail("Begin", begin.toString())
 			    .detail("End", end.toString())
 			    .detail("Reached", limits.isReached())
@@ -651,11 +652,11 @@ class RYWImpl {
 			    .detail("Requests", requestCount);*/
 
 			if (!result.size() && actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
-				return RangeResultRef(false, false);
+				co_return RangeResultRef(false, false);
 			}
 
 			if (end.offset <= 1 && end.getKey() == allKeys.begin) {
-				return RangeResultRef(readToBegin, readThroughEnd);
+				co_return RangeResultRef(readToBegin, readThroughEnd);
 			}
 
 			if ((begin.offset >= end.offset && begin.getKey() >= end.getKey()) ||
@@ -664,10 +665,10 @@ class RYWImpl {
 					break;
 				if (!result.size())
 					break;
-				Key resolvedEnd =
-				    wait(read(ryw,
-				              GetKeyReq(end),
-				              pit)); // do not worry about iterator invalidation, because we are breaking for the loop
+				Key resolvedEnd = co_await read(
+				    ryw,
+				    GetKeyReq(end),
+				    pit); // do not worry about iterator invalidation, because we are breaking for the loop
 				if (resolvedEnd == allKeys.begin)
 					readToBegin = true;
 				if (resolvedEnd == ryw->getMaxReadKey())
@@ -679,7 +680,7 @@ class RYWImpl {
 			if (!it.is_unreadable() && !it.is_unknown_range() && it.beginKey() > itEnd.beginKey()) {
 				if (end.isFirstGreaterOrEqual())
 					break;
-				return RangeResultRef(readToBegin, readThroughEnd);
+				co_return RangeResultRef(readToBegin, readThroughEnd);
 			}
 
 			if (limits.isReached() && itemsPastEnd >= 1 - end.offset)
@@ -702,7 +703,7 @@ class RYWImpl {
 				if (it.beginKey() < itEnd.beginKey())
 					singleClears = std::min(skipUncached(ucEnd, itEnd, BUGGIFY ? 0 : clearLimit + 100), clearLimit);
 
-				state KeySelector read_end;
+				KeySelector read_end;
 				if (ucEnd != itEnd) {
 					Key k = ucEnd.endKey().toStandaloneStringRef();
 					read_end = KeySelector(firstGreaterOrEqual(k), k.arena());
@@ -722,7 +723,7 @@ class RYWImpl {
 
 				additionalRows += singleClears;
 
-				state KeySelector read_begin;
+				KeySelector read_begin;
 				if (begin.isFirstGreaterOrEqual()) {
 					Key k = it.beginKey() > begin.getKey() ? it.beginKey().toStandaloneStringRef()
 					                                       : Key(begin.getKey(), begin.arena());
@@ -745,7 +746,7 @@ class RYWImpl {
 					read_end.arena().dependsOn(read_begin.arena());
 				}
 
-				state GetRangeLimits requestLimit = limits;
+				GetRangeLimits requestLimit = limits;
 				setRequestLimits(requestLimit, additionalRows, 2 - read_begin.offset, requestCount);
 				requestCount++;
 
@@ -756,7 +757,7 @@ class RYWImpl {
 
 				additionalRows = 0;
 				RangeResult snapshot_read =
-				    wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::True, Reverse::False));
+				    co_await ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::True, Reverse::False);
 				KeyRangeRef range = getKnownKeyRange(snapshot_read, read_begin, read_end, ryw->arena);
 
 				//TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough);
@@ -810,7 +811,7 @@ class RYWImpl {
 		result.readThroughEnd = !result.more && readThroughEnd;
 		result.arena().dependsOn(ryw->arena);
 
-		return result;
+		co_return result;
 	}
 
 	static KeyRangeRef getKnownKeyRangeBack(RangeResultRef data, KeySelector begin, KeySelector end, Arena& arena) {
@@ -893,22 +894,22 @@ class RYWImpl {
 		return singleEmpty;
 	}
 
-	ACTOR template <class Iter>
+	template <class Iter>
 	static Future<RangeResult> getRangeValueBack(ReadYourWritesTransaction* ryw,
 	                                             KeySelector begin,
 	                                             KeySelector end,
 	                                             GetRangeLimits limits,
 	                                             Iter* pit) {
-		state Iter& it(*pit);
-		state Iter itEnd(*pit);
-		state RangeResult result;
-		state int64_t additionalRows = 0;
-		state int itemsPastBegin = 0;
-		state int requestCount = 0;
-		state bool readToBegin = false;
-		state bool readThroughEnd = false;
-		state int actualBeginOffset = begin.offset;
-		state int actualEndOffset = end.offset;
+		Iter& it(*pit);
+		Iter itEnd(*pit);
+		RangeResult result;
+		int64_t additionalRows = 0;
+		int itemsPastBegin = 0;
+		int requestCount = 0;
+		bool readToBegin = false;
+		bool readThroughEnd = false;
+		int actualBeginOffset = begin.offset;
+		int actualEndOffset = end.offset;
 		// state UID randomID = nondeterministicRandom()->randomUniqueID();
 
 		resolveKeySelectorFromCache(end, it, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualEndOffset);
@@ -916,23 +917,23 @@ class RYWImpl {
 		    begin, itEnd, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualBeginOffset);
 
 		if (actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
-			return RangeResultRef(false, false);
+			co_return RangeResultRef(false, false);
 		} else if ((begin.isFirstGreaterOrEqual() && begin.getKey() == ryw->getMaxReadKey()) ||
 		           (end.isFirstGreaterOrEqual() && end.getKey() == allKeys.begin)) {
-			return RangeResultRef(readToBegin, readThroughEnd);
+			co_return RangeResultRef(readToBegin, readThroughEnd);
 		}
 
 		if (!begin.isFirstGreaterOrEqual() && begin.getKey() > end.getKey()) {
-			Key resolvedBegin = wait(read(ryw, GetKeyReq(begin), pit));
+			Key resolvedBegin = co_await read(ryw, GetKeyReq(begin), pit);
 			if (resolvedBegin == allKeys.begin)
 				readToBegin = true;
 			if (resolvedBegin == ryw->getMaxReadKey())
 				readThroughEnd = true;
 
 			if (resolvedBegin >= end.getKey() && end.offset <= 1) {
-				return RangeResultRef(false, false);
+				co_return RangeResultRef(false, false);
 			} else if (resolvedBegin == ryw->getMaxReadKey()) {
-				return RangeResultRef(readToBegin, readThroughEnd);
+				co_return RangeResultRef(readToBegin, readThroughEnd);
 			}
 
 			resolveKeySelectorFromCache(end, it, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualEndOffset);
@@ -942,7 +943,7 @@ class RYWImpl {
 
 		//TraceEvent("RYWSelectorsStartReverse", randomID).detail("ByteLimit", limits.bytes).detail("RowLimit", limits.rows);
 
-		loop {
+		while (true) {
 			/*TraceEvent("RYWSelectors", randomID).detail("Begin", begin.toString())
 			    .detail("End", end.toString())
 			    .detail("Reached", limits.isReached())
@@ -955,11 +956,11 @@ class RYWImpl {
 			    .detail("Requests", requestCount);*/
 
 			if (!result.size() && actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
-				return RangeResultRef(false, false);
+				co_return RangeResultRef(false, false);
 			}
 
 			if (!begin.isBackward() && begin.getKey() >= ryw->getMaxReadKey()) {
-				return RangeResultRef(readToBegin, readThroughEnd);
+				co_return RangeResultRef(readToBegin, readThroughEnd);
 			}
 
 			if ((begin.offset >= end.offset && begin.getKey() >= end.getKey()) ||
@@ -968,10 +969,10 @@ class RYWImpl {
 					break;
 				if (!result.size())
 					break;
-				Key resolvedBegin =
-				    wait(read(ryw,
-				              GetKeyReq(begin),
-				              pit)); // do not worry about iterator invalidation, because we are breaking for the loop
+				Key resolvedBegin = co_await read(
+				    ryw,
+				    GetKeyReq(begin),
+				    pit); // do not worry about iterator invalidation, because we are breaking for the loop
 				if (resolvedBegin == allKeys.begin)
 					readToBegin = true;
 				if (resolvedBegin == ryw->getMaxReadKey())
@@ -984,7 +985,7 @@ class RYWImpl {
 			    it.beginKey() < itEnd.beginKey()) {
 				if (begin.isFirstGreaterOrEqual())
 					break;
-				return RangeResultRef(readToBegin, readThroughEnd);
+				co_return RangeResultRef(readToBegin, readThroughEnd);
 			}
 
 			if (limits.isReached() && itemsPastBegin >= begin.offset - 1)
@@ -1008,7 +1009,7 @@ class RYWImpl {
 				if (it.beginKey() > itEnd.beginKey())
 					singleClears = std::min(skipUncachedBack(ucEnd, itEnd, BUGGIFY ? 0 : clearLimit + 100), clearLimit);
 
-				state KeySelector read_begin;
+				KeySelector read_begin;
 				if (ucEnd != itEnd) {
 					Key k = ucEnd.beginKey().toStandaloneStringRef();
 					read_begin = KeySelector(firstGreaterOrEqual(k), k.arena());
@@ -1027,7 +1028,7 @@ class RYWImpl {
 
 				additionalRows += singleClears;
 
-				state KeySelector read_end;
+				KeySelector read_end;
 				if (end.isFirstGreaterOrEqual()) {
 					Key k = it.endKey() < end.getKey() ? it.endKey().toStandaloneStringRef() : end.getKey();
 					end = KeySelector(firstGreaterOrEqual(k), k.arena());
@@ -1049,7 +1050,7 @@ class RYWImpl {
 					read_begin.arena().dependsOn(read_end.arena());
 				}
 
-				state GetRangeLimits requestLimit = limits;
+				GetRangeLimits requestLimit = limits;
 				setRequestLimits(requestLimit, additionalRows, read_end.offset, requestCount);
 				requestCount++;
 
@@ -1060,7 +1061,7 @@ class RYWImpl {
 
 				additionalRows = 0;
 				RangeResult snapshot_read =
-				    wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::True, Reverse::True));
+				    co_await ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::True, Reverse::True);
 				KeyRangeRef range = getKnownKeyRangeBack(snapshot_read, read_begin, read_end, ryw->arena);
 
 				//TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough);
@@ -1123,7 +1124,7 @@ class RYWImpl {
 		result.readThroughEnd = readThroughEnd;
 		result.arena().dependsOn(ryw->arena);
 
-		return result;
+		co_return result;
 	}
 
 #ifndef __INTEL_COMPILER
@@ -1142,22 +1143,22 @@ class RYWImpl {
 		// read.limits, it);
 	};
 
-	ACTOR template <bool backwards>
+	template <bool backwards>
 	static Future<MappedRangeResult> readThrough(ReadYourWritesTransaction* ryw,
 	                                             GetMappedRangeReq<backwards> read,
 	                                             Snapshot snapshot) {
 		if (backwards && read.end.offset > 1) {
 			// FIXME: Optimistically assume that this will not run into the system keys, and only reissue if the result
 			// actually does.
-			Key key = wait(ryw->tr.getKey(read.end, snapshot));
+			Key key = co_await ryw->tr.getKey(read.end, snapshot);
 			if (key > ryw->getMaxReadKey())
 				read.end = firstGreaterOrEqual(ryw->getMaxReadKey());
 			else
 				read.end = KeySelector(firstGreaterOrEqual(key), key.arena());
 		}
-		MappedRangeResult v = wait(ryw->tr.getMappedRange(
-		    read.begin, read.end, read.mapper, read.limits, snapshot, backwards ? Reverse::True : Reverse::False));
-		return v;
+		MappedRangeResult v = co_await ryw->tr.getMappedRange(
+		    read.begin, read.end, read.mapper, read.limits, snapshot, backwards ? Reverse::True : Reverse::False);
+		co_return v;
 	}
 
 	template <bool backwards>
@@ -1281,11 +1282,11 @@ class RYWImpl {
 		triggerWatches(ryw, singleKeyRange(key), val, valueKnown);
 	}
 
-	ACTOR static Future<Void> watch(ReadYourWritesTransaction* ryw, Key key) {
-		state Future<Optional<Value>> val;
-		state Future<Void> watchFuture;
-		state Reference<Watch> watch(new Watch(key));
-		state Promise<Void> done;
+	static Future<Void> watch(ReadYourWritesTransaction* ryw, Key key) {
+		Future<Optional<Value>> val;
+		Future<Void> watchFuture;
+		Reference<Watch> watch(new Watch(key));
+		Promise<Void> done;
 
 		ryw->reading.add(done.getFuture());
 
@@ -1298,7 +1299,7 @@ class RYWImpl {
 		}
 
 		try {
-			wait(ryw->resetPromise.getFuture() || success(val) || watch->onChangeTrigger.getFuture());
+			co_await (ryw->resetPromise.getFuture() || success(val) || watch->onChangeTrigger.getFuture());
 		} catch (Error& e) {
 			done.send(Void());
 			throw;
@@ -1308,7 +1309,7 @@ class RYWImpl {
 			done.send(Void());
 			if (watch->onChangeTrigger.getFuture().isError())
 				throw watch->onChangeTrigger.getFuture().getError();
-			return Void();
+			co_return;
 		}
 
 		watch->valuePresent = true;
@@ -1318,7 +1319,7 @@ class RYWImpl {
 		                          (watch->value.present() && watch->setValue.get() != watch->value.get()))) {
 			watch->onChangeTrigger.send(Void());
 			done.send(Void());
-			return Void();
+			co_return;
 		}
 
 		try {
@@ -1329,35 +1330,33 @@ class RYWImpl {
 		}
 		done.send(Void());
 
-		wait(watchFuture);
-
-		return Void();
+		co_await watchFuture;
 	}
 
-	ACTOR static void simulateTimeoutInFlightCommit(ReadYourWritesTransaction* ryw_) {
-		state Reference<ReadYourWritesTransaction> ryw = Reference<ReadYourWritesTransaction>::addRef(ryw_);
+	static Future<Void> simulateTimeoutInFlightCommit(Uncancellable, ReadYourWritesTransaction* ryw_) {
+		Reference<ReadYourWritesTransaction> ryw = Reference<ReadYourWritesTransaction>::addRef(ryw_);
 		ASSERT(ryw->options.timeoutInSeconds > 0);
 		// An actual in-flight commit (i.e. one that's past the point where cancelling the transaction would stop it)
 		// would already have a read version. We need to get a read version too, otherwise committing a conflicting
 		// transaction may not ensure this transaction is no longer in-flight, since this transaction could get a read
 		// version _after_.
-		wait(success(ryw->getReadVersion()));
+		co_await success(ryw->getReadVersion());
 		if (!ryw->resetPromise.isSet())
 			ryw->resetPromise.sendError(transaction_timed_out());
-		wait(delay(deterministicRandom()->random01() * 5));
+		co_await delay(deterministicRandom()->random01() * 5);
 		TraceEvent("ClientBuggifyInFlightCommit").log();
-		wait(ryw->tr.commit());
+		co_await ryw->tr.commit();
 	}
 
-	ACTOR static Future<Void> commit(ReadYourWritesTransaction* ryw) {
+	static Future<Void> commit(ReadYourWritesTransaction* ryw) {
 		try {
 			ryw->commitStarted = true;
 
 			if (ryw->options.specialKeySpaceChangeConfiguration)
-				wait(ryw->getDatabase()->specialKeySpace->commit(ryw));
+				co_await ryw->getDatabase()->specialKeySpace->commit(ryw);
 
 			Future<Void> ready = ryw->reading;
-			wait(ryw->resetPromise.getFuture() || ready);
+			co_await (ryw->resetPromise.getFuture() || ready);
 
 			if (ryw->options.readYourWritesDisabled) {
 
@@ -1375,10 +1374,10 @@ class RYWImpl {
 				if (ryw->resetPromise.isSet())
 					throw ryw->resetPromise.getFuture().getError();
 				if (CLIENT_BUGGIFY && ryw->options.timeoutInSeconds > 0) {
-					simulateTimeoutInFlightCommit(ryw);
+					simulateTimeoutInFlightCommit(Uncancellable(), ryw);
 					throw transaction_timed_out();
 				}
-				wait(ryw->resetPromise.getFuture() || ryw->tr.commit());
+				co_await (ryw->resetPromise.getFuture() || ryw->tr.commit());
 
 				ryw->debugLogRetries();
 
@@ -1386,7 +1385,7 @@ class RYWImpl {
 					ryw->reset();
 				}
 
-				return Void();
+				co_return;
 			}
 
 			ryw->writeRangeToNativeTransaction(KeyRangeRef(StringRef(), allKeys.end));
@@ -1399,17 +1398,17 @@ class RYWImpl {
 			}
 
 			if (CLIENT_BUGGIFY && ryw->options.timeoutInSeconds > 0) {
-				simulateTimeoutInFlightCommit(ryw);
+				simulateTimeoutInFlightCommit(Uncancellable(), ryw);
 				throw transaction_timed_out();
 			}
-			wait(ryw->resetPromise.getFuture() || ryw->tr.commit());
+			co_await (ryw->resetPromise.getFuture() || ryw->tr.commit());
 
 			ryw->debugLogRetries();
 			if (!ryw->tr.apiVersionAtLeast(410)) {
 				ryw->reset();
 			}
 
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			if (!ryw->tr.apiVersionAtLeast(410)) {
 				ryw->commitStarted = false;
@@ -1428,21 +1427,21 @@ class RYWImpl {
 	// the future in the output. If commitFuture isn't specified, then the transaction will
 	// be reported uncommitted. In that case, an optional error can be provided to indicate
 	// why the transaction was uncommitted.
-	ACTOR static Future<Void> printDebugMessages(ReadYourWritesTransaction* self,
-	                                             Optional<Future<Void>> commitFuture,
-	                                             Optional<Error> error = Optional<Error>()) {
-		state std::string prefix;
-		state std::string commitResult;
-		state ErrorOr<Void> result;
-		state std::vector<BaseTraceEvent> debugTraces = std::move(self->debugTraces);
-		state std::vector<std::string> debugMessages = std::move(self->debugMessages);
+	static Future<Void> printDebugMessages(ReadYourWritesTransaction* self,
+	                                       Optional<Future<Void>> commitFuture,
+	                                       Optional<Error> error = Optional<Error>()) {
+		std::string prefix;
+		std::string commitResult;
+		ErrorOr<Void> result;
+		std::vector<BaseTraceEvent> debugTraces = std::move(self->debugTraces);
+		std::vector<std::string> debugMessages = std::move(self->debugMessages);
 
 		self->debugTraces.clear();
 		self->debugMessages.clear();
 
 		if (commitFuture.present()) {
 			try {
-				wait(store(result, errorOr(commitFuture.get())));
+				result = co_await errorOr(commitFuture.get());
 			} catch (Error& e) {
 				result = e;
 			}
@@ -1492,11 +1491,9 @@ class RYWImpl {
 		if (result.isError()) {
 			throw result.getError();
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> onError(ReadYourWritesTransaction* ryw, Error e) {
+	static Future<Void> onError(ReadYourWritesTransaction* ryw, Error e) {
 		if (ryw->debugTraces.size() > 0 || ryw->debugMessages.size() > 0) {
 			// printDebugMessages returns a future but will not block if called with an empty second argument
 			ASSERT(printDebugMessages(ryw, {}, e).isReady());
@@ -1514,12 +1511,12 @@ class RYWImpl {
 				throw e;
 			}
 
-			wait(ryw->resetPromise.getFuture() || ryw->tr.onError(e));
+			co_await (ryw->resetPromise.getFuture() || ryw->tr.onError(e));
 
 			ryw->debugLogRetries(e);
 
 			ryw->resetRyow();
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			if (!ryw->resetPromise.isSet()) {
 				if (ryw->tr.apiVersionAtLeast(610)) {
@@ -1556,9 +1553,9 @@ ReadYourWritesTransaction::ReadYourWritesTransaction(Database const& cx)
 	applyPersistentOptions();
 }
 
-ACTOR Future<Void> timebomb(double endTime, Promise<Void> resetPromise) {
+Future<Void> timebomb(double endTime, Promise<Void> resetPromise) {
 	while (now() < endTime) {
-		wait(delayUntil(std::min(endTime + 0.0001, now() + CLIENT_KNOBS->TRANSACTION_TIMEOUT_DELAY_INTERVAL)));
+		co_await delayUntil(std::min(endTime + 0.0001, now() + CLIENT_KNOBS->TRANSACTION_TIMEOUT_DELAY_INTERVAL));
 	}
 	if (!resetPromise.isSet())
 		resetPromise.sendError(transaction_timed_out());
@@ -1590,32 +1587,32 @@ Optional<Value> getValueFromJSON(StatusObject statusObj) {
 	}
 }
 
-ACTOR Future<Optional<Value>> getJSON(Database db, std::string jsonField = "") {
-	StatusObject statusObj = wait(StatusClient::statusFetcher(db, jsonField));
-	return getValueFromJSON(statusObj);
+Future<Optional<Value>> getJSON(Database db, std::string jsonField) {
+	StatusObject statusObj = co_await StatusClient::statusFetcher(db, jsonField);
+	co_return getValueFromJSON(statusObj);
 }
 
-ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> connRecord) {
-	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
-	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(connRecord, clusterInterface);
-
-	loop {
-		choose {
-			when(std::vector<ClientWorkerInterface> workers =
-			         wait(clusterInterface->get().present()
-			                  ? brokenPromiseToNever(
-			                        clusterInterface->get().get().getClientWorkers.getReply(GetClientWorkersRequest()))
-			                  : Never())) {
-				RangeResult result;
-				for (auto& it : workers) {
-					result.push_back_deep(
-					    result.arena(),
-					    KeyValueRef(it.address().toString(), BinaryWriter::toValue(it, IncludeVersion())));
-				}
-
-				return result;
+Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> connRecord) {
+	Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
+	Future<Void> leaderMon = monitorLeader<ClusterInterface>(connRecord, clusterInterface);
+
+	while (true) {
+		Future<std::vector<ClientWorkerInterface>> workersFuture =
+		    clusterInterface->get().present()
+		        ? brokenPromiseToNever(
+		              clusterInterface->get().get().getClientWorkers.getReply(GetClientWorkersRequest()))
+		        : Never();
+		auto res = co_await race(workersFuture, clusterInterface->onChange());
+		if (res.index() == 0) {
+			std::vector<ClientWorkerInterface> workers = std::get<0>(std::move(res));
+
+			RangeResult result;
+			for (auto& it : workers) {
+				result.push_back_deep(
+				    result.arena(), KeyValueRef(it.address().toString(), BinaryWriter::toValue(it, IncludeVersion())));
 			}
-			when(wait(clusterInterface->onChange())) {}
+
+			co_return result;
 		}
 	}
 }
diff --git a/fdbclient/RestoreInterface.cpp b/fdbclient/RestoreInterface.cpp
index 5e52c708768..96d81225bb4 100644
--- a/fdbclient/RestoreInterface.cpp
+++ b/fdbclient/RestoreInterface.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbclient/RestoreInterface.h"
+#include "RestoreInterface.h"
 #include "flow/serialize.h"
 
 const KeyRef restoreRequestDoneKey = "\xff\x02/restoreRequestDone"_sr;
diff --git a/fdbclient/include/fdbclient/RestoreInterface.h b/fdbclient/RestoreInterface.h
similarity index 100%
rename from fdbclient/include/fdbclient/RestoreInterface.h
rename to fdbclient/RestoreInterface.h
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
deleted file mode 100644
index e7dc5918ac5..00000000000
--- a/fdbclient/S3BlobStore.actor.cpp
+++ /dev/null
@@ -1,2600 +0,0 @@
-/*
- * S3BlobStore.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/S3BlobStore.h"
-
-#include <sstream>
-#include "fdbrpc/HTTP.h"
-#include "fdbclient/ClientKnobs.h"
-#include "fdbclient/Knobs.h"
-#include "flow/FastRef.h"
-#include "flow/IConnection.h"
-#include "flow/Trace.h"
-#include "flow/flow.h"
-#include "flow/genericactors.actor.h"
-
-// S3BlobStoreEndpoint::ConnectionPoolData destructor implementation
-S3BlobStoreEndpoint::ConnectionPoolData::~ConnectionPoolData() {
-	// In simulation, explicitly close all pooled connections before destruction.
-	// This satisfies Sim2Conn's assertion: !opened || closedByCaller
-	// Without this, connections would be destroyed without being closed, causing assertion failures.
-	if (g_network && g_network->isSimulated()) {
-		while (!pool.empty()) {
-			ReusableConnection& rconn = pool.front();
-			if (rconn.conn.isValid()) {
-				rconn.conn->close();
-			}
-			pool.pop();
-		}
-	}
-}
-
-#include "md5/md5.h"
-#include "libb64/encode.h"
-#include "fdbclient/sha1/SHA1.h"
-#include <climits>
-#include <iostream>
-#include <time.h>
-#include <iomanip>
-#include <openssl/sha.h>
-#include <openssl/evp.h>
-#include <openssl/hmac.h>
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/classification.hpp>
-#include <boost/algorithm/string.hpp>
-#include <boost/algorithm/string.hpp>
-#include <boost/algorithm/hex.hpp>
-#include "flow/IAsyncFile.h"
-#include "flow/Hostname.h"
-#include "flow/UnitTest.h"
-#include "rapidxml/rapidxml.hpp"
-#ifdef WITH_AWS_BACKUP
-#include "fdbclient/FDBAWSCredentialsProvider.h"
-#endif
-
-#include "flow/actorcompiler.h" // has to be last include
-
-using namespace rapidxml;
-
-json_spirit::mObject S3BlobStoreEndpoint::Stats::getJSON() {
-	json_spirit::mObject o;
-
-	o["requests_failed"] = requests_failed;
-	o["requests_successful"] = requests_successful;
-	o["bytes_sent"] = bytes_sent;
-
-	return o;
-}
-
-S3BlobStoreEndpoint::Stats S3BlobStoreEndpoint::Stats::operator-(const Stats& rhs) {
-	Stats r;
-	r.requests_failed = requests_failed - rhs.requests_failed;
-	r.requests_successful = requests_successful - rhs.requests_successful;
-	r.bytes_sent = bytes_sent - rhs.bytes_sent;
-	return r;
-}
-
-S3BlobStoreEndpoint::Stats S3BlobStoreEndpoint::s_stats;
-
-S3BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
-	secure_connection = 1;
-	connect_tries = CLIENT_KNOBS->BLOBSTORE_CONNECT_TRIES;
-	connect_timeout = CLIENT_KNOBS->BLOBSTORE_CONNECT_TIMEOUT;
-	max_connection_life = CLIENT_KNOBS->BLOBSTORE_MAX_CONNECTION_LIFE;
-	request_tries = CLIENT_KNOBS->BLOBSTORE_REQUEST_TRIES;
-	request_timeout_min = CLIENT_KNOBS->BLOBSTORE_REQUEST_TIMEOUT_MIN;
-	requests_per_second = CLIENT_KNOBS->BLOBSTORE_REQUESTS_PER_SECOND;
-	concurrent_requests = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_REQUESTS;
-	list_requests_per_second = CLIENT_KNOBS->BLOBSTORE_LIST_REQUESTS_PER_SECOND;
-	write_requests_per_second = CLIENT_KNOBS->BLOBSTORE_WRITE_REQUESTS_PER_SECOND;
-	read_requests_per_second = CLIENT_KNOBS->BLOBSTORE_READ_REQUESTS_PER_SECOND;
-	delete_requests_per_second = CLIENT_KNOBS->BLOBSTORE_DELETE_REQUESTS_PER_SECOND;
-	multipart_max_part_size = CLIENT_KNOBS->BLOBSTORE_MULTIPART_MAX_PART_SIZE;
-	multipart_min_part_size = CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE;
-	concurrent_uploads = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_UPLOADS;
-	concurrent_lists = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_LISTS;
-	concurrent_reads_per_file = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_READS_PER_FILE;
-	concurrent_writes_per_file = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
-	enable_read_cache = CLIENT_KNOBS->BLOBSTORE_ENABLE_READ_CACHE;
-	read_block_size = CLIENT_KNOBS->BLOBSTORE_READ_BLOCK_SIZE;
-	read_ahead_blocks = CLIENT_KNOBS->BLOBSTORE_READ_AHEAD_BLOCKS;
-	read_cache_blocks_per_file = CLIENT_KNOBS->BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
-	max_send_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
-	max_recv_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
-	max_delay_retryable_error = CLIENT_KNOBS->BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
-	max_delay_connection_failed = CLIENT_KNOBS->BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
-	sdk_auth = false;
-	enable_object_integrity_check = CLIENT_KNOBS->BLOBSTORE_ENABLE_OBJECT_INTEGRITY_CHECK;
-	global_connection_pool = CLIENT_KNOBS->BLOBSTORE_GLOBAL_CONNECTION_POOL;
-}
-
-bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
-#define TRY_PARAM(n, sn)                                                                                               \
-	if (name == #n || name == #sn) {                                                                                   \
-		n = value;                                                                                                     \
-		return true;                                                                                                   \
-	}
-	TRY_PARAM(secure_connection, sc)
-	TRY_PARAM(connect_tries, ct);
-	TRY_PARAM(connect_timeout, cto);
-	TRY_PARAM(max_connection_life, mcl);
-	TRY_PARAM(request_tries, rt);
-	TRY_PARAM(request_timeout_min, rtom);
-	// TODO: For backward compatibility because request_timeout was renamed to request_timeout_min
-	if (name == "request_timeout"_sr || name == "rto"_sr) {
-		request_timeout_min = value;
-		return true;
-	}
-	TRY_PARAM(requests_per_second, rps);
-	TRY_PARAM(list_requests_per_second, lrps);
-	TRY_PARAM(write_requests_per_second, wrps);
-	TRY_PARAM(read_requests_per_second, rrps);
-	TRY_PARAM(delete_requests_per_second, drps);
-	TRY_PARAM(concurrent_requests, cr);
-	TRY_PARAM(multipart_max_part_size, maxps);
-	TRY_PARAM(multipart_min_part_size, minps);
-	TRY_PARAM(multipart_retry_delay_ms, mrd);
-	TRY_PARAM(concurrent_uploads, cu);
-	TRY_PARAM(concurrent_lists, cl);
-	TRY_PARAM(concurrent_reads_per_file, crpf);
-	TRY_PARAM(concurrent_writes_per_file, cwpf);
-	TRY_PARAM(enable_read_cache, erc);
-	TRY_PARAM(read_block_size, rbs);
-	TRY_PARAM(read_ahead_blocks, rab);
-	TRY_PARAM(read_cache_blocks_per_file, rcb);
-	TRY_PARAM(max_send_bytes_per_second, sbps);
-	TRY_PARAM(max_recv_bytes_per_second, rbps);
-	TRY_PARAM(max_delay_retryable_error, dre);
-	TRY_PARAM(max_delay_connection_failed, dcf);
-	TRY_PARAM(sdk_auth, sa);
-	TRY_PARAM(enable_object_integrity_check, eoic);
-	TRY_PARAM(global_connection_pool, gcp);
-#undef TRY_PARAM
-	return false;
-}
-
-// Returns an S3 Blob URL parameter string that specifies all of the non-default options for the endpoint using option
-// short names.
-std::string S3BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
-	static BlobKnobs defaults;
-	std::string r;
-#define _CHECK_PARAM(n, sn)                                                                                            \
-	if (n != defaults.n) {                                                                                             \
-		r += format("%s%s=%d", r.empty() ? "" : "&", #sn, n);                                                          \
-	}
-	_CHECK_PARAM(secure_connection, sc);
-	_CHECK_PARAM(connect_tries, ct);
-	_CHECK_PARAM(connect_timeout, cto);
-	_CHECK_PARAM(max_connection_life, mcl);
-	_CHECK_PARAM(request_tries, rt);
-	_CHECK_PARAM(request_timeout_min, rto);
-	_CHECK_PARAM(requests_per_second, rps);
-	_CHECK_PARAM(list_requests_per_second, lrps);
-	_CHECK_PARAM(write_requests_per_second, wrps);
-	_CHECK_PARAM(read_requests_per_second, rrps);
-	_CHECK_PARAM(delete_requests_per_second, drps);
-	_CHECK_PARAM(concurrent_requests, cr);
-	_CHECK_PARAM(multipart_max_part_size, maxps);
-	_CHECK_PARAM(multipart_min_part_size, minps);
-	_CHECK_PARAM(concurrent_uploads, cu);
-	_CHECK_PARAM(concurrent_lists, cl);
-	_CHECK_PARAM(concurrent_reads_per_file, crpf);
-	_CHECK_PARAM(concurrent_writes_per_file, cwpf);
-	_CHECK_PARAM(enable_read_cache, erc);
-	_CHECK_PARAM(read_block_size, rbs);
-	_CHECK_PARAM(read_ahead_blocks, rab);
-	_CHECK_PARAM(read_cache_blocks_per_file, rcb);
-	_CHECK_PARAM(max_send_bytes_per_second, sbps);
-	_CHECK_PARAM(max_recv_bytes_per_second, rbps);
-	_CHECK_PARAM(sdk_auth, sa);
-	_CHECK_PARAM(enable_object_integrity_check, eoic);
-	_CHECK_PARAM(global_connection_pool, gcp);
-	_CHECK_PARAM(max_delay_retryable_error, dre);
-	_CHECK_PARAM(max_delay_connection_failed, dcf);
-#undef _CHECK_PARAM
-	return r;
-}
-
-std::string guessRegionFromDomain(std::string domain) {
-	// Special case for localhost/127.0.0.1 to prevent basic_string exception
-	if (domain == "127.0.0.1" || domain == "localhost") {
-		return "us-east-1";
-	}
-
-	static const std::vector<const char*> knownServices = { "s3.", "cos.", "oss-", "obs." };
-	boost::algorithm::to_lower(domain);
-
-	for (int i = 0; i < knownServices.size(); ++i) {
-		const char* service = knownServices[i];
-
-		std::size_t p = domain.find(service);
-		if (p == std::string::npos || (p >= 1 && domain[p - 1] != '.')) {
-			// eg. 127.0.0.1, example.com, s3-service.example.com, mys3.example.com
-			continue;
-		}
-
-		StringRef h = StringRef(domain).substr(p);
-
-		if (!h.startsWith("oss-"_sr)) {
-			h.eat(service); // ignore s3 service
-		}
-
-		return h.eat(".").toString();
-	}
-
-	return "";
-}
-
-Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string& url,
-                                                               const Optional<std::string>& proxy,
-                                                               std::string* resourceFromURL,
-                                                               std::string* error,
-                                                               ParametersT* ignored_parameters) {
-	if (resourceFromURL)
-		resourceFromURL->clear();
-
-	try {
-		// Replace HTML-encoded ampersands with raw ampersands
-		std::string decoded_url = url;
-		size_t pos = 0;
-		while ((pos = decoded_url.find("&amp;", pos)) != std::string::npos) {
-			decoded_url.replace(pos, 5, "&");
-			pos += 1;
-		}
-
-		// Also handle double-encoded ampersands
-		pos = 0;
-		while ((pos = decoded_url.find("&amp;amp;", pos)) != std::string::npos) {
-			decoded_url.replace(pos, 9, "&");
-			pos += 1;
-		}
-
-		StringRef t(decoded_url);
-		StringRef prefix = t.eat("://");
-		if (prefix != "blobstore"_sr)
-			throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str());
-
-		Optional<std::string> proxyHost, proxyPort;
-		if (proxy.present()) {
-			StringRef proxyRef(proxy.get());
-			if (proxy.get().find("://") != std::string::npos) {
-				StringRef proxyPrefix = proxyRef.eat("://");
-				if (proxyPrefix != "http"_sr) {
-					throw format("Invalid proxy URL prefix '%s'. Either don't use a prefix, or use http://",
-					             proxyPrefix.toString().c_str());
-				}
-			}
-			std::string proxyBody = proxyRef.eat().toString();
-			if (!Hostname::isHostname(proxyBody) && !NetworkAddress::parseOptional(proxyBody).present()) {
-				throw format("'%s' is not a valid value for proxy. Format should be either IP:port or host:port.",
-				             proxyBody.c_str());
-			}
-			StringRef p(proxyBody);
-			proxyHost = p.eat(":").toString();
-			proxyPort = p.eat().toString();
-		}
-
-		Optional<StringRef> cred;
-		if (url.find("@") != std::string::npos) {
-			cred = t.eat("@");
-		}
-		uint8_t foundSeparator = 0;
-		StringRef hostPort = t.eatAny("/?", &foundSeparator);
-		StringRef resource;
-		if (foundSeparator == '/') {
-			resource = t.eat("?");
-		}
-
-		// hostPort is at least a host or IP address, optionally followed by :portNumber or :serviceName
-		StringRef h(hostPort);
-		StringRef host = h.eat(":");
-		if (host.size() == 0)
-			throw std::string("host cannot be empty");
-
-		StringRef service = h.eat();
-
-		std::string region = guessRegionFromDomain(host.toString());
-
-		BlobKnobs knobs;
-		HTTP::Headers extraHeaders;
-		while (1) {
-			StringRef name = t.eat("=");
-			if (name.size() == 0)
-				break;
-			StringRef value = t.eat("&");
-
-			// Special case for header
-			if (name == "header"_sr) {
-				StringRef originalValue = value;
-				StringRef headerFieldName = value.eat(":");
-				StringRef headerFieldValue = value;
-				if (headerFieldName.size() == 0 || headerFieldValue.size() == 0) {
-					throw format("'%s' is not a valid value for '%s' parameter.  Format is <FieldName>:<FieldValue> "
-					             "where strings are not empty.",
-					             originalValue.toString().c_str(),
-					             name.toString().c_str());
-				}
-				std::string& fieldValue = extraHeaders[headerFieldName.toString()];
-				// RFC 2616 section 4.2 says header field names can repeat but only if it is valid to concatenate their
-				// values with comma separation
-				if (!fieldValue.empty()) {
-					fieldValue.append(",");
-				}
-				fieldValue.append(headerFieldValue.toString());
-				continue;
-			}
-
-			// overwrite s3 region from parameter
-			if (name == "region"_sr) {
-				region = value.toString();
-				continue;
-			}
-
-			// See if the parameter is a knob
-			// First try setting a dummy value (all knobs are currently numeric) just to see if this parameter is known
-			// to S3BlobStoreEndpoint. If it is, then we will set it to a good value or throw below, so the dummy set
-			// has no bad side effects.
-			bool known = knobs.set(name, 0);
-
-			// If the parameter is not known to S3BlobStoreEndpoint then throw unless there is an ignored_parameters set
-			// to add it to
-			if (!known) {
-				if (ignored_parameters == nullptr) {
-					throw format("%s is not a valid parameter name", name.toString().c_str());
-				}
-				(*ignored_parameters)[name.toString()] = value.toString();
-				continue;
-			}
-
-			// The parameter is known to S3BlobStoreEndpoint so it must be numeric and valid.
-			char* valueEnd = nullptr;
-			std::string s = value.toString();
-			long int ivalue = strtol(s.c_str(), &valueEnd, 10);
-			if (*valueEnd || (ivalue == 0 && s != "0") ||
-			    (((ivalue == LONG_MAX) || (ivalue == LONG_MIN)) && errno == ERANGE))
-				throw format("%s is not a valid value for %s", s.c_str(), name.toString().c_str());
-
-			// It should not be possible for this set to fail now since the dummy set above had to have worked.
-			ASSERT(knobs.set(name, ivalue));
-		}
-
-		if (resourceFromURL != nullptr)
-			*resourceFromURL = resource.toString();
-
-		Optional<S3BlobStoreEndpoint::Credentials> creds;
-		if (cred.present()) {
-			StringRef c(cred.get());
-			StringRef key = c.eat(":");
-			StringRef secret = c.eat(":");
-			StringRef securityToken = c.eat();
-			creds = S3BlobStoreEndpoint::Credentials{ key.toString(), secret.toString(), securityToken.toString() };
-		}
-
-		if (region.empty() && CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER) {
-			throw std::string(
-			    "Failed to get region from host or parameter in url, region is required for aws v4 signature");
-		}
-
-		return makeReference<S3BlobStoreEndpoint>(
-		    host.toString(), service.toString(), region, proxyHost, proxyPort, creds, knobs, extraHeaders);
-
-	} catch (std::string& err) {
-		if (error != nullptr)
-			*error = err;
-		TraceEvent(SevWarnAlways, "S3BlobStoreEndpointBadURL")
-		    .suppressFor(60)
-		    .detail("Description", err)
-		    .detail("Format", getURLFormat())
-		    .detail("URL", url);
-		throw backup_invalid_url();
-	}
-}
-
-std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::string params) const {
-	std::string hostPort = host;
-	if (!service.empty()) {
-		hostPort.append(":");
-		hostPort.append(service);
-	}
-
-	// If secret isn't being looked up from credentials files then it was passed explicitly in the URL so show it here.
-	std::string credsString;
-	if (credentials.present()) {
-		if (!lookupKey) {
-			credsString = credentials.get().key;
-		}
-		if (!lookupSecret) {
-			credsString += ":" + credentials.get().secret;
-			if (!credentials.get().securityToken.empty()) {
-				credsString += ":" + credentials.get().securityToken;
-			}
-		}
-		credsString += "@";
-	}
-
-	std::string r = format("blobstore://%s%s/%s", credsString.c_str(), hostPort.c_str(), resource.c_str());
-
-	// Get params that are deviations from knob defaults
-	std::string knobParams = knobs.getURLParameters();
-	if (!knobParams.empty()) {
-		if (!params.empty()) {
-			params.append("&");
-		}
-		params.append(knobParams);
-	}
-
-	for (const auto& [k, v] : extraHeaders) {
-		if (!params.empty()) {
-			params.append("&");
-		}
-		params.append("header=");
-		params.append(k);
-		params.append(":");
-		params.append(v);
-	}
-
-	if (!params.empty()) {
-		r.append("?").append(params);
-	}
-
-	return r;
-}
-
-std::string S3BlobStoreEndpoint::constructResourcePath(const std::string& bucket, const std::string& object) const {
-	std::string resource;
-
-	if (host.find(bucket + ".") != 0) {
-		resource += std::string("/") + bucket; // not virtual hosting mode
-	}
-
-	if (!object.empty()) {
-		// S3 object keys should not start with '/'. Strip any leading slashes.
-		std::string cleanedObject = object;
-		while (!cleanedObject.empty() && cleanedObject[0] == '/') {
-			cleanedObject = cleanedObject.substr(1);
-		}
-		if (!cleanedObject.empty()) {
-			resource += "/";
-			resource += cleanedObject;
-		}
-	}
-
-	return resource;
-}
-
-// Forward declaration for doRequest_impl to fix compilation order issue
-ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                               std::string verb,
-                                                               std::string resource,
-                                                               HTTP::Headers headers,
-                                                               UnsentPacketQueue* pContent,
-                                                               int contentLen,
-                                                               std::set<unsigned int> successCodes);
-
-ACTOR Future<bool> bucketExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {
-	wait(b->requestRateRead->getAllowance(1));
-
-	state std::string resource = b->constructResourcePath(bucket, "");
-	state HTTP::Headers headers;
-
-	try {
-		Reference<HTTP::IncomingResponse> r =
-		    wait(doRequest_impl(b, "HEAD", resource, headers, nullptr, 0, { 200, 404 }));
-		return r->code == 200;
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled)
-			throw;
-		TraceEvent(SevError, "S3ClientBucketExistsError")
-		    .detail("Bucket", bucket)
-		    .detail("Host", b->host)
-		    .errorUnsuppressed(e);
-		throw;
-	}
-}
-
-Future<bool> S3BlobStoreEndpoint::bucketExists(std::string const& bucket) {
-	return bucketExists_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket);
-}
-
-ACTOR Future<bool> objectExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
-	wait(b->requestRateRead->getAllowance(1));
-
-	state std::string resource = b->constructResourcePath(bucket, object);
-	state HTTP::Headers headers;
-
-	Reference<HTTP::IncomingResponse> r = wait(doRequest_impl(b, "HEAD", resource, headers, nullptr, 0, { 200, 404 }));
-	return r->code == 200;
-}
-
-Future<bool> S3BlobStoreEndpoint::objectExists(std::string const& bucket, std::string const& object) {
-	return objectExists_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
-}
-
-ACTOR Future<Void> deleteObject_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
-	wait(b->requestRateDelete->getAllowance(1));
-
-	state std::string resource = b->constructResourcePath(bucket, object);
-	state HTTP::Headers headers;
-	// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered
-	// successful
-	Reference<HTTP::IncomingResponse> r =
-	    wait(doRequest_impl(b, "DELETE", resource, headers, nullptr, 0, { 200, 204, 404 }));
-
-	// But if the object already did not exist then the 'delete' is assumed to be successful but a warning is logged.
-	if (r->code == 404) {
-		TraceEvent(SevWarnAlways, "S3BlobStoreEndpointDeleteObjectMissing")
-		    .detail("Host", b->host)
-		    .detail("Bucket", bucket)
-		    .detail("Object", object);
-	}
-
-	return Void();
-}
-
-Future<Void> S3BlobStoreEndpoint::deleteObject(std::string const& bucket, std::string const& object) {
-	return deleteObject_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
-}
-
-ACTOR Future<Void> deleteRecursively_impl(Reference<S3BlobStoreEndpoint> b,
-                                          std::string bucket,
-                                          std::string prefix,
-                                          int* pNumDeleted,
-                                          int64_t* pBytesDeleted) {
-	state PromiseStream<S3BlobStoreEndpoint::ListResult> resultStream;
-	// Start a recursive parallel listing which will send results to resultStream as they are received
-	state Future<Void> done = b->listObjectsStream(bucket, resultStream, prefix, '/', std::numeric_limits<int>::max());
-	// Wrap done in an actor which will send end_of_stream since listObjectsStream() does not (so that many calls can
-	// write to the same stream)
-	done = map(done, [=](Void) mutable {
-		resultStream.sendError(end_of_stream());
-		return Void();
-	});
-
-	state std::list<Future<Void>> deleteFutures;
-	try {
-		loop {
-			choose {
-				// Throw if done throws, otherwise don't stop until end_of_stream
-				when(wait(done)) {
-					done = Never();
-				}
-
-				when(S3BlobStoreEndpoint::ListResult list = waitNext(resultStream.getFuture())) {
-					for (auto& object : list.objects) {
-						deleteFutures.push_back(map(b->deleteObject(bucket, object.name), [=](Void) -> Void {
-							if (pNumDeleted != nullptr) {
-								++*pNumDeleted;
-							}
-							if (pBytesDeleted != nullptr) {
-								*pBytesDeleted += object.size;
-							}
-							return Void();
-						}));
-					}
-				}
-			}
-
-			// This is just a precaution to avoid having too many outstanding delete actors waiting to run
-			while (deleteFutures.size() > CLIENT_KNOBS->BLOBSTORE_CONCURRENT_REQUESTS) {
-				wait(deleteFutures.front());
-				deleteFutures.pop_front();
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_end_of_stream)
-			throw;
-	}
-
-	while (deleteFutures.size() > 0) {
-		wait(deleteFutures.front());
-		deleteFutures.pop_front();
-	}
-
-	return Void();
-}
-
-Future<Void> S3BlobStoreEndpoint::deleteRecursively(std::string const& bucket,
-                                                    std::string prefix,
-                                                    int* pNumDeleted,
-                                                    int64_t* pBytesDeleted) {
-	return deleteRecursively_impl(
-	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, prefix, pNumDeleted, pBytesDeleted);
-}
-
-ACTOR Future<Void> createBucket_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {
-	state UnsentPacketQueue packets;
-	wait(b->requestRateWrite->getAllowance(1));
-
-	bool exists = wait(b->bucketExists(bucket));
-	if (!exists) {
-		state std::string resource = b->constructResourcePath(bucket, "");
-		state HTTP::Headers headers;
-
-		std::string region = b->getRegion();
-		if (region.empty()) {
-			Reference<HTTP::IncomingResponse> r =
-			    wait(doRequest_impl(b, "PUT", resource, headers, nullptr, 0, { 200, 409 }));
-		} else {
-			Standalone<StringRef> body(
-			    format("<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
-			           "  <LocationConstraint>%s</LocationConstraint>"
-			           "</CreateBucketConfiguration>",
-			           region.c_str()));
-			PacketWriter pw(packets.getWriteBuffer(), nullptr, Unversioned());
-			pw.serializeBytes(body);
-
-			Reference<HTTP::IncomingResponse> r =
-			    wait(doRequest_impl(b, "PUT", resource, headers, &packets, body.size(), { 200, 409 }));
-		}
-	}
-	return Void();
-}
-
-Future<Void> S3BlobStoreEndpoint::createBucket(std::string const& bucket) {
-	return createBucket_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket);
-}
-
-ACTOR Future<int64_t> objectSize_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
-	wait(b->requestRateRead->getAllowance(1));
-
-	state std::string resource = b->constructResourcePath(bucket, object);
-	state HTTP::Headers headers;
-
-	Reference<HTTP::IncomingResponse> r = wait(doRequest_impl(b, "HEAD", resource, headers, nullptr, 0, { 200, 404 }));
-	if (r->code == 404)
-		throw file_not_found();
-	return r->data.contentLen;
-}
-
-Future<int64_t> S3BlobStoreEndpoint::objectSize(std::string const& bucket, std::string const& object) {
-	return objectSize_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
-}
-
-// Try to read a file, parse it as JSON, and return the resulting document.
-// It will NOT throw if any errors are encountered, it will just return an empty
-// JSON object and will log trace events for the errors encountered.
-ACTOR Future<Optional<json_spirit::mObject>> tryReadJSONFile(std::string path) {
-	state std::string content;
-
-	// Event type to be logged in the event of an exception
-	state const char* errorEventType = "BlobCredentialFileError";
-
-	try {
-		state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-		    path, IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
-		state int64_t size = wait(f->size());
-		state Standalone<StringRef> buf = makeString(size);
-		int r = wait(f->read(mutateString(buf), size, 0));
-		ASSERT(r == size);
-		content = buf.toString();
-
-		// Any exceptions from hehre forward are parse failures
-		errorEventType = "BlobCredentialFileParseFailed";
-		json_spirit::mValue json;
-		json_spirit::read_string(content, json);
-		if (json.type() == json_spirit::obj_type)
-			return json.get_obj();
-		else
-			TraceEvent(SevWarn, "BlobCredentialFileNotJSONObject").suppressFor(60).detail("File", path);
-
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled)
-			TraceEvent(SevWarn, errorEventType).errorUnsuppressed(e).suppressFor(60).detail("File", path);
-	}
-
-	return Optional<json_spirit::mObject>();
-}
-
-// If the credentials expire, the connection will eventually fail and be discarded from the pool, and then a new
-// connection will be constructed, which will call this again to get updated credentials
-static S3BlobStoreEndpoint::Credentials getSecretSdk() {
-#ifdef WITH_AWS_BACKUP
-	double elapsed = -timer_monotonic();
-	Aws::Auth::AWSCredentials awsCreds = FDBAWSCredentialsProvider::getAwsCredentials();
-	elapsed += timer_monotonic();
-
-	if (awsCreds.IsEmpty()) {
-		TraceEvent(SevWarn, "S3BlobStoreAWSCredsEmpty");
-		throw backup_auth_missing();
-	}
-
-	S3BlobStoreEndpoint::Credentials fdbCreds;
-	fdbCreds.key = awsCreds.GetAWSAccessKeyId();
-	fdbCreds.secret = awsCreds.GetAWSSecretKey();
-	fdbCreds.securityToken = awsCreds.GetSessionToken();
-
-	TraceEvent("S3BlobStoreGotSdkCredentials").suppressFor(60).detail("Duration", elapsed);
-
-	return fdbCreds;
-#else
-	TraceEvent(SevError, "S3BlobStoreNoSDK").log();
-	throw backup_auth_missing();
-#endif
-}
-
-ACTOR Future<Void> updateSecret_impl(Reference<S3BlobStoreEndpoint> b) {
-	if (b->knobs.sdk_auth) {
-		b->credentials = getSecretSdk();
-		return Void();
-	}
-	std::vector<std::string>* pFiles = (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
-	if (pFiles == nullptr)
-		return Void();
-
-	if (!b->credentials.present()) {
-		return Void();
-	}
-
-	state std::vector<Future<Optional<json_spirit::mObject>>> reads;
-	for (auto& f : *pFiles)
-		reads.push_back(tryReadJSONFile(f));
-
-	wait(waitForAll(reads));
-
-	std::string accessKey = b->lookupKey ? "" : b->credentials.get().key;
-	std::string credentialsFileKey = accessKey + "@" + b->host;
-
-	int invalid = 0;
-
-	for (auto& f : reads) {
-		// If value not present then the credentials file wasn't readable or valid.  Continue to check other results.
-		if (!f.get().present()) {
-			TraceEvent(SevWarn, "S3BlobStoreAuthMissingNotPresent")
-			    .detail("Endpoint", b->host)
-			    .detail("Service", b->service);
-			++invalid;
-			continue;
-		}
-
-		JSONDoc doc(f.get().get());
-		if (doc.has("accounts") && doc.last().type() == json_spirit::obj_type) {
-			JSONDoc accounts(doc.last().get_obj());
-			if (accounts.has(credentialsFileKey, false) && accounts.last().type() == json_spirit::obj_type) {
-				JSONDoc account(accounts.last());
-				S3BlobStoreEndpoint::Credentials creds = b->credentials.get();
-				if (b->lookupKey) {
-					std::string apiKey;
-					if (account.tryGet("api_key", apiKey))
-						creds.key = apiKey;
-					else
-						continue;
-				}
-				if (b->lookupSecret) {
-					std::string secret;
-					if (account.tryGet("secret", secret))
-						creds.secret = secret;
-					else
-						continue;
-				}
-				std::string token;
-				if (account.tryGet("token", token))
-					creds.securityToken = token;
-				b->credentials = creds;
-				return Void();
-			} else {
-				TraceEvent(SevWarn, "S3BlobStoreAuthFoundAccountFAILED")
-				    .detail("CredentialFileKey", credentialsFileKey);
-			}
-		}
-	}
-
-	// If any sources were invalid
-	if (invalid > 0) {
-		TraceEvent(SevWarn, "S3BlobStoreInvalidCredentials")
-		    .detail("Endpoint", b->host)
-		    .detail("Service", b->service)
-		    .detail("Invalid", invalid);
-		throw backup_auth_unreadable();
-	}
-
-	// All sources were valid but didn't contain the desired info
-	TraceEvent(SevWarn, "S3BlobStoreAuthMissing")
-	    .detail("Endpoint", b->host)
-	    .detail("Service", b->service)
-	    .detail("Reason", "No valid credentials found");
-	throw backup_auth_missing();
-}
-
-Future<Void> S3BlobStoreEndpoint::updateSecret() {
-	return updateSecret_impl(Reference<S3BlobStoreEndpoint>::addRef(this));
-}
-
-ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3BlobStoreEndpoint> b,
-                                                                   bool* reusingConn) {
-	// First try to get a connection from the pool
-	*reusingConn = false;
-	while (!b->connectionPool->pool.empty()) {
-		S3BlobStoreEndpoint::ReusableConnection rconn = b->connectionPool->pool.front();
-		b->connectionPool->pool.pop();
-
-		// If the connection expires in the future then return it
-		if (rconn.expirationTime > now()) {
-			*reusingConn = true;
-			++b->blobStats->reusedConnections;
-			TraceEvent("S3BlobStoreEndpointReusingConnected")
-			    .suppressFor(60)
-			    .detail("RemoteEndpoint", rconn.conn->getPeerAddress())
-			    .detail("ExpiresIn", rconn.expirationTime - now())
-			    .detail("Proxy", b->proxyHost.orDefault(""));
-			return rconn;
-		}
-		++b->blobStats->expiredConnections;
-	}
-	++b->blobStats->newConnections;
-	std::string host = b->host, service = b->service;
-	TraceEvent(SevDebug, "S3BlobStoreEndpointBuildingNewConnection")
-	    .detail("UseProxy", b->useProxy)
-	    .detail("TLS", b->knobs.secure_connection == 1)
-	    .detail("Host", host)
-	    .detail("Service", service)
-	    .log();
-	if (service.empty()) {
-		if (b->useProxy) {
-			fprintf(stderr, "ERROR: Port can't be empty when using HTTP proxy.\n");
-			throw connection_failed();
-		}
-		service = b->knobs.secure_connection ? "https" : "http";
-	}
-	bool isTLS = b->knobs.isTLS();
-	state Reference<IConnection> conn;
-	if (b->useProxy) {
-		if (isTLS) {
-			Reference<IConnection> _conn =
-			    wait(HTTP::proxyConnect(host, service, b->proxyHost.get(), b->proxyPort.get()));
-			conn = _conn;
-		} else {
-			host = b->proxyHost.get();
-			service = b->proxyPort.get();
-			Reference<IConnection> _conn = wait(INetworkConnections::net()->connect(host, service, false));
-			conn = _conn;
-		}
-	} else {
-		wait(store(conn, INetworkConnections::net()->connect(host, service, isTLS)));
-	}
-
-	// Ensure connection is valid before handshake
-	ASSERT(conn.isValid());
-
-	wait(conn->connectHandshake());
-
-	TraceEvent("S3BlobStoreEndpointNewConnectionSuccess")
-	    .suppressFor(60)
-	    .detail("RemoteEndpoint", conn->getPeerAddress())
-	    .detail("ExpiresIn", b->knobs.max_connection_life)
-	    .detail("Proxy", b->proxyHost.orDefault(""));
-
-	if (b->lookupKey || b->lookupSecret || b->knobs.sdk_auth)
-		wait(b->updateSecret());
-
-	return S3BlobStoreEndpoint::ReusableConnection({ conn, now() + b->knobs.max_connection_life });
-}
-
-Future<S3BlobStoreEndpoint::ReusableConnection> S3BlobStoreEndpoint::connect(bool* reusing) {
-	return connect_impl(Reference<S3BlobStoreEndpoint>::addRef(this), reusing);
-}
-
-void S3BlobStoreEndpoint::returnConnection(ReusableConnection& rconn) {
-	// If it expires in the future then add it to the pool in the front
-	if (rconn.expirationTime > now()) {
-		connectionPool->pool.push(rconn);
-	} else {
-		++blobStats->expiredConnections;
-	}
-	rconn.conn = Reference<IConnection>();
-}
-
-std::string awsCanonicalURI(const std::string& resource, std::vector<std::string>& queryParameters, bool isV4) {
-	StringRef resourceRef(resource);
-	resourceRef.eat("/");
-	std::string canonicalURI("/" + resourceRef.toString());
-	size_t q = canonicalURI.find_last_of('?');
-	if (q != canonicalURI.npos)
-		canonicalURI.resize(q);
-	if (isV4) {
-		canonicalURI = HTTP::awsV4URIEncode(canonicalURI, false);
-	} else {
-		canonicalURI = HTTP::urlEncode(canonicalURI);
-	}
-
-	// Create the canonical query string
-	std::string queryString;
-	q = resource.find_last_of('?');
-	if (q != queryString.npos)
-		queryString = resource.substr(q + 1);
-
-	StringRef qStr(queryString);
-	StringRef queryParameter;
-	while ((queryParameter = qStr.eat("&")) != StringRef()) {
-		StringRef param = queryParameter.eat("=");
-		StringRef value = queryParameter.eat();
-
-		if (isV4) {
-			queryParameters.push_back(HTTP::awsV4URIEncode(param.toString(), true) + "=" +
-			                          HTTP::awsV4URIEncode(value.toString(), true));
-		} else {
-			queryParameters.push_back(HTTP::urlEncode(param.toString()) + "=" + HTTP::urlEncode(value.toString()));
-		}
-	}
-
-	return canonicalURI;
-}
-
-// ref: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-std::string parseErrorCodeFromS3(std::string xmlResponse) {
-	// Copy XML string to a modifiable buffer
-	try {
-		std::vector<char> xmlBuffer(xmlResponse.begin(), xmlResponse.end());
-		xmlBuffer.push_back('\0'); // Ensure null-terminated string
-		// Parse the XML
-		xml_document<> doc;
-		doc.parse<0>(&xmlBuffer[0]);
-		// Find the root node
-		xml_node<>* root = doc.first_node("Error");
-		if (!root) {
-			TraceEvent(SevWarn, "ParseS3XMLResponseNoError").detail("Response", xmlResponse).log();
-			return "";
-		}
-		// Find the <Code> node
-		xml_node<>* codeNode = root->first_node("Code");
-		if (!codeNode) {
-			TraceEvent(SevWarn, "ParseS3XMLResponseNoErrorCode").detail("Response", xmlResponse).log();
-			return "";
-		}
-		return std::string(codeNode->value());
-	} catch (Error e) {
-		TraceEvent("BackupParseS3ErrorCodeFailure").errorUnsuppressed(e);
-		throw backup_parse_s3_response_failure();
-	} catch (...) {
-		throw backup_parse_s3_response_failure();
-	}
-}
-
-bool isS3TokenError(const std::string& s3Error) {
-	return s3Error == "InvalidToken" || s3Error == "ExpiredToken";
-}
-
-void setHeaders(Reference<S3BlobStoreEndpoint> bstore, Reference<HTTP::OutgoingRequest> req) {
-	// Finish/update the request headers (which includes Date header)
-	// This must be done AFTER the connection is ready because if credentials are coming from disk they are
-	// refreshed when a new connection is established and setAuthHeaders() would need the updated secret.
-	if (bstore->credentials.present() && !bstore->credentials.get().securityToken.empty())
-		req->data.headers["x-amz-security-token"] = bstore->credentials.get().securityToken;
-	if (CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER) {
-		bstore->setV4AuthHeaders(req->verb, req->resource, req->data.headers);
-	} else {
-		bstore->setAuthHeaders(req->verb, req->resource, req->data.headers);
-	}
-}
-
-std::string getCanonicalURI(Reference<S3BlobStoreEndpoint> bstore, Reference<HTTP::OutgoingRequest> req) {
-	std::vector<std::string> queryParameters;
-	std::string canonicalURI =
-	    awsCanonicalURI(req->resource, queryParameters, CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER);
-	if (!queryParameters.empty()) {
-		canonicalURI += "?";
-		// Use raw ampersands when joining query parameters
-		for (size_t i = 0; i < queryParameters.size(); ++i) {
-			if (i > 0) {
-				canonicalURI += "&";
-			}
-			canonicalURI += queryParameters[i];
-		}
-	}
-
-	if (bstore->useProxy && bstore->knobs.secure_connection == 0) {
-		// Has to be in absolute-form.
-		canonicalURI = "http://" + bstore->host + ":" + bstore->service + canonicalURI;
-	}
-	return canonicalURI;
-}
-
-void populateDryrunRequest(Reference<HTTP::OutgoingRequest> dryrunRequest,
-                           Reference<S3BlobStoreEndpoint> bstore,
-                           std::string bucket) {
-	// dryrun with a check bucket exist request, to avoid sending duplicate data
-	HTTP::Headers headers;
-	dryrunRequest->verb = "GET";
-	dryrunRequest->data.contentLen = 0;
-	dryrunRequest->data.headers = headers;
-	dryrunRequest->data.headers["Host"] = bstore->host;
-	dryrunRequest->data.headers["Accept"] = "application/xml";
-
-	dryrunRequest->resource = bstore->constructResourcePath(bucket, "");
-}
-
-bool isWriteRequest(std::string verb) {
-	return verb == "POST" || verb == "PUT";
-}
-
-std::string parseBucketFromURI(std::string uri) {
-	if (uri.size() <= 1 || uri[0] != '/') {
-		// there is no bucket in the uri
-		return "";
-	}
-	uri = uri.substr(1);
-	size_t secondSlash = uri.find('/');
-	if (secondSlash == std::string::npos) {
-		return uri;
-	}
-	return uri.substr(0, secondSlash);
-}
-
-// Do a request, get a Response.
-// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
-// itself must live for the life of this actor and be destroyed by the caller
-ACTOR Future<Reference<HTTP::IncomingResponse>> doRequest_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                               std::string verb,
-                                                               std::string resource,
-                                                               HTTP::Headers headers,
-                                                               UnsentPacketQueue* pContent,
-                                                               int contentLen,
-                                                               std::set<unsigned int> successCodes) {
-
-	state UnsentPacketQueue contentCopy;
-	state UnsentPacketQueue dryrunContentCopy; // NonCopyable state var so must be declared at top of actor
-	state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
-	state Reference<HTTP::OutgoingRequest> dryrunRequest = makeReference<HTTP::OutgoingRequest>();
-	req->verb = verb;
-	req->data.content = &contentCopy;
-	req->data.contentLen = contentLen;
-
-	req->data.headers = headers;
-	req->data.headers["Host"] = bstore->host;
-	req->data.headers["Accept"] = "application/xml";
-
-	// Avoid to send request with an empty resource.
-	if (resource.empty()) {
-		resource = "/";
-	}
-
-	// Merge extraHeaders into headers
-	for (const auto& [k, v] : bstore->extraHeaders) {
-		std::string& fieldValue = req->data.headers[k];
-		if (!fieldValue.empty()) {
-			fieldValue.append(",");
-		}
-		fieldValue.append(v);
-	}
-
-	// For requests with content to upload, the request timeout should be at least twice the amount of time
-	// it would take to upload the content given the upload bandwidth and concurrency limits.
-	int bandwidthThisRequest = 1 + bstore->knobs.max_send_bytes_per_second / bstore->knobs.concurrent_uploads;
-	int contentUploadSeconds = contentLen / bandwidthThisRequest;
-	state int requestTimeout = std::max(bstore->knobs.request_timeout_min, 3 * contentUploadSeconds);
-
-	wait(bstore->concurrentRequests.take());
-	state FlowLock::Releaser globalReleaser(bstore->concurrentRequests, 1);
-
-	state int maxTries = std::min(bstore->knobs.request_tries, bstore->knobs.connect_tries);
-	state int thisTry = 1;
-	state int badRequestCode = 400;
-	state bool s3TokenError = false;
-	state double nextRetryDelay = 2.0;
-
-	loop {
-		state Optional<Error> err;
-		state Optional<NetworkAddress> remoteAddress;
-		state bool connectionEstablished = false;
-		state Reference<HTTP::IncomingResponse> r;
-		state Reference<HTTP::IncomingResponse> dryrunR;
-		state std::string canonicalURI = resource;
-		// Set the resource on each loop so we don't double-encode when we set it to `getCanonicalURI` below.
-		req->resource = resource;
-
-		// Reset headers to initial state for this retry attempt to prevent header accumulation
-		// across retries and potential map corruption
-		req->data.headers = headers;
-		req->data.headers["Host"] = bstore->host;
-		req->data.headers["Accept"] = "application/xml";
-		// Re-merge extraHeaders
-		for (const auto& [k, v] : bstore->extraHeaders) {
-			std::string& fieldValue = req->data.headers[k];
-			if (!fieldValue.empty()) {
-				fieldValue.append(",");
-			}
-			fieldValue.append(v);
-		}
-
-		state UID connID = UID();
-		state double reqStartTimer;
-		state double connectStartTimer = g_network->timer();
-		state bool reusingConn = false;
-		state bool fastRetry = false;
-		state bool simulateS3TokenError = false;
-		state S3BlobStoreEndpoint::ReusableConnection rconn; // Moved outside try block for error handler access
-
-		try {
-			// Start connecting
-			Future<S3BlobStoreEndpoint::ReusableConnection> frconn = bstore->connect(&reusingConn);
-
-			// Make a shallow copy of the queue by calling addref() on each buffer in the chain and then prepending that
-			// chain to contentCopy
-			req->data.content->discardAll();
-			if (pContent != nullptr) {
-				PacketBuffer* pFirst = pContent->getUnsent();
-				PacketBuffer* pLast = nullptr;
-				for (PacketBuffer* p = pFirst; p != nullptr; p = p->nextPacketBuffer()) {
-					p->addref();
-					// Also reset the sent count on each buffer
-					p->bytes_sent = 0;
-					pLast = p;
-				}
-				req->data.content->prependWriteBuffer(pFirst, pLast);
-			}
-
-			// Finish connecting, do request
-			wait(store(rconn, timeoutError(frconn, bstore->knobs.connect_timeout)));
-			connectionEstablished = true;
-			connID = rconn.conn->getDebugID();
-			reqStartTimer = g_network->timer();
-			TraceEvent(SevDebug, "S3BlobStoreEndpointConnected")
-			    .suppressFor(60)
-			    .detail("RemoteEndpoint", rconn.conn->getPeerAddress())
-			    .detail("Reusing", reusingConn)
-			    .detail("ConnID", connID)
-			    .detail("Verb", req->verb)
-			    .detail("Resource", resource)
-			    .detail("Proxy", bstore->proxyHost.orDefault(""));
-
-			try {
-				if (s3TokenError && isWriteRequest(req->verb) && CLIENT_KNOBS->BACKUP_ALLOW_DRYRUN) {
-					// if it is a write request with s3TokenError, retry with a HEAD dryrun request
-					// to avoid sending duplicate data indefinitly to save network bandwidth
-					// because it might due to expired or invalid S3 token from the disk
-					state std::string bucket = parseBucketFromURI(resource);
-					if (bucket.empty()) {
-						TraceEvent(SevError, "EmptyBucketRequest")
-						    .detail("S3TokenError", s3TokenError)
-						    .detail("Verb", req->verb)
-						    .detail("Resource", resource)
-						    .log();
-						throw bucket_not_in_url();
-					}
-					dryrunRequest->data.content = &dryrunContentCopy;
-					dryrunRequest->data.content->discardAll(); // this should always be empty
-					populateDryrunRequest(dryrunRequest, bstore, bucket);
-					setHeaders(bstore, dryrunRequest);
-					dryrunRequest->resource = getCanonicalURI(bstore, dryrunRequest);
-					TraceEvent("RetryS3RequestDueToTokenIssue")
-					    .detail("S3TokenError", s3TokenError)
-					    .detail("OriginalResource", resource)
-					    .detail("DryrunResource", dryrunRequest->resource)
-					    .detail("Bucket", bucket)
-					    .detail("V4", CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER)
-					    .log();
-					wait(bstore->requestRate->getAllowance(1));
-					Future<Reference<HTTP::IncomingResponse>> dryrunResponse = HTTP::doRequest(
-					    rconn.conn, dryrunRequest, bstore->sendRate, &bstore->s_stats.bytes_sent, bstore->recvRate);
-					Reference<HTTP::IncomingResponse> _dryrunR = wait(timeoutError(dryrunResponse, requestTimeout));
-					dryrunR = _dryrunR;
-					// Only parse S3 error code for error responses (4xx/5xx), not successful responses (2xx)
-					std::string s3Error;
-					if (dryrunR->code >= 400) {
-						s3Error = parseErrorCodeFromS3(dryrunR->data.content);
-					}
-					if (dryrunR->code == badRequestCode && isS3TokenError(s3Error)) {
-						// authentication fails and s3 token error persists, retry with a HEAD dryrun request
-						// to avoid sending duplicate data indefinitly to save network bandwidth
-						// because it might be due to expired or invalid S3 token from the disk
-						wait(delay(bstore->knobs.max_delay_retryable_error));
-					} else if (dryrunR->code == 200 || dryrunR->code == 404) {
-						// authentication has passed, and bucket existence has been verified(200 or 404)
-						// it might work now(or it might be another error) thus reset s3TokenError.
-						TraceEvent("S3TokenIssueResolved")
-						    .detail("HttpCode", dryrunR->code)
-						    .detail("HttpResponseContent", dryrunR->data.content)
-						    .detail("S3Error", s3Error)
-						    .detail("URI", dryrunRequest->resource)
-						    .log();
-						s3TokenError = false;
-					} else {
-						TraceEvent(SevError, "S3UnexpectedError")
-						    .detail("HttpCode", dryrunR->code)
-						    .detail("HttpResponseContent", dryrunR->data.content)
-						    .detail("S3Error", s3Error)
-						    .detail("URI", dryrunRequest->resource)
-						    .log();
-						throw http_bad_response();
-					}
-					continue;
-				}
-			} catch (Error& e) {
-				// retry with GET failed, but continue to do original request anyway
-				if (e.code() != error_code_actor_cancelled) {
-					TraceEvent(SevError, "ErrorDuringRetryS3TokenIssue").errorUnsuppressed(e);
-				} else {
-					throw;
-				}
-			}
-			setHeaders(bstore, req);
-			req->resource = getCanonicalURI(bstore, req);
-
-			remoteAddress = rconn.conn->getPeerAddress();
-			wait(bstore->requestRate->getAllowance(1));
-
-			state Future<Reference<HTTP::IncomingResponse>> reqF =
-			    HTTP::doRequest(rconn.conn, req, bstore->sendRate, &bstore->s_stats.bytes_sent, bstore->recvRate);
-			// if we reused a connection from the pool, and immediately got an error, retry immediately discarding
-			// the connection
-			if (reqF.isReady() && reusingConn) {
-				fastRetry = true;
-			}
-
-			Reference<HTTP::IncomingResponse> _r = wait(timeoutError(reqF, requestTimeout));
-			// Don't simulate token errors for multipart complete operations (POST with uploadId but no partNumber)
-			// because changing a successful 200 to 400 after the server has already completed and removed
-			// the upload causes the client to infinitely retry with a phantom upload ID. This seems too much of
-			// an artifical manufacture.
-			bool isMultipartComplete = verb == "POST" && resource.find("uploadId=") != std::string::npos &&
-			                           resource.find("partNumber=") == std::string::npos;
-			if (g_network->isSimulated() && BUGGIFY && deterministicRandom()->random01() < 0.1 &&
-			    !isMultipartComplete) {
-				// simulate an error from s3
-				_r->code = badRequestCode;
-				simulateS3TokenError = true;
-			}
-			r = _r;
-
-			// Since the response was parsed successfully (which is why we are here) reuse the connection unless we
-			// received the "Connection: close" header.
-			if (r->data.headers["Connection"] != "close") {
-				bstore->returnConnection(rconn);
-			} else {
-				++bstore->blobStats->expiredConnections;
-			}
-			rconn.conn.clear();
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "S3BlobStoreDoRequestError")
-			    .errorUnsuppressed(e)
-			    .detail("Verb", verb)
-			    .detail("Resource", resource);
-			// Close the connection on error to satisfy Sim2Conn's assertion in simulation.
-			// If rconn holds a connection that hasn't been returned to the pool,
-			// we must close it before it's destroyed (when rconn goes out of scope).
-			// This sets closedByCaller = true in Sim2Conn, preventing assertion failures.
-			if (connectionEstablished && rconn.conn.isValid()) {
-				rconn.conn->close();
-				rconn.conn.clear();
-			}
-			if (e.code() == error_code_actor_cancelled)
-				throw;
-			err = e;
-		}
-
-		double end = g_network->timer();
-		double connectDuration = reqStartTimer - connectStartTimer;
-		double reqDuration = end - reqStartTimer;
-		bstore->blobStats->requestLatency.addMeasurement(reqDuration);
-
-		// If err is not present then r is valid.
-		// If r->code is in successCodes then record the successful request and return r.
-		if (!err.present() && successCodes.count(r->code) != 0) {
-			TraceEvent(SevDebug, "S3BlobStoreDoRequestSuccessful")
-			    .detail("Verb", verb)
-			    .detail("Error", err.present())
-			    .detail("ErrorString", err.present() ? err.get().name() : "")
-			    .detail("Resource", resource)
-			    .detail("ResponseCode", r->code)
-			    .detail("ResponseContentSize", r->data.content.size())
-			    .log();
-			bstore->s_stats.requests_successful++;
-			++bstore->blobStats->requestsSuccessful;
-			return r;
-		}
-
-		// Otherwise, this request is considered failed.  Update failure count.
-		bstore->s_stats.requests_failed++;
-		++bstore->blobStats->requestsFailed;
-
-		// All errors in err are potentially retryable as well as certain HTTP response codes...
-		bool retryable = err.present() || r->code == 500 || r->code == 502 || r->code == 503 || r->code == 429;
-		// But only if our previous attempt was not the last allowable try.
-		retryable = retryable && (thisTry < maxTries);
-
-		if (!retryable || !err.present()) {
-			fastRetry = false;
-		}
-
-		TraceEvent event(SevWarn,
-		                 retryable ? (fastRetry ? "S3BlobStoreEndpointRequestFailedFastRetryable"
-		                                        : "S3BlobStoreEndpointRequestFailedRetryable")
-		                           : "S3BlobStoreEndpointRequestFailed");
-
-		bool connectionFailed = false;
-		// Attach err to trace event if present, otherwise extract some stuff from the response
-		if (err.present()) {
-			event.errorUnsuppressed(err.get());
-			if (err.get().code() == error_code_connection_failed) {
-				connectionFailed = true;
-			}
-		}
-		event.suppressFor(60);
-
-		if (!err.present()) {
-			event.detail("ResponseCode", r->code);
-			// Only parse S3 error code for real error responses (4xx/5xx), not successful responses (2xx)
-			// Skip parsing for simulated errors where response content is still binary data
-			std::string s3Error;
-			if (r->code >= 400 && !simulateS3TokenError) {
-				s3Error = parseErrorCodeFromS3(r->data.content);
-			}
-			event.detail("S3ErrorCode", s3Error);
-			if (r->code == badRequestCode) {
-				if (isS3TokenError(s3Error) || simulateS3TokenError) {
-					s3TokenError = true;
-				}
-				TraceEvent(SevWarnAlways, "S3BlobStoreBadRequest")
-				    .detail("HttpCode", r->code)
-				    .detail("HttpResponseContent", r->data.content)
-				    .detail("S3Error", s3Error)
-				    .log();
-			}
-		}
-
-		event.detail("ConnectionEstablished", connectionEstablished);
-		event.detail("ReusingConn", reusingConn);
-		if (connectionEstablished) {
-			event.detail("ConnID", connID);
-			event.detail("ConnectDuration", connectDuration);
-			event.detail("ReqDuration", reqDuration);
-		}
-
-		if (remoteAddress.present())
-			event.detail("RemoteEndpoint", remoteAddress.get());
-		else
-			event.detail("RemoteHost", bstore->host);
-
-		event.detail("Verb", verb)
-		    .detail("Resource", resource)
-		    .detail("ThisTry", thisTry)
-		    .detail("URI", canonicalURI)
-		    .detail("Proxy", bstore->proxyHost.orDefault(""));
-
-		// If r is not valid or not code 429 then increment the try count.  429's will not count against the attempt
-		// limit. Also skip incrementing the retry count for fast retries
-		if (!fastRetry && (!r || r->code != 429))
-			++thisTry;
-
-		if (fastRetry) {
-			++bstore->blobStats->fastRetries;
-			wait(delay(0));
-		} else if (retryable || s3TokenError) {
-			// We will wait delay seconds before the next retry, start with nextRetryDelay.
-			double delay = nextRetryDelay;
-			// conenctionFailed is treated specially as we know proxy to AWS can only serve 1 request per connection
-			// so there is no point of waiting too long, instead retry more aggressively
-			double limit =
-			    connectionFailed ? bstore->knobs.max_delay_connection_failed : bstore->knobs.max_delay_retryable_error;
-			// Double but limit the *next* nextRetryDelay.
-			nextRetryDelay = std::min(nextRetryDelay * 2, limit);
-			// If r is valid then obey the Retry-After response header if present.
-			if (r) {
-				auto iRetryAfter = r->data.headers.find("Retry-After");
-				if (iRetryAfter != r->data.headers.end()) {
-					event.detail("RetryAfterHeader", iRetryAfter->second);
-					char* pEnd;
-					double retryAfter = strtod(iRetryAfter->second.c_str(), &pEnd);
-					if (*pEnd) // If there were other characters then don't trust the parsed value, use a probably safe
-					           // value of 5 minutes.
-						retryAfter = 300;
-					// Update delay
-					delay = std::max(delay, retryAfter);
-				}
-			}
-			// Log the delay then wait.
-
-			event.detail("RetryDelay", delay);
-			wait(::delay(delay));
-		} else {
-			// We can't retry, so throw something.
-
-			// This error code means the authentication header was not accepted, likely the account or key is wrong.
-			if (r && r->code == 406)
-				throw http_not_accepted();
-
-			if (r && r->code == 401)
-				throw http_auth_failed();
-
-			// Recognize and throw specific errors
-			if (err.present()) {
-				int code = err.get().code();
-
-				// If we get a timed_out error during the the connect() phase, we'll call that connection_failed despite
-				// the fact that there was technically never a 'connection' to begin with.  It differentiates between an
-				// active connection timing out vs a connection timing out, though not between an active connection
-				// failing vs connection attempt failing.
-				// TODO:  Add more error types?
-				if (code == error_code_timed_out && !connectionEstablished) {
-					TraceEvent(SevWarn, "S3BlobStoreEndpointConnectTimeout")
-					    .suppressFor(60)
-					    .detail("Timeout", requestTimeout);
-					throw connection_failed();
-				}
-
-				if (code == error_code_timed_out || code == error_code_connection_failed ||
-				    code == error_code_lookup_failed)
-					throw err.get();
-			}
-
-			throw http_request_failed();
-		}
-	}
-}
-
-Future<Reference<HTTP::IncomingResponse>> S3BlobStoreEndpoint::doRequest(std::string const& verb,
-                                                                         std::string const& resource,
-                                                                         const HTTP::Headers& headers,
-                                                                         UnsentPacketQueue* pContent,
-                                                                         int contentLen,
-                                                                         std::set<unsigned int> successCodes) {
-	return doRequest_impl(
-	    Reference<S3BlobStoreEndpoint>::addRef(this), verb, resource, headers, pContent, contentLen, successCodes);
-}
-
-ACTOR Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                          std::string bucket,
-                                          PromiseStream<S3BlobStoreEndpoint::ListResult> results,
-                                          Optional<std::string> prefix,
-                                          Optional<char> delimiter,
-                                          int maxDepth,
-                                          std::function<bool(std::string const&)> recurseFilter) {
-	state std::string resource = bstore->constructResourcePath(bucket, "");
-	// In virtual hosting mode, constructResourcePath returns "" for empty object.
-	// We need to ensure the query string starts with "/" to form a valid HTTP request.
-	// Commit 15dd76a7f9 switched from ListObjectsV1 to V2 and accidentally removed the leading "/".
-	if (resource.empty()) {
-		resource = "/";
-	}
-	resource.append("?list-type=2&max-keys=").append(std::to_string(CLIENT_KNOBS->BLOBSTORE_LIST_MAX_KEYS_PER_PAGE));
-
-	if (prefix.present())
-		resource.append("&prefix=").append(prefix.get());
-	if (delimiter.present())
-		resource.append("&delimiter=").append(std::string(1, delimiter.get()));
-
-	state std::string continuationToken;
-	state bool more = true;
-	state std::vector<Future<Void>> subLists;
-
-	while (more) {
-		wait(bstore->concurrentLists.take());
-		state FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
-
-		state HTTP::Headers headers;
-		state std::string fullResource = resource;
-		if (!continuationToken.empty()) {
-			fullResource.append("&continuation-token=").append(continuationToken);
-		}
-
-		Reference<HTTP::IncomingResponse> r =
-		    wait(doRequest_impl(bstore, "GET", fullResource, headers, nullptr, 0, { 200, 404 }));
-		listReleaser.release();
-
-		try {
-			S3BlobStoreEndpoint::ListResult listResult;
-
-			// If we got a 404, throw an error to indicate the resource doesn't exist
-			if (r->code == 404) {
-				TraceEvent(SevError, "S3BlobStoreResourceNotFound")
-				    .detail("Bucket", bucket)
-				    .detail("Prefix", prefix.present() ? prefix.get() : "")
-				    .detail("Resource", fullResource);
-				throw resource_not_found();
-			}
-
-			xml_document<> doc;
-
-			// Copy content because rapidxml will modify it during parse
-			std::string content = r->data.content;
-			doc.parse<0>((char*)content.c_str());
-
-			// There should be exactly one node
-			xml_node<>* result = doc.first_node();
-			if (result == nullptr || strcmp(result->name(), "ListBucketResult") != 0) {
-				throw http_bad_response();
-			}
-
-			xml_node<>* n = result->first_node();
-			more = false;
-			continuationToken.clear();
-
-			while (n != nullptr) {
-				const char* name = n->name();
-				if (strcmp(name, "IsTruncated") == 0) {
-					const char* val = n->value();
-					if (strcmp(val, "true") == 0) {
-						more = true;
-					} else if (strcmp(val, "false") == 0) {
-						more = false;
-					} else {
-						throw http_bad_response();
-					}
-				} else if (strcmp(name, "NextContinuationToken") == 0) {
-					if (n->value() != nullptr) {
-						continuationToken = n->value();
-					}
-				} else if (strcmp(name, "Contents") == 0) {
-					S3BlobStoreEndpoint::ObjectInfo object;
-
-					xml_node<>* key = n->first_node("Key");
-					if (key == nullptr) {
-						throw http_bad_response();
-					}
-					// URL decode the object name since S3 XML responses contain URL-encoded names
-					object.name = HTTP::urlDecode(key->value());
-
-					xml_node<>* size = n->first_node("Size");
-					if (size == nullptr) {
-						throw http_bad_response();
-					}
-					object.size = strtoull(size->value(), nullptr, 10);
-
-					listResult.objects.push_back(object);
-				} else if (strcmp(name, "CommonPrefixes") == 0) {
-					xml_node<>* prefixNode = n->first_node("Prefix");
-					while (prefixNode != nullptr) {
-						const char* prefix = prefixNode->value();
-						// If recursing, queue a sub-request, otherwise add the common prefix to the result.
-						if (maxDepth > 0) {
-							if (!recurseFilter || recurseFilter(prefix)) {
-								// For recursive listing, don't use delimiter in sub-requests to get individual files
-								subLists.push_back(bstore->listObjectsStream(
-								    bucket, results, prefix, Optional<char>(), maxDepth - 1, recurseFilter));
-							}
-						} else {
-							listResult.commonPrefixes.push_back(prefix);
-						}
-						prefixNode = prefixNode->next_sibling("Prefix");
-					}
-				}
-				n = n->next_sibling();
-			}
-
-			results.send(listResult);
-		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevWarn, "S3BlobStoreEndpointListResultParseError")
-				    .errorUnsuppressed(e)
-				    .suppressFor(60)
-				    .detail("Resource", fullResource);
-			}
-			throw http_bad_response();
-		}
-	}
-
-	wait(waitForAll(subLists));
-	return Void();
-}
-
-Future<Void> S3BlobStoreEndpoint::listObjectsStream(std::string const& bucket,
-                                                    PromiseStream<ListResult> results,
-                                                    Optional<std::string> prefix,
-                                                    Optional<char> delimiter,
-                                                    int maxDepth,
-                                                    std::function<bool(std::string const&)> recurseFilter) {
-	return listObjectsStream_impl(
-	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, results, prefix, delimiter, maxDepth, recurseFilter);
-}
-
-ACTOR Future<S3BlobStoreEndpoint::ListResult> listObjects_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                               std::string bucket,
-                                                               Optional<std::string> prefix,
-                                                               Optional<char> delimiter,
-                                                               int maxDepth,
-                                                               std::function<bool(std::string const&)> recurseFilter) {
-	state S3BlobStoreEndpoint::ListResult results;
-	state PromiseStream<S3BlobStoreEndpoint::ListResult> resultStream;
-	state Future<Void> done =
-	    bstore->listObjectsStream(bucket, resultStream, prefix, delimiter, maxDepth, recurseFilter);
-	// Wrap done in an actor which sends end_of_stream because list does not so that many lists can write to the same
-	// stream
-	done = map(done, [=](Void) mutable {
-		resultStream.sendError(end_of_stream());
-		return Void();
-	});
-
-	try {
-		loop {
-			choose {
-				// Throw if done throws, otherwise don't stop until end_of_stream
-				when(wait(done)) {
-					done = Never();
-				}
-
-				when(S3BlobStoreEndpoint::ListResult info = waitNext(resultStream.getFuture())) {
-					results.commonPrefixes.insert(
-					    results.commonPrefixes.end(), info.commonPrefixes.begin(), info.commonPrefixes.end());
-					results.objects.insert(results.objects.end(), info.objects.begin(), info.objects.end());
-				}
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_end_of_stream)
-			throw;
-	}
-
-	return results;
-}
-
-Future<S3BlobStoreEndpoint::ListResult> S3BlobStoreEndpoint::listObjects(
-    std::string const& bucket,
-    Optional<std::string> prefix,
-    Optional<char> delimiter,
-    int maxDepth,
-    std::function<bool(std::string const&)> recurseFilter) {
-	return listObjects_impl(
-	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, prefix, delimiter, maxDepth, recurseFilter);
-}
-
-ACTOR Future<std::vector<std::string>> listBuckets_impl(Reference<S3BlobStoreEndpoint> bstore) {
-	state std::string resource = "/?marker=";
-	state std::string lastName;
-	state bool more = true;
-	state std::vector<std::string> buckets;
-
-	while (more) {
-		wait(bstore->concurrentLists.take());
-		state FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
-
-		state HTTP::Headers headers;
-		state std::string fullResource = resource + lastName;
-		Reference<HTTP::IncomingResponse> r =
-		    wait(doRequest_impl(bstore, "GET", fullResource, headers, nullptr, 0, { 200 }));
-		listReleaser.release();
-
-		try {
-			xml_document<> doc;
-
-			// Copy content because rapidxml will modify it during parse
-			std::string content = r->data.content;
-			doc.parse<0>((char*)content.c_str());
-
-			// There should be exactly one node
-			xml_node<>* result = doc.first_node();
-			if (result == nullptr || strcmp(result->name(), "ListAllMyBucketsResult") != 0) {
-				throw http_bad_response();
-			}
-
-			more = false;
-			xml_node<>* truncated = result->first_node("IsTruncated");
-			if (truncated != nullptr && strcmp(truncated->value(), "true") == 0) {
-				more = true;
-			}
-
-			xml_node<>* bucketsNode = result->first_node("Buckets");
-			if (bucketsNode != nullptr) {
-				xml_node<>* bucketNode = bucketsNode->first_node("Bucket");
-				while (bucketNode != nullptr) {
-					xml_node<>* nameNode = bucketNode->first_node("Name");
-					if (nameNode == nullptr) {
-						throw http_bad_response();
-					}
-					const char* name = nameNode->value();
-					buckets.push_back(name);
-
-					bucketNode = bucketNode->next_sibling("Bucket");
-				}
-			}
-
-			if (more) {
-				lastName = buckets.back();
-			}
-
-		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled)
-				TraceEvent(SevWarn, "S3BlobStoreEndpointListBucketResultParseError")
-				    .errorUnsuppressed(e)
-				    .suppressFor(60)
-				    .detail("Resource", fullResource);
-			throw http_bad_response();
-		}
-	}
-
-	return buckets;
-}
-
-Future<std::vector<std::string>> S3BlobStoreEndpoint::listBuckets() {
-	return listBuckets_impl(Reference<S3BlobStoreEndpoint>::addRef(this));
-}
-
-std::string S3BlobStoreEndpoint::hmac_sha1(Credentials const& creds, std::string const& msg) {
-	std::string key = creds.secret;
-
-	// Hash key to shorten it if it is longer than SHA1 block size
-	if (key.size() > 64) {
-		key = SHA1::from_string(key);
-	}
-
-	// Pad key up to SHA1 block size if needed
-	key.append(64 - key.size(), '\0');
-
-	std::string kipad = key;
-	for (int i = 0; i < 64; ++i)
-		kipad[i] ^= '\x36';
-
-	std::string kopad = key;
-	for (int i = 0; i < 64; ++i)
-		kopad[i] ^= '\x5c';
-
-	kipad.append(msg);
-	std::string hkipad = SHA1::from_string(kipad);
-	kopad.append(hkipad);
-	return SHA1::from_string(kopad);
-}
-
-static void sha256(const unsigned char* data, const size_t len, unsigned char* hash) {
-	SHA256_CTX sha256;
-	SHA256_Init(&sha256);
-	SHA256_Update(&sha256, data, len);
-	SHA256_Final(hash, &sha256);
-}
-
-std::string sha256_hex(std::string str) {
-	unsigned char hash[SHA256_DIGEST_LENGTH];
-	sha256((const unsigned char*)str.c_str(), str.size(), hash);
-	std::stringstream ss;
-	for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) {
-		ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i];
-	}
-	return ss.str();
-}
-
-// Return base64'd SHA256 hash of input string.
-std::string sha256_base64(std::string str) {
-	unsigned char hash[SHA256_DIGEST_LENGTH];
-	sha256((const unsigned char*)str.c_str(), str.size(), hash);
-	std::string hashAsStr = std::string((char*)hash, SHA256_DIGEST_LENGTH);
-	std::string sig = base64::encoder::from_string(hashAsStr);
-	// base64 encoded blocks end in \n so remove last character.
-	sig.resize(sig.size() - 1);
-	return sig;
-}
-
-std::string hmac_sha256_hex(std::string key, std::string msg) {
-	unsigned char hash[32];
-
-	HMAC_CTX* hmac = HMAC_CTX_new();
-	HMAC_Init_ex(hmac, &key[0], key.length(), EVP_sha256(), NULL);
-	HMAC_Update(hmac, (unsigned char*)&msg[0], msg.length());
-	unsigned int len = 32;
-	HMAC_Final(hmac, hash, &len);
-	HMAC_CTX_free(hmac);
-
-	std::stringstream ss;
-	ss << std::hex << std::setfill('0');
-	for (int i = 0; i < len; i++) {
-		ss << std::hex << std::setw(2) << (unsigned int)hash[i];
-	}
-	return (ss.str());
-}
-
-std::string hmac_sha256(std::string key, std::string msg) {
-	unsigned char hash[32];
-
-	HMAC_CTX* hmac = HMAC_CTX_new();
-	HMAC_Init_ex(hmac, &key[0], key.length(), EVP_sha256(), NULL);
-	HMAC_Update(hmac, (unsigned char*)&msg[0], msg.length());
-	unsigned int len = 32;
-	HMAC_Final(hmac, hash, &len);
-	HMAC_CTX_free(hmac);
-
-	std::stringstream ss;
-	ss << std::setfill('0');
-	for (int i = 0; i < len; i++) {
-		ss << hash[i];
-	}
-	return (ss.str());
-}
-
-// Date and Time parameters are used for unit testing
-void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
-                                           std::string const& resource,
-                                           HTTP::Headers& headers,
-                                           std::string date,
-                                           std::string datestamp) {
-	if (!credentials.present()) {
-		return;
-	}
-	Credentials creds = credentials.get();
-	// std::cout << "========== Starting===========" << std::endl;
-	std::string accessKey = creds.key;
-	std::string secretKey = creds.secret;
-	// Create a date for headers and the credential string
-	std::string amzDate;
-	std::string dateStamp;
-	if (date.empty() || datestamp.empty()) {
-		time_t ts;
-		time(&ts);
-		char dateBuf[20];
-		// ISO 8601 format YYYYMMDD'T'HHMMSS'Z'
-		strftime(dateBuf, 20, "%Y%m%dT%H%M%SZ", gmtime(&ts));
-		amzDate = dateBuf;
-		strftime(dateBuf, 20, "%Y%m%d", gmtime(&ts));
-		dateStamp = dateBuf;
-	} else {
-		amzDate = date;
-		dateStamp = datestamp;
-	}
-
-	// ************* TASK 1: CREATE A CANONICAL REQUEST *************
-	// Create Create canonical URI--the part of the URI from domain to query string (use '/' if no path)
-	std::vector<std::string> queryParameters;
-	std::string canonicalURI = awsCanonicalURI(resource, queryParameters, true);
-
-	std::string canonicalQueryString;
-	if (!queryParameters.empty()) {
-		std::sort(queryParameters.begin(), queryParameters.end());
-		canonicalQueryString = boost::algorithm::join(queryParameters, "&");
-	}
-
-	using namespace boost::algorithm;
-	// Create the canonical headers and signed headers
-	ASSERT(!headers["Host"].empty());
-	// Be careful. There is x-amz-content-sha256 for auth and then
-	// x-amz-checksum-sha256 for object integrity check.
-	headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD";
-	headers["x-amz-date"] = amzDate;
-	std::vector<std::pair<std::string, std::string>> headersList;
-	headersList.push_back({ "host", trim_copy(headers["Host"]) + "\n" });
-	if (headers.find("Content-Type") != headers.end())
-		headersList.push_back({ "content-type", trim_copy(headers["Content-Type"]) + "\n" });
-	if (headers.find("Content-MD5") != headers.end())
-		headersList.push_back({ "content-md5", trim_copy(headers["Content-MD5"]) + "\n" });
-	for (auto h : headers) {
-		if (StringRef(h.first).startsWith("x-amz"_sr))
-			headersList.push_back({ to_lower_copy(h.first), trim_copy(h.second) + "\n" });
-	}
-	std::sort(headersList.begin(), headersList.end());
-	std::string canonicalHeaders;
-	std::string signedHeaders;
-	for (auto& i : headersList) {
-		canonicalHeaders += i.first + ":" + i.second;
-		signedHeaders += i.first + ";";
-	}
-	signedHeaders.pop_back();
-	std::string canonicalRequest = verb + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders +
-	                               "\n" + signedHeaders + "\n" + headers["x-amz-content-sha256"];
-
-	// ************* TASK 2: CREATE THE STRING TO SIGN*************
-	std::string algorithm = "AWS4-HMAC-SHA256";
-	std::string credentialScope = dateStamp + "/" + region + "/s3/" + "aws4_request";
-	std::string stringToSign =
-	    algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + sha256_hex(canonicalRequest);
-
-	// ************* TASK 3: CALCULATE THE SIGNATURE *************
-	// Create the signing key using the function defined above.
-	std::string signingKey =
-	    hmac_sha256(hmac_sha256(hmac_sha256(hmac_sha256("AWS4" + secretKey, dateStamp), region), "s3"), "aws4_request");
-	// Sign the string_to_sign using the signing_key
-	std::string signature = hmac_sha256_hex(signingKey, stringToSign);
-	// ************* TASK 4: ADD SIGNING INFORMATION TO THE Header *************
-	std::string authorizationHeader = algorithm + " " + "Credential=" + accessKey + "/" + credentialScope + ", " +
-	                                  "SignedHeaders=" + signedHeaders + ", " + "Signature=" + signature;
-	headers["Authorization"] = authorizationHeader;
-}
-
-void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) {
-	if (!credentials.present()) {
-		return;
-	}
-	Credentials creds = credentials.get();
-
-	std::string& date = headers["Date"];
-
-	char dateBuf[64];
-	time_t ts;
-	time(&ts);
-	strftime(dateBuf, 64, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&ts));
-	date = dateBuf;
-
-	std::string msg;
-	msg.append(verb);
-	msg.append("\n");
-	auto contentMD5 = headers.find("Content-MD5");
-	if (contentMD5 != headers.end())
-		msg.append(contentMD5->second);
-	msg.append("\n");
-	auto contentType = headers.find("Content-Type");
-	if (contentType != headers.end())
-		msg.append(contentType->second);
-	msg.append("\n");
-	msg.append(date);
-	msg.append("\n");
-	for (auto h : headers) {
-		StringRef name = h.first;
-		if (name.startsWith("x-amz"_sr) || name.startsWith("x-icloud"_sr)) {
-			msg.append(h.first);
-			msg.append(":");
-			msg.append(h.second);
-			msg.append("\n");
-		}
-	}
-
-	msg.append(resource);
-	if (verb == "GET") {
-		size_t q = resource.find_last_of('?');
-		if (q != resource.npos)
-			msg.resize(msg.size() - (resource.size() - q));
-	}
-
-	std::string sig = base64::encoder::from_string(hmac_sha1(creds, msg));
-	// base64 encoded blocks end in \n so remove it.
-	sig.resize(sig.size() - 1);
-	std::string auth = "AWS ";
-	auth.append(creds.key);
-	auth.append(":");
-	auth.append(sig);
-	headers["Authorization"] = auth;
-}
-
-ACTOR Future<std::string> readEntireFile_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                              std::string bucket,
-                                              std::string object) {
-	wait(bstore->requestRateRead->getAllowance(1));
-
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	state HTTP::Headers headers;
-	// Set this header on the GET for it to volunteer saved checksum in the response headers.
-	// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax
-	headers["x-amz-checksum-mode"] = "ENABLED";
-	Reference<HTTP::IncomingResponse> r =
-	    wait(doRequest_impl(bstore, "GET", resource, headers, nullptr, 0, { 200, 404 }));
-	if (r->code == 404)
-		throw file_not_found();
-	if (bstore->knobs.enable_object_integrity_check) {
-		// Verify the content. We set 'x-amz-checksum-mode' on the GET request above so
-		// the server will return the sha256 checksum we set when we uploaded the object in the
-		// GET response headers. See
-		// https://stackoverflow.com/questions/36540234/what-is-the-difference-getcontentmd5-and-getetag-of-aws-s3-putobjectresult
-		std::string checksumSHA256 = r->data.headers["x-amz-checksum-sha256"];
-		if (checksumSHA256.empty()) {
-			// This is what is thrown elsewhere when no expected etag.
-			throw http_bad_response();
-		}
-		// Calculate the sha256 checksum of the content and compare it to the checksum returned by the server.
-		std::string contentSHA256 = sha256_base64(r->data.content);
-		if (checksumSHA256 != contentSHA256) {
-			throw checksum_failed();
-		}
-	}
-	return r->data.content;
-}
-
-Future<std::string> S3BlobStoreEndpoint::readEntireFile(std::string const& bucket, std::string const& object) {
-	return readEntireFile_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
-}
-
-ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                  std::string bucket,
-                                                  std::string object,
-                                                  UnsentPacketQueue* pContent,
-                                                  int contentLen,
-                                                  std::string contentHash) {
-	if (contentLen > bstore->knobs.multipart_max_part_size)
-		throw file_too_large();
-
-	wait(bstore->requestRateWrite->getAllowance(1));
-	wait(bstore->concurrentUploads.take());
-	state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
-
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	state HTTP::Headers headers;
-	// contentHash is calculated by the caller. It is md5 or sha256 dependent on
-	// enable_object_integrity_check setting. If the hash (md5 or sha256) we
-	// volunteer doesn't match that calculated serverside, the upload fails with:
-	// InvalidDigest</Code><Message>The Content-MD5 you specified was invalid.</Message>
-	if (bstore->knobs.enable_object_integrity_check) {
-		headers["x-amz-checksum-sha256"] = contentHash;
-		headers["x-amz-checksum-algorithm"] = "SHA256";
-	} else {
-		headers["Content-MD5"] = contentHash;
-	}
-	if (!CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE.empty())
-		headers["x-amz-server-side-encryption"] = CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE;
-	Reference<HTTP::IncomingResponse> r =
-	    wait(doRequest_impl(bstore, "PUT", resource, headers, pContent, contentLen, { 200 }));
-	return Void();
-}
-
-ACTOR Future<Void> writeEntireFile_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                        std::string bucket,
-                                        std::string object,
-                                        std::string content) {
-	state UnsentPacketQueue packets;
-	if (content.size() > bstore->knobs.multipart_max_part_size)
-		throw file_too_large();
-
-	PacketWriter pw(packets.getWriteBuffer(content.size()), nullptr, Unversioned());
-	pw.serializeBytes(content);
-
-	// Yield because we may have just had to copy several MB's into packet buffer chain and next we have to calculate an
-	// MD5 sum of it.
-	// TODO:  If this actor is used to send large files then combine the summing and packetization into a loop with a
-	// yield() every 20k or so.
-	wait(yield());
-
-	// If enable_object_integrity_check is true, calculate the sha256 sum of the content.
-	// Otherwise, do md5. Save the calculated hash to contentHash. Whichever, when we
-	// upload to the server, it will check the hash -- md5 or sha256 -- and if a mismatch,
-	// the upload will fail. The difference is that sha256 is a better hash than md5 and
-	// when enable_object_integrity_check is set, we will verify the sha256 hash of the
-	// content when we download it too; we do not do this latter when
-	// enable_object_integrity_check is false (the etag returned in the GET response is an
-	// md5 most of the time but NOT always so it can't be relied upon. See
-	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
-	std::string contentHash;
-	if (CLIENT_KNOBS->BLOBSTORE_ENABLE_OBJECT_INTEGRITY_CHECK) {
-		// If the content is small enough to fit in a single packet then we can calculate the sha256 sum now.
-		contentHash = sha256_base64(content);
-	} else {
-		MD5_CTX sum;
-		::MD5_Init(&sum);
-		::MD5_Update(&sum, content.data(), content.size());
-		std::string sumBytes;
-		sumBytes.resize(16);
-		::MD5_Final((unsigned char*)sumBytes.data(), &sum);
-		contentHash = base64::encoder::from_string(sumBytes);
-		contentHash.resize(contentHash.size() - 1);
-	}
-
-	wait(writeEntireFileFromBuffer_impl(bstore, bucket, object, &packets, content.size(), contentHash));
-	return Void();
-}
-
-Future<Void> S3BlobStoreEndpoint::writeEntireFile(std::string const& bucket,
-                                                  std::string const& object,
-                                                  std::string const& content) {
-	return writeEntireFile_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, content);
-}
-
-Future<Void> S3BlobStoreEndpoint::writeEntireFileFromBuffer(std::string const& bucket,
-                                                            std::string const& object,
-                                                            UnsentPacketQueue* pContent,
-                                                            int contentLen,
-                                                            std::string const& contentHash) {
-	return writeEntireFileFromBuffer_impl(
-	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, pContent, contentLen, contentHash);
-}
-
-ACTOR Future<int> readObject_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                  std::string bucket,
-                                  std::string object,
-                                  void* data,
-                                  int length,
-                                  int64_t offset) {
-	try {
-		if (length <= 0) {
-			TraceEvent(SevWarn, "S3BlobStoreReadObjectEmptyRead").detail("Length", length);
-			return 0;
-		}
-
-		// Log rate limiter state
-		wait(bstore->requestRateRead->getAllowance(1));
-
-		state std::string resource = bstore->constructResourcePath(bucket, object);
-		state HTTP::Headers headers;
-		headers["Range"] = format("bytes=%lld-%lld", offset, offset + length - 1);
-
-		// Attempt the request
-		state Reference<HTTP::IncomingResponse> r;
-		Reference<HTTP::IncomingResponse> _r =
-		    wait(doRequest_impl(bstore, "GET", resource, headers, nullptr, 0, { 200, 206, 404 }));
-		r = _r;
-
-		if (r->code == 404) {
-			throw file_not_found();
-		}
-
-		// Verify response has content
-		if (r->data.contentLen != r->data.content.size()) {
-			TraceEvent(SevWarn, "S3BlobStoreReadObjectContentLengthMismatch")
-			    .detail("Expected", r->data.contentLen)
-			    .detail("Actual", r->data.content.size());
-			throw io_error();
-		}
-
-		// Copy the output bytes, server could have sent more or less bytes than requested so copy at most length
-		// bytes
-		int bytesToCopy = std::min<int64_t>(r->data.contentLen, length);
-		memcpy(data, r->data.content.data(), bytesToCopy);
-		// Return the number of bytes actually copied
-		return bytesToCopy;
-	} catch (Error& e) {
-		TraceEvent(SevWarn, "S3BlobStoreEndpoint_ReadError")
-		    .error(e)
-		    .detail("Bucket", bucket)
-		    .detail("Object", object)
-		    .detail("Length", length)
-		    .detail("Offset", offset);
-		throw;
-	}
-}
-
-Future<int> S3BlobStoreEndpoint::readObject(std::string const& bucket,
-                                            std::string const& object,
-                                            void* data,
-                                            int length,
-                                            int64_t offset) {
-	return readObject_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, data, length, offset);
-}
-
-ACTOR static Future<std::string> beginMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                           std::string bucket,
-                                                           std::string object) {
-	wait(bstore->requestRateWrite->getAllowance(1));
-
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	resource += "?uploads";
-	state HTTP::Headers headers;
-	if (!CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE.empty())
-		headers["x-amz-server-side-encryption"] = CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE;
-	if (bstore->knobs.enable_object_integrity_check) {
-		// CRITICAL: Adding x-amz-checksum-algorithm to multipart initiation means ALL parts
-		// must include checksums and completion request must include ChecksumSHA256 tags.
-		// AWS S3 will reject completion if any part checksum is missing.
-		headers["x-amz-checksum-algorithm"] = "SHA256";
-	}
-	Reference<HTTP::IncomingResponse> r = wait(doRequest_impl(bstore, "POST", resource, headers, nullptr, 0, { 200 }));
-
-	try {
-		xml_document<> doc;
-		// Copy content because rapidxml will modify it during parse
-		std::string content = r->data.content;
-
-		doc.parse<0>((char*)content.c_str());
-
-		// There should be exactly one node
-		xml_node<>* result = doc.first_node();
-		if (result != nullptr && strcmp(result->name(), "InitiateMultipartUploadResult") == 0) {
-			xml_node<>* id = result->first_node("UploadId");
-			if (id != nullptr) {
-				return id->value();
-			}
-		}
-	} catch (...) {
-	}
-	throw http_bad_response();
-}
-
-Future<std::string> S3BlobStoreEndpoint::beginMultiPartUpload(std::string const& bucket, std::string const& object) {
-	return beginMultiPartUpload_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
-}
-
-ACTOR Future<std::string> uploadPart_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                          std::string bucket,
-                                          std::string object,
-                                          std::string uploadID,
-                                          unsigned int partNumber,
-                                          UnsentPacketQueue* pContent,
-                                          int contentLen,
-                                          std::string contentMD5) {
-	wait(bstore->requestRateWrite->getAllowance(1));
-	wait(bstore->concurrentUploads.take());
-	state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
-
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	resource += format("?partNumber=%d&uploadId=%s", partNumber, uploadID.c_str());
-	state HTTP::Headers headers;
-	// Send hash for content so blobstore can verify it
-	// Use SHA256 if integrity check is enabled, otherwise use MD5
-	if (bstore->knobs.enable_object_integrity_check) {
-		headers["x-amz-checksum-sha256"] = contentMD5;
-		headers["x-amz-checksum-algorithm"] = "SHA256";
-	} else {
-		headers["Content-MD5"] = contentMD5;
-	}
-	state Reference<HTTP::IncomingResponse> r =
-	    wait(doRequest_impl(bstore, "PUT", resource, headers, pContent, contentLen, { 200 }));
-	// TODO:  In the event that the client times out just before the request completes (so the client is unaware) then
-	// the next retry will see error 400.  That could be detected and handled gracefully by retrieving the etag for the
-	// successful request.
-
-	// For uploads, Blobstore returns an MD5 sum of uploaded content so check it.
-	// Only verify MD5 when not using integrity check (SHA256 verification is done by AWS)
-	if (!bstore->knobs.enable_object_integrity_check) {
-		if (!HTTP::verifyMD5(&r->data, false, contentMD5))
-			throw checksum_failed();
-	}
-	// Note: When using SHA256, AWS handles the verification internally
-	// and will return an error if the checksum doesn't match
-
-	// No etag -> bad response.
-	std::string etag = r->data.headers["ETag"];
-	if (etag.empty())
-		throw http_bad_response();
-
-	return etag;
-}
-
-Future<std::string> S3BlobStoreEndpoint::uploadPart(std::string const& bucket,
-                                                    std::string const& object,
-                                                    std::string const& uploadID,
-                                                    unsigned int partNumber,
-                                                    UnsentPacketQueue* pContent,
-                                                    int contentLen,
-                                                    std::string const& contentMD5) {
-	return uploadPart_impl(Reference<S3BlobStoreEndpoint>::addRef(this),
-	                       bucket,
-	                       object,
-	                       uploadID,
-	                       partNumber,
-	                       pContent,
-	                       contentLen,
-	                       contentMD5);
-}
-
-ACTOR Future<Optional<std::string>> finishMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                               std::string bucket,
-                                                               std::string object,
-                                                               std::string uploadID,
-                                                               S3BlobStoreEndpoint::MultiPartSetT parts) {
-	state UnsentPacketQueue part_list; // NonCopyable state var so must be declared at top of actor
-	wait(bstore->requestRateWrite->getAllowance(1));
-
-	std::string manifest = "<CompleteMultipartUpload>";
-	for (auto& p : parts) {
-		manifest += format("<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>", p.first, p.second.etag.c_str());
-		// Include checksum if integrity check is enabled and checksum is present
-		if (bstore->knobs.enable_object_integrity_check && !p.second.checksum.empty()) {
-			manifest += format("<ChecksumSHA256>%s</ChecksumSHA256>", p.second.checksum.c_str());
-		}
-		manifest += "</Part>\n";
-	}
-	manifest += "</CompleteMultipartUpload>";
-
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	resource += format("?uploadId=%s", uploadID.c_str());
-	state HTTP::Headers headers;
-	PacketWriter pw(part_list.getWriteBuffer(manifest.size()), nullptr, Unversioned());
-	pw.serializeBytes(manifest);
-	Reference<HTTP::IncomingResponse> r =
-	    wait(doRequest_impl(bstore, "POST", resource, headers, &part_list, manifest.size(), { 200 }));
-
-	// The XML response contains a ChecksumSHA256 field, but this is just a hash of the multipart
-	// structure, not the actual object content, so it's useless for integrity verification.
-	// We skip parsing it to avoid wasted CPU cycles.
-
-	// TODO:  In the event that the client times out just before the request completes (so the client is unaware) then
-	// the next retry will see error 400.  That could be detected and handled gracefully by HEAD'ing the object before
-	// upload to get its (possibly nonexistent) eTag, then if an error 400 is seen then retrieve the eTag again and if
-	// it has changed then consider the finish complete.
-	return Optional<std::string>();
-}
-
-Future<Optional<std::string>> S3BlobStoreEndpoint::finishMultiPartUpload(std::string const& bucket,
-                                                                         std::string const& object,
-                                                                         std::string const& uploadID,
-                                                                         MultiPartSetT const& parts) {
-	return finishMultiPartUpload_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, uploadID, parts);
-}
-
-ACTOR Future<Void> abortMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                             std::string bucket,
-                                             std::string object,
-                                             std::string uploadID) {
-	wait(bstore->requestRateWrite->getAllowance(1));
-
-	std::string resource = bstore->constructResourcePath(bucket, object);
-	resource += format("?uploadId=%s", uploadID.c_str());
-
-	HTTP::Headers headers;
-	Reference<HTTP::IncomingResponse> r =
-	    wait(doRequest_impl(bstore, "DELETE", resource, headers, nullptr, 0, { 200, 204 }));
-	return Void();
-}
-
-Future<Void> S3BlobStoreEndpoint::abortMultiPartUpload(std::string const& bucket,
-                                                       std::string const& object,
-                                                       std::string const& uploadID) {
-	return abortMultiPartUpload_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, uploadID);
-}
-
-// Forward declarations
-ACTOR Future<std::map<std::string, std::string>> getObjectTags_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                                    std::string bucket,
-                                                                    std::string object);
-
-ACTOR Future<Void> putObjectTags_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                      std::string bucket,
-                                      std::string object,
-                                      std::map<std::string, std::string> tags) {
-	state UnsentPacketQueue packets;
-	wait(bstore->requestRateWrite->getAllowance(1));
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	resource += "?tagging";
-	state int maxRetries = 5;
-	state int retryCount = 0;
-	state double backoff = 1.0;
-	state double maxBackoff = 8.0;
-
-	loop {
-		try {
-			std::string manifest = "<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet>";
-			for (auto itr = tags.begin(); itr != tags.end(); ++itr) {
-				manifest += "<Tag><Key>" + itr->first + "</Key><Value>" + itr->second + "</Value></Tag>";
-			}
-			manifest += "</TagSet></Tagging>";
-
-			PacketWriter pw(packets.getWriteBuffer(manifest.size()), nullptr, Unversioned());
-			pw.serializeBytes(manifest);
-
-			HTTP::Headers headers;
-			headers["Content-Type"] = "application/xml";
-			headers["Content-Length"] = format("%d", manifest.size());
-
-			Reference<HTTP::IncomingResponse> r =
-			    wait(doRequest_impl(bstore, "PUT", resource, headers, &packets, manifest.size(), { 200 }));
-
-			// Verify tags were written correctly
-			std::map<std::string, std::string> verifyTags = wait(getObjectTags_impl(bstore, bucket, object));
-			if (verifyTags == tags) {
-				return Void();
-			}
-
-			if (++retryCount >= maxRetries) {
-				TraceEvent(SevWarn, "S3BlobStorePutTagsMaxRetriesExceeded")
-				    .detail("Bucket", bucket)
-				    .detail("Object", object);
-				throw operation_failed();
-			}
-
-			// Implement exponential backoff with jitter
-			wait(delay(backoff * (0.9 + 0.2 * deterministicRandom()->random01())));
-			backoff = std::min(backoff * 2, maxBackoff);
-
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
-			}
-
-			TraceEvent(SevWarn, "S3BlobStorePutTagsError")
-			    .error(e)
-			    .detail("Bucket", bucket)
-			    .detail("Object", object)
-			    .detail("RetryCount", retryCount);
-
-			if (++retryCount >= maxRetries) {
-				throw;
-			}
-
-			// Implement exponential backoff with jitter for errors
-			wait(delay(backoff * (0.9 + 0.2 * deterministicRandom()->random01())));
-			backoff = std::min(backoff * 2, maxBackoff);
-		}
-	}
-}
-
-Future<Void> S3BlobStoreEndpoint::putObjectTags(std::string const& bucket,
-                                                std::string const& object,
-                                                std::map<std::string, std::string> const& tags) {
-	return putObjectTags_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, tags);
-}
-
-ACTOR Future<std::map<std::string, std::string>> getObjectTags_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                                    std::string bucket,
-                                                                    std::string object) {
-	wait(bstore->requestRateRead->getAllowance(1));
-
-	state std::string resource = bstore->constructResourcePath(bucket, object);
-	resource += "?tagging";
-	state HTTP::Headers headers;
-
-	Reference<HTTP::IncomingResponse> r = wait(doRequest_impl(bstore, "GET", resource, headers, nullptr, 0, { 200 }));
-
-	rapidxml::xml_document<> doc;
-	doc.parse<rapidxml::parse_default>((char*)r->data.content.c_str());
-
-	std::map<std::string, std::string> tags;
-
-	// Find the Tagging node (with or without namespace)
-	rapidxml::xml_node<>* tagging = doc.first_node();
-	while (tagging && strcmp(tagging->name(), "Tagging") != 0) {
-		tagging = tagging->next_sibling();
-	}
-
-	if (tagging) {
-		// Find TagSet node
-		rapidxml::xml_node<>* tagSet = tagging->first_node();
-		while (tagSet && strcmp(tagSet->name(), "TagSet") != 0) {
-			tagSet = tagSet->next_sibling();
-		}
-
-		if (tagSet) {
-			// Iterate through Tag nodes
-			for (rapidxml::xml_node<>* tag = tagSet->first_node(); tag; tag = tag->next_sibling()) {
-				if (strcmp(tag->name(), "Tag") == 0) {
-					std::string key, value;
-					// Find Key and Value nodes
-					for (rapidxml::xml_node<>* node = tag->first_node(); node; node = node->next_sibling()) {
-						if (strcmp(node->name(), "Key") == 0) {
-							key = node->value();
-						} else if (strcmp(node->name(), "Value") == 0) {
-							value = node->value();
-						}
-					}
-					if (!key.empty()) {
-						tags[key] = value;
-					}
-				}
-			}
-		}
-	}
-
-	return tags;
-}
-
-Future<std::map<std::string, std::string>> S3BlobStoreEndpoint::getObjectTags(std::string const& bucket,
-                                                                              std::string const& object) {
-	return getObjectTags_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
-}
-
-TEST_CASE("/backup/s3/v4headers") {
-	S3BlobStoreEndpoint::Credentials creds{ "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "" };
-	// GET without query parameters
-	{
-		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
-		std::string verb("GET");
-		std::string resource("/test.txt");
-		HTTP::Headers headers;
-		headers["Host"] = "s3.amazonaws.com";
-		s3.setV4AuthHeaders(verb, resource, headers, "20130524T000000Z", "20130524");
-		ASSERT(headers["Authorization"] ==
-		       "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/amazonaws/s3/aws4_request, "
-		       "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
-		       "Signature=c6037f4b174f2019d02d7085a611cef8adfe1efe583e220954dc85d59cd31ba3");
-		ASSERT(headers["x-amz-date"] == "20130524T000000Z");
-	}
-
-	// GET with query parameters
-	{
-		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
-		std::string verb("GET");
-		std::string resource("/test/examplebucket?Action=DescribeRegions&Version=2013-10-15");
-		HTTP::Headers headers;
-		headers["Host"] = "s3.amazonaws.com";
-		s3.setV4AuthHeaders(verb, resource, headers, "20130524T000000Z", "20130524");
-		ASSERT(headers["Authorization"] ==
-		       "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/amazonaws/s3/aws4_request, "
-		       "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
-		       "Signature=426f04e71e191fbc30096c306fe1b11ce8f026a7be374541862bbee320cce71c");
-		ASSERT(headers["x-amz-date"] == "20130524T000000Z");
-	}
-
-	// POST
-	{
-		S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "443", "us-west-2", "proxy", "port", creds);
-		std::string verb("POST");
-		std::string resource("/simple.json");
-		HTTP::Headers headers;
-		headers["Host"] = "s3.us-west-2.amazonaws.com";
-		headers["Content-Type"] = "Application/x-amz-json-1.0";
-		s3.setV4AuthHeaders(verb, resource, headers, "20130524T000000Z", "20130524");
-		ASSERT(headers["Authorization"] ==
-		       "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-west-2/s3/aws4_request, "
-		       "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, "
-		       "Signature=cf095e36bed9cd3139c2e8b3e20c296a79d8540987711bf3a0d816b19ae00314");
-		ASSERT(headers["x-amz-date"] == "20130524T000000Z");
-		ASSERT(headers["Host"] == "s3.us-west-2.amazonaws.com");
-		ASSERT(headers["Content-Type"] == "Application/x-amz-json-1.0");
-	}
-
-	return Void();
-}
-
-TEST_CASE("/backup/s3/guess_region") {
-	std::string url = "blobstore://s3.us-west-2.amazonaws.com/resource_name?bucket=bucket_name&sa=1";
-
-	std::string resource;
-	std::string error;
-	S3BlobStoreEndpoint::ParametersT parameters;
-	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
-	ASSERT(s3->getRegion() == "us-west-2");
-
-	url = "blobstore://s3.us-west-2.amazonaws.com/resource_name?bucket=bucket_name&sc=922337203685477580700";
-	try {
-		s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
-		ASSERT(false); // not reached
-	} catch (Error& e) {
-		// conversion of 922337203685477580700 to long int will overflow
-		ASSERT_EQ(e.code(), error_code_backup_invalid_url);
-	}
-	return Void();
-}
-
-TEST_CASE("/backup/s3/virtual_hosting_list_resource_path") {
-	// Test that listObjectsStream_impl constructs valid HTTP resource paths
-	// for virtual hosting mode URLs with path prefixes.
-	//
-	// Regression test for commit 15dd76a7f9 which accidentally removed the leading "/"
-	// from the query string when switching from ListObjectsV1 to ListObjectsV2.
-	//
-	// For URL: blobstore://@bucket.s3.region.amazonaws.com/prefix/path?bucket=...
-	// constructResourcePath(bucket, "") returns "" in virtual hosting mode.
-	// We must ensure the final resource starts with "/" before the query string.
-
-	std::string url = "blobstore://@test-bucket.s3.us-west-2.amazonaws.com/bulkload/path?bucket=test-bucket";
-	std::string resource;
-	std::string error;
-	S3BlobStoreEndpoint::ParametersT parameters;
-	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
-
-	ASSERT(s3.isValid());
-	ASSERT(resource == "bulkload/path"); // Path prefix extracted from URL (without leading '/')
-
-	// In virtual hosting mode, constructResourcePath returns "" for empty object
-	std::string listResource = s3->constructResourcePath("test-bucket", "");
-	ASSERT(listResource == ""); // Virtual hosting mode doesn't include bucket in path
-
-	// listObjectsStream_impl should add "/" before query string to form valid HTTP request
-	// Expected: GET /bulkload/path?list-type=2&... (but we can't test the full actor here)
-	// We can only verify that constructResourcePath returns "" as expected.
-	// The fix in listObjectsStream_impl checks if resource.empty() and sets it to "/" before
-	// appending the query string, ensuring the final request is well-formed.
-
-	// Test non-virtual hosting mode for comparison
-	url = "blobstore://s3.us-west-2.amazonaws.com/resource_path?bucket=test-bucket";
-	s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
-	ASSERT(s3.isValid());
-	ASSERT(resource == "resource_path"); // Path extracted from URL (without leading '/')
-
-	// In non-virtual hosting mode, constructResourcePath includes bucket
-	listResource = s3->constructResourcePath("test-bucket", "");
-	ASSERT(listResource == "/test-bucket"); // Bucket is part of path
-
-	return Void();
-}
-
-TEST_CASE("/backup/s3/constructResourcePath") {
-	// Test that leading slashes in object keys are properly stripped
-	// S3 object keys should not start with '/', but if passed, we normalize them
-
-	std::string url = "blobstore://s3.us-west-2.amazonaws.com?bucket=test-bucket";
-	std::string resource;
-	std::string error;
-	S3BlobStoreEndpoint::ParametersT parameters;
-	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
-
-	// Test normal object key (no leading slash)
-	ASSERT(s3->constructResourcePath("test-bucket", "normal/path/file.txt") == "/test-bucket/normal/path/file.txt");
-
-	// Test single leading slash - should be stripped
-	ASSERT(s3->constructResourcePath("test-bucket", "/leading/slash/file.txt") ==
-	       "/test-bucket/leading/slash/file.txt");
-
-	// Test multiple leading slashes - all should be stripped
-	ASSERT(s3->constructResourcePath("test-bucket", "///multiple/slashes.txt") == "/test-bucket/multiple/slashes.txt");
-
-	// Test object key that is only slashes - should result in empty object path
-	ASSERT(s3->constructResourcePath("test-bucket", "///") == "/test-bucket");
-
-	// Test empty object key
-	ASSERT(s3->constructResourcePath("test-bucket", "") == "/test-bucket");
-
-	// Test virtual hosting mode (bucket in hostname)
-	url = "blobstore://test-bucket.s3.us-west-2.amazonaws.com";
-	s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
-
-	// Virtual hosting mode doesn't include bucket in path
-	ASSERT(s3->constructResourcePath("test-bucket", "normal/file.txt") == "/normal/file.txt");
-	ASSERT(s3->constructResourcePath("test-bucket", "/leading/slash.txt") == "/leading/slash.txt");
-	ASSERT(s3->constructResourcePath("test-bucket", "") == "");
-
-	return Void();
-}
diff --git a/fdbclient/S3BlobStore.cpp b/fdbclient/S3BlobStore.cpp
new file mode 100644
index 00000000000..e87791466c7
--- /dev/null
+++ b/fdbclient/S3BlobStore.cpp
@@ -0,0 +1,1698 @@
+/*
+ * S3BlobStore.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/S3BlobStore.h"
+
+#include <sstream>
+#include "fdbrpc/HTTP.h"
+#include "fdbclient/Knobs.h"
+#include "flow/FastRef.h"
+#include "flow/IConnection.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+#include "flow/genericactors.actor.h"
+
+#include "md5/md5.h"
+#include "libb64/encode.h"
+#include "sha1/SHA1.h"
+#include <climits>
+#include <iostream>
+#include <time.h>
+#include <iomanip>
+#include <openssl/sha.h>
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/hex.hpp>
+#include "flow/IAsyncFile.h"
+#include "flow/Hostname.h"
+#include "flow/UnitTest.h"
+#include "rapidxml/rapidxml.hpp"
+#ifdef WITH_AWS_BACKUP
+#include "FDBAWSCredentialsProvider.h"
+#endif
+
+#include "flow/CoroUtils.h"
+
+using namespace rapidxml;
+
+std::string S3BlobStoreEndpoint::guessRegionFromDomain(std::string domain) {
+	// Special case for localhost/127.0.0.1 to prevent basic_string exception
+	if (domain == "127.0.0.1" || domain == "localhost") {
+		return "us-east-1";
+	}
+
+	static const std::vector<const char*> knownServices = { "s3.", "cos.", "oss-", "obs." };
+	boost::algorithm::to_lower(domain);
+
+	for (const auto& service : knownServices) {
+		std::size_t p = domain.find(service);
+		if (p == std::string::npos || (p >= 1 && domain[p - 1] != '.')) {
+			// eg. 127.0.0.1, example.com, s3-service.example.com, mys3.example.com
+			continue;
+		}
+
+		StringRef h = StringRef(domain).substr(p);
+
+		if (!h.startsWith("oss-"_sr)) {
+			h.eat(service); // ignore s3 service
+		}
+
+		return h.eat(".").toString();
+	}
+
+	return "";
+}
+
+static Optional<S3BlobStoreEndpoint::Credentials> parseS3Credentials(Optional<StringRef> const& credString) {
+	if (!credString.present()) {
+		return Optional<S3BlobStoreEndpoint::Credentials>();
+	}
+	StringRef c = credString.get();
+	StringRef key = c.eat(":");
+	StringRef secret = c.eat(":");
+	StringRef securityToken = c.eat();
+	return S3BlobStoreEndpoint::Credentials{ key.toString(), secret.toString(), securityToken.toString() };
+}
+
+S3BlobStoreEndpoint::S3BlobStoreEndpoint(std::string const& host,
+                                         std::string const& service,
+                                         std::string region,
+                                         Optional<std::string> const& proxyHost,
+                                         Optional<std::string> const& proxyPort,
+                                         Optional<StringRef> const& creds,
+                                         BlobKnobs const& knobs,
+                                         HTTP::Headers extraHeaders)
+  : IBlobStoreEndpoint(host,
+                       service,
+                       region.empty() ? guessRegionFromDomain(host) : region,
+                       proxyHost,
+                       proxyPort,
+                       knobs,
+                       extraHeaders),
+    credentials(parseS3Credentials(creds)), lookupKey(credentials.present() && credentials.get().key.empty()),
+    lookupSecret(credentials.present() && credentials.get().secret.empty()) {
+	if (this->region.empty() && CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER) {
+		throw std::string(
+		    "Failed to get region from host or parameter in url, region is required for aws v4 signature");
+	}
+}
+
+std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::string params) const {
+	// The base class produces blobstore://@host/resource?params.
+	// For S3, we replace the empty credential slot with explicit credentials when they weren't
+	// looked up from credential files (i.e., they were passed in the URL).
+	std::string url = IBlobStoreEndpoint::getResourceURL(resource, params);
+
+	// Build the S3 credential prefix to replace the bare "@"
+	std::string credsString;
+	if (credentials.present()) {
+		if (!lookupKey) {
+			credsString = credentials.get().key;
+		}
+		if (!lookupSecret) {
+			credsString += ":" + credentials.get().secret;
+			if (!credentials.get().securityToken.empty()) {
+				credsString += ":" + credentials.get().securityToken;
+			}
+		}
+	}
+
+	// Replace "blobstore://@" with "blobstore://<creds>@"
+	const std::string placeholder = "blobstore://@";
+	size_t pos = url.find(placeholder);
+	if (pos != std::string::npos) {
+		url.replace(pos, placeholder.size(), "blobstore://" + credsString + "@");
+	}
+
+	return url;
+}
+
+std::string S3BlobStoreEndpoint::constructResourcePath(const std::string& bucket, const std::string& object) const {
+	std::string resource;
+
+	if (host.find(bucket + ".") != 0) {
+		resource += std::string("/") + bucket; // not virtual hosting mode
+	}
+
+	if (!object.empty()) {
+		// S3 object keys should not start with '/'. Strip any leading slashes.
+		std::string cleanedObject = object;
+		while (!cleanedObject.empty() && cleanedObject[0] == '/') {
+			cleanedObject = cleanedObject.substr(1);
+		}
+		if (!cleanedObject.empty()) {
+			resource += "/";
+			resource += cleanedObject;
+		}
+	}
+
+	return resource;
+}
+
+Future<bool> bucketExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {
+	co_await b->requestRateRead->getAllowance(1);
+
+	std::string resource = b->constructResourcePath(bucket, "");
+	HTTP::Headers headers;
+
+	try {
+		Reference<HTTP::IncomingResponse> r =
+		    co_await b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 });
+		co_return r->code == 200;
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled)
+			throw;
+		TraceEvent(SevError, "S3ClientBucketExistsError")
+		    .detail("Bucket", bucket)
+		    .detail("Host", b->host)
+		    .errorUnsuppressed(e);
+		throw;
+	}
+}
+
+Future<bool> S3BlobStoreEndpoint::bucketExists(std::string const& bucket) {
+	return bucketExists_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket);
+}
+
+Future<bool> objectExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
+	co_await b->requestRateRead->getAllowance(1);
+
+	std::string resource = b->constructResourcePath(bucket, object);
+	HTTP::Headers headers;
+
+	Reference<HTTP::IncomingResponse> r = co_await b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 });
+	co_return r->code == 200;
+}
+
+Future<bool> S3BlobStoreEndpoint::objectExists(std::string const& bucket, std::string const& object) {
+	return objectExists_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
+}
+
+Future<Void> deleteObject_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
+	co_await b->requestRateDelete->getAllowance(1);
+
+	std::string resource = b->constructResourcePath(bucket, object);
+	HTTP::Headers headers;
+	// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered
+	// successful
+	Reference<HTTP::IncomingResponse> r =
+	    co_await b->doRequest("DELETE", resource, headers, nullptr, 0, { 200, 204, 404 });
+
+	// But if the object already did not exist then the 'delete' is assumed to be successful but a warning is logged.
+	if (r->code == 404) {
+		TraceEvent(SevWarnAlways, "S3BlobStoreEndpointDeleteObjectMissing")
+		    .detail("Host", b->host)
+		    .detail("Bucket", bucket)
+		    .detail("Object", object);
+	}
+}
+
+Future<Void> S3BlobStoreEndpoint::deleteObject(std::string const& bucket, std::string const& object) {
+	return deleteObject_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
+}
+
+Future<Void> createBucket_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {
+	UnsentPacketQueue packets;
+	co_await b->requestRateWrite->getAllowance(1);
+
+	bool exists = co_await b->bucketExists(bucket);
+	if (!exists) {
+		std::string resource = b->constructResourcePath(bucket, "");
+		HTTP::Headers headers;
+
+		std::string region = b->getRegion();
+		if (region.empty()) {
+			Reference<HTTP::IncomingResponse> r =
+			    co_await b->doRequest("PUT", resource, headers, nullptr, 0, { 200, 409 });
+		} else {
+			Standalone<StringRef> body(
+			    format("<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
+			           "  <LocationConstraint>%s</LocationConstraint>"
+			           "</CreateBucketConfiguration>",
+			           region.c_str()));
+			PacketWriter pw(packets.getWriteBuffer(), nullptr, Unversioned());
+			pw.serializeBytes(body);
+
+			Reference<HTTP::IncomingResponse> r =
+			    co_await b->doRequest("PUT", resource, headers, &packets, body.size(), { 200, 409 });
+		}
+	}
+}
+
+Future<Void> S3BlobStoreEndpoint::createBucket(std::string const& bucket) {
+	return createBucket_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket);
+}
+
+Future<int64_t> objectSize_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
+	co_await b->requestRateRead->getAllowance(1);
+
+	std::string resource = b->constructResourcePath(bucket, object);
+	HTTP::Headers headers;
+
+	Reference<HTTP::IncomingResponse> r = co_await b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 });
+	if (r->code == 404)
+		throw file_not_found();
+	co_return r->data.contentLen;
+}
+
+Future<int64_t> S3BlobStoreEndpoint::objectSize(std::string const& bucket, std::string const& object) {
+	return objectSize_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
+}
+
+// If the credentials expire, the connection will eventually fail and be discarded from the pool, and then a new
+// connection will be constructed, which will call this again to get updated credentials
+static S3BlobStoreEndpoint::Credentials getSecretSdk() {
+#ifdef WITH_AWS_BACKUP
+	double elapsed = -timer_monotonic();
+	Aws::Auth::AWSCredentials awsCreds = FDBAWSCredentialsProvider::getAwsCredentials();
+	elapsed += timer_monotonic();
+
+	if (awsCreds.IsEmpty()) {
+		TraceEvent(SevWarn, "S3BlobStoreAWSCredsEmpty");
+		throw backup_auth_missing();
+	}
+
+	S3BlobStoreEndpoint::Credentials fdbCreds;
+	fdbCreds.key = awsCreds.GetAWSAccessKeyId();
+	fdbCreds.secret = awsCreds.GetAWSSecretKey();
+	fdbCreds.securityToken = awsCreds.GetSessionToken();
+
+	TraceEvent("S3BlobStoreGotSdkCredentials").suppressFor(60).detail("Duration", elapsed);
+
+	return fdbCreds;
+#else
+	TraceEvent(SevError, "S3BlobStoreNoSDK").log();
+	throw backup_auth_missing();
+#endif
+}
+
+Future<Void> S3BlobStoreEndpoint::updateSecret() {
+	if (knobs.sdk_auth) {
+		credentials = getSecretSdk();
+		co_return;
+	}
+	if (!credentials.present()) {
+		co_return;
+	}
+	co_await IBlobStoreEndpoint::updateSecret();
+}
+
+bool S3BlobStoreEndpoint::extractCredentialFields(JSONDoc& account) {
+	Credentials creds = credentials.get();
+	if (lookupKey) {
+		std::string apiKey;
+		if (account.tryGet("api_key", apiKey))
+			creds.key = apiKey;
+		else
+			return false;
+	}
+	if (lookupSecret) {
+		std::string secret;
+		if (account.tryGet("secret", secret))
+			creds.secret = secret;
+		else
+			return false;
+	}
+	std::string token;
+	if (account.tryGet("token", token))
+		creds.securityToken = token;
+	credentials = creds;
+	return true;
+}
+
+std::string S3BlobStoreEndpoint::credentialFileKey() const {
+	std::string accessKey = lookupKey ? "" : credentials.get().key;
+	return accessKey + "@" + host;
+}
+
+bool S3BlobStoreEndpoint::lookupSecretOnEachRequest() {
+	return lookupKey || lookupSecret || knobs.sdk_auth;
+}
+
+std::string awsCanonicalURI(const std::string& resource, std::vector<std::string>& queryParameters, bool isV4) {
+	StringRef resourceRef(resource);
+	resourceRef.eat("/");
+	std::string canonicalURI("/" + resourceRef.toString());
+	size_t q = canonicalURI.find_last_of('?');
+	if (q != canonicalURI.npos)
+		canonicalURI.resize(q);
+	if (isV4) {
+		canonicalURI = HTTP::awsV4URIEncode(canonicalURI, false);
+	} else {
+		canonicalURI = HTTP::urlEncode(canonicalURI);
+	}
+
+	// Create the canonical query string
+	std::string queryString;
+	q = resource.find_last_of('?');
+	if (q != queryString.npos)
+		queryString = resource.substr(q + 1);
+
+	StringRef qStr(queryString);
+	StringRef queryParameter;
+	while (!(queryParameter = qStr.eat("&")).empty()) {
+		StringRef param = queryParameter.eat("=");
+		StringRef value = queryParameter.eat();
+
+		if (isV4) {
+			queryParameters.push_back(HTTP::awsV4URIEncode(param.toString(), true) + "=" +
+			                          HTTP::awsV4URIEncode(value.toString(), true));
+		} else {
+			queryParameters.push_back(HTTP::urlEncode(param.toString()) + "=" + HTTP::urlEncode(value.toString()));
+		}
+	}
+
+	return canonicalURI;
+}
+
+// ref: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+// Returns the S3 error code string from an XML error response, or "" if the response
+// cannot be parsed. This is best-effort for logging/diagnostics only — many HTTP error
+// responses (502/503 from load balancers, empty bodies, HTML responses) are not XML.
+std::string parseErrorCodeFromS3(const std::string& response) {
+	if (response.empty()) {
+		return "";
+	}
+	try {
+		std::vector<char> xmlBuffer(response.begin(), response.end());
+		xmlBuffer.push_back('\0');
+		xml_document<> doc;
+		doc.parse<0>(&xmlBuffer[0]);
+		xml_node<>* root = doc.first_node("Error");
+		if (!root) {
+			return "";
+		}
+		xml_node<>* codeNode = root->first_node("Code");
+		if (!codeNode) {
+			return "";
+		}
+		return std::string(codeNode->value());
+	} catch (...) {
+		// Parse failures are expected for non-XML responses (HTML error pages, empty bodies, etc.)
+		TraceEvent("ParseS3ErrorCodeNonXML")
+		    .suppressFor(60)
+		    .detail("ResponseSize", response.size())
+		    .detail("ResponseHead", response.substr(0, 200));
+		return "";
+	}
+}
+
+bool isS3TokenError(const std::string& s3Error) {
+	return s3Error == "InvalidToken" || s3Error == "ExpiredToken";
+}
+
+// S3 returns HTTP 400 for token/auth errors (InvalidToken, ExpiredToken).
+// Used by processRequestFailure, preRetryCheck, and simulateRequestFailure.
+constexpr int s3BadRequestCode = 400;
+
+void S3BlobStoreEndpoint::setRequestHeaders(std::string const& verb,
+                                            std::string const& resource,
+                                            HTTP::Headers& headers) {
+	headers["Accept"] = "application/xml";
+	if (credentials.present() && !credentials.get().securityToken.empty())
+		headers["x-amz-security-token"] = credentials.get().securityToken;
+	if (CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER) {
+		setV4AuthHeaders(verb, resource, headers);
+	} else {
+		setAuthHeaders(verb, resource, headers);
+	}
+}
+
+std::string S3BlobStoreEndpoint::normalizeResourceForRequest(std::string const& resource) {
+	std::vector<std::string> queryParameters;
+	std::string canonicalURI = awsCanonicalURI(resource, queryParameters, CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER);
+	if (!queryParameters.empty()) {
+		canonicalURI += "?";
+		for (size_t i = 0; i < queryParameters.size(); ++i) {
+			if (i > 0) {
+				canonicalURI += "&";
+			}
+			canonicalURI += queryParameters[i];
+		}
+	}
+	return canonicalURI;
+}
+
+void populateDryrunRequest(Reference<HTTP::OutgoingRequest> dryrunRequest,
+                           Reference<S3BlobStoreEndpoint> bstore,
+                           std::string bucket) {
+	// dryrun with a check bucket exist request, to avoid sending duplicate data
+	HTTP::Headers headers;
+	dryrunRequest->verb = "GET";
+	dryrunRequest->data.contentLen = 0;
+	dryrunRequest->data.headers = headers;
+	dryrunRequest->data.headers["Host"] = bstore->host;
+	dryrunRequest->data.headers["Accept"] = "application/xml";
+
+	dryrunRequest->resource = bstore->constructResourcePath(bucket, "");
+}
+
+bool isWriteRequest(std::string verb) {
+	return verb == "POST" || verb == "PUT";
+}
+
+std::string parseBucketFromURI(std::string uri) {
+	if (uri.size() <= 1 || uri[0] != '/') {
+		// there is no bucket in the uri
+		return "";
+	}
+	uri = uri.substr(1);
+	size_t secondSlash = uri.find('/');
+	if (secondSlash == std::string::npos) {
+		return uri;
+	}
+	return uri.substr(0, secondSlash);
+}
+
+void S3BlobStoreEndpoint::simulateRequestFailure(std::string const& verb,
+                                                 std::string const& resource,
+                                                 Reference<HTTP::IncomingResponse>& r) {
+	simulatedTokenError = false;
+	if (!g_network->isSimulated() || !BUGGIFY || deterministicRandom()->random01() >= 0.1) {
+		return;
+	}
+	// Don't simulate token errors for multipart complete operations (POST with uploadId but no partNumber)
+	// because changing a successful 200 to 400 after the server has already completed and removed
+	// the upload causes the client to infinitely retry with a phantom upload ID.
+	bool isMultipartComplete = verb == "POST" && resource.find("uploadId=") != std::string::npos &&
+	                           resource.find("partNumber=") == std::string::npos;
+	if (!isMultipartComplete) {
+		r->code = s3BadRequestCode;
+		simulatedTokenError = true;
+	}
+}
+
+void S3BlobStoreEndpoint::processRequestFailure(Reference<HTTP::IncomingResponse> const& r,
+                                                TraceEvent& event,
+                                                bool& retryExtended) {
+	// Only parse S3 error code for error responses (4xx/5xx), not successful responses (2xx).
+	// Skip parsing for simulated errors where response content is still binary data.
+	std::string s3Error;
+	if (r->code >= 400 && !simulatedTokenError) {
+		s3Error = parseErrorCodeFromS3(r->data.content);
+	}
+	event.detail("S3ErrorCode", s3Error);
+	event.detail("HttpResponseContent", r->data.content);
+
+	if (r->code == s3BadRequestCode && (isS3TokenError(s3Error) || simulatedTokenError)) {
+		retryExtended = true;
+	}
+}
+
+Future<bool> S3BlobStoreEndpoint::preRetryCheck(std::string const& verb,
+                                                std::string const& resource,
+                                                ReusableConnection& rconn,
+                                                int requestTimeout,
+                                                bool& retryExtended) {
+	if (!isWriteRequest(verb) || !CLIENT_KNOBS->BACKUP_ALLOW_DRYRUN) {
+		co_return true;
+	}
+
+	std::string bucket = parseBucketFromURI(resource);
+	if (bucket.empty()) {
+		TraceEvent(SevError, "EmptyBucketRequest").detail("Verb", verb).detail("Resource", resource);
+		throw bucket_not_in_url();
+	}
+
+	// Send a cheap HEAD request to verify credentials before resending a potentially large upload
+	UnsentPacketQueue dryrunContentCopy;
+	auto dryrunRequest = makeReference<HTTP::OutgoingRequest>();
+	dryrunRequest->data.content = &dryrunContentCopy;
+	populateDryrunRequest(dryrunRequest, Reference<S3BlobStoreEndpoint>::addRef(this), bucket);
+	setRequestHeaders(dryrunRequest->verb, dryrunRequest->resource, dryrunRequest->data.headers);
+	dryrunRequest->resource = normalizeResourceForRequest(dryrunRequest->resource);
+
+	TraceEvent("RetryS3RequestDueToTokenIssue")
+	    .detail("OriginalResource", resource)
+	    .detail("DryrunResource", dryrunRequest->resource)
+	    .detail("Bucket", bucket)
+	    .detail("V4", CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER);
+
+	co_await requestRate->getAllowance(1);
+	Future<Reference<HTTP::IncomingResponse>> dryrunResponse =
+	    HTTP::doRequest(rconn.conn, dryrunRequest, sendRate, &s_stats.bytes_sent, recvRate);
+	Reference<HTTP::IncomingResponse> dryrunR = co_await timeoutError(dryrunResponse, requestTimeout);
+
+	std::string s3Error;
+	if (dryrunR->code >= 400) {
+		s3Error = parseErrorCodeFromS3(dryrunR->data.content);
+	}
+
+	if (dryrunR->code == s3BadRequestCode && isS3TokenError(s3Error)) {
+		// Token still bad — delay and skip this iteration
+		co_await ::delay(knobs.max_delay_retryable_error);
+		co_return false;
+	} else if (dryrunR->code == 200 || dryrunR->code == 404) {
+		// Credentials are good now — proceed with the real request
+		TraceEvent("S3TokenIssueResolved").detail("HttpCode", dryrunR->code).detail("URI", dryrunRequest->resource);
+		retryExtended = false;
+		co_return true;
+	} else {
+		TraceEvent(SevError, "S3UnexpectedError")
+		    .detail("HttpCode", dryrunR->code)
+		    .detail("HttpResponseContent", dryrunR->data.content)
+		    .detail("S3Error", s3Error)
+		    .detail("URI", dryrunRequest->resource);
+		throw http_bad_response();
+	}
+}
+
+Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                    std::string bucket,
+                                    PromiseStream<S3BlobStoreEndpoint::ListResult> results,
+                                    Optional<std::string> prefix,
+                                    Optional<char> delimiter,
+                                    int maxDepth,
+                                    std::function<bool(std::string const&)> recurseFilterUnsafe) {
+	// C++20 coroutine safety: copy const& params to survive across suspension.
+	auto recurseFilter = recurseFilterUnsafe;
+
+	std::string resource = bstore->constructResourcePath(bucket, "");
+	// In virtual hosting mode, constructResourcePath returns "" for empty object.
+	// We need to ensure the query string starts with "/" to form a valid HTTP request.
+	// Commit 15dd76a7f9 switched from ListObjectsV1 to V2 and accidentally removed the leading "/".
+	if (resource.empty()) {
+		resource = "/";
+	}
+	resource.append("?list-type=2&max-keys=").append(std::to_string(CLIENT_KNOBS->BLOBSTORE_LIST_MAX_KEYS_PER_PAGE));
+
+	if (prefix.present())
+		resource.append("&prefix=").append(prefix.get());
+	if (delimiter.present())
+		resource.append("&delimiter=").append(std::string(1, delimiter.get()));
+
+	std::string continuationToken;
+	bool more = true;
+	std::vector<Future<Void>> subLists;
+
+	while (more) {
+		co_await bstore->concurrentLists.take();
+		FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
+
+		HTTP::Headers headers;
+		std::string fullResource = resource;
+		if (!continuationToken.empty()) {
+			fullResource.append("&continuation-token=").append(continuationToken);
+		}
+
+		Reference<HTTP::IncomingResponse> r =
+		    co_await bstore->doRequest("GET", fullResource, headers, nullptr, 0, { 200, 404 });
+		listReleaser.release();
+
+		try {
+			S3BlobStoreEndpoint::ListResult listResult;
+
+			// If we got a 404, throw an error to indicate the resource doesn't exist
+			if (r->code == 404) {
+				TraceEvent(SevError, "S3BlobStoreResourceNotFound")
+				    .detail("Bucket", bucket)
+				    .detail("Prefix", prefix.present() ? prefix.get() : "")
+				    .detail("Resource", fullResource);
+				throw resource_not_found();
+			}
+
+			xml_document<> doc;
+
+			// Copy content because rapidxml will modify it during parse
+			std::string content = r->data.content;
+			doc.parse<0>((char*)content.c_str());
+
+			// There should be exactly one node
+			xml_node<>* result = doc.first_node();
+			if (result == nullptr || strcmp(result->name(), "ListBucketResult") != 0) {
+				throw http_bad_response();
+			}
+
+			xml_node<>* n = result->first_node();
+			more = false;
+			continuationToken.clear();
+
+			while (n != nullptr) {
+				const char* name = n->name();
+				if (strcmp(name, "IsTruncated") == 0) {
+					const char* val = n->value();
+					if (strcmp(val, "true") == 0) {
+						more = true;
+					} else if (strcmp(val, "false") == 0) {
+						more = false;
+					} else {
+						throw http_bad_response();
+					}
+				} else if (strcmp(name, "NextContinuationToken") == 0) {
+					if (n->value() != nullptr) {
+						continuationToken = n->value();
+					}
+				} else if (strcmp(name, "Contents") == 0) {
+					S3BlobStoreEndpoint::ObjectInfo object;
+
+					xml_node<>* key = n->first_node("Key");
+					if (key == nullptr) {
+						throw http_bad_response();
+					}
+					// URL decode the object name since S3 XML responses contain URL-encoded names
+					object.name = HTTP::urlDecode(key->value());
+
+					xml_node<>* size = n->first_node("Size");
+					if (size == nullptr) {
+						throw http_bad_response();
+					}
+					object.size = strtoull(size->value(), nullptr, 10);
+
+					listResult.objects.push_back(object);
+				} else if (strcmp(name, "CommonPrefixes") == 0) {
+					xml_node<>* prefixNode = n->first_node("Prefix");
+					while (prefixNode != nullptr) {
+						const char* prefix = prefixNode->value();
+						// If recursing, queue a sub-request, otherwise add the common prefix to the result.
+						if (maxDepth > 0) {
+							if (!recurseFilter || recurseFilter(prefix)) {
+								// For recursive listing, don't use delimiter in sub-requests to get individual files
+								subLists.push_back(bstore->listObjectsStream(
+								    bucket, results, prefix, Optional<char>(), maxDepth - 1, recurseFilter));
+							}
+						} else {
+							listResult.commonPrefixes.push_back(prefix);
+						}
+						prefixNode = prefixNode->next_sibling("Prefix");
+					}
+				}
+				n = n->next_sibling();
+			}
+
+			results.send(listResult);
+		} catch (Error& e) {
+			if (e.code() != error_code_actor_cancelled) {
+				TraceEvent(SevWarn, "S3BlobStoreEndpointListResultParseError")
+				    .errorUnsuppressed(e)
+				    .suppressFor(60)
+				    .detail("Resource", fullResource);
+			}
+			throw http_bad_response();
+		}
+	}
+
+	co_await waitForAll(subLists);
+}
+
+Future<Void> S3BlobStoreEndpoint::listObjectsStream(std::string const& bucket,
+                                                    PromiseStream<ListResult> results,
+                                                    Optional<std::string> prefix,
+                                                    Optional<char> delimiter,
+                                                    int maxDepth,
+                                                    std::function<bool(std::string const&)> recurseFilter) {
+	return listObjectsStream_impl(
+	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, results, prefix, delimiter, maxDepth, recurseFilter);
+}
+
+AsyncResult<std::vector<std::string>> listBuckets_impl(Reference<S3BlobStoreEndpoint> bstore) {
+	std::string resource = "/?marker=";
+	std::string lastName;
+	bool more = true;
+	std::vector<std::string> buckets;
+
+	while (more) {
+		co_await bstore->concurrentLists.take();
+		FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
+
+		HTTP::Headers headers;
+		std::string fullResource = resource + lastName;
+		Reference<HTTP::IncomingResponse> r =
+		    co_await bstore->doRequest("GET", fullResource, headers, nullptr, 0, { 200 });
+		listReleaser.release();
+
+		try {
+			xml_document<> doc;
+
+			// Copy content because rapidxml will modify it during parse
+			std::string content = r->data.content;
+			doc.parse<0>((char*)content.c_str());
+
+			// There should be exactly one node
+			xml_node<>* result = doc.first_node();
+			if (result == nullptr || strcmp(result->name(), "ListAllMyBucketsResult") != 0) {
+				throw http_bad_response();
+			}
+
+			more = false;
+			xml_node<>* truncated = result->first_node("IsTruncated");
+			if (truncated != nullptr && strcmp(truncated->value(), "true") == 0) {
+				more = true;
+			}
+
+			xml_node<>* bucketsNode = result->first_node("Buckets");
+			if (bucketsNode != nullptr) {
+				xml_node<>* bucketNode = bucketsNode->first_node("Bucket");
+				while (bucketNode != nullptr) {
+					xml_node<>* nameNode = bucketNode->first_node("Name");
+					if (nameNode == nullptr) {
+						throw http_bad_response();
+					}
+					const char* name = nameNode->value();
+					buckets.push_back(name);
+
+					bucketNode = bucketNode->next_sibling("Bucket");
+				}
+			}
+
+			if (more) {
+				lastName = buckets.back();
+			}
+
+		} catch (Error& e) {
+			if (e.code() != error_code_actor_cancelled)
+				TraceEvent(SevWarn, "S3BlobStoreEndpointListBucketResultParseError")
+				    .errorUnsuppressed(e)
+				    .suppressFor(60)
+				    .detail("Resource", fullResource);
+			throw http_bad_response();
+		}
+	}
+
+	co_return buckets;
+}
+
+AsyncResult<std::vector<std::string>> S3BlobStoreEndpoint::listBuckets() {
+	return listBuckets_impl(Reference<S3BlobStoreEndpoint>::addRef(this));
+}
+
+std::string S3BlobStoreEndpoint::hmac_sha1(Credentials const& creds, std::string const& msg) {
+	std::string key = creds.secret;
+
+	// Hash key to shorten it if it is longer than SHA1 block size
+	if (key.size() > 64) {
+		key = SHA1::from_string(key);
+	}
+
+	// Pad key up to SHA1 block size if needed
+	key.append(64 - key.size(), '\0');
+
+	std::string kipad = key;
+	for (int i = 0; i < 64; ++i)
+		kipad[i] ^= '\x36';
+
+	std::string kopad = key;
+	for (int i = 0; i < 64; ++i)
+		kopad[i] ^= '\x5c';
+
+	kipad.append(msg);
+	std::string hkipad = SHA1::from_string(kipad);
+	kopad.append(hkipad);
+	return SHA1::from_string(kopad);
+}
+
+static void sha256(const unsigned char* data, const size_t len, unsigned char* hash) {
+	SHA256_CTX sha256;
+	SHA256_Init(&sha256);
+	SHA256_Update(&sha256, data, len);
+	SHA256_Final(hash, &sha256);
+}
+
+std::string sha256_hex(std::string str) {
+	unsigned char hash[SHA256_DIGEST_LENGTH];
+	sha256((const unsigned char*)str.c_str(), str.size(), hash);
+	std::stringstream ss;
+	for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) {
+		ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i];
+	}
+	return ss.str();
+}
+
+// Return base64'd SHA256 hash of input string.
+std::string sha256_base64(std::string str) {
+	unsigned char hash[SHA256_DIGEST_LENGTH];
+	sha256((const unsigned char*)str.c_str(), str.size(), hash);
+	std::string hashAsStr = std::string((char*)hash, SHA256_DIGEST_LENGTH);
+	std::string sig = base64::encoder::from_string(hashAsStr);
+	// base64 encoded blocks end in \n so remove last character.
+	sig.resize(sig.size() - 1);
+	return sig;
+}
+
+std::string hmac_sha256_hex(std::string key, std::string msg) {
+	unsigned char hash[32];
+
+	HMAC_CTX* hmac = HMAC_CTX_new();
+	HMAC_Init_ex(hmac, &key[0], key.length(), EVP_sha256(), nullptr);
+	HMAC_Update(hmac, (unsigned char*)&msg[0], msg.length());
+	unsigned int len = 32;
+	HMAC_Final(hmac, hash, &len);
+	HMAC_CTX_free(hmac);
+
+	std::stringstream ss;
+	ss << std::hex << std::setfill('0');
+	for (int i = 0; i < len; i++) {
+		ss << std::hex << std::setw(2) << (unsigned int)hash[i];
+	}
+	return (ss.str());
+}
+
+std::string hmac_sha256(std::string key, std::string msg) {
+	unsigned char hash[32];
+
+	HMAC_CTX* hmac = HMAC_CTX_new();
+	HMAC_Init_ex(hmac, &key[0], key.length(), EVP_sha256(), nullptr);
+	HMAC_Update(hmac, (unsigned char*)&msg[0], msg.length());
+	unsigned int len = 32;
+	HMAC_Final(hmac, hash, &len);
+	HMAC_CTX_free(hmac);
+
+	std::stringstream ss;
+	ss << std::setfill('0');
+	for (int i = 0; i < len; i++) {
+		ss << hash[i];
+	}
+	return (ss.str());
+}
+
+// Date and Time parameters are used for unit testing
+void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
+                                           std::string const& resource,
+                                           HTTP::Headers& headers,
+                                           std::string date,
+                                           std::string datestamp) {
+	if (!credentials.present()) {
+		return;
+	}
+	Credentials creds = credentials.get();
+	// std::cout << "========== Starting===========" << std::endl;
+	std::string accessKey = creds.key;
+	std::string secretKey = creds.secret;
+	// Create a date for headers and the credential string
+	std::string amzDate;
+	std::string dateStamp;
+	if (date.empty() || datestamp.empty()) {
+		time_t ts;
+		time(&ts);
+		char dateBuf[20];
+		// ISO 8601 format YYYYMMDD'T'HHMMSS'Z'
+		strftime(dateBuf, 20, "%Y%m%dT%H%M%SZ", gmtime(&ts));
+		amzDate = dateBuf;
+		strftime(dateBuf, 20, "%Y%m%d", gmtime(&ts));
+		dateStamp = dateBuf;
+	} else {
+		amzDate = date;
+		dateStamp = datestamp;
+	}
+
+	// ************* TASK 1: CREATE A CANONICAL REQUEST *************
+	// Create Create canonical URI--the part of the URI from domain to query string (use '/' if no path)
+	std::vector<std::string> queryParameters;
+	std::string canonicalURI = awsCanonicalURI(resource, queryParameters, true);
+
+	std::string canonicalQueryString;
+	if (!queryParameters.empty()) {
+		std::sort(queryParameters.begin(), queryParameters.end());
+		canonicalQueryString = boost::algorithm::join(queryParameters, "&");
+	}
+
+	using namespace boost::algorithm;
+	// Create the canonical headers and signed headers
+	ASSERT(!headers["Host"].empty());
+	// Be careful. There is x-amz-content-sha256 for auth and then
+	// x-amz-checksum-sha256 for object integrity check.
+	headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD";
+	headers["x-amz-date"] = amzDate;
+	std::vector<std::pair<std::string, std::string>> headersList;
+	headersList.push_back({ "host", trim_copy(headers["Host"]) + "\n" });
+	if (headers.contains("Content-Type"))
+		headersList.push_back({ "content-type", trim_copy(headers["Content-Type"]) + "\n" });
+	if (headers.contains("Content-MD5"))
+		headersList.push_back({ "content-md5", trim_copy(headers["Content-MD5"]) + "\n" });
+	for (const auto& [headerName, headerValue] : headers) {
+		if (StringRef(headerName).startsWith("x-amz"_sr))
+			headersList.push_back({ to_lower_copy(headerName), trim_copy(headerValue) + "\n" });
+	}
+	std::sort(headersList.begin(), headersList.end());
+	std::string canonicalHeaders;
+	std::string signedHeaders;
+	for (const auto& [headerName, headerValue] : headersList) {
+		canonicalHeaders += headerName + ":" + headerValue;
+		signedHeaders += headerName + ";";
+	}
+	signedHeaders.pop_back();
+	std::string canonicalRequest = verb + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders +
+	                               "\n" + signedHeaders + "\n" + headers["x-amz-content-sha256"];
+
+	// ************* TASK 2: CREATE THE STRING TO SIGN*************
+	std::string algorithm = "AWS4-HMAC-SHA256";
+	std::string credentialScope = dateStamp + "/" + region + "/s3/" + "aws4_request";
+	std::string stringToSign =
+	    algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + sha256_hex(canonicalRequest);
+
+	// ************* TASK 3: CALCULATE THE SIGNATURE *************
+	// Create the signing key using the function defined above.
+	std::string signingKey =
+	    hmac_sha256(hmac_sha256(hmac_sha256(hmac_sha256("AWS4" + secretKey, dateStamp), region), "s3"), "aws4_request");
+	// Sign the string_to_sign using the signing_key
+	std::string signature = hmac_sha256_hex(signingKey, stringToSign);
+	// ************* TASK 4: ADD SIGNING INFORMATION TO THE Header *************
+	std::string authorizationHeader = algorithm + " " + "Credential=" + accessKey + "/" + credentialScope + ", " +
+	                                  "SignedHeaders=" + signedHeaders + ", " + "Signature=" + signature;
+	headers["Authorization"] = authorizationHeader;
+}
+
+void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) {
+	if (!credentials.present()) {
+		return;
+	}
+	Credentials creds = credentials.get();
+
+	std::string& date = headers["Date"];
+
+	char dateBuf[64];
+	time_t ts;
+	time(&ts);
+	strftime(dateBuf, 64, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&ts));
+	date = dateBuf;
+
+	std::string msg;
+	msg.append(verb);
+	msg.append("\n");
+	auto contentMD5 = headers.find("Content-MD5");
+	if (contentMD5 != headers.end())
+		msg.append(contentMD5->second);
+	msg.append("\n");
+	auto contentType = headers.find("Content-Type");
+	if (contentType != headers.end())
+		msg.append(contentType->second);
+	msg.append("\n");
+	msg.append(date);
+	msg.append("\n");
+	for (const auto& [headerName, headerValue] : headers) {
+		StringRef name = headerName;
+		if (name.startsWith("x-amz"_sr) || name.startsWith("x-icloud"_sr)) {
+			msg.append(headerName);
+			msg.append(":");
+			msg.append(headerValue);
+			msg.append("\n");
+		}
+	}
+
+	msg.append(resource);
+	if (verb == "GET") {
+		size_t q = resource.find_last_of('?');
+		if (q != resource.npos)
+			msg.resize(msg.size() - (resource.size() - q));
+	}
+
+	std::string sig = base64::encoder::from_string(hmac_sha1(creds, msg));
+	// base64 encoded blocks end in \n so remove it.
+	sig.resize(sig.size() - 1);
+	std::string auth = "AWS ";
+	auth.append(creds.key);
+	auth.append(":");
+	auth.append(sig);
+	headers["Authorization"] = auth;
+}
+
+AsyncResult<std::string> readEntireFile_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                             std::string bucket,
+                                             std::string object) {
+	co_await bstore->requestRateRead->getAllowance(1);
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	HTTP::Headers headers;
+	// Set this header on the GET for it to volunteer saved checksum in the response headers.
+	// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax
+	headers["x-amz-checksum-mode"] = "ENABLED";
+	Reference<HTTP::IncomingResponse> r =
+	    co_await bstore->doRequest("GET", resource, headers, nullptr, 0, { 200, 404 });
+	if (r->code == 404)
+		throw file_not_found();
+	if (bstore->knobs.enable_object_integrity_check) {
+		// Verify the content. We set 'x-amz-checksum-mode' on the GET request above so
+		// the server will return the sha256 checksum we set when we uploaded the object in the
+		// GET response headers. See
+		// https://stackoverflow.com/questions/36540234/what-is-the-difference-getcontentmd5-and-getetag-of-aws-s3-putobjectresult
+		std::string checksumSHA256 = r->data.headers["x-amz-checksum-sha256"];
+		if (checksumSHA256.empty()) {
+			// This is what is thrown elsewhere when no expected etag.
+			throw http_bad_response();
+		}
+		// Calculate the sha256 checksum of the content and compare it to the checksum returned by the server.
+		std::string contentSHA256 = sha256_base64(r->data.content);
+		if (checksumSHA256 != contentSHA256) {
+			throw checksum_failed();
+		}
+	}
+	co_return r->data.content;
+}
+
+AsyncResult<std::string> S3BlobStoreEndpoint::readEntireFile(std::string const& bucket, std::string const& object) {
+	return readEntireFile_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
+}
+
+Future<Void> writeEntireFileFromBuffer_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                            std::string bucket,
+                                            std::string object,
+                                            UnsentPacketQueue* pContent,
+                                            int contentLen,
+                                            std::string contentHash) {
+	if (contentLen > bstore->knobs.multipart_max_part_size)
+		throw file_too_large();
+
+	co_await bstore->requestRateWrite->getAllowance(1);
+	co_await bstore->concurrentUploads.take();
+	FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	HTTP::Headers headers;
+	// contentHash is calculated by the caller. It is md5 or sha256 dependent on
+	// enable_object_integrity_check setting. If the hash (md5 or sha256) we
+	// volunteer doesn't match that calculated serverside, the upload fails with:
+	// InvalidDigest</Code><Message>The Content-MD5 you specified was invalid.</Message>
+	if (bstore->knobs.enable_object_integrity_check) {
+		headers["x-amz-checksum-sha256"] = contentHash;
+		headers["x-amz-checksum-algorithm"] = "SHA256";
+	} else {
+		headers["Content-MD5"] = contentHash;
+	}
+	if (!CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE.empty())
+		headers["x-amz-server-side-encryption"] = CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE;
+	Reference<HTTP::IncomingResponse> r =
+	    co_await bstore->doRequest("PUT", resource, headers, pContent, contentLen, { 200 });
+}
+
+Future<Void> S3BlobStoreEndpoint::writeEntireFileFromBuffer(std::string const& bucket,
+                                                            std::string const& object,
+                                                            UnsentPacketQueue* pContent,
+                                                            int contentLen,
+                                                            std::string const& contentHash) {
+	return writeEntireFileFromBuffer_impl(
+	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, pContent, contentLen, contentHash);
+}
+
+Future<int> readObject_impl(Reference<S3BlobStoreEndpoint> bstore,
+                            std::string bucket,
+                            std::string object,
+                            void* data,
+                            int length,
+                            int64_t offset) {
+	try {
+		if (length <= 0) {
+			TraceEvent(SevWarn, "S3BlobStoreReadObjectEmptyRead").detail("Length", length);
+			co_return 0;
+		}
+
+		// Log rate limiter state
+		co_await bstore->requestRateRead->getAllowance(1);
+
+		std::string resource = bstore->constructResourcePath(bucket, object);
+		HTTP::Headers headers;
+		headers["Range"] = format("bytes=%lld-%lld", offset, offset + length - 1);
+
+		// Attempt the request
+		Reference<HTTP::IncomingResponse> r;
+		Reference<HTTP::IncomingResponse> _r =
+		    co_await bstore->doRequest("GET", resource, headers, nullptr, 0, { 200, 206, 404 });
+		r = _r;
+
+		if (r->code == 404) {
+			throw file_not_found();
+		}
+
+		// Verify response has content
+		if (r->data.contentLen != r->data.content.size()) {
+			TraceEvent(SevWarn, "S3BlobStoreReadObjectContentLengthMismatch")
+			    .detail("Expected", r->data.contentLen)
+			    .detail("Actual", r->data.content.size());
+			throw io_error();
+		}
+
+		// Copy the output bytes, server could have sent more or less bytes than requested so copy at most length
+		// bytes
+		int bytesToCopy = std::min<int64_t>(r->data.contentLen, length);
+		memcpy(data, r->data.content.data(), bytesToCopy);
+		// Return the number of bytes actually copied
+		co_return bytesToCopy;
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "S3BlobStoreEndpoint_ReadError")
+		    .error(e)
+		    .detail("Bucket", bucket)
+		    .detail("Object", object)
+		    .detail("Length", length)
+		    .detail("Offset", offset);
+		throw;
+	}
+}
+
+Future<int> S3BlobStoreEndpoint::readObject(std::string const& bucket,
+                                            std::string const& object,
+                                            void* data,
+                                            int length,
+                                            int64_t offset) {
+	return readObject_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, data, length, offset);
+}
+
+static Future<std::string> beginMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                                     std::string bucket,
+                                                     std::string object) {
+	co_await bstore->requestRateWrite->getAllowance(1);
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	resource += "?uploads";
+	HTTP::Headers headers;
+	if (!CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE.empty())
+		headers["x-amz-server-side-encryption"] = CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE;
+	if (bstore->knobs.enable_object_integrity_check) {
+		// CRITICAL: Adding x-amz-checksum-algorithm to multipart initiation means ALL parts
+		// must include checksums and completion request must include ChecksumSHA256 tags.
+		// AWS S3 will reject completion if any part checksum is missing.
+		headers["x-amz-checksum-algorithm"] = "SHA256";
+	}
+	Reference<HTTP::IncomingResponse> r = co_await bstore->doRequest("POST", resource, headers, nullptr, 0, { 200 });
+
+	try {
+		xml_document<> doc;
+		// Copy content because rapidxml will modify it during parse
+		std::string content = r->data.content;
+
+		doc.parse<0>((char*)content.c_str());
+
+		// There should be exactly one node
+		xml_node<>* result = doc.first_node();
+		if (result != nullptr && strcmp(result->name(), "InitiateMultipartUploadResult") == 0) {
+			xml_node<>* id = result->first_node("UploadId");
+			if (id != nullptr) {
+				co_return id->value();
+			}
+		}
+	} catch (...) {
+	}
+	throw http_bad_response();
+}
+
+Future<std::string> S3BlobStoreEndpoint::beginMultiPartUpload(std::string const& bucket, std::string const& object) {
+	return beginMultiPartUpload_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
+}
+
+Future<std::string> uploadPart_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                    std::string bucket,
+                                    std::string object,
+                                    std::string uploadID,
+                                    unsigned int partNumber,
+                                    UnsentPacketQueue* pContent,
+                                    int contentLen,
+                                    std::string contentMD5) {
+	co_await bstore->requestRateWrite->getAllowance(1);
+	co_await bstore->concurrentUploads.take();
+	FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	resource += format("?partNumber=%d&uploadId=%s", partNumber, uploadID.c_str());
+	HTTP::Headers headers;
+	// Send hash for content so blobstore can verify it
+	// Use SHA256 if integrity check is enabled, otherwise use MD5
+	if (bstore->knobs.enable_object_integrity_check) {
+		headers["x-amz-checksum-sha256"] = contentMD5;
+		headers["x-amz-checksum-algorithm"] = "SHA256";
+	} else {
+		headers["Content-MD5"] = contentMD5;
+	}
+	Reference<HTTP::IncomingResponse> r =
+	    co_await bstore->doRequest("PUT", resource, headers, pContent, contentLen, { 200 });
+	// TODO:  In the event that the client times out just before the request completes (so the client is unaware) then
+	// the next retry will see error 400.  That could be detected and handled gracefully by retrieving the etag for the
+	// successful request.
+
+	// For uploads, Blobstore returns an MD5 sum of uploaded content so check it.
+	// Only verify MD5 when not using integrity check (SHA256 verification is done by AWS)
+	if (!bstore->knobs.enable_object_integrity_check) {
+		if (!HTTP::verifyMD5(&r->data, false, contentMD5))
+			throw checksum_failed();
+	}
+	// Note: When using SHA256, AWS handles the verification internally
+	// and will return an error if the checksum doesn't match
+
+	// No etag -> bad response.
+	std::string etag = r->data.headers["ETag"];
+	if (etag.empty())
+		throw http_bad_response();
+
+	co_return etag;
+}
+
+Future<std::string> S3BlobStoreEndpoint::uploadPart(std::string const& bucket,
+                                                    std::string const& object,
+                                                    std::string const& uploadID,
+                                                    unsigned int partNumber,
+                                                    UnsentPacketQueue* pContent,
+                                                    int contentLen,
+                                                    std::string const& contentMD5) {
+	return uploadPart_impl(Reference<S3BlobStoreEndpoint>::addRef(this),
+	                       bucket,
+	                       object,
+	                       uploadID,
+	                       partNumber,
+	                       pContent,
+	                       contentLen,
+	                       contentMD5);
+}
+
+Future<Optional<std::string>> finishMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                                         std::string bucket,
+                                                         std::string object,
+                                                         std::string uploadID,
+                                                         S3BlobStoreEndpoint::MultiPartSetT parts,
+                                                         int64_t /*totalSize*/) {
+	UnsentPacketQueue part_list; // NonCopyable state var so must be declared at top of actor
+	co_await bstore->requestRateWrite->getAllowance(1);
+
+	std::string manifest = "<CompleteMultipartUpload>";
+	for (auto& p : parts) {
+		manifest += format("<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>", p.first, p.second.etag.c_str());
+		// Include checksum if integrity check is enabled and checksum is present
+		if (bstore->knobs.enable_object_integrity_check && !p.second.checksum.empty()) {
+			manifest += format("<ChecksumSHA256>%s</ChecksumSHA256>", p.second.checksum.c_str());
+		}
+		manifest += "</Part>\n";
+	}
+	manifest += "</CompleteMultipartUpload>";
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	resource += format("?uploadId=%s", uploadID.c_str());
+	HTTP::Headers headers;
+	PacketWriter pw(part_list.getWriteBuffer(manifest.size()), nullptr, Unversioned());
+	pw.serializeBytes(manifest);
+	Reference<HTTP::IncomingResponse> r =
+	    co_await bstore->doRequest("POST", resource, headers, &part_list, manifest.size(), { 200 });
+
+	// The XML response contains a ChecksumSHA256 field, but this is just a hash of the multipart
+	// structure, not the actual object content, so it's useless for integrity verification.
+	// We skip parsing it to avoid wasted CPU cycles.
+
+	// TODO:  In the event that the client times out just before the request completes (so the client is unaware) then
+	// the next retry will see error 400.  That could be detected and handled gracefully by HEAD'ing the object before
+	// upload to get its (possibly nonexistent) eTag, then if an error 400 is seen then retrieve the eTag again and if
+	// it has changed then consider the finish complete.
+	co_return Optional<std::string>();
+}
+
+Future<Optional<std::string>> S3BlobStoreEndpoint::finishMultiPartUpload(std::string const& bucket,
+                                                                         std::string const& object,
+                                                                         std::string const& uploadID,
+                                                                         MultiPartSetT const& parts,
+                                                                         int64_t totalSize) {
+	return finishMultiPartUpload_impl(
+	    Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, uploadID, parts, totalSize);
+}
+
+Future<Void> abortMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                       std::string bucket,
+                                       std::string object,
+                                       std::string uploadID) {
+	co_await bstore->requestRateWrite->getAllowance(1);
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	resource += format("?uploadId=%s", uploadID.c_str());
+
+	HTTP::Headers headers;
+	Reference<HTTP::IncomingResponse> r =
+	    co_await bstore->doRequest("DELETE", resource, headers, nullptr, 0, { 200, 204 });
+}
+
+Future<Void> S3BlobStoreEndpoint::abortMultiPartUpload(std::string const& bucket,
+                                                       std::string const& object,
+                                                       std::string const& uploadID) {
+	return abortMultiPartUpload_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, uploadID);
+}
+
+// Forward declarations
+Future<std::map<std::string, std::string>> getObjectTags_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                                              std::string bucket,
+                                                              std::string object);
+
+Future<Void> putObjectTags_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                std::string bucket,
+                                std::string object,
+                                std::map<std::string, std::string> tags) {
+	UnsentPacketQueue packets;
+	co_await bstore->requestRateWrite->getAllowance(1);
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	resource += "?tagging";
+	int maxRetries = 5;
+	int retryCount = 0;
+	double backoff = 1.0;
+	double maxBackoff = 8.0;
+
+	while (true) {
+		Error err;
+		try {
+			std::string manifest = "<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet>";
+			for (auto itr = tags.begin(); itr != tags.end(); ++itr) {
+				manifest += "<Tag><Key>" + itr->first + "</Key><Value>" + itr->second + "</Value></Tag>";
+			}
+			manifest += "</TagSet></Tagging>";
+
+			PacketWriter pw(packets.getWriteBuffer(manifest.size()), nullptr, Unversioned());
+			pw.serializeBytes(manifest);
+
+			HTTP::Headers headers;
+			headers["Content-Type"] = "application/xml";
+			headers["Content-Length"] = format("%d", manifest.size());
+
+			Reference<HTTP::IncomingResponse> r =
+			    co_await bstore->doRequest("PUT", resource, headers, &packets, manifest.size(), { 200 });
+
+			// Verify tags were written correctly
+			std::map<std::string, std::string> verifyTags = co_await getObjectTags_impl(bstore, bucket, object);
+			if (verifyTags == tags) {
+				co_return;
+			}
+
+			if (++retryCount >= maxRetries) {
+				TraceEvent(SevWarn, "S3BlobStorePutTagsMaxRetriesExceeded")
+				    .detail("Bucket", bucket)
+				    .detail("Object", object);
+				throw operation_failed();
+			}
+
+			// Implement exponential backoff with jitter
+			co_await delay(backoff * (0.9 + 0.2 * deterministicRandom()->random01()));
+			backoff = std::min(backoff * 2, maxBackoff);
+			continue;
+		} catch (Error& e) {
+			err = e;
+		}
+
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+
+		TraceEvent(SevWarn, "S3BlobStorePutTagsError")
+		    .error(err)
+		    .detail("Bucket", bucket)
+		    .detail("Object", object)
+		    .detail("RetryCount", retryCount);
+
+		if (++retryCount >= maxRetries) {
+			throw err;
+		}
+
+		// Implement exponential backoff with jitter for errors
+		co_await delay(backoff * (0.9 + 0.2 * deterministicRandom()->random01()));
+		backoff = std::min(backoff * 2, maxBackoff);
+	}
+}
+
+Future<Void> S3BlobStoreEndpoint::putObjectTags(std::string const& bucket,
+                                                std::string const& object,
+                                                std::map<std::string, std::string> const& tags) {
+	return putObjectTags_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object, tags);
+}
+
+Future<std::map<std::string, std::string>> getObjectTags_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                                              std::string bucket,
+                                                              std::string object) {
+	co_await bstore->requestRateRead->getAllowance(1);
+
+	std::string resource = bstore->constructResourcePath(bucket, object);
+	resource += "?tagging";
+	HTTP::Headers headers;
+
+	Reference<HTTP::IncomingResponse> r = co_await bstore->doRequest("GET", resource, headers, nullptr, 0, { 200 });
+
+	rapidxml::xml_document<> doc;
+	doc.parse<rapidxml::parse_default>((char*)r->data.content.c_str());
+
+	std::map<std::string, std::string> tags;
+
+	// Find the Tagging node (with or without namespace)
+	rapidxml::xml_node<>* tagging = doc.first_node();
+	while (tagging && strcmp(tagging->name(), "Tagging") != 0) {
+		tagging = tagging->next_sibling();
+	}
+
+	if (tagging) {
+		// Find TagSet node
+		rapidxml::xml_node<>* tagSet = tagging->first_node();
+		while (tagSet && strcmp(tagSet->name(), "TagSet") != 0) {
+			tagSet = tagSet->next_sibling();
+		}
+
+		if (tagSet) {
+			// Iterate through Tag nodes
+			for (rapidxml::xml_node<>* tag = tagSet->first_node(); tag; tag = tag->next_sibling()) {
+				if (strcmp(tag->name(), "Tag") == 0) {
+					std::string key, value;
+					// Find Key and Value nodes
+					for (rapidxml::xml_node<>* node = tag->first_node(); node; node = node->next_sibling()) {
+						if (strcmp(node->name(), "Key") == 0) {
+							key = node->value();
+						} else if (strcmp(node->name(), "Value") == 0) {
+							value = node->value();
+						}
+					}
+					if (!key.empty()) {
+						tags[key] = value;
+					}
+				}
+			}
+		}
+	}
+
+	co_return tags;
+}
+
+Future<std::map<std::string, std::string>> S3BlobStoreEndpoint::getObjectTags(std::string const& bucket,
+                                                                              std::string const& object) {
+	return getObjectTags_impl(Reference<S3BlobStoreEndpoint>::addRef(this), bucket, object);
+}
+
+TEST_CASE("/backup/s3/v4headers") {
+	std::string credStr = "AKIAIOSFODNN7EXAMPLE:wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY:";
+	Optional<StringRef> creds = StringRef(credStr);
+	// GET without query parameters
+	{
+		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
+		std::string verb("GET");
+		std::string resource("/test.txt");
+		HTTP::Headers headers;
+		headers["Host"] = "s3.amazonaws.com";
+		s3.setV4AuthHeaders(verb, resource, headers, "20130524T000000Z", "20130524");
+		ASSERT(headers["Authorization"] ==
+		       "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/amazonaws/s3/aws4_request, "
+		       "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+		       "Signature=c6037f4b174f2019d02d7085a611cef8adfe1efe583e220954dc85d59cd31ba3");
+		ASSERT(headers["x-amz-date"] == "20130524T000000Z");
+	}
+
+	// GET with query parameters
+	{
+		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
+		std::string verb("GET");
+		std::string resource("/test/examplebucket?Action=DescribeRegions&Version=2013-10-15");
+		HTTP::Headers headers;
+		headers["Host"] = "s3.amazonaws.com";
+		s3.setV4AuthHeaders(verb, resource, headers, "20130524T000000Z", "20130524");
+		ASSERT(headers["Authorization"] ==
+		       "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/amazonaws/s3/aws4_request, "
+		       "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+		       "Signature=426f04e71e191fbc30096c306fe1b11ce8f026a7be374541862bbee320cce71c");
+		ASSERT(headers["x-amz-date"] == "20130524T000000Z");
+	}
+
+	// POST
+	{
+		S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "443", "us-west-2", "proxy", "port", creds);
+		std::string verb("POST");
+		std::string resource("/simple.json");
+		HTTP::Headers headers;
+		headers["Host"] = "s3.us-west-2.amazonaws.com";
+		headers["Content-Type"] = "Application/x-amz-json-1.0";
+		s3.setV4AuthHeaders(verb, resource, headers, "20130524T000000Z", "20130524");
+		ASSERT(headers["Authorization"] ==
+		       "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-west-2/s3/aws4_request, "
+		       "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, "
+		       "Signature=cf095e36bed9cd3139c2e8b3e20c296a79d8540987711bf3a0d816b19ae00314");
+		ASSERT(headers["x-amz-date"] == "20130524T000000Z");
+		ASSERT(headers["Host"] == "s3.us-west-2.amazonaws.com");
+		ASSERT(headers["Content-Type"] == "Application/x-amz-json-1.0");
+	}
+
+	return Void();
+}
+
+TEST_CASE("/backup/s3/guess_region") {
+	std::string url = "blobstore://s3.us-west-2.amazonaws.com/resource_name?bucket=bucket_name&sa=1";
+
+	std::string resource;
+	std::string error;
+	S3BlobStoreEndpoint::ParametersT parameters;
+	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+	ASSERT(s3->getRegion() == "us-west-2");
+
+	url = "blobstore://s3.us-west-2.amazonaws.com/resource_name?bucket=bucket_name&sc=922337203685477580700";
+	try {
+		s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+		ASSERT(false); // not reached
+	} catch (Error& e) {
+		// conversion of 922337203685477580700 to long int will overflow
+		ASSERT_EQ(e.code(), error_code_backup_invalid_url);
+	}
+	return Void();
+}
+
+TEST_CASE("/backup/s3/virtual_hosting_list_resource_path") {
+	// Test that listObjectsStream_impl constructs valid HTTP resource paths
+	// for virtual hosting mode URLs with path prefixes.
+	//
+	// Regression test for commit 15dd76a7f9 which accidentally removed the leading "/"
+	// from the query string when switching from ListObjectsV1 to ListObjectsV2.
+	//
+	// For URL: blobstore://@bucket.s3.region.amazonaws.com/prefix/path?bucket=...
+	// constructResourcePath(bucket, "") returns "" in virtual hosting mode.
+	// We must ensure the final resource starts with "/" before the query string.
+
+	std::string url = "blobstore://@test-bucket.s3.us-west-2.amazonaws.com/bulkload/path?bucket=test-bucket";
+	std::string resource;
+	std::string error;
+	S3BlobStoreEndpoint::ParametersT parameters;
+	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+
+	ASSERT(s3.isValid());
+	ASSERT(resource == "bulkload/path"); // Path prefix extracted from URL (without leading '/')
+
+	// In virtual hosting mode, constructResourcePath returns "" for empty object
+	std::string listResource = s3->constructResourcePath("test-bucket", "");
+	ASSERT(listResource.empty()); // Virtual hosting mode doesn't include bucket in path
+
+	// listObjectsStream_impl should add "/" before query string to form valid HTTP request
+	// Expected: GET /bulkload/path?list-type=2&... (but we can't test the full actor here)
+	// We can only verify that constructResourcePath returns "" as expected.
+	// The fix in listObjectsStream_impl checks if resource.empty() and sets it to "/" before
+	// appending the query string, ensuring the final request is well-formed.
+
+	// Test non-virtual hosting mode for comparison
+	url = "blobstore://s3.us-west-2.amazonaws.com/resource_path?bucket=test-bucket";
+	s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+	ASSERT(s3.isValid());
+	ASSERT(resource == "resource_path"); // Path extracted from URL (without leading '/')
+
+	// In non-virtual hosting mode, constructResourcePath includes bucket
+	listResource = s3->constructResourcePath("test-bucket", "");
+	ASSERT(listResource == "/test-bucket"); // Bucket is part of path
+
+	return Void();
+}
+
+TEST_CASE("/backup/s3/constructResourcePath") {
+	// Test that leading slashes in object keys are properly stripped
+	// S3 object keys should not start with '/', but if passed, we normalize them
+
+	std::string url = "blobstore://s3.us-west-2.amazonaws.com?bucket=test-bucket";
+	std::string resource;
+	std::string error;
+	S3BlobStoreEndpoint::ParametersT parameters;
+	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+
+	// Test normal object key (no leading slash)
+	ASSERT(s3->constructResourcePath("test-bucket", "normal/path/file.txt") == "/test-bucket/normal/path/file.txt");
+
+	// Test single leading slash - should be stripped
+	ASSERT(s3->constructResourcePath("test-bucket", "/leading/slash/file.txt") ==
+	       "/test-bucket/leading/slash/file.txt");
+
+	// Test multiple leading slashes - all should be stripped
+	ASSERT(s3->constructResourcePath("test-bucket", "///multiple/slashes.txt") == "/test-bucket/multiple/slashes.txt");
+
+	// Test object key that is only slashes - should result in empty object path
+	ASSERT(s3->constructResourcePath("test-bucket", "///") == "/test-bucket");
+
+	// Test empty object key
+	ASSERT(s3->constructResourcePath("test-bucket", "") == "/test-bucket");
+
+	// Test virtual hosting mode (bucket in hostname)
+	url = "blobstore://test-bucket.s3.us-west-2.amazonaws.com";
+	s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+
+	// Virtual hosting mode doesn't include bucket in path
+	ASSERT(s3->constructResourcePath("test-bucket", "normal/file.txt") == "/normal/file.txt");
+	ASSERT(s3->constructResourcePath("test-bucket", "/leading/slash.txt") == "/leading/slash.txt");
+	ASSERT(s3->constructResourcePath("test-bucket", "").empty());
+
+	return Void();
+}
+
+TEST_CASE("/backup/s3/parseErrorCodeFromS3") {
+	// Valid S3 error XML — should extract the error code
+	{
+		std::string xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+		                  "<Error><Code>AccessDenied</Code>"
+		                  "<Message>Access Denied</Message></Error>";
+		ASSERT(parseErrorCodeFromS3(xml) == "AccessDenied");
+	}
+
+	// InvalidToken error
+	{
+		std::string xml = "<Error><Code>InvalidToken</Code>"
+		                  "<Message>The provided token is malformed or otherwise invalid.</Message></Error>";
+		ASSERT(parseErrorCodeFromS3(xml) == "InvalidToken");
+	}
+
+	// ExpiredToken error
+	{
+		std::string xml = "<Error><Code>ExpiredToken</Code><Message>Token expired</Message></Error>";
+		ASSERT(parseErrorCodeFromS3(xml) == "ExpiredToken");
+	}
+
+	// Missing <Code> node — should return ""
+	{
+		std::string xml = "<Error><Message>Something went wrong</Message></Error>";
+		ASSERT(parseErrorCodeFromS3(xml).empty());
+	}
+
+	// No <Error> root — should return ""
+	{
+		std::string xml = "<ListBucketResult><Name>bucket</Name></ListBucketResult>";
+		ASSERT(parseErrorCodeFromS3(xml).empty());
+	}
+
+	// Empty response — should return ""
+	{
+		ASSERT(parseErrorCodeFromS3("").empty());
+	}
+
+	// HTML error page from load balancer (502/503) — should return "", not throw
+	{
+		std::string html = "<html><body><h1>502 Bad Gateway</h1></body></html>";
+		ASSERT(parseErrorCodeFromS3(html).empty());
+	}
+
+	// Completely invalid / garbage — should return "", not throw
+	{
+		ASSERT(parseErrorCodeFromS3("not xml at all {{{").empty());
+	}
+
+	// Partial XML — should return "", not throw
+	{
+		ASSERT(parseErrorCodeFromS3("<Error><Code>Incomple").empty());
+	}
+
+	// Plain text error — should return "", not throw
+	{
+		ASSERT(parseErrorCodeFromS3("Internal Server Error").empty());
+	}
+
+	return Void();
+}
diff --git a/fdbclient/S3Client.actor.cpp b/fdbclient/S3Client.cpp
similarity index 62%
rename from fdbclient/S3Client.actor.cpp
rename to fdbclient/S3Client.cpp
index 7854d46594c..629051580f8 100644
--- a/fdbclient/S3Client.actor.cpp
+++ b/fdbclient/S3Client.cpp
@@ -1,5 +1,5 @@
 /*
- * S3Client.actor.cpp
+ * S3Client.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,19 +20,15 @@
 
 #include <string>
 #include <vector>
-#include <unordered_set>
 #include <algorithm>
-#include <sstream>
-#include <iomanip>
 
 #ifdef _WIN32
 #include <io.h>
 #endif
 
-#include "fdbclient/S3Client.actor.h"
+#include "fdbclient/S3Client.h"
 #include "flow/IAsyncFile.h"
 #include "flow/Trace.h"
-#include "flow/Traceable.h"
 #include "flow/flow.h"
 #include "flow/xxhash.h"
 #include "flow/Error.h"
@@ -40,13 +36,11 @@
 #include <openssl/sha.h>
 #include "libb64/encode.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 // Configuration constants
 #define S3_CHECKSUM_TAG_NAME "xxhash64"
 #define S3_CHECKSUM_FILE_SUFFIX ".checksum"
 
-typedef XXH64_state_t XXHashState;
+using XXHashState = XXH64_state_t;
 
 using ::format; // Use FoundationDB's format, not std::format
 
@@ -87,23 +81,23 @@ struct PartConfig {
 // Uses xxhash library because it's fast (supposedly) and used elsewhere in fdb.
 // If size is -1, the function will determine the file size automatically.
 // Returns a hex string representation of the xxhash64 checksum.
-ACTOR Future<std::string> calculateFileChecksum(Reference<IAsyncFile> file, int64_t size) {
-	state int64_t pos = 0;
-	state XXH64_state_t* hashState = XXH64_createState();
-	state std::shared_ptr<std::vector<uint8_t>> buffer = std::make_shared<std::vector<uint8_t>>(65536);
-	state int readSize;
+AsyncResult<std::string> calculateFileChecksum(Reference<IAsyncFile> file, int64_t size) {
+	int64_t pos = 0;
+	XXH64_state_t* hashState = XXH64_createState();
+	std::shared_ptr<std::vector<uint8_t>> buffer = std::make_shared<std::vector<uint8_t>>(65536);
+	int readSize{ 0 };
 
 	XXH64_reset(hashState, 0);
 
 	try {
 		if (size == -1) {
-			int64_t s = wait(file->size());
+			int64_t s = co_await file->size();
 			size = s;
 		}
 
 		while (pos < size) {
 			readSize = std::min<int64_t>(buffer->size(), size - pos);
-			int bytesRead = wait(uncancellable(holdWhile(buffer, file->read(buffer->data(), readSize, pos))));
+			int bytesRead = co_await uncancellable(holdWhile(buffer, file->read(buffer->data(), readSize, pos)));
 			if (bytesRead != readSize) {
 				XXH64_freeState(hashState);
 				TraceEvent(SevError, "S3ClientCalculateChecksumReadError")
@@ -118,7 +112,7 @@ ACTOR Future<std::string> calculateFileChecksum(Reference<IAsyncFile> file, int6
 
 		uint64_t hash = XXH64_digest(hashState);
 		XXH64_freeState(hashState);
-		return format("%016llx", hash);
+		co_return format("%016llx", hash);
 	} catch (Error& e) {
 		XXH64_freeState(hashState);
 		throw;
@@ -185,29 +179,29 @@ bool isRetryableError(int errorCode) {
 }
 
 // Write checksum with configurable fallback strategy
-ACTOR static Future<Void> writeChecksumWithFallback(Reference<S3BlobStoreEndpoint> endpoint,
-                                                    std::string bucket,
-                                                    std::string objectName,
-                                                    std::string checksum,
-                                                    PartConfig config) {
+static Future<Void> writeChecksumWithFallback(Reference<S3BlobStoreEndpoint> endpoint,
+                                              std::string bucket,
+                                              std::string objectName,
+                                              std::string checksum,
+                                              PartConfig config) {
 	if (!config.enableChecksumValidation) {
 		TraceEvent(SevWarn, "S3ClientChecksumValidationDisabled")
 		    .suppressFor(60)
 		    .detail("Bucket", bucket)
 		    .detail("Object", objectName);
-		return Void(); // Skip checksum storage if disabled
+		co_return; // Skip checksum storage if disabled
 	}
 
 	// Always try tags first
 	try {
-		state std::map<std::string, std::string> tags;
+		std::map<std::string, std::string> tags;
 		tags[S3_CHECKSUM_TAG_NAME] = checksum;
-		wait(endpoint->putObjectTags(bucket, objectName, tags));
+		co_await endpoint->putObjectTags(bucket, objectName, tags);
 		TraceEvent(SevDebug, "S3ClientChecksumStoredAsTags")
 		    .detail("Bucket", bucket)
 		    .detail("Object", objectName)
 		    .detail("Checksum", checksum);
-		return Void();
+		co_return;
 	} catch (Error& e) {
 		if (e.code() != error_code_http_bad_response && e.code() != error_code_file_not_found) {
 			throw;
@@ -219,37 +213,36 @@ ACTOR static Future<Void> writeChecksumWithFallback(Reference<S3BlobStoreEndpoin
 	}
 
 	// Use companion file (either by preference or as fallback)
-	wait(endpoint->writeEntireFile(bucket, objectName + S3_CHECKSUM_FILE_SUFFIX, checksum));
+	co_await endpoint->writeEntireFile(bucket, objectName + S3_CHECKSUM_FILE_SUFFIX, checksum);
 	TraceEvent(SevDebug, "S3ClientChecksumStoredAsFile")
 	    .detail("Bucket", bucket)
 	    .detail("Object", objectName)
 	    .detail("ChecksumFile", objectName + S3_CHECKSUM_FILE_SUFFIX)
 	    .detail("Checksum", checksum);
-	return Void();
 }
 
 // Read checksum with configurable fallback strategy
-ACTOR static Future<Optional<std::string>> readChecksumWithFallback(Reference<S3BlobStoreEndpoint> endpoint,
-                                                                    std::string bucket,
-                                                                    std::string objectName,
-                                                                    PartConfig config) {
+static AsyncResult<Optional<std::string>> readChecksumWithFallback(Reference<S3BlobStoreEndpoint> endpoint,
+                                                                   std::string bucket,
+                                                                   std::string objectName,
+                                                                   PartConfig config) {
 	if (!config.enableChecksumValidation) {
 		TraceEvent(SevDebug, "S3ClientChecksumValidationDisabled")
 		    .detail("Bucket", bucket)
 		    .detail("Object", objectName);
-		return Optional<std::string>(); // Skip checksum validation if disabled
+		co_return Optional<std::string>(); // Skip checksum validation if disabled
 	}
 
 	// Always try tags first
 	try {
-		state std::map<std::string, std::string> tags = wait(endpoint->getObjectTags(bucket, objectName));
+		std::map<std::string, std::string> tags = co_await endpoint->getObjectTags(bucket, objectName);
 		auto it = tags.find(S3_CHECKSUM_TAG_NAME);
 		if (it != tags.end() && !it->second.empty()) {
 			TraceEvent(SevDebug, "S3ClientChecksumFoundInTags")
 			    .detail("Bucket", bucket)
 			    .detail("Object", objectName)
 			    .detail("Checksum", it->second);
-			return Optional<std::string>(it->second);
+			co_return Optional<std::string>(it->second);
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_http_bad_response && e.code() != error_code_file_not_found) {
@@ -263,13 +256,13 @@ ACTOR static Future<Optional<std::string>> readChecksumWithFallback(Reference<S3
 
 	// Try companion file
 	try {
-		std::string checksum = wait(endpoint->readEntireFile(bucket, objectName + S3_CHECKSUM_FILE_SUFFIX));
+		std::string checksum = co_await endpoint->readEntireFile(bucket, objectName + S3_CHECKSUM_FILE_SUFFIX);
 		TraceEvent(SevDebug, "S3ClientChecksumFoundInFile")
 		    .detail("Bucket", bucket)
 		    .detail("Object", objectName)
 		    .detail("ChecksumFile", objectName + S3_CHECKSUM_FILE_SUFFIX)
 		    .detail("Checksum", checksum);
-		return Optional<std::string>(checksum);
+		co_return Optional<std::string>(checksum);
 	} catch (Error& e) {
 		if (e.code() != error_code_file_not_found) {
 			throw;
@@ -280,24 +273,24 @@ ACTOR static Future<Optional<std::string>> readChecksumWithFallback(Reference<S3
 	    .detail("Bucket", bucket)
 	    .detail("Object", objectName)
 	    .detail("ChecksumValidationEnabled", config.enableChecksumValidation ? "true" : "false");
-	return Optional<std::string>();
+	co_return Optional<std::string>();
 }
 
 // Upload a part of a multipart upload with configurable retry logic.
 
-ACTOR static Future<PartState> uploadPart(Reference<S3BlobStoreEndpoint> endpoint,
-                                          std::string bucket,
-                                          std::string objectName,
-                                          std::string uploadID,
-                                          Reference<IAsyncFile> file,
-                                          PartState part,
-                                          PartConfig config) {
-	state double startTime = now();
-	state PartState resultPart = part;
-	state int attempt = 0;
-	state int maxRetries = config.maxPartRetries;
-	state int delayMs = config.baseRetryDelayMs;
-	state UnsentPacketQueue packets;
+static Future<PartState> uploadPart(Reference<S3BlobStoreEndpoint> endpoint,
+                                    std::string bucket,
+                                    std::string objectName,
+                                    std::string uploadID,
+                                    Reference<IAsyncFile> file,
+                                    PartState part,
+                                    PartConfig config) {
+	double startTime = now();
+	PartState resultPart = part;
+	int attempt = 0;
+	int maxRetries = config.maxPartRetries;
+	int delayMs = config.baseRetryDelayMs;
+	UnsentPacketQueue packets;
 
 	TraceEvent(SevDebug, "S3ClientUploadPartStart")
 	    .detail("Bucket", StringRef(bucket))
@@ -307,13 +300,14 @@ ACTOR static Future<PartState> uploadPart(Reference<S3BlobStoreEndpoint> endpoin
 	    .detail("Size", resultPart.size)
 	    .detail("MaxRetries", maxRetries);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			// Read part data from file (automatic memory management)
-			state std::shared_ptr<std::string> partData = std::make_shared<std::string>(resultPart.size, '\0');
+			std::shared_ptr<std::string> partData = std::make_shared<std::string>(resultPart.size, '\0');
 
-			int bytesRead = wait(
-			    uncancellable(holdWhile(partData, file->read(&(*partData)[0], resultPart.size, resultPart.offset))));
+			int bytesRead = co_await uncancellable(
+			    holdWhile(partData, file->read(&(*partData)[0], resultPart.size, resultPart.offset)));
 			if (bytesRead != resultPart.size) {
 				TraceEvent(SevError, "S3ClientUploadPartReadError")
 				    .detail("Expected", resultPart.size)
@@ -354,13 +348,13 @@ ACTOR static Future<PartState> uploadPart(Reference<S3BlobStoreEndpoint> endpoin
 			PacketWriter pw(packets.getWriteBuffer(resultPart.partData.size()), nullptr, Unversioned());
 			pw.serializeBytes(resultPart.partData);
 
-			std::string etag = wait(endpoint->uploadPart(bucket,
-			                                             objectName,
-			                                             uploadID,
-			                                             resultPart.partNumber,
-			                                             &packets,
-			                                             resultPart.partData.size(),
-			                                             resultPart.checksum));
+			std::string etag = co_await endpoint->uploadPart(bucket,
+			                                                 objectName,
+			                                                 uploadID,
+			                                                 resultPart.partNumber,
+			                                                 &packets,
+			                                                 resultPart.partData.size(),
+			                                                 resultPart.checksum);
 
 			resultPart.etag = etag;
 			resultPart.completed = true;
@@ -372,60 +366,60 @@ ACTOR static Future<PartState> uploadPart(Reference<S3BlobStoreEndpoint> endpoin
 			    .detail("Duration", now() - startTime)
 			    .detail("Size", resultPart.size)
 			    .detail("Attempts", attempt + 1);
-			return resultPart;
+			co_return resultPart;
 		} catch (Error& e) {
-			attempt++;
-			if (attempt >= maxRetries || !isRetryableError(e.code())) {
-				TraceEvent(SevWarnAlways, "S3ClientUploadPartFailed")
-				    .detail("Bucket", StringRef(bucket))
-				    .detail("Object", StringRef(objectName))
-				    .detail("PartNumber", part.partNumber)
-				    .detail("ErrorCode", e.code())
-				    .detail("Attempts", attempt)
-				    .detail("MaxRetries", maxRetries)
-				    .detail("FinalError", e.what());
-				throw;
-			}
+			err = e;
+		}
 
-			TraceEvent(SevDebug, "S3ClientUploadPartRetry")
+		attempt++;
+		if (attempt >= maxRetries || !isRetryableError(err.code())) {
+			TraceEvent(SevWarnAlways, "S3ClientUploadPartFailed")
 			    .detail("Bucket", StringRef(bucket))
 			    .detail("Object", StringRef(objectName))
 			    .detail("PartNumber", part.partNumber)
-			    .detail("Attempt", attempt)
-			    .detail("Error", e.what())
-			    .detail("DelayMs", delayMs);
-
-			wait(delay(delayMs / 1000.0));
-			delayMs = std::min(delayMs * 2, config.maxRetryDelayMs);
+			    .detail("ErrorCode", err.code())
+			    .detail("Attempts", attempt)
+			    .detail("MaxRetries", maxRetries)
+			    .detail("FinalError", err.what());
+			throw err;
 		}
+
+		TraceEvent(SevDebug, "S3ClientUploadPartRetry")
+		    .detail("Bucket", StringRef(bucket))
+		    .detail("Object", StringRef(objectName))
+		    .detail("PartNumber", part.partNumber)
+		    .detail("Attempt", attempt)
+		    .detail("Error", err.what())
+		    .detail("DelayMs", delayMs);
+
+		co_await delay(delayMs / 1000.0);
+		delayMs = std::min(delayMs * 2, config.maxRetryDelayMs);
 	}
 }
 
 // Copy filepath to bucket at resource in s3.
-ACTOR static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
-                                     std::string bucket,
-                                     std::string objectName,
-                                     std::string filepath,
-                                     PartConfig config = PartConfig()) {
-	state double startTime = now();
-	state Reference<IAsyncFile> file;
-	state std::string uploadID;
-	state std::vector<PartState> parts;
-	state int64_t size;
-	state XXH64_state_t* hashState = XXH64_createState();
-	state int retries = 0;
-	state int64_t offset;
-	state int partNumber;
-	state int maxConcurrentUploads;
-	state std::vector<Future<PartState>> activeFutures;
-	state std::vector<int> activePartIndices;
-	state int64_t partSize;
-	state int numParts;
-	state std::string checksum;
-	state PartState part;
-	state std::map<std::string, std::string> tags;
-
-	loop {
+static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
+                               std::string bucket,
+                               std::string objectName,
+                               std::string filepath,
+                               PartConfig config = PartConfig()) {
+	double startTime = now();
+	Reference<IAsyncFile> file;
+	std::string uploadID;
+	std::vector<PartState> parts;
+	int64_t size{ 0 };
+	XXH64_state_t* hashState = XXH64_createState();
+	int retries = 0;
+	int64_t offset{ 0 };
+	int partNumber{ 0 };
+	int maxConcurrentUploads{ 0 };
+	std::vector<Future<PartState>> activeFutures;
+	std::vector<int> activePartIndices;
+	int numParts{ 0 };
+	std::string checksum;
+
+	while (true) {
+		Error err;
 		try {
 			TraceEvent(s3VerboseEventSev(), "S3ClientCopyUpFileStart")
 			    .detail("Bucket", bucket)
@@ -439,14 +433,14 @@ ACTOR static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
 			}
 			XXH64_reset(hashState, 0);
 
-			Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-			    filepath, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO, 0644));
+			Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+			    filepath, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO, 0644);
 			file = f;
 
-			int64_t fileSize = wait(file->size());
+			int64_t fileSize = co_await file->size();
 			size = fileSize;
 			// Start multipart upload
-			std::string id = wait(endpoint->beginMultiPartUpload(bucket, objectName));
+			std::string id = co_await endpoint->beginMultiPartUpload(bucket, objectName);
 			uploadID = id;
 
 			offset = 0;
@@ -459,24 +453,20 @@ ACTOR static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
 			while (offset < size) {
 				// Fill up to maxConcurrentUploads active uploads
 				while (activeFutures.size() < maxConcurrentUploads && offset < size) {
-					partSize = std::min(config.partSizeBytes, size - offset);
-
-					part = PartState();
-					part.partNumber = partNumber;
-					part.offset = offset;
-					part.size = partSize;
+					const int64_t currentPartSize = std::min(config.partSizeBytes, size - offset);
+					PartState part(partNumber, offset, currentPartSize);
 					parts.push_back(part);
 
 					activeFutures.push_back(uploadPart(endpoint, bucket, objectName, uploadID, file, part, config));
 					activePartIndices.push_back(partNumber - 1); // Store index into parts array
 
-					offset += partSize;
+					offset += currentPartSize;
 					partNumber++;
 				}
 
 				// Wait for all active uploads to complete
 				if (!activeFutures.empty()) {
-					std::vector<PartState> completedParts = wait(getAll(activeFutures));
+					std::vector<PartState> completedParts = co_await getAll(activeFutures);
 					// Update parts with completion status
 					for (int i = 0; i < completedParts.size(); i++) {
 						parts[activePartIndices[i]] = completedParts[i];
@@ -502,7 +492,7 @@ ACTOR static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
 			}
 
 			Optional<std::string> s3Checksum =
-			    wait(endpoint->finishMultiPartUpload(bucket, objectName, uploadID, etagMap));
+			    co_await endpoint->finishMultiPartUpload(bucket, objectName, uploadID, etagMap);
 
 			// Log the S3 checksum if present
 			if (s3Checksum.present()) {
@@ -538,7 +528,7 @@ ACTOR static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
 			file = Reference<IAsyncFile>();
 
 			// Store the checksum using configurable fallback strategy
-			wait(writeChecksumWithFallback(endpoint, bucket, objectName, checksum, config));
+			co_await writeChecksumWithFallback(endpoint, bucket, objectName, checksum, config);
 
 			TraceEvent(s3VerboseEventSev(), "S3ClientCopyUpFileEnd")
 			    .detail("Bucket", bucket)
@@ -551,100 +541,99 @@ ACTOR static Future<Void> copyUpFile(Reference<S3BlobStoreEndpoint> endpoint,
 
 			break; // Success - exit retry loop
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
-			}
-			// File-level retry for specific errors, matching download behavior
-			if ((e.code() == error_code_file_not_found || e.code() == error_code_http_request_failed ||
-			     e.code() == error_code_io_error) &&
-			    retries < config.maxFileRetries) { // Use configurable retry limit
-				state Error retryError = e;
-				TraceEvent(SevWarn, "S3ClientCopyUpFileRetry")
-				    .errorUnsuppressed(retryError)
-				    .detail("Bucket", bucket)
-				    .detail("Object", objectName)
-				    .detail("FilePath", filepath)
-				    .detail("Retries", retries);
-				retries++;
+			err = e;
+		}
 
-				// Cleanup before retry
-				XXH64_freeState(hashState);
-				hashState = nullptr;
-				parts.clear();
-				activeFutures.clear();
-				activePartIndices.clear();
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		// File-level retry for specific errors, matching download behavior
+		if ((err.code() == error_code_file_not_found || err.code() == error_code_http_request_failed ||
+		     err.code() == error_code_io_error) &&
+		    retries < config.maxFileRetries) { // Use configurable retry limit
+			Error retryError = err;
+			TraceEvent(SevWarn, "S3ClientCopyUpFileRetry")
+			    .errorUnsuppressed(retryError)
+			    .detail("Bucket", bucket)
+			    .detail("Object", objectName)
+			    .detail("FilePath", filepath)
+			    .detail("Retries", retries);
+			retries++;
 
-				if (file) {
-					file = Reference<IAsyncFile>();
-				}
+			// Cleanup before retry
+			XXH64_freeState(hashState);
+			hashState = nullptr;
+			parts.clear();
+			activeFutures.clear();
+			activePartIndices.clear();
 
-				// Attempt to abort the upload but only if we have a valid uploadID
-				if (!uploadID.empty()) {
-					try {
-						wait(endpoint->abortMultiPartUpload(bucket, objectName, uploadID));
-					} catch (Error& abortError) {
-						TraceEvent(SevWarn, "S3ClientCopyUpFileAbortError")
-						    .error(abortError)
-						    .detail("Bucket", bucket)
-						    .detail("Object", objectName)
-						    .detail("UploadID", uploadID)
-						    .detail("OriginalError", retryError.what());
-					}
-					uploadID = "";
-				}
+			if (file) {
+				file = Reference<IAsyncFile>();
+			}
 
-				if (g_network->isSimulated()) {
-					wait(delay(0));
-					continue;
+			// Attempt to abort the upload but only if we have a valid uploadID
+			if (!uploadID.empty()) {
+				try {
+					co_await endpoint->abortMultiPartUpload(bucket, objectName, uploadID);
+				} catch (Error& abortError) {
+					TraceEvent(SevWarn, "S3ClientCopyUpFileAbortError")
+					    .error(abortError)
+					    .detail("Bucket", bucket)
+					    .detail("Object", objectName)
+					    .detail("UploadID", uploadID)
+					    .detail("OriginalError", retryError.what());
 				}
-				wait(delay(1.0 * retries)); // Linear backoff like download
-			} else {
-				state Error err = e;
-				XXH64_freeState(hashState);
-				hashState = nullptr;
-				TraceEvent(SevWarnAlways, "S3ClientCopyUpFileError")
-				    .detail("Filepath", filepath)
-				    .detail("Bucket", bucket)
-				    .detail("ObjectName", objectName)
-				    .detail("Error", err.what())
-				    .detail("Attempts", retries + 1);
+				uploadID = "";
+			}
 
-				// Close file before abort attempt
-				file = Reference<IAsyncFile>();
+			if (g_network->isSimulated()) {
+				co_await delay(0);
+				continue;
+			}
+			co_await delay(1.0 * retries); // Linear backoff like download
+		} else {
+			XXH64_freeState(hashState);
+			hashState = nullptr;
+			TraceEvent(SevWarnAlways, "S3ClientCopyUpFileError")
+			    .detail("Filepath", filepath)
+			    .detail("Bucket", bucket)
+			    .detail("ObjectName", objectName)
+			    .detail("Error", err.what())
+			    .detail("Attempts", retries + 1);
 
-				// Attempt to abort the upload but do not wait for it
-				if (!uploadID.empty()) {
-					try {
-						wait(endpoint->abortMultiPartUpload(bucket, objectName, uploadID));
-					} catch (Error& abortError) {
-						// Log abort failure but throw original error
-						TraceEvent(SevWarnAlways, "S3ClientCopyUpFileAbortError")
-						    .error(abortError)
-						    .detail("Bucket", bucket)
-						    .detail("Object", objectName)
-						    .detail("OriginalError", err.what());
-					}
+			// Close file before abort attempt
+			file = Reference<IAsyncFile>();
+
+			// Attempt to abort the upload but do not wait for it
+			if (!uploadID.empty()) {
+				try {
+					co_await endpoint->abortMultiPartUpload(bucket, objectName, uploadID);
+				} catch (Error& abortError) {
+					// Log abort failure but throw original error
+					TraceEvent(SevWarnAlways, "S3ClientCopyUpFileAbortError")
+					    .error(abortError)
+					    .detail("Bucket", bucket)
+					    .detail("Object", objectName)
+					    .detail("OriginalError", err.what());
 				}
 			}
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> copyUpFile(std::string filepath, std::string s3url) {
+Future<Void> copyUpFile(std::string filepath, std::string s3url) {
 	std::string resource;
 	S3BlobStoreEndpoint::ParametersT parameters;
 	Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
-	wait(copyUpFile(endpoint, parameters["bucket"], resource, filepath));
-	return Void();
+	co_await copyUpFile(endpoint, parameters["bucket"], resource, filepath);
 }
 
-ACTOR Future<Void> copyUpDirectory(std::string dirpath, std::string s3url) {
-	state std::string resource;
+Future<Void> copyUpDirectory(std::string dirpath, std::string s3url) {
+	std::string resource;
 	S3BlobStoreEndpoint::ParametersT parameters;
-	state Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
-	state std::string bucket = parameters["bucket"];
-	state std::vector<std::string> files;
+	Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
+	std::string bucket = parameters["bucket"];
+	std::vector<std::string> files;
 	platform::findFilesRecursively(dirpath, files);
 	TraceEvent(s3VerboseEventSev(), "S3ClientUploadDirStart")
 	    .detail("Filecount", files.size())
@@ -653,62 +642,60 @@ ACTOR Future<Void> copyUpDirectory(std::string dirpath, std::string s3url) {
 	for (const auto& file : files) {
 		std::string filepath = file;
 		std::string s3path = resource + "/" + file.substr(dirpath.size() + 1);
-		wait(copyUpFile(endpoint, bucket, s3path, filepath));
+		co_await copyUpFile(endpoint, bucket, s3path, filepath);
 	}
 	TraceEvent(s3VerboseEventSev(), "S3ClientUploadDirEnd").detail("Bucket", bucket).detail("Resource", resource);
-	return Void();
 }
 
-ACTOR Future<Void> copyUpBulkDumpFileSet(std::string s3url,
-                                         BulkLoadFileSet sourceFileSet,
-                                         BulkLoadFileSet destinationFileSet) {
-	state std::string resource;
+Future<Void> copyUpBulkDumpFileSet(std::string s3url,
+                                   BulkLoadFileSet sourceFileSet,
+                                   BulkLoadFileSet destinationFileSet) {
+	std::string resource;
 	S3BlobStoreEndpoint::ParametersT parameters;
-	state Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
-	state std::string bucket = parameters["bucket"];
+	Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
+	std::string bucket = parameters["bucket"];
 
 	TraceEvent(s3VerboseEventSev(), "S3ClientCopyUpBulkDumpFileSetStart")
 	    .detail("Bucket", bucket)
 	    .detail("SourceFileSet", sourceFileSet.toString())
 	    .detail("DestinationFileSet", destinationFileSet.toString());
-	state int pNumDeleted = 0;
-	state int64_t pBytesDeleted = 0;
-	state std::string batch_dir = joinPath(getPath(s3url), destinationFileSet.getRelativePath());
+	int pNumDeleted = 0;
+	int64_t pBytesDeleted = 0;
+	std::string batch_dir = joinPath(getPath(s3url), destinationFileSet.getRelativePath());
 
 	// Delete the batch dir if it exists already (need to check bucket exists else 404 and s3blobstore errors out).
-	bool exists = wait(endpoint->bucketExists(bucket));
+	bool exists = co_await endpoint->bucketExists(bucket);
 	if (exists) {
-		wait(endpoint->deleteRecursively(bucket, batch_dir, &pNumDeleted, &pBytesDeleted));
+		co_await endpoint->deleteRecursively(bucket, batch_dir, &pNumDeleted, &pBytesDeleted);
 	}
 	// Destination for manifest file.
 	auto destinationManifestPath = joinPath(batch_dir, destinationFileSet.getManifestFileName());
-	wait(copyUpFile(endpoint, bucket, destinationManifestPath, sourceFileSet.getManifestFileFullPath()));
+	co_await copyUpFile(endpoint, bucket, destinationManifestPath, sourceFileSet.getManifestFileFullPath());
 	if (sourceFileSet.hasDataFile()) {
 		auto destinationDataPath = joinPath(batch_dir, destinationFileSet.getDataFileName());
-		wait(copyUpFile(endpoint, bucket, destinationDataPath, sourceFileSet.getDataFileFullPath()));
+		co_await copyUpFile(endpoint, bucket, destinationDataPath, sourceFileSet.getDataFileFullPath());
 	}
 	if (sourceFileSet.hasByteSampleFile()) {
 		ASSERT(sourceFileSet.hasDataFile());
 		auto destinationByteSamplePath = joinPath(batch_dir, destinationFileSet.getByteSampleFileName());
-		wait(copyUpFile(endpoint, bucket, destinationByteSamplePath, sourceFileSet.getBytesSampleFileFullPath()));
+		co_await copyUpFile(endpoint, bucket, destinationByteSamplePath, sourceFileSet.getBytesSampleFileFullPath());
 	}
 	TraceEvent(s3VerboseEventSev(), "S3ClientCopyUpBulkDumpFileSetEnd")
 	    .detail("BatchDir", batch_dir)
 	    .detail("NumDeleted", pNumDeleted)
 	    .detail("BytesDeleted", pBytesDeleted);
-	return Void();
 }
 
-ACTOR static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpoint,
-                                            std::string bucket,
-                                            std::string objectName,
-                                            Reference<IAsyncFile> file,
-                                            PartState part,
-                                            PartConfig config) {
-	state PartState resultPart = part;
-	state int attempt = 0;
-	state int maxRetries = config.maxPartRetries;
-	state int delayMs = config.baseRetryDelayMs;
+static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpoint,
+                                      std::string bucket,
+                                      std::string objectName,
+                                      Reference<IAsyncFile> file,
+                                      PartState part,
+                                      PartConfig config) {
+	PartState resultPart = part;
+	int attempt = 0;
+	int maxRetries = config.maxPartRetries;
+	int delayMs = config.baseRetryDelayMs;
 
 	TraceEvent(SevDebug, "S3ClientDownloadPartStart")
 	    .detail("Bucket", bucket)
@@ -717,11 +704,11 @@ ACTOR static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpo
 	    .detail("Offset", resultPart.offset)
 	    .detail("Size", resultPart.size);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			state std::vector<uint8_t> buffer;
-			state int64_t totalBytesRead = 0;
-			buffer.resize(resultPart.size);
+			std::shared_ptr<std::vector<uint8_t>> buffer = std::make_shared<std::vector<uint8_t>>(resultPart.size);
+			int64_t totalBytesRead = 0;
 
 			// Add range validation
 			if (resultPart.offset < 0 || resultPart.size <= 0) {
@@ -732,11 +719,14 @@ ACTOR static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpo
 			}
 
 			while (totalBytesRead < resultPart.size) {
-				int bytesRead = wait(endpoint->readObject(bucket,
-				                                          objectName,
-				                                          buffer.data() + totalBytesRead,
-				                                          resultPart.size - totalBytesRead,
-				                                          resultPart.offset + totalBytesRead));
+				// Keep the shared buffer alive until the async read has fully settled, even if this actor is cancelled.
+				int bytesRead =
+				    co_await uncancellable(holdWhile(buffer,
+				                                     endpoint->readObject(bucket,
+				                                                          objectName,
+				                                                          buffer->data() + totalBytesRead,
+				                                                          resultPart.size - totalBytesRead,
+				                                                          resultPart.offset + totalBytesRead)));
 				if (bytesRead == 0) {
 					// Avoid infinite loop if server closes connection prematurely
 					TraceEvent(SevError, "S3ClientDownloadPartUnexpectedEOF")
@@ -758,7 +748,7 @@ ACTOR static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpo
 
 			// Verify checksum if provided (currently only MD5 is used for download verification)
 			if (!resultPart.checksum.empty()) {
-				std::string calculatedMD5 = HTTP::computeMD5Sum(std::string((char*)buffer.data(), totalBytesRead));
+				std::string calculatedMD5 = HTTP::computeMD5Sum(std::string((char*)buffer->data(), totalBytesRead));
 				if (resultPart.checksum != calculatedMD5) {
 					TraceEvent(SevWarnAlways, "S3ClientDownloadPartChecksumMismatch")
 					    .detail("Expected", resultPart.checksum)
@@ -767,7 +757,8 @@ ACTOR static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpo
 				}
 			}
 
-			wait(file->write(buffer.data(), totalBytesRead, resultPart.offset));
+			// Keep the shared buffer alive until the async write has fully settled, even if this actor is cancelled.
+			co_await uncancellable(holdWhile(buffer, file->write(buffer->data(), totalBytesRead, resultPart.offset)));
 
 			resultPart.completed = true;
 			TraceEvent(SevDebug, "S3ClientDownloadPartEnd")
@@ -777,61 +768,64 @@ ACTOR static Future<PartState> downloadPart(Reference<S3BlobStoreEndpoint> endpo
 			    .detail("Offset", resultPart.offset)
 			    .detail("Size", resultPart.size)
 			    .detail("Attempts", attempt + 1);
-			return resultPart;
+			co_return resultPart;
 		} catch (Error& e) {
-			attempt++;
-			if (attempt >= maxRetries || !isRetryableError(e.code())) {
-				TraceEvent(SevWarnAlways, "S3ClientDownloadPartFailed")
-				    .detail("Bucket", bucket)
-				    .detail("Object", objectName)
-				    .detail("PartNumber", part.partNumber)
-				    .detail("ErrorCode", e.code())
-				    .detail("Attempts", attempt)
-				    .detail("FinalError", e.what());
-				throw;
-			}
+			err = e;
+		}
 
-			TraceEvent(SevInfo, "S3ClientDownloadPartRetry")
+		attempt++;
+		if (attempt >= maxRetries || !isRetryableError(err.code())) {
+			TraceEvent(SevWarnAlways, "S3ClientDownloadPartFailed")
 			    .detail("Bucket", bucket)
 			    .detail("Object", objectName)
 			    .detail("PartNumber", part.partNumber)
-			    .detail("Attempt", attempt)
-			    .detail("Error", e.what())
-			    .detail("DelayMs", delayMs);
-
-			wait(delay(delayMs / 1000.0));
-			delayMs = std::min(delayMs * 2, config.maxRetryDelayMs); // Use configurable cap
+			    .detail("ErrorCode", err.code())
+			    .detail("Attempts", attempt)
+			    .detail("FinalError", err.what());
+			throw err;
 		}
+
+		TraceEvent(SevInfo, "S3ClientDownloadPartRetry")
+		    .detail("Bucket", bucket)
+		    .detail("Object", objectName)
+		    .detail("PartNumber", part.partNumber)
+		    .detail("Attempt", attempt)
+		    .detail("Error", err.what())
+		    .detail("DelayMs", delayMs);
+
+		co_await delay(delayMs / 1000.0);
+		delayMs = std::min(delayMs * 2, config.maxRetryDelayMs); // Use configurable cap
 	}
 }
 
-ACTOR static Future<Optional<std::string>> getExpectedChecksum(Reference<S3BlobStoreEndpoint> endpoint,
-                                                               std::string bucket,
-                                                               std::string objectName) {
+static AsyncResult<Optional<std::string>> getExpectedChecksum(Reference<S3BlobStoreEndpoint> endpoint,
+                                                              std::string bucket,
+                                                              std::string objectName) {
 	PartConfig config; // Use default configuration
-	Optional<std::string> result = wait(readChecksumWithFallback(endpoint, bucket, objectName, config));
-	return result;
+	Optional<std::string> result = co_await readChecksumWithFallback(endpoint, bucket, objectName, config);
+	co_return result;
 }
 
 // Copy down file from s3 to filepath.
-ACTOR static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
-                                       std::string bucket,
-                                       std::string objectName,
-                                       std::string filepath,
-                                       PartConfig config = PartConfig()) {
-	state Reference<IAsyncFile> file;
-	state std::vector<PartState> parts;
-	state int64_t fileSize = 0;
-	state int64_t offset = 0;
-	state int partNumber = 1;
-	state int64_t partSize;
-	state std::string expectedChecksum;
-	state int retries = 0;
-	state int maxConcurrentDownloads;
-	state std::vector<Future<PartState>> activeDownloadFutures;
-	state std::vector<int> activePartIndices;
-
-	loop {
+static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
+                                 std::string bucket,
+                                 std::string objectName,
+                                 std::string filepath,
+                                 PartConfig config = PartConfig()) {
+	Reference<IAsyncFile> file;
+	std::vector<PartState> parts;
+	int64_t fileSize = 0;
+	int64_t offset = 0;
+	int partNumber = 1;
+	int64_t partSize{ 0 };
+	std::string expectedChecksum;
+	int retries = 0;
+	int maxConcurrentDownloads{ 0 };
+	std::vector<Future<PartState>> activeDownloadFutures;
+	std::vector<int> activePartIndices;
+
+	while (true) {
+		Error err;
 		try {
 			TraceEvent(s3VerboseEventSev(), "S3ClientCopyDownFileStart")
 			    .detail("Bucket", bucket)
@@ -842,7 +836,7 @@ ACTOR static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
 			TraceEvent(SevDebug, "S3ClientCopyDownFileBeforeObjectSize")
 			    .detail("Bucket", bucket)
 			    .detail("Object", objectName);
-			int64_t s = wait(endpoint->objectSize(bucket, objectName));
+			int64_t s = co_await endpoint->objectSize(bucket, objectName);
 			TraceEvent(SevDebug, "S3ClientCopyDownFileAfterObjectSize")
 			    .detail("Bucket", bucket)
 			    .detail("Object", objectName)
@@ -862,15 +856,15 @@ ACTOR static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
 
 			int numParts = (fileSize + config.partSizeBytes - 1) / config.partSizeBytes;
 			parts.reserve(numParts);
-			Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
+			Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
 			    filepath,
 			    IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED |
 			        IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_NO_AIO,
-			    0644));
+			    0644);
 			file = f;
 
-			wait(file->truncate(fileSize));
-			wait(file->truncate(fileSize));
+			co_await file->truncate(fileSize);
+			co_await file->truncate(fileSize);
 
 			offset = 0;
 			partNumber = 1;
@@ -893,7 +887,7 @@ ACTOR static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
 
 				// Wait for all active downloads to complete
 				if (!activeDownloadFutures.empty()) {
-					std::vector<PartState> completedParts = wait(getAll(activeDownloadFutures));
+					std::vector<PartState> completedParts = co_await getAll(activeDownloadFutures);
 					// Update parts with completion status
 					for (int i = 0; i < completedParts.size(); i++) {
 						parts[activePartIndices[i]] = completedParts[i];
@@ -912,15 +906,15 @@ ACTOR static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
 				}
 			}
 
-			wait(file->truncate(fileSize));
-			wait(file->sync());
+			co_await file->truncate(fileSize);
+			co_await file->sync();
 
 			// Get and verify checksum using the helper
-			Optional<std::string> cs = wait(getExpectedChecksum(endpoint, bucket, objectName));
+			Optional<std::string> cs = co_await getExpectedChecksum(endpoint, bucket, objectName);
 
 			if (cs.present()) {
 				expectedChecksum = cs.get();
-				state std::string actualChecksum = wait(calculateFileChecksum(file, fileSize));
+				std::string actualChecksum = co_await calculateFileChecksum(file, fileSize);
 				if (actualChecksum != expectedChecksum) {
 					TraceEvent(SevWarnAlways, "S3ClientCopyDownFileChecksumMismatch")
 					    .detail("Expected", expectedChecksum)
@@ -940,65 +934,65 @@ ACTOR static Future<Void> copyDownFile(Reference<S3BlobStoreEndpoint> endpoint,
 
 			break; // Success
 		} catch (Error& e) {
-			if ((e.code() == error_code_file_not_found || e.code() == error_code_http_request_failed ||
-			     e.code() == error_code_io_error) &&
-			    retries < config.maxFileRetries) {
-				TraceEvent(SevWarn, "S3ClientCopyDownFileRetry")
-				    .errorUnsuppressed(e)
-				    .detail("Bucket", bucket)
-				    .detail("Object", objectName)
-				    .detail("FilePath", filepath)
-				    .detail("Retries", retries);
-				retries++;
-
-				// Cleanup state for retry
-				parts.clear();
-				activeDownloadFutures.clear();
-				activePartIndices.clear();
-
-				if (file) {
-					try {
-						file = Reference<IAsyncFile>();
-						IAsyncFileSystem::filesystem()->deleteFile(filepath, true);
-					} catch (Error& cleanupError) {
-						TraceEvent(SevWarnAlways, "S3ClientCopyDownFileCleanupError")
-						    .detail("FilePath", filepath)
-						    .errorUnsuppressed(cleanupError);
-					}
-				}
-				if (g_network->isSimulated()) {
-					wait(delay(0));
-					continue;
+			err = e;
+		}
+
+		if ((err.code() == error_code_file_not_found || err.code() == error_code_http_request_failed ||
+		     err.code() == error_code_io_error) &&
+		    retries < config.maxFileRetries) {
+			TraceEvent(SevWarn, "S3ClientCopyDownFileRetry")
+			    .errorUnsuppressed(err)
+			    .detail("Bucket", bucket)
+			    .detail("Object", objectName)
+			    .detail("FilePath", filepath)
+			    .detail("Retries", retries);
+			retries++;
+
+			// Cleanup state for retry
+			parts.clear();
+			activeDownloadFutures.clear();
+			activePartIndices.clear();
+
+			if (file) {
+				try {
+					file = Reference<IAsyncFile>();
+					IAsyncFileSystem::filesystem()->deleteFile(filepath, true);
+				} catch (Error& cleanupError) {
+					TraceEvent(SevWarnAlways, "S3ClientCopyDownFileCleanupError")
+					    .detail("FilePath", filepath)
+					    .errorUnsuppressed(cleanupError);
 				}
-				wait(delay(1.0 * retries));
-			} else {
-				state Error err = e;
-				TraceEvent(SevWarnAlways, "S3ClientCopyDownFileError")
-				    .detail("Bucket", bucket)
-				    .detail("ObjectName", objectName)
-				    .errorUnsuppressed(err)
-				    .detail("FilePath", filepath)
-				    .detail("FileSize", fileSize);
-
-				if (file) {
-					try {
-						wait(file->sync());
-						file = Reference<IAsyncFile>();
-						IAsyncFileSystem::filesystem()->deleteFile(filepath, false);
-					} catch (Error& e2) {
-						TraceEvent(SevWarnAlways, "S3ClientCopyDownFileCleanupError")
-						    .detail("FilePath", filepath)
-						    .errorUnsuppressed(e2);
-					}
+			}
+			if (g_network->isSimulated()) {
+				co_await delay(0);
+				continue;
+			}
+			co_await delay(1.0 * retries);
+		} else {
+			TraceEvent(SevWarnAlways, "S3ClientCopyDownFileError")
+			    .detail("Bucket", bucket)
+			    .detail("ObjectName", objectName)
+			    .errorUnsuppressed(err)
+			    .detail("FilePath", filepath)
+			    .detail("FileSize", fileSize);
+
+			if (file) {
+				try {
+					co_await file->sync();
+					file = Reference<IAsyncFile>();
+					IAsyncFileSystem::filesystem()->deleteFile(filepath, false);
+				} catch (Error& e2) {
+					TraceEvent(SevWarnAlways, "S3ClientCopyDownFileCleanupError")
+					    .detail("FilePath", filepath)
+					    .errorUnsuppressed(e2);
 				}
-				throw err;
 			}
+			throw err;
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> copyDownFile(std::string s3url, std::string filepath) {
+Future<Void> copyDownFile(std::string s3url, std::string filepath) {
 	TraceEvent(SevDebug, "S3ClientCopyDownFileWrapperStart").detail("S3URL", s3url).detail("FilePath", filepath);
 	std::string resource;
 	S3BlobStoreEndpoint::ParametersT parameters;
@@ -1008,18 +1002,17 @@ ACTOR Future<Void> copyDownFile(std::string s3url, std::string filepath) {
 	    .detail("S3URL", s3url)
 	    .detail("Resource", resource)
 	    .detail("Bucket", parameters["bucket"]);
-	wait(copyDownFile(endpoint, parameters["bucket"], resource, filepath));
+	co_await copyDownFile(endpoint, parameters["bucket"], resource, filepath);
 	TraceEvent(SevDebug, "S3ClientCopyDownFileWrapperEnd").detail("S3URL", s3url).detail("FilePath", filepath);
-	return Void();
 }
 
-ACTOR Future<Void> copyDownDirectory(std::string s3url, std::string dirpath) {
-	state std::string resource;
+Future<Void> copyDownDirectory(std::string s3url, std::string dirpath) {
+	std::string resource;
 	S3BlobStoreEndpoint::ParametersT parameters;
-	state Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
-	state std::string bucket = parameters["bucket"];
-	S3BlobStoreEndpoint::ListResult items = wait(endpoint->listObjects(bucket, resource));
-	state std::vector<S3BlobStoreEndpoint::ObjectInfo> objects = items.objects;
+	Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
+	std::string bucket = parameters["bucket"];
+	S3BlobStoreEndpoint::ListResult items = co_await endpoint->listObjects(bucket, resource);
+	std::vector<S3BlobStoreEndpoint::ObjectInfo> objects = items.objects;
 	TraceEvent(s3VerboseEventSev(), "S3ClientDownDirectoryStart")
 	    .detail("Filecount", objects.size())
 	    .detail("Bucket", bucket)
@@ -1027,27 +1020,25 @@ ACTOR Future<Void> copyDownDirectory(std::string s3url, std::string dirpath) {
 	for (const auto& object : objects) {
 		std::string filepath = dirpath + "/" + object.name.substr(resource.size());
 		std::string s3path = object.name;
-		wait(copyDownFile(endpoint, bucket, s3path, filepath));
+		co_await copyDownFile(endpoint, bucket, s3path, filepath);
 	}
 	TraceEvent(s3VerboseEventSev(), "S3ClientDownDirectoryEnd").detail("Bucket", bucket).detail("Resource", resource);
-	return Void();
 }
 
-ACTOR Future<Void> deleteResource(std::string s3url) {
-	state std::string resource;
+Future<Void> deleteResource(std::string s3url) {
+	std::string resource;
 	S3BlobStoreEndpoint::ParametersT parameters;
 	Reference<S3BlobStoreEndpoint> endpoint = getEndpoint(s3url, resource, parameters);
-	state std::string bucket = parameters["bucket"];
-	wait(endpoint->deleteRecursively(bucket, resource));
-	return Void();
+	std::string bucket = parameters["bucket"];
+	co_await endpoint->deleteRecursively(bucket, resource);
 }
 
-ACTOR Future<Void> listFiles(std::string s3url, int maxDepth) {
+Future<Void> listFiles(std::string s3url, int maxDepth) {
 	try {
-		state std::string resource;
-		state std::string error;
-		state S3BlobStoreEndpoint::ParametersT parameters;
-		state Reference<S3BlobStoreEndpoint> bstore = getEndpoint(s3url, resource, parameters);
+		std::string resource;
+		std::string error;
+		S3BlobStoreEndpoint::ParametersT parameters;
+		Reference<S3BlobStoreEndpoint> bstore = getEndpoint(s3url, resource, parameters);
 
 		if (!bstore) {
 			TraceEvent(SevError, "S3ClientListingFailed").detail("Error", error);
@@ -1055,23 +1046,23 @@ ACTOR Future<Void> listFiles(std::string s3url, int maxDepth) {
 		}
 
 		// Get bucket directly from parameters
-		state std::string bucket = parameters["bucket"];
+		std::string bucket = parameters["bucket"];
 
 		// Check if bucket exists first
-		bool exists = wait(bstore->bucketExists(bucket));
+		bool exists = co_await bstore->bucketExists(bucket);
 		if (!exists) {
 			std::cerr << "ERROR: Bucket '" << bucket << "' does not exist" << std::endl;
 			throw http_request_failed();
 		}
 
 		// Let S3BlobStoreEndpoint handle the resource path construction
-		state Optional<char> delimiter;
+		Optional<char> delimiter;
 		if (maxDepth <= 1) {
 			delimiter = Optional<char>('/');
 		}
 
 		// Use listObjects with the resource path directly, letting S3BlobStoreEndpoint handle URL construction
-		state S3BlobStoreEndpoint::ListResult result = wait(bstore->listObjects(bucket, resource, delimiter, maxDepth));
+		S3BlobStoreEndpoint::ListResult result = co_await bstore->listObjects(bucket, resource, delimiter, maxDepth);
 
 		// Format and display the objects
 		std::cout << "Contents of " << s3url << ":" << std::endl;
@@ -1142,20 +1133,19 @@ ACTOR Future<Void> listFiles(std::string s3url, int maxDepth) {
 		}
 		throw;
 	}
-	return Void();
 }
 
-ACTOR Future<std::vector<std::string>> listFiles_impl(Reference<S3BlobStoreEndpoint> bstore,
-                                                      std::string bucket,
-                                                      std::string path) {
-	wait(bstore->requestRateRead->getAllowance(1));
+AsyncResult<std::vector<std::string>> listFiles_impl(Reference<S3BlobStoreEndpoint> bstore,
+                                                     std::string bucket,
+                                                     std::string path) {
+	co_await bstore->requestRateRead->getAllowance(1);
 
-	state std::string resource = bstore->constructResourcePath(bucket, path);
-	state HTTP::Headers headers;
-	state std::string fullResource = resource + "?list-type=2&prefix=" + path;
+	std::string resource = bstore->constructResourcePath(bucket, path);
+	HTTP::Headers headers;
+	std::string fullResource = resource + "?list-type=2&prefix=" + path;
 
 	Reference<HTTP::IncomingResponse> r =
-	    wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, { 200, 404 }));
+	    co_await bstore->doRequest("GET", fullResource, headers, nullptr, 0, { 200, 404 });
 
 	if (r->code == 404) {
 		TraceEvent(SevWarn, "S3ClientListFilesNotFound").detail("Bucket", bucket).detail("Path", path);
@@ -1193,7 +1183,7 @@ ACTOR Future<std::vector<std::string>> listFiles_impl(Reference<S3BlobStoreEndpo
 			n = n->next_sibling();
 		}
 
-		return files;
+		co_return files;
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "S3ClientListFilesError").error(e).detail("Bucket", bucket).detail("Path", path);
 		throw;
diff --git a/fdbclient/S3Client_cli.actor.cpp b/fdbclient/S3Client_cli.cpp
similarity index 91%
rename from fdbclient/S3Client_cli.actor.cpp
rename to fdbclient/S3Client_cli.cpp
index 1fab153ca8c..dedb0d0540f 100644
--- a/fdbclient/S3Client_cli.actor.cpp
+++ b/fdbclient/S3Client_cli.cpp
@@ -18,43 +18,32 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ClientKnobCollection.h"
-#include <algorithm>
-#include <cctype>
 #include <cstdlib>
-#include <fstream>
 #include <iostream>
 #include <limits>
 #include <memory>
 #include <string>
 #include <vector>
-#include <fcntl.h>
 #include <filesystem>
 
 #ifdef _WIN32
 #include <io.h>
 #endif
 
-#include <boost/algorithm/hex.hpp>
 #include "fdbclient/BuildFlags.h"
-#include "fdbclient/BackupContainerFileSystem.h"
 #import "fdbclient/BackupTLSConfig.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/Knobs.h"
+#include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/versions.h"
-#include "fdbclient/S3Client.actor.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/S3Client.h"
 #include "flow/Platform.h"
 #include "flow/ArgParseUtil.h"
 #include "flow/FastRef.h"
 #include "flow/Trace.h"
 #include "flow/flow.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "SimpleOpt/SimpleOpt.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 // CLI for S3Client.
 
 extern const char* getSourceVersion();
@@ -180,7 +169,7 @@ struct Params : public ReferenceCounted<Params> {
 	void updateKnobs() {
 		// Set default to 'true' for blobstore_enable_object_integrity_check if not explicitly set
 		bool blobstore_enable_object_integrity_check_set = false;
-		for (const std::pair p : knobs) {
+		for (const auto& p : knobs) {
 			if (p.first == blobstore_enable_object_integrity_check) {
 				blobstore_enable_object_integrity_check_set = true;
 				break;
@@ -189,10 +178,10 @@ struct Params : public ReferenceCounted<Params> {
 		if (!blobstore_enable_object_integrity_check_set) {
 			knobs.push_back(std::pair(blobstore_enable_object_integrity_check, "true"));
 		}
-		IKnobCollection::setupKnobs(knobs);
+		setupClientKnobs(knobs);
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
+		initializeClientKnobs(Randomize::False, IsSimulated::False);
 	}
 };
 
@@ -326,51 +315,48 @@ static int parseCommandLine(Reference<Params> param, CSimpleOpt* args) {
 }
 
 // Method called by main. Figures which of copy_up or copy_down to call.
-ACTOR Future<Void> run(Reference<Params> params) {
+Future<Void> run(Reference<Params> params) {
 	if (params->command == "cp") {
 		if (params->whichIsBlobstoreURL == 1) {
 			if (std::filesystem::is_directory(params->src)) {
-				wait(copyUpDirectory(params->src, params->tgt));
+				co_await copyUpDirectory(params->src, params->tgt);
 			} else {
-				wait(copyUpFile(params->src, params->tgt));
+				co_await copyUpFile(params->src, params->tgt);
 			}
 		} else {
 			if (std::filesystem::is_directory(params->tgt)) {
-				wait(copyDownDirectory(params->src, params->tgt));
+				co_await copyDownDirectory(params->src, params->tgt);
 			} else {
-				wait(copyDownFile(params->src, params->tgt));
+				co_await copyDownFile(params->src, params->tgt);
 			}
 		}
 	} else if (params->command == "rm") {
-		wait(deleteResource(params->src));
+		co_await deleteResource(params->src);
 	} else if (params->command == "ls") {
 		if (params->ls_recursive) {
-			wait(listFiles(params->src, std::numeric_limits<int>::max()));
+			co_await listFiles(params->src, std::numeric_limits<int>::max());
 		} else {
-			wait(listFiles(params->src, 1));
+			co_await listFiles(params->src, 1);
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> deleteResource(std::string src) {
+Future<Void> deleteResource(std::string src) {
 	try {
-		wait(::deleteResource(src));
+		co_await ::deleteResource(src);
 	} catch (Error& e) {
 		// Rethrow the error to ensure it's handled by the main error handler
 		throw;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> listFiles(std::string src, int maxDepth) {
+Future<Void> listFiles(std::string src, int maxDepth) {
 	try {
-		wait(::listFiles(src, maxDepth));
+		co_await ::listFiles(src, maxDepth);
 	} catch (Error& e) {
 		// Rethrow the error to ensure it's handled by the main error handler
 		throw;
 	}
-	return Void();
 }
 } // namespace s3client_cli
 
@@ -442,7 +428,7 @@ int main(int argc, char** argv) {
 				fprintf(stderr, "ERROR: proxy format should be either IP:port or host:port\n");
 				flushAndExit(FDB_EXIT_ERROR);
 			}
-			Optional<std::string>* pProxy = (Optional<std::string>*)g_network->global(INetwork::enProxy);
+			auto* pProxy = static_cast<Optional<std::string>*>(g_network->global(INetwork::enProxy));
 			*pProxy = param->proxy.get();
 		}
 
@@ -451,7 +437,7 @@ int main(int argc, char** argv) {
 		    .detail("SourceVersion", getSourceVersion())
 		    .detail("Version", FDB_VT_VERSION)
 		    .detail("PackageName", FDB_VT_PACKAGE_NAME)
-		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
+		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
 		    .setMaxFieldLength(10000)
 		    .detail("CommandLine", commandLine)
 		    .setMaxFieldLength(0)
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index b0496352a95..d88ab83a70e 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -429,6 +429,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
                         "trace_log_file_write_error",
                         "trace_log_could_not_create_file",
                         "trace_log_writer_thread_unresponsive",
+                        "exclude_from_tlog_recruitment_low_disk",
                         "process_error",
                         "io_error",
                         "io_timeout",
@@ -534,9 +535,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed",
-                  "blob_worker_lag",
-                  "blob_worker_missing"
+                  "storage_server_list_fetch_failed"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -557,9 +556,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed",
-                  "blob_worker_lag",
-                  "blob_worker_missing"
+                  "storage_server_list_fetch_failed"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -682,7 +679,6 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
                   "unreachable_cluster_controller_worker",
                   "unreachable_dataDistributor_worker",
                   "unreachable_ratekeeper_worker",
-                  "unreachable_blobManager_worker",
                   "unreachable_consistencyScan_worker",
                   "unreadable_configuration",
                   "full_replication_timeout",
@@ -704,8 +700,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
                   "primary_dc_missing",
                   "fetch_primary_dc_timeout",
                   "fetch_storage_wiggler_stats_timeout",
-                  "fetch_consistency_scan_status_timeout",
-                  "metacluster_metrics_missing"
+                  "fetch_consistency_scan_status_timeout"
                ]
             },
             "issues":[
@@ -715,7 +710,8 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
                         "incorrect_cluster_file_contents",
                         "trace_log_file_write_error",
                         "trace_log_could_not_create_file",
-                        "trace_log_writer_thread_unresponsive"
+                        "trace_log_writer_thread_unresponsive",
+                        "exclude_from_tlog_recruitment_low_disk"
                      ]
                   },
                   "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."
@@ -969,6 +965,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
          "grv_proxies":1,
          "proxies":6,
          "backup_worker_enabled":1,
+         "range_backup_worker_enabled":1,
          "perpetual_storage_wiggle":0,
          "perpetual_storage_wiggle_locality":"0",
          "perpetual_storage_wiggle_engine":{
@@ -1379,9 +1376,7 @@ const KeyRef JSONSchemas::faultToleranceStatusSchema = R"statusSchema(
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed",
-                  "blob_worker_lag",
-                  "blob_worker_missing"
+                  "storage_server_list_fetch_failed"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -1402,9 +1397,7 @@ const KeyRef JSONSchemas::faultToleranceStatusSchema = R"statusSchema(
                   "log_server_min_free_space",
                   "log_server_min_free_space_ratio",
                   "storage_server_durability_lag",
-                  "storage_server_list_fetch_failed",
-                  "blob_worker_lag",
-                  "blob_worker_missing"
+                  "storage_server_list_fetch_failed"
                ]
             },
             "description":"The database is not being saturated by the workload."
@@ -1461,7 +1454,6 @@ const KeyRef JSONSchemas::faultToleranceStatusSchema = R"statusSchema(
                   "unreachable_cluster_controller_worker",
                   "unreachable_dataDistributor_worker",
                   "unreachable_ratekeeper_worker",
-                  "unreachable_blobManager_worker",
                   "unreachable_consistencyScan_worker",
                   "unreadable_configuration",
                   "full_replication_timeout",
@@ -1483,8 +1475,7 @@ const KeyRef JSONSchemas::faultToleranceStatusSchema = R"statusSchema(
                   "primary_dc_missing",
                   "fetch_primary_dc_timeout",
                   "fetch_storage_wiggler_stats_timeout",
-                  "fetch_consistency_scan_status_timeout",
-                  "metacluster_metrics_missing"
+                  "fetch_consistency_scan_status_timeout"
                ]
             },
             "issues":[
diff --git a/fdbclient/ServerKnobCollection.cpp b/fdbclient/ServerKnobCollection.cpp
deleted file mode 100644
index ad0bc522b2f..00000000000
--- a/fdbclient/ServerKnobCollection.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * ServerKnobCollection.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/ServerKnobCollection.h"
-
-ServerKnobCollection::ServerKnobCollection(Randomize randomize, IsSimulated isSimulated)
-  : clientKnobCollection(randomize, isSimulated),
-    serverKnobs(randomize, &clientKnobCollection.getMutableClientKnobs(), isSimulated) {}
-
-void ServerKnobCollection::initialize(Randomize randomize, IsSimulated isSimulated) {
-	clientKnobCollection.initialize(randomize, isSimulated);
-	serverKnobs.initialize(randomize, &clientKnobCollection.getMutableClientKnobs(), isSimulated);
-}
-
-void ServerKnobCollection::reset(Randomize randomize, IsSimulated isSimulated) {
-	clientKnobCollection.reset(randomize, isSimulated);
-	serverKnobs.reset(randomize, &clientKnobCollection.getMutableClientKnobs(), isSimulated);
-}
-
-Optional<KnobValue> ServerKnobCollection::tryParseKnobValue(std::string const& knobName,
-                                                            std::string const& knobValue) const {
-	auto result = clientKnobCollection.tryParseKnobValue(knobName, knobValue);
-	if (result.present()) {
-		return result;
-	}
-	auto parsedKnobValue = serverKnobs.parseKnobValue(knobName, knobValue);
-	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
-		return KnobValueRef::create(parsedKnobValue);
-	}
-	return {};
-}
-
-bool ServerKnobCollection::trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) {
-	// Do not short circuit by directly returning:
-	//     clientKnobCollection.trySetKnob(knobName, knobValue) || knobValue.visitSetKnob(knobName, serverKnobs)
-	// This is because some knobs have the same name in client and server e.g. MAX_WRITE_TRANSACTION_LIFE_VERSIONS
-	// When setting such knobs, we want both client and server knob to have their value updated
-	// Short circuiting would mean that server knob named FOO won't be updated if client knob FOO was updated
-	// Instead, we attempt setting client and server knobs in separate statements, and return true
-	// if at least one of the set attempts was succesful.
-	const bool setClientKnob = clientKnobCollection.trySetKnob(knobName, knobValue);
-	const bool setServerKnob = knobValue.visitSetKnob(knobName, serverKnobs);
-	return setClientKnob || setServerKnob;
-}
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.cpp
similarity index 85%
rename from fdbclient/SpecialKeySpace.actor.cpp
rename to fdbclient/SpecialKeySpace.cpp
index e7193504955..7858be1fc78 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.cpp
@@ -1,5 +1,5 @@
 /*
- * SpecialKeySpace.actor.cpp
+ * SpecialKeySpace.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -33,12 +33,13 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/ProcessInterface.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/SpecialKeySpace.h"
 #include "flow/Arena.h"
 #include "flow/UnitTest.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/StatusClient.h"
+#include "flow/CoroUtils.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
@@ -47,7 +48,7 @@ const std::string kTracingTokenKey = "token";
 
 static bool isAlphaNumeric(const std::string& key) {
 	// [A-Za-z0-9_]+
-	if (!key.size())
+	if (key.empty())
 		return false;
 	for (const char& c : key) {
 		if (!((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'))
@@ -122,19 +123,19 @@ RangeResult rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, c
 // The cache object is used to cache the first read result from the rpc call during the key resolution,
 // then when we need to do key resolution or result filtering,
 // we, instead of rpc call, read from this cache object have consistent results
-ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* skrImpl,
-                                                 ReadYourWritesTransaction* ryw,
-                                                 KeySelector* ks,
-                                                 KeyRangeMap<Optional<RangeResult>>* cache) {
+Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* skrImpl,
+                                           ReadYourWritesTransaction* ryw,
+                                           KeySelector* ks,
+                                           KeyRangeMap<Optional<RangeResult>>* cache) {
 	// should be removed before calling
 	ASSERT(!ks->orEqual);
 
 	// never being called if KeySelector is already normalized
 	ASSERT(ks->offset != 1);
 
-	state Key startKey(skrImpl->getKeyRange().begin);
-	state Key endKey(skrImpl->getKeyRange().end);
-	state RangeResult result;
+	Key startKey(skrImpl->getKeyRange().begin);
+	Key endKey(skrImpl->getKeyRange().end);
+	RangeResult result;
 
 	if (ks->offset < 1) {
 		// less than the given key
@@ -158,17 +159,17 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
 	GetRangeLimits limitsHint(ks->offset >= 1 ? ks->offset : 1 - ks->offset);
 
 	if (skrImpl->isAsync()) {
-		const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(skrImpl);
-		RangeResult result_ = wait(ptr->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint, cache));
+		auto* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(skrImpl);
+		RangeResult result_ = co_await ptr->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint, cache);
 		result = result_;
 	} else {
-		RangeResult result_ = wait(skrImpl->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint));
+		RangeResult result_ = co_await skrImpl->getRange(ryw, KeyRangeRef(startKey, endKey), limitsHint);
 		result = result_;
 	}
 
-	if (result.size() == 0) {
+	if (result.empty()) {
 		TraceEvent(SevDebug, "ZeroElementsIntheRange").detail("Start", startKey).detail("End", endKey);
-		return Void();
+		co_return;
 	}
 	// Note : KeySelector::setKey has byte limit according to the knobs, customize it if needed
 	if (ks->offset < 1) {
@@ -194,7 +195,6 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
 	    .detail("NormalizedOffset", ks->offset)
 	    .detail("SpecialKeyRangeStart", skrImpl->getKeyRange().begin)
 	    .detail("SpecialKeyRangeEnd", skrImpl->getKeyRange().end);
-	return Void();
 }
 
 // This function will normalize the given KeySelector to a standard KeySelector:
@@ -205,24 +205,24 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl*
 // to maintain; Thus, separate each part to make the code easy to understand and more compact
 // Boundary is the range of the legal key space, which, by default is the range of the module
 // And (\xff\xff, \xff\xff\xff) if SPECIAL_KEY_SPACE_RELAXED is turned on
-ACTOR Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks,
-                                             ReadYourWritesTransaction* ryw,
-                                             KeySelector* ks,
-                                             KeyRangeRef boundary,
-                                             int* actualOffset,
-                                             RangeResult* result,
-                                             KeyRangeMap<Optional<RangeResult>>* cache) {
+Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks,
+                                       ReadYourWritesTransaction* ryw,
+                                       KeySelector* ks,
+                                       KeyRangeRef boundary,
+                                       int* actualOffset,
+                                       RangeResult* result,
+                                       KeyRangeMap<Optional<RangeResult>>* cache) {
 	// If offset < 1, where we need to move left, iter points to the range containing at least one smaller key
 	// (It's a wasting of time to walk through the range whose begin key is same as ks->key)
 	// (rangeContainingKeyBefore itself handles the case where ks->key == Key())
 	// Otherwise, we only need to move right if offset > 1, iter points to the range containing the key
 	// Since boundary.end is always a key in the RangeMap, it is always safe to move right
-	state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::iterator iter =
+	RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::iterator iter =
 	    ks->offset < 1 ? sks->getReadImpls().rangeContainingKeyBefore(ks->getKey())
 	                   : sks->getReadImpls().rangeContaining(ks->getKey());
 	while ((ks->offset < 1 && iter->begin() >= boundary.begin) || (ks->offset > 1 && iter->begin() < boundary.end)) {
 		if (iter->value() != nullptr) {
-			wait(moveKeySelectorOverRangeActor(iter->value(), ryw, ks, cache));
+			co_await moveKeySelectorOverRangeActor(iter->value(), ryw, ks, cache);
 		}
 		// Check if we can still move the iterator left
 		if (ks->offset < 1) {
@@ -253,7 +253,6 @@ ACTOR Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks,
 		}
 		ks->offset = 1;
 	}
-	return Void();
 }
 
 SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool testOnly)
@@ -270,55 +269,55 @@ SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool
 }
 
 void SpecialKeySpace::modulesBoundaryInit() {
-	for (const auto& pair : moduleToBoundary) {
-		ASSERT(range.contains(pair.second));
+	for (const auto& [module, moduleBoundary] : moduleToBoundary) {
+		ASSERT(range.contains(moduleBoundary));
 		// Make sure the module is not overlapping with any registered read modules
 		// Note: same like ranges, one module's end cannot be another module's start, relax the condition if needed
-		ASSERT(modules.rangeContaining(pair.second.begin) == modules.rangeContaining(pair.second.end) &&
-		       modules[pair.second.begin] == SpecialKeySpace::MODULE::UNKNOWN);
-		modules.insert(pair.second, pair.first);
+		ASSERT(modules.rangeContaining(moduleBoundary.begin) == modules.rangeContaining(moduleBoundary.end) &&
+		       modules[moduleBoundary.begin] == SpecialKeySpace::MODULE::UNKNOWN);
+		modules.insert(moduleBoundary, module);
 		// Note: Due to underlying implementation, the insertion here is important to make cross_module_read being
 		// handled correctly
-		readImpls.insert(pair.second, nullptr);
-		writeImpls.insert(pair.second, nullptr);
+		readImpls.insert(moduleBoundary, nullptr);
+		writeImpls.insert(moduleBoundary, nullptr);
 	}
 }
 
-ACTOR Future<RangeResult> SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks,
-                                                         ReadYourWritesTransaction* ryw,
-                                                         KeySelector begin,
-                                                         KeySelector end,
-                                                         GetRangeLimits limits,
-                                                         Reverse reverse) {
+Future<RangeResult> SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks,
+                                                   ReadYourWritesTransaction* ryw,
+                                                   KeySelector begin,
+                                                   KeySelector end,
+                                                   GetRangeLimits limits,
+                                                   Reverse reverse) {
 	ASSERT(ryw);
-	choose {
-		when(RangeResult result =
-		         wait(SpecialKeySpace::getRangeAggregationActor(sks, ryw, begin, end, limits, reverse))) {
-			return result;
-		}
-		when(wait(ryw->resetFuture())) {
-			throw internal_error();
-		}
+	auto choice = co_await race(SpecialKeySpace::getRangeAggregationActor(sks, ryw, begin, end, limits, reverse),
+	                            ryw->resetFuture());
+	if (choice.index() == 0) {
+		co_return std::get<0>(std::move(choice));
+	}
+	if (choice.index() == 1) {
+		throw internal_error();
 	}
+	UNREACHABLE();
 }
 
-ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySpace* sks,
-                                                                    ReadYourWritesTransaction* ryw,
-                                                                    KeySelector begin,
-                                                                    KeySelector end,
-                                                                    GetRangeLimits limits,
-                                                                    Reverse reverse) {
+Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySpace* sks,
+                                                              ReadYourWritesTransaction* ryw,
+                                                              KeySelector begin,
+                                                              KeySelector end,
+                                                              GetRangeLimits limits,
+                                                              Reverse reverse) {
 	// This function handles ranges which cover more than one keyrange and aggregates all results
 	// KeySelector, GetRangeLimits and reverse are all handled here
-	state RangeResult result;
-	state RangeResult pairs;
-	state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::iterator iter;
-	state int actualBeginOffset;
-	state int actualEndOffset;
-	state KeyRangeRef moduleBoundary;
+	RangeResult result;
+	RangeResult pairs;
+	RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::iterator iter;
+	int actualBeginOffset{ 0 };
+	int actualEndOffset{ 0 };
+	KeyRangeRef moduleBoundary;
 	// used to cache results from potential first async read
 	// the current implementation will read the whole range result to save in the cache
-	state KeyRangeMap<Optional<RangeResult>> cache(Optional<RangeResult>(), specialKeys.end);
+	KeyRangeMap<Optional<RangeResult>> cache(Optional<RangeResult>(), specialKeys.end);
 
 	if (ryw->specialKeySpaceRelaxed()) {
 		moduleBoundary = sks->range;
@@ -339,20 +338,20 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 		}
 	}
 
-	wait(normalizeKeySelectorActor(sks, ryw, &begin, moduleBoundary, &actualBeginOffset, &result, &cache));
-	wait(normalizeKeySelectorActor(sks, ryw, &end, moduleBoundary, &actualEndOffset, &result, &cache));
+	co_await normalizeKeySelectorActor(sks, ryw, &begin, moduleBoundary, &actualBeginOffset, &result, &cache);
+	co_await normalizeKeySelectorActor(sks, ryw, &end, moduleBoundary, &actualEndOffset, &result, &cache);
 	// Handle all corner cases like what RYW does
 	// return if range inverted
 	if (actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
 		CODE_PROBE(true, "inverted range");
-		return RangeResultRef(false, false);
+		co_return RangeResultRef(false, false);
 	}
 	// If touches begin or end, return with readToBegin and readThroughEnd flags
 	if (begin.getKey() == moduleBoundary.end || end.getKey() == moduleBoundary.begin) {
 		CODE_PROBE(true, "query touches begin or end");
-		return result;
+		co_return result;
 	}
-	state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::Ranges ranges =
+	RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::Ranges ranges =
 	    sks->getReadImpls().intersectingRanges(KeyRangeRef(begin.getKey(), end.getKey()));
 
 	// TODO : workaround to write this two together to make the code compact
@@ -367,11 +366,11 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 			KeyRef keyStart = kr.contains(begin.getKey()) ? begin.getKey() : kr.begin;
 			KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end;
 			if (iter->value()->isAsync() && cache.rangeContaining(keyStart).value().present()) {
-				const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
-				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache));
+				auto* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
+				RangeResult pairs_ = co_await ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache);
 				pairs = pairs_;
 			} else {
-				RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits));
+				RangeResult pairs_ = co_await iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits);
 				pairs = pairs_;
 			}
 			result.arena().dependsOn(pairs.arena());
@@ -386,7 +385,7 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 				if (limits.isReached()) {
 					result.more = true;
 					result.readToBegin = false;
-					return result;
+					co_return result;
 				};
 			}
 		}
@@ -398,31 +397,31 @@ ACTOR Future<RangeResult> SpecialKeySpace::getRangeAggregationActor(SpecialKeySp
 			KeyRef keyStart = kr.contains(begin.getKey()) ? begin.getKey() : kr.begin;
 			KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end;
 			if (iter->value()->isAsync() && cache.rangeContaining(keyStart).value().present()) {
-				const SpecialKeyRangeAsyncImpl* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
-				RangeResult pairs_ = wait(ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache));
+				auto* ptr = dynamic_cast<const SpecialKeyRangeAsyncImpl*>(iter->value());
+				RangeResult pairs_ = co_await ptr->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits, &cache);
 				pairs = pairs_;
 			} else {
-				RangeResult pairs_ = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits));
+				RangeResult pairs_ = co_await iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd), limits);
 				pairs = pairs_;
 			}
 			result.arena().dependsOn(pairs.arena());
 			// limits handler
-			for (int i = 0; i < pairs.size(); ++i) {
-				ASSERT(iter->range().contains(pairs[i].key));
-				result.push_back(result.arena(), pairs[i]);
+			for (const auto& keyValue : pairs) {
+				ASSERT(iter->range().contains(keyValue.key));
+				result.push_back(result.arena(), keyValue);
 				// Note : behavior here is even the last k-v pair makes total bytes larger than specified, it's still
 				// returned. In other words, the total size of the returned value (less the last entry) will be less
 				// than byteLimit
-				limits.decrement(pairs[i]);
+				limits.decrement(keyValue);
 				if (limits.isReached()) {
 					result.more = true;
 					result.readThroughEnd = false;
-					return result;
+					co_return result;
 				};
 			}
 		}
 	}
-	return result;
+	co_return result;
 }
 
 Future<RangeResult> SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw,
@@ -449,20 +448,18 @@ Future<RangeResult> SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw,
 	return checkRYWValid(this, ryw, begin, end, limits, reverse);
 }
 
-ACTOR Future<Optional<Value>> SpecialKeySpace::getActor(SpecialKeySpace* sks,
-                                                        ReadYourWritesTransaction* ryw,
-                                                        KeyRef key) {
+Future<Optional<Value>> SpecialKeySpace::getActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeyRef key) {
 	// use getRange to workaround this
-	RangeResult result = wait(sks->getRange(ryw,
-	                                        KeySelector(firstGreaterOrEqual(key)),
-	                                        KeySelector(firstGreaterOrEqual(keyAfter(key))),
-	                                        GetRangeLimits(CLIENT_KNOBS->TOO_MANY),
-	                                        Reverse::False));
+	RangeResult result = co_await sks->getRange(ryw,
+	                                            KeySelector(firstGreaterOrEqual(key)),
+	                                            KeySelector(firstGreaterOrEqual(keyAfter(key))),
+	                                            GetRangeLimits(CLIENT_KNOBS->TOO_MANY),
+	                                            Reverse::False);
 	ASSERT(result.size() <= 1);
-	if (result.size()) {
-		return Optional<Value>(result[0].value);
+	if (!result.empty()) {
+		co_return Optional<Value>(result[0].value);
 	} else {
-		return Optional<Value>();
+		co_return Optional<Value>();
 	}
 }
 
@@ -575,11 +572,11 @@ KeyRange SpecialKeySpace::decode(const KeyRangeRef& kr) {
 	return KeyRangeRef(begin->value()->decode(kr.begin), begin->value()->decode(kr.end));
 }
 
-ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw) {
-	state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
+Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw) {
+	RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
 	    ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys);
-	state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
-	state std::vector<SpecialKeyRangeRWImpl*> writeModulePtrs;
+	RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
+	std::vector<SpecialKeyRangeRWImpl*> writeModulePtrs;
 	std::unordered_set<SpecialKeyRangeRWImpl*> deduplicate;
 	while (iter != ranges.end()) {
 		std::pair<bool, Optional<Value>> entry = iter->value();
@@ -593,9 +590,9 @@ ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction*
 		++iter;
 	}
 
-	state std::vector<SpecialKeyRangeRWImpl*>::const_iterator it;
+	std::vector<SpecialKeyRangeRWImpl*>::const_iterator it;
 	for (it = writeModulePtrs.begin(); it != writeModulePtrs.end(); ++it) {
-		Optional<std::string> msg = wait((*it)->commit(ryw));
+		Optional<std::string> msg = co_await (*it)->commit(ryw);
 		if (msg.present()) {
 			ryw->setSpecialKeySpaceErrorMsg(msg.get());
 			TraceEvent(SevDebug, "SpecialKeySpaceManagementAPIError")
@@ -604,7 +601,6 @@ ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction*
 			throw special_keys_api_failure();
 		}
 	}
-	return Void();
 }
 
 Future<Void> SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) {
@@ -647,9 +643,9 @@ Future<RangeResult> SKSCTestAsyncReadImpl::getRange(ReadYourWritesTransaction* r
 
 ReadConflictRangeImpl::ReadConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-ACTOR static Future<RangeResult> getReadConflictRangeImpl(ReadYourWritesTransaction* ryw, KeyRange kr) {
-	wait(ryw->pendingReads());
-	return ryw->getReadConflictRangeIntersecting(kr);
+static Future<RangeResult> getReadConflictRangeImpl(ReadYourWritesTransaction* ryw, KeyRange kr) {
+	co_await ryw->pendingReads();
+	co_return ryw->getReadConflictRangeIntersecting(kr);
 }
 
 Future<RangeResult> ReadConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw,
@@ -688,12 +684,13 @@ Future<RangeResult> ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw
 	return result;
 }
 
-ACTOR Future<RangeResult> ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
-	loop {
+Future<RangeResult> ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+	while (true) {
+		Error err;
 		try {
 			auto keys = kr.removePrefix(ddStatsRange.begin);
 			Standalone<VectorRef<DDMetricsRef>> resultWithoutPrefix =
-			    wait(waitDataDistributionMetricsList(ryw->getDatabase(), keys, CLIENT_KNOBS->TOO_MANY));
+			    co_await waitDataDistributionMetricsList(ryw->getDatabase(), keys, CLIENT_KNOBS->TOO_MANY);
 			RangeResult result;
 			for (const auto& ddMetricsRef : resultWithoutPrefix) {
 				// each begin key is the previous end key, thus we only encode the begin key in the result
@@ -707,17 +704,17 @@ ACTOR Future<RangeResult> ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw,
 				ValueRef bytes(result.arena(), statsString);
 				result.push_back(result.arena(), KeyValueRef(beginKey, bytes));
 			}
-			return result;
+			co_return result;
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_dd_not_found) {
-				TraceEvent(SevWarnAlways, "DataDistributorNotPresent")
-				    .detail("Operation", "DDMetricsReqestThroughSpecialKeys");
-				wait(delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				continue;
-			}
-			throw err;
+			err = e;
+		}
+		if (err.code() == error_code_dd_not_found) {
+			TraceEvent(SevWarnAlways, "DataDistributorNotPresent")
+			    .detail("Operation", "DDMetricsReqestThroughSpecialKeys");
+			co_await delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			continue;
 		}
+		throw err;
 	}
 }
 
@@ -833,16 +830,16 @@ RangeResult rywGetRange(ReadYourWritesTransaction* ryw, const KeyRangeRef& kr, c
 }
 
 // read from those readwrite modules in which special keys have one-to-one mapping with real persisted keys
-ACTOR Future<RangeResult> rwModuleWithMappingGetRangeActor(ReadYourWritesTransaction* ryw,
-                                                           const SpecialKeyRangeRWImpl* impl,
-                                                           KeyRangeRef kr) {
-	RangeResult resultWithoutPrefix =
-	    wait(ryw->getTransaction().getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY));
+Future<RangeResult> rwModuleWithMappingGetRangeActor(ReadYourWritesTransaction* ryw,
+                                                     const SpecialKeyRangeRWImpl* impl,
+                                                     KeyRangeRef kr) {
+	RangeResult resultWithoutPrefix = co_await ryw->getTransaction().getRange(
+	    ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY);
 	RangeResult result;
 	for (const KeyValueRef& kv : resultWithoutPrefix)
 		result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value));
-	return rywGetRange(ryw, kr, result);
+	co_return rywGetRange(ryw, kr, result);
 }
 
 ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -904,14 +901,14 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw,
 	return true;
 }
 
-ACTOR Future<bool> checkExclusion(Database db,
-                                  std::vector<AddressExclusion> addresses,
-                                  std::set<AddressExclusion> exclusions,
-                                  std::unordered_set<std::string> localities,
-                                  bool markFailed,
-                                  Optional<std::string>* msg) {
-	StatusObject status = wait(StatusClient::statusFetcher(db));
-	state std::string errorString =
+Future<bool> checkExclusion(Database db,
+                            std::vector<AddressExclusion> addresses,
+                            std::set<AddressExclusion> exclusions,
+                            std::unordered_set<std::string> localities,
+                            bool markFailed,
+                            Optional<std::string>* msg) {
+	StatusObject status = co_await StatusClient::statusFetcher(db);
+	std::string errorString =
 	    "ERROR: Could not calculate the impact of this exclude on the total available space in the cluster.\n"
 	    "Please try the exclude again in 30 seconds.\n"
 	    "Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without checking available "
@@ -923,31 +920,31 @@ ACTOR Future<bool> checkExclusion(Database db,
 	if (!statusObj.get("cluster", statusObjCluster)) {
 		*msg = ManagementAPIError::toJsonString(
 		    false, markFailed ? "exclude failed" : "exclude", errorString + "Field 'cluster' missing.\n");
-		return false;
+		co_return false;
 	}
 
 	StatusObjectReader processesMap;
 	if (!statusObjCluster.get("processes", processesMap)) {
 		*msg = ManagementAPIError::toJsonString(
 		    false, markFailed ? "exclude failed" : "exclude", errorString + "Field 'processes' missing.\n");
-		return false;
+		co_return false;
 	}
 
-	state int ssTotalCount = 0;
-	state int ssExcludedCount = 0;
+	int ssTotalCount = 0;
+	int ssExcludedCount = 0;
 
-	state std::unordered_set<std::string> diskLocalities;
-	state int64_t totalKvStoreFreeBytes = 0;
-	state int64_t totalKvStoreFreeBytesNotExcluded = 0;
-	state int64_t totalKvStoreUsedBytes = 0;
-	state int64_t totalKvStoreUsedBytesNotExcluded = 0;
-	state int64_t totalKvStoreAvailableBytes = 0;
-	state std::map<std::string, std::vector<std::string>> parsedLocalities;
+	std::unordered_set<std::string> diskLocalities;
+	int64_t totalKvStoreFreeBytes = 0;
+	int64_t totalKvStoreFreeBytesNotExcluded = 0;
+	int64_t totalKvStoreUsedBytes = 0;
+	int64_t totalKvStoreUsedBytesNotExcluded = 0;
+	int64_t totalKvStoreAvailableBytes = 0;
+	std::map<std::string, std::vector<std::string>> parsedLocalities;
 	// Convert the passed localities into a map of vectors to make it easier to check if a process
 	// is excluded by locality.
 	for (const auto& locality : localities) {
 		std::pair<std::string, std::string> locality_key_value = decodeLocality(locality);
-		if (locality_key_value.first == "") {
+		if (locality_key_value.first.empty()) {
 			continue;
 		}
 
@@ -962,7 +959,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 	// be empty and if the addresses are not empty, we are not using the locality based exclusion. So this
 	// step is a safeguard for future changes where the provided addresses are not empty and locality based
 	// exclusions are used.
-	state std::vector<AddressExclusion> excludedAddresses;
+	std::vector<AddressExclusion> excludedAddresses;
 	if (markFailed && !addresses.empty()) {
 		excludedAddresses = addresses;
 	}
@@ -982,7 +979,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 				if (!process.get("address", addrStr)) {
 					*msg =
 					    ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
-					return false;
+					co_return false;
 				}
 				NetworkAddress addr = NetworkAddress::parse(addrStr);
 				excluded = addressExcluded(exclusions, addr);
@@ -1037,7 +1034,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 						    false,
 						    markFailed ? "exclude failed" : "exclude",
 						    errorString + "Field 'kvstore_used_bytes' missing for storage role.\n");
-						return false;
+						co_return false;
 					}
 
 					int64_t free_bytes;
@@ -1046,7 +1043,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 						    false,
 						    markFailed ? "exclude failed" : "exclude",
 						    errorString + "Field 'kvstore_free_bytes' missing for storage role.\n");
-						return false;
+						co_return false;
 					}
 
 					int64_t available_bytes;
@@ -1055,7 +1052,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 						    false,
 						    markFailed ? "exclude failed" : "exclude",
 						    errorString + "Field 'kvstore_available_bytes' missing for storage role.\n");
-						return false;
+						co_return false;
 					}
 
 					totalKvStoreUsedBytes += used_bytes;
@@ -1082,7 +1079,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 	catch (...) {
 		*msg = ManagementAPIError::toJsonString(
 		    false, markFailed ? "exclude failed" : "exclude", errorString + "General exception raised.\n");
-		return false;
+		co_return false;
 	}
 
 	// If the exclusion command only contains processes that serve a non storage role we can skip the free capacity
@@ -1112,14 +1109,14 @@ ACTOR Future<bool> checkExclusion(Database db,
 			    "Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without "
 			    "checking available space.\n";
 			*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp);
-			return false;
+			co_return false;
 		}
 	}
 
 	if (markFailed) {
-		state bool safe;
+		bool safe{ false };
 		try {
-			bool _safe = wait(checkSafeExclusions(db, addresses));
+			bool _safe = co_await checkSafeExclusions(db, addresses);
 			safe = _safe;
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -1135,11 +1132,11 @@ ACTOR Future<bool> checkExclusion(Database db,
 			                   "Call set(\"0xff0xff/management/failed/<ADDRESS...>\", ...) to exclude without "
 			                   "performing safety checks.\n";
 			*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp);
-			return false;
+			co_return false;
 		}
 	}
 
-	return true;
+	co_return true;
 }
 
 void includeServers(ReadYourWritesTransaction* ryw) {
@@ -1179,28 +1176,28 @@ void includeServers(ReadYourWritesTransaction* ryw) {
 	}
 }
 
-ACTOR Future<Optional<std::string>> excludeCommitActor(ReadYourWritesTransaction* ryw, bool failed) {
+Future<Optional<std::string>> excludeCommitActor(ReadYourWritesTransaction* ryw, bool failed) {
 	// parse network addresses
-	state Optional<std::string> result;
-	state std::vector<AddressExclusion> addresses;
-	state std::set<AddressExclusion> exclusions;
-	state std::unordered_set<std::string> localities;
+	Optional<std::string> result;
+	std::vector<AddressExclusion> addresses;
+	std::set<AddressExclusion> exclusions;
+	std::unordered_set<std::string> localities;
 
 	if (!parseNetWorkAddrFromKeys(ryw, failed, addresses, exclusions, result))
-		return result;
+		co_return result;
 	// If force option is not set, we need to do safety check
 	auto force = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
 	    failed ? "failed" : "excluded", "force")];
 	// Only do safety check when we have servers to be excluded and the force option key is not set
-	if (addresses.size() && !(force.first && force.second.present())) {
-		bool safe = wait(checkExclusion(ryw->getDatabase(), addresses, exclusions, localities, failed, &result));
+	if (!addresses.empty() && !(force.first && force.second.present())) {
+		bool safe = co_await checkExclusion(ryw->getDatabase(), addresses, exclusions, localities, failed, &result);
 		if (!safe)
-			return result;
+			co_return result;
 	}
-	wait(excludeServers(&(ryw->getTransaction()), addresses, failed));
+	co_await excludeServers(&(ryw->getTransaction()), addresses, failed);
 	includeServers(ryw);
 
-	return result;
+	co_return result;
 }
 
 Future<Optional<std::string>> ExcludeServersRangeImpl::commit(ReadYourWritesTransaction* ryw) {
@@ -1235,32 +1232,32 @@ Future<Optional<std::string>> FailedServersRangeImpl::commit(ReadYourWritesTrans
 	return excludeCommitActor(ryw, true);
 }
 
-ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
-	state RangeResult result;
-	state Transaction& tr = ryw->getTransaction();
+Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
+	RangeResult result;
+	Transaction& tr = ryw->getTransaction();
 	tr.setOption(FDBTransactionOptions::RAW_ACCESS);
 	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state Future<std::vector<AddressExclusion>> fExclusions = getAllExcludedServers(&tr);
-	state Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(&tr);
-	state Future<RangeResult> fServerList = tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-	state Future<Optional<Standalone<StringRef>>> fLogsKey = tr.get(logsKey);
-	wait(success(fExclusions) && success(fExcludedLocalities) && success(fServerList));
+	Future<std::vector<AddressExclusion>> fExclusions = getAllExcludedServers(&tr);
+	Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(&tr);
+	Future<RangeResult> fServerList = tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+	Future<Optional<Standalone<StringRef>>> fLogsKey = tr.get(logsKey);
+	co_await (success(fExclusions) && success(fExcludedLocalities) && success(fServerList));
 
-	state std::vector<AddressExclusion> excl = fExclusions.get();
-	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
-	state std::set<NetworkAddress> inProgressExclusion;
+	std::vector<AddressExclusion> excl = fExclusions.get();
+	std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
+	std::set<NetworkAddress> inProgressExclusion;
 	// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed
 	// recovery Check that there aren't any storage servers with addresses violating the exclusions
-	state RangeResult serverList = fServerList.get();
+	RangeResult serverList = fServerList.get();
 	ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
 	// We have to make use of the localities here to verify if a server is still in the server list,
 	// even if it might be missing in the workers as the server is not running anymore.
-	state std::vector<std::string> excludedLocalities = fExcludedLocalities.get();
+	std::vector<std::string> excludedLocalities = fExcludedLocalities.get();
 	// Decode the excluded localities to check if any server is excluded by locality.
-	state std::vector<std::pair<std::string, std::string>> decodedExcludedLocalities;
+	std::vector<std::pair<std::string, std::string>> decodedExcludedLocalities;
 	for (auto& excludedLocality : excludedLocalities) {
 		decodedExcludedLocalities.push_back(decodeLocality(excludedLocality));
 	}
@@ -1293,21 +1290,21 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 		}
 	}
 
-	wait(success(fLogsKey));
+	co_await fLogsKey;
 	Optional<Standalone<StringRef>> value = fLogsKey.get();
 	ASSERT(value.present());
 	// TODO(jscheuermann): The logs key range doesn't hold any information about localities. This is a limitation
 	// for locality based exclusions. The problematic edge case here is a log server that still has mutation on it
 	// but is currently not part of the worker list, e.g. because it was shutdown or is partitioned.
 	auto logs = decodeLogsValue(value.get());
-	for (auto const& log : logs.first) {
-		if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
-			inProgressExclusion.insert(log.second);
+	for (const auto& [_logId, logAddress] : logs.first) {
+		if (logAddress == NetworkAddress() || addressExcluded(exclusions, logAddress)) {
+			inProgressExclusion.insert(logAddress);
 		}
 	}
-	for (auto const& log : logs.second) {
-		if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
-			inProgressExclusion.insert(log.second);
+	for (const auto& [_logId, logAddress] : logs.second) {
+		if (logAddress == NetworkAddress() || addressExcluded(exclusions, logAddress)) {
+			inProgressExclusion.insert(logAddress);
 		}
 	}
 
@@ -1325,7 +1322,7 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 		}
 	}
 
-	return result;
+	co_return result;
 }
 
 ExclusionInProgressRangeImpl::ExclusionInProgressRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
@@ -1336,9 +1333,9 @@ Future<RangeResult> ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransac
 	return ExclusionInProgressActor(ryw, getKeyRange().begin, kr);
 }
 
-ACTOR Future<RangeResult> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
+Future<RangeResult> getProcessClassActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
 	ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
-	std::vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
+	std::vector<ProcessData> _workers = co_await getWorkers(&ryw->getTransaction());
 	auto workers = _workers; // strip const
 	// Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5
 	std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) {
@@ -1359,17 +1356,17 @@ ACTOR Future<RangeResult> getProcessClassActor(ReadYourWritesTransaction* ryw, K
 			result.push_back(result.arena(), KeyValueRef(k, v));
 		}
 	}
-	return rywGetRange(ryw, kr, result);
+	co_return rywGetRange(ryw, kr, result);
 }
 
-ACTOR Future<Optional<std::string>> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) {
+Future<Optional<std::string>> processClassCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef range) {
 	// enable related options
 	ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	ryw->setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 	ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
-	std::vector<ProcessData> workers = wait(
-	    getWorkers(&ryw->getTransaction())); // make sure we use the Transaction object to avoid used_during_commit()
+	std::vector<ProcessData> workers = co_await getWorkers(
+	    &ryw->getTransaction()); // make sure we use the Transaction object to avoid used_during_commit()
 
 	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
 	auto iter = ranges.begin();
@@ -1400,7 +1397,7 @@ ACTOR Future<Optional<std::string>> processClassCommitActor(ReadYourWritesTransa
 		}
 		++iter;
 	}
-	return Optional<std::string>();
+	co_return Optional<std::string>();
 }
 
 ProcessClassRangeImpl::ProcessClassRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -1459,9 +1456,9 @@ void ProcessClassRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef&
 	    ryw, "setclass", "Clear range operation is meaningless thus forbidden for setclass");
 }
 
-ACTOR Future<RangeResult> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
+Future<RangeResult> getProcessClassSourceActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
 	ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
-	std::vector<ProcessData> _workers = wait(getWorkers(&ryw->getTransaction()));
+	std::vector<ProcessData> _workers = co_await getWorkers(&ryw->getTransaction());
 	auto workers = _workers; // strip const
 	// Note : the sort by string is anti intuition, ex. 1.1.1.1:11 < 1.1.1.1:5
 	std::sort(workers.begin(), workers.end(), [](const ProcessData& lhs, const ProcessData& rhs) {
@@ -1484,7 +1481,7 @@ ACTOR Future<RangeResult> getProcessClassSourceActor(ReadYourWritesTransaction*
 			result.arena().dependsOn(v.arena());
 		}
 	}
-	return result;
+	co_return result;
 }
 
 ProcessClassSourceRangeImpl::ProcessClassSourceRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
@@ -1495,16 +1492,16 @@ Future<RangeResult> ProcessClassSourceRangeImpl::getRange(ReadYourWritesTransact
 	return getProcessClassSourceActor(ryw, getKeyRange().begin, kr);
 }
 
-ACTOR Future<RangeResult> getLockedKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+Future<RangeResult> getLockedKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
-	Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(databaseLockedKey);
 	RangeResult result;
 	if (val.present()) {
 		UID uid = UID::fromString(BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()).toString());
 		result.push_back_deep(result.arena(), KeyValueRef(kr.begin, Value(uid.toString())));
 	}
-	return result;
+	co_return result;
 }
 
 LockDatabaseImpl::LockDatabaseImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -1527,11 +1524,11 @@ Future<RangeResult> LockDatabaseImpl::getRange(ReadYourWritesTransaction* ryw,
 	}
 }
 
-ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransaction* ryw, UID uid) {
-	state Optional<std::string> msg;
+Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransaction* ryw, UID uid) {
+	Optional<std::string> msg;
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
-	Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(databaseLockedKey);
 
 	if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != uid) {
 		// check database not locked
@@ -1546,17 +1543,17 @@ ACTOR Future<Optional<std::string>> lockDatabaseCommitActor(ReadYourWritesTransa
 		ryw->getTransaction().addWriteConflictRange(normalKeys);
 	}
 
-	return msg;
+	co_return msg;
 }
 
-ACTOR Future<Optional<std::string>> unlockDatabaseCommitActor(ReadYourWritesTransaction* ryw) {
+Future<Optional<std::string>> unlockDatabaseCommitActor(ReadYourWritesTransaction* ryw) {
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
-	Optional<Value> val = wait(ryw->getTransaction().get(databaseLockedKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(databaseLockedKey);
 	if (val.present()) {
 		ryw->getTransaction().clear(singleKeyRange(databaseLockedKey));
 	}
-	return Optional<std::string>();
+	co_return Optional<std::string>();
 }
 
 Future<Optional<std::string>> LockDatabaseImpl::commit(ReadYourWritesTransaction* ryw) {
@@ -1576,17 +1573,17 @@ Future<Optional<std::string>> LockDatabaseImpl::commit(ReadYourWritesTransaction
 	}
 }
 
-ACTOR Future<RangeResult> getConsistencyCheckKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+Future<RangeResult> getConsistencyCheckKeyActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 	ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	Optional<Value> val = wait(ryw->getTransaction().get(fdbShouldConsistencyCheckBeSuspended));
+	Optional<Value> val = co_await ryw->getTransaction().get(fdbShouldConsistencyCheckBeSuspended);
 	bool ccSuspendSetting = val.present() ? BinaryReader::fromStringRef<bool>(val.get(), Unversioned()) : false;
 	RangeResult result;
 	if (ccSuspendSetting) {
 		result.push_back_deep(result.arena(), KeyValueRef(kr.begin, ValueRef()));
 	}
-	return result;
+	co_return result;
 }
 
 ConsistencyCheckImpl::ConsistencyCheckImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -1668,14 +1665,13 @@ void GlobalConfigImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, co
 // Writes global configuration changes to durable memory. Also writes the
 // changes made in the transaction to a recent history set, and updates the
 // latest version which the global configuration was updated at.
-ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* globalConfig,
-                                                            ReadYourWritesTransaction* ryw) {
-	state Transaction& tr = ryw->getTransaction();
+Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* globalConfig, ReadYourWritesTransaction* ryw) {
+	Transaction& tr = ryw->getTransaction();
 	ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
 
 	// History should only contain three most recent updates. If it currently
 	// has three items, remove the oldest to make room for a new item.
-	RangeResult history = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY));
+	RangeResult history = co_await tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY);
 	constexpr int kGlobalConfigMaxHistorySize = 3;
 	if (history.size() > kGlobalConfigMaxHistorySize - 1) {
 		for (int i = 0; i < history.size() - (kGlobalConfigMaxHistorySize - 1); ++i) {
@@ -1689,9 +1685,9 @@ ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* gl
 	// Transform writes from the special-key-space (\xff\xff/global_config/) to
 	// the system key space (\xff/globalConfig/), and writes mutations to
 	// latest version history.
-	state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
+	RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
 	    ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys);
-	state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
+	RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
 	while (iter != ranges.end()) {
 		std::pair<bool, Optional<Value>> entry = iter->value();
 		if (entry.first) {
@@ -1709,7 +1705,7 @@ ACTOR Future<Optional<std::string>> globalConfigCommitActor(GlobalConfigImpl* gl
 	}
 	GlobalConfig::applyChanges(tr, insertions, clears);
 
-	return Optional<std::string>();
+	co_return Optional<std::string>();
 }
 
 // Called when a transaction includes keys in the global configuration special-key-space range.
@@ -1792,9 +1788,9 @@ void TracingOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key
 
 CoordinatorsImpl::CoordinatorsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
-ACTOR Future<RangeResult> coordinatorsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
-	state ClusterConnectionString cs = ryw->getDatabase()->getConnectionRecord()->getConnectionString();
-	state std::vector<NetworkAddress> coordinator_processes = wait(cs.tryResolveHostnames());
+Future<RangeResult> coordinatorsGetRangeActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
+	ClusterConnectionString cs = ryw->getDatabase()->getConnectionRecord()->getConnectionString();
+	std::vector<NetworkAddress> coordinator_processes = co_await cs.tryResolveHostnames();
 	RangeResult result;
 	Key cluster_description_key = prefix.withSuffix("cluster_description"_sr);
 	if (kr.contains(cluster_description_key)) {
@@ -1807,7 +1803,7 @@ ACTOR Future<RangeResult> coordinatorsGetRangeActor(ReadYourWritesTransaction* r
 	          [](const NetworkAddress& lhs, const NetworkAddress& rhs) { return lhs.toString() < rhs.toString(); });
 	std::string processes_str;
 	for (const auto& w : coordinator_processes) {
-		if (processes_str.size())
+		if (!processes_str.empty())
 			processes_str += ",";
 		processes_str += w.toString();
 	}
@@ -1815,7 +1811,7 @@ ACTOR Future<RangeResult> coordinatorsGetRangeActor(ReadYourWritesTransaction* r
 	if (kr.contains(processes_key)) {
 		result.push_back_deep(result.arena(), KeyValueRef(processes_key, Value(processes_str)));
 	}
-	return rywGetRange(ryw, kr, result);
+	co_return rywGetRange(ryw, kr, result);
 }
 
 Future<RangeResult> CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw,
@@ -1825,12 +1821,12 @@ Future<RangeResult> CoordinatorsImpl::getRange(ReadYourWritesTransaction* ryw,
 	return coordinatorsGetRangeActor(ryw, prefix, kr);
 }
 
-ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
-	state ClusterConnectionString conn; // We don't care about the Key here.
-	state std::vector<std::string> process_address_or_hostname_strs;
-	state Optional<std::string> msg;
-	state int index;
-	state bool parse_error = false;
+static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+	ClusterConnectionString conn; // We don't care about the Key here.
+	std::vector<std::string> process_address_or_hostname_strs;
+	Optional<std::string> msg;
+	int index{ 0 };
+	bool parse_error = false;
 
 	// check update for coordinators
 	Key processes_key = "processes"_sr.withPrefix(kr.begin);
@@ -1839,8 +1835,8 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 		ASSERT(processes_entry.second.present()); // no clear should be seen here
 		auto processesStr = processes_entry.second.get().toString();
 		boost::split(process_address_or_hostname_strs, processesStr, [](char c) { return c == ','; });
-		if (!process_address_or_hostname_strs.size()) {
-			return ManagementAPIError::toJsonString(
+		if (process_address_or_hostname_strs.empty()) {
+			co_return ManagementAPIError::toJsonString(
 			    false,
 			    "coordinators",
 			    "New coordinators\' processes are empty, please specify new processes\' network addresses with format "
@@ -1866,7 +1862,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 			if (parse_error) {
 				std::string error = "ERROR: \'" + process_address_or_hostname_strs[index] +
 				                    "\' is not a valid network endpoint address\n";
-				return ManagementAPIError::toJsonString(false, "coordinators", error);
+				co_return ManagementAPIError::toJsonString(false, "coordinators", error);
 			}
 		}
 	}
@@ -1882,7 +1878,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 			newName = entry.second.get().toString();
 		} else {
 			// throw the error
-			return ManagementAPIError::toJsonString(
+			co_return ManagementAPIError::toJsonString(
 			    false, "coordinators", "Cluster description must match [A-Za-z0-9_]+");
 		}
 	}
@@ -1891,7 +1887,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 	    .detail("NewConnectionString", conn.toString())
 	    .detail("Description", entry.first ? entry.second.get().toString() : "");
 
-	Optional<CoordinatorsResult> r = wait(changeQuorumChecker(&ryw->getTransaction(), &conn, newName));
+	Optional<CoordinatorsResult> r = co_await changeQuorumChecker(&ryw->getTransaction(), &conn, newName);
 
 	TraceEvent(SevDebug, "SKSChangeCoordinatorsFinish")
 	    .detail("Result", r.present() ? static_cast<int>(r.get()) : -1); // -1 means success
@@ -1906,7 +1902,7 @@ ACTOR static Future<Optional<std::string>> coordinatorsCommitActor(ReadYourWrite
 		}
 		msg = ManagementAPIError::toJsonString(retriable, "coordinators", ManagementAPI::generateErrorMessage(res));
 	}
-	return msg;
+	co_return msg;
 }
 
 Future<Optional<std::string>> CoordinatorsImpl::commit(ReadYourWritesTransaction* ryw) {
@@ -1924,31 +1920,28 @@ void CoordinatorsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key)
 
 CoordinatorsAutoImpl::CoordinatorsAutoImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
 
-ACTOR static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
-	state RangeResult res;
-	state std::string autoCoordinatorsKey;
-	state Transaction& tr = ryw->getTransaction();
+static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+	RangeResult res;
+	std::string autoCoordinatorsKey;
+	Transaction& tr = ryw->getTransaction();
 
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 	tr.setOption(FDBTransactionOptions::RAW_ACCESS);
 	tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	Optional<Value> currentKey = wait(tr.get(coordinatorsKey));
+	Optional<Value> currentKey = co_await tr.get(coordinatorsKey);
 
 	if (!currentKey.present()) {
 		ryw->setSpecialKeySpaceErrorMsg(
 		    ManagementAPIError::toJsonString(false, "auto_coordinators", "The coordinator key does not exist"));
 		throw special_keys_api_failure();
 	}
-	state ClusterConnectionString old(currentKey.get().toString());
-	state CoordinatorsResult result = CoordinatorsResult::SUCCESS;
+	ClusterConnectionString old(currentKey.get().toString());
+	CoordinatorsResult result = CoordinatorsResult::SUCCESS;
 
-	std::vector<NetworkAddress> oldCoordinators = wait(old.tryResolveHostnames());
-	std::vector<NetworkAddress> _desiredCoordinators = wait(autoQuorumChange()->getDesiredCoordinators(
-	    &tr,
-	    oldCoordinators,
-	    Reference<ClusterConnectionMemoryRecord>(new ClusterConnectionMemoryRecord(old)),
-	    result));
+	std::vector<NetworkAddress> oldCoordinators = co_await old.tryResolveHostnames();
+	std::vector<NetworkAddress> _desiredCoordinators = co_await autoQuorumChange()->getDesiredCoordinators(
+	    &tr, oldCoordinators, makeReference<ClusterConnectionMemoryRecord>(old), result);
 
 	if (result == CoordinatorsResult::NOT_ENOUGH_MACHINES) {
 		// we could get not_enough_machines if we happen to see the database while the cluster controller is updating
@@ -1962,21 +1955,21 @@ ACTOR static Future<RangeResult> CoordinatorsAutoImplActor(ReadYourWritesTransac
 
 	if (result == CoordinatorsResult::SAME_NETWORK_ADDRESSES) {
 		for (const auto& host : old.hostnames) {
-			autoCoordinatorsKey += autoCoordinatorsKey.size() ? "," : "";
+			autoCoordinatorsKey += autoCoordinatorsKey.empty() ? "" : ",";
 			autoCoordinatorsKey += host.toString();
 		}
 		for (const auto& coord : old.coords) {
-			autoCoordinatorsKey += autoCoordinatorsKey.size() ? "," : "";
+			autoCoordinatorsKey += autoCoordinatorsKey.empty() ? "" : ",";
 			autoCoordinatorsKey += coord.toString();
 		}
 	} else {
 		for (const auto& address : _desiredCoordinators) {
-			autoCoordinatorsKey += autoCoordinatorsKey.size() ? "," : "";
+			autoCoordinatorsKey += autoCoordinatorsKey.empty() ? "" : ",";
 			autoCoordinatorsKey += address.toString();
 		}
 	}
 	res.push_back_deep(res.arena(), KeyValueRef(kr.begin, Value(autoCoordinatorsKey)));
-	return res;
+	co_return res;
 }
 
 Future<RangeResult> CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ryw,
@@ -1987,17 +1980,17 @@ Future<RangeResult> CoordinatorsAutoImpl::getRange(ReadYourWritesTransaction* ry
 	return CoordinatorsAutoImplActor(ryw, kr);
 }
 
-ACTOR static Future<RangeResult> getMinCommitVersionActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+static Future<RangeResult> getMinCommitVersionActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
-	Optional<Value> val = wait(ryw->getTransaction().get(minRequiredCommitVersionKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(minRequiredCommitVersionKey);
 	RangeResult result;
 	if (val.present()) {
 		Version minRequiredCommitVersion = BinaryReader::fromStringRef<Version>(val.get(), Unversioned());
 		ValueRef version(result.arena(), boost::lexical_cast<std::string>(minRequiredCommitVersion));
 		result.push_back_deep(result.arena(), KeyValueRef(kr.begin, version));
 	}
-	return result;
+	co_return result;
 }
 
 AdvanceVersionImpl::AdvanceVersionImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -2020,10 +2013,10 @@ Future<RangeResult> AdvanceVersionImpl::getRange(ReadYourWritesTransaction* ryw,
 	}
 }
 
-ACTOR static Future<Optional<std::string>> advanceVersionCommitActor(ReadYourWritesTransaction* ryw, Version v) {
-	Optional<Standalone<StringRef>> versionEpochValue = wait(ryw->getTransaction().get(versionEpochKey));
+static Future<Optional<std::string>> advanceVersionCommitActor(ReadYourWritesTransaction* ryw, Version v) {
+	Optional<Standalone<StringRef>> versionEpochValue = co_await ryw->getTransaction().get(versionEpochKey);
 	if (versionEpochValue.present()) {
-		return ManagementAPIError::toJsonString(
+		co_return ManagementAPIError::toJsonString(
 		    false, "advanceversion", "Illegal to modify the version while the version epoch is enabled");
 	}
 
@@ -2036,19 +2029,19 @@ ACTOR static Future<Optional<std::string>> advanceVersionCommitActor(ReadYourWri
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 	TraceEvent(SevDebug, "AdvanceVersion").detail("MaxAllowedVersion", maxAllowedVerion);
 	if (v > maxAllowedVerion) {
-		return ManagementAPIError::toJsonString(
+		co_return ManagementAPIError::toJsonString(
 		    false,
 		    "advanceversion",
 		    "The given version is larger than the maximum allowed value(2**63-1-version_per_second*3600*24*365*1000)");
 	}
-	Version rv = wait(ryw->getTransaction().getReadVersion());
+	Version rv = co_await ryw->getTransaction().getReadVersion();
 	if (rv <= v) {
 		ryw->getTransaction().set(minRequiredCommitVersionKey, BinaryWriter::toValue(v + 1, Unversioned()));
 	} else {
-		return ManagementAPIError::toJsonString(
+		co_return ManagementAPIError::toJsonString(
 		    false, "advanceversion", "Current read version is larger than the given version");
 	}
-	return Optional<std::string>();
+	co_return Optional<std::string>();
 }
 
 Future<Optional<std::string>> AdvanceVersionImpl::commit(ReadYourWritesTransaction* ryw) {
@@ -2070,17 +2063,17 @@ Future<Optional<std::string>> AdvanceVersionImpl::commit(ReadYourWritesTransacti
 	return Optional<std::string>();
 }
 
-ACTOR static Future<RangeResult> getVersionEpochActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+static Future<RangeResult> getVersionEpochActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
-	Optional<Value> val = wait(ryw->getTransaction().get(versionEpochKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(versionEpochKey);
 	RangeResult result;
 	if (val.present()) {
 		int64_t versionEpoch = BinaryReader::fromStringRef<int64_t>(val.get(), Unversioned());
 		ValueRef version(result.arena(), boost::lexical_cast<std::string>(versionEpoch));
 		result.push_back_deep(result.arena(), KeyValueRef(kr.begin, version));
 	}
-	return result;
+	co_return result;
 }
 
 VersionEpochImpl::VersionEpochImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -2312,23 +2305,21 @@ void parse(std::vector<StringRef>::iterator it, std::vector<StringRef>::iterator
 	}
 }
 
-ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransaction* ryw,
-                                                           KeyRef prefix,
-                                                           KeyRangeRef kr) {
-	state RangeResult result;
+static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
+	RangeResult result;
 
 	// Set default values for all fields. The default will be used if the field
 	// is missing in the key.
-	state NetworkAddress host;
-	state WaitState waitStateStart = WaitState{ 0 };
-	state WaitState waitStateEnd = WaitState{ 2 };
-	state time_t timeStart = 0;
-	state time_t timeEnd = std::numeric_limits<time_t>::max();
-	state int seqStart = 0;
-	state int seqEnd = std::numeric_limits<int>::max();
-
-	state std::vector<StringRef> beginValues = kr.begin.removePrefix(prefix).splitAny("/"_sr);
-	state std::vector<StringRef> endValues = kr.end.removePrefix(prefix).splitAny("/"_sr);
+	NetworkAddress host;
+	WaitState waitStateStart = WaitState{ 0 };
+	WaitState waitStateEnd = WaitState{ 2 };
+	time_t timeStart = 0;
+	time_t timeEnd = std::numeric_limits<time_t>::max();
+	int seqStart = 0;
+	int seqEnd = std::numeric_limits<int>::max();
+
+	std::vector<StringRef> beginValues = kr.begin.removePrefix(prefix).splitAny("/"_sr);
+	std::vector<StringRef> endValues = kr.end.removePrefix(prefix).splitAny("/"_sr);
 	// Require index (either "state" or "time") and address:port.
 	if (beginValues.size() < 2 || endValues.size() < 2) {
 		ryw->setSpecialKeySpaceErrorMsg(
@@ -2336,7 +2327,7 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 		throw special_keys_api_failure();
 	}
 
-	state NetworkAddress endRangeHost;
+	NetworkAddress endRangeHost;
 	try {
 		if (SpecialKeySpace::getActorLineageApiCommandRange("state").contains(kr)) {
 			// For the range \xff\xff/actor_lineage/state/ip:port/wait-state/time/seq
@@ -2375,9 +2366,9 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 
 	// Open endpoint to target process on each call. This can be optimized at
 	// some point...
-	state ProcessInterface process;
+	ProcessInterface process;
 	process.getInterface = RequestStream<GetProcessInterfaceRequest>(Endpoint::wellKnown({ host }, WLTOKEN_PROCESS));
-	ProcessInterface p = wait(retryBrokenPromise(process.getInterface, GetProcessInterfaceRequest{}));
+	ProcessInterface p = co_await retryBrokenPromise(process.getInterface, GetProcessInterfaceRequest{});
 	process = p;
 
 	ActorLineageRequest actorLineageRequest;
@@ -2385,7 +2376,7 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 	actorLineageRequest.waitStateEnd = waitStateEnd;
 	actorLineageRequest.timeStart = timeStart;
 	actorLineageRequest.timeEnd = timeEnd;
-	ActorLineageReply reply = wait(process.actorLineage.getReply(actorLineageRequest));
+	ActorLineageReply reply = co_await process.actorLineage.getReply(actorLineageRequest);
 
 	time_t dt = 0;
 	int seq = -1;
@@ -2428,7 +2419,7 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 			result.push_back_deep(result.arena(), KeyValueRef(streamKey.str(), stream.str()));
 		}
 
-		if (sample.data.size() == 0) {
+		if (sample.data.empty()) {
 			std::ostringstream streamKey;
 			if (SpecialKeySpace::getActorLineageApiCommandRange("state").contains(kr)) {
 				streamKey << SpecialKeySpace::getActorLineageApiCommandPrefix("state").toString() << host.toString()
@@ -2444,7 +2435,7 @@ ACTOR static Future<RangeResult> actorLineageGetRangeActor(ReadYourWritesTransac
 		}
 	}
 
-	return result;
+	co_return result;
 }
 
 Future<RangeResult> ActorLineageImpl::getRange(ReadYourWritesTransaction* ryw,
@@ -2527,14 +2518,12 @@ MaintenanceImpl::MaintenanceImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 // we will calculate the remaining time(truncated to integer, the same as fdbcli) and return back as the value
 // If the zoneId is the special one `ignoreSSFailuresZoneString`,
 // value will be 0 (same as fdbcli)
-ACTOR static Future<RangeResult> MaintenanceGetRangeActor(ReadYourWritesTransaction* ryw,
-                                                          KeyRef prefix,
-                                                          KeyRangeRef kr) {
-	state RangeResult result;
+static Future<RangeResult> MaintenanceGetRangeActor(ReadYourWritesTransaction* ryw, KeyRef prefix, KeyRangeRef kr) {
+	RangeResult result;
 	// zoneId
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
-	Optional<Value> val = wait(ryw->getTransaction().get(healthyZoneKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(healthyZoneKey);
 	if (val.present()) {
 		auto healthyZone = decodeHealthyZoneValue(val.get());
 		if ((healthyZone.first == ignoreSSFailuresZoneString) ||
@@ -2550,7 +2539,7 @@ ACTOR static Future<RangeResult> MaintenanceGetRangeActor(ReadYourWritesTransact
 			}
 		}
 	}
-	return rywGetRange(ryw, kr, result);
+	co_return rywGetRange(ryw, kr, result);
 }
 
 Future<RangeResult> MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw,
@@ -2564,16 +2553,16 @@ Future<RangeResult> MaintenanceImpl::getRange(ReadYourWritesTransaction* ryw,
 // In addition, if the zoneId now is 'ignoreSSFailuresZoneString',
 // which means the data distribution is disabled for storage failures.
 // Only clear this specific key is allowed, any other operations will throw error
-ACTOR static Future<Optional<std::string>> maintenanceCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+static Future<Optional<std::string>> maintenanceCommitActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
 	// read
 	ryw->getTransaction().setOption(FDBTransactionOptions::LOCK_AWARE);
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 	ryw->getTransaction().setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	Optional<Value> val = wait(ryw->getTransaction().get(healthyZoneKey));
+	Optional<Value> val = co_await ryw->getTransaction().get(healthyZoneKey);
 	Optional<std::pair<Key, Version>> healthyZone =
 	    val.present() ? decodeHealthyZoneValue(val.get()) : Optional<std::pair<Key, Version>>();
 
-	state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
+	RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
 	    ryw->getSpecialKeySpaceWriteMap().containedRanges(kr);
 	Key zoneId;
 	double seconds;
@@ -2586,7 +2575,7 @@ ACTOR static Future<Optional<std::string>> maintenanceCommitActor(ReadYourWrites
 			continue;
 		if (iter->value().second.present()) {
 			if (isSet)
-				return Optional<std::string>(ManagementAPIError::toJsonString(
+				co_return Optional<std::string>(ManagementAPIError::toJsonString(
 				    false, "maintenance", "Multiple zones given for maintenance, only one allowed at the same time"));
 			isSet = true;
 			zoneId = iter->begin().removePrefix(kr.begin);
@@ -2602,11 +2591,11 @@ ACTOR static Future<Optional<std::string>> maintenanceCommitActor(ReadYourWrites
 		if (healthyZone.present() && healthyZone.get().first == ignoreSSFailuresZoneString) {
 			std::string msg = "Maintenance mode cannot be used while data distribution is disabled for storage "
 			                  "server failures.";
-			return Optional<std::string>(ManagementAPIError::toJsonString(false, "maintenance", msg));
+			co_return Optional<std::string>(ManagementAPIError::toJsonString(false, "maintenance", msg));
 		} else if (seconds < 0) {
 			std::string msg =
 			    "The specified maintenance time " + boost::lexical_cast<std::string>(seconds) + " is a negative value";
-			return Optional<std::string>(ManagementAPIError::toJsonString(false, "maintenance", msg));
+			co_return Optional<std::string>(ManagementAPIError::toJsonString(false, "maintenance", msg));
 		} else {
 			TraceEvent(SevDebug, "SKSMaintenanceSet").detail("ZoneId", zoneId.toString());
 			ryw->getTransaction().set(healthyZoneKey,
@@ -2615,7 +2604,7 @@ ACTOR static Future<Optional<std::string>> maintenanceCommitActor(ReadYourWrites
 			                                               (seconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)));
 		}
 	}
-	return Optional<std::string>();
+	co_return Optional<std::string>();
 }
 
 Future<Optional<std::string>> MaintenanceImpl::commit(ReadYourWritesTransaction* ryw) {
@@ -2625,19 +2614,19 @@ Future<Optional<std::string>> MaintenanceImpl::commit(ReadYourWritesTransaction*
 DataDistributionImpl::DataDistributionImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
 
 // Read the system keys dataDistributionModeKey and rebalanceDDIgnoreKey
-ACTOR static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTransaction* ryw,
-                                                               KeyRef prefix,
-                                                               KeyRangeRef kr) {
-	state RangeResult result;
+static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTransaction* ryw,
+                                                         KeyRef prefix,
+                                                         KeyRangeRef kr) {
+	RangeResult result;
 	// dataDistributionModeKey
-	state Key modeKey = "mode"_sr.withPrefix(prefix);
+	Key modeKey = "mode"_sr.withPrefix(prefix);
 
 	ryw->getTransaction().setOption(FDBTransactionOptions::RAW_ACCESS);
 
 	if (kr.contains(modeKey)) {
 		auto entry = ryw->getSpecialKeySpaceWriteMap()[modeKey];
 		if (ryw->readYourWritesDisabled() || !entry.first) {
-			Optional<Value> f = wait(ryw->getTransaction().get(dataDistributionModeKey));
+			Optional<Value> f = co_await ryw->getTransaction().get(dataDistributionModeKey);
 			int mode = -1;
 			if (f.present()) {
 				mode = BinaryReader::fromStringRef<int>(f.get(), Unversioned());
@@ -2646,17 +2635,17 @@ ACTOR static Future<RangeResult> DataDistributionGetRangeActor(ReadYourWritesTra
 		}
 	}
 	// rebalanceDDIgnoreKey
-	state Key rebalanceIgnoredKey = "rebalance_ignored"_sr.withPrefix(prefix);
+	Key rebalanceIgnoredKey = "rebalance_ignored"_sr.withPrefix(prefix);
 	if (kr.contains(rebalanceIgnoredKey)) {
 		auto entry = ryw->getSpecialKeySpaceWriteMap()[rebalanceIgnoredKey];
 		if (ryw->readYourWritesDisabled() || !entry.first) {
-			Optional<Value> f = wait(ryw->getTransaction().get(rebalanceDDIgnoreKey));
+			Optional<Value> f = co_await ryw->getTransaction().get(rebalanceDDIgnoreKey);
 			if (f.present()) {
 				result.push_back_deep(result.arena(), KeyValueRef(rebalanceIgnoredKey, Value()));
 			}
 		}
 	}
-	return rywGetRange(ryw, kr, result);
+	co_return rywGetRange(ryw, kr, result);
 }
 
 Future<RangeResult> DataDistributionImpl::getRange(ReadYourWritesTransaction* ryw,
@@ -2809,11 +2798,11 @@ bool parseLocalitiesFromKeys(ReadYourWritesTransaction* ryw,
 
 // On commit, parses the special exclusion keys and get the localities to be excluded, check for exclusions
 // and add them to the exclusion list. Also, clears the special management api keys with includeLocalities.
-ACTOR Future<Optional<std::string>> excludeLocalityCommitActor(ReadYourWritesTransaction* ryw, bool failed) {
-	state Optional<std::string> result;
-	state std::unordered_set<std::string> localities;
-	state std::vector<AddressExclusion> addresses;
-	state std::set<AddressExclusion> exclusions;
+Future<Optional<std::string>> excludeLocalityCommitActor(ReadYourWritesTransaction* ryw, bool failed) {
+	Optional<std::string> result;
+	std::unordered_set<std::string> localities;
+	std::vector<AddressExclusion> addresses;
+	std::set<AddressExclusion> exclusions;
 
 	ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
 	ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -2824,21 +2813,21 @@ ACTOR Future<Optional<std::string>> excludeLocalityCommitActor(ReadYourWritesTra
 	// If we fetch the workers here and in the status, we are bascially fetching the same information twice
 	// which is fine under normal circumstances, but there is no need to do that.
 	if (!parseLocalitiesFromKeys(ryw, failed, localities, result))
-		return result;
+		co_return result;
 	// If force option is not set, we need to do safety check
 	auto force = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
 	    failed ? "failed_locality" : "excluded_locality", "force")];
 	// only do safety check when we have localities to be excluded and the force option key is not set
-	if (localities.size() && !(force.first && force.second.present())) {
-		bool safe = wait(checkExclusion(ryw->getDatabase(), addresses, exclusions, localities, failed, &result));
+	if (!localities.empty() && !(force.first && force.second.present())) {
+		bool safe = co_await checkExclusion(ryw->getDatabase(), addresses, exclusions, localities, failed, &result);
 		if (!safe)
-			return result;
+			co_return result;
 	}
 
-	wait(excludeLocalities(&ryw->getTransaction(), localities, failed));
+	co_await excludeLocalities(&ryw->getTransaction(), localities, failed);
 	includeLocalities(ryw);
 
-	return result;
+	co_return result;
 }
 
 ExcludedLocalitiesRangeImpl::ExcludedLocalitiesRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@@ -2899,37 +2888,34 @@ Future<Optional<std::string>> FailedLocalitiesRangeImpl::commit(ReadYourWritesTr
 	return excludeLocalityCommitActor(ryw, true);
 }
 
-// Defined in ReadYourWrites.actor.cpp
-ACTOR Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
 // Defined in NativeAPI.actor.cpp
-ACTOR Future<bool> verifyInterfaceActor(Reference<FlowLock> connectLock, ClientWorkerInterface workerInterf);
+Future<bool> verifyInterfaceActor(Reference<FlowLock> const& connectLock, ClientWorkerInterface const& workerInterf);
 
-ACTOR static Future<RangeResult> workerInterfacesImplGetRangeActor(ReadYourWritesTransaction* ryw,
-                                                                   KeyRef prefix,
-                                                                   KeyRangeRef kr) {
+static Future<RangeResult> workerInterfacesImplGetRangeActor(ReadYourWritesTransaction* ryw,
+                                                             KeyRef prefix,
+                                                             KeyRangeRef kr) {
 	if (!ryw->getDatabase().getPtr() || !ryw->getDatabase()->getConnectionRecord())
-		return RangeResult();
+		co_return RangeResult();
 
-	state RangeResult interfs = wait(getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord()));
+	RangeResult interfs = co_await getWorkerInterfaces(ryw->getDatabase()->getConnectionRecord());
 	// for options' special keys, the boolean flag indicates if it's a SET operation
 	auto [verify, _] = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
 	    "worker_interfaces", "verify")];
-	state RangeResult result;
+	RangeResult result;
 	if (verify) {
 		// if verify option is set, we try to talk to every worker and only returns those we can talk to
 		Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
-		state std::vector<Future<bool>> verifyInterfs;
+		std::vector<Future<bool>> verifyInterfs;
 		for (const auto& [k_, value] : interfs) {
 			auto k = k_.withPrefix(prefix);
 			if (kr.contains(k)) {
-				ClientWorkerInterface workerInterf =
-				    BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion());
+				auto workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion());
 				verifyInterfs.push_back(verifyInterfaceActor(connectLock, workerInterf));
 			} else {
 				verifyInterfs.push_back(false);
 			}
 		}
-		wait(waitForAll(verifyInterfs));
+		co_await waitForAll(verifyInterfs);
 		// state int index;
 		for (int index = 0; index < interfs.size(); index++) {
 			if (verifyInterfs[index].get()) {
@@ -2946,7 +2932,7 @@ ACTOR static Future<RangeResult> workerInterfacesImplGetRangeActor(ReadYourWrite
 		}
 	}
 	std::sort(result.begin(), result.end(), KeyValueRef::OrderByKey{});
-	return result;
+	co_return result;
 }
 
 WorkerInterfacesSpecialKeyImpl::WorkerInterfacesSpecialKeyImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
@@ -2957,17 +2943,15 @@ Future<RangeResult> WorkerInterfacesSpecialKeyImpl::getRange(ReadYourWritesTrans
 	return workerInterfacesImplGetRangeActor(ryw, getKeyRange().begin, kr);
 }
 
-ACTOR Future<Optional<Value>> getJSON(Database db, std::string jsonField = "");
-
-ACTOR static Future<RangeResult> FaultToleranceMetricsImplActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
-	state RangeResult res;
+static Future<RangeResult> FaultToleranceMetricsImplActor(ReadYourWritesTransaction* ryw, KeyRangeRef kr) {
+	RangeResult res;
 	if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionRecord()) {
-		Optional<Value> val = wait(getJSON(ryw->getDatabase(), "fault_tolerance"));
+		Optional<Value> val = co_await getJSON(ryw->getDatabase(), "fault_tolerance");
 		if (val.present()) {
 			res.push_back_deep(res.arena(), KeyValueRef(kr.begin, val.get()));
 		}
 	}
-	return res;
+	co_return res;
 }
 
 FaultToleranceMetricsImpl::FaultToleranceMetricsImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
@@ -2980,16 +2964,16 @@ Future<RangeResult> FaultToleranceMetricsImpl::getRange(ReadYourWritesTransactio
 	return FaultToleranceMetricsImplActor(ryw, kr);
 }
 
-ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
-                                               KeySelector begin,
-                                               KeySelector end,
-                                               GetRangeLimits limits,
-                                               Reverse reverse,
-                                               RangeResult result) {
-	if (!result.size()) {
-		RangeResult testResult = wait(ryw->getRange(begin, end, limits, Snapshot::True, reverse));
+Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
+                                         KeySelector begin,
+                                         KeySelector end,
+                                         GetRangeLimits limits,
+                                         Reverse reverse,
+                                         RangeResult result) {
+	if (result.empty()) {
+		RangeResult testResult = co_await ryw->getRange(begin, end, limits, Snapshot::True, reverse);
 		ASSERT(testResult == result);
-		return Void();
+		co_return;
 	}
 
 	if (reverse) {
@@ -3025,12 +3009,12 @@ ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
 	std::vector<Key> keys;
 	keys = { deterministicRandom()->randomChoice(candidateKeys), deterministicRandom()->randomChoice(candidateKeys) };
 	std::sort(keys.begin(), keys.end());
-	state KeySelector testBegin = firstGreaterOrEqual(keys[0]);
-	state KeySelector testEnd = firstGreaterOrEqual(keys[1]);
+	KeySelector testBegin = firstGreaterOrEqual(keys[0]);
+	KeySelector testEnd = firstGreaterOrEqual(keys[1]);
 
 	// Generate expected result. Linear time is ok here since we're in simulation, and there's a benefit to keeping this
 	// simple (as we're using it as an test oracle)
-	state RangeResult expectedResult;
+	RangeResult expectedResult;
 	// The reverse parameter should be the same as for the original read, so if
 	// reverse is true then the results are _already_ in reverse order.
 	for (const auto& kr : result) {
@@ -3040,7 +3024,7 @@ ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
 	}
 
 	// Test
-	RangeResult testResult = wait(ryw->getRange(testBegin, testEnd, limits, Snapshot::True, reverse));
+	RangeResult testResult = co_await ryw->getRange(testBegin, testEnd, limits, Snapshot::True, reverse);
 	if (testResult != expectedResult) {
 		fmt::print("Reverse: {}\n", static_cast<bool>(reverse));
 		fmt::print("Original range: [{}, {})\n", begin.toString(), end.toString());
@@ -3059,5 +3043,4 @@ ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
 		}
 		ASSERT(testResult == expectedResult);
 	}
-	return Void();
 }
diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.cpp
similarity index 80%
rename from fdbclient/StatusClient.actor.cpp
rename to fdbclient/StatusClient.cpp
index 74e4b269d2b..ce5df725ab6 100644
--- a/fdbclient/StatusClient.actor.cpp
+++ b/fdbclient/StatusClient.cpp
@@ -1,5 +1,5 @@
 /*
- * StatusClient.actor.cpp
+ * StatusClient.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,11 +26,10 @@
 #include "fdbclient/Status.h"
 #include "fdbclient/json_spirit/json_spirit_writer_template.h"
 #include "fdbclient/json_spirit/json_spirit_reader_template.h"
-#include "fdbrpc/genericactors.actor.h"
+#include "fdbrpc/genericactors.h"
+#include "flow/CoroUtils.h"
 #include <cstdint>
 
-#include "flow/actorcompiler.h" // has to be last include
-
 json_spirit::mValue readJSONStrictly(const std::string& s) {
 	json_spirit::mValue val;
 	std::string::const_iterator i = s.begin();
@@ -303,14 +302,14 @@ void JSONDoc::mergeValueInto(json_spirit::mValue& dst, const json_spirit::mValue
 
 // Check if a quorum of coordination servers is reachable
 // Will not throw, will just return non-present Optional if error
-ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<IClusterConnectionRecord> connRecord,
-                                                                     bool* quorum_reachable,
-                                                                     int* coordinatorsFaultTolerance) {
+AsyncResult<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<IClusterConnectionRecord> connRecord,
+                                                                    bool* quorum_reachable,
+                                                                    int* coordinatorsFaultTolerance) {
 	try {
-		state ClientCoordinators coord(connRecord);
-		state StatusObject statusObj;
+		ClientCoordinators coord(connRecord);
+		StatusObject statusObj;
 
-		state std::vector<Future<Optional<LeaderInfo>>> leaderServers;
+		std::vector<Future<Optional<LeaderInfo>>> leaderServers;
 		leaderServers.reserve(coord.clientLeaderServers.size());
 		for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
 			if (coord.clientLeaderServers[i].hostname.present()) {
@@ -325,7 +324,7 @@ ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<I
 			}
 		}
 
-		state std::vector<Future<ProtocolInfoReply>> coordProtocols;
+		std::vector<Future<ProtocolInfoReply>> coordProtocols;
 		coordProtocols.reserve(coord.clientLeaderServers.size());
 		for (int i = 0; i < coord.clientLeaderServers.size(); i++) {
 			if (coord.clientLeaderServers[i].hostname.present()) {
@@ -338,9 +337,9 @@ ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<I
 			}
 		}
 
-		wait(smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.5) &&
-		         smartQuorum(coordProtocols, coordProtocols.size() / 2 + 1, 1.5) ||
-		     delay(2.0));
+		co_await (smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.5) &&
+		              smartQuorum(coordProtocols, coordProtocols.size() / 2 + 1, 1.5) ||
+		          delay(2.0));
 
 		statusObj["quorum_reachable"] = *quorum_reachable =
 		    quorum(leaderServers, leaderServers.size() / 2 + 1).isReady();
@@ -368,24 +367,24 @@ ACTOR Future<Optional<StatusObject>> clientCoordinatorsStatusFetcher(Reference<I
 		statusObj["coordinators"] = coordsStatus;
 
 		*coordinatorsFaultTolerance = (leaderServers.size() - 1) / 2 - coordinatorsUnavailable;
-		return statusObj;
-	} catch (Error& e) {
+		co_return statusObj;
+	} catch (Error&) {
 		*quorum_reachable = false;
-		return Optional<StatusObject>();
+		co_return Optional<StatusObject>();
 	}
 }
 
 // Client section of the json output
 // Will NOT throw, errors will be put into messages array
-ACTOR Future<StatusObject> clientStatusFetcher(Reference<IClusterConnectionRecord> connRecord,
-                                               StatusArray* messages,
-                                               bool* quorum_reachable,
-                                               int* coordinatorsFaultTolerance) {
-	state StatusObject statusObj;
+AsyncResult<StatusObject> clientStatusFetcher(Reference<IClusterConnectionRecord> connRecord,
+                                              StatusArray* messages,
+                                              bool* quorum_reachable,
+                                              int* coordinatorsFaultTolerance) {
+	StatusObject statusObj;
 
-	state Optional<StatusObject> coordsStatusObj =
-	    wait(clientCoordinatorsStatusFetcher(connRecord, quorum_reachable, coordinatorsFaultTolerance));
-	state bool contentsUpToDate = wait(connRecord->upToDate());
+	Optional<StatusObject> coordsStatusObj =
+	    co_await clientCoordinatorsStatusFetcher(connRecord, quorum_reachable, coordinatorsFaultTolerance);
+	bool contentsUpToDate = co_await connRecord->upToDate();
 
 	if (coordsStatusObj.present()) {
 		statusObj["coordinators"] = coordsStatusObj.get();
@@ -402,7 +401,7 @@ ACTOR Future<StatusObject> clientStatusFetcher(Reference<IClusterConnectionRecor
 	statusObj["cluster_file"] = statusObjClusterFile;
 
 	if (!contentsUpToDate) {
-		ClusterConnectionString storedConnectionString = wait(connRecord->getStoredConnectionString());
+		ClusterConnectionString storedConnectionString = co_await connRecord->getStoredConnectionString();
 		std::string description = "Cluster file contents do not match current cluster connection string.";
 		description += "\nThe file contains the connection string: ";
 		description += storedConnectionString.toString().c_str();
@@ -416,50 +415,46 @@ ACTOR Future<StatusObject> clientStatusFetcher(Reference<IClusterConnectionRecor
 		messages->push_back(makeMessage(MessageType::INCORRECT_CLUSTER_FILE_CONTENTS, description.c_str()));
 	}
 
-	return statusObj;
+	co_return statusObj;
 }
 
 // Cluster section of json output
-ACTOR Future<Optional<StatusObject>> clusterStatusFetcher(ClusterInterface cI,
-                                                          StatusArray* messages,
-                                                          std::string statusField) {
-	state StatusRequest req(statusField);
-	state Future<Void> clusterTimeout = delay(30.0);
-	state Optional<StatusObject> oStatusObj;
-
-	wait(delay(0.0)); // make sure the cluster controller is marked as not failed
-
-	state Future<ErrorOr<StatusReply>> statusReply = cI.databaseStatus.tryGetReply(req);
-	loop {
-		choose {
-			when(ErrorOr<StatusReply> result = wait(statusReply)) {
-				if (result.isError()) {
-					if (result.getError().code() == error_code_request_maybe_delivered)
-						messages->push_back(makeMessage(MessageType::UNREACHABLE_CLUSTER_CONTROLLER,
-						                                ("Unable to communicate with the cluster controller at " +
-						                                 cI.address().toString() + " to get status.")
-						                                    .c_str()));
-					else if (result.getError().code() == error_code_server_overloaded)
-						messages->push_back(makeMessage(MessageType::SERVER_OVERLOADED,
-						                                "The cluster controller is currently processing too many "
-						                                "status requests and is unable to respond"));
-					else
-						messages->push_back(makeMessage(MessageType::STATUS_INCOMPLETE_ERROR,
-						                                "Cluster encountered an error fetching status."));
-				} else {
-					oStatusObj = result.get().statusObj;
-				}
-				break;
-			}
-			when(wait(clusterTimeout)) {
+AsyncResult<Optional<StatusObject>> clusterStatusFetcher(ClusterInterface cI,
+                                                         StatusArray* messages,
+                                                         std::string statusField) {
+	StatusRequest req(statusField);
+	Future<Void> clusterTimeout = delay(CLIENT_KNOBS->STATUS_TIMEOUT);
+	Optional<StatusObject> oStatusObj;
+
+	co_await delay(0.0); // make sure the cluster controller is marked as not failed
+
+	Future<ErrorOr<StatusReply>> statusReply = cI.databaseStatus.tryGetReply(req);
+	auto res = co_await race(statusReply, clusterTimeout);
+	if (res.index() == 0) {
+		ErrorOr<StatusReply> result = std::get<0>(std::move(res));
+		if (result.isError()) {
+			if (result.getError().code() == error_code_request_maybe_delivered)
+				messages->push_back(makeMessage(MessageType::UNREACHABLE_CLUSTER_CONTROLLER,
+				                                ("Unable to communicate with the cluster controller at " +
+				                                 cI.address().toString() + " to get status.")
+				                                    .c_str()));
+			else if (result.getError().code() == error_code_server_overloaded)
+				messages->push_back(makeMessage(MessageType::SERVER_OVERLOADED,
+				                                "The cluster controller is currently processing too many "
+				                                "status requests and is unable to respond"));
+			else
 				messages->push_back(
-				    makeMessage(MessageType::STATUS_INCOMPLETE_TIMEOUT, "Timed out fetching cluster status."));
-				break;
-			}
+				    makeMessage(MessageType::STATUS_INCOMPLETE_ERROR, "Cluster encountered an error fetching status."));
+		} else {
+			oStatusObj = result.get().statusObj;
 		}
+	} else if (res.index() == 1) {
+		messages->push_back(makeMessage(MessageType::STATUS_INCOMPLETE_TIMEOUT, "Timed out fetching cluster status."));
+	} else {
+		UNREACHABLE();
 	}
 
-	return oStatusObj;
+	co_return oStatusObj;
 }
 
 // Create and return a database_status section.
@@ -510,26 +505,25 @@ StatusObject getClientDatabaseStatus(StatusObjectReader client, StatusObjectRead
 	return databaseStatus;
 }
 
-ACTOR Future<StatusObject> statusFetcherImpl(Reference<IClusterConnectionRecord> connRecord,
-                                             Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface,
-                                             std::string statusField) {
+AsyncResult<StatusObject> statusFetcherImpl(Reference<IClusterConnectionRecord> connRecord,
+                                            Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface,
+                                            std::string statusField) {
 	if (!g_network)
 		throw network_not_setup();
 
-	state StatusObject statusObj;
-	state StatusObject statusObjClient;
-	state StatusArray clientMessages;
+	StatusObject statusObj;
+	StatusObject statusObjClient;
+	StatusArray clientMessages;
 
 	// This could be read from the JSON but doing so safely is ugly so using a real var.
-	state bool quorum_reachable = false;
-	state int coordinatorsFaultTolerance = 0;
+	bool quorum_reachable = false;
+	int coordinatorsFaultTolerance = 0;
 
 	try {
-		state int64_t clientTime = g_network->timer();
+		int64_t clientTime = g_network->timer();
 
-		StatusObject _statusObjClient =
-		    wait(clientStatusFetcher(connRecord, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance));
-		statusObjClient = _statusObjClient;
+		statusObjClient =
+		    co_await clientStatusFetcher(connRecord, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance);
 
 		if (clientTime != -1)
 			statusObjClient["timestamp"] = clientTime;
@@ -544,19 +538,17 @@ ACTOR Future<StatusObject> statusFetcherImpl(Reference<IClusterConnectionRecord>
 		// it.
 	}
 
-	state StatusObject statusObjCluster;
+	StatusObject statusObjCluster;
 
 	if (quorum_reachable) {
 		try {
-
-			state Future<Void> interfaceTimeout = delay(2.0);
-
-			loop {
+			Future<Void> interfaceTimeout = delay(2.0);
+			while (true) {
 				if (clusterInterface->get().present()) {
 					Optional<StatusObject> _statusObjCluster =
-					    wait(clusterStatusFetcher(clusterInterface->get().get(), &clientMessages, statusField));
+					    co_await clusterStatusFetcher(clusterInterface->get().get(), &clientMessages, statusField);
 					if (_statusObjCluster.present()) {
-						statusObjCluster = _statusObjCluster.get();
+						statusObjCluster = std::move(_statusObjCluster).get();
 						// TODO: this is a temporary fix, getting the number of available coordinators should move to
 						// the server side
 						if (statusObjCluster.count("fault_tolerance")) {
@@ -578,14 +570,12 @@ ACTOR Future<StatusObject> statusFetcherImpl(Reference<IClusterConnectionRecord>
 					// else clusterStatusFetcher added a message
 					break;
 				}
-				choose {
-					when(wait(clusterInterface->onChange())) {}
-					when(wait(interfaceTimeout)) {
-						clientMessages.push_back(makeMessage(MessageType::NO_CLUSTER_CONTROLLER,
-						                                     "Unable to locate a cluster controller within 2 seconds.  "
-						                                     "Check that there are server processes running."));
-						break;
-					}
+				auto res = co_await race(clusterInterface->onChange(), interfaceTimeout);
+				if (res.index() == 1) {
+					clientMessages.push_back(makeMessage(MessageType::NO_CLUSTER_CONTROLLER,
+					                                     "Unable to locate a cluster controller within 2 seconds.  "
+					                                     "Check that there are server processes running."));
+					break;
 				}
 			}
 
@@ -614,21 +604,21 @@ ACTOR Future<StatusObject> statusFetcherImpl(Reference<IClusterConnectionRecord>
 	if (layers_valid.is_null())
 		layers_valid = false;
 
-	return statusObj;
+	co_return statusObj;
 }
 
-ACTOR Future<Void> timeoutMonitorLeader(Database db) {
-	state Future<Void> leadMon = monitorLeader<ClusterInterface>(db->getConnectionRecord(), db->statusClusterInterface);
-	loop {
-		wait(delay(CLIENT_KNOBS->STATUS_IDLE_TIMEOUT + 0.00001 + db->lastStatusFetch - now()));
+Future<Void> timeoutMonitorLeader(Database db) {
+	Future<Void> leadMon = monitorLeader<ClusterInterface>(db->getConnectionRecord(), db->statusClusterInterface);
+	while (true) {
+		co_await delay(CLIENT_KNOBS->STATUS_IDLE_TIMEOUT + 0.00001 + db->lastStatusFetch - now());
 		if (now() - db->lastStatusFetch > CLIENT_KNOBS->STATUS_IDLE_TIMEOUT) {
 			db->statusClusterInterface = Reference<AsyncVar<Optional<ClusterInterface>>>();
-			return Void();
+			co_return;
 		}
 	}
 }
 
-Future<StatusObject> StatusClient::statusFetcher(Database db, std::string statusField) {
+AsyncResult<StatusObject> StatusClient::statusFetcher(Database db, std::string statusField) {
 	db->lastStatusFetch = now();
 	if (!db->statusClusterInterface) {
 		db->statusClusterInterface = makeReference<AsyncVar<Optional<ClusterInterface>>>();
diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp
index bc02abf772f..9fda4145ed8 100644
--- a/fdbclient/StorageServerInterface.cpp
+++ b/fdbclient/StorageServerInterface.cpp
@@ -29,6 +29,75 @@
 // Includes template specializations for all tss operations on storage server types.
 // New StorageServerInterface reply types must be added here or it won't compile.
 
+void StorageServerInterface::initEndpointsFromGetValue() {
+	getKey = PublicRequestStream<struct GetKeyRequest>(getValue.getEndpoint().getAdjustedEndpoint(1));
+	getKeyValues = PublicRequestStream<struct GetKeyValuesRequest>(getValue.getEndpoint().getAdjustedEndpoint(2));
+	getShardState = RequestStream<struct GetShardStateRequest>(getValue.getEndpoint().getAdjustedEndpoint(3));
+	waitMetrics = PublicRequestStream<struct WaitMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(4));
+	splitMetrics = RequestStream<struct SplitMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(5));
+	getStorageMetrics = RequestStream<struct GetStorageMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(6));
+	waitFailure = RequestStream<ReplyPromise<Void>>(getValue.getEndpoint().getAdjustedEndpoint(7));
+	getQueuingMetrics =
+	    RequestStream<struct StorageQueuingMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(8));
+	getKeyValueStoreType =
+	    RequestStream<ReplyPromise<KeyValueStoreType>>(getValue.getEndpoint().getAdjustedEndpoint(9));
+	watchValue = PublicRequestStream<struct WatchValueRequest>(getValue.getEndpoint().getAdjustedEndpoint(10));
+	getReadHotRanges = RequestStream<struct ReadHotSubRangeRequest>(getValue.getEndpoint().getAdjustedEndpoint(11));
+	getRangeSplitPoints = RequestStream<struct SplitRangeRequest>(getValue.getEndpoint().getAdjustedEndpoint(12));
+	getKeyValuesStream =
+	    PublicRequestStream<struct GetKeyValuesStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(13));
+	getMappedKeyValues =
+	    PublicRequestStream<struct GetMappedKeyValuesRequest>(getValue.getEndpoint().getAdjustedEndpoint(14));
+	changeFeedStream = RequestStream<struct ChangeFeedStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(15));
+	overlappingChangeFeeds =
+	    RequestStream<struct OverlappingChangeFeedsRequest>(getValue.getEndpoint().getAdjustedEndpoint(16));
+	changeFeedPop = RequestStream<struct ChangeFeedPopRequest>(getValue.getEndpoint().getAdjustedEndpoint(17));
+	changeFeedVersionUpdate =
+	    RequestStream<struct ChangeFeedVersionUpdateRequest>(getValue.getEndpoint().getAdjustedEndpoint(18));
+	checkpoint = RequestStream<struct GetCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(19));
+	fetchCheckpoint = RequestStream<struct FetchCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(20));
+	fetchCheckpointKeyValues =
+	    RequestStream<struct FetchCheckpointKeyValuesRequest>(getValue.getEndpoint().getAdjustedEndpoint(21));
+	updateCommitCostRequest =
+	    RequestStream<struct UpdateCommitCostRequest>(getValue.getEndpoint().getAdjustedEndpoint(22));
+	auditStorage = RequestStream<struct AuditStorageRequest>(getValue.getEndpoint().getAdjustedEndpoint(23));
+	getHotShards = RequestStream<struct GetHotShardsRequest>(getValue.getEndpoint().getAdjustedEndpoint(24));
+	getCheckSum = RequestStream<struct GetStorageCheckSumRequest>(getValue.getEndpoint().getAdjustedEndpoint(25));
+	bulkdump = RequestStream<struct BulkDumpRequest>(getValue.getEndpoint().getAdjustedEndpoint(26));
+}
+
+void StorageServerInterface::initEndpoints() {
+	std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
+	streams.push_back(getValue.getReceiver(TaskPriority::LoadBalancedEndpoint));
+	streams.push_back(getKey.getReceiver(TaskPriority::LoadBalancedEndpoint));
+	streams.push_back(getKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint));
+	streams.push_back(getShardState.getReceiver());
+	streams.push_back(waitMetrics.getReceiver());
+	streams.push_back(splitMetrics.getReceiver());
+	streams.push_back(getStorageMetrics.getReceiver());
+	streams.push_back(waitFailure.getReceiver());
+	streams.push_back(getQueuingMetrics.getReceiver());
+	streams.push_back(getKeyValueStoreType.getReceiver());
+	streams.push_back(watchValue.getReceiver());
+	streams.push_back(getReadHotRanges.getReceiver());
+	streams.push_back(getRangeSplitPoints.getReceiver());
+	streams.push_back(getKeyValuesStream.getReceiver(TaskPriority::LoadBalancedEndpoint));
+	streams.push_back(getMappedKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint));
+	streams.push_back(changeFeedStream.getReceiver());
+	streams.push_back(overlappingChangeFeeds.getReceiver());
+	streams.push_back(changeFeedPop.getReceiver());
+	streams.push_back(changeFeedVersionUpdate.getReceiver());
+	streams.push_back(checkpoint.getReceiver());
+	streams.push_back(fetchCheckpoint.getReceiver());
+	streams.push_back(fetchCheckpointKeyValues.getReceiver());
+	streams.push_back(updateCommitCostRequest.getReceiver());
+	streams.push_back(auditStorage.getReceiver());
+	streams.push_back(getHotShards.getReceiver());
+	streams.push_back(getCheckSum.getReceiver());
+	streams.push_back(bulkdump.getReceiver());
+	FlowTransport::transport().addEndpoints(streams);
+}
+
 // if size + hex of checksum is shorter than value, record that instead of actual value. break-even point is 12
 // characters
 std::string traceChecksumValue(const ValueRef& s) {
diff --git a/fdbclient/Subspace.cpp b/fdbclient/Subspace.cpp
index dfdb213e1a1..3ec4da6cd9e 100644
--- a/fdbclient/Subspace.cpp
+++ b/fdbclient/Subspace.cpp
@@ -38,7 +38,7 @@ Subspace::Subspace(StringRef const& rawPrefix) {
 	this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
 }
 
-Subspace::~Subspace() {}
+Subspace::~Subspace() = default;
 
 Key Subspace::key() const {
 	return Key(StringRef(rawPrefix.begin(), rawPrefix.size()), rawPrefix.arena());
@@ -87,4 +87,4 @@ Subspace Subspace::subspace(Tuple const& tuple) const {
 
 Subspace Subspace::get(Tuple const& tuple) const {
 	return subspace(tuple);
-}
\ No newline at end of file
+}
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index 8fbfce948f6..258209c8c3a 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -18,12 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/Arena.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/serialize.h"
 #include "flow/UnitTest.h"
 
@@ -66,13 +66,13 @@ const KeyRef keyServersKeyServersKey = keyServersKeyServersKeys.begin;
 const UID anonymousShardId = UID(0x666666, 0x88888888);
 const uint64_t emptyShardId = 0x2222222;
 
-const Key keyServersKey(const KeyRef& k) {
+Key keyServersKey(const KeyRef& k) {
 	return k.withPrefix(keyServersPrefix);
 }
-const KeyRef keyServersKey(const KeyRef& k, Arena& arena) {
+KeyRef keyServersKey(const KeyRef& k, Arena& arena) {
 	return k.withPrefix(keyServersPrefix, arena);
 }
-const Value keyServersValue(RangeResult result, const std::vector<UID>& src, const std::vector<UID>& dest) {
+Value keyServersValue(RangeResult result, const std::vector<UID>& src, const std::vector<UID>& dest) {
 	if (!CLIENT_KNOBS->TAG_ENCODE_KEY_SERVERS) {
 		BinaryWriter wr(IncludeVersion(ProtocolVersion::withKeyServerValue()));
 		wr << src << dest;
@@ -97,10 +97,7 @@ const Value keyServersValue(RangeResult result, const std::vector<UID>& src, con
 	return keyServersValue(srcTag, destTag);
 }
 
-const Value keyServersValue(const std::vector<UID>& src,
-                            const std::vector<UID>& dest,
-                            const UID& srcID,
-                            const UID& destID) {
+Value keyServersValue(const std::vector<UID>& src, const std::vector<UID>& dest, const UID& srcID, const UID& destID) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withShardEncodeLocationMetaData()));
 	if (dest.empty()) {
 		ASSERT(!destID.isValid());
@@ -111,7 +108,7 @@ const Value keyServersValue(const std::vector<UID>& src,
 	return wr.toValue();
 }
 
-const Value keyServersValue(const std::vector<Tag>& srcTag, const std::vector<Tag>& destTag) {
+Value keyServersValue(const std::vector<Tag>& srcTag, const std::vector<Tag>& destTag) {
 	// src and dest are expected to be sorted
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withKeyServerValueV2()));
 	wr << srcTag << destTag;
@@ -123,7 +120,7 @@ void decodeKeyServersValue(RangeResult result,
                            std::vector<UID>& src,
                            std::vector<UID>& dest,
                            bool missingIsError) {
-	if (value.size() == 0) {
+	if (value.empty()) {
 		src.clear();
 		dest.clear();
 		return;
@@ -186,7 +183,7 @@ void decodeKeyServersValue(RangeResult result,
 	srcID = UID();
 	destID = UID();
 
-	if (value.size() == 0) {
+	if (value.empty()) {
 		return;
 	}
 
@@ -217,7 +214,7 @@ void decodeKeyServersValue(std::map<Tag, UID> const& tag_uid,
 	static std::vector<Tag> srcTag, destTag;
 	src.clear();
 	dest.clear();
-	if (value.size() == 0) {
+	if (value.empty()) {
 		return;
 	}
 
@@ -275,7 +272,7 @@ void decodeKeyServersValue(std::map<Tag, UID> const& tag_uid,
 }
 
 bool isSystemKey(KeyRef key) {
-	return key.size() && key[0] == systemKeys.begin[0];
+	return !key.empty() && key[0] == systemKeys.begin[0];
 }
 
 const KeyRangeRef conflictingKeysRange =
@@ -291,7 +288,7 @@ const KeyRangeRef writeConflictRangeKeysRange = KeyRangeRef("\xff\xff/transactio
 
 const KeyRef accumulativeChecksumKey = "\xff\xff/accumulativeChecksum"_sr;
 
-const Value accumulativeChecksumValue(const AccumulativeChecksumState& acsState) {
+Value accumulativeChecksumValue(const AccumulativeChecksumState& acsState) {
 	return ObjectWriter::toValue(acsState, IncludeVersion());
 }
 
@@ -309,7 +306,7 @@ const KeyRef auditRangePrefix = auditRanges.begin;
 const KeyRangeRef auditServers = KeyRangeRef("\xff/auditServers/"_sr, "\xff/auditServers0"_sr);
 const KeyRef auditServerPrefix = auditServers.begin;
 
-const Key auditKey(const AuditType type, const UID& auditId) {
+Key auditKey(const AuditType type, const UID& auditId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditPrefix);
 	wr << static_cast<uint8_t>(type);
@@ -318,7 +315,7 @@ const Key auditKey(const AuditType type, const UID& auditId) {
 	return wr.toValue();
 }
 
-const KeyRange auditKeyRange(const AuditType type) {
+KeyRange auditKeyRange(const AuditType type) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditPrefix);
 	wr << static_cast<uint8_t>(type);
@@ -326,7 +323,7 @@ const KeyRange auditKeyRange(const AuditType type) {
 	return prefixRange(wr.toValue());
 }
 
-const Key auditRangeBasedProgressPrefixFor(const AuditType type, const UID& auditId) {
+Key auditRangeBasedProgressPrefixFor(const AuditType type, const UID& auditId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditRangePrefix);
 	wr << static_cast<uint8_t>(type);
@@ -336,7 +333,7 @@ const Key auditRangeBasedProgressPrefixFor(const AuditType type, const UID& audi
 	return wr.toValue();
 }
 
-const KeyRange auditRangeBasedProgressRangeFor(const AuditType type, const UID& auditId) {
+KeyRange auditRangeBasedProgressRangeFor(const AuditType type, const UID& auditId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditRangePrefix);
 	wr << static_cast<uint8_t>(type);
@@ -346,7 +343,7 @@ const KeyRange auditRangeBasedProgressRangeFor(const AuditType type, const UID&
 	return prefixRange(wr.toValue());
 }
 
-const KeyRange auditRangeBasedProgressRangeFor(const AuditType type) {
+KeyRange auditRangeBasedProgressRangeFor(const AuditType type) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditRangePrefix);
 	wr << static_cast<uint8_t>(type);
@@ -354,7 +351,7 @@ const KeyRange auditRangeBasedProgressRangeFor(const AuditType type) {
 	return prefixRange(wr.toValue());
 }
 
-const Key auditServerBasedProgressPrefixFor(const AuditType type, const UID& auditId, const UID& serverId) {
+Key auditServerBasedProgressPrefixFor(const AuditType type, const UID& auditId, const UID& serverId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditServerPrefix);
 	wr << static_cast<uint8_t>(type);
@@ -366,7 +363,7 @@ const Key auditServerBasedProgressPrefixFor(const AuditType type, const UID& aud
 	return wr.toValue();
 }
 
-const KeyRange auditServerBasedProgressRangeFor(const AuditType type, const UID& auditId) {
+KeyRange auditServerBasedProgressRangeFor(const AuditType type, const UID& auditId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditServerPrefix);
 	wr << static_cast<uint8_t>(type);
@@ -376,7 +373,7 @@ const KeyRange auditServerBasedProgressRangeFor(const AuditType type, const UID&
 	return prefixRange(wr.toValue());
 }
 
-const KeyRange auditServerBasedProgressRangeFor(const AuditType type) {
+KeyRange auditServerBasedProgressRangeFor(const AuditType type) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(auditServerPrefix);
 	wr << static_cast<uint8_t>(type);
@@ -384,7 +381,7 @@ const KeyRange auditServerBasedProgressRangeFor(const AuditType type) {
 	return prefixRange(wr.toValue());
 }
 
-const Value auditStorageStateValue(const AuditStorageState& auditStorageState) {
+Value auditStorageStateValue(const AuditStorageState& auditStorageState) {
 	return ObjectWriter::toValue(auditStorageState, IncludeVersion());
 }
 
@@ -397,14 +394,14 @@ AuditStorageState decodeAuditStorageState(const ValueRef& value) {
 
 const KeyRef checkpointPrefix = "\xff/checkpoint/"_sr;
 
-const Key checkpointKeyFor(UID checkpointID) {
+Key checkpointKeyFor(UID checkpointID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(checkpointPrefix);
 	wr << checkpointID;
 	return wr.toValue();
 }
 
-const Value checkpointValue(const CheckpointMetaData& checkpoint) {
+Value checkpointValue(const CheckpointMetaData& checkpoint) {
 	return ObjectWriter::toValue(checkpoint, IncludeVersion());
 }
 
@@ -424,14 +421,14 @@ CheckpointMetaData decodeCheckpointValue(const ValueRef& value) {
 
 // "\xff/dataMoves/[[UID]] := [[DataMoveMetaData]]"
 const KeyRangeRef dataMoveKeys("\xff/dataMoves/"_sr, "\xff/dataMoves0"_sr);
-const Key dataMoveKeyFor(UID dataMoveId) {
+Key dataMoveKeyFor(UID dataMoveId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(dataMoveKeys.begin);
 	wr << dataMoveId;
 	return wr.toValue();
 }
 
-const Value dataMoveValue(const DataMoveMetaData& dataMoveMetaData) {
+Value dataMoveValue(const DataMoveMetaData& dataMoveMetaData) {
 	return ObjectWriter::toValue(dataMoveMetaData, IncludeVersion());
 }
 
@@ -452,8 +449,8 @@ DataMoveMetaData decodeDataMoveValue(const ValueRef& value) {
 const KeyRangeRef ddStatsRange =
     KeyRangeRef("\xff\xff/metrics/data_distribution_stats/"_sr, "\xff\xff/metrics/data_distribution_stats/\xff\xff"_sr);
 
-const Value logsValue(const std::vector<std::pair<UID, NetworkAddress>>& logs,
-                      const std::vector<std::pair<UID, NetworkAddress>>& oldLogs) {
+Value logsValue(const std::vector<std::pair<UID, NetworkAddress>>& logs,
+                const std::vector<std::pair<UID, NetworkAddress>>& oldLogs) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withLogsValue()));
 	wr << logs;
 	wr << oldLogs;
@@ -475,11 +472,11 @@ const ValueRef serverKeysTrue = "1"_sr, // compatible with what was serverKeysTr
     serverKeysTrueEmptyRange = "3"_sr, // the server treats the range as empty.
     serverKeysFalse;
 
-const UID newDataMoveId(const uint64_t physicalShardId,
-                        AssignEmptyRange assignEmptyRange,
-                        const DataMoveType type,
-                        const DataMovementReason reason,
-                        UnassignShard unassignShard) {
+UID newDataMoveId(const uint64_t physicalShardId,
+                  AssignEmptyRange assignEmptyRange,
+                  const DataMoveType type,
+                  const DataMovementReason reason,
+                  UnassignShard unassignShard) {
 	uint64_t split = 0;
 	if (assignEmptyRange) {
 		split = emptyShardId;
@@ -499,7 +496,7 @@ const UID newDataMoveId(const uint64_t physicalShardId,
 	return UID(physicalShardId, split);
 }
 
-const Key serverKeysKey(UID serverID, const KeyRef& key) {
+Key serverKeysKey(UID serverID, const KeyRef& key) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverKeysPrefix);
 	wr << serverID;
@@ -507,7 +504,7 @@ const Key serverKeysKey(UID serverID, const KeyRef& key) {
 	wr.serializeBytes(key);
 	return wr.toValue();
 }
-const Key serverKeysPrefixFor(UID serverID) {
+Key serverKeysPrefixFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverKeysPrefix);
 	wr << serverID;
@@ -541,7 +538,7 @@ bool serverHasKey(ValueRef storedValue) {
 	return assigned;
 }
 
-const Value serverKeysValue(const UID& id) {
+Value serverKeysValue(const UID& id) {
 	if (!id.isValid()) {
 		return serverKeysFalse;
 	}
@@ -607,7 +604,7 @@ void decodeServerKeysValue(const ValueRef& value,
                            DataMovementReason& dataMoveReason) {
 	dataMoveType = DataMoveType::LOGICAL;
 	dataMoveReason = DataMovementReason::INVALID;
-	if (value.size() == 0) {
+	if (value.empty()) {
 		assigned = false;
 		emptyRange = false;
 		id = UID();
@@ -631,54 +628,11 @@ void decodeServerKeysValue(const ValueRef& value,
 	}
 }
 
-const KeyRef cacheKeysPrefix = "\xff\x02/cacheKeys/"_sr;
-
-const Key cacheKeysKey(uint16_t idx, const KeyRef& key) {
-	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(cacheKeysPrefix);
-	wr << idx;
-	wr.serializeBytes("/"_sr);
-	wr.serializeBytes(key);
-	return wr.toValue();
-}
-const Key cacheKeysPrefixFor(uint16_t idx) {
-	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(cacheKeysPrefix);
-	wr << idx;
-	wr.serializeBytes("/"_sr);
-	return wr.toValue();
-}
-uint16_t cacheKeysDecodeIndex(const KeyRef& key) {
-	uint16_t idx;
-	BinaryReader rd(key.removePrefix(cacheKeysPrefix), Unversioned());
-	rd >> idx;
-	return idx;
-}
-KeyRef cacheKeysDecodeKey(const KeyRef& key) {
-	return key.substr(cacheKeysPrefix.size() + sizeof(uint16_t) + 1);
-}
-
-const KeyRef cacheChangeKey = "\xff\x02/cacheChangeKey"_sr;
-const KeyRangeRef cacheChangeKeys("\xff\x02/cacheChangeKeys/"_sr, "\xff\x02/cacheChangeKeys0"_sr);
-const KeyRef cacheChangePrefix = cacheChangeKeys.begin;
-const Key cacheChangeKeyFor(uint16_t idx) {
-	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(cacheChangePrefix);
-	wr << idx;
-	return wr.toValue();
-}
-uint16_t cacheChangeKeyDecodeIndex(const KeyRef& key) {
-	uint16_t idx;
-	BinaryReader rd(key.removePrefix(cacheChangePrefix), Unversioned());
-	rd >> idx;
-	return idx;
-}
-
 const KeyRangeRef tssMappingKeys("\xff/tss/"_sr, "\xff/tss0"_sr);
 
 const KeyRangeRef tssQuarantineKeys("\xff/tssQ/"_sr, "\xff/tssQ0"_sr);
 
-const Key tssQuarantineKeyFor(UID serverID) {
+Key tssQuarantineKeyFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(tssQuarantineKeys.begin);
 	wr << serverID;
@@ -720,28 +674,28 @@ const KeyRef serverTagConflictPrefix = serverTagConflictKeys.begin;
 const KeyRangeRef serverTagHistoryKeys("\xff/serverTagHistory/"_sr, "\xff/serverTagHistory0"_sr);
 const KeyRef serverTagHistoryPrefix = serverTagHistoryKeys.begin;
 
-const Key serverTagKeyFor(UID serverID) {
+Key serverTagKeyFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverTagKeys.begin);
 	wr << serverID;
 	return wr.toValue();
 }
 
-const Key serverTagHistoryKeyFor(UID serverID) {
+Key serverTagHistoryKeyFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverTagHistoryKeys.begin);
 	wr << serverID;
 	return addVersionStampAtEnd(wr.toValue());
 }
 
-const KeyRange serverTagHistoryRangeFor(UID serverID) {
+KeyRange serverTagHistoryRangeFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverTagHistoryKeys.begin);
 	wr << serverID;
 	return prefixRange(wr.toValue());
 }
 
-const KeyRange serverTagHistoryRangeBefore(UID serverID, Version version) {
+KeyRange serverTagHistoryRangeBefore(UID serverID, Version version) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverTagHistoryKeys.begin);
 	wr << serverID;
@@ -754,7 +708,7 @@ const KeyRange serverTagHistoryRangeBefore(UID serverID, Version version) {
 	return KeyRangeRef(wr.toValue(), versionStr.withPrefix(wr.toValue()));
 }
 
-const Value serverTagValue(Tag tag) {
+Value serverTagValue(Tag tag) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withServerTagValue()));
 	wr << tag;
 	return wr.toValue();
@@ -782,7 +736,7 @@ Tag decodeServerTagValue(ValueRef const& value) {
 	return s;
 }
 
-const Key serverTagConflictKeyFor(Tag tag) {
+Key serverTagConflictKeyFor(Tag tag) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverTagConflictKeys.begin);
 	wr << tag;
@@ -792,14 +746,14 @@ const Key serverTagConflictKeyFor(Tag tag) {
 const KeyRangeRef tagLocalityListKeys("\xff/tagLocalityList/"_sr, "\xff/tagLocalityList0"_sr);
 const KeyRef tagLocalityListPrefix = tagLocalityListKeys.begin;
 
-const Key tagLocalityListKeyFor(Optional<Value> dcID) {
+Key tagLocalityListKeyFor(Optional<Value> dcID) {
 	BinaryWriter wr(AssumeVersion(currentProtocolVersion()));
 	wr.serializeBytes(tagLocalityListKeys.begin);
 	wr << dcID;
 	return wr.toValue();
 }
 
-const Value tagLocalityListValue(int8_t const& tagLocality) {
+Value tagLocalityListValue(int8_t const& tagLocality) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagLocalityListValue()));
 	wr << tagLocality;
 	return wr.toValue();
@@ -820,14 +774,14 @@ int8_t decodeTagLocalityListValue(ValueRef const& value) {
 const KeyRangeRef datacenterReplicasKeys("\xff\x02/datacenterReplicas/"_sr, "\xff\x02/datacenterReplicas0"_sr);
 const KeyRef datacenterReplicasPrefix = datacenterReplicasKeys.begin;
 
-const Key datacenterReplicasKeyFor(Optional<Value> dcID) {
+Key datacenterReplicasKeyFor(Optional<Value> dcID) {
 	BinaryWriter wr(AssumeVersion(currentProtocolVersion()));
 	wr.serializeBytes(datacenterReplicasKeys.begin);
 	wr << dcID;
 	return wr.toValue();
 }
 
-const Value datacenterReplicasValue(int const& replicas) {
+Value datacenterReplicasValue(int const& replicas) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withDatacenterReplicasValue()));
 	wr << replicas;
 	return wr.toValue();
@@ -848,12 +802,12 @@ int decodeDatacenterReplicasValue(ValueRef const& value) {
 //    "\xff\x02/tLogDatacenters/[[datacenterID]]"
 extern const KeyRangeRef tLogDatacentersKeys;
 extern const KeyRef tLogDatacentersPrefix;
-const Key tLogDatacentersKeyFor(Optional<Value> dcID);
+Key tLogDatacentersKeyFor(Optional<Value> dcID);
 
 const KeyRangeRef tLogDatacentersKeys("\xff\x02/tLogDatacenters/"_sr, "\xff\x02/tLogDatacenters0"_sr);
 const KeyRef tLogDatacentersPrefix = tLogDatacentersKeys.begin;
 
-const Key tLogDatacentersKeyFor(Optional<Value> dcID) {
+Key tLogDatacentersKeyFor(Optional<Value> dcID) {
 	BinaryWriter wr(AssumeVersion(currentProtocolVersion()));
 	wr.serializeBytes(tLogDatacentersKeys.begin);
 	wr << dcID;
@@ -872,14 +826,14 @@ const KeyRef primaryDatacenterKey = "\xff/primaryDatacenter"_sr;
 const KeyRangeRef serverListKeys("\xff/serverList/"_sr, "\xff/serverList0"_sr);
 const KeyRef serverListPrefix = serverListKeys.begin;
 
-const Key serverListKeyFor(UID serverID) {
+Key serverListKeyFor(UID serverID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(serverListKeys.begin);
 	wr << serverID;
 	return wr.toValue();
 }
 
-const Value serverListValue(StorageServerInterface const& server) {
+Value serverListValue(StorageServerInterface const& server) {
 	auto protocolVersion = currentProtocolVersion();
 	protocolVersion.addObjectSerializerFlag();
 	return ObjectWriter::toValue(server, IncludeVersion(protocolVersion));
@@ -928,14 +882,14 @@ const KeyRef processClassChangeKey = "\xff/processClassChanges"_sr;
 const KeyRef processClassVersionKey = "\xff/processClassChangesVersion"_sr;
 const ValueRef processClassVersionValue = "1"_sr;
 
-const Key processClassKeyFor(StringRef processID) {
+Key processClassKeyFor(StringRef processID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(processClassKeys.begin);
 	wr << processID;
 	return wr.toValue();
 }
 
-const Value processClassValue(ProcessClass const& processClass) {
+Value processClassValue(ProcessClass const& processClass) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withProcessClassValue()));
 	wr << processClass;
 	return wr.toValue();
@@ -966,6 +920,7 @@ const KeyRangeRef configKeys("\xff/conf/"_sr, "\xff/conf0"_sr);
 const KeyRef configKeysPrefix = configKeys.begin;
 
 const KeyRef backupWorkerEnabledKey("\xff/conf/backup_worker_enabled"_sr);
+const KeyRef rangeBackupWorkerEnabledKey("\xff/conf/range_backup_worker_enabled"_sr);
 const KeyRef perpetualStorageWiggleKey("\xff/conf/perpetual_storage_wiggle"_sr);
 const KeyRef perpetualStorageWiggleLocalityKey("\xff/conf/perpetual_storage_wiggle_locality"_sr);
 // The below two are there for compatible upgrade and downgrade. After 7.3, the perpetual wiggle related keys should use
@@ -1046,14 +1001,14 @@ const KeyRef globalConfigVersionKey = "\xff/globalConfig/v"_sr;
 const KeyRangeRef workerListKeys("\xff/worker/"_sr, "\xff/worker0"_sr);
 const KeyRef workerListPrefix = workerListKeys.begin;
 
-const Key workerListKeyFor(StringRef processID) {
+Key workerListKeyFor(StringRef processID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(workerListKeys.begin);
 	wr << processID;
 	return wr.toValue();
 }
 
-const Value workerListValue(ProcessData const& processData) {
+Value workerListValue(ProcessData const& processData) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withWorkerListValue()));
 	wr << processData;
 	return wr.toValue();
@@ -1077,16 +1032,15 @@ const KeyRangeRef backupProgressKeys("\xff\x02/backupProgress/"_sr, "\xff\x02/ba
 const KeyRef backupProgressPrefix = backupProgressKeys.begin;
 const KeyRef backupStartedKey = "\xff\x02/backupStarted"_sr;
 extern const KeyRef backupPausedKey = "\xff\x02/backupPaused"_sr;
-extern const KeyRef backupWorkerMaxNoopVersionKey = "\xff\x02/backupWorkerMaxNoopVersion"_sr;
 
-const Key backupProgressKeyFor(UID workerID) {
+Key backupProgressKeyFor(UID workerID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(backupProgressPrefix);
 	wr << workerID;
 	return wr.toValue();
 }
 
-const Value backupProgressValue(const WorkerBackupStatus& status) {
+Value backupProgressValue(const WorkerBackupStatus& status) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withBackupProgressValue()));
 	wr << status;
 	return wr.toValue();
@@ -1115,7 +1069,7 @@ Value encodeBackupStartedValue(const std::vector<std::pair<UID, Version>>& ids)
 std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& value) {
 	std::vector<std::pair<UID, Version>> ids;
 	BinaryReader reader(value, IncludeVersion());
-	if (value.size() > 0)
+	if (!value.empty())
 		reader >> ids;
 	return ids;
 }
@@ -1124,6 +1078,43 @@ bool mutationForKey(const MutationRef& m, const KeyRef& key) {
 	return isSingleKeyMutation((MutationRef::Type)m.type) && m.param1 == key;
 }
 
+// Backup keys related to Range Partitioned.
+const KeyRef backupRangePartitionedMapUploadedPrefix = "\xff\x02/backupRangePartitionedMapUploaded/"_sr;
+const KeyRangeRef backupRangePartitionedProgressKeys("\xff\x02/backupRangePartitionedProgress/"_sr,
+                                                     "\xff\x02/backupRangePartitionedProgress0"_sr);
+const KeyRef backupRangePartitionedProgressPrefix = backupRangePartitionedProgressKeys.begin;
+
+Key backupRangePartitionedMapUploadedKeyFor(Version v) {
+	return backupRangePartitionedMapUploadedPrefix.withSuffix(format("%lld", v));
+}
+
+Key backupRangePartitionedProgressKey(UID workerID) {
+	BinaryWriter wr(Unversioned());
+	wr.serializeBytes(backupRangePartitionedProgressPrefix);
+	wr << workerID;
+	return wr.toValue();
+}
+
+Value backupRangePartitionedProgressValue(const WorkerBackupStatus& status) {
+	BinaryWriter wr(IncludeVersion(ProtocolVersion::withBackupProgressValue()));
+	wr << status;
+	return wr.toValue();
+}
+
+UID decodeBackupRangePartitionedProgressKey(const KeyRef& key) {
+	UID serverID;
+	BinaryReader rd(key.removePrefix(backupRangePartitionedProgressPrefix), Unversioned());
+	rd >> serverID;
+	return serverID;
+}
+
+WorkerBackupStatus decodeBackupRangePartitionedProgressValue(const ValueRef& value) {
+	WorkerBackupStatus status;
+	BinaryReader reader(value, IncludeVersion());
+	reader >> status;
+	return status;
+}
+
 const KeyRef previousCoordinatorsKey = "\xff/previousCoordinators"_sr;
 const KeyRef coordinatorsKey = "\xff/coordinators"_sr;
 const KeyRef logsKey = "\xff/logs"_sr;
@@ -1157,7 +1148,7 @@ const KeyRef bulkLoadModeKey = "\xff/bulkLoadMode"_sr;
 const KeyRangeRef bulkLoadTaskKeys = KeyRangeRef("\xff/bulkLoadTask/"_sr, "\xff/bulkLoadTask0"_sr);
 const KeyRef bulkLoadTaskPrefix = bulkLoadTaskKeys.begin;
 
-const Value bulkLoadTaskStateValue(const BulkLoadTaskState& bulkLoadTaskState) {
+Value bulkLoadTaskStateValue(const BulkLoadTaskState& bulkLoadTaskState) {
 	return ObjectWriter::toValue(bulkLoadTaskState, IncludeVersion());
 }
 
@@ -1168,7 +1159,7 @@ BulkLoadTaskState decodeBulkLoadTaskState(const ValueRef& value) {
 	return bulkLoadTaskState;
 }
 
-const Value ssBulkLoadMetadataValue(const SSBulkLoadMetadata& ssBulkLoadMetadata) {
+Value ssBulkLoadMetadataValue(const SSBulkLoadMetadata& ssBulkLoadMetadata) {
 	return ObjectWriter::toValue(ssBulkLoadMetadata, IncludeVersion());
 }
 
@@ -1182,7 +1173,7 @@ SSBulkLoadMetadata decodeSSBulkLoadMetadata(const ValueRef& value) {
 const KeyRangeRef bulkLoadJobKeys = KeyRangeRef("\xff/bulkLoadJob/"_sr, "\xff/bulkLoadJob0"_sr);
 const KeyRef bulkLoadJobPrefix = bulkLoadJobKeys.begin;
 
-const Value bulkLoadJobValue(const BulkLoadJobState& bulkLoadJobState) {
+Value bulkLoadJobValue(const BulkLoadJobState& bulkLoadJobState) {
 	return ObjectWriter::toValue(bulkLoadJobState, IncludeVersion());
 }
 
@@ -1198,7 +1189,7 @@ const KeyRef bulkLoadJobHistoryPrefix = bulkLoadJobHistoryKeys.begin;
 
 // BulkLoad job with the same jobId can run for multiple times, we only keep the latest one
 // in the history.
-const Key bulkLoadJobHistoryKeyFor(const UID& jobId) {
+Key bulkLoadJobHistoryKeyFor(const UID& jobId) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(bulkLoadJobHistoryPrefix);
 	wr.serializeBytes(StringRef(jobId.toString()));
@@ -1210,7 +1201,7 @@ const KeyRef bulkDumpModeKey = "\xff/bulkDumpMode"_sr;
 const KeyRangeRef bulkDumpKeys = KeyRangeRef("\xff/bulkDump/"_sr, "\xff/bulkDump0"_sr);
 const KeyRef bulkDumpPrefix = bulkDumpKeys.begin;
 
-const Value bulkDumpStateValue(const BulkDumpState& bulkDumpState) {
+Value bulkDumpStateValue(const BulkDumpState& bulkDumpState) {
 	return ObjectWriter::toValue(bulkDumpState, IncludeVersion());
 }
 
@@ -1221,13 +1212,35 @@ BulkDumpState decodeBulkDumpState(const ValueRef& value) {
 	return bulkDumpState;
 }
 
+// BulkDump owner tracking - stored separately for backward compatibility
+const KeyRangeRef bulkDumpOwnerKeys = KeyRangeRef("\xff/bulkDumpOwner/"_sr, "\xff/bulkDumpOwner0"_sr);
+const KeyRef bulkDumpOwnerPrefix = bulkDumpOwnerKeys.begin;
+
+Key bulkDumpOwnerKeyFor(const UID& jobId) {
+	BinaryWriter wr(Unversioned());
+	wr.serializeBytes(bulkDumpOwnerPrefix);
+	wr << jobId;
+	return wr.toValue();
+}
+
+// BulkLoad owner tracking - stored separately for backward compatibility
+const KeyRangeRef bulkLoadOwnerKeys = KeyRangeRef("\xff/bulkLoadOwner/"_sr, "\xff/bulkLoadOwner0"_sr);
+const KeyRef bulkLoadOwnerPrefix = bulkLoadOwnerKeys.begin;
+
+Key bulkLoadOwnerKeyFor(const UID& jobId) {
+	BinaryWriter wr(Unversioned());
+	wr.serializeBytes(bulkLoadOwnerPrefix);
+	wr << jobId;
+	return wr.toValue();
+}
+
 // Range Lock
 const std::string rangeLockNameForBulkLoad = "BulkLoad";
 
 const KeyRangeRef rangeLockKeys = KeyRangeRef("\xff/rangeLock/"_sr, "\xff/rangeLock0"_sr);
 const KeyRef rangeLockPrefix = rangeLockKeys.begin;
 
-const Value rangeLockStateSetValue(const RangeLockStateSet& rangeLockStateSet) {
+Value rangeLockStateSetValue(const RangeLockStateSet& rangeLockStateSet) {
 	return ObjectWriter::toValue(rangeLockStateSet, IncludeVersion());
 }
 
@@ -1241,14 +1254,14 @@ RangeLockStateSet decodeRangeLockStateSet(const ValueRef& value) {
 const KeyRangeRef rangeLockOwnerKeys = KeyRangeRef("\xff/rangeLockOwner/"_sr, "\xff/rangeLockOwner0"_sr);
 const KeyRef rangeLockOwnerPrefix = rangeLockOwnerKeys.begin;
 
-const Key rangeLockOwnerKeyFor(const RangeLockOwnerName& ownerUniqueID) {
+Key rangeLockOwnerKeyFor(const RangeLockOwnerName& ownerUniqueID) {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(rangeLockOwnerPrefix);
 	wr.serializeBytes(StringRef(ownerUniqueID));
 	return wr.toValue();
 }
 
-const Value rangeLockOwnerValue(const RangeLockOwner& rangeLockOwner) {
+Value rangeLockOwnerValue(const RangeLockOwner& rangeLockOwner) {
 	return ObjectWriter::toValue(rangeLockOwner, IncludeVersion());
 }
 
@@ -1270,7 +1283,7 @@ const KeyRef tagThrottleCountKey = "\xff\x02/throttledTags/manualThrottleCount"_
 
 // Client status info prefix
 const KeyRangeRef fdbClientInfoPrefixRange("\xff\x02/fdbClientInfo/"_sr, "\xff\x02/fdbClientInfo0"_sr);
-// See remaining fields in GlobalConfig.actor.h
+// See remaining fields in GlobalConfig.h
 
 // ConsistencyCheck settings
 const KeyRef fdbShouldConsistencyCheckBeSuspended = "\xff\x02/ConsistencyCheck/Suspend"_sr;
@@ -1477,13 +1490,11 @@ const KeyRef mustContainSystemMutationsKey = "\xff/mustContainSystemMutations"_s
 
 const KeyRangeRef monitorConfKeys("\xff\x02/monitorConf/"_sr, "\xff\x02/monitorConf0"_sr);
 
-const KeyRef restoreRequestDoneKey = "\xff\x02/restoreRequestDone"_sr;
-
 const KeyRef healthyZoneKey = "\xff\x02/healthyZone"_sr;
 const StringRef ignoreSSFailuresZoneString = "IgnoreSSFailures"_sr;
 const KeyRef rebalanceDDIgnoreKey = "\xff\x02/rebalanceDDIgnored"_sr;
 
-const Value healthyZoneValue(StringRef const& zoneId, Version version) {
+Value healthyZoneValue(StringRef const& zoneId, Version version) {
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withHealthyZoneValue()));
 	wr << zoneId;
 	wr << version;
@@ -1586,11 +1597,13 @@ TEST_CASE("noSim/SystemData/compat/KeyServers") {
 	std::sort(src.begin(), src.end());
 	std::sort(dest.begin(), dest.end());
 	RangeResult idTag;
-	for (int i = 0; i < src.size(); ++i) {
-		idTag.push_back_deep(idTag.arena(), KeyValueRef(serverTagKeyFor(src[i]), serverTagValue(uid_tag[src[i]])));
+	for (const auto& srcServerId : src) {
+		idTag.push_back_deep(idTag.arena(),
+		                     KeyValueRef(serverTagKeyFor(srcServerId), serverTagValue(uid_tag[srcServerId])));
 	}
-	for (int i = 0; i < dest.size(); ++i) {
-		idTag.push_back_deep(idTag.arena(), KeyValueRef(serverTagKeyFor(dest[i]), serverTagValue(uid_tag[dest[i]])));
+	for (const auto& destServerId : dest) {
+		idTag.push_back_deep(idTag.arena(),
+		                     KeyValueRef(serverTagKeyFor(destServerId), serverTagValue(uid_tag[destServerId])));
 	}
 
 	auto decodeAndVerify =
diff --git a/fdbclient/TagThrottle.actor.cpp b/fdbclient/TagThrottle.cpp
similarity index 73%
rename from fdbclient/TagThrottle.actor.cpp
rename to fdbclient/TagThrottle.cpp
index 94dba48d66c..4fa35d53dce 100644
--- a/fdbclient/TagThrottle.actor.cpp
+++ b/fdbclient/TagThrottle.cpp
@@ -1,5 +1,5 @@
 /*
- * TagThrottle.actor.cpp
+ * TagThrottle.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbclient/CommitProxyInterface.h"
-#include "fdbclient/DatabaseContext.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbclient/Tuple.h"
-
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbclient/TagThrottle.h"
 
 double const ClientTagThrottleLimits::NO_EXPIRATION = std::numeric_limits<double>::max();
 
@@ -97,7 +92,7 @@ Key TagThrottleKey::toKey() const {
 
 	for (auto tag : tags) {
 		*(str++) = (uint8_t)tag.size();
-		if (tag.size() > 0) {
+		if (!tag.empty()) {
 			memcpy(str, tag.begin(), tag.size());
 			str += tag.size();
 		}
@@ -108,8 +103,8 @@ Key TagThrottleKey::toKey() const {
 
 TagThrottleKey TagThrottleKey::fromKey(const KeyRef& key) {
 	const uint8_t* str = key.substr(tagThrottleKeysPrefix.size()).begin();
-	TagThrottleType throttleType = TagThrottleType(*(str++));
-	TransactionPriority priority = TransactionPriority(*(str++));
+	auto throttleType = TagThrottleType(*(str++));
+	auto priority = TransactionPriority(*(str++));
 	TagSet tags;
 
 	while (str < key.end()) {
@@ -128,43 +123,6 @@ TagThrottleValue TagThrottleValue::fromValue(const ValueRef& value) {
 	return throttleValue;
 }
 
-KeyRangeRef const tagQuotaKeys = KeyRangeRef("\xff/tagQuota/"_sr, "\xff/tagQuota0"_sr);
-KeyRef const tagQuotaPrefix = tagQuotaKeys.begin;
-
-Key ThrottleApi::getTagQuotaKey(TransactionTagRef tag) {
-	return tag.withPrefix(tagQuotaPrefix);
-}
-
-bool ThrottleApi::TagQuotaValue::isValid() const {
-	return reservedQuota <= totalQuota && reservedQuota >= 0;
-}
-
-Value ThrottleApi::TagQuotaValue::toValue() const {
-	return Tuple::makeTuple(reservedQuota, totalQuota).pack();
-}
-
-ThrottleApi::TagQuotaValue ThrottleApi::TagQuotaValue::fromValue(ValueRef value) {
-	auto tuple = Tuple::unpack(value);
-	if (tuple.size() != 2) {
-		throw invalid_throttle_quota_value();
-	}
-	TagQuotaValue result;
-	try {
-		result.reservedQuota = tuple.getInt(0);
-		result.totalQuota = tuple.getInt(1);
-	} catch (Error& e) {
-		TraceEvent(SevWarnAlways, "TagQuotaValueFailedToDeserialize").error(e);
-		throw invalid_throttle_quota_value();
-	}
-	if (!result.isValid()) {
-		TraceEvent(SevWarnAlways, "TagQuotaValueInvalidQuotas")
-		    .detail("ReservedQuota", result.reservedQuota)
-		    .detail("TotalQuota", result.totalQuota);
-		throw invalid_throttle_quota_value();
-	}
-	return result;
-}
-
 TEST_CASE("TagSet/toString") {
 	{
 		TagSet tagSet;
diff --git a/fdbclient/TaskBucket.actor.cpp b/fdbclient/TaskBucket.cpp
similarity index 68%
rename from fdbclient/TaskBucket.actor.cpp
rename to fdbclient/TaskBucket.cpp
index ee30a45855c..bac6a8335b7 100644
--- a/fdbclient/TaskBucket.actor.cpp
+++ b/fdbclient/TaskBucket.cpp
@@ -1,5 +1,5 @@
 /*
- * TaskBucket.actor.cpp
+ * TaskBucket.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,7 +21,6 @@
 #include "fdbclient/TaskBucket.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 Reference<TaskFuture> Task::getDoneFuture(Reference<FutureBucket> fb) {
 	return fb->unpack(params[reservedTaskParamKeyDone]);
@@ -44,22 +43,20 @@ struct UnblockFutureTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		state Reference<TaskFuture> future = futureBucket->unpack(task->params[Task::reservedTaskParamKeyFuture]);
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		Reference<TaskFuture> future = futureBucket->unpack(task->params[Task::reservedTaskParamKeyFuture]);
 
 		futureBucket->setOptions(tr);
 
 		tr->clear(future->blocks.pack(task->params[Task::reservedTaskParamKeyBlockID]));
 
-		bool is_set = wait(future->isSet(tr));
+		bool is_set = co_await future->isSet(tr);
 		if (is_set) {
-			wait(future->performAllActions(tr, taskBucket));
+			co_await future->performAllActions(tr, taskBucket);
 		}
-
-		return Void();
 	}
 };
 StringRef UnblockFutureTaskFunc::name = "UnblockFuture"_sr;
@@ -122,13 +119,13 @@ Key Task::reservedTaskParamValidValue = "_validvalue"_sr;
 // be set for non-default constructor arguments.  To change this behavior look at all
 // Task() default constructions to see if they require params to be empty and call clear.
 Task::Task(Value type, uint32_t version, Value done, unsigned int priority) : extendMutex(1) {
-	if (type.size())
+	if (!type.empty())
 		params[Task::reservedTaskParamKeyType] = type;
 
 	if (version > 0)
 		params[Task::reservedTaskParamKeyVersion] = BinaryWriter::toValue(version, Unversioned());
 
-	if (done.size())
+	if (!done.empty())
 		params[Task::reservedTaskParamKeyDone] = done;
 
 	priority = std::min<int64_t>(priority, CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY);
@@ -159,36 +156,35 @@ unsigned int Task::getPriority() const {
 
 class TaskBucketImpl {
 public:
-	ACTOR static Future<Optional<Key>> getTaskKey(Reference<ReadYourWritesTransaction> tr,
-	                                              Reference<TaskBucket> taskBucket,
-	                                              int priority = 0) {
+	static Future<Optional<Key>> getTaskKey(Reference<ReadYourWritesTransaction> tr,
+	                                        Reference<TaskBucket> taskBucket,
+	                                        int priority = 0) {
 		Standalone<StringRef> uid = StringRef(deterministicRandom()->randomUniqueID().toString());
 
 		// Get keyspace for the specified priority level
-		state Subspace space = taskBucket->getAvailableSpace(priority);
+		Subspace space = taskBucket->getAvailableSpace(priority);
 		{
 			// Get a task key that is <= a random UID task key, if successful then return it
 			RangeResult value =
-			    wait(tr->getRange(KeyRangeRef(space.key(), space.pack(uid)), 1, Snapshot::True, Reverse::True));
+			    co_await tr->getRange(KeyRangeRef(space.key(), space.pack(uid)), 1, Snapshot::True, Reverse::True);
 			if (!value.empty()) {
-				return Optional<Key>(value[0].key);
+				co_return Optional<Key>(value[0].key);
 			}
 		}
 
 		{
 			// Get a task key that is <= the maximum possible UID, if successful return it.
-			RangeResult value =
-			    wait(tr->getRange(KeyRangeRef(space.key(), space.pack(maxUIDKey)), 1, Snapshot::True, Reverse::True));
+			RangeResult value = co_await tr->getRange(
+			    KeyRangeRef(space.key(), space.pack(maxUIDKey)), 1, Snapshot::True, Reverse::True);
 			if (!value.empty()) {
-				return Optional<Key>(value[0].key);
+				co_return Optional<Key>(value[0].key);
 			}
 		}
 
-		return Optional<Key>();
+		co_return Optional<Key>();
 	}
 
-	ACTOR static Future<Reference<Task>> getOne(Reference<ReadYourWritesTransaction> tr,
-	                                            Reference<TaskBucket> taskBucket) {
+	static Future<Reference<Task>> getOne(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
 		if (taskBucket->priority_batch)
 			tr->setOption(FDBTransactionOptions::PRIORITY_BATCH);
 
@@ -197,29 +193,28 @@ class TaskBucketImpl {
 		// give it some chances for the timed out tasks to get into the task loop in the case of
 		// many other new tasks get added so that the timed out tasks never get chances to re-run
 		if (deterministicRandom()->random01() < CLIENT_KNOBS->TASKBUCKET_CHECK_TIMEOUT_CHANCE) {
-			bool anyTimeouts = wait(requeueTimedOutTasks(tr, taskBucket));
+			bool anyTimeouts = co_await requeueTimedOutTasks(tr, taskBucket);
 			CODE_PROBE(anyTimeouts, "Found a task that timed out");
 		}
 
-		state std::vector<Future<Optional<Key>>> taskKeyFutures(CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY + 1);
+		std::vector<Future<Optional<Key>>> taskKeyFutures(CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY + 1);
 
 		// Start looking for a task at each priority, highest first
-		state int pri;
-		for (pri = CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY; pri >= 0; --pri)
+		for (int pri = CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY; pri >= 0; --pri)
 			taskKeyFutures[pri] = getTaskKey(tr, taskBucket, pri);
 
 		// Task key and subspace it is located in.
-		state Optional<Key> taskKey;
-		state Subspace availableSpace;
+		Optional<Key> taskKey;
+		Subspace availableSpace;
 
 		// In priority order from highest to lowest, wait for fetch to finish and if it found a task then cancel the
 		// rest.
-		for (pri = CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY; pri >= 0; --pri) {
+		for (int pri = CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY; pri >= 0; --pri) {
 			// If we already have a task key then cancel this fetch
 			if (taskKey.present())
 				taskKeyFutures[pri].cancel();
 			else {
-				Optional<Key> key = wait(taskKeyFutures[pri]);
+				Optional<Key> key = co_await taskKeyFutures[pri];
 				if (key.present()) {
 					taskKey = key;
 					availableSpace = taskBucket->getAvailableSpace(pri);
@@ -229,26 +224,26 @@ class TaskBucketImpl {
 
 		// If we don't have a task key, requeue timed out tasks and try again by calling self.
 		if (!taskKey.present()) {
-			bool anyTimeouts = wait(requeueTimedOutTasks(tr, taskBucket));
+			bool anyTimeouts = co_await requeueTimedOutTasks(tr, taskBucket);
 			// If there were timeouts, try to get a task since there should now be one in one of the available spaces.
 			if (anyTimeouts) {
 				CODE_PROBE(true, "Try to get one task from timeouts subspace");
-				Reference<Task> task = wait(getOne(tr, taskBucket));
-				return task;
+				Reference<Task> task = co_await getOne(tr, taskBucket);
+				co_return task;
 			}
-			return Reference<Task>();
+			co_return Reference<Task>();
 		}
 
 		// Now we know the task key is present and we have the available space for the task's priority
-		state Tuple t = availableSpace.unpack(taskKey.get());
-		state Key taskUID = t.getString(0);
-		state Subspace taskAvailableSpace = availableSpace.get(taskUID);
+		Tuple t = availableSpace.unpack(taskKey.get());
+		Key taskUID = t.getString(0);
+		Subspace taskAvailableSpace = availableSpace.get(taskUID);
 
-		state Reference<Task> task(new Task());
+		Reference<Task> task(new Task());
 		task->key = taskUID;
 
-		state RangeResult values = wait(tr->getRange(taskAvailableSpace.range(), CLIENT_KNOBS->TOO_MANY));
-		Version version = wait(tr->getReadVersion());
+		RangeResult values = co_await tr->getRange(taskAvailableSpace.range(), CLIENT_KNOBS->TOO_MANY);
+		Version version = co_await tr->getReadVersion();
 		task->timeoutVersion =
 		    version + (uint64_t)(taskBucket->timeout *
 		                         (CLIENT_KNOBS->TASKBUCKET_TIMEOUT_JITTER_OFFSET +
@@ -266,19 +261,19 @@ class TaskBucketImpl {
 		tr->clear(taskAvailableSpace.range());
 		tr->set(taskBucket->active.key(), deterministicRandom()->randomUniqueID().toString());
 
-		return task;
+		co_return task;
 	}
 
 	// Verify that the user configured task verification key still has the user specified value
-	ACTOR static Future<bool> taskVerify(Reference<TaskBucket> tb,
-	                                     Reference<ReadYourWritesTransaction> tr,
-	                                     Reference<Task> task) {
+	static Future<bool> taskVerify(Reference<TaskBucket> tb,
+	                               Reference<ReadYourWritesTransaction> tr,
+	                               Reference<Task> task) {
 
 		if (task->params.find(Task::reservedTaskParamValidKey) == task->params.end()) {
 			TraceEvent("TaskBucketTaskVerifyInvalidTask")
 			    .detail("Task", task->params[Task::reservedTaskParamKeyType])
 			    .detail("ReservedTaskParamValidKey", "missing");
-			return false;
+			co_return false;
 		}
 
 		if (task->params.find(Task::reservedTaskParamValidValue) == task->params.end()) {
@@ -286,12 +281,12 @@ class TaskBucketImpl {
 			    .detail("Task", task->params[Task::reservedTaskParamKeyType])
 			    .detail("ReservedTaskParamValidKey", task->params[Task::reservedTaskParamValidKey])
 			    .detail("ReservedTaskParamValidValue", "missing");
-			return false;
+			co_return false;
 		}
 
 		tb->setOptions(tr);
 
-		Optional<Value> keyValue = wait(tr->get(task->params[Task::reservedTaskParamValidKey]));
+		Optional<Value> keyValue = co_await tr->get(task->params[Task::reservedTaskParamValidKey]);
 
 		if (!keyValue.present()) {
 			TraceEvent("TaskBucketTaskVerifyInvalidTask")
@@ -299,7 +294,7 @@ class TaskBucketImpl {
 			    .detail("ReservedTaskParamValidKey", task->params[Task::reservedTaskParamValidKey])
 			    .detail("ReservedTaskParamValidValue", task->params[Task::reservedTaskParamValidValue])
 			    .detail("KeyValue", "missing");
-			return false;
+			co_return false;
 		}
 
 		if (keyValue.get().compare(StringRef(task->params[Task::reservedTaskParamValidValue]))) {
@@ -308,74 +303,70 @@ class TaskBucketImpl {
 			    .detail("ReservedTaskParamValidKey", task->params[Task::reservedTaskParamValidKey])
 			    .detail("ReservedTaskParamValidValue", task->params[Task::reservedTaskParamValidValue])
 			    .detail("KeyValue", keyValue.get());
-			return false;
+			co_return false;
 		}
 
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<bool> taskVerify(Reference<TaskBucket> tb, Database cx, Reference<Task> task) {
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-
+	static Future<bool> taskVerify(Reference<TaskBucket> tb, Database cx, Reference<Task> task) {
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+			Error err;
 			try {
-				bool verified = wait(taskVerify(tb, tr, task));
-				return verified;
+				bool verified = co_await taskVerify(tb, tr, task);
+				co_return verified;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> finishTaskRun(Reference<ReadYourWritesTransaction> tr,
-	                                        Reference<TaskBucket> taskBucket,
-	                                        Reference<FutureBucket> futureBucket,
-	                                        Reference<Task> task,
-	                                        Reference<TaskFuncBase> taskFunc,
-	                                        VerifyTask verifyTask) {
-		bool isFinished = wait(taskBucket->isFinished(tr, task));
+	static Future<Void> finishTaskRun(Reference<ReadYourWritesTransaction> tr,
+	                                  Reference<TaskBucket> taskBucket,
+	                                  Reference<FutureBucket> futureBucket,
+	                                  Reference<Task> task,
+	                                  Reference<TaskFuncBase> taskFunc,
+	                                  VerifyTask verifyTask) {
+		bool isFinished = co_await taskBucket->isFinished(tr, task);
 		if (isFinished) {
-			return Void();
+			co_return;
 		}
 
-		state bool validTask = true;
+		bool validTask = true;
 		if (verifyTask) {
-			bool _validTask = wait(taskVerify(taskBucket, tr, task));
+			bool _validTask = co_await taskVerify(taskBucket, tr, task);
 			validTask = _validTask;
 		}
 		if (!validTask) {
-			wait(taskBucket->finish(tr, task));
+			co_await taskBucket->finish(tr, task);
 		} else {
-			wait(taskFunc->finish(tr, taskBucket, futureBucket, task));
+			co_await taskFunc->finish(tr, taskBucket, futureBucket, task);
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<bool> doOne(Database cx,
-	                                Reference<TaskBucket> taskBucket,
-	                                Reference<FutureBucket> futureBucket) {
-		state Reference<Task> task = wait(taskBucket->getOne(cx));
-		bool result = wait(taskBucket->doTask(cx, futureBucket, task));
-		return result;
+	static Future<bool> doOne(Database cx, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket) {
+		Reference<Task> task = co_await taskBucket->getOne(cx);
+		bool result = co_await taskBucket->doTask(cx, futureBucket, task);
+		co_return result;
 	}
 
-	ACTOR static Future<Void> extendTimeoutRepeatedly(Database cx,
-	                                                  Reference<TaskBucket> taskBucket,
-	                                                  Reference<Task> task) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state double start = now();
-		state Version versionNow = wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
+	static Future<Void> extendTimeoutRepeatedly(Database cx, Reference<TaskBucket> taskBucket, Reference<Task> task) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		double start = now();
+		Version versionNow = co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			taskBucket->setOptions(tr);
 			return map(tr->getReadVersion(), [=](Version v) { return v; });
-		}));
+		});
 
-		loop {
-			state FlowLock::Releaser releaser;
+		while (true) {
+			FlowLock::Releaser releaser;
 
 			// Wait until we are half way to the timeout version of this task
-			wait(delay(0.8 * (BUGGIFY ? (2 * deterministicRandom()->random01()) : 1.0) *
-			           (double)(task->timeoutVersion - (uint64_t)versionNow) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
+			co_await delay(0.8 * (BUGGIFY ? (2 * deterministicRandom()->random01()) : 1.0) *
+			               (double)(task->timeoutVersion - (uint64_t)versionNow) /
+			               CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
 
 			if (now() - start > 300) {
 				TraceEvent(SevWarnAlways, "TaskBucketLongExtend")
@@ -385,122 +376,128 @@ class TaskBucketImpl {
 				    .detail("Priority", task->getPriority());
 			}
 			// Take the extendMutex lock until we either succeed or stop trying to extend due to failure
-			wait(task->extendMutex.take());
+			co_await task->extendMutex.take();
 			releaser = FlowLock::Releaser(task->extendMutex, 1);
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr->reset();
 					taskBucket->setOptions(tr);
 
 					// Attempt to extend the task's timeout
-					state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::False));
-					wait(tr->commit());
+					Version newTimeout = co_await taskBucket->extendTimeout(tr, task, UpdateParams::False);
+					co_await tr->commit();
 					task->timeoutVersion = newTimeout;
 					versionNow = tr->getCommittedVersion();
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 				}
+				co_await tr->onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<bool> doTask(Database cx,
-	                                 Reference<TaskBucket> taskBucket,
-	                                 Reference<FutureBucket> futureBucket,
-	                                 Reference<Task> task) {
-		state Reference<TaskFuncBase> taskFunc;
-		state VerifyTask verifyTask(false);
+	static Future<bool> doTask(Database cx,
+	                           Reference<TaskBucket> taskBucket,
+	                           Reference<FutureBucket> futureBucket,
+	                           Reference<Task> task) {
+		Reference<TaskFuncBase> taskFunc;
+		VerifyTask verifyTask(false);
 
 		if (!task || !TaskFuncBase::isValidTask(task))
-			return false;
+			co_return false;
 
+		Error err;
 		try {
 			taskFunc = TaskFuncBase::create(task->params[Task::reservedTaskParamKeyType]);
 			if (taskFunc) {
 				verifyTask.set(task->params.find(Task::reservedTaskParamValidKey) != task->params.end());
 
 				if (verifyTask) {
-					loop {
-						state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+					while (true) {
+						Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 						taskBucket->setOptions(tr);
-
+						Error innerErr;
 						try {
-							bool validTask = wait(taskVerify(taskBucket, tr, task));
+							bool validTask = co_await taskVerify(taskBucket, tr, task);
 
 							if (!validTask) {
-								bool isFinished = wait(taskBucket->isFinished(tr, task));
+								bool isFinished = co_await taskBucket->isFinished(tr, task);
 								if (!isFinished) {
-									wait(taskBucket->finish(tr, task));
+									co_await taskBucket->finish(tr, task);
 								}
-								wait(tr->commit());
-								return true;
+								co_await tr->commit();
+								co_return true;
 							}
 							break;
 						} catch (Error& e) {
-							wait(tr->onError(e));
+							innerErr = e;
 						}
+						co_await tr->onError(innerErr);
 					}
 				}
 
-				wait(taskFunc->execute(cx, taskBucket, futureBucket, task) ||
-				     extendTimeoutRepeatedly(cx, taskBucket, task));
+				co_await (taskFunc->execute(cx, taskBucket, futureBucket, task) ||
+				          extendTimeoutRepeatedly(cx, taskBucket, task));
 
 				if (BUGGIFY)
-					wait(delay(10.0));
-				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
+					co_await delay(10.0);
+				co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 					return finishTaskRun(tr, taskBucket, futureBucket, task, taskFunc, verifyTask);
-				}));
+				});
 			}
+			co_return true;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "TaskBucketExecuteFailure")
-			    .error(e)
-			    .detail("TaskUID", task->key)
+			err = e;
+		}
+		TraceEvent(SevWarn, "TaskBucketExecuteFailure")
+		    .error(err)
+		    .detail("TaskUID", task->key)
+		    .detail("TaskType", task->params[Task::reservedTaskParamKeyType].printable())
+		    .detail("Priority", task->getPriority());
+		try {
+			co_await taskFunc->handleError(cx, task, err);
+		} catch (Error& handleErr) {
+			TraceEvent(SevWarn, "TaskBucketExecuteFailureLogErrorFailed")
+			    .error(handleErr) // output handleError() error instead of original task error
+			    .detail("TaskUID", task->key.printable())
 			    .detail("TaskType", task->params[Task::reservedTaskParamKeyType].printable())
 			    .detail("Priority", task->getPriority());
-			try {
-				wait(taskFunc->handleError(cx, task, e));
-			} catch (Error& e) {
-				TraceEvent(SevWarn, "TaskBucketExecuteFailureLogErrorFailed")
-				    .error(e) // output handleError() error instead of original task error
-				    .detail("TaskUID", task->key.printable())
-				    .detail("TaskType", task->params[Task::reservedTaskParamKeyType].printable())
-				    .detail("Priority", task->getPriority());
-			}
 		}
 
 		// Return true to indicate that we did work.
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<Void> dispatch(Database cx,
-	                                   Reference<TaskBucket> taskBucket,
-	                                   Reference<FutureBucket> futureBucket,
-	                                   std::shared_ptr<double const> pollDelay,
-	                                   int maxConcurrentTasks) {
-		state std::vector<Future<bool>> tasks(maxConcurrentTasks);
+	static Future<Void> dispatch(Database cx,
+	                             Reference<TaskBucket> taskBucket,
+	                             Reference<FutureBucket> futureBucket,
+	                             std::shared_ptr<double const> pollDelay,
+	                             int maxConcurrentTasks) {
+		std::vector<Future<bool>> tasks(maxConcurrentTasks);
 		for (auto& f : tasks)
 			f = Never();
 
 		// Since the futures have to be kept in a vector to be compatible with waitForAny(), we'll keep a queue
 		// of available slots in it.  Initially, they're all available.
-		state std::vector<int> availableSlots;
+		std::vector<int> availableSlots;
 		availableSlots.reserve(tasks.size());
 		for (int i = 0; i < tasks.size(); ++i)
 			availableSlots.push_back(i);
 
-		state std::vector<Future<Reference<Task>>> getTasks;
-		state unsigned int getBatchSize = 1;
+		std::vector<Future<Reference<Task>>> getTasks;
+		unsigned int getBatchSize = 1;
 
-		loop {
+		while (true) {
 			// Start running tasks while slots are available and we keep finding work to do
 			++taskBucket->dispatchSlotChecksStarted;
 			while (!availableSlots.empty()) {
 				getTasks.clear();
 				for (int i = 0, imax = std::min<unsigned int>(getBatchSize, availableSlots.size()); i < imax; ++i)
 					getTasks.push_back(taskBucket->getOne(cx));
-				wait(waitForAllReady(getTasks));
+				co_await waitForAllReady(getTasks);
 
 				bool done = false;
 				for (int i = 0; i < getTasks.size(); ++i) {
@@ -540,7 +537,7 @@ class TaskBucketImpl {
 				w = w ||
 				    delay(*pollDelay * (0.9 + deterministicRandom()->random01() / 5)); // Jittered by 20 %, so +/- 10%
 			}
-			wait(w);
+			co_await w;
 
 			// Check all of the task slots, any that are finished should be replaced with Never() and their slots added
 			// back to availableSlots
@@ -553,40 +550,42 @@ class TaskBucketImpl {
 		}
 	}
 
-	ACTOR static Future<Void> watchPaused(Database cx,
-	                                      Reference<TaskBucket> taskBucket,
-	                                      Reference<AsyncVar<bool>> paused) {
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	static Future<Void> watchPaused(Database cx, Reference<TaskBucket> taskBucket, Reference<AsyncVar<bool>> paused) {
+		while (true) {
+			auto tr = makeReference<ReadYourWritesTransaction>(cx);
+			Error err;
 			try {
 				taskBucket->setOptions(tr);
-				Optional<Value> pausedVal = wait(tr->get(taskBucket->pauseKey));
+				Optional<Value> pausedVal = co_await tr->get(taskBucket->pauseKey);
 				paused->set(pausedVal.present());
-				state Future<Void> watchPausedFuture = tr->watch(taskBucket->pauseKey);
-				wait(tr->commit());
-				wait(watchPausedFuture);
+				Future<Void> watchPausedFuture = tr->watch(taskBucket->pauseKey);
+				co_await tr->commit();
+				co_await watchPausedFuture;
+				continue;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> run(Database cx,
-	                              Reference<TaskBucket> taskBucket,
-	                              Reference<FutureBucket> futureBucket,
-	                              std::shared_ptr<double const> pollDelay,
-	                              int maxConcurrentTasks) {
-		state Reference<AsyncVar<bool>> paused = makeReference<AsyncVar<bool>>(true);
-		state Future<Void> watchPausedFuture = watchPaused(cx, taskBucket, paused);
+	static Future<Void> run(Database cx,
+	                        Reference<TaskBucket> taskBucket,
+	                        Reference<FutureBucket> futureBucket,
+	                        std::shared_ptr<double const> pollDelay,
+	                        int maxConcurrentTasks) {
+		auto paused = makeReference<AsyncVar<bool>>(true);
+		Future<Void> watchPausedFuture = watchPaused(cx, taskBucket, paused);
 		taskBucket->metricLogger = taskBucket->cc.traceCounters(
 		    "TaskBucketMetrics", taskBucket->dbgid, CLIENT_KNOBS->TASKBUCKET_LOGGING_DELAY);
-		loop {
+		while (true) {
 			while (paused->get()) {
-				wait(paused->onChange() || watchPausedFuture);
+				co_await (paused->onChange() || watchPausedFuture);
 			}
 
-			wait(dispatch(cx, taskBucket, futureBucket, pollDelay, maxConcurrentTasks) || paused->onChange() ||
-			     watchPausedFuture);
+			co_await race(dispatch(cx, taskBucket, futureBucket, pollDelay, maxConcurrentTasks),
+			              paused->onChange(),
+			              watchPausedFuture);
 		}
 	}
 
@@ -602,148 +601,149 @@ class TaskBucketImpl {
 		return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return addIdle(tr, taskBucket); });
 	}
 
-	ACTOR static Future<bool> isEmpty(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
+	static Future<bool> isEmpty(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
 		taskBucket->setOptions(tr);
 
 		// Check all available priorities for keys
-		state std::vector<Future<RangeResult>> resultFutures;
+		std::vector<Future<RangeResult>> resultFutures;
 		for (int pri = 0; pri <= CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY; ++pri)
 			resultFutures.push_back(tr->getRange(taskBucket->getAvailableSpace(pri).range(), 1));
 
 		// If any priority levels have any keys then the taskbucket is not empty so return false
-		state int i;
-		for (i = 0; i < resultFutures.size(); ++i) {
-			RangeResult results = wait(resultFutures[i]);
-			if (results.size() > 0)
-				return false;
+		for (int i = 0; i < resultFutures.size(); ++i) {
+			RangeResult results = co_await resultFutures[i];
+			if (!results.empty())
+				co_return false;
 		}
 
-		RangeResult values = wait(tr->getRange(taskBucket->timeouts.range(), 1));
-		if (values.size() > 0)
-			return false;
+		RangeResult values = co_await tr->getRange(taskBucket->timeouts.range(), 1);
+		if (!values.empty())
+			co_return false;
 
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<bool> isBusy(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
+	static Future<bool> isBusy(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
 		taskBucket->setOptions(tr);
 
 		// Check all available priorities for emptiness
-		state std::vector<Future<RangeResult>> resultFutures;
+		std::vector<Future<RangeResult>> resultFutures;
 		for (int pri = 0; pri <= CLIENT_KNOBS->TASKBUCKET_MAX_PRIORITY; ++pri)
 			resultFutures.push_back(tr->getRange(taskBucket->getAvailableSpace(pri).range(), 1));
 
 		// If any priority levels have any keys then return true as the level is 'busy'
-		state int i;
-		for (i = 0; i < resultFutures.size(); ++i) {
-			RangeResult results = wait(resultFutures[i]);
-			if (results.size() > 0)
-				return true;
+		for (int i = 0; i < resultFutures.size(); ++i) {
+			RangeResult results = co_await resultFutures[i];
+			if (!results.empty())
+				co_return true;
 		}
 
-		return false;
+		co_return false;
 	}
 
 	// Verify that the task's keys are still in the timeout space at the expected timeout prefix
-	ACTOR static Future<bool> isFinished(Reference<ReadYourWritesTransaction> tr,
-	                                     Reference<TaskBucket> taskBucket,
-	                                     Reference<Task> task) {
+	static Future<bool> isFinished(Reference<ReadYourWritesTransaction> tr,
+	                               Reference<TaskBucket> taskBucket,
+	                               Reference<Task> task) {
 		taskBucket->setOptions(tr);
 
 		Tuple t = Tuple::makeTuple(task->timeoutVersion, task->key);
-		RangeResult values = wait(tr->getRange(taskBucket->timeouts.range(t), 1));
-		if (values.size() > 0)
-			return false;
+		RangeResult values = co_await tr->getRange(taskBucket->timeouts.range(t), 1);
+		if (!values.empty())
+			co_return false;
 
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<bool> getActiveKey(Reference<ReadYourWritesTransaction> tr,
-	                                       Reference<TaskBucket> taskBucket,
-	                                       Optional<Value> startingValue) {
+	static Future<bool> getActiveKey(Reference<ReadYourWritesTransaction> tr,
+	                                 Reference<TaskBucket> taskBucket,
+	                                 Optional<Value> startingValue) {
 		taskBucket->setOptions(tr);
 
-		Optional<Value> new_value = wait(tr->get(taskBucket->active.key()));
+		Optional<Value> new_value = co_await tr->get(taskBucket->active.key());
 		if (new_value != startingValue) {
-			return true;
+			co_return true;
 		}
-		return false;
+		co_return false;
 	}
 
-	ACTOR static Future<bool> checkActive(Database cx, Reference<TaskBucket> taskBucket) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Optional<Value> startingValue;
+	static Future<bool> checkActive(Database cx, Reference<TaskBucket> taskBucket) {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Optional<Value> startingValue;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				taskBucket->setOptions(tr);
 
-				bool is_busy = wait(isBusy(tr, taskBucket));
+				bool is_busy = co_await isBusy(tr, taskBucket);
 				if (!is_busy) {
-					wait(success(addIdle(tr, taskBucket)));
+					co_await addIdle(tr, taskBucket);
 				}
 
-				Optional<Value> val = wait(tr->get(taskBucket->active.key()));
+				Optional<Value> val = co_await tr->get(taskBucket->active.key());
 				startingValue = val;
 
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		state int idx = 0;
+		int idx = 0;
 		for (; idx < CLIENT_KNOBS->TASKBUCKET_CHECK_ACTIVE_AMOUNT; ++idx) {
-			tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-			loop {
+			tr = makeReference<ReadYourWritesTransaction>(cx);
+			while (true) {
+				Error err;
 				try {
 					taskBucket->setOptions(tr);
 
-					wait(delay(CLIENT_KNOBS->TASKBUCKET_CHECK_ACTIVE_DELAY));
-					bool isActiveKey = wait(getActiveKey(tr, taskBucket, startingValue));
+					co_await delay(CLIENT_KNOBS->TASKBUCKET_CHECK_ACTIVE_DELAY);
+					bool isActiveKey = co_await getActiveKey(tr, taskBucket, startingValue);
 					if (isActiveKey) {
 						CODE_PROBE(true, "checkActive return true");
-						return true;
+						co_return true;
 					}
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 				}
+				co_await tr->onError(err);
 			}
 		}
 
 		CODE_PROBE(true, "checkActive return false");
-		return false;
+		co_return false;
 	}
 
-	ACTOR static Future<int64_t> getTaskCount(Reference<ReadYourWritesTransaction> tr,
-	                                          Reference<TaskBucket> taskBucket) {
+	static Future<int64_t> getTaskCount(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
 		taskBucket->setOptions(tr);
 
-		Optional<Value> val = wait(tr->get(taskBucket->prefix.pack("task_count"_sr)));
+		Optional<Value> val = co_await tr->get(taskBucket->prefix.pack("task_count"_sr));
 
 		if (!val.present())
-			return 0;
+			co_return 0;
 
 		ASSERT(val.get().size() == sizeof(int64_t));
 
 		int64_t intValue = 0;
 		memcpy(&intValue, val.get().begin(), val.get().size());
 
-		return intValue;
+		co_return intValue;
 	}
 
 	// Looks for tasks that have timed out and returns them to be available tasks.
 	// Returns True if any tasks were affected.
-	ACTOR static Future<bool> requeueTimedOutTasks(Reference<ReadYourWritesTransaction> tr,
-	                                               Reference<TaskBucket> taskBucket) {
+	static Future<bool> requeueTimedOutTasks(Reference<ReadYourWritesTransaction> tr,
+	                                         Reference<TaskBucket> taskBucket) {
 		CODE_PROBE(true, "Looks for tasks that have timed out and returns them to be available tasks.");
-		Version end = wait(tr->getReadVersion());
-		state KeyRange range(
+		Version end = co_await tr->getReadVersion();
+		KeyRange range(
 		    KeyRangeRef(taskBucket->timeouts.get(0).range().begin, taskBucket->timeouts.get(end).range().end));
 
-		RangeResult values = wait(tr->getRange(range, CLIENT_KNOBS->TASKBUCKET_MAX_TASK_KEYS));
+		RangeResult values = co_await tr->getRange(range, CLIENT_KNOBS->TASKBUCKET_MAX_TASK_KEYS);
 
 		// Keys will be tuples of (taskUID, param) -> paramValue
 		// Unfortunately we need to know the priority parameter for a taskUID before we can know which available-tasks
@@ -779,22 +779,22 @@ class TaskBucketImpl {
 			for (auto& p : task.params)
 				tr->set(space.pack(p.key), p.value);
 
-			if (values.size() > 0) {
+			if (!values.empty()) {
 				tr->clear(range);
-				return true;
+				co_return true;
 			}
-			return false;
+			co_return false;
 		}
 
-		ASSERT(lastKey != Key());
+		ASSERT(!lastKey.empty());
 		tr->clear(KeyRangeRef(range.begin, lastKey));
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<Void> debugPrintRange(Reference<ReadYourWritesTransaction> tr, Subspace subspace, Key msg) {
+	static Future<Void> debugPrintRange(Reference<ReadYourWritesTransaction> tr, Subspace subspace, Key msg) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		RangeResult values = wait(tr->getRange(subspace.range(), CLIENT_KNOBS->TOO_MANY));
+		RangeResult values = co_await tr->getRange(subspace.range(), CLIENT_KNOBS->TOO_MANY);
 		TraceEvent("TaskBucketDebugPrintRange")
 		    .detail("Key", subspace.key())
 		    .detail("Count", values.size())
@@ -807,24 +807,22 @@ class TaskBucketImpl {
 		        .detail("Key", s.key)
 		        .detail("Value", s.value);
 		}*/
-
-		return Void();
 	}
 
-	ACTOR static Future<Version> extendTimeout(Reference<ReadYourWritesTransaction> tr,
-	                                           Reference<TaskBucket> taskBucket,
-	                                           Reference<Task> task,
-	                                           UpdateParams updateParams,
-	                                           Version newTimeoutVersion) {
+	static Future<Version> extendTimeout(Reference<ReadYourWritesTransaction> tr,
+	                                     Reference<TaskBucket> taskBucket,
+	                                     Reference<Task> task,
+	                                     UpdateParams updateParams,
+	                                     Version newTimeoutVersion) {
 		taskBucket->setOptions(tr);
 
 		// First make sure it's safe to keep running
-		wait(taskBucket->keepRunning(tr, task));
+		co_await taskBucket->keepRunning(tr, task);
 
 		// This is where the task definition currently exists
-		state Subspace oldTimeoutSpace = taskBucket->timeouts.get(task->timeoutVersion).get(task->key);
+		Subspace oldTimeoutSpace = taskBucket->timeouts.get(task->timeoutVersion).get(task->key);
 		// Update the task's timeout
-		Version version = wait(tr->getReadVersion());
+		Version version = co_await tr->getReadVersion();
 
 		if (newTimeoutVersion == invalidVersion)
 			newTimeoutVersion = version + taskBucket->timeout;
@@ -838,7 +836,7 @@ class TaskBucketImpl {
 		}
 
 		// This is where the task definition is being moved to
-		state Subspace newTimeoutSpace = taskBucket->timeouts.get(newTimeoutVersion).get(task->key);
+		Subspace newTimeoutSpace = taskBucket->timeouts.get(newTimeoutVersion).get(task->key);
 
 		tr->addReadConflictRange(oldTimeoutSpace.range());
 		tr->addWriteConflictRange(newTimeoutSpace.range());
@@ -852,7 +850,7 @@ class TaskBucketImpl {
 		} else {
 			CODE_PROBE(true, "Extended a task without updating parameters");
 			// Otherwise, read and transplant the params from the old to new timeout spaces
-			RangeResult params = wait(tr->getRange(oldTimeoutSpace.range(), CLIENT_KNOBS->TOO_MANY));
+			RangeResult params = co_await tr->getRange(oldTimeoutSpace.range(), CLIENT_KNOBS->TOO_MANY);
 			for (auto& kv : params) {
 				Tuple paramKey = oldTimeoutSpace.unpack(kv.key);
 				tr->set(newTimeoutSpace.pack(paramKey), kv.value);
@@ -861,7 +859,7 @@ class TaskBucketImpl {
 
 		tr->clear(oldTimeoutSpace.range());
 
-		return newTimeoutVersion;
+		co_return newTimeoutVersion;
 	}
 };
 
@@ -877,7 +875,7 @@ TaskBucket::TaskBucket(const Subspace& subspace,
     timeouts(prefix.get("to"_sr)), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS), system_access(sysAccess),
     priority_batch(priorityBatch), lockAware(lockAware) {}
 
-TaskBucket::~TaskBucket() {}
+TaskBucket::~TaskBucket() = default;
 
 Future<Void> TaskBucket::clear(Reference<ReadYourWritesTransaction> tr) {
 	setOptions(tr);
@@ -928,13 +926,13 @@ void TaskBucket::setValidationCondition(Reference<Task> task, KeyRef vKey, KeyRe
 	task->params[Task::reservedTaskParamValidValue] = vValue;
 }
 
-ACTOR static Future<Key> actorAddTask(TaskBucket* tb,
-                                      Reference<ReadYourWritesTransaction> tr,
-                                      Reference<Task> task,
-                                      KeyRef validationKey) {
+static Future<Key> actorAddTask(TaskBucket* tb,
+                                Reference<ReadYourWritesTransaction> tr,
+                                Reference<Task> task,
+                                KeyRef validationKey) {
 	tb->setOptions(tr);
 
-	Optional<Value> validationValue = wait(tr->get(validationKey));
+	Optional<Value> validationValue = co_await tr->get(validationKey);
 
 	if (!validationValue.present()) {
 		TraceEvent(SevError, "TaskBucketAddTaskInvalidKey")
@@ -945,7 +943,7 @@ ACTOR static Future<Key> actorAddTask(TaskBucket* tb,
 
 	TaskBucket::setValidationCondition(task, validationKey, validationValue.get());
 
-	return tb->addTask(tr, task);
+	co_return tb->addTask(tr, task);
 }
 
 Future<Key> TaskBucket::addTask(Reference<ReadYourWritesTransaction> tr, Reference<Task> task, KeyRef validationKey) {
@@ -1032,18 +1030,18 @@ Future<Void> TaskBucket::debugPrintRange(Reference<ReadYourWritesTransaction> tr
 
 class FutureBucketImpl {
 public:
-	ACTOR static Future<bool> isEmpty(Reference<ReadYourWritesTransaction> tr, Reference<FutureBucket> futureBucket) {
+	static Future<bool> isEmpty(Reference<ReadYourWritesTransaction> tr, Reference<FutureBucket> futureBucket) {
 		futureBucket->setOptions(tr);
 
-		Key lastKey = wait(tr->getKey(lastLessOrEqual(futureBucket->prefix.pack(maxUIDKey))));
-		return !futureBucket->prefix.contains(lastKey);
+		Key lastKey = co_await tr->getKey(lastLessOrEqual(futureBucket->prefix.pack(maxUIDKey)));
+		co_return !futureBucket->prefix.contains(lastKey);
 	}
 };
 
 FutureBucket::FutureBucket(const Subspace& subspace, AccessSystemKeys sysAccess, LockAware lockAware)
   : prefix(subspace), system_access(sysAccess), lockAware(lockAware) {}
 
-FutureBucket::~FutureBucket() {}
+FutureBucket::~FutureBucket() = default;
 
 Future<Void> FutureBucket::clear(Reference<ReadYourWritesTransaction> tr) {
 	setOptions(tr);
@@ -1071,29 +1069,27 @@ Reference<TaskFuture> FutureBucket::unpack(Key key) {
 
 class TaskFutureImpl {
 public:
-	ACTOR static Future<Void> join(Reference<ReadYourWritesTransaction> tr,
-	                               Reference<TaskBucket> taskBucket,
-	                               Reference<TaskFuture> taskFuture,
-	                               std::vector<Reference<TaskFuture>> vectorFuture) {
+	static Future<Void> join(Reference<ReadYourWritesTransaction> tr,
+	                         Reference<TaskBucket> taskBucket,
+	                         Reference<TaskFuture> taskFuture,
+	                         std::vector<Reference<TaskFuture>> vectorFuture) {
 		taskFuture->futureBucket->setOptions(tr);
 
-		bool is_set = wait(isSet(tr, taskFuture));
+		bool is_set = co_await isSet(tr, taskFuture);
 		// A taskFuture cannot be already set for it to be joined with others.
 		if (is_set) {
-			return Void();
+			co_return;
 		}
 
 		tr->clear(taskFuture->blocks.pack(StringRef()));
 
-		wait(_join(tr, taskBucket, taskFuture, vectorFuture));
-
-		return Void();
+		co_await _join(tr, taskBucket, taskFuture, vectorFuture);
 	}
 
-	ACTOR static Future<Void> _join(Reference<ReadYourWritesTransaction> tr,
-	                                Reference<TaskBucket> taskBucket,
-	                                Reference<TaskFuture> taskFuture,
-	                                std::vector<Reference<TaskFuture>> vectorFuture) {
+	static Future<Void> _join(Reference<ReadYourWritesTransaction> tr,
+	                          Reference<TaskBucket> taskBucket,
+	                          Reference<TaskFuture> taskFuture,
+	                          std::vector<Reference<TaskFuture>> vectorFuture) {
 		std::vector<Future<Void>> onSetFutures;
 		for (int i = 0; i < vectorFuture.size(); ++i) {
 			Key key = StringRef(deterministicRandom()->randomUniqueID().toString());
@@ -1105,32 +1101,30 @@ class TaskFutureImpl {
 			onSetFutures.push_back(vectorFuture[i]->onSet(tr, taskBucket, task));
 		}
 
-		wait(waitForAll(onSetFutures));
-
-		return Void();
+		co_await waitForAll(onSetFutures);
 	}
 
-	ACTOR static Future<bool> isSet(Reference<ReadYourWritesTransaction> tr, Reference<TaskFuture> taskFuture) {
+	static Future<bool> isSet(Reference<ReadYourWritesTransaction> tr, Reference<TaskFuture> taskFuture) {
 		taskFuture->futureBucket->setOptions(tr);
 
-		RangeResult values = wait(tr->getRange(taskFuture->blocks.range(), 1));
-		if (values.size() > 0)
-			return false;
+		RangeResult values = co_await tr->getRange(taskFuture->blocks.range(), 1);
+		if (!values.empty())
+			co_return false;
 
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<Void> onSet(Reference<ReadYourWritesTransaction> tr,
-	                                Reference<TaskBucket> taskBucket,
-	                                Reference<TaskFuture> taskFuture,
-	                                Reference<Task> task) {
+	static Future<Void> onSet(Reference<ReadYourWritesTransaction> tr,
+	                          Reference<TaskBucket> taskBucket,
+	                          Reference<TaskFuture> taskFuture,
+	                          Reference<Task> task) {
 		taskFuture->futureBucket->setOptions(tr);
 
-		bool is_set = wait(isSet(tr, taskFuture));
+		bool is_set = co_await isSet(tr, taskFuture);
 
 		if (is_set) {
 			CODE_PROBE(true, "is_set == true");
-			wait(performAction(tr, taskBucket, taskFuture, task));
+			co_await performAction(tr, taskBucket, taskFuture, task);
 		} else {
 			CODE_PROBE(true, "is_set == false");
 			Subspace callbackSpace =
@@ -1139,50 +1133,44 @@ class TaskFutureImpl {
 				tr->set(callbackSpace.pack(v.key), v.value);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> set(Reference<ReadYourWritesTransaction> tr,
-	                              Reference<TaskBucket> taskBucket,
-	                              Reference<TaskFuture> taskFuture) {
+	static Future<Void> set(Reference<ReadYourWritesTransaction> tr,
+	                        Reference<TaskBucket> taskBucket,
+	                        Reference<TaskFuture> taskFuture) {
 		taskFuture->futureBucket->setOptions(tr);
 
 		tr->clear(taskFuture->blocks.range());
 
-		wait(performAllActions(tr, taskBucket, taskFuture));
-
-		return Void();
+		co_await performAllActions(tr, taskBucket, taskFuture);
 	}
 
-	ACTOR static Future<Void> performAction(Reference<ReadYourWritesTransaction> tr,
-	                                        Reference<TaskBucket> taskBucket,
-	                                        Reference<TaskFuture> taskFuture,
-	                                        Reference<Task> task) {
+	static Future<Void> performAction(Reference<ReadYourWritesTransaction> tr,
+	                                  Reference<TaskBucket> taskBucket,
+	                                  Reference<TaskFuture> taskFuture,
+	                                  Reference<Task> task) {
 		taskFuture->futureBucket->setOptions(tr);
 
 		if (task && TaskFuncBase::isValidTask(task)) {
 			Reference<TaskFuncBase> taskFunc = TaskFuncBase::create(task->params[Task::reservedTaskParamKeyType]);
 			if (taskFunc.getPtr()) {
-				wait(taskFunc->finish(tr, taskBucket, taskFuture->futureBucket, task));
+				co_await taskFunc->finish(tr, taskBucket, taskFuture->futureBucket, task);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> performAllActions(Reference<ReadYourWritesTransaction> tr,
-	                                            Reference<TaskBucket> taskBucket,
-	                                            Reference<TaskFuture> taskFuture) {
+	static Future<Void> performAllActions(Reference<ReadYourWritesTransaction> tr,
+	                                      Reference<TaskBucket> taskBucket,
+	                                      Reference<TaskFuture> taskFuture) {
 		taskFuture->futureBucket->setOptions(tr);
 
-		RangeResult values = wait(tr->getRange(taskFuture->callbacks.range(), CLIENT_KNOBS->TOO_MANY));
+		RangeResult values = co_await tr->getRange(taskFuture->callbacks.range(), CLIENT_KNOBS->TOO_MANY);
 		tr->clear(taskFuture->callbacks.range());
 
 		std::vector<Future<Void>> actions;
 
-		if (values.size() != 0) {
-			state Reference<Task> task(new Task());
+		if (!values.empty()) {
+			Reference<Task> task(new Task());
 			Key lastTaskID;
 			for (auto& s : values) {
 				Tuple t = taskFuture->callbacks.unpack(s.key);
@@ -1190,7 +1178,7 @@ class TaskFutureImpl {
 				Key key = t.getString(1);
 				// If we see a new task ID and the old one isn't empty then process the task accumulated so far and make
 				// a new task
-				if (taskID.size() != 0 && taskID != lastTaskID) {
+				if (!taskID.empty() && taskID != lastTaskID) {
 					actions.push_back(performAction(tr, taskBucket, taskFuture, task));
 					task = makeReference<Task>();
 				}
@@ -1201,32 +1189,28 @@ class TaskFutureImpl {
 			actions.push_back(performAction(tr, taskBucket, taskFuture, task));
 		}
 
-		wait(waitForAll(actions));
-
-		return Void();
+		co_await waitForAll(actions);
 	}
 
-	ACTOR static Future<Void> onSetAddTask(Reference<ReadYourWritesTransaction> tr,
-	                                       Reference<TaskBucket> taskBucket,
-	                                       Reference<TaskFuture> taskFuture,
-	                                       Reference<Task> task) {
+	static Future<Void> onSetAddTask(Reference<ReadYourWritesTransaction> tr,
+	                                 Reference<TaskBucket> taskBucket,
+	                                 Reference<TaskFuture> taskFuture,
+	                                 Reference<Task> task) {
 		taskFuture->futureBucket->setOptions(tr);
 
 		task->params[Task::reservedTaskParamKeyAddTask] = task->params[Task::reservedTaskParamKeyType];
 		task->params[Task::reservedTaskParamKeyType] = "AddTask"_sr;
-		wait(onSet(tr, taskBucket, taskFuture, task));
-
-		return Void();
+		co_await onSet(tr, taskBucket, taskFuture, task);
 	}
 
-	ACTOR static Future<Void> onSetAddTask(Reference<ReadYourWritesTransaction> tr,
-	                                       Reference<TaskBucket> taskBucket,
-	                                       Reference<TaskFuture> taskFuture,
-	                                       Reference<Task> task,
-	                                       KeyRef validationKey) {
+	static Future<Void> onSetAddTask(Reference<ReadYourWritesTransaction> tr,
+	                                 Reference<TaskBucket> taskBucket,
+	                                 Reference<TaskFuture> taskFuture,
+	                                 Reference<Task> task,
+	                                 KeyRef validationKey) {
 		taskFuture->futureBucket->setOptions(tr);
 
-		Optional<Value> validationValue = wait(tr->get(validationKey));
+		Optional<Value> validationValue = co_await tr->get(validationKey);
 
 		if (!validationValue.present()) {
 			TraceEvent(SevError, "TaskBucketOnSetAddTaskInvalidKey")
@@ -1238,9 +1222,7 @@ class TaskFutureImpl {
 		task->params[Task::reservedTaskParamValidKey] = validationKey;
 		task->params[Task::reservedTaskParamValidValue] = validationValue.get();
 
-		wait(onSetAddTask(tr, taskBucket, taskFuture, task));
-
-		return Void();
+		co_await onSetAddTask(tr, taskBucket, taskFuture, task);
 	}
 
 	static Future<Void> onSetAddTask(Reference<ReadYourWritesTransaction> tr,
@@ -1257,25 +1239,25 @@ class TaskFutureImpl {
 		return onSetAddTask(tr, taskBucket, taskFuture, task);
 	}
 
-	ACTOR static Future<Reference<TaskFuture>> joinedFuture(Reference<ReadYourWritesTransaction> tr,
-	                                                        Reference<TaskBucket> taskBucket,
-	                                                        Reference<TaskFuture> taskFuture) {
+	static Future<Reference<TaskFuture>> joinedFuture(Reference<ReadYourWritesTransaction> tr,
+	                                                  Reference<TaskBucket> taskBucket,
+	                                                  Reference<TaskFuture> taskFuture) {
 		taskFuture->futureBucket->setOptions(tr);
 
 		std::vector<Reference<TaskFuture>> vectorFuture;
 		// the next line means generate a new task future with different key,
 		// but share the same prefix of futureBucket with the input taskFuture
-		state Reference<TaskFuture> future = taskFuture->futureBucket->future(tr);
+		Reference<TaskFuture> future = taskFuture->futureBucket->future(tr);
 		vectorFuture.push_back(future);
-		wait(join(tr, taskBucket, taskFuture, vectorFuture));
-		return future;
+		co_await join(tr, taskBucket, taskFuture, vectorFuture);
+		co_return future;
 	}
 };
 
-TaskFuture::TaskFuture() {}
+TaskFuture::TaskFuture() = default;
 
 TaskFuture::TaskFuture(const Reference<FutureBucket> bucket, Key k) : futureBucket(bucket), key(k) {
-	if (k.size() == 0) {
+	if (k.empty()) {
 		key = deterministicRandom()->randomUniqueID().toString();
 	}
 
@@ -1284,7 +1266,7 @@ TaskFuture::TaskFuture(const Reference<FutureBucket> bucket, Key k) : futureBuck
 	callbacks = prefix.get("cb"_sr);
 }
 
-TaskFuture::~TaskFuture() {}
+TaskFuture::~TaskFuture() = default;
 
 void TaskFuture::addBlock(Reference<ReadYourWritesTransaction> tr, StringRef block_id) {
 	tr->set(blocks.pack(block_id), ""_sr);
@@ -1341,11 +1323,11 @@ Future<Reference<TaskFuture>> TaskFuture::joinedFuture(Reference<ReadYourWritesT
 	return TaskFutureImpl::joinedFuture(tr, taskBucket, Reference<TaskFuture>::addRef(this));
 }
 
-ACTOR Future<Key> getCompletionKey(TaskCompletionKey* self, Future<Reference<TaskFuture>> f) {
-	Reference<TaskFuture> taskFuture = wait(f);
+Future<Key> getCompletionKey(TaskCompletionKey* self, Future<Reference<TaskFuture>> f) {
+	Reference<TaskFuture> taskFuture = co_await f;
 	self->joinFuture.clear();
 	self->key = taskFuture->key;
-	return self->key.get();
+	co_return self->key.get();
 }
 
 Future<Key> TaskCompletionKey::get(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket) {
diff --git a/fdbclient/TestKnobCollection.cpp b/fdbclient/TestKnobCollection.cpp
deleted file mode 100644
index 7e6f782fd0e..00000000000
--- a/fdbclient/TestKnobCollection.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * TestKnobCollection.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/TestKnobCollection.h"
-
-TestKnobCollection::TestKnobCollection(Randomize randomize, IsSimulated isSimulated)
-  : serverKnobCollection(randomize, isSimulated) {
-	initialize(randomize, isSimulated);
-}
-
-void TestKnobCollection::initialize(Randomize randomize, IsSimulated isSimulated) {
-	serverKnobCollection.initialize(randomize, isSimulated);
-	testKnobs.initialize();
-}
-
-void TestKnobCollection::reset(Randomize randomize, IsSimulated isSimulated) {
-	serverKnobCollection.reset(randomize, isSimulated);
-	testKnobs.reset();
-}
-
-void TestKnobCollection::clearTestKnobs() {
-	testKnobs.reset();
-}
-
-Optional<KnobValue> TestKnobCollection::tryParseKnobValue(std::string const& knobName,
-                                                          std::string const& knobValue) const {
-	auto result = serverKnobCollection.tryParseKnobValue(knobName, knobValue);
-	if (result.present()) {
-		return result;
-	}
-	auto parsedKnobValue = testKnobs.parseKnobValue(knobName, knobValue);
-	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
-		return KnobValueRef::create(parsedKnobValue);
-	}
-	return {};
-}
-
-bool TestKnobCollection::trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) {
-	return serverKnobCollection.trySetKnob(knobName, knobValue) || knobValue.visitSetKnob(knobName, testKnobs);
-}
-
-#define init(knob, value) initKnob(knob, value, #knob)
-
-TestKnobs::TestKnobs() {
-	initialize();
-}
-
-void TestKnobs::initialize() {
-	init(TEST_LONG, 0);
-	init(TEST_INT, 0);
-	init(TEST_DOUBLE, 0.0);
-	init(TEST_BOOL, false);
-	init(TEST_STRING, "");
-}
-
-bool TestKnobs::operator==(TestKnobs const& rhs) const {
-	return (TEST_LONG == rhs.TEST_LONG) && (TEST_INT == rhs.TEST_INT) && (TEST_DOUBLE == rhs.TEST_DOUBLE) &&
-	       (TEST_BOOL == rhs.TEST_BOOL) && (TEST_STRING == rhs.TEST_STRING);
-}
-
-bool TestKnobs::operator!=(TestKnobs const& rhs) const {
-	return !(*this == rhs);
-}
diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp
index 4d79566f4c9..5e0a8810202 100644
--- a/fdbclient/ThreadSafeTransaction.cpp
+++ b/fdbclient/ThreadSafeTransaction.cpp
@@ -24,7 +24,7 @@
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/versions.h"
-#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/GenericManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "flow/Arena.h"
 #include "flow/ProtocolVersion.h"
@@ -46,12 +46,12 @@ ThreadFuture<Reference<IDatabase>> ThreadSafeDatabase::createFromExistingDatabas
 		db->checkDeferredError();
 		DatabaseContext* cx = db.getPtr();
 		cx->addref();
-		return Future<Reference<IDatabase>>(Reference<IDatabase>(new ThreadSafeDatabase(cx)));
+		return Future<Reference<IDatabase>>(makeReference<ThreadSafeDatabase>(cx));
 	});
 }
 
 Reference<ITransaction> ThreadSafeDatabase::createTransaction() {
-	return Reference<ITransaction>(new ThreadSafeTransaction(db));
+	return makeReference<ThreadSafeTransaction>(db);
 }
 
 void ThreadSafeDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
@@ -142,8 +142,8 @@ ThreadSafeDatabase::ThreadSafeDatabase(ConnectionRecordType connectionRecordType
 			Reference<IClusterConnectionRecord> connectionRecord =
 			    connectionRecordType == ConnectionRecordType::FILE
 			        ? Reference<IClusterConnectionRecord>(ClusterConnectionFile::openOrDefault(connectionRecordString))
-			        : Reference<IClusterConnectionRecord>(
-			              new ClusterConnectionMemoryRecord(ClusterConnectionString(connectionRecordString)));
+			        : Reference<IClusterConnectionRecord>(makeReference<ClusterConnectionMemoryRecord>(
+			              ClusterConnectionString(connectionRecordString)));
 
 			Database::createDatabase(connectionRecord, apiVersion, IsInternal::False, LocalityData(), db).extractPtr();
 		} catch (Error& e) {
@@ -592,13 +592,13 @@ void ThreadSafeApi::stopNetwork() {
 }
 
 Reference<IDatabase> ThreadSafeApi::createDatabase(const char* clusterFilePath) {
-	return Reference<IDatabase>(
-	    new ThreadSafeDatabase(ThreadSafeDatabase::ConnectionRecordType::FILE, clusterFilePath, apiVersion.version()));
+	return makeReference<ThreadSafeDatabase>(
+	    ThreadSafeDatabase::ConnectionRecordType::FILE, clusterFilePath, apiVersion.version());
 }
 
 Reference<IDatabase> ThreadSafeApi::createDatabaseFromConnectionString(const char* connectionString) {
-	return Reference<IDatabase>(new ThreadSafeDatabase(
-	    ThreadSafeDatabase::ConnectionRecordType::CONNECTION_STRING, connectionString, apiVersion.version()));
+	return makeReference<ThreadSafeDatabase>(
+	    ThreadSafeDatabase::ConnectionRecordType::CONNECTION_STRING, connectionString, apiVersion.version());
 }
 
 void ThreadSafeApi::addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) {
diff --git a/fdbclient/Tracing.actor.cpp b/fdbclient/Tracing.cpp
similarity index 91%
rename from fdbclient/Tracing.actor.cpp
rename to fdbclient/Tracing.cpp
index 4343e044fad..9ba14e6135d 100644
--- a/fdbclient/Tracing.actor.cpp
+++ b/fdbclient/Tracing.cpp
@@ -1,5 +1,5 @@
 /*
- * Tracing.actor.cpp
+ * Tracing.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,18 +24,15 @@
 #include "flow/UnitTest.h"
 #include "flow/Knobs.h"
 #include "flow/IConnection.h"
-#include "fdbclient/IKnobCollection.h"
 #include "flow/network.h"
 #include <functional>
 #include <iomanip>
 #include <memory>
 #include "flow/IUDPSocket.h"
 
-#include "flow/actorcompiler.h" // has to be last include
+#include "flow/CoroUtils.h"
 
-#ifdef NO_INTELLISENSE
 namespace {
-#endif
 
 // Initial size of buffer used to store serialized traces. Buffer will be
 // resized when necessary.
@@ -83,23 +80,27 @@ struct LogfileTracer : ITracer {
 };
 
 // A server listening for UDP trace messages, run only in simulation.
-ACTOR Future<Void> simulationStartServer() {
+Future<Void> simulationStartServer() {
+	// Preserve the actor scheduling boundary so UDP listener setup does not run synchronously
+	// from the first sampled Span destructor after coroutine conversion.
+	co_await delay(0);
+
 	// We're going to force the address to be loopback regardless of FLOW_KNOBS->TRACING_UDP_LISTENER_ADDR
 	// because we're in simulation testing mode.
 	TraceEvent(SevInfo, "UDPServerStarted")
 	    .detail("Address", "127.0.0.1")
 	    .detail("Port", FLOW_KNOBS->TRACING_UDP_LISTENER_PORT);
-	state NetworkAddress localAddress =
+	NetworkAddress localAddress =
 	    NetworkAddress::parse("127.0.0.1:" + std::to_string(FLOW_KNOBS->TRACING_UDP_LISTENER_PORT));
-	state Reference<IUDPSocket> serverSocket = wait(INetworkConnections::net()->createUDPSocket(localAddress));
+	Reference<IUDPSocket> serverSocket = co_await INetworkConnections::net()->createUDPSocket(localAddress);
 	serverSocket->bind(localAddress);
 
-	state Standalone<StringRef> packetString = makeString(IUDPSocket::MAX_PACKET_SIZE);
-	state uint8_t* packet = mutateString(packetString);
+	auto packet = std::make_unique<uint8_t[]>(IUDPSocket::MAX_PACKET_SIZE);
 
-	loop {
-		int size = wait(serverSocket->receive(packet, packet + IUDPSocket::MAX_PACKET_SIZE));
-		auto message = packetString.substr(0, size);
+	while (true) {
+		uint8_t* packetBegin = packet.get();
+		int size = co_await serverSocket->receive(packetBegin, packetBegin + IUDPSocket::MAX_PACKET_SIZE);
+		auto message = StringRef(packetBegin, size);
 
 		// For now, just check the first byte in the message matches. Data is
 		// currently written as an array, so first byte should match msgpack
@@ -112,10 +113,10 @@ ACTOR Future<Void> simulationStartServer() {
 /*
 // Runs on an interval, printing debug information and performing other
 // connection tasks.
-ACTOR Future<Void> traceLog(int* pendingMessages, bool* sendError) {
-    state bool sendErrorReset = false;
+Future<Void> traceLog(int* pendingMessages, bool* sendError) {
+    bool sendErrorReset = false;
 
-    loop {
+    while (true) {
         TraceEvent("TracingSpanQueueSize").detail("PendingMessages", *pendingMessages);
 
         // Wait at least one full loop before attempting to send messages
@@ -127,7 +128,7 @@ ACTOR Future<Void> traceLog(int* pendingMessages, bool* sendError) {
             sendErrorReset = true;
         }
 
-        wait(delay(kQueueSizeLogInterval));
+        co_await delay(kQueueSizeLogInterval);
     }
 }
 */
@@ -163,7 +164,8 @@ struct UDPTracer : public ITracer {
 	// Writes the given vector of linked SpanContext's to the request. If the vector is
 	// empty, the request is not modified.
 	inline void serialize_vector(const SmallVectorRef<SpanContext>& vec, MsgpackBuffer& buf) {
-		int size = vec.size();
+		// Each SpanContext produces 3 MsgPack elements: traceID.first, traceID.second, spanID
+		int size = vec.size() * 3;
 		if (size <= 15) {
 			buf.write_byte(static_cast<uint8_t>(size) | 0b10010000);
 		} else if (size <= 65535) {
@@ -185,7 +187,8 @@ struct UDPTracer : public ITracer {
 	// Writes the given vector of linked SpanEventRef's to the request. If the vector is
 	// empty, the request is not modified.
 	inline void serialize_vector(const SmallVectorRef<SpanEventRef>& vec, MsgpackBuffer& buf) {
-		int size = vec.size();
+		// Each SpanEventRef produces 3 MsgPack elements: name, time, attributes
+		int size = vec.size() * 3;
 		if (size <= 15) {
 			buf.write_byte(static_cast<uint8_t>(size) | 0b10010000);
 		} else if (size <= 65535) {
@@ -223,10 +226,10 @@ struct UDPTracer : public ITracer {
 };
 
 #ifndef WIN32
-ACTOR Future<Void> fastTraceLogger(int* unreadyMessages, int* failedMessages, int* totalMessages, bool* sendError) {
-	state bool sendErrorReset = false;
+Future<Void> fastTraceLogger(int* unreadyMessages, int* failedMessages, int* totalMessages, bool* sendError) {
+	bool sendErrorReset = false;
 
-	loop {
+	while (true) {
 		TraceEvent("TracingSpanStats")
 		    .detail("UnreadyMessages", *unreadyMessages)
 		    .detail("FailedMessages", *failedMessages)
@@ -240,7 +243,7 @@ ACTOR Future<Void> fastTraceLogger(int* unreadyMessages, int* failedMessages, in
 			sendErrorReset = true;
 		}
 
-		wait(delay(kQueueSizeLogInterval));
+		co_await delay(kQueueSizeLogInterval);
 	}
 }
 
@@ -264,10 +267,20 @@ struct FastUDPTracer : public UDPTracer {
 				// Force loopback when in simulation mode
 				destAddr = "127.0.0.1";
 			}
-			NetworkAddress destAddress =
-			    NetworkAddress::parse(destAddr + ":" + std::to_string(FLOW_KNOBS->TRACING_UDP_LISTENER_PORT));
-
-			socket_ = INetworkConnections::net()->createUDPSocket(destAddress);
+			// NetworkAddress::parse() only accepts literal IP addresses, not hostnames.
+			// If the knob is set to a DNS name, parse() throws connection_string_invalid.
+			// Catch it here and disable tracing gracefully instead of crashing the process.
+			try {
+				NetworkAddress destAddress =
+				    NetworkAddress::parse(destAddr + ":" + std::to_string(FLOW_KNOBS->TRACING_UDP_LISTENER_PORT));
+				socket_ = INetworkConnections::net()->createUDPSocket(destAddress);
+			} catch (Error& e) {
+				TraceEvent(SevWarnAlways, "TracingInvalidListenerAddress")
+				    .error(e)
+				    .detail("Address", destAddr)
+				    .detail("Hint", "TRACING_UDP_LISTENER_ADDR must be a literal IP address, not a hostname");
+				send_error_ = true; // disable tracing for this process
+			}
 		});
 
 		if (size == 0) {
@@ -275,16 +288,15 @@ struct FastUDPTracer : public UDPTracer {
 		}
 
 		++total_messages_;
+		if (send_error_) {
+			return;
+		}
 		if (!socket_.isReady()) {
 			++unready_socket_messages_;
 			return;
 		} else if (socket_fd_ == -1) {
 			socket_fd_ = socket_.get()->native_handle();
 		}
-
-		if (send_error_) {
-			return;
-		}
 	}
 
 	void write() {
@@ -322,9 +334,7 @@ struct FastUDPTracer : public UDPTracer {
 
 ITracer* g_tracer = new NoopTracer();
 
-#ifdef NO_INTELLISENSE
 } // namespace
-#endif
 
 void openTracer(TracerType type) {
 	if (g_tracer->type() == type) {
@@ -349,7 +359,7 @@ void openTracer(TracerType type) {
 	}
 }
 
-ITracer::~ITracer() {}
+ITracer::~ITracer() = default;
 
 Span& Span::operator=(Span&& o) {
 	if (begin > 0.0 && context.isSampled()) {
@@ -607,7 +617,7 @@ TEST_CASE("/flow/Tracing/FastUDPMessagePackEncoding") {
 	ASSERT(data[70] == 0xcc);
 	ASSERT(data[71] == static_cast<uint8_t>(SpanStatus::OK));
 	// Linked SpanContext
-	ASSERT(data[72] == 0b10010001);
+	ASSERT(data[72] == 0b10010011); // 3 elements: traceID.first, traceID.second, spanID
 	ASSERT(data[73] == 0xcf);
 	ASSERT(swapUint64BE(&data[74]) == 200);
 	ASSERT(data[82] == 0xcf);
@@ -646,7 +656,7 @@ TEST_CASE("/flow/Tracing/FastUDPMessagePackEncoding") {
 	ASSERT(data[72] == 0xcc);
 	ASSERT(data[73] == static_cast<uint8_t>(SpanStatus::OK));
 	// Linked SpanContext
-	ASSERT(data[74] == 0b10010001);
+	ASSERT(data[74] == 0b10010011); // 3 elements: traceID.first, traceID.second, spanID
 	ASSERT(data[75] == 0xcf);
 	ASSERT(swapUint64BE(&data[76]) == 300);
 	ASSERT(data[84] == 0xcf);
@@ -654,7 +664,7 @@ TEST_CASE("/flow/Tracing/FastUDPMessagePackEncoding") {
 	ASSERT(data[93] == 0xcf);
 	ASSERT(swapUint64BE(&data[94]) == 400);
 	// Events
-	ASSERT(data[102] == 0b10010001); // empty
+	ASSERT(data[102] == 0b10010011); // 3 elements: name, time, attributes
 	ASSERT(readMPString(&data[103]) == "event1");
 	ASSERT(data[110] == 0xcb);
 	ASSERT(swapDoubleBE(&data[111]) == 100.101);
diff --git a/fdbclient/VersionedMap.cpp b/fdbclient/VersionedMap.cpp
deleted file mode 100644
index 7507ee30581..00000000000
--- a/fdbclient/VersionedMap.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * VersionedMap.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/VersionedMap.h"
-#include "flow/TreeBenchmark.h"
-#include "flow/UnitTest.h"
-
-template <typename K>
-struct VersionedMapHarness {
-	using map = VersionedMap<K, int>;
-	using key_type = K;
-
-	struct result {
-		typename map::iterator it;
-
-		result(typename map::iterator it) : it(it) {}
-
-		result& operator++() {
-			++it;
-			return *this;
-		}
-
-		const K& operator*() const { return it.key(); }
-
-		const K& operator->() const { return it.key(); }
-
-		bool operator==(result const& k) const { return it == k.it; }
-		bool operator!=(result const& k) const { return !(*this == k); }
-	};
-
-	map s;
-
-	void insert(K const& k) { s.insert(k, 1); }
-	result find(K const& k) const { return result(s.atLatest().find(k)); }
-	result not_found() const { return result(s.atLatest().end()); }
-	result begin() const { return result(s.atLatest().begin()); }
-	result end() const { return result(s.atLatest().end()); }
-	result lower_bound(K const& k) const { return result(s.atLatest().lower_bound(k)); }
-	result upper_bound(K const& k) const { return result(s.atLatest().upper_bound(k)); }
-	void erase(K const& k) { s.erase(k); }
-};
-
-TEST_CASE("performance/map/int/VersionedMap") {
-	VersionedMapHarness<int> tree;
-
-	treeBenchmark(tree, *randomInt);
-
-	return Void();
-}
-
-TEST_CASE("performance/map/StringRef/VersionedMap") {
-	Arena arena;
-	VersionedMapHarness<StringRef> tree;
-
-	treeBenchmark(tree, [&arena]() { return randomStr(arena); });
-
-	return Void();
-}
-
-void forceLinkVersionedMapTests() {}
diff --git a/fdbclient/WriteMap.cpp b/fdbclient/WriteMap.cpp
index c36258cdc70..ca0ad365f2e 100644
--- a/fdbclient/WriteMap.cpp
+++ b/fdbclient/WriteMap.cpp
@@ -104,22 +104,26 @@ void WriteMap::mutate(KeyRef key, MutationRef::Type operation, ValueRef param, b
 			it.tree.clear();
 			OperationStack op(RYWMutation(Optional<StringRef>(), MutationRef::SetValue));
 			coalesceOver(op, RYWMutation(param, operation), *arena);
-			PTreeImpl::insert(
-			    writes,
-			    ver,
-			    WriteMapEntry(
-			        key, std::move(op), true, following_conflict, is_conflict, following_unreadable, is_unreadable));
+			PTreeImpl::insert(writes,
+			                  ver,
+			                  WriteMapEntry(key,
+			                                std::move(op),
+			                                FollowingKeysCleared::True,
+			                                FollowingKeysConflict(following_conflict),
+			                                IsConflict(is_conflict),
+			                                FollowingKeysUnreadable(following_unreadable),
+			                                IsUnreadable(is_unreadable)));
 		} else {
 			it.tree.clear();
 			PTreeImpl::insert(writes,
 			                  ver,
 			                  WriteMapEntry(key,
 			                                OperationStack(RYWMutation(param, operation)),
-			                                is_cleared,
-			                                following_conflict,
-			                                is_conflict,
-			                                following_unreadable,
-			                                is_unreadable));
+			                                FollowingKeysCleared(is_cleared),
+			                                FollowingKeysConflict(following_conflict),
+			                                IsConflict(is_conflict),
+			                                FollowingKeysUnreadable(following_unreadable),
+			                                IsUnreadable(is_unreadable)));
 		}
 	} else {
 		if (!it.is_unreadable() &&
@@ -130,11 +134,11 @@ void WriteMap::mutate(KeyRef key, MutationRef::Type operation, ValueRef param, b
 			                  ver,
 			                  WriteMapEntry(key,
 			                                OperationStack(RYWMutation(param, operation)),
-			                                is_cleared,
-			                                following_conflict,
-			                                is_conflict,
-			                                following_unreadable,
-			                                is_unreadable));
+			                                FollowingKeysCleared(is_cleared),
+			                                FollowingKeysConflict(following_conflict),
+			                                IsConflict(is_conflict),
+			                                FollowingKeysUnreadable(following_unreadable),
+			                                IsUnreadable(is_unreadable)));
 		} else {
 			WriteMapEntry e(it.entry());
 			e.is_conflict = is_conflict;
@@ -192,14 +196,26 @@ void WriteMap::clear(KeyRangeRef keys, bool addConflict) {
 	                  ExtStringRef(keys.end, end_coalesce_clear ? 1 : 0));
 
 	if (insert_begin)
-		PTreeImpl::insert(writes, ver, WriteMapEntry(keys.begin, OperationStack(), true, true, true, false, false));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(keys.begin,
+		                                OperationStack(),
+		                                FollowingKeysCleared::True,
+		                                FollowingKeysConflict::True,
+		                                IsConflict::True,
+		                                FollowingKeysUnreadable::False,
+		                                IsUnreadable::False));
 
 	if (insert_end)
-		PTreeImpl::insert(
-		    writes,
-		    ver,
-		    WriteMapEntry(
-		        keys.end, OperationStack(), end_cleared, end_conflict, end_conflict, end_unreadable, end_unreadable));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(keys.end,
+		                                OperationStack(),
+		                                FollowingKeysCleared(end_cleared),
+		                                FollowingKeysConflict(end_conflict),
+		                                IsConflict(end_conflict),
+		                                FollowingKeysUnreadable(end_unreadable),
+		                                IsUnreadable(end_unreadable)));
 }
 
 void WriteMap::addUnmodifiedAndUnreadableRange(KeyRangeRef keys) {
@@ -231,14 +247,26 @@ void WriteMap::addUnmodifiedAndUnreadableRange(KeyRangeRef keys) {
 	                  ExtStringRef(keys.end, end_coalesce_unmodified ? 1 : 0));
 
 	if (insert_begin)
-		PTreeImpl::insert(writes, ver, WriteMapEntry(keys.begin, OperationStack(), false, false, false, true, true));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(keys.begin,
+		                                OperationStack(),
+		                                FollowingKeysCleared::False,
+		                                FollowingKeysConflict::False,
+		                                IsConflict::False,
+		                                FollowingKeysUnreadable::True,
+		                                IsUnreadable::True));
 
 	if (insert_end)
-		PTreeImpl::insert(
-		    writes,
-		    ver,
-		    WriteMapEntry(
-		        keys.end, OperationStack(), end_cleared, end_conflict, end_conflict, end_unreadable, end_unreadable));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(keys.end,
+		                                OperationStack(),
+		                                FollowingKeysCleared(end_cleared),
+		                                FollowingKeysConflict(end_conflict),
+		                                IsConflict(end_conflict),
+		                                FollowingKeysUnreadable(end_unreadable),
+		                                IsUnreadable(end_unreadable)));
 }
 
 void WriteMap::addConflictRange(KeyRangeRef keys) {
@@ -256,11 +284,11 @@ void WriteMap::addConflictRange(KeyRangeRef keys) {
 		}
 		insertions.push_back(WriteMapEntry(keys.begin,
 		                                   it.is_operation() ? OperationStack(it.op()) : OperationStack(),
-		                                   it.entry().following_keys_cleared,
-		                                   true,
-		                                   true,
-		                                   it.entry().following_keys_unreadable,
-		                                   it.entry().is_unreadable));
+		                                   FollowingKeysCleared(it.entry().following_keys_cleared),
+		                                   FollowingKeysConflict::True,
+		                                   IsConflict::True,
+		                                   FollowingKeysUnreadable(it.entry().following_keys_unreadable),
+		                                   IsUnreadable(it.entry().is_unreadable)));
 	}
 
 	while (it.endKey() < keys.end) {
@@ -282,8 +310,13 @@ void WriteMap::addConflictRange(KeyRangeRef keys) {
 		++it;
 
 		if (!it.keyAtBegin() || it.beginKey() != keys.end) {
-			insertions.push_back(
-			    WriteMapEntry(keys.end, OperationStack(), isCleared, false, false, followingUnreadable, isUnreadable));
+			insertions.push_back(WriteMapEntry(keys.end,
+			                                   OperationStack(),
+			                                   FollowingKeysCleared(isCleared),
+			                                   FollowingKeysConflict::False,
+			                                   IsConflict::False,
+			                                   FollowingKeysUnreadable(followingUnreadable),
+			                                   IsUnreadable(isUnreadable)));
 		}
 	}
 
@@ -577,20 +610,28 @@ void WriteMap::clearNoConflict(KeyRangeRef keys) {
 	                  ExtStringRef(keys.end, end_coalesce_clear ? 1 : 0));
 
 	for (int i = 0; i < conflict_ranges.size(); i++) {
-		PTreeImpl::insert(
-		    writes,
-		    ver,
-		    WriteMapEntry(
-		        conflict_ranges[i].toArenaOrRef(*arena), OperationStack(), true, conflicted, conflicted, false, false));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(conflict_ranges[i].toArenaOrRef(*arena),
+		                                OperationStack(),
+		                                FollowingKeysCleared::True,
+		                                FollowingKeysConflict(conflicted),
+		                                IsConflict(conflicted),
+		                                FollowingKeysUnreadable::False,
+		                                IsUnreadable::False));
 		conflicted = !conflicted;
 	}
 
 	ASSERT(conflicted != lastConflicted);
 
 	if (insert_end)
-		PTreeImpl::insert(
-		    writes,
-		    ver,
-		    WriteMapEntry(
-		        keys.end, OperationStack(), end_cleared, end_conflict, end_conflict, end_unreadable, end_unreadable));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(keys.end,
+		                                OperationStack(),
+		                                FollowingKeysCleared(end_cleared),
+		                                FollowingKeysConflict(end_conflict),
+		                                IsConflict(end_conflict),
+		                                FollowingKeysUnreadable(end_unreadable),
+		                                IsUnreadable(end_unreadable)));
 }
diff --git a/flowbench/BenchIdempotencyIds.cpp b/fdbclient/bench/BenchIdempotencyIds.cpp
similarity index 100%
rename from flowbench/BenchIdempotencyIds.cpp
rename to fdbclient/bench/BenchIdempotencyIds.cpp
diff --git a/flowbench/BenchIterate.cpp b/fdbclient/bench/BenchIterate.cpp
similarity index 98%
rename from flowbench/BenchIterate.cpp
rename to fdbclient/bench/BenchIterate.cpp
index 1f6103c7ab6..15a461e2bbf 100644
--- a/flowbench/BenchIterate.cpp
+++ b/fdbclient/bench/BenchIterate.cpp
@@ -23,9 +23,9 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/MutationList.h"
+#include "GlobalData.h"
 #include "flow/Arena.h"
 #include "flow/FastAlloc.h"
-#include "flowbench/GlobalData.h"
 
 void populate(Standalone<VectorRef<MutationRef>>& mutations, size_t items, size_t size, KeyRef key, ValueRef value) {
 	mutations = Standalone<VectorRef<MutationRef>>{};
diff --git a/fdbserver/include/fdbserver/ServerDBInfo.h b/fdbclient/bench/BenchMain.cpp
similarity index 82%
rename from fdbserver/include/fdbserver/ServerDBInfo.h
rename to fdbclient/bench/BenchMain.cpp
index 4871485ff28..1502e9db3ed 100644
--- a/fdbserver/include/fdbserver/ServerDBInfo.h
+++ b/fdbclient/bench/BenchMain.cpp
@@ -1,5 +1,5 @@
 /*
- * ServerDBInfo.h
+ * BenchMain.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,8 @@
  * limitations under the License.
  */
 
-#ifndef FDBSERVER_SERVERDBINFO_H
-#define FDBSERVER_SERVERDBINFO_H
-#pragma once
+#include "flow/BenchMain.h"
 
-#include "fdbserver/ServerDBInfo.actor.h"
-
-#endif
+int main(int argc, char** argv) {
+	return runBenchmarks(argc, argv);
+}
diff --git a/flowbench/BenchMetadataCheck.cpp b/fdbclient/bench/BenchMetadataCheck.cpp
similarity index 100%
rename from flowbench/BenchMetadataCheck.cpp
rename to fdbclient/bench/BenchMetadataCheck.cpp
diff --git a/flowbench/BenchPopulate.cpp b/fdbclient/bench/BenchPopulate.cpp
similarity index 98%
rename from flowbench/BenchPopulate.cpp
rename to fdbclient/bench/BenchPopulate.cpp
index 3b9b3c6eadf..3298359539c 100644
--- a/flowbench/BenchPopulate.cpp
+++ b/fdbclient/bench/BenchPopulate.cpp
@@ -22,9 +22,9 @@
 
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
+#include "GlobalData.h"
 #include "flow/Arena.h"
 #include "flow/FastAlloc.h"
-#include "flowbench/GlobalData.h"
 
 static constexpr bool EMPLACE_BACK = true;
 static constexpr bool PUSH_BACK = false;
diff --git a/flowbench/BenchTempTagMessages.cpp b/fdbclient/bench/BenchTempTagMessages.cpp
similarity index 100%
rename from flowbench/BenchTempTagMessages.cpp
rename to fdbclient/bench/BenchTempTagMessages.cpp
diff --git a/flowbench/BenchVersionVector.cpp b/fdbclient/bench/BenchVersionVector.cpp
similarity index 100%
rename from flowbench/BenchVersionVector.cpp
rename to fdbclient/bench/BenchVersionVector.cpp
diff --git a/flowbench/BenchVersionVectorSerialization.cpp b/fdbclient/bench/BenchVersionVectorSerialization.cpp
similarity index 100%
rename from flowbench/BenchVersionVectorSerialization.cpp
rename to fdbclient/bench/BenchVersionVectorSerialization.cpp
diff --git a/fdbclient/bench/BenchVersionedMap.cpp b/fdbclient/bench/BenchVersionedMap.cpp
new file mode 100644
index 00000000000..064ee56bc26
--- /dev/null
+++ b/fdbclient/bench/BenchVersionedMap.cpp
@@ -0,0 +1,564 @@
+/*
+ * BenchVersionedMap.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "fdbclient/VersionedMap.h"
+
+#include <algorithm>
+#include <climits>
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace {
+
+constexpr uint32_t versionedMapBenchmarkSeed = 12345;
+
+void resetVersionedMapBenchmarkRandom() {
+	// VersionedMap priorities come from deterministicRandom(); keep benchmark tree shape stable across iterations.
+	setThreadLocalDeterministicRandomSeed(versionedMapBenchmarkSeed);
+}
+
+template <typename K>
+struct VersionedMapHarness {
+	using map = VersionedMap<K, int>;
+	using key_type = K;
+
+	struct result {
+		typename map::iterator it;
+
+		explicit result(typename map::iterator it) : it(it) {}
+
+		result& operator++() {
+			++it;
+			return *this;
+		}
+
+		const K& operator*() const { return it.key(); }
+
+		const K& operator->() const { return it.key(); }
+
+		bool operator==(result const& k) const { return it == k.it; }
+		bool operator!=(result const& k) const { return !(*this == k); }
+	};
+
+	map s;
+
+	void insert(K const& k) { s.insert(k, 1); }
+	result find(K const& k) const { return result(s.latestFind(k)); }
+	result not_found() const { return result(s.latestEnd()); }
+	result begin() const { return result(s.latestBegin()); }
+	result end() const { return result(s.latestEnd()); }
+	result lower_bound(K const& k) const { return result(s.latestLowerBound(k)); }
+	result upper_bound(K const& k) const { return result(s.latestUpperBound(k)); }
+	result last_less_or_equal(K const& k) const { return result(s.latestLastLessOrEqual(k)); }
+	result last_less(K const& k) const { return result(s.latestLastLess(k)); }
+	void erase(K const& k) { s.erase(k); }
+};
+
+template <class K>
+void sortUnique(std::vector<K>& keys) {
+	std::sort(keys.begin(), keys.end());
+	keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
+}
+
+struct IntFixture {
+	using key_type = int;
+
+	std::vector<int> keys;
+	VersionedMapHarness<int> tree;
+
+	static std::unique_ptr<IntFixture> create(int64_t keyCount, bool populateTree, bool uniqueSortedKeys = false) {
+		auto fixture = std::make_unique<IntFixture>();
+		resetVersionedMapBenchmarkRandom();
+		fixture->keys.reserve(keyCount);
+		std::mt19937_64 rng(0x4d595df4d0f33173ULL);
+		std::uniform_int_distribution<int> dist(0, INT_MAX);
+		for (int64_t i = 0; i < keyCount; ++i) {
+			fixture->keys.push_back(dist(rng));
+		}
+		if (populateTree) {
+			for (const auto& key : fixture->keys) {
+				fixture->tree.insert(key);
+			}
+		}
+		if (uniqueSortedKeys) {
+			// Duplicate inserts update the existing tree node, so the tree and key list both contain the same unique
+			// keys after this. Sorting after population preserves the random insertion stream used to build the tree.
+			sortUnique(fixture->keys);
+		}
+		return fixture;
+	}
+};
+
+struct StringRefFixture {
+	using key_type = StringRef;
+
+	Arena arena;
+	std::vector<StringRef> keys;
+	VersionedMapHarness<StringRef> tree;
+
+	static std::unique_ptr<StringRefFixture> create(int64_t keyCount,
+	                                                bool populateTree,
+	                                                bool uniqueSortedKeys = false) {
+		auto fixture = std::make_unique<StringRefFixture>();
+		resetVersionedMapBenchmarkRandom();
+		fixture->keys.reserve(keyCount);
+		std::mt19937_64 rng(0x46a2f9b1d8c3e57bULL);
+		constexpr char alphabet[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+		constexpr int keySize = 100;
+		std::string key;
+		key.resize(keySize);
+		for (int64_t i = 0; i < keyCount; ++i) {
+			for (int j = 0; j < keySize; ++j) {
+				key[j] = alphabet[rng() % (sizeof(alphabet) - 1)];
+			}
+			fixture->keys.push_back(StringRef(fixture->arena, key));
+		}
+		if (populateTree) {
+			for (const auto& keyRef : fixture->keys) {
+				fixture->tree.insert(keyRef);
+			}
+		}
+		if (uniqueSortedKeys) {
+			// Duplicate inserts update the existing tree node, so the tree and key list both contain the same unique
+			// keys after this. Sorting after population preserves the random insertion stream used to build the tree.
+			sortUnique(fixture->keys);
+		}
+		return fixture;
+	}
+};
+
+template <class Fixture>
+static void bench_versioned_map_insert(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, false);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			fixture->tree.insert(key);
+		}
+		benchmark::DoNotOptimize(&fixture->tree);
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(keyCount * state.iterations());
+}
+
+template <class Fixture>
+static void bench_versioned_map_find(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			auto found = fixture->tree.find(key);
+			ASSERT(found != fixture->tree.not_found());
+			benchmark::DoNotOptimize(found);
+		}
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(keyCount * state.iterations());
+}
+
+template <class Fixture>
+static void bench_versioned_map_lower_bound(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			auto found = fixture->tree.lower_bound(key);
+			ASSERT(found != fixture->tree.not_found());
+			benchmark::DoNotOptimize(found);
+		}
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(keyCount * state.iterations());
+}
+
+template <class Fixture>
+static void bench_versioned_map_upper_bound(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			auto found = fixture->tree.upper_bound(key);
+			benchmark::DoNotOptimize(found);
+		}
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(keyCount * state.iterations());
+}
+
+template <class Fixture>
+static void bench_versioned_map_last_less_or_equal(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			auto found = fixture->tree.last_less_or_equal(key);
+			ASSERT(found != fixture->tree.not_found());
+			ASSERT(key == *found);
+			benchmark::DoNotOptimize(found);
+		}
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(keyCount * state.iterations());
+}
+
+template <class Fixture>
+static void bench_versioned_map_last_less(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	int64_t foundCount = 0;
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true, true);
+		state.ResumeTiming();
+
+		for (int64_t i = 1; i < fixture->keys.size(); ++i) {
+			auto found = fixture->tree.last_less(fixture->keys[i]);
+			ASSERT(found != fixture->tree.not_found());
+			ASSERT(fixture->keys[i - 1] == *found);
+			benchmark::DoNotOptimize(found);
+		}
+		foundCount += fixture->keys.size() - 1;
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(foundCount);
+}
+
+template <class Fixture>
+static void bench_versioned_map_scan(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	int64_t scanned = 0;
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true, true);
+		state.ResumeTiming();
+
+		auto iter = fixture->tree.lower_bound(fixture->keys.front());
+		for (const auto& key : fixture->keys) {
+			ASSERT(iter != fixture->tree.end());
+			ASSERT(key == *iter);
+			++iter;
+		}
+		scanned += fixture->keys.size();
+		ASSERT(iter == fixture->tree.end());
+		benchmark::DoNotOptimize(iter);
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(scanned);
+}
+
+template <class Fixture>
+static void bench_versioned_map_find_sorted(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	int64_t foundCount = 0;
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true, true);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			auto found = fixture->tree.find(key);
+			ASSERT(found != fixture->tree.end());
+			benchmark::DoNotOptimize(found);
+		}
+		foundCount += fixture->keys.size();
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(foundCount);
+}
+
+template <class Fixture>
+static void bench_versioned_map_erase(benchmark::State& state) {
+	const int64_t keyCount = state.range(0);
+	int64_t erased = 0;
+	for (auto _ : state) {
+		state.PauseTiming();
+		auto fixture = Fixture::create(keyCount, true, true);
+		std::mt19937_64 rng(0x91e10da5c79e7b1dULL);
+		std::shuffle(fixture->keys.begin(), fixture->keys.end(), rng);
+		state.ResumeTiming();
+
+		for (const auto& key : fixture->keys) {
+			fixture->tree.erase(key);
+		}
+		erased += fixture->keys.size();
+		ASSERT(fixture->tree.begin() == fixture->tree.end());
+		benchmark::DoNotOptimize(&fixture->tree);
+
+		state.PauseTiming();
+		fixture.reset();
+		state.ResumeTiming();
+	}
+	state.SetItemsProcessed(erased);
+}
+
+static void setKOpsCounter(benchmark::State& state, const char* name, int64_t opCount, double elapsed) {
+	state.counters[name] = elapsed == 0 ? 0 : (opCount / 1000.0) / elapsed;
+}
+
+static void bench_versioned_map_multiversion_int(benchmark::State& state) {
+	constexpr int initialKeys = 20000;
+	constexpr int versions = 50000;
+	constexpr int retainedVersions = 5000;
+	constexpr int forgetEvery = 1000;
+	constexpr int scanWidth = 8;
+
+	double populateElapsed = 0;
+	double createElapsed = 0;
+	double latestInsertElapsed = 0;
+	double latestEraseElapsed = 0;
+	double historicalFindElapsed = 0;
+	double historicalLowerBoundElapsed = 0;
+	double historicalScanElapsed = 0;
+	double compactForgetElapsed = 0;
+
+	int64_t populateOps = 0;
+	int64_t createOps = 0;
+	int64_t latestInsertOps = 0;
+	int64_t latestEraseOps = 0;
+	int64_t historicalFindOps = 0;
+	int64_t historicalLowerBoundOps = 0;
+	int64_t historicalScanOps = 0;
+	int64_t compactForgetOps = 0;
+
+	for (auto _ : state) {
+		resetVersionedMapBenchmarkRandom();
+		double totalStart = timer();
+		VersionedMap<int, int> map;
+		map.createNewVersion(1);
+
+		double start = timer();
+		for (int key = 0; key < initialKeys; ++key) {
+			map.insert(key, key);
+		}
+		populateElapsed += timer() - start;
+		populateOps += initialKeys;
+
+		for (int version = 2; version <= versions; ++version) {
+			start = timer();
+			map.createNewVersion(version);
+			createElapsed += timer() - start;
+			++createOps;
+
+			start = timer();
+			map.insert(version % initialKeys, version);
+			map.insert(initialKeys + version, version);
+			latestInsertElapsed += timer() - start;
+			latestInsertOps += 2;
+
+			if (version > retainedVersions + 1) {
+				start = timer();
+				map.erase(initialKeys + version - retainedVersions);
+				latestEraseElapsed += timer() - start;
+				++latestEraseOps;
+			}
+
+			Version readVersion = std::max<Version>(map.getOldestVersion(), version - retainedVersions / 2);
+			readVersion = std::max<Version>(readVersion, 1);
+
+			start = timer();
+			auto historicalFind = map.at(readVersion).find(int(readVersion % initialKeys));
+			ASSERT(historicalFind);
+			historicalFindElapsed += timer() - start;
+			++historicalFindOps;
+
+			start = timer();
+			auto historicalLower = map.at(readVersion).lower_bound(int((readVersion * 17) % initialKeys));
+			ASSERT(historicalLower);
+			historicalLowerBoundElapsed += timer() - start;
+			++historicalLowerBoundOps;
+
+			start = timer();
+			auto scan = map.at(readVersion).lower_bound(int((readVersion * 31) % (initialKeys - scanWidth)));
+			for (int i = 0; i < scanWidth; ++i) {
+				ASSERT(scan);
+				++scan;
+				++historicalScanOps;
+			}
+			historicalScanElapsed += timer() - start;
+
+			if (version > retainedVersions && version % forgetEvery == 0) {
+				Version cutoff = version - retainedVersions;
+				start = timer();
+				map.compact(cutoff);
+				map.forgetVersionsBefore(cutoff);
+				compactForgetElapsed += timer() - start;
+				compactForgetOps += 2;
+			}
+		}
+
+		benchmark::DoNotOptimize(map.getLatestVersion());
+		double iterationElapsed = timer() - totalStart;
+		state.SetIterationTime(iterationElapsed);
+	}
+
+	state.SetItemsProcessed((populateOps + createOps + latestInsertOps + latestEraseOps + historicalFindOps +
+	                         historicalLowerBoundOps + historicalScanOps + compactForgetOps));
+	setKOpsCounter(state, "populate_kops", populateOps, populateElapsed);
+	setKOpsCounter(state, "create_version_kops", createOps, createElapsed);
+	setKOpsCounter(state, "latest_insert_kops", latestInsertOps, latestInsertElapsed);
+	setKOpsCounter(state, "latest_erase_kops", latestEraseOps, latestEraseElapsed);
+	setKOpsCounter(state, "historical_find_kops", historicalFindOps, historicalFindElapsed);
+	setKOpsCounter(state, "historical_lower_bound_kops", historicalLowerBoundOps, historicalLowerBoundElapsed);
+	setKOpsCounter(state, "historical_scan_kops", historicalScanOps, historicalScanElapsed);
+	setKOpsCounter(state, "compact_forget_kops", compactForgetOps, compactForgetElapsed);
+}
+
+} // namespace
+
+BENCHMARK_TEMPLATE(bench_versioned_map_insert, IntFixture)
+    ->Name("VersionedMap/int/insert")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_find, IntFixture)
+    ->Name("VersionedMap/int/find")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_lower_bound, IntFixture)
+    ->Name("VersionedMap/int/lower_bound")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_upper_bound, IntFixture)
+    ->Name("VersionedMap/int/upper_bound")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_last_less_or_equal, IntFixture)
+    ->Name("VersionedMap/int/last_less_or_equal")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_last_less, IntFixture)
+    ->Name("VersionedMap/int/last_less")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_scan, IntFixture)
+    ->Name("VersionedMap/int/scan")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_find_sorted, IntFixture)
+    ->Name("VersionedMap/int/find_sorted")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_erase, IntFixture)
+    ->Name("VersionedMap/int/erase")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+
+BENCHMARK_TEMPLATE(bench_versioned_map_insert, StringRefFixture)
+    ->Name("VersionedMap/StringRef/insert")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_find, StringRefFixture)
+    ->Name("VersionedMap/StringRef/find")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_lower_bound, StringRefFixture)
+    ->Name("VersionedMap/StringRef/lower_bound")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_upper_bound, StringRefFixture)
+    ->Name("VersionedMap/StringRef/upper_bound")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_last_less_or_equal, StringRefFixture)
+    ->Name("VersionedMap/StringRef/last_less_or_equal")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_last_less, StringRefFixture)
+    ->Name("VersionedMap/StringRef/last_less")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_scan, StringRefFixture)
+    ->Name("VersionedMap/StringRef/scan")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_find_sorted, StringRefFixture)
+    ->Name("VersionedMap/StringRef/find_sorted")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+BENCHMARK_TEMPLATE(bench_versioned_map_erase, StringRefFixture)
+    ->Name("VersionedMap/StringRef/erase")
+    ->Arg(1000000)
+    ->Iterations(3)
+    ->Unit(benchmark::kMillisecond);
+
+BENCHMARK(bench_versioned_map_multiversion_int)
+    ->Name("VersionedMap/int/multiversion")
+    ->Iterations(3)
+    ->UseManualTime()
+    ->Unit(benchmark::kMillisecond);
diff --git a/fdbclient/bench/CMakeLists.txt b/fdbclient/bench/CMakeLists.txt
new file mode 100644
index 00000000000..63fef43158d
--- /dev/null
+++ b/fdbclient/bench/CMakeLists.txt
@@ -0,0 +1,12 @@
+include(FDBBenchmark)
+
+fdb_find_sources(FDBCLIENT_BENCH_SRCS)
+add_flow_target(EXECUTABLE NAME fdbclient_bench SRCS ${FDBCLIENT_BENCH_SRCS})
+
+fdb_setup_googlebenchmark()
+
+target_include_directories(
+  fdbclient_bench
+  PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
+
+target_link_libraries(fdbclient_bench PRIVATE Threads::Threads fdb_google_benchmark fdbclient)
diff --git a/flowbench/GlobalData.cpp b/fdbclient/bench/GlobalData.cpp
similarity index 98%
rename from flowbench/GlobalData.cpp
rename to fdbclient/bench/GlobalData.cpp
index 7585f310e73..69b811a4ed3 100644
--- a/flowbench/GlobalData.cpp
+++ b/fdbclient/bench/GlobalData.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/FDBTypes.h"
+#include "GlobalData.h"
 #include "flow/IRandom.h"
 
 static constexpr size_t globalDataSize = 1 << 20;
diff --git a/flowbench/include/flowbench/GlobalData.h b/fdbclient/bench/GlobalData.h
similarity index 88%
rename from flowbench/include/flowbench/GlobalData.h
rename to fdbclient/bench/GlobalData.h
index 5bed599c814..8d9e9955136 100644
--- a/flowbench/include/flowbench/GlobalData.h
+++ b/fdbclient/bench/GlobalData.h
@@ -18,18 +18,17 @@
  * limitations under the License.
  */
 
-#ifndef __FDBBENCH_GLOBALDATA_H__
-#define __FDBBENCH_GLOBALDATA_H__
+#ifndef FDB_FDBCLIENT_BENCH_GLOBALDATA_H
+#define FDB_FDBCLIENT_BENCH_GLOBALDATA_H
 
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
+#include "flow/flow.h"
 
 KeyValueRef getKV(size_t keySize, size_t valueSize);
 KeyRef getKey(size_t keySize);
 
-// Pre-generate a vector of T using a lambda then return them
-// via next() one by one with wrap-around
 template <typename T>
 struct InputGenerator {
 	InputGenerator() {}
diff --git a/fdbclient/include/fdbclient/AnnotateActor.h b/fdbclient/include/fdbclient/AnnotateActor.h
index df753d974fa..902ae0d86c5 100644
--- a/fdbclient/include/fdbclient/AnnotateActor.h
+++ b/fdbclient/include/fdbclient/AnnotateActor.h
@@ -33,7 +33,7 @@ struct AnnotateActor {
 
 	AnnotateActor() : set(false) {}
 
-	AnnotateActor(LineageReference* lineage) : set(false) {
+	explicit AnnotateActor(LineageReference* lineage) : set(false) {
 #ifdef ENABLE_SAMPLING
 		if (lineage->getPtr() != 0) {
 			index = g_network->getActorLineageSet().insert(*lineage);
diff --git a/fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h b/fdbclient/include/fdbclient/AsyncFileBlobStore.h
similarity index 62%
rename from fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h
rename to fdbclient/include/fdbclient/AsyncFileBlobStore.h
index 5ecea287e7e..81930e529a0 100644
--- a/fdbclient/include/fdbclient/AsyncFileS3BlobStore.actor.h
+++ b/fdbclient/include/fdbclient/AsyncFileBlobStore.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileS3BlobStore.actor.h
+ * AsyncFileBlobStore.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,14 +20,6 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H)
-#define FDBRPC_ASYNCFILEBLOBSTORE_ACTOR_G_H
-#include "fdbclient/AsyncFileS3BlobStore.actor.g.h"
-#elif !defined(FDBRPC_ASYNCFILES3BLOBSTORE_ACTOR_H)
-#define FDBRPC_ASYNCFILES3BLOBSTORE_ACTOR_H
-
 #include <sstream>
 #include <time.h>
 
@@ -35,33 +27,32 @@
 #include "flow/serialize.h"
 #include "flow/Net2Packet.h"
 #include "flow/IRateControl.h"
-#include "fdbclient/S3BlobStore.h"
+#include "fdbclient/IBlobStore.h"
 #include "md5/md5.h"
 #include "libb64/encode.h"
 #include <openssl/sha.h>
-#include "flow/actorcompiler.h" // This must be the last #include.
 
-ACTOR template <typename T>
+template <typename T>
 static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
 	try {
-		wait(success(f) || p.getFuture());
-		return f.get();
+		co_await (success(f) || p.getFuture());
+		co_return f.get();
 	} catch (Error& e) {
 		if (p.canBeSet())
 			p.sendError(e);
 		throw;
 	}
 }
-// This class represents a write-only file that lives in an S3-style blob store.  It writes using the REST API,
+// This class represents a write-only file that lives in a blob store.  It writes using the REST API,
 // using multi-part upload and beginning to transfer each part as soon as it is large enough.
 // All write operations file operations must be sequential and contiguous.
-// Limits on part sizes, upload speed, and concurrent uploads are taken from the S3BlobStoreEndpoint being used.
-class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCounted<AsyncFileS3BlobStoreWrite> {
+// Limits on part sizes, upload speed, and concurrent uploads are taken from the IBlobStoreEndpoint being used.
+class AsyncFileBlobStoreWrite final : public IAsyncFile, public ReferenceCounted<AsyncFileBlobStoreWrite> {
 public:
-	void addref() override { ReferenceCounted<AsyncFileS3BlobStoreWrite>::addref(); }
-	void delref() override { ReferenceCounted<AsyncFileS3BlobStoreWrite>::delref(); }
+	void addref() override { ReferenceCounted<AsyncFileBlobStoreWrite>::addref(); }
+	void delref() override { ReferenceCounted<AsyncFileBlobStoreWrite>::delref(); }
 
-	virtual StringRef getClassName() override { return "AsyncFileS3BlobStoreWrite"_sr; }
+	StringRef getClassName() override { return "AsyncFileBlobStoreWrite"_sr; }
 
 	struct Part : ReferenceCounted<Part> {
 		Part(int n, int minSize, bool useSHA256)
@@ -120,8 +111,8 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 
 	Future<int> read(void* data, int length, int64_t offset) override { throw file_not_readable(); }
 
-	ACTOR static Future<Void> write_impl(Reference<AsyncFileS3BlobStoreWrite> f, const uint8_t* data, int length) {
-		state Part* p = f->m_parts.back().getPtr();
+	static Future<Void> write_impl(Reference<AsyncFileBlobStoreWrite> f, const uint8_t* data, int length) {
+		Part* p = f->m_parts.back().getPtr();
 		// If this write will cause the part to cross the min part size boundary then write to the boundary and start a
 		// new part.
 		while (p->length + length >= f->m_bstore->knobs.multipart_min_part_size) {
@@ -134,12 +125,11 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 			data = (const uint8_t*)data + finishlen;
 
 			// End current part (and start new one)
-			wait(f->endCurrentPart(f.getPtr(), true));
+			co_await f->endCurrentPart(f.getPtr(), true);
 			p = f->m_parts.back().getPtr();
 		}
 
 		p->write((const uint8_t*)data, length);
-		return Void();
 	}
 
 	Future<Void> write(void const* data, int length, int64_t offset) override {
@@ -148,7 +138,7 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 		m_cursor += length;
 
 		return m_error.getFuture() ||
-		       write_impl(Reference<AsyncFileS3BlobStoreWrite>::addRef(this), (const uint8_t*)data, length);
+		       write_impl(Reference<AsyncFileBlobStoreWrite>::addRef(this), (const uint8_t*)data, length);
 	}
 
 	Future<Void> truncate(int64_t size) override {
@@ -157,54 +147,52 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 		return Void();
 	}
 
-	ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) {
+	static Future<std::string> doPartUpload(AsyncFileBlobStoreWrite* f, Part* p) {
 		p->finalizeChecksum();
-		std::string upload_id = wait(f->getUploadID());
-		std::string etag = wait(f->m_bstore->uploadPart(
-		    f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->checksumString));
-		return etag;
+		std::string upload_id = co_await f->getUploadID();
+		std::string etag = co_await f->m_bstore->uploadPart(
+		    f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->checksumString);
+		co_return etag;
 	}
 
-	ACTOR static Future<Void> doFinishUpload(AsyncFileS3BlobStoreWrite* f) {
+	static Future<Void> doFinishUpload(AsyncFileBlobStoreWrite* f) {
 		// If there is only 1 part then it has not yet been uploaded so just write the whole file at once.
 		if (f->m_parts.size() == 1) {
 			Reference<Part> part = f->m_parts.back();
 			part->finalizeChecksum();
-			wait(f->m_bstore->writeEntireFileFromBuffer(
-			    f->m_bucket, f->m_object, &part->content, part->length, part->checksumString));
-			return Void();
+			co_await f->m_bstore->writeEntireFileFromBuffer(
+			    f->m_bucket, f->m_object, &part->content, part->length, part->checksumString);
+			co_return;
 		}
 
 		// There are at least 2 parts.  End the last part (which could be empty)
-		wait(f->endCurrentPart(f));
+		co_await f->endCurrentPart(f);
 
-		state S3BlobStoreEndpoint::MultiPartSetT partSet;
-		state std::vector<Reference<Part>>::iterator p;
+		IBlobStoreEndpoint::MultiPartSetT partSet;
+		std::vector<Reference<Part>>::iterator p;
 
 		// Wait for all the parts to be done to get their ETags, populate the partSet required to finish the object
 		// upload.
 		for (p = f->m_parts.begin(); p != f->m_parts.end(); ++p) {
-			std::string tag = wait((*p)->etag);
+			std::string tag = co_await (*p)->etag;
 			if ((*p)->length > 0) { // The last part might be empty and has to be omitted.
-				partSet[(*p)->number] = S3BlobStoreEndpoint::PartInfo(tag, (*p)->checksumString);
+				partSet[(*p)->number] = IBlobStoreEndpoint::PartInfo(tag, (*p)->checksumString);
 			}
 		}
 
 		// No need to wait for the upload ID here because the above loop waited for all the parts and each part required
 		// the upload ID so it is ready
-		Optional<std::string> checksumSHA256 =
-		    wait(f->m_bstore->finishMultiPartUpload(f->m_bucket, f->m_object, f->m_upload_id.get(), partSet));
+		Optional<std::string> checksumSHA256 = co_await f->m_bstore->finishMultiPartUpload(
+		    f->m_bucket, f->m_object, f->m_upload_id.get(), partSet, f->m_cursor);
 
 		// Log the checksum if present - this is just a hash of the multipart structure, not the object content
 		if (checksumSHA256.present()) {
-			TraceEvent(SevDebug, "AsyncFileS3BlobStoreMultipartUploadChecksum")
+			TraceEvent(SevDebug, "AsyncFileBlobStoreMultipartUploadChecksum")
 			    .detail("Bucket", f->m_bucket)
 			    .detail("Object", f->m_object)
 			    .detail("ChecksumSHA256", checksumSHA256.get())
 			    .detail("Note", "This is a hash of the multipart structure, not object content");
 		}
-
-		return Void();
 	}
 
 	// Ready once all data has been sent AND acknowledged from the remote side
@@ -237,7 +225,7 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 
 	int64_t debugFD() const override { return -1; }
 
-	~AsyncFileS3BlobStoreWrite() override {
+	~AsyncFileBlobStoreWrite() override {
 		m_upload_id.cancel();
 		m_finished.cancel();
 		m_parts.clear(); // Contains futures
@@ -246,7 +234,7 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 	std::string getFilename() const override { return m_object; }
 
 private:
-	Reference<S3BlobStoreEndpoint> m_bstore;
+	Reference<IBlobStoreEndpoint> m_bstore;
 	std::string m_bucket;
 	std::string m_object;
 
@@ -259,12 +247,12 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 	FlowLock m_concurrentUploads;
 
 	// End the current part and start uploading it, but also wait for a part to finish if too many are in transit.
-	ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) {
+	static Future<Void> endCurrentPart(AsyncFileBlobStoreWrite* f, bool startNew = false) {
 		if (f->m_parts.back()->length == 0)
-			return Void();
+			co_return;
 
 		// Wait for an upload slot to be available
-		wait(f->m_concurrentUploads.take());
+		co_await f->m_concurrentUploads.take();
 
 		// Do the upload, and if it fails forward errors to m_error and also stop if anything else sends an error to
 		// m_error Also, hold a releaser for the concurrent upload slot while all that is going on.
@@ -274,11 +262,9 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 
 		// Make a new part to write to
 		if (startNew)
-			f->m_parts.push_back(Reference<Part>(new Part(f->m_parts.size() + 1,
-			                                              f->m_bstore->knobs.multipart_min_part_size,
-			                                              f->m_bstore->knobs.enable_object_integrity_check)));
-
-		return Void();
+			f->m_parts.push_back(makeReference<Part>(f->m_parts.size() + 1,
+			                                         f->m_bstore->knobs.multipart_min_part_size,
+			                                         f->m_bstore->knobs.enable_object_integrity_check));
 	}
 
 	Future<std::string> getUploadID() {
@@ -288,7 +274,7 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 	}
 
 public:
-	AsyncFileS3BlobStoreWrite(Reference<S3BlobStoreEndpoint> bstore, std::string bucket, std::string object)
+	AsyncFileBlobStoreWrite(Reference<IBlobStoreEndpoint> bstore, std::string bucket, std::string object)
 	  : m_bstore(bstore), m_bucket(bucket), m_object(object), m_cursor(0),
 	    m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
 
@@ -298,46 +284,13 @@ class AsyncFileS3BlobStoreWrite final : public IAsyncFile, public ReferenceCount
 	}
 };
 
-// Different Download Approaches:
-//
-// 1. AsyncFileS3BlobStoreRead::read
-//    - Always uses range requests (Range: bytes=0-24)
-//    - ❌ NO checksum verification - can't use x-amz-checksum-mode: ENABLED
-//    - Used by backup/restore operations through BackupContainerS3BlobStore::readFile()
-//
-// 2. S3BlobStoreEndpoint::readEntireFile (for small files):
-//    - ✅ HAS checksum verification using x-amz-checksum-mode: ENABLED
-//    - Used for small file downloads where full object retrieval is efficient
-//
-// 3. S3Client::copyDownFile (for large files):
-//    - Uses range-based downloads in parallel parts
-//    - ✅ Validates overall file checksum after download using XXH64 stored in tags/companion files
-//
-// Why Range Requests Can't Use S3 Checksums:
-// - S3 limitation: x-amz-checksum-mode: ENABLED only works for full object downloads
-// - Range requests (Range: bytes=X-Y) cannot be checksum-verified by S3 because:
-//   * The checksum is calculated for the entire object, not arbitrary byte ranges
-//   * S3 doesn't know how to verify partial content against a full-object checksum
-//
-// - Upload: SHA256 checksums stored with multipart uploads ✅
-// - Download:
-//   * Small files: S3 SHA256 verification ✅
-//   * Large files: Custom XXH64 verification after complete download ✅
-//   * Range requests: No S3 verification ❌ (but still get FoundationDB's own checksums)
-//
-// - Range requests are more efficient for large files and allow parallel downloads
-// - S3's checksum verification wouldn't work with range requests anyway
-// - You still get protection through FoundationDB's XXH64 checksums
-// - The multipart upload checksums primarily protect against transmission errors during upload
-// - The XXH64 checksums are used to verify the download.
-
-// This class represents a read-only file that lives in an S3-style blob store.  It reads using the REST API.
-class AsyncFileS3BlobStoreRead final : public IAsyncFile, public ReferenceCounted<AsyncFileS3BlobStoreRead> {
+// This class represents a read-only file that lives in a blob store.  It reads using the REST API.
+class AsyncFileBlobStoreRead final : public IAsyncFile, public ReferenceCounted<AsyncFileBlobStoreRead> {
 public:
-	void addref() override { ReferenceCounted<AsyncFileS3BlobStoreRead>::addref(); }
-	void delref() override { ReferenceCounted<AsyncFileS3BlobStoreRead>::delref(); }
+	void addref() override { ReferenceCounted<AsyncFileBlobStoreRead>::addref(); }
+	void delref() override { ReferenceCounted<AsyncFileBlobStoreRead>::delref(); }
 
-	virtual StringRef getClassName() override { return "AsyncFileS3BlobStoreRead"_sr; }
+	StringRef getClassName() override { return "AsyncFileBlobStoreRead"_sr; }
 
 	Future<int> read(void* data, int length, int64_t offset) override;
 
@@ -359,16 +312,13 @@ class AsyncFileS3BlobStoreRead final : public IAsyncFile, public ReferenceCounte
 
 	std::string getFilename() const override { return m_object; }
 
-	~AsyncFileS3BlobStoreRead() override {}
+	~AsyncFileBlobStoreRead() override = default;
 
-	Reference<S3BlobStoreEndpoint> m_bstore;
+	Reference<IBlobStoreEndpoint> m_bstore;
 	std::string m_bucket;
 	std::string m_object;
 	mutable Future<int64_t> m_size;
 
-	AsyncFileS3BlobStoreRead(Reference<S3BlobStoreEndpoint> bstore, std::string bucket, std::string object)
+	AsyncFileBlobStoreRead(Reference<IBlobStoreEndpoint> bstore, std::string bucket, std::string object)
 	  : m_bstore(bstore), m_bucket(bucket), m_object(object) {}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/AsyncTaskThread.h b/fdbclient/include/fdbclient/AsyncTaskThread.h
deleted file mode 100644
index 62101ad9470..00000000000
--- a/fdbclient/include/fdbclient/AsyncTaskThread.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * AsyncTaskThread.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __ASYNC_TASK_THREAD_H__
-#define __ASYNC_TASK_THREAD_H__
-
-#include <condition_variable>
-#include <memory>
-#include <mutex>
-#include <thread>
-
-#include "flow/network.h"
-#include "flow/ThreadHelper.actor.h"
-#include "flow/ThreadSafeQueue.h"
-
-class IAsyncTask {
-public:
-	virtual void operator()() = 0;
-	virtual ~IAsyncTask() = default;
-	virtual bool isTerminate() const = 0;
-};
-
-template <class F>
-class AsyncTask final : public IAsyncTask {
-	F func;
-
-public:
-	AsyncTask(const F& func) : func(func) {}
-
-	void operator()() override { func(); }
-	bool isTerminate() const override { return false; }
-};
-
-class AsyncTaskThread {
-	ThreadSafeQueue<std::unique_ptr<IAsyncTask>> queue;
-	std::condition_variable cv;
-	std::mutex m;
-	std::thread thread;
-
-	static void run(AsyncTaskThread* self);
-
-	template <class F>
-	void addTask(const F& func) {
-		bool wakeUp = false;
-		{
-			std::lock_guard<std::mutex> g(m);
-			wakeUp = queue.push(std::make_unique<AsyncTask<F>>(func));
-		}
-		if (wakeUp) {
-			cv.notify_one();
-		}
-	}
-
-	static const double meanDelay;
-
-public:
-	AsyncTaskThread();
-
-	// Warning: This destructor can hang if a task hangs, so it is
-	// up to the caller to prevent tasks from hanging indefinitely
-	~AsyncTaskThread();
-
-	template <class F>
-	auto execAsync(const F& func, TaskPriority priority = TaskPriority::DefaultOnMainThread)
-	    -> Future<decltype(func())> {
-		if (g_network->isSimulated()) {
-			return map(delayJittered(meanDelay), [func](Void _) { return func(); });
-		}
-		Promise<decltype(func())> promise;
-		addTask([promise, func, priority] {
-			try {
-				auto funcResult = func();
-				onMainThreadVoid([promise, funcResult] { promise.send(funcResult); }, priority);
-			} catch (Error& e) {
-				TraceEvent("ErrorExecutingAsyncTask").error(e);
-				onMainThreadVoid([promise, e] { promise.sendError(e); }, priority);
-			}
-		});
-		return promise.getFuture();
-	}
-};
-
-#endif
diff --git a/fdbclient/include/fdbclient/AuditUtils.actor.h b/fdbclient/include/fdbclient/AuditUtils.actor.h
deleted file mode 100644
index 48311a80b9b..00000000000
--- a/fdbclient/include/fdbclient/AuditUtils.actor.h
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * AuditUtils.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_AUDITUTILS_ACTOR_G_H)
-#define FDBCLIENT_AUDITUTILS_ACTOR_G_H
-#include "fdbclient/AuditUtils.actor.g.h"
-#elif !defined(FDBCLIENT_AUDITUTILS_ACTOR_H)
-#define FDBCLIENT_AUDITUTILS_ACTOR_H
-#pragma once
-
-#include "fdbclient/Audit.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbrpc/fdbrpc.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-struct MoveKeyLockInfo {
-	UID prevOwner, myOwner, prevWrite;
-};
-
-ACTOR Future<Void> cancelAuditMetadata(Database cx, AuditType auditType, UID auditId);
-ACTOR Future<UID> persistNewAuditState(Database cx, AuditStorageState auditState, MoveKeyLockInfo lock, bool ddEnabled);
-ACTOR Future<Void> persistAuditState(Database cx,
-                                     AuditStorageState auditState,
-                                     std::string context,
-                                     MoveKeyLockInfo lock,
-                                     bool ddEnabled);
-ACTOR Future<AuditStorageState> getAuditState(Database cx, AuditType type, UID id);
-ACTOR Future<std::vector<AuditStorageState>> getAuditStates(Database cx,
-                                                            AuditType auditType,
-                                                            bool newFirst,
-                                                            Optional<int> num = Optional<int>(),
-                                                            Optional<AuditPhase> phase = Optional<AuditPhase>());
-
-ACTOR Future<Void> persistAuditStateByRange(Database cx, AuditStorageState auditState);
-ACTOR Future<std::vector<AuditStorageState>> getAuditStateByRange(Database cx,
-                                                                  AuditType type,
-                                                                  UID auditId,
-                                                                  KeyRange range);
-ACTOR Future<Void> persistAuditStateByServer(Database cx, AuditStorageState auditState);
-ACTOR Future<std::vector<AuditStorageState>> getAuditStateByServer(Database cx,
-                                                                   AuditType type,
-                                                                   UID auditId,
-                                                                   UID auditServerId,
-                                                                   KeyRange range);
-ACTOR Future<Void> clearAuditMetadataForType(Database cx,
-                                             AuditType auditType,
-                                             UID maxAuditIdToClear,
-                                             int numFinishAuditToKeep);
-ACTOR Future<bool> checkStorageServerRemoved(Database cx, UID ssid);
-AuditPhase stringToAuditPhase(std::string auditPhaseStr);
-ACTOR Future<bool> checkAuditProgressCompleteByRange(Database cx,
-                                                     AuditType auditType,
-                                                     UID auditId,
-                                                     KeyRange auditRange);
-ACTOR Future<bool> checkAuditProgressCompleteByServer(Database cx,
-                                                      AuditType auditType,
-                                                      UID auditId,
-                                                      KeyRange auditRange,
-                                                      UID serverId,
-                                                      std::shared_ptr<AsyncVar<int>> checkProgressBudget);
-ACTOR Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
-                                                               MoveKeyLockInfo lock,
-                                                               bool ddEnabled,
-                                                               UID dataDistributorId,
-                                                               int persistFinishAuditCount);
-
-struct AuditGetServerKeysRes {
-	KeyRange completeRange;
-	Version readAtVersion;
-	UID serverId;
-	std::vector<KeyRange> ownRanges;
-	int64_t readBytes;
-	AuditGetServerKeysRes() = default;
-	AuditGetServerKeysRes(KeyRange completeRange,
-	                      Version readAtVersion,
-	                      UID serverId,
-	                      std::vector<KeyRange> ownRanges,
-	                      int64_t readBytes)
-	  : completeRange(completeRange), readAtVersion(readAtVersion), serverId(serverId), ownRanges(ownRanges),
-	    readBytes(readBytes) {}
-};
-
-struct AuditGetKeyServersRes {
-	KeyRange completeRange;
-	Version readAtVersion;
-	int64_t readBytes;
-	std::unordered_map<UID, std::vector<KeyRange>> rangeOwnershipMap;
-	AuditGetKeyServersRes() = default;
-	AuditGetKeyServersRes(KeyRange completeRange,
-	                      Version readAtVersion,
-	                      std::unordered_map<UID, std::vector<KeyRange>> rangeOwnershipMap,
-	                      int64_t readBytes)
-	  : completeRange(completeRange), readAtVersion(readAtVersion), rangeOwnershipMap(rangeOwnershipMap),
-	    readBytes(readBytes) {}
-};
-
-std::vector<KeyRange> coalesceRangeList(std::vector<KeyRange> ranges);
-Optional<std::pair<KeyRange, KeyRange>> rangesSame(std::vector<KeyRange> rangesA, std::vector<KeyRange> rangesB);
-ACTOR Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID, Transaction* tr, KeyRange range);
-ACTOR Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId, Transaction* tr, KeyRange range);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/AuditUtils.h b/fdbclient/include/fdbclient/AuditUtils.h
new file mode 100644
index 00000000000..6421d94a79c
--- /dev/null
+++ b/fdbclient/include/fdbclient/AuditUtils.h
@@ -0,0 +1,203 @@
+/*
+ * AuditUtils.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef FDBCLIENT_AUDITUTILS_H
+#define FDBCLIENT_AUDITUTILS_H
+
+#include "fdbclient/Audit.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbrpc/fdbrpc.h"
+
+struct MoveKeyLockInfo {
+	UID prevOwner, myOwner, prevWrite;
+};
+
+// Cancel an in-progress audit by setting its persisted state to Failed and clearing progress metadata.
+Future<Void> cancelAuditMetadata(Database cx, AuditType auditType, UID auditId);
+
+// Persist a new audit's initial state to the database. Returns the assigned audit ID.
+Future<UID> persistNewAuditState(Database cx, AuditStorageState auditState, MoveKeyLockInfo lock, bool ddEnabled);
+
+// Update an existing audit's persisted state (e.g. phase transitions).
+Future<Void> persistAuditState(Database cx,
+                               AuditStorageState auditState,
+                               std::string context,
+                               MoveKeyLockInfo lock,
+                               bool ddEnabled);
+
+// Read a single audit's state by type and ID.
+Future<AuditStorageState> getAuditState(Database cx, AuditType type, UID id);
+
+// List audit states for a given type, optionally filtered by phase, ordered by ID.
+AsyncResult<std::vector<AuditStorageState>> getAuditStates(Database cx,
+                                                           AuditType auditType,
+                                                           bool newFirst,
+                                                           Optional<int> num = Optional<int>(),
+                                                           Optional<AuditPhase> phase = Optional<AuditPhase>());
+
+// Persist per-range audit progress (which sub-ranges have been validated).
+Future<Void> persistAuditStateByRange(Database cx, AuditStorageState auditState);
+
+// Read per-range audit progress for the given audit within a key range.
+Future<std::vector<AuditStorageState>> getAuditStateByRange(Database cx, AuditType type, UID auditId, KeyRange range);
+
+// Persist per-server audit progress.
+Future<Void> persistAuditStateByServer(Database cx, AuditStorageState auditState);
+
+// Read per-server audit progress for a specific server within a key range.
+Future<std::vector<AuditStorageState>> getAuditStateByServer(Database cx,
+                                                             AuditType type,
+                                                             UID auditId,
+                                                             UID auditServerId,
+                                                             KeyRange range);
+
+// Clear persisted audit metadata for completed/failed audits older than maxAuditIdToClear,
+// keeping the most recent numFinishAuditToKeep finished audits.
+Future<Void> clearAuditMetadataForType(Database cx,
+                                       AuditType auditType,
+                                       UID maxAuditIdToClear,
+                                       int numFinishAuditToKeep);
+
+// Check whether a storage server has been removed from the server list.
+Future<bool> checkStorageServerRemoved(Database cx, UID ssid);
+
+// Parse a human-readable audit phase string (e.g. "running", "complete") into an AuditPhase enum.
+AuditPhase stringToAuditPhase(std::string auditPhaseStr);
+
+// Check whether all sub-ranges of auditRange have completed range-based audit progress.
+Future<bool> checkAuditProgressCompleteByRange(Database cx, AuditType auditType, UID auditId, KeyRange auditRange);
+
+// Check whether a specific server's audit progress is complete for the given range.
+Future<bool> checkAuditProgressCompleteByServer(Database cx,
+                                                AuditType auditType,
+                                                UID auditId,
+                                                KeyRange auditRange,
+                                                UID serverId,
+                                                std::shared_ptr<AsyncVar<int>> checkProgressBudget);
+
+// Initialize audit metadata on DD startup: resume incomplete audits and clean up old ones.
+Future<std::vector<AuditStorageState>> initAuditMetadata(Database cx,
+                                                         MoveKeyLockInfo lock,
+                                                         bool ddEnabled,
+                                                         UID dataDistributorId,
+                                                         int persistFinishAuditCount);
+
+// Result of reading a single server's owned ranges from the ServerKeys system key space.
+struct AuditGetServerKeysRes {
+	KeyRange completeRange; // the sub-range that was fully read in this batch
+	Version readAtVersion;
+	UID serverId;
+	std::vector<KeyRange> ownRanges; // ranges this server owns according to ServerKeys
+	int64_t readBytes;
+	AuditGetServerKeysRes() = default;
+	AuditGetServerKeysRes(KeyRange completeRange,
+	                      Version readAtVersion,
+	                      UID serverId,
+	                      std::vector<KeyRange> ownRanges,
+	                      int64_t readBytes)
+	  : completeRange(completeRange), readAtVersion(readAtVersion), serverId(serverId), ownRanges(ownRanges),
+	    readBytes(readBytes) {}
+};
+
+// Result of reading the KeyServers system key space: maps each server to the ranges it owns.
+struct AuditGetKeyServersRes {
+	KeyRange completeRange; // the sub-range that was fully read in this batch
+	Version readAtVersion;
+	int64_t readBytes;
+	std::unordered_map<UID, std::vector<KeyRange>> rangeOwnershipMap;
+	AuditGetKeyServersRes() = default;
+	AuditGetKeyServersRes(KeyRange completeRange,
+	                      Version readAtVersion,
+	                      std::unordered_map<UID, std::vector<KeyRange>> rangeOwnershipMap,
+	                      int64_t readBytes)
+	  : completeRange(completeRange), readAtVersion(readAtVersion), rangeOwnershipMap(rangeOwnershipMap),
+	    readBytes(readBytes) {}
+};
+
+// Merge overlapping or adjacent ranges in the input list into a minimal set of non-overlapping
+// ranges covering the same key space.
+std::vector<KeyRange> coalesceRangeList(std::vector<KeyRange> ranges);
+
+// Compare two sorted, non-overlapping range lists for equivalence. Handles different split
+// points that cover the same total range. Returns the first mismatched pair, or empty if equal.
+Optional<std::pair<KeyRange, KeyRange>> rangesSame(std::vector<KeyRange> rangesA, std::vector<KeyRange> rangesB);
+
+// A single consistency error found by checkLocationMetadataConsistency.
+struct LocationMetadataError {
+	std::string message;
+	UID serverId;
+	Optional<KeyRange> mismatchedRangeByKeyServer;
+	Optional<KeyRange> mismatchedRangeByServerKey;
+};
+
+// Bidirectional consistency check between the KeyServers and ServerKeys views of shard
+// ownership. Returns an error for each server that is missing from one side or has
+// mismatched range assignments. An empty result means the two views are consistent.
+std::vector<LocationMetadataError> checkLocationMetadataConsistency(
+    const std::unordered_map<UID, std::vector<KeyRange>>& mapFromKeyServers,
+    const std::unordered_map<UID, std::vector<KeyRange>>& mapFromServerKeys,
+    KeyRange claimRange);
+
+// Output of buildLocationMetadataMaps: the processed per-server range maps ready for
+// consistency checking, plus counts of validated entries.
+struct LocationMetadataMaps {
+	std::unordered_map<UID, std::vector<KeyRange>> fromKeyServers;
+	std::unordered_map<UID, std::vector<KeyRange>> fromServerKeys;
+	int64_t numValidatedKeyServers = 0;
+	int64_t numValidatedServerKeys = 0;
+};
+
+// Process raw per-server ownership data into maps suitable for checkLocationMetadataConsistency.
+// For KeyServers data: coalesces overlapping ranges then intersects with claimRange.
+// For ServerKeys data: intersects each range with claimRange.
+// If traceId is provided, emits SevVerbose trace events for each range added.
+LocationMetadataMaps buildLocationMetadataMaps(
+    const std::unordered_map<UID, std::vector<KeyRange>>& mapFromKeyServersRaw,
+    const std::unordered_map<UID, std::vector<KeyRange>>& serverOwnRangesMap,
+    KeyRange claimRange,
+    Optional<UID> traceId = Optional<UID>());
+
+// Parse a ServerKeys KRM result into the list of ranges where serverHasKey() is true.
+std::vector<KeyRange> buildOwnRangesFromServerKeysResult(const RangeResult& krmResult);
+
+// Output of buildOwnershipMapFromKeyServersResult: per-server range ownership plus
+// shard statistics from the KeyServers system key space.
+struct KeyServersOwnershipMap {
+	std::unordered_map<UID, std::vector<KeyRange>> rangeOwnershipMap;
+	int64_t totalShardsCount = 0;
+	int64_t shardsInAnonymousPhysicalShardCount = 0;
+};
+
+// Parse a KeyServers KRM result into a per-server ownership map. Decodes each entry's
+// src and dest server lists via decodeKeyServersValue and merges them. Also counts total
+// shards and shards using the anonymous physical shard ID.
+KeyServersOwnershipMap buildOwnershipMapFromKeyServersResult(const RangeResult& krmResult,
+                                                             const RangeResult& uidToTagMap);
+
+// Read a single server's owned ranges from the ServerKeys system key space via transaction.
+Future<AuditGetServerKeysRes> getThisServerKeysFromServerKeys(UID serverID, Transaction* tr, KeyRange range);
+
+// Read the KeyServers system key space via transaction, producing a per-server ownership map.
+Future<AuditGetKeyServersRes> getShardMapFromKeyServers(UID auditServerId, Transaction* tr, KeyRange range);
+
+#endif
diff --git a/fdbclient/include/fdbclient/BackupAgent.actor.h b/fdbclient/include/fdbclient/BackupAgent.h
similarity index 86%
rename from fdbclient/include/fdbclient/BackupAgent.actor.h
rename to fdbclient/include/fdbclient/BackupAgent.h
index 0f62c647907..8955b096da7 100644
--- a/fdbclient/include/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/include/fdbclient/BackupAgent.h
@@ -1,5 +1,5 @@
 /*
- * BackupAgent.actor.h
+ * BackupAgent.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,6 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H)
-#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
-#include "fdbclient/BackupAgent.actor.g.h"
-#elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H)
-#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
 
 #include <ctime>
 #include <climits>
@@ -33,9 +28,8 @@
 #include "fdbclient/TaskBucket.h"
 #include "fdbclient/Notified.h"
 #include "flow/IAsyncFile.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/BackupContainer.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 FDB_BOOLEAN_PARAM(LockDB);
 FDB_BOOLEAN_PARAM(UnlockDB);
@@ -45,8 +39,6 @@ FDB_BOOLEAN_PARAM(WaitForComplete);
 FDB_BOOLEAN_PARAM(ForceAction);
 FDB_BOOLEAN_PARAM(Terminator);
 FDB_BOOLEAN_PARAM(IncrementalBackupOnly);
-FDB_BOOLEAN_PARAM(UsePartitionedLog);
-FDB_BOOLEAN_PARAM(TransformPartitionedLog);
 FDB_BOOLEAN_PARAM(OnlyApplyMutationLogs);
 FDB_BOOLEAN_PARAM(InconsistentSnapshotOnly);
 FDB_BOOLEAN_PARAM(ShowErrors);
@@ -61,6 +53,8 @@ FDB_BOOLEAN_PARAM(ReadLowPriority);
 
 extern Optional<std::string> fileBackupAgentProxy;
 
+constexpr int DEFAULT_ENCRYPTION_BLOCK_SIZE = 1048576;
+
 class BackupAgentBase : NonCopyable {
 public:
 	// Time formatter for anything backup or restore related
@@ -150,7 +144,7 @@ class FileBackupAgent : public BackupAgentBase {
 public:
 	FileBackupAgent();
 
-	FileBackupAgent(FileBackupAgent&& r) noexcept
+	explicit(false) FileBackupAgent(FileBackupAgent&& r) noexcept
 	  : subspace(std::move(r.subspace)), config(std::move(r.config)), lastRestorable(std::move(r.lastRestorable)),
 	    taskBucket(std::move(r.taskBucket)), futureBucket(std::move(r.futureBucket)) {}
 
@@ -177,24 +171,6 @@ class FileBackupAgent : public BackupAgentBase {
 	static StringRef restoreStateText(ERestoreState id);
 	static Key getPauseKey();
 
-	// parallel restore
-	Future<Void> parallelRestoreFinish(Database cx, UID randomUID, UnlockDB = UnlockDB::True);
-	Future<Void> submitParallelRestore(Database cx,
-	                                   Key backupTag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   Key bcUrl,
-	                                   Optional<std::string> proxy,
-	                                   Version targetVersion,
-	                                   LockDB lockDB,
-	                                   UID randomUID,
-	                                   Key addPrefix,
-	                                   Key removePrefix);
-	Future<Void> atomicParallelRestore(Database cx,
-	                                   Key tagName,
-	                                   Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                   Key addPrefix,
-	                                   Key removePrefix);
-
 	// restore() will
 	//   - make sure that url is readable and appears to be a complete backup
 	//   - make sure the requested TargetVersion is valid
@@ -297,9 +273,10 @@ class FileBackupAgent : public BackupAgentBase {
 	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                          StopWhenDone = StopWhenDone::True,
-	                          UsePartitionedLog = UsePartitionedLog::False,
+	                          MutationLogType mutationLogType = MutationLogType::DEFAULT,
 	                          IncrementalBackupOnly = IncrementalBackupOnly::False,
 	                          Optional<std::string> const& encryptionKeyFileName = {},
+	                          int encryptionBlockSize = 0,
 	                          int snapshotMode = 0);
 	// snapshotMode: 0=RANGEFILE (default), 1=BULKDUMP, 2=BOTH
 	Future<Void> submitBackup(Database cx,
@@ -310,9 +287,10 @@ class FileBackupAgent : public BackupAgentBase {
 	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                          StopWhenDone stopWhenDone = StopWhenDone::True,
-	                          UsePartitionedLog partitionedLog = UsePartitionedLog::False,
+	                          MutationLogType mutationLogType = MutationLogType::DEFAULT,
 	                          IncrementalBackupOnly incrementalBackupOnly = IncrementalBackupOnly::False,
 	                          Optional<std::string> const& encryptionKeyFileName = {},
+	                          int encryptionBlockSize = 0,
 	                          int snapshotMode = 0) {
 		// Note: Do NOT call checkAndDisableBackupWorkers here. That function is for cleanup
 		// when backups END (abort/discontinue), not when they START. Calling it here causes
@@ -327,9 +305,10 @@ class FileBackupAgent : public BackupAgentBase {
 			                    tagName,
 			                    backupRanges,
 			                    stopWhenDone,
-			                    partitionedLog,
+			                    mutationLogType,
 			                    incrementalBackupOnly,
 			                    encryptionKeyFileName,
+			                    encryptionBlockSize,
 			                    snapshotMode);
 		});
 	}
@@ -338,7 +317,7 @@ class FileBackupAgent : public BackupAgentBase {
 	Future<Void> discontinueBackup(Database cx, Key tagName) {
 		return runRYWTransaction(
 		           cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); }) +
-		       checkAndDisableBackupWorkers(cx);
+		       checkAndDisableBackupWorkers(cx) + checkAndDisableRangeBackupWorkers(cx);
 	}
 
 	// Terminate an ongoing backup, without waiting for the backup to finish.
@@ -353,12 +332,15 @@ class FileBackupAgent : public BackupAgentBase {
 		// First abort the backup, then check and disable backup workers if needed.
 		return runRYWTransaction(cx,
 		                         [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); }) +
-		       checkAndDisableBackupWorkers(cx);
+		       checkAndDisableBackupWorkers(cx) + checkAndDisableRangeBackupWorkers(cx);
 	}
 
 	// Disable backup workers if no active partitioned backup is running.
 	Future<Void> checkAndDisableBackupWorkers(Database cx);
 
+	// Disable range backup workers if no active range-partitioned backup is running.
+	Future<Void> checkAndDisableRangeBackupWorkers(Database cx);
+
 	Future<std::string> getStatus(Database cx, ShowErrors, std::string tagName);
 	Future<std::string> getStatusJSON(Database cx, std::string tagName);
 
@@ -412,7 +394,7 @@ class DatabaseBackupAgent : public BackupAgentBase {
 	DatabaseBackupAgent();
 	explicit DatabaseBackupAgent(Database src);
 
-	DatabaseBackupAgent(DatabaseBackupAgent&& r) noexcept
+	explicit(false) DatabaseBackupAgent(DatabaseBackupAgent&& r) noexcept
 	  : subspace(std::move(r.subspace)), states(std::move(r.states)), config(std::move(r.config)),
 	    errors(std::move(r.errors)), ranges(std::move(r.ranges)), tagNames(std::move(r.tagNames)),
 	    sourceStates(std::move(r.sourceStates)), sourceTagNames(std::move(r.sourceTagNames)),
@@ -587,37 +569,37 @@ std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
 Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
 Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
 Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
-ACTOR Future<Void> readCommitted(Database cx,
-                                 PromiseStream<RangeResultWithVersion> results,
-                                 Reference<FlowLock> lock,
-                                 KeyRangeRef range,
-                                 Terminator terminator = Terminator::True,
-                                 AccessSystemKeys systemAccess = AccessSystemKeys::False,
-                                 LockAware lockAware = LockAware::False,
-                                 ReadLowPriority readLowPriority = ReadLowPriority::False);
-ACTOR Future<Void> readCommitted(Database cx,
-                                 PromiseStream<RCGroup> results,
-                                 Future<Void> active,
-                                 Reference<FlowLock> lock,
-                                 KeyRangeRef range,
-                                 std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
-                                 Terminator terminator = Terminator::True,
-                                 AccessSystemKeys systemAccess = AccessSystemKeys::False,
-                                 LockAware lockAware = LockAware::False,
-                                 ReadLowPriority readLowPriority = ReadLowPriority::False);
+Future<Void> readCommitted(Database cx,
+                           PromiseStream<RangeResultWithVersion> results,
+                           Reference<FlowLock> lock,
+                           KeyRangeRef range,
+                           Terminator terminator = Terminator::True,
+                           AccessSystemKeys systemAccess = AccessSystemKeys::False,
+                           LockAware lockAware = LockAware::False,
+                           ReadLowPriority readLowPriority = ReadLowPriority::False);
+Future<Void> readCommitted(Database cx,
+                           PromiseStream<RCGroup> results,
+                           Future<Void> active,
+                           Reference<FlowLock> lock,
+                           KeyRangeRef range,
+                           std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
+                           Terminator terminator = Terminator::True,
+                           AccessSystemKeys systemAccess = AccessSystemKeys::False,
+                           LockAware lockAware = LockAware::False,
+                           ReadLowPriority readLowPriority = ReadLowPriority::False);
 
 // Applies the mutations between the beginVersion and endVersion to the database during a restore.
-ACTOR Future<Void> applyMutations(Database cx,
-                                  Key uid,
-                                  Key addPrefix,
-                                  Key removePrefix,
-                                  Version beginVersion,
-                                  Version* endVersion,
-                                  PublicRequestStream<CommitTransactionRequest> commit,
-                                  NotifiedVersion* committedVersion,
-                                  Reference<KeyRangeMap<Version>> keyVersion,
-                                  bool provisionalProxy);
-ACTOR Future<Void> cleanupBackup(Database cx, DeleteData deleteData);
+Future<Void> applyMutations(Database cx,
+                            Key uid,
+                            Key addPrefix,
+                            Key removePrefix,
+                            Version beginVersion,
+                            Version* endVersion,
+                            PublicRequestStream<CommitTransactionRequest> commit,
+                            NotifiedVersion* committedVersion,
+                            Reference<KeyRangeMap<Version>> keyVersion,
+                            bool provisionalProxy);
+Future<Void> cleanupBackup(Database cx, DeleteData deleteData);
 
 using EBackupState = BackupAgentBase::EnumState;
 template <>
@@ -660,12 +642,12 @@ class KeyBackedTag : public KeyBackedProperty<UidAndAbortedFlagT> {
 typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
 // Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
 class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
-	ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
-	                                                           Reference<ReadYourWritesTransaction> tr,
-	                                                           Snapshot snapshot);
+	static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
+	                                                     Reference<ReadYourWritesTransaction> tr,
+	                                                     Snapshot snapshot);
 
 public:
-	TagUidMap(const StringRef& prefix) : TagMap("tag->uid/"_sr.withPrefix(prefix)), prefix(prefix) {}
+	explicit TagUidMap(const StringRef& prefix) : TagMap("tag->uid/"_sr.withPrefix(prefix)), prefix(prefix) {}
 
 	Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr,
 	                                         Snapshot snapshot = Snapshot::False) {
@@ -685,7 +667,7 @@ class KeyBackedTaskConfig : public KeyBackedClass {
 		static TaskParam<UID> uid() { return __FUNCTION__sr; }
 	} TaskParams;
 
-	KeyBackedTaskConfig(StringRef prefix, UID uid = UID())
+	explicit KeyBackedTaskConfig(StringRef prefix, UID uid = UID())
 	  : KeyBackedClass(prefix), uid(uid), configSpace(uidPrefixKey("uid->config/"_sr.withPrefix(prefix), uid)) {}
 
 	KeyBackedTaskConfig(StringRef prefix, Reference<Task> task)
@@ -787,6 +769,7 @@ inline Standalone<StringRef> TupleCodec<Reference<IBackupContainer>>::pack(Refer
 		tuple.append(StringRef());
 	}
 
+	tuple.append((int64_t)bc->getEncryptionBlockSize());
 	return tuple.pack();
 }
 template <>
@@ -806,13 +789,17 @@ inline Reference<IBackupContainer> TupleCodec<Reference<IBackupContainer>>::unpa
 		proxy = t.getString(2).toString();
 	}
 
-	return IBackupContainer::openContainer(url, proxy, encryptionKeyFileName);
+	int blockSize = 0;
+	if (t.size() > 3) {
+		blockSize = (int)t.getInt(3);
+	}
+	return IBackupContainer::openContainer(url, proxy, encryptionKeyFileName, blockSize);
 }
 
 class BackupConfig : public KeyBackedTaskConfig {
 public:
-	BackupConfig(UID uid = UID()) : KeyBackedTaskConfig(fileBackupPrefixRange.begin, uid) {}
-	BackupConfig(Reference<Task> task) : KeyBackedTaskConfig(fileBackupPrefixRange.begin, task) {}
+	explicit BackupConfig(UID uid = UID()) : KeyBackedTaskConfig(fileBackupPrefixRange.begin, uid) {}
+	explicit BackupConfig(Reference<Task> task) : KeyBackedTaskConfig(fileBackupPrefixRange.begin, task) {}
 
 	// rangeFileMap maps a keyrange file's End to its Begin and Filename
 	struct RangeSlice {
@@ -918,8 +905,8 @@ class BackupConfig : public KeyBackedTaskConfig {
 		return configSpace.pack(__FUNCTION__sr);
 	}
 
-	// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
-	KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(__FUNCTION__sr); }
+	// Mutation log type: 0 - DEFAULT (backup v1), 1 = PARTITIONED_LOG, 2 = RANGE_PARTITIONED_LOG.
+	KeyBackedProperty<MutationLogType> mutationLogType() { return configSpace.pack(__FUNCTION__sr); }
 
 	// Set to true if only requesting incremental backup without base snapshot.
 	KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(__FUNCTION__sr); }
@@ -945,6 +932,10 @@ class BackupConfig : public KeyBackedTaskConfig {
 	// BulkDump total keys - stored when BulkDump snapshot completes
 	KeyBackedProperty<int64_t> bulkDumpTotalKeys() { return configSpace.pack(__FUNCTION__sr); }
 
+	// BulkDump snapshot end version - set when BulkDump task completes
+	// Used for mode=BOTH to track that BulkDump data is available
+	KeyBackedProperty<Version> bulkDumpSnapshotEndVersion() { return configSpace.pack(__FUNCTION__sr); }
+
 	// Latest version for which all prior versions have had their log copy tasks completed
 	KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(__FUNCTION__sr); }
 
@@ -961,15 +952,55 @@ class BackupConfig : public KeyBackedTaskConfig {
 		tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		auto lastLog = latestLogEndVersion().get(tr);
 		auto firstSnapshot = firstSnapshotEndVersion().get(tr);
-		auto plogEnabled = partitionedLogEnabled().get(tr);
+		auto mutLogType = mutationLogType().get(tr);
 		auto workerVersion = latestBackupWorkerSavedVersion().get(tr);
 		auto incrementalBackup = incrementalBackupOnly().get(tr);
-		return map(success(lastLog) && success(firstSnapshot) && success(plogEnabled) && success(workerVersion) &&
-		               success(incrementalBackup),
+		auto snapMode = snapshotMode().get(tr);
+		auto bulkDumpSnapshot = bulkDumpSnapshotEndVersion().get(tr);
+		return map(success(lastLog) && success(firstSnapshot) && success(mutLogType) && success(workerVersion) &&
+		               success(incrementalBackup) && success(snapMode) && success(bulkDumpSnapshot),
 		           [=](Void) -> Optional<Version> {
+			           MutationLogType mutationLogType =
+			               mutLogType.get().present() ? mutLogType.get().get() : MutationLogType::DEFAULT;
 			           // The latest log greater than the oldest snapshot is the restorable version
 			           Optional<Version> logVersion =
-			               plogEnabled.get().present() && plogEnabled.get().get() ? workerVersion.get() : lastLog.get();
+			               mutationLogType == MutationLogType::PARTITIONED_LOG ? workerVersion.get() : lastLog.get();
+
+			           // For mode=BOTH (2), require both rangefile and bulkdump snapshots to be complete
+			           int mode = snapMode.get().present() ? snapMode.get().get() : 0;
+			           if (mode == 2) {
+				           // BOTH mode: need both firstSnapshotEndVersion (rangefile) and bulkDumpSnapshotEndVersion
+				           if (!firstSnapshot.get().present() || !bulkDumpSnapshot.get().present()) {
+					           // Incremental-only backup doesn't need snapshots
+					           if (logVersion.present() && incrementalBackup.isReady() &&
+					               incrementalBackup.get().present() && incrementalBackup.get().get()) {
+						           return logVersion.get() - 1;
+					           }
+					           return {}; // Not restorable until both complete
+				           }
+				           // Use the minimum of both as the effective first snapshot version
+				           Version effectiveFirstSnapshot =
+				               std::min(firstSnapshot.get().get(), bulkDumpSnapshot.get().get());
+				           if (logVersion.present() && logVersion.get() > effectiveFirstSnapshot) {
+					           return std::max(logVersion.get() - 1, effectiveFirstSnapshot);
+				           }
+				           return {};
+			           }
+
+			           // For mode=BULKDUMP (1), use bulkDumpSnapshotEndVersion
+			           if (mode == 1) {
+				           if (logVersion.present() && bulkDumpSnapshot.get().present() &&
+				               logVersion.get() > bulkDumpSnapshot.get().get()) {
+					           return std::max(logVersion.get() - 1, bulkDumpSnapshot.get().get());
+				           }
+				           if (logVersion.present() && incrementalBackup.isReady() &&
+				               incrementalBackup.get().present() && incrementalBackup.get().get()) {
+					           return logVersion.get() - 1;
+				           }
+				           return {};
+			           }
+
+			           // For mode=RANGEFILE (0, default), use existing logic with firstSnapshotEndVersion
 			           if (logVersion.present() && firstSnapshot.get().present() &&
 			               logVersion.get() > firstSnapshot.get().get()) {
 				           return std::max(logVersion.get() - 1, firstSnapshot.get().get());
@@ -1012,7 +1043,8 @@ class BackupConfig : public KeyBackedTaskConfig {
 
 // Helper class for reading restore data from a buffer and throwing the right errors.
 struct StringRefReader {
-	StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {}
+	explicit StringRefReader(StringRef s = StringRef(), Error e = Error())
+	  : rptr(s.begin()), end(s.end()), failure_error(e) {}
 
 	// Return remainder of data as a StringRef
 	StringRef remainder() { return StringRef(rptr, end - rptr); }
@@ -1052,31 +1084,22 @@ struct StringRefReader {
 namespace fileBackup {
 Standalone<VectorRef<KeyValueRef>> decodeRangeFileBlock(const Standalone<StringRef>& buf);
 
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
-                                                                      int64_t offset,
-                                                                      int len,
-                                                                      Database cx);
+Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
+                                                                int64_t offset,
+                                                                int len,
+                                                                Database cx);
 
 Standalone<VectorRef<KeyValueRef>> decodeMutationLogFileBlock(const Standalone<StringRef>& buf);
 
 // Reads a mutation log block from file and parses into batch mutation blocks for further parsing.
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
-                                                                            int64_t offset,
-                                                                            int len);
+Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
+                                                                      int64_t offset,
+                                                                      int len);
 
 // Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
 Value makePadding(int size);
 } // namespace fileBackup
 
-// For fast restore simulation test
-// For testing addPrefix feature in fast restore.
-// Transform db content in restoreRanges by removePrefix and then addPrefix.
-// Assume: DB is locked
-ACTOR Future<Void> transformRestoredDatabase(Database cx,
-                                             Standalone<VectorRef<KeyRangeRef>> backupRanges,
-                                             Key addPrefix,
-                                             Key removePrefix);
-
 void simulateBlobFailure();
 
 // Add the set of ranges that are backed up in a default backup to the given vector. This consists of all normal keys
@@ -1114,6 +1137,3 @@ bool isDefaultBackup(Container ranges) {
 
 // Returns a key-range used to denote that a shared mutation stream belongs to the default backup set.
 KeyRangeRef const& getDefaultBackupSharedRange();
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/BackupContainer.h b/fdbclient/include/fdbclient/BackupContainer.h
index dba89dff315..4a4c54b9195 100644
--- a/fdbclient/include/fdbclient/BackupContainer.h
+++ b/fdbclient/include/fdbclient/BackupContainer.h
@@ -33,8 +33,8 @@ FDB_BOOLEAN_PARAM(IncludeKeyRangeMap);
 
 class ReadYourWritesTransaction;
 
-Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const& v, Reference<ReadYourWritesTransaction> const& tr);
-Future<Version> timeKeeperVersionFromDatetime(std::string const& datetime, Database const& db);
+Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version v, Reference<ReadYourWritesTransaction> tr);
+Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Database db);
 
 // Helper function to check if a URL is a blobstore:// URL
 bool isBlobstoreUrl(const std::string& url);
@@ -45,7 +45,7 @@ bool isBlobstoreUrl(const std::string& url);
 // TODO: Move the log file and range file format encoding/decoding stuff to this file and behind interfaces.
 class IBackupFile {
 public:
-	IBackupFile(const std::string& fileName) : m_fileName(fileName) {}
+	explicit IBackupFile(const std::string& fileName) : m_fileName(fileName) {}
 	virtual ~IBackupFile() {}
 	// Backup files are append-only and cannot have more than 1 append outstanding at once.
 	virtual Future<Void> append(const void* data, int len) = 0;
@@ -69,6 +69,9 @@ static const uint32_t BACKUP_AGENT_MLOG_VERSION = 2001;
 // Mutation log version written by BackupWorker
 static const uint32_t PARTITIONED_MLOG_VERSION = 4110;
 
+// Mutation log version written by BackupWorker for range partitioned logs
+static const uint32_t RANGE_PARTITIONED_MLOG_VERSION = 5001;
+
 // Snapshot file version written by FileBackupAgent
 static const uint32_t BACKUP_AGENT_SNAPSHOT_FILE_VERSION = 1001;
 
@@ -132,7 +135,9 @@ struct KeyspaceSnapshotFile {
 	std::string fileName;
 	int64_t totalSize;
 	Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
+	std::string snapshotType; // "bulkdump" or "rangefile" (empty if not present in filename)
 	bool isSingleVersion() const { return beginVersion == endVersion; }
+	bool isBulkDump() const { return snapshotType == "bulkdump"; }
 	double expiredPct(Optional<Version> expiredEnd) const {
 		double pctExpired = 0;
 		if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
@@ -181,9 +186,25 @@ struct BackupFileList {
 	void toStream(FILE* fout) const;
 };
 
+// Mutation log types for backup
+enum class MutationLogType { DEFAULT = 0, PARTITIONED_LOG, RANGE_PARTITIONED_LOG };
+
+inline std::string mutationLogTypeToString(MutationLogType type) {
+	switch (type) {
+	case MutationLogType::DEFAULT:
+		return "default";
+	case MutationLogType::PARTITIONED_LOG:
+		return "partitioned-log-experimental";
+	case MutationLogType::RANGE_PARTITIONED_LOG:
+		return "range-partitioned-log-experimental";
+	default:
+		return "unknown";
+	}
+}
+
 // The byte counts here only include usable log files and byte counts from kvrange manifests
 struct BackupDescription {
-	BackupDescription() : snapshotBytes(0) {}
+	BackupDescription() : snapshotBytes(0), mutationLogType(MutationLogType::DEFAULT) {}
 	std::string url;
 	Optional<std::string> proxy;
 	std::vector<KeyspaceSnapshotFile> snapshots;
@@ -203,8 +224,9 @@ struct BackupDescription {
 	// The minimum version which this backup can be used to restore to
 	Optional<Version> minRestorableVersion;
 	std::string extendedDetail; // Freeform container-specific info.
-	bool partitioned; // If this backup contains partitioned mutation logs.
+	MutationLogType mutationLogType;
 	bool fileLevelEncryption; // If this backup contains encrypted files.
+	int encryptionBlockSize; // Block size used for file encryption, 0 if not encrypted.
 
 	// Resolves the versions above to timestamps using a given database's TimeKeeper data.
 	// toString will use this information if present.
@@ -269,6 +291,12 @@ class IBackupContainer {
 	                                                          uint16_t tagId,
 	                                                          int totalTags) = 0;
 
+	virtual Future<Reference<IBackupFile>> writeRangePartitionedLogFile(Version beginVersion,
+	                                                                    Version endVersion,
+	                                                                    Version baseVersion,
+	                                                                    int32_t partitionId,
+	                                                                    int blockSize) = 0;
+
 	// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
 	// snapshot of the key ranges this backup is targeting.
 	// For BulkDump snapshots, pass SnapshotMetadata with snapshotType="bulkdump" and the job ID.
@@ -280,6 +308,9 @@ class IBackupContainer {
 	    IncludeKeyRangeMap includeKeyRangeMap,
 	    Optional<SnapshotMetadata> metadata = Optional<SnapshotMetadata>()) = 0;
 
+	// Write a partition list file which contains the partition info (e.g. key ranges and partition id).
+	virtual Future<Void> writePartitionListFile(Version v, std::string contents) = 0;
+
 	// Open a file for read by name
 	virtual Future<Reference<IAsyncFile>> readFile(const std::string& name) = 0;
 
@@ -331,7 +362,8 @@ class IBackupContainer {
 	// Get an IBackupContainer based on a container spec string
 	static Reference<IBackupContainer> openContainer(const std::string& url,
 	                                                 const Optional<std::string>& proxy,
-	                                                 const Optional<std::string>& encryptionKeyFileName);
+	                                                 const Optional<std::string>& encryptionKeyFileName,
+	                                                 int encryptionBlockSize);
 	static std::vector<std::string> getURLFormats();
 	static Future<std::vector<std::string>> listContainers(const std::string& baseURL,
 	                                                       const Optional<std::string>& proxy);
@@ -340,12 +372,15 @@ class IBackupContainer {
 	Optional<std::string> const& getProxy() const { return proxy; }
 	Optional<std::string> const& getEncryptionKeyFileName() const { return encryptionKeyFileName; }
 
-	virtual Future<Void> writeEncryptionMetadata() = 0;
+	virtual Future<Void> writeEncryptionMetadata(int encryptionBlockSize) = 0;
 
 	static std::string lastOpenError;
 
 	virtual Future<Void> encryptionSetupComplete() const = 0;
 
+	virtual int getEncryptionBlockSize() const { return 0; }
+	virtual void setEncryptionBlockSize(int blockSize) {}
+
 	// TODO: change the following back to `private` once blob obj access is refactored
 protected:
 	std::string URL;
diff --git a/fdbclient/include/fdbclient/BackupContainerFileSystem.h b/fdbclient/include/fdbclient/BackupContainerFileSystem.h
index 49450e2f99f..775f9068ba5 100644
--- a/fdbclient/include/fdbclient/BackupContainerFileSystem.h
+++ b/fdbclient/include/fdbclient/BackupContainerFileSystem.h
@@ -78,12 +78,6 @@ class BackupContainerFileSystem : public IBackupContainer {
 	Future<Void> create() override = 0;
 	Future<bool> exists() override = 0;
 
-	// TODO: refactor this to separate out the "deal with blob store" stuff from the backup business logic
-	static Reference<BackupContainerFileSystem> openContainerFS(const std::string& url,
-	                                                            const Optional<std::string>& proxy,
-	                                                            const Optional<std::string>& encryptionKeyFileName,
-	                                                            bool isBackup = true);
-
 	// Get a list of fileNames and their sizes in the container under the given path
 	// Although not required, an implementation can avoid traversing unwanted subfolders
 	// by calling folderPathFilter(absoluteFolderPath) and checking for a false return value.
@@ -115,6 +109,12 @@ class BackupContainerFileSystem : public IBackupContainer {
 	                                                  uint16_t tagId,
 	                                                  int totalTags) final;
 
+	Future<Reference<IBackupFile>> writeRangePartitionedLogFile(Version beginVersion,
+	                                                            Version endVersion,
+	                                                            Version baseVersion,
+	                                                            int32_t partitionId,
+	                                                            int blockSize) final;
+
 	Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
 	                                              int snapshotFileCount,
 	                                              Version fileVersion,
@@ -129,9 +129,13 @@ class BackupContainerFileSystem : public IBackupContainer {
 	                                       IncludeKeyRangeMap IncludeKeyRangeMap,
 	                                       Optional<SnapshotMetadata> metadata = Optional<SnapshotMetadata>()) final;
 
+	Future<Void> writePartitionListFile(Version v, std::string contents) override;
+
 	// List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion.
-	// "partitioned" flag indicates if new partitioned mutation logs or old logs should be listed.
-	Future<std::vector<LogFile>> listLogFiles(Version beginVersion, Version targetVersion, bool partitioned);
+	// "mutationLogType" value indicates which mutation log files should be listed.
+	Future<std::vector<LogFile>> listLogFiles(Version beginVersion,
+	                                          Version targetVersion,
+	                                          MutationLogType mutationLogType);
 
 	// List range files, unsorted, which contain data at or between beginVersion and endVersion
 	// Note: The contents of each top level snapshot.N folder do not necessarily constitute a valid snapshot
@@ -164,11 +168,14 @@ class BackupContainerFileSystem : public IBackupContainer {
 	                                                  Version beginVersion) final;
 	static Future<Void> createTestEncryptionKeyFile(std::string const& filename);
 
-	Future<Void> writeEncryptionMetadata() override;
+	Future<Void> writeEncryptionMetadata(int encryptionBlockSize) override;
 
 	// Waits for encryption initialization to complete by reading encryption key file during container opening.
 	Future<Void> encryptionSetupComplete() const override;
 
+	int getEncryptionBlockSize() const override { return encryptionBlockSize; }
+	void setEncryptionBlockSize(int blockSize) override { encryptionBlockSize = blockSize; }
+
 protected:
 	// Returns true if an encryption key file was provided.
 	bool usesEncryption() const;
@@ -177,6 +184,8 @@ class BackupContainerFileSystem : public IBackupContainer {
 
 	Future<Void> writeEntireFileFallback(const std::string& fileName, const std::string& fileContents);
 
+	int encryptionBlockSize = 0;
+
 private:
 	struct VersionProperty {
 		VersionProperty(Reference<BackupContainerFileSystem> bc, const std::string& name)
@@ -203,7 +212,8 @@ class BackupContainerFileSystem : public IBackupContainer {
 	VersionProperty expiredEndVersion();
 	VersionProperty unreliableEndVersion();
 	VersionProperty logType();
-	VersionProperty fileLevelEncryption();
+
+	static std::string encryptionMetadataFileName();
 
 	// List range files, unsorted, which contain data at or between beginVersion and endVersion
 	// NOTE: This reads the range file folder schema from FDB 6.0.15 and earlier and is provided for backward
diff --git a/fdbclient/include/fdbclient/BuildIdempotencyIdMutations.h b/fdbclient/include/fdbclient/BuildIdempotencyIdMutations.h
index d6caade37cf..08fa4a8a24c 100644
--- a/fdbclient/include/fdbclient/BuildIdempotencyIdMutations.h
+++ b/fdbclient/include/fdbclient/BuildIdempotencyIdMutations.h
@@ -22,7 +22,7 @@
 #define FDBCLIENT_BUILD_IDEMPOTENCY_ID_MUTATIONS_H
 
 #include "fdbclient/CommitProxyInterface.h"
-#include "fdbclient/IdempotencyId.actor.h"
+#include "fdbclient/IdempotencyId.h"
 
 #pragma once
 
diff --git a/fdbclient/include/fdbclient/BulkDumping.h b/fdbclient/include/fdbclient/BulkDumping.h
index 1aa56b4ff53..5751bf2b7ab 100644
--- a/fdbclient/include/fdbclient/BulkDumping.h
+++ b/fdbclient/include/fdbclient/BulkDumping.h
@@ -141,6 +141,8 @@ struct BulkDumpState {
 
 	BulkLoadManifest getManifest() const { return manifest; }
 
+	double getSubmitTime() const { return now(); }
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, jobId, jobRange, phase, taskId, manifest);
diff --git a/fdbclient/include/fdbclient/BulkLoading.h b/fdbclient/include/fdbclient/BulkLoading.h
index f1b2fa6b372..61a37ef9a13 100644
--- a/fdbclient/include/fdbclient/BulkLoading.h
+++ b/fdbclient/include/fdbclient/BulkLoading.h
@@ -23,8 +23,9 @@
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Platform.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/Trace.h"
+#include "flow/Util.h"
 #include <cstdint>
 #include <string>
 #pragma once
@@ -188,7 +189,7 @@ struct BulkLoadFileSet {
 		}
 	}
 
-	BulkLoadFileSet(const std::string& rootPath) : rootPath(rootPath) {
+	explicit BulkLoadFileSet(const std::string& rootPath) : rootPath(rootPath) {
 		if (rootPath.empty()) {
 			TraceEvent(SevError, "BulkLoadFileSetProvideInvalidPath")
 			    .suppressFor(10.0)
@@ -379,7 +380,7 @@ struct BulkLoadManifest {
 	// [ByteSampleOverhead]: 100, [ByteSampleMinimalProbability]: 0.500000, [loadType]: 1, [TransportMethod]: 1"
 	// To decode the string, we firstly split the string by ", ". Then, for each part, we remove "[*]: ". Finally, we
 	// convert the remain string for each part to the fields defined in the BulkLoadManifest.
-	BulkLoadManifest(const std::string& rawString) {
+	explicit BulkLoadManifest(const std::string& rawString) {
 		try {
 			std::vector<std::string> parts = splitString(rawString, ", ");
 			formatVersion = std::stoi(stringRemovePrefix(parts[0], "[FormatVersion]: "));
@@ -565,7 +566,7 @@ struct BulkLoadManifestSet {
 
 	BulkLoadManifestSet() = default;
 
-	BulkLoadManifestSet(int inputMaxCount) { maxCount = inputMaxCount; }
+	explicit BulkLoadManifestSet(int inputMaxCount) { maxCount = inputMaxCount; }
 
 	bool isValid() const {
 		if (maxCount == 0) {
@@ -866,7 +867,7 @@ struct BulkLoadJobState {
 		                  ", [JobRange]: " + jobRange.toString() +
 		                  ", [Phase]: " + convertBulkLoadJobPhaseToString(phase) +
 		                  ", [TransportMethod]: " + convertBulkLoadTransportMethodToString(transportMethod);
-		res = res + ", [SubmitTime]: " + std::to_string(submitTime);
+		res = res + ", [SubmitTime]: " + formatTimeISO8601(submitTime);
 		res = res + ", [SinceSubmitMins]: " + std::to_string((now() - submitTime) / 60.0);
 		if (taskCount.present()) {
 			res = res + ", [TaskCount]: " + std::to_string(taskCount.get());
@@ -964,7 +965,7 @@ struct BulkLoadJobManifestFileHeader {
 	// "[FormatVersion]: 1, [ManifestCount]: 2"
 	// To decode the string, we firstly split the string by ", ". Then, for each part, we remove "[*]: ". Finally,
 	// we convert the remain string for each part to the fields defined in the BulkLoadJobManifestFileHeader.
-	BulkLoadJobManifestFileHeader(const std::string& rawString) {
+	explicit BulkLoadJobManifestFileHeader(const std::string& rawString) {
 		try {
 			std::vector<std::string> parts = splitString(rawString, ", ");
 			formatVersion = std::stoi(stringRemovePrefix(parts[0], "[FormatVersion]: "));
@@ -1016,7 +1017,7 @@ struct BulkLoadJobFileManifestEntry {
 	// 50705947, [Bytes]: 6889" To decode the string, we firstly split the string by ", ". Then, for each part, we
 	// remove "[*]: ". Finally, we convert the remain string for each part to the fields defined in the
 	// BulkLoadJobFileManifestEntry.
-	BulkLoadJobFileManifestEntry(const std::string& rawString) {
+	explicit BulkLoadJobFileManifestEntry(const std::string& rawString) {
 		try {
 			std::vector<std::string> parts = splitString(rawString, ", ");
 			if (parts.size() != 5) {
@@ -1042,7 +1043,7 @@ struct BulkLoadJobFileManifestEntry {
 	}
 
 	// Used when dumping
-	BulkLoadJobFileManifestEntry(const BulkLoadManifest& manifest)
+	explicit BulkLoadJobFileManifestEntry(const BulkLoadManifest& manifest)
 	  : beginKey(manifest.getBeginKey()), endKey(manifest.getEndKey()),
 	    manifestRelativePath(manifest.getManifestRelativePath()), version(manifest.getVersion()),
 	    bytes(manifest.getTotalBytes()) {
@@ -1093,7 +1094,7 @@ struct SSBulkLoadMetadata {
 
 	SSBulkLoadMetadata() : dataMoveId(UID()), conductBulkLoad(false) {};
 
-	SSBulkLoadMetadata(const UID& dataMoveId) : dataMoveId(dataMoveId) {
+	explicit SSBulkLoadMetadata(const UID& dataMoveId) : dataMoveId(dataMoveId) {
 		conductBulkLoad = getConductBulkLoadFromDataMoveId(dataMoveId);
 		return;
 	}
@@ -1159,4 +1160,4 @@ BulkLoadJobState createBulkLoadJob(const UID& dumpJobIdToLoad,
                                    const KeyRange& range,
                                    const std::string& jobRoot,
                                    const BulkLoadTransportMethod& transportMethod);
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbclient/include/fdbclient/ClientKnobCollection.h b/fdbclient/include/fdbclient/ClientKnobCollection.h
deleted file mode 100644
index 0002e181436..00000000000
--- a/fdbclient/include/fdbclient/ClientKnobCollection.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * ClientKnobCollection.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBCLIENT_CLIENTKNOBCOLLECTION_H
-#define FDBCLIENT_CLIENTKNOBCOLLECTION_H
-
-#pragma once
-
-#include "fdbclient/ClientKnobs.h"
-#include "fdbclient/IKnobCollection.h"
-#include "flow/Knobs.h"
-
-/*
- * Stores both flow knobs and client knobs, attempting to access server knobs or test knobs
- * results in a run-time error
- */
-class ClientKnobCollection : public IKnobCollection {
-	FlowKnobs flowKnobs;
-	ClientKnobs clientKnobs;
-
-public:
-	ClientKnobCollection(Randomize randomize, IsSimulated isSimulated);
-	void initialize(Randomize randomize, IsSimulated isSimulated) override;
-	void reset(Randomize randomize, IsSimulated isSimulated) override;
-	FlowKnobs const& getFlowKnobs() const override { return flowKnobs; }
-	ClientKnobs const& getClientKnobs() const override { return clientKnobs; }
-	ClientKnobs& getMutableClientKnobs() { return clientKnobs; }
-	ServerKnobs const& getServerKnobs() const override { throw internal_error(); }
-	TestKnobs const& getTestKnobs() const override { throw internal_error(); }
-	Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const override;
-	bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
-};
-
-#endif // FDBCLIENT_CLIENTKNOBCCOLLECTION_H
diff --git a/fdbclient/include/fdbclient/ClientKnobs.h b/fdbclient/include/fdbclient/ClientKnobs.h
deleted file mode 100644
index 5bc1ea35301..00000000000
--- a/fdbclient/include/fdbclient/ClientKnobs.h
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * ClientKnobs.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBCLIENT_KNOBS_H
-#define FDBCLIENT_KNOBS_H
-#pragma once
-
-#include "flow/BooleanParam.h"
-#include "flow/Knobs.h"
-#include "flow/flow.h"
-
-FDB_BOOLEAN_PARAM(Randomize);
-FDB_BOOLEAN_PARAM(IsSimulated);
-
-// Returns a deterministically random transaction timeout value for simulation testing.
-// More weight is given to the famous 5s timeout, but [1, 10] range is returned with lower weight.
-// This is only be used in simulation.
-int getSimulatedTxnTimeoutSeconds();
-
-class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ClientKnobs : public KnobsImpl<ClientKnobs> {
-public:
-	int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
-
-	double SYSTEM_MONITOR_INTERVAL;
-	double NETWORK_BUSYNESS_MONITOR_INTERVAL; // The interval in which we should update the network busyness metric
-	double TSS_METRICS_LOGGING_INTERVAL;
-
-	double FAILURE_MAX_DELAY;
-	double FAILURE_MIN_DELAY;
-	double RECOVERY_DELAY_START_GENERATION;
-	double RECOVERY_DELAY_SECONDS_PER_GENERATION;
-	double MAX_GENERATIONS;
-	double MAX_GENERATIONS_OVERRIDE;
-	double MAX_GENERATIONS_SIM;
-
-	double COORDINATOR_RECONNECTION_DELAY;
-	int CLIENT_EXAMPLE_AMOUNT;
-	double MAX_CLIENT_STATUS_AGE;
-	int MAX_COMMIT_PROXY_CONNECTIONS;
-	int MAX_GRV_PROXY_CONNECTIONS;
-	double STATUS_IDLE_TIMEOUT;
-	double GRPC_CTL_SERVICE_DEFAULT_TIMEOUT; // Default timeout for gRPC FDBCTL service requests
-	bool SEND_ENTIRE_VERSION_VECTOR;
-
-	// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
-	double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
-	                                 // mostly wrong (e.g. dumping the database after a test)
-	double FUTURE_VERSION_RETRY_DELAY;
-	double GRV_ERROR_RETRY_DELAY;
-	int REPLY_BYTE_LIMIT;
-	double DEFAULT_BACKOFF;
-	double DEFAULT_MAX_BACKOFF;
-	double BACKOFF_GROWTH_RATE;
-	double RESOURCE_CONSTRAINED_MAX_BACKOFF;
-	int PROXY_COMMIT_OVERHEAD_BYTES;
-	double SHARD_STAT_SMOOTH_AMOUNT;
-	int INIT_MID_SHARD_BYTES;
-
-	int TRANSACTION_SIZE_LIMIT;
-	int64_t KEY_SIZE_LIMIT;
-	int64_t SYSTEM_KEY_SIZE_LIMIT;
-	int64_t VALUE_SIZE_LIMIT;
-	int64_t SPLIT_KEY_SIZE_LIMIT;
-	int METADATA_VERSION_CACHE_SIZE;
-	int64_t CHANGE_FEED_LOCATION_LIMIT;
-	int64_t CHANGE_FEED_CACHE_SIZE;
-	double CHANGE_FEED_POP_TIMEOUT;
-	int64_t CHANGE_FEED_STREAM_MIN_BYTES;
-	double CHANGE_FEED_START_INTERVAL;
-	bool CHANGE_FEED_COALESCE_LOCATIONS;
-	int64_t CHANGE_FEED_CACHE_FLUSH_BYTES;
-	double CHANGE_FEED_CACHE_EXPIRE_TIME;
-	int64_t CHANGE_FEED_CACHE_LIMIT_BYTES;
-
-	int MAX_BATCH_SIZE;
-	double GRV_BATCH_TIMEOUT;
-	int BROADCAST_BATCH_SIZE;
-	double TRANSACTION_TIMEOUT_DELAY_INTERVAL;
-
-	// When locationCache in DatabaseContext gets to be this size, items will be evicted
-	int LOCATION_CACHE_EVICTION_SIZE;
-	int LOCATION_CACHE_EVICTION_SIZE_SIM;
-	double LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD;
-	double LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL;
-
-	int GET_RANGE_SHARD_LIMIT;
-	int WARM_RANGE_SHARD_LIMIT;
-	int STORAGE_METRICS_SHARD_LIMIT;
-	int SHARD_COUNT_LIMIT;
-	double STORAGE_METRICS_UNFAIR_SPLIT_LIMIT;
-	double STORAGE_METRICS_TOO_MANY_SHARDS_DELAY;
-	double AGGREGATE_HEALTH_METRICS_MAX_STALENESS;
-	double DETAILED_HEALTH_METRICS_MAX_STALENESS;
-	double MID_SHARD_SIZE_MAX_STALENESS;
-	bool TAG_ENCODE_KEY_SERVERS;
-	int64_t RANGESTREAM_FRAGMENT_SIZE;
-	int RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT;
-	bool QUARANTINE_TSS_ON_MISMATCH;
-	double CHANGE_FEED_EMPTY_BATCH_TIME;
-
-	// KeyRangeMap
-	int KRM_GET_RANGE_LIMIT;
-	int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
-	                               // entries will be returned from an attempt to read a key range map
-
-	int DEFAULT_MAX_OUTSTANDING_WATCHES;
-	int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
-	double WATCH_POLLING_TIME;
-	double NO_RECENT_UPDATES_DURATION;
-	double FAST_WATCH_TIMEOUT;
-	double WATCH_TIMEOUT;
-
-	double IS_ACCEPTABLE_DELAY;
-
-	// Versions -- knobs that control 5s timeout
-	int64_t VERSIONS_PER_SECOND; // Copy of SERVER_KNOBS, as we can't link with it.
-	int64_t MAX_WRITE_TRANSACTION_LIFE_VERSIONS; // Copy of SERVER_KNOBS, as we can't link with it.
-
-	// Core
-	int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
-	int LOG_RANGE_BLOCK_SIZE;
-	int MUTATION_BLOCK_SIZE;
-	double MAX_VERSION_CACHE_LAG; // The upper bound in seconds for OK amount of staleness when using a cached RV
-	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
-	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
-	bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
-	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
-	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
-	                                           // specified here, ratekeeper is throttling and not a false positive
-
-	// Taskbucket
-	double TASKBUCKET_LOGGING_DELAY;
-	int TASKBUCKET_MAX_PRIORITY;
-	double TASKBUCKET_CHECK_TIMEOUT_CHANCE;
-	double TASKBUCKET_TIMEOUT_JITTER_OFFSET;
-	double TASKBUCKET_TIMEOUT_JITTER_RANGE;
-	double TASKBUCKET_CHECK_ACTIVE_DELAY;
-	int TASKBUCKET_CHECK_ACTIVE_AMOUNT;
-	int TASKBUCKET_TIMEOUT_VERSIONS;
-	int TASKBUCKET_MAX_TASK_KEYS;
-
-	// Backup
-	int BACKUP_LOCAL_FILE_WRITE_BLOCK;
-	int BACKUP_CONCURRENT_DELETES;
-	int BACKUP_SIMULATED_LIMIT_BYTES;
-	int BACKUP_GET_RANGE_LIMIT_BYTES;
-	int BACKUP_LOCK_BYTES;
-	double BACKUP_RANGE_TIMEOUT;
-	double BACKUP_RANGE_MINWAIT;
-	double BULKDUMP_JOB_TIMEOUT;
-	double BULKLOAD_JOB_TIMEOUT;
-	int BACKUP_SNAPSHOT_DISPATCH_INTERVAL_SEC;
-	int BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
-	int BACKUP_SHARD_TASK_LIMIT;
-	double BACKUP_AGGREGATE_POLL_RATE;
-	double BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL;
-	int BACKUP_LOG_WRITE_BATCH_MAX_SIZE;
-	int BACKUP_MAX_LOG_RANGES;
-	int BACKUP_SIM_COPY_LOG_RANGES;
-	int BACKUP_VERSION_DELAY;
-	int BACKUP_MAP_KEY_LOWER_LIMIT;
-	int BACKUP_MAP_KEY_UPPER_LIMIT;
-	int BACKUP_COPY_TASKS;
-	int BACKUP_BLOCK_SIZE;
-	int COPY_LOG_BLOCK_SIZE;
-	int COPY_LOG_BLOCKS_PER_TASK;
-	int COPY_LOG_PREFETCH_BLOCKS;
-	int COPY_LOG_READ_AHEAD_BYTES;
-	double COPY_LOG_TASK_DURATION_NANOS;
-	int BACKUP_TASKS_PER_AGENT;
-	int BACKUP_POLL_PROGRESS_SECONDS;
-	int SIM_BACKUP_TASKS_PER_AGENT;
-	int BACKUP_RANGEFILE_BLOCK_SIZE;
-	int BACKUP_LOGFILE_BLOCK_SIZE;
-	int BACKUP_DISPATCH_ADDTASK_SIZE;
-	bool BACKUP_ALLOW_DRYRUN;
-	int RESTORE_DISPATCH_ADDTASK_SIZE;
-	int RESTORE_DISPATCH_BATCH_SIZE;
-	int RESTORE_WRITE_TX_SIZE;
-	int RESTORE_PARTITIONED_BATCH_VERSION_SIZE;
-	int APPLY_MAX_LOCK_BYTES;
-	int APPLY_MIN_LOCK_BYTES;
-	int APPLY_BLOCK_SIZE;
-	double APPLY_MAX_DECAY_RATE;
-	double APPLY_MAX_INCREASE_FACTOR;
-	double BACKUP_ERROR_DELAY;
-	double BACKUP_STATUS_DELAY;
-	double BACKUP_STATUS_JITTER;
-	double MIN_CLEANUP_SECONDS;
-	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
-	int RESTORE_RANGES_READ_BATCH;
-
-	bool BACKUP_CONTAINER_LOCAL_ALLOW_RELATIVE_PATH;
-	bool ENABLE_REPLICA_CONSISTENCY_CHECK_ON_BACKUP_READS;
-	int BACKUP_CONSISTENCY_CHECK_REQUIRED_REPLICAS;
-	bool BACKUP_READS_USE_LOW_PRIORITY;
-	int BULKLOAD_JOB_HISTORY_COUNT_MAX; // the max number of bulk load job history to keep. The oldest job history will
-	                                    // be removed when the count exceeds this value. Set to 0 to disable history.
-	                                    // Do not set the value to a large number, e.g. <= 10.
-	int BULKLOAD_VERBOSE_LEVEL; // Set to 1 to minimize the verbosity. Set to 5 to turn on all events for performance
-	                            // insights. Set to 10 to turn on all events.
-	int S3CLIENT_VERBOSE_LEVEL; // Similar to BULKLOAD_VERBOSE_LEVEL
-
-	// Configuration
-	int32_t DEFAULT_AUTO_COMMIT_PROXIES;
-	int32_t DEFAULT_AUTO_GRV_PROXIES;
-	int32_t DEFAULT_COMMIT_GRV_PROXIES_RATIO;
-	int32_t DEFAULT_MAX_GRV_PROXIES;
-	int32_t DEFAULT_AUTO_RESOLVERS;
-	int32_t DEFAULT_AUTO_LOGS;
-
-	double GLOBAL_CONFIG_REFRESH_BACKOFF;
-	double GLOBAL_CONFIG_REFRESH_MAX_BACKOFF;
-	double GLOBAL_CONFIG_REFRESH_TIMEOUT;
-
-	// Dynamic Knobs
-	double COMMIT_QUORUM_TIMEOUT;
-	double GET_GENERATION_QUORUM_TIMEOUT;
-	double GET_KNOB_TIMEOUT;
-	double TIMEOUT_RETRY_UPPER_BOUND;
-
-	// Client Status Info
-	double CSI_SAMPLING_PROBABILITY;
-	int64_t CSI_SIZE_LIMIT;
-	double CSI_STATUS_DELAY;
-
-	bool HTTP_REQUEST_AWS_V4_HEADER; // setting this knob to true will enable AWS V4 style header.
-	std::string BLOBSTORE_ENCRYPTION_TYPE;
-	int BLOBSTORE_CONNECT_TRIES;
-	int BLOBSTORE_CONNECT_TIMEOUT;
-	int BLOBSTORE_MAX_CONNECTION_LIFE;
-	int BLOBSTORE_REQUEST_TRIES;
-	int BLOBSTORE_REQUEST_TIMEOUT_MIN;
-	int BLOBSTORE_REQUESTS_PER_SECOND;
-	int BLOBSTORE_LIST_REQUESTS_PER_SECOND;
-	int BLOBSTORE_WRITE_REQUESTS_PER_SECOND;
-	int BLOBSTORE_READ_REQUESTS_PER_SECOND;
-	int BLOBSTORE_DELETE_REQUESTS_PER_SECOND;
-	int BLOBSTORE_CONCURRENT_REQUESTS;
-	int BLOBSTORE_MULTIPART_MAX_PART_SIZE;
-	int BLOBSTORE_MULTIPART_MIN_PART_SIZE;
-	int BLOBSTORE_MULTIPART_RETRY_DELAY_MS;
-	int BLOBSTORE_CONCURRENT_UPLOADS;
-	int BLOBSTORE_CONCURRENT_LISTS;
-	int BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
-	int BLOBSTORE_CONCURRENT_READS_PER_FILE;
-	int BLOBSTORE_ENABLE_READ_CACHE;
-	int BLOBSTORE_READ_BLOCK_SIZE;
-	int BLOBSTORE_READ_AHEAD_BLOCKS;
-	int BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
-	int BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
-	int BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
-	int BLOBSTORE_LIST_MAX_KEYS_PER_PAGE;
-	bool BLOBSTORE_GLOBAL_CONNECTION_POOL;
-	bool BLOBSTORE_ENABLE_LOGGING;
-	double BLOBSTORE_STATS_LOGGING_INTERVAL;
-	double BLOBSTORE_LATENCY_LOGGING_INTERVAL;
-	double BLOBSTORE_LATENCY_LOGGING_ACCURACY;
-	int BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
-	int BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
-	bool
-	    BLOBSTORE_ENABLE_OBJECT_INTEGRITY_CHECK; // Enable integrity check of download. When not set, on upload, we
-	                                             // we volunteer an md5 of the content and upload will fail if our
-	                                             // md5 doesn't match that calculated by the server. When this flag
-	                                             // is set, we hash the content using sha256 and have the server store
-	                                             // the hash in the object metadata. On download, if this flag is set
-	                                             // we will check the serverside proffered hash against that we
-	                                             // calculate on the received content.  If no match, throw an error. See
-	                                             // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
-	                                             // This download check only works for small files. For large files,
-	                                             // we run a separate checksum. See design/s3-checksumming.md
-	                                             //
-	int CONSISTENCY_CHECK_RATE_LIMIT_MAX; // Available in both normal and urgent mode
-	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; // Available in normal mode
-	int CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME; // Available in urgent mode
-	int CONSISTENCY_CHECK_URGENT_BATCH_SHARD_COUNT; // Available in urgent mode
-	int CONSISTENCY_CHECK_URGENT_RETRY_DEPTH_MAX; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_0; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_0; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_1; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_1; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_2;
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_2;
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_3;
-	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_3;
-
-	// fdbcli
-	int CLI_CONNECT_PARALLELISM;
-	double CLI_CONNECT_TIMEOUT;
-	bool CLI_PRINT_INVALID_CONFIGURATION;
-
-	// trace
-	int TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH;
-
-	// transaction tags
-	int MAX_TRANSACTION_TAG_LENGTH;
-	int MAX_TAGS_PER_TRANSACTION;
-	int COMMIT_SAMPLE_COST; // The expectation of sampling is every COMMIT_SAMPLE_COST sample once
-	int INCOMPLETE_SHARD_PLUS; // The size of (possible) incomplete shard when estimate clear range
-	double READ_TAG_SAMPLE_RATE; // Communicated to clients from cluster
-	double TAG_THROTTLE_SMOOTHING_WINDOW;
-	double TAG_THROTTLE_RECHECK_INTERVAL;
-	double TAG_THROTTLE_EXPIRATION_INTERVAL;
-	int64_t TAG_THROTTLING_PAGE_SIZE; // Used to round up the cost of operations
-	// Cost multiplier for writes (because write operations are more expensive than reads):
-	double GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO;
-	// Maximum duration that a transaction can be tag throttled by proxy before being rejected
-	double PROXY_MAX_TAG_THROTTLE_DURATION;
-
-	// Enable to automatically retry transactions in the presence of transaction_lock_rejection error
-	// Set to false only for the rangeLocking simulation test
-	bool TRANSACTION_LOCK_REJECTION_RETRIABLE;
-
-	// busyness reporting
-	double BUSYNESS_SPIKE_START_THRESHOLD;
-	double BUSYNESS_SPIKE_SATURATED_THRESHOLD;
-
-	// The coordinator key/value in storage server might be inconsistent to the value stored in the cluster file.
-	// This might happen when a recovery is happening together with a cluster controller coordinator key change.
-	// During this process the database will be marked as "bad state" in changeQuorumChecker while later it will be
-	// available again. Using a backoffed retry when it happens.
-	int CHANGE_QUORUM_BAD_STATE_RETRY_TIMES;
-	double CHANGE_QUORUM_BAD_STATE_RETRY_DELAY;
-
-	bool ENABLE_MUTATION_CHECKSUM;
-	// Enable to start accumulative checksum population and validation
-	bool ENABLE_ACCUMULATIVE_CHECKSUM;
-	// Enable to logging verbose trace events related to the accumulative checksum
-	bool ENABLE_ACCUMULATIVE_CHECKSUM_LOGGING;
-
-	ClientKnobs(Randomize randomize, IsSimulated isSimulated);
-	void initialize(Randomize randomize, IsSimulated isSimulated);
-};
-
-#endif
diff --git a/fdbclient/include/fdbclient/ClientVersion.h b/fdbclient/include/fdbclient/ClientVersion.h
index 6de3f065913..5b0d2535923 100644
--- a/fdbclient/include/fdbclient/ClientVersion.h
+++ b/fdbclient/include/fdbclient/ClientVersion.h
@@ -36,7 +36,7 @@ struct ClientVersionRef {
 	    protocolVersion(arena, cv.protocolVersion) {}
 	ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
 	  : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
-	ClientVersionRef(StringRef versionString) {
+	explicit ClientVersionRef(StringRef versionString) {
 		std::vector<StringRef> parts = versionString.splitAny(","_sr);
 		if (parts.size() != 3) {
 			initUnknown();
diff --git a/fdbclient/include/fdbclient/CommitProxyInterface.h b/fdbclient/include/fdbclient/CommitProxyInterface.h
index 983c6de7a5b..05fe9d81919 100644
--- a/fdbclient/include/fdbclient/CommitProxyInterface.h
+++ b/fdbclient/include/fdbclient/CommitProxyInterface.h
@@ -27,15 +27,10 @@
 
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/EncryptKeyProxyInterface.h"
 #include "fdbclient/GlobalConfig.h"
 #include "fdbclient/GrvProxyInterface.h"
-#include "fdbclient/IdempotencyId.actor.h"
+#include "fdbclient/IdempotencyId.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbclient/VersionVector.h"
-#include "fdbrpc/Stats.h"
-#include "fdbrpc/TimedRequest.h"
 
 struct CommitProxyInterface {
 	constexpr static FileIdentifier file_identifier = 8954922;
@@ -45,11 +40,10 @@ struct CommitProxyInterface {
 	Optional<Key> processId;
 	bool provisional;
 	PublicRequestStream<struct CommitTransactionRequest> commit;
-	PublicRequestStream<struct GetReadVersionRequest>
-	    getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
-	                              // committed (by a commit response) when this request was sent
-	                              //   (at some point between when this request is sent and when its response is
-	                              //   received, the latest version reported committed)
+	// Reserved to preserve the historical adjusted-endpoint numbering for the
+	// commit proxy interface. Commit proxies do not serve GRV traffic; clients
+	// use GrvProxyInterface::getConsistentReadVersion for that path.
+	PublicRequestStream<struct GetReadVersionRequest> legacyGetConsistentReadVersion;
 	PublicRequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations;
 	RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo;
 
@@ -74,7 +68,7 @@ struct CommitProxyInterface {
 	void serialize(Archive& ar) {
 		serializer(ar, processId, provisional, commit);
 		if (Archive::isDeserializing) {
-			getConsistentReadVersion =
+			legacyGetConsistentReadVersion =
 			    PublicRequestStream<struct GetReadVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(1));
 			getKeyServersLocations =
 			    PublicRequestStream<struct GetKeyServerLocationsRequest>(commit.getEndpoint().getAdjustedEndpoint(2));
@@ -98,7 +92,7 @@ struct CommitProxyInterface {
 	void initEndpoints() {
 		std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
 		streams.push_back(commit.getReceiver(TaskPriority::ReadSocket));
-		streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
+		streams.push_back(legacyGetConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
 		streams.push_back(getKeyServersLocations.getReceiver(
 		    TaskPriority::ReadSocket)); // priority lowered to TaskPriority::DefaultEndpoint on the proxy
 		streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
@@ -126,7 +120,6 @@ struct ClientDBInfo {
 	Optional<Value> forward;
 	std::vector<VersionHistory> history;
 	UID clusterId;
-	Optional<EncryptKeyProxyInterface> encryptKeyProxy;
 
 	ClusterType clusterType = ClusterType::STANDALONE;
 
@@ -140,7 +133,7 @@ struct ClientDBInfo {
 		if constexpr (!is_fb_function<Archive>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(ar, grvProxies, commitProxies, id, forward, history, encryptKeyProxy, clusterId, clusterType);
+		serializer(ar, grvProxies, commitProxies, id, forward, history, clusterId, clusterType);
 	}
 };
 
@@ -208,7 +201,7 @@ struct CommitTransactionRequest : TimedRequest {
 	bool verify() const { return true; }
 
 	CommitTransactionRequest() : CommitTransactionRequest(SpanContext()) {}
-	CommitTransactionRequest(SpanContext const& context) : spanContext(context), flags(0) {}
+	explicit CommitTransactionRequest(SpanContext const& context) : spanContext(context), flags(0) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -230,121 +223,6 @@ static inline int getBytes(CommitTransactionRequest const& r) {
 	return total;
 }
 
-struct GetReadVersionReply : public BasicLoadBalancedReply {
-	constexpr static FileIdentifier file_identifier = 15709388;
-	Version version;
-	bool locked;
-	Optional<Value> metadataVersion;
-	int64_t midShardSize = 0;
-	bool rkDefaultThrottled = false;
-	bool rkBatchThrottled = false;
-
-	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
-	double proxyTagThrottledDuration{ 0.0 };
-
-	VersionVector ssVersionVectorDelta;
-	UID proxyId; // GRV proxy ID to detect old GRV proxies at client side
-
-	GetReadVersionReply() : version(invalidVersion), locked(false) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar,
-		           BasicLoadBalancedReply::processBusyTime,
-		           version,
-		           locked,
-		           metadataVersion,
-		           tagThrottleInfo,
-		           midShardSize,
-		           rkDefaultThrottled,
-		           rkBatchThrottled,
-		           ssVersionVectorDelta,
-		           proxyId,
-		           proxyTagThrottledDuration);
-	}
-};
-
-struct GetReadVersionRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 838566;
-	enum {
-		PRIORITY_SYSTEM_IMMEDIATE =
-		    15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
-		PRIORITY_DEFAULT = 8 << 24,
-		PRIORITY_BATCH = 1 << 24
-	};
-	enum {
-		FLAG_USE_MIN_KNOWN_COMMITTED_VERSION = 4,
-		FLAG_USE_PROVISIONAL_PROXIES = 2,
-		FLAG_CAUSAL_READ_RISKY = 1,
-		FLAG_PRIORITY_MASK = PRIORITY_SYSTEM_IMMEDIATE,
-	};
-
-	SpanContext spanContext;
-	uint32_t transactionCount;
-	uint32_t flags;
-	TransactionPriority priority;
-
-	TransactionTagMap<uint32_t> tags;
-	// Not serialized, because this field does not need to be sent to master.
-	// It is used for reporting to clients the amount of time spent delayed by
-	// the TagQueue
-	double proxyTagThrottledDuration{ 0.0 };
-
-	Optional<UID> debugID;
-	ReplyPromise<GetReadVersionReply> reply;
-
-	Version maxVersion; // max version in the client's version vector cache
-
-	GetReadVersionRequest() : transactionCount(1), flags(0), maxVersion(invalidVersion) {}
-	GetReadVersionRequest(SpanContext spanContext,
-	                      uint32_t transactionCount,
-	                      TransactionPriority priority,
-	                      Version maxVersion,
-	                      uint32_t flags = 0,
-	                      TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
-	                      Optional<UID> debugID = Optional<UID>())
-	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority), tags(tags),
-	    debugID(debugID), maxVersion(maxVersion) {
-		flags = flags & ~FLAG_PRIORITY_MASK;
-		switch (priority) {
-		case TransactionPriority::BATCH:
-			flags |= PRIORITY_BATCH;
-			break;
-		case TransactionPriority::DEFAULT:
-			flags |= PRIORITY_DEFAULT;
-			break;
-		case TransactionPriority::IMMEDIATE:
-			flags |= PRIORITY_SYSTEM_IMMEDIATE;
-			break;
-		default:
-			ASSERT(false);
-		}
-	}
-
-	bool verify() const { return true; }
-
-	bool operator<(GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
-
-	bool isTagged() const { return !tags.empty(); }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext, maxVersion);
-
-		if (ar.isDeserializing) {
-			if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
-				priority = TransactionPriority::IMMEDIATE;
-			} else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
-				priority = TransactionPriority::DEFAULT;
-			} else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
-				priority = TransactionPriority::BATCH;
-			} else {
-				priority = TransactionPriority::DEFAULT;
-			}
-		}
-	}
-};
-
 struct GetKeyServerLocationsReply {
 	constexpr static FileIdentifier file_identifier = 10636023;
 	Arena arena;
@@ -597,38 +475,6 @@ struct ExclusionSafetyCheckRequest {
 	}
 };
 
-struct GlobalConfigRefreshReply {
-	constexpr static FileIdentifier file_identifier = 12680327;
-	Arena arena;
-	Version version;
-	RangeResultRef result;
-
-	GlobalConfigRefreshReply() {}
-	GlobalConfigRefreshReply(Arena const& arena, Version version, RangeResultRef result)
-	  : arena(arena), version(version), result(result) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, result, version, arena);
-	}
-};
-
-struct GlobalConfigRefreshRequest {
-	constexpr static FileIdentifier file_identifier = 2828131;
-	Version lastKnown;
-	ReplyPromise<GlobalConfigRefreshReply> reply;
-
-	GlobalConfigRefreshRequest() {}
-	explicit GlobalConfigRefreshRequest(Version lastKnown) : lastKnown(lastKnown) {}
-
-	bool verify() const noexcept { return true; }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, lastKnown, reply);
-	}
-};
-
 struct SetThrottledShardReply {
 	constexpr static FileIdentifier file_identifier = 2828140;
 
diff --git a/fdbclient/include/fdbclient/CommitTransaction.h b/fdbclient/include/fdbclient/CommitTransaction.h
index 26922fde2cd..968766b3cae 100644
--- a/fdbclient/include/fdbclient/CommitTransaction.h
+++ b/fdbclient/include/fdbclient/CommitTransaction.h
@@ -113,16 +113,6 @@ struct MutationRef {
 		       (accumulativeChecksumIndex.present() ? sizeof(uint16_t) + 1 : 1);
 	}
 	int expectedSize() const { return param1.size() + param2.size(); }
-	int weightedTotalSize() const {
-		// AtomicOp can cause more workload to FDB cluster than the same-size set mutation;
-		// Amplify atomicOp size to consider such extra workload.
-		// A good value for FASTRESTORE_ATOMICOP_WEIGHT needs experimental evaluations.
-		if (isAtomicOp()) {
-			return totalSize() * CLIENT_KNOBS->FASTRESTORE_ATOMICOP_WEIGHT;
-		} else {
-			return totalSize();
-		}
-	}
 
 	std::string toString() const {
 		std::string checksumStr;
diff --git a/fdbclient/include/fdbclient/ConsistencyScanInterface.actor.h b/fdbclient/include/fdbclient/ConsistencyScanInterface.h
similarity index 76%
rename from fdbclient/include/fdbclient/ConsistencyScanInterface.actor.h
rename to fdbclient/include/fdbclient/ConsistencyScanInterface.h
index e7b6045a692..fb277767b32 100644
--- a/fdbclient/include/fdbclient/ConsistencyScanInterface.actor.h
+++ b/fdbclient/include/fdbclient/ConsistencyScanInterface.h
@@ -18,26 +18,21 @@
  * limitations under the License.
  */
 
+#pragma once
+
 #include "fdbclient/SystemData.h"
 #include "fdbclient/json_spirit/json_spirit_value.h"
 #include "flow/serialize.h"
 #include "fmt/core.h"
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_G_H)
-#define FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_G_H
-#include "fdbclient/ConsistencyScanInterface.actor.g.h"
-#elif !defined(FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_H)
-#define FDBCLIENT_CONSISTENCYSCANINTERFACE_ACTOR_H
 
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/Locality.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/KeyBackedRangeMap.actor.h"
-
-#include "flow/actorcompiler.h" // must be last include
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/KeyBackedRangeMap.h"
 
 struct ConsistencyScanInterface {
 	constexpr static FileIdentifier file_identifier = 4983265;
@@ -87,7 +82,7 @@ struct HaltConsistencyScanRequest {
 //
 // The class trigger will only be fired by changes to Config or RangeConfig
 struct ConsistencyScanState : public KeyBackedClass {
-	ConsistencyScanState(Key prefix = SystemKey("\xff/consistencyScanState"_sr)) : KeyBackedClass(prefix) {}
+	explicit ConsistencyScanState(Key prefix = SystemKey("\xff/consistencyScanState"_sr)) : KeyBackedClass(prefix) {}
 
 	struct Config {
 		constexpr static FileIdentifier file_identifier = 23123;
@@ -312,60 +307,16 @@ struct ConsistencyScanState : public KeyBackedClass {
 	// History of scan round stats stored by their start version
 	StatsHistoryMap roundStatsHistory() { return { subspace.pack(__FUNCTION__sr), IncludeVersion() }; }
 
-	ACTOR static Future<Void> clearStatsActor(ConsistencyScanState* self, Reference<ReadYourWritesTransaction> tr) {
+	Future<Void> clearStats(Reference<ReadYourWritesTransaction> tr) {
 		// read the keyspaces so the transaction conflicts on write (key-backed properties don't expose conflict ranges,
 		// and the extra work here is negligible because this is a rare manual command so performance is not a huge
 		// concern)
-		wait(success(self->currentRoundStats().getD(tr)) && success(self->lifetimeStats().getD(tr)) &&
-		     success(self->roundStatsHistory().getRange(tr, {}, {}, 1, Snapshot::False, Reverse::False)));
+		co_await (success(currentRoundStats().getD(tr)) && success(lifetimeStats().getD(tr)) &&
+		          success(roundStatsHistory().getRange(tr, {}, {}, 1, Snapshot::False, Reverse::False)));
 
 		// update each of the stats keyspaces to empty
-		self->currentRoundStats().set(tr, ConsistencyScanState::RoundStats());
-		self->lifetimeStats().set(tr, ConsistencyScanState::LifetimeStats());
-		self->roundStatsHistory().erase(tr, 0, MAX_VERSION);
-		return Void();
+		currentRoundStats().set(tr, ConsistencyScanState::RoundStats());
+		lifetimeStats().set(tr, ConsistencyScanState::LifetimeStats());
+		roundStatsHistory().erase(tr, 0, MAX_VERSION);
 	}
-
-	Future<Void> clearStats(Reference<ReadYourWritesTransaction> tr) { return clearStatsActor(this, tr); }
 };
-
-/////////////////////
-// Code below this line is not used by the Consistency Scan Role, only the ConsistencyCheck Workload.
-
-ACTOR Future<Version> getVersion(Database cx);
-ACTOR Future<bool> getKeyServers(
-    Database cx,
-    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
-    KeyRangeRef kr,
-    bool performQuiescentChecks,
-    bool failureIsError,
-    bool* success);
-ACTOR Future<bool> getKeyLocations(Database cx,
-                                   std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
-                                   Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise,
-                                   bool performQuiescentChecks,
-                                   bool* success);
-ACTOR Future<Void> checkDataConsistency(Database cx,
-                                        VectorRef<KeyValueRef> keyLocations,
-                                        DatabaseConfiguration configuration,
-                                        std::map<UID, StorageServerInterface> tssMapping,
-                                        bool performQuiescentChecks,
-                                        bool performTSSCheck,
-                                        bool firstClient,
-                                        bool failureIsError,
-                                        int clientId,
-                                        int clientCount,
-                                        bool distributed,
-                                        bool shuffleShards,
-                                        int shardSampleFactor,
-                                        int64_t sharedRandomNumber,
-                                        int64_t repetitions,
-                                        int64_t* bytesReadInPreviousRound,
-                                        int restart,
-                                        int64_t maxRate,
-                                        int64_t targetInterval,
-                                        bool* success);
-
-#include "flow/unactorcompiler.h"
-
-#endif // FDBCLIENT_CONSISTENCYSCANINTERFACE_H
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/CoordinationInterface.h b/fdbclient/include/fdbclient/CoordinationInterface.h
index fd73ff6ac6f..25a90807c20 100644
--- a/fdbclient/include/fdbclient/CoordinationInterface.h
+++ b/fdbclient/include/fdbclient/CoordinationInterface.h
@@ -39,9 +39,9 @@ struct ClientLeaderRegInterface {
 	Optional<Hostname> hostname;
 
 	ClientLeaderRegInterface() {}
-	ClientLeaderRegInterface(NetworkAddress remote);
-	ClientLeaderRegInterface(INetwork* local);
-	ClientLeaderRegInterface(Hostname hostname) : hostname(hostname) {}
+	explicit ClientLeaderRegInterface(NetworkAddress remote);
+	explicit ClientLeaderRegInterface(INetwork* local);
+	explicit ClientLeaderRegInterface(Hostname hostname) : hostname(hostname) {}
 
 	bool operator==(const ClientLeaderRegInterface& rhs) const {
 		return getLeader == rhs.getLeader && openDatabase == rhs.openDatabase;
@@ -66,7 +66,7 @@ class ClusterConnectionString {
 	constexpr static FileIdentifier file_identifier = 13602011;
 
 	ClusterConnectionString() {}
-	ClusterConnectionString(const std::string& connectionString);
+	explicit ClusterConnectionString(const std::string& connectionString);
 	ClusterConnectionString(const std::vector<NetworkAddress>& coordinators, Key key);
 	ClusterConnectionString(const std::vector<Hostname>& hosts, Key key);
 
@@ -119,7 +119,7 @@ FDB_BOOLEAN_PARAM(ConnectionStringNeedsPersisted);
 // one that is only stored in memory.
 class IClusterConnectionRecord {
 public:
-	IClusterConnectionRecord(ConnectionStringNeedsPersisted connectionStringNeedsPersisted)
+	explicit IClusterConnectionRecord(ConnectionStringNeedsPersisted connectionStringNeedsPersisted)
 	  : connectionStringNeedsPersisted(connectionStringNeedsPersisted) {}
 	virtual ~IClusterConnectionRecord() {}
 
@@ -194,7 +194,7 @@ struct LeaderInfo {
 	bool forward;
 
 	LeaderInfo() : forward(false) {}
-	LeaderInfo(UID changeID) : changeID(changeID), forward(false) {}
+	explicit LeaderInfo(UID changeID) : changeID(changeID), forward(false) {}
 
 	bool operator<(LeaderInfo const& r) const { return changeID < r.changeID; }
 	bool operator>(LeaderInfo const& r) const { return r < *this; }
diff --git a/fdbclient/include/fdbclient/DataDistributionConfig.actor.h b/fdbclient/include/fdbclient/DataDistributionConfig.h
similarity index 81%
rename from fdbclient/include/fdbclient/DataDistributionConfig.actor.h
rename to fdbclient/include/fdbclient/DataDistributionConfig.h
index b53c7613a17..420348f5c82 100644
--- a/fdbclient/include/fdbclient/DataDistributionConfig.actor.h
+++ b/fdbclient/include/fdbclient/DataDistributionConfig.h
@@ -1,5 +1,5 @@
 /*
- * DataDistributionConfig.actor.h
+ * DataDistributionConfig.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,23 +18,18 @@
  * limitations under the License.
  */
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_DATA_DISTRIBUTION_CONFIG_ACTOR_G_H)
-#define FDBCLIENT_DATA_DISTRIBUTION_CONFIG_ACTOR_G_H
-#include "fdbclient/DataDistributionConfig.actor.g.h"
-#elif !defined(FDBCLIENT_DATA_DISTRIBUTION_CONFIG_ACTOR_H)
-#define FDBCLIENT_DATA_DISTRIBUTION_CONFIG_ACTOR_H
+#pragma once
 
 #include "flow/serialize.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/KeyBackedRangeMap.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/KeyBackedRangeMap.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/RunTransaction.actor.h"
+#include "fdbclient/RunTransaction.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/json_spirit/json_spirit_value.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // DD Configuration for a key range range.
 // There are many properties here, all Optional, because for a given key range range a property
@@ -42,7 +37,7 @@
 struct DDRangeConfig {
 	constexpr static FileIdentifier file_identifier = 9193856;
 
-	DDRangeConfig(Optional<int> replicationFactor = {}, Optional<int> teamID = {})
+	explicit(false) DDRangeConfig(Optional<int> replicationFactor = {}, Optional<int> teamID = {})
 	  : replicationFactor(replicationFactor), teamID(teamID) {}
 
 	Optional<int> replicationFactor;
@@ -93,7 +88,7 @@ template <>
 struct fmt::formatter<DDRangeConfig> : FormatUsingTraceable<DDRangeConfig> {};
 
 struct DDConfiguration : public KeyBackedClass {
-	DDConfiguration(KeyRef prefix = SystemKey("\xff\x02/ddconfig/"_sr)) : KeyBackedClass(prefix) {}
+	explicit DDConfiguration(KeyRef prefix = SystemKey("\xff\x02/ddconfig/"_sr)) : KeyBackedClass(prefix) {}
 
 	// RangeConfigMap is a  KeyBackedRangeMap of DDRangeConfig values describing various option overrides for key ranges
 	typedef KeyBackedRangeMap<Key,
@@ -109,6 +104,3 @@ struct DDConfiguration : public KeyBackedClass {
 
 	static json_spirit::mValue toJSON(RangeConfigMapSnapshot const& snapshot, bool includeDefaultRanges = false);
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/DatabaseConfiguration.h b/fdbclient/include/fdbclient/DatabaseConfiguration.h
index 817d5c87bfb..1b3ce10c634 100644
--- a/fdbclient/include/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/include/fdbclient/DatabaseConfiguration.h
@@ -170,41 +170,7 @@ struct DatabaseConfiguration {
 	// replicated to remote, and will be incremented later. `forAvailability` is set to true
 	// if we want to account the number for machines that can recruit new tLogs/SS after failures.
 	// Killing an entire datacenter counts as killing one zone in modes that support it.
-	int32_t maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const {
-		int worstSatelliteTLogReplicationFactor = regions.size() ? std::numeric_limits<int>::max() : 0;
-		int regionsWithNonNegativePriority = 0;
-		for (auto& r : regions) {
-			if (r.priority >= 0) {
-				regionsWithNonNegativePriority++;
-			}
-			worstSatelliteTLogReplicationFactor = std::min(
-			    worstSatelliteTLogReplicationFactor, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
-			if (r.satelliteTLogUsableDcsFallback > 0) {
-				worstSatelliteTLogReplicationFactor =
-				    std::min(worstSatelliteTLogReplicationFactor,
-				             r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
-			}
-		}
-
-		if (worstSatelliteTLogReplicationFactor <= 0) {
-			// HA is not enabled in this database. Return single cluster zone failures to tolerate.
-			return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1);
-		}
-
-		// Compute HA enabled database zone failure tolerance.
-		auto isGeoReplicatedData = [this, &fullyReplicatedRegions]() {
-			return usableRegions > 1 && fullyReplicatedRegions > 1;
-		};
-
-		if (isGeoReplicatedData() && (!forAvailability || regionsWithNonNegativePriority > 1)) {
-			return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum,
-			                             worstSatelliteTLogReplicationFactor - 1),
-			                    storageTeamSize - 1);
-		}
-		// Primary and Satellite tLogs are synchronously replicated, hence we can lose all but 1.
-		return std::min(tLogReplicationFactor + worstSatelliteTLogReplicationFactor - 1 - tLogWriteAntiQuorum,
-		                storageTeamSize - 1);
-	}
+	int32_t maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const;
 
 	// CommitProxy Servers
 	int32_t commitProxyCount;
@@ -243,6 +209,7 @@ struct DatabaseConfiguration {
 
 	// Backup Workers
 	bool backupWorkerEnabled;
+	bool rangeBackupWorkerEnabled;
 
 	// Data centers
 	int32_t usableRegions; // Number of regions which have a replica of the database.
diff --git a/fdbclient/include/fdbclient/DatabaseContext.h b/fdbclient/include/fdbclient/DatabaseContext.h
index 0104dcb30f3..b79688508fe 100644
--- a/fdbclient/include/fdbclient/DatabaseContext.h
+++ b/fdbclient/include/fdbclient/DatabaseContext.h
@@ -24,10 +24,11 @@
 #include "flow/ApiVersion.h"
 #include "flow/FastAlloc.h"
 #include "flow/FastRef.h"
-#include "fdbclient/GlobalConfig.actor.h"
+#include "fdbclient/GlobalConfig.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
+#include <compare>
 #include <vector>
 #include <unordered_map>
 #pragma once
@@ -36,13 +37,12 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/CommitProxyInterface.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/SpecialKeySpace.h"
 #include "fdbclient/VersionVector.h"
-#include "fdbclient/IKeyValueStore.actor.h"
 #include "fdbrpc/QueueModel.h"
 #include "fdbrpc/MultiInterface.h"
-#include "flow/TDMetric.actor.h"
-#include "fdbclient/EventTypes.actor.h"
+#include "flow/TDMetric.h"
+#include "fdbclient/EventTypes.h"
 #include "fdbrpc/Smoother.h"
 #include "fdbrpc/DDSketch.h"
 
@@ -89,7 +89,7 @@ class ClientTagThrottleData : NonCopyable {
 	Smoother smoothReleased;
 
 public:
-	ClientTagThrottleData(ClientTagThrottleLimits const& limits)
+	explicit ClientTagThrottleData(ClientTagThrottleLimits const& limits)
 	  : tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()),
 	    smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
 	    smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {
@@ -152,7 +152,7 @@ class WatchMetadata : public ReferenceCounted<WatchMetadata> {
 
 	Reference<const WatchParameters> parameters;
 
-	WatchMetadata(Reference<const WatchParameters> parameters) : parameters(parameters) {}
+	explicit WatchMetadata(Reference<const WatchParameters> parameters) : parameters(parameters) {}
 };
 
 struct MutationAndVersionStream {
@@ -237,7 +237,7 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 	Future<HealthMetrics> getHealthMetrics(bool detailed);
 	// Get storage stats of a storage server from the cached healthy metrics if now() - lastUpdate < maxStaleness.
 	// Otherwise, ask GRVProxy for the up-to-date health metrics.
-	Future<Optional<HealthMetrics::StorageStats>> getStorageStats(const UID& id, double maxStaleness);
+	Future<Optional<HealthMetrics::StorageStats>> getStorageStats(UID id, double maxStaleness);
 	// Pass a negative value for `shardLimit` to indicate no limit on the shard number.
 	Future<StorageMetrics> getStorageMetrics(
 	    KeyRange const& keys,
@@ -379,7 +379,9 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 		TagSet tags;
 		Optional<UID> debugID;
 
-		VersionRequest(SpanContext spanContext, TagSet tags = TagSet(), Optional<UID> debugID = Optional<UID>())
+		explicit VersionRequest(SpanContext spanContext,
+		                        TagSet tags = TagSet(),
+		                        Optional<UID> debugID = Optional<UID>())
 		  : spanContext(spanContext), tags(tags), debugID(debugID) {}
 	};
 
@@ -388,7 +390,16 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 		PromiseStream<VersionRequest> stream;
 		Future<Void> actor;
 	};
-	std::map<uint32_t, VersionBatcher> versionBatcher;
+	struct VersionBatcherKey {
+		uint32_t flags;
+		Optional<int64_t> maxGrvQueueDelayMS;
+
+		VersionBatcherKey(uint32_t flags, Optional<int64_t> maxGrvQueueDelayMS)
+		  : flags(flags), maxGrvQueueDelayMS(maxGrvQueueDelayMS) {}
+
+		std::strong_ordering operator<=>(VersionBatcherKey const& rhs) const = default;
+	};
+	std::map<VersionBatcherKey, VersionBatcher> versionBatcher;
 
 	AsyncTrigger connectionFileChangedTrigger;
 
@@ -418,10 +429,6 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 	// map from tssid -> metrics for that tss pair
 	std::unordered_map<UID, Reference<TSSMetrics>> tssMetrics;
 
-	IKeyValueStore* storage = nullptr;
-
-	void setStorage(IKeyValueStore* storage);
-
 	// map from ssid -> ss tag
 	// @note this map allows the client to identify the latest commit versions
 	// of storage servers (note that "ssVersionVectorCache" identifies storage
@@ -515,7 +522,7 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 	TaskPriority taskID;
 
 	Int64MetricHandle getValueSubmitted;
-	EventMetricHandle<GetValueComplete> getValueCompleted;
+	EventMetricHandle<GetValueCompleteDescriptor> getValueCompleted;
 
 	Reference<AsyncVar<ClientDBInfo>> clientInfo;
 	Future<Void> clientInfoMonitor;
@@ -670,7 +677,8 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 
 // Similar to tr.onError(), but doesn't require a DatabaseContext.
 struct Backoff {
-	Backoff(double backoff = CLIENT_KNOBS->DEFAULT_BACKOFF, double maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF)
+	explicit Backoff(double backoff = CLIENT_KNOBS->DEFAULT_BACKOFF,
+	                 double maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF)
 	  : backoff(backoff), maxBackoff(maxBackoff) {}
 
 	Future<Void> onError() {
diff --git a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
deleted file mode 100644
index 505aa117b21..00000000000
--- a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * EncryptKeyProxyInterface.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_ENCRYPTKEYPROXYINTERFACE_H
-#define FDBSERVER_ENCRYPTKEYPROXYINTERFACE_H
-#pragma once
-
-#include "fdbclient/FDBTypes.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbrpc/Locality.h"
-#include "flow/Arena.h"
-#include "flow/EncryptUtils.h"
-#include "flow/FileIdentifier.h"
-#include "flow/IRandom.h"
-#include "flow/network.h"
-
-#include <limits>
-
-#define DEBUG_ENCRYPT_KEY_PROXY false
-
-struct KMSHealthStatus {
-	constexpr static FileIdentifier file_identifier = 2378149;
-	bool canConnectToKms;
-	bool canConnectToEKP;
-	double lastUpdatedTS;
-	std::string kmsConnectorType;
-	std::vector<std::string> restKMSUrls;
-	bool kmsStable = true;
-
-	KMSHealthStatus() : canConnectToKms(false), canConnectToEKP(false), lastUpdatedTS(-1), kmsStable(true) {}
-
-	bool operator==(const KMSHealthStatus& other) {
-		return canConnectToKms == other.canConnectToKms && canConnectToEKP == other.canConnectToEKP;
-	}
-
-	bool healthnessChanged(const KMSHealthStatus& other) {
-		return canConnectToKms != other.canConnectToKms || canConnectToEKP != other.canConnectToEKP;
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "CanConnectToKms(" << canConnectToKms << ")"
-		   << ", CanConnectToEKP(" << canConnectToEKP << ")"
-		   << ", LastUpdatedTS(" << lastUpdatedTS << ")"
-		   << ", KMSConnectorType(" << kmsConnectorType << ")"
-		   << ", RESTKmsUrls(";
-		bool firstUrl = true;
-		for (const auto& url : restKMSUrls) {
-			if (!firstUrl) {
-				ss << ", ";
-			}
-			ss << url;
-			firstUrl = false;
-		}
-
-		ss << "), KmsStable(" << kmsStable << ")";
-		return ss.str();
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, canConnectToKms, canConnectToEKP, lastUpdatedTS, kmsConnectorType, restKMSUrls, kmsStable);
-	}
-};
-
-struct EncryptKeyProxyInterface {
-	constexpr static FileIdentifier file_identifier = 1303419;
-	struct LocalityData locality;
-	UID myId;
-	RequestStream<ReplyPromise<Void>> waitFailure;
-	RequestStream<struct HaltEncryptKeyProxyRequest> haltEncryptKeyProxy;
-	RequestStream<struct EKPGetBaseCipherKeysByIdsRequest> getBaseCipherKeysByIds;
-	RequestStream<struct EKPGetLatestBaseCipherKeysRequest> getLatestBaseCipherKeys;
-	RequestStream<struct EKPGetLatestBlobMetadataRequest> getLatestBlobMetadata;
-	RequestStream<struct EncryptKeyProxyHealthStatusRequest> getHealthStatus;
-
-	EncryptKeyProxyInterface() {}
-	explicit EncryptKeyProxyInterface(const struct LocalityData& loc, UID id) : locality(loc), myId(id) {}
-
-	NetworkAddress address() const { return haltEncryptKeyProxy.getEndpoint().getPrimaryAddress(); }
-	NetworkAddressList addresses() const { return haltEncryptKeyProxy.getEndpoint().addresses; }
-
-	UID id() const { return myId; }
-
-	bool operator==(const EncryptKeyProxyInterface& toCompare) const { return myId == toCompare.myId; }
-	bool operator!=(const EncryptKeyProxyInterface& toCompare) const { return !(*this == toCompare); }
-
-	template <class Archive>
-	void serialize(Archive& ar) {
-		if constexpr (!is_fb_function<Archive>) {
-			ASSERT(ar.protocolVersion().isValid());
-		}
-		serializer(ar, locality, myId, waitFailure);
-		if (Archive::isDeserializing) {
-			haltEncryptKeyProxy =
-			    RequestStream<struct HaltEncryptKeyProxyRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(1));
-			getBaseCipherKeysByIds = RequestStream<struct EKPGetBaseCipherKeysByIdsRequest>(
-			    waitFailure.getEndpoint().getAdjustedEndpoint(2));
-			getLatestBaseCipherKeys = RequestStream<struct EKPGetLatestBaseCipherKeysRequest>(
-			    waitFailure.getEndpoint().getAdjustedEndpoint(3));
-			getLatestBlobMetadata =
-			    RequestStream<struct EKPGetLatestBlobMetadataRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(4));
-			getHealthStatus = RequestStream<struct EncryptKeyProxyHealthStatusRequest>(
-			    waitFailure.getEndpoint().getAdjustedEndpoint(5));
-		}
-	}
-
-	void initEndpoints() {
-		std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
-		streams.push_back(waitFailure.getReceiver());
-		streams.push_back(haltEncryptKeyProxy.getReceiver(TaskPriority::DefaultPromiseEndpoint));
-		streams.push_back(getBaseCipherKeysByIds.getReceiver(TaskPriority::Worker));
-		streams.push_back(getLatestBaseCipherKeys.getReceiver(TaskPriority::Worker));
-		streams.push_back(getLatestBlobMetadata.getReceiver(TaskPriority::Worker));
-		streams.push_back(getHealthStatus.getReceiver(TaskPriority::Worker));
-		FlowTransport::transport().addEndpoints(streams);
-	}
-};
-
-struct HaltEncryptKeyProxyRequest {
-	constexpr static FileIdentifier file_identifier = 2378138;
-	UID requesterID;
-	ReplyPromise<Void> reply;
-
-	HaltEncryptKeyProxyRequest() : requesterID(deterministicRandom()->randomUniqueID()) {}
-	explicit HaltEncryptKeyProxyRequest(UID uid) : requesterID(uid) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, requesterID, reply);
-	}
-};
-
-struct EncryptKeyProxyHealthStatusRequest {
-	constexpr static FileIdentifier file_identifier = 2378139;
-	ReplyPromise<KMSHealthStatus> reply;
-
-	EncryptKeyProxyHealthStatusRequest() {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
-struct EKPBaseCipherDetails {
-	constexpr static FileIdentifier file_identifier = 2149615;
-	int64_t encryptDomainId;
-	uint64_t baseCipherId;
-	Standalone<StringRef> baseCipherKey;
-	EncryptCipherKeyCheckValue baseCipherKCV;
-	int64_t refreshAt;
-	int64_t expireAt;
-
-	EKPBaseCipherDetails()
-	  : encryptDomainId(0), baseCipherId(0), baseCipherKey(Standalone<StringRef>()), baseCipherKCV(0), refreshAt(0),
-	    expireAt(-1) {}
-	explicit EKPBaseCipherDetails(int64_t dId,
-	                              uint64_t id,
-	                              Standalone<StringRef> key,
-	                              EncryptCipherKeyCheckValue cipherKCV)
-	  : encryptDomainId(dId), baseCipherId(id), baseCipherKey(key), baseCipherKCV(cipherKCV),
-	    refreshAt(std::numeric_limits<int64_t>::max()), expireAt(std::numeric_limits<int64_t>::max()) {}
-	explicit EKPBaseCipherDetails(int64_t dId,
-	                              uint64_t id,
-	                              Standalone<StringRef> key,
-	                              EncryptCipherKeyCheckValue cipherKCV,
-	                              int64_t refAt,
-	                              int64_t expAt)
-	  : encryptDomainId(dId), baseCipherId(id), baseCipherKey(key), baseCipherKCV(cipherKCV), refreshAt(refAt),
-	    expireAt(expAt) {}
-
-	bool operator==(const EKPBaseCipherDetails& r) const {
-		return encryptDomainId == r.encryptDomainId && baseCipherId == r.baseCipherId && refreshAt == r.refreshAt &&
-		       expireAt == r.expireAt && baseCipherKey.toString() == r.baseCipherKey.toString();
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptDomainId, baseCipherId, baseCipherKey, baseCipherKCV, refreshAt, expireAt);
-	}
-};
-
-struct EKPGetBaseCipherKeysByIdsReply {
-	constexpr static FileIdentifier file_identifier = 9485259;
-	std::vector<EKPBaseCipherDetails> baseCipherDetails;
-	int numHits;
-	Optional<Error> error;
-
-	EKPGetBaseCipherKeysByIdsReply() : numHits(0) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, baseCipherDetails, numHits, error);
-	}
-};
-
-struct EKPGetBaseCipherKeysRequestInfo {
-	constexpr static FileIdentifier file_identifier = 2180516;
-	// Encryption cipher domain identifier
-	EncryptCipherDomainId domainId;
-	// Encryption cipher KMS assigned identifier
-	EncryptCipherBaseKeyId baseCipherId;
-
-	EKPGetBaseCipherKeysRequestInfo()
-	  : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
-	EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId)
-	  : domainId(dId), baseCipherId(bCId) {}
-
-	bool operator==(const EKPGetBaseCipherKeysRequestInfo& info) const {
-		return domainId == info.domainId && baseCipherId == info.baseCipherId;
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, domainId, baseCipherId);
-	}
-};
-
-struct EKPGetBaseCipherKeysByIdsRequest {
-	constexpr static FileIdentifier file_identifier = 4930263;
-	std::vector<EKPGetBaseCipherKeysRequestInfo> baseCipherInfos;
-	Optional<UID> debugId;
-	ReplyPromise<EKPGetBaseCipherKeysByIdsReply> reply;
-
-	EKPGetBaseCipherKeysByIdsRequest() {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, baseCipherInfos, debugId, reply);
-	}
-};
-
-struct EKPGetLatestBaseCipherKeysReply {
-	constexpr static FileIdentifier file_identifier = 4831583;
-	std::vector<EKPBaseCipherDetails> baseCipherDetails;
-	int numHits;
-	Optional<Error> error;
-
-	EKPGetLatestBaseCipherKeysReply() : numHits(0) {}
-	explicit EKPGetLatestBaseCipherKeysReply(const std::vector<EKPBaseCipherDetails>& cipherDetails)
-	  : baseCipherDetails(cipherDetails), numHits(0) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, baseCipherDetails, numHits, error);
-	}
-};
-
-struct EKPGetBaseCipherKeysRequestInfo_Hash {
-	std::size_t operator()(const EKPGetBaseCipherKeysRequestInfo& info) const {
-		boost::hash<std::pair<EncryptCipherDomainId, EncryptCipherBaseKeyId>> hasher;
-		return hasher(std::make_pair(info.domainId, info.baseCipherId));
-	}
-};
-
-struct EKPGetLatestBaseCipherKeysRequest {
-	constexpr static FileIdentifier file_identifier = 1910123;
-	std::vector<EncryptCipherDomainId> encryptDomainIds;
-	Optional<UID> debugId;
-	ReplyPromise<EKPGetLatestBaseCipherKeysReply> reply;
-
-	EKPGetLatestBaseCipherKeysRequest() {}
-	explicit EKPGetLatestBaseCipherKeysRequest(const std::vector<EncryptCipherDomainId>& ids) : encryptDomainIds(ids) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptDomainIds, debugId, reply);
-	}
-};
-
-// partition and credentials information for a given blob domain
-
-struct EKPGetLatestBlobMetadataReply {
-	constexpr static FileIdentifier file_identifier = 5761581;
-	// stuff deleted.  This RPC should not be happening.
-
-	EKPGetLatestBlobMetadataReply() {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar);
-	}
-};
-
-struct EKPGetLatestBlobMetadataRequest {
-	constexpr static FileIdentifier file_identifier = 3821549;
-	std::vector<EncryptCipherDomainId> domainIds;
-	Optional<UID> debugId;
-	ReplyPromise<EKPGetLatestBlobMetadataReply> reply;
-
-	EKPGetLatestBlobMetadataRequest() {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, domainIds, debugId, reply);
-	}
-};
-
-#endif
diff --git a/fdbclient/include/fdbclient/EventTypes.h b/fdbclient/include/fdbclient/EventTypes.h
new file mode 100644
index 00000000000..69b248aada3
--- /dev/null
+++ b/fdbclient/include/fdbclient/EventTypes.h
@@ -0,0 +1,34 @@
+/*
+ * EventTypes.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "flow/flow.h"
+#include "flow/TDMetric.h"
+
+struct GetValueCompleteDescriptor {
+	int64_t latency;
+};
+
+template <>
+struct Descriptor<GetValueCompleteDescriptor>
+  : DescribeType<GetValueCompleteDescriptor,
+                 "GetValueComplete",
+                 DescribeField<&GetValueCompleteDescriptor::latency, "latency", "ns">> {};
diff --git a/fdbclient/include/fdbclient/FDBTypes.h b/fdbclient/include/fdbclient/FDBTypes.h
index 5eced645fe7..e775e10d31d 100644
--- a/fdbclient/include/fdbclient/FDBTypes.h
+++ b/fdbclient/include/fdbclient/FDBTypes.h
@@ -981,7 +981,7 @@ struct KeyValueStoreType {
 	};
 
 	KeyValueStoreType() : type(END) {}
-	KeyValueStoreType(StoreType type) : type(type) {
+	explicit(false) KeyValueStoreType(StoreType type) : type(type) {
 		if ((uint32_t)type > END)
 			this->type = END;
 	}
@@ -1038,7 +1038,7 @@ struct TLogVersion {
 	} version;
 
 	TLogVersion() : version(UNSET) {}
-	TLogVersion(Version v) : version(v) {}
+	explicit(false) TLogVersion(Version v) : version(v) {}
 
 	operator Version() const { return version; }
 
@@ -1083,7 +1083,7 @@ struct TLogSpillType {
 	};
 
 	TLogSpillType() : type(DEFAULT) {}
-	TLogSpillType(SpillType type) : type(type) {
+	explicit(false) TLogSpillType(SpillType type) : type(type) {
 		if ((uint32_t)type >= END) {
 			this->type = UNSET;
 		}
@@ -1163,7 +1163,7 @@ struct StorageBytes {
 };
 struct LogMessageVersion {
 	// Each message pushed into the log system has a unique, totally ordered LogMessageVersion
-	// See ILogSystem::push() for how these are assigned
+	// See LogSystem::push() for how these are assigned
 	Version version;
 	uint32_t sub;
 
@@ -1392,7 +1392,7 @@ struct StorageMigrationType {
 	enum MigrationType { DEFAULT = 1, UNSET = 0, DISABLED = 1, AGGRESSIVE = 2, GRADUAL = 3, END = 4 };
 
 	StorageMigrationType() : type(UNSET) {}
-	StorageMigrationType(MigrationType type) : type(type) {
+	explicit(false) StorageMigrationType(MigrationType type) : type(type) {
 		if ((uint32_t)type >= END) {
 			this->type = UNSET;
 		}
@@ -1436,7 +1436,7 @@ struct EncryptionAtRestModeDeprecated {
 	};
 
 	EncryptionAtRestModeDeprecated() : mode(DISABLED) {}
-	EncryptionAtRestModeDeprecated(Mode mode) : mode(mode) {
+	explicit(false) EncryptionAtRestModeDeprecated(Mode mode) : mode(mode) {
 		if ((uint32_t)mode >= END) {
 			this->mode = DISABLED;
 		}
@@ -1517,10 +1517,6 @@ struct Traceable<EncryptionAtRestModeDeprecated> : std::true_type {
 	static std::string toString(const EncryptionAtRestModeDeprecated& mode) { return mode.toString(); }
 };
 
-typedef StringRef ClusterNameRef;
-typedef Standalone<ClusterNameRef> ClusterName;
-
-// TODO(gglass): delete metacluster code and tenant code and reassess the need for this enum
 enum class ClusterType { STANDALONE, LEGACY_UNUSED_METACLUSTER_MANAGEMENT, LEGACY_UNUSED_METACLUSTER_DATA };
 
 struct GRVCacheSpace {
@@ -1579,9 +1575,9 @@ struct StorageMetadataType {
 	bool wrongConfiguredForWiggle = false;
 
 	StorageMetadataType() : createdTime(0) {}
-	StorageMetadataType(double t,
-	                    KeyValueStoreType storeType = KeyValueStoreType::END,
-	                    bool wrongConfiguredForWiggle = false)
+	explicit StorageMetadataType(double t,
+	                             KeyValueStoreType storeType = KeyValueStoreType::END,
+	                             bool wrongConfiguredForWiggle = false)
 	  : createdTime(t), storeType(storeType), wrongConfiguredForWiggle(wrongConfiguredForWiggle) {}
 
 	static double currentTime() { return g_network->timer(); }
@@ -1622,7 +1618,7 @@ struct StorageWiggleValue {
 	constexpr static FileIdentifier file_identifier = 732124;
 	UID id; // storage id
 
-	StorageWiggleValue(UID id = UID(0, 0)) : id(id) {}
+	explicit StorageWiggleValue(UID id = UID(0, 0)) : id(id) {}
 
 	// To change this serialization, ProtocolVersion::PerpetualWiggleMetadata must be updated, and downgrades need
 	// to be considered
@@ -1649,13 +1645,13 @@ struct ReadOptions {
 	Optional<UID> debugID;
 	Optional<Version> consistencyCheckStartVersion;
 
-	ReadOptions(Optional<UID> debugID = Optional<UID>(),
-	            ReadType type = ReadType::NORMAL,
-	            CacheResult cache = CacheResult::True,
-	            Optional<Version> version = Optional<Version>())
+	explicit ReadOptions(Optional<UID> debugID = Optional<UID>(),
+	                     ReadType type = ReadType::NORMAL,
+	                     CacheResult cache = CacheResult::True,
+	                     Optional<Version> version = Optional<Version>())
 	  : type(type), cacheResult(cache), debugID(debugID), consistencyCheckStartVersion(version) {}
 
-	ReadOptions(ReadType type, CacheResult cache = CacheResult::True) : ReadOptions({}, type, cache) {}
+	explicit ReadOptions(ReadType type, CacheResult cache = CacheResult::True) : ReadOptions({}, type, cache) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -1665,11 +1661,12 @@ struct ReadOptions {
 
 // Can be used to identify types (e.g. IDatabase) that can be used to create transactions with a `createTransaction`
 // function
-template <typename, typename = void>
+template <typename>
 struct transaction_creator_traits : std::false_type {};
 
 template <typename T>
-struct transaction_creator_traits<T, std::void_t<typename T::TransactionT>> : std::true_type {};
+    requires requires { typename T::TransactionT; }
+struct transaction_creator_traits<T> : std::true_type {};
 
 template <typename T>
 struct transaction_creator_traits<Reference<T>> : transaction_creator_traits<T> {};
@@ -1692,7 +1689,7 @@ struct Versionstamp {
 
 	Versionstamp() {}
 	Versionstamp(Version version, uint16_t batchNumber) : version(version), batchNumber(batchNumber) {}
-	Versionstamp(Standalone<StringRef> str) {
+	explicit Versionstamp(Standalone<StringRef> str) {
 		ASSERT(str.size() == sizeof(Version) + sizeof(batchNumber));
 		version = bigEndian64(*reinterpret_cast<const Version*>(str.begin()));
 		batchNumber = bigEndian16(*reinterpret_cast<const uint16_t*>(str.begin() + sizeof(Version)));
diff --git a/fdbclient/include/fdbclient/GenericManagementAPI.actor.h b/fdbclient/include/fdbclient/GenericManagementAPI.h
similarity index 75%
rename from fdbclient/include/fdbclient/GenericManagementAPI.actor.h
rename to fdbclient/include/fdbclient/GenericManagementAPI.h
index 7e9bd7f80d5..762e6e3fadc 100644
--- a/fdbclient/include/fdbclient/GenericManagementAPI.actor.h
+++ b/fdbclient/include/fdbclient/GenericManagementAPI.h
@@ -1,5 +1,5 @@
 /*
- * GenericManagementAPI.actor.h
+ * GenericManagementAPI.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,6 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GENERIC_MANAGEMENT_API_ACTOR_G_H)
-#define FDBCLIENT_GENERIC_MANAGEMENT_API_ACTOR_G_H
-#include "fdbclient/GenericManagementAPI.actor.g.h"
-#elif !defined(FDBCLIENT_GENERIC_MANAGEMENT_API_ACTOR_H)
-#define FDBCLIENT_GENERIC_MANAGEMENT_API_ACTOR_H
 
 /* This file defines "management" interfaces that have been templated to support both IClientAPI
 and Native version of databases, transactions, etc., and includes functions for performing cluster
@@ -41,8 +36,7 @@ the contents of the system key space.
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/StorageWiggleMetrics.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbclient/StorageWiggleMetrics.h"
 
 // ConfigurationResult enumerates normal outcomes of changeConfig() and various error
 // conditions specific to it.  changeConfig may also throw an Error to report other problems.
@@ -70,7 +64,8 @@ enum class ConfigurationResult {
 	DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL,
 	DATABASE_IS_REGISTERED,
 	INVALID_STORAGE_TYPE,
-	BACKUP_WORKER_ENABLED_RESTRICTED
+	BACKUP_WORKER_ENABLED_RESTRICTED,
+	RANGE_BACKUP_WORKER_ENABLED_RESTRICTED
 };
 
 enum class CoordinatorsResult {
@@ -134,17 +129,17 @@ ConfigureAutoResult parseConfig(StatusObject const& status);
 // Management API written in template code to support both IClientAPI and NativeAPI
 namespace ManagementAPI {
 
-ACTOR template <class Tr>
+template <class Tr>
 Future<std::vector<ProcessData>> getWorkers(Reference<Tr> tr,
                                             typename Tr::template FutureT<RangeResult> processClassesF,
                                             typename Tr::template FutureT<RangeResult> processDataF) {
 	// processClassesF and processDataF are used to hold standalone memory
 	processClassesF = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
 	processDataF = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
-	state Future<RangeResult> processClasses = safeThreadFutureToFuture(processClassesF);
-	state Future<RangeResult> processData = safeThreadFutureToFuture(processDataF);
+	Future<RangeResult> processClasses = safeThreadFutureToFuture(processClassesF);
+	Future<RangeResult> processData = safeThreadFutureToFuture(processDataF);
 
-	wait(success(processClasses) && success(processData));
+	co_await (success(processClasses) && success(processData));
 	ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
 
@@ -168,31 +163,31 @@ Future<std::vector<ProcessData>> getWorkers(Reference<Tr> tr,
 			results.push_back(data);
 	}
 
-	return results;
+	co_return results;
 }
 
 // All versions of changeConfig apply the given set of configuration tokens to the database, and return a
 // ConfigurationResult (or error).
 
 // Accepts a full configuration in key/value format (from buildConfiguration)
-ACTOR template <class DB>
+template <class DB>
 Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string, std::string> m, bool force) {
-	state StringRef initIdKey = "\xff/init_id"_sr;
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	StringRef initIdKey = "\xff/init_id"_sr;
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
 
 	if (!m.size()) {
-		return ConfigurationResult::NO_OPTIONS_PROVIDED;
+		co_return ConfigurationResult::NO_OPTIONS_PROVIDED;
 	}
 
 	// make sure we have essential configuration options
 	std::string initKey = configKeysPrefix.toString() + "initialized";
-	state bool creating = m.count(initKey) != 0;
-	state Optional<UID> locked;
+	bool creating = m.count(initKey) != 0;
+	Optional<UID> locked;
 	{
 		auto iter = m.find(databaseLockedKey.toString());
 		if (iter != m.end()) {
 			if (!creating) {
-				return ConfigurationResult::LOCKED_NOT_NEW;
+				co_return ConfigurationResult::LOCKED_NOT_NEW;
 			}
 			locked = UID::fromString(iter->second);
 			m.erase(iter);
@@ -201,20 +196,22 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 	if (creating) {
 		m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString();
 		if (!isCompleteConfiguration(m)) {
-			return ConfigurationResult::INCOMPLETE_CONFIGURATION;
+			co_return ConfigurationResult::INCOMPLETE_CONFIGURATION;
 		}
 	}
 
-	state Future<Void> tooLong = delay(60);
-	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
-	state bool oldReplicationUsesDcId = false;
+	Future<Void> tooLong = delay(60);
+	Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
+	bool oldReplicationUsesDcId = false;
 	// the caller need to reset the perpetual wiggle stats if pw=0 in case the reset txn on DD side is cancelled
 	// due to DD can die at the same time
-	state bool resetPPWStats = false;
-	state bool warnPPWGradual = false;
-	state bool warnShardedRocksDBIsExperimental = false;
+	bool resetPPWStats = false;
+	bool warnPPWGradual = false;
+	bool warnShardedRocksDBIsExperimental = false;
 
-	loop {
+	while (true) {
+		Error err;
+		bool hasErr = false;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -222,32 +219,32 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 			tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 
 			if (!creating && !force) {
-				state typename DB::TransactionT::template FutureT<RangeResult> fConfigF =
+				typename DB::TransactionT::template FutureT<RangeResult> fConfigF =
 				    tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fConfig = safeThreadFutureToFuture(fConfigF);
-				state typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
-				state typename DB::TransactionT::template FutureT<RangeResult> processDataF;
-				state Future<std::vector<ProcessData>> fWorkers = getWorkers(tr, processClassesF, processDataF);
-				wait(success(fConfig) || tooLong);
+				Future<RangeResult> fConfig = safeThreadFutureToFuture(fConfigF);
+				typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
+				typename DB::TransactionT::template FutureT<RangeResult> processDataF;
+				Future<std::vector<ProcessData>> fWorkers = getWorkers(tr, processClassesF, processDataF);
+				co_await (success(fConfig) || tooLong);
 
 				if (!fConfig.isReady()) {
-					return ConfigurationResult::DATABASE_UNAVAILABLE;
+					co_return ConfigurationResult::DATABASE_UNAVAILABLE;
 				}
 
 				if (fConfig.isReady()) {
 					ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY);
-					state DatabaseConfiguration oldConfig;
+					DatabaseConfiguration oldConfig;
 					oldConfig.fromKeyValues((VectorRef<KeyValueRef>)fConfig.get());
-					state DatabaseConfiguration newConfig = oldConfig;
+					DatabaseConfiguration newConfig = oldConfig;
 					for (auto kv : m) {
 						newConfig.set(kv.first, kv.second);
 					}
 					if (!newConfig.isValid()) {
-						return ConfigurationResult::INVALID_CONFIGURATION;
+						co_return ConfigurationResult::INVALID_CONFIGURATION;
 					}
 
 					if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) {
-						return ConfigurationResult::REGION_REPLICATION_MISMATCH;
+						co_return ConfigurationResult::REGION_REPLICATION_MISMATCH;
 					}
 
 					oldReplicationUsesDcId =
@@ -261,7 +258,7 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 						}
 						for (auto& it : oldConfig.regions) {
 							if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) {
-								return ConfigurationResult::REGIONS_CHANGED;
+								co_return ConfigurationResult::REGIONS_CHANGED;
 							}
 						}
 
@@ -273,23 +270,23 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 							}
 						}
 						if (activeRegionCount > 1) {
-							return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
+							co_return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
 						}
 					}
 
-					state typename DB::TransactionT::template FutureT<RangeResult> fServerListF =
+					typename DB::TransactionT::template FutureT<RangeResult> fServerListF =
 					    tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-					state Future<RangeResult> fServerList =
+					Future<RangeResult> fServerList =
 					    (newConfig.regions.size()) ? safeThreadFutureToFuture(fServerListF) : Future<RangeResult>();
 
 					if (newConfig.usableRegions == 2) {
 						if (oldReplicationUsesDcId) {
-							state typename DB::TransactionT::template FutureT<RangeResult> fLocalityListF =
+							typename DB::TransactionT::template FutureT<RangeResult> fLocalityListF =
 							    tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
-							state Future<RangeResult> fLocalityList = safeThreadFutureToFuture(fLocalityListF);
-							wait(success(fLocalityList) || tooLong);
+							Future<RangeResult> fLocalityList = safeThreadFutureToFuture(fLocalityListF);
+							co_await (success(fLocalityList) || tooLong);
 							if (!fLocalityList.isReady()) {
-								return ConfigurationResult::DATABASE_UNAVAILABLE;
+								co_return ConfigurationResult::DATABASE_UNAVAILABLE;
 							}
 							RangeResult localityList = fLocalityList.get();
 							ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY);
@@ -304,28 +301,27 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 
 							for (auto& it : newConfig.regions) {
 								if (localityDcIds.count(it.dcId) == 0) {
-									return ConfigurationResult::DCID_MISSING;
+									co_return ConfigurationResult::DCID_MISSING;
 								}
 							}
 						} else {
 							// all regions with priority >= 0 must be fully replicated
-							state std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>>
-							    replicasFuturesF;
-							state std::vector<Future<Optional<Value>>> replicasFutures;
+							std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>> replicasFuturesF;
+							std::vector<Future<Optional<Value>>> replicasFutures;
 							for (auto& it : newConfig.regions) {
 								if (it.priority >= 0) {
 									replicasFuturesF.push_back(tr->get(datacenterReplicasKeyFor(it.dcId)));
 									replicasFutures.push_back(safeThreadFutureToFuture(replicasFuturesF.back()));
 								}
 							}
-							wait(waitForAll(replicasFutures) || tooLong);
+							co_await (waitForAll(replicasFutures) || tooLong);
 
 							for (auto& it : replicasFutures) {
 								if (!it.isReady()) {
-									return ConfigurationResult::DATABASE_UNAVAILABLE;
+									co_return ConfigurationResult::DATABASE_UNAVAILABLE;
 								}
 								if (!it.get().present()) {
-									return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
+									co_return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
 								}
 							}
 						}
@@ -333,9 +329,9 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 
 					if (newConfig.regions.size()) {
 						// all storage servers must be in one of the regions
-						wait(success(fServerList) || tooLong);
+						co_await (success(fServerList) || tooLong);
 						if (!fServerList.isReady()) {
-							return ConfigurationResult::DATABASE_UNAVAILABLE;
+							co_return ConfigurationResult::DATABASE_UNAVAILABLE;
 						}
 						RangeResult serverList = fServerList.get();
 						ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
@@ -352,13 +348,13 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 							}
 						}
 						if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) {
-							return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
+							co_return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
 						}
 					}
 
-					wait(success(fWorkers) || tooLong);
+					co_await (success(fWorkers) || tooLong);
 					if (!fWorkers.isReady()) {
-						return ConfigurationResult::DATABASE_UNAVAILABLE;
+						co_return ConfigurationResult::DATABASE_UNAVAILABLE;
 					}
 
 					if (newConfig.regions.size()) {
@@ -371,7 +367,7 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 						for (auto& region : newConfig.regions) {
 							if (dcId_zoneIds[region.dcId].size() <
 							    std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
-								return ConfigurationResult::NOT_ENOUGH_WORKERS;
+								co_return ConfigurationResult::NOT_ENOUGH_WORKERS;
 							}
 							if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
 								int totalSatelliteProcesses = 0;
@@ -379,7 +375,7 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 									totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size();
 								}
 								if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) {
-									return ConfigurationResult::NOT_ENOUGH_WORKERS;
+									co_return ConfigurationResult::NOT_ENOUGH_WORKERS;
 								}
 							}
 						}
@@ -391,17 +387,17 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 							}
 						}
 						if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
-							return ConfigurationResult::NOT_ENOUGH_WORKERS;
+							co_return ConfigurationResult::NOT_ENOUGH_WORKERS;
 						}
 					}
 
 					if (!newConfig.storageServerStoreType.isValid() || !newConfig.tLogDataStoreType.isValid()) {
-						return ConfigurationResult::INVALID_STORAGE_TYPE;
+						co_return ConfigurationResult::INVALID_STORAGE_TYPE;
 					}
 
 					if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
 					    newConfig.storageMigrationType == StorageMigrationType::DISABLED) {
-						return ConfigurationResult::STORAGE_MIGRATION_DISABLED;
+						co_return ConfigurationResult::STORAGE_MIGRATION_DISABLED;
 					} else if (newConfig.storageMigrationType == StorageMigrationType::GRADUAL &&
 					           newConfig.perpetualStorageWiggleSpeed == 0) {
 						warnPPWGradual = true;
@@ -440,82 +436,100 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 					}
 				}
 
-				// Clear backup progress when backup workers are disabled
+				// Clear partitioned backup progress when backup workers are disabled
 				if (i->first == backupWorkerEnabledKey && i->second == "0") {
 					tr->clear(backupProgressKeys);
 					TraceEvent("BackupWorkerProgressCleared");
 				}
+
+				// Clear range partitioned backup progress when range partitioned backup workers are disabled
+				if (i->first == rangeBackupWorkerEnabledKey && i->second == "0") {
+					tr->clear(backupProgressKeys);
+					TraceEvent("RangePartitionedBackupWorkerProgressCleared");
+				}
 			}
 
 			if (!creating && resetPPWStats) {
-				state StorageWiggleData wiggleData;
-				wait(wiggleData.resetStorageWiggleMetrics(tr, PrimaryRegion(true)));
-				wait(wiggleData.resetStorageWiggleMetrics(tr, PrimaryRegion(false)));
+				StorageWiggleData wiggleData;
+				co_await wiggleData.resetStorageWiggleMetrics(tr, PrimaryRegion(true));
+				co_await wiggleData.resetStorageWiggleMetrics(tr, PrimaryRegion(false));
 			}
 
 			tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
 			tr->set(moveKeysLockOwnerKey, versionKey);
 
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 			break;
 		} catch (Error& e) {
-			state Error e1(e);
-			if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) {
+			err = e;
+			hasErr = true;
+		}
+		if (hasErr) {
+			Error e1(err);
+			if ((err.code() == error_code_not_committed || err.code() == error_code_transaction_too_old) && creating) {
 				// The database now exists.  Determine whether we created it or it was already existing/created by
 				// someone else.  The latter is an error.
 				tr->reset();
-				loop {
+				while (true) {
+					Error err;
+					bool hasErr = false;
 					try {
 						tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 						tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 						tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 						tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 
-						state typename DB::TransactionT::template FutureT<Optional<Value>> vF = tr->get(initIdKey);
-						Optional<Value> v = wait(safeThreadFutureToFuture(vF));
+						typename DB::TransactionT::template FutureT<Optional<Value>> vF = tr->get(initIdKey);
+						Optional<Value> v = co_await safeThreadFutureToFuture(vF);
 						if (v != m[initIdKey.toString()])
-							return ConfigurationResult::DATABASE_ALREADY_CREATED;
+							co_return ConfigurationResult::DATABASE_ALREADY_CREATED;
 						else if (m[configKeysPrefix.toString() + "storage_engine"] ==
 						         std::to_string(KeyValueStoreType::SSD_SHARDED_ROCKSDB))
-							return ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
+							co_return ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
 						else
-							return ConfigurationResult::DATABASE_CREATED;
+							co_return ConfigurationResult::DATABASE_CREATED;
 					} catch (Error& e2) {
-						wait(safeThreadFutureToFuture(tr->onError(e2)));
+						err = e2;
+						hasErr = true;
+					}
+					if (hasErr) {
+						co_await safeThreadFutureToFuture(tr->onError(err));
 					}
 				}
 			}
-			wait(safeThreadFutureToFuture(tr->onError(e1)));
+			co_await safeThreadFutureToFuture(tr->onError(e1));
 		}
 	}
 
 	if (warnPPWGradual) {
-		return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL;
+		co_return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL;
 	} else if (warnShardedRocksDBIsExperimental) {
-		return ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
+		co_return ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
 	} else {
-		return ConfigurationResult::SUCCESS;
+		co_return ConfigurationResult::SUCCESS;
 	}
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<ConfigurationResult> autoConfig(Reference<DB> db, ConfigureAutoResult conf) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
-	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
 
 	if (!conf.address_class.size())
-		return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type
+		co_return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type
 
-	loop {
+	while (true) {
+		Error err;
+		bool hasErr = false;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
 
-			state typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
-			state typename DB::TransactionT::template FutureT<RangeResult> processDataF;
-			std::vector<ProcessData> workers = wait(getWorkers(tr, processClassesF, processDataF));
+			typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
+			typename DB::TransactionT::template FutureT<RangeResult> processDataF;
+			std::vector<ProcessData> workers = co_await getWorkers(tr, processClassesF, processDataF);
 			std::map<NetworkAddress, Optional<Standalone<StringRef>>> address_processId;
 			for (auto& w : workers) {
 				address_processId[w.address] = w.locality.processId();
@@ -550,7 +564,7 @@ Future<ConfigurationResult> autoConfig(Reference<DB> db, ConfigureAutoResult con
 				std::map<std::string, std::string> m;
 				auto r = buildConfiguration(modes, m);
 				if (r != ConfigurationResult::SUCCESS)
-					return r;
+					co_return r;
 
 				for (auto& kv : m)
 					tr->set(kv.first, kv.second);
@@ -559,10 +573,14 @@ Future<ConfigurationResult> autoConfig(Reference<DB> db, ConfigureAutoResult con
 			tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
 			tr->set(moveKeysLockOwnerKey, versionKey);
 
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return ConfigurationResult::SUCCESS;
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return ConfigurationResult::SUCCESS;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
+			hasErr = true;
+		}
+		if (hasErr) {
+			co_await safeThreadFutureToFuture(tr->onError(err));
 		}
 	}
 }
@@ -600,6 +618,3 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db,
 std::string generateErrorMessage(const CoordinatorsResult& res);
 
 } // namespace ManagementAPI
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/GlobalConfig.actor.h b/fdbclient/include/fdbclient/GlobalConfig.actor.h
deleted file mode 100644
index ce199053bbe..00000000000
--- a/fdbclient/include/fdbclient/GlobalConfig.actor.h
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * GlobalConfig.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GLOBALCONFIG_ACTOR_G_H)
-#define FDBCLIENT_GLOBALCONFIG_ACTOR_G_H
-#include "fdbclient/GlobalConfig.actor.g.h"
-#elif !defined(FDBCLIENT_GLOBALCONFIG_ACTOR_H)
-#define FDBCLIENT_GLOBALCONFIG_ACTOR_H
-
-#include <any>
-#include <map>
-#include <optional>
-#include <type_traits>
-#include <unordered_map>
-
-#include "fdbclient/CommitProxyInterface.h"
-#include "fdbclient/GlobalConfig.h"
-#include "fdbclient/ReadYourWrites.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-// The global configuration is a series of typed key-value pairs synced to all
-// nodes (server and client) in an FDB cluster in an eventually consistent
-// manner. Only small key-value pairs should be stored in global configuration;
-// an excessive amount of data can cause synchronization slowness.
-
-// Keys
-extern const KeyRef fdbClientInfoTxnSampleRate;
-extern const KeyRef fdbClientInfoTxnSizeLimit;
-
-extern const KeyRef transactionTagSampleRate;
-extern const KeyRef transactionTagSampleCost;
-
-extern const KeyRef samplingFrequency;
-extern const KeyRef samplingWindow;
-
-// Structure used to hold the values stored by global configuration. The arena
-// is used as memory to store both the key and the value (the value is only
-// stored in the arena if it is an object; primitives are just copied).
-struct ConfigValue : ReferenceCounted<ConfigValue> {
-	Arena arena;
-	std::any value;
-
-	ConfigValue() {}
-	ConfigValue(Arena&& a, std::any&& v) : arena(a), value(v) {}
-};
-
-class GlobalConfig : NonCopyable {
-public:
-	// Requires a database pointer to allow global configuration to run
-	// transactions on the database.
-	explicit GlobalConfig(DatabaseContext* cx);
-
-	// Requires an AsyncVar object to watch for changes on. The ClientDBInfo pointer
-	// should point to a ClientDBInfo object which will contain the updated
-	// global configuration history when the given AsyncVar changes. This
-	// function should be called whenever the database object changes, in order
-	// to allow global configuration to run transactions on the latest
-	// database.
-	template <class T>
-	void init(Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
-		_updater = updater(this, dbInfo);
-		// Bind changes in `db` to the `dbInfoChanged` AsyncTrigger.
-		// TODO: Change AsyncTrigger to a Reference
-		_forward = forward(db, std::addressof(dbInfoChanged));
-	}
-
-	// Given a list of insertions and clears, applies the necessary changes to
-	// the given transaction to update the global configuration keyspace. Keys
-	// in the list of mutations should not include the global configuration
-	// prefix (`\xff\xff/global_config/`). The caller must still commit the
-	// given transaction in order to persist the changes.
-	static void applyChanges(Transaction& tr,
-	                         const VectorRef<KeyValueRef>& insertions,
-	                         const VectorRef<KeyRangeRef>& clears);
-
-	// Use this function to turn a global configuration key defined above into
-	// the full path needed to set the value in the database.
-	//
-	// For example, given "config/a", returns "\xff\xff/global_config/config/a".
-	static Key prefixedKey(KeyRef key);
-
-	// Get a value from the framework. Values are returned as a ConfigValue
-	// reference which also contains the arena holding the object. As long as
-	// the caller keeps the ConfigValue reference, the value is guaranteed to
-	// be readable. An empty reference is returned if the value does not exist.
-	const Reference<ConfigValue> get(KeyRef name);
-	const std::map<KeyRef, Reference<ConfigValue>> get(KeyRangeRef range);
-
-	// For arithmetic value types, returns a copy of the value for the given
-	// key, or the supplied default value if the framework does not know about
-	// the key.
-	template <typename T, typename std::enable_if<std::is_arithmetic<T>{}, bool>::type = true>
-	const T get(KeyRef name, T defaultVal) {
-		try {
-			auto configValue = get(name);
-			if (configValue.isValid()) {
-				if (configValue->value.has_value()) {
-					return std::any_cast<T>(configValue->value);
-				}
-			}
-
-			return defaultVal;
-		} catch (Error& e) {
-			throw;
-		}
-	}
-
-	// Trying to write into the global configuration keyspace? To write data,
-	// submit a transaction to \xff\xff/global_config/<your-key> with
-	// <your-value> encoded using the FDB tuple typecodes. Use the helper
-	// function `prefixedKey` to correctly prefix your global configuration
-	// key.
-
-	// Triggers the returned future when the global configuration singleton has
-	// been created and is ready.
-	Future<Void> onInitialized();
-
-	// Triggers the returned future when any key-value pair in the global
-	// configuration changes.
-	Future<Void> onChange();
-
-	// Calls \ref fn when the value associated with \ref key is changed. \ref
-	// key should be one of the string literals defined at the top of
-	// GlobalConfig.actor.cpp, to ensure memory validity. \ref fn is passed the
-	// updated value for the key, or an empty optional if the key has been
-	// cleared. If the value is an allocated object, its memory remains in the
-	// control of global configuration.
-	void trigger(KeyRef key, std::function<void(std::optional<std::any>)> fn);
-
-private:
-	// The functions below only affect the local copy of the global
-	// configuration keyspace! To insert or remove values across all nodes you
-	// must use a transaction (see the note above).
-
-	// Inserts the given key-value pair into the local copy of the global
-	// configuration keyspace, overwriting the old key-value pair if it exists.
-	// `value` must be encoded using the FDB tuple typecodes.
-	void insert(KeyRef key, ValueRef value);
-	// Removes the given key (and associated value) from the local copy of the
-	// global configuration keyspace.
-	void erase(Key key);
-	// Removes the given key range (and associated values) from the local copy
-	// of the global configuration keyspace.
-	void erase(KeyRangeRef range);
-
-	// Updates local copy of global configuration by reading the entire key-range
-	// from storage (proxied through the GrvProxies). Returns the version of the
-	// refreshed data.
-	ACTOR static Future<Version> refresh(GlobalConfig* self, Version lastKnown, Version largestSeen);
-	ACTOR static Future<Void> updater(GlobalConfig* self, const ClientDBInfo* dbInfo);
-
-	DatabaseContext* cx;
-	AsyncTrigger dbInfoChanged;
-	Future<Void> _forward;
-	Future<Void> _updater;
-	Promise<Void> initialized;
-	AsyncTrigger configChanged;
-	std::unordered_map<StringRef, Reference<ConfigValue>> data;
-	Version lastUpdate;
-	// The key should be a global config string literal key (see the top of this class).
-	std::unordered_map<KeyRef, std::function<void(std::optional<std::any>)>> callbacks;
-};
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbclient/include/fdbclient/GlobalConfig.h b/fdbclient/include/fdbclient/GlobalConfig.h
index ac7f680f18e..eaad72b186a 100644
--- a/fdbclient/include/fdbclient/GlobalConfig.h
+++ b/fdbclient/include/fdbclient/GlobalConfig.h
@@ -20,8 +20,21 @@
 
 #pragma once
 
+#include <any>
+#include <functional>
+#include <map>
+#include <memory>
+#include <optional>
+#include <type_traits>
+#include <unordered_map>
+
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
+#include "flow/genericactors.actor.h"
+
+class DatabaseContext;
+struct ClientDBInfo;
+class Transaction;
 
 // Used to store a list of mutations made to the global configuration at a
 // specific version.
@@ -29,7 +42,7 @@ struct VersionHistory {
 	constexpr static FileIdentifier file_identifier = 5863456;
 
 	VersionHistory() {}
-	VersionHistory(Version v) : version(v) {}
+	explicit VersionHistory(Version v) : version(v) {}
 
 	Version version;
 	Standalone<VectorRef<MutationRef>> mutations;
@@ -43,3 +56,147 @@ struct VersionHistory {
 		serializer(ar, mutations, version);
 	}
 };
+
+// The global configuration is a series of typed key-value pairs synced to all
+// nodes (server and client) in an FDB cluster in an eventually consistent
+// manner. Only small key-value pairs should be stored in global configuration;
+// an excessive amount of data can cause synchronization slowness.
+
+// Keys
+extern const KeyRef fdbClientInfoTxnSampleRate;
+extern const KeyRef fdbClientInfoTxnSizeLimit;
+
+extern const KeyRef transactionTagSampleRate;
+extern const KeyRef transactionTagSampleCost;
+
+extern const KeyRef samplingFrequency;
+extern const KeyRef samplingWindow;
+
+// Structure used to hold the values stored by global configuration. The arena
+// is used as memory to store both the key and the value (the value is only
+// stored in the arena if it is an object; primitives are just copied).
+struct ConfigValue : ReferenceCounted<ConfigValue> {
+	Arena arena;
+	std::any value;
+
+	ConfigValue() {}
+	ConfigValue(Arena&& a, std::any&& v) : arena(a), value(v) {}
+};
+
+class GlobalConfig : NonCopyable {
+public:
+	// Requires a database pointer to allow global configuration to run
+	// transactions on the database.
+	explicit GlobalConfig(DatabaseContext* cx);
+
+	// Requires an AsyncVar object to watch for changes on. The ClientDBInfo pointer
+	// should point to a ClientDBInfo object which will contain the updated
+	// global configuration history when the given AsyncVar changes. This
+	// function should be called whenever the database object changes, in order
+	// to allow global configuration to run transactions on the latest
+	// database.
+	template <class T>
+	void init(Reference<AsyncVar<T> const> db, const ClientDBInfo* dbInfo) {
+		_updater = updater(dbInfo);
+		// Bind changes in `db` to the `dbInfoChanged` AsyncTrigger.
+		// TODO: Change AsyncTrigger to a Reference
+		_forward = forward(db, std::addressof(dbInfoChanged));
+	}
+
+	// Given a list of insertions and clears, applies the necessary changes to
+	// the given transaction to update the global configuration keyspace. Keys
+	// in the list of mutations should not include the global configuration
+	// prefix (`\xff\xff/global_config/`). The caller must still commit the
+	// given transaction in order to persist the changes.
+	static void applyChanges(Transaction& tr,
+	                         const VectorRef<KeyValueRef>& insertions,
+	                         const VectorRef<KeyRangeRef>& clears);
+
+	// Use this function to turn a global configuration key defined above into
+	// the full path needed to set the value in the database.
+	//
+	// For example, given "config/a", returns "\xff\xff/global_config/config/a".
+	static Key prefixedKey(KeyRef key);
+
+	// Get a value from the framework. Values are returned as a ConfigValue
+	// reference which also contains the arena holding the object. As long as
+	// the caller keeps the ConfigValue reference, the value is guaranteed to
+	// be readable. An empty reference is returned if the value does not exist.
+	Reference<ConfigValue> get(KeyRef name);
+	std::map<KeyRef, Reference<ConfigValue>> get(KeyRangeRef range);
+
+	// For arithmetic value types, returns a copy of the value for the given
+	// key, or the supplied default value if the framework does not know about
+	// the key.
+	template <typename T>
+	    requires(std::is_arithmetic_v<T>)
+	const T get(KeyRef name, T defaultVal) {
+		try {
+			auto configValue = get(name);
+			if (configValue.isValid()) {
+				if (configValue->value.has_value()) {
+					return std::any_cast<T>(configValue->value);
+				}
+			}
+
+			return defaultVal;
+		} catch (Error& e) {
+			throw;
+		}
+	}
+
+	// Trying to write into the global configuration keyspace? To write data,
+	// submit a transaction to \xff\xff/global_config/<your-key> with
+	// <your-value> encoded using the FDB tuple typecodes. Use the helper
+	// function `prefixedKey` to correctly prefix your global configuration
+	// key.
+
+	// Triggers the returned future when the global configuration singleton has
+	// been created and is ready.
+	Future<Void> onInitialized();
+
+	// Triggers the returned future when any key-value pair in the global
+	// configuration changes.
+	Future<Void> onChange();
+
+	// Calls \ref fn when the value associated with \ref key is changed. \ref
+	// key should be one of the string literals defined at the top of
+	// GlobalConfig.cpp, to ensure memory validity. \ref fn is passed the
+	// updated value for the key, or an empty optional if the key has been
+	// cleared. If the value is an allocated object, its memory remains in the
+	// control of global configuration.
+	void trigger(KeyRef key, std::function<void(std::optional<std::any>)> fn);
+
+private:
+	// The functions below only affect the local copy of the global
+	// configuration keyspace. To insert or remove values across all nodes you
+	// must use a transaction (see the note above).
+
+	// Inserts the given key-value pair into the local copy of the global
+	// configuration keyspace, overwriting the old key-value pair if it exists.
+	// `value` must be encoded using the FDB tuple typecodes.
+	void insert(KeyRef key, ValueRef value);
+	// Removes the given key (and associated value) from the local copy of the
+	// global configuration keyspace.
+	void erase(Key key);
+	// Removes the given key range (and associated values) from the local copy
+	// of the global configuration keyspace.
+	void erase(KeyRangeRef range);
+
+	// Updates local copy of global configuration by reading the entire key-range
+	// from storage (proxied through the GrvProxies). Returns the version of the
+	// refreshed data.
+	Future<Version> refresh(Version lastKnown, Version largestSeen);
+	Future<Void> updater(const ClientDBInfo* dbInfo);
+
+	DatabaseContext* cx;
+	AsyncTrigger dbInfoChanged;
+	Future<Void> _forward;
+	Future<Void> _updater;
+	Promise<Void> initialized;
+	AsyncTrigger configChanged;
+	std::unordered_map<StringRef, Reference<ConfigValue>> data;
+	Version lastUpdate;
+	// The key should be a global config string literal key (see the top of this file).
+	std::unordered_map<KeyRef, std::function<void(std::optional<std::any>)>> callbacks;
+};
diff --git a/fdbclient/include/fdbclient/GrvProxyInterface.h b/fdbclient/include/fdbclient/GrvProxyInterface.h
index 347e5279a56..8c3872db015 100644
--- a/fdbclient/include/fdbclient/GrvProxyInterface.h
+++ b/fdbclient/include/fdbclient/GrvProxyInterface.h
@@ -21,10 +21,158 @@
 #ifndef FDBCLIENT_GRVPROXYINTERFACE_H
 #define FDBCLIENT_GRVPROXYINTERFACE_H
 #pragma once
+#include "fdbclient/TagThrottle.h"
+#include "fdbclient/VersionVector.h"
 #include "flow/FileIdentifier.h"
 #include "fdbrpc/fdbrpc.h"
+#include "fdbrpc/LoadBalance.actor.h"
+#include "fdbrpc/Stats.h"
+#include "fdbrpc/TimedRequest.h"
 #include "fdbclient/FDBTypes.h"
 
+struct GetReadVersionReply : public BasicLoadBalancedReply {
+	constexpr static FileIdentifier file_identifier = 15709388;
+	Version version;
+	bool locked;
+	Optional<Value> metadataVersion;
+	int64_t midShardSize = 0;
+	bool rkDefaultThrottled = false;
+	bool rkBatchThrottled = false;
+
+	TransactionTagMap<ClientTagThrottleLimits> tagThrottleInfo;
+
+	VersionVector ssVersionVectorDelta;
+	UID proxyId; // GRV proxy ID to detect old GRV proxies at client side
+
+	GetReadVersionReply() : version(invalidVersion), locked(false) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar,
+		           BasicLoadBalancedReply::processBusyTime,
+		           version,
+		           locked,
+		           metadataVersion,
+		           tagThrottleInfo,
+		           midShardSize,
+		           rkDefaultThrottled,
+		           rkBatchThrottled,
+		           ssVersionVectorDelta,
+		           proxyId);
+	}
+};
+
+struct GetReadVersionRequest : TimedRequest {
+	constexpr static FileIdentifier file_identifier = 838566;
+	enum {
+		PRIORITY_SYSTEM_IMMEDIATE =
+		    15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
+		PRIORITY_DEFAULT = 8 << 24,
+		PRIORITY_BATCH = 1 << 24
+	};
+	enum {
+		FLAG_USE_MIN_KNOWN_COMMITTED_VERSION = 4,
+		FLAG_USE_PROVISIONAL_PROXIES = 2,
+		FLAG_CAUSAL_READ_RISKY = 1,
+		FLAG_PRIORITY_MASK = PRIORITY_SYSTEM_IMMEDIATE,
+	};
+
+	SpanContext spanContext;
+	uint32_t transactionCount;
+	uint32_t flags;
+	TransactionPriority priority;
+
+	TransactionTagMap<uint32_t> tags;
+
+	Optional<UID> debugID;
+	ReplyPromise<GetReadVersionReply> reply;
+
+	Version maxVersion; // max version in the client's version vector cache
+	Optional<int64_t> maxGrvQueueDelayMS; // max tolerated GRV proxy queue delay due to ratekeeper throttling
+
+	GetReadVersionRequest() : transactionCount(1), flags(0), maxVersion(invalidVersion) {}
+	GetReadVersionRequest(SpanContext spanContext,
+	                      uint32_t transactionCount,
+	                      TransactionPriority priority,
+	                      Version maxVersion,
+	                      uint32_t flags = 0,
+	                      TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
+	                      Optional<UID> debugID = Optional<UID>(),
+	                      Optional<int64_t> maxGrvQueueDelayMS = Optional<int64_t>())
+	  : spanContext(spanContext), transactionCount(transactionCount), flags(flags), priority(priority), tags(tags),
+	    debugID(debugID), maxVersion(maxVersion), maxGrvQueueDelayMS(maxGrvQueueDelayMS) {
+		this->flags &= ~FLAG_PRIORITY_MASK;
+		switch (priority) {
+		case TransactionPriority::BATCH:
+			this->flags |= PRIORITY_BATCH;
+			break;
+		case TransactionPriority::DEFAULT:
+			this->flags |= PRIORITY_DEFAULT;
+			break;
+		case TransactionPriority::IMMEDIATE:
+			this->flags |= PRIORITY_SYSTEM_IMMEDIATE;
+			break;
+		default:
+			ASSERT(false);
+		}
+	}
+
+	bool verify() const { return true; }
+
+	bool operator<(GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
+
+	bool isTagged() const { return !tags.empty(); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext, maxVersion, maxGrvQueueDelayMS);
+
+		if (ar.isDeserializing) {
+			if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
+				priority = TransactionPriority::IMMEDIATE;
+			} else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
+				priority = TransactionPriority::DEFAULT;
+			} else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
+				priority = TransactionPriority::BATCH;
+			} else {
+				priority = TransactionPriority::DEFAULT;
+			}
+		}
+	}
+};
+
+struct GlobalConfigRefreshReply {
+	constexpr static FileIdentifier file_identifier = 12680327;
+	Arena arena;
+	Version version;
+	RangeResultRef result;
+
+	GlobalConfigRefreshReply() {}
+	GlobalConfigRefreshReply(Arena const& arena, Version version, RangeResultRef result)
+	  : arena(arena), version(version), result(result) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, result, version, arena);
+	}
+};
+
+struct GlobalConfigRefreshRequest {
+	constexpr static FileIdentifier file_identifier = 2828131;
+	Version lastKnown;
+	ReplyPromise<GlobalConfigRefreshReply> reply;
+
+	GlobalConfigRefreshRequest() {}
+	explicit GlobalConfigRefreshRequest(Version lastKnown) : lastKnown(lastKnown) {}
+
+	bool verify() const noexcept { return true; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, lastKnown, reply);
+	}
+};
+
 // GrvProxy is proxy primarily specializing on serving GetReadVersion. It also
 // serves health metrics since it communicates with RateKeeper to gather health
 // information of the cluster, and handles proxied GlobalConfig requests.
diff --git a/fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h b/fdbclient/include/fdbclient/HighContentionPrefixAllocator.h
similarity index 66%
rename from fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h
rename to fdbclient/include/fdbclient/HighContentionPrefixAllocator.h
index d685efdc2b0..88e41857078 100644
--- a/fdbclient/include/fdbclient/HighContentionPrefixAllocator.actor.h
+++ b/fdbclient/include/fdbclient/HighContentionPrefixAllocator.h
@@ -1,5 +1,5 @@
 /*
- * HighContentionPrefixAllocator.actor.h
+ * HighContentionPrefixAllocator.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,24 +20,15 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_HIGHCONTENTIONPREFIXALLOCATOR_ACTOR_G_H)
-#define FDBCLIENT_HIGHCONTENTIONPREFIXALLOCATOR_ACTOR_G_H
-#include "fdbclient/HighContentionPrefixAllocator.actor.g.h"
-#elif !defined(FDBCLIENT_HIGHCONTENTIONPREFIXALLOCATOR_ACTOR_H)
-#define FDBCLIENT_HIGHCONTENTIONPREFIXALLOCATOR_ACTOR_H
-
 #include "fdbclient/ClientBooleanParams.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/Subspace.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 class HighContentionPrefixAllocator {
 public:
-	HighContentionPrefixAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
+	explicit HighContentionPrefixAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
 
 	template <class TransactionT>
 	Future<Standalone<StringRef>> allocate(Reference<TransactionT> tr) {
@@ -59,22 +50,22 @@ class HighContentionPrefixAllocator {
 	Subspace counters;
 	Subspace recent;
 
-	ACTOR template <class TransactionT>
+	template <class TransactionT>
 	Future<Standalone<StringRef>> allocate(HighContentionPrefixAllocator* self, Reference<TransactionT> tr) {
-		state int64_t start = 0;
-		state int64_t window = 0;
+		int64_t start = 0;
+		int64_t window = 0;
 
-		loop {
-			state typename TransactionT::template FutureT<RangeResult> rangeFuture =
+		while (true) {
+			typename TransactionT::template FutureT<RangeResult> rangeFuture =
 			    tr->getRange(self->counters.range(), 1, Snapshot::True, Reverse::True);
-			RangeResult range = wait(safeThreadFutureToFuture(rangeFuture));
+			RangeResult range = co_await safeThreadFutureToFuture(rangeFuture);
 
 			if (range.size() > 0) {
 				start = self->counters.unpack(range[0].key).getInt(0);
 			}
 
-			state bool windowAdvanced = false;
-			loop {
+			bool windowAdvanced = false;
+			while (true) {
 				// if thread safety is needed, this should be locked {
 				if (windowAdvanced) {
 					tr->clear(KeyRangeRef(self->counters.key(), self->counters.get(start).key()));
@@ -85,11 +76,11 @@ class HighContentionPrefixAllocator {
 				int64_t inc = 1;
 				tr->atomicOp(self->counters.get(start).key(), StringRef((uint8_t*)&inc, 8), MutationRef::AddValue);
 
-				state typename TransactionT::template FutureT<Optional<Value>> countFuture =
+				typename TransactionT::template FutureT<Optional<Value>> countFuture =
 				    tr->get(self->counters.get(start).key(), Snapshot::True);
 				// }
 
-				Optional<Value> countValue = wait(safeThreadFutureToFuture(countFuture));
+				Optional<Value> countValue = co_await safeThreadFutureToFuture(countFuture);
 
 				int64_t count = 0;
 				if (countValue.present()) {
@@ -108,20 +99,20 @@ class HighContentionPrefixAllocator {
 				windowAdvanced = true;
 			}
 
-			loop {
-				state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
+			while (true) {
+				int64_t candidate = deterministicRandom()->randomInt(start, start + window);
 
 				// if thread safety is needed, this should be locked {
-				state typename TransactionT::template FutureT<RangeResult> latestCounterFuture =
+				typename TransactionT::template FutureT<RangeResult> latestCounterFuture =
 				    tr->getRange(self->counters.range(), 1, Snapshot::True, Reverse::True);
-				state typename TransactionT::template FutureT<Optional<Value>> candidateValueFuture =
+				typename TransactionT::template FutureT<Optional<Value>> candidateValueFuture =
 				    tr->get(self->recent.get(candidate).key());
 				tr->setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 				tr->set(self->recent.get(candidate).key(), ValueRef());
 				// }
 
-				wait(success(safeThreadFutureToFuture(latestCounterFuture)) &&
-				     success(safeThreadFutureToFuture(candidateValueFuture)));
+				co_await (success(safeThreadFutureToFuture(latestCounterFuture)) &&
+				          success(safeThreadFutureToFuture(candidateValueFuture)));
 
 				int64_t currentWindowStart = 0;
 				if (latestCounterFuture.get().size() > 0) {
@@ -134,12 +125,9 @@ class HighContentionPrefixAllocator {
 
 				if (!candidateValueFuture.get().present()) {
 					tr->addWriteConflictRange(singleKeyRange(self->recent.get(candidate).key()));
-					return Tuple::makeTuple(candidate).pack();
+					co_return Tuple::makeTuple(candidate).pack();
 				}
 			}
 		}
 	}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/IBlobStore.h b/fdbclient/include/fdbclient/IBlobStore.h
new file mode 100644
index 00000000000..3c5f88629b0
--- /dev/null
+++ b/fdbclient/include/fdbclient/IBlobStore.h
@@ -0,0 +1,522 @@
+/*
+ * IBlobStore.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <functional>
+#include <map>
+#include <queue>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "fdbclient/Knobs.h"
+#include "fdbclient/JSONDoc.h"
+#include "fdbrpc/HTTP.h"
+#include "fdbrpc/Stats.h"
+#include "flow/IRandom.h"
+#include "flow/IRateControl.h"
+#include "flow/IConnection.h"
+#include "flow/Net2Packet.h"
+#include "flow/flow.h"
+
+#include <boost/functional/hash.hpp>
+
+// Blob store endpoint configuration knobs, shared across all implementations.
+struct BlobKnobs {
+	BlobKnobs();
+	int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min,
+	    requests_per_second, list_requests_per_second, write_requests_per_second, read_requests_per_second,
+	    delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
+	    concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file, enable_read_cache,
+	    read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second,
+	    max_recv_bytes_per_second, sdk_auth, enable_object_integrity_check, global_connection_pool,
+	    max_delay_retryable_error, max_delay_connection_failed, multipart_retry_delay_ms;
+
+	bool set(StringRef name, int value);
+	std::string getURLParameters() const;
+	static std::vector<std::string> getKnobDescriptions() {
+		return {
+			"secure_connection (or sc)             Set 1 for secure connection and 0 for insecure connection.",
+			"connect_tries (or ct)                 Number of times to try to connect for each request.",
+			"connect_timeout (or cto)              Number of seconds to wait for a connect request to succeed.",
+			"max_connection_life (or mcl)          Maximum number of seconds to use a single TCP connection.",
+			"request_tries (or rt)                 Number of times to try each request until a parsable HTTP "
+			"response other than 429 is received.",
+			"request_timeout_min (or rtom)         Number of seconds to wait for a request to succeed after a "
+			"connection is established.",
+			"requests_per_second (or rps)          Max number of requests to start per second.",
+			"list_requests_per_second (or lrps)    Max number of list requests to start per second.",
+			"write_requests_per_second (or wrps)   Max number of write requests to start per second.",
+			"read_requests_per_second (or rrps)    Max number of read requests to start per second.",
+			"delete_requests_per_second (or drps)  Max number of delete requests to start per second.",
+			"multipart_max_part_size (or maxps)    Max part size for multipart uploads.",
+			"multipart_min_part_size (or minps)    Min part size for multipart uploads.",
+			"multipart_retry_delay_ms (or mrd)     Delay in milliseconds between retries for multipart uploads.",
+			"concurrent_requests (or cr)           Max number of total requests in progress at once, regardless of "
+			"operation-specific concurrency limits.",
+			"concurrent_uploads (or cu)            Max concurrent uploads (part or whole) that can be in progress "
+			"at once.",
+			"concurrent_lists (or cl)              Max concurrent list operations that can be in progress at once.",
+			"concurrent_reads_per_file (or crps)   Max concurrent reads in progress for any one file.",
+			"concurrent_writes_per_file (or cwps)  Max concurrent uploads in progress for any one file.",
+			"enable_read_cache (or erc)            Whether read block caching is enabled.",
+			"read_block_size (or rbs)              Block size in bytes to be used for reads.",
+			"read_ahead_blocks (or rab)            Number of blocks to read ahead of requested offset.",
+			"read_cache_blocks_per_file (or rcb)   Size of the read cache for a file in blocks.",
+			"max_send_bytes_per_second (or sbps)   Max send bytes per second for all requests combined.",
+			"max_recv_bytes_per_second (or rbps)   Max receive bytes per second for all requests combined (NOT YET "
+			"USED).",
+			"max_delay_retryable_error (or dre)    Max seconds to delay before retry when see a retryable error.",
+			"max_delay_connection_failed (or dcf)  Max seconds to delay before retry when see a connection "
+			"failure.",
+			"sdk_auth (or sa)                      Use AWS SDK to resolve credentials. Only valid if "
+			"BUILD_AWS_BACKUP is enabled.",
+			"enable_object_integrity_check (or eoic) Enable integrity check on GET requests (Default: false).",
+			"global_connection_pool (or gcp)       Enable shared connection pool between all blobstore instances.",
+			"provider (or p)                       Blob store provider: s3 (default) or gcs."
+		};
+	}
+
+	bool isTLS() const { return secure_connection == 1; }
+};
+
+// Unique key that identifies interchangeable connections for the same settings and destination.
+// FIXME: can we define std::hash as a struct member of IBlobStoreEndpoint?
+struct BlobStoreConnectionPoolKey {
+	std::string host;
+	std::string service;
+	std::string region;
+	bool isTLS;
+
+	BlobStoreConnectionPoolKey(const std::string& host,
+	                           const std::string& service,
+	                           const std::string& region,
+	                           bool isTLS)
+	  : host(host), service(service), region(region), isTLS(isTLS) {}
+
+	bool operator==(const BlobStoreConnectionPoolKey& other) const {
+		return isTLS == other.isTLS && host == other.host && service == other.service && region == other.region;
+	}
+};
+
+namespace std {
+template <>
+struct hash<BlobStoreConnectionPoolKey> {
+	std::size_t operator()(const BlobStoreConnectionPoolKey& key) const {
+		std::size_t seed = 0;
+		boost::hash_combine(seed, std::hash<std::string>{}(key.host));
+		boost::hash_combine(seed, std::hash<std::string>{}(key.service));
+		boost::hash_combine(seed, std::hash<std::string>{}(key.region));
+		boost::hash_combine(seed, std::hash<bool>{}(key.isTLS));
+		return seed;
+	}
+};
+} // namespace std
+
+// Representation of all the things you need to connect to a blob store instance.
+// Reference counted because a very large number of them could be needed.
+class IBlobStoreEndpoint {
+public:
+	struct Stats {
+		Stats() : requests_successful(0), requests_failed(0), bytes_sent(0) {}
+		Stats operator-(const Stats& rhs);
+		void clear() { memset(this, 0, sizeof(*this)); }
+		json_spirit::mObject getJSON();
+
+		int64_t requests_successful;
+		int64_t requests_failed;
+		int64_t bytes_sent;
+	};
+
+	static Stats s_stats;
+
+	struct BlobStats {
+		UID id;
+		CounterCollection cc;
+		Counter requestsSuccessful;
+		Counter requestsFailed;
+		Counter newConnections;
+		Counter expiredConnections;
+		Counter reusedConnections;
+		Counter fastRetries;
+
+		LatencySample requestLatency;
+
+		// init not in static codepath, to avoid initialization race issues and so no blob connections means no
+		// unnecessary blob stats traces
+		BlobStats()
+		  : id(deterministicRandom()->randomUniqueID()), cc("BlobStoreStats", id.toString()),
+		    requestsSuccessful("RequestsSuccessful", cc), requestsFailed("RequestsFailed", cc),
+		    newConnections("NewConnections", cc), expiredConnections("ExpiredConnections", cc),
+		    reusedConnections("ReusedConnections", cc), fastRetries("FastRetries", cc),
+		    requestLatency("BlobStoreRequestLatency",
+		                   id,
+		                   CLIENT_KNOBS->BLOBSTORE_LATENCY_LOGGING_INTERVAL,
+		                   CLIENT_KNOBS->BLOBSTORE_LATENCY_LOGGING_ACCURACY) {}
+	};
+
+	struct ReusableConnection {
+		Reference<IConnection> conn;
+		double expirationTime;
+		// CROSS_PROCESS_FIX: Track which process created this connection
+		NetworkAddress creatingProcess;
+		ReusableConnection() : expirationTime(0) {
+			if (g_network && g_network->isSimulated()) {
+				creatingProcess = g_network->getLocalAddress();
+			}
+		}
+		ReusableConnection(Reference<IConnection> c, double exp) : conn(c), expirationTime(exp) {
+			if (g_network && g_network->isSimulated()) {
+				creatingProcess = g_network->getLocalAddress();
+			}
+		}
+
+		// CROSS_PROCESS_FIX: Copy constructor with cross-process detection
+		explicit(false) ReusableConnection(const ReusableConnection& other)
+		  : conn(other.conn), expirationTime(other.expirationTime), creatingProcess(other.creatingProcess) {
+			if (g_network && g_network->isSimulated() && creatingProcess.isValid() &&
+			    creatingProcess != g_network->getLocalAddress()) {
+				// Cross-process copy detected - invalidate the connection to prevent sharing
+				conn = Reference<IConnection>();
+				expirationTime = 0; // Mark as expired
+			}
+		}
+
+		// CROSS_PROCESS_FIX: Assignment operator with cross-process detection
+		ReusableConnection& operator=(const ReusableConnection& other) {
+			if (this != &other) {
+				conn = other.conn;
+				expirationTime = other.expirationTime;
+				creatingProcess = other.creatingProcess;
+				if (g_network && g_network->isSimulated() && creatingProcess.isValid() &&
+				    creatingProcess != g_network->getLocalAddress()) {
+					// Cross-process assignment detected - invalidate the connection to prevent sharing
+					conn = Reference<IConnection>();
+					expirationTime = 0; // Mark as expired
+				}
+			}
+			return *this;
+		}
+	};
+
+	// Reference counted connection pool queue
+	struct ConnectionPoolData : NonCopyable, ReferenceCounted<ConnectionPoolData> {
+		std::queue<ReusableConnection> pool;
+
+		ConnectionPoolData() = default;
+
+		// Destructor implementation in BlobStoreCommon.cpp
+		// In simulation, explicitly closes all pooled connections before destruction
+		~ConnectionPoolData();
+	};
+
+	struct ObjectInfo {
+		std::string name;
+		int64_t size;
+	};
+
+	struct ListResult {
+		std::vector<std::string> commonPrefixes;
+		std::vector<ObjectInfo> objects;
+	};
+
+	struct PartInfo {
+		std::string etag;
+		std::string checksum; // MD5 or SHA256 depending on integrity check setting
+		PartInfo() = default;
+		explicit PartInfo(std::string e, std::string c = "") : etag(e), checksum(c) {}
+	};
+
+	using ParametersT = std::map<std::string, std::string>;
+	using MultiPartSetT = std::map<int, PartInfo>;
+
+	// Global connection pool for multiple blobstore endpoints with same connection settings and request destination.
+	// NOTE: This is disabled by default (BLOBSTORE_GLOBAL_CONNECTION_POOL=false), so each endpoint gets its own pool
+	static std::unordered_map<BlobStoreConnectionPoolKey, Reference<ConnectionPoolData>>& getGlobalConnectionPool() {
+		// Use process address as key to separate connection pools per simulated process
+		static std::map<NetworkAddress, std::unordered_map<BlobStoreConnectionPoolKey, Reference<ConnectionPoolData>>>
+		    processConnectionPools;
+
+		NetworkAddress currentProcess;
+		if (g_network && g_network->isSimulated()) {
+			currentProcess = g_network->getLocalAddress();
+		}
+		return processConnectionPools[currentProcess];
+	}
+
+protected:
+	IBlobStoreEndpoint(std::string const& host,
+	                   std::string const& service,
+	                   std::string const& region,
+	                   Optional<std::string> const& proxyHost,
+	                   Optional<std::string> const& proxyPort,
+	                   BlobKnobs const& knobs = BlobKnobs(),
+	                   HTTP::Headers extraHeaders = HTTP::Headers())
+	  : host(host), service(service), region(region), proxyHost(proxyHost), proxyPort(proxyPort),
+	    useProxy(proxyHost.present() && proxyPort.present()), knobs(knobs), extraHeaders(extraHeaders),
+	    requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
+	    requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)),
+	    requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)),
+	    requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)),
+	    requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)),
+	    sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
+	    recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests),
+	    concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) {
+
+		if (host.empty() || (proxyHost.present() != proxyPort.present()))
+			throw connection_string_invalid();
+
+		// set connection pool instance
+		if (useProxy || !knobs.global_connection_pool) {
+			// don't use global connection pool if there's a proxy, as it complicates the logic
+			// FIXME: handle proxies?
+			connectionPool = makeReference<ConnectionPoolData>();
+		} else {
+			BlobStoreConnectionPoolKey key(host, service, region, knobs.isTLS());
+			auto it = getGlobalConnectionPool().find(key);
+			if (it != getGlobalConnectionPool().end()) {
+				connectionPool = it->second;
+			} else {
+				connectionPool = makeReference<ConnectionPoolData>();
+				getGlobalConnectionPool().insert({ key, connectionPool });
+			}
+		}
+		ASSERT(connectionPool.isValid());
+
+		maybeStartStatsLogger();
+	}
+
+public:
+	virtual ~IBlobStoreEndpoint() = default;
+	virtual void addref() = 0;
+	virtual void delref() = 0;
+
+	static std::string getURLFormat(bool withResource = false) {
+		const char* resource = "";
+		if (withResource)
+			resource = "<name>";
+		return format("blobstore://<credentials>@<host>[:<port>]/"
+		              "%s[?<param>=<value>[&<param>=<value>]...]",
+		              resource);
+	}
+
+	// Parse url and return an IBlobStoreEndpoint.
+	// If the url has parameters that cannot be consumed then an error will be thrown unless
+	// ignored_parameters is given in which case the unconsumed parameters will be added to it.
+	static Reference<IBlobStoreEndpoint> fromString(const std::string& url,
+	                                                const Optional<std::string>& proxy,
+	                                                std::string* resourceFromURL,
+	                                                std::string* error,
+	                                                ParametersT* ignored_parameters);
+
+	std::string getHost() const { return host; }
+	std::string getRegion() const { return region; }
+
+	// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL
+	// parameters in addition to the passed params string
+	virtual std::string getResourceURL(std::string resource, std::string params) const;
+
+	// Check if an object exists in a bucket
+	virtual Future<bool> objectExists(std::string const& bucket, std::string const& object) = 0;
+
+	// Get the size of an object in a bucket
+	virtual Future<int64_t> objectSize(std::string const& bucket, std::string const& object) = 0;
+
+	// Read an arbitrary segment of an object
+	virtual Future<int> readObject(std::string const& bucket,
+	                               std::string const& object,
+	                               void* data,
+	                               int length,
+	                               int64_t offset) = 0;
+
+	// Delete an object in a bucket
+	virtual Future<Void> deleteObject(std::string const& bucket, std::string const& object) = 0;
+
+	// Delete all objects in a bucket under a prefix.  Note this is not atomic as blob store does not
+	// support this operation directly. This method is just a convenience method that lists and deletes
+	// all of the objects in the bucket under the given prefix.
+	// Since it can take a while, if a pNumDeleted and/or pBytesDeleted are provided they will be incremented every time
+	// a deletion of an object completes.
+	Future<Void> deleteRecursively(std::string const& bucket,
+	                               std::string prefix = "",
+	                               int* pNumDeleted = nullptr,
+	                               int64_t* pBytesDeleted = nullptr);
+
+	Future<Void> writeEntireFile(std::string const& bucket, std::string const& object, std::string const& content);
+	virtual Future<Void> writeEntireFileFromBuffer(std::string const& bucket,
+	                                               std::string const& object,
+	                                               UnsentPacketQueue* pContent,
+	                                               int contentLen,
+	                                               std::string const& contentHash) = 0;
+
+	// MultiPart upload methods
+	// Returns UploadID
+	virtual Future<std::string> beginMultiPartUpload(std::string const& bucket, std::string const& object) = 0;
+	// Returns eTag
+	virtual Future<std::string> uploadPart(std::string const& bucket,
+	                                       std::string const& object,
+	                                       std::string const& uploadID,
+	                                       unsigned int partNumber,
+	                                       UnsentPacketQueue* pContent,
+	                                       int contentLen,
+	                                       std::string const& contentHash) = 0;
+	virtual Future<Optional<std::string>> finishMultiPartUpload(std::string const& bucket,
+	                                                            std::string const& object,
+	                                                            std::string const& uploadID,
+	                                                            MultiPartSetT const& parts,
+	                                                            int64_t totalSize = 0) = 0;
+
+	// Get bucket contents via a stream, since listing large buckets will take many serial blob requests.
+	// If a delimiter is passed then common prefixes will be read in parallel, recursively, depending on recurseFilter.
+	// recurseFilter must be a function that takes a string and returns true if it passes.  The default behavior is
+	// to assume true.
+	virtual Future<Void> listObjectsStream(std::string const& bucket,
+	                                       PromiseStream<ListResult> results,
+	                                       Optional<std::string> prefix = {},
+	                                       Optional<char> delimiter = {},
+	                                       int maxDepth = 0,
+	                                       std::function<bool(std::string const&)> recurseFilter = nullptr) = 0;
+
+	// Get a list of the files in a bucket, see listObjectsStream for more argument detail.
+	AsyncResult<ListResult> listObjects(std::string const& bucket,
+	                                    Optional<std::string> prefix = {},
+	                                    Optional<char> delimiter = {},
+	                                    int maxDepth = 0,
+	                                    std::function<bool(std::string const&)> recurseFilter = nullptr);
+
+	// Create a bucket if it does not already exist.
+	virtual Future<Void> createBucket(std::string const& bucket) = 0;
+
+	// Check if a bucket exists
+	virtual Future<bool> bucketExists(std::string const& bucket) = 0;
+
+	// Get a list of all buckets
+	virtual AsyncResult<std::vector<std::string>> listBuckets() = 0;
+
+	// Read an entire small file and return its contents
+	virtual AsyncResult<std::string> readEntireFile(std::string const& bucket, std::string const& object) = 0;
+
+	// Provider-specific credential refresh from files or SDK.
+	// Default implementation reads credential files and calls extractCredentialFields.
+	virtual Future<Void> updateSecret();
+
+	// Extract credential fields from a JSON account object.
+	// Returns true if credentials were successfully extracted, false to continue searching.
+	virtual bool extractCredentialFields(JSONDoc& account) = 0;
+
+	virtual std::string credentialFileKey() const { return "@" + host; }
+
+	virtual bool lookupSecretOnEachRequest() { return true; }
+
+	virtual void setRequestHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) = 0;
+
+	// Transform resource URI for the actual HTTP request (e.g., URI encoding, proxy absolute-form).
+	virtual std::string normalizeResourceForRequest(std::string const& resource) { return resource; }
+
+	// In simulation, optionally mutate a successful response to inject provider-specific errors.
+	// Called after receiving the response, before success/failure handling.
+	virtual void simulateRequestFailure(std::string const& verb,
+	                                    std::string const& resource,
+	                                    Reference<HTTP::IncomingResponse>& r) {}
+
+	// Called on non-success responses during doRequest retry loop.
+	// Set retryExtended=true to extend retries beyond maxTries.
+	virtual void processRequestFailure(Reference<HTTP::IncomingResponse> const& r,
+	                                   TraceEvent& event,
+	                                   bool& retryExtended) {}
+
+	// Called before sending the main request when retryExtended is true.
+	// Returns true to proceed with the request, false to skip this iteration.
+	virtual Future<bool> preRetryCheck(std::string const& verb,
+	                                   std::string const& resource,
+	                                   ReusableConnection& rconn,
+	                                   int requestTimeout,
+	                                   bool& retryExtended) {
+		co_return true;
+	}
+
+	// Do an HTTP request to the blob store, read the response. Handles connection, retry, and authentication.
+	Future<Reference<HTTP::IncomingResponse>> doRequest(std::string const& verb,
+	                                                    std::string const& resource,
+	                                                    const HTTP::Headers& headers,
+	                                                    UnsentPacketQueue* pContent,
+	                                                    int contentLen,
+	                                                    std::set<unsigned int> successCodes);
+
+	// Shared connection management
+	Future<ReusableConnection> connect(bool* reusingConn);
+	void returnConnection(ReusableConnection& rconn);
+
+	// Shared state
+	BlobKnobs knobs;
+	HTTP::Headers extraHeaders;
+	std::string host;
+	std::string service;
+	std::string region;
+	Optional<std::string> proxyHost;
+	Optional<std::string> proxyPort;
+	bool useProxy;
+
+	// FIXME: add periodic connection reaper to pool
+	Reference<ConnectionPoolData> connectionPool;
+
+	// Speed and concurrency limits
+	Reference<IRateControl> requestRate;
+	Reference<IRateControl> requestRateList;
+	Reference<IRateControl> requestRateWrite;
+	Reference<IRateControl> requestRateRead;
+	Reference<IRateControl> requestRateDelete;
+	Reference<IRateControl> sendRate;
+	Reference<IRateControl> recvRate;
+	FlowLock concurrentRequests;
+	FlowLock concurrentUploads;
+	FlowLock concurrentLists;
+
+	// null when initialized, so no blob stats until a blob connection is used
+	std::unique_ptr<BlobStats> blobStats;
+	Future<Void> statsLogger;
+
+	void maybeStartStatsLogger() {
+		if (!blobStats && CLIENT_KNOBS->BLOBSTORE_ENABLE_LOGGING) {
+			blobStats = std::make_unique<BlobStats>();
+			specialCounter(blobStats->cc, "GlobalConnectionPoolCount", [this]() {
+				return this->getGlobalConnectionPool().size();
+			});
+			specialCounter(blobStats->cc, "GlobalConnectionPoolSize", [this]() {
+				// FIXME: could track this explicitly via an int variable with extra logic, but this should be small and
+				// infrequent
+				int totalConnections = 0;
+				for (auto& it : this->getGlobalConnectionPool()) {
+					totalConnections += it.second->pool.size();
+				}
+				return totalConnections;
+			});
+
+			statsLogger = blobStats->cc.traceCounters(
+			    "BlobStoreMetrics", blobStats->id, CLIENT_KNOBS->BLOBSTORE_STATS_LOGGING_INTERVAL, "BlobStoreMetrics");
+		}
+	}
+	// Try to read a file, parse it as JSON, and return the resulting document.
+	// It will NOT throw if any errors are encountered, it will just return an empty
+	// JSON object and will log trace events for the errors encountered.
+	static Future<Optional<json_spirit::mObject>> tryReadJSONFile(std::string path);
+};
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/IKnobCollection.h b/fdbclient/include/fdbclient/IKnobCollection.h
deleted file mode 100644
index b8fdc4b099d..00000000000
--- a/fdbclient/include/fdbclient/IKnobCollection.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * IKnobCollection.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBCLIENT_IKNOBCOLLECTION_H
-#define FDBCLIENT_IKNOBCOLLECTION_H
-
-#pragma once
-
-#include <memory>
-
-#include "fdbclient/ClientKnobs.h"
-#include "fdbclient/KnobValue.h"
-#include "fdbclient/ServerKnobs.h"
-#include "flow/Knobs.h"
-
-/*
- * Each IKnobCollection instantiation stores several Knobs objects, and when parsing or setting a knob,
- * these Knobs objects will be traversed in order, until the requested knob name is found. The order of traversal is:
- *  - FlowKnobs
- *  - ClientKnobs
- *  - ServerKnobs
- *  - TestKnobs
- */
-class IKnobCollection {
-	static std::unique_ptr<IKnobCollection>& globalKnobCollection();
-
-public:
-	virtual ~IKnobCollection() = default;
-
-	enum class Type {
-		CLIENT,
-		SERVER,
-		TEST,
-	};
-
-	static std::unique_ptr<IKnobCollection> create(Type, Randomize, IsSimulated);
-	virtual void initialize(Randomize randomize, IsSimulated isSimulated) = 0;
-	virtual void reset(Randomize randomize, IsSimulated isSimulated) = 0;
-	virtual FlowKnobs const& getFlowKnobs() const = 0;
-	virtual ClientKnobs const& getClientKnobs() const = 0;
-	virtual ServerKnobs const& getServerKnobs() const = 0;
-	virtual class TestKnobs const& getTestKnobs() const = 0;
-	virtual void clearTestKnobs() {}
-	virtual Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const = 0;
-	KnobValue parseKnobValue(std::string const& knobName, std::string const& knobValue) const;
-	static KnobValue parseKnobValue(std::string const& knobName, std::string const& knobValue, Type);
-	// Result indicates whether or not knob was successfully set:
-	virtual bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) = 0;
-	void setKnob(std::string const& knobName, KnobValueRef const& knobValue);
-
-	static void setGlobalKnobCollection(Type, Randomize, IsSimulated);
-	static IKnobCollection const& getGlobalKnobCollection();
-	static IKnobCollection& getMutableGlobalKnobCollection();
-
-	// Sets up a list of <knob, value> pairs. If encounter a failure,
-	// immediately throws the error.
-	static void setupKnobs(const std::vector<std::pair<std::string, std::string>>& knobs);
-
-	// KnobValue mutations are no longer emitted via config DB.
-};
-
-#endif // FDBCLIENT_IKNOBCOLLECTION_H
diff --git a/fdbclient/include/fdbclient/IdempotencyId.actor.h b/fdbclient/include/fdbclient/IdempotencyId.h
similarity index 89%
rename from fdbclient/include/fdbclient/IdempotencyId.actor.h
rename to fdbclient/include/fdbclient/IdempotencyId.h
index 30ef23c90f8..5162dea9894 100644
--- a/fdbclient/include/fdbclient/IdempotencyId.actor.h
+++ b/fdbclient/include/fdbclient/IdempotencyId.h
@@ -1,5 +1,5 @@
 /*
- * IdempotencyId.actor.h
+ * IdempotencyId.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,14 +18,6 @@
  * limitations under the License.
  */
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_IDEMPOTENCY_ID_ACTOR_G_H)
-#define FDBCLIENT_IDEMPOTENCY_ID_ACTOR_G_H
-#include "fdbclient/IdempotencyId.actor.g.h"
-#elif !defined(FDBCLIENT_IDEMPOTENCY_ID_ACTOR_H)
-#define FDBCLIENT_IDEMPOTENCY_ID_ACTOR_H
-
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
@@ -34,7 +26,6 @@
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/serialize.h"
-#include "flow/actorcompiler.h" // this has to be the last include
 
 struct CommitResult {
 	Version commitVersion;
@@ -94,9 +85,9 @@ struct IdempotencyIdRef {
 
 	bool operator==(const IdempotencyIdRef& other) const { return asStringRefUnsafe() == other.asStringRefUnsafe(); }
 
-	IdempotencyIdRef(IdempotencyIdRef&& other) = default;
+	explicit(false) IdempotencyIdRef(IdempotencyIdRef&& other) = default;
 	IdempotencyIdRef& operator=(IdempotencyIdRef&& other) = default;
-	IdempotencyIdRef(const IdempotencyIdRef& other) = default;
+	explicit(false) IdempotencyIdRef(const IdempotencyIdRef& other) = default;
 	IdempotencyIdRef& operator=(const IdempotencyIdRef& other) = default;
 
 	template <class Archive>
@@ -189,14 +180,11 @@ KeyRangeRef makeIdempotencySingleKeyRange(Arena& arena, Version version, uint8_t
 
 void decodeIdempotencyKey(KeyRef key, Version& commitVersion, uint8_t& highOrderBatchIndex);
 
-ACTOR Future<JsonBuilderObject> getIdmpKeyStatus(Database db);
+AsyncResult<JsonBuilderObject> getIdmpKeyStatus(Database db);
 
 // Delete zero or more idempotency ids older than minAgeSeconds
 //
 // Normally idempotency ids are deleted as part of the normal commit process, so this only needs to clean ids that
 // leaked during a failure scenario. The rate of leaked idempotency ids should be low. The rate is zero during normal
 // operation, and proportional to the number of in-flight transactions during a failure scenario.
-ACTOR Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds);
-
-#include "flow/unactorcompiler.h"
-#endif
+Future<Void> cleanIdempotencyIds(Database db, double minAgeSeconds);
diff --git a/fdbclient/include/fdbclient/JSONDoc.h b/fdbclient/include/fdbclient/JSONDoc.h
index ce43c7ce81f..4f0975413d2 100644
--- a/fdbclient/include/fdbclient/JSONDoc.h
+++ b/fdbclient/include/fdbclient/JSONDoc.h
@@ -71,19 +71,19 @@ struct JSONDoc {
 
 	// Construction from const json_spirit::mObject, trivial and will never throw.
 	// Resulting JSONDoc will not allow modifications.
-	JSONDoc(const json_spirit::mObject& o) : pObj(&o), wpObj(nullptr) {}
+	explicit(false) JSONDoc(const json_spirit::mObject& o) : pObj(&o), wpObj(nullptr) {}
 
 	// Construction from json_spirit::mObject.  Allows modifications.
-	JSONDoc(json_spirit::mObject& o) : pObj(&o), wpObj(&o) {}
+	explicit(false) JSONDoc(json_spirit::mObject& o) : pObj(&o), wpObj(&o) {}
 
 	// Construction from const json_spirit::mValue (which is a Variant type) which will try to
 	// convert it to an mObject.  This will throw if that fails, just as it would
 	// if the caller called get_obj() itself and used the previous constructor instead.
-	JSONDoc(const json_spirit::mValue& v) : pObj(&v.get_obj()), wpObj(nullptr) {}
+	explicit(false) JSONDoc(const json_spirit::mValue& v) : pObj(&v.get_obj()), wpObj(nullptr) {}
 
 	// Construction from non-const json_spirit::mValue - will convert the mValue to
 	// an object if it isn't already and then attach to it.
-	JSONDoc(json_spirit::mValue& v) {
+	explicit(false) JSONDoc(json_spirit::mValue& v) {
 		if (v.type() != json_spirit::obj_type)
 			v = json_spirit::mObject();
 		wpObj = &v.get_obj();
diff --git a/fdbclient/include/fdbclient/KeyBackedRangeMap.actor.h b/fdbclient/include/fdbclient/KeyBackedRangeMap.h
similarity index 86%
rename from fdbclient/include/fdbclient/KeyBackedRangeMap.actor.h
rename to fdbclient/include/fdbclient/KeyBackedRangeMap.h
index ade92d0a438..7740d93a9b5 100644
--- a/fdbclient/include/fdbclient/KeyBackedRangeMap.actor.h
+++ b/fdbclient/include/fdbclient/KeyBackedRangeMap.h
@@ -1,5 +1,5 @@
 /*
- * KeyBackedRangeMap.actor.h
+ * KeyBackedRangeMap.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,15 +19,9 @@
  */
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_KEYBACKEDRANGEMAP_ACTOR_G_H)
-#define FDBCLIENT_KEYBACKEDRANGEMAP_ACTOR_G_H
-#include "fdbclient/KeyBackedRangeMap.actor.g.h"
-#elif !defined(FDBCLIENT_KEYBACKEDRANGEMAP_ACTOR_H)
-#define FDBCLIENT_KEYBACKEDRANGEMAP_ACTOR_H
-
 #include "flow/FastRef.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/KeyBackedTypes.h"
+#include "flow/CoroUtils.h"
 
 // A local in-memory representation of a KeyBackedRangeMap snapshot
 // It is invalid to look up a range in this map which not within the range of
@@ -132,19 +126,19 @@ class KeyBackedRangeMap {
 	  : kvMap(prefix, trigger, valueCodec) {}
 
 	// Get the RangeValue for the range that contains key, if there is a begin and end in the map which contain key
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<Optional<RangeValue>> getRangeForKey(KeyBackedRangeMap self,
 	                                                   Transaction tr,
 	                                                   KeyType key,
 	                                                   Snapshot snapshot = Snapshot::False) {
-		state Future<Optional<typename Map::KVType>> begin = self.kvMap.seekLessOrEqual(tr, key, snapshot);
-		state Future<Optional<typename Map::KVType>> end = self.kvMap.seekGreaterThan(tr, key, snapshot);
-		wait(success(begin) && success(end));
+		Future<Optional<typename Map::KVType>> begin = self.kvMap.seekLessOrEqual(tr, key, snapshot);
+		Future<Optional<typename Map::KVType>> end = self.kvMap.seekGreaterThan(tr, key, snapshot);
+		co_await (success(begin) && success(end));
 
 		if (begin.get().present() && end.get().present()) {
-			return RangeValue{ { begin.get()->key, end.get()->key }, begin.get()->value };
+			co_return RangeValue{ { begin.get()->key, end.get()->key }, begin.get()->value };
 		}
-		return Optional<RangeValue>();
+		co_return Optional<RangeValue>();
 	}
 
 	template <class Transaction>
@@ -158,7 +152,7 @@ class KeyBackedRangeMap {
 	// the the range with the given value.
 	// Adjacent ranges that are identical will be coalesced in the update transaction.
 	// Since the transaction type may not be RYW, this method must take care to not rely on reading its own updates.
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<Void> updateRangeActor(KeyBackedRangeMap self,
 	                                     Transaction tr,
 	                                     KeyType begin,
@@ -176,23 +170,23 @@ class KeyBackedRangeMap {
 		//     can be removed, which is the case if its value matches the previous boundary's value
 		//   - compare the boundary after the last modified boundary to see if it can be removed because
 		//     its value matches the last modified boundary's value
-		Optional<typename Map::KVType> beginKV = wait(self.kvMap.seekLessThan(tr, begin));
-		state KeySelector rangeBegin = KeySelector::firstGreaterOrEqual(beginKV.present() ? beginKV->key : begin);
+		Optional<typename Map::KVType> beginKV = co_await self.kvMap.seekLessThan(tr, begin);
+		KeySelector rangeBegin = KeySelector::firstGreaterOrEqual(beginKV.present() ? beginKV->key : begin);
 
 		// rangeEnd is past end so the result will include end if it exists
-		state KeySelector rangeEnd = KeySelector::firstGreaterThan(end);
+		KeySelector rangeEnd = KeySelector::firstGreaterThan(end);
 
-		state int readSize = BUGGIFY ? 1 : 100000;
-		state Future<RangeResultType> boundariesFuture =
+		int readSize = BUGGIFY ? 1 : 100000;
+		Future<RangeResultType> boundariesFuture =
 		    self.kvMap.getRange(tr, rangeBegin, rangeEnd, GetRangeLimits(readSize));
 
 		// As we walk through the range boundaries, keep two values about the last visited boundary
-		state ValueType original; // value prior to modification
-		state Optional<ValueType> previous; // value after modification
+		ValueType original{}; // value prior to modification
+		Optional<ValueType> previous; // value after modification
 		// Indicates that begin has been either seen/updated or created.
-		state bool beginDone = false;
+		bool beginDone = false;
 		// Indicates that end was found
-		state bool endFound = false;
+		bool endFound = false;
 
 		// The results will contain
 		//   A) 0 or 1 key < begin    Use this to initialize previous.
@@ -200,9 +194,9 @@ class KeyBackedRangeMap {
 		//                            Possibly delete it if update matches previous range.
 		//   C) >=0 keys > begin and < end   Update these, delete if they match previous
 		//   D) 0 or 1 key == end.  Delete this key if it matches previous. Set to default if it doesn't exist.
-		loop {
+		while (true) {
 			kbt_debug("RANGEMAP updateRange loop\n");
-			RangeResultType boundaries = wait(boundariesFuture);
+			RangeResultType boundaries = co_await boundariesFuture;
 			for (auto const& bv : boundaries.results) {
 				kbt_debug("RANGEMAP updateRange   result key={} value={}\n", bv.first, bv.second);
 				// Should never see a result past the end key
@@ -293,8 +287,6 @@ class KeyBackedRangeMap {
 			kbt_debug("RANGEMAP updateRange set end\n");
 			self.kvMap.set(tr, end, original);
 		}
-
-		return Void();
 	}
 
 	template <class Transaction>
@@ -312,7 +304,7 @@ class KeyBackedRangeMap {
 		}
 	}
 
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<Reference<LocalSnapshot>> getSnapshotActor(KeyBackedRangeMap self,
 	                                                         Transaction tr,
 	                                                         KeyType begin,
@@ -322,20 +314,20 @@ class KeyBackedRangeMap {
 		// The range read should start at KeySelector::lastLessOrEqual(begin) to get the range which covers begin.
 		// However this could touch a key outside of the map subspace which can lead to various errors.
 		// Use seekLessOrEqual() to find a begin key safely.
-		Optional<typename Map::KVType> beginKV = wait(self.kvMap.seekLessOrEqual(tr, begin));
-		state KeySelector rangeBegin = KeySelector::firstGreaterOrEqual(beginKV.present() ? beginKV->key : begin);
+		Optional<typename Map::KVType> beginKV = co_await self.kvMap.seekLessOrEqual(tr, begin);
+		KeySelector rangeBegin = KeySelector::firstGreaterOrEqual(beginKV.present() ? beginKV->key : begin);
 
 		// rangeEnd is past end so the result will include end if it exists
-		state KeySelector rangeEnd = KeySelector::firstGreaterThan(end);
+		KeySelector rangeEnd = KeySelector::firstGreaterThan(end);
 
-		state int readSize = BUGGIFY ? 1 : 100000;
-		state Future<RangeResultType> boundariesFuture =
+		int readSize = BUGGIFY ? 1 : 100000;
+		Future<RangeResultType> boundariesFuture =
 		    self.kvMap.getRange(tr, rangeBegin, rangeEnd, GetRangeLimits(readSize));
 
-		state Reference<LocalSnapshot> result = makeReference<LocalSnapshot>();
-		loop {
+		Reference<LocalSnapshot> result = makeReference<LocalSnapshot>();
+		while (true) {
 			kbt_debug("RANGEMAP snapshot loop\n");
-			RangeResultType boundaries = wait(boundariesFuture);
+			RangeResultType boundaries = co_await boundariesFuture;
 			for (auto const& bv : boundaries.results) {
 				result->map[bv.first] = bv.second;
 			}
@@ -360,7 +352,7 @@ class KeyBackedRangeMap {
 		}
 
 		kbt_debug("RANGEMAP snapshot end\n");
-		return result;
+		co_return result;
 	}
 
 	// Return a LocalSnapshot of all ranges from the map which cover the range of begin through end.
@@ -380,7 +372,3 @@ class KeyBackedRangeMap {
 private:
 	Map kvMap;
 };
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbclient/include/fdbclient/KeyBackedTypes.actor.h b/fdbclient/include/fdbclient/KeyBackedTypes.h
similarity index 91%
rename from fdbclient/include/fdbclient/KeyBackedTypes.actor.h
rename to fdbclient/include/fdbclient/KeyBackedTypes.h
index e4994492aae..ef240c61d58 100644
--- a/fdbclient/include/fdbclient/KeyBackedTypes.actor.h
+++ b/fdbclient/include/fdbclient/KeyBackedTypes.h
@@ -1,5 +1,5 @@
 /*
- * KeyBackedTypes.actor.h
+ * KeyBackedTypes.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,19 +20,13 @@
 
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_KEYBACKEDTYPES_ACTOR_G_H)
-#define FDBCLIENT_KEYBACKEDTYPES_ACTOR_G_H
-#include "fdbclient/KeyBackedTypes.actor.g.h"
-#elif !defined(FDBCLIENT_KEYBACKEDTYPES_ACTOR_H)
-#define FDBCLIENT_KEYBACKEDTYPES_ACTOR_H
-
 #include <utility>
 #include <vector>
 #include <ranges>
 
 #include "fdbclient/ClientBooleanParams.h"
 #include "fdbclient/CommitTransaction.h"
-#include "fdbclient/RunTransaction.actor.h"
+#include "fdbclient/RunTransaction.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/GenericTransactionHelper.h"
@@ -40,12 +34,11 @@
 #include "fdbclient/TupleVersionstamp.h"
 #include "flow/ObjectSerializer.h"
 #include "flow/Platform.h"
+#include "flow/CoroUtils.h"
 #include "flow/genericactors.actor.h"
 #include "flow/serialize.h"
 #include "flow/ThreadHelper.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 #define KEYBACKEDTYPES_DEBUG 0
 #if KEYBACKEDTYPES_DEBUG || !defined(NO_INTELLISENSE)
 #define kbt_debug fmt::print
@@ -203,7 +196,8 @@ inline KeyRange TupleCodec<KeyRange>::unpack(Standalone<StringRef> const& val) {
 }
 
 template <class Enum>
-struct TupleCodec<Enum, std::enable_if_t<std::is_enum_v<Enum>>> {
+    requires(std::is_enum_v<Enum>)
+struct TupleCodec<Enum> {
 	static inline Standalone<StringRef> pack(Enum const& val) {
 		return Tuple::makeTuple(static_cast<int64_t>(val)).pack();
 	}
@@ -226,7 +220,7 @@ struct BinaryCodec {
 // Codec for using Flatbuffer compatible types via ObjectWriter/ObjectReader
 template <typename T, typename VersionOptions>
 struct ObjectCodec {
-	ObjectCodec(VersionOptions vo) : vo(vo) {}
+	explicit(false) ObjectCodec(VersionOptions vo) : vo(vo) {}
 	VersionOptions vo;
 
 	inline Standalone<StringRef> pack(T const& val) const { return ObjectWriter::toValue<T>(val, vo); }
@@ -247,7 +241,7 @@ class WatchableTrigger {
 	Key key;
 
 public:
-	WatchableTrigger(Key k) : key(k) {}
+	explicit WatchableTrigger(Key k) : key(k) {}
 
 	template <class Transaction>
 	void update(Transaction tr, AddConflictRange conflict = AddConflictRange::False) {
@@ -279,21 +273,11 @@ class WatchableTrigger {
 		return holdWhile(watchFuture, safeThreadFutureToFuture(watchFuture));
 	}
 
-// Forward declaration of this static template actor does not work correctly, so this actor is forward declared
-// in a way that works for both compiling and in IDEs.
-#if defined(NO_INTELLISENSE)
 	template <class DB>
-	static Future<Version> onChangeActor(WatchableTrigger const& self,
-	                                     Reference<DB> const& db,
-	                                     Optional<Version> const& initialVersion,
-	                                     Promise<Version> const& watching);
-#else
-	ACTOR template <class DB>
 	static Future<Version> onChangeActor(WatchableTrigger self,
 	                                     Reference<DB> db,
 	                                     Optional<Version> initialVersion,
 	                                     Promise<Version> watching);
-#endif
 
 	// Watch the trigger until it changes.  The result will be ready when it is observed that the trigger value's
 	// version is greater than initialVersion, and the observed trigger value version will be returned.
@@ -313,42 +297,45 @@ class WatchableTrigger {
 	}
 };
 
-ACTOR template <class DB>
+template <class DB>
 Future<Version> WatchableTrigger::onChangeActor(WatchableTrigger self,
                                                 Reference<DB> db,
                                                 Optional<Version> initialVersion,
                                                 Promise<Version> watching) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
 
-	loop {
+	while (true) {
 		if constexpr (can_set_transaction_options<DB>) {
 			db->setOptions(tr);
 		}
+		Error err;
 		try {
 			// If the initialVersion is not set yet, then initialize it with the read version
 			if (!initialVersion.present()) {
-				wait(store(initialVersion, safeThreadFutureToFuture(tr->getReadVersion())));
+				co_await store(initialVersion, safeThreadFutureToFuture(tr->getReadVersion()));
 			}
 			if (watching.canBeSet()) {
 				watching.send(*initialVersion);
 			}
 
 			// Get the trigger's latest value.
-			Optional<Versionstamp> currentVal = wait(self.get(tr));
+			Optional<Versionstamp> currentVal = co_await self.get(tr);
 
 			// If the trigger has a value and its version is > initialVersion then the trigger has fired so break
 			if (currentVal.present() && currentVal->version > *initialVersion) {
-				return currentVal->version;
+				co_return currentVal->version;
 			}
 
 			// Otherwise, watch the key and repeat the loop once the watch fires
-			state Future<Void> watch = self.watch(tr);
-			wait(safeThreadFutureToFuture(tr->commit()));
-			wait(watch);
+			Future<Void> watch = self.watch(tr);
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_await watch;
 			tr->reset();
+			continue;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -357,7 +344,8 @@ Future<Version> WatchableTrigger::onChangeActor(WatchableTrigger self,
 template <typename T, typename Codec = TupleCodec<T>, bool SystemAccess = true>
 class KeyBackedProperty {
 public:
-	KeyBackedProperty(KeyRef key = invalidKey, Optional<WatchableTrigger> trigger = {}, Codec codec = {})
+	explicit(false)
+	    KeyBackedProperty(KeyRef key = invalidKey, Optional<WatchableTrigger> trigger = {}, Codec codec = {})
 	  : key(key), trigger(trigger), codec(codec) {}
 
 	template <class Transaction>
@@ -424,8 +412,8 @@ class KeyBackedProperty {
 	}
 
 	template <class TransactionContext>
-	std::enable_if_t<is_transaction_creator<TransactionContext>, Future<Void>> set(TransactionContext tcx,
-	                                                                               T const& val) {
+	    requires(is_transaction_creator<TransactionContext>)
+	Future<Void> set(TransactionContext tcx, T const& val) {
 		return runTransaction(tcx, [this, val](decltype(tcx->createTransaction()) tr) {
 			if constexpr (SystemAccess) {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -437,7 +425,8 @@ class KeyBackedProperty {
 	}
 
 	template <class Transaction>
-	std::enable_if_t<!is_transaction_creator<Transaction>, void> set(Transaction tr, T const& val) {
+	    requires(!is_transaction_creator<Transaction>)
+	void set(Transaction tr, T const& val) {
 		tr->set(key, packValue(val));
 		if (trigger.present()) {
 			trigger->update(tr);
@@ -445,7 +434,8 @@ class KeyBackedProperty {
 	}
 
 	template <class TransactionContext>
-	std::enable_if_t<is_transaction_creator<TransactionContext>, Future<Void>> clear(TransactionContext tcx) {
+	    requires(is_transaction_creator<TransactionContext>)
+	Future<Void> clear(TransactionContext tcx) {
 		return runTransaction(tcx, [this](decltype(tcx->createTransaction()) tr) {
 			if constexpr (SystemAccess) {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -457,7 +447,8 @@ class KeyBackedProperty {
 	}
 
 	template <class Transaction>
-	std::enable_if_t<!is_transaction_creator<Transaction>, void> clear(Transaction tr) {
+	    requires(!is_transaction_creator<Transaction>)
+	void clear(Transaction tr) {
 		tr->clear(key);
 		if (trigger.present()) {
 			trigger->update(tr);
@@ -496,7 +487,8 @@ class KeyBackedBinaryValue : public KeyBackedProperty<T, BinaryCodec<T>> {
 	typedef KeyBackedProperty<T, BinaryCodec<T>> Base;
 
 public:
-	KeyBackedBinaryValue(KeyRef key = invalidKey, Optional<WatchableTrigger> trigger = {}) : Base(key, trigger) {}
+	explicit(false) KeyBackedBinaryValue(KeyRef key = invalidKey, Optional<WatchableTrigger> trigger = {})
+	  : Base(key, trigger) {}
 
 	template <class Transaction>
 	void atomicOp(Transaction tr, T const& val, MutationRef::Type type) {
@@ -562,7 +554,8 @@ template <typename _KeyType,
           typename ValueCodec = TupleCodec<_ValueType>>
 class KeyBackedMap {
 public:
-	KeyBackedMap(KeyRef prefix = invalidKey, Optional<WatchableTrigger> trigger = {}, ValueCodec valueCodec = {})
+	explicit(false)
+	    KeyBackedMap(KeyRef prefix = invalidKey, Optional<WatchableTrigger> trigger = {}, ValueCodec valueCodec = {})
 	  : subspace(prefixRange(prefix)), trigger(trigger), valueCodec(valueCodec) {}
 
 	typedef _KeyType KeyType;
@@ -600,9 +593,9 @@ class KeyBackedMap {
 			    map(safeThreadFutureToFuture(getRangeFuture),
 			        [prefix = subspace.begin, valueCodec = valueCodec](RangeResult const& kvs) -> RangeResultType {
 				        RangeResultType rangeResult;
-				        for (int i = 0; i < kvs.size(); ++i) {
-					        KeyType key = KeyCodec::unpack(kvs[i].key.removePrefix(prefix));
-					        ValueType val = valueCodec.unpack(kvs[i].value);
+				        for (const auto& keyValue : kvs) {
+					        KeyType key = KeyCodec::unpack(keyValue.key.removePrefix(prefix));
+					        ValueType val = valueCodec.unpack(keyValue.value);
 					        rangeResult.results.push_back(PairType(key, val));
 				        }
 				        rangeResult.more = kvs.more;
@@ -611,7 +604,7 @@ class KeyBackedMap {
 		}
 	}
 
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<RangeResultType> getRangeActor(KeyBackedMap self,
 	                                             Transaction tr,
 	                                             KeySelector begin,
@@ -623,18 +616,18 @@ class KeyBackedMap {
 		          begin.pack(self.subspace.begin).toString(),
 		          end.pack(self.subspace.begin).toString());
 
-		state ::KeySelector ksBegin = begin.pack(self.subspace.begin);
-		state ::KeySelector ksEnd = end.pack(self.subspace.begin);
-		state typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
+		::KeySelector ksBegin = begin.pack(self.subspace.begin);
+		::KeySelector ksEnd = end.pack(self.subspace.begin);
+		typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
 		    tr->getRange(ksBegin, ksEnd, limits, snapshot, reverse);
 
 		// Since the getRange result must be filtered to keys within the map subspace, it is possible that the given
 		// TypedKeySelectors and GetRangeLimits yields a result in which no KV pairs are within the map subspace.  If
 		// this happens, we can't return any map KV pairs for the caller to base the next request on, so we will loop
 		// and continue with the next request here.
-		state RangeResultType rangeResult;
-		loop {
-			RangeResult kvs = wait(safeThreadFutureToFuture(getRangeFuture));
+		RangeResultType rangeResult{};
+		while (true) {
+			RangeResult kvs = co_await safeThreadFutureToFuture(getRangeFuture);
 			kbt_debug("MAP GETRANGE KeySelectors {} - {} results={} more={}\n",
 			          begin.pack(self.subspace.begin).toString(),
 			          end.pack(self.subspace.begin).toString(),
@@ -680,7 +673,7 @@ class KeyBackedMap {
 			getRangeFuture = tr->getRange(ksBegin, ksEnd, limits, snapshot, reverse);
 		}
 
-		return rangeResult;
+		co_return rangeResult;
 	}
 
 	// GetRange with typed KeySelectors
@@ -698,7 +691,7 @@ class KeyBackedMap {
 	// These operation can be accomplished using KeySelectors however they run the risk of touching keys outside of
 	// map subspace, which can cause problems if this touches an offline range or a key which is unreadable by range
 	// read operations due to having been modified with a version stamp operation in the current transaction.
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<Optional<KVType>> seek(KeyBackedMap self,
 	                                     Transaction tr,
 	                                     KeyType query,
@@ -727,15 +720,15 @@ class KeyBackedMap {
 			end = self.subspace.end;
 		}
 
-		state typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
+		typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
 		    tr->getRange(KeyRangeRef(begin, end), 1, snapshot, Reverse{ lessThan });
 
-		RangeResult kvs = wait(safeThreadFutureToFuture(getRangeFuture));
+		RangeResult kvs = co_await safeThreadFutureToFuture(getRangeFuture);
 		if (kvs.empty()) {
-			return Optional<KVType>();
+			co_return Optional<KVType>();
 		}
 
-		return self.unpackKV(kvs.front());
+		co_return self.unpackKV(kvs.front());
 	}
 
 	template <class Transaction>
@@ -896,7 +889,7 @@ class KeyBackedObjectMap
 template <typename _ValueType, typename Codec = TupleCodec<_ValueType>>
 class KeyBackedSet {
 public:
-	KeyBackedSet(KeyRef key = invalidKey, Optional<WatchableTrigger> trigger = {})
+	explicit(false) KeyBackedSet(KeyRef key = invalidKey, Optional<WatchableTrigger> trigger = {})
 	  : subspace(prefixRange(key)), trigger(trigger) {}
 
 	typedef _ValueType ValueType;
@@ -928,7 +921,7 @@ class KeyBackedSet {
 		                     }));
 	}
 
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<RangeResultType> getRangeActor(KeyBackedSet self,
 	                                             Transaction tr,
 	                                             KeySelector begin,
@@ -940,18 +933,18 @@ class KeyBackedSet {
 		          begin.pack(self.subspace.begin).toString(),
 		          end.pack(self.subspace.begin).toString());
 
-		state ::KeySelector ksBegin = begin.pack(self.subspace.begin);
-		state ::KeySelector ksEnd = end.pack(self.subspace.begin);
-		state typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
+		::KeySelector ksBegin = begin.pack(self.subspace.begin);
+		::KeySelector ksEnd = end.pack(self.subspace.begin);
+		typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
 		    tr->getRange(ksBegin, ksEnd, limits, snapshot, reverse);
 
 		// Since the getRange result must be filtered to keys within the map subspace, it is possible that the given
 		// TypedKeySelectors and GetRangeLimits yields a result in which no KV pairs are within the map subspace.  If
 		// this happens, we can't return any map KV pairs for the caller to base the next request on, so we will loop
 		// and continue with the next request here.
-		state RangeResultType rangeResult;
-		loop {
-			RangeResult kvs = wait(safeThreadFutureToFuture(getRangeFuture));
+		RangeResultType rangeResult{};
+		while (true) {
+			RangeResult kvs = co_await safeThreadFutureToFuture(getRangeFuture);
 			kbt_debug("MAP GETRANGE KeySelectors {} - {} results={} more={}\n",
 			          begin.pack(self.subspace.begin).toString(),
 			          end.pack(self.subspace.begin).toString(),
@@ -995,7 +988,7 @@ class KeyBackedSet {
 			getRangeFuture = tr->getRange(ksBegin, ksEnd, limits, snapshot, reverse);
 		}
 
-		return rangeResult;
+		co_return rangeResult;
 	}
 
 	// GetRange with typed KeySelectors
@@ -1013,7 +1006,7 @@ class KeyBackedSet {
 	// These operation can be accomplished using KeySelectors however they run the risk of touching keys outside of
 	// map subspace, which can cause problems if this touches an offline range or a key which is unreadable by range
 	// read operations due to having been modified with a version stamp operation in the current transaction.
-	ACTOR template <class Transaction>
+	template <class Transaction>
 	static Future<Optional<ValueType>> seek(KeyBackedSet self,
 	                                        Transaction tr,
 	                                        ValueType query,
@@ -1042,15 +1035,15 @@ class KeyBackedSet {
 			end = self.subspace.end;
 		}
 
-		state typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
+		typename transaction_future_type<Transaction, RangeResult>::type getRangeFuture =
 		    tr->getRange(KeyRangeRef(begin, end), 1, snapshot, Reverse{ lessThan });
 
-		RangeResult kvs = wait(safeThreadFutureToFuture(getRangeFuture));
+		RangeResult kvs = co_await safeThreadFutureToFuture(getRangeFuture);
 		if (kvs.empty()) {
-			return Optional<ValueType>();
+			co_return Optional<ValueType>();
 		}
 
-		return self.unpackKey(kvs.front().key);
+		co_return self.unpackKey(kvs.front().key);
 	}
 
 	template <class Transaction>
@@ -1143,13 +1136,9 @@ class KeyBackedSet {
 
 class KeyBackedClass {
 public:
-	KeyBackedClass(StringRef prefix, Optional<Key> triggerOverride = {})
+	explicit KeyBackedClass(StringRef prefix, Optional<Key> triggerOverride = {})
 	  : subspace(prefix), trigger(triggerOverride.orDefault(subspace.pack("_changeTrigger"_sr))) {}
 
 	Subspace subspace;
 	WatchableTrigger trigger;
 };
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbclient/include/fdbclient/KeyRangeMap.h b/fdbclient/include/fdbclient/KeyRangeMap.h
index 7a90b7f2923..49b44970000 100644
--- a/fdbclient/include/fdbclient/KeyRangeMap.h
+++ b/fdbclient/include/fdbclient/KeyRangeMap.h
@@ -107,7 +107,7 @@ class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, Metr
 		RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
 	}
 
-	CoalescedKeyRangeMap(CoalescedKeyRangeMap&& source) = default;
+	explicit(false) CoalescedKeyRangeMap(CoalescedKeyRangeMap&& source) = default;
 
 	void insert(const KeyRangeRef& keys, const Val& value);
 	void insert(const KeyRef& key, const Val& value);
@@ -131,26 +131,26 @@ class KeyRangeActorMap {
 // krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions
 class Transaction;
 class ReadYourWritesTransaction;
-Future<RangeResult> krmGetRanges(Transaction* const& tr,
-                                 Key const& mapPrefix,
-                                 KeyRange const& keys,
-                                 int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-                                 int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
-Future<RangeResult> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
-                                 Key const& mapPrefix,
-                                 KeyRange const& keys,
-                                 int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-                                 int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
-Future<RangeResult> krmGetRangesUnaligned(Transaction* const& tr,
-                                          Key const& mapPrefix,
-                                          KeyRange const& keys,
-                                          int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-                                          int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
-Future<RangeResult> krmGetRangesUnaligned(Reference<ReadYourWritesTransaction> const& tr,
-                                          Key const& mapPrefix,
-                                          KeyRange const& keys,
-                                          int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-                                          int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+Future<RangeResult> krmGetRanges(Transaction* tr,
+                                 Key mapPrefix,
+                                 KeyRange keys,
+                                 int limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+                                 int limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+Future<RangeResult> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
+                                 Key mapPrefix,
+                                 KeyRange keys,
+                                 int limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+                                 int limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+Future<RangeResult> krmGetRangesUnaligned(Transaction* tr,
+                                          Key mapPrefix,
+                                          KeyRange keys,
+                                          int limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+                                          int limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
+Future<RangeResult> krmGetRangesUnaligned(Reference<ReadYourWritesTransaction> tr,
+                                          Key mapPrefix,
+                                          KeyRange keys,
+                                          int limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+                                          int limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 void krmSetPreviouslyEmptyRange(Transaction* tr,
                                 const KeyRef& mapPrefix,
                                 const KeyRangeRef& keys,
@@ -162,11 +162,8 @@ void krmSetPreviouslyEmptyRange(struct CommitTransactionRef& tr,
                                 const KeyRangeRef& keys,
                                 const ValueRef& newValue,
                                 const ValueRef& oldEndValue);
-Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value);
-Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> const& tr,
-                         Key const& mapPrefix,
-                         KeyRange const& range,
-                         Value const& value);
+Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value);
+Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value);
 Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
                                    Key const& mapPrefix,
                                    KeyRange const& range,
diff --git a/fdbclient/include/fdbclient/Knobs.h b/fdbclient/include/fdbclient/Knobs.h
index c2e32c523ec..493506f69d5 100644
--- a/fdbclient/include/fdbclient/Knobs.h
+++ b/fdbclient/include/fdbclient/Knobs.h
@@ -20,10 +20,344 @@
 
 #pragma once
 
-#include "fdbclient/IKnobCollection.h"
+#include <vector>
 
-#define CLIENT_KNOBS (&IKnobCollection::getGlobalKnobCollection().getClientKnobs())
+#include "fdbclient/KnobValue.h"
+#include "flow/BooleanParam.h"
+#include "flow/Knobs.h"
+#include "flow/flow.h"
+
+FDB_BOOLEAN_PARAM(Randomize);
+FDB_BOOLEAN_PARAM(IsSimulated);
+
+// Returns a deterministically random transaction timeout value for simulation testing.
+// More weight is given to the famous 5s timeout, but [1, 10] range is returned with lower weight.
+// This is only be used in simulation.
+int getSimulatedTxnTimeoutSeconds();
+
+class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ClientKnobs : public KnobsImpl<ClientKnobs> {
+public:
+	int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
+
+	double SYSTEM_MONITOR_INTERVAL;
+	double NETWORK_BUSYNESS_MONITOR_INTERVAL; // The interval in which we should update the network busyness metric
+	double TSS_METRICS_LOGGING_INTERVAL;
+
+	double FAILURE_MAX_DELAY;
+	double FAILURE_MIN_DELAY;
+	double RECOVERY_DELAY_START_GENERATION;
+	double RECOVERY_DELAY_SECONDS_PER_GENERATION;
+	double MAX_GENERATIONS;
+	double MAX_GENERATIONS_OVERRIDE;
+	double MAX_GENERATIONS_SIM;
+
+	double COORDINATOR_RECONNECTION_DELAY;
+	int CLIENT_EXAMPLE_AMOUNT;
+	double MAX_CLIENT_STATUS_AGE;
+	int MAX_COMMIT_PROXY_CONNECTIONS;
+	int MAX_GRV_PROXY_CONNECTIONS;
+	double STATUS_IDLE_TIMEOUT;
+	double STATUS_TIMEOUT;
+	double GRPC_CTL_SERVICE_DEFAULT_TIMEOUT; // Default timeout for gRPC FDBCTL service requests
+	bool SEND_ENTIRE_VERSION_VECTOR;
+
+	// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
+	double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
+	                                 // mostly wrong (e.g. dumping the database after a test)
+	double FUTURE_VERSION_RETRY_DELAY;
+	double GRV_ERROR_RETRY_DELAY;
+	int REPLY_BYTE_LIMIT;
+	double DEFAULT_BACKOFF;
+	double DEFAULT_MAX_BACKOFF;
+	double BACKOFF_GROWTH_RATE;
+	double RESOURCE_CONSTRAINED_MAX_BACKOFF;
+	int PROXY_COMMIT_OVERHEAD_BYTES;
+	double SHARD_STAT_SMOOTH_AMOUNT;
+	int INIT_MID_SHARD_BYTES;
+
+	int TRANSACTION_SIZE_LIMIT;
+	int64_t KEY_SIZE_LIMIT;
+	int64_t SYSTEM_KEY_SIZE_LIMIT;
+	int64_t VALUE_SIZE_LIMIT;
+	int64_t SPLIT_KEY_SIZE_LIMIT;
+	int METADATA_VERSION_CACHE_SIZE;
+	int64_t CHANGE_FEED_LOCATION_LIMIT;
+	int64_t CHANGE_FEED_CACHE_SIZE;
+	double CHANGE_FEED_POP_TIMEOUT;
+	int64_t CHANGE_FEED_STREAM_MIN_BYTES;
+	double CHANGE_FEED_START_INTERVAL;
+	bool CHANGE_FEED_COALESCE_LOCATIONS;
+	int64_t CHANGE_FEED_CACHE_FLUSH_BYTES;
+	double CHANGE_FEED_CACHE_EXPIRE_TIME;
+	int64_t CHANGE_FEED_CACHE_LIMIT_BYTES;
+
+	int MAX_BATCH_SIZE;
+	double GRV_BATCH_TIMEOUT;
+	int BROADCAST_BATCH_SIZE;
+	double TRANSACTION_TIMEOUT_DELAY_INTERVAL;
+
+	// When locationCache in DatabaseContext gets to be this size, items will be evicted
+	int LOCATION_CACHE_EVICTION_SIZE;
+	int LOCATION_CACHE_EVICTION_SIZE_SIM;
+	double LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD;
+	double LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL;
+
+	int GET_RANGE_SHARD_LIMIT;
+	int WARM_RANGE_SHARD_LIMIT;
+	int STORAGE_METRICS_SHARD_LIMIT;
+	int SHARD_COUNT_LIMIT;
+	double STORAGE_METRICS_UNFAIR_SPLIT_LIMIT;
+	double STORAGE_METRICS_TOO_MANY_SHARDS_DELAY;
+	double AGGREGATE_HEALTH_METRICS_MAX_STALENESS;
+	double DETAILED_HEALTH_METRICS_MAX_STALENESS;
+	double MID_SHARD_SIZE_MAX_STALENESS;
+	bool TAG_ENCODE_KEY_SERVERS;
+	bool QUARANTINE_TSS_ON_MISMATCH;
+	double CHANGE_FEED_EMPTY_BATCH_TIME;
+
+	// KeyRangeMap
+	int KRM_GET_RANGE_LIMIT;
+	int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
+	                               // entries will be returned from an attempt to read a key range map
+
+	int DEFAULT_MAX_OUTSTANDING_WATCHES;
+	int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
+	double WATCH_POLLING_TIME;
+	double NO_RECENT_UPDATES_DURATION;
+	double FAST_WATCH_TIMEOUT;
+	double WATCH_TIMEOUT;
+
+	double IS_ACCEPTABLE_DELAY;
+
+	// Versions -- knobs that control 5s timeout
+	int64_t VERSIONS_PER_SECOND; // Copy of SERVER_KNOBS, as we can't link with it.
+	int64_t MAX_WRITE_TRANSACTION_LIFE_VERSIONS; // Copy of SERVER_KNOBS, as we can't link with it.
+
+	// Core
+	int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
+	int LOG_RANGE_BLOCK_SIZE;
+	int MUTATION_BLOCK_SIZE;
+	double MAX_VERSION_CACHE_LAG; // The upper bound in seconds for OK amount of staleness when using a cached RV
+	double MAX_PROXY_CONTACT_LAG; // The upper bound in seconds for how often we want a response from the GRV proxies
+	double DEBUG_USE_GRV_CACHE_CHANCE; // Debug setting to change the chance for a regular GRV request to use the cache
+	bool FORCE_GRV_CACHE_OFF; // Panic button to turn off cache. Holds priority over other options.
+	double GRV_CACHE_RK_COOLDOWN; // Required number of seconds to pass after throttling to re-allow cache use
+	double GRV_SUSTAINED_THROTTLING_THRESHOLD; // If ALL GRV requests have been throttled in the last number of seconds
+	                                           // specified here, ratekeeper is throttling and not a false positive
+
+	// Taskbucket
+	double TASKBUCKET_LOGGING_DELAY;
+	int TASKBUCKET_MAX_PRIORITY;
+	double TASKBUCKET_CHECK_TIMEOUT_CHANCE;
+	double TASKBUCKET_TIMEOUT_JITTER_OFFSET;
+	double TASKBUCKET_TIMEOUT_JITTER_RANGE;
+	double TASKBUCKET_CHECK_ACTIVE_DELAY;
+	int TASKBUCKET_CHECK_ACTIVE_AMOUNT;
+	int TASKBUCKET_TIMEOUT_VERSIONS;
+	int TASKBUCKET_MAX_TASK_KEYS;
+
+	// Backup
+	int BACKUP_LOCAL_FILE_WRITE_BLOCK;
+	int BACKUP_CONCURRENT_DELETES;
+	int BACKUP_SIMULATED_LIMIT_BYTES;
+	int BACKUP_GET_RANGE_LIMIT_BYTES;
+	int BACKUP_LOCK_BYTES;
+	double BACKUP_RANGE_TIMEOUT;
+	double BACKUP_RANGE_MINWAIT;
+	double BULKDUMP_JOB_TIMEOUT;
+	double BULKLOAD_JOB_TIMEOUT;
+	int BACKUP_SNAPSHOT_DISPATCH_INTERVAL_SEC;
+	int BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
+	int BACKUP_SHARD_TASK_LIMIT;
+	double BACKUP_AGGREGATE_POLL_RATE;
+	double BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL;
+	int BACKUP_LOG_WRITE_BATCH_MAX_SIZE;
+	int BACKUP_MAX_LOG_RANGES;
+	int BACKUP_SIM_COPY_LOG_RANGES;
+	int BACKUP_VERSION_DELAY;
+	int BACKUP_MAP_KEY_LOWER_LIMIT;
+	int BACKUP_MAP_KEY_UPPER_LIMIT;
+	int BACKUP_COPY_TASKS;
+	int BACKUP_BLOCK_SIZE;
+	int COPY_LOG_BLOCK_SIZE;
+	int COPY_LOG_BLOCKS_PER_TASK;
+	int COPY_LOG_PREFETCH_BLOCKS;
+	int COPY_LOG_READ_AHEAD_BYTES;
+	double COPY_LOG_TASK_DURATION_SECONDS;
+	int BACKUP_TASKS_PER_AGENT;
+	int BACKUP_POLL_PROGRESS_SECONDS;
+	int SIM_BACKUP_TASKS_PER_AGENT;
+	int BACKUP_RANGEFILE_BLOCK_SIZE;
+	int BACKUP_LOGFILE_BLOCK_SIZE;
+	int BACKUP_DISPATCH_ADDTASK_SIZE;
+	bool BACKUP_ALLOW_DRYRUN;
+	int RESTORE_DISPATCH_ADDTASK_SIZE;
+	int RESTORE_DISPATCH_BATCH_SIZE;
+	int RESTORE_WRITE_TX_SIZE;
+	int RESTORE_PARTITIONED_BATCH_VERSION_SIZE;
+	int APPLY_MAX_LOCK_BYTES;
+	int APPLY_MIN_LOCK_BYTES;
+	int APPLY_BLOCK_SIZE;
+	double APPLY_MAX_DECAY_RATE;
+	double APPLY_MAX_INCREASE_FACTOR;
+	double BACKUP_ERROR_DELAY;
+	double BACKUP_STATUS_DELAY;
+	double BACKUP_STATUS_JITTER;
+	double MIN_CLEANUP_SECONDS;
+	int RESTORE_RANGES_READ_BATCH;
+
+	// interval for version directory bucketing in range-partitioned backup.
+	int64_t BACKUP_RANGE_PARTITIONED_VDIR_INTERVAL;
+	bool BACKUP_CONTAINER_LOCAL_ALLOW_RELATIVE_PATH;
+	bool ENABLE_REPLICA_CONSISTENCY_CHECK_ON_BACKUP_READS;
+	int BACKUP_CONSISTENCY_CHECK_REQUIRED_REPLICAS;
+	bool BACKUP_READS_USE_LOW_PRIORITY;
+	int BULKLOAD_JOB_HISTORY_COUNT_MAX; // the max number of bulk load job history to keep. The oldest job history will
+	                                    // be removed when the count exceeds this value. Set to 0 to disable history.
+	                                    // Do not set the value to a large number, e.g. <= 10.
+	int BULKLOAD_VERBOSE_LEVEL; // Set to 1 to minimize the verbosity. Set to 5 to turn on all events for performance
+	                            // insights. Set to 10 to turn on all events.
+	int S3CLIENT_VERBOSE_LEVEL; // Similar to BULKLOAD_VERBOSE_LEVEL
+
+	// Configuration
+	int32_t DEFAULT_AUTO_COMMIT_PROXIES;
+	int32_t DEFAULT_AUTO_GRV_PROXIES;
+	int32_t DEFAULT_COMMIT_GRV_PROXIES_RATIO;
+	int32_t DEFAULT_MAX_GRV_PROXIES;
+	int32_t DEFAULT_AUTO_RESOLVERS;
+	int32_t DEFAULT_AUTO_LOGS;
+
+	double GLOBAL_CONFIG_REFRESH_BACKOFF;
+	double GLOBAL_CONFIG_REFRESH_MAX_BACKOFF;
+	double GLOBAL_CONFIG_REFRESH_TIMEOUT;
+
+	// Dynamic Knobs
+	double COMMIT_QUORUM_TIMEOUT;
+	double GET_GENERATION_QUORUM_TIMEOUT;
+	double GET_KNOB_TIMEOUT;
+	double TIMEOUT_RETRY_UPPER_BOUND;
+
+	// Client Status Info
+	double CSI_SAMPLING_PROBABILITY;
+	int64_t CSI_SIZE_LIMIT;
+	double CSI_STATUS_DELAY;
+
+	bool HTTP_REQUEST_AWS_V4_HEADER; // setting this knob to true will enable AWS V4 style header.
+	std::string BLOBSTORE_ENCRYPTION_TYPE;
+	int BLOBSTORE_CONNECT_TRIES;
+	int BLOBSTORE_CONNECT_TIMEOUT;
+	int BLOBSTORE_MAX_CONNECTION_LIFE;
+	int BLOBSTORE_REQUEST_TRIES;
+	int BLOBSTORE_REQUEST_TIMEOUT_MIN;
+	int BLOBSTORE_REQUESTS_PER_SECOND;
+	int BLOBSTORE_LIST_REQUESTS_PER_SECOND;
+	int BLOBSTORE_WRITE_REQUESTS_PER_SECOND;
+	int BLOBSTORE_READ_REQUESTS_PER_SECOND;
+	int BLOBSTORE_DELETE_REQUESTS_PER_SECOND;
+	int BLOBSTORE_CONCURRENT_REQUESTS;
+	int BLOBSTORE_MULTIPART_MAX_PART_SIZE;
+	int BLOBSTORE_MULTIPART_MIN_PART_SIZE;
+	int BLOBSTORE_MULTIPART_RETRY_DELAY_MS;
+	int BLOBSTORE_CONCURRENT_UPLOADS;
+	int BLOBSTORE_CONCURRENT_LISTS;
+	int BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
+	int BLOBSTORE_CONCURRENT_READS_PER_FILE;
+	int BLOBSTORE_ENABLE_READ_CACHE;
+	int BLOBSTORE_READ_BLOCK_SIZE;
+	int BLOBSTORE_READ_AHEAD_BLOCKS;
+	int BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
+	int BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
+	int BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
+	int BLOBSTORE_LIST_MAX_KEYS_PER_PAGE;
+	bool BLOBSTORE_GLOBAL_CONNECTION_POOL;
+	bool BLOBSTORE_ENABLE_LOGGING;
+	double BLOBSTORE_STATS_LOGGING_INTERVAL;
+	double BLOBSTORE_LATENCY_LOGGING_INTERVAL;
+	double BLOBSTORE_LATENCY_LOGGING_ACCURACY;
+	int BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
+	int BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
+	bool
+	    BLOBSTORE_ENABLE_OBJECT_INTEGRITY_CHECK; // Enable integrity check of download. When not set, on upload, we
+	                                             // we volunteer an md5 of the content and upload will fail if our
+	                                             // md5 doesn't match that calculated by the server. When this flag
+	                                             // is set, we hash the content using sha256 and have the server store
+	                                             // the hash in the object metadata. On download, if this flag is set
+	                                             // we will check the serverside proffered hash against that we
+	                                             // calculate on the received content.  If no match, throw an error. See
+	                                             // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+	                                             // This download check only works for small files. For large files,
+	                                             // we run a separate checksum. See design/s3-checksumming.md
+	                                             //
+	int CONSISTENCY_CHECK_RATE_LIMIT_MAX; // Available in both normal and urgent mode
+	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; // Available in normal mode
+	int CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME; // Available in urgent mode
+	int CONSISTENCY_CHECK_URGENT_BATCH_SHARD_COUNT; // Available in urgent mode
+	int CONSISTENCY_CHECK_URGENT_RETRY_DEPTH_MAX; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_0; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_0; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_1; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_1; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_2;
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_2;
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_3;
+	std::string CONSISTENCY_CHECK_URGENT_RANGE_END_3;
+
+	// fdbcli
+	int CLI_CONNECT_PARALLELISM;
+	double CLI_CONNECT_TIMEOUT;
+	bool CLI_PRINT_INVALID_CONFIGURATION;
+
+	// trace
+	int TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH;
+
+	// transaction tags
+	int MAX_TRANSACTION_TAG_LENGTH;
+	int MAX_TAGS_PER_TRANSACTION;
+	int COMMIT_SAMPLE_COST; // The expectation of sampling is every COMMIT_SAMPLE_COST sample once
+	int INCOMPLETE_SHARD_PLUS; // The size of (possible) incomplete shard when estimate clear range
+	double READ_TAG_SAMPLE_RATE; // Communicated to clients from cluster
+	double TAG_THROTTLE_SMOOTHING_WINDOW;
+	double TAG_THROTTLE_RECHECK_INTERVAL;
+	double TAG_THROTTLE_EXPIRATION_INTERVAL;
+	int64_t TAG_THROTTLING_PAGE_SIZE; // Used to round up the cost of operations
+	// Cost multiplier for writes (because write operations are more expensive than reads):
+	double TAG_THROTTLING_RW_FUNGIBILITY_RATIO;
+
+	// Enable to automatically retry transactions in the presence of transaction_lock_rejection error
+	// Set to false only for the rangeLocking simulation test
+	bool TRANSACTION_LOCK_REJECTION_RETRIABLE;
+
+	// busyness reporting
+	double BUSYNESS_SPIKE_START_THRESHOLD;
+	double BUSYNESS_SPIKE_SATURATED_THRESHOLD;
+
+	// The coordinator key/value in storage server might be inconsistent to the value stored in the cluster file.
+	// This might happen when a recovery is happening together with a cluster controller coordinator key change.
+	// During this process the database will be marked as "bad state" in changeQuorumChecker while later it will be
+	// available again. Using a backoffed retry when it happens.
+	int CHANGE_QUORUM_BAD_STATE_RETRY_TIMES;
+	double CHANGE_QUORUM_BAD_STATE_RETRY_DELAY;
+
+	bool ENABLE_MUTATION_CHECKSUM;
+	// Enable to start accumulative checksum population and validation
+	bool ENABLE_ACCUMULATIVE_CHECKSUM;
+	// Enable to logging verbose trace events related to the accumulative checksum
+	bool ENABLE_ACCUMULATIVE_CHECKSUM_LOGGING;
+
+	ClientKnobs(Randomize randomize, IsSimulated isSimulated);
+	void initialize(Randomize randomize, IsSimulated isSimulated);
+};
+
+extern ClientKnobs const* CLIENT_KNOBS;
 
 inline const ClientKnobs& getClientKnobs() {
 	return *CLIENT_KNOBS;
 }
+
+void resetClientKnobs(Randomize randomize, IsSimulated isSimulated);
+void initializeClientKnobs(Randomize randomize, IsSimulated isSimulated);
+Optional<KnobValue> tryParseClientKnobValue(std::string const& knobName, std::string const& knobValue);
+KnobValue parseClientKnobValue(std::string const& knobName, std::string const& knobValue);
+bool trySetClientKnob(std::string const& knobName, KnobValueRef const& knobValue);
+void setClientKnob(std::string const& knobName, KnobValueRef const& knobValue);
+void setupClientKnobs(std::vector<std::pair<std::string, std::string>> const& knobs);
diff --git a/fdbclient/include/fdbclient/ManagementAPI.actor.h b/fdbclient/include/fdbclient/ManagementAPI.actor.h
deleted file mode 100644
index fbf1ed08bf1..00000000000
--- a/fdbclient/include/fdbclient/ManagementAPI.actor.h
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * ManagementAPI.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_MANAGEMENT_API_ACTOR_G_H)
-#define FDBCLIENT_MANAGEMENT_API_ACTOR_G_H
-#include "fdbclient/ManagementAPI.actor.g.h"
-#elif !defined(FDBCLIENT_MANAGEMENT_API_ACTOR_H)
-#define FDBCLIENT_MANAGEMENT_API_ACTOR_H
-
-/* This file defines "management" interfaces for configuration, coordination changes, and
-the inclusion and exclusion of servers. It is used to implement fdbcli management commands
-and by test workloads that simulate such. It isn't exposed to C clients or anywhere outside
-our code base and doesn't need to be versioned. It doesn't do anything you can't do with the
-standard API and some knowledge of the contents of the system key space.
-*/
-
-#include <string>
-#include <map>
-#include "fdbclient/GenericManagementAPI.actor.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/RangeLock.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/MonitorLeader.h"
-#include "flow/actorcompiler.h" // has to be last include
-
-ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Transaction* tr, bool useSystemPriority = false);
-ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx, bool useSystemPriority = false);
-ACTOR Future<Void> waitForFullReplication(Database cx);
-
-struct IQuorumChange : ReferenceCounted<IQuorumChange> {
-	virtual ~IQuorumChange() {}
-	virtual Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
-	                                                                   std::vector<NetworkAddress> oldCoordinators,
-	                                                                   Reference<IClusterConnectionRecord>,
-	                                                                   CoordinatorsResult&) = 0;
-	virtual std::string getDesiredClusterKeyName() const { return std::string(); }
-};
-
-// Change to use the given set of coordination servers
-ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
-                                                               ClusterConnectionString* conn,
-                                                               std::string newName);
-ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
-Reference<IQuorumChange> autoQuorumChange(int desired = -1);
-Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
-
-// Exclude the given set of servers from use as state servers.  Returns as soon as the change is durable, without
-// necessarily waiting for the servers to be evacuated.  A NetworkAddress with a port of 0 means all servers on the
-// given IP.
-ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
-ACTOR Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed = false);
-
-// Exclude the servers matching the given set of localities from use as state servers.  Returns as soon as the change
-// is durable, without necessarily waiting for the servers to be evacuated.
-ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> localities, bool failed = false);
-ACTOR Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string> localities, bool failed = false);
-
-// Remove the given servers from the exclusion list.  A NetworkAddress with a port of 0 means all servers on the given
-// IP.  A NetworkAddress() means all servers (don't exclude anything)
-ACTOR Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
-
-// Remove the given localities from the exclusion list.
-ACTOR Future<Void> includeLocalities(Database cx,
-                                     std::vector<std::string> localities,
-                                     bool failed = false,
-                                     bool includeAll = false);
-
-// Set the process class of processes with the given address.  A NetworkAddress with a port of 0 means all servers on
-// the given IP.
-ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
-
-// Get the current list of excluded servers including both "exclude" and "failed".
-ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx);
-ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr);
-
-// Get the current list of excluded servers.
-ACTOR Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* tr);
-
-// Get the current list of failed servers.
-ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr);
-
-// Get the current list of excluded localities
-ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Database cx);
-ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr);
-
-// Get the current list of excluded localities.
-ACTOR Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr);
-
-// Get the current list of failed localities.
-ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr);
-
-// Decodes the locality string to a pair of locality prefix and its value.
-// The prefix could be dcid, processid, machineid, processid.
-std::pair<std::string, std::string> decodeLocality(const std::string& locality);
-std::set<AddressExclusion> getServerAddressesByLocality(
-    const std::map<std::string, StorageServerInterface> server_interfaces,
-    const std::string& locality);
-std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers, const std::string& locality);
-
-// Check for the given, previously excluded servers to be evacuated (no longer used for state).  If waitForExclusion
-// is true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance,
-// until and unless any of them are explicitly included with includeServers()
-ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
-                                                                std::vector<AddressExclusion> servers,
-                                                                bool waitForAllExcluded);
-ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
-                                                   std::set<AddressExclusion>* exclusions,
-                                                   std::set<NetworkAddress>* inProgressExclusion);
-
-// Gets a list of all workers in the cluster (excluding testers)
-ACTOR Future<std::vector<ProcessData>> getWorkers(Database cx);
-ACTOR Future<std::vector<ProcessData>> getWorkers(Transaction* tr);
-
-ACTOR Future<Void> timeKeeperSetDisable(Database cx);
-
-ACTOR Future<Void> lockDatabase(Transaction* tr, UID id);
-ACTOR Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id);
-ACTOR Future<Void> lockDatabase(Database cx, UID id);
-
-ACTOR Future<Void> unlockDatabase(Transaction* tr, UID id);
-ACTOR Future<Void> unlockDatabase(Reference<ReadYourWritesTransaction> tr, UID id);
-ACTOR Future<Void> unlockDatabase(Database cx, UID id);
-
-ACTOR Future<Void> checkDatabaseLock(Transaction* tr, UID id);
-ACTOR Future<Void> checkDatabaseLock(Reference<ReadYourWritesTransaction> tr, UID id);
-
-ACTOR Future<Void> advanceVersion(Database cx, Version v);
-
-ACTOR Future<int> setDDMode(Database cx, int mode);
-
-ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
-
-// Start an audit on range of the specific type.
-ACTOR Future<UID> auditStorage(Reference<IClusterConnectionRecord> clusterFile,
-                               KeyRange range,
-                               AuditType type,
-                               KeyValueStoreType engineType,
-                               double timeoutSeconds);
-// Cancel an audit given type and id
-ACTOR Future<UID> cancelAuditStorage(Reference<IClusterConnectionRecord> clusterFile,
-                                     AuditType type,
-                                     UID auditId,
-                                     double timeoutSeconds);
-
-// Set bulk load mode
-// When the mode is on, DD will periodically check if there is any bulkload task to do by scaning the metadata.
-ACTOR Future<int> setBulkLoadMode(Database cx, int mode);
-
-// Get bulk load mode value.
-ACTOR Future<int> getBulkLoadMode(Database cx);
-
-// Create a bulkload task submission transaction without commit
-// Used by ManagementAPI and bulkdumpRestore at DD
-ACTOR Future<Void> setBulkLoadSubmissionTransaction(Transaction* tr, BulkLoadTaskState bulkLoadTask);
-
-// Create an bulkload task acknowledge transaction without commit
-// Used by ManagementAPI and bulkdumpRestore at DD
-ACTOR Future<Void> setBulkLoadFinalizeTransaction(Transaction* tr, KeyRange range, UID taskId);
-
-// Get bulk load task for the input range and taskId
-ACTOR Future<BulkLoadTaskState> getBulkLoadTask(Transaction* tr,
-                                                KeyRange range,
-                                                UID taskId,
-                                                std::vector<BulkLoadPhase> phases);
-
-// Add bulkLoad job to history map
-ACTOR Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState jobState);
-
-// Get all past bulkLoad jobs from history map
-ACTOR Future<std::vector<BulkLoadJobState>> getBulkLoadJobFromHistory(Database cx);
-
-// Erase all bulkLoad job history metadata if jobId is not provided. Otherwise, erase the job with the given jobId.
-ACTOR Future<Void> clearBulkLoadJobHistory(Database cx, Optional<UID> jobId = Optional<UID>());
-
-// Get the current running bulk load job
-// Set lockAware=true when checking during database restore (when database is locked).
-ACTOR Future<Optional<BulkLoadJobState>> getRunningBulkLoadJob(Database cx, bool lockAware = false);
-
-// Cancel bulkLoad job for the given jobId
-ACTOR Future<Void> cancelBulkLoadJob(Database cx, UID jobId);
-
-// Acknowledge all bulk load tasks that are in the Error phase.
-// After acknowledge, the write traffic to the task's range is turned on and the task's metadata is cleared by the bulk
-// load engine.
-ACTOR Future<Void> acknowledgeAllErrorBulkLoadTasks(Database cx, UID jobId, KeyRange jobRange);
-
-// Submit a BulkLoad job: loading data from a remote folder using bulkloading mechanism.
-// There is at most one BulkLoad or one BulkDump job at a time.
-// If there is any existing BulkLoad or BulkDump job, reject the new job.
-// Set lockAware=true when submitting during database restore (when database is locked).
-ACTOR Future<Void> submitBulkLoadJob(Database cx, BulkLoadJobState jobState, bool lockAware = false);
-
-// Return the existing BulkLoad job metadata (if any)
-ACTOR Future<Optional<BulkLoadJobState>> getSubmittedBulkLoadJob(Transaction* tr);
-
-// Set bulk dump mode. When the mode is on, DD will periodically check if there is any bulkdump task to do by scaning
-// the metadata.
-ACTOR Future<int> setBulkDumpMode(Database cx, int mode);
-
-// Get bulk dump mode value.
-ACTOR Future<int> getBulkDumpMode(Database cx);
-
-// TODO(BulkDump): Cancel or clear the BulkDump job
-ACTOR Future<Void> cancelBulkDumpJob(Database cx, UID jobId);
-
-// Submit a bulkdump job: dumping data to a remote folder by storage servers.
-// There is at most one BulkLoad or one BulkDump job at a time.
-// If there is any existing BulkLoad or BulkDump job, reject the new job.
-ACTOR Future<Void> submitBulkDumpJob(Database cx, BulkDumpState bulkDumpJob);
-
-// Return the existing job metadata
-ACTOR Future<Optional<BulkDumpState>> getSubmittedBulkDumpJob(Transaction* tr);
-
-// Get total number of completed tasks within the input range
-ACTOR Future<size_t> getBulkDumpCompleteTaskCount(Database cx, KeyRange rangeToRead);
-
-// Persist a rangeLock owner to database metadata
-// A range can only be locked by a registered owner
-ACTOR Future<Void> registerRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID, std::string description);
-
-// Remove an owner form the database metadata
-ACTOR Future<Void> removeRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID);
-
-// Get all registered rangeLock owner
-ACTOR Future<std::vector<RangeLockOwner>> getAllRangeLockOwners(Database cx);
-
-// Get a rangeLock owner by ownerUniqueID
-ACTOR Future<Optional<RangeLockOwner>> getRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID);
-
-// Block write traffic to a user range (the input range must be within normalKeys).
-// One transaction can call releaseExclusiveReadLockOnRange at most for one time.
-ACTOR Future<Void> takeExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID);
-
-ACTOR Future<Void> takeExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID);
-
-// Unblock a user range (the input range must be within normalKeys).
-// One transaction can call releaseExclusiveReadLockOnRange at most for one time.
-ACTOR Future<Void> releaseExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID);
-
-ACTOR Future<Void> releaseExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID);
-
-// Get locked ranges within the input range (the input range must be within normalKeys)
-ACTOR Future<std::vector<std::pair<KeyRange, RangeLockState>>> findExclusiveReadLockOnRange(
-    Database cx,
-    KeyRange range,
-    Optional<RangeLockOwnerName> ownerName = Optional<RangeLockOwnerName>());
-
-// Clear all exclusive read lock by the input user. Not transactional.
-ACTOR Future<Void> releaseExclusiveReadLockByUser(Database cx, RangeLockOwnerName ownerUniqueID);
-
-ACTOR Future<Void> printHealthyZone(Database cx);
-ACTOR Future<bool> clearHealthyZone(Database cx, bool printWarning = false, bool clearSSFailureZoneString = false);
-ACTOR Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds, bool printWarning = false);
-
-ACTOR Future<Void> waitForPrimaryDC(Database cx, StringRef dcId);
-
-// Gets the cluster connection string
-ACTOR Future<Optional<ClusterConnectionString>> getConnectionString(Database cx);
-
-void schemaCoverage(std::string const& spath, bool covered = true);
-bool schemaMatch(json_spirit::mValue const& schema,
-                 json_spirit::mValue const& result,
-                 std::string& errorStr,
-                 Severity sev = SevError,
-                 bool checkCoverage = false,
-                 std::string path = std::string(),
-                 std::string schema_path = std::string());
-
-// execute payload in 'snapCmd' on all the coordinators, TLogs and
-// storage nodes
-ACTOR Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
-
-ACTOR Future<Void> disableBackupWorker(Database cx);
-ACTOR Future<Void> enableBackupWorker(Database cx);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/ManagementAPI.h b/fdbclient/include/fdbclient/ManagementAPI.h
new file mode 100644
index 00000000000..35496efda33
--- /dev/null
+++ b/fdbclient/include/fdbclient/ManagementAPI.h
@@ -0,0 +1,419 @@
+/*
+ * ManagementAPI.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+/* This file defines "management" interfaces for configuration, coordination changes, and
+the inclusion and exclusion of servers. It is used to implement fdbcli management commands
+and by test workloads that simulate such. It isn't exposed to C clients or anywhere outside
+our code base and doesn't need to be versioned. It doesn't do anything you can't do with the
+standard API and some knowledge of the contents of the system key space.
+*/
+
+#include <string>
+#include <map>
+#include "fdbclient/GenericManagementAPI.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/RangeLock.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/DatabaseConfiguration.h"
+#include "fdbclient/MonitorLeader.h"
+
+Future<DatabaseConfiguration> getDatabaseConfiguration(Transaction* tr, bool useSystemPriority = false);
+Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx, bool useSystemPriority = false);
+Future<Void> waitForFullReplication(Database cx);
+
+struct IQuorumChange : ReferenceCounted<IQuorumChange> {
+	virtual ~IQuorumChange() {}
+	virtual Future<std::vector<NetworkAddress>> getDesiredCoordinators(Transaction* tr,
+	                                                                   std::vector<NetworkAddress> oldCoordinators,
+	                                                                   Reference<IClusterConnectionRecord>,
+	                                                                   CoordinatorsResult&) = 0;
+	virtual std::string getDesiredClusterKeyName() const { return std::string(); }
+};
+
+// Change to use the given set of coordination servers
+Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
+                                                         ClusterConnectionString* conn,
+                                                         std::string newName);
+Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> change);
+Reference<IQuorumChange> autoQuorumChange(int desired = -1);
+Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
+
+// Exclude the given set of servers from use as state servers.  Returns as soon as the change is durable, without
+// necessarily waiting for the servers to be evacuated.  A NetworkAddress with a port of 0 means all servers on the
+// given IP.
+Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
+Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed = false);
+
+// Exclude the servers matching the given set of localities from use as state servers.  Returns as soon as the change
+// is durable, without necessarily waiting for the servers to be evacuated.
+Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> localities, bool failed = false);
+Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string> localities, bool failed = false);
+
+// Remove the given servers from the exclusion list.  A NetworkAddress with a port of 0 means all servers on the given
+// IP.  A NetworkAddress() means all servers (don't exclude anything)
+Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
+
+// Remove the given localities from the exclusion list.
+Future<Void> includeLocalities(Database cx,
+                               std::vector<std::string> localities,
+                               bool failed = false,
+                               bool includeAll = false);
+
+// Set the process class of processes with the given address.  A NetworkAddress with a port of 0 means all servers on
+// the given IP.
+Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
+
+// Get the current list of excluded servers including both "exclude" and "failed".
+Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx);
+Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr);
+
+// Get the current list of excluded servers.
+Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* tr);
+
+// Get the current list of failed servers.
+Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr);
+
+// Get the current list of excluded localities
+Future<std::vector<std::string>> getAllExcludedLocalities(Database cx);
+Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr);
+
+// Get the current list of excluded localities.
+Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr);
+
+// Get the current list of failed localities.
+Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr);
+
+// Decodes the locality string to a pair of locality prefix and its value.
+// The prefix could be dcid, processid, machineid, processid.
+std::pair<std::string, std::string> decodeLocality(const std::string& locality);
+std::set<AddressExclusion> getServerAddressesByLocality(
+    const std::map<std::string, StorageServerInterface> server_interfaces,
+    const std::string& locality);
+std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers, const std::string& locality);
+
+// Check for the given, previously excluded servers to be evacuated (no longer used for state).  If waitForExclusion
+// is true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance,
+// until and unless any of them are explicitly included with includeServers()
+Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
+                                                          std::vector<AddressExclusion> servers,
+                                                          bool waitForAllExcluded);
+Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
+                                             std::set<AddressExclusion>* exclusions,
+                                             std::set<NetworkAddress>* inProgressExclusion);
+
+// Gets a list of all workers in the cluster (excluding testers)
+Future<std::vector<ProcessData>> getWorkers(Database cx);
+Future<std::vector<ProcessData>> getWorkers(Transaction* tr);
+
+Future<Void> timeKeeperSetDisable(Database cx);
+
+Future<Void> lockDatabase(Transaction* tr, UID id);
+Future<Void> lockDatabase(Reference<ReadYourWritesTransaction> tr, UID id);
+Future<Void> lockDatabase(Database cx, UID id);
+
+Future<Void> unlockDatabase(Transaction* tr, UID id);
+Future<Void> unlockDatabase(Reference<ReadYourWritesTransaction> tr, UID id);
+Future<Void> unlockDatabase(Database cx, UID id);
+
+Future<Void> checkDatabaseLock(Transaction* tr, UID id);
+Future<Void> checkDatabaseLock(Reference<ReadYourWritesTransaction> tr, UID id);
+
+Future<Void> advanceVersion(Database cx, Version v);
+
+Future<int> setDDMode(Database cx, int mode);
+
+Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
+
+// Start an audit on range of the specific type.
+Future<UID> auditStorage(Reference<IClusterConnectionRecord> clusterFile,
+                         KeyRange range,
+                         AuditType type,
+                         KeyValueStoreType engineType,
+                         double timeoutSeconds);
+// Cancel an audit given type and id
+Future<UID> cancelAuditStorage(Reference<IClusterConnectionRecord> clusterFile,
+                               AuditType type,
+                               UID auditId,
+                               double timeoutSeconds);
+
+// Set bulk load mode
+// When the mode is on, DD will periodically check if there is any bulkload task to do by scaning the metadata.
+Future<int> setBulkLoadMode(Database cx, int mode);
+
+// Get bulk load mode value.
+Future<int> getBulkLoadMode(Database cx);
+
+// Create a bulkload task submission transaction without commit
+// Used by ManagementAPI and bulkdumpRestore at DD
+Future<Void> setBulkLoadSubmissionTransaction(Transaction* tr, BulkLoadTaskState bulkLoadTask);
+
+// Create an bulkload task acknowledge transaction without commit
+// Used by ManagementAPI and bulkdumpRestore at DD
+Future<Void> setBulkLoadFinalizeTransaction(Transaction* tr, KeyRange range, UID taskId);
+
+// Get bulk load task for the input range and taskId
+Future<BulkLoadTaskState> getBulkLoadTask(Transaction* tr,
+                                          KeyRange range,
+                                          UID taskId,
+                                          std::vector<BulkLoadPhase> phases);
+
+// Add bulkLoad job to history map
+Future<Void> addBulkLoadJobToHistory(Transaction* tr, BulkLoadJobState jobState);
+
+// Get all past bulkLoad jobs from history map
+AsyncResult<std::vector<BulkLoadJobState>> getBulkLoadJobFromHistory(Database cx);
+
+// Erase all bulkLoad job history metadata if jobId is not provided. Otherwise, erase the job with the given jobId.
+Future<Void> clearBulkLoadJobHistory(Database cx, Optional<UID> jobId = Optional<UID>());
+
+// Get the current running bulk load job
+// Set lockAware=true when checking during database restore (when database is locked).
+Future<Optional<BulkLoadJobState>> getRunningBulkLoadJob(Database cx, bool lockAware = false);
+
+// Cancel bulkLoad job for the given jobId
+Future<Void> cancelBulkLoadJob(Database cx, UID jobId);
+
+// Acknowledge all bulk load tasks that are in the Error phase.
+// After acknowledge, the write traffic to the task's range is turned on and the task's metadata is cleared by the bulk
+// load engine.
+Future<Void> acknowledgeAllErrorBulkLoadTasks(Database cx, UID jobId, KeyRange jobRange);
+
+// Submit a BulkLoad job: loading data from a remote folder using bulkloading mechanism.
+// There is at most one BulkLoad or one BulkDump job at a time.
+// If there is any existing BulkLoad or BulkDump job, reject the new job.
+// Set lockAware=true when submitting during database restore (when database is locked).
+Future<Void> submitBulkLoadJob(Database cx, BulkLoadJobState jobState, bool lockAware = false);
+
+// Return the existing BulkLoad job metadata (if any)
+Future<Optional<BulkLoadJobState>> getSubmittedBulkLoadJob(Transaction* tr);
+
+// Set bulk dump mode. When the mode is on, DD will periodically check if there is any bulkdump task to do by scaning
+// the metadata.
+Future<int> setBulkDumpMode(Database cx, int mode);
+
+// Get bulk dump mode value.
+Future<int> getBulkDumpMode(Database cx);
+
+// TODO(BulkDump): Cancel or clear the BulkDump job
+Future<Void> cancelBulkDumpJob(Database cx, UID jobId);
+
+// Submit a bulkdump job: dumping data to a remote folder by storage servers.
+// There is at most one BulkLoad or one BulkDump job at a time.
+// If there is any existing BulkLoad or BulkDump job, reject the new job.
+Future<Void> submitBulkDumpJob(Database cx, BulkDumpState bulkDumpJob);
+
+// Return the existing job metadata
+Future<Optional<BulkDumpState>> getSubmittedBulkDumpJob(Transaction* tr);
+
+// Get total number of completed tasks within the input range
+Future<size_t> getBulkDumpCompleteTaskCount(Database cx, KeyRange rangeToRead);
+
+// ==================== BulkDump/BulkLoad Progress Tracking ====================
+
+// Represents a stalled task for observability
+struct BulkLoadStalledTask {
+	UID taskId;
+	KeyRange range;
+	UID storageServerId;
+	double stalledSeconds;
+	int restartCount;
+	std::string lastError;
+
+	BulkLoadStalledTask() : stalledSeconds(0), restartCount(0) {}
+	BulkLoadStalledTask(UID taskId,
+	                    KeyRange range,
+	                    UID storageServerId,
+	                    double stalledSeconds,
+	                    int restartCount,
+	                    std::string lastError)
+	  : taskId(taskId), range(range), storageServerId(storageServerId), stalledSeconds(stalledSeconds),
+	    restartCount(restartCount), lastError(lastError) {}
+};
+
+// Aggregated progress for a BulkDump job
+struct BulkDumpProgress {
+	UID jobId;
+	KeyRange jobRange;
+	int totalTasks;
+	int completeTasks;
+	int runningTasks;
+	int errorTasks;
+	int64_t totalBytes;
+	int64_t completedBytes;
+	double elapsedSeconds;
+	double startTime; // For computing current throughput
+	std::vector<BulkLoadStalledTask> stalledTasks;
+
+	BulkDumpProgress()
+	  : totalTasks(0), completeTasks(0), runningTasks(0), errorTasks(0), totalBytes(0), completedBytes(0),
+	    elapsedSeconds(0), startTime(0) {}
+
+	double progressPercent() const { return totalTasks > 0 ? 100.0 * completeTasks / totalTasks : 0; }
+
+	double avgBytesPerSecond() const { return elapsedSeconds > 0 ? completedBytes / elapsedSeconds : 0; }
+
+	Optional<double> etaSeconds() const {
+		double rate = avgBytesPerSecond();
+		if (rate > 0 && completedBytes < totalBytes) {
+			return (totalBytes - completedBytes) / rate;
+		}
+		return {};
+	}
+};
+
+// Aggregated progress for a BulkLoad job
+struct BulkLoadProgress {
+	UID jobId;
+	KeyRange jobRange;
+	int totalTasks;
+	int submittedTasks;
+	int triggeredTasks;
+	int runningTasks;
+	int completeTasks;
+	int errorTasks;
+	int64_t totalBytes;
+	int64_t completedBytes;
+	double elapsedSeconds;
+	double startTime;
+	std::vector<BulkLoadStalledTask> stalledTasks;
+
+	BulkLoadProgress()
+	  : totalTasks(0), submittedTasks(0), triggeredTasks(0), runningTasks(0), completeTasks(0), errorTasks(0),
+	    totalBytes(0), completedBytes(0), elapsedSeconds(0), startTime(0) {}
+
+	double progressPercent() const { return totalTasks > 0 ? 100.0 * completeTasks / totalTasks : 0; }
+
+	double avgBytesPerSecond() const { return elapsedSeconds > 0 ? completedBytes / elapsedSeconds : 0; }
+
+	Optional<double> etaSeconds() const {
+		double rate = avgBytesPerSecond();
+		if (rate > 0 && completedBytes < totalBytes) {
+			return (totalBytes - completedBytes) / rate;
+		}
+		return {};
+	}
+};
+
+// Get aggregated progress for the current BulkDump job (if any)
+// Reads task metadata from system keys and computes aggregate metrics
+Future<Optional<BulkDumpProgress>> getBulkDumpProgress(Database cx);
+
+// Get aggregated progress for the current BulkLoad job (if any)
+// Reads task metadata from system keys and computes aggregate metrics
+Future<Optional<BulkLoadProgress>> getBulkLoadProgress(Database cx);
+
+// Threshold in seconds for considering a task "stalled".
+// SST file downloads from blobstore can take 5-10 minutes per task,
+// so this needs to be well above normal download times.
+constexpr double BULK_TASK_STALL_THRESHOLD_SECONDS = 600.0;
+
+// BulkDump owner tracking - stored in separate system keys for backward compatibility
+struct BulkDumpOwnerInfo {
+	UID ownerUID;
+	std::string ownerType; // e.g., "backup", "dr", "migration"
+	std::string ownerName; // e.g., backup tag
+	double submitTime;
+
+	constexpr static FileIdentifier file_identifier = 1384510;
+
+	BulkDumpOwnerInfo() : submitTime(0) {}
+	BulkDumpOwnerInfo(UID uid, std::string type, std::string name, double time)
+	  : ownerUID(uid), ownerType(type), ownerName(name), submitTime(time) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, ownerUID, ownerType, ownerName, submitTime);
+	}
+};
+
+// Generic bulk owner tracking (internal implementation)
+Future<Void> setBulkOwner(Database cx, UID jobId, BulkDumpOwnerInfo ownerInfo, bool isBulkDump);
+Future<Optional<BulkDumpOwnerInfo>> getBulkOwner(Database cx, UID jobId, bool isBulkDump);
+
+// Public API wrappers (for backward compatibility)
+Future<Void> setBulkDumpOwner(Database cx, UID jobId, BulkDumpOwnerInfo ownerInfo);
+Future<Optional<BulkDumpOwnerInfo>> getBulkDumpOwner(Database cx, UID jobId);
+Future<Void> setBulkLoadOwner(Database cx, UID jobId, BulkDumpOwnerInfo ownerInfo);
+Future<Optional<BulkDumpOwnerInfo>> getBulkLoadOwner(Database cx, UID jobId);
+
+// ==================== End Progress Tracking ====================
+
+// Persist a rangeLock owner to database metadata
+// A range can only be locked by a registered owner
+Future<Void> registerRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID, std::string description);
+
+// Remove an owner form the database metadata
+Future<Void> removeRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID);
+
+// Get all registered rangeLock owner
+AsyncResult<std::vector<RangeLockOwner>> getAllRangeLockOwners(Database cx);
+
+// Get a rangeLock owner by ownerUniqueID
+Future<Optional<RangeLockOwner>> getRangeLockOwner(Database cx, RangeLockOwnerName ownerUniqueID);
+
+// Block write traffic to a user range (the input range must be within normalKeys).
+// One transaction can call releaseExclusiveReadLockOnRange at most for one time.
+Future<Void> takeExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID);
+
+Future<Void> takeExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID);
+
+// Unblock a user range (the input range must be within normalKeys).
+// One transaction can call releaseExclusiveReadLockOnRange at most for one time.
+Future<Void> releaseExclusiveReadLockOnRange(Transaction* tr, KeyRange range, RangeLockOwnerName ownerUniqueID);
+
+Future<Void> releaseExclusiveReadLockOnRange(Database cx, KeyRange range, RangeLockOwnerName ownerUniqueID);
+
+// Get locked ranges within the input range (the input range must be within normalKeys)
+Future<std::vector<std::pair<KeyRange, RangeLockState>>> findExclusiveReadLockOnRange(
+    Database cx,
+    KeyRange range,
+    Optional<RangeLockOwnerName> ownerName = Optional<RangeLockOwnerName>());
+
+// Clear all exclusive read lock by the input user. Not transactional.
+Future<Void> releaseExclusiveReadLockByUser(Database cx, RangeLockOwnerName ownerUniqueID);
+
+Future<Void> printHealthyZone(Database cx);
+Future<bool> clearHealthyZone(Database cx, bool printWarning = false, bool clearSSFailureZoneString = false);
+Future<bool> setHealthyZone(Database cx, StringRef zoneId, double seconds, bool printWarning = false);
+
+Future<Void> waitForPrimaryDC(Database cx, StringRef dcId);
+
+// Gets the cluster connection string
+Future<Optional<ClusterConnectionString>> getConnectionString(Database cx);
+
+void schemaCoverage(std::string const& spath, bool covered = true);
+bool schemaMatch(json_spirit::mValue const& schema,
+                 json_spirit::mValue const& result,
+                 std::string& errorStr,
+                 Severity sev = SevError,
+                 bool checkCoverage = false,
+                 std::string path = std::string(),
+                 std::string schema_path = std::string());
+
+// execute payload in 'snapCmd' on all the coordinators, TLogs and
+// storage nodes
+Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
+
+Future<Void> disableBackupWorker(Database cx);
+Future<Void> enableBackupWorker(Database cx);
+Future<Void> disableRangeBackupWorker(Database cx);
+Future<Void> enableRangeBackupWorker(Database cx);
diff --git a/fdbclient/include/fdbclient/MonitorLeader.h b/fdbclient/include/fdbclient/MonitorLeader.h
index 9aa2dc837a1..607452ba971 100644
--- a/fdbclient/include/fdbclient/MonitorLeader.h
+++ b/fdbclient/include/fdbclient/MonitorLeader.h
@@ -75,19 +75,18 @@ Future<Void> monitorLeader(Reference<IClusterConnectionRecord> const& connFile,
 // This is one place where the leader election algorithm is run. The coodinator contacts all coodinators to collect
 // nominees, the nominee with the most nomination is the leader, and collects client data from the leader. This function
 // also monitors the change of the leader.
-Future<Void> monitorLeaderAndGetClientInfo(Key const& clusterKey,
-                                           std::vector<Hostname> const& hostnames,
-                                           std::vector<NetworkAddress> const& coordinators,
-                                           ClientData* const& clientData,
-                                           Reference<AsyncVar<Optional<LeaderInfo>>> const& leaderInfo);
-
-Future<Void> monitorProxies(
-    Reference<AsyncVar<Reference<IClusterConnectionRecord>>> const& connRecord,
-    Reference<AsyncVar<ClientDBInfo>> const& clientInfo,
-    Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> const& coordinator,
-    Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions,
-    Key const& traceLogGroup,
-    IsInternal const& internal);
+Future<Void> monitorLeaderAndGetClientInfo(Key clusterKey,
+                                           std::vector<Hostname> hostnames,
+                                           std::vector<NetworkAddress> coordinators,
+                                           ClientData* clientData,
+                                           Reference<AsyncVar<Optional<LeaderInfo>>> leaderInfo);
+
+Future<Void> monitorProxies(Reference<AsyncVar<Reference<IClusterConnectionRecord>>> connRecord,
+                            Reference<AsyncVar<ClientDBInfo>> clientInfo,
+                            Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
+                            Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
+                            Key traceLogGroup,
+                            IsInternal internal);
 
 void shrinkProxyList(ClientDBInfo& ni,
                      std::vector<UID>& lastCommitProxyUIDs,
@@ -99,8 +98,8 @@ void shrinkProxyList(ClientDBInfo& ni,
 #pragma region Implementation
 #endif
 
-Future<Void> monitorLeaderInternal(Reference<IClusterConnectionRecord> const& connRecord,
-                                   Reference<AsyncVar<Value>> const& outSerializedLeaderInfo);
+Future<Void> monitorLeaderInternal(Reference<IClusterConnectionRecord> connRecord,
+                                   Reference<AsyncVar<Value>> outSerializedLeaderInfo);
 
 template <class LeaderInterface>
 struct LeaderDeserializer {
@@ -110,8 +109,8 @@ struct LeaderDeserializer {
 	}
 };
 
-Future<Void> asyncDeserializeClusterInterface(const Reference<AsyncVar<Value>>& serializedInfo,
-                                              const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader);
+Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> serializedInfo,
+                                              Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader);
 
 template <>
 struct LeaderDeserializer<ClusterInterface> {
diff --git a/fdbclient/include/fdbclient/MultiVersionTransaction.h b/fdbclient/include/fdbclient/MultiVersionTransaction.h
index f4c351b3ce0..ffbf8a008d2 100644
--- a/fdbclient/include/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/include/fdbclient/MultiVersionTransaction.h
@@ -393,7 +393,7 @@ class DLDatabase : public IDatabase, ThreadSafeReferenceCounted<DLDatabase> {
 // The DL prefix stands for "dynamic library".
 class DLApi : public IClientApi {
 public:
-	DLApi(std::string fdbCPath, bool unlinkOnLoad = false);
+	explicit DLApi(std::string fdbCPath, bool unlinkOnLoad = false);
 
 	void selectApiVersion(int apiVersion) override;
 	const char* getClientVersion() override;
@@ -585,7 +585,7 @@ struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted<ClientInfo> {
 	ClientInfo()
 	  : ClientDesc(std::string(), false, false), protocolVersion(0), api(nullptr), failed(true), initialized(false),
 	    threadIndex(0) {}
-	ClientInfo(IClientApi* api)
+	explicit ClientInfo(IClientApi* api)
 	  : ClientDesc("internal", false, false), protocolVersion(0), api(api), failed(false), initialized(false),
 	    threadIndex(0) {}
 	ClientInfo(IClientApi* api, std::string libPath, bool useFutureVersion, int threadIndex)
diff --git a/fdbclient/include/fdbclient/MutationLogReader.actor.h b/fdbclient/include/fdbclient/MutationLogReader.h
similarity index 78%
rename from fdbclient/include/fdbclient/MutationLogReader.actor.h
rename to fdbclient/include/fdbclient/MutationLogReader.h
index ea56b81775c..cf27f7934e9 100644
--- a/fdbclient/include/fdbclient/MutationLogReader.actor.h
+++ b/fdbclient/include/fdbclient/MutationLogReader.h
@@ -1,5 +1,5 @@
 /*
- * MutationLogReader.actor.h
+ * MutationLogReader.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,18 +20,11 @@
 
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_MUTATION_LOG_READER_ACTOR_G_H)
-#define FDBCLIENT_MUTATION_LOG_READER_ACTOR_G_H
-#include "fdbclient/MutationLogReader.actor.g.h"
-#elif !defined(FDBCLIENT_MUTATION_LOG_READER_ACTOR_H)
-#define FDBCLIENT_MUTATION_LOG_READER_ACTOR_H
-
 #include <deque>
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "flow/flow.h"
 #include "flow/ActorCollection.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 namespace mutation_log_reader {
 
@@ -68,7 +61,7 @@ class PipelinedReader {
 
 	void startReading(Database cx);
 	Future<Void> getNext(Database cx);
-	ACTOR static Future<Void> getNext_impl(PipelinedReader* self, Database cx);
+	static Future<Void> getNext_impl(PipelinedReader* self, Database cx);
 
 	void release() { readerLimit.release(); }
 
@@ -109,22 +102,22 @@ class MutationLogReader : public ReferenceCounted<MutationLogReader> {
 		}
 	}
 
-	ACTOR static Future<Reference<MutationLogReader>> Create(Database cx,
-	                                                         Version bv,
-	                                                         Version ev,
-	                                                         Key uid,
-	                                                         Key beginKey,
-	                                                         unsigned pd) {
-		state Reference<MutationLogReader> self(new MutationLogReader(cx, bv, ev, uid, beginKey, pd));
-		wait(self->initializePQ(self.getPtr()));
-		return self;
+	static Future<Reference<MutationLogReader>> Create(Database cx,
+	                                                   Version bv,
+	                                                   Version ev,
+	                                                   Key uid,
+	                                                   Key beginKey,
+	                                                   unsigned pd) {
+		Reference<MutationLogReader> self(new MutationLogReader(cx, bv, ev, uid, beginKey, pd));
+		co_await self->initializePQ(self.getPtr());
+		co_return self;
 	}
 
 	Future<Standalone<RangeResultRef>> getNext();
 
 private:
-	ACTOR static Future<Void> initializePQ(MutationLogReader* self);
-	ACTOR static Future<Standalone<RangeResultRef>> getNext_impl(MutationLogReader* self);
+	static Future<Void> initializePQ(MutationLogReader* self);
+	static Future<Standalone<RangeResultRef>> getNext_impl(MutationLogReader* self);
 
 	std::vector<std::unique_ptr<mutation_log_reader::PipelinedReader>> pipelinedReaders;
 	std::priority_queue<mutation_log_reader::RangeResultBlock> priorityQueue;
@@ -133,6 +126,3 @@ class MutationLogReader : public ReferenceCounted<MutationLogReader> {
 	unsigned pipelineDepth;
 	unsigned finished;
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/NativeAPI.actor.h b/fdbclient/include/fdbclient/NativeAPI.actor.h
index 9921b7e0616..82317ea2c43 100644
--- a/fdbclient/include/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/include/fdbclient/NativeAPI.actor.h
@@ -28,7 +28,7 @@
 #include "flow/BooleanParam.h"
 #include "flow/flow.h"
 #include "flow/WipedString.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/IRandom.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/CommitProxyInterface.h"
@@ -110,8 +110,8 @@ class Database {
 
 	Database() {} // an uninitialized database can be destructed or reassigned safely; that's it
 	void operator=(Database const& rhs) { db = rhs.db; }
-	Database(Database const& rhs) : db(rhs.db) {}
-	Database(Database&& r) noexcept : db(std::move(r.db)) {}
+	explicit(false) Database(Database const& rhs) : db(rhs.db) {}
+	explicit(false) Database(Database&& r) noexcept : db(std::move(r.db)) {}
 	void operator=(Database&& r) noexcept { db = std::move(r.db); }
 
 	// For internal use by the native client:
@@ -175,6 +175,7 @@ struct TransactionOptions {
 	bool rawAccess : 1;
 	bool bypassStorageQuota : 1;
 	bool enableReplicaConsistencyCheck : 1;
+	Optional<int64_t> maxGrvQueueDelayMS;
 	int requiredReplicas;
 
 	TransactionPriority priority;
@@ -184,7 +185,7 @@ struct TransactionOptions {
 
 	// update clear function if you add a new field
 
-	TransactionOptions(Database const& cx);
+	explicit TransactionOptions(Database const& cx);
 	TransactionOptions();
 
 	void reset(Database const& cx);
@@ -199,7 +200,7 @@ struct TransactionLogInfo : public ReferenceCounted<TransactionLogInfo>, NonCopy
 	enum LoggingLocation { DONT_LOG = 0, TRACE_LOG = 1, DATABASE = 2 };
 
 	TransactionLogInfo() : logLocation(DONT_LOG), maxFieldLength(0) {}
-	TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {}
+	explicit TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {}
 	TransactionLogInfo(std::string id, LoggingLocation location)
 	  : logLocation(location), maxFieldLength(0), identifier(id) {}
 
@@ -245,7 +246,7 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
 	Optional<ReadOptions> readOptions;
 
 	Watch() : valuePresent(false), setPresent(false), watchFuture(Never()) {}
-	Watch(Key key) : key(key), valuePresent(false), setPresent(false), watchFuture(Never()) {}
+	explicit Watch(Key key) : key(key), valuePresent(false), setPresent(false), watchFuture(Never()) {}
 	Watch(Key key, Optional<Value> val)
 	  : key(key), value(val), valuePresent(true), setPresent(false), watchFuture(Never()) {}
 
@@ -269,8 +270,6 @@ struct TransactionState : ReferenceCounted<TransactionState> {
 	// after rounding up to the nearest page size and applying a write penalty
 	int64_t totalCost = 0;
 
-	double proxyTagThrottledDuration = 0.0;
-
 	int numErrors = 0;
 	double startTime = 0;
 	Promise<Standalone<StringRef>> versionstampPromise;
@@ -543,9 +542,7 @@ Future<Void> Database::run(Fun fun) {
 }
 
 ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanContext spanContext);
-ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
-                                                                                  KeyRange keys,
-                                                                                  int shardLimit);
+Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx, KeyRange keys, int shardLimit);
 
 int64_t extractIntOption(Optional<StringRef> value,
                          int64_t minValue = std::numeric_limits<int64_t>::min(),
@@ -553,7 +550,7 @@ int64_t extractIntOption(Optional<StringRef> value,
 
 // Takes a snapshot of the cluster, specifically the following persistent
 // states: coordinator, TLog and storage state
-ACTOR Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
+Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
 
 // Adds necessary mutation(s) to the transaction, so that *one* checkpoint will be created for
 // each and every shards overlapping with `ranges`.
@@ -572,7 +569,7 @@ Future<Void> createCheckpoint(Reference<ReadYourWritesTransaction> tr,
 // Gets checkpoint metadata for `ranges` at the specific version, with the particular format.
 // Returns a list of [range, checkpoint], where the `checkpoint` has data over `range`.
 // checkpoint_not_found() error will be returned if the specific checkpoint cannot be found.
-ACTOR Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpointMetaData(
+Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpointMetaData(
     Database cx,
     std::vector<KeyRange> ranges,
     Version version,
@@ -581,15 +578,15 @@ ACTOR Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpoint
     double timeout = 5.0);
 
 // Checks with Data Distributor that it is safe to mark all servers in exclusions as failed
-ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions);
+Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions);
 
 // Measured in bytes, rounded up to the nearest page size. Multiply by fungibility ratio
 // because writes are more expensive than reads.
 inline uint64_t getWriteOperationCost(uint64_t bytes) {
 	if (bytes == 0) {
-		return CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
+		return CLIENT_KNOBS->TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
 	} else {
-		return CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE *
+		return CLIENT_KNOBS->TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE *
 		       ((bytes - 1) / CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE + 1);
 	}
 }
@@ -606,11 +603,11 @@ inline uint64_t getReadOperationCost(uint64_t bytes) {
 // Create a transaction to set the value of system key \xff/conf/perpetual_storage_wiggle. If enable == true, the value
 // will be 1. Otherwise, the value will be 0. The caller should take care of the reset of StorageWiggleMetrics if
 // necessary. Returns the FDB version at which the transaction was committed.
-ACTOR Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware = LockAware::False);
+Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware = LockAware::False);
 
-ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
-                                                                                      bool primary,
-                                                                                      bool use_system_priority);
+Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
+                                                                                bool primary,
+                                                                                bool use_system_priority);
 
 // Returns the maximum legal size of a key. This size will be determined by the prefix of the passed in key
 // (system keys have a larger maximum size). This should be used for generic max key size requests.
@@ -629,12 +626,12 @@ int64_t getMaxClearKeySize(KeyRef const& key);
 struct KeyRangeLocationInfo;
 // Return the aggregated StorageMetrics of range keys to the caller. The locations tell which interface should
 // serve the request. The final result is within (min-permittedError/2, max + permittedError/2) if valid.
-ACTOR Future<Optional<StorageMetrics>> waitStorageMetricsWithLocation(Version version,
-                                                                      KeyRange keys,
-                                                                      std::vector<KeyRangeLocationInfo> locations,
-                                                                      StorageMetrics min,
-                                                                      StorageMetrics max,
-                                                                      StorageMetrics permittedError);
+Future<Optional<StorageMetrics>> waitStorageMetricsWithLocation(Version version,
+                                                                KeyRange keys,
+                                                                std::vector<KeyRangeLocationInfo> locations,
+                                                                StorageMetrics min,
+                                                                StorageMetrics max,
+                                                                StorageMetrics permittedError);
 
 // Return the suggested split points from storage server.The locations tell which interface should
 // serve the request. `limit` is the current estimated storage metrics of `keys`.The returned points, if present,
@@ -646,19 +643,20 @@ ACTOR Future<Optional<Standalone<VectorRef<KeyRef>>>> splitStorageMetricsWithLoc
     StorageMetrics estimated,
     Optional<int> minSplitBytes);
 
+Future<RangeResult> getWorkerInterfaces(Reference<IClusterConnectionRecord> clusterRecord);
+
 namespace NativeAPI {
-ACTOR Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(
-    Transaction* tr);
+Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(Transaction* tr);
 }
-ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
-                                                           Key key,
-                                                           SpanContext spanContext,
-                                                           Optional<UID> debugID,
-                                                           UseProvisionalProxies useProvisionalProxies,
-                                                           Reverse isBackward,
-                                                           Version version);
-
-ACTOR Future<Void> refreshTransaction(DatabaseContext* self, Transaction* tr);
+Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
+                                                     Key key,
+                                                     SpanContext spanContext,
+                                                     Optional<UID> debugID,
+                                                     UseProvisionalProxies useProvisionalProxies,
+                                                     Reverse isBackward,
+                                                     Version version);
+
+Future<Void> refreshTransaction(DatabaseContext* self, Transaction* tr);
 
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbclient/include/fdbclient/Notified.h b/fdbclient/include/fdbclient/Notified.h
index b055a5ce7d1..edcbd521dde 100644
--- a/fdbclient/include/fdbclient/Notified.h
+++ b/fdbclient/include/fdbclient/Notified.h
@@ -23,7 +23,7 @@
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/swift_support.h"
 #include <queue>
 
@@ -76,7 +76,7 @@ struct Notified {
 
 	void operator=(const ValueType& v) { set(v); }
 
-	Notified(Notified&& r) noexcept : waiting(std::move(r.waiting)), val(std::move(r.val)) {}
+	explicit(false) Notified(Notified&& r) noexcept : waiting(std::move(r.waiting)), val(std::move(r.val)) {}
 	void operator=(Notified&& r) noexcept {
 		waiting = std::move(r.waiting);
 		val = std::move(r.val);
diff --git a/fdbclient/include/fdbclient/ParallelStream.actor.h b/fdbclient/include/fdbclient/ParallelStream.actor.h
deleted file mode 100644
index a4c6ea02e98..00000000000
--- a/fdbclient/include/fdbclient/ParallelStream.actor.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * ParallelStream.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_PARALLEL_STREAM_ACTOR_G_H)
-#define FDBCLIENT_PARALLEL_STREAM_ACTOR_G_H
-#include "fdbclient/ParallelStream.actor.g.h"
-#elif !defined(FDBCLIENT_PARALLEL_STREAM_ACTOR_H)
-#define FDBCLIENT_PARALLEL_STREAM_ACTOR_H
-
-#include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // must be last include
-
-// ParallelStream is used to fetch data from multiple streams in parallel and then merge them back into a single stream
-// in order.
-template <class T>
-class ParallelStream {
-	Reference<BoundedFlowLock> semaphore;
-	struct FragmentConstructorTag {
-		explicit FragmentConstructorTag() = default;
-	};
-
-public:
-	// A Fragment is a single stream that will get results to be merged back into the main output stream
-	class Fragment : public ReferenceCounted<Fragment> {
-		Reference<BoundedFlowLock> semaphore;
-		PromiseStream<T> stream;
-		BoundedFlowLock::Releaser releaser;
-		friend class ParallelStream;
-
-	public:
-		Fragment(Reference<BoundedFlowLock> semaphore, int64_t permitNumber, FragmentConstructorTag)
-		  : semaphore(semaphore), releaser(semaphore.getPtr(), permitNumber) {}
-		template <class U>
-		void send(U&& value) {
-			stream.send(std::forward<U>(value));
-		}
-		void sendError(Error e) { stream.sendError(e); }
-		void finish() {
-			releaser.release(); // Release before destruction to free up pending fragments
-			stream.sendError(end_of_stream());
-		}
-		Future<Void> onEmpty() { return stream.onEmpty(); }
-	};
-
-private:
-	PromiseStream<Reference<Fragment>> fragments;
-	size_t fragmentsProcessed{ 0 };
-	PromiseStream<T> results;
-	Future<Void> flusher;
-
-public:
-	// A background actor which take results from the oldest fragment and sends them to the main output stream
-	ACTOR static Future<Void> flushToClient(ParallelStream<T>* self) {
-		state const int messagesBetweenYields = 1000;
-		state int messagesSinceYield = 0;
-		try {
-			loop {
-				state Reference<Fragment> fragment = waitNext(self->fragments.getFuture());
-				loop {
-					try {
-						wait(self->results.onEmpty());
-						T value = waitNext(fragment->stream.getFuture());
-						self->results.send(value);
-						if (++messagesSinceYield == messagesBetweenYields) {
-							wait(yield());
-							messagesSinceYield = 0;
-						}
-					} catch (Error& e) {
-						if (e.code() == error_code_end_of_stream) {
-							fragment.clear();
-							break;
-						} else {
-							throw e;
-						}
-					}
-				}
-			}
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
-			}
-			self->results.sendError(e);
-			return Void();
-		}
-	}
-
-	ParallelStream(PromiseStream<T> results, size_t bufferLimit) : results(results) {
-		semaphore = makeReference<BoundedFlowLock>(1, bufferLimit);
-		flusher = flushToClient(this);
-	}
-
-	// Creates a fragment to get merged into the main output stream
-	ACTOR static Future<Fragment*> createFragmentImpl(ParallelStream<T>* self) {
-		int64_t permitNumber = wait(self->semaphore->take());
-		auto fragment = makeReference<Fragment>(self->semaphore, permitNumber, FragmentConstructorTag());
-		self->fragments.send(fragment);
-		return fragment.getPtr();
-	}
-
-	Future<Fragment*> createFragment() { return createFragmentImpl(this); }
-
-	Future<Void> finish() {
-		fragments.sendError(end_of_stream());
-		return flusher;
-	}
-};
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbclient/include/fdbclient/RandomKeyValueUtils.h b/fdbclient/include/fdbclient/RandomKeyValueUtils.h
index 5a0ed3097e5..8b75b145c64 100644
--- a/fdbclient/include/fdbclient/RandomKeyValueUtils.h
+++ b/fdbclient/include/fdbclient/RandomKeyValueUtils.h
@@ -68,7 +68,7 @@ struct RandomIntGenerator : IGenerator<unsigned int> {
 		}
 	}
 
-	RandomIntGenerator(unsigned int only = 0) : min(only), max(only) {}
+	explicit RandomIntGenerator(unsigned int only = 0) : min(only), max(only) {}
 	RandomIntGenerator(unsigned int first, unsigned int last, bool skewTowardFirst = false) : min(first), max(last) {
 		if (first != last && skewTowardFirst) {
 			skew = (first < last) ? SMALL : LARGE;
@@ -77,9 +77,9 @@ struct RandomIntGenerator : IGenerator<unsigned int> {
 			std::swap(min, max);
 		}
 	}
-	RandomIntGenerator(const char* cstr) : RandomIntGenerator(std::string(cstr)) {}
-	RandomIntGenerator(std::string str) : RandomIntGenerator(StringRef(str)) {}
-	RandomIntGenerator(StringRef str) {
+	explicit(false) RandomIntGenerator(const char* cstr) : RandomIntGenerator(std::string(cstr)) {}
+	explicit(false) RandomIntGenerator(std::string str) : RandomIntGenerator(StringRef(str)) {}
+	explicit(false) RandomIntGenerator(StringRef str) {
 		bool skewTowardFirst = false;
 		if (!str.empty() && str[0] == '^') {
 			skewTowardFirst = true;
@@ -143,9 +143,9 @@ struct RandomIntGenerator : IGenerator<unsigned int> {
 struct RandomStringGenerator : IKeyGenerator {
 	RandomStringGenerator() {}
 	RandomStringGenerator(RandomIntGenerator size, RandomIntGenerator byteset) : size(size), bytes(byteset) {}
-	RandomStringGenerator(const char* cstr) : RandomStringGenerator(std::string(cstr)) {}
-	RandomStringGenerator(std::string str) : RandomStringGenerator(StringRef(str)) {}
-	RandomStringGenerator(StringRef str) {
+	explicit(false) RandomStringGenerator(const char* cstr) : RandomStringGenerator(std::string(cstr)) {}
+	explicit(false) RandomStringGenerator(std::string str) : RandomStringGenerator(StringRef(str)) {}
+	explicit(false) RandomStringGenerator(StringRef str) {
 		StringRef sSize = str.eat("/");
 		StringRef sBytes = str;
 		if (sBytes.size() == 0) {
@@ -179,7 +179,7 @@ struct RandomStringGenerator : IKeyGenerator {
 // It uses a large pre-generated string and generates random substrings from it.
 struct RandomValueGenerator {
 	template <typename... Args>
-	RandomValueGenerator(Args&&... args) : strings(std::forward<Args>(args)...) {
+	explicit RandomValueGenerator(Args&&... args) : strings(std::forward<Args>(args)...) {
 		// Make a similar RandomStringGenerator to generate the noise block from
 		noise = RandomStringGenerator(RandomIntGenerator(std::max<int>(2e6, strings.size.max)), strings.bytes).next();
 	}
@@ -282,11 +282,11 @@ struct RandomStringSetGenerator : public RandomStringSetGeneratorBase {
 	  : indexGen(indexGen), stringGen(stringGen) {
 		init(indexGen, stringGen);
 	}
-	RandomStringSetGenerator(const char* cstr) : RandomStringSetGenerator(std::string(cstr)) {}
-	RandomStringSetGenerator(std::string str) : RandomStringSetGenerator(StringRef(str)) {}
-	RandomStringSetGenerator(StringRef str) {
+	explicit(false) RandomStringSetGenerator(const char* cstr) : RandomStringSetGenerator(std::string(cstr)) {}
+	explicit(false) RandomStringSetGenerator(std::string str) : RandomStringSetGenerator(StringRef(str)) {}
+	explicit(false) RandomStringSetGenerator(StringRef str) {
 		indexGen = str.eat("::");
-		stringGen = str;
+		stringGen = StringGenT(str);
 		init(indexGen, stringGen);
 	}
 
@@ -302,9 +302,9 @@ typedef RandomStringSetGenerator<RandomStringGenerator> RandomKeySetGenerator;
 // String Definition Format: RandomKeySet[,RandomKeySet]...
 struct RandomKeyTupleGenerator : public IKeyGenerator {
 	RandomKeyTupleGenerator() {};
-	RandomKeyTupleGenerator(std::vector<RandomKeySetGenerator> tupleParts) : tuples(tupleParts) {}
-	RandomKeyTupleGenerator(std::string s) : RandomKeyTupleGenerator(StringRef(s)) {}
-	RandomKeyTupleGenerator(StringRef s) {
+	explicit RandomKeyTupleGenerator(std::vector<RandomKeySetGenerator> tupleParts) : tuples(tupleParts) {}
+	explicit RandomKeyTupleGenerator(std::string s) : RandomKeyTupleGenerator(StringRef(s)) {}
+	explicit RandomKeyTupleGenerator(StringRef s) {
 		while (!s.empty()) {
 			tuples.push_back(s.eat(","));
 		}
diff --git a/fdbclient/include/fdbclient/ReadYourWrites.h b/fdbclient/include/fdbclient/ReadYourWrites.h
index e17a7d2aa82..ea55ce94cce 100644
--- a/fdbclient/include/fdbclient/ReadYourWrites.h
+++ b/fdbclient/include/fdbclient/ReadYourWrites.h
@@ -147,7 +147,7 @@ class ReadYourWritesTransaction final : NonCopyable,
 	// These are to permit use as state variables in actors:
 	ReadYourWritesTransaction() : cache(&arena), writes(&arena) {}
 	void operator=(ReadYourWritesTransaction&& r) noexcept;
-	ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept;
+	explicit(false) ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept;
 
 	void cancel();
 	void reset();
@@ -205,7 +205,8 @@ class ReadYourWritesTransaction final : NonCopyable,
 	template <typename Type>
 	using FutureT = Future<Type>;
 
-	virtual void debugTrace(BaseTraceEvent&& event);
+	// not virtual because final class
+	void debugTrace(BaseTraceEvent&& event);
 	void debugPrint(std::string const& message);
 
 	// Used by ThreadSafeTransaction for exceptions thrown in void methods.
@@ -267,4 +268,6 @@ class ReadYourWritesTransaction final : NonCopyable,
 	ReadYourWritesTransactionOptions options;
 };
 
+Future<Optional<Value>> getJSON(Database db, std::string jsonField = "");
+
 #endif
diff --git a/fdbclient/include/fdbclient/RunRYWTransaction.actor.h b/fdbclient/include/fdbclient/RunRYWTransaction.actor.h
deleted file mode 100644
index bd05b8e5248..00000000000
--- a/fdbclient/include/fdbclient/RunRYWTransaction.actor.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * RunRYWTransaction.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_RUNRYWTRANSACTION_ACTOR_G_H)
-#define FDBCLIENT_RUNRYWTRANSACTION_ACTOR_G_H
-#include "fdbclient/RunRYWTransaction.actor.g.h"
-#elif !defined(FDBCLIENT_RUNRYWTRANSACTION_ACTOR_H)
-#define FDBCLIENT_RUNRYWTRANSACTION_ACTOR_H
-
-#include <utility>
-
-#include "flow/flow.h"
-#include "fdbclient/RunTransaction.actor.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// Runs a RYW transaction in a retry loop on the given Database.
-//
-// Takes a function func that accepts a Reference<ReadYourWritesTransaction> as a parameter and returns a non-Void
-// Future. This function is run inside the transaction, and when the transaction is successfully committed the result of
-// the function is returned.
-//
-// The supplied function should be idempotent. Otherwise, outcome of this function will depend on how many times the
-// transaction is retried.
-ACTOR template <class Function>
-Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())> runRYWTransaction(
-    Database cx,
-    Function func) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	loop {
-		try {
-			state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result =
-			    wait(func(tr));
-			wait(tr->commit());
-			return result;
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-}
-
-// Debug version of runRYWTransaction. It logs the function name and the committed version of the transaction.
-// Note the function name is required, e.g., taskFunc->getName() for TaskFuncBase.
-ACTOR template <class Function>
-Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())>
-runRYWTransactionDebug(Database cx, StringRef name, Function func) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	loop {
-		try {
-			// func should be idempodent; otherwise, retry will get undefined result
-			state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result =
-			    wait(func(tr));
-			wait(tr->commit());
-			TraceEvent("DebugRunRYWTransaction")
-			    .detail("Function", name)
-			    .detail("CommitVersion", tr->getCommittedVersion());
-			return result;
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-}
-
-// Runs a RYW transaction in a retry loop on the given Database.
-//
-// Takes a function func that accepts a Reference<ReadYourWritesTransaction> as a parameter and returns a Void
-// Future. This function is run inside the transaction.
-//
-// The supplied function should be idempotent. Otherwise, outcome of this function will depend on how many times the
-// transaction is retried.
-ACTOR template <class Function>
-Future<Void> runRYWTransactionVoid(Database cx, Function func) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	loop {
-		try {
-			wait(func(tr));
-			wait(tr->commit());
-			return Void();
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-}
-
-ACTOR template <class Function>
-Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())>
-runRYWTransactionFailIfLocked(Database cx, Function func) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	loop {
-		try {
-			state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result =
-			    wait(func(tr));
-			wait(tr->commit());
-			return result;
-		} catch (Error& e) {
-			if (e.code() == error_code_database_locked)
-				throw;
-			wait(tr->onError(e));
-		}
-	}
-}
-
-ACTOR template <class Function>
-Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())> runRYWTransactionNoRetry(
-    Database cx,
-    Function func) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result = wait(func(tr));
-	wait(tr->commit());
-	return result;
-}
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/RunRYWTransaction.h b/fdbclient/include/fdbclient/RunRYWTransaction.h
new file mode 100644
index 00000000000..9b9bca3a78a
--- /dev/null
+++ b/fdbclient/include/fdbclient/RunRYWTransaction.h
@@ -0,0 +1,137 @@
+/*
+ * RunRYWTransaction.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <utility>
+
+#include "flow/flow.h"
+#include "fdbclient/RunTransaction.h"
+#include "fdbclient/ReadYourWrites.h"
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsubobject-linkage"
+#endif
+
+template <class Function>
+using RunRYWTransactionResult = decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue());
+
+// Runs a RYW transaction in a retry loop on the given Database.
+//
+// Takes a function func that accepts a Reference<ReadYourWritesTransaction> as a parameter and returns a non-Void
+// Future. This function is run inside the transaction, and when the transaction is successfully committed the result of
+// the function is returned.
+//
+// The supplied function should be idempotent. Otherwise, outcome of this function will depend on how many times the
+// transaction is retried.
+template <class Function>
+Future<RunRYWTransactionResult<Function>> runRYWTransaction(Database cx, Function func, ExplicitVoid = {}) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	while (true) {
+		Error err;
+		try {
+			auto const result = co_await func(tr);
+			co_await tr->commit();
+			co_return result;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr->onError(err);
+	}
+}
+
+// Debug version of runRYWTransaction. It logs the function name and the committed version of the transaction.
+// Note the function name is required, e.g., taskFunc->getName() for TaskFuncBase.
+template <class Function>
+Future<RunRYWTransactionResult<Function>> runRYWTransactionDebug(Database cx,
+                                                                 StringRef name,
+                                                                 Function func,
+                                                                 ExplicitVoid = {}) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	while (true) {
+		Error err;
+		try {
+			// func should be idempodent; otherwise, retry will get undefined result
+			auto const result = co_await func(tr);
+			co_await tr->commit();
+			TraceEvent("DebugRunRYWTransaction")
+			    .detail("Function", name)
+			    .detail("CommitVersion", tr->getCommittedVersion());
+			co_return result;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr->onError(err);
+	}
+}
+
+// Runs a RYW transaction in a retry loop on the given Database.
+//
+// Takes a function func that accepts a Reference<ReadYourWritesTransaction> as a parameter and returns a Void
+// Future. This function is run inside the transaction.
+//
+// The supplied function should be idempotent. Otherwise, outcome of this function will depend on how many times the
+// transaction is retried.
+template <class Function>
+Future<Void> runRYWTransactionVoid(Database cx, Function func) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	while (true) {
+		Error err;
+		try {
+			co_await func(tr);
+			co_await tr->commit();
+			co_return;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr->onError(err);
+	}
+}
+
+template <class Function>
+Future<RunRYWTransactionResult<Function>> runRYWTransactionFailIfLocked(Database cx, Function func, ExplicitVoid = {}) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	while (true) {
+		Error err;
+		try {
+			auto const result = co_await func(tr);
+			co_await tr->commit();
+			co_return result;
+		} catch (Error& e) {
+			err = e;
+		}
+		if (err.code() == error_code_database_locked)
+			throw err;
+		co_await tr->onError(err);
+	}
+}
+
+template <class Function>
+Future<RunRYWTransactionResult<Function>> runRYWTransactionNoRetry(Database cx, Function func, ExplicitVoid = {}) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	auto const result = co_await func(tr);
+	co_await tr->commit();
+	co_return result;
+}
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
diff --git a/fdbclient/include/fdbclient/RunTransaction.actor.h b/fdbclient/include/fdbclient/RunTransaction.h
similarity index 69%
rename from fdbclient/include/fdbclient/RunTransaction.actor.h
rename to fdbclient/include/fdbclient/RunTransaction.h
index d4d81cc9164..a9b99fe7897 100644
--- a/fdbclient/include/fdbclient/RunTransaction.actor.h
+++ b/fdbclient/include/fdbclient/RunTransaction.h
@@ -1,5 +1,5 @@
 /*
- * RunTransaction.actor.h
+ * RunTransaction.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,19 +20,15 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_RUNTRANSACTION_ACTOR_G_H)
-#define FDBCLIENT_RUNTRANSACTION_ACTOR_G_H
-#include "fdbclient/RunTransaction.actor.g.h"
-#elif !defined(FDBCLIENT_RUNTRANSACTION_ACTOR_H)
-#define FDBCLIENT_RUNTRANSACTION_ACTOR_H
-
 #include <utility>
 
 #include "flow/flow.h"
 #include "fdbclient/FDBOptions.g.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsubobject-linkage"
+#endif
 
 template <typename, typename = void>
 struct transaction_option_setter : std::false_type {};
@@ -43,43 +39,47 @@ struct transaction_option_setter<Reference<T>> : transaction_option_setter<T> {}
 template <typename T>
 constexpr bool can_set_transaction_options = transaction_option_setter<T>::value;
 
-ACTOR template <class Function, class DB>
-Future<decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue())> runTransaction(
-    Reference<DB> db,
-    Function func) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
-	loop {
+template <class Function, class DB>
+using RunTransactionResult = decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue());
+
+template <class Function, class DB>
+Future<RunTransactionResult<Function, DB>> runTransaction(Reference<DB> db, Function func, ExplicitVoid = {}) {
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
+	while (true) {
 		if constexpr (can_set_transaction_options<DB>) {
 			db->setOptions(tr);
 		}
 
+		Error err;
 		try {
 			// func should be idempotent; otherwise, retry will get undefined result
-			state decltype(std::declval<Function>()(Reference<typename DB::TransactionT>()).getValue()) result =
-			    wait(func(tr));
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return result;
+			auto const result = co_await func(tr);
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return result;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR template <class Function, class DB>
+template <class Function, class DB>
 Future<Void> runTransactionVoid(Reference<DB> db, Function func) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
-	loop {
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
+	while (true) {
 		if constexpr (can_set_transaction_options<DB>) {
 			db->setOptions(tr);
 		}
+		Error err;
 		try {
 			// func should be idempotent; otherwise, retry will get undefined result
-			wait(func(tr));
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return Void();
+			co_await func(tr);
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -129,8 +129,12 @@ auto SystemDB(Reference<DB> db, bool write = false, bool lockAware = false, bool
 // SystemDB with all options true
 template <typename DB>
 auto SystemDBWriteLockedNow(Reference<DB> db) {
-	return makeReference<SystemTransactionGenerator<DB>>(db, true, true, true);
+	constexpr bool write = true;
+	constexpr bool lockAware = true;
+	constexpr bool immediate = true;
+	return makeReference<SystemTransactionGenerator<DB>>(db, write, lockAware, immediate);
 }
 
-#include "flow/unactorcompiler.h"
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
 #endif
diff --git a/fdbclient/include/fdbclient/S3BlobStore.h b/fdbclient/include/fdbclient/S3BlobStore.h
index 3faa0c18967..abe52746202 100644
--- a/fdbclient/include/fdbclient/S3BlobStore.h
+++ b/fdbclient/include/fdbclient/S3BlobStore.h
@@ -21,119 +21,15 @@
 #pragma once
 
 #include <map>
-#include <unordered_map>
 #include <functional>
-#include <string>
-#include "flow/IRandom.h"
-#include "flow/flow.h"
 #include "flow/Net2Packet.h"
-#include "fdbclient/Knobs.h"
-#include "flow/IRateControl.h"
+#include "fdbclient/IBlobStore.h"
 #include "fdbrpc/HTTP.h"
-#include "fdbrpc/Stats.h"
-#include "fdbclient/JSONDoc.h"
-#include "flow/IConnection.h"
 
-#include <boost/functional/hash.hpp>
-
-// unique key that identifies interchangeable connections for the same settings and destination
-// FIXME: can we define std::hash as a struct member of a S3BlobStoreEndpoint?
-struct BlobStoreConnectionPoolKey {
-	std::string host;
-	std::string service;
-	std::string region;
-	bool isTLS;
-
-	BlobStoreConnectionPoolKey(const std::string& host,
-	                           const std::string& service,
-	                           const std::string& region,
-	                           bool isTLS)
-	  : host(host), service(service), region(region), isTLS(isTLS) {}
-
-	bool operator==(const BlobStoreConnectionPoolKey& other) const {
-		return isTLS == other.isTLS && host == other.host && service == other.service && region == other.region;
-	}
-};
-
-namespace std {
-template <>
-struct hash<BlobStoreConnectionPoolKey> {
-	std::size_t operator()(const BlobStoreConnectionPoolKey& key) const {
-		std::size_t seed = 0;
-		boost::hash_combine(seed, std::hash<std::string>{}(key.host));
-		boost::hash_combine(seed, std::hash<std::string>{}(key.service));
-		boost::hash_combine(seed, std::hash<std::string>{}(key.region));
-		boost::hash_combine(seed, std::hash<bool>{}(key.isTLS));
-		return seed;
-	}
-};
-} // namespace std
-
-// Representation of all the things you need to connect to a blob store instance with some credentials.
-// Reference counted because a very large number of them could be needed.
-class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
+class S3BlobStoreEndpoint : public IBlobStoreEndpoint, ReferenceCounted<S3BlobStoreEndpoint> {
 public:
-	struct Stats {
-		Stats() : requests_successful(0), requests_failed(0), bytes_sent(0) {}
-		Stats operator-(const Stats& rhs);
-		void clear() { memset(this, 0, sizeof(*this)); }
-		json_spirit::mObject getJSON();
-
-		int64_t requests_successful;
-		int64_t requests_failed;
-		int64_t bytes_sent;
-	};
-
-	static Stats s_stats;
-
-	struct BlobStats {
-		UID id;
-		CounterCollection cc;
-		Counter requestsSuccessful;
-		Counter requestsFailed;
-		Counter newConnections;
-		Counter expiredConnections;
-		Counter reusedConnections;
-		Counter fastRetries;
-
-		LatencySample requestLatency;
-
-		// init not in static codepath, to avoid initialization race issues and so no blob connections means no
-		// unnecessary blob stats traces
-		BlobStats()
-		  : id(deterministicRandom()->randomUniqueID()), cc("BlobStoreStats", id.toString()),
-		    requestsSuccessful("RequestsSuccessful", cc), requestsFailed("RequestsFailed", cc),
-		    newConnections("NewConnections", cc), expiredConnections("ExpiredConnections", cc),
-		    reusedConnections("ReusedConnections", cc), fastRetries("FastRetries", cc),
-		    requestLatency("BlobStoreRequestLatency",
-		                   id,
-		                   CLIENT_KNOBS->BLOBSTORE_LATENCY_LOGGING_INTERVAL,
-		                   CLIENT_KNOBS->BLOBSTORE_LATENCY_LOGGING_ACCURACY) {}
-	};
-	// null when initialized, so no blob stats until a blob connection is used
-	std::unique_ptr<BlobStats> blobStats;
-	Future<Void> statsLogger;
-
-	void maybeStartStatsLogger() {
-		if (!blobStats && CLIENT_KNOBS->BLOBSTORE_ENABLE_LOGGING) {
-			blobStats = std::make_unique<BlobStats>();
-			specialCounter(blobStats->cc, "GlobalConnectionPoolCount", [this]() {
-				return this->getGlobalConnectionPool().size();
-			});
-			specialCounter(blobStats->cc, "GlobalConnectionPoolSize", [this]() {
-				// FIXME: could track this explicitly via an int variable with extra logic, but this should be small and
-				// infrequent
-				int totalConnections = 0;
-				for (auto& it : this->getGlobalConnectionPool()) {
-					totalConnections += it.second->pool.size();
-				}
-				return totalConnections;
-			});
-
-			statsLogger = blobStats->cc.traceCounters(
-			    "BlobStoreMetrics", blobStats->id, CLIENT_KNOBS->BLOBSTORE_STATS_LOGGING_INTERVAL, "BlobStoreMetrics");
-		}
-	}
+	void addref() override { ReferenceCounted<S3BlobStoreEndpoint>::addref(); }
+	void delref() override { ReferenceCounted<S3BlobStoreEndpoint>::delref(); }
 
 	struct Credentials {
 		std::string key;
@@ -141,235 +37,50 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 		std::string securityToken;
 	};
 
-	struct BlobKnobs {
-		BlobKnobs();
-		int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min,
-		    requests_per_second, list_requests_per_second, write_requests_per_second, read_requests_per_second,
-		    delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
-		    concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file,
-		    enable_read_cache, read_block_size, read_ahead_blocks, read_cache_blocks_per_file,
-		    max_send_bytes_per_second, max_recv_bytes_per_second, sdk_auth, enable_object_integrity_check,
-		    global_connection_pool, max_delay_retryable_error, max_delay_connection_failed, multipart_retry_delay_ms;
-
-		bool set(StringRef name, int value);
-		std::string getURLParameters() const;
-		static std::vector<std::string> getKnobDescriptions() {
-			return {
-				"secure_connection (or sc)             Set 1 for secure connection and 0 for insecure connection.",
-				"connect_tries (or ct)                 Number of times to try to connect for each request.",
-				"connect_timeout (or cto)              Number of seconds to wait for a connect request to succeed.",
-				"max_connection_life (or mcl)          Maximum number of seconds to use a single TCP connection.",
-				"request_tries (or rt)                 Number of times to try each request until a parsable HTTP "
-				"response other than 429 is received.",
-				"request_timeout_min (or rtom)         Number of seconds to wait for a request to succeed after a "
-				"connection is established.",
-				"requests_per_second (or rps)          Max number of requests to start per second.",
-				"list_requests_per_second (or lrps)    Max number of list requests to start per second.",
-				"write_requests_per_second (or wrps)   Max number of write requests to start per second.",
-				"read_requests_per_second (or rrps)    Max number of read requests to start per second.",
-				"delete_requests_per_second (or drps)  Max number of delete requests to start per second.",
-				"multipart_max_part_size (or maxps)    Max part size for multipart uploads.",
-				"multipart_min_part_size (or minps)    Min part size for multipart uploads.",
-				"multipart_retry_delay_ms (or mrd)     Delay in milliseconds between retries for multipart uploads.",
-				"concurrent_requests (or cr)           Max number of total requests in progress at once, regardless of "
-				"operation-specific concurrency limits.",
-				"concurrent_uploads (or cu)            Max concurrent uploads (part or whole) that can be in progress "
-				"at once.",
-				"concurrent_lists (or cl)              Max concurrent list operations that can be in progress at once.",
-				"concurrent_reads_per_file (or crps)   Max concurrent reads in progress for any one file.",
-				"concurrent_writes_per_file (or cwps)  Max concurrent uploads in progress for any one file.",
-				"enable_read_cache (or erc)            Whether read block caching is enabled.",
-				"read_block_size (or rbs)              Block size in bytes to be used for reads.",
-				"read_ahead_blocks (or rab)            Number of blocks to read ahead of requested offset.",
-				"read_cache_blocks_per_file (or rcb)   Size of the read cache for a file in blocks.",
-				"max_send_bytes_per_second (or sbps)   Max send bytes per second for all requests combined.",
-				"max_recv_bytes_per_second (or rbps)   Max receive bytes per second for all requests combined (NOT YET "
-				"USED).",
-				"max_delay_retryable_error (or dre)    Max seconds to delay before retry when see a retryable error.",
-				"max_delay_connection_failed (or dcf)  Max seconds to delay before retry when see a connection "
-				"failure.",
-				"sdk_auth (or sa)                      Use AWS SDK to resolve credentials. Only valid if "
-				"BUILD_AWS_BACKUP is enabled.",
-				"enable_object_integrity_check (or eoic) Enable integrity check on GET requests (Default: false).",
-				"global_connection_pool (or gcp)       Enable shared connection pool between all blobstore instances."
-			};
-		}
-
-		bool isTLS() const { return secure_connection == 1; }
-	};
-
-	struct ReusableConnection {
-		Reference<IConnection> conn;
-		double expirationTime;
-		// CROSS_PROCESS_FIX: Track which process created this connection
-		NetworkAddress creatingProcess;
-		ReusableConnection() : expirationTime(0) {
-			if (g_network && g_network->isSimulated()) {
-				creatingProcess = g_network->getLocalAddress();
-			}
-		}
-		ReusableConnection(Reference<IConnection> c, double exp) : conn(c), expirationTime(exp) {
-			if (g_network && g_network->isSimulated()) {
-				creatingProcess = g_network->getLocalAddress();
-			}
-		}
-
-		// CROSS_PROCESS_FIX: Copy constructor with cross-process detection
-		ReusableConnection(const ReusableConnection& other)
-		  : conn(other.conn), expirationTime(other.expirationTime), creatingProcess(other.creatingProcess) {
-			if (g_network && g_network->isSimulated() && creatingProcess.isValid() &&
-			    creatingProcess != g_network->getLocalAddress()) {
-				// Cross-process copy detected - invalidate the connection to prevent sharing
-				conn = Reference<IConnection>();
-				expirationTime = 0; // Mark as expired
-			}
-		}
-
-		// CROSS_PROCESS_FIX: Assignment operator with cross-process detection
-		ReusableConnection& operator=(const ReusableConnection& other) {
-			if (this != &other) {
-				conn = other.conn;
-				expirationTime = other.expirationTime;
-				creatingProcess = other.creatingProcess;
-				if (g_network && g_network->isSimulated() && creatingProcess.isValid() &&
-				    creatingProcess != g_network->getLocalAddress()) {
-					// Cross-process assignment detected - invalidate the connection to prevent sharing
-					conn = Reference<IConnection>();
-					expirationTime = 0; // Mark as expired
-				}
-			}
-			return *this;
-		}
-	};
-
-	// basically, reference counted queue with option to add other fields
-	struct ConnectionPoolData : NonCopyable, ReferenceCounted<ConnectionPoolData> {
-		std::queue<ReusableConnection> pool;
-
-		ConnectionPoolData() {}
-
-		// Destructor implementation in S3BlobStore.actor.cpp
-		// In simulation, explicitly closes all pooled connections before destruction
-		~ConnectionPoolData();
-	};
-
-	// global connection pool for multiple blobstore endpoints with same connection settings and request destination
-	// NOTE: This is disabled by default (BLOBSTORE_GLOBAL_CONNECTION_POOL=false), so each endpoint gets its own pool
-	static std::unordered_map<BlobStoreConnectionPoolKey, Reference<ConnectionPoolData>>& getGlobalConnectionPool() {
-		// Use process address as key to separate connection pools per simulated process
-		static std::map<NetworkAddress, std::unordered_map<BlobStoreConnectionPoolKey, Reference<ConnectionPoolData>>>
-		    processConnectionPools;
-
-		NetworkAddress currentProcess;
-		if (g_network && g_network->isSimulated()) {
-			currentProcess = g_network->getLocalAddress();
-		}
-		return processConnectionPools[currentProcess];
-	}
-
 	S3BlobStoreEndpoint(std::string const& host,
 	                    std::string const& service,
 	                    std::string region,
 	                    Optional<std::string> const& proxyHost,
 	                    Optional<std::string> const& proxyPort,
-	                    Optional<Credentials> const& creds,
+	                    Optional<StringRef> const& creds,
 	                    BlobKnobs const& knobs = BlobKnobs(),
-	                    HTTP::Headers extraHeaders = HTTP::Headers())
-	  : host(host), service(service), region(region), proxyHost(proxyHost), proxyPort(proxyPort),
-	    useProxy(proxyHost.present() && proxyPort.present()), credentials(creds),
-	    lookupKey(creds.present() && creds.get().key.empty()),
-	    lookupSecret(creds.present() && creds.get().secret.empty()), knobs(knobs), extraHeaders(extraHeaders),
-	    requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
-	    requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)),
-	    requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)),
-	    requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)),
-	    requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)),
-	    sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
-	    recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests),
-	    concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) {
-
-		if (host.empty() || (proxyHost.present() != proxyPort.present()))
-			throw connection_string_invalid();
+	                    HTTP::Headers extraHeaders = HTTP::Headers());
 
-		// set connection pool instance
-		if (useProxy || !knobs.global_connection_pool) {
-			// don't use global connection pool if there's a proxy, as it complicates the logic
-			// FIXME: handle proxies?
-			connectionPool = makeReference<ConnectionPoolData>();
-		} else {
-			BlobStoreConnectionPoolKey key(host, service, region, knobs.isTLS());
-			auto it = getGlobalConnectionPool().find(key);
-			if (it != getGlobalConnectionPool().end()) {
-				connectionPool = it->second;
-			} else {
-				connectionPool = makeReference<ConnectionPoolData>();
-				getGlobalConnectionPool().insert({ key, connectionPool });
-			}
-		}
-		ASSERT(connectionPool.isValid());
-
-		maybeStartStatsLogger();
-	}
-
-	static std::string getURLFormat(bool withResource = false) {
-		const char* resource = "";
-		if (withResource)
-			resource = "<name>";
-		return format("blobstore://<api_key>:<secret>:<security_token>@<host>[:<port>]/"
-		              "%s[?<param>=<value>[&<param>=<value>]...]",
-		              resource);
-	}
+	// Infer the cloud region from a hostname by matching known service prefixes
+	// (e.g. "s3.us-west-2.amazonaws.com" -> "us-west-2", "cos.ap-beijing.myqcloud.com" -> "ap-beijing").
+	// Returns "" if no pattern matches.
+	static std::string guessRegionFromDomain(std::string domain);
 
-	typedef std::map<std::string, std::string> ParametersT;
-
-	// Parse url and return a S3BlobStoreEndpoint
-	// If the url has parameters that S3BlobStoreEndpoint can't consume then an error will be thrown unless
-	// ignored_parameters is given in which case the unconsumed parameters will be added to it.
+	// Convenience: parse URL and return as S3BlobStoreEndpoint (downcasts from IBlobStoreEndpoint::fromString).
+	// Callers that only need the IBlobStoreEndpoint interface should use IBlobStoreEndpoint::fromString directly.
 	static Reference<S3BlobStoreEndpoint> fromString(const std::string& url,
 	                                                 const Optional<std::string>& proxy,
 	                                                 std::string* resourceFromURL,
 	                                                 std::string* error,
-	                                                 ParametersT* ignored_parameters);
-
-	// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL
-	// parameters in addition to the passed params string
-	std::string getResourceURL(std::string resource, std::string params) const;
-
-	// Construct a resource path for S3 operations
-	std::string constructResourcePath(const std::string& bucket, const std::string& object) const;
-
-	// FIXME: add periodic connection reaper to pool
-	// local connection pool for this blobstore
-	Reference<ConnectionPoolData> connectionPool;
-	Future<ReusableConnection> connect(bool* reusingConn);
-	void returnConnection(ReusableConnection& conn);
+	                                                 ParametersT* ignored_parameters) {
+		Reference<IBlobStoreEndpoint> base =
+		    IBlobStoreEndpoint::fromString(url, proxy, resourceFromURL, error, ignored_parameters);
+		auto* endpoint = dynamic_cast<S3BlobStoreEndpoint*>(base.getPtr());
+		if (base && !endpoint) {
+			if (error) {
+				*error = "Parsed blob store endpoint is not an S3BlobStoreEndpoint";
+			}
+			return Reference<S3BlobStoreEndpoint>();
+		}
+		return Reference<S3BlobStoreEndpoint>::addRef(endpoint);
+	}
 
-	std::string host;
-	std::string service;
-	std::string region;
-	Optional<std::string> proxyHost;
-	Optional<std::string> proxyPort;
-	bool useProxy;
+	Future<Void> updateSecret() override;
+	bool extractCredentialFields(JSONDoc& account) override;
+	std::string credentialFileKey() const override;
+	bool lookupSecretOnEachRequest() override;
+	void setRequestHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) override;
+	std::string normalizeResourceForRequest(std::string const& resource) override;
 
 	Optional<Credentials> credentials;
 	bool lookupKey;
 	bool lookupSecret;
-	BlobKnobs knobs;
-	HTTP::Headers extraHeaders;
-
-	// Speed and concurrency limits
-	Reference<IRateControl> requestRate;
-	Reference<IRateControl> requestRateList;
-	Reference<IRateControl> requestRateWrite;
-	Reference<IRateControl> requestRateRead;
-	Reference<IRateControl> requestRateDelete;
-	Reference<IRateControl> sendRate;
-	Reference<IRateControl> recvRate;
-	FlowLock concurrentRequests;
-	FlowLock concurrentUploads;
-	FlowLock concurrentLists;
-
-	Future<Void> updateSecret();
+	bool simulatedTokenError = false; // Set by simulateRequestFailure for BUGGIFY testing
 
 	// Calculates the authentication string from the secret key
 	static std::string hmac_sha1(Credentials const& creds, std::string const& msg);
@@ -384,95 +95,73 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 	                      std::string date = "",
 	                      std::string datestamp = "");
 
-	std::string getHost() const { return host; }
-
-	std::string getRegion() const { return region; }
-
-	// Do an HTTP request to the Blob Store, read the response.  Handles authentication.
-	// Every blob store interaction should ultimately go through this function
+	// doRequest hooks for S3 token error recovery
+	void simulateRequestFailure(std::string const& verb,
+	                            std::string const& resource,
+	                            Reference<HTTP::IncomingResponse>& r) override;
+	void processRequestFailure(Reference<HTTP::IncomingResponse> const& r,
+	                           TraceEvent& event,
+	                           bool& retryExtended) override;
+	Future<bool> preRetryCheck(std::string const& verb,
+	                           std::string const& resource,
+	                           ReusableConnection& rconn,
+	                           int requestTimeout,
+	                           bool& retryExtended) override;
 
-	Future<Reference<HTTP::IncomingResponse>> doRequest(std::string const& verb,
-	                                                    std::string const& resource,
-	                                                    const HTTP::Headers& headers,
-	                                                    UnsentPacketQueue* pContent,
-	                                                    int contentLen,
-	                                                    std::set<unsigned int> successCodes);
-
-	struct ObjectInfo {
-		std::string name;
-		int64_t size;
-	};
+	// Get a normalized version of this URL with the given resource and any non-default BlobKnob values as URL
+	// parameters in addition to the passed params string
+	std::string getResourceURL(std::string resource, std::string params) const override;
 
-	struct ListResult {
-		std::vector<std::string> commonPrefixes;
-		std::vector<ObjectInfo> objects;
-	};
+	// Construct a resource path for S3 operations
+	std::string constructResourcePath(const std::string& bucket, const std::string& object) const;
 
-	// Get bucket contents via a stream, since listing large buckets will take many serial blob requests
+	// Get bucket contents via a stream, since listing large buckets will take many serial blob requests.
 	// If a delimiter is passed then common prefixes will be read in parallel, recursively, depending on recurseFilter.
-	// Recursefilter is a must be a function that takes a string and returns true if it passes.  The default behavior is
+	// recurseFilter must be a function that takes a string and returns true if it passes.  The default behavior is
 	// to assume true.
 	Future<Void> listObjectsStream(std::string const& bucket,
 	                               PromiseStream<ListResult> results,
 	                               Optional<std::string> prefix = {},
 	                               Optional<char> delimiter = {},
 	                               int maxDepth = 0,
-	                               std::function<bool(std::string const&)> recurseFilter = nullptr);
-
-	// Get a list of the files in a bucket, see listObjectsStream for more argument detail.
-	Future<ListResult> listObjects(std::string const& bucket,
-	                               Optional<std::string> prefix = {},
-	                               Optional<char> delimiter = {},
-	                               int maxDepth = 0,
-	                               std::function<bool(std::string const&)> recurseFilter = nullptr);
+	                               std::function<bool(std::string const&)> recurseFilter = nullptr) override;
 
 	// Get a list of all buckets
-	Future<std::vector<std::string>> listBuckets();
+	AsyncResult<std::vector<std::string>> listBuckets() override;
 
 	// Check if a bucket exists
-	Future<bool> bucketExists(std::string const& bucket);
+	Future<bool> bucketExists(std::string const& bucket) override;
 
 	// Check if an object exists in a bucket
-	Future<bool> objectExists(std::string const& bucket, std::string const& object);
+	Future<bool> objectExists(std::string const& bucket, std::string const& object) override;
 
 	// Get the size of an object in a bucket
-	Future<int64_t> objectSize(std::string const& bucket, std::string const& object);
+	Future<int64_t> objectSize(std::string const& bucket, std::string const& object) override;
 
 	// Read an arbitrary segment of an object
 	Future<int> readObject(std::string const& bucket,
 	                       std::string const& object,
 	                       void* data,
 	                       int length,
-	                       int64_t offset);
+	                       int64_t offset) override;
 
 	// Delete an object in a bucket
-	Future<Void> deleteObject(std::string const& bucket, std::string const& object);
-
-	// Delete all objects in a bucket under a prefix.  Note this is not atomic as blob store does not
-	// support this operation directly. This method is just a convenience method that lists and deletes
-	// all of the objects in the bucket under the given prefix.
-	// Since it can take a while, if a pNumDeleted and/or pBytesDeleted are provided they will be incremented every time
-	// a deletion of an object completes.
-	Future<Void> deleteRecursively(std::string const& bucket,
-	                               std::string prefix = "",
-	                               int* pNumDeleted = nullptr,
-	                               int64_t* pBytesDeleted = nullptr);
+	Future<Void> deleteObject(std::string const& bucket, std::string const& object) override;
 
-	// Create a bucket if it does not already exists.
-	Future<Void> createBucket(std::string const& bucket);
+	// Create a bucket if it does not already exist.
+	Future<Void> createBucket(std::string const& bucket) override;
 
 	// Useful methods for working with tiny files
-	Future<std::string> readEntireFile(std::string const& bucket, std::string const& object);
-	Future<Void> writeEntireFile(std::string const& bucket, std::string const& object, std::string const& content);
+	AsyncResult<std::string> readEntireFile(std::string const& bucket, std::string const& object) override;
 	Future<Void> writeEntireFileFromBuffer(std::string const& bucket,
 	                                       std::string const& object,
 	                                       UnsentPacketQueue* pContent,
 	                                       int contentLen,
-	                                       std::string const& contentHash);
+	                                       std::string const& contentHash) override;
 
 	// MultiPart upload methods
 	// Returns UploadID
-	Future<std::string> beginMultiPartUpload(std::string const& bucket, std::string const& object);
+	Future<std::string> beginMultiPartUpload(std::string const& bucket, std::string const& object) override;
 	// Returns eTag
 	Future<std::string> uploadPart(std::string const& bucket,
 	                               std::string const& object,
@@ -480,19 +169,13 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 	                               unsigned int partNumber,
 	                               UnsentPacketQueue* pContent,
 	                               int contentLen,
-	                               std::string const& contentHash);
-
-	struct PartInfo {
-		std::string etag;
-		std::string checksum; // MD5 or SHA256 depending on integrity check setting
-		PartInfo() = default;
-		PartInfo(std::string e, std::string c = "") : etag(e), checksum(c) {}
-	};
-	typedef std::map<int, PartInfo> MultiPartSetT;
+	                               std::string const& contentHash) override;
 	Future<Optional<std::string>> finishMultiPartUpload(std::string const& bucket,
 	                                                    std::string const& object,
 	                                                    std::string const& uploadID,
-	                                                    MultiPartSetT const& parts);
+	                                                    MultiPartSetT const& parts,
+	                                                    int64_t totalSize = 0) override;
+
 	Future<Void> abortMultiPartUpload(std::string const& bucket,
 	                                  std::string const& object,
 	                                  std::string const& uploadID);
diff --git a/fdbclient/include/fdbclient/S3Client.actor.h b/fdbclient/include/fdbclient/S3Client.h
similarity index 81%
rename from fdbclient/include/fdbclient/S3Client.actor.h
rename to fdbclient/include/fdbclient/S3Client.h
index d24a62f24e2..076e1314a0e 100644
--- a/fdbclient/include/fdbclient/S3Client.actor.h
+++ b/fdbclient/include/fdbclient/S3Client.h
@@ -1,5 +1,5 @@
 /*
- * S3Client.actor.h
+ * S3Client.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,17 +20,10 @@
 
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_S3CLIENT_ACTOR_G_H)
-#define FDBCLIENT_S3CLIENT_ACTOR_G_H
-#include "fdbclient/S3Client.actor.g.h"
-#elif !defined(FDBCLIENT_S3CLIENT_ACTOR_H)
-#define FDBCLIENT_S3CLIENT_ACTOR_H
-
 #include <string>
 #include "fdbclient/S3BlobStore.h"
 #include "fdbclient/BulkDumping.h"
 #include "flow/Error.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // FDB S3 Client. Includes copying files and directories to and from s3.
 // Uses the S3BlobStoreEndpoint to interact with s3. The s3url is of the form
@@ -81,28 +74,28 @@ const std::string BLOBSTORE_PREFIX = "blobstore://";
 // dirpath: Local directory path to copy from
 // s3url: S3 URL to copy to (must include bucket parameter)
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> copyUpDirectory(std::string dirpath, std::string s3url);
+Future<Void> copyUpDirectory(std::string dirpath, std::string s3url);
 
 // Copy filepath to bucket at resource in s3.
 // Uses multipart upload for large files.
 // filepath: Local file path to copy from
 // s3url: S3 URL to copy to (must include bucket parameter)
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> copyUpFile(std::string filepath, std::string s3url);
+Future<Void> copyUpFile(std::string filepath, std::string s3url);
 
 // Copy the file from s3 down to the local filesystem.
 // Overwrites existing file. Uses multipart download for large files.
 // s3url: S3 URL to copy from (must include bucket parameter)
 // filepath: Local file path to copy to
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> copyDownFile(std::string s3url, std::string filepath);
+Future<Void> copyDownFile(std::string s3url, std::string filepath);
 
 // Copy down the directory content from s3 to the local filesystem.
 // Recursively copies all files and subdirectories.
 // s3url: S3 URL to copy from (must include bucket parameter)
 // dirpath: Local directory path to copy to
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> copyDownDirectory(std::string s3url, std::string dirpath);
+Future<Void> copyDownDirectory(std::string s3url, std::string dirpath);
 
 // Upload the source file set after clearing any existing files at the destination.
 // Used for bulk operations like backup and restore.
@@ -110,27 +103,27 @@ ACTOR Future<Void> copyDownDirectory(std::string s3url, std::string dirpath);
 // sourceFileSet: Source file set to copy
 // destinationFileSet: Destination file set to create
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> copyUpBulkDumpFileSet(std::string s3url,
-                                         BulkLoadFileSet sourceFileSet,
-                                         BulkLoadFileSet destinationFileSet);
+Future<Void> copyUpBulkDumpFileSet(std::string s3url,
+                                   BulkLoadFileSet sourceFileSet,
+                                   BulkLoadFileSet destinationFileSet);
 
 // Delete the file or directory at s3url -- recursively.
 // s3url: S3 URL to delete (must include bucket parameter)
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> deleteResource(std::string s3url);
+Future<Void> deleteResource(std::string s3url);
 
 // Calculate the xxhash64 checksum of a file.
 // Used for verifying data integrity during uploads and downloads.
 // file: File handle to calculate checksum for
 // size: Size of the file in bytes, or -1 to determine automatically
-// Returns a Future that completes with the hex string representation of the checksum
-ACTOR Future<std::string> calculateFileChecksum(Reference<IAsyncFile> file, int64_t size = -1);
+// Returns an AsyncResult that completes with the hex string representation of the checksum
+AsyncResult<std::string> calculateFileChecksum(Reference<IAsyncFile> file, int64_t size = -1);
 
 // List files and directories at the given S3 URL
 // s3url: S3 URL to list (must include bucket parameter)
 // maxDepth: Maximum depth to recurse (default: 1)
 // Returns a Future that completes when the operation is done
-ACTOR Future<Void> listFiles(std::string s3url, int maxDepth = 1);
+Future<Void> listFiles(std::string s3url, int maxDepth = 1);
 
 // Get the endpoint for the given s3url.
 // Populates parameters and resource with parse of s3url.
@@ -141,6 +134,3 @@ ACTOR Future<Void> listFiles(std::string s3url, int maxDepth = 1);
 Reference<S3BlobStoreEndpoint> getEndpoint(const std::string& s3url,
                                            std::string& resource,
                                            S3BlobStoreEndpoint::ParametersT& parameters);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/ServerKnobCollection.h b/fdbclient/include/fdbclient/ServerKnobCollection.h
deleted file mode 100644
index 5177f4b312b..00000000000
--- a/fdbclient/include/fdbclient/ServerKnobCollection.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * ServerKnobCollection.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBCLIENT_SERVERKNOBCOLLECTION_H
-#define FDBCLIENT_SERVERKNOBCOLLECTION_H
-
-#pragma once
-
-#include "fdbclient/ClientKnobCollection.h"
-#include "fdbclient/IKnobCollection.h"
-
-/*
- * Stores both flow knobs, client knobs, and server knobs. Attempting to access test knobs
- * results in a run-time error
- */
-class ServerKnobCollection : public IKnobCollection {
-	ClientKnobCollection clientKnobCollection;
-	ServerKnobs serverKnobs;
-
-public:
-	ServerKnobCollection(Randomize randomize, IsSimulated isSimulated);
-	void initialize(Randomize randomize, IsSimulated isSimulated) override;
-	void reset(Randomize randomize, IsSimulated isSimulated) override;
-	FlowKnobs const& getFlowKnobs() const override { return clientKnobCollection.getFlowKnobs(); }
-	ClientKnobs const& getClientKnobs() const override { return clientKnobCollection.getClientKnobs(); }
-	ServerKnobs const& getServerKnobs() const override { return serverKnobs; }
-	TestKnobs const& getTestKnobs() const override { throw internal_error(); }
-	Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const override;
-	bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
-};
-
-#endif // FDBCLIENT_SERVERKNOBCOLLECTION_H
diff --git a/fdbclient/include/fdbclient/SimpleIni.h b/fdbclient/include/fdbclient/SimpleIni.h
index b0f75fe7591..9ba1caf1ff7 100644
--- a/fdbclient/include/fdbclient/SimpleIni.h
+++ b/fdbclient/include/fdbclient/SimpleIni.h
@@ -297,10 +297,11 @@ class CSimpleIniTempl {
 		const SI_CHAR* pComment;
 		int nOrder;
 
-		Entry(const SI_CHAR* a_pszItem = NULL, int a_nOrder = 0) : pItem(a_pszItem), pComment(NULL), nOrder(a_nOrder) {}
+		explicit(false) Entry(const SI_CHAR* a_pszItem = NULL, int a_nOrder = 0)
+		  : pItem(a_pszItem), pComment(NULL), nOrder(a_nOrder) {}
 		Entry(const SI_CHAR* a_pszItem, const SI_CHAR* a_pszComment, int a_nOrder)
 		  : pItem(a_pszItem), pComment(a_pszComment), nOrder(a_nOrder) {}
-		Entry(const Entry& rhs) { operator=(rhs); }
+		explicit(false) Entry(const Entry& rhs) { operator=(rhs); }
 		Entry& operator=(const Entry& rhs) {
 			pItem = rhs.pItem;
 			pComment = rhs.pComment;
@@ -354,7 +355,7 @@ class CSimpleIniTempl {
 		virtual void Write(const char* a_pBuf) = 0;
 
 	private:
-		OutputWriter(const OutputWriter&); // disable
+		explicit(false) OutputWriter(const OutputWriter&); // disable
 		OutputWriter& operator=(const OutputWriter&); // disable
 	};
 
@@ -363,11 +364,11 @@ class CSimpleIniTempl {
 		FILE* m_file;
 
 	public:
-		FileWriter(FILE* a_file) : m_file(a_file) {}
+		explicit FileWriter(FILE* a_file) : m_file(a_file) {}
 		void Write(const char* a_pBuf) { fputs(a_pBuf, m_file); }
 
 	private:
-		FileWriter(const FileWriter&); // disable
+		explicit(false) FileWriter(const FileWriter&); // disable
 		FileWriter& operator=(const FileWriter&); // disable
 	};
 
@@ -376,11 +377,11 @@ class CSimpleIniTempl {
 		std::string& m_string;
 
 	public:
-		StringWriter(std::string& a_string) : m_string(a_string) {}
+		explicit StringWriter(std::string& a_string) : m_string(a_string) {}
 		void Write(const char* a_pBuf) { m_string.append(a_pBuf); }
 
 	private:
-		StringWriter(const StringWriter&); // disable
+		explicit(false) StringWriter(const StringWriter&); // disable
 		StringWriter& operator=(const StringWriter&); // disable
 	};
 
@@ -390,11 +391,11 @@ class CSimpleIniTempl {
 		std::ostream& m_ostream;
 
 	public:
-		StreamWriter(std::ostream& a_ostream) : m_ostream(a_ostream) {}
+		explicit StreamWriter(std::ostream& a_ostream) : m_ostream(a_ostream) {}
 		void Write(const char* a_pBuf) { m_ostream << a_pBuf; }
 
 	private:
-		StreamWriter(const StreamWriter&); // disable
+		explicit(false) StreamWriter(const StreamWriter&); // disable
 		StreamWriter& operator=(const StreamWriter&); // disable
 	};
 #endif // SI_SUPPORT_IOSTREAMS
@@ -405,8 +406,8 @@ class CSimpleIniTempl {
 	class Converter : private SI_CONVERTER {
 	public:
 		using SI_CONVERTER::SizeToStore;
-		Converter(bool a_bStoreIsUtf8) : SI_CONVERTER(a_bStoreIsUtf8) { m_scratch.resize(1024); }
-		Converter(const Converter& rhs) { operator=(rhs); }
+		explicit Converter(bool a_bStoreIsUtf8) : SI_CONVERTER(a_bStoreIsUtf8) { m_scratch.resize(1024); }
+		explicit(false) Converter(const Converter& rhs) { operator=(rhs); }
 		Converter& operator=(const Converter& rhs) {
 			m_scratch = rhs.m_scratch;
 			return *this;
@@ -436,7 +437,7 @@ class CSimpleIniTempl {
 	    @param a_bMultiKey   See the method SetMultiKey() for details.
 	    @param a_bMultiLine  See the method SetMultiLine() for details.
 	 */
-	CSimpleIniTempl(bool a_bIsUtf8 = false, bool a_bMultiKey = false, bool a_bMultiLine = false);
+	explicit CSimpleIniTempl(bool a_bIsUtf8 = false, bool a_bMultiKey = false, bool a_bMultiLine = false);
 
 	/** Destructor */
 	~CSimpleIniTempl();
@@ -1011,7 +1012,7 @@ class CSimpleIniTempl {
 
 private:
 	// copying is not permitted
-	CSimpleIniTempl(const CSimpleIniTempl&); // disabled
+	explicit(false) CSimpleIniTempl(const CSimpleIniTempl&); // disabled
 	CSimpleIniTempl& operator=(const CSimpleIniTempl&); // disabled
 
 	/** Parse the data looking for a file comment and store it if found.
@@ -1862,9 +1863,9 @@ SI_Error CSimpleIniTempl<SI_CHAR, SI_STRLESS, SI_CONVERTER>::SetLongValue(const
 	// convert to an ASCII string
 	char szInput[64];
 #if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
-	sprintf_s(szInput, a_bUseHex ? "0x%lx" : "%ld", a_nValue);
+	snprintf_s(szInput, sizeof(szInput), a_bUseHex ? "0x%lx" : "%ld", a_nValue);
 #else // !__STDC_WANT_SECURE_LIB__
-	sprintf(szInput, a_bUseHex ? "0x%lx" : "%ld", a_nValue);
+	snprintf(szInput, sizeof(szInput), a_bUseHex ? "0x%lx" : "%ld", a_nValue);
 #endif // __STDC_WANT_SECURE_LIB__
 
 	// convert to output text
@@ -1914,13 +1915,8 @@ SI_Error CSimpleIniTempl<SI_CHAR, SI_STRLESS, SI_CONVERTER>::SetDoubleValue(cons
 	if (!a_pSection || !a_pKey)
 		return SI_FAIL;
 
-	// convert to an ASCII string
 	char szInput[64];
-#if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
-	sprintf_s(szInput, "%f", a_nValue);
-#else // !__STDC_WANT_SECURE_LIB__
-	sprintf(szInput, "%f", a_nValue);
-#endif // __STDC_WANT_SECURE_LIB__
+	snprintf(szInput, sizeof(szInput), "%f", a_nValue);
 
 	// convert to output text
 	SI_CHAR szOutput[64];
@@ -2436,10 +2432,10 @@ class SI_ConvertA {
 	SI_ConvertA() {}
 
 public:
-	SI_ConvertA(bool a_bStoreIsUtf8) : m_bStoreIsUtf8(a_bStoreIsUtf8) {}
+	explicit SI_ConvertA(bool a_bStoreIsUtf8) : m_bStoreIsUtf8(a_bStoreIsUtf8) {}
 
 	/* copy and assignment */
-	SI_ConvertA(const SI_ConvertA& rhs) { operator=(rhs); }
+	explicit(false) SI_ConvertA(const SI_ConvertA& rhs) { operator=(rhs); }
 	SI_ConvertA& operator=(const SI_ConvertA& rhs) {
 		m_bStoreIsUtf8 = rhs.m_bStoreIsUtf8;
 		return *this;
@@ -2555,10 +2551,10 @@ class SI_ConvertW {
 	SI_ConvertW() {}
 
 public:
-	SI_ConvertW(bool a_bStoreIsUtf8) : m_bStoreIsUtf8(a_bStoreIsUtf8) {}
+	explicit SI_ConvertW(bool a_bStoreIsUtf8) : m_bStoreIsUtf8(a_bStoreIsUtf8) {}
 
 	/* copy and assignment */
-	SI_ConvertW(const SI_ConvertW& rhs) { operator=(rhs); }
+	explicit(false) SI_ConvertW(const SI_ConvertW& rhs) { operator=(rhs); }
 	SI_ConvertW& operator=(const SI_ConvertW& rhs) {
 		m_bStoreIsUtf8 = rhs.m_bStoreIsUtf8;
 		return *this;
@@ -2736,10 +2732,10 @@ class SI_ConvertW {
 	SI_ConvertW() : m_pEncoding(NULL), m_pConverter(NULL) {}
 
 public:
-	SI_ConvertW(bool a_bStoreIsUtf8) : m_pConverter(NULL) { m_pEncoding = a_bStoreIsUtf8 ? "UTF-8" : NULL; }
+	explicit SI_ConvertW(bool a_bStoreIsUtf8) : m_pConverter(NULL) { m_pEncoding = a_bStoreIsUtf8 ? "UTF-8" : NULL; }
 
 	/* copy and assignment */
-	SI_ConvertW(const SI_ConvertW& rhs) { operator=(rhs); }
+	explicit(false) SI_ConvertW(const SI_ConvertW& rhs) { operator=(rhs); }
 	SI_ConvertW& operator=(const SI_ConvertW& rhs) {
 		m_pEncoding = rhs.m_pEncoding;
 		m_pConverter = NULL;
@@ -2943,10 +2939,10 @@ class SI_ConvertW {
 	SI_ConvertW() {}
 
 public:
-	SI_ConvertW(bool a_bStoreIsUtf8) { m_uCodePage = a_bStoreIsUtf8 ? CP_UTF8 : CP_ACP; }
+	explicit SI_ConvertW(bool a_bStoreIsUtf8) { m_uCodePage = a_bStoreIsUtf8 ? CP_UTF8 : CP_ACP; }
 
 	/* copy and assignment */
-	SI_ConvertW(const SI_ConvertW& rhs) { operator=(rhs); }
+	explicit(false) SI_ConvertW(const SI_ConvertW& rhs) { operator=(rhs); }
 	SI_ConvertW& operator=(const SI_ConvertW& rhs) {
 		m_uCodePage = rhs.m_uCodePage;
 		return *this;
diff --git a/fdbclient/include/fdbclient/SnapshotCache.h b/fdbclient/include/fdbclient/SnapshotCache.h
index d6b9b90c642..e64ed5a23e0 100644
--- a/fdbclient/include/fdbclient/SnapshotCache.h
+++ b/fdbclient/include/fdbclient/SnapshotCache.h
@@ -29,7 +29,8 @@
 
 struct ExtStringRef {
 	ExtStringRef() : extra_zero_bytes(0) {}
-	ExtStringRef(StringRef const& s, int extra_zero_bytes = 0) : base(s), extra_zero_bytes(extra_zero_bytes) {}
+	explicit(false) ExtStringRef(StringRef const& s, int extra_zero_bytes = 0)
+	  : base(s), extra_zero_bytes(extra_zero_bytes) {}
 
 	Standalone<StringRef> toStandaloneStringRef() const {
 		auto s = makeString(size());
@@ -180,7 +181,7 @@ class SnapshotCache {
 		// Note that an uncached range stops at the next individual key that is known, even though we might want to read
 		// through that key.  In RYWIterator, it might also stop at a dependent write which is not known at all!
 
-		iterator(SnapshotCache* cache) : parent(cache), it(cache->entries.begin()), offset(0) {
+		explicit iterator(SnapshotCache* cache) : parent(cache), it(cache->entries.begin()), offset(0) {
 			//++*this;  // gives begin
 		}
 
@@ -311,7 +312,7 @@ class SnapshotCache {
 		entries.insert(Entry(allKeys.end, afterAllKeys, VectorRef<KeyValueRef>()), NoMetric(), true);
 	}
 	// Visual Studio refuses to generate these, apparently despite the standard
-	SnapshotCache(SnapshotCache&& r) noexcept : arena(r.arena), entries(std::move(r.entries)) {}
+	explicit(false) SnapshotCache(SnapshotCache&& r) noexcept : arena(r.arena), entries(std::move(r.entries)) {}
 	SnapshotCache& operator=(SnapshotCache&& r) noexcept {
 		entries = std::move(r.entries);
 		arena = r.arena;
diff --git a/fdbclient/include/fdbclient/SpecialKeySpace.actor.h b/fdbclient/include/fdbclient/SpecialKeySpace.h
similarity index 90%
rename from fdbclient/include/fdbclient/SpecialKeySpace.actor.h
rename to fdbclient/include/fdbclient/SpecialKeySpace.h
index 340f22dbb13..7dd0715b776 100644
--- a/fdbclient/include/fdbclient/SpecialKeySpace.actor.h
+++ b/fdbclient/include/fdbclient/SpecialKeySpace.h
@@ -1,5 +1,5 @@
 /*
- * SpecialKeySpace.actor.h
+ * SpecialKeySpace.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,18 +20,11 @@
 
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_SPECIALKEYSPACE_ACTOR_G_H)
-#define FDBCLIENT_SPECIALKEYSPACE_ACTOR_G_H
-#include "fdbclient/SpecialKeySpace.actor.g.h"
-#elif !defined(FDBCLIENT_SPECIALKEYSPACE_ACTOR_H)
-#define FDBCLIENT_SPECIALKEYSPACE_ACTOR_H
-
 #include "flow/flow.h"
 #include "flow/Arena.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 class SpecialKeyRangeReadImpl {
 public:
@@ -131,11 +124,11 @@ class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {
 
 	bool isAsync() const override { return true; }
 
-	ACTOR static Future<RangeResult> getRangeAsyncActor(const SpecialKeyRangeReadImpl* skrAyncImpl,
-	                                                    ReadYourWritesTransaction* ryw,
-	                                                    KeyRangeRef kr,
-	                                                    GetRangeLimits limits,
-	                                                    KeyRangeMap<Optional<RangeResult>>* cache) {
+	static Future<RangeResult> getRangeAsyncActor(const SpecialKeyRangeReadImpl* skrAyncImpl,
+	                                              ReadYourWritesTransaction* ryw,
+	                                              KeyRangeRef kr,
+	                                              GetRangeLimits limits,
+	                                              KeyRangeMap<Optional<RangeResult>>* cache) {
 		ASSERT(skrAyncImpl->getKeyRange().contains(kr));
 		ASSERT(cache != nullptr);
 		ASSERT(cache->rangeContaining(kr.begin) == cache->rangeContainingKeyBefore(kr.end));
@@ -143,7 +136,7 @@ class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {
 			// For simplicity, every time we need to cache, we read the whole range
 			// Although sometimes the range can be narrowed,
 			// there is not a general way to do it in complicated scenarios
-			RangeResult result_ = wait(skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange(), limits));
+			RangeResult result_ = co_await skrAyncImpl->getRange(ryw, skrAyncImpl->getKeyRange(), limits);
 			cache->insert(skrAyncImpl->getKeyRange(), result_);
 		}
 		const auto& allResults = (*cache)[kr.begin].get();
@@ -155,9 +148,9 @@ class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {
 		if (start < end) {
 			RangeResult result = RangeResultRef(allResults.slice(start, end), false);
 			result.arena().dependsOn(allResults.arena());
-			return result;
-		} else
-			return RangeResult();
+			co_return result;
+		}
+		co_return RangeResult();
 	}
 };
 
@@ -187,7 +180,7 @@ class SpecialKeySpace {
 		READWRITE // The underlying special key range can be called with get, getRange, set, clear
 	};
 
-	SpecialKeySpace(KeyRef spaceStartKey = Key(), KeyRef spaceEndKey = normalKeys.end, bool testOnly = true);
+	explicit SpecialKeySpace(KeyRef spaceStartKey = Key(), KeyRef spaceEndKey = normalKeys.end, bool testOnly = true);
 
 	Future<Optional<Value>> get(ReadYourWritesTransaction* ryw, const Key& key);
 
@@ -235,20 +228,20 @@ class SpecialKeySpace {
 	static const std::set<std::string>& getTracingOptions() { return tracingOptions; }
 
 private:
-	ACTOR static Future<Optional<Value>> getActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeyRef key);
-
-	ACTOR static Future<RangeResult> checkRYWValid(SpecialKeySpace* sks,
-	                                               ReadYourWritesTransaction* ryw,
-	                                               KeySelector begin,
-	                                               KeySelector end,
-	                                               GetRangeLimits limits,
-	                                               Reverse reverse);
-	ACTOR static Future<RangeResult> getRangeAggregationActor(SpecialKeySpace* sks,
-	                                                          ReadYourWritesTransaction* ryw,
-	                                                          KeySelector begin,
-	                                                          KeySelector end,
-	                                                          GetRangeLimits limits,
-	                                                          Reverse reverse);
+	static Future<Optional<Value>> getActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeyRef key);
+
+	static Future<RangeResult> checkRYWValid(SpecialKeySpace* sks,
+	                                         ReadYourWritesTransaction* ryw,
+	                                         KeySelector begin,
+	                                         KeySelector end,
+	                                         GetRangeLimits limits,
+	                                         Reverse reverse);
+	static Future<RangeResult> getRangeAggregationActor(SpecialKeySpace* sks,
+	                                                    ReadYourWritesTransaction* ryw,
+	                                                    KeySelector begin,
+	                                                    KeySelector end,
+	                                                    GetRangeLimits limits,
+	                                                    Reverse reverse);
 
 	KeyRangeMap<SpecialKeyRangeReadImpl*> readImpls;
 	KeyRangeMap<SpecialKeySpace::MODULE> modules;
@@ -568,12 +561,9 @@ class FaultToleranceMetricsImpl : public SpecialKeyRangeReadImpl {
 // If the underlying set of key-value pairs of a key space is not changing, then we expect repeating a read to give the
 // same result. Additionally, we can generate the expected result of any read if that read is reading a subrange. This
 // actor performs a read of an arbitrary subrange of [begin, end) and validates the results.
-ACTOR Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
-                                               KeySelector begin,
-                                               KeySelector end,
-                                               GetRangeLimits limits,
-                                               Reverse reverse,
-                                               RangeResult result);
-
-#include "flow/unactorcompiler.h"
-#endif
+Future<Void> validateSpecialSubrangeRead(ReadYourWritesTransaction* ryw,
+                                         KeySelector begin,
+                                         KeySelector end,
+                                         GetRangeLimits limits,
+                                         Reverse reverse,
+                                         RangeResult result);
diff --git a/fdbclient/include/fdbclient/Status.h b/fdbclient/include/fdbclient/Status.h
index 405dd9a8407..c998d7d61bb 100644
--- a/fdbclient/include/fdbclient/Status.h
+++ b/fdbclient/include/fdbclient/Status.h
@@ -34,7 +34,7 @@ struct StatusObject : json_spirit::mObject {
 	typedef json_spirit::mArray Array;
 
 	StatusObject() {}
-	StatusObject(json_spirit::mObject const& o) : json_spirit::mObject(o) {}
+	explicit(false) StatusObject(json_spirit::mObject const& o) : json_spirit::mObject(o) {}
 };
 
 template <class Ar>
@@ -61,12 +61,12 @@ void save(Ar& ar, StatusObject const& statusObj) {
 
 struct StatusArray : json_spirit::mArray {
 	StatusArray() {}
-	StatusArray(json_spirit::mArray const& o) : json_spirit::mArray(o.begin(), o.end()) {}
+	explicit(false) StatusArray(json_spirit::mArray const& o) : json_spirit::mArray(o.begin(), o.end()) {}
 };
 
 struct StatusValue : json_spirit::mValue {
 	StatusValue() {}
-	StatusValue(json_spirit::mValue const& o) : json_spirit::mValue(o) {}
+	explicit(false) StatusValue(json_spirit::mValue const& o) : json_spirit::mValue(o) {}
 };
 
 enum class MessageType {
diff --git a/fdbclient/include/fdbclient/StatusClient.h b/fdbclient/include/fdbclient/StatusClient.h
index b9176d55a53..ccd64b911e1 100644
--- a/fdbclient/include/fdbclient/StatusClient.h
+++ b/fdbclient/include/fdbclient/StatusClient.h
@@ -36,7 +36,7 @@ class StatusClient {
 	// only.
 	// @out status json
 	// @note we don't have support to fetch arbitrary status json fields at this point.
-	static Future<StatusObject> statusFetcher(Database db, std::string statusField = "");
+	static AsyncResult<StatusObject> statusFetcher(Database db, std::string statusField = "");
 };
 
 #endif
diff --git a/fdbclient/include/fdbclient/StorageCheckpoint.h b/fdbclient/include/fdbclient/StorageCheckpoint.h
index 8ed71aab797..28ba9d37bed 100644
--- a/fdbclient/include/fdbclient/StorageCheckpoint.h
+++ b/fdbclient/include/fdbclient/StorageCheckpoint.h
@@ -187,7 +187,7 @@ struct DataMoveMetaData {
 	DataMoveMetaData(UID id, KeyRange range) : id(id), version(invalidVersion), priority(0), mode(0) {
 		this->ranges.push_back(range);
 	}
-	DataMoveMetaData(UID id) : id(id), version(invalidVersion), priority(0), mode(0) {}
+	explicit DataMoveMetaData(UID id) : id(id), version(invalidVersion), priority(0), mode(0) {}
 
 	Phase getPhase() const { return static_cast<Phase>(phase); }
 
diff --git a/fdbclient/include/fdbclient/StorageServerInterface.h b/fdbclient/include/fdbclient/StorageServerInterface.h
index 230a2d63b42..9be4826d51a 100644
--- a/fdbclient/include/fdbclient/StorageServerInterface.h
+++ b/fdbclient/include/fdbclient/StorageServerInterface.h
@@ -36,7 +36,7 @@
 #include "fdbrpc/TimedRequest.h"
 #include "fdbrpc/TSSComparison.h"
 #include "fdbclient/CommitTransaction.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TagThrottle.h"
 #include "fdbclient/Tracing.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/VersionVector.h"
@@ -129,6 +129,7 @@ struct StorageServerInterface {
 	RequestStream<struct BulkDumpRequest> bulkdump;
 
 private:
+	void initEndpointsFromGetValue();
 	bool acceptingRequests;
 
 public:
@@ -156,82 +157,12 @@ struct StorageServerInterface {
 		serializer(ar, uniqueID, locality, getValue, tssPairID, acceptingRequests);
 
 		if (Ar::isDeserializing) {
-			getKey = PublicRequestStream<struct GetKeyRequest>(getValue.getEndpoint().getAdjustedEndpoint(1));
-			getKeyValues =
-			    PublicRequestStream<struct GetKeyValuesRequest>(getValue.getEndpoint().getAdjustedEndpoint(2));
-			getShardState = RequestStream<struct GetShardStateRequest>(getValue.getEndpoint().getAdjustedEndpoint(3));
-			waitMetrics = PublicRequestStream<struct WaitMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(4));
-			splitMetrics = RequestStream<struct SplitMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(5));
-			getStorageMetrics =
-			    RequestStream<struct GetStorageMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(6));
-			waitFailure = RequestStream<ReplyPromise<Void>>(getValue.getEndpoint().getAdjustedEndpoint(7));
-			getQueuingMetrics =
-			    RequestStream<struct StorageQueuingMetricsRequest>(getValue.getEndpoint().getAdjustedEndpoint(8));
-			getKeyValueStoreType =
-			    RequestStream<ReplyPromise<KeyValueStoreType>>(getValue.getEndpoint().getAdjustedEndpoint(9));
-			watchValue = PublicRequestStream<struct WatchValueRequest>(getValue.getEndpoint().getAdjustedEndpoint(10));
-			getReadHotRanges =
-			    RequestStream<struct ReadHotSubRangeRequest>(getValue.getEndpoint().getAdjustedEndpoint(11));
-			getRangeSplitPoints =
-			    RequestStream<struct SplitRangeRequest>(getValue.getEndpoint().getAdjustedEndpoint(12));
-			getKeyValuesStream =
-			    PublicRequestStream<struct GetKeyValuesStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(13));
-			getMappedKeyValues =
-			    PublicRequestStream<struct GetMappedKeyValuesRequest>(getValue.getEndpoint().getAdjustedEndpoint(14));
-			changeFeedStream =
-			    RequestStream<struct ChangeFeedStreamRequest>(getValue.getEndpoint().getAdjustedEndpoint(15));
-			overlappingChangeFeeds =
-			    RequestStream<struct OverlappingChangeFeedsRequest>(getValue.getEndpoint().getAdjustedEndpoint(16));
-			changeFeedPop = RequestStream<struct ChangeFeedPopRequest>(getValue.getEndpoint().getAdjustedEndpoint(17));
-			changeFeedVersionUpdate =
-			    RequestStream<struct ChangeFeedVersionUpdateRequest>(getValue.getEndpoint().getAdjustedEndpoint(18));
-			checkpoint = RequestStream<struct GetCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(19));
-			fetchCheckpoint =
-			    RequestStream<struct FetchCheckpointRequest>(getValue.getEndpoint().getAdjustedEndpoint(20));
-			fetchCheckpointKeyValues =
-			    RequestStream<struct FetchCheckpointKeyValuesRequest>(getValue.getEndpoint().getAdjustedEndpoint(21));
-			updateCommitCostRequest =
-			    RequestStream<struct UpdateCommitCostRequest>(getValue.getEndpoint().getAdjustedEndpoint(22));
-			auditStorage = RequestStream<struct AuditStorageRequest>(getValue.getEndpoint().getAdjustedEndpoint(23));
-			getHotShards = RequestStream<struct GetHotShardsRequest>(getValue.getEndpoint().getAdjustedEndpoint(24));
-			getCheckSum =
-			    RequestStream<struct GetStorageCheckSumRequest>(getValue.getEndpoint().getAdjustedEndpoint(25));
-			bulkdump = RequestStream<struct BulkDumpRequest>(getValue.getEndpoint().getAdjustedEndpoint(26));
+			initEndpointsFromGetValue();
 		}
 	}
 	bool operator==(StorageServerInterface const& s) const { return uniqueID == s.uniqueID; }
 	bool operator<(StorageServerInterface const& s) const { return uniqueID < s.uniqueID; }
-	void initEndpoints() {
-		std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
-		streams.push_back(getValue.getReceiver(TaskPriority::LoadBalancedEndpoint));
-		streams.push_back(getKey.getReceiver(TaskPriority::LoadBalancedEndpoint));
-		streams.push_back(getKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint));
-		streams.push_back(getShardState.getReceiver());
-		streams.push_back(waitMetrics.getReceiver());
-		streams.push_back(splitMetrics.getReceiver());
-		streams.push_back(getStorageMetrics.getReceiver());
-		streams.push_back(waitFailure.getReceiver());
-		streams.push_back(getQueuingMetrics.getReceiver());
-		streams.push_back(getKeyValueStoreType.getReceiver());
-		streams.push_back(watchValue.getReceiver());
-		streams.push_back(getReadHotRanges.getReceiver());
-		streams.push_back(getRangeSplitPoints.getReceiver());
-		streams.push_back(getKeyValuesStream.getReceiver(TaskPriority::LoadBalancedEndpoint));
-		streams.push_back(getMappedKeyValues.getReceiver(TaskPriority::LoadBalancedEndpoint));
-		streams.push_back(changeFeedStream.getReceiver());
-		streams.push_back(overlappingChangeFeeds.getReceiver());
-		streams.push_back(changeFeedPop.getReceiver());
-		streams.push_back(changeFeedVersionUpdate.getReceiver());
-		streams.push_back(checkpoint.getReceiver());
-		streams.push_back(fetchCheckpoint.getReceiver());
-		streams.push_back(fetchCheckpointKeyValues.getReceiver());
-		streams.push_back(updateCommitCostRequest.getReceiver());
-		streams.push_back(auditStorage.getReceiver());
-		streams.push_back(getHotShards.getReceiver());
-		streams.push_back(getCheckSum.getReceiver());
-		streams.push_back(bulkdump.getReceiver());
-		FlowTransport::transport().addEndpoints(streams);
-	}
+	void initEndpoints();
 };
 
 struct StorageInfo : NonCopyable, public ReferenceCounted<StorageInfo> {
@@ -243,8 +174,8 @@ struct StorageInfo : NonCopyable, public ReferenceCounted<StorageInfo> {
 struct StorageServerMetaInfo : public StorageServerInterface {
 	Optional<StorageMetadataType> metadata;
 
-	StorageServerMetaInfo(const StorageServerInterface& interface,
-	                      Optional<StorageMetadataType> metadata = Optional<StorageMetadataType>())
+	explicit StorageServerMetaInfo(const StorageServerInterface& interface,
+	                               Optional<StorageMetadataType> metadata = Optional<StorageMetadataType>())
 	  : StorageServerInterface(interface), metadata(metadata) {}
 };
 
@@ -475,7 +406,7 @@ struct GetKeyValuesStreamReply : public ReplyPromiseStreamReply {
 	bool cached = false;
 
 	GetKeyValuesStreamReply() : version(invalidVersion), more(false), cached(false) {}
-	GetKeyValuesStreamReply(GetKeyValuesReply r)
+	explicit GetKeyValuesStreamReply(GetKeyValuesReply r)
 	  : arena(r.arena), data(r.data), version(r.version), more(r.more), cached(r.cached) {}
 
 	int expectedSize() const { return sizeof(GetKeyValuesStreamReply) + data.expectedSize(); }
@@ -821,7 +752,7 @@ struct ReadHotSubRangeRequest {
 	int chunkCount = 1;
 
 	ReadHotSubRangeRequest() {}
-	ReadHotSubRangeRequest(KeyRangeRef const& keys, SplitType type = SplitType::BYTES, int chunkCount = 1)
+	explicit ReadHotSubRangeRequest(KeyRangeRef const& keys, SplitType type = SplitType::BYTES, int chunkCount = 1)
 	  : keys(arena, keys), type(type), chunkCount(chunkCount) {}
 
 	template <class Ar>
@@ -971,7 +902,7 @@ struct FetchCheckpointReply : public ReplyPromiseStreamReply {
 	Standalone<StringRef> data;
 
 	FetchCheckpointReply() {}
-	FetchCheckpointReply(StringRef token) : token(token) {}
+	explicit FetchCheckpointReply(StringRef token) : token(token) {}
 
 	int expectedSize() const { return data.expectedSize(); }
 
diff --git a/fdbclient/include/fdbclient/StorageWiggleMetrics.actor.h b/fdbclient/include/fdbclient/StorageWiggleMetrics.h
similarity index 90%
rename from fdbclient/include/fdbclient/StorageWiggleMetrics.actor.h
rename to fdbclient/include/fdbclient/StorageWiggleMetrics.h
index fdc919be6c0..c1f15763b2d 100644
--- a/fdbclient/include/fdbclient/StorageWiggleMetrics.actor.h
+++ b/fdbclient/include/fdbclient/StorageWiggleMetrics.h
@@ -1,5 +1,5 @@
 /*
- * StorageWiggleMetrics.actor.h
+ * StorageWiggleMetrics.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,19 +19,13 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_STORAGEWIGGLEMETRICS_ACTOR_G_H)
-#define FDBCLIENT_STORAGEWIGGLEMETRICS_ACTOR_G_H
-#include "fdbclient/StorageWiggleMetrics.actor.g.h"
-#elif !defined(FDBCLIENT_STORAGEWIGGLEMETRICS_ACTOR_H)
-#define FDBCLIENT_STORAGEWIGGLEMETRICS_ACTOR_H
 
 #include "fdbrpc/Smoother.h"
 #include "flow/ObjectSerializer.h"
 #include "flow/serialize.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/RunTransaction.actor.h"
-#include "flow/actorcompiler.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/RunTransaction.h"
 
 FDB_BOOLEAN_PARAM(PrimaryRegion);
 
@@ -118,22 +112,22 @@ struct StorageWiggleDelay {
 
 namespace {
 // Persistent the total delay time to the database, and return accumulated delay time.
-ACTOR template <class TrType>
+template <class TrType>
 Future<double> addPerpetualWiggleDelay_impl(
     TrType tr,
     KeyBackedObjectProperty<StorageWiggleDelay, decltype(IncludeVersion())> delayProperty,
     double secDelta) {
 
-	state StorageWiggleDelay delayObj = wait(delayProperty.getD(tr, Snapshot::False));
+	StorageWiggleDelay delayObj = co_await delayProperty.getD(tr, Snapshot::False);
 	delayObj.delaySeconds += secDelta;
 	delayProperty.set(tr, delayObj);
 
-	return delayObj.delaySeconds;
+	co_return delayObj.delaySeconds;
 }
 
 // set all fields except for smoothed durations to default values. If the metrics is not given, load from system key
 // space
-ACTOR template <class TrType>
+template <class TrType>
 Future<Void> resetStorageWiggleMetrics_impl(
     TrType tr,
     KeyBackedObjectProperty<StorageWiggleMetrics, decltype(IncludeVersion())> metricsProperty,
@@ -141,14 +135,13 @@ Future<Void> resetStorageWiggleMetrics_impl(
 	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 	if (!metrics.present()) {
-		wait(store(metrics, metricsProperty.get(tr)));
+		metrics = co_await metricsProperty.get(tr);
 	}
 
 	if (metrics.present()) {
 		metrics.get().reset();
 		metricsProperty.set(tr, metrics.get());
 	}
-	return Void();
 }
 } // namespace
 
@@ -159,7 +152,7 @@ class StorageWiggleData {
 
 public:
 	struct DataForDc : public KeyBackedClass {
-		DataForDc(StringRef prefix) : KeyBackedClass(prefix) {}
+		explicit DataForDc(StringRef prefix) : KeyBackedClass(prefix) {}
 
 		auto storageWiggleDelay() const {
 			auto key = subspace.pack("storageWiggleDelay"_sr);
@@ -216,7 +209,7 @@ class StorageWiggleData {
 		return resetStorageWiggleMetrics_impl(tr, storageWiggleMetrics(primary), metrics);
 	}
 
-	ACTOR template <typename TrType>
+	template <typename TrType>
 	static Future<Void> updateStorageWiggleMetrics_impl(
 	    KeyBackedProperty<Value, NullCodec> wiggleSpeed,
 	    KeyBackedObjectProperty<StorageWiggleMetrics, decltype(IncludeVersion())> storageMetrics,
@@ -225,13 +218,12 @@ class StorageWiggleData {
 	    PrimaryRegion primary) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		Optional<Value> v = wait(wiggleSpeed.get(tr));
+		Optional<Value> v = co_await wiggleSpeed.get(tr);
 		if (v.present() && v == "1"_sr) {
 			storageMetrics.set(tr, metrics);
 		} else {
 			CODE_PROBE(true, "Intend to update StorageWiggleMetrics after PW disabled");
 		}
-		return Void();
 	}
 
 	// update the serialized metrics when the perpetual wiggle is enabled
@@ -241,6 +233,3 @@ class StorageWiggleData {
 		    perpetualWiggleSpeed(), storageWiggleMetrics(primary), tr, metrics, primary);
 	}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
\ No newline at end of file
diff --git a/fdbclient/include/fdbclient/Subspace.h b/fdbclient/include/fdbclient/Subspace.h
index 2d68e0476c5..da62e2ee470 100644
--- a/fdbclient/include/fdbclient/Subspace.h
+++ b/fdbclient/include/fdbclient/Subspace.h
@@ -28,8 +28,8 @@
 
 class Subspace {
 public:
-	Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
-	Subspace(StringRef const& rawPrefix);
+	explicit Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
+	explicit(false) Subspace(StringRef const& rawPrefix);
 
 	virtual ~Subspace();
 
@@ -70,4 +70,4 @@ class Subspace {
 	Standalone<VectorRef<uint8_t>> rawPrefix;
 };
 
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbclient/include/fdbclient/SystemData.h b/fdbclient/include/fdbclient/SystemData.h
index 19d1029595b..e9e13f6d4e4 100644
--- a/fdbclient/include/fdbclient/SystemData.h
+++ b/fdbclient/include/fdbclient/SystemData.h
@@ -79,13 +79,9 @@ enum class DataMovementReason : uint8_t {
 // SystemKey is just a Key but with a special type so that instances of it can be found easily throughput the code base
 // and in simulation constructions will verify that no SystemKey is a direct prefix of any other.
 struct SystemKey : Key {
-	SystemKey(Key const& k);
+	explicit SystemKey(Key const& k);
 };
 
-struct RestoreLoaderInterface;
-struct RestoreApplierInterface;
-struct RestoreMasterInterface;
-
 extern const KeyRangeRef normalKeys; // '' to systemKeys.begin
 extern const KeyRangeRef systemKeys; // [FF] to [FF][FF]
 extern const KeyRangeRef nonMetadataSystemKeys; // [FF][00] to [FF][01]
@@ -109,16 +105,13 @@ extern const KeyRef keyServersPrefix, keyServersEnd, keyServersKeyServersKey;
 // temporary ID, until a permanent ID is assigned to it.
 extern const UID anonymousShardId;
 extern const uint64_t assignedEmptyShardId;
-const Key keyServersKey(const KeyRef& k);
-const KeyRef keyServersKey(const KeyRef& k, Arena& arena);
-const Value keyServersValue(RangeResult result,
-                            const std::vector<UID>& src,
-                            const std::vector<UID>& dest = std::vector<UID>());
-const Value keyServersValue(const std::vector<Tag>& srcTag, const std::vector<Tag>& destTag = std::vector<Tag>());
-const Value keyServersValue(const std::vector<UID>& src,
-                            const std::vector<UID>& dest,
-                            const UID& srcID,
-                            const UID& destID);
+Key keyServersKey(const KeyRef& k);
+KeyRef keyServersKey(const KeyRef& k, Arena& arena);
+Value keyServersValue(RangeResult result,
+                      const std::vector<UID>& src,
+                      const std::vector<UID>& dest = std::vector<UID>());
+Value keyServersValue(const std::vector<Tag>& srcTag, const std::vector<Tag>& destTag = std::vector<Tag>());
+Value keyServersValue(const std::vector<UID>& src, const std::vector<UID>& dest, const UID& srcID, const UID& destID);
 // `result` must be the full result of getting serverTagKeys
 void decodeKeyServersValue(RangeResult result,
                            const ValueRef& value,
@@ -139,7 +132,7 @@ void decodeKeyServersValue(RangeResult result,
 bool isSystemKey(KeyRef key);
 
 extern const KeyRef accumulativeChecksumKey;
-const Value accumulativeChecksumValue(const AccumulativeChecksumState& acsState);
+Value accumulativeChecksumValue(const AccumulativeChecksumState& acsState);
 AccumulativeChecksumState decodeAccumulativeChecksum(const ValueRef& value);
 
 extern const KeyRangeRef auditKeys;
@@ -148,34 +141,34 @@ extern const KeyRangeRef auditRanges;
 extern const KeyRef auditRangePrefix;
 
 // Key for a particular audit
-const Key auditKey(const AuditType type, const UID& auditId);
+Key auditKey(const AuditType type, const UID& auditId);
 // KeyRange for whole audit
-const KeyRange auditKeyRange(const AuditType type);
+KeyRange auditKeyRange(const AuditType type);
 // Prefix for audit work progress by range
-const Key auditRangeBasedProgressPrefixFor(const AuditType type, const UID& auditId);
+Key auditRangeBasedProgressPrefixFor(const AuditType type, const UID& auditId);
 // Range for audit work progress by range
-const KeyRange auditRangeBasedProgressRangeFor(const AuditType type, const UID& auditId);
-const KeyRange auditRangeBasedProgressRangeFor(const AuditType type);
+KeyRange auditRangeBasedProgressRangeFor(const AuditType type, const UID& auditId);
+KeyRange auditRangeBasedProgressRangeFor(const AuditType type);
 // Prefix for audit work progress by server
-const Key auditServerBasedProgressPrefixFor(const AuditType type, const UID& auditId, const UID& serverId);
+Key auditServerBasedProgressPrefixFor(const AuditType type, const UID& auditId, const UID& serverId);
 // Range for audit work progress by server
-const KeyRange auditServerBasedProgressRangeFor(const AuditType type, const UID& auditId);
-const KeyRange auditServerBasedProgressRangeFor(const AuditType type);
+KeyRange auditServerBasedProgressRangeFor(const AuditType type, const UID& auditId);
+KeyRange auditServerBasedProgressRangeFor(const AuditType type);
 
-const Value auditStorageStateValue(const AuditStorageState& auditStorageState);
+Value auditStorageStateValue(const AuditStorageState& auditStorageState);
 AuditStorageState decodeAuditStorageState(const ValueRef& value);
 
 // "\xff/checkpoint/[[UID]] := [[CheckpointMetaData]]"
 extern const KeyRef checkpointPrefix;
-const Key checkpointKeyFor(UID checkpointID);
-const Value checkpointValue(const CheckpointMetaData& checkpoint);
+Key checkpointKeyFor(UID checkpointID);
+Value checkpointValue(const CheckpointMetaData& checkpoint);
 UID decodeCheckpointKey(const KeyRef& key);
 CheckpointMetaData decodeCheckpointValue(const ValueRef& value);
 
 // "\xff/dataMoves/[[UID]] := [[DataMoveMetaData]]"
 extern const KeyRangeRef dataMoveKeys;
-const Key dataMoveKeyFor(UID dataMoveId);
-const Value dataMoveValue(const DataMoveMetaData& dataMove);
+Key dataMoveKeyFor(UID dataMoveId);
+Value dataMoveValue(const DataMoveMetaData& dataMove);
 UID decodeDataMoveKey(const KeyRef& key);
 DataMoveMetaData decodeDataMoveValue(const ValueRef& value);
 
@@ -187,17 +180,17 @@ DataMoveMetaData decodeDataMoveValue(const ValueRef& value);
 extern const KeyRangeRef serverKeysRange;
 extern const KeyRef serverKeysPrefix;
 extern const ValueRef serverKeysTrue, serverKeysTrueEmptyRange, serverKeysFalse;
-const UID newDataMoveId(const uint64_t physicalShardId,
-                        AssignEmptyRange assignEmptyRange,
-                        const DataMoveType type,
-                        const DataMovementReason reason,
-                        UnassignShard unassignShard = UnassignShard::False);
-const Key serverKeysKey(UID serverID, const KeyRef& keys);
-const Key serverKeysPrefixFor(UID serverID);
+UID newDataMoveId(const uint64_t physicalShardId,
+                  AssignEmptyRange assignEmptyRange,
+                  const DataMoveType type,
+                  const DataMovementReason reason,
+                  UnassignShard unassignShard = UnassignShard::False);
+Key serverKeysKey(UID serverID, const KeyRef& keys);
+Key serverKeysPrefixFor(UID serverID);
 UID serverKeysDecodeServer(const KeyRef& key);
 std::pair<UID, Key> serverKeysDecodeServerBegin(const KeyRef& key);
 bool serverHasKey(ValueRef storedValue);
-const Value serverKeysValue(const UID& id);
+Value serverKeysValue(const UID& id);
 void decodeDataMoveId(const UID& id,
                       bool& assigned,
                       bool& emptyRange,
@@ -223,7 +216,7 @@ extern const KeyRangeRef tssMappingKeys;
 // For quarantining a misbehaving TSS.
 extern const KeyRangeRef tssQuarantineKeys;
 
-const Key tssQuarantineKeyFor(UID serverID);
+Key tssQuarantineKeyFor(UID serverID);
 UID decodeTssQuarantineKey(KeyRef const&);
 
 // \xff/tssMismatch/[[Tuple<TSSStorageUID, timestamp, mismatchUID>]] := [[TraceEventString]]
@@ -251,23 +244,23 @@ extern const KeyRef serverTagConflictPrefix;
 extern const KeyRangeRef serverTagHistoryKeys;
 extern const KeyRef serverTagHistoryPrefix;
 
-const Key serverTagKeyFor(UID serverID);
-const Key serverTagHistoryKeyFor(UID serverID);
-const KeyRange serverTagHistoryRangeFor(UID serverID);
-const KeyRange serverTagHistoryRangeBefore(UID serverID, Version version);
-const Value serverTagValue(Tag);
+Key serverTagKeyFor(UID serverID);
+Key serverTagHistoryKeyFor(UID serverID);
+KeyRange serverTagHistoryRangeFor(UID serverID);
+KeyRange serverTagHistoryRangeBefore(UID serverID, Version version);
+Value serverTagValue(Tag);
 UID decodeServerTagKey(KeyRef const&);
 Version decodeServerTagHistoryKey(KeyRef const&);
 Tag decodeServerTagValue(ValueRef const&);
-const Key serverTagConflictKeyFor(Tag);
+Key serverTagConflictKeyFor(Tag);
 
 //    "\xff/tagLocalityList/[[datacenterID]]" := "[[tagLocality]]"
 //	Provides the tagLocality for the given datacenterID
 //	See "FDBTypes.h" struct Tag for more details on tagLocality
 extern const KeyRangeRef tagLocalityListKeys;
 extern const KeyRef tagLocalityListPrefix;
-const Key tagLocalityListKeyFor(Optional<Value> dcID);
-const Value tagLocalityListValue(int8_t const&);
+Key tagLocalityListKeyFor(Optional<Value> dcID);
+Value tagLocalityListValue(int8_t const&);
 Optional<Value> decodeTagLocalityListKey(KeyRef const&);
 int8_t decodeTagLocalityListValue(ValueRef const&);
 
@@ -276,8 +269,8 @@ int8_t decodeTagLocalityListValue(ValueRef const&);
 //	Used in the initialization of the Data Distributor.
 extern const KeyRangeRef datacenterReplicasKeys;
 extern const KeyRef datacenterReplicasPrefix;
-const Key datacenterReplicasKeyFor(Optional<Value> dcID);
-const Value datacenterReplicasValue(int const&);
+Key datacenterReplicasKeyFor(Optional<Value> dcID);
+Value datacenterReplicasValue(int const&);
 Optional<Value> decodeDatacenterReplicasKey(KeyRef const&);
 int decodeDatacenterReplicasValue(ValueRef const&);
 
@@ -286,7 +279,7 @@ int decodeDatacenterReplicasValue(ValueRef const&);
 //	(as opposed to having no value at all)
 extern const KeyRangeRef tLogDatacentersKeys;
 extern const KeyRef tLogDatacentersPrefix;
-const Key tLogDatacentersKeyFor(Optional<Value> dcID);
+Key tLogDatacentersKeyFor(Optional<Value> dcID);
 Optional<Value> decodeTLogDatacentersKey(KeyRef const&);
 
 extern const KeyRef primaryDatacenterKey;
@@ -297,8 +290,8 @@ extern const KeyRef primaryDatacenterKey;
 //    have a new ID.  When removed from here, a storage server may release all resources and destroy itself.
 extern const KeyRangeRef serverListKeys;
 extern const KeyRef serverListPrefix;
-const Key serverListKeyFor(UID serverID);
-const Value serverListValue(StorageServerInterface const&);
+Key serverListKeyFor(UID serverID);
+Value serverListValue(StorageServerInterface const&);
 UID decodeServerListKey(KeyRef const&);
 StorageServerInterface decodeServerListValue(ValueRef const&);
 
@@ -312,8 +305,8 @@ extern const KeyRef processClassPrefix;
 extern const KeyRef processClassChangeKey;
 extern const KeyRef processClassVersionKey;
 extern const ValueRef processClassVersionValue;
-const Key processClassKeyFor(StringRef processID);
-const Value processClassValue(ProcessClass const&);
+Key processClassKeyFor(StringRef processID);
+Value processClassValue(ProcessClass const&);
 Key decodeProcessClassKey(KeyRef const&);
 ProcessClass decodeProcessClassValue(ValueRef const&);
 UID decodeProcessClassKeyOld(KeyRef const& key);
@@ -325,6 +318,7 @@ extern const KeyRangeRef configKeys;
 extern const KeyRef configKeysPrefix;
 
 extern const KeyRef backupWorkerEnabledKey;
+extern const KeyRef rangeBackupWorkerEnabledKey;
 extern const KeyRef perpetualStorageWiggleKey;
 extern const KeyRef perpetualStorageWiggleLocalityKey;
 extern const KeyRef perpetualStorageWiggleIDPrefix;
@@ -411,8 +405,8 @@ extern const KeyRef globalConfigVersionKey;
 //	and are currently (recently) available
 extern const KeyRangeRef workerListKeys;
 extern const KeyRef workerListPrefix;
-const Key workerListKeyFor(StringRef processID);
-const Value workerListValue(ProcessData const&);
+Key workerListKeyFor(StringRef processID);
+Value workerListValue(ProcessData const&);
 Key decodeWorkerListKey(KeyRef const&);
 ProcessData decodeWorkerListValue(ValueRef const&);
 
@@ -421,11 +415,24 @@ ProcessData decodeWorkerListValue(ValueRef const&);
 //	See "FDBTypes.h" struct WorkerBackupStatus for more details on the return type value.
 extern const KeyRangeRef backupProgressKeys;
 extern const KeyRef backupProgressPrefix;
-const Key backupProgressKeyFor(UID workerID);
-const Value backupProgressValue(const WorkerBackupStatus& status);
+Key backupProgressKeyFor(UID workerID);
+Value backupProgressValue(const WorkerBackupStatus& status);
 UID decodeBackupProgressKey(const KeyRef& key);
 WorkerBackupStatus decodeBackupProgressValue(const ValueRef& value);
 
+//   "\xff\x02/backupRangePartitionedProgress/[[workerID]]" := "[[WorkerBackupStatus]]"
+extern const KeyRangeRef backupRangePartitionedProgressKeys;
+extern const KeyRef backupRangePartitionedProgressPrefix;
+Key backupRangePartitionedProgressKey(UID workerID);
+Value backupRangePartitionedProgressValue(const WorkerBackupStatus& status);
+UID decodeBackupRangePartitionedProgressKey(const KeyRef& key);
+WorkerBackupStatus decodeBackupRangePartitionedProgressValue(const ValueRef& value);
+
+// The key to signal when partition map has been uploaded for a given version.
+//    "\xff\x02/backupRangePartitionedMapUploaded/<version>" := "1"
+extern const KeyRef backupRangePartitionedMapUploadedPrefix;
+Key backupRangePartitionedMapUploadedKeyFor(Version v);
+
 // The key to signal backup workers a new backup job is submitted.
 //    "\xff\x02/backupStarted" := "[[vector<UID,Version1>]]"
 extern const KeyRef backupStartedKey;
@@ -438,9 +445,6 @@ std::vector<std::pair<UID, Version>> decodeBackupStartedValue(const ValueRef& va
 // 1 = Send a signal to pause/already paused.
 extern const KeyRef backupPausedKey;
 
-// The key to store the maximum version that backup workers popped in NOOP mode.
-extern const KeyRef backupWorkerMaxNoopVersionKey;
-
 //	"\xff/previousCoordinators" = "[[ClusterConnectionString]]"
 //	Set to the encoded structure of the cluster's previous set of coordinators.
 //	Changed when performing quorumChange.
@@ -467,8 +471,8 @@ extern const KeyRef minRequiredCommitVersionKey;
 //	number of microseconds since the Unix epoch.
 extern const KeyRef versionEpochKey;
 
-const Value logsValue(const std::vector<std::pair<UID, NetworkAddress>>& logs,
-                      const std::vector<std::pair<UID, NetworkAddress>>& oldLogs);
+Value logsValue(const std::vector<std::pair<UID, NetworkAddress>>& logs,
+                const std::vector<std::pair<UID, NetworkAddress>>& oldLogs);
 std::pair<std::vector<std::pair<UID, NetworkAddress>>, std::vector<std::pair<UID, NetworkAddress>>> decodeLogsValue(
     const ValueRef& value);
 
@@ -495,37 +499,48 @@ extern const UID dataDistributionModeLock;
 extern const KeyRef bulkLoadModeKey;
 extern const KeyRangeRef bulkLoadTaskKeys;
 extern const KeyRef bulkLoadTaskPrefix;
-const Value bulkLoadTaskStateValue(const BulkLoadTaskState& bulkLoadTaskState);
+Value bulkLoadTaskStateValue(const BulkLoadTaskState& bulkLoadTaskState);
 BulkLoadTaskState decodeBulkLoadTaskState(const ValueRef& value);
 
-const Value ssBulkLoadMetadataValue(const SSBulkLoadMetadata& ssBulkLoadMetadata);
+Value ssBulkLoadMetadataValue(const SSBulkLoadMetadata& ssBulkLoadMetadata);
 SSBulkLoadMetadata decodeSSBulkLoadMetadata(const ValueRef& value);
 
 extern const KeyRangeRef bulkLoadJobKeys;
 extern const KeyRef bulkLoadJobPrefix;
-const Value bulkLoadJobValue(const BulkLoadJobState& bulkLoadJobState);
+Value bulkLoadJobValue(const BulkLoadJobState& bulkLoadJobState);
 BulkLoadJobState decodeBulkLoadJobState(const ValueRef& value);
 
 extern const KeyRangeRef bulkLoadJobHistoryKeys;
 extern const KeyRef bulkLoadJobHistoryPrefix;
-const Key bulkLoadJobHistoryKeyFor(const UID& jobId);
+Key bulkLoadJobHistoryKeyFor(const UID& jobId);
 
 extern const KeyRef bulkDumpModeKey;
 extern const KeyRangeRef bulkDumpKeys;
 extern const KeyRef bulkDumpPrefix;
-const Value bulkDumpStateValue(const BulkDumpState& bulkDumpState);
+Value bulkDumpStateValue(const BulkDumpState& bulkDumpState);
 BulkDumpState decodeBulkDumpState(const ValueRef& value);
 
+// BulkDump owner tracking - stored separately for backward compatibility
+// "\xff/bulkDumpOwner/[[jobId]]" := "[[BulkDumpOwnerInfo]]"
+extern const KeyRangeRef bulkDumpOwnerKeys;
+extern const KeyRef bulkDumpOwnerPrefix;
+
+// "\xff/bulkLoadOwner/[[jobId]]" := "[[BulkDumpOwnerInfo]]" (reuses same struct)
+extern const KeyRangeRef bulkLoadOwnerKeys;
+extern const KeyRef bulkLoadOwnerPrefix;
+Key bulkDumpOwnerKeyFor(const UID& jobId);
+Key bulkLoadOwnerKeyFor(const UID& jobId);
+
 extern const std::string rangeLockNameForBulkLoad;
 extern const KeyRangeRef rangeLockKeys;
 extern const KeyRef rangeLockPrefix;
-const Value rangeLockStateSetValue(const RangeLockStateSet& rangeLockStateSet);
+Value rangeLockStateSetValue(const RangeLockStateSet& rangeLockStateSet);
 RangeLockStateSet decodeRangeLockStateSet(const ValueRef& value);
 
 extern const KeyRangeRef rangeLockOwnerKeys;
 extern const KeyRef rangeLockOwnerPrefix;
-const Key rangeLockOwnerKeyFor(const RangeLockOwnerName& ownerUniqueID);
-const Value rangeLockOwnerValue(const RangeLockOwner& rangeLockOwner);
+Key rangeLockOwnerKeyFor(const RangeLockOwnerName& ownerUniqueID);
+Value rangeLockOwnerValue(const RangeLockOwner& rangeLockOwner);
 RangeLockOwner decodeRangeLockOwner(const ValueRef& value);
 
 // Keys to view and control tag throttling
@@ -536,8 +551,6 @@ extern const KeyRef tagThrottleSignalKey;
 extern const KeyRef tagThrottleAutoEnabledKey;
 extern const KeyRef tagThrottleLimitKey;
 extern const KeyRef tagThrottleCountKey;
-extern const KeyRangeRef tagQuotaKeys;
-extern const KeyRef tagQuotaPrefix;
 
 // Log Range constant variables
 // Used in the backup pipeline to track mutations
@@ -668,7 +681,7 @@ namespace DDIgnore {
 enum IgnoreType : uint8_t { NONE = 0, REBALANCE_DISK = 1, REBALANCE_READ = 2, ALL = 3 };
 }
 
-const Value healthyZoneValue(StringRef const& zoneId, Version version);
+Value healthyZoneValue(StringRef const& zoneId, Version version);
 std::pair<Key, Version> decodeHealthyZoneValue(ValueRef const&);
 
 // All mutations done to this range are blindly copied into txnStateStore.
diff --git a/fdbclient/include/fdbclient/TagThrottle.actor.h b/fdbclient/include/fdbclient/TagThrottle.h
similarity index 77%
rename from fdbclient/include/fdbclient/TagThrottle.actor.h
rename to fdbclient/include/fdbclient/TagThrottle.h
index 47ccb818687..6a924f4a34a 100644
--- a/fdbclient/include/fdbclient/TagThrottle.actor.h
+++ b/fdbclient/include/fdbclient/TagThrottle.h
@@ -1,5 +1,5 @@
 /*
- * TagThrottle.actor.h
+ * TagThrottle.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,6 @@
  */
 
 #include "flow/Arena.h"
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_TAG_THROTTLE_ACTOR_G_H)
-#define FDBCLIENT_TAG_THROTTLE_ACTOR_G_H
-#include "fdbclient/TagThrottle.actor.g.h"
-#elif !defined(FDBCLIENT_TAG_THROTTLE_ACTOR_H)
-#define FDBCLIENT_TAG_THROTTLE_ACTOR_H
 
 #pragma once
 
@@ -35,7 +30,6 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/CommitTransaction.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 typedef StringRef TransactionTagRef;
 typedef Standalone<TransactionTagRef> TransactionTag;
@@ -247,79 +241,83 @@ namespace ThrottleApi {
 // The template functions can be called with Native API like DatabaseContext, Transaction/ReadYourWritesTransaction
 // or using IClientAPI like IDatabase, ITransaction
 
-ACTOR template <class Tr>
+template <class Tr>
 Future<Optional<bool>> getValidAutoEnabled(Reference<Tr> tr) {
 	// hold the returned standalone object's memory
-	state typename Tr::template FutureT<Optional<Value>> valueF = tr->get(tagThrottleAutoEnabledKey);
-	Optional<Value> value = wait(safeThreadFutureToFuture(valueF));
+	typename Tr::template FutureT<Optional<Value>> valueF = tr->get(tagThrottleAutoEnabledKey);
+	Optional<Value> value = co_await safeThreadFutureToFuture(valueF);
 	if (!value.present()) {
-		return {};
+		co_return Optional<bool>();
 	} else if (value.get() == "1"_sr) {
-		return true;
+		co_return true;
 	} else if (value.get() == "0"_sr) {
-		return false;
+		co_return false;
 	} else {
 		TraceEvent(SevWarnAlways, "InvalidAutoTagThrottlingValue").detail("Value", value.get());
-		return {};
+		co_return Optional<bool>();
 	}
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<std::vector<TagThrottleInfo>> getRecommendedTags(Reference<DB> db, int limit) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
-	loop {
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			Optional<bool> enableAuto = wait(getValidAutoEnabled(tr));
+			Optional<bool> enableAuto = co_await getValidAutoEnabled(tr);
 			if (!enableAuto.present()) {
 				tr->reset();
-				wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF));
+				co_await delay(CLIENT_KNOBS->DEFAULT_BACKOFF);
 				continue;
 			} else if (enableAuto.get()) {
-				return std::vector<TagThrottleInfo>();
+				co_return std::vector<TagThrottleInfo>();
 			}
-			state typename DB::TransactionT::template FutureT<RangeResult> f =
+			typename DB::TransactionT::template FutureT<RangeResult> f =
 			    tr->getRange(KeyRangeRef(tagThrottleAutoKeysPrefix, tagThrottleKeys.end), limit);
-			RangeResult throttles = wait(safeThreadFutureToFuture(f));
+			RangeResult throttles = co_await safeThreadFutureToFuture(f);
 			std::vector<TagThrottleInfo> results;
 			for (auto throttle : throttles) {
 				results.push_back(TagThrottleInfo(TagThrottleKey::fromKey(throttle.key),
 				                                  TagThrottleValue::fromValue(throttle.value)));
 			}
-			return results;
+			co_return results;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<std::vector<TagThrottleInfo>>
 getThrottledTags(Reference<DB> db, int limit, ContainsRecommended containsRecommended = ContainsRecommended::False) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
-	state Optional<bool> reportAuto;
-	loop {
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Optional<bool> reportAuto;
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			wait(store(reportAuto, getValidAutoEnabled(tr)));
+			reportAuto = co_await getValidAutoEnabled(tr);
 			if (!reportAuto.present()) {
 				tr->reset();
-				wait(delay(CLIENT_KNOBS->DEFAULT_BACKOFF));
+				co_await delay(CLIENT_KNOBS->DEFAULT_BACKOFF);
 				continue;
 			}
-			state typename DB::TransactionT::template FutureT<RangeResult> f = tr->getRange(
+			typename DB::TransactionT::template FutureT<RangeResult> f = tr->getRange(
 			    reportAuto.get() ? tagThrottleKeys : KeyRangeRef(tagThrottleKeysPrefix, tagThrottleAutoKeysPrefix),
 			    limit);
-			RangeResult throttles = wait(safeThreadFutureToFuture(f));
+			RangeResult throttles = co_await safeThreadFutureToFuture(f);
 			std::vector<TagThrottleInfo> results;
 			for (auto throttle : throttles) {
 				results.push_back(TagThrottleInfo(TagThrottleKey::fromKey(throttle.key),
 				                                  TagThrottleValue::fromValue(throttle.value)));
 			}
-			return results;
+			co_return results;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -328,12 +326,12 @@ void signalThrottleChange(Reference<Tr> tr) {
 	tr->atomicOp(tagThrottleSignalKey, "XXXXXXXXXX\x00\x00\x00\x00"_sr, MutationRef::SetVersionstampedValue);
 }
 
-ACTOR template <class Tr>
+template <class Tr>
 Future<Void> updateThrottleCount(Reference<Tr> tr, int64_t delta) {
-	state typename Tr::template FutureT<Optional<Value>> countVal = tr->get(tagThrottleCountKey);
-	state typename Tr::template FutureT<Optional<Value>> limitVal = tr->get(tagThrottleLimitKey);
+	typename Tr::template FutureT<Optional<Value>> countVal = tr->get(tagThrottleCountKey);
+	typename Tr::template FutureT<Optional<Value>> limitVal = tr->get(tagThrottleLimitKey);
 
-	wait(success(safeThreadFutureToFuture(countVal)) && success(safeThreadFutureToFuture(limitVal)));
+	co_await (success(safeThreadFutureToFuture(countVal)) && success(safeThreadFutureToFuture(limitVal)));
 
 	int64_t count = 0;
 	int64_t limit = 0;
@@ -358,29 +356,29 @@ Future<Void> updateThrottleCount(Reference<Tr> tr, int64_t delta) {
 	writer << count;
 
 	tr->set(tagThrottleCountKey, writer.toValue());
-	return Void();
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<bool> unthrottleMatchingThrottles(Reference<DB> db,
                                          KeyRef beginKey,
                                          KeyRef endKey,
                                          Optional<TransactionPriority> priority,
                                          bool onlyExpiredThrottles) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
 
-	state KeySelector begin = firstGreaterOrEqual(beginKey);
-	state KeySelector end = firstGreaterOrEqual(endKey);
+	KeySelector begin = firstGreaterOrEqual(beginKey);
+	KeySelector end = firstGreaterOrEqual(endKey);
 
-	state bool removed = false;
+	bool removed = false;
 
-	loop {
+	while (true) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		Error err;
 		try {
 			// holds memory of the RangeResult
-			state typename DB::TransactionT::template FutureT<RangeResult> f = tr->getRange(begin, end, 1000);
-			state RangeResult tags = wait(safeThreadFutureToFuture(f));
-			state uint64_t unthrottledTags = 0;
+			typename DB::TransactionT::template FutureT<RangeResult> f = tr->getRange(begin, end, 1000);
+			RangeResult tags = co_await safeThreadFutureToFuture(f);
+			uint64_t unthrottledTags = 0;
 			uint64_t manualUnthrottledTags = 0;
 			for (auto tag : tags) {
 				if (onlyExpiredThrottles) {
@@ -405,24 +403,25 @@ Future<bool> unthrottleMatchingThrottles(Reference<DB> db,
 			}
 
 			if (manualUnthrottledTags > 0) {
-				wait(updateThrottleCount(tr, -manualUnthrottledTags));
+				co_await updateThrottleCount(tr, -manualUnthrottledTags);
 			}
 
 			if (unthrottledTags > 0) {
 				signalThrottleChange(tr);
 			}
 
-			wait(safeThreadFutureToFuture(tr->commit()));
+			co_await safeThreadFutureToFuture(tr->commit());
 
 			if (!tags.more) {
-				return removed;
+				co_return removed;
 			}
 
 			ASSERT(tags.size() > 0);
 			begin = KeySelector(firstGreaterThan(tags[tags.size() - 1].key), tags.arena());
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
@@ -448,14 +447,14 @@ Future<bool> unthrottleAll(Reference<DB> db,
 	return unthrottleMatchingThrottles(db, begin, end, priority, false);
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<bool> unthrottleTags(Reference<DB> db,
                             TagSet tags,
                             Optional<TagThrottleType> throttleType,
                             Optional<TransactionPriority> priority) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
 
-	state std::vector<Key> keys;
+	std::vector<Key> keys;
 	for (auto p : allTransactionPriorities) {
 		if (!priority.present() || priority.get() == p) {
 			if (!throttleType.present() || throttleType.get() == TagThrottleType::AUTO) {
@@ -467,20 +466,21 @@ Future<bool> unthrottleTags(Reference<DB> db,
 		}
 	}
 
-	state bool removed = false;
+	bool removed = false;
 
-	loop {
+	while (true) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		Error err;
 		try {
-			state std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>> valueFutures;
-			state std::vector<Future<Optional<Value>>> values;
+			std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>> valueFutures;
+			std::vector<Future<Optional<Value>>> values;
 			values.reserve(keys.size());
 			for (auto key : keys) {
 				valueFutures.push_back(tr->get(key));
 				values.push_back(safeThreadFutureToFuture(valueFutures.back()));
 			}
 
-			wait(waitForAll(values));
+			co_await waitForAll(values);
 
 			int delta = 0;
 			for (int i = 0; i < values.size(); ++i) {
@@ -500,21 +500,22 @@ Future<bool> unthrottleTags(Reference<DB> db,
 			}
 
 			if (delta != 0) {
-				wait(updateThrottleCount(tr, delta));
+				co_await updateThrottleCount(tr, delta);
 			}
 			if (removed) {
 				signalThrottleChange(tr);
-				wait(safeThreadFutureToFuture(tr->commit()));
+				co_await safeThreadFutureToFuture(tr->commit());
 			}
 
-			return removed;
+			co_return removed;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<Void> throttleTags(Reference<DB> db,
                           TagSet tags,
                           double tpsRate,
@@ -523,8 +524,8 @@ Future<Void> throttleTags(Reference<DB> db,
                           TransactionPriority priority,
                           Optional<double> expirationTime = Optional<double>(),
                           Optional<TagThrottledReason> reason = Optional<TagThrottledReason>()) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
-	state Key key = TagThrottleKey(tags, throttleType, priority).toKey();
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Key key = TagThrottleKey(tags, throttleType, priority).toKey();
 
 	ASSERT(initialDuration > 0);
 
@@ -537,17 +538,18 @@ Future<Void> throttleTags(Reference<DB> db,
 	                          reason.present() ? reason.get() : TagThrottledReason::UNSET);
 	BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
 	wr << throttle;
-	state Value value = wr.toValue();
+	Value value = wr.toValue();
 
-	loop {
+	while (true) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		Error err;
 		try {
 			if (throttleType == TagThrottleType::MANUAL) {
 				// hold the returned standalone object's memory
-				state typename DB::TransactionT::template FutureT<Optional<Value>> oldThrottleF = tr->get(key);
-				Optional<Value> oldThrottle = wait(safeThreadFutureToFuture(oldThrottleF));
+				typename DB::TransactionT::template FutureT<Optional<Value>> oldThrottleF = tr->get(key);
+				Optional<Value> oldThrottle = co_await safeThreadFutureToFuture(oldThrottleF);
 				if (!oldThrottle.present()) {
-					wait(updateThrottleCount(tr, 1));
+					co_await updateThrottleCount(tr, 1);
 				}
 			}
 
@@ -557,61 +559,40 @@ Future<Void> throttleTags(Reference<DB> db,
 				signalThrottleChange(tr);
 			}
 
-			wait(safeThreadFutureToFuture(tr->commit()));
-			return Void();
+			co_await safeThreadFutureToFuture(tr->commit());
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-ACTOR template <class DB>
+template <class DB>
 Future<Void> enableAuto(Reference<DB> db, bool enabled) {
-	state Reference<typename DB::TransactionT> tr = db->createTransaction();
+	Reference<typename DB::TransactionT> tr = db->createTransaction();
 
-	loop {
+	while (true) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		Error err;
 		try {
 			// hold the returned standalone object's memory
-			state typename DB::TransactionT::template FutureT<Optional<Value>> valueF =
-			    tr->get(tagThrottleAutoEnabledKey);
-			Optional<Value> value = wait(safeThreadFutureToFuture(valueF));
+			typename DB::TransactionT::template FutureT<Optional<Value>> valueF = tr->get(tagThrottleAutoEnabledKey);
+			Optional<Value> value = co_await safeThreadFutureToFuture(valueF);
 			if (!value.present() || (enabled && value.get() != "1"_sr) || (!enabled && value.get() != "0"_sr)) {
 				tr->set(tagThrottleAutoEnabledKey, enabled ? "1"_sr : "0"_sr);
 				signalThrottleChange<typename DB::TransactionT>(tr);
 
-				wait(safeThreadFutureToFuture(tr->commit()));
+				co_await safeThreadFutureToFuture(tr->commit());
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(safeThreadFutureToFuture(tr->onError(e)));
+			err = e;
 		}
+		co_await safeThreadFutureToFuture(tr->onError(err));
 	}
 }
 
-class TagQuotaValue {
-public:
-	int64_t reservedQuota{ 0 };
-	int64_t totalQuota{ 0 };
-	bool isValid() const;
-	Value toValue() const;
-	static TagQuotaValue fromValue(ValueRef);
-};
-
-Key getTagQuotaKey(TransactionTagRef);
-
-template <class Tr>
-void setTagQuota(Reference<Tr> tr, TransactionTagRef tag, int64_t reservedQuota, int64_t totalQuota) {
-	TagQuotaValue tagQuotaValue;
-	tagQuotaValue.reservedQuota = reservedQuota;
-	tagQuotaValue.totalQuota = totalQuota;
-	if (!tagQuotaValue.isValid()) {
-		throw invalid_throttle_quota_value();
-	}
-	tr->set(getTagQuotaKey(tag), tagQuotaValue.toValue());
-	signalThrottleChange(tr);
-}
-
 }; // namespace ThrottleApi
 
 template <class Value>
@@ -622,6 +603,3 @@ using PrioritizedTransactionTagMap = std::map<TransactionPriority, TransactionTa
 
 template <class Value>
 using UIDTransactionTagMap = std::unordered_map<UID, TransactionTagMap<Value>>;
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/TaskBucket.h b/fdbclient/include/fdbclient/TaskBucket.h
index d3f67ff9169..b470068eed2 100644
--- a/fdbclient/include/fdbclient/TaskBucket.h
+++ b/fdbclient/include/fdbclient/TaskBucket.h
@@ -28,9 +28,9 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbclient/Subspace.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 
 class FutureBucket;
 class TaskFuture;
@@ -56,7 +56,7 @@ FDB_BOOLEAN_PARAM(UpdateParams);
 //       same Task
 class Task : public ReferenceCounted<Task> {
 public:
-	Task(Value type = StringRef(), uint32_t version = 0, Value done = StringRef(), unsigned int priority = 0);
+	explicit Task(Value type = StringRef(), uint32_t version = 0, Value done = StringRef(), unsigned int priority = 0);
 	~Task() {};
 
 	// Methods that safely read values from task's params
@@ -102,7 +102,7 @@ class Task : public ReferenceCounted<Task> {
 template <typename T>
 class TaskParam {
 public:
-	TaskParam(StringRef key) : key(key) {}
+	explicit(false) TaskParam(StringRef key) : key(key) {}
 	T get(Reference<Task> task) const { return TupleCodec<T>::unpack(task->params[key]); }
 	void set(Reference<Task> task, T const& val) const { task->params[key] = TupleCodec<T>::pack(val); }
 	bool exists(Reference<Task> task) const { return task->params.find(key) != task->params.end(); }
@@ -139,10 +139,10 @@ class FutureBucket;
 // instance may declare the Task a failure and move it back to the available subspace.
 class TaskBucket : public ReferenceCounted<TaskBucket> {
 public:
-	TaskBucket(const Subspace& subspace,
-	           AccessSystemKeys = AccessSystemKeys::False,
-	           PriorityBatch = PriorityBatch::False,
-	           LockAware = LockAware::False);
+	explicit TaskBucket(const Subspace& subspace,
+	                    AccessSystemKeys = AccessSystemKeys::False,
+	                    PriorityBatch = PriorityBatch::False,
+	                    LockAware = LockAware::False);
 	virtual ~TaskBucket();
 
 	void setOptions(Reference<ReadYourWritesTransaction> tr) {
@@ -311,7 +311,9 @@ class TaskFuture;
 
 class FutureBucket : public ReferenceCounted<FutureBucket> {
 public:
-	FutureBucket(const Subspace& subspace, AccessSystemKeys = AccessSystemKeys::False, LockAware = LockAware::False);
+	explicit FutureBucket(const Subspace& subspace,
+	                      AccessSystemKeys = AccessSystemKeys::False,
+	                      LockAware = LockAware::False);
 	virtual ~FutureBucket();
 
 	void setOptions(Reference<ReadYourWritesTransaction> tr) {
@@ -350,7 +352,7 @@ class FutureBucket : public ReferenceCounted<FutureBucket> {
 class TaskFuture : public ReferenceCounted<TaskFuture> {
 public:
 	TaskFuture();
-	TaskFuture(const Reference<FutureBucket> bucket, Standalone<StringRef> key = Standalone<StringRef>());
+	explicit TaskFuture(const Reference<FutureBucket> bucket, Standalone<StringRef> key = Standalone<StringRef>());
 	virtual ~TaskFuture();
 
 	Future<bool> isSet(Reference<ReadYourWritesTransaction> tr);
@@ -499,8 +501,8 @@ struct TaskCompletionKey {
 	TaskCompletionKey() {}
 
 private:
-	TaskCompletionKey(Reference<TaskFuture> f) : joinFuture(f) {}
-	TaskCompletionKey(Key k) : key(k) {}
+	explicit TaskCompletionKey(Reference<TaskFuture> f) : joinFuture(f) {}
+	explicit TaskCompletionKey(Key k) : key(k) {}
 };
 
 #endif
diff --git a/fdbclient/include/fdbclient/TestKnobCollection.h b/fdbclient/include/fdbclient/TestKnobCollection.h
deleted file mode 100644
index 2a88616fa63..00000000000
--- a/fdbclient/include/fdbclient/TestKnobCollection.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * TestKnobCollection.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBCLIENT_TESTKNOBCOLLECTION_H
-#define FDBCLIENT_TESTKNOBCOLLECTION_H
-
-#pragma once
-
-#include "fdbclient/IKnobCollection.h"
-#include "fdbclient/ServerKnobCollection.h"
-
-class TestKnobs : public KnobsImpl<TestKnobs> {
-public:
-	TestKnobs();
-
-	// Nonatomic test knobs
-	int64_t TEST_LONG;
-	int TEST_INT;
-	double TEST_DOUBLE;
-	bool TEST_BOOL;
-	std::string TEST_STRING;
-
-	bool operator==(TestKnobs const&) const;
-	bool operator!=(TestKnobs const&) const;
-	void initialize();
-};
-
-/*
- * Stores both flow knobs, client knobs, server knobs, and test knobs. As the
- * name implies, this class is only meant to be used for testing.
- */
-class TestKnobCollection : public IKnobCollection {
-	ServerKnobCollection serverKnobCollection;
-	TestKnobs testKnobs;
-
-public:
-	TestKnobCollection(Randomize randomize, IsSimulated isSimulated);
-	void initialize(Randomize randomize, IsSimulated isSimulated) override;
-	void reset(Randomize randomize, IsSimulated isSimulated) override;
-	FlowKnobs const& getFlowKnobs() const override { return serverKnobCollection.getFlowKnobs(); }
-	ClientKnobs const& getClientKnobs() const override { return serverKnobCollection.getClientKnobs(); }
-	ServerKnobs const& getServerKnobs() const override { return serverKnobCollection.getServerKnobs(); }
-	TestKnobs const& getTestKnobs() const override { return testKnobs; }
-	void clearTestKnobs() override;
-	Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const override;
-	bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
-};
-
-#endif // FDBCLIENT_TESTKNOBCOLLECTION_H
diff --git a/fdbclient/include/fdbclient/ThreadSafeTransaction.h b/fdbclient/include/fdbclient/ThreadSafeTransaction.h
index 23b65fa7ab4..94699beefb7 100644
--- a/fdbclient/include/fdbclient/ThreadSafeTransaction.h
+++ b/fdbclient/include/fdbclient/ThreadSafeTransaction.h
@@ -71,7 +71,7 @@ class ThreadSafeDatabase : public IDatabase, public ThreadSafeReferenceCounted<T
 public: // Internal use only
 	enum class ConnectionRecordType { FILE, CONNECTION_STRING };
 	ThreadSafeDatabase(ConnectionRecordType connectionRecordType, std::string connectionRecord, int apiVersion);
-	ThreadSafeDatabase(DatabaseContext* db) : db(db) {}
+	explicit ThreadSafeDatabase(DatabaseContext* db) : db(db) {}
 	DatabaseContext* unsafeGetPtr() const { return db; }
 };
 
@@ -156,7 +156,7 @@ class ThreadSafeTransaction : public ITransaction, ThreadSafeReferenceCounted<Th
 	// These are to permit use as state variables in actors:
 	ThreadSafeTransaction() : tr(nullptr), initialized(std::make_shared<std::atomic_bool>(false)) {}
 	void operator=(ThreadSafeTransaction&& r) noexcept;
-	ThreadSafeTransaction(ThreadSafeTransaction&& r) noexcept;
+	explicit(false) ThreadSafeTransaction(ThreadSafeTransaction&& r) noexcept;
 
 	void reset() override;
 
diff --git a/fdbclient/include/fdbclient/Tracing.h b/fdbclient/include/fdbclient/Tracing.h
index cc8321a7d5e..ba4a274a719 100644
--- a/fdbclient/include/fdbclient/Tracing.h
+++ b/fdbclient/include/fdbclient/Tracing.h
@@ -50,7 +50,7 @@ struct SpanContext {
 	SpanContext() : traceID(UID()), spanID(0), m_Flags(TraceFlags::unsampled) {}
 	SpanContext(UID traceID, uint64_t spanID, TraceFlags flags) : traceID(traceID), spanID(spanID), m_Flags(flags) {}
 	SpanContext(UID traceID, uint64_t spanID) : traceID(traceID), spanID(spanID), m_Flags(TraceFlags::unsampled) {}
-	SpanContext(const SpanContext& span) = default;
+	explicit(false) SpanContext(const SpanContext& span) = default;
 	bool isSampled() const { return (m_Flags & TraceFlags::sampled) == TraceFlags::sampled; }
 	std::string toString() const { return format("%016llx%016llx%016llx", traceID.first(), traceID.second(), spanID); };
 	bool isValid() const { return traceID.first() != 0 && traceID.second() != 0 && spanID != 0; }
@@ -160,7 +160,7 @@ class Span {
 	explicit Span(const Location& location) : Span(location, SpanContext()) {}
 
 	Span(const Span&) = delete;
-	Span(Span&& o) {
+	explicit(false) Span(Span&& o) {
 		arena = std::move(o.arena);
 		context = o.context;
 		location = o.location;
diff --git a/fdbclient/include/fdbclient/TransactionLineage.h b/fdbclient/include/fdbclient/TransactionLineage.h
index ec0d439755c..e1ba1a040d7 100644
--- a/fdbclient/include/fdbclient/TransactionLineage.h
+++ b/fdbclient/include/fdbclient/TransactionLineage.h
@@ -108,7 +108,8 @@ class ScopedLineage {
 		}
 		getCurrentLineage()->modify(member) = before;
 	}
-	ScopedLineage(ScopedLineage<T, V>&& o) : before(std::move(o.before)), member(o.member), valid(o.valid) {
+	explicit(false) ScopedLineage(ScopedLineage<T, V>&& o)
+	  : before(std::move(o.before)), member(o.member), valid(o.valid) {
 		o.release();
 	}
 	ScopedLineage& operator=(ScopedLineage<T, V>&& o) {
diff --git a/fdbclient/include/fdbclient/Tuple.h b/fdbclient/include/fdbclient/Tuple.h
index 76e102a48b3..52ea0bf417b 100644
--- a/fdbclient/include/fdbclient/Tuple.h
+++ b/fdbclient/include/fdbclient/Tuple.h
@@ -121,7 +121,7 @@ struct Tuple {
 	}
 
 private:
-	Tuple(const StringRef& data, bool exclude_incomplete = false, bool exclude_user_type = false);
+	explicit Tuple(const StringRef& data, bool exclude_incomplete = false, bool exclude_user_type = false);
 	Standalone<VectorRef<uint8_t>> data;
 	std::vector<size_t> offsets;
 };
diff --git a/fdbclient/include/fdbclient/TupleVersionstamp.h b/fdbclient/include/fdbclient/TupleVersionstamp.h
index bfcb6a97f43..4c6e7f16ef7 100644
--- a/fdbclient/include/fdbclient/TupleVersionstamp.h
+++ b/fdbclient/include/fdbclient/TupleVersionstamp.h
@@ -33,7 +33,7 @@ struct TupleVersionstamp {
 	    "\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00"_sr;
 
 	TupleVersionstamp() : data(DEFAULT_VERSIONSTAMP) {}
-	TupleVersionstamp(StringRef);
+	explicit TupleVersionstamp(StringRef);
 	TupleVersionstamp(int64_t version, uint16_t batchNumber, uint16_t userVersion = 0);
 
 	int64_t getVersion() const;
diff --git a/fdbclient/include/fdbclient/VersionVector.h b/fdbclient/include/fdbclient/VersionVector.h
index 112cfb4284b..57160149a0e 100644
--- a/fdbclient/include/fdbclient/VersionVector.h
+++ b/fdbclient/include/fdbclient/VersionVector.h
@@ -43,7 +43,7 @@ struct VersionVector {
 	                    // version in the "versions" map.)
 
 	VersionVector() : maxVersion(invalidVersion), cachedEncodedSize(InvalidEncodedSize) {}
-	VersionVector(Version version) : maxVersion(version), cachedEncodedSize(InvalidEncodedSize) {}
+	explicit VersionVector(Version version) : maxVersion(version), cachedEncodedSize(InvalidEncodedSize) {}
 
 private:
 	// Only invoked by getDelta() and applyDelta(), where tag has been validated
diff --git a/fdbclient/include/fdbclient/VersionedMap.actor.h b/fdbclient/include/fdbclient/VersionedMap.actor.h
deleted file mode 100644
index 6be5db37718..00000000000
--- a/fdbclient/include/fdbclient/VersionedMap.actor.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * VersionedMap.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_VERSIONEDMAP_ACTOR_G_H)
-#define FDBCLIENT_VERSIONEDMAP_ACTOR_G_H
-#include "fdbclient/VersionedMap.actor.g.h"
-#elif !defined(FDBCLIENT_VERSIONEDMAP_ACTOR_H)
-#define FDBCLIENT_VERSIONEDMAP_ACTOR_H
-
-#include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR template <class Tree>
-Future<Void> deferredCleanupActor(std::vector<Tree> toFree, TaskPriority taskID = TaskPriority::DefaultYield) {
-	state int freeCount = 0;
-	while (!toFree.empty()) {
-		Tree a = std::move(toFree.back());
-		toFree.pop_back();
-
-		for (int c = 0; c < 3; c++) {
-			if (a->pointer[c] && a->pointer[c]->isSoleOwner())
-				toFree.push_back(std::move(a->pointer[c]));
-		}
-
-		if (++freeCount % 100 == 0)
-			wait(yield(taskID));
-	}
-
-	return Void();
-}
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbclient/include/fdbclient/VersionedMap.h b/fdbclient/include/fdbclient/VersionedMap.h
index 2eb1db58fd7..02f036bba37 100644
--- a/fdbclient/include/fdbclient/VersionedMap.h
+++ b/fdbclient/include/fdbclient/VersionedMap.h
@@ -26,7 +26,55 @@
 #include "flow/IndexedSet.h"
 #include "fdbclient/FDBTypes.h"
 #include "flow/IRandom.h"
-#include "fdbclient/VersionedMap.actor.h"
+#include <unordered_set>
+
+// Drains old PTree roots without recursive Reference destruction, including if
+// the cleanup coroutine is cancelled while suspended at yield().
+template <class Tree>
+class DeferredCleanupWorklist {
+public:
+	explicit DeferredCleanupWorklist(std::vector<Tree>&& toFree) : toFree(std::move(toFree)) {}
+	~DeferredCleanupWorklist() { drain(); }
+
+	bool empty() const { return toFree.empty(); }
+	void cleanupOne() {
+		Tree a = std::move(toFree.back());
+		toFree.pop_back();
+
+		auto* node = a.extractPtr();
+		if (node == nullptr || !node->delref_no_destroy()) {
+			return;
+		}
+
+		for (auto& child : node->pointer) {
+			if (child) {
+				toFree.push_back(std::move(child));
+			}
+		}
+		delete node;
+	}
+
+private:
+	void drain() {
+		while (!toFree.empty()) {
+			cleanupOne();
+		}
+	}
+
+	std::vector<Tree> toFree;
+};
+
+template <class Tree>
+Future<Void> deferredCleanupActor(std::vector<Tree> toFree, TaskPriority taskID = TaskPriority::DefaultYield) {
+	DeferredCleanupWorklist<Tree> cleanup(std::move(toFree));
+	int freeCount = 0;
+	while (!cleanup.empty()) {
+		cleanup.cleanupOne();
+
+		if (++freeCount % 100 == 0)
+			co_await yield(taskID);
+	}
+}
 
 // PTree is a persistent balanced binary tree implementation. It is based on a treap as a way to guarantee O(1) space
 // for node insertion (rotating is asymptotically cheap), but the constant factors are very large.
@@ -63,6 +111,9 @@ struct PTree : public ReferenceCounted<PTree<T>>, FastAllocated<PTree<T>>, NonCo
 	PTree(const T& data, Version ver) : lastUpdateVersion(ver), updated(false), data(data) {
 		priority = deterministicRandom()->randomUInt32();
 	}
+	PTree(const T& data, Version ver, IRandom& random) : lastUpdateVersion(ver), updated(false), data(data) {
+		priority = random.randomUInt32();
+	}
 	PTree(uint32_t pri, T const& data, Reference<PTree> const& left, Reference<PTree> const& right, Version ver)
 	  : priority(pri), lastUpdateVersion(ver), updated(false), data(data) {
 		pointer[0] = left;
@@ -70,7 +121,7 @@ struct PTree : public ReferenceCounted<PTree<T>>, FastAllocated<PTree<T>>, NonCo
 	}
 
 private:
-	PTree(PTree const&);
+	explicit(false) PTree(PTree const&);
 };
 
 template <class T>
@@ -79,8 +130,8 @@ class PTreeFinger {
 	// This finger size supports trees with up to exp(96/4.3) ~= 4,964,514,749 entries.
 	// The number 4.3 comes from here: https://en.wikipedia.org/wiki/Random_binary_tree#The_longest_path
 	// see also: check().
-	static constexpr size_t N = 96;
-	PTreeFingerEntry entries_[N];
+	static constexpr size_t pathCapacity = 96;
+	PTreeFingerEntry entries_[pathCapacity];
 	size_t size_ = 0;
 	size_t bound_sz_ = 0;
 
@@ -88,8 +139,8 @@ class PTreeFinger {
 	PTreeFinger() {}
 
 	// Explicit copy constructors ensure we copy the live values in entries_.
-	PTreeFinger(PTreeFinger const& f) { *this = f; }
-	PTreeFinger(PTreeFinger&& f) { *this = f; }
+	explicit(false) PTreeFinger(PTreeFinger const& f) { *this = f; }
+	explicit(false) PTreeFinger(PTreeFinger&& f) { *this = f; }
 
 	PTreeFinger& operator=(PTreeFinger const& f) {
 		size_ = f.size_;
@@ -113,12 +164,12 @@ class PTreeFinger {
 
 	void resize(size_t sz) {
 		size_ = sz;
-		ASSERT(size_ < N);
+		ASSERT(size_ < pathCapacity);
 	}
 
 	void push_back(PTree<T> const* node) {
 		entries_[size_++] = { node };
-		ASSERT(size_ < N);
+		ASSERT(size_ < pathCapacity);
 	}
 
 	void push_for_bound(PTree<T> const* node, bool less) {
@@ -279,19 +330,19 @@ T get(PTreeFinger<T>& f) {
 	return f.back()->data;
 }
 
-// Modifies p to point to a PTree with x inserted
+// Modifies p to point to a PTree with x inserted, using a caller-owned priority generator.
 template <class T>
-void insert(Reference<PTree<T>>& p, Version at, const T& x) {
+void insert(Reference<PTree<T>>& p, Version at, const T& x, IRandom& priorityRandom) {
 	if (!p) {
-		p = makeReference<PTree<T>>(x, at);
+		p = makeReference<PTree<T>>(x, at, priorityRandom);
 	} else {
 		int c = ::compare(x, p->data);
 		if (c == 0) {
 			p = makeReference<PTree<T>>(p->priority, x, p->left(at), p->right(at), at);
 		} else {
-			const bool direction = !(c < 0);
+			const bool direction = c >= 0;
 			Reference<PTree<T>> child = p->child(direction, at);
-			insert(child, at, x);
+			insert(child, at, x, priorityRandom);
 			p = update(p, direction, child, at);
 			if (p->child(direction, at)->priority > p->priority)
 				rotate(p, at, !direction);
@@ -299,6 +350,13 @@ void insert(Reference<PTree<T>>& p, Version at, const T& x) {
 	}
 }
 
+// Modifies p to point to a PTree with x inserted.
+template <class T>
+void insert(Reference<PTree<T>>& p, Version at, const T& x) {
+	auto priorityRandom = deterministicRandom();
+	insert(p, at, x, *priorityRandom);
+}
+
 template <class T>
 Reference<PTree<T>> firstNode(const Reference<PTree<T>>& p, Version at) {
 	if (!p)
@@ -338,12 +396,16 @@ void last(const Reference<PTree<T>>& p, Version at, PTreeFinger<T>& f) {
 // modifies p to point to a PTree with the root of p removed
 template <class T>
 void removeRoot(Reference<PTree<T>>& p, Version at) {
-	if (!p->right(at))
+	const auto& right = p->right(at);
+	if (!right)
 		p = p->left(at);
-	else if (!p->left(at))
-		p = p->right(at);
 	else {
-		bool direction = p->right(at)->priority < p->left(at)->priority;
+		const auto& left = p->left(at);
+		if (!left) {
+			p = right;
+			return;
+		}
+		bool direction = right->priority < left->priority;
 		rotate(p, at, direction);
 		Reference<PTree<T>> child = p->child(direction, at);
 		removeRoot(child, at);
@@ -473,11 +535,13 @@ void demoteRoot(Reference<PTree<T>>& p, Version at) {
 		ASSERT(false);
 
 	uint32_t priority[2];
-	for (int i = 0; i < 2; i++)
-		if (p->child(i, at))
-			priority[i] = p->child(i, at)->priority;
+	for (int i = 0; i < 2; i++) {
+		const auto& child = p->child(i, at);
+		if (child)
+			priority[i] = child->priority;
 		else
 			priority[i] = 0;
+	}
 
 	bool higherDirection = priority[1] > priority[0];
 
@@ -498,7 +562,7 @@ Reference<PTree<T>> append(const Reference<PTree<T>>& left, const Reference<PTre
 	if (!right)
 		return left;
 
-	Reference<PTree<T>> r = makeReference<PTree<T>>(lastNode(left, at)->data, at);
+	auto r = makeReference<PTree<T>>(lastNode(left, at)->data, at);
 	if (EXPENSIVE_VALIDATION) {
 		ASSERT(r->data < firstNode(right, at)->data);
 	}
@@ -613,10 +677,13 @@ void check(const Reference<PTree<T>>& p) {
 // This essentially gets rid of node versions that will never be read (beyond 5s worth of versions)
 // TODO look into making this per-version compaction. (We could keep track of updated nodes at each version for example)
 template <class T>
-void compact(Reference<PTree<T>>& p, Version newOldestVersion) {
+void compact(Reference<PTree<T>>& p, Version newOldestVersion, std::unordered_set<PTree<T>*>& visited) {
 	if (!p) {
 		return;
 	}
+	if (!visited.insert(p.getPtr()).second) {
+		return;
+	}
 	if (p->updated && p->lastUpdateVersion <= newOldestVersion) {
 		/* If the node has been updated, figure out which pointer was replaced. And replace that pointer with the
 		   updated pointer. Then we can get rid of the updated child pointer and then make room in the node for future
@@ -629,8 +696,14 @@ void compact(Reference<PTree<T>>& p, Version newOldestVersion) {
 	}
 	Reference<PTree<T>> left = p->left(newOldestVersion);
 	Reference<PTree<T>> right = p->right(newOldestVersion);
-	compact(left, newOldestVersion);
-	compact(right, newOldestVersion);
+	compact(left, newOldestVersion, visited);
+	compact(right, newOldestVersion, visited);
+}
+
+template <class T>
+void compact(Reference<PTree<T>>& p, Version newOldestVersion) {
+	std::unordered_set<PTree<T>*> visited;
+	compact(p, newOldestVersion, visited);
 }
 
 } // namespace PTreeImpl
@@ -670,6 +743,7 @@ class VersionedMap : NonCopyable {
 	typedef Reference<PTreeT> Tree;
 
 	Version oldestVersion, latestVersion;
+	Reference<IRandom> priorityRandom;
 
 	// This deque keeps track of PTree root nodes at various versions. Since the
 	// versions increase monotonically, the deque is implicitly sorted and hence
@@ -691,12 +765,16 @@ class VersionedMap : NonCopyable {
 	static const int overheadPerItem = nextFastAllocatedSize(sizeof(PTreeT)) * 4;
 	struct iterator;
 
-	VersionedMap() : oldestVersion(0), latestVersion(0) { roots.emplace_back(0, Tree()); }
-	VersionedMap(VersionedMap&& v) noexcept
-	  : oldestVersion(v.oldestVersion), latestVersion(v.latestVersion), roots(std::move(v.roots)) {}
+	VersionedMap() : oldestVersion(0), latestVersion(0), priorityRandom(deterministicRandom()) {
+		roots.emplace_back(0, Tree());
+	}
+	explicit(false) VersionedMap(VersionedMap&& v) noexcept
+	  : oldestVersion(v.oldestVersion), latestVersion(v.latestVersion), priorityRandom(std::move(v.priorityRandom)),
+	    roots(std::move(v.roots)) {}
 	void operator=(VersionedMap&& v) noexcept {
 		oldestVersion = v.oldestVersion;
 		latestVersion = v.latestVersion;
+		priorityRandom = std::move(v.priorityRandom);
 		roots = std::move(v.roots);
 	}
 
@@ -770,8 +848,10 @@ class VersionedMap : NonCopyable {
 	// insert() and erase() invalidate atLatest() and all iterators into it
 	void insert(const K& k, const T& t) { insert(k, t, latestVersion); }
 	void insert(const K& k, const T& t, Version insertAt) {
-		PTreeImpl::insert(
-		    roots.back().second, latestVersion, MapPair<K, std::pair<T, Version>>(k, std::make_pair(t, insertAt)));
+		PTreeImpl::insert(roots.back().second,
+		                  latestVersion,
+		                  MapPair<K, std::pair<T, Version>>(k, std::make_pair(t, insertAt)),
+		                  *priorityRandom);
 	}
 	void erase(const K& begin, const K& end) { PTreeImpl::remove(roots.back().second, latestVersion, begin, end); }
 	void erase(const K& key) { // key must be present
@@ -790,9 +870,10 @@ class VersionedMap : NonCopyable {
 		ASSERT(newOldestVersion <= latestVersion);
 		// auto newBegin = roots.lower_bound(newOldestVersion);
 		auto newBegin = lower_bound(roots.begin(), roots.end(), newOldestVersion, rootsComparator());
+		std::unordered_set<PTreeT*> visited;
 		for (auto root = roots.begin(); root != newBegin; ++root) {
 			if (root->second)
-				PTreeImpl::compact(root->second, newOldestVersion);
+				PTreeImpl::compact(root->second, newOldestVersion, visited);
 		}
 		// printf("\nPrinting the tree at latest version after compaction.\n");
 		// PTreeImpl::printTreeDetails(roots.back().second(), 0);
@@ -843,60 +924,127 @@ class VersionedMap : NonCopyable {
 		PTreeFingerT finger;
 	};
 
+	static iterator beginAt(Tree const& root, Version at) {
+		iterator i(root, at);
+		PTreeImpl::first(root, at, i.finger);
+		return i;
+	}
+
+	static iterator endAt(Tree const& root, Version at) { return iterator(root, at); }
+
+	// Returns x such that key==*x, or end()
+	template <class X>
+	static iterator findAt(Tree const& root, Version at, const X& key) {
+		iterator i(root, at);
+		PTreeImpl::lower_bound(root, at, key, i.finger);
+		if (i && i.key() == key)
+			return i;
+		else
+			return endAt(root, at);
+	}
+
+	// Returns the smallest x such that *x>=key, or end()
+	template <class X>
+	static iterator lowerBoundAt(Tree const& root, Version at, const X& key) {
+		iterator i(root, at);
+		PTreeImpl::lower_bound(root, at, key, i.finger);
+		return i;
+	}
+
+	// Returns the smallest x such that *x>key, or end()
+	template <class X>
+	static iterator upperBoundAt(Tree const& root, Version at, const X& key) {
+		iterator i(root, at);
+		PTreeImpl::upper_bound(root, at, key, i.finger);
+		return i;
+	}
+
+	// Returns the largest x such that *x<=key, or end()
+	template <class X>
+	static iterator lastLessOrEqualAt(Tree const& root, Version at, const X& key) {
+		iterator i(root, at);
+		PTreeImpl::upper_bound(root, at, key, i.finger);
+		--i;
+		return i;
+	}
+
+	// Returns the largest x such that *x<key, or end()
+	template <class X>
+	static iterator lastLessAt(Tree const& root, Version at, const X& key) {
+		iterator i(root, at);
+		PTreeImpl::lower_bound(root, at, key, i.finger);
+		--i;
+		return i;
+	}
+
+	iterator latestBegin() const { return beginAt(roots.back().second, latestVersion); }
+	iterator latestEnd() const { return endAt(roots.back().second, latestVersion); }
+
+	// Returns x such that key==*x, or latestEnd()
+	template <class X>
+	iterator latestFind(const X& key) const {
+		return findAt(roots.back().second, latestVersion, key);
+	}
+
+	// Returns the smallest x such that *x>=key, or latestEnd()
+	template <class X>
+	iterator latestLowerBound(const X& key) const {
+		return lowerBoundAt(roots.back().second, latestVersion, key);
+	}
+
+	// Returns the smallest x such that *x>key, or latestEnd()
+	template <class X>
+	iterator latestUpperBound(const X& key) const {
+		return upperBoundAt(roots.back().second, latestVersion, key);
+	}
+
+	// Returns the largest x such that *x<=key, or latestEnd()
+	template <class X>
+	iterator latestLastLessOrEqual(const X& key) const {
+		return lastLessOrEqualAt(roots.back().second, latestVersion, key);
+	}
+
+	// Returns the largest x such that *x<key, or latestEnd()
+	template <class X>
+	iterator latestLastLess(const X& key) const {
+		return lastLessAt(roots.back().second, latestVersion, key);
+	}
+
 	class ViewAtVersion {
 	public:
 		ViewAtVersion(Tree const& root, Version at) : root(root), at(at) {}
 
-		iterator begin() const {
-			iterator i(root, at);
-			PTreeImpl::first(root, at, i.finger);
-			return i;
-		}
-		iterator end() const { return iterator(root, at); }
+		iterator begin() const { return VersionedMap::beginAt(root, at); }
+		iterator end() const { return VersionedMap::endAt(root, at); }
 
 		// Returns x such that key==*x, or end()
 		template <class X>
 		iterator find(const X& key) const {
-			iterator i(root, at);
-			PTreeImpl::lower_bound(root, at, key, i.finger);
-			if (i && i.key() == key)
-				return i;
-			else
-				return end();
+			return VersionedMap::findAt(root, at, key);
 		}
 
 		// Returns the smallest x such that *x>=key, or end()
 		template <class X>
 		iterator lower_bound(const X& key) const {
-			iterator i(root, at);
-			PTreeImpl::lower_bound(root, at, key, i.finger);
-			return i;
+			return VersionedMap::lowerBoundAt(root, at, key);
 		}
 
 		// Returns the smallest x such that *x>key, or end()
 		template <class X>
 		iterator upper_bound(const X& key) const {
-			iterator i(root, at);
-			PTreeImpl::upper_bound(root, at, key, i.finger);
-			return i;
+			return VersionedMap::upperBoundAt(root, at, key);
 		}
 
 		// Returns the largest x such that *x<=key, or end()
 		template <class X>
 		iterator lastLessOrEqual(const X& key) const {
-			iterator i(root, at);
-			PTreeImpl::upper_bound(root, at, key, i.finger);
-			--i;
-			return i;
+			return VersionedMap::lastLessOrEqualAt(root, at, key);
 		}
 
 		// Returns the largest x such that *x<key, or end()
 		template <class X>
 		iterator lastLess(const X& key) const {
-			iterator i(root, at);
-			PTreeImpl::lower_bound(root, at, key, i.finger);
-			--i;
-			return i;
+			return VersionedMap::lastLessAt(root, at, key);
 		}
 
 		void validate() {
diff --git a/fdbclient/include/fdbclient/WriteMap.h b/fdbclient/include/fdbclient/WriteMap.h
index 8521fd64de6..2e4efe2d766 100644
--- a/fdbclient/include/fdbclient/WriteMap.h
+++ b/fdbclient/include/fdbclient/WriteMap.h
@@ -66,6 +66,12 @@ class OperationStack {
 	bool operator==(const OperationStack& r) const;
 };
 
+FDB_BOOLEAN_PARAM(FollowingKeysCleared);
+FDB_BOOLEAN_PARAM(FollowingKeysConflict);
+FDB_BOOLEAN_PARAM(IsConflict);
+FDB_BOOLEAN_PARAM(FollowingKeysUnreadable);
+FDB_BOOLEAN_PARAM(IsUnreadable);
+
 struct WriteMapEntry {
 	KeyRef key;
 	OperationStack stack;
@@ -77,11 +83,11 @@ struct WriteMapEntry {
 
 	WriteMapEntry(KeyRef const& key,
 	              OperationStack&& stack,
-	              bool following_keys_cleared,
-	              bool following_keys_conflict,
-	              bool is_conflict,
-	              bool following_keys_unreadable,
-	              bool is_unreadable)
+	              FollowingKeysCleared following_keys_cleared,
+	              FollowingKeysConflict following_keys_conflict,
+	              IsConflict is_conflict,
+	              FollowingKeysUnreadable following_keys_unreadable,
+	              IsUnreadable is_unreadable)
 	  : key(key), stack(std::move(stack)), following_keys_cleared(following_keys_cleared),
 	    following_keys_conflict(following_keys_conflict), is_conflict(is_conflict),
 	    following_keys_unreadable(following_keys_unreadable), is_unreadable(is_unreadable) {}
@@ -127,14 +133,36 @@ class WriteMap {
 
 public:
 	explicit WriteMap(Arena* arena) : arena(arena), writeMapEmpty(true), ver(-1), scratch_iterator(this) {
-		PTreeImpl::insert(
-		    writes, ver, WriteMapEntry(allKeys.begin, OperationStack(), false, false, false, false, false));
-		PTreeImpl::insert(writes, ver, WriteMapEntry(allKeys.end, OperationStack(), false, false, false, false, false));
-		PTreeImpl::insert(
-		    writes, ver, WriteMapEntry(afterAllKeys, OperationStack(), false, false, false, false, false));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(allKeys.begin,
+		                                OperationStack(),
+		                                FollowingKeysCleared::False,
+		                                FollowingKeysConflict::False,
+		                                IsConflict::False,
+		                                FollowingKeysUnreadable::False,
+		                                IsUnreadable::False));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(allKeys.end,
+		                                OperationStack(),
+		                                FollowingKeysCleared::False,
+		                                FollowingKeysConflict::False,
+		                                IsConflict::False,
+		                                FollowingKeysUnreadable::False,
+		                                IsUnreadable::False));
+		PTreeImpl::insert(writes,
+		                  ver,
+		                  WriteMapEntry(afterAllKeys,
+		                                OperationStack(),
+		                                FollowingKeysCleared::False,
+		                                FollowingKeysConflict::False,
+		                                IsConflict::False,
+		                                FollowingKeysUnreadable::False,
+		                                IsUnreadable::False));
 	}
 
-	WriteMap(WriteMap&& r) noexcept
+	explicit(false) WriteMap(WriteMap&& r) noexcept
 	  : arena(r.arena), writeMapEmpty(r.writeMapEmpty), writes(std::move(r.writes)), ver(r.ver),
 	    scratch_iterator(std::move(r.scratch_iterator)) {}
 
diff --git a/fdbclient/include/fdbclient/json_spirit/json_spirit_reader_template.h b/fdbclient/include/fdbclient/json_spirit/json_spirit_reader_template.h
index 9cd03ce9d31..203d6ecbf06 100644
--- a/fdbclient/include/fdbclient/json_spirit/json_spirit_reader_template.h
+++ b/fdbclient/include/fdbclient/json_spirit/json_spirit_reader_template.h
@@ -205,7 +205,7 @@ class Semantic_actions {
 	typedef typename Config_type::Array_type Array_type;
 	typedef typename String_type::value_type Char_type;
 
-	Semantic_actions(Value_type& value) : value_(value), current_p_(0) {}
+	explicit Semantic_actions(Value_type& value) : value_(value), current_p_(0) {}
 
 	void begin_obj(Char_type c) {
 		assert(c == '{');
@@ -335,7 +335,7 @@ class Json_grammar : public spirit_namespace::grammar<Json_grammar<Value_type, I
 public:
 	typedef Semantic_actions<Value_type, Iter_type> Semantic_actions_t;
 
-	Json_grammar(Semantic_actions_t& semantic_actions) : actions_(semantic_actions) {}
+	explicit Json_grammar(Semantic_actions_t& semantic_actions) : actions_(semantic_actions) {}
 
 	static void throw_not_value(Iter_type begin, Iter_type end) { throw_error(begin, "not a value"); }
 
@@ -352,7 +352,7 @@ class Json_grammar : public spirit_namespace::grammar<Json_grammar<Value_type, I
 	template <typename ScannerT>
 	class definition {
 	public:
-		definition(const Json_grammar& self) {
+		explicit definition(const Json_grammar& self) {
 			using namespace spirit_namespace;
 
 			typedef typename Value_type::String_type::value_type Char_type;
@@ -437,7 +437,7 @@ struct Multi_pass_iters {
 	typedef std::istream_iterator<Char_type, Char_type> istream_iter;
 	typedef spirit_namespace::multi_pass<istream_iter> Mp_iter;
 
-	Multi_pass_iters(Istream_type& is) {
+	explicit Multi_pass_iters(Istream_type& is) {
 		is.unsetf(std::ios::skipws);
 
 		begin_ = spirit_namespace::make_multi_pass(istream_iter(is));
diff --git a/fdbclient/include/fdbclient/json_spirit/json_spirit_value.h b/fdbclient/include/fdbclient/json_spirit/json_spirit_value.h
index a37fc000bd0..b82ac9c32c0 100644
--- a/fdbclient/include/fdbclient/json_spirit/json_spirit_value.h
+++ b/fdbclient/include/fdbclient/json_spirit/json_spirit_value.h
@@ -45,23 +45,24 @@ class Value_impl {
 	typedef typename String_type::const_pointer Const_str_ptr; // eg const char*
 
 	Value_impl(); // creates null value
-	Value_impl(Const_str_ptr value);
-	Value_impl(const String_type& value);
-	Value_impl(const Object& value);
-	Value_impl(const Array& value);
-	Value_impl(bool value);
-	Value_impl(int value);
-	Value_impl(int64_t value);
-	Value_impl(uint64_t value);
-	Value_impl(double value);
+	explicit(false) Value_impl(Const_str_ptr value);
+	explicit(false) Value_impl(const String_type& value);
+	explicit(false) Value_impl(const Object& value);
+	explicit(false) Value_impl(const Array& value);
+	explicit(false) Value_impl(bool value);
+	explicit(false) Value_impl(int value);
+	explicit(false) Value_impl(int64_t value);
+	explicit(false) Value_impl(uint64_t value);
+	explicit(false) Value_impl(double value);
 
 	template <class Iter>
 	Value_impl(Iter first, Iter last); // constructor from containers, e.g. std::vector or std::list
 
 	template <BOOST_VARIANT_ENUM_PARAMS(typename T)>
-	Value_impl(const boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>& variant); // constructor for compatible variant types
+	explicit(false) Value_impl(
+	    const boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>& variant); // constructor for compatible variant types
 
-	Value_impl(const Value_impl& other);
+	explicit(false) Value_impl(const Value_impl& other);
 
 	bool operator==(const Value_impl& lhs) const;
 
diff --git a/fdbclient/sha1/SHA1.cpp b/fdbclient/sha1/SHA1.cpp
index af31c019186..5ce7380d09d 100644
--- a/fdbclient/sha1/SHA1.cpp
+++ b/fdbclient/sha1/SHA1.cpp
@@ -13,10 +13,8 @@
         -- Volker Grabsch <vog@notjusthosting.com>
 */
 
-#include "fdbclient/sha1/SHA1.h"
+#include "SHA1.h"
 #include <sstream>
-#include <iomanip>
-#include <fstream>
 
 /* Help macros */
 #define SHA1_ROL(value, bits) (((value) << (bits)) | (((value) & 0xffffffff) >> (32 - (bits))))
diff --git a/fdbclient/include/fdbclient/sha1/SHA1.h b/fdbclient/sha1/SHA1.h
similarity index 100%
rename from fdbclient/include/fdbclient/sha1/SHA1.h
rename to fdbclient/sha1/SHA1.h
diff --git a/fdbclient/tests/gcp_fixture.sh b/fdbclient/tests/gcp_fixture.sh
new file mode 100644
index 00000000000..af6b56247fd
--- /dev/null
+++ b/fdbclient/tests/gcp_fixture.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+#
+# Functions for dealing w/ gcp/GCS
+#
+# Here is how to use this fixture:
+#
+# - set GCS_FDB_BUCKET and GCS_APPLICATION_TOKEN
+#
+#  # First source this fixture.
+#  if ! source "${cwd}/gcp_fixture.sh"; then
+#    err "Failed to source gcp_fixture.sh"
+#    exit 1
+#  fi
+#  # Save off the return from create_gcp_dir. You'll need it for shutdown.
+#  if ! TEST_SCRATCH_DIR=$( create_gcp_dir "${scratch_dir}" ); then
+#    err "Failed creating local gcp_dir"
+#    exit 1
+#  fi
+#  readonly TEST_SCRATCH_DIR
+#  if ! readarray -t configs < <(gcp_setup "${build_dir}" "${TEST_SCRATCH_DIR}"); then
+#    err "Failed gcp_setup"
+#    return 1
+#  fi
+#  readonly host="${configs[0]}"
+#  ...etc.
+#  # When done, call shutdown_gcp
+#  shutdown_gcp "${TEST_SCRATCH_DIR}"
+#
+
+# Cleanup any mess we've made. For calling from signal trap on exit.
+# $1 The gcp scratch directory to clean up on exit.
+function shutdown_gcp {
+  local local_scratch_dir="${1}"
+  if [[ -d "${local_scratch_dir}" ]]; then
+    rm -rf "${local_scratch_dir}"
+  fi
+}
+
+# Create directory for gcp test to use writing temporary data.
+# Call shutdown_gcp to clean up the directory created here for test.
+# $1 Directory where we want to write data and logs.
+# check $? error code on return.
+function create_gcp_dir {
+  local dir="${1}"
+  local gcp_dir
+  gcp_dir=$(mktemp -d "${dir}/gcp.$$.XXXX")
+  # Exit if the temp directory wasn't created successfully.
+  if [[ ! -d "${gcp_dir}" ]]; then
+    echo "ERROR: Failed create of gcp directory ${gcp_dir}" >&2
+    return 1
+  fi
+  echo "${gcp_dir}"
+}
+
+# Write out blob_credentials
+# $1 The token to use querying credentials
+# $2 Hostname going to gcp
+# $3 The scratch dir to write the blob credentials file into
+# Echos the blob_credentials file path.
+function write_gcp_blob_credentials {
+  local token="${1}"
+  local host="${2}"
+  local dir="${3}"
+  blob_credentials_str="{\"accounts\": { \"@${host}\": {\"token\": \"${token}\"}}}"
+  readonly blob_credentials_file="${dir}/blob_credentials.json"
+  echo "${blob_credentials_str}" > "${blob_credentials_file}"
+  echo "${blob_credentials_file}"
+}
+
+# Set up GCS access.
+# $1 build_dir
+# $2 gcp_dir (This is what is returned when you call create_gcp_dir
+# so call it first).
+# Returns array of configurations to use contacting GCS.
+function gcp_setup {
+  if [ -z "$GCS_APPLICATION_TOKEN" ]; then
+    echo "Error: GCS_APPLICATION_TOKEN is not set."
+    exit 1
+  fi
+  if [ -z "$GCS_FDB_BUCKET" ]; then
+    echo "Error: GCS_FDB_BUCKET is not set."
+    exit 1
+  fi
+  local local_build_dir="${1}"
+  local local_gcp_dir="${2}"
+
+  readonly host="storage.googleapis.com"
+  if ! blob_credentials_file=$(write_gcp_blob_credentials "${GCS_APPLICATION_TOKEN}" "${host}" "${local_gcp_dir}"); then
+    err "Failed to write credentials file"
+    exit 1
+  fi
+  results_array=("@${host}" "${GCS_FDB_BUCKET}" "${blob_credentials_file}")
+  printf "%s\n" "${results_array[@]}"
+}
diff --git a/fdbclient/tests/s3client_test.sh b/fdbclient/tests/s3client_test.sh
index 0c001c53b6f..496e77734c3 100755
--- a/fdbclient/tests/s3client_test.sh
+++ b/fdbclient/tests/s3client_test.sh
@@ -35,6 +35,7 @@ function filter_http_debug {
   echo "${output}" | grep -v "Contents of" | grep -v "^$" | grep -v "^[[:space:]]*$" | grep -v "HTTP" | grep -v "^\[.*\]" | grep -v "^Request Header:" | grep -v "^Response Header:" | grep -v "^Response Code:" | grep -v "^Response ContentLen:" | grep -v "^-- RESPONSE CONTENT--" | grep -v "^--------" | grep -v "^<?xml" | grep -v "^<.*>" | grep -v "^'$" | grep -v "^++"
 }
 
+
 # Run s3client with proper TLS CA file handling
 # This function handles the case where TLS_CA_FILE might be empty
 function run_s3client {
@@ -250,7 +251,7 @@ function test_nonexistent_bucket {
       # 1. The command succeeded (status 0)
       # 2. The output contains the URL header
       # 3. There are no objects listed (no lines after the header)
-      if echo "${output}" | grep -q "Contents of" &&
+      if output_contains "${output}" "Contents of" &&
          [[ $(filter_http_debug "${output}" | wc -l) -eq 0 ]]; then
         # Success - we have the header and no other non-empty lines (excluding HTTP debug lines)
         return 0
@@ -293,7 +294,7 @@ function test_nonexistent_resource {
     local filtered_output
     filtered_output=$(filter_http_debug "${output}")
     
-    if ! (echo "${output}" | grep -q "Contents of" &&
+    if ! (output_contains "${output}" "Contents of" &&
           [[ -z "$(echo "${filtered_output}" | tr -d '[:space:]')" ]]); then
       err "Failed to detect non-existent resource in S3"
       return 1
@@ -307,8 +308,8 @@ function test_nonexistent_resource {
       # 1. The command succeeded (status 0)
       # 2. The output contains the URL header
       # 3. There are no actual object listings (no lines starting with FILE or DIR)
-      if echo "${output}" | grep -q "Contents of" && 
-         ! echo "${output}" | grep -q "^FILE\|^DIR"; then
+      if output_contains "${output}" "Contents of" && 
+         ! output_matches "${output}" "^FILE\|^DIR"; then
         # Success - we have the header and no object listings
         return 0
       else
@@ -367,8 +368,8 @@ function test_empty_bucket {
     local filtered_output
     filtered_output=$(filter_http_debug "${output}")
     
-    if echo "${output}" | grep -q "No objects found" || 
-       (echo "${output}" | grep -q "Contents of" && 
+    if output_contains "${output}" "No objects found" ||
+       (output_contains "${output}" "Contents of" && 
         [[ -z "$(echo "${filtered_output}" | tr -d '[:space:]')" ]]); then
       log "Successfully handled empty bucket listing"
       return 0
@@ -466,7 +467,7 @@ function test_list_with_files {
 
     local missing=0
     for i in $(seq 1 "${file_count}"); do
-      if ! echo "${output}" | grep -q "ls_test/file${i}"; then
+      if ! output_contains "${output}" "ls_test/file${i}"; then
         err "Missing file${i} in ls output"
         missing=1
       fi
@@ -538,7 +539,7 @@ function test_list_with_files {
   local output
   local status
   output=$(run_s3client "${s3client}" "${credentials}" "${logsdir}" "false" \
-    --knob_blobstore_list_max_keys_per_page=100 ls --recursive "${url}" 2>&1)
+    --knob_blobstore_list_max_keys_per_page=100 ls --recursive "${url}" 2>/dev/null)
   status=$?
 
   local missing=0
@@ -549,10 +550,10 @@ function test_list_with_files {
 
     for i in $(seq 1 "${files_per_level}"); do
       local expected="${current_path}/file${current_depth}_${i}"
-      if ! echo "${output}" | grep -q "${expected}"; then
+      if ! output_contains "${output}" "${expected}"; then
         err "Missing ${expected} in ls output"
         log "=== DEBUG: Recursive ls output ==="
-        echo "${output}" | grep -v "HTTP" | head -30
+        awk '!/HTTP/ { print; if (++count == 30) exit }' <<<"${output}"
         log "=== END DEBUG ==="
         missing=1
       fi
@@ -580,7 +581,7 @@ function test_list_with_files {
   
   while [[ $non_recursive_attempt -lt $non_recursive_attempts ]]; do
     output=$(run_s3client "${s3client}" "${credentials}" "${logsdir}" "false" \
-      --knob_blobstore_list_max_keys_per_page=10 ls "${url}" 2>&1)
+      --knob_blobstore_list_max_keys_per_page=10 ls "${url}" 2>/dev/null)
     status=$?
     
     # Reset missing flag for this attempt
@@ -799,6 +800,32 @@ if [[ "${USE_S3}" == "true" ]]; then
     query_str='bucket='"${bucket}"'&region='"${region}"'&secure_connection=1'
   fi
   path_prefix="bulkload/test/s3client"
+elif [[ "${USE_MOCK_GCS:-false}" == "true" ]]; then
+  log "Testing GCS against MockS3Server"
+  # shellcheck source=/dev/null
+  if ! source "${cwd}/mocks3_fixture.sh"; then
+    err "Failed to source mocks3_fixture.sh"
+    exit 1
+  fi
+  if ! TEST_SCRATCH_DIR=$(mktemp -d "${scratch_dir}/mockgcs_test.XXXXXX"); then
+    err "Failed create of the mockgcs test dir." >&2
+    exit 1
+  fi
+  readonly TEST_SCRATCH_DIR
+  if ! start_mocks3 "${build_dir}" "${TEST_SCRATCH_DIR}/mocks3_data"; then
+    err "Failed to start MockS3Server"
+    exit 1
+  fi
+  readonly host="${MOCKS3_HOST}:${MOCKS3_PORT}"
+  readonly bucket="test-bucket"
+  readonly region=""
+  # GCS-format credential file with Bearer token.
+  # Key by hostname only (not host:port) — GCSBlobStoreEndpoint::credentialFileKey() is "@" + parsed host.
+  readonly blob_credentials_file="${TEST_SCRATCH_DIR}/blob_credentials.json"
+  echo "{\"accounts\": {\"@${MOCKS3_HOST}\": {\"token\": \"mock-gcs-token\"}}}" > "${blob_credentials_file}"
+  query_str='bucket='"${bucket}"'&p=gcs&secure_connection=0'
+  path_prefix="gcsclient"
+
 else
   log "Testing against MockS3Server"
   # Now source in the mocks3 fixture so we can use its methods in the below.
diff --git a/fdbclient/tests/tests_common.sh b/fdbclient/tests/tests_common.sh
index 821059d515d..e41332a07b8 100644
--- a/fdbclient/tests/tests_common.sh
+++ b/fdbclient/tests/tests_common.sh
@@ -96,6 +96,44 @@ function err {
   echo "$(date -Iseconds) ERROR: ${*}" >&2
 }
 
+# Check whether captured output contains a literal string.
+# Avoids echo|grep -q because pipefail can turn grep's early-exit into
+# a false negative via SIGPIPE on the upstream echo.
+# $1 output to search
+# $2 literal string to find
+function output_contains {
+  local output="$1"
+  local needle="$2"
+  grep -Fq -- "${needle}" <<<"${output}"
+}
+
+# Check whether captured output matches a basic regex.
+# $1 output to search
+# $2 regex pattern
+function output_matches {
+  local output="$1"
+  local pattern="$2"
+  grep -q -- "${pattern}" <<<"${output}"
+}
+
+# Case-insensitive regex match.
+# $1 output to search
+# $2 regex pattern
+function output_matches_i {
+  local output="$1"
+  local pattern="$2"
+  grep -qi -- "${pattern}" <<<"${output}"
+}
+
+# Extended regex match.
+# $1 output to search
+# $2 extended regex pattern
+function output_matches_E {
+  local output="$1"
+  local pattern="$2"
+  grep -qE -- "${pattern}" <<<"${output}"
+}
+
 # Check if test data should be preserved (PRESERVE_TEST_DATA=1)
 # If yes, prints preservation message and returns 0 (should skip cleanup)
 # If no, returns 1 (should continue with normal cleanup)
@@ -288,7 +326,7 @@ function log_test_result {
 # $1 Dir to search under.
 function grep_for_severity40 {
   local dir="${1}"
-  if grep -r -C 3 -e "Severity=\"40\"" "${dir}"; then
+  if grep -r --exclude-dir=encryption_mismatch_logs -C 3 -e "Severity=\"40\"" "${dir}"; then
     err "Found 'Severity=40' errors"
     return 1
   fi
@@ -324,18 +362,50 @@ function get_use_s3_default {
   fi
 }
 
-# Common S3/MockS3 environment setup - shared across all S3 tests
-# Prerequisites: USE_S3 and TLS_CA_FILE must be set before calling this function
+# Detect which blob store provider to use based on environment variables.
+# Sets USE_GCS global (USE_S3 must already be set).
+function detect_blobstore_provider {
+  USE_GCS="$( if [[ -n "${GCS_FDB_BUCKET:-}" && -n "${GCS_APPLICATION_TOKEN:-}" ]]; then echo "true"; else echo "false"; fi )"
+  readonly USE_GCS
+}
+
+# Common blobstore environment setup - shared across all blob store tests
+# Supports S3, GCS, and MockS3Server.
+# Prerequisites: USE_S3, USE_GCS, and TLS_CA_FILE must be set before calling this function
 #   (use get_use_s3_default and setup_tls_ca_file)
 # $1 build directory, $2 scratch directory, $3 path prefix (used for temp dir naming)
 # Sets global variables: TEST_SCRATCH_DIR, host, bucket, region, blob_credentials_file, query_str
-# Exports: FDB_BLOB_CREDENTIALS, FDB_TLS_CA_FILE (if using real S3)
+# Exports: FDB_BLOB_CREDENTIALS, FDB_TLS_CA_FILE (if using a real cloud provider)
 function setup_s3_environment {
   local local_build_dir="${1}"
   local local_scratch_dir="${2}"
   local local_path_prefix="${3}"
 
-  if [[ "${USE_S3}" == "true" ]]; then
+  if [[ "${USE_GCS:-false}" == "true" ]]; then
+    log "Testing against GCS"
+    if ! source "${TESTS_COMMON_DIR}/gcp_fixture.sh"; then
+      err "Failed to source gcp_fixture.sh"
+      exit 1
+    fi
+    if ! TEST_SCRATCH_DIR=$( create_gcp_dir "${local_scratch_dir}" ); then
+      err "Failed creating local gcp_dir"
+      exit 1
+    fi
+    if ! readarray -t configs < <(gcp_setup "${local_build_dir}" "${TEST_SCRATCH_DIR}"); then
+      err "Failed gcp_setup"
+      return 1
+    fi
+    host="${configs[0]}"
+    bucket="${configs[1]}"
+    blob_credentials_file="${configs[2]}"
+    region=""
+    query_str="bucket=${bucket}&p=gcs&secure_connection=1"
+    export FDB_BLOB_CREDENTIALS="${blob_credentials_file}"
+    if [[ -n "${TLS_CA_FILE:-}" ]]; then
+      export FDB_TLS_CA_FILE="${TLS_CA_FILE}"
+    fi
+
+  elif [[ "${USE_S3}" == "true" ]]; then
     log "Testing against s3"
     # Source AWS fixture (use TESTS_COMMON_DIR for reliable path resolution)
     if ! source "${TESTS_COMMON_DIR}/aws_fixture.sh"; then
@@ -359,6 +429,7 @@ function setup_s3_environment {
     if [[ -n "${TLS_CA_FILE:-}" ]]; then
       export FDB_TLS_CA_FILE="${TLS_CA_FILE}"
     fi
+
   else
     log "Testing against MockS3Server"
     # Source MockS3 fixture (use TESTS_COMMON_DIR for reliable path resolution)
@@ -391,9 +462,9 @@ function setup_s3_environment {
   readonly query_str
 }
 
-# Setup TLS CA file for S3 connections
+# Setup TLS CA file for cloud provider connections (S3, GCS)
 function setup_tls_ca_file {
-  if [[ "${USE_S3}" == "true" ]]; then
+  if [[ "${USE_S3}" == "true" || "${USE_GCS:-false}" == "true" ]]; then
     # Try to find a valid TLS CA file if not explicitly set
     if [[ -z "${TLS_CA_FILE:-}" ]]; then
       # Common locations for TLS CA files on different systems
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index ca2c966ee7c..013681b942f 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -321,7 +321,7 @@ description is not currently required but encouraged.
     <Option name="used_during_commit_protection_disable" code="701"
             description="By default, operations that are performed on a transaction while it is being committed will not only fail themselves, but they will attempt to fail other in-flight operations (such as the commit) as well. This behavior is intended to help developers discover situations where operations could be unintentionally executed after the transaction has been reset. Setting this option removes that protection, causing only the offending operation to fail."/>
     <Option name="read_lock_aware" code="702"
-            description="The transaction can read from locked databases."/>
+            description="The transaction can read from locked databases. This option will make the transaction read-only if the lock_aware option is not set."/>
     <Option name="first_in_batch" code="710"
             description="No other transactions will be applied before this transaction within the same commit version."
             hidden="true" />
@@ -348,6 +348,9 @@ description is not currently required but encouraged.
     <Option name="skip_grv_cache" code="1102"
             description="Specifically instruct this transaction to NOT use cached GRV. Primarily used for the read version cache's background updater to avoid attempting to read a cached entry in specific situations."
             hidden="true"/>
+    <Option name="max_grv_queue_delay" code="1103"
+            paramType="Int" paramDescription="value in milliseconds of maximum estimated GRV queue delay"
+            description="Rejects a GRV request with ``transaction_grv_queue_rejected`` if the GRV proxy estimates that ratekeeper throttling will keep the request queued longer than this limit. Valid parameter values are ``[0, INT_MAX]``. The estimate is advisory and only accounts for time spent queued at the GRV proxy due to ratekeeper throttling." />
     <Option name="authorization_token" code="2000"
             description="Attach given authorization token to the transaction such that subsequent tenant-aware requests are authorized"
             paramType="String" paramDescription="A JSON Web Token authorized to access data belonging to one or more tenants, indicated by 'tenants' claim of the token's payload."
diff --git a/fdbctl/CMakeLists.txt b/fdbctl/CMakeLists.txt
index 54cbdf7a088..545227e7f7e 100644
--- a/fdbctl/CMakeLists.txt
+++ b/fdbctl/CMakeLists.txt
@@ -2,7 +2,7 @@ fdb_find_sources(FDBCTL_SRCS)
 
 add_flow_target(STATIC_LIBRARY NAME fdbctl
   SRCS ${FDBCTL_SRCS})
-target_include_directories(fdbctl PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_include_directories(fdbctl PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
 target_link_libraries(fdbctl PRIVATE fdbclient)
 
 if (WITH_GRPC)
diff --git a/fdbctl/ControlCommands.cpp b/fdbctl/ControlCommands.cpp
index 3c735aa88a7..05a630c3522 100644
--- a/fdbctl/ControlCommands.cpp
+++ b/fdbctl/ControlCommands.cpp
@@ -19,7 +19,7 @@
  */
 #ifdef FLOW_GRPC_ENABLED
 #include "fdbctl/ControlCommands.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/Schemas.h"
 #include "fmt/format.h"
 #include <boost/algorithm/string.hpp>
@@ -72,13 +72,13 @@ Future<grpc::Status> changeCoordinators(Reference<IDatabase> db,
 	loop {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		try {
-			if (req->has_cluster_description() && req->cluster_description().size()) {
+			if (req->has_cluster_description() && !req->cluster_description().empty()) {
 				tr->set(special_keys::clusterDescriptionSpecialKey, req->cluster_description());
 			}
 
 			if (req->automatic_coordinators()) {
 				ASSERT(req->new_coordinator_addresses_size() == 0);
-				if (!auto_coordinators_str.size()) {
+				if (auto_coordinators_str.empty()) {
 					ThreadFuture<Optional<Value>> auto_coordinatorsF =
 					    tr->get(special_keys::coordinatorsAutoSpecialKey);
 					Optional<Value> auto_coordinators = co_await safeThreadFutureToFuture(auto_coordinatorsF);
@@ -95,7 +95,7 @@ Future<grpc::Status> changeCoordinators(Reference<IDatabase> db,
 					try {
 						if (Hostname::isHostname(new_coord)) {
 							const auto& hostname = Hostname::parse(new_coord);
-							if (new_coordinators_hostnames.count(hostname)) {
+							if (new_coordinators_hostnames.contains(hostname)) {
 								co_return grpc::Status(
 								    grpc::StatusCode::INVALID_ARGUMENT,
 								    fmt::format("passed redundant coordinators: {}", hostname.toString()));
@@ -104,7 +104,7 @@ Future<grpc::Status> changeCoordinators(Reference<IDatabase> db,
 							newCoordinatorsList.push_back(hostname.toString());
 						} else {
 							const auto& addr = NetworkAddress::parse(new_coord);
-							if (new_coordinators_addresses.count(addr)) {
+							if (new_coordinators_addresses.contains(addr)) {
 								co_return grpc::Status(
 								    grpc::StatusCode::INVALID_ARGUMENT,
 								    fmt::format("passed redundant coordinators: {}", addr.toString()));
@@ -362,8 +362,7 @@ Future<grpc::Status> include(Reference<IDatabase> db, const IncludeRequest* req,
 void addInterfacesFromKVs(RangeResult& kvs,
                           std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface) {
 	for (const auto& kv : kvs) {
-		ClientWorkerInterface workerInterf =
-		    BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
+		auto workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(kv.value, IncludeVersion());
 		ClientLeaderRegInterface leaderInterf(workerInterf.address());
 		StringRef ip_port = (kv.key.endsWith(":tls"_sr) ? kv.key.removeSuffix(":tls"_sr) : kv.key)
 		                        .removePrefix("\xff\xff/worker_interfaces/"_sr);
@@ -404,7 +403,7 @@ Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
 Future<grpc::Status> kill(Reference<IDatabase> db, const KillRequest* req, KillReply* rep) {
 	// TODO: Handle duration_seconds
 	// TODO: What if asked to kill itself?
-	if (req->all() && req->addresses().size() > 0) {
+	if (req->all() && !req->addresses().empty()) {
 		co_return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "ERROR: both fields `all` and `addresses` set");
 	}
 
@@ -412,7 +411,7 @@ Future<grpc::Status> kill(Reference<IDatabase> db, const KillRequest* req, KillR
 	std::map<Key, std::pair<Value, ClientLeaderRegInterface>> address_interface;
 	co_await getWorkerInterfaces(tr, &address_interface, true);
 
-	if (address_interface.size() == 0) {
+	if (address_interface.empty()) {
 		// TODO: Return more elaborate result.
 		co_return grpc::Status::OK;
 	}
@@ -422,9 +421,9 @@ Future<grpc::Status> kill(Reference<IDatabase> db, const KillRequest* req, KillR
 		for (const auto& [address, _] : address_interface) {
 			addressesVec.push_back(address.toString());
 		}
-	} else if (req->addresses().size() > 0) {
+	} else if (!req->addresses().empty()) {
 		for (const auto& addr : req->addresses()) {
-			if (!address_interface.count(StringRef(addr))) {
+			if (!address_interface.contains(StringRef(addr))) {
 				co_return grpc::Status(grpc::StatusCode::UNKNOWN, fmt::format("process `{}' not recognized", addr));
 			}
 			addressesVec.push_back(addr);
@@ -469,7 +468,7 @@ Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			ThreadFuture<RangeResult> serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-			co_await success(safeThreadFutureToFuture(serverListF));
+			co_await safeThreadFutureToFuture(serverListF);
 			ASSERT(!serverListF.get().more);
 			ASSERT_LT(serverListF.get().size(), CLIENT_KNOBS->TOO_MANY);
 			RangeResult serverList = serverListF.get();
@@ -503,8 +502,8 @@ Future<bool> getWorkersProcessData(Reference<IDatabase> db, std::vector<ProcessD
 			ThreadFuture<RangeResult> processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
 			ThreadFuture<RangeResult> processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
 
-			co_await success(safeThreadFutureToFuture(processClasses));
-			co_await success(safeThreadFutureToFuture(processData));
+			co_await safeThreadFutureToFuture(processClasses);
+			co_await safeThreadFutureToFuture(processData);
 
 			ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY);
diff --git a/fdbctl/ExcludeCommand.cpp b/fdbctl/ExcludeCommand.cpp
index 77e9a5ee181..6f4cff3bf79 100644
--- a/fdbctl/ExcludeCommand.cpp
+++ b/fdbctl/ExcludeCommand.cpp
@@ -22,12 +22,11 @@ I * Licensed under the Apache License, Version 2.0 (the "License");
 #include "fdbctl/ControlCommands.h"
 
 #include "fdbclient/IClientApi.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/Schemas.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/genericactors.actor.h"
-#include "fdbctl/ControlCommands.h"
 #include "fmt/format.h"
 #include <boost/algorithm/string.hpp>
 
@@ -178,7 +177,7 @@ Future<Void> excludeServersAndLocalities(Reference<IDatabase> db,
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		try {
-			if (force && servers.size())
+			if (force && !servers.empty())
 				tr->set(failed ? special_keys::failedForceOptionSpecialKey
 				               : special_keys::excludedForceOptionSpecialKey,
 				        ValueRef());
@@ -187,7 +186,7 @@ Future<Void> excludeServersAndLocalities(Reference<IDatabase> db,
 				                  : special_keys::excludedServersSpecialKeyRange.begin.withSuffix(s.toString());
 				tr->set(addr, ValueRef());
 			}
-			if (force && localities.size())
+			if (force && !localities.empty())
 				tr->set(failed ? special_keys::failedLocalityForceOptionSpecialKey
 				               : special_keys::excludedLocalityForceOptionSpecialKey,
 				        ValueRef());
@@ -284,7 +283,7 @@ Future<grpc::Status> exclude(Reference<IDatabase> db, const ExcludeRequest* req,
 		Future<bool> future_workers = utils::getWorkersProcessData(db, &workers);
 		Future<Void> future_server_interfaces = utils::getStorageServerInterfaces(db, &server_interfaces);
 
-		co_await success(future_workers);
+		co_await future_workers;
 		co_await future_server_interfaces;
 
 		bool force = req->force();
@@ -358,7 +357,7 @@ Future<grpc::Status> exclude(Reference<IDatabase> db, const ExcludeRequest* req,
 			auto worker = workerPorts.find(addr.ip);
 			if (worker == workerPorts.end())
 				absentExclusions.insert(addr);
-			else if (addr.port > 0 && worker->second.count(addr.port) == 0)
+			else if (addr.port > 0 && !worker->second.contains(addr.port))
 				absentExclusions.insert(addr);
 		}
 
@@ -419,4 +418,4 @@ Future<grpc::Status> excludeStatus(Reference<IDatabase> db, const ExcludeStatusR
 
 } // namespace fdbctl
 
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbctl/include/fdbctl/ControlService.h b/fdbctl/include/fdbctl/ControlService.h
index 18d19bfc43c..e262ea86a66 100644
--- a/fdbctl/include/fdbctl/ControlService.h
+++ b/fdbctl/include/fdbctl/ControlService.h
@@ -25,7 +25,6 @@
 #include <grpcpp/support/status.h>
 
 #include "fdbctl/ControlCommands.h"
-#include "fdbclient/ClientKnobs.h"
 #include "fdbclient/IClientApi.h"
 #include "fdbclient/Knobs.h"
 #include "flow/ThreadHelper.actor.h"
@@ -72,7 +71,7 @@ Future<grpc::Status> grpcHandlerWrapper(Reference<IDatabase> db,
 
 class ControlServiceImpl final : public fdbctl::ControlService::Service {
 public:
-	ControlServiceImpl(Reference<IDatabase> db);
+	explicit ControlServiceImpl(Reference<IDatabase> db);
 
 	DEFINE_GRPC_HANDLER(GetCoordinators, getCoordinators);
 	DEFINE_GRPC_HANDLER(ChangeCoordinators, changeCoordinators);
diff --git a/fdbctl/protos/control_service.proto b/fdbctl/protos/control_service.proto
index 22a6b96b69f..19d0e27240e 100644
--- a/fdbctl/protos/control_service.proto
+++ b/fdbctl/protos/control_service.proto
@@ -253,7 +253,7 @@ message ConfigureRequest {
 // from error_definitions.h?
 message ConfigureReply {
     // Result codes indicating the outcome of the configuration operation
-    // Refer: GenericManagementAPI.actor.h::ConfigurationResult
+    // Refer: GenericManagementAPI.h::ConfigurationResult
     enum Result {
         SUCCESS = 0;                                  // Configuration succeeded
         NO_OPTIONS_PROVIDED = 1;                      // No configuration options were provided
diff --git a/fdbkubernetesmonitor/CMakeLists.txt b/fdbkubernetesmonitor/CMakeLists.txt
new file mode 100644
index 00000000000..0acc95129d3
--- /dev/null
+++ b/fdbkubernetesmonitor/CMakeLists.txt
@@ -0,0 +1,43 @@
+find_program(GO_EXECUTABLE go HINTS /usr/local/go/bin/)
+if(NOT GO_EXECUTABLE)
+  message(WARNING "Go not found. The 'fdb-kubernetes-monitor' target will not be available.")
+  return()
+endif()
+
+if(WIN32)
+  return()
+endif()
+
+file(GLOB_RECURSE FDB_K8S_MONITOR_GO_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.go")
+set(FDB_K8S_MONITOR_GO_MOD "${CMAKE_CURRENT_SOURCE_DIR}/go.mod")
+set(FDB_K8S_MONITOR_GO_SUM "${CMAKE_CURRENT_SOURCE_DIR}/go.sum")
+
+set(FDB_K8S_MONITOR_BINARY "${CMAKE_BINARY_DIR}/bin/fdb-kubernetes-monitor")
+
+add_custom_command(
+  OUTPUT ${FDB_K8S_MONITOR_BINARY}
+  COMMAND ${GO_EXECUTABLE} build -o ${FDB_K8S_MONITOR_BINARY} .
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS ${FDB_K8S_MONITOR_GO_SOURCES} ${FDB_K8S_MONITOR_GO_MOD} ${FDB_K8S_MONITOR_GO_SUM}
+  COMMENT "Building fdb-kubernetes-monitor")
+
+add_custom_target(fdb-kubernetes-monitor ALL
+  DEPENDS ${FDB_K8S_MONITOR_BINARY}
+  SOURCES ${FDB_K8S_MONITOR_GO_SOURCES})
+
+add_test(
+  NAME fdb-kubernetes-monitor-go-tests
+  COMMAND ${GO_EXECUTABLE} test -race ./...
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+# gofmt should be installed with go, but it's better to check if the binary is available.
+find_program(GOFMT_EXECUTABLE gofmt HINTS /usr/local/go/bin/)
+if(GOFMT_EXECUTABLE)
+  add_test(
+    NAME fdb-kubernetes-monitor-go-fmt
+    # gofmt -d doesn't return a non-zero exit code, so we have to check if the output is empty.
+    COMMAND bash -c "out=$(\"${GOFMT_EXECUTABLE}\" -d \"${CMAKE_CURRENT_SOURCE_DIR}\"); test -z \"$out\" || { echo \"$out\"; exit 1; }"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+else()
+  message(WARNING "gofmt not found. The 'fdb-kubernetes-monitor-go-fmt' test will not be available.")
+endif()
diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md
index c2618a21958..3237703f9e2 100644
--- a/fdbkubernetesmonitor/README.md
+++ b/fdbkubernetesmonitor/README.md
@@ -14,8 +14,8 @@ To test this, run the following commands from the root of the FoundationDB repos
 
 ```bash
 mkdir -p website
-docker build -t foundationdb/fdb-kubernetes-monitor:7.1.5 --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.1.5 --build-arg FDB_LIBRARY_VERSIONS="7.1.5 6.3.24 6.2.30" -f packaging/docker/Dockerfile .
-docker build -t foundationdb/fdb-kubernetes-monitor:7.1.6 --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.1.6 --build-arg FDB_LIBRARY_VERSIONS="7.1.6 6.3.24 6.2.30" -f packaging/docker/Dockerfile .
+docker build -t foundationdb/fdb-kubernetes-monitor:7.3.73 --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.3.73 -f packaging/docker/Dockerfile .
+docker build -t foundationdb/fdb-kubernetes-monitor:7.3.75 --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.3.75 -f packaging/docker/Dockerfile .
 kubectl apply -f packaging/docker/kubernetes/test_config.yaml
 # Wait for the pods to become ready
 kubectl rollout status sts/fdb-kubernetes-example
@@ -55,7 +55,7 @@ The following steps assume that you already have a [local development](https://g
 ```bash
 mkdir -p website
 # Change this version if you want to create a cluster with a different version
-docker build -t foundationdb/fdb-kubernetes-monitor:7.1.11 --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.1.11 --build-arg FDB_LIBRARY_VERSIONS="7.1.11 6.3.24 6.2.30" -f packaging/docker/Dockerfile .
+docker build -t foundationdb/fdb-kubernetes-monitor:7.3.73 --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.3.73 -f packaging/docker/Dockerfile .
 ```
 
 Depending on the Kubernetes setup you use you might have to push the newly build image to a local registry.
diff --git a/fdbkubernetesmonitor/api/version_test.go b/fdbkubernetesmonitor/api/version_test.go
index 9062a5a6233..94957578abd 100644
--- a/fdbkubernetesmonitor/api/version_test.go
+++ b/fdbkubernetesmonitor/api/version_test.go
@@ -21,16 +21,13 @@
 package api
 
 import (
-	//"encoding/json"
-	//"os"
+	"encoding/json"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("[api] FDBVersion", func() {
-	// TODO test a json marshal and unmarshl!
-
 	When("checking if the protocol and the version are compatible", func() {
 		It("should return the correct compatibility", func() {
 			version := Version{Major: 6, Minor: 2, Patch: 20}
@@ -80,6 +77,18 @@ var _ = Describe("[api] FDBVersion", func() {
 			Expect(err.Error()).To(Equal("could not parse FDB version from 6.2"))
 		})
 
+		It("should marshal and unmarshal JSON strings", func() {
+			version := Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 39}
+			data, err := json.Marshal(&version)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(data).To(Equal([]byte(`"7.1.0-rc39"`)))
+
+			var decoded Version
+			err = json.Unmarshal(data, &decoded)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decoded).To(Equal(version))
+		})
+
 		It("should format the version correctly", func() {
 			version := Version{Major: 6, Minor: 2, Patch: 11}
 			Expect(version.String()).To(Equal("6.2.11"))
diff --git a/fdbkubernetesmonitor/go.mod b/fdbkubernetesmonitor/go.mod
index 3a469e57561..d33e317e43c 100644
--- a/fdbkubernetesmonitor/go.mod
+++ b/fdbkubernetesmonitor/go.mod
@@ -19,40 +19,38 @@
 
 module github.com/apple/foundationdb/fdbkubernetesmonitor
 
-go 1.24.0
-
-toolchain go1.24.4
+go 1.25.0
 
 require (
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-logr/logr v1.4.3
 	github.com/go-logr/zapr v1.3.0
-	github.com/onsi/ginkgo/v2 v2.23.4
-	github.com/onsi/gomega v1.37.0
-	github.com/prometheus/client_golang v1.22.0
-	github.com/spf13/pflag v1.0.6
+	github.com/onsi/ginkgo/v2 v2.27.2
+	github.com/onsi/gomega v1.38.2
+	github.com/prometheus/client_golang v1.23.2
+	github.com/spf13/pflag v1.0.9
 	go.uber.org/zap v1.27.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
-	k8s.io/api v0.33.2
-	k8s.io/apimachinery v0.33.2
-	k8s.io/client-go v0.33.2
-	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
-	sigs.k8s.io/controller-runtime v0.21.0
+	k8s.io/api v0.35.0
+	k8s.io/apimachinery v0.35.0
+	k8s.io/client-go v0.35.0
+	k8s.io/utils v0.0.0-20251002143259-bc988d571ff4
+	sigs.k8s.io/controller-runtime v0.23.1
 )
 
 require (
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
-	github.com/fxamacker/cbor/v2 v2.8.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.1 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.1 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -60,33 +58,34 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/net v0.41.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
-	golang.org/x/tools v0.34.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
-	google.golang.org/protobuf v1.36.6 // indirect
-	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.33.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20250626183228-af0a60a813f8 // indirect
-	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect
-	sigs.k8s.io/yaml v1.5.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )
diff --git a/fdbkubernetesmonitor/go.sum b/fdbkubernetesmonitor/go.sum
index 62f66222201..5f1ac5f0f21 100644
--- a/fdbkubernetesmonitor/go.sum
+++ b/fdbkubernetesmonitor/go.sum
@@ -1,7 +1,7 @@
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
-github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -13,8 +13,14 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU=
-github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
+github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs=
+github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
+github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
+github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
+github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
+github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -27,11 +33,10 @@ github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZ
 github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
-github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
+github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -43,10 +48,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
+github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -57,134 +62,116 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
 github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
+github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
+github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
+github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
+github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
-github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
-github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
-github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
+github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
+github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
+github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
-github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
-github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
-github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
-github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
-github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
-go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE=
-go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
-golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
+golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
 golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
 golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
-golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
+golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
 gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0=
 gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
-gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
+gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.33.2 h1:YgwIS5jKfA+BZg//OQhkJNIfie/kmRsO0BmNaVSimvY=
-k8s.io/api v0.33.2/go.mod h1:fhrbphQJSM2cXzCWgqU29xLDuks4mu7ti9vveEnpSXs=
-k8s.io/apiextensions-apiserver v0.33.2 h1:6gnkIbngnaUflR3XwE1mCefN3YS8yTD631JXQhsU6M8=
-k8s.io/apiextensions-apiserver v0.33.2/go.mod h1:IvVanieYsEHJImTKXGP6XCOjTwv2LUMos0YWc9O+QP8=
-k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY=
-k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
-k8s.io/client-go v0.33.2 h1:z8CIcc0P581x/J1ZYf4CNzRKxRvQAwoAolYPbtQes+E=
-k8s.io/client-go v0.33.2/go.mod h1:9mCgT4wROvL948w6f6ArJNb7yQd7QsvqavDeZHvNmHo=
+k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY=
+k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA=
+k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4=
+k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU=
+k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8=
+k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
+k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE=
+k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kube-openapi v0.0.0-20250626183228-af0a60a813f8 h1:BbZLZQv9i6ROWiPa5sX3LGf3Sc0Ir7dtHwG2RXGmvRs=
-k8s.io/kube-openapi v0.0.0-20250626183228-af0a60a813f8/go.mod h1:4ZFs+FSGR/oEzT6/XaRojsx96IHMbqK7wOGt+fcEuRk=
-k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y=
-k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8=
-sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
-sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
-sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
-sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE=
+k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
+k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck=
+k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE=
+sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
 sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
 sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
-sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI=
-sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
-sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
-sigs.k8s.io/yaml v1.5.0 h1:M10b2U7aEUY6hRtU870n2VTPgR5RZiL/I6Lcc2F4NUQ=
-sigs.k8s.io/yaml v1.5.0/go.mod h1:wZs27Rbxoai4C0f8/9urLZtZtF3avA3gKvGyPdDqTO4=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
diff --git a/fdbkubernetesmonitor/internal/certloader/certloader.go b/fdbkubernetesmonitor/internal/certloader/certloader.go
index 6cd081a54f5..b88364e132f 100644
--- a/fdbkubernetesmonitor/internal/certloader/certloader.go
+++ b/fdbkubernetesmonitor/internal/certloader/certloader.go
@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -11,6 +12,8 @@ import (
 
 // CertLoader is used to reload certificates if needed.
 type CertLoader struct {
+	// mu protects cachedCert and cachedCertModTime from concurrent access.
+	mu sync.Mutex
 	// CertFile specifies the path to the x509 certificate.
 	CertFile string
 	// CertFile specifies the path to the x509 private key.
@@ -41,16 +44,23 @@ func (certLoader *CertLoader) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certi
 		return nil, err
 	}
 
-	if certLoader.cachedCert == nil || stat.ModTime().After(certLoader.cachedCertModTime) {
-		certLoader.logger.Info("loading new certificates", "certFile", certLoader.CertFile, "keyFile", certLoader.KeyFile, "cachedModificationTime", certLoader.cachedCertModTime.String(), "currentModificationTime", stat.ModTime().String())
-		pair, err := tls.LoadX509KeyPair(certLoader.CertFile, certLoader.KeyFile)
-		if err != nil {
-			return nil, fmt.Errorf("failed loading tls key pair: %w", err)
-		}
+	// Lock to avoid concurrent access to cachedCert and cachedCertModTime.
+	certLoader.mu.Lock()
+	defer certLoader.mu.Unlock()
+	cached := certLoader.cachedCert
+	modTime := certLoader.cachedCertModTime
 
-		certLoader.cachedCert = &pair
-		certLoader.cachedCertModTime = stat.ModTime()
+	if cached != nil && !stat.ModTime().After(modTime) {
+		return cached, nil
 	}
 
+	certLoader.logger.Info("loading new certificates", "certFile", certLoader.CertFile, "keyFile", certLoader.KeyFile, "cachedModificationTime", certLoader.cachedCertModTime.String(), "currentModificationTime", stat.ModTime().String())
+	pair, err := tls.LoadX509KeyPair(certLoader.CertFile, certLoader.KeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed loading tls key pair: %w", err)
+	}
+
+	certLoader.cachedCert = &pair
+	certLoader.cachedCertModTime = stat.ModTime()
 	return certLoader.cachedCert, nil
 }
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index f82e00def97..2bcafbe745c 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -28,6 +28,7 @@ import (
 	"strconv"
 	"time"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/client-go/util/retry"
 
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
@@ -48,12 +49,6 @@ var _ toolscache.ResourceEventHandler = (*kubernetesClient)(nil)
 
 // kubernetesClient is a wrapper around the Kubernetes API.
 type kubernetesClient struct {
-	// podMetadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according Pod.
-	podMetadata *metav1.PartialObjectMetadata
-
-	// nodeMetadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according node that hosts the Pod.
-	nodeMetadata *metav1.PartialObjectMetadata
-
 	// TimestampFeed is a channel where the kubernetes client will send updates with
 	// the values from api.OutdatedConfigMapAnnotation. If the api.IsolateProcessGroupAnnotation is changed the current
 	// timestamp will be sent to the channel to force the monitor to change its configuration accordingly.
@@ -65,8 +60,15 @@ type kubernetesClient struct {
 	// Adds the controller runtime client to the kubernetesClient.
 	client.Client
 
-	// patchType defines the patch type that should be used for patching the pod metadata.
-	patchType client.Patch
+	// namespace defines the namespace where the pod that hosts the fdbkubernetesmonitor instance is running in.
+	namespace string
+
+	// podName defines the Pod name where the fdbkubernetesmonitor instance is running in.
+	podName string
+
+	// nodeName defines the node name of the Pod where the fdbkubernetesmonitor instance is running in.
+	// This value is only set if the fdbkubernetesmonitor should watch node events.
+	nodeName string
 }
 
 func setupCache(namespace string, podName string, nodeName string) (client.WithWatch, cache.Cache, error) {
@@ -127,14 +129,17 @@ func createPodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 	}
 
 	podClient := &kubernetesClient{
-		podMetadata:   nil,
-		nodeMetadata:  nil,
+		podName:       podName,
+		namespace:     namespace,
 		TimestampFeed: make(chan int64, 10),
 		Logger:        logger,
-		patchType:     client.Apply,
 		Client:        internalClient,
 	}
 
+	if enableNodeWatcher {
+		podClient.nodeName = nodeName
+	}
+
 	// Fetch the informer for the Pod resource.
 	podInformer, err := internalCache.GetInformer(ctx, &corev1.Pod{})
 	if err != nil {
@@ -170,30 +175,36 @@ func createPodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 	// This should be fairly quick as no informers are provided by default.
 	_ = internalCache.WaitForCacheSync(ctx)
 
-	// Fetch the current metadata before returning the kubernetesClient
-	currentPodMetadata := &metav1.PartialObjectMetadata{}
-	currentPodMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod"))
-	err = podClient.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: podName}, currentPodMetadata)
+	return podClient, nil
+}
+
+// getPodMetadata returns a copy of the pod metadata.
+func (podClient *kubernetesClient) getPodMetadata(ctx context.Context) (*metav1.PartialObjectMetadata, error) {
+	metadata := &metav1.PartialObjectMetadata{}
+	metadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod"))
+	err := podClient.Client.Get(ctx, client.ObjectKey{Namespace: podClient.namespace, Name: podClient.podName}, metadata)
 	if err != nil {
 		return nil, err
 	}
 
-	podClient.podMetadata = currentPodMetadata
+	return metadata, nil
+}
 
-	// Only if the fdb-kubernetes-monitor should update the node information, add the watcher here by fetching the node
-	// information once during start up.
-	if enableNodeWatcher {
-		currentNodeMetadata := &metav1.PartialObjectMetadata{}
-		currentNodeMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node"))
-		err = podClient.Client.Get(ctx, client.ObjectKey{Name: nodeName}, currentNodeMetadata)
-		if err != nil {
-			return nil, err
-		}
+// getNodeMetadata returns a copy of the node metadata.
+func (podClient *kubernetesClient) getNodeMetadata(ctx context.Context) (*metav1.PartialObjectMetadata, error) {
+	// Only if the fdb-kubernetes-monitor is watching node events we should be returning the node metadata.
+	if podClient.nodeName == "" {
+		return nil, nil
+	}
 
-		podClient.nodeMetadata = currentNodeMetadata
+	metadata := &metav1.PartialObjectMetadata{}
+	metadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node"))
+	err := podClient.Client.Get(ctx, client.ObjectKey{Name: podClient.nodeName}, metadata)
+	if err != nil {
+		return nil, err
 	}
 
-	return podClient, nil
+	return metadata, nil
 }
 
 // retrieveEnvironmentVariables extracts the environment variables we have for
@@ -214,7 +225,7 @@ func retrieveEnvironmentVariables(monitor *monitor, argument api.Argument, targe
 
 // updateAnnotations updates annotations on the pod after loading new
 // configuration.
-func (podClient *kubernetesClient) updateAnnotations(monitor *monitor) error {
+func (podClient *kubernetesClient) updateAnnotations(ctx context.Context, monitor *monitor) error {
 	environment := make(map[string]string)
 	for _, argument := range monitor.activeConfiguration.Arguments {
 		retrieveEnvironmentVariables(monitor, argument, environment)
@@ -225,7 +236,7 @@ func (podClient *kubernetesClient) updateAnnotations(monitor *monitor) error {
 		return err
 	}
 
-	return podClient.updateAnnotationsOnPod(map[string]string{
+	return podClient.updateAnnotationsOnPod(ctx, map[string]string{
 		api.CurrentConfigurationAnnotation: string(monitor.activeConfigurationBytes),
 		api.EnvironmentAnnotation:          string(jsonEnvironment),
 	})
@@ -233,20 +244,25 @@ func (podClient *kubernetesClient) updateAnnotations(monitor *monitor) error {
 
 // updateFdbClusterTimestampAnnotation updates the ClusterFileChangeDetectedAnnotation annotation on the pod
 // after a change to the fdb.cluster file was detected, e.g. because the coordinators were changed.
-func (podClient *kubernetesClient) updateFdbClusterTimestampAnnotation() error {
-	return podClient.updateAnnotationsOnPod(map[string]string{
+func (podClient *kubernetesClient) updateFdbClusterTimestampAnnotation(ctx context.Context) error {
+	return podClient.updateAnnotationsOnPod(ctx, map[string]string{
 		api.ClusterFileChangeDetectedAnnotation: strconv.FormatInt(time.Now().Unix(), 10),
 	})
 }
 
 // updateAnnotationsOnPod will update the annotations with the provided annotationChanges. If an annotation exists, it
 // will be updated if the annotation is absent it will be added.
-func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
-	if podClient.podMetadata == nil {
+func (podClient *kubernetesClient) updateAnnotationsOnPod(ctx context.Context, annotationChanges map[string]string) error {
+	meta, err := podClient.getPodMetadata(ctx)
+	if err != nil {
+		return err
+	}
+
+	if meta == nil {
 		return fmt.Errorf("pod client has no metadata present")
 	}
 
-	if !podClient.podMetadata.DeletionTimestamp.IsZero() {
+	if !meta.DeletionTimestamp.IsZero() {
 		return fmt.Errorf("pod is marked for deletion, cannot update annotations")
 	}
 
@@ -254,7 +270,7 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 	return retry.OnError(retry.DefaultRetry, func(err error) bool {
 		podClient.Logger.Error(err, "could not update annotations of pod")
 		// If the pod is marked for deletion, we don't have to retry the patch.
-		if !podClient.podMetadata.DeletionTimestamp.IsZero() {
+		if !meta.DeletionTimestamp.IsZero() {
 			return false
 		}
 
@@ -265,7 +281,7 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 
 		return true
 	}, func() error {
-		currentAnnotations := podClient.podMetadata.Annotations
+		currentAnnotations := meta.Annotations
 		if len(currentAnnotations) == 0 {
 			currentAnnotations = map[string]string{}
 		}
@@ -285,63 +301,51 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 			return nil
 		}
 
-		return podClient.Patch(context.Background(), &corev1.Pod{
+		unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&corev1.Pod{
 			TypeMeta: metav1.TypeMeta{
 				Kind:       "Pod",
 				APIVersion: "v1",
 			},
 			ObjectMeta: metav1.ObjectMeta{
-				Namespace:   podClient.podMetadata.Namespace,
-				Name:        podClient.podMetadata.Name,
+				Namespace:   meta.Namespace,
+				Name:        meta.Name,
 				Annotations: currentAnnotations,
 			},
-		}, podClient.patchType, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
+		})
+		if err != nil {
+			return err
+		}
+
+		return podClient.Apply(
+			context.Background(),
+			client.ApplyConfigurationFromUnstructured(&unstructured.Unstructured{Object: unstructuredPod}),
+			client.FieldOwner("fdb-kubernetes-monitor"),
+			client.ForceOwnership)
 	})
 }
 
 // OnAdd is called when an object is added.
-func (podClient *kubernetesClient) OnAdd(obj interface{}, isInInitialList bool) {
-	switch castedObj := obj.(type) {
-	case *corev1.Pod:
-		podClient.Logger.Info("Got event for OnAdd for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
-		podClient.podMetadata = &metav1.PartialObjectMetadata{
-			TypeMeta:   castedObj.TypeMeta,
-			ObjectMeta: castedObj.ObjectMeta,
-		}
-	case *corev1.Node:
-		podClient.Logger.Info("Got event for OnAdd for Node resource", "name", castedObj.Name)
-		podClient.nodeMetadata = &metav1.PartialObjectMetadata{
-			TypeMeta:   castedObj.TypeMeta,
-			ObjectMeta: castedObj.ObjectMeta,
-		}
-	}
-}
+func (podClient *kubernetesClient) OnAdd(_ interface{}, _ bool) {}
 
 // OnUpdate is also called when a re-list happens, and it will
 // get called even if nothing changed. This is useful for periodically
 // evaluating or syncing something.
-func (podClient *kubernetesClient) OnUpdate(_, newObj interface{}) {
+func (podClient *kubernetesClient) OnUpdate(prevObj, newObj interface{}) {
 	switch castedObj := newObj.(type) {
 	case *corev1.Pod:
 		podClient.Logger.Info("Got event for OnUpdate for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace, "generation", castedObj.Generation)
-		var previousAnnotations map[string]string
-		if podClient.podMetadata != nil {
-			previousAnnotations = podClient.podMetadata.Annotations
-		}
-
-		podClient.podMetadata = &metav1.PartialObjectMetadata{
-			TypeMeta:   castedObj.TypeMeta,
-			ObjectMeta: castedObj.ObjectMeta,
-		}
-
-		if podClient.podMetadata.Annotations == nil {
-			return
+		// If the IsolateProcessGroupAnnotation changes, force a reload of the configuration to make sure the processes
+		// will be shutdown or started.
+		var previousIsolateProcessGroupAnnotationValue string
+		prevPod, ok := prevObj.(*corev1.Pod)
+		if ok && prevPod != nil {
+			previousAnnotations := prevPod.Annotations
+			if previousAnnotations != nil {
+				previousIsolateProcessGroupAnnotationValue = previousAnnotations[api.IsolateProcessGroupAnnotation]
+			}
 		}
 
-		// If the IsolateProcessGroupAnnotation changes force a reload of the configuration to make sure the processes
-		// will be shutdown.
-		previousIsolateProcessGroupAnnotationValue := previousAnnotations[api.IsolateProcessGroupAnnotation]
-		newIsolateProcessGroupAnnotationValue := podClient.podMetadata.Annotations[api.IsolateProcessGroupAnnotation]
+		newIsolateProcessGroupAnnotationValue := castedObj.Annotations[api.IsolateProcessGroupAnnotation]
 		if previousIsolateProcessGroupAnnotationValue != newIsolateProcessGroupAnnotationValue {
 			podClient.Logger.Info("Got change in isolate process group annotation", "previous", previousIsolateProcessGroupAnnotationValue, "new", newIsolateProcessGroupAnnotationValue)
 			podClient.TimestampFeed <- time.Now().Unix()
@@ -349,7 +353,7 @@ func (podClient *kubernetesClient) OnUpdate(_, newObj interface{}) {
 			return
 		}
 
-		annotation := podClient.podMetadata.Annotations[api.OutdatedConfigMapAnnotation]
+		annotation := castedObj.Annotations[api.OutdatedConfigMapAnnotation]
 		if annotation == "" {
 			return
 		}
@@ -361,12 +365,6 @@ func (podClient *kubernetesClient) OnUpdate(_, newObj interface{}) {
 		}
 
 		podClient.TimestampFeed <- timestamp
-	case *corev1.Node:
-		podClient.Logger.Info("Got event for OnUpdate for Node resource", "name", castedObj.Name)
-		podClient.nodeMetadata = &metav1.PartialObjectMetadata{
-			TypeMeta:   castedObj.TypeMeta,
-			ObjectMeta: castedObj.ObjectMeta,
-		}
 	}
 }
 
@@ -374,13 +372,4 @@ func (podClient *kubernetesClient) OnUpdate(_, newObj interface{}) {
 // it will get an object of type DeletedFinalStateUnknown. This can
 // happen if the watch is closed and misses the delete event and we don't
 // notice the deletion until the subsequent re-list.
-func (podClient *kubernetesClient) OnDelete(obj interface{}) {
-	switch castedObj := obj.(type) {
-	case *corev1.Pod:
-		podClient.Logger.Info("Got event for OnDelete for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
-		podClient.podMetadata = nil
-	case *corev1.Node:
-		podClient.Logger.Info("Got event for OnDelete for Node resource", "name", castedObj.Name)
-		podClient.nodeMetadata = nil
-	}
-}
+func (podClient *kubernetesClient) OnDelete(_ interface{}) {}
diff --git a/fdbkubernetesmonitor/kubernetes_test.go b/fdbkubernetesmonitor/kubernetes_test.go
index b455ab20641..8caefbc522e 100644
--- a/fdbkubernetesmonitor/kubernetes_test.go
+++ b/fdbkubernetesmonitor/kubernetes_test.go
@@ -96,8 +96,12 @@ var _ = Describe("Testing FDB Pod client", func() {
 			})
 
 			It("should have the metadata for the pod but not the node", func() {
-				Expect(podClient.podMetadata).NotTo(BeNil())
-				Expect(podClient.nodeMetadata).To(BeNil())
+				podMetadata, err := podClient.getPodMetadata(context.Background())
+				Expect(podMetadata).NotTo(BeNil())
+				Expect(err).NotTo(HaveOccurred())
+				nodeMetadata, err := podClient.getNodeMetadata(context.Background())
+				Expect(nodeMetadata).To(BeNil())
+				Expect(err).NotTo(HaveOccurred())
 				Expect(internalCache.InformersByGVK).To(HaveLen(1))
 			})
 		})
@@ -108,8 +112,12 @@ var _ = Describe("Testing FDB Pod client", func() {
 			})
 
 			It("should have the metadata for the pod and node", func() {
-				Expect(podClient.podMetadata).NotTo(BeNil())
-				Expect(podClient.nodeMetadata).NotTo(BeNil())
+				podMetadata, err := podClient.getPodMetadata(context.Background())
+				Expect(podMetadata).NotTo(BeNil())
+				Expect(err).NotTo(HaveOccurred())
+				nodeMetadata, err := podClient.getNodeMetadata(context.Background())
+				Expect(nodeMetadata).NotTo(BeNil())
+				Expect(err).NotTo(HaveOccurred())
 				Expect(internalCache.InformersByGVK).To(HaveLen(2))
 			})
 		})
@@ -148,38 +156,6 @@ var _ = Describe("Testing FDB Pod client", func() {
 				Expect(err).NotTo(HaveOccurred())
 			})
 
-			When("an AddEvent is handled", func() {
-				BeforeEach(func() {
-					fakeInformer.Add(pod)
-				})
-
-				It("should update the pod information", func() {
-					Expect(podClient.podMetadata).NotTo(BeNil())
-					Expect(podClient.podMetadata.Name).To(Equal(podName))
-					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-						"testing": "testing",
-					}))
-				})
-			})
-
-			When("an UpdateEvent is handled", func() {
-				BeforeEach(func() {
-					fakeInformer.Update(nil, pod)
-				})
-
-				It("should update the pod information", func() {
-					Expect(podClient.podMetadata).NotTo(BeNil())
-					Expect(podClient.podMetadata.Name).To(Equal(podName))
-					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-						"testing": "testing",
-					}))
-
-					Expect(podClient.TimestampFeed).NotTo(Receive())
-				})
-			})
-
 			When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation", func() {
 				var timestamp int64
 
@@ -192,13 +168,6 @@ var _ = Describe("Testing FDB Pod client", func() {
 				})
 
 				It("should update the pod information and receive an event", func() {
-					Expect(podClient.podMetadata).NotTo(BeNil())
-					Expect(podClient.podMetadata.Name).To(Equal(podName))
-					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-						"testing": "testing",
-					}))
-
 					Expect(podClient.TimestampFeed).To(Receive(&timestamp))
 				})
 			})
@@ -212,84 +181,36 @@ var _ = Describe("Testing FDB Pod client", func() {
 				})
 
 				It("should update the Pod information", func() {
-					Expect(podClient.podMetadata).NotTo(BeNil())
-					Expect(podClient.podMetadata.Name).To(Equal(podName))
-					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-						"testing": "testing",
-					}))
-
 					Expect(podClient.TimestampFeed).NotTo(Receive())
 				})
 			})
 
-			When("a DeleteEvent is handled", func() {
+			When("an UpdateEvent is handled that updates the IsolateProcessGroupAnnotation where the value is changed", func() {
 				BeforeEach(func() {
-					fakeInformer.Delete(pod)
-				})
-
-				It("should remove the pod information", func() {
-					Expect(podClient.podMetadata).To(BeNil())
-				})
-			})
-		})
-
-		When("events for the node are received", func() {
-			var fakeInformer *controllertest.FakeInformer
-			var node *corev1.Node
-
-			BeforeEach(func() {
-				node = &corev1.Node{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      podName,
-						Namespace: namespace,
-						Labels: map[string]string{
-							"testing": "testing",
-						},
-					},
-				}
-				var err error
-				fakeInformer, err = internalCache.FakeInformerFor(context.Background(), node)
-				Expect(err).NotTo(HaveOccurred())
-			})
+					oldPod := pod.DeepCopy()
+					pod.Annotations = map[string]string{
+						api.IsolateProcessGroupAnnotation: "true",
+					}
 
-			When("an AddEvent is handled", func() {
-				BeforeEach(func() {
-					fakeInformer.Add(node)
+					fakeInformer.Update(oldPod, pod)
 				})
 
-				It("should update the node information", func() {
-					Expect(podClient.nodeMetadata).NotTo(BeNil())
-					Expect(podClient.nodeMetadata.Name).To(Equal(podName))
-					Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
-					Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
-						"testing": "testing",
-					}))
+				It("should update the Pod information", func() {
+					Expect(podClient.TimestampFeed).To(Receive())
 				})
 			})
 
-			When("an UpdateEvent is handled", func() {
+			When("an UpdateEvent is handled that updates the IsolateProcessGroupAnnotation where the value is not changed", func() {
 				BeforeEach(func() {
-					fakeInformer.Update(nil, node)
-				})
-
-				It("should update the node information", func() {
-					Expect(podClient.nodeMetadata).NotTo(BeNil())
-					Expect(podClient.nodeMetadata.Name).To(Equal(podName))
-					Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
-					Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
-						"testing": "testing",
-					}))
-				})
-			})
+					pod.Annotations = map[string]string{
+						api.IsolateProcessGroupAnnotation: "true",
+					}
 
-			When("a DeleteEvent is handled", func() {
-				BeforeEach(func() {
-					fakeInformer.Delete(node)
+					fakeInformer.Update(pod.DeepCopy(), pod)
 				})
 
-				It("should remove the node information", func() {
-					Expect(podClient.nodeMetadata).To(BeNil())
+				It("should update the Pod information", func() {
+					Expect(podClient.TimestampFeed).NotTo(Receive())
 				})
 			})
 		})
@@ -308,13 +229,9 @@ var _ = Describe("Testing FDB Pod client", func() {
 				return fakeClient, internalCache, nil
 			})
 
-			// Currently the patch type is not supported in the fake client:
-			// https://github.com/kubernetes/kubernetes/issues/115598
-			podClient.patchType = client.Merge
-
 			Expect(err).NotTo(HaveOccurred())
 			// Execute the update annotations.
-			Expect(podClient.updateAnnotations(mon)).NotTo(HaveOccurred())
+			Expect(podClient.updateAnnotations(context.Background(), mon)).NotTo(HaveOccurred())
 		})
 
 		When("no additional env variables are set", func() {
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index f161ed1acfb..5efbd149a27 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -174,7 +174,11 @@ func main() {
 			logger.Error(err, "Error parsing container version", "currentContainerVersion", currentContainerVersion)
 			os.Exit(1)
 		}
-		startMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, promConfig, enablePprof, parsedVersion, enableNodeWatch)
+
+		ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+		defer stop()
+
+		startMonitor(ctx, logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, promConfig, enablePprof, parsedVersion, enableNodeWatch)
 	case executionModeInit:
 		err = copyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 0f45aff47d6..05076b4a1ac 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -27,11 +27,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"net/http"
 	"net/http/pprof"
 	"os"
 	"os/exec"
-	"os/signal"
 	"path"
 	"strconv"
 	"strings"
@@ -131,7 +131,7 @@ func startMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 		currentContainerVersion: currentContainerVersion,
 	}
 
-	go func() { mon.watchPodTimestamps() }()
+	go func() { mon.watchPodTimestamps(ctx) }()
 
 	mux := http.NewServeMux()
 	// Enable pprof endpoints for debugging purposes.
@@ -163,6 +163,7 @@ func startMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 			certLoader := certloader.NewCertLoader(logger, promConfig.certPath, promConfig.keyPath)
 			tlsConfig := &tls.Config{
 				GetCertificate: certLoader.GetCertificate,
+				MinVersion:     tls.VersionTLS12,
 			}
 			server := &http.Server{
 				Addr:      promConfig.listenAddr,
@@ -182,19 +183,24 @@ func startMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 		}
 	}()
 
-	mon.run()
+	mon.run(ctx)
 }
 
 // updateCustomEnvironment will add the node labels and their values to the custom environment map. All the generated
 // environment variables will start with NODE_LABEL and "/" and "." will be replaced in the key as "_", e.g. from the
 // label "foundationdb.org/testing = awesome" the env variables NODE_LABEL_FOUNDATIONDB_ORG_TESTING = awesome" will be
 // generated.
-func (monitor *monitor) updateCustomEnvironmentFromNodeMetadata() {
-	if monitor.podClient.nodeMetadata == nil {
-		return
+func (monitor *monitor) updateCustomEnvironmentFromNodeMetadata(ctx context.Context) error {
+	nodeMeta, err := monitor.podClient.getNodeMetadata(ctx)
+	if err != nil {
+		return err
 	}
 
-	nodeLabels := monitor.podClient.nodeMetadata.Labels
+	if nodeMeta == nil {
+		return nil
+	}
+
+	nodeLabels := nodeMeta.Labels
 	for key, value := range nodeLabels {
 		sanitizedKey := strings.ReplaceAll(key, "/", "_")
 		sanitizedKey = strings.ReplaceAll(sanitizedKey, ".", "_")
@@ -214,10 +220,12 @@ func (monitor *monitor) updateCustomEnvironmentFromNodeMetadata() {
 		monitor.customEnvironment[envKey] = value
 		continue
 	}
+
+	return nil
 }
 
 // readConfiguration reads the latest configuration from the monitor file.
-func (monitor *monitor) readConfiguration() (*api.ProcessConfiguration, []byte) {
+func (monitor *monitor) readConfiguration(ctx context.Context) (*api.ProcessConfiguration, []byte) {
 	file, err := os.Open(monitor.configFile)
 	if err != nil {
 		monitor.logger.Error(err, "Error reading monitor config file", "monitorConfigPath", monitor.configFile)
@@ -259,7 +267,11 @@ func (monitor *monitor) readConfiguration() (*api.ProcessConfiguration, []byte)
 		return nil, nil
 	}
 
-	monitor.updateCustomEnvironmentFromNodeMetadata()
+	err = monitor.updateCustomEnvironmentFromNodeMetadata(ctx)
+	if err != nil {
+		monitor.logger.Error(err, "Error updating custom environment from node metadata", "configuration", configuration, "binaryPath", configuration.BinaryPath)
+		return nil, nil
+	}
 	_, err = configuration.GenerateArguments(1, monitor.customEnvironment)
 	if err != nil {
 		monitor.logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
@@ -269,7 +281,7 @@ func (monitor *monitor) readConfiguration() (*api.ProcessConfiguration, []byte)
 	if configuration.ShouldRunServers() {
 		// In case that the process is isolated we don't want to start the servers and we should terminate the running fdbserver
 		// instances.
-		if monitor.processIsIsolated() {
+		if monitor.processIsIsolated(ctx) {
 			configuration.RunServers = pointer.Bool(false)
 		}
 	}
@@ -278,8 +290,8 @@ func (monitor *monitor) readConfiguration() (*api.ProcessConfiguration, []byte)
 }
 
 // loadConfiguration loads the latest configuration from the config file.
-func (monitor *monitor) loadConfiguration() {
-	configuration, configurationBytes := monitor.readConfiguration()
+func (monitor *monitor) loadConfiguration(ctx context.Context) {
+	configuration, configurationBytes := monitor.readConfiguration(ctx)
 	if configuration == nil || len(configurationBytes) == 0 {
 		return
 	}
@@ -287,7 +299,7 @@ func (monitor *monitor) loadConfiguration() {
 	monitor.acceptConfiguration(configuration, configurationBytes)
 	// Always update the annotations if needed to handle cases where the fdb-kubernetes-monitor
 	// has loaded the new configuration but was not able to update the annotations.
-	err := monitor.podClient.updateAnnotations(monitor)
+	err := monitor.podClient.updateAnnotations(ctx, monitor)
 	if err != nil {
 		monitor.logger.Error(err, "Error updating pod annotations")
 	}
@@ -338,7 +350,7 @@ func (monitor *monitor) acceptConfiguration(configuration *api.ProcessConfigurat
 
 	// If the monitor has running processes but the processes shouldn't be running, kill them with SIGTERM.
 	if hasRunningProcesses && !monitor.activeConfiguration.ShouldRunServers() {
-		monitor.sendSignalToProcesses(syscall.SIGTERM)
+		monitor.signalProcesses(syscall.SIGTERM, monitor.processIDs)
 	}
 }
 
@@ -373,10 +385,18 @@ func (monitor *monitor) runProcess(processNumber int) {
 			errorCounter = 0
 		}
 
-		arguments, err := monitor.activeConfiguration.GenerateArguments(processNumber, monitor.customEnvironment)
+		monitor.mutex.Lock()
+		config := monitor.activeConfiguration
+		customEnv := make(map[string]string, len(monitor.customEnvironment))
+		for k, v := range monitor.customEnvironment {
+			customEnv[k] = v
+		}
+		monitor.mutex.Unlock()
+
+		arguments, err := config.GenerateArguments(processNumber, customEnv)
 		if err != nil {
 			backoffDuration := getBackoffDuration(errorCounter)
-			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.activeConfiguration, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
+			logger.Error(err, "Error generating arguments for subprocess", "configuration", config, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
 			time.Sleep(backoffDuration)
 			errorCounter++
 			continue
@@ -408,7 +428,7 @@ func (monitor *monitor) runProcess(processNumber int) {
 		}
 
 		// Update the prometheus metrics for the process.
-		monitor.metrics.registerProcessStartup(processNumber, monitor.activeConfiguration.Version.String())
+		monitor.metrics.registerProcessStartup(processNumber, config.Version.String())
 
 		if cmd.Process != nil {
 			pid = cmd.Process.Pid
@@ -484,16 +504,21 @@ func (monitor *monitor) processRequired(processNumber int) bool {
 }
 
 // processIsIsolated returns true if the IsolateProcessGroupAnnotation is set to "true".
-func (monitor *monitor) processIsIsolated() bool {
-	if monitor.podClient.podMetadata == nil {
+func (monitor *monitor) processIsIsolated(ctx context.Context) bool {
+	podMeta, err := monitor.podClient.getPodMetadata(ctx)
+	if err != nil {
+		return false
+	}
+
+	if podMeta == nil {
 		return false
 	}
 
-	if monitor.podClient.podMetadata.Annotations == nil {
+	if podMeta.Annotations == nil {
 		return false
 	}
 
-	val, ok := monitor.podClient.podMetadata.Annotations[api.IsolateProcessGroupAnnotation]
+	val, ok := podMeta.Annotations[api.IsolateProcessGroupAnnotation]
 	if !ok {
 		return false
 	}
@@ -514,8 +539,37 @@ func (monitor *monitor) updateProcessID(processNumber int, pid int) {
 	monitor.processIDs[processNumber] = pid
 }
 
+// addWatcher adds a watch for the provided file. If an error is returned it will be retried for a maximum of 10 retries.
+// The delay between the retries increases with each retry and the maximum delay is 1 second.
+func addWatcher(fileName string, watcher *fsnotify.Watcher) error {
+	var retryCnt int
+	maxRetryCnt := 10
+
+	baseDelay := 100 * time.Millisecond
+	maxDelay := 1 * time.Second
+
+	var err error
+	for retryCnt < maxRetryCnt {
+		err = watcher.Add(fileName)
+		if err == nil {
+			return nil
+		}
+
+		retryCnt++
+
+		// Double the delay each attempt (100ms → 200ms → 400ms → …)
+		delay := min(baseDelay*(1<<retryCnt), maxDelay)
+		// Add jitter to avoid thundering herd
+		delay += time.Duration(rand.Int64N(int64(delay) / 2))
+
+		time.Sleep(delay)
+	}
+
+	return err
+}
+
 // watchConfiguration detects changes to the monitor configuration file.
-func (monitor *monitor) watchConfiguration(watcher *fsnotify.Watcher) {
+func (monitor *monitor) watchConfiguration(ctx context.Context, watcher *fsnotify.Watcher) {
 	for {
 		select {
 		case event, ok := <-watcher.Events:
@@ -525,38 +579,44 @@ func (monitor *monitor) watchConfiguration(watcher *fsnotify.Watcher) {
 
 			monitor.logger.Info("Detected event on monitor conf file or cluster file", "event", event)
 			if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
-				monitor.handleFileChange(event.Name)
+				monitor.handleFileChange(ctx, event.Name)
 			} else if event.Op&fsnotify.Remove == fsnotify.Remove {
-				err := watcher.Add(event.Name)
+				err := addWatcher(event.Name, watcher)
 				if err != nil {
 					panic(err)
 				}
-				monitor.handleFileChange(event.Name)
+				monitor.handleFileChange(ctx, event.Name)
 			}
 		case err, ok := <-watcher.Errors:
 			if !ok {
 				return
 			}
 			monitor.logger.Error(err, "Error watching for file system events")
+		// Stop watching if the context is shutdown.
+		case <-ctx.Done():
+			return
 		}
 	}
 }
 
 // handleFileChange will perform the required action based on the changed/modified file.
-func (monitor *monitor) handleFileChange(changedFile string) {
+func (monitor *monitor) handleFileChange(ctx context.Context, changedFile string) {
 	if changedFile == fdbClusterFilePath {
-		err := monitor.podClient.updateFdbClusterTimestampAnnotation()
+		err := monitor.podClient.updateFdbClusterTimestampAnnotation(ctx)
 		if err != nil {
 			monitor.logger.Error(err, fmt.Sprintf("could not update %s annotation", api.ClusterFileChangeDetectedAnnotation))
 		}
 		return
 	}
 
-	monitor.loadConfiguration()
+	monitor.loadConfiguration(ctx)
 }
 
-func (monitor *monitor) sendSignalToProcesses(signal os.Signal) {
-	for processNumber, processID := range monitor.processIDs {
+// signalProcesses sends the given signal to all processes in the provided slice.
+// Callers that already hold monitor.mutex should pass monitor.processIDs directly.
+// Callers that do not hold the mutex should snapshot processIDs under the mutex first.
+func (monitor *monitor) signalProcesses(signal os.Signal, processIDs []int) {
+	for processNumber, processID := range processIDs {
 		if processID <= 0 {
 			continue
 		}
@@ -577,34 +637,43 @@ func (monitor *monitor) sendSignalToProcesses(signal os.Signal) {
 }
 
 // run runs the monitor loop.
-func (monitor *monitor) run() {
+func (monitor *monitor) run(ctx context.Context) {
 	done := make(chan bool, 1)
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 
 	go func() {
-		latestSignal := <-signals
-		monitor.logger.Info("Received system signal", "signal", latestSignal)
+		<-ctx.Done()
+		monitor.logger.Info("Received system signal")
 
-		// Reset the processCount to 0 to make sure the monitor doesn't try to restart the processes.
-		monitor.processCount = 0
-		monitor.sendSignalToProcesses(latestSignal)
-
-		annotations := monitor.podClient.podMetadata.Annotations
-		if len(annotations) > 0 {
-			delayValue, ok := annotations[api.DelayShutdownAnnotation]
-			if ok {
-				delay, err := time.ParseDuration(delayValue)
-				if err == nil {
-					time.Sleep(delay)
+		var delayShutdown time.Duration
+		// We cannot use ctx here as the context as cancelled.
+		podMeta, err := monitor.podClient.getPodMetadata(context.Background())
+		if err != nil {
+			monitor.logger.Error(err, "Could not read pod metadata for delayed shutdown")
+		}
+
+		if podMeta != nil {
+			if delayValue, ok := podMeta.Annotations[api.DelayShutdownAnnotation]; ok {
+				if delay, err := time.ParseDuration(delayValue); err == nil {
+					delayShutdown = delay
 				}
 			}
 		}
 
+		// Reset processCount and snapshot processIDs under the mutex, then signal
+		// outside the lock to avoid holding it during OS calls.
+		monitor.mutex.Lock()
+		monitor.processCount = 0
+		ids := make([]int, len(monitor.processIDs))
+		copy(ids, monitor.processIDs)
+		monitor.mutex.Unlock()
+		monitor.signalProcesses(syscall.SIGTERM, ids)
+
+		time.Sleep(delayShutdown)
+
 		done <- true
 	}()
 
-	monitor.loadConfiguration()
+	monitor.loadConfiguration(ctx)
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		panic(err)
@@ -616,12 +685,12 @@ func (monitor *monitor) run() {
 	}
 
 	defer func(watcher *fsnotify.Watcher) {
-		err := watcher.Close()
+		err = watcher.Close()
 		if err != nil {
 			monitor.logger.Error(err, "could not close watcher")
 		}
 	}(watcher)
-	go func() { monitor.watchConfiguration(watcher) }()
+	go func() { monitor.watchConfiguration(ctx, watcher) }()
 
 	// The cluster file will be created and managed by the fdbserver processes, so we have to wait until the fdbserver
 	// processes have been started. Except for the initial cluster creation this file should be present as soon as the
@@ -646,10 +715,18 @@ func (monitor *monitor) run() {
 }
 
 // watchPodTimestamps watches the timestamp feed to reload the configuration.
-func (monitor *monitor) watchPodTimestamps() {
-	for timestamp := range monitor.podClient.TimestampFeed {
-		if timestamp > monitor.lastConfigurationTime.Unix() {
-			monitor.loadConfiguration()
+func (monitor *monitor) watchPodTimestamps(ctx context.Context) {
+	for {
+		select {
+		case timestamp, ok := <-monitor.podClient.TimestampFeed:
+			if !ok {
+				return
+			}
+			if timestamp > monitor.lastConfigurationTime.Unix() {
+				monitor.loadConfiguration(ctx)
+			}
+		case <-ctx.Done():
+			return
 		}
 	}
 }
diff --git a/fdbkubernetesmonitor/monitor_test.go b/fdbkubernetesmonitor/monitor_test.go
index 7ee9db84acf..f97eb78d4b6 100644
--- a/fdbkubernetesmonitor/monitor_test.go
+++ b/fdbkubernetesmonitor/monitor_test.go
@@ -2,7 +2,7 @@
 //
 // This source file is part of the FoundationDB open source project
 //
-// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+// Copyright 2021-2026 Apple Inc. and the FoundationDB project authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,12 +20,15 @@
 package main
 
 import (
+	"context"
 	"os"
 	"path"
 	"time"
 
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/json"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -47,7 +50,7 @@ var _ = Describe("Testing FDB Kubernetes monitor", func() {
 		})
 
 		JustBeforeEach(func() {
-			mon.updateCustomEnvironmentFromNodeMetadata()
+			Expect(mon.updateCustomEnvironmentFromNodeMetadata(context.Background())).To(Succeed())
 		})
 
 		When("no node metadata is present", func() {
@@ -59,15 +62,17 @@ var _ = Describe("Testing FDB Kubernetes monitor", func() {
 
 		When("node metadata is present", func() {
 			BeforeEach(func() {
-				mon.podClient.nodeMetadata = &metav1.PartialObjectMetadata{
+				mon.podClient.nodeName = "test-node"
+				mon.podClient.Client = fake.NewFakeClient(&corev1.Node{
 					ObjectMeta: metav1.ObjectMeta{
+						Name: "test-node",
 						Labels: map[string]string{
 							// Examples are taken from: https://kubernetes.io/docs/reference/labels-annotations-taints/
 							"topology.kubernetes.io/zone": "us-east-1c",
 							"kubernetes.io/hostname":      "ip-172-20-114-199.ec2.internal",
 						},
 					},
-				}
+				})
 			})
 
 			It("should add the new entries", func() {
@@ -118,16 +123,25 @@ var _ = Describe("Testing FDB Kubernetes monitor", func() {
 			Expect(os.WriteFile(fdbserverPath, []byte(""), 0700)).NotTo(HaveOccurred())
 			configurationFilePath = path.Join(tmpDir, "config.json")
 			mon = &monitor{
-				logger:                  GinkgoLogr,
-				configFile:              configurationFilePath,
-				customEnvironment:       map[string]string{},
-				podClient:               &kubernetesClient{},
+				logger:            GinkgoLogr,
+				configFile:        configurationFilePath,
+				customEnvironment: map[string]string{},
+				podClient: &kubernetesClient{
+					podName:   "test-pod",
+					namespace: "test",
+					Client: fake.NewFakeClient(&corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "test-pod",
+							Namespace: "test",
+						},
+					}),
+				},
 				currentContainerVersion: defaultVersion,
 			}
 		})
 
 		JustBeforeEach(func() {
-			configuration, configurationBytes = mon.readConfiguration()
+			configuration, configurationBytes = mon.readConfiguration(context.Background())
 		})
 
 		When("the configuration file is empty", func() {
@@ -166,15 +180,15 @@ var _ = Describe("Testing FDB Kubernetes monitor", func() {
 
 			When("the pod has the isolate annotation set to true", func() {
 				BeforeEach(func() {
-					mon.podClient = &kubernetesClient{
-						podMetadata: &metav1.PartialObjectMetadata{
-							ObjectMeta: metav1.ObjectMeta{
-								Annotations: map[string]string{
-									api.IsolateProcessGroupAnnotation: "true",
-								},
+					Expect(mon.podClient.Client.Update(context.Background(), &corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "test-pod",
+							Namespace: "test",
+							Annotations: map[string]string{
+								api.IsolateProcessGroupAnnotation: "true",
 							},
 						},
-					}
+					})).To(Succeed())
 				})
 
 				It("should return the configuration with runServers set to false", func() {
diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp
index 480ae62b37a..9ae25e8083f 100644
--- a/fdbmonitor/fdbmonitor.cpp
+++ b/fdbmonitor/fdbmonitor.cpp
@@ -201,7 +201,7 @@ int main(int argc, char** argv) {
 
 	/* open and lock our lockfile for mutual exclusion */
 	std::string lockfileDir = parentDirectory(lockfile, true);
-	if (lockfileDir.size() == 0) {
+	if (lockfileDir.empty()) {
 		log_msg(SevError, "Unable to determine parent directory of lockfile %s\n", lockfile.c_str());
 		exit(1);
 	}
@@ -367,7 +367,7 @@ int main(int argc, char** argv) {
 				end_time = std::min(i.second->fork_retry_time, end_time);
 			}
 			// If process has a resident memory limit and is currently running
-			if (i.second->memory_rss > 0 && id_pid.count(i.first) > 0) {
+			if (i.second->memory_rss > 0 && id_pid.contains(i.first)) {
 				need_rss_check = true;
 			}
 		}
@@ -453,7 +453,7 @@ int main(int argc, char** argv) {
 				}
 				break;
 			case EVFILT_READ:
-				Command* cmd = (Command*)ev.udata;
+				auto* cmd = (Command*)ev.udata;
 				for (int i = 0; i < 2; i++) {
 					if (ev.ident == cmd->pipes[i][0]) {
 						read_child_output(cmd, i, watched_fds);
@@ -470,7 +470,7 @@ int main(int argc, char** argv) {
 			last_rss_check = timer();
 			std::vector<ProcessID> oom_ids;
 			for (auto& i : id_command) {
-				if (id_pid.count(i.first) == 0) {
+				if (!id_pid.contains(i.first)) {
 					// process is not running
 					continue;
 				}
@@ -494,7 +494,7 @@ int main(int argc, char** argv) {
 			for (auto& id : oom_ids) {
 				kill_process(id, false /*wait*/, false /*cleanup*/);
 			}
-			if (oom_ids.size() > 0) {
+			if (!oom_ids.empty()) {
 				child_exited = true;
 			}
 		}
@@ -570,11 +570,12 @@ int main(int argc, char** argv) {
 					log_err("read", errno, "Error reading inotify message");
 
 				while (i < len) {
-					struct inotify_event* event = (struct inotify_event*)&buf[i];
+					auto* event = (struct inotify_event*)&buf[i];
 
 					auto search = additional_watch_wds.find(event->wd);
 					if (event->wd != conffile_wd) {
-						if (search != additional_watch_wds.end() && event->len && search->second.count(event->name)) {
+						if (search != additional_watch_wds.end() && event->len &&
+						    search->second.contains(event->name)) {
 							log_msg(SevInfo,
 							        "Changes detected on watched symlink `%s': (%d, %#010x)\n",
 							        event->name,
@@ -596,7 +597,7 @@ int main(int argc, char** argv) {
 							conffile = confpath.substr(confdir.size());
 
 							// Remove all the old watches
-							for (auto wd : additional_watch_wds) {
+							for (const auto& wd : additional_watch_wds) {
 								if (inotify_rm_watch(ifd, wd.first) < 0) {
 									// log_err("inotify_rm_watch", errno, "Unable to remove symlink watch %d",
 									// wd.first); exit(1);
diff --git a/fdbmonitor/fdbmonitor_lib.cpp b/fdbmonitor/fdbmonitor_lib.cpp
index 57443f900eb..55a5f91f402 100644
--- a/fdbmonitor/fdbmonitor_lib.cpp
+++ b/fdbmonitor/fdbmonitor_lib.cpp
@@ -278,9 +278,9 @@ uint64_t parseWithSuffix(const char* to_parse, const char* default_unit) {
 std::string joinPath(std::string const& directory, std::string const& filename) {
 	auto d = directory;
 	auto f = filename;
-	while (f.size() && (f[0] == '/' || f[0] == CANONICAL_PATH_SEPARATOR))
+	while (!f.empty() && (f[0] == '/' || f[0] == CANONICAL_PATH_SEPARATOR))
 		f = f.substr(1);
-	while (d.size() && (d.back() == '/' || d.back() == CANONICAL_PATH_SEPARATOR))
+	while (!d.empty() && (d.back() == '/' || d.back() == CANONICAL_PATH_SEPARATOR))
 		d = d.substr(0, d.size() - 1);
 	return d + CANONICAL_PATH_SEPARATOR + f;
 }
@@ -297,7 +297,7 @@ std::string cleanPath(std::string const& path) {
 		}
 		std::string part = path.substr(i, sep - i);
 		i = sep + 1;
-		if (part.size() == 0 || (part.size() == 1 && part[0] == '.'))
+		if (part.empty() || (part.size() == 1 && part[0] == '.'))
 			continue;
 		if (part == "..") {
 			if (!finalParts.empty() && finalParts.back() != "..") {
@@ -605,7 +605,7 @@ bool argv_equal(const char** a1, const char** a2) {
 	int i = 0;
 
 	while (a1[i] && a2[i]) {
-		if (strcmp(a1[i], a2[i]))
+		if (strcmp(a1[i], a2[i]) != 0)
 			return false;
 		i++;
 	}
@@ -675,12 +675,12 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb
 		/* Any change to uid or gid requires the process to be restarted to take effect */
 		if (uid != _uid || gid != _gid) {
 			std::vector<ProcessID> kill_ids;
-			for (auto i : id_pid) {
+			for (const auto& i : id_pid) {
 				if (id_command[i.first]->kill_on_configuration_change) {
 					kill_ids.push_back(i.first);
 				}
 			}
-			for (auto i : kill_ids) {
+			for (const auto& i : kill_ids) {
 				kill_process(i);
 				id_command.erase(i);
 			}
@@ -693,7 +693,7 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb
 	std::list<ProcessID> kill_ids;
 	std::list<std::pair<ProcessID, Command*>> start_ids;
 
-	for (auto i : id_pid) {
+	for (const auto& i : id_pid) {
 		if (!loadedConf || ini.GetSectionSize(id_command[i.first]->ssection.c_str()) == -1) {
 			/* Process no longer configured; deconfigure it and kill it if required */
 			log_msg(SevInfo, "Deconfigured %s\n", id_command[i.first]->ssection.c_str());
@@ -726,10 +726,10 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb
 		}
 	}
 
-	for (auto i : kill_ids)
+	for (const auto& i : kill_ids)
 		kill_process(i);
 
-	for (auto i : start_ids) {
+	for (const auto& i : start_ids) {
 		start_process(i.second, i.first, uid, gid, 0, mask);
 	}
 
@@ -738,10 +738,10 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb
 	if (loadedConf) {
 		CSimpleIniA::TNamesDepend sections;
 		ini.GetAllSections(sections);
-		for (auto i : sections) {
+		for (const auto& i : sections) {
 			if (auto dot = strrchr(i.pItem, '.')) {
 				ProcessID id = i.pItem;
-				if (!id_pid.count(id)) {
+				if (!id_pid.contains(id)) {
 					/* Found something we haven't yet started */
 					Command* cmd;
 
diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.cpp
similarity index 99%
rename from fdbrpc/AsyncFileCached.actor.cpp
rename to fdbrpc/AsyncFileCached.cpp
index a045b619fa8..fd7d97b371d 100644
--- a/fdbrpc/AsyncFileCached.actor.cpp
+++ b/fdbrpc/AsyncFileCached.cpp
@@ -1,5 +1,5 @@
 /*
- * AsyncFileCached.actor.cpp
+ * AsyncFileCached.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbrpc/AsyncFileCached.actor.h"
+#include "fdbrpc/AsyncFileCached.h"
 
 // Page caches used in non-simulated environments
 Optional<Reference<EvictablePageCache>> pc4k, pc64k;
diff --git a/fdbrpc/include/fdbrpc/AsyncFileChaos.h b/fdbrpc/AsyncFileChaos.h
similarity index 100%
rename from fdbrpc/include/fdbrpc/AsyncFileChaos.h
rename to fdbrpc/AsyncFileChaos.h
diff --git a/fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h b/fdbrpc/AsyncFileEIO.h
similarity index 71%
rename from fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h
rename to fdbrpc/AsyncFileEIO.h
index 10d7d51579f..7a06470ce98 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileEIO.actor.h
+++ b/fdbrpc/AsyncFileEIO.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileEIO.actor.h
+ * AsyncFileEIO.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,14 +24,6 @@
 
 #define Net2AsyncFile AsyncFileEIO
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_ASYNCFILEEIO_ACTOR_G_H)
-#define FLOW_ASYNCFILEEIO_ACTOR_G_H
-#include "fdbrpc/AsyncFileEIO.actor.g.h"
-#elif !defined(FLOW_ASYNCFILEEIO_ACTOR_H)
-#define FLOW_ASYNCFILEEIO_ACTOR_H
-
 #include <fcntl.h>
 #include <sys/stat.h>
 
@@ -39,8 +31,7 @@
 #include "flow/flow.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/IAsyncFile.h"
-#include "flow/TDMetric.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TDMetric.h"
 
 class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 
@@ -75,17 +66,17 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		return true;
 	}
 
-	ACTOR static Future<Reference<IAsyncFile>> open(std::string filename, int flags, int mode, void* ignore) {
+	static Future<Reference<IAsyncFile>> open(std::string filename, int flags, int mode, void* ignore) {
 		std::string open_filename = filename;
 		if (flags & OPEN_ATOMIC_WRITE_AND_CREATE) {
 			ASSERT((flags & OPEN_CREATE) && (flags & OPEN_READWRITE) && !(flags & OPEN_EXCLUSIVE));
 			open_filename = filename + ".part";
 		}
 
-		state Promise<Void> p;
-		state eio_req* r = eio_open(open_filename.c_str(), openFlags(flags), mode, 0, eio_callback, &p);
+		Promise<Void> p;
+		eio_req* r = eio_open(open_filename.c_str(), openFlags(flags), mode, 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			eio_cancel(r);
 			throw;
@@ -113,7 +104,7 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 			throw lock_file_failure();
 		}
 
-		return Reference<IAsyncFile>(new AsyncFileEIO(r->result, flags, filename));
+		co_return Reference<IAsyncFile>(makeReference<AsyncFileEIO>(r->result, flags, filename));
 	}
 	static Future<Void> deleteFile(std::string filename, bool mustBeDurable) {
 		::deleteFile(filename);
@@ -124,36 +115,35 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 			return Void();
 	}
 
-	ACTOR static Future<Void> renameFile(std::string from, std::string to) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
-		state eio_req* r = eio_rename(from.c_str(), to.c_str(), 0, eio_callback, &p);
+	static Future<Void> renameFile(std::string from, std::string to) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
+		eio_req* r = eio_rename(from.c_str(), to.c_str(), 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
 			throw;
 		}
+		Optional<Error> err;
 		try {
-			state int result = r->result;
+			int result = r->result;
 			if (result == -1) {
 				TraceEvent(SevError, "FileRenameError").detail("Errno", r->errorno);
 				throw internal_error();
-			} else {
-				wait(delay(0, taskID));
-				return Void();
 			}
 		} catch (Error& e) {
-			state Error _e = e;
-			wait(delay(0, taskID));
-			throw _e;
+			err = e;
 		}
+		co_await delay(0, taskID);
+		if (err.present())
+			throw err.get();
 	}
 
-	ACTOR static Future<std::time_t> lastWriteTime(std::string filename) {
-		EIO_STRUCT_STAT statdata = wait(stat_impl(filename));
-		return statdata.st_mtime;
+	static Future<std::time_t> lastWriteTime(std::string filename) {
+		EIO_STRUCT_STAT statdata = co_await stat_impl(filename);
+		co_return statdata.st_mtime;
 	}
 
 	void addref() override { ReferenceCounted<AsyncFileEIO>::addref(); }
@@ -198,20 +188,19 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 	}
 	std::string getFilename() const override { return filename; }
 
-	ACTOR static Future<Void> async_fsync_parent(std::string filename) {
+	static Future<Void> async_fsync_parent(std::string filename) {
 		std::string folder = parentDirectory(filename);
 		TraceEvent("FSyncParentDir").detail("Folder", folder).detail("File", filename);
-		state int folderFD = ::open(folder.c_str(), O_DIRECTORY | O_CLOEXEC, 0);
+		int folderFD = ::open(folder.c_str(), O_DIRECTORY | O_CLOEXEC, 0);
 		if (folderFD < 0)
 			throw io_error();
 		try {
-			wait(async_fsync(folderFD)); // not sure if fdatasync on the folder has the same effect
+			co_await async_fsync(folderFD); // not sure if fdatasync on the folder has the same effect
 		} catch (...) {
 			close(folderFD);
 			throw;
 		}
 		close(folderFD);
-		return Void();
 	}
 
 	static Future<Void> async_fdatasync(int fd) {
@@ -222,11 +211,9 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		// Used by AsyncFileKAIO, since kernel AIO doesn't really implement fsync yet
 		return sync_impl(fd, makeReference<ErrorInfo>(), true);
 	}
-	ACTOR static Future<Void> waitAndAtomicRename(Future<Void> fsync,
-	                                              std::string part_filename,
-	                                              std::string final_filename) {
+	static Future<Void> waitAndAtomicRename(Future<Void> fsync, std::string part_filename, std::string final_filename) {
 		// First wait for the data in the part file to be durable
-		wait(fsync);
+		co_await fsync;
 
 		// rename() is atomic
 		if (rename(part_filename.c_str(), final_filename.c_str())) {
@@ -235,9 +222,7 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		}
 
 		// fsync the parent directory to make it durable as well
-		wait(async_fsync_parent(final_filename));
-
-		return Void();
+		co_await async_fsync_parent(final_filename);
 	}
 
 	// Run the given function on the EIO thread pool and return its result
@@ -246,9 +231,12 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		return dispatch_impl(func);
 	}
 
-	~AsyncFileEIO() override { close_impl(fd); }
+	~AsyncFileEIO() override { close_impl(Uncancellable(), fd); }
 
 private:
+	template <class P, class... Args>
+	friend Reference<P> makeReference(Args&&... args);
+
 	struct ErrorInfo : ReferenceCounted<ErrorInfo>, FastAllocated<ErrorInfo> {
 		Error err;
 		void set(const Error& e) {
@@ -322,50 +310,50 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 			throw e;
 	}
 
-	ACTOR static void close_impl(int fd) {
-		state Promise<Void> p;
-		state eio_req* r = eio_close(fd, 0, eio_callback, &p);
-		wait(p.getFuture());
+	static Future<Void> close_impl(Uncancellable, int fd) {
+		Promise<Void> p;
+		eio_req* r = eio_close(fd, 0, eio_callback, &p);
+		co_await p.getFuture();
 		if (r->result)
 			error("CloseError", fd, r);
 		TraceEvent("AsyncFileClosed").suppressFor(1.0).detail("Fd", fd);
 	}
 
-	ACTOR static Future<int> read_impl(int fd, void* data, int length, int64_t offset) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
+	static Future<int> read_impl(int fd, void* data, int length, int64_t offset) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
 		// fprintf(stderr, "eio_read (fd=%d length=%d offset=%lld)\n", fd, length, offset);
-		state eio_req* r = eio_read(fd, data, length, offset, 0, eio_callback, &p);
+		eio_req* r = eio_read(fd, data, length, offset, 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
 			throw;
 		}
+		int result = r->result;
+		Optional<Error> err;
 		try {
-			state int result = r->result;
 			// printf("eio read: %d/%d\n", r->result, length);
 			if (result == -1) {
 				error("ReadError", fd, r);
 				throw internal_error();
-			} else {
-				wait(delay(0, taskID));
-				return result;
 			}
-		} catch (Error& _e) {
-			state Error e = _e;
-			wait(delay(0, taskID));
-			throw e;
+		} catch (Error& e) {
+			err = e;
 		}
+		co_await delay(0, taskID);
+		if (err.present())
+			throw err.get();
+		co_return result;
 	}
 
-	ACTOR static Future<Void> write_impl(int fd, Reference<ErrorInfo> err, StringRef data, int64_t offset) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
-		state eio_req* r = eio_write(fd, (void*)data.begin(), data.size(), offset, 0, eio_callback, &p);
+	static Future<Void> write_impl(int fd, Reference<ErrorInfo> err, StringRef data, int64_t offset) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
+		eio_req* r = eio_write(fd, (void*)data.begin(), data.size(), offset, 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
@@ -373,16 +361,15 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		}
 		if (r->result != data.size())
 			error("WriteError", fd, r, err);
-		wait(delay(0, taskID));
-		return Void();
+		co_await delay(0, taskID);
 	}
 
-	ACTOR static Future<Void> truncate_impl(int fd, Reference<ErrorInfo> err, int64_t size) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
-		state eio_req* r = eio_ftruncate(fd, size, 0, eio_callback, &p);
+	static Future<Void> truncate_impl(int fd, Reference<ErrorInfo> err, int64_t size) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
+		eio_req* r = eio_ftruncate(fd, size, 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
@@ -390,8 +377,7 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		}
 		if (r->result)
 			error("TruncateError", fd, r, err);
-		wait(delay(0, taskID));
-		return Void();
+		co_await delay(0, taskID);
 	}
 
 	static eio_req* start_fsync(int fd, Promise<Void>& p, bool sync_metadata) {
@@ -415,39 +401,39 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 #endif
 	}
 
-	ACTOR static Future<Void> sync_impl(int fd, Reference<ErrorInfo> err, bool sync_metadata = false) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
-		state eio_req* r = start_fsync(fd, p, sync_metadata);
+	static Future<Void> sync_impl(int fd, Reference<ErrorInfo> err, bool sync_metadata = false) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
+		eio_req* r = start_fsync(fd, p, sync_metadata);
 
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
 			throw;
 		}
+		Optional<Error> caughtErr;
 		try {
 			// Report any errors from prior write() or truncate() calls
 			err->report();
 
 			if (r->result)
 				error("SyncError", fd, r);
-			wait(delay(0, taskID));
-			return Void();
-		} catch (Error& _e) {
-			state Error e = _e;
-			wait(delay(0, taskID));
-			throw e;
+		} catch (Error& e) {
+			caughtErr = e;
 		}
+		co_await delay(0, taskID);
+		if (caughtErr.present())
+			throw caughtErr.get();
 	}
 
-	ACTOR static Future<int64_t> size_impl(int fd) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
-		state eio_req* r = eio_fstat(fd, 0, eio_callback, &p);
+	static Future<int64_t> size_impl(int fd) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
+		eio_req* r = eio_fstat(fd, 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
@@ -458,18 +444,18 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		EIO_STRUCT_STAT* statdata = (EIO_STRUCT_STAT*)r->ptr2;
 		if (!statdata)
 			error("FStatBufferError", fd, r);
-		state int64_t size = statdata->st_size;
-		wait(delay(0, taskID));
-		return size;
+		int64_t size = statdata->st_size;
+		co_await delay(0, taskID);
+		co_return size;
 	}
 
-	ACTOR static Future<EIO_STRUCT_STAT> stat_impl(std::string filename) {
-		state TaskPriority taskID = g_network->getCurrentTask();
-		state Promise<Void> p;
-		state EIO_STRUCT_STAT statdata;
-		state eio_req* r = eio_stat(filename.c_str(), 0, eio_callback, &p);
+	static Future<EIO_STRUCT_STAT> stat_impl(std::string filename) {
+		TaskPriority taskID = g_network->getCurrentTask();
+		Promise<Void> p;
+		EIO_STRUCT_STAT statdata;
+		eio_req* r = eio_stat(filename.c_str(), 0, eio_callback, &p);
 		try {
-			wait(p.getFuture());
+			co_await p.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
@@ -480,16 +466,16 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		if (!r->ptr2)
 			error("StatBufferError", 0, r);
 		statdata = *EIO_STAT_BUF(r);
-		wait(delay(0, taskID));
-		return statdata;
+		co_await delay(0, taskID);
+		co_return statdata;
 	}
 
-	ACTOR template <class R>
+	template <class R>
 	static Future<R> dispatch_impl(std::function<R()> func) {
-		state Dispatch<R> data(func);
-		state TaskPriority taskID = g_network->getCurrentTask();
+		Dispatch<R> data(func);
+		TaskPriority taskID = g_network->getCurrentTask();
 
-		state eio_req* r = eio_custom(
+		eio_req* r = eio_custom(
 		    [](eio_req* req) {
 			    // Runs on the eio thread pool
 			    auto data = reinterpret_cast<Dispatch<R>*>(req->data);
@@ -516,31 +502,31 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 		    },
 		    &data);
 		try {
-			wait(data.done.getFuture());
+			co_await data.done.getFuture();
 		} catch (...) {
 			g_network->setCurrentTask(taskID);
 			eio_cancel(r);
 			throw;
 		}
 
-		wait(delay(0, taskID));
+		co_await delay(0, taskID);
 		if (data.result.isError())
 			throw data.result.getError();
-		return data.result.get();
+		co_return data.result.get();
 	}
 
 	static std::atomic<int32_t> want_poll;
 
-	ACTOR static void poll_eio() {
+	static Future<Void> poll_eio(Uncancellable) {
 		while (eio_poll() == -1)
-			wait(yield());
+			co_await yield();
 		want_poll = 0;
 	}
 
 	static void eio_want_poll() {
 		want_poll = 1;
 		// SOMEDAY: nullptr for deferred error, no analysis of correctness (itp)
-		onMainThreadVoid([]() { poll_eio(); }, TaskPriority::PollEIO);
+		onMainThreadVoid([]() { poll_eio(Uncancellable()); }, TaskPriority::PollEIO);
 	}
 
 	static int eio_callback(eio_req* req) {
@@ -561,6 +547,4 @@ class AsyncFileEIO : public IAsyncFile, public ReferenceCounted<AsyncFileEIO> {
 std::atomic<int32_t> AsyncFileEIO::want_poll = 0;
 #endif
 
-#include "flow/unactorcompiler.h"
-#endif
 #endif
diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.cpp
similarity index 57%
rename from fdbrpc/AsyncFileEncrypted.actor.cpp
rename to fdbrpc/AsyncFileEncrypted.cpp
index 43da1e01dd9..e509f22a773 100644
--- a/fdbrpc/AsyncFileEncrypted.actor.cpp
+++ b/fdbrpc/AsyncFileEncrypted.cpp
@@ -1,5 +1,5 @@
 /*
- * AsyncFileEncrypted.actor.cpp
+ * AsyncFileEncrypted.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,7 +22,6 @@
 #include "flow/StreamCipher.h"
 #include "flow/UnitTest.h"
 #include "flow/xxhash.h"
-#include "flow/actorcompiler.h" // must be last include
 
 class AsyncFileEncryptedImpl {
 public:
@@ -30,7 +29,8 @@ class AsyncFileEncryptedImpl {
 	// the filename.
 	static auto getFirstBlockIV(const std::string& filename) {
 		StreamCipher::IV iv;
-		auto salt = basename(filename);
+		auto slashPos = filename.rfind('/');
+		auto salt = (slashPos == std::string::npos) ? filename : filename.substr(slashPos + 1);
 		auto pos = salt.find('.');
 		salt = salt.substr(0, pos);
 		auto hash = XXH3_128bits(salt.c_str(), salt.size());
@@ -44,53 +44,42 @@ class AsyncFileEncryptedImpl {
 		return iv;
 	}
 
-	// Read a single block of size ENCRYPTION_BLOCK_SIZE bytes, and decrypt.
-	ACTOR static Future<Standalone<StringRef>> readBlock(AsyncFileEncrypted* self, uint32_t block) {
-		state Arena arena;
-		state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE];
-		int bytes = wait(uncancellable(holdWhile(arena,
-		                                         self->file->read(encrypted,
-		                                                          FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE,
-		                                                          FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * block))));
+	// Read a single block of size encryptionBlockSize bytes, and decrypt.
+	static Future<Standalone<StringRef>> readBlock(AsyncFileEncrypted* self, uint32_t block) {
+		Arena arena;
+		auto* encrypted = new (arena) unsigned char[self->encryptionBlockSize];
+		int bytes = co_await uncancellable(holdWhile(
+		    arena, self->file->read(encrypted, self->encryptionBlockSize, self->encryptionBlockSize * block)));
 		StreamCipherKey const* cipherKey = StreamCipherKey::getGlobalCipherKey();
 		DecryptionStreamCipher decryptor(cipherKey, self->getIV(block));
 		auto decrypted = decryptor.decrypt(encrypted, bytes, arena);
-		return Standalone<StringRef>(decrypted, arena);
+		co_return Standalone<StringRef>(decrypted, arena);
 	}
 
-	ACTOR static Future<int> read(Reference<AsyncFileEncrypted> self, void* data, int length, int64_t offset) {
+	static Future<int> read(Reference<AsyncFileEncrypted> self, void* data, int length, int64_t offset) {
 		if (self->fileSize == -1) {
-			state int64_t fileSize = wait(self->file->size());
+			int64_t fileSize = co_await self->file->size();
 			self->fileSize = fileSize;
 		}
 		if (offset >= self->fileSize) {
-			return 0;
+			co_return 0;
 		}
 		if (offset + length > self->fileSize) {
 			length = self->fileSize - offset;
 		}
-		state uint32_t firstBlock = offset / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
-		state uint32_t lastBlock = (offset + length - 1) / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE;
-		state uint32_t block;
-		state unsigned char* output = reinterpret_cast<unsigned char*>(data);
-		state int bytesRead = 0;
+		uint32_t firstBlock = offset / self->encryptionBlockSize;
+		uint32_t lastBlock = (offset + length - 1) / self->encryptionBlockSize;
+		uint32_t block{ 0 };
+		auto* output = reinterpret_cast<unsigned char*>(data);
+		int bytesRead = 0;
 		ASSERT(self->mode == AsyncFileEncrypted::Mode::READ_ONLY);
 		for (block = firstBlock; block <= lastBlock; ++block) {
-			state Standalone<StringRef> plaintext;
-
-			auto cachedBlock = self->readBuffers.get(block);
-			if (cachedBlock.present()) {
-				plaintext = cachedBlock.get();
-			} else {
-				wait(store(plaintext, readBlock(self.getPtr(), block)));
-				self->readBuffers.insert(block, plaintext);
-			}
-			auto start = (block == firstBlock) ? plaintext.begin() + (offset % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE)
-			                                   : plaintext.begin();
-			auto end = (block == lastBlock)
-			               ? plaintext.begin() + ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE)
-			               : plaintext.end();
-			if ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE == 0) {
+			Standalone<StringRef> plaintext = co_await readBlock(self.getPtr(), block);
+			auto start =
+			    (block == firstBlock) ? plaintext.begin() + (offset % self->encryptionBlockSize) : plaintext.begin();
+			auto end = (block == lastBlock) ? plaintext.begin() + ((offset + length) % self->encryptionBlockSize)
+			                                : plaintext.end();
+			if ((offset + length) % self->encryptionBlockSize == 0) {
 				end = plaintext.end();
 			}
 
@@ -105,16 +94,16 @@ class AsyncFileEncryptedImpl {
 			output += (end - start);
 			bytesRead += (end - start);
 		}
-		return bytesRead;
+		co_return bytesRead;
 	}
 
-	ACTOR static Future<Void> write(Reference<AsyncFileEncrypted> self, void const* data, int length, int64_t offset) {
+	static Future<Void> write(Reference<AsyncFileEncrypted> self, void const* data, int length, int64_t offset) {
 		ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY);
 		// All writes must append to the end of the file:
-		ASSERT_EQ(offset, self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock);
-		state unsigned char const* input = reinterpret_cast<unsigned char const*>(data);
+		ASSERT_EQ(offset, self->currentBlock * self->encryptionBlockSize + self->offsetInBlock);
+		auto const* input = reinterpret_cast<unsigned char const*>(data);
 		while (length > 0) {
-			const auto chunkSize = std::min(length, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE - self->offsetInBlock);
+			const auto chunkSize = std::min(length, self->encryptionBlockSize - self->offsetInBlock);
 			Arena arena;
 			auto encrypted = self->encryptor->encrypt(input, chunkSize, arena);
 			std::copy(encrypted.begin(), encrypted.end(), &self->writeBuffer[self->offsetInBlock]);
@@ -122,8 +111,8 @@ class AsyncFileEncryptedImpl {
 			self->offsetInBlock += chunkSize;
 			length -= chunkSize;
 			input += chunkSize;
-			if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) {
-				wait(self->writeLastBlockToFile());
+			if (self->offsetInBlock == self->encryptionBlockSize) {
+				co_await self->writeLastBlockToFile();
 				self->offsetInBlock = 0;
 				ASSERT_LT(self->currentBlock, std::numeric_limits<uint32_t>::max());
 				++self->currentBlock;
@@ -131,34 +120,32 @@ class AsyncFileEncryptedImpl {
 				                                                           self->getIV(self->currentBlock));
 			}
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> sync(Reference<AsyncFileEncrypted> self) {
+	static Future<Void> sync(Reference<AsyncFileEncrypted> self) {
 		ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY);
-		wait(self->writeLastBlockToFile());
-		wait(self->file->sync());
-		return Void();
+		co_await self->writeLastBlockToFile();
+		co_await self->file->sync();
 	}
 
-	ACTOR static Future<Void> zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) {
+	static Future<Void> zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) {
 		ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY);
 		// TODO: Could optimize this
 		Arena arena;
 		auto zeroes = new (arena) unsigned char[length];
 		memset(zeroes, 0, length);
-		wait(uncancellable(holdWhile(arena, self->write(zeroes, length, offset))));
-		return Void();
+		co_await uncancellable(holdWhile(arena, self->write(zeroes, length, offset)));
 	}
 };
 
-AsyncFileEncrypted::AsyncFileEncrypted(Reference<IAsyncFile> file, Mode mode)
-  : file(file), mode(mode), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS), currentBlock(0) {
+AsyncFileEncrypted::AsyncFileEncrypted(Reference<IAsyncFile> file, Mode mode, int encryptionBlockSize)
+  : file(file), mode(mode), currentBlock(0), encryptionBlockSize(encryptionBlockSize) {
+	ASSERT(encryptionBlockSize > 0);
 	firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename());
 	if (mode == Mode::APPEND_ONLY) {
 		encryptor =
 		    std::make_unique<EncryptionStreamCipher>(StreamCipherKey::getGlobalCipherKey(), getIV(currentBlock));
-		writeBuffer = std::vector<unsigned char>(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0);
+		writeBuffer = std::vector<unsigned char>(encryptionBlockSize, 0);
 	}
 }
 
@@ -231,73 +218,43 @@ StreamCipher::IV AsyncFileEncrypted::getIV(uint32_t block) const {
 Future<Void> AsyncFileEncrypted::writeLastBlockToFile() {
 	// The source buffer for the write is owned by *this so this must be kept alive by reference count until the write
 	// is finished.
-	return uncancellable(
-	    holdWhile(Reference<AsyncFileEncrypted>::addRef(this),
-	              file->write(&writeBuffer[0], offsetInBlock, currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE)));
-}
-
-size_t AsyncFileEncrypted::RandomCache::evict() {
-	ASSERT_EQ(vec.size(), maxSize);
-	auto index = deterministicRandom()->randomInt(0, maxSize);
-	hashMap.erase(vec[index]);
-	return index;
-}
-
-AsyncFileEncrypted::RandomCache::RandomCache(size_t maxSize) : maxSize(maxSize) {
-	vec.reserve(maxSize);
-}
-
-void AsyncFileEncrypted::RandomCache::insert(uint32_t block, const Standalone<StringRef>& value) {
-	auto [_, found] = hashMap.insert({ block, value });
-	if (found) {
-		return;
-	} else if (vec.size() < maxSize) {
-		vec.push_back(block);
-	} else {
-		auto index = evict();
-		vec[index] = block;
-	}
-}
-
-Optional<Standalone<StringRef>> AsyncFileEncrypted::RandomCache::get(uint32_t block) const {
-	auto it = hashMap.find(block);
-	if (it == hashMap.end()) {
-		return {};
-	} else {
-		return it->second;
-	}
+	return uncancellable(holdWhile(Reference<AsyncFileEncrypted>::addRef(this),
+	                               file->write(&writeBuffer[0], offsetInBlock, currentBlock * encryptionBlockSize)));
 }
 
 // This test writes random data into an encrypted file in random increments,
 // then reads this data back from the file in random increments, then confirms that
 // the bytes read match the bytes written.
 TEST_CASE("fdbrpc/AsyncFileEncrypted") {
-	state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000);
-	state std::vector<unsigned char> writeBuffer(bytes, 0);
-	deterministicRandom()->randomBytes(&writeBuffer.front(), bytes);
-	state std::vector<unsigned char> readBuffer(bytes, 0);
+	int encryptionBlockSize = 4096;
+	const int bytes = encryptionBlockSize * deterministicRandom()->randomInt(0, 1000);
+	std::vector<unsigned char> writeBuffer(bytes, 0);
+	if (bytes > 0) {
+		deterministicRandom()->randomBytes(writeBuffer.data(), bytes);
+	}
+	std::vector<unsigned char> readBuffer(bytes, 0);
 	ASSERT(g_network->isSimulated());
 	StreamCipherKey::initializeGlobalRandomTestKey();
 	int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE |
-	            IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED |
-	            IAsyncFile::OPEN_NO_AIO;
-	state Reference<IAsyncFile> file =
-	    wait(IAsyncFileSystem::filesystem()->open(joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600));
-	state int bytesWritten = 0;
-	state int chunkSize;
+	            IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO;
+	Reference<IAsyncFile> rawFile = co_await IAsyncFileSystem::filesystem()->open(
+	    joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600);
+	Reference<IAsyncFile> file =
+	    makeReference<AsyncFileEncrypted>(rawFile, AsyncFileEncrypted::Mode::APPEND_ONLY, encryptionBlockSize);
+	int bytesWritten = 0;
+	int chunkSize;
 	while (bytesWritten < bytes) {
 		chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten);
-		wait(file->write(&writeBuffer[bytesWritten], chunkSize, bytesWritten));
+		co_await file->write(&writeBuffer[bytesWritten], chunkSize, bytesWritten);
 		bytesWritten += chunkSize;
 	}
-	wait(file->sync());
-	state int bytesRead = 0;
+	co_await file->sync();
+	int bytesRead = 0;
 	while (bytesRead < bytes) {
 		chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead);
-		int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead));
+		int bytesReadInChunk = co_await file->read(&readBuffer[bytesRead], chunkSize, bytesRead);
 		ASSERT_EQ(bytesReadInChunk, chunkSize);
 		bytesRead += bytesReadInChunk;
 	}
 	ASSERT(writeBuffer == readBuffer);
-	return Void();
 }
diff --git a/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.h
similarity index 86%
rename from fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h
rename to fdbrpc/AsyncFileKAIO.h
index 4a5616012cc..ca9d7b17c2b 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h
+++ b/fdbrpc/AsyncFileKAIO.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileKAIO.actor.h
+ * AsyncFileKAIO.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,14 +21,6 @@
 #pragma once
 #ifdef __linux__
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_ASYNCFILEKAIO_ACTOR_G_H)
-#define FLOW_ASYNCFILEKAIO_ACTOR_G_H
-#include "fdbrpc/AsyncFileKAIO.actor.g.h"
-#elif !defined(FLOW_ASYNCFILEKAIO_ACTOR_H)
-#define FLOW_ASYNCFILEKAIO_ACTOR_H
-
 #include "flow/IAsyncFile.h"
 
 #include <stdio.h>
@@ -36,26 +28,34 @@
 #include <sys/stat.h>
 #include <sys/eventfd.h>
 #include <sys/syscall.h>
-#include "fdbrpc/linux_kaio.h"
+#include "linux_kaio.h"
 #include "flow/Knobs.h"
 #include "fdbrpc/Stats.h"
-#include "flow/UnitTest.h"
 #include "crc32/crc32c.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // Set this to true to enable detailed KAIO request logging, which currently is written to a hardcoded location
 // /data/v7/fdb/
 #define KAIO_LOGGING 0
 
-DESCR struct SlowAioSubmit {
-	int64_t submitDuration; // ns
-	int64_t truncateDuration; // ns
+struct SlowAioSubmit {
+	int64_t submitDuration;
+	int64_t truncateDuration;
 	int64_t numTruncates;
 	int64_t truncateBytes;
 	int64_t largestTruncate;
 };
 
+template <>
+struct Descriptor<SlowAioSubmit>
+  : DescribeType<SlowAioSubmit,
+                 "SlowAioSubmit",
+                 DescribeField<&SlowAioSubmit::submitDuration, "submitDuration", "ns">,
+                 DescribeField<&SlowAioSubmit::truncateDuration, "truncateDuration", "ns">,
+                 DescribeField<&SlowAioSubmit::numTruncates, "numTruncates">,
+                 DescribeField<&SlowAioSubmit::truncateBytes, "truncateBytes">,
+                 DescribeField<&SlowAioSubmit::largestTruncate, "largestTruncate">> {};
+
 class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFileKAIO> {
 public:
 	virtual StringRef getClassName() override { return "AsyncFileKAIO"_sr; }
@@ -208,7 +208,7 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 		}
 		setTimeout(ioTimeout);
 		ctx.evfd = ev->getFD();
-		poll(ev);
+		poll(Uncancellable(), ev);
 
 		g_network->setGlobal(INetwork::enRunCycleFunc, (flowGlobalType)&AsyncFileKAIO::launch);
 	}
@@ -345,12 +345,11 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 		return Void();
 	}
 
-	ACTOR static Future<Void> throwErrorIfFailed(Reference<AsyncFileKAIO> self, Future<Void> sync) {
-		wait(sync);
+	static Future<Void> throwErrorIfFailed(Reference<AsyncFileKAIO> self, Future<Void> sync) {
+		co_await sync;
 		if (self->failed) {
 			throw io_timeout();
 		}
-		return Void();
 	}
 
 	Future<Void> sync() override {
@@ -528,8 +527,8 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 
 		TaskPriority getTask() const { return static_cast<TaskPriority>((prio >> 32) + 1); }
 
-		ACTOR static void deliver(Promise<int> result, bool failed, int r, TaskPriority task) {
-			wait(delay(0, task));
+		static Future<Void> deliver(Uncancellable, Promise<int> result, bool failed, int r, TaskPriority task) {
+			co_await delay(0, task);
 			if (failed)
 				result.sendError(io_timeout());
 			else if (r < 0)
@@ -554,7 +553,7 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 				    .detail("Size", fst.st_size)
 				    .detail("Filename", owner->filename);
 			}
-			deliver(result, owner->failed, r, getTask());
+			deliver(Uncancellable(), result, owner->failed, r, getTask());
 			delete this;
 		}
 
@@ -720,21 +719,21 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 		return oflags;
 	}
 
-	ACTOR static void poll(Reference<IEventFD> ev) {
-		loop {
-			wait(success(ev->read()));
+	static Future<Void> poll(Uncancellable, Reference<IEventFD> ev) {
+		while (true) {
+			co_await ev->read();
 
-			wait(delay(0, TaskPriority::DiskIOComplete));
+			co_await delay(0, TaskPriority::DiskIOComplete);
 
-			linux_ioresult ev[FLOW_KNOBS->MAX_OUTSTANDING];
+			std::vector<linux_ioresult> events(FLOW_KNOBS->MAX_OUTSTANDING);
 			timespec tm;
 			tm.tv_sec = 0;
 			tm.tv_nsec = 0;
 
 			int n;
 
-			loop {
-				n = io_getevents(ctx.iocx, 0, FLOW_KNOBS->MAX_OUTSTANDING, ev, &tm);
+			while (true) {
+				n = io_getevents(ctx.iocx, 0, FLOW_KNOBS->MAX_OUTSTANDING, events.data(), &tm);
 				if (n >= 0 || errno != EINTR)
 					break;
 			}
@@ -766,9 +765,9 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 			}
 
 			for (int i = 0; i < n; i++) {
-				IOBlock* iob = static_cast<IOBlock*>(ev[i].iocb);
+				IOBlock* iob = static_cast<IOBlock*>(events[i].iocb);
 
-				KAIOLogBlockEvent(iob, OpLogEntry::COMPLETE, ev[i].result);
+				KAIOLogBlockEvent(iob, OpLogEntry::COMPLETE, events[i].result);
 
 				if (ctx.ioTimeout > 0) {
 					ctx.removeFromRequestList(iob);
@@ -783,7 +782,7 @@ class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted<AsyncFile
 					break;
 				}
 
-				iob->setResult(ev[i].result);
+				iob->setResult(events[i].result);
 			}
 		}
 	}
@@ -852,93 +851,6 @@ void AsyncFileKAIO::KAIOLogEvent(FILE* logFile,
 }
 #endif
 
-ACTOR Future<Void> runTestOps(Reference<IAsyncFile> f, int numIterations, int fileSize, bool expectedToSucceed) {
-	state void* buf =
-	    FastAllocator<4096>::allocate(); // we leak this if there is an error, but that shouldn't be a big deal
-	state int iteration = 0;
-
-	state bool opTimedOut = false;
-
-	for (; iteration < numIterations; ++iteration) {
-		state std::vector<Future<Void>> futures;
-		state int numOps = deterministicRandom()->randomInt(1, 20);
-		for (; numOps > 0; --numOps) {
-			if (deterministicRandom()->coinflip()) {
-				futures.push_back(
-				    success(f->read(buf, 4096, deterministicRandom()->randomInt(0, fileSize) / 4096 * 4096)));
-			} else {
-				futures.push_back(f->write(buf, 4096, deterministicRandom()->randomInt(0, fileSize) / 4096 * 4096));
-			}
-		}
-		state int fIndex = 0;
-		for (; fIndex < futures.size(); ++fIndex) {
-			try {
-				wait(futures[fIndex]);
-			} catch (Error& e) {
-				ASSERT(!expectedToSucceed);
-				ASSERT(e.code() == error_code_io_timeout);
-				opTimedOut = true;
-			}
-		}
-
-		try {
-			wait(f->sync() && delay(0.1));
-			ASSERT(expectedToSucceed);
-		} catch (Error& e) {
-			ASSERT(!expectedToSucceed && e.code() == error_code_io_timeout);
-		}
-	}
-
-	FastAllocator<4096>::release(buf);
-
-	ASSERT(expectedToSucceed || opTimedOut);
-	return Void();
-}
-
-TEST_CASE("/fdbrpc/AsyncFileKAIO/RequestList") {
-	// This test does nothing in simulation because simulation doesn't support AsyncFileKAIO
-	if (!g_network->isSimulated()) {
-		state Reference<IAsyncFile> f;
-		try {
-			Reference<IAsyncFile> f_ = wait(
-			    AsyncFileKAIO::open("/tmp/__KAIO_TEST_FILE__",
-			                        IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,
-			                        0666,
-			                        nullptr));
-			f = f_;
-			state int fileSize = 2 << 27; // ~100MB
-			wait(f->truncate(fileSize));
-
-			// Test that the request list works as intended with default timeout
-			AsyncFileKAIO::setTimeout(0.0);
-			wait(runTestOps(f, 100, fileSize, true));
-			ASSERT(!((AsyncFileKAIO*)f.getPtr())->failed);
-
-			// Test that the request list works as intended with long timeout
-			AsyncFileKAIO::setTimeout(20.0);
-			wait(runTestOps(f, 100, fileSize, true));
-			ASSERT(!((AsyncFileKAIO*)f.getPtr())->failed);
-
-			// Test that requests timeout correctly
-			AsyncFileKAIO::setTimeout(0.0001);
-			wait(runTestOps(f, 10, fileSize, false));
-			ASSERT(((AsyncFileKAIO*)f.getPtr())->failed);
-		} catch (Error& e) {
-			state Error err = e;
-			if (f) {
-				wait(AsyncFileEIO::deleteFile(f->getFilename(), true));
-			}
-			throw err;
-		}
-
-		wait(AsyncFileEIO::deleteFile(f->getFilename(), true));
-	}
-
-	return Void();
-}
-
 AsyncFileKAIO::Context AsyncFileKAIO::ctx;
 
-#include "flow/unactorcompiler.h"
-#endif
 #endif
diff --git a/fdbrpc/AsyncFileNonDurable.actor.cpp b/fdbrpc/AsyncFileNonDurable.actor.cpp
deleted file mode 100644
index f19eebfba1c..00000000000
--- a/fdbrpc/AsyncFileNonDurable.actor.cpp
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * AsyncFileNonDurable.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbrpc/AsyncFileNonDurable.actor.h"
-
-#include "fdbrpc/SimulatorMachineInfo.h"
-#include "fdbrpc/SimulatorProcessInfo.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-std::map<std::string, Future<Void>> AsyncFileNonDurable::filesBeingDeleted;
-
-Future<Void> waitShutdownSignal() {
-	return success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture());
-}
-
-ACTOR Future<Void> sendOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, TaskPriority taskID) {
-	wait(g_simulator->onProcess(process, taskID));
-	promise.send(Void());
-	return Void();
-}
-
-ACTOR Future<Void> sendErrorOnProcess(ISimulator::ProcessInfo* process,
-                                      Promise<Void> promise,
-                                      Error e,
-                                      TaskPriority taskID) {
-	wait(g_simulator->onProcess(process, taskID));
-	promise.sendError(e);
-	return Void();
-}
-
-ACTOR Future<Void> AsyncFileDetachable::doShutdown(AsyncFileDetachable* self) {
-	wait(success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture()));
-	self->file = Reference<IAsyncFile>();
-	return Void();
-}
-
-ACTOR Future<Reference<IAsyncFile>> AsyncFileDetachable::open(Future<Reference<IAsyncFile>> wrappedFile) {
-	choose {
-		when(wait(success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture()))) {
-			throw io_error().asInjectedFault();
-		}
-		when(Reference<IAsyncFile> f = wait(wrappedFile)) {
-			return makeReference<AsyncFileDetachable>(f);
-		}
-	}
-}
-
-Future<int> AsyncFileDetachable::read(void* data, int length, int64_t offset) {
-	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
-		return io_error().asInjectedFault();
-	return sendErrorOnShutdown(file->read(data, length, offset), assertOnReadWriteCancel);
-}
-
-Future<Void> AsyncFileDetachable::write(void const* data, int length, int64_t offset) {
-	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
-		return io_error().asInjectedFault();
-	return sendErrorOnShutdown(file->write(data, length, offset), assertOnReadWriteCancel);
-}
-
-Future<Void> AsyncFileDetachable::truncate(int64_t size) {
-	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
-		return io_error().asInjectedFault();
-	return sendErrorOnShutdown(file->truncate(size));
-}
-
-Future<Void> AsyncFileDetachable::sync() {
-	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
-		return io_error().asInjectedFault();
-	return sendErrorOnShutdown(file->sync());
-}
-
-Future<int64_t> AsyncFileDetachable::size() const {
-	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
-		return io_error().asInjectedFault();
-	return sendErrorOnShutdown(file->size());
-}
-
-ACTOR Future<Reference<IAsyncFile>> AsyncFileNonDurable::open(std::string filename,
-                                                              std::string actualFilename,
-                                                              Future<Reference<IAsyncFile>> wrappedFile,
-                                                              Reference<DiskParameters> diskParameters,
-                                                              bool aio) {
-	state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-	state TaskPriority currentTaskID = g_network->getCurrentTask();
-	state Future<Void> shutdown = success(currentProcess->shutdownSignal.getFuture());
-
-	//TraceEvent("AsyncFileNonDurableOpenBegin").detail("Filename", filename).detail("Addr", g_simulator->getCurrentProcess()->address);
-	wait(g_simulator->onMachine(currentProcess));
-	try {
-		wait(success(wrappedFile) || shutdown);
-
-		if (shutdown.isReady())
-			throw io_error().asInjectedFault();
-
-		state Reference<IAsyncFile> file = wrappedFile.get();
-
-		// If we are in the process of deleting a file, we can't let someone else modify it at the same time.  We
-		// therefore block the creation of new files until deletion is complete
-		state std::map<std::string, Future<Void>>::iterator deletedFile =
-		    AsyncFileNonDurable::filesBeingDeleted.find(filename);
-		if (deletedFile != AsyncFileNonDurable::filesBeingDeleted.end()) {
-			//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete1").detail("Filename", filename);
-			wait(deletedFile->second || shutdown);
-			//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete2").detail("Filename", filename);
-			if (shutdown.isReady())
-				throw io_error().asInjectedFault();
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-		}
-
-		state Reference<AsyncFileNonDurable> nonDurableFile(
-		    new AsyncFileNonDurable(filename, actualFilename, file, diskParameters, currentProcess->address, aio));
-
-		// Causes the approximateSize member to be set
-		state Future<int64_t> sizeFuture = nonDurableFile->size();
-		wait(success(sizeFuture) || shutdown);
-
-		if (shutdown.isReady())
-			throw io_error().asInjectedFault();
-
-		//TraceEvent("AsyncFileNonDurableOpenComplete").detail("Filename", filename);
-
-		wait(g_simulator->onProcess(currentProcess, currentTaskID));
-
-		return nonDurableFile;
-	} catch (Error& e) {
-		state Error err = e;
-		std::string currentFilename =
-		    (wrappedFile.isReady() && !wrappedFile.isError()) ? wrappedFile.get()->getFilename() : actualFilename;
-		currentProcess->machine->openFiles.erase(currentFilename);
-		//TraceEvent("AsyncFileNonDurableOpenError").errorUnsuppressed(e).detail("Filename", filename).detail("Address", currentProcess->address).detail("Addr", g_simulator->getCurrentProcess()->address);
-		wait(g_simulator->onProcess(currentProcess, currentTaskID));
-		throw err;
-	}
-}
-
-ACTOR Future<int> AsyncFileNonDurable::read(AsyncFileNonDurable* self, void* data, int length, int64_t offset) {
-	state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-	state TaskPriority currentTaskID = g_network->getCurrentTask();
-	wait(g_simulator->onMachine(currentProcess));
-
-	try {
-		state int rep = wait(self->onRead(self, data, length, offset));
-		wait(g_simulator->onProcess(currentProcess, currentTaskID));
-		return rep;
-	} catch (Error& e) {
-		state Error err = e;
-		wait(g_simulator->onProcess(currentProcess, currentTaskID));
-		throw err;
-	}
-}
-
-ACTOR Future<Void> AsyncFileNonDurable::closeFile(AsyncFileNonDurable* self) {
-	state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-	state std::string filename = self->filename;
-
-	g_simulator->getMachineByNetworkAddress(self->openedAddress)->deletingOrClosingFiles.insert(self->getFilename());
-
-	wait(g_simulator->onMachine(currentProcess));
-	try {
-		// Make sure all writes have gone through.
-		Promise<bool> startSyncPromise = self->startSyncPromise;
-		self->startSyncPromise = Promise<bool>();
-		startSyncPromise.send(true);
-
-		std::vector<Future<Void>> outstandingModifications;
-
-		for (auto itr = self->pendingModifications.ranges().begin(); itr != self->pendingModifications.ranges().end();
-		     ++itr)
-			if (itr->value().isValid() && !itr->value().isReady())
-				outstandingModifications.push_back(itr->value());
-
-		// Ignore errors here so that all modifications can finish
-		wait(waitForAllReady(outstandingModifications));
-
-		// Make sure we aren't in the process of killing the file
-		if (self->killed.isSet())
-			wait(self->killComplete.getFuture());
-
-		// Remove this file from the filesBeingDeleted map so that new files can be created with this filename
-		g_simulator->getMachineByNetworkAddress(self->openedAddress)->closingFiles.erase(self->getFilename());
-		g_simulator->getMachineByNetworkAddress(self->openedAddress)->deletingOrClosingFiles.erase(self->getFilename());
-		AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
-		//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
-
-		delete self;
-		return Void();
-	} catch (Error& e) {
-		state Error err = e;
-		throw err;
-	}
-}
-
-void AsyncFileNonDurable::removeOpenFile(std::string filename, AsyncFileNonDurable* file) {
-	auto& openFiles = g_simulator->getCurrentProcess()->machine->openFiles;
-
-	auto iter = openFiles.find(filename);
-
-	// Various actions (e.g. simulated delete) can remove a file from openFiles prematurely, so it may already
-	// be gone. Renamed files (from atomic write and create) will also be present under only one of the two
-	// names.
-	if (iter != openFiles.end()) {
-		// even if the filename exists, it doesn't mean that it references the same file. It could be that the
-		// file was renamed and later a file with the same name was opened.
-		if (iter->second.getPtrIfReady().orDefault(nullptr) == file) {
-			openFiles.erase(iter);
-		}
-	}
-}
diff --git a/fdbrpc/AsyncFileNonDurable.cpp b/fdbrpc/AsyncFileNonDurable.cpp
new file mode 100644
index 00000000000..2e090458a3f
--- /dev/null
+++ b/fdbrpc/AsyncFileNonDurable.cpp
@@ -0,0 +1,215 @@
+/*
+ * AsyncFileNonDurable.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbrpc/AsyncFileNonDurable.h"
+
+#include "fdbrpc/SimulatorMachineInfo.h"
+#include "fdbrpc/SimulatorProcessInfo.h"
+
+#include "flow/CoroUtils.h"
+
+std::map<std::string, Future<Void>> AsyncFileNonDurable::filesBeingDeleted;
+
+Future<Void> waitShutdownSignal() {
+	return success(g_simulator->getCurrentProcess()->shutdownSignal.getFuture());
+}
+
+Future<Void> sendOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, TaskPriority taskID) {
+	co_await g_simulator->onProcess(process, taskID);
+	promise.send(Void());
+}
+
+Future<Void> sendErrorOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, Error e, TaskPriority taskID) {
+	co_await g_simulator->onProcess(process, taskID);
+	promise.sendError(e);
+}
+
+Future<Void> AsyncFileDetachable::doShutdown(AsyncFileDetachable* self) {
+	co_await g_simulator->getCurrentProcess()->shutdownSignal.getFuture();
+	self->file = Reference<IAsyncFile>();
+}
+
+Future<Reference<IAsyncFile>> AsyncFileDetachable::open(Future<Reference<IAsyncFile>> wrappedFile) {
+	auto res = co_await race(g_simulator->getCurrentProcess()->shutdownSignal.getFuture(), wrappedFile);
+	if (res.index() == 0) {
+		throw io_error().asInjectedFault();
+	} else if (res.index() == 1) {
+		Reference<IAsyncFile> f = std::get<1>(std::move(res));
+		co_return makeReference<AsyncFileDetachable>(f);
+	}
+	UNREACHABLE();
+}
+
+Future<int> AsyncFileDetachable::read(void* data, int length, int64_t offset) {
+	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		return io_error().asInjectedFault();
+	return sendErrorOnShutdown(file->read(data, length, offset), assertOnReadWriteCancel);
+}
+
+Future<Void> AsyncFileDetachable::write(void const* data, int length, int64_t offset) {
+	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		return io_error().asInjectedFault();
+	return sendErrorOnShutdown(file->write(data, length, offset), assertOnReadWriteCancel);
+}
+
+Future<Void> AsyncFileDetachable::truncate(int64_t size) {
+	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		return io_error().asInjectedFault();
+	return sendErrorOnShutdown(file->truncate(size));
+}
+
+Future<Void> AsyncFileDetachable::sync() {
+	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		return io_error().asInjectedFault();
+	return sendErrorOnShutdown(file->sync());
+}
+
+Future<int64_t> AsyncFileDetachable::size() const {
+	if (!file.getPtr() || g_simulator->getCurrentProcess()->shutdownSignal.getFuture().isReady())
+		return io_error().asInjectedFault();
+	return sendErrorOnShutdown(file->size());
+}
+
+Future<Reference<IAsyncFile>> AsyncFileNonDurable::open(std::string filename,
+                                                        std::string actualFilename,
+                                                        Future<Reference<IAsyncFile>> wrappedFile,
+                                                        Reference<DiskParameters> diskParameters,
+                                                        bool aio) {
+	ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+	TaskPriority currentTaskID = g_network->getCurrentTask();
+	Future<Void> shutdown = success(currentProcess->shutdownSignal.getFuture());
+
+	//TraceEvent("AsyncFileNonDurableOpenBegin").detail("Filename", filename).detail("Addr", g_simulator->getCurrentProcess()->address);
+	co_await g_simulator->onMachine(currentProcess);
+	Error err;
+	try {
+		co_await (success(wrappedFile) || shutdown);
+
+		if (shutdown.isReady())
+			throw io_error().asInjectedFault();
+
+		Reference<IAsyncFile> file = wrappedFile.get();
+
+		// If we are in the process of deleting a file, we can't let someone else modify it at the same time.  We
+		// therefore block the creation of new files until deletion is complete
+		auto deletedFile = AsyncFileNonDurable::filesBeingDeleted.find(filename);
+		if (deletedFile != AsyncFileNonDurable::filesBeingDeleted.end()) {
+			//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete1").detail("Filename", filename);
+			co_await (deletedFile->second || shutdown);
+			//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete2").detail("Filename", filename);
+			if (shutdown.isReady())
+				throw io_error().asInjectedFault();
+			co_await g_simulator->onProcess(currentProcess, currentTaskID);
+		}
+
+		Reference<AsyncFileNonDurable> nonDurableFile(
+		    new AsyncFileNonDurable(filename, actualFilename, file, diskParameters, currentProcess->address, aio));
+
+		// Causes the approximateSize member to be set
+		Future<int64_t> sizeFuture = nonDurableFile->size();
+		co_await (success(sizeFuture) || shutdown);
+
+		if (shutdown.isReady())
+			throw io_error().asInjectedFault();
+
+		//TraceEvent("AsyncFileNonDurableOpenComplete").detail("Filename", filename);
+
+		co_await g_simulator->onProcess(currentProcess, currentTaskID);
+		co_return nonDurableFile;
+	} catch (Error& e) {
+		err = e;
+	}
+
+	std::string currentFilename =
+	    (wrappedFile.isReady() && !wrappedFile.isError()) ? wrappedFile.get()->getFilename() : actualFilename;
+	currentProcess->machine->openFiles.erase(currentFilename);
+	//TraceEvent("AsyncFileNonDurableOpenError").errorUnsuppressed(e).detail("Filename", filename).detail("Address", currentProcess->address).detail("Addr", g_simulator->getCurrentProcess()->address);
+	co_await g_simulator->onProcess(currentProcess, currentTaskID);
+	throw err;
+}
+
+Future<int> AsyncFileNonDurable::read(AsyncFileNonDurable* self, void* data, int length, int64_t offset) {
+	ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+	TaskPriority currentTaskID = g_network->getCurrentTask();
+	co_await g_simulator->onMachine(currentProcess);
+
+	Error err;
+	try {
+		int rep = co_await self->onRead(self, data, length, offset);
+		co_await g_simulator->onProcess(currentProcess, currentTaskID);
+		co_return rep;
+	} catch (Error& e) {
+		err = e;
+	}
+
+	co_await g_simulator->onProcess(currentProcess, currentTaskID);
+	throw err;
+}
+
+Future<Void> AsyncFileNonDurable::closeFile(AsyncFileNonDurable* self) {
+	ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+
+	g_simulator->getMachineByNetworkAddress(self->openedAddress)->deletingOrClosingFiles.insert(self->getFilename());
+
+	co_await g_simulator->onMachine(currentProcess);
+	// Make sure all writes have gone through.
+	Promise<bool> startSyncPromise = self->startSyncPromise;
+	self->startSyncPromise = Promise<bool>();
+	startSyncPromise.send(true);
+
+	std::vector<Future<Void>> outstandingModifications;
+
+	for (auto itr = self->pendingModifications.ranges().begin(); itr != self->pendingModifications.ranges().end();
+	     ++itr)
+		if (itr->value().isValid() && !itr->value().isReady())
+			outstandingModifications.push_back(itr->value());
+
+	// Ignore errors here so that all modifications can finish
+	co_await waitForAllReady(outstandingModifications);
+
+	// Make sure we aren't in the process of killing the file
+	if (self->killed.isSet())
+		co_await self->killComplete.getFuture();
+
+	// Remove this file from the filesBeingDeleted map so that new files can be created with this filename
+	g_simulator->getMachineByNetworkAddress(self->openedAddress)->closingFiles.erase(self->getFilename());
+	g_simulator->getMachineByNetworkAddress(self->openedAddress)->deletingOrClosingFiles.erase(self->getFilename());
+	AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
+	//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
+
+	delete self;
+}
+
+void AsyncFileNonDurable::removeOpenFile(std::string filename, AsyncFileNonDurable* file) {
+	auto& openFiles = g_simulator->getCurrentProcess()->machine->openFiles;
+
+	auto iter = openFiles.find(filename);
+
+	// Various actions (e.g. simulated delete) can remove a file from openFiles prematurely, so it may already
+	// be gone. Renamed files (from atomic write and create) will also be present under only one of the two
+	// names.
+	if (iter != openFiles.end()) {
+		// even if the filename exists, it doesn't mean that it references the same file. It could be that the
+		// file was renamed and later a file with the same name was opened.
+		if (iter->second.getPtrIfReady().orDefault(nullptr) == file) {
+			openFiles.erase(iter);
+		}
+	}
+}
diff --git a/fdbrpc/include/fdbrpc/AsyncFileWinASIO.actor.h b/fdbrpc/AsyncFileWinASIO.h
similarity index 93%
rename from fdbrpc/include/fdbrpc/AsyncFileWinASIO.actor.h
rename to fdbrpc/AsyncFileWinASIO.h
index d9cbe35c955..b94fc7acc31 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileWinASIO.actor.h
+++ b/fdbrpc/AsyncFileWinASIO.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileWinASIO.actor.h
+ * AsyncFileWinASIO.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,14 +24,6 @@
 
 #define Net2AsyncFile AsyncFileWinASIO
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_ASYNCFILEWINASIO_ACTOR_G_H)
-#define FLOW_ASYNCFILEWINASIO_ACTOR_G_H
-#include "fdbrpc/AsyncFileWinASIO.actor.g.h"
-#elif !defined(FLOW_ASYNCFILEWINASIO_ACTOR_H)
-#define FLOW_ASYNCFILEWINASIO_ACTOR_H
-
 #include <Windows.h>
 #include <boost/bind/bind.hpp>
 #undef min
@@ -77,7 +69,7 @@ class AsyncFileWinASIO final : public IAsyncFile, public ReferenceCounted<AsyncF
 			    .detail("Mode", mode);
 			return e;
 		}
-		return Reference<IAsyncFile>(new AsyncFileWinASIO(*ios, h, flags, filename));
+		return Reference<IAsyncFile>(makeReference<AsyncFileWinASIO>(*ios, h, flags, filename));
 	}
 	static Future<Void> deleteFile(std::string filename, bool mustBeDurable) {
 		::deleteFile(filename);
@@ -205,6 +197,9 @@ class AsyncFileWinASIO final : public IAsyncFile, public ReferenceCounted<AsyncF
 	~AsyncFileWinASIO() {}
 
 private:
+	template <class P, class... Args>
+	friend Reference<P> makeReference(Args&&... args);
+
 	boost::asio::windows::random_access_handle file;
 	int flags;
 	std::string filename;
@@ -214,4 +209,3 @@ class AsyncFileWinASIO final : public IAsyncFile, public ReferenceCounted<AsyncF
 };
 
 #endif
-#endif
diff --git a/fdbrpc/AsyncFileWriteChecker.cpp b/fdbrpc/AsyncFileWriteChecker.cpp
index ce99d593874..bcb03dc2291 100644
--- a/fdbrpc/AsyncFileWriteChecker.cpp
+++ b/fdbrpc/AsyncFileWriteChecker.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbrpc/AsyncFileWriteChecker.actor.h"
+#include "AsyncFileWriteChecker.h"
 #include "flow/UnitTest.h"
 
 Optional<int> AsyncFileWriteChecker::checksumHistoryBudget = {};
@@ -39,7 +39,7 @@ class LRU2 {
 			page = _page;
 			writeInfo = _writeInfo;
 			next = nullptr;
-			prev = NULL;
+			prev = nullptr;
 		}
 	};
 
@@ -62,7 +62,7 @@ class LRU2 {
 	}
 
 public:
-	LRU2(std::string _fileName) {
+	explicit LRU2(std::string _fileName) {
 		fileName = _fileName;
 		maxFullPagePlusOne = 0;
 		start = new node(0, AsyncFileWriteChecker::WriteInfo());
@@ -100,7 +100,7 @@ class LRU2 {
 	}
 
 	uint32_t randomPage() {
-		if (m.size() == 0) {
+		if (m.empty()) {
 			return 0;
 		}
 		auto it = m.begin();
@@ -144,7 +144,7 @@ class LRU2 {
 	}
 
 	uint32_t leastRecentlyUsedPage() {
-		if (m.size() == 0) {
+		if (m.empty()) {
 			return 0;
 		}
 		return end->prev->page;
@@ -226,4 +226,4 @@ TEST_CASE("/fdbrpc/AsyncFileWriteChecker/LRU") {
 		i += 1;
 	}
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbrpc/include/fdbrpc/AsyncFileWriteChecker.actor.h b/fdbrpc/AsyncFileWriteChecker.h
similarity index 92%
rename from fdbrpc/include/fdbrpc/AsyncFileWriteChecker.actor.h
rename to fdbrpc/AsyncFileWriteChecker.h
index c146392c5f5..ec999af521e 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileWriteChecker.actor.h
+++ b/fdbrpc/AsyncFileWriteChecker.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileWriteChecker.actor.h
+ * AsyncFileWriteChecker.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -17,11 +17,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#if defined(NO_INTELLISENSE) && !defined(ASYNC_FILE_WRITE_CHECKER_ACTOR_G_H)
-#define ASYNC_FILE_WRITE_CHECKER_ACTOR_G_H
-#include "fdbrpc/AsyncFileWriteChecker.actor.g.h"
-#elif !defined(ASYNC_FILE_WRITE_CHECKER_ACTOR_H)
-#define ASYNC_FILE_WRITE_CHECKER_ACTOR_H
 
 #include "flow/IAsyncFile.h"
 #include "crc32/crc32c.h"
@@ -30,7 +25,6 @@
 #include <memcheck.h>
 #endif
 
-#include "flow/actorcompiler.h"
 static double millisecondsPerSecond = 1000;
 
 // this class does checksum for the wrapped IAsyncFile in read and writes opertions.
@@ -117,7 +111,7 @@ class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFi
 		std::unordered_map<uint32_t, AsyncFileWriteChecker::WriteInfo> pageContents;
 
 	public:
-		LRU(std::string _fileName) {
+		explicit LRU(std::string _fileName) {
 			step = 0;
 			fileName = _fileName;
 		}
@@ -188,7 +182,7 @@ class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFi
 		}
 	};
 
-	AsyncFileWriteChecker(Reference<IAsyncFile> f) : m_f(f), lru(f->getFilename()) {
+	explicit AsyncFileWriteChecker(Reference<IAsyncFile> f) : m_f(f), lru(f->getFilename()) {
 		// Initialize the static history budget the first time (and only the first time) a file is opened.
 		if (!checksumHistoryBudget.present()) {
 			checksumHistoryBudget = FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY;
@@ -221,26 +215,26 @@ class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFi
 	static Optional<int> checksumHistoryBudget;
 	static int checksumHistoryPageSize;
 
-	ACTOR Future<Void> sweep(AsyncFileWriteChecker* self) {
-		loop {
+	Future<Void> sweep(AsyncFileWriteChecker* self) {
+		while (true) {
 			// for each page, read and do checksum
 			// scan from the least recently used, thus it is safe to quit if data has not been synced
-			state uint32_t page = self->lru.leastRecentlyUsedPage();
+			uint32_t page = self->lru.leastRecentlyUsedPage();
 			while (self->writing.find(page) != self->writing.end() || page == 0) {
 				// avoid concurrent ops
-				wait(delay(FLOW_KNOBS->ASYNC_FILE_WRITE_CHEKCER_CHECKING_DELAY));
+				co_await delay(FLOW_KNOBS->ASYNC_FILE_WRITE_CHEKCER_CHECKING_DELAY);
 				continue;
 			}
 			int64_t offset = page * checksumHistoryPageSize;
 			// perform a read to verify checksum, it will remove the entry upon success
-			wait(success(self->read(self->pageBuffer, checksumHistoryPageSize, offset)));
+			co_await self->read(self->pageBuffer, checksumHistoryPageSize, offset);
 		}
 	}
 
-	ACTOR Future<Void> runChecksumLogger(AsyncFileWriteChecker* self) {
-		state double delayDuration = FLOW_KNOBS->ASYNC_FILE_WRITE_CHEKCER_LOGGING_INTERVAL;
-		loop {
-			wait(delay(delayDuration));
+	Future<Void> runChecksumLogger(AsyncFileWriteChecker* self) {
+		double delayDuration = FLOW_KNOBS->ASYNC_FILE_WRITE_CHEKCER_LOGGING_INTERVAL;
+		while (true) {
+			co_await delay(delayDuration);
 			// TODO: add more stats, such as total checked, current entries, budget
 			TraceEvent("AsyncFileWriteChecker")
 			    .detail("Delay", delayDuration)
@@ -338,6 +332,3 @@ class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFi
 		return pages;
 	}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
\ No newline at end of file
diff --git a/fdbrpc/Base64Decode.cpp b/fdbrpc/Base64Decode.cpp
index f39496c2e00..8a267402fd3 100644
--- a/fdbrpc/Base64Decode.cpp
+++ b/fdbrpc/Base64Decode.cpp
@@ -20,8 +20,8 @@
 
 #include <string_view>
 #include <fmt/format.h>
-#include "fdbrpc/Base64Encode.h"
-#include "fdbrpc/Base64Decode.h"
+#include "Base64Encode.h"
+#include "Base64Decode.h"
 #include "flow/flow.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
diff --git a/fdbrpc/include/fdbrpc/Base64Decode.h b/fdbrpc/Base64Decode.h
similarity index 100%
rename from fdbrpc/include/fdbrpc/Base64Decode.h
rename to fdbrpc/Base64Decode.h
diff --git a/fdbrpc/Base64Encode.cpp b/fdbrpc/Base64Encode.cpp
index 0b75fb20b64..3a584c2fe72 100644
--- a/fdbrpc/Base64Encode.cpp
+++ b/fdbrpc/Base64Encode.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbrpc/Base64Encode.h"
+#include "Base64Encode.h"
 
 namespace {
 
diff --git a/fdbrpc/include/fdbrpc/Base64Encode.h b/fdbrpc/Base64Encode.h
similarity index 100%
rename from fdbrpc/include/fdbrpc/Base64Encode.h
rename to fdbrpc/Base64Encode.h
diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt
index d6a7c8adee9..0a283a73818 100644
--- a/fdbrpc/CMakeLists.txt
+++ b/fdbrpc/CMakeLists.txt
@@ -1,4 +1,5 @@
 fdb_find_sources(FDBRPC_SRCS)
+list(REMOVE_ITEM FDBRPC_SRCS LinkTest.cpp)
 
 set(COMPILE_EIO OFF)
 
@@ -26,7 +27,7 @@ add_flow_target(STATIC_LIBRARY NAME fdbrpc_sampling
   DISABLE_ACTOR_DIAGNOSTICS ${FDBRPC_SRCS_DISABLE_ACTOR_DIAGNOSTICS})
 
 add_flow_target(LINK_TEST NAME fdbrpclinktest SRCS LinkTest.cpp)
-target_link_libraries(fdbrpclinktest PRIVATE fdbrpc rapidjson)
+target_link_libraries(fdbrpclinktest PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,fdbrpc>" rapidjson)
 target_include_directories(fdbrpclinktest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/libeio)
 
 if(COMPILE_EIO)
@@ -60,8 +61,13 @@ if(${COROUTINE_IMPL} STREQUAL libcoro)
   endif()
 endif()
 
-target_include_directories(fdbrpc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-target_include_directories(fdbrpc PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/libeio)
+target_include_directories(fdbrpc
+  PUBLIC
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    "${CMAKE_CURRENT_BINARY_DIR}/include"
+  PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}"
+    "${CMAKE_CURRENT_SOURCE_DIR}/libeio")
 target_link_libraries(fdbrpc PUBLIC flow libb64 md5 PRIVATE rapidjson)
 
 if (WITH_GRPC)
@@ -78,7 +84,13 @@ if (WITH_GRPC)
   target_link_libraries(fdbrpc_sampling PUBLIC proto_fdbrpc_test proto_fdbrpc_file_transfer)
 endif()
 
-target_include_directories(fdbrpc_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/libeio)
+target_include_directories(fdbrpc_sampling
+  PUBLIC
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    "${CMAKE_CURRENT_BINARY_DIR}/include"
+  PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}"
+    "${CMAKE_CURRENT_SOURCE_DIR}/libeio")
 target_link_libraries(fdbrpc_sampling PUBLIC flow_sampling libb64 md5 PRIVATE rapidjson)
 
 if(${COROUTINE_IMPL} STREQUAL libcoro)
@@ -96,4 +108,9 @@ if(WIN32)
   add_dependencies(fdbrpc_sampling_actors fdbrpc_actors)
 endif()
 
+if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
+                                     # x-compiling.
+  add_subdirectory(bench EXCLUDE_FROM_ALL)
+endif()
+
 add_subdirectory(tests)
diff --git a/fdbrpc/include/fdbrpc/ContinuousSample.h b/fdbrpc/ContinuousSample.h
similarity index 100%
rename from fdbrpc/include/fdbrpc/ContinuousSample.h
rename to fdbrpc/ContinuousSample.h
diff --git a/fdbrpc/DDSketchTest.actor.cpp b/fdbrpc/DDSketchTest.cpp
similarity index 97%
rename from fdbrpc/DDSketchTest.actor.cpp
rename to fdbrpc/DDSketchTest.cpp
index f2ecb48bd29..3f1a4ee7fc4 100644
--- a/fdbrpc/DDSketchTest.actor.cpp
+++ b/fdbrpc/DDSketchTest.cpp
@@ -4,7 +4,6 @@
 #include "flow/UnitTest.h"
 #include <limits>
 #include <random>
-#include "flow/actorcompiler.h" // has to be last include
 void forceLinkDDSketchTests() {}
 
 TEST_CASE("/fdbrpc/ddsketch/accuracy") {
diff --git a/fdbrpc/FailureMonitor.actor.cpp b/fdbrpc/FailureMonitor.cpp
similarity index 89%
rename from fdbrpc/FailureMonitor.actor.cpp
rename to fdbrpc/FailureMonitor.cpp
index d537261fae0..75579ec0ed9 100644
--- a/fdbrpc/FailureMonitor.actor.cpp
+++ b/fdbrpc/FailureMonitor.cpp
@@ -1,5 +1,5 @@
 /*
- * FailureMonitor.actor.cpp
+ * FailureMonitor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,27 +19,26 @@
  */
 
 #include "fdbrpc/FailureMonitor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
-ACTOR Future<Void> waitForStateEqual(IFailureMonitor* monitor, Endpoint endpoint, FailureStatus status) {
-	loop {
+Future<Void> waitForStateEqual(IFailureMonitor* monitor, Endpoint endpoint, FailureStatus status) {
+	while (true) {
 		Future<Void> change = monitor->onStateChanged(endpoint);
 		if (monitor->getState(endpoint) == status)
-			return Void();
-		wait(change);
+			co_return;
+		co_await change;
 	}
 }
 
-ACTOR Future<Void> waitForContinuousFailure(IFailureMonitor* monitor,
-                                            Endpoint endpoint,
-                                            double sustainedFailureDuration,
-                                            double slope) {
-	state double startT = now();
+Future<Void> waitForContinuousFailure(IFailureMonitor* monitor,
+                                      Endpoint endpoint,
+                                      double sustainedFailureDuration,
+                                      double slope) {
+	double startT = now();
 
-	loop {
-		wait(monitor->onFailed(endpoint));
+	while (true) {
+		co_await monitor->onFailed(endpoint);
 		if (monitor->permanentlyFailed(endpoint))
-			return Void();
+			co_return;
 
 		// X == sustainedFailureDuration + slope * (now()-startT+X)
 		double waitDelay = (sustainedFailureDuration + slope * (now() - startT)) / (1 - slope);
@@ -50,12 +49,11 @@ ACTOR Future<Void> waitForContinuousFailure(IFailureMonitor* monitor,
 		             FLOW_KNOBS->SERVER_REQUEST_INTERVAL)) // We will not get a failure monitoring update in this amount
 		                                                   // of time, so there is no point in waiting for changes
 			waitDelay = 0;
-		choose {
-			when(wait(monitor->onStateEqual(endpoint, FailureStatus(false)))) {
-			} // SOMEDAY: Use onStateChanged() for efficiency
-			when(wait(delay(waitDelay))) {
-				return Void();
-			}
+
+		// SOMEDAY: Use onStateChanged() for efficiency
+		if (auto healthy = co_await timeout(monitor->onStateEqual(endpoint, FailureStatus(false)), waitDelay);
+		    !healthy.present()) {
+			co_return;
 		}
 	}
 }
@@ -156,7 +154,7 @@ void SimpleFailureMonitor::notifyDisconnect(NetworkAddress const& address) {
 Future<Void> SimpleFailureMonitor::onDisconnectOrFailure(Endpoint const& endpoint) {
 	// If the endpoint or address is already failed, return right away
 	auto i = addressStatus.find(endpoint.getPrimaryAddress());
-	if (i == addressStatus.end() || i->second.isFailed() || failedEndpoints.count(endpoint)) {
+	if (i == addressStatus.end() || i->second.isFailed() || failedEndpoints.contains(endpoint)) {
 		TraceEvent event("AlreadyDisconnected");
 		if (endpoint.token.first() == 0xffffffffffffffff) {
 			// well known endpoint
@@ -187,14 +185,14 @@ Future<Void> SimpleFailureMonitor::onStateChanged(Endpoint const& endpoint) {
 	//   failure status for that endpoint can never change (and we could be spuriously triggered by setStatus)
 	// Also returns spuriously when notifyDisconnect is called (which doesn't actually change the state), but callers
 	//   check the state so it's OK
-	if (failedEndpoints.count(endpoint))
+	if (failedEndpoints.contains(endpoint))
 		return Never();
 	else
 		return endpointKnownFailed.onChange(endpoint);
 }
 
 FailureStatus SimpleFailureMonitor::getState(Endpoint const& endpoint) const {
-	if (failedEndpoints.count(endpoint))
+	if (failedEndpoints.contains(endpoint))
 		return FailureStatus(true);
 	else {
 		auto a = addressStatus.find(endpoint.getPrimaryAddress());
@@ -214,7 +212,7 @@ FailureStatus SimpleFailureMonitor::getState(NetworkAddress const& address) cons
 }
 
 bool SimpleFailureMonitor::onlyEndpointFailed(Endpoint const& endpoint) const {
-	if (!failedEndpoints.count(endpoint))
+	if (!failedEndpoints.contains(endpoint))
 		return false;
 	auto a = addressStatus.find(endpoint.getPrimaryAddress());
 	if (a == addressStatus.end())
@@ -224,7 +222,7 @@ bool SimpleFailureMonitor::onlyEndpointFailed(Endpoint const& endpoint) const {
 }
 
 bool SimpleFailureMonitor::permanentlyFailed(Endpoint const& endpoint) const {
-	return failedEndpoints.count(endpoint);
+	return failedEndpoints.contains(endpoint);
 }
 
 bool SimpleFailureMonitor::knownUnauthorized(Endpoint const& endpoint) const {
diff --git a/fdbrpc/FileTransfer.cpp b/fdbrpc/FileTransfer.cpp
index 709e3dc6c3d..479507d7439 100644
--- a/fdbrpc/FileTransfer.cpp
+++ b/fdbrpc/FileTransfer.cpp
@@ -20,7 +20,7 @@
 #ifdef FLOW_GRPC_ENABLED
 #include <fstream>
 
-#include <fdbrpc/FileTransfer.h>
+#include "FileTransfer.h"
 #include "flow/IRandom.h"
 #include "crc32/crc32c.h"
 
diff --git a/fdbrpc/include/fdbrpc/FileTransfer.h b/fdbrpc/FileTransfer.h
similarity index 96%
rename from fdbrpc/include/fdbrpc/FileTransfer.h
rename to fdbrpc/FileTransfer.h
index baf2861fdf1..deaa66601a2 100644
--- a/fdbrpc/include/fdbrpc/FileTransfer.h
+++ b/fdbrpc/FileTransfer.h
@@ -57,7 +57,8 @@ class FileTransferServiceImpl final : public fdbrpc::FileTransferService::Servic
 
 class FileTransferClient {
 public:
-	FileTransferClient(std::shared_ptr<grpc::Channel> channel) : stub_(fdbrpc::FileTransferService::NewStub(channel)) {}
+	explicit FileTransferClient(std::shared_ptr<grpc::Channel> channel)
+	  : stub_(fdbrpc::FileTransferService::NewStub(channel)) {}
 
 	std::optional<fdbrpc::GetFileInfoReply> GetFileInfo(const std::string& filename, bool get_crc_checksum = false);
 
diff --git a/fdbrpc/FlowGrpc.cpp b/fdbrpc/FlowGrpc.cpp
index 5cd720e1a5d..092eca2235b 100644
--- a/fdbrpc/FlowGrpc.cpp
+++ b/fdbrpc/FlowGrpc.cpp
@@ -93,7 +93,7 @@ Future<Void> GrpcServer::runInternal() {
 			co_await delay(CONFIG_STARTUP_DELAY_BETWEEN_RESTART);
 
 			// gRPC can't run a server without registered service.
-			if (registered_services_.size() > 0) {
+			if (!registered_services_.empty()) {
 				break;
 			} else {
 				next = on_services_changed_.onTrigger();
@@ -200,7 +200,7 @@ void GrpcServer::registerRoleServices(const UID& owner_id, const ServiceList& se
 	on_services_changed_.trigger();
 }
 
-Future<Void> GrpcServer::deregisterRoleServices(const UID& owner_id) {
+Future<Void> GrpcServer::deregisterRoleServices(UID owner_id) {
 	ASSERT(g_network->isOnMainThread());
 	co_await stopServer();
 	registered_services_.erase(owner_id);
diff --git a/fdbrpc/FlowGrpcTests.actor.cpp b/fdbrpc/FlowGrpcTests.actor.cpp
index 4215f52b19e..7bf99f5d90a 100644
--- a/fdbrpc/FlowGrpcTests.actor.cpp
+++ b/fdbrpc/FlowGrpcTests.actor.cpp
@@ -21,7 +21,7 @@
 #include <cstdio>
 
 #include "fdbrpc/FlowGrpc.h"
-#include "fdbrpc/FlowGrpcTests.h"
+#include "FlowGrpcTests.h"
 #include "flow/UnitTest.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
diff --git a/fdbrpc/FlowGrpcTests.cpp b/fdbrpc/FlowGrpcTests.cpp
index 7e1b8713165..f1c0d017c7f 100644
--- a/fdbrpc/FlowGrpcTests.cpp
+++ b/fdbrpc/FlowGrpcTests.cpp
@@ -23,10 +23,10 @@
 #include <thread>
 #include <mutex>
 #include "fdbrpc/FlowGrpc.h"
-#include "fdbrpc/FlowGrpcTests.h"
-#include "fdbrpc/FileTransfer.h"
+#include "FlowGrpcTests.h"
+#include "FileTransfer.h"
 #include "flow/UnitTest.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/flow.h"
 
 // So that tests are not optimized out. :/
diff --git a/fdbrpc/include/fdbrpc/FlowGrpcTests.h b/fdbrpc/FlowGrpcTests.h
similarity index 96%
rename from fdbrpc/include/fdbrpc/FlowGrpcTests.h
rename to fdbrpc/FlowGrpcTests.h
index a7bbeb302aa..9598141f613 100644
--- a/fdbrpc/include/fdbrpc/FlowGrpcTests.h
+++ b/fdbrpc/FlowGrpcTests.h
@@ -86,7 +86,7 @@ class TestEchoServiceImpl final : public TestEchoService::Service {
 
 class EchoClient {
 public:
-	EchoClient(const shared_ptr<Channel>& channel) : stub_(TestEchoService::NewStub(channel)) {}
+	explicit EchoClient(const shared_ptr<Channel>& channel) : stub_(TestEchoService::NewStub(channel)) {}
 
 	std::string Echo(const std::string& message) {
 		EchoRequest request;
diff --git a/fdbrpc/FlowTests.actor.cpp b/fdbrpc/FlowTests.actor.cpp
index 0ec05c2818b..bb5565fde63 100644
--- a/fdbrpc/FlowTests.actor.cpp
+++ b/fdbrpc/FlowTests.actor.cpp
@@ -31,7 +31,7 @@
 #include "flow/WriteOnlySet.h"
 #include "fdbrpc/fdbrpc.h"
 #include "flow/IAsyncFile.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbrpc/grpc/AsyncTaskExecutor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -373,7 +373,7 @@ struct Int {
 	constexpr static FileIdentifier file_identifier = 12345;
 	uint32_t value;
 	Int() = default;
-	Int(uint32_t value) : value(value) {}
+	explicit(false) Int(uint32_t value) : value(value) {}
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, value);
@@ -557,7 +557,8 @@ TEST_CASE("/flow/flow/callbacks") {
 
 	onReady(std::move(f), [&result](int x) { result = x; }, [&result](Error e) { result = -1; });
 	onReady(p.getFuture(), [&happened](int) { happened = true; }, [&happened](Error) { happened = true; });
-	ASSERT(!f.isValid());
+	ASSERT(
+	    !f.isValid()); // NOLINT(bugprone-use-after-move): this test intentionally checks the moved-from Future state.
 	ASSERT(p.isValid() && !p.isSet() && p.getFutureReferenceCount() == 1);
 	ASSERT(result == 0 && !happened);
 
@@ -574,7 +575,8 @@ TEST_CASE("/flow/flow/callbacks") {
 	f = p.getFuture();
 	result = 0;
 	onReady(std::move(f), [&result](int x) { result = x; }, [&result](Error e) { result = -e.code(); });
-	ASSERT(!f.isValid());
+	ASSERT(
+	    !f.isValid()); // NOLINT(bugprone-use-after-move): this test intentionally checks the moved-from Future state.
 	ASSERT(p.isValid() && !p.isSet() && p.getFutureReferenceCount() == 1);
 	ASSERT(result == 0);
 
@@ -1394,8 +1396,8 @@ TEST_CASE("/flow/DeterministicRandom/SignedOverflow") {
 struct Tracker {
 	int copied;
 	bool moved;
-	Tracker(int copied = 0) : copied(copied), moved(false) {}
-	Tracker(Tracker&& other) : Tracker(other.copied) {
+	explicit Tracker(int copied = 0) : copied(copied), moved(false) {}
+	explicit(false) Tracker(Tracker&& other) : Tracker(other.copied) {
 		ASSERT(!other.moved);
 		other.moved = true;
 	}
@@ -1406,7 +1408,7 @@ struct Tracker {
 		this->copied = other.copied;
 		return *this;
 	}
-	Tracker(const Tracker& other) : Tracker(other.copied + 1) { ASSERT(!other.moved); }
+	explicit(false) Tracker(const Tracker& other) : Tracker(other.copied + 1) { ASSERT(!other.moved); }
 	Tracker& operator=(const Tracker& other) {
 		ASSERT(!other.moved);
 		this->moved = false;
@@ -1487,7 +1489,8 @@ TEST_CASE("/flow/flow/PromiseStream/move2") {
 	stream.send(Tracker{});
 	Tracker tracker = waitNext(stream.getFuture());
 	Tracker movedTracker = std::move(tracker);
-	ASSERT(tracker.moved);
+	ASSERT(
+	    tracker.moved); // NOLINT(bugprone-use-after-move): this test intentionally checks the moved-from Tracker state.
 	ASSERT(!movedTracker.moved);
 	ASSERT(movedTracker.copied == 0);
 	return Void();
@@ -1722,6 +1725,42 @@ TEST_CASE("/flow/thread/ThreadReturnPromiseStream_Error") {
 	return Void();
 }
 
+TEST_CASE("/fdbrpc/waitValueOrSignal/peerDisconnect") {
+	// Test that waitValueOrSignal detects peer disconnect and returns request_maybe_delivered.
+	// This reproduces the bug where DD would hang forever because waitValueOrSignal didn't
+	// watch peer->disconnect, so dead connections (e.g., from K8s NAT timeouts) were never detected.
+
+	// Construct a minimal Peer. We pass nullptr for TransportData because this test only relies on
+	// peer->disconnect, and PeerHolder only touches outstandingReplies. Note that Peer construction
+	// also updates the global failure monitor status for fakeAddr.
+	NetworkAddress fakeAddr = NetworkAddress::parse("1.2.3.4:1234");
+	state Reference<Peer> peer = makeReference<Peer>(nullptr, fakeAddr);
+
+	// Create a value future that never resolves (simulating a stuck RPC to unreachable storage server)
+	state Promise<Void> neverReply;
+
+	// No failure signal either (simulating failure monitor not yet detecting the failure)
+	state Endpoint ep;
+
+	// Call waitValueOrSignal with the peer
+	state Future<ErrorOr<Void>> result =
+	    waitValueOrSignal(neverReply.getFuture(), Never(), ep, ReplyPromise<Void>(), peer);
+
+	// Result should not be ready yet - the reply hasn't come and disconnect hasn't fired
+	ASSERT(!result.isReady());
+
+	// Simulate peer disconnection (as would happen when connectionReader/connectionKeeper detects failure)
+	peer->disconnect.send(Void());
+
+	// Now waitValueOrSignal should detect the disconnect and return request_maybe_delivered
+	ASSERT(result.isReady());
+	ErrorOr<Void> r = result.get();
+	ASSERT(r.isError());
+	ASSERT(r.getError().code() == error_code_request_maybe_delivered);
+
+	return Void();
+}
+
 TEST_CASE("/flow/IThreadPool/ThreadReturnPromiseStream_DestroyPromise") {
 	noUnseed = true;
 
@@ -1790,3 +1829,91 @@ TEST_CASE("/flow/IThreadPool/ThreadReturnPromiseStream_DestroyPromise") {
 
 	return Void();
 }
+
+TEST_CASE("/fdbrpc/waitValueOrSignal/noPeerFallback") {
+	// Test that waitValueOrSignal still works correctly when no peer is provided (the default).
+	// The broken_promise path should still be handled: when the reply promise breaks,
+	// the endpoint should be marked as not found and value set to Never().
+
+	state Promise<Void> reply;
+
+	// Call waitValueOrSignal without a peer (default behavior)
+	state Future<ErrorOr<Void>> result = waitValueOrSignal(reply.getFuture(), Never(), Endpoint());
+
+	ASSERT(!result.isReady());
+
+	// Break the promise (simulating endpoint failure)
+	reply.sendError(broken_promise());
+
+	// waitValueOrSignal should handle broken_promise by setting value = Never() and looping.
+	// Since signal is Never() and there's no peer disconnect, it should now wait forever.
+	// We verify it doesn't crash and the result is NOT ready (it's stuck in the loop).
+	wait(delay(0.1));
+	ASSERT(!result.isReady());
+
+	return Void();
+}
+
+TEST_CASE("/fdbrpc/waitValueOrSignal/retryOnDisconnect") {
+	// End-to-end test of the retry pattern that loadBalance uses:
+	// 1. First attempt: RPC to peer1, peer1 disconnects → request_maybe_delivered
+	// 2. Second attempt: RPC to peer2, peer2 responds → success
+	// 3. Verify numAttempts > 1
+	//
+	// This reproduces the exact scenario in production: DD sends waitMetrics to a storage
+	// server, the K8s NAT kills the connection at ~3 minutes, and the fix ensures the
+	// request_maybe_delivered error propagates so loadBalance can retry on another server.
+
+	NetworkAddress addr1 = NetworkAddress::parse("1.2.3.4:1234");
+	NetworkAddress addr2 = NetworkAddress::parse("1.2.3.5:1234");
+	state Reference<Peer> peer1 = makeReference<Peer>(nullptr, addr1);
+	state Reference<Peer> peer2 = makeReference<Peer>(nullptr, addr2);
+
+	state int numAttempts = 0;
+
+	// --- Attempt 1: peer1 disconnects mid-request (simulating K8s NAT timeout) ---
+	{
+		state Promise<Void> reply1;
+		state Endpoint ep1;
+
+		state Future<ErrorOr<Void>> result1 =
+		    waitValueOrSignal(reply1.getFuture(), Never(), ep1, ReplyPromise<Void>(), peer1);
+
+		numAttempts++;
+		ASSERT(!result1.isReady());
+
+		// Connection killed by external factor (K8s NAT, network partition, etc.)
+		peer1->disconnect.send(Void());
+
+		// waitValueOrSignal must detect the disconnect and return request_maybe_delivered
+		ASSERT(result1.isReady());
+		ErrorOr<Void> r1 = result1.get();
+		ASSERT(r1.isError());
+		ASSERT(r1.getError().code() == error_code_request_maybe_delivered);
+	}
+
+	// --- Attempt 2: retry to peer2, which responds successfully ---
+	// This is what loadBalance does: on request_maybe_delivered, pick next alternative and retry
+	{
+		state Promise<Void> reply2;
+		state Endpoint ep2;
+
+		state Future<ErrorOr<Void>> result2 =
+		    waitValueOrSignal(reply2.getFuture(), Never(), ep2, ReplyPromise<Void>(), peer2);
+
+		numAttempts++;
+
+		// Second server is healthy and responds
+		reply2.send(Void());
+
+		ASSERT(result2.isReady());
+		ErrorOr<Void> r2 = result2.get();
+		ASSERT(r2.present()); // Success!
+	}
+
+	// The critical assertion: retries happened (numAttempts > 1)
+	ASSERT_GE(numAttempts, 2);
+	TraceEvent("WaitValueOrSignalRetryTest").detail("NumAttempts", numAttempts);
+
+	return Void();
+}
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.cpp
similarity index 87%
rename from fdbrpc/FlowTransport.actor.cpp
rename to fdbrpc/FlowTransport.cpp
index 72be7f726e6..42c35f3554e 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.cpp
@@ -1,5 +1,5 @@
 /*
- * FlowTransport.actor.cpp
+ * FlowTransport.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -33,29 +33,26 @@
 #include <memcheck.h>
 #endif
 
-#include <boost/unordered_map.hpp>
-
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/HealthMonitor.h"
-#include "fdbrpc/JsonWebKeySet.h"
-#include "fdbrpc/genericactors.actor.h"
+#include "JsonWebKeySet.h"
+#include "fdbrpc/genericactors.h"
 #include "fdbrpc/IPAllowList.h"
 #include "fdbrpc/simulator.h"
 #include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
 #include "flow/Error.h"
 #include "flow/flow.h"
 #include "flow/Net2Packet.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/ObjectSerializer.h"
 #include "flow/Platform.h"
 #include "flow/ProtocolVersion.h"
-#include "flow/UnitTest.h"
-#include "flow/WatchFile.actor.h"
+#include "flow/WatchFile.h"
 #include "flow/IConnection.h"
 #define XXH_INLINE_ALL
 #include "flow/xxhash.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 void removeCachedDNS(const std::string& host, const std::string& service) {
 	INetworkConnections::net()->removeCachedDNS(host, service);
@@ -230,7 +227,7 @@ void EndpointMap::remove(Endpoint::Token const& token, NetworkMessageReceiver* r
 }
 
 struct EndpointNotFoundReceiver final : NetworkMessageReceiver {
-	EndpointNotFoundReceiver(EndpointMap& endpoints) {
+	explicit EndpointNotFoundReceiver(EndpointMap& endpoints) {
 		endpoints.insertWellKnown(
 		    this, Endpoint::wellKnownToken(WLTOKEN_ENDPOINT_NOT_FOUND), TaskPriority::DefaultEndpoint);
 	}
@@ -256,7 +253,7 @@ struct PingRequest {
 };
 
 struct PingReceiver final : NetworkMessageReceiver {
-	PingReceiver(EndpointMap& endpoints) {
+	explicit PingReceiver(EndpointMap& endpoints) {
 		endpoints.insertWellKnown(this, Endpoint::wellKnownToken(WLTOKEN_PING_PACKET), TaskPriority::ReadSocket);
 	}
 	void receive(ArenaObjectReader& reader) override {
@@ -271,7 +268,7 @@ struct PingReceiver final : NetworkMessageReceiver {
 };
 
 struct UnauthorizedEndpointReceiver final : NetworkMessageReceiver {
-	UnauthorizedEndpointReceiver(EndpointMap& endpoints) {
+	explicit UnauthorizedEndpointReceiver(EndpointMap& endpoints) {
 		endpoints.insertWellKnown(
 		    this, Endpoint::wellKnownToken(WLTOKEN_UNAUTHORIZED_ENDPOINT), TaskPriority::ReadSocket);
 	}
@@ -294,7 +291,7 @@ struct UnauthorizedEndpointReceiver final : NetworkMessageReceiver {
 class NetworkAddressCachedString {
 public:
 	NetworkAddressCachedString() { setAddressList(NetworkAddressList()); }
-	NetworkAddressCachedString(NetworkAddressList const& list) { setAddressList(list); }
+	explicit NetworkAddressCachedString(NetworkAddressList const& list) { setAddressList(list); }
 	NetworkAddressList const& getAddressList() const { return addressList; }
 	void setAddressList(NetworkAddressList const& list) {
 		cachedStr = Standalone<StringRef>(StringRef(list.address.toString()));
@@ -387,9 +384,9 @@ struct ConnectionLogWriter : IThreadPoolReceiver {
 	std::string fileName;
 	std::fstream file;
 
-	ConnectionLogWriter(const std::string baseDir) : baseDir(baseDir) {}
+	explicit ConnectionLogWriter(const std::string baseDir) : baseDir(baseDir) {}
 
-	virtual ~ConnectionLogWriter() {
+	~ConnectionLogWriter() override {
 		if (file.is_open())
 			file.close();
 	}
@@ -400,12 +397,12 @@ struct ConnectionLogWriter : IThreadPoolReceiver {
 		AppendAction(std::string localAddr, std::deque<TransportData::ConnectionHistoryEntry>&& entries)
 		  : localAddr(localAddr), entries(std::move(entries)) {}
 
-		double getTimeEstimate() const { return 2; }
+		double getTimeEstimate() const override { return 2; }
 	};
 
 	std::string newFileName() const { return baseDir + "fdb-connection-log-" + time_str() + ".csv"; }
 
-	void init() { fileName = newFileName(); }
+	void init() override { fileName = newFileName(); }
 
 	std::string time_str() const { return std::to_string(now()); }
 
@@ -443,23 +440,23 @@ struct ConnectionLogWriter : IThreadPoolReceiver {
 	}
 };
 
-ACTOR Future<Void> connectionHistoryLogger(TransportData* self) {
+Future<Void> connectionHistoryLogger(TransportData* self) {
 	if (!FLOW_KNOBS->LOG_CONNECTION_ATTEMPTS_ENABLED) {
-		return Void();
+		co_return;
 	}
 
-	state Future<Void> next = Void();
+	Future<Void> next = Void();
 
 	// One thread ensures async serialized execution on the log file.
 	if (g_network->isSimulated()) {
-		self->connectionLogWriterThread = Reference<IThreadPool>(new DummyThreadPool());
+		self->connectionLogWriterThread = makeReference<DummyThreadPool>();
 	} else {
 		self->connectionLogWriterThread = createGenericThreadPool();
 	}
 
 	self->connectionLogWriterThread->addThread(new ConnectionLogWriter(FLOW_KNOBS->CONNECTION_LOG_DIRECTORY));
-	loop {
-		wait(next);
+	while (true) {
+		co_await next;
 		next = delay(FLOW_KNOBS->LOG_CONNECTION_INTERVAL_SECS);
 		if (self->connectionHistory.size() == 0) {
 			continue;
@@ -472,9 +469,9 @@ ACTOR Future<Void> connectionHistoryLogger(TransportData* self) {
 	}
 }
 
-ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
-	state NetworkAddress lastAddress = NetworkAddress();
-	loop {
+Future<Void> pingLatencyLogger(TransportData* self) {
+	NetworkAddress lastAddress = NetworkAddress();
+	while (true) {
 		if (self->orderedAddresses.size()) {
 			auto it = self->orderedAddresses.upper_bound(lastAddress);
 			if (it == self->orderedAddresses.end()) {
@@ -525,12 +522,12 @@ ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
 				peer->lastLoggedBytesReceived = peer->bytesReceived;
 				peer->lastLoggedBytesSent = peer->bytesSent;
 				peer->timeoutCount = 0;
-				wait(delay(FLOW_KNOBS->PING_LOGGING_INTERVAL));
+				co_await delay(FLOW_KNOBS->PING_LOGGING_INTERVAL);
 			} else if (it == self->orderedAddresses.begin()) {
-				wait(delay(FLOW_KNOBS->PING_LOGGING_INTERVAL));
+				co_await delay(FLOW_KNOBS->PING_LOGGING_INTERVAL);
 			}
 		} else {
-			wait(delay(FLOW_KNOBS->PING_LOGGING_INTERVAL));
+			co_await delay(FLOW_KNOBS->PING_LOGGING_INTERVAL);
 		}
 	}
 }
@@ -613,10 +610,10 @@ struct ConnectPacket {
 
 #pragma pack(pop)
 
-ACTOR static Future<Void> connectionReader(TransportData* transport,
-                                           Reference<IConnection> conn,
-                                           Reference<struct Peer> peer,
-                                           Promise<Reference<struct Peer>> onConnected);
+static Future<Void> connectionReader(TransportData* transport,
+                                     Reference<IConnection> conn,
+                                     Reference<struct Peer> peer,
+                                     Promise<Reference<struct Peer>> onConnected);
 
 static void sendLocal(TransportData* self, ISerializeSource const& what, const Endpoint& destination);
 static ReliablePacket* sendPacket(TransportData* self,
@@ -625,11 +622,11 @@ static ReliablePacket* sendPacket(TransportData* self,
                                   const Endpoint& destination,
                                   bool reliable);
 
-ACTOR Future<Void> connectionMonitor(Reference<Peer> peer) {
-	state Endpoint remotePingEndpoint({ peer->destination }, Endpoint::wellKnownToken(WLTOKEN_PING_PACKET));
+Future<Void> connectionMonitor(Reference<Peer> peer) {
+	Endpoint remotePingEndpoint({ peer->destination }, Endpoint::wellKnownToken(WLTOKEN_PING_PACKET));
 	// set this to not immediately close the connection as idle if the peer already existed
 	peer->lastDataPacketSentTime = now();
-	loop {
+	while (true) {
 		if (!FlowTransport::isClient() && !peer->destination.isPublic() && peer->compatible) {
 			// Don't send ping messages to clients unless necessary. Instead monitor incoming client pings.
 			// We ignore this block for incompatible clients because pings from server would trigger the
@@ -651,10 +648,10 @@ ACTOR Future<Void> connectionMonitor(Reference<Peer> peer) {
 			// the version reported in ConnectPacket.
 			//
 			// TLDR; We can probably close the connection now as we don't have any older clusters running either.
-			state double lastRefreshed = now();
-			state int64_t lastBytesReceived = peer->bytesReceived;
-			loop {
-				wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME, TaskPriority::ReadSocket));
+			double lastRefreshed = now();
+			int64_t lastBytesReceived = peer->bytesReceived;
+			while (true) {
+				co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME, TaskPriority::ReadSocket);
 				if (lastBytesReceived < peer->bytesReceived) {
 					lastRefreshed = now();
 					lastBytesReceived = peer->bytesReceived;
@@ -673,7 +670,7 @@ ACTOR Future<Void> connectionMonitor(Reference<Peer> peer) {
 		// FIXME: Why should actors are about stacks?  Is there a better way to deal with this than having to
 		// anticipate and insert non-functional (from the point of what this actor is supposed to accomplish)
 		// delays like this just to avoid distributed (across multiple actors) logic bugs and memory leaks?
-		wait(delay(0, TaskPriority::ReadSocket));
+		co_await delay(0, TaskPriority::ReadSocket);
 
 		if (peer->reliable.empty() && peer->unsent.empty() && peer->outstandingReplies == 0) {
 			if (peer->peerReferences == 0 &&
@@ -688,57 +685,58 @@ ACTOR Future<Void> connectionMonitor(Reference<Peer> peer) {
 			}
 		}
 
-		wait(delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME, TaskPriority::ReadSocket));
+		co_await delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME, TaskPriority::ReadSocket);
 
 		// TODO: Stop monitoring and close the connection with no onDisconnect requests outstanding
-		state PingRequest pingRequest;
+		PingRequest pingRequest;
 		FlowTransport::transport().sendUnreliable(SerializeSource<PingRequest>(pingRequest), remotePingEndpoint, true);
-		state int64_t startingBytes = peer->bytesReceived;
-		state int timeouts = 0;
-		state double startTime = now();
-		loop {
-			choose {
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
-					peer->timeoutCount++;
-					if (startingBytes == peer->bytesReceived) {
-						if (peer->destination.isPublic()) {
-							peer->pingLatencies.addSample(now() - startTime);
-						}
-						TraceEvent("ConnectionTimeout").suppressFor(1.0).detail("WithAddr", peer->destination);
-						throw connection_failed();
-					}
-					if (timeouts > 1) {
-						TraceEvent(SevWarnAlways, "ConnectionSlowPing")
-						    .suppressFor(1.0)
-						    .detail("WithAddr", peer->destination)
-						    .detail("Timeouts", timeouts);
-					}
-					startingBytes = peer->bytesReceived;
-					timeouts++;
-				}
-				when(wait(pingRequest.reply.getFuture())) {
+		int64_t startingBytes = peer->bytesReceived;
+		int timeouts = 0;
+		double startTime = now();
+		while (true) {
+			auto res = co_await race(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT),
+			                         pingRequest.reply.getFuture(),
+			                         peer->resetPing.onTrigger());
+			if (res.index() == 0) {
+				peer->timeoutCount++;
+				if (startingBytes == peer->bytesReceived) {
 					if (peer->destination.isPublic()) {
 						peer->pingLatencies.addSample(now() - startTime);
 					}
-					break;
+					TraceEvent("ConnectionTimeout").suppressFor(1.0).detail("WithAddr", peer->destination);
+					throw connection_failed();
 				}
-				when(wait(peer->resetPing.onTrigger())) {
-					break;
+				if (timeouts > 1) {
+					TraceEvent(SevWarnAlways, "ConnectionSlowPing")
+					    .suppressFor(1.0)
+					    .detail("WithAddr", peer->destination)
+					    .detail("Timeouts", timeouts);
+				}
+				startingBytes = peer->bytesReceived;
+				timeouts++;
+			} else if (res.index() == 1) {
+				if (peer->destination.isPublic()) {
+					peer->pingLatencies.addSample(now() - startTime);
 				}
+				break;
+			} else if (res.index() == 2) {
+				break;
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
 }
 
-ACTOR Future<Void> connectionWriter(Reference<Peer> self, Reference<IConnection> conn) {
-	state double lastWriteTime = now();
-	loop {
-		wait(delayJittered(
+Future<Void> connectionWriter(Reference<Peer> self, Reference<IConnection> conn) {
+	double lastWriteTime = now();
+	while (true) {
+		co_await delayJittered(
 		    std::max<double>(FLOW_KNOBS->MIN_COALESCE_DELAY, FLOW_KNOBS->MAX_COALESCE_DELAY - (now() - lastWriteTime)),
-		    TaskPriority::WriteSocket));
+		    TaskPriority::WriteSocket);
 
 		// Send until there is nothing left to send
-		loop {
+		while (true) {
 			lastWriteTime = now();
 
 			int sent = conn->write(self->unsent.getUnsent(), /* limit= */ FLOW_KNOBS->MAX_PACKET_SEND_BYTES);
@@ -754,22 +752,22 @@ ACTOR Future<Void> connectionWriter(Reference<Peer> self, Reference<IConnection>
 
 			CODE_PROBE(
 			    true, "We didn't write everything, so apparently the write buffer is full.  Wait for it to be nonfull");
-			wait(conn->onWritable());
-			wait(yield(TaskPriority::WriteSocket));
+			co_await conn->onWritable();
+			co_await yield(TaskPriority::WriteSocket);
 		}
 
 		// Wait until there is something to send
 		while (self->unsent.empty())
-			wait(self->dataToSend.onTrigger());
+			co_await self->dataToSend.onTrigger();
 	}
 }
 
-ACTOR Future<Void> delayedHealthUpdate(NetworkAddress address, bool* tooManyConnectionsClosed) {
-	state double start = now();
-	loop {
+Future<Void> delayedHealthUpdate(NetworkAddress address, bool* tooManyConnectionsClosed) {
+	double start = now();
+	while (true) {
 		if (FLOW_KNOBS->HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS &&
 		    FlowTransport::transport().healthMonitor()->tooManyConnectionsClosed(address) && address.isPublic()) {
-			wait(delayJittered(FLOW_KNOBS->MAX_RECONNECTION_TIME * 2.0));
+			co_await delayJittered(FLOW_KNOBS->MAX_RECONNECTION_TIME * 2.0);
 		} else {
 			if (*tooManyConnectionsClosed) {
 				TraceEvent("TooManyConnectionsClosedMarkAvailable")
@@ -783,24 +781,23 @@ ACTOR Future<Void> delayedHealthUpdate(NetworkAddress address, bool* tooManyConn
 			break;
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
-                                    Reference<IConnection> conn = Reference<IConnection>(),
-                                    Future<Void> reader = Void()) {
+Future<Void> connectionKeeper(Reference<Peer> self,
+                              Reference<IConnection> conn = Reference<IConnection>(),
+                              Future<Void> reader = Void()) {
 	TraceEvent(SevDebug, "ConnectionKeeper", conn ? conn->getDebugID() : UID())
 	    .detail("PeerAddr", self->destination)
 	    .detail("PeerAddress", self->destination)
 	    .detail("ConnSet", (bool)conn);
 	ASSERT_WE_THINK(FlowTransport::transport().getLocalAddress() != self->destination);
 
-	state Future<Void> delayedHealthUpdateF;
-	state Optional<double> firstConnFailedTime = Optional<double>();
-	state int retryConnect = false;
-	state bool tooManyConnectionsClosed = false;
+	Future<Void> delayedHealthUpdateF;
+	Optional<double> firstConnFailedTime = Optional<double>();
+	int retryConnect = false;
+	bool tooManyConnectionsClosed = false;
 
-	loop {
+	while (true) {
 		try {
 			delayedHealthUpdateF = Future<Void>();
 
@@ -816,19 +813,20 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 						                    : delay(FLOW_KNOBS->SERVER_REQUEST_INTERVAL);
 					}
 
-					choose {
-						when(wait(self->dataToSend.onTrigger())) {}
-						when(wait(retryConnectF)) {
-							break;
-						}
+					auto res = co_await race(self->dataToSend.onTrigger(), retryConnectF);
+					if (res.index() == 1) {
+						break;
+					} else if (res.index() != 0) {
+						UNREACHABLE();
 					}
 				}
 
 				ASSERT(self->destination.isPublic());
 				self->outgoingConnectionIdle = false;
-				wait(delayJittered(std::max(0.0,
-				                            self->lastConnectTime + self->reconnectionDelay -
-				                                now()))); // Don't connect() to the same peer more than once per 2 sec
+				co_await delayJittered(
+				    std::max(0.0,
+				             self->lastConnectTime + self->reconnectionDelay -
+				                 now())); // Don't connect() to the same peer more than once per 2 sec
 				self->lastConnectTime = now();
 
 				TraceEvent("ConnectingTo", conn ? conn->getDebugID() : UID())
@@ -841,46 +839,42 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 				                                                                                        : "FAILED");
 				++self->connectOutgoingCount;
 				try {
-					choose {
-						when(Reference<IConnection> _conn =
-						         wait(INetworkConnections::net()->connect(self->destination))) {
-							conn = _conn;
-							static SimpleCounter<int64_t>* countOutgoingConnectionCreated =
-							    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/OutgoingConnectionCreated");
-							countOutgoingConnectionCreated->increment(1);
-							wait(conn->connectHandshake());
-							static SimpleCounter<int64_t>* countOutgoingConnectionHandshakeComplete =
-							    SimpleCounter<int64_t>::makeCounter(
-							        "/Transport/TLS/OutgoingConnectionHandshakeComplete");
-							countOutgoingConnectionHandshakeComplete->increment(1);
-							self->connectLatencies.addSample(now() - self->lastConnectTime);
-							if (FlowTransport::isClient()) {
-								IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(false));
-							}
-							if (self->unsent.empty()) {
-								delayedHealthUpdateF =
-								    delayedHealthUpdate(self->destination, &tooManyConnectionsClosed);
-								choose {
-									when(wait(delayedHealthUpdateF)) {
-										conn->close();
-										conn = Reference<IConnection>();
-										retryConnect = false;
-										continue;
-									}
-									when(wait(self->dataToSend.onTrigger())) {}
-								}
-							}
-
-							TraceEvent("ConnectionExchangingConnectPacket", conn->getDebugID())
-							    .suppressFor(1.0)
-							    .detail("PeerAddr", self->destination)
-							    .detail("PeerAddress", self->destination);
-							self->prependConnectPacket();
-							reader = connectionReader(self->transport, conn, self, Promise<Reference<Peer>>());
+					auto maybeConn = co_await timeout(INetworkConnections::net()->connect(self->destination),
+					                                  FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
+					if (maybeConn.present()) {
+						conn = maybeConn.get();
+						static SimpleCounter<int64_t>* countOutgoingConnectionCreated =
+						    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/OutgoingConnectionCreated");
+						countOutgoingConnectionCreated->increment(1);
+						co_await conn->connectHandshake();
+						static SimpleCounter<int64_t>* countOutgoingConnectionHandshakeComplete =
+						    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/OutgoingConnectionHandshakeComplete");
+						countOutgoingConnectionHandshakeComplete->increment(1);
+						self->connectLatencies.addSample(now() - self->lastConnectTime);
+						if (FlowTransport::isClient()) {
+							IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(false));
 						}
-						when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
-							throw connection_failed();
+						if (self->unsent.empty()) {
+							delayedHealthUpdateF = delayedHealthUpdate(self->destination, &tooManyConnectionsClosed);
+							auto healthRes = co_await race(delayedHealthUpdateF, self->dataToSend.onTrigger());
+							if (healthRes.index() == 0) {
+								conn->close();
+								conn = Reference<IConnection>();
+								retryConnect = false;
+								continue;
+							} else if (healthRes.index() != 1) {
+								UNREACHABLE();
+							}
 						}
+
+						TraceEvent("ConnectionExchangingConnectPacket", conn->getDebugID())
+						    .suppressFor(1.0)
+						    .detail("PeerAddr", self->destination)
+						    .detail("PeerAddress", self->destination);
+						self->prependConnectPacket();
+						reader = connectionReader(self->transport, conn, self, Promise<Reference<Peer>>());
+					} else {
+						throw connection_failed();
 					}
 				} catch (Error& e) {
 					++self->connectFailedCount;
@@ -905,8 +899,8 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 				if (!delayedHealthUpdateF.isValid())
 					delayedHealthUpdateF = delayedHealthUpdate(self->destination, &tooManyConnectionsClosed);
 				self->connected = true;
-				wait(connectionWriter(self, conn) || reader || connectionMonitor(self) ||
-				     self->resetConnection.onTrigger());
+				co_await (connectionWriter(self, conn) || reader || connectionMonitor(self) ||
+				          self->resetConnection.onTrigger());
 				TraceEvent("ConnectionReset", conn ? conn->getDebugID() : UID())
 				    .suppressFor(1.0)
 				    .detail("PeerAddr", self->destination)
@@ -1043,7 +1037,7 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 				self->connect.cancel();
 				self->transport->peers.erase(self->destination);
 				self->transport->orderedAddresses.erase(self->destination);
-				return Void();
+				co_return;
 			}
 		}
 	}
@@ -1055,8 +1049,8 @@ Peer::Peer(TransportData* transport, NetworkAddress const& destination)
     bytesReceived(0), bytesSent(0), lastDataPacketSentTime(now()), outstandingReplies(0),
     pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SKETCH_ACCURACY : 0.1), lastLoggedTime(0.0),
     lastLoggedBytesReceived(0), lastLoggedBytesSent(0), timeoutCount(0),
-    protocolVersion(Reference<AsyncVar<Optional<ProtocolVersion>>>(new AsyncVar<Optional<ProtocolVersion>>())),
-    connectOutgoingCount(0), connectIncomingCount(0), connectFailedCount(0),
+    protocolVersion(makeReference<AsyncVar<Optional<ProtocolVersion>>>()), connectOutgoingCount(0),
+    connectIncomingCount(0), connectFailedCount(0),
     connectLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SKETCH_ACCURACY : 0.1) {
 	IFailureMonitor::failureMonitor().setStatus(destination, FailureStatus(false));
 }
@@ -1157,8 +1151,8 @@ void Peer::onIncomingConnection(Reference<Peer> self, Reference<IConnection> con
 }
 
 TransportData::~TransportData() {
-	for (auto& p : peers) {
-		p.second->connect.cancel();
+	for (auto& [_peerAddress, peer] : peers) {
+		peer->connect.cancel();
 	}
 }
 
@@ -1177,21 +1171,22 @@ static bool checkCompatible(const PeerCompatibilityPolicy& policy, ProtocolVersi
 // This actor looks up the task associated with an endpoint
 // and sends the message to it. The actual deserialization will
 // be done by that task (see NetworkMessageReceiver).
-ACTOR static void deliver(TransportData* self,
-                          Endpoint destination,
-                          TaskPriority priority,
-                          ArenaReader reader,
-                          NetworkAddress peerAddress,
-                          bool isTrustedPeer,
-                          InReadSocket inReadSocket,
-                          Future<Void> disconnect) {
+static Future<Void> deliver(Uncancellable,
+                            TransportData* self,
+                            Endpoint destination,
+                            TaskPriority priority,
+                            ArenaReader reader,
+                            NetworkAddress peerAddress,
+                            bool isTrustedPeer,
+                            InReadSocket inReadSocket,
+                            Future<Void> disconnect) {
 	// We want to run the task at the right priority. If the priority is higher than the current priority (which is
 	// ReadSocket) we can just upgrade. Otherwise we'll context switch so that we don't block other tasks that might run
 	// with a higher priority. ReplyPromiseStream needs to guarantee that messages are received in the order they were
 	// sent, so we are using orderedDelay.
 	// NOTE: don't skip delay(0) when it's local deliver since it could cause out of order object deconstruction.
 	if (priority < TaskPriority::ReadSocket || !inReadSocket) {
-		wait(orderedDelay(0, priority));
+		co_await orderedDelay(0, priority);
 	} else {
 		g_network->setCurrentTask(priority);
 	}
@@ -1199,7 +1194,7 @@ ACTOR static void deliver(TransportData* self,
 	auto receiver = self->endpoints.get(destination.token);
 	if (receiver && (isTrustedPeer || receiver->isPublic())) {
 		if (!checkCompatible(receiver->peerCompatibilityPolicy(), reader.protocolVersion())) {
-			return;
+			co_return;
 		}
 		try {
 			ASSERT(g_currentDeliveryPeerAddress == NetworkAddressList());
@@ -1277,7 +1272,7 @@ static void scanPackets(TransportData* transport,
 	uint8_t* p = unprocessed_begin;
 
 	const bool checksumEnabled = !peerAddress.isTLS();
-	loop {
+	while (true) {
 		uint32_t packetLen;
 		XXH64_hash_t packetChecksum;
 
@@ -1397,7 +1392,8 @@ static void scanPackets(TransportData* transport,
 		// we have many messages to UnknownEndpoint we want to optimize earlier. As deliver is an actor it
 		// will allocate some state on the heap and this prevents it from doing that.
 		if (priority != TaskPriority::UnknownEndpoint || (token.first() & TOKEN_STREAM_FLAG) != 0) {
-			deliver(transport,
+			deliver(Uncancellable(),
+			        transport,
 			        Endpoint({ peerAddress }, token),
 			        priority,
 			        std::move(reader),
@@ -1435,30 +1431,30 @@ static int getNewBufferSize(const uint8_t* begin,
 
 // This actor exists whenever there is an open or opening connection, whether incoming or outgoing
 // For incoming connections conn is set and peer is initially nullptr; for outgoing connections it is the reverse
-ACTOR static Future<Void> connectionReader(TransportData* transport,
-                                           Reference<IConnection> conn,
-                                           Reference<Peer> peer,
-                                           Promise<Reference<Peer>> onConnected) {
-
-	state Arena arena;
-	state uint8_t* unprocessed_begin = nullptr;
-	state uint8_t* unprocessed_end = nullptr;
-	state uint8_t* buffer_end = nullptr;
-	state bool expectConnectPacket = true;
-	state bool compatible = false;
-	state bool incompatiblePeerCounted = false;
-	state NetworkAddress peerAddress;
-	state ProtocolVersion peerProtocolVersion;
-	state bool trusted = transport->allowList(conn->getPeerAddress().ip) && conn->hasTrustedPeer();
+static Future<Void> connectionReader(TransportData* transport,
+                                     Reference<IConnection> conn,
+                                     Reference<Peer> peer,
+                                     Promise<Reference<Peer>> onConnected) {
+
+	Arena arena;
+	uint8_t* unprocessed_begin = nullptr;
+	uint8_t* unprocessed_end = nullptr;
+	uint8_t* buffer_end = nullptr;
+	bool expectConnectPacket = true;
+	bool compatible = false;
+	bool incompatiblePeerCounted = false;
+	NetworkAddress peerAddress;
+	ProtocolVersion peerProtocolVersion;
+	bool trusted = transport->allowList(conn->getPeerAddress().ip) && conn->hasTrustedPeer();
 	peerAddress = conn->getPeerAddress();
 
 	if (!peer) {
 		ASSERT(!peerAddress.isPublic());
 	}
 	try {
-		loop {
-			loop {
-				state int readAllBytes = buffer_end - unprocessed_end;
+		while (true) {
+			while (true) {
+				int readAllBytes = buffer_end - unprocessed_end;
 				if (readAllBytes < FLOW_KNOBS->MIN_PACKET_BUFFER_FREE_BYTES) {
 					Arena newArena;
 					const int unproc_len = unprocessed_end - unprocessed_begin;
@@ -1475,15 +1471,15 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 					readAllBytes = buffer_end - unprocessed_end;
 				}
 
-				state int totalReadBytes = 0;
+				int totalReadBytes = 0;
 				while (true) {
 					const int len = std::min<int>(buffer_end - unprocessed_end, FLOW_KNOBS->MAX_PACKET_SEND_BYTES);
 					if (len == 0)
 						break;
-					state int readBytes = conn->read(unprocessed_end, unprocessed_end + len);
+					int readBytes = conn->read(unprocessed_end, unprocessed_end + len);
 					if (readBytes == 0)
 						break;
-					wait(yield(TaskPriority::ReadSocket));
+					co_await yield(TaskPriority::ReadSocket);
 					totalReadBytes += readBytes;
 					unprocessed_end += readBytes;
 				}
@@ -1492,7 +1488,7 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 				}
 				if (totalReadBytes == 0)
 					break;
-				state bool readWillBlock = totalReadBytes != readAllBytes;
+				bool readWillBlock = totalReadBytes != readAllBytes;
 
 				if (expectConnectPacket && unprocessed_end - unprocessed_begin >= CONNECT_PACKET_V0_SIZE) {
 					// FIXME: this block might go in a separate function whose purpose, obviously, would
@@ -1603,7 +1599,7 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 								incompatiblePeerCounted = true;
 							}
 							onConnected.send(peer);
-							wait(delay(0)); // Check for cancellation
+							co_await delay(0); // Check for cancellation
 						}
 						peer->protocolVersion->set(peerProtocolVersion);
 					}
@@ -1629,12 +1625,12 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 				if (readWillBlock)
 					break;
 
-				wait(yield(TaskPriority::ReadSocket));
+				co_await yield(TaskPriority::ReadSocket);
 			}
 
-			wait(conn->onReadable());
-			wait(delay(0, TaskPriority::ReadSocket)); // We don't want to call conn->read directly from the reactor - we
-			                                          // could get stuck in the reactor reading 1 packet at a time
+			co_await conn->onReadable();
+			co_await delay(0, TaskPriority::ReadSocket); // We don't want to call conn->read directly from the reactor -
+			                                             // we could get stuck in the reactor reading 1 packet at a time
 		}
 	} catch (Error& e) {
 		if (incompatiblePeerCounted) {
@@ -1645,36 +1641,37 @@ ACTOR static Future<Void> connectionReader(TransportData* transport,
 	}
 }
 
-ACTOR static Future<Void> connectionIncoming(TransportData* self, Reference<IConnection> conn) {
-	state TransportData::ConnectionHistoryEntry entry;
+static Future<Void> connectionIncoming(TransportData* self, Reference<IConnection> conn) {
+	TransportData::ConnectionHistoryEntry entry;
 	entry.time = now();
 	entry.addr = conn->getPeerAddress();
 	try {
-		wait(conn->acceptHandshake());
+		co_await conn->acceptHandshake();
 		static SimpleCounter<int64_t>* countIncomingConnectionHandshakeAccepted =
 		    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/IncomingConnectionHandshakeAccepted");
 		countIncomingConnectionHandshakeAccepted->increment(1);
-		state Promise<Reference<Peer>> onConnected;
-		state Future<Void> reader = connectionReader(self, conn, Reference<Peer>(), onConnected);
+		Promise<Reference<Peer>> onConnected;
+		Future<Void> reader = connectionReader(self, conn, Reference<Peer>(), onConnected);
 		if (FLOW_KNOBS->LOG_CONNECTION_ATTEMPTS_ENABLED) {
 			entry.failed = false;
 			self->connectionHistory.push_back(entry);
 		}
-		choose {
-			when(wait(reader)) {
-				ASSERT(false);
-				return Void();
-			}
-			when(Reference<Peer> p = wait(onConnected.getFuture())) {
-				p->onIncomingConnection(p, conn, reader);
-			}
-			when(wait(delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
-				CODE_PROBE(true, "Incoming connection timed out");
-				static SimpleCounter<int64_t>* countIncomingConnectionTimedout =
-				    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/IncomingConnectionTimedout");
-				countIncomingConnectionTimedout->increment(1);
-				throw timed_out();
-			}
+		auto res =
+		    co_await race(reader, onConnected.getFuture(), delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
+		if (res.index() == 0) {
+			ASSERT(false);
+			co_return;
+		} else if (res.index() == 1) {
+			Reference<Peer> p = std::get<1>(std::move(res));
+			p->onIncomingConnection(p, conn, reader);
+		} else if (res.index() == 2) {
+			CODE_PROBE(true, "Incoming connection timed out");
+			static SimpleCounter<int64_t>* countIncomingConnectionTimedout =
+			    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/IncomingConnectionTimedout");
+			countIncomingConnectionTimedout->increment(1);
+			throw timed_out();
+		} else {
+			UNREACHABLE();
 		}
 		static SimpleCounter<int64_t>* countIncomingConnectionConnected =
 		    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/IncomingConnectionConnected");
@@ -1695,24 +1692,15 @@ ACTOR static Future<Void> connectionIncoming(TransportData* self, Reference<ICon
 		}
 		conn->close();
 	}
-
-	return Void();
 }
 
-ACTOR static Future<Void> listen(TransportData* self, NetworkAddress listenAddr) {
-	state ActorCollectionNoErrors
+static Future<Void> listenImpl(TransportData* self, NetworkAddress listenAddr, Reference<IListener> listener) {
+	ActorCollectionNoErrors
 	    incoming; // Actors monitoring incoming connections that haven't yet been associated with a peer
-	state Reference<IListener> listener = INetworkConnections::net()->listen(listenAddr);
-	if (!g_network->isSimulated() && self->localAddresses.getAddressList().address.port == 0) {
-		TraceEvent(SevInfo, "UpdatingListenAddress")
-		    .detail("AssignedListenAddress", listener->getListenAddress().toString());
-		self->localAddresses.setNetworkAddress(listener->getListenAddress());
-		setTraceLocalAddress(listener->getListenAddress());
-	}
-	state uint64_t connectionCount = 0;
+	uint64_t connectionCount = 0;
 	try {
-		loop {
-			Reference<IConnection> conn = wait(listener->accept());
+		while (true) {
+			Reference<IConnection> conn = co_await listener->accept();
 			static SimpleCounter<int64_t>* countIncomingConnectionCreated =
 			    SimpleCounter<int64_t>::makeCounter("/Transport/TLS/IncomingConnectionCreated");
 			countIncomingConnectionCreated->increment(1);
@@ -1725,7 +1713,7 @@ ACTOR static Future<Void> listen(TransportData* self, NetworkAddress listenAddr)
 			}
 			connectionCount++;
 			if (connectionCount % (FLOW_KNOBS->ACCEPT_BATCH_SIZE) == 0) {
-				wait(delay(0, TaskPriority::AcceptSocket));
+				co_await delay(0, TaskPriority::AcceptSocket);
 			}
 		}
 	} catch (Error& e) {
@@ -1734,6 +1722,17 @@ ACTOR static Future<Void> listen(TransportData* self, NetworkAddress listenAddr)
 	}
 }
 
+static Future<Void> listen(TransportData* self, NetworkAddress listenAddr) {
+	Reference<IListener> listener = INetworkConnections::net()->listen(listenAddr);
+	if (!g_network->isSimulated() && self->localAddresses.getAddressList().address.port == 0) {
+		TraceEvent(SevInfo, "UpdatingListenAddress")
+		    .detail("AssignedListenAddress", listener->getListenAddress().toString());
+		self->localAddresses.setNetworkAddress(listener->getListenAddress());
+		setTraceLocalAddress(listener->getListenAddress());
+	}
+	return listenImpl(self, listenAddr, listener);
+}
+
 Reference<Peer> TransportData::getPeer(NetworkAddress const& address) {
 	auto peer = peers.find(address);
 	if (peer != peers.end()) {
@@ -1785,9 +1784,9 @@ void TransportData::applyPublicKeySet(StringRef jwkSetString) {
 	}
 }
 
-ACTOR static Future<Void> multiVersionCleanupWorker(TransportData* self) {
-	loop {
-		wait(delay(FLOW_KNOBS->CONNECTION_CLEANUP_DELAY));
+static Future<Void> multiVersionCleanupWorker(TransportData* self) {
+	while (true) {
+		co_await delay(FLOW_KNOBS->CONNECTION_CLEANUP_DELAY);
 		bool foundIncompatible = false;
 		for (auto it = self->incompatiblePeers.begin(); it != self->incompatiblePeers.end();) {
 			if (self->multiVersionConnections.count(it->second.first)) {
@@ -1818,8 +1817,8 @@ FlowTransport::FlowTransport(uint64_t transportId, int maxWellKnownEndpoints, IP
   : self(new TransportData(transportId, maxWellKnownEndpoints, allowList)) {
 	self->multiVersionCleanup = multiVersionCleanupWorker(self);
 	if (g_network->isSimulated()) {
-		for (auto const& p : g_simulator->authKeys) {
-			self->publicKeys.emplace(p.first, p.second.toPublic());
+		for (const auto& [keyName, key] : g_simulator->authKeys) {
+			self->publicKeys.emplace(keyName, key.toPublic());
 		}
 	}
 }
@@ -1961,7 +1960,8 @@ static void sendLocal(TransportData* self, ISerializeSource const& what, const E
 	ASSERT(copy.size() > 0);
 	TaskPriority priority = self->endpoints.getPriority(destination.token);
 	if (priority != TaskPriority::UnknownEndpoint || (destination.token.first() & TOKEN_STREAM_FLAG) != 0) {
-		deliver(self,
+		deliver(Uncancellable(),
+		        self,
 		        destination,
 		        priority,
 		        ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion())),
@@ -2229,33 +2229,33 @@ void FlowTransport::loadPublicKeyFile(const std::string& filePath) {
 	}
 }
 
-ACTOR static Future<Void> watchPublicKeyJwksFile(std::string filePath, TransportData* self) {
-	state AsyncTrigger fileChanged;
-	state Future<Void> fileWatch;
-	state unsigned errorCount = 0; // error since watch start or last successful refresh
+static Future<Void> watchPublicKeyJwksFile(std::string filePath, TransportData* self) {
+	AsyncTrigger fileChanged;
+	unsigned errorCount = 0; // error since watch start or last successful refresh
 
 	// Make sure this watch setup does not break due to async file system initialization not having been called
-	loop {
+	while (true) {
 		if (IAsyncFileSystem::filesystem())
 			break;
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 	const int& intervalSeconds = FLOW_KNOBS->PUBLIC_KEY_FILE_REFRESH_INTERVAL_SECONDS;
-	fileWatch = watchFileForChanges(filePath, &fileChanged, &intervalSeconds, "AuthzPublicKeySetRefreshStatError");
-	loop {
+	[[maybe_unused]] Future<Void> fileWatch =
+	    watchFileForChanges(filePath, &fileChanged, &intervalSeconds, "AuthzPublicKeySetRefreshStatError");
+	while (true) {
 		try {
-			wait(fileChanged.onTrigger());
-			state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
-			    filePath, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
-			state int64_t filesize = wait(file->size());
-			state std::string json(filesize, '\0');
+			co_await fileChanged.onTrigger();
+			Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
+			    filePath, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0);
+			int64_t filesize = co_await file->size();
+			std::string json(filesize, '\0');
 			if (filesize > FLOW_KNOBS->PUBLIC_KEY_FILE_MAX_SIZE)
 				throw file_too_large();
 			if (filesize <= 0) {
 				TraceEvent(SevWarn, "AuthzPublicKeySetEmpty").suppressFor(60);
 				continue;
 			}
-			wait(success(file->read(&json[0], filesize, 0)));
+			co_await file->read(&json[0], filesize, 0);
 			self->applyPublicKeySet(StringRef(json));
 			errorCount = 0;
 		} catch (Error& e) {
diff --git a/fdbrpc/HTTP.actor.cpp b/fdbrpc/HTTP.cpp
similarity index 80%
rename from fdbrpc/HTTP.actor.cpp
rename to fdbrpc/HTTP.cpp
index 018eeeef6b9..aeb7594811b 100644
--- a/fdbrpc/HTTP.actor.cpp
+++ b/fdbrpc/HTTP.cpp
@@ -1,5 +1,5 @@
 /*
- * HTTP.actor.cpp
+ * HTTP.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -30,9 +30,6 @@
 #include <cctype>
 #include <sstream>
 #include "flow/IConnection.h"
-#include <unordered_map>
-
-#include "flow/actorcompiler.h" // has to be last include
 
 namespace HTTP {
 
@@ -40,14 +37,14 @@ namespace HTTP {
 std::string awsV4URIEncode(const std::string& s, bool encodeSlash) {
 	std::string o;
 	o.reserve(s.size() * 3);
-	char buf[4];
+	char buf[32];
 	for (auto c : s) {
 		if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~')
 			o.append(&c, 1);
 		else if (c == '/')
 			o.append(encodeSlash ? "%2F" : "/");
 		else {
-			sprintf(buf, "%%%.02X", c);
+			snprintf(buf, sizeof(buf), "%%%.02X", c);
 			o.append(buf);
 		}
 	}
@@ -57,12 +54,12 @@ std::string awsV4URIEncode(const std::string& s, bool encodeSlash) {
 std::string urlEncode(const std::string& s) {
 	std::string o;
 	o.reserve(s.size() * 3);
-	char buf[4];
+	char buf[32];
 	for (auto c : s)
 		if (std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':')
 			o.append(&c, 1);
 		else {
-			sprintf(buf, "%%%.02X", c);
+			snprintf(buf, sizeof(buf), "%%%.02X", c);
 			o.append(buf);
 		}
 	return o;
@@ -154,8 +151,8 @@ bool verifyMD5(HTTPData<std::string>* data, bool fail_if_header_missing, Optiona
 std::string IncomingResponse::toString() const {
 	std::string r = fmt::format("Response Code: {0}\n", code);
 	r += fmt::format("Response ContentLen: {0}\n", data.contentLen);
-	for (auto h : data.headers)
-		r += fmt::format("Response Header: {0}: {1}\n", h.first, h.second);
+	for (const auto& [headerName, headerValue] : data.headers)
+		r += fmt::format("Response Header: {0}: {1}\n", headerName, headerValue);
 	r.append("-- RESPONSE CONTENT--\n");
 	// Limit the length of the response content to 1024 bytes for logging.
 	// No one wants 40MB of content dumped to the console.
@@ -171,10 +168,10 @@ std::string IncomingResponse::toString() const {
 }
 
 void writeHeaders(HTTP::Headers const& headers, PacketWriter& writer) {
-	for (auto h : headers) {
-		writer.serializeBytes(h.first);
+	for (const auto& [headerName, headerValue] : headers) {
+		writer.serializeBytes(headerName);
 		writer.serializeBytes(": "_sr);
-		writer.serializeBytes(h.second);
+		writer.serializeBytes(headerValue);
 		writer.serializeBytes("\r\n"_sr);
 	}
 	writer.serializeBytes("\r\n"_sr);
@@ -205,7 +202,7 @@ PacketBuffer* writeResponseHeader(Reference<OutgoingResponse> response, PacketBu
 	return writer.finish();
 }
 
-ACTOR Future<Void> writeResponse(Reference<IConnection> conn, Reference<OutgoingResponse> response) {
+Future<Void> writeResponse(Reference<IConnection> conn, Reference<OutgoingResponse> response) {
 
 	// Write headers to a packet buffer chain
 	ASSERT(response.isValid());
@@ -214,7 +211,7 @@ ACTOR Future<Void> writeResponse(Reference<IConnection> conn, Reference<Outgoing
 	PacketBuffer* pLast = writeResponseHeader(response, pFirst);
 	// Prepend headers to content packer buffer chain
 	response->data.content->prependWriteBuffer(pFirst, pLast);
-	loop {
+	while (true) {
 		int trySend = FLOW_KNOBS->HTTP_SEND_SIZE;
 		if ((!g_network->isSimulated() || !g_simulator->speedUpSimulation) && BUGGIFY_WITH_PROB(0.01)) {
 			trySend = deterministicRandom()->randomInt(1, 10);
@@ -222,18 +219,18 @@ ACTOR Future<Void> writeResponse(Reference<IConnection> conn, Reference<Outgoing
 		int len = conn->write(response->data.content->getUnsent(), trySend);
 		response->data.content->sent(len);
 		if (response->data.content->empty()) {
-			return Void();
+			co_return;
 		}
 
-		wait(conn->onWritable());
-		wait(yield(TaskPriority::WriteSocket));
+		co_await conn->onWritable();
+		co_await yield(TaskPriority::WriteSocket);
 	}
 }
 
 // Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
 // Returns the number of bytes read.
-ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string* buf, int maxlen) {
-	loop {
+Future<int> read_into_string(Reference<IConnection> conn, std::string* buf, int maxlen) {
+	while (true) {
 		// Read into buffer
 		int originalSize = buf->size();
 		// TODO:  resize is zero-initializing the space we're about to overwrite, so do something else, which probably
@@ -246,59 +243,55 @@ ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string* buf
 
 		// Make sure data was actually read, it's possible for there to be none.
 		if (len > 0)
-			return len;
+			co_return len;
 
 		// Wait for connection to have something to read
-		wait(conn->onReadable());
-		wait(delay(0, TaskPriority::ReadSocket));
+		co_await conn->onReadable();
+		co_await delay(0, TaskPriority::ReadSocket);
 	}
 }
 
 // Returns the position of delim within buf, relative to pos.  If delim is not found, continues to read from conn until
 // either it is found or the connection ends, at which point connection_failed is thrown and buf contains
 // everything that was read up to that point.
-ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
-                                                const char* delim,
-                                                std::string* buf,
-                                                size_t pos) {
-	state size_t sPos = pos;
-	state int lookBack = strlen(delim) - 1;
+Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
+                                          const char* delim,
+                                          std::string* buf,
+                                          size_t pos) {
+	size_t sPos = pos;
+	int lookBack = strlen(delim) - 1;
 	ASSERT(lookBack >= 0);
 
-	loop {
+	while (true) {
 		size_t endPos = buf->find(delim, sPos);
 		if (endPos != std::string::npos) {
-			return endPos - pos;
+			co_return endPos - pos;
 		}
 		// Next search will start at the current end of the buffer - delim size + 1
 		if (buf->size() >= lookBack) {
 			sPos = buf->size() - lookBack;
 		}
-		wait(success(read_into_string(conn, buf, FLOW_KNOBS->HTTP_READ_SIZE)));
+		co_await read_into_string(conn, buf, FLOW_KNOBS->HTTP_READ_SIZE);
 	}
 }
 
 // Reads from conn (as needed) until there are at least len bytes starting at pos in buf
-ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string* buf, size_t pos) {
-	state int stop_size = pos + len;
+Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string* buf, size_t pos) {
+	int stop_size = pos + len;
 	while (buf->size() < stop_size) {
-		wait(success(read_into_string(conn, buf, FLOW_KNOBS->HTTP_READ_SIZE)));
+		co_await read_into_string(conn, buf, FLOW_KNOBS->HTTP_READ_SIZE);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
-                                              Headers* headers,
-                                              std::string* buf,
-                                              size_t* pos) {
-	loop {
+Future<Void> read_http_response_headers(Reference<IConnection> conn, Headers* headers, std::string* buf, size_t* pos) {
+	while (true) {
 		// Get a line, reading more data from conn if necessary
-		size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
+		size_t lineLen = co_await read_delimited_into_string(conn, "\r\n", buf, *pos);
 		// If line is empty we have reached the end of the headers.
 		if (lineLen == 0) {
 			// Increment pos to move past the empty line.
 			*pos += 2;
-			return Void();
+			co_return;
 		}
 
 		int nameEnd = -1, valueStart = -1, valueEnd = -1;
@@ -345,25 +338,25 @@ ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
 // buf has the http header line in it at *pos offset.
 // FIXME: should this throw a different error for http requests? Or should we rename http_bad_response to
 // http_bad_<something>?
-ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
-                                Reference<IConnection> conn,
-                                std::string* buf,
-                                size_t* pos,
-                                bool content_optional,
-                                bool skipCheckMD5) {
+Future<Void> readHTTPData(HTTPData<std::string>* r,
+                          Reference<IConnection> conn,
+                          std::string* buf,
+                          size_t* pos,
+                          bool content_optional,
+                          bool skipCheckMD5) {
 	// Read headers
 	r->headers.clear();
 
-	wait(read_http_response_headers(conn, &r->headers, buf, pos));
+	co_await read_http_response_headers(conn, &r->headers, buf, pos);
 
 	auto i = r->headers.find("Content-Length");
 	if (i != r->headers.end()) {
-		r->contentLen = strtoll(i->second.c_str(), NULL, 10);
+		r->contentLen = strtoll(i->second.c_str(), nullptr, 10);
 	} else {
 		r->contentLen = -1; // Content length unknown
 	}
 
-	state std::string transferEncoding;
+	std::string transferEncoding;
 	i = r->headers.find("Transfer-Encoding");
 	if (i != r->headers.end()) {
 		transferEncoding = i->second;
@@ -374,7 +367,7 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 	// If this is allowed to be a header-only response and the buffer has been fully processed then stop.  Otherwise,
 	// there must be response content.
 	if (content_optional && *pos == buf->size()) {
-		return Void();
+		co_return;
 	}
 
 	// There should be content (or at least metadata describing that there is no content.
@@ -385,7 +378,7 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 		*pos = 0;
 
 		// Read until there are at least contentLen bytes available at pos
-		wait(read_fixed_into_string(conn, r->contentLen, &r->content, *pos));
+		co_await read_fixed_into_string(conn, r->contentLen, &r->content, *pos);
 
 		// There shouldn't be any bytes after content.
 		if (r->content.size() != r->contentLen) {
@@ -401,11 +394,11 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 		r->content = buf->substr(*pos);
 		*pos = 0;
 
-		loop {
+		while (true) {
 			{
 				// Read the line that contains the chunk length as text in hex
-				size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, *pos));
-				state int chunkLen = strtol(r->content.substr(*pos, lineLen).c_str(), nullptr, 16);
+				size_t lineLen = co_await read_delimited_into_string(conn, "\r\n", &r->content, *pos);
+				int chunkLen = strtol(r->content.substr(*pos, lineLen).c_str(), nullptr, 16);
 
 				// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the
 				// content buffer
@@ -416,13 +409,13 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 					break;
 
 				// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
-				wait(read_fixed_into_string(conn, chunkLen, &r->content, *pos));
+				co_await read_fixed_into_string(conn, chunkLen, &r->content, *pos);
 				*pos += chunkLen;
 			}
 
 			{
 				// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
-				size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, *pos));
+				size_t lineLen = co_await read_delimited_into_string(conn, "\r\n", &r->content, *pos);
 				if (lineLen != 0)
 					throw http_bad_response();
 
@@ -435,7 +428,7 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 		r->contentLen = *pos;
 
 		// Next is the post-chunk header block, so read that.
-		wait(read_http_response_headers(conn, &r->headers, &r->content, pos));
+		co_await read_http_response_headers(conn, &r->headers, &r->content, pos);
 
 		// If the header parsing did not consume all of the buffer then something is wrong
 		if (*pos != r->content.size()) {
@@ -456,9 +449,9 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 	}
 
 	// If there is actual response content, check the MD5 sum against the Content-MD5 response header
-	if (r->content.size() > 0) {
+	if (!r->content.empty()) {
 		if (skipCheckMD5) {
-			return Void();
+			co_return;
 		}
 
 		if (!HTTP::verifyMD5(r, false)) { // false arg means do not fail if the Content-MD5 header is missing.
@@ -468,19 +461,17 @@ ACTOR Future<Void> readHTTPData(HTTPData<std::string>* r,
 			throw http_bad_response();
 		}
 	}
-
-	return Void();
 }
 
 // Reads an HTTP request from a network connection
 // If the connection fails while being read the exception will emitted
 // If the response is not parsable or complete in some way, http_bad_response will be thrown
-ACTOR Future<Void> read_http_request(Reference<HTTP::IncomingRequest> r, Reference<IConnection> conn) {
-	state std::string buf;
-	state size_t pos = 0;
+Future<Void> read_http_request(Reference<HTTP::IncomingRequest> r, Reference<IConnection> conn) {
+	std::string buf;
+	size_t pos = 0;
 
 	// Read HTTP response code and version line
-	size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
+	size_t lineLen = co_await read_delimited_into_string(conn, "\r\n", &buf, pos);
 
 	// FIXME: this is pretty inefficient with 2 copies, but sscanf isn't the best with strings
 	std::string requestLine = buf.substr(0, lineLen);
@@ -535,8 +526,7 @@ ACTOR Future<Void> read_http_request(Reference<HTTP::IncomingRequest> r, Referen
 	// Move position past the line found and the delimiter length
 	pos += lineLen + 2;
 
-	wait(readHTTPData(&r->data, conn, &buf, &pos, false, false));
-	return Void();
+	co_await readHTTPData(&r->data, conn, &buf, &pos, false, false);
 }
 
 Future<Void> HTTP::IncomingRequest::read(Reference<IConnection> conn, bool header_only) {
@@ -557,14 +547,12 @@ void HTTP::OutgoingResponse::reset() {
 // Reads an HTTP response from a network connection
 // If the connection fails while being read the exception will emitted
 // If the response is not parsable or complete in some way, http_bad_response will be thrown
-ACTOR Future<Void> read_http_response(Reference<HTTP::IncomingResponse> r,
-                                      Reference<IConnection> conn,
-                                      bool header_only) {
-	state std::string buf;
-	state size_t pos = 0;
+Future<Void> read_http_response(Reference<HTTP::IncomingResponse> r, Reference<IConnection> conn, bool header_only) {
+	std::string buf;
+	size_t pos = 0;
 
 	// Read HTTP response line
-	size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
+	size_t lineLen = co_await read_delimited_into_string(conn, "\r\n", &buf, pos);
 
 	int reachedEnd = -1;
 	if (sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd) < 2 || reachedEnd < 0) {
@@ -580,9 +568,7 @@ ACTOR Future<Void> read_http_response(Reference<HTTP::IncomingResponse> r,
 
 	bool skipCheckMD5 = r->code == 206 && FLOW_KNOBS->HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT;
 
-	wait(readHTTPData(&r->data, conn, &buf, &pos, header_only, skipCheckMD5));
-
-	return Void();
+	co_await readHTTPData(&r->data, conn, &buf, &pos, header_only, skipCheckMD5);
 }
 
 Future<Void> HTTP::IncomingResponse::read(Reference<IConnection> conn, bool header_only) {
@@ -593,20 +579,20 @@ Future<Void> HTTP::IncomingResponse::read(Reference<IConnection> conn, bool head
 // Request content is provided as UnsentPacketQueue in req, which will be depleted as bytes are sent but the queue
 // itself must live for the life of this actor and be destroyed by the caller
 // TODO:  pSent is very hackish, do something better.
-ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnection> conn,
-                                                               Reference<OutgoingRequest> request,
-                                                               Reference<IRateControl> sendRate,
-                                                               int64_t* pSent,
-                                                               Reference<IRateControl> recvRate) {
-	state TraceEvent event(SevDebug, "HTTPRequest");
+Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnection> conn,
+                                                         Reference<OutgoingRequest> request,
+                                                         Reference<IRateControl> sendRate,
+                                                         int64_t* pSent,
+                                                         Reference<IRateControl> recvRate) {
+	TraceEvent event(SevDebug, "HTTPRequest");
 
 	// There is no standard http request id header field, so either a global default can be set via a knob
 	// or it can be set per-request with the requestIDHeader argument (which overrides the default)
-	state std::string requestIDHeader = FLOW_KNOBS->HTTP_REQUEST_ID_HEADER;
+	std::string requestIDHeader = FLOW_KNOBS->HTTP_REQUEST_ID_HEADER;
 
-	state bool earlyResponse = false;
-	state int total_sent = 0;
-	state double send_start;
+	bool earlyResponse = false;
+	int total_sent = 0;
+	double send_start{ 0 };
 
 	event.detail("DebugID", conn->getDebugID());
 	event.detail("RemoteAddress", conn->getPeerAddress());
@@ -615,7 +601,7 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnec
 	event.detail("RequestContentLen", request->data.contentLen);
 
 	try {
-		state std::string requestID;
+		std::string requestID;
 		if (!requestIDHeader.empty()) {
 			requestID = deterministicRandom()->randomUniqueID().toString();
 			requestID = requestID.insert(20, "-");
@@ -645,15 +631,15 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnec
 				fmt::print("Request Header: {}: {}\n", h.first, h.second);
 		}
 
-		state Reference<HTTP::IncomingResponse> r(new HTTP::IncomingResponse());
+		Reference<HTTP::IncomingResponse> r(new HTTP::IncomingResponse());
 		event.detail("IsHeaderOnlyResponse", request->isHeaderOnlyResponse());
-		state Future<Void> responseReading = r->read(conn, request->isHeaderOnlyResponse());
+		Future<Void> responseReading = r->read(conn, request->isHeaderOnlyResponse());
 		event.detail("ResponseReading", responseReading.isReady());
 
 		send_start = timer();
 
 		// too many state things here to refactor this with writing the response
-		loop {
+		while (true) {
 			// If we already got a response, before finishing sending the request, then close the connection,
 			// set the Connection header to "close" as a hint to the caller that this connection can't be used
 			// again, and break out of the send loop.
@@ -673,11 +659,11 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnec
 				break;
 			}
 
-			state int trySend = FLOW_KNOBS->HTTP_SEND_SIZE;
+			int trySend = FLOW_KNOBS->HTTP_SEND_SIZE;
 			if ((!g_network->isSimulated() || !g_simulator->speedUpSimulation) && BUGGIFY_WITH_PROB(0.01)) {
 				trySend = deterministicRandom()->randomInt(1, 10);
 			}
-			wait(sendRate->getAllowance(trySend));
+			co_await sendRate->getAllowance(trySend);
 			int len = conn->write(request->data.content->getUnsent(), trySend);
 			if (pSent != nullptr)
 				*pSent += len;
@@ -688,11 +674,11 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnec
 				break;
 			}
 
-			wait(conn->onWritable());
-			wait(yield(TaskPriority::WriteSocket));
+			co_await conn->onWritable();
+			co_await yield(TaskPriority::WriteSocket);
 		}
 
-		wait(responseReading);
+		co_await responseReading;
 		double elapsed = timer() - send_start;
 
 		event.detail("ResponseCode", r->code);
@@ -760,7 +746,7 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestActor(Reference<IConnec
 			throw err.get();
 		}
 
-		return r;
+		co_return r;
 	} catch (Error& e) {
 		double elapsed = timer() - send_start;
 		// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
@@ -789,18 +775,16 @@ Future<Reference<IncomingResponse>> doRequest(Reference<IConnection> conn,
 	return doRequestActor(conn, request, sendRate, pSent, recvRate);
 }
 
-ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
-                                           std::string remoteHost,
-                                           std::string remoteService) {
-	state int requestTimeout = 60;
-	state int maxTries = FLOW_KNOBS->RESTCLIENT_CONNECT_TRIES;
-	state int thisTry = 1;
-	state double nextRetryDelay = 2.0;
-	state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
-	state int64_t bytes_sent = 0;
-	state UnsentPacketQueue empty_packet_queue;
-
-	state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
+Future<Void> sendProxyConnectRequest(Reference<IConnection> conn, std::string remoteHost, std::string remoteService) {
+	int requestTimeout = 60;
+	int maxTries = FLOW_KNOBS->RESTCLIENT_CONNECT_TRIES;
+	int thisTry = 1;
+	double nextRetryDelay = 2.0;
+	Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+	int64_t bytes_sent = 0;
+	UnsentPacketQueue empty_packet_queue;
+
+	auto req = makeReference<HTTP::OutgoingRequest>();
 	req->verb = HTTP_VERB_CONNECT;
 	req->resource = remoteHost + ":" + remoteService;
 	req->data.content = &empty_packet_queue;
@@ -809,16 +793,13 @@ ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
 	req->data.headers["Accept"] = "application/xml";
 	req->data.headers["Proxy-Connection"] = "Keep-Alive";
 
-	loop {
-		state Optional<Error> err;
-
-		state Reference<HTTP::IncomingResponse> r;
+	while (true) {
+		Optional<Error> err;
+		Reference<HTTP::IncomingResponse> r;
 
 		try {
-			Future<Reference<HTTP::IncomingResponse>> f =
-			    HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate);
-			Reference<HTTP::IncomingResponse> _r = wait(timeoutError(f, requestTimeout));
-			r = _r;
+			r = co_await timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate),
+			                          requestTimeout);
 		} catch (Error& e) {
 			TraceEvent("ProxyRequestFailed").errorUnsuppressed(e);
 			if (e.code() == error_code_actor_cancelled)
@@ -832,7 +813,7 @@ ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
 			TraceEvent(SevDebug, "ProxyRequestSuccess")
 			    .detail("RemoteHost", remoteHost)
 			    .detail("RemoteService", remoteService);
-			return Void();
+			co_return;
 		}
 
 		// All errors in err are potentially retryable as well as certain HTTP response codes...
@@ -882,7 +863,7 @@ ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
 
 			// Log the delay then wait.
 			event.detail("RetryDelay", delay);
-			wait(::delay(delay));
+			co_await ::delay(delay);
 		} else {
 			// We can't retry, so throw something.
 
@@ -898,23 +879,23 @@ ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
 	}
 }
 
-ACTOR Future<Reference<IConnection>> proxyConnectImpl(std::string remoteHost,
-                                                      std::string remoteService,
-                                                      std::string proxyHost,
-                                                      std::string proxyService) {
-	state NetworkAddress remoteEndpoint =
-	    wait(map(INetworkConnections::net()->resolveTCPEndpoint(remoteHost, remoteService),
-	             [=](std::vector<NetworkAddress> const& addresses) -> NetworkAddress {
-		             NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
-		             addr.fromHostname = true;
-		             addr.flags = NetworkAddress::FLAG_TLS;
-		             return addr;
-	             }));
-	state Reference<IConnection> connection = wait(INetworkConnections::net()->connect(proxyHost, proxyService));
-	wait(sendProxyConnectRequest(connection, remoteHost, remoteService));
+Future<Reference<IConnection>> proxyConnectImpl(std::string remoteHost,
+                                                std::string remoteService,
+                                                std::string proxyHost,
+                                                std::string proxyService) {
+	NetworkAddress remoteEndpoint =
+	    co_await map(INetworkConnections::net()->resolveTCPEndpoint(remoteHost, remoteService),
+	                 [=](std::vector<NetworkAddress> const& addresses) -> NetworkAddress {
+		                 NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
+		                 addr.fromHostname = true;
+		                 addr.flags = NetworkAddress::FLAG_TLS;
+		                 return addr;
+	                 });
+	Reference<IConnection> connection = co_await INetworkConnections::net()->connect(proxyHost, proxyService);
+	co_await sendProxyConnectRequest(connection, remoteHost, remoteService);
 	boost::asio::ip::tcp::socket socket = std::move(connection->getSocket());
-	Reference<IConnection> remoteConnection = wait(INetworkConnections::net()->connect(remoteEndpoint, &socket));
-	return remoteConnection;
+	Reference<IConnection> remoteConnection = co_await INetworkConnections::net()->connect(remoteEndpoint, &socket);
+	co_return remoteConnection;
 }
 
 Future<Reference<IConnection>> proxyConnect(const std::string& remoteHost,
diff --git a/fdbrpc/HTTPServer.actor.cpp b/fdbrpc/HTTPServer.cpp
similarity index 67%
rename from fdbrpc/HTTPServer.actor.cpp
rename to fdbrpc/HTTPServer.cpp
index 6b55013e15d..8ab4f4bddcb 100644
--- a/fdbrpc/HTTPServer.actor.cpp
+++ b/fdbrpc/HTTPServer.cpp
@@ -1,5 +1,5 @@
 /*
- * HTTPServer.actor.cpp
+ * HTTPServer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,22 +22,19 @@
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "fdbrpc/simulator.h"
-#include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR Future<Void> callbackHandler(Reference<IConnection> conn,
-                                   Future<Void> readRequestDone,
-                                   Reference<HTTP::IRequestHandler> requestHandler,
-                                   Reference<HTTP::IncomingRequest> req,
-                                   FlowMutex* mutex) {
-	state Reference<HTTP::OutgoingResponse> response = makeReference<HTTP::OutgoingResponse>();
-	state UnsentPacketQueue content;
+Future<Void> callbackHandler(Reference<IConnection> conn,
+                             Future<Void> readRequestDone,
+                             Reference<HTTP::IRequestHandler> requestHandler,
+                             Reference<HTTP::IncomingRequest> req,
+                             FlowMutex* mutex) {
+	auto response = makeReference<HTTP::OutgoingResponse>();
+	UnsentPacketQueue content;
 	response->data.content = &content;
 	response->data.contentLen = 0;
 
 	try {
-		wait(readRequestDone);
-		wait(requestHandler->handleRequest(req, response));
+		co_await readRequestDone;
+		co_await requestHandler->handleRequest(req, response);
 	} catch (Error& e) {
 		if (e.code() == error_code_operation_cancelled) {
 			throw e;
@@ -46,7 +43,7 @@ ACTOR Future<Void> callbackHandler(Reference<IConnection> conn,
 		if (e.code() == error_code_connection_failed) {
 			TraceEvent(SevWarn, "HTTPServerConnHandlerFailed").error(e);
 			// return, client will retry
-			return Void();
+			co_return;
 		}
 		if (e.code() == error_code_http_request_failed || e.code() == error_code_http_bad_response) {
 			TraceEvent(SevWarn, "HTTPServerConnHandlerInternalError").errorUnsuppressed(e);
@@ -60,36 +57,35 @@ ACTOR Future<Void> callbackHandler(Reference<IConnection> conn,
 	}
 	// take out response mutex to ensure no parallel writers to response connection
 	// FIXME: is this necessary? I think it is
-	state FlowMutex::Lock lock = wait(mutex->take());
+	FlowMutex::Lock lock = co_await mutex->take();
 	try {
-		wait(response->write(conn));
+		co_await response->write(conn);
 	} catch (Error& e) {
 		lock.release();
 		if (e.code() == error_code_connection_failed) {
 			// connection back to client failed, end. They will retry if they still need the response.
 			TraceEvent("HTTPServerConnHandlerResponseError").errorUnsuppressed(e);
-			return Void();
+			co_return;
 		}
 		TraceEvent("HTTPServerConnHandlerResponseUnexpectedError").errorUnsuppressed(e);
 		throw e;
 	}
 	lock.release();
-	return Void();
 }
 
-ACTOR Future<Void> connectionHandler(Reference<HTTP::SimServerContext> server,
-                                     Reference<IConnection> conn,
-                                     Reference<HTTP::IRequestHandler> requestHandler) {
+Future<Void> connectionHandler(Reference<HTTP::SimServerContext> server,
+                               Reference<IConnection> conn,
+                               Reference<HTTP::IRequestHandler> requestHandler) {
 	try {
 		// TODO do we actually have multiple requests on a connection? how does this work
-		state FlowMutex responseMutex;
-		state Future<Void> readPrevRequest = Future<Void>(Void());
-		wait(conn->acceptHandshake());
-		loop {
-			wait(readPrevRequest);
-			wait(delay(0));
-			wait(conn->onReadable());
-			state Reference<HTTP::IncomingRequest> req = makeReference<HTTP::IncomingRequest>();
+		FlowMutex responseMutex;
+		Future<Void> readPrevRequest = Future<Void>(Void());
+		co_await conn->acceptHandshake();
+		while (true) {
+			co_await readPrevRequest;
+			co_await delay(0);
+			co_await conn->onReadable();
+			auto req = makeReference<HTTP::IncomingRequest>();
 			readPrevRequest = req->read(conn, false);
 			server->actors.add(callbackHandler(conn, readPrevRequest, requestHandler, req, &responseMutex));
 		}
@@ -103,22 +99,21 @@ ACTOR Future<Void> connectionHandler(Reference<HTTP::SimServerContext> server,
 		}
 		conn->close();
 	}
-	return Void();
 }
 
-ACTOR Future<Void> listenActor(Reference<HTTP::SimServerContext> server,
-                               Reference<HTTP::IRequestHandler> requestHandler,
-                               NetworkAddress addr,
-                               Reference<IListener> listener) {
+Future<Void> listenActor(Reference<HTTP::SimServerContext> server,
+                         Reference<HTTP::IRequestHandler> requestHandler,
+                         NetworkAddress addr,
+                         Reference<IListener> listener) {
 	TraceEvent(SevDebug, "HTTPServerListenStart", server->dbgid).detail("ListenAddress", addr.toString());
 
-	wait(requestHandler->init());
+	co_await requestHandler->init();
 
 	TraceEvent(SevDebug, "HTTPServerListenInitialized", server->dbgid).detail("ListenAddress", addr.toString());
 
 	try {
-		loop {
-			Reference<IConnection> conn = wait(listener->accept());
+		while (true) {
+			Reference<IConnection> conn = co_await listener->accept();
 			if (!server->running) {
 				TraceEvent("HTTPServerExitedAfterAccept", server->dbgid);
 				break;
@@ -131,8 +126,6 @@ ACTOR Future<Void> listenActor(Reference<HTTP::SimServerContext> server,
 		TraceEvent(SevError, "HTTPListenError", server->dbgid).error(e);
 		throw;
 	}
-
-	return Void();
 }
 
 void HTTP::SimServerContext::registerNewServer(NetworkAddress addr, Reference<HTTP::IRequestHandler> requestHandler) {
@@ -184,9 +177,9 @@ Future<Void> HTTP::registerAlwaysFailHTTPHandler() {
 
 // unit test stuff
 
-ACTOR Future<Void> helloWorldServerCallback(Reference<HTTP::IncomingRequest> req,
-                                            Reference<HTTP::OutgoingResponse> response) {
-	wait(delay(0));
+Future<Void> helloWorldServerCallback(Reference<HTTP::IncomingRequest> req,
+                                      Reference<HTTP::OutgoingResponse> response) {
+	co_await delay(0);
 	ASSERT_EQ(req->verb, HTTP::HTTP_VERB_POST);
 	ASSERT_EQ(req->resource, "/hello-world");
 	ASSERT_EQ(req->data.headers.size(), 2);
@@ -207,8 +200,6 @@ ACTOR Future<Void> helloWorldServerCallback(Reference<HTTP::IncomingRequest> req
 	PacketWriter pw(response->data.content->getWriteBuffer(hello.size()), nullptr, Unversioned());
 	pw.serializeBytes(hello);
 	response->data.contentLen = hello.size();
-
-	return Void();
 }
 
 struct HelloWorldRequestHandler : HTTP::IRequestHandler, ReferenceCounted<HelloWorldRequestHandler> {
@@ -222,9 +213,9 @@ struct HelloWorldRequestHandler : HTTP::IRequestHandler, ReferenceCounted<HelloW
 	void delref() override { ReferenceCounted<HelloWorldRequestHandler>::delref(); }
 };
 
-ACTOR Future<Void> helloErrorServerCallback(Reference<HTTP::IncomingRequest> req,
-                                            Reference<HTTP::OutgoingResponse> response) {
-	wait(delay(0));
+Future<Void> helloErrorServerCallback(Reference<HTTP::IncomingRequest> req,
+                                      Reference<HTTP::OutgoingResponse> response) {
+	co_await delay(0);
 	if (deterministicRandom()->coinflip()) {
 		throw http_bad_response();
 	} else {
@@ -243,9 +234,9 @@ struct HelloErrorRequestHandler : HTTP::IRequestHandler, ReferenceCounted<HelloE
 	void delref() override { ReferenceCounted<HelloErrorRequestHandler>::delref(); }
 };
 
-ACTOR Future<Void> helloBadMD5ServerCallback(Reference<HTTP::IncomingRequest> req,
-                                             Reference<HTTP::OutgoingResponse> response) {
-	wait(delay(0));
+Future<Void> helloBadMD5ServerCallback(Reference<HTTP::IncomingRequest> req,
+                                       Reference<HTTP::OutgoingResponse> response) {
+	co_await delay(0);
 	ASSERT_EQ(req->verb, HTTP::HTTP_VERB_GET);
 	ASSERT_EQ(req->resource, "/hello-world");
 	ASSERT_EQ(req->data.headers.size(), 1);
@@ -266,8 +257,6 @@ ACTOR Future<Void> helloBadMD5ServerCallback(Reference<HTTP::IncomingRequest> re
 	PacketWriter pw(response->data.content->getWriteBuffer(hello.size()), nullptr, Unversioned());
 	pw.serializeBytes(hello);
 	response->data.contentLen = hello.size();
-
-	return Void();
 }
 
 struct HelloBadMD5RequestHandler : HTTP::IRequestHandler, ReferenceCounted<HelloBadMD5RequestHandler> {
@@ -281,46 +270,48 @@ struct HelloBadMD5RequestHandler : HTTP::IRequestHandler, ReferenceCounted<Hello
 	void delref() override { ReferenceCounted<HelloBadMD5RequestHandler>::delref(); }
 };
 
-typedef std::function<Future<Reference<HTTP::IncomingResponse>>(Reference<IConnection> conn)> DoRequestFunction;
+using DoRequestFunction = std::function<Future<Reference<HTTP::IncomingResponse>>(Reference<IConnection> conn)>;
 
 // handles retrying on timeout and reinitializing connection like other users of HTTP (S3BlobStore, RestClient)
-ACTOR Future<Reference<HTTP::IncomingResponse>> doRequestTest(std::string hostname,
-                                                              std::string service,
-                                                              DoRequestFunction reqFunction) {
-	state Reference<IConnection> conn;
-	loop {
+Future<Reference<HTTP::IncomingResponse>> doRequestTest(std::string hostname,
+                                                        std::string service,
+                                                        DoRequestFunction reqFunction) {
+	Reference<IConnection> conn;
+	while (true) {
+		Error err;
 		try {
 			if (!conn) {
-				wait(store(conn, INetworkConnections::net()->connect(hostname, service, false)));
+				conn = co_await INetworkConnections::net()->connect(hostname, service, false);
 				ASSERT(conn.isValid());
-				wait(conn->connectHandshake());
+				co_await conn->connectHandshake();
 			}
 
-			Future<Reference<HTTP::IncomingResponse>> f = reqFunction(conn);
-			Reference<HTTP::IncomingResponse> response = wait(f);
+			Reference<HTTP::IncomingResponse> response = co_await reqFunction(conn);
 			conn->close();
-			return response;
+			co_return response;
 		} catch (Error& e) {
-			if (conn) {
-				conn->close();
-			}
-			if (e.code() != error_code_timed_out && e.code() != error_code_connection_failed &&
-			    e.code() != error_code_lookup_failed) {
-				throw e;
-			}
-			// request got timed out or connection could not be established, close conn and try again
-			conn.clear();
-			wait(delay(0.1));
+			err = e;
 		}
+
+		if (conn) {
+			conn->close();
+		}
+		if (err.code() != error_code_timed_out && err.code() != error_code_connection_failed &&
+		    err.code() != error_code_lookup_failed) {
+			throw err;
+		}
+		// Request timed out or the connection could not be established, so clear conn and try again.
+		conn.clear();
+		co_await delay(0.1);
 	}
 }
 
-ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloWorldReq(Reference<IConnection> conn) {
-	state UnsentPacketQueue content;
-	state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
+Future<Reference<HTTP::IncomingResponse>> doHelloWorldReq(Reference<IConnection> conn) {
+	UnsentPacketQueue content;
+	auto req = makeReference<HTTP::OutgoingRequest>();
 
-	state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
-	state int64_t bytes_sent = 0;
+	Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+	int64_t bytes_sent = 0;
 
 	req->verb = HTTP::HTTP_VERB_POST;
 	req->resource = "/hello-world";
@@ -335,7 +326,7 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloWorldReq(Reference<IConne
 	pw.serializeBytes(hello);
 
 	Reference<HTTP::IncomingResponse> response =
-	    wait(timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 30.0));
+	    co_await timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 30.0);
 
 	std::string expectedContent = "Hello World Response!";
 
@@ -350,15 +341,15 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloWorldReq(Reference<IConne
 	ASSERT_EQ(response->data.contentLen, response->data.content.size());
 	ASSERT_EQ(response->data.content, expectedContent);
 
-	return response;
+	co_return response;
 }
 
-ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloWorldErrorReq(Reference<IConnection> conn) {
-	state UnsentPacketQueue content;
-	state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
+Future<Reference<HTTP::IncomingResponse>> doHelloWorldErrorReq(Reference<IConnection> conn) {
+	UnsentPacketQueue content;
+	auto req = makeReference<HTTP::OutgoingRequest>();
 
-	state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
-	state int64_t bytes_sent = 0;
+	Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+	int64_t bytes_sent = 0;
 
 	req->verb = HTTP::HTTP_VERB_GET;
 	req->resource = "/hello-error";
@@ -367,19 +358,19 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloWorldErrorReq(Reference<I
 	req->data.contentLen = 0;
 
 	Reference<HTTP::IncomingResponse> response =
-	    wait(timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 30.0));
+	    co_await timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 30.0);
 
 	ASSERT(response->code == 500);
 
-	return response;
+	co_return response;
 }
 
-ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloBadMD5Req(Reference<IConnection> conn) {
-	state UnsentPacketQueue content;
-	state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
+Future<Reference<HTTP::IncomingResponse>> doHelloBadMD5Req(Reference<IConnection> conn) {
+	UnsentPacketQueue content;
+	auto req = makeReference<HTTP::OutgoingRequest>();
 
-	state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
-	state int64_t bytes_sent = 0;
+	Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+	int64_t bytes_sent = 0;
 
 	req->verb = HTTP::HTTP_VERB_GET;
 	req->resource = "/hello-world";
@@ -393,56 +384,53 @@ ACTOR Future<Reference<HTTP::IncomingResponse>> doHelloBadMD5Req(Reference<IConn
 	pw.serializeBytes(hello);
 
 	Reference<HTTP::IncomingResponse> response =
-	    wait(timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 30.0));
+	    co_await timeoutError(HTTP::doRequest(conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 30.0);
 
 	// should have gotten error
 	ASSERT(false);
 
-	return response;
+	co_return response;
 }
 
 // can't run as regular unit test right now because it needs special setup
 TEST_CASE("/HTTP/Server/HelloWorld") {
 	ASSERT(g_network->isSimulated());
 	fmt::print("Registering sim server\n");
-	state std::string hostname = "helloworld-" + deterministicRandom()->randomUniqueID().toString();
-	wait(g_simulator->registerSimHTTPServer(hostname, "80", makeReference<HelloWorldRequestHandler>()));
+	std::string hostname = "helloworld-" + deterministicRandom()->randomUniqueID().toString();
+	co_await g_simulator->registerSimHTTPServer(hostname, "80", makeReference<HelloWorldRequestHandler>());
 	fmt::print("Registered sim server\n");
 
-	wait(success(doRequestTest(hostname, "80", doHelloWorldReq)));
+	co_await doRequestTest(hostname, "80", doHelloWorldReq);
 
 	fmt::print("Done Hello\n");
-	return Void();
 }
 
 TEST_CASE("/HTTP/Server/HelloError") {
 	ASSERT(g_network->isSimulated());
 	fmt::print("Registering sim server\n");
-	state std::string hostname = "helloerror-" + deterministicRandom()->randomUniqueID().toString();
-	wait(g_simulator->registerSimHTTPServer(hostname, "80", makeReference<HelloErrorRequestHandler>()));
+	std::string hostname = "helloerror-" + deterministicRandom()->randomUniqueID().toString();
+	co_await g_simulator->registerSimHTTPServer(hostname, "80", makeReference<HelloErrorRequestHandler>());
 	fmt::print("Registered sim server\n");
 
-	wait(success(doRequestTest(hostname, "80", doHelloWorldErrorReq)));
+	co_await doRequestTest(hostname, "80", doHelloWorldErrorReq);
 
 	fmt::print("Done Error\n");
-	return Void();
 }
 
 TEST_CASE("/HTTP/Server/HelloBadMD5") {
 	ASSERT(g_network->isSimulated());
 	fmt::print("Registering sim server\n");
-	state std::string hostname = "hellobadmd5-" + deterministicRandom()->randomUniqueID().toString();
-	wait(g_simulator->registerSimHTTPServer(hostname, "80", makeReference<HelloBadMD5RequestHandler>()));
+	std::string hostname = "hellobadmd5-" + deterministicRandom()->randomUniqueID().toString();
+	co_await g_simulator->registerSimHTTPServer(hostname, "80", makeReference<HelloBadMD5RequestHandler>());
 	fmt::print("Registered sim server\n");
 
 	// TODO refactor this into ASSERT_ERROR()?
 	try {
-		wait(success(doRequestTest(hostname, "80", doHelloBadMD5Req)));
+		co_await doRequestTest(hostname, "80", doHelloBadMD5Req);
 		ASSERT(false);
 	} catch (Error& e) {
 		ASSERT(e.code() == error_code_http_bad_response);
 	}
 
 	fmt::print("Done Bad MD5\n");
-	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbrpc/HealthMonitor.actor.cpp b/fdbrpc/HealthMonitor.cpp
similarity index 95%
rename from fdbrpc/HealthMonitor.actor.cpp
rename to fdbrpc/HealthMonitor.cpp
index 481bb2ebbb4..452edeec18b 100644
--- a/fdbrpc/HealthMonitor.actor.cpp
+++ b/fdbrpc/HealthMonitor.cpp
@@ -1,5 +1,5 @@
 /*
- * HealthMonitor.actor.cpp
+ * HealthMonitor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,8 +18,6 @@
  * limitations under the License.
  */
 
-#include "fdbrpc/FailureMonitor.h"
-#include "fdbrpc/FlowTransport.h"
 #include "fdbrpc/HealthMonitor.h"
 
 void HealthMonitor::reportPeerClosed(const NetworkAddress& peerAddress) {
diff --git a/fdbrpc/IPAllowList.cpp b/fdbrpc/IPAllowList.cpp
index b31d67be0da..e80d10ee9ac 100644
--- a/fdbrpc/IPAllowList.cpp
+++ b/fdbrpc/IPAllowList.cpp
@@ -180,8 +180,8 @@ IPAddress parseAddr(std::string const& str) {
 
 struct SubNetTest {
 	AuthAllowedSubnet subnet;
-	SubNetTest(AuthAllowedSubnet&& subnet) : subnet(std::move(subnet)) {}
-	SubNetTest(AuthAllowedSubnet const& subnet) : subnet(subnet) {}
+	explicit SubNetTest(AuthAllowedSubnet&& subnet) : subnet(std::move(subnet)) {}
+	explicit SubNetTest(AuthAllowedSubnet const& subnet) : subnet(subnet) {}
 	template <bool V4>
 	static SubNetTest randomSubNetImpl() {
 		constexpr int width = V4 ? 4 : 16;
diff --git a/fdbrpc/JsonWebKeySet.cpp b/fdbrpc/JsonWebKeySet.cpp
index fffe1c3e5d6..62e5c3f5653 100644
--- a/fdbrpc/JsonWebKeySet.cpp
+++ b/fdbrpc/JsonWebKeySet.cpp
@@ -25,9 +25,9 @@
 #include "flow/MkCert.h"
 #include "flow/PKey.h"
 #include "flow/UnitTest.h"
-#include "fdbrpc/Base64Encode.h"
-#include "fdbrpc/Base64Decode.h"
-#include "fdbrpc/JsonWebKeySet.h"
+#include "Base64Encode.h"
+#include "Base64Decode.h"
+#include "JsonWebKeySet.h"
 #include <openssl/bn.h>
 #include <openssl/ec.h>
 #include <openssl/err.h>
diff --git a/fdbrpc/include/fdbrpc/JsonWebKeySet.h b/fdbrpc/JsonWebKeySet.h
similarity index 92%
rename from fdbrpc/include/fdbrpc/JsonWebKeySet.h
rename to fdbrpc/JsonWebKeySet.h
index d1671b5e98e..077236b2428 100644
--- a/fdbrpc/include/fdbrpc/JsonWebKeySet.h
+++ b/fdbrpc/JsonWebKeySet.h
@@ -30,9 +30,9 @@ struct PublicOrPrivateKey {
 
 	PublicOrPrivateKey() noexcept = default;
 
-	PublicOrPrivateKey(PublicKey key) noexcept : key(std::in_place_type<PublicKey>, key) {}
+	explicit PublicOrPrivateKey(PublicKey key) noexcept : key(std::in_place_type<PublicKey>, key) {}
 
-	PublicOrPrivateKey(PrivateKey key) noexcept : key(std::in_place_type<PrivateKey>, key) {}
+	explicit PublicOrPrivateKey(PrivateKey key) noexcept : key(std::in_place_type<PrivateKey>, key) {}
 
 	bool isPublic() const noexcept { return std::holds_alternative<PublicKey>(key); }
 
diff --git a/fdbrpc/Locality.cpp b/fdbrpc/Locality.cpp
index ff19da67583..e44e7077d72 100644
--- a/fdbrpc/Locality.cpp
+++ b/fdbrpc/Locality.cpp
@@ -20,6 +20,8 @@
 
 #include "fdbrpc/Locality.h"
 
+#include <cstdio>
+
 const UID LocalityData::UNSET_ID = UID(0x0ccb4e0feddb5583, 0x010f6b77d9d10ece);
 alignas(8) const StringRef LocalityData::keyProcessId = "processid"_sr;
 alignas(8) const StringRef LocalityData::keyZoneId = "zoneid"_sr;
@@ -28,6 +30,186 @@ alignas(8) const StringRef LocalityData::keyMachineId = "machineid"_sr;
 alignas(8) const StringRef LocalityData::keyDataHallId = "data_hall"_sr;
 alignas(8) const StringRef LocalityData::ExcludeLocalityPrefix = "locality_"_sr;
 
+ProcessClass::ProcessClass(std::string s, ClassSource source) : _source(source) {
+	if (s == "storage")
+		_class = StorageClass;
+	else if (s == "transaction")
+		_class = TransactionClass;
+	else if (s == "resolution")
+		_class = ResolutionClass;
+	else if (s == "commit_proxy")
+		_class = CommitProxyClass;
+	else if (s == "proxy") {
+		_class = CommitProxyClass;
+		printf("WARNING: 'proxy' machine class is deprecated and will be automatically converted "
+		       "'commit_proxy' machine class. Please use 'grv_proxy' or 'commit_proxy' specifically\n");
+	} else if (s == "grv_proxy")
+		_class = GrvProxyClass;
+	else if (s == "master")
+		_class = MasterClass;
+	else if (s == "test")
+		_class = TesterClass;
+	else if (s == "unset")
+		_class = UnsetClass;
+	else if (s == "stateless")
+		_class = StatelessClass;
+	else if (s == "log")
+		_class = LogClass;
+	else if (s == "router")
+		_class = LogRouterClass;
+	else if (s == "cluster_controller")
+		_class = ClusterControllerClass;
+	else if (s == "fast_restore")
+		ASSERT(false); // deprecated
+	else if (s == "data_distributor")
+		_class = DataDistributorClass;
+	else if (s == "coordinator")
+		_class = CoordinatorClass;
+	else if (s == "ratekeeper")
+		_class = RatekeeperClass;
+	else if (s == "consistency_scan")
+		_class = ConsistencyScanClass;
+	else if (s == "blob_manager")
+		_class = BlobManagerClass;
+	else if (s == "blob_worker")
+		_class = BlobWorkerClass;
+	else if (s == "backup")
+		_class = BackupClass;
+	else if (s == "encrypt_key_proxy")
+		_class = EncryptKeyProxyClass;
+	else if (s == "sim_http_server")
+		_class = SimHTTPServerClass;
+	else
+		_class = InvalidClass;
+}
+
+ProcessClass::ProcessClass(std::string classStr, std::string sourceStr) {
+	if (classStr == "storage")
+		_class = StorageClass;
+	else if (classStr == "transaction")
+		_class = TransactionClass;
+	else if (classStr == "resolution")
+		_class = ResolutionClass;
+	else if (classStr == "commit_proxy")
+		_class = CommitProxyClass;
+	else if (classStr == "proxy") {
+		_class = CommitProxyClass;
+		printf("WARNING: 'proxy' machine class is deprecated and will be automatically converted "
+		       "'commit_proxy' machine class. Please use 'grv_proxy' or 'commit_proxy' specifically\n");
+	} else if (classStr == "grv_proxy")
+		_class = GrvProxyClass;
+	else if (classStr == "master")
+		_class = MasterClass;
+	else if (classStr == "test")
+		_class = TesterClass;
+	else if (classStr == "unset")
+		_class = UnsetClass;
+	else if (classStr == "stateless")
+		_class = StatelessClass;
+	else if (classStr == "log")
+		_class = LogClass;
+	else if (classStr == "router")
+		_class = LogRouterClass;
+	else if (classStr == "cluster_controller")
+		_class = ClusterControllerClass;
+	else if (classStr == "fast_restore")
+		ASSERT(false); // deprecated
+	else if (classStr == "data_distributor")
+		_class = DataDistributorClass;
+	else if (classStr == "coordinator")
+		_class = CoordinatorClass;
+	else if (classStr == "ratekeeper")
+		_class = RatekeeperClass;
+	else if (classStr == "consistency_scan")
+		_class = ConsistencyScanClass;
+	else if (classStr == "blob_manager")
+		_class = BlobManagerClass;
+	else if (classStr == "blob_worker")
+		_class = BlobWorkerClass;
+	else if (classStr == "backup")
+		_class = BackupClass;
+	else if (classStr == "encrypt_key_proxy")
+		_class = EncryptKeyProxyClass;
+	else if (classStr == "sim_http_server")
+		_class = SimHTTPServerClass;
+	else
+		_class = InvalidClass;
+
+	if (sourceStr == "command_line")
+		_source = CommandLineSource;
+	else if (sourceStr == "configure_auto")
+		_source = AutoSource;
+	else if (sourceStr == "set_class")
+		_source = DBSource;
+	else
+		_source = InvalidSource;
+}
+
+std::string ProcessClass::toString() const {
+	switch (_class) {
+	case UnsetClass:
+		return "unset";
+	case StorageClass:
+		return "storage";
+	case TransactionClass:
+		return "transaction";
+	case ResolutionClass:
+		return "resolution";
+	case CommitProxyClass:
+		return "commit_proxy";
+	case GrvProxyClass:
+		return "grv_proxy";
+	case MasterClass:
+		return "master";
+	case TesterClass:
+		return "test";
+	case StatelessClass:
+		return "stateless";
+	case LogClass:
+		return "log";
+	case LogRouterClass:
+		return "router";
+	case ClusterControllerClass:
+		return "cluster_controller";
+	case FastRestoreClass:
+		ASSERT(false);
+		return "fast_restore";
+	case DataDistributorClass:
+		return "data_distributor";
+	case CoordinatorClass:
+		return "coordinator";
+	case RatekeeperClass:
+		return "ratekeeper";
+	case ConsistencyScanClass:
+		return "consistency_scan";
+	case BlobManagerClass:
+		return "blob_manager";
+	case BlobWorkerClass:
+		return "blob_worker";
+	case BackupClass:
+		return "backup";
+	case EncryptKeyProxyClass:
+		return "encrypt_key_proxy";
+	case SimHTTPServerClass:
+		return "sim_http_server";
+	default:
+		return "invalid";
+	}
+}
+
+std::string ProcessClass::sourceString() const {
+	switch (_source) {
+	case CommandLineSource:
+		return "command_line";
+	case AutoSource:
+		return "configure_auto";
+	case DBSource:
+		return "set_class";
+	default:
+		return "invalid";
+	}
+}
+
 ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const {
 	switch (role) {
 	case ProcessClass::Storage:
diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp
index c65ace83159..fff7601f7d7 100644
--- a/fdbrpc/Net2FileSystem.cpp
+++ b/fdbrpc/Net2FileSystem.cpp
@@ -21,7 +21,6 @@
 #include "fdbrpc/Net2FileSystem.h"
 
 // Define boost::asio::io_service
-#include <algorithm>
 #ifndef BOOST_SYSTEM_NO_LIB
 #define BOOST_SYSTEM_NO_LIB
 #endif
@@ -35,15 +34,99 @@
 
 #define FILESYSTEM_IMPL 1
 
-#include "fdbrpc/AsyncFileCached.actor.h"
-#include "fdbrpc/AsyncFileChaos.h"
-#include "fdbrpc/AsyncFileEIO.actor.h"
-#include "fdbrpc/AsyncFileEncrypted.h"
-#include "fdbrpc/AsyncFileWinASIO.actor.h"
-#include "fdbrpc/AsyncFileKAIO.actor.h"
+#include "fdbrpc/AsyncFileCached.h"
+#include "AsyncFileChaos.h"
+#include "AsyncFileEIO.h"
+#include "AsyncFileWinASIO.h"
+#include "AsyncFileKAIO.h"
 #include "flow/AsioReactor.h"
 #include "flow/Platform.h"
-#include "fdbrpc/AsyncFileWriteChecker.actor.h"
+#include "AsyncFileWriteChecker.h"
+#include "flow/UnitTest.h"
+
+#ifdef __linux__
+namespace {
+Future<Void> runAsyncFileKAIOTestOps(Reference<IAsyncFile> f, int numIterations, int fileSize, bool expectedToSucceed) {
+	void* buf = FastAllocator<4096>::allocate(); // we leak this if there is an error, but that shouldn't be a big deal
+
+	bool opTimedOut = false;
+
+	for (int iteration = 0; iteration < numIterations; ++iteration) {
+		std::vector<Future<Void>> futures;
+		for (int numOps = deterministicRandom()->randomInt(1, 20); numOps > 0; --numOps) {
+			if (deterministicRandom()->coinflip()) {
+				futures.push_back(
+				    success(f->read(buf, 4096, deterministicRandom()->randomInt(0, fileSize) / 4096 * 4096)));
+			} else {
+				futures.push_back(f->write(buf, 4096, deterministicRandom()->randomInt(0, fileSize) / 4096 * 4096));
+			}
+		}
+		for (int fIndex = 0; fIndex < futures.size(); ++fIndex) {
+			try {
+				co_await futures[fIndex];
+			} catch (Error& e) {
+				ASSERT(!expectedToSucceed);
+				ASSERT(e.code() == error_code_io_timeout);
+				opTimedOut = true;
+			}
+		}
+
+		try {
+			co_await (f->sync() && delay(0.1));
+			ASSERT(expectedToSucceed);
+		} catch (Error& e) {
+			ASSERT(!expectedToSucceed && e.code() == error_code_io_timeout);
+		}
+	}
+
+	FastAllocator<4096>::release(buf);
+
+	ASSERT(expectedToSucceed || opTimedOut);
+}
+} // namespace
+
+TEST_CASE("/fdbrpc/AsyncFileKAIO/RequestList") {
+	// This test does nothing in simulation because simulation doesn't support AsyncFileKAIO
+	if (!g_network->isSimulated()) {
+		Reference<IAsyncFile> f;
+		Optional<Error> err;
+		try {
+			f = co_await AsyncFileKAIO::open("/tmp/__KAIO_TEST_FILE__",
+			                                 IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_READWRITE |
+			                                     IAsyncFile::OPEN_CREATE,
+			                                 0666,
+			                                 nullptr);
+			int fileSize = 2 << 27; // ~100MB
+			co_await f->truncate(fileSize);
+
+			// Test that the request list works as intended with default timeout
+			AsyncFileKAIO::setTimeout(0.0);
+			co_await runAsyncFileKAIOTestOps(f, 100, fileSize, true);
+			ASSERT(!((AsyncFileKAIO*)f.getPtr())->failed);
+
+			// Test that the request list works as intended with long timeout
+			AsyncFileKAIO::setTimeout(20.0);
+			co_await runAsyncFileKAIOTestOps(f, 100, fileSize, true);
+			ASSERT(!((AsyncFileKAIO*)f.getPtr())->failed);
+
+			// Test that requests timeout correctly
+			AsyncFileKAIO::setTimeout(0.0001);
+			co_await runAsyncFileKAIOTestOps(f, 10, fileSize, false);
+			ASSERT(((AsyncFileKAIO*)f.getPtr())->failed);
+		} catch (Error& e) {
+			err = e;
+		}
+		if (err.present()) {
+			if (f) {
+				co_await AsyncFileEIO::deleteFile(f->getFilename(), true);
+			}
+			throw err.get();
+		}
+
+		co_await AsyncFileEIO::deleteFile(f->getFilename(), true);
+	}
+}
+#endif // __linux__
 
 // Opens a file for asynchronous I/O
 Future<Reference<class IAsyncFile>> Net2FileSystem::open(const std::string& filename, int64_t flags, int64_t mode) {
@@ -82,15 +165,11 @@ Future<Reference<class IAsyncFile>> Net2FileSystem::open(const std::string& file
 		    mode,
 		    static_cast<boost::asio::io_service*>((void*)g_network->global(INetwork::enASIOService)));
 	if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
-		f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
-	if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES)
-		f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileChaos(r)); });
-	if (flags & IAsyncFile::OPEN_ENCRYPTED)
-		f = map(f, [flags](Reference<IAsyncFile> r) {
-			auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY
-			                                               : AsyncFileEncrypted::Mode::READ_ONLY;
-			return Reference<IAsyncFile>(new AsyncFileEncrypted(r, mode));
+		f = map(f, [=](Reference<IAsyncFile> r) -> Reference<IAsyncFile> {
+			return makeReference<AsyncFileWriteChecker>(r);
 		});
+	if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES)
+		f = map(f, [=](Reference<IAsyncFile> r) -> Reference<IAsyncFile> { return makeReference<AsyncFileChaos>(r); });
 	return f;
 }
 
diff --git a/fdbrpc/Net2FileSystemTests.cpp b/fdbrpc/Net2FileSystemTests.cpp
new file mode 100644
index 00000000000..953d1920e8d
--- /dev/null
+++ b/fdbrpc/Net2FileSystemTests.cpp
@@ -0,0 +1,156 @@
+/*
+ * Net2FileSystemTests.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/IAsyncFile.h"
+#include "flow/IRandom.h"
+#include "flow/Platform.h"
+#include "flow/UnitTest.h"
+
+#include <array>
+#include <cstring>
+#include <memory>
+
+void forceLinkNet2FileSystemTests() {}
+
+const static unsigned int ONE_MEGABYTE = 1 << 20;
+const static unsigned int FOUR_KILOBYTES = 4 << 10;
+
+static Future<Void> writeRangeWithByte(Reference<IAsyncFile> f, int64_t offset, int64_t length, int fixedByte) {
+	int64_t pos = offset;
+	void* data = aligned_alloc(ONE_MEGABYTE, ONE_MEGABYTE);
+	memset(data, fixedByte, ONE_MEGABYTE);
+
+	while (pos < offset + length) {
+		int len = std::min<int64_t>(ONE_MEGABYTE, offset + length - pos);
+		co_await f->write(data, len, pos);
+		pos += len;
+		co_await yield();
+	}
+
+	aligned_free(data);
+}
+
+TEST_CASE("/fileio/zero") {
+	std::string filename = "/tmp/__ZEROJUNK__";
+	Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+	    filename, IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0);
+
+	// Verify that we can grow a file with zero().
+	co_await f->sync();
+	co_await f->zeroRange(0, ONE_MEGABYTE);
+	int64_t size = co_await f->size();
+	ASSERT(ONE_MEGABYTE == size);
+
+	// Verify that zero() does, in fact, zero.
+	co_await writeRangeWithByte(f, 0, ONE_MEGABYTE, 0xff);
+	co_await f->zeroRange(0, ONE_MEGABYTE);
+	uint8_t* page = (uint8_t*)malloc(FOUR_KILOBYTES);
+	int n = co_await f->read(page, FOUR_KILOBYTES, 0);
+	ASSERT(n == FOUR_KILOBYTES);
+	for (int i = 0; i < FOUR_KILOBYTES; i++) {
+		ASSERT(page[i] == 0);
+	}
+	free(page);
+
+	// Destruct our file and remove it.
+	f.clear();
+	co_await IAsyncFileSystem::filesystem()->deleteFile(filename, true);
+}
+
+TEST_CASE("/fileio/incrementalDelete") {
+	// about 5GB
+	int64_t fileSize = 5e9;
+	std::string filename = "/tmp/__JUNK__";
+	Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+	    filename, IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0);
+	co_await f->sync();
+	co_await f->truncate(fileSize);
+	// close the file by deleting the reference
+	f.clear();
+	co_await IAsyncFileSystem::filesystem()->incrementalDeleteFile(filename, true);
+}
+
+TEST_CASE("/fileio/rename") {
+	// create a file
+	int64_t fileSize = 100e6;
+	std::string filename = "/tmp/__JUNK__." + deterministicRandom()->randomUniqueID().toString();
+	std::string renamedFile = "/tmp/__RENAMED_JUNK__." + deterministicRandom()->randomUniqueID().toString();
+	std::unique_ptr<char[]> data(new char[4096]);
+	std::unique_ptr<char[]> readData(new char[4096]);
+	Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+	    filename,
+	    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE,
+	    0644);
+	co_await f->sync();
+	co_await f->truncate(fileSize);
+	memset(data.get(), 0, 4096);
+	// write a random string at the beginning of the file which we can verify after rename
+	for (int i = 0; i < 16; ++i) {
+		data[i] = deterministicRandom()->randomAlphaNumeric();
+	}
+	// write first and block
+	co_await f->write(data.get(), 4096, 0);
+	co_await f->write(data.get(), 4096, fileSize - 4096);
+	co_await f->sync();
+	// close file
+	f.clear();
+	co_await IAsyncFileSystem::filesystem()->renameFile(filename, renamedFile);
+	f = co_await IAsyncFileSystem::filesystem()->open(renamedFile, IAsyncFile::OPEN_READONLY, 0);
+
+	// verify rename happened
+	bool renamedExists = false;
+	auto bName = basename(renamedFile);
+	auto files = platform::listFiles("/tmp/");
+	for (const auto& file : files) {
+		if (file == bName) {
+			renamedExists = true;
+		}
+		ASSERT(file != filename);
+	}
+	ASSERT(renamedExists);
+
+	// verify magic string at beginning of file
+	int length = co_await f->read(readData.get(), 4096, 0);
+	ASSERT(length == 4096);
+	ASSERT(memcmp(readData.get(), data.get(), 4096) == 0);
+	// close the file
+	f.clear();
+
+	// clean up
+	co_await IAsyncFileSystem::filesystem()->deleteFile(renamedFile, true);
+}
+
+// Truncating to extend size should zero the new data
+TEST_CASE("/fileio/truncateAndRead") {
+	std::string filename = "/tmp/__JUNK__";
+	Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+	    filename, IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0);
+	std::array<char, 4096> data;
+	co_await f->sync();
+	co_await f->truncate(4096);
+	int length = co_await f->read(&data[0], 4096, 0);
+	ASSERT(length == 4096);
+	for (auto c : data) {
+		ASSERT(c == '\0');
+	}
+	// close the file by deleting the reference
+	f.clear();
+	co_await IAsyncFileSystem::filesystem()->incrementalDeleteFile(filename, true);
+}
diff --git a/fdbrpc/ReplicationPolicy.cpp b/fdbrpc/ReplicationPolicy.cpp
index ca77f69bbac..59c1a913da2 100644
--- a/fdbrpc/ReplicationPolicy.cpp
+++ b/fdbrpc/ReplicationPolicy.cpp
@@ -38,7 +38,7 @@ bool IReplicationPolicy::validateFull(bool solved,
 	std::vector<LocalityEntry> totalSolution(solutionSet);
 
 	// Append the also servers, if any
-	if (alsoServers.size()) {
+	if (!alsoServers.empty()) {
 		totalSolution.reserve(totalSolution.size() + alsoServers.size());
 		totalSolution.insert(totalSolution.end(), alsoServers.begin(), alsoServers.end());
 	}
@@ -77,7 +77,7 @@ bool PolicyOne::selectReplicas(Reference<LocalitySet>& fromServers,
                                std::vector<LocalityEntry> const& alsoServers,
                                std::vector<LocalityEntry>& results) {
 	int totalUsed = 0;
-	if (alsoServers.size()) {
+	if (!alsoServers.empty()) {
 		totalUsed++;
 	} else if (fromServers->size()) {
 		auto randomEntry = fromServers->random();
@@ -89,7 +89,7 @@ bool PolicyOne::selectReplicas(Reference<LocalitySet>& fromServers,
 
 bool PolicyOne::validate(std::vector<LocalityEntry> const& solutionSet,
                          Reference<LocalitySet> const& fromServers) const {
-	return ((solutionSet.size() > 0) && (fromServers->size() > 0));
+	return ((!solutionSet.empty()) && (fromServers->size() > 0));
 }
 
 PolicyAcross::PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy)
@@ -99,7 +99,7 @@ PolicyAcross::PolicyAcross(int count, std::string const& attribKey, Reference<IR
 
 PolicyAcross::PolicyAcross() : _policy(new PolicyOne()) {}
 
-PolicyAcross::~PolicyAcross() {}
+PolicyAcross::~PolicyAcross() = default;
 
 // Debug purpose only
 // Trace all record entries to help debug
@@ -235,7 +235,7 @@ bool PolicyAcross::selectReplicas(Reference<LocalitySet>& fromServers,
 	}
 
 	// Process the remaining results, if present
-	if ((count < _count) && (_addedResults.size())) {
+	if ((count < _count) && (!_addedResults.empty())) {
 		// Sort the added results array
 		std::sort(_addedResults.begin(), _addedResults.end(), PolicyAcross::compareAddedResults);
 		if (g_replicationdebug > 0) {
@@ -349,21 +349,14 @@ void testPolicySerialization(Reference<IReplicationPolicy>& policy) {
 }
 
 void testReplicationPolicy(int nTests) {
-	Reference<IReplicationPolicy> policy =
-	    Reference<IReplicationPolicy>(new PolicyAcross(1, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
+	Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(1, "data_hall", makeReference<PolicyOne>());
 	testPolicySerialization(policy);
 
-	policy = Reference<IReplicationPolicy>(
-	    new PolicyAnd({ Reference<IReplicationPolicy>(
-	                        new PolicyAcross(2,
-	                                         "data_center",
-	                                         Reference<IReplicationPolicy>(new PolicyAcross(
-	                                             3, "rack", Reference<IReplicationPolicy>(new PolicyOne()))))),
-	                    Reference<IReplicationPolicy>(
-	                        new PolicyAcross(2,
-	                                         "data_center",
-	                                         Reference<IReplicationPolicy>(new PolicyAcross(
-	                                             2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne()))))) }));
+	policy = Reference<IReplicationPolicy>(new PolicyAnd(
+	    { makeReference<PolicyAcross>(
+	          2, "data_center", makeReference<PolicyAcross>(3, "rack", makeReference<PolicyOne>())),
+	      makeReference<PolicyAcross>(
+	          2, "data_center", makeReference<PolicyAcross>(2, "data_hall", makeReference<PolicyOne>())) }));
 
 	testPolicySerialization(policy);
 }
diff --git a/fdbrpc/ReplicationUtils.cpp b/fdbrpc/ReplicationUtils.cpp
index 1f655155a24..409a33e73a2 100644
--- a/fdbrpc/ReplicationUtils.cpp
+++ b/fdbrpc/ReplicationUtils.cpp
@@ -58,10 +58,10 @@ double ratePolicy(Reference<LocalitySet>& localitySet,
 		int largestMode = 0;
 		LocalityEntry largestEntry;
 
-		for (auto& counterItem : counterMap) {
-			if (counterItem.second > largestMode) {
-				largestMode = counterItem.second;
-				largestEntry = counterItem.first;
+		for (const auto& [entry, entryCount] : counterMap) {
+			if (entryCount > largestMode) {
+				largestMode = entryCount;
+				largestEntry = entry;
 			}
 		}
 		rating = (double)largestMode / (double)uniqueResults;
@@ -90,8 +90,8 @@ int mostUsedZoneCount(Reference<LocalitySet>& logServerSet, std::vector<Locality
 		entries[value.get()]++;
 	}
 	int maxEntries = 0;
-	for (auto it : entries) {
-		maxEntries = std::max(maxEntries, it.second);
+	for (const auto& [_zoneId, entryCount] : entries) {
+		maxEntries = std::max(maxEntries, entryCount);
 	}
 	return maxEntries;
 }
@@ -121,15 +121,15 @@ bool findBestPolicySetSimple(int targetUniqueValueCount,
 	ASSERT_WE_THINK(uniqueValueCount == entries.size());
 	std::vector<std::vector<int>> randomizedEntries;
 	randomizedEntries.resize(entries.size());
-	for (auto it : entries) {
-		randomizedEntries.push_back(it.second);
+	for (const auto& [_zoneId, entryIndexes] : entries) {
+		randomizedEntries.push_back(entryIndexes);
 	}
 	deterministicRandom()->randomShuffle(randomizedEntries);
 
 	desired = std::max(desired, targetUniqueValueCount);
 	auto it = randomizedEntries.begin();
 	while (bestSet.size() < desired) {
-		if (it->size()) {
+		if (!it->empty()) {
 			bestSet.push_back(mutableEntries[it->back()]);
 			it->pop_back();
 		}
@@ -238,7 +238,7 @@ bool findBestPolicySet(std::vector<LocalityEntry>& bestResults,
 				break;
 		}
 	} else if (policy->name() == "Across") {
-		PolicyAcross* pa = (PolicyAcross*)policy.getPtr();
+		auto* pa = (PolicyAcross*)policy.getPtr();
 		std::set<std::string> attributeKeys;
 		pa->attributeKeys(&attributeKeys);
 		if (pa->embeddedPolicyName() == "One" && attributeKeys.size() == 1 &&
@@ -375,8 +375,8 @@ bool validateAllCombinations(std::vector<LocalityData>& offendingCombo,
 		bValid = false;
 	} else {
 		bool bIsValidGroup;
-		Reference<LocalitySet> localSet = Reference<LocalitySet>(new LocalityGroup());
-		LocalityGroup* localGroup = (LocalityGroup*)localSet.getPtr();
+		Reference<LocalitySet> localSet = makeReference<LocalityGroup>();
+		auto* localGroup = (LocalityGroup*)localSet.getPtr();
 		localGroup->deep_copy(localitySet);
 
 		std::vector<LocalityEntry> localityGroupEntries = localGroup->getEntries();
@@ -405,7 +405,7 @@ bool validateAllCombinations(std::vector<LocalityData>& offendingCombo,
 			bool result = localSet->selectReplicas(policy, localityGroupEntries, resultEntries);
 			ASSERT(result);
 
-			bIsValidGroup = resultEntries.size() == 0;
+			bIsValidGroup = resultEntries.empty();
 
 			if (((bCheckIfValid) && (!bIsValidGroup)) || ((!bCheckIfValid) && (bIsValidGroup))) {
 				offendingCombo.reserve(nCombinationSize);
@@ -460,7 +460,7 @@ Reference<LocalitySet> createTestLocalityMap(std::vector<repTestType>& indexes,
                                              int independentItems,
                                              int independentTotal) {
 	Reference<LocalitySet> buildServer(new LocalityMap<repTestType>());
-	LocalityMap<repTestType>* serverMap = (LocalityMap<repTestType>*)buildServer.getPtr();
+	auto* serverMap = (LocalityMap<repTestType>*)buildServer.getPtr();
 	int serverValue;
 	std::string dcText, szText, rackText, slotText, independentName, independentText;
 
@@ -552,7 +552,7 @@ bool testPolicy(Reference<LocalitySet> servers,
                 Reference<IReplicationPolicy> const& policy,
                 std::vector<LocalityEntry> const& including,
                 bool validate) {
-	LocalityMap<repTestType>* serverMap = (LocalityMap<repTestType>*)servers.getPtr();
+	auto* serverMap = (LocalityMap<repTestType>*)servers.getPtr();
 	std::string outputText, includeText;
 	std::vector<LocalityEntry> entryResults;
 	std::vector<repTestType*> results;
@@ -592,14 +592,14 @@ bool testPolicy(Reference<LocalitySet> servers,
 	valid = (validate) ? policy->validateFull(solved, entryResults, including, servers) : true;
 
 	if (g_replicationdebug > 0) {
-		if (including.size()) {
+		if (!including.empty()) {
 			includeText = " with ";
 			for (auto& entry : including) {
 				includeText += " " + servers->getEntryInfo(entry);
 			}
 		}
 
-		if (results.size()) {
+		if (!results.empty()) {
 			outputText = policy->info() + includeText + " -> ";
 			int count = 0;
 			for (auto& entry : entryResults) {
@@ -626,153 +626,98 @@ std::vector<Reference<IReplicationPolicy>> const& getStaticPolicies() {
 	if (staticPolicies.empty()) {
 		staticPolicies = {
 
-			Reference<IReplicationPolicy>(new PolicyOne()),
+			makeReference<PolicyOne>(),
 
 			// 1 'dc^2 x 1'
-			Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>(new PolicyOne()))),
+			makeReference<PolicyAcross>(2, "dc", makeReference<PolicyOne>()),
 
 			// 2 'dc^3 x 1'
-			Reference<IReplicationPolicy>(new PolicyAcross(3, "dc", Reference<IReplicationPolicy>(new PolicyOne()))),
+			makeReference<PolicyAcross>(3, "dc", makeReference<PolicyOne>()),
 
 			// 3 'sz^3 x 1'
-			Reference<IReplicationPolicy>(new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
+			makeReference<PolicyAcross>(3, "sz", makeReference<PolicyOne>()),
 
 			// 4 'dc^1 x az^3 x 1'
-			Reference<IReplicationPolicy>(
-			    new PolicyAcross(1,
-			                     "dc",
-			                     Reference<IReplicationPolicy>(
-			                         new PolicyAcross(3, "az", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(1, "dc", makeReference<PolicyAcross>(3, "az", makeReference<PolicyOne>())),
 
 			// 5 '(sz^3 x rack^2 x 1) + (dc^2 x az^3 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(3,
-			                                         "sz",
-			                                         Reference<IReplicationPolicy>(new PolicyAcross(
-			                                             2, "rack", Reference<IReplicationPolicy>(new PolicyOne()))))),
-			                    Reference<IReplicationPolicy>(new PolicyAcross(
-			                        2,
-			                        "dc",
-			                        Reference<IReplicationPolicy>(new PolicyAcross(
-			                            3, "az", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(
+			                        3, "sz", makeReference<PolicyAcross>(2, "rack", makeReference<PolicyOne>())),
+			                    makeReference<PolicyAcross>(
+			                        2, "dc", makeReference<PolicyAcross>(3, "az", makeReference<PolicyOne>())) })),
 
 			// 6 '(sz^1 x 1)'
-			Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
+			makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()),
 
 			// 7 '(sz^1 x 1) + (sz^1 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()) })),
 
 			// 8 '(sz^2 x 1) + (sz^2 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()) })),
 
 			// 9 '(dc^1 x sz^2 x 1)'
-			Reference<IReplicationPolicy>(
-			    new PolicyAcross(1,
-			                     "dc",
-			                     Reference<IReplicationPolicy>(
-			                         new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(1, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())),
 
 			// 10 '(dc^2 x sz^2 x 1)'
-			Reference<IReplicationPolicy>(
-			    new PolicyAcross(2,
-			                     "dc",
-			                     Reference<IReplicationPolicy>(
-			                         new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(2, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())),
 
 			// 11 '(dc^1 x sz^2 x 1) + (dc^2 x sz^2 x 1)'
-			Reference<IReplicationPolicy>(new PolicyAnd(
-			    { Reference<IReplicationPolicy>(
-			          new PolicyAcross(1,
-			                           "dc",
-			                           Reference<IReplicationPolicy>(
-			                               new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
-			      Reference<IReplicationPolicy>(
-			          new PolicyAcross(2,
-			                           "dc",
-			                           Reference<IReplicationPolicy>(new PolicyAcross(
-			                               2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			Reference<IReplicationPolicy>(
+			    new PolicyAnd({ makeReference<PolicyAcross>(
+			                        1, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())),
+			                    makeReference<PolicyAcross>(
+			                        2, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())) })),
 
 			// 12 '(dc^2 x sz^2 x 1) + (dc^1 x sz^2 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2,
-			                                         "dc",
-			                                         Reference<IReplicationPolicy>(new PolicyAcross(
-			                                             2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
-			                    Reference<IReplicationPolicy>(new PolicyAcross(
-			                        1,
-			                        "dc",
-			                        Reference<IReplicationPolicy>(new PolicyAcross(
-			                            2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(
+			                        2, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())),
+			                    makeReference<PolicyAcross>(
+			                        1, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())) })),
 
 			// 13 '(sz^2 x 1) + (dc^1 x sz^2 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(new PolicyAcross(
-			                        1,
-			                        "dc",
-			                        Reference<IReplicationPolicy>(new PolicyAcross(
-			                            2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(
+			                        1, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())) })),
 
 			// 14 '(sz^2 x 1) + (dc^2 x sz^2 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(new PolicyAcross(
-			                        2,
-			                        "dc",
-			                        Reference<IReplicationPolicy>(new PolicyAcross(
-			                            2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(
+			                        2, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())) })),
 
 			// 15 '(sz^3 x 1) + (dc^2 x sz^2 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(new PolicyAcross(
-			                        2,
-			                        "dc",
-			                        Reference<IReplicationPolicy>(new PolicyAcross(
-			                            2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(3, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(
+			                        2, "dc", makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>())) })),
 
 			// 16 '(sz^1 x 1) + (sz^2 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()) })),
 
 			// 17 '(sz^2 x 1) + (sz^3 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(3, "sz", makeReference<PolicyOne>()) })),
 
 			// 18 '(sz^1 x 1) + (sz^2 x 1) + (sz^3 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(2, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(3, "sz", makeReference<PolicyOne>()) })),
 
 			// 19 '(sz^1 x 1) + (machine^1 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(1, "zoneid", makeReference<PolicyOne>()) })),
 
 			// '(dc^1 x 1) + (sz^1 x 1) + (machine^1 x 1)'
 			//	Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "dc",
@@ -781,35 +726,19 @@ std::vector<Reference<IReplicationPolicy>> const& getStaticPolicies() {
 			//"zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
 
 			// 20 '(dc^1 x sz^3 x 1)'
-			Reference<IReplicationPolicy>(
-			    new PolicyAcross(1,
-			                     "dc",
-			                     Reference<IReplicationPolicy>(
-			                         new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(1, "dc", makeReference<PolicyAcross>(3, "sz", makeReference<PolicyOne>())),
 
 			// 21 '(dc^2 x sz^3 x 1)'
-			Reference<IReplicationPolicy>(
-			    new PolicyAcross(2,
-			                     "dc",
-			                     Reference<IReplicationPolicy>(
-			                         new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(2, "dc", makeReference<PolicyAcross>(3, "sz", makeReference<PolicyOne>())),
 
 			// 22 '(dc^2 x az^3 x 1)'
-			Reference<IReplicationPolicy>(
-			    new PolicyAcross(2,
-			                     "dc",
-			                     Reference<IReplicationPolicy>(
-			                         new PolicyAcross(3, "az", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(2, "dc", makeReference<PolicyAcross>(3, "az", makeReference<PolicyOne>())),
 
 			// 23 '(sz^1 x 1) + (dc^2 x az^3 x 1)'
 			Reference<IReplicationPolicy>(
-			    new PolicyAnd({ Reference<IReplicationPolicy>(
-			                        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))),
-			                    Reference<IReplicationPolicy>(new PolicyAcross(
-			                        2,
-			                        "dc",
-			                        Reference<IReplicationPolicy>(new PolicyAcross(
-			                            3, "az", Reference<IReplicationPolicy>(new PolicyOne()))))) })),
+			    new PolicyAnd({ makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()),
+			                    makeReference<PolicyAcross>(
+			                        2, "dc", makeReference<PolicyAcross>(3, "az", makeReference<PolicyOne>())) })),
 
 			// 'dc^1 x (az^2 x 1) + (sz^2 x 1)'
 			//	Reference<IReplicationPolicy>( new PolicyAcross(1, "dc", Reference<IReplicationPolicy>(new
@@ -818,24 +747,16 @@ std::vector<Reference<IReplicationPolicy>> const& getStaticPolicies() {
 			// PolicyOne())))}))) ),
 
 			// 24 Require backtracking
-			Reference<IReplicationPolicy>(new PolicyAcross(
-			    8,
-			    "zoneid",
-			    Reference<IReplicationPolicy>(
-			        new PolicyAcross(1, "az", Reference<IReplicationPolicy>(new PolicyOne()))))),
+			makeReference<PolicyAcross>(8, "zoneid", makeReference<PolicyAcross>(1, "az", makeReference<PolicyOne>())),
 
 			// 25
-			Reference<IReplicationPolicy>(new PolicyAcross(
-			    8,
-			    "zoneid",
-			    Reference<IReplicationPolicy>(
-			        new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne())))))
+			makeReference<PolicyAcross>(8, "zoneid", makeReference<PolicyAcross>(1, "sz", makeReference<PolicyOne>()))
 		};
 	}
 	return staticPolicies;
 }
 
-Reference<IReplicationPolicy> const randomAcrossPolicy(LocalitySet const& serverSet) {
+Reference<IReplicationPolicy> randomAcrossPolicy(LocalitySet const& serverSet) {
 	int usedKeyTotal, keysUsed, keyIndex, valueTotal, maxValueTotal, maxKeyTotal, skips, lastKeyIndex;
 	std::vector<std::string> keyArray(serverSet.getGroupKeyMap()->_lookuparray);
 	std::set<std::string> valueSet;
@@ -896,7 +817,7 @@ Reference<IReplicationPolicy> const randomAcrossPolicy(LocalitySet const& server
 			valueTotal = deterministicRandom()->randomInt(1, valueSet.size() + 2);
 			if ((valueTotal > maxValueTotal) && (deterministicRandom()->random01() > .25))
 				valueTotal = maxValueTotal;
-			policy = Reference<IReplicationPolicy>(new PolicyAcross(valueTotal, keyText, policy));
+			policy = makeReference<PolicyAcross>(valueTotal, keyText, policy);
 			if (g_replicationdebug > 1) {
 				printf("  item%3d: (%3d =>%3d) %-10s  =>%4d\n",
 				       keysUsed + 1,
@@ -1002,7 +923,7 @@ int testReplication() {
 			alsoSize = maxAlsoSize;
 		}
 
-		if ((!alsoSize) && (alsoServers.size() > 0)) {
+		if ((!alsoSize) && (!alsoServers.empty())) {
 			alsoServers.clear();
 		} else {
 			alsoServers = testServers->getEntries();
diff --git a/fdbrpc/SimExternalConnection.actor.cpp b/fdbrpc/SimExternalConnection.cpp
similarity index 74%
rename from fdbrpc/SimExternalConnection.actor.cpp
rename to fdbrpc/SimExternalConnection.cpp
index 7ee4f507e5d..8a88d44c215 100644
--- a/fdbrpc/SimExternalConnection.actor.cpp
+++ b/fdbrpc/SimExternalConnection.cpp
@@ -1,5 +1,5 @@
 /*
- * SimExternalConnection.actor.cpp
+ * SimExternalConnection.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -31,13 +31,14 @@
 #include <boost/range.hpp>
 #include <thread>
 
-#include "fdbrpc/SimExternalConnection.h"
+#include "SimExternalConnection.h"
+#include "flow/Hostname.h"
+#include "flow/IConnection.h"
 #include "flow/Net2Packet.h"
 #include "flow/Platform.h"
 #include "flow/SendBufferIterator.h"
 #include "flow/UnitTest.h"
-#include "flow/IConnection.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/network.h"
 
 using namespace boost::asio;
 
@@ -45,16 +46,15 @@ static io_service ios;
 
 class SimExternalConnectionImpl {
 public:
-	ACTOR static Future<Void> onReadable(SimExternalConnection* self) {
-		wait(delayJittered(0.1));
+	static Future<Void> onReadable(SimExternalConnection* self) {
+		co_await delayJittered(0.1);
 		if (self->readBuffer.empty()) {
-			wait(self->onReadableTrigger.onTrigger());
+			co_await self->onReadableTrigger.onTrigger();
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Reference<IConnection>> connect(NetworkAddress toAddr) {
-		wait(delayJittered(0.1));
+	static Future<Reference<IConnection>> connect(NetworkAddress toAddr) {
+		co_await delayJittered(0.1);
 		ip::tcp::socket socket(ios);
 		auto ip = toAddr.ip;
 		ip::address address;
@@ -66,9 +66,9 @@ class SimExternalConnectionImpl {
 		boost::system::error_code err;
 		socket.connect(ip::tcp::endpoint(address, toAddr.port), err);
 		if (err) {
-			return Reference<IConnection>();
+			co_return Reference<IConnection>();
 		} else {
-			return Reference<IConnection>(new SimExternalConnection(std::move(socket)));
+			co_return Reference<IConnection>(makeReference<SimExternalConnection>(std::move(socket)));
 		}
 	}
 };
@@ -166,16 +166,15 @@ std::vector<NetworkAddress> SimExternalConnection::resolveTCPEndpointBlocking(co
 		dnsCache->add(host, service, addrs);
 		return addrs;
 	} catch (...) {
-		dnsCache->remove(host, service);
 		throw lookup_failed();
 	}
 }
 
-ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host,
-                                                                        std::string service,
-                                                                        DNSCache* dnsCache) {
-	wait(delayJittered(0.1));
-	return SimExternalConnection::resolveTCPEndpointBlocking(host, service, dnsCache);
+static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host,
+                                                                  std::string service,
+                                                                  DNSCache* dnsCache) {
+	co_await delayJittered(0.1);
+	co_return SimExternalConnection::resolveTCPEndpointBlocking(host, service, dnsCache);
 }
 
 Future<std::vector<NetworkAddress>> SimExternalConnection::resolveTCPEndpoint(const std::string& host,
@@ -199,7 +198,7 @@ static void testEchoServer() {
 	ip::tcp::acceptor acceptor(ios, ip::tcp::endpoint(ip::tcp::v4(), testEchoServerPort));
 	ip::tcp::socket socket(ios);
 	acceptor.accept(socket);
-	loop {
+	while (true) {
 		char readBuffer[readBufferSize];
 		boost::system::error_code err;
 		auto length = socket.read_some(mutable_buffers_1(readBuffer, readBufferSize), err);
@@ -212,56 +211,74 @@ static void testEchoServer() {
 }
 
 TEST_CASE("fdbrpc/SimExternalClient") {
-	state const size_t maxDataLength = 10000;
-	state std::thread serverThread([] { return testEchoServer(); });
-	state UnsentPacketQueue packetQueue;
-	state Reference<IConnection> externalConn;
-	loop {
-		Reference<IConnection> _externalConn =
-		    wait(INetworkConnections::net()->connect("localhost", std::to_string(testEchoServerPort)));
-		if (_externalConn.isValid()) {
-			externalConn = std::move(_externalConn);
+	const size_t maxDataLength = 10000;
+	std::thread serverThread([] { return testEchoServer(); });
+	UnsentPacketQueue packetQueue;
+	Reference<IConnection> externalConn;
+	while (true) {
+		Reference<IConnection> connected =
+		    co_await INetworkConnections::net()->connect("localhost", std::to_string(testEchoServerPort));
+		if (connected.isValid()) {
+			externalConn = std::move(connected);
 			break;
 		}
 		// Wait until server is ready
 		threadSleep(0.01);
 	}
-	state Standalone<StringRef> data(
+	Standalone<StringRef> data(
 	    deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, maxDataLength + 1)));
 	PacketWriter packetWriter(packetQueue.getWriteBuffer(data.size()), nullptr, Unversioned());
 	packetWriter.serializeBytes(data);
-	wait(externalConn->onWritable());
+	co_await externalConn->onWritable();
 	externalConn->write(packetQueue.getUnsent());
-	wait(externalConn->onReadable());
+	co_await externalConn->onReadable();
 	std::vector<uint8_t> vec(data.size());
-	externalConn->read(&vec[0], &vec[0] + vec.size());
+	if (!vec.empty()) {
+		externalConn->read(vec.data(), vec.data() + vec.size());
+	}
 	externalConn->close();
-	StringRef echo(&vec[0], vec.size());
+	StringRef echo(vec.empty() ? reinterpret_cast<const uint8_t*>("") : vec.data(), vec.size());
 	ASSERT(echo.toString() == data.toString());
 	serverThread.join();
-	return Void();
 }
 
 TEST_CASE("fdbrpc/MockDNS") {
-	state std::vector<NetworkAddress> networkAddresses;
-	state NetworkAddress address1(IPAddress(0x13131313), 1);
+	std::vector<NetworkAddress> networkAddresses;
+	NetworkAddress address1(IPAddress(0x13131313), 1);
 	networkAddresses.push_back(address1);
 	INetworkConnections::net()->addMockTCPEndpoint("testhost1", "port1", networkAddresses);
-	state std::vector<NetworkAddress> resolvedNetworkAddresses =
-	    wait(INetworkConnections::net()->resolveTCPEndpoint("testhost1", "port1"));
+	std::vector<NetworkAddress> resolvedNetworkAddresses =
+	    co_await INetworkConnections::net()->resolveTCPEndpoint("testhost1", "port1");
 	ASSERT(resolvedNetworkAddresses.size() == 1);
 	ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address1) !=
 	       resolvedNetworkAddresses.end());
 	INetworkConnections::net()->removeMockTCPEndpoint("testhost1", "port1");
-	state NetworkAddress address2(IPAddress(0x14141414), 2);
+	NetworkAddress address2(IPAddress(0x14141414), 2);
 	networkAddresses.push_back(address2);
 	INetworkConnections::net()->addMockTCPEndpoint("testhost1", "port1", networkAddresses);
-	wait(store(resolvedNetworkAddresses, INetworkConnections::net()->resolveTCPEndpoint("testhost1", "port1")));
+	resolvedNetworkAddresses = co_await INetworkConnections::net()->resolveTCPEndpoint("testhost1", "port1");
 	ASSERT(resolvedNetworkAddresses.size() == 2);
 	ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address2) !=
 	       resolvedNetworkAddresses.end());
+}
 
-	return Void();
+TEST_CASE("/fdbrpc/Hostname/hostname") {
+	if (!g_network->isSimulated()) {
+		co_return;
+	}
+
+	Hostname hostname = Hostname::parse("host-name:1234");
+	NetworkAddress addressSource = NetworkAddress::parse("127.0.0.0:1234");
+	INetworkConnections::net()->addMockTCPEndpoint(hostname.host, hostname.service, { addressSource });
+
+	Optional<NetworkAddress> optionalAddress = co_await hostname.resolve();
+	ASSERT(optionalAddress.present() && optionalAddress.get() == addressSource);
+
+	optionalAddress = hostname.resolveBlocking();
+	ASSERT(optionalAddress.present() && optionalAddress.get() == addressSource);
+
+	NetworkAddress address = co_await hostname.resolveWithRetry();
+	ASSERT(address == addressSource);
 }
 
 void forceLinkSimExternalConnectionTests() {}
diff --git a/fdbrpc/include/fdbrpc/SimExternalConnection.h b/fdbrpc/SimExternalConnection.h
similarity index 94%
rename from fdbrpc/include/fdbrpc/SimExternalConnection.h
rename to fdbrpc/SimExternalConnection.h
index ebb0b527444..7943886378a 100644
--- a/fdbrpc/include/fdbrpc/SimExternalConnection.h
+++ b/fdbrpc/SimExternalConnection.h
@@ -31,11 +31,13 @@
 
 class SimExternalConnection final : public IConnection, public ReferenceCounted<SimExternalConnection> {
 	boost::asio::ip::tcp::socket socket;
-	SimExternalConnection(boost::asio::ip::tcp::socket&& socket);
+	explicit SimExternalConnection(boost::asio::ip::tcp::socket&& socket);
 	UID dbgid;
 	std::deque<uint8_t> readBuffer;
 	AsyncTrigger onReadableTrigger;
 	friend class SimExternalConnectionImpl;
+	template <class P, class... Args>
+	friend Reference<P> makeReference(Args&&... args);
 
 public:
 	void addref() override { return ReferenceCounted<SimExternalConnection>::addref(); }
diff --git a/fdbrpc/Stats.actor.cpp b/fdbrpc/Stats.cpp
similarity index 92%
rename from fdbrpc/Stats.actor.cpp
rename to fdbrpc/Stats.cpp
index e27abd4cad3..8ea1ea558d8 100644
--- a/fdbrpc/Stats.actor.cpp
+++ b/fdbrpc/Stats.cpp
@@ -1,5 +1,5 @@
 /*
- * Stats.actor.cpp
+ * Stats.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,12 +22,11 @@
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
 #include "flow/OTELMetrics.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/Trace.h"
 #include "flow/flow.h"
 #include "flow/network.h"
 #include <string>
-#include "flow/actorcompiler.h" // has to be last include
 
 Counter::Counter(std::string const& name, CounterCollection& collection, bool skipTraceOnSilentInterval)
   : name(name), interval_start(0), last_event(0), interval_sq_time(0), roughness_interval_start(0), interval_delta(0),
@@ -107,12 +106,14 @@ void CounterCollection::logToTraceEvent(TraceEvent& te) {
 				metrics->sumMap[c->id].points.back().addAttribute("ip", ip_str);
 				metrics->sumMap[c->id].points.back().addAttribute("port", port_str);
 				metrics->sumMap[c->id].points.back().startTime = logTime;
+				break;
 			}
 			case MetricsDataModel::STATSD: {
 				std::vector<std::pair<std::string, std::string>> statsd_attributes{ { "ip", ip_str },
 					                                                                { "port", port_str } };
 				metrics->statsd_message.push_back(createStatsdMessage(
 				    c->getName(), StatsDMetric::COUNTER, std::to_string(val) /*, statsd_attributes*/));
+				break;
 			}
 			case MetricsDataModel::NONE:
 			default: {
@@ -128,25 +129,25 @@ void CounterCollection::logToTraceEvent(TraceEvent& te) {
 
 class CounterCollectionImpl {
 public:
-	ACTOR static Future<Void> traceCounters(CounterCollection* counters,
-	                                        std::string traceEventName,
-	                                        UID traceEventID,
-	                                        double interval,
-	                                        std::string trackLatestName,
-	                                        std::function<void(TraceEvent&)> decorator) {
-		wait(delay(0)); // Give an opportunity for all members used in special counters to be initialized
+	static Future<Void> traceCounters(CounterCollection* counters,
+	                                  std::string traceEventName,
+	                                  UID traceEventID,
+	                                  double interval,
+	                                  std::string trackLatestName,
+	                                  std::function<void(TraceEvent&)> decorator) {
+		co_await delay(0); // Give an opportunity for all members used in special counters to be initialized
 
 		for (ICounter* c : counters->counters)
 			c->resetInterval();
 
-		state Reference<EventCacheHolder> traceEventHolder;
+		Reference<EventCacheHolder> traceEventHolder;
 		if (!trackLatestName.empty()) {
 			traceEventHolder = makeReference<EventCacheHolder>(trackLatestName);
 		}
 
-		state double last_interval = now();
+		double last_interval = now();
 
-		loop {
+		while (true) {
 			TraceEvent te(traceEventName.c_str(), traceEventID);
 			te.detail("Elapsed", now() - last_interval);
 
@@ -158,7 +159,7 @@ class CounterCollectionImpl {
 			}
 
 			last_interval = now();
-			wait(delay(interval, TaskPriority::FlushTrace));
+			co_await delay(interval, TaskPriority::FlushTrace);
 		}
 	}
 };
@@ -183,8 +184,8 @@ LatencyBands::LatencyBands(std::string const& name,
   : name(name), id(id), loggingInterval(loggingInterval), decorator(decorator) {}
 
 void LatencyBands::addThreshold(double value) {
-	if (value > 0 && bands.count(value) == 0) {
-		if (bands.size() == 0) {
+	if (value > 0 && !bands.contains(value)) {
+		if (bands.empty()) {
 			ASSERT(!cc && !filteredCount);
 			cc = std::make_unique<CounterCollection>(name, id.toString());
 			logger = cc->traceCounters(name, id, loggingInterval, id.toString() + "/" + name, decorator);
@@ -199,7 +200,7 @@ void LatencyBands::addThreshold(double value) {
 void LatencyBands::addMeasurement(double measurement, int count, Filtered filtered) {
 	if (filtered && filteredCount) {
 		(*filteredCount) += count;
-	} else if (bands.size() > 0) {
+	} else if (!bands.empty()) {
 		auto itr = bands.upper_bound(measurement);
 		ASSERT(itr != bands.end());
 		(*itr->second) += count;
@@ -289,6 +290,7 @@ void LatencySample::logSample() {
 			createOtelGauge(p95id, name + "p95", p95);
 			createOtelGauge(p99id, name + "p99", p99);
 			createOtelGauge(p999id, name + "p99_9", p99_9);
+			break;
 		}
 		case MetricsDataModel::STATSD: {
 			std::vector<std::pair<std::string, std::string>> statsd_attributes{ { "ip", ip_str },
@@ -303,6 +305,7 @@ void LatencySample::logSample() {
 			    createStatsdMessage(name + "p99", StatsDMetric::GAUGE, std::to_string(p99) /*, statsd_attributes*/);
 			auto p999_gauge =
 			    createStatsdMessage(name + "p99.9", StatsDMetric::GAUGE, std::to_string(p99_9) /*, statsd_attributes*/);
+			break;
 		}
 		case MetricsDataModel::NONE:
 		default: {
diff --git a/flowbench/BenchAsyncFileWriteCheckerLRU.cpp b/fdbrpc/bench/BenchAsyncFileWriteCheckerLRU.cpp
similarity index 86%
rename from flowbench/BenchAsyncFileWriteCheckerLRU.cpp
rename to fdbrpc/bench/BenchAsyncFileWriteCheckerLRU.cpp
index 213921397c7..d8d7c9d2671 100644
--- a/flowbench/BenchAsyncFileWriteCheckerLRU.cpp
+++ b/fdbrpc/bench/BenchAsyncFileWriteCheckerLRU.cpp
@@ -1,9 +1,6 @@
 #include "benchmark/benchmark.h"
-#include "crc32/crc32c.h"
-#include "flow/Hash3.h"
-#include "flow/xxhash.h"
-#include "flowbench/GlobalData.h"
-#include "fdbrpc/AsyncFileWriteChecker.actor.h"
+#include "AsyncFileWriteChecker.h"
+#include "flow/IRandom.h"
 
 #include <stdint.h>
 
@@ -44,4 +41,4 @@ static void lru_test(benchmark::State& state) {
 	}
 }
 
-BENCHMARK(lru_test);
\ No newline at end of file
+BENCHMARK(lru_test);
diff --git a/flowbench/BenchIONet2.actor.cpp b/fdbrpc/bench/BenchIONet2.actor.cpp
similarity index 98%
rename from flowbench/BenchIONet2.actor.cpp
rename to fdbrpc/bench/BenchIONet2.actor.cpp
index d2b6f8dba83..2a81890d98d 100644
--- a/flowbench/BenchIONet2.actor.cpp
+++ b/fdbrpc/bench/BenchIONet2.actor.cpp
@@ -49,7 +49,7 @@ static inline TaskPriority getRandomTaskPriority(DeterministicRandom& rand) {
 ACTOR static Future<Void> benchIONet2Actor(benchmark::State* benchState) {
 	state size_t actorCount = benchState->range(0);
 	state uint32_t sum;
-	state int seed = platform::getRandomSeed();
+	state uint64_t seed = platform::getRandomSeed();
 	state std::unique_ptr<char[]> data(new char[4096]);
 	memset(data.get(), 0, 4096);
 	while (benchState->KeepRunning()) {
diff --git a/fdbrpc/bench/BenchMain.cpp b/fdbrpc/bench/BenchMain.cpp
new file mode 100644
index 00000000000..174c2b249f3
--- /dev/null
+++ b/fdbrpc/bench/BenchMain.cpp
@@ -0,0 +1,35 @@
+/*
+ * BenchMain.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbrpc/Net2FileSystem.h"
+#include "flow/BenchMain.h"
+
+namespace {
+
+void initializeNet2FileSystem() {
+	g_network->addStopCallback(Net2FileSystem::stop);
+	Net2FileSystem::newFileSystem();
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+	return runBenchmarks(argc, argv, initializeNet2FileSystem);
+}
diff --git a/flowbench/BenchSamples.cpp b/fdbrpc/bench/BenchSamples.cpp
similarity index 98%
rename from flowbench/BenchSamples.cpp
rename to fdbrpc/bench/BenchSamples.cpp
index 1a76fc7cbef..cdf321378d7 100644
--- a/flowbench/BenchSamples.cpp
+++ b/fdbrpc/bench/BenchSamples.cpp
@@ -19,11 +19,12 @@
  */
 
 #include "benchmark/benchmark.h"
+#include "BenchSupport.h"
+
 #include "flow/IRandom.h"
-#include "flowbench/GlobalData.h"
 #include "fdbrpc/Stats.h"
 #include "fdbrpc/DDSketch.h"
-#include "fdbrpc/ContinuousSample.h"
+#include "ContinuousSample.h"
 #include "flow/Histogram.h"
 
 static void bench_ddsketchUnsigned(benchmark::State& state) {
diff --git a/flowbench/BenchSelectReplicas.cpp b/fdbrpc/bench/BenchSelectReplicas.cpp
similarity index 91%
rename from flowbench/BenchSelectReplicas.cpp
rename to fdbrpc/bench/BenchSelectReplicas.cpp
index e183d3d2f0f..e5afb61924c 100644
--- a/flowbench/BenchSelectReplicas.cpp
+++ b/fdbrpc/bench/BenchSelectReplicas.cpp
@@ -27,8 +27,7 @@
 
 static void bench_select_replicas(int repCount, benchmark::State& state) {
 
-	Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
-	    new PolicyAcross(repCount, "rack", Reference<IReplicationPolicy>(new PolicyOne())));
+	Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(repCount, "rack", makeReference<PolicyOne>());
 
 	// Pre-warm the depth cache to avoid measuring lazy initialization overhead
 	policy->depth();
@@ -47,7 +46,7 @@ static void bench_select_replicas(int repCount, benchmark::State& state) {
 	    createTestLocalityMap(indexes, dcTotal, szTotal, rackTotal, slotTotal, independentItems, independentTotal);
 	LocalityGroup* fromServersGroup = (LocalityGroup*)fromServersSet.getPtr();
 
-	const Reference<LocalitySet> alreadyServersSet = Reference<LocalitySet>(new LocalityGroup());
+	const Reference<LocalitySet> alreadyServersSet = makeReference<LocalityGroup>();
 
 	alreadyServersSet->deep_copy(*fromServersGroup);
 	std::vector<LocalityEntry> localityGroupEntries;
diff --git a/contrib/mockkms/utils.go b/fdbrpc/bench/BenchSupport.h
similarity index 57%
rename from contrib/mockkms/utils.go
rename to fdbrpc/bench/BenchSupport.h
index e811c91658e..d75c639f362 100644
--- a/contrib/mockkms/utils.go
+++ b/fdbrpc/bench/BenchSupport.h
@@ -1,5 +1,5 @@
 /*
- * utils.go
+ * BenchSupport.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,34 +18,37 @@
  * limitations under the License.
  */
 
-package main
+#ifndef FDB_FDBRPC_BENCH_SUPPORT_H
+#define FDB_FDBRPC_BENCH_SUPPORT_H
 
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-)
+#pragma once
 
-type ErrorDetail struct {
-	Detail string `json:"details"`
-}
+#include <vector>
 
-type ErrorResponse struct {
-	Err ErrorDetail `json:"error"`
-}
+#include "flow/flow.h"
 
-func sendErrorResponse(w http.ResponseWriter, err error) {
-	e := ErrorDetail{}
-	e.Detail = fmt.Sprintf("Error: %s", err.Error())
-	resp := ErrorResponse{
-		Err: e,
+template <class T>
+struct InputGenerator {
+	InputGenerator() = default;
+
+	template <class Fn>
+	InputGenerator(int n, Fn genFunc) : lastIndex(-1) {
+		ASSERT(n > 0);
+		data.reserve(n);
+		for (int i = 0; i < n; ++i) {
+			data.push_back(genFunc());
+		}
 	}
 
-	mResp, err := json.Marshal(resp)
-	if err != nil {
-		log.Printf("Error marshalling error response %s", err.Error())
-		panic(err)
+	const T& next() {
+		if (++lastIndex == data.size()) {
+			lastIndex = 0;
+		}
+		return data[lastIndex];
 	}
-	fmt.Fprintf(w, string(mResp))
-}
+
+	std::vector<T> data;
+	int lastIndex;
+};
+
+#endif
diff --git a/fdbrpc/bench/CMakeLists.txt b/fdbrpc/bench/CMakeLists.txt
new file mode 100644
index 00000000000..ce58923bbed
--- /dev/null
+++ b/fdbrpc/bench/CMakeLists.txt
@@ -0,0 +1,14 @@
+include(FDBBenchmark)
+
+fdb_find_sources(FDBRPC_BENCH_SRCS)
+add_flow_target(EXECUTABLE NAME fdbrpc_bench SRCS ${FDBRPC_BENCH_SRCS})
+
+fdb_setup_googlebenchmark()
+
+target_include_directories(
+  fdbrpc_bench
+  PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}"
+          "${CMAKE_CURRENT_SOURCE_DIR}/.."
+          "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+target_link_libraries(fdbrpc_bench PRIVATE Threads::Threads fdb_google_benchmark flow fdbrpc)
diff --git a/fdbrpc/dsltest.actor.cpp b/fdbrpc/dsltest.actor.cpp
index 224039cc292..dc53149cf53 100644
--- a/fdbrpc/dsltest.actor.cpp
+++ b/fdbrpc/dsltest.actor.cpp
@@ -28,6 +28,10 @@
 #include "flow/ThreadHelper.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+//
+// TODO: IS THIS FILE NEEDED?  IS IT RUN ON A RECURRING BASIS?  IF NOT, DELETE IT.
+//
+
 void* allocateLargePages(int total);
 
 bool testFuzzActor(Future<int> (*actor)(FutureStream<int> const&, PromiseStream<int> const&, Future<Void> const&),
@@ -296,7 +300,7 @@ class TestBuffer : public ReferenceCounted<TestBuffer> {
 	const uint8_t* end() const { return data + length; }
 
 private:
-	TestBuffer(int length) noexcept : length(length) {}
+	explicit TestBuffer(int length) noexcept : length(length) {}
 	int length;
 	uint8_t data[1];
 };
@@ -630,7 +634,7 @@ void showArena(ArenaBlock* a, ArenaBlock* parent) {
 	if (!a->isTiny()) {
 		int o = a->nextBlockOffset;
 		while (o) {
-			ArenaBlockRef* r = (ArenaBlockRef*)((char*)a->getData() + o);
+			auto* r = (ArenaBlockRef*)((char*)a->getData() + o);
 
 			// If alignedBuffer is valid then print its pointer and size, else recurse
 			if (r->aligned4kBufferSize != 0) {
@@ -978,8 +982,8 @@ ACTOR [[flow_allow_discard]] Future<int> introFirst(Future<int> a, Future<int> b
 
 struct AddReply {
 	int sum;
-	AddReply() {}
-	AddReply(int x) : sum(x) {}
+	AddReply() = default;
+	explicit(false) AddReply(int x) : sum(x) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -991,7 +995,7 @@ struct AddRequest {
 	int a, b;
 	Promise<AddReply> reply; // Self-addressed envelope
 
-	AddRequest() {}
+	AddRequest() = default;
 	AddRequest(int a, int b) : a(a), b(b) {}
 
 	template <class Ar>
@@ -1188,8 +1192,6 @@ void asyncMapTest() {
 	printf("  onChange(present, set): %0.1fM/sec\n", 1.0 / (timer() - startt));
 }
 
-extern void net2_test();
-
 void dsltest() {
 	double startt, endt;
 
@@ -1197,9 +1199,6 @@ void dsltest() {
 
 	asyncMapTest();
 
-	net2_test();
-	// sleeptest();
-
 	Future<Void> ctf = cycleTime(1000, 1000);
 	ctf.get();
 
diff --git a/fdbrpc/include/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/include/fdbrpc/AsyncFileCached.h
similarity index 88%
rename from fdbrpc/include/fdbrpc/AsyncFileCached.actor.h
rename to fdbrpc/include/fdbrpc/AsyncFileCached.h
index b355ecb2a20..383151c5508 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileCached.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileCached.actor.h
+ * AsyncFileCached.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,23 +20,14 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_ASYNCFILECACHED_ACTOR_G_H)
-#define FLOW_ASYNCFILECACHED_ACTOR_G_H
-#include "fdbrpc/AsyncFileCached.actor.g.h"
-#elif !defined(FLOW_ASYNCFILECACHED_ACTOR_H)
-#define FLOW_ASYNCFILECACHED_ACTOR_H
-
 #include <boost/intrusive/list.hpp>
 #include <type_traits>
 
 #include "flow/flow.h"
 #include "flow/IAsyncFile.h"
 #include "flow/Knobs.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/network.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace bi = boost::intrusive;
 struct EvictablePage {
@@ -48,7 +39,7 @@ struct EvictablePage {
 	virtual bool evict() = 0; // true if page was evicted, false if it isn't immediately evictable (but will be evicted
 	                          // regardless if possible)
 
-	EvictablePage(Reference<EvictablePageCache> pageCache) : data(0), index(-1), pageCache(pageCache) {}
+	explicit EvictablePage(Reference<EvictablePageCache> pageCache) : data(0), index(-1), pageCache(pageCache) {}
 	virtual ~EvictablePage();
 };
 
@@ -138,6 +129,8 @@ struct AFCPage;
 
 class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFileCached> {
 	friend struct AFCPage;
+	template <class P, class... Args>
+	friend Reference<P> makeReference(Args&&... args);
 
 public:
 	virtual StringRef getClassName() override { return "AsyncFileCached"_sr; }
@@ -179,11 +172,13 @@ class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFi
 		return tag(f, length);
 	}
 
-	ACTOR static Future<Void> write_impl(AsyncFileCached* self, void const* data, int length, int64_t offset) {
+	static Future<Void> write_impl(AsyncFileCached* self, void const* data, int length, int64_t offset) {
 		// If there is a truncate in progress before the the write position then we must
 		// wait for it to complete.
-		if (length + offset > self->currentTruncateSize)
-			wait(self->currentTruncate);
+		if (length + offset > self->currentTruncateSize) {
+			Future<Void> currentTruncate = self->currentTruncate;
+			co_await currentTruncate;
+		}
 		++self->countFileCacheWrites;
 		++self->countCacheWrites;
 		Future<Void> f = read_write_impl<true>(self, static_cast<const uint8_t*>(data), length, offset);
@@ -191,8 +186,7 @@ class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFi
 			++self->countFileCacheWritesBlocked;
 			++self->countCacheWritesBlocked;
 		}
-		wait(f);
-		return Void();
+		co_await f;
 	}
 
 	Future<Void> write(void const* data, int length, int64_t offset) override {
@@ -210,12 +204,13 @@ class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFi
 
 	// This wrapper for the actual truncation operation enforces ordering of truncates.
 	// It maintains currentTruncate and currentTruncateSize so writers can wait behind truncates that would affect them.
-	ACTOR static Future<Void> truncate_impl(AsyncFileCached* self, int64_t size) {
-		wait(self->currentTruncate);
+	static Future<Void> truncate_impl(AsyncFileCached* self, int64_t size) {
+		Future<Void> previousTruncate = self->currentTruncate;
+		co_await previousTruncate;
 		self->currentTruncateSize = size;
 		self->currentTruncate = self->changeFileSize(size);
-		wait(self->currentTruncate);
-		return Void();
+		Future<Void> currentTruncate = self->currentTruncate;
+		co_await currentTruncate;
 	}
 
 	Future<Void> sync() override { return waitAndSync(this, flush()); }
@@ -330,22 +325,23 @@ class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFi
 	static Future<Reference<IAsyncFile>> open_impl(std::string filename, int flags, int mode);
 
 	// Opens a file that uses the FDB in-memory page cache
-	ACTOR static Future<Reference<IAsyncFile>> open_impl(std::string filename,
-	                                                     int flags,
-	                                                     int mode,
-	                                                     Reference<EvictablePageCache> pageCache) {
+	static Future<Reference<IAsyncFile>> open_impl(std::string filename,
+	                                               int flags,
+	                                               int mode,
+	                                               Reference<EvictablePageCache> pageCache) {
 		try {
 			TraceEvent("AFCUnderlyingOpenBegin").detail("Filename", filename);
 			if (flags & IAsyncFile::OPEN_CACHED_READ_ONLY)
 				flags = (flags & ~IAsyncFile::OPEN_READWRITE) | IAsyncFile::OPEN_READONLY;
 			else
 				flags = (flags & ~IAsyncFile::OPEN_READONLY) | IAsyncFile::OPEN_READWRITE;
-			state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-			    filename, flags | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, mode));
+			Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+			    filename, flags | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, mode);
 			TraceEvent("AFCUnderlyingOpenEnd").detail("Filename", filename);
-			int64_t l = wait(f->size());
+			int64_t l = co_await f->size();
 			TraceEvent("AFCUnderlyingSize").detail("Filename", filename).detail("Size", l);
-			return Reference<AsyncFileCached>(new AsyncFileCached(f, filename, l, pageCache)).castTo<IAsyncFile>();
+			auto cachedFile = makeReference<AsyncFileCached>(f, filename, l, pageCache);
+			co_return Reference<IAsyncFile>(cachedFile);
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled)
 				openFiles.erase(filename);
@@ -357,10 +353,9 @@ class AsyncFileCached final : public IAsyncFile, public ReferenceCounted<AsyncFi
 
 	Future<Void> quiesce();
 
-	ACTOR static Future<Void> waitAndSync(AsyncFileCached* self, Future<Void> flush) {
-		wait(flush);
-		wait(self->uncached->sync());
-		return Void();
+	static Future<Void> waitAndSync(AsyncFileCached* self, Future<Void> flush) {
+		co_await flush;
+		co_await self->uncached->sync();
 	}
 
 	template <bool writing>
@@ -431,10 +426,10 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 		return notReading;
 	}
 
-	ACTOR static Future<Void> waitAndWrite(AFCPage* self, void const* data, int length, int offset) {
-		wait(self->notReading);
+	static Future<Void> waitAndWrite(AFCPage* self, void const* data, int length, int offset) {
+		Future<Void> notReading = self->notReading;
+		co_await notReading;
 		memcpy(static_cast<uint8_t*>(self->data) + offset, data, length);
-		return Void();
 	}
 
 	Future<Void> readZeroCopy() {
@@ -487,18 +482,18 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 		return notReading;
 	}
 
-	ACTOR static Future<Void> waitAndRead(AFCPage* self, void* data, int length, int offset) {
-		wait(self->notReading);
+	static Future<Void> waitAndRead(AFCPage* self, void* data, int length, int offset) {
+		Future<Void> notReading = self->notReading;
+		co_await notReading;
 		memcpy(data, static_cast<uint8_t const*>(self->data) + offset, length);
-		return Void();
 	}
 
-	ACTOR static Future<Void> readThrough(AFCPage* self) {
+	static Future<Void> readThrough(AFCPage* self) {
 		ASSERT(!self->valid);
-		state void* dst = self->data;
+		void* dst = self->data;
 		if (self->pageOffset < self->owner->prevLength) {
 			try {
-				int _ = wait(self->owner->uncached->read(dst, self->pageCache->pageSize, self->pageOffset));
+				int _ = co_await self->owner->uncached->read(dst, self->pageCache->pageSize, self->pageOffset);
 				if (_ != self->pageCache->pageSize)
 					TraceEvent("ReadThroughShortRead")
 					    .detail("ReadAmount", _)
@@ -513,18 +508,18 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 		// If the memory we read into wasn't orphaned while we were waiting on the read then set valid to true
 		if (dst == self->data)
 			self->valid = true;
-		return Void();
 	}
 
-	ACTOR static Future<Void> writeThrough(AFCPage* self, Promise<Void> writing) {
+	static Future<Void> writeThrough(AFCPage* self, Promise<Void> writing) {
 		// writeThrough can be called on a page that is not dirty, just to wait for a previous writeThrough to finish.
 		// In that case we don't want to do any disk I/O
 		try {
-			state bool dirty = self->dirty;
+			bool dirty = self->dirty;
 			++self->writeThroughCount;
 			self->updateFlushableIndex();
 
-			wait(self->notReading && self->notFlushing);
+			Future<Void> readyToWrite = self->notReading && self->notFlushing;
+			co_await readyToWrite;
 
 			if (dirty) {
 				// Wait for rate control if it is set
@@ -536,7 +531,7 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 						            FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_IO_SIZE; // round up
 						ASSERT(allowance > 0);
 					}
-					wait(self->owner->getRateControl()->getAllowance(allowance));
+					co_await self->owner->getRateControl()->getAllowance(allowance);
 				}
 
 				if (self->pageOffset + self->pageCache->pageSize > self->owner->length) {
@@ -548,7 +543,7 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 
 				auto f = self->owner->uncached->write(self->data, self->pageCache->pageSize, self->pageOffset);
 
-				wait(f);
+				co_await f;
 			}
 		} catch (Error& e) {
 			--self->writeThroughCount;
@@ -563,8 +558,6 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 		                      // overlapping write and sync operations
 
 		self->pageCache->try_evict();
-
-		return Void();
 	}
 
 	Future<Void> flush() {
@@ -607,10 +600,10 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 		return truncate_impl(this);
 	}
 
-	ACTOR static Future<Void> truncate_impl(AFCPage* self) {
-		wait(self->notReading && self->notFlushing && yield());
+	static Future<Void> truncate_impl(AFCPage* self) {
+		Future<Void> readyToTruncate = self->notReading && self->notFlushing && yield();
+		co_await readyToTruncate;
 		delete self;
-		return Void();
 	}
 
 	AFCPage(AsyncFileCached* owner, int64_t offset)
@@ -665,6 +658,3 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
 	int flushableIndex; // index in owner->flushable[]
 	int zeroCopyRefCount; // references held by "zero-copy" reads
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbrpc/include/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/include/fdbrpc/AsyncFileEncrypted.h
index af0c0eb8579..dea56152d50 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileEncrypted.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileEncrypted.h
@@ -26,8 +26,6 @@
 #include "flow/IRandom.h"
 #include "flow/StreamCipher.h"
 
-#include <array>
-
 /*
  * Append-only file encrypted using AES-256-GCM.
  * */
@@ -46,28 +44,16 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted<AsyncFileE
 	friend class AsyncFileEncryptedImpl;
 	int64_t fileSize = -1;
 
-	// Reading:
-	class RandomCache {
-		size_t maxSize;
-		std::vector<uint32_t> vec;
-		std::unordered_map<uint32_t, Standalone<StringRef>> hashMap;
-		size_t evict();
-
-	public:
-		RandomCache(size_t maxSize);
-		void insert(uint32_t block, const Standalone<StringRef>& value);
-		Optional<Standalone<StringRef>> get(uint32_t block) const;
-	} readBuffers;
-
 	// Writing (append only):
 	std::unique_ptr<EncryptionStreamCipher> encryptor;
 	uint32_t currentBlock{ 0 };
 	int offsetInBlock{ 0 };
 	std::vector<unsigned char> writeBuffer;
+	int encryptionBlockSize{ 0 };
 	Future<Void> initialize();
 
 public:
-	AsyncFileEncrypted(Reference<IAsyncFile>, Mode);
+	AsyncFileEncrypted(Reference<IAsyncFile>, Mode, int);
 	void addref() override;
 	void delref() override;
 	Future<int> read(void* data, int length, int64_t offset) override;
diff --git a/fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/include/fdbrpc/AsyncFileNonDurable.h
similarity index 81%
rename from fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h
rename to fdbrpc/include/fdbrpc/AsyncFileNonDurable.h
index d327f1fd4b4..d8b3f158ea1 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileNonDurable.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileNonDurable.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileNonDurable.actor.h
+ * AsyncFileNonDurable.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,42 +20,33 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_ASYNCFILENONDURABLE_ACTOR_G_H)
-#define FLOW_ASYNCFILENONDURABLE_ACTOR_G_H
-#include "fdbrpc/AsyncFileNonDurable.actor.g.h"
-#elif !defined(FLOW_ASYNCFILENONDURABLE_ACTOR_H)
-#define FLOW_ASYNCFILENONDURABLE_ACTOR_H
-
 #include "flow/flow.h"
 #include "flow/IAsyncFile.h"
 #include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/TraceFileIO.h"
 #include "fdbrpc/RangeMap.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 #undef max
 #undef min
 
-ACTOR Future<Void> sendOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, TaskPriority taskID);
-ACTOR Future<Void> sendErrorOnProcess(ISimulator::ProcessInfo* process,
-                                      Promise<Void> promise,
-                                      Error e,
-                                      TaskPriority taskID);
+Future<Void> sendOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, TaskPriority taskID);
+Future<Void> sendErrorOnProcess(ISimulator::ProcessInfo* process, Promise<Void> promise, Error e, TaskPriority taskID);
 
 extern Future<Void> waitShutdownSignal();
 
-ACTOR template <class T>
+template <class T>
 Future<T> sendErrorOnShutdown(Future<T> in, bool assertOnCancel = false) {
 	try {
-		choose {
-			when(wait(waitShutdownSignal())) {
-				throw io_error().asInjectedFault();
-			}
-			when(T rep = wait(in)) {
-				return rep;
+		auto res = co_await race(waitShutdownSignal(), in);
+		if (res.index() == 0) {
+			throw io_error().asInjectedFault();
+		} else {
+			if constexpr (std::is_same_v<T, Void>) {
+				co_return;
+			} else {
+				co_return std::get<1>(std::move(res));
 			}
 		}
 	} catch (Error& e) {
@@ -77,8 +68,8 @@ class AsyncFileDetachable final : public IAsyncFile, public ReferenceCounted<Asy
 		shutdown = doShutdown(this);
 	}
 
-	ACTOR Future<Void> doShutdown(AsyncFileDetachable* self);
-	ACTOR static Future<Reference<IAsyncFile>> open(Future<Reference<IAsyncFile>> wrappedFile);
+	Future<Void> doShutdown(AsyncFileDetachable* self);
+	static Future<Reference<IAsyncFile>> open(Future<Reference<IAsyncFile>> wrappedFile);
 
 	void addref() override { ReferenceCounted<AsyncFileDetachable>::addref(); }
 	void delref() override { ReferenceCounted<AsyncFileDetachable>::delref(); }
@@ -189,11 +180,11 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 	static std::map<std::string, Future<Void>> filesBeingDeleted;
 
 	// Creates a new AsyncFileNonDurable which wraps the provided IAsyncFile
-	ACTOR static Future<Reference<IAsyncFile>> open(std::string filename,
-	                                                std::string actualFilename,
-	                                                Future<Reference<IAsyncFile>> wrappedFile,
-	                                                Reference<DiskParameters> diskParameters,
-	                                                bool aio);
+	static Future<Reference<IAsyncFile>> open(std::string filename,
+	                                          std::string actualFilename,
+	                                          Future<Reference<IAsyncFile>> wrappedFile,
+	                                          Reference<DiskParameters> diskParameters,
+	                                          bool aio);
 
 	~AsyncFileNonDurable() override {
 		//TraceEvent("AsyncFileNonDurable_Destroy", id).detail("Filename", filename);
@@ -311,66 +302,64 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 	}
 
 	// Checks if the file is killed.  If so, then the current sync is completed if running and then an error is thrown
-	ACTOR static Future<Void> checkKilled(AsyncFileNonDurable const* self, std::string context) {
+	static Future<Void> checkKilled(AsyncFileNonDurable const* self, std::string context) {
 		if (self->killed.isSet()) {
 			//TraceEvent("AsyncFileNonDurable_KilledInCheck", self->id).detail("In", context).detail("Filename", self->filename);
-			wait(self->killComplete.getFuture());
+			co_await self->killComplete.getFuture();
 			TraceEvent("AsyncFileNonDurable_KilledFileOperation", self->id)
 			    .detail("In", context)
 			    .detail("Filename", self->filename);
 			CODE_PROBE(true, "AsyncFileNonDurable operation killed", probe::decoration::rare);
 			throw io_error().asInjectedFault();
 		}
-
-		return Void();
 	}
 
 	// Passes along reads straight to the underlying file, waiting for any outstanding changes that could affect the
 	// results
-	ACTOR Future<int> onRead(AsyncFileNonDurable* self, void* data, int length, int64_t offset) {
-		wait(checkKilled(self, "Read"));
+	Future<int> onRead(AsyncFileNonDurable* self, void* data, int length, int64_t offset) {
+		co_await checkKilled(self, "Read");
 		std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(offset, length);
-		wait(waitForAll(priorModifications));
-		state Future<int> readFuture = self->file->read(data, length, offset);
-		wait(success(readFuture) || self->killed.getFuture());
+		co_await waitForAll(priorModifications);
+		Future<int> readFuture = self->file->read(data, length, offset);
+		co_await (success(readFuture) || self->killed.getFuture());
 
 		// throws if we were killed
-		wait(checkKilled(self, "ReadEnd"));
+		co_await checkKilled(self, "ReadEnd");
 
 		debugFileCheck("AsyncFileNonDurableRead", self->filename, data, offset, length);
 
 		// if(g_simulator->getCurrentProcess()->rebooting)
 		//TraceEvent("AsyncFileNonDurable_ReadEnd", self->id).detail("Filename", self->filename);
 
-		return readFuture.get();
+		co_return readFuture.get();
 	}
 
-	ACTOR Future<int> read(AsyncFileNonDurable* self, void* data, int length, int64_t offset);
+	Future<int> read(AsyncFileNonDurable* self, void* data, int length, int64_t offset);
 
 	// Delays writes a random amount of time before passing them through to the underlying file.
 	// If a kill interrupts the delay, then the output could be the correct write, part of the write,
 	// or none of the write.  It may also corrupt parts of sectors which have not been written correctly
-	ACTOR Future<Void> write(AsyncFileNonDurable* self,
-	                         Promise<Void> writeStarted,
-	                         Future<Future<Void>> ownFuture,
-	                         void const* data,
-	                         int length,
-	                         int64_t offset) {
-		state Standalone<StringRef> dataCopy(StringRef((uint8_t*)data, length));
-		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator->onMachine(currentProcess));
-
-		state double delayDuration =
+	Future<Void> write(AsyncFileNonDurable* self,
+	                   Promise<Void> writeStarted,
+	                   Future<Future<Void>> ownFuture,
+	                   void const* data,
+	                   int length,
+	                   int64_t offset) {
+		Standalone<StringRef> dataCopy(StringRef((uint8_t*)data, length));
+		ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+		TaskPriority currentTaskID = g_network->getCurrentTask();
+		co_await g_simulator->onMachine(currentProcess);
+
+		double delayDuration =
 		    g_simulator->speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay);
 
-		state Future<bool> startSyncFuture = self->startSyncPromise.getFuture();
+		Future<bool> startSyncFuture = self->startSyncPromise.getFuture();
 
 		try {
 			//TraceEvent("AsyncFileNonDurable_Write", self->id).detail("Delay", delayDuration).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset);
-			wait(checkKilled(self, "Write"));
+			co_await checkKilled(self, "Write");
 
-			Future<Void> writeEnded = wait(ownFuture);
+			Future<Void> writeEnded = co_await ownFuture;
 			std::vector<Future<Void>> priorModifications =
 			    self->getModificationsAndInsert(offset, length, true, writeEnded);
 			self->minSizeAfterPendingModifications = std::max(self->minSizeAfterPendingModifications, offset + length);
@@ -383,7 +372,7 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, length) ||
 				                             self->killed.getFuture());
 
-			wait(waitForAll(priorModifications));
+			co_await waitForAll(priorModifications);
 
 			self->approximateSize = std::max(self->approximateSize, length + offset);
 
@@ -396,12 +385,9 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 		//TraceEvent("AsyncFileNonDurable_WriteDoneWithPreviousMods", self->id).detail("Delay", delayDuration).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset);
 
 		// Wait a random amount of time or until a sync/kill is issued
-		state bool saveDurable = true;
-		choose {
-			when(wait(delay(delayDuration))) {}
-			when(bool durable = wait(startSyncFuture)) {
-				saveDurable = durable;
-			}
+		bool saveDurable = true;
+		if (auto const res = co_await timeout(startSyncFuture, delayDuration); res.present()) {
+			saveDurable = res.get();
 		}
 
 		debugFileCheck("AsyncFileNonDurableWriteAfterWait", self->filename, dataCopy.begin(), offset, length);
@@ -520,35 +506,34 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 			writeOffset += pageLength;
 		}
 
-		wait(waitForAll(writeFutures));
+		co_await waitForAll(writeFutures);
 		//TraceEvent("AsyncFileNonDurable_WriteDone", self->id).detail("Delay", delayDuration).detail("Filename", self->filename).detail("WriteLength", length).detail("Offset", offset);
-		return Void();
 	}
 
 	// Delays truncates a random amount of time before passing them through to the underlying file.
 	// If a kill interrupts the delay, then the truncate may or may not be performed
-	ACTOR Future<Void> truncate(AsyncFileNonDurable* self,
-	                            Promise<Void> truncateStarted,
-	                            Future<Future<Void>> ownFuture,
-	                            int64_t size) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator->onMachine(currentProcess));
-
-		state double delayDuration =
+	Future<Void> truncate(AsyncFileNonDurable* self,
+	                      Promise<Void> truncateStarted,
+	                      Future<Future<Void>> ownFuture,
+	                      int64_t size) {
+		ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+		TaskPriority currentTaskID = g_network->getCurrentTask();
+		co_await g_simulator->onMachine(currentProcess);
+
+		double delayDuration =
 		    g_simulator->speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay);
-		state Future<bool> startSyncFuture = self->startSyncPromise.getFuture();
+		Future<bool> startSyncFuture = self->startSyncPromise.getFuture();
 
 		try {
 			//TraceEvent("AsyncFileNonDurable_Truncate", self->id).detail("Delay", delayDuration).detail("Filename", self->filename);
-			wait(checkKilled(self, "Truncate"));
+			co_await checkKilled(self, "Truncate");
 
-			state Future<Void> truncateEnded = wait(ownFuture);
+			Future<Void> truncateEnded = co_await ownFuture;
 
 			// Need to know the size of the file directly before this truncate
 			// takes effect to see what range it modifies.
 			if (!self->minSizeAfterPendingModificationsIsExact) {
-				wait(success(self->size()));
+				co_await self->size();
 			}
 			ASSERT(self->minSizeAfterPendingModificationsIsExact);
 			int64_t beginModifiedRange = std::min(size, self->minSizeAfterPendingModifications);
@@ -564,7 +549,7 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, 0) || self->killed.getFuture());
 
-			wait(waitForAll(priorModifications));
+			co_await waitForAll(priorModifications);
 
 			self->approximateSize = size;
 
@@ -575,43 +560,38 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 		}
 
 		// Wait a random amount of time or until a sync/kill is issued
-		state bool saveDurable = true;
-		choose {
-			when(wait(delay(delayDuration))) {}
-			when(bool durable = wait(startSyncFuture)) {
-				saveDurable = durable;
-			}
+		bool saveDurable = true;
+		if (auto const res = co_await timeout(startSyncFuture, delayDuration); res.present()) {
+			saveDurable = res.get();
 		}
 
 		if (g_network->check_yield(TaskPriority::DefaultYield)) {
-			wait(delay(0, TaskPriority::DefaultYield));
+			co_await delay(0, TaskPriority::DefaultYield);
 		}
 
 		// If performing a durable truncate, then pass it through to the file.  Otherwise, pass it through with a 1/2
 		// chance
 		if (saveDurable || self->killMode == NO_CORRUPTION || deterministicRandom()->random01() < 0.5)
-			wait(self->file->truncate(size));
+			co_await self->file->truncate(size);
 		else {
 			TraceEvent("AsyncFileNonDurable_DroppedTruncate", self->id).detail("Size", size);
 			CODE_PROBE(true, "AsyncFileNonDurable dropped truncate", probe::decoration::rare);
 		}
-
-		return Void();
 	}
 
 	// Waits for delayed modifications to the file to complete and then syncs the underlying file
 	// If durable is false, then some of the delayed modifications will not be applied or will be
 	// applied incorrectly
-	ACTOR Future<Void> onSync(AsyncFileNonDurable* self, bool durable) {
+	Future<Void> onSync(AsyncFileNonDurable* self, bool durable) {
 		//TraceEvent("AsyncFileNonDurable_ImplSync", self->id).detail("Filename", self->filename).detail("Durable", durable);
 		ASSERT(durable || !self->killed.isSet()); // this file is kill()ed only once
 
 		if (durable) {
 			self->hasBeenSynced = true;
-			wait(waitUntilDiskReady(self->diskParameters, 0, true) || self->killed.getFuture());
+			co_await (waitUntilDiskReady(self->diskParameters, 0, true) || self->killed.getFuture());
 		}
 
-		wait(checkKilled(self, durable ? "Sync" : "Kill"));
+		co_await checkKilled(self, durable ? "Sync" : "Kill");
 
 		if (!durable)
 			self->killed.send(Void());
@@ -643,14 +623,14 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 		self->startSyncPromise = Promise<bool>();
 
 		// Writes will be durable in a kill with a 10% probability
-		state bool writeDurable = durable || deterministicRandom()->random01() < 0.1;
+		bool writeDurable = durable || deterministicRandom()->random01() < 0.1;
 		startSyncPromise.send(writeDurable);
 
 		// Wait for outstanding writes to complete
 		if (durable)
-			wait(allModifications);
+			co_await allModifications;
 		else
-			wait(success(errorOr(allModifications)));
+			co_await errorOr(allModifications);
 
 		if (!durable) {
 			// Sometimes sync the file if writes were made durably.  Before a file is first synced, it is stored in a
@@ -658,7 +638,7 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 			// simulate a failure to fsync the directory storing the file
 			if (self->hasBeenSynced && writeDurable && deterministicRandom()->random01() < 0.5) {
 				CODE_PROBE(true, "AsyncFileNonDurable kill was durable and synced", probe::decoration::rare);
-				wait(success(errorOr(self->file->sync())));
+				co_await errorOr(self->file->sync());
 			}
 
 			// Setting this promise could trigger the deletion of the AsyncFileNonDurable; after this none of its
@@ -668,68 +648,65 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 		}
 		// A killed file cannot be allowed to report that it successfully synced
 		else {
-			wait(checkKilled(self, "SyncEnd"));
-			wait(self->file->sync());
+			co_await checkKilled(self, "SyncEnd");
+			co_await self->file->sync();
 			//TraceEvent("AsyncFileNonDurable_ImplSyncEnd", self->id).detail("Filename", self->filename).detail("Durable", durable);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> sync(AsyncFileNonDurable* self, bool durable) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-		state TaskPriority currentTaskID = g_network->getCurrentTask();
-		wait(g_simulator->onMachine(currentProcess));
+	Future<Void> sync(AsyncFileNonDurable* self, bool durable) {
+		ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+		TaskPriority currentTaskID = g_network->getCurrentTask();
+		co_await g_simulator->onMachine(currentProcess);
 
+		Error err;
 		try {
-			wait(self->onSync(self, durable));
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-
-			return Void();
+			co_await self->onSync(self, durable);
+			co_await g_simulator->onProcess(currentProcess, currentTaskID);
+			co_return;
 		} catch (Error& e) {
-			state Error err = e;
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-			throw err;
+			err = e;
 		}
+
+		co_await g_simulator->onProcess(currentProcess, currentTaskID);
+		throw err;
 	}
 
 	// Passes along size requests to the underlying file, augmenting with any writes past the end of the file
-	ACTOR static Future<int64_t> onSize(AsyncFileNonDurable const* self) {
+	static Future<int64_t> onSize(AsyncFileNonDurable const* self) {
 		//TraceEvent("AsyncFileNonDurable_Size", self->id).detail("Filename", self->filename);
-		wait(checkKilled(self, "Size"));
-		state Future<int64_t> sizeFuture = self->file->size();
-		wait(success(sizeFuture) || self->killed.getFuture());
+		co_await checkKilled(self, "Size");
+		Future<int64_t> sizeFuture = self->file->size();
+		co_await (success(sizeFuture) || self->killed.getFuture());
 
-		wait(checkKilled(self, "SizeEnd"));
+		co_await checkKilled(self, "SizeEnd");
 
 		// Include any modifications which extend past the end of the file
 		self->approximateSize = self->minSizeAfterPendingModifications =
 		    std::max<int64_t>(sizeFuture.get(), self->minSizeAfterPendingModifications);
 		self->minSizeAfterPendingModificationsIsExact = true;
-		return self->approximateSize;
+		co_return self->approximateSize;
 	}
 
-	ACTOR static Future<int64_t> size(AsyncFileNonDurable const* self) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-		state TaskPriority currentTaskID = g_network->getCurrentTask();
+	static Future<int64_t> size(AsyncFileNonDurable const* self) {
+		ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+		TaskPriority currentTaskID = g_network->getCurrentTask();
 
-		wait(g_simulator->onMachine(currentProcess));
+		co_await g_simulator->onMachine(currentProcess);
 
+		Error err;
 		try {
-			state int64_t rep = wait(onSize(self));
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-
-			return rep;
+			int64_t rep = co_await onSize(self);
+			co_await g_simulator->onProcess(currentProcess, currentTaskID);
+			co_return rep;
 		} catch (Error& e) {
-			state Error err = e;
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-			throw err;
+			err = e;
 		}
+
+		co_await g_simulator->onProcess(currentProcess, currentTaskID);
+		throw err;
 	}
 
 	// Finishes all outstanding actors on an AsyncFileNonDurable and then deletes it
-	ACTOR Future<Void> closeFile(AsyncFileNonDurable* self);
+	Future<Void> closeFile(AsyncFileNonDurable* self);
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbrpc/include/fdbrpc/AsyncFileReadAhead.actor.h b/fdbrpc/include/fdbrpc/AsyncFileReadAhead.h
similarity index 82%
rename from fdbrpc/include/fdbrpc/AsyncFileReadAhead.actor.h
rename to fdbrpc/include/fdbrpc/AsyncFileReadAhead.h
index d964ba8230f..ec4a69af8fa 100644
--- a/fdbrpc/include/fdbrpc/AsyncFileReadAhead.actor.h
+++ b/fdbrpc/include/fdbrpc/AsyncFileReadAhead.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFileReadAhead.actor.h
+ * AsyncFileReadAhead.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,17 +20,8 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBRPC_ASYNCFILEREADAHEAD_ACTOR_G_H)
-#define FDBRPC_ASYNCFILEREADAHEAD_ACTOR_G_H
-#include "fdbrpc/AsyncFileReadAhead.actor.g.h"
-#elif !defined(FDBRPC_ASYNCFILEREADAHEAD_ACTOR_H)
-#define FDBRPC_ASYNCFILEREADAHEAD_ACTOR_H
-
 #include "flow/flow.h"
 #include "flow/IAsyncFile.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // Read-only file type that wraps another file instance, reads in large blocks, and reads ahead of the actual range
 // requested
@@ -42,19 +33,19 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 	virtual StringRef getClassName() override { return "AsyncFileReadAheadCache"_sr; }
 
 	struct CacheBlock : ReferenceCounted<CacheBlock> {
-		CacheBlock(int size = 0) : data(new uint8_t[size]), len(size) {}
+		explicit CacheBlock(int size = 0) : data(new uint8_t[size]), len(size) {}
 		~CacheBlock() { delete[] data; }
 		uint8_t* data;
 		int len;
 	};
 
 	// Read from the underlying file to a CacheBlock
-	ACTOR static Future<Reference<CacheBlock>> readBlock(AsyncFileReadAheadCache* f, int length, int64_t offset) {
-		wait(f->m_max_concurrent_reads.take());
+	static Future<Reference<CacheBlock>> readBlock(AsyncFileReadAheadCache* f, int length, int64_t offset) {
+		co_await f->m_max_concurrent_reads.take();
 
-		state Reference<CacheBlock> block(new CacheBlock(length));
+		Reference<CacheBlock> block(new CacheBlock(length));
 		try {
-			int len = wait(uncancellable(holdWhile(block, f->m_f->read(block->data, length, offset))));
+			int len = co_await uncancellable(holdWhile(block, f->m_f->read(block->data, length, offset)));
 			block->len = len;
 		} catch (Error& e) {
 			f->m_max_concurrent_reads.release(1);
@@ -62,17 +53,17 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 		}
 
 		f->m_max_concurrent_reads.release(1);
-		return block;
+		co_return block;
 	}
 
-	ACTOR static Future<int> read_impl(Reference<AsyncFileReadAheadCache> f, void* data, int length, int64_t offset) {
+	static Future<int> read_impl(Reference<AsyncFileReadAheadCache> f, void* data, int length, int64_t offset) {
 		// Make sure range is valid for the file
-		int64_t fileSize = wait(f->size());
+		int64_t fileSize = co_await f->size();
 		if (offset >= fileSize)
-			return 0; // TODO:  Should this throw since the input isn't really valid?
+			co_return 0; // TODO:  Should this throw since the input isn't really valid?
 
 		if (length == 0) {
-			return 0;
+			co_return 0;
 		}
 
 		// If reading past the end then clip length to just read to the end
@@ -80,22 +71,22 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 			length = fileSize - offset; // Length is at least 1 since offset < fileSize
 
 		// Calculate block range for the blocks that contain this data
-		state int firstBlockNum = offset / f->m_block_size;
+		int firstBlockNum = offset / f->m_block_size;
 		ASSERT(f->m_block_size > 0);
-		state int lastBlockNum = (offset + length - 1) / f->m_block_size;
+		int lastBlockNum = (offset + length - 1) / f->m_block_size;
 
 		// Start reads (if needed) of the block range required for this read, plus the read ahead blocks
 		// The futures for the read started will be stored in the cache but since things can be evicted from
 		// the cache while we're wait()ing we also will keep a local cache of futures for the blocks
 		// we need (not the read ahead blocks).
-		state std::map<int, Future<Reference<CacheBlock>>> localCache;
+		std::map<int, Future<Reference<CacheBlock>>> localCache;
 
 		// Start blocks up to the read ahead size beyond the last needed block but don't go past the end of the file
-		state int lastBlockNumInFile = ((fileSize + f->m_block_size - 1) / f->m_block_size) - 1;
+		int lastBlockNumInFile = ((fileSize + f->m_block_size - 1) / f->m_block_size) - 1;
 		ASSERT(lastBlockNum <= lastBlockNumInFile);
 		int lastBlockToStart = std::min<int>(lastBlockNum + f->m_read_ahead_blocks, lastBlockNumInFile);
 
-		state int blockNum;
+		int blockNum{ 0 };
 		for (blockNum = firstBlockNum; blockNum <= lastBlockToStart; ++blockNum) {
 			Future<Reference<CacheBlock>> fblock;
 
@@ -115,10 +106,10 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 		}
 
 		// Read block(s) and copy data
-		state int wpos = 0;
+		int wpos = 0;
 		for (blockNum = firstBlockNum; blockNum <= lastBlockNum; ++blockNum) {
 			// Wait for block to be ready
-			Reference<CacheBlock> block = wait(localCache[blockNum]);
+			Reference<CacheBlock> block = co_await localCache[blockNum];
 
 			// Calculate the block-relative read range.  It's a given that the offset / length range touches this block
 			// so readStart will never be greater than blocksize (though it could be past the actual end of a short
@@ -168,7 +159,7 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 			}
 		}
 
-		return wpos;
+		co_return wpos;
 	}
 
 	Future<int> read(void* data, int length, int64_t offset) override {
@@ -216,6 +207,3 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 	  : m_f(f), m_block_size(blockSize), m_read_ahead_blocks(readAheadBlocks),
 	    m_cache_block_limit(std::max<int>(1, cacheSizeBlocks)), m_max_concurrent_reads(maxConcurrentReads) {}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbrpc/include/fdbrpc/DDSketch.h b/fdbrpc/include/fdbrpc/DDSketch.h
index cc66eb23061..481a8af40c1 100644
--- a/fdbrpc/include/fdbrpc/DDSketch.h
+++ b/fdbrpc/include/fdbrpc/DDSketch.h
@@ -245,7 +245,7 @@ class DDSketch : public DDSketchBase<DDSketch<T>, T> {
 template <class T>
 class DDSketchSlow : public DDSketchBase<DDSketchSlow<T>, T> {
 public:
-	DDSketchSlow(double errorGuarantee = 0.1)
+	explicit DDSketchSlow(double errorGuarantee = 0.1)
 	  : DDSketchBase<DDSketchSlow<T>, T>(errorGuarantee), gamma((1.0 + errorGuarantee) / (1.0 - errorGuarantee)),
 	    logGamma(log(gamma)) {
 		offset = getIndex(1.0 / DDSketchBase<DDSketch<T>, T>::EPS) + 5;
diff --git a/fdbrpc/include/fdbrpc/FlowGrpc.h b/fdbrpc/include/fdbrpc/FlowGrpc.h
index 83a75ec8a54..2a6a4769a8f 100644
--- a/fdbrpc/include/fdbrpc/FlowGrpc.h
+++ b/fdbrpc/include/fdbrpc/FlowGrpc.h
@@ -26,7 +26,7 @@
 
 #include "flow/flow.h"
 #include "flow/NetworkAddress.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbrpc/grpc/AsyncGrpcClient.h"
 #include "fdbrpc/grpc/AsyncTaskExecutor.h"
 #include "fdbrpc/grpc/Credentials.h"
@@ -110,7 +110,7 @@ class GrpcServer {
 public:
 	using ServiceList = std::vector<std::shared_ptr<grpc::Service>>;
 
-	GrpcServer(const NetworkAddress& addr, std::shared_ptr<GrpcCredentialProvider> provider = nullptr);
+	explicit GrpcServer(const NetworkAddress& addr, std::shared_ptr<GrpcCredentialProvider> provider = nullptr);
 	~GrpcServer();
 
 	// Returns the singleton instance.
@@ -150,9 +150,9 @@ class GrpcServer {
 	void registerService(std::shared_ptr<grpc::Service> service);
 	void registerRoleServices(const UID& owner_id, const ServiceList& services);
 
-	// Removes services associated with given `owner_id` from the server. Returns future that is fulfilled onced the
+	// Removes services associated with given `owner_id` from the server. Returns future that is fulfilled once the
 	// services are no longer alive (however, server may not have restarted yet).
-	Future<Void> deregisterRoleServices(const UID& owner_id);
+	Future<Void> deregisterRoleServices(UID owner_id);
 	void deregisterRoleServicesSync(const UID& owner_id);
 
 	// Returns `true` if TLS is enabled.
diff --git a/fdbrpc/include/fdbrpc/FlowProcess.actor.h b/fdbrpc/include/fdbrpc/FlowProcess.actor.h
deleted file mode 100644
index 6271349b60a..00000000000
--- a/fdbrpc/include/fdbrpc/FlowProcess.actor.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * FlowProcess.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#if defined(NO_INTELLISENSE) && !defined(FDBRPC_FLOW_PROCESS_ACTOR_G_H)
-#define FDBRPC_FLOW_PROCESS_ACTOR_G_H
-#include "fdbrpc/FlowProcess.actor.g.h"
-#elif !defined(FDBRPC_FLOW_PROCESS_ACTOR_H)
-#define FDBRPC_FLOW_PROCESS_ACTOR_H
-
-#include "fdbrpc/fdbrpc.h"
-
-#include <string>
-#include <map>
-
-#include <flow/actorcompiler.h> // has to be last include
-
-struct FlowProcessInterface {
-	constexpr static FileIdentifier file_identifier = 3491839;
-	RequestStream<struct FlowProcessRegistrationRequest> registerProcess;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, registerProcess);
-	}
-};
-
-struct FlowProcessRegistrationRequest {
-	constexpr static FileIdentifier file_identifier = 3411838;
-	Standalone<StringRef> flowProcessInterface;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, flowProcessInterface);
-	}
-};
-
-class FlowProcess {
-
-public:
-	virtual ~FlowProcess() {}
-	virtual StringRef name() const = 0;
-	virtual StringRef serializedInterface() const = 0;
-	virtual Future<Void> run() = 0;
-	virtual void registerEndpoint(Endpoint p) = 0;
-};
-
-struct IProcessFactory {
-	static FlowProcess* create(std::string const& name) {
-		auto it = factories().find(name);
-		if (it == factories().end())
-			return nullptr; // or throw?
-		return it->second->create();
-	}
-	static std::map<std::string, IProcessFactory*>& factories() {
-		static std::map<std::string, IProcessFactory*> theFactories;
-		return theFactories;
-	}
-
-	virtual FlowProcess* create() = 0;
-
-	virtual const char* getName() = 0;
-};
-
-template <class ProcessType>
-struct ProcessFactory : IProcessFactory {
-	ProcessFactory(const char* name) : name(name) { factories()[name] = this; }
-	FlowProcess* create() override { return new ProcessType(); }
-	const char* getName() override { return this->name; }
-
-private:
-	const char* name;
-};
-
-#include <flow/unactorcompiler.h>
-#endif
diff --git a/fdbrpc/include/fdbrpc/LoadBalance.actor.h b/fdbrpc/include/fdbrpc/LoadBalance.actor.h
index 072516dcadd..85d83c0190e 100644
--- a/fdbrpc/include/fdbrpc/LoadBalance.actor.h
+++ b/fdbrpc/include/fdbrpc/LoadBalance.actor.h
@@ -150,7 +150,7 @@ Future<Void> tssComparison(Req req,
 			if (!TSS_doCompare(src.get(), tss.get().get())) {
 				CODE_PROBE(true, "TSS Mismatch");
 				state TraceEvent mismatchEvent(
-				    (g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
+				    (simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch))
 				        ? SevWarnAlways
 				        : SevError,
 				    LB_mismatchTraceName(req, TSS_COMPARISON));
@@ -213,11 +213,12 @@ Future<Void> tssComparison(Req req,
 						tssData.metrics->recordDetailedMismatchData(mismatchUID, mismatchEvent.getFields().toString());
 
 						// record a summarized trace event instead
-						TraceEvent summaryEvent((g_network->isSimulated() &&
-						                         g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations)
-						                            ? SevWarnAlways
-						                            : SevError,
-						                        LB_mismatchTraceName(req, TSS_COMPARISON));
+						TraceEvent summaryEvent(
+						    (g_network->isSimulated() &&
+						     simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch))
+						        ? SevWarnAlways
+						        : SevError,
+						    LB_mismatchTraceName(req, TSS_COMPARISON));
 						summaryEvent.detail("TSSID", tssData.tssId).detail("MismatchId", mismatchUID);
 					}
 				} else {
@@ -246,42 +247,6 @@ Future<Void> tssComparison(Req req,
 	return Void();
 }
 
-ACTOR template <class Resp>
-Future<Void> waitForQuorumReplies(std::vector<Future<Optional<ErrorOr<Resp>>>>* replies, int required) {
-	state int outstandingReplies = (int)replies->size();
-	state int requiredReplies = std::min(required, (int)replies->size());
-	state std::vector<Future<Optional<ErrorOr<Resp>>>> ongoingReplies;
-	loop {
-		ongoingReplies.clear();
-		for (auto& reply : (*replies)) {
-			if (!reply.isReady()) {
-				ongoingReplies.push_back(reply);
-			}
-		}
-		ASSERT(ongoingReplies.size() == outstandingReplies);
-
-		if (requiredReplies == 0 || outstandingReplies == 0) {
-			break;
-		}
-
-		wait(quorum(ongoingReplies, std::min(requiredReplies, outstandingReplies)));
-
-		for (auto& reply : ongoingReplies) {
-			if (reply.isReady()) {
-				outstandingReplies--;
-				if (!reply.isError() && reply.get().present() && !reply.get().get().isError()) {
-					Optional<LoadBalancedReply> lbReply = getLoadBalancedReply(&reply.get().get().get());
-					if (lbReply.present() && !lbReply.get().error.present()) {
-						requiredReplies--;
-					}
-				}
-			}
-		}
-	}
-
-	return Void();
-}
-
 ACTOR template <class Req, class Resp, class Interface, class Multi, bool P>
 Future<Void> replicaComparison(Req req,
                                Future<ErrorOr<Resp>> fSource,
@@ -465,7 +430,7 @@ struct RequestData : NonCopyable {
 	bool compareReplicas = false;
 	Future<Void> comparisonResult;
 
-	RequestData(bool compareReplicas = false) : compareReplicas(compareReplicas) {}
+	explicit RequestData(bool compareReplicas = false) : compareReplicas(compareReplicas) {}
 
 	// Whether or not the response future is valid
 	// This is true once setupRequest is called, even though at that point the response is Never().
@@ -549,14 +514,14 @@ struct RequestData : NonCopyable {
 		if (backoff > 0) {
 			response = mapAsync(delay(backoff), [this, stream, &request, model, alternatives, channel](Void _) {
 				requestStarted = true;
-				modelHolder = Reference<ModelHolder>(new ModelHolder(model, stream->getEndpoint().token.first()));
+				modelHolder = makeReference<ModelHolder>(model, stream->getEndpoint().token.first());
 				Future<Reply> resp = stream->tryGetReply(request);
 				maybeDuplicateTSSRequest(stream, request, model, resp, alternatives, channel);
 				return resp;
 			});
 		} else {
 			requestStarted = true;
-			modelHolder = Reference<ModelHolder>(new ModelHolder(model, stream->getEndpoint().token.first()));
+			modelHolder = makeReference<ModelHolder>(model, stream->getEndpoint().token.first());
 			response = stream->tryGetReply(request);
 			maybeDuplicateTSSRequest(stream, request, model, response, alternatives, channel);
 		}
@@ -688,19 +653,16 @@ struct RequestData : NonCopyable {
 // interfaces. If too many interfaces in the same DC are bad, try remote interfaces.
 // If compareReplicas is set, does a consistency check by fetching and comparing results from storage
 // replicas (as many as specified by "requiredReplicas") and throws an exception if an inconsistency is found.
-// FIXME: reformat this minus the long inline comment about one parameter, so that the indentation of
-// the parameters is more to the right and not confusingly lined up with the code of this function.
 ACTOR template <class Interface, class Request, class Multi, bool P>
-Future<REPLY_TYPE(Request)> loadBalance(
-    Reference<MultiInterface<Multi>> alternatives,
-    RequestStream<Request, P> Interface::* channel,
-    Request request = Request(),
-    TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint,
-    AtMostOnce atMostOnce =
-        AtMostOnce::False, // if true, throws request_maybe_delivered() instead of retrying automatically
-    QueueModel* model = nullptr,
-    bool compareReplicas = false,
-    int requiredReplicas = 0) {
+Future<REPLY_TYPE(Request)> loadBalance(Reference<MultiInterface<Multi>> alternatives,
+                                        RequestStream<Request, P> Interface::* channel,
+                                        Request request = Request(),
+                                        TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint,
+                                        // If true, throws request_maybe_delivered() instead of retrying automatically.
+                                        AtMostOnce atMostOnce = AtMostOnce::False,
+                                        QueueModel* model = nullptr,
+                                        bool compareReplicas = false,
+                                        int requiredReplicas = 0) {
 
 	state RequestData<Request, Interface, Multi, P> firstRequestData(compareReplicas);
 	state RequestData<Request, Interface, Multi, P> secondRequestData(compareReplicas);
diff --git a/fdbrpc/include/fdbrpc/Locality.h b/fdbrpc/include/fdbrpc/Locality.h
index fa903767952..428281df6d0 100644
--- a/fdbrpc/include/fdbrpc/Locality.h
+++ b/fdbrpc/include/fdbrpc/Locality.h
@@ -39,7 +39,7 @@ struct ProcessClass {
 		LogClass,
 		ClusterControllerClass,
 		LogRouterClass,
-		FastRestoreClass,
+		FastRestoreClass, // deprecated
 		DataDistributorClass,
 		CoordinatorClass,
 		RatekeeperClass,
@@ -126,124 +126,22 @@ struct ProcessClass {
 public:
 	ProcessClass() : _class(UnsetClass), _source(CommandLineSource) {}
 	ProcessClass(ClassType type, ClassSource source) : _class(type), _source(source) {}
-	// clang-format off
-	explicit ProcessClass( std::string s, ClassSource source ) : _source( source ) {
-		if (s=="storage") _class = StorageClass;
-		else if (s=="transaction") _class = TransactionClass;
-		else if (s=="resolution") _class = ResolutionClass;
-		else if (s=="commit_proxy") _class = CommitProxyClass;
-		else if (s=="proxy") {
-			_class = CommitProxyClass;
-			printf("WARNING: 'proxy' machine class is deprecated and will be automatically converted "
-					"'commit_proxy' machine class. Please use 'grv_proxy' or 'commit_proxy' specifically\n");
-		}
-		else if (s=="grv_proxy") _class = GrvProxyClass;
-		else if (s=="master") _class = MasterClass;
-		else if (s=="test") _class = TesterClass;
-		else if (s=="unset") _class = UnsetClass;
-		else if (s=="stateless") _class = StatelessClass;
-		else if (s=="log") _class = LogClass;
-		else if (s=="router") _class = LogRouterClass;
-		else if (s=="cluster_controller") _class = ClusterControllerClass;
-		else if (s=="fast_restore") _class = FastRestoreClass;
-		else if (s=="data_distributor") _class = DataDistributorClass;
-		else if (s=="coordinator") _class = CoordinatorClass;
-		else if (s=="ratekeeper") _class = RatekeeperClass;
-		else if (s=="consistency_scan") _class = ConsistencyScanClass;
-		else if (s=="blob_manager") _class = BlobManagerClass;
-		else if (s=="blob_worker") _class = BlobWorkerClass;
-		else if (s=="backup") _class = BackupClass;
-		else if (s=="encrypt_key_proxy") _class = EncryptKeyProxyClass;
-		else if (s=="sim_http_server") _class = SimHTTPServerClass;
-		else _class = InvalidClass;
-	}
+	explicit ProcessClass(std::string s, ClassSource source);
 
-	ProcessClass( std::string classStr, std::string sourceStr ) {
-		if (classStr=="storage") _class = StorageClass;
-		else if (classStr=="transaction") _class = TransactionClass;
-		else if (classStr=="resolution") _class = ResolutionClass;
-		else if (classStr=="commit_proxy") _class = CommitProxyClass;
-		else if (classStr=="proxy") {
-			_class = CommitProxyClass;
-			printf("WARNING: 'proxy' machine class is deprecated and will be automatically converted "
-					"'commit_proxy' machine class. Please use 'grv_proxy' or 'commit_proxy' specifically\n");
-		}
-		else if (classStr=="grv_proxy") _class = GrvProxyClass;
-		else if (classStr=="master") _class = MasterClass;
-		else if (classStr=="test") _class = TesterClass;
-		else if (classStr=="unset") _class = UnsetClass;
-		else if (classStr=="stateless") _class = StatelessClass;
-		else if (classStr=="log") _class = LogClass;
-		else if (classStr=="router") _class = LogRouterClass;
-		else if (classStr=="cluster_controller") _class = ClusterControllerClass;
-		else if (classStr=="fast_restore") _class = FastRestoreClass;
-		else if (classStr=="data_distributor") _class = DataDistributorClass;
-		else if (classStr=="coordinator") _class = CoordinatorClass;
-		else if (classStr=="ratekeeper") _class = RatekeeperClass;
-		else if (classStr=="consistency_scan") _class = ConsistencyScanClass;
-		else if (classStr=="blob_manager") _class = BlobManagerClass;
-		else if (classStr=="blob_worker") _class = BlobWorkerClass;
-		else if (classStr=="backup") _class = BackupClass;
-		else if (classStr=="encrypt_key_proxy") _class = EncryptKeyProxyClass;
-		else if (classStr=="sim_http_server") _class = SimHTTPServerClass;
-		else _class = InvalidClass;
-
-		if (sourceStr=="command_line") _source = CommandLineSource;
-		else if (sourceStr=="configure_auto") _source = AutoSource;
-		else if (sourceStr=="set_class") _source = DBSource;
-		else _source = InvalidSource;
-	}
+	ProcessClass(std::string classStr, std::string sourceStr);
 
 	ClassType classType() const { return (ClassType)_class; }
 	ClassSource classSource() const { return (ClassSource)_source; }
 
-	bool operator == ( const ClassType& rhs ) const { return _class == rhs; }
-	bool operator != ( const ClassType& rhs ) const { return _class != rhs; }
+	bool operator==(const ClassType& rhs) const { return _class == rhs; }
+	bool operator!=(const ClassType& rhs) const { return _class != rhs; }
 
-	bool operator == ( const ProcessClass& rhs ) const { return _class == rhs._class && _source == rhs._source; }
-	bool operator != ( const ProcessClass& rhs ) const { return _class != rhs._class || _source != rhs._source; }
+	bool operator==(const ProcessClass& rhs) const { return _class == rhs._class && _source == rhs._source; }
+	bool operator!=(const ProcessClass& rhs) const { return _class != rhs._class || _source != rhs._source; }
 
-	std::string toString() const {
-		switch (_class) {
-			case UnsetClass: return "unset";
-			case StorageClass: return "storage";
-			case TransactionClass: return "transaction";
-			case ResolutionClass: return "resolution";
-			case CommitProxyClass: return "commit_proxy";
-			case GrvProxyClass: return "grv_proxy";
-			case MasterClass: return "master";
-			case TesterClass: return "test";
-			case StatelessClass: return "stateless";
-			case LogClass: return "log";
-			case LogRouterClass: return "router";
-			case ClusterControllerClass: return "cluster_controller";
-			case FastRestoreClass: return "fast_restore";
-			case DataDistributorClass: return "data_distributor";
-			case CoordinatorClass: return "coordinator";
-			case RatekeeperClass: return "ratekeeper";
-			case ConsistencyScanClass: return "consistency_scan";
-			case BlobManagerClass: return "blob_manager";
-			case BlobWorkerClass: return "blob_worker";
-			case BackupClass: return "backup";
-			case EncryptKeyProxyClass: return "encrypt_key_proxy";
-			case SimHTTPServerClass: return "sim_http_server";
-			default: return "invalid";
-		}
-	}
-	// clang-format on
-
-	std::string sourceString() const {
-		switch (_source) {
-		case CommandLineSource:
-			return "command_line";
-		case AutoSource:
-			return "configure_auto";
-		case DBSource:
-			return "set_class";
-		default:
-			return "invalid";
-		}
-	}
+	std::string toString() const;
+
+	std::string sourceString() const;
 
 	Fitness machineClassFitness(ClusterRole role) const;
 
@@ -447,7 +345,8 @@ struct LBLocalityData {
 // Template specialization that only works for interfaces with a .locality member.
 //   If an interface has a .locality it must also have a .address()
 template <class Interface>
-struct LBLocalityData<Interface, typename std::enable_if<Interface::LocationAwareLoadBalance>::type> {
+    requires(static_cast<bool>(Interface::LocationAwareLoadBalance))
+struct LBLocalityData<Interface> {
 	enum { Present = 1 };
 	static LocalityData getLocality(Interface const& i) { return i.locality; }
 	static NetworkAddress getAddress(Interface const& i) { return i.address(); }
diff --git a/fdbrpc/include/fdbrpc/MultiInterface.h b/fdbrpc/include/fdbrpc/MultiInterface.h
index a231809a2b2..cd3c2399984 100644
--- a/fdbrpc/include/fdbrpc/MultiInterface.h
+++ b/fdbrpc/include/fdbrpc/MultiInterface.h
@@ -61,7 +61,7 @@ struct ReferencedInterface : public ReferenceCounted<ReferencedInterface<T>> {
 	T interf;
 	int8_t distance; // one of enum values in struct LBDistance
 	std::string toString() const { return interf.toString(); }
-	ReferencedInterface(T const& interf, LocalityData const& locality = LocalityData()) : interf(interf) {
+	explicit ReferencedInterface(T const& interf, LocalityData const& locality = LocalityData()) : interf(interf) {
 		distance = LBLocalityData<T>::Present ? loadBalanceDistance(locality,
 		                                                            LBLocalityData<T>::getLocality(interf),
 		                                                            LBLocalityData<T>::getAddress(interf))
@@ -184,7 +184,7 @@ class ModelInterface : public ReferenceCounted<ModelInterface<T>> {
 
 template <class T>
 class MultiInterface : public ReferenceCounted<MultiInterface<T>> {
-	MultiInterface(const std::vector<T>& v, LocalityData const& locality = LocalityData()) {
+	explicit MultiInterface(const std::vector<T>& v, LocalityData const& locality = LocalityData()) {
 		// This version of MultInterface is no longer used, but was kept around because of templating
 		ASSERT(false);
 	}
@@ -195,7 +195,7 @@ class MultiInterface : public ReferenceCounted<MultiInterface<T>> {
 template <class T>
 class MultiInterface<ReferencedInterface<T>> : public ReferenceCounted<MultiInterface<ReferencedInterface<T>>> {
 public:
-	MultiInterface(const std::vector<Reference<ReferencedInterface<T>>>& v) : alternatives(v), bestCount(0) {
+	explicit MultiInterface(const std::vector<Reference<ReferencedInterface<T>>>& v) : alternatives(v), bestCount(0) {
 		deterministicRandom()->randomShuffle(alternatives);
 		if (LBLocalityData<T>::Present) {
 			std::stable_sort(alternatives.begin(), alternatives.end(), ReferencedInterface<T>::sort_by_distance);
diff --git a/fdbrpc/include/fdbrpc/Net2FileSystem.h b/fdbrpc/include/fdbrpc/Net2FileSystem.h
index bc590bdadf3..dcb60b6c26a 100644
--- a/fdbrpc/include/fdbrpc/Net2FileSystem.h
+++ b/fdbrpc/include/fdbrpc/Net2FileSystem.h
@@ -46,7 +46,7 @@ class Net2FileSystem final : public IAsyncFileSystem {
 	// void init();
 	static void stop();
 
-	Net2FileSystem(double ioTimeout = 0.0, const std::string& fileSystemPath = "");
+	explicit Net2FileSystem(double ioTimeout = 0.0, const std::string& fileSystemPath = "");
 
 	~Net2FileSystem() override {}
 
diff --git a/fdbrpc/include/fdbrpc/PerfMetric.h b/fdbrpc/include/fdbrpc/PerfMetric.h
index eaab2310e56..a6b80a24c77 100644
--- a/fdbrpc/include/fdbrpc/PerfMetric.h
+++ b/fdbrpc/include/fdbrpc/PerfMetric.h
@@ -59,7 +59,7 @@ struct PerfMetric {
 };
 
 struct PerfIntCounter {
-	PerfIntCounter(std::string name) : name(name), value(0) {}
+	explicit PerfIntCounter(std::string name) : name(name), value(0) {}
 	PerfIntCounter(std::string name, std::vector<PerfIntCounter*>& v) : name(name), value(0) { v.push_back(this); }
 	void operator+=(int64_t delta) { value += delta; }
 	void operator++() { value += 1; }
@@ -73,7 +73,7 @@ struct PerfIntCounter {
 };
 
 struct PerfDoubleCounter {
-	PerfDoubleCounter(std::string name) : name(name), value(0) {}
+	explicit PerfDoubleCounter(std::string name) : name(name), value(0) {}
 	PerfDoubleCounter(std::string name, std::vector<PerfDoubleCounter*>& v) : name(name), value(0) {
 		v.push_back(this);
 	}
diff --git a/fdbrpc/include/fdbrpc/RangeMap.h b/fdbrpc/include/fdbrpc/RangeMap.h
index a001c993d36..0084d0c71dd 100644
--- a/fdbrpc/include/fdbrpc/RangeMap.h
+++ b/fdbrpc/include/fdbrpc/RangeMap.h
@@ -211,7 +211,7 @@ class RangeMap {
 
 	void operator=(RangeMap&& r) noexcept { map = std::move(r.map); }
 
-	RangeMap(RangeMap&& source) = default;
+	explicit(false) RangeMap(RangeMap&& source) = default;
 	// void clear( const Val& value ) { ranges.clear(); ranges.insert(std::make_pair(Key(),value)); }
 	void clear() { map.clear(); }
 
diff --git a/fdbrpc/include/fdbrpc/Replication.h b/fdbrpc/include/fdbrpc/Replication.h
index 7a08814e6ef..db2acf810a0 100644
--- a/fdbrpc/include/fdbrpc/Replication.h
+++ b/fdbrpc/include/fdbrpc/Replication.h
@@ -28,11 +28,11 @@
 
 struct LocalitySet : public ReferenceCounted<LocalitySet> {
 public:
-	LocalitySet(LocalitySet const& source)
+	explicit(false) LocalitySet(LocalitySet const& source)
 	  : _keymap(source._keymap), _entryArray(source._entryArray), _mutableEntryArray(source._mutableEntryArray),
 	    _keyValueArray(source._keyValueArray), _keyIndexArray(source._keyIndexArray), _cacheArray(source._cacheArray),
 	    _localitygroup(source._localitygroup), _cachehits(source._cachehits), _cachemisses(source._cachemisses) {}
-	LocalitySet(LocalitySet& localityGroup)
+	explicit(false) LocalitySet(LocalitySet& localityGroup)
 	  : _keymap(new StringToIntMap()), _localitygroup(&localityGroup), _cachehits(0), _cachemisses(0) {}
 	virtual ~LocalitySet() {}
 
@@ -466,7 +466,7 @@ struct LocalitySet : public ReferenceCounted<LocalitySet> {
 		Reference<LocalitySet> _resultset;
 		LocalityCacheRecord(AttribRecord const& attribute, Reference<LocalitySet> resultset)
 		  : _attribute(attribute), _resultset(resultset) {}
-		LocalityCacheRecord(LocalityCacheRecord const& source)
+		explicit(false) LocalityCacheRecord(LocalityCacheRecord const& source)
 		  : _attribute(source._attribute), _resultset(source._resultset) {}
 		virtual ~LocalityCacheRecord() {}
 		LocalityCacheRecord& operator=(LocalityCacheRecord const& source) {
@@ -518,7 +518,7 @@ struct LocalitySet : public ReferenceCounted<LocalitySet> {
 
 struct LocalityGroup : public LocalitySet {
 	LocalityGroup() : LocalitySet(*this), _valuemap(new StringToIntMap()) {}
-	LocalityGroup(LocalityGroup const& source)
+	explicit(false) LocalityGroup(LocalityGroup const& source)
 	  : LocalitySet(source), _recordArray(source._recordArray), _valuemap(source._valuemap) {}
 	~LocalityGroup() override {}
 
@@ -596,7 +596,7 @@ struct LocalityGroup : public LocalitySet {
 template <class V>
 struct LocalityMap : public LocalityGroup {
 	LocalityMap() : LocalityGroup() {}
-	LocalityMap(LocalityMap const& source) : LocalityGroup(source), _objectArray(source._objectArray) {}
+	explicit(false) LocalityMap(LocalityMap const& source) : LocalityGroup(source), _objectArray(source._objectArray) {}
 	~LocalityMap() override {}
 
 	bool selectReplicas(Reference<IReplicationPolicy> const& policy,
diff --git a/fdbrpc/include/fdbrpc/ReplicationPolicy.h b/fdbrpc/include/fdbrpc/ReplicationPolicy.h
index 941955d981e..c1f7cac0ef1 100644
--- a/fdbrpc/include/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/include/fdbrpc/ReplicationPolicy.h
@@ -216,7 +216,8 @@ struct PolicyAcross final : IReplicationPolicy {
 
 struct PolicyAnd final : IReplicationPolicy {
 	friend struct serializable_traits<PolicyAnd*>;
-	PolicyAnd(std::vector<Reference<IReplicationPolicy>> policies) : _policies(policies), _sortedPolicies(policies) {
+	explicit PolicyAnd(std::vector<Reference<IReplicationPolicy>> policies)
+	  : _policies(policies), _sortedPolicies(policies) {
 		// Sort the policy array
 		std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
 	}
diff --git a/fdbrpc/include/fdbrpc/ReplicationTypes.h b/fdbrpc/include/fdbrpc/ReplicationTypes.h
index 766eaee7697..b4d69f2593c 100644
--- a/fdbrpc/include/fdbrpc/ReplicationTypes.h
+++ b/fdbrpc/include/fdbrpc/ReplicationTypes.h
@@ -68,8 +68,8 @@ struct KeyValueMap final : public ReferenceCounted<KeyValueMap> {
 	std::vector<AttribRecord> _keyvaluearray;
 
 	KeyValueMap() {}
-	KeyValueMap(const LocalityData& data);
-	KeyValueMap(const KeyValueMap& entry) : _keyvaluearray(entry._keyvaluearray) {}
+	explicit KeyValueMap(const LocalityData& data);
+	explicit(false) KeyValueMap(const KeyValueMap& entry) : _keyvaluearray(entry._keyvaluearray) {}
 	KeyValueMap& operator=(KeyValueMap const& source) {
 		_keyvaluearray = source._keyvaluearray;
 		return *this;
@@ -110,7 +110,8 @@ struct LocalityRecord final : public ReferenceCounted<LocalityRecord> {
 	LocalityEntry _entryIndex;
 	LocalityRecord(Reference<KeyValueMap> const& dataMap, int arrayIndex)
 	  : _dataMap(dataMap), _entryIndex(arrayIndex) {}
-	LocalityRecord(LocalityRecord const& entry) : _dataMap(entry._dataMap), _entryIndex(entry._entryIndex) {}
+	explicit(false) LocalityRecord(LocalityRecord const& entry)
+	  : _dataMap(entry._dataMap), _entryIndex(entry._entryIndex) {}
 	LocalityRecord& operator=(LocalityRecord const& source) {
 		_dataMap = source._dataMap;
 		_entryIndex = source._entryIndex;
@@ -143,7 +144,8 @@ struct StringToIntMap final : public ReferenceCounted<StringToIntMap> {
 	std::map<std::string, int> _hashmap;
 	std::vector<std::string> _lookuparray;
 	StringToIntMap() {}
-	StringToIntMap(StringToIntMap const& source) : _hashmap(source._hashmap), _lookuparray(source._lookuparray) {}
+	explicit(false) StringToIntMap(StringToIntMap const& source)
+	  : _hashmap(source._hashmap), _lookuparray(source._lookuparray) {}
 	StringToIntMap& operator=(StringToIntMap const& source) {
 		_hashmap = source._hashmap;
 		_lookuparray = source._lookuparray;
diff --git a/fdbrpc/include/fdbrpc/SimulatorKillType.h b/fdbrpc/include/fdbrpc/SimulatorKillType.h
index ce384781586..3b4e525ce93 100644
--- a/fdbrpc/include/fdbrpc/SimulatorKillType.h
+++ b/fdbrpc/include/fdbrpc/SimulatorKillType.h
@@ -23,8 +23,8 @@
 
 namespace simulator {
 
-// Order matters!
-// FIXME: explain why it matters.
+// Order matters: simulation code compares kill types to decide whether one fault mode is
+// more destructive than another.
 enum KillType {
 	KillInstantly,
 	InjectFaults,
diff --git a/fdbrpc/include/fdbrpc/SimulatorProcessInfo.h b/fdbrpc/include/fdbrpc/SimulatorProcessInfo.h
index 49cbec09e4d..ea0cedada22 100644
--- a/fdbrpc/include/fdbrpc/SimulatorProcessInfo.h
+++ b/fdbrpc/include/fdbrpc/SimulatorProcessInfo.h
@@ -27,7 +27,7 @@
 #include "flow/NetworkAddress.h"
 #include "flow/IConnection.h"
 #include "flow/IUDPSocket.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/ChaosMetrics.h"
 #include "flow/Histogram.h"
 
diff --git a/fdbrpc/include/fdbrpc/Stats.h b/fdbrpc/include/fdbrpc/Stats.h
index beee64eb630..df5fa120c1a 100644
--- a/fdbrpc/include/fdbrpc/Stats.h
+++ b/fdbrpc/include/fdbrpc/Stats.h
@@ -44,7 +44,7 @@ MyCounters() : foo("foo", cc), bar("bar", cc), baz("baz", cc) {}
 #include <cstdint>
 #include <cstddef>
 #include "flow/flow.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "fdbrpc/DDSketch.h"
 
 struct ICounter : public IMetric {
@@ -86,7 +86,7 @@ class CounterCollection {
 	double logTime;
 
 public:
-	CounterCollection(std::string const& name, std::string const& id = std::string())
+	explicit CounterCollection(std::string const& name, std::string const& id = std::string())
 	  : name(name), id(id), logTime(0) {}
 	~CounterCollection() {
 		for (auto c : countersToRemove)
@@ -220,7 +220,7 @@ class LatencyBands {
 	    double loggingInterval,
 	    std::function<void(TraceEvent&)> const& decorator = [](auto&) {});
 
-	LatencyBands(LatencyBands&&) = default;
+	explicit(false) LatencyBands(LatencyBands&&) = default;
 	LatencyBands& operator=(LatencyBands&&) = default;
 
 	void addThreshold(double value);
diff --git a/fdbrpc/include/fdbrpc/TimedRequest.h b/fdbrpc/include/fdbrpc/TimedRequest.h
index e03b99c65b8..21248981807 100644
--- a/fdbrpc/include/fdbrpc/TimedRequest.h
+++ b/fdbrpc/include/fdbrpc/TimedRequest.h
@@ -34,6 +34,8 @@ class TimedRequest {
 		return _requestTime;
 	}
 
+	void setRequestTime(double requestTime) { _requestTime = requestTime; }
+
 	TimedRequest() {
 		if (!FlowTransport::isClient()) {
 			_requestTime = g_network->timer();
diff --git a/fdbrpc/include/fdbrpc/fdbrpc.h b/fdbrpc/include/fdbrpc/fdbrpc.h
index ce9e73fe0b6..e0c5be397b2 100644
--- a/fdbrpc/include/fdbrpc/fdbrpc.h
+++ b/fdbrpc/include/fdbrpc/fdbrpc.h
@@ -27,7 +27,7 @@
 #include "flow/serialize.h"
 #include "fdbrpc/FlowTransport.h" // NetworkMessageReceiver Endpoint
 #include "fdbrpc/FailureMonitor.h"
-#include "fdbrpc/networksender.actor.h"
+#include "fdbrpc/networksender.h"
 #include "fdbrpc/simulator.h"
 #ifdef WITH_SWIFT
 #include <swift/bridging>
@@ -158,14 +158,14 @@ class ReplyPromise final : public ComposedIdentifier<T, 1> {
 	explicit ReplyPromise(const PeerCompatibilityPolicy& policy) : ReplyPromise() {
 		sav->setPeerCompatibilityPolicy(policy);
 	}
-	ReplyPromise(const ReplyPromise& rhs) : sav(rhs.sav) { sav->addPromiseRef(); }
-	ReplyPromise(ReplyPromise&& rhs) noexcept : sav(rhs.sav) { rhs.sav = 0; }
+	explicit(false) ReplyPromise(const ReplyPromise& rhs) : sav(rhs.sav) { sav->addPromiseRef(); }
+	explicit(false) ReplyPromise(ReplyPromise&& rhs) noexcept : sav(rhs.sav) { rhs.sav = 0; }
 	~ReplyPromise() {
 		if (sav)
 			sav->delPromiseRef();
 	}
 
-	ReplyPromise(const Endpoint& endpoint) : sav(new NetSAV<T>(0, 1, endpoint)) {}
+	explicit ReplyPromise(const Endpoint& endpoint) : sav(new NetSAV<T>(0, 1, endpoint)) {}
 	const Endpoint& getEndpoint(TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint) const {
 		return sav->getEndpoint(taskID);
 	}
@@ -215,7 +215,7 @@ void load(Ar& ar, ReplyPromise<T>& value) {
 	ar >> token;
 	Endpoint endpoint = FlowTransport::transport().loadedEndpoint(token);
 	value = ReplyPromise<T>(endpoint);
-	networkSender(value.getFuture(), endpoint);
+	networkSender(Uncancellable(), value.getFuture(), endpoint);
 }
 
 template <class T>
@@ -227,7 +227,7 @@ struct serializable_traits<ReplyPromise<T>> : std::true_type {
 			serializer(ar, token);
 			auto endpoint = FlowTransport::transport().loadedEndpoint(token);
 			p = ReplyPromise<T>(endpoint);
-			networkSender(p.getFuture(), endpoint);
+			networkSender(Uncancellable(), p.getFuture(), endpoint);
 		} else {
 			const auto& ep = p.getEndpoint().token;
 			serializer(ar, ep);
@@ -310,7 +310,8 @@ struct AcknowledgementReceiver final : FlowReceiver, FastAllocated<Acknowledgeme
 	Future<Void> failures;
 
 	AcknowledgementReceiver() : ready(nullptr) {}
-	AcknowledgementReceiver(const Endpoint& remoteEndpoint) : FlowReceiver(remoteEndpoint, false), ready(nullptr) {}
+	explicit AcknowledgementReceiver(const Endpoint& remoteEndpoint)
+	  : FlowReceiver(remoteEndpoint, false), ready(nullptr) {}
 
 	bool isPublic() const override { return true; }
 
@@ -502,13 +503,13 @@ class ReplyPromiseStream {
 		return FutureStream<T>(queue);
 	}
 	ReplyPromiseStream() : queue(new NetNotifiedQueueWithAcknowledgements<T>(0, 1)), errors(new SAV<Void>(0, 1)) {}
-	ReplyPromiseStream(const ReplyPromiseStream& rhs) : queue(rhs.queue), errors(rhs.errors) {
+	explicit(false) ReplyPromiseStream(const ReplyPromiseStream& rhs) : queue(rhs.queue), errors(rhs.errors) {
 		queue->addPromiseRef();
 		if (errors) {
 			errors->addPromiseRef();
 		}
 	}
-	ReplyPromiseStream(ReplyPromiseStream&& rhs) noexcept : queue(rhs.queue), errors(rhs.errors) {
+	explicit(false) ReplyPromiseStream(ReplyPromiseStream&& rhs) noexcept : queue(rhs.queue), errors(rhs.errors) {
 		rhs.queue = nullptr;
 		rhs.errors = nullptr;
 	}
@@ -687,7 +688,8 @@ struct HasVerify_t<T, decltype(void(std::declval<T>().verify()), 0)> : std::true
 template <class T>
 constexpr bool HasVerify = HasVerify_t<T>::value;
 
-// FIXME: explain what IsPublic means here
+// IsPublic streams accept messages from clients outside the cluster and verify
+// each request before delivering it to the local queue.
 template <class T, bool IsPublic>
 struct NetNotifiedQueue final : NotifiedQueue<T>, FlowReceiver, FastAllocated<NetNotifiedQueue<T, IsPublic>> {
 	using FastAllocated<NetNotifiedQueue<T, IsPublic>>::operator new;
@@ -848,7 +850,7 @@ class RequestStream {
 			} else {
 				Reference<Peer> peer =
 				    FlowTransport::transport().sendUnreliable(SerializeSource<T>(value), getEndpoint(), true);
-				endStreamOnDisconnect(disc, p, getEndpoint(), peer);
+				endStreamOnDisconnect(Uncancellable(), disc, p, getEndpoint(), peer);
 			}
 			return p;
 		} else {
@@ -905,8 +907,8 @@ class RequestStream {
 	explicit RequestStream(PeerCompatibilityPolicy policy) : RequestStream() {
 		queue->setPeerCompatibilityPolicy(policy);
 	}
-	RequestStream(const RequestStream& rhs) : queue(rhs.queue) { queue->addPromiseRef(); }
-	RequestStream(RequestStream&& rhs) noexcept : queue(rhs.queue) { rhs.queue = 0; }
+	explicit(false) RequestStream(const RequestStream& rhs) : queue(rhs.queue) { queue->addPromiseRef(); }
+	explicit(false) RequestStream(RequestStream&& rhs) noexcept : queue(rhs.queue) { rhs.queue = 0; }
 	void operator=(const RequestStream& rhs) {
 		rhs.queue->addPromiseRef();
 		if (queue)
@@ -948,7 +950,8 @@ class RequestStream {
 	NetNotifiedQueue<T, IsPublic>* queue;
 };
 
-// FIXME: explain what Public and Private mean here
+// Public request streams require T::verify() and reject unauthorized messages.
+// Private request streams are used for trusted intra-cluster traffic.
 template <class T>
 using PrivateRequestStream = RequestStream<T, false>;
 template <class T>
@@ -989,4 +992,4 @@ struct serializable_traits<RequestStream<T, P>> : std::true_type {
 };
 
 #endif
-#include "fdbrpc/genericactors.actor.h"
+#include "fdbrpc/genericactors.h"
diff --git a/fdbrpc/include/fdbrpc/genericactors.actor.h b/fdbrpc/include/fdbrpc/genericactors.h
similarity index 53%
rename from fdbrpc/include/fdbrpc/genericactors.actor.h
rename to fdbrpc/include/fdbrpc/genericactors.h
index 0ae5739f9a4..3cf65870ffc 100644
--- a/fdbrpc/include/fdbrpc/genericactors.actor.h
+++ b/fdbrpc/include/fdbrpc/genericactors.h
@@ -1,5 +1,5 @@
 /*
- * genericactors.actor.h
+ * genericactors.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,7 @@
  * limitations under the License.
  */
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBRPC_GENERICACTORS_ACTOR_G_H)
-#define FDBRPC_GENERICACTORS_ACTOR_G_H
-#include "fdbrpc/genericactors.actor.g.h"
-#elif !defined(RPCGENERICACTORS_ACTOR_H)
-#define RPCGENERICACTORS_ACTOR_H
+#pragma once
 
 #include "flow/genericactors.actor.h"
 #include "flow/CodeProbe.h"
@@ -33,74 +27,63 @@
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/WellKnownEndpoints.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include <type_traits>
+
+#include "flow/CoroUtils.h"
 
 // To avoid directly access INetworkConnection::net()->removeCachedDNS(), which will require heavy include budget, put
-// the call to FlowTransport.actor.cpp as a external function.
+// the call to FlowTransport.cpp as a external function.
 extern void removeCachedDNS(const std::string& host, const std::string& service);
 
-ACTOR template <class Req, bool P>
+template <class Req, bool P>
 Future<REPLY_TYPE(Req)> retryBrokenPromise(RequestStream<Req, P> to, Req request) {
 	// Like to.getReply(request), except that a broken_promise exception results in retrying request immediately.
 	// Suitable for use with well known endpoints, which are likely to return to existence after the other process
 	// restarts. Not normally useful for ordinary endpoints, which conventionally are permanently destroyed after
 	// replying with broken_promise.
-	loop {
+	while (true) {
 		try {
-			REPLY_TYPE(Req) reply = wait(to.getReply(request));
-			return reply;
+			co_return co_await to.getReply(request);
 		} catch (Error& e) {
 			if (e.code() != error_code_broken_promise)
 				throw;
-			resetReply(request);
-			wait(delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			CODE_PROBE(true, "retryBrokenPromise");
 		}
+		resetReply(request);
+		co_await delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+		CODE_PROBE(true, "retryBrokenPromise");
 	}
 }
 
-ACTOR template <class Req, bool P>
+template <class Req, bool P>
 Future<REPLY_TYPE(Req)> retryBrokenPromise(RequestStream<Req, P> to, Req request, TaskPriority taskID) {
 	// Like to.getReply(request), except that a broken_promise exception results in retrying request immediately.
 	// Suitable for use with well known endpoints, which are likely to return to existence after the other process
 	// restarts. Not normally useful for ordinary endpoints, which conventionally are permanently destroyed after
 	// replying with broken_promise.
-	loop {
+	while (true) {
 		try {
-			REPLY_TYPE(Req) reply = wait(to.getReply(request, taskID));
-			return reply;
+			co_return co_await to.getReply(request, taskID);
 		} catch (Error& e) {
 			if (e.code() != error_code_broken_promise)
 				throw;
-			resetReply(request);
-			wait(delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY, taskID));
-			CODE_PROBE(true, "retryBrokenPromise with taskID");
 		}
+		resetReply(request);
+		co_await delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY, taskID);
+		CODE_PROBE(true, "retryBrokenPromise with taskID");
 	}
 }
 
-ACTOR template <class Req>
-Future<Void> tryInitializeRequestStream(RequestStream<Req>* stream, Hostname hostname, WellKnownEndpoints token) {
-	Optional<NetworkAddress> address = wait(hostname.resolve());
-	if (!address.present()) {
-		return Void();
-	}
-	ASSERT(stream != nullptr);
-	*stream = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
-	return Void();
-}
-
-ACTOR template <class Req>
+template <class Req>
 Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request, Hostname hostname, WellKnownEndpoints token) {
 	// A wrapper of tryGetReply(request), except that the request is sent to an address resolved from a hostname.
 	// If resolving fails, return lookup_failed().
 	// Otherwise, return tryGetReply(request).
-	Optional<NetworkAddress> address = wait(hostname.resolve());
+	Optional<NetworkAddress> address = co_await hostname.resolve();
 	if (!address.present()) {
-		return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
+		co_return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
 	}
 	RequestStream<Req> to(Endpoint::wellKnown({ address.get() }, token));
-	state ErrorOr<REPLY_TYPE(Req)> reply = wait(to.tryGetReply(request));
+	ErrorOr<REPLY_TYPE(Req)> reply = co_await to.tryGetReply(request);
 	if (reply.isError()) {
 		resetReply(request);
 		if (reply.getError().code() == error_code_request_maybe_delivered) {
@@ -108,10 +91,10 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request, Hostname h
 			removeCachedDNS(hostname.host, hostname.service);
 		}
 	}
-	return reply;
+	co_return reply;
 }
 
-ACTOR template <class Req>
+template <class Req>
 Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request,
                                                          Hostname hostname,
                                                          WellKnownEndpoints token,
@@ -119,12 +102,12 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request,
 	// A wrapper of tryGetReply(request), except that the request is sent to an address resolved from a hostname.
 	// If resolving fails, return lookup_failed().
 	// Otherwise, return tryGetReply(request).
-	Optional<NetworkAddress> address = wait(hostname.resolve());
+	Optional<NetworkAddress> address = co_await hostname.resolve();
 	if (!address.present()) {
-		return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
+		co_return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
 	}
 	RequestStream<Req> to(Endpoint::wellKnown({ address.get() }, token));
-	state ErrorOr<REPLY_TYPE(Req)> reply = wait(to.tryGetReply(request, taskID));
+	ErrorOr<REPLY_TYPE(Req)> reply = co_await to.tryGetReply(request, taskID);
 	if (reply.isError()) {
 		resetReply(request);
 		if (reply.getError().code() == error_code_request_maybe_delivered) {
@@ -132,59 +115,63 @@ Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(Req request,
 			removeCachedDNS(hostname.host, hostname.service);
 		}
 	}
-	return reply;
+	co_return reply;
 }
 
-ACTOR template <class Req>
-Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request, Hostname hostname, WellKnownEndpoints token) {
+template <class Req>
+Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request,
+                                                  Hostname hostname,
+                                                  WellKnownEndpoints token,
+                                                  ExplicitVoid = {}) {
 	// Like tryGetReplyFromHostname, except that request_maybe_delivered results in re-resolving the hostname.
 	// Suitable for use with hostname, where RequestStream is NOT initialized yet.
 	// Not normally useful for endpoints initialized with NetworkAddress.
-	state double reconnectInterval = FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL;
-	state std::unique_ptr<RequestStream<Req>> to;
-	loop {
-		NetworkAddress address = wait(hostname.resolveWithRetry());
+	double reconnectInterval = FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL;
+	std::unique_ptr<RequestStream<Req>> to;
+	while (true) {
+		NetworkAddress address = co_await hostname.resolveWithRetry();
 		if (to == nullptr || to->getEndpoint().getPrimaryAddress() != address) {
 			to = std::make_unique<RequestStream<Req>>(Endpoint::wellKnown({ address }, token));
 		}
-		state ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request));
+		ErrorOr<REPLY_TYPE(Req)> reply = co_await to->tryGetReply(request);
 		if (reply.isError()) {
 			resetReply(request);
 			if (reply.getError().code() == error_code_request_maybe_delivered) {
 				// Connection failure.
-				wait(delay(reconnectInterval));
+				co_await delay(reconnectInterval);
 				reconnectInterval = std::min(2 * reconnectInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
 				removeCachedDNS(hostname.host, hostname.service);
 			} else {
 				throw reply.getError();
 			}
 		} else {
-			return reply.get();
+			co_return reply.get();
 		}
 	}
 }
 
-ACTOR template <class Req>
+template <class Req>
 Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request,
                                                   Hostname hostname,
                                                   WellKnownEndpoints token,
-                                                  TaskPriority taskID) {
+                                                  TaskPriority taskID,
+                                                  ExplicitVoid = {}) {
 	// Like tryGetReplyFromHostname, except that request_maybe_delivered results in re-resolving the hostname.
 	// Suitable for use with hostname, where RequestStream is NOT initialized yet.
 	// Not normally useful for endpoints initialized with NetworkAddress.
-	state double reconnectInitInterval = FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL;
-	state std::unique_ptr<RequestStream<Req>> to;
-	loop {
-		NetworkAddress address = wait(hostname.resolveWithRetry());
+	double reconnectInitInterval = FLOW_KNOBS->HOSTNAME_RECONNECT_INIT_INTERVAL;
+	std::unique_ptr<RequestStream<Req>> to;
+	while (true) {
+		NetworkAddress address = co_await hostname.resolveWithRetry();
 		if (to == nullptr || to->getEndpoint().getPrimaryAddress() != address) {
 			to = std::make_unique<RequestStream<Req>>(Endpoint::wellKnown({ address }, token));
 		}
-		state ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request, taskID));
+		ErrorOr<REPLY_TYPE(Req)> reply = co_await to->tryGetReply(request, taskID);
 		if (reply.isError()) {
 			resetReply(request);
 			if (reply.getError().code() == error_code_request_maybe_delivered) {
 				// Connection failure.
-				wait(delay(reconnectInitInterval));
+				co_await delay(reconnectInitInterval);
 				reconnectInitInterval =
 				    std::min(2 * reconnectInitInterval, FLOW_KNOBS->HOSTNAME_RECONNECT_MAX_INTERVAL);
 				removeCachedDNS(hostname.host, hostname.service);
@@ -192,121 +179,132 @@ Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(Req request,
 				throw reply.getError();
 			}
 		} else {
-			return reply.get();
+			co_return reply.get();
 		}
 	}
 }
 
-ACTOR template <class T>
-Future<T> timeoutWarning(Future<T> what, double time, PromiseStream<Void> output) {
-	state Future<Void> end = delay(time);
-	loop choose {
-		when(T t = wait(what)) {
-			return t;
-		}
-		when(wait(end)) {
+template <class T>
+Future<T> timeoutWarning(Future<T> what, double time, PromiseStream<Void> output, ExplicitVoid = {}) {
+	Future<Void> end = delay(time);
+	while (true) {
+		auto res = co_await race(what, end);
+		if (res.index() == 0) {
+			co_return std::get<0>(std::move(res));
+		} else if (res.index() == 1) {
 			output.send(Void());
 			end = delay(time);
 		}
 	}
 }
 
-ACTOR template <class T>
-void forwardPromise(Promise<T> output, Future<T> input) {
+template <class T>
+Future<Void> forwardPromise(Uncancellable, Promise<T> output, Future<T> input, ExplicitVoid = {}) {
 	try {
-		T value = wait(input);
-		output.send(value);
+		T value = co_await input;
+		output.send(std::move(value));
 	} catch (Error& err) {
 		output.sendError(err);
 	}
+	co_return Void();
 }
 
-ACTOR template <class T>
-void forwardPromise(ReplyPromise<T> output, Future<T> input) {
+template <class T>
+Future<Void> forwardPromise(Uncancellable, ReplyPromise<T> output, Future<T> input, ExplicitVoid = {}) {
 	try {
-		T value = wait(input);
-		output.send(value);
+		T value = co_await input;
+		output.send(std::move(value));
 	} catch (Error& err) {
 		output.sendError(err);
 	}
+	co_return Void();
 }
 
-ACTOR template <class T>
-void forwardPromise(PromiseStream<T> output, Future<T> input) {
+template <class T>
+Future<Void> forwardPromise(Uncancellable, PromiseStream<T> output, Future<T> input, ExplicitVoid = {}) {
 	try {
-		T value = wait(input);
-		output.send(value);
+		T value = co_await input;
+		output.send(std::move(value));
 	} catch (Error& e) {
 		output.sendError(e);
 	}
+	co_return Void();
 }
 
-ACTOR template <class T>
-Future<Void> broadcast(Future<T> input, std::vector<Promise<T>> output) {
-	T value = wait(input);
-	for (int i = 0; i < output.size(); i++)
+template <class T>
+Future<Void> broadcast(Future<T> input, std::vector<Promise<T>> output, ExplicitVoid = {}) {
+	T value = co_await input;
+	for (int i = 0; i < output.size(); i++) {
 		output[i].send(value);
-	return Void();
+	}
+	co_return Void();
 }
 
-ACTOR template <class T>
-Future<Void> broadcast(Future<T> input, std::vector<ReplyPromise<T>> output) {
-	T value = wait(input);
-	for (int i = 0; i < output.size(); i++)
+template <class T>
+Future<Void> broadcast(Future<T> input, std::vector<ReplyPromise<T>> output, ExplicitVoid = {}) {
+	T value = co_await input;
+	for (int i = 0; i < output.size(); i++) {
 		output[i].send(value);
-	return Void();
+	}
+	co_return Void();
 }
 
-ACTOR template <class T>
-Future<Void> incrementalBroadcast(Future<T> input, std::vector<Promise<T>> output, int batchSize) {
-	state T value = wait(input);
-	state int i = 0;
-	for (; i < output.size(); i++) {
+template <class T>
+Future<Void> incrementalBroadcast(Future<T> input, std::vector<Promise<T>> output, int batchSize, ExplicitVoid = {}) {
+	T value = co_await input;
+	for (int i = 0; i < output.size(); i++) {
 		output[i].send(value);
 		if ((i + 1) % batchSize == 0) {
-			wait(delay(0));
+			co_await delay(0);
 		}
 	}
-	return Void();
+	co_return Void();
 }
 
-ACTOR template <class T>
-Future<Void> incrementalBroadcast(Future<T> input, std::vector<ReplyPromise<T>> output, int batchSize) {
-	state T value = wait(input);
-	state int i = 0;
-	for (; i < output.size(); i++) {
+template <class T>
+Future<Void> incrementalBroadcast(Future<T> input,
+                                  std::vector<ReplyPromise<T>> output,
+                                  int batchSize,
+                                  ExplicitVoid = {}) {
+	T value = co_await input;
+	for (int i = 0; i < output.size(); i++) {
 		output[i].send(value);
 		if ((i + 1) % batchSize == 0) {
-			wait(delay(0));
+			co_await delay(0);
 		}
 	}
-	return Void();
+	co_return Void();
 }
 
-ACTOR template <class T>
-Future<Void> incrementalBroadcastWithError(Future<T> input, std::vector<Promise<T>> output, int batchSize) {
-	state int i = 0;
+template <class T>
+Future<Void> incrementalBroadcastWithError(Future<T> input,
+                                           std::vector<Promise<T>> output,
+                                           int batchSize,
+                                           ExplicitVoid = {}) {
+	int i = 0;
+	Error err;
 	try {
-		state T value = wait(input);
+		T value = co_await input;
 		for (; i < output.size(); i++) {
 			output[i].send(value);
 			if ((i + 1) % batchSize == 0) {
-				wait(delay(0));
+				co_await delay(0);
 			}
 		}
-	} catch (Error& _e) {
-		if (_e.code() == error_code_operation_cancelled) {
-			throw _e;
-		}
-		state Error e = _e;
-		for (; i < output.size(); i++) {
-			output[i].sendError(e);
-			if ((i + 1) % batchSize == 0) {
-				wait(delay(0));
-			}
+		co_return Void();
+	} catch (Error& e) {
+		err = e;
+	}
+	if (err.code() == error_code_operation_cancelled) {
+		throw err;
+	}
+	for (; i < output.size(); i++) {
+		output[i].sendError(err);
+		if ((i + 1) % batchSize == 0) {
+			co_await delay(0);
 		}
 	}
-	return Void();
+	co_return Void();
 }
 
 // Needed for the call to endpointNotFound()
@@ -326,65 +324,77 @@ struct PeerHolder {
 	}
 };
 
-// Implements getReplyStream, this a void actor with the same lifetime as the input ReplyPromiseStream.
-// Because this actor holds a reference to the stream, normally it would be impossible to know when there are no other
-// references. To get around this, there is a SAV inside the stream that has one less promise reference than it should
-// (caused by getErrorFutureAndDelPromiseRef()). When that SAV gets a broken promise because no one besides this void
-// actor is referencing it, this void actor will get a broken_promise dropping the final reference to the full
+// Implements getReplyStream, this coroutine has the same lifetime as the input ReplyPromiseStream.
+// Because this coroutine holds a reference to the stream, normally it would be impossible to know when there are no
+// other references. To get around this, there is a SAV inside the stream that has one less promise reference than it
+// should (caused by getErrorFutureAndDelPromiseRef()). When that SAV gets a broken promise because no one besides this
+// coroutine is referencing it, this coroutine will get a broken_promise dropping the final reference to the full
 // ReplyPromiseStream
-ACTOR template <class X>
-void endStreamOnDisconnect(Future<Void> signal,
-                           ReplyPromiseStream<X> stream,
-                           Endpoint endpoint,
-                           Reference<Peer> peer = Reference<Peer>()) {
-	state PeerHolder holder = PeerHolder(peer);
+template <class X>
+Future<Void> endStreamOnDisconnect(Uncancellable,
+                                   Future<Void> signal,
+                                   ReplyPromiseStream<X> stream,
+                                   Endpoint endpoint,
+                                   Reference<Peer> peer = Reference<Peer>()) {
+	PeerHolder holder = PeerHolder(peer);
 	stream.setRequestStreamEndpoint(endpoint);
+	Error err;
 	try {
-		choose {
-			when(wait(signal)) {
-				stream.sendError(connection_failed());
-			}
-			when(wait(peer.isValid() ? peer->disconnect.getFuture() : Never())) {
-				stream.sendError(connection_failed());
-			}
-			when(wait(stream.getErrorFutureAndDelPromiseRef())) {}
+		auto res = co_await race(
+		    signal, peer.isValid() ? peer->disconnect.getFuture() : Never(), stream.getErrorFutureAndDelPromiseRef());
+		if (res.index() == 0) {
+			stream.sendError(connection_failed());
+		} else if (res.index() == 1) {
+			stream.sendError(connection_failed());
 		}
+		co_return;
 	} catch (Error& e) {
-		if (e.code() == error_code_broken_promise) {
-			// getErrorFutureAndDelPromiseRef returned, wait on stream connect or error
-			if (!stream.connected()) {
-				wait(signal || stream.onConnected());
-			}
+		err = e;
+	}
+	if (err.code() == error_code_broken_promise) {
+		// getErrorFutureAndDelPromiseRef returned, wait on stream connect or error
+		if (!stream.connected()) {
+			co_await (signal || stream.onConnected());
 		}
-		// Notify BEFORE dropping last reference, causing broken_promise to send on stream before destructor is called
-		stream.notifyFailed();
 	}
+	// Notify BEFORE dropping last reference, causing broken_promise to send on stream before destructor is called
+	stream.notifyFailed();
 }
 
 // Implements tryGetReply, getReplyUnlessFailedFor
-ACTOR template <class X>
+template <class X>
 Future<ErrorOr<X>> waitValueOrSignal(Future<X> value,
                                      Future<Void> signal,
                                      Endpoint endpoint,
                                      ReplyPromise<X> holdme = ReplyPromise<X>(),
                                      Reference<Peer> peer = Reference<Peer>()) {
-	state PeerHolder holder = PeerHolder(peer);
-	loop {
+	PeerHolder holder = PeerHolder(peer);
+	while (true) {
 		try {
-			choose {
-				when(X x = wait(value)) {
-					return x;
-				}
-				when(wait(signal)) {
-					return ErrorOr<X>(IFailureMonitor::failureMonitor().knownUnauthorized(endpoint)
-					                      ? unauthorized_attempt()
-					                      : request_maybe_delivered());
-				}
+			auto res = co_await race(value, signal, peer.isValid() ? peer->disconnect.getFuture() : Never());
+			if (res.index() == 0) {
+				X x = std::get<0>(std::move(res));
+
+				co_return x;
+			} else if (res.index() == 1) {
+
+				co_return ErrorOr<X>(IFailureMonitor::failureMonitor().knownUnauthorized(endpoint)
+				                         ? unauthorized_attempt()
+				                         : request_maybe_delivered());
+			} else if (res.index() == 2) {
+				CODE_PROBE(true, "waitValueOrSignal detected peer disconnect");
+				TraceEvent("WaitValueOrSignalPeerDisconnect")
+				    .suppressFor(1.0)
+				    .detail("Endpoint", endpoint.getPrimaryAddress())
+				    .detail("Token", endpoint.token);
+				co_return ErrorOr<X>(IFailureMonitor::failureMonitor().knownUnauthorized(endpoint)
+				                         ? unauthorized_attempt()
+				                         : request_maybe_delivered());
 			}
 		} catch (Error& e) {
 			if (signal.isError()) {
 				TraceEvent(SevError, "WaitValueOrSignalError").error(signal.getError());
-				return ErrorOr<X>(internal_error());
+				co_return ErrorOr<X>(internal_error());
 			}
 
 			if (e.code() == error_code_actor_cancelled)
@@ -393,34 +403,34 @@ Future<ErrorOr<X>> waitValueOrSignal(Future<X> value,
 			// broken_promise error normally means an endpoint failure, which in tryGetReply has the same semantics as
 			// receiving the failure signal
 			if (e.code() != error_code_broken_promise || signal.isError())
-				return ErrorOr<X>(e);
+				co_return ErrorOr<X>(e);
 			IFailureMonitor::failureMonitor().endpointNotFound(endpoint);
 			value = Never();
 		}
 	}
 }
 
-ACTOR template <class T>
-Future<T> sendCanceler(ReplyPromise<T> reply, ReliablePacket* send, Endpoint endpoint) {
-	state bool didCancelReliable = false;
+template <class T>
+Future<T> sendCanceler(ReplyPromise<T> reply, ReliablePacket* send, Endpoint endpoint, ExplicitVoid = {}) {
+	bool didCancelReliable = false;
 	try {
-		loop {
+		while (true) {
 			if (IFailureMonitor::failureMonitor().permanentlyFailed(endpoint)) {
 				FlowTransport::transport().cancelReliable(send);
 				didCancelReliable = true;
 				if (IFailureMonitor::failureMonitor().knownUnauthorized(endpoint)) {
 					throw unauthorized_attempt();
 				} else {
-					wait(Never());
+					co_await Future<Void>(Never());
 				}
 			}
-			choose {
-				when(T t = wait(reply.getFuture())) {
-					FlowTransport::transport().cancelReliable(send);
-					didCancelReliable = true;
-					return t;
-				}
-				when(wait(IFailureMonitor::failureMonitor().onStateChanged(endpoint))) {}
+			auto res = co_await race(reply.getFuture(), IFailureMonitor::failureMonitor().onStateChanged(endpoint));
+			if (res.index() == 0) {
+				T t = std::get<0>(std::move(res));
+
+				FlowTransport::transport().cancelReliable(send);
+				didCancelReliable = true;
+				co_return t;
 			}
 		}
 	} catch (Error& e) {
@@ -434,11 +444,10 @@ Future<T> sendCanceler(ReplyPromise<T> reply, ReliablePacket* send, Endpoint end
 	}
 }
 
-ACTOR template <class X>
-Future<X> reportEndpointFailure(Future<X> value, Endpoint endpoint) {
+template <class X>
+Future<X> reportEndpointFailure(Future<X> value, Endpoint endpoint, ExplicitVoid = {}) {
 	try {
-		X x = wait(value);
-		return x;
+		co_return co_await value;
 	} catch (Error& e) {
 		if (e.code() == error_code_broken_promise) {
 			IFailureMonitor::failureMonitor().endpointNotFound(endpoint);
@@ -446,7 +455,3 @@ Future<X> reportEndpointFailure(Future<X> value, Endpoint endpoint) {
 		throw;
 	}
 }
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbrpc/include/fdbrpc/grpc/AsyncGrpcClient.h b/fdbrpc/include/fdbrpc/grpc/AsyncGrpcClient.h
index d3c0456401a..72a3bff7eac 100644
--- a/fdbrpc/include/fdbrpc/grpc/AsyncGrpcClient.h
+++ b/fdbrpc/include/fdbrpc/grpc/AsyncGrpcClient.h
@@ -108,7 +108,7 @@ class AsyncGrpcClient {
 				promise.send(response);
 			} else {
 				// std::cout << "Error: " << status.error_message() << std::endl;
-				promise.sendError(grpc_error()); // TODO (Vishesh): Propogate the gRPC error codes.
+				promise.sendError(grpc_error()); // TODO (Vishesh): Propagate the gRPC error codes.
 			}
 		});
 
@@ -141,7 +141,7 @@ class AsyncGrpcClient {
 				std::cout << "sent end of stream from client.\n";
 				promise.sendError(end_of_stream());
 			} else {
-				promise.sendError(grpc_error()); // TODO (Vishesh): Propogate the gRPC error codes.
+				promise.sendError(grpc_error()); // TODO (Vishesh): Propagate the gRPC error codes.
 			}
 		});
 
@@ -164,7 +164,7 @@ class AsyncGrpcClient {
 	// 		if (status.ok()) {
 	// 			promise->sendError(end_of_stream());
 	// 		} else {
-	// 			promise->sendError(grpc_error()); // TODO (Vishesh): Propogate the gRPC error codes.
+	// 			promise->sendError(grpc_error()); // TODO (Vishesh): Propagate the gRPC error codes.
 	// 		}
 	// 	});
 
diff --git a/fdbrpc/include/fdbrpc/grpc/AsyncTaskExecutor.h b/fdbrpc/include/fdbrpc/grpc/AsyncTaskExecutor.h
index d31bedeb34f..ea0bc64c96d 100644
--- a/fdbrpc/include/fdbrpc/grpc/AsyncTaskExecutor.h
+++ b/fdbrpc/include/fdbrpc/grpc/AsyncTaskExecutor.h
@@ -59,7 +59,6 @@ concept IsVoidReturn = std::is_void_v<std::invoke_result_t<Func>>;
 //     });
 //
 // - TODO: Move this to more standard location inside codebase.
-// - TODO: Remove AsyncTaskThread which has similar purpose however implemented differently?
 class AsyncTaskExecutor {
 public:
 	explicit AsyncTaskExecutor(int num_threads) {
@@ -119,7 +118,8 @@ class AsyncTaskExecutor {
 
 // `ThreadAction` implementation for tasks that return non-void values.
 template <typename Func>
-struct AsyncTaskExecutor::Action<Func, typename std::enable_if_t<!IsVoidReturn<Func>>> : ThreadAction {
+    requires(!IsVoidReturn<Func>)
+struct AsyncTaskExecutor::Action<Func> : ThreadAction {
 	using Ret = typename std::invoke_result<Func>::type;
 
 	Action(Func&& fn) : fn_(std::move(fn)) {}
@@ -156,7 +156,8 @@ struct AsyncTaskExecutor::Action<Func, typename std::enable_if_t<!IsVoidReturn<F
 
 // `ThreadAction` implementation for tasks that return void.
 template <typename Func>
-struct AsyncTaskExecutor::Action<Func, typename std::enable_if_t<IsVoidReturn<Func>>> : ThreadAction {
+    requires(IsVoidReturn<Func>)
+struct AsyncTaskExecutor::Action<Func> : ThreadAction {
 	using Ret = typename std::invoke_result<Func>::type;
 
 	Action(Func&& fn) : fn_(std::move(fn)) {}
diff --git a/fdbrpc/include/fdbrpc/grpc/Credentials.h b/fdbrpc/include/fdbrpc/grpc/Credentials.h
index 44463384957..785ad322431 100644
--- a/fdbrpc/include/fdbrpc/grpc/Credentials.h
+++ b/fdbrpc/include/fdbrpc/grpc/Credentials.h
@@ -30,7 +30,7 @@
 #include <grpcpp/security/tls_credentials_options.h>
 
 #include "flow/Knobs.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 
 namespace ge = grpc::experimental;
 
@@ -77,7 +77,7 @@ class GrpcInsecureCredentialProvider : public GrpcCredentialProvider {
 // a valid certificate for authentication.
 class GrpcTlsCredentialProvider : public GrpcCredentialProvider {
 public:
-	GrpcTlsCredentialProvider(TLSConfig* config)
+	explicit GrpcTlsCredentialProvider(TLSConfig* config)
 	  : provider_(std::make_shared<ge::FileWatcherCertificateProvider>(config->getKeyPathSync(),
 	                                                                   config->getCertificatePathSync(),
 	                                                                   config->getCAPathSync(),
diff --git a/fdbrpc/include/fdbrpc/networksender.actor.h b/fdbrpc/include/fdbrpc/networksender.h
similarity index 62%
rename from fdbrpc/include/fdbrpc/networksender.actor.h
rename to fdbrpc/include/fdbrpc/networksender.h
index 6da9252086f..9bd62d4a5d2 100644
--- a/fdbrpc/include/fdbrpc/networksender.actor.h
+++ b/fdbrpc/include/fdbrpc/networksender.h
@@ -1,5 +1,5 @@
 /*
- * networksender.actor.h
+ * networksender.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,33 +18,28 @@
  * limitations under the License.
  */
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBRPC_NETWORKSENDER_ACTOR_G_H)
-#define FDBRPC_NETWORKSENDER_ACTOR_G_H
-#include "fdbrpc/networksender.actor.g.h"
-#elif !defined(RPCNETWORKSENDER_ACTOR_H)
-#define RPCNETWORKSENDER_ACTOR_H
+#ifndef FDBRPC_NETWORKSENDER_H
+#define FDBRPC_NETWORKSENDER_H
+#pragma once
 
 #include "fdbrpc/FlowTransport.h"
 #include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
-// This actor is used by FlowTransport to serialize the response to a ReplyPromise across the network
-ACTOR template <class T>
-void networkSender(Future<T> input, Endpoint endpoint) {
+// Used by FlowTransport to serialize the response to a ReplyPromise across the network.
+template <class T>
+Future<Void> networkSender(Uncancellable, Future<T> input, Endpoint endpoint, ExplicitVoid = {}) {
 	try {
-		T value = wait(input);
+		T value = co_await input;
 		FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<EnsureTable<T>>>(value), endpoint, false);
 	} catch (Error& err) {
 		// if (err.code() == error_code_broken_promise) return;
 		if (err.code() == error_code_never_reply) {
-			return;
+			co_return Void();
 		}
 		ASSERT(err.code() != error_code_actor_cancelled);
 		FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<EnsureTable<T>>>(err), endpoint, false);
 	}
+	co_return Void();
 }
-#include "flow/unactorcompiler.h"
 
 #endif
diff --git a/fdbrpc/include/fdbrpc/simulator.h b/fdbrpc/include/fdbrpc/simulator.h
index 2945b3dee9c..d772e35523e 100644
--- a/fdbrpc/include/fdbrpc/simulator.h
+++ b/fdbrpc/include/fdbrpc/simulator.h
@@ -37,7 +37,7 @@
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/Locality.h"
 #include "flow/IAsyncFile.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "fdbrpc/HTTP.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/Locality.h"
@@ -56,6 +56,45 @@ constexpr double DISABLE_CONNECTION_FAILURE_FOREVER = 1e6;
 // Minimum interval to disable connection failures. If less than this, connection failures are always disabled.
 constexpr double DISABLE_CONNECTION_FAILURE_MIN_INTERVAL = 1e-3;
 
+class ISimulationPolicy : public ReferenceCounted<ISimulationPolicy> {
+public:
+	using KillType = simulator::KillType;
+	using ProcessInfo = simulator::ProcessInfo;
+
+	enum class Capability {
+		WarnOnStorageMismatch,
+		StorageReplicaFaultInjection,
+		StorageReplicaDelay,
+		StorageReplicaMutationDrop,
+		LimitStorageServerReadBytes
+	};
+
+	virtual bool shouldProtectNewProcess(ProcessInfo const&) const { return false; }
+	virtual bool isAvailable(std::vector<ProcessInfo*> const&,
+	                         std::vector<ProcessInfo*> const& availableProcesses,
+	                         std::vector<ProcessInfo*> const& deadProcesses) const {
+		return canKillProcesses(availableProcesses, deadProcesses, KillType::KillInstantly, nullptr);
+	}
+	virtual bool datacenterDead(Optional<Standalone<StringRef>>, std::vector<ProcessInfo*> const&) const {
+		return false;
+	}
+	virtual bool shouldRunVersionValidation() const { return true; }
+	virtual bool canSwapToMachine(Optional<Standalone<StringRef>> const&) const { return true; }
+	virtual bool checkInjectedCorruption(NetworkAddress const&) const { return false; }
+	virtual bool hasCapability(Capability) const { return false; }
+	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses,
+	                              std::vector<ProcessInfo*> const& deadProcesses,
+	                              KillType kt,
+	                              KillType* newKillType) const {
+		if (newKillType) {
+			*newKillType = kt;
+		}
+		return true;
+	}
+
+	virtual ~ISimulationPolicy() = default;
+};
+
 class ISimulator : public INetwork {
 
 public:
@@ -63,30 +102,6 @@ class ISimulator : public INetwork {
 	using ProcessInfo = simulator::ProcessInfo;
 	using MachineInfo = simulator::MachineInfo;
 
-	// Order matters! all modes >= 2 are fault injection modes
-	enum TSSMode { Disabled, EnabledNormal, EnabledAddDelay, EnabledDropMutations };
-
-	enum class BackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB };
-	enum class ExtraDatabaseMode { Disabled, LocalOrSingle, Single, Local, Multiple };
-
-	static ExtraDatabaseMode stringToExtraDatabaseMode(const std::string& databaseMode) {
-		if (databaseMode == "Disabled") {
-			return ExtraDatabaseMode::Disabled;
-		} else if (databaseMode == "LocalOrSingle") {
-			return ExtraDatabaseMode::LocalOrSingle;
-		} else if (databaseMode == "Single") {
-			return ExtraDatabaseMode::Single;
-		} else if (databaseMode == "Local") {
-			return ExtraDatabaseMode::Local;
-		} else if (databaseMode == "Multiple") {
-			return ExtraDatabaseMode::Multiple;
-		} else {
-			TraceEvent(SevError, "UnknownExtraDatabaseMode").detail("DatabaseMode", databaseMode);
-			ASSERT(false);
-			throw internal_error();
-		}
-	};
-
 	ProcessInfo* getProcess(Endpoint const& endpoint) { return getProcessByAddress(endpoint.getPrimaryAddress()); }
 
 	// Returns currently executing process in simulation.
@@ -271,7 +286,8 @@ class ISimulator : public INetwork {
 		allSwapsDisabled = false;
 	}
 	bool canSwapToMachine(Optional<Standalone<StringRef>> zoneId) const {
-		return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled && extraDatabases.empty();
+		return swapsDisabled.count(zoneId) == 0 && !allSwapsDisabled &&
+		       (!simulationPolicy || simulationPolicy->canSwapToMachine(zoneId));
 	}
 	void enableSwapsToAll() {
 		swapsDisabled.clear();
@@ -301,45 +317,17 @@ class ISimulator : public INetwork {
 	                                           std::string service,
 	                                           Reference<HTTP::IRequestHandler> requestHandler) = 0;
 
-	int desiredCoordinators;
-	int physicalDatacenters;
+	void setSimulationPolicy(Reference<ISimulationPolicy> policy) { simulationPolicy = policy; }
+	Reference<ISimulationPolicy> getSimulationPolicy() const { return simulationPolicy; }
+
+	void protectAddress(NetworkAddress const& address) { protectedAddresses.insert(address); }
+	bool isProtectedAddress(NetworkAddress const& address) const { return protectedAddresses.contains(address); }
+	size_t protectedAddressCount() const { return protectedAddresses.size(); }
+
 	int processesPerMachine;
 	int listenersPerProcess;
 
-	// We won't kill machines in this set, but we might reboot
-	// them.  This is a conservative mechanism to prevent the
-	// simulator from killing off important processes and rendering
-	// the cluster unrecoverable, e.g. a quorum of coordinators.
-	std::set<NetworkAddress> protectedAddresses;
-
 	std::map<NetworkAddress, ProcessInfo*> currentlyRebootingProcesses;
-	std::vector<std::string> extraDatabases;
-	Reference<IReplicationPolicy> storagePolicy;
-	Reference<IReplicationPolicy> tLogPolicy;
-	int32_t tLogWriteAntiQuorum;
-	Optional<Standalone<StringRef>> primaryDcId;
-	Reference<IReplicationPolicy> remoteTLogPolicy;
-	int32_t usableRegions;
-	bool quiesced = false;
-	std::string disablePrimary;
-	std::string disableRemote;
-	std::string originalRegions;
-	std::string startingDisabledConfiguration;
-	bool allowLogSetKills;
-	Optional<Standalone<StringRef>> remoteDcId;
-	bool hasSatelliteReplication;
-	Reference<IReplicationPolicy> satelliteTLogPolicy;
-	Reference<IReplicationPolicy> satelliteTLogPolicyFallback;
-	int32_t satelliteTLogWriteAntiQuorum;
-	int32_t satelliteTLogWriteAntiQuorumFallback;
-	std::vector<Optional<Standalone<StringRef>>> primarySatelliteDcIds;
-	std::vector<Optional<Standalone<StringRef>>> remoteSatelliteDcIds;
-	TSSMode tssMode;
-	std::map<NetworkAddress, bool> corruptWorkerMap;
-
-	// Used by workloads that perform reconfigurations
-	int testerCount;
-	std::string connectionString;
 
 	bool isStopped;
 	double lastConnectionFailure;
@@ -347,77 +335,17 @@ class ISimulator : public INetwork {
 	bool speedUpSimulation;
 	double connectionFailureEnableTime; // Last time connection failure is enabled.
 	double connectionFailureDisableTime = 0; // Latest time connection failure should be disabled.
-	bool disableTLogRecoveryFinish;
-	BackupAgentType backupAgents;
-	BackupAgentType drAgents;
-	bool willRestart = false;
-	bool restarted = false;
-	bool isConsistencyChecked = false;
-
-	bool hasDiffProtocolProcess; // true if simulator is testing a process with a different version
-	bool setDiffProtocol; // true if a process with a different protocol version has been started
-
-	bool allowStorageMigrationTypeChange = false;
-	double injectTargetedSSRestartTime = std::numeric_limits<double>::max();
-	double injectSSDelayTime = std::numeric_limits<double>::max();
-	double injectTargetedBMRestartTime = std::numeric_limits<double>::max();
-	double injectTargetedBWRestartTime = std::numeric_limits<double>::max();
-
-	enum SimConsistencyScanState {
-		DisabledStart = 0,
-		Enabling = 1,
-		Enabled = 2,
-		Enabled_InjectCorruption = 3,
-		Enabled_FoundCorruption = 4,
-		Complete = 5,
-		DisabledEnd = 6
-	};
-	SimConsistencyScanState consistencyScanState = SimConsistencyScanState::DisabledStart;
-
-	// check that validates that the state transition is valid
-	bool updateConsistencyScanState(SimConsistencyScanState expectedCurrent, SimConsistencyScanState desired) {
-		if (consistencyScanState == expectedCurrent && desired > consistencyScanState) {
-			consistencyScanState = desired;
-
-			if (desired == SimConsistencyScanState::Enabled_FoundCorruption) {
-				// reset other metadata
-				consistencyScanInjectedCorruptionType = {};
-				consistencyScanCorruptRequestKey = {};
-				consistencyScanCorruptor = {};
-			}
-
-			return true;
-		}
-		return false;
-	}
-
-	// Inject corruption only in consistency scan reads to ensure scan finds it.
-	enum SimConsistencyScanCorruptionType { FlipMoreFlag = 0, AddToEmpty = 1, RemoveLastRow = 2, ChangeFirstValue = 3 };
-	Optional<SimConsistencyScanCorruptionType> consistencyScanInjectedCorruptionType;
-	Optional<UID> consistencyScanInjectedCorruptionDestination;
-	Optional<bool> doInjectConsistencyScanCorruption;
-	Optional<Standalone<StringRef>> consistencyScanCorruptRequestKey;
-	Optional<std::pair<UID, NetworkAddress>> consistencyScanCorruptor;
 
 	std::unordered_map<Standalone<StringRef>, PrivateKey> authKeys;
 
 	std::set<std::pair<std::string, unsigned>> corruptedBlocks;
 
-	// Validate at-rest encryption guarantees. If enabled, tests should inject a known 'marker' in Key and/or Values
-	// inserted into FDB by the workload. On shutdown, all test generated files (under simfdb/) are scanned to find if
-	// 'plaintext marker' is present.
-	Optional<std::string> dataAtRestPlaintextMarker;
-
 	std::unordered_map<std::string, Reference<HTTP::SimRegisteredHandlerContext>> httpHandlers;
 	std::vector<std::pair<ProcessInfo*, Reference<HTTP::SimServerContext>>> httpServerProcesses;
 	std::set<IPAddress> httpServerIps;
 	int nextHTTPPort = 5000;
 	bool httpProtected = false;
 
-	// Truly simulator-global registry for MockS3ServerChaos to prevent duplicate registrations
-	// across all simulated processes (must stay in sync with httpHandlers)
-	std::set<std::string> registeredMockS3ChaosServers;
-
 	flowGlobalType global(int id) const final;
 	void setGlobal(size_t id, flowGlobalType v) final;
 
@@ -439,6 +367,10 @@ class ISimulator : public INetwork {
 	Mutex mutex;
 
 private:
+	Reference<ISimulationPolicy> simulationPolicy;
+	// We won't kill machines in this set, but we might reboot them. This lets
+	// higher-level simulation policies protect important processes.
+	std::set<NetworkAddress> protectedAddresses;
 	std::set<Optional<Standalone<StringRef>>> swapsDisabled;
 	std::map<NetworkAddress, int> excludedAddresses;
 	std::map<NetworkAddress, int> clearedAddresses;
@@ -451,6 +383,13 @@ class ISimulator : public INetwork {
 
 extern ISimulator* g_simulator;
 
+inline bool simulationPolicyHasCapability(ISimulationPolicy::Capability capability) {
+	if (!g_network || !g_network->isSimulated() || !g_simulator || !g_simulator->getSimulationPolicy()) {
+		return false;
+	}
+	return g_simulator->getSimulationPolicy()->hasCapability(capability);
+}
+
 void startNewSimulator(bool printSimTime);
 
 // Parameters used to simulate disk performance
diff --git a/fdbrpc/libeio/eio.c b/fdbrpc/libeio/eio.c
index cb7a10396ff..cfac51cbc8b 100644
--- a/fdbrpc/libeio/eio.c
+++ b/fdbrpc/libeio/eio.c
@@ -57,7 +57,6 @@
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <limits.h>
@@ -1255,7 +1254,7 @@ static int eio__realpath(struct tmpbuf* tmpbuf, eio_wd wd, const char* path) {
 
     if (fd >= 0)
       {
-        sprintf (tmp1, "/proc/self/fd/%d", fd);
+        snprintf(tmp1, PATH_MAX, "/proc/self/fd/%d", fd);
         req->result = readlink (tmp1, res, PATH_MAX);
         close (fd);
 
diff --git a/fdbrpc/include/fdbrpc/linux_kaio.h b/fdbrpc/linux_kaio.h
similarity index 100%
rename from fdbrpc/include/fdbrpc/linux_kaio.h
rename to fdbrpc/linux_kaio.h
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.cpp
similarity index 80%
rename from fdbrpc/sim2.actor.cpp
rename to fdbrpc/sim2.cpp
index 79f6f55cdd6..260c54eeb57 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.cpp
@@ -1,5 +1,5 @@
 /*
- * sim2.actor.cpp
+ * sim2.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -33,18 +33,17 @@
 #ifndef BOOST_REGEX_NO_LIB
 #define BOOST_REGEX_NO_LIB
 #endif
-#include "fdbrpc/SimExternalConnection.h"
+#include "SimExternalConnection.h"
 #include "flow/ActorCollection.h"
 #include "flow/IRandom.h"
 #include "flow/CodeProbe.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/Util.h"
 #include "flow/IAsyncFile.h"
-#include "fdbrpc/AsyncFileCached.actor.h"
-#include "fdbrpc/AsyncFileEncrypted.h"
+#include "fdbrpc/AsyncFileCached.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbrpc/AsyncFileNonDurable.actor.h"
-#include "fdbrpc/AsyncFileChaos.h"
+#include "fdbrpc/AsyncFileNonDurable.h"
+#include "AsyncFileChaos.h"
 #include "crc32/crc32c.h"
 #include "fdbrpc/TraceFileIO.h"
 #include "flow/flow.h"
@@ -52,12 +51,10 @@
 #include "flow/swift/ABI/Task.h"
 #include "flow/genericactors.actor.h"
 #include "flow/network.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbrpc/Net2FileSystem.h"
-#include "fdbrpc/Replication.h"
-#include "fdbrpc/ReplicationUtils.h"
-#include "fdbrpc/AsyncFileWriteChecker.actor.h"
-#include "fdbrpc/genericactors.actor.h"
+#include "AsyncFileWriteChecker.h"
+#include "fdbrpc/genericactors.h"
 #include "fdbrpc/WellKnownEndpoints.h"
 #include "flow/FaultInjection.h"
 #include "flow/TaskQueue.h"
@@ -71,11 +68,9 @@ thread_local ISimulator::ProcessInfo* ISimulator::currentProcess = nullptr;
 thread_local bool ISimulator::isMainThread = false;
 
 ISimulator::ISimulator()
-  : desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1), usableRegions(1),
-    allowLogSetKills(true), tssMode(TSSMode::Disabled), isStopped(false), lastConnectionFailure(0),
+  : processesPerMachine(0), listenersPerProcess(1), isStopped(false), lastConnectionFailure(0),
     connectionFailuresDisableDuration(0), speedUpSimulation(false), connectionFailureEnableTime(0),
-    disableTLogRecoveryFinish(false), backupAgents(BackupAgentType::WaitForType),
-    drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {}
+    allSwapsDisabled(false) {}
 ISimulator::~ISimulator() = default;
 
 bool simulator_should_inject_fault(const char* context, const char* file, int line, int error_code) {
@@ -130,10 +125,7 @@ double ISimulator::checkDisabled(const std::string& desc) const {
 }
 
 bool ISimulator::checkInjectedCorruption() {
-	auto iter = corruptWorkerMap.find(currentProcess->address);
-	if (iter != corruptWorkerMap.end())
-		return iter->second;
-	return false;
+	return simulationPolicy && simulationPolicy->checkInjectedCorruption(currentProcess->address);
 }
 
 flowGlobalType ISimulator::global(int id) const {
@@ -210,14 +202,14 @@ struct SimClogging {
 		if (!g_simulator->speedUpSimulation && !stableConnection)
 			t += clogPairLatency[pair];
 
-		if (!g_simulator->speedUpSimulation && !stableConnection && clogPairUntil.count(pair))
+		if (!g_simulator->speedUpSimulation && !stableConnection && clogPairUntil.contains(pair))
 			t = std::max(t, clogPairUntil[pair]);
 
 		auto p = std::make_pair(from, to);
-		if (!g_simulator->speedUpSimulation && !stableConnection && clogProcessPairUntil.count(p))
+		if (!g_simulator->speedUpSimulation && !stableConnection && clogProcessPairUntil.contains(p))
 			t = std::max(t, clogProcessPairUntil[p]);
 
-		if (!g_simulator->speedUpSimulation && !stableConnection && clogRecvUntil.count(to.ip))
+		if (!g_simulator->speedUpSimulation && !stableConnection && clogRecvUntil.contains(to.ip))
 			t = std::max(t, clogRecvUntil[to.ip]);
 
 		return t - tnow;
@@ -300,7 +292,7 @@ struct SimClogging {
 SimClogging g_clogging;
 
 struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
-	Sim2Conn(ISimulator::ProcessInfo* process)
+	explicit Sim2Conn(ISimulator::ProcessInfo* process)
 	  : opened(false), closedByCaller(false), stableConnection(false), trustedPeer(true), process(process),
 	    dbgid(deterministicRandom()->randomUniqueID()), stopReceive(Never()) {
 		pipes = sender(this) && receiver(this);
@@ -464,20 +456,20 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 		peer.clear();
 	}
 
-	ACTOR static Future<Void> sender(Sim2Conn* self) {
+	static Future<Void> sender(Sim2Conn* self) {
 		loop {
-			wait(self->writtenBytes.onChange()); // takes place on peer!
+			co_await self->writtenBytes.onChange(); // takes place on peer!
 			ASSERT(g_simulator->getCurrentProcess() == self->peerProcess);
-			wait(delay(.002 * deterministicRandom()->random01()));
+			co_await delay(.002 * deterministicRandom()->random01());
 			self->sentBytes.set(self->writtenBytes.get()); // or possibly just some sometimes...
 		}
 	}
-	ACTOR static Future<Void> receiver(Sim2Conn* self) {
+	static Future<Void> receiver(Sim2Conn* self) {
 		loop {
 			if (self->sentBytes.get() != self->receivedBytes.get())
-				wait(g_simulator->onProcess(self->peerProcess));
+				co_await g_simulator->onProcess(self->peerProcess);
 			while (self->sentBytes.get() == self->receivedBytes.get())
-				wait(self->sentBytes.onChange());
+				co_await self->sentBytes.onChange();
 			ASSERT(g_simulator->getCurrentProcess() == self->peerProcess);
 
 			// Simulated network disconnection. Make sure to only throw connection_failed() on the sender process.
@@ -489,33 +481,33 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 				throw connection_failed();
 			}
 
-			state int64_t pos =
+			int64_t pos =
 			    deterministicRandom()->random01() < .5
 			        ? self->sentBytes.get()
 			        : deterministicRandom()->randomInt64(self->receivedBytes.get(), self->sentBytes.get() + 1);
-			wait(delay(g_clogging.getSendDelay(
-			    self->peerProcess->address, self->process->address, self->isStableConnection())));
-			wait(g_simulator->onProcess(self->process));
+			co_await delay(g_clogging.getSendDelay(
+			    self->peerProcess->address, self->process->address, self->isStableConnection()));
+			co_await g_simulator->onProcess(self->process);
 			ASSERT(g_simulator->getCurrentProcess() == self->process);
-			wait(delay(g_clogging.getRecvDelay(
-			    self->peerProcess->address, self->process->address, self->isStableConnection())));
+			co_await delay(g_clogging.getRecvDelay(
+			    self->peerProcess->address, self->process->address, self->isStableConnection()));
 			ASSERT(g_simulator->getCurrentProcess() == self->process);
 			if (self->stopReceive.isReady()) {
-				wait(Future<Void>(Never()));
+				co_await Future<Void>(Never());
 			}
 			self->receivedBytes.set(pos);
-			wait(Future<Void>(Void())); // Prior notification can delete self and cancel this actor
+			co_await Future<Void>(Void()); // Prior notification can delete self and cancel this actor
 			ASSERT(g_simulator->getCurrentProcess() == self->process);
 		}
 	}
-	ACTOR static Future<Void> whenReadable(Sim2Conn* self) {
+	static Future<Void> whenReadable(Sim2Conn* self) {
 		try {
 			loop {
 				if (self->readBytes.get() != self->receivedBytes.get()) {
 					ASSERT(g_simulator->getCurrentProcess() == self->process);
-					return Void();
+					co_return;
 				}
-				wait(self->receivedBytes.onChange());
+				co_await self->receivedBytes.onChange();
 				self->rollRandomClose();
 			}
 		} catch (Error& e) {
@@ -523,23 +515,23 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 			throw;
 		}
 	}
-	ACTOR static Future<Void> whenWritable(Sim2Conn* self) {
+	static Future<Void> whenWritable(Sim2Conn* self) {
 		try {
 			loop {
 				if (!self->peer)
-					return Void();
+					co_return;
 				if (self->peer->availableSendBufferForPeer() > 0) {
 					ASSERT(g_simulator->getCurrentProcess() == self->process);
-					return Void();
+					co_return;
 				}
 				try {
-					wait(self->peer->receivedBytes.onChange());
+					co_await self->peer->receivedBytes.onChange();
 					ASSERT(g_simulator->getCurrentProcess() == self->peerProcess);
 				} catch (Error& e) {
 					if (e.code() != error_code_broken_promise)
 						throw;
 				}
-				wait(g_simulator->onProcess(self->process));
+				co_await g_simulator->onProcess(self->process);
 			}
 		} catch (Error& e) {
 			ASSERT(g_simulator->getCurrentProcess() == self->process);
@@ -573,18 +565,19 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 		}
 	}
 
-	ACTOR static Future<Void> trackLeakedConnection(Sim2Conn* self) {
+	static Future<Void> trackLeakedConnection(Sim2Conn* self) {
 		// FIXME: we could also just implement connection idle closing for sim http server instead
-		if (g_simulator->httpServerIps.count(self->process->address.ip)) {
-			return Void();
+		if (g_simulator->httpServerIps.contains(self->process->address.ip)) {
+			co_return;
 		}
-		wait(g_simulator->onProcess(self->process));
+		co_await g_simulator->onProcess(self->process);
 
 		if (self->process->address.isPublic()) {
-			wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * 1.5 +
-			           FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME * 2.1 + FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
+			co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT *
+			                   1.5 +
+			               FLOW_KNOBS->CONNECTION_MONITOR_LOOP_TIME * 2.1 + FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
 		} else {
-			wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * 1.5));
+			co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_IDLE_TIMEOUT * 1.5);
 		}
 		TraceEvent(SevError, "LeakedConnection", self->dbgid)
 		    .error(connection_leaked())
@@ -594,7 +587,7 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 		    .detail("PeerAddress", self->peerEndpoint)
 		    .detail("PeerId", self->peerId)
 		    .detail("Opened", self->opened);
-		return Void();
+		co_return;
 	}
 };
 
@@ -629,18 +622,18 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 public:
 	static void init() {}
 
-	virtual StringRef getClassName() override { return "SimpleFile"_sr; }
+	StringRef getClassName() override { return "SimpleFile"_sr; }
 
 	static bool should_poll() { return false; }
 
-	ACTOR static Future<Reference<IAsyncFile>> open(
+	static Future<Reference<IAsyncFile>> open(
 	    std::string filename,
 	    int flags,
 	    int mode,
 	    Reference<DiskParameters> diskParameters = makeReference<DiskParameters>(25000, 150000000),
 	    bool delayOnWrite = true) {
-		state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-		state TaskPriority currentTaskID = g_network->getCurrentTask();
+		ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+		TaskPriority currentTaskID = g_network->getCurrentTask();
 
 		if (++openFileCount >= 6000) {
 			TraceEvent(SevError, "TooManyFiles").log();
@@ -655,10 +648,11 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		// filename.  We add ".part" below, so we need to stay under 250.
 		ASSERT(basename(filename).size() < 250);
 
-		wait(g_simulator->onMachine(currentProcess));
+		co_await g_simulator->onMachine(currentProcess);
+		Error err;
 		try {
-			wait(delay(FLOW_KNOBS->MIN_OPEN_TIME +
-			           deterministicRandom()->random01() * (FLOW_KNOBS->MAX_OPEN_TIME - FLOW_KNOBS->MIN_OPEN_TIME)));
+			co_await delay(FLOW_KNOBS->MIN_OPEN_TIME +
+			               deterministicRandom()->random01() * (FLOW_KNOBS->MAX_OPEN_TIME - FLOW_KNOBS->MIN_OPEN_TIME));
 
 			std::string open_filename = filename;
 			if (flags & OPEN_ATOMIC_WRITE_AND_CREATE) {
@@ -680,14 +674,14 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 
 			platform::makeTemporary(open_filename.c_str());
 			SimpleFile* simpleFile = new SimpleFile(h, diskParameters, delayOnWrite, filename, open_filename, flags);
-			state Reference<IAsyncFile> file = Reference<IAsyncFile>(simpleFile);
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-			return file;
+			Reference<IAsyncFile> file = Reference<IAsyncFile>(simpleFile);
+			co_await g_simulator->onProcess(currentProcess, currentTaskID);
+			co_return file;
 		} catch (Error& e) {
-			state Error err = e;
-			wait(g_simulator->onProcess(currentProcess, currentTaskID));
-			throw err;
+			err = e;
 		}
+		co_await g_simulator->onProcess(currentProcess, currentTaskID);
+		throw err;
 	}
 
 	void addref() override { ReferenceCounted<SimpleFile>::addref(); }
@@ -753,10 +747,10 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		return outFlags;
 	}
 
-	ACTOR static Future<int> read_impl(SimpleFile* self, void* data, int length, int64_t offset) {
+	static Future<int> read_impl(SimpleFile* self, void* data, int length, int64_t offset) {
 		ASSERT((self->flags & IAsyncFile::OPEN_NO_AIO) != 0 ||
 		       ((uintptr_t)data % 4096 == 0 && length % 4096 == 0 && offset % 4096 == 0)); // Required by KAIO.
-		state UID opId = deterministicRandom()->randomUniqueID();
+		UID opId = deterministicRandom()->randomUniqueID();
 		if (randLog)
 			fmt::print(randLog,
 			           "SFR1 {0} {1} {2} {3} {4}\n",
@@ -766,7 +760,7 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 			           length,
 			           offset);
 
-		wait(waitUntilDiskReady(self->diskParameters, length));
+		co_await waitUntilDiskReady(self->diskParameters, length);
 
 		if (_lseeki64(self->h, offset, SEEK_SET) == -1) {
 			TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 1);
@@ -795,11 +789,11 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		INJECT_FAULT(io_timeout, "SimpleFile::read"); // SimpleFile::read io_timeout injected
 		INJECT_FAULT(io_error, "SimpleFile::read"); // SimpleFile::read io_error injected
 
-		return read_bytes;
+		co_return read_bytes;
 	}
 
-	ACTOR static Future<Void> write_impl(SimpleFile* self, StringRef data, int64_t offset) {
-		state UID opId = deterministicRandom()->randomUniqueID();
+	static Future<Void> write_impl(SimpleFile* self, StringRef data, int64_t offset) {
+		UID opId = deterministicRandom()->randomUniqueID();
 		if (randLog) {
 			uint32_t a = crc32c_append(0, data.begin(), data.size());
 			fmt::print(randLog,
@@ -813,7 +807,7 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		}
 
 		if (self->delayOnWrite)
-			wait(waitUntilDiskReady(self->diskParameters, data.size()));
+			co_await waitUntilDiskReady(self->diskParameters, data.size());
 
 		if (_lseeki64(self->h, offset, SEEK_SET) == -1) {
 			TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 3);
@@ -844,11 +838,11 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		INJECT_FAULT(io_timeout, "SimpleFile::write"); // SimpleFile::write inject io_timeout
 		INJECT_FAULT(io_error, "SimpleFile::write"); // SimpleFile::write inject io_error
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> truncate_impl(SimpleFile* self, int64_t size) {
-		state UID opId = deterministicRandom()->randomUniqueID();
+	static Future<Void> truncate_impl(SimpleFile* self, int64_t size) {
+		UID opId = deterministicRandom()->randomUniqueID();
 		if (randLog)
 			fmt::print(
 			    randLog, "SFT1 {0} {1} {2} {3}\n", self->dbgId.shortString(), self->filename, opId.shortString(), size);
@@ -859,7 +853,7 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		}
 
 		if (self->delayOnWrite)
-			wait(waitUntilDiskReady(self->diskParameters, 0));
+			co_await waitUntilDiskReady(self->diskParameters, 0);
 
 		if (_chsize(self->h, (long)size) == -1) {
 			TraceEvent(SevWarn, "SimpleFileIOError")
@@ -881,12 +875,12 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		INJECT_FAULT(io_timeout, "SimpleFile::truncate"); // SimpleFile::truncate inject io_timeout
 		INJECT_FAULT(io_error, "SimpleFile::truncate"); // SimpleFile::truncate inject io_error
 
-		return Void();
+		co_return;
 	}
 
 	// Simulated sync does not actually do anything besides wait a random amount of time
-	ACTOR static Future<Void> sync_impl(SimpleFile* self) {
-		state UID opId = deterministicRandom()->randomUniqueID();
+	static Future<Void> sync_impl(SimpleFile* self) {
+		UID opId = deterministicRandom()->randomUniqueID();
 		if (randLog)
 			fprintf(randLog,
 			        "SFC1 %s %s %s\n",
@@ -895,14 +889,14 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 			        opId.shortString().c_str());
 
 		if (self->delayOnWrite)
-			wait(waitUntilDiskReady(self->diskParameters, 0, true));
+			co_await waitUntilDiskReady(self->diskParameters, 0, true);
 
 		if (self->flags & OPEN_ATOMIC_WRITE_AND_CREATE) {
 			self->flags &= ~OPEN_ATOMIC_WRITE_AND_CREATE;
 			auto& machineCache = g_simulator->getCurrentProcess()->machine->openFiles;
 			std::string sourceFilename = self->filename + ".part";
 
-			if (machineCache.count(sourceFilename)) {
+			if (machineCache.contains(sourceFilename)) {
 				// it seems gcc has some trouble with these types. Aliasing with typename is ugly, but seems to work.
 				using block_value_type = typename decltype(g_simulator->corruptedBlocks)::key_type::second_type;
 				TraceEvent("SimpleFileRename")
@@ -940,11 +934,11 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		INJECT_FAULT(io_timeout, "SimpleFile::sync"); // SimpleFile::sync inject io_timeout
 		INJECT_FAULT(io_error, "SimpleFile::sync"); // SimpleFile::sync inject io_errot
 
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<int64_t> size_impl(SimpleFile const* self) {
-		state UID opId = deterministicRandom()->randomUniqueID();
+	static Future<int64_t> size_impl(SimpleFile const* self) {
+		UID opId = deterministicRandom()->randomUniqueID();
 		if (randLog)
 			fprintf(randLog,
 			        "SFS1 %s %s %s\n",
@@ -952,7 +946,7 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 			        self->filename.c_str(),
 			        opId.shortString().c_str());
 
-		wait(waitUntilDiskReady(self->diskParameters, 0));
+		co_await waitUntilDiskReady(self->diskParameters, 0);
 
 		int64_t pos = _lseeki64(self->h, 0L, SEEK_END);
 		if (pos == -1) {
@@ -965,7 +959,7 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 			    randLog, "SFS2 {0} {1} {2} {3}\n", self->dbgId.shortString(), self->filename, opId.shortString(), pos);
 		INJECT_FAULT(io_error, "SimpleFile::size"); // SimpleFile::size inject io_error
 
-		return pos;
+		co_return pos;
 	}
 };
 
@@ -975,14 +969,14 @@ struct SimDiskSpace {
 	double lastUpdate;
 };
 
-void doReboot(ISimulator::ProcessInfo* const& p, ISimulator::KillType const& kt);
+Future<Void> doReboot(Uncancellable, ISimulator::ProcessInfo* p, ISimulator::KillType kt);
 
 struct Sim2Listener final : IListener, ReferenceCounted<Sim2Listener> {
 	explicit Sim2Listener(ISimulator::ProcessInfo* process, const NetworkAddress& listenAddr)
 	  : process(process), address(listenAddr) {}
 
 	void incomingConnection(double seconds, Reference<IConnection> conn) { // Called by another process!
-		incoming(Reference<Sim2Listener>::addRef(this), seconds, conn);
+		incoming(Uncancellable(), Reference<Sim2Listener>::addRef(this), seconds, conn);
 	}
 
 	void addref() override { ReferenceCounted<Sim2Listener>::addref(); }
@@ -996,20 +990,23 @@ struct Sim2Listener final : IListener, ReferenceCounted<Sim2Listener> {
 	ISimulator::ProcessInfo* process;
 	PromiseStream<Reference<IConnection>> nextConnection;
 
-	ACTOR static void incoming(Reference<Sim2Listener> self, double seconds, Reference<IConnection> conn) {
-		wait(g_simulator->onProcess(self->process));
-		wait(delay(seconds));
+	static Future<Void> incoming(Uncancellable,
+	                             Reference<Sim2Listener> self,
+	                             double seconds,
+	                             Reference<IConnection> conn) {
+		co_await g_simulator->onProcess(self->process);
+		co_await delay(seconds);
 		if (((Sim2Conn*)conn.getPtr())->isPeerGone() && deterministicRandom()->random01() < 0.5)
-			return;
+			co_return;
 		TraceEvent("Sim2IncomingConn", conn->getDebugID())
 		    .detail("ListenAddress", self->getListenAddress())
 		    .detail("PeerAddress", conn->getPeerAddress());
 		self->nextConnection.send(conn);
 	}
-	ACTOR static Future<Reference<IConnection>> popOne(FutureStream<Reference<IConnection>> conns) {
-		Reference<IConnection> c = waitNext(conns);
+	static Future<Reference<IConnection>> popOne(FutureStream<Reference<IConnection>> conns) {
+		Reference<IConnection> c = co_await conns;
 		((Sim2Conn*)c.getPtr())->opened = true;
-		return c;
+		co_return c;
 	}
 
 	NetworkAddress address;
@@ -1059,7 +1056,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		                     // as infinite
 			return Never();
 
-		PromiseTask* t = new PromiseTask(machine);
+		auto* t = new PromiseTask(machine);
 		if (seconds <= TIME_EPS) {
 			taskQueue.addReady(taskID, t);
 		} else {
@@ -1074,10 +1071,10 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		}
 		return t->promise.getFuture();
 	}
-	ACTOR static Future<Void> checkShutdown(Sim2* self, TaskPriority taskID) {
-		wait(success(self->getCurrentProcess()->shutdownSignal.getFuture()));
+	static Future<Void> checkShutdown(Sim2* self, TaskPriority taskID) {
+		co_await self->getCurrentProcess()->shutdownSignal.getFuture();
 		self->setCurrentTask(taskID);
-		return Void();
+		co_return;
 	}
 	Future<class Void> yield(TaskPriority taskID) override {
 		if (taskID == TaskPriority::DefaultYield)
@@ -1107,7 +1104,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 	Future<Reference<IConnection>> connect(NetworkAddress toAddr,
 	                                       boost::asio::ip::tcp::socket* existingSocket = nullptr) override {
 		ASSERT(existingSocket == nullptr);
-		if (!addressMap.count(toAddr)) {
+		if (!addressMap.contains(toAddr)) {
 			return waitForProcessAndConnect(toAddr, this);
 		}
 		auto peerp = getProcessByAddress(toAddr);
@@ -1134,7 +1131,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 
 	Future<Reference<IConnection>> connectExternal(NetworkAddress toAddr) override {
 		// If sim http connection, do connect instead of external connect
-		if (httpServerIps.count(toAddr.ip)) {
+		if (httpServerIps.contains(toAddr.ip)) {
 			return connect(toAddr);
 		}
 		return SimExternalConnection::connect(toAddr);
@@ -1204,32 +1201,31 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		}
 		return SimExternalConnection::resolveTCPEndpointBlocking(host, service, &dnsCache);
 	}
-	ACTOR static Future<Reference<IConnection>> onConnect(Future<Void> ready, Reference<Sim2Conn> conn) {
-		wait(ready);
+	static Future<Reference<IConnection>> onConnect(Future<Void> ready, Reference<Sim2Conn> conn) {
+		co_await ready;
 		if (conn->isPeerGone()) {
 			conn.clear();
 			if (FLOW_KNOBS->SIM_CONNECT_ERROR_MODE == 1 ||
 			    (FLOW_KNOBS->SIM_CONNECT_ERROR_MODE == 2 && deterministicRandom()->random01() > 0.5)) {
 				throw connection_failed();
 			}
-			wait(Never());
+			co_await Future<Void>(Never());
 		}
 		conn->opened = true;
-		return conn;
+		co_return conn;
 	}
 	Reference<IListener> listen(NetworkAddress localAddr) override {
 		Reference<IListener> listener(getCurrentProcess()->getListener(localAddr));
 		ASSERT(listener);
 		return listener;
 	}
-	ACTOR static Future<Reference<IConnection>> waitForProcessAndConnect(NetworkAddress toAddr,
-	                                                                     INetworkConnections* self) {
+	static Future<Reference<IConnection>> waitForProcessAndConnect(NetworkAddress toAddr, INetworkConnections* self) {
 		// We have to be able to connect to processes that don't yet exist, so we do some silly polling
 		loop {
-			wait(::delay(0.1 * deterministicRandom()->random01()));
-			if (g_sim2.addressMap.count(toAddr)) {
-				Reference<IConnection> c = wait(self->connect(toAddr));
-				return c;
+			co_await ::delay(0.1 * deterministicRandom()->random01());
+			if (g_sim2.addressMap.contains(toAddr)) {
+				Reference<IConnection> c = co_await self->connect(toAddr);
+				co_return c;
 			}
 		}
 	}
@@ -1262,7 +1258,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 
 	// Starts a new thread, making sure to set any thread local state
 	THREAD_FUNC simStartThread(void* arg) {
-		SimThreadArgs* simArgs = (SimThreadArgs*)arg;
+		auto* simArgs = static_cast<SimThreadArgs*>(arg);
 		ISimulator::currentProcess = simArgs->currentProcess;
 		simArgs->func(simArgs->arg);
 
@@ -1271,7 +1267,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 	}
 
 	THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg, int stackSize, const char* name) override {
-		SimThreadArgs* simArgs = new SimThreadArgs(func, arg);
+		auto* simArgs = new SimThreadArgs(func, arg);
 		return ::startThread(simStartThread, simArgs, stackSize, name);
 	}
 
@@ -1326,43 +1322,44 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		return addr.ip == getCurrentProcess()->address.ip;
 	}
 
-	ACTOR static Future<Void> deleteFileImpl(Sim2* self, std::string filename, bool mustBeDurable) {
+	static Future<Void> deleteFileImpl(Sim2* self, std::string filename, bool mustBeDurable) {
 		// This is a _rudimentary_ simulation of the untrustworthiness of non-durable deletes and the possibility of
 		// rebooting during a durable one.  It isn't perfect: for example, on real filesystems testing
 		// for the existence of a non-durably deleted file BEFORE a reboot will show that it apparently doesn't exist.
-		if (g_simulator->getCurrentProcess()->machine->openFiles.count(filename)) {
+		if (g_simulator->getCurrentProcess()->machine->openFiles.contains(filename)) {
 			g_simulator->getCurrentProcess()->machine->openFiles.erase(filename);
 			g_simulator->getCurrentProcess()->machine->deletingOrClosingFiles.insert(filename);
 		}
 		if (mustBeDurable || deterministicRandom()->random01() < 0.5) {
-			state ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
-			state TaskPriority currentTaskID = g_network->getCurrentTask();
+			ISimulator::ProcessInfo* currentProcess = g_simulator->getCurrentProcess();
+			TaskPriority currentTaskID = g_network->getCurrentTask();
 			TraceEvent(SevDebug, "Sim2DeleteFileImpl")
 			    .detail("CurrentProcess", currentProcess->toString())
 			    .detail("Filename", filename)
 			    .detail("Durable", mustBeDurable);
-			wait(g_simulator->onMachine(currentProcess));
+			co_await g_simulator->onMachine(currentProcess);
+			Error err;
 			try {
-				wait(::delay(0.05 * deterministicRandom()->random01()));
+				co_await ::delay(0.05 * deterministicRandom()->random01());
 				if (!currentProcess->rebooting) {
 					auto f = IAsyncFileSystem::filesystem(self->net2)->deleteFile(filename, false);
 					ASSERT(f.isReady());
-					wait(::delay(0.05 * deterministicRandom()->random01()));
+					co_await ::delay(0.05 * deterministicRandom()->random01());
 					CODE_PROBE(true, "Simulated durable delete", probe::context::sim2, probe::assert::simOnly);
 				}
-				wait(g_simulator->onProcess(currentProcess, currentTaskID));
-				return Void();
+				co_await g_simulator->onProcess(currentProcess, currentTaskID);
+				co_return;
 			} catch (Error& e) {
-				state Error err = e;
-				wait(g_simulator->onProcess(currentProcess, currentTaskID));
-				throw err;
+				err = e;
 			}
+			co_await g_simulator->onProcess(currentProcess, currentTaskID);
+			throw err;
 		} else {
 			TraceEvent(SevDebug, "Sim2DeleteFileImplNonDurable")
 			    .detail("Filename", filename)
 			    .detail("Durable", mustBeDurable);
 			CODE_PROBE(true, "Simulated non-durable delete", probe::context::sim2, probe::assert::simOnly);
-			return Void();
+			co_return;
 		}
 	}
 
@@ -1452,11 +1449,10 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		}
 
 		// FIXME: why would a ProcessInfo be called `m`?
-		ProcessInfo* m =
-		    new ProcessInfo(name, locality, startingClass, addresses, this, dataFolder, coordinationFolder);
+		auto* m = new ProcessInfo(name, locality, startingClass, addresses, this, dataFolder, coordinationFolder);
 		for (int processPort = port; processPort < port + listenPerProcess; ++processPort) {
 			NetworkAddress address(ip, processPort, true, sslEnabled && processPort == port);
-			m->listenerMap[address] = Reference<IListener>(new Sim2Listener(m, address));
+			m->listenerMap[address] = makeReference<Sim2Listener>(m, address);
 			addressMap[address] = m;
 		}
 		m->machine = &machine;
@@ -1485,8 +1481,9 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		    .detail("Cleared", m->cleared)
 		    .detail("DrProcess", m->drProcess);
 
-		if (std::string(name) == "remote flow process") {
-			protectedAddresses.insert(m->address);
+		if (std::string(name) == "remote flow process" ||
+		    (getSimulationPolicy() && getSimulationPolicy()->shouldProtectNewProcess(*m))) {
+			protectAddress(m->address);
 			TraceEvent(SevDebug, "NewFlowProcessProtected").detail("Address", m->address);
 		}
 
@@ -1496,7 +1493,8 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 	}
 	bool isAvailable() const override {
 		std::vector<ProcessInfo*> processesLeft, processesDead;
-		for (auto processInfo : getAllProcesses()) {
+		auto processes = getAllProcesses();
+		for (auto processInfo : processes) {
 			if (processInfo->isAvailableClass()) {
 				if (processInfo->isExcluded() || processInfo->isCleared() || !processInfo->isAvailable()) {
 					processesDead.push_back(processInfo);
@@ -1505,7 +1503,10 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				}
 			}
 		}
-		return canKillProcesses(processesLeft, processesDead, KillType::KillInstantly, nullptr);
+		if (getSimulationPolicy()) {
+			return getSimulationPolicy()->isAvailable(processes, processesLeft, processesDead);
+		}
+		return true;
 	}
 
 	std::vector<AddressExclusion> getAllAddressesInDCToExclude(Optional<Standalone<StringRef>> dcId) const override {
@@ -1522,36 +1523,10 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 	}
 
 	bool datacenterDead(Optional<Standalone<StringRef>> dcId) const override {
-		if (!dcId.present()) {
-			return false;
-		}
-
-		LocalityGroup primaryProcessesLeft, primaryProcessesDead;
-		std::vector<LocalityData> primaryLocalitiesDead, primaryLocalitiesLeft;
-
-		for (auto processInfo : getAllProcesses()) {
-			if (!processInfo->isSpawnedKVProcess() && processInfo->isAvailableClass() &&
-			    processInfo->locality.dcId() == dcId) {
-				if (processInfo->isExcluded() || processInfo->isCleared() || !processInfo->isAvailable()) {
-					primaryProcessesDead.add(processInfo->locality);
-					primaryLocalitiesDead.push_back(processInfo->locality);
-				} else {
-					primaryProcessesLeft.add(processInfo->locality);
-					primaryLocalitiesLeft.push_back(processInfo->locality);
-				}
-			}
+		if (getSimulationPolicy()) {
+			return getSimulationPolicy()->datacenterDead(dcId, getAllProcesses());
 		}
-
-		std::vector<LocalityData> badCombo;
-		bool primaryTLogsDead =
-		    tLogWriteAntiQuorum
-		        ? !validateAllCombinations(
-		              badCombo, primaryProcessesDead, tLogPolicy, primaryLocalitiesLeft, tLogWriteAntiQuorum, false)
-		        : primaryProcessesDead.validate(tLogPolicy);
-		if (usableRegions > 1 && remoteTLogPolicy && !primaryTLogsDead) {
-			primaryTLogsDead = primaryProcessesDead.validate(remoteTLogPolicy);
-		}
-		return primaryTLogsDead || primaryProcessesDead.validate(storagePolicy);
+		return false;
 	}
 
 	// The following function will determine if the specified configuration of available and dead processes can allow
@@ -1560,224 +1535,13 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 	                      std::vector<ProcessInfo*> const& deadProcesses,
 	                      KillType kt,
 	                      KillType* newKillType) const override {
-		bool canSurvive = true;
-		int nQuorum = ((desiredCoordinators + 1) / 2) * 2 - 1;
-
-		KillType newKt = kt;
-		if ((kt == KillType::KillInstantly) || (kt == KillType::InjectFaults) || (kt == KillType::FailDisk) ||
-		    (kt == KillType::RebootAndDelete) || (kt == KillType::RebootProcessAndDelete)) {
-			LocalityGroup primaryProcessesLeft, primaryProcessesDead;
-			LocalityGroup primarySatelliteProcessesLeft, primarySatelliteProcessesDead;
-			LocalityGroup remoteProcessesLeft, remoteProcessesDead;
-			LocalityGroup remoteSatelliteProcessesLeft, remoteSatelliteProcessesDead;
-
-			std::vector<LocalityData> primaryLocalitiesDead, primaryLocalitiesLeft;
-			std::vector<LocalityData> primarySatelliteLocalitiesDead, primarySatelliteLocalitiesLeft;
-			std::vector<LocalityData> remoteLocalitiesDead, remoteLocalitiesLeft;
-			std::vector<LocalityData> remoteSatelliteLocalitiesDead, remoteSatelliteLocalitiesLeft;
-
-			std::vector<LocalityData> badCombo;
-			std::set<Optional<Standalone<StringRef>>> uniqueMachines;
-
-			if (!primaryDcId.present() || usableRegions == 1) {
-				for (auto processInfo : availableProcesses) {
-					primaryProcessesLeft.add(processInfo->locality);
-					primaryLocalitiesLeft.push_back(processInfo->locality);
-					uniqueMachines.insert(processInfo->locality.zoneId());
-				}
-				for (auto processInfo : deadProcesses) {
-					primaryProcessesDead.add(processInfo->locality);
-					primaryLocalitiesDead.push_back(processInfo->locality);
-				}
-			} else {
-				for (auto processInfo : availableProcesses) {
-					uniqueMachines.insert(processInfo->locality.zoneId());
-					if (processInfo->locality.dcId() == primaryDcId) {
-						primaryProcessesLeft.add(processInfo->locality);
-						primaryLocalitiesLeft.push_back(processInfo->locality);
-					}
-					if (processInfo->locality.dcId() == remoteDcId) {
-						remoteProcessesLeft.add(processInfo->locality);
-						remoteLocalitiesLeft.push_back(processInfo->locality);
-					}
-					if (std::find(primarySatelliteDcIds.begin(),
-					              primarySatelliteDcIds.end(),
-					              processInfo->locality.dcId()) != primarySatelliteDcIds.end()) {
-						primarySatelliteProcessesLeft.add(processInfo->locality);
-						primarySatelliteLocalitiesLeft.push_back(processInfo->locality);
-					}
-					if (std::find(remoteSatelliteDcIds.begin(),
-					              remoteSatelliteDcIds.end(),
-					              processInfo->locality.dcId()) != remoteSatelliteDcIds.end()) {
-						remoteSatelliteProcessesLeft.add(processInfo->locality);
-						remoteSatelliteLocalitiesLeft.push_back(processInfo->locality);
-					}
-				}
-				for (auto processInfo : deadProcesses) {
-					if (processInfo->locality.dcId() == primaryDcId) {
-						primaryProcessesDead.add(processInfo->locality);
-						primaryLocalitiesDead.push_back(processInfo->locality);
-					}
-					if (processInfo->locality.dcId() == remoteDcId) {
-						remoteProcessesDead.add(processInfo->locality);
-						remoteLocalitiesDead.push_back(processInfo->locality);
-					}
-					if (std::find(primarySatelliteDcIds.begin(),
-					              primarySatelliteDcIds.end(),
-					              processInfo->locality.dcId()) != primarySatelliteDcIds.end()) {
-						primarySatelliteProcessesDead.add(processInfo->locality);
-						primarySatelliteLocalitiesDead.push_back(processInfo->locality);
-					}
-					if (std::find(remoteSatelliteDcIds.begin(),
-					              remoteSatelliteDcIds.end(),
-					              processInfo->locality.dcId()) != remoteSatelliteDcIds.end()) {
-						remoteSatelliteProcessesDead.add(processInfo->locality);
-						remoteSatelliteLocalitiesDead.push_back(processInfo->locality);
-					}
-				}
-			}
-
-			bool tooManyDead = false;
-			bool notEnoughLeft = false;
-			bool primaryTLogsDead =
-			    tLogWriteAntiQuorum
-			        ? !validateAllCombinations(
-			              badCombo, primaryProcessesDead, tLogPolicy, primaryLocalitiesLeft, tLogWriteAntiQuorum, false)
-			        : primaryProcessesDead.validate(tLogPolicy);
-			if (usableRegions > 1 && remoteTLogPolicy && !primaryTLogsDead) {
-				primaryTLogsDead = primaryProcessesDead.validate(remoteTLogPolicy);
-			}
-
-			if (!primaryDcId.present()) {
-				tooManyDead = primaryTLogsDead || primaryProcessesDead.validate(storagePolicy);
-				notEnoughLeft =
-				    !primaryProcessesLeft.validate(tLogPolicy) || !primaryProcessesLeft.validate(storagePolicy);
-			} else {
-				bool remoteTLogsDead = tLogWriteAntiQuorum ? !validateAllCombinations(badCombo,
-				                                                                      remoteProcessesDead,
-				                                                                      tLogPolicy,
-				                                                                      remoteLocalitiesLeft,
-				                                                                      tLogWriteAntiQuorum,
-				                                                                      false)
-				                                           : remoteProcessesDead.validate(tLogPolicy);
-				if (usableRegions > 1 && remoteTLogPolicy && !remoteTLogsDead) {
-					remoteTLogsDead = remoteProcessesDead.validate(remoteTLogPolicy);
-				}
-
-				if (!hasSatelliteReplication) {
-					if (usableRegions > 1) {
-						tooManyDead = primaryTLogsDead || remoteTLogsDead ||
-						              (primaryProcessesDead.validate(storagePolicy) &&
-						               remoteProcessesDead.validate(storagePolicy));
-						notEnoughLeft = !primaryProcessesLeft.validate(tLogPolicy) ||
-						                !primaryProcessesLeft.validate(remoteTLogPolicy) ||
-						                !primaryProcessesLeft.validate(storagePolicy) ||
-						                !remoteProcessesLeft.validate(tLogPolicy) ||
-						                !remoteProcessesLeft.validate(remoteTLogPolicy) ||
-						                !remoteProcessesLeft.validate(storagePolicy);
-					} else {
-						tooManyDead = primaryTLogsDead || remoteTLogsDead ||
-						              primaryProcessesDead.validate(storagePolicy) ||
-						              remoteProcessesDead.validate(storagePolicy);
-						notEnoughLeft = !primaryProcessesLeft.validate(tLogPolicy) ||
-						                !primaryProcessesLeft.validate(storagePolicy) ||
-						                !remoteProcessesLeft.validate(tLogPolicy) ||
-						                !remoteProcessesLeft.validate(storagePolicy);
-					}
-				} else {
-					bool primarySatelliteTLogsDead =
-					    satelliteTLogWriteAntiQuorumFallback
-					        ? !validateAllCombinations(badCombo,
-					                                   primarySatelliteProcessesDead,
-					                                   satelliteTLogPolicyFallback,
-					                                   primarySatelliteLocalitiesLeft,
-					                                   satelliteTLogWriteAntiQuorumFallback,
-					                                   false)
-					        : primarySatelliteProcessesDead.validate(satelliteTLogPolicyFallback);
-					// Ignore remoteSatelliteTLogsDead because remote satellites are not used and
-					// not affecting recovery.
-					/* bool remoteSatelliteTLogsDead =
-					    satelliteTLogWriteAntiQuorumFallback
-					        ? !validateAllCombinations(badCombo,
-					                                   remoteSatelliteProcessesDead,
-					                                   satelliteTLogPolicyFallback,
-					                                   remoteSatelliteLocalitiesLeft,
-					                                   satelliteTLogWriteAntiQuorumFallback,
-					                                   false)
-					        : remoteSatelliteProcessesDead.validate(satelliteTLogPolicyFallback); */
-
-					if (usableRegions > 1) {
-						notEnoughLeft = !primaryProcessesLeft.validate(tLogPolicy) ||
-						                !primaryProcessesLeft.validate(remoteTLogPolicy) ||
-						                !primaryProcessesLeft.validate(storagePolicy) ||
-						                !primarySatelliteProcessesLeft.validate(satelliteTLogPolicy) ||
-						                !remoteProcessesLeft.validate(tLogPolicy) ||
-						                !remoteProcessesLeft.validate(remoteTLogPolicy) ||
-						                !remoteProcessesLeft.validate(storagePolicy) ||
-						                !remoteSatelliteProcessesLeft.validate(satelliteTLogPolicy);
-					} else {
-						notEnoughLeft = !primaryProcessesLeft.validate(tLogPolicy) ||
-						                !primaryProcessesLeft.validate(storagePolicy) ||
-						                !primarySatelliteProcessesLeft.validate(satelliteTLogPolicy) ||
-						                !remoteProcessesLeft.validate(tLogPolicy) ||
-						                !remoteProcessesLeft.validate(storagePolicy) ||
-						                !remoteSatelliteProcessesLeft.validate(satelliteTLogPolicy);
-					}
-
-					if (usableRegions > 1 && allowLogSetKills) {
-						tooManyDead = (primaryTLogsDead && primarySatelliteTLogsDead) || remoteTLogsDead ||
-						              (primaryTLogsDead && remoteTLogsDead) ||
-						              (primaryProcessesDead.validate(storagePolicy) &&
-						               remoteProcessesDead.validate(storagePolicy));
-					} else {
-						tooManyDead = primaryTLogsDead || remoteTLogsDead ||
-						              primaryProcessesDead.validate(storagePolicy) ||
-						              remoteProcessesDead.validate(storagePolicy);
-					}
-				}
-			}
-
-			// Reboot if dead machines do fulfill policies
-			if (tooManyDead || (usableRegions > 1 && notEnoughLeft)) {
-				newKt = KillType::Reboot;
-				canSurvive = false;
-				TraceEvent("KillChanged")
-				    .detail("KillType", kt)
-				    .detail("NewKillType", newKt)
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("Reason", "Too many dead processes that cannot satisfy tLogPolicy.");
-			}
-			// Reboot and Delete if remaining machines do NOT fulfill policies
-			else if ((kt < KillType::RebootAndDelete) && notEnoughLeft) {
-				newKt = KillType::RebootAndDelete;
-				canSurvive = false;
-				TraceEvent("KillChanged")
-				    .detail("KillType", kt)
-				    .detail("NewKillType", newKt)
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("Reason", "Not enough tLog left to satisfy tLogPolicy.");
-			} else if ((kt < KillType::RebootAndDelete) && (nQuorum > uniqueMachines.size())) {
-				newKt = KillType::RebootAndDelete;
-				canSurvive = false;
-				TraceEvent("KillChanged")
-				    .detail("KillType", kt)
-				    .detail("NewKillType", newKt)
-				    .detail("StoragePolicy", storagePolicy->info())
-				    .detail("Quorum", nQuorum)
-				    .detail("Machines", uniqueMachines.size())
-				    .detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
-			} else {
-				TraceEvent("CanSurviveKills")
-				    .detail("KillType", kt)
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("StoragePolicy", storagePolicy->info())
-				    .detail("Quorum", nQuorum)
-				    .detail("Machines", uniqueMachines.size());
-			}
+		if (getSimulationPolicy()) {
+			return getSimulationPolicy()->canKillProcesses(availableProcesses, deadProcesses, kt, newKillType);
+		}
+		if (newKillType) {
+			*newKillType = kt;
 		}
-		if (newKillType)
-			*newKillType = newKt;
-		return canSurvive;
+		return true;
 	}
 
 	void destroyProcess(ISimulator::ProcessInfo* p) override {
@@ -1826,7 +1590,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			    .detail("ZoneId", machine->locality.zoneId())
 			    .detail("Process", machine->toString())
 			    .detail("Rebooting", machine->rebooting)
-			    .detail("Protected", protectedAddresses.count(machine->address))
+			    .detail("Protected", isProtectedAddress(machine->address))
 			    .backtrace();
 			// This will remove all the "tracked" messages that came from the machine being killed
 			if (!machine->isSpawnedKVProcess())
@@ -1839,7 +1603,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			    .detail("ZoneId", machine->locality.zoneId())
 			    .detail("Process", machine->toString())
 			    .detail("Rebooting", machine->rebooting)
-			    .detail("Protected", protectedAddresses.count(machine->address))
+			    .detail("Protected", isProtectedAddress(machine->address))
 			    .backtrace();
 			should_inject_fault = simulator_should_inject_fault;
 			machine->fault_injection_r = deterministicRandom()->randomUniqueID().first();
@@ -1852,16 +1616,16 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			    .detail("ZoneId", machine->locality.zoneId())
 			    .detail("Process", machine->toString())
 			    .detail("Rebooting", machine->rebooting)
-			    .detail("Protected", protectedAddresses.count(machine->address))
+			    .detail("Protected", isProtectedAddress(machine->address))
 			    .backtrace();
 			machine->failedDisk = true;
 		} else {
 			ASSERT(false);
 		}
-		ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting || machine->isSpawnedKVProcess());
+		ASSERT(!isProtectedAddress(machine->address) || machine->rebooting || machine->isSpawnedKVProcess());
 	}
 	void rebootProcess(ProcessInfo* process, KillType kt) override {
-		if (kt == KillType::RebootProcessAndDelete && protectedAddresses.count(process->address)) {
+		if (kt == KillType::RebootProcessAndDelete && isProtectedAddress(process->address)) {
 			TraceEvent("RebootChanged")
 			    .detail("ZoneId", process->locality.describeZone())
 			    .detail("KillType", KillType::RebootProcess)
@@ -1869,14 +1633,14 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			    .detail("Reason", "Protected process");
 			kt = KillType::RebootProcess;
 		}
-		doReboot(process, kt);
+		doReboot(Uncancellable(), process, kt);
 	}
 	void rebootProcess(Optional<Standalone<StringRef>> zoneId, bool allProcesses) override {
 		if (allProcesses) {
 			auto processes = getAllProcesses();
 			for (int i = 0; i < processes.size(); i++)
 				if (processes[i]->locality.zoneId() == zoneId && !processes[i]->rebooting)
-					doReboot(processes[i], KillType::RebootProcess);
+					doReboot(Uncancellable(), processes[i], KillType::RebootProcess);
 		} else {
 			auto processes = getAllProcesses();
 			for (int i = 0; i < processes.size(); i++) {
@@ -1884,14 +1648,14 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 					swapAndPop(&processes, i--);
 				}
 			}
-			if (processes.size())
-				doReboot(deterministicRandom()->randomChoice(processes), KillType::RebootProcess);
+			if (!processes.empty())
+				doReboot(Uncancellable(), deterministicRandom()->randomChoice(processes), KillType::RebootProcess);
 		}
 	}
 	void killProcess(ProcessInfo* machine, KillType kt) override {
 		TraceEvent("AttemptingKillProcess").detail("ProcessInfo", machine->toString());
 		// Refuse to kill a protected process.
-		if (kt < KillType::RebootAndDelete && protectedAddresses.count(machine->address) == 0) {
+		if (kt < KillType::RebootAndDelete && !isProtectedAddress(machine->address)) {
 			killProcess_internal(machine, kt);
 		}
 	}
@@ -1900,7 +1664,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			std::vector<ProcessInfo*>& processes = machines[addressMap[address]->locality.machineId()].processes;
 			for (auto& process : processes) {
 				// Refuse to kill a protected process.
-				if (protectedAddresses.count(process->address) == 0) {
+				if (!isProtectedAddress(process->address)) {
 					killProcess_internal(process, kt);
 				}
 			}
@@ -1980,7 +1744,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		KillType originalKt = kt;
 		// Reboot if any of the processes are protected and count the number of processes not rebooting
 		for (auto& process : machines[machineId].processes) {
-			if (protectedAddresses.count(process->address) && kt != KillType::RebootProcessAndSwitch) {
+			if (isProtectedAddress(process->address) && kt != KillType::RebootProcessAndSwitch) {
 				kt = KillType::Reboot;
 			}
 
@@ -2023,7 +1787,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 					} else if (!processInfo->isAvailable()) {
 						processesDead.push_back(processInfo);
 						unavailable++;
-					} else if (protectedAddresses.count(processInfo->address)) {
+					} else if (isProtectedAddress(processInfo->address)) {
 						processesLeft.push_back(processInfo);
 						protectedWorker++;
 					} else if (processInfo->locality.machineId() != machineId) {
@@ -2046,9 +1810,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				    .detail("Unavailable", unavailable)
 				    .detail("Excluded", excluded)
 				    .detail("Cleared", cleared)
-				    .detail("ProtectedTotal", protectedAddresses.size())
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("StoragePolicy", storagePolicy->info());
+				    .detail("ProtectedTotal", protectedAddressCount());
 			} else if ((kt == KillType::KillInstantly) || (kt == KillType::InjectFaults) ||
 			           (kt == KillType::FailDisk)) {
 				TraceEvent("DeadMachine")
@@ -2057,9 +1819,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				    .detail("ProcessesLeft", processesLeft.size())
 				    .detail("ProcessesDead", processesDead.size())
 				    .detail("TotalProcesses", machines.size())
-				    .detail("ProcessesPerMachine", processesPerMachine)
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("StoragePolicy", storagePolicy->info());
+				    .detail("ProcessesPerMachine", processesPerMachine);
 				for (auto process : processesLeft) {
 					TraceEvent("DeadMachineSurvivors")
 					    .detail("MachineId", machineId)
@@ -2083,9 +1843,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				    .detail("ProcessesLeft", processesLeft.size())
 				    .detail("ProcessesDead", processesDead.size())
 				    .detail("TotalProcesses", machines.size())
-				    .detail("ProcessesPerMachine", processesPerMachine)
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("StoragePolicy", storagePolicy->info());
+				    .detail("ProcessesPerMachine", processesPerMachine);
 				for (auto process : processesLeft) {
 					TraceEvent("ClearMachineSurvivors")
 					    .detail("MachineId", machineId)
@@ -2168,7 +1926,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				    .detail("Cleared", process->cleared)
 				    .detail("Rebooting", process->rebooting);
 				if (process->startingClass != ProcessClass::TesterClass)
-					doReboot(process, kt);
+					doReboot(Uncancellable(), process, kt);
 			}
 		}
 
@@ -2201,7 +1959,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			auto processMachineId = procRecord->locality.machineId();
 			ASSERT(processMachineId.present());
 			if (processDcId.present() && (processDcId == dcId)) {
-				if ((kt != KillType::Reboot) && (protectedAddresses.count(procRecord->address))) {
+				if ((kt != KillType::Reboot) && (isProtectedAddress(procRecord->address))) {
 					kt = KillType::Reboot;
 					TraceEvent(SevWarn, "DcKillChanged")
 					    .detail("DataCenter", dcId)
@@ -2229,7 +1987,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				if (processInfo->isAvailableClass()) {
 					if (processInfo->isExcluded() || processInfo->isCleared() || !processInfo->isAvailable()) {
 						processesDead.push_back(processInfo);
-					} else if (protectedAddresses.count(processInfo->address) ||
+					} else if (isProtectedAddress(processInfo->address) ||
 					           datacenterMachines.find(processInfo->locality.machineId()) == datacenterMachines.end()) {
 						processesLeft.push_back(processInfo);
 					} else {
@@ -2250,9 +2008,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 				    .detail("DcZones", datacenterMachines.size())
 				    .detail("DcProcesses", dcProcesses)
 				    .detail("ProcessesDead", processesDead.size())
-				    .detail("ProcessesLeft", processesLeft.size())
-				    .detail("TLogPolicy", tLogPolicy->info())
-				    .detail("StoragePolicy", storagePolicy->info());
+				    .detail("ProcessesLeft", processesLeft.size());
 				for (auto process : processesLeft) {
 					TraceEvent("DeadDcSurvivors")
 					    .detail("MachineId", process->locality.machineId())
@@ -2411,17 +2167,16 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		machines.erase(machineId);
 	}
 
-	// Assumes the simulator is already onProcess for proc
-	// FIXME: delete the above comment and add an ASSERT about this condition.
 	void startRequestHandlerOnProcess(ProcessInfo* process,
 	                                  Reference<HTTP::SimServerContext> serverContext,
 	                                  Reference<HTTP::SimRegisteredHandlerContext> handlerContext) {
+		ASSERT(getCurrentProcess() == process);
 		try {
 			NetworkAddress addr = NetworkAddress(g_simulator->getCurrentProcess()->address.ip,
 			                                     handlerContext->port,
 			                                     true /* isPublic*/,
 			                                     false /*isTLS*/);
-			process->listenerMap[addr] = Reference<IListener>(new Sim2Listener(process, addr));
+			process->listenerMap[addr] = makeReference<Sim2Listener>(process, addr);
 			addressMap[addr] = process;
 			handlerContext->addAddress(addr);
 			serverContext->registerNewServer(addr, handlerContext->requestHandler->clone());
@@ -2443,7 +2198,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 			fmt::print("SimHTTPServer protecting {0}\n", p->address.toString());
 			TraceEvent(SevDebug, "HTTPProcessProtected").detail("Address", p->address);
 			g_simulator->httpProtected = true;
-			protectedAddresses.insert(p->address);
+			protectAddress(p->address);
 		}
 		// make sure this process isn't already added
 		for (int i = 0; i < httpServerProcesses.size(); i++) {
@@ -2476,35 +2231,33 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		}
 	}
 
-	ACTOR static Future<Void> registerSimHTTPServerActor(Sim2* self,
-	                                                     std::string hostname,
-	                                                     std::string service,
-	                                                     Reference<HTTP::IRequestHandler> requestHandler) {
+	static Future<Void> registerSimHTTPServerActor(Sim2* self,
+	                                               std::string hostname,
+	                                               std::string service,
+	                                               Reference<HTTP::IRequestHandler> requestHandler) {
 		std::string id = hostname + ":" + service;
-		ASSERT(!self->httpHandlers.count(id));
+		ASSERT(!self->httpHandlers.contains(id));
 
 		// check not too many servers
 		ASSERT(self->httpHandlers.size() < 1000);
-		state Reference<HTTP::SimRegisteredHandlerContext> handlerContext =
+		auto handlerContext =
 		    makeReference<HTTP::SimRegisteredHandlerContext>(hostname, service, self->nextHTTPPort++, requestHandler);
 		self->httpHandlers.insert({ id, handlerContext });
 
 		// start process on all running HTTP servers
-		state ProcessInfo* callingProcess = self->getCurrentProcess();
-		state int i = 0;
+		ProcessInfo* callingProcess = self->getCurrentProcess();
 
 		// copy the processes before waits just to ensure no races with addSimHTTPProcess
-		state std::vector<std::pair<ProcessInfo*, Reference<HTTP::SimServerContext>>> procsCopy =
-		    self->httpServerProcesses;
-		for (; i < procsCopy.size(); i++) {
-			state ProcessInfo* serverProcess = procsCopy[i].first;
-			wait(self->onProcess(serverProcess, TaskPriority::DefaultYield));
+		std::vector<std::pair<ProcessInfo*, Reference<HTTP::SimServerContext>>> procsCopy = self->httpServerProcesses;
+		for (int i = 0; i < procsCopy.size(); i++) {
+			ProcessInfo* serverProcess = procsCopy[i].first;
+			co_await self->onProcess(serverProcess, TaskPriority::DefaultYield);
 			self->startRequestHandlerOnProcess(serverProcess, procsCopy[i].second, handlerContext);
 		}
 
-		wait(self->onProcess(callingProcess, TaskPriority::DefaultYield));
+		co_await self->onProcess(callingProcess, TaskPriority::DefaultYield);
 
-		return Void();
+		co_return;
 	}
 
 	// starts a numAddresses http servers with the dns alias hostname:service with the provided server callback
@@ -2514,7 +2267,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		return registerSimHTTPServerActor(this, hostname, service, requestHandler);
 	}
 
-	Sim2(bool printSimTime)
+	explicit Sim2(bool printSimTime)
 	  : time(0.0), timerTime(0.0), currentTaskID(TaskPriority::Zero), yielded(false), yield_limit(0),
 	    printSimTime(printSimTime) {
 		// Not letting currentProcess be nullptr eliminates some annoying special cases
@@ -2582,7 +2335,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		ASSERT(getCurrentProcess());
 		ASSERT(taskID >= TaskPriority::Min && taskID <= TaskPriority::Max);
 
-		PromiseTask* p = new PromiseTask(getCurrentProcess(), std::move(signal));
+		auto* p = new PromiseTask(getCurrentProcess(), std::move(signal));
 		taskQueue.addReadyThreadSafe(isOnMainThread(), taskID, p);
 	}
 	bool isOnMainThread() const override { return ISimulator::isMainThread; }
@@ -2648,29 +2401,29 @@ class UDPSimSocket : public IUDPSocket, ReferenceCounted<UDPSimSocket> {
 	bool isClosed() const { return closed.getFuture().isReady(); }
 	Future<Void> onClosed() const { return closed.getFuture(); }
 
-	ACTOR static Future<Void> cleanupPeerSocket(UDPSimSocket* self) {
-		wait(self->peerSocket.get()->onClosed());
+	static Future<Void> cleanupPeerSocket(UDPSimSocket* self) {
+		co_await self->peerSocket.get()->onClosed();
 		self->peerSocket.reset();
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<Void> send(UDPSimSocket* self,
-	                               Reference<UDPSimSocket> peerSocket,
-	                               uint8_t const* begin,
-	                               uint8_t const* end) {
-		state Packet packet(std::make_shared<std::vector<uint8_t>>());
+	static Future<Void> send(UDPSimSocket* self,
+	                         Reference<UDPSimSocket> peerSocket,
+	                         uint8_t const* begin,
+	                         uint8_t const* end) {
+		Packet packet(std::make_shared<std::vector<uint8_t>>());
 		packet->resize(end - begin);
 		std::copy(begin, end, packet->begin());
-		wait(delay(.002 * deterministicRandom()->random01()));
+		co_await delay(.002 * deterministicRandom()->random01());
 		peerSocket->recvBuffer.emplace_back(self->_localAddress, std::move(packet));
 		peerSocket->writtenPackets.set(peerSocket->writtenPackets.get() + 1);
-		return Void();
+		co_return;
 	}
 
-	ACTOR static Future<int> receiveFrom(UDPSimSocket* self, uint8_t* begin, uint8_t* end, NetworkAddress* sender) {
-		state TaskPriority currentTaskID = g_sim2.getCurrentTask();
-		wait(self->writtenPackets.onChange());
-		wait(g_sim2.onProcess(self->process, currentTaskID));
+	static Future<int> receiveFrom(UDPSimSocket* self, uint8_t* begin, uint8_t* end, NetworkAddress* sender) {
+		TaskPriority currentTaskID = g_sim2.getCurrentTask();
+		co_await self->writtenPackets.onChange();
+		co_await g_sim2.onProcess(self->process, currentTaskID);
 		auto packet = self->recvBuffer.front().second;
 		int sz = packet->size();
 		ASSERT(sz <= end - begin);
@@ -2679,7 +2432,7 @@ class UDPSimSocket : public IUDPSocket, ReferenceCounted<UDPSimSocket> {
 		}
 		std::copy(packet->begin(), packet->end(), begin);
 		self->recvBuffer.pop_front();
-		return sz;
+		co_return sz;
 	}
 
 public:
@@ -2688,7 +2441,7 @@ class UDPSimSocket : public IUDPSocket, ReferenceCounted<UDPSimSocket> {
 	    peerAddress(peerAddress), actors(false), _localAddress(localAddress) {
 		g_sim2.addressMap.emplace(_localAddress, process);
 		ASSERT(process->boundUDPSockets.find(localAddress) == process->boundUDPSockets.end());
-		process->boundUDPSockets.emplace(localAddress, this);
+		process->boundUDPSockets.emplace(localAddress, Reference<IUDPSocket>::addRef(this));
 	}
 	~UDPSimSocket() override {
 		if (!closed.getFuture().isReady()) {
@@ -2801,7 +2554,7 @@ Future<Reference<IUDPSocket>> Sim2::createUDPSocket(NetworkAddress toAddr) {
 	while (process->boundUDPSockets.find(localAddress) != process->boundUDPSockets.end()) {
 		localAddress.port = deterministicRandom()->randomInt(40000, 60000);
 	}
-	return Reference<IUDPSocket>(new UDPSimSocket(localAddress, toAddr));
+	return Reference<IUDPSocket>(makeReference<UDPSimSocket>(localAddress, toAddr));
 }
 
 Future<Reference<IUDPSocket>> Sim2::createUDPSocket(bool isV6) {
@@ -2834,7 +2587,7 @@ void startNewSimulator(bool printSimTime) {
 	    deterministicRandom()->coinflip() ? 0 : DISABLE_CONNECTION_FAILURE_FOREVER;
 }
 
-ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
+Future<Void> doReboot(Uncancellable, ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 	TraceEvent("RebootingProcessAttempt")
 	    .detail("ZoneId", p->locality.zoneId())
 	    .detail("KillType", kt)
@@ -2846,7 +2599,7 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 	    .detail("Rebooting", p->rebooting)
 	    .detail("TaskPriorityDefaultDelay", TaskPriority::DefaultDelay);
 
-	wait(g_sim2.delay(0, TaskPriority::DefaultDelay, p)); // Switch to the machine in question
+	co_await g_sim2.delay(0, TaskPriority::DefaultDelay, p); // Switch to the machine in question
 
 	try {
 		ASSERT(kt == ISimulator::KillType::RebootProcess || kt == ISimulator::KillType::Reboot ||
@@ -2878,13 +2631,13 @@ ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
 			TraceEvent(SevDebug, "DoRebootFailed")
 			    .detail("Rebooting", p->rebooting)
 			    .detail("Reliable", p->isReliable());
-			return;
+			co_return;
 		} else if (p->isSpawnedKVProcess()) {
 			TraceEvent(SevDebug, "DoRebootFailed").detail("Name", p->name).detail("Address", p->address);
-			return;
-		} else if (p->getChilds().size()) {
+			co_return;
+		} else if (!p->getChilds().empty()) {
 			TraceEvent(SevDebug, "DoRebootFailedOnParentProcess").detail("Address", p->address);
-			return;
+			co_return;
 		}
 
 		TraceEvent("RebootingProcess")
@@ -3034,8 +2787,9 @@ Future<Reference<class IAsyncFile>> Sim2FileSystem::open(const std::string& file
 
 			f = SimpleFile::open(filename, flags, mode, diskParameters, false);
 			if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) {
-				f = map(f,
-				        [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+				f = map(f, [=](Reference<IAsyncFile> r) -> Reference<IAsyncFile> {
+					return makeReference<AsyncFileWriteChecker>(r);
+				});
 			}
 
 			f = AsyncFileNonDurable::open(
@@ -3048,13 +2802,8 @@ Future<Reference<class IAsyncFile>> Sim2FileSystem::open(const std::string& file
 
 		f = AsyncFileDetachable::open(f);
 		if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES)
-			f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileChaos(r)); });
-		if (flags & IAsyncFile::OPEN_ENCRYPTED)
-			f = map(f, [flags](Reference<IAsyncFile> r) {
-				auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY
-				                                               : AsyncFileEncrypted::Mode::READ_ONLY;
-				return Reference<IAsyncFile>(new AsyncFileEncrypted(r, mode));
-			});
+			f = map(f,
+			        [=](Reference<IAsyncFile> r) -> Reference<IAsyncFile> { return makeReference<AsyncFileChaos>(r); });
 		return f;
 	} else
 		return AsyncFileCached::open(filename, flags, mode);
@@ -3066,8 +2815,8 @@ Future<Void> Sim2FileSystem::deleteFile(const std::string& filename, bool mustBe
 	return Sim2::deleteFileImpl(&g_sim2, filename, mustBeDurable);
 }
 
-ACTOR Future<Void> renameFileImpl(std::string from, std::string to) {
-	wait(delay(0.5 * deterministicRandom()->random01()));
+Future<Void> renameFileImpl(std::string from, std::string to) {
+	co_await delay(0.5 * deterministicRandom()->random01());
 	// rename all keys in the corrupted list
 	// first we have to delete all corruption of the destination, since this file will be unlinked if it exists
 	TraceEvent("RenamingFile").detail("From", from).detail("To", to).log();
@@ -3085,8 +2834,8 @@ ACTOR Future<Void> renameFileImpl(std::string from, std::string to) {
 	g_simulator->corruptedBlocks.erase(begin, end);
 	// do the rename
 	::renameFile(from, to);
-	wait(delay(0.5 * deterministicRandom()->random01()));
-	return Void();
+	co_await delay(0.5 * deterministicRandom()->random01());
+	co_return;
 }
 
 Future<Void> Sim2FileSystem::renameFile(std::string const& from, std::string const& to) {
diff --git a/fdbrpc/sim_validation.cpp b/fdbrpc/sim_validation.cpp
index d0093af0cc7..a8390289db4 100644
--- a/fdbrpc/sim_validation.cpp
+++ b/fdbrpc/sim_validation.cpp
@@ -28,6 +28,14 @@ static std::map<std::string, int64_t> validationData;
 static std::map<int64_t, double> timedVersionsValidationData;
 static std::set<UID> disabledMachines;
 
+static bool versionValidationDisabled() {
+	if (!g_network->isSimulated()) {
+		return true;
+	}
+	auto policy = g_simulator->getSimulationPolicy();
+	return policy && !policy->shouldRunVersionValidation();
+}
+
 void debug_setVersionCheckEnabled(UID uid, bool enabled) {
 	if (enabled)
 		disabledMachines.erase(uid);
@@ -43,7 +51,7 @@ void debug_advanceCommittedVersions(UID id, int64_t minVersion, int64_t maxVersi
 }
 
 void debug_advanceVersion(UID id, int64_t version, const char* suffix) {
-	if (!disabledMachines.count(id)) {
+	if (!disabledMachines.contains(id)) {
 		auto& entry = validationData[id.toString() + suffix];
 		if (version > entry)
 			entry = version;
@@ -51,13 +59,13 @@ void debug_advanceVersion(UID id, int64_t version, const char* suffix) {
 }
 
 void debug_advanceMinCommittedVersion(UID id, int64_t version) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return;
 	debug_advanceVersion(id, version, "min");
 }
 
 void debug_advanceMaxCommittedVersion(UID id, int64_t version) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return;
 	debug_advanceVersion(id, version, "max");
 }
@@ -67,11 +75,11 @@ bool debug_checkPartRestoredVersion(UID id,
                                     std::string context,
                                     std::string minormax,
                                     Severity sev = SevError) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return false;
-	if (disabledMachines.count(id))
+	if (disabledMachines.contains(id))
 		return false;
-	if (!validationData.count(id.toString() + minormax)) {
+	if (!validationData.contains(id.toString() + minormax)) {
 		TraceEvent(SevWarn, (context + "UnknownVersion").c_str(), id).detail("RestoredVersion", version);
 		return false;
 	}
@@ -88,33 +96,33 @@ bool debug_checkPartRestoredVersion(UID id,
 }
 
 bool debug_checkRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return false;
 	return debug_checkPartRestoredVersion(id, version, context, "min", sev) ||
 	       debug_checkPartRestoredVersion(id, version, context, "max", sev);
 }
 
 void debug_removeVersions(UID id) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return;
 	validationData.erase(id.toString() + "min");
 	validationData.erase(id.toString() + "max");
 }
 
 bool debug_versionsExist(UID id) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return false;
-	return validationData.count(id.toString() + "min") != 0 || validationData.count(id.toString() + "max") != 0;
+	return validationData.contains(id.toString() + "min") || validationData.contains(id.toString() + "max");
 }
 
 bool debug_checkMinRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return false;
 	return debug_checkPartRestoredVersion(id, version, context, "min", sev);
 }
 
 bool debug_checkMaxRestoredVersion(UID id, int64_t version, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return false;
 	return debug_checkPartRestoredVersion(id, version, context, "max", sev);
 }
@@ -129,15 +137,15 @@ void debug_setCheckRelocationDuration(bool check) {
 	checkRelocationDuration = check;
 }
 void debug_advanceVersionTimestamp(int64_t version, double t) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return;
 	timedVersionsValidationData[version] = t;
 }
 
 bool debug_checkVersionTime(int64_t version, double t, std::string context, Severity sev) {
-	if (!g_network->isSimulated() || !g_simulator->extraDatabases.empty())
+	if (versionValidationDisabled())
 		return false;
-	if (!timedVersionsValidationData.count(version)) {
+	if (!timedVersionsValidationData.contains(version)) {
 		TraceEvent(SevWarn, (context + "UnknownTime").c_str())
 		    .detail("VersionChecking", version)
 		    .detail("TimeChecking", t);
diff --git a/fdbrpc/swift_sim2_hooks.cpp b/fdbrpc/swift_sim2_hooks.cpp
index b7f700c9c8c..63dfd22a929 100644
--- a/fdbrpc/swift_sim2_hooks.cpp
+++ b/fdbrpc/swift_sim2_hooks.cpp
@@ -21,7 +21,7 @@
 #include "fdbrpc/simulator.h"
 #include "flow/flow.h"
 #include "flow/network.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/swift_concurrency_hooks.h"
 #include "flow/swift.h"
 #include "flow/swift/ABI/Task.h"
diff --git a/fdbrpc/tests/AuthzTlsTest.actor.cpp b/fdbrpc/tests/AuthzTlsTest.cpp
similarity index 98%
rename from fdbrpc/tests/AuthzTlsTest.actor.cpp
rename to fdbrpc/tests/AuthzTlsTest.cpp
index cac52014c81..915f2deb199 100644
--- a/fdbrpc/tests/AuthzTlsTest.actor.cpp
+++ b/fdbrpc/tests/AuthzTlsTest.cpp
@@ -41,9 +41,7 @@
 #include "flow/Error.h"
 #include "flow/MkCert.h"
 #include "flow/ScopeExit.h"
-#include "flow/TLSConfig.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TLSConfig.h"
 
 using namespace std::literals::string_view_literals;
 
@@ -181,11 +179,11 @@ struct fmt::formatter<Result> : fmt::formatter<std::string> {
 	}
 };
 
-ACTOR template <class T>
-Future<T> stopNetworkAfter(Future<T> what) {
-	T t = wait(what);
+template <class T>
+Future<T> stopNetworkAfter(Future<T> what, ExplicitVoid = {}) {
+	T t = co_await what;
 	g_network->stop();
-	return t;
+	co_return t;
 }
 
 // Reflective struct containing information about the requester from a server PoV
@@ -214,7 +212,7 @@ struct SessionProbeRequest {
 };
 
 struct SessionProbeReceiver final : NetworkMessageReceiver {
-	SessionProbeReceiver() {}
+	SessionProbeReceiver() = default;
 	void receive(ArenaObjectReader& reader) override {
 		SessionProbeRequest req;
 		reader.deserialize(req);
@@ -252,9 +250,9 @@ void runServer(const Endpoint& endpoint, int addrPipe, int completionPipe) {
 	return;
 }
 
-ACTOR Future<Void> waitAndPrintResponse(Future<SessionInfo> response, Result* rc) {
+Future<Void> waitAndPrintResponse(Future<SessionInfo> response, Result* rc) {
 	try {
-		SessionInfo info = wait(response);
+		SessionInfo info = co_await response;
 		log("Probe response: trusted={} peerAddress={}", info.isPeerTrusted, info.peerAddress.toString());
 		*rc = info.isPeerTrusted ? Result::TRUSTED : Result::UNTRUSTED;
 	} catch (Error& err) {
@@ -266,7 +264,6 @@ ACTOR Future<Void> waitAndPrintResponse(Future<SessionInfo> response, Result* rc
 			*rc = Result::TIMEOUT;
 		}
 	}
-	return Void();
 }
 
 // int runAsServer(TLSCreds creds, int addrPipe, int completionPipe, Result expect) {}
diff --git a/fdbrpc/tests/CMakeLists.txt b/fdbrpc/tests/CMakeLists.txt
index 732dcd37617..a0ba2a2dbba 100644
--- a/fdbrpc/tests/CMakeLists.txt
+++ b/fdbrpc/tests/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(NOT WIN32)
-  add_flow_target(EXECUTABLE NAME authz_tls_unittest SRCS AuthzTlsTest.actor.cpp)
+  add_flow_target(EXECUTABLE NAME authz_tls_unittest SRCS AuthzTlsTest.cpp)
   target_link_libraries(authz_tls_unittest PRIVATE flow fdbrpc fmt::fmt)
   if(NOT OPEN_FOR_IDE)
     add_test(NAME authorization_tls_unittest
@@ -11,5 +11,5 @@ endif()
 if(WITH_GRPC)
   generate_grpc_protobuf(fdbrpc.test protos/echo.proto)
 endif()
-add_flow_target(EXECUTABLE NAME fdbrpc_bench SRCS fdbrpc_bench.actor.cpp)
-target_link_libraries(fdbrpc_bench PUBLIC flow fdbrpc boost_target_program_options)
+add_flow_target(EXECUTABLE NAME fdbrpc_transport_bench SRCS fdbrpc_bench.cpp)
+target_link_libraries(fdbrpc_transport_bench PUBLIC flow fdbrpc boost_target_program_options)
diff --git a/fdbrpc/tests/fdbrpc_bench.actor.cpp b/fdbrpc/tests/fdbrpc_bench.cpp
similarity index 77%
rename from fdbrpc/tests/fdbrpc_bench.actor.cpp
rename to fdbrpc/tests/fdbrpc_bench.cpp
index 55aae1d1434..0edd9121c94 100644
--- a/fdbrpc/tests/fdbrpc_bench.actor.cpp
+++ b/fdbrpc/tests/fdbrpc_bench.cpp
@@ -1,5 +1,5 @@
 /*
- * fdbrpc_bench.actor.cpp
+ * fdbrpc_bench.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,10 +23,9 @@
 
 #include "flow/flow.h"
 #include "flow/Platform.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/FlowTransport.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 namespace fdbrpc_bench {
 NetworkAddress serverAddress;
@@ -72,7 +71,7 @@ struct EchoRequest {
 // where each entry counts the number of hits each second.
 class StatCounter {
 public:
-	StatCounter(int size = 10) : vals(size) {}
+	explicit StatCounter(int size = 10) : vals(size) {}
 
 	// Returns the average number of hits per seconds over last `size` seconds.
 	int avg() {
@@ -110,33 +109,60 @@ class StatCounter {
 	std::vector<std::pair<int64_t, int>> vals;
 };
 
-ACTOR Future<Void> echoServer() {
-	state EchoServerInterface echoServer;
-	state StatCounter c;
-	echoServer.getInterface.makeWellKnownEndpoint(WLTOKEN_ECHO_SERVER, TaskPriority::DefaultEndpoint);
-	state Future<Void> next = delay(10);
-	loop {
-		try {
-			choose {
-				when(GetInterfaceRequest req = waitNext(echoServer.getInterface.getFuture())) {
-					req.reply.send(echoServer);
-				}
-				when(EchoRequest req = waitNext(echoServer.echo.getFuture())) {
-					req.reply.send(req.message);
-					c.inc();
-				}
-				when(wait(next)) {
-					next = delay(10);
-					std::cout << "Throughput: " << c.avg() << " req/sec" << std::endl;
-				}
+class EchoServer {
+	static void rethrowUnexpectedError(const Error& e) {
+		if (e.code() != error_code_operation_obsolete) {
+			fprintf(stderr, "Error: %s\n", e.what());
+			throw e;
+		}
+	}
+
+	Future<Void> serveGetInterfaceReqs() {
+		while (true) {
+			try {
+				GetInterfaceRequest req = co_await interf.getInterface.getFuture();
+				req.reply.send(interf);
+			} catch (Error& e) {
+				rethrowUnexpectedError(e);
+			}
+		}
+	}
+
+	Future<Void> serveEchoReqs() {
+		while (true) {
+			try {
+				EchoRequest req = co_await interf.echo.getFuture();
+				req.reply.send(req.message);
+				counter.inc();
+			} catch (Error& e) {
+				rethrowUnexpectedError(e);
 			}
-		} catch (Error& e) {
-			if (e.code() != error_code_operation_obsolete) {
-				fprintf(stderr, "Error: %s\n", e.what());
-				throw e;
+		}
+	}
+
+	Future<Void> printThroughput() {
+		while (true) {
+			try {
+				co_await delay(10);
+				std::cout << "Throughput: " << counter.avg() << " req/sec" << std::endl;
+			} catch (Error& e) {
+				rethrowUnexpectedError(e);
 			}
 		}
 	}
+
+	EchoServerInterface interf;
+	StatCounter counter;
+
+public:
+	EchoServer() { interf.getInterface.makeWellKnownEndpoint(WLTOKEN_ECHO_SERVER, TaskPriority::DefaultEndpoint); }
+
+	Future<Void> run() { co_await race(serveGetInterfaceReqs(), serveEchoReqs(), printThroughput()); }
+};
+
+Future<Void> echoServer() {
+	EchoServer server;
+	co_await server.run();
 }
 
 int payload_size_bytes = 1024 * 10;
@@ -156,27 +182,26 @@ std::string randString(int size) {
 	return result;
 }
 
-ACTOR Future<Void> echoClient() {
+Future<Void> echoClient() {
 	std::cout << "Starting client. Payload size: " << payload_size_bytes << " bytes" << std::endl;
-	state EchoServerInterface server;
+	EchoServerInterface server;
 	server.getInterface =
 	    RequestStream<GetInterfaceRequest>(Endpoint::wellKnown({ serverAddress }, WLTOKEN_ECHO_SERVER));
-	EchoServerInterface s = wait(server.getInterface.getReply(GetInterfaceRequest()));
-	server = s;
-	state std::string payload = randString(payload_size_bytes);
+	server = co_await server.getInterface.getReply(GetInterfaceRequest());
+	std::string payload = randString(payload_size_bytes);
 
 	while (true) {
-		state int duration_seconds = 10;
-		state int request_count = 0;
+		int duration_seconds = 10;
+		int request_count = 0;
 
-		state std::chrono::time_point<std::chrono::steady_clock> start_time = std::chrono::steady_clock::now();
-		state std::chrono::time_point<std::chrono::steady_clock> end_time =
+		std::chrono::time_point<std::chrono::steady_clock> start_time = std::chrono::steady_clock::now();
+		std::chrono::time_point<std::chrono::steady_clock> end_time =
 		    start_time + std::chrono::seconds(duration_seconds);
 
 		while (std::chrono::steady_clock::now() < end_time) {
 			EchoRequest echoRequest;
 			echoRequest.message = payload;
-			std::string echoMessage = wait(server.echo.getReply(echoRequest));
+			co_await server.echo.getReply(echoRequest);
 			++request_count;
 		}
 		std::cout << "Sent " << request_count << " requests in " << request_count / duration_seconds << " /second"
@@ -229,7 +254,6 @@ int main(int argc, char* argv[]) {
 	}
 
 	bool isServer = (mode == "server");
-	std::string port;
 	std::vector<std::function<Future<Void>()>> toRun;
 	auto actor = actors.find(mode);
 	toRun.push_back(actor->second);
diff --git a/fdbserver/BackupWorkerRangePartitioned.actor.cpp b/fdbserver/BackupWorkerRangePartitioned.actor.cpp
deleted file mode 100644
index 17f9d04382f..00000000000
--- a/fdbserver/BackupWorkerRangePartitioned.actor.cpp
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * BackupWorkerRangePartitioned.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbclient/DatabaseContext.h"
-#include "fdbserver/Knobs.h"
-#include "fdbclient/Tracing.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-#define SevDebugMemory SevVerbose
-
-struct RangePartitionedVersionedMessage {
-	LogMessageVersion version;
-	StringRef message;
-	VectorRef<Tag> tags;
-	Arena arena;
-
-	RangePartitionedVersionedMessage(LogMessageVersion v, StringRef m, const VectorRef<Tag>& t, const Arena& a)
-	  : version(v), message(m), tags(t), arena(a) {}
-
-	Version getVersion() const { return version.version; }
-	size_t getEstimatedSize() const { return message.size() + TagsAndMessage::getHeaderSize(6); }
-};
-
-struct BackupRangePartitionedData {
-	const UID myId;
-	const Tag tag; // tag for this backup worker
-	const int totalTags; // Total backup worker tags
-	const Version startVersion; // This worker's start version
-	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
-	const LogEpoch recruitedEpoch; // current epoch whose tLogs are receiving mutations
-	const LogEpoch backupEpoch; // the epoch workers should pull mutations
-	Version minKnownCommittedVersion;
-	Version savedVersion; // Largest version saved to blob storage
-	NotifiedVersion pulledVersion;
-	AsyncVar<Reference<ILogSystem>> logSystem;
-	AsyncVar<bool> paused; // Track if "backupPausedKey" is set.
-	Reference<FlowLock> lock;
-	AsyncTrigger doneTrigger;
-	std::vector<RangePartitionedVersionedMessage> messages;
-	// Key range to partition ID map, used to determine which partition a mutation belongs to based on its key.
-	KeyRangeMap<uint64_t> keyRangeToPartition;
-
-	explicit BackupRangePartitionedData(UID id,
-	                                    Reference<AsyncVar<ServerDBInfo> const> db,
-	                                    const InitializeBackupRequest& req)
-	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
-	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), pulledVersion(0), paused(false),
-	    lock(new FlowLock(SERVER_KNOBS->BACKUP_WORKER_LOCK_BYTES)) {
-
-		// TODO akanksha: Initialize keyRangeToPartition map. Each backworker will receive a partition map
-		// which is BackupTag ->  list<Partition> mapping and Partition contains partition Id and key range. We can
-		// construct the keyRangeToPartition map from that.
-	}
-
-	bool pullFinished() const { return endVersion.present() && pulledVersion.get() > endVersion.get(); }
-
-	void eraseMessagesAfterEndVersion() {
-		ASSERT(endVersion.present());
-		const Version ver = endVersion.get();
-		while (!messages.empty()) {
-			if (messages.back().getVersion() > ver) {
-				size_t bytes = messages.back().getEstimatedSize();
-				TraceEvent(SevDebugMemory, "BWRangePartitionedMemory", myId).detail("Release", bytes);
-				lock->release(bytes);
-				messages.pop_back();
-			} else {
-				break;
-			}
-		}
-	}
-};
-
-ACTOR Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db,
-                                LogEpoch recoveryCount,
-                                BackupRangePartitionedData* self) {
-	loop {
-		bool isDisplaced =
-		    db->get().recoveryCount > recoveryCount && db->get().recoveryState != RecoveryState::UNINITIALIZED;
-		if (isDisplaced) {
-			TraceEvent("BWRangePartitionedDisplaced", self->myId)
-			    .detail("RecoveryCount", recoveryCount)
-			    .detail("RecoveryState", (int)db->get().recoveryState);
-			throw worker_removed();
-		}
-		wait(db->onChange());
-	}
-}
-
-ACTOR Future<Void> backupWorkerRangePartitioned(BackupInterface interf,
-                                                InitializeBackupRequest req,
-                                                Reference<AsyncVar<ServerDBInfo> const> db) {
-	state BackupRangePartitionedData self(interf.id(), db, req);
-	state PromiseStream<Future<Void>> addActor;
-	state Future<Void> error = actorCollection(addActor.getFuture());
-	state Future<Void> dbInfoChange = Void();
-	state Future<Void> done;
-
-	TraceEvent("BWRangePartitionedStart", self.myId)
-	    .detail("Tag", req.routerTag.toString())
-	    .detail("TotalTags", req.totalTags)
-	    .detail("StartVersion", req.startVersion)
-	    .detail("EndVersion", req.endVersion.present() ? req.endVersion.get() : -1)
-	    .detail("LogEpoch", req.recruitedEpoch)
-	    .detail("BackupEpoch", req.backupEpoch);
-
-	try {
-		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
-		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
-
-		loop choose {
-			when(wait(dbInfoChange)) {
-				dbInfoChange = db->onChange();
-				Reference<ILogSystem> ls = ILogSystem::fromServerDBInfo(self.myId, db->get(), true);
-			}
-			when(wait(done)) {
-				TraceEvent("BWRangePartitionedDone", self.myId).detail("BackupEpoch", self.backupEpoch);
-				// Notify master so that this worker can be removed from log system, then this
-				// worker (for an old epoch's unfinished work) can safely exit.
-				wait(brokenPromiseToNever(db->get().clusterInterface.notifyBackupWorkerDone.getReply(
-				    BackupWorkerDoneRequest(self.myId, self.backupEpoch))));
-				break;
-			}
-			when(wait(error)) {}
-		}
-	} catch (Error& e) {
-		state Error err = e;
-		if (e.code() == error_code_worker_removed) {
-			try {
-				wait(done);
-			} catch (Error& e) {
-				TraceEvent("BWRangePartitionedShutdownError", self.myId).errorUnsuppressed(e);
-			}
-		}
-		TraceEvent("BWRangePartitionedTerminated", self.myId).errorUnsuppressed(err);
-		if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
-			throw err;
-		}
-	}
-	return Void();
-}
-
-// Pulls data from TLog servers.
-ACTOR Future<Void> pullAsyncData(BackupRangePartitionedData* self) {
-	state Future<Void> logSystemChange = Void();
-	state Reference<ILogSystem::IPeekCursor> cursor;
-
-	state Version tagAt = std::max({ self->pulledVersion.get(), self->startVersion, self->savedVersion });
-
-	TraceEvent("BWRangePartitionedPull", self->myId)
-	    .detail("Tag", self->tag)
-	    .detail("Version", tagAt)
-	    .detail("StartVersion", self->startVersion)
-	    .detail("SavedVersion", self->savedVersion);
-
-	loop {
-		while (self->paused.get()) {
-			wait(self->paused.onChange());
-		}
-
-		loop choose {
-			when(wait(cursor ? cursor->getMore(TaskPriority::TLogCommit) : Never())) {
-				DisabledTraceEvent("BWRangePartitionedGotMore", self->myId)
-				    .detail("Tag", self->tag)
-				    .detail("CursorVersion", cursor->version().version);
-				break;
-			}
-			when(wait(logSystemChange)) {
-				if (self->logSystem.get()) {
-					// TODO akanksha: Use peekSingle as of now instead of peekLogRouter and later confirm if it works as
-					// expected.
-					cursor = self->logSystem.get()->peekSingle(self->myId, tagAt, self->tag);
-				} else {
-					cursor = Reference<ILogSystem::IPeekCursor>();
-				}
-				logSystemChange = self->logSystem.onChange();
-			}
-		}
-
-		if (cursor->popped() > 0) {
-			TraceEvent(SevError, "BWRangePartitionedDataPopped", self->myId)
-			    .detail("Popped", cursor->popped())
-			    .detail("Expected", tagAt);
-			throw worker_removed();
-		}
-
-		self->minKnownCommittedVersion =
-		    std::max(self->minKnownCommittedVersion, cursor->getMinKnownCommittedVersion());
-		state int64_t peekedBytes = 0;
-
-		// Hold messages until we know how many we can take, self->messages always
-		// contains messages that we have reserved memory for. Therefore, lock->release()
-		// will always encounter message with reserved memory.
-		state std::vector<RangePartitionedVersionedMessage> tmpMessages;
-
-		// Messages may be prefetched in peek here, but uncommitted messages should not be uploaded in uploadData().
-		while (cursor->hasMessage()) {
-			auto msg = RangePartitionedVersionedMessage(
-			    cursor->version(), cursor->getMessage(), cursor->getTags(), cursor->arena());
-			tmpMessages.emplace_back(std::move(msg));
-			peekedBytes += tmpMessages.back().getEstimatedSize();
-			cursor->nextMessage();
-		}
-
-		if (peekedBytes > 0) {
-			TraceEvent(SevDebugMemory, "BWRangePartitionedMemory", self->myId)
-			    .detail("Take", peekedBytes)
-			    .detail("Current", self->lock->activePermits());
-			wait(self->lock->take(TaskPriority::DefaultYield, peekedBytes));
-			self->messages.insert(self->messages.end(),
-			                      std::make_move_iterator(tmpMessages.begin()),
-			                      std::make_move_iterator(tmpMessages.end()));
-		}
-
-		tagAt = cursor->version().version;
-		self->pulledVersion.set(tagAt);
-		TraceEvent("BWRangePartitionedGot", self->myId).suppressFor(1.0).detail("LatestPulledVersion", tagAt);
-
-		// For older epochs, we may have an end version to stop at.
-		if (self->pullFinished()) {
-			self->eraseMessagesAfterEndVersion();
-			self->doneTrigger.trigger();
-			TraceEvent("BWRangePartitionedFinishPull", self->myId)
-			    .detail("Tag", self->tag.toString())
-			    .detail("VersionGot", tagAt)
-			    .detail("EndVersion", self->endVersion.get())
-			    .detail("LogEpoch", self->recruitedEpoch)
-			    .detail("BackupEpoch", self->backupEpoch);
-			return Void();
-		}
-		wait(yield());
-	}
-}
\ No newline at end of file
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index dccaf9e6026..3c910e68eb4 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -1,11 +1,20 @@
 fdb_find_sources(FDBSERVER_SRCS)
-
-if(${COROUTINE_IMPL} STREQUAL libcoro)
-  list(APPEND FDBSERVER_SRCS coroimpl/CoroFlowCoro.actor.cpp)
-else()
-  list(APPEND FDBSERVER_SRCS coroimpl/CoroFlow.actor.cpp)
-endif()
-
+list(FILTER FDBSERVER_SRCS EXCLUDE REGEX "^workloads/")
+include(AddFdbServerLinkTest)
+
+function(configure_fdbserver_common_includes target_name)
+  target_include_directories(${target_name} PRIVATE
+    ${CMAKE_SOURCE_DIR}/bindings/c
+    ${CMAKE_BINARY_DIR}/bindings/c)
+endfunction()
+
+function(configure_fdbserver_target_includes target_name)
+  configure_fdbserver_common_includes(${target_name})
+  target_include_directories(${target_name} PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+  )
+endfunction()
 
 if(WITH_ROCKSDB)
   find_package(LZ4 REQUIRED)
@@ -15,6 +24,27 @@ if(WITH_ROCKSDB)
   endif()
 endif()
 
+add_subdirectory(core)
+add_subdirectory(kvstore)
+add_subdirectory(logsystem)
+add_subdirectory(mocks3)
+add_subdirectory(clustercontroller)
+add_subdirectory(backupworker)
+add_subdirectory(commitproxy)
+add_subdirectory(coordinator)
+add_subdirectory(datadistributor)
+add_subdirectory(consistencyscan)
+add_subdirectory(grvproxy)
+add_subdirectory(logrouter)
+add_subdirectory(ratekeeper)
+add_subdirectory(resolver)
+add_subdirectory(sequencer)
+add_subdirectory(storageserver)
+add_subdirectory(tester)
+add_subdirectory(tlog)
+add_subdirectory(worker)
+add_subdirectory(workloads)
+
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/workloads)
 
 add_flow_target(EXECUTABLE NAME fdbserver SRCS ${FDBSERVER_SRCS})
@@ -26,7 +56,6 @@ if (WITH_SWIFT)
   add_definitions(-DWITH_SWIFT)
 
   add_library(fdbserver_swift STATIC
-          masterserver.swift
           swift_fdbserver_cxx_swift_value_conformance.swift
           swift_fdbserver_collections.swift
           swift_fdbserver_stream_support.swift
@@ -120,18 +149,40 @@ if (WITH_SWIFT)
   # Generate the module map for FDBServer.
   include(GenerateModulemap)
   set(FDBSERVER_BINARY_DIR "${CMAKE_BINARY_DIR}/fdbserver")
+  get_target_property(FDBSERVER_MODULE_HEADERS fdbserver HEADER_FILES)
+  if(NOT FDBSERVER_MODULE_HEADERS)
+    set(FDBSERVER_MODULE_HEADERS "")
+  endif()
   generate_modulemap("${CMAKE_BINARY_DIR}/fdbserver/include" "FDBServer" fdbserver
+     HEADERS ${FDBSERVER_MODULE_HEADERS}
      OMIT
      ArtMutationBuffer.h # actually a textual include
      art_impl.h # actually a textual include
   )
 endif()
 
-target_include_directories(fdbserver PRIVATE
-  ${CMAKE_SOURCE_DIR}/bindings/c
-  ${CMAKE_BINARY_DIR}/bindings/c
-  ${CMAKE_CURRENT_SOURCE_DIR}/include
-  ${CMAKE_CURRENT_BINARY_DIR}/include)
+configure_fdbserver_target_includes(fdbserver)
+target_link_libraries(fdbserver PRIVATE
+  fdbserver_mocks3
+  "$<LINK_LIBRARY:WHOLE_ARCHIVE,fdbserver_workloads>"
+  fdbserver_worker
+  fdbserver_backupworker
+  fdbserver_clustercontroller
+  fdbserver_commitproxy
+  fdbserver_consistencyscan
+  fdbserver_coordinator
+  fdbserver_datadistributor
+  fdbserver_grvproxy
+  fdbserver_kvstore
+  fdbserver_logrouter
+  fdbserver_logsystem
+  fdbserver_ratekeeper
+  fdbserver_resolver
+  fdbserver_sequencer
+  fdbserver_storageserver
+  fdbserver_tester
+  fdbserver_tlog
+  fdbserver_core)
 if (WITH_ROCKSDB)
   add_dependencies(fdbserver rocksdb)
   if(WITH_LIBURING)
@@ -151,7 +202,7 @@ if(USE_JEMALLOC)
   target_link_libraries(fdbserver PRIVATE jemalloc::jemalloc)
 endif()
 
-target_link_libraries(fdbserver PRIVATE toml11_target rapidjson)
+target_link_libraries(fdbserver PRIVATE toml11::toml11 rapidjson)
 if(WITH_ROCKSDB)
   target_compile_definitions(fdbserver PRIVATE WITH_ROCKSDB)
 endif()
diff --git a/fdbserver/GlobalTagThrottler.actor.cpp b/fdbserver/GlobalTagThrottler.actor.cpp
deleted file mode 100644
index 09500086e95..00000000000
--- a/fdbserver/GlobalTagThrottler.actor.cpp
+++ /dev/null
@@ -1,1146 +0,0 @@
-/*
- * GlobalTagThrottler.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbrpc/Smoother.h"
-#include "fdbserver/ServerThroughputTracker.h"
-#include "fdbserver/TagThrottler.h"
-
-#include <limits>
-
-#include "flow/actorcompiler.h" // must be last include
-
-// In the function names below, several terms are used repeatedly. The context-specific are defined here:
-//
-// Cost: Every read or write operation has an associated cost, determined by the number of bytes accessed.
-//       Global tag throttling quotas are specified in terms of the amount of this cost that can be consumed
-//       per second. In the global tag throttler, cost refers to the per second rate of cost consumption.
-//
-// TPS: Transactions per second. Quotas are not specified in terms of TPS, but the limits given to clients must
-//      be specified in terms of TPS because throttling is performed at the front end of transactions (before costs are
-//      known).
-//
-// Total: Refers to the total quota specified by clients through the global tag throttling API. The sum of the
-//        costs of all operations (cluster-wide) with a particular tag cannot exceed the tag's specified total quota,
-//        even if the cluster has no saturated processes.
-//
-// Desired TPS: Assuming that a tag is able to achieve its total quota, this is the TPS it would be able to perform.
-//
-// Reserved: Refers to the reserved quota specified by clients through the global tag throttling API. As long as the
-//           sum of the costs of all operations (cluster-wide) with a particular tag are not above the tag's
-//           specified reserved quota, the tag should not experience any throttling from the global tag throttler.
-//
-// Current [Cost|TPS]: Measuring the current throughput on the cluster, independent of any specified quotas.
-//
-// ThrottlingRatio: Based on the health of each storage server, a throttling ratio is provided,
-//                  informing the global tag throttler what ratio of the current throughput can be maintained.
-//
-// Limiting [Cost|TPS]: Based on the health of storage servers, a limiting throughput may be enforced.
-//
-// Target [Cost|TPS]: Based on reserved, limiting, and desired throughputs, this is the target throughput
-//                    that the global tag throttler aims to achieve (across all clients).
-//
-// PerClient TPS: Because the target throughput must be shared across multiple clients, and all clients must
-//           be given the same limits, a per-client limit is calculated based on the current and target throughputs.
-
-class GlobalTagThrottlerImpl {
-	template <class K, class V>
-	static Optional<V> tryGet(std::unordered_map<K, V> const& m, K const& k) {
-		auto it = m.find(k);
-		if (it == m.end()) {
-			return {};
-		} else {
-			return it->second;
-		}
-	}
-
-	static Optional<double> getMin(Optional<double> a, Optional<double> b) {
-		if (a.present() && b.present()) {
-			return std::min(a.get(), b.get());
-		} else if (a.present()) {
-			return a;
-		} else {
-			return b;
-		}
-	}
-
-	static Optional<double> getMax(Optional<double> a, Optional<double> b) {
-		if (a.present() && b.present()) {
-			return std::max(a.get(), b.get());
-		} else if (a.present()) {
-			return a;
-		} else {
-			return b;
-		}
-	}
-
-	enum class LimitType { RESERVED, TOTAL };
-	enum class OpType { READ, WRITE };
-
-	class ThroughputCounters {
-		Smoother readCost;
-		Smoother writeCost;
-
-	public:
-		ThroughputCounters()
-		  : readCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME),
-		    writeCost(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {}
-
-		void updateCost(double newCost, OpType opType) {
-			if (opType == OpType::READ) {
-				readCost.setTotal(newCost);
-			} else {
-				writeCost.setTotal(newCost);
-			}
-		}
-
-		double getCost() const { return readCost.smoothTotal() + writeCost.smoothTotal(); }
-	};
-
-	// Track various statistics per tag, aggregated across all storage servers
-	class PerTagStatistics {
-		Optional<ThrottleApi::TagQuotaValue> quota;
-		HoltLinearSmoother transactionCounter;
-		Smoother perClientRate;
-		Smoother targetRate;
-		double transactionsLastAdded;
-		double lastLogged;
-
-	public:
-		explicit PerTagStatistics()
-		  : transactionCounter(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME,
-		                       SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME),
-		    perClientRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME),
-		    targetRate(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME), transactionsLastAdded(now()),
-		    lastLogged(0) {}
-
-		Optional<ThrottleApi::TagQuotaValue> getQuota() const { return quota; }
-
-		void setQuota(ThrottleApi::TagQuotaValue quota) { this->quota = quota; }
-
-		void clearQuota() { quota = {}; }
-
-		void addTransactions(int count) {
-			transactionsLastAdded = now();
-			transactionCounter.addDelta(count);
-		}
-
-		double getTransactionRate() const { return transactionCounter.smoothRate(); }
-
-		ClientTagThrottleLimits updateAndGetPerClientLimit(double targetTps) {
-			auto newPerClientRate = std::max(
-			    SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MIN_RATE,
-			    std::min(targetTps, (targetTps / transactionCounter.smoothRate()) * perClientRate.smoothTotal()));
-			perClientRate.setTotal(newPerClientRate);
-			return ClientTagThrottleLimits(
-			    std::max(perClientRate.smoothTotal(), SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MIN_RATE),
-			    ClientTagThrottleLimits::NO_EXPIRATION);
-		}
-
-		double updateAndGetTargetLimit(double targetTps) {
-			targetRate.setTotal(targetTps);
-			return targetRate.smoothTotal();
-		}
-
-		bool recentTransactionsAdded() const {
-			return now() - transactionsLastAdded < SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER;
-		}
-
-		bool canLog() const { return now() - lastLogged > SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TRACE_INTERVAL; }
-
-		void updateLastLogged() { lastLogged = now(); }
-	};
-
-	struct StorageServerInfo {
-		Optional<Standalone<StringRef>> zoneId;
-		Optional<double> throttlingRatio;
-	};
-
-	Database db;
-	UID id;
-	int maxFallingBehind{ 0 };
-	double limitingThreshold{ 0.0 };
-	uint64_t throttledTagChangeId{ 0 };
-	uint32_t lastBusyTagCount{ 0 };
-
-	std::unordered_map<UID, StorageServerInfo> ssInfos;
-	std::unordered_map<TransactionTag, PerTagStatistics> tagStatistics;
-	ServerThroughputTracker throughputTracker;
-
-	// For transactions with the provided tag, returns the average cost of all transactions
-	// across the cluster. The minimum cost is one page. If the transaction rate is too low,
-	// return an empty Optional, because no accurate estimation can be made.
-	Optional<double> getAverageTransactionCost(TransactionTag tag, TraceEvent& te) const {
-		auto const cost = throughputTracker.getThroughput(tag);
-		auto const stats = tryGet(tagStatistics, tag);
-		if (!stats.present()) {
-			return CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-		}
-		auto const transactionRate = stats.get().getTransactionRate();
-		te.detail("TransactionRate", transactionRate);
-		te.detail("Cost", cost);
-		if (transactionRate < SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MIN_RATE) {
-			return {};
-		} else {
-			return std::max(static_cast<double>(CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE), cost / transactionRate);
-		}
-	}
-
-	Optional<double> getQuota(TransactionTag tag, LimitType limitType) const {
-		auto const stats = tryGet(tagStatistics, tag);
-		if (!stats.present()) {
-			return {};
-		}
-		auto const quota = stats.get().getQuota();
-		if (!quota.present()) {
-			return {};
-		}
-		if (limitType == LimitType::TOTAL) {
-			return quota.get().totalQuota;
-		} else {
-			return quota.get().reservedQuota;
-		}
-	}
-
-	Optional<double> getLimitingThrottlingRatio(TransactionTag tag) const {
-		std::unordered_map<Optional<Standalone<StringRef>>, double> zoneThrottlingRatios;
-		for (auto const& [id, ssInfo] : ssInfos) {
-			// Ignore storage servers where throttlingId is not significantly contributing
-			auto const throughput = throughputTracker.getThroughput(id, tag);
-			if (!throughput.present() || throughput.get() < limitingThreshold) {
-				continue;
-			}
-
-			if (ssInfo.throttlingRatio.present()) {
-				auto const [it, inserted] =
-				    zoneThrottlingRatios.try_emplace(ssInfo.zoneId, ssInfo.throttlingRatio.get());
-				if (!inserted) {
-					auto& zoneThrottlingRatio = it->second;
-					zoneThrottlingRatio = std::min<double>(zoneThrottlingRatio, ssInfo.throttlingRatio.get());
-				}
-			}
-		}
-		if (zoneThrottlingRatios.size() <= maxFallingBehind) {
-			return {};
-		} else {
-			std::priority_queue<double, std::vector<double>> maxHeap;
-			ASSERT_GE(maxFallingBehind, 0);
-			for (auto const& [_, throttlingRatio] : zoneThrottlingRatios) {
-				if (maxHeap.size() < maxFallingBehind + 1) {
-					maxHeap.push(throttlingRatio);
-				} else if (maxHeap.top() > throttlingRatio) {
-					maxHeap.pop();
-					maxHeap.push(throttlingRatio);
-				}
-			}
-			return maxHeap.top();
-		}
-	}
-
-	// Return the limiting transaction rate, aggregated across all storage servers.
-	// The limits from the worst maxFallingBehind zones are
-	// ignored, because we do not non-workload related issues (e.g. slow disks)
-	// to affect tag throttling. If more than maxFallingBehind zones are at
-	// or near saturation, this indicates that throttling should take place.
-	Optional<double> getLimitingTps(TransactionTag tag) const {
-		auto const stats = tryGet(tagStatistics, tag);
-		if (!stats.present()) {
-			return {};
-		}
-		auto const limitingThrottlingRatio = getLimitingThrottlingRatio(tag);
-		if (!limitingThrottlingRatio.present()) {
-			return {};
-		}
-		return stats.get().getTransactionRate() * limitingThrottlingRatio.get();
-	}
-
-	void removeUnseenQuotas(std::unordered_set<TransactionTag> const& tagsWithQuota) {
-		for (auto& [tag, stats] : tagStatistics) {
-			if (!tagsWithQuota.count(tag)) {
-				stats.clearQuota();
-			}
-		}
-	}
-
-	ACTOR static Future<Void> monitorThrottlingChanges(GlobalTagThrottlerImpl* self) {
-		state std::unordered_set<TransactionTag> tagsWithQuota;
-
-		loop {
-			state ReadYourWritesTransaction tr(self->db);
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-
-					tagsWithQuota.clear();
-					state RangeResult currentQuotas = wait(tr.getRange(tagQuotaKeys, CLIENT_KNOBS->TOO_MANY));
-					TraceEvent("GlobalTagThrottler_ReadCurrentQuotas", self->id).detail("Size", currentQuotas.size());
-					for (auto const kv : currentQuotas) {
-						auto const tag = kv.key.removePrefix(tagQuotaPrefix);
-						auto const quota = ThrottleApi::TagQuotaValue::fromValue(kv.value);
-						self->tagStatistics[tag].setQuota(quota);
-						tagsWithQuota.insert(tag);
-					}
-					self->removeUnseenQuotas(tagsWithQuota);
-					self->removeExpiredTags();
-					++self->throttledTagChangeId;
-					wait(delay(5.0));
-					break;
-				} catch (Error& e) {
-					TraceEvent("GlobalTagThrottler_MonitoringChangesError", self->id).error(e);
-					wait(tr.onError(e));
-				}
-			}
-		}
-	}
-
-	Optional<double> getTargetTps(TransactionTag tag, bool& isBusy, TraceEvent& te) {
-		auto const limitingTps = getLimitingTps(tag);
-		auto const averageTransactionCost = getAverageTransactionCost(tag, te);
-		auto const totalQuota = getQuota(tag, LimitType::TOTAL);
-		auto const reservedQuota = getQuota(tag, LimitType::RESERVED);
-		if (!averageTransactionCost.present() || !totalQuota.present() || !reservedQuota.present()) {
-			return {};
-		}
-		auto const desiredTps = totalQuota.get() / averageTransactionCost.get();
-		auto const reservedTps = reservedQuota.get() / averageTransactionCost.get();
-		auto const targetTps = getMax(reservedTps, getMin(desiredTps, limitingTps));
-
-		isBusy = limitingTps.present() && limitingTps.get() < desiredTps;
-
-		te.detail("Tag", tag)
-		    .detail("TargetTps", targetTps)
-		    .detail("AverageTransactionCost", averageTransactionCost)
-		    .detail("LimitingTps", limitingTps)
-		    .detail("ReservedTps", reservedTps)
-		    .detail("DesiredTps", desiredTps)
-		    .detail("NumStorageServers", throughputTracker.storageServersTracked())
-		    .detail("TotalQuota", totalQuota)
-		    .detail("ReservedQuota", reservedQuota);
-
-		return targetTps;
-	}
-
-public:
-	GlobalTagThrottlerImpl(Database db, UID id, int maxFallingBehind, double limitingThreshold)
-	  : db(db), id(id), maxFallingBehind(maxFallingBehind), limitingThreshold(limitingThreshold) {}
-	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
-	void addRequests(TransactionTag tag, int count) {
-		auto it = tagStatistics.find(tag);
-		if (it == tagStatistics.end()) {
-			if (tagStatistics.size() == SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED) {
-				CODE_PROBE(true,
-				           "Global tag throttler ignoring transactions because maximum number of trackable tags has "
-				           "been reached");
-				TraceEvent("GlobalTagThrottler_IgnoringRequests")
-				    .suppressFor(60.0)
-				    .detail("Tag", tag)
-				    .detail("Count", count);
-			} else {
-				tagStatistics[tag].addTransactions(static_cast<double>(count));
-			}
-		} else {
-			it->second.addTransactions(static_cast<double>(count));
-		}
-	}
-	uint64_t getThrottledTagChangeId() const { return throttledTagChangeId; }
-
-	TransactionTagMap<double> getProxyRates(int numProxies) {
-		TransactionTagMap<double> result;
-		lastBusyTagCount = 0;
-
-		for (auto& [tag, stats] : tagStatistics) {
-			// Currently there is no differentiation between batch priority and default priority transactions
-			TraceEvent te("GlobalTagThrottler_GotRate", id);
-			bool const traceEnabled = stats.canLog();
-			if (!traceEnabled) {
-				te.disable();
-			}
-			bool isBusy{ false };
-			auto const targetTps = getTargetTps(tag, isBusy, te);
-			if (isBusy) {
-				++lastBusyTagCount;
-			}
-			if (targetTps.present()) {
-				auto const smoothedTargetTps = stats.updateAndGetTargetLimit(targetTps.get());
-				te.detail("SmoothedTargetTps", smoothedTargetTps).detail("NumProxies", numProxies);
-				result[tag] = std::max(1.0, smoothedTargetTps / numProxies);
-				if (traceEnabled) {
-					stats.updateLastLogged();
-				}
-			} else {
-				te.disable();
-			}
-		}
-
-		return result;
-	}
-
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() {
-		PrioritizedTransactionTagMap<ClientTagThrottleLimits> result;
-		lastBusyTagCount = 0;
-
-		for (auto& [tag, stats] : tagStatistics) {
-			// Currently there is no differentiation between batch priority and default priority transactions
-			bool isBusy{ false };
-			TraceEvent te("GlobalTagThrottler_GotClientRate", id);
-			bool const traceEnabled = stats.canLog();
-			if (!traceEnabled) {
-				te.disable();
-			}
-			auto const targetTps = getTargetTps(tag, isBusy, te);
-
-			if (isBusy) {
-				++lastBusyTagCount;
-			}
-
-			if (targetTps.present()) {
-				auto const clientRate = stats.updateAndGetPerClientLimit(targetTps.get());
-				result[TransactionPriority::BATCH][tag] = result[TransactionPriority::DEFAULT][tag] = clientRate;
-				te.detail("ClientTps", clientRate.tpsRate);
-				if (traceEnabled) {
-					stats.updateLastLogged();
-				}
-			} else {
-				te.disable();
-			}
-		}
-		return result;
-	}
-
-	int64_t autoThrottleCount() const {
-		int64_t result{ 0 };
-		for (const auto& [tag, stats] : tagStatistics) {
-			if (stats.getQuota().present()) {
-				++result;
-			}
-		}
-		return result;
-	}
-	uint32_t busyReadTagCount() const { return lastBusyTagCount; }
-	uint32_t busyWriteTagCount() const { return lastBusyTagCount; }
-	int64_t manualThrottleCount() const { return 0; }
-
-	void updateThrottling(Map<UID, StorageQueueInfo> const& sqInfos) {
-		for (auto it = sqInfos.begin(); it != sqInfos.end(); ++it) {
-			auto& ss = it->value;
-			auto& ssInfo = ssInfos[ss.id];
-			ssInfo.throttlingRatio =
-			    ss.getTagThrottlingRatio(SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES,
-			                             SERVER_KNOBS->AUTO_TAG_THROTTLE_SPRING_BYTES_STORAGE_SERVER);
-			ssInfo.zoneId = ss.locality.zoneId();
-		}
-		throughputTracker.update(sqInfos);
-	}
-
-	void setQuota(TransactionTagRef tag, ThrottleApi::TagQuotaValue const& tagQuotaValue) {
-		tagStatistics[tag].setQuota(tagQuotaValue);
-	}
-
-	void removeQuota(TransactionTagRef tag) { tagStatistics[tag].clearQuota(); }
-
-	void removeExpiredTags() {
-		for (auto it = tagStatistics.begin(); it != tagStatistics.end();) {
-			const auto& [tag, stats] = *it;
-			if (!stats.recentTransactionsAdded()) {
-				throughputTracker.removeTag(tag);
-				it = tagStatistics.erase(it);
-			} else {
-				++it;
-			}
-		}
-	}
-
-	uint32_t tagsTracked() const { return tagStatistics.size(); }
-};
-
-GlobalTagThrottler::GlobalTagThrottler(Database db, UID id, int maxFallingBehind, double limitingThreshold)
-  : impl(PImpl<GlobalTagThrottlerImpl>::create(db, id, maxFallingBehind, limitingThreshold)) {}
-
-GlobalTagThrottler::~GlobalTagThrottler() = default;
-
-Future<Void> GlobalTagThrottler::monitorThrottlingChanges() {
-	return impl->monitorThrottlingChanges();
-}
-void GlobalTagThrottler::addRequests(TransactionTag tag, int count) {
-	return impl->addRequests(tag, count);
-}
-uint64_t GlobalTagThrottler::getThrottledTagChangeId() const {
-	return impl->getThrottledTagChangeId();
-}
-PrioritizedTransactionTagMap<ClientTagThrottleLimits> GlobalTagThrottler::getClientRates() {
-	return impl->getClientRates();
-}
-TransactionTagMap<double> GlobalTagThrottler::getProxyRates(int numProxies) {
-	return impl->getProxyRates(numProxies);
-}
-int64_t GlobalTagThrottler::autoThrottleCount() const {
-	return impl->autoThrottleCount();
-}
-uint32_t GlobalTagThrottler::busyReadTagCount() const {
-	return impl->busyReadTagCount();
-}
-uint32_t GlobalTagThrottler::busyWriteTagCount() const {
-	return impl->busyWriteTagCount();
-}
-int64_t GlobalTagThrottler::manualThrottleCount() const {
-	return impl->manualThrottleCount();
-}
-bool GlobalTagThrottler::isAutoThrottlingEnabled() const {
-	return true;
-}
-void GlobalTagThrottler::updateThrottling(Map<UID, StorageQueueInfo> const& sqInfos) {
-	return impl->updateThrottling(sqInfos);
-}
-
-void GlobalTagThrottler::setQuota(TransactionTagRef tag, ThrottleApi::TagQuotaValue const& tagQuotaValue) {
-	return impl->setQuota(tag, tagQuotaValue);
-}
-
-void GlobalTagThrottler::removeQuota(TransactionTagRef tag) {
-	return impl->removeQuota(tag);
-}
-
-uint32_t GlobalTagThrottler::tagsTracked() const {
-	return impl->tagsTracked();
-}
-
-void GlobalTagThrottler::removeExpiredTags() {
-	return impl->removeExpiredTags();
-}
-
-namespace {
-
-enum class LimitType { RESERVED, TOTAL };
-enum class OpType { READ, WRITE };
-
-Optional<double> getTPSLimit(GlobalTagThrottler& globalTagThrottler, TransactionTag tag) {
-	auto clientRates = globalTagThrottler.getClientRates();
-	auto it1 = clientRates.find(TransactionPriority::DEFAULT);
-	if (it1 != clientRates.end()) {
-		auto it2 = it1->second.find(tag);
-		if (it2 != it1->second.end()) {
-			return it2->second.tpsRate;
-		}
-	}
-	return {};
-}
-
-class MockStorageServer {
-	class Cost {
-		Smoother smoother;
-
-	public:
-		Cost() : smoother(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {}
-		Cost& operator+=(double delta) {
-			smoother.addDelta(delta);
-			return *this;
-		}
-		double smoothRate() const { return smoother.smoothRate(); }
-	};
-
-	UID id;
-	// pages/second that this storage server can handle
-	double capacity;
-	std::map<TransactionTag, Cost> readCosts, writeCosts;
-	Cost totalReadCost, totalWriteCost;
-
-public:
-	explicit MockStorageServer(UID id, double capacity) : id(id), capacity(capacity) { ASSERT_GT(capacity, 0); }
-	void addReadCost(TransactionTag tag, double cost) {
-		readCosts[tag] += cost;
-		totalReadCost += cost;
-	}
-	void addWriteCost(TransactionTag tag, double cost) {
-		writeCosts[tag] += cost;
-		totalWriteCost += cost;
-	}
-
-	void setCapacity(double value) { capacity = value; }
-
-	StorageQueueInfo getStorageQueueInfo() const {
-		StorageQueueInfo result(id, LocalityData({}, Value(id.toString()), {}, {}));
-		for (const auto& [tag, readCost] : readCosts) {
-			double fractionalBusyness{ 0.0 }; // unused for global tag throttling
-			result.busiestReadTags.emplace_back(tag, readCost.smoothRate(), fractionalBusyness);
-		}
-		for (const auto& [tag, writeCost] : writeCosts) {
-			double fractionalBusyness{ 0.0 }; // unused for global tag throttling
-			result.busiestWriteTags.emplace_back(tag, writeCost.smoothRate(), fractionalBusyness);
-		}
-		result.lastReply.bytesInput = ((totalReadCost.smoothRate() + totalWriteCost.smoothRate()) /
-		                               (capacity * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE)) *
-		                              SERVER_KNOBS->AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES;
-		result.valid = true;
-		return result;
-	}
-};
-
-class StorageServerCollection {
-	std::vector<MockStorageServer> storageServers;
-
-public:
-	StorageServerCollection(size_t size, double capacity) {
-		ASSERT_GT(size, 0);
-		storageServers.reserve(size);
-		for (int i = 0; i < size; ++i) {
-			storageServers.emplace_back(UID(i, i), capacity);
-		}
-	}
-
-	void addCost(TransactionTag tag,
-	             double pagesPerSecond,
-	             std::vector<int> const& storageServerIndices,
-	             OpType opType) {
-		if (storageServerIndices.empty()) {
-			auto costPerSS = CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE * (pagesPerSecond / storageServers.size());
-			if (opType == OpType::WRITE) {
-				costPerSS *= CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO;
-			}
-			for (auto& storageServer : storageServers) {
-				if (opType == OpType::READ) {
-					storageServer.addReadCost(tag, costPerSS);
-				} else {
-					storageServer.addWriteCost(tag, costPerSS);
-				}
-			}
-		} else {
-			auto const costPerSS =
-			    CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE * (pagesPerSecond / storageServerIndices.size());
-			for (auto i : storageServerIndices) {
-				if (opType == OpType::READ) {
-					storageServers[i].addReadCost(tag, costPerSS);
-				} else {
-					storageServers[i].addWriteCost(tag, costPerSS);
-				}
-			}
-		}
-	}
-
-	void setCapacity(int index, double value) { storageServers[index].setCapacity(value); }
-
-	Map<UID, StorageQueueInfo> getStorageQueueInfos() const {
-		Map<UID, StorageQueueInfo> result;
-		for (const auto& storageServer : storageServers) {
-			auto const sqInfo = storageServer.getStorageQueueInfo();
-			result.insert(mapPair(sqInfo.id, sqInfo));
-		}
-		return result;
-	}
-};
-
-ACTOR Future<Void> runClient(GlobalTagThrottler* globalTagThrottler,
-                             StorageServerCollection* storageServers,
-                             TransactionTag tag,
-                             double tpsRate,
-                             double costPerTransaction,
-                             OpType opType,
-                             std::vector<int> storageServerIndices = std::vector<int>()) {
-	loop {
-		auto tpsLimit = getTPSLimit(*globalTagThrottler, tag);
-		state double enforcedRate = tpsLimit.present() ? std::min<double>(tpsRate, tpsLimit.get()) : tpsRate;
-		wait(delay(1 / enforcedRate));
-		storageServers->addCost(tag, costPerTransaction, storageServerIndices, opType);
-		globalTagThrottler->addRequests(tag, 1);
-	}
-}
-
-ACTOR template <class Check>
-Future<Void> monitorActor(GlobalTagThrottler* globalTagThrottler, Check check) {
-	state int successes = 0;
-	loop {
-		wait(delay(1.0));
-		if (check(*globalTagThrottler)) {
-			// Wait for 10 consecutive successes so we're certain
-			// than a stable equilibrium has been reached
-			if (++successes == 10) {
-				return Void();
-			}
-		} else {
-			successes = 0;
-		}
-	}
-}
-
-bool isNear(double a, double b) {
-	return abs(a - b) < 0.1 * a;
-}
-
-bool isNear(Optional<double> a, Optional<double> b) {
-	if (a.present()) {
-		return b.present() && isNear(a.get(), b.get());
-	} else {
-		return !b.present();
-	}
-}
-
-bool targetRateIsNear(GlobalTagThrottler& globalTagThrottler, TransactionTag tag, Optional<double> expected) {
-	Optional<double> rate;
-	auto const targetRates = globalTagThrottler.getProxyRates(1);
-	auto it = targetRates.find(tag);
-	if (it != targetRates.end()) {
-		rate = it->second;
-	}
-	TraceEvent("GlobalTagThrottling_RateMonitor")
-	    .detail("Tag", tag)
-	    .detail("CurrentTPSRate", rate)
-	    .detail("ExpectedTPSRate", expected);
-	return isNear(rate, expected);
-}
-
-bool totalTargetRateIsNear(GlobalTagThrottler& globalTagThrottler, double expected) {
-	auto const targetRates = globalTagThrottler.getProxyRates(1);
-	double targetRateSum = 0.0;
-	for (auto const& [_, targetRate] : targetRates) {
-		targetRateSum += targetRate;
-	}
-	TraceEvent("GlobalTagThrottling_TotalRateMonitor")
-	    .detail("NumTags", targetRates.size())
-	    .detail("CurrentTotalTPSRate", targetRateSum)
-	    .detail("ExpectedTotalTPSRate", expected);
-	return isNear(targetRateSum, expected);
-}
-
-bool clientRateIsNear(GlobalTagThrottler& globalTagThrottler, TransactionTag tag, Optional<double> expected) {
-	Optional<double> rate;
-	auto clientRates = globalTagThrottler.getClientRates();
-	auto it1 = clientRates.find(TransactionPriority::DEFAULT);
-	if (it1 != clientRates.end()) {
-		auto it2 = it1->second.find(tag);
-		if (it2 != it1->second.end()) {
-			rate = it2->second.tpsRate;
-		}
-	}
-	TraceEvent("GlobalTagThrottling_ClientRateMonitor")
-	    .detail("Tag", tag)
-	    .detail("CurrentTPSRate", rate)
-	    .detail("ExpectedTPSRate", expected);
-	return isNear(rate, expected);
-}
-
-ACTOR Future<Void> updateGlobalTagThrottler(GlobalTagThrottler* globalTagThrottler,
-                                            StorageServerCollection const* storageServers) {
-	loop {
-		wait(delay(1.0));
-		auto const storageQueueInfos = storageServers->getStorageQueueInfos();
-		globalTagThrottler->updateThrottling(storageServers->getStorageQueueInfos());
-	}
-}
-
-GlobalTagThrottler getTestGlobalTagThrottler(int maxFallingBehind = 0) {
-	return GlobalTagThrottler(
-	    Database{}, UID{}, maxFallingBehind, /*limitingThreshold=*/CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
-}
-
-} // namespace
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second.
-// Client attempts 5 6-byte read transactions per second.
-// Limit should adjust to allow 100/6 transactions per second.
-TEST_CASE("/GlobalTagThrottler/Simple") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::READ);
-	state Future<Void> monitor =
-	    monitorActor(&globalTagThrottler, [testTag](auto& gtt) { return targetRateIsNear(gtt, testTag, 100.0 / 6.0); });
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second.
-// Client attempts 5 6-byte write transactions per second.
-// Limit should adjust to allow 100/(6*<fungibility_ratio>) transactions per second.
-TEST_CASE("/GlobalTagThrottler/WriteThrottling") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::WRITE);
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag](auto& gtt) {
-		return targetRateIsNear(gtt, testTag, 100.0 / (6.0 * CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO));
-	});
-
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second for each tag.
-// 2 clients each attempt 5 6-byte read transactions per second.
-// Both limits should adjust to allow 100/6 transactions per second.
-TEST_CASE("/GlobalTagThrottler/MultiTagThrottling") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag1 = "sampleTag1"_sr;
-	TransactionTag testTag2 = "sampleTag2"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag1, tagQuotaValue);
-	globalTagThrottler.setQuota(testTag2, tagQuotaValue);
-	state std::vector<Future<Void>> futures;
-	state std::vector<Future<Void>> monitorFutures;
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag1, 5.0, 6.0, OpType::READ));
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag2, 5.0, 6.0, OpType::READ));
-	futures.push_back(updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
-		return targetRateIsNear(gtt, testTag1, 100.0 / 6.0) && targetRateIsNear(gtt, testTag2, 100.0 / 6.0);
-	});
-	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second.
-// Client attempts 20 10-byte read transactions per second.
-// Limit should adjust to allow 100/10 transactions per second.
-TEST_CASE("/GlobalTagThrottler/AttemptWorkloadAboveQuota") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 20.0, 10.0, OpType::READ);
-	state Future<Void> monitor =
-	    monitorActor(&globalTagThrottler, [testTag](auto& gtt) { return targetRateIsNear(gtt, testTag, 10.0); });
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second.
-// 2 clients each attempt 5 6-byte transactions per second.
-// Limit should adjust to allow 100/6 transactions per second.
-TEST_CASE("/GlobalTagThrottler/MultiClientThrottling") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::READ);
-	state Future<Void> client2 = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag](auto& gtt) {
-		return targetRateIsNear(gtt, testTag, 100.0 / 6.0) && clientRateIsNear(gtt, testTag, 100.0 / 6.0);
-	});
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || client2 || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second.
-// 2 clients each attempt 20 10-page transactions per second.
-// Target rate should adjust to allow 100/10 transactions per second.
-// Each client is throttled to only perform (100/10)/2 transactions per second.
-TEST_CASE("/GlobalTagThrottler/MultiClientThrottling2") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 20.0, 10.0, OpType::READ);
-	state Future<Void> client2 = runClient(&globalTagThrottler, &storageServers, testTag, 20.0, 10.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag](auto& gtt) {
-		return targetRateIsNear(gtt, testTag, 10.0) && clientRateIsNear(gtt, testTag, 5.0);
-	});
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota set to 100 pages/second.
-// One client attempts 5 5-page read transactions per second.
-// Another client attempts 25 5-page read transactions per second.
-// Target rate should adjust to allow 100/5 transactions per second.
-// This 20 transactions/second limit is split with a distribution of (5, 15) between the 2 clients.
-TEST_CASE("/GlobalTagThrottler/SkewedMultiClientThrottling") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 5.0, OpType::READ);
-	state Future<Void> client2 = runClient(&globalTagThrottler, &storageServers, testTag, 25.0, 5.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag](auto& gtt) {
-		return targetRateIsNear(gtt, testTag, 20.0) && clientRateIsNear(gtt, testTag, 15.0);
-	});
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota is initially set to 100 pages/second.
-// Client attempts 5 6-page transactions per second.
-// Test that the tag throttler can reach equilibrium, then adjust to a new equilibrium once the quota is changed
-// Target rate should adjust to allow 100/6 transactions per second.
-// Total quota is modified to 50 pages/second.
-// Target rate should adjust to allow 50/6 transactions per second.
-TEST_CASE("/GlobalTagThrottler/UpdateQuota") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	state ThrottleApi::TagQuotaValue tagQuotaValue;
-	state TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(
-	    &globalTagThrottler, [](auto& gtt) { return targetRateIsNear(gtt, "sampleTag1"_sr, 100.0 / 6.0); });
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	tagQuotaValue.totalQuota = 50 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	monitor =
-	    monitorActor(&globalTagThrottler, [](auto& gtt) { return targetRateIsNear(gtt, "sampleTag1"_sr, 50.0 / 6.0); });
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 100 pages/second each.
-// Total quota is initially set to 100 pages/second.
-// Client attempts 5 6-page read transactions per second.
-// Target limit adjusts to allow 100/6 transactions per second.
-// Then Quota is removed.
-// Target limit is removed as a result.
-TEST_CASE("/GlobalTagThrottler/RemoveQuota") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 100);
-	state ThrottleApi::TagQuotaValue tagQuotaValue;
-	state TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(
-	    &globalTagThrottler, [](auto& gtt) { return targetRateIsNear(gtt, "sampleTag1"_sr, 100.0 / 6.0); });
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	globalTagThrottler.removeQuota(testTag);
-	monitor = monitorActor(&globalTagThrottler, [](auto& gtt) { return targetRateIsNear(gtt, "sampleTag1"_sr, {}); });
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 5 pages/second each.
-// Total quota is set to 100 pages/second.
-// Client attempts 10 6-page transactions per second
-// Target is adjusted to 50/6 transactions per second, to match the total capacity all storage servers.
-TEST_CASE("/GlobalTagThrottler/ActiveThrottling") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 5);
-	state ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 10.0, 6.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag](auto& gtt) {
-		return targetRateIsNear(gtt, testTag, 50 / 6.0) && gtt.busyReadTagCount() == 1;
-	});
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// 10 storage servers can handle 5 pages/second each.
-// Total quota is set to 50 pages/second for one tag, 100 pages/second for another.
-// For each tag, a client attempts to execute 10 6-page read transactions per second.
-// Target rates are adjusted to utilize the full 50 pages/second capacity of the
-// storage servers.
-TEST_CASE("/GlobalTagThrottler/MultiTagActiveThrottling1") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 5);
-	state ThrottleApi::TagQuotaValue tagQuotaValue1;
-	state ThrottleApi::TagQuotaValue tagQuotaValue2;
-	TransactionTag testTag1 = "sampleTag1"_sr;
-	TransactionTag testTag2 = "sampleTag2"_sr;
-	tagQuotaValue1.totalQuota = 50 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	tagQuotaValue2.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag1, tagQuotaValue1);
-	globalTagThrottler.setQuota(testTag2, tagQuotaValue2);
-	std::vector<Future<Void>> futures;
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag1, 10.0, 6.0, OpType::READ));
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag2, 10.0, 6.0, OpType::READ));
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
-		return totalTargetRateIsNear(gtt, 50 / 6.0) && gtt.busyReadTagCount() == 2;
-	});
-	futures.push_back(updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
-	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
-	return Void();
-}
-
-// 3 storage servers can handle 50 pages/second each.
-// Total quota is set to 100 pages/second for each tag.
-// Each client attempts 10 6-page read transactions per second.
-// This workload is sent to 2 storage servers per client (with an overlap of one storage server).
-// The total target rate summed across both tags is adjusted to match the throughput that the
-// busiest server can handle (50/3 transactions per second).
-TEST_CASE("/GlobalTagThrottler/MultiTagActiveThrottling2") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(3, 50);
-	state ThrottleApi::TagQuotaValue tagQuotaValue1;
-	state ThrottleApi::TagQuotaValue tagQuotaValue2;
-	TransactionTag testTag1 = "sampleTag1"_sr;
-	TransactionTag testTag2 = "sampleTag2"_sr;
-	tagQuotaValue1.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	tagQuotaValue2.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag1, tagQuotaValue1);
-	globalTagThrottler.setQuota(testTag2, tagQuotaValue2);
-	std::vector<Future<Void>> futures;
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag1, 10.0, 6.0, OpType::READ, { 0, 1 }));
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag2, 10.0, 6.0, OpType::READ, { 1, 2 }));
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
-		return totalTargetRateIsNear(gtt, 50 / 3.0) && gtt.busyReadTagCount() == 2;
-	});
-	futures.push_back(updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
-	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
-	return Void();
-}
-
-// 3 storage servers can handle 50 pages/second each.
-// Total quota is set to 100 pages/second for each tag.
-// One client attempts 10 6-page read transactions per second, all directed towards a single storage server.
-// Another client, using a different tag, attempts 10 6-page read transactions split across the other two storage
-// servers. Target rates adjust to 50/6 and 100/6 transactions per second for the two clients, based on the capacities
-// of the storage servers being accessed.
-TEST_CASE("/GlobalTagThrottler/MultiTagActiveThrottling3") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(3, 50);
-	state ThrottleApi::TagQuotaValue tagQuotaValue1;
-	state ThrottleApi::TagQuotaValue tagQuotaValue2;
-	TransactionTag testTag1 = "sampleTag1"_sr;
-	TransactionTag testTag2 = "sampleTag2"_sr;
-	tagQuotaValue1.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	tagQuotaValue2.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag1, tagQuotaValue1);
-	globalTagThrottler.setQuota(testTag2, tagQuotaValue2);
-	std::vector<Future<Void>> futures;
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag1, 10.0, 6.0, OpType::READ, { 0 }));
-	futures.push_back(runClient(&globalTagThrottler, &storageServers, testTag2, 10.0, 6.0, OpType::READ, { 1, 2 }));
-	state Future<Void> monitor = monitorActor(&globalTagThrottler, [testTag1, testTag2](auto& gtt) {
-		return targetRateIsNear(gtt, testTag1, 50 / 6.0) && targetRateIsNear(gtt, testTag2, 100 / 6.0) &&
-		       gtt.busyReadTagCount() == 1;
-	});
-	futures.push_back(updateGlobalTagThrottler(&globalTagThrottler, &storageServers));
-	wait(timeoutError(waitForAny(futures) || monitor, 600.0));
-	return Void();
-}
-
-// 10 storage servers can serve 5 pages/second each.
-// Total quota is set to 100 pages/second.
-// Reserved quota is set to 70 pages/second.
-// A client attempts to execute 10 6-page read transactions per second.
-// Despite the storage server only having capacity to serve 50/6 transactions per second,
-//   the reserved quota will ensure the target rate adjusts to 70/6 transactions per second.
-TEST_CASE("/GlobalTagThrottler/ReservedQuota") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 5);
-	state ThrottleApi::TagQuotaValue tagQuotaValue;
-	TransactionTag testTag = "sampleTag1"_sr;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	tagQuotaValue.reservedQuota = 70 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 10.0, 6.0, OpType::READ);
-	state Future<Void> monitor =
-	    monitorActor(&globalTagThrottler, [testTag](auto& gtt) { return targetRateIsNear(gtt, testTag, 70 / 6.0); });
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
-
-// Test that tags are expired iff a sufficient amount of time has passed since the
-// last transaction with that tag
-TEST_CASE("/GlobalTagThrottler/ExpireTags") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 5);
-	TransactionTag testTag = "sampleTag1"_sr;
-
-	state Future<Void> client =
-	    timeout(runClient(&globalTagThrottler, &storageServers, testTag, 10.0, 6.0, OpType::READ), 60.0, Void());
-	state Future<Void> updater = timeout(updateGlobalTagThrottler(&globalTagThrottler, &storageServers), 60.0, Void());
-	wait(client && updater);
-	client.cancel();
-	updater.cancel();
-	ASSERT_EQ(globalTagThrottler.tagsTracked(), 1);
-	globalTagThrottler.removeExpiredTags();
-	ASSERT_EQ(globalTagThrottler.tagsTracked(), 1);
-	wait(delay(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER + 1.0));
-	ASSERT_EQ(globalTagThrottler.tagsTracked(), 1);
-	globalTagThrottler.removeExpiredTags();
-	ASSERT_EQ(globalTagThrottler.tagsTracked(), 0);
-	return Void();
-}
-
-// Test that the number of tags tracked does not grow beyond SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED
-TEST_CASE("/GlobalTagThrottler/TagLimit") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler();
-	state StorageServerCollection storageServers(10, 5);
-	std::vector<Future<Void>> futures;
-	for (int i = 0; i < 2 * SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED; ++i) {
-		Arena arena;
-		TransactionTag tag = makeString(8, arena);
-		deterministicRandom()->randomBytes(mutateString(tag), tag.size());
-		futures.push_back(runClient(&globalTagThrottler, &storageServers, tag, 1.0, 6.0, OpType::READ));
-	}
-	wait(timeout(waitForAll(futures), 60.0, Void()));
-	ASSERT_EQ(globalTagThrottler.tagsTracked(), SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED);
-	return Void();
-}
-
-// 9 storage servers can handle 100 pages/second each.
-// 1 unhealthy storage server can only handle 1 page/second.
-// Total quota is set to 100 pages/second.
-// Client attempts 5 6-page transactions per second.
-// Target rate adjusts to 100/6 transactions per second, ignoring the worst storage server.
-// Then, a second storage server becomes unhealthy and can only handle 1 page/second.
-// Target rate adjusts down to 10/6 transactions per second, because only one bad zone can be ignored.
-TEST_CASE("/GlobalTagThrottler/IgnoreWorstZone") {
-	state GlobalTagThrottler globalTagThrottler = getTestGlobalTagThrottler(/*maxFallingBehind=*/1);
-	state StorageServerCollection storageServers(10, 100);
-	state TransactionTag testTag = "sampleTag1"_sr;
-	storageServers.setCapacity(0, 1);
-	ThrottleApi::TagQuotaValue tagQuotaValue;
-	tagQuotaValue.totalQuota = 100 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-	globalTagThrottler.setQuota(testTag, tagQuotaValue);
-	state Future<Void> client = runClient(&globalTagThrottler, &storageServers, testTag, 5.0, 6.0, OpType::READ);
-	state Future<Void> monitor = monitorActor(
-	    &globalTagThrottler, [](auto& gtt) { return targetRateIsNear(gtt, "sampleTag1"_sr, 100.0 / 6.0); });
-	state Future<Void> updater = updateGlobalTagThrottler(&globalTagThrottler, &storageServers);
-	wait(timeoutError(monitor || client || updater, 600.0));
-	storageServers.setCapacity(1, 1);
-	monitor =
-	    monitorActor(&globalTagThrottler, [](auto& gtt) { return targetRateIsNear(gtt, "sampleTag1"_sr, 10.0 / 6.0); });
-	wait(timeoutError(monitor || client || updater, 600.0));
-	return Void();
-}
diff --git a/fdbserver/GrvProxyTagThrottler.actor.cpp b/fdbserver/GrvProxyTagThrottler.actor.cpp
deleted file mode 100644
index c40086c5540..00000000000
--- a/fdbserver/GrvProxyTagThrottler.actor.cpp
+++ /dev/null
@@ -1,512 +0,0 @@
-/*
- * GrvProxyTagThrottler.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/Knobs.h"
-#include "fdbserver/GrvProxyTagThrottler.h"
-#include "fdbserver/Knobs.h"
-#include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // must be last include
-
-uint64_t GrvProxyTagThrottler::DelayedRequest::lastSequenceNumber = 0;
-
-void GrvProxyTagThrottler::DelayedRequest::updateProxyTagThrottledDuration(LatencyBandsMap& latencyBandsMap) {
-	req.proxyTagThrottledDuration = now() - startTime;
-	auto const& [tag, count] = *req.tags.begin();
-	latencyBandsMap.addMeasurement(tag, req.proxyTagThrottledDuration, count);
-}
-
-bool GrvProxyTagThrottler::DelayedRequest::isMaxThrottled(double maxThrottleDuration) const {
-	return now() - startTime > maxThrottleDuration;
-}
-
-void GrvProxyTagThrottler::TagQueue::setRate(double rate) {
-	if (rateInfo.present()) {
-		rateInfo.get().setRate(rate);
-	} else {
-		rateInfo = GrvTransactionRateInfo(
-		    SERVER_KNOBS->TAG_THROTTLE_RATE_WINDOW, SERVER_KNOBS->TAG_THROTTLE_MAX_EMPTY_QUEUE_BUDGET, rate);
-	}
-}
-
-bool GrvProxyTagThrottler::TagQueue::isMaxThrottled(double maxThrottleDuration) const {
-	return !requests.empty() && requests.front().isMaxThrottled(maxThrottleDuration);
-}
-
-void GrvProxyTagThrottler::TagQueue::rejectRequests(LatencyBandsMap& latencyBandsMap) {
-	CODE_PROBE(true, "GrvProxyTagThrottler rejecting requests");
-	while (!requests.empty()) {
-		auto& delayedReq = requests.front();
-		delayedReq.updateProxyTagThrottledDuration(latencyBandsMap);
-		delayedReq.req.reply.sendError(proxy_tag_throttled());
-		requests.pop_front();
-	}
-}
-
-void GrvProxyTagThrottler::TagQueue::endReleaseWindow(int64_t numStarted, double elapsed) {
-	if (rateInfo.present()) {
-		CODE_PROBE(requests.empty(), "Tag queue ending release window with empty request queue");
-		CODE_PROBE(!requests.empty(), "Tag queue ending release window with requests still queued");
-		rateInfo.get().endReleaseWindow(numStarted, requests.empty(), elapsed);
-	}
-}
-
-GrvProxyTagThrottler::GrvProxyTagThrottler(double maxThrottleDuration)
-  : maxThrottleDuration(maxThrottleDuration),
-    latencyBandsMap("GrvProxyTagThrottler",
-                    deterministicRandom()->randomUniqueID(),
-                    SERVER_KNOBS->GLOBAL_TAG_THROTTLING_PROXY_LOGGING_INTERVAL,
-                    SERVER_KNOBS->GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED) {}
-
-void GrvProxyTagThrottler::updateRates(TransactionTagMap<double> const& newRates) {
-	for (const auto& [tag, rate] : newRates) {
-		auto it = queues.find(tag);
-		if (it == queues.end()) {
-			queues[tag] = TagQueue(rate);
-		} else {
-			it->second.setRate(rate);
-		}
-	}
-
-	// Clean up tags that did not appear in newRates
-	for (auto& [tag, queue] : queues) {
-		if (newRates.find(tag) == newRates.end()) {
-			queue.rateInfo.reset();
-		}
-	}
-
-	// TODO: Use std::erase_if in C++20
-	for (auto it = queues.begin(); it != queues.end();) {
-		const auto& [tag, queue] = *it;
-		if (queue.requests.empty() && !queue.rateInfo.present()) {
-			it = queues.erase(it);
-		} else {
-			++it;
-		}
-	}
-}
-
-void GrvProxyTagThrottler::addRequest(GetReadVersionRequest const& req) {
-	ASSERT(req.isTagged());
-	auto const& tag = req.tags.begin()->first;
-	if (req.tags.size() > 1) {
-		// The GrvProxyTagThrottler assumes that each GetReadVersionRequest
-		// has at most one tag. If a transaction uses multiple tags and
-		// SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES is enabled, there may be
-		// unexpected behaviour, because only one tag is used for throttling.
-		TraceEvent(SevWarnAlways, "GrvProxyTagThrottler_MultipleTags")
-		    .suppressFor(60.0)
-		    .detail("NumTags", req.tags.size())
-		    .detail("UsingTag", tag);
-	}
-	queues[tag].requests.emplace_back(req);
-}
-
-GrvProxyTagThrottler::ReleaseTransactionsResult GrvProxyTagThrottler::releaseTransactions(
-    double elapsed,
-    Deque<GetReadVersionRequest>& outBatchPriority,
-    Deque<GetReadVersionRequest>& outDefaultPriority) {
-	ReleaseTransactionsResult result;
-
-	// Pointer to a TagQueue with some extra metadata stored alongside
-	struct TagQueueHandle {
-		// Store pointers here to avoid frequent std::unordered_map lookups
-		TagQueue* queue;
-		// Cannot be stored directly because we need to
-		uint32_t* numReleased;
-		// Sequence number of the first queued request
-		int64_t nextSeqNo;
-		bool operator>(TagQueueHandle const& rhs) const { return nextSeqNo > rhs.nextSeqNo; }
-		explicit TagQueueHandle(TagQueue& queue, uint32_t& numReleased) : queue(&queue), numReleased(&numReleased) {
-			ASSERT(!this->queue->requests.empty());
-			nextSeqNo = this->queue->requests.front().sequenceNumber;
-		}
-	};
-
-	// Priority queue of queues for each tag, ordered by the sequence number of the
-	// next request to process in each queue
-	std::priority_queue<TagQueueHandle, std::vector<TagQueueHandle>, std::greater<TagQueueHandle>> pqOfQueues;
-
-	// Track transactions released for each tag
-	std::vector<std::pair<TransactionTag, uint32_t>> transactionsReleased;
-	transactionsReleased.reserve(queues.size());
-	auto const transactionsReleasedInitialCapacity = transactionsReleased.capacity();
-
-	for (auto& [tag, queue] : queues) {
-		if (queue.rateInfo.present()) {
-			queue.rateInfo.get().startReleaseWindow();
-		}
-		if (!queue.requests.empty()) {
-			// First place the count in the transactionsReleased object,
-			// then pass a reference to the count to the TagQueueHandle object
-			// emplaced into pqOfQueues.
-			//
-			// Because we've reserved enough space in transactionsReleased
-			// to avoid resizing, this reference should remain valid.
-			// This allows each TagQueueHandle to update its number of
-			// numReleased counter without incurring the cost of a std::unordered_map lookup.
-			auto& [_, count] = transactionsReleased.emplace_back(tag, 0);
-			pqOfQueues.emplace(queue, count);
-		}
-	}
-
-	while (!pqOfQueues.empty()) {
-		auto tagQueueHandle = pqOfQueues.top();
-		pqOfQueues.pop();
-		// Used to determine when it is time to start processing another tag
-		auto const nextQueueSeqNo =
-		    pqOfQueues.empty() ? std::numeric_limits<int64_t>::max() : pqOfQueues.top().nextSeqNo;
-
-		while (!tagQueueHandle.queue->requests.empty()) {
-			auto& delayedReq = tagQueueHandle.queue->requests.front();
-			auto count = delayedReq.req.tags.begin()->second;
-			ASSERT_EQ(tagQueueHandle.nextSeqNo, delayedReq.sequenceNumber);
-			if (tagQueueHandle.queue->rateInfo.present() &&
-			    !tagQueueHandle.queue->rateInfo.get().canStart(*(tagQueueHandle.numReleased), count)) {
-				// Cannot release any more transaction from this tag (don't push the tag queue handle back into
-				// pqOfQueues)
-				CODE_PROBE(true, "GrvProxyTagThrottler throttling transaction");
-				if (tagQueueHandle.queue->isMaxThrottled(maxThrottleDuration)) {
-					// Requests in this queue have been throttled too long and errors
-					// should be sent to clients.
-					result.rejectedRequests += tagQueueHandle.queue->requests.size();
-					tagQueueHandle.queue->rejectRequests(latencyBandsMap);
-				}
-				break;
-			} else {
-				if (tagQueueHandle.nextSeqNo < nextQueueSeqNo) {
-					// Releasing transaction
-					*(tagQueueHandle.numReleased) += count;
-					delayedReq.updateProxyTagThrottledDuration(latencyBandsMap);
-					if (delayedReq.req.priority == TransactionPriority::BATCH) {
-						result.batchPriorityTransactionsReleased += delayedReq.req.transactionCount;
-						++result.batchPriorityRequestsReleased;
-						outBatchPriority.push_back(delayedReq.req);
-					} else if (delayedReq.req.priority == TransactionPriority::DEFAULT) {
-						result.defaultPriorityTransactionsReleased += delayedReq.req.transactionCount;
-						++result.defaultPriorityRequestsReleased;
-						outDefaultPriority.push_back(delayedReq.req);
-					} else {
-						// Immediate priority transactions should bypass the GrvProxyTagThrottler
-						ASSERT(false);
-					}
-					tagQueueHandle.queue->requests.pop_front();
-					if (!tagQueueHandle.queue->requests.empty()) {
-						tagQueueHandle.nextSeqNo = tagQueueHandle.queue->requests.front().sequenceNumber;
-					}
-				} else {
-					CODE_PROBE(true, "GrvProxyTagThrottler switching tags to preserve FIFO");
-					pqOfQueues.push(tagQueueHandle);
-					break;
-				}
-			}
-		}
-	}
-
-	// End release windows for all tag queues
-	{
-		TransactionTagMap<uint32_t> transactionsReleasedMap;
-		for (const auto& [tag, count] : transactionsReleased) {
-			transactionsReleasedMap[tag] = count;
-		}
-		for (auto& [tag, queue] : queues) {
-			queue.endReleaseWindow(transactionsReleasedMap[tag], elapsed);
-		}
-	}
-	// If the capacity is increased, that means the vector has been illegally resized, potentially
-	// corrupting memory
-	ASSERT_EQ(transactionsReleased.capacity(), transactionsReleasedInitialCapacity);
-
-	return result;
-}
-
-void GrvProxyTagThrottler::addLatencyBandThreshold(double value) {
-	CODE_PROBE(size() > 0, "GrvProxyTagThrottler adding latency bands while actively throttling");
-	latencyBandsMap.addThreshold(value);
-}
-
-uint32_t GrvProxyTagThrottler::size() const {
-	return queues.size();
-}
-
-ACTOR static Future<Void> mockClient(GrvProxyTagThrottler* throttler,
-                                     TransactionPriority priority,
-                                     TagSet tagSet,
-                                     int batchSize,
-                                     double desiredRate,
-                                     TransactionTagMap<uint32_t>* counters) {
-	state Future<Void> timer;
-	state TransactionTagMap<uint32_t> tags;
-	for (const auto& tag : tagSet) {
-		tags[tag] = batchSize;
-	}
-	loop {
-		timer = delayJittered(static_cast<double>(batchSize) / desiredRate);
-		GetReadVersionRequest req;
-		req.tags = tags;
-		req.priority = priority;
-		throttler->addRequest(req);
-		wait(success(req.reply.getFuture()) && timer);
-		for (auto& [tag, _] : tags) {
-			(*counters)[tag] += batchSize;
-		}
-	}
-}
-
-ACTOR static Future<Void> mockFifoClient(GrvProxyTagThrottler* throttler) {
-	state TransactionTagMap<uint32_t> tagSet1;
-	state TransactionTagMap<uint32_t> tagSet2;
-	state std::vector<GetReadVersionRequest> reqs;
-	state int i = 0;
-	// Used to track the order in which replies are received
-	state std::vector<int> replyIndices;
-
-	// Tag half of requests with one tag, half with another, then randomly shuffle
-	tagSet1["sampleTag1"_sr] = 1;
-	tagSet2["sampleTag2"_sr] = 1;
-	for (i = 0; i < 2000; ++i) {
-		auto& req = reqs.emplace_back();
-		req.priority = TransactionPriority::DEFAULT;
-		if (i < 1000) {
-			req.tags = tagSet1;
-		} else {
-			req.tags = tagSet2;
-		}
-	}
-	deterministicRandom()->randomShuffle(reqs);
-
-	// Send requests to throttler and assert that responses are received in FIFO order
-	for (const auto& req : reqs) {
-		throttler->addRequest(req);
-	}
-	state std::vector<Future<Void>> futures;
-	for (int j = 0; j < 2000; ++j) {
-		// Flow hack to capture replyIndices
-		auto* _replyIndices = &replyIndices;
-		futures.push_back(map(reqs[j].reply.getFuture(), [_replyIndices, j](auto const&) {
-			(*_replyIndices).push_back(j);
-			return Void();
-		}));
-	}
-	wait(waitForAll(futures));
-	for (i = 0; i < 2000; ++i) {
-		ASSERT_EQ(replyIndices[i], i);
-	}
-	return Void();
-}
-
-ACTOR static Future<Void> mockServer(GrvProxyTagThrottler* throttler) {
-	state Deque<GetReadVersionRequest> outBatchPriority;
-	state Deque<GetReadVersionRequest> outDefaultPriority;
-	loop {
-		state double elapsed = (0.009 + 0.002 * deterministicRandom()->random01());
-		wait(delay(elapsed));
-		throttler->releaseTransactions(elapsed, outBatchPriority, outDefaultPriority);
-		while (!outBatchPriority.empty()) {
-			outBatchPriority.front().reply.send(GetReadVersionReply{});
-			outBatchPriority.pop_front();
-		}
-		while (!outDefaultPriority.empty()) {
-			outDefaultPriority.front().reply.send(GetReadVersionReply{});
-			outDefaultPriority.pop_front();
-		}
-	}
-}
-
-static TransactionTag getRandomTag() {
-	TransactionTag result;
-	auto arr = new (result.arena()) uint8_t[32];
-	for (int i = 0; i < 32; ++i) {
-		arr[i] = (uint8_t)deterministicRandom()->randomInt(0, 256);
-	}
-	result.contents() = TransactionTagRef(arr, 32);
-	return result;
-}
-
-static bool isNear(double desired, int64_t actual) {
-	return std::abs(desired - actual) * 10 < desired;
-}
-
-// Rate limit set at 10, but client attempts 20 transactions per second.
-// Client should be throttled to only 10 transactions per second.
-TEST_CASE("/GrvProxyTagThrottler/Simple") {
-	state GrvProxyTagThrottler throttler(5.0);
-	state TagSet tagSet;
-	state TransactionTagMap<uint32_t> counters;
-	{
-		TransactionTagMap<double> rates;
-		rates["sampleTag"_sr] = 10.0;
-		throttler.updateRates(rates);
-	}
-	tagSet.addTag("sampleTag"_sr);
-
-	state Future<Void> client = mockClient(&throttler, TransactionPriority::DEFAULT, tagSet, 1, 20.0, &counters);
-	state Future<Void> server = mockServer(&throttler);
-	wait(timeout(client && server, 60.0, Void()));
-	TraceEvent("TagQuotaTest_Simple").detail("Counter", counters["sampleTag"_sr]);
-	ASSERT(isNear(counters["sampleTag"_sr], 60.0 * 10.0));
-	return Void();
-}
-
-// Clients share the available 30 transaction/second budget
-TEST_CASE("/GrvProxyTagThrottler/MultiClient") {
-	state GrvProxyTagThrottler throttler(5.0);
-	state TagSet tagSet;
-	state TransactionTagMap<uint32_t> counters;
-	{
-		TransactionTagMap<double> rates;
-		rates["sampleTag"_sr] = 30.0;
-		throttler.updateRates(rates);
-	}
-	tagSet.addTag("sampleTag"_sr);
-
-	state std::vector<Future<Void>> clients;
-	clients.reserve(10);
-	for (int i = 0; i < 10; ++i) {
-		clients.push_back(mockClient(&throttler, TransactionPriority::DEFAULT, tagSet, 1, 10.0, &counters));
-	}
-
-	state Future<Void> server = mockServer(&throttler);
-	wait(timeout(waitForAll(clients) && server, 60.0, Void()));
-	TraceEvent("TagQuotaTest_MultiClient").detail("Counter", counters["sampleTag"_sr]);
-	ASSERT(isNear(counters["sampleTag"_sr], 60.0 * 30.0));
-	return Void();
-}
-
-// Each tag receives 10 transaction/second budget
-TEST_CASE("/GrvProxyTagThrottler/MultiTag") {
-	state GrvProxyTagThrottler throttler(5.0);
-	state TagSet tagSet1;
-	state TagSet tagSet2;
-	state TransactionTagMap<uint32_t> counters;
-	{
-		TransactionTagMap<double> rates;
-		rates["sampleTag1"_sr] = 10.0;
-		rates["sampleTag2"_sr] = 10.0;
-		throttler.updateRates(rates);
-	}
-	tagSet1.addTag("sampleTag1"_sr);
-	tagSet2.addTag("sampleTag2"_sr);
-	state Future<Void> client1 = mockClient(&throttler, TransactionPriority::DEFAULT, tagSet1, 5, 20.0, &counters);
-	state Future<Void> client2 = mockClient(&throttler, TransactionPriority::DEFAULT, tagSet2, 5, 20.0, &counters);
-	state Future<Void> server = mockServer(&throttler);
-	wait(timeout(client1 && client2 && server, 60.0, Void()));
-	TraceEvent("TagQuotaTest_MultiTag")
-	    .detail("Counter1", counters["sampleTag1"_sr])
-	    .detail("Counter2", counters["sampleTag2"_sr]);
-	ASSERT(isNear(counters["sampleTag1"_sr], 60 * 10.0));
-	ASSERT(isNear(counters["sampleTag2"_sr], 60 * 10.0));
-	return Void();
-}
-
-// Test processing GetReadVersionRequests that batch several transactions
-TEST_CASE("/GrvProxyTagThrottler/Batch") {
-	state GrvProxyTagThrottler throttler(5.0);
-	state TagSet tagSet;
-	state TransactionTagMap<uint32_t> counters;
-	{
-		TransactionTagMap<double> rates;
-		rates["sampleTag"_sr] = 10.0;
-		throttler.updateRates(rates);
-	}
-	tagSet.addTag("sampleTag"_sr);
-
-	state Future<Void> client = mockClient(&throttler, TransactionPriority::DEFAULT, tagSet, 5, 20.0, &counters);
-	state Future<Void> server = mockServer(&throttler);
-	wait(timeout(client && server, 60.0, Void()));
-
-	TraceEvent("TagQuotaTest_Batch").detail("Counter", counters["sampleTag"_sr]);
-	ASSERT(isNear(counters["sampleTag"_sr], 60.0 * 10.0));
-	return Void();
-}
-
-// Tests cleanup of tags that are no longer throttled.
-TEST_CASE("/GrvProxyTagThrottler/Cleanup1") {
-	GrvProxyTagThrottler throttler(5.0);
-	for (int i = 0; i < 1000; ++i) {
-		auto const tag = getRandomTag();
-		TransactionTagMap<double> rates;
-		rates[tag] = 10.0;
-		throttler.updateRates(rates);
-		ASSERT_EQ(throttler.size(), 1);
-	}
-	return Void();
-}
-
-// Tests cleanup of tags once queues have been emptied
-TEST_CASE("/GrvProxyTagThrottler/Cleanup2") {
-	GrvProxyTagThrottler throttler(5.0);
-	{
-		GetReadVersionRequest req;
-		req.tags["sampleTag"_sr] = 1;
-		req.priority = TransactionPriority::DEFAULT;
-		throttler.addRequest(req);
-	}
-	ASSERT_EQ(throttler.size(), 1);
-	throttler.updateRates(TransactionTagMap<double>{});
-	ASSERT_EQ(throttler.size(), 1);
-	{
-		Deque<GetReadVersionRequest> outBatchPriority;
-		Deque<GetReadVersionRequest> outDefaultPriority;
-		throttler.releaseTransactions(0.1, outBatchPriority, outDefaultPriority);
-	}
-	// Calling updates cleans up the queues in throttler
-	throttler.updateRates(TransactionTagMap<double>{});
-	ASSERT_EQ(throttler.size(), 0);
-	return Void();
-}
-
-// Tests that unthrottled transactions are released in FIFO order, even when they
-// have different tags
-TEST_CASE("/GrvProxyTagThrottler/Fifo") {
-	state GrvProxyTagThrottler throttler(5.0);
-	state Future<Void> server = mockServer(&throttler);
-	wait(mockFifoClient(&throttler));
-	return Void();
-}
-
-// Tests that while throughput is low, the tag throttler
-// does not accumulate too much budget.
-//
-// A server is setup to server 100 transactions per second,
-// then runs idly for 60 seconds. Then a client starts
-// and attempts 200 transactions per second for 60 seconds.
-// The server throttles the client to only achieve
-// 10 transactions per second during this 60 second window.
-// If the throttler is allowed to accumulate budget indefinitely
-// during the idle 60 seconds, this test will fail.
-TEST_CASE("/GrvProxyTagThrottler/LimitedIdleBudget") {
-	state GrvProxyTagThrottler throttler(5.0);
-	state TagSet tagSet;
-	state TransactionTagMap<uint32_t> counters;
-	{
-		TransactionTagMap<double> rates;
-		rates["sampleTag"_sr] = 100.0;
-		throttler.updateRates(rates);
-	}
-	tagSet.addTag("sampleTag"_sr);
-
-	state Future<Void> server = mockServer(&throttler);
-	wait(delay(60.0));
-	state Future<Void> client = mockClient(&throttler, TransactionPriority::DEFAULT, tagSet, 1, 200.0, &counters);
-	wait(timeout(client && server, 60.0, Void()));
-	TraceEvent("TagQuotaTest_LimitedIdleBudget").detail("Counter", counters["sampleTag"_sr]);
-	ASSERT(isNear(counters["sampleTag"_sr], 60.0 * 100.0));
-	return Void();
-}
diff --git a/fdbserver/LatencyBandsMap.actor.cpp b/fdbserver/LatencyBandsMap.actor.cpp
deleted file mode 100644
index b7b1eedc0e2..00000000000
--- a/fdbserver/LatencyBandsMap.actor.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * LatencyBandsMap.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/LatencyBandsMap.h"
-#include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // must be last include
-
-class LatencyBandsMapImpl {
-public:
-	ACTOR static Future<Void> expireOldTagsActor(LatencyBandsMap* self) {
-		loop {
-			wait(delay(5.0));
-			for (auto it = self->map.begin(); it != self->map.end();) {
-				const auto& [tag, expirableBands] = *it;
-				if (now() - expirableBands.lastUpdated > SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER) {
-					CODE_PROBE(true, "LatencyBandsMap erasing expired tag");
-					it = self->map.erase(it);
-				} else {
-					++it;
-				}
-			}
-		}
-	}
-};
-
-LatencyBandsMap::ExpirableBands::ExpirableBands(LatencyBands&& bands)
-  : latencyBands(std::move(bands)), lastUpdated(now()) {}
-
-Optional<LatencyBands*> LatencyBandsMap::getLatencyBands(TransactionTag tag) {
-	if (map.size() == maxSize && !map.contains(tag)) {
-		CODE_PROBE(true, "LatencyBandsMap reached maxSize");
-		return {};
-	}
-	auto const [it, inserted] = map.try_emplace(
-	    tag, LatencyBands(name, id, loggingInterval, [tag](auto& te) { te.detail("Tag", printable(tag)); }));
-	auto& expirableBands = it->second;
-	if (inserted) {
-		for (const auto& threshold : thresholds) {
-			expirableBands.latencyBands.addThreshold(threshold);
-		}
-	}
-	expirableBands.lastUpdated = now();
-	return &expirableBands.latencyBands;
-}
-
-LatencyBandsMap::LatencyBandsMap(std::string const& name, UID id, double loggingInterval, int maxSize)
-  : name(name), id(id), loggingInterval(loggingInterval), maxSize(maxSize) {
-	expireOldTags = LatencyBandsMapImpl::expireOldTagsActor(this);
-}
-
-void LatencyBandsMap::addMeasurement(TransactionTag tag, double value, int count) {
-	auto bands = getLatencyBands(tag);
-	if (bands.present()) {
-		bands.get()->addMeasurement(value, count);
-	}
-}
-
-void LatencyBandsMap::addThreshold(double value) {
-	thresholds.push_back(value);
-	for (auto& [tag, expirableBands] : map) {
-		expirableBands.latencyBands.addThreshold(value);
-	}
-}
-
-TEST_CASE("/fdbserver/LatencyBandsMap/Simple") {
-	state LatencyBandsMap latencyBandsMap("TestLatencyBandsMap", deterministicRandom()->randomUniqueID(), 10.0, 100);
-	state Standalone<VectorRef<TransactionTagRef>> tags;
-	tags.push_back_deep(tags.arena(), "a"_sr);
-	tags.push_back_deep(tags.arena(), "b"_sr);
-	tags.push_back_deep(tags.arena(), "c"_sr);
-	latencyBandsMap.addThreshold(0.1);
-	latencyBandsMap.addThreshold(0.2);
-	latencyBandsMap.addThreshold(0.4);
-	state int measurements = 0;
-	loop {
-		wait(delayJittered(0.1));
-		auto const tag = deterministicRandom()->randomChoice(tags);
-		latencyBandsMap.addMeasurement(tag, deterministicRandom()->random01());
-		if (++measurements == 1000) {
-			return Void();
-		}
-	}
-}
-
-TEST_CASE("/fdbserver/LatencyBandsMap/MaxSize") {
-	LatencyBandsMap latencyBandsMap("TestLatencyBandsMap", deterministicRandom()->randomUniqueID(), 10.0, 2);
-	latencyBandsMap.addMeasurement("a"_sr, deterministicRandom()->random01());
-	latencyBandsMap.addMeasurement("b"_sr, deterministicRandom()->random01());
-	latencyBandsMap.addMeasurement("c"_sr, deterministicRandom()->random01());
-	ASSERT_EQ(latencyBandsMap.size(), 2);
-	return Void();
-}
-
-TEST_CASE("/fdbserver/LatencyBandsMap/Expire") {
-	state LatencyBandsMap latencyBandsMap("TestLatencyBandsMap", deterministicRandom()->randomUniqueID(), 10.0, 100);
-	latencyBandsMap.addMeasurement("a"_sr, deterministicRandom()->random01());
-	latencyBandsMap.addMeasurement("b"_sr, deterministicRandom()->random01());
-	latencyBandsMap.addMeasurement("c"_sr, deterministicRandom()->random01());
-	latencyBandsMap.addThreshold(0.1);
-	latencyBandsMap.addThreshold(0.2);
-	latencyBandsMap.addThreshold(0.4);
-	ASSERT_EQ(latencyBandsMap.size(), 3);
-	state int waitIterations = 0;
-	loop {
-		wait(delay(1.0));
-		latencyBandsMap.addMeasurement("a"_sr, deterministicRandom()->random01());
-		latencyBandsMap.addMeasurement("b"_sr, deterministicRandom()->random01());
-		if (++waitIterations == 2 * SERVER_KNOBS->GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER) {
-			break;
-		}
-	}
-	ASSERT_EQ(latencyBandsMap.size(), 2);
-	return Void();
-}
diff --git a/fdbserver/RemoteIKeyValueStore.actor.cpp b/fdbserver/RemoteIKeyValueStore.actor.cpp
deleted file mode 100644
index 98c4bd584c8..00000000000
--- a/fdbserver/RemoteIKeyValueStore.actor.cpp
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * RemoteIKeyValueStore.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "flow/ActorCollection.h"
-#include "flow/Error.h"
-#include "flow/Platform.h"
-#include "flow/Trace.h"
-#include "fdbrpc/FlowProcess.actor.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RemoteIKeyValueStore.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-StringRef KeyValueStoreProcess::_name = "KeyValueStoreProcess"_sr;
-
-// A guard for guaranteed killing of machine after runIKVS returns
-struct AfterReturn {
-	IKeyValueStore* kvStore;
-	UID id;
-	AfterReturn() : kvStore(nullptr) {}
-	AfterReturn(IKeyValueStore* store, UID& uid) : kvStore(store), id(uid) {}
-	~AfterReturn() {
-		TraceEvent(SevDebug, "RemoteKVStoreAfterReturn")
-		    .detail("Valid", kvStore != nullptr ? "True" : "False")
-		    .detail("UID", id)
-		    .log();
-		if (kvStore != nullptr) {
-			kvStore->close();
-		}
-	}
-	// called when we already explicitly closed the kv store
-	void invalidate() { kvStore = nullptr; }
-};
-
-ACTOR void sendCommitReply(IKVSCommitRequest commitReq, IKeyValueStore* kvStore, Future<Void> onClosed) {
-	try {
-		choose {
-			when(wait(onClosed)) {
-				commitReq.reply.sendError(remote_kvs_cancelled());
-			}
-			when(wait(kvStore->commit(commitReq.sequential))) {
-				StorageBytes storageBytes = kvStore->getStorageBytes();
-				commitReq.reply.send(IKVSCommitReply(storageBytes));
-			}
-		}
-	} catch (Error& e) {
-		TraceEvent(SevDebug, "RemoteKVSCommitReplyError").errorUnsuppressed(e);
-		commitReq.reply.sendError(e.code() == error_code_actor_cancelled ? remote_kvs_cancelled() : e);
-	}
-}
-
-ACTOR template <class T>
-Future<Void> cancellableForwardPromise(ReplyPromise<T> output, Future<T> input) {
-	try {
-		T value = wait(input);
-		output.send(value);
-	} catch (Error& e) {
-		TraceEvent(SevDebug, "CancellableForwardPromiseError").errorUnsuppressed(e).backtrace();
-		output.sendError(e.code() == error_code_actor_cancelled ? remote_kvs_cancelled() : e);
-	}
-	return Void();
-}
-
-ACTOR Future<Void> runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterface) {
-	state IKeyValueStore* kvStore = openKVStore(openReq.storeType,
-	                                            openReq.filename,
-	                                            openReq.logID,
-	                                            openReq.memoryLimit,
-	                                            openReq.checkChecksums,
-	                                            openReq.checkIntegrity);
-	state UID kvsId(ikvsInterface.id());
-	state ActorCollection actors(false);
-	state AfterReturn guard(kvStore, kvsId);
-	state Promise<Void> onClosed;
-	TraceEvent(SevDebug, "RemoteKVStoreInitializing").detail("UID", kvsId);
-	wait(kvStore->init());
-	openReq.reply.send(ikvsInterface);
-	TraceEvent(SevInfo, "RemoteKVStoreInitialized").detail("IKVSInterfaceUID", kvsId);
-
-	loop {
-		try {
-			choose {
-				when(IKVSGetValueRequest getReq = waitNext(ikvsInterface.getValue.getFuture())) {
-					actors.add(cancellableForwardPromise(getReq.reply, kvStore->readValue(getReq.key, getReq.options)));
-				}
-				when(IKVSSetRequest req = waitNext(ikvsInterface.set.getFuture())) {
-					kvStore->set(req.keyValue);
-				}
-				when(IKVSClearRequest req = waitNext(ikvsInterface.clear.getFuture())) {
-					kvStore->clear(req.range);
-				}
-				when(IKVSCommitRequest commitReq = waitNext(ikvsInterface.commit.getFuture())) {
-					sendCommitReply(commitReq, kvStore, onClosed.getFuture());
-				}
-				when(IKVSReadValuePrefixRequest readPrefixReq = waitNext(ikvsInterface.readValuePrefix.getFuture())) {
-					actors.add(cancellableForwardPromise(
-					    readPrefixReq.reply,
-					    kvStore->readValuePrefix(readPrefixReq.key, readPrefixReq.maxLength, readPrefixReq.options)));
-				}
-				when(IKVSReadRangeRequest readRangeReq = waitNext(ikvsInterface.readRange.getFuture())) {
-					actors.add(cancellableForwardPromise(
-					    readRangeReq.reply,
-					    fmap([](const RangeResult& result) { return IKVSReadRangeReply(result); },
-					         kvStore->readRange(readRangeReq.keys,
-					                            readRangeReq.rowLimit,
-					                            readRangeReq.byteLimit,
-					                            readRangeReq.options))));
-				}
-				when(IKVSGetStorageByteRequest req = waitNext(ikvsInterface.getStorageBytes.getFuture())) {
-					StorageBytes storageBytes = kvStore->getStorageBytes();
-					req.reply.send(storageBytes);
-				}
-				when(IKVSGetErrorRequest getFutureReq = waitNext(ikvsInterface.getError.getFuture())) {
-					actors.add(cancellableForwardPromise(getFutureReq.reply, kvStore->getError()));
-				}
-				when(IKVSOnClosedRequest onClosedReq = waitNext(ikvsInterface.onClosed.getFuture())) {
-					// onClosed request is not cancelled even this actor is cancelled
-					forwardPromise(onClosedReq.reply, kvStore->onClosed());
-				}
-				when(IKVSDisposeRequest disposeReq = waitNext(ikvsInterface.dispose.getFuture())) {
-					(void)disposeReq; // suppress unused variable warning
-					TraceEvent(SevDebug, "RemoteIKVSDisposeReceivedRequest").detail("UID", kvsId);
-					kvStore->dispose();
-					guard.invalidate();
-					onClosed.send(Void());
-					return Void();
-				}
-				when(IKVSCloseRequest closeReq = waitNext(ikvsInterface.close.getFuture())) {
-					(void)closeReq; // suppress unused variable warning
-					TraceEvent(SevDebug, "RemoteIKVSCloseReceivedRequest").detail("UID", kvsId);
-					kvStore->close();
-					guard.invalidate();
-					onClosed.send(Void());
-					return Void();
-				}
-			}
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				TraceEvent(SevInfo, "RemoteKVStoreCancelled").detail("UID", kvsId).backtrace();
-				onClosed.send(Void());
-				return Void();
-			} else {
-				TraceEvent(SevError, "RemoteKVStoreError").error(e).detail("UID", kvsId).backtrace();
-				throw;
-			}
-		}
-	}
-}
-
-ACTOR static Future<int> flowProcessRunner(RemoteIKeyValueStore* self, Promise<Void> ready) {
-	state FlowProcessInterface processInterface;
-	state Future<int> process;
-
-	auto path = abspath(getExecPath());
-	auto endpoint = processInterface.registerProcess.getEndpoint();
-	auto address = endpoint.addresses.address.toString();
-	auto token = endpoint.token;
-
-	// port 0 means we will find a random available port number for it
-	std::string flowProcessAddr = g_network->getLocalAddress().ip.toString().append(":0");
-	std::vector<std::string> args = { "bin/fdbserver",
-		                              "-r",
-		                              "flowprocess",
-		                              "-C",
-		                              SERVER_KNOBS->CONN_FILE,
-		                              "--logdir",
-		                              SERVER_KNOBS->LOG_DIRECTORY,
-		                              "--trace-format",
-		                              getTraceFormatExtension(),
-		                              "-p",
-		                              flowProcessAddr,
-		                              "--process-name",
-		                              KeyValueStoreProcess::_name.toString(),
-		                              "--process-endpoint",
-		                              format("%s,%lu,%lu", address.c_str(), token.first(), token.second()) };
-	// For remote IKV store, we need to make sure the shutdown signal is sent back until we can destroy it in the
-	// simulation
-	process = spawnProcess(path, args, -1.0, false, 0.01 /*not used*/, self);
-	choose {
-		when(FlowProcessRegistrationRequest req = waitNext(processInterface.registerProcess.getFuture())) {
-			self->consumeInterface(req.flowProcessInterface);
-			ready.send(Void());
-		}
-		when(int res = wait(process)) {
-			// 0 means process normally shut down; non-zero means errors
-			// process should not shut down normally before not ready
-			ASSERT(res);
-			return res;
-		}
-	}
-	int res = wait(process);
-	return res;
-}
-
-ACTOR static Future<Void> initializeRemoteKVStore(RemoteIKeyValueStore* self, OpenKVStoreRequest openKVSReq) {
-	TraceEvent(SevInfo, "WaitingOnFlowProcess").detail("StoreType", openKVSReq.storeType).log();
-	Promise<Void> ready;
-	self->returnCode = flowProcessRunner(self, ready);
-	wait(ready.getFuture());
-	IKVSInterface ikvsInterface = wait(self->kvsProcess.openKVStore.getReply(openKVSReq));
-	TraceEvent(SevInfo, "IKVSInterfaceReceived").detail("UID", ikvsInterface.id());
-	self->interf = ikvsInterface;
-	self->interf.storeType = openKVSReq.storeType;
-	return Void();
-}
-
-IKeyValueStore* openRemoteKVStore(KeyValueStoreType storeType,
-                                  std::string const& filename,
-                                  UID logID,
-                                  int64_t memoryLimit,
-                                  bool checkChecksums,
-                                  bool checkIntegrity) {
-	RemoteIKeyValueStore* self = new RemoteIKeyValueStore();
-	self->initialized = initializeRemoteKVStore(
-	    self, OpenKVStoreRequest(storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity));
-	return self;
-}
-
-Future<Void> runFlowProcess(std::string const& name, Endpoint endpoint) {
-	TraceEvent(SevInfo, "RunFlowProcessStart").log();
-	FlowProcess* self = IProcessFactory::create(name.c_str());
-	self->registerEndpoint(endpoint);
-	RequestStream<FlowProcessRegistrationRequest> registerProcess(endpoint);
-	FlowProcessRegistrationRequest req;
-	req.flowProcessInterface = self->serializedInterface();
-	registerProcess.send(req);
-	TraceEvent(SevDebug, "FlowProcessInitFinished").log();
-	return self->run();
-}
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
deleted file mode 100644
index 4b078ce6cdf..00000000000
--- a/fdbserver/RestoreApplier.actor.cpp
+++ /dev/null
@@ -1,825 +0,0 @@
-/*
- * RestoreApplier.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file defines the functions used by the RestoreApplier role.
-// RestoreApplier role starts at restoreApplierCore actor
-
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/SystemData.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RestoreCommon.actor.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreApplier.actor.h"
-
-#include "flow/network.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
-                                                          Reference<RestoreApplierData> self);
-ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req,
-                                                 Reference<RestoreApplierData> self,
-                                                 Database cx);
-void handleUpdateRateRequest(RestoreUpdateRateRequest req, Reference<RestoreApplierData> self);
-
-ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int nodeIndex, Database cx) {
-	state Reference<RestoreApplierData> self = makeReference<RestoreApplierData>(applierInterf.id(), nodeIndex);
-	state ActorCollection actors(false);
-	state Future<Void> exitRole = Never();
-
-	actors.add(updateProcessMetrics(self));
-	actors.add(traceProcessMetrics(self, "RestoreApplier"));
-	actors.add(traceRoleVersionBatchProgress(self, "RestoreApplier"));
-
-	loop {
-		state std::string requestTypeStr = "[Init]";
-
-		try {
-			choose {
-				when(RestoreSimpleRequest req = waitNext(applierInterf.heartbeat.getFuture())) {
-					requestTypeStr = "heartbeat";
-					actors.add(handleHeartbeat(req, applierInterf.id()));
-				}
-				when(RestoreSendVersionedMutationsRequest req =
-				         waitNext(applierInterf.sendMutationVector.getFuture())) {
-					requestTypeStr = "sendMutationVector";
-					actors.add(handleSendMutationVectorRequest(req, self));
-				}
-				when(RestoreVersionBatchRequest req = waitNext(applierInterf.applyToDB.getFuture())) {
-					requestTypeStr = "applyToDB";
-					actors.add(handleApplyToDBRequest(
-					    req, self, cx)); // TODO: Check how FDB uses TaskPriority for ACTORS. We may need to add
-					                     // priority here to avoid requests at later VB block requests at earlier VBs
-				}
-				when(RestoreUpdateRateRequest req = waitNext(applierInterf.updateRate.getFuture())) {
-					requestTypeStr = "updateRate";
-					handleUpdateRateRequest(req, self);
-				}
-				when(RestoreVersionBatchRequest req = waitNext(applierInterf.initVersionBatch.getFuture())) {
-					requestTypeStr = "initVersionBatch";
-					actors.add(handleInitVersionBatchRequest(req, self));
-				}
-				when(RestoreFinishRequest req = waitNext(applierInterf.finishRestore.getFuture())) {
-					requestTypeStr = "finishRestore";
-					actors.clear(false); // cancel all pending actors
-					handleFinishRestoreRequest(req, self);
-					if (req.terminate) {
-						exitRole = Void();
-					}
-				}
-				when(wait(actors.getResult())) {}
-				when(wait(exitRole)) {
-					TraceEvent("RestoreApplierCoreExitRole", self->id());
-					break;
-				}
-			}
-			//TraceEvent("RestoreApplierCore", self->id()).detail("Request", requestTypeStr); // For debug only
-		} catch (Error& e) {
-			bool isError = e.code() != error_code_operation_cancelled;
-			TraceEvent(isError ? SevError : SevWarnAlways, "FastRestoreApplierError", self->id())
-			    .errorUnsuppressed(e)
-			    .detail("RequestType", requestTypeStr);
-			actors.clear(false);
-			break;
-		}
-	}
-
-	return Void();
-}
-
-// The actor may be invoked multiple times and executed async.
-// No race condition as long as we do not wait or yield when operate the shared
-// data. Multiple such actors can run on different fileIDs.
-// Different files may contain mutations of the same commit versions, but with
-// different subsequence number.
-// Only one actor can process mutations from the same file.
-ACTOR static Future<Void> handleSendMutationVectorRequest(RestoreSendVersionedMutationsRequest req,
-                                                          Reference<RestoreApplierData> self) {
-	state Reference<ApplierBatchData> batchData; // initialized as nullptr
-	state bool printTrace = false;
-	state NotifiedVersion* curMsgIndex = nullptr;
-
-	if (req.batchIndex <= self->finishedBatch.get()) { // Handle duplicate request from batchIndex that has finished
-		TraceEvent(SevWarn, "FastRestoreApplierRestoreSendVersionedMutationsRequestTooLate")
-		    .detail("RequestBatchIndex", req.batchIndex)
-		    .detail("FinishedBatchIndex", self->finishedBatch.get());
-		req.reply.send(RestoreCommonReply(self->id(), true));
-		ASSERT_WE_THINK(false); // Test to see if simulation can reproduce this
-		return Void();
-	}
-
-	batchData = self->batch[req.batchIndex];
-
-	ASSERT(batchData.isValid());
-	ASSERT(self->finishedBatch.get() < req.batchIndex);
-	// wait(delay(0.0, TaskPriority::RestoreApplierReceiveMutations)); // This hurts performance from 100MB/s to 60MB/s
-	// on circus
-
-	batchData->receiveMutationReqs += 1;
-	// Trace when the receive phase starts at a VB and when it finishes.
-	// This can help check if receiveMutations block applyMutation phase.
-	// If so, we need more sophisticated scheduler to ensure priority execution
-	printTrace = (batchData->receiveMutationReqs % SERVER_KNOBS->FASTRESTORE_NUM_TRACE_EVENTS == 0);
-	TraceEvent(printTrace ? SevInfo : SevFRDebugInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("RestoreAsset", req.asset.toString())
-	    .detail("RestoreAssetMesssageIndex", batchData->processedFileState[req.asset].get())
-	    .detail("Request", req.toString())
-	    .detail("CurrentMemory", getSystemStatistics().processMemory)
-	    .detail("PreviousVersionBatchState", batchData->vbState.get())
-	    .detail("ReceiveMutationRequests", batchData->receiveMutationReqs);
-
-	wait(isSchedulable(self, req.batchIndex, __FUNCTION__));
-
-	ASSERT(batchData.isValid());
-	ASSERT(req.batchIndex > self->finishedBatch.get());
-	// Assume: processedFileState[req.asset] will not be erased while the actor is active.
-	// Note: Insert new items into processedFileState will not invalidate the reference.
-	curMsgIndex = &batchData->processedFileState[req.asset];
-	wait(curMsgIndex->whenAtLeast(req.msgIndex - 1));
-	batchData->vbState = ApplierVersionBatchState::RECEIVE_MUTATIONS;
-
-	state bool isDuplicated = true;
-	if (curMsgIndex->get() == req.msgIndex - 1) {
-		isDuplicated = false;
-
-		for (int mIndex = 0; mIndex < req.versionedMutations.size(); mIndex++) {
-			const VersionedMutationSerialized& versionedMutation = req.versionedMutations[mIndex];
-			TraceEvent(SevFRDebugInfo, "FastRestoreApplierPhaseReceiveMutations", self->id())
-			    .detail("RestoreAsset", req.asset.toString())
-			    .detail("Version", versionedMutation.version.toString())
-			    .detail("Index", mIndex)
-			    .detail("MutationReceived", versionedMutation.mutation.toString());
-			batchData->receivedBytes += versionedMutation.mutation.totalSize();
-			batchData->counters.receivedBytes += versionedMutation.mutation.totalSize();
-			batchData->counters.receivedWeightedBytes +=
-			    versionedMutation.mutation.weightedTotalSize(); // atomicOp will be amplified
-			batchData->counters.receivedMutations += 1;
-			batchData->counters.receivedAtomicOps +=
-			    isAtomicOp((MutationRef::Type)versionedMutation.mutation.type) ? 1 : 0;
-			// Sanity check
-			ASSERT_WE_THINK(req.asset.isInVersionRange(versionedMutation.version.version));
-			ASSERT_WE_THINK(req.asset.isInKeyRange(
-			    versionedMutation.mutation)); // mutation is already applied removePrefix and addPrefix
-
-			// Note: Log and range mutations may be delivered out of order. Can we handle it?
-			batchData->addMutation(versionedMutation.mutation, versionedMutation.version);
-
-			ASSERT(versionedMutation.mutation.type != MutationRef::SetVersionstampedKey &&
-			       versionedMutation.mutation.type != MutationRef::SetVersionstampedValue);
-		}
-		curMsgIndex->set(req.msgIndex);
-	}
-
-	req.reply.send(RestoreCommonReply(self->id(), isDuplicated));
-	TraceEvent(printTrace ? SevInfo : SevFRDebugInfo, "FastRestoreApplierPhaseReceiveMutationsDone", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("RestoreAsset", req.asset.toString())
-	    .detail("ProcessedMessageIndex", curMsgIndex->get())
-	    .detail("Request", req.toString());
-	return Void();
-}
-
-// Clear all ranges in input ranges
-ACTOR static Future<Void> applyClearRangeMutations(Standalone<VectorRef<KeyRangeRef>> ranges,
-                                                   double delayTime,
-                                                   Database cx,
-                                                   UID applierID,
-                                                   int batchIndex,
-                                                   ApplierBatchData::Counters* cc) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state int retries = 0;
-	state double numOps = 0;
-	wait(delay(delayTime + deterministicRandom()->random01() * delayTime));
-	TraceEvent(delayTime > 5 ? SevWarnAlways : SevDebug, "FastRestoreApplierClearRangeMutationsStart", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Ranges", ranges.size())
-	    .detail("DelayTime", delayTime);
-	if (SERVER_KNOBS->FASTRESTORE_NOT_WRITE_DB) {
-		TraceEvent("FastRestoreApplierClearRangeMutationsNotWriteDB", applierID)
-		    .detail("BatchIndex", batchIndex)
-		    .detail("Ranges", ranges.size());
-		ASSERT(!g_network->isSimulated());
-		return Void();
-	}
-
-	loop {
-		try {
-			// TODO: Consider clearrange traffic in write traffic control
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			for (auto& range : ranges) {
-				debugFRMutation("FastRestoreApplierApplyClearRangeMutation",
-				                0,
-				                MutationRef(MutationRef::ClearRange, range.begin, range.end));
-				tr->clear(range);
-				cc->clearOps += 1;
-				++numOps;
-				if (numOps >= SERVER_KNOBS->FASTRESTORE_TXN_CLEAR_MAX) {
-					TraceEvent(SevWarn, "FastRestoreApplierClearRangeMutationsTooManyClearsInTxn")
-					    .suppressFor(5.0)
-					    .detail("Clears", numOps)
-					    .detail("Ranges", ranges.size())
-					    .detail("Range", range.toString());
-				}
-			}
-			wait(tr->commit());
-			cc->clearTxns += 1;
-			break;
-		} catch (Error& e) {
-			retries++;
-			if (retries > SERVER_KNOBS->FASTRESTORE_TXN_RETRY_MAX) {
-				TraceEvent(SevWarnAlways, "RestoreApplierApplyClearRangeMutationsStuck", applierID)
-				    .error(e)
-				    .detail("BatchIndex", batchIndex)
-				    .detail("ClearRanges", ranges.size());
-			}
-			wait(tr->onError(e));
-		}
-	}
-	return Void();
-}
-
-// Get keys in incompleteStagingKeys and precompute the stagingKey which is stored in batchData->stagingKeys
-ACTOR static Future<Void> getAndComputeStagingKeys(
-    std::map<Key, std::map<Key, StagingKey>::iterator> incompleteStagingKeys,
-    double delayTime,
-    Database cx,
-    UID applierID,
-    int batchIndex,
-    ApplierBatchData::Counters* cc) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state std::vector<Future<Optional<Value>>> fValues(incompleteStagingKeys.size(), Never());
-	state int retries = 0;
-	state UID randomID = deterministicRandom()->randomUniqueID();
-
-	wait(delay(delayTime + deterministicRandom()->random01() * delayTime));
-
-	if (SERVER_KNOBS->FASTRESTORE_NOT_WRITE_DB) { // Get dummy value to short-circut DB
-		TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStartNotUseDB", applierID)
-		    .detail("RandomUID", randomID)
-		    .detail("BatchIndex", batchIndex)
-		    .detail("GetKeys", incompleteStagingKeys.size())
-		    .detail("DelayTime", delayTime);
-		ASSERT(!g_network->isSimulated());
-		for (auto& key : incompleteStagingKeys) {
-			MutationRef m(MutationRef::SetValue, key.first, "0"_sr);
-			key.second->second.add(m, LogMessageVersion(1));
-			key.second->second.precomputeResult("GetAndComputeStagingKeys", applierID, batchIndex);
-		}
-		return Void();
-	}
-
-	TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID)
-	    .detail("RandomUID", randomID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("GetKeys", incompleteStagingKeys.size())
-	    .detail("DelayTime", delayTime);
-
-	loop {
-		try {
-			int i = 0;
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			for (auto& key : incompleteStagingKeys) {
-				fValues[i++] = tr->get(key.first);
-				cc->fetchKeys += 1;
-			}
-			wait(waitForAll(fValues));
-			cc->fetchTxns += 1;
-			break;
-		} catch (Error& e) {
-			cc->fetchTxnRetries += 1;
-			if (retries++ > incompleteStagingKeys.size()) {
-				if (e.code() != error_code_actor_cancelled) {
-					TraceEvent(SevWarnAlways, "GetAndComputeStagingKeys", applierID)
-					    .errorUnsuppressed(e)
-					    .suppressFor(1.0)
-					    .detail("RandomUID", randomID)
-					    .detail("BatchIndex", batchIndex);
-				}
-			}
-			wait(tr->onError(e));
-		}
-	}
-
-	ASSERT(fValues.size() == incompleteStagingKeys.size());
-	int i = 0;
-	for (auto& key : incompleteStagingKeys) {
-		if (!fValues[i].get().present()) { // Key not exist in DB
-			// if condition: fValues[i].Valid() && fValues[i].isReady() && !fValues[i].isError() &&
-			TraceEvent(SevDebug, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB", applierID)
-			    .suppressFor(5.0)
-			    .detail("BatchIndex", batchIndex)
-			    .detail("Key", key.first)
-			    .detail("IsReady", fValues[i].isReady())
-			    .detail("PendingMutations", key.second->second.pendingMutations.size())
-			    .detail("StagingKeyType", getTypeString(key.second->second.type));
-			for (auto& vm : key.second->second.pendingMutations) {
-				TraceEvent(SevDebug, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB")
-				    .detail("PendingMutationVersion", vm.first.toString())
-				    .detail("PendingMutation", vm.second.toString());
-			}
-			key.second->second.precomputeResult("GetAndComputeStagingKeysNoBaseValueInDB", applierID, batchIndex);
-		} else {
-			// The key's version ideally should be the most recently committed version.
-			// But as long as it is > 1 and less than the start version of the version batch, it is the same result.
-			MutationRef m(MutationRef::SetValue, key.first, fValues[i].get().get());
-			key.second->second.add(m, LogMessageVersion(1));
-			key.second->second.precomputeResult("GetAndComputeStagingKeys", applierID, batchIndex);
-		}
-		i++;
-	}
-
-	TraceEvent("FastRestoreApplierGetAndComputeStagingKeysDone", applierID)
-	    .detail("RandomUID", randomID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("GetKeys", incompleteStagingKeys.size())
-	    .detail("DelayTime", delayTime);
-
-	return Void();
-}
-
-ACTOR static Future<Void> precomputeMutationsResult(Reference<ApplierBatchData> batchData,
-                                                    UID applierID,
-                                                    int64_t batchIndex,
-                                                    Database cx) {
-	// Apply range mutations (i.e., clearRange) to database cx
-	TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResultStart", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Step", "Applying clear range mutations to DB")
-	    .detail("ClearRanges", batchData->stagingKeyRanges.size());
-	state std::vector<Future<Void>> fClearRanges;
-	Standalone<VectorRef<KeyRangeRef>> clearRanges;
-	double curTxnSize = 0;
-	{
-		double delayTime = 0;
-		for (auto& rangeMutation : batchData->stagingKeyRanges) {
-			KeyRangeRef range(rangeMutation.mutation.param1, rangeMutation.mutation.param2);
-			debugFRMutation("FastRestoreApplierPrecomputeMutationsResultClearRange",
-			                rangeMutation.version.version,
-			                MutationRef(MutationRef::ClearRange, range.begin, range.end));
-			clearRanges.push_back_deep(clearRanges.arena(), range);
-			curTxnSize += range.expectedSize();
-			if (curTxnSize >= SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES) {
-				fClearRanges.push_back(
-				    applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
-				delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
-				clearRanges = Standalone<VectorRef<KeyRangeRef>>();
-				curTxnSize = 0;
-			}
-		}
-		if (curTxnSize > 0) {
-			fClearRanges.push_back(
-			    applyClearRangeMutations(clearRanges, delayTime, cx, applierID, batchIndex, &batchData->counters));
-		}
-	}
-
-	// Apply range mutations (i.e., clearRange) to stagingKeyRanges
-	TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Step", "Applying clear range mutations to staging keys")
-	    .detail("ClearRanges", batchData->stagingKeyRanges.size())
-	    .detail("FutureClearRanges", fClearRanges.size());
-	for (auto& rangeMutation : batchData->stagingKeyRanges) {
-		ASSERT(rangeMutation.mutation.param1 <= rangeMutation.mutation.param2);
-		std::map<Key, StagingKey>::iterator lb = batchData->stagingKeys.lower_bound(rangeMutation.mutation.param1);
-		std::map<Key, StagingKey>::iterator ub = batchData->stagingKeys.lower_bound(rangeMutation.mutation.param2);
-		while (lb != ub) {
-			if (lb->first >= rangeMutation.mutation.param2) {
-				TraceEvent(SevError, "FastRestoreApplerPhasePrecomputeMutationsResultIncorrectUpperBound")
-				    .detail("Key", lb->first)
-				    .detail("ClearRangeUpperBound", rangeMutation.mutation.param2)
-				    .detail("UsedUpperBound", ub->first);
-			}
-			// We make the beginKey = endKey for the ClearRange on purpose so that
-			// we can sanity check ClearRange mutation when we apply it to DB.
-			MutationRef clearKey(MutationRef::ClearRange, lb->first, lb->first);
-			lb->second.add(clearKey, rangeMutation.version);
-			lb++;
-		}
-	}
-	TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Step", "Wait on applying clear range mutations to DB")
-	    .detail("FutureClearRanges", fClearRanges.size());
-
-	wait(waitForAll(fClearRanges));
-	TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Step", "Getting and computing staging keys")
-	    .detail("StagingKeys", batchData->stagingKeys.size());
-
-	// Get keys in stagingKeys which does not have a baseline key by reading database cx, and precompute the key's value
-	std::vector<Future<Void>> fGetAndComputeKeys;
-	std::map<Key, std::map<Key, StagingKey>::iterator> incompleteStagingKeys;
-	std::map<Key, StagingKey>::iterator stagingKeyIter = batchData->stagingKeys.begin();
-	int numKeysInBatch = 0;
-	int numGetTxns = 0;
-	{
-		double delayTime = 0; // Start transactions at different time to avoid overwhelming FDB.
-		for (; stagingKeyIter != batchData->stagingKeys.end(); stagingKeyIter++) {
-			if (!stagingKeyIter->second.hasBaseValue()) {
-				incompleteStagingKeys.emplace(stagingKeyIter->first, stagingKeyIter);
-				numKeysInBatch++;
-			}
-			if (numKeysInBatch == SERVER_KNOBS->FASTRESTORE_APPLIER_FETCH_KEYS_SIZE) {
-				fGetAndComputeKeys.push_back(getAndComputeStagingKeys(
-				    incompleteStagingKeys, delayTime, cx, applierID, batchIndex, &batchData->counters));
-				numGetTxns++;
-				delayTime += SERVER_KNOBS->FASTRESTORE_TXN_EXTRA_DELAY;
-				numKeysInBatch = 0;
-				incompleteStagingKeys.clear();
-			}
-		}
-		if (numKeysInBatch > 0) {
-			numGetTxns++;
-			fGetAndComputeKeys.push_back(getAndComputeStagingKeys(
-			    incompleteStagingKeys, delayTime, cx, applierID, batchIndex, &batchData->counters));
-		}
-	}
-
-	TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResult", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Step", "Compute the other staging keys")
-	    .detail("StagingKeys", batchData->stagingKeys.size())
-	    .detail("GetStagingKeyBatchTxns", numGetTxns);
-	// Pre-compute pendingMutations to other keys in stagingKeys that has base value
-	for (stagingKeyIter = batchData->stagingKeys.begin(); stagingKeyIter != batchData->stagingKeys.end();
-	     stagingKeyIter++) {
-		if (stagingKeyIter->second.hasBaseValue()) {
-			stagingKeyIter->second.precomputeResult("HasBaseValue", applierID, batchIndex);
-		}
-	}
-
-	TraceEvent("FastRestoreApplierGetAndComputeStagingKeysWaitOn", applierID).log();
-	wait(waitForAll(fGetAndComputeKeys));
-
-	// Sanity check all stagingKeys have been precomputed
-	ASSERT_WE_THINK(batchData->allKeysPrecomputed());
-
-	TraceEvent("FastRestoreApplerPhasePrecomputeMutationsResultDone", applierID).detail("BatchIndex", batchIndex);
-
-	return Void();
-}
-
-bool okToReleaseTxns(double targetMB, double applyingDataBytes) {
-	return applyingDataBytes < targetMB * 1024 * 1024;
-}
-
-ACTOR static Future<Void> shouldReleaseTransaction(double* targetMB,
-                                                   double* applyingDataBytes,
-                                                   AsyncTrigger* releaseTxns) {
-	loop {
-		if (okToReleaseTxns(*targetMB, *applyingDataBytes)) {
-			break;
-		} else {
-			wait(releaseTxns->onTrigger());
-			wait(delay(0.0)); // Avoid all waiting txns are triggered at the same time and all decide to proceed before
-			                  // applyingDataBytes has a chance to update
-		}
-	}
-	return Void();
-}
-
-// Apply mutations in batchData->stagingKeys [begin, end).
-ACTOR static Future<Void> applyStagingKeysBatch(std::map<Key, StagingKey>::iterator begin,
-                                                std::map<Key, StagingKey>::iterator end,
-                                                Database cx,
-                                                UID applierID,
-                                                ApplierBatchData::Counters* cc,
-                                                double* appliedBytes,
-                                                double* applyingDataBytes,
-                                                double* targetMB,
-                                                AsyncTrigger* releaseTxnTrigger) {
-	if (SERVER_KNOBS->FASTRESTORE_NOT_WRITE_DB) {
-		TraceEvent("FastRestoreApplierPhaseApplyStagingKeysBatchSkipped", applierID).detail("Begin", begin->first);
-		ASSERT(!g_network->isSimulated());
-		return Void();
-	}
-	wait(shouldReleaseTransaction(targetMB, applyingDataBytes, releaseTxnTrigger));
-
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state int sets = 0;
-	state int clears = 0;
-	state Key endKey = begin->first;
-	state double txnSize = 0;
-	state double txnSizeUsed = 0; // txn size accounted in applyingDataBytes
-	TraceEvent(SevFRDebugInfo, "FastRestoreApplierPhaseApplyStagingKeysBatch", applierID).detail("Begin", begin->first);
-	loop {
-		try {
-			txnSize = 0;
-			txnSizeUsed = 0;
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::map<Key, StagingKey>::iterator iter = begin;
-			while (iter != end) {
-				if (iter->second.type == MutationRef::SetValue) {
-					tr->set(iter->second.key, iter->second.val);
-					txnSize += iter->second.totalSize();
-					cc->appliedMutations += 1;
-					TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseApplyStagingKeysBatch", applierID)
-					    .detail("SetKey", iter->second.key);
-					sets++;
-				} else if (iter->second.type == MutationRef::ClearRange) {
-					if (iter->second.key != iter->second.val) {
-						TraceEvent(SevError, "FastRestoreApplierPhaseApplyStagingKeysBatchClearTooMuchData", applierID)
-						    .detail("KeyBegin", iter->second.key)
-						    .detail("KeyEnd", iter->second.val)
-						    .detail("Version", iter->second.version.version)
-						    .detail("SubVersion", iter->second.version.sub);
-					}
-					tr->clear(singleKeyRange(iter->second.key));
-					txnSize += iter->second.totalSize();
-					cc->appliedMutations += 1;
-					TraceEvent(SevFRMutationInfo, "FastRestoreApplierPhaseApplyStagingKeysBatch", applierID)
-					    .detail("ClearKey", iter->second.key);
-					clears++;
-				} else {
-					ASSERT(false);
-				}
-				endKey = iter != end ? iter->first : endKey;
-				iter++;
-				if (sets > 10000000 || clears > 10000000) {
-					TraceEvent(SevError, "FastRestoreApplierPhaseApplyStagingKeysBatchInfiniteLoop", applierID)
-					    .detail("Begin", begin->first)
-					    .detail("Sets", sets)
-					    .detail("Clears", clears);
-				}
-			}
-			TraceEvent(SevFRDebugInfo, "FastRestoreApplierPhaseApplyStagingKeysBatchPrecommit", applierID)
-			    .detail("Begin", begin->first)
-			    .detail("End", endKey)
-			    .detail("Sets", sets)
-			    .detail("Clears", clears);
-			tr->addWriteConflictRange(KeyRangeRef(begin->first, keyAfter(endKey))); // Reduce resolver load
-			txnSizeUsed = txnSize;
-			*applyingDataBytes += txnSizeUsed; // Must account for applying bytes before wait for write traffic control
-			wait(tr->commit());
-			cc->appliedTxns += 1;
-			cc->appliedBytes += txnSize;
-			*appliedBytes += txnSize;
-			*applyingDataBytes -= txnSizeUsed;
-			if (okToReleaseTxns(*targetMB, *applyingDataBytes)) {
-				releaseTxnTrigger->trigger();
-			}
-			break;
-		} catch (Error& e) {
-			cc->appliedTxnRetries += 1;
-			wait(tr->onError(e));
-			*applyingDataBytes -= txnSizeUsed;
-		}
-	}
-	return Void();
-}
-
-// Apply mutations in stagingKeys in batches in parallel
-ACTOR static Future<Void> applyStagingKeys(Reference<ApplierBatchData> batchData,
-                                           UID applierID,
-                                           int64_t batchIndex,
-                                           Database cx) {
-	std::map<Key, StagingKey>::iterator begin = batchData->stagingKeys.begin();
-	std::map<Key, StagingKey>::iterator cur = begin;
-	state int txnBatches = 0;
-	double txnSize = 0;
-	std::vector<Future<Void>> fBatches;
-	TraceEvent("FastRestoreApplerPhaseApplyStagingKeysStart", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("StagingKeys", batchData->stagingKeys.size());
-	batchData->totalBytesToWrite = 0;
-	while (cur != batchData->stagingKeys.end()) {
-		txnSize += cur->second.totalSize(); // should be consistent with receivedBytes accounting method
-		if (txnSize > SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES) {
-			fBatches.push_back(applyStagingKeysBatch(begin,
-			                                         cur,
-			                                         cx,
-			                                         applierID,
-			                                         &batchData->counters,
-			                                         &batchData->appliedBytes,
-			                                         &batchData->applyingDataBytes,
-			                                         &batchData->targetWriteRateMB,
-			                                         &batchData->releaseTxnTrigger));
-			batchData->totalBytesToWrite += txnSize;
-			begin = cur;
-			txnSize = 0;
-			txnBatches++;
-		}
-		cur++;
-	}
-	if (begin != batchData->stagingKeys.end()) {
-		fBatches.push_back(applyStagingKeysBatch(begin,
-		                                         cur,
-		                                         cx,
-		                                         applierID,
-		                                         &batchData->counters,
-		                                         &batchData->appliedBytes,
-		                                         &batchData->applyingDataBytes,
-		                                         &batchData->targetWriteRateMB,
-		                                         &batchData->releaseTxnTrigger));
-		batchData->totalBytesToWrite += txnSize;
-		txnBatches++;
-	}
-
-	wait(waitForAll(fBatches));
-
-	TraceEvent("FastRestoreApplerPhaseApplyStagingKeysDone", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("StagingKeys", batchData->stagingKeys.size())
-	    .detail("TransactionBatches", txnBatches)
-	    .detail("TotalBytesToWrite", batchData->totalBytesToWrite);
-	return Void();
-}
-
-// Write mutations to the destination DB
-ACTOR Future<Void> writeMutationsToDB(UID applierID,
-                                      int64_t batchIndex,
-                                      Reference<ApplierBatchData> batchData,
-                                      Database cx) {
-	TraceEvent("FastRestoreApplierPhaseApplyTxnStart", applierID).detail("BatchIndex", batchIndex);
-	wait(precomputeMutationsResult(batchData, applierID, batchIndex, cx));
-
-	wait(applyStagingKeys(batchData, applierID, batchIndex, cx));
-	TraceEvent("FastRestoreApplierPhaseApplyTxnDone", applierID)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("AppliedBytes", batchData->appliedBytes)
-	    .detail("ReceivedBytes", batchData->receivedBytes);
-
-	return Void();
-}
-
-void handleUpdateRateRequest(RestoreUpdateRateRequest req, Reference<RestoreApplierData> self) {
-	TraceEvent ev("FastRestoreApplierUpdateRateRequest", self->id());
-	ev.suppressFor(10)
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("FinishedBatch", self->finishedBatch.get())
-	    .detail("WriteMB", req.writeMB);
-	double remainingDataMB = 0;
-	if (self->finishedBatch.get() == req.batchIndex - 1) { // current applying batch
-		Reference<ApplierBatchData> batchData = self->batch[req.batchIndex];
-		ASSERT(batchData.isValid());
-		batchData->targetWriteRateMB = req.writeMB;
-		remainingDataMB = batchData->totalBytesToWrite > 0
-		                      ? std::max(0.0, batchData->totalBytesToWrite - batchData->appliedBytes) / 1024 / 1024
-		                      : batchData->receivedBytes / 1024 / 1024;
-		ev.detail("TotalBytesToWrite", batchData->totalBytesToWrite)
-		    .detail("AppliedBytes", batchData->appliedBytes)
-		    .detail("ReceivedBytes", batchData->receivedBytes)
-		    .detail("TargetWriteRateMB", batchData->targetWriteRateMB)
-		    .detail("RemainingDataMB", remainingDataMB);
-	}
-	req.reply.send(RestoreUpdateRateReply(self->id(), remainingDataMB));
-
-	return;
-}
-
-ACTOR static Future<Void> traceRate(const char* context,
-                                    Reference<ApplierBatchData> batchData,
-                                    int batchIndex,
-                                    UID nodeID,
-                                    NotifiedVersion* finishedVB,
-                                    bool once = false) {
-	loop {
-		if ((finishedVB->get() != batchIndex - 1) || !batchData.isValid()) {
-			break;
-		}
-		TraceEvent(context, nodeID)
-		    .suppressFor(10)
-		    .detail("BatchIndex", batchIndex)
-		    .detail("FinishedBatchIndex", finishedVB->get())
-		    .detail("TotalDataToWriteMB", batchData->totalBytesToWrite / 1024 / 1024)
-		    .detail("AppliedBytesMB", batchData->appliedBytes / 1024 / 1024)
-		    .detail("TargetBytesMB", batchData->targetWriteRateMB)
-		    .detail("InflightBytesMB", batchData->applyingDataBytes)
-		    .detail("ReceivedBytes", batchData->receivedBytes);
-		if (once) {
-			break;
-		}
-		wait(delay(5.0));
-	}
-
-	return Void();
-}
-
-ACTOR static Future<Void> handleApplyToDBRequest(RestoreVersionBatchRequest req,
-                                                 Reference<RestoreApplierData> self,
-                                                 Database cx) {
-	TraceEvent("FastRestoreApplierPhaseHandleApplyToDBStart", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("FinishedBatch", self->finishedBatch.get());
-
-	// Ensure batch (i-1) is applied before batch i
-	// TODO: Add a counter to warn when too many requests are waiting on the actor
-	wait(self->finishedBatch.whenAtLeast(req.batchIndex - 1));
-
-	state bool isDuplicated = true;
-	if (self->finishedBatch.get() == req.batchIndex - 1) {
-		// duplicate request from earlier version batch will be ignored
-		state Reference<ApplierBatchData> batchData = self->batch[req.batchIndex];
-		ASSERT(batchData.isValid());
-		TraceEvent("FastRestoreApplierPhaseHandleApplyToDBRunning", self->id())
-		    .detail("BatchIndex", req.batchIndex)
-		    .detail("FinishedBatch", self->finishedBatch.get())
-		    .detail("HasStarted", batchData->dbApplier.present())
-		    .detail("WroteToDBDone", batchData->dbApplier.present() ? batchData->dbApplier.get().isReady() : 0)
-		    .detail("PreviousVersionBatchState", batchData->vbState.get());
-
-		ASSERT(batchData.isValid());
-		if (!batchData->dbApplier.present()) {
-			isDuplicated = false;
-			batchData->dbApplier = Never();
-			batchData->dbApplier = writeMutationsToDB(self->id(), req.batchIndex, batchData, cx);
-			batchData->vbState = ApplierVersionBatchState::WRITE_TO_DB;
-			batchData->rateTracer = traceRate("FastRestoreApplierTransactionRateControl",
-			                                  batchData,
-			                                  req.batchIndex,
-			                                  self->id(),
-			                                  &self->finishedBatch);
-		}
-
-		ASSERT(batchData->dbApplier.present());
-		ASSERT(!batchData->dbApplier.get().isError()); // writeMutationsToDB actor cannot have error.
-		                                               // We cannot blindly retry because it is not idempotent
-
-		wait(batchData->dbApplier.get());
-
-		// Multiple actors can wait on req.batchIndex-1;
-		// Avoid setting finishedBatch when finishedBatch > req.batchIndex
-		if (self->finishedBatch.get() == req.batchIndex - 1) {
-			batchData->rateTracer = traceRate("FastRestoreApplierTransactionRateControlDone",
-			                                  batchData,
-			                                  req.batchIndex,
-			                                  self->id(),
-			                                  &self->finishedBatch,
-			                                  true /*print once*/); // Track the last rate info
-			self->finishedBatch.set(req.batchIndex);
-			// self->batch[req.batchIndex]->vbState = ApplierVersionBatchState::DONE;
-			// Free memory for the version batch
-			self->batch.erase(req.batchIndex);
-			if (self->delayedActors > 0) {
-				self->checkMemory.trigger();
-			}
-		}
-	}
-
-	req.reply.send(RestoreCommonReply(self->id(), isDuplicated));
-
-	TraceEvent("FastRestoreApplierPhaseHandleApplyToDBDone", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("FinishedBatch", self->finishedBatch.get())
-	    .detail("IsDuplicated", isDuplicated);
-
-	return Void();
-}
-
-// Copy from WriteDuringRead.actor.cpp with small modifications
-// Not all AtomicOps are handled in this function: SetVersionstampedKey, SetVersionstampedValue, and CompareAndClear
-Value applyAtomicOp(Optional<StringRef> existingValue, Value value, MutationRef::Type type) {
-	Arena arena;
-	if (type == MutationRef::AddValue)
-		return doLittleEndianAdd(existingValue, value, arena);
-	else if (type == MutationRef::AppendIfFits)
-		return doAppendIfFits(existingValue, value, arena);
-	else if (type == MutationRef::And || type == MutationRef::AndV2)
-		return doAndV2(existingValue, value, arena);
-	else if (type == MutationRef::Or)
-		return doOr(existingValue, value, arena);
-	else if (type == MutationRef::Xor)
-		return doXor(existingValue, value, arena);
-	else if (type == MutationRef::Max)
-		return doMax(existingValue, value, arena);
-	else if (type == MutationRef::Min || type == MutationRef::MinV2)
-		return doMinV2(existingValue, value, arena);
-	else if (type == MutationRef::ByteMin)
-		return doByteMin(existingValue, value, arena);
-	else if (type == MutationRef::ByteMax)
-		return doByteMax(existingValue, value, arena);
-	else {
-		TraceEvent(SevError, "ApplyAtomicOpUnhandledType")
-		    .detail("TypeCode", (int)type)
-		    .detail("TypeName", getTypeString(type));
-		ASSERT(false);
-	}
-	return Value();
-}
diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp
deleted file mode 100644
index 8a6a0e6a547..00000000000
--- a/fdbserver/RestoreCommon.actor.cpp
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * RestoreCommon.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file implements the functions defined in RestoreCommon.actor.h
-// The functions in this file are copied from BackupAgent
-
-#include "fdbserver/RestoreCommon.actor.h"
-
-// Backup agent header
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/SystemData.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// Split RestoreConfigFR defined in FileBackupAgent.actor.cpp to declaration in Restore.actor.h and implementation in
-// RestoreCommon.actor.cpp
-
-KeyBackedProperty<ERestoreState> RestoreConfigFR::stateEnum() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-Future<StringRef> RestoreConfigFR::stateText(Reference<ReadYourWritesTransaction> tr) {
-	return map(stateEnum().getD(tr), [](ERestoreState s) -> StringRef { return FileBackupAgent::restoreStateText(s); });
-}
-KeyBackedProperty<Key> RestoreConfigFR::addPrefix() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-KeyBackedProperty<Key> RestoreConfigFR::removePrefix() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
-KeyBackedProperty<KeyRange> RestoreConfigFR::restoreRange() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-KeyBackedProperty<std::vector<KeyRange>> RestoreConfigFR::restoreRanges() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-KeyBackedProperty<Key> RestoreConfigFR::batchFuture() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-KeyBackedProperty<Version> RestoreConfigFR::restoreVersion() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-
-KeyBackedProperty<Reference<IBackupContainer>> RestoreConfigFR::sourceContainer() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-// Get the source container as a bare URL, without creating a container instance
-KeyBackedProperty<Value> RestoreConfigFR::sourceContainerURL() {
-	return configSpace.pack("sourceContainer"_sr);
-}
-
-// Total bytes written by all log and range restore tasks.
-KeyBackedBinaryValue<int64_t> RestoreConfigFR::bytesWritten() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-// File blocks that have had tasks created for them by the Dispatch task
-KeyBackedBinaryValue<int64_t> RestoreConfigFR::filesBlocksDispatched() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-// File blocks whose tasks have finished
-KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileBlocksFinished() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-// Total number of files in the fileMap
-KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileCount() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-// Total number of file blocks in the fileMap
-KeyBackedBinaryValue<int64_t> RestoreConfigFR::fileBlockCount() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-
-Future<std::vector<KeyRange>> RestoreConfigFR::getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr) {
-	return getRestoreRangesOrDefault_impl(this, tr);
-}
-
-ACTOR Future<std::vector<KeyRange>> RestoreConfigFR::getRestoreRangesOrDefault_impl(
-    RestoreConfigFR* self,
-    Reference<ReadYourWritesTransaction> tr) {
-	state std::vector<KeyRange> ranges = wait(self->restoreRanges().getD(tr));
-	if (ranges.empty()) {
-		state KeyRange range = wait(self->restoreRange().getD(tr));
-		ranges.push_back(range);
-	}
-	return ranges;
-}
-
-KeyBackedSet<RestoreConfigFR::RestoreFile> RestoreConfigFR::fileSet() {
-	return configSpace.pack(__FUNCTION__sr);
-}
-
-Future<bool> RestoreConfigFR::isRunnable(Reference<ReadYourWritesTransaction> tr) {
-	return map(stateEnum().getD(tr), [](ERestoreState s) -> bool {
-		return s != ERestoreState::ABORTED && s != ERestoreState::COMPLETED && s != ERestoreState::UNINITIALIZED;
-	});
-}
-
-Future<Void> RestoreConfigFR::logError(Database cx, Error e, std::string const& details, void* taskInstance) {
-	if (!uid.isValid()) {
-		TraceEvent(SevError, "FileRestoreErrorNoUID").error(e).detail("Description", details);
-		return Void();
-	}
-	TraceEvent t(SevWarn, "FileRestoreError");
-	t.error(e).detail("RestoreUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
-	// key_not_found could happen
-	if (e.code() == error_code_key_not_found)
-		t.backtrace();
-
-	return updateErrorInfo(cx, e, details);
-}
-
-Key RestoreConfigFR::mutationLogPrefix() {
-	return uidPrefixKey(applyLogKeys.begin, uid);
-}
-
-Key RestoreConfigFR::applyMutationsMapPrefix() {
-	return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
-}
-
-ACTOR Future<int64_t> RestoreConfigFR::getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid) {
-	// Both of these are snapshot reads
-	state Future<Optional<Value>> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::True);
-	state Future<Optional<Value>> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::True);
-	wait(success(beginVal) && success(endVal));
-
-	if (!beginVal.get().present() || !endVal.get().present())
-		return 0;
-
-	Version beginVersion = BinaryReader::fromStringRef<Version>(beginVal.get().get(), Unversioned());
-	Version endVersion = BinaryReader::fromStringRef<Version>(endVal.get().get(), Unversioned());
-	return endVersion - beginVersion;
-}
-
-Future<int64_t> RestoreConfigFR::getApplyVersionLag(Reference<ReadYourWritesTransaction> tr) {
-	return getApplyVersionLag_impl(tr, uid);
-}
-
-void RestoreConfigFR::initApplyMutations(Reference<ReadYourWritesTransaction> tr, Key addPrefix, Key removePrefix) {
-	// Set these because they have to match the applyMutations values.
-	this->addPrefix().set(tr, addPrefix);
-	this->removePrefix().set(tr, removePrefix);
-
-	clearApplyMutationsKeys(tr);
-
-	// Initialize add/remove prefix, range version map count and set the map's start key to InvalidVersion
-	tr->set(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid), addPrefix);
-	tr->set(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid), removePrefix);
-	int64_t startCount = 0;
-	tr->set(uidPrefixKey(applyMutationsKeyVersionCountRange.begin, uid), StringRef((uint8_t*)&startCount, 8));
-	Key mapStart = uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
-	tr->set(mapStart, BinaryWriter::toValue<Version>(invalidVersion, Unversioned()));
-}
-
-void RestoreConfigFR::clearApplyMutationsKeys(Reference<ReadYourWritesTransaction> tr) {
-	tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
-
-	// Clear add/remove prefix keys
-	tr->clear(uidPrefixKey(applyMutationsAddPrefixRange.begin, uid));
-	tr->clear(uidPrefixKey(applyMutationsRemovePrefixRange.begin, uid));
-
-	// Clear range version map and count key
-	tr->clear(uidPrefixKey(applyMutationsKeyVersionCountRange.begin, uid));
-	Key mapStart = uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid);
-	tr->clear(KeyRangeRef(mapStart, strinc(mapStart)));
-
-	// Clear any loaded mutations that have not yet been applied
-	Key mutationPrefix = mutationLogPrefix();
-	tr->clear(KeyRangeRef(mutationPrefix, strinc(mutationPrefix)));
-
-	// Clear end and begin versions (intentionally in this order)
-	tr->clear(uidPrefixKey(applyMutationsEndRange.begin, uid));
-	tr->clear(uidPrefixKey(applyMutationsBeginRange.begin, uid));
-}
-
-void RestoreConfigFR::setApplyBeginVersion(Reference<ReadYourWritesTransaction> tr, Version ver) {
-	tr->set(uidPrefixKey(applyMutationsBeginRange.begin, uid), BinaryWriter::toValue(ver, Unversioned()));
-}
-
-void RestoreConfigFR::setApplyEndVersion(Reference<ReadYourWritesTransaction> tr, Version ver) {
-	tr->set(uidPrefixKey(applyMutationsEndRange.begin, uid), BinaryWriter::toValue(ver, Unversioned()));
-}
-
-Future<Version> RestoreConfigFR::getApplyEndVersion(Reference<ReadYourWritesTransaction> tr) {
-	return map(tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid)), [=](Optional<Value> const& value) -> Version {
-		return value.present() ? BinaryReader::fromStringRef<Version>(value.get(), Unversioned()) : 0;
-	});
-}
-
-// Meng: Change RestoreConfigFR to Reference<RestoreConfigFR> because FastRestore pass the Reference<RestoreConfigFR>
-// around
-ACTOR Future<std::string> RestoreConfigFR::getProgress_impl(Reference<RestoreConfigFR> restore,
-                                                            Reference<ReadYourWritesTransaction> tr) {
-	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-
-	state Future<int64_t> fileCount = restore->fileCount().getD(tr);
-	state Future<int64_t> fileBlockCount = restore->fileBlockCount().getD(tr);
-	state Future<int64_t> fileBlocksDispatched = restore->filesBlocksDispatched().getD(tr);
-	state Future<int64_t> fileBlocksFinished = restore->fileBlocksFinished().getD(tr);
-	state Future<int64_t> bytesWritten = restore->bytesWritten().getD(tr);
-	state Future<StringRef> status = restore->stateText(tr);
-	state Future<Version> lag = restore->getApplyVersionLag(tr);
-	state Future<std::string> tag = restore->tag().getD(tr);
-	state Future<std::pair<std::string, Version>> lastError = restore->lastError().getD(tr);
-
-	// restore might no longer be valid after the first wait so make sure it is not needed anymore.
-	state UID uid = restore->getUid();
-	wait(success(fileCount) && success(fileBlockCount) && success(fileBlocksDispatched) &&
-	     success(fileBlocksFinished) && success(bytesWritten) && success(status) && success(lag) && success(tag) &&
-	     success(lastError));
-
-	std::string errstr = "None";
-	if (lastError.get().second != 0)
-		errstr = format("'%s' %llds ago.\n",
-		                lastError.get().first.c_str(),
-		                (tr->getReadVersion().get() - lastError.get().second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND);
-
-	TraceEvent("FileRestoreProgress")
-	    .detail("RestoreUID", uid)
-	    .detail("Tag", tag.get())
-	    .detail("State", status.get().toString())
-	    .detail("FileCount", fileCount.get())
-	    .detail("FileBlocksFinished", fileBlocksFinished.get())
-	    .detail("FileBlocksTotal", fileBlockCount.get())
-	    .detail("FileBlocksInProgress", fileBlocksDispatched.get() - fileBlocksFinished.get())
-	    .detail("BytesWritten", bytesWritten.get())
-	    .detail("ApplyLag", lag.get())
-	    .detail("TaskInstance", THIS_ADDR)
-	    .backtrace();
-
-	return format("Tag: %s  UID: %s  State: %s  Blocks: %lld/%lld  BlocksInProgress: %lld  Files: %lld  BytesWritten: "
-	              "%lld  ApplyVersionLag: %lld  LastError: %s",
-	              tag.get().c_str(),
-	              uid.toString().c_str(),
-	              status.get().toString().c_str(),
-	              fileBlocksFinished.get(),
-	              fileBlockCount.get(),
-	              fileBlocksDispatched.get() - fileBlocksFinished.get(),
-	              fileCount.get(),
-	              bytesWritten.get(),
-	              lag.get(),
-	              errstr.c_str());
-}
-Future<std::string> RestoreConfigFR::getProgress(Reference<ReadYourWritesTransaction> tr) {
-	Reference<RestoreConfigFR> restore = Reference<RestoreConfigFR>(this);
-	return getProgress_impl(restore, tr);
-}
-
-// Meng: Change RestoreConfigFR to Reference<RestoreConfigFR>
-ACTOR Future<std::string> RestoreConfigFR::getFullStatus_impl(Reference<RestoreConfigFR> restore,
-                                                              Reference<ReadYourWritesTransaction> tr) {
-	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-
-	state Future<std::vector<KeyRange>> ranges = restore->getRestoreRangesOrDefault(tr);
-	state Future<Key> addPrefix = restore->addPrefix().getD(tr);
-	state Future<Key> removePrefix = restore->removePrefix().getD(tr);
-	state Future<Key> url = restore->sourceContainerURL().getD(tr);
-	state Future<Version> restoreVersion = restore->restoreVersion().getD(tr);
-	state Future<std::string> progress = restore->getProgress(tr);
-
-	// restore might no longer be valid after the first wait so make sure it is not needed anymore.
-	wait(success(ranges) && success(addPrefix) && success(removePrefix) && success(url) && success(restoreVersion) &&
-	     success(progress));
-
-	std::string returnStr;
-	returnStr = format("%s  URL: %s", progress.get().c_str(), url.get().toString().c_str());
-	for (auto& range : ranges.get()) {
-		returnStr += format("  Range: '%s'-'%s'", printable(range.begin).c_str(), printable(range.end).c_str());
-	}
-	returnStr += format("  AddPrefix: '%s'  RemovePrefix: '%s'  Version: %lld",
-	                    printable(addPrefix.get()).c_str(),
-	                    printable(removePrefix.get()).c_str(),
-	                    restoreVersion.get());
-	return returnStr;
-}
-Future<std::string> RestoreConfigFR::getFullStatus(Reference<ReadYourWritesTransaction> tr) {
-	Reference<RestoreConfigFR> restore = Reference<RestoreConfigFR>(this);
-	return getFullStatus_impl(restore, tr);
-}
-
-std::string RestoreConfigFR::toString() {
-	std::stringstream ss;
-	ss << "uid:" << uid.toString() << " prefix:" << subspace.key().contents().toString();
-	return ss.str();
-}
-
-// parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfigFR is copied
-// The implementation of parallelFileRestore is copied from FileBackupAgent.actor.cpp
-// parallelFileRestore is copied from FileBackupAgent.actor.cpp for the same reason as RestoreConfigFR is copied
-namespace parallelFileRestore {
-
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file,
-                                                                    int64_t offset,
-                                                                    int len) {
-	state Standalone<StringRef> buf = makeString(len);
-	int rLen = wait(file->read(mutateString(buf), len, offset));
-	if (rLen != len)
-		throw restore_bad_read();
-
-	simulateBlobFailure();
-
-	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state StringRefReader reader(buf, restore_corrupted_data());
-
-	try {
-		// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
-		if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
-			throw restore_unsupported_file_version();
-
-		// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
-		while (1) {
-			// If eof reached or first key len bytes is 0xFF then end of block was reached.
-			if (reader.eof() || *reader.rptr == 0xFF)
-				break;
-
-			// Read key and value.  If anything throws then there is a problem.
-			uint32_t kLen = reader.consumeNetworkUInt32();
-			const uint8_t* k = reader.consume(kLen);
-			uint32_t vLen = reader.consumeNetworkUInt32();
-			const uint8_t* v = reader.consume(vLen);
-
-			results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
-		}
-
-		// Make sure any remaining bytes in the block are 0xFF
-		for (auto b : reader.remainder())
-			if (b != 0xFF)
-				throw restore_corrupted_data_padding();
-
-		return results;
-
-	} catch (Error& e) {
-		TraceEvent(SevError, "FileRestoreCorruptLogFileBlock")
-		    .error(e)
-		    .detail("Filename", file->getFilename())
-		    .detail("BlockOffset", offset)
-		    .detail("BlockLen", len)
-		    .detail("ErrorRelativeOffset", reader.rptr - buf.begin())
-		    .detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
-		throw;
-	}
-}
-
-} // namespace parallelFileRestore
diff --git a/fdbserver/RestoreController.actor.cpp b/fdbserver/RestoreController.actor.cpp
deleted file mode 100644
index 08af0204543..00000000000
--- a/fdbserver/RestoreController.actor.cpp
+++ /dev/null
@@ -1,1222 +0,0 @@
-/*
- * RestoreController.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file implements the functions for RestoreController role
-
-#include "fdbrpc/RangeMap.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/SystemData.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreCommon.actor.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreController.actor.h"
-#include "fdbserver/RestoreApplier.actor.h"
-#include "fdbserver/RestoreLoader.actor.h"
-
-#include "flow/Platform.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// TODO: Support [[maybe_unused]] attribute for actors
-// ACTOR static Future<Void> clearDB(Database cx);
-ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
-                                                std::vector<RestoreFileFR>* rangeFiles,
-                                                std::vector<RestoreFileFR>* logFiles,
-                                                Version* minRangeVersion,
-                                                Database cx,
-                                                RestoreRequest request);
-ACTOR static Future<Void> buildRangeVersions(KeyRangeMap<Version>* pRangeVersions,
-                                             std::vector<RestoreFileFR>* pRangeFiles,
-                                             Key url,
-                                             Optional<std::string> proxy,
-                                             Database cx);
-
-ACTOR static Future<Version> processRestoreRequest(Reference<RestoreControllerData> self,
-                                                   Database cx,
-                                                   RestoreRequest request);
-ACTOR static Future<Void> startProcessRestoreRequests(Reference<RestoreControllerData> self, Database cx);
-ACTOR static Future<Void> distributeWorkloadPerVersionBatch(Reference<RestoreControllerData> self,
-                                                            int batchIndex,
-                                                            Database cx,
-                                                            RestoreRequest request,
-                                                            VersionBatch versionBatch);
-
-ACTOR static Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> controllerWorker,
-                                              Reference<RestoreControllerData> controllerData);
-ACTOR static Future<Void> distributeRestoreSysInfo(Reference<RestoreControllerData> controllerData,
-                                                   KeyRangeMap<Version>* pRangeVersions);
-
-ACTOR static Future<std::vector<RestoreRequest>> collectRestoreRequests(Database cx);
-ACTOR static Future<Void> initializeVersionBatch(std::map<UID, RestoreApplierInterface> appliersInterf,
-                                                 std::map<UID, RestoreLoaderInterface> loadersInterf,
-                                                 int batchIndex);
-ACTOR static Future<Void> notifyApplierToApplyMutations(Reference<ControllerBatchData> batchData,
-                                                        Reference<ControllerBatchStatus> batchStatus,
-                                                        std::map<UID, RestoreApplierInterface> appliersInterf,
-                                                        int batchIndex,
-                                                        NotifiedVersion* finishedBatch);
-ACTOR static Future<Void> notifyLoadersVersionBatchFinished(std::map<UID, RestoreLoaderInterface> loadersInterf,
-                                                            int batchIndex);
-ACTOR static Future<Void> notifyRestoreCompleted(Reference<RestoreControllerData> self, bool terminate);
-ACTOR static Future<Void> signalRestoreCompleted(Reference<RestoreControllerData> self, Database cx);
-// TODO: Support [[maybe_unused]] attribute for actors
-// ACTOR static Future<Void> updateHeartbeatTime(Reference<RestoreControllerData> self);
-ACTOR static Future<Void> checkRolesLiveness(Reference<RestoreControllerData> self);
-
-void splitKeyRangeForAppliers(Reference<ControllerBatchData> batchData,
-                              std::map<UID, RestoreApplierInterface> appliersInterf,
-                              int batchIndex);
-
-ACTOR Future<Void> sampleBackups(Reference<RestoreControllerData> self, RestoreControllerInterface ci) {
-	loop {
-		try {
-			RestoreSamplesRequest req = waitNext(ci.samples.getFuture());
-			TraceEvent(SevDebug, "FastRestoreControllerSampleBackups")
-			    .detail("SampleID", req.id)
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("Samples", req.samples.size());
-			ASSERT(req.batchIndex <= self->batch.size()); // batchIndex starts from 1
-
-			Reference<ControllerBatchData> batch = self->batch[req.batchIndex];
-			ASSERT(batch.isValid());
-			if (batch->sampleMsgs.find(req.id) != batch->sampleMsgs.end()) {
-				req.reply.send(RestoreCommonReply(req.id));
-				continue;
-			}
-			batch->sampleMsgs.insert(req.id);
-			for (auto& m : req.samples) {
-				batch->samples.addMetric(m.key, m.size);
-				batch->samplesSize += m.size;
-			}
-			req.reply.send(RestoreCommonReply(req.id));
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "FastRestoreControllerSampleBackupsError", self->id()).error(e);
-			break;
-		}
-	}
-
-	return Void();
-}
-
-ACTOR Future<Void> startRestoreController(Reference<RestoreWorkerData> controllerWorker, Database cx) {
-	ASSERT(controllerWorker.isValid());
-	ASSERT(controllerWorker->controllerInterf.present());
-	state Reference<RestoreControllerData> self =
-	    makeReference<RestoreControllerData>(controllerWorker->controllerInterf.get().id());
-	state Future<Void> error = actorCollection(self->addActor.getFuture());
-
-	try {
-		// recruitRestoreRoles must come after controllerWorker has finished collectWorkerInterface
-		wait(recruitRestoreRoles(controllerWorker, self));
-
-		// self->addActor.send(updateHeartbeatTime(self));
-		self->addActor.send(checkRolesLiveness(self));
-		self->addActor.send(updateProcessMetrics(self));
-		self->addActor.send(traceProcessMetrics(self, "RestoreController"));
-		self->addActor.send(sampleBackups(self, controllerWorker->controllerInterf.get()));
-
-		wait(startProcessRestoreRequests(self, cx) || error);
-	} catch (Error& e) {
-		if (e.code() != error_code_operation_cancelled) {
-			TraceEvent(SevError, "FastRestoreControllerStart").error(e).detail("Reason", "Unexpected unhandled error");
-		}
-	}
-
-	return Void();
-}
-
-// RestoreWorker that has restore controller role: Recruite a role for each worker
-ACTOR Future<Void> recruitRestoreRoles(Reference<RestoreWorkerData> controllerWorker,
-                                       Reference<RestoreControllerData> controllerData) {
-	state int nodeIndex = 0;
-	state RestoreRole role = RestoreRole::Invalid;
-
-	TraceEvent("FastRestoreController", controllerData->id())
-	    .detail("RecruitRestoreRoles", controllerWorker->workerInterfaces.size())
-	    .detail("NumLoaders", SERVER_KNOBS->FASTRESTORE_NUM_LOADERS)
-	    .detail("NumAppliers", SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS);
-	ASSERT(controllerData->loadersInterf.empty() && controllerData->appliersInterf.empty());
-	ASSERT(controllerWorker->controllerInterf.present());
-
-	ASSERT(controllerData.isValid());
-	ASSERT(SERVER_KNOBS->FASTRESTORE_NUM_LOADERS > 0 && SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS > 0);
-	// We assign 1 role per worker for now
-	ASSERT(SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS <=
-	       controllerWorker->workerInterfaces.size());
-
-	// Assign a role to each worker
-	std::vector<std::pair<UID, RestoreRecruitRoleRequest>> requests;
-	for (auto& workerInterf : controllerWorker->workerInterfaces) {
-		if (nodeIndex >= 0 && nodeIndex < SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS) {
-			// [0, numApplier) are appliers
-			role = RestoreRole::Applier;
-		} else if (nodeIndex >= SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS &&
-		           nodeIndex < SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS) {
-			// [numApplier, numApplier + numLoader) are loaders
-			role = RestoreRole::Loader;
-		} else {
-			break;
-		}
-
-		TraceEvent("FastRestoreController", controllerData->id())
-		    .detail("WorkerNode", workerInterf.first)
-		    .detail("NodeRole", role)
-		    .detail("NodeIndex", nodeIndex);
-		requests.emplace_back(workerInterf.first,
-		                      RestoreRecruitRoleRequest(controllerWorker->controllerInterf.get(), role, nodeIndex));
-		nodeIndex++;
-	}
-
-	state std::vector<RestoreRecruitRoleReply> replies;
-	wait(getBatchReplies(&RestoreWorkerInterface::recruitRole, controllerWorker->workerInterfaces, requests, &replies));
-	for (auto& reply : replies) {
-		if (reply.role == RestoreRole::Applier) {
-			ASSERT_WE_THINK(reply.applier.present());
-			controllerData->appliersInterf[reply.applier.get().id()] = reply.applier.get();
-		} else if (reply.role == RestoreRole::Loader) {
-			ASSERT_WE_THINK(reply.loader.present());
-			controllerData->loadersInterf[reply.loader.get().id()] = reply.loader.get();
-		} else {
-			TraceEvent(SevError, "FastRestoreController").detail("RecruitRestoreRolesInvalidRole", reply.role);
-		}
-	}
-	controllerData->recruitedRoles.send(Void());
-	TraceEvent("FastRestoreRecruitRestoreRolesDone", controllerData->id())
-	    .detail("Workers", controllerWorker->workerInterfaces.size())
-	    .detail("RecruitedRoles", replies.size());
-
-	return Void();
-}
-
-ACTOR Future<Void> distributeRestoreSysInfo(Reference<RestoreControllerData> controllerData,
-                                            KeyRangeMap<Version>* pRangeVersions) {
-	ASSERT(controllerData.isValid());
-	ASSERT(!controllerData->loadersInterf.empty());
-	RestoreSysInfo sysInfo(controllerData->appliersInterf);
-	// Construct serializable KeyRange versions
-	Standalone<VectorRef<std::pair<KeyRangeRef, Version>>> rangeVersionsVec;
-	auto ranges = pRangeVersions->ranges();
-	int i = 0;
-	for (auto r = ranges.begin(); r != ranges.end(); ++r) {
-		rangeVersionsVec.push_back(rangeVersionsVec.arena(),
-		                           std::make_pair(KeyRangeRef(r->begin(), r->end()), r->value()));
-		TraceEvent("DistributeRangeVersions")
-		    .detail("RangeIndex", i++)
-		    .detail("RangeBegin", r->begin())
-		    .detail("RangeEnd", r->end())
-		    .detail("RangeVersion", r->value());
-	}
-	std::vector<std::pair<UID, RestoreSysInfoRequest>> requests;
-	for (auto& loader : controllerData->loadersInterf) {
-		requests.emplace_back(loader.first, RestoreSysInfoRequest(sysInfo, rangeVersionsVec));
-	}
-
-	TraceEvent("FastRestoreDistributeRestoreSysInfoToLoaders", controllerData->id())
-	    .detail("Loaders", controllerData->loadersInterf.size());
-	wait(sendBatchRequests(&RestoreLoaderInterface::updateRestoreSysInfo, controllerData->loadersInterf, requests));
-	TraceEvent("FastRestoreDistributeRestoreSysInfoToLoadersDone", controllerData->id())
-	    .detail("Loaders", controllerData->loadersInterf.size());
-
-	return Void();
-}
-
-// The server of the restore controller. It drives the restore progress with the following steps:
-// 1) Lock database and clear the normal keyspace
-// 2) Wait on each RestoreRequest, which is sent by RestoreTool operated by DBA
-// 3) Process each restore request in actor processRestoreRequest;
-// 3.1) Sample workload to decide the key range for each applier, which is implemented as a dummy sampling;
-// 3.2) Send each loader the map of key-range to applier interface;
-// 3.3) Construct requests of which file should be loaded by which loader, and send requests to loaders;
-// 4) After process all restore requests, finish restore by cleaning up the restore related system key
-//    and ask all restore roles to quit.
-ACTOR Future<Void> startProcessRestoreRequests(Reference<RestoreControllerData> self, Database cx) {
-	state std::vector<RestoreRequest> restoreRequests = wait(collectRestoreRequests(cx));
-	state int restoreIndex = 0;
-
-	TraceEvent("FastRestoreControllerWaitOnRestoreRequests", self->id())
-	    .detail("RestoreRequests", restoreRequests.size());
-
-	// TODO: Sanity check restoreRequests' key ranges do not overlap
-
-	// Step: Perform the restore requests
-	try {
-		for (restoreIndex = 0; restoreIndex < restoreRequests.size(); restoreIndex++) {
-			state RestoreRequest request = restoreRequests[restoreIndex];
-			state KeyRange range = request.range.removePrefix(request.removePrefix).withPrefix(request.addPrefix);
-			TraceEvent("FastRestoreControllerProcessRestoreRequests", self->id())
-			    .detail("RestoreRequestInfo", request.toString())
-			    .detail("TransformedKeyRange", range);
-			// TODO: Initialize controllerData and all loaders and appliers' data for each restore request!
-			self->resetPerRestoreRequest();
-
-			// clear the key range that will be restored
-			wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-				tr->clear(range);
-				return Void();
-			}));
-
-			wait(success(processRestoreRequest(self, cx, request)));
-			wait(notifyRestoreCompleted(self, false));
-		}
-	} catch (Error& e) {
-		if (restoreIndex < restoreRequests.size()) {
-			TraceEvent(SevError, "FastRestoreControllerProcessRestoreRequestsFailed", self->id())
-			    .error(e)
-			    .detail("RestoreRequest", restoreRequests[restoreIndex].toString());
-		} else {
-			TraceEvent(SevError, "FastRestoreControllerProcessRestoreRequestsFailed", self->id())
-			    .error(e)
-			    .detail("RestoreRequests", restoreRequests.size())
-			    .detail("RestoreIndex", restoreIndex);
-		}
-	}
-
-	// Step: Notify all restore requests have been handled by cleaning up the restore keys
-	wait(signalRestoreCompleted(self, cx));
-
-	TraceEvent("FastRestoreControllerRestoreCompleted", self->id());
-
-	return Void();
-}
-
-ACTOR static Future<Void> monitorFinishedVersion(Reference<RestoreControllerData> self, RestoreRequest request) {
-	loop {
-		TraceEvent("FastRestoreMonitorFinishedVersion", self->id())
-		    .detail("RestoreRequest", request.toString())
-		    .detail("BatchIndex", self->finishedBatch.get());
-		wait(delay(SERVER_KNOBS->FASTRESTORE_VB_MONITOR_DELAY));
-	}
-}
-
-ACTOR static Future<Version> processRestoreRequest(Reference<RestoreControllerData> self,
-                                                   Database cx,
-                                                   RestoreRequest request) {
-	state std::vector<RestoreFileFR> rangeFiles;
-	state std::vector<RestoreFileFR> logFiles;
-	state std::vector<RestoreFileFR> allFiles;
-	state Version minRangeVersion = MAX_VERSION;
-
-	self->initBackupContainer(request.url, request.proxy);
-
-	// Get all backup files' description and save them to files
-	state Version targetVersion =
-	    wait(collectBackupFiles(self->bc, &rangeFiles, &logFiles, &minRangeVersion, cx, request));
-	ASSERT(targetVersion > 0);
-	ASSERT(minRangeVersion != MAX_VERSION); // otherwise, all mutations will be skipped
-
-	std::sort(rangeFiles.begin(), rangeFiles.end());
-	std::sort(logFiles.begin(), logFiles.end(), [](RestoreFileFR const& f1, RestoreFileFR const& f2) -> bool {
-		return std::tie(f1.endVersion, f1.beginVersion, f1.fileIndex, f1.fileName) <
-		       std::tie(f2.endVersion, f2.beginVersion, f2.fileIndex, f2.fileName);
-	});
-
-	// Build range versions: version of key ranges in range file
-	state KeyRangeMap<Version> rangeVersions(minRangeVersion, allKeys.end);
-	if (SERVER_KNOBS->FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE) {
-		wait(buildRangeVersions(&rangeVersions, &rangeFiles, request.url, request.proxy, cx));
-	} else {
-		// Debug purpose, dump range versions
-		auto ranges = rangeVersions.ranges();
-		int i = 0;
-		for (auto r = ranges.begin(); r != ranges.end(); ++r) {
-			TraceEvent(SevDebug, "SingleRangeVersion")
-			    .detail("RangeIndex", i++)
-			    .detail("RangeBegin", r->begin())
-			    .detail("RangeEnd", r->end())
-			    .detail("RangeVersion", r->value());
-		}
-	}
-
-	wait(distributeRestoreSysInfo(self, &rangeVersions));
-
-	// Divide files into version batches.
-	self->buildVersionBatches(rangeFiles, logFiles, &self->versionBatches, targetVersion);
-	self->dumpVersionBatches(self->versionBatches);
-
-	state std::vector<Future<Void>> fBatches;
-	state std::vector<VersionBatch> versionBatches; // To randomize invoking order of version batchs
-	for (auto& vb : self->versionBatches) {
-		versionBatches.push_back(vb.second);
-	}
-
-	// releaseVBOutOfOrder can only be true in simulation
-	state bool releaseVBOutOfOrder = g_network->isSimulated() ? deterministicRandom()->random01() < 0.5 : false;
-	ASSERT(g_network->isSimulated() || !releaseVBOutOfOrder);
-	if (releaseVBOutOfOrder) {
-		// Randomize invoking order of version batches
-		int permTimes = deterministicRandom()->randomInt(0, 100);
-		while (permTimes-- > 0) {
-			std::next_permutation(versionBatches.begin(), versionBatches.end());
-		}
-	}
-
-	self->addActor.send(monitorFinishedVersion(self, request));
-	state std::vector<VersionBatch>::iterator versionBatch = versionBatches.begin();
-	for (; versionBatch != versionBatches.end(); versionBatch++) {
-		while (self->runningVersionBatches.get() >= SERVER_KNOBS->FASTRESTORE_VB_PARALLELISM && !releaseVBOutOfOrder) {
-			// Control how many batches can be processed in parallel. Avoid dead lock due to OOM on loaders
-			TraceEvent("FastRestoreControllerDispatchVersionBatches")
-			    .detail("WaitOnRunningVersionBatches", self->runningVersionBatches.get());
-			wait(self->runningVersionBatches.onChange());
-		}
-		int batchIndex = versionBatch->batchIndex;
-		TraceEvent("FastRestoreControllerDispatchVersionBatches")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("BatchSize", versionBatch->size)
-		    .detail("RunningVersionBatches", self->runningVersionBatches.get())
-		    .detail("VersionBatches", versionBatches.size());
-		self->batch[batchIndex] = makeReference<ControllerBatchData>();
-		self->batchStatus[batchIndex] = makeReference<ControllerBatchStatus>();
-		fBatches.push_back(distributeWorkloadPerVersionBatch(self, batchIndex, cx, request, *versionBatch));
-		// Wait a bit to give the current version batch a head start from the next version batch
-		wait(delay(SERVER_KNOBS->FASTRESTORE_VB_LAUNCH_DELAY));
-	}
-
-	try {
-		wait(waitForAll(fBatches));
-	} catch (Error& e) {
-		TraceEvent(SevError, "FastRestoreControllerDispatchVersionBatchesUnexpectedError").error(e);
-	}
-
-	TraceEvent("FastRestoreController").detail("RestoreToVersion", request.targetVersion);
-	return request.targetVersion;
-}
-
-ACTOR static Future<Void> loadFilesOnLoaders(Reference<ControllerBatchData> batchData,
-                                             Reference<ControllerBatchStatus> batchStatus,
-                                             std::map<UID, RestoreLoaderInterface> loadersInterf,
-                                             int batchIndex,
-                                             Database cx,
-                                             RestoreRequest request,
-                                             VersionBatch versionBatch,
-                                             bool isRangeFile) {
-	// set is internally sorted
-	std::set<RestoreFileFR>* files = isRangeFile ? &versionBatch.rangeFiles : &versionBatch.logFiles;
-
-	TraceEvent("FastRestoreControllerPhaseLoadFilesStart")
-	    .detail("RestoreRequestID", request.randomUid)
-	    .detail("BatchIndex", batchIndex)
-	    .detail("FileTypeLoadedInVersionBatch", isRangeFile)
-	    .detail("BeginVersion", versionBatch.beginVersion)
-	    .detail("EndVersion", versionBatch.endVersion)
-	    .detail("Files", (files != nullptr ? files->size() : -1));
-
-	std::vector<std::pair<UID, RestoreLoadFileRequest>> requests;
-	std::map<UID, RestoreLoaderInterface>::iterator loader = loadersInterf.begin();
-	state std::vector<RestoreAsset> assets; // all assets loaded, used for sanity check restore progress
-
-	// Balance workload on loaders for parsing range and log files across version batches
-	int random = deterministicRandom()->randomInt(0, loadersInterf.size());
-	while (random-- > 0) {
-		loader++;
-	}
-
-	int paramIdx = 0;
-	for (auto& file : *files) {
-		// TODO: Allow empty files in version batch; Filter out them here.
-		if (loader == loadersInterf.end()) {
-			loader = loadersInterf.begin();
-		}
-		// Prepare loading
-		LoadingParam param;
-		param.url = request.url;
-		param.isRangeFile = file.isRange;
-		param.rangeVersion = file.isRange ? file.version : -1;
-		param.blockSize = file.blockSize;
-
-		param.asset.uid = deterministicRandom()->randomUniqueID();
-		param.asset.filename = file.fileName;
-		param.asset.fileIndex = file.fileIndex;
-		param.asset.partitionId = file.partitionId;
-		param.asset.offset = 0;
-		param.asset.len = file.fileSize;
-		param.asset.range = request.range;
-		param.asset.beginVersion = versionBatch.beginVersion;
-		param.asset.endVersion = (isRangeFile || request.targetVersion == -1)
-		                             ? versionBatch.endVersion
-		                             : std::min(versionBatch.endVersion, request.targetVersion + 1);
-		param.asset.addPrefix = request.addPrefix;
-		param.asset.removePrefix = request.removePrefix;
-		param.asset.batchIndex = batchIndex;
-
-		TraceEvent("FastRestoreControllerPhaseLoadFiles")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("LoadParamIndex", paramIdx)
-		    .detail("LoaderID", loader->first.toString())
-		    .detail("LoadParam", param.toString());
-		ASSERT_WE_THINK(param.asset.len > 0);
-		ASSERT_WE_THINK(param.asset.offset >= 0);
-		ASSERT_WE_THINK(param.asset.offset <= file.fileSize);
-		ASSERT_WE_THINK(param.asset.beginVersion <= param.asset.endVersion);
-
-		requests.emplace_back(loader->first, RestoreLoadFileRequest(batchIndex, param));
-		// Restore asset should only be loaded exactly once.
-		if (batchStatus->raStatus.find(param.asset) != batchStatus->raStatus.end()) {
-			TraceEvent(SevError, "FastRestoreControllerPhaseLoadFiles")
-			    .detail("LoadingParam", param.toString())
-			    .detail("RestoreAssetAlreadyProcessed", batchStatus->raStatus[param.asset]);
-		}
-		batchStatus->raStatus[param.asset] = RestoreAssetStatus::Loading;
-		assets.push_back(param.asset);
-		++loader;
-		++paramIdx;
-	}
-	TraceEvent(files->size() != paramIdx ? SevError : SevInfo, "FastRestoreControllerPhaseLoadFiles")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Files", files->size())
-	    .detail("LoadParams", paramIdx);
-
-	state std::vector<RestoreLoadFileReply> replies;
-	// Wait on the batch of load files or log files
-	wait(getBatchReplies(
-	    &RestoreLoaderInterface::loadFile, loadersInterf, requests, &replies, TaskPriority::RestoreLoaderLoadFiles));
-
-	TraceEvent("FastRestoreControllerPhaseLoadFilesReply")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("SamplingReplies", replies.size());
-	for (auto& reply : replies) {
-		// Update and sanity check restore asset's status
-		RestoreAssetStatus status = batchStatus->raStatus[reply.param.asset];
-		if (status == RestoreAssetStatus::Loading && !reply.isDuplicated) {
-			batchStatus->raStatus[reply.param.asset] = RestoreAssetStatus::Loaded;
-		} else if (status == RestoreAssetStatus::Loading && reply.isDuplicated) {
-			// Duplicate request wait on the restore asset to be processed before it replies
-			batchStatus->raStatus[reply.param.asset] = RestoreAssetStatus::Loaded;
-			TraceEvent(SevWarn, "FastRestoreControllerPhaseLoadFilesReply")
-			    .detail("RestoreAsset", reply.param.asset.toString())
-			    .detail("DuplicateRequestArriveEarly", "RestoreAsset should have been processed");
-		} else if (status == RestoreAssetStatus::Loaded && reply.isDuplicated) {
-			TraceEvent(SevDebug, "FastRestoreControllerPhaseLoadFilesReply")
-			    .detail("RestoreAsset", reply.param.asset.toString())
-			    .detail("RequestIgnored", "Loading request was sent more than once");
-		} else {
-			TraceEvent(SevError, "FastRestoreControllerPhaseLoadFilesReply")
-			    .detail("RestoreAsset", reply.param.asset.toString())
-			    .detail("UnexpectedReply", reply.toString());
-		}
-	}
-
-	// Sanity check: all restore assets status should be Loaded
-	for (auto& asset : assets) {
-		if (batchStatus->raStatus[asset] != RestoreAssetStatus::Loaded) {
-			TraceEvent(SevError, "FastRestoreControllerPhaseLoadFilesReply")
-			    .detail("RestoreAsset", asset.toString())
-			    .detail("UnexpectedStatus", batchStatus->raStatus[asset]);
-		}
-	}
-
-	TraceEvent("FastRestoreControllerPhaseLoadFilesDone")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("FileTypeLoadedInVersionBatch", isRangeFile)
-	    .detail("BeginVersion", versionBatch.beginVersion)
-	    .detail("EndVersion", versionBatch.endVersion);
-	return Void();
-}
-
-// Ask loaders to send its buffered mutations to appliers
-ACTOR static Future<Void> sendMutationsFromLoaders(Reference<ControllerBatchData> batchData,
-                                                   Reference<ControllerBatchStatus> batchStatus,
-                                                   std::map<UID, RestoreLoaderInterface> loadersInterf,
-                                                   int batchIndex,
-                                                   bool useRangeFile) {
-	TraceEvent("FastRestoreControllerPhaseSendMutationsFromLoadersStart")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("UseRangeFiles", useRangeFile)
-	    .detail("Loaders", loadersInterf.size());
-
-	std::vector<std::pair<UID, RestoreSendMutationsToAppliersRequest>> requests;
-	for (auto& loader : loadersInterf) {
-		requests.emplace_back(
-		    loader.first, RestoreSendMutationsToAppliersRequest(batchIndex, batchData->rangeToApplier, useRangeFile));
-		batchStatus->loadStatus[loader.first] =
-		    useRangeFile ? RestoreSendStatus::SendingRanges : RestoreSendStatus::SendingLogs;
-	}
-	state std::vector<RestoreCommonReply> replies;
-	wait(getBatchReplies(&RestoreLoaderInterface::sendMutations,
-	                     loadersInterf,
-	                     requests,
-	                     &replies,
-	                     TaskPriority::RestoreLoaderSendMutations));
-
-	TraceEvent("FastRestoreControllerPhaseSendMutationsFromLoadersDone")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("UseRangeFiles", useRangeFile)
-	    .detail("Loaders", loadersInterf.size());
-
-	return Void();
-}
-
-// Process a version batch. Phases (loading files, send mutations) should execute in order
-ACTOR static Future<Void> distributeWorkloadPerVersionBatch(Reference<RestoreControllerData> self,
-                                                            int batchIndex,
-                                                            Database cx,
-                                                            RestoreRequest request,
-                                                            VersionBatch versionBatch) {
-	state Reference<ControllerBatchData> batchData = self->batch[batchIndex];
-	state Reference<ControllerBatchStatus> batchStatus = self->batchStatus[batchIndex];
-	state double startTime = now();
-
-	TraceEvent("FastRestoreControllerDispatchVersionBatchesStart", self->id())
-	    .detail("BatchIndex", batchIndex)
-	    .detail("BatchSize", versionBatch.size)
-	    .detail("RunningVersionBatches", self->runningVersionBatches.get());
-
-	self->runningVersionBatches.set(self->runningVersionBatches.get() + 1);
-
-	// In case sampling data takes too much memory on controller
-	wait(isSchedulable(self, batchIndex, __FUNCTION__));
-
-	wait(initializeVersionBatch(self->appliersInterf, self->loadersInterf, batchIndex));
-
-	ASSERT(!versionBatch.isEmpty());
-	ASSERT(self->loadersInterf.size() > 0);
-	ASSERT(self->appliersInterf.size() > 0);
-
-	// Parse log files and send mutations to appliers before we parse range files
-	// TODO: Allow loading both range and log files in parallel
-	ASSERT(batchData->samples.empty());
-	ASSERT(batchData->samplesSize < 1 && batchData->samplesSize > -1); // samplesSize should be 0
-	ASSERT(batchStatus->raStatus.empty());
-	ASSERT(batchStatus->loadStatus.empty());
-	ASSERT(batchStatus->applyStatus.empty());
-
-	// New backup has subversion to order mutations at the same version. For mutations at the same version,
-	// range file's mutations have the largest subversion and larger than log file's.
-	// SOMEDAY: Extend subversion to old-style backup.
-	wait(
-	    loadFilesOnLoaders(batchData, batchStatus, self->loadersInterf, batchIndex, cx, request, versionBatch, false) &&
-	    loadFilesOnLoaders(batchData, batchStatus, self->loadersInterf, batchIndex, cx, request, versionBatch, true));
-
-	ASSERT(batchData->rangeToApplier.empty());
-	splitKeyRangeForAppliers(batchData, self->appliersInterf, batchIndex);
-
-	// Ask loaders to send parsed mutations to appliers;
-	// log mutations should be applied before range mutations at the same version, which is ensured by LogMessageVersion
-	wait(sendMutationsFromLoaders(batchData, batchStatus, self->loadersInterf, batchIndex, false) &&
-	     sendMutationsFromLoaders(batchData, batchStatus, self->loadersInterf, batchIndex, true));
-
-	// Synchronization point for version batch pipelining.
-	// self->finishedBatch will continuously increase by 1 per version batch.
-	wait(notifyApplierToApplyMutations(batchData, batchStatus, self->appliersInterf, batchIndex, &self->finishedBatch));
-
-	wait(notifyLoadersVersionBatchFinished(self->loadersInterf, batchIndex));
-
-	self->runningVersionBatches.set(self->runningVersionBatches.get() - 1);
-
-	if (self->delayedActors > 0) {
-		self->checkMemory.trigger();
-	}
-
-	TraceEvent("FastRestoreControllerDispatchVersionBatchesDone", self->id())
-	    .detail("BatchIndex", batchIndex)
-	    .detail("BatchSize", versionBatch.size)
-	    .detail("RunningVersionBatches", self->runningVersionBatches.get())
-	    .detail("Latency", now() - startTime);
-
-	return Void();
-}
-
-// Decide which key range should be taken by which applier
-// Input: samples in batchData
-// Output: rangeToApplier in batchData
-void splitKeyRangeForAppliers(Reference<ControllerBatchData> batchData,
-                              std::map<UID, RestoreApplierInterface> appliersInterf,
-                              int batchIndex) {
-	ASSERT(batchData->samplesSize >= 0);
-	// Sanity check: samples should not be used after freed
-	ASSERT((batchData->samplesSize > 0 && !batchData->samples.empty()) ||
-	       (batchData->samplesSize == 0 && batchData->samples.empty()));
-	int numAppliers = appliersInterf.size();
-	double slotSize = std::max(batchData->samplesSize / numAppliers, 1.0);
-	double cumulativeSize = slotSize;
-	TraceEvent("FastRestoreControllerPhaseCalculateApplierKeyRangesStart")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("SamplingSize", batchData->samplesSize)
-	    .detail("SlotSize", slotSize);
-
-	std::set<Key> keyrangeSplitter; // unique key to split key range for appliers
-	keyrangeSplitter.insert(allKeys.begin); // First slot
-	TraceEvent("FastRestoreControllerPhaseCalculateApplierKeyRanges")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("CumulativeSize", cumulativeSize)
-	    .detail("Slot", 0)
-	    .detail("LowerBoundKey", allKeys.begin);
-	int slotIdx = 1;
-	while (cumulativeSize < batchData->samplesSize) {
-		IndexedSet<Key, int64_t>::iterator lowerBound = batchData->samples.index(cumulativeSize);
-		if (lowerBound == batchData->samples.end()) {
-			break;
-		}
-		keyrangeSplitter.insert(*lowerBound);
-		TraceEvent("FastRestoreControllerPhaseCalculateApplierKeyRanges")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("CumulativeSize", cumulativeSize)
-		    .detail("Slot", slotIdx++)
-		    .detail("LowerBoundKey", lowerBound->toString());
-		cumulativeSize += slotSize;
-	}
-	if (keyrangeSplitter.size() < numAppliers) {
-		TraceEvent(SevWarnAlways, "FastRestoreControllerPhaseCalculateApplierKeyRanges")
-		    .detail("NotAllAppliersAreUsed", keyrangeSplitter.size())
-		    .detail("NumAppliers", numAppliers);
-	} else if (keyrangeSplitter.size() > numAppliers) {
-		bool expected = (keyrangeSplitter.size() == numAppliers + 1);
-		TraceEvent(expected ? SevWarn : SevError, "FastRestoreControllerPhaseCalculateApplierKeyRanges")
-		    .detail("TooManySlotsThanAppliers", keyrangeSplitter.size())
-		    .detail("NumAppliers", numAppliers)
-		    .detail("SamplingSize", batchData->samplesSize)
-		    .detail("PerformanceMayDegrade", "Last applier handles more data than others");
-	}
-
-	std::set<Key>::iterator splitter = keyrangeSplitter.begin();
-	batchData->rangeToApplier.clear();
-	for (auto& applier : appliersInterf) {
-		if (splitter == keyrangeSplitter.end()) {
-			break; // Not all appliers will be used
-		}
-		batchData->rangeToApplier[*splitter] = applier.first;
-		splitter++;
-	}
-	ASSERT(batchData->rangeToApplier.size() > 0);
-	ASSERT(batchData->sanityCheckApplierKeyRange());
-	batchData->logApplierKeyRange(batchIndex);
-	TraceEvent("FastRestoreControllerPhaseCalculateApplierKeyRangesDone")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("SamplingSize", batchData->samplesSize)
-	    .detail("SlotSize", slotSize);
-	batchData->samples.clear();
-}
-
-ACTOR static Future<std::vector<RestoreRequest>> collectRestoreRequests(Database cx) {
-	state std::vector<RestoreRequest> restoreRequests;
-	state Future<Void> watch4RestoreRequest;
-	state ReadYourWritesTransaction tr(cx);
-
-	// restoreRequestTriggerKey should already been set
-	loop {
-		try {
-			TraceEvent("FastRestoreControllerPhaseCollectRestoreRequestsWait").log();
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-
-			// Sanity check
-			Optional<Value> numRequests = wait(tr.get(restoreRequestTriggerKey));
-			ASSERT(numRequests.present());
-
-			RangeResult restoreRequestValues = wait(tr.getRange(restoreRequestKeys, CLIENT_KNOBS->TOO_MANY));
-			ASSERT(!restoreRequestValues.more);
-			if (restoreRequestValues.size()) {
-				for (auto& it : restoreRequestValues) {
-					restoreRequests.push_back(decodeRestoreRequestValue(it.value));
-					TraceEvent("FastRestoreControllerPhaseCollectRestoreRequests")
-					    .detail("RestoreRequest", restoreRequests.back().toString());
-				}
-				break;
-			} else {
-				TraceEvent(SevError, "FastRestoreControllerPhaseCollectRestoreRequestsEmptyRequests").log();
-				wait(delay(5.0));
-			}
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-
-	return restoreRequests;
-}
-
-// Collect the backup files' description into output_files by reading the backupContainer bc.
-// Returns the restore target version.
-ACTOR static Future<Version> collectBackupFiles(Reference<IBackupContainer> bc,
-                                                std::vector<RestoreFileFR>* rangeFiles,
-                                                std::vector<RestoreFileFR>* logFiles,
-                                                Version* minRangeVersion,
-                                                Database cx,
-                                                RestoreRequest request) {
-	state BackupDescription desc = wait(bc->describeBackup());
-
-	// Convert version to real time for operators to read the BackupDescription desc.
-	wait(desc.resolveVersionTimes(cx));
-
-	if (request.targetVersion == invalidVersion && desc.maxRestorableVersion.present()) {
-		request.targetVersion = desc.maxRestorableVersion.get();
-	}
-
-	TraceEvent("FastRestoreControllerPhaseCollectBackupFilesStart")
-	    .detail("TargetVersion", request.targetVersion)
-	    .detail("BackupDesc", desc.toString())
-	    .detail("UseRangeFile", SERVER_KNOBS->FASTRESTORE_USE_RANGE_FILE)
-	    .detail("UseLogFile", SERVER_KNOBS->FASTRESTORE_USE_LOG_FILE);
-	if (g_network->isSimulated()) {
-		std::cout << "Restore to version: " << request.targetVersion << "\nBackupDesc: \n" << desc.toString() << "\n\n";
-	}
-
-	state VectorRef<KeyRangeRef> restoreRanges;
-	restoreRanges.add(request.range);
-	Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(request.targetVersion, restoreRanges));
-
-	if (!restorable.present()) {
-		TraceEvent(SevWarn, "FastRestoreControllerPhaseCollectBackupFiles")
-		    .detail("NotRestorable", request.targetVersion);
-		throw restore_missing_data();
-	}
-
-	ASSERT(rangeFiles->empty());
-	ASSERT(logFiles->empty());
-
-	std::set<RestoreFileFR> uniqueRangeFiles;
-	std::set<RestoreFileFR> uniqueLogFiles;
-	double rangeSize = 0;
-	double logSize = 0;
-	*minRangeVersion = MAX_VERSION;
-	if (SERVER_KNOBS->FASTRESTORE_USE_RANGE_FILE) {
-		for (const RangeFile& f : restorable.get().ranges) {
-			TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles")
-			    .detail("RangeFile", f.toString());
-			if (f.fileSize <= 0) {
-				continue;
-			}
-			RestoreFileFR file(f);
-			TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles")
-			    .detail("RangeFileFR", file.toString());
-			uniqueRangeFiles.insert(file);
-			rangeSize += file.fileSize;
-			*minRangeVersion = std::min(*minRangeVersion, file.version);
-		}
-	}
-	if (MAX_VERSION == *minRangeVersion) {
-		*minRangeVersion = 0; // If no range file, range version must be 0 so that we apply all mutations
-	}
-
-	if (SERVER_KNOBS->FASTRESTORE_USE_LOG_FILE) {
-		for (const LogFile& f : restorable.get().logs) {
-			TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles").detail("LogFile", f.toString());
-			if (f.fileSize <= 0) {
-				continue;
-			}
-			RestoreFileFR file(f);
-			TraceEvent(SevFRDebugInfo, "FastRestoreControllerPhaseCollectBackupFiles")
-			    .detail("LogFileFR", file.toString());
-			logFiles->push_back(file);
-			uniqueLogFiles.insert(file);
-			logSize += file.fileSize;
-		}
-	}
-
-	// Assign unique range files and log files to output
-	rangeFiles->assign(uniqueRangeFiles.begin(), uniqueRangeFiles.end());
-	logFiles->assign(uniqueLogFiles.begin(), uniqueLogFiles.end());
-
-	TraceEvent("FastRestoreControllerPhaseCollectBackupFilesDone")
-	    .detail("BackupDesc", desc.toString())
-	    .detail("RangeFiles", rangeFiles->size())
-	    .detail("LogFiles", logFiles->size())
-	    .detail("RangeFileBytes", rangeSize)
-	    .detail("LogFileBytes", logSize)
-	    .detail("UseRangeFile", SERVER_KNOBS->FASTRESTORE_USE_RANGE_FILE)
-	    .detail("UseLogFile", SERVER_KNOBS->FASTRESTORE_USE_LOG_FILE);
-	return request.targetVersion;
-}
-
-// By the first and last block of *file to get (beginKey, endKey);
-// set (beginKey, endKey) and file->version to pRangeVersions
-ACTOR static Future<Void> insertRangeVersion(KeyRangeMap<Version>* pRangeVersions,
-                                             RestoreFileFR* file,
-                                             Reference<IBackupContainer> bc,
-                                             Database cx) {
-	TraceEvent("FastRestoreControllerDecodeRangeVersion").detail("File", file->toString());
-	RangeFile rangeFile = { file->version, (uint32_t)file->blockSize, file->fileName, file->fileSize };
-
-	// First and last key are the range for this file: endKey is exclusive
-	KeyRange fileRange = wait(bc->getSnapshotFileKeyRange(rangeFile, cx));
-	TraceEvent("FastRestoreControllerInsertRangeVersion")
-	    .detail("DecodedRangeFile", file->fileName)
-	    .detail("KeyRange", fileRange)
-	    .detail("Version", file->version);
-	// Update version for pRangeVersions's ranges in fileRange
-	auto ranges = pRangeVersions->modify(fileRange);
-	for (auto r = ranges.begin(); r != ranges.end(); ++r) {
-		r->value() = std::max(r->value(), file->version);
-	}
-
-	if (SERVER_KNOBS->FASTRESTORE_DUMP_INSERT_RANGE_VERSION) {
-		// Dump the new key ranges for debugging purpose.
-		ranges = pRangeVersions->ranges();
-		int i = 0;
-		for (auto r = ranges.begin(); r != ranges.end(); ++r) {
-			TraceEvent(SevDebug, "RangeVersionsAfterUpdate")
-			    .detail("File", file->toString())
-			    .detail("FileRange", fileRange.toString())
-			    .detail("FileVersion", file->version)
-			    .detail("RangeIndex", i++)
-			    .detail("RangeBegin", r->begin())
-			    .detail("RangeEnd", r->end())
-			    .detail("RangeVersion", r->value());
-		}
-	}
-
-	return Void();
-}
-
-// Build the version skyline of snapshot ranges by parsing range files;
-// Expensive and slow operation that should not run in real prod.
-ACTOR static Future<Void> buildRangeVersions(KeyRangeMap<Version>* pRangeVersions,
-                                             std::vector<RestoreFileFR>* pRangeFiles,
-                                             Key url,
-                                             Optional<std::string> proxy,
-                                             Database cx) {
-	if (!g_network->isSimulated()) {
-		TraceEvent(SevError, "ExpensiveBuildRangeVersions")
-		    .detail("Reason", "Parsing all range files is slow and memory intensive");
-		return Void();
-	}
-	Reference<IBackupContainer> bc = IBackupContainer::openContainer(url.toString(), proxy, {});
-
-	// Key ranges not in range files are empty;
-	// Assign highest version to avoid applying any mutation in these ranges
-	state int fileIndex = 0;
-	state std::vector<Future<Void>> fInsertRangeVersions;
-	for (; fileIndex < pRangeFiles->size(); ++fileIndex) {
-		fInsertRangeVersions.push_back(insertRangeVersion(pRangeVersions, &pRangeFiles->at(fileIndex), bc, cx));
-	}
-
-	wait(waitForAll(fInsertRangeVersions));
-
-	return Void();
-}
-
-/*
-ACTOR static Future<Void> clearDB(Database cx) {
-    wait(runRYWTransaction(cx, [](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-        tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-        tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-        tr->clear(normalKeys);
-        return Void();
-    }));
-
-    return Void();
-}
-*/
-
-ACTOR static Future<Void> initializeVersionBatch(std::map<UID, RestoreApplierInterface> appliersInterf,
-                                                 std::map<UID, RestoreLoaderInterface> loadersInterf,
-                                                 int batchIndex) {
-	TraceEvent("FastRestoreControllerPhaseInitVersionBatchForAppliersStart")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Appliers", appliersInterf.size());
-	std::vector<std::pair<UID, RestoreVersionBatchRequest>> requestsToAppliers;
-	requestsToAppliers.reserve(appliersInterf.size());
-	for (auto& applier : appliersInterf) {
-		requestsToAppliers.emplace_back(applier.first, RestoreVersionBatchRequest(batchIndex));
-	}
-	wait(sendBatchRequests(&RestoreApplierInterface::initVersionBatch, appliersInterf, requestsToAppliers));
-
-	TraceEvent("FastRestoreControllerPhaseInitVersionBatchForLoaders")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("Loaders", loadersInterf.size());
-	std::vector<std::pair<UID, RestoreVersionBatchRequest>> requestsToLoaders;
-	requestsToLoaders.reserve(loadersInterf.size());
-	for (auto& loader : loadersInterf) {
-		requestsToLoaders.emplace_back(loader.first, RestoreVersionBatchRequest(batchIndex));
-	}
-	wait(sendBatchRequests(&RestoreLoaderInterface::initVersionBatch, loadersInterf, requestsToLoaders));
-
-	TraceEvent("FastRestoreControllerPhaseInitVersionBatchForAppliersDone").detail("BatchIndex", batchIndex);
-	return Void();
-}
-
-// Calculate the amount of data each applier should keep outstanding to DB;
-// This is the amount of data that are in in-progress transactions.
-ACTOR static Future<Void> updateApplierWriteBW(Reference<ControllerBatchData> batchData,
-                                               std::map<UID, RestoreApplierInterface> appliersInterf,
-                                               int batchIndex) {
-	state std::unordered_map<UID, double> applierRemainMB;
-	state double totalRemainMB = SERVER_KNOBS->FASTRESTORE_WRITE_BW_MB;
-	state double standardAvgBW = SERVER_KNOBS->FASTRESTORE_WRITE_BW_MB / SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS;
-	state int loopCount = 0;
-	state std::vector<RestoreUpdateRateReply> replies;
-	state std::vector<std::pair<UID, RestoreUpdateRateRequest>> requests;
-	for (auto& applier : appliersInterf) {
-		applierRemainMB[applier.first] = SERVER_KNOBS->FASTRESTORE_WRITE_BW_MB / SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS;
-	}
-
-	loop {
-		requests.clear();
-		for (auto& applier : appliersInterf) {
-			double writeRate = totalRemainMB > 1 ? (applierRemainMB[applier.first] / totalRemainMB) *
-			                                           SERVER_KNOBS->FASTRESTORE_WRITE_BW_MB
-			                                     : standardAvgBW;
-			requests.emplace_back(applier.first, RestoreUpdateRateRequest(batchIndex, writeRate));
-		}
-		replies.clear();
-		wait(getBatchReplies(
-		    &RestoreApplierInterface::updateRate,
-		    appliersInterf,
-		    requests,
-		    &replies,
-		    TaskPriority::DefaultEndpoint)); // DefaultEndpoint has higher priority than fast restore endpoints
-		ASSERT(replies.size() == requests.size());
-		totalRemainMB = 0;
-		for (int i = 0; i < replies.size(); i++) {
-			UID& applierID = requests[i].first;
-			applierRemainMB[applierID] = replies[i].remainMB;
-			totalRemainMB += replies[i].remainMB;
-		}
-		ASSERT(totalRemainMB >= 0);
-		double delayTime = SERVER_KNOBS->FASTRESTORE_RATE_UPDATE_SECONDS;
-		if (loopCount == 0) { // First loop: Need to update writeRate quicker
-			delayTime = 0.2;
-		}
-		loopCount++;
-		wait(delay(delayTime));
-	}
-}
-
-// Ask each applier to apply its received mutations to DB
-// NOTE: Controller cannot start applying mutations at batchIndex until all appliers have applied for (batchIndex - 1)
-//       because appliers at different batchIndex may have overlapped key ranges.
-ACTOR static Future<Void> notifyApplierToApplyMutations(Reference<ControllerBatchData> batchData,
-                                                        Reference<ControllerBatchStatus> batchStatus,
-                                                        std::map<UID, RestoreApplierInterface> appliersInterf,
-                                                        int batchIndex,
-                                                        NotifiedVersion* finishedBatch) {
-	TraceEvent("FastRestoreControllerPhaseApplyToDBStart")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("FinishedBatch", finishedBatch->get());
-
-	wait(finishedBatch->whenAtLeast(batchIndex - 1));
-
-	state Future<Void> updateRate;
-
-	if (finishedBatch->get() == batchIndex - 1) {
-		// Prepare the applyToDB requests
-		std::vector<std::pair<UID, RestoreVersionBatchRequest>> requests;
-
-		TraceEvent("FastRestoreControllerPhaseApplyToDBRunning")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("Appliers", appliersInterf.size());
-		for (auto& applier : appliersInterf) {
-			ASSERT(batchStatus->applyStatus.find(applier.first) == batchStatus->applyStatus.end());
-			requests.emplace_back(applier.first, RestoreVersionBatchRequest(batchIndex));
-			batchStatus->applyStatus[applier.first] = RestoreApplyStatus::Applying;
-		}
-		state std::vector<RestoreCommonReply> replies;
-		// The actor at each batchIndex should only occur once.
-		// Use batchData->applyToDB just in case the actor at a batchIndex is executed more than once.
-		if (!batchData->applyToDB.present()) {
-			batchData->applyToDB = Never();
-			batchData->applyToDB = getBatchReplies(&RestoreApplierInterface::applyToDB,
-			                                       appliersInterf,
-			                                       requests,
-			                                       &replies,
-			                                       TaskPriority::RestoreApplierWriteDB);
-			updateRate = updateApplierWriteBW(batchData, appliersInterf, batchIndex);
-		} else {
-			TraceEvent(SevError, "FastRestoreControllerPhaseApplyToDB")
-			    .detail("BatchIndex", batchIndex)
-			    .detail("Attention", "Actor should not be invoked twice for the same batch index");
-		}
-
-		ASSERT(batchData->applyToDB.present());
-		ASSERT(!batchData->applyToDB.get().isError());
-		wait(batchData->applyToDB.get());
-
-		// Sanity check all appliers have applied data to destination DB
-		for (auto& reply : replies) {
-			if (batchStatus->applyStatus[reply.id] == RestoreApplyStatus::Applying) {
-				batchStatus->applyStatus[reply.id] = RestoreApplyStatus::Applied;
-				if (reply.isDuplicated) {
-					TraceEvent(SevWarn, "FastRestoreControllerPhaseApplyToDB")
-					    .detail("Applier", reply.id)
-					    .detail("DuplicateRequestReturnEarlier", "Apply db request should have been processed");
-				}
-			}
-		}
-		for (auto& applier : appliersInterf) {
-			if (batchStatus->applyStatus[applier.first] != RestoreApplyStatus::Applied) {
-				TraceEvent(SevError, "FastRestoreControllerPhaseApplyToDB")
-				    .detail("Applier", applier.first)
-				    .detail("ApplyStatus", batchStatus->applyStatus[applier.first]);
-			}
-		}
-		finishedBatch->set(batchIndex);
-	}
-
-	TraceEvent("FastRestoreControllerPhaseApplyToDBDone")
-	    .detail("BatchIndex", batchIndex)
-	    .detail("FinishedBatch", finishedBatch->get());
-
-	return Void();
-}
-
-// Notify loaders that all data in the version batch has been applied to DB.
-ACTOR static Future<Void> notifyLoadersVersionBatchFinished(std::map<UID, RestoreLoaderInterface> loadersInterf,
-                                                            int batchIndex) {
-	TraceEvent("FastRestoreControllerPhaseNotifyLoadersVersionBatchFinishedStart").detail("BatchIndex", batchIndex);
-	std::vector<std::pair<UID, RestoreVersionBatchRequest>> requestsToLoaders;
-	requestsToLoaders.reserve(loadersInterf.size());
-	for (auto& loader : loadersInterf) {
-		requestsToLoaders.emplace_back(loader.first, RestoreVersionBatchRequest(batchIndex));
-	}
-	wait(sendBatchRequests(&RestoreLoaderInterface::finishVersionBatch, loadersInterf, requestsToLoaders));
-	TraceEvent("FastRestoreControllerPhaseNotifyLoadersVersionBatchFinishedDone").detail("BatchIndex", batchIndex);
-
-	return Void();
-}
-
-// Ask all loaders and appliers to perform housecleaning at the end of a restore request
-// Terminate those roles if terminate = true
-ACTOR static Future<Void> notifyRestoreCompleted(Reference<RestoreControllerData> self, bool terminate = false) {
-	std::vector<std::pair<UID, RestoreFinishRequest>> requests;
-	TraceEvent("FastRestoreControllerPhaseNotifyRestoreCompletedStart").log();
-	for (auto& loader : self->loadersInterf) {
-		requests.emplace_back(loader.first, RestoreFinishRequest(terminate));
-	}
-
-	Future<Void> endLoaders = sendBatchRequests(&RestoreLoaderInterface::finishRestore, self->loadersInterf, requests);
-
-	requests.clear();
-	for (auto& applier : self->appliersInterf) {
-		requests.emplace_back(applier.first, RestoreFinishRequest(terminate));
-	}
-	Future<Void> endAppliers =
-	    sendBatchRequests(&RestoreApplierInterface::finishRestore, self->appliersInterf, requests);
-
-	// If terminate = true, loaders and appliers exits immediately after it receives the request. Controller may not
-	// receive acks.
-	if (!terminate) {
-		wait(endLoaders && endAppliers);
-	}
-
-	TraceEvent("FastRestoreControllerPhaseNotifyRestoreCompletedDone").log();
-
-	return Void();
-}
-
-// Register the restoreRequestDoneKey to signal the end of restore
-ACTOR static Future<Void> signalRestoreCompleted(Reference<RestoreControllerData> self, Database cx) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-
-	wait(notifyRestoreCompleted(self, true)); // notify workers the restore has completed
-
-	wait(delay(5.0)); // Give some time for loaders and appliers to exit
-
-	// Notify tester that the restore has finished
-	loop {
-		try {
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr->clear(restoreRequestTriggerKey);
-			tr->clear(restoreRequestKeys);
-			Version readVersion = wait(tr->getReadVersion());
-			tr->set(restoreRequestDoneKey, restoreRequestDoneVersionValue(readVersion));
-			wait(tr->commit());
-			break;
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-
-	TraceEvent("FastRestoreControllerAllRestoreCompleted").log();
-
-	return Void();
-}
-
-/*
-// Update the most recent time when controller receives heartbeat from each loader and applier
-// TODO: Replace the heartbeat mechanism with FDB failure monitoring mechanism
-ACTOR static Future<Void> updateHeartbeatTime(Reference<RestoreControllerData> self) {
-    wait(self->recruitedRoles.getFuture());
-
-    int numRoles = self->loadersInterf.size() + self->appliersInterf.size();
-    state std::map<UID, RestoreLoaderInterface>::iterator loader = self->loadersInterf.begin();
-    state std::map<UID, RestoreApplierInterface>::iterator applier = self->appliersInterf.begin();
-    state std::vector<Future<RestoreCommonReply>> fReplies(numRoles, Never()); // TODO: Reserve memory for this vector
-    state std::vector<UID> nodes;
-    state int index = 0;
-    state Future<Void> fTimeout = Void();
-
-    // Initialize nodes only once
-    std::transform(self->loadersInterf.begin(),
-                   self->loadersInterf.end(),
-                   std::back_inserter(nodes),
-                   [](const std::pair<UID, RestoreLoaderInterface>& in) { return in.first; });
-    std::transform(self->appliersInterf.begin(),
-                   self->appliersInterf.end(),
-                   std::back_inserter(nodes),
-                   [](const std::pair<UID, RestoreApplierInterface>& in) { return in.first; });
-
-    loop {
-        loader = self->loadersInterf.begin();
-        applier = self->appliersInterf.begin();
-        index = 0;
-        std::fill(fReplies.begin(), fReplies.end(), Never());
-        // ping loaders and appliers
-        while (loader != self->loadersInterf.end()) {
-            fReplies[index] = loader->second.heartbeat.getReply(RestoreSimpleRequest());
-            loader++;
-            index++;
-        }
-        while (applier != self->appliersInterf.end()) {
-            fReplies[index] = applier->second.heartbeat.getReply(RestoreSimpleRequest());
-            applier++;
-            index++;
-        }
-
-        fTimeout = delay(SERVER_KNOBS->FASTRESTORE_HEARTBEAT_DELAY);
-
-        // Here we have to handle error, otherwise controller worker will fail and exit.
-        try {
-            wait(waitForAll(fReplies) || fTimeout);
-        } catch (Error& e) {
-            // This should be an ignorable error.
-            TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "FastRestoreUpdateHeartbeatError").error(e);
-        }
-
-        // Update the most recent heart beat time for each role
-        for (int i = 0; i < fReplies.size(); ++i) {
-            if (!fReplies[i].isError() && fReplies[i].isReady()) {
-                double currentTime = now();
-                auto item = self->rolesHeartBeatTime.emplace(nodes[i], currentTime);
-                item.first->second = currentTime;
-            }
-        }
-        wait(fTimeout); // Ensure not updating heartbeat too quickly
-    }
-}
-*/
-
-// Check if a restore role dies or disconnected
-ACTOR static Future<Void> checkRolesLiveness(Reference<RestoreControllerData> self) {
-	loop {
-		wait(delay(SERVER_KNOBS->FASTRESTORE_HEARTBEAT_MAX_DELAY));
-		for (auto& role : self->rolesHeartBeatTime) {
-			if (now() - role.second > SERVER_KNOBS->FASTRESTORE_HEARTBEAT_MAX_DELAY) {
-				TraceEvent(SevWarnAlways, "FastRestoreUnavailableRole", role.first)
-				    .detail("Delta", now() - role.second)
-				    .detail("LastAliveTime", role.second);
-			}
-		}
-	}
-}
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
deleted file mode 100644
index 9b8370f0e78..00000000000
--- a/fdbserver/RestoreLoader.actor.cpp
+++ /dev/null
@@ -1,1583 +0,0 @@
-/*
- * RestoreLoader.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// NOTE: this file represents fast restore functionality which is deprecated and
-// slated for removal.
-
-#include "fdbclient/CommitProxyInterface.h"
-#include "flow/UnitTest.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/DatabaseContext.h"
-#include "fdbserver/RestoreLoader.actor.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/StorageMetrics.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// SerializedMutationListMap: Buffered mutation lists from data blocks in log files
-// Key is the signature/version of the mutation list; Value.first is the mutation list which may come from multiple
-// data blocks of log file; Value.second is the largest part number of the mutation list, which is used to sanity check
-// the data blocks for the same mutation list are concatenated in increasing order of part number.
-typedef std::map<Standalone<StringRef>, std::pair<Standalone<StringRef>, uint32_t>> SerializedMutationListMap;
-
-std::vector<UID> getApplierIDs(std::map<Key, UID>& rangeToApplier);
-void splitMutation(const KeyRangeMap<UID>& krMap,
-                   MutationRef m,
-                   Arena& mvector_arena,
-                   VectorRef<MutationRef>& mvector,
-                   Arena& nodeIDs_arena,
-                   VectorRef<UID>& nodeIDs);
-ACTOR Future<Void> _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
-                                            std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-                                            SerializedMutationListMap* mutationMap,
-                                            std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-                                            LoaderCounters* cc,
-                                            RestoreAsset asset,
-                                            Database cx);
-
-void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<RestoreLoaderData> self);
-ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self, Database cx);
-ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequest req,
-                                              Reference<RestoreLoaderData> self,
-                                              Database cx);
-ACTOR Future<Void> sendMutationsToApplier(
-    std::priority_queue<RestoreLoaderSchedSendLoadParamRequest>* sendLoadParamQueue,
-    std::map<int, int>* inflightSendLoadParamReqs,
-    NotifiedVersion* finishedBatch,
-    VersionedMutationsMap* pkvOps,
-    int batchIndex,
-    RestoreAsset asset,
-    bool isRangeFile,
-    std::map<Key, UID>* pRangeToApplier,
-    std::map<UID, RestoreApplierInterface>* pApplierInterfaces,
-    Database cx);
-ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
-                                                           SerializedMutationListMap* mutationMap,
-                                                           Reference<IBackupContainer> bc,
-                                                           RestoreAsset asset);
-ACTOR static Future<Void> parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
-                                                          SerializedMutationListMap* mutationMap,
-                                                          Reference<IBackupContainer> bc,
-                                                          RestoreAsset asset);
-ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
-    std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-    std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-    LoaderCounters* cc,
-    Reference<IBackupContainer> bc,
-    Version version,
-    RestoreAsset asset,
-    Database cx);
-ACTOR Future<Void> handleFinishVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreLoaderData> self);
-
-// Dispatch requests based on node's business (i.e, cpu usage for now) and requests' priorities
-// Requests for earlier version batches are preferred; which is equivalent to
-// sendMuttionsRequests are preferred than loadingFileRequests
-ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self, Database cx) {
-	try {
-		state int curVBInflightReqs = 0;
-		state int sendLoadParams = 0;
-		state int lastLoadReqs = 0;
-		loop {
-			TraceEvent(SevVerbose, "FastRestoreLoaderDispatchRequests", self->id())
-			    .detail("SendingQueue", self->sendingQueue.size())
-			    .detail("LoadingQueue", self->loadingQueue.size())
-			    .detail("SendingLoadParamQueue", self->sendLoadParamQueue.size())
-			    .detail("InflightSendingReqs", self->inflightSendingReqs)
-			    .detail("InflightSendingReqsThreshold", SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_SEND_REQS)
-			    .detail("InflightLoadingReqs", self->inflightLoadingReqs)
-			    .detail("InflightLoadingReqsThreshold", SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS)
-			    .detail("LastLoadFileRequests", lastLoadReqs)
-			    .detail("LoadFileRequestsBatchThreshold", SERVER_KNOBS->FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE)
-			    .detail("LastDispatchSendLoadParamReqsForCurrentVB", curVBInflightReqs)
-			    .detail("LastDispatchSendLoadParamReqsForFutureVB", sendLoadParams)
-			    .detail("CpuUsage", self->cpuUsage)
-			    .detail("TargetCpuUsage", SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT)
-			    .detail("MaxCpuUsage", SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT);
-
-			// TODO: Pop old requests whose version batch <= finishedBatch.get()
-			// TODO2: Simulate delayed request can be too old by introducing artificial delay
-			if (SERVER_KNOBS->FASTRESTORE_EXPENSIVE_VALIDATION) {
-				// Sanity check: All requests before and in finishedBatch must have been processed; otherwise,
-				// those requests may cause segmentation fault after applier remove the batch data
-				if (!self->loadingQueue.empty() && self->loadingQueue.top().batchIndex <= self->finishedBatch.get()) {
-					// Still has pending requests from earlier batchIndex  and current batchIndex, which should not
-					// happen
-					TraceEvent(SevError, "FastRestoreLoaderSchedulerHasOldLoadFileRequests")
-					    .detail("FinishedBatchIndex", self->finishedBatch.get())
-					    .detail("PendingRequest", self->loadingQueue.top().toString());
-				}
-				if (!self->sendingQueue.empty() && self->sendingQueue.top().batchIndex <= self->finishedBatch.get()) {
-					TraceEvent(SevError, "FastRestoreLoaderSchedulerHasOldSendRequests")
-					    .detail("FinishedBatchIndex", self->finishedBatch.get())
-					    .detail("PendingRequest", self->sendingQueue.top().toString());
-				}
-				if (!self->sendLoadParamQueue.empty() &&
-				    self->sendLoadParamQueue.top().batchIndex <= self->finishedBatch.get()) {
-					TraceEvent(SevError, "FastRestoreLoaderSchedulerHasOldSendLoadParamRequests")
-					    .detail("FinishedBatchIndex", self->finishedBatch.get())
-					    .detail("PendingRequest", self->sendLoadParamQueue.top().toString());
-				}
-			}
-
-			if (!self->sendingQueue.empty()) {
-				// Only release one sendMutationRequest at a time because it sends all data for a version batch
-				// and it takes large amount of resource
-				const RestoreSendMutationsToAppliersRequest& req = self->sendingQueue.top();
-				// Dispatch the request if it is the next version batch to process or if cpu usage is low
-				if (req.batchIndex - 1 == self->finishedSendingVB ||
-				    self->cpuUsage < SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT) {
-					self->addActor.send(handleSendMutationsRequest(req, self, cx));
-					self->sendingQueue.pop();
-				}
-			}
-			// When shall the node pause the process of other requests, e.g., load file requests
-			// TODO: Revisit if we should have (self->inflightSendingReqs > 0 && self->inflightLoadingReqs > 0)
-			if ((self->inflightSendingReqs > 0 && self->inflightLoadingReqs > 0) &&
-			    (self->inflightSendingReqs >= SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_SEND_REQS ||
-			     self->inflightLoadingReqs >= SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS ||
-			     (self->inflightSendingReqs >= 1 &&
-			      self->cpuUsage >= SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT) ||
-			     self->cpuUsage >= SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT)) {
-				if (self->inflightSendingReqs >= SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_SEND_REQS) {
-					TraceEvent(SevWarn, "FastRestoreLoaderTooManyInflightRequests")
-					    .detail("VersionBatchesBlockedAtSendingMutationsToAppliers", self->inflightSendingReqs)
-					    .detail("CpuUsage", self->cpuUsage)
-					    .detail("InflightSendingReq", self->inflightSendingReqs)
-					    .detail("InflightSendingReqThreshold", SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_SEND_REQS)
-					    .detail("InflightLoadingReq", self->inflightLoadingReqs)
-					    .detail("InflightLoadingReqThreshold", SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS);
-				}
-				wait(delay(SERVER_KNOBS->FASTRESTORE_SCHED_UPDATE_DELAY));
-				updateProcessStats(self);
-				continue;
-			}
-			// Dispatch queued requests of sending mutations per loading param
-			while (!self->sendLoadParamQueue.empty()) { // dispatch current VB first
-				const RestoreLoaderSchedSendLoadParamRequest& req = self->sendLoadParamQueue.top();
-				if (req.batchIndex - 1 > self->finishedSendingVB) { // future VB
-					break;
-				} else {
-					req.toSched.send(Void());
-					self->sendLoadParamQueue.pop();
-				}
-			}
-			sendLoadParams = 0;
-			curVBInflightReqs = self->inflightSendLoadParamReqs[self->finishedSendingVB + 1];
-			while (!self->sendLoadParamQueue.empty()) {
-				const RestoreLoaderSchedSendLoadParamRequest& req = self->sendLoadParamQueue.top();
-				if (curVBInflightReqs >= SERVER_KNOBS->FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD ||
-				    sendLoadParams >= SERVER_KNOBS->FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH) {
-					// Too many future VB requests are released
-					break;
-				} else {
-					req.toSched.send(Void());
-					self->sendLoadParamQueue.pop();
-					sendLoadParams++;
-				}
-			}
-
-			// Dispatch loading backup file requests
-			lastLoadReqs = 0;
-			while (!self->loadingQueue.empty()) {
-				if (lastLoadReqs >= SERVER_KNOBS->FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE) {
-					break;
-				}
-				const RestoreLoadFileRequest& req = self->loadingQueue.top();
-				if (req.batchIndex <= self->finishedBatch.get()) {
-					TraceEvent(SevError, "FastRestoreLoaderDispatchRestoreLoadFileRequestTooOld")
-					    .detail("FinishedBatchIndex", self->finishedBatch.get())
-					    .detail("RequestBatchIndex", req.batchIndex);
-					req.reply.send(RestoreLoadFileReply(req.param, true));
-					self->loadingQueue.pop();
-					ASSERT(false); // Check if this ever happens easily
-				} else {
-					self->addActor.send(handleLoadFileRequest(req, self, cx));
-					self->loadingQueue.pop();
-					lastLoadReqs++;
-				}
-			}
-
-			if (self->cpuUsage >= SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT) {
-				wait(delay(SERVER_KNOBS->FASTRESTORE_SCHED_UPDATE_DELAY));
-			}
-			updateProcessStats(self);
-
-			if (self->loadingQueue.empty() && self->sendingQueue.empty() && self->sendLoadParamQueue.empty()) {
-				TraceEvent(SevVerbose, "FastRestoreLoaderDispatchRequestsWaitOnRequests", self->id())
-				    .detail("HasPendingRequests", self->hasPendingRequests->get());
-				self->hasPendingRequests->set(false);
-				wait(self->hasPendingRequests->onChange()); // CAREFUL:Improper req release may cause restore stuck here
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled) {
-			TraceEvent(SevError, "FastRestoreLoaderDispatchRequests").errorUnsuppressed(e);
-			throw e;
-		}
-	}
-	return Void();
-}
-
-ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf,
-                                     int nodeIndex,
-                                     Database cx,
-                                     RestoreControllerInterface ci) {
-	state Reference<RestoreLoaderData> self = makeReference<RestoreLoaderData>(loaderInterf.id(), nodeIndex, ci);
-	state Future<Void> error = actorCollection(self->addActor.getFuture());
-	state ActorCollection actors(false); // actors whose errors can be ignored
-	state Future<Void> exitRole = Never();
-	state bool hasQueuedRequests = false;
-
-	actors.add(updateProcessMetrics(self));
-	actors.add(traceProcessMetrics(self, "RestoreLoader"));
-
-	self->addActor.send(dispatchRequests(self, cx));
-
-	loop {
-		state std::string requestTypeStr = "[Init]";
-
-		try {
-			choose {
-				when(RestoreSimpleRequest req = waitNext(loaderInterf.heartbeat.getFuture())) {
-					requestTypeStr = "heartbeat";
-					actors.add(handleHeartbeat(req, loaderInterf.id()));
-				}
-				when(RestoreSysInfoRequest req = waitNext(loaderInterf.updateRestoreSysInfo.getFuture())) {
-					requestTypeStr = "updateRestoreSysInfo";
-					handleRestoreSysInfoRequest(req, self);
-				}
-				when(RestoreLoadFileRequest req = waitNext(loaderInterf.loadFile.getFuture())) {
-					requestTypeStr = "loadFile";
-					hasQueuedRequests = !self->loadingQueue.empty() || !self->sendingQueue.empty();
-					self->initBackupContainer(req.param.url, req.param.proxy);
-					self->loadingQueue.push(req);
-					if (!hasQueuedRequests) {
-						self->hasPendingRequests->set(true);
-					}
-				}
-				when(RestoreSendMutationsToAppliersRequest req = waitNext(loaderInterf.sendMutations.getFuture())) {
-					requestTypeStr = "sendMutations";
-					hasQueuedRequests = !self->loadingQueue.empty() || !self->sendingQueue.empty();
-					self->sendingQueue.push(req);
-					if (!hasQueuedRequests) {
-						self->hasPendingRequests->set(true);
-					}
-				}
-				when(RestoreVersionBatchRequest req = waitNext(loaderInterf.initVersionBatch.getFuture())) {
-					requestTypeStr = "initVersionBatch";
-					actors.add(handleInitVersionBatchRequest(req, self));
-				}
-				when(RestoreVersionBatchRequest req = waitNext(loaderInterf.finishVersionBatch.getFuture())) {
-					requestTypeStr = "finishVersionBatch";
-					actors.add(handleFinishVersionBatchRequest(req, self));
-				}
-				when(RestoreFinishRequest req = waitNext(loaderInterf.finishRestore.getFuture())) {
-					requestTypeStr = "finishRestore";
-					handleFinishRestoreRequest(req, self);
-					if (req.terminate) {
-						exitRole = Void();
-					}
-				}
-				when(wait(actors.getResult())) {}
-				when(wait(exitRole)) {
-					TraceEvent("FastRestoreLoaderCoreExitRole", self->id());
-					break;
-				}
-				when(wait(error)) {
-					TraceEvent("FastRestoreLoaderActorCollectionError", self->id());
-				}
-			}
-		} catch (Error& e) {
-			bool isError = e.code() != error_code_operation_cancelled; // == error_code_broken_promise
-			TraceEvent(isError ? SevError : SevWarnAlways, "FastRestoreLoaderError", self->id())
-			    .errorUnsuppressed(e)
-			    .detail("RequestType", requestTypeStr);
-			actors.clear(false);
-			break;
-		}
-	}
-
-	return Void();
-}
-
-static inline bool _logMutationTooOld(KeyRangeMap<Version>* pRangeVersions, KeyRangeRef keyRange, Version v) {
-	ASSERT(pRangeVersions != nullptr);
-	auto ranges = pRangeVersions->intersectingRanges(keyRange);
-	Version minVersion = MAX_VERSION;
-	for (auto r = ranges.begin(); r != ranges.end(); ++r) {
-		minVersion = std::min(minVersion, r->value());
-	}
-	ASSERT(minVersion != MAX_VERSION); // pRangeVersions is initialized as entire keyspace, ranges cannot be empty
-	return minVersion >= v;
-}
-
-static inline bool logMutationTooOld(KeyRangeMap<Version>* pRangeVersions, MutationRef mutation, Version v) {
-	return isRangeMutation(mutation)
-	           ? _logMutationTooOld(pRangeVersions, KeyRangeRef(mutation.param1, mutation.param2), v)
-	           : _logMutationTooOld(pRangeVersions, KeyRangeRef(singleKeyRange(mutation.param1)), v);
-}
-
-// Assume: Only update the local data if it (applierInterf) has not been set
-void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<RestoreLoaderData> self) {
-	TraceEvent("FastRestoreLoader", self->id()).detail("HandleRestoreSysInfoRequest", self->id());
-	ASSERT(self.isValid());
-
-	// The loader has received the appliers interfaces
-	if (!self->appliersInterf.empty()) {
-		req.reply.send(RestoreCommonReply(self->id()));
-		return;
-	}
-
-	self->appliersInterf = req.sysInfo.appliers;
-	// Update rangeVersions
-	ASSERT(req.rangeVersions.size() > 0); // At least the min version of range files will be used
-	ASSERT(self->rangeVersions.size() == 1); // rangeVersions has not been set
-	for (auto rv = req.rangeVersions.begin(); rv != req.rangeVersions.end(); ++rv) {
-		self->rangeVersions.insert(rv->first, rv->second);
-	}
-
-	// Debug message for range version in each loader
-	auto ranges = self->rangeVersions.ranges();
-	int i = 0;
-	for (auto r = ranges.begin(); r != ranges.end(); ++r) {
-		TraceEvent("FastRestoreLoader", self->id())
-		    .detail("RangeIndex", i++)
-		    .detail("RangeBegin", r->begin())
-		    .detail("RangeEnd", r->end())
-		    .detail("Version", r->value());
-	}
-
-	req.reply.send(RestoreCommonReply(self->id()));
-}
-
-// Parse a data block in a partitioned mutation log file and store mutations
-// into "kvOpsIter" and samples into "samplesIter".
-ACTOR static Future<Void> _parsePartitionedLogFileOnLoader(
-    KeyRangeMap<Version>* pRangeVersions,
-    NotifiedVersion* processedFileOffset,
-    std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-    std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-    LoaderCounters* cc,
-    Reference<IBackupContainer> bc,
-    RestoreAsset asset,
-    Database cx) {
-	state Standalone<StringRef> buf = makeString(asset.len);
-	state Reference<IAsyncFile> file = wait(bc->readFile(asset.filename));
-	int rLen = wait(file->read(mutateString(buf), asset.len, asset.offset));
-	if (rLen != asset.len)
-		throw restore_bad_read();
-
-	simulateBlobFailure();
-
-	TraceEvent("FastRestoreLoaderDecodingLogFile")
-	    .detail("BatchIndex", asset.batchIndex)
-	    .detail("Filename", asset.filename)
-	    .detail("Offset", asset.offset)
-	    .detail("Length", asset.len);
-
-	state Arena tempArena;
-	state StringRefReader reader(buf, restore_corrupted_data());
-	try {
-		// Read block header
-		if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
-			throw restore_unsupported_file_version();
-
-		state VersionedMutationsMap* kvOps = &kvOpsIter->second;
-		while (1) {
-			// If eof reached or first key len bytes is 0xFF then end of block was reached.
-			if (reader.eof() || *reader.rptr == 0xFF)
-				break;
-
-			// Deserialize messages written in saveMutationsToFile().
-			state LogMessageVersion msgVersion;
-			msgVersion.version = reader.consumeNetworkUInt64();
-			msgVersion.sub = reader.consumeNetworkUInt32();
-			state int msgSize = reader.consumeNetworkInt32();
-			state const uint8_t* message = reader.consume(msgSize);
-
-			// Skip mutations out of the version range
-			if (!asset.isInVersionRange(msgVersion.version)) {
-				wait(yield()); // avoid potential stack overflows
-				continue;
-			}
-
-			state VersionedMutationsMap::iterator it;
-			bool inserted;
-			std::tie(it, inserted) = kvOps->emplace(msgVersion, MutationsVec());
-			// A clear mutation can be split into multiple mutations with the same (version, sub).
-			// See saveMutationsToFile(). Current tests only use one key range per backup, thus
-			// only one clear mutation is generated (i.e., always inserted).
-			ASSERT(inserted);
-
-			ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
-			state MutationRef mutation;
-			rd >> mutation;
-
-			// Skip mutation whose commitVesion < range kv's version
-			if (logMutationTooOld(pRangeVersions, mutation, msgVersion.version)) {
-				cc->oldLogMutations += 1;
-				wait(yield()); // avoid potential stack overflows
-				continue;
-			}
-
-			// Should this mutation be skipped?
-			if (mutation.param1 >= asset.range.end ||
-			    (isRangeMutation(mutation) && mutation.param2 < asset.range.begin) ||
-			    (!isRangeMutation(mutation) && mutation.param1 < asset.range.begin)) {
-				wait(yield()); // avoid potential stack overflows
-				continue;
-			}
-
-			// Only apply mutation within the asset.range
-			ASSERT(asset.removePrefix.size() == 0);
-			if (isRangeMutation(mutation)) {
-				mutation.param1 = mutation.param1 >= asset.range.begin ? mutation.param1 : asset.range.begin;
-				mutation.param2 = mutation.param2 < asset.range.end ? mutation.param2 : asset.range.end;
-				// Remove prefix or add prefix when we restore to a new key space
-				if (asset.hasPrefix()) { // Avoid creating new Key
-					mutation.param1 =
-					    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-					mutation.param2 =
-					    mutation.param2.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-				}
-			} else {
-				if (asset.hasPrefix()) { // Avoid creating new Key
-					mutation.param1 =
-					    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-				}
-			}
-
-			TraceEvent(SevFRMutationInfo, "FastRestoreDecodePartitionedLogFile")
-			    .detail("CommitVersion", msgVersion.toString())
-			    .detail("ParsedMutation", mutation.toString());
-			it->second.push_back_deep(it->second.arena(), mutation);
-			cc->loadedLogBytes += mutation.totalSize();
-			// Sampling data similar to SS sample kvs
-			ByteSampleInfo sampleInfo = isKeyValueInSample(KeyValueRef(mutation.param1, mutation.param2));
-			if (sampleInfo.inSample) {
-				cc->sampledLogBytes += sampleInfo.sampledSize;
-				samplesIter->second.push_back_deep(samplesIter->second.arena(),
-				                                   SampledMutation(mutation.param1, sampleInfo.sampledSize));
-			}
-		}
-
-		// Make sure any remaining bytes in the block are 0xFF
-		for (auto b : reader.remainder()) {
-			if (b != 0xFF)
-				throw restore_corrupted_data_padding();
-		}
-	} catch (Error& e) {
-		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
-		    .error(e)
-		    .detail("BatchIndex", asset.batchIndex)
-		    .detail("Filename", file->getFilename())
-		    .detail("BlockOffset", asset.offset)
-		    .detail("BlockLen", asset.len);
-		throw;
-	}
-	return Void();
-}
-
-// wrapper of _parsePartitionedLogFileOnLoader to retry on blob error
-ACTOR static Future<Void> parsePartitionedLogFileOnLoader(
-    KeyRangeMap<Version>* pRangeVersions,
-    NotifiedVersion* processedFileOffset,
-    std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-    std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-    LoaderCounters* cc,
-    Reference<IBackupContainer> bc,
-    RestoreAsset asset,
-    Database cx) {
-	state int readFileRetries = 0;
-	loop {
-		try {
-			// Ensure data blocks in the same file are processed in order
-			wait(processedFileOffset->whenAtLeast(asset.offset));
-			ASSERT(processedFileOffset->get() == asset.offset);
-
-			wait(_parsePartitionedLogFileOnLoader(
-			    pRangeVersions, processedFileOffset, kvOpsIter, samplesIter, cc, bc, asset, cx));
-			processedFileOffset->set(asset.offset + asset.len);
-
-			TraceEvent("FastRestoreLoaderDecodingLogFileDone")
-			    .detail("BatchIndex", asset.batchIndex)
-			    .detail("Filename", asset.filename)
-			    .detail("Offset", asset.offset)
-			    .detail("Length", asset.len);
-			break;
-		} catch (Error& e) {
-			if (e.code() == error_code_restore_bad_read || e.code() == error_code_restore_unsupported_file_version ||
-			    e.code() == error_code_restore_corrupted_data_padding) { // no retriable error
-				TraceEvent(SevError, "FastRestoreFileRestoreCorruptedPartitionedLogFileBlock").error(e);
-				throw;
-			} else if (e.code() == error_code_http_request_failed || e.code() == error_code_connection_failed ||
-			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
-				// blob http request failure, retry
-				TraceEvent(SevWarnAlways, "FastRestoreDecodedPartitionedLogFileConnectionFailure")
-				    .error(e)
-				    .detail("Retries", ++readFileRetries);
-				wait(delayJittered(0.1));
-			} else {
-				TraceEvent(SevError, "FastRestoreParsePartitionedLogFileOnLoaderUnexpectedError").error(e);
-				throw;
-			}
-		}
-	}
-	return Void();
-}
-
-ACTOR Future<Void> _processLoadingParam(KeyRangeMap<Version>* pRangeVersions,
-                                        LoadingParam param,
-                                        Reference<LoaderBatchData> batchData,
-                                        UID loaderID,
-                                        Reference<IBackupContainer> bc,
-                                        Database cx) {
-	// Temporary data structure for parsing log files into (version, <K, V, mutationType>)
-	// Must use StandAlone to save mutations, otherwise, the mutationref memory will be corrupted
-	// mutationMap: Key is the unique identifier for a batch of mutation logs at the same version
-	state SerializedMutationListMap mutationMap;
-	state NotifiedVersion processedFileOffset(0);
-	state std::vector<Future<Void>> fileParserFutures;
-	state std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsPerLPIter = batchData->kvOpsPerLP.end();
-	state std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter = batchData->sampleMutations.end();
-
-	TraceEvent("FastRestoreLoaderProcessLoadingParam", loaderID)
-	    .detail("BatchIndex", param.asset.batchIndex)
-	    .detail("LoadingParam", param.toString());
-	ASSERT(param.blockSize > 0);
-	ASSERT(param.asset.offset % param.blockSize == 0); // Parse file must be at block boundary.
-	ASSERT(batchData->kvOpsPerLP.find(param) == batchData->kvOpsPerLP.end());
-
-	// NOTE: map's iterator is guaranteed to be stable, but pointer may not.
-	bool inserted;
-	std::tie(kvOpsPerLPIter, inserted) = batchData->kvOpsPerLP.emplace(param, VersionedMutationsMap());
-	ASSERT(inserted);
-	std::tie(samplesIter, inserted) = batchData->sampleMutations.emplace(param, SampledMutationsVec());
-	ASSERT(inserted);
-
-	for (int64_t j = param.asset.offset; j < param.asset.len; j += param.blockSize) {
-		RestoreAsset subAsset = param.asset;
-		subAsset.offset = j;
-		subAsset.len = std::min<int64_t>(param.blockSize, param.asset.len - j);
-		if (param.isRangeFile) {
-			fileParserFutures.push_back(_parseRangeFileToMutationsOnLoader(
-			    kvOpsPerLPIter, samplesIter, &batchData->counters, bc, param.rangeVersion.get(), subAsset, cx));
-		} else {
-			// TODO: Sanity check the log file's range is overlapped with the restored version range
-			if (param.isPartitionedLog()) {
-				fileParserFutures.push_back(parsePartitionedLogFileOnLoader(pRangeVersions,
-				                                                            &processedFileOffset,
-				                                                            kvOpsPerLPIter,
-				                                                            samplesIter,
-				                                                            &batchData->counters,
-				                                                            bc,
-				                                                            subAsset,
-				                                                            cx));
-			} else {
-				fileParserFutures.push_back(
-				    parseLogFileToMutationsOnLoader(&processedFileOffset, &mutationMap, bc, subAsset));
-			}
-		}
-	}
-	wait(waitForAll(fileParserFutures));
-
-	if (!param.isRangeFile && !param.isPartitionedLog()) {
-		wait(_parseSerializedMutation(
-		    pRangeVersions, kvOpsPerLPIter, &mutationMap, samplesIter, &batchData->counters, param.asset, cx));
-	}
-
-	TraceEvent("FastRestoreLoaderProcessLoadingParamDone", loaderID)
-	    .detail("BatchIndex", param.asset.batchIndex)
-	    .detail("LoadingParam", param.toString());
-
-	return Void();
-}
-
-// A loader can process multiple RestoreLoadFileRequest in parallel.
-ACTOR Future<Void> handleLoadFileRequest(RestoreLoadFileRequest req, Reference<RestoreLoaderData> self, Database cx) {
-	state Reference<LoaderBatchData> batchData = self->batch[req.batchIndex];
-	state bool isDuplicated = true;
-	state bool printTrace = false;
-	ASSERT(batchData.isValid());
-	ASSERT(req.batchIndex > self->finishedBatch.get());
-	bool paramExist = batchData->processedFileParams.find(req.param) != batchData->processedFileParams.end();
-	bool isReady = paramExist ? batchData->processedFileParams[req.param].isReady() : false;
-
-	batchData->loadFileReqs += 1;
-	printTrace = (batchData->loadFileReqs % 10 == 1);
-	// TODO: Make the actor priority lower than sendMutation priority. (Unsure it will help performance though)
-	TraceEvent(printTrace ? SevInfo : SevFRDebugInfo, "FastRestoreLoaderPhaseLoadFile", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("ProcessLoadParam", req.param.toString())
-	    .detail("NotProcessed", !paramExist)
-	    .detail("Processed", isReady)
-	    .detail("CurrentMemory", getSystemStatistics().processMemory);
-	// Loader destroy batchData once the batch finishes and self->finishedBatch.set(req.batchIndex);
-	ASSERT(self->finishedBatch.get() < req.batchIndex);
-
-	wait(isSchedulable(self, req.batchIndex, __FUNCTION__));
-
-	if (batchData->processedFileParams.find(req.param) == batchData->processedFileParams.end()) {
-		TraceEvent(SevFRDebugInfo, "FastRestoreLoadFile", self->id())
-		    .detail("BatchIndex", req.batchIndex)
-		    .detail("ProcessLoadParam", req.param.toString());
-		ASSERT(batchData->sampleMutations.find(req.param) == batchData->sampleMutations.end());
-		batchData->processedFileParams[req.param] =
-		    _processLoadingParam(&self->rangeVersions, req.param, batchData, self->id(), self->bc, cx);
-		self->inflightLoadingReqs++;
-		isDuplicated = false;
-	} else {
-		TraceEvent(SevFRDebugInfo, "FastRestoreLoadFile", self->id())
-		    .detail("BatchIndex", req.batchIndex)
-		    .detail("WaitOnProcessLoadParam", req.param.toString());
-	}
-	auto it = batchData->processedFileParams.find(req.param);
-	ASSERT(it != batchData->processedFileParams.end());
-	wait(it->second); // wait on the processing of the req.param.
-
-	// Send sampled mutations back to controller:  batchData->sampleMutations[req.param]
-	std::vector<Future<RestoreCommonReply>> fSendSamples;
-	SampledMutationsVec& samples = batchData->sampleMutations[req.param];
-	SampledMutationsVec sampleBatch = SampledMutationsVec(); // sampleBatch: Standalone pointer to the created object
-	long sampleBatchSize = 0;
-	for (int i = 0; i < samples.size(); ++i) {
-		sampleBatchSize += samples[i].totalSize();
-		sampleBatch.push_back_deep(sampleBatch.arena(), samples[i]); // TODO: may not need deep copy
-		if (sampleBatchSize >= SERVER_KNOBS->FASTRESTORE_SAMPLE_MSG_BYTES) {
-			fSendSamples.push_back(self->ci.samples.getReply(
-			    RestoreSamplesRequest(deterministicRandom()->randomUniqueID(), req.batchIndex, sampleBatch)));
-			sampleBatchSize = 0;
-			sampleBatch = SampledMutationsVec();
-		}
-	}
-	if (sampleBatchSize > 0) {
-		fSendSamples.push_back(self->ci.samples.getReply(
-		    RestoreSamplesRequest(deterministicRandom()->randomUniqueID(), req.batchIndex, sampleBatch)));
-		sampleBatchSize = 0;
-	}
-
-	state int samplesMessages = fSendSamples.size();
-	try {
-		wait(waitForAll(fSendSamples));
-	} catch (Error& e) { // In case ci.samples throws broken_promise due to unstable network
-		if (e.code() == error_code_broken_promise || e.code() == error_code_operation_cancelled) {
-			TraceEvent(SevWarnAlways, "FastRestoreLoaderPhaseLoadFileSendSamples")
-			    .errorUnsuppressed(e)
-			    .detail("SamplesMessages", samplesMessages);
-		} else {
-			TraceEvent(SevError, "FastRestoreLoaderPhaseLoadFileSendSamplesUnexpectedError").errorUnsuppressed(e);
-		}
-	}
-
-	// Ack restore controller the param is processed
-	self->inflightLoadingReqs--;
-	req.reply.send(RestoreLoadFileReply(req.param, isDuplicated));
-	TraceEvent(printTrace ? SevInfo : SevFRDebugInfo, "FastRestoreLoaderPhaseLoadFileDone", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("ProcessLoadParam", req.param.toString());
-
-	return Void();
-}
-
-// Send buffered mutations to appliers.
-// Do not need to block on low memory usage because this actor should not increase memory usage.
-ACTOR Future<Void> handleSendMutationsRequest(RestoreSendMutationsToAppliersRequest req,
-                                              Reference<RestoreLoaderData> self,
-                                              Database cx) {
-	state Reference<LoaderBatchData> batchData;
-	state Reference<LoaderBatchStatus> batchStatus;
-	state bool isDuplicated = true;
-
-	if (req.batchIndex <= self->finishedBatch.get()) {
-		TraceEvent(SevWarn, "FastRestoreLoaderRestoreSendMutationsToAppliersRequestTooOld")
-		    .detail("FinishedBatchIndex", self->finishedBatch.get())
-		    .detail("RequestBatchIndex", req.batchIndex);
-		req.reply.send(RestoreCommonReply(self->id(), isDuplicated));
-		return Void();
-	}
-
-	batchData = self->batch[req.batchIndex];
-	batchStatus = self->status[req.batchIndex];
-	ASSERT(batchData.isValid() && batchStatus.isValid());
-	// Loader destroy batchData once the batch finishes and self->finishedBatch.set(req.batchIndex);
-	ASSERT(req.batchIndex > self->finishedBatch.get());
-	TraceEvent("FastRestoreLoaderPhaseSendMutations", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("UseRangeFile", req.useRangeFile)
-	    .detail("LoaderSendStatus", batchStatus->toString());
-	// The VB must finish loading phase before it can send mutations; update finishedLoadingVB for scheduler
-	self->finishedLoadingVB = std::max(self->finishedLoadingVB, req.batchIndex);
-
-	// Ensure each file is sent exactly once by using batchStatus->sendAllLogs and batchStatus->sendAllRanges
-	if (!req.useRangeFile) {
-		if (!batchStatus->sendAllLogs.present()) { // Has not sent
-			batchStatus->sendAllLogs = Never();
-			isDuplicated = false;
-			TraceEvent(SevInfo, "FastRestoreSendMutationsProcessLogRequest", self->id())
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("UseRangeFile", req.useRangeFile);
-		} else if (!batchStatus->sendAllLogs.get().isReady()) { // In the process of sending
-			TraceEvent(SevDebug, "FastRestoreSendMutationsWaitDuplicateLogRequest", self->id())
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("UseRangeFile", req.useRangeFile);
-			wait(batchStatus->sendAllLogs.get());
-		} else { // Already sent
-			TraceEvent(SevDebug, "FastRestoreSendMutationsSkipDuplicateLogRequest", self->id())
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("UseRangeFile", req.useRangeFile);
-		}
-	} else {
-		if (!batchStatus->sendAllRanges.present()) {
-			batchStatus->sendAllRanges = Never();
-			isDuplicated = false;
-			TraceEvent(SevInfo, "FastRestoreSendMutationsProcessRangeRequest", self->id())
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("UseRangeFile", req.useRangeFile);
-		} else if (!batchStatus->sendAllRanges.get().isReady()) {
-			TraceEvent(SevDebug, "FastRestoreSendMutationsWaitDuplicateRangeRequest", self->id())
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("UseRangeFile", req.useRangeFile);
-			wait(batchStatus->sendAllRanges.get());
-		} else {
-			TraceEvent(SevDebug, "FastRestoreSendMutationsSkipDuplicateRangeRequest", self->id())
-			    .detail("BatchIndex", req.batchIndex)
-			    .detail("UseRangeFile", req.useRangeFile);
-		}
-	}
-
-	if (!isDuplicated) {
-		self->inflightSendingReqs++;
-		std::vector<Future<Void>> fSendMutations;
-		batchData->rangeToApplier = req.rangeToApplier;
-		for (auto& [loadParam, kvOps] : batchData->kvOpsPerLP) {
-			if (loadParam.isRangeFile == req.useRangeFile) {
-				// Send the parsed mutation to applier who will apply the mutation to DB
-				fSendMutations.push_back(sendMutationsToApplier(&self->sendLoadParamQueue,
-				                                                &self->inflightSendLoadParamReqs,
-				                                                &self->finishedBatch,
-				                                                &kvOps,
-				                                                req.batchIndex,
-				                                                loadParam.asset,
-				                                                loadParam.isRangeFile,
-				                                                &batchData->rangeToApplier,
-				                                                &self->appliersInterf,
-				                                                cx));
-			}
-		}
-		wait(waitForAll(fSendMutations));
-		self->inflightSendingReqs--;
-		if (req.useRangeFile) {
-			batchStatus->sendAllRanges = Void(); // Finish sending kvs parsed from range files
-		} else {
-			batchStatus->sendAllLogs = Void();
-		}
-		if ((batchStatus->sendAllRanges.present() && batchStatus->sendAllRanges.get().isReady()) &&
-		    (batchStatus->sendAllLogs.present() && batchStatus->sendAllLogs.get().isReady())) {
-			// Both log and range files have been sent.
-			self->finishedSendingVB = std::max(self->finishedSendingVB, req.batchIndex);
-			batchData->kvOpsPerLP.clear();
-		}
-	}
-
-	TraceEvent("FastRestoreLoaderPhaseSendMutationsDone", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("UseRangeFile", req.useRangeFile)
-	    .detail("LoaderSendStatus", batchStatus->toString());
-	req.reply.send(RestoreCommonReply(self->id(), isDuplicated));
-	return Void();
-}
-
-void buildApplierRangeMap(KeyRangeMap<UID>* krMap, std::map<Key, UID>* pRangeToApplier) {
-	std::map<Key, UID>::iterator beginKey = pRangeToApplier->begin();
-	std::map<Key, UID>::iterator endKey = std::next(beginKey, 1);
-	while (endKey != pRangeToApplier->end()) {
-		krMap->insert(KeyRangeRef(beginKey->first, endKey->first), beginKey->second);
-		beginKey = endKey;
-		endKey++;
-	}
-	if (beginKey != pRangeToApplier->end()) {
-		krMap->insert(KeyRangeRef(beginKey->first, allKeys.end), beginKey->second);
-	}
-}
-
-// Assume: kvOps data are from the same RestoreAsset.
-// Input: pkvOps: versioned kv mutation for the asset in the version batch (batchIndex)
-//   isRangeFile: is pkvOps from range file? Let receiver (applier) know if the mutation is log mutation;
-//   pRangeToApplier: range to applierID mapping, deciding which applier is responsible for which range
-//   pApplierInterfaces: applier interfaces to send the mutations to
-ACTOR Future<Void> sendMutationsToApplier(
-    std::priority_queue<RestoreLoaderSchedSendLoadParamRequest>* sendLoadParamQueue,
-    std::map<int, int>* inflightSendLoadParamReqs,
-    NotifiedVersion* finishedBatch,
-    VersionedMutationsMap* pkvOps,
-    int batchIndex,
-    RestoreAsset asset,
-    bool isRangeFile,
-    std::map<Key, UID>* pRangeToApplier,
-    std::map<UID, RestoreApplierInterface>* pApplierInterfaces,
-    Database cx) {
-	state VersionedMutationsMap& kvOps = *pkvOps;
-	state VersionedMutationsMap::iterator kvOp = kvOps.begin();
-	state int kvCount = 0;
-	state int splitMutationIndex = 0;
-	state Version msgIndex = 1; // Monotonically increased index for send message, must start at 1
-	state std::vector<UID> applierIDs = getApplierIDs(*pRangeToApplier);
-	state double msgSize = 0; // size of mutations in the message
-	state Arena arena;
-
-	// Wait for scheduler to kick it off
-	Promise<Void> toSched;
-	sendLoadParamQueue->push(RestoreLoaderSchedSendLoadParamRequest(batchIndex, toSched, now()));
-	wait(toSched.getFuture());
-	if (finishedBatch->get() >= batchIndex) {
-		TraceEvent(SevError, "FastRestoreLoaderSendMutationToApplierLateRequest")
-		    .detail("FinishedBatchIndex", finishedBatch->get())
-		    .detail("RequestBatchIndex", batchIndex);
-		ASSERT(false);
-		return Void();
-	}
-
-	(*inflightSendLoadParamReqs)[batchIndex]++;
-
-	TraceEvent("FastRestoreLoaderSendMutationToApplier")
-	    .detail("IsRangeFile", isRangeFile)
-	    .detail("EndVersion", asset.endVersion)
-	    .detail("RestoreAsset", asset.toString());
-
-	// There should be no mutation at asset.endVersion version because it is exclusive
-	if (kvOps.lower_bound(LogMessageVersion(asset.endVersion)) != kvOps.end()) {
-		TraceEvent(SevError, "FastRestoreLoaderSendMutationToApplier")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("RestoreAsset", asset.toString())
-		    .detail("IsRangeFile", isRangeFile)
-		    .detail("Data loss at version", asset.endVersion);
-	} else {
-		// Ensure there is a mutation request sent at endVersion, so that applier can advance its notifiedVersion
-		kvOps[LogMessageVersion(asset.endVersion)] = MutationsVec(); // Empty mutation vector will be handled by applier
-	}
-
-	splitMutationIndex = 0;
-	kvCount = 0;
-
-	// applierVersionedMutationsBuffer is the mutation-and-its-version vector to be sent to each applier
-	state std::map<UID, VersionedMutationsVec> applierVersionedMutationsBuffer;
-	state int mIndex = 0;
-	state LogMessageVersion commitVersion;
-	state std::vector<Future<Void>> fSends;
-	for (auto& applierID : applierIDs) {
-		applierVersionedMutationsBuffer[applierID] = VersionedMutationsVec();
-	}
-	state KeyRangeMap<UID> krMap;
-	buildApplierRangeMap(&krMap, pRangeToApplier);
-	for (kvOp = kvOps.begin(); kvOp != kvOps.end(); kvOp++) {
-		commitVersion = kvOp->first;
-		ASSERT(commitVersion.version >= asset.beginVersion);
-		ASSERT(commitVersion.version <= asset.endVersion); // endVersion is an empty commit to ensure progress
-		for (mIndex = 0; mIndex < kvOp->second.size(); mIndex++) {
-			state MutationRef kvm = kvOp->second[mIndex];
-			// Send the mutation to applier
-			if (isRangeMutation(kvm)) {
-				MutationsVec mvector;
-				Standalone<VectorRef<UID>> nodeIDs;
-				// Because using a vector of mutations causes overhead, and the range mutation should happen rarely;
-				// We handle the range mutation and key mutation differently for the benefit of avoiding memory copy
-				splitMutation(krMap, kvm, mvector.arena(), mvector.contents(), nodeIDs.arena(), nodeIDs.contents());
-				ASSERT(mvector.size() == nodeIDs.size());
-
-				if (MUTATION_TRACKING_ENABLED) {
-					TraceEvent&& e = debugMutation("RestoreLoaderDebugSplit", commitVersion.version, kvm);
-					if (e.isEnabled()) {
-						int i = 0;
-						for (auto& [key, uid] : *pRangeToApplier) {
-							e.detail(format("Range%d", i).c_str(), printable(key))
-							    .detail(format("UID%d", i).c_str(), uid.toString());
-							i++;
-						}
-					}
-				}
-				for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) {
-					MutationRef mutation = mvector[splitMutationIndex];
-					UID applierID = nodeIDs[splitMutationIndex];
-					DEBUG_MUTATION("RestoreLoaderSplitMutation", commitVersion.version, mutation)
-					    .detail("CommitVersion", commitVersion.toString());
-					// CAREFUL: The split mutations' lifetime is shorter than the for-loop
-					// Must use deep copy for split mutations
-					applierVersionedMutationsBuffer[applierID].push_back_deep(
-					    applierVersionedMutationsBuffer[applierID].arena(),
-					    VersionedMutationSerialized(mutation, commitVersion));
-					msgSize += mutation.expectedSize();
-
-					kvCount++;
-				}
-			} else { // mutation operates on a particular key
-				std::map<Key, UID>::iterator itlow = pRangeToApplier->upper_bound(kvm.param1);
-				--itlow; // make sure itlow->first <= m.param1
-				ASSERT(itlow->first <= kvm.param1);
-				UID applierID = itlow->second;
-				kvCount++;
-
-				DEBUG_MUTATION("RestoreLoaderSendMutation", commitVersion.version, kvm)
-				    .detail("Applier", applierID)
-				    .detail("SubVersion", commitVersion.toString());
-				// kvm data is saved in pkvOps in batchData, so shallow copy is ok here.
-				applierVersionedMutationsBuffer[applierID].push_back(applierVersionedMutationsBuffer[applierID].arena(),
-				                                                     VersionedMutationSerialized(kvm, commitVersion));
-				msgSize += kvm.expectedSize();
-			}
-
-			// Batch mutations at multiple versions up to FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES size
-			// to improve bandwidth from a loader to appliers
-			if (msgSize >= SERVER_KNOBS->FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES) {
-				std::vector<std::pair<UID, RestoreSendVersionedMutationsRequest>> requests;
-				requests.reserve(applierIDs.size());
-				for (const UID& applierID : applierIDs) {
-					requests.emplace_back(
-					    applierID,
-					    RestoreSendVersionedMutationsRequest(
-					        batchIndex, asset, msgIndex, isRangeFile, applierVersionedMutationsBuffer[applierID]));
-				}
-				TraceEvent(SevInfo, "FastRestoreLoaderSendMutationToApplier")
-				    .detail("MessageIndex", msgIndex)
-				    .detail("RestoreAsset", asset.toString())
-				    .detail("Requests", requests.size());
-				fSends.push_back(sendBatchRequests(&RestoreApplierInterface::sendMutationVector,
-				                                   *pApplierInterfaces,
-				                                   requests,
-				                                   TaskPriority::RestoreLoaderSendMutations));
-				msgIndex++;
-				msgSize = 0;
-				for (auto& applierID : applierIDs) {
-					applierVersionedMutationsBuffer[applierID] = VersionedMutationsVec();
-				}
-			}
-		} // Mutations at the same LogMessageVersion
-	} // all versions of mutations in the same file
-
-	// Send the remaining mutations in the applierMutationsBuffer
-	if (msgSize > 0) {
-		// TODO: Sanity check each asset has been received exactly once!
-		std::vector<std::pair<UID, RestoreSendVersionedMutationsRequest>> requests;
-		requests.reserve(applierIDs.size());
-		for (const UID& applierID : applierIDs) {
-			requests.emplace_back(
-			    applierID,
-			    RestoreSendVersionedMutationsRequest(
-			        batchIndex, asset, msgIndex, isRangeFile, applierVersionedMutationsBuffer[applierID]));
-		}
-		TraceEvent(SevInfo, "FastRestoreLoaderSendMutationToApplier")
-		    .detail("MessageIndex", msgIndex)
-		    .detail("RestoreAsset", asset.toString())
-		    .detail("Requests", requests.size());
-		fSends.push_back(sendBatchRequests(&RestoreApplierInterface::sendMutationVector,
-		                                   *pApplierInterfaces,
-		                                   requests,
-		                                   TaskPriority::RestoreLoaderSendMutations));
-	}
-	wait(waitForAll(fSends));
-
-	(*inflightSendLoadParamReqs)[batchIndex]--;
-
-	if (finishedBatch->get() < batchIndex) {
-		kvOps = VersionedMutationsMap(); // Free memory for parsed mutations at the restore asset.
-		TraceEvent("FastRestoreLoaderSendMutationToApplierDone")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("RestoreAsset", asset.toString())
-		    .detail("Mutations", kvCount);
-	} else {
-		TraceEvent(SevWarnAlways, "FastRestoreLoaderSendMutationToApplierDoneTooLate")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("FinishedBatchIndex", finishedBatch->get())
-		    .detail("RestoreAsset", asset.toString())
-		    .detail("Mutations", kvCount);
-	}
-
-	return Void();
-}
-
-// Splits a clear range mutation for Appliers and puts results of split mutations and
-// Applier IDs into "mvector" and "nodeIDs" on return.
-void splitMutation(const KeyRangeMap<UID>& krMap,
-                   MutationRef m,
-                   Arena& mvector_arena,
-                   VectorRef<MutationRef>& mvector,
-                   Arena& nodeIDs_arena,
-                   VectorRef<UID>& nodeIDs) {
-	TraceEvent(SevVerbose, "FastRestoreSplitMutation").detail("Mutation", m);
-	ASSERT(mvector.empty());
-	ASSERT(nodeIDs.empty());
-	auto r = krMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
-	for (auto i = r.begin(); i != r.end(); ++i) {
-		// Calculate the overlap range
-		KeyRef rangeBegin = m.param1 > i->range().begin ? m.param1 : i->range().begin;
-		KeyRef rangeEnd = m.param2 < i->range().end ? m.param2 : i->range().end;
-		KeyRange krange1(KeyRangeRef(rangeBegin, rangeEnd));
-		mvector.push_back_deep(mvector_arena, MutationRef(MutationRef::ClearRange, rangeBegin, rangeEnd));
-		nodeIDs.push_back(nodeIDs_arena, i->cvalue());
-	}
-}
-
-// key_input format:
-// [logRangeMutation.first][hash_value_of_commit_version:1B][bigEndian64(commitVersion)][bigEndian32(part)]
-// value_input: serialized binary of mutations at the same version
-bool concatenateBackupMutationForLogFile(SerializedMutationListMap* pMutationMap,
-                                         Standalone<StringRef> key_input,
-                                         Standalone<StringRef> val_input,
-                                         const RestoreAsset& asset) {
-	SerializedMutationListMap& mutationMap = *pMutationMap;
-	const int key_prefix_len = sizeof(uint8_t) + sizeof(Version) + sizeof(uint32_t);
-
-	StringRefReader readerKey(key_input, restore_corrupted_data()); // read key_input!
-	int logRangeMutationFirstLength = key_input.size() - key_prefix_len;
-	bool concatenated = false;
-
-	ASSERT_WE_THINK(key_input.size() >= key_prefix_len);
-
-	if (logRangeMutationFirstLength > 0) {
-		// Strip out the [logRangeMutation.first]; otherwise, the following readerKey.consume will produce wrong value
-		readerKey.consume(logRangeMutationFirstLength);
-	}
-
-	readerKey.consume<uint8_t>(); // uint8_t hashValue = readerKey.consume<uint8_t>()
-	Version commitVersion = readerKey.consumeNetworkUInt64();
-	// Skip mutations not in [asset.beginVersion, asset.endVersion), which is what we are only processing right now
-	if (!asset.isInVersionRange(commitVersion)) {
-		return false;
-	}
-
-	uint32_t part = readerKey.consumeNetworkUInt32();
-	// Use commitVersion as id
-	Standalone<StringRef> id = StringRef((uint8_t*)&commitVersion, sizeof(Version));
-
-	auto it = mutationMap.find(id);
-	if (it == mutationMap.end()) {
-		mutationMap.emplace(id, std::make_pair(val_input, 0));
-		if (part != 0) {
-			TraceEvent(SevError, "FastRestoreLoader")
-			    .detail("FirstPartNotZero", part)
-			    .detail("KeyInput", getHexString(key_input));
-		}
-	} else { // Concatenate the val string with the same commitVersion
-		it->second.first =
-		    it->second.first.contents().withSuffix(val_input.contents()); // Assign the new Areana to the map's value
-		auto& currentPart = it->second.second;
-		if (part != (currentPart + 1)) {
-			// Check if the same range or log file has been processed more than once!
-			TraceEvent(SevError, "FastRestoreLoader")
-			    .detail("CurrentPart1", currentPart)
-			    .detail("CurrentPart2", part)
-			    .detail("KeyInput", getHexString(key_input))
-			    .detail("Hint", "Check if the same range or log file has been processed more than once");
-		}
-		currentPart = part;
-		concatenated = true;
-	}
-
-	return concatenated;
-}
-
-// Parse the kv pair (version, serialized_mutation), which are the results parsed from log file, into
-// (version, <K, V, mutationType>) pair;
-// Put the parsed versioned mutations into *pkvOps.
-//
-// Input key: [commitVersion_of_the_mutation_batch:uint64_t];
-// Input value: [includeVersion:uint64_t][val_length:uint32_t][encoded_list_of_mutations], where
-// includeVersion is the serialized version in the batch commit. It is not the commitVersion in Input key.
-//
-// val_length is always equal to (val.size() - 12); otherwise,
-// we may not get the entire mutation list for the version encoded_list_of_mutations:
-// [mutation1][mutation2]...[mutationk], where
-//	a mutation is encoded as [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][keyContent][valueContent]
-ACTOR Future<Void> _parseSerializedMutation(KeyRangeMap<Version>* pRangeVersions,
-                                            std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-                                            SerializedMutationListMap* pmutationMap,
-                                            std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-                                            LoaderCounters* cc,
-                                            RestoreAsset asset,
-                                            Database cx) {
-	state VersionedMutationsMap* kvOps = &kvOpsIter->second;
-	state SampledMutationsVec* samples = &samplesIter->second;
-	state SerializedMutationListMap::iterator mutationMapIterator = pmutationMap->begin();
-
-	TraceEvent(SevFRMutationInfo, "FastRestoreLoaderParseSerializedLogMutation")
-	    .detail("BatchIndex", asset.batchIndex)
-	    .detail("RestoreAsset", asset.toString());
-
-	state Arena tempArena;
-	loop {
-		if (mutationMapIterator == pmutationMap->end()) {
-			break;
-		}
-		// NOTE: this variable used to be called `k` and unintentionally shadowed a
-		// DIFFERENT state variable called `k` below.  This bug was exposed by removal
-		// of an intervening wait call.
-		StringRef outer_k = mutationMapIterator->first.contents();
-		state StringRef val = mutationMapIterator->second.first.contents();
-
-		StringRefReader kReader(outer_k, restore_corrupted_data());
-		state uint64_t commitVersion = kReader.consume<uint64_t>(); // Consume little Endian data
-		// We have already filter the commit not in [beginVersion, endVersion) when we concatenate kv pair in log file
-		ASSERT_WE_THINK(asset.isInVersionRange(commitVersion));
-
-		state StringRefReader vReader(val, restore_corrupted_data());
-		vReader.consume<uint64_t>(); // Consume the includeVersion
-		// TODO(xumengpanda): verify the protocol version is compatible and raise error if needed
-
-		// Parse little endian value, confirmed it is correct!
-		uint32_t val_length_decoded = vReader.consume<uint32_t>();
-		ASSERT(val_length_decoded == val.size() - sizeof(uint64_t) - sizeof(uint32_t));
-
-		state int sub = 0;
-		loop {
-			// stop when reach the end of the string
-			if (vReader.eof()) { //|| *reader.rptr == 0xFF
-				break;
-			}
-
-			state uint32_t type = vReader.consume<uint32_t>();
-			state uint32_t kLen = vReader.consume<uint32_t>();
-			state uint32_t vLen = vReader.consume<uint32_t>();
-			state const uint8_t* k = vReader.consume(kLen);
-			state const uint8_t* v = vReader.consume(vLen);
-
-			state MutationRef mutation((MutationRef::Type)type, KeyRef(k, kLen), KeyRef(v, vLen));
-			// Should this mutation be skipped?
-			// Skip mutation whose commitVesion < range kv's version
-			if (logMutationTooOld(pRangeVersions, mutation, commitVersion)) {
-				cc->oldLogMutations += 1;
-			} else {
-				if (mutation.param1 >= asset.range.end ||
-				    (isRangeMutation(mutation) && mutation.param2 < asset.range.begin) ||
-				    (!isRangeMutation(mutation) && mutation.param1 < asset.range.begin)) {
-				} else {
-					// Only apply mutation within the asset.range and apply removePrefix and addPrefix
-					ASSERT(asset.removePrefix.size() == 0);
-					if (isRangeMutation(mutation)) {
-						mutation.param1 = mutation.param1 >= asset.range.begin ? mutation.param1 : asset.range.begin;
-						mutation.param2 = mutation.param2 < asset.range.end ? mutation.param2 : asset.range.end;
-						// Remove prefix or add prefix if we restore data to a new key space
-						if (asset.hasPrefix()) { // Avoid creating new Key
-							mutation.param1 =
-							    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-							mutation.param2 =
-							    mutation.param2.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-						}
-					} else {
-						if (asset.hasPrefix()) { // Avoid creating new Key
-							mutation.param1 =
-							    mutation.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-						}
-					}
-
-					cc->loadedLogBytes += mutation.totalSize();
-
-					TraceEvent(SevFRMutationInfo, "FastRestoreDecodeLogFile")
-					    .detail("CommitVersion", commitVersion)
-					    .detail("ParsedMutation", mutation.toString());
-
-					auto it = kvOps->insert(std::make_pair(LogMessageVersion(commitVersion, sub++), MutationsVec()));
-					ASSERT(it.second); // inserted is true
-					ASSERT(sub <
-					       std::numeric_limits<int32_t>::max()); // range file mutation uses int32_max as subversion
-					it.first->second.push_back_deep(it.first->second.arena(), mutation);
-
-					// Sampling data similar to how SS sample bytes
-					ByteSampleInfo sampleInfo = isKeyValueInSample(KeyValueRef(mutation.param1, mutation.param2));
-					if (sampleInfo.inSample) {
-						cc->sampledLogBytes += sampleInfo.sampledSize;
-						samples->push_back_deep(samples->arena(),
-						                        SampledMutation(mutation.param1, sampleInfo.sampledSize));
-					}
-					ASSERT_WE_THINK(kLen >= 0 && kLen < val.size());
-					ASSERT_WE_THINK(vLen >= 0 && vLen < val.size());
-				}
-			}
-		}
-		mutationMapIterator++;
-	}
-	return Void();
-}
-
-// Parsing the data blocks in a range file
-// kvOpsIter: saves the parsed versioned-mutations for the specific LoadingParam;
-// samplesIter: saves the sampled mutations from the parsed versioned-mutations;
-// bc: backup container to read the backup file
-// version: the version the parsed mutations should be at
-// asset: RestoreAsset about which backup data should be parsed
-ACTOR static Future<Void> _parseRangeFileToMutationsOnLoader(
-    std::map<LoadingParam, VersionedMutationsMap>::iterator kvOpsIter,
-    std::map<LoadingParam, SampledMutationsVec>::iterator samplesIter,
-    LoaderCounters* cc,
-    Reference<IBackupContainer> bc,
-    Version version,
-    RestoreAsset asset,
-    Database cx) {
-	state VersionedMutationsMap& kvOps = kvOpsIter->second;
-	state SampledMutationsVec& sampleMutations = samplesIter->second;
-
-	TraceEvent(SevFRDebugInfo, "FastRestoreDecodedRangeFile")
-	    .detail("BatchIndex", asset.batchIndex)
-	    .detail("Filename", asset.filename)
-	    .detail("Version", version)
-	    .detail("BeginVersion", asset.beginVersion)
-	    .detail("EndVersion", asset.endVersion)
-	    .detail("RestoreAsset", asset.toString());
-	// Sanity check the range file is within the restored version range
-	ASSERT_WE_THINK(asset.isInVersionRange(version));
-
-	state Standalone<VectorRef<KeyValueRef>> blockData;
-	// should retry here
-	state int readFileRetries = 0;
-	loop {
-		try {
-			// The set of key value version is rangeFile.version. the key-value set in the same range file has the same
-			// version
-			Reference<IAsyncFile> inFile = wait(bc->readFile(asset.filename));
-			Standalone<VectorRef<KeyValueRef>> kvs =
-			    wait(fileBackup::decodeRangeFileBlock(inFile, asset.offset, asset.len, cx));
-			TraceEvent("FastRestoreLoaderDecodedRangeFile")
-			    .detail("BatchIndex", asset.batchIndex)
-			    .detail("Filename", asset.filename)
-			    .detail("DataSize", kvs.contents().size());
-			blockData = kvs;
-			break;
-		} catch (Error& e) {
-			if (e.code() == error_code_restore_bad_read || e.code() == error_code_restore_unsupported_file_version ||
-			    e.code() == error_code_restore_corrupted_data_padding) { // no retriable error
-				TraceEvent(SevError, "FastRestoreFileRestoreCorruptedRangeFileBlock").error(e);
-				throw;
-			} else if (e.code() == error_code_http_request_failed || e.code() == error_code_connection_failed ||
-			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
-				// blob http request failure, retry
-				TraceEvent(SevWarnAlways, "FastRestoreDecodedRangeFileConnectionFailure")
-				    .error(e)
-				    .detail("Retries", ++readFileRetries);
-				wait(delayJittered(0.1));
-			} else {
-				TraceEvent(SevError, "FastRestoreParseRangeFileOnLoaderUnexpectedError").error(e);
-				throw;
-			}
-		}
-	}
-
-	// First and last key are the range for this file
-	KeyRange fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
-
-	// If fileRange doesn't intersect restore range then we're done.
-	if (!fileRange.intersects(asset.range)) {
-		return Void();
-	}
-
-	// We know the file range intersects the restore range but there could still be keys outside the restore range.
-	// Find the subvector of kv pairs that intersect the restore range.
-	// Note that the first and last keys are just the range endpoints for this file.
-	// They are metadata, not the real data.
-	int rangeStart = 1;
-	int rangeEnd = blockData.size() - 1; // The rangeStart and rangeEnd is [,)
-
-	// Slide start from beginning, stop if something in range is found
-	// Move rangeStart and rangeEnd until they is within restoreRange
-	while (rangeStart < rangeEnd && !asset.range.contains(blockData[rangeStart].key)) {
-		++rangeStart;
-	}
-	// Side end from back, stop if something at (rangeEnd-1) is found in range
-	while (rangeEnd > rangeStart && !asset.range.contains(blockData[rangeEnd - 1].key)) {
-		--rangeEnd;
-	}
-
-	// Now data only contains the kv mutation within restoreRange
-	VectorRef<KeyValueRef> data = blockData.slice(rangeStart, rangeEnd);
-
-	// Note we give INT_MAX as the sub sequence number to override any log mutations.
-	const LogMessageVersion msgVersion(version, std::numeric_limits<int32_t>::max());
-
-	// Convert KV in data into SET mutations of different keys in kvOps
-	Arena tempArena;
-	for (const KeyValueRef& kv : data) {
-		// NOTE: The KV pairs in range files are the real KV pairs in original DB.
-		MutationRef m(MutationRef::Type::SetValue, kv.key, kv.value);
-		// Remove prefix or add prefix in case we restore data to a different sub keyspace
-		if (asset.hasPrefix()) { // Avoid creating new Key
-			ASSERT(asset.removePrefix.size() == 0);
-			m.param1 = m.param1.removePrefix(asset.removePrefix).withPrefix(asset.addPrefix, tempArena);
-		}
-
-		cc->loadedRangeBytes += m.totalSize();
-
-		// We cache all kv operations into kvOps, and apply all kv operations later in one place
-		auto it = kvOps.insert(std::make_pair(msgVersion, MutationsVec()));
-		TraceEvent(SevFRMutationInfo, "FastRestoreDecodeRangeFile")
-		    .detail("BatchIndex", asset.batchIndex)
-		    .detail("CommitVersion", version)
-		    .detail("ParsedMutationKV", m.toString());
-
-		it.first->second.push_back_deep(it.first->second.arena(), m);
-		// Sampling (FASTRESTORE_SAMPLING_PERCENT%) data
-		ByteSampleInfo sampleInfo = isKeyValueInSample(KeyValueRef(m.param1, m.param2));
-		if (sampleInfo.inSample) {
-			cc->sampledRangeBytes += sampleInfo.sampledSize;
-			sampleMutations.push_back_deep(sampleMutations.arena(), SampledMutation(m.param1, sampleInfo.sampledSize));
-		}
-	}
-
-	return Void();
-}
-
-// Parse data blocks in a log file into a vector of <string, string> pairs.
-// Each pair.second contains the mutations at a version encoded in pair.first;
-// Step 1: decodeLogFileBlock into <string, string> pairs;
-// Step 2: Concatenate the second of pairs with the same pair.first.
-// pProcessedFileOffset: ensure each data block is processed in order exactly once;
-// pMutationMap: concatenated mutation list string at the mutation's commit version
-ACTOR static Future<Void> _parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
-                                                           SerializedMutationListMap* pMutationMap,
-                                                           Reference<IBackupContainer> bc,
-                                                           RestoreAsset asset) {
-	Reference<IAsyncFile> inFile = wait(bc->readFile(asset.filename));
-	// decodeLogFileBlock() must read block by block!
-	state Standalone<VectorRef<KeyValueRef>> data =
-	    wait(parallelFileRestore::decodeLogFileBlock(inFile, asset.offset, asset.len));
-	TraceEvent("FastRestoreLoaderDecodeLogFile")
-	    .detail("BatchIndex", asset.batchIndex)
-	    .detail("RestoreAsset", asset.toString())
-	    .detail("DataSize", data.contents().size());
-
-	// Ensure data blocks in the same file are processed in order
-	wait(pProcessedFileOffset->whenAtLeast(asset.offset));
-
-	if (pProcessedFileOffset->get() == asset.offset) {
-		for (const KeyValueRef& kv : data) {
-			// Concatenate the backup param1 and param2 (KV) at the same version.
-			concatenateBackupMutationForLogFile(pMutationMap, kv.key, kv.value, asset);
-		}
-		pProcessedFileOffset->set(asset.offset + asset.len);
-	}
-
-	return Void();
-}
-
-// retry on _parseLogFileToMutationsOnLoader
-ACTOR static Future<Void> parseLogFileToMutationsOnLoader(NotifiedVersion* pProcessedFileOffset,
-                                                          SerializedMutationListMap* pMutationMap,
-                                                          Reference<IBackupContainer> bc,
-                                                          RestoreAsset asset) {
-	state int readFileRetries = 0;
-	loop {
-		try {
-			wait(_parseLogFileToMutationsOnLoader(pProcessedFileOffset, pMutationMap, bc, asset));
-			break;
-		} catch (Error& e) {
-			if (e.code() == error_code_restore_bad_read || e.code() == error_code_restore_unsupported_file_version ||
-			    e.code() == error_code_restore_corrupted_data_padding) { // non retriable error
-				TraceEvent(SevError, "FastRestoreFileRestoreCorruptedLogFileBlock").error(e);
-				throw;
-			} else if (e.code() == error_code_http_request_failed || e.code() == error_code_connection_failed ||
-			           e.code() == error_code_timed_out || e.code() == error_code_lookup_failed) {
-				// blob http request failure, retry
-				TraceEvent(SevWarnAlways, "FastRestoreDecodedLogFileConnectionFailure")
-				    .error(e)
-				    .detail("Retries", ++readFileRetries);
-				wait(delayJittered(0.1));
-			} else {
-				TraceEvent(SevError, "FastRestoreParseLogFileToMutationsOnLoaderUnexpectedError").error(e);
-				throw;
-			}
-		}
-	}
-	return Void();
-}
-
-// Return applier IDs that are used to apply key-values
-std::vector<UID> getApplierIDs(std::map<Key, UID>& rangeToApplier) {
-	std::vector<UID> applierIDs;
-	applierIDs.reserve(rangeToApplier.size());
-	for (auto& applier : rangeToApplier) {
-		applierIDs.push_back(applier.second);
-	}
-
-	ASSERT(!applierIDs.empty());
-	return applierIDs;
-}
-
-// Notify loaders that the version batch (index) has been applied.
-// This affects which version batch each loader can release actors even when the worker has low memory
-ACTOR Future<Void> handleFinishVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreLoaderData> self) {
-	// Ensure batch (i-1) is applied before batch i
-	TraceEvent("FastRestoreLoaderHandleFinishVersionBatch", self->id())
-	    .detail("FinishedBatchIndex", self->finishedBatch.get())
-	    .detail("RequestedBatchIndex", req.batchIndex);
-	wait(self->finishedBatch.whenAtLeast(req.batchIndex - 1));
-	if (self->finishedBatch.get() == req.batchIndex - 1) {
-		// Sanity check: All requests before and in this batchIndex must have been processed; otherwise,
-		// those requests may cause segmentation fault after applier remove the batch data
-		while (!self->loadingQueue.empty() && self->loadingQueue.top().batchIndex <= req.batchIndex) {
-			// Still has pending requests from earlier batchIndex  and current batchIndex, which should not happen
-			TraceEvent(SevWarn, "FastRestoreLoaderHasPendingLoadFileRequests")
-			    .detail("PendingRequest", self->loadingQueue.top().toString());
-			self->loadingQueue.pop();
-		}
-		while (!self->sendingQueue.empty() && self->sendingQueue.top().batchIndex <= req.batchIndex) {
-			TraceEvent(SevWarn, "FastRestoreLoaderHasPendingSendRequests")
-			    .detail("PendingRequest", self->sendingQueue.top().toString());
-			self->sendingQueue.pop();
-		}
-		while (!self->sendLoadParamQueue.empty() && self->sendLoadParamQueue.top().batchIndex <= req.batchIndex) {
-			TraceEvent(SevWarn, "FastRestoreLoaderHasPendingSendLoadParamRequests")
-			    .detail("PendingRequest", self->sendLoadParamQueue.top().toString());
-			self->sendLoadParamQueue.pop();
-		}
-
-		self->finishedBatch.set(req.batchIndex);
-		// Clean up batchData
-		self->batch.erase(req.batchIndex);
-		self->status.erase(req.batchIndex);
-	}
-	if (self->delayedActors > 0) {
-		self->checkMemory.trigger();
-	}
-	req.reply.send(RestoreCommonReply(self->id(), false));
-	return Void();
-}
-
-namespace {
-
-void oldSplitMutation(std::map<Key, UID>* pRangeToApplier,
-                      MutationRef m,
-                      Arena& mvector_arena,
-                      VectorRef<MutationRef>& mvector,
-                      Arena& nodeIDs_arena,
-                      VectorRef<UID>& nodeIDs) {
-	// mvector[i] should be mapped to nodeID[i]
-	ASSERT(mvector.empty());
-	ASSERT(nodeIDs.empty());
-	// key range [m->param1, m->param2)
-	std::map<Key, UID>::iterator itlow, itup; // we will return [itlow, itup)
-	itlow = pRangeToApplier->lower_bound(m.param1); // lower_bound returns the iterator that is >= m.param1
-	if (itlow == pRangeToApplier->end()) {
-		--itlow;
-		mvector.push_back_deep(mvector_arena, m);
-		nodeIDs.push_back(nodeIDs_arena, itlow->second);
-		return;
-	}
-	if (itlow->first > m.param1) {
-		if (itlow != pRangeToApplier->begin()) {
-			--itlow;
-		}
-	}
-
-	itup = pRangeToApplier->upper_bound(m.param2); // return rmap::end if no key is after m.param2.
-	ASSERT(itup == pRangeToApplier->end() || itup->first > m.param2);
-
-	std::map<Key, UID>::iterator itApplier;
-	while (itlow != itup) {
-		Standalone<MutationRef> curm; // current mutation
-		curm.type = m.type;
-		// The first split mutation should starts with m.first.
-		// The later ones should start with the rangeToApplier boundary.
-		if (m.param1 > itlow->first) {
-			curm.param1 = m.param1;
-		} else {
-			curm.param1 = itlow->first;
-		}
-		itApplier = itlow;
-		itlow++;
-		if (itlow == itup) {
-			ASSERT(m.param2 <= allKeys.end);
-			curm.param2 = m.param2;
-		} else if (m.param2 < itlow->first) {
-			UNREACHABLE();
-			curm.param2 = m.param2;
-		} else {
-			curm.param2 = itlow->first;
-		}
-		ASSERT(curm.param1 <= curm.param2);
-		// itup > m.param2: (itup-1) may be out of mutation m's range
-		// Ensure the added mutations have overlap with mutation m
-		if (m.param1 < curm.param2 && m.param2 > curm.param1) {
-			mvector.push_back_deep(mvector_arena, curm);
-			nodeIDs.push_back(nodeIDs_arena, itApplier->second);
-		}
-	}
-}
-
-// Test splitMutation
-TEST_CASE("/FastRestore/RestoreLoader/splitMutation") {
-	std::map<Key, UID> rangeToApplier;
-	MutationsVec mvector;
-	Standalone<VectorRef<UID>> nodeIDs;
-
-	// Prepare RangeToApplier
-	rangeToApplier.emplace(allKeys.begin, deterministicRandom()->randomUniqueID());
-	int numAppliers = deterministicRandom()->randomInt(1, 50);
-	for (int i = 0; i < numAppliers; ++i) {
-		Key k = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(1, 1000)));
-		UID node = deterministicRandom()->randomUniqueID();
-		rangeToApplier.emplace(k, node);
-		TraceEvent("RangeToApplier").detail("Key", k).detail("Node", node);
-	}
-	Key k1 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(1, 500)));
-	Key k2 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(1, 1000)));
-	Key beginK = k1 < k2 ? k1 : k2;
-	Key endK = k1 < k2 ? k2 : k1;
-	Standalone<MutationRef> mutation(MutationRef(MutationRef::ClearRange, beginK.contents(), endK.contents()));
-
-	// Method 1: Use old splitMutation
-	oldSplitMutation(
-	    &rangeToApplier, mutation, mvector.arena(), mvector.contents(), nodeIDs.arena(), nodeIDs.contents());
-	ASSERT(mvector.size() == nodeIDs.size());
-
-	// Method 2: Use new intersection based method
-	KeyRangeMap<UID> krMap;
-	buildApplierRangeMap(&krMap, &rangeToApplier);
-
-	MutationsVec mvector2;
-	Standalone<VectorRef<UID>> nodeIDs2;
-	splitMutation(krMap, mutation, mvector2.arena(), mvector2.contents(), nodeIDs2.arena(), nodeIDs2.contents());
-	ASSERT(mvector2.size() == nodeIDs2.size());
-
-	ASSERT(mvector.size() == mvector2.size());
-	int splitMutationIndex = 0;
-	for (; splitMutationIndex < mvector.size(); splitMutationIndex++) {
-		MutationRef result = mvector[splitMutationIndex];
-		MutationRef result2 = mvector2[splitMutationIndex];
-		UID applierID = nodeIDs[splitMutationIndex];
-		UID applierID2 = nodeIDs2[splitMutationIndex];
-		KeyRange krange(KeyRangeRef(result.param1, result.param2));
-		KeyRange krange2(KeyRangeRef(result2.param1, result2.param2));
-		TraceEvent("Result")
-		    .detail("KeyRange1", krange)
-		    .detail("KeyRange2", krange2)
-		    .detail("ApplierID1", applierID)
-		    .detail("ApplierID2", applierID2);
-		if (krange != krange2 || applierID != applierID2) {
-			TraceEvent(SevError, "IncorrectResult")
-			    .detail("Mutation", mutation)
-			    .detail("KeyRange1", krange)
-			    .detail("KeyRange2", krange2)
-			    .detail("ApplierID1", applierID)
-			    .detail("ApplierID2", applierID2);
-		}
-	}
-
-	return Void();
-}
-
-} // namespace
diff --git a/fdbserver/RestoreRoleCommon.actor.cpp b/fdbserver/RestoreRoleCommon.actor.cpp
deleted file mode 100644
index 67e82743e7e..00000000000
--- a/fdbserver/RestoreRoleCommon.actor.cpp
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * RestoreRoleCommon.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreLoader.actor.h"
-#include "fdbserver/RestoreApplier.actor.h"
-#include "fdbserver/RestoreController.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-class Database;
-struct RestoreWorkerData;
-
-// id is the id of the worker to be monitored
-// This actor is used for both restore loader and restore applier
-ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id) {
-	wait(delayJittered(5.0)); // Random jitter reduces heat beat monitor's pressure
-	req.reply.send(RestoreCommonReply(id));
-	return Void();
-}
-
-void handleFinishRestoreRequest(const RestoreFinishRequest& req, Reference<RestoreRoleData> self) {
-	self->resetPerRestoreRequest();
-	TraceEvent("FastRestoreRolePhaseFinishRestoreRequest", self->id())
-	    .detail("FinishRestoreRequest", req.terminate)
-	    .detail("Role", getRoleStr(self->role));
-
-	req.reply.send(RestoreCommonReply(self->id()));
-}
-
-// Multiple version batches may execute in parallel and init their version batches
-ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self) {
-	TraceEvent("FastRestoreRolePhaseInitVersionBatch", self->id())
-	    .detail("BatchIndex", req.batchIndex)
-	    .detail("Role", getRoleStr(self->role))
-	    .detail("VersionBatchNotifiedVersion", self->versionBatchId.get());
-	// Loader destroy batchData once the batch finishes and self->finishedBatch.set(req.batchIndex);
-	ASSERT(self->finishedBatch.get() < req.batchIndex);
-
-	// batchId is continuous. (req.batchIndex-1) is the id of the just finished batch.
-	wait(self->versionBatchId.whenAtLeast(req.batchIndex - 1));
-
-	if (self->versionBatchId.get() == req.batchIndex - 1) {
-		self->initVersionBatch(req.batchIndex);
-		self->setVersionBatchState(req.batchIndex, ApplierVersionBatchState::INIT);
-		TraceEvent("FastRestoreInitVersionBatch")
-		    .detail("BatchIndex", req.batchIndex)
-		    .detail("Role", getRoleStr(self->role))
-		    .detail("Node", self->id());
-		self->versionBatchId.set(req.batchIndex);
-	}
-
-	req.reply.send(RestoreCommonReply(self->id()));
-	return Void();
-}
-
-void updateProcessStats(Reference<RestoreRoleData> self) {
-	if (g_network->isSimulated()) {
-		// memUsage and cpuUsage are not relevant in the simulator,
-		// and relying on the actual values could break seed determinism
-		if (deterministicRandom()->random01() < 0.2) { // not fully utilized cpu
-			self->cpuUsage = deterministicRandom()->random01() * SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT;
-		} else if (deterministicRandom()->random01() < 0.6) { // achieved target cpu but cpu is not busy
-			self->cpuUsage = SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT +
-			                 deterministicRandom()->random01() * (SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT -
-			                                                      SERVER_KNOBS->FASTRESTORE_SCHED_TARGET_CPU_PERCENT);
-		} else { // reach desired max cpu usage; use max cpu as 200 to simulate incorrect cpu profiling
-			self->cpuUsage =
-			    SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT +
-			    deterministicRandom()->random01() * (200 - SERVER_KNOBS->FASTRESTORE_SCHED_MAX_CPU_PERCENT);
-		}
-		self->memory = 100.0;
-		self->residentMemory = 100.0;
-		return;
-	}
-
-	SystemStatistics sysStats = getSystemStatistics();
-	if (sysStats.initialized) {
-		self->cpuUsage = 100 * sysStats.processCPUSeconds / sysStats.elapsed;
-		self->memory = sysStats.processMemory;
-		self->residentMemory = sysStats.processResidentMemory;
-	}
-}
-
-// An actor is schedulable to run if the current worker has enough resources, i.e.,
-// the worker's memory usage is below the threshold;
-// Exception: If the actor is working on the current version batch, we have to schedule
-// the actor to run to avoid dead-lock.
-// Future: When we release the actors that are blocked by memory usage, we should release them
-// in increasing order of their version batch.
-ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name) {
-	self->delayedActors++;
-	state double memoryThresholdBytes = SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT * 1024 * 1024;
-	loop {
-		double memory = getSystemStatistics().processMemory;
-		if (g_network->isSimulated() && BUGGIFY) {
-			// Intentionally randomly block actors for low memory reason.
-			// memory will be larger than threshold when deterministicRandom()->random01() > 1/2
-			if (deterministicRandom()->random01() < 0.4) { // enough memory
-				memory = SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT * deterministicRandom()->random01();
-			} else { // used too much memory, needs throttling
-				memory = SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT +
-				         deterministicRandom()->random01() * SERVER_KNOBS->FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT;
-			}
-		}
-		if (memory < memoryThresholdBytes || self->finishedBatch.get() + 1 == actorBatchIndex) {
-			if (memory >= memoryThresholdBytes) {
-				TraceEvent(SevWarn, "FastRestoreMemoryUsageAboveThreshold", self->id())
-				    .suppressFor(5.0)
-				    .detail("Role", getRoleStr(self->role))
-				    .detail("BatchIndex", actorBatchIndex)
-				    .detail("FinishedBatch", self->finishedBatch.get())
-				    .detail("Actor", name)
-				    .detail("Memory", memory);
-			}
-			self->delayedActors--;
-			break;
-		} else {
-			TraceEvent(SevInfo, "FastRestoreMemoryUsageAboveThresholdWait", self->id())
-			    .suppressFor(5.0)
-			    .detail("Role", getRoleStr(self->role))
-			    .detail("BatchIndex", actorBatchIndex)
-			    .detail("Actor", name)
-			    .detail("CurrentMemory", memory);
-			// TODO: Set FASTRESTORE_WAIT_FOR_MEMORY_LATENCY to a large value. It should be able to avoided
-			wait(delay(SERVER_KNOBS->FASTRESTORE_WAIT_FOR_MEMORY_LATENCY) || self->checkMemory.onTrigger());
-		}
-	}
-	return Void();
-}
-
-// Updated process metrics will be used by scheduler for throttling as well
-ACTOR Future<Void> updateProcessMetrics(Reference<RestoreRoleData> self) {
-	loop {
-		updateProcessStats(self);
-		wait(delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL));
-	}
-}
-
-ACTOR Future<Void> traceProcessMetrics(Reference<RestoreRoleData> self, std::string role) {
-	loop {
-		TraceEvent("FastRestoreTraceProcessMetrics", self->nodeID)
-		    .detail("Role", role)
-		    .detail("PipelinedMaxVersionBatchIndex", self->versionBatchId.get())
-		    .detail("FinishedVersionBatchIndex", self->finishedBatch.get())
-		    .detail("CurrentVersionBatchPhase", self->getVersionBatchState(self->finishedBatch.get() + 1))
-		    .detail("CpuUsage", self->cpuUsage)
-		    .detail("UsedMemory", self->memory)
-		    .detail("ResidentMemory", self->residentMemory);
-		wait(delay(SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY));
-	}
-}
-
-ACTOR Future<Void> traceRoleVersionBatchProgress(Reference<RestoreRoleData> self, std::string role) {
-	loop {
-		int batchIndex = self->finishedBatch.get();
-		int maxBatchIndex = self->versionBatchId.get();
-		int maxPrintBatchIndex = batchIndex + SERVER_KNOBS->FASTRESTORE_VB_PARALLELISM;
-
-		TraceEvent ev("FastRestoreVersionBatchProgressState", self->nodeID);
-		ev.detail("Role", role)
-		    .detail("Node", self->nodeID)
-		    .detail("FinishedBatch", batchIndex)
-		    .detail("InitializedBatch", maxBatchIndex);
-		while (batchIndex <= maxBatchIndex) {
-			if (batchIndex > maxPrintBatchIndex) {
-				ev.detail("SkipVersionBatches", maxBatchIndex - batchIndex + 1);
-				break;
-			}
-			std::stringstream typeName;
-			typeName << "VersionBatch" << batchIndex;
-			ev.detail(typeName.str(), self->getVersionBatchState(batchIndex));
-			batchIndex++;
-		}
-
-		wait(delay(SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY));
-	}
-}
-
-//-------Helper functions
-std::string getHexString(StringRef input) {
-	std::stringstream ss;
-	for (int i = 0; i < input.size(); i++) {
-		if (i % 4 == 0)
-			ss << " ";
-		if (i == 12) { // The end of 12bytes, which is the version size for value
-			ss << "|";
-		}
-		if (i == (12 + 12)) { // The end of version + header
-			ss << "@";
-		}
-		ss << std::setfill('0') << std::setw(2) << std::hex
-		   << (int)input[i]; // [] operator moves the pointer in step of unit8
-	}
-	return ss.str();
-}
diff --git a/fdbserver/RestoreUtil.actor.cpp b/fdbserver/RestoreUtil.actor.cpp
deleted file mode 100644
index ec2496919bf..00000000000
--- a/fdbserver/RestoreUtil.actor.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * RestoreUtil.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file implements the functions defined in RestoreUtil.h
-
-#include "fdbserver/RestoreUtil.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-const std::vector<std::string> RestoreRoleStr = { "Invalid", "Controller", "Loader", "Applier" };
-int numRoles = RestoreRoleStr.size();
-
-// Similar to debugMutation(), we use debugFRMutation to track mutations for fast restore systems only.
-#if CENABLED(0, NOT_IN_CLEAN)
-StringRef debugFRKey = "\xff\xff\xff\xff"_sr;
-
-// Track any mutation in fast restore that has overlap with debugFRKey
-bool debugFRMutation(const char* context, Version version, MutationRef const& mutation) {
-	if (mutation.type != mutation.ClearRange && mutation.param1 == debugFRKey) { // Single key mutation
-		TraceEvent("FastRestoreMutationTracking")
-		    .detail("At", context)
-		    .detail("Version", version)
-		    .detail("MutationType", getTypeString((MutationRef::Type)mutation.type))
-		    .detail("Key", mutation.param1)
-		    .detail("Value", mutation.param2);
-	} else if (mutation.type == mutation.ClearRange && debugFRKey >= mutation.param1 &&
-	           debugFRKey < mutation.param2) { // debugFRKey is in the range mutation
-		TraceEvent("FastRestoreMutationTracking")
-		    .detail("At", context)
-		    .detail("Version", version)
-		    .detail("MutationType", getTypeString((MutationRef::Type)mutation.type))
-		    .detail("Begin", mutation.param1)
-		    .detail("End", mutation.param2);
-	} else
-		return false;
-
-	return true;
-}
-#else
-// Default implementation.
-bool debugFRMutation(const char* context, Version version, MutationRef const& mutation) {
-	return false;
-}
-#endif
-
-std::string getRoleStr(RestoreRole role) {
-	if ((int)role >= numRoles || (int)role < 0) {
-		printf("[ERROR] role:%d is out of scope\n", (int)role);
-		return "[Unset]";
-	}
-	return RestoreRoleStr[(int)role];
-}
-
-bool isRangeMutation(MutationRef m) {
-	if (m.type == MutationRef::Type::ClearRange) {
-		ASSERT(m.type != MutationRef::Type::DebugKeyRange);
-		return true;
-	} else {
-		ASSERT(m.type == MutationRef::Type::SetValue || isAtomicOp((MutationRef::Type)m.type));
-		return false;
-	}
-}
diff --git a/fdbserver/RestoreWorker.actor.cpp b/fdbserver/RestoreWorker.actor.cpp
deleted file mode 100644
index d883e786a55..00000000000
--- a/fdbserver/RestoreWorker.actor.cpp
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * RestoreWorker.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <ctime>
-#include <climits>
-#include <numeric>
-#include <algorithm>
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/classification.hpp>
-
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/SystemData.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/MutationList.h"
-#include "fdbclient/BackupContainer.h"
-#include "flow/ApiVersion.h"
-#include "flow/IAsyncFile.h"
-#include "fdbrpc/simulator.h"
-#include "flow/genericactors.actor.h"
-#include "flow/Hash3.h"
-#include "flow/ActorCollection.h"
-#include "fdbserver/RestoreWorker.actor.h"
-#include "fdbserver/RestoreController.actor.h"
-#include "fdbrpc/SimulatorProcessInfo.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-class RestoreConfigFR;
-struct RestoreWorkerData; // Only declare the struct exist but we cannot use its field
-
-ACTOR Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req,
-                                                 Reference<RestoreWorkerData> self,
-                                                 RestoreWorkerInterface workerInterf,
-                                                 Database cx);
-ACTOR Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self);
-void handleRecruitRoleRequest(RestoreRecruitRoleRequest req,
-                              Reference<RestoreWorkerData> self,
-                              ActorCollection* actors,
-                              Database cx);
-ACTOR Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self,
-                                                 Database cx,
-                                                 int min_num_workers = 2);
-ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader,
-                                 Database cx,
-                                 RestoreWorkerInterface myWorkerInterf);
-ACTOR Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self,
-                                            RestoreWorkerInterface workerInterf,
-                                            Database cx);
-
-// Remove the worker interface from restoreWorkerKey and remove its roles interfaces from their keys.
-ACTOR Future<Void> handlerTerminateWorkerRequest(RestoreSimpleRequest req,
-                                                 Reference<RestoreWorkerData> self,
-                                                 RestoreWorkerInterface workerInterf,
-                                                 Database cx) {
-	wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-		tr->clear(restoreWorkerKeyFor(workerInterf.id()));
-		return Void();
-	}));
-
-	TraceEvent("FastRestoreWorker").detail("HandleTerminateWorkerReq", self->id());
-
-	return Void();
-}
-
-// Assume only 1 role on a restore worker.
-// Future: Multiple roles in a restore worker
-void handleRecruitRoleRequest(RestoreRecruitRoleRequest req,
-                              Reference<RestoreWorkerData> self,
-                              ActorCollection* actors,
-                              Database cx) {
-	// Future: Allow multiple restore roles on a restore worker. The design should easily allow this.
-	ASSERT(!self->loaderInterf.present() || !self->applierInterf.present()); // Only one role per worker for now
-	// Already recruited a role
-	if (self->loaderInterf.present()) {
-		ASSERT(req.role == RestoreRole::Loader);
-		req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Loader, self->loaderInterf.get()));
-		return;
-	} else if (self->applierInterf.present()) {
-		req.reply.send(RestoreRecruitRoleReply(self->id(), RestoreRole::Applier, self->applierInterf.get()));
-		return;
-	}
-
-	if (req.role == RestoreRole::Loader) {
-		ASSERT(!self->loaderInterf.present());
-		self->controllerInterf = req.ci;
-		self->loaderInterf = RestoreLoaderInterface();
-		self->loaderInterf.get().initEndpoints();
-		RestoreLoaderInterface& recruited = self->loaderInterf.get();
-		DUMPTOKEN(recruited.heartbeat);
-		DUMPTOKEN(recruited.updateRestoreSysInfo);
-		DUMPTOKEN(recruited.initVersionBatch);
-		DUMPTOKEN(recruited.loadFile);
-		DUMPTOKEN(recruited.sendMutations);
-		DUMPTOKEN(recruited.initVersionBatch);
-		DUMPTOKEN(recruited.finishVersionBatch);
-		DUMPTOKEN(recruited.collectRestoreRoleInterfaces);
-		DUMPTOKEN(recruited.finishRestore);
-		actors->add(restoreLoaderCore(self->loaderInterf.get(), req.nodeIndex, cx, req.ci));
-		TraceEvent("FastRestoreWorker").detail("RecruitedLoaderNodeIndex", req.nodeIndex);
-		req.reply.send(
-		    RestoreRecruitRoleReply(self->loaderInterf.get().id(), RestoreRole::Loader, self->loaderInterf.get()));
-	} else if (req.role == RestoreRole::Applier) {
-		ASSERT(!self->applierInterf.present());
-		self->controllerInterf = req.ci;
-		self->applierInterf = RestoreApplierInterface();
-		self->applierInterf.get().initEndpoints();
-		RestoreApplierInterface& recruited = self->applierInterf.get();
-		DUMPTOKEN(recruited.heartbeat);
-		DUMPTOKEN(recruited.sendMutationVector);
-		DUMPTOKEN(recruited.applyToDB);
-		DUMPTOKEN(recruited.initVersionBatch);
-		DUMPTOKEN(recruited.collectRestoreRoleInterfaces);
-		DUMPTOKEN(recruited.finishRestore);
-		actors->add(restoreApplierCore(self->applierInterf.get(), req.nodeIndex, cx));
-		TraceEvent("FastRestoreWorker").detail("RecruitedApplierNodeIndex", req.nodeIndex);
-		req.reply.send(
-		    RestoreRecruitRoleReply(self->applierInterf.get().id(), RestoreRole::Applier, self->applierInterf.get()));
-	} else {
-		TraceEvent(SevError, "FastRestoreWorkerHandleRecruitRoleRequestUnknownRole").detail("Request", req.toString());
-	}
-
-	return;
-}
-
-// Read restoreWorkersKeys from DB to get each restore worker's workerInterface and set it to self->workerInterfaces;
-// This is done before we assign restore roles for restore workers.
-ACTOR Future<Void> collectRestoreWorkerInterface(Reference<RestoreWorkerData> self, Database cx, int min_num_workers) {
-	state Transaction tr(cx);
-	state std::vector<RestoreWorkerInterface> agents; // agents is cmdsInterf
-
-	loop {
-		try {
-			self->workerInterfaces.clear();
-			agents.clear();
-			tr.reset();
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			RangeResult agentValues = wait(tr.getRange(restoreWorkersKeys, CLIENT_KNOBS->TOO_MANY));
-			ASSERT(!agentValues.more);
-			// If agentValues.size() < min_num_workers, we should wait for coming workers to register their
-			// workerInterface before we read them once for all
-			if (agentValues.size() >= min_num_workers) {
-				for (auto& it : agentValues) {
-					agents.push_back(BinaryReader::fromStringRef<RestoreWorkerInterface>(it.value, IncludeVersion()));
-					// Save the RestoreWorkerInterface for the later operations
-					self->workerInterfaces.insert(std::make_pair(agents.back().id(), agents.back()));
-				}
-				break;
-			}
-			TraceEvent("FastRestoreWorker")
-			    .suppressFor(10.0)
-			    .detail("NotEnoughWorkers", agentValues.size())
-			    .detail("MinWorkers", min_num_workers);
-			wait(delay(5.0));
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-	ASSERT(agents.size() >= min_num_workers); // ASSUMPTION: We must have at least 1 loader and 1 applier
-
-	TraceEvent("FastRestoreWorker").detail("CollectWorkerInterfaceNumWorkers", self->workerInterfaces.size());
-
-	return Void();
-}
-
-// Periodically send worker heartbeat to
-ACTOR Future<Void> monitorWorkerLiveness(Reference<RestoreWorkerData> self) {
-	ASSERT(!self->workerInterfaces.empty());
-
-	state std::map<UID, RestoreWorkerInterface>::iterator workerInterf;
-	loop {
-		std::vector<std::pair<UID, RestoreSimpleRequest>> requests;
-		for (auto& worker : self->workerInterfaces) {
-			requests.emplace_back(worker.first, RestoreSimpleRequest());
-		}
-		wait(sendBatchRequests(&RestoreWorkerInterface::heartbeat, self->workerInterfaces, requests));
-		wait(delay(60.0));
-	}
-}
-
-// RestoreWorkerLeader is the worker that runs RestoreController role
-ACTOR Future<Void> startRestoreWorkerLeader(Reference<RestoreWorkerData> self,
-                                            RestoreWorkerInterface workerInterf,
-                                            Database cx) {
-	// We must wait for enough time to make sure all restore workers have registered their workerInterfaces into the DB
-	TraceEvent("FastRestoreWorker")
-	    .detail("Controller", workerInterf.id())
-	    .detail("WaitForRestoreWorkerInterfaces",
-	            SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS);
-	wait(delay(10.0));
-	TraceEvent("FastRestoreWorker")
-	    .detail("Controller", workerInterf.id())
-	    .detail("CollectRestoreWorkerInterfaces",
-	            SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS);
-
-	wait(collectRestoreWorkerInterface(
-	    self, cx, SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS));
-
-	// TODO: Needs to keep this monitor's future. May use actorCollection
-	state Future<Void> workersFailureMonitor = monitorWorkerLiveness(self);
-
-	RestoreControllerInterface recruited;
-	DUMPTOKEN(recruited.samples);
-
-	self->controllerInterf = recruited;
-	wait(startRestoreController(self, cx) || workersFailureMonitor);
-
-	return Void();
-}
-
-ACTOR Future<Void> startRestoreWorker(Reference<RestoreWorkerData> self, RestoreWorkerInterface interf, Database cx) {
-	state double lastLoopTopTime = now();
-	state ActorCollection actors(false); // Collect the main actor for each role
-	state Future<Void> exitRole = Never();
-
-	loop {
-		double loopTopTime = now();
-		double elapsedTime = loopTopTime - lastLoopTopTime;
-		if (elapsedTime > 0.050) {
-			if (deterministicRandom()->random01() < 0.01)
-				TraceEvent(SevWarn, "SlowRestoreWorkerLoopx100")
-				    .detail("NodeDesc", self->describeNode())
-				    .detail("Elapsed", elapsedTime);
-		}
-		lastLoopTopTime = loopTopTime;
-		state std::string requestTypeStr = "[Init]";
-
-		try {
-			choose {
-				when(RestoreSimpleRequest req = waitNext(interf.heartbeat.getFuture())) {
-					requestTypeStr = "heartbeat";
-					actors.add(handleHeartbeat(req, interf.id()));
-				}
-				when(RestoreRecruitRoleRequest req = waitNext(interf.recruitRole.getFuture())) {
-					requestTypeStr = "recruitRole";
-					handleRecruitRoleRequest(req, self, &actors, cx);
-				}
-				when(RestoreSimpleRequest req = waitNext(interf.terminateWorker.getFuture())) {
-					// Destroy the worker at the end of the restore
-					requestTypeStr = "terminateWorker";
-					exitRole = handlerTerminateWorkerRequest(req, self, interf, cx);
-				}
-				when(wait(exitRole)) {
-					TraceEvent("FastRestoreWorkerCoreExitRole", self->id());
-					break;
-				}
-			}
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "FastRestoreWorkerError").errorUnsuppressed(e).detail("RequestType", requestTypeStr);
-			break;
-		}
-	}
-
-	return Void();
-}
-
-ACTOR static Future<Void> waitOnRestoreRequests(Database cx, UID nodeID = UID()) {
-	state ReadYourWritesTransaction tr(cx);
-	state Optional<Value> numRequests;
-
-	// wait for the restoreRequestTriggerKey to be set by the client/test workload
-	TraceEvent("FastRestoreWaitOnRestoreRequest", nodeID).log();
-	loop {
-		try {
-			tr.reset();
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> _numRequests = wait(tr.get(restoreRequestTriggerKey));
-			numRequests = _numRequests;
-			if (!numRequests.present()) {
-				state Future<Void> watchForRestoreRequest = tr.watch(restoreRequestTriggerKey);
-				wait(tr.commit());
-				TraceEvent(SevInfo, "FastRestoreWaitOnRestoreRequestTriggerKey", nodeID).log();
-				wait(watchForRestoreRequest);
-				TraceEvent(SevInfo, "FastRestoreDetectRestoreRequestTriggerKeyChanged", nodeID).log();
-			} else {
-				TraceEvent(SevInfo, "FastRestoreRestoreRequestTriggerKey", nodeID)
-				    .detail("TriggerKey", numRequests.get().toString());
-				break;
-			}
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
-
-	return Void();
-}
-
-// RestoreController is the leader
-ACTOR Future<Void> monitorleader(Reference<AsyncVar<RestoreWorkerInterface>> leader,
-                                 Database cx,
-                                 RestoreWorkerInterface myWorkerInterf) {
-	wait(delay(SERVER_KNOBS->FASTRESTORE_MONITOR_LEADER_DELAY));
-	TraceEvent("FastRestoreWorker", myWorkerInterf.id()).detail("MonitorLeader", "StartLeaderElection");
-	state int count = 0;
-	state RestoreWorkerInterface leaderInterf;
-	state ReadYourWritesTransaction tr(cx); // MX: Somewhere here program gets stuck
-	loop {
-		try {
-			count++;
-			tr.reset();
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> leaderValue = wait(tr.get(restoreLeaderKey));
-			TraceEvent(SevInfo, "FastRestoreLeaderElection")
-			    .detail("Round", count)
-			    .detail("LeaderExisted", leaderValue.present());
-			if (leaderValue.present()) {
-				leaderInterf = BinaryReader::fromStringRef<RestoreWorkerInterface>(leaderValue.get(), IncludeVersion());
-				// Register my interface as an worker if I am not the leader
-				if (leaderInterf != myWorkerInterf) {
-					tr.set(restoreWorkerKeyFor(myWorkerInterf.id()), restoreWorkerInterfaceValue(myWorkerInterf));
-				}
-			} else {
-				// Workers compete to be the leader
-				tr.set(restoreLeaderKey,
-				       BinaryWriter::toValue(myWorkerInterf,
-				                             IncludeVersion(ProtocolVersion::withRestoreWorkerInterfaceValue())));
-				leaderInterf = myWorkerInterf;
-			}
-			wait(tr.commit());
-			leader->set(leaderInterf);
-			break;
-		} catch (Error& e) {
-			TraceEvent(SevInfo, "FastRestoreLeaderElection").detail("ErrorCode", e.code()).detail("Error", e.what());
-			wait(tr.onError(e));
-		}
-	}
-
-	TraceEvent("FastRestoreWorker", myWorkerInterf.id())
-	    .detail("MonitorLeader", "FinishLeaderElection")
-	    .detail("Leader", leaderInterf.id())
-	    .detail("IamLeader", leaderInterf == myWorkerInterf);
-	return Void();
-}
-
-ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality) {
-	state ActorCollection actors(false);
-	state Future<Void> myWork = Never();
-	state Reference<AsyncVar<RestoreWorkerInterface>> leader = makeReference<AsyncVar<RestoreWorkerInterface>>();
-	state RestoreWorkerInterface myWorkerInterf;
-	state Reference<RestoreWorkerData> self = makeReference<RestoreWorkerData>();
-
-	myWorkerInterf.initEndpoints();
-	self->workerID = myWorkerInterf.id();
-
-	// Protect restore worker from being killed in simulation;
-	// Future: Remove the protection once restore can tolerate failure
-	if (g_network->isSimulated()) {
-		auto addresses = g_simulator->getProcessByAddress(myWorkerInterf.address())->addresses;
-
-		g_simulator->protectedAddresses.insert(addresses.address);
-		if (addresses.secondaryAddress.present()) {
-			g_simulator->protectedAddresses.insert(addresses.secondaryAddress.get());
-		}
-		ISimulator::ProcessInfo* p = g_simulator->getProcessByAddress(myWorkerInterf.address());
-		TraceEvent("ProtectRestoreWorker")
-		    .detail("Address", addresses.toString())
-		    .detail("IsReliable", p->isReliable())
-		    .detail("ReliableInfo", p->getReliableInfo())
-		    .backtrace();
-		ASSERT(p->isReliable());
-	}
-
-	TraceEvent("FastRestoreWorkerKnobs", myWorkerInterf.id())
-	    .detail("FailureTimeout", SERVER_KNOBS->FASTRESTORE_FAILURE_TIMEOUT)
-	    .detail("HeartBeat", SERVER_KNOBS->FASTRESTORE_HEARTBEAT_INTERVAL)
-	    .detail("SamplePercentage", SERVER_KNOBS->FASTRESTORE_SAMPLING_PERCENT)
-	    .detail("NumLoaders", SERVER_KNOBS->FASTRESTORE_NUM_LOADERS)
-	    .detail("NumAppliers", SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS)
-	    .detail("TxnBatchSize", SERVER_KNOBS->FASTRESTORE_TXN_BATCH_MAX_BYTES)
-	    .detail("VersionBatchSize", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES);
-
-	wait(waitOnRestoreRequests(cx, myWorkerInterf.id()));
-
-	wait(monitorleader(leader, cx, myWorkerInterf));
-
-	TraceEvent("FastRestoreWorker", myWorkerInterf.id()).detail("LeaderElection", "WaitForLeader");
-	if (leader->get() == myWorkerInterf) {
-		// Restore controller worker: doLeaderThings();
-		myWork = startRestoreWorkerLeader(self, myWorkerInterf, cx);
-	} else {
-		// Restore normal worker (for RestoreLoader and RestoreApplier roles): doWorkerThings();
-		myWork = startRestoreWorker(self, myWorkerInterf, cx);
-	}
-
-	wait(myWork);
-	return Void();
-}
-
-ACTOR Future<Void> restoreWorker(Reference<IClusterConnectionRecord> connRecord,
-                                 LocalityData locality,
-                                 std::string coordFolder) {
-	try {
-		Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
-		wait(reportErrors(_restoreWorker(cx, locality), "RestoreWorker"));
-	} catch (Error& e) {
-		TraceEvent("FastRestoreWorker").detail("Error", e.what());
-		throw e;
-	}
-
-	return Void();
-}
diff --git a/fdbserver/RestoreWorkerInterface.actor.cpp b/fdbserver/RestoreWorkerInterface.actor.cpp
deleted file mode 100644
index 87638dc0997..00000000000
--- a/fdbserver/RestoreWorkerInterface.actor.cpp
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * RestoreWorkerInterface.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-#include "flow/actorcompiler.h" // must be last include
-
-const KeyRef restoreLeaderKey = "\xff\x02/restoreLeader"_sr;
-const KeyRangeRef restoreWorkersKeys("\xff\x02/restoreWorkers/"_sr, "\xff\x02/restoreWorkers0"_sr);
-const KeyRef restoreStatusKey = "\xff\x02/restoreStatus/"_sr;
-const KeyRangeRef restoreApplierKeys("\xff\x02/restoreApplier/"_sr, "\xff\x02/restoreApplier0"_sr);
-const KeyRef restoreApplierTxnValue = "1"_sr;
-
-// restoreApplierKeys: track atomic transaction progress to ensure applying atomicOp exactly once
-// Version and batchIndex are passed in as LittleEndian,
-// they must be converted to BigEndian to maintain ordering in lexical order
-const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version) {
-	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(restoreApplierKeys.begin);
-	wr << applierID << bigEndian64(batchIndex) << bigEndian64(version);
-	return wr.toValue();
-}
-
-std::tuple<UID, int64_t, Version> decodeRestoreApplierKey(ValueRef const& key) {
-	BinaryReader rd(key, Unversioned());
-	UID applierID;
-	int64_t batchIndex;
-	Version version;
-	rd >> applierID >> batchIndex >> version;
-	return std::make_tuple(applierID, bigEndian64(batchIndex), bigEndian64(version));
-}
-
-// Encode restore worker key for workerID
-const Key restoreWorkerKeyFor(UID const& workerID) {
-	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(restoreWorkersKeys.begin);
-	wr << workerID;
-	return wr.toValue();
-}
-
-// Encode restore agent value
-const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& cmdInterf) {
-	BinaryWriter wr(IncludeVersion(ProtocolVersion::withRestoreWorkerInterfaceValue()));
-	wr << cmdInterf;
-	return wr.toValue();
-}
-
-RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value) {
-	RestoreWorkerInterface s;
-	BinaryReader reader(value, IncludeVersion());
-	reader >> s;
-	return s;
-}
-
-Value restoreRequestDoneVersionValue(Version readVersion) {
-	BinaryWriter wr(IncludeVersion(ProtocolVersion::withRestoreRequestDoneVersionValue()));
-	wr << readVersion;
-	return wr.toValue();
-}
-Version decodeRestoreRequestDoneVersionValue(ValueRef const& value) {
-	Version v;
-	BinaryReader reader(value, IncludeVersion());
-	reader >> v;
-	return v;
-}
-
-RestoreRequest decodeRestoreRequestValue(ValueRef const& value) {
-	RestoreRequest s;
-	BinaryReader reader(value, IncludeVersion());
-	reader >> s;
-	return s;
-}
-
-// TODO: Register restore performance data to restoreStatus key
-const Key restoreStatusKeyFor(StringRef statusType) {
-	BinaryWriter wr(Unversioned());
-	wr.serializeBytes(restoreStatusKey);
-	wr << statusType;
-	return wr.toValue();
-}
-
-const Value restoreStatusValue(double val) {
-	BinaryWriter wr(IncludeVersion(ProtocolVersion::withRestoreStatusValue()));
-	wr << StringRef(std::to_string(val));
-	return wr.toValue();
-}
diff --git a/fdbserver/ServerThroughputTracker.cpp b/fdbserver/ServerThroughputTracker.cpp
deleted file mode 100644
index fe288624bd8..00000000000
--- a/fdbserver/ServerThroughputTracker.cpp
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * ServerThroughputTracker.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/ServerThroughputTracker.h"
-
-namespace {
-
-template <class K, class V, class H>
-static Optional<V> tryGet(std::unordered_map<K, V, H> const& m, K const& k) {
-	auto it = m.find(k);
-	if (it == m.end()) {
-		return {};
-	} else {
-		return it->second;
-	}
-}
-
-} // namespace
-
-ServerThroughputTracker::ThroughputCounters::ThroughputCounters()
-  : readThroughput(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME),
-    writeThroughput(SERVER_KNOBS->GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME) {}
-
-std::vector<TransactionTag> ServerThroughputTracker::getTagsAffectingStorageServer(UID storageServerId) const {
-	std::vector<TransactionTag> result;
-	auto const tagToThroughputCounters = tryGet(throughput, storageServerId);
-	if (!tagToThroughputCounters.present()) {
-		return {};
-	} else {
-		result.reserve(tagToThroughputCounters.get().size());
-		for (const auto& [tag, _] : tagToThroughputCounters.get()) {
-			result.push_back(tag);
-		}
-	}
-	return result;
-}
-
-void ServerThroughputTracker::ThroughputCounters::updateThroughput(double newThroughput, OpType opType) {
-	if (opType == OpType::READ) {
-		readThroughput.setTotal(newThroughput);
-	} else {
-		writeThroughput.setTotal(newThroughput);
-	}
-}
-
-double ServerThroughputTracker::ThroughputCounters::getThroughput() const {
-	return readThroughput.smoothTotal() + writeThroughput.smoothTotal();
-}
-
-ServerThroughputTracker::~ServerThroughputTracker() = default;
-
-// Exponentially decay throughput statistics for tags that were not reported.
-// If reported throughput for a particular tag is too low, stop tracking this
-// tag.
-void ServerThroughputTracker::cleanupUnseenTags(TransactionTagMap<ThroughputCounters>& tagToThroughputCounters,
-                                                std::unordered_set<TransactionTag> const& seenReadTags,
-                                                std::unordered_set<TransactionTag> const& seenWriteTags) {
-	auto it = tagToThroughputCounters.begin();
-	while (it != tagToThroughputCounters.end()) {
-		auto& [tag, throughputCounters] = *it;
-		bool seen = false;
-		if (seenReadTags.contains(tag)) {
-			seen = true;
-		} else {
-			throughputCounters.updateThroughput(0, OpType::READ);
-		}
-		if (seenWriteTags.contains(tag)) {
-			seen = true;
-		} else {
-			throughputCounters.updateThroughput(0, OpType::WRITE);
-		}
-		if (!seen && throughputCounters.getThroughput() < SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FORGET_SS_THRESHOLD) {
-			it = tagToThroughputCounters.erase(it);
-		} else {
-			++it;
-		}
-	}
-}
-
-// For unseen storage servers, exponentially decay throughput statistics for every tag.
-// If the reported throughput for a particular tag is too low, stop tracking this
-// tag. If no more tags are being tracked for a particular storage server,
-// stop tracking this storage server altogether.
-void ServerThroughputTracker::cleanupUnseenStorageServers(std::unordered_set<UID> const& seen) {
-	auto it1 = throughput.begin();
-	while (it1 != throughput.end()) {
-		auto& [ssId, tagToThroughputCounters] = *it1;
-		if (seen.contains(ssId)) {
-			++it1;
-		} else {
-			auto it2 = tagToThroughputCounters.begin();
-			while (it2 != tagToThroughputCounters.end()) {
-				auto& throughputCounters = it2->second;
-				throughputCounters.updateThroughput(0, OpType::READ);
-				throughputCounters.updateThroughput(0, OpType::WRITE);
-				if (throughputCounters.getThroughput() < SERVER_KNOBS->GLOBAL_TAG_THROTTLING_FORGET_SS_THRESHOLD) {
-					it2 = tagToThroughputCounters.erase(it2);
-				} else {
-					++it2;
-				}
-			}
-			if (tagToThroughputCounters.empty()) {
-				it1 = throughput.erase(it1);
-			} else {
-				++it1;
-			}
-		}
-	}
-}
-
-void ServerThroughputTracker::update(Map<UID, StorageQueueInfo> const& sqInfos) {
-	std::unordered_set<UID> seenStorageServerIds;
-	for (auto it = sqInfos.begin(); it != sqInfos.end(); ++it) {
-		auto const& ss = it->value;
-		if (!ss.valid) {
-			continue;
-		}
-		seenStorageServerIds.insert(ss.id);
-		std::unordered_set<TransactionTag> seenReadTags, seenWriteTags;
-		auto& tagToThroughputCounters = throughput[ss.id];
-		for (const auto& busyReader : ss.busiestReadTags) {
-			seenReadTags.insert(busyReader.tag);
-			tagToThroughputCounters[busyReader.tag].updateThroughput(busyReader.rate, OpType::READ);
-		}
-		for (const auto& busyWriter : ss.busiestWriteTags) {
-			seenWriteTags.insert(busyWriter.tag);
-			tagToThroughputCounters[busyWriter.tag].updateThroughput(busyWriter.rate, OpType::WRITE);
-		}
-
-		cleanupUnseenTags(tagToThroughputCounters, seenReadTags, seenWriteTags);
-	}
-	cleanupUnseenStorageServers(seenStorageServerIds);
-}
-
-Optional<double> ServerThroughputTracker::getThroughput(UID storageServerId, TransactionTag const& tag) const {
-	auto const tagToThroughputCounters = tryGet(throughput, storageServerId);
-	if (!tagToThroughputCounters.present()) {
-		return {};
-	}
-	auto const throughputCounter = tryGet(tagToThroughputCounters.get(), tag);
-	if (!throughputCounter.present()) {
-		return {};
-	}
-	return throughputCounter.get().getThroughput();
-}
-
-double ServerThroughputTracker::getThroughput(TransactionTag const& tag) const {
-	double result{ 0.0 };
-	for (auto const& [ssId, _] : throughput) {
-		result += getThroughput(ssId, tag).orDefault(0);
-	}
-	return result;
-}
-
-Optional<double> ServerThroughputTracker::getThroughput(UID storageServerId) const {
-	auto tagToThroughputCounters = tryGet(throughput, storageServerId);
-	if (!tagToThroughputCounters.present()) {
-		return {};
-	}
-	double result = 0;
-	for (const auto& [_, throughputCounters] : tagToThroughputCounters.get()) {
-		result += throughputCounters.getThroughput();
-	}
-	return result;
-}
-
-void ServerThroughputTracker::removeTag(TransactionTag const& tag) {
-	for (auto& [ss, tagToCounters] : throughput) {
-		tagToCounters.erase(tag);
-	}
-}
-
-int ServerThroughputTracker::storageServersTracked() const {
-	return throughput.size();
-}
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.cpp
similarity index 84%
rename from fdbserver/SimulatedCluster.actor.cpp
rename to fdbserver/SimulatedCluster.cpp
index 24e24795ae8..0fd39c9880f 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.cpp
@@ -1,5 +1,5 @@
 /*
- * SimulatedCluster.actor.cpp
+ * SimulatedCluster.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -37,16 +37,17 @@
 #include "fdbclient/ClusterConnectionFile.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/tester/tester.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/worker/Worker.h"
 #include "fdbclient/ClusterInterface.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/CoordinationInterface.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/CoordinationInterface.h"
 #include "fdbclient/SimpleIni.h"
-#include "fdbrpc/AsyncFileNonDurable.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbrpc/AsyncFileNonDurable.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/versions.h"
 #include "flow/IRandom.h"
 #include "flow/MkCert.h"
@@ -58,10 +59,11 @@
 #include "flow/TypeTraits.h"
 #include "flow/FaultInjection.h"
 #include "flow/CodeProbeUtils.h"
-#include "fdbserver/SimulatedCluster.h"
+#include "fdbserver/datadistributor/SimulatedCluster.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
 #include "flow/IConnection.h"
-#include "fdbserver/MockGlobalState.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/datadistributor/MockGlobalState.h"
+#include "flow/CoroUtils.h"
 
 #undef max
 #undef min
@@ -71,10 +73,6 @@ extern const char* getSourceVersion();
 
 using namespace std::literals;
 
-bool isSimulatorProcessUnreliable() {
-	return g_network->isSimulated() && !g_simulator->getCurrentProcess()->isReliable();
-}
-
 namespace {
 
 constexpr bool hasRocksDB =
@@ -115,7 +113,22 @@ std::string describe(int const& val) {
 
 template <>
 std::string describe(const SimulationStorageEngine& val) {
-	return std::to_string(static_cast<uint32_t>(val));
+	switch (val) {
+	case SimulationStorageEngine::SSD:
+		return "ssd";
+	case SimulationStorageEngine::MEMORY:
+		return "memory";
+	case SimulationStorageEngine::RADIX_TREE:
+		return "memory-radixtree";
+	case SimulationStorageEngine::REDWOOD:
+		return "ssd-redwood-1";
+	case SimulationStorageEngine::ROCKSDB:
+		return "ssd-rocksdb-v1";
+	case SimulationStorageEngine::SHARDED_ROCKSDB:
+		return "ssd-sharded-rocksdb";
+	default:
+		return std::to_string(static_cast<uint32_t>(val));
+	}
 }
 
 namespace {
@@ -125,6 +138,28 @@ bool isValidSimulationStorageEngineValue(const std::underlying_type_t<Simulation
 	                   SimulationStorageEngine::SIMULATION_STORAGE_ENGINE_INVALID_VALUE);
 }
 
+SimulationStorageEngine parseSimulationStorageEngine(const toml::value& value) {
+	if (value.type() == toml::value_t::integer) {
+		const auto intVal = static_cast<uint8_t>(value.as_integer());
+		ASSERT(isValidSimulationStorageEngineValue(intVal));
+		return static_cast<SimulationStorageEngine>(intVal);
+	}
+
+	ASSERT(value.type() == toml::value_t::string);
+	const auto& stringVal = value.as_string();
+	static const std::unordered_map<std::string, SimulationStorageEngine> STORAGE_ENGINE_NAME_MAPPER = {
+		{ "ssd", SimulationStorageEngine::SSD },
+		{ "memory", SimulationStorageEngine::MEMORY },
+		{ "memory-radixtree", SimulationStorageEngine::RADIX_TREE },
+		{ "ssd-redwood-1", SimulationStorageEngine::REDWOOD },
+		{ "ssd-rocksdb-v1", SimulationStorageEngine::ROCKSDB },
+		{ "ssd-sharded-rocksdb", SimulationStorageEngine::SHARDED_ROCKSDB },
+	};
+	const auto iter = STORAGE_ENGINE_NAME_MAPPER.find(stringVal);
+	ASSERT(iter != STORAGE_ENGINE_NAME_MAPPER.end());
+	return iter->second;
+}
+
 constexpr int MACHINE_REBOOT_TIME = 10;
 
 bool destructed = false;
@@ -149,7 +184,7 @@ class TestConfig : public BasicTestConfig {
 
 		struct visitor {
 			const value_type& value;
-			visitor(const value_type& v) : value(v) {}
+			explicit visitor(const value_type& v) : value(v) {}
 			void operator()(int* val) const { *val = value.as_integer(); }
 			void operator()(Optional<int>* val) const { *val = value.as_integer(); }
 			void operator()(float* val) const { *val = value.as_floating(); }
@@ -185,9 +220,7 @@ class TestConfig : public BasicTestConfig {
 			void operator()(std::set<SimulationStorageEngine>* val) const {
 				auto arr = value.as_array();
 				for (const auto& i : arr) {
-					const auto intVal = static_cast<uint8_t>(i.as_integer());
-					ASSERT(isValidSimulationStorageEngineValue(intVal));
-					val->insert(static_cast<SimulationStorageEngine>(intVal));
+					val->insert(parseSimulationStorageEngine(i));
 				}
 			}
 			void operator()(Optional<std::set<SimulationStorageEngine>>* val) const {
@@ -260,8 +293,8 @@ class TestConfig : public BasicTestConfig {
 	public:
 		~ConfigBuilder() {
 			TraceEvent evt("SimulatorConfigFromToml");
-			for (const auto& p : confMap) {
-				std::visit(trace_visitor(std::string(p.first), evt), p.second);
+			for (const auto& [configKey, configValue] : confMap) {
+				std::visit(trace_visitor(std::string(configKey), evt), configValue);
 			}
 		}
 
@@ -301,7 +334,7 @@ class TestConfig : public BasicTestConfig {
 		while (ifs.good()) {
 			getline(ifs, cline);
 			std::string line = removeWhitespace(std::string(cline));
-			if (!line.size() || line.find(';') == 0)
+			if (line.empty() || line.find(';') == 0)
 				continue;
 
 			size_t found = line.find('=');
@@ -312,7 +345,7 @@ class TestConfig : public BasicTestConfig {
 			std::string value = removeWhitespace(line.substr(found + 1));
 
 			if (attrib == "extraDatabaseMode") {
-				extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(value);
+				extraDatabaseMode = stringToFDBExtraDatabaseMode(value);
 			}
 
 			if (attrib == "extraDatabaseCount") {
@@ -360,9 +393,6 @@ class TestConfig : public BasicTestConfig {
 			if (attrib == "disableHostname") {
 				disableHostname = strcmp(value.c_str(), "true") == 0;
 			}
-			if (attrib == "disableRemoteKVS") {
-				disableRemoteKVS = strcmp(value.c_str(), "true") == 0;
-			}
 			if (attrib == "restartInfoLocation") {
 				isFirstTestInRestart = true;
 			}
@@ -391,7 +421,7 @@ class TestConfig : public BasicTestConfig {
 
 public:
 	int extraDB = 0;
-	ISimulator::ExtraDatabaseMode extraDatabaseMode = ISimulator::ExtraDatabaseMode::Disabled;
+	FDBExtraDatabaseMode extraDatabaseMode = FDBExtraDatabaseMode::Disabled;
 	// The number of extra database used if the database mode is MULTIPLE
 	int extraDatabaseCount = 1;
 	bool extraDatabaseBackupAgents = false;
@@ -404,17 +434,9 @@ class TestConfig : public BasicTestConfig {
 	bool disableTss = false;
 	// 7.1 cannot be downgraded to 7.0 and below after enabling hostname, so disable hostname for 7.0 downgrade tests
 	bool disableHostname = false;
-	// remote key value store is a child process spawned by the SS process to run the storage engine
-	bool disableRemoteKVS = false;
 	// Storage Engine Types: Verify match with SimulationConfig::generateNormalConfig
-	//	0 = "ssd"
-	//	1 = "memory"
-	//	2 = "memory-radixtree"
-	//	3 = "ssd-redwood-1"
-	//	4 = "ssd-rocksdb-v1"
-	//	5 = "ssd-sharded-rocksdb"
-	// Requires a comma-separated list of numbers WITHOUT whitespaces
-	// See SimulationStorageEngine for more details
+	// TOML tests may use either the canonical string names or legacy integer values. Restart tests
+	// parsed by older binaries must keep using legacy integers for compatibility.
 	std::set<SimulationStorageEngine> storageEngineExcludeTypes;
 	Optional<int> datacenters, stderrSeverity, processesPerMachine;
 	// Set the maximum TLog version that can be selected for a test
@@ -481,7 +503,6 @@ class TestConfig : public BasicTestConfig {
 		    .add("maxTLogVersion", &maxTLogVersion)
 		    .add("disableTss", &disableTss)
 		    .add("disableHostname", &disableHostname)
-		    .add("disableRemoteKVS", &disableRemoteKVS)
 		    .add("simpleConfig", &simpleConfig)
 		    .add("singleRegion", &singleRegion)
 		    .add("generateFearless", &generateFearless)
@@ -533,7 +554,7 @@ class TestConfig : public BasicTestConfig {
 				isFirstTestInRestart = tomlKeyPresent(file, "restartInfoLocation");
 			}
 			if (!extraDatabaseModeStr.empty()) {
-				extraDatabaseMode = ISimulator::stringToExtraDatabaseMode(extraDatabaseModeStr);
+				extraDatabaseMode = stringToFDBExtraDatabaseMode(extraDatabaseModeStr);
 			}
 		} catch (std::exception& e) {
 			std::cerr << e.what() << std::endl;
@@ -554,7 +575,7 @@ class TestConfig : public BasicTestConfig {
 	}
 
 	TestConfig() = default;
-	TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {}
+	explicit(false) TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {}
 };
 
 template <class T>
@@ -567,60 +588,60 @@ T simulate(const T& in) {
 	return out;
 }
 
-ACTOR Future<Void> runBackup(Reference<IClusterConnectionRecord> connRecord) {
-	state Future<Void> agentFuture;
-	state FileBackupAgent fileAgent;
+Future<Void> runBackup(Reference<IClusterConnectionRecord> connRecord) {
+	Future<Void> agentFuture;
+	FileBackupAgent fileAgent;
 
-	while (g_simulator->backupAgents == ISimulator::BackupAgentType::WaitForType) {
-		wait(delay(1.0));
+	while (fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::WaitForType) {
+		co_await delay(1.0);
 	}
 
-	if (g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) {
+	if (fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::BackupToFile) {
 		Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION);
 
 		TraceEvent("SimBackupAgentsStarting").log();
 		agentFuture =
 		    fileAgent.run(cx, 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT);
 
-		while (g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) {
-			wait(delay(1.0));
+		while (fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::BackupToFile) {
+			co_await delay(1.0);
 		}
 
 		TraceEvent("SimBackupAgentsStopping").log();
 		agentFuture.cancel();
 	}
 
-	wait(Future<Void>(Never()));
+	co_await Future<Void>(Never());
 	throw internal_error();
 }
 
-ACTOR Future<Void> runDr(Reference<IClusterConnectionRecord> connRecord) {
-	state std::vector<Future<Void>> agentFutures;
+Future<Void> runDr(Reference<IClusterConnectionRecord> connRecord) {
+	std::vector<Future<Void>> agentFutures;
 
-	while (g_simulator->drAgents == ISimulator::BackupAgentType::WaitForType) {
-		wait(delay(1.0));
+	while (fdbSimulationPolicyState().drAgents == FDBBackupAgentType::WaitForType) {
+		co_await delay(1.0);
 	}
 
-	if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
-		ASSERT(g_simulator->extraDatabases.size() == 1);
+	if (fdbSimulationPolicyState().drAgents == FDBBackupAgentType::BackupToDB) {
+		ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
 		Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION);
 
-		state Database drDatabase = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+		Database drDatabase = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 
 		TraceEvent("StartingDrAgents")
 		    .detail("ConnectionString", connRecord->getConnectionString().toString())
-		    .detail("ExtraString", g_simulator->extraDatabases[0]);
+		    .detail("ExtraString", fdbSimulationPolicyState().extraDatabases[0]);
 
-		state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
-		state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(drDatabase);
+		DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
+		DatabaseBackupAgent extraAgent = DatabaseBackupAgent(drDatabase);
 
 		auto drPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
 
 		agentFutures.push_back(extraAgent.run(cx, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
 		agentFutures.push_back(dbAgent.run(drDatabase, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT));
 
-		while (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
-			wait(delay(1.0));
+		while (fdbSimulationPolicyState().drAgents == FDBBackupAgentType::BackupToDB) {
+			co_await delay(1.0);
 		}
 
 		TraceEvent("StoppingDrAgents").log();
@@ -630,17 +651,17 @@ ACTOR Future<Void> runDr(Reference<IClusterConnectionRecord> connRecord) {
 		}
 	}
 
-	wait(Future<Void>(Never()));
+	co_await Future<Void>(Never());
 	throw internal_error();
 }
 
-ACTOR Future<Void> runSimHTTPServer() {
+Future<Void> runSimHTTPServer() {
 	TraceEvent("SimHTTPServerStarting");
-	state Reference<HTTP::SimServerContext> context = makeReference<HTTP::SimServerContext>();
+	auto context = makeReference<HTTP::SimServerContext>();
 	g_simulator->addSimHTTPProcess(context);
 
 	try {
-		wait(context->actors.getResult());
+		co_await context->actors.getResult();
 	} catch (Error& e) {
 		TraceEvent("SimHTTPServerDied").errorUnsuppressed(e);
 		context->stop();
@@ -669,32 +690,32 @@ bool processRunHTTPServer(ProcessMode mode) {
 // SOMEDAY: when a process can be rebooted in isolation from the other on that machine,
 //  a loop{} will be needed around the waiting on simulatedFDBD(). For now this simply
 //  takes care of house-keeping such as context switching and file closing.
-ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConnectionRecord> connRecord,
-                                                         IPAddress ip,
-                                                         bool sslEnabled,
-                                                         uint16_t port,
-                                                         uint16_t listenPerProcess,
-                                                         LocalityData localities,
-                                                         ProcessClass processClass,
-                                                         std::string* dataFolder,
-                                                         std::string* coordFolder,
-                                                         std::string baseFolder,
-                                                         ClusterConnectionString connStr,
-                                                         ClusterConnectionString otherConnStr,
-                                                         bool useSeedFile,
-                                                         ProcessMode processMode,
-                                                         std::string whitelistBinPaths,
-                                                         ProtocolVersion protocolVersion,
-                                                         bool isDr) {
-	state ISimulator::ProcessInfo* simProcess = g_simulator->getCurrentProcess();
-	state UID randomId = nondeterministicRandom()->randomUniqueID();
-	state int cycles = 0;
-	state IPAllowList allowList;
+Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConnectionRecord> connRecord,
+                                                   IPAddress ip,
+                                                   bool sslEnabled,
+                                                   uint16_t port,
+                                                   uint16_t listenPerProcess,
+                                                   LocalityData localities,
+                                                   ProcessClass processClass,
+                                                   std::string* dataFolder,
+                                                   std::string* coordFolder,
+                                                   std::string baseFolder,
+                                                   ClusterConnectionString connStr,
+                                                   ClusterConnectionString otherConnStr,
+                                                   bool useSeedFile,
+                                                   ProcessMode processMode,
+                                                   std::string whitelistBinPaths,
+                                                   ProtocolVersion protocolVersion,
+                                                   bool isDr) {
+	ISimulator::ProcessInfo* simProcess = g_simulator->getCurrentProcess();
+	UID randomId = nondeterministicRandom()->randomUniqueID();
+	int cycles = 0;
+	IPAllowList allowList;
 
 	allowList.addTrustedSubnet("0.0.0.0/2"sv);
 	allowList.addTrustedSubnet("abcd::/16"sv);
 
-	loop {
+	while (true) {
 		auto waitTime =
 		    SERVER_KNOBS->MIN_REBOOT_TIME +
 		    (SERVER_KNOBS->MAX_REBOOT_TIME - SERVER_KNOBS->MIN_REBOOT_TIME) * deterministicRandom()->random01();
@@ -708,23 +729,23 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 		    .detail("Port", port)
 		    .detail("IsDr", isDr);
 
-		wait(delay(waitTime));
-
-		state ISimulator::ProcessInfo* process = g_simulator->newProcess("Server",
-		                                                                 ip,
-		                                                                 port,
-		                                                                 sslEnabled,
-		                                                                 listenPerProcess,
-		                                                                 localities,
-		                                                                 processClass,
-		                                                                 dataFolder->c_str(),
-		                                                                 coordFolder->c_str(),
-		                                                                 protocolVersion,
-		                                                                 isDr);
-		wait(g_simulator->onProcess(
+		co_await delay(waitTime);
+
+		ISimulator::ProcessInfo* process = g_simulator->newProcess("Server",
+		                                                           ip,
+		                                                           port,
+		                                                           sslEnabled,
+		                                                           listenPerProcess,
+		                                                           localities,
+		                                                           processClass,
+		                                                           dataFolder->c_str(),
+		                                                           coordFolder->c_str(),
+		                                                           protocolVersion,
+		                                                           isDr);
+		co_await g_simulator->onProcess(
 		    process,
-		    TaskPriority::DefaultYield)); // Now switch execution to the process on which we will run
-		state Future<ISimulator::KillType> onShutdown = process->onShutdown();
+		    TaskPriority::DefaultYield); // Now switch execution to the process on which we will run
+		Future<ISimulator::KillType> onShutdown = process->onShutdown();
 
 		try {
 			TraceEvent("SimulatedRebooterStarting")
@@ -756,8 +777,8 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 				bool client = processClass == ProcessClass::TesterClass || processMode == BackupAgentOnly ||
 				              processMode == SimHTTPServer;
 				FlowTransport::createInstance(client, 1, WLTOKEN_RESERVED_COUNT, &allowList);
-				for (const auto& p : g_simulator->authKeys) {
-					FlowTransport::transport().addPublicKey(p.first, p.second.toPublic());
+				for (const auto& [keyName, key] : g_simulator->authKeys) {
+					FlowTransport::transport().addPublicKey(keyName, key.toPublic());
 				}
 				Sim2FileSystem::newFileSystem();
 
@@ -803,7 +824,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 					// rebooting flag is still false, which breaks the assertion in g_simulator->destroyProcess
 					process->rebooting = true;
 				}
-				wait(waitForAny(futures));
+				co_await waitForAny(futures);
 			} catch (Error& e) {
 				// If in simulation, if we make it here with an error other than io_timeout but enASIOTimedOut is set
 				// then somewhere an io_timeout was converted to a different error.
@@ -865,9 +886,9 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 		    .detail("Excluded", process->excluded)
 		    .detail("Rebooting", process->rebooting)
 		    .detail("ZoneId", localities.zoneId());
-		wait(g_simulator->onProcess(simProcess));
+		co_await g_simulator->onProcess(simProcess);
 
-		wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY)); // One last chance for the process to clean up?
+		co_await delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY); // One last chance for the process to clean up?
 
 		g_simulator->destroyProcess(
 		    process); // Leak memory here; the process may be used in other parts of the simulation
@@ -889,7 +910,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 			    .detail("Excluded", process->excluded)
 			    .detail("ZoneId", localities.zoneId())
 			    .detail("KillType", shutdownResult);
-			return onShutdown.get();
+			co_return shutdownResult;
 		}
 
 		if (onShutdown.get() == ISimulator::KillType::RebootProcessAndDelete) {
@@ -907,8 +928,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 				writeFile(joinPath(*dataFolder, "fdb.cluster"), connStr.toString());
 				connRecord = makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"));
 			} else {
-				connRecord =
-				    makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"), connStr.toString());
+				connRecord = makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"), connStr);
 			}
 		} else if (onShutdown.get() == ISimulator::KillType::RebootProcessAndSwitch) {
 			TraceEvent("SimulatedFDBDRebootAndSwitch")
@@ -926,7 +946,8 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 			if (otherConnStr.toString().size() > 1) {
 				std::string newConnStr =
 				    g_simulator->hasSwitchedCluster(process->address) ? otherConnStr.toString() : connStr.toString();
-				connRecord = makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"), newConnStr);
+				connRecord = makeReference<ClusterConnectionFile>(joinPath(*dataFolder, "fdb.cluster"),
+				                                                  ClusterConnectionString(newConnStr));
 			}
 		} else {
 			TraceEvent("SimulatedFDBDJustRepeat")
@@ -943,25 +964,25 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 std::map<Optional<Standalone<StringRef>>, std::vector<std::vector<std::string>>> availableFolders;
 // process count is no longer needed because it is now the length of the vector of ip's, because it was one ip per
 // process
-ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
-                                    ClusterConnectionString otherConnStr,
-                                    std::vector<IPAddress> ips,
-                                    bool sslEnabled,
-                                    LocalityData localities,
-                                    ProcessClass processClass,
-                                    std::string baseFolder,
-                                    bool restarting,
-                                    bool useSeedFile,
-                                    ProcessMode processMode,
-                                    bool sslOnly,
-                                    std::string whitelistBinPaths,
-                                    ProtocolVersion protocolVersion,
-                                    bool isDr) {
-	state int bootCount = 0;
-	state std::vector<std::string> myFolders;
-	state std::vector<std::string> coordFolders;
-	state UID randomId = nondeterministicRandom()->randomUniqueID();
-	state int listenPerProcess = (sslEnabled && !sslOnly) ? 2 : 1;
+Future<Void> simulatedMachine(ClusterConnectionString connStr,
+                              ClusterConnectionString otherConnStr,
+                              std::vector<IPAddress> ips,
+                              bool sslEnabled,
+                              LocalityData localities,
+                              ProcessClass processClass,
+                              std::string baseFolder,
+                              bool restarting,
+                              bool useSeedFile,
+                              ProcessMode processMode,
+                              bool sslOnly,
+                              std::string whitelistBinPaths,
+                              ProtocolVersion protocolVersion,
+                              bool isDr) {
+	int bootCount = 0;
+	std::vector<std::string> myFolders;
+	std::vector<std::string> coordFolders;
+	UID randomId = nondeterministicRandom()->randomUniqueID();
+	int listenPerProcess = (sslEnabled && !sslOnly) ? 2 : 1;
 
 	try {
 		CSimpleIni ini;
@@ -978,7 +999,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				if (i == 0) {
 					std::string coordinationFolder =
 					    ini.GetValue(printable(localities.machineId()).c_str(), "coordinationFolder", "");
-					if (!coordinationFolder.size())
+					if (coordinationFolder.empty())
 						coordinationFolder = ini.GetValue(
 						    printable(localities.machineId()).c_str(),
 						    format("c%d", i * listenPerProcess).c_str(),
@@ -1002,8 +1023,8 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			}
 		}
 
-		loop {
-			state std::vector<Future<ISimulator::KillType>> processes;
+		while (true) {
+			std::vector<Future<ISimulator::KillType>> processes;
 			for (int i = 0; i < ips.size(); i++) {
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
 				ProcessMode ipProcessMode =
@@ -1015,13 +1036,14 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				if (ipProcessMode != SimHTTPServer) {
 					// Fall back to use seed string if fdb.cluster not present
 					// It can happen when a process failed before it persisted the connection string to disk
-					clusterFile = Reference<IClusterConnectionRecord>(
-					    useSeedFile || !fileExists(path) ? new ClusterConnectionFile(path, connStr.toString())
-					                                     : new ClusterConnectionFile(path));
+					clusterFile = Reference<IClusterConnectionRecord>(useSeedFile || !fileExists(path)
+					                                                      ? new ClusterConnectionFile(path, connStr)
+					                                                      : new ClusterConnectionFile(path));
 				}
 				const int listenPort = i * listenPerProcess + 1;
 
-				if (g_simulator->hasDiffProtocolProcess && !g_simulator->setDiffProtocol && ipProcessMode == FDBDOnly) {
+				if (fdbSimulationPolicyState().hasDiffProtocolProcess && !fdbSimulationPolicyState().setDiffProtocol &&
+				    ipProcessMode == FDBDOnly) {
 					processes.push_back(simulatedFDBDRebooter(clusterFile,
 					                                          ips[i],
 					                                          sslEnabled,
@@ -1039,7 +1061,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 					                                          whitelistBinPaths,
 					                                          protocolVersion,
 					                                          isDr));
-					g_simulator->setDiffProtocol = true;
+					fdbSimulationPolicyState().setDiffProtocol = true;
 				} else {
 					processes.push_back(simulatedFDBDRebooter(clusterFile,
 					                                          ips[i],
@@ -1085,7 +1107,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			    .detail("DataHall", localities.dataHallId())
 			    .detail("Locality", localities.toString());
 
-			wait(waitForAll(processes));
+			co_await waitForAll(processes);
 
 			TraceEvent("SimulatedMachineRebootStart", randomId)
 			    .detail("Folder0", myFolders[0])
@@ -1109,19 +1131,19 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				for (auto fileItr = files.begin(); fileItr != files.end(); ++fileItr)
 					killFutures.push_back((*fileItr)->kill());
 
-				wait(waitForAll(killFutures));
+				co_await waitForAll(killFutures);
 			}
 
-			state std::set<std::string> filenames;
-			state std::string closingStr;
+			std::set<std::string> filenames;
+			std::string closingStr;
 			auto& machineCache = g_simulator->getMachineById(localities.machineId())->openFiles;
-			for (auto it : machineCache) {
-				filenames.insert(it.first);
-				closingStr += it.first + ", ";
-				ASSERT(it.second.get().canGet());
+			for (auto& [filename, openFile] : machineCache) {
+				filenames.insert(filename);
+				closingStr += filename + ", ";
+				ASSERT(openFile.get().canGet());
 			}
 
-			for (auto it : g_simulator->getMachineById(localities.machineId())->deletingOrClosingFiles) {
+			for (const auto& it : g_simulator->getMachineById(localities.machineId())->deletingOrClosingFiles) {
 				filenames.insert(it);
 				closingStr += it + ", ";
 			}
@@ -1143,9 +1165,9 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			//   without delay(0)-equivalents, so delay(0) a few times waiting for it to achieve that goal.
 			// After an injected fault:
 			//   The process is expected to shut down eventually, but not necessarily instantly.  Wait up to 60 seconds.
-			state int shutdownDelayCount = 0;
-			state double backoff = 0;
-			loop {
+			int shutdownDelayCount = 0;
+			double backoff = 0;
+			while (true) {
 				auto& machineCache = g_simulator->getMachineById(localities.machineId())->closingFiles;
 
 				if (!machineCache.empty()) {
@@ -1169,7 +1191,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 					ASSERT(false);
 				}
 
-				wait(delay(backoff));
+				co_await delay(backoff);
 				backoff = std::min(backoff + 1.0, 6.0);
 			}
 
@@ -1184,14 +1206,14 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			// ASSERT( this machine is rebooting );
 
 			// Since processes can end with different codes, take the highest (least severe) to detmine what to do
-			state ISimulator::KillType killType = processes[0].get();
+			ISimulator::KillType killType = processes[0].get();
 			for (int i = 1; i < ips.size(); i++)
 				killType = std::max(processes[i].get(), killType);
 
 			CODE_PROBE(true, "Simulated machine has been rebooted");
 
-			state bool swap = killType == ISimulator::KillType::Reboot && BUGGIFY_WITH_PROB(0.75) &&
-			                  g_simulator->canSwapToMachine(localities.zoneId());
+			bool swap = killType == ISimulator::KillType::Reboot && BUGGIFY_WITH_PROB(0.75) &&
+			            g_simulator->canSwapToMachine(localities.zoneId());
 			if (swap)
 				availableFolders[localities.dcId()].push_back(myFolders);
 
@@ -1205,7 +1227,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 			    .detail("DataHall", localities.dataHallId())
 			    .detail("MachineIPs", toIPVectorString(ips));
 
-			wait(delay(rebootTime));
+			co_await delay(rebootTime);
 
 			if (swap) {
 				auto& avail = availableFolders[localities.dcId()];
@@ -1224,7 +1246,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr,
 				}
 				myFolders = toRebootFrom;
 				if (!useSeedFile) {
-					for (auto f : toRebootFrom) {
+					for (const auto& f : toRebootFrom) {
 						if (!fileExists(joinPath(f, "fdb.cluster"))) {
 							writeFile(joinPath(f, "fdb.cluster"), connStr.toString());
 						}
@@ -1267,18 +1289,19 @@ IPAddress makeIPAddressForSim(bool isIPv6, std::array<int, 4> parts) {
 }
 
 #include "fdbclient/MonitorLeader.h"
+#include "fdbserver/core/TesterInterface.h"
 
 // Configures the system according to the given specifications in order to run
 // simulation, but with the additional consideration that it is meant to act
 // like a "rebooted" machine, mostly used for restarting tests.
-ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActors,
-                                          std::string baseFolder,
-                                          int* pTesterCount,
-                                          Optional<ClusterConnectionString>* pConnString,
-                                          Standalone<StringRef>* pStartingConfiguration,
-                                          TestConfig* testConfig,
-                                          std::string whitelistBinPaths,
-                                          ProtocolVersion protocolVersion) {
+Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActors,
+                                    std::string baseFolder,
+                                    int* pTesterCount,
+                                    Optional<ClusterConnectionString>* pConnString,
+                                    Standalone<StringRef>* pStartingConfiguration,
+                                    TestConfig* testConfig,
+                                    std::string whitelistBinPaths,
+                                    ProtocolVersion protocolVersion) {
 	CSimpleIni ini;
 	ini.SetUnicode();
 	ini.LoadFile(joinPath(baseFolder, "restartInfo.ini").c_str());
@@ -1303,12 +1326,12 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 		int testerCount = atoi(ini.GetValue("META", "testerCount"));
 		auto tssModeStr = ini.GetValue("META", "tssMode");
 		if (tssModeStr != nullptr) {
-			g_simulator->tssMode = (ISimulator::TSSMode)atoi(tssModeStr);
+			fdbSimulationPolicyState().tssMode = static_cast<FDBTSSMode>(atoi(tssModeStr));
 		}
 		ClusterConnectionString conn(ini.GetValue("META", "connectionString"));
-		if (testConfig->extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local) {
-			g_simulator->extraDatabases.clear();
-			g_simulator->extraDatabases.push_back(conn.toString());
+		if (testConfig->extraDatabaseMode == FDBExtraDatabaseMode::Local) {
+			fdbSimulationPolicyState().extraDatabases.clear();
+			fdbSimulationPolicyState().extraDatabases.push_back(conn.toString());
 		}
 		if (!testConfig->disableHostname) {
 			auto mockDNSStr = ini.GetValue("META", "mockDNS");
@@ -1316,11 +1339,6 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 				INetworkConnections::net()->parseMockDNSFromString(mockDNSStr);
 			}
 		}
-		auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
-		if (testConfig->disableRemoteKVS) {
-			g_knobs.setKnob("remote_kv_store", KnobValueRef::create(bool{ false }));
-			TraceEvent(SevDebug, "DisableRemoteKVS");
-		}
 		*pConnString = conn;
 		*pTesterCount = testerCount;
 		bool usingSSL = conn.toString().find(":tls") != std::string::npos || listenersPerProcess > 1;
@@ -1348,8 +1366,7 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 				zoneId = Standalone<StringRef>(zoneIdStr);
 			}
 
-			ProcessClass::ClassType cType =
-			    (ProcessClass::ClassType)(atoi(ini.GetValue(machineIdString.c_str(), "mClass")));
+			auto cType = static_cast<ProcessClass::ClassType>(atoi(ini.GetValue(machineIdString.c_str(), "mClass")));
 			// using specialized class types can lead to nondeterministic recruitment
 			if (cType == ProcessClass::MasterClass || cType == ProcessClass::ResolutionClass) {
 				cType = ProcessClass::StatelessClass;
@@ -1421,11 +1438,11 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 			    processClass == ProcessClass::TesterClass ? "SimulatedTesterMachine" : "SimulatedMachine"));
 		}
 
-		g_simulator->desiredCoordinators = desiredCoordinators;
+		fdbSimulationPolicyState().desiredCoordinators = desiredCoordinators;
 		g_simulator->processesPerMachine = processesPerMachine;
 
 		uniquify(dcIds);
-		if (!BUGGIFY && dcIds.size() == 2 && dcIds[0] != "" && dcIds[1] != "") {
+		if (!BUGGIFY && dcIds.size() == 2 && !dcIds[0].empty() && !dcIds[1].empty()) {
 			StatusObject primaryObj;
 			StatusObject primaryDcObj;
 			primaryDcObj["id"] = dcIds[0];
@@ -1452,25 +1469,23 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
 			    json_spirit::write_string(json_spirit::mValue(regionArr), json_spirit::Output_options::none);
 		}
 
-		g_simulator->restarted = true;
+		fdbSimulationPolicyState().restarted = true;
 
 		TraceEvent("RestartSimulatorSettings")
-		    .detail("DesiredCoordinators", g_simulator->desiredCoordinators)
+		    .detail("DesiredCoordinators", fdbSimulationPolicyState().desiredCoordinators)
 		    .detail("ProcessesPerMachine", g_simulator->processesPerMachine)
 		    .detail("ListenersPerProcess", listenersPerProcess);
 	} catch (Error& e) {
 		TraceEvent(SevError, "RestartSimulationError").error(e);
 	}
 
-	wait(delay(1.0));
-
-	return Void();
+	co_await delay(1.0);
 }
 
 // Configuration details compiled in a structure used when setting up a simulated cluster
 struct SimulationConfig : public BasicSimulationConfig {
 	explicit SimulationConfig(const TestConfig& testConfig);
-	ISimulator::ExtraDatabaseMode extraDatabaseMode;
+	FDBExtraDatabaseMode extraDatabaseMode;
 	int extraDatabaseCount;
 	bool generateFearless;
 
@@ -1504,12 +1519,8 @@ void SimulationConfig::set_config(std::string config) {
 	std::map<std::string, std::string> hack_map;
 	const auto buildResult = buildConfiguration(config, hack_map);
 	ASSERT(buildResult != ConfigurationResult::NO_OPTIONS_PROVIDED);
-	for (auto kv : hack_map)
-		db.set(kv.first, kv.second);
-}
-
-[[maybe_unused]] StringRef StringRefOf(const char* s) {
-	return StringRef((uint8_t*)s, strlen(s));
+	for (const auto& [configKey, configValue] : hack_map)
+		db.set(configKey, configValue);
 }
 
 // Set the randomly generated options of the config. Compiled here to easily observe and trace random options
@@ -1539,10 +1550,6 @@ void SimulationConfig::setRandomConfig() {
 		// TraceEvent("SimulatedConfigRandom").detail("PerpetualWiggle", 1);
 		set_config("perpetual_storage_wiggle=1");
 	}
-
-	if (deterministicRandom()->random01() < 0.5) {
-		set_config("backup_worker_enabled:=1");
-	}
 }
 
 // Overwrite DB with simple options, used when simpleConfig is true in the TestConfig
@@ -1654,7 +1661,7 @@ const std::vector<SimulationStorageEngine> SIMULATION_STORAGE_ENGINE = {
 std::string getExcludedStorageEngineTypesInString(const std::set<SimulationStorageEngine>& excluded) {
 	std::string str;
 	for (const auto& e : excluded) {
-		str += std::to_string(static_cast<uint32_t>(e));
+		str += describe(e);
 		str += ',';
 	}
 	if (!excluded.empty())
@@ -1691,7 +1698,7 @@ SimulationStorageEngine chooseSimulationStorageEngine(const TestConfig& testConf
 		for (const auto& storageEngineExcluded : testConfig.storageEngineExcludeTypes) {
 			storageEngineAvailable.erase(storageEngineExcluded);
 		}
-		ASSERT(storageEngineAvailable.size() > 0);
+		ASSERT(!storageEngineAvailable.empty());
 		std::vector<SimulationStorageEngine> storageEngineCandidates;
 		for (const auto& storageEngine : storageEngineAvailable) {
 			if (storageEngine == SimulationStorageEngine::MEMORY) {
@@ -2015,18 +2022,20 @@ void SimulationConfig::setRegions(const TestConfig& testConfig) {
 	}
 
 	if (needsRemote) {
-		g_simulator->originalRegions =
+		fdbSimulationPolicyState().originalRegions =
 		    "regions=" + json_spirit::write_string(json_spirit::mValue(regionArr), json_spirit::Output_options::none);
 
 		StatusArray disablePrimary = regionArr;
 		disablePrimary[0].get_obj()["datacenters"].get_array()[0].get_obj()["priority"] = -1;
-		g_simulator->disablePrimary = "regions=" + json_spirit::write_string(json_spirit::mValue(disablePrimary),
-		                                                                     json_spirit::Output_options::none);
+		fdbSimulationPolicyState().disablePrimary =
+		    "regions=" +
+		    json_spirit::write_string(json_spirit::mValue(disablePrimary), json_spirit::Output_options::none);
 
 		StatusArray disableRemote = regionArr;
 		disableRemote[1].get_obj()["datacenters"].get_array()[0].get_obj()["priority"] = -1;
-		g_simulator->disableRemote = "regions=" + json_spirit::write_string(json_spirit::mValue(disableRemote),
-		                                                                    json_spirit::Output_options::none);
+		fdbSimulationPolicyState().disableRemote =
+		    "regions=" +
+		    json_spirit::write_string(json_spirit::mValue(disableRemote), json_spirit::Output_options::none);
 	} else {
 		// In order to generate a starting configuration with the remote disabled, do not apply the region
 		// configuration to the DatabaseConfiguration until after creating the starting conf string.
@@ -2059,8 +2068,7 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
 		                         ((db.minDatacentersRequired() > 0) ? datacenters : 1) *
 		                             std::max(3, db.minZonesRequiredPerDatacenter()));
 		machine_count = deterministicRandom()->randomInt(
-		    machine_count,
-		    std::max(machine_count + 1, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6));
+		    machine_count, std::max(machine_count + 1, extraDatabaseMode == FDBExtraDatabaseMode::Disabled ? 10 : 6));
 		// generateMachineTeamTestConfig set up the number of servers per machine and the number of machines such that
 		// if we do not remove the surplus server and machine teams, the simulation test will report error.
 		// This is needed to make sure the number of server (and machine) teams is no larger than the desired number.
@@ -2070,9 +2078,9 @@ void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
 			// while the max possible machine team number is 10.
 			// If machine_count > 5, we can still test the effectivenss of machine teams
 			// Note: machine_count may be much larger than 5 because we may have a big replication factor
-			machine_count = std::max(machine_count,
-			                         deterministicRandom()->randomInt(
-			                             5, extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 10 : 6));
+			machine_count = std::max(
+			    machine_count,
+			    deterministicRandom()->randomInt(5, extraDatabaseMode == FDBExtraDatabaseMode::Disabled ? 10 : 6));
 		}
 	}
 	machine_count += datacenters * (testConfig.extraMachineCountDC + testConfig.extraStorageMachineCountPerDC);
@@ -2100,7 +2108,7 @@ void SimulationConfig::setProcessesPerMachine(const TestConfig& testConfig) {
 		processes_per_machine = 1;
 	} else {
 		processes_per_machine = deterministicRandom()->randomInt(
-		    1, (extraDatabaseMode == ISimulator::ExtraDatabaseMode::Disabled ? 28 : 14) / machine_count + 2);
+		    1, (extraDatabaseMode == FDBExtraDatabaseMode::Disabled ? 28 : 14) / machine_count + 2);
 	}
 }
 
@@ -2124,16 +2132,18 @@ void SimulationConfig::setTss(const TestConfig& testConfig) {
 		double tssRandom = deterministicRandom()->random01();
 		if (tssRandom > 0.5 || !faultInjectionActivated) {
 			// normal tss mode
-			g_simulator->tssMode = ISimulator::TSSMode::EnabledNormal;
+			fdbSimulationPolicyState().tssMode = FDBTSSMode::EnabledNormal;
 		} else if (tssRandom < 0.25 && !testConfig.isFirstTestInRestart) {
 			// fault injection - don't enable in first test in restart because second test won't know it intentionally
 			// lost data
-			g_simulator->tssMode = ISimulator::TSSMode::EnabledDropMutations;
+			fdbSimulationPolicyState().tssMode = FDBTSSMode::EnabledDropMutations;
 		} else {
 			// delay injection
-			g_simulator->tssMode = ISimulator::TSSMode::EnabledAddDelay;
+			fdbSimulationPolicyState().tssMode = FDBTSSMode::EnabledAddDelay;
 		}
-		printf("enabling tss for simulation in mode %d: %s\n", g_simulator->tssMode, confStr.c_str());
+		printf("enabling tss for simulation in mode %d: %s\n",
+		       static_cast<int>(fdbSimulationPolicyState().tssMode),
+		       confStr.c_str());
 	}
 }
 
@@ -2186,7 +2196,6 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
                           std::string whitelistBinPaths,
                           TestConfig testConfig,
                           ProtocolVersion protocolVersion) {
-	auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
 	// SOMEDAY: this does not test multi-interface configurations
 	SimulationConfig simconfig(testConfig);
 
@@ -2203,10 +2212,6 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	if (testConfig.configureLocked) {
 		startingConfigString += " locked";
 	}
-	if (testConfig.disableRemoteKVS) {
-		g_knobs.setKnob("remote_kv_store", KnobValueRef::create(bool{ false }));
-		TraceEvent(SevDebug, "DisableRemoteKVS");
-	}
 	startingConfigString += DatabaseConfiguration::configureStringFromJSON(startingConfigJSON);
 
 	// Set a random locality for the perpetual wiggle if any locality is set in the config
@@ -2226,10 +2231,11 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		startingConfigString += format(" tss_storage_engine:=%d", simconfig.db.testingStorageServerStoreType);
 	}
 
-	if (g_simulator->originalRegions != "") {
-		simconfig.set_config(g_simulator->originalRegions);
-		g_simulator->startingDisabledConfiguration = startingConfigString + " " + g_simulator->disableRemote;
-		startingConfigString += " " + g_simulator->originalRegions;
+	if (!fdbSimulationPolicyState().originalRegions.empty()) {
+		simconfig.set_config(fdbSimulationPolicyState().originalRegions);
+		fdbSimulationPolicyState().startingDisabledConfiguration =
+		    startingConfigString + " " + fdbSimulationPolicyState().disableRemote;
+		startingConfigString += " " + fdbSimulationPolicyState().originalRegions;
 	}
 
 	TraceEvent("SimulatorConfig").setMaxFieldLength(10000).detail("ConfigString", StringRef(startingConfigString));
@@ -2263,11 +2269,11 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	    useHostname ? NetworkAddressFromHostname::True : NetworkAddressFromHostname::False;
 
 	int extraDatabaseCount = 0;
-	bool useLocalDatabase = (testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::LocalOrSingle && BUGGIFY) ||
-	                        testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Local;
-	if (!useLocalDatabase && testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) {
+	bool useLocalDatabase = (testConfig.extraDatabaseMode == FDBExtraDatabaseMode::LocalOrSingle && BUGGIFY) ||
+	                        testConfig.extraDatabaseMode == FDBExtraDatabaseMode::Local;
+	if (!useLocalDatabase && testConfig.extraDatabaseMode != FDBExtraDatabaseMode::Disabled) {
 		extraDatabaseCount =
-		    testConfig.extraDatabaseMode == ISimulator::ExtraDatabaseMode::Multiple && testConfig.extraDatabaseCount > 0
+		    testConfig.extraDatabaseMode == FDBExtraDatabaseMode::Multiple && testConfig.extraDatabaseCount > 0
 		        ? testConfig.extraDatabaseCount
 		        : 1;
 	}
@@ -2382,7 +2388,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		}
 	}
 
-	ASSERT(coordinatorAddresses.size() > 0);
+	ASSERT(!coordinatorAddresses.empty());
 
 	// Mark a random majority of the coordinators as protected, so
 	// we won't accidentally kill off a quorum and render the
@@ -2392,10 +2398,10 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		TraceEvent("ProtectCoordinator")
 		    .detail("Address", coordinatorAddresses[i])
 		    .detail("Coordinators", describe(coordinatorAddresses));
-		g_simulator->protectedAddresses.insert(NetworkAddress(
+		g_simulator->protectAddress(NetworkAddress(
 		    coordinatorAddresses[i].ip, coordinatorAddresses[i].port, true, coordinatorAddresses[i].isTLS()));
 		if (coordinatorAddresses[i].port == 2) {
-			g_simulator->protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
+			g_simulator->protectAddress(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
 		}
 	}
 	deterministicRandom()->randomShuffle(coordinatorAddresses);
@@ -2405,10 +2411,10 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 			TraceEvent("ProtectCoordinator")
 			    .detail("Address", coordinators[i])
 			    .detail("Coordinators", describe(coordinators));
-			g_simulator->protectedAddresses.insert(
+			g_simulator->protectAddress(
 			    NetworkAddress(coordinators[i].ip, coordinators[i].port, true, coordinators[i].isTLS()));
 			if (coordinators[i].port == 2) {
-				g_simulator->protectedAddresses.insert(NetworkAddress(coordinators[i].ip, 1, true, true));
+				g_simulator->protectAddress(NetworkAddress(coordinators[i].ip, 1, true, true));
 			}
 		}
 	}
@@ -2420,12 +2426,12 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	}
 
 	if (useLocalDatabase) {
-		g_simulator->extraDatabases.push_back(
+		fdbSimulationPolicyState().extraDatabases.push_back(
 		    useHostname ? ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr).toString()
 		                : ClusterConnectionString(coordinatorAddresses, "TestCluster:0"_sr).toString());
-	} else if (testConfig.extraDatabaseMode != ISimulator::ExtraDatabaseMode::Disabled) {
+	} else if (testConfig.extraDatabaseMode != FDBExtraDatabaseMode::Disabled) {
 		for (int i = 0; i < extraDatabaseCount; ++i) {
-			g_simulator->extraDatabases.push_back(
+			fdbSimulationPolicyState().extraDatabases.push_back(
 			    useHostname
 			        ? ClusterConnectionString(extraCoordinatorHostnames[i], StringRef(format("ExtraCluster%04d:0", i)))
 			              .toString()
@@ -2440,7 +2446,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	    .detail("String", conn.toString())
 	    .detail("ConfigString", startingConfigString);
 
-	bool requiresExtraDBMachines = !g_simulator->extraDatabases.empty() && !useLocalDatabase;
+	bool requiresExtraDBMachines = !fdbSimulationPolicyState().extraDatabases.empty() && !useLocalDatabase;
 	int assignedMachines = 0;
 	bool gradualMigrationPossible = true;
 	std::vector<ProcessClass::ClassType> processClassesSubSet = { ProcessClass::UnsetClass,
@@ -2485,7 +2491,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		for (int machine = 0; machine < totalMachines; machine++) {
 			Standalone<StringRef> machineId(deterministicRandom()->randomUniqueID().toString());
 			if (machine == 0 || machineCount - dataCenters <= 4 || assignedMachines != 4 ||
-			    simconfig.db.regions.size() || deterministicRandom()->random01() < 0.5) {
+			    !simconfig.db.regions.empty() || deterministicRandom()->random01() < 0.5) {
 				zoneId = deterministicRandom()->randomUniqueID().toString();
 				newZoneId = deterministicRandom()->randomUniqueID().toString();
 			}
@@ -2496,7 +2502,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 				if (assignedMachines < 4)
 					processClass = ProcessClass((ProcessClass::ClassType)deterministicRandom()->randomInt(0, 2),
 					                            ProcessClass::CommandLineSource); // Unset or Storage
-				else if (assignedMachines == 4 && !simconfig.db.regions.size())
+				else if (assignedMachines == 4 && simconfig.db.regions.empty())
 					processClass = ProcessClass(
 					    processClassesSubSet[deterministicRandom()->randomInt(0, processClassesSubSet.size())],
 					    ProcessClass::CommandLineSource); // Unset or Stateless
@@ -2547,8 +2553,9 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 			localities.set("data_hall"_sr, dcUID);
 			systemActors->push_back(reportErrors(
 			    simulatedMachine(conn,
-			                     requiresExtraDBMachines ? ClusterConnectionString(g_simulator->extraDatabases.at(0))
-			                                             : ClusterConnectionString(),
+			                     requiresExtraDBMachines
+			                         ? ClusterConnectionString(fdbSimulationPolicyState().extraDatabases.at(0))
+			                         : ClusterConnectionString(),
 			                     ips,
 			                     sslEnabled,
 			                     localities,
@@ -2565,7 +2572,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 
 			if (requiresExtraDBMachines) {
 				int cluster = 4;
-				for (auto extraDatabase : g_simulator->extraDatabases) {
+				for (const auto& extraDatabase : fdbSimulationPolicyState().extraDatabases) {
 					std::vector<IPAddress> extraIps;
 					extraIps.reserve(processesPerMachine);
 					for (int i = 0; i < processesPerMachine; i++) {
@@ -2617,13 +2624,13 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		    .detail("DcCoordinators", dcCoordinators);
 	}
 
-	g_simulator->desiredCoordinators = coordinatorCount;
-	g_simulator->physicalDatacenters = dataCenters;
+	fdbSimulationPolicyState().desiredCoordinators = coordinatorCount;
+	fdbSimulationPolicyState().physicalDatacenters = dataCenters;
 	g_simulator->processesPerMachine = processesPerMachine;
 
 	TraceEvent("SetupSimulatorSettings")
-	    .detail("DesiredCoordinators", g_simulator->desiredCoordinators)
-	    .detail("PhysicalDatacenters", g_simulator->physicalDatacenters)
+	    .detail("DesiredCoordinators", fdbSimulationPolicyState().desiredCoordinators)
+	    .detail("PhysicalDatacenters", fdbSimulationPolicyState().physicalDatacenters)
 	    .detail("ProcessesPerMachine", g_simulator->processesPerMachine);
 
 	// SOMEDAY: add locality for testers to simulate network topology
@@ -2654,17 +2661,17 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		                 "SimulatedTesterMachine"));
 	}
 
-	if (g_simulator->setDiffProtocol) {
+	if (fdbSimulationPolicyState().setDiffProtocol) {
 		--(*pTesterCount);
 	}
 
 	*pStartingConfiguration = startingConfigString;
 
 	// save some state that we only need when restarting the simulator.
-	g_simulator->connectionString = conn.toString();
-	g_simulator->testerCount = testerCount;
-	g_simulator->allowStorageMigrationTypeChange = gradualMigrationPossible;
-	g_simulator->willRestart = testConfig.isFirstTestInRestart;
+	fdbSimulationPolicyState().connectionString = conn.toString();
+	fdbSimulationPolicyState().testerCount = testerCount;
+	fdbSimulationPolicyState().allowStorageMigrationTypeChange = gradualMigrationPossible;
+	fdbSimulationPolicyState().willRestart = testConfig.isFirstTestInRestart;
 
 	TraceEvent("SimulatedClusterStarted")
 	    .detail("DataCenters", dataCenters)
@@ -2679,48 +2686,52 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 
 using namespace std::literals;
 
-// Populates the TestConfig fields according to what is found in the test file.
-[[maybe_unused]] void checkTestConf(const char* testFile, TestConfig* testConfig) {}
-
 // Actor that waits for a specified simulation time and then resets the random seed
-ACTOR Future<Void> reseedRandomAtTime(double waitTime, uint32_t newSeed) {
-	wait(delay(waitTime));
+Future<Void> reseedRandomAtTime(double waitTime, uint64_t newSeed) {
+	co_await delay(waitTime);
 	TraceEvent("ResettingRandomSeed").detail("WaitTime", waitTime).detail("NewSeed", newSeed);
 	deterministicRandom()->resetSeed(newSeed);
-	return Void();
 }
 
 } // namespace
 
-ACTOR void simulationSetupAndRun(std::string dataFolder,
-                                 const char* testFile,
-                                 bool rebooting,
-                                 bool restoring,
-                                 std::string whitelistBinPaths,
-                                 double reseedTime) {
-	state std::vector<Future<Void>> systemActors;
-	state Optional<ClusterConnectionString> connectionString;
-	state Standalone<StringRef> startingConfiguration;
-	state int testerCount = 1;
-	state TestConfig testConfig;
-	state IPAllowList allowList;
+static Future<Void> simulationSetupAndRunImpl(std::string dataFolder,
+                                              const char* testFile,
+                                              bool rebooting,
+                                              bool restoring,
+                                              std::string whitelistBinPaths,
+                                              double reseedTime);
+
+void simulationSetupAndRun(std::string const& dataFolder,
+                           const char* const& testFile,
+                           bool const& rebooting,
+                           bool const& restoring,
+                           std::string const& whitelistBinPath,
+                           double const& reseedTime) {
+	uncancellable(simulationSetupAndRunImpl(dataFolder, testFile, rebooting, restoring, whitelistBinPath, reseedTime));
+}
+
+static Future<Void> simulationSetupAndRunImpl(std::string dataFolder,
+                                              const char* testFile,
+                                              bool rebooting,
+                                              bool restoring,
+                                              std::string whitelistBinPaths,
+                                              double reseedTime) {
+	std::vector<Future<Void>> systemActors;
+	Optional<ClusterConnectionString> connectionString;
+	Standalone<StringRef> startingConfiguration;
+	int testerCount = 1;
+	TestConfig testConfig;
+	IPAllowList allowList;
 	testConfig.readFromConfig(testFile);
-	g_simulator->hasDiffProtocolProcess = testConfig.startIncompatibleProcess;
-	g_simulator->setDiffProtocol = false;
+	fdbSimulationPolicyState().hasDiffProtocolProcess = testConfig.startIncompatibleProcess;
+	fdbSimulationPolicyState().setDiffProtocol = false;
 	if (testConfig.injectTargetedSSRestart && deterministicRandom()->random01() < 0.25) {
-		g_simulator->injectTargetedSSRestartTime = 60.0 + 340.0 * deterministicRandom()->random01();
+		fdbSimulationPolicyState().injectTargetedSSRestartTime = 60.0 + 340.0 * deterministicRandom()->random01();
 	}
 
 	if (testConfig.injectSSDelay && deterministicRandom()->random01() < 0.25) {
-		g_simulator->injectSSDelayTime = 60.0 + 240.0 * deterministicRandom()->random01();
-	}
-
-	if (deterministicRandom()->random01() < 0.25) {
-		g_simulator->injectTargetedBMRestartTime = 60.0 + 340.0 * deterministicRandom()->random01();
-	}
-
-	if (deterministicRandom()->random01() < 0.25) {
-		g_simulator->injectTargetedBWRestartTime = 60.0 + 340.0 * deterministicRandom()->random01();
+		fdbSimulationPolicyState().injectSSDelayTime = 60.0 + 240.0 * deterministicRandom()->random01();
 	}
 
 	// Build simulator allow list
@@ -2738,7 +2749,7 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 		testConfig.storageEngineExcludeTypes.insert(SimulationStorageEngine::SHARDED_ROCKSDB);
 	}
 
-	state ProtocolVersion protocolVersion = currentProtocolVersion();
+	ProtocolVersion protocolVersion = currentProtocolVersion();
 	if (testConfig.startIncompatibleProcess) {
 		// isolates right most 1 bit of compatibleProtocolVersionMask to make this protocolVersion incompatible
 		uint64_t minAddToMakeIncompatible =
@@ -2763,23 +2774,27 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 	                            currentProtocolVersion(),
 	                            false);
 	testSystem->excludeFromRestarts = true;
-	wait(g_simulator->onProcess(testSystem, TaskPriority::DefaultYield));
+	co_await g_simulator->onProcess(testSystem, TaskPriority::DefaultYield);
 	Sim2FileSystem::newFileSystem();
 	FlowTransport::createInstance(true, 1, WLTOKEN_RESERVED_COUNT, &allowList);
 	CODE_PROBE(true, "Simulation start");
 
+	bool testsFinished = false;
+	int timeoutSeconds = isGeneralBuggifyEnabled() ? testConfig.simulationBuggifyRunTestsTimeoutSeconds
+	                                               : testConfig.simulationNormalRunTestsTimeoutSeconds;
+
 	try {
 		// systemActors.push_back( startSystemMonitor(dataFolder) );
 		if (rebooting) {
-			wait(timeoutError(restartSimulatedSystem(&systemActors,
-			                                         dataFolder,
-			                                         &testerCount,
-			                                         &connectionString,
-			                                         &startingConfiguration,
-			                                         &testConfig,
-			                                         whitelistBinPaths,
-			                                         protocolVersion),
-			                  100.0));
+			co_await timeoutError(restartSimulatedSystem(&systemActors,
+			                                             dataFolder,
+			                                             &testerCount,
+			                                             &connectionString,
+			                                             &startingConfiguration,
+			                                             &testConfig,
+			                                             whitelistBinPaths,
+			                                             protocolVersion),
+			                      100.0);
 			// FIXME: snapshot restore does not support multi-region restore, hence restore it as single region always
 			if (restoring) {
 				startingConfiguration = "usable_regions=1"_sr;
@@ -2794,31 +2809,32 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 			                     whitelistBinPaths,
 			                     testConfig,
 			                     protocolVersion);
-			wait(delay(1.0)); // FIXME: WHY!!!  //wait for machines to boot
+			co_await delay(1.0); // FIXME: WHY!!!  //wait for machines to boot
 			// Yeah right, I agree, Whyyyyy?  Since it's 1s, should we poll some condition?  Is there a risk
 			// 1s is not enough and random weird bad stuff happens after this?
 		}
 
-		wait(HTTP::registerAlwaysFailHTTPHandler());
+		TraceEvent("TestProgress").detail("Progress", "StartedSimulatedSystem");
+
+		co_await HTTP::registerAlwaysFailHTTPHandler();
 
 		std::string clusterFileDir = joinPath(dataFolder, deterministicRandom()->randomUniqueID().toString());
 		platform::createDirectory(clusterFileDir);
 		writeFile(joinPath(clusterFileDir, "fdb.cluster"), connectionString.get().toString());
-		state Reference<ClusterConnectionFile> connFile =
-		    makeReference<ClusterConnectionFile>(joinPath(clusterFileDir, "fdb.cluster"));
+		auto connFile = makeReference<ClusterConnectionFile>(joinPath(clusterFileDir, "fdb.cluster"));
 		if (rebooting) {
 			// protect coordinators for restarting tests
 			std::vector<NetworkAddress> coordinatorAddresses =
-			    wait(connFile->getConnectionString().tryResolveHostnames());
-			ASSERT(coordinatorAddresses.size() > 0);
+			    co_await connFile->getConnectionString().tryResolveHostnames();
+			ASSERT(!coordinatorAddresses.empty());
 			for (int i = 0; i < (coordinatorAddresses.size() / 2) + 1; i++) {
 				TraceEvent("ProtectCoordinator")
 				    .detail("Address", coordinatorAddresses[i])
 				    .detail("Coordinators", describe(coordinatorAddresses));
-				g_simulator->protectedAddresses.insert(NetworkAddress(
+				g_simulator->protectAddress(NetworkAddress(
 				    coordinatorAddresses[i].ip, coordinatorAddresses[i].port, true, coordinatorAddresses[i].isTLS()));
 				if (coordinatorAddresses[i].port == 2) {
-					g_simulator->protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
+					g_simulator->protectAddress(NetworkAddress(coordinatorAddresses[i].ip, 1, true, true));
 				}
 			}
 		}
@@ -2826,7 +2842,7 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 		// If reseedTime is set, schedule a random seed reset at a random time between [0, reseedTime]
 		if (reseedTime >= 0.0) {
 			double actualReseedTime = nondeterministicRandom()->random01() * reseedTime;
-			uint32_t newSeed = platform::getRandomSeed();
+			uint64_t newSeed = platform::getRandomSeed();
 			TraceEvent("SchedulingRandomSeedReset")
 			    .detail("ReseedTimeMax", reseedTime)
 			    .detail("ActualReseedTime", actualReseedTime)
@@ -2834,6 +2850,8 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 			systemActors.push_back(reseedRandomAtTime(actualReseedTime, newSeed));
 		}
 
+		TraceEvent("TestProgress").detail("Progress", "AboutToRunTests");
+
 		Future<Void> runTestsF = runTests(connFile,
 		                                  TEST_TYPE_FROM_FILE,
 		                                  TEST_ON_TESTERS,
@@ -2843,17 +2861,16 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 		                                  LocalityData(),
 		                                  UnitTestParameters(),
 		                                  rebooting);
-		wait(testConfig.longRunningTest
-		         ? runTestsF
-		         : timeoutError(runTestsF,
-		                        isGeneralBuggifyEnabled() ? testConfig.simulationBuggifyRunTestsTimeoutSeconds
-		                                                  : testConfig.simulationNormalRunTestsTimeoutSeconds));
+		co_await (testConfig.longRunningTest ? runTestsF : timeoutError(runTestsF, timeoutSeconds));
+
+		testsFinished = true;
+		TraceEvent("TestProgress").detail("Progress", "RunTestsFinished");
 	} catch (Error& e) {
-		auto timeoutVal = isGeneralBuggifyEnabled() ? testConfig.simulationBuggifyRunTestsTimeoutSeconds
-		                                            : testConfig.simulationNormalRunTestsTimeoutSeconds;
-		auto msg = fmt::format("Timeout after {} simulated seconds", timeoutVal);
-		ProcessEvents::trigger("Timeout"_sr, StringRef(msg), e);
-		TraceEvent(SevError, "SetupAndRunError").error(e);
+		if (e.code() == error_code_timed_out) {
+			auto msg = fmt::format("Timeout after {} simulated seconds", timeoutSeconds);
+			ProcessEvents::trigger("Timeout"_sr, StringRef(msg), e);
+		}
+		TraceEvent(SevError, "SetupAndRunError").error(e).detail("TestsFinished", testsFinished);
 	}
 
 	TraceEvent("TracingMissingCodeProbes").log();
@@ -2861,7 +2878,7 @@ ACTOR void simulationSetupAndRun(std::string dataFolder,
 	TraceEvent("SimulatedSystemDestruct").log();
 	g_simulator->stop();
 	destructed = true;
-	wait(Never());
+	co_await Future<Void>(Never());
 	ASSERT(false);
 }
 
@@ -2894,8 +2911,3 @@ int getMaxSatelliteLogs() {
 	// Cap at 6 (the original max) and ensure at least 1
 	return std::max(1, std::min(6, minSatelliteMachines));
 }
-
-BasicSimulationConfig generateBasicSimulationConfig(const BasicTestConfig& testConfig) {
-	TestConfig config(testConfig);
-	return SimulationConfig(config);
-}
diff --git a/fdbserver/TSSMappingUtil.actor.cpp b/fdbserver/TSSMappingUtil.actor.cpp
deleted file mode 100644
index e0e218241b7..00000000000
--- a/fdbserver/TSSMappingUtil.actor.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * TSSMappingUtil.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/SystemData.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbserver/TSSMappingUtil.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
-                                     std::map<UID, StorageServerInterface>* tssMapping) {
-	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
-	state KeyBackedMap<UID, UID>::RangeResultType uidMapping =
-	    wait(tssMapDB.getRange(tr, UID(), Optional<UID>(), CLIENT_KNOBS->TOO_MANY));
-	ASSERT(uidMapping.results.size() < CLIENT_KNOBS->TOO_MANY);
-
-	state std::map<UID, StorageServerInterface> mapping;
-	state std::vector<std::pair<UID, UID>>::iterator mapItr;
-	for (mapItr = uidMapping.results.begin(); mapItr != uidMapping.results.end(); ++mapItr) {
-		state UID ssId = mapItr->first;
-		Optional<Value> v = wait(tr->get(serverListKeyFor(mapItr->second)));
-		(*tssMapping)[ssId] = decodeServerListValue(v.get());
-	}
-	return Void();
-}
-
-ACTOR Future<Void> readTSSMapping(Transaction* tr, std::map<UID, StorageServerInterface>* tssMapping) {
-	state RangeResult mappingList = wait(tr->getRange(tssMappingKeys, CLIENT_KNOBS->TOO_MANY));
-	ASSERT(!mappingList.more && mappingList.size() < CLIENT_KNOBS->TOO_MANY);
-
-	for (auto& it : mappingList) {
-		state UID ssId = TupleCodec<UID>::unpack(it.key.removePrefix(tssMappingKeys.begin));
-		UID tssId = TupleCodec<UID>::unpack(it.value);
-		Optional<Value> v = wait(tr->get(serverListKeyFor(tssId)));
-		(*tssMapping)[ssId] = decodeServerListValue(v.get());
-	}
-	return Void();
-}
-
-ACTOR Future<Void> removeTSSPairsFromCluster(Database cx, std::vector<std::pair<UID, UID>> pairsToRemove) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
-	loop {
-		try {
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			for (auto& tssPair : pairsToRemove) {
-				// DO NOT remove server list key - that'll break a bunch of stuff. DD will eventually call
-				// removeStorageServer
-				tr->clear(serverTagKeyFor(tssPair.second));
-				tssMapDB.erase(tr, tssPair.first);
-			}
-			wait(tr->commit());
-			break;
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-	return Void();
-}
diff --git a/fdbserver/BackupPartitionMap.actor.cpp b/fdbserver/backupworker/BackupPartitionMap.cpp
similarity index 69%
rename from fdbserver/BackupPartitionMap.actor.cpp
rename to fdbserver/backupworker/BackupPartitionMap.cpp
index 16f369c9e76..f7d18f0e3fc 100644
--- a/fdbserver/BackupPartitionMap.actor.cpp
+++ b/fdbserver/backupworker/BackupPartitionMap.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupPartitionMap.actor.cpp
+ * BackupPartitionMap.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,20 +18,39 @@
  * limitations under the License.
  */
 
-#include "fdbserver/DDShardTracker.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "BackupPartitionMap.h"
+#include "fdbclient/JsonBuilder.h"
+#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/SystemData.h"
+
+std::string serializePartitionListJSON(PartitionMap const& partitionMap) {
+	JsonBuilderObject root;
+	JsonBuilderArray partitionsArray;
+	for (const auto& [tag, partitionList] : partitionMap) {
+		for (const auto& partition : partitionList) {
+			JsonBuilderObject partitionObj;
+			partitionObj["partitionId"] = partition.partitionId;
+			partitionObj["beginKey"] = partition.ranges.begin.printable();
+			partitionObj["endKey"] = partition.ranges.end.printable();
+			partitionsArray.push_back(partitionObj);
+		}
+	}
+	root["partitions"] = partitionsArray;
+	root["totalPartitions"] = partitionsArray.size();
+	return root.getJson();
+}
 
 // KeyRangeMap guarantees that key ranges are contiguous with no gaps in shards.
-ACTOR Future<std::vector<KeyRange>> calculateBackupPartitions(KeyRangeMap<ShardTrackedData>* shards) {
+Future<std::vector<KeyRange>> calculateBackupPartitionKeyRanges(KeyRangeMap<ShardTrackedData>* shards) {
 	// TODO akanksha: Hardcoded for now.
-	state const int NUM_PARTITIONS = 100;
-	state std::vector<std::pair<KeyRange, int64_t>> userShards; // Pair of shard key range and shard size in bytes.
-	state int64_t totalBytes = 0;
+	const int NUM_PARTITIONS = 100;
+	std::vector<std::pair<KeyRange, int64_t>> userShards; // Pair of shard key range and shard size in bytes.
+	int64_t totalBytes = 0;
 
 	// Step 1: Collect shard sizes
-	loop {
-		state bool needWait = false;
-		state Future<Void> onChange;
+	while (true) {
+		bool needWait = false;
+		Future<Void> onChange;
 		userShards.clear();
 		totalBytes = 0;
 		for (auto it : shards->intersectingRanges(normalKeys)) {
@@ -51,29 +70,29 @@ ACTOR Future<std::vector<KeyRange>> calculateBackupPartitions(KeyRangeMap<ShardT
 		if (!needWait) {
 			break;
 		}
-		wait(onChange);
+		co_await onChange;
 	}
 
 	// Step 2: Partition the shards
 	// Integer division is acceptable here as any rounding remainder is added to the last partition.
 	int64_t targetBytesPerPartition = totalBytes / NUM_PARTITIONS;
-	state std::vector<KeyRange> partitions;
-	state int64_t currentPartitionBytes = 0;
-	state Key partitionStart = normalKeys.begin;
+	std::vector<KeyRange> partitionKeyRanges;
+	int64_t currentPartitionBytes = 0;
+	Key partitionStart = normalKeys.begin;
 
 	for (int i = 0; i < userShards.size(); i++) {
 		currentPartitionBytes += userShards[i].second;
 		// Checks if new partition should be started.
 		if ((currentPartitionBytes >= targetBytesPerPartition) || (i == userShards.size() - 1)) {
-			partitions.push_back(KeyRangeRef(partitionStart, userShards[i].first.end));
+			partitionKeyRanges.push_back(KeyRangeRef(partitionStart, userShards[i].first.end));
 			partitionStart = userShards[i].first.end;
 			currentPartitionBytes = 0;
 		}
 	}
-	return partitions;
+	co_return partitionKeyRanges;
 }
 
-TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/NoUserShards") {
+TEST_CASE("/BackupPartitionMap/calculateBackupPartitionKeyRanges/NoUserShards") {
 	ShardTrackedData defaultData;
 	StorageMetrics zeroMetrics;
 	zeroMetrics.bytes = 0;
@@ -89,15 +108,14 @@ TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/NoUserShards") {
 	systemData.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>(zeroShard);
 	shards.insert(systemKeys, systemData);
 
-	std::vector<KeyRange> partitions = wait(calculateBackupPartitions(&shards));
+	std::vector<KeyRange> partitions = co_await calculateBackupPartitionKeyRanges(&shards);
 
 	ASSERT(partitions.size() == 1);
 	ASSERT(partitions[0].begin == normalKeys.begin);
 	ASSERT(partitions[0].end == normalKeys.end);
-	return Void();
 }
 
-TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/SingleShard") {
+TEST_CASE("/BackupPartitionMap/calculateBackupPartitionKeyRanges/SingleShard") {
 	ShardTrackedData defaultData;
 	defaultData.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
 	KeyRangeMap<ShardTrackedData> shards(defaultData);
@@ -109,34 +127,33 @@ TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/SingleShard") {
 	data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>(shardMetrics);
 	shards.insert(normalKeys, data);
 
-	std::vector<KeyRange> partitions = wait(calculateBackupPartitions(&shards));
+	std::vector<KeyRange> partitions = co_await calculateBackupPartitionKeyRanges(&shards);
 	ASSERT(partitions.size() == 1);
 	ASSERT(partitions[0].begin == normalKeys.begin);
 	ASSERT(partitions[0].end == normalKeys.end);
-	return Void();
 }
 
-TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/VaryingSizes") {
+TEST_CASE("/BackupPartitionMap/calculateBackupPartitionKeyRanges/VaryingSizes") {
 	ShardTrackedData defaultData;
 	defaultData.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
 	KeyRangeMap<ShardTrackedData> shards(defaultData);
-	state Key key1 = "a"_sr;
-	state Key key2 = "b"_sr;
-	state Key key3 = "c"_sr;
-	state Key key4 = normalKeys.end;
+	Key key1 = "a"_sr;
+	Key key2 = "b"_sr;
+	Key key3 = "c"_sr;
+	Key key4 = normalKeys.end;
 	std::vector<std::pair<KeyRange, int64_t>> testShards = { { KeyRangeRef(normalKeys.begin, key1), 50000 },
 		                                                     { KeyRangeRef(key1, key2), 200000 },
 		                                                     { KeyRangeRef(key2, key3), 10000 },
 		                                                     { KeyRangeRef(key3, key4), 90000 } };
-	for (const auto& shard : testShards) {
+	for (const auto& [range, bytes] : testShards) {
 		ShardTrackedData data;
 		StorageMetrics metrics;
-		metrics.bytes = shard.second;
+		metrics.bytes = bytes;
 		ShardMetrics shardMetrics(metrics, 0.0, 1);
 		data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>(shardMetrics);
-		shards.insert(shard.first, data);
+		shards.insert(range, data);
 	}
-	std::vector<KeyRange> partitions = wait(calculateBackupPartitions(&shards));
+	std::vector<KeyRange> partitions = co_await calculateBackupPartitionKeyRanges(&shards);
 
 	ASSERT(partitions.size() == 4);
 	ASSERT(partitions[0].begin == normalKeys.begin);
@@ -147,43 +164,41 @@ TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/VaryingSizes") {
 	ASSERT(partitions[2].end == key3);
 	ASSERT(partitions[3].begin == key3);
 	ASSERT(partitions[3].end == key4);
-	return Void();
 }
 
-TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/ZeroSizeShards") {
+TEST_CASE("/BackupPartitionMap/calculateBackupPartitionKeyRanges/ZeroSizeShards") {
 	ShardTrackedData defaultData;
 	defaultData.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
 	KeyRangeMap<ShardTrackedData> shards(defaultData);
-	state Key key1 = "a"_sr;
-	state Key key2 = "b"_sr;
-	state Key key3 = normalKeys.end;
+	Key key1 = "a"_sr;
+	Key key2 = "b"_sr;
+	Key key3 = normalKeys.end;
 
 	std::vector<std::pair<KeyRange, int64_t>> testShards = { { KeyRangeRef(normalKeys.begin, key1), 0 },
 		                                                     { KeyRangeRef(key1, key2), 1000000 },
 		                                                     { KeyRangeRef(key2, key3), 0 } };
 
-	for (const auto& shard : testShards) {
+	for (const auto& [range, bytes] : testShards) {
 		ShardTrackedData data;
 		StorageMetrics metrics;
-		metrics.bytes = shard.second;
+		metrics.bytes = bytes;
 		ShardMetrics shardMetrics(metrics, 0.0, 1);
 		data.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>(shardMetrics);
-		shards.insert(shard.first, data);
+		shards.insert(range, data);
 	}
-	std::vector<KeyRange> partitions = wait(calculateBackupPartitions(&shards));
+	std::vector<KeyRange> partitions = co_await calculateBackupPartitionKeyRanges(&shards);
 	ASSERT(partitions.size() == 2);
 	ASSERT(partitions[0].begin == normalKeys.begin);
 	ASSERT(partitions[0].end == key2);
 	ASSERT(partitions[1].begin == key2);
 	ASSERT(partitions[1].end == normalKeys.end);
-	return Void();
 }
 
-ACTOR Future<Void> testAsyncMetricsUpdate() {
-	state ShardTrackedData defaultData;
+Future<Void> testAsyncMetricsUpdate() {
+	ShardTrackedData defaultData;
 	defaultData.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
-	state KeyRangeMap<ShardTrackedData> shards(defaultData);
-	state Key splitKey = "split"_sr;
+	KeyRangeMap<ShardTrackedData> shards(defaultData);
+	Key splitKey = "split"_sr;
 	ShardTrackedData emptyData;
 	emptyData.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>();
 	shards.insert(KeyRangeRef(normalKeys.begin, splitKey), emptyData);
@@ -195,29 +210,27 @@ ACTOR Future<Void> testAsyncMetricsUpdate() {
 	dataWithMetrics.stats = makeReference<AsyncVar<Optional<ShardMetrics>>>(shardMetrics);
 	shards.insert(KeyRangeRef(splitKey, normalKeys.end), dataWithMetrics);
 
-	state Future<std::vector<KeyRange>> resultFuture = calculateBackupPartitions(&shards);
+	Future<std::vector<KeyRange>> resultFuture = calculateBackupPartitionKeyRanges(&shards);
 
-	wait(delay(0.1));
+	co_await delay(0.1);
 	ASSERT(!resultFuture.isReady());
 	StorageMetrics newMetrics;
 	newMetrics.bytes = 50000;
 	ShardMetrics newShardMetrics(newMetrics, 0.0, 1);
 	shards.rangeContaining(normalKeys.begin)->value().stats->set(newShardMetrics);
-	std::vector<KeyRange> partitions = wait(resultFuture);
+	std::vector<KeyRange> partitions = co_await resultFuture;
 	ASSERT(partitions.size() == 2);
 	ASSERT(partitions[0].begin == normalKeys.begin);
 	ASSERT(partitions[0].end == splitKey);
 	ASSERT(partitions[1].begin == splitKey);
 	ASSERT(partitions[1].end == normalKeys.end);
-	return Void();
 }
 
-TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/WaitForAsyncMetrics") {
-	wait(testAsyncMetricsUpdate());
-	return Void();
+TEST_CASE("/BackupPartitionMap/calculateBackupPartitionKeyRanges/WaitForAsyncMetrics") {
+	co_await testAsyncMetricsUpdate();
 }
 
-TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/MultipleSmallShards") {
+TEST_CASE("/BackupPartitionMap/calculateBackupPartitionKeyRanges/MultipleSmallShards") {
 	ShardTrackedData defaultData;
 	StorageMetrics defaultMetrics;
 	defaultMetrics.bytes = 0;
@@ -237,7 +250,7 @@ TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/MultipleSmallShards") {
 		shards.insert(KeyRangeRef(start, end), data);
 	}
 
-	std::vector<KeyRange> partitions = wait(calculateBackupPartitions(&shards));
+	std::vector<KeyRange> partitions = co_await calculateBackupPartitionKeyRanges(&shards);
 
 	ASSERT(partitions.size() == 100);
 	ASSERT(partitions[0].begin == normalKeys.begin);
@@ -245,5 +258,4 @@ TEST_CASE("/BackupPartitionMap/calculateBackupPartitions/MultipleSmallShards") {
 	for (int i = 1; i < partitions.size(); i++) {
 		ASSERT(partitions[i - 1].end == partitions[i].begin);
 	}
-	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbserver/backupworker/BackupPartitionMap.h b/fdbserver/backupworker/BackupPartitionMap.h
new file mode 100644
index 00000000000..a9217ab57f5
--- /dev/null
+++ b/fdbserver/backupworker/BackupPartitionMap.h
@@ -0,0 +1,44 @@
+/*
+ * BackupPartitionMap.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbserver/core/ShardMetrics.h"
+#include <unordered_map>
+#include <vector>
+
+struct Partition {
+	int32_t partitionId;
+	KeyRange ranges;
+
+	Partition() : partitionId(-1) {}
+	Partition(int32_t id, KeyRange r) : partitionId(id), ranges(r) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, partitionId, ranges);
+	}
+};
+
+typedef std::vector<Partition> PartitionList;
+typedef std::unordered_map<Tag, PartitionList> PartitionMap;
+
+std::string serializePartitionListJSON(PartitionMap const& PartitionMap);
diff --git a/fdbserver/backupworker/BackupRangePartitionedProgress.cpp b/fdbserver/backupworker/BackupRangePartitionedProgress.cpp
new file mode 100644
index 00000000000..120a9aba644
--- /dev/null
+++ b/fdbserver/backupworker/BackupRangePartitionedProgress.cpp
@@ -0,0 +1,73 @@
+/*
+ * BackupRangePartitionedProgress.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "BackupRangePartitionedProgress.h"
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/SystemData.h"
+
+void BackupRangePartitionedProgress::addBackupStatus(const WorkerBackupStatus& status) {
+	auto& it = progress[status.epoch];
+	auto lb = it.lower_bound(status.tag);
+	if (lb != it.end() && status.tag == lb->first) {
+		if (lb->second < status.version) {
+			lb->second = status.version;
+		}
+	} else {
+		it.insert(lb, { status.tag, status.version });
+	}
+}
+
+Future<Void> getBackupRangePartitionedProgress(Database cx,
+                                               UID dbgid,
+                                               Reference<BackupRangePartitionedProgress> bStatus,
+                                               Severity severity) {
+	Transaction tr(cx);
+
+	while (true) {
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+
+			RangeResult results = co_await tr.getRange(backupRangePartitionedProgressKeys, CLIENT_KNOBS->TOO_MANY);
+			ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
+
+			for (auto& it : results) {
+				const UID workerID = decodeBackupRangePartitionedProgressKey(it.key);
+				const WorkerBackupStatus status = decodeBackupRangePartitionedProgressValue(it.value);
+				bStatus->addBackupStatus(status);
+
+				TraceEvent(severity, "GotBackupRangePartitionedProgress", dbgid)
+				    .detail("BackupWorker", workerID)
+				    .detail("Epoch", status.epoch)
+				    .detail("Version", status.version)
+				    .detail("Tag", status.tag.toString())
+				    .detail("TotalTags", status.totalTags);
+			}
+			co_return;
+		} catch (Error& e) {
+			err = e;
+		}
+
+		co_await tr.onError(err);
+	}
+}
diff --git a/fdbserver/backupworker/BackupRangePartitionedProgress.h b/fdbserver/backupworker/BackupRangePartitionedProgress.h
new file mode 100644
index 00000000000..7f5f33592a7
--- /dev/null
+++ b/fdbserver/backupworker/BackupRangePartitionedProgress.h
@@ -0,0 +1,65 @@
+/*
+ * BackupRangePartitionedProgress.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <tuple>
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+
+class BackupRangePartitionedProgress : NonCopyable, ReferenceCounted<BackupRangePartitionedProgress> {
+public:
+	explicit BackupRangePartitionedProgress(UID id) : dbgid(id) {}
+	~BackupRangePartitionedProgress() {}
+
+	// Adds a backup status. If the tag already has an entry, then the max of
+	// savedVersion is used.
+	void addBackupStatus(const WorkerBackupStatus& status);
+
+	// Returns progress for an epoch.
+	std::map<Tag, Version> getEpochStatus(LogEpoch epoch) const {
+		const auto it = progress.find(epoch);
+		if (it == progress.end())
+			return {};
+		return it->second;
+	}
+
+	void addref() { ReferenceCounted<BackupRangePartitionedProgress>::addref(); }
+
+	void delref() { ReferenceCounted<BackupRangePartitionedProgress>::delref(); }
+
+private:
+	// Used for logging and debugging purpose to identify which backup progress it is.
+	const UID dbgid;
+
+	// Backup progress saved in the system keyspace. Note there can be multiple
+	// progress status for a tag in an epoch due to later epoch trying to fill
+	// the gap. "progress" MUST be iterated in ascending order.
+	std::map<LogEpoch, std::map<Tag, Version>> progress;
+};
+
+Future<Void> getBackupRangePartitionedProgress(Database cx,
+                                               UID dbgid,
+                                               Reference<BackupRangePartitionedProgress> bStatus,
+                                               Severity severity);
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/backupworker/BackupWorker.cpp
similarity index 63%
rename from fdbserver/BackupWorker.actor.cpp
rename to fdbserver/backupworker/BackupWorker.cpp
index 1ba3819af57..ecabea2dce0 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/backupworker/BackupWorker.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupWorker.actor.cpp
+ * BackupWorker.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,25 +18,26 @@
  * limitations under the License.
  */
 
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/BackupInterface.h"
-#include "fdbserver/BackupProgress.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogProtocolMessage.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/BackupInterface.h"
+#include "fdbserver/core/BackupProgress.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/LogProtocolMessage.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/backupworker/BackupWorker.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/Error.h"
 
 #include "flow/IRandom.h"
 #include "fdbclient/Tracing.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 #define SevDebugMemory SevVerbose
 
@@ -112,15 +113,12 @@ struct BackupData {
 	LogEpoch oldestBackupEpoch = 0; // oldest epoch that still has data on tLogs for backup to pull
 	Version minKnownCommittedVersion;
 	Version savedVersion; // Largest version saved to blob storage
-	Version popVersion; // Largest version popped in NOOP mode, can be larger than savedVersion.
 	Reference<AsyncVar<ServerDBInfo> const> db;
-	AsyncVar<Reference<ILogSystem>> logSystem;
+	AsyncVar<Reference<LogSystem>> logSystem;
 	Database cx;
 	std::vector<VersionedMessage> messages;
 	NotifiedVersion pulledVersion;
-	bool pulling = false;
 	bool stopped = false;
-	bool exitEarly = false; // If the worker is on an old epoch and all backups starts a version >= the endVersion
 	AsyncVar<bool> paused; // Track if "backupPausedKey" is set.
 	Reference<FlowLock> lock;
 
@@ -153,9 +151,8 @@ struct BackupData {
 			return _waitReady(this);
 		}
 
-		ACTOR static Future<Void> _waitReady(PerBackupInfo* info) {
-			wait(success(info->container) && success(info->ranges));
-			return Void();
+		static Future<Void> _waitReady(PerBackupInfo* info) {
+			co_await (success(info->container) && success(info->ranges));
 		}
 
 		// Update the number of backup workers in the BackupConfig. Each worker
@@ -164,24 +161,23 @@ struct BackupData {
 		// (i.e., the "submitBackup" call is successful). Worker 0 then sets
 		// the "allWorkerStarted" flag, which in turn unblocks
 		// StartFullBackupTaskFunc::_execute.
-		ACTOR static Future<Void> _updateStartedWorkers(PerBackupInfo* info, BackupData* self, UID uid) {
-			state BackupConfig config(uid);
-			state Future<Void> watchFuture;
-			state bool updated = false;
-			state bool firstWorker = info->self->tag.id == 0;
-			state bool allUpdated = false;
-			state Optional<std::vector<std::pair<int64_t, int64_t>>> workers;
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
-
-			loop {
+		static Future<Void> _updateStartedWorkers(PerBackupInfo* info, BackupData* self, UID uid) {
+			BackupConfig config(uid);
+			Future<Void> watchFuture;
+			bool updated = false;
+			const bool firstWorker = info->self->tag.id == 0;
+			bool allUpdated = false;
+			Optional<std::vector<std::pair<int64_t, int64_t>>> workers;
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+
+			while (true) {
+				Error err;
 				try {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 					tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-					Optional<std::vector<std::pair<int64_t, int64_t>>> tmp =
-					    wait(config.startedBackupWorkers().get(tr));
-					workers = tmp;
+					workers = co_await config.startedBackupWorkers().get(tr);
 					if (!updated) {
 						if (workers.present()) {
 							workers.get().emplace_back(self->recruitedEpoch, (int64_t)self->tag.id);
@@ -190,13 +186,14 @@ struct BackupData {
 							workers = Optional<std::vector<std::pair<int64_t, int64_t>>>(v);
 						}
 					}
+
 					if (firstWorker) {
 						if (!workers.present()) {
 							TraceEvent("BackupWorkerDetectAbortedJob", self->myId).detail("BackupID", uid);
-							return Void();
+							co_return;
 						}
-						ASSERT(workers.present() && workers.get().size() > 0);
-						std::vector<std::pair<int64_t, int64_t>>& v = workers.get();
+						ASSERT(workers.present() && !workers.get().empty());
+						auto& v = workers.get();
 						v.erase(std::remove_if(v.begin(),
 						                       v.end(),
 						                       [epoch = self->recruitedEpoch](const std::pair<int64_t, int64_t>& p) {
@@ -204,7 +201,7 @@ struct BackupData {
 						                       }),
 						        v.end());
 						std::set<int64_t> tags;
-						for (auto p : v) {
+						for (const auto& p : v) {
 							tags.insert(p.second);
 						}
 						if (self->totalTags == tags.size()) {
@@ -214,36 +211,37 @@ struct BackupData {
 							// monitor all workers' updates
 							watchFuture = tr->watch(config.startedBackupWorkers().key);
 						}
-						ASSERT(workers.present() && workers.get().size() > 0);
+						ASSERT(workers.present() && !workers.get().empty());
 						if (!updated) {
 							config.startedBackupWorkers().set(tr, workers.get());
 						}
-						for (auto p : workers.get()) {
+						for (const auto& p : workers.get()) {
 							TraceEvent("BackupWorkerDebugTag", self->myId)
 							    .detail("Epoch", p.first)
 							    .detail("TagID", p.second);
 						}
-						wait(tr->commit());
+						co_await tr->commit();
 
 						updated = true; // Only set to true after commit.
 						if (allUpdated) {
 							break;
 						}
-						wait(watchFuture);
+						co_await watchFuture;
 						tr->reset();
+						continue;
 					} else {
-						ASSERT(workers.present() && workers.get().size() > 0);
+						ASSERT(workers.present() && !workers.get().empty());
 						config.startedBackupWorkers().set(tr, workers.get());
-						wait(tr->commit());
+						co_await tr->commit();
 						break;
 					}
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 					allUpdated = false;
 				}
+				co_await tr->onError(err);
 			}
 			TraceEvent("BackupWorkerSetReady", self->myId).detail("BackupID", uid).detail("TagId", self->tag.id);
-			return Void();
 		}
 
 		BackupData* self = nullptr;
@@ -265,15 +263,12 @@ struct BackupData {
 
 	CounterCollection cc;
 	Future<Void> logger;
-	Future<Void> noopPopper; // holds actor to save progress in NOOP mode
-	AsyncVar<Version> popTrigger; // trigger to pop version in NOOP mode
 
 	explicit BackupData(UID id, Reference<AsyncVar<ServerDBInfo> const> db, const InitializeBackupRequest& req)
 	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
 	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
-	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), popVersion(req.startVersion - 1),
-	    db(db), pulledVersion(0), paused(false), lock(new FlowLock(SERVER_KNOBS->BACKUP_WORKER_LOCK_BYTES)),
-	    cc("BackupWorker", myId.toString()) {
+	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), db(db), pulledVersion(0),
+	    paused(false), lock(new FlowLock(SERVER_KNOBS->BACKUP_WORKER_LOCK_BYTES)), cc("BackupWorker", myId.toString()) {
 		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True);
 
 		specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; });
@@ -283,15 +278,11 @@ struct BackupData {
 		specialCounter(cc, "AvailableBytes", [this]() { return this->lock->available(); });
 		logger =
 		    cc.traceCounters("BackupWorkerMetrics", myId, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, "BackupWorkerMetrics");
-		popTrigger.set(invalidVersion);
-		noopPopper = _noopPopper(this);
 	}
 
 	bool pullFinished() const { return endVersion.present() && pulledVersion.get() > endVersion.get(); }
 
-	bool allMessageSaved() const {
-		return (endVersion.present() && savedVersion >= endVersion.get()) || stopped || exitEarly;
-	}
+	bool allMessageSaved() const { return (endVersion.present() && savedVersion >= endVersion.get()) || stopped; }
 
 	Version maxPopVersion() const { return endVersion.present() ? endVersion.get() : minKnownCommittedVersion; }
 
@@ -338,11 +329,8 @@ struct BackupData {
 		}
 		ASSERT_WE_THINK(backupEpoch == oldestBackupEpoch);
 		const Tag popTag = logSystem.get()->getPseudoPopTag(tag, ProcessClass::BackupClass);
-		DisabledTraceEvent("BackupWorkerPop", myId)
-		    .detail("Tag", popTag)
-		    .detail("SavedVersion", savedVersion)
-		    .detail("PopVersion", popVersion);
-		logSystem.get()->pop(std::max(popVersion, savedVersion), popTag);
+		DisabledTraceEvent("BackupWorkerPop", myId).detail("Tag", popTag).detail("SavedVersion", savedVersion);
+		logSystem.get()->pop(savedVersion, popTag);
 	}
 
 	void stop() {
@@ -393,7 +381,7 @@ struct BackupData {
 	// to start new backups and stop ones not in the active state.
 	void onBackupChanges(const std::vector<std::pair<UID, Version>>& uidVersions) {
 		std::set<UID> stopList;
-		for (auto it : backups) {
+		for (const auto& it : backups) {
 			stopList.insert(it.first);
 		}
 
@@ -430,58 +418,7 @@ struct BackupData {
 			changedTrigger.trigger();
 	}
 
-	// Update the NOOP popped version so that when backup is started or resumed,
-	// the worker can ignore any versions that are already popped.
-	ACTOR static Future<Void> _saveNoopVersion(BackupData* self, Version poppedVersion) {
-		state Transaction tr(self->cx);
-
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-
-				Optional<Value> noopValue = wait(tr.get(backupWorkerMaxNoopVersionKey));
-				if (noopValue.present()) {
-					Version noopVersion = BinaryReader::fromStringRef<Version>(noopValue.get(), Unversioned());
-					if (poppedVersion > noopVersion) {
-						tr.set(backupWorkerMaxNoopVersionKey, BinaryWriter::toValue(poppedVersion, Unversioned()));
-					}
-				} else {
-					tr.set(backupWorkerMaxNoopVersionKey, BinaryWriter::toValue(poppedVersion, Unversioned()));
-				}
-
-				wait(tr.commit());
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> _noopPopper(BackupData* self) {
-		state Future<Void> onChange = self->popTrigger.onChange();
-
-		loop {
-			wait(onChange);
-			onChange = self->popTrigger.onChange();
-			if (!self->pulling) {
-				// Save the noop pop version, which sets min version for
-				// the next backup job. Note this version may change after the wait.
-				state Version popVersion = self->popTrigger.get();
-				ASSERT(self->popVersion <= popVersion);
-				wait(_saveNoopVersion(self, popVersion));
-				self->popVersion = popVersion;
-				TraceEvent("BackupWorkerNoopPop", self->myId)
-				    .detail("Tag", self->tag)
-				    .detail("SavedVersion", self->savedVersion)
-				    .detail("PopVersion", popVersion);
-				self->pop();
-			}
-		}
-	}
-
-	ACTOR static Future<Void> _waitAllInfoReady(BackupData* self) {
+	static Future<Void> _waitAllInfoReady(BackupData* self) {
 		std::vector<Future<Void>> all;
 		for (auto it = self->backups.begin(); it != self->backups.end();) {
 			if (it->second.stopped) {
@@ -493,8 +430,7 @@ struct BackupData {
 			all.push_back(it->second.waitReady());
 			it++;
 		}
-		wait(waitForAll(all));
-		return Void();
+		co_await waitForAll(all);
 	}
 
 	Future<Void> waitAllInfoReady() { return _waitAllInfoReady(this); }
@@ -506,56 +442,60 @@ struct BackupData {
 		}
 		return true;
 	}
+};
 
-	ACTOR static Future<Version> _getMinKnownCommittedVersion(BackupData* self) {
-		state Span span("BA:GetMinCommittedVersion"_loc);
-		loop {
+// If the worker is on an old epoch and all backups starts a version >= the endVersion
+// it will exit early.
+static Future<bool> shouldBackupWorkerExitEarly(BackupData* self) {
+	while (true) {
+		ReadYourWritesTransaction tr(self->cx);
+		while (true) {
+			Error err;
 			try {
-				GetReadVersionRequest request(span.context,
-				                              0,
-				                              TransactionPriority::DEFAULT,
-				                              invalidVersion,
-				                              GetReadVersionRequest::FLAG_USE_MIN_KNOWN_COMMITTED_VERSION);
-				choose {
-					when(wait(self->cx->onProxiesChanged())) {}
-					when(GetReadVersionReply reply =
-					         wait(basicLoadBalance(self->cx->getGrvProxies(UseProvisionalProxies::False),
-					                               &GrvProxyInterface::getConsistentReadVersion,
-					                               request,
-					                               self->cx->taskID))) {
-						self->cx->ssVersionVectorCache.applyDelta(reply.ssVersionVectorDelta);
-						return reply.version;
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				Optional<Value> value = co_await tr.get(backupStartedKey);
+				std::vector<std::pair<UID, Version>> uidVersions;
+				if (value.present()) {
+					bool shouldExit = self->endVersion.present();
+					uidVersions = decodeBackupStartedValue(value.get());
+					TraceEvent e("BackupWorkerGotStartKey", self->myId);
+					int i = 1;
+					for (auto [uid, version] : uidVersions) {
+						e.detail(format("BackupID%d", i), uid).detail(format("Version%d", i), version);
+						i++;
+						if (shouldExit && version < self->endVersion.get()) {
+							shouldExit = false;
+						}
 					}
+					self->onBackupChanges(uidVersions);
+					co_return shouldExit;
 				}
+
+				TraceEvent("BackupWorkerEmptyStartKey", self->myId);
+				Future<Void> watchFuture = tr.watch(backupStartedKey);
+				co_await tr.commit();
+				co_await watchFuture;
+				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_batch_transaction_throttled ||
-				    e.code() == error_code_grv_proxy_memory_limit_exceeded) {
-					// GRV Proxy returns an error
-					wait(delayJittered(CLIENT_KNOBS->GRV_ERROR_RETRY_DELAY));
-				} else {
-					throw;
-				}
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
+}
 
-	Future<Version> getMinKnownCommittedVersion() { return _getMinKnownCommittedVersion(this); }
-};
-
-// Monitors "backupStartedKey". If "present" is true, wait until the key is set;
-// otherwise, wait until the key is cleared. If "watch" is false, do not perform
-// the wait for key set/clear events. Returns if key present.
-ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool present, bool watch) {
-	loop {
-		state ReadYourWritesTransaction tr(self->cx);
-
-		loop {
+// Monitors "backupStartedKey".
+static Future<Void> monitorBackupStartedKeyChanges(BackupData* self) {
+	while (true) {
+		ReadYourWritesTransaction tr(self->cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> value = wait(tr.get(backupStartedKey));
+				Optional<Value> value = co_await tr.get(backupStartedKey);
 				std::vector<std::pair<UID, Version>> uidVersions;
-				bool shouldExit = self->endVersion.present();
 				if (value.present()) {
 					uidVersions = decodeBackupStartedValue(value.get());
 					TraceEvent e("BackupWorkerGotStartKey", self->myId);
@@ -563,59 +503,48 @@ ACTOR Future<bool> monitorBackupStartedKeyChanges(BackupData* self, bool present
 					for (auto [uid, version] : uidVersions) {
 						e.detail(format("BackupID%d", i), uid).detail(format("Version%d", i), version);
 						i++;
-						if (shouldExit && version < self->endVersion.get()) {
-							shouldExit = false;
-						}
-					}
-					self->exitEarly = shouldExit;
-					self->onBackupChanges(uidVersions);
-					if (present || !watch)
-						return true;
-				} else {
-					TraceEvent("BackupWorkerEmptyStartKey", self->myId).log();
-					self->onBackupChanges(uidVersions);
-
-					self->exitEarly = shouldExit;
-					if (!present || !watch) {
-						return false;
 					}
 				}
 
-				state Future<Void> watchFuture = tr.watch(backupStartedKey);
-				wait(tr.commit());
-				wait(watchFuture);
+				self->onBackupChanges(uidVersions);
+				Future<Void> watchFuture = tr.watch(backupStartedKey);
+				co_await tr.commit();
+				co_await watchFuture;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 }
 
 // Set "latestBackupWorkerSavedVersion" key for backups
-ACTOR Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedLogVersions) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedLogVersions) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-			state std::vector<Future<Optional<Version>>> prevVersions;
-			state std::vector<BackupConfig> versionConfigs;
-			state std::vector<Future<Optional<bool>>> allWorkersReady;
+			std::vector<Future<Optional<Version>>> prevVersions;
+			std::vector<BackupConfig> versionConfigs;
+			std::vector<Future<Optional<bool>>> allWorkersReady;
 			for (const auto& [uid, version] : savedLogVersions) {
 				versionConfigs.emplace_back(uid);
 				prevVersions.push_back(versionConfigs.back().latestBackupWorkerSavedVersion().get(tr));
 				allWorkersReady.push_back(versionConfigs.back().allWorkerStarted().get(tr));
 			}
 
-			wait(waitForAll(prevVersions) && waitForAll(allWorkersReady));
+			co_await (waitForAll(prevVersions) && waitForAll(allWorkersReady));
 
 			for (int i = 0; i < prevVersions.size(); i++) {
-				if (!allWorkersReady[i].get().present() || !allWorkersReady[i].get().get())
+				if (!allWorkersReady[i].get().present() || !allWorkersReady[i].get().get()) {
 					continue;
+				}
 
 				const Version current = savedLogVersions[versionConfigs[i].getUid()];
 				if (prevVersions[i].get().present()) {
@@ -634,11 +563,12 @@ ACTOR Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedL
 					versionConfigs[i].latestBackupWorkerSavedVersion().set(tr, current);
 				}
 			}
-			wait(tr->commit());
-			return Void();
+			co_await tr->commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
@@ -646,23 +576,23 @@ ACTOR Future<Void> setBackupKeys(BackupData* self, std::map<UID, Version> savedL
 // version key is set by one process, which is stored in each BackupConfig in
 // the system space. The client can know if a backup is restorable by checking
 // log saved version > snapshot version.
-ACTOR Future<Void> monitorBackupProgress(BackupData* self) {
-	state Future<Void> interval;
+Future<Void> monitorBackupProgress(BackupData* self) {
+	Future<Void> interval;
 
-	loop {
+	while (true) {
 		interval = delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0);
 		while (self->backups.empty() || !self->logSystem.get()) {
-			wait(self->changedTrigger.onTrigger() || self->logSystem.onChange());
+			co_await (self->changedTrigger.onTrigger() || self->logSystem.onChange());
 		}
 
 		// check all workers have started by checking their progress is larger
 		// than the backup's start version.
-		state Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
-		wait(getBackupProgress(self->cx, self->myId, progress, /*logging=*/false));
-		state std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
-		state std::map<UID, Version> savedLogVersions;
+		Reference<BackupProgress> progress(new BackupProgress(self->myId, {}));
+		co_await getBackupProgress(self->cx, self->myId, progress, SevDebug);
+		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
+		std::map<UID, Version> savedLogVersions;
 		if (tagVersions.size() != self->totalTags) {
-			wait(interval);
+			co_await interval;
 			continue;
 		}
 
@@ -680,15 +610,16 @@ ACTOR Future<Void> monitorBackupProgress(BackupData* self) {
 		}
 		Future<Void> setKeys = savedLogVersions.empty() ? Void() : setBackupKeys(self, savedLogVersions);
 
-		wait(interval && setKeys);
+		co_await (interval && setKeys);
 	}
 }
 
-ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
-	state Transaction tr(self->cx);
-	state Key key = backupProgressKeyFor(self->myId);
+Future<Void> saveProgress(BackupData* self, Version backupVersion) {
+	Transaction tr(self->cx);
+	Key key = backupProgressKeyFor(self->myId);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			// It's critical to save progress immediately so that after a master
 			// recovery, the new master can know the progress so far.
@@ -697,32 +628,33 @@ ACTOR Future<Void> saveProgress(BackupData* self, Version backupVersion) {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			// CHECK: Don't save progress if backup workers are disabled
-			Optional<Value> backupWorkerEnabled = wait(tr.get(backupWorkerEnabledKey));
+			Optional<Value> backupWorkerEnabled = co_await tr.get(backupWorkerEnabledKey);
 			if (!backupWorkerEnabled.present() || backupWorkerEnabled.get() == "0"_sr) {
 				TraceEvent("BackupWorkerProgressSkipped", self->myId).detail("Reason", "BackupWorkersDisabled");
-				return Void();
+				co_return;
 			}
 
 			WorkerBackupStatus status(self->backupEpoch, backupVersion, self->tag, self->totalTags);
 			tr.set(key, backupProgressValue(status));
 			tr.addReadConflictRange(singleKeyRange(key));
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
 // Write a mutation to a log file. Note the mutation can be different from
 // message.message for clear mutations.
-ACTOR Future<Void> addMutation(Reference<IBackupFile> logFile,
-                               VersionedMessage message,
-                               StringRef mutation,
-                               int64_t* blockEnd,
-                               int blockSize) {
+Future<Void> addMutation(Reference<IBackupFile> logFile,
+                         VersionedMessage message,
+                         StringRef mutation,
+                         int64_t* blockEnd,
+                         int blockSize) {
 	// format: version, subversion, messageSize, message
-	state int bytes = sizeof(Version) + sizeof(uint32_t) + sizeof(int) + mutation.size();
+	int bytes = sizeof(Version) + sizeof(uint32_t) + sizeof(int) + mutation.size();
 
 	// Convert to big Endianness for version.version, version.sub, and msgSize
 	// The decoder assumes 0xFF is the end, so little endian can easily be
@@ -730,34 +662,34 @@ ACTOR Future<Void> addMutation(Reference<IBackupFile> logFile,
 	// the first byte is not 0xFF (should always be 0x00).
 	BinaryWriter wr(Unversioned());
 	wr << bigEndian64(message.version.version) << bigEndian32(message.version.sub) << bigEndian32(mutation.size());
-	state Standalone<StringRef> header = wr.toValue();
+	Standalone<StringRef> header = wr.toValue();
 
 	// Start a new block if needed
 	if (logFile->size() + bytes > *blockEnd) {
 		// Write padding if needed
 		const int bytesLeft = *blockEnd - logFile->size();
 		if (bytesLeft > 0) {
-			state Value paddingFFs = fileBackup::makePadding(bytesLeft);
-			wait(logFile->append(paddingFFs.begin(), bytesLeft));
+			Value paddingFFs = fileBackup::makePadding(bytesLeft);
+			co_await logFile->append(paddingFFs.begin(), bytesLeft);
 		}
 
 		*blockEnd += blockSize;
 		// write block Header
-		wait(logFile->append((uint8_t*)&PARTITIONED_MLOG_VERSION, sizeof(PARTITIONED_MLOG_VERSION)));
+		co_await logFile->append((uint8_t*)&PARTITIONED_MLOG_VERSION, sizeof(PARTITIONED_MLOG_VERSION));
 	}
 
-	wait(logFile->append((void*)header.begin(), header.size()));
-	wait(logFile->append(mutation.begin(), mutation.size()));
-	return Void();
+	co_await logFile->append((void*)header.begin(), header.size());
+	co_await logFile->append(mutation.begin(), mutation.size());
 }
 
-ACTOR static Future<Void> updateLogBytesWritten(BackupData* self,
-                                                std::vector<UID> backupUids,
-                                                std::vector<Reference<IBackupFile>> logFiles) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+static Future<Void> updateLogBytesWritten(BackupData* self,
+                                          std::vector<UID> backupUids,
+                                          std::vector<Reference<IBackupFile>> logFiles) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
 
 	ASSERT(backupUids.size() == logFiles.size());
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -767,31 +699,32 @@ ACTOR static Future<Void> updateLogBytesWritten(BackupData* self,
 				BackupConfig config(backupUids[i]);
 				config.logBytesWritten().atomicOp(tr, logFiles[i]->size(), MutationRef::AddValue);
 			}
-			wait(tr->commit());
-			return Void();
+			co_await tr->commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
 // Saves messages in the range of [0, numMsg) to a file and then remove these
 // messages. The file content format is a sequence of (Version, sub#, msgSize, message).
 // Note only ready backups are saved.
-ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int numMsg) {
-	state int blockSize = SERVER_KNOBS->BACKUP_FILE_BLOCK_BYTES;
-	state std::vector<Future<Reference<IBackupFile>>> logFileFutures;
-	state std::vector<Reference<IBackupFile>> logFiles;
-	state std::vector<int64_t> blockEnds;
-	state std::vector<UID> activeUids; // active Backups' UIDs
-	state std::vector<Version> beginVersions; // logFiles' begin versions
-	state KeyRangeMap<std::set<int>> keyRangeMap; // range to index in logFileFutures, logFiles, & blockEnds
-	state std::vector<Standalone<StringRef>> mutations;
-	state int idx;
+Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int numMsg) {
+	int blockSize = SERVER_KNOBS->BACKUP_FILE_BLOCK_BYTES;
+	std::vector<Future<Reference<IBackupFile>>> logFileFutures;
+	std::vector<Reference<IBackupFile>> logFiles;
+	std::vector<int64_t> blockEnds;
+	std::vector<UID> activeUids; // active Backups' UIDs
+	std::vector<Version> beginVersions; // logFiles' begin versions
+	KeyRangeMap<std::set<int>> keyRangeMap; // range to index in logFileFutures, logFiles, & blockEnds
+	std::vector<Standalone<StringRef>> mutations;
+	int idx{ 0 };
 
 	// Make sure all backups are ready, otherwise mutations will be lost.
 	while (!self->isAllInfoReady()) {
-		wait(self->waitAllInfoReady());
+		co_await self->waitAllInfoReady();
 	}
 
 	for (auto it = self->backups.begin(); it != self->backups.end();) {
@@ -809,7 +742,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 				// True-up first mutation log's begin version
 				it->second.lastSavedVersion = self->messages[0].getVersion();
 			} else {
-				it->second.lastSavedVersion = std::max({ self->popVersion, self->savedVersion, self->startVersion });
+				it->second.lastSavedVersion = std::max({ self->savedVersion, self->startVersion });
 			}
 			TraceEvent("BackupWorkerTrueUp", self->myId).detail("LastSavedVersion", it->second.lastSavedVersion);
 		}
@@ -823,7 +756,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 	}
 
 	keyRangeMap.coalesce(allKeys);
-	wait(waitForAll(logFileFutures));
+	co_await waitForAll(logFileFutures);
 
 	std::transform(logFileFutures.begin(),
 	               logFileFutures.end(),
@@ -878,7 +811,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 				}
 			}
 		}
-		wait(waitForAll(adds));
+		co_await waitForAll(adds);
 		mutations.clear();
 	}
 
@@ -887,7 +820,7 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		return f->finish();
 	});
 
-	wait(waitForAll(finished));
+	co_await waitForAll(finished);
 
 	for (const auto& file : logFiles) {
 		TraceEvent("CloseMutationFile", self->myId)
@@ -899,19 +832,18 @@ ACTOR Future<Void> saveMutationsToFile(BackupData* self, Version popVersion, int
 		self->backups[uid].lastSavedVersion = popVersion + 1;
 	}
 
-	wait(updateLogBytesWritten(self, activeUids, logFiles));
-	return Void();
+	co_await updateLogBytesWritten(self, activeUids, logFiles);
 }
 
 // Uploads self->messages to cloud storage and updates savedVersion.
-ACTOR Future<Void> uploadData(BackupData* self) {
-	state Version popVersion = invalidVersion;
+Future<Void> uploadData(BackupData* self) {
+	Version popVersion = invalidVersion;
 
-	loop {
+	while (true) {
 		// Too large uploadDelay will delay popping tLog data for too long.
-		state Future<Void> uploadDelay = delay(SERVER_KNOBS->BACKUP_UPLOAD_DELAY);
+		Future<Void> uploadDelay = delay(SERVER_KNOBS->BACKUP_UPLOAD_DELAY);
 
-		state int numMsg = 0;
+		int numMsg = 0;
 		Version lastPopVersion = popVersion;
 		// index of last version's end position in self->messages
 		int lastVersionIndex = 0;
@@ -941,26 +873,20 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 			// queuing more, then we are stuck. This could suggest the lock capacity is too small.
 			ASSERT(numMsg > 0 || self->lock->waiters() == 0);
 		}
-		if (((numMsg > 0 || popVersion > lastPopVersion) && self->pulling) || self->pullFinished()) {
+		if ((numMsg > 0 || popVersion > lastPopVersion) || self->pullFinished()) {
 			TraceEvent("BackupWorkerSave", self->myId)
 			    .detail("Version", popVersion)
 			    .detail("LastPopVersion", lastPopVersion)
-			    .detail("Pulling", self->pulling)
 			    .detail("SavedVersion", self->savedVersion)
 			    .detail("NumMsg", numMsg)
 			    .detail("MsgQ", self->messages.size());
 			// save an empty file for old epochs so that log file versions are continuous
-			wait(saveMutationsToFile(self, popVersion, numMsg));
+			co_await saveMutationsToFile(self, popVersion, numMsg);
 			self->eraseMessages(numMsg);
 		}
 
-		// If transition into NOOP mode, should clear messages
-		if (!self->pulling && self->backupEpoch == self->recruitedEpoch) {
-			self->eraseMessages(self->messages.size());
-		}
-
-		if (popVersion > self->savedVersion && popVersion > self->popVersion) {
-			wait(saveProgress(self, popVersion));
+		if (popVersion > self->savedVersion) {
+			co_await saveProgress(self, popVersion);
 			TraceEvent("BackupWorkerSavedProgress", self->myId)
 			    .detail("Tag", self->tag.toString())
 			    .detail("Version", popVersion)
@@ -970,75 +896,48 @@ ACTOR Future<Void> uploadData(BackupData* self) {
 		}
 
 		if (self->allMessageSaved()) {
-			return Void();
+			co_return;
 		}
 
 		if (!self->pullFinished()) {
-			wait(uploadDelay || self->doneTrigger.onTrigger());
-		}
-	}
-}
-
-ACTOR static Future<Version> getNoopVersion(BackupData* self) {
-	state Transaction tr(self->cx);
-
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
-
-			Optional<Value> noopValue = wait(tr.get(backupWorkerMaxNoopVersionKey));
-			if (noopValue.present()) {
-				return BinaryReader::fromStringRef<Version>(noopValue.get(), Unversioned());
-			} else {
-				return invalidVersion;
-			}
-		} catch (Error& e) {
-			wait(tr.onError(e));
+			co_await (uploadDelay || self->doneTrigger.onTrigger());
 		}
 	}
 }
 
 // Pulls data from TLog servers using LogRouter tag.
-ACTOR Future<Void> pullAsyncData(BackupData* self) {
-	state Future<Void> logSystemChange = Void();
-	state Reference<ILogSystem::IPeekCursor> r;
-
-	// Going out of noop mode, the popVersion could be larger than
-	// savedVersion or ongoing pop version, i.e., popTrigger.get(),
-	// and we can't peek messages between savedVersion and popVersion.
-	state Version tagAt = std::max({ self->pulledVersion.get(),
-	                                 self->startVersion,
-	                                 self->savedVersion,
-	                                 self->popVersion,
-	                                 self->popTrigger.get() });
+Future<Void> pullAsyncData(BackupData* self) {
+	Future<Void> logSystemChange = Void();
+	Reference<IPeekCursor> r;
+
+	Version tagAt = std::max({ self->pulledVersion.get(), self->startVersion, self->savedVersion });
 
 	TraceEvent("BackupWorkerPull", self->myId)
 	    .detail("Tag", self->tag)
 	    .detail("Version", tagAt)
-	    .detail("PopVersion", self->popVersion)
-	    .detail("TriggerVersion", self->popTrigger.get())
 	    .detail("StartVersion", self->startVersion)
 	    .detail("SavedVersion", self->savedVersion);
-	loop {
+	while (true) {
 		while (self->paused.get()) {
-			wait(self->paused.onChange());
+			co_await self->paused.onChange();
 		}
 
-		loop choose {
-			when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
+		while (true) {
+			Future<Void> getMoreFuture = r ? r->getMore(TaskPriority::TLogCommit) : Never();
+			auto const res = co_await race(getMoreFuture, logSystemChange);
+			if (res.index() == 0) {
+				// getMoreFuture finished
 				DisabledTraceEvent("BackupWorkerGotMore", self->myId)
 				    .detail("Tag", self->tag)
 				    .detail("CursorVersion", r->version().version);
 				break;
-			}
-			when(wait(logSystemChange)) {
+			} else {
+				// logSystemChange detected
 				if (self->logSystem.get()) {
 					r = self->logSystem.get()->peekLogRouter(
 					    self->myId, tagAt, self->tag, SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED);
 				} else {
-					r = Reference<ILogSystem::IPeekCursor>();
+					r = Reference<IPeekCursor>();
 				}
 				logSystemChange = self->logSystem.onChange();
 			}
@@ -1046,32 +945,23 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		// When TLog sets popped version, it means mutations between popped() and tagAt are unavailable
 		// on the TLog. So, we should stop pulling data from the TLog.
 		if (r->popped() > 0) {
-			Version maxNoopVersion = wait(getNoopVersion(self));
-			Severity sev = maxNoopVersion != invalidVersion && maxNoopVersion < r->popped() ? SevError : SevWarnAlways;
-			TraceEvent(sev, "BackupWorkerPullMissingMutations", self->myId)
+			TraceEvent(SevError, "BackupWorkerPullMissingMutations", self->myId)
 			    .detail("Tag", self->tag)
 			    .detail("BackupEpoch", self->backupEpoch)
 			    .detail("Popped", r->popped())
-			    .detail("NoopPoppedVersion", maxNoopVersion)
 			    .detail("ExpectedPeekVersion", tagAt)
 			    .detail("RecruitedEpoch", self->recruitedEpoch);
-			ASSERT(self->backupEpoch < self->recruitedEpoch && maxNoopVersion >= r->popped());
-			// This can only happen when the backup was in NOOP mode in the previous epoch,
-			// where NOOP mode popped version is larger than the expected peek version.
-			// CC recruits this worker from epoch's begin version, which is lower than the
-			// noop popped version. So it's ok for this worker to continue from the popped
-			// version. Max noop popped version (maybe from a different tag) should be larger
-			// than the popped version.
+			ASSERT(true);
 		}
 		self->minKnownCommittedVersion = std::max(self->minKnownCommittedVersion, r->getMinKnownCommittedVersion());
 
 		// Note we aggressively peek (uncommitted) messages, but only committed
 		// messages/mutations will be flushed to disk/blob in uploadData().
-		state int64_t peekedBytes = 0;
+		int64_t peekedBytes = 0;
 		// Hold messages until we know how many we can take, self->messages always
 		// contains messages that we have reserved memory for. Therefore, lock->release()
 		// will always encounter message with reserved memory.
-		state std::vector<VersionedMessage> tmpMessages;
+		std::vector<VersionedMessage> tmpMessages;
 		while (r->hasMessage()) {
 			tmpMessages.emplace_back(r->version(), r->getMessage(), r->getTags(), r->arena());
 			peekedBytes += tmpMessages.back().getEstimatedSize();
@@ -1081,7 +971,7 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 			TraceEvent(SevDebugMemory, "BackupWorkerMemory", self->myId)
 			    .detail("Take", peekedBytes)
 			    .detail("Current", self->lock->activePermits());
-			wait(self->lock->take(TaskPriority::DefaultYield, peekedBytes));
+			co_await self->lock->take(TaskPriority::DefaultYield, peekedBytes);
 			self->messages.insert(self->messages.end(),
 			                      std::make_move_iterator(tmpMessages.begin()),
 			                      std::make_move_iterator(tmpMessages.end()));
@@ -1098,69 +988,14 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 			    .detail("VersionGot", tagAt)
 			    .detail("EndVersion", self->endVersion.get())
 			    .detail("MsgQ", self->messages.size());
-			return Void();
+			co_return;
 		}
-		wait(yield());
+		co_await yield();
 	}
 }
 
-ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self, bool keyPresent) {
-	state Future<Void> pullFinished = Void();
-
-	loop {
-		state Future<bool> present = monitorBackupStartedKeyChanges(self, !keyPresent, /*watch=*/true);
-		if (keyPresent) {
-			pullFinished = pullAsyncData(self);
-			self->pulling = true;
-			wait(success(present) || pullFinished);
-			if (pullFinished.isReady()) {
-				self->pulling = false;
-				return Void(); // backup is done for some old epoch.
-			}
-
-			// Even though the snapshot is done, mutation logs may not be written
-			// out yet. We need to make sure mutations up to this point is written.
-			Version currentVersion = wait(self->getMinKnownCommittedVersion());
-			wait(self->pulledVersion.whenAtLeast(currentVersion));
-			pullFinished = Future<Void>(); // cancels pullAsyncData()
-			self->pulling = false;
-			TraceEvent("BackupWorkerPaused", self->myId).detail("Reason", "NoBackup");
-		} else {
-			// Backup key is not present, enter this NOOP POP mode.
-			state Future<Version> committedVersion = self->getMinKnownCommittedVersion();
-
-			loop choose {
-				when(wait(success(present))) {
-					break;
-				}
-				when(wait(success(committedVersion) || delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
-					if (committedVersion.isReady()) {
-						Version newPopVersion =
-						    std::max({ self->popVersion, self->savedVersion, committedVersion.get() });
-						self->minKnownCommittedVersion =
-						    std::max(committedVersion.get(), self->minKnownCommittedVersion);
-						if (newPopVersion < self->popTrigger.get()) {
-							// this can happen if a different GRV proxy replies
-							DisabledTraceEvent("BackupWorkerSkipTrigger", self->myId)
-							    .detail("Version", newPopVersion)
-							    .detail("OldPop", self->popTrigger.get());
-						} else {
-							self->popTrigger.set(newPopVersion);
-						}
-						committedVersion = Never();
-					} else {
-						committedVersion = self->getMinKnownCommittedVersion();
-					}
-				}
-			}
-		}
-		ASSERT(!keyPresent == present.get());
-		keyPresent = !keyPresent;
-	}
-}
-
-ACTOR Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db, LogEpoch recoveryCount, BackupData* self) {
-	loop {
+Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db, LogEpoch recoveryCount, BackupData* self) {
+	while (true) {
 		bool isDisplaced =
 		    db->get().recoveryCount > recoveryCount && db->get().recoveryState != RecoveryState::UNINITIALIZED;
 		if (isDisplaced) {
@@ -1172,21 +1007,22 @@ ACTOR Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db, LogE
 			    .detail("RecoveryState", (int)db->get().recoveryState);
 			throw worker_removed();
 		}
-		wait(db->onChange());
+		co_await db->onChange();
 	}
 }
 
-ACTOR static Future<Void> monitorWorkerPause(BackupData* self) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
-	state Future<Void> watch;
+static Future<Void> monitorWorkerPause(BackupData* self) {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->cx));
+	Future<Void> watch;
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-			Optional<Value> value = wait(tr->get(backupPausedKey));
+			Optional<Value> value = co_await tr->get(backupPausedKey);
 			bool paused = value.present() && value.get() == "1"_sr;
 			if (self->paused.get() != paused) {
 				TraceEvent(paused ? "BackupWorkerPaused" : "BackupWorkerResumed", self->myId).log();
@@ -1194,24 +1030,27 @@ ACTOR static Future<Void> monitorWorkerPause(BackupData* self) {
 			}
 
 			watch = tr->watch(backupPausedKey);
-			wait(tr->commit());
-			wait(watch);
+			co_await tr->commit();
+			co_await watch;
 			tr->reset();
+			continue;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR Future<Void> backupWorker(BackupInterface interf,
-                                InitializeBackupRequest req,
-                                Reference<AsyncVar<ServerDBInfo> const> db) {
-	state BackupData self(interf.id(), db, req);
-	state PromiseStream<Future<Void>> addActor;
-	state Future<Void> error = actorCollection(addActor.getFuture());
-	state Future<Void> dbInfoChange = Void();
-	state Future<Void> pull;
-	state Future<Void> done;
+Future<Void> backupWorker(BackupInterface interf,
+                          InitializeBackupRequest req,
+                          Reference<AsyncVar<ServerDBInfo> const> db) {
+	BackupData self(interf.id(), db, req);
+	PromiseStream<Future<Void>> addActor;
+	Future<Void> error = actorCollection(addActor.getFuture());
+	Future<Void> dbInfoChange = Void();
+	Future<Void> pull;
+	Future<Void> done;
+	Error err;
 
 	TraceEvent("BackupWorkerStart", self.myId)
 	    .detail("Tag", req.routerTag.toString())
@@ -1228,20 +1067,22 @@ ACTOR Future<Void> backupWorker(BackupInterface interf,
 		}
 		addActor.send(monitorWorkerPause(&self));
 
-		// Check if backup key is present to avoid race between this check and
-		// noop pop as well as upload data: pop or skip upload before knowing
-		// there are backup keys. Set the "exitEarly" flag if needed.
-		bool present = wait(monitorBackupStartedKeyChanges(&self, true, false));
-		TraceEvent("BackupWorkerWaitKey", self.myId).detail("Present", present).detail("ExitEarly", self.exitEarly);
+		// If the worker is on an old epoch and all backups starts a version >= the endVersion
+		bool exitEarly = co_await shouldBackupWorkerExitEarly(&self);
+		TraceEvent("BackupWorkerExitEarly", self.myId).detail("ExitEarly", exitEarly);
+		if (!exitEarly) {
+			addActor.send(monitorBackupStartedKeyChanges(&self));
+		}
 
-		pull = self.exitEarly ? Void() : monitorBackupKeyOrPullData(&self, present);
+		pull = exitEarly ? Void() : pullAsyncData(&self);
 		addActor.send(pull);
-		done = self.exitEarly ? Void() : uploadData(&self);
+		done = exitEarly ? Void() : uploadData(&self);
 
-		loop choose {
-			when(wait(dbInfoChange)) {
+		while (true) {
+			auto res = co_await race(dbInfoChange, done, error);
+			if (res.index() == 0) {
 				dbInfoChange = db->onChange();
-				Reference<ILogSystem> ls = ILogSystem::fromServerDBInfo(self.myId, db->get(), true);
+				Reference<LogSystem> ls = makeLogSystemFromServerDBInfo(self.myId, db->get(), true);
 				bool hasPseudoLocality = ls.isValid() && ls->hasPseudoLocality(tagLocalityBackup);
 				if (hasPseudoLocality) {
 					self.logSystem.set(ls);
@@ -1251,32 +1092,31 @@ ACTOR Future<Void> backupWorker(BackupInterface interf,
 				    .detail("HasBackupLocality", hasPseudoLocality)
 				    .detail("OldestBackupEpoch", self.oldestBackupEpoch)
 				    .detail("Tag", self.tag.toString());
-			}
-			when(wait(done)) {
+			} else if (res.index() == 1) {
 				TraceEvent("BackupWorkerDone", self.myId).detail("BackupEpoch", self.backupEpoch);
 				// Notify master so that this worker can be removed from log system, then this
 				// worker (for an old epoch's unfinished work) can safely exit.
-				wait(brokenPromiseToNever(db->get().clusterInterface.notifyBackupWorkerDone.getReply(
-				    BackupWorkerDoneRequest(self.myId, self.backupEpoch))));
+				co_await brokenPromiseToNever(db->get().clusterInterface.notifyBackupWorkerDone.getReply(
+				    BackupWorkerDoneRequest(self.myId, self.backupEpoch)));
 				break;
 			}
-			when(wait(error)) {}
 		}
+		co_return;
 	} catch (Error& e) {
-		state Error err = e;
-		if (e.code() == error_code_worker_removed) {
-			pull = Void(); // cancels pulling
-			self.stop();
-			try {
-				wait(done);
-			} catch (Error& e) {
-				TraceEvent("BackupWorkerShutdownError", self.myId).errorUnsuppressed(e);
-			}
-		}
-		TraceEvent("BackupWorkerTerminated", self.myId).errorUnsuppressed(err);
-		if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
-			throw err;
+		err = e;
+	}
+
+	if (err.code() == error_code_worker_removed) {
+		pull = Void(); // cancels pulling
+		self.stop();
+		try {
+			co_await done;
+		} catch (Error& shutdownErr) {
+			TraceEvent("BackupWorkerShutdownError", self.myId).errorUnsuppressed(shutdownErr);
 		}
 	}
-	return Void();
+	TraceEvent("BackupWorkerTerminated", self.myId).errorUnsuppressed(err);
+	if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
+		throw err;
+	}
 }
diff --git a/fdbserver/backupworker/BackupWorkerRangePartitioned.cpp b/fdbserver/backupworker/BackupWorkerRangePartitioned.cpp
new file mode 100644
index 00000000000..c80bf696f37
--- /dev/null
+++ b/fdbserver/backupworker/BackupWorkerRangePartitioned.cpp
@@ -0,0 +1,1024 @@
+/*
+ * BackupWorkerRangePartitioned.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/BackupAgent.h"
+#include "fdbclient/BackupContainer.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/JsonBuilder.h"
+#include "fdbclient/SystemData.h"
+#include "fdbclient/Tracing.h"
+#include "BackupPartitionMap.h"
+#include "BackupRangePartitionedProgress.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "flow/CoroUtils.h"
+
+#define SevDebugMemory SevVerbose
+
+struct RangePartitionedVersionedMessage {
+	LogMessageVersion version;
+	StringRef message;
+	VectorRef<Tag> tags;
+	Arena arena;
+
+	RangePartitionedVersionedMessage(LogMessageVersion v, StringRef m, const VectorRef<Tag>& t, const Arena& a)
+	  : version(v), message(m), tags(t), arena(a) {}
+
+	Version getVersion() const { return version.version; }
+	size_t getEstimatedSize() const { return message.size() + TagsAndMessage::getHeaderSize(6); }
+
+	// Returns true if the message is a mutation that could be backed up (normal keys, system key backup ranges, or the
+	// metadata version key).
+	bool isCandidateBackupMessage(MutationRef* m) {
+		// TODO akanksha: Implement this function to filter out messages that are not mutations or not relevant to
+		// backup. Need to figure out the what those message can be.
+		return true;
+	}
+};
+
+struct RangePartitionedLogFileInfo {
+	UID backupUid;
+	int32_t partitionId;
+	KeyRange fileKeyRange;
+	Version beginVersion;
+	Reference<IBackupFile> file;
+	int64_t blockEnd = 0;
+};
+
+struct BackupRangePartitionedData {
+	const UID myId;
+	const Tag tag; // tag for this backup worker
+	const int totalTags; // Total backup worker tags
+	const Version startVersion; // This worker's start version
+	const Optional<Version> endVersion; // old epoch's end version (inclusive), or empty for current epoch
+	const LogEpoch recruitedEpoch; // current epoch whose tLogs are receiving mutations
+	const LogEpoch backupEpoch; // the epoch workers should pull mutations
+	// TODO akanksha: Update oldestBackupEpoch wherever needed.
+	LogEpoch oldestBackupEpoch = 0; // oldest epoch that still has data on tLogs for backup to pull
+	// Minimumum known committed version in StorageServers.
+	Version minKnownCommittedVersion;
+	Version savedVersion; // Largest version saved to blob storage
+	NotifiedVersion pulledVersion;
+	Version logFolderBaseVersion;
+	AsyncVar<Reference<LogSystem>> logSystem;
+	AsyncVar<bool> paused; // Track if "backupPausedKey" is set.
+	Reference<FlowLock> lock;
+	AsyncTrigger doneTrigger;
+	AsyncTrigger changedTrigger;
+	Database cx;
+	std::vector<RangePartitionedVersionedMessage> messages;
+	// Key range to partition ID map, used to determine which partition a mutation belongs to based on its key.
+	KeyRangeMap<int> keyRangeToPartitionId;
+	// Partition ID to key range map for easy lookup of partition's key range.
+	std::unordered_map<int, KeyRange> partitionToKeyRange;
+	// KeyRange to backup UID and partition id map needed to create log files for the right backup and partition.
+	KeyRangeMap<std::vector<std::pair<UID, int32_t>>> keyRangeToBackupAssignment;
+	// TODO akanksha: Once end to end implementation complete, check if stopped need to be added or not.
+
+	struct PerBackupInfo {
+		PerBackupInfo() = default;
+		PerBackupInfo(BackupRangePartitionedData* data, UID uid, Version v) : self(data), startVersion(v) {
+			// Open the container and get the key ranges.
+			BackupConfig config(uid);
+			container = config.backupContainer().get(data->cx.getReference());
+			ranges = config.backupRanges().get(data->cx.getReference());
+			TraceEvent("BWRangePartitionedAddBackup", data->myId).detail("BackupID", uid).detail("Version", v);
+		}
+
+		BackupRangePartitionedData* self = nullptr;
+		Future<Optional<std::vector<KeyRange>>> ranges; // Key ranges of this backup
+		Future<Optional<Reference<IBackupContainer>>> container;
+		// Backup request's commit version. Mutations are logged at some version after this.
+		Version startVersion = invalidVersion;
+		// The next log's begin version.
+		Version nextFileBeginVersion = invalidVersion;
+		bool stopped = false;
+
+		bool isReady() const { return stopped || (container.isReady() && ranges.isReady()); }
+
+		Future<Void> waitReady() {
+			if (stopped)
+				return Void();
+			return _waitReady(this);
+		}
+
+		static Future<Void> _waitReady(PerBackupInfo* info) {
+			co_await (success(info->container) && success(info->ranges));
+		}
+	};
+
+	// TODO akanksha: Add backups in this map when backup worker receives backup request.
+	std::unordered_map<UID, PerBackupInfo> backups; // Backup UID to infos
+
+	explicit BackupRangePartitionedData(UID id,
+	                                    Reference<AsyncVar<ServerDBInfo> const> db,
+	                                    const InitializeBackupRequest& req)
+	  : myId(id), tag(req.routerTag), totalTags(req.totalTags), startVersion(req.startVersion),
+	    endVersion(req.endVersion), recruitedEpoch(req.recruitedEpoch), backupEpoch(req.backupEpoch),
+	    minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), pulledVersion(0),
+	    logFolderBaseVersion(invalidVersion), paused(false),
+	    lock(new FlowLock(SERVER_KNOBS->BACKUP_WORKER_LOCK_BYTES)) {
+		cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True);
+	}
+
+	bool pullFinished() const { return endVersion.present() && pulledVersion.get() > endVersion.get(); }
+
+	Version maxPopVersion() const { return endVersion.present() ? endVersion.get() : minKnownCommittedVersion; }
+
+	bool allMessageSaved() const { return (endVersion.present() && savedVersion >= endVersion.get()); }
+
+	// Erases messages and updates lock with memory released.
+	void eraseMessages(int num) {
+		ASSERT(num <= messages.size());
+		if (num == 0)
+			return;
+
+		// Accumulate erased message sizes
+		int64_t bytes = 0;
+		for (int i = 0; i < num; i++) {
+			bytes += messages[i].getEstimatedSize();
+		}
+		TraceEvent(SevDebugMemory, "BWRangePartitionedMemory", myId)
+		    .detail("Release", bytes)
+		    .detail("Total", lock->activePermits());
+		lock->release(bytes);
+		messages.erase(messages.begin(), messages.begin() + num);
+	}
+
+	void eraseMessagesAfterEndVersion() {
+		ASSERT(endVersion.present());
+		const Version ver = endVersion.get();
+		while (!messages.empty()) {
+			if (messages.back().getVersion() > ver) {
+				size_t bytes = messages.back().getEstimatedSize();
+				TraceEvent(SevDebugMemory, "BWRangePartitionedMemory", myId).detail("Release", bytes);
+				lock->release(bytes);
+				messages.pop_back();
+			} else {
+				break;
+			}
+		}
+	}
+
+	// Inserts a backup's single range into rangeMap.
+	template <class T>
+	void insertRange(KeyRangeMap<std::set<T>>& keyRangeMap, KeyRangeRef range, T value) {
+		for (auto& logRange : keyRangeMap.modify(range)) {
+			logRange->value().insert(value);
+		}
+		for (auto& logRange : keyRangeMap.modify(singleKeyRange(metadataVersionKey))) {
+			logRange->value().insert(value);
+		}
+		TraceEvent("BackupWorkerInsertRange", myId)
+		    .detail("Value", value)
+		    .detail("Begin", range.begin)
+		    .detail("End", range.end);
+	}
+
+	// Finds the intersection between a vector of ranges and a target range.
+	// Returns the union of all intersecting portions as a single range.
+	// Returns empty Optional if there are no intersections.
+	Optional<KeyRange> getKeyRangeIntersection(const std::vector<KeyRange>& ranges, const KeyRange& target) {
+		Optional<KeyRange> result;
+
+		for (const auto& range : ranges) {
+			KeyRange intersection = range & target;
+			if (intersection.empty()) {
+				continue;
+			}
+
+			if (!result.present()) {
+				result = intersection;
+			} else {
+				KeyRef newBegin = std::min(result.get().begin, intersection.begin);
+				KeyRef newEnd = std::max(result.get().end, intersection.end);
+				result = KeyRange(KeyRangeRef(newBegin, newEnd));
+			}
+		}
+		return result;
+	}
+
+	void pop() {
+		if (!logSystem.get()) {
+			return;
+		}
+		logSystem.get()->pop(savedVersion, tag);
+	}
+
+	static Future<Void> _waitAllInfoReady(BackupRangePartitionedData* self) {
+		std::vector<Future<Void>> all;
+		for (auto it = self->backups.begin(); it != self->backups.end();) {
+			all.push_back(it->second.waitReady());
+			it++;
+		}
+		co_await waitForAll(all);
+	}
+
+	Future<Void> waitAllInfoReady() { return _waitAllInfoReady(this); }
+
+	bool isAllInfoReady() const {
+		for (const auto& [uid, info] : backups) {
+			if (!info.isReady())
+				return false;
+		}
+		return true;
+	}
+};
+
+static Future<Void> computeKeyRangeToBackupAssignment(BackupRangePartitionedData* self) {
+	self->keyRangeToBackupAssignment = KeyRangeMap<std::vector<std::pair<UID, int32_t>>>();
+
+	while (!self->isAllInfoReady()) {
+		co_await self->waitAllInfoReady();
+	}
+
+	for (auto& [uid, info] : self->backups) {
+		const auto& backupRanges = info.ranges.get().get();
+
+		for (auto iter : self->keyRangeToPartitionId.ranges()) {
+			int32_t partitionId = iter.value();
+			KeyRange partitionRange = iter.range();
+
+			Optional<KeyRange> intersection = self->getKeyRangeIntersection(backupRanges, partitionRange);
+			if (!intersection.present())
+				continue;
+
+			std::pair<UID, int32_t> bk{ uid, partitionId };
+			for (auto& range : self->keyRangeToBackupAssignment.modify(intersection.get())) {
+				range->value().push_back(bk);
+			}
+		}
+	}
+	self->keyRangeToBackupAssignment.coalesce(allKeys);
+}
+
+static Future<Void> onBackupChanges(BackupRangePartitionedData* self,
+                                    std::vector<std::pair<UID, Version>> uidVersions) {
+	std::unordered_set<UID> activeUids;
+	for (const auto& [uid, version] : uidVersions) {
+		activeUids.insert(uid);
+	}
+
+	bool modified = false;
+	bool hasNewBackup = false;
+	Version newBackupsMinVersion = std::numeric_limits<Version>::max();
+
+	// Add any new backups
+	for (const auto& [uid, version] : uidVersions) {
+		if (self->backups.find(uid) == self->backups.end()) {
+			self->backups.emplace(uid, BackupRangePartitionedData::PerBackupInfo(self, uid, version));
+			modified = true;
+			newBackupsMinVersion = std::min(newBackupsMinVersion, version);
+			hasNewBackup = true;
+		}
+	}
+
+	// Remove backups that are no longer active.
+	for (auto it = self->backups.begin(); it != self->backups.end(); it++) {
+		if (activeUids.find(it->first) == activeUids.end()) {
+			it->second.stopped = true;
+			it = self->backups.erase(it);
+			modified = true;
+		}
+	}
+
+	if (hasNewBackup && self->backupEpoch < self->recruitedEpoch && self->savedVersion + 1 == self->startVersion) {
+		// Advance savedVersion to minimize version ranges in case backupEpoch's progress is not saved. Master may set a
+		// very low startVersion that is already popped. Advance the version is safe because these versions are not
+		// popped -- if they are popped, their progress should be already recorded and Master would use a higher version
+		// than minVersion.
+		self->savedVersion = std::max(newBackupsMinVersion, self->savedVersion);
+	}
+	if (modified) {
+		self->changedTrigger.trigger();
+		co_await computeKeyRangeToBackupAssignment(self);
+	}
+}
+
+Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db,
+                          LogEpoch recoveryCount,
+                          BackupRangePartitionedData* self) {
+	while (true) {
+		bool isDisplaced =
+		    db->get().recoveryCount > recoveryCount && db->get().recoveryState != RecoveryState::UNINITIALIZED;
+		if (isDisplaced) {
+			TraceEvent("BWRangePartitionedDisplaced", self->myId)
+			    .detail("RecoveryCount", recoveryCount)
+			    .detail("RecoveryState", (int)db->get().recoveryState);
+			throw worker_removed();
+		}
+		co_await db->onChange();
+	}
+}
+
+Future<Version> pullPartitionMapFromTLog(BackupRangePartitionedData* self, PartitionMap* outPartitionMap) {
+	Reference<IPeekCursor> cursor;
+	Version partitionMapVersion = invalidVersion;
+	Future<Void> logSystemChange = Void();
+
+	while (true) {
+		while (true) {
+			auto res = co_await race(cursor ? cursor->getMore() : Never(), logSystemChange);
+			if (res.index() == 0) {
+				break;
+			} else if (res.index() == 1) {
+				if (self->logSystem.get()) {
+					cursor = self->logSystem.get()->peekSingle(self->myId, self->startVersion, self->tag);
+				} else {
+					cursor = Reference<IPeekCursor>();
+				}
+				logSystemChange = self->logSystem.onChange();
+			} else {
+				UNREACHABLE();
+			}
+		}
+		co_await cursor->getMore();
+		if (!cursor->hasMessage()) {
+			continue;
+		}
+		for (; cursor->hasMessage(); cursor->nextMessage()) {
+			Version msgVersion = cursor->version().version;
+			StringRef message = cursor->getMessage();
+			Arena arena = cursor->arena();
+			ArenaReader reader(arena, message, AssumeVersion(g_network->protocolVersion()));
+			if (reader.protocolVersion().hasSpanContext() && SpanContextMessage::isNextIn(reader)) {
+				cursor->nextMessage();
+				continue;
+			}
+			if (reader.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(reader)) {
+				cursor->nextMessage();
+				continue;
+			}
+			// TODO akanksha: Uncomment once PartitionMapMessage is implemented.
+			// bool isPartitionMap = PartitionMapMessage::isNextIn(reader);
+			bool isPartitionMap = true;
+			if (!isPartitionMap) {
+				TraceEvent(SevError, "BWRangeParitionedPartitionMapNotReceived", self->myId)
+				    .detail("Version", msgVersion)
+				    .detail("Tag", self->tag.toString())
+				    .detail("MessageSize", message.size());
+				throw worker_removed();
+			}
+
+			// TODO akanksha: 1. std::unordered_map is not supported by ArenaReader right now, need to implement custom
+			// deserialization logic for it. Or we can switch to std::map which is supported by ArenaReader.
+			// 2. Uncomment and update the code once the deserialization logic is implemented.
+			/*
+			PartitionMapMessage pmMsg;
+			reader >> pmMsg;
+			*outPartitionMap  = pmMsg.partitionMap;
+			*/
+			partitionMapVersion = msgVersion;
+			co_return partitionMapVersion;
+		}
+	}
+}
+
+Future<Void> uploadPartitionList(BackupRangePartitionedData* self, PartitionMap partitionMap) {
+	std::vector<Future<Void>> fileFutures;
+	auto it = self->backups.begin();
+
+	std::string jsonContent = serializePartitionListJSON(partitionMap);
+
+	for (; it != self->backups.end();) {
+		if (!it->second.container.get().present()) {
+			TraceEvent("BWRangePartitionedRemoveContainer", self->myId).detail("BackupId", it->first);
+			it = self->backups.erase(it);
+			continue;
+		}
+		Reference<IBackupContainer> container = it->second.container.get().get();
+		fileFutures.push_back(container->writePartitionListFile(self->logFolderBaseVersion, jsonContent));
+		it++;
+	}
+	if (fileFutures.empty()) {
+		TraceEvent("BWRangePartitionedNoContainers", self->myId);
+		co_return;
+	}
+
+	co_await waitForAll(fileFutures);
+}
+
+// TODO akanksha -> Need to figure out if
+// 1. For new requests -> PartitionMap in TLOG will be same for all containers
+// 2. For older epochs with different containers is PartitionMap specific to container or same for all.
+// Right now assumption is that PartitionMap will be passed by TLOG with the first message after start version for both
+// older epochs and newer epochs.
+Future<Void> waitAndProcessPartitionMap(BackupRangePartitionedData* self) {
+	TraceEvent("BWRangeParitionedWaitingForPartitionMap", self->myId)
+	    .detail("Tag", self->tag.toString())
+	    .detail("StartVersion", self->startVersion);
+	PartitionMap partitionMap;
+
+	Version partitionMapVersion = co_await pullPartitionMapFromTLog(self, &partitionMap);
+	self->logFolderBaseVersion = partitionMapVersion + 1;
+
+	ASSERT(partitionMap.find(self->tag) != partitionMap.end());
+	TraceEvent("BWRangeParitionedPulledPartitionMap", self->myId)
+	    .detail("Version", partitionMapVersion)
+	    .detail("NumTags", partitionMap.size())
+	    .detail("Tag", self->tag.toString())
+	    .detail("NumPartitions", partitionMap[self->tag].size());
+
+	self->keyRangeToPartitionId.clear();
+	ASSERT_GT(partitionMap[self->tag].size(), 0);
+	for (auto& partition : partitionMap[self->tag]) {
+		self->keyRangeToPartitionId.insert(partition.ranges, partition.partitionId);
+	}
+
+	Key doneKey = backupRangePartitionedMapUploadedKeyFor(partitionMapVersion);
+	ReadYourWritesTransaction tr(self->cx);
+
+	// TODO akanksha: Check what will be the tag id once tags are implemented for backup workers and update the
+	// condition accordingly.
+	// Also add background actor to clean up the done key at regular
+	if (self->tag.id == 0) {
+		co_await uploadPartitionList(self, partitionMap);
+		TraceEvent("BWRangePartitionedPartitionMapUploaded", self->myId)
+		    .detail("Version", partitionMapVersion)
+		    .detail("NumBackups", self->backups.size());
+		while (true) {
+			Error err;
+			try {
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr.set(doneKey, "1"_sr);
+				co_await tr.commit();
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	} else {
+		// All other backup workers waits for done key to be set by the worker with tag id 0, then start pulling
+		// mutations. This is to make sure partition map is uploaded before any worker starts pulling mutations.
+		while (true) {
+			Error err;
+			try {
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE); // More efficient for reads
+				Optional<Value> v = co_await tr.get(doneKey);
+				if (v.present()) {
+					break;
+				}
+				Future<Void> watchFuture = tr.watch(doneKey);
+				co_await tr.commit();
+				co_await watchFuture;
+				tr.reset();
+				continue;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+	self->pulledVersion.set(partitionMapVersion);
+	self->savedVersion = partitionMapVersion;
+	self->pop();
+	co_await computeKeyRangeToBackupAssignment(self);
+}
+
+// Pulls mutations from TLog servers.
+Future<Void> pullAsyncData(BackupRangePartitionedData* self) {
+	Future<Void> logSystemChange = Void();
+	Reference<IPeekCursor> cursor;
+
+	Version tagAt = std::max({ self->pulledVersion.get(), self->startVersion, self->savedVersion });
+
+	TraceEvent("BWRangePartitionedPull", self->myId)
+	    .detail("Tag", self->tag)
+	    .detail("Version", tagAt)
+	    .detail("StartVersion", self->startVersion)
+	    .detail("SavedVersion", self->savedVersion);
+
+	while (true) {
+		while (self->paused.get()) {
+			co_await self->paused.onChange();
+		}
+
+		while (true) {
+			auto res = co_await race(cursor ? cursor->getMore(TaskPriority::TLogCommit) : Never(), logSystemChange);
+			if (res.index() == 0) {
+				DisabledTraceEvent("BWRangePartitionedGotMore", self->myId)
+				    .detail("Tag", self->tag)
+				    .detail("CursorVersion", cursor->version().version);
+				break;
+			} else if (res.index() == 1) {
+				if (self->logSystem.get()) {
+					// TODO akanksha: Use peekSingle as of now instead of peekLogRouter and later confirm if it works as
+					// expected.
+					cursor = self->logSystem.get()->peekSingle(self->myId, tagAt, self->tag);
+				} else {
+					cursor = Reference<IPeekCursor>();
+				}
+				logSystemChange = self->logSystem.onChange();
+			} else {
+				UNREACHABLE();
+			}
+		}
+
+		if (cursor->popped() > 0) {
+			TraceEvent(SevError, "BWRangePartitionedDataPopped", self->myId)
+			    .detail("Popped", cursor->popped())
+			    .detail("Expected", tagAt);
+			throw worker_removed();
+		}
+
+		self->minKnownCommittedVersion =
+		    std::max(self->minKnownCommittedVersion, cursor->getMinKnownCommittedVersion());
+		int64_t peekedBytes = 0;
+
+		// Hold messages until we know how many we can take, self->messages always
+		// contains messages that we have reserved memory for. Therefore, lock->release()
+		// will always encounter message with reserved memory.
+		std::vector<RangePartitionedVersionedMessage> tmpMessages;
+
+		// Messages may be prefetched in peek here, but uncommitted messages should not be uploaded in uploadData().
+		while (cursor->hasMessage()) {
+			auto msg = RangePartitionedVersionedMessage(
+			    cursor->version(), cursor->getMessage(), cursor->getTags(), cursor->arena());
+			tmpMessages.emplace_back(std::move(msg));
+			peekedBytes += tmpMessages.back().getEstimatedSize();
+			cursor->nextMessage();
+		}
+
+		if (peekedBytes > 0) {
+			TraceEvent(SevDebugMemory, "BWRangePartitionedMemory", self->myId)
+			    .detail("Take", peekedBytes)
+			    .detail("Current", self->lock->activePermits());
+			co_await self->lock->take(TaskPriority::DefaultYield, peekedBytes);
+			self->messages.insert(self->messages.end(),
+			                      std::make_move_iterator(tmpMessages.begin()),
+			                      std::make_move_iterator(tmpMessages.end()));
+		}
+
+		tagAt = cursor->version().version;
+		self->pulledVersion.set(tagAt);
+		TraceEvent("BWRangePartitionedGot", self->myId).suppressFor(1.0).detail("LatestPulledVersion", tagAt);
+
+		// For older epochs, we may have an end version to stop at.
+		if (self->pullFinished()) {
+			self->eraseMessagesAfterEndVersion();
+			self->doneTrigger.trigger();
+			TraceEvent("BWRangePartitionedFinishPull", self->myId)
+			    .detail("Tag", self->tag.toString())
+			    .detail("VersionGot", tagAt)
+			    .detail("EndVersion", self->endVersion.get())
+			    .detail("LogEpoch", self->recruitedEpoch)
+			    .detail("BackupEpoch", self->backupEpoch);
+			co_return;
+		}
+		co_await yield();
+	}
+}
+
+Future<Void> writeFileHeader(Reference<IBackupFile> logFile, int32_t partitionId, KeyRange range) {
+	co_await logFile->append((uint8_t*)&RANGE_PARTITIONED_MLOG_VERSION, sizeof(RANGE_PARTITIONED_MLOG_VERSION));
+
+	BinaryWriter wr(Unversioned());
+	wr << partitionId << range.begin << range.end;
+	Standalone<StringRef> header = wr.toValue();
+	co_await logFile->append(header.begin(), header.size());
+}
+
+Future<Void> addMutation(Reference<IBackupFile> logFile,
+                         RangePartitionedVersionedMessage message,
+                         StringRef mutation,
+                         int64_t* blockEnd,
+                         int blockSize) {
+	// Format: version, subversion, messageSize, message
+	int bytes = sizeof(Version) + sizeof(uint32_t) + sizeof(int) + mutation.size();
+
+	// Convert to big Endianness for version.version, version.sub, and msgSize
+	// The decoder assumes 0xFF is the end, so little endian can easily be
+	// mistaken as the end. In contrast, big endian for version almost guarantee
+	// the first byte is not 0xFF (should always be 0x00).
+	BinaryWriter wr(Unversioned());
+	wr << bigEndian64(message.version.version) << bigEndian32(message.version.sub) << bigEndian32(mutation.size());
+	Standalone<StringRef> mutationHeader = wr.toValue();
+
+	// Start a new block if needed
+	if (logFile->size() + bytes > *blockEnd) {
+		const int bytesLeft = *blockEnd - logFile->size();
+		if (bytesLeft > 0) {
+			Value paddingFFs = fileBackup::makePadding(bytesLeft);
+			co_await logFile->append(paddingFFs.begin(), bytesLeft);
+		}
+
+		*blockEnd += blockSize;
+		// Block header.
+		co_await logFile->append((uint8_t*)&RANGE_PARTITIONED_MLOG_VERSION, sizeof(RANGE_PARTITIONED_MLOG_VERSION));
+	}
+	co_await logFile->append((void*)mutationHeader.begin(), mutationHeader.size());
+	co_await logFile->append(mutation.begin(), mutation.size());
+}
+
+Future<Void> saveMutationsToFile(BackupRangePartitionedData* self, Version lastVersionInFile, int numMsg) {
+	// Make sure all backups are ready, otherwise mutations will be lost.
+	while (!self->isAllInfoReady()) {
+		co_await self->waitAllInfoReady();
+	}
+
+	std::vector<RangePartitionedLogFileInfo> activeFiles;
+	// Map of (backupUid, partitionId) -> index into activeFiles.
+	std::map<std::pair<UID, int32_t>, int> fileIndexByBackupPartition;
+	int blockSize = SERVER_KNOBS->BACKUP_FILE_BLOCK_BYTES;
+	std::vector<Future<Reference<IBackupFile>>> fileFutures;
+
+	for (auto entry = self->keyRangeToBackupAssignment.ranges().begin();
+	     entry != self->keyRangeToBackupAssignment.ranges().end();
+	     ++entry) {
+		for (const auto& [backupUid, partitionId] : entry->value()) {
+
+			auto it = self->backups.find(backupUid);
+			if (it == self->backups.end() || !it->second.container.get().present()) {
+				TraceEvent("BWRangePartitionedRemoveContainerInFileCreation", self->myId).detail("BackupId", backupUid);
+				continue;
+			}
+
+			std::pair<UID, int32_t> bpKey(backupUid, partitionId);
+			if (fileIndexByBackupPartition.contains(bpKey)) {
+				continue;
+			}
+
+			Version fileEndVersion = lastVersionInFile + 1;
+			if (it->second.nextFileBeginVersion == invalidVersion) {
+				it->second.nextFileBeginVersion = self->savedVersion + 1;
+			}
+			Version beginVersion = it->second.nextFileBeginVersion;
+
+			RangePartitionedLogFileInfo lf;
+			lf.backupUid = it->first;
+			lf.partitionId = partitionId;
+			lf.fileKeyRange = entry->range();
+			lf.beginVersion = beginVersion;
+			lf.blockEnd = 0;
+
+			activeFiles.push_back(lf);
+			fileIndexByBackupPartition[bpKey] = activeFiles.size() - 1;
+			fileFutures.push_back(it->second.container.get().get()->writeRangePartitionedLogFile(
+			    beginVersion, fileEndVersion, self->logFolderBaseVersion, partitionId, blockSize));
+		}
+	}
+
+	if (fileFutures.empty()) {
+		co_return;
+	}
+	co_await waitForAll(fileFutures);
+
+	std::vector<Future<Void>> headerWrites;
+	int i;
+	for (i = 0; i < activeFiles.size(); i++) {
+		activeFiles[i].file = fileFutures[i].get();
+		headerWrites.push_back(
+		    writeFileHeader(activeFiles[i].file, activeFiles[i].partitionId, activeFiles[i].fileKeyRange));
+	}
+	co_await waitForAll(headerWrites);
+
+	if (activeFiles.empty()) {
+		co_return;
+	}
+
+	// Process mutations
+	int idx;
+	for (idx = 0; idx < numMsg; idx++) {
+		auto& message = self->messages[idx];
+		MutationRef m;
+		if (!message.isCandidateBackupMessage(&m)) {
+			continue;
+		}
+
+		DEBUG_MUTATION("BWRangeParitionedAddMutation", message.version.version, m, self->myId)
+		    .detail("KCV", self->minKnownCommittedVersion)
+		    .detail("SavedVersion", self->savedVersion);
+
+		std::vector<Future<Void>> adds;
+		if (m.type != MutationRef::Type::ClearRange) {
+			for (const auto& entry : self->keyRangeToBackupAssignment[m.param1]) {
+				auto it = fileIndexByBackupPartition.find(entry);
+				ASSERT(it != fileIndexByBackupPartition.end());
+
+				int fileIdx = it->second;
+				auto& lf = activeFiles[fileIdx];
+				// Different backups may have different start version so need this check before writing.
+				if (message.getVersion() >= lf.beginVersion) {
+					adds.push_back(addMutation(lf.file, message, message.message, &lf.blockEnd, blockSize));
+				}
+			}
+		} else {
+			KeyRangeRef mutationRange(m.param1, m.param2);
+			std::unordered_set<int> writtenFiles;
+			for (auto range : self->keyRangeToBackupAssignment.intersectingRanges(mutationRange)) {
+				for (const auto& entry : range.value()) {
+					auto it = fileIndexByBackupPartition.find(entry);
+					ASSERT(it != fileIndexByBackupPartition.end());
+
+					int fileIdx = it->second;
+					// For ClearRange, we only need to write the full mutation once for each file.
+					if (writtenFiles.find(fileIdx) != writtenFiles.end()) {
+						continue;
+					}
+					auto& lf = activeFiles[fileIdx];
+					if (message.getVersion() >= lf.beginVersion) {
+						adds.push_back(addMutation(lf.file, message, message.message, &lf.blockEnd, blockSize));
+					}
+					writtenFiles.insert(fileIdx);
+				}
+			}
+		}
+		if (!adds.empty()) {
+			co_await waitForAll(adds);
+		}
+	}
+
+	// Finish files
+	// TODO akanksha: Add FileLevel checksum.
+	std::vector<Future<Void>> finished;
+	for (auto& lf : activeFiles) {
+		finished.push_back(lf.file->finish());
+	}
+	co_await waitForAll(finished);
+
+	for (auto& lf : activeFiles) {
+		self->backups[lf.backupUid].nextFileBeginVersion = lastVersionInFile + 1;
+	}
+}
+
+static Future<Void> updateLogBytesWritten(BackupRangePartitionedData* self,
+                                          std::vector<UID> backupUids,
+                                          std::vector<Reference<IBackupFile>> logFiles) {
+	// TODO akanksha: Implement in next PR.
+	co_return;
+}
+
+static Future<bool> shouldBackupWorkerExitEarly(BackupRangePartitionedData* self) {
+	// TODO akanksha: Implement in next PR.
+	co_return true;
+}
+
+static Future<Void> monitorBackupStartedKeyChanges(BackupRangePartitionedData* self) {
+	// TODO akanksha: Implement in next PR.
+	co_return;
+}
+
+Future<Void> setBackupKeys(BackupRangePartitionedData* self, std::map<UID, Version> savedLogVersions) {
+	// TODO akanksha: Implement in next PR.
+	co_return;
+}
+
+static Future<Void> monitorWorkerPause(BackupRangePartitionedData* self) {
+	co_return;
+}
+
+Future<Void> monitorBackupRangePartitionedProgress(BackupRangePartitionedData* self) {
+	Future<Void> interval;
+
+	while (true) {
+		interval = delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL / 2.0);
+		while (self->backups.empty() || !self->logSystem.get()) {
+			co_await (self->changedTrigger.onTrigger() || self->logSystem.onChange());
+		}
+
+		// check all workers have started by checking their progress is larger
+		// than the backup's start version.
+		Reference<BackupRangePartitionedProgress> progress(new BackupRangePartitionedProgress(self->myId));
+		co_await getBackupRangePartitionedProgress(self->cx, self->myId, progress, SevDebug);
+
+		std::map<Tag, Version> tagVersions = progress->getEpochStatus(self->recruitedEpoch);
+		if (tagVersions.size() != self->totalTags) {
+			co_await interval;
+			continue;
+		}
+
+		std::map<UID, Version> savedLogVersions;
+		// update progress so far if previous epochs are done.
+		if (self->recruitedEpoch == self->oldestBackupEpoch) {
+			Version v = std::numeric_limits<Version>::max();
+			// Find the version we can gurantee is fully backed up for all backup workers.
+			for (const auto& [tag, version] : tagVersions) {
+				v = std::min(v, version);
+			}
+
+			for (auto& [uid, info] : self->backups) {
+				savedLogVersions.emplace(uid, v);
+				TraceEvent("BWRangePartitionedSavedBackupVersion", self->myId)
+				    .detail("BackupID", uid)
+				    .detail("Version", v);
+			}
+		}
+
+		// TODO akanksha: Implement and explain what setBackupKeys does in next PR.
+		Future<Void> setKeys = savedLogVersions.empty() ? Void() : setBackupKeys(self, savedLogVersions);
+		co_await (interval && setKeys);
+	}
+}
+
+Future<Void> saveProgress(BackupRangePartitionedData* self, Version backupVersion) {
+	Transaction tr(self->cx);
+	Key key = backupRangePartitionedProgressKey(self->myId);
+
+	while (true) {
+		Error err;
+		try {
+			// It's critical to save progress immediately so that after a master
+			// recovery, the new master can know the progress so far.
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+
+			// CHECK: Don't save progress if backup workers are disabled
+			Optional<Value> backupWorkerEnabled = co_await tr.get(rangeBackupWorkerEnabledKey);
+			if (!backupWorkerEnabled.present() || backupWorkerEnabled.get() == "0"_sr) {
+				TraceEvent("BWRangePartitionedProgressSkipped", self->myId).detail("Reason", "BackupWorkersDisabled");
+				co_return;
+			}
+
+			WorkerBackupStatus status(self->backupEpoch, backupVersion, self->tag, self->totalTags);
+			tr.set(key, backupRangePartitionedProgressValue(status));
+			tr.addReadConflictRange(singleKeyRange(key));
+			co_await tr.commit();
+			co_return;
+		} catch (Error& e) {
+			err = e;
+		}
+		co_await tr.onError(err);
+	}
+}
+
+// Uploads self->messages to storage and updates savedVersion.
+Future<Void> uploadData(BackupRangePartitionedData* self) {
+	// Version up to which messages will be popped from tlog.
+	Version popVersion = invalidVersion;
+
+	while (true) {
+		// Too large uploadDelay will delay popping tLog data for too long.
+		Future<Void> uploadDelay = delay(SERVER_KNOBS->BACKUP_UPLOAD_DELAY);
+
+		int numMsg = 0;
+		// Last Version that we popped from tlog.
+		Version lastPopVersion = popVersion;
+		// index of last version's end position in self->messages. i.e. the first index of next version???
+		int lastVersionIndex = 0;
+		// Version just before current popVersion - to find version boundaries.
+		Version lastVersion = invalidVersion;
+
+		for (auto& message : self->messages) {
+			// message may be prefetched in peek; uncommitted message should not be uploaded.
+			const Version msgVersion = message.getVersion();
+			if (msgVersion > self->maxPopVersion()) {
+				break;
+			}
+			if (msgVersion > popVersion) {
+				lastVersionIndex = numMsg;
+				lastVersion = popVersion;
+				popVersion = msgVersion;
+			}
+			numMsg++;
+		}
+		if (self->pullFinished()) {
+			popVersion = self->endVersion.get();
+		} else {
+			// make sure file is saved on version boundary
+			popVersion = lastVersion;
+			numMsg = lastVersionIndex;
+
+			// If we aren't able to process any messages and the lock is blocking us from
+			// queuing more, then we are stuck. This could suggest the lock capacity is too small.
+			ASSERT(numMsg > 0 || self->lock->waiters() == 0);
+		}
+
+		// TODO akanksha: Removed redundant check popVersion > lastPopVersion. Remove todo after testing completes.
+		if (numMsg > 0 || self->pullFinished()) {
+			TraceEvent("BWRangePartitionedSave", self->myId)
+			    .detail("Version", popVersion)
+			    .detail("LastPopVersion", lastPopVersion)
+			    .detail("SavedVersion", self->savedVersion)
+			    .detail("NumMsg", numMsg)
+			    .detail("MsgQ", self->messages.size());
+			// save an empty file for old epochs so that log file versions are continuous
+			co_await saveMutationsToFile(self, popVersion, numMsg);
+			self->eraseMessages(numMsg);
+		}
+
+		if (popVersion > self->savedVersion) {
+			co_await saveProgress(self, popVersion);
+			TraceEvent("BWRangePartitionedSavedProgress", self->myId)
+			    .detail("Tag", self->tag.toString())
+			    .detail("Version", popVersion)
+			    .detail("MsgQ", self->messages.size());
+			self->savedVersion = popVersion;
+			self->pop();
+		}
+
+		if (self->allMessageSaved()) {
+			co_return;
+		}
+
+		if (!self->pullFinished()) {
+			co_await (uploadDelay || self->doneTrigger.onTrigger());
+		}
+	}
+}
+
+Future<Void> backupWorkerRangePartitioned(BackupInterface interf,
+                                          InitializeBackupRequest req,
+                                          Reference<AsyncVar<ServerDBInfo> const> db) {
+	BackupRangePartitionedData self(interf.id(), db, req);
+	PromiseStream<Future<Void>> addActor;
+	Future<Void> error = actorCollection(addActor.getFuture());
+	Future<Void> dbInfoChange = Void();
+	Future<Void> pull;
+	Future<Void> done;
+	Error err;
+
+	TraceEvent("BWRangePartitionedStart", self.myId)
+	    .detail("Tag", req.routerTag.toString())
+	    .detail("TotalTags", req.totalTags)
+	    .detail("StartVersion", req.startVersion)
+	    .detail("EndVersion", req.endVersion.present() ? req.endVersion.get() : -1)
+	    .detail("LogEpoch", req.recruitedEpoch)
+	    .detail("BackupEpoch", req.backupEpoch);
+
+	try {
+		addActor.send(checkRemoved(db, req.recruitedEpoch, &self));
+		addActor.send(waitFailureServer(interf.waitFailure.getFuture()));
+
+		if (req.recruitedEpoch == req.backupEpoch && req.routerTag.id == 0) {
+			addActor.send(monitorBackupRangePartitionedProgress(&self));
+		}
+
+		// If the worker is on an old epoch and all backups starts a version >= the endVersion
+		bool exitEarly = co_await shouldBackupWorkerExitEarly(&self);
+		TraceEvent("BWRangePartitionedExitEarly", self.myId).detail("ExitEarly", exitEarly);
+		if (!exitEarly) {
+			addActor.send(monitorBackupStartedKeyChanges(&self));
+		}
+
+		pull = exitEarly ? Void() : pullAsyncData(&self);
+		addActor.send(pull);
+		done = exitEarly ? Void() : uploadData(&self);
+
+		while (true) {
+			auto res = co_await race(dbInfoChange, done, error);
+			if (res.index() == 0) {
+				dbInfoChange = db->onChange();
+				Reference<LogSystem> ls = makeLogSystemFromServerDBInfo(self.myId, db->get(), true);
+
+				if (ls.isValid()) {
+					self.logSystem.set(ls);
+					self.oldestBackupEpoch = std::max(self.oldestBackupEpoch, ls->getOldestBackupEpoch());
+					TraceEvent("BWRangePartitionedLogSystemUpdate", self.myId)
+					    .detail("Tag", self.tag.toString())
+					    .detail("TagLocality", self.tag.locality)
+					    .detail("OldestEpoch", self.oldestBackupEpoch);
+				} else {
+					TraceEvent("BWRangePartitionedNoLogSystem", self.myId);
+				}
+			} else if (res.index() == 1) {
+				TraceEvent("BWRangePartitionedDone", self.myId).detail("BackupEpoch", self.backupEpoch);
+				// Notify master so that this worker can be removed from log system, then this
+				// worker (for an old epoch's unfinished work) can safely exit.
+				co_await brokenPromiseToNever(db->get().clusterInterface.notifyBackupWorkerDone.getReply(
+				    BackupWorkerDoneRequest(self.myId, self.backupEpoch)));
+				break;
+			} else if (res.index() != 2) {
+				UNREACHABLE();
+			}
+		}
+		co_return;
+	} catch (Error& e) {
+		err = e;
+	}
+
+	if (err.code() == error_code_worker_removed) {
+		pull = Void(); // cancels pulling
+		try {
+			co_await done;
+		} catch (Error& shutdownErr) {
+			TraceEvent("BWRangePartitionedShutdownError", self.myId).errorUnsuppressed(shutdownErr);
+		}
+	}
+	TraceEvent("BWRangePartitionedTerminated", self.myId).errorUnsuppressed(err);
+	if (err.code() != error_code_actor_cancelled && err.code() != error_code_worker_removed) {
+		throw err;
+	}
+}
diff --git a/fdbserver/backupworker/CMakeLists.txt b/fdbserver/backupworker/CMakeLists.txt
new file mode 100644
index 00000000000..fa4e4374396
--- /dev/null
+++ b/fdbserver/backupworker/CMakeLists.txt
@@ -0,0 +1,15 @@
+fdb_find_sources(FDBSERVER_BACKUPWORKER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_backupworker SRCS ${FDBSERVER_BACKUPWORKER_SRCS})
+add_fdbserver_link_test(fdbserver_backupworkerlinktest
+  fdbserver_backupworker
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_backupworker)
+target_include_directories(fdbserver_backupworker
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_backupworker PRIVATE fdbserver_core fdbserver_logsystem)
diff --git a/fdbserver/backupworker/include/fdbserver/backupworker/BackupWorker.h b/fdbserver/backupworker/include/fdbserver/backupworker/BackupWorker.h
new file mode 100644
index 00000000000..e7170f24827
--- /dev/null
+++ b/fdbserver/backupworker/include/fdbserver/backupworker/BackupWorker.h
@@ -0,0 +1,29 @@
+/*
+ * BackupWorker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/BackupInterface.h"
+#include "flow/flow.h"
+
+struct InitializeBackupRequest;
+struct ServerDBInfo;
+
+Future<Void> backupWorker(BackupInterface bi, InitializeBackupRequest req, Reference<AsyncVar<ServerDBInfo> const> db);
diff --git a/fdbserver/clustercontroller/CMakeLists.txt b/fdbserver/clustercontroller/CMakeLists.txt
new file mode 100644
index 00000000000..c98434860f0
--- /dev/null
+++ b/fdbserver/clustercontroller/CMakeLists.txt
@@ -0,0 +1,15 @@
+fdb_find_sources(FDBSERVER_CLUSTERCONTROLLER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_clustercontroller SRCS ${FDBSERVER_CLUSTERCONTROLLER_SRCS})
+add_fdbserver_link_test(fdbserver_clustercontrollerlinktest
+  fdbserver_clustercontroller
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_clustercontroller)
+target_include_directories(fdbserver_clustercontroller
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_clustercontroller PRIVATE fdbserver_core fdbserver_logsystem)
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/clustercontroller/ClusterController.actor.cpp
similarity index 87%
rename from fdbserver/ClusterController.actor.cpp
rename to fdbserver/clustercontroller/ClusterController.actor.cpp
index 8875198b62d..644abd0d919 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/clustercontroller/ClusterController.actor.cpp
@@ -32,57 +32,123 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/Locality.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/BackupInterface.h"
-#include "fdbserver/BackupProgress.actor.h"
-#include "fdbserver/CoordinatedState.h"
-#include "fdbserver/CoordinationInterface.h" // copy constructors for ServerCoordinators class
-#include "fdbserver/ClusterController.actor.h"
-#include "fdbserver/ClusterRecovery.actor.h"
-#include "fdbserver/DataDistributorInterface.h"
-#include "fdbserver/DBCoreState.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/LeaderElection.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemConfig.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/SingletonRoles.h"
-#include "fdbserver/Status.actor.h"
-#include "fdbserver/LatencyBandConfig.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbserver/RecoveryState.h"
+#include "fdbserver/core/BackupInterface.h"
+#include "fdbserver/core/CoordinatedState.h"
+#include "fdbserver/core/CoordinationInterface.h" // copy constructors for ServerCoordinators class
+#include "fdbserver/clustercontroller/ClusterController.h"
+#include "ClusterController.h"
+#include "ClusterRecovery.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/core/LeaderElection.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/core/LogSystemConfig.h"
+#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "SingletonRoles.h"
+#include "Status.h"
+#include "fdbserver/core/LatencyBandConfig.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbserver/core/RecoveryState.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/Replication.h"
 #include "fdbrpc/ReplicationUtils.h"
 #include "fdbrpc/sim_validation.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbserver/logsystem/ApplyMetadataMutation.h"
+#include "fdbserver/core/BackupProgress.h"
+#include "fdbserver/core/DBCoreState.h"
+#include "fdbserver/core/MoveKeys.h"
 #include "flow/Error.h"
 #include "flow/Trace.h"
 #include "flow/Util.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-ACTOR Future<Optional<Value>> getPreviousCoordinators(ClusterControllerData* self) {
-	state ReadYourWritesTransaction tr(self->db.db);
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			Optional<Value> previousCoordinators = wait(tr.get(previousCoordinatorsKey));
-			return previousCoordinators;
-		} catch (Error& e) {
-			wait(tr.onError(e));
-		}
-	}
+static bool storageTeamOneReplicaLeftIsCritical(DatabaseConfiguration const& configuration) {
+	return configuration.initialized && configuration.storageTeamSize == 3;
+}
+
+ClusterControllerData::ClusterControllerData(ClusterControllerFullInterface const& ccInterface,
+                                             LocalityData const& locality,
+                                             ServerCoordinators const& coordinators,
+                                             Reference<AsyncVar<Optional<UID>>> clusterId)
+  : gotProcessClasses(false), gotFullyRecoveredConfig(false), shouldCommitSuicide(false),
+    clusterControllerProcessId(locality.processId()), clusterControllerDcId(locality.dcId()), id(ccInterface.id()),
+    clusterId(clusterId), ac(false),
+    clusterHealthWorkerEventProvider(makeReference<cluster_health::WorkerEventProvider>()),
+    clusterHealthMonitor(cluster_health::Monitor::create(clusterHealthWorkerEventProvider)),
+    outstandingRequestChecker(Void()), outstandingRemoteRequestChecker(Void()), startTime(now()),
+    goodRecruitmentTime(Never()), goodRemoteRecruitmentTime(Never()), dcLogServerVersionDifference(0),
+    dcStorageServerVersionDifference(0), datacenterVersionDifference(0), versionDifferenceUpdated(false),
+    remoteDCMonitorStarted(false), remoteTransactionSystemDegraded(false), recruitDistributor(false),
+    recruitRatekeeper(false), recruitConsistencyScan(false),
+    clusterControllerMetrics("ClusterController", id.toString()),
+    openDatabaseRequests("OpenDatabaseRequests", clusterControllerMetrics),
+    registerWorkerRequests("RegisterWorkerRequests", clusterControllerMetrics),
+    getWorkersRequests("GetWorkersRequests", clusterControllerMetrics),
+    getClientWorkersRequests("GetClientWorkersRequests", clusterControllerMetrics),
+    registerMasterRequests("RegisterMasterRequests", clusterControllerMetrics),
+    statusRequests("StatusRequests", clusterControllerMetrics),
+    recruitedMasterWorkerEventHolder(makeReference<EventCacheHolder>("RecruitedMasterWorker")) {
+	(void)coordinators;
+	auto serverInfo = ServerDBInfo();
+	serverInfo.id = deterministicRandom()->randomUniqueID();
+	serverInfo.infoGeneration = ++db.dbInfoCount;
+	serverInfo.masterLifetime.ccID = id;
+	serverInfo.clusterInterface = ccInterface;
+	serverInfo.myLocality = locality;
+	db.serverInfo->set(serverInfo);
+	cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::True);
+
+	specialCounter(clusterControllerMetrics, "ClientCount", [this]() { return db.clientCount; });
+	updateClusterHealthMonitorInputs();
+}
+
+void ClusterControllerData::updateClusterHealthMonitorInputs() {
+	std::vector<WorkerDetails> workers;
+	workers.reserve(id_worker.size());
+	for (auto const& [processId, workerInfo] : id_worker) {
+		(void)processId;
+		workers.push_back(workerInfo.details);
+	}
+	clusterHealthWorkerEventProvider->setWorkers(std::move(workers));
+	clusterHealthWorkerEventProvider->setRecoveryState(db.serverInfo->get().recoveryState);
+	clusterHealthWorkerEventProvider->setStorageTeamOneReplicaLeftIsCritical(
+	    storageTeamOneReplicaLeftIsCritical(db.config));
+
+	Optional<WorkerInterface> ratekeeperWorker;
+	if (db.serverInfo->get().ratekeeper.present()) {
+		auto workerIt = id_worker.find(db.serverInfo->get().ratekeeper.get().locality.processId());
+		if (workerIt != id_worker.end()) {
+			ratekeeperWorker = workerIt->second.details.interf;
+		}
+	}
+	clusterHealthWorkerEventProvider->setRatekeeperWorker(std::move(ratekeeperWorker));
+
+	Optional<WorkerInterface> dataDistributorWorker;
+	if (db.serverInfo->get().distributor.present()) {
+		auto workerIt = id_worker.find(db.serverInfo->get().distributor.get().locality.processId());
+		if (workerIt != id_worker.end()) {
+			dataDistributorWorker = workerIt->second.details.interf;
+		}
+	}
+	clusterHealthWorkerEventProvider->setDataDistributorWorker(std::move(dataDistributorWorker));
+
+	std::vector<StorageServerInterface> storageServers;
+	storageServers.reserve(storageStatusInfos.size());
+	for (auto const& storageStatusInfo : storageStatusInfos) {
+		storageServers.push_back(storageStatusInfo);
+	}
+	clusterHealthWorkerEventProvider->setStorageServers(std::move(storageServers));
+
+	clusterHealthWorkerEventProvider->setTLogs(db.serverInfo->get().logSystemConfig.allPresentLogs());
 }
 
 bool ClusterControllerData::processesInSameDC(const NetworkAddress& addr1, const NetworkAddress& addr2) const {
@@ -219,20 +285,20 @@ bool ClusterControllerData::remoteTransactionSystemContainsDegradedServers() {
 }
 
 // Recruit failed log routers in parallel
-ACTOR Future<Void> recruitFailedLogRouters(ClusterControllerData* cluster,
-                                           ClusterControllerData::DBInfo* db,
-                                           std::vector<int> tagIds,
-                                           int logSetIndex,
-                                           Reference<ILogSystem> logSystem,
-                                           LogSystemConfig config) {
-	state Optional<Key> targetDcId =
+Future<Void> recruitFailedLogRouters(ClusterControllerData* cluster,
+                                     ClusterControllerData::DBInfo* db,
+                                     std::vector<int> tagIds,
+                                     int logSetIndex,
+                                     Reference<LogSystem> logSystem,
+                                     LogSystemConfig config) {
+	Optional<Key> targetDcId =
 	    db->recoveryData->remoteDcIds.size() ? db->recoveryData->remoteDcIds[0] : Optional<Key>();
 
 	// Use getWorkersForRoleInDatacenter to get workers for all log routers at once
 	ClusterControllerData::WorkerUsages id_used;
 	cluster->updateKnownIds(&id_used);
 
-	state std::vector<WorkerDetails> workers =
+	std::vector<WorkerDetails> workers =
 	    cluster->getWorkersForRoleInDatacenter(targetDcId, ProcessClass::LogRouter, tagIds.size(), db->config, id_used);
 
 	if (workers.size() < tagIds.size()) {
@@ -251,7 +317,7 @@ ACTOR Future<Void> recruitFailedLogRouters(ClusterControllerData* cluster,
 	    .detail("TargetDcId", targetDcId);
 
 	// Build requests for all log routers and send them in parallel
-	state std::vector<Future<ErrorOr<TLogInterface>>> recruitments;
+	std::vector<Future<ErrorOr<TLogInterface>>> recruitments;
 
 	for (int i = 0; i < tagIds.size(); i++) {
 		Tag routerTag = Tag(tagLocalityLogRouter, tagIds[i]);
@@ -293,8 +359,8 @@ ACTOR Future<Void> recruitFailedLogRouters(ClusterControllerData* cluster,
 	}
 
 	// Wait for all recruitments to complete
-	state Future<Void> timeout = delay(SERVER_KNOBS->CC_RERECRUIT_LOG_ROUTER_TIMEOUT);
-	wait(waitForAll(recruitments) || timeout);
+	Future<Void> timeout = delay(SERVER_KNOBS->CC_RERECRUIT_LOG_ROUTER_TIMEOUT);
+	co_await (waitForAll(recruitments) || timeout);
 
 	if (timeout.isReady()) {
 		TraceEvent(SevWarn, "LogRoutersRecruitmentTimeout", cluster->id)
@@ -330,17 +396,16 @@ ACTOR Future<Void> recruitFailedLogRouters(ClusterControllerData* cluster,
 	db->recoveryData->registrationTrigger.trigger();
 
 	TraceEvent("LogRoutersRecruitmentComplete", cluster->id).detail("Count", tagIds.size());
-
-	return Void();
 }
 
-ACTOR Future<std::vector<int>> monitorLogRouters(Reference<ILogSystem> logSystem, int logSetIndex) {
-	state std::vector<Future<Void>> failures;
-	state LogSystemConfig config = logSystem->getLogSystemConfig();
-	state std::vector<int> failedTagIds;
+Future<std::vector<int>> monitorLogRouters(Reference<LogSystem> logSystem, int logSetIndex) {
+	std::vector<Future<Void>> failures;
+	LogSystemConfig config = logSystem->getLogSystemConfig();
+	std::vector<int> failedTagIds;
 
 	if (logSetIndex == -1) {
-		return Never();
+		co_await Future<Void>(Never());
+		UNREACHABLE();
 	}
 
 	ASSERT_WE_THINK(logSetIndex >= 0 && logSetIndex < config.tLogs.size());
@@ -357,17 +422,18 @@ ACTOR Future<std::vector<int>> monitorLogRouters(Reference<ILogSystem> logSystem
 	}
 
 	if (failures.empty()) {
-		return Never();
+		co_await Future<Void>(Never());
+		UNREACHABLE();
 	}
 
-	wait(quorum(failures, 1));
+	co_await quorum(failures, 1);
 
 	for (int i = 0; i < failures.size(); i++) {
 		if (failures[i].isReady() || failures[i].isError()) {
 			failedTagIds.push_back(i);
 		}
 	}
-	return failedTagIds;
+	co_return failedTagIds;
 }
 
 // When in fully_recovered state, the cluster controller will monitor log routers
@@ -438,7 +504,7 @@ Future<Void> monitorAndRecruitLogRouters(ClusterControllerData* self) {
 
 		ASSERT(self->db.recoveryData.isValid());
 		uint64_t recoveryCount = self->db.recoveryData->cstate.myDBState.recoveryCount;
-		Reference<ILogSystem> logSystem = self->db.recoveryData->logSystem;
+		Reference<LogSystem> logSystem = self->db.recoveryData->logSystem;
 		LogSystemConfig config = logSystem->getLogSystemConfig();
 
 		// Find the log set with log routers (should be remote/satellite)
@@ -933,16 +999,16 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 }
 
-ACTOR Future<Void> doCheckOutstandingRequests(ClusterControllerData* self) {
+Future<Void> doCheckOutstandingRequests(ClusterControllerData* self) {
 	try {
-		wait(delay(SERVER_KNOBS->CHECK_OUTSTANDING_INTERVAL));
+		co_await delay(SERVER_KNOBS->CHECK_OUTSTANDING_INTERVAL);
 		while (now() - self->lastRecruitTime < SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY ||
 		       !self->goodRecruitmentTime.isReady()) {
 			if (now() - self->lastRecruitTime < SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY) {
-				wait(delay(SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY + 0.001 - (now() - self->lastRecruitTime)));
+				co_await delay(SERVER_KNOBS->SINGLETON_RECRUIT_BME_DELAY + 0.001 - (now() - self->lastRecruitTime));
 			}
 			if (!self->goodRecruitmentTime.isReady()) {
-				wait(self->goodRecruitmentTime);
+				co_await self->goodRecruitmentTime;
 			}
 		}
 
@@ -961,14 +1027,13 @@ ACTOR Future<Void> doCheckOutstandingRequests(ClusterControllerData* self) {
 			TraceEvent(SevError, "CheckOutstandingError").error(e);
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> doCheckOutstandingRemoteRequests(ClusterControllerData* self) {
+Future<Void> doCheckOutstandingRemoteRequests(ClusterControllerData* self) {
 	try {
-		wait(delay(SERVER_KNOBS->CHECK_OUTSTANDING_INTERVAL));
+		co_await delay(SERVER_KNOBS->CHECK_OUTSTANDING_INTERVAL);
 		while (!self->goodRemoteRecruitmentTime.isReady()) {
-			wait(self->goodRemoteRecruitmentTime);
+			co_await self->goodRemoteRecruitmentTime;
 		}
 
 		checkOutstandingRemoteRecruitmentRequests(self);
@@ -977,7 +1042,6 @@ ACTOR Future<Void> doCheckOutstandingRemoteRequests(ClusterControllerData* self)
 			TraceEvent(SevError, "CheckOutstandingError").error(e);
 		}
 	}
-	return Void();
 }
 
 void checkOutstandingRequests(ClusterControllerData* self) {
@@ -990,14 +1054,14 @@ void checkOutstandingRequests(ClusterControllerData* self) {
 	}
 }
 
-ACTOR Future<Void> rebootAndCheck(ClusterControllerData* cluster, Optional<Standalone<StringRef>> processID) {
+Future<Void> rebootAndCheck(ClusterControllerData* cluster, Optional<Standalone<StringRef>> processID) {
 	{
 		ASSERT(processID.present());
 		auto watcher = cluster->id_worker.find(processID);
 		ASSERT(watcher != cluster->id_worker.end());
 
 		watcher->second.reboots++;
-		wait(delay(g_network->isSimulated() ? SERVER_KNOBS->SIM_SHUTDOWN_TIMEOUT : SERVER_KNOBS->SHUTDOWN_TIMEOUT));
+		co_await delay(g_network->isSimulated() ? SERVER_KNOBS->SIM_SHUTDOWN_TIMEOUT : SERVER_KNOBS->SHUTDOWN_TIMEOUT);
 	}
 
 	{
@@ -1008,8 +1072,6 @@ ACTOR Future<Void> rebootAndCheck(ClusterControllerData* cluster, Optional<Stand
 				checkOutstandingRequests(cluster);
 		}
 	}
-
-	return Void();
 }
 
 ACTOR Future<Void> workerAvailabilityWatch(WorkerInterface worker,
@@ -1060,6 +1122,7 @@ ACTOR Future<Void> workerAvailabilityWatch(WorkerInterface worker,
 						cluster->addr_locality.erase(worker.secondaryAddress().get());
 					}
 				}
+				cluster->updateClusterHealthMonitorInputs();
 				cluster->updateWorkerList.set(worker.locality.processId(), Optional<ProcessData>());
 				return Void();
 			}
@@ -1084,18 +1147,6 @@ struct FailureStatusInfo {
 	}
 };
 
-ACTOR Future<std::vector<TLogInterface>> requireAll(std::vector<Future<Optional<std::vector<TLogInterface>>>> in) {
-	state std::vector<TLogInterface> out;
-	state int i;
-	for (i = 0; i < in.size(); i++) {
-		Optional<std::vector<TLogInterface>> x = wait(in[i]);
-		if (!x.present())
-			throw recruitment_failed();
-		out.insert(out.end(), x.get().begin(), x.get().end());
-	}
-	return out;
-}
-
 void clusterRecruitStorage(ClusterControllerData* self, RecruitStorageRequest req) {
 	try {
 		if (!self->gotProcessClasses && !req.criticalRecruitment)
@@ -1154,6 +1205,8 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 	db->recoveryStalled = req.recoveryStalled;
 	if (req.configuration.present()) {
 		db->config = req.configuration.get();
+		self->clusterHealthWorkerEventProvider->setStorageTeamOneReplicaLeftIsCritical(
+		    storageTeamOneReplicaLeftIsCritical(db->config));
 
 		if (req.recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
 			self->gotFullyRecoveredConfig = true;
@@ -1233,6 +1286,7 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 		dbInfo.id = deterministicRandom()->randomUniqueID();
 		dbInfo.infoGeneration = ++self->db.dbInfoCount;
 		self->db.serverInfo->set(dbInfo);
+		self->updateClusterHealthMonitorInputs();
 	}
 
 	checkOutstandingRequests(self);
@@ -1399,6 +1453,7 @@ void registerWorker(RegisterWorkerRequest req, ClusterControllerData* self) {
 		    w.locality.processId() == self->db.serverInfo->get().master.locality.processId()) {
 			self->masterProcessId = w.locality.processId();
 		}
+		self->updateClusterHealthMonitorInputs();
 		self->updateDBInfoEndpoints.insert(w.updateServerDBInfo.getEndpoint());
 		self->updateDBInfo.trigger();
 		checkOutstandingRequests(self);
@@ -1426,6 +1481,7 @@ void registerWorker(RegisterWorkerRequest req, ClusterControllerData* self) {
 		}
 
 		// Re-registrations must refresh DBInfo delivery, otherwise rebooted workers can miss role updates.
+		self->updateClusterHealthMonitorInputs();
 		self->updateDBInfoEndpoints.insert(w.updateServerDBInfo.getEndpoint());
 		self->updateDBInfo.trigger();
 		checkOutstandingRequests(self);
@@ -1465,39 +1521,40 @@ void registerWorker(RegisterWorkerRequest req, ClusterControllerData* self) {
 
 #define TIME_KEEPER_VERSION "1"_sr
 
-ACTOR Future<Void> timeKeeperSetVersion(ClusterControllerData* self) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
-	loop {
+Future<Void> timeKeeperSetVersion(ClusterControllerData* self) {
+	auto tr = makeReference<ReadYourWritesTransaction>(self->cx);
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->set(timeKeeperVersionKey, TIME_KEEPER_VERSION);
-			wait(tr->commit());
+			co_await tr->commit();
 			break;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
-
-	return Void();
 }
 
 // This actor periodically gets read version and writes it to cluster with current timestamp as key. To avoid
 // running out of space, it limits the max number of entries and clears old entries on each update. This mapping is
 // used from backup and restore to get the version information for a timestamp.
-ACTOR Future<Void> timeKeeper(ClusterControllerData* self) {
-	state KeyBackedMap<int64_t, Version> versionMap(timeKeeperPrefixRange.begin);
+Future<Void> timeKeeper(ClusterControllerData* self) {
+	KeyBackedMap<int64_t, Version> versionMap(timeKeeperPrefixRange.begin);
 
 	TraceEvent("TimeKeeperStarted").log();
 
-	wait(timeKeeperSetVersion(self));
+	co_await timeKeeperSetVersion(self);
 
-	loop {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
-		loop {
+	while (true) {
+		auto tr = makeReference<ReadYourWritesTransaction>(self->cx);
+		while (true) {
+			Error err;
 			try {
-				state UID debugID = deterministicRandom()->randomUniqueID();
+				UID debugID = deterministicRandom()->randomUniqueID();
 				if (!g_network->isSimulated()) {
 					// This is done to provide an arbitrary logged transaction every ~10s.
 					// FIXME: replace or augment this with logging on the proxy which tracks
@@ -1508,7 +1565,7 @@ ACTOR Future<Void> timeKeeper(ClusterControllerData* self) {
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-				Optional<Value> disableValue = wait(tr->get(timeKeeperDisableKey));
+				Optional<Value> disableValue = co_await tr->get(timeKeeperDisableKey);
 				if (disableValue.present()) {
 					break;
 				}
@@ -1524,30 +1581,31 @@ ACTOR Future<Void> timeKeeper(ClusterControllerData* self) {
 					versionMap.erase(tr, 0, ttl);
 				}
 
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		wait(delay(SERVER_KNOBS->TIME_KEEPER_DELAY));
+		co_await delay(SERVER_KNOBS->TIME_KEEPER_DELAY);
 	}
 }
 
-ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
-                                ClusterControllerData* self,
-                                ServerCoordinators coordinators) {
+Future<Void> statusServer(FutureStream<StatusRequest> requests,
+                          ClusterControllerData* self,
+                          ServerCoordinators coordinators) {
 	// Seconds since the END of the last GetStatus executed
-	state double last_request_time = 0.0;
+	double last_request_time = 0.0;
 
 	// Place to accumulate a batch of requests to respond to
-	state std::vector<StatusRequest> requests_batch;
+	std::vector<StatusRequest> requests_batch;
 
-	loop {
+	while (true) {
 		try {
 			// Wait til first request is ready
-			StatusRequest req = waitNext(requests);
+			StatusRequest req = co_await requests;
 			++self->statusRequests;
 			requests_batch.push_back(req);
 
@@ -1556,7 +1614,7 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 
 			// Wait if needed to satisfy min_time knob, also allows more requests to queue up.
 			double minwait = std::max(next_allowed_request_time - now(), 0.0);
-			wait(delay(minwait));
+			co_await delay(minwait);
 
 			// Get all requests that are ready right *now*, before GetStatus() begins.
 			// All of these requests will be responded to with the next GetStatus() result.
@@ -1597,18 +1655,21 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 				}
 			}
 
-			state ErrorOr<StatusReply> result = wait(errorOr(clusterGetStatus(self->db.serverInfo,
-			                                                                  self->cx,
-			                                                                  workers,
-			                                                                  workerIssues,
-			                                                                  self->storageStatusInfos,
-			                                                                  &self->db.clientStatus,
-			                                                                  coordinators,
-			                                                                  incompatibleConnections,
-			                                                                  self->datacenterVersionDifference,
-			                                                                  self->dcLogServerVersionDifference,
-			                                                                  self->dcStorageServerVersionDifference,
-			                                                                  self->excludedDegradedServers)));
+			ErrorOr<StatusReply> result = co_await errorOr(
+			    clusterGetStatus(self->db.serverInfo,
+			                     self->cx,
+			                     workers,
+			                     workerIssues,
+			                     self->storageStatusInfos,
+			                     &self->db.clientStatus,
+			                     coordinators,
+			                     incompatibleConnections,
+			                     self->datacenterVersionDifference,
+			                     self->dcLogServerVersionDifference,
+			                     self->dcStorageServerVersionDifference,
+			                     self->excludedDegradedServers,
+			                     std::max(CLIENT_KNOBS->STATUS_TIMEOUT - SERVER_KNOBS->STATUS_TIMEOUT_BUFFER,
+			                              SERVER_KNOBS->STATUS_TIMEOUT_BUFFER)));
 
 			if (result.isError() && result.getError().code() == error_code_actor_cancelled)
 				throw result.getError();
@@ -1617,7 +1678,7 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 			// requests
 			last_request_time = now();
 
-			state Optional<StatusReply> faultToleranceRelatedStatus;
+			Optional<StatusReply> faultToleranceRelatedStatus;
 			while (!requests_batch.empty()) {
 				if (result.isError())
 					requests_batch.back().reply.sendError(result.getError());
@@ -1631,7 +1692,7 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 					requests_batch.back().reply.send(faultToleranceRelatedStatus.get());
 				}
 				requests_batch.pop_back();
-				wait(yield());
+				co_await yield();
 			}
 			faultToleranceRelatedStatus.reset();
 		} catch (Error& e) {
@@ -1641,20 +1702,21 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 	}
 }
 
-ACTOR Future<Void> monitorProcessClasses(ClusterControllerData* self) {
+Future<Void> monitorProcessClasses(ClusterControllerData* self) {
 
-	state ReadYourWritesTransaction trVer(self->db.db);
-	loop {
+	ReadYourWritesTransaction trVer(self->db.db);
+	while (true) {
+		Error err;
 		try {
 			trVer.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			trVer.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-			Optional<Value> val = wait(trVer.get(processClassVersionKey));
+			Optional<Value> val = co_await trVer.get(processClassVersionKey);
 
 			if (val.present())
 				break;
 
-			RangeResult processClasses = wait(trVer.getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult processClasses = co_await trVer.getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!processClasses.more && processClasses.size() < CLIENT_KNOBS->TOO_MANY);
 
 			trVer.clear(processClassKeys);
@@ -1664,22 +1726,24 @@ ACTOR Future<Void> monitorProcessClasses(ClusterControllerData* self) {
 				trVer.set(processClassKeyFor(processUid.toString()), it.value);
 			}
 
-			wait(trVer.commit());
+			co_await trVer.commit();
 			TraceEvent("ProcessClassUpgrade").log();
 			break;
 		} catch (Error& e) {
-			wait(trVer.onError(e));
+			err = e;
 		}
+		co_await trVer.onError(err);
 	}
 
-	loop {
-		state ReadYourWritesTransaction tr(self->db.db);
+	while (true) {
+		ReadYourWritesTransaction tr(self->db.db);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				RangeResult processClasses = wait(tr.getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult processClasses = co_await tr.getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!processClasses.more && processClasses.size() < CLIENT_KNOBS->TOO_MANY);
 
 				if (processClasses != self->lastProcessClasses || !self->gotProcessClasses) {
@@ -1718,27 +1782,29 @@ ACTOR Future<Void> monitorProcessClasses(ClusterControllerData* self) {
 					checkOutstandingRequests(self);
 				}
 
-				state Future<Void> watchFuture = tr.watch(processClassChangeKey);
-				wait(tr.commit());
-				wait(watchFuture);
+				Future<Void> watchFuture = tr.watch(processClassChangeKey);
+				co_await tr.commit();
+				co_await watchFuture;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 }
 
-ACTOR Future<Void> monitorServerInfoConfig(ClusterControllerData::DBInfo* db) {
-	loop {
-		state ReadYourWritesTransaction tr(db->db);
-		loop {
+Future<Void> monitorServerInfoConfig(ClusterControllerData::DBInfo* db) {
+	while (true) {
+		ReadYourWritesTransaction tr(db->db);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 
-				Optional<Value> configVal = wait(tr.get(latencyBandConfigKey));
+				Optional<Value> configVal = co_await tr.get(latencyBandConfigKey);
 				Optional<LatencyBandConfig> config;
 				if (configVal.present()) {
 					config = LatencyBandConfig::parse(configVal.get());
@@ -1753,32 +1819,34 @@ ACTOR Future<Void> monitorServerInfoConfig(ClusterControllerData::DBInfo* db) {
 					db->serverInfo->set(serverInfo);
 				}
 
-				state Future<Void> configChangeFuture = tr.watch(latencyBandConfigKey);
+				Future<Void> configChangeFuture = tr.watch(latencyBandConfigKey);
 
-				wait(tr.commit());
-				wait(configChangeFuture);
+				co_await tr.commit();
+				co_await configChangeFuture;
 
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 }
 
 // Monitors storage metadata changes and updates to storage servers.
-ACTOR Future<Void> monitorStorageMetadata(ClusterControllerData* self) {
-	state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
-	                                                                                           IncludeVersion());
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
-	state std::vector<StorageServerMetaInfo> servers;
-	loop {
+Future<Void> monitorStorageMetadata(ClusterControllerData* self) {
+	KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
+	                                                                                     IncludeVersion());
+	auto tr = makeReference<ReadYourWritesTransaction>(self->cx);
+	std::vector<StorageServerMetaInfo> servers;
+	while (true) {
+		Error err;
 		try {
 			servers.clear();
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			state RangeResult serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult serverList = co_await tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
 			servers.reserve(serverList.size());
@@ -1786,7 +1854,7 @@ ACTOR Future<Void> monitorStorageMetadata(ClusterControllerData* self) {
 				servers.push_back(StorageServerMetaInfo(decodeServerListValue(ss.value)));
 			}
 
-			state RangeResult serverMetadata = wait(tr->getRange(serverMetadataKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult serverMetadata = co_await tr->getRange(serverMetadataKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!serverMetadata.more && serverMetadata.size() < CLIENT_KNOBS->TOO_MANY);
 			std::map<UID, StorageMetadataType> idMetadata;
 			for (const auto& sm : serverMetadata) {
@@ -1801,15 +1869,18 @@ ACTOR Future<Void> monitorStorageMetadata(ClusterControllerData* self) {
 				}
 			}
 
-			state Future<Void> watchFuture = tr->watch(serverMetadataChangeKey);
-			wait(tr->commit());
+			Future<Void> watchFuture = tr->watch(serverMetadataChangeKey);
+			co_await tr->commit();
 
 			self->storageStatusInfos = std::move(servers);
-			wait(watchFuture);
+			self->updateClusterHealthMonitorInputs();
+			co_await watchFuture;
 			tr->reset();
+			continue;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
@@ -1818,22 +1889,23 @@ ACTOR Future<Void> monitorStorageMetadata(ClusterControllerData* self) {
 // all processes in the system by updating the ClientDBInfo object. The
 // GlobalConfig actor class contains the functionality to read the latest
 // history and update the processes local view.
-ACTOR Future<Void> monitorGlobalConfig(ClusterControllerData::DBInfo* db) {
-	loop {
-		state ReadYourWritesTransaction tr(db->db);
-		state ClientDBInfo clientInfo;
-		loop {
+Future<Void> monitorGlobalConfig(ClusterControllerData::DBInfo* db) {
+	while (true) {
+		ReadYourWritesTransaction tr(db->db);
+		ClientDBInfo clientInfo;
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				state Optional<Value> globalConfigVersion = wait(tr.get(globalConfigVersionKey));
+				Optional<Value> globalConfigVersion = co_await tr.get(globalConfigVersionKey);
 
 				if (globalConfigVersion.present()) {
 					// Since the history keys end with versionstamps, they
 					// should be sorted correctly (versionstamps are stored in
 					// big-endian order).
 					RangeResult globalConfigHistory =
-					    wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY));
+					    co_await tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY);
 					// If the global configuration version key has been set,
 					// the history should contain at least one item.
 					ASSERT(globalConfigHistory.size() > 0);
@@ -1896,22 +1968,23 @@ ACTOR Future<Void> monitorGlobalConfig(ClusterControllerData::DBInfo* db) {
 					db->clientInfo->set(clientInfo);
 				}
 
-				state Future<Void> globalConfigFuture = tr.watch(globalConfigVersionKey);
-				wait(tr.commit());
-				wait(globalConfigFuture);
+				Future<Void> globalConfigFuture = tr.watch(globalConfigVersionKey);
+				co_await tr.commit();
+				co_await globalConfigFuture;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 }
 
-ACTOR Future<Void> updatedChangingDatacenters(ClusterControllerData* self) {
+Future<Void> updatedChangingDatacenters(ClusterControllerData* self) {
 	// do not change the cluster controller until all the processes have had a chance to register
-	wait(delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
-	loop {
-		state Future<Void> onChange = self->desiredDcIds.onChange();
+	co_await delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY);
+	while (true) {
+		Future<Void> onChange = self->desiredDcIds.onChange();
 		if (!self->desiredDcIds.get().present()) {
 			self->changingDcIds.set(std::make_pair(false, self->desiredDcIds.get()));
 		} else {
@@ -1930,7 +2003,7 @@ ACTOR Future<Void> updatedChangingDatacenters(ClusterControllerData* self) {
 					worker.reply.send(RegisterWorkerReply(worker.details.processClass, worker.priorityInfo));
 				}
 			} else {
-				state int currentFit = ProcessClass::BestFit;
+				int currentFit = ProcessClass::BestFit;
 				while (currentFit <= ProcessClass::NeverAssign) {
 					bool updated = false;
 					for (auto& it : self->id_worker) {
@@ -1951,14 +2024,14 @@ ACTOR Future<Void> updatedChangingDatacenters(ClusterControllerData* self) {
 						}
 					}
 					if (updated && currentFit < ProcessClass::NeverAssign) {
-						wait(delay(SERVER_KNOBS->CC_CLASS_DELAY));
+						co_await delay(SERVER_KNOBS->CC_CLASS_DELAY);
 					}
 					currentFit++;
 				}
 			}
 		}
 
-		wait(onChange);
+		co_await onChange;
 	}
 }
 
@@ -2023,9 +2096,9 @@ ACTOR Future<Void> updatedChangedDatacenters(ClusterControllerData* self) {
 	}
 }
 
-ACTOR Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self) {
-	state double lastLogTime = 0;
-	loop {
+Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self) {
+	double lastLogTime = 0;
+	while (true) {
 		self->versionDifferenceUpdated = false;
 		if (self->db.serverInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS &&
 		    self->db.config.usableRegions == 1) {
@@ -2040,12 +2113,12 @@ ACTOR Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self
 				checkOutstandingRequests(self);
 			}
 
-			wait(self->db.serverInfo->onChange());
+			co_await self->db.serverInfo->onChange();
 			continue;
 		}
 
-		state Optional<TLogInterface> primaryLog;
-		state Optional<TLogInterface> remoteLog;
+		Optional<TLogInterface> primaryLog;
+		Optional<TLogInterface> remoteLog;
 		if (self->db.serverInfo->get().recoveryState >= RecoveryState::ALL_LOGS_RECRUITED) {
 			for (auto& logSet : self->db.serverInfo->get().logSystemConfig.tLogs) {
 				if (logSet.isLocal && logSet.locality != tagLocalitySatellite) {
@@ -2068,20 +2141,20 @@ ACTOR Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self
 		}
 
 		if (!primaryLog.present() || !remoteLog.present() || !self->db.serverInfo->get().ratekeeper.present()) {
-			wait(self->db.serverInfo->onChange());
+			co_await self->db.serverInfo->onChange();
 			continue;
 		}
 
-		state Future<Void> onChange = self->db.serverInfo->onChange();
-		loop {
-			state Future<TLogQueuingMetricsReply> primaryMetrics =
+		Future<Void> onChange = self->db.serverInfo->onChange();
+		while (true) {
+			Future<TLogQueuingMetricsReply> primaryMetrics =
 			    brokenPromiseToNever(primaryLog.get().getQueuingMetrics.getReply(TLogQueuingMetricsRequest()));
-			state Future<TLogQueuingMetricsReply> remoteMetrics =
+			Future<TLogQueuingMetricsReply> remoteMetrics =
 			    brokenPromiseToNever(remoteLog.get().getQueuingMetrics.getReply(TLogQueuingMetricsRequest()));
-			state Future<GetSSVersionLagReply> ssVersionLagReply = brokenPromiseToNever(
+			Future<GetSSVersionLagReply> ssVersionLagReply = brokenPromiseToNever(
 			    self->db.serverInfo->get().ratekeeper.get().getSSVersionLag.getReply(GetSSVersionLagRequest()));
 
-			wait((success(primaryMetrics) && success(remoteMetrics) && success(ssVersionLagReply)) || onChange);
+			co_await ((success(primaryMetrics) && success(remoteMetrics) && success(ssVersionLagReply)) || onChange);
 			if (onChange.isReady()) {
 				break;
 			}
@@ -2113,7 +2186,7 @@ ACTOR Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self
 				}
 			}
 
-			wait(delay(SERVER_KNOBS->VERSION_LAG_METRIC_INTERVAL) || onChange);
+			co_await (delay(SERVER_KNOBS->VERSION_LAG_METRIC_INTERVAL) || onChange);
 			if (onChange.isReady()) {
 				break;
 			}
@@ -2123,10 +2196,10 @@ ACTOR Future<Void> updateDatacenterVersionDifference(ClusterControllerData* self
 
 // A background actor that periodically checks remote DC health, and `checkOutstandingRequests` if remote DC
 // recovers.
-ACTOR Future<Void> updateRemoteDCHealth(ClusterControllerData* self) {
+Future<Void> updateRemoteDCHealth(ClusterControllerData* self) {
 	// The purpose of the initial delay is to wait for the cluster to achieve a steady state before checking remote
 	// DC health, since remote DC healthy may trigger a failover, and we don't want that to happen too frequently.
-	wait(delay(SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY));
+	co_await delay(SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY);
 
 	self->remoteDCMonitorStarted = true;
 
@@ -2136,40 +2209,88 @@ ACTOR Future<Void> updateRemoteDCHealth(ClusterControllerData* self) {
 		checkOutstandingRequests(self);
 	}
 
-	loop {
+	while (true) {
 		bool oldRemoteTransactionSystemDegraded = self->remoteTransactionSystemDegraded;
 		self->remoteTransactionSystemDegraded = self->remoteTransactionSystemContainsDegradedServers();
 
 		if (oldRemoteTransactionSystemDegraded && !self->remoteTransactionSystemDegraded) {
 			checkOutstandingRequests(self);
 		}
-		wait(delay(SERVER_KNOBS->CHECK_REMOTE_HEALTH_INTERVAL));
+		co_await delay(SERVER_KNOBS->CHECK_REMOTE_HEALTH_INTERVAL);
 	}
 }
 
-ACTOR Future<Void> doEmptyCommit(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> monitorRatekeeperTpsLimit(ClusterControllerData* self) {
+	while (true) {
+		try {
+			co_await delay(1.0);
+
+			if (self->db.config.usableRegions <= 1 ||
+			    self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+				self->ratekeeperMonitor.resetZeroRatekeeperTpsLimitObservation();
+				continue;
+			}
+
+			HealthMetrics healthMetrics = co_await self->db.db->getHealthMetrics(/*detailed=*/false);
+			if (!self->ratekeeperMonitor.hasSustainedZeroRatekeeperTpsLimit(healthMetrics.tpsLimit)) {
+				continue;
+			}
+
+			if (self->canSafelyTriggerFailoverToRemoteDc()) {
+				if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER &&
+				    self->triggerFailoverToRemoteDc("RatekeeperZeroTpsLimit", healthMetrics.tpsLimit)) {
+					CODE_PROBE(true, "Ratekeeper zero TPS limit triggered region failover", probe::decoration::rare);
+					self->ratekeeperMonitor.resetZeroRatekeeperTpsLimitObservation();
+				} else if (!SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER) {
+					TraceEvent(SevWarn, "RatekeeperZeroTpsLimitSuggestFailover", self->id)
+					    .suppressFor(std::max(1.0, SERVER_KNOBS->CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION))
+					    .detail("TPSLimit", healthMetrics.tpsLimit)
+					    .detail("ZeroTpsDuration", self->ratekeeperMonitor.getZeroRatekeeperTpsLimitDuration());
+				}
+			} else {
+				TraceEvent(SevWarn, "RatekeeperZeroTpsLimitSuggestFailover", self->id)
+				    .suppressFor(std::max(1.0, SERVER_KNOBS->CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION))
+				    .detail("TPSLimit", healthMetrics.tpsLimit)
+				    .detail("VersionDifferenceUpdated", self->versionDifferenceUpdated)
+				    .detail("DatacenterVersionDifference", self->datacenterVersionDifference)
+				    .detail("RemoteDcHealthy", self->remoteDCIsHealthy())
+				    .detail("ZeroTpsDuration", self->ratekeeperMonitor.getZeroRatekeeperTpsLimitDuration());
+			}
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "MonitorRatekeeperTpsLimitError", self->id).error(e);
+			self->ratekeeperMonitor.resetZeroRatekeeperTpsLimitObservation();
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
+		}
+	}
+}
+
+Future<Void> doEmptyCommit(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.makeSelfConflicting();
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<Void> handleForcedRecoveries(ClusterControllerData* self, ClusterControllerFullInterface interf) {
-	loop {
-		state ForceRecoveryRequest req = waitNext(interf.clientInterface.forceRecovery.getFuture());
+Future<Void> handleForcedRecoveries(ClusterControllerData* self, ClusterControllerFullInterface interf) {
+	while (true) {
+		ForceRecoveryRequest req = co_await interf.clientInterface.forceRecovery.getFuture();
 		TraceEvent("ForcedRecoveryStart", self->id)
 		    .detail("ClusterControllerDcId", self->clusterControllerDcId)
 		    .detail("DcId", req.dcId.printable());
-		state Future<Void> fCommit = doEmptyCommit(self->cx);
-		wait(fCommit || delay(SERVER_KNOBS->FORCE_RECOVERY_CHECK_DELAY));
+		Future<Void> fCommit = doEmptyCommit(self->cx);
+		co_await (fCommit || delay(SERVER_KNOBS->FORCE_RECOVERY_CHECK_DELAY));
 		if (!fCommit.isReady() || fCommit.isError()) {
 			if (self->clusterControllerDcId != req.dcId) {
 				std::vector<Optional<Key>> dcPriority;
@@ -2180,7 +2301,7 @@ ACTOR Future<Void> handleForcedRecoveries(ClusterControllerData* self, ClusterCo
 				self->db.forceRecovery = true;
 				self->db.forceMasterFailure.trigger();
 			}
-			wait(fCommit);
+			co_await fCommit;
 		}
 		TraceEvent("ForcedRecoveryFinish", self->id).log();
 		self->db.forceRecovery = false;
@@ -2188,13 +2309,13 @@ ACTOR Future<Void> handleForcedRecoveries(ClusterControllerData* self, ClusterCo
 	}
 }
 
-ACTOR Future<Void> triggerAuditStorage(ClusterControllerData* self, TriggerAuditRequest req) {
-	state UID auditId;
+Future<Void> triggerAuditStorage(ClusterControllerData* self, TriggerAuditRequest req) {
+	UID auditId;
 	ASSERT(!req.cancel);
 	try {
 		while (self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS ||
 		       !self->db.serverInfo->get().distributor.present()) {
-			wait(self->db.serverInfo->onChange());
+			co_await self->db.serverInfo->onChange();
 		}
 		TraceEvent(SevVerbose, "CCTriggerAuditStorageBegin", self->id)
 		    .detail("Range", req.range)
@@ -2202,7 +2323,7 @@ ACTOR Future<Void> triggerAuditStorage(ClusterControllerData* self, TriggerAudit
 		    .detail("KeyValueStoreType", req.engineType.toString())
 		    .detail("DDId", self->db.serverInfo->get().distributor.get().id());
 		TriggerAuditRequest fReq(req.getType(), req.range, req.engineType);
-		UID auditId_ = wait(self->db.serverInfo->get().distributor.get().triggerAudit.getReply(fReq));
+		UID auditId_ = co_await self->db.serverInfo->get().distributor.get().triggerAudit.getReply(fReq);
 		auditId = auditId_;
 		TraceEvent(SevVerbose, "CCTriggerAuditStorageEnd", self->id)
 		    .detail("AuditID", auditId)
@@ -2219,23 +2340,21 @@ ACTOR Future<Void> triggerAuditStorage(ClusterControllerData* self, TriggerAudit
 		    .detail("KeyValueStoreType", req.engineType.toString());
 		req.reply.sendError(audit_storage_failed());
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> cancelAuditStorage(ClusterControllerData* self, TriggerAuditRequest req) {
+Future<Void> cancelAuditStorage(ClusterControllerData* self, TriggerAuditRequest req) {
 	ASSERT(req.cancel);
 	try {
 		while (self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS ||
 		       !self->db.serverInfo->get().distributor.present()) {
-			wait(self->db.serverInfo->onChange());
+			co_await self->db.serverInfo->onChange();
 		}
 		TraceEvent(SevVerbose, "CCCancelAuditStorageBegin", self->id)
 		    .detail("AuditID", req.id)
 		    .detail("AuditType", req.getType())
 		    .detail("DDId", self->db.serverInfo->get().distributor.get().id());
 		TriggerAuditRequest fReq(req.getType(), req.id);
-		UID auditId = wait(self->db.serverInfo->get().distributor.get().triggerAudit.getReply(fReq));
+		UID auditId = co_await self->db.serverInfo->get().distributor.get().triggerAudit.getReply(fReq);
 		TraceEvent(SevVerbose, "CCCancelAuditStorageEnd", self->id)
 		    .detail("ReturnedAuditID", auditId)
 		    .detail("AuditID", auditId)
@@ -2249,13 +2368,11 @@ ACTOR Future<Void> cancelAuditStorage(ClusterControllerData* self, TriggerAuditR
 		    .detail("AuditType", req.getType());
 		req.reply.sendError(cancel_audit_storage_failed());
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> handleTriggerAuditStorage(ClusterControllerData* self, ClusterControllerFullInterface interf) {
-	loop {
-		TriggerAuditRequest req = waitNext(interf.clientInterface.triggerAudit.getFuture());
+Future<Void> handleTriggerAuditStorage(ClusterControllerData* self, ClusterControllerFullInterface interf) {
+	while (true) {
+		TriggerAuditRequest req = co_await interf.clientInterface.triggerAudit.getFuture();
 		TraceEvent(SevVerbose, "CCTriggerAuditStorageReceived", self->id)
 		    .detail("ClusterControllerDcId", self->clusterControllerDcId)
 		    .detail("Range", req.range)
@@ -2270,23 +2387,23 @@ ACTOR Future<Void> handleTriggerAuditStorage(ClusterControllerData* self, Cluste
 	}
 }
 
-ACTOR Future<Void> startDataDistributor(ClusterControllerData* self, double waitTime) {
+Future<Void> startDataDistributor(ClusterControllerData* self, double waitTime) {
 	// If master fails at the same time, give it a chance to clear master PID.
 	// Also wait to avoid too many consecutive recruits in a small time window.
-	wait(delay(waitTime));
+	co_await delay(waitTime);
 
 	TraceEvent("CCStartDataDistributor", self->id).log();
-	loop {
+	while (true) {
 		try {
-			state bool noDistributor = !self->db.serverInfo->get().distributor.present();
+			bool noDistributor = !self->db.serverInfo->get().distributor.present();
 			while (!self->masterProcessId.present() ||
 			       self->masterProcessId != self->db.serverInfo->get().master.locality.processId() ||
 			       self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-				wait(self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
+				co_await (self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
 			}
 			if (noDistributor && self->db.serverInfo->get().distributor.present()) {
 				// Existing distributor registers while waiting, so skip.
-				return Void();
+				co_return;
 			}
 
 			auto idUsed = self->getUsedIds();
@@ -2296,7 +2413,7 @@ ACTOR Future<Void> startDataDistributor(ClusterControllerData* self, double wait
 			                                                                self->db.config,
 			                                                                idUsed);
 			InitializeDataDistributorRequest req(deterministicRandom()->randomUniqueID());
-			state WorkerDetails worker = ddWorker.worker;
+			WorkerDetails worker = ddWorker.worker;
 			if (self->onMasterIsBetter(worker, ProcessClass::DataDistributor)) {
 				worker = self->id_worker[self->masterProcessId.get()].details;
 			}
@@ -2306,8 +2423,8 @@ ACTOR Future<Void> startDataDistributor(ClusterControllerData* self, double wait
 			    .detail("Addr", worker.interf.address())
 			    .detail("DDID", req.reqId);
 
-			ErrorOr<DataDistributorInterface> ddInterf = wait(worker.interf.dataDistributor.getReplyUnlessFailedFor(
-			    req, SERVER_KNOBS->WAIT_FOR_DISTRIBUTOR_JOIN_DELAY, 0));
+			ErrorOr<DataDistributorInterface> ddInterf = co_await worker.interf.dataDistributor.getReplyUnlessFailedFor(
+			    req, SERVER_KNOBS->WAIT_FOR_DISTRIBUTOR_JOIN_DELAY, 0);
 
 			if (ddInterf.present()) {
 				self->recruitDistributor.set(false);
@@ -2329,7 +2446,7 @@ ACTOR Future<Void> startDataDistributor(ClusterControllerData* self, double wait
 					self->db.setDistributor(ddInterf.get());
 				}
 				checkOutstandingRequests(self);
-				return Void();
+				co_return;
 			}
 		} catch (Error& e) {
 			TraceEvent("CCDataDistributorRecruitError", self->id).error(e);
@@ -2337,7 +2454,7 @@ ACTOR Future<Void> startDataDistributor(ClusterControllerData* self, double wait
 				throw;
 			}
 		}
-		wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+		co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 	}
 }
 
@@ -2371,23 +2488,23 @@ ACTOR Future<Void> monitorDataDistributor(ClusterControllerData* self) {
 	}
 }
 
-ACTOR Future<Void> startRatekeeper(ClusterControllerData* self, double waitTime) {
+Future<Void> startRatekeeper(ClusterControllerData* self, double waitTime) {
 	// If master fails at the same time, give it a chance to clear master PID.
 	// Also wait to avoid too many consecutive recruits in a small time window.
-	wait(delay(waitTime));
+	co_await delay(waitTime);
 
 	TraceEvent("CCStartRatekeeper", self->id).log();
-	loop {
+	while (true) {
 		try {
-			state bool no_ratekeeper = !self->db.serverInfo->get().ratekeeper.present();
+			bool no_ratekeeper = !self->db.serverInfo->get().ratekeeper.present();
 			while (!self->masterProcessId.present() ||
 			       self->masterProcessId != self->db.serverInfo->get().master.locality.processId() ||
 			       self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-				wait(self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
+				co_await (self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
 			}
 			if (no_ratekeeper && self->db.serverInfo->get().ratekeeper.present()) {
 				// Existing ratekeeper registers while waiting, so skip.
-				return Void();
+				co_return;
 			}
 
 			ClusterControllerData::WorkerUsages id_used = self->getUsedIds();
@@ -2397,7 +2514,7 @@ ACTOR Future<Void> startRatekeeper(ClusterControllerData* self, double waitTime)
 			                                                                self->db.config,
 			                                                                id_used);
 			InitializeRatekeeperRequest req(deterministicRandom()->randomUniqueID());
-			state WorkerDetails worker = rkWorker.worker;
+			WorkerDetails worker = rkWorker.worker;
 			if (self->onMasterIsBetter(worker, ProcessClass::Ratekeeper)) {
 				worker = self->id_worker[self->masterProcessId.get()].details;
 			}
@@ -2407,8 +2524,8 @@ ACTOR Future<Void> startRatekeeper(ClusterControllerData* self, double waitTime)
 			    .detail("Addr", worker.interf.address())
 			    .detail("RKID", req.reqId);
 
-			ErrorOr<RatekeeperInterface> interf = wait(
-			    worker.interf.ratekeeper.getReplyUnlessFailedFor(req, SERVER_KNOBS->WAIT_FOR_RATEKEEPER_JOIN_DELAY, 0));
+			ErrorOr<RatekeeperInterface> interf = co_await worker.interf.ratekeeper.getReplyUnlessFailedFor(
+			    req, SERVER_KNOBS->WAIT_FOR_RATEKEEPER_JOIN_DELAY, 0);
 			if (interf.present()) {
 				self->recruitRatekeeper.set(false);
 				self->recruitingRatekeeperID = interf.get().id();
@@ -2427,7 +2544,7 @@ ACTOR Future<Void> startRatekeeper(ClusterControllerData* self, double waitTime)
 					self->db.setRatekeeper(interf.get());
 				}
 				checkOutstandingRequests(self);
-				return Void();
+				co_return;
 			}
 		} catch (Error& e) {
 			TraceEvent("CCRatekeeperRecruitError", self->id).error(e);
@@ -2435,7 +2552,7 @@ ACTOR Future<Void> startRatekeeper(ClusterControllerData* self, double waitTime)
 				throw;
 			}
 		}
-		wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+		co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 	}
 }
 
@@ -2463,20 +2580,20 @@ ACTOR Future<Void> monitorRatekeeper(ClusterControllerData* self) {
 	}
 }
 
-ACTOR Future<Void> startConsistencyScan(ClusterControllerData* self) {
-	wait(delay(0.0)); // If master fails at the same time, give it a chance to clear master PID.
+Future<Void> startConsistencyScan(ClusterControllerData* self) {
+	co_await delay(0.0); // If master fails at the same time, give it a chance to clear master PID.
 	TraceEvent("CCStartConsistencyScan", self->id).log();
-	loop {
+	while (true) {
 		try {
-			state bool no_consistencyScan = !self->db.serverInfo->get().consistencyScan.present();
+			bool no_consistencyScan = !self->db.serverInfo->get().consistencyScan.present();
 			while (!self->masterProcessId.present() ||
 			       self->masterProcessId != self->db.serverInfo->get().master.locality.processId() ||
 			       self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-				wait(self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
+				co_await (self->db.serverInfo->onChange() || delay(SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY));
 			}
 			if (no_consistencyScan && self->db.serverInfo->get().consistencyScan.present()) {
 				// Existing consistencyScan registers while waiting, so skip.
-				return Void();
+				co_return;
 			}
 
 			auto id_used = self->getUsedIds();
@@ -2487,7 +2604,7 @@ ACTOR Future<Void> startConsistencyScan(ClusterControllerData* self) {
 			                                                                id_used);
 
 			InitializeConsistencyScanRequest req(deterministicRandom()->randomUniqueID());
-			state WorkerDetails worker = csWorker.worker;
+			WorkerDetails worker = csWorker.worker;
 			if (self->onMasterIsBetter(worker, ProcessClass::ConsistencyScan)) {
 				worker = self->id_worker[self->masterProcessId.get()].details;
 			}
@@ -2497,8 +2614,8 @@ ACTOR Future<Void> startConsistencyScan(ClusterControllerData* self) {
 			    .detail("Addr", worker.interf.address())
 			    .detail("CSID", req.reqId);
 
-			ErrorOr<ConsistencyScanInterface> interf = wait(worker.interf.consistencyScan.getReplyUnlessFailedFor(
-			    req, SERVER_KNOBS->WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY, 0));
+			ErrorOr<ConsistencyScanInterface> interf = co_await worker.interf.consistencyScan.getReplyUnlessFailedFor(
+			    req, SERVER_KNOBS->WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY, 0);
 			if (interf.present()) {
 				self->recruitConsistencyScan.set(false);
 				self->recruitingConsistencyScanID = interf.get().id();
@@ -2517,7 +2634,7 @@ ACTOR Future<Void> startConsistencyScan(ClusterControllerData* self) {
 					self->db.setConsistencyScan(interf.get());
 				}
 				checkOutstandingRequests(self);
-				return Void();
+				co_return;
 			} else {
 				TraceEvent("CCConsistencyScanRecruitEmpty", self->id).log();
 			}
@@ -2527,7 +2644,7 @@ ACTOR Future<Void> startConsistencyScan(ClusterControllerData* self) {
 				throw;
 			}
 		}
-		wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+		co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 	}
 }
 
@@ -2557,34 +2674,6 @@ ACTOR Future<Void> monitorConsistencyScan(ClusterControllerData* self) {
 	}
 }
 
-ACTOR Future<Void> monitorEncryptKeyProxy(ClusterControllerData* self) {
-	state EncryptionAtRestModeDeprecated encryptMode = wait(self->encryptionAtRestModeDeprecated.getFuture());
-	if (!encryptMode.isEncryptionEnabled()) {
-		TraceEvent("EKPNotConfigured");
-		return Void();
-	} else {
-		TraceEvent(SevError, "WeBoguslyThinkEKPIsConfigured");
-		return Void();
-	}
-}
-
-ACTOR Future<Void> stopConsistencyScan(Database db) {
-	state ConsistencyScanState cs = ConsistencyScanState();
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	loop {
-		try {
-			SystemDBWriteLockedNow(db.getReference())->setOptions(tr);
-			state ConsistencyScanState::Config config = wait(ConsistencyScanState().config().getD(tr));
-			config.enabled = false;
-			cs.config().set(tr, config);
-			wait(tr->commit());
-			return Void();
-		} catch (Error& e) {
-			wait(tr->onError(e));
-		}
-	}
-}
-
 ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 	state Future<Void> dbInfoChange = self->db.serverInfo->onChange();
 	state Future<Void> updateDBInfo = self->updateDBInfo.onTrigger();
@@ -2652,11 +2741,11 @@ static void invalidateExcludedProcessComplaints(ClusterControllerData* self) {
 }
 
 // The actor that periodically monitors the health of tracked workers.
-ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
-	loop {
+Future<Void> workerHealthMonitor(ClusterControllerData* self) {
+	while (true) {
 		try {
 			while (!self->goodRecruitmentTime.isReady()) {
-				wait(lowPriorityDelay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL));
+				co_await lowPriorityDelay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL);
 			}
 			self->degradationInfo = self->getDegradationInfo();
 
@@ -2719,22 +2808,18 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 						TraceEvent(SevWarn, "DegradedServerDetectedAndSuggestRecovery").log();
 					}
 				} else if (self->shouldTriggerFailoverDueToDegradedServers()) {
-					double ccUpTime = now() - machineStartTime();
-					if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER &&
-					    ccUpTime > SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
+					const bool canSafelyFailover = self->canSafelyTriggerFailoverToRemoteDc();
+					const double ccUpTime = machineStartTime() == 0 ? 0.0 : now() - machineStartTime();
+					if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER && canSafelyFailover) {
 						TraceEvent(SevWarn, "DegradedServerDetectedAndTriggerFailover").log();
-						std::vector<Optional<Key>> dcPriority;
-						auto remoteDcId = self->db.config.regions[0].dcId == self->clusterControllerDcId.get()
-						                      ? self->db.config.regions[1].dcId
-						                      : self->db.config.regions[0].dcId;
-
-						// Switch the current primary DC and remote DC in desiredDcIds, so that the remote DC
-						// becomes the new primary, and the primary DC becomes the new remote.
-						dcPriority.push_back(remoteDcId);
-						dcPriority.push_back(self->clusterControllerDcId);
-						self->desiredDcIds.set(dcPriority);
+						self->triggerFailoverToRemoteDc("DegradedServers");
 					} else {
-						TraceEvent(SevWarn, "DegradedServerDetectedAndSuggestFailover").detail("CCUpTime", ccUpTime);
+						TraceEvent(SevWarn, "DegradedServerDetectedAndSuggestFailover")
+						    .detail("CCUpTime", ccUpTime)
+						    .detail("CanSafelyFailover", canSafelyFailover)
+						    .detail("VersionDifferenceUpdated", self->versionDifferenceUpdated)
+						    .detail("DatacenterVersionDifference", self->datacenterVersionDifference)
+						    .detail("RemoteDcHealthy", self->remoteDCIsHealthy());
 					}
 				}
 			}
@@ -2745,7 +2830,7 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 				checkOutstandingRequests(self);
 			}
 
-			wait(delay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL));
+			co_await delay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL);
 		} catch (Error& e) {
 			TraceEvent(SevWarnAlways, "ClusterControllerHealthMonitorError").error(e);
 			if (e.code() == error_code_actor_cancelled) {
@@ -2757,15 +2842,16 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 
 // Update the DBInfo state with this processes cluster ID. If this process does
 // not have a cluster ID and one does not exist in the database, generate one.
-ACTOR Future<Void> updateClusterId(ClusterControllerData* self) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
-	loop {
+Future<Void> updateClusterId(ClusterControllerData* self) {
+	auto tr = makeReference<ReadYourWritesTransaction>(self->cx);
+	while (true) {
+		Error err;
 		try {
-			state Optional<UID> durableClusterId = self->clusterId->get();
+			Optional<UID> durableClusterId = self->clusterId->get();
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			Optional<Value> clusterIdVal = wait(tr->get(clusterIdKey));
+			Optional<Value> clusterIdVal = co_await tr->get(clusterIdKey);
 
 			if (clusterIdVal.present()) {
 				UID clusterId = BinaryReader::fromStringRef<UID>(clusterIdVal.get(), IncludeVersion());
@@ -2784,7 +2870,7 @@ ACTOR Future<Void> updateClusterId(ClusterControllerData* self) {
 				ASSERT(!durableClusterId.present());
 				durableClusterId = deterministicRandom()->randomUniqueID();
 				tr->set(clusterIdKey, BinaryWriter::toValue(durableClusterId.get(), IncludeVersion()));
-				wait(tr->commit());
+				co_await tr->commit();
 			}
 			auto serverInfo = self->db.serverInfo->get();
 			if (!serverInfo.client.clusterId.isValid()) {
@@ -2798,18 +2884,19 @@ ACTOR Future<Void> updateClusterId(ClusterControllerData* self) {
 				clientInfo.clusterId = durableClusterId.get();
 				self->db.clientInfo->set(clientInfo);
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
-ACTOR Future<Void> handleGetEncryptionAtRestMode(ClusterControllerData* self, ClusterControllerFullInterface ccInterf) {
-	loop {
-		state GetEncryptionAtRestModeRequest req = waitNext(ccInterf.getEncryptionAtRestMode.getFuture());
+Future<Void> handleGetEncryptionAtRestMode(ClusterControllerData* self, ClusterControllerFullInterface ccInterf) {
+	while (true) {
+		GetEncryptionAtRestModeRequest req = co_await ccInterf.getEncryptionAtRestMode.getFuture();
 		TraceEvent("HandleGetEncryptionAtRestModeStart").detail("TlogId", req.tlogId);
-		EncryptionAtRestModeDeprecated mode = wait(self->encryptionAtRestModeDeprecated.getFuture());
+		EncryptionAtRestModeDeprecated mode = co_await self->encryptionAtRestModeDeprecated.getFuture();
 		GetEncryptionAtRestModeResponse resp;
 		resp.mode = mode;
 		req.reply.send(resp);
@@ -2827,8 +2914,6 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	state uint64_t step = 0;
 	state Future<ErrorOr<Void>> error = errorOr(actorCollection(self.addActor.getFuture()));
 
-	// EncryptKeyProxy is necessary for TLog recovery, recruit it as the first process
-	self.addActor.send(monitorEncryptKeyProxy(&self));
 	self.addActor.send(clusterWatchDatabase(&self, &self.db, coordinators)); // Start the master database
 	self.addActor.send(self.updateWorkerList.init(self.db.db));
 	self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(), &self, coordinators));
@@ -2840,12 +2925,16 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	self.addActor.send(updatedChangingDatacenters(&self));
 	self.addActor.send(updatedChangedDatacenters(&self));
 	self.addActor.send(updateDatacenterVersionDifference(&self));
+	if (SERVER_KNOBS->CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION > 0) {
+		self.addActor.send(monitorRatekeeperTpsLimit(&self));
+	}
 	self.addActor.send(handleForcedRecoveries(&self, interf));
 	self.addActor.send(handleTriggerAuditStorage(&self, interf));
 	self.addActor.send(monitorDataDistributor(&self));
 	self.addActor.send(monitorRatekeeper(&self));
 
 	self.addActor.send(monitorConsistencyScan(&self));
+	self.addActor.send(self.clusterHealthMonitor.run());
 	self.addActor.send(dbInfoUpdater(&self));
 	self.addActor.send(updateClusterId(&self));
 	self.addActor.send(handleGetEncryptionAtRestMode(&self, interf));
@@ -2946,13 +3035,13 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	}
 }
 
-ACTOR Future<Void> replaceInterface(ClusterControllerFullInterface interf) {
-	loop {
+Future<Void> replaceInterface(ClusterControllerFullInterface interf) {
+	while (true) {
 		if (interf.hasMessage()) {
-			wait(delay(SERVER_KNOBS->REPLACE_INTERFACE_DELAY));
-			return Void();
+			co_await delay(SERVER_KNOBS->REPLACE_INTERFACE_DELAY);
+			co_return;
 		}
-		wait(delay(SERVER_KNOBS->REPLACE_INTERFACE_CHECK_DELAY));
+		co_await delay(SERVER_KNOBS->REPLACE_INTERFACE_CHECK_DELAY);
 	}
 }
 
@@ -3009,16 +3098,16 @@ ACTOR Future<Void> clusterController(ServerCoordinators coordinators,
 	}
 }
 
-ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> connRecord,
-                                     Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
-                                     Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
-                                     LocalityData locality,
-                                     Reference<AsyncVar<Optional<UID>>> clusterId) {
-	state bool hasConnected = false;
-	loop {
+Future<Void> clusterController(Reference<IClusterConnectionRecord> connRecord,
+                               Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
+                               Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
+                               LocalityData locality,
+                               Reference<AsyncVar<Optional<UID>>> clusterId) {
+	bool hasConnected = false;
+	while (true) {
 		try {
 			ServerCoordinators coordinators(connRecord);
-			wait(clusterController(coordinators, currentCC, hasConnected, asyncPriorityInfo, locality, clusterId));
+			co_await clusterController(coordinators, currentCC, hasConnected, asyncPriorityInfo, locality, clusterId);
 			hasConnected = true;
 		} catch (Error& e) {
 			if (e.code() != error_code_coordinators_changed)
diff --git a/fdbserver/include/fdbserver/ClusterController.actor.h b/fdbserver/clustercontroller/ClusterController.h
similarity index 96%
rename from fdbserver/include/fdbserver/ClusterController.actor.h
rename to fdbserver/clustercontroller/ClusterController.h
index cf6094ec41e..d7f2ebf88d7 100644
--- a/fdbserver/include/fdbserver/ClusterController.actor.h
+++ b/fdbserver/clustercontroller/ClusterController.h
@@ -1,5 +1,5 @@
 /*
- * ClusterController.actor.h
+ * ClusterController.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,6 @@
  * limitations under the License.
  */
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERCONTROLLER_ACTOR_G_H)
-#define FDBSERVER_CLUSTERCONTROLLER_ACTOR_G_H
-#include "fdbserver/ClusterController.actor.g.h"
-#elif !defined(FDBSERVER_CLUSTERCONTROLLER_ACTOR_H)
-#define FDBSERVER_CLUSTERCONTROLLER_ACTOR_H
 #pragma once
 
 #include <utility>
@@ -33,14 +26,15 @@
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbrpc/Replication.h"
 #include "fdbrpc/ReplicationUtils.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "ClusterHealthMonitor.h"
+#include "RatekeeperMonitor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "fdbrpc/Locality.h"
+#include "flow/CoroUtils.h"
 #include "flow/NetworkAddress.h"
 #include "flow/SystemMonitor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 struct WorkerInfo : NonCopyable {
 	Future<Void> watcher;
 	ReplyPromise<RegisterWorkerReply> reply;
@@ -70,7 +64,7 @@ struct WorkerInfo : NonCopyable {
 	  : watcher(watcher), reply(reply), gen(gen), reboots(0), initialClass(initialClass), priorityInfo(priorityInfo),
 	    details(interf, processClass, degraded, recoveredDiskFiles), issues(issues) {}
 
-	WorkerInfo(WorkerInfo&& r) noexcept
+	explicit(false) WorkerInfo(WorkerInfo&& r) noexcept
 	  : watcher(std::move(r.watcher)), reply(std::move(r.reply)), gen(r.gen), reboots(r.reboots),
 	    initialClass(r.initialClass), priorityInfo(r.priorityInfo), details(std::move(r.details)),
 	    haltRatekeeper(r.haltRatekeeper), haltDistributor(r.haltDistributor),
@@ -99,13 +93,21 @@ struct WorkerFitnessInfo {
 	  : worker(worker), fitness(fitness), used(used) {}
 };
 
+inline bool hasWorkerIssue(const Standalone<VectorRef<StringRef>>& issues, StringRef issue) {
+	return std::find(issues.begin(), issues.end(), issue) != issues.end();
+}
+
+inline bool excludesFromTLogRecruitmentDueToLowDisk(const Standalone<VectorRef<StringRef>>& issues) {
+	return hasWorkerIssue(issues, "exclude_from_tlog_recruitment_low_disk"_sr);
+}
+
 struct RecruitWorkersInfo : ReferenceCounted<RecruitWorkersInfo> {
 	RecruitFromConfigurationRequest req;
 	RecruitFromConfigurationReply rep;
 	AsyncTrigger waitForCompletion;
 	Optional<UID> dbgId;
 
-	RecruitWorkersInfo(RecruitFromConfigurationRequest const& req) : req(req) {}
+	explicit RecruitWorkersInfo(RecruitFromConfigurationRequest const& req) : req(req) {}
 };
 
 struct RecruitRemoteWorkersInfo : ReferenceCounted<RecruitRemoteWorkersInfo> {
@@ -114,7 +116,7 @@ struct RecruitRemoteWorkersInfo : ReferenceCounted<RecruitRemoteWorkersInfo> {
 	AsyncTrigger waitForCompletion;
 	Optional<UID> dbgId;
 
-	RecruitRemoteWorkersInfo(RecruitRemoteFromConfigurationRequest const& req) : req(req) {}
+	explicit RecruitRemoteWorkersInfo(RecruitRemoteFromConfigurationRequest const& req) : req(req) {}
 };
 
 struct ClusterRecoveryData;
@@ -197,9 +199,9 @@ class ClusterControllerData {
 			clientInfo->set(newClientInfo);
 		}
 
-		ACTOR static Future<Void> countClients(DBInfo* self) {
-			loop {
-				wait(delay(SERVER_KNOBS->CC_PRUNE_CLIENTS_INTERVAL));
+		static Future<Void> countClients(DBInfo* self) {
+			while (true) {
+				co_await delay(SERVER_KNOBS->CC_PRUNE_CLIENTS_INTERVAL);
 
 				self->clientCount = 0;
 				for (auto itr = self->clientStatus.begin(); itr != self->clientStatus.end();) {
@@ -226,36 +228,40 @@ class ClusterControllerData {
 		std::map<Optional<Standalone<StringRef>>, Optional<ProcessData>> delta;
 		AsyncVar<bool> anyDelta;
 
-		ACTOR static Future<Void> update(UpdateWorkerList* self, Database db) {
+		static Future<Void> update(UpdateWorkerList* self, Database db) {
 			// The Database we are using is based on worker registrations to this cluster controller, which come only
 			// from master servers that we started, so it shouldn't be possible for multiple cluster controllers to
 			// fight.
-			state Transaction tr(db);
-			loop {
+			Transaction tr(db);
+			while (true) {
+				Error err;
 				try {
 					tr.clear(workerListKeys);
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 
-			loop {
+			while (true) {
 				tr.reset();
 
 				// Wait for some changes
-				while (!self->anyDelta.get())
-					wait(self->anyDelta.onChange());
+				while (!self->anyDelta.get()) {
+					co_await self->anyDelta.onChange();
+				}
 				self->anyDelta.set(false);
 
-				state std::map<Optional<Standalone<StringRef>>, Optional<ProcessData>> delta;
+				std::map<Optional<Standalone<StringRef>>, Optional<ProcessData>> delta;
 				delta.swap(self->delta);
 
 				TraceEvent("UpdateWorkerList").detail("DeltaCount", delta.size());
 
 				// Do a transaction to write the changes
-				loop {
+				while (true) {
+					Error err;
 					try {
 						for (auto w = delta.begin(); w != delta.end(); ++w) {
 							if (w->second.present()) {
@@ -263,11 +269,12 @@ class ClusterControllerData {
 							} else
 								tr.clear(workerListKeyFor(w->first.get()));
 						}
-						wait(tr.commit());
+						co_await tr.commit();
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 				}
 			}
 		}
@@ -386,7 +393,7 @@ class ClusterControllerData {
 	    Optional<Optional<Standalone<StringRef>>> const& dcId = Optional<Optional<Standalone<StringRef>>>()) {
 		std::map<ProcessClass::Fitness, std::vector<WorkerDetails>> fitness_workers;
 		std::vector<WorkerDetails> results;
-		Reference<LocalitySet> logServerSet = Reference<LocalitySet>(new LocalityMap<WorkerDetails>());
+		Reference<LocalitySet> logServerSet = makeReference<LocalityMap<WorkerDetails>>();
 		LocalityMap<WorkerDetails>* logServerMap = (LocalityMap<WorkerDetails>*)logServerSet.getPtr();
 		bool bCompleted = false;
 
@@ -660,6 +667,16 @@ class ClusterControllerData {
 				    SevDebug, id, "complex", "Worker is not in the target DC", worker_details, fitness, dcIds);
 				continue;
 			}
+			if (excludesFromTLogRecruitmentDueToLowDisk(worker_info.issues)) {
+				logWorkerUnavailable(SevInfo,
+				                     id,
+				                     "complex",
+				                     "Worker is excluded from TLog recruitment due to low disk",
+				                     worker_details,
+				                     fitness,
+				                     dcIds);
+				continue;
+			}
 			if (!allowDegraded && worker_details.degraded) {
 				logWorkerUnavailable(
 				    SevInfo, id, "complex", "Worker is degraded and not allowed", worker_details, fitness, dcIds);
@@ -913,6 +930,16 @@ class ClusterControllerData {
 				    SevDebug, id, "simple", "Worker is not in the target DC", worker_details, fitness, dcIds);
 				continue;
 			}
+			if (excludesFromTLogRecruitmentDueToLowDisk(worker_info.issues)) {
+				logWorkerUnavailable(SevInfo,
+				                     id,
+				                     "simple",
+				                     "Worker is excluded from TLog recruitment due to low disk",
+				                     worker_details,
+				                     fitness,
+				                     dcIds);
+				continue;
+			}
 
 			// This worker is a candidate for TLog recruitment.
 			bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId;
@@ -1011,7 +1038,7 @@ class ClusterControllerData {
 		std::map<std::tuple<ProcessClass::Fitness, int, bool, bool, unsigned>, std::vector<WorkerDetails>>
 		    fitness_workers;
 		std::vector<WorkerDetails> results;
-		Reference<LocalitySet> logServerSet = Reference<LocalitySet>(new LocalityMap<WorkerDetails>());
+		Reference<LocalitySet> logServerSet = makeReference<LocalityMap<WorkerDetails>>();
 		LocalityMap<WorkerDetails>* logServerMap = (LocalityMap<WorkerDetails>*)logServerSet.getPtr();
 		bool bCompleted = false;
 		desired = std::max(required, desired);
@@ -1066,6 +1093,16 @@ class ClusterControllerData {
 				    SevDebug, id, "deprecated", "Worker is not in the target DC", worker_details, fitness, dcIds);
 				continue;
 			}
+			if (excludesFromTLogRecruitmentDueToLowDisk(worker_info.issues)) {
+				logWorkerUnavailable(SevInfo,
+				                     id,
+				                     "deprecated",
+				                     "Worker is excluded from TLog recruitment due to low disk",
+				                     worker_details,
+				                     fitness,
+				                     dcIds);
+				continue;
+			}
 
 			// This worker is a candidate for TLog recruitment.
 			bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId;
@@ -1374,7 +1411,7 @@ class ClusterControllerData {
 	                                                      bool& satelliteFallback,
 	                                                      bool checkStable = false) {
 		int startDC = 0;
-		loop {
+		while (true) {
 			if (startDC > 0 && startDC >= region.satellites.size() + 1 -
 			                                  (satelliteFallback ? region.satelliteTLogUsableDcsFallback
 			                                                     : region.satelliteTLogUsableDcs)) {
@@ -3231,6 +3268,76 @@ class ClusterControllerData {
 		return !remoteTransactionSystemContainsDegradedServers();
 	}
 
+	// Returns true if the cluster controller can safely trigger a failover to the other region.
+	bool canSafelyTriggerFailoverToRemoteDc() {
+		if (db.config.usableRegions <= 1 || db.config.regions.size() < 2 || !clusterControllerDcId.present()) {
+			return false;
+		}
+
+		if (machineStartTime() == 0 || now() - machineStartTime() < SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
+			return false;
+		}
+
+		if (db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			return false;
+		}
+
+		auto ccWorker = id_worker.find(clusterControllerProcessId);
+		if (ccWorker == id_worker.end() || ccWorker->second.priorityInfo.isExcluded) {
+			return false;
+		}
+
+		const bool clusterControllerInPrimary = db.config.regions[0].dcId == clusterControllerDcId.get();
+		const bool clusterControllerInRemote = db.config.regions[1].dcId == clusterControllerDcId.get();
+		if (!clusterControllerInPrimary && !clusterControllerInRemote) {
+			return false;
+		}
+
+		const int targetRegionIndex = clusterControllerInPrimary ? 1 : 0;
+		if (db.config.regions[targetRegionIndex].priority < 0) {
+			return false;
+		}
+
+		return versionDifferenceUpdated && datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE &&
+		       remoteDCIsHealthy();
+	}
+
+	bool triggerFailoverToRemoteDc(const char* reason, Optional<double> tpsLimit = Optional<double>()) {
+		if (db.config.usableRegions <= 1 || db.config.regions.size() < 2 || !clusterControllerDcId.present()) {
+			return false;
+		}
+
+		Optional<Key> remoteDcId;
+		if (db.config.regions[0].dcId == clusterControllerDcId.get()) {
+			remoteDcId = db.config.regions[1].dcId;
+		} else if (db.config.regions[1].dcId == clusterControllerDcId.get()) {
+			remoteDcId = db.config.regions[0].dcId;
+		} else {
+			return false;
+		}
+
+		std::vector<Optional<Key>> dcPriority;
+		dcPriority.push_back(remoteDcId);
+		dcPriority.push_back(clusterControllerDcId);
+
+		if (desiredDcIds.get().present() && desiredDcIds.get().get() == dcPriority) {
+			return false;
+		}
+
+		TraceEvent(SevWarn, "ClusterControllerTriggerFailover", id)
+		    .detail("Reason", reason)
+		    .detail("CurrentClusterControllerDcId", printable(clusterControllerDcId))
+		    .detail("TargetPrimaryDcId", printable(remoteDcId));
+		if (tpsLimit.present()) {
+			TraceEvent(SevInfo, "ClusterControllerTriggerFailoverMetrics", id)
+			    .detail("Reason", reason)
+			    .detail("TPSLimit", tpsLimit.get())
+			    .detail("ZeroTpsDuration", ratekeeperMonitor.getZeroRatekeeperTpsLimitDuration());
+		}
+		desiredDcIds.set(dcPriority);
+		return true;
+	}
+
 	// Returns true when the cluster controller should trigger a recovery due to degraded servers used in the
 	// transaction system in the primary data center.
 	bool shouldTriggerRecoveryDueToDegradedServers() {
@@ -3256,8 +3363,8 @@ class ClusterControllerData {
 		    .detail("TransactionSystemContainsDegradedServers", txnSystemContainsDegradedServers);
 		return txnSystemContainsDegradedServers;
 	}
-	// Returns true when the cluster controller should trigger a failover due to degraded servers used in the
-	// transaction system in the primary data center, and no degradation in the remote data center.
+	// Returns true when degraded servers in the primary transaction system make a remote failover desirable.
+	// Call `canSafelyTriggerFailoverToRemoteDc()` before actually triggering the failover.
 	bool shouldTriggerFailoverDueToDegradedServers() {
 		if (db.config.usableRegions <= 1) {
 			return false;
@@ -3332,6 +3439,8 @@ class ClusterControllerData {
 	std::vector<std::pair<RecruitStorageRequest, double>> outstandingStorageRequests;
 	ActorCollection ac;
 	UpdateWorkerList updateWorkerList;
+	Reference<cluster_health::WorkerEventProvider> clusterHealthWorkerEventProvider;
+	cluster_health::Monitor clusterHealthMonitor;
 	Future<Void> outstandingRequestChecker;
 	Future<Void> outstandingRemoteRequestChecker;
 	AsyncTrigger updateDBInfo;
@@ -3355,6 +3464,7 @@ class ClusterControllerData {
 
 	bool remoteDCMonitorStarted;
 	bool remoteTransactionSystemDegraded;
+	RatekeeperMonitor ratekeeperMonitor;
 
 	// recruitX is used to signal when role X needs to be (re)recruited.
 	// recruitingXID is used to track the ID of X's interface which is being recruited.
@@ -3408,33 +3518,9 @@ class ClusterControllerData {
 	ClusterControllerData(ClusterControllerFullInterface const& ccInterface,
 	                      LocalityData const& locality,
 	                      ServerCoordinators const& coordinators,
-	                      Reference<AsyncVar<Optional<UID>>> clusterId)
-	  : gotProcessClasses(false), gotFullyRecoveredConfig(false), shouldCommitSuicide(false),
-	    clusterControllerProcessId(locality.processId()), clusterControllerDcId(locality.dcId()), id(ccInterface.id()),
-	    clusterId(clusterId), ac(false), outstandingRequestChecker(Void()), outstandingRemoteRequestChecker(Void()),
-	    startTime(now()), goodRecruitmentTime(Never()), goodRemoteRecruitmentTime(Never()),
-	    dcLogServerVersionDifference(0), dcStorageServerVersionDifference(0), datacenterVersionDifference(0),
-	    versionDifferenceUpdated(false), remoteDCMonitorStarted(false), remoteTransactionSystemDegraded(false),
-	    recruitDistributor(false), recruitRatekeeper(false), recruitConsistencyScan(false),
-	    clusterControllerMetrics("ClusterController", id.toString()),
-	    openDatabaseRequests("OpenDatabaseRequests", clusterControllerMetrics),
-	    registerWorkerRequests("RegisterWorkerRequests", clusterControllerMetrics),
-	    getWorkersRequests("GetWorkersRequests", clusterControllerMetrics),
-	    getClientWorkersRequests("GetClientWorkersRequests", clusterControllerMetrics),
-	    registerMasterRequests("RegisterMasterRequests", clusterControllerMetrics),
-	    statusRequests("StatusRequests", clusterControllerMetrics),
-	    recruitedMasterWorkerEventHolder(makeReference<EventCacheHolder>("RecruitedMasterWorker")) {
-		auto serverInfo = ServerDBInfo();
-		serverInfo.id = deterministicRandom()->randomUniqueID();
-		serverInfo.infoGeneration = ++db.dbInfoCount;
-		serverInfo.masterLifetime.ccID = id;
-		serverInfo.clusterInterface = ccInterface;
-		serverInfo.myLocality = locality;
-		db.serverInfo->set(serverInfo);
-		cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::True);
-
-		specialCounter(clusterControllerMetrics, "ClientCount", [this]() { return db.clientCount; });
-	}
+	                      Reference<AsyncVar<Optional<UID>>> clusterId);
+
+	void updateClusterHealthMonitorInputs();
 
 	~ClusterControllerData() {
 		ac.clear(false);
@@ -3443,7 +3529,3 @@ class ClusterControllerData {
 };
 
 void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest const& req);
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbserver/clustercontroller/ClusterHealthIFactor.cpp b/fdbserver/clustercontroller/ClusterHealthIFactor.cpp
new file mode 100644
index 00000000000..28d236d4a3a
--- /dev/null
+++ b/fdbserver/clustercontroller/ClusterHealthIFactor.cpp
@@ -0,0 +1,287 @@
+/*
+ * ClusterHealthIFactor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <limits>
+
+#include "fdbserver/core/RecoveryState.h"
+#include "flow/Trace.h"
+
+#include "ClusterHealthIFactor.h"
+#include "ClusterHealthMonitor.h"
+
+namespace cluster_health {
+
+namespace {
+
+WorkerEvents filterEmptyEvents(WorkerEvents const& events) {
+	WorkerEvents filteredEvents;
+	for (auto const& [address, traceEvent] : events) {
+		if (traceEvent.size() > 0) {
+			filteredEvents.emplace(address, traceEvent);
+		}
+	}
+	return filteredEvents;
+}
+
+Future<Level> fetchSpaceLevel(LatestWorkerEvents eventsAndErrors,
+                              std::string const& availableBytesField,
+                              std::string const& totalBytesField,
+                              double interventionThreshold,
+                              double criticalInterventionThreshold,
+                              char const* failureTraceEventName) {
+	if (!eventsAndErrors.present()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	auto const& [events, _errors] = eventsAndErrors.get();
+	WorkerEvents filteredEvents = filterEmptyEvents(events);
+	if (filteredEvents.empty()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	double minAvailableSpaceRatio = 1.0;
+
+	try {
+		for (auto const& [address, traceEvent] : filteredEvents) {
+			(void)address;
+			double available = traceEvent.getDouble(availableBytesField);
+			double total = std::max(1.0, traceEvent.getDouble(totalBytesField));
+			minAvailableSpaceRatio = std::min(minAvailableSpaceRatio, available / total);
+		}
+
+		if (minAvailableSpaceRatio < criticalInterventionThreshold) {
+			co_return Level::CRITICAL_INTERVENTION_REQUIRED;
+		}
+		if (minAvailableSpaceRatio < interventionThreshold) {
+			co_return Level::INTERVENTION_REQUIRED;
+		}
+		co_return Level::HEALTHY;
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, failureTraceEventName).error(e);
+		co_return Level::METRICS_MISSING;
+	}
+}
+
+} // namespace
+
+StorageSpaceFactor::StorageSpaceFactor(double interventionThreshold, double criticalInterventionThreshold)
+  : interventionThreshold(interventionThreshold), criticalInterventionThreshold(criticalInterventionThreshold) {}
+
+std::string_view StorageSpaceFactor::getName() const {
+	return "StorageSpace";
+}
+
+Future<Level> StorageSpaceFactor::fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider,
+                                             TrackCodeProbes trackCodeProbes) {
+	Level level = co_await fetchSpaceLevel(co_await workerEventProvider->getLatestStorageServerEvents("StorageMetrics"),
+	                                       "KvstoreBytesAvailable",
+	                                       "KvstoreBytesTotal",
+	                                       interventionThreshold,
+	                                       criticalInterventionThreshold,
+	                                       "StorageSpaceFactorFetchFailed");
+	CODE_PROBE(trackCodeProbes && level == Level::HEALTHY, "ClusterHealth StorageSpaceFactor returns HEALTHY");
+	CODE_PROBE(trackCodeProbes && level == Level::INTERVENTION_REQUIRED,
+	           "ClusterHealth StorageSpaceFactor returns INTERVENTION_REQUIRED");
+	CODE_PROBE(trackCodeProbes && level == Level::CRITICAL_INTERVENTION_REQUIRED,
+	           "ClusterHealth StorageSpaceFactor returns CRITICAL_INTERVENTION_REQUIRED");
+	co_return level;
+}
+
+TLogSpaceFactor::TLogSpaceFactor(double interventionThreshold, double criticalInterventionThreshold)
+  : interventionThreshold(interventionThreshold), criticalInterventionThreshold(criticalInterventionThreshold) {}
+
+std::string_view TLogSpaceFactor::getName() const {
+	return "TLogSpace";
+}
+
+Future<Level> TLogSpaceFactor::fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider,
+                                          TrackCodeProbes trackCodeProbes) {
+	Level level = co_await fetchSpaceLevel(co_await workerEventProvider->getLatestTLogEvents("TLogMetrics"),
+	                                       "QueueDiskBytesAvailable",
+	                                       "QueueDiskBytesTotal",
+	                                       interventionThreshold,
+	                                       criticalInterventionThreshold,
+	                                       "TLogSpaceFactorFetchFailed");
+	CODE_PROBE(trackCodeProbes && level == Level::HEALTHY, "ClusterHealth TLogSpaceFactor returns HEALTHY");
+	CODE_PROBE(trackCodeProbes && level == Level::INTERVENTION_REQUIRED,
+	           "ClusterHealth TLogSpaceFactor returns INTERVENTION_REQUIRED");
+	CODE_PROBE(trackCodeProbes && level == Level::CRITICAL_INTERVENTION_REQUIRED,
+	           "ClusterHealth TLogSpaceFactor returns CRITICAL_INTERVENTION_REQUIRED");
+	co_return level;
+}
+
+std::string_view StorageReplicationFactor::getName() const {
+	return "StorageReplication";
+}
+
+Future<Level> StorageReplicationFactor::fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider,
+                                                   TrackCodeProbes trackCodeProbes) {
+	auto eventsAndErrors = co_await workerEventProvider->getLatestDataDistributorEvents("MovingData");
+	if (!eventsAndErrors.present()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	auto const& [events, _errors] = eventsAndErrors.get();
+	WorkerEvents filteredEvents = filterEmptyEvents(events);
+	if (filteredEvents.empty()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	try {
+		int64_t queuedOrInFlightRepairMoves = 0;
+		int64_t zeroReplicaTeams = 0;
+		int64_t oneReplicaTeams = 0;
+		for (auto const& [address, traceEvent] : filteredEvents) {
+			(void)address;
+			int64_t inQueue = traceEvent.getInt64("InQueue");
+			int64_t inFlight = traceEvent.getInt64("InFlight");
+			int64_t priorityTeamUnhealthy = traceEvent.getInt64("PriorityTeamUnhealthy");
+			int64_t priorityTeam2Left = traceEvent.getInt64("PriorityTeam2Left");
+			int64_t priorityTeam1Left = traceEvent.getInt64("PriorityTeam1Left");
+			int64_t priorityTeam0Left = traceEvent.getInt64("PriorityTeam0Left");
+
+			zeroReplicaTeams += priorityTeam0Left;
+			oneReplicaTeams += priorityTeam1Left;
+			if (inQueue > 0 || inFlight > 0) {
+				queuedOrInFlightRepairMoves +=
+				    priorityTeamUnhealthy + priorityTeam2Left + priorityTeam1Left + priorityTeam0Left;
+			}
+		}
+
+		if (zeroReplicaTeams > 0) {
+			CODE_PROBE(trackCodeProbes, "ClusterHealth StorageReplicationFactor returns OUTAGE");
+			co_return Level::OUTAGE;
+		}
+		if (oneReplicaTeams > 0 && workerEventProvider->shouldTreatStorageTeamOneReplicaLeftAsCritical()) {
+			CODE_PROBE(trackCodeProbes,
+			           "ClusterHealth StorageReplicationFactor returns CRITICAL_INTERVENTION_REQUIRED");
+			co_return Level::CRITICAL_INTERVENTION_REQUIRED;
+		}
+		if (queuedOrInFlightRepairMoves > 0) {
+			CODE_PROBE(trackCodeProbes, "ClusterHealth StorageReplicationFactor returns SELF_HEALING");
+			co_return Level::SELF_HEALING;
+		}
+		CODE_PROBE(trackCodeProbes, "ClusterHealth StorageReplicationFactor returns HEALTHY");
+		co_return Level::HEALTHY;
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "StorageReplicationFactorFetchFailed").error(e);
+		co_return Level::METRICS_MISSING;
+	}
+}
+
+std::string_view RecoveryStateFactor::getName() const {
+	return "RecoveryState";
+}
+
+Future<Level> RecoveryStateFactor::fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider,
+                                              TrackCodeProbes trackCodeProbes) {
+	Optional<RecoveryState> recoveryState = workerEventProvider->getRecoveryState();
+	if (!recoveryState.present()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	Level level = Level::HEALTHY;
+	if (recoveryState.get() < RecoveryState::ACCEPTING_COMMITS) {
+		level = Level::OUTAGE;
+	} else if (recoveryState.get() < RecoveryState::FULLY_RECOVERED) {
+		level = Level::SELF_HEALING;
+	}
+
+	CODE_PROBE(trackCodeProbes && level == Level::OUTAGE, "ClusterHealth RecoveryStateFactor returns OUTAGE");
+	CODE_PROBE(trackCodeProbes && level == Level::SELF_HEALING,
+	           "ClusterHealth RecoveryStateFactor returns SELF_HEALING");
+	CODE_PROBE(trackCodeProbes && level == Level::HEALTHY, "ClusterHealth RecoveryStateFactor returns HEALTHY");
+	co_return level;
+}
+
+std::string_view ProcessErrorsFactor::getName() const {
+	return "ProcessErrors";
+}
+
+Future<Level> ProcessErrorsFactor::fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider,
+                                              TrackCodeProbes trackCodeProbes) {
+	auto eventsAndErrors = co_await workerEventProvider->getLatestEvents("");
+	if (!eventsAndErrors.present()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	auto const& [events, errors] = eventsAndErrors.get();
+	WorkerEvents filteredEvents = filterEmptyEvents(events);
+	if (filteredEvents.empty()) {
+		bool const hadSuccessfulRequest = events.size() > errors.size();
+		CODE_PROBE(trackCodeProbes && hadSuccessfulRequest, "ClusterHealth ProcessErrorsFactor returns HEALTHY");
+		co_return hadSuccessfulRequest ? Level::HEALTHY : Level::METRICS_MISSING;
+	}
+	CODE_PROBE(trackCodeProbes, "ClusterHealth ProcessErrorsFactor returns CRITICAL_INTERVENTION_REQUIRED");
+	co_return Level::CRITICAL_INTERVENTION_REQUIRED;
+}
+
+RkThrottlingFactor::RkThrottlingFactor(double criticalTpsLimitToReleasedTpsRatioThreshold)
+  : criticalTpsLimitToReleasedTpsRatioThreshold(criticalTpsLimitToReleasedTpsRatioThreshold) {}
+
+std::string_view RkThrottlingFactor::getName() const {
+	return "RkThrottling";
+}
+
+Future<Level> RkThrottlingFactor::fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider,
+                                             TrackCodeProbes trackCodeProbes) {
+	auto eventsAndErrors = co_await workerEventProvider->getLatestRatekeeperEvents("RkUpdate");
+	if (!eventsAndErrors.present()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	auto const& [events, _errors] = eventsAndErrors.get();
+	WorkerEvents filteredEvents = filterEmptyEvents(events);
+	if (filteredEvents.empty()) {
+		co_return Level::METRICS_MISSING;
+	}
+
+	try {
+		double minTpsLimitToReleasedTpsRatio = std::numeric_limits<double>::infinity();
+		for (auto const& [address, traceEvent] : filteredEvents) {
+			(void)address;
+			double tpsLimit = traceEvent.getDouble("TPSLimit");
+			if (tpsLimit == 0.0) {
+				CODE_PROBE(trackCodeProbes, "ClusterHealth RkThrottlingFactor returns OUTAGE");
+				co_return Level::OUTAGE;
+			}
+
+			double releasedTps = traceEvent.getDouble("ReleasedTPS");
+			if (releasedTps == 0.0) {
+				continue;
+			}
+
+			minTpsLimitToReleasedTpsRatio = std::min(minTpsLimitToReleasedTpsRatio, tpsLimit / releasedTps);
+		}
+
+		if (minTpsLimitToReleasedTpsRatio < criticalTpsLimitToReleasedTpsRatioThreshold) {
+			CODE_PROBE(trackCodeProbes, "ClusterHealth RkThrottlingFactor returns CRITICAL_INTERVENTION_REQUIRED");
+			co_return Level::CRITICAL_INTERVENTION_REQUIRED;
+		}
+		CODE_PROBE(trackCodeProbes, "ClusterHealth RkThrottlingFactor returns HEALTHY");
+		co_return Level::HEALTHY;
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "RkThrottlingFactorFetchFailed").error(e);
+		co_return Level::METRICS_MISSING;
+	}
+}
+
+} // namespace cluster_health
diff --git a/fdbserver/clustercontroller/ClusterHealthIFactor.h b/fdbserver/clustercontroller/ClusterHealthIFactor.h
new file mode 100644
index 00000000000..533e4bba5e0
--- /dev/null
+++ b/fdbserver/clustercontroller/ClusterHealthIFactor.h
@@ -0,0 +1,94 @@
+/*
+ * ClusterHealthIFactor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string_view>
+
+#include "flow/BooleanParam.h"
+#include "flow/flow.h"
+
+namespace cluster_health {
+
+enum class Level;
+class IWorkerEventProvider;
+FDB_BOOLEAN_PARAM(TrackCodeProbes);
+
+class IFactor {
+public:
+	virtual ~IFactor() = default;
+	virtual std::string_view getName() const = 0;
+	virtual Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) = 0;
+};
+
+// Evaluates storage-server free-space pressure from StorageMetrics events.
+class StorageSpaceFactor final : public IFactor {
+	double interventionThreshold;
+	double criticalInterventionThreshold;
+
+public:
+	StorageSpaceFactor(double interventionThreshold, double criticalInterventionThreshold);
+	std::string_view getName() const override;
+	Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) override;
+};
+
+// Evaluates TLog queue-disk free-space pressure from TLogMetrics events.
+class TLogSpaceFactor final : public IFactor {
+	double interventionThreshold;
+	double criticalInterventionThreshold;
+
+public:
+	TLogSpaceFactor(double interventionThreshold, double criticalInterventionThreshold);
+	std::string_view getName() const override;
+	Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) override;
+};
+
+// Evaluates whether data distribution is restoring or has lost storage replication.
+class StorageReplicationFactor final : public IFactor {
+public:
+	std::string_view getName() const override;
+	Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) override;
+};
+
+// Evaluates cluster recovery progress from cluster controller state.
+class RecoveryStateFactor final : public IFactor {
+public:
+	std::string_view getName() const override;
+	Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) override;
+};
+
+// Evaluates whether any worker is currently reporting a latest process error.
+class ProcessErrorsFactor final : public IFactor {
+public:
+	std::string_view getName() const override;
+	Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) override;
+};
+
+// Evaluates ratekeeper throttling severity from RkUpdate events.
+class RkThrottlingFactor final : public IFactor {
+	double criticalTpsLimitToReleasedTpsRatioThreshold;
+
+public:
+	explicit RkThrottlingFactor(double criticalTpsLimitToReleasedTpsRatioThreshold);
+	std::string_view getName() const override;
+	Future<Level> fetchLevel(Reference<IWorkerEventProvider const> workerEventProvider, TrackCodeProbes) override;
+};
+
+} // namespace cluster_health
diff --git a/fdbserver/clustercontroller/ClusterHealthMonitor.cpp b/fdbserver/clustercontroller/ClusterHealthMonitor.cpp
new file mode 100644
index 00000000000..33c03ab3fd4
--- /dev/null
+++ b/fdbserver/clustercontroller/ClusterHealthMonitor.cpp
@@ -0,0 +1,274 @@
+/*
+ * ClusterHealthMonitor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unordered_map>
+#include <utility>
+
+#include "fmt/format.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/WorkerEvents.h"
+#include "flow/Trace.h"
+#include "flow/genericactors.actor.h"
+
+#include "ClusterHealthIFactor.h"
+#include "ClusterHealthMonitor.h"
+
+namespace cluster_health {
+
+namespace {
+
+AsyncResult<LatestWorkerEvents> latestEventOnWorker(WorkerInterface worker, std::string eventName) {
+	try {
+		EventLogRequest req = eventName.empty() ? EventLogRequest() : EventLogRequest(Standalone<StringRef>(eventName));
+		ErrorOr<TraceEventFields> traceEvent =
+		    co_await errorOr(timeoutError(worker.eventLogRequest.getReply(req), 2.0));
+
+		WorkerEvents results;
+		std::set<std::string> failed;
+		if (traceEvent.isError()) {
+			failed.insert(worker.address().toString());
+			results[worker.address()] = TraceEventFields();
+		} else {
+			results[worker.address()] = traceEvent.get();
+		}
+		co_return std::make_pair(std::move(results), std::move(failed));
+	} catch (Error& e) {
+		ASSERT(e.code() ==
+		       error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above
+		throw;
+	}
+}
+
+template <class Interface>
+AsyncResult<LatestWorkerEvents> latestEventOnInterfaces(
+    std::vector<Interface> interfaces,
+    std::unordered_map<NetworkAddress, WorkerInterface> addressWorkers,
+    std::string eventName) {
+	try {
+		std::vector<Future<ErrorOr<TraceEventFields>>> eventTraces;
+		std::vector<NetworkAddress> addresses;
+		std::set<std::string> failed;
+		WorkerEvents results;
+
+		for (auto const& interf : interfaces) {
+			auto workerIt = addressWorkers.find(interf.address());
+			if (workerIt == addressWorkers.end()) {
+				failed.insert(interf.address().toString());
+				results[interf.address()] = TraceEventFields();
+				continue;
+			}
+
+			addresses.push_back(interf.address());
+			eventTraces.push_back(
+			    errorOr(timeoutError(workerIt->second.eventLogRequest.getReply(EventLogRequest(
+			                             Standalone<StringRef>(interf.id().toString() + "/" + eventName))),
+			                         2.0)));
+		}
+
+		co_await waitForAll(eventTraces);
+
+		for (int i = 0; i < eventTraces.size(); ++i) {
+			ErrorOr<TraceEventFields> const& traceEvent = eventTraces[i].get();
+			if (traceEvent.isError()) {
+				failed.insert(addresses[i].toString());
+				results[addresses[i]] = TraceEventFields();
+			} else {
+				results[addresses[i]] = traceEvent.get();
+			}
+		}
+
+		co_return std::make_pair(std::move(results), std::move(failed));
+	} catch (Error& e) {
+		ASSERT(e.code() ==
+		       error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above
+		throw;
+	}
+}
+
+uint8_t levelToInt(Level level) {
+	switch (level) {
+	case Level::HEALTHY:
+		return 100;
+	case Level::SELF_HEALING:
+		return 80;
+	case Level::INTERVENTION_REQUIRED:
+		return 60;
+	case Level::CRITICAL_INTERVENTION_REQUIRED:
+		return 40;
+	case Level::METRICS_MISSING:
+		return 20;
+	case Level::OUTAGE:
+		return 0;
+	}
+
+	UNREACHABLE();
+}
+
+std::string_view levelToStr(Level level) {
+	switch (level) {
+	case Level::OUTAGE:
+		return "Outage";
+	case Level::CRITICAL_INTERVENTION_REQUIRED:
+		return "CriticalInterventionRequired";
+	case Level::INTERVENTION_REQUIRED:
+		return "InterventionRequired";
+	case Level::SELF_HEALING:
+		return "SelfHealing";
+	case Level::METRICS_MISSING:
+		return "MetricsMissing";
+	case Level::HEALTHY:
+		return "Healthy";
+	}
+
+	UNREACHABLE();
+}
+
+} // namespace
+
+void WorkerEventProvider::setWorkers(std::vector<WorkerDetails> workers) {
+	this->workers = std::move(workers);
+}
+
+void WorkerEventProvider::setRecoveryState(RecoveryState recoveryState) {
+	this->recoveryState = recoveryState;
+}
+
+void WorkerEventProvider::setStorageTeamOneReplicaLeftIsCritical(bool storageTeamOneReplicaLeftIsCritical) {
+	this->storageTeamOneReplicaLeftIsCritical = storageTeamOneReplicaLeftIsCritical;
+}
+
+void WorkerEventProvider::setRatekeeperWorker(Optional<WorkerInterface> ratekeeperWorker) {
+	this->ratekeeperWorker = std::move(ratekeeperWorker);
+}
+
+void WorkerEventProvider::setDataDistributorWorker(Optional<WorkerInterface> dataDistributorWorker) {
+	this->dataDistributorWorker = std::move(dataDistributorWorker);
+}
+
+void WorkerEventProvider::setStorageServers(std::vector<StorageServerInterface> storageServers) {
+	this->storageServers = std::move(storageServers);
+}
+
+void WorkerEventProvider::setTLogs(std::vector<TLogInterface> tlogs) {
+	this->tlogs = std::move(tlogs);
+}
+
+Optional<RecoveryState> WorkerEventProvider::getRecoveryState() const {
+	return recoveryState;
+}
+
+bool WorkerEventProvider::shouldTreatStorageTeamOneReplicaLeftAsCritical() const {
+	return storageTeamOneReplicaLeftIsCritical;
+}
+
+AsyncResult<LatestWorkerEvents> WorkerEventProvider::getLatestEvents(std::string const& eventName) const {
+	return latestEventOnWorkers(workers, eventName);
+}
+
+AsyncResult<LatestWorkerEvents> WorkerEventProvider::getLatestRatekeeperEvents(std::string const& eventName) const {
+	if (!ratekeeperWorker.present()) {
+		co_return LatestWorkerEvents();
+	}
+	co_return co_await latestEventOnWorker(ratekeeperWorker.get(), eventName);
+}
+
+AsyncResult<LatestWorkerEvents> WorkerEventProvider::getLatestDataDistributorEvents(
+    std::string const& eventName) const {
+	if (!dataDistributorWorker.present()) {
+		co_return LatestWorkerEvents();
+	}
+	co_return co_await latestEventOnWorker(dataDistributorWorker.get(), eventName);
+}
+
+AsyncResult<LatestWorkerEvents> WorkerEventProvider::getLatestStorageServerEvents(std::string const& eventName) const {
+	std::unordered_map<NetworkAddress, WorkerInterface> addressWorkers;
+	addressWorkers.reserve(workers.size());
+	for (auto const& worker : workers) {
+		addressWorkers.emplace(worker.interf.address(), worker.interf);
+	}
+	return latestEventOnInterfaces(storageServers, std::move(addressWorkers), eventName);
+}
+
+AsyncResult<LatestWorkerEvents> WorkerEventProvider::getLatestTLogEvents(std::string const& eventName) const {
+	std::unordered_map<NetworkAddress, WorkerInterface> addressWorkers;
+	addressWorkers.reserve(workers.size());
+	for (auto const& worker : workers) {
+		addressWorkers.emplace(worker.interf.address(), worker.interf);
+	}
+	return latestEventOnInterfaces(tlogs, std::move(addressWorkers), eventName);
+}
+
+Monitor::Monitor(std::vector<std::unique_ptr<IFactor>>&& factors,
+                 Reference<IWorkerEventProvider const> workerEventProvider)
+  : factors(std::move(factors)), workerEventProvider(workerEventProvider) {}
+
+Future<Void> Monitor::run() {
+	if (!SERVER_KNOBS->CLUSTER_HEALTH_METRIC_ENABLE) {
+		co_return;
+	}
+
+	Future<Void> timer = Void();
+	while (true) {
+		co_await timer;
+		timer = delay(SERVER_KNOBS->CLUSTER_HEALTH_METRIC_POLL_INTERVAL);
+
+		std::vector<Future<Level>> levelFutures;
+		levelFutures.reserve(factors.size());
+		for (auto const& factor : factors) {
+			levelFutures.push_back(factor->fetchLevel(workerEventProvider, TrackCodeProbes::True));
+		}
+		co_await waitForAll(levelFutures);
+
+		Optional<int> limitingIndex;
+		Level limitingLevel = Level::HEALTHY;
+		TraceEvent traceEvent("ClusterHealthMetric");
+		for (int i = 0; i < factors.size(); ++i) {
+			Level level = levelFutures[i].get();
+			traceEvent.detail(fmt::format("Factor{}", factors[i]->getName()), levelToStr(level));
+			if (!limitingIndex.present() || levelToInt(level) < levelToInt(limitingLevel)) {
+				limitingIndex = i;
+				limitingLevel = level;
+			}
+		}
+
+		traceEvent.detail("Aggregate", levelToStr(limitingLevel));
+		traceEvent.detail("AggregateValue", levelToInt(limitingLevel));
+		if (limitingIndex.present() && limitingLevel != Level::HEALTHY) {
+			traceEvent.detail("LimitingFactor", factors[*limitingIndex]->getName());
+		}
+	}
+}
+
+Monitor Monitor::create(Reference<IWorkerEventProvider const> workerEventProvider) {
+	std::vector<std::unique_ptr<IFactor>> factors;
+	factors.push_back(
+	    std::make_unique<StorageSpaceFactor>(SERVER_KNOBS->CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD,
+	                                         SERVER_KNOBS->CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD));
+	factors.push_back(std::make_unique<TLogSpaceFactor>(SERVER_KNOBS->CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD,
+	                                                    SERVER_KNOBS->CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD));
+	factors.push_back(std::make_unique<StorageReplicationFactor>());
+	factors.push_back(std::make_unique<RecoveryStateFactor>());
+	factors.push_back(std::make_unique<ProcessErrorsFactor>());
+	factors.push_back(std::make_unique<RkThrottlingFactor>(
+	    SERVER_KNOBS->CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD));
+	return Monitor(std::move(factors), workerEventProvider);
+}
+
+} // namespace cluster_health
diff --git a/fdbserver/clustercontroller/ClusterHealthMonitor.h b/fdbserver/clustercontroller/ClusterHealthMonitor.h
new file mode 100644
index 00000000000..94921827670
--- /dev/null
+++ b/fdbserver/clustercontroller/ClusterHealthMonitor.h
@@ -0,0 +1,107 @@
+/*
+ * ClusterHealthMonitor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "ClusterHealthIFactor.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/core/WorkerEvents.h"
+#include "flow/flow.h"
+
+namespace cluster_health {
+
+enum class Level {
+	OUTAGE,
+	CRITICAL_INTERVENTION_REQUIRED,
+	INTERVENTION_REQUIRED,
+	SELF_HEALING,
+	METRICS_MISSING,
+	HEALTHY
+};
+
+using LatestWorkerEvents = Optional<std::pair<WorkerEvents, std::set<std::string>>>;
+
+// Abstracts access to health-monitor inputs so factors can be evaluated in production and tests.
+class IWorkerEventProvider {
+public:
+	virtual ~IWorkerEventProvider() = default;
+	virtual void addref() const = 0;
+	virtual void delref() const = 0;
+	virtual Optional<RecoveryState> getRecoveryState() const = 0;
+	virtual bool shouldTreatStorageTeamOneReplicaLeftAsCritical() const = 0;
+	virtual AsyncResult<LatestWorkerEvents> getLatestEvents(std::string const& eventName) const = 0;
+	virtual AsyncResult<LatestWorkerEvents> getLatestRatekeeperEvents(std::string const& eventName) const = 0;
+	virtual AsyncResult<LatestWorkerEvents> getLatestDataDistributorEvents(std::string const& eventName) const = 0;
+	virtual AsyncResult<LatestWorkerEvents> getLatestStorageServerEvents(std::string const& eventName) const = 0;
+	virtual AsyncResult<LatestWorkerEvents> getLatestTLogEvents(std::string const& eventName) const = 0;
+};
+
+// Production event provider backed by worker event-log RPCs.
+class WorkerEventProvider final : public IWorkerEventProvider, public ReferenceCounted<WorkerEventProvider> {
+	std::vector<WorkerDetails> workers;
+	Optional<RecoveryState> recoveryState;
+	bool storageTeamOneReplicaLeftIsCritical = false;
+	Optional<WorkerInterface> ratekeeperWorker;
+	Optional<WorkerInterface> dataDistributorWorker;
+	std::vector<StorageServerInterface> storageServers;
+	std::vector<TLogInterface> tlogs;
+
+public:
+	void addref() const override { ReferenceCounted<WorkerEventProvider>::addref(); }
+	void delref() const override { ReferenceCounted<WorkerEventProvider>::delref(); }
+	void setWorkers(std::vector<WorkerDetails> workers);
+	void setRecoveryState(RecoveryState recoveryState);
+	void setStorageTeamOneReplicaLeftIsCritical(bool storageTeamOneReplicaLeftIsCritical);
+	void setRatekeeperWorker(Optional<WorkerInterface> ratekeeperWorker);
+	void setDataDistributorWorker(Optional<WorkerInterface> dataDistributorWorker);
+	void setStorageServers(std::vector<StorageServerInterface> storageServers);
+	void setTLogs(std::vector<TLogInterface> tlogs);
+	Optional<RecoveryState> getRecoveryState() const override;
+	bool shouldTreatStorageTeamOneReplicaLeftAsCritical() const override;
+	AsyncResult<LatestWorkerEvents> getLatestEvents(std::string const& eventName) const override;
+	AsyncResult<LatestWorkerEvents> getLatestRatekeeperEvents(std::string const& eventName) const override;
+	AsyncResult<LatestWorkerEvents> getLatestDataDistributorEvents(std::string const& eventName) const override;
+	AsyncResult<LatestWorkerEvents> getLatestStorageServerEvents(std::string const& eventName) const override;
+	AsyncResult<LatestWorkerEvents> getLatestTLogEvents(std::string const& eventName) const override;
+};
+
+// Periodically evaluates factors and logs the aggregate cluster-health metric.
+class Monitor {
+	std::vector<std::unique_ptr<IFactor>> factors;
+	Reference<IWorkerEventProvider const> workerEventProvider;
+
+	Monitor(std::vector<std::unique_ptr<IFactor>>&& factors, Reference<IWorkerEventProvider const> workerEventProvider);
+
+public:
+	explicit(false) Monitor(Monitor&&) noexcept = default;
+	Monitor& operator=(Monitor&&) noexcept = default;
+
+	static Monitor create(Reference<IWorkerEventProvider const> workerEventProvider);
+	Future<Void> run();
+};
+
+} // namespace cluster_health
diff --git a/fdbserver/clustercontroller/ClusterHealthMonitorTesting.cpp b/fdbserver/clustercontroller/ClusterHealthMonitorTesting.cpp
new file mode 100644
index 00000000000..83596f5104c
--- /dev/null
+++ b/fdbserver/clustercontroller/ClusterHealthMonitorTesting.cpp
@@ -0,0 +1,407 @@
+/*
+ * ClusterHealthMonitorTesting.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <map>
+
+#include "fdbserver/core/RecoveryState.h"
+#include "flow/UnitTest.h"
+
+#include "ClusterHealthIFactor.h"
+#include "ClusterHealthMonitor.h"
+
+namespace cluster_health {
+
+namespace {
+
+class FakeWorkerEventProvider final : public IWorkerEventProvider, public ReferenceCounted<FakeWorkerEventProvider> {
+	Optional<RecoveryState> recoveryState;
+	bool storageTeamOneReplicaLeftIsCritical = false;
+	std::map<std::string, LatestWorkerEvents> latestEventsByName;
+	std::map<std::string, LatestWorkerEvents> latestRatekeeperEventsByName;
+	std::map<std::string, LatestWorkerEvents> latestDataDistributorEventsByName;
+	std::map<std::string, LatestWorkerEvents> latestStorageServerEventsByName;
+	std::map<std::string, LatestWorkerEvents> latestTLogEventsByName;
+
+public:
+	void addref() const override { ReferenceCounted<FakeWorkerEventProvider>::addref(); }
+	void delref() const override { ReferenceCounted<FakeWorkerEventProvider>::delref(); }
+
+	void setLatestEvents(std::string eventName, LatestWorkerEvents latestEvents) {
+		latestEventsByName[std::move(eventName)] = std::move(latestEvents);
+	}
+
+	void setRecoveryState(RecoveryState recoveryState) { this->recoveryState = recoveryState; }
+
+	void setStorageTeamOneReplicaLeftIsCritical(bool storageTeamOneReplicaLeftIsCritical) {
+		this->storageTeamOneReplicaLeftIsCritical = storageTeamOneReplicaLeftIsCritical;
+	}
+
+	void setLatestRatekeeperEvents(std::string eventName, LatestWorkerEvents latestEvents) {
+		latestRatekeeperEventsByName[std::move(eventName)] = std::move(latestEvents);
+	}
+
+	void setLatestDataDistributorEvents(std::string eventName, LatestWorkerEvents latestEvents) {
+		latestDataDistributorEventsByName[std::move(eventName)] = std::move(latestEvents);
+	}
+
+	void setLatestStorageServerEvents(std::string eventName, LatestWorkerEvents latestEvents) {
+		latestStorageServerEventsByName[std::move(eventName)] = std::move(latestEvents);
+	}
+
+	void setLatestTLogEvents(std::string eventName, LatestWorkerEvents latestEvents) {
+		latestTLogEventsByName[std::move(eventName)] = std::move(latestEvents);
+	}
+
+	AsyncResult<LatestWorkerEvents> getLatestEvents(std::string const& eventName) const override {
+		auto it = latestEventsByName.find(eventName);
+		if (it == latestEventsByName.end()) {
+			co_return LatestWorkerEvents();
+		}
+		co_return it->second;
+	}
+
+	Optional<RecoveryState> getRecoveryState() const override { return recoveryState; }
+
+	bool shouldTreatStorageTeamOneReplicaLeftAsCritical() const override { return storageTeamOneReplicaLeftIsCritical; }
+
+	AsyncResult<LatestWorkerEvents> getLatestRatekeeperEvents(std::string const& eventName) const override {
+		auto it = latestRatekeeperEventsByName.find(eventName);
+		if (it != latestRatekeeperEventsByName.end()) {
+			co_return it->second;
+		}
+		co_return co_await getLatestEvents(eventName);
+	}
+
+	AsyncResult<LatestWorkerEvents> getLatestDataDistributorEvents(std::string const& eventName) const override {
+		auto it = latestDataDistributorEventsByName.find(eventName);
+		if (it != latestDataDistributorEventsByName.end()) {
+			co_return it->second;
+		}
+		co_return co_await getLatestEvents(eventName);
+	}
+
+	AsyncResult<LatestWorkerEvents> getLatestStorageServerEvents(std::string const& eventName) const override {
+		auto it = latestStorageServerEventsByName.find(eventName);
+		if (it == latestStorageServerEventsByName.end()) {
+			co_return LatestWorkerEvents();
+		}
+		co_return it->second;
+	}
+
+	AsyncResult<LatestWorkerEvents> getLatestTLogEvents(std::string const& eventName) const override {
+		auto it = latestTLogEventsByName.find(eventName);
+		if (it == latestTLogEventsByName.end()) {
+			co_return LatestWorkerEvents();
+		}
+		co_return it->second;
+	}
+};
+
+TraceEventFields makeSpaceMetrics(std::string const& availableBytesField,
+                                  std::string const& totalBytesField,
+                                  double availableBytes,
+                                  double totalBytes) {
+	TraceEventFields traceEventFields;
+	traceEventFields.addField(availableBytesField, std::to_string(availableBytes));
+	traceEventFields.addField(totalBytesField, std::to_string(totalBytes));
+	return traceEventFields;
+}
+
+LatestWorkerEvents makeLatestWorkerEvents(TraceEventFields traceEventFields) {
+	WorkerEvents events;
+	events.emplace(NetworkAddress(IPAddress(0x01010101), 1), std::move(traceEventFields));
+	return LatestWorkerEvents(std::make_pair(std::move(events), std::set<std::string>{}));
+}
+
+LatestWorkerEvents makeLatestWorkerEvents(WorkerEvents events) {
+	return LatestWorkerEvents(std::make_pair(std::move(events), std::set<std::string>{}));
+}
+
+LatestWorkerEvents makeLatestWorkerEvents(NetworkAddress address, TraceEventFields traceEventFields) {
+	WorkerEvents events;
+	events.emplace(address, std::move(traceEventFields));
+	return LatestWorkerEvents(std::make_pair(std::move(events), std::set<std::string>{}));
+}
+
+LatestWorkerEvents makeLatestWorkerEvents(WorkerEvents events, std::set<std::string> errors) {
+	return LatestWorkerEvents(std::make_pair(std::move(events), std::move(errors)));
+}
+
+class MovingDataMetricsBuilder {
+	TraceEventFields traceEventFields;
+
+	MovingDataMetricsBuilder& setField(char const* fieldName, int64_t value) {
+		traceEventFields.addField(fieldName, std::to_string(value));
+		return *this;
+	}
+
+	TraceEventFields buildWithDefault(char const* fieldName, int64_t defaultValue, TraceEventFields fields) const {
+		std::string value;
+		if (!fields.tryGetValue(fieldName, value)) {
+			fields.addField(fieldName, std::to_string(defaultValue));
+		}
+		return fields;
+	}
+
+public:
+	MovingDataMetricsBuilder& inQueue(int64_t value) { return setField("InQueue", value); }
+	MovingDataMetricsBuilder& inFlight(int64_t value) { return setField("InFlight", value); }
+	MovingDataMetricsBuilder& priorityTeamUnhealthy(int64_t value) { return setField("PriorityTeamUnhealthy", value); }
+	MovingDataMetricsBuilder& priorityTeam2Left(int64_t value) { return setField("PriorityTeam2Left", value); }
+	MovingDataMetricsBuilder& priorityTeam1Left(int64_t value) { return setField("PriorityTeam1Left", value); }
+	MovingDataMetricsBuilder& priorityTeam0Left(int64_t value) { return setField("PriorityTeam0Left", value); }
+
+	TraceEventFields build() const {
+		TraceEventFields fields = traceEventFields;
+		fields = buildWithDefault("InQueue", 0, std::move(fields));
+		fields = buildWithDefault("InFlight", 0, std::move(fields));
+		fields = buildWithDefault("PriorityTeamUnhealthy", 0, std::move(fields));
+		fields = buildWithDefault("PriorityTeam2Left", 0, std::move(fields));
+		fields = buildWithDefault("PriorityTeam1Left", 0, std::move(fields));
+		fields = buildWithDefault("PriorityTeam0Left", 0, std::move(fields));
+		return fields;
+	}
+};
+
+TraceEventFields makeProcessErrorMetrics(std::string const& type, std::string const& error) {
+	TraceEventFields traceEventFields;
+	traceEventFields.addField("Type", type);
+	traceEventFields.addField("Error", error);
+	return traceEventFields;
+}
+
+TraceEventFields makeRkUpdateMetrics(double releasedTps, double tpsLimit) {
+	TraceEventFields traceEventFields;
+	traceEventFields.addField("ReleasedTPS", std::to_string(releasedTps));
+	traceEventFields.addField("TPSLimit", std::to_string(tpsLimit));
+	return traceEventFields;
+}
+
+} // namespace
+
+TEST_CASE("/fdbserver/clustercontroller/ClusterHealthMonitor/StorageSpaceFactor") {
+	StorageSpaceFactor factor(/*interventionThreshold=*/0.20, /*criticalInterventionThreshold=*/0.10);
+	auto provider = makeReference<FakeWorkerEventProvider>();
+	Level level;
+
+	provider->setLatestStorageServerEvents(
+	    "StorageMetrics",
+	    makeLatestWorkerEvents(makeSpaceMetrics("KvstoreBytesAvailable", "KvstoreBytesTotal", 50, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestStorageServerEvents(
+	    "StorageMetrics",
+	    makeLatestWorkerEvents(makeSpaceMetrics("KvstoreBytesAvailable", "KvstoreBytesTotal", 15, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::INTERVENTION_REQUIRED);
+
+	provider->setLatestStorageServerEvents(
+	    "StorageMetrics",
+	    makeLatestWorkerEvents(makeSpaceMetrics("KvstoreBytesAvailable", "KvstoreBytesTotal", 5, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::CRITICAL_INTERVENTION_REQUIRED);
+
+	WorkerEvents mixedRoleEvents;
+	mixedRoleEvents.emplace(NetworkAddress(IPAddress(0x01010101), 1), TraceEventFields());
+	mixedRoleEvents.emplace(NetworkAddress(IPAddress(0x02020202), 2),
+	                        makeSpaceMetrics("KvstoreBytesAvailable", "KvstoreBytesTotal", 50, 100));
+	provider->setLatestStorageServerEvents("StorageMetrics", makeLatestWorkerEvents(std::move(mixedRoleEvents)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestStorageServerEvents("StorageMetrics", LatestWorkerEvents());
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+}
+
+TEST_CASE("/fdbserver/clustercontroller/ClusterHealthMonitor/TLogSpaceFactor") {
+	TLogSpaceFactor factor(/*interventionThreshold=*/0.20, /*criticalInterventionThreshold=*/0.10);
+	auto provider = makeReference<FakeWorkerEventProvider>();
+	Level level;
+
+	provider->setLatestTLogEvents(
+	    "TLogMetrics",
+	    makeLatestWorkerEvents(makeSpaceMetrics("QueueDiskBytesAvailable", "QueueDiskBytesTotal", 50, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestTLogEvents(
+	    "TLogMetrics",
+	    makeLatestWorkerEvents(makeSpaceMetrics("QueueDiskBytesAvailable", "QueueDiskBytesTotal", 15, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::INTERVENTION_REQUIRED);
+
+	provider->setLatestTLogEvents(
+	    "TLogMetrics",
+	    makeLatestWorkerEvents(makeSpaceMetrics("QueueDiskBytesAvailable", "QueueDiskBytesTotal", 5, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::CRITICAL_INTERVENTION_REQUIRED);
+
+	provider->setLatestTLogEvents("TLogMetrics", LatestWorkerEvents());
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+}
+
+TEST_CASE("/fdbserver/clustercontroller/ClusterHealthMonitor/StorageReplicationFactor") {
+	StorageReplicationFactor factor;
+	auto provider = makeReference<FakeWorkerEventProvider>();
+	Level level;
+
+	provider->setLatestEvents("MovingData", makeLatestWorkerEvents(MovingDataMetricsBuilder().build()));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestEvents(
+	    "MovingData", makeLatestWorkerEvents(MovingDataMetricsBuilder().inQueue(1).priorityTeamUnhealthy(1).build()));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::SELF_HEALING);
+
+	provider->setLatestEvents(
+	    "MovingData", makeLatestWorkerEvents(MovingDataMetricsBuilder().inFlight(1).priorityTeam1Left(1).build()));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::SELF_HEALING);
+
+	provider->setStorageTeamOneReplicaLeftIsCritical(true);
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::CRITICAL_INTERVENTION_REQUIRED);
+
+	provider->setLatestEvents(
+	    "MovingData", makeLatestWorkerEvents(MovingDataMetricsBuilder().inQueue(1).priorityTeam0Left(1).build()));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::OUTAGE);
+
+	WorkerEvents staleWorkerEvents;
+	staleWorkerEvents.emplace(NetworkAddress(IPAddress(0x01010101), 1),
+	                          MovingDataMetricsBuilder().inQueue(1).priorityTeam0Left(1).build());
+	staleWorkerEvents.emplace(NetworkAddress(IPAddress(0x02020202), 2), MovingDataMetricsBuilder().build());
+	provider->setLatestEvents("MovingData", makeLatestWorkerEvents(std::move(staleWorkerEvents)));
+	provider->setLatestDataDistributorEvents(
+	    "MovingData",
+	    makeLatestWorkerEvents(NetworkAddress(IPAddress(0x02020202), 2), MovingDataMetricsBuilder().build()));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestEvents("MovingData", LatestWorkerEvents());
+	provider->setLatestDataDistributorEvents("MovingData", LatestWorkerEvents());
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+}
+
+TEST_CASE("/fdbserver/clustercontroller/ClusterHealthMonitor/RecoveryStateFactor") {
+	RecoveryStateFactor factor;
+	auto provider = makeReference<FakeWorkerEventProvider>();
+	Level level;
+
+	provider->setRecoveryState(RecoveryState::FULLY_RECOVERED);
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setRecoveryState(RecoveryState::ACCEPTING_COMMITS);
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::SELF_HEALING);
+
+	provider->setRecoveryState(RecoveryState::ALL_LOGS_RECRUITED);
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::SELF_HEALING);
+
+	provider->setRecoveryState(RecoveryState::RECOVERY_TRANSACTION);
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::OUTAGE);
+
+	provider->setRecoveryState(RecoveryState::FULLY_RECOVERED);
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	auto missingProvider = makeReference<FakeWorkerEventProvider>();
+	level = co_await factor.fetchLevel(missingProvider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+}
+
+TEST_CASE("/fdbserver/clustercontroller/ClusterHealthMonitor/ProcessErrorsFactor") {
+	ProcessErrorsFactor factor;
+	auto provider = makeReference<FakeWorkerEventProvider>();
+	Level level;
+
+	provider->setLatestEvents("", makeLatestWorkerEvents(makeProcessErrorMetrics("OpenClusterIdError", "io_error")));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::CRITICAL_INTERVENTION_REQUIRED);
+
+	WorkerEvents emptyLatestErrors;
+	emptyLatestErrors.emplace(NetworkAddress(IPAddress(0x01010101), 1), TraceEventFields());
+	provider->setLatestEvents("", makeLatestWorkerEvents(std::move(emptyLatestErrors)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	WorkerEvents partiallyFailedLatestErrors;
+	partiallyFailedLatestErrors.emplace(NetworkAddress(IPAddress(0x01010101), 1), TraceEventFields());
+	partiallyFailedLatestErrors.emplace(NetworkAddress(IPAddress(0x02020202), 2), TraceEventFields());
+	provider->setLatestEvents("", makeLatestWorkerEvents(std::move(partiallyFailedLatestErrors), { "2.2.2.2:2" }));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestEvents("", makeLatestWorkerEvents(WorkerEvents(), { "1.1.1.1:1" }));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+
+	provider->setLatestEvents("", LatestWorkerEvents());
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+}
+
+TEST_CASE("/fdbserver/clustercontroller/ClusterHealthMonitor/RkThrottlingFactor") {
+	RkThrottlingFactor factor(/*criticalTpsLimitToReleasedTpsRatioThreshold=*/1.20);
+	auto provider = makeReference<FakeWorkerEventProvider>();
+	Level level;
+
+	provider->setLatestEvents("RkUpdate", makeLatestWorkerEvents(makeRkUpdateMetrics(100, 125)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestEvents("RkUpdate", makeLatestWorkerEvents(makeRkUpdateMetrics(100, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::CRITICAL_INTERVENTION_REQUIRED);
+
+	provider->setLatestEvents("RkUpdate", makeLatestWorkerEvents(makeRkUpdateMetrics(0, 100)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestEvents("RkUpdate", makeLatestWorkerEvents(makeRkUpdateMetrics(0, 0)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::OUTAGE);
+
+	WorkerEvents staleWorkerEvents;
+	staleWorkerEvents.emplace(NetworkAddress(IPAddress(0x01010101), 1), makeRkUpdateMetrics(100, 0));
+	staleWorkerEvents.emplace(NetworkAddress(IPAddress(0x02020202), 2), makeRkUpdateMetrics(100, 200));
+	provider->setLatestEvents("RkUpdate", makeLatestWorkerEvents(std::move(staleWorkerEvents)));
+	provider->setLatestRatekeeperEvents(
+	    "RkUpdate", makeLatestWorkerEvents(NetworkAddress(IPAddress(0x02020202), 2), makeRkUpdateMetrics(100, 200)));
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::HEALTHY);
+
+	provider->setLatestEvents("RkUpdate", LatestWorkerEvents());
+	provider->setLatestRatekeeperEvents("RkUpdate", LatestWorkerEvents());
+	level = co_await factor.fetchLevel(provider, TrackCodeProbes::False);
+	ASSERT_EQ(level, Level::METRICS_MISSING);
+}
+
+} // namespace cluster_health
+
+void forceLinkClusterHealthMonitorTests() {}
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/clustercontroller/ClusterRecovery.cpp
similarity index 75%
rename from fdbserver/ClusterRecovery.actor.cpp
rename to fdbserver/clustercontroller/ClusterRecovery.cpp
index fdeafcdd729..6d3c826a67d 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/clustercontroller/ClusterRecovery.cpp
@@ -1,5 +1,5 @@
 /*
- * ClusterRecovery.actor.cpp
+ * ClusterRecovery.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,17 +23,19 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/sim_validation.h"
-#include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/BackupProgress.actor.h"
-#include "fdbserver/ClusterRecovery.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/WaitFailure.h"
+#include "fdbserver/logsystem/ApplyMetadataMutation.h"
+#include "fdbserver/core/BackupProgress.h"
+#include "ClusterRecovery.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/SeedShardServers.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
 #include "flow/Error.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/Trace.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 static std::set<int> const& normalClusterRecoveryErrors() {
 	static std::set<int> s;
@@ -57,32 +59,25 @@ static std::set<int> const& normalClusterRecoveryErrors() {
 	return s;
 }
 
-ACTOR Future<Void> recoveryTerminateOnConflict(UID dbgid,
-                                               Promise<Void> fullyRecovered,
-                                               Future<Void> onConflict,
-                                               Future<Void> switchedState) {
-	choose {
-		when(wait(onConflict)) {
-			if (!fullyRecovered.isSet()) {
-				TraceEvent("RecoveryTerminated", dbgid).detail("Reason", "Conflict");
-				CODE_PROBE(true, "Coordinated state conflict, recovery terminating");
-				throw worker_removed();
-			}
-			return Void();
-		}
-		when(wait(switchedState)) {
-			return Void();
-		}
+Future<Void> recoveryTerminateOnConflict(UID dbgid,
+                                         Promise<Void> fullyRecovered,
+                                         Future<Void> onConflict,
+                                         Future<Void> switchedState) {
+	auto const res = co_await race(onConflict, switchedState);
+	if (res.index() == 0 && !fullyRecovered.isSet()) {
+		TraceEvent("RecoveryTerminated", dbgid).detail("Reason", "Conflict");
+		CODE_PROBE(true, "Coordinated state conflict, recovery terminating");
+		throw worker_removed();
 	}
 }
 
-ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
-                                    ClusterControllerData::DBInfo* db,
-                                    MasterInterface* newMaster) {
-	state Future<ErrorOr<MasterInterface>> fNewMaster;
-	state WorkerFitnessInfo masterWorker;
+Future<Void> recruitNewMaster(ClusterControllerData* cluster,
+                              ClusterControllerData::DBInfo* db,
+                              MasterInterface* newMaster) {
+	Future<ErrorOr<MasterInterface>> fNewMaster;
+	WorkerFitnessInfo masterWorker;
 
-	loop {
+	while (true) {
 		// We must recruit the master in the same data center as the cluster controller.
 		// This should always be possible, because we can recruit the master on the same process as the cluster
 		// controller.
@@ -96,7 +91,7 @@ ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
 		    !cluster->goodRecruitmentTime.isReady()) {
 			TraceEvent("RecruitNewMaster", cluster->id)
 			    .detail("Fitness", masterWorker.worker.processClass.machineClassFitness(ProcessClass::Master));
-			wait(delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+			co_await delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 			continue;
 		}
 		RecruitMasterRequest rmq;
@@ -106,7 +101,7 @@ ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
 		cluster->masterProcessId = masterWorker.worker.interf.locality.processId();
 		cluster->db.unfinishedRecoveries++;
 		fNewMaster = masterWorker.worker.interf.master.tryGetReply(rmq);
-		wait(ready(fNewMaster) || db->forceMasterFailure.onTrigger());
+		co_await (ready(fNewMaster) || db->forceMasterFailure.onTrigger());
 		if (fNewMaster.isReady() && fNewMaster.get().present()) {
 			TraceEvent("RecruitNewMaster", cluster->id).detail("Recruited", fNewMaster.get().get().id());
 
@@ -117,77 +112,80 @@ ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
 
 			*newMaster = fNewMaster.get().get();
 
-			return Void();
+			co_return;
 		} else {
 			CODE_PROBE(true, "clusterWatchDatabase() !newMaster.present()");
-			wait(delay(SERVER_KNOBS->MASTER_SPIN_DELAY));
+			co_await delay(SERVER_KNOBS->MASTER_SPIN_DELAY);
 		}
 	}
 }
 
-ACTOR Future<Void> clusterRecruitFromConfiguration(ClusterControllerData* self, Reference<RecruitWorkersInfo> req) {
-	// At the moment this doesn't really need to be an actor (it always completes immediately)
+Future<Void> clusterRecruitFromConfiguration(ClusterControllerData* self, Reference<RecruitWorkersInfo> req) {
 	CODE_PROBE(true, "ClusterController RecruitTLogsRequest");
-	loop {
+	while (true) {
+		Error err;
 		try {
 			req->rep = self->findWorkersForConfiguration(req->req);
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			if (e.code() == error_code_no_more_servers && self->goodRecruitmentTime.isReady()) {
-				self->outstandingRecruitmentRequests.push_back(req);
-				TraceEvent(SevWarn, "RecruitFromConfigurationNotAvailable", self->id).error(e);
-				wait(req->waitForCompletion.onTrigger());
-				return Void();
-			} else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) {
-				// recruitment not good enough, try again
-				TraceEvent("RecruitFromConfigurationRetry", self->id)
-				    .error(e)
-				    .detail("GoodRecruitmentTimeReady", self->goodRecruitmentTime.isReady());
-				while (!self->goodRecruitmentTime.isReady()) {
-					wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
-				}
-			} else {
-				TraceEvent(SevError, "RecruitFromConfigurationError", self->id).error(e);
-				throw;
+			err = e;
+		}
+
+		if (err.code() == error_code_no_more_servers && self->goodRecruitmentTime.isReady()) {
+			self->outstandingRecruitmentRequests.push_back(req);
+			TraceEvent(SevWarn, "RecruitFromConfigurationNotAvailable", self->id).error(err);
+			co_await req->waitForCompletion.onTrigger();
+			co_return;
+		} else if (err.code() == error_code_operation_failed || err.code() == error_code_no_more_servers) {
+			// recruitment not good enough, try again
+			TraceEvent("RecruitFromConfigurationRetry", self->id)
+			    .error(err)
+			    .detail("GoodRecruitmentTimeReady", self->goodRecruitmentTime.isReady());
+			while (!self->goodRecruitmentTime.isReady()) {
+				co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 			}
+		} else {
+			TraceEvent(SevError, "RecruitFromConfigurationError", self->id).error(err);
+			throw err;
 		}
-		wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+		co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 	}
 }
 
-ACTOR Future<RecruitRemoteFromConfigurationReply> clusterRecruitRemoteFromConfiguration(
+Future<RecruitRemoteFromConfigurationReply> clusterRecruitRemoteFromConfiguration(
     ClusterControllerData* self,
     Reference<RecruitRemoteWorkersInfo> req) {
-	// At the moment this doesn't really need to be an actor (it always completes immediately)
+	// At the moment this doesn't really need to be a coroutine (it always completes immediately)
 	CODE_PROBE(true, "ClusterController RecruitTLogsRequest Remote");
-	loop {
+	while (true) {
+		Error err;
 		try {
-			auto rep = self->findRemoteWorkersForConfiguration(req->req);
-			return rep;
+			co_return self->findRemoteWorkersForConfiguration(req->req);
 		} catch (Error& e) {
-			if (e.code() == error_code_no_more_servers && self->goodRemoteRecruitmentTime.isReady()) {
-				self->outstandingRemoteRecruitmentRequests.push_back(req);
-				TraceEvent(SevWarn, "RecruitRemoteFromConfigurationNotAvailable", self->id).error(e);
-				wait(req->waitForCompletion.onTrigger());
-				return req->rep;
-			} else if (e.code() == error_code_operation_failed || e.code() == error_code_no_more_servers) {
-				// recruitment not good enough, try again
-				TraceEvent("RecruitRemoteFromConfigurationRetry", self->id)
-				    .error(e)
-				    .detail("GoodRecruitmentTimeReady", self->goodRemoteRecruitmentTime.isReady());
-				while (!self->goodRemoteRecruitmentTime.isReady()) {
-					wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
-				}
-			} else {
-				TraceEvent(SevError, "RecruitRemoteFromConfigurationError", self->id).error(e);
-				throw;
+			err = e;
+		}
+		if (err.code() == error_code_no_more_servers && self->goodRemoteRecruitmentTime.isReady()) {
+			self->outstandingRemoteRecruitmentRequests.push_back(req);
+			TraceEvent(SevWarn, "RecruitRemoteFromConfigurationNotAvailable", self->id).error(err);
+			co_await req->waitForCompletion.onTrigger();
+			co_return req->rep;
+		} else if (err.code() == error_code_operation_failed || err.code() == error_code_no_more_servers) {
+			// recruitment not good enough, try again
+			TraceEvent("RecruitRemoteFromConfigurationRetry", self->id)
+			    .error(err)
+			    .detail("GoodRecruitmentTimeReady", self->goodRemoteRecruitmentTime.isReady());
+			while (!self->goodRemoteRecruitmentTime.isReady()) {
+				co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 			}
+		} else {
+			TraceEvent(SevError, "RecruitRemoteFromConfigurationError", self->id).error(err);
+			throw err;
 		}
-		wait(lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY));
+		co_await lowPriorityDelay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY);
 	}
 }
 
-ACTOR Future<Void> newCommitProxies(Reference<ClusterRecoveryData> self, RecruitFromConfigurationReply recr) {
+Future<Void> newCommitProxies(Reference<ClusterRecoveryData> self, RecruitFromConfigurationReply recr) {
 	std::vector<Future<CommitProxyInterface>> initializationReplies;
 	for (int i = 0; i < recr.commitProxies.size(); i++) {
 		InitializeCommitProxyRequest req;
@@ -208,15 +206,13 @@ ACTOR Future<Void> newCommitProxies(Reference<ClusterRecoveryData> self, Recruit
 		                    commit_proxy_failed()));
 	}
 
-	std::vector<CommitProxyInterface> newRecruits = wait(getAll(initializationReplies));
+	std::vector<CommitProxyInterface> newRecruits = co_await getAll(initializationReplies);
 	TraceEvent("CommitProxyInitializationComplete", self->dbgid).log();
 	// It is required for the correctness of COMMIT_ON_FIRST_PROXY that self->commitProxies[0] is the firstCommitProxy.
-	self->commitProxies = newRecruits;
-
-	return Void();
+	self->commitProxies = std::move(newRecruits);
 }
 
-ACTOR Future<Void> newGrvProxies(Reference<ClusterRecoveryData> self, RecruitFromConfigurationReply recr) {
+Future<Void> newGrvProxies(Reference<ClusterRecoveryData> self, RecruitFromConfigurationReply recr) {
 	std::vector<Future<GrvProxyInterface>> initializationReplies;
 	for (int i = 0; i < recr.grvProxies.size(); i++) {
 		InitializeGrvProxyRequest req;
@@ -230,13 +226,12 @@ ACTOR Future<Void> newGrvProxies(Reference<ClusterRecoveryData> self, RecruitFro
 		                    grv_proxy_failed()));
 	}
 
-	std::vector<GrvProxyInterface> newRecruits = wait(getAll(initializationReplies));
+	std::vector<GrvProxyInterface> newRecruits = co_await getAll(initializationReplies);
 	TraceEvent("GrvProxyInitializationComplete", self->dbgid).log();
-	self->grvProxies = newRecruits;
-	return Void();
+	self->grvProxies = std::move(newRecruits);
 }
 
-ACTOR Future<Void> newResolvers(Reference<ClusterRecoveryData> self, RecruitFromConfigurationReply recr) {
+Future<Void> newResolvers(Reference<ClusterRecoveryData> self, RecruitFromConfigurationReply recr) {
 	std::vector<Future<ResolverInterface>> initializationReplies;
 	for (int i = 0; i < recr.resolvers.size(); i++) {
 		InitializeResolverRequest req;
@@ -251,20 +246,18 @@ ACTOR Future<Void> newResolvers(Reference<ClusterRecoveryData> self, RecruitFrom
 		                    resolver_failed()));
 	}
 
-	std::vector<ResolverInterface> newRecruits = wait(getAll(initializationReplies));
+	std::vector<ResolverInterface> newRecruits = co_await getAll(initializationReplies);
 	TraceEvent("ResolverInitializationComplete", self->dbgid).log();
-	self->resolvers = newRecruits;
-
-	return Void();
+	self->resolvers = std::move(newRecruits);
 }
 
-ACTOR Future<Void> newTLogServers(Reference<ClusterRecoveryData> self,
-                                  RecruitFromConfigurationReply recr,
-                                  Reference<ILogSystem> oldLogSystem,
-                                  std::vector<Standalone<CommitTransactionRef>>* initialConfChanges) {
+Future<Void> newTLogServers(Reference<ClusterRecoveryData> self,
+                            RecruitFromConfigurationReply recr,
+                            Reference<LogSystem> oldLogSystem,
+                            std::vector<Standalone<CommitTransactionRef>>* initialConfChanges) {
 	TraceEvent("NewTLogServersStarted", self->dbgid).detail("UsableRegions", self->configuration.usableRegions);
 	if (self->configuration.usableRegions > 1) {
-		state Optional<Key> remoteDcId = self->remoteDcIds.size() ? self->remoteDcIds[0] : Optional<Key>();
+		Optional<Key> remoteDcId = self->remoteDcIds.size() ? self->remoteDcIds[0] : Optional<Key>();
 		if (!self->dcId_locality.contains(recr.dcId)) {
 			int8_t loc = self->getNextLocality();
 			Standalone<CommitTransactionRef> tr;
@@ -300,53 +293,52 @@ ACTOR Future<Void> newTLogServers(Reference<ClusterRecoveryData> self,
 		        std::max<int>(1, self->configuration.desiredLogRouterCount / std::max<int>(1, recr.tLogs.size())),
 		    exclusionWorkerIds);
 		remoteRecruitReq.dbgId = self->dbgid;
-		state Reference<RecruitRemoteWorkersInfo> recruitWorkersInfo =
-		    makeReference<RecruitRemoteWorkersInfo>(remoteRecruitReq);
+		auto recruitWorkersInfo = makeReference<RecruitRemoteWorkersInfo>(remoteRecruitReq);
 		recruitWorkersInfo->dbgId = self->dbgid;
 		Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers =
 		    clusterRecruitRemoteFromConfiguration(self->controllerData, recruitWorkersInfo);
 
 		self->primaryLocality = self->dcId_locality[recr.dcId];
-		self->logSystem = Reference<ILogSystem>(); // Cancels the actors in the previous log system.
-		Reference<ILogSystem> newLogSystem = wait(oldLogSystem->newEpoch(recr,
-		                                                                 fRemoteWorkers,
-		                                                                 self->configuration,
-		                                                                 self->cstate.myDBState.recoveryCount + 1,
-		                                                                 self->recoveryTransactionVersion,
-		                                                                 self->primaryLocality,
-		                                                                 self->dcId_locality[remoteDcId],
-		                                                                 self->allTags,
-		                                                                 self->recruitmentStalled));
+		self->logSystem = Reference<LogSystem>(); // Cancels the actors in the previous log system.
+		Reference<LogSystem> newLogSystem = co_await oldLogSystem->newEpoch(recr,
+		                                                                    fRemoteWorkers,
+		                                                                    self->configuration,
+		                                                                    self->cstate.myDBState.recoveryCount + 1,
+		                                                                    self->recoveryTransactionVersion,
+		                                                                    self->primaryLocality,
+		                                                                    self->dcId_locality[remoteDcId],
+		                                                                    self->allTags,
+		                                                                    self->recruitmentStalled);
 		self->logSystem = newLogSystem;
 	} else {
 		self->primaryLocality = tagLocalitySpecial;
-		self->logSystem = Reference<ILogSystem>(); // Cancels the actors in the previous log system.
-		Reference<ILogSystem> newLogSystem = wait(oldLogSystem->newEpoch(recr,
-		                                                                 Never(),
-		                                                                 self->configuration,
-		                                                                 self->cstate.myDBState.recoveryCount + 1,
-		                                                                 self->recoveryTransactionVersion,
-		                                                                 self->primaryLocality,
-		                                                                 tagLocalitySpecial,
-		                                                                 self->allTags,
-		                                                                 self->recruitmentStalled));
+		self->logSystem = Reference<LogSystem>(); // Cancels the actors in the previous log system.
+		Reference<LogSystem> newLogSystem = co_await oldLogSystem->newEpoch(recr,
+		                                                                    Never(),
+		                                                                    self->configuration,
+		                                                                    self->cstate.myDBState.recoveryCount + 1,
+		                                                                    self->recoveryTransactionVersion,
+		                                                                    self->primaryLocality,
+		                                                                    tagLocalitySpecial,
+		                                                                    self->allTags,
+		                                                                    self->recruitmentStalled);
 		self->logSystem = newLogSystem;
 	}
 	TraceEvent("NewTLogServersFinished", self->dbgid);
-	return Void();
 }
 
-ACTOR Future<Void> newSeedServers(Reference<ClusterRecoveryData> self,
-                                  RecruitFromConfigurationReply recruits,
-                                  std::vector<StorageServerInterface>* servers) {
+Future<Void> newSeedServers(Reference<ClusterRecoveryData> self,
+                            RecruitFromConfigurationReply recruits,
+                            std::vector<StorageServerInterface>* servers) {
 	// This is only necessary if the database is at version 0
 	servers->clear();
-	if (self->lastEpochEnd)
-		return Void();
+	if (self->lastEpochEnd) {
+		co_return;
+	}
 
-	state int idx = 0;
-	state std::map<Optional<Value>, Tag> dcId_tags;
-	state int8_t nextLocality = 0;
+	int idx = 0;
+	std::map<Optional<Value>, Tag> dcId_tags;
+	int8_t nextLocality = 0;
 	while (idx < recruits.storageServers.size()) {
 		TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_SS_RECRUITMENT_EVENT_NAME).c_str(),
 		           self->dbgid)
@@ -361,7 +353,7 @@ ACTOR Future<Void> newSeedServers(Reference<ClusterRecoveryData> self,
 		isr.interfaceId = deterministicRandom()->randomUniqueID();
 		isr.initialClusterVersion = self->recoveryTransactionVersion;
 
-		ErrorOr<InitializeStorageReply> newServer = wait(recruits.storageServers[idx].storage.tryGetReply(isr));
+		ErrorOr<InitializeStorageReply> newServer = co_await recruits.storageServers[idx].storage.tryGetReply(isr);
 
 		if (newServer.isError()) {
 			if (!newServer.isError(error_code_recruitment_failed) &&
@@ -369,7 +361,7 @@ ACTOR Future<Void> newSeedServers(Reference<ClusterRecoveryData> self,
 				throw newServer.getError();
 
 			CODE_PROBE(true, "initial storage recuitment loop failed to get new server");
-			wait(delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY));
+			co_await delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY);
 		} else {
 			if (!dcId_tags.contains(recruits.storageServers[idx].locality.dcId())) {
 				dcId_tags[recruits.storageServers[idx].locality.dcId()] = Tag(nextLocality, 0);
@@ -392,8 +384,6 @@ ACTOR Future<Void> newSeedServers(Reference<ClusterRecoveryData> self,
 	TraceEvent("ClusterRecoveryRecruitedInitialStorageServers", self->dbgid)
 	    .detail("TargetCount", self->configuration.storageTeamSize)
 	    .detail("Servers", describe(*servers));
-
-	return Void();
 }
 
 Future<Void> waitCommitProxyFailure(std::vector<CommitProxyInterface> const& commitProxies) {
@@ -434,9 +424,9 @@ Future<Void> waitResolverFailure(std::vector<ResolverInterface> const& resolvers
 	return tagError<Void>(quorum(failed, 1), resolver_failed());
 }
 
-ACTOR Future<Void> rejoinRequestHandler(Reference<ClusterRecoveryData> self) {
-	loop {
-		TLogRejoinRequest req = waitNext(self->clusterController.tlogRejoin.getFuture());
+Future<Void> rejoinRequestHandler(Reference<ClusterRecoveryData> self) {
+	while (true) {
+		TLogRejoinRequest req = co_await self->clusterController.tlogRejoin.getFuture();
 		TraceEvent(SevDebug, "TLogRejoinRequestHandler")
 		    .detail("MasterLifeTime", self->dbInfo->get().masterLifetime.toString());
 		req.reply.send(true);
@@ -444,15 +434,15 @@ ACTOR Future<Void> rejoinRequestHandler(Reference<ClusterRecoveryData> self) {
 }
 
 // Keeps the coordinated state (cstate) updated as the set of recruited tlogs change through recovery.
-ACTOR Future<Void> trackTlogRecovery(Reference<ClusterRecoveryData> self,
-                                     Reference<AsyncVar<Reference<ILogSystem>>> oldLogSystems,
-                                     Future<Void> minRecoveryDuration) {
-	state Future<Void> rejoinRequests = Never();
-	state DBRecoveryCount recoverCount = self->cstate.myDBState.recoveryCount + 1;
-	state DatabaseConfiguration configuration =
+Future<Void> trackTlogRecovery(Reference<ClusterRecoveryData> self,
+                               Reference<AsyncVar<Reference<LogSystem>>> oldLogSystems,
+                               Future<Void> minRecoveryDuration) {
+	Future<Void> rejoinRequests = Never();
+	DBRecoveryCount recoverCount = self->cstate.myDBState.recoveryCount + 1;
+	DatabaseConfiguration configuration =
 	    self->configuration; // self-configuration can be changed by configurationMonitor so we need a copy
-	loop {
-		state DBCoreState newState;
+	while (true) {
+		DBCoreState newState;
 		self->logSystem->toCoreState(newState);
 		// We can't purge old generations until we have the new state durable on coordinators,
 		// otherwise old tlogs can be removed before the new state is written,
@@ -460,15 +450,14 @@ ACTOR Future<Void> trackTlogRecovery(Reference<ClusterRecoveryData> self,
 		self->logSystem->purgeOldRecoveredGenerationsCoreState(newState);
 		newState.recoveryCount = recoverCount;
 
-		state Future<Void> changed = self->logSystem->onCoreStateChanged();
+		Future<Void> changed = self->logSystem->onCoreStateChanged();
 
 		ASSERT(newState.tLogs[0].tLogWriteAntiQuorum == configuration.tLogWriteAntiQuorum &&
 		       newState.tLogs[0].tLogReplicationFactor == configuration.tLogReplicationFactor);
 
-		state bool allLogs =
-		    newState.tLogs.size() ==
-		    configuration.expectedLogSets(self->primaryDcId.size() ? self->primaryDcId[0] : Optional<Key>());
-		state bool finalUpdate = !newState.oldTLogData.size() && allLogs;
+		bool allLogs = newState.tLogs.size() ==
+		               configuration.expectedLogSets(self->primaryDcId.size() ? self->primaryDcId[0] : Optional<Key>());
+		bool finalUpdate = !newState.oldTLogData.size() && allLogs;
 		TraceEvent("TrackTLogRecovery")
 		    .detail("FinalUpdate", finalUpdate)
 		    .detail("NewState.tlogs", newState.tLogs.size())
@@ -476,14 +465,14 @@ ACTOR Future<Void> trackTlogRecovery(Reference<ClusterRecoveryData> self,
 		    .detail("Expected.tlogs",
 		            configuration.expectedLogSets(self->primaryDcId.size() ? self->primaryDcId[0] : Optional<Key>()))
 		    .detail("RecoveryCount", newState.recoveryCount);
-		wait(self->cstate.write(newState, finalUpdate));
+		co_await self->cstate.write(newState, finalUpdate);
 		// Purge in memory state after durability to avoid race conditions.
 		self->logSystem->purgeOldRecoveredGenerationsInMemory(newState);
 		if (self->cstateUpdated.canBeSet()) {
 			self->cstateUpdated.send(Void());
 		}
 
-		wait(minRecoveryDuration);
+		co_await minRecoveryDuration;
 		self->logSystem->coreStateWritten(newState);
 
 		if (self->recoveryReadyForCommits.canBeSet()) {
@@ -523,19 +512,19 @@ ACTOR Future<Void> trackTlogRecovery(Reference<ClusterRecoveryData> self,
 		if (finalUpdate) {
 			oldLogSystems->get()->stopRejoins();
 			rejoinRequests = rejoinRequestHandler(self);
-			return Void();
+			co_return;
 		}
 
-		wait(changed);
+		co_await changed;
 	}
 }
 
-ACTOR Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
-	loop {
-		ChangeCoordinatorsRequest req = waitNext(self->clusterController.changeCoordinators.getFuture());
+Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
+	while (true) {
+		ChangeCoordinatorsRequest req = co_await self->clusterController.changeCoordinators.getFuture();
 		TraceEvent("ChangeCoordinators", self->dbgid).log();
 		++self->changeCoordinatorsRequests;
-		state ChangeCoordinatorsRequest changeCoordinatorsRequest = req;
+		ChangeCoordinatorsRequest changeCoordinatorsRequest = req;
 		if (self->masterInterface.id() != changeCoordinatorsRequest.masterId) {
 			// Make sure the request is coming from a proxy from the same
 			// generation. If not, throw coordinators_changed - this is OK
@@ -557,18 +546,18 @@ ACTOR Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
 		self->controllerData->shouldCommitSuicide = true;
 
 		while (!self->cstate.previousWrite.isReady()) {
-			wait(self->cstate.previousWrite);
-			wait(delay(
-			    0)); // if a new core state is ready to be written, have that take priority over our finalizing write;
+			co_await self->cstate.previousWrite;
+			co_await delay(
+			    0); // if a new core state is ready to be written, have that take priority over our finalizing write;
 		}
 
 		if (!self->cstate.fullyRecovered.isSet()) {
-			wait(self->cstate.write(self->cstate.myDBState, true));
+			co_await self->cstate.write(self->cstate.myDBState, true);
 		}
 
 		try {
 			ClusterConnectionString conn(changeCoordinatorsRequest.newConnectionString.toString());
-			wait(self->cstate.move(conn));
+			co_await self->cstate.move(conn);
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled)
 				changeCoordinatorsRequest.reply.sendError(e);
@@ -580,14 +569,15 @@ ACTOR Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
 	}
 }
 
-ACTOR Future<Void> configurationMonitor(Reference<ClusterRecoveryData> self, Database cx) {
-	loop {
-		state ReadYourWritesTransaction tr(cx);
+Future<Void> configurationMonitor(Reference<ClusterRecoveryData> self, Database cx) {
+	while (true) {
+		ReadYourWritesTransaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult results = co_await tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
 
 				DatabaseConfiguration conf;
@@ -606,36 +596,31 @@ ACTOR Future<Void> configurationMonitor(Reference<ClusterRecoveryData> self, Dat
 					self->registrationTrigger.trigger();
 				}
 
-				state Future<Void> watchFuture =
-				    tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
-				    tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
-				    tr.watch(failedLocalityVersionKey);
-				wait(tr.commit());
-				wait(watchFuture);
+				Future<Void> watchFuture = tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
+				                           tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
+				                           tr.watch(failedLocalityVersionKey);
+				co_await tr.commit();
+				co_await watchFuture;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 }
 
-// Returns the minimum backup version and the maximum backup worker noop version.
-ACTOR static Future<std::pair<Optional<Version>, Optional<Version>>> getMinBackupVersion(
-    Reference<ClusterRecoveryData> self,
-    Database cx) {
-	loop {
-		state ReadYourWritesTransaction tr(cx);
+// Returns the minimum backup version.
+static Future<Optional<Version>> getMinBackupVersion(Reference<ClusterRecoveryData> self, Database cx) {
+	while (true) {
+		ReadYourWritesTransaction tr(cx);
 
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			state Future<Optional<Value>> fValue = tr.get(backupStartedKey);
-			state Future<Optional<Value>> fNoopValue = tr.get(backupWorkerMaxNoopVersionKey);
-			wait(success(fValue) && success(fNoopValue));
-			Optional<Value> value = fValue.get();
-			Optional<Value> noopValue = fNoopValue.get();
-			Optional<Version> minVersion, noopVersion;
+			Optional<Value> value = co_await tr.get(backupStartedKey);
+			Optional<Version> minVersion;
 			if (value.present()) {
 				auto uidVersions = decodeBackupStartedValue(value.get());
 				TraceEvent e("GotBackupStartKey", self->dbgid);
@@ -648,38 +633,37 @@ ACTOR static Future<std::pair<Optional<Version>, Optional<Version>>> getMinBacku
 			} else {
 				TraceEvent("EmptyBackupStartKey", self->dbgid).log();
 			}
-			if (noopValue.present()) {
-				noopVersion = BinaryReader::fromStringRef<Version>(noopValue.get(), Unversioned());
-			}
-			return std::make_pair(minVersion, noopVersion);
+
+			co_return minVersion;
 
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR static Future<Void> recruitBackupWorkers(Reference<ClusterRecoveryData> self, Database cx) {
+static Future<Void> recruitBackupWorkers(Reference<ClusterRecoveryData> self, Database cx) {
 	ASSERT(self->backupWorkers.size() > 0);
 
 	// Avoid race between a backup worker's save progress and the reads below.
-	wait(delay(SERVER_KNOBS->SECONDS_BEFORE_RECRUIT_BACKUP_WORKER));
+	co_await delay(SERVER_KNOBS->SECONDS_BEFORE_RECRUIT_BACKUP_WORKER);
 
-	state LogEpoch epoch = self->cstate.myDBState.recoveryCount;
-	state Reference<BackupProgress> backupProgress(
+	LogEpoch epoch = self->cstate.myDBState.recoveryCount;
+	Reference<BackupProgress> backupProgress(
 	    new BackupProgress(self->dbgid, self->logSystem->getOldEpochTagsVersionsInfo()));
-	state Future<Void> gotProgress = getBackupProgress(cx, self->dbgid, backupProgress, /*logging=*/true);
-	state std::vector<Future<InitializeBackupReply>> initializationReplies;
+	Future<Void> gotProgress = getBackupProgress(cx, self->dbgid, backupProgress, SevInfo);
+	std::vector<Future<InitializeBackupReply>> initializationReplies;
 
-	state std::vector<std::pair<UID, Tag>> idsTags; // worker IDs and tags for current epoch
-	state int logRouterTags = self->logSystem->getLogRouterTags();
+	std::vector<std::pair<UID, Tag>> idsTags; // worker IDs and tags for current epoch
+	int logRouterTags = self->logSystem->getLogRouterTags();
 	idsTags.reserve(logRouterTags);
 	for (int i = 0; i < logRouterTags; i++) {
 		idsTags.emplace_back(deterministicRandom()->randomUniqueID(), Tag(tagLocalityLogRouter, i));
 	}
 
 	const Version startVersion = self->logSystem->getBackupStartVersion();
-	state int i = 0;
+	int i = 0;
 	for (; i < logRouterTags; i++) {
 		const auto& worker = self->backupWorkers[i % self->backupWorkers.size()];
 		InitializeBackupRequest req(idsTags[i].first);
@@ -700,23 +684,18 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<ClusterRecoveryData> se
 		                    backup_worker_failed()));
 	}
 
-	state Future<std::pair<Optional<Version>, Optional<Version>>> fMinVersion = getMinBackupVersion(self, cx);
-	wait(gotProgress && success(fMinVersion));
-	Optional<Version> minVersion = fMinVersion.get().first;
-	Optional<Version> noopVersion = fMinVersion.get().second;
-	TraceEvent("MinBackupVersion", self->dbgid)
-	    .detail("Version", minVersion.present() ? minVersion.get() : -1)
-	    .detail("NoopVersion", noopVersion.present() ? noopVersion.get() : -1);
+	Future<Optional<Version>> fMinVersion = getMinBackupVersion(self, cx);
+	co_await (gotProgress && success(fMinVersion));
+	Optional<Version> minVersion = fMinVersion.get();
+	TraceEvent("MinBackupVersion", self->dbgid).detail("Version", minVersion.present() ? minVersion.get() : -1);
 
 	std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> toRecruit =
 	    backupProgress->getUnfinishedBackup();
 	for (const auto& [epochVersionTags, tagVersions] : toRecruit) {
 		const Version oldEpochEnd = std::get<1>(epochVersionTags);
-		if ((!minVersion.present() || minVersion.get() + 1 >= oldEpochEnd) ||
-		    (noopVersion.present() && noopVersion.get() >= oldEpochEnd)) {
+		if (!minVersion.present() || minVersion.get() + 1 >= oldEpochEnd) {
 			TraceEvent("SkipBackupRecruitment", self->dbgid)
 			    .detail("MinVersion", minVersion.present() ? minVersion.get() : -1)
-			    .detail("NoopVersion", noopVersion.present() ? noopVersion.get() : -1)
 			    .detail("Epoch", epoch)
 			    .detail("OldEpoch", std::get<0>(epochVersionTags))
 			    .detail("OldEpochEnd", oldEpochEnd);
@@ -746,18 +725,18 @@ ACTOR static Future<Void> recruitBackupWorkers(Reference<ClusterRecoveryData> se
 		}
 	}
 
-	std::vector<InitializeBackupReply> newRecruits = wait(getAll(initializationReplies));
+	std::vector<InitializeBackupReply> newRecruits = co_await getAll(initializationReplies);
 	self->logSystem->setBackupWorkers(newRecruits);
 	TraceEvent("BackupRecruitmentDone", self->dbgid).log();
 	self->registrationTrigger.trigger();
-	return Void();
 }
 
-ACTOR Future<Void> updateLogsValue(Reference<ClusterRecoveryData> self, Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> updateLogsValue(Reference<ClusterRecoveryData> self, Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			Optional<Standalone<StringRef>> value = wait(tr.get(logsKey));
+			Optional<Standalone<StringRef>> value = co_await tr.get(logsKey);
 			ASSERT(value.present());
 			auto logs = decodeLogsValue(value.get());
 
@@ -781,15 +760,16 @@ ACTOR Future<Void> updateLogsValue(Reference<ClusterRecoveryData> self, Database
 
 			if (!found) {
 				CODE_PROBE(true, "old master attempted to change logsKey", probe::decoration::rare);
-				return Void();
+				co_return;
 			}
 
 			tr.set(logsKey, self->logSystem->getLogsValue());
-			wait(tr.commit());
-			return Void();
+			co_await tr.commit();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -819,14 +799,14 @@ void sendMasterRegistration(ClusterRecoveryData* self,
 	clusterRegisterMaster(self->controllerData, masterReq);
 }
 
-ACTOR Future<Void> updateRegistration(Reference<ClusterRecoveryData> self, Reference<ILogSystem> logSystem) {
-	state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
-	state Future<Void> trigger = self->registrationTrigger.onTrigger();
-	state Future<Void> updateLogsKey;
+Future<Void> updateRegistration(Reference<ClusterRecoveryData> self, Reference<LogSystem> logSystem) {
+	Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
+	Future<Void> trigger = self->registrationTrigger.onTrigger();
+	Future<Void> updateLogsKey;
 
-	loop {
-		wait(trigger);
-		wait(delay(.001)); // Coalesce multiple changes
+	while (true) {
+		co_await trigger;
+		co_await delay(.001); // Coalesce multiple changes
 
 		trigger = self->registrationTrigger.onTrigger();
 
@@ -864,33 +844,30 @@ ACTOR Future<Void> updateRegistration(Reference<ClusterRecoveryData> self, Refer
 	}
 }
 
-ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<ClusterRecoveryData> parent,
-                                                                 Future<Void> activate) {
-	wait(activate);
-
-	// Register a fake commit proxy (to be provided right here) to make ourselves available to clients
-	parent->provisionalCommitProxies = std::vector<CommitProxyInterface>(1);
-	parent->provisionalCommitProxies[0].provisional = true;
-	parent->provisionalCommitProxies[0].initEndpoints();
-	parent->provisionalGrvProxies = std::vector<GrvProxyInterface>(1);
-	parent->provisionalGrvProxies[0].provisional = true;
-	parent->provisionalGrvProxies[0].initEndpoints();
-	state Future<Void> waitCommitProxyFailure =
-	    waitFailureServer(parent->provisionalCommitProxies[0].waitFailure.getFuture());
-	state Future<Void> waitGrvProxyFailure =
-	    waitFailureServer(parent->provisionalGrvProxies[0].waitFailure.getFuture());
-	parent->registrationTrigger.trigger();
-
-	auto lockedKey = parent->txnStateStore->readValue(databaseLockedKey).get();
-	state bool locked = lockedKey.present() && lockedKey.get().size();
-
-	state Optional<Value> metadataVersion = parent->txnStateStore->readValue(metadataVersionKey).get();
-
-	// We respond to a minimal subset of the commit proxy protocol.  Our sole purpose is to receive a single write-only
-	// transaction which might repair our configuration, and return it.
-	loop choose {
-		when(GetReadVersionRequest req =
-		         waitNext(parent->provisionalGrvProxies[0].getConsistentReadVersion.getFuture())) {
+class ProvisionalMaster {
+	Reference<ClusterRecoveryData> parent;
+	bool locked = false;
+	Optional<Value> metadataVersion;
+
+	void initializeProxies() {
+		// Register a fake commit proxy (to be provided right here) to make ourselves available to clients
+		parent->provisionalCommitProxies = std::vector<CommitProxyInterface>(1);
+		parent->provisionalCommitProxies[0].provisional = true;
+		parent->provisionalCommitProxies[0].initEndpoints();
+		parent->provisionalGrvProxies = std::vector<GrvProxyInterface>(1);
+		parent->provisionalGrvProxies[0].provisional = true;
+		parent->provisionalGrvProxies[0].initEndpoints();
+		parent->registrationTrigger.trigger();
+
+		auto lockedKey = parent->txnStateStore->readValue(databaseLockedKey).get();
+		locked = lockedKey.present() && lockedKey.get().size();
+
+		metadataVersion = parent->txnStateStore->readValue(metadataVersionKey).get();
+	}
+
+	Future<Void> serveGetReadVersions() {
+		while (true) {
+			GetReadVersionRequest req = co_await parent->provisionalGrvProxies[0].getConsistentReadVersion.getFuture();
 			if ((req.flags & GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY) &&
 			    (req.flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES) && parent->lastEpochEnd) {
 				GetReadVersionReply rep;
@@ -899,10 +876,15 @@ ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<Clust
 				rep.metadataVersion = metadataVersion;
 				rep.proxyId = parent->provisionalGrvProxies[0].id();
 				req.reply.send(rep);
-			} else
+			} else {
 				req.reply.send(Never()); // We can't perform causally consistent reads without recovering
+			}
 		}
-		when(CommitTransactionRequest req = waitNext(parent->provisionalCommitProxies[0].commit.getFuture())) {
+	}
+
+	Future<Standalone<CommitTransactionRef>> serveCommits() {
+		while (true) {
+			CommitTransactionRequest req = co_await parent->provisionalCommitProxies[0].commit.getFuture();
 			req.reply.send(Never()); // don't reply (clients always get commit_unknown_result)
 			auto t = &req.transaction;
 			if (t->read_snapshot == parent->lastEpochEnd && //< So no transactions can fall between the read snapshot
@@ -927,22 +909,52 @@ ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<Clust
 						out.mutations.append_deep(out.arena(), t->mutations.begin(), t->mutations.size());
 						out.write_conflict_ranges.append_deep(
 						    out.arena(), t->write_conflict_ranges.begin(), t->write_conflict_ranges.size());
-						return out;
+						co_return out;
 					}
 				}
 			}
 		}
-		when(GetKeyServerLocationsRequest req =
-		         waitNext(parent->provisionalCommitProxies[0].getKeyServersLocations.getFuture())) {
+	}
+
+	Future<Void> serveGetKeyServerLocations() {
+		while (true) {
+			GetKeyServerLocationsRequest req =
+			    co_await parent->provisionalCommitProxies[0].getKeyServersLocations.getFuture();
 			req.reply.send(Never());
 		}
-		when(wait(waitCommitProxyFailure)) {
-			throw worker_removed();
-		}
-		when(wait(waitGrvProxyFailure)) {
-			throw worker_removed();
-		}
 	}
+
+	Future<Void> waitForCommitProxyFailure() {
+		co_await waitFailureServer(parent->provisionalCommitProxies[0].waitFailure.getFuture());
+		throw worker_removed();
+	}
+
+	Future<Void> waitForGrvProxyFailure() {
+		co_await waitFailureServer(parent->provisionalGrvProxies[0].waitFailure.getFuture());
+		throw worker_removed();
+	}
+
+public:
+	explicit ProvisionalMaster(Reference<ClusterRecoveryData> parent) : parent(parent) {}
+
+	Future<Standalone<CommitTransactionRef>> run(Future<Void> activate) {
+		co_await activate;
+		initializeProxies();
+
+		auto result = co_await race(serveCommits(),
+		                            serveGetReadVersions(),
+		                            serveGetKeyServerLocations(),
+		                            waitForCommitProxyFailure(),
+		                            waitForGrvProxyFailure());
+		ASSERT(result.index() == 0);
+		co_return std::get<0>(std::move(result));
+	}
+};
+
+Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<ClusterRecoveryData> parent,
+                                                           Future<Void> activate) {
+	ProvisionalMaster master(parent);
+	co_return co_await master.run(activate);
 }
 
 // Monitors the initialization of transaction system roles (commit proxies, GRV proxies, resolvers, TLogs, LogRouters)
@@ -952,7 +964,7 @@ ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<Clust
 // - By default: base timeout (30s) doubles with each unfinished recovery: 30s, 60s, 120s, 240s, up to max (300s)
 // - This prevents rapid recovery retry loops while allowing quick initial attempts
 // - All timeout values are configurable via SERVER_KNOBS
-ACTOR Future<Void> monitorInitializingTxnSystem(int unfinishedRecoveries) {
+Future<Void> monitorInitializingTxnSystem(int unfinishedRecoveries) {
 	// Validate parameters to prevent overflow and ensure exponential backoff works correctly
 	// With growth factor <= 10 and unfinishedRecoveries <= 100, max scaling factor is 10^100
 	const bool validParameters = unfinishedRecoveries >= 1 && SERVER_KNOBS->CC_RECOVERY_INIT_REQ_TIMEOUT > 0 &&
@@ -969,7 +981,7 @@ ACTOR Future<Void> monitorInitializingTxnSystem(int unfinishedRecoveries) {
 		    .detail("MaxUnfinishedRecoveries", SERVER_KNOBS->CC_RECOVERY_INIT_REQ_MAX_UNFINISHED_RECOVERIES);
 		ASSERT_WE_THINK(false); // it is expected for these parameters to always be valid so we assert/crash in
 		                        // simulation if that's not the case
-		return Never();
+		co_await Future<Void>(Never());
 	}
 
 	const bool tooManyUnfinishedRecoveries =
@@ -981,9 +993,9 @@ ACTOR Future<Void> monitorInitializingTxnSystem(int unfinishedRecoveries) {
 		    .detail("MaxTimeout", SERVER_KNOBS->CC_RECOVERY_INIT_REQ_MAX_TIMEOUT)
 		    .detail("UnfinishedRecoveries", unfinishedRecoveries)
 		    .detail("MaxUnfinishedRecoveries", SERVER_KNOBS->CC_RECOVERY_INIT_REQ_MAX_UNFINISHED_RECOVERIES);
-		return Never(); // if there have been too many recoveries, clearly something is wrong. At this point, an
-		                // operator needs to look into the issue rather than us relying on this timeout monitor.
-		                // Triggering more timeouts can make the situation worse.
+		co_await Future<Void>(Never()); // if there have been too many recoveries, clearly something is wrong. At this
+		                                // point, an operator needs to look into the issue rather than us relying on
+		                                // this timeout monitor. Triggering more timeouts can make the situation worse.
 	}
 
 	// Calculate timeout with exponential backoff
@@ -999,16 +1011,16 @@ ACTOR Future<Void> monitorInitializingTxnSystem(int unfinishedRecoveries) {
 	    .detail("ScalingFactor", scalingFactor)
 	    .detail("ScaledTimeout", scaledTimeout);
 
-	wait(delay(scaledTimeout));
+	co_await delay(scaledTimeout);
 
 	TraceEvent("InitializingTxnSystemTimeoutTriggered");
 	throw cluster_recovery_failed();
 }
 
-ACTOR Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
+Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
     Reference<ClusterRecoveryData> self,
     std::vector<StorageServerInterface>* seedServers,
-    Reference<ILogSystem> oldLogSystem) {
+    Reference<LogSystem> oldLogSystem) {
 	if (!self->configuration.isValid()) {
 		RecoveryStatus::RecoveryStatus status;
 		if (self->configuration.initialized) {
@@ -1031,7 +1043,7 @@ ACTOR Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
 		    .detail("StatusCode", status)
 		    .detail("Status", RecoveryStatus::names[status])
 		    .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey);
-		return Never();
+		co_await Future<Void>(Never());
 	} else {
 		TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(),
 		           self->dbgid)
@@ -1052,10 +1064,10 @@ ACTOR Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
 	}
 
 	RecruitFromConfigurationRequest recruitReq(self->configuration, self->lastEpochEnd == 0, maxLogRouters);
-	state Reference<RecruitWorkersInfo> recruitWorkersInfo = makeReference<RecruitWorkersInfo>(recruitReq);
+	auto recruitWorkersInfo = makeReference<RecruitWorkersInfo>(recruitReq);
 	recruitWorkersInfo->dbgId = self->dbgid;
-	wait(clusterRecruitFromConfiguration(self->controllerData, recruitWorkersInfo));
-	state RecruitFromConfigurationReply recruits = recruitWorkersInfo->rep;
+	co_await clusterRecruitFromConfiguration(self->controllerData, recruitWorkersInfo);
+	RecruitFromConfigurationReply recruits = recruitWorkersInfo->rep;
 
 	std::string primaryDcIds, remoteDcIds;
 
@@ -1103,32 +1115,32 @@ ACTOR Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
 	// brand new database we are sort of lying that we are past the recruitment phase.  In a perfect world we would
 	// split that up so that the recruitment part happens above (in parallel with recruiting the transaction
 	// servers?).
-	wait(newSeedServers(self, recruits, seedServers));
+	co_await newSeedServers(self, recruits, seedServers);
 
-	state std::vector<Standalone<CommitTransactionRef>> confChanges;
+	std::vector<Standalone<CommitTransactionRef>> confChanges;
 	Future<Void> txnSystemInitialized =
 	    traceAfter(newCommitProxies(self, recruits), "CommitProxiesInitialized") &&
 	    traceAfter(newGrvProxies(self, recruits), "GRVProxiesInitialized") &&
 	    traceAfter(newResolvers(self, recruits), "ResolversInitialized") &&
 	    traceAfter(newTLogServers(self, recruits, oldLogSystem, &confChanges), "TLogServersInitialized");
-	wait(txnSystemInitialized || monitorInitializingTxnSystem(self->controllerData->db.unfinishedRecoveries));
+	co_await (txnSystemInitialized || monitorInitializingTxnSystem(self->controllerData->db.unfinishedRecoveries));
 
 	// Update recovery related information to the newly elected sequencer (master) process.
-	wait(brokenPromiseToNever(
+	co_await brokenPromiseToNever(
 	    self->masterInterface.updateRecoveryData.getReply(UpdateRecoveryDataRequest(self->recoveryTransactionVersion,
 	                                                                                self->lastEpochEnd,
 	                                                                                self->commitProxies,
 	                                                                                self->resolvers,
 	                                                                                self->versionEpoch,
-	                                                                                self->primaryLocality))));
+	                                                                                self->primaryLocality)));
 
-	return confChanges;
+	co_return confChanges;
 }
 
-ACTOR Future<Void> updateLocalityForDcId(Optional<Key> dcId,
-                                         Reference<ILogSystem> oldLogSystem,
-                                         Reference<AsyncVar<PeekTxsInfo>> locality) {
-	loop {
+Future<Void> updateLocalityForDcId(Optional<Key> dcId,
+                                   Reference<LogSystem> oldLogSystem,
+                                   Reference<AsyncVar<PeekTxsInfo>> locality) {
+	while (true) {
 		std::pair<int8_t, int8_t> loc = oldLogSystem->getLogSystemConfig().getLocalityForDcId(dcId);
 		Version ver = locality->get().knownCommittedVersion;
 		if (ver == invalidVersion) {
@@ -1141,16 +1153,16 @@ ACTOR Future<Void> updateLocalityForDcId(Optional<Key> dcId,
 		    .detail("Locality0", loc.first)
 		    .detail("Locality1", loc.second)
 		    .detail("Version", ver);
-		wait(oldLogSystem->onLogSystemConfigChange() || oldLogSystem->onKnownCommittedVersionChange());
+		co_await (oldLogSystem->onLogSystemConfigChange() || oldLogSystem->onKnownCommittedVersionChange());
 	}
 }
 
-ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> self,
-                                              Reference<ILogSystem> oldLogSystem,
-                                              Version txsPoppedVersion) {
-	state Reference<AsyncVar<PeekTxsInfo>> myLocality = Reference<AsyncVar<PeekTxsInfo>>(
-	    new AsyncVar<PeekTxsInfo>(PeekTxsInfo(tagLocalityInvalid, tagLocalityInvalid, invalidVersion)));
-	state Future<Void> localityUpdater =
+Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> self,
+                                        Reference<LogSystem> oldLogSystem,
+                                        Version txsPoppedVersion) {
+	Reference<AsyncVar<PeekTxsInfo>> myLocality =
+	    makeReference<AsyncVar<PeekTxsInfo>>(PeekTxsInfo(tagLocalityInvalid, tagLocalityInvalid, invalidVersion));
+	Future<Void> localityUpdater =
 	    updateLocalityForDcId(self->masterInterface.locality.dcId(), oldLogSystem, myLocality);
 	// Peek the txnStateTag in oldLogSystem and recover self->txnStateStore
 
@@ -1169,14 +1181,14 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 	                                             self->dbInfo,
 	                                             self->dbgid,
 	                                             self->memoryLimit,
-	                                             /*disableSnapshot=*/false,
-	                                             /*replaceContent=*/false,
-	                                             /*exactRecovery=*/true);
+	                                             DisableSnapshot::False,
+	                                             ReplaceContent::False,
+	                                             ExactRecovery::True);
 
 	// Version 0 occurs at the version epoch. The version epoch is the number
 	// of microseconds since the Unix epoch. It can be set through fdbcli.
 	self->versionEpoch.reset();
-	Optional<Standalone<StringRef>> versionEpochValue = wait(self->txnStateStore->readValue(versionEpochKey));
+	Optional<Standalone<StringRef>> versionEpochValue = co_await self->txnStateStore->readValue(versionEpochKey);
 	if (versionEpochValue.present()) {
 		self->versionEpoch = BinaryReader::fromStringRef<int64_t>(versionEpochValue.get(), Unversioned());
 	}
@@ -1185,7 +1197,7 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 	// that we may recover to, as they embed the version in user-readable data and require that no
 	// transactions will be committed at a lower version.
 	Optional<Standalone<StringRef>> requiredCommitVersion =
-	    wait(self->txnStateStore->readValue(minRequiredCommitVersionKey));
+	    co_await self->txnStateStore->readValue(minRequiredCommitVersionKey);
 	Version minRequiredCommitVersion = -1;
 	if (requiredCommitVersion.present()) {
 		minRequiredCommitVersion = BinaryReader::fromStringRef<Version>(requiredCommitVersion.get(), Unversioned());
@@ -1214,7 +1226,9 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 		// When the version epoch is enabled, versions stay in sync with time.
 		// An offline cluster could see a large version jump when it comes back
 		// online, so test this behavior in simulation.
-		if (BUGGIFY) {
+		// Only inflate on first generation (no old TLog data) to avoid compound
+		// version growth across rapid recovery loops that can reach 20+ billion.
+		if (BUGGIFY && self->cstate.myDBState.oldTLogData.empty()) {
 			self->recoveryTransactionVersion += deterministicRandom()->randomInt64(0, 10000000);
 		}
 	}
@@ -1224,7 +1238,7 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 	    .detail("LastEpochEnd", self->lastEpochEnd)
 	    .detail("RecoveryTransactionVersion", self->recoveryTransactionVersion);
 
-	RangeResult rawConf = wait(self->txnStateStore->readRange(configKeys));
+	RangeResult rawConf = co_await self->txnStateStore->readRange(configKeys);
 	self->configuration.fromKeyValues(rawConf.castTo<VectorRef<KeyValueRef>>());
 	self->originalConfiguration = self->configuration;
 	self->hasConfiguration = true;
@@ -1236,13 +1250,13 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 	    .detail("Conf", self->configuration.toString())
 	    .trackLatest(self->recoveredConfigEventHolder->trackingKey);
 
-	RangeResult rawLocalities = wait(self->txnStateStore->readRange(tagLocalityListKeys));
+	RangeResult rawLocalities = co_await self->txnStateStore->readRange(tagLocalityListKeys);
 	self->dcId_locality.clear();
 	for (auto& kv : rawLocalities) {
 		self->dcId_locality[decodeTagLocalityListKey(kv.key)] = decodeTagLocalityListValue(kv.value);
 	}
 
-	RangeResult rawTags = wait(self->txnStateStore->readRange(serverTagKeys));
+	RangeResult rawTags = co_await self->txnStateStore->readRange(serverTagKeys);
 	self->allTags.clear();
 
 	if (self->forceRecovery) {
@@ -1259,7 +1273,7 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 		}
 	}
 
-	RangeResult rawHistoryTags = wait(self->txnStateStore->readRange(serverTagHistoryKeys));
+	RangeResult rawHistoryTags = co_await self->txnStateStore->readRange(serverTagHistoryKeys);
 	for (auto& kv : rawHistoryTags) {
 		self->allTags.push_back(decodeServerTagValue(kv.value));
 	}
@@ -1272,23 +1286,21 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 	                             // the discardCommit() below and not need a writable log adapter
 
 	TraceEvent("RTSSComplete", self->dbgid).log();
-
-	return Void();
 }
 
-ACTOR Future<Void> sendInitialCommitToResolvers(Reference<ClusterRecoveryData> self) {
-	state KeyRange txnKeys = allKeys;
-	state Sequence txnSequence = 0;
+Future<Void> sendInitialCommitToResolvers(Reference<ClusterRecoveryData> self) {
+	KeyRange txnKeys = allKeys;
+	Sequence txnSequence = 0;
 	ASSERT(self->recoveryTransactionVersion);
 
-	state RangeResult data =
+	RangeResult data =
 	    self->txnStateStore
 	        ->readRange(txnKeys, BUGGIFY ? 3 : SERVER_KNOBS->DESIRED_TOTAL_BYTES, SERVER_KNOBS->DESIRED_TOTAL_BYTES)
 	        .get();
-	state std::vector<Future<Void>> txnReplies;
-	state int64_t dataOutstanding = 0;
+	std::vector<Future<Void>> txnReplies;
+	int64_t dataOutstanding = 0;
 
-	state std::vector<Endpoint> endpoints;
+	std::vector<Endpoint> endpoints;
 	for (auto& it : self->commitProxies) {
 		endpoints.push_back(it.txnState.getEndpoint());
 	}
@@ -1298,7 +1310,7 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<ClusterRecoveryData> s
 			endpoints.push_back(it.txnState.getEndpoint());
 		}
 	}
-	loop {
+	while (true) {
 		if (!data.size())
 			break;
 		((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
@@ -1319,14 +1331,14 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<ClusterRecoveryData> s
 		txnSequence++;
 
 		if (dataOutstanding > SERVER_KNOBS->MAX_TXS_SEND_MEMORY) {
-			wait(waitForAll(txnReplies));
+			co_await waitForAll(txnReplies);
 			txnReplies = std::vector<Future<Void>>();
 			dataOutstanding = 0;
 		}
 
-		wait(yield());
+		co_await yield();
 	}
-	wait(waitForAll(txnReplies));
+	co_await waitForAll(txnReplies);
 	TraceEvent("RecoveryInternal", self->dbgid)
 	    .detail("StatusCode", RecoveryStatus::recovery_transaction)
 	    .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
@@ -1344,35 +1356,34 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<ClusterRecoveryData> s
 		replies.push_back(brokenPromiseToNever(r.resolve.getReply(req)));
 	}
 
-	wait(waitForAll(replies));
+	co_await waitForAll(replies);
 	TraceEvent("RecoveryInternal", self->dbgid)
 	    .detail("StatusCode", RecoveryStatus::recovery_transaction)
 	    .detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
 	    .detail("RecoveryTxnVersion", self->recoveryTransactionVersion)
 	    .detail("LastEpochEnd", self->lastEpochEnd)
 	    .detail("Step", "InitializedAllResolvers");
-	return Void();
 }
 
-ACTOR Future<Void> triggerUpdates(Reference<ClusterRecoveryData> self, Reference<ILogSystem> oldLogSystem) {
-	loop {
-		wait(oldLogSystem->onLogSystemConfigChange() || self->cstate.fullyRecovered.getFuture() ||
-		     self->recruitmentStalled->onChange());
+Future<Void> triggerUpdates(Reference<ClusterRecoveryData> self, Reference<LogSystem> oldLogSystem) {
+	while (true) {
+		co_await race(oldLogSystem->onLogSystemConfigChange(),
+		              self->cstate.fullyRecovered.getFuture(),
+		              self->recruitmentStalled->onChange());
 		if (self->cstate.fullyRecovered.isSet())
-			return Void();
+			co_return;
 
 		self->registrationTrigger.trigger();
 	}
 }
 
-ACTOR Future<Void> discardCommit(IKeyValueStore* store, LogSystemDiskQueueAdapter* adapter) {
-	state Future<LogSystemDiskQueueAdapter::CommitMessage> fcm = adapter->getCommitMessage();
-	state Future<Void> committed = store->commit();
-	LogSystemDiskQueueAdapter::CommitMessage cm = wait(fcm);
+Future<Void> discardCommit(IKeyValueStore* store, LogSystemDiskQueueAdapter* adapter) {
+	Future<LogSystemDiskQueueAdapter::CommitMessage> fcm = adapter->getCommitMessage();
+	Future<Void> committed = store->commit();
+	LogSystemDiskQueueAdapter::CommitMessage cm = co_await fcm;
 	ASSERT(!committed.isReady());
 	cm.acknowledge.send(Void());
 	ASSERT(committed.isReady());
-	return Void();
 }
 
 void updateConfigForForcedRecovery(Reference<ClusterRecoveryData> self,
@@ -1414,11 +1425,11 @@ void updateConfigForForcedRecovery(Reference<ClusterRecoveryData> self,
 	initialConfChanges->push_back(regionCommit);
 }
 
-ACTOR Future<Void> recoverFrom(Reference<ClusterRecoveryData> self,
-                               Reference<ILogSystem> oldLogSystem,
-                               std::vector<StorageServerInterface>* seedServers,
-                               std::vector<Standalone<CommitTransactionRef>>* initialConfChanges,
-                               Future<Version> poppedTxsVersion) {
+Future<Void> recoverFrom(Reference<ClusterRecoveryData> self,
+                         Reference<LogSystem> oldLogSystem,
+                         std::vector<StorageServerInterface>* seedServers,
+                         std::vector<Standalone<CommitTransactionRef>>* initialConfChanges,
+                         Future<Version> poppedTxsVersion) {
 	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid)
 	    .detail("StatusCode", RecoveryStatus::reading_transaction_system_state)
 	    .detail("Status", RecoveryStatus::names[RecoveryStatus::reading_transaction_system_state])
@@ -1426,10 +1437,10 @@ ACTOR Future<Void> recoverFrom(Reference<ClusterRecoveryData> self,
 	self->hasConfiguration = false;
 
 	if (BUGGIFY)
-		wait(delay(10.0));
+		co_await delay(10.0);
 
-	Version txsPoppedVersion = wait(poppedTxsVersion);
-	wait(readTransactionSystemState(self, oldLogSystem, txsPoppedVersion));
+	Version txsPoppedVersion = co_await poppedTxsVersion;
+	co_await readTransactionSystemState(self, oldLogSystem, txsPoppedVersion);
 	for (auto& itr : *initialConfChanges) {
 		for (auto& m : itr.mutations) {
 			self->configuration.applyMutation(m);
@@ -1446,63 +1457,60 @@ ACTOR Future<Void> recoverFrom(Reference<ClusterRecoveryData> self,
 	// second, a provisional master is initialized, and an "emergency transaction" is submitted that might change the
 	// configuration so that we can finish recovery.
 
-	state std::map<Optional<Value>, int8_t> originalLocalityMap = self->dcId_locality;
-	state Future<std::vector<Standalone<CommitTransactionRef>>> recruitments =
+	std::map<Optional<Value>, int8_t> originalLocalityMap = self->dcId_locality;
+	Future<std::vector<Standalone<CommitTransactionRef>>> recruitments =
 	    recruitEverything(self, seedServers, oldLogSystem);
-	state double provisionalDelay = SERVER_KNOBS->PROVISIONAL_START_DELAY;
-	loop {
-		state Future<Standalone<CommitTransactionRef>> provisional = provisionalMaster(self, delay(provisionalDelay));
+	double provisionalDelay = SERVER_KNOBS->PROVISIONAL_START_DELAY;
+	while (true) {
+		Future<Standalone<CommitTransactionRef>> provisional = provisionalMaster(self, delay(provisionalDelay));
 		provisionalDelay =
 		    std::min(SERVER_KNOBS->PROVISIONAL_MAX_DELAY, provisionalDelay * SERVER_KNOBS->PROVISIONAL_DELAY_GROWTH);
-		choose {
-			when(std::vector<Standalone<CommitTransactionRef>> confChanges = wait(recruitments)) {
-				initialConfChanges->insert(initialConfChanges->end(), confChanges.begin(), confChanges.end());
-				provisional.cancel();
-				break;
-			}
-			when(Standalone<CommitTransactionRef> _req = wait(provisional)) {
-				state Standalone<CommitTransactionRef> req = _req; // mutable
-				CODE_PROBE(true, "Emergency transaction processing during recovery");
-				TraceEvent("EmergencyTransaction", self->dbgid).log();
-				for (auto m = req.mutations.begin(); m != req.mutations.end(); ++m)
-					TraceEvent("EmergencyTransactionMutation", self->dbgid)
-					    .detail("MType", m->type)
-					    .detail("P1", m->param1)
-					    .detail("P2", m->param2);
-
-				DatabaseConfiguration oldConf = self->configuration;
+		auto res = co_await race(recruitments, provisional);
+		if (res.index() == 0) {
+			std::vector<Standalone<CommitTransactionRef>> confChanges = std::get<0>(std::move(res));
+			initialConfChanges->insert(initialConfChanges->end(), confChanges.begin(), confChanges.end());
+			provisional.cancel();
+			break;
+		} else if (res.index() == 1) {
+			Standalone<CommitTransactionRef> req = std::get<1>(std::move(res));
+			CODE_PROBE(true, "Emergency transaction processing during recovery");
+			TraceEvent("EmergencyTransaction", self->dbgid).log();
+			for (auto m = req.mutations.begin(); m != req.mutations.end(); ++m)
+				TraceEvent("EmergencyTransactionMutation", self->dbgid)
+				    .detail("MType", m->type)
+				    .detail("P1", m->param1)
+				    .detail("P2", m->param2);
+
+			DatabaseConfiguration oldConf = self->configuration;
+			self->configuration = self->originalConfiguration;
+			for (auto& m : req.mutations)
+				self->configuration.applyMutation(m);
+
+			initialConfChanges->clear();
+			if (self->originalConfiguration.isValid() &&
+			    self->configuration.usableRegions != self->originalConfiguration.usableRegions) {
+				TraceEvent(SevWarnAlways, "CannotChangeUsableRegions", self->dbgid).log();
 				self->configuration = self->originalConfiguration;
-				for (auto& m : req.mutations)
-					self->configuration.applyMutation(m);
-
-				initialConfChanges->clear();
-				if (self->originalConfiguration.isValid() &&
-				    self->configuration.usableRegions != self->originalConfiguration.usableRegions) {
-					TraceEvent(SevWarnAlways, "CannotChangeUsableRegions", self->dbgid).log();
-					self->configuration = self->originalConfiguration;
-				} else {
-					initialConfChanges->push_back(req);
-				}
-				if (self->forceRecovery) {
-					updateConfigForForcedRecovery(self, initialConfChanges);
-				}
+			} else {
+				initialConfChanges->push_back(req);
+			}
+			if (self->forceRecovery) {
+				updateConfigForForcedRecovery(self, initialConfChanges);
+			}
 
-				if (self->configuration != oldConf) { // confChange does not trigger when including servers
-					self->dcId_locality = originalLocalityMap;
-					recruitments = recruitEverything(self, seedServers, oldLogSystem);
-				}
+			if (self->configuration != oldConf) { // confChange does not trigger when including servers
+				self->dcId_locality = originalLocalityMap;
+				recruitments = recruitEverything(self, seedServers, oldLogSystem);
 			}
 		}
 
 		provisional.cancel();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
-	state TraceInterval recoveryInterval("ClusterRecovery");
-	state double recoverStartTime = now();
+Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
+	TraceInterval recoveryInterval("ClusterRecovery");
+	double recoverStartTime = now();
 
 	self->addActor.send(waitFailureServer(self->masterInterface.waitFailure.getFuture()));
 
@@ -1514,7 +1522,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	    .detail("Status", RecoveryStatus::names[RecoveryStatus::reading_coordinated_state])
 	    .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey);
 
-	wait(self->cstate.read());
+	co_await self->cstate.read();
 
 	if (self->cstate.prevDBState.lowestCompatibleProtocolVersion > currentProtocolVersion()) {
 		TraceEvent(SevWarnAlways, "IncompatibleProtocolVersion", self->dbgid).log();
@@ -1547,7 +1555,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 			            "Recovery stopped because too many recoveries have happened since the last time the cluster "
 			            "was fully_recovered. Set --knob-max-generations-override on your server processes to a value "
 			            "larger than OldGenerations to resume recovery once the underlying problem has been fixed.");
-			wait(Future<Void>(Never()));
+			co_await Future<Void>(Never());
 		} else if (self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION) {
 			TraceEvent(SevError, "RecoveryDelayedTooManyOldGenerations")
 			    .detail("OldGenerations", self->cstate.myDBState.oldTLogData.size())
@@ -1555,22 +1563,21 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 			            "Recovery is delayed because too many recoveries have happened since the last time the cluster "
 			            "was fully_recovered. Set --knob-max-generations-override on your server processes to a value "
 			            "larger than OldGenerations to resume recovery once the underlying problem has been fixed.");
-			wait(delay(CLIENT_KNOBS->RECOVERY_DELAY_SECONDS_PER_GENERATION *
-			           (self->cstate.myDBState.oldTLogData.size() - CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION)));
+			co_await delay(CLIENT_KNOBS->RECOVERY_DELAY_SECONDS_PER_GENERATION *
+			               (self->cstate.myDBState.oldTLogData.size() - CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION));
 		}
 		if (g_network->isSimulated() && self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_SIM) {
 			disableConnectionFailures("TooManyGenerations");
 		}
 	}
 
-	state Reference<AsyncVar<Reference<ILogSystem>>> oldLogSystems(new AsyncVar<Reference<ILogSystem>>);
-	state Future<Void> recoverAndEndEpoch =
-	    ILogSystem::recoverAndEndEpoch(oldLogSystems,
-	                                   self->dbgid,
-	                                   self->cstate.prevDBState,
-	                                   self->clusterController.tlogRejoin.getFuture(),
-	                                   self->controllerData->db.serverInfo->get().myLocality,
-	                                   std::addressof(self->forceRecovery));
+	Reference<AsyncVar<Reference<LogSystem>>> oldLogSystems(new AsyncVar<Reference<LogSystem>>);
+	Future<Void> recoverAndEndEpoch = recoverAndEndLogSystemEpoch(oldLogSystems,
+	                                                              self->dbgid,
+	                                                              self->cstate.prevDBState,
+	                                                              self->clusterController.tlogRejoin.getFuture(),
+	                                                              self->controllerData->db.serverInfo->get().myLocality,
+	                                                              std::addressof(self->forceRecovery));
 
 	DBCoreState newState = self->cstate.myDBState;
 	newState.recoveryCount++;
@@ -1581,7 +1588,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 		newState.newestProtocolVersion = currentProtocolVersion();
 		newState.lowestCompatibleProtocolVersion = minCompatibleProtocolVersion;
 	}
-	wait(self->cstate.write(newState) || recoverAndEndEpoch);
+	co_await (self->cstate.write(newState) || recoverAndEndEpoch);
 
 	TraceEvent("ProtocolVersionCompatibilityChecked", self->dbgid)
 	    .detail("NewestProtocolVersion", self->cstate.myDBState.newestProtocolVersion)
@@ -1590,14 +1597,14 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	self->recoveryState = RecoveryState::RECRUITING;
 
-	state std::vector<StorageServerInterface> seedServers;
-	state std::vector<Standalone<CommitTransactionRef>> initialConfChanges;
-	state Future<Void> logChanges;
-	state Future<Void> minRecoveryDuration;
-	state Future<Version> poppedTxsVersion;
+	std::vector<StorageServerInterface> seedServers;
+	std::vector<Standalone<CommitTransactionRef>> initialConfChanges;
+	Future<Void> logChanges;
+	Future<Void> minRecoveryDuration;
+	Future<Version> poppedTxsVersion;
 
-	loop {
-		Reference<ILogSystem> oldLogSystem = oldLogSystems->get();
+	while (true) {
+		Reference<LogSystem> oldLogSystem = oldLogSystems->get();
 		if (oldLogSystem) {
 			logChanges = triggerUpdates(self, oldLogSystem);
 			if (!minRecoveryDuration.isValid()) {
@@ -1606,24 +1613,26 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 			}
 		}
 
-		state Future<Void> reg = oldLogSystem ? updateRegistration(self, oldLogSystem) : Never();
+		Future<Void> reg = oldLogSystem ? updateRegistration(self, oldLogSystem) : Never();
 		self->registrationTrigger.trigger();
 
-		choose {
-			when(wait(oldLogSystem
-			              ? recoverFrom(self, oldLogSystem, &seedServers, &initialConfChanges, poppedTxsVersion)
-			              : Never())) {
-				reg.cancel();
-				break;
-			}
-			when(wait(oldLogSystems->onChange())) {}
-			when(wait(reg)) {
-				throw internal_error();
-			}
-			when(wait(recoverAndEndEpoch)) {
-				throw internal_error();
-			}
+		auto res = co_await race(
+		    oldLogSystem ? recoverFrom(self, oldLogSystem, &seedServers, &initialConfChanges, poppedTxsVersion)
+		                 : Never(),
+		    oldLogSystems->onChange(),
+		    reg,
+		    recoverAndEndEpoch);
+		if (res.index() == 0) {
+			reg.cancel();
+			break;
 		}
+		if (res.index() == 1) {
+			continue;
+		}
+		if (res.index() == 2 || res.index() == 3) {
+			throw internal_error();
+		}
+		UNREACHABLE();
 	}
 
 	if (self->neverCreated) {
@@ -1649,7 +1658,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	    .trackLatest(self->clusterRecoveryStateEventHolder->trackingKey);
 
 	// Recovery transaction
-	state bool debugResult = debug_checkMinRestoredVersion(UID(), self->lastEpochEnd, "DBRecovery", SevWarn);
+	bool debugResult = debug_checkMinRestoredVersion(UID(), self->lastEpochEnd, "DBRecovery", SevWarn);
 
 	CommitTransactionRequest recoveryCommitRequest;
 	recoveryCommitRequest.flags = recoveryCommitRequest.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
@@ -1746,7 +1755,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_COMMIT_EVENT_NAME).c_str(), self->dbgid);
 
-	state Future<ErrorOr<CommitID>> recoveryCommit = self->commitProxies[0].commit.tryGetReply(recoveryCommitRequest);
+	Future<ErrorOr<CommitID>> recoveryCommit = self->commitProxies[0].commit.tryGetReply(recoveryCommitRequest);
 	self->addActor.send(self->logSystem->onError());
 	self->addActor.send(waitResolverFailure(self->resolvers));
 	self->addActor.send(waitCommitProxyFailure(self->commitProxies));
@@ -1754,13 +1763,13 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 	self->addActor.send(reportErrors(updateRegistration(self, self->logSystem), "UpdateRegistration", self->dbgid));
 	self->registrationTrigger.trigger();
 
-	wait(traceAfter(discardCommit(self->txnStateStore, self->txnStateLogAdapter), "DiscardCommitFinished"));
+	co_await traceAfter(discardCommit(self->txnStateStore, self->txnStateLogAdapter), "DiscardCommitFinished");
 
 	// Wait for the recovery transaction to complete.
 	// SOMEDAY: For faster recovery, do this and setDBState asynchronously and don't wait for them
 	// unless we want to change TLogs
-	wait(traceAfter(success(recoveryCommit), "RecoveryCommitFinished") &&
-	     traceAfter(sendInitialCommitToResolvers(self), "InitialCommitToResolversFinished"));
+	co_await (traceAfter(success(recoveryCommit), "RecoveryCommitFinished") &&
+	          traceAfter(sendInitialCommitToResolvers(self), "InitialCommitToResolversFinished"));
 	if (recoveryCommit.isReady() && recoveryCommit.get().isError()) {
 		TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_COMMIT_EVENT_NAME).c_str(),
 		           self->dbgid)
@@ -1791,7 +1800,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 
 	self->addActor.send(trackTlogRecovery(self, oldLogSystems, minRecoveryDuration));
 	debug_advanceMaxCommittedVersion(UID(), self->recoveryTransactionVersion);
-	wait(self->cstateUpdated.getFuture());
+	co_await self->cstateUpdated.getFuture();
 	debug_advanceMinCommittedVersion(UID(), self->recoveryTransactionVersion);
 
 	if (debugResult) {
@@ -1837,20 +1846,18 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 		self->logSystem->setOldestBackupEpoch(self->cstate.myDBState.recoveryCount);
 	}
 
-	wait(Future<Void>(Never()));
+	co_await Future<Void>(Never());
 	throw internal_error();
 }
 
-ACTOR Future<Void> cleanupRecoveryActorCollection(Reference<ClusterRecoveryData> self) {
+Future<Void> cleanupRecoveryActorCollection(Reference<ClusterRecoveryData> self) {
 	if (self.isValid()) {
-		wait(delay(0.0));
+		co_await delay(0.0);
 
 		while (!self->addActor.isEmpty()) {
 			self->addActor.getFuture().pop();
 		}
 	}
-
-	return Void();
 }
 
 bool isNormalClusterRecoveryError(const Error& error) {
diff --git a/fdbserver/include/fdbserver/ClusterRecovery.actor.h b/fdbserver/clustercontroller/ClusterRecovery.h
similarity index 84%
rename from fdbserver/include/fdbserver/ClusterRecovery.actor.h
rename to fdbserver/clustercontroller/ClusterRecovery.h
index 943bbbee71a..09499d5dec3 100644
--- a/fdbserver/include/fdbserver/ClusterRecovery.actor.h
+++ b/fdbserver/clustercontroller/ClusterRecovery.h
@@ -1,5 +1,5 @@
 /*
- * ClusterRecovery.actor.h
+ * ClusterRecovery.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,37 +20,27 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
 #include "flow/Trace.h"
 #include <utility>
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERRECOVERY_ACTOR_G_H)
-#define FDBSERVER_CLUSTERRECOVERY_ACTOR_G_H
-#include "fdbserver/ClusterRecovery.actor.g.h"
-#elif !defined(FDBSERVER_CLUSTERRECOVERY_ACTOR_H)
-#define FDBSERVER_CLUSTERRECOVERY_ACTOR_H
-
 #include "fdbclient/DatabaseContext.h"
 #include "fdbrpc/Replication.h"
 #include "fdbrpc/ReplicationUtils.h"
-#include "fdbserver/CoordinatedState.h"
-#include "fdbserver/CoordinationInterface.h" // copy constructors for ServerCoordinators class
-#include "fdbserver/ClusterController.actor.h"
-#include "fdbserver/DBCoreState.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemConfig.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/CoordinatedState.h"
+#include "fdbserver/core/CoordinationInterface.h" // copy constructors for ServerCoordinators class
+#include "ClusterController.h"
+#include "fdbserver/core/DBCoreState.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/core/LogSystemConfig.h"
+#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "flow/CoroUtils.h"
 #include "flow/Error.h"
 #include "flow/SystemMonitor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 class ClusterControllerData;
-
 typedef enum {
 	CLUSTER_RECOVERY_STATE_EVENT_NAME,
 	CLUSTER_RECOVERY_COMMIT_TLOG_EVENT_NAME,
@@ -68,10 +58,10 @@ typedef enum {
 	CLUSTER_RECOVERY_LAST // Always the last entry
 } ClusterRecoveryEventType;
 
-ACTOR Future<Void> recoveryTerminateOnConflict(UID dbgid,
-                                               Promise<Void> fullyRecovered,
-                                               Future<Void> onConflict,
-                                               Future<Void> switchedState);
+Future<Void> recoveryTerminateOnConflict(UID dbgid,
+                                         Promise<Void> fullyRecovered,
+                                         Future<Void> onConflict,
+                                         Future<Void> switchedState);
 std::string& getRecoveryEventName(ClusterRecoveryEventType type);
 
 class ReusableCoordinatedState : NonCopyable {
@@ -106,8 +96,8 @@ class ReusableCoordinatedState : NonCopyable {
 	Promise<Void> switchedState;
 	UID dbgid;
 
-	ACTOR Future<Void> _read(ReusableCoordinatedState* self) {
-		Value prevDBStateRaw = wait(self->cstate.read());
+	Future<Void> _read(ReusableCoordinatedState* self) {
+		Value prevDBStateRaw = co_await self->cstate.read();
 		Future<Void> onConflict = recoveryTerminateOnConflict(
 		    self->dbgid, self->fullyRecovered, self->cstate.onConflict(), self->switchedState.getFuture());
 		if (onConflict.isReady() && onConflict.isError()) {
@@ -119,13 +109,11 @@ class ReusableCoordinatedState : NonCopyable {
 			self->prevDBState = BinaryReader::fromStringRef<DBCoreState>(prevDBStateRaw, IncludeVersion());
 			self->myDBState = self->prevDBState;
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _write(ReusableCoordinatedState* self, DBCoreState newState, bool finalWrite) {
+	Future<Void> _write(ReusableCoordinatedState* self, DBCoreState newState, bool finalWrite) {
 		if (self->finalWriteStarted) {
-			wait(Future<Void>(Never()));
+			co_await Future<Void>(Never());
 		}
 
 		if (finalWrite) {
@@ -139,11 +127,11 @@ class ReusableCoordinatedState : NonCopyable {
 			// TODO(gglass): figure out what the above means post-encryption-at-rest deletion.
 			// It's not clear the protocol versioning scheme contemplates the possibility of features being removed.
 			if (SERVER_KNOBS->RECORD_RECOVER_AT_IN_CSTATE) {
-				wait(self->cstate.setExclusive(
-				    BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withGcTxnGenerations()))));
+				co_await self->cstate.setExclusive(
+				    BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withGcTxnGenerations())));
 			} else {
-				wait(self->cstate.setExclusive(
-				    BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withEncryptionAtRest()))));
+				co_await self->cstate.setExclusive(
+				    BinaryWriter::toValue(newState, IncludeVersion(ProtocolVersion::withEncryptionAtRest())));
 			}
 		} catch (Error& e) {
 			CODE_PROBE(true, "Master displaced during writeMasterState");
@@ -155,7 +143,7 @@ class ReusableCoordinatedState : NonCopyable {
 		if (!finalWrite) {
 			self->switchedState.send(Void());
 			self->cstate = MovableCoordinatedState(self->coordinators);
-			Value rereadDBStateRaw = wait(self->cstate.read());
+			Value rereadDBStateRaw = co_await self->cstate.read();
 			DBCoreState readState;
 			if (rereadDBStateRaw.size())
 				readState = BinaryReader::fromStringRef<DBCoreState>(rereadDBStateRaw, IncludeVersion());
@@ -171,8 +159,6 @@ class ReusableCoordinatedState : NonCopyable {
 		} else {
 			self->fullyRecovered.send(Void());
 		}
-
-		return Void();
 	}
 };
 
@@ -200,7 +186,7 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted<ClusterRecoveryData>
 
 	ServerCoordinators coordinators;
 
-	Reference<ILogSystem> logSystem;
+	Reference<LogSystem> logSystem;
 	double lastVersionTime;
 	LogSystemDiskQueueAdapter* txnStateLogAdapter;
 	IKeyValueStore* txnStateStore;
@@ -318,13 +304,11 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted<ClusterRecoveryData>
 	}
 };
 
-ACTOR Future<Void> recruitNewMaster(ClusterControllerData* cluster,
-                                    ClusterControllerData::DBInfo* db,
-                                    MasterInterface* newMaster);
-ACTOR Future<Void> cleanupRecoveryActorCollection(Reference<ClusterRecoveryData> self);
-ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self);
+Future<Void> recruitNewMaster(ClusterControllerData* cluster,
+                              ClusterControllerData::DBInfo* db,
+                              MasterInterface* newMaster);
+Future<Void> cleanupRecoveryActorCollection(Reference<ClusterRecoveryData> self);
+Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self);
 bool isNormalClusterRecoveryError(const Error&);
 
-#include "flow/unactorcompiler.h"
-
-#endif
+#include "fdbserver/core/MoveKeys.h"
diff --git a/fdbserver/clustercontroller/RatekeeperMonitor.cpp b/fdbserver/clustercontroller/RatekeeperMonitor.cpp
new file mode 100644
index 00000000000..24726a81693
--- /dev/null
+++ b/fdbserver/clustercontroller/RatekeeperMonitor.cpp
@@ -0,0 +1,62 @@
+/*
+ * RatekeeperMonitor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "RatekeeperMonitor.h"
+#include "flow/UnitTest.h"
+
+bool RatekeeperMonitor::hasSustainedZeroRatekeeperTpsLimit(double tpsLimit,
+                                                           double currentTime,
+                                                           double zeroTpsLimitDuration) {
+	if (zeroTpsLimitDuration <= 0 || tpsLimit > 0) {
+		resetZeroRatekeeperTpsLimitObservation();
+		return false;
+	}
+
+	if (!zeroRatekeeperTpsLimitStartTime.present()) {
+		zeroRatekeeperTpsLimitStartTime = currentTime;
+		return false;
+	}
+
+	return currentTime - zeroRatekeeperTpsLimitStartTime.get() >= zeroTpsLimitDuration;
+}
+
+TEST_CASE("/fdbserver/clustercontroller/hasSustainedZeroRatekeeperTpsLimit") {
+	RatekeeperMonitor monitor;
+
+	constexpr double duration = 5.0;
+
+	ASSERT(!monitor.hasSustainedZeroRatekeeperTpsLimit(0.0, 100.0, duration));
+	ASSERT(monitor.getZeroRatekeeperTpsLimitStartTime().present());
+	ASSERT_EQ(monitor.getZeroRatekeeperTpsLimitStartTime().get(), 100.0);
+	ASSERT(!monitor.hasSustainedZeroRatekeeperTpsLimit(0.0, 104.9, duration));
+	ASSERT(monitor.hasSustainedZeroRatekeeperTpsLimit(0.0, 105.0, duration));
+
+	ASSERT(!monitor.hasSustainedZeroRatekeeperTpsLimit(1.0, 106.0, duration));
+	ASSERT(!monitor.getZeroRatekeeperTpsLimitStartTime().present());
+
+	ASSERT(!monitor.hasSustainedZeroRatekeeperTpsLimit(0.0, 107.0, duration));
+	ASSERT(monitor.getZeroRatekeeperTpsLimitStartTime().present());
+	ASSERT_EQ(monitor.getZeroRatekeeperTpsLimitStartTime().get(), 107.0);
+
+	ASSERT(!monitor.hasSustainedZeroRatekeeperTpsLimit(0.0, 200.0, 0.0));
+	ASSERT(!monitor.getZeroRatekeeperTpsLimitStartTime().present());
+
+	return Void();
+}
diff --git a/fdbserver/clustercontroller/RatekeeperMonitor.h b/fdbserver/clustercontroller/RatekeeperMonitor.h
new file mode 100644
index 00000000000..690f900045b
--- /dev/null
+++ b/fdbserver/clustercontroller/RatekeeperMonitor.h
@@ -0,0 +1,46 @@
+/*
+ * RatekeeperMonitor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/Knobs.h"
+#include "flow/Optional.h"
+#include "flow/flow.h"
+
+class ClusterControllerData;
+
+class RatekeeperMonitor {
+public:
+	void resetZeroRatekeeperTpsLimitObservation() { zeroRatekeeperTpsLimitStartTime = Optional<double>(); }
+
+	bool hasSustainedZeroRatekeeperTpsLimit(
+	    double tpsLimit,
+	    double currentTime = now(),
+	    double zeroTpsLimitDuration = SERVER_KNOBS->CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION);
+
+	double getZeroRatekeeperTpsLimitDuration(double currentTime = now()) const {
+		return zeroRatekeeperTpsLimitStartTime.present() ? currentTime - zeroRatekeeperTpsLimitStartTime.get() : 0.0;
+	}
+
+	Optional<double> getZeroRatekeeperTpsLimitStartTime() const { return zeroRatekeeperTpsLimitStartTime; }
+
+private:
+	Optional<double> zeroRatekeeperTpsLimitStartTime;
+};
diff --git a/fdbserver/include/fdbserver/SingletonRoles.h b/fdbserver/clustercontroller/SingletonRoles.h
similarity index 90%
rename from fdbserver/include/fdbserver/SingletonRoles.h
rename to fdbserver/clustercontroller/SingletonRoles.h
index efb6bfbcd8f..0f44d029295 100644
--- a/fdbserver/include/fdbserver/SingletonRoles.h
+++ b/fdbserver/clustercontroller/SingletonRoles.h
@@ -20,9 +20,9 @@
 
 #pragma once
 
-#include "fdbserver/ClusterController.actor.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/DataDistributorInterface.h"
+#include "ClusterController.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbserver/core/DataDistributorInterface.h"
 
 // This is used to artificially amplify the used count for processes
 // occupied by non-singletons. This ultimately makes it less desirable
@@ -45,7 +45,7 @@ class Singleton {
 
 struct RatekeeperSingleton : Singleton<RatekeeperInterface> {
 
-	RatekeeperSingleton(const Optional<RatekeeperInterface>& interface) : Singleton(interface) {}
+	explicit RatekeeperSingleton(const Optional<RatekeeperInterface>& interface) : Singleton(interface) {}
 
 	Role getRole() const { return Role::RATEKEEPER; }
 	ProcessClass::ClusterRole getClusterRole() const { return ProcessClass::Ratekeeper; }
@@ -70,7 +70,7 @@ struct RatekeeperSingleton : Singleton<RatekeeperInterface> {
 
 struct DataDistributorSingleton : Singleton<DataDistributorInterface> {
 
-	DataDistributorSingleton(const Optional<DataDistributorInterface>& interface) : Singleton(interface) {}
+	explicit DataDistributorSingleton(const Optional<DataDistributorInterface>& interface) : Singleton(interface) {}
 
 	Role getRole() const { return Role::DATA_DISTRIBUTOR; }
 	ProcessClass::ClusterRole getClusterRole() const { return ProcessClass::DataDistributor; }
@@ -95,7 +95,7 @@ struct DataDistributorSingleton : Singleton<DataDistributorInterface> {
 
 struct ConsistencyScanSingleton : Singleton<ConsistencyScanInterface> {
 
-	ConsistencyScanSingleton(const Optional<ConsistencyScanInterface>& interface) : Singleton(interface) {}
+	explicit ConsistencyScanSingleton(const Optional<ConsistencyScanInterface>& interface) : Singleton(interface) {}
 
 	Role getRole() const { return Role::CONSISTENCYSCAN; }
 	ProcessClass::ClusterRole getClusterRole() const { return ProcessClass::ConsistencyScan; }
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/clustercontroller/Status.cpp
similarity index 77%
rename from fdbserver/Status.actor.cpp
rename to fdbserver/clustercontroller/Status.cpp
index e2358015af4..0ff20ac85c6 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/clustercontroller/Status.cpp
@@ -1,5 +1,5 @@
 /*
- * Status.actor.cpp
+ * Status.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,28 +22,29 @@
 #include "fdbclient/json_spirit/json_spirit_value.h"
 #include "flow/genericactors.actor.h"
 #include "fmt/format.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbserver/Status.actor.h"
+#include "fdbclient/BackupAgent.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "Status.h"
 #include "flow/ITrace.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/Trace.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/WorkerEvents.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include <time.h>
-#include "fdbserver/ClusterRecovery.actor.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbclient/ConsistencyScanInterface.actor.h"
+#include "ClusterRecovery.h"
+#include "fdbclient/ClusterConnectionMemoryRecord.h"
+#include "fdbserver/core/CoordinationInterface.h"
+#include "fdbclient/ConsistencyScanInterface.h"
 #include "flow/UnitTest.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/RatekeeperLimitReasons.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbclient/JsonBuilder.h"
-#include "fdbclient/StorageWiggleMetrics.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/StorageWiggleMetrics.h"
 
 const char* RecoveryStatus::names[] = { "reading_coordinated_state",
 	                                    "locking_coordinated_state",
@@ -100,73 +101,53 @@ const char* RecoveryStatus::descriptions[] = {
 static_assert(sizeof(RecoveryStatus::descriptions) == sizeof(RecoveryStatus::descriptions[0]) * RecoveryStatus::END,
               "RecoveryStatus::descriptions[] size");
 
-// From Ratekeeper.actor.cpp
-extern int limitReasonEnd;
-extern const char* limitReasonName[];
-extern const char* limitReasonDesc[];
-
-typedef std::map<std::string, TraceEventFields> EventMap;
+using EventMap = std::map<std::string, TraceEventFields>;
 
 struct StorageServerStatusInfo : public StorageServerMetaInfo {
 	EventMap eventMap;
-	StorageServerStatusInfo(const StorageServerMetaInfo& info) : StorageServerMetaInfo(info, info.metadata) {}
+	explicit StorageServerStatusInfo(const StorageServerMetaInfo& info) : StorageServerMetaInfo(info, info.metadata) {}
+};
+
+// Shared output state for clusterGetStatus() and clusterGetStatusImpl().
+// The inner status collection can outlive the outer deadline race, so these
+// objects must not be stack locals referenced through raw pointers.
+struct ClusterGetStatusState : ReferenceCounted<ClusterGetStatusState> {
+	JsonBuilderObject statusObj;
+	JsonBuilderArray messages;
+	std::set<std::string> statusIncompleteReasons;
+};
+
+// Owns the transactions used by the latency probe child actors. Those actors
+// may continue running after latencyProbeFetcher() times out, so the
+// transactions cannot live as stack locals in the parent coroutine frame.
+struct LatencyProbeState : ReferenceCounted<LatencyProbeState> {
+	explicit LatencyProbeState(Database cx) : trImmediate(cx), trDefault(cx), trBatch(cx), trWrite(cx) {}
+
+	Transaction trImmediate;
+	Transaction trDefault;
+	Transaction trBatch;
+	Transaction trWrite;
 };
 
-ACTOR static Future<Optional<TraceEventFields>> latestEventOnWorker(WorkerInterface worker, std::string eventName) {
+static AsyncResult<Optional<TraceEventFields>> latestEventOnWorker(WorkerInterface worker, std::string eventName) {
 	try {
 		EventLogRequest req =
 		    eventName.size() > 0 ? EventLogRequest(Standalone<StringRef>(eventName)) : EventLogRequest();
-		ErrorOr<TraceEventFields> eventTrace = wait(errorOr(timeoutError(worker.eventLogRequest.getReply(req), 2.0)));
+		ErrorOr<TraceEventFields> eventTrace =
+		    co_await errorOr(timeoutError(worker.eventLogRequest.getReply(req), 2.0));
 
 		if (eventTrace.isError()) {
-			return Optional<TraceEventFields>();
+			co_return Optional<TraceEventFields>();
 		}
-		return eventTrace.get();
+		co_return eventTrace.get();
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
 			throw;
-		return Optional<TraceEventFields>();
+		co_return Optional<TraceEventFields>();
 	}
 }
 
-ACTOR Future<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventOnWorkers(
-    std::vector<WorkerDetails> workers,
-    std::string eventName) {
-	try {
-		state std::vector<Future<ErrorOr<TraceEventFields>>> eventTraces;
-		for (int c = 0; c < workers.size(); c++) {
-			EventLogRequest req =
-			    eventName.size() > 0 ? EventLogRequest(Standalone<StringRef>(eventName)) : EventLogRequest();
-			eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0)));
-		}
-
-		wait(waitForAll(eventTraces));
-
-		std::set<std::string> failed;
-		WorkerEvents results;
-
-		for (int i = 0; i < eventTraces.size(); i++) {
-			const ErrorOr<TraceEventFields>& v = eventTraces[i].get();
-			if (v.isError()) {
-				failed.insert(workers[i].interf.address().toString());
-				results[workers[i].interf.address()] = TraceEventFields();
-			} else {
-				results[workers[i].interf.address()] = v.get();
-			}
-		}
-
-		std::pair<WorkerEvents, std::set<std::string>> val;
-		val.first = results;
-		val.second = failed;
-
-		return val;
-	} catch (Error& e) {
-		ASSERT(e.code() ==
-		       error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above
-		throw;
-	}
-}
-static Future<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestErrorOnWorkers(
+static AsyncResult<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestErrorOnWorkers(
     std::vector<WorkerDetails> workers) {
 	return latestEventOnWorkers(workers, "");
 }
@@ -196,7 +177,7 @@ class StatusCounter {
 public:
 	StatusCounter() : hz(0), roughness(0), counter(0) {}
 	StatusCounter(double hz, double roughness, int64_t counter) : hz(hz), roughness(roughness), counter(counter) {}
-	StatusCounter(const std::string& parsableText) { parseText(parsableText); }
+	explicit(false) StatusCounter(const std::string& parsableText) { parseText(parsableText); }
 
 	StatusCounter& parseText(const std::string& parsableText) {
 		sscanf(parsableText.c_str(), "%lf %lf %" SCNd64 "", &hz, &roughness, &counter);
@@ -794,7 +775,7 @@ struct RolesInfo {
 	}
 };
 
-ACTOR static Future<JsonBuilderObject> processStatusFetcher(
+static AsyncResult<JsonBuilderObject> processStatusFetcher(
     Reference<AsyncVar<ServerDBInfo>> db,
     std::vector<WorkerDetails> workers,
     WorkerEvents pMetrics,
@@ -815,15 +796,15 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
     Optional<Key> healthyZone,
     std::set<std::string>* incomplete_reasons) {
 
-	state JsonBuilderObject processMap;
+	JsonBuilderObject processMap;
 
 	// construct a map from a process address to a status object containing a trace file open error
 	// this is later added to the messages subsection
-	state std::map<std::string, JsonBuilderObject> tracefileOpenErrorMap;
-	state WorkerEvents::iterator traceFileErrorsItr;
+	std::map<std::string, JsonBuilderObject> tracefileOpenErrorMap;
+	WorkerEvents::iterator traceFileErrorsItr;
 	for (traceFileErrorsItr = traceFileOpenErrors.begin(); traceFileErrorsItr != traceFileOpenErrors.end();
 	     ++traceFileErrorsItr) {
-		wait(yield());
+		co_await yield();
 		if (traceFileErrorsItr->second.size()) {
 			try {
 				// Have event fields, parse it and turn it into a message object describing the trace file opening error
@@ -846,11 +827,11 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		}
 	}
 
-	state std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo> machineMemoryUsage;
-	state std::vector<WorkerDetails>::iterator workerItr;
+	std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo> machineMemoryUsage;
+	std::vector<WorkerDetails>::iterator workerItr;
 	for (workerItr = workers.begin(); workerItr != workers.end(); ++workerItr) {
-		wait(yield());
-		state std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo>::iterator memInfo =
+		co_await yield();
+		std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo>::iterator memInfo =
 		    machineMemoryUsage.insert(std::make_pair(workerItr->interf.locality.machineId(), MachineMemoryInfo()))
 		        .first;
 		try {
@@ -871,7 +852,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		}
 	}
 
-	state RolesInfo roles;
+	RolesInfo roles;
 
 	roles.addRole("master", db->get().master);
 	roles.addRole("cluster_controller", db->get().clusterInterface.clientInterface);
@@ -896,8 +877,8 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		}
 	}
 
-	state std::vector<OldTLogConf>::const_iterator oldTLogIter;
-	state std::vector<OldTLogConf> oldTLogs = db->get().logSystemConfig.oldTLogs;
+	std::vector<OldTLogConf>::const_iterator oldTLogIter;
+	std::vector<OldTLogConf> oldTLogs = db->get().logSystemConfig.oldTLogs;
 	for (oldTLogIter = oldTLogs.begin(); oldTLogIter != oldTLogs.end(); ++oldTLogIter) {
 		for (auto& tLogSet : oldTLogIter->tLogs) {
 			for (auto& it : tLogSet.tLogs) {
@@ -911,57 +892,57 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 				}
 			}
 		}
-		wait(yield());
+		co_await yield();
 	}
 
 	for (const auto& coordinator : coordinatorAddresses) {
 		roles.addCoordinatorRole(coordinator);
 	}
 
-	state std::vector<std::pair<CommitProxyInterface, EventMap>>::iterator commit_proxy;
+	std::vector<std::pair<CommitProxyInterface, EventMap>>::iterator commit_proxy;
 	for (commit_proxy = commitProxies.begin(); commit_proxy != commitProxies.end(); ++commit_proxy) {
 		roles.addRole("commit_proxy", commit_proxy->first, commit_proxy->second);
-		wait(yield());
+		co_await yield();
 	}
 
-	state std::vector<std::pair<GrvProxyInterface, EventMap>>::iterator grvProxy;
+	std::vector<std::pair<GrvProxyInterface, EventMap>>::iterator grvProxy;
 	for (grvProxy = grvProxies.begin(); grvProxy != grvProxies.end(); ++grvProxy) {
 		roles.addRole("grv_proxy", grvProxy->first, grvProxy->second);
-		wait(yield());
+		co_await yield();
 	}
 
-	state std::vector<std::pair<TLogInterface, EventMap>>::iterator log;
-	state Version maxTLogVersion = 0;
+	std::vector<std::pair<TLogInterface, EventMap>>::iterator log;
+	Version maxTLogVersion = 0;
 
 	// Get largest TLog version
 	for (log = tLogs.begin(); log != tLogs.end(); ++log) {
 		Version tLogVersion = 0;
 		roles.addRole("log", log->first, log->second, &tLogVersion);
 		maxTLogVersion = std::max(maxTLogVersion, tLogVersion);
-		wait(yield());
+		co_await yield();
 	}
 
-	state std::vector<StorageServerStatusInfo>::iterator ss;
-	state std::map<NetworkAddress, double> ssLag;
-	state double lagSeconds;
+	std::vector<StorageServerStatusInfo>::iterator ss;
+	std::map<NetworkAddress, double> ssLag;
+	double lagSeconds{ 0 };
 	for (ss = storageServers.begin(); ss != storageServers.end(); ++ss) {
 		roles.addRole("storage", *ss, maxTLogVersion, &lagSeconds);
 		if (lagSeconds != -1.0) {
 			ssLag[ss->address()] = lagSeconds;
 		}
-		wait(yield());
+		co_await yield();
 	}
 
-	state std::vector<ResolverInterface>::const_iterator res;
-	state std::vector<ResolverInterface> resolvers = db->get().resolvers;
+	std::vector<ResolverInterface>::const_iterator res;
+	std::vector<ResolverInterface> resolvers = db->get().resolvers;
 	for (res = resolvers.begin(); res != resolvers.end(); ++res) {
 		roles.addRole("resolver", *res);
-		wait(yield());
+		co_await yield();
 	}
 
 	for (workerItr = workers.begin(); workerItr != workers.end(); ++workerItr) {
-		wait(yield());
-		state JsonBuilderObject statusObj;
+		co_await yield();
+		JsonBuilderObject statusObj;
 		try {
 			ASSERT(pMetrics.contains(workerItr->interf.address()));
 
@@ -1154,7 +1135,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 
 		processMap[printable(workerItr->interf.locality.processId())] = statusObj;
 	}
-	return processMap;
+	co_return processMap;
 }
 
 static JsonBuilderObject grayFailureStatus(const std::unordered_map<NetworkAddress, double>& excludedDegradedServers) {
@@ -1238,31 +1219,31 @@ static JsonBuilderObject clientStatusFetcher(
 	return clientStatus;
 }
 
-ACTOR static Future<JsonBuilderObject> recoveryStateStatusFetcher(Database cx,
-                                                                  WorkerDetails ccWorker,
-                                                                  WorkerDetails mWorker,
-                                                                  int workerCount,
-                                                                  std::set<std::string>* incomplete_reasons,
-                                                                  int* statusCode) {
-	state JsonBuilderObject message;
-	state Transaction tr(cx);
+static AsyncResult<JsonBuilderObject> recoveryStateStatusFetcher(Database cx,
+                                                                 WorkerDetails ccWorker,
+                                                                 WorkerDetails mWorker,
+                                                                 int workerCount,
+                                                                 std::set<std::string>* incomplete_reasons,
+                                                                 int* statusCode) {
+	JsonBuilderObject message;
+	Transaction tr(cx);
 	try {
-		state Future<TraceEventFields> mdActiveGensF =
+		Future<TraceEventFields> mdActiveGensF =
 		    timeoutError(ccWorker.interf.eventLogRequest.getReply(EventLogRequest(StringRef(
 		                     getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_GENERATION_EVENT_NAME)))),
 		                 1.0);
-		state Future<TraceEventFields> mdF =
+		Future<TraceEventFields> mdF =
 		    timeoutError(ccWorker.interf.eventLogRequest.getReply(EventLogRequest(StringRef(
 		                     getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME)))),
 		                 1.0);
-		state Future<TraceEventFields> mDBAvailableF =
+		Future<TraceEventFields> mDBAvailableF =
 		    timeoutError(ccWorker.interf.eventLogRequest.getReply(EventLogRequest(StringRef(
 		                     getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_AVAILABLE_EVENT_NAME)))),
 		                 1.0);
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-		state Future<ErrorOr<Version>> rvF = errorOr(timeoutError(tr.getReadVersion(), 1.0));
+		Future<ErrorOr<Version>> rvF = errorOr(timeoutError(tr.getReadVersion(), 1.0));
 
-		wait(success(mdActiveGensF) && success(mdF) && success(rvF) && success(mDBAvailableF));
+		co_await (success(mdActiveGensF) && success(mdF) && success(rvF) && success(mDBAvailableF));
 
 		const TraceEventFields& md = mdF.get();
 		int mStatusCode = md.getInt("StatusCode");
@@ -1322,96 +1303,103 @@ ACTOR static Future<JsonBuilderObject> recoveryStateStatusFetcher(Database cx,
 		incomplete_reasons->insert("Recovery Status unavailable.");
 	}
 
-	return message;
+	co_return message;
 }
 
-ACTOR static Future<double> doGrvProbe(
-    Transaction* tr,
+static Future<double> doGrvProbe(
+    Reference<LatencyProbeState> probeState,
+    Transaction LatencyProbeState::* trMember,
     Optional<FDBTransactionOptions::Option> priority = Optional<FDBTransactionOptions::Option>()) {
-	state double start = g_network->timer_monotonic();
+	Transaction& tr = probeState.getPtr()->*trMember;
+	double start = g_network->timer_monotonic();
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			if (priority.present()) {
-				tr->setOption(priority.get());
+				tr.setOption(priority.get());
 			}
 
-			wait(success(tr->getReadVersion()));
-			return g_network->timer_monotonic() - start;
+			co_await tr.getReadVersion();
+			co_return g_network->timer_monotonic() - start;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR static Future<double> doReadProbe(Future<double> grvProbe, Transaction* tr) {
-	ErrorOr<double> grv = wait(errorOr(grvProbe));
+static Future<double> doReadProbe(Reference<LatencyProbeState> probeState,
+                                  Future<double> grvProbe,
+                                  Transaction LatencyProbeState::* trMember) {
+	Transaction& tr = probeState.getPtr()->*trMember;
+	ErrorOr<double> grv = co_await errorOr(grvProbe);
 	if (grv.isError()) {
 		throw grv.getError();
 	}
 
-	state double start = g_network->timer_monotonic();
+	double start = g_network->timer_monotonic();
 
-	loop {
-		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+	while (true) {
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		Error err;
 		try {
-			Optional<Standalone<StringRef>> _ = wait(tr->get("\xff/StatusJsonTestKey62793"_sr));
-			return g_network->timer_monotonic() - start;
+			Optional<Standalone<StringRef>> _ = co_await tr.get("\xff/StatusJsonTestKey62793"_sr);
+			co_return g_network->timer_monotonic() - start;
 		} catch (Error& e) {
-			wait(tr->onError(e));
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			err = e;
 		}
+		co_await tr.onError(err);
+		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 	}
 }
 
-ACTOR static Future<double> doCommitProbe(Future<double> grvProbe, Transaction* sourceTr, Transaction* tr) {
-	ErrorOr<double> grv = wait(errorOr(grvProbe));
+static Future<double> doCommitProbe(Reference<LatencyProbeState> probeState,
+                                    Future<double> grvProbe,
+                                    Transaction LatencyProbeState::* sourceTrMember,
+                                    Transaction LatencyProbeState::* trMember) {
+	Transaction& sourceTr = probeState.getPtr()->*sourceTrMember;
+	Transaction& tr = probeState.getPtr()->*trMember;
+	ErrorOr<double> grv = co_await errorOr(grvProbe);
 	if (grv.isError()) {
 		throw grv.getError();
 	}
 
-	ASSERT(sourceTr->getReadVersion().isReady());
-	tr->setVersion(sourceTr->getReadVersion().get());
-	tr->getDatabase()->ssVersionVectorCache = sourceTr->getDatabase()->ssVersionVectorCache;
-	tr->trState->readVersionObtainedFromGrvProxy = sourceTr->trState->readVersionObtainedFromGrvProxy;
+	ASSERT(sourceTr.getReadVersion().isReady());
+	tr.setVersion(sourceTr.getReadVersion().get());
+	tr.getDatabase()->ssVersionVectorCache = sourceTr.getDatabase()->ssVersionVectorCache;
+	tr.trState->readVersionObtainedFromGrvProxy = sourceTr.trState->readVersionObtainedFromGrvProxy;
 
-	state double start = g_network->timer_monotonic();
+	double start = g_network->timer_monotonic();
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr->makeSelfConflicting();
-			wait(tr->commit());
-			return g_network->timer_monotonic() - start;
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.makeSelfConflicting();
+			co_await tr.commit();
+			co_return g_network->timer_monotonic() - start;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR static Future<Void> doProbe(Future<double> probe,
-                                  int timeoutSeconds,
-                                  const char* prefix,
-                                  const char* description,
-                                  JsonBuilderObject* probeObj,
-                                  JsonBuilderArray* messages,
-                                  std::set<std::string>* incomplete_reasons,
-                                  bool* isAvailable = nullptr) {
-	choose {
-		when(ErrorOr<double> result = wait(errorOr(probe))) {
-			if (result.isError()) {
-				if (isAvailable != nullptr) {
-					*isAvailable = false;
-				}
-				incomplete_reasons->insert(format(
-				    "Unable to retrieve latency probe information (%s: %s).", description, result.getError().what()));
-			} else {
-				(*probeObj)[format("%s_seconds", prefix).c_str()] = result.get();
-			}
-		}
-		when(wait(delay(timeoutSeconds))) {
+static Future<Void> doProbe(Future<double> probe,
+                            int timeoutSeconds,
+                            const char* prefix,
+                            const char* description,
+                            JsonBuilderObject* probeObj,
+                            JsonBuilderArray* messages,
+                            std::set<std::string>* incomplete_reasons,
+                            bool* isAvailable = nullptr) {
+	try {
+		if (auto const latency = co_await timeout(probe, timeoutSeconds); latency.present()) {
+			(*probeObj)[format("%s_seconds", prefix).c_str()] = latency.get();
+		} else {
 			if (isAvailable != nullptr) {
 				*isAvailable = false;
 			}
@@ -1419,29 +1407,34 @@ ACTOR static Future<Void> doProbe(Future<double> probe,
 			    JsonString::makeMessage(format("%s_probe_timeout", prefix).c_str(),
 			                            format("Unable to %s after %d seconds.", description, timeoutSeconds).c_str()));
 		}
+	} catch (Error& e) {
+		if (isAvailable != nullptr) {
+			*isAvailable = false;
+		}
+		incomplete_reasons->insert(
+		    format("Unable to retrieve latency probe information (%s: %s).", description, e.what()));
 	}
-
-	return Void();
+	co_return;
 }
 
-ACTOR static Future<JsonBuilderObject> latencyProbeFetcher(Database cx,
-                                                           JsonBuilderArray* messages,
-                                                           std::set<std::string>* incomplete_reasons,
-                                                           bool* isAvailable) {
-	state Transaction trImmediate(cx);
-	state Transaction trDefault(cx);
-	state Transaction trBatch(cx);
-	state Transaction trWrite(cx);
+static AsyncResult<JsonBuilderObject> latencyProbeFetcher(Database cx,
+                                                          JsonBuilderArray* messages,
+                                                          std::set<std::string>* incomplete_reasons,
+                                                          bool* isAvailable) {
+	auto probeState = makeReference<LatencyProbeState>(cx);
 
-	state JsonBuilderObject statusObj;
+	JsonBuilderObject statusObj;
 
 	try {
-		Future<double> immediateGrvProbe = doGrvProbe(&trImmediate, FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-		Future<double> defaultGrvProbe = doGrvProbe(&trDefault);
-		Future<double> batchGrvProbe = doGrvProbe(&trBatch, FDBTransactionOptions::PRIORITY_BATCH);
+		Future<double> immediateGrvProbe =
+		    doGrvProbe(probeState, &LatencyProbeState::trImmediate, FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		Future<double> defaultGrvProbe = doGrvProbe(probeState, &LatencyProbeState::trDefault);
+		Future<double> batchGrvProbe =
+		    doGrvProbe(probeState, &LatencyProbeState::trBatch, FDBTransactionOptions::PRIORITY_BATCH);
 
-		Future<double> readProbe = doReadProbe(immediateGrvProbe, &trImmediate);
-		Future<double> commitProbe = doCommitProbe(immediateGrvProbe, &trImmediate, &trWrite);
+		Future<double> readProbe = doReadProbe(probeState, immediateGrvProbe, &LatencyProbeState::trImmediate);
+		Future<double> commitProbe =
+		    doCommitProbe(probeState, immediateGrvProbe, &LatencyProbeState::trImmediate, &LatencyProbeState::trWrite);
 
 		int timeoutSeconds = 5;
 
@@ -1473,7 +1466,7 @@ ACTOR static Future<JsonBuilderObject> latencyProbeFetcher(Database cx,
 		probes.push_back(doProbe(
 		    commitProbe, timeoutSeconds, "commit", "commit", &statusObj, messages, incomplete_reasons, isAvailable));
 
-		wait(waitForAll(probes));
+		co_await waitForAll(probes);
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
 			throw;
@@ -1481,20 +1474,22 @@ ACTOR static Future<JsonBuilderObject> latencyProbeFetcher(Database cx,
 		incomplete_reasons->insert(format("Unable to retrieve latency probe information (%s).", e.what()));
 	}
 
-	return statusObj;
+	co_return statusObj;
 }
 
-ACTOR static Future<JsonBuilderObject> versionEpochStatusFetcher(Database cx,
-                                                                 std::set<std::string>* incomplete_reasons) {
-	state JsonBuilderObject message;
+static AsyncResult<JsonBuilderObject> versionEpochStatusFetcher(Database cx,
+                                                                std::set<std::string>* incomplete_reasons) {
+	JsonBuilderObject message;
 	try {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				Optional<Value> versionEpochVal = wait(timeoutError(BUGGIFY ? Never() : tr.get(versionEpochKey), 5.0));
+				Optional<Value> versionEpochVal =
+				    co_await timeoutError(BUGGIFY ? Never() : tr.get(versionEpochKey), 5.0);
 				message["enabled"] = versionEpochVal.present();
 				if (!versionEpochVal.present()) {
 					break;
@@ -1503,8 +1498,9 @@ ACTOR static Future<JsonBuilderObject> versionEpochStatusFetcher(Database cx,
 				message["epoch"] = std::to_string(versionEpoch);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
@@ -1512,21 +1508,22 @@ ACTOR static Future<JsonBuilderObject> versionEpochStatusFetcher(Database cx,
 		}
 		incomplete_reasons->insert(format("Unable to retrieve version epoch information (%s).", e.what()));
 	}
-	return message;
+	co_return message;
 }
 
-ACTOR static Future<Void> consistencyCheckStatusFetcher(Database cx,
-                                                        JsonBuilderArray* messages,
-                                                        std::set<std::string>* incomplete_reasons) {
+static Future<Void> consistencyCheckStatusFetcher(Database cx,
+                                                  JsonBuilderArray* messages,
+                                                  std::set<std::string>* incomplete_reasons) {
 	try {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				Optional<Value> ccSuspendVal =
-				    wait(timeoutError(BUGGIFY ? Never() : tr.get(fdbShouldConsistencyCheckBeSuspended), 5.0));
+				    co_await timeoutError(BUGGIFY ? Never() : tr.get(fdbShouldConsistencyCheckBeSuspended), 5.0);
 				bool ccSuspend = ccSuspendVal.present()
 				                     ? BinaryReader::fromStringRef<bool>(ccSuspendVal.get(), Unversioned())
 				                     : false;
@@ -1536,16 +1533,17 @@ ACTOR static Future<Void> consistencyCheckStatusFetcher(Database cx,
 				}
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_timed_out) {
-					messages->push_back(
-					    JsonString::makeMessage("consistencycheck_suspendkey_fetch_timeout",
-					                            format("Timed out trying to fetch `%s` from the database.",
-					                                   printable(fdbShouldConsistencyCheckBeSuspended).c_str())
-					                                .c_str()));
-					break;
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.code() == error_code_timed_out) {
+				messages->push_back(
+				    JsonString::makeMessage("consistencycheck_suspendkey_fetch_timeout",
+				                            format("Timed out trying to fetch `%s` from the database.",
+				                                   printable(fdbShouldConsistencyCheckBeSuspended).c_str())
+				                                .c_str()));
+				break;
 			}
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
@@ -1553,7 +1551,6 @@ ACTOR static Future<Void> consistencyCheckStatusFetcher(Database cx,
 		}
 		incomplete_reasons->insert(format("Unable to retrieve consistency check settings (%s).", e.what()));
 	}
-	return Void();
 }
 
 struct LogRangeAndUID {
@@ -1571,22 +1568,23 @@ struct LogRangeAndUID {
 	}
 };
 
-ACTOR static Future<Void> logRangeWarningFetcher(Database cx,
-                                                 JsonBuilderArray* messages,
-                                                 std::set<std::string>* incomplete_reasons) {
+static Future<Void> logRangeWarningFetcher(Database cx,
+                                           JsonBuilderArray* messages,
+                                           std::set<std::string>* incomplete_reasons) {
 	try {
-		state Transaction tr(cx);
-		state Future<Void> timeoutFuture = timeoutError(Future<Void>(Never()), 5.0);
-		loop {
+		Transaction tr(cx);
+		Future<Void> timeoutFuture = timeoutError(Future<Void>(Never()), 5.0);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-				state Future<RangeResult> existingDestUidValues =
+				Future<RangeResult> existingDestUidValues =
 				    tr.getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> existingLogRanges = tr.getRange(logRangesRange, CLIENT_KNOBS->TOO_MANY);
-				wait((success(existingDestUidValues) && success(existingLogRanges)) || timeoutFuture);
+				Future<RangeResult> existingLogRanges = tr.getRange(logRangesRange, CLIENT_KNOBS->TOO_MANY);
+				co_await ((success(existingDestUidValues) && success(existingLogRanges)) || timeoutFuture);
 
 				std::set<LogRangeAndUID> loggingRanges;
 				for (auto& it : existingLogRanges.get()) {
@@ -1620,19 +1618,19 @@ ACTOR static Future<Void> logRangeWarningFetcher(Database cx,
 						existingRanges.insert(rangePair);
 					}
 				}
-				wait(tr.commit() || timeoutFuture);
+				co_await (tr.commit() || timeoutFuture);
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_timed_out) {
-					messages->push_back(
-					    JsonString::makeMessage("duplicate_mutation_fetch_timeout",
-					                            format("Timed out trying to fetch `%s` from the database.",
-					                                   printable(destUidLookupPrefix).c_str())
-					                                .c_str()));
-					break;
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.code() == error_code_timed_out) {
+				messages->push_back(JsonString::makeMessage(
+				    "duplicate_mutation_fetch_timeout",
+				    format("Timed out trying to fetch `%s` from the database.", printable(destUidLookupPrefix).c_str())
+				        .c_str()));
+				break;
 			}
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
@@ -1640,7 +1638,6 @@ ACTOR static Future<Void> logRangeWarningFetcher(Database cx,
 		}
 		incomplete_reasons->insert(format("Unable to retrieve log ranges (%s).", e.what()));
 	}
-	return Void();
 }
 
 struct ProtocolVersionData {
@@ -1654,27 +1651,27 @@ struct ProtocolVersionData {
 	    lowestCompatibleProtocolVersion(lowestCompatibleProtocolVersionValue) {}
 };
 
-ACTOR Future<ProtocolVersionData> getNewestProtocolVersion(Database cx, WorkerDetails ccWorker) {
+Future<ProtocolVersionData> getNewestProtocolVersion(Database cx, WorkerDetails ccWorker) {
 
 	try {
-		state Future<TraceEventFields> swVersionF = timeoutError(
+		Future<TraceEventFields> swVersionF = timeoutError(
 		    ccWorker.interf.eventLogRequest.getReply(EventLogRequest("SWVersionCompatibilityChecked"_sr)), 1.0);
 
-		wait(success(swVersionF));
+		co_await swVersionF;
 		const TraceEventFields& swVersionTrace = swVersionF.get();
 		int64_t newestProtocolVersionValue =
 		    std::stoull(swVersionTrace.getValue("NewestProtocolVersion").c_str(), nullptr, 16);
 		int64_t lowestCompatibleProtocolVersionValue =
 		    std::stoull(swVersionTrace.getValue("LowestCompatibleProtocolVersion").c_str(), nullptr, 16);
 
-		return ProtocolVersionData(newestProtocolVersionValue, lowestCompatibleProtocolVersionValue);
+		co_return ProtocolVersionData(newestProtocolVersionValue, lowestCompatibleProtocolVersionValue);
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
 			throw;
 
 		TraceEvent(SevWarnAlways, "SWVersionStatusFailed").error(e);
 
-		return ProtocolVersionData();
+		co_return ProtocolVersionData();
 	}
 }
 
@@ -1691,98 +1688,98 @@ struct LoadConfigurationResult {
 	  : fullReplication(true), healthyZoneSeconds(0), rebalanceDDIgnored(false), dataDistributionDisabled(false) {}
 };
 
-ACTOR static Future<std::pair<Optional<DatabaseConfiguration>, Optional<LoadConfigurationResult>>>
+static Future<std::pair<Optional<DatabaseConfiguration>, Optional<LoadConfigurationResult>>>
 loadConfiguration(Database cx, JsonBuilderArray* messages, std::set<std::string>* status_incomplete_reasons) {
-	state Optional<DatabaseConfiguration> result;
-	state Optional<LoadConfigurationResult> loadResult;
-	state Transaction tr(cx);
-	state Future<Void> getConfTimeout = delay(5.0);
+	Optional<DatabaseConfiguration> result;
+	Optional<LoadConfigurationResult> loadResult;
+	Transaction tr(cx);
+	Future<Void> getConfTimeout = delay(5.0);
 
-	loop {
+	while (true) {
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		tr.setOption(FDBTransactionOptions::CAUSAL_READ_RISKY);
+		Error err;
 		try {
-			choose {
-				when(RangeResult res = wait(tr.getRange(configKeys, SERVER_KNOBS->CONFIGURATION_ROWS_TO_FETCH))) {
-					DatabaseConfiguration configuration;
-					if (res.size() == SERVER_KNOBS->CONFIGURATION_ROWS_TO_FETCH) {
-						status_incomplete_reasons->insert("Too many configuration parameters set.");
-					} else {
-						configuration.fromKeyValues((VectorRef<KeyValueRef>)res);
-					}
-
-					result = configuration;
+			auto configResult =
+			    co_await race(tr.getRange(configKeys, SERVER_KNOBS->CONFIGURATION_ROWS_TO_FETCH), getConfTimeout);
+			if (configResult.index() == 0) {
+				RangeResult res = std::get<0>(std::move(configResult));
+				DatabaseConfiguration configuration;
+				if (res.size() == SERVER_KNOBS->CONFIGURATION_ROWS_TO_FETCH) {
+					status_incomplete_reasons->insert("Too many configuration parameters set.");
+				} else {
+					configuration.fromKeyValues((VectorRef<KeyValueRef>)res);
 				}
-				when(wait(getConfTimeout)) {
-					if (!result.present()) {
-						messages->push_back(JsonString::makeMessage("unreadable_configuration",
-						                                            "Unable to read database configuration."));
-					} else {
-						messages->push_back(
-						    JsonString::makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
-					}
-					break;
+
+				result = configuration;
+			} else {
+				if (!result.present()) {
+					messages->push_back(
+					    JsonString::makeMessage("unreadable_configuration", "Unable to read database configuration."));
+				} else {
+					messages->push_back(
+					    JsonString::makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
 				}
+				break;
 			}
 
 			ASSERT(result.present());
-			state std::vector<Future<Optional<Value>>> replicasFutures;
+			std::vector<Future<Optional<Value>>> replicasFutures;
 			for (auto& region : result.get().regions) {
 				replicasFutures.push_back(tr.get(datacenterReplicasKeyFor(region.dcId)));
 			}
-			state Future<Optional<Value>> healthyZoneValue = tr.get(healthyZoneKey);
-			state Future<Optional<Value>> rebalanceDDIgnored = tr.get(rebalanceDDIgnoreKey);
-			state Future<Optional<Value>> ddModeKey = tr.get(dataDistributionModeKey);
-
-			choose {
-				when(wait(waitForAll(replicasFutures) && success(healthyZoneValue) && success(rebalanceDDIgnored) &&
-				          success(ddModeKey))) {
-					int unreplicated = 0;
-					for (int i = 0; i < result.get().regions.size(); i++) {
-						if (!replicasFutures[i].get().present() ||
-						    decodeDatacenterReplicasValue(replicasFutures[i].get().get()) <
-						        result.get().storageTeamSize) {
-							unreplicated++;
-						}
-					}
-					LoadConfigurationResult res;
-					res.fullReplication = (!unreplicated || (result.get().usableRegions == 1 &&
-					                                         unreplicated < result.get().regions.size()));
-					if (healthyZoneValue.get().present()) {
-						auto healthyZone = decodeHealthyZoneValue(healthyZoneValue.get().get());
-						if (healthyZone.first == ignoreSSFailuresZoneString) {
-							res.healthyZone = healthyZone.first;
-						} else if (healthyZone.second > tr.getReadVersion().get()) {
-							res.healthyZone = healthyZone.first;
-							res.healthyZoneSeconds =
-							    (healthyZone.second - tr.getReadVersion().get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
-						}
-					}
-					res.rebalanceDDIgnored = rebalanceDDIgnored.get().present();
-					if (res.rebalanceDDIgnored) {
-						res.rebalanceDDIgnoreHex = rebalanceDDIgnored.get().get().toHexString();
+			Future<Optional<Value>> healthyZoneValue = tr.get(healthyZoneKey);
+			Future<Optional<Value>> rebalanceDDIgnored = tr.get(rebalanceDDIgnoreKey);
+			Future<Optional<Value>> ddModeKey = tr.get(dataDistributionModeKey);
+			Future<Void> replicaStateReady = waitForAll(replicasFutures) && success(healthyZoneValue) &&
+			                                 success(rebalanceDDIgnored) && success(ddModeKey);
+
+			auto replicaLoadResult = co_await race(replicaStateReady, getConfTimeout);
+			if (replicaLoadResult.index() == 0) {
+				int unreplicated = 0;
+				for (int i = 0; i < result.get().regions.size(); ++i) {
+					if (!replicasFutures[i].get().present() ||
+					    decodeDatacenterReplicasValue(replicasFutures[i].get().get()) < result.get().storageTeamSize) {
+						unreplicated++;
 					}
-					if (ddModeKey.get().present()) {
-						BinaryReader rd(ddModeKey.get().get(), Unversioned());
-						int currentMode;
-						rd >> currentMode;
-						if (currentMode == 0) {
-							res.dataDistributionDisabled = true;
-						}
+				}
+				LoadConfigurationResult res;
+				res.fullReplication =
+				    (!unreplicated || (result.get().usableRegions == 1 && unreplicated < result.get().regions.size()));
+				if (healthyZoneValue.get().present()) {
+					auto healthyZone = decodeHealthyZoneValue(healthyZoneValue.get().get());
+					if (healthyZone.first == ignoreSSFailuresZoneString) {
+						res.healthyZone = healthyZone.first;
+					} else if (healthyZone.second > tr.getReadVersion().get()) {
+						res.healthyZone = healthyZone.first;
+						res.healthyZoneSeconds =
+						    (healthyZone.second - tr.getReadVersion().get()) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
 					}
-					loadResult = res;
 				}
-				when(wait(getConfTimeout)) {
-					messages->push_back(
-					    JsonString::makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
+				res.rebalanceDDIgnored = rebalanceDDIgnored.get().present();
+				if (res.rebalanceDDIgnored) {
+					res.rebalanceDDIgnoreHex = rebalanceDDIgnored.get().get().toHexString();
 				}
+				if (ddModeKey.get().present()) {
+					BinaryReader rd(ddModeKey.get().get(), Unversioned());
+					int currentMode;
+					rd >> currentMode;
+					if (currentMode == 0) {
+						res.dataDistributionDisabled = true;
+					}
+				}
+				loadResult = res;
+			} else {
+				messages->push_back(
+				    JsonString::makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
 			}
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return std::make_pair(result, loadResult);
+	co_return std::make_pair(result, loadResult);
 }
 
 static JsonBuilderObject configurationFetcher(Optional<DatabaseConfiguration> conf,
@@ -1796,7 +1793,7 @@ static JsonBuilderObject configurationFetcher(Optional<DatabaseConfiguration> co
 
 			JsonBuilderArray excludedServersArr;
 			std::set<AddressExclusion> excludedServers = configuration.getExcludedServers();
-			for (std::set<AddressExclusion>::iterator it = excludedServers.begin(); it != excludedServers.end(); it++) {
+			for (auto it = excludedServers.begin(); it != excludedServers.end(); it++) {
 				JsonBuilderObject statusObj;
 				statusObj["address"] = it->toString();
 				excludedServersArr.push_back(statusObj);
@@ -1820,10 +1817,10 @@ static JsonBuilderObject configurationFetcher(Optional<DatabaseConfiguration> co
 	return statusObj;
 }
 
-ACTOR static Future<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
-                                                         DatabaseConfiguration configuration,
-                                                         int* minStorageReplicasRemaining) {
-	state JsonBuilderObject statusObjData;
+static AsyncResult<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
+                                                        DatabaseConfiguration configuration,
+                                                        int* minStorageReplicasRemaining) {
+	JsonBuilderObject statusObjData;
 
 	try {
 		std::vector<Future<TraceEventFields>> futures;
@@ -1840,7 +1837,7 @@ ACTOR static Future<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 		futures.push_back(
 		    timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest("TotalDataInFlightRemote"_sr)), 1.0));
 
-		std::vector<TraceEventFields> dataInfo = wait(getAll(futures));
+		std::vector<TraceEventFields> dataInfo = co_await getAll(futures);
 
 		TraceEventFields startingStats = dataInfo[0];
 		TraceEventFields dataStats = dataInfo[1];
@@ -1850,7 +1847,7 @@ ACTOR static Future<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 			stateSectionObj["name"] = "initializing";
 			stateSectionObj["description"] = "(Re)initializing automatic data distribution";
 			statusObjData["state"] = stateSectionObj;
-			return statusObjData;
+			co_return statusObjData;
 		}
 
 		TraceEventFields md = dataInfo[2];
@@ -1986,38 +1983,38 @@ ACTOR static Future<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 		// not from the "cluster" perspective - from the client perspective it is not but that is indicated elsewhere.
 	}
 
-	return statusObjData;
+	co_return statusObjData;
 }
 
-ACTOR template <class iface>
-static Future<std::vector<std::pair<iface, EventMap>>> getServerMetrics(
+template <class iface>
+static AsyncResult<std::vector<std::pair<iface, EventMap>>> getServerMetrics(
     std::vector<iface> servers,
     std::unordered_map<NetworkAddress, WorkerInterface> address_workers,
     std::vector<std::string> eventNames) {
-	state std::vector<Future<Optional<TraceEventFields>>> futures;
+	std::vector<AsyncResult<Optional<TraceEventFields>>> futures;
 	for (auto s : servers) {
 		for (auto name : eventNames) {
 			futures.push_back(latestEventOnWorker(address_workers[s.address()], s.id().toString() + "/" + name));
 		}
 	}
 
-	wait(waitForAll(futures));
+	std::vector<Optional<TraceEventFields>> eventTraces = co_await getAll(std::move(futures));
 
 	std::vector<std::pair<iface, EventMap>> results;
-	auto futureItr = futures.begin();
+	auto eventTraceItr = eventTraces.begin();
 
 	for (int i = 0; i < servers.size(); i++) {
 		EventMap serverResults;
 		for (auto name : eventNames) {
-			ASSERT(futureItr != futures.end());
-			serverResults[name] = futureItr->get().present() ? futureItr->get().get() : TraceEventFields();
-			++futureItr;
+			ASSERT(eventTraceItr != eventTraces.end());
+			serverResults[name] = eventTraceItr->present() ? eventTraceItr->get() : TraceEventFields();
+			++eventTraceItr;
 		}
 
 		results.emplace_back(servers[i], serverResults);
 	}
 
-	return results;
+	co_return results;
 }
 
 namespace {
@@ -2027,63 +2024,63 @@ const std::vector<std::string> STORAGE_SERVER_METRICS_LIST{ "StorageMetrics", "R
 
 } // namespace
 
-ACTOR static Future<std::vector<StorageServerStatusInfo>> getStorageServerStatusInfos(
+static Future<std::vector<StorageServerStatusInfo>> getStorageServerStatusInfos(
     std::vector<StorageServerMetaInfo> storageMetadatas,
     std::unordered_map<NetworkAddress, WorkerInterface> address_workers,
     WorkerDetails rkWorker) {
-	state std::vector<StorageServerStatusInfo> servers;
+	std::vector<StorageServerStatusInfo> servers;
 	servers.reserve(storageMetadatas.size());
 	for (const auto& meta : storageMetadatas) {
 		servers.push_back(StorageServerStatusInfo(meta));
 	}
-	state std::vector<std::pair<StorageServerStatusInfo, EventMap>> results;
-	wait(store(results, getServerMetrics(servers, address_workers, STORAGE_SERVER_METRICS_LIST)));
+	std::vector<std::pair<StorageServerStatusInfo, EventMap>> results =
+	    co_await getServerMetrics(servers, address_workers, STORAGE_SERVER_METRICS_LIST);
 	for (int i = 0; i < results.size(); ++i) {
 		servers[i].eventMap = std::move(results[i].second);
 	}
-	return servers;
+	co_return servers;
 }
 
-ACTOR static Future<std::vector<std::pair<TLogInterface, EventMap>>> getTLogsAndMetrics(
+static Future<std::vector<std::pair<TLogInterface, EventMap>>> getTLogsAndMetrics(
     Reference<AsyncVar<ServerDBInfo>> db,
     std::unordered_map<NetworkAddress, WorkerInterface> address_workers) {
 	std::vector<TLogInterface> servers = db->get().logSystemConfig.allPresentLogs();
+	std::vector<std::string> eventNames{ "TLogMetrics" };
 	std::vector<std::pair<TLogInterface, EventMap>> results =
-	    wait(getServerMetrics(servers, address_workers, std::vector<std::string>{ "TLogMetrics" }));
+	    co_await getServerMetrics(servers, address_workers, std::move(eventNames));
 
-	return results;
+	co_return results;
 }
 
 // Returns list of tuples of grv proxy interfaces and their latency metrics
-ACTOR static Future<std::vector<std::pair<CommitProxyInterface, EventMap>>> getCommitProxiesAndMetrics(
+static Future<std::vector<std::pair<CommitProxyInterface, EventMap>>> getCommitProxiesAndMetrics(
     Reference<AsyncVar<ServerDBInfo>> db,
     std::unordered_map<NetworkAddress, WorkerInterface> address_workers) {
+	std::vector<std::string> eventNames{"CommitLatencyMetrics", 
+										"CommitLatencyBands", 
+										"CommitBatchingWindowSize",
+										"CommitBatchTransactions",
+										"CommitBatchBytes",
+										"CommitBatchingWaiting",
+										"CommitPreresolutionLatency",
+										"CommitResolutionLatency",
+										"CommitPostresolutionLatency",
+										"CommitTLogLoggingLatency",
+										"CommitReplyLatency"
+										};
 	std::vector<std::pair<CommitProxyInterface, EventMap>> results =
-	    wait(getServerMetrics(db->get().client.commitProxies,
-	                          address_workers,
-	                          std::vector<std::string>{ "CommitLatencyMetrics",
-	                                                    "CommitLatencyBands",
-	                                                    "CommitBatchingWindowSize",
-	                                                    "CommitBatchTransactions",
-	                                                    "CommitBatchBytes",
-	                                                    "CommitBatchingWaiting",
-	                                                    "CommitPreresolutionLatency",
-	                                                    "CommitResolutionLatency",
-	                                                    "CommitPostresolutionLatency",
-	                                                    "CommitTLogLoggingLatency",
-	                                                    "CommitReplyLatency" }));
-
-	return results;
+	    co_await getServerMetrics(db->get().client.commitProxies, address_workers, std::move(eventNames));
+
+	co_return results;
 }
 
-ACTOR static Future<std::vector<std::pair<GrvProxyInterface, EventMap>>> getGrvProxiesAndMetrics(
+static Future<std::vector<std::pair<GrvProxyInterface, EventMap>>> getGrvProxiesAndMetrics(
     Reference<AsyncVar<ServerDBInfo>> db,
     std::unordered_map<NetworkAddress, WorkerInterface> address_workers) {
-	std::vector<std::pair<GrvProxyInterface, EventMap>> results = wait(
-	    getServerMetrics(db->get().client.grvProxies,
-	                     address_workers,
-	                     std::vector<std::string>{ "GRVLatencyMetrics", "GRVLatencyBands", "GRVBatchLatencyMetrics" }));
-	return results;
+	std::vector<std::string> eventNames{ "GRVLatencyMetrics", "GRVLatencyBands", "GRVBatchLatencyMetrics" };
+	std::vector<std::pair<GrvProxyInterface, EventMap>> results =
+	    co_await getServerMetrics(db->get().client.grvProxies, address_workers, std::move(eventNames));
+	co_return results;
 }
 
 // Returns the number of zones eligible for recruiting new tLogs after zone failures, to maintain the current
@@ -2160,7 +2157,7 @@ JsonBuilderObject getPerfLimit(TraceEventFields const& ratekeeper, double transP
 	return perfLimit;
 }
 
-ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
+static AsyncResult<JsonBuilderObject> workloadStatusFetcher(
     Reference<AsyncVar<ServerDBInfo>> db,
     std::vector<WorkerDetails> workers,
     WorkerDetails mWorker,
@@ -2169,15 +2166,15 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
     JsonBuilderObject* data_overlay,
     std::set<std::string>* incomplete_reasons,
     Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture) {
-	state JsonBuilderObject statusObj;
-	state JsonBuilderObject operationsObj;
-	state JsonBuilderObject bytesObj;
-	state JsonBuilderObject keysObj;
+	JsonBuilderObject statusObj;
+	JsonBuilderObject operationsObj;
+	JsonBuilderObject bytesObj;
+	JsonBuilderObject keysObj;
 
 	// Writes and conflicts
 	try {
-		state std::vector<Future<TraceEventFields>> commitProxyStatFutures;
-		state std::vector<Future<TraceEventFields>> grvProxyStatFutures;
+		std::vector<Future<TraceEventFields>> commitProxyStatFutures;
+		std::vector<Future<TraceEventFields>> grvProxyStatFutures;
 		std::map<NetworkAddress, WorkerDetails> workersMap;
 		for (auto const& w : workers) {
 			workersMap[w.interf.address()] = w;
@@ -2198,8 +2195,8 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 			else
 				throw all_alternatives_failed(); // We need data from all proxies for this result to be trustworthy
 		}
-		state std::vector<TraceEventFields> commitProxyStats = wait(getAll(commitProxyStatFutures));
-		state std::vector<TraceEventFields> grvProxyStats = wait(getAll(grvProxyStatFutures));
+		std::vector<TraceEventFields> commitProxyStats = co_await getAll(commitProxyStatFutures);
+		std::vector<TraceEventFields> grvProxyStats = co_await getAll(grvProxyStatFutures);
 
 		StatusCounter txnStartOut;
 		StatusCounter txnSystemPriorityStartOut;
@@ -2256,11 +2253,11 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 
 	// Transactions
 	try {
-		state Future<TraceEventFields> f1 =
+		Future<TraceEventFields> f1 =
 		    timeoutError(rkWorker.interf.eventLogRequest.getReply(EventLogRequest("RkUpdate"_sr)), 1.0);
-		state Future<TraceEventFields> f2 =
+		Future<TraceEventFields> f2 =
 		    timeoutError(rkWorker.interf.eventLogRequest.getReply(EventLogRequest("RkUpdateBatch"_sr)), 1.0);
-		wait(success(f1) && success(f2));
+		co_await (success(f1) && success(f2));
 		TraceEventFields ratekeeper = f1.get();
 		TraceEventFields batchRatekeeper = f2.get();
 
@@ -2341,7 +2338,7 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 
 	// Reads
 	try {
-		ErrorOr<std::vector<StorageServerStatusInfo>> storageServers = wait(storageServerFuture);
+		ErrorOr<std::vector<StorageServerStatusInfo>> storageServers = co_await storageServerFuture;
 		if (!storageServers.present()) {
 			throw storageServers.getError();
 		}
@@ -2379,19 +2376,19 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
 	statusObj["keys"] = keysObj;
 	statusObj["bytes"] = bytesObj;
 
-	return statusObj;
+	co_return statusObj;
 }
 
-ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher(
+static AsyncResult<JsonBuilderObject> clusterSummaryStatisticsFetcher(
     WorkerEvents pMetrics,
     Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture,
     Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tlogFuture,
     std::set<std::string>* incomplete_reasons) {
-	state JsonBuilderObject statusObj;
+	JsonBuilderObject statusObj;
 	try {
-		state JsonBuilderObject cacheStatistics;
+		JsonBuilderObject cacheStatistics;
 
-		ErrorOr<std::vector<StorageServerStatusInfo>> storageServers = wait(storageServerFuture);
+		ErrorOr<std::vector<StorageServerStatusInfo>> storageServers = co_await storageServerFuture;
 
 		if (!storageServers.present()) {
 			throw storageServers.getError();
@@ -2414,7 +2411,7 @@ ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher(
 		cacheStatistics["storage_hit_rate"] =
 		    (storageCacheMissesHz == 0) ? 1.0 : storageCacheHitsHz / (storageCacheHitsHz + storageCacheMissesHz);
 
-		ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>> tlogServers = wait(tlogFuture);
+		ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>> tlogServers = co_await tlogFuture;
 
 		if (!tlogServers.present()) {
 			throw tlogServers.getError();
@@ -2444,7 +2441,7 @@ ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher(
 		incomplete_reasons->insert("Unknown cache statistics.");
 	}
 
-	return statusObj;
+	co_return statusObj;
 }
 
 static JsonBuilderObject tlogFetcher(int* logFaultTolerance,
@@ -2462,7 +2459,7 @@ static JsonBuilderObject tlogFetcher(int* logFaultTolerance,
 	for (const auto& tLogSet : tLogs) {
 		if (tLogSet.tLogs.size() == 0) {
 			// We can have LogSets where there are no tLogs but some LogRouters. It's the way
-			// recruiting is implemented for old LogRouters in TagPartitionedLogSystem, where
+			// recruiting is implemented for old LogRouters in LogSystem, where
 			// it adds an empty LogSet for missing locality.
 			continue;
 		}
@@ -2643,6 +2640,10 @@ static std::string getIssueDescription(std::string name) {
 		       "its "
 		       "parent directory are writable and that the cluster file has not been overwritten externally.";
 	}
+	if (name == "exclude_from_tlog_recruitment_low_disk") {
+		return "Process is temporarily excluded from TLog recruitment because its available disk space is below the "
+		       "minimum TLog threshold.";
+	}
 
 	// FIXME: name and description will be the same unless the message is 'incorrect_cluster_file_contents', which
 	// is currently the only possible message
@@ -2712,44 +2713,44 @@ static JsonBuilderArray getClientIssuesAsMessages(
 	return issuesList;
 }
 
-ACTOR Future<JsonBuilderObject> layerStatusFetcher(Database cx,
-                                                   JsonBuilderArray* messages,
-                                                   std::set<std::string>* incomplete_reasons) {
-	state StatusObject result;
-	state JSONDoc json(result);
-	state double tStart = now();
+AsyncResult<JsonBuilderObject> layerStatusFetcher(Database cx,
+                                                  JsonBuilderArray* messages,
+                                                  std::set<std::string>* incomplete_reasons) {
+	StatusObject result;
+	JSONDoc json(result);
+	double tStart = now();
 
 	try {
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				int64_t timeout_ms = 3000;
 				tr.setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t*)&timeout_ms, sizeof(int64_t)));
 
 				std::string jsonPrefix = layerStatusMetaPrefixRange.begin.toString() + "json/";
-				RangeResult jsonLayers = wait(tr.getRange(KeyRangeRef(jsonPrefix, strinc(jsonPrefix)), 1000));
+				RangeResult jsonLayers = co_await tr.getRange(KeyRangeRef(jsonPrefix, strinc(jsonPrefix)), 1000);
 				// TODO:  Also fetch other linked subtrees of meta keys
 
-				state std::vector<Future<RangeResult>> docFutures;
-				state int i;
-				for (i = 0; i < jsonLayers.size(); ++i)
+				std::vector<Future<RangeResult>> docFutures;
+				for (int i = 0; i < jsonLayers.size(); ++i) {
 					docFutures.push_back(
 					    tr.getRange(KeyRangeRef(jsonLayers[i].value, strinc(jsonLayers[i].value)), 1000));
+				}
 
 				result.clear();
 				JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get();
 
-				for (i = 0; i < docFutures.size(); ++i) {
-					state RangeResult docs = wait(docFutures[i]);
-					state int j;
-					for (j = 0; j < docs.size(); ++j) {
-						state json_spirit::mValue doc;
+				for (int i = 0; i < docFutures.size(); ++i) {
+					RangeResult docs = co_await docFutures[i];
+					for (int j = 0; j < docs.size(); ++j) {
+						json_spirit::mValue doc;
 						try {
 							json_spirit::read_string(docs[j].value.toString(), doc);
-							wait(yield());
+							co_await yield();
 							json.absorb(doc.get_obj());
-							wait(yield());
+							co_await yield();
 						} catch (Error& e) {
 							TraceEvent(SevWarn, "LayerStatusBadJSON").detail("Key", docs[j].key);
 						}
@@ -2758,8 +2759,9 @@ ACTOR Future<JsonBuilderObject> layerStatusFetcher(Database cx,
 				json.create("_valid") = true;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "LayerStatusError").error(e);
@@ -2777,68 +2779,69 @@ ACTOR Future<JsonBuilderObject> layerStatusFetcher(Database cx,
 	TraceEvent("LayerStatusFetcher")
 	    .detail("Duration", now() - tStart)
 	    .detail("StatusSize", statusObj.getFinalLength());
-	return statusObj;
+	co_return statusObj;
 }
 
-ACTOR Future<JsonBuilderObject> lockedStatusFetcher(Database cx,
-                                                    JsonBuilderArray* messages,
-                                                    std::set<std::string>* incomplete_reasons) {
-	state JsonBuilderObject statusObj;
-
-	state Transaction tr(cx);
-	state int timeoutSeconds = 5;
-	state Future<Void> getTimeout = delay(timeoutSeconds);
+AsyncResult<JsonBuilderObject> lockedStatusFetcher(Database cx,
+                                                   JsonBuilderArray* messages,
+                                                   std::set<std::string>* incomplete_reasons) {
+	JsonBuilderObject statusObj;
+	Transaction tr(cx);
+	int timeoutSeconds = 5;
+	Future<Void> getTimeout = delay(timeoutSeconds);
 
-	loop {
+	while (true) {
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+		Error err;
 		try {
-			choose {
-				when(Optional<Value> lockUID = wait(tr.get(databaseLockedKey))) {
-					if (lockUID.present()) {
-						statusObj["locked"] = true;
-						statusObj["lock_uid"] =
-						    BinaryReader::fromStringRef<UID>(lockUID.get().substr(10), Unversioned()).toString();
-					} else {
-						statusObj["locked"] = false;
-					}
-				}
-				when(wait(getTimeout)) {
-					incomplete_reasons->insert(
-					    format("Unable to determine if database is locked after %d seconds.", timeoutSeconds));
+			auto res = co_await race(tr.get(databaseLockedKey), getTimeout);
+			if (res.index() == 0) {
+				Optional<Value> lockUID = std::get<0>(std::move(res));
+				if (lockUID.present()) {
+					statusObj["locked"] = true;
+					statusObj["lock_uid"] =
+					    BinaryReader::fromStringRef<UID>(lockUID.get().substr(10), Unversioned()).toString();
+				} else {
+					statusObj["locked"] = false;
 				}
+			} else {
+				incomplete_reasons->insert(
+				    format("Unable to determine if database is locked after %d seconds.", timeoutSeconds));
 			}
 			break;
 		} catch (Error& e) {
-			try {
-				wait(tr.onError(e));
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw;
+			err = e;
+		}
+		try {
+			co_await tr.onError(err);
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled)
+				throw;
 
-				incomplete_reasons->insert(format("Unable to determine if database is locked (%s).", e.what()));
-				break;
-			}
+			incomplete_reasons->insert(format("Unable to determine if database is locked (%s).", e.what()));
+			break;
 		}
 	}
-	return statusObj;
+	co_return statusObj;
 }
 
-ACTOR Future<Optional<Value>> getActivePrimaryDC(Database cx, int* fullyReplicatedRegions, JsonBuilderArray* messages) {
-	state ReadYourWritesTransaction tr(cx);
+Future<Optional<Value>> getActivePrimaryDC(Database cx, int* fullyReplicatedRegions, JsonBuilderArray* messages) {
+	ReadYourWritesTransaction tr(cx);
 
-	state Future<Void> readTimeout = delay(5); // so that we won't loop forever
-	loop {
+	Future<Void> readTimeout = delay(5); // so that we won't loop forever
+	while (true) {
+		Error err;
 		try {
 			if (readTimeout.isReady()) {
 				throw timed_out();
 			}
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			state Future<RangeResult> fReplicaKeys = tr.getRange(datacenterReplicasKeys, CLIENT_KNOBS->TOO_MANY);
-			state Future<Optional<Value>> fPrimaryDatacenterKey = tr.get(primaryDatacenterKey);
-			wait(timeoutError(success(fPrimaryDatacenterKey) && success(fReplicaKeys), 5));
+			Future<RangeResult> fReplicaKeys = tr.getRange(datacenterReplicasKeys, CLIENT_KNOBS->TOO_MANY);
+			Future<Optional<Value>> fPrimaryDatacenterKey = tr.get(primaryDatacenterKey);
+			co_await timeoutError(success(fPrimaryDatacenterKey) && success(fReplicaKeys), 5);
 
 			*fullyReplicatedRegions = fReplicaKeys.get().size();
 
@@ -2846,58 +2849,59 @@ ACTOR Future<Optional<Value>> getActivePrimaryDC(Database cx, int* fullyReplicat
 				messages->push_back(
 				    JsonString::makeMessage("primary_dc_missing", "Unable to determine primary datacenter."));
 			}
-			return fPrimaryDatacenterKey.get();
+			co_return fPrimaryDatacenterKey.get();
 		} catch (Error& e) {
-			if (e.code() == error_code_timed_out) {
-				messages->push_back(
-				    JsonString::makeMessage("fetch_primary_dc_timeout", "Fetching primary DC timed out."));
-				return Optional<Value>();
-			} else {
-				wait(tr.onError(e));
-			}
+			err = e;
+		}
+		if (err.code() == error_code_timed_out) {
+			messages->push_back(JsonString::makeMessage("fetch_primary_dc_timeout", "Fetching primary DC timed out."));
+			co_return Optional<Value>();
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<std::pair<Optional<StorageWiggleMetrics>, Optional<StorageWiggleMetrics>>> readStorageWiggleMetrics(
+Future<std::pair<Optional<StorageWiggleMetrics>, Optional<StorageWiggleMetrics>>> readStorageWiggleMetrics(
     Database cx,
     bool use_system_priority) {
-	state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-	state Optional<StorageWiggleMetrics> primaryV;
-	state Optional<StorageWiggleMetrics> remoteV;
-	state StorageWiggleData wiggleState;
-	loop {
+	Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+	Optional<StorageWiggleMetrics> primaryV;
+	Optional<StorageWiggleMetrics> remoteV;
+	StorageWiggleData wiggleState;
+	while (true) {
+		Error err;
 		try {
 			if (use_system_priority) {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			}
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
-			wait(store(primaryV, wiggleState.storageWiggleMetrics(PrimaryRegion(true)).get(tr)) &&
-			     store(remoteV, wiggleState.storageWiggleMetrics(PrimaryRegion(false)).get(tr)));
-			return std::make_pair(primaryV, remoteV);
+			co_await (store(primaryV, wiggleState.storageWiggleMetrics(PrimaryRegion(true)).get(tr)) &&
+			          store(remoteV, wiggleState.storageWiggleMetrics(PrimaryRegion(false)).get(tr)));
+			co_return std::make_pair(primaryV, remoteV);
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
 // read storageWigglerStats through Read-only tx, then convert it to JSON field
-ACTOR Future<JsonBuilderObject> storageWigglerStatsFetcher(Optional<DataDistributorInterface> ddWorker,
-                                                           DatabaseConfiguration conf,
-                                                           Database cx,
-                                                           bool use_system_priority,
-                                                           JsonBuilderArray* messages) {
-
-	state Future<GetStorageWigglerStateReply> stateFut;
-	state Future<std::pair<Optional<StorageWiggleMetrics>, Optional<StorageWiggleMetrics>>> wiggleMetricsFut =
+AsyncResult<JsonBuilderObject> storageWigglerStatsFetcher(Optional<DataDistributorInterface> ddWorker,
+                                                          DatabaseConfiguration conf,
+                                                          Database cx,
+                                                          bool use_system_priority,
+                                                          JsonBuilderArray* messages) {
+
+	Future<GetStorageWigglerStateReply> stateFut;
+	Future<std::pair<Optional<StorageWiggleMetrics>, Optional<StorageWiggleMetrics>>> wiggleMetricsFut =
 	    timeoutError(readStorageWiggleMetrics(cx, use_system_priority), 2.0);
-	state JsonBuilderObject res;
+	JsonBuilderObject res;
 	if (ddWorker.present()) {
 		stateFut = timeoutError(ddWorker.get().storageWigglerState.getReply(GetStorageWigglerStateRequest()), 2.0);
-		wait(ready(stateFut));
+		co_await ready(stateFut);
 	} else {
-		return res;
+		co_return res;
 	}
 
 	try {
@@ -2905,12 +2909,12 @@ ACTOR Future<JsonBuilderObject> storageWigglerStatsFetcher(Optional<DataDistribu
 			throw timed_out();
 		}
 
-		wait(success(wiggleMetricsFut) && success(stateFut));
+		co_await (success(wiggleMetricsFut) && success(stateFut));
 		auto [primaryV, remoteV] = wiggleMetricsFut.get();
 		if (primaryV.present()) {
 			auto obj = primaryV.get().toJSON();
 			auto& reply = stateFut.get();
-			obj["state"] = StorageWiggler::getWiggleStateStr(static_cast<StorageWiggler::State>(reply.primary));
+			obj["state"] = StorageWigglerState::toString(static_cast<StorageWigglerState::Value>(reply.primary));
 			obj["last_state_change_timestamp"] = reply.lastStateChangePrimary;
 			obj["last_state_change_datetime"] = epochsToGMTString(reply.lastStateChangePrimary);
 			res["primary"] = obj;
@@ -2918,50 +2922,43 @@ ACTOR Future<JsonBuilderObject> storageWigglerStatsFetcher(Optional<DataDistribu
 		if (conf.regions.size() > 1 && remoteV.present()) {
 			auto obj = remoteV.get().toJSON();
 			auto& reply = stateFut.get();
-			obj["state"] = StorageWiggler::getWiggleStateStr(static_cast<StorageWiggler::State>(reply.remote));
+			obj["state"] = StorageWigglerState::toString(static_cast<StorageWigglerState::Value>(reply.remote));
 			obj["last_state_change_timestamp"] = reply.lastStateChangeRemote;
 			obj["last_state_change_datetime"] = epochsToGMTString(reply.lastStateChangeRemote);
 			res["remote"] = obj;
 		}
-		return res;
+		co_return res;
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled)
 			throw;
 		messages->push_back(JsonString::makeMessage("fetch_storage_wiggler_stats_timeout",
 		                                            "Fetching storage wiggler stats timed out."));
 	}
-	return res;
+	co_return res;
 }
 
 // constructs the cluster section of the json status output
-ACTOR Future<StatusReply> clusterGetStatus(
-    Reference<AsyncVar<ServerDBInfo>> db,
-    Database cx,
-    std::vector<WorkerDetails> workers,
-    std::vector<ProcessIssues> workerIssues,
-    std::vector<StorageServerMetaInfo> storageMetadatas,
-    std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* clientStatus,
-    ServerCoordinators coordinators,
-    std::vector<NetworkAddress> incompatibleConnections,
-    Version datacenterVersionDifference,
-    Version dcLogServerVersionDifference,
-    Version dcStorageServerVersionDifference,
-    std::unordered_map<NetworkAddress, double> excludedDegradedServers) {
+static AsyncResult<Void> clusterGetStatusImpl(Reference<ClusterGetStatusState> statusState,
+                                              Reference<AsyncVar<ServerDBInfo>> db,
+                                              Database cx,
+                                              std::vector<WorkerDetails> workers,
+                                              std::vector<ProcessIssues> workerIssues,
+                                              std::vector<StorageServerMetaInfo> storageMetadatas,
+                                              ServerCoordinators coordinators,
+                                              std::unordered_map<NetworkAddress, double> excludedDegradedServers) {
+	JsonBuilderObject& statusObj = statusState->statusObj;
+	JsonBuilderArray& messages = statusState->messages;
+	std::set<std::string>& status_incomplete_reasons = statusState->statusIncompleteReasons;
 
-	state double tStart = timer();
+	double tStart = timer();
 
-	state JsonBuilderArray messages;
-	state std::set<std::string> status_incomplete_reasons;
-	state WorkerDetails mWorker; // Master worker
-	state WorkerDetails ccWorker; // Cluster-Controller worker
-	state WorkerDetails ddWorker; // DataDistributor worker
-	state WorkerDetails rkWorker; // Ratekeeper worker
-	state WorkerDetails csWorker; // ConsistencyScan worker
+	WorkerDetails mWorker; // Master worker
+	WorkerDetails ccWorker; // Cluster-Controller worker
+	WorkerDetails ddWorker; // DataDistributor worker
+	WorkerDetails rkWorker; // Ratekeeper worker
+	WorkerDetails csWorker; // ConsistencyScan worker
 
 	try {
-
-		state JsonBuilderObject statusObj;
-
 		// Get the master Worker interface
 		Optional<WorkerDetails> _mWorker = getWorker(workers, db->get().master.address());
 		if (_mWorker.present()) {
@@ -3022,17 +3019,17 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		// Get latest events for various event types from ALL workers
 		// WorkerEvents is a map of worker's NetworkAddress to its event string
 		// The pair represents worker responses and a set of worker NetworkAddress strings which did not respond.
-		std::vector<Future<Optional<std::pair<WorkerEvents, std::set<std::string>>>>> futures;
-		futures.push_back(latestEventOnWorkers(workers, "MachineMetrics"));
-		futures.push_back(latestEventOnWorkers(workers, "ProcessMetrics"));
-		futures.push_back(latestEventOnWorkers(workers, "NetworkMetrics"));
-		futures.push_back(latestErrorOnWorkers(workers)); // Get all latest errors.
-		futures.push_back(latestEventOnWorkers(workers, "TraceFileOpenError"));
-		futures.push_back(latestEventOnWorkers(workers, "ProgramStart"));
+		std::vector<AsyncResult<Optional<std::pair<WorkerEvents, std::set<std::string>>>>> workerEventFetchers;
+		workerEventFetchers.push_back(latestEventOnWorkers(workers, "MachineMetrics"));
+		workerEventFetchers.push_back(latestEventOnWorkers(workers, "ProcessMetrics"));
+		workerEventFetchers.push_back(latestEventOnWorkers(workers, "NetworkMetrics"));
+		workerEventFetchers.push_back(latestErrorOnWorkers(workers)); // Get all latest errors.
+		workerEventFetchers.push_back(latestEventOnWorkers(workers, "TraceFileOpenError"));
+		workerEventFetchers.push_back(latestEventOnWorkers(workers, "ProgramStart"));
 
 		// Wait for all response pairs.
-		state std::vector<Optional<std::pair<WorkerEvents, std::set<std::string>>>> workerEventsVec =
-		    wait(getAll(futures));
+		std::vector<Optional<std::pair<WorkerEvents, std::set<std::string>>>> workerEventsVec =
+		    co_await getAll(std::move(workerEventFetchers));
 
 		// Create a unique set of all workers who were unreachable for 1 or more of the event requests above.
 		// Since each event request is independent and to all workers, workers can have responded to some
@@ -3059,68 +3056,83 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			messages.push_back(message);
 		}
 
-		state ProtocolVersionData protocolVersion = wait(getNewestProtocolVersion(cx, ccWorker));
+		ProtocolVersionData protocolVersion = co_await getNewestProtocolVersion(cx, ccWorker);
 
 		// construct status information for cluster subsections
-		state int statusCode = (int)RecoveryStatus::END;
-		state Future<JsonBuilderObject> recoveryStateStatusFuture =
-		    recoveryStateStatusFetcher(cx, ccWorker, mWorker, workers.size(), &status_incomplete_reasons, &statusCode);
+		int statusCode = (int)RecoveryStatus::END;
+		std::vector<AsyncResult<ErrorOr<JsonBuilderObject>>> clusterSubsectionFetchers;
+		clusterSubsectionFetchers.push_back(errorOr(recoveryStateStatusFetcher(
+		    cx, ccWorker, mWorker, workers.size(), &status_incomplete_reasons, &statusCode)));
+		clusterSubsectionFetchers.push_back(errorOr(timeoutError(getIdmpKeyStatus(cx), 5.0)));
+		clusterSubsectionFetchers.push_back(errorOr(versionEpochStatusFetcher(cx, &status_incomplete_reasons)));
 
-		state Future<JsonBuilderObject> idmpKeyStatusFuture = getIdmpKeyStatus(cx);
+		std::vector<ErrorOr<JsonBuilderObject>> clusterSubsectionStatuses =
+		    co_await getAll(std::move(clusterSubsectionFetchers));
 
-		state Future<JsonBuilderObject> versionEpochStatusFuture =
-		    versionEpochStatusFetcher(cx, &status_incomplete_reasons);
+		if (clusterSubsectionStatuses[0].isError()) {
+			throw clusterSubsectionStatuses[0].getError();
+		}
+		if (clusterSubsectionStatuses[2].isError()) {
+			throw clusterSubsectionStatuses[2].getError();
+		}
 
-		wait(waitForAll<JsonBuilderObject>(
-		    { recoveryStateStatusFuture, idmpKeyStatusFuture, versionEpochStatusFuture }));
+		statusObj["newest_protocol_version"] = format("%" PRIx64, protocolVersion.newestProtocolVersion.version());
+		statusObj["lowest_compatible_protocol_version"] =
+		    format("%" PRIx64, protocolVersion.lowestCompatibleProtocolVersion.version());
 
-		state JsonBuilderObject recoveryStateStatus = recoveryStateStatusFuture.get();
-		state JsonBuilderObject idmpKeyStatus = idmpKeyStatusFuture.get();
-		state JsonBuilderObject versionEpochStatus = versionEpochStatusFuture.get();
+		statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version());
+		statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString();
+		statusObj["bounce_impact"] = getBounceImpactInfo(statusCode);
+
+		JsonBuilderObject recoveryStateStatus = clusterSubsectionStatuses[0].get();
+		JsonBuilderObject idmpKeyStatus;
+		if (clusterSubsectionStatuses[1].present()) {
+			idmpKeyStatus = clusterSubsectionStatuses[1].get();
+		} else {
+			status_incomplete_reasons.insert("Unable to retrieve idempotency ID status.");
+		}
+		JsonBuilderObject versionEpochStatus = clusterSubsectionStatuses[2].get();
+
+		if (!recoveryStateStatus.empty()) {
+			statusObj["recovery_state"] = recoveryStateStatus;
+		}
+
+		statusObj["idempotency_ids"] = idmpKeyStatus;
+		statusObj["version_epoch"] = versionEpochStatus;
 
 		// machine metrics
-		state WorkerEvents mMetrics = workerEventsVec[0].present() ? workerEventsVec[0].get().first : WorkerEvents();
+		WorkerEvents mMetrics = workerEventsVec[0].present() ? workerEventsVec[0].get().first : WorkerEvents();
 		// process metrics
-		state WorkerEvents pMetrics = workerEventsVec[1].present() ? workerEventsVec[1].get().first : WorkerEvents();
-		state WorkerEvents networkMetrics =
-		    workerEventsVec[2].present() ? workerEventsVec[2].get().first : WorkerEvents();
-		state WorkerEvents latestError = workerEventsVec[3].present() ? workerEventsVec[3].get().first : WorkerEvents();
-		state WorkerEvents traceFileOpenErrors =
+		WorkerEvents pMetrics = workerEventsVec[1].present() ? workerEventsVec[1].get().first : WorkerEvents();
+		WorkerEvents networkMetrics = workerEventsVec[2].present() ? workerEventsVec[2].get().first : WorkerEvents();
+		WorkerEvents latestError = workerEventsVec[3].present() ? workerEventsVec[3].get().first : WorkerEvents();
+		WorkerEvents traceFileOpenErrors =
 		    workerEventsVec[4].present() ? workerEventsVec[4].get().first : WorkerEvents();
-		state WorkerEvents programStarts =
-		    workerEventsVec[5].present() ? workerEventsVec[5].get().first : WorkerEvents();
+		WorkerEvents programStarts = workerEventsVec[5].present() ? workerEventsVec[5].get().first : WorkerEvents();
 
 		if (db->get().recoveryCount > 0) {
 			statusObj["generation"] = db->get().recoveryCount;
 		}
 
-		state std::map<std::string, std::vector<JsonBuilderObject>> processIssues =
-		    getProcessIssuesAsMessages(workerIssues);
-		state std::vector<StorageServerStatusInfo> storageServers;
-		state std::vector<std::pair<TLogInterface, EventMap>> tLogs;
-		state std::vector<std::pair<CommitProxyInterface, EventMap>> commitProxies;
-		state std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies;
+		std::map<std::string, std::vector<JsonBuilderObject>> processIssues = getProcessIssuesAsMessages(workerIssues);
+		std::vector<StorageServerStatusInfo> storageServers;
+		std::vector<std::pair<TLogInterface, EventMap>> tLogs;
+		std::vector<std::pair<CommitProxyInterface, EventMap>> commitProxies;
+		std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies;
 
-		state JsonBuilderObject qos;
-		state JsonBuilderObject dataOverlay;
-		state JsonBuilderObject metacluster;
-		state JsonBuilderObject storageWiggler;
-		state std::unordered_set<UID> wiggleServers;
+		JsonBuilderObject qos;
+		JsonBuilderObject dataOverlay;
+		JsonBuilderObject metacluster;
+		JsonBuilderObject storageWiggler;
+		std::unordered_set<UID> wiggleServers;
 
-		statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version());
-		statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString();
-		statusObj["bounce_impact"] = getBounceImpactInfo(statusCode);
-		statusObj["newest_protocol_version"] = format("%" PRIx64, protocolVersion.newestProtocolVersion.version());
-		statusObj["lowest_compatible_protocol_version"] =
-		    format("%" PRIx64, protocolVersion.lowestCompatibleProtocolVersion.version());
-
-		state Optional<DatabaseConfiguration> configuration;
-		state Optional<LoadConfigurationResult> loadResult;
-		state std::unordered_map<NetworkAddress, WorkerInterface> address_workers;
+		Optional<DatabaseConfiguration> configuration;
+		Optional<LoadConfigurationResult> loadResult;
+		std::unordered_map<NetworkAddress, WorkerInterface> address_workers;
 
 		if (statusCode != RecoveryStatus::configuration_missing) {
 			std::pair<Optional<DatabaseConfiguration>, Optional<LoadConfigurationResult>> loadResults =
-			    wait(loadConfiguration(cx, &messages, &status_incomplete_reasons));
+			    co_await loadConfiguration(cx, &messages, &status_incomplete_reasons);
 			configuration = loadResults.first;
 			loadResult = loadResults.second;
 		}
@@ -3147,19 +3159,19 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		statusObj["machines"] = machineStatusFetcher(mMetrics, workers, configuration, &status_incomplete_reasons);
 
-		state std::vector<NetworkAddress> coordinatorAddresses;
+		std::vector<NetworkAddress> coordinatorAddresses;
 		if (configuration.present()) {
 			// Do the latency probe by itself to avoid interference from other status activities
-			state bool isAvailable = true;
+			bool isAvailable = true;
 			JsonBuilderObject latencyProbeResults =
-			    wait(latencyProbeFetcher(cx, &messages, &status_incomplete_reasons, &isAvailable));
+			    co_await latencyProbeFetcher(cx, &messages, &status_incomplete_reasons, &isAvailable);
 
 			statusObj["database_available"] = isAvailable;
 			if (!latencyProbeResults.empty()) {
 				statusObj["latency_probe"] = latencyProbeResults;
 			}
 
-			state std::vector<Future<Void>> warningFutures;
+			std::vector<Future<Void>> warningFutures;
 			if (isAvailable) {
 				warningFutures.push_back(consistencyCheckStatusFetcher(cx, &messages, &status_incomplete_reasons));
 				if (!SERVER_KNOBS->DISABLE_DUPLICATE_LOG_WARNING) {
@@ -3174,38 +3186,40 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				address_workers[worker.interf.address()] = worker.interf;
 			}
 
-			state Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture =
+			Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture =
 			    errorOr(getStorageServerStatusInfos(storageMetadatas, address_workers, rkWorker));
-			state Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tLogFuture =
+			Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tLogFuture =
 			    errorOr(getTLogsAndMetrics(db, address_workers));
-			state Future<ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>>> commitProxyFuture =
+			Future<ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>>> commitProxyFuture =
 			    errorOr(getCommitProxiesAndMetrics(db, address_workers));
-			state Future<ErrorOr<std::vector<std::pair<GrvProxyInterface, EventMap>>>> grvProxyFuture =
+			Future<ErrorOr<std::vector<std::pair<GrvProxyInterface, EventMap>>>> grvProxyFuture =
 			    errorOr(getGrvProxiesAndMetrics(db, address_workers));
 
-			state int minStorageReplicasRemaining = -1;
-			state int fullyReplicatedRegions = -1;
+			int minStorageReplicasRemaining = -1;
+			int fullyReplicatedRegions = -1;
 			// NOTE: here we should start all the transaction before wait in order to overlay latency
-			state Future<Optional<Value>> primaryDCFO = getActivePrimaryDC(cx, &fullyReplicatedRegions, &messages);
-			state std::vector<Future<JsonBuilderObject>> futures2;
-			futures2.push_back(dataStatusFetcher(ddWorker, configuration.get(), &minStorageReplicasRemaining));
-			futures2.push_back(workloadStatusFetcher(
+			Future<Optional<Value>> primaryDCFO = getActivePrimaryDC(cx, &fullyReplicatedRegions, &messages);
+			std::vector<AsyncResult<JsonBuilderObject>> statusSectionFetchers;
+			statusSectionFetchers.push_back(
+			    dataStatusFetcher(ddWorker, configuration.get(), &minStorageReplicasRemaining));
+			statusSectionFetchers.push_back(workloadStatusFetcher(
 			    db, workers, mWorker, rkWorker, &qos, &dataOverlay, &status_incomplete_reasons, storageServerFuture));
-			futures2.push_back(layerStatusFetcher(cx, &messages, &status_incomplete_reasons));
-			futures2.push_back(lockedStatusFetcher(cx, &messages, &status_incomplete_reasons));
-			futures2.push_back(
+			statusSectionFetchers.push_back(layerStatusFetcher(cx, &messages, &status_incomplete_reasons));
+			statusSectionFetchers.push_back(lockedStatusFetcher(cx, &messages, &status_incomplete_reasons));
+			statusSectionFetchers.push_back(
 			    clusterSummaryStatisticsFetcher(pMetrics, storageServerFuture, tLogFuture, &status_incomplete_reasons));
 
 			if (configuration.get().perpetualStorageWiggleSpeed > 0) {
-				state Future<std::vector<std::pair<UID, StorageWiggleValue>>> primaryWiggleValues;
-				state Future<std::vector<std::pair<UID, StorageWiggleValue>>> remoteWiggleValues;
+				Future<std::vector<std::pair<UID, StorageWiggleValue>>> primaryWiggleValues;
+				Future<std::vector<std::pair<UID, StorageWiggleValue>>> remoteWiggleValues;
+				AsyncResult<JsonBuilderObject> storageWigglerFetcher;
 				double timeout = g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01) ? 0.0 : 2.0;
 				primaryWiggleValues = timeoutError(readStorageWiggleValues(cx, true, true), timeout);
 				remoteWiggleValues = timeoutError(readStorageWiggleValues(cx, false, true), timeout);
-				wait(store(
-				         storageWiggler,
-				         storageWigglerStatsFetcher(db->get().distributor, configuration.get(), cx, true, &messages)) &&
-				     ready(primaryWiggleValues) && ready(remoteWiggleValues));
+				storageWigglerFetcher =
+				    storageWigglerStatsFetcher(db->get().distributor, configuration.get(), cx, true, &messages);
+				co_await (ready(primaryWiggleValues) && ready(remoteWiggleValues));
+				storageWiggler = co_await std::move(storageWigglerFetcher);
 
 				if (primaryWiggleValues.canGet()) {
 					for (auto& p : primaryWiggleValues.get())
@@ -3224,11 +3238,15 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				}
 			}
 
-			state std::vector<JsonBuilderObject> workerStatuses = wait(getAll(futures2));
-			wait(success(primaryDCFO));
+			std::vector<JsonBuilderObject> workerStatuses = co_await getAll(std::move(statusSectionFetchers));
+			co_await primaryDCFO;
+
+			if (primaryDCFO.get().present()) {
+				statusObj["active_primary_dc"] = primaryDCFO.get().get();
+			}
 
 			ErrorOr<std::vector<NetworkAddress>> addresses =
-			    wait(errorOr(timeoutError(coordinators.ccr->getConnectionString().tryResolveHostnames(), 5.0)));
+			    co_await errorOr(timeoutError(coordinators.ccr->getConnectionString().tryResolveHostnames(), 5.0));
 			if (addresses.present()) {
 				coordinatorAddresses = std::move(addresses.get());
 			} else {
@@ -3253,12 +3271,8 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			                                fullyReplicatedRegions,
 			                                loadResult.present() && loadResult.get().healthyZone.present());
 
-			state JsonBuilderObject configObj =
-			    configurationFetcher(configuration, coordinators, &status_incomplete_reasons);
+			JsonBuilderObject configObj = configurationFetcher(configuration, coordinators, &status_incomplete_reasons);
 
-			if (primaryDCFO.get().present()) {
-				statusObj["active_primary_dc"] = primaryDCFO.get().get();
-			}
 			// configArr could be empty
 			if (!configObj.empty()) {
 				statusObj["configuration"] = configObj;
@@ -3274,8 +3288,6 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			if (!qos.empty())
 				statusObj["qos"] = qos;
 
-			statusObj["version_epoch"] = versionEpochStatus;
-
 			// Merge dataOverlay into data
 			JsonBuilderObject& clusterDataSection = workerStatuses[0];
 
@@ -3298,7 +3310,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			}
 
 			// Need storage servers now for processStatusFetcher() below.
-			ErrorOr<std::vector<StorageServerStatusInfo>> _storageServers = wait(storageServerFuture);
+			ErrorOr<std::vector<StorageServerStatusInfo>> _storageServers = co_await storageServerFuture;
 			if (_storageServers.present()) {
 				storageServers = _storageServers.get();
 			} else {
@@ -3307,7 +3319,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			}
 
 			// ...also tlogs
-			ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>> _tLogs = wait(tLogFuture);
+			ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>> _tLogs = co_await tLogFuture;
 			if (_tLogs.present()) {
 				tLogs = _tLogs.get();
 			} else {
@@ -3316,7 +3328,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			}
 
 			// ...also commit proxies
-			ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>> _commitProxies = wait(commitProxyFuture);
+			ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>> _commitProxies = co_await commitProxyFuture;
 			if (_commitProxies.present()) {
 				commitProxies = _commitProxies.get();
 			} else {
@@ -3325,7 +3337,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			}
 
 			// ...also grv proxies
-			ErrorOr<std::vector<std::pair<GrvProxyInterface, EventMap>>> _grvProxies = wait(grvProxyFuture);
+			ErrorOr<std::vector<std::pair<GrvProxyInterface, EventMap>>> _grvProxies = co_await grvProxyFuture;
 			if (_grvProxies.present()) {
 				grvProxies = _grvProxies.get();
 			} else {
@@ -3333,7 +3345,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				    JsonBuilder::makeMessage("grv_proxies_error", "Timed out trying to retrieve grv proxies."));
 			}
 
-			wait(waitForAll(warningFutures));
+			co_await waitForAll(warningFutures);
 		} else {
 			// Set layers status to { _valid: false, error: "configurationMissing"}
 			JsonBuilderObject layers;
@@ -3343,36 +3355,26 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		}
 
 		JsonBuilderObject processStatus =
-		    wait(processStatusFetcher(db,
-		                              workers,
-		                              pMetrics,
-		                              mMetrics,
-		                              networkMetrics,
-		                              latestError,
-		                              traceFileOpenErrors,
-		                              programStarts,
-		                              processIssues,
-		                              storageServers,
-		                              tLogs,
-		                              commitProxies,
-		                              grvProxies,
-		                              coordinators,
-		                              coordinatorAddresses,
-		                              cx,
-		                              configuration,
-		                              loadResult.present() ? loadResult.get().healthyZone : Optional<Key>(),
-		                              &status_incomplete_reasons));
+		    co_await processStatusFetcher(db,
+		                                  workers,
+		                                  pMetrics,
+		                                  mMetrics,
+		                                  networkMetrics,
+		                                  latestError,
+		                                  traceFileOpenErrors,
+		                                  programStarts,
+		                                  processIssues,
+		                                  storageServers,
+		                                  tLogs,
+		                                  commitProxies,
+		                                  grvProxies,
+		                                  coordinators,
+		                                  coordinatorAddresses,
+		                                  cx,
+		                                  configuration,
+		                                  loadResult.present() ? loadResult.get().healthyZone : Optional<Key>(),
+		                                  &status_incomplete_reasons);
 		statusObj["processes"] = processStatus;
-		statusObj["clients"] = clientStatusFetcher(clientStatus);
-
-		JsonBuilderArray incompatibleConnectionsArray;
-		for (auto it : incompatibleConnections) {
-			incompatibleConnectionsArray.push_back(it.toString());
-		}
-		statusObj["incompatible_connections"] = incompatibleConnectionsArray;
-		statusObj["datacenter_lag"] = getLagObject(datacenterVersionDifference);
-		statusObj["logserver_lag"] = getLagObject(dcLogServerVersionDifference);
-		statusObj["storageserver_lag"] = getLagObject(dcStorageServerVersionDifference);
 
 		int activeTSSCount = 0;
 		JsonBuilderArray wiggleServerAddress;
@@ -3404,36 +3406,22 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		}
 		statusObj["degraded_processes"] = totalDegraded;
 
-		if (!recoveryStateStatus.empty())
-			statusObj["recovery_state"] = recoveryStateStatus;
-
-		statusObj["idempotency_ids"] = idmpKeyStatus;
-
-		// cluster messages subsection;
-		JsonBuilderArray clientIssuesArr = getClientIssuesAsMessages(clientStatus);
-		if (clientIssuesArr.size() > 0) {
-			JsonBuilderObject clientIssueMessage =
-			    JsonBuilder::makeMessage("client_issues", "Some clients of this cluster have issues.");
-			clientIssueMessage["issues"] = clientIssuesArr;
-			messages.push_back(clientIssueMessage);
-		}
-
 		// Fetch Consistency Scan Information
 		try {
-			state ConsistencyScanState cs;
-			state ConsistencyScanState::Config config;
-			state ConsistencyScanState::RoundStats currentRound;
-			state ConsistencyScanState::LifetimeStats lifetime;
-			state ConsistencyScanState::StatsHistoryMap::RangeResultType olderStats;
+			ConsistencyScanState cs;
+			ConsistencyScanState::Config config;
+			ConsistencyScanState::RoundStats currentRound;
+			ConsistencyScanState::LifetimeStats lifetime;
+			ConsistencyScanState::StatsHistoryMap::RangeResultType olderStats;
 			auto sysDB = SystemDBWriteLockedNow(cx.getReference());
 
 			// TODO: Use the same transaction.
-			wait(timeoutError(
+			co_await timeoutError(
 			    store(config, cs.config().getD(sysDB)) && store(currentRound, cs.currentRoundStats().getD(sysDB)) &&
 			        store(lifetime, cs.lifetimeStats().getD(sysDB)) &&
 			        store(olderStats,
 			              cs.roundStatsHistory().getRange(sysDB, {}, {}, 5, Snapshot::False, Reverse::True)),
-			    2.0));
+			    2.0);
 			json_spirit::mObject csDoc;
 			csDoc["configuration"] = config.toJSON();
 			csDoc["lifetime_stats"] = lifetime.toJSON();
@@ -3452,21 +3440,6 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			                                           "Fetching consistency scan state timed out."));
 		}
 
-		// Create the status_incomplete message if there were any reasons that the status is incomplete.
-		if (!status_incomplete_reasons.empty()) {
-			JsonBuilderObject incomplete_message =
-			    JsonBuilder::makeMessage("status_incomplete", "Unable to retrieve all status information.");
-			// Make a JSON array of all of the reasons in the status_incomplete_reasons set.
-			JsonBuilderArray reasons;
-			for (auto i : status_incomplete_reasons) {
-				reasons.push_back(JsonBuilderObject().setKey("description", i));
-			}
-			incomplete_message["reasons"] = reasons;
-			messages.push_back(incomplete_message);
-		}
-
-		statusObj["messages"] = messages;
-
 		int64_t clusterTime = g_network->timer();
 		if (clusterTime != -1) {
 			statusObj["cluster_controller_timestamp"] = clusterTime;
@@ -3480,14 +3453,85 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		    .detail("Duration", timer() - tStart)
 		    .detail("StatusSize", statusObj.getFinalLength());
 
-		return StatusReply(statusObj.getJson());
-
+		co_return;
 	} catch (Error& e) {
 		TraceEvent(SevError, "StatusError").error(e);
 		throw;
 	}
 }
 
+AsyncResult<StatusReply> clusterGetStatus(
+    Reference<AsyncVar<ServerDBInfo>> db,
+    Database cx,
+    std::vector<WorkerDetails> workers,
+    std::vector<ProcessIssues> workerIssues,
+    std::vector<StorageServerMetaInfo> storageMetadatas,
+    std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* clientStatus,
+    ServerCoordinators coordinators,
+    std::vector<NetworkAddress> incompatibleConnections,
+    Version datacenterVersionDifference,
+    Version dcLogServerVersionDifference,
+    Version dcStorageServerVersionDifference,
+    std::unordered_map<NetworkAddress, double> excludedDegradedServers,
+    double deadlineTimeout) {
+	auto statusState = makeReference<ClusterGetStatusState>();
+	JsonBuilderObject& statusObj = statusState->statusObj;
+	JsonBuilderArray& messages = statusState->messages;
+	std::set<std::string>& status_incomplete_reasons = statusState->statusIncompleteReasons;
+
+	auto result = co_await race(
+	    errorOr(clusterGetStatusImpl(
+	        statusState, db, cx, workers, workerIssues, storageMetadatas, coordinators, excludedDegradedServers)),
+	    delay(deadlineTimeout));
+
+	if (result.index() == 0) {
+		ErrorOr<Void> statusResult = std::get<0>(std::move(result));
+		if (statusResult.isError()) {
+			status_incomplete_reasons.insert(
+			    fmt::format("Status collection threw: {}", statusResult.getError().what()));
+		}
+	} else {
+		status_incomplete_reasons.insert("Status collection deadline exceeded.");
+	}
+
+	// Other construction bits
+	statusObj["clients"] = clientStatusFetcher(clientStatus);
+
+	// cluster messages subsection;
+	JsonBuilderArray clientIssuesArr = getClientIssuesAsMessages(clientStatus);
+	if (clientIssuesArr.size() > 0) {
+		JsonBuilderObject clientIssueMessage =
+		    JsonBuilder::makeMessage("client_issues", "Some clients of this cluster have issues.");
+		clientIssueMessage["issues"] = clientIssuesArr;
+		messages.push_back(clientIssueMessage);
+	}
+
+	JsonBuilderArray incompatibleConnectionsArray;
+	for (auto it : incompatibleConnections) {
+		incompatibleConnectionsArray.push_back(it.toString());
+	}
+	statusObj["incompatible_connections"] = incompatibleConnectionsArray;
+	statusObj["datacenter_lag"] = getLagObject(datacenterVersionDifference);
+	statusObj["logserver_lag"] = getLagObject(dcLogServerVersionDifference);
+	statusObj["storageserver_lag"] = getLagObject(dcStorageServerVersionDifference);
+
+	// Create the status_incomplete message if there were any reasons that the status is incomplete.
+	if (!status_incomplete_reasons.empty()) {
+		JsonBuilderObject incomplete_message =
+		    JsonBuilder::makeMessage("status_incomplete", "Unable to retrieve all status information.");
+		// Make a JSON array of all of the reasons in the status_incomplete_reasons set.
+		JsonBuilderArray reasons;
+		for (auto i : status_incomplete_reasons) {
+			reasons.push_back(JsonBuilderObject().setKey("description", i));
+		}
+		incomplete_message["reasons"] = reasons;
+		messages.push_back(incomplete_message);
+	}
+	statusObj["messages"] = messages;
+
+	co_return StatusReply(statusObj.getJson());
+}
+
 StatusReply clusterGetFaultToleranceStatus(const std::string& statusStr) {
 	double tStart = timer();
 
@@ -3869,3 +3913,111 @@ TEST_CASE("/status/json/merging") {
 
 	return Void();
 }
+
+// Test that clusterGetStatus returns partial results within the specified deadline
+// even when the database is unavailable and transactions hang forever.
+TEST_CASE("/fdbserver/clustercontroller/clusterGetStatusTimeout") {
+	// Create mock ServerDBInfo with fake interfaces that will never respond
+	NetworkAddress masterAddr(IPAddress(0x01010101), 1);
+	NetworkAddress ccAddr(IPAddress(0x09090909), 1);
+	NetworkAddress proxyAddr(IPAddress(0x07070707), 1);
+	UID testUID(1, 2);
+
+	ServerDBInfo testDbInfo;
+	testDbInfo.clusterInterface.changeCoordinators =
+	    RequestStream<struct ChangeCoordinatorsRequest>(Endpoint({ ccAddr }, testUID));
+
+	MasterInterface mInterface;
+	mInterface.getCommitVersion = RequestStream<struct GetCommitVersionRequest>(Endpoint({ masterAddr }, testUID));
+	testDbInfo.master = mInterface;
+
+	// Add a GRV proxy that points to a non-existent address; any GRV request will hang.
+	GrvProxyInterface proxyInterf;
+	proxyInterf.getConsistentReadVersion =
+	    PublicRequestStream<struct GetReadVersionRequest>(Endpoint({ proxyAddr }, testUID));
+	testDbInfo.client.grvProxies.push_back(proxyInterf);
+
+	testDbInfo.recoveryState = RecoveryState::ACCEPTING_COMMITS;
+	testDbInfo.recoveryCount = 1;
+
+	auto db = makeReference<AsyncVar<ServerDBInfo>>(testDbInfo);
+	Database cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True);
+
+	// Create minimal ServerCoordinators
+	ServerCoordinators coordinators(
+	    Reference<IClusterConnectionRecord>(new ClusterConnectionMemoryRecord(ClusterConnectionString())));
+
+	// Empty containers for other parameters
+	std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>> clientStatus;
+
+	// Call clusterGetStatus with a short timeout (1 second) to force the
+	// timeout path to trigger, since internal timeouts take ~13 seconds.
+	double startTime = now();
+	AsyncResult<StatusReply> statusFuture = clusterGetStatus(db,
+	                                                         cx,
+	                                                         std::vector<WorkerDetails>(),
+	                                                         std::vector<ProcessIssues>(),
+	                                                         std::vector<StorageServerMetaInfo>(),
+	                                                         &clientStatus,
+	                                                         coordinators,
+	                                                         std::vector<NetworkAddress>(),
+	                                                         Version(0),
+	                                                         Version(0),
+	                                                         Version(0),
+	                                                         std::unordered_map<NetworkAddress, double>(),
+	                                                         1.0);
+	// Since we set the timeout above to 1 second, 5 seconds should be more than enough to complete the test.
+	// If it takes any longer, then the test case has failed
+	const double MAX_ACCEPTABLE_DELAY = 5.0;
+	auto result = co_await race(std::move(statusFuture), delay(MAX_ACCEPTABLE_DELAY));
+	if (result.index() != 0) {
+		TraceEvent(SevError, "ClusterGetStatusTimeoutTestFailed").detail("Elapsed", now() - startTime);
+		ASSERT(false);
+	}
+
+	StatusReply reply = std::get<0>(std::move(result));
+	double elapsed = now() - startTime;
+
+	TraceEvent("ClusterGetStatusTimeoutTest")
+	    .detail("Elapsed", elapsed)
+	    .detail("StatusSize", reply.statusStr.size())
+	    .detail("StatusJSON", reply.statusStr);
+	fmt::print("=== Status JSON ({:.2f}s) ===\n{}\n=== End Status JSON ===\n", elapsed, reply.statusStr);
+
+	ASSERT_LT(elapsed, MAX_ACCEPTABLE_DELAY);
+
+	json_spirit::mValue mv = readJSONStrictly(reply.statusStr);
+	auto obj = mv.get_obj();
+
+	// Fields set in the outer clusterGetStatus are always present after deadline.
+	ASSERT(obj.count("messages") > 0);
+	ASSERT(obj.count("clients") > 0);
+	ASSERT(obj.count("incompatible_connections") > 0);
+	ASSERT(obj.count("datacenter_lag") > 0);
+
+	// Verify the deadline-exceeded message is in messages.
+	auto& messagesArr = obj["messages"].get_array();
+	bool foundTimeoutMsg = false;
+	bool foundIncomplete = false;
+	for (auto& msg : messagesArr) {
+		auto& msgObj = msg.get_obj();
+		if (msgObj.count("name") && msgObj["name"].get_str() == "status_incomplete") {
+			foundIncomplete = true;
+			if (msgObj.count("reasons")) {
+				for (auto& reason : msgObj["reasons"].get_array()) {
+					auto& reasonObj = reason.get_obj();
+					if (reasonObj.count("description") &&
+					    reasonObj["description"].get_str() == "Status collection deadline exceeded.") {
+						foundTimeoutMsg = true;
+					}
+				}
+			}
+		}
+	}
+	fmt::print("Found status_incomplete message: {}\n", foundIncomplete);
+	fmt::print("Found deadline exceeded reason: {}\n", foundTimeoutMsg);
+	ASSERT(foundIncomplete);
+	ASSERT(foundTimeoutMsg);
+
+	co_return;
+}
diff --git a/fdbserver/include/fdbserver/Status.actor.h b/fdbserver/clustercontroller/Status.h
similarity index 50%
rename from fdbserver/include/fdbserver/Status.actor.h
rename to fdbserver/clustercontroller/Status.h
index 66ea19db563..827ab5f6b13 100644
--- a/fdbserver/include/fdbserver/Status.actor.h
+++ b/fdbserver/clustercontroller/Status.h
@@ -18,20 +18,14 @@
  * limitations under the License.
  */
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_ACTOR_STATUS_G_H)
-#define FDBSERVER_ACTOR_STATUS_G_H
-#include "fdbserver/Status.actor.g.h"
-#elif !defined(FDBSERVER_ACTOR_STATUS_H)
-#define FDBSERVER_ACTOR_STATUS_H
 #pragma once
 
 #include "fdbrpc/fdbrpc.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/CoordinationInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/MasterInterface.h"
 #include "fdbclient/ClusterInterface.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 struct ProcessIssues {
 	NetworkAddress address;
 	Standalone<VectorRef<StringRef>> issues;
@@ -39,27 +33,19 @@ struct ProcessIssues {
 	ProcessIssues(NetworkAddress address, Standalone<VectorRef<StringRef>> issues) : address(address), issues(issues) {}
 };
 
-Future<StatusReply> clusterGetStatus(
-    Reference<AsyncVar<struct ServerDBInfo>> const& db,
-    Database const& cx,
-    std::vector<WorkerDetails> const& workers,
-    std::vector<ProcessIssues> const& workerIssues,
-    std::vector<StorageServerMetaInfo> const& storageMetadatas,
-    std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* const& clientStatus,
-    ServerCoordinators const& coordinators,
-    std::vector<NetworkAddress> const& incompatibleConnections,
-    Version const& datacenterVersionDifference,
-    Version const& dcLogServerVersionDifference,
-    Version const& dcStorageServerVersionDifference,
-    std::unordered_map<NetworkAddress, double /* latest time at which address was excluded */> const&
-        excludedDegradedServers);
-
-StatusReply clusterGetFaultToleranceStatus(const std::string& statusString);
-
-struct WorkerEvents : std::map<NetworkAddress, TraceEventFields> {};
-ACTOR Future<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventOnWorkers(
+AsyncResult<StatusReply> clusterGetStatus(
+    Reference<AsyncVar<struct ServerDBInfo>> db,
+    Database cx,
     std::vector<WorkerDetails> workers,
-    std::string eventName);
+    std::vector<ProcessIssues> workerIssues,
+    std::vector<StorageServerMetaInfo> storageMetadatas,
+    std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* clientStatus,
+    ServerCoordinators coordinators,
+    std::vector<NetworkAddress> incompatibleConnections,
+    Version datacenterVersionDifference,
+    Version dcLogServerVersionDifference,
+    Version dcStorageServerVersionDifference,
+    std::unordered_map<NetworkAddress, double /* latest time at which address was excluded */> excludedDegradedServers,
+    double deadlineTimeout);
 
-#include "flow/unactorcompiler.h"
-#endif
+StatusReply clusterGetFaultToleranceStatus(const std::string& statusString);
diff --git a/fdbserver/clustercontroller/include/fdbserver/clustercontroller/ClusterController.h b/fdbserver/clustercontroller/include/fdbserver/clustercontroller/ClusterController.h
new file mode 100644
index 00000000000..255efe18699
--- /dev/null
+++ b/fdbserver/clustercontroller/include/fdbserver/clustercontroller/ClusterController.h
@@ -0,0 +1,34 @@
+/*
+ * ClusterController.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbrpc/Locality.h"
+#include "flow/flow.h"
+
+struct ClusterControllerFullInterface;
+
+Future<Void> clusterController(Reference<IClusterConnectionRecord> ccr,
+                               Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
+                               Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
+                               LocalityData locality,
+                               Reference<AsyncVar<Optional<UID>>> clusterId);
diff --git a/fdbserver/commitproxy/CMakeLists.txt b/fdbserver/commitproxy/CMakeLists.txt
new file mode 100644
index 00000000000..ffc577c083c
--- /dev/null
+++ b/fdbserver/commitproxy/CMakeLists.txt
@@ -0,0 +1,17 @@
+fdb_find_sources(FDBSERVER_COMMITPROXY_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_commitproxy SRCS ${FDBSERVER_COMMITPROXY_SRCS})
+add_fdbserver_link_test(fdbserver_commitproxylinktest
+  fdbserver_commitproxy
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_commitproxy)
+target_include_directories(fdbserver_commitproxy
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_include_directories(fdbserver_commitproxy
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(fdbserver_commitproxy PRIVATE fdbserver_core fdbserver_kvstore fdbserver_logsystem)
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/commitproxy/CommitProxyServer.cpp
similarity index 73%
rename from fdbserver/CommitProxyServer.actor.cpp
rename to fdbserver/commitproxy/CommitProxyServer.cpp
index 7c304f1b513..ee36d8b4ea4 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/commitproxy/CommitProxyServer.cpp
@@ -1,5 +1,5 @@
 /*
- * CommitProxyServer.actor.cpp
+ * CommitProxyServer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,12 +25,12 @@
 
 #include "fdbclient/AccumulativeChecksum.h"
 #include "fdbclient/Atomic.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BuildIdempotencyIdMutations.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/IdempotencyId.actor.h"
+#include "fdbclient/IdempotencyId.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/NativeAPI.actor.h"
@@ -38,68 +38,38 @@
 #include "fdbclient/Tracing.h"
 #include "fdbclient/TransactionLineage.h"
 #include "fdbrpc/sim_validation.h"
-#include "fdbserver/AccumulativeChecksumUtil.h"
-#include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/ConflictSet.h"
-#include "fdbserver/DataDistributorInterface.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/ProxyCommitData.actor.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/AccumulativeChecksumUtil.h"
+#include "fdbserver/logsystem/ApplyMetadataMutation.h"
+#include "fdbserver/core/ConflictBatch.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/kvstore/FDBExecHelper.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/MutationTracking.h"
+#include "ProxyCommitData.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/commitproxy/CommitProxyServer.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
 #include "flow/CodeProbe.h"
+#include "flow/CoroUtils.h"
 #include "flow/EncryptUtils.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
 #include "flow/Trace.h"
 #include "flow/network.h"
-
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 using WriteMutationRefVar = std::variant<MutationRef, VectorRef<MutationRef>>;
 
-ACTOR Future<Void> broadcastTxnRequest(TxnStateRequest req, int sendAmount, bool sendReply) {
-	state ReplyPromise<Void> reply = req.reply;
-	resetReply(req);
-	std::vector<Future<Void>> replies;
-	int currentStream = 0;
-	std::vector<Endpoint> broadcastEndpoints = req.broadcastInfo;
-	for (int i = 0; i < sendAmount && currentStream < broadcastEndpoints.size(); i++) {
-		std::vector<Endpoint> endpoints;
-		RequestStream<TxnStateRequest> cur(broadcastEndpoints[currentStream++]);
-		while (currentStream < broadcastEndpoints.size() * (i + 1) / sendAmount) {
-			endpoints.push_back(broadcastEndpoints[currentStream++]);
-		}
-		req.broadcastInfo = endpoints;
-		replies.push_back(brokenPromiseToNever(cur.getReply(req)));
-		resetReply(req);
-	}
-	wait(waitForAll(replies));
-	if (sendReply) {
-		reply.send(Void());
-	}
-	return Void();
-}
-
-ACTOR void discardCommit(UID id, Future<LogSystemDiskQueueAdapter::CommitMessage> fcm, Future<Void> dummyCommitState) {
-	ASSERT(!dummyCommitState.isReady());
-	LogSystemDiskQueueAdapter::CommitMessage cm = wait(fcm);
-	TraceEvent("Discarding", id).detail("Count", cm.messages.size());
-	cm.acknowledge.send(Void());
-	ASSERT(dummyCommitState.isReady());
-}
-
 struct ResolutionRequestBuilder {
 	const ProxyCommitData* self;
 
@@ -280,92 +250,93 @@ Future<Void> makeBatchTimeoutFuture(double baseDelay) {
 	           : delay(targetDelay, TaskPriority::ProxyCommitBatcher);
 }
 
-ACTOR Future<Void> commitBatcher(ProxyCommitData* commitData,
-                                 PromiseStream<std::pair<std::vector<CommitTransactionRequest>, int>> out,
-                                 FutureStream<CommitTransactionRequest> in,
-                                 int desiredBytes,
-                                 int64_t memBytesLimit) {
-	wait(delayJittered(commitData->commitBatchInterval, TaskPriority::ProxyCommitBatcher));
+Future<Void> commitBatcher(ProxyCommitData* commitData,
+                           PromiseStream<std::pair<std::vector<CommitTransactionRequest>, int>> out,
+                           FutureStream<CommitTransactionRequest> in,
+                           int desiredBytes,
+                           int64_t memBytesLimit) {
+	co_await delayJittered(commitData->commitBatchInterval, TaskPriority::ProxyCommitBatcher);
 
-	state double lastBatch = 0;
-	state int64_t batchesInProgress = 0;
+	double lastBatch = 0;
+	int64_t batchesInProgress = 0;
 
-	loop {
-		state Future<Void> timeout;
-		state std::vector<CommitTransactionRequest> batch;
-		state int batchBytes = 0;
+	while (true) {
+		Future<Void> timeout;
+		std::vector<CommitTransactionRequest> batch;
+		int batchBytes = 0;
 		// TODO: Enable this assertion (currently failing with gcc)
 		// static_assert(std::is_nothrow_move_constructible_v<CommitTransactionRequest>);
 
-		timeout = makeBatchTimeoutFuture(SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL);
+	timeout = makeBatchTimeoutFuture(SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL);
 
 		while (!timeout.isReady() &&
-		       !(batch.size() == SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_COUNT_MAX || batchBytes >= desiredBytes) &&
+			   !(batch.size() == SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_COUNT_MAX || batchBytes >= desiredBytes) &&
 		       !(SERVER_KNOBS->COMMIT_BATCH_MAX_IN_PROGRESS > 0 &&
 		         batchesInProgress < SERVER_KNOBS->COMMIT_BATCH_MAX_IN_PROGRESS && batch.size())) {
-			choose {
-				when(CommitTransactionRequest req = waitNext(in)) {
-					// WARNING: this code is run at a high priority, so it needs to do as little work as possible
-					int bytes = getBytes(req);
-
-					// Drop requests if memory is under severe pressure
-					if (commitData->commitBatchesMemBytesCount + bytes > memBytesLimit) {
-						++commitData->stats.txnCommitErrors;
-						req.reply.sendError(commit_proxy_memory_limit_exceeded());
-						TraceEvent(SevWarnAlways, "ProxyCommitBatchMemoryThresholdExceeded")
-						    .suppressFor(60)
-						    .detail("MemBytesCount", commitData->commitBatchesMemBytesCount)
-						    .detail("MemLimit", memBytesLimit);
-						continue;
-					}
-
-					if (bytes > FLOW_KNOBS->PACKET_WARNING) {
-						TraceEvent(SevWarn, "LargeTransaction")
-						    .suppressFor(1.0)
-						    .detail("Size", bytes)
-						    .detail("Client", req.reply.getEndpoint().getPrimaryAddress());
-					}
-
-					++commitData->stats.txnCommitIn;
-					commitData->stats.uniqueClients.insert(req.reply.getEndpoint().getPrimaryAddress());
+			auto res = co_await race(in, timeout, commitData->triggerCommit.onChange(), commitData->committedBatches.getFuture());
+			if (res.index() == 0) {
+				CommitTransactionRequest req = std::get<0>(std::move(res));
+
+				// WARNING: this code is run at a high priority, so it needs to do as little work as possible
+				int bytes = getBytes(req);
+
+				// Drop requests if memory is under severe pressure
+				if (commitData->commitBatchesMemBytesCount + bytes > memBytesLimit) {
+					++commitData->stats.txnCommitErrors;
+					req.reply.sendError(commit_proxy_memory_limit_exceeded());
+					TraceEvent(SevWarnAlways, "ProxyCommitBatchMemoryThresholdExceeded")
+					    .suppressFor(60)
+					    .detail("MemBytesCount", commitData->commitBatchesMemBytesCount)
+					    .detail("MemLimit", memBytesLimit);
+					continue;
+				}
 
-					if (req.debugID.present()) {
-						g_traceBatch.addEvent("CommitDebug", req.debugID.get().first(), "CommitProxyServer.batcher");
-					}
+				if (bytes > FLOW_KNOBS->PACKET_WARNING) {
+					TraceEvent(SevWarn, "LargeTransaction")
+					    .suppressFor(1.0)
+					    .detail("Size", bytes)
+					    .detail("Client", req.reply.getEndpoint().getPrimaryAddress());
+				}
 
-					if (!batch.size())
-						timeout =
-						    makeBatchTimeoutFuture(std::max(commitData->commitBatchInterval - (now() - lastBatch),
-						                                    SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE));
-
-					if ((batchBytes + bytes > CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT || req.firstInBatch()) &&
-					    batch.size()) {
-						commitData->triggerCommit.set(false);
-						batchesInProgress++;
-						out.send({ std::move(batch), batchBytes });
-						lastBatch = now();
-						timeout = makeBatchTimeoutFuture(commitData->commitBatchInterval);
-						batch.clear();
-						batchBytes = 0;
-					}
+				++commitData->stats.txnCommitIn;
+				commitData->stats.uniqueClients.insert(req.reply.getEndpoint().getPrimaryAddress());
 
-					batch.push_back(req);
-					batchBytes += bytes;
-					commitData->commitBatchesMemBytesCount += bytes;
+				if (req.debugID.present()) {
+					g_traceBatch.addEvent("CommitDebug", req.debugID.get().first(), "CommitProxyServer.batcher");
 				}
-				when(wait(timeout)) {}
-				when(wait(commitData->triggerCommit.onChange())) {
-					ASSERT(commitData->triggerCommit.get());
-					double commitTime = lastBatch + SERVER_KNOBS->COMMIT_TRIGGER_DELAY;
-					if (now() > commitTime) {
-						break;
-					}
 
-					timeout = timeout || makeBatchTimeoutFuture(commitTime - now());
+				if (!batch.size())
+					timeout =
+					    makeBatchTimeoutFuture(std::max(commitData->commitBatchInterval - (now() - lastBatch),
+						                                SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE));
+
+				if ((batchBytes + bytes > CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT || req.firstInBatch()) && batch.size()) {
+					commitData->triggerCommit.set(false);
+					batchesInProgress++;
+					out.send({ std::move(batch), batchBytes });
+					lastBatch = now();
+					timeout = makeBatchTimeoutFuture(commitData->commitBatchInterval);
+					batch.clear();
+					batchBytes = 0;
 				}
-				when(waitNext(commitData->committedBatches.getFuture())) {
-					batchesInProgress--;
+
+				batch.push_back(req);
+				batchBytes += bytes;
+				commitData->commitBatchesMemBytesCount += bytes;
+			} else if (res.index() == 1) {
+			} else if (res.index() == 2) {
+				ASSERT(commitData->triggerCommit.get());
+				double commitTime = lastBatch + SERVER_KNOBS->COMMIT_TRIGGER_DELAY;
+				if (now() > commitTime) {
+					break;
 				}
+
+				timeout = timeout || makeBatchTimeoutFuture(commitTime - now());
+			} else if (res.index() == 3) {
+				ASSERT(batchesInProgress > 0);
+				--batchesInProgress;
+			} else {
+				UNREACHABLE();
 			}
 		}
 		batchesInProgress++;
@@ -403,33 +374,32 @@ bool isWhitelisted(const std::vector<Standalone<StringRef>>& binPathVec, StringR
 	return std::find(binPathVec.begin(), binPathVec.end(), binPath) != binPathVec.end();
 }
 
-ACTOR Future<Void> addBackupMutations(ProxyCommitData* self,
-                                      const std::map<Key, MutationListRef>* logRangeMutations,
-                                      LogPushData* toCommit,
-                                      Version commitVersion,
-                                      double* computeDuration,
-                                      double* computeStart) {
-	state std::map<Key, MutationListRef>::const_iterator logRangeMutation = logRangeMutations->cbegin();
-	state int32_t version = commitVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
-	state int yieldBytes = 0;
-	state BinaryWriter valueWriter(Unversioned());
+Future<Void> addBackupMutations(ProxyCommitData* self,
+                                const std::map<Key, MutationListRef>* logRangeMutations,
+                                LogPushData* toCommit,
+                                Version commitVersion,
+                                double* computeDuration,
+                                double* computeStart) {
+	const int32_t version = commitVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
+	int yieldBytes = 0;
 
 	toCommit->addTransactionInfo(SpanContext());
 
 	// Serialize the log range mutations within the map
-	for (; logRangeMutation != logRangeMutations->cend(); ++logRangeMutation) {
+	for (auto logRangeMutation = logRangeMutations->cbegin(); logRangeMutation != logRangeMutations->cend();
+	     ++logRangeMutation) {
 		// FIXME: this is re-implementing the serialize function of MutationListRef in order to have a yield
 		// this is 0x0FDB00A200090001
-		valueWriter = BinaryWriter(IncludeVersion(ProtocolVersion::withBackupMutations()));
+		BinaryWriter valueWriter(IncludeVersion(ProtocolVersion::withBackupMutations()));
 		valueWriter << logRangeMutation->second.totalSize(); // this is int32 by default
 
-		state MutationListRef::Blob* blobIter = logRangeMutation->second.blob_begin;
+		MutationListRef::Blob* blobIter = logRangeMutation->second.blob_begin;
 		while (blobIter) {
 			if (yieldBytes > SERVER_KNOBS->DESIRED_TOTAL_BYTES) {
 				yieldBytes = 0;
 				if (g_network->check_yield(TaskPriority::ProxyCommitYield1)) {
 					*computeDuration += g_network->timer_monotonic() - *computeStart;
-					wait(delay(0, TaskPriority::ProxyCommitYield1));
+					co_await delay(0, TaskPriority::ProxyCommitYield1);
 					*computeStart = g_network->timer_monotonic();
 				}
 			}
@@ -488,22 +458,34 @@ ACTOR Future<Void> addBackupMutations(ProxyCommitData* self,
 			//			}
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> releaseResolvingAfter(ProxyCommitData* self, Future<Void> releaseDelay, int64_t localBatchNumber) {
-	wait(releaseDelay);
+static Future<Void> releaseResolvingAfterImpl(ProxyCommitData* self,
+                                              Future<Void> releaseDelay,
+                                              int64_t localBatchNumber) {
+	co_await releaseDelay;
 	ASSERT(self->latestLocalCommitBatchResolving.get() == localBatchNumber - 1);
 	self->latestLocalCommitBatchResolving.set(localBatchNumber);
-	return Void();
+	co_return;
+}
+
+Future<Void> releaseResolvingAfter(ProxyCommitData* self, Future<Void> releaseDelay, int64_t localBatchNumber) {
+	if (releaseDelay.isReady()) {
+		releaseDelay.get();
+		ASSERT(self->latestLocalCommitBatchResolving.get() == localBatchNumber - 1);
+		self->latestLocalCommitBatchResolving.set(localBatchNumber);
+		return Void();
+	}
+
+	return releaseResolvingAfterImpl(self, releaseDelay, localBatchNumber);
 }
 
-ACTOR static Future<ResolveTransactionBatchReply> trackResolutionMetrics(Reference<Histogram> dist,
-                                                                         Future<ResolveTransactionBatchReply> in) {
-	state double startTime = g_network->timer_monotonic();
-	ResolveTransactionBatchReply reply = wait(in);
+static Future<ResolveTransactionBatchReply> trackResolutionMetrics(Reference<Histogram> dist,
+                                                                   Future<ResolveTransactionBatchReply> in) {
+	double startTime = g_network->timer_monotonic();
+	ResolveTransactionBatchReply reply = co_await in;
 	dist->sampleSeconds(g_network->timer_monotonic() - startTime);
-	return reply;
+	co_return reply;
 }
 
 namespace CommitBatch {
@@ -817,15 +799,15 @@ double computeReleaseDelay(CommitBatchContext* self, double latencyBucket) {
 	                self->batchOperations * self->pProxyCommitData->commitComputePerOperation[latencyBucket]);
 }
 
-ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
+Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 
-	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
-	state std::vector<CommitTransactionRequest>& trs = self->trs;
-	state const int64_t localBatchNumber = self->localBatchNumber;
-	state const int latencyBucket = self->latencyBucket;
-	state const Optional<UID>& debugID = self->debugID;
-	state Span span("MP:preresolutionProcessing"_loc, self->span.context);
-	state double startTime = g_network->timer_monotonic();
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+	std::vector<CommitTransactionRequest>& trs = self->trs;
+	const int64_t localBatchNumber = self->localBatchNumber;
+	const int latencyBucket = self->latencyBucket;
+	const Optional<UID>& debugID = self->debugID;
+	Span span("MP:preresolutionProcessing"_loc, self->span.context);
+	double startTime = g_network->timer_monotonic();
 
 	if (self->localBatchNumber - self->pProxyCommitData->latestLocalCommitBatchResolving.get() >
 	        SERVER_KNOBS->RESET_MASTER_BATCHES &&
@@ -839,7 +821,7 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 
 	// Pre-resolution the commits
 	CODE_PROBE(pProxyCommitData->latestLocalCommitBatchResolving.get() < localBatchNumber - 1, "Wait for local batch");
-	wait(pProxyCommitData->latestLocalCommitBatchResolving.whenAtLeast(localBatchNumber - 1));
+	co_await pProxyCommitData->latestLocalCommitBatchResolving.whenAtLeast(localBatchNumber - 1);
 	double queuingDelay = g_network->timer_monotonic() - startTime;
 	pProxyCommitData->stats.computeLatency.addMeasurement(queuingDelay);
 	pProxyCommitData->stats.commitBatchQueuingDist->sampleSeconds(queuingDelay);
@@ -856,7 +838,7 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 		ASSERT(pProxyCommitData->latestLocalCommitBatchResolving.get() == localBatchNumber - 1);
 		pProxyCommitData->latestLocalCommitBatchResolving.set(localBatchNumber);
 
-		wait(pProxyCommitData->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber - 1));
+		co_await pProxyCommitData->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber - 1);
 		ASSERT(pProxyCommitData->latestLocalCommitBatchLogging.get() == localBatchNumber - 1);
 		pProxyCommitData->latestLocalCommitBatchLogging.set(localBatchNumber);
 		for (const auto& tr : trs) {
@@ -866,7 +848,7 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 		pProxyCommitData->stats.txnCommitOut += trs.size();
 		pProxyCommitData->stats.txnRejectedForQueuedTooLong += trs.size();
 		self->rejected = true;
-		return Void();
+		co_return;
 	}
 
 	self->releaseDelay = delay(computeReleaseDelay(self, latencyBucket), TaskPriority::ProxyMasterVersionReply);
@@ -888,9 +870,9 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 	                            pProxyCommitData->commitVersionRequestNumber++,
 	                            pProxyCommitData->mostRecentProcessedRequestNumber,
 	                            pProxyCommitData->dbgid);
-	state double beforeGettingCommitVersion = g_network->timer_monotonic();
-	GetCommitVersionReply versionReply = wait(brokenPromiseToNever(
-	    pProxyCommitData->master.getCommitVersion.getReply(req, TaskPriority::ProxyMasterVersionReply)));
+	double beforeGettingCommitVersion = g_network->timer_monotonic();
+	GetCommitVersionReply versionReply = co_await brokenPromiseToNever(
+	    pProxyCommitData->master.getCommitVersion.getReply(req, TaskPriority::ProxyMasterVersionReply));
 
 	pProxyCommitData->mostRecentProcessedRequestNumber = versionReply.requestNum;
 
@@ -909,24 +891,22 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 		for (auto r = rs.begin(); r != rs.end(); ++r)
 			r->value().emplace_back(versionReply.resolverChangesVersion, it.dest);
 	}
-
+	
 	pProxyCommitData->stats.commitPreresolutionLatency.addMeasurement(now() - startTime);
 	//TraceEvent("ProxyGotVer", pProxyContext->dbgid).detail("Commit", commitVersion).detail("Prev", prevVersion);
 
 	if (debugID.present()) {
 		g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.GotCommitVersion");
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> getResolution(CommitBatchContext* self) {
-	state double resolutionStart = g_network->timer_monotonic();
+Future<Void> getResolution(CommitBatchContext* self) {
+	double resolutionStart = g_network->timer_monotonic();
 	// Sending these requests is the fuzzy border between phase 1 and phase 2; it could conceivably overlap with
 	// resolution processing but is still using CPU
-	state ProxyCommitData* pProxyCommitData = self->pProxyCommitData;
+	ProxyCommitData* pProxyCommitData = self->pProxyCommitData;
 	std::vector<CommitTransactionRequest>& trs = self->trs;
-	state Span span("MP:getResolution"_loc, self->span.context);
+	Span span("MP:getResolution"_loc, self->span.context);
 
 	ResolutionRequestBuilder requests(pProxyCommitData,
 	                                  self->commitVersion,
@@ -952,12 +932,23 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 
 	pProxyCommitData->stats.txnCommitResolving += trs.size();
 	std::vector<Future<ResolveTransactionBatchReply>> replies;
-	for (int r = 0; r < pProxyCommitData->resolvers.size(); r++) {
-		requests.requests[r].debugID = self->debugID;
-		requests.requests[r].writtenTags = self->writtenTagsPreResolution;
-		replies.push_back(trackResolutionMetrics(pProxyCommitData->stats.resolverDist[r],
-		                                         brokenPromiseToNever(pProxyCommitData->resolvers[r].resolve.getReply(
-		                                             requests.requests[r], TaskPriority::ProxyResolverReply))));
+	Future<ResolveTransactionBatchReply> singleResolverReply;
+	double singleResolverStart = 0;
+	if (pProxyCommitData->resolvers.size() == 1) {
+		requests.requests[0].debugID = self->debugID;
+		requests.requests[0].writtenTags = self->writtenTagsPreResolution;
+		singleResolverStart = g_network->timer_monotonic();
+		singleResolverReply = brokenPromiseToNever(
+		    pProxyCommitData->resolvers[0].resolve.getReply(requests.requests[0], TaskPriority::ProxyResolverReply));
+	} else {
+		for (int r = 0; r < pProxyCommitData->resolvers.size(); r++) {
+			requests.requests[r].debugID = self->debugID;
+			requests.requests[r].writtenTags = self->writtenTagsPreResolution;
+			replies.push_back(
+			    trackResolutionMetrics(pProxyCommitData->stats.resolverDist[r],
+			                           brokenPromiseToNever(pProxyCommitData->resolvers[r].resolve.getReply(
+			                               requests.requests[r], TaskPriority::ProxyResolverReply))));
+		}
 	}
 
 	self->transactionResolverMap.swap(requests.transactionResolverMap);
@@ -981,16 +972,24 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 	}
 
 	// Wait for the final resolution
-	std::vector<ResolveTransactionBatchReply> resolutionResp = wait(getAll(replies));
-	self->resolution.swap(*const_cast<std::vector<ResolveTransactionBatchReply>*>(&resolutionResp));
+	if (pProxyCommitData->resolvers.size() == 1) {
+		ResolveTransactionBatchReply resolutionResp = co_await singleResolverReply;
+		pProxyCommitData->stats.resolverDist[0]->sampleSeconds(g_network->timer_monotonic() - singleResolverStart);
+		self->resolution.clear();
+		self->resolution.push_back(std::move(resolutionResp));
+	} else {
+		std::vector<ResolveTransactionBatchReply> resolutionResp = co_await getAll(replies);
+		self->resolution = std::move(resolutionResp);
+	}
+
+	double resolutionDuration = g_network->timer_monotonic() - resolutionStart;
 
-	self->pProxyCommitData->stats.resolutionDist->sampleSeconds(g_network->timer_monotonic() - resolutionStart);
+	self->pProxyCommitData->stats.commitResolutionLatency.addMeasurement(resolutionDuration);
+	self->pProxyCommitData->stats.resolutionDist->sampleSeconds(resolutionDuration);
 	if (self->debugID.present()) {
 		g_traceBatch.addEvent(
 		    "CommitDebug", self->debugID.get().first(), "CommitProxyServer.commitBatch.AfterResolution");
 	}
-
-	return Void();
 }
 
 void assertResolutionStateMutationsSizeConsistent(const std::vector<ResolveTransactionBatchReply>& resolution) {
@@ -1075,7 +1074,7 @@ void applyMetadataEffect(CommitBatchContext* self) {
 
 			if (committed) {
 				applyMetadataMutations(SpanContext(),
-				                       *self->pProxyCommitData,
+				                       self->pProxyCommitData->getApplyMetadataProxyContext(),
 				                       self->arena,
 				                       self->pProxyCommitData->logSystem,
 				                       self->resolution[0].stateMutations[versionIndex][transactionIndex].mutations,
@@ -1119,7 +1118,7 @@ void determineCommittedTransactions(CommitBatchContext* self) {
 	// Thus, we use this nextTr to track the correct transaction index on each resolver.
 	self->nextTr.resize(self->resolution.size());
 	for (int t = 0; t < trs.size(); t++) {
-		uint8_t commit = ConflictBatch::TransactionCommitted;
+		uint8_t commit = ConflictBatchStatus::TransactionCommitted;
 		for (int r : self->transactionResolverMap[t]) {
 			commit = std::min(self->resolution[r].committed[self->nextTr[r]++], commit);
 		}
@@ -1137,7 +1136,7 @@ void determineCommittedTransactions(CommitBatchContext* self) {
 	    pProxyCommitData->txnStateStore->readValue(mustContainSystemMutationsKey).get();
 	if (mustContainSystemKey.present() && mustContainSystemKey.get().size()) {
 		for (int t = 0; t < trs.size(); t++) {
-			if (self->committed[t] == ConflictBatch::TransactionCommitted) {
+			if (self->committed[t] == ConflictBatchStatus::TransactionCommitted) {
 				bool foundSystem = false;
 				for (auto& m : trs[t].transaction.mutations) {
 					if ((m.type == MutationRef::ClearRange ? m.param2 : m.param1) >= nonMetadataSystemKeys.end) {
@@ -1146,7 +1145,7 @@ void determineCommittedTransactions(CommitBatchContext* self) {
 					}
 				}
 				if (!foundSystem) {
-					self->committed[t] = ConflictBatch::TransactionConflict;
+					self->committed[t] = ConflictBatchStatus::TransactionConflict;
 				}
 			}
 		}
@@ -1155,16 +1154,25 @@ void determineCommittedTransactions(CommitBatchContext* self) {
 
 // This first pass through committed transactions deals with "metadata" effects (modifications of txnStateStore, changes
 // to storage servers' responsibilities)
-ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self) {
-	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+Future<Void> changeCoordinatorsForMetadata(CommitBatchContext* self) {
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+	co_await brokenPromiseToNever(pProxyCommitData->db->get().clusterInterface.changeCoordinators.getReply(
+	    ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get(),
+	                              self->pProxyCommitData->master.id())));
+	ASSERT(false); // ChangeCoordinatorsRequest should always throw
+}
+
+Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self) {
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
 	auto& trs = self->trs;
 
 	int t;
 	for (t = 0; t < trs.size() && !self->forceRecovery; t++) {
-		if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || trs[t].isLockAware())) {
+		if (self->committed[t] == ConflictBatchStatus::TransactionCommitted &&
+		    (!self->locked || trs[t].isLockAware())) {
 			self->commitCount++;
 			applyMetadataMutations(trs[t].spanContext,
-			                       *pProxyCommitData,
+			                       pProxyCommitData->getApplyMetadataProxyContext(),
 			                       self->arena,
 			                       pProxyCommitData->logSystem,
 			                       trs[t].transaction.mutations,
@@ -1177,7 +1185,7 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 		}
 
 		if (self->firstStateMutations) {
-			ASSERT(self->committed[t] == ConflictBatch::TransactionCommitted);
+			ASSERT(self->committed[t] == ConflictBatchStatus::TransactionCommitted);
 			self->firstStateMutations = false;
 			self->forceRecovery = false;
 		}
@@ -1185,7 +1193,7 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 
 	if (self->forceRecovery) {
 		for (; t < trs.size(); t++)
-			self->committed[t] = ConflictBatch::TransactionConflict;
+			self->committed[t] = ConflictBatchStatus::TransactionConflict;
 		TraceEvent(SevWarn, "RestartingTxnSubsystem", pProxyCommitData->dbgid).detail("Stage", "AwaitCommit");
 	}
 	if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS) {
@@ -1224,10 +1232,7 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 
 	if (!self->isMyFirstBatch &&
 	    pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get() != self->previousCoordinators.get()) {
-		wait(brokenPromiseToNever(pProxyCommitData->db->get().clusterInterface.changeCoordinators.getReply(
-		    ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get(),
-		                              self->pProxyCommitData->master.id()))));
-		ASSERT(false); // ChangeCoordinatorsRequest should always throw
+		return changeCoordinatorsForMetadata(self);
 	}
 
 	return Void();
@@ -1311,7 +1316,7 @@ void rejectMutationsForReadLockOnRange(CommitBatchContext* self) {
 	ASSERT(pProxyCommitData->rangeLock != nullptr);
 	std::vector<CommitTransactionRequest>& trs = self->trs;
 	for (int i = self->transactionNum; i < trs.size(); i++) {
-		if (self->committed[i] != ConflictBatch::TransactionCommitted) {
+		if (self->committed[i] != ConflictBatchStatus::TransactionCommitted) {
 			continue;
 		} else if (trs[i].isLockAware()) {
 			continue; // rangeLock is transparent to lock-aware transactions
@@ -1327,7 +1332,7 @@ void rejectMutationsForReadLockOnRange(CommitBatchContext* self) {
 			}
 			bool shouldReject = pProxyCommitData->rangeLock->isLocked(rangeToCheck);
 			if (shouldReject) {
-				self->committed[i] = ConflictBatch::TransactionLockReject;
+				self->committed[i] = ConflictBatchStatus::TransactionLockReject;
 				trs[i].reply.sendError(transaction_rejected_range_locked());
 				break;
 			}
@@ -1337,20 +1342,20 @@ void rejectMutationsForReadLockOnRange(CommitBatchContext* self) {
 
 /// This second pass through committed transactions assigns the actual mutations to the appropriate storage servers'
 /// tags
-ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
-	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
-	state std::vector<CommitTransactionRequest>& trs = self->trs;
+Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+	std::vector<CommitTransactionRequest>& trs = self->trs;
 
 	for (; self->transactionNum < trs.size(); self->transactionNum++) {
-		if (!(self->committed[self->transactionNum] == ConflictBatch::TransactionCommitted &&
+		if (!(self->committed[self->transactionNum] == ConflictBatchStatus::TransactionCommitted &&
 		      (!self->locked || trs[self->transactionNum].isLockAware()))) {
 			continue;
 		}
 
-		state bool checkSample = trs[self->transactionNum].commitCostEstimation.present();
-		state Optional<ClientTrCommitCostEstimation>* trCost = &trs[self->transactionNum].commitCostEstimation;
-		state int mutationNum = 0;
-		state VectorRef<MutationRef>* pMutations = &trs[self->transactionNum].transaction.mutations;
+		bool checkSample = trs[self->transactionNum].commitCostEstimation.present();
+		Optional<ClientTrCommitCostEstimation>* trCost = &trs[self->transactionNum].commitCostEstimation;
+		int mutationNum = 0;
+		VectorRef<MutationRef>* pMutations = &trs[self->transactionNum].transaction.mutations;
 
 		self->toCommit.addTransactionInfo(trs[self->transactionNum].spanContext);
 
@@ -1359,14 +1364,14 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 				self->yieldBytes = 0;
 				if (g_network->check_yield(TaskPriority::ProxyCommitYield1)) {
 					self->computeDuration += g_network->timer_monotonic() - self->computeStart;
-					wait(delay(0, TaskPriority::ProxyCommitYield1));
+					co_await delay(0, TaskPriority::ProxyCommitYield1);
 					self->computeStart = g_network->timer_monotonic();
 				}
 			}
 
-			state MutationRef m = (*pMutations)[mutationNum];
-			state Arena arena;
-			state MutationRef writtenMutation;
+			MutationRef m = (*pMutations)[mutationNum];
+			Arena arena;
+			MutationRef writtenMutation;
 			self->mutationCount++;
 			self->mutationBytes += m.expectedSize();
 			self->yieldBytes += m.expectedSize();
@@ -1523,24 +1528,22 @@ ACTOR Future<Void> assignMutationsToStorageServers(CommitBatchContext* self) {
 			    trs[self->transactionNum].commitCostEstimation.get().expensiveCostEstCount;
 		}
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> postResolution(CommitBatchContext* self) {
-	state double postResolutionStart = g_network->timer_monotonic();
-	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
-	state std::vector<CommitTransactionRequest>& trs = self->trs;
-	state const int64_t localBatchNumber = self->localBatchNumber;
-	state const Optional<UID>& debugID = self->debugID;
-	state Span span("MP:postResolution"_loc, self->span.context);
+Future<Void> postResolution(CommitBatchContext* self) {
+	double postResolutionStart = g_network->timer_monotonic();
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+	std::vector<CommitTransactionRequest>& trs = self->trs;
+	const int64_t localBatchNumber = self->localBatchNumber;
+	const Optional<UID>& debugID = self->debugID;
+	Span span("MP:postResolution"_loc, self->span.context);
 
 	bool queuedCommits = pProxyCommitData->latestLocalCommitBatchLogging.get() < localBatchNumber - 1;
 	CODE_PROBE(queuedCommits, "Queuing post-resolution commit processing");
-	wait(pProxyCommitData->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber - 1));
-	state double postResolutionQueuing = g_network->timer_monotonic();
+	co_await pProxyCommitData->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber - 1);
+	double postResolutionQueuing = g_network->timer_monotonic();
 	pProxyCommitData->stats.postResolutionDist->sampleSeconds(postResolutionQueuing - postResolutionStart);
-	wait(yield(TaskPriority::ProxyCommitYield1));
+	co_await yield(TaskPriority::ProxyCommitYield1);
 
 	self->computeStart = g_network->timer_monotonic();
 
@@ -1571,11 +1574,11 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	}
 
 	if (self->forceRecovery) {
-		wait(Future<Void>(Never()));
+		co_await Future<Void>(Never());
 	}
 
 	// First pass
-	wait(applyMetadataToCommittedTransactions(self));
+	co_await applyMetadataToCommittedTransactions(self);
 
 	if (debugID.present()) {
 		g_traceBatch.addEvent(
@@ -1590,7 +1593,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	}
 
 	// Second pass
-	wait(assignMutationsToStorageServers(self));
+	co_await assignMutationsToStorageServers(self);
 
 	if (debugID.present()) {
 		g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "CommitProxyServer.commitBatch.AssignMutationToSS");
@@ -1598,12 +1601,12 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 
 	// Serialize and backup the mutations as a single mutation
 	if ((pProxyCommitData->vecBackupKeys.size() > 1) && self->logRangeMutations.size()) {
-		wait(addBackupMutations(pProxyCommitData,
-		                        &self->logRangeMutations,
-		                        &self->toCommit,
-		                        self->commitVersion,
-		                        &self->computeDuration,
-		                        &self->computeStart));
+		co_await addBackupMutations(pProxyCommitData,
+		                            &self->logRangeMutations,
+		                            &self->toCommit,
+		                            self->commitVersion,
+		                            &self->computeDuration,
+		                            &self->computeStart);
 	}
 
 	// When version vector is enabled, idempotency entries should only be created or cleared
@@ -1614,7 +1617,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	                            self->idempotencyKVBuilder,
 	                            self->commitVersion,
 	                            self->committed,
-	                            ConflictBatch::TransactionCommitted,
+	                            ConflictBatchStatus::TransactionCommitted,
 	                            self->locked,
 	                            [&](const KeyValue& kv) {
 		                            MutationRef idempotencyIdSet;
@@ -1641,8 +1644,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST ||
 	    pProxyCommitData->db->get().logSystemConfig.numLogs() == self->tpcvMap.size()) {
-		state int i = 0;
-		for (i = 0; i < pProxyCommitData->idempotencyClears.size(); i++) {
+		for (int i = 0; i < pProxyCommitData->idempotencyClears.size(); i++) {
 			auto& tags = pProxyCommitData->tagsForKey(pProxyCommitData->idempotencyClears[i].param1);
 			self->toCommit.addTags(tags);
 			if (pProxyCommitData->acsBuilder != nullptr) {
@@ -1672,7 +1674,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 	if (pProxyCommitData->committedVersion.get() <
 	    self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) {
 		self->computeDuration += g_network->timer_monotonic() - self->computeStart;
-		state Span waitVersionSpan;
+		Span waitVersionSpan;
 		while (pProxyCommitData->committedVersion.get() <
 		       self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) {
 			// This should be *extremely* rare in the real world, but knob buggification should make it happen in
@@ -1680,29 +1682,35 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 			CODE_PROBE(true, "Semi-committed pipeline limited by MVCC window");
 			//TraceEvent("ProxyWaitingForCommitted", pProxyCommitData->dbgid).detail("CommittedVersion", pProxyCommitData->committedVersion.get()).detail("NeedToCommit", commitVersion);
 			waitVersionSpan = Span("MP:overMaxReadTransactionLifeVersions"_loc, span.context);
-			choose {
-				when(wait(pProxyCommitData->committedVersion.whenAtLeast(
-				    self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS))) {
-					wait(yield());
-					break;
+			// @todo probably there is no need to get the (entire) version vector from the sequencer
+			// in this case, and if so, consider adding a flag to the request to tell the sequencer
+			// to not send the version vector information.
+			auto res =
+			    co_await race(pProxyCommitData->committedVersion.whenAtLeast(
+			                      self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS),
+			                  pProxyCommitData->cx->onProxiesChanged(),
+			                  pProxyCommitData->master.getLiveCommittedVersion.getReply(
+			                      GetRawCommittedVersionRequest(waitVersionSpan.context, debugID, invalidVersion),
+			                      TaskPriority::GetLiveCommittedVersionReply));
+			if (res.index() == 0) {
+				co_await yield();
+				break;
+			} else if (res.index() == 1) {
+			} else if (res.index() == 2) {
+				GetRawCommittedVersionReply v = std::get<2>(std::move(res));
+
+				if (v.version > pProxyCommitData->committedVersion.get()) {
+					pProxyCommitData->locked = v.locked;
+					pProxyCommitData->metadataVersion = v.metadataVersion;
+					pProxyCommitData->committedVersion.set(v.version);
 				}
-				when(wait(pProxyCommitData->cx->onProxiesChanged())) {}
-				// @todo probably there is no need to get the (entire) version vector from the sequencer
-				// in this case, and if so, consider adding a flag to the request to tell the sequencer
-				// to not send the version vector information.
-				when(GetRawCommittedVersionReply v = wait(pProxyCommitData->master.getLiveCommittedVersion.getReply(
-				         GetRawCommittedVersionRequest(waitVersionSpan.context, debugID, invalidVersion),
-				         TaskPriority::GetLiveCommittedVersionReply))) {
-					if (v.version > pProxyCommitData->committedVersion.get()) {
-						pProxyCommitData->locked = v.locked;
-						pProxyCommitData->metadataVersion = v.metadataVersion;
-						pProxyCommitData->committedVersion.set(v.version);
-					}
 
-					if (pProxyCommitData->committedVersion.get() <
-					    self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)
-						wait(delay(SERVER_KNOBS->PROXY_SPIN_DELAY));
+				if (pProxyCommitData->committedVersion.get() <
+				    self->commitVersion - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) {
+					co_await delay(SERVER_KNOBS->PROXY_SPIN_DELAY);
 				}
+			} else {
+				UNREACHABLE();
 			}
 		}
 		waitVersionSpan = Span{};
@@ -1750,10 +1758,10 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 		// Issue acs mutation at the end of this commit batch
 		addAccumulativeChecksumMutations(self);
 	}
-	const auto versionSet = ILogSystem::PushVersionSet{ self->prevVersion,
-		                                                self->commitVersion,
-		                                                pProxyCommitData->committedVersion.get(),
-		                                                pProxyCommitData->minKnownCommittedVersion };
+	const auto versionSet = LogPushVersionSet{ self->prevVersion,
+		                                       self->commitVersion,
+		                                       pProxyCommitData->committedVersion.get(),
+		                                       pProxyCommitData->minKnownCommittedVersion };
 	self->loggingComplete =
 	    pProxyCommitData->logSystem->push(versionSet, self->toCommit, span.context, self->debugID, tpcvMap);
 
@@ -1801,24 +1809,25 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 		}
 	}
 
-	pProxyCommitData->stats.processingMutationDist->sampleSeconds(g_network->timer_monotonic() - postResolutionQueuing);
-	return Void();
+	double postResolutionEnd = g_network->timer_monotonic();
+
+	pProxyCommitData->stats.commitPostresolutionLatency.addMeasurement(postResolutionEnd - postResolutionStart);
+	pProxyCommitData->stats.processingMutationDist->sampleSeconds(postResolutionEnd - postResolutionQueuing);
 }
 
-ACTOR Future<Void> transactionLogging(CommitBatchContext* self) {
-	state double tLoggingStart = g_network->timer_monotonic();
-	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
-	state Span span("MP:transactionLogging"_loc, self->span.context);
+Future<Void> transactionLogging(CommitBatchContext* self) {
+	double tLoggingStart = g_network->timer_monotonic();
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+	Span span("MP:transactionLogging"_loc, self->span.context);
 
 	try {
-		choose {
-			when(Version ver = wait(self->loggingComplete)) {
-				if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
-					pProxyCommitData->minKnownCommittedVersion =
-					    std::max(pProxyCommitData->minKnownCommittedVersion, ver);
-				}
+		auto res = co_await race(self->loggingComplete,
+		                         pProxyCommitData->committedVersion.whenAtLeast(self->commitVersion + 1));
+		if (res.index() == 0) {
+			Version ver = std::get<0>(std::move(res));
+			if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
+				pProxyCommitData->minKnownCommittedVersion = std::max(pProxyCommitData->minKnownCommittedVersion, ver);
 			}
-			when(wait(pProxyCommitData->committedVersion.whenAtLeast(self->commitVersion + 1))) {}
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_broken_promise) {
@@ -1830,7 +1839,7 @@ ACTOR Future<Void> transactionLogging(CommitBatchContext* self) {
 	pProxyCommitData->lastCommitLatency = now() - self->commitStartTime;
 	pProxyCommitData->lastCommitTime = std::max(pProxyCommitData->lastCommitTime.get(), self->commitStartTime);
 
-	wait(yield(TaskPriority::ProxyCommitYield2));
+	co_await yield(TaskPriority::ProxyCommitYield2);
 
 	if (pProxyCommitData->popRemoteTxs &&
 	    self->msg.popTo > (pProxyCommitData->txsPopVersions.size() ? pProxyCommitData->txsPopVersions.back().second
@@ -1843,16 +1852,18 @@ ACTOR Future<Void> transactionLogging(CommitBatchContext* self) {
 		pProxyCommitData->txsPopVersions.emplace_back(self->commitVersion, self->msg.popTo);
 	}
 	pProxyCommitData->logSystem->popTxs(self->msg.popTo);
-	pProxyCommitData->stats.tlogLoggingDist->sampleSeconds(g_network->timer_monotonic() - tLoggingStart);
-	return Void();
+	double tLoggingDuration = g_network->timer_monotonic() - tLoggingStart;
+
+	pProxyCommitData->stats.commitTLogLoggingLatency.addMeasurement(tLoggingDuration);
+	pProxyCommitData->stats.tlogLoggingDist->sampleSeconds(tLoggingDuration);
 }
 
-ACTOR Future<Void> reply(CommitBatchContext* self) {
-	state double replyStart = g_network->timer_monotonic();
-	state ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
-	state Span span("MP:reply"_loc, self->span.context);
+Future<Void> reply(CommitBatchContext* self) {
+	double replyStart = g_network->timer_monotonic();
+	ProxyCommitData* const pProxyCommitData = self->pProxyCommitData;
+	Span span("MP:reply"_loc, self->span.context);
 
-	state const Optional<UID>& debugID = self->debugID;
+	const Optional<UID>& debugID = self->debugID;
 
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		// Version vector/unicast is disabled: Logging completed, so the current version (and all versions prior to
@@ -1884,18 +1895,18 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 	           "later version was reported committed first");
 
 	if (self->commitVersion >= pProxyCommitData->committedVersion.get()) {
-		state Optional<std::set<Tag>> writtenTags;
+		Optional<std::set<Tag>> writtenTags;
 		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
 			writtenTags = self->writtenTags;
 		}
-		wait(pProxyCommitData->master.reportLiveCommittedVersion.getReply(
+		co_await pProxyCommitData->master.reportLiveCommittedVersion.getReply(
 		    ReportRawCommittedVersionRequest(self->commitVersion,
 		                                     self->lockedAfter,
 		                                     self->metadataVersionAfter,
 		                                     pProxyCommitData->minKnownCommittedVersion,
 		                                     self->prevVersion,
 		                                     writtenTags),
-		    TaskPriority::ProxyMasterVersionReply));
+		    TaskPriority::ProxyMasterVersionReply);
 	}
 
 	if (debugID.present()) {
@@ -1942,15 +1953,15 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 	std::unordered_map<uint8_t, int16_t> idCountsForKey;
 	for (int t = 0; t < self->trs.size(); t++) {
 		auto& tr = self->trs[t];
-		if (self->committed[t] == ConflictBatch::TransactionCommitted && (!self->locked || tr.isLockAware())) {
+		if (self->committed[t] == ConflictBatchStatus::TransactionCommitted && (!self->locked || tr.isLockAware())) {
 			ASSERT_WE_THINK(self->commitVersion != invalidVersion);
 			if (self->trs[t].idempotencyId.valid()) {
 				idCountsForKey[uint8_t(t >> 8)] += 1;
 			}
 			tr.reply.send(CommitID(self->commitVersion, t, self->metadataVersionAfter));
-		} else if (self->committed[t] == ConflictBatch::TransactionTooOld) {
+		} else if (self->committed[t] == ConflictBatchStatus::TransactionTooOld) {
 			tr.reply.sendError(transaction_too_old());
-		} else if (self->committed[t] == ConflictBatch::TransactionLockReject) {
+		} else if (self->committed[t] == ConflictBatchStatus::TransactionLockReject) {
 			// We already sent the error
 			ASSERT(tr.reply.isSet());
 		} else {
@@ -2045,13 +2056,15 @@ ACTOR Future<Void> reply(CommitBatchContext* self) {
 	pProxyCommitData->stats.commitBatchingWindowSize.addMeasurement(pProxyCommitData->commitBatchInterval);
 	pProxyCommitData->commitBatchesMemBytesCount -= self->currentBatchMemBytesCount;
 	ASSERT_ABORT(pProxyCommitData->commitBatchesMemBytesCount >= 0);
-	wait(self->releaseFuture);
-	pProxyCommitData->stats.replyCommitDist->sampleSeconds(g_network->timer_monotonic() - replyStart);
-	return Void();
+	co_await self->releaseFuture;
+	double replyDuration = g_network->timer_monotonic() - replyStart;
+
+	pProxyCommitData->stats.commitReplyLatency.addMeasurement(replyDuration);
+	pProxyCommitData->stats.replyCommitDist->sampleSeconds(replyDuration);
 }
 
 // Commit one batch of transactions trs
-ACTOR Future<Void> commitBatchImpl(CommitBatchContext* pContext) {
+Future<Void> commitBatchImpl(CommitBatchContext* pContext) {
 	// WARNING: this code is run at a high priority (until the first delay(0)), so it needs to do as little work as
 	// possible
 
@@ -2060,62 +2073,61 @@ ACTOR Future<Void> commitBatchImpl(CommitBatchContext* pContext) {
 
 	// Active load balancing runs at a very high priority (to obtain accurate estimate of memory used by commit batches)
 	// so we need to downgrade here
-	wait(delay(0, TaskPriority::ProxyCommit));
+	co_await delay(0, TaskPriority::ProxyCommit);
 
 	pContext->pProxyCommitData->lastVersionTime = pContext->startTime;
 	++pContext->pProxyCommitData->stats.commitBatchIn;
 	pContext->setupTraceBatch();
+	pContext->pProxyCommitData->stats.commitBatchBytes.addMeasurement(pContext->currentBatchMemBytesCount);
+	pContext->pProxyCommitData->stats.commitBatchTransactions.addMeasurement(pContext->trs.size());
 
 	/////// Phase 1: Pre-resolution processing (CPU bound except waiting for a version # which is separately pipelined
 	/// and *should* be available by now (unless empty commit); ordered; currently atomic but could yield)
 	pContext->stage = PRE_RESOLUTION;
-	wait(CommitBatch::preresolutionProcessing(pContext));
+	co_await CommitBatch::preresolutionProcessing(pContext);
 	if (pContext->rejected) {
 		pContext->pProxyCommitData->commitBatchesMemBytesCount -= pContext->currentBatchMemBytesCount;
-		return Void();
+		co_return;
 	}
 
 	/////// Phase 2: Resolution (waiting on the network; pipelined)
 	pContext->stage = RESOLUTION;
-	wait(CommitBatch::getResolution(pContext));
+	co_await CommitBatch::getResolution(pContext);
 
 	////// Phase 3: Post-resolution processing (CPU bound except for very rare situations; ordered; currently atomic but
 	/// doesn't need to be)
 	pContext->stage = POST_RESOLUTION;
-	wait(CommitBatch::postResolution(pContext));
+	co_await CommitBatch::postResolution(pContext);
 
 	/////// Phase 4: Logging (network bound; pipelined up to MAX_READ_TRANSACTION_LIFE_VERSIONS (limited by loop above))
 	pContext->stage = TRANSACTION_LOGGING;
-	wait(CommitBatch::transactionLogging(pContext));
+	co_await CommitBatch::transactionLogging(pContext);
 
 	/////// Phase 5: Replies (CPU bound; no particular order required, though ordered execution would be best for
 	/// latency)
 	pContext->stage = REPLY;
-	wait(CommitBatch::reply(pContext));
+	co_await CommitBatch::reply(pContext);
 
 	pContext->stage = COMPLETE;
-	return Void();
 }
 
 } // namespace CommitBatch
 
-ACTOR Future<Void> commitBatch(ProxyCommitData* pCommitData,
-                               std::vector<CommitTransactionRequest>* trs,
-                               int currentBatchMemBytesCount) {
+Future<Void> commitBatch(ProxyCommitData* pCommitData,
+                         std::vector<CommitTransactionRequest>* trs,
+                         int currentBatchMemBytesCount) {
 
-	state CommitBatch::CommitBatchContext context(pCommitData, trs, currentBatchMemBytesCount);
-	state Future<Void> commit = CommitBatch::commitBatchImpl(&context);
+	CommitBatch::CommitBatchContext context(pCommitData, trs, currentBatchMemBytesCount);
+	Future<Void> commit = CommitBatch::commitBatchImpl(&context);
 
 	try {
-		wait(timeoutError(commit, SERVER_KNOBS->COMMIT_PROXY_LIVENESS_TIMEOUT));
+		co_await timeoutError(commit, SERVER_KNOBS->COMMIT_PROXY_LIVENESS_TIMEOUT);
 	} catch (Error& err) {
 		TraceEvent(SevInfo, "CommitBatchFailed", pCommitData->dbgid)
 		    .detail("Stage", context.stage)
 		    .detail("ErrorCode", err.code());
 		throw failed_to_progress();
 	}
-
-	return Void();
 }
 
 // Add tss mapping data to the reply, if any of the included storage servers have a TSS pair
@@ -2142,14 +2154,14 @@ void addTagMapping(GetKeyServerLocationsReply& reply, ProxyCommitData* commitDat
 	}
 }
 
-ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsRequest req, ProxyCommitData* commitData) {
+static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsRequest req, ProxyCommitData* commitData) {
 	// We can't respond to these requests until we have valid txnStateStore
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKeyServersLocations;
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 
-	wait(commitData->validState.getFuture());
+	co_await commitData->validState.getFuture();
 
-	wait(delay(0, TaskPriority::DefaultEndpoint));
+	co_await delay(0, TaskPriority::DefaultEndpoint);
 
 	std::unordered_set<UID> tssMappingsIncluded;
 	GetKeyServerLocationsReply rep;
@@ -2199,14 +2211,13 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
 	addTagMapping(rep, commitData);
 	req.reply.send(rep);
 	++commitData->stats.keyServerLocationOut;
-	return Void();
 }
 
-ACTOR static Future<Void> readRequestServer(CommitProxyInterface proxy,
-                                            PromiseStream<Future<Void>> addActor,
-                                            ProxyCommitData* commitData) {
-	loop {
-		GetKeyServerLocationsRequest req = waitNext(proxy.getKeyServersLocations.getFuture());
+static Future<Void> readRequestServer(CommitProxyInterface proxy,
+                                      PromiseStream<Future<Void>> addActor,
+                                      ProxyCommitData* commitData) {
+	while (true) {
+		GetKeyServerLocationsRequest req = co_await proxy.getKeyServersLocations.getFuture();
 		// WARNING: this code is run at a high priority, so it needs to do as little work as possible
 		if (req.limit != CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT && // Always do data distribution requests
 		    (commitData->stats.keyServerLocationIn.getValue() - commitData->stats.keyServerLocationOut.getValue() >
@@ -2222,14 +2233,14 @@ ACTOR static Future<Void> readRequestServer(CommitProxyInterface proxy,
 	}
 }
 
-ACTOR static Future<Void> rejoinServer(CommitProxyInterface proxy, ProxyCommitData* commitData) {
+static Future<Void> rejoinServer(CommitProxyInterface proxy, ProxyCommitData* commitData) {
 	// We can't respond to these requests until we have valid txnStateStore
-	wait(commitData->validState.getFuture());
+	co_await commitData->validState.getFuture();
 
 	TraceEvent("ProxyReadyForReads", proxy.id()).log();
 
-	loop {
-		GetStorageServerRejoinInfoRequest req = waitNext(proxy.getStorageServerRejoinInfo.getFuture());
+	while (true) {
+		GetStorageServerRejoinInfoRequest req = co_await proxy.getStorageServerRejoinInfo.getFuture();
 		if (commitData->txnStateStore->readValue(serverListKeyFor(req.id)).get().present()) {
 			GetStorageServerRejoinInfoReply rep;
 			rep.version = commitData->version.get();
@@ -2290,33 +2301,30 @@ ACTOR static Future<Void> rejoinServer(CommitProxyInterface proxy, ProxyCommitDa
 	}
 }
 
-ACTOR Future<Void> ddMetricsRequestServer(CommitProxyInterface proxy, Reference<AsyncVar<ServerDBInfo> const> db) {
-	loop {
-		choose {
-			when(state GetDDMetricsRequest req = waitNext(proxy.getDDMetrics.getFuture())) {
-				if (!db->get().distributor.present()) {
-					req.reply.sendError(dd_not_found());
-					continue;
-				}
-				ErrorOr<GetDataDistributorMetricsReply> reply =
-				    wait(errorOr(db->get().distributor.get().dataDistributorMetrics.getReply(
-				        GetDataDistributorMetricsRequest(req.keys, req.shardLimit))));
-				if (reply.isError()) {
-					req.reply.sendError(reply.getError());
-				} else {
-					GetDDMetricsReply newReply;
-					newReply.storageMetricsList = reply.get().storageMetricsList;
-					req.reply.send(newReply);
-				}
-			}
+Future<Void> ddMetricsRequestServer(CommitProxyInterface proxy, Reference<AsyncVar<ServerDBInfo> const> db) {
+	while (true) {
+		GetDDMetricsRequest req = co_await proxy.getDDMetrics.getFuture();
+		if (!db->get().distributor.present()) {
+			req.reply.sendError(dd_not_found());
+			continue;
+		}
+		ErrorOr<GetDataDistributorMetricsReply> reply =
+		    co_await errorOr(db->get().distributor.get().dataDistributorMetrics.getReply(
+		        GetDataDistributorMetricsRequest(req.keys, req.shardLimit)));
+		if (reply.isError()) {
+			req.reply.sendError(reply.getError());
+		} else {
+			GetDDMetricsReply newReply;
+			newReply.storageMetricsList = reply.get().storageMetricsList;
+			req.reply.send(newReply);
 		}
 	}
 }
 
-ACTOR Future<Void> monitorRemoteCommitted(ProxyCommitData* self) {
-	loop {
-		wait(delay(0)); // allow this actor to be cancelled if we are removed after db changes.
-		state Optional<std::vector<OptionalInterface<TLogInterface>>> remoteLogs;
+Future<Void> monitorRemoteCommitted(ProxyCommitData* self) {
+	while (true) {
+		co_await delay(0); // allow this actor to be cancelled if we are removed after db changes.
+		Optional<std::vector<OptionalInterface<TLogInterface>>> remoteLogs;
 		if (self->db->get().recoveryState >= RecoveryState::ALL_LOGS_RECRUITED) {
 			for (auto& logSet : self->db->get().logSystemConfig.tLogs) {
 				if (!logSet.isLocal) {
@@ -2333,19 +2341,19 @@ ACTOR Future<Void> monitorRemoteCommitted(ProxyCommitData* self) {
 		}
 
 		if (!remoteLogs.present()) {
-			wait(self->db->onChange());
+			co_await self->db->onChange();
 			continue;
 		}
 		self->popRemoteTxs = true;
 
-		state Future<Void> onChange = self->db->onChange();
-		loop {
-			state std::vector<Future<TLogQueuingMetricsReply>> replies;
+		Future<Void> onChange = self->db->onChange();
+		while (true) {
+			std::vector<Future<TLogQueuingMetricsReply>> replies;
 			for (auto& it : remoteLogs.get()) {
 				replies.push_back(
 				    brokenPromiseToNever(it.interf().getQueuingMetrics.getReply(TLogQueuingMetricsRequest())));
 			}
-			wait(waitForAll(replies) || onChange);
+			co_await (waitForAll(replies) || onChange);
 
 			if (onChange.isReady()) {
 				break;
@@ -2363,7 +2371,7 @@ ACTOR Future<Void> monitorRemoteCommitted(ProxyCommitData* self) {
 				self->txsPopVersions.pop_front();
 			}
 
-			wait(delay(SERVER_KNOBS->UPDATE_REMOTE_LOG_VERSION_INTERVAL) || onChange);
+			co_await (delay(SERVER_KNOBS->UPDATE_REMOTE_LOG_VERSION_INTERVAL) || onChange);
 			if (onChange.isReady()) {
 				break;
 			}
@@ -2371,7 +2379,7 @@ ACTOR Future<Void> monitorRemoteCommitted(ProxyCommitData* self) {
 	}
 }
 
-ACTOR Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* commitData) {
+Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* commitData) {
 	TraceEvent("SnapCommitProxy_SnapReqEnter")
 	    .detail("SnapPayload", snapReq.snapPayload)
 	    .detail("SnapUID", snapReq.snapUID);
@@ -2412,33 +2420,35 @@ ACTOR Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co
 			throw snap_log_anti_quorum_unsupported();
 		}
 
-		state int snapReqRetry = 0;
-		state double snapRetryBackoff = FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY;
-		loop {
+		int snapReqRetry = 0;
+		double snapRetryBackoff = FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY;
+		while (true) {
 			// send a snap request to DD
 			if (!commitData->db->get().distributor.present()) {
 				TraceEvent(SevWarnAlways, "DataDistributorNotPresent").detail("Operation", "SnapRequest");
 				throw dd_not_found();
 			}
+			Error err;
 			try {
 				Future<ErrorOr<Void>> ddSnapReq =
 				    commitData->db->get().distributor.get().distributorSnapReq.tryGetReply(
 				        DistributorSnapRequest(snapReq.snapPayload, snapReq.snapUID));
-				wait(throwErrorOr(ddSnapReq));
+				co_await throwErrorOr(ddSnapReq);
 				break;
 			} catch (Error& e) {
-				TraceEvent("SnapCommitProxy_DDSnapResponseError")
-				    .errorUnsuppressed(e)
-				    .detail("SnapPayload", snapReq.snapPayload)
-				    .detail("SnapUID", snapReq.snapUID)
-				    .detail("Retry", snapReqRetry);
-				// Retry if we have network issues
-				if (e.code() != error_code_request_maybe_delivered ||
-				    ++snapReqRetry > SERVER_KNOBS->SNAP_NETWORK_FAILURE_RETRY_LIMIT)
-					throw e;
-				wait(delay(snapRetryBackoff));
-				snapRetryBackoff = snapRetryBackoff * 2; // exponential backoff
+				err = e;
 			}
+			TraceEvent("SnapCommitProxy_DDSnapResponseError")
+			    .errorUnsuppressed(err)
+			    .detail("SnapPayload", snapReq.snapPayload)
+			    .detail("SnapUID", snapReq.snapUID)
+			    .detail("Retry", snapReqRetry);
+			// Retry if we have network issues
+			if (err.code() != error_code_request_maybe_delivered ||
+			    ++snapReqRetry > SERVER_KNOBS->SNAP_NETWORK_FAILURE_RETRY_LIMIT)
+				throw err;
+			co_await delay(snapRetryBackoff);
+			snapRetryBackoff = snapRetryBackoff * 2; // exponential backoff
 		}
 		snapReq.reply.send(Void());
 	} catch (Error& e) {
@@ -2455,71 +2465,87 @@ ACTOR Future<Void> proxySnapCreate(ProxySnapRequest snapReq, ProxyCommitData* co
 	TraceEvent("SnapCommitProxy_SnapReqExit")
 	    .detail("SnapPayload", snapReq.snapPayload)
 	    .detail("SnapUID", snapReq.snapUID);
-	return Void();
 }
 
-ACTOR Future<Void> proxyCheckSafeExclusion(Reference<AsyncVar<ServerDBInfo> const> db,
-                                           ExclusionSafetyCheckRequest req) {
+Future<Void> proxyCheckSafeExclusion(Reference<AsyncVar<ServerDBInfo> const> db, ExclusionSafetyCheckRequest req) {
 	TraceEvent("SafetyCheckCommitProxyBegin").log();
-	state ExclusionSafetyCheckReply reply(false);
+	ExclusionSafetyCheckReply reply(false);
 	if (!db->get().distributor.present()) {
 		TraceEvent(SevWarnAlways, "DataDistributorNotPresent").detail("Operation", "ExclusionSafetyCheck");
 		req.reply.send(reply);
-		return Void();
+		co_return;
 	}
 	try {
-		state Future<ErrorOr<DistributorExclusionSafetyCheckReply>> ddSafeFuture =
+		Future<ErrorOr<DistributorExclusionSafetyCheckReply>> ddSafeFuture =
 		    db->get().distributor.get().distributorExclCheckReq.tryGetReply(
 		        DistributorExclusionSafetyCheckRequest(req.exclusions));
-		DistributorExclusionSafetyCheckReply _reply = wait(throwErrorOr(ddSafeFuture));
-		reply.safe = _reply.safe;
+		DistributorExclusionSafetyCheckReply ddReply = co_await throwErrorOr(ddSafeFuture);
+		reply.safe = ddReply.safe;
 	} catch (Error& e) {
 		TraceEvent("SafetyCheckCommitProxyResponseError").error(e);
 		if (e.code() != error_code_operation_cancelled) {
 			req.reply.sendError(e);
-			return Void();
+			co_return;
 		} else {
 			throw e;
 		}
 	}
 	TraceEvent("SafetyCheckCommitProxyFinish").log();
 	req.reply.send(reply);
-	return Void();
 }
 
-ACTOR Future<Void> reportTxnTagCommitCost(UID myID,
-                                          Reference<AsyncVar<ServerDBInfo> const> db,
-                                          UIDTransactionTagMap<TransactionCommitCostEstimation>* ssTrTagCommitCost) {
-	state Future<Void> nextRequestTimer = Never();
-	state Future<Void> nextReply = Never();
-	if (db->get().ratekeeper.present())
-		nextRequestTimer = Void();
-	loop choose {
-		when(wait(db->onChange())) {
+class TxnTagCommitCostReporter {
+	UID myID;
+	Reference<AsyncVar<ServerDBInfo> const> db;
+	UIDTransactionTagMap<TransactionCommitCostEstimation>* ssTrTagCommitCost;
+
+	Future<Void> traceRatekeeperChanges() {
+		while (true) {
+			co_await db->onChange();
 			if (db->get().ratekeeper.present()) {
 				TraceEvent("ProxyRatekeeperChanged", myID).detail("RKID", db->get().ratekeeper.get().id());
-				nextRequestTimer = Void();
 			} else {
 				TraceEvent("ProxyRatekeeperDied", myID).log();
-				nextRequestTimer = Never();
 			}
 		}
-		when(wait(nextRequestTimer)) {
-			nextRequestTimer = Never();
-			if (db->get().ratekeeper.present()) {
-				nextReply = brokenPromiseToNever(db->get().ratekeeper.get().reportCommitCostEstimation.getReply(
-				    ReportCommitCostEstimationRequest(std::move(*ssTrTagCommitCost))));
-				ssTrTagCommitCost->clear();
-			} else {
-				nextReply = Never();
+	}
+
+	Future<Void> reportCommitCosts() {
+		bool sendImmediately = db->get().ratekeeper.present();
+		while (true) {
+			if (!sendImmediately) {
+				co_await db->onChange();
 			}
-		}
-		when(wait(nextReply)) {
-			nextReply = Never();
-			nextRequestTimer = delay(SERVER_KNOBS->REPORT_TRANSACTION_COST_ESTIMATION_DELAY);
+			sendImmediately = false;
+
+			if (!db->get().ratekeeper.present()) {
+				continue;
+			}
+
+			auto reply = brokenPromiseToNever(db->get().ratekeeper.get().reportCommitCostEstimation.getReply(
+			    ReportCommitCostEstimationRequest(std::move(*ssTrTagCommitCost))));
+			ssTrTagCommitCost->clear();
+
+			auto dbChanged = db->onChange();
+			auto replyOrChange = co_await race(reply, dbChanged);
+			if (replyOrChange.index() == 1) {
+				sendImmediately = true;
+				continue;
+			}
+
+			co_await race(delay(SERVER_KNOBS->REPORT_TRANSACTION_COST_ESTIMATION_DELAY), dbChanged);
+			sendImmediately = true;
 		}
 	}
-}
+
+public:
+	TxnTagCommitCostReporter(UID myID,
+	                         Reference<AsyncVar<ServerDBInfo> const> db,
+	                         UIDTransactionTagMap<TransactionCommitCostEstimation>* ssTrTagCommitCost)
+	  : myID(myID), db(db), ssTrTagCommitCost(ssTrTagCommitCost) {}
+
+	Future<Void> run() { co_await race(traceRatekeeperChanges(), reportCommitCosts()); }
+};
 
 namespace {
 struct ExpireServerEntry {
@@ -2552,50 +2578,15 @@ struct hash<IdempotencyKey> {
 
 } // namespace std
 
-ACTOR static Future<Void> idempotencyIdsExpireServer(
-    Database db,
-    PublicRequestStream<ExpireIdempotencyIdRequest> expireIdempotencyId,
-    PromiseStream<ExpectedIdempotencyIdCountForKey> expectedIdempotencyIdCountForKey,
-    Standalone<VectorRef<MutationRef>>* idempotencyClears) {
-	state std::unordered_map<IdempotencyKey, ExpireServerEntry> idStatus;
-	state std::unordered_map<IdempotencyKey, ExpireServerEntry>::iterator iter;
-	state int64_t purgeBefore;
-	state IdempotencyKey key;
-	state ExpireServerEntry* status = nullptr;
-	state Future<Void> purgeOld = Void();
-	loop {
-		choose {
-			when(ExpireIdempotencyIdRequest req = waitNext(expireIdempotencyId.getFuture())) {
-				key = IdempotencyKey{ req.commitVersion, req.batchIndexHighByte };
-				status = &idStatus[key];
-				status->receivedCount += 1;
-				CODE_PROBE(status->expectedCount == 0, "ExpireIdempotencyIdRequest received before count is known");
-				if (status->expectedCount > 0) {
-					ASSERT_LE(status->receivedCount, status->expectedCount);
-				}
-			}
-			when(ExpectedIdempotencyIdCountForKey req = waitNext(expectedIdempotencyIdCountForKey.getFuture())) {
-				key = IdempotencyKey{ req.commitVersion, req.batchIndexHighByte };
-				status = &idStatus[key];
-				ASSERT_EQ(status->expectedCount, 0);
-				status->expectedCount = req.idempotencyIdCount;
-			}
-			when(wait(purgeOld)) {
-				purgeOld = delay(SERVER_KNOBS->IDEMPOTENCY_ID_IN_MEMORY_LIFETIME);
-				purgeBefore = now() - SERVER_KNOBS->IDEMPOTENCY_ID_IN_MEMORY_LIFETIME;
-				for (iter = idStatus.begin(); iter != idStatus.end();) {
-					// We have exclusive access to idStatus in this when block, so iter will still be valid after the
-					// wait
-					wait(yield());
-					if (iter->second.timeReceived < purgeBefore) {
-						iter = idStatus.erase(iter);
-					} else {
-						++iter;
-					}
-				}
-				continue;
-			}
-		}
+namespace {
+
+class IdempotencyIdsExpireServer {
+	PublicRequestStream<ExpireIdempotencyIdRequest> expireIdempotencyId;
+	PromiseStream<ExpectedIdempotencyIdCountForKey> expectedIdempotencyIdCountForKey;
+	Standalone<VectorRef<MutationRef>>* idempotencyClears;
+	std::unordered_map<IdempotencyKey, ExpireServerEntry> idStatus;
+
+	void updateStatus(IdempotencyKey key, ExpireServerEntry* status) {
 		if (status->initialized) {
 			if (status->receivedCount == status->expectedCount) {
 				auto keyRange =
@@ -2609,9 +2600,65 @@ ACTOR static Future<Void> idempotencyIdsExpireServer(
 			status->initialized = true;
 		}
 	}
-}
 
-namespace {
+	Future<Void> receiveExpireRequests() {
+		while (true) {
+			ExpireIdempotencyIdRequest req = co_await expireIdempotencyId.getFuture();
+			IdempotencyKey key{ req.commitVersion, req.batchIndexHighByte };
+			ExpireServerEntry* status = &idStatus[key];
+			status->receivedCount += 1;
+			CODE_PROBE(status->expectedCount == 0, "ExpireIdempotencyIdRequest received before count is known");
+			if (status->expectedCount > 0) {
+				ASSERT_LE(status->receivedCount, status->expectedCount);
+			}
+			updateStatus(key, status);
+		}
+	}
+
+	Future<Void> receiveExpectedCounts() {
+		while (true) {
+			ExpectedIdempotencyIdCountForKey req = co_await expectedIdempotencyIdCountForKey.getFuture();
+			IdempotencyKey key{ req.commitVersion, req.batchIndexHighByte };
+			ExpireServerEntry* status = &idStatus[key];
+			ASSERT_EQ(status->expectedCount, 0);
+			status->expectedCount = req.idempotencyIdCount;
+			updateStatus(key, status);
+		}
+	}
+
+	Future<Void> purgeOldEntries() {
+		while (true) {
+			co_await delay(SERVER_KNOBS->IDEMPOTENCY_ID_IN_MEMORY_LIFETIME);
+			int64_t purgeBefore = now() - SERVER_KNOBS->IDEMPOTENCY_ID_IN_MEMORY_LIFETIME;
+			std::vector<IdempotencyKey> keys;
+			keys.reserve(idStatus.size());
+			for (const auto& entry : idStatus) {
+				keys.push_back(entry.first);
+			}
+
+			std::vector<IdempotencyKey> keysToErase;
+			for (const auto& key : keys) {
+				co_await yield();
+				auto status = idStatus.find(key);
+				if (status != idStatus.end() && status->second.timeReceived < purgeBefore) {
+					keysToErase.push_back(key);
+				}
+			}
+			for (const auto& key : keysToErase) {
+				idStatus.erase(key);
+			}
+		}
+	}
+
+public:
+	IdempotencyIdsExpireServer(PublicRequestStream<ExpireIdempotencyIdRequest> expireIdempotencyId,
+	                           PromiseStream<ExpectedIdempotencyIdCountForKey> expectedIdempotencyIdCountForKey,
+	                           Standalone<VectorRef<MutationRef>>* idempotencyClears)
+	  : expireIdempotencyId(expireIdempotencyId), expectedIdempotencyIdCountForKey(expectedIdempotencyIdCountForKey),
+	    idempotencyClears(idempotencyClears) {}
+
+	Future<Void> run() { co_await race(receiveExpireRequests(), receiveExpectedCounts(), purgeOldEntries()); }
+};
 
 struct TransactionStateResolveContext {
 	// Maximum sequence for txnStateRequest, this is defined when the request last flag is set.
@@ -2643,17 +2690,17 @@ struct TransactionStateResolveContext {
 	}
 };
 
-ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolveContext* pContext) {
-	state KeyRange txnKeys = allKeys;
-	state std::map<Tag, UID> tag_uid;
+Future<Void> processCompleteTransactionStateRequest(TransactionStateResolveContext* pContext) {
+	KeyRange txnKeys = allKeys;
+	std::map<Tag, UID> tag_uid;
 
 	RangeResult UIDtoTagMap = pContext->pTxnStateStore->readRange(serverTagKeys).get();
 	for (const KeyValueRef& kv : UIDtoTagMap) {
 		tag_uid[decodeServerTagValue(kv.value)] = decodeServerTagKey(kv.key);
 	}
 
-	loop {
-		wait(yield());
+	while (true) {
+		co_await yield();
 
 		RangeResult data =
 		    pContext->pTxnStateStore
@@ -2664,11 +2711,10 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 
 		((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
 
-		MutationsVec mutations;
+		Standalone<VectorRef<MutationRef>> mutations;
 		std::vector<std::pair<MapPair<Key, ServerCacheInfo>, int>> keyInfoData;
 		std::vector<UID> src, dest;
 		ServerCacheInfo info;
-		// NOTE: An ACTOR will be compiled into several classes, the this pointer is from one of them.
 		auto updateTagInfo = [pContext = pContext](const std::vector<UID>& uids,
 		                                           std::vector<Tag>& tags,
 		                                           std::vector<Reference<StorageInfo>>& storageInfoItems) {
@@ -2716,9 +2762,9 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 		Arena arena;
 		bool confChanges;
 		applyMetadataMutations(SpanContext(),
-		                       *pContext->pCommitData,
+		                       pContext->pCommitData->getApplyMetadataProxyContext(),
 		                       arena,
-		                       Reference<ILogSystem>(),
+		                       Reference<LogSystem>(),
 		                       mutations,
 		                       /* pToCommit= */ nullptr,
 		                       confChanges,
@@ -2726,30 +2772,27 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(TransactionStateResolv
 		                       /* popVersion= */ 0,
 		                       /* initialCommit= */ true,
 		                       /* provisionalCommitProxy */ pContext->pCommitData->provisional);
-	} // loop
+	}
 
 	auto lockedKey = pContext->pTxnStateStore->readValue(databaseLockedKey).get();
 	pContext->pCommitData->locked = lockedKey.present() && lockedKey.get().size();
 	pContext->pCommitData->metadataVersion = pContext->pTxnStateStore->readValue(metadataVersionKey).get();
 
 	pContext->pTxnStateStore->enableSnapshot();
-
-	return Void();
 }
 
-ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveContext* pContext,
-                                                      TxnStateRequest request) {
+Future<Void> processTransactionStateRequestPart(TransactionStateResolveContext* pContext, TxnStateRequest request) {
 	ASSERT(pContext->pCommitData != nullptr);
 	ASSERT(pContext->pActors != nullptr);
 
 	if (pContext->receivedSequences.contains(request.sequence)) {
 		if (pContext->receivedSequences.size() == pContext->maxSequence) {
-			wait(pContext->txnRecovery);
+			co_await pContext->txnRecovery;
 		}
 		// This part is already received. Still we will re-broadcast it to other CommitProxies
 		pContext->pActors->send(broadcastTxnRequest(request, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
-		wait(yield());
-		return Void();
+		co_await yield();
+		co_return;
 	}
 
 	if (request.last) {
@@ -2772,13 +2815,12 @@ ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveCon
 		// Received all components of the txnStateRequest
 		ASSERT(!pContext->processed);
 		pContext->txnRecovery = processCompleteTransactionStateRequest(pContext);
-		wait(pContext->txnRecovery);
+		co_await pContext->txnRecovery;
 		pContext->processed = true;
 	}
 
 	pContext->pActors->send(broadcastTxnRequest(request, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
-	wait(yield());
-	return Void();
+	co_await yield();
 }
 
 } // anonymous namespace
@@ -2801,25 +2843,19 @@ ACTOR Future<Void> processTransactionStateRequestPart(TransactionStateResolveCon
 // `BURSTINESS_METRICS_ENABLED` knob to false. The reporting interval can be
 // adjusted by modifying the knob `BURSTINESS_METRICS_LOG_INTERVAL`.
 //
-ACTOR Future<Void> logDetailedMetrics(ProxyCommitData* commitData) {
-	state double startTime = 0;
-	state int64_t commitBatchInBaseline = 0;
-	state int64_t txnCommitInBaseline = 0;
-	state int64_t mutationsBaseline = 0;
-	state int64_t mutationBytesBaseline = 0;
-
-	loop {
+Future<Void> logDetailedMetrics(ProxyCommitData* commitData) {
+	while (true) {
 		if (!SERVER_KNOBS->BURSTINESS_METRICS_ENABLED) {
-			return Void();
+			co_return;
 		}
 
-		startTime = now();
-		commitBatchInBaseline = commitData->stats.commitBatchIn.getValue();
-		txnCommitInBaseline = commitData->stats.txnCommitIn.getValue();
-		mutationsBaseline = commitData->stats.mutations.getValue();
-		mutationBytesBaseline = commitData->stats.mutationBytes.getValue();
+		double startTime = now();
+		int64_t commitBatchInBaseline = commitData->stats.commitBatchIn.getValue();
+		int64_t txnCommitInBaseline = commitData->stats.txnCommitIn.getValue();
+		int64_t mutationsBaseline = commitData->stats.mutations.getValue();
+		int64_t mutationBytesBaseline = commitData->stats.mutationBytes.getValue();
 
-		wait(delay(SERVER_KNOBS->BURSTINESS_METRICS_LOG_INTERVAL));
+		co_await delay(SERVER_KNOBS->BURSTINESS_METRICS_LOG_INTERVAL);
 
 		int64_t commitBatchInReal = commitData->stats.commitBatchIn.getValue();
 		int64_t txnCommitInReal = commitData->stats.txnCommitIn.getValue();
@@ -2843,131 +2879,31 @@ ACTOR Future<Void> logDetailedMetrics(ProxyCommitData* commitData) {
 	}
 }
 
-ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
-                                         MasterInterface master,
-                                         LifetimeToken masterLifetime,
-                                         Reference<AsyncVar<ServerDBInfo> const> db,
-                                         LogEpoch epoch,
-                                         Version recoveryTransactionVersion,
-                                         bool firstProxy,
-                                         std::string whitelistBinPaths,
-                                         bool provisional,
-                                         uint16_t commitProxyIndex) {
-	state ProxyCommitData commitData(proxy.id(),
-	                                 master,
-	                                 proxy.getConsistentReadVersion,
-	                                 recoveryTransactionVersion,
-	                                 proxy.commit,
-	                                 db,
-	                                 firstProxy,
-	                                 provisional,
-	                                 commitProxyIndex,
-	                                 epoch);
-
-	state Future<Sequence> sequenceFuture = (Sequence)0;
-	state PromiseStream<std::pair<std::vector<CommitTransactionRequest>, int>> batchedCommits;
-	state Future<Void> commitBatcherActor;
-	state Future<Void> lastCommitComplete = Void();
-
-	state PromiseStream<Future<Void>> addActor;
-	state Future<Void> onError = transformError(actorCollection(addActor.getFuture()), broken_promise(), tlog_failed());
-
-	addActor.send(waitFailureServer(proxy.waitFailure.getFuture()));
-	addActor.send(traceRole(Role::COMMIT_PROXY, proxy.id()));
-
-	//TraceEvent("CommitProxyInit1", proxy.id());
-
-	// Wait until we can load the "real" logsystem, since we don't support switching them currently
-	while (!(masterLifetime.isEqual(commitData.db->get().masterLifetime) &&
-	         commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION)) {
-		//TraceEvent("ProxyInit2", proxy.id()).detail("LSEpoch", db->get().logSystemConfig.epoch).detail("Need", epoch);
-		wait(commitData.db->onChange());
-	}
-	state Future<Void> dbInfoChange = commitData.db->onChange();
-	//TraceEvent("ProxyInit3", proxy.id());
-
-	commitData.resolvers = commitData.db->get().resolvers;
-	commitData.localTLogCount = commitData.db->get().logSystemConfig.numLogs();
-	ASSERT(commitData.resolvers.size() != 0);
-	for (int i = 0; i < commitData.resolvers.size(); ++i) {
-		commitData.stats.resolverDist.push_back(Histogram::getHistogram(
-		    "CommitProxy"_sr, "ToResolver_" + commitData.resolvers[i].id().toString(), Histogram::Unit::milliseconds));
-	}
-
-	// Initialize keyResolvers map
-	auto rs = commitData.keyResolvers.modify(SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? normalKeys : allKeys);
-	for (auto r = rs.begin(); r != rs.end(); ++r)
-		r->value().emplace_back(0, 0);
-	commitData.systemKeyVersions.push_back(0);
-
-	commitData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), commitData.db->get(), false, addActor);
-	commitData.logAdapter =
-	    new LogSystemDiskQueueAdapter(commitData.logSystem, Reference<AsyncVar<PeekTxsInfo>>(), 1, false);
-	commitData.txnStateStore = keyValueStoreLogSystem(
-	    commitData.logAdapter, commitData.db, proxy.id(), 2e9, /*doc=*/true, /*ument=*/true, /*constants=*/true);
-	createWhitelistBinPathVec(whitelistBinPaths, commitData.whitelistedBinPathVec);
-
-	commitData.updateLatencyBandConfig(commitData.db->get().latencyBandConfig);
-
-	// ((SERVER_MEM_LIMIT * COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) / COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR) is
-	// only a approximate formula for limiting the memory used. COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR is an
-	// estimate based on experiments and not an accurate one.
-	state int64_t commitBatchesMemoryLimit = SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT;
-	if (SERVER_KNOBS->SERVER_MEM_LIMIT > 0) {
-		commitBatchesMemoryLimit = std::min(
-		    commitBatchesMemoryLimit,
-		    static_cast<int64_t>((SERVER_KNOBS->SERVER_MEM_LIMIT * SERVER_KNOBS->COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) /
-		                         SERVER_KNOBS->COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR));
-	}
-	TraceEvent(SevInfo, "CommitBatchesMemoryLimit").detail("BytesLimit", commitBatchesMemoryLimit);
-
-	// Initialize RangeLock
-	if (commitData.rangeLockEnabled()) {
-		commitData.rangeLock = std::make_shared<RangeLock>(&commitData);
-		TraceEvent(SevInfo, "CommitProxyRangeLockEnabled", commitData.dbgid);
-	}
-
-	addActor.send(monitorRemoteCommitted(&commitData));
-	addActor.send(readRequestServer(proxy, addActor, &commitData));
-	addActor.send(rejoinServer(proxy, &commitData));
-	addActor.send(ddMetricsRequestServer(proxy, db));
-	addActor.send(reportTxnTagCommitCost(proxy.id(), db, &commitData.ssTrTagCommitCost));
-	addActor.send(logDetailedMetrics(&commitData));
-
-	auto openDb = openDBOnServer(db);
-
-	if (firstProxy) {
-		addActor.send(recurringAsync(
-		    [openDb = openDb]() { return cleanIdempotencyIds(openDb, SERVER_KNOBS->IDEMPOTENCY_IDS_MIN_AGE_SECONDS); },
-		    SERVER_KNOBS->IDEMPOTENCY_IDS_CLEANER_POLLING_INTERVAL,
-		    true,
-		    SERVER_KNOBS->IDEMPOTENCY_IDS_CLEANER_POLLING_INTERVAL));
-	}
-	addActor.send(idempotencyIdsExpireServer(
-	    openDb, proxy.expireIdempotencyId, commitData.expectedIdempotencyIdCountForKey, &commitData.idempotencyClears));
-
-	// wait for txnStateStore recovery
-	wait(success(commitData.txnStateStore->readValue(StringRef())));
-
-	int commitBatchByteLimit =
-	    (int)std::min<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MAX,
-	                          std::max<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MIN,
-	                                           SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE *
-	                                               pow(commitData.db->get().client.commitProxies.size(),
-	                                                   SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER)));
-
-	commitBatcherActor = commitBatcher(
-	    &commitData, batchedCommits, proxy.commit.getFuture(), commitBatchByteLimit, commitBatchesMemoryLimit);
-
-	// This has to be declared after the commitData.txnStateStore get initialized
-	state TransactionStateResolveContext transactionStateResolveContext(&commitData, &addActor);
-
-	loop choose {
-		when(wait(dbInfoChange)) {
+class CommitProxyServerCore {
+	CommitProxyInterface proxy;
+	LifetimeToken masterLifetime;
+	Reference<AsyncVar<ServerDBInfo> const> db;
+	bool firstProxy;
+	std::string whitelistBinPaths;
+
+	ProxyCommitData commitData;
+	PromiseStream<std::pair<std::vector<CommitTransactionRequest>, int>> batchedCommits;
+	Future<Void> commitBatcherActor;
+	Future<Void> lastCommitComplete = Void();
+	TxnTagCommitCostReporter txnTagCommitCostReporter;
+	IdempotencyIdsExpireServer idempotencyIdsExpireServer;
+	PromiseStream<Future<Void>> addActor;
+	Future<Void> onError;
+	Future<Void> dbInfoChange;
+	TransactionStateResolveContext transactionStateResolveContext;
+
+	Future<Void> serveDbInfoChanges() {
+		while (true) {
+			co_await dbInfoChange;
 			dbInfoChange = commitData.db->onChange();
 			if (masterLifetime.isEqual(commitData.db->get().masterLifetime) &&
 			    commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION) {
-				commitData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), commitData.db->get(), false, addActor);
+				commitData.logSystem = makeLogSystemFromServerDBInfo(proxy.id(), commitData.db->get(), false, addActor);
 				for (auto it : commitData.tag_popped) {
 					commitData.logSystem->pop(it.second, it.first);
 				}
@@ -2976,9 +2912,14 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 
 			commitData.updateLatencyBandConfig(commitData.db->get().latencyBandConfig);
 		}
-		when(wait(onError)) {}
-		when(std::pair<std::vector<CommitTransactionRequest>, int> batchedRequests =
-		         waitNext(batchedCommits.getFuture())) {
+	}
+
+	Future<Void> serveActorErrors() { co_await onError; }
+
+	Future<Void> serveBatchedCommits() {
+		while (true) {
+			std::pair<std::vector<CommitTransactionRequest>, int> batchedRequests = co_await batchedCommits.getFuture();
+
 			// WARNING: this code is run at a high priority, so it needs to do as little work as possible
 			/*
 			TraceEvent("CommitProxyCTR", proxy.id())
@@ -2996,28 +2937,46 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 			const std::vector<CommitTransactionRequest>& trs = batchedRequests.first;
 			const int batchBytes = batchedRequests.second;
 			if (trs.size() ||
-			    (commitData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS &&
-			     masterLifetime.isEqual(commitData.db->get().masterLifetime) && lastCommitComplete.isReady())) {
-
+				(commitData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS &&
+				masterLifetime.isEqual(commitData.db->get().masterLifetime) && lastCommitComplete.isReady())) {
 				lastCommitComplete =
-				    commitBatch(&commitData,
-				                const_cast<std::vector<CommitTransactionRequest>*>(&batchedRequests.first),
-				                batchBytes);
-
+					tag(commitBatch(&commitData,
+									const_cast<std::vector<CommitTransactionRequest>*>(
+										&batchedRequests.first),
+									batchBytes),
+						Void(),
+						commitData.committedBatches);
 				addActor.send(lastCommitComplete);
-			}
+			} else
+				commitData.committedBatches.send(Void());
 		}
-		when(ProxySnapRequest snapReq = waitNext(proxy.proxySnapReq.getFuture())) {
+	}
+
+	Future<Void> serveProxySnapRequests() {
+		while (true) {
+			ProxySnapRequest snapReq = co_await proxy.proxySnapReq.getFuture();
 			TraceEvent(SevDebug, "SnapMasterEnqueue").log();
 			addActor.send(proxySnapCreate(snapReq, &commitData));
 		}
-		when(ExclusionSafetyCheckRequest exclCheckReq = waitNext(proxy.exclusionSafetyCheckReq.getFuture())) {
+	}
+
+	Future<Void> serveExclusionSafetyCheckRequests() {
+		while (true) {
+			ExclusionSafetyCheckRequest exclCheckReq = co_await proxy.exclusionSafetyCheckReq.getFuture();
 			addActor.send(proxyCheckSafeExclusion(db, exclCheckReq));
 		}
-		when(TxnStateRequest request = waitNext(proxy.txnState.getFuture())) {
+	}
+
+	Future<Void> serveTxnStateRequests() {
+		while (true) {
+			TxnStateRequest request = co_await proxy.txnState.getFuture();
 			addActor.send(processTransactionStateRequestPart(&transactionStateResolveContext, request));
 		}
-		when(SetThrottledShardRequest request = waitNext(proxy.setThrottledShard.getFuture())) {
+	}
+
+	Future<Void> serveSetThrottledShardRequests() {
+		while (true) {
+			SetThrottledShardRequest request = co_await proxy.setThrottledShard.getFuture();
 			for (auto& shard : request.throttledShards) {
 				auto it = commitData.hotShards.begin();
 				for (; it != commitData.hotShards.end(); ++it) {
@@ -3033,17 +2992,154 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
 			// TraceEvent(SevDebug, "ReceivedSetThrottledShards").detail("NumHotShards", commitData.hotShards.size());
 		}
 	}
-}
+
+public:
+	CommitProxyServerCore(CommitProxyInterface proxy,
+	                      MasterInterface master,
+	                      LifetimeToken masterLifetime,
+	                      Reference<AsyncVar<ServerDBInfo> const> db,
+	                      LogEpoch epoch,
+	                      Version recoveryTransactionVersion,
+	                      bool firstProxy,
+	                      std::string whitelistBinPaths,
+	                      bool provisional,
+	                      uint16_t commitProxyIndex)
+	  : proxy(proxy), masterLifetime(masterLifetime), db(db), firstProxy(firstProxy),
+	    whitelistBinPaths(std::move(whitelistBinPaths)), commitData(proxy.id(),
+	                                                                master,
+	                                                                recoveryTransactionVersion,
+	                                                                proxy.commit,
+	                                                                db,
+	                                                                firstProxy,
+	                                                                provisional,
+	                                                                commitProxyIndex,
+	                                                                epoch),
+	    txnTagCommitCostReporter(proxy.id(), db, &commitData.ssTrTagCommitCost),
+	    idempotencyIdsExpireServer(proxy.expireIdempotencyId,
+	                               commitData.expectedIdempotencyIdCountForKey,
+	                               &commitData.idempotencyClears),
+	    onError(transformError(actorCollection(addActor.getFuture()), broken_promise(), tlog_failed())) {}
+
+	Future<Void> run() {
+		addActor.send(waitFailureServer(proxy.waitFailure.getFuture()));
+		addActor.send(traceRole(Role::COMMIT_PROXY, proxy.id()));
+
+		//TraceEvent("CommitProxyInit1", proxy.id());
+
+		// Wait until we can load the "real" logsystem, since we don't support switching them currently
+		while (!(masterLifetime.isEqual(commitData.db->get().masterLifetime) &&
+		         commitData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION)) {
+			//TraceEvent("ProxyInit2", proxy.id()).detail("LSEpoch", db->get().logSystemConfig.epoch).detail("Need", epoch);
+			co_await commitData.db->onChange();
+		}
+		dbInfoChange = commitData.db->onChange();
+		//TraceEvent("ProxyInit3", proxy.id());
+
+		commitData.resolvers = commitData.db->get().resolvers;
+		commitData.localTLogCount = commitData.db->get().logSystemConfig.numLogs();
+		ASSERT(commitData.resolvers.size() != 0);
+		for (int i = 0; i < commitData.resolvers.size(); ++i) {
+			commitData.stats.resolverDist.push_back(
+			    Histogram::getHistogram("CommitProxy"_sr,
+			                            "ToResolver_" + commitData.resolvers[i].id().toString(),
+			                            Histogram::Unit::milliseconds));
+		}
+
+		// Initialize keyResolvers map
+		auto rs =
+		    commitData.keyResolvers.modify(SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? normalKeys : allKeys);
+		for (auto r = rs.begin(); r != rs.end(); ++r)
+			r->value().emplace_back(0, 0);
+		commitData.systemKeyVersions.push_back(0);
+
+		commitData.logSystem = makeLogSystemFromServerDBInfo(proxy.id(), commitData.db->get(), false, addActor);
+		commitData.logAdapter =
+		    new LogSystemDiskQueueAdapter(commitData.logSystem, Reference<AsyncVar<PeekTxsInfo>>(), 1, false);
+		commitData.txnStateStore = keyValueStoreLogSystem(commitData.logAdapter,
+		                                                  commitData.db,
+		                                                  proxy.id(),
+		                                                  2e9,
+		                                                  DisableSnapshot::True,
+		                                                  ReplaceContent::True,
+		                                                  ExactRecovery::True);
+		createWhitelistBinPathVec(whitelistBinPaths, commitData.whitelistedBinPathVec);
+
+		commitData.updateLatencyBandConfig(commitData.db->get().latencyBandConfig);
+
+		// ((SERVER_MEM_LIMIT * COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) / COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR) is
+		// only a approximate formula for limiting the memory used. COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR is an
+		// estimate based on experiments and not an accurate one.
+		int64_t commitBatchesMemoryLimit = SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT;
+		if (SERVER_KNOBS->SERVER_MEM_LIMIT > 0) {
+			commitBatchesMemoryLimit =
+			    std::min(commitBatchesMemoryLimit,
+			             static_cast<int64_t>(
+			                 (SERVER_KNOBS->SERVER_MEM_LIMIT * SERVER_KNOBS->COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) /
+			                 SERVER_KNOBS->COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR));
+		}
+		TraceEvent(SevInfo, "CommitBatchesMemoryLimit").detail("BytesLimit", commitBatchesMemoryLimit);
+
+		// Initialize RangeLock
+		if (commitData.rangeLockEnabled()) {
+			commitData.rangeLock = std::make_shared<RangeLock>(&commitData);
+			TraceEvent(SevInfo, "CommitProxyRangeLockEnabled", commitData.dbgid);
+		}
+
+		addActor.send(monitorRemoteCommitted(&commitData));
+		addActor.send(readRequestServer(proxy, addActor, &commitData));
+		addActor.send(rejoinServer(proxy, &commitData));
+		addActor.send(ddMetricsRequestServer(proxy, db));
+		addActor.send(txnTagCommitCostReporter.run());
+		addActor.send(logDetailedMetrics(&commitData));
+
+		auto openDb = openDBOnServer(db);
+
+		if (firstProxy) {
+			addActor.send(recurringAsync(
+			    [openDb = openDb]() {
+				    return cleanIdempotencyIds(openDb, SERVER_KNOBS->IDEMPOTENCY_IDS_MIN_AGE_SECONDS);
+			    },
+			    SERVER_KNOBS->IDEMPOTENCY_IDS_CLEANER_POLLING_INTERVAL,
+			    true,
+			    SERVER_KNOBS->IDEMPOTENCY_IDS_CLEANER_POLLING_INTERVAL));
+		}
+		addActor.send(idempotencyIdsExpireServer.run());
+
+		// wait for txnStateStore recovery
+		co_await success(commitData.txnStateStore->readValue(StringRef()));
+
+		int commitBatchByteLimit =
+		    (int)std::min<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MAX,
+		                          std::max<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MIN,
+		                                           SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE *
+		                                               pow(commitData.db->get().client.commitProxies.size(),
+		                                                   SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER)));
+
+		commitBatcherActor = commitBatcher(
+		    &commitData, batchedCommits, proxy.commit.getFuture(), commitBatchByteLimit, commitBatchesMemoryLimit);
+
+		// This has to be initialized after the commitData.txnStateStore gets initialized.
+		transactionStateResolveContext = TransactionStateResolveContext(&commitData, &addActor);
+
+		co_await race(serveDbInfoChanges(),
+		              serveActorErrors(),
+		              serveBatchedCommits(),
+		              serveProxySnapRequests(),
+		              serveExclusionSafetyCheckRequests(),
+		              serveTxnStateRequests(),
+		              serveSetThrottledShardRequests());
+	}
+};
 
 // only update the local Db info if the CP is not removed
-ACTOR Future<Void> updateLocalDbInfo(Reference<AsyncVar<ServerDBInfo> const> in,
-                                     Reference<AsyncVar<ServerDBInfo>> out,
-                                     uint64_t recoveryCount,
-                                     CommitProxyInterface myInterface) {
+Future<Void> updateLocalDbInfo(Reference<AsyncVar<ServerDBInfo> const> in,
+                               Reference<AsyncVar<ServerDBInfo>> out,
+                               uint64_t recoveryCount,
+                               CommitProxyInterface myInterface) {
 	// whether this CP already receive the db info including itself
-	state bool firstValidDbInfo = false;
+	bool firstValidDbInfo = false;
 
-	loop {
+	while (true) {
 		bool isIncluded =
 		    std::count(in->get().client.commitProxies.begin(), in->get().client.commitProxies.end(), myInterface);
 		if (in->get().recoveryCount >= recoveryCount && !isIncluded) {
@@ -3068,27 +3164,27 @@ ACTOR Future<Void> updateLocalDbInfo(Reference<AsyncVar<ServerDBInfo> const> in,
 			}
 		}
 
-		wait(in->onChange());
+		co_await in->onChange();
 	}
 }
 
-ACTOR Future<Void> commitProxyServer(CommitProxyInterface proxy,
-                                     InitializeCommitProxyRequest req,
-                                     Reference<AsyncVar<ServerDBInfo> const> db,
-                                     std::string whitelistBinPaths) {
+Future<Void> commitProxyServer(CommitProxyInterface proxy,
+                               InitializeCommitProxyRequest req,
+                               Reference<AsyncVar<ServerDBInfo> const> db,
+                               std::string whitelistBinPaths) {
 	try {
-		state Reference<AsyncVar<ServerDBInfo>> localDb = makeReference<AsyncVar<ServerDBInfo>>();
-		state Future<Void> core = commitProxyServerCore(proxy,
-		                                                req.master,
-		                                                req.masterLifetime,
-		                                                localDb,
-		                                                req.recoveryCount,
-		                                                req.recoveryTransactionVersion,
-		                                                req.firstProxy,
-		                                                whitelistBinPaths,
-		                                                proxy.provisional,
-		                                                req.commitProxyIndex);
-		wait(core || updateLocalDbInfo(db, localDb, req.recoveryCount, proxy));
+		auto localDb = makeReference<AsyncVar<ServerDBInfo>>();
+		CommitProxyServerCore core(proxy,
+		                           req.master,
+		                           req.masterLifetime,
+		                           localDb,
+		                           req.recoveryCount,
+		                           req.recoveryTransactionVersion,
+		                           req.firstProxy,
+		                           std::move(whitelistBinPaths),
+		                           proxy.provisional,
+		                           req.commitProxyIndex);
+		co_await race(core.run(), updateLocalDbInfo(db, localDb, req.recoveryCount, proxy));
 	} catch (Error& e) {
 		Severity sev = e.code() == error_code_failed_to_progress ? SevWarnAlways : SevInfo;
 		TraceEvent(sev, "CommitProxyTerminated", proxy.id()).errorUnsuppressed(e);
@@ -3101,7 +3197,6 @@ ACTOR Future<Void> commitProxyServer(CommitProxyInterface proxy,
 		}
 		CODE_PROBE(e.code() == error_code_failed_to_progress, "Commit proxy failed to progress");
 	}
-	return Void();
 }
 
 void forceLinkCommitProxyTests() {}
diff --git a/fdbserver/include/fdbserver/ProxyCommitData.actor.h b/fdbserver/commitproxy/ProxyCommitData.h
similarity index 87%
rename from fdbserver/include/fdbserver/ProxyCommitData.actor.h
rename to fdbserver/commitproxy/ProxyCommitData.h
index c9ede9b12f0..743fa050919 100644
--- a/fdbserver/include/fdbserver/ProxyCommitData.actor.h
+++ b/fdbserver/commitproxy/ProxyCommitData.h
@@ -1,5 +1,5 @@
 /*
- * ProxyCommitData.actor.h
+ * ProxyCommitData.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,26 +19,20 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H)
-#define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H
-#include "fdbserver/ProxyCommitData.actor.g.h"
-#elif !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_H)
-#define FDBSERVER_PROXYCOMMITDATA_ACTOR_H
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/RangeLock.h"
 #include "fdbrpc/Stats.h"
-#include "fdbserver/AccumulativeChecksumUtil.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/ResolverInterface.h"
+#include "fdbserver/core/AccumulativeChecksumUtil.h"
+#include "fdbserver/logsystem/ApplyMetadataMutation.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/ResolverInterface.h"
+#include "../logsystem/include/fdbserver/logsystem/LogSystem.h"
 #include "flow/IRandom.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-DESCR struct SingleKeyMutation {
+struct SingleKeyMutationDescriptor {
 	Standalone<StringRef> shardBegin;
 	Standalone<StringRef> shardEnd;
 	int64_t tag1;
@@ -46,11 +40,17 @@ DESCR struct SingleKeyMutation {
 	int64_t tag3;
 };
 
-struct ApplyMutationsData {
-	Future<Void> worker;
-	Version endVersion;
-	Reference<KeyRangeMap<Version>> keyVersion;
-};
+template <>
+struct Descriptor<SingleKeyMutationDescriptor>
+  : DescribeType<SingleKeyMutationDescriptor,
+                 "SingleKeyMutation",
+                 DescribeField<&SingleKeyMutationDescriptor::shardBegin, "shardBegin">,
+                 DescribeField<&SingleKeyMutationDescriptor::shardEnd, "shardEnd">,
+                 DescribeField<&SingleKeyMutationDescriptor::tag1, "tag1">,
+                 DescribeField<&SingleKeyMutationDescriptor::tag2, "tag2">,
+                 DescribeField<&SingleKeyMutationDescriptor::tag3, "tag3">> {};
+
+class LogSystemDiskQueueAdapter;
 
 struct ProxyStats {
 	CounterCollection cc;
@@ -232,7 +232,7 @@ struct ProxyCommitData {
 	MasterInterface master;
 	std::vector<ResolverInterface> resolvers;
 	LogSystemDiskQueueAdapter* logAdapter;
-	Reference<ILogSystem> logSystem;
+	Reference<LogSystem> logSystem;
 	IKeyValueStore* txnStateStore;
 	NotifiedVersion committedVersion; // Provided that this recovery has succeeded or will succeed, this version is
 	                                  // fully committed (durable)
@@ -262,12 +262,10 @@ struct ProxyCommitData {
 	NotifiedVersion latestLocalCommitBatchResolving;
 	NotifiedVersion latestLocalCommitBatchLogging;
 
-	PublicRequestStream<GetReadVersionRequest> getConsistentReadVersion;
 	PublicRequestStream<CommitTransactionRequest> commit;
 	Database cx;
 	Reference<AsyncVar<ServerDBInfo> const> db;
-	EventMetricHandle<SingleKeyMutation> singleKeyMutationEvent;
-
+	EventMetricHandle<SingleKeyMutationDescriptor> singleKeyMutationEvent;
 	std::map<UID, Reference<StorageInfo>> storageCache;
 	std::unordered_map<UID, StorageServerInterface> tssMapping;
 	std::map<Tag, Version> tag_popped;
@@ -354,7 +352,6 @@ struct ProxyCommitData {
 
 	ProxyCommitData(UID dbgid,
 	                MasterInterface master,
-	                PublicRequestStream<GetReadVersionRequest> getConsistentReadVersion,
 	                Version recoveryTransactionVersion,
 	                PublicRequestStream<CommitTransactionRequest> commit,
 	                Reference<AsyncVar<ServerDBInfo> const> db,
@@ -368,8 +365,7 @@ struct ProxyCommitData {
 	    lastVersionTime(0), commitVersionRequestNumber(1), mostRecentProcessedRequestNumber(0), firstProxy(firstProxy),
 	    provisional(provisional), lastCoalesceTime(0), locked(false),
 	    commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN), localCommitBatchesStarted(0),
-	    getConsistentReadVersion(getConsistentReadVersion), commit(commit),
-	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True)), db(db),
+	    commit(commit), cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True)), db(db),
 	    singleKeyMutationEvent("SingleKeyMutation"_sr), lastTxsPop(0), popRemoteTxs(false), lastStartCommit(0),
 	    lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0), lastMasterReset(now()),
 	    lastResolverReset(now()), commitProxyIndex(commitProxyIndex),
@@ -381,14 +377,20 @@ struct ProxyCommitData {
 	    lastShardMove(invalidVersion), epoch(epoch) {
 		commitComputePerOperation.resize(SERVER_KNOBS->PROXY_COMPUTE_BUCKETS, 0.0);
 	}
+
+	// Build the core-owned view consumed by applyMetadataMutations() without
+	// exposing the full ProxyCommitData type outside commitproxy.
+	ApplyMetadataProxyContext getApplyMetadataProxyContext();
 };
-struct RangeLock {
+struct RangeLock : ApplyMetadataRangeLock {
 public:
-	RangeLock(ProxyCommitData* const pProxyCommitData) : pProxyCommitData(pProxyCommitData) {
+	explicit RangeLock(ProxyCommitData* const pProxyCommitData) : pProxyCommitData(pProxyCommitData) {
 		coreMap.insert(allKeys, RangeLockStateSet());
 	}
 
-	bool pendingRequest() const { return currentRangeLockStartKey.present(); }
+	~RangeLock() override = default;
+
+	bool pendingRequest() const override { return currentRangeLockStartKey.present(); }
 
 	void initKeyPoint(const Key& key, const Value& value) {
 		ASSERT(pProxyCommitData != nullptr && pProxyCommitData->rangeLockEnabled());
@@ -401,14 +403,14 @@ struct RangeLock {
 		return;
 	}
 
-	void setPendingRequest(const Key& startKey, const RangeLockStateSet& lockSetState) {
+	void setPendingRequest(const Key& startKey, const RangeLockStateSet& lockSetState) override {
 		ASSERT(pProxyCommitData != nullptr && pProxyCommitData->rangeLockEnabled());
 		ASSERT(!pendingRequest());
 		currentRangeLockStartKey = std::make_pair(startKey, lockSetState);
 		return;
 	}
 
-	void consumePendingRequest(const Key& endKey) {
+	void consumePendingRequest(const Key& endKey) override {
 		ASSERT(pProxyCommitData != nullptr && pProxyCommitData->rangeLockEnabled());
 		ASSERT(pendingRequest());
 		ASSERT(endKey <= normalKeys.end);
@@ -452,5 +454,23 @@ struct RangeLock {
 	ProxyCommitData* const pProxyCommitData;
 };
 
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_PROXYCOMMITDATA_H
+inline ApplyMetadataProxyContext ProxyCommitData::getApplyMetadataProxyContext() {
+	return { .dbgid = dbgid,
+		     .txnStateStore = txnStateStore,
+		     .vecBackupKeys = &vecBackupKeys,
+		     .keyInfo = &keyInfo,
+		     .uid_applyMutationsData = firstProxy ? &uid_applyMutationsData : nullptr,
+		     .commit = commit,
+		     .cx = cx,
+		     .committedVersion = &committedVersion,
+		     .storageCache = &storageCache,
+		     .tag_popped = &tag_popped,
+		     .tssMapping = &tssMapping,
+		     .commitProxyIndex = commitProxyIndex,
+		     .acsBuilder = acsBuilder,
+		     .epoch = epoch,
+		     // applyMetadataMutations() only borrows the interface during the
+		     // synchronous call, so there is no need to propagate shared
+		     // ownership through the context object.
+		     .rangeLock = rangeLock.get() };
+}
diff --git a/fdbserver/commitproxy/include/fdbserver/commitproxy/CommitProxyServer.h b/fdbserver/commitproxy/include/fdbserver/commitproxy/CommitProxyServer.h
new file mode 100644
index 00000000000..f2b27f74877
--- /dev/null
+++ b/fdbserver/commitproxy/include/fdbserver/commitproxy/CommitProxyServer.h
@@ -0,0 +1,32 @@
+/*
+ * CommitProxyServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/CommitProxyInterface.h"
+#include "flow/flow.h"
+
+struct InitializeCommitProxyRequest;
+struct ServerDBInfo;
+
+Future<Void> commitProxyServer(CommitProxyInterface proxy,
+                               InitializeCommitProxyRequest req,
+                               Reference<AsyncVar<ServerDBInfo> const> db,
+                               std::string whitelistBinPaths);
diff --git a/fdbserver/consistencyscan/CMakeLists.txt b/fdbserver/consistencyscan/CMakeLists.txt
new file mode 100644
index 00000000000..b6474d84675
--- /dev/null
+++ b/fdbserver/consistencyscan/CMakeLists.txt
@@ -0,0 +1,12 @@
+fdb_find_sources(FDBSERVER_CONSISTENCYSCAN_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_consistencyscan SRCS ${FDBSERVER_CONSISTENCYSCAN_SRCS})
+add_fdbserver_link_test(fdbserver_consistencyscanlinktest
+  fdbserver_consistencyscan
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_consistencyscan)
+target_include_directories(fdbserver_consistencyscan
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(fdbserver_consistencyscan PRIVATE fdbserver_core)
diff --git a/fdbserver/ConsistencyScan.actor.cpp b/fdbserver/consistencyscan/ConsistencyScan.cpp
similarity index 75%
rename from fdbserver/ConsistencyScan.actor.cpp
rename to fdbserver/consistencyscan/ConsistencyScan.cpp
index f2fa12981f1..b8232710f54 100644
--- a/fdbserver/ConsistencyScan.actor.cpp
+++ b/fdbserver/consistencyscan/ConsistencyScan.cpp
@@ -1,5 +1,5 @@
 /*
- * ConsistencyScan.actor.cpp
+ * ConsistencyScan.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,7 +23,9 @@
 #include "fdbclient/json_spirit/json_spirit_value.h"
 #include "fdbclient/json_spirit/json_spirit_writer_options.h"
 #include "fdbclient/json_spirit/json_spirit_writer_template.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/consistencyscan/ConsistencyScan.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/IRandom.h"
 #include "flow/IndexedSet.h"
 #include "fdbrpc/FailureMonitor.h"
@@ -32,19 +34,19 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbclient/DataDistributionConfig.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbclient/TagThrottle.h"
+#include "fdbclient/DataDistributionConfig.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/StorageMetrics.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/ShardSizing.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/QuietDatabase.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/Trace.h"
-#include "fdbserver/QuietDatabase.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 #define DEBUG_SCAN_PROGRESS false
 
@@ -89,18 +91,18 @@ struct ConsistencyScanMemoryState : public ReferenceCounted<ConsistencyScanMemor
 
 // TODO: test the test and write a canary key that the storage servers intentionally get wrong
 // Get database KV bytes size from Status JSON at cluster.data.total_kv_size_bytes
-ACTOR Future<Void> pollDatabaseSize(Reference<ConsistencyScanMemoryState> memState, double interval) {
-	loop {
-		loop {
+Future<Void> pollDatabaseSize(Reference<ConsistencyScanMemoryState> memState, double interval) {
+	while (true) {
+		while (true) {
 			if (memState->dbInfo->get().distributor.present()) {
 				break;
 			}
-			wait(memState->dbInfo->onChange());
+			co_await memState->dbInfo->onChange();
 		}
 
 		try {
-			std::vector<WorkerDetails> workers = wait(getWorkers(memState->dbInfo));
-			state Optional<WorkerInterface> ddWorkerInterf;
+			std::vector<WorkerDetails> workers = co_await getWorkers(memState->dbInfo);
+			Optional<WorkerInterface> ddWorkerInterf;
 			// dbInfo could have changed while getting workers
 			if (memState->dbInfo->get().distributor.present()) {
 				for (int i = 0; i < workers.size(); i++) {
@@ -116,15 +118,15 @@ ACTOR Future<Void> pollDatabaseSize(Reference<ConsistencyScanMemoryState> memSta
 
 			if (ddWorkerInterf.present()) {
 				++memState->stats.databasePollSuccesses;
-				TraceEventFields md = wait(timeoutError(
-				    ddWorkerInterf.get().eventLogRequest.getReply(EventLogRequest("DDTrackerStats"_sr)), 5.0));
+				TraceEventFields md = co_await timeoutError(
+				    ddWorkerInterf.get().eventLogRequest.getReply(EventLogRequest("DDTrackerStats"_sr)), 5.0);
 				memState->databaseSize.set(md.getInt64("TotalSizeBytes"));
 				CODE_PROBE(true, "Consistency Scan got DB size from DD Worker");
 
 				TraceEvent("ConsistencyScan_DatabaseSize", memState->csId)
 				    .detail("EstimatedSize", memState->databaseSize.get());
 
-				wait(delay(interval));
+				co_await delay(interval);
 			}
 
 		} catch (Error& e) {
@@ -135,27 +137,26 @@ ACTOR Future<Void> pollDatabaseSize(Reference<ConsistencyScanMemoryState> memSta
 			++memState->stats.databasePollErrors;
 		}
 
-		wait(delay(5.0));
+		co_await delay(5.0);
 	}
 }
 
-ACTOR Future<std::vector<StorageServerInterface>> loadShardInterfaces(Reference<ConsistencyScanMemoryState> memState,
-                                                                      Reference<ReadYourWritesTransaction> tr,
-                                                                      Future<RangeResult> readShardBoundaries,
-                                                                      Database db) {
-	state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+Future<std::vector<StorageServerInterface>> loadShardInterfaces(Reference<ReadYourWritesTransaction> tr,
+                                                                Future<RangeResult> readShardBoundaries,
+                                                                Database db) {
+	RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-	state RangeResult shardBoundaries = wait(readShardBoundaries);
+	RangeResult shardBoundaries = co_await readShardBoundaries;
 	ASSERT(shardBoundaries.size() == 2);
 
-	state std::vector<UID> sourceStorageServers;
-	state std::vector<UID> destStorageServers;
+	std::vector<UID> sourceStorageServers;
+	std::vector<UID> destStorageServers;
 
 	// FIXME: cache storage server interfaces in and/or tag list in memState? then have to deal with invalidating cache
 	// though
 	decodeKeyServersValue(UIDtoTagMap, shardBoundaries[0].value, sourceStorageServers, destStorageServers, true);
 
-	state std::unordered_map<UID, Tag> storageServerToTagMap;
+	std::unordered_map<UID, Tag> storageServerToTagMap;
 	if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
 		storageServerToTagMap.reserve(UIDtoTagMap.size());
 		for (auto& it : UIDtoTagMap) {
@@ -164,18 +165,18 @@ ACTOR Future<std::vector<StorageServerInterface>> loadShardInterfaces(Reference<
 	}
 
 	// if shard move in progress, use destination
-	state std::vector<UID> storageServers = (!destStorageServers.empty()) ? destStorageServers : sourceStorageServers;
+	std::vector<UID> storageServers = (!destStorageServers.empty()) ? destStorageServers : sourceStorageServers;
 
-	state std::vector<Future<Optional<Value>>> serverListEntries;
+	std::vector<Future<Optional<Value>>> serverListEntries;
 	serverListEntries.reserve(storageServers.size());
-	state std::vector<StorageServerInterface> storageServerInterfaces;
+	std::vector<StorageServerInterface> storageServerInterfaces;
 	storageServerInterfaces.reserve(storageServers.size());
 
 	for (auto id : storageServers) {
 		serverListEntries.push_back(tr->get(serverListKeyFor(id)));
 	}
 
-	state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+	std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 	for (auto& it : serverListValues) {
 		ASSERT(it.present());
 		storageServerInterfaces.push_back(decodeServerListValue(it.get()));
@@ -193,21 +194,21 @@ ACTOR Future<std::vector<StorageServerInterface>> loadShardInterfaces(Reference<
 		}
 	}
 
-	return storageServerInterfaces;
+	co_return storageServerInterfaces;
 }
 
 // returns error count
-ACTOR Future<int> consistencyCheckReadData(UID myId,
-                                           Database cx,
-                                           KeyRange range,
-                                           Version version,
-                                           std::vector<StorageServerInterface>* storageServerInterfaces,
-                                           std::vector<Future<ErrorOr<GetKeyValuesReply>>>* keyValueFutures,
-                                           Optional<int>* firstValidServer,
-                                           int64_t* totalReadAmount,
-                                           Optional<Version> consistencyCheckStartVersion) {
+Future<int> consistencyCheckReadData(UID myId,
+                                     Database cx,
+                                     KeyRange range,
+                                     Version version,
+                                     std::vector<StorageServerInterface>* storageServerInterfaces,
+                                     std::vector<Future<ErrorOr<GetKeyValuesReply>>>* keyValueFutures,
+                                     Optional<int>* firstValidServer,
+                                     int64_t* totalReadAmount,
+                                     Optional<Version> consistencyCheckStartVersion) {
 	ASSERT(!range.empty());
-	state GetKeyValuesRequest req;
+	GetKeyValuesRequest req;
 	req.begin = firstGreaterOrEqual(range.begin);
 	req.end = firstGreaterOrEqual(range.end);
 	req.limit = 1e4;
@@ -242,8 +243,7 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 
 	// Try getting the entries in the specified range
 
-	state int j = 0;
-	for (j = 0; j < storageServerInterfaces->size(); j++) {
+	for (int j = 0; j < storageServerInterfaces->size(); j++) {
 		resetReply(req);
 		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
 			cx->getLatestCommitVersion((*storageServerInterfaces)[j], req.version, req.ssLatestCommitVersions);
@@ -251,19 +251,20 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 		keyValueFutures->push_back((*storageServerInterfaces)[j].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 	}
 
-	wait(waitForAll(*keyValueFutures));
+	co_await waitForAll(*keyValueFutures);
 
 	bool expectInjected =
 	    storageServerInterfaces->size() > 1 && g_network->isSimulated() && consistencyCheckStartVersion.present() &&
-	    g_simulator->consistencyScanState == ISimulator::SimConsistencyScanState::Enabled_InjectCorruption &&
-	    g_simulator->consistencyScanCorruptRequestKey.present() && g_simulator->consistencyScanCorruptor.present() &&
-	    g_simulator->consistencyScanCorruptor.get().first == myId &&
-	    range.contains(g_simulator->consistencyScanCorruptRequestKey.get());
+	    fdbSimulationPolicyState().consistencyScanState == FDBSimConsistencyScanState::Enabled_InjectCorruption &&
+	    fdbSimulationPolicyState().consistencyScanCorruptRequestKey.present() &&
+	    fdbSimulationPolicyState().consistencyScanCorruptor.present() &&
+	    fdbSimulationPolicyState().consistencyScanCorruptor.get().first == myId &&
+	    range.contains(fdbSimulationPolicyState().consistencyScanCorruptRequestKey.get());
 	if (expectInjected) {
 		TraceEvent(SevWarnAlways, "ConsistencyScanExpectingInjectedCorruption", myId)
 		    .detail("StorageServers", storageServerInterfaces->size())
-		    .detail("ScanState", g_simulator->consistencyScanState)
-		    .detail("CorruptKey", g_simulator->consistencyScanCorruptRequestKey.get())
+		    .detail("ScanState", fdbSimulationPolicyState().consistencyScanState)
+		    .detail("CorruptKey", fdbSimulationPolicyState().consistencyScanCorruptRequestKey.get())
 		    .detail("ReqRange", range)
 		    .detail("CCStartVersion",
 		            consistencyCheckStartVersion.present() ? consistencyCheckStartVersion.get() : invalidVersion);
@@ -272,12 +273,12 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 	// Read the resulting entries
 	bool allSucceeded = true;
 	bool foundInjected = false;
-	for (j = 0; j < storageServerInterfaces->size(); j++) {
+	for (int j = 0; j < storageServerInterfaces->size(); j++) {
 		ErrorOr<GetKeyValuesReply> rangeResult = (*keyValueFutures)[j].get();
 
 		// Compare the results with other storage servers
 		if (rangeResult.present() && !rangeResult.get().error.present()) {
-			state GetKeyValuesReply current = rangeResult.get();
+			GetKeyValuesReply current = rangeResult.get();
 			DisabledTraceEvent("ConsistencyCheck_GetKeyValuesStream", myId)
 			    .detail("DataSize", current.data.size())
 			    .detail(format("StorageServer%d", j).c_str(), (*storageServerInterfaces)[j].id());
@@ -389,9 +390,9 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 
 					bool isTss = (*storageServerInterfaces)[j].isTss() ||
 					             (*storageServerInterfaces)[firstValidServer->get()].isTss();
-					bool isExpectedTSSMismatch = g_network->isSimulated() &&
-					                             g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations &&
-					                             isTss;
+					bool isExpectedTSSMismatch =
+					    g_network->isSimulated() &&
+					    simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch) && isTss;
 					// It's possible that the storage servers are inconsistent in KillRegion
 					// workload where a forced recovery is performed. The killed storage server
 					// in the killed region returned first with a higher version, and a later
@@ -409,23 +410,24 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 
 					if (!isTss && !isFailed && expectInjected) {
 						// Ensure the only corruption we see is the one that was expected to be injected
-						ASSERT(g_simulator->consistencyScanInjectedCorruptionType.present());
-						if (g_simulator->consistencyScanInjectedCorruptionType.get() ==
-						    ISimulator::SimConsistencyScanCorruptionType::FlipMoreFlag) {
+						ASSERT(fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.present());
+						if (fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.get() ==
+						    FDBSimConsistencyScanCorruptionType::FlipMoreFlag) {
 							// only more flag should be different
 							expectInjected = (current.more != reference.more && currentUniques == 0 &&
 							                  referenceUniques == 0 && valueMismatches == 0);
-						} else if (g_simulator->consistencyScanInjectedCorruptionType.get() ==
-						           ISimulator::SimConsistencyScanCorruptionType::AddToEmpty) {
+						} else if (fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.get() ==
+						           FDBSimConsistencyScanCorruptionType::AddToEmpty) {
 							expectInjected =
 							    (current.more == reference.more && current.data.size() + reference.data.size() == 1);
 							if (expectInjected) {
 								KeyValueRef& kv = current.data.empty() ? reference.data[0] : current.data[0];
-								expectInjected = kv.key == g_simulator->consistencyScanCorruptRequestKey.get() &&
-								                 kv.value == "consistencyCheckCorruptValue"_sr;
+								expectInjected =
+								    kv.key == fdbSimulationPolicyState().consistencyScanCorruptRequestKey.get() &&
+								    kv.value == "consistencyCheckCorruptValue"_sr;
 							}
-						} else if (g_simulator->consistencyScanInjectedCorruptionType.get() ==
-						           ISimulator::SimConsistencyScanCorruptionType::RemoveLastRow) {
+						} else if (fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.get() ==
+						           FDBSimConsistencyScanCorruptionType::RemoveLastRow) {
 							expectInjected = (current.more == reference.more && valueMismatches == 0 &&
 							                  currentUniques + referenceUniques == 1 &&
 							                  std::abs(current.data.size() - reference.data.size()) == 1);
@@ -438,8 +440,8 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 									}
 								}
 							}
-						} else if (g_simulator->consistencyScanInjectedCorruptionType.get() ==
-						           ISimulator::SimConsistencyScanCorruptionType::ChangeFirstValue) {
+						} else if (fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.get() ==
+						           FDBSimConsistencyScanCorruptionType::ChangeFirstValue) {
 							expectInjected =
 							    (current.more == reference.more && valueMismatches == 1 && currentUniques == 0 &&
 							     referenceUniques == 0 && valueMismatchKey == current.data[0].key &&
@@ -484,18 +486,19 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 						CODE_PROBE(true, "consistency check detected injected corruption");
 						// we found the injected corruption, clear the state
 						TraceEvent(SevWarnAlways, "ConsistencyScanFoundInjectedCorruption", myId)
-						    .detail("CorruptionType", g_simulator->consistencyScanInjectedCorruptionType.get())
+						    .detail("CorruptionType",
+						            fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.get())
 						    .detail("Version", req.version);
-						g_simulator->updateConsistencyScanState(
-						    ISimulator::SimConsistencyScanState::Enabled_InjectCorruption,
-						    ISimulator::SimConsistencyScanState::Enabled_FoundCorruption);
+						fdbSimulationPolicyState().updateConsistencyScanState(
+						    FDBSimConsistencyScanState::Enabled_InjectCorruption,
+						    FDBSimConsistencyScanState::Enabled_FoundCorruption);
 					}
 
 					if (!isExpectedTSSMismatch) {
 						int issues = currentUniques + referenceUniques + valueMismatches +
 						             ((current.more == reference.more) ? 0 : 1);
 						ASSERT(issues > 0);
-						return issues;
+						co_return issues;
 					}
 
 					if (isFailed) {
@@ -517,8 +520,8 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 			           "consistency check potentially missed injected corruption due to failures",
 			           probe::decoration::rare);
 			TraceEvent(SevInfo, "ConsistencyCheck_MissedCorruptionDueToFailures");
-			g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Enabled_InjectCorruption,
-			                                        ISimulator::SimConsistencyScanState::Enabled_FoundCorruption);
+			fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::Enabled_InjectCorruption,
+			                                                      FDBSimConsistencyScanState::Enabled_FoundCorruption);
 		}
 	}
 
@@ -527,34 +530,32 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
 	    .detail("Version", version)
 	    .detail("FirstValidServer", firstValidServer->present() ? firstValidServer->get() : -1);
 
-	return 0;
+	co_return 0;
 }
 
-ACTOR Future<Void> consistencyScanCore(Database db,
-                                       Reference<ConsistencyScanMemoryState> memState,
-                                       ConsistencyScanState cs) {
+Future<Void> consistencyScanCore(Database db, Reference<ConsistencyScanMemoryState> memState, ConsistencyScanState cs) {
 	TraceEvent("ConsistencyScanCoreStart").log();
 
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	state Reference<SystemTransactionGenerator<DatabaseContext>> systemDB = SystemDBWriteLockedNow(db.getReference());
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
+	Reference<SystemTransactionGenerator<DatabaseContext>> systemDB = SystemDBWriteLockedNow(db.getReference());
 
 	// Infrequently poll the database size to use in speed calculations.
 	// TODO knob and buggify
-	state Future<Void> pollSize = pollDatabaseSize(memState, g_network->isSimulated() ? 15 : 900);
+	Future<Void> pollSize = pollDatabaseSize(memState, g_network->isSimulated() ? 15 : 900);
 
-	state int64_t readRateLimit = 0;
-	state Reference<IRateControl> readRateControl;
+	int64_t readRateLimit = 0;
+	Reference<IRateControl> readRateControl;
 
 	CODE_PROBE(true, "Running Consistency Scan");
 
 	// Main loop, this is safe to restart at any time.
-	loop {
+	while (true) {
 		// The last known config and its read version
-		state ConsistencyScanState::Config config;
-		state Optional<Versionstamp> configVersion;
+		ConsistencyScanState::Config config;
+		Optional<Versionstamp> configVersion;
 
-		state ConsistencyScanState::LifetimeStats statsLifetime;
-		state ConsistencyScanState::RoundStats statsCurrentRound;
+		ConsistencyScanState::LifetimeStats statsLifetime;
+		ConsistencyScanState::RoundStats statsCurrentRound;
 
 		if (DEBUG_SCAN_PROGRESS) {
 			TraceEvent(SevDebug, "ConsistencyScanProgressLoopStart", memState->csId);
@@ -562,15 +563,16 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 
 		// Read/watch the config until the scan is enabled.
 		tr->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				systemDB->setOptions(tr);
 
 				// Get the config and store the cs trigger version so we can detect updates later.
 				// Get the ConsistencyScanState config and the ConsistencyScanState trigger will update when any of its
 				// configuration members change.
-				wait(store(config, cs.config().getD(tr)));
-				wait(store(configVersion, cs.trigger.get(tr)));
+				config = co_await cs.config().getD(tr);
+				configVersion = co_await cs.trigger.get(tr);
 
 				if (DEBUG_SCAN_PROGRESS) {
 					TraceEvent(SevDebug, "ConsistencyScanProgressGotConfig", memState->csId)
@@ -581,8 +583,8 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 				// *owns* these things and will update them but not read them again because nothing else should be
 				// writing them.
 				if (config.enabled) {
-					wait(store(statsLifetime, cs.lifetimeStats().getD(tr)) &&
-					     store(statsCurrentRound, cs.currentRoundStats().getD(tr)));
+					co_await (store(statsLifetime, cs.lifetimeStats().getD(tr)) &&
+					          store(statsCurrentRound, cs.currentRoundStats().getD(tr)));
 
 					// If the current round is complete OR
 					// If the current round has a nonzero start version but it is < minStartVersion
@@ -618,8 +620,8 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 				                      tr->getReadVersion().get() - (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 60 * 60 *
 				                                                    24 * config.roundHistoryDays)));
 
-				state Future<Void> watch = config.enabled ? Void() : cs.trigger.watch(tr);
-				wait(tr->commit());
+				Future<Void> watch = config.enabled ? Void() : cs.trigger.watch(tr);
+				co_await tr->commit();
 
 				if (config.enabled) {
 					break;
@@ -629,12 +631,15 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 					TraceEvent(SevDebug, "ConsistencyScanProgressDisabledWatching", memState->csId);
 				}
 
-				wait(watch);
+				co_await watch;
 				tr->reset();
+				continue;
 			} catch (Error& e) {
-				TraceEvent("ConsistencyScan_MainLoopError", memState->csId).error(e);
-				wait(tr->onError(e));
+				err = e;
 			}
+
+			TraceEvent("ConsistencyScan_MainLoopError", memState->csId).error(err);
+			co_await tr->onError(err);
 		}
 
 		if (DEBUG_SCAN_PROGRESS) {
@@ -643,18 +648,18 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 
 		// Wait for disk usage if we don't know it yet
 		if (memState->databaseSize.get() < 0) {
-			wait(memState->databaseSize.onChange());
+			co_await memState->databaseSize.onChange();
 		}
 
 		if (DEBUG_SCAN_PROGRESS) {
 			TraceEvent(SevDebug, "ConsistencyScanProgressGotDBSize", memState->csId);
 		}
 
-		state bool restartMainLoop = false;
-		state Future<Void> delayBeforeMainLoopRestart = Void();
+		bool restartMainLoop = false;
+		Future<Void> delayBeforeMainLoopRestart = Void();
 
 		// Scan loop.  Each iteration makes incremental, durable progress on the scan.
-		loop {
+		while (true) {
 			if (DEBUG_SCAN_PROGRESS) {
 				TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopStart", memState->csId);
 			}
@@ -691,33 +696,34 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 				if (DEBUG_SCAN_PROGRESS) {
 					TraceEvent("ConsistencyScan_ChangeRate", memState->csId).detail("RateBytes", readRateLimit);
 				}
-				readRateControl = Reference<IRateControl>(new SpeedLimit(readRateLimit, 1));
+				readRateControl = makeReference<SpeedLimit>(readRateLimit, 1);
 				memState->stats.targetRate = configuredRate;
 			}
 
 			// This will store how many total bytes we read from all replicas in this loop iteration, including retries,
 			// because that's what our bandwidth limiting should be based on. We will wait on the rate control *after*
 			// reading using the actual amount that we read
-			state int totalReadBytesFromStorageServers = 0;
+			int totalReadBytesFromStorageServers = 0;
 
 			// We only want to update initialRoundState with *durable* progress, so if the loop below retries it must
 			// start from the same initial state, so save it here and restore it at the start of the transaction.
-			state ConsistencyScanState::RoundStats savedCurrentRoundState = statsCurrentRound;
+			ConsistencyScanState::RoundStats savedCurrentRoundState = statsCurrentRound;
 
 			tr->reset();
-			loop {
+			while (true) {
+				Error err;
 				try {
 					++memState->stats.scanLoops;
 					statsCurrentRound = savedCurrentRoundState;
 					systemDB->setOptions(tr);
 
 					// The shard boundaries for the shard that lastEndKey is in
-					state RangeResult shardBoundaries;
-					state std::vector<StorageServerInterface> storageServerInterfaces;
+					RangeResult shardBoundaries;
+					std::vector<StorageServerInterface> storageServerInterfaces;
 					// The consistency scan config update version
-					state Optional<Versionstamp> csVersion;
+					Optional<Versionstamp> csVersion;
 					// The range config for the range that we're about to read
-					state Optional<ConsistencyScanState::RangeConfigMap::RangeValue> configRange;
+					Optional<ConsistencyScanState::RangeConfigMap::RangeValue> configRange;
 
 					if (DEBUG_SCAN_PROGRESS) {
 						TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopLoadingState", memState->csId);
@@ -731,10 +737,10 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 					                 firstGreaterThan(allKeys.end.withPrefix(keyServersPrefix)),
 					                 limits,
 					                 Snapshot::True);
-					wait(store(shardBoundaries, readShardBoundaries) &&
-					     store(storageServerInterfaces, loadShardInterfaces(memState, tr, readShardBoundaries, db)) &&
-					     store(csVersion, cs.trigger.get(tr)) &&
-					     store(configRange, cs.rangeConfig().getRangeForKey(tr, statsCurrentRound.lastEndKey)));
+					co_await (store(shardBoundaries, readShardBoundaries) &&
+					          store(storageServerInterfaces, loadShardInterfaces(tr, readShardBoundaries, db)) &&
+					          store(csVersion, cs.trigger.get(tr)) &&
+					          store(configRange, cs.rangeConfig().getRangeForKey(tr, statsCurrentRound.lastEndKey)));
 
 					if (DEBUG_SCAN_PROGRESS) {
 						TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopLoadedState", memState->csId);
@@ -750,12 +756,11 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 						break;
 					}
 
-					state Key beginKey = shardBoundaries[0].key.removePrefix(keyServersPrefix);
+					Key beginKey = shardBoundaries[0].key.removePrefix(keyServersPrefix);
 					if (beginKey < statsCurrentRound.lastEndKey) {
 						beginKey = statsCurrentRound.lastEndKey;
 					}
-					state KeyRange targetRange =
-					    KeyRangeRef(beginKey, shardBoundaries[1].key.removePrefix(keyServersPrefix));
+					KeyRange targetRange = KeyRangeRef(beginKey, shardBoundaries[1].key.removePrefix(keyServersPrefix));
 
 					bool scanRange = true;
 					statsCurrentRound.lastProgressVersion = tr->getReadVersion().get();
@@ -794,32 +799,32 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 					}
 
 					// If we've skipped to the end of all records then there are no more records to scan
-					state bool noMoreRecords = statsCurrentRound.lastEndKey == allKeys.end;
-					state Optional<Error> failedRequest;
+					bool noMoreRecords = statsCurrentRound.lastEndKey == allKeys.end;
+					Optional<Error> failedRequest;
 
 					if (scanRange) {
 						// inject corruption if desired in simulation
 						if (g_network->isSimulated() &&
-						    g_simulator->consistencyScanState == ISimulator::SimConsistencyScanState::Enabled &&
-						    g_simulator->doInjectConsistencyScanCorruption.present() &&
-						    g_simulator->doInjectConsistencyScanCorruption.get() &&
+						    fdbSimulationPolicyState().consistencyScanState == FDBSimConsistencyScanState::Enabled &&
+						    fdbSimulationPolicyState().doInjectConsistencyScanCorruption.present() &&
+						    fdbSimulationPolicyState().doInjectConsistencyScanCorruption.get() &&
 						    (deterministicRandom()->random01() < 0.1 || targetRange.end == allKeys.end)) {
 							if (storageServerInterfaces.size() > 1) {
 								// sometimes do the first key in the range, sometimes try to do a later key in the range
 								Key corruptKey = targetRange.begin;
-								if (deterministicRandom()->coinflip() && targetRange.end.size() > 0) {
+								if (deterministicRandom()->coinflip() && !targetRange.end.empty()) {
 									Key corruptKey2 = targetRange.end.substr(0, targetRange.end.size() - 1);
 									if (corruptKey2 > corruptKey) {
 										corruptKey = corruptKey2;
 									}
 								}
-								g_simulator->consistencyScanCorruptRequestKey = corruptKey;
+								fdbSimulationPolicyState().consistencyScanCorruptRequestKey = corruptKey;
 								std::pair<UID, NetworkAddress> meCorruptor = { memState->csId,
 									                                           g_network->getLocalAddress() };
-								g_simulator->consistencyScanCorruptor = meCorruptor;
-								g_simulator->updateConsistencyScanState(
-								    ISimulator::SimConsistencyScanState::Enabled,
-								    ISimulator::SimConsistencyScanState::Enabled_InjectCorruption);
+								fdbSimulationPolicyState().consistencyScanCorruptor = meCorruptor;
+								fdbSimulationPolicyState().updateConsistencyScanState(
+								    FDBSimConsistencyScanState::Enabled,
+								    FDBSimConsistencyScanState::Enabled_InjectCorruption);
 								TraceEvent(
 								    SevWarnAlways, "ConsistencyScanProgressScanRangeInjectCorruption", memState->csId)
 								    .detail("StartKey", targetRange.begin)
@@ -828,9 +833,9 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 							} else {
 								// since we can't notice the corruption with only 1 server, just skip it
 								CODE_PROBE(true, "skipping consistency scan corruption check because only 1 replica");
-								g_simulator->updateConsistencyScanState(
-								    ISimulator::SimConsistencyScanState::Enabled,
-								    ISimulator::SimConsistencyScanState::Enabled_FoundCorruption);
+								fdbSimulationPolicyState().updateConsistencyScanState(
+								    FDBSimConsistencyScanState::Enabled,
+								    FDBSimConsistencyScanState::Enabled_FoundCorruption);
 								TraceEvent(SevWarnAlways,
 								           "ConsistencyScanProgressScanRangeSkipInjectCorruption",
 								           memState->csId)
@@ -844,11 +849,11 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 
 						// Stats for *this unit of durable progress only*, which will be committed atomically with the
 						// endKey update. Total bytes from all replicas read
-						state int64_t replicatedBytesRead = 0;
+						int64_t replicatedBytesRead = 0;
 						// Logical KV bytes scanned regardless of how many replicas exist of each key
-						state int64_t logicalBytesRead = 0;
+						int64_t logicalBytesRead = 0;
 						// Number of key differences found (essentially a count of comparisons that failed)
-						state int errors = 0;
+						int errors = 0;
 						// FIXME: break the round after some amount of time or number of failed requests regardless of
 						// progress
 
@@ -862,20 +867,20 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 						// TODO: Also read from blob as one of the replicas?  If so, maybe separately track blob errors
 						// where blob disagrees from the other replicas, which would also be a general ++error
 
-						loop {
-							state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
-							state Optional<int> firstValidServer;
+						while (true) {
+							std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+							Optional<int> firstValidServer;
 							memState->stats.requests += storageServerInterfaces.size();
-							state int64_t replicatedBytesReadThisLoop = 0;
-							int newErrors = wait(consistencyCheckReadData(memState->csId,
-							                                              db,
-							                                              targetRange,
-							                                              tr->getReadVersion().get(),
-							                                              &storageServerInterfaces,
-							                                              &keyValueFutures,
-							                                              &firstValidServer,
-							                                              &replicatedBytesReadThisLoop,
-							                                              statsCurrentRound.startVersion));
+							int64_t replicatedBytesReadThisLoop = 0;
+							int newErrors = co_await consistencyCheckReadData(memState->csId,
+							                                                  db,
+							                                                  targetRange,
+							                                                  tr->getReadVersion().get(),
+							                                                  &storageServerInterfaces,
+							                                                  &keyValueFutures,
+							                                                  &firstValidServer,
+							                                                  &replicatedBytesReadThisLoop,
+							                                                  statsCurrentRound.startVersion);
 							errors += newErrors;
 							memState->stats.inconsistencies += newErrors;
 
@@ -905,7 +910,7 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 								} else {
 									VectorRef<KeyValueRef> result =
 									    keyValueFutures[firstValidServer.get()].get().get().data;
-									ASSERT(result.size() > 0);
+									ASSERT(!result.empty());
 									statsCurrentRound.lastEndKey = keyAfter(result.back().key);
 									targetRange = KeyRangeRef(statsCurrentRound.lastEndKey, targetRange.end);
 									if (targetRange.empty()) {
@@ -922,7 +927,7 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 									if (i == firstValidServer.get()) {
 										logicalBytesRead += rangeResult.data.expectedSize();
 									}
-									Key storageNextKey = (rangeResult.more && rangeResult.data.size() > 0)
+									Key storageNextKey = (rangeResult.more && !rangeResult.data.empty())
 									                         ? keyAfter(rangeResult.data.back().key)
 									                         : targetRange.end;
 									if (storageNextKey > nextKey) {
@@ -963,7 +968,7 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 							ratio = std::max(0.0, std::min(1.0, ratio));
 							int sleepBytes = (int)(totalReadBytesFromStorageServers * ratio);
 							totalReadBytesFromStorageServers -= sleepBytes;
-							wait(readRateControl->getAllowance(sleepBytes));
+							co_await readRateControl->getAllowance(sleepBytes);
 						}
 
 						statsCurrentRound.errorCount += errors;
@@ -1011,20 +1016,22 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 					//            json_spirit::write_string(json_spirit::mValue(statsCurrentRound.toJSON())));
 					break;
 				} catch (Error& e) {
-					TraceEvent("ConsistencyScan_ScanLoopError", memState->csId).errorUnsuppressed(e);
-					if (e.code() == error_code_future_version || e.code() == error_code_wrong_shard_server ||
-					    e.code() == error_code_all_alternatives_failed || e.code() == error_code_process_behind) {
-						// sleep for a bit extra
-						totalReadBytesFromStorageServers += 100000;
-						break;
-					} else {
-						try {
-							wait(tr->onError(e));
-						} catch (Error& e2) {
-							TraceEvent(SevError, "ConsistencyScan_UnexpectedError", memState->csId).error(e2);
-							throw e2;
-						}
-					}
+					err = e;
+				}
+
+				TraceEvent("ConsistencyScan_ScanLoopError", memState->csId).errorUnsuppressed(err);
+				if (err.code() == error_code_future_version || err.code() == error_code_wrong_shard_server ||
+				    err.code() == error_code_all_alternatives_failed || err.code() == error_code_process_behind) {
+					// sleep for a bit extra
+					totalReadBytesFromStorageServers += 100000;
+					break;
+				}
+
+				try {
+					co_await tr->onError(err);
+				} catch (Error& e2) {
+					TraceEvent(SevError, "ConsistencyScan_UnexpectedError", memState->csId).error(e2);
+					throw e2;
 				}
 			} // Transaction retry loop for making one increment of progress
 
@@ -1037,14 +1044,15 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 					    .detail("ProgressKey", statsCurrentRound.lastEndKey);
 				}
 				statsCurrentRound.lastProgressTime = now();
-				loop {
+				while (true) {
 					if (DEBUG_SCAN_PROGRESS) {
 						TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopSaveStart", memState->csId)
 						    .detail("ProgressKey", statsCurrentRound.lastEndKey);
 					}
+					Error err;
 					try {
 						systemDB->setOptions(tr);
-						Optional<Versionstamp> csVersion2 = wait(cs.trigger.get(tr));
+						Optional<Versionstamp> csVersion2 = co_await cs.trigger.get(tr);
 						if (csVersion2 != configVersion) {
 							if (DEBUG_SCAN_PROGRESS) {
 								TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopVersionMismatch2", memState->csId);
@@ -1055,7 +1063,7 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 						cs.currentRoundStats().set(tr, statsCurrentRound);
 						cs.lifetimeStats().set(tr, statsLifetime);
 
-						wait(tr->commit());
+						co_await tr->commit();
 						if (DEBUG_SCAN_PROGRESS) {
 							TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopDurable", memState->csId)
 							    .detail("BytesRead", statsCurrentRound.logicalBytesScanned)
@@ -1063,13 +1071,15 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 						}
 						break;
 					} catch (Error& e) {
-						if (DEBUG_SCAN_PROGRESS) {
-							TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopSaveError", memState->csId)
-							    .errorUnsuppressed(e)
-							    .detail("ProgressKey", statsCurrentRound.lastEndKey);
-						}
-						wait(tr->onError(e));
+						err = e;
+					}
+
+					if (DEBUG_SCAN_PROGRESS) {
+						TraceEvent(SevDebug, "ConsistencyScanProgressScanLoopSaveError", memState->csId)
+						    .errorUnsuppressed(err)
+						    .detail("ProgressKey", statsCurrentRound.lastEndKey);
 					}
+					co_await tr->onError(err);
 				}
 			} else {
 				CODE_PROBE(true, "consistency scan loop no progress");
@@ -1080,7 +1090,7 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 			}
 
 			// Wait for the rate control to generate enough budget to match what we read.
-			wait(readRateControl->getAllowance(totalReadBytesFromStorageServers));
+			co_await readRateControl->getAllowance(totalReadBytesFromStorageServers);
 
 			if (DEBUG_SCAN_PROGRESS) {
 				TraceEvent(SevDebug, "ConsistencyScanProgressRateLimited", memState->csId);
@@ -1091,7 +1101,7 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 					TraceEvent(SevDebug, "ConsistencyScanProgressWait", memState->csId);
 				}
 				memState->stats.waitingBetweenRounds = true;
-				wait(delayBeforeMainLoopRestart);
+				co_await delayBeforeMainLoopRestart;
 				memState->stats.waitingBetweenRounds = false;
 				break;
 			}
@@ -1099,58 +1109,58 @@ ACTOR Future<Void> consistencyScanCore(Database db,
 	} // Main loop
 }
 
-ACTOR Future<Void> sometimesRandomlyClearStatsInSim(Database db, Reference<ConsistencyScanMemoryState> memState) {
+Future<Void> sometimesRandomlyClearStatsInSim(Database db, Reference<ConsistencyScanMemoryState> memState) {
 	ASSERT(g_network->isSimulated());
 
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	state ConsistencyScanState cs;
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
+	ConsistencyScanState cs;
 
 	if (BUGGIFY_WITH_PROB(0.1) && !g_simulator->speedUpSimulation) {
 		TraceEvent("ConsistencyScan_RandomStatClearWaiting", memState->csId).log();
-		wait(delay(deterministicRandom()->randomInt(1, 60)));
+		co_await delay(deterministicRandom()->randomInt(1, 60));
 
 		TraceEvent("ConsistencyScan_RandomStatClearing", memState->csId).log();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				SystemDBWriteLockedNow(db.getReference())->setOptions(tr);
-				wait(cs.clearStats(tr));
-				wait(tr->commit());
+				co_await cs.clearStats(tr);
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+
+			co_await tr->onError(err);
 		}
 
 		TraceEvent("ConsistencyScan_RandomStatCleared", memState->csId).log();
 	}
-
-	return Void();
 }
 
 void resetSimCorruptionCheckOnDeath(Reference<ConsistencyScanMemoryState> memState) {
 	if (!g_network->isSimulated()) {
 		return;
 	}
-	if (g_simulator->consistencyScanCorruptor.present() &&
-	    g_simulator->consistencyScanCorruptor.get().first == memState->csId) {
+	if (fdbSimulationPolicyState().consistencyScanCorruptor.present() &&
+	    fdbSimulationPolicyState().consistencyScanCorruptor.get().first == memState->csId) {
 		TraceEvent("ConsistencyScan_ResetCorruptionOnDeath");
 		CODE_PROBE(
 		    true, "Consistency Scan skipped corruption check because scan died in the middle", probe::decoration::rare);
-		g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Enabled_InjectCorruption,
-		                                        ISimulator::SimConsistencyScanState::Enabled_FoundCorruption);
+		fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::Enabled_InjectCorruption,
+		                                                      FDBSimConsistencyScanState::Enabled_FoundCorruption);
 	}
 }
 
-ACTOR Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	state Database db = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
-	state ActorCollection actors;
-	state Reference<ConsistencyScanMemoryState> memState =
-	    makeReference<ConsistencyScanMemoryState>(dbInfo, csInterf.id());
+Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	Database db = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
+	ActorCollection actors;
+	auto memState = makeReference<ConsistencyScanMemoryState>(dbInfo, csInterf.id());
 
 	TraceEvent("ConsistencyScan_Start", csInterf.id()).log();
 	actors.add(traceRole(Role::CONSISTENCYSCAN, csInterf.id()));
 	actors.add(waitFailureServer(csInterf.waitFailure.getFuture()));
-	state Future<Void> core = consistencyScanCore(db, memState, ConsistencyScanState());
+	Future<Void> core = consistencyScanCore(db, memState, ConsistencyScanState());
 
 	if (g_network->isSimulated()) {
 		actors.add(sometimesRandomlyClearStatsInSim(db, memState));
@@ -1158,41 +1168,32 @@ ACTOR Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<
 
 		// Check if previous consistency scan was terminated without getting actor_cancelled in the middle of a
 		// corruption injection
-		if (g_simulator->consistencyScanCorruptor.present() &&
-		    csInterf.id() != g_simulator->consistencyScanCorruptor.get().first &&
-		    g_simulator->getProcessByAddress(g_simulator->consistencyScanCorruptor.get().second)->failed &&
-		    g_simulator->consistencyScanState == ISimulator::SimConsistencyScanState::Enabled_InjectCorruption) {
+		if (fdbSimulationPolicyState().consistencyScanCorruptor.present() &&
+		    csInterf.id() != fdbSimulationPolicyState().consistencyScanCorruptor.get().first &&
+		    g_simulator->getProcessByAddress(fdbSimulationPolicyState().consistencyScanCorruptor.get().second)
+		        ->failed &&
+		    fdbSimulationPolicyState().consistencyScanState == FDBSimConsistencyScanState::Enabled_InjectCorruption) {
 			TraceEvent("ConsistencyScan_ResetCorruptionOnPreviousScanDeath");
 			// noop if state isn't already Enabled_InjectCorruption
-			g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Enabled_InjectCorruption,
-			                                        ISimulator::SimConsistencyScanState::Enabled_FoundCorruption);
+			fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::Enabled_InjectCorruption,
+			                                                      FDBSimConsistencyScanState::Enabled_FoundCorruption);
 		}
 	}
 
-	loop {
-		try {
-			loop choose {
-				when(wait(core)) {
-					// This actor never returns so the only way out is throwing an exception.
-					ASSERT(false);
-				}
-				when(HaltConsistencyScanRequest req = waitNext(csInterf.haltConsistencyScan.getFuture())) {
-					resetSimCorruptionCheckOnDeath(memState);
-					req.reply.send(Void());
-					core = Void();
-					TraceEvent("ConsistencyScan_Halted", csInterf.id()).detail("ReqID", req.requesterID);
-					return Void();
-				}
-				when(wait(actors.getResult())) {
-					ASSERT(false);
-					throw internal_error();
-				}
-			}
-		} catch (Error& err) {
-			resetSimCorruptionCheckOnDeath(memState);
-			TraceEvent("ConsistencyScan_Error", csInterf.id()).errorUnsuppressed(err);
-			throw;
-		}
+	try {
+		auto res = co_await race(core, csInterf.haltConsistencyScan.getFuture(), actors.getResult());
+		// core or actors.getResult() never return so the only way out is throwing an exception.
+		ASSERT_EQ(res.index(), 1);
+		HaltConsistencyScanRequest req = std::get<1>(std::move(res));
+
+		resetSimCorruptionCheckOnDeath(memState);
+		req.reply.send(Void());
+		core = Void();
+		TraceEvent("ConsistencyScan_Halted", csInterf.id()).detail("ReqID", req.requesterID);
+	} catch (Error& err) {
+		resetSimCorruptionCheckOnDeath(memState);
+		TraceEvent("ConsistencyScan_Error", csInterf.id()).errorUnsuppressed(err);
+		throw;
 	}
 }
 
@@ -1201,16 +1202,19 @@ ACTOR Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<
 // It is only used by workloads/ConsistencyCheck
 
 // Gets a version at which to read from the storage servers
-ACTOR Future<Version> getVersion(Database cx) {
-	loop {
-		state Transaction tr(cx);
+Future<Version> getStorageServerReadVersion(Database cx) {
+	while (true) {
+		Transaction tr(cx);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		Error err;
 		try {
-			Version version = wait(tr.getReadVersion());
-			return version;
+			Version version = co_await tr.getReadVersion();
+			co_return version;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+
+		co_await tr.onError(err);
 	}
 }
 
@@ -1228,26 +1232,25 @@ void testFailure(std::string message, bool performQuiescentChecks, bool* success
 // Get a list of storage servers(persisting keys within range "kr") from the master and compares them with the
 // TLogs. If this is a quiescent check, then each commit proxy needs to respond, otherwise only one needs to
 // respond. Returns false if there is a failure (in this case, keyServersPromise will never be set)
-ACTOR Future<bool> getKeyServers(
+Future<bool> getKeyServers(
     Database cx,
     Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
     KeyRangeRef kr,
     bool performQuiescentChecks,
     bool failureIsError,
     bool* success) {
-	state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
+	std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
 
 	// Try getting key server locations from the master proxies
-	state std::vector<Future<ErrorOr<GetKeyServerLocationsReply>>> keyServerLocationFutures;
-	state Key begin = kr.begin;
-	state Key end = kr.end;
-	state int limitKeyServers = BUGGIFY ? 1 : 100;
-	state Span span(SpanContext(deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64()),
-	                "WL:ConsistencyCheck"_loc);
+	std::vector<Future<ErrorOr<GetKeyServerLocationsReply>>> keyServerLocationFutures;
+	Key begin = kr.begin;
+	Key end = kr.end;
+	int limitKeyServers = BUGGIFY ? 1 : 100;
+	Span span(SpanContext(deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64()),
+	          "WL:ConsistencyCheck"_loc);
 
 	while (begin < end) {
-		state Reference<CommitProxyInfo> commitProxyInfo =
-		    wait(cx->getCommitProxiesFuture(UseProvisionalProxies::False));
+		Reference<CommitProxyInfo> commitProxyInfo = co_await cx->getCommitProxiesFuture(UseProvisionalProxies::False);
 		keyServerLocationFutures.clear();
 		for (int i = 0; i < commitProxyInfo->size(); i++)
 			keyServerLocationFutures.push_back(
@@ -1258,66 +1261,66 @@ ACTOR Future<bool> getKeyServers(
 			            2,
 			            0));
 
-		state bool keyServersInsertedForThisIteration = false;
-		choose {
-			when(wait(waitForAll(keyServerLocationFutures))) {
-				// Read the key server location results
-				for (int i = 0; i < keyServerLocationFutures.size(); i++) {
-					ErrorOr<GetKeyServerLocationsReply> shards = keyServerLocationFutures[i].get();
-
-					// If performing quiescent check, then all master proxies should be reachable.  Otherwise, only
-					// one needs to be reachable
-					if (performQuiescentChecks && !shards.present()) {
-						TraceEvent("ConsistencyCheck_CommitProxyUnavailable")
-						    .error(shards.getError())
-						    .detail("CommitProxyID", commitProxyInfo->getId(i));
-						testFailure("Commit proxy unavailable", performQuiescentChecks, success, failureIsError);
-						return false;
-					}
+		bool keyServersInsertedForThisIteration = false;
 
-					// Get the list of shards if one was returned.  If not doing a quiescent check, we can break if
-					// it is. If we are doing a quiescent check, then we only need to do this for the first shard.
-					if (shards.present() && !keyServersInsertedForThisIteration) {
-						keyServers.insert(keyServers.end(), shards.get().results.begin(), shards.get().results.end());
-						keyServersInsertedForThisIteration = true;
-						begin = shards.get().results.back().first.end;
+		auto res = co_await race(waitForAll(keyServerLocationFutures), cx->onProxiesChanged());
+		if (res.index() == 0) {
 
-						if (!performQuiescentChecks)
-							break;
-					}
-				} // End of For
+			// Read the key server location results
+			for (int i = 0; i < keyServerLocationFutures.size(); i++) {
+				ErrorOr<GetKeyServerLocationsReply> shards = keyServerLocationFutures[i].get();
+
+				// If performing quiescent check, then all master proxies should be reachable.  Otherwise, only
+				// one needs to be reachable
+				if (performQuiescentChecks && !shards.present()) {
+					TraceEvent("ConsistencyCheck_CommitProxyUnavailable")
+					    .error(shards.getError())
+					    .detail("CommitProxyID", commitProxyInfo->getId(i));
+					testFailure("Commit proxy unavailable", performQuiescentChecks, success, failureIsError);
+					co_return false;
+				}
+
+				// Get the list of shards if one was returned.  If not doing a quiescent check, we can break if
+				// it is. If we are doing a quiescent check, then we only need to do this for the first shard.
+				if (shards.present() && !keyServersInsertedForThisIteration) {
+					keyServers.insert(keyServers.end(), shards.get().results.begin(), shards.get().results.end());
+					keyServersInsertedForThisIteration = true;
+					begin = shards.get().results.back().first.end;
+
+					if (!performQuiescentChecks)
+						break;
+				}
 			}
-			when(wait(cx->onProxiesChanged())) {}
-		} // End of choose
+		}
 
 		if (!keyServersInsertedForThisIteration) // Retry the entire workflow
-			wait(delay(1.0));
+			co_await delay(1.0);
 
 	} // End of while
 
 	keyServersPromise.send(keyServers);
-	return true;
+	co_return true;
 }
 
 // Retrieves the locations of all shards in the database
 // Returns false if there is a failure (in this case, keyLocationPromise will never be set)
-ACTOR Future<bool> getKeyLocations(Database cx,
-                                   std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
-                                   Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise,
-                                   bool performQuiescentChecks,
-                                   bool* success) {
-	state Standalone<VectorRef<KeyValueRef>> keyLocations;
-	state Key beginKey = allKeys.begin.withPrefix(keyServersPrefix);
-	state Key endKey = allKeys.end.withPrefix(keyServersPrefix);
-	state int i = 0;
-	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+Future<bool> getKeyLocations(Database cx,
+                             std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
+                             Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise,
+                             bool performQuiescentChecks,
+                             bool* success) {
+	Standalone<VectorRef<KeyValueRef>> keyLocations;
+	Key beginKey = allKeys.begin.withPrefix(keyServersPrefix);
+	Key endKey = allKeys.end.withPrefix(keyServersPrefix);
+	Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
 
 	// If the responses are too big, we may use multiple requests to get the key locations.  Each request begins
 	// where the last left off
-	for (; i < shards.size(); i++) {
+	for (int i = 0; i < shards.size(); i++) {
 		while (beginKey < std::min<KeyRef>(shards[i].first.end, endKey)) {
+			Error err;
 			try {
-				Version version = wait(getVersion(cx));
+				Version version = co_await getStorageServerReadVersion(cx);
 
 				GetKeyValuesRequest req;
 				req.begin = firstGreaterOrEqual(beginKey);
@@ -1328,7 +1331,7 @@ ACTOR Future<bool> getKeyLocations(Database cx,
 				req.tags = TagSet();
 
 				// Try getting the shard locations from the key servers
-				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+				std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
 				for (const auto& kv : shards[i].second) {
 					resetReply(req);
 					if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
@@ -1337,7 +1340,7 @@ ACTOR Future<bool> getKeyLocations(Database cx,
 					keyValueFutures.push_back(kv.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 				}
 
-				wait(waitForAll(keyValueFutures));
+				co_await waitForAll(keyValueFutures);
 
 				int firstValidStorageServer = -1;
 
@@ -1363,7 +1366,7 @@ ACTOR Future<bool> getKeyLocations(Database cx,
 						    .detail("StorageServer1", shards[i].second[firstValidStorageServer].id())
 						    .detail("StorageServer2", shards[i].second[j].id());
 						testFailure("Key servers inconsistent", performQuiescentChecks, success, true);
-						return false;
+						co_return false;
 					}
 				}
 
@@ -1374,7 +1377,7 @@ ACTOR Future<bool> getKeyLocations(Database cx,
 				                std::min<KeyRef>(shards[i].first.end, endKey).removePrefix(keyServersPrefix)),
 				    RangeResultRef(keyValueResponse.data, keyValueResponse.more));
 
-				if (keyValueResponse.data.size() && beginKey == keyValueResponse.data[0].key) {
+				if (!keyValueResponse.data.empty() && beginKey == keyValueResponse.data[0].key) {
 					keyLocations.push_back_deep(keyLocations.arena(), currentLocations[0]);
 				}
 
@@ -1393,33 +1396,35 @@ ACTOR Future<bool> getKeyLocations(Database cx,
 				// If this is the last iteration, then push the allKeys.end KV pair
 				if (beginKey >= endKey)
 					keyLocations.push_back_deep(keyLocations.arena(), currentLocations.end()[-1]);
+				continue;
 			} catch (Error& e) {
-				state Error err = e;
-				wait(onErrorTr.onError(err));
-				TraceEvent("ConsistencyCheck_RetryGetKeyLocations").error(err);
+				err = e;
 			}
+
+			co_await onErrorTr.onError(err);
+			TraceEvent("ConsistencyCheck_RetryGetKeyLocations").error(err);
 		}
 	}
 
 	keyLocationPromise.send(keyLocations);
-	return true;
+	co_return true;
 }
 
 // Retrieves a vector of the storage servers' estimates for the size of a particular shard
 // If a storage server can't be reached, its estimate will be -1
 // If there is an error, then the returned vector will have 0 size
-ACTOR Future<std::pair<std::vector<int64_t>, StorageMetrics>> getStorageSizeEstimate(
+Future<std::pair<std::vector<int64_t>, StorageMetrics>> getStorageSizeEstimate(
     std::vector<StorageServerInterface> storageServers,
     KeyRangeRef shard) {
-	state std::vector<int64_t> estimatedBytes;
-	state StorageMetrics metrics;
+	std::vector<int64_t> estimatedBytes;
+	StorageMetrics metrics;
 
-	state WaitMetricsRequest req;
+	WaitMetricsRequest req;
 	req.keys = shard;
 	req.max.bytes = -1;
 	req.min.bytes = 0;
 
-	state std::vector<Future<ErrorOr<StorageMetrics>>> metricFutures;
+	std::vector<Future<ErrorOr<StorageMetrics>>> metricFutures;
 
 	try {
 		// Check the size of the shard on each storage server
@@ -1429,7 +1434,7 @@ ACTOR Future<std::pair<std::vector<int64_t>, StorageMetrics>> getStorageSizeEsti
 		}
 
 		// Wait for the storage servers to respond
-		wait(waitForAll(metricFutures));
+		co_await waitForAll(metricFutures);
 
 		int firstValidStorageServer = -1;
 
@@ -1477,60 +1482,63 @@ ACTOR Future<std::pair<std::vector<int64_t>, StorageMetrics>> getStorageSizeEsti
 		estimatedBytes.clear();
 	}
 
-	return std::make_pair(estimatedBytes, metrics);
+	co_return std::make_pair(estimatedBytes, metrics);
 }
 
-ACTOR Future<int64_t> getDatabaseSize(Database cx) {
+Future<int64_t> getDatabaseSize(Database cx) {
 	// This is too expensive and probably won't complete fast enough on a real cluster
 	ASSERT(g_network->isSimulated());
 
-	state Transaction tr(cx);
+	Transaction tr(cx);
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			StorageMetrics metrics =
-			    wait(tr.getDatabase()->getStorageMetrics(KeyRangeRef(allKeys.begin, keyServersPrefix), 100000));
-			return metrics.bytes;
+			    co_await tr.getDatabase()->getStorageMetrics(KeyRangeRef(allKeys.begin, keyServersPrefix), 100000);
+			co_return metrics.bytes;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+
+		co_await tr.onError(err);
 	}
 }
 
 // Checks that the data in each shard is the same on each storage server that it resides on.  Also performs some
 // sanity checks on the sizes of shards and storage servers. Returns false if there is a failure
 // TODO: Future optimization: Use streaming reads
-ACTOR Future<Void> checkDataConsistency(Database cx,
-                                        VectorRef<KeyValueRef> keyLocations,
-                                        DatabaseConfiguration configuration,
-                                        std::map<UID, StorageServerInterface> tssMapping,
-                                        bool performQuiescentChecks,
-                                        bool performTSSCheck,
-                                        bool firstClient,
-                                        bool failureIsError,
-                                        int clientId,
-                                        int clientCount,
-                                        bool distributed,
-                                        bool shuffleShards,
-                                        int shardSampleFactor,
-                                        int64_t sharedRandomNumber,
-                                        int64_t repetitions,
-                                        int64_t* bytesReadInPrevRound,
-                                        int restart,
-                                        int64_t maxRate,
-                                        int64_t targetInterval,
-                                        bool* success) {
+Future<Void> checkDataConsistency(Database cx,
+                                  VectorRef<KeyValueRef> keyLocations,
+                                  DatabaseConfiguration configuration,
+                                  std::map<UID, StorageServerInterface> tssMapping,
+                                  bool performQuiescentChecks,
+                                  bool performTSSCheck,
+                                  bool firstClient,
+                                  bool failureIsError,
+                                  int clientId,
+                                  int clientCount,
+                                  bool distributed,
+                                  bool shuffleShards,
+                                  int shardSampleFactor,
+                                  int64_t sharedRandomNumber,
+                                  int64_t repetitions,
+                                  int64_t* bytesReadInPrevRound,
+                                  int restart,
+                                  int64_t maxRate,
+                                  int64_t targetInterval,
+                                  bool* success) {
 	// Stores the total number of bytes on each storage server
 	// In a distributed test, this will be an estimated size
-	state std::map<UID, int64_t> storageServerSizes;
+	std::map<UID, int64_t> storageServerSizes;
 
 	// Iterate through each shard, checking its values on all of its storage servers
 	// If shardSampleFactor > 1, then not all shards are processed
 	// Also, in a distributed data consistency check, each client processes a subset of the shards
 	// Note: this may cause some shards to be processed more than once or not at all in a non-quiescent database
-	state int effectiveClientCount = distributed ? clientCount : 1;
-	state int i = clientId * (shardSampleFactor + 1);
-	state int64_t rateLimitForThisRound =
+	int effectiveClientCount = distributed ? clientCount : 1;
+	int i = clientId * (shardSampleFactor + 1);
+	int64_t rateLimitForThisRound =
 	    *bytesReadInPrevRound == 0
 	        ? maxRate
 	        : std::min(maxRate, static_cast<int64_t>(ceil(*bytesReadInPrevRound / (float)targetInterval)));
@@ -1540,19 +1548,19 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 	    .detail("TargetInterval", targetInterval)
 	    .detail("MaxRate", maxRate);
 	ASSERT(rateLimitForThisRound >= 0 && rateLimitForThisRound <= maxRate);
-	state Reference<IRateControl> rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
-	state double rateLimiterStartTime = now();
-	state int64_t bytesReadInthisRound = 0;
-	state double rateLimiterCumulatedWaitTime = 0;
-	state bool decideToRunAtMaxRate = false;
-
-	state double dbSize = 100e12;
-	state int ssCount = 1e6;
+	Reference<IRateControl> rateLimiter = makeReference<SpeedLimit>(rateLimitForThisRound, 1);
+	double rateLimiterStartTime = now();
+	int64_t bytesReadInthisRound = 0;
+	double rateLimiterCumulatedWaitTime = 0;
+	bool decideToRunAtMaxRate = false;
+
+	double dbSize = 100e12;
+	int ssCount = 1e6;
 	if (g_network->isSimulated()) {
 		// This call will get all shard ranges in the database, which is too expensive on real clusters.
-		int64_t _dbSize = wait(getDatabaseSize(cx));
+		int64_t _dbSize = co_await getDatabaseSize(cx);
 		dbSize = _dbSize;
-		std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+		std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
 		ssCount = 0;
 		for (auto& it : storageServers) {
 			if (!it.isTss()) {
@@ -1561,14 +1569,14 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 		}
 	}
 
-	state std::vector<KeyRangeRef> ranges;
+	std::vector<KeyRangeRef> ranges;
 
 	for (int k = 0; k < keyLocations.size() - 1; k++) {
 		KeyRangeRef range(keyLocations[k].key, keyLocations[k + 1].key);
 		ranges.push_back(range);
 	}
 
-	state std::vector<int> shardOrder;
+	std::vector<int> shardOrder;
 	shardOrder.reserve(ranges.size());
 	for (int k = 0; k < ranges.size(); k++)
 		shardOrder.push_back(k);
@@ -1577,34 +1585,34 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 		sharedRandom.randomShuffle(shardOrder);
 	}
 
-	state Reference<DDConfiguration::RangeConfigMapSnapshot> userRangeConfig =
-	    wait(DDConfiguration().userRangeConfig().getSnapshot(
-	        SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end));
-	state int customReplicatedShards = 0;
-	state int underReplicatedShards = 0;
-	state int64_t numCheckedShard = 0;
-	state int64_t numCheckedReadShard = 0;
+	Reference<DDConfiguration::RangeConfigMapSnapshot> userRangeConfig =
+	    co_await DDConfiguration().userRangeConfig().getSnapshot(
+	        SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end);
+	int customReplicatedShards = 0;
+	int underReplicatedShards = 0;
+	int64_t numCheckedShard = 0;
+	int64_t numCheckedReadShard = 0;
 
 	for (; i < ranges.size(); i++) {
-		state int shard = shardOrder[i];
+		int shard = shardOrder[i];
 
-		state KeyRangeRef range = ranges[shard];
-		state std::vector<UID> sourceStorageServers;
-		state std::vector<UID> destStorageServers;
-		state Transaction tr(cx);
+		KeyRangeRef range = ranges[shard];
+		std::vector<UID> sourceStorageServers;
+		std::vector<UID> destStorageServers;
+		Transaction tr(cx);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		state int bytesReadInRange = 0;
+		int bytesReadInRange = 0;
 
-		RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+		RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 		ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 		decodeKeyServersValue(UIDtoTagMap, keyLocations[shard].value, sourceStorageServers, destStorageServers, false);
 
 		// If the destStorageServers is non-empty, then this shard is being relocated
-		state bool isRelocating = destStorageServers.size() > 0;
+		bool isRelocating = !destStorageServers.empty();
 
-		state double shardCheckStartTime = now();
-		state double rateLimiterWaitTimeForThisShard = 0;
-		state double dataConsistencyCheckTimeForThisShard = 0;
+		double shardCheckStartTime = now();
+		double rateLimiterWaitTimeForThisShard = 0;
+		double dataConsistencyCheckTimeForThisShard = 0;
 
 		int desiredReplicas = configuration.storageTeamSize;
 		if (ddLargeTeamEnabled()) {
@@ -1636,7 +1644,7 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 					    .detail("CustomBegin", configuredRange.begin)
 					    .detail("CustomEnd", configuredRange.end);
 					testFailure("Custom shard boundary violated", performQuiescentChecks, success, failureIsError);
-					return Void();
+					co_return;
 				}
 
 				desiredReplicas = std::max(desiredReplicas, it->value().replicationFactor.orDefault(0));
@@ -1695,19 +1703,20 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 				TraceEvent("IncorrectSizeTeamInfo").detail("ServerUID", id).detail("TeamIndex", k++);
 			}
 			testFailure("Invalid team size", performQuiescentChecks, success, failureIsError);
-			return Void();
+			co_return;
 		}
 
-		state std::vector<UID> storageServers = (isRelocating) ? destStorageServers : sourceStorageServers;
-		state std::vector<StorageServerInterface> storageServerInterfaces;
+		std::vector<UID> storageServers = (isRelocating) ? destStorageServers : sourceStorageServers;
+		std::vector<StorageServerInterface> storageServerInterfaces;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				std::vector<Future<Optional<Value>>> serverListEntries;
 				serverListEntries.reserve(storageServers.size());
 				for (int s = 0; s < storageServers.size(); s++)
 					serverListEntries.push_back(tr.get(serverListKeyFor(storageServers[s])));
-				state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+				std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 				for (int s = 0; s < serverListValues.size(); s++) {
 					if (serverListValues[s].present())
 						storageServerInterfaces.push_back(decodeServerListValue(serverListValues[s].get()));
@@ -1720,8 +1729,10 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+
+			co_await tr.onError(err);
 		}
 
 		// add TSS to end of list, if configured and if not relocating
@@ -1738,17 +1749,17 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 		}
 
 		std::pair<std::vector<int64_t>, StorageMetrics> estimatedBytesAndMetrics =
-		    wait(getStorageSizeEstimate(storageServerInterfaces, range));
-		state std::vector<int64_t> estimatedBytes = estimatedBytesAndMetrics.first;
-		state StorageMetrics estimated = estimatedBytesAndMetrics.second;
+		    co_await getStorageSizeEstimate(storageServerInterfaces, range);
+		std::vector<int64_t> estimatedBytes = estimatedBytesAndMetrics.first;
+		StorageMetrics estimated = estimatedBytesAndMetrics.second;
 
 		// Gets permitted size range of shard
 		int64_t maxShardSize = getMaxShardSize(dbSize);
-		state ShardSizeBounds shardBounds = getShardSizeBounds(range, maxShardSize);
+		ShardSizeBounds shardBounds = getShardSizeBounds(range, maxShardSize);
 
 		if (firstClient) {
 			// If there was an error retrieving shard estimated size
-			if (performQuiescentChecks && estimatedBytes.size() == 0)
+			if (performQuiescentChecks && estimatedBytes.empty())
 				testFailure("Error fetching storage metrics", performQuiescentChecks, success, failureIsError);
 
 			// If running a distributed test, storage server size is an accumulation of shard estimates
@@ -1760,43 +1771,44 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 		// The first client may need to skip the rest of the loop contents if it is just processing this shard to
 		// get a size estimate
 		if (!firstClient || shard % (effectiveClientCount * shardSampleFactor) == 0) {
-			state int shardKeys = 0;
-			state int shardBytes = 0;
-			state int sampledBytes = 0;
-			state int splitBytes = 0;
-			state int firstKeySampledBytes = 0;
-			state int sampledKeys = 0;
-			state int sampledKeysWithProb = 0;
-			state double shardVariance = 0;
-			state bool canSplit = false;
-			state Key lastSampleKey;
-			state Key lastStartSampleKey;
-			state int64_t totalReadAmount = 0;
-			state KeyRange readRange = range;
-			state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+			int shardKeys = 0;
+			int shardBytes = 0;
+			int sampledBytes = 0;
+			int splitBytes = 0;
+			int firstKeySampledBytes = 0;
+			int sampledKeys = 0;
+			int sampledKeysWithProb = 0;
+			double shardVariance = 0;
+			bool canSplit = false;
+			Key lastSampleKey;
+			Key lastStartSampleKey;
+			int64_t totalReadAmount = 0;
+			KeyRange readRange = range;
+			Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
 
 			// Read a limited number of entries at a time, repeating until all keys in the shard have been read
-			loop {
+			while (true) {
+				Error err;
 				try {
 					lastSampleKey = lastStartSampleKey;
 
 					// Get the min version of the storage servers
-					Version version = wait(getVersion(cx));
+					Version version = co_await getStorageServerReadVersion(cx);
 
-					state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
-					state Optional<int> firstValidServer;
-					state double dataConsistencyCheckBeginTime = now();
+					std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+					Optional<int> firstValidServer;
+					double dataConsistencyCheckBeginTime = now();
 
 					totalReadAmount = 0;
-					int failures = wait(consistencyCheckReadData(UID(),
-					                                             cx,
-					                                             readRange,
-					                                             version,
-					                                             &storageServerInterfaces,
-					                                             &keyValueFutures,
-					                                             &firstValidServer,
-					                                             &totalReadAmount,
-					                                             {}));
+					int failures = co_await consistencyCheckReadData(UID(),
+					                                                 cx,
+					                                                 readRange,
+					                                                 version,
+					                                                 &storageServerInterfaces,
+					                                                 &keyValueFutures,
+					                                                 &firstValidServer,
+					                                                 &totalReadAmount,
+					                                                 {});
 					if (failures > 0) {
 						testFailure("Data inconsistent", performQuiescentChecks, success, true);
 					}
@@ -1826,19 +1838,19 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 							if (e.code() == error_code_request_maybe_delivered) {
 								// SS in the team may be removed and we get this error.
 								*success = false;
-								return Void();
+								co_return;
 							}
 							// All shards should be available in quiscence
 							if (performQuiescentChecks && !storageServerInterfaces[i].isTss()) {
 								testFailure(
 								    "Storage server unavailable", performQuiescentChecks, success, failureIsError);
-								return Void();
+								co_return;
 							}
 						}
 					}
 
 					if (firstValidServer.present()) {
-						state VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer.get()].get().get().data;
+						VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer.get()].get().get().data;
 
 						// Calculate the size of the shard, the variance of the shard size estimate, and the correct
 						// shard size estimate
@@ -1897,8 +1909,8 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 						TraceEvent("ConsistencyCheck_RateLimit")
 						    .detail("RateLimitForThisRound", rateLimitForThisRound)
 						    .detail("TotalAmountRead", totalReadAmount);
-						state double rateLimiterBeforeWaitTime = now();
-						wait(rateLimiter->getAllowance(totalReadAmount));
+						double rateLimiterBeforeWaitTime = now();
+						co_await rateLimiter->getAllowance(totalReadAmount);
 						double rateLimiterCurrentWaitTime = now() - rateLimiterBeforeWaitTime;
 						rateLimiterWaitTimeForThisShard += rateLimiterCurrentWaitTime;
 						rateLimiterCumulatedWaitTime += rateLimiterCurrentWaitTime;
@@ -1906,7 +1918,7 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 						// Set ratelimit to max allowed if current round has been going on for a while
 						if (now() - rateLimiterStartTime > 1.1 * targetInterval && rateLimitForThisRound != maxRate) {
 							rateLimitForThisRound = maxRate;
-							rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
+							rateLimiter = makeReference<SpeedLimit>(rateLimitForThisRound, 1);
 							rateLimiterStartTime = now();
 							TraceEvent(SevInfo, "ConsistencyCheck_RateLimitSetMaxForThisRound")
 							    .detail("RateLimit", rateLimitForThisRound);
@@ -1922,25 +1934,29 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 					// Advance to the next set of entries
 					if (firstValidServer.present() && keyValueFutures[firstValidServer.get()].get().get().more) {
 						VectorRef<KeyValueRef> result = keyValueFutures[firstValidServer.get()].get().get().data;
-						ASSERT(result.size() > 0);
+						ASSERT(!result.empty());
 						ASSERT(result[result.size() - 1].key != allKeys.end);
 						readRange = KeyRangeRef(keyAfter(result[result.size() - 1].key), range.end);
 						lastStartSampleKey = lastSampleKey;
 						if (readRange.empty()) {
 							break;
 						}
-					} else
-						break;
-				} catch (Error& e) {
-					state Error err = e;
-					if (e.code() == error_code_wrong_shard_server) {
-						// consistencyCheckReadData throws this error when a storage server is dead in simulation.
-						wait(delay(1.0));
 					} else {
-						wait(onErrorTr.onError(err));
+						break;
 					}
-					TraceEvent("ConsistencyCheck_RetryDataConsistency").error(err);
+
+					continue;
+				} catch (Error& e) {
+					err = e;
+				}
+
+				if (err.code() == error_code_wrong_shard_server) {
+					// consistencyCheckReadData throws this error when a storage server is dead in simulation.
+					co_await delay(1.0);
+				} else {
+					co_await onErrorTr.onError(err);
 				}
+				TraceEvent("ConsistencyCheck_RetryDataConsistency").error(err);
 			}
 
 			canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
@@ -1971,9 +1987,11 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 						}
 
 						break;
-					} else if (estimatedBytes[j] < 0 && ((g_network->isSimulated() &&
-					                                      g_simulator->tssMode <= ISimulator::TSSMode::EnabledNormal) ||
-					                                     !storageServerInterfaces[j].isTss())) {
+					} else if (estimatedBytes[j] < 0 &&
+					           ((g_network->isSimulated() &&
+					             !simulationPolicyHasCapability(
+					                 ISimulationPolicy::Capability::StorageReplicaFaultInjection)) ||
+					            !storageServerInterfaces[j].isTss())) {
 						// Ignore a non-responding TSS outside of simulation, or if tss fault injection is enabled
 						break;
 					}
@@ -2024,7 +2042,7 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 				splitMetrics.iosPerKSecond = splitMetrics.infinity;
 				splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
 
-				Standalone<VectorRef<KeyRef>> splits = wait(cx->splitStorageMetrics(range, splitMetrics, estimated));
+				Standalone<VectorRef<KeyRef>> splits = co_await cx->splitStorageMetrics(range, splitMetrics, estimated);
 				if (splits.size() <= 2) {
 					// Because the range's begin and end is included in splits, so this is the case
 					// the returned split is unfair and is discarded.
@@ -2053,7 +2071,7 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 				            performQuiescentChecks,
 				            success,
 				            failureIsError);
-				return Void();
+				co_return;
 			}
 		}
 
@@ -2093,7 +2111,7 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 		    .detail("ShardsOnLargeTeams", customReplicatedShards)
 		    .detail("Limit", SERVER_KNOBS->DD_MAX_SHARDS_ON_LARGE_TEAMS);
 		testFailure("Too many shards on large teams", performQuiescentChecks, success, failureIsError);
-		return Void();
+		co_return;
 	}
 
 	if (customReplicatedShards < SERVER_KNOBS->DD_MAX_SHARDS_ON_LARGE_TEAMS && underReplicatedShards > 0) {
@@ -2102,7 +2120,7 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 		    .detail("Limit", SERVER_KNOBS->DD_MAX_SHARDS_ON_LARGE_TEAMS)
 		    .detail("UnderReplicated", underReplicatedShards);
 		testFailure("Too few shards on large teams", performQuiescentChecks, success, failureIsError);
-		return Void();
+		co_return;
 	}
 
 	if (customReplicatedShards == SERVER_KNOBS->DD_MAX_SHARDS_ON_LARGE_TEAMS && underReplicatedShards > 0) {
@@ -2110,5 +2128,4 @@ ACTOR Future<Void> checkDataConsistency(Database cx,
 	}
 
 	*bytesReadInPrevRound = bytesReadInthisRound;
-	return Void();
 }
diff --git a/fdbserver/consistencyscan/include/fdbserver/consistencyscan/ConsistencyScan.h b/fdbserver/consistencyscan/include/fdbserver/consistencyscan/ConsistencyScan.h
new file mode 100644
index 00000000000..2a6c26e29e1
--- /dev/null
+++ b/fdbserver/consistencyscan/include/fdbserver/consistencyscan/ConsistencyScan.h
@@ -0,0 +1,64 @@
+/*
+ * ConsistencyScan.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/ConsistencyScanInterface.h"
+#include "flow/flow.h"
+
+struct ServerDBInfo;
+
+Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
+
+// These helpers are used by the ConsistencyCheck workload and implemented in
+// fdbserver/consistencyscan/ConsistencyScan.cpp.
+Future<Version> getVersion(Database cx);
+Future<bool> getKeyServers(
+    Database cx,
+    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
+    KeyRangeRef kr,
+    bool performQuiescentChecks,
+    bool failureIsError,
+    bool* success);
+Future<bool> getKeyLocations(Database cx,
+                             std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> shards,
+                             Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise,
+                             bool performQuiescentChecks,
+                             bool* success);
+Future<Void> checkDataConsistency(Database cx,
+                                  VectorRef<KeyValueRef> keyLocations,
+                                  DatabaseConfiguration configuration,
+                                  std::map<UID, StorageServerInterface> tssMapping,
+                                  bool performQuiescentChecks,
+                                  bool performTSSCheck,
+                                  bool firstClient,
+                                  bool failureIsError,
+                                  int clientId,
+                                  int clientCount,
+                                  bool distributed,
+                                  bool shuffleShards,
+                                  int shardSampleFactor,
+                                  int64_t sharedRandomNumber,
+                                  int64_t repetitions,
+                                  int64_t* bytesReadInPreviousRound,
+                                  int restart,
+                                  int64_t maxRate,
+                                  int64_t targetInterval,
+                                  bool* success);
diff --git a/fdbserver/coordinator/CMakeLists.txt b/fdbserver/coordinator/CMakeLists.txt
new file mode 100644
index 00000000000..baacbdae28a
--- /dev/null
+++ b/fdbserver/coordinator/CMakeLists.txt
@@ -0,0 +1,16 @@
+fdb_find_sources(FDBSERVER_COORDINATOR_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_coordinator SRCS ${FDBSERVER_COORDINATOR_SRCS})
+add_fdbserver_link_test(fdbserver_coordinatorlinktest
+  fdbserver_coordinator
+  fdbserver_kvstore
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_coordinator)
+target_include_directories(fdbserver_coordinator
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(fdbserver_coordinator PRIVATE fdbserver_core fdbserver_kvstore)
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/coordinator/Coordination.cpp
similarity index 57%
rename from fdbserver/Coordination.actor.cpp
rename to fdbserver/coordinator/Coordination.cpp
index a442a8bcf44..5726d75a068 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/coordinator/Coordination.cpp
@@ -1,5 +1,5 @@
 /*
- * Coordination.actor.cpp
+ * Coordination.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,37 +20,35 @@
 
 #include <cstdint>
 
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/OnDemandStore.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/Status.actor.h"
+#include "fdbserver/coordinator/CoordinationServer.h"
+#include "fdbserver/core/Knobs.h"
+#include "OnDemandStore.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/UnitTest.h"
 #include "flow/IndexedSet.h"
+#include "flow/genericactors.actor.h"
 #include "fdbclient/MonitorLeader.h"
 #include "flow/network.h"
+#include "flow/CoroUtils.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// This module implements coordinationServer() and the interfaces in CoordinationInterface.h
+// This module implements coordinationServer() plus the interfaces in CoordinationInterface.h
 
 namespace {
 
 const std::string fileCoordinatorPrefix = "coordination-";
 
+} // namespace
+
 class LivenessChecker {
 	double threshold;
 	AsyncVar<double> lastTime;
-	ACTOR static Future<Void> checkStuck(LivenessChecker const* self) {
-		loop {
-			choose {
-				when(wait(delayUntil(self->lastTime.get() + self->threshold))) {
-					return Void();
-				}
-				when(wait(self->lastTime.onChange())) {}
+	static Future<Void> checkStuck(LivenessChecker const* self) {
+		while (true) {
+			auto res = co_await race(delayUntil(self->lastTime.get() + self->threshold), self->lastTime.onChange());
+			if (res.index() == 0) {
+				co_return;
 			}
 		}
 	}
@@ -63,8 +61,6 @@ class LivenessChecker {
 	Future<Void> checkStuck() const { return checkStuck(this); }
 };
 
-} // namespace
-
 struct GenerationRegVal {
 	UniqueGeneration readGen, writeGen;
 	Optional<Value> val;
@@ -77,53 +73,33 @@ struct GenerationRegVal {
 	}
 };
 
-GenerationRegInterface::GenerationRegInterface(NetworkAddress const& remote)
-  : read(Endpoint::wellKnown({ remote }, WLTOKEN_GENERATIONREG_READ)),
-    write(Endpoint::wellKnown({ remote }, WLTOKEN_GENERATIONREG_WRITE)) {}
-
-GenerationRegInterface::GenerationRegInterface(INetwork* local) {
-	read.makeWellKnownEndpoint(WLTOKEN_GENERATIONREG_READ, TaskPriority::Coordination);
-	write.makeWellKnownEndpoint(WLTOKEN_GENERATIONREG_WRITE, TaskPriority::Coordination);
-}
-
-LeaderElectionRegInterface::LeaderElectionRegInterface(NetworkAddress const& remote)
-  : ClientLeaderRegInterface(remote), candidacy(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_CANDIDACY)),
-    electionResult(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT)),
-    leaderHeartbeat(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT)),
-    forward(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_FORWARD)) {}
-
-LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : ClientLeaderRegInterface(local) {
-	candidacy.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_CANDIDACY, TaskPriority::Coordination);
-	electionResult.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT, TaskPriority::Coordination);
-	leaderHeartbeat.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT, TaskPriority::Coordination);
-	forward.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_FORWARD, TaskPriority::Coordination);
-}
+class LocalGenerationReg {
+public:
+	LocalGenerationReg(GenerationRegInterface interf, OnDemandStore* pstore)
+	  : readReqs(interf.read.getFuture()), writeReqs(interf.write.getFuture()), pStore(pstore),
+	    storeLock(new FlowLock(1)) {}
 
-ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr) : ClientCoordinators(ccr) {
-	ClusterConnectionString cs = ccr->getConnectionString();
-	for (auto h : cs.hostnames) {
-		leaderElectionServers.emplace_back(h);
-		stateServers.emplace_back(h);
+	Future<Void> run() {
+		return serveReadReqs(readReqs, pStore, storeLock) || serveWriteReqs(writeReqs, pStore, storeLock);
 	}
-	for (auto s : cs.coords) {
-		leaderElectionServers.emplace_back(s);
-		stateServers.emplace_back(s);
-	}
-}
 
-ACTOR Future<Void> localGenerationReg(GenerationRegInterface interf, OnDemandStore* pstore) {
-	state GenerationRegVal v;
-	state OnDemandStore& store = *pstore;
-	// SOMEDAY: concurrent access to different keys?
-	loop choose {
-		when(GenerationRegReadRequest _req = waitNext(interf.read.getFuture())) {
+private:
+	static Future<Void> serveReadReqs(FutureStream<GenerationRegReadRequest> readReqs,
+	                                  OnDemandStore* pstore,
+	                                  Reference<FlowLock> storeLock) {
+		OnDemandStore& store = *pstore;
+		while (true) {
+			GenerationRegReadRequest req = co_await readReqs;
 			TraceEvent("GenerationRegReadRequest")
-			    .detail("From", _req.reply.getEndpoint().getPrimaryAddress())
-			    .detail("K", _req.key);
-			state GenerationRegReadRequest req = _req;
-			Optional<Value> rawV = wait(store->readValue(req.key));
-			v = rawV.present() ? BinaryReader::fromStringRef<GenerationRegVal>(rawV.get(), IncludeVersion())
-			                   : GenerationRegVal();
+			    .detail("From", req.reply.getEndpoint().getPrimaryAddress())
+			    .detail("K", req.key);
+			// SOMEDAY: concurrent access to different keys?
+			co_await storeLock->take();
+			FlowLock::Releaser storeLockReleaser(*storeLock);
+			Optional<Value> rawV = co_await store->readValue(req.key);
+			GenerationRegVal v = rawV.present()
+			                         ? BinaryReader::fromStringRef<GenerationRegVal>(rawV.get(), IncludeVersion())
+			                         : GenerationRegVal();
 			TraceEvent("GenerationRegReadReply")
 			    .detail("RVSize", rawV.present() ? rawV.get().size() : -1)
 			    .detail("VWG", v.writeGen.generation);
@@ -131,21 +107,31 @@ ACTOR Future<Void> localGenerationReg(GenerationRegInterface interf, OnDemandSto
 				v.readGen = req.gen;
 				store->set(KeyValueRef(
 				    req.key, BinaryWriter::toValue(v, IncludeVersion(ProtocolVersion::withGenerationRegVal()))));
-				wait(store->commit());
+				co_await store->commit();
 			}
 			req.reply.send(GenerationRegReadReply(v.val, v.writeGen, v.readGen));
 		}
-		when(GenerationRegWriteRequest _wrq = waitNext(interf.write.getFuture())) {
-			state GenerationRegWriteRequest wrq = _wrq;
-			Optional<Value> rawV = wait(store->readValue(wrq.kv.key));
-			v = rawV.present() ? BinaryReader::fromStringRef<GenerationRegVal>(rawV.get(), IncludeVersion())
-			                   : GenerationRegVal();
+	}
+
+	static Future<Void> serveWriteReqs(FutureStream<GenerationRegWriteRequest> writeReqs,
+	                                   OnDemandStore* pstore,
+	                                   Reference<FlowLock> storeLock) {
+		OnDemandStore& store = *pstore;
+		while (true) {
+			GenerationRegWriteRequest wrq = co_await writeReqs;
+			// SOMEDAY: concurrent access to different keys?
+			co_await storeLock->take();
+			FlowLock::Releaser storeLockReleaser(*storeLock);
+			Optional<Value> rawV = co_await store->readValue(wrq.kv.key);
+			GenerationRegVal v = rawV.present()
+			                         ? BinaryReader::fromStringRef<GenerationRegVal>(rawV.get(), IncludeVersion())
+			                         : GenerationRegVal();
 			if (v.readGen <= wrq.gen && v.writeGen < wrq.gen) {
 				v.writeGen = wrq.gen;
 				v.val = wrq.kv.value;
 				store->set(KeyValueRef(
 				    wrq.kv.key, BinaryWriter::toValue(v, IncludeVersion(ProtocolVersion::withGenerationRegVal()))));
-				wait(store->commit());
+				co_await store->commit();
 				TraceEvent("GenerationRegWrote")
 				    .detail("From", wrq.reply.getEndpoint().getPrimaryAddress())
 				    .detail("Key", wrq.kv.key)
@@ -163,18 +149,24 @@ ACTOR Future<Void> localGenerationReg(GenerationRegInterface interf, OnDemandSto
 			}
 		}
 	}
-}
+
+	FutureStream<GenerationRegReadRequest> readReqs;
+	FutureStream<GenerationRegWriteRequest> writeReqs;
+	OnDemandStore* pStore;
+	Reference<FlowLock> storeLock;
+};
 
 TEST_CASE("/fdbserver/Coordination/localGenerationReg/simple") {
-	state GenerationRegInterface reg;
-	state OnDemandStore store(params.getDataDir(), deterministicRandom()->randomUniqueID(), fileCoordinatorPrefix);
-	state Future<Void> actor = localGenerationReg(reg, &store);
-	state Key the_key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, 10)));
+	GenerationRegInterface reg;
+	OnDemandStore store(params.getDataDir(), deterministicRandom()->randomUniqueID(), fileCoordinatorPrefix);
+	LocalGenerationReg generationReg(reg, &store);
+	Future<Void> actor = generationReg.run();
+	Key the_key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, 10)));
 
-	state UniqueGeneration firstGen(0, deterministicRandom()->randomUniqueID());
+	UniqueGeneration firstGen(0, deterministicRandom()->randomUniqueID());
 
 	{
-		GenerationRegReadReply r = wait(reg.read.getReply(GenerationRegReadRequest(the_key, firstGen)));
+		GenerationRegReadReply r = co_await reg.read.getReply(GenerationRegReadRequest(the_key, firstGen));
 		//   If there was no prior write(_,_,0) or a data loss fault,
 		//     returns (Optional(),0,gen2)
 		ASSERT(!r.value.present());
@@ -184,13 +176,13 @@ TEST_CASE("/fdbserver/Coordination/localGenerationReg/simple") {
 
 	{
 		UniqueGeneration g =
-		    wait(reg.write.getReply(GenerationRegWriteRequest(KeyValueRef(the_key, "Value1"_sr), firstGen)));
+		    co_await reg.write.getReply(GenerationRegWriteRequest(KeyValueRef(the_key, "Value1"_sr), firstGen));
 		//   (gen1==gen is considered a "successful" write)
 		ASSERT(g == firstGen);
 	}
 
 	{
-		GenerationRegReadReply r = wait(reg.read.getReply(GenerationRegReadRequest(the_key, UniqueGeneration())));
+		GenerationRegReadReply r = co_await reg.read.getReply(GenerationRegReadRequest(the_key, UniqueGeneration()));
 		// read(key,gen2) returns (value,gen,rgen).
 		//     There was some earlier or concurrent write(key,value,gen).
 		ASSERT(r.value == "Value1"_sr);
@@ -205,16 +197,15 @@ TEST_CASE("/fdbserver/Coordination/localGenerationReg/simple") {
 
 		ASSERT(!actor.isReady());
 	}
-	return Void();
 }
 
-ACTOR Future<Void> openDatabase(ClientData* db,
-                                int* clientCount,
-                                Reference<AsyncVar<bool>> hasConnectedClients,
-                                OpenDatabaseCoordRequest req,
-                                Future<Void> checkStuck) {
-	state ErrorOr<CachedSerialization<ClientDBInfo>> replyContents;
-	state Future<Void> clientInfoOnChange = db->clientInfo->onChange();
+Future<Void> openDatabase(ClientData* db,
+                          int* clientCount,
+                          Reference<AsyncVar<bool>> hasConnectedClients,
+                          OpenDatabaseCoordRequest req,
+                          Future<Void> checkStuck) {
+	ErrorOr<CachedSerialization<ClientDBInfo>> replyContents;
+	Future<Void> clientInfoOnChange = db->clientInfo->onChange();
 
 	++(*clientCount);
 	hasConnectedClients->set(true);
@@ -226,22 +217,24 @@ ACTOR Future<Void> openDatabase(ClientData* db,
 
 	while (!db->clientInfo->get().read().id.isValid() || (db->clientInfo->get().read().id == req.knownClientInfoID &&
 	                                                      !db->clientInfo->get().read().forward.present())) {
-		choose {
-			when(wait(checkStuck)) {
-				replyContents = failed_to_progress();
-				break;
-			}
-			when(wait(yieldedFuture(clientInfoOnChange))) {
-				clientInfoOnChange = db->clientInfo->onChange();
+		auto res = co_await race(
+		    checkStuck, yieldedFuture(clientInfoOnChange), delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL));
+		if (res.index() == 0) {
+			// checkStuck fired:
+			replyContents = failed_to_progress();
+			break;
+		} else if (res.index() == 1) {
+			// clientInfoOnChange fired:
+			clientInfoOnChange = db->clientInfo->onChange();
+			replyContents = db->clientInfo->get();
+		} else if (res.index() == 2) {
+			// delay fired:
+			if (db->clientInfo->get().read().id.isValid()) {
 				replyContents = db->clientInfo->get();
 			}
-			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) {
-				if (db->clientInfo->get().read().id.isValid()) {
-					replyContents = db->clientInfo->get();
-				}
-				// Otherwise, we still break out of the loop and return a default_error_or.
-				break;
-			} // The client might be long gone!
+			// Otherwise, we still break out of the loop and return a default_error_or.
+			// The client might be long gone!
+			break;
 		}
 	}
 
@@ -258,26 +251,23 @@ ACTOR Future<Void> openDatabase(ClientData* db,
 	if (--(*clientCount) == 0) {
 		hasConnectedClients->set(false);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> remoteMonitorLeader(int* clientCount,
-                                       Reference<AsyncVar<bool>> hasConnectedClients,
-                                       Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader,
-                                       ElectionResultRequest req) {
-	state Future<Void> currentElectedLeaderOnChange = currentElectedLeader->onChange();
+Future<Void> remoteMonitorLeader(int* clientCount,
+                                 Reference<AsyncVar<bool>> hasConnectedClients,
+                                 Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader,
+                                 ElectionResultRequest req) {
+	Future<Void> currentElectedLeaderOnChange = currentElectedLeader->onChange();
 	++(*clientCount);
 	hasConnectedClients->set(true);
 
 	while (!currentElectedLeader->get().present() || req.knownLeader == currentElectedLeader->get().get().changeID) {
-		choose {
-			when(wait(yieldedFuture(currentElectedLeaderOnChange))) {
-				currentElectedLeaderOnChange = currentElectedLeader->onChange();
-			}
-			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) {
-				break;
-			}
+		auto res = co_await race(yieldedFuture(currentElectedLeaderOnChange),
+		                         delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL));
+		if (res.index() == 0) {
+			currentElectedLeaderOnChange = currentElectedLeader->onChange();
+		} else {
+			break;
 		}
 	}
 
@@ -286,36 +276,75 @@ ACTOR Future<Void> remoteMonitorLeader(int* clientCount,
 	if (--(*clientCount) == 0) {
 		hasConnectedClients->set(false);
 	}
-
-	return Void();
 }
 
-// This actor implements a *single* leader-election register (essentially, it ignores
+// This class implements a *single* leader-election register (essentially, it ignores
 // the .key member of each request).  It returns any time the leader election is in the
 // default state, so that only active registers consume memory.
-ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
-	state std::set<LeaderInfo> availableCandidates;
-	state std::set<LeaderInfo> availableLeaders;
-	state Optional<LeaderInfo> currentNominee;
-	state Deque<ReplyPromise<Optional<LeaderInfo>>> notify;
-	state Future<Void> nextInterval;
-	state double candidateDelay = SERVER_KNOBS->CANDIDATE_MIN_DELAY;
-	state int leaderIntervalCount = 0;
-	state Future<Void> notifyCheck =
-	    delay(SERVER_KNOBS->NOTIFICATION_FULL_CLEAR_TIME / SERVER_KNOBS->MIN_NOTIFICATIONS);
-	state ClientData clientData;
-	state int clientCount = 0;
-	state Reference<AsyncVar<bool>> hasConnectedClients = makeReference<AsyncVar<bool>>(false);
-	state ActorCollection actors(false);
-	state Future<Void> leaderMon;
-	state AsyncVar<Value> leaderInterface;
-	state Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader =
-	    makeReference<AsyncVar<Optional<LeaderInfo>>>();
-	state LivenessChecker canConnectToLeader(SERVER_KNOBS->COORDINATOR_LEADER_CONNECTION_TIMEOUT);
-	state Future<Void> hasConnectedClientsOnChange = hasConnectedClients->onChange();
-
-	loop choose {
-		when(OpenDatabaseCoordRequest req = waitNext(interf.openDatabase.getFuture())) {
+class LeaderRegister : public ReferenceCounted<LeaderRegister>, NonCopyable {
+	LeaderElectionRegInterface interf;
+	Key key;
+	std::set<LeaderInfo> availableCandidates;
+	std::set<LeaderInfo> availableLeaders;
+	Optional<LeaderInfo> currentNominee;
+	Deque<ReplyPromise<Optional<LeaderInfo>>> notify;
+	Future<Void> nextInterval;
+	AsyncVar<int> nextIntervalGeneration;
+	double candidateDelay;
+	int leaderIntervalCount;
+	Future<Void> notifyCheck;
+	ClientData clientData;
+	int clientCount;
+	Reference<AsyncVar<bool>> hasConnectedClients;
+	ActorCollection actors;
+	Future<Void> leaderMon;
+	Reference<AsyncVar<Optional<LeaderInfo>>> currentElectedLeader;
+	LivenessChecker canConnectToLeader;
+	Future<Void> hasConnectedClientsOnChange;
+
+	void setNextInterval(Future<Void> interval) {
+		nextInterval = interval;
+		nextIntervalGeneration.set(nextIntervalGeneration.get() + 1);
+	}
+
+	void ensureNextInterval() {
+		if (!nextInterval.isValid()) {
+			setNextInterval(delay(0));
+		}
+	}
+
+	void clearNextInterval() {
+		if (nextInterval.isValid()) {
+			setNextInterval(Future<Void>());
+		}
+	}
+
+	void sendNotifications(Optional<LeaderInfo> const& nominee) {
+		for (unsigned int i = 0; i < notify.size(); i++) {
+			notify[i].send(nominee);
+		}
+		notify.clear();
+	}
+
+	void sendNotifications(LeaderInfo const& info) {
+		for (unsigned int i = 0; i < notify.size(); i++) {
+			notify[i].send(info);
+		}
+		notify.clear();
+	}
+
+	bool checkNotificationLimit() {
+		if (notify.size() > SERVER_KNOBS->MAX_NOTIFICATIONS) {
+			TraceEvent(SevWarnAlways, "TooManyNotifications").detail("Amount", notify.size());
+			sendNotifications(currentNominee);
+			return true;
+		}
+		return false;
+	}
+
+	Future<Void> serveOpenDatabaseRequests() {
+		while (true) {
+			OpenDatabaseCoordRequest req = co_await interf.openDatabase.getFuture();
 			if (clientData.clientInfo->get().read().id.isValid() &&
 			    clientData.clientInfo->get().read().id != req.knownClientInfoID &&
 			    !clientData.clientInfo->get().read().forward.present()) {
@@ -329,7 +358,11 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				    openDatabase(&clientData, &clientCount, hasConnectedClients, req, canConnectToLeader.checkStuck()));
 			}
 		}
-		when(ElectionResultRequest req = waitNext(interf.electionResult.getFuture())) {
+	}
+
+	Future<Void> serveElectionResultRequests() {
+		while (true) {
+			ElectionResultRequest req = co_await interf.electionResult.getFuture();
 			if (currentElectedLeader->get().present() &&
 			    req.knownLeader != currentElectedLeader->get().get().changeID) {
 				req.reply.send(currentElectedLeader->get());
@@ -341,43 +374,41 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				actors.add(remoteMonitorLeader(&clientCount, hasConnectedClients, currentElectedLeader, req));
 			}
 		}
-		when(GetLeaderRequest req = waitNext(interf.getLeader.getFuture())) {
+	}
+
+	Future<Void> serveGetLeaderRequests() {
+		while (true) {
+			GetLeaderRequest req = co_await interf.getLeader.getFuture();
 			if (currentNominee.present() && currentNominee.get().changeID != req.knownLeader) {
 				req.reply.send(currentNominee.get());
 			} else {
 				notify.push_back(req.reply);
-				if (notify.size() > SERVER_KNOBS->MAX_NOTIFICATIONS) {
-					TraceEvent(SevWarnAlways, "TooManyNotifications").detail("Amount", notify.size());
-					for (uint32_t i = 0; i < notify.size(); i++)
-						notify[i].send(currentNominee.get());
-					notify.clear();
-				} else if (!nextInterval.isValid()) {
-					nextInterval = delay(0);
+				if (!checkNotificationLimit() && !nextInterval.isValid()) {
+					ensureNextInterval();
 				}
 			}
 		}
-		when(CandidacyRequest req = waitNext(interf.candidacy.getFuture())) {
-			if (!nextInterval.isValid()) {
-				nextInterval = delay(0);
-			}
+	}
+
+	Future<Void> serveCandidacyRequests() {
+		while (true) {
+			CandidacyRequest req = co_await interf.candidacy.getFuture();
+			ensureNextInterval();
 			availableCandidates.erase(LeaderInfo(req.prevChangeID));
 			availableCandidates.insert(req.myInfo);
 			if (currentNominee.present() && currentNominee.get().changeID != req.knownLeader) {
 				req.reply.send(currentNominee.get());
 			} else {
 				notify.push_back(req.reply);
-				if (notify.size() > SERVER_KNOBS->MAX_NOTIFICATIONS) {
-					TraceEvent(SevWarnAlways, "TooManyNotifications").detail("Amount", notify.size());
-					for (uint32_t i = 0; i < notify.size(); i++)
-						notify[i].send(currentNominee.get());
-					notify.clear();
-				}
+				checkNotificationLimit();
 			}
 		}
-		when(LeaderHeartbeatRequest req = waitNext(interf.leaderHeartbeat.getFuture())) {
-			if (!nextInterval.isValid()) {
-				nextInterval = delay(0);
-			}
+	}
+
+	Future<Void> serveLeaderHeartbeatRequests() {
+		while (true) {
+			LeaderHeartbeatRequest req = co_await interf.leaderHeartbeat.getFuture();
+			ensureNextInterval();
 			// TODO: use notify to only send a heartbeat once per interval
 			availableLeaders.erase(LeaderInfo(req.prevChangeID));
 			availableLeaders.insert(req.myInfo);
@@ -387,24 +418,39 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 			}
 			req.reply.send(LeaderHeartbeatReply{ isCurrentLeader });
 		}
-		when(ForwardRequest req = waitNext(interf.forward.getFuture())) {
+	}
+
+	Future<Void> serveForwardRequests() {
+		while (true) {
+			ForwardRequest req = co_await interf.forward.getFuture();
 			LeaderInfo newInfo;
 			newInfo.forward = true;
 			newInfo.serializedInfo = req.conn.toString();
-			for (unsigned int i = 0; i < notify.size(); i++)
-				notify[i].send(newInfo);
-			notify.clear();
+			sendNotifications(newInfo);
 			ClientDBInfo outInfo;
 			outInfo.id = deterministicRandom()->randomUniqueID();
 			outInfo.forward = req.conn.toString();
 			clientData.clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
 			req.reply.send(Void());
 			if (!hasConnectedClients->get()) {
-				return Void();
+				co_return;
 			}
-			nextInterval = Future<Void>();
+			clearNextInterval();
 		}
-		when(wait(nextInterval.isValid() ? nextInterval : Never())) {
+	}
+
+	Future<Void> monitorNextInterval() {
+		while (true) {
+			int generation = nextIntervalGeneration.get();
+			auto res =
+			    co_await race(nextInterval.isValid() ? nextInterval : Never(), nextIntervalGeneration.onChange());
+			if (res.index() == 1 || generation != nextIntervalGeneration.get()) {
+				continue;
+			}
+			if (res.index() != 0) {
+				UNREACHABLE();
+			}
+
 			if (!availableLeaders.size() && !availableCandidates.size() && !notify.size() &&
 			    !currentNominee.present()) {
 				// Our state is back to the initial state, so we can safely stop this actor
@@ -412,9 +458,9 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				    .detail("Key", key)
 				    .detail("HasConnectedClients", hasConnectedClients->get());
 				if (!hasConnectedClients->get()) {
-					return Void();
+					co_return;
 				} else {
-					nextInterval = Future<Void>();
+					clearNextInterval();
 				}
 			} else {
 				Optional<LeaderInfo> nextNominee;
@@ -440,20 +486,18 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 					    .detail("NextNominee", nextNominee.present() ? nextNominee.get().changeID : UID())
 					    .detail("CurrentNominee", currentNominee.present() ? currentNominee.get().changeID : UID())
 					    .detail("Key", printable(key));
-					for (unsigned int i = 0; i < notify.size(); i++)
-						notify[i].send(nextNominee);
-					notify.clear();
+					sendNotifications(nextNominee);
 				}
 
 				currentNominee = nextNominee;
 
 				if (availableLeaders.size()) {
-					nextInterval = delay(SERVER_KNOBS->POLLING_FREQUENCY);
+					setNextInterval(delay(SERVER_KNOBS->POLLING_FREQUENCY));
 					if (leaderIntervalCount++ > 5) {
 						candidateDelay = SERVER_KNOBS->CANDIDATE_MIN_DELAY;
 					}
 				} else {
-					nextInterval = delay(candidateDelay);
+					setNextInterval(delay(candidateDelay));
 					candidateDelay = std::min(SERVER_KNOBS->CANDIDATE_MAX_DELAY,
 					                          candidateDelay * SERVER_KNOBS->CANDIDATE_GROWTH_RATE);
 					leaderIntervalCount = 0;
@@ -463,7 +507,11 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				availableCandidates.clear();
 			}
 		}
-		when(wait(notifyCheck)) {
+	}
+
+	Future<Void> monitorNotifyCheck() {
+		while (true) {
+			co_await notifyCheck;
 			notifyCheck = delay(SERVER_KNOBS->NOTIFICATION_FULL_CLEAR_TIME /
 			                    std::max<double>(SERVER_KNOBS->MIN_NOTIFICATIONS, notify.size()));
 			if (!notify.empty() && currentNominee.present()) {
@@ -471,16 +519,44 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				notify.pop_front();
 			}
 		}
-		when(wait(hasConnectedClientsOnChange)) {
+	}
+
+	Future<Void> monitorConnectedClients() {
+		while (true) {
+			co_await hasConnectedClientsOnChange;
 			hasConnectedClientsOnChange = hasConnectedClients->onChange();
 			if (!hasConnectedClients->get() && !nextInterval.isValid()) {
 				TraceEvent("LeaderRegisterUnneeded").detail("Key", key);
-				return Void();
+				co_return;
 			}
 		}
-		when(wait(actors.getResult())) {}
 	}
-}
+
+	Future<Void> monitorActors() { co_await actors.getResult(); }
+
+public:
+	LeaderRegister(LeaderElectionRegInterface interf, Key key)
+	  : interf(interf), key(key), nextIntervalGeneration(0), candidateDelay(SERVER_KNOBS->CANDIDATE_MIN_DELAY),
+	    leaderIntervalCount(0),
+	    notifyCheck(delay(SERVER_KNOBS->NOTIFICATION_FULL_CLEAR_TIME / SERVER_KNOBS->MIN_NOTIFICATIONS)),
+	    clientCount(0), hasConnectedClients(makeReference<AsyncVar<bool>>(false)), actors(false),
+	    currentElectedLeader(makeReference<AsyncVar<Optional<LeaderInfo>>>()),
+	    canConnectToLeader(SERVER_KNOBS->COORDINATOR_LEADER_CONNECTION_TIMEOUT),
+	    hasConnectedClientsOnChange(hasConnectedClients->onChange()) {}
+
+	static Future<Void> run(Reference<LeaderRegister> self) {
+		co_await race(self->serveOpenDatabaseRequests(),
+		              self->serveElectionResultRequests(),
+		              self->serveGetLeaderRequests(),
+		              self->serveCandidacyRequests(),
+		              self->serveLeaderHeartbeatRequests(),
+		              self->serveForwardRequests(),
+		              self->monitorNextInterval(),
+		              self->monitorNotifyCheck(),
+		              self->monitorConnectedClients(),
+		              self->monitorActors());
+	}
+};
 
 // Generation register values are stored without prefixing in the coordinated state, but always begin with an
 // alphanumeric character (they are always derived from a ClusterConnectionString key). Forwarding values are stored in
@@ -503,15 +579,15 @@ struct LeaderRegisterCollection {
 	OnDemandStore* pStore;
 	Map<Key, double> forwardStartTime;
 
-	LeaderRegisterCollection(OnDemandStore* pStore) : actors(false), pStore(pStore) {}
+	explicit LeaderRegisterCollection(OnDemandStore* pStore) : actors(false), pStore(pStore) {}
 
-	ACTOR static Future<Void> init(LeaderRegisterCollection* self) {
+	static Future<Void> init(LeaderRegisterCollection* self) {
 		if (!self->pStore->exists())
-			return Void();
+			co_return;
 		OnDemandStore& store = *self->pStore;
-		state Future<Standalone<RangeResultRef>> forwardingInfoF = store->readRange(fwdKeys);
-		state Future<Standalone<RangeResultRef>> forwardingTimeF = store->readRange(fwdTimeKeys);
-		wait(success(forwardingInfoF) && success(forwardingTimeF));
+		Future<Standalone<RangeResultRef>> forwardingInfoF = store->readRange(fwdKeys);
+		Future<Standalone<RangeResultRef>> forwardingTimeF = store->readRange(fwdTimeKeys);
+		co_await (success(forwardingInfoF) && success(forwardingTimeF));
 		Standalone<RangeResultRef> forwardingInfo = forwardingInfoF.get();
 		Standalone<RangeResultRef> forwardingTime = forwardingTimeF.get();
 		for (int i = 0; i < forwardingInfo.size(); i++) {
@@ -524,7 +600,6 @@ struct LeaderRegisterCollection {
 			double time = BinaryReader::fromStringRef<double>(forwardingTime[i].value, Unversioned());
 			self->forwardStartTime[forwardingTime[i].key.removePrefix(fwdTimeKeys.begin)] = time;
 		}
-		return Void();
 	}
 
 	Future<Void> onError() const { return actors.getResult(); }
@@ -551,11 +626,11 @@ struct LeaderRegisterCollection {
 	// When the lead coordinator changes, store the new connection ID in the "fwd" keyspace.
 	// If a request arrives using an old connection id, resend it to the new coordinator using the stored connection id.
 	// Store when this change took place in the fwdTime keyspace.
-	ACTOR static Future<Void> setForward(LeaderRegisterCollection* self,
-	                                     KeyRef key,
-	                                     ClusterConnectionString conn,
-	                                     ForwardRequest req,
-	                                     UID id) {
+	static Future<Void> setForward(LeaderRegisterCollection* self,
+	                               KeyRef key,
+	                               ClusterConnectionString conn,
+	                               ForwardRequest req,
+	                               UID id) {
 		double forwardTime = now();
 		LeaderInfo forwardInfo;
 		forwardInfo.forward = true;
@@ -565,17 +640,17 @@ struct LeaderRegisterCollection {
 		OnDemandStore& store = *self->pStore;
 		store->set(KeyValueRef(key.withPrefix(fwdKeys.begin), conn.toString()));
 		store->set(KeyValueRef(key.withPrefix(fwdTimeKeys.begin), BinaryWriter::toValue(forwardTime, Unversioned())));
-		wait(store->commit());
+		co_await store->commit();
 		// Do not process a forwarding request until after it has been made durable in case the coordinator restarts
 		self->getInterface(req.key, id).forward.send(req);
-		return Void();
 	}
 
 	LeaderElectionRegInterface& getInterface(KeyRef key, UID id) {
 		auto i = registerInterfaces.find(key);
 		if (i == registerInterfaces.end()) {
 			Key k = key;
-			Future<Void> a = wrap(this, k, leaderRegister(registerInterfaces[k], k), id);
+			Future<Void> a =
+			    wrap(this, k, LeaderRegister::run(makeReference<LeaderRegister>(registerInterfaces[k], k)), id);
 			if (a.isError())
 				throw a.getError();
 			ASSERT(!a.isReady());
@@ -586,12 +661,12 @@ struct LeaderRegisterCollection {
 		return i->value;
 	}
 
-	ACTOR static Future<Void> wrap(LeaderRegisterCollection* self, Key key, Future<Void> actor, UID id) {
-		state Error e;
+	static Future<Void> wrap(LeaderRegisterCollection* self, Key key, Future<Void> actor, UID id) {
+		Error e;
 		try {
 			// FIXME: Get worker ID here
 			startRole(Role::COORDINATOR, id, UID());
-			wait(actor || traceRole(Role::COORDINATOR, id));
+			co_await (actor || traceRole(Role::COORDINATOR, id));
 			endRole(Role::COORDINATOR, id, "Coordinator changed");
 		} catch (Error& err) {
 			endRole(Role::COORDINATOR, id, err.what(), err.code() == error_code_actor_cancelled, err);
@@ -602,7 +677,6 @@ struct LeaderRegisterCollection {
 		self->registerInterfaces.erase(key);
 		if (e.code() != invalid_error_code)
 			throw e;
-		return Void();
 	}
 };
 
@@ -612,25 +686,26 @@ StringRef getClusterDescriptor(Key key) {
 	return str.eat(":");
 }
 
-// leaderServer multiplexes multiple leaderRegisters onto a single LeaderElectionRegInterface,
-// creating and destroying them on demand.
-ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
-                                OnDemandStore* pStore,
-                                UID id,
-                                Reference<IClusterConnectionRecord> ccr) {
-	state LeaderRegisterCollection regs(pStore);
-	state ActorCollection forwarders(false);
+class LeaderServer {
+	LeaderElectionRegInterface interf;
+	UID id;
+	Reference<IClusterConnectionRecord> ccr;
+	LeaderRegisterCollection regs;
+	ActorCollection forwarders;
 
-	wait(LeaderRegisterCollection::init(&regs));
-
-	loop choose {
-		when(CheckDescriptorMutableRequest req = waitNext(interf.checkDescriptorMutable.getFuture())) {
+	Future<Void> serveCheckDescriptorMutableRequests() {
+		while (true) {
+			CheckDescriptorMutableRequest req = co_await interf.checkDescriptorMutable.getFuture();
 			// Note the response returns the value of a knob enforced by checking only one coordinator. It is not
 			// quorum based.
 			CheckDescriptorMutableReply rep(SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT);
 			req.reply.send(rep);
 		}
-		when(OpenDatabaseCoordRequest req = waitNext(interf.openDatabase.getFuture())) {
+	}
+
+	Future<Void> serveOpenDatabaseRequests() {
+		while (true) {
+			OpenDatabaseCoordRequest req = co_await interf.openDatabase.getFuture();
 			Optional<LeaderInfo> forward = regs.getForward(req.clusterKey);
 			if (forward.present()) {
 				ClientDBInfo info;
@@ -652,7 +727,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				}
 			}
 		}
-		when(ElectionResultRequest req = waitNext(interf.electionResult.getFuture())) {
+	}
+
+	Future<Void> serveElectionResultRequests() {
+		while (true) {
+			ElectionResultRequest req = co_await interf.electionResult.getFuture();
 			Optional<LeaderInfo> forward = regs.getForward(req.key);
 			if (forward.present()) {
 				req.reply.send(forward.get());
@@ -671,7 +750,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				}
 			}
 		}
-		when(GetLeaderRequest req = waitNext(interf.getLeader.getFuture())) {
+	}
+
+	Future<Void> serveGetLeaderRequests() {
+		while (true) {
+			GetLeaderRequest req = co_await interf.getLeader.getFuture();
 			Optional<LeaderInfo> forward = regs.getForward(req.key);
 			if (forward.present())
 				req.reply.send(forward.get());
@@ -689,7 +772,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				}
 			}
 		}
-		when(CandidacyRequest req = waitNext(interf.candidacy.getFuture())) {
+	}
+
+	Future<Void> serveCandidacyRequests() {
+		while (true) {
+			CandidacyRequest req = co_await interf.candidacy.getFuture();
 			Optional<LeaderInfo> forward = regs.getForward(req.key);
 			if (forward.present())
 				req.reply.send(forward.get());
@@ -706,7 +793,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				}
 			}
 		}
-		when(LeaderHeartbeatRequest req = waitNext(interf.leaderHeartbeat.getFuture())) {
+	}
+
+	Future<Void> serveLeaderHeartbeatRequests() {
+		while (true) {
+			LeaderHeartbeatRequest req = co_await interf.leaderHeartbeat.getFuture();
 			Optional<LeaderInfo> forward = regs.getForward(req.key);
 			if (forward.present())
 				req.reply.send(LeaderHeartbeatReply{ false });
@@ -723,7 +814,11 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				}
 			}
 		}
-		when(ForwardRequest req = waitNext(interf.forward.getFuture())) {
+	}
+
+	Future<Void> serveForwardRequests() {
+		while (true) {
+			ForwardRequest req = co_await interf.forward.getFuture();
 			Optional<LeaderInfo> forward = regs.getForward(req.key);
 			if (forward.present()) {
 				req.reply.send(Void());
@@ -741,104 +836,135 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 				}
 			}
 		}
-		when(wait(forwarders.getResult())) {
-			ASSERT(false);
-			throw internal_error();
-		}
 	}
+
+	Future<Void> monitorForwarders() {
+		co_await forwarders.getResult();
+		ASSERT(false);
+		throw internal_error();
+	}
+
+public:
+	LeaderServer(LeaderElectionRegInterface interf,
+	             OnDemandStore* pStore,
+	             UID id,
+	             Reference<IClusterConnectionRecord> ccr)
+	  : interf(interf), id(id), ccr(ccr), regs(pStore), forwarders(false) {}
+
+	Future<Void> run() {
+		co_await LeaderRegisterCollection::init(&regs);
+		co_await race(serveCheckDescriptorMutableRequests(),
+		              serveOpenDatabaseRequests(),
+		              serveElectionResultRequests(),
+		              serveGetLeaderRequests(),
+		              serveCandidacyRequests(),
+		              serveLeaderHeartbeatRequests(),
+		              serveForwardRequests(),
+		              monitorForwarders());
+	}
+};
+
+// leaderServer multiplexes multiple leaderRegisters onto a single LeaderElectionRegInterface,
+// creating and destroying them on demand.
+Future<Void> leaderServer(LeaderElectionRegInterface interf,
+                          OnDemandStore* pStore,
+                          UID id,
+                          Reference<IClusterConnectionRecord> ccr) {
+	LeaderServer server(interf, pStore, id, ccr);
+	co_await server.run();
 }
 
-ACTOR Future<Void> coordinationServer(std::string dataFolder, Reference<IClusterConnectionRecord> ccr) {
-	state UID myID = deterministicRandom()->randomUniqueID();
-	state LeaderElectionRegInterface myLeaderInterface(g_network);
-	state GenerationRegInterface myInterface(g_network);
-	state OnDemandStore store(dataFolder, myID, fileCoordinatorPrefix);
+Future<Void> coordinationServer(std::string dataFolder, Reference<IClusterConnectionRecord> ccr) {
+	UID myID = deterministicRandom()->randomUniqueID();
+	LeaderElectionRegInterface myLeaderInterface(g_network);
+	GenerationRegInterface myInterface(g_network);
+	OnDemandStore store(dataFolder, myID, fileCoordinatorPrefix);
 	TraceEvent("CoordinationServer", myID)
 	    .detail("MyInterfaceAddr", myInterface.read.getEndpoint().getPrimaryAddress())
 	    .detail("Folder", dataFolder);
 
+	Error err;
 	try {
-		wait(localGenerationReg(myInterface, &store) || leaderServer(myLeaderInterface, &store, myID, ccr) ||
-		     store.getError());
+		LocalGenerationReg generationReg(myInterface, &store);
+		co_await (generationReg.run() || leaderServer(myLeaderInterface, &store, myID, ccr) || store.getError());
 		throw internal_error();
 	} catch (Error& e) {
-		state Error err = e;
-		TraceEvent("CoordinationServerError", myID).errorUnsuppressed(e);
-
-		// Handle a rare issue where the coordinator crashes during creation of
-		// its disk queue files. A disk queue consists of two files, created in
-		// the following manner:
-		//
-		//   1. Create two .part files.
-		//   2. Rename each .part file to remove its .part suffix.
-		//
-		// Step 2 can crash in between removing the .part suffix of the first
-		// and second file. If this occurs, the disk queue will refuse to open
-		// on subsequent attempts because it only sees one of its files,
-		// causing a process crash with the file_not_found error. Normally this
-		// behavior is fine, but in simulation it can occasionally cause
-		// problems. Simulation does not take into account injected errors can
-		// cause permanent process death. In most cases this is true. But if
-		// this inconsistent disk queue state occurs on a coordinator, the
-		// coordinator will enter a reboot loop, continuously failing to open
-		// its disk queue and crashing. If the simulation run has a small
-		// number of processes and another process is colocated with the
-		// coordinator, the run may get stuck because the coordinator crashing
-		// brings the other role offline as well. This has been observed in a
-		// stuck simulation run where a tlog colocated with a coordinator that
-		// ran into this issue meant the tlog replication policy couldn't be
-		// achieved.
-		//
-		// As a short term fix, catch file_not_found and fix inconsistent disk
-		// queue state on the coordinator. In the long term, we should either
-		// modify simulation to consider injected errors as fatal or allow the
-		// coordinator to manually fix disk queue state in real clusters.
-		if (g_network->isSimulated() && g_simulator->speedUpSimulation && e.code() == error_code_file_not_found) {
-			state std::vector<Future<Reference<IAsyncFile>>> fs;
-			fs.reserve(2);
-			for (int i = 0; i < 2; ++i) {
-				std::string file = joinPath(dataFolder, format("%s%d.fdq", fileCoordinatorPrefix.c_str(), i));
-				fs.push_back(
-				    IAsyncFileSystem::filesystem()->open(file,
-				                                         IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED |
-				                                             IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK,
-				                                         0));
-			}
-			wait(waitForAllReady(fs));
-
-			// Make sure one of the disk queue files is missing. This should be
-			// the only cause of the file_not_found error.
-			ASSERT(((fs[0].isError() && fs[0].getError().code() == error_code_file_not_found) ||
-			        (fs[1].isError() && fs[1].getError().code() == error_code_file_not_found)) &&
-			       fs[0].isError() != fs[1].isError());
-
-			TraceEvent(SevWarnAlways, "CoordinatorDiskQueueInconsistentState")
-			    .detail("File0Missing", fs[0].isError())
-			    .detail("File1Missing", fs[1].isError());
-			;
-
-			// Remove the remaining disk queue file to allow the coordinator to
-			// create new files on next boot.
-			if (fs[0].isError()) {
-				ASSERT(fs[0].getError().code() == error_code_file_not_found);
-				wait(IAsyncFileSystem::filesystem()->deleteFile(joinPath(dataFolder, fileCoordinatorPrefix + "1.fdq"),
-				                                                true));
-			}
-			if (fs[1].isError()) {
-				ASSERT(fs[1].getError().code() == error_code_file_not_found);
-				wait(IAsyncFileSystem::filesystem()->deleteFile(joinPath(dataFolder, fileCoordinatorPrefix + "0.fdq"),
-				                                                true));
-			}
+		err = e;
+	}
+	TraceEvent("CoordinationServerError", myID).errorUnsuppressed(err);
+
+	// Handle a rare issue where the coordinator crashes during creation of
+	// its disk queue files. A disk queue consists of two files, created in
+	// the following manner:
+	//
+	//   1. Create two .part files.
+	//   2. Rename each .part file to remove its .part suffix.
+	//
+	// Step 2 can crash in between removing the .part suffix of the first
+	// and second file. If this occurs, the disk queue will refuse to open
+	// on subsequent attempts because it only sees one of its files,
+	// causing a process crash with the file_not_found error. Normally this
+	// behavior is fine, but in simulation it can occasionally cause
+	// problems. Simulation does not take into account injected errors can
+	// cause permanent process death. In most cases this is true. But if
+	// this inconsistent disk queue state occurs on a coordinator, the
+	// coordinator will enter a reboot loop, continuously failing to open
+	// its disk queue and crashing. If the simulation run has a small
+	// number of processes and another process is colocated with the
+	// coordinator, the run may get stuck because the coordinator crashing
+	// brings the other role offline as well. This has been observed in a
+	// stuck simulation run where a tlog colocated with a coordinator that
+	// ran into this issue meant the tlog replication policy couldn't be
+	// achieved.
+	//
+	// As a short term fix, catch file_not_found and fix inconsistent disk
+	// queue state on the coordinator. In the long term, we should either
+	// modify simulation to consider injected errors as fatal or allow the
+	// coordinator to manually fix disk queue state in real clusters.
+	if (g_network->isSimulated() && g_simulator->speedUpSimulation && err.code() == error_code_file_not_found) {
+		std::vector<Future<Reference<IAsyncFile>>> fs;
+		fs.reserve(2);
+		for (int i = 0; i < 2; ++i) {
+			std::string file = joinPath(dataFolder, format("%s%d.fdq", fileCoordinatorPrefix.c_str(), i));
+			fs.push_back(IAsyncFileSystem::filesystem()->open(file,
+			                                                  IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED |
+			                                                      IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK,
+			                                                  0));
+		}
+		co_await waitForAllReady(fs);
+
+		// Make sure one of the disk queue files is missing. This should be
+		// the only cause of the file_not_found error.
+		ASSERT(((fs[0].isError() && fs[0].getError().code() == error_code_file_not_found) ||
+		        (fs[1].isError() && fs[1].getError().code() == error_code_file_not_found)) &&
+		       fs[0].isError() != fs[1].isError());
+
+		TraceEvent(SevWarnAlways, "CoordinatorDiskQueueInconsistentState")
+		    .detail("File0Missing", fs[0].isError())
+		    .detail("File1Missing", fs[1].isError());
+		;
+
+		// Remove the remaining disk queue file to allow the coordinator to
+		// create new files on next boot.
+		if (fs[0].isError()) {
+			ASSERT(fs[0].getError().code() == error_code_file_not_found);
+			co_await IAsyncFileSystem::filesystem()->deleteFile(joinPath(dataFolder, fileCoordinatorPrefix + "1.fdq"),
+			                                                    true);
+		}
+		if (fs[1].isError()) {
+			ASSERT(fs[1].getError().code() == error_code_file_not_found);
+			co_await IAsyncFileSystem::filesystem()->deleteFile(joinPath(dataFolder, fileCoordinatorPrefix + "0.fdq"),
+			                                                    true);
 		}
-
-		throw err;
 	}
+
+	throw err;
 }
 
-ACTOR Future<Void> changeClusterDescription(std::string datafolder, KeyRef newClusterKey, KeyRef oldClusterKey) {
-	state UID myID = deterministicRandom()->randomUniqueID();
-	state OnDemandStore store(datafolder, myID, fileCoordinatorPrefix);
-	RangeResult res = wait(store->readRange(allKeys));
+Future<Void> changeClusterDescription(std::string datafolder, KeyRef newClusterKey, KeyRef oldClusterKey) {
+	UID myID = deterministicRandom()->randomUniqueID();
+	OnDemandStore store(datafolder, myID, fileCoordinatorPrefix);
+	RangeResult res = co_await store->readRange(allKeys);
 	// Context, in coordinators' kv-store
 	// cluster description and the random id are always appear together as the clusterKey
 	// The old cluster key, (call it oldCKey) below can appear in the following scenarios:
@@ -867,7 +993,7 @@ ACTOR Future<Void> changeClusterDescription(std::string datafolder, KeyRef newCl
 			store->set(KeyValueRef(newClusterKey, value));
 		} else {
 			// parse the value part
-			GenerationRegVal regVal = BinaryReader::fromStringRef<GenerationRegVal>(value, IncludeVersion());
+			auto regVal = BinaryReader::fromStringRef<GenerationRegVal>(value, IncludeVersion());
 			if (regVal.val.present()) {
 				Optional<Value> newVal = updateCCSInMovableValue(regVal.val.get(), oldClusterKey, newClusterKey);
 				if (newVal.present()) {
@@ -878,8 +1004,7 @@ ACTOR Future<Void> changeClusterDescription(std::string datafolder, KeyRef newCl
 			}
 		}
 	}
-	wait(store->commit());
-	return Void();
+	co_await store->commit();
 }
 
 Future<Void> coordChangeClusterKey(std::string dataFolder, KeyRef newClusterKey, KeyRef oldClusterKey) {
diff --git a/fdbserver/OnDemandStore.actor.cpp b/fdbserver/coordinator/OnDemandStore.cpp
similarity index 88%
rename from fdbserver/OnDemandStore.actor.cpp
rename to fdbserver/coordinator/OnDemandStore.cpp
index 122868492cf..fd4258724d3 100644
--- a/fdbserver/OnDemandStore.actor.cpp
+++ b/fdbserver/coordinator/OnDemandStore.cpp
@@ -1,5 +1,5 @@
 /*
- * OnDemandStore.actor.cpp
+ * OnDemandStore.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbserver/OnDemandStore.h"
-#include "flow/actorcompiler.h" // must be last include
+#include "OnDemandStore.h"
 
-ACTOR static Future<Void> onErr(Future<Future<Void>> e) {
-	Future<Void> f = wait(e);
-	wait(f);
-	return Void();
+static Future<Void> onErr(Future<Future<Void>> e) {
+	Future<Void> f = co_await std::move(e);
+	co_await f;
 }
 
 void OnDemandStore::open() {
@@ -70,6 +68,7 @@ void OnDemandStore::dispose() {
 		store = nullptr;
 	}
 }
+
 void OnDemandStore::close() {
 	if (store) {
 		store->close();
diff --git a/fdbserver/include/fdbserver/OnDemandStore.h b/fdbserver/coordinator/OnDemandStore.h
similarity index 96%
rename from fdbserver/include/fdbserver/OnDemandStore.h
rename to fdbserver/coordinator/OnDemandStore.h
index 55ab0a6e72f..179f49d5087 100644
--- a/fdbserver/include/fdbserver/OnDemandStore.h
+++ b/fdbserver/coordinator/OnDemandStore.h
@@ -23,7 +23,7 @@
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/Platform.h"
-#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
 
 // Create a key value store if and only if it is actually used
 class OnDemandStore : public IClosable, NonCopyable {
diff --git a/fdbserver/coordinator/include/fdbserver/coordinator/CoordinationServer.h b/fdbserver/coordinator/include/fdbserver/coordinator/CoordinationServer.h
new file mode 100644
index 00000000000..2ce476a9ecf
--- /dev/null
+++ b/fdbserver/coordinator/include/fdbserver/coordinator/CoordinationServer.h
@@ -0,0 +1,30 @@
+/*
+ * CoordinationServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_COORDINATOR_COORDINATIONSERVER_H
+#define FDBSERVER_COORDINATOR_COORDINATIONSERVER_H
+#pragma once
+
+#include "fdbserver/core/CoordinationInterface.h"
+
+Future<Void> coordinationServer(std::string dataFolder, Reference<IClusterConnectionRecord> ccf);
+Future<Void> coordChangeClusterKey(std::string dataFolder, KeyRef newClusterKey, KeyRef oldClusterKey);
+
+#endif
diff --git a/fdbserver/AccumulativeChecksumUtil.cpp b/fdbserver/core/AccumulativeChecksumUtil.cpp
similarity index 99%
rename from fdbserver/AccumulativeChecksumUtil.cpp
rename to fdbserver/core/AccumulativeChecksumUtil.cpp
index e975a567720..301d053c9bc 100644
--- a/fdbserver/AccumulativeChecksumUtil.cpp
+++ b/fdbserver/core/AccumulativeChecksumUtil.cpp
@@ -18,8 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/AccumulativeChecksumUtil.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/AccumulativeChecksumUtil.h"
+#include "fdbserver/core/Knobs.h"
 
 void updateMutationWithAcsAndAddMutationToAcsBuilder(std::shared_ptr<AccumulativeChecksumBuilder> acsBuilder,
                                                      MutationRef& mutation,
@@ -181,7 +181,7 @@ Optional<AccumulativeChecksumState> AccumulativeChecksumValidator::processAccumu
 		existInTable = false;
 	}
 	// Calculate acs value by mutation buffer and compare it with acs value in acs mutation
-	if (mutationBuffer.size() == 0) {
+	if (mutationBuffer.empty()) {
 		TraceEvent(SevError, "AcsValidatorCorruptionDetected", ssid)
 		    .detail("Reason", "Mutation buffer is empty when processAccumulativeChecksum")
 		    .detail("AcsTag", tag)
diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/core/BackupProgress.cpp
similarity index 87%
rename from fdbserver/BackupProgress.actor.cpp
rename to fdbserver/core/BackupProgress.cpp
index c1fc9331785..383b2e335ab 100644
--- a/fdbserver/BackupProgress.actor.cpp
+++ b/fdbserver/core/BackupProgress.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupProgress.actor.cpp
+ * BackupProgress.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,12 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbserver/BackupProgress.actor.h"
+#include <algorithm>
+
+#include "fdbserver/core/BackupProgress.h"
 
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 void BackupProgress::addBackupStatus(const WorkerBackupStatus& status) {
 	auto& it = progress[status.epoch];
@@ -118,7 +119,6 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 				if (savedMore > 0) {
 					// The logRouterTags are the same
 					// ASSERT(info.logRouterTags == epochTags[rit->first]);
-
 					updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, adjustedBeginVersion, epoch);
 					if (tags.empty())
 						break;
@@ -144,46 +144,47 @@ std::map<std::tuple<LogEpoch, Version, int>, std::map<Tag, Version>> BackupProgr
 }
 
 // Save each tag's savedVersion for all epochs into "bStatus".
-ACTOR Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupProgress> bStatus, bool logging) {
-	state Transaction tr(cx);
+Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupProgress> bStatus, Severity severity) {
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			state Future<Optional<Value>> fValue = tr.get(backupStartedKey);
-			state RangeResult results = wait(tr.getRange(backupProgressKeys, CLIENT_KNOBS->TOO_MANY));
+			Future<Optional<Value>> fValue = tr.get(backupStartedKey);
+			RangeResult results = co_await tr.getRange(backupProgressKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
 
-			Optional<Value> value = wait(fValue);
+			Optional<Value> value = co_await fValue;
 			bStatus->setBackupStartedValue(value);
 			for (auto& it : results) {
 				const UID workerID = decodeBackupProgressKey(it.key);
 				const WorkerBackupStatus status = decodeBackupProgressValue(it.value);
 				bStatus->addBackupStatus(status);
-				if (logging) {
-					TraceEvent("GotBackupProgress", dbgid)
-					    .detail("BackupWorker", workerID)
-					    .detail("Epoch", status.epoch)
-					    .detail("Version", status.version)
-					    .detail("Tag", status.tag.toString())
-					    .detail("TotalTags", status.totalTags);
-				}
+				TraceEvent(severity, "GotBackupProgress", dbgid)
+				    .detail("BackupWorker", workerID)
+				    .detail("Epoch", status.epoch)
+				    .detail("Version", status.version)
+				    .detail("Tag", status.tag.toString())
+				    .detail("TotalTags", status.totalTags);
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+
+		co_await tr.onError(err);
 	}
 }
 
 TEST_CASE("/BackupProgress/Unfinished") {
-	std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> epochInfos;
+	std::map<LogEpoch, EpochTagsVersionsInfo> epochInfos;
 
 	const int epoch1 = 2, begin1 = 1, end1 = 100;
 	const Tag tag1(tagLocalityLogRouter, 0);
-	epochInfos.insert({ epoch1, ILogSystem::EpochTagsVersionsInfo(1, begin1, end1) });
+	epochInfos.insert({ epoch1, EpochTagsVersionsInfo(1, begin1, end1) });
 	BackupProgress progress(UID(0, 0), epochInfos);
 	progress.setBackupStartedValue(Optional<Value>("1"_sr));
 
diff --git a/fdbserver/BulkDumpUtil.actor.cpp b/fdbserver/core/BulkDumpUtil.cpp
similarity index 67%
rename from fdbserver/BulkDumpUtil.actor.cpp
rename to fdbserver/core/BulkDumpUtil.cpp
index eab796c99e9..254689ae87b 100644
--- a/fdbserver/BulkDumpUtil.actor.cpp
+++ b/fdbserver/core/BulkDumpUtil.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkDumpUtils.actor.cpp
+ * BulkDumpUtils.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,14 +22,12 @@
 #include "fdbclient/BulkLoading.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/S3Client.actor.h"
-#include "fdbserver/BulkDumpUtil.actor.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
-
+#include "fdbclient/S3Client.h"
+#include "fdbserver/core/BulkDumpUtil.h"
+#include "fdbserver/core/BulkLoadUtil.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
+#include "fdbserver/core/StorageMetrics.h"
 SSBulkDumpTask getSSBulkDumpTask(const std::map<std::string, std::vector<StorageServerInterface>>& locations,
                                  const BulkDumpState& bulkDumpState) {
 	StorageServerInterface targetServer;
@@ -112,21 +110,21 @@ void writeKVSToSSTFile(std::string filePath, std::map<Key, Value>& sortedKVS, UI
 	return;
 }
 
-ACTOR Future<BulkLoadManifest> dumpDataFileToLocalDirectory(UID logId,
-                                                            std::shared_ptr<RangeDumpRawData> rangeDumpRawData,
-                                                            BulkLoadFileSet localFileSet,
-                                                            BulkLoadFileSet remoteFileSet,
-                                                            BulkLoadByteSampleSetting byteSampleSetting,
-                                                            Version dumpVersion,
-                                                            KeyRange dumpRange,
-                                                            BulkLoadType dumpType,
-                                                            BulkLoadTransportMethod transportMethod) {
+Future<BulkLoadManifest> dumpDataFileToLocalDirectory(UID logId,
+                                                      std::shared_ptr<RangeDumpRawData> rangeDumpRawData,
+                                                      BulkLoadFileSet localFileSet,
+                                                      BulkLoadFileSet remoteFileSet,
+                                                      BulkLoadByteSampleSetting byteSampleSetting,
+                                                      Version dumpVersion,
+                                                      KeyRange dumpRange,
+                                                      BulkLoadType dumpType,
+                                                      BulkLoadTransportMethod transportMethod) {
 	// Step 1: Clean up local folder
 	resetFileFolder((abspath(localFileSet.getFolder())));
 
 	// Step 2: Dump data to file
 	bool containDataFile = false;
-	if (rangeDumpRawData->kvs.size() > 0) {
+	if (!rangeDumpRawData->kvs.empty()) {
 		writeKVSToSSTFile(abspath(localFileSet.getDataFileFullPath()), rangeDumpRawData->kvs, logId);
 		containDataFile = true;
 	} else {
@@ -136,7 +134,7 @@ ACTOR Future<BulkLoadManifest> dumpDataFileToLocalDirectory(UID logId,
 
 	// Step 3: Dump sample to file
 	bool containByteSampleFile = false;
-	if (rangeDumpRawData->sampled.size() > 0) {
+	if (!rangeDumpRawData->sampled.empty()) {
 		writeKVSToSSTFile(abspath(localFileSet.getBytesSampleFileFullPath()), rangeDumpRawData->sampled, logId);
 		containByteSampleFile = true;
 	} else {
@@ -157,19 +155,19 @@ ACTOR Future<BulkLoadManifest> dumpDataFileToLocalDirectory(UID logId,
 	                              containDataFile ? remoteFileSet.getDataFileName() : std::string(),
 	                              containByteSampleFile ? remoteFileSet.getByteSampleFileName() : std::string(),
 	                              BulkLoadChecksum());
-	state BulkLoadManifest manifestMetadata(fileSetRemote,
-	                                        dumpRange.begin,
-	                                        dumpRange.end,
-	                                        dumpVersion,
-	                                        rangeDumpRawData->kvsBytes,
-	                                        rangeDumpRawData->kvs.size(),
-	                                        byteSampleSetting,
-	                                        dumpType,
-	                                        transportMethod);
-	state std::string manifestStr = manifestMetadata.toString();
-	state std::shared_ptr<std::string> manifest = std::make_shared<std::string>(std::move(manifestStr));
-	wait(writeBulkFileBytes(abspath(localFileSet.getManifestFileFullPath()), manifest));
-	return manifestMetadata;
+	BulkLoadManifest manifestMetadata(fileSetRemote,
+	                                  dumpRange.begin,
+	                                  dumpRange.end,
+	                                  dumpVersion,
+	                                  rangeDumpRawData->kvsBytes,
+	                                  rangeDumpRawData->kvs.size(),
+	                                  byteSampleSetting,
+	                                  dumpType,
+	                                  transportMethod);
+	std::string manifestStr = manifestMetadata.toString();
+	std::shared_ptr<std::string> manifest = std::make_shared<std::string>(std::move(manifestStr));
+	co_await writeBulkFileBytes(abspath(localFileSet.getManifestFileFullPath()), manifest);
+	co_return manifestMetadata;
 }
 
 // Validate the invariant of filenames. Source is the file stored locally. Destination is the file going to move to.
@@ -197,44 +195,42 @@ bool validateSourceDestinationFileSets(const BulkLoadFileSet& source, const Bulk
 }
 
 // Copy files between local file folders, used to mock blobstore in the test.
-ACTOR Future<Void> bulkDumpTransportCP_impl(BulkLoadFileSet srcFileSet,
-                                            BulkLoadFileSet destFileSet,
-                                            size_t fileBytesMax,
-                                            UID logId) {
+Future<Void> bulkDumpTransportCP_impl(BulkLoadFileSet srcFileSet,
+                                      BulkLoadFileSet destFileSet,
+                                      size_t fileBytesMax,
+                                      UID logId) {
 	// Clear remote existing folder
 	resetFileFolder(abspath(destFileSet.getFolder()));
 	// Copy bulk dump files to the remote folder
 	ASSERT(srcFileSet.hasManifestFile() && destFileSet.hasManifestFile());
-	wait(copyBulkFile(
-	    abspath(srcFileSet.getManifestFileFullPath()), abspath(destFileSet.getManifestFileFullPath()), fileBytesMax));
+	co_await copyBulkFile(
+	    abspath(srcFileSet.getManifestFileFullPath()), abspath(destFileSet.getManifestFileFullPath()), fileBytesMax);
 	if (srcFileSet.hasDataFile()) {
 		ASSERT(destFileSet.hasDataFile());
-		wait(copyBulkFile(
-		    abspath(srcFileSet.getDataFileFullPath()), abspath(destFileSet.getDataFileFullPath()), fileBytesMax));
+		co_await copyBulkFile(
+		    abspath(srcFileSet.getDataFileFullPath()), abspath(destFileSet.getDataFileFullPath()), fileBytesMax);
 	}
 	if (srcFileSet.hasByteSampleFile()) {
 		ASSERT(srcFileSet.hasDataFile() && destFileSet.hasByteSampleFile());
-		wait(copyBulkFile(abspath(srcFileSet.getBytesSampleFileFullPath()),
-		                  abspath(destFileSet.getBytesSampleFileFullPath()),
-		                  fileBytesMax));
+		co_await copyBulkFile(abspath(srcFileSet.getBytesSampleFileFullPath()),
+		                      abspath(destFileSet.getBytesSampleFileFullPath()),
+		                      fileBytesMax);
 	}
-	return Void();
 }
 
 // Dump files to blobstore.
-ACTOR Future<Void> bulkDumpTransportBlobstore_impl(BulkLoadFileSet sourceFileSet,
-                                                   BulkLoadFileSet destinationFileSet,
-                                                   size_t fileBytesMax,
-                                                   UID logId) {
+Future<Void> bulkDumpTransportBlobstore_impl(BulkLoadFileSet sourceFileSet,
+                                             BulkLoadFileSet destinationFileSet,
+                                             size_t fileBytesMax,
+                                             UID logId) {
 	// TODO(BulkDump): Make use of fileBytesMax
-	wait(copyUpBulkDumpFileSet(destinationFileSet.getRootPath(), sourceFileSet, destinationFileSet));
-	return Void();
+	co_await copyUpBulkDumpFileSet(destinationFileSet.getRootPath(), sourceFileSet, destinationFileSet);
 }
 
-ACTOR Future<Void> uploadBulkDumpFileSet(BulkLoadTransportMethod transportMethod,
-                                         BulkLoadFileSet sourceFileSet,
-                                         BulkLoadFileSet destinationFileSet,
-                                         UID logId) {
+Future<Void> uploadBulkDumpFileSet(BulkLoadTransportMethod transportMethod,
+                                   BulkLoadFileSet sourceFileSet,
+                                   BulkLoadFileSet destinationFileSet,
+                                   UID logId) {
 	// Validate file names of source and destination
 	if (!validateSourceDestinationFileSets(sourceFileSet, destinationFileSet)) {
 		TraceEvent(SevWarnAlways, "SSBulkDumpUploadFilesError", logId)
@@ -245,33 +241,33 @@ ACTOR Future<Void> uploadBulkDumpFileSet(BulkLoadTransportMethod transportMethod
 	}
 	// Upload to blobstore or mock file copy
 	if (transportMethod == BulkLoadTransportMethod::BLOBSTORE) {
-		wait(bulkDumpTransportBlobstore_impl(
-		    sourceFileSet, destinationFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId));
+		co_await bulkDumpTransportBlobstore_impl(
+		    sourceFileSet, destinationFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId);
 	} else if (transportMethod == BulkLoadTransportMethod::CP) {
-		wait(bulkDumpTransportCP_impl(sourceFileSet, destinationFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId));
+		co_await bulkDumpTransportCP_impl(
+		    sourceFileSet, destinationFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId);
 	} else {
 		TraceEvent(SevError, "SSBulkDumpUploadFilesError", logId)
 		    .detail("Reason", "Transport method is not implemented")
 		    .detail("TransportMethod", transportMethod);
 		ASSERT(false);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> uploadBulkDumpJobManifestFile(BulkLoadTransportMethod transportMethod,
-                                                 std::string localJobManifestFilePath,
-                                                 std::string remoteFolder,
-                                                 std::string remoteJobManifestFileName,
-                                                 UID logId) {
+Future<Void> uploadBulkDumpJobManifestFile(BulkLoadTransportMethod transportMethod,
+                                           std::string localJobManifestFilePath,
+                                           std::string remoteFolder,
+                                           std::string remoteJobManifestFileName,
+                                           UID logId) {
 	auto remoteJobManifestFilePath = appendToPath(remoteFolder, remoteJobManifestFileName);
 	TraceEvent(SevInfo, "UploadBulkDumpJobManifest", logId)
 	    .detail("RemoteJobManifestFilePath", remoteJobManifestFilePath);
 	if (transportMethod == BulkLoadTransportMethod::BLOBSTORE) {
-		wait(copyUpFile(localJobManifestFilePath, remoteJobManifestFilePath));
+		co_await copyUpFile(localJobManifestFilePath, remoteJobManifestFilePath);
 	} else if (transportMethod == BulkLoadTransportMethod::CP) {
-		wait(copyBulkFile(abspath(localJobManifestFilePath),
-		                  abspath(remoteJobManifestFilePath),
-		                  SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX));
+		co_await copyBulkFile(abspath(localJobManifestFilePath),
+		                      abspath(remoteJobManifestFilePath),
+		                      SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX);
 	} else {
 		TraceEvent(SevError, "UploadBulkDumpJobManifestFileError", logId)
 		    .detail("Reason", "Transport method is not implemented")
@@ -279,25 +275,28 @@ ACTOR Future<Void> uploadBulkDumpJobManifestFile(BulkLoadTransportMethod transpo
 		ASSERT(false);
 	}
 	// TODO(BulkDump): check uploaded file exist
-	return Void();
 }
 
-ACTOR Future<Void> persistCompleteBulkDumpRange(Database cx, BulkDumpState bulkDumpState) {
-	state Transaction tr(cx);
+Future<Void> persistCompleteBulkDumpRange(Database cx, BulkDumpState bulkDumpState) {
+	Transaction tr(cx);
 	ASSERT(bulkDumpState.isMetadataValid());
-	state Key beginKey = bulkDumpState.getRange().begin;
-	state Key endKey = bulkDumpState.getRange().end;
-	state KeyRange rangeToPersist;
-	state RangeResult result;
-	loop {
+	Key beginKey = bulkDumpState.getRange().begin;
+	Key endKey = bulkDumpState.getRange().end;
+	KeyRange rangeToPersist;
+	RangeResult result;
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			rangeToPersist = Standalone(KeyRangeRef(beginKey, endKey));
-			wait(store(result, krmGetRanges(&tr, bulkDumpPrefix, rangeToPersist)));
+			result = co_await krmGetRanges(&tr, bulkDumpPrefix, rangeToPersist);
+			if (result.empty()) {
+				throw bulkdump_task_outdated();
+			}
 			bool anyNew = false;
-			for (int i = 0; i < result.size() - 1; i++) {
+			for (int i = 0; i < static_cast<int>(result.size()) - 1; i++) {
 				if (result[i].value.empty()) { // has been cancelled
 					throw bulkdump_task_outdated();
 				}
@@ -317,8 +316,8 @@ ACTOR Future<Void> persistCompleteBulkDumpRange(Database cx, BulkDumpState bulkD
 			if (!anyNew) {
 				throw bulkdump_task_outdated();
 			}
-			wait(krmSetRange(&tr, bulkDumpPrefix, bulkDumpState.getRange(), bulkDumpStateValue(bulkDumpState)));
-			wait(tr.commit());
+			co_await krmSetRange(&tr, bulkDumpPrefix, bulkDumpState.getRange(), bulkDumpStateValue(bulkDumpState));
+			co_await tr.commit();
 			beginKey = result[result.size() - 1].key;
 			if (beginKey >= endKey) {
 				break;
@@ -326,8 +325,8 @@ ACTOR Future<Void> persistCompleteBulkDumpRange(Database cx, BulkDumpState bulkD
 				tr.reset();
 			}
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbserver/BulkLoadUtil.actor.cpp b/fdbserver/core/BulkLoadUtil.cpp
similarity index 56%
rename from fdbserver/BulkLoadUtil.actor.cpp
rename to fdbserver/core/BulkLoadUtil.cpp
index 5d76270e753..49e06e6244a 100644
--- a/fdbserver/BulkLoadUtil.actor.cpp
+++ b/fdbserver/core/BulkLoadUtil.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkLoadUtils.actor.cpp
+ * BulkLoadUtil.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,21 +21,20 @@
 #include "fdbclient/BulkLoading.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/S3Client.actor.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/StorageMetrics.actor.h"
+#include "fdbclient/S3Client.h"
+#include "fdbserver/core/BulkLoadUtil.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
+#include "fdbserver/core/StorageMetrics.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
 
-ACTOR Future<Void> readBulkFileBytes(std::string path, int64_t maxLength, std::shared_ptr<std::string> output) {
+Future<Void> readBulkFileBytes(std::string path, int64_t maxLength, std::shared_ptr<std::string> output) {
 	try {
 		output->clear();
-		state int64_t chunkSize = SERVER_KNOBS->BULKLOAD_ASYNC_READ_WRITE_BLOCK_SIZE;
-		state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
-		    abspath(path), IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0644));
-		state int64_t fileSize = wait(file->size());
+		int64_t chunkSize = SERVER_KNOBS->BULKLOAD_ASYNC_READ_WRITE_BLOCK_SIZE;
+		Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
+		    abspath(path), IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0644);
+		int64_t fileSize = co_await file->size();
 		if (fileSize > maxLength) {
 			TraceEvent(SevError, "ReadBulkFileBytesTooLarge")
 			    .detail("FileSize", fileSize)
@@ -45,14 +44,14 @@ ACTOR Future<Void> readBulkFileBytes(std::string path, int64_t maxLength, std::s
 		output->reserve(fileSize); // Pre-allocate the full size
 
 		// Read in chunks to avoid memory pressure
-		state int64_t offset = 0;
-		state int64_t remaining = fileSize;
-		state std::shared_ptr<std::string> chunk = std::make_shared<std::string>();
+		int64_t offset = 0;
+		int64_t remaining = fileSize;
+		auto chunk = std::make_shared<std::string>();
 		while (remaining > 0) {
-			state int64_t bytesToRead = std::min(chunkSize, remaining);
+			int64_t bytesToRead = std::min(chunkSize, remaining);
 			chunk->clear();
 			chunk->resize(bytesToRead);
-			state int bytesRead = wait(uncancellable(holdWhile(chunk, file->read(chunk->data(), bytesToRead, offset))));
+			int bytesRead = co_await uncancellable(holdWhile(chunk, file->read(chunk->data(), bytesToRead, offset)));
 			if (bytesRead != bytesToRead) {
 				TraceEvent(SevError, "ReadBulkFileBytesError")
 				    .detail("BytesRead", bytesRead)
@@ -63,62 +62,60 @@ ACTOR Future<Void> readBulkFileBytes(std::string path, int64_t maxLength, std::s
 			offset += bytesRead;
 			remaining -= bytesRead;
 		}
-		return Void();
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "ReadBulkFileBytesError").error(e).detail("Path", path).detail("MaxLength", maxLength);
 		throw e;
 	}
 }
 
-ACTOR Future<Void> writeBulkFileBytes(std::string path, std::shared_ptr<std::string> content) {
+Future<Void> writeBulkFileBytes(std::string path, std::shared_ptr<std::string> content) {
 	try {
-		state int64_t chunkSize = SERVER_KNOBS->BULKLOAD_ASYNC_READ_WRITE_BLOCK_SIZE;
-		state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
+		int64_t chunkSize = SERVER_KNOBS->BULKLOAD_ASYNC_READ_WRITE_BLOCK_SIZE;
+		Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
 		    abspath(path),
 		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,
-		    0644));
+		    0644);
 
 		// For large files, write in chunks to avoid memory pressure
-		state int64_t offset = 0;
-		state int64_t remaining = content->size();
+		int64_t offset = 0;
+		int64_t remaining = content->size();
 
 		while (remaining > 0) {
-			state int64_t bytesToWrite = std::min(chunkSize, remaining);
-			wait(uncancellable(holdWhile(content, file->write(content->data() + offset, bytesToWrite, offset))));
+			int64_t bytesToWrite = std::min(chunkSize, remaining);
+			co_await uncancellable(holdWhile(content, file->write(content->data() + offset, bytesToWrite, offset)));
 			offset += bytesToWrite;
 			remaining -= bytesToWrite;
 		}
 
 		// Ensure the file size is correct and data is synced
-		wait(file->truncate(content->size()));
-		wait(file->sync());
-		return Void();
+		co_await file->truncate(content->size());
+		co_await file->sync();
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "WriteBulkFileBytesError").error(e).detail("Path", path);
 		throw e;
 	}
 }
 
-ACTOR Future<Void> copyBulkFile(std::string fromFile, std::string toFile, size_t fileBytesMax) {
-	state std::shared_ptr<std::string> content = std::make_shared<std::string>();
-	wait(readBulkFileBytes(abspath(fromFile), fileBytesMax, /*output=*/content));
-	wait(writeBulkFileBytes(toFile, content));
-	return Void();
+Future<Void> copyBulkFile(std::string fromFile, std::string toFile, size_t fileBytesMax) {
+	auto content = std::make_shared<std::string>();
+	co_await readBulkFileBytes(abspath(fromFile), fileBytesMax, /*output=*/content);
+	co_await writeBulkFileBytes(toFile, content);
 }
 
-ACTOR Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
-                                                                 UID dataMoveId,
-                                                                 Version atLeastVersion,
-                                                                 UID logId) {
-	state Transaction tr(cx);
-	state int retryCount = 0;
-	state int metadataRetryCount = 0;
-	state double startTime = now();
+Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
+                                                           UID dataMoveId,
+                                                           Version atLeastVersion,
+                                                           UID logId) {
+	Transaction tr(cx);
+	int retryCount = 0;
+	int metadataRetryCount = 0;
+	double startTime = now();
 	tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			state Optional<Value> val = wait(tr.get(dataMoveKeyFor(dataMoveId)));
+			Optional<Value> val = co_await tr.get(dataMoveKeyFor(dataMoveId));
 			ASSERT(tr.getReadVersion().isReady());
 			if (tr.getReadVersion().get() < atLeastVersion) {
 				retryCount++;
@@ -130,14 +127,14 @@ ACTOR Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
 					    .detail("RetryCount", retryCount)
 					    .detail("ElapsedSec", now() - startTime);
 				}
-				wait(delay(0.1));
+				co_await delay(0.1);
 				tr.reset();
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				continue;
 			}
 			if (val.present()) {
-				state DataMoveMetaData dataMoveMetaData = decodeDataMoveValue(val.get());
+				DataMoveMetaData dataMoveMetaData = decodeDataMoveValue(val.get());
 				if (dataMoveMetaData.bulkLoadTaskState.present()) {
 					if (metadataRetryCount > 0 || retryCount > 0) {
 						TraceEvent(SevInfo, "SSBulkLoadTaskGotMetadata", logId)
@@ -146,28 +143,28 @@ ACTOR Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
 						    .detail("VersionRetryCount", retryCount)
 						    .detail("ElapsedSec", now() - startTime);
 					}
-					return dataMoveMetaData.bulkLoadTaskState.get();
-				} else {
-					metadataRetryCount++;
-					if (metadataRetryCount % 100 == 0) {
-						TraceEvent(SevWarn, "SSBulkLoadTaskWaitingForMetadata", logId)
-						    .detail("DataMoveID", dataMoveId)
-						    .detail("DataMovePhase", static_cast<int>(dataMoveMetaData.getPhase()))
-						    .detail("MetadataRetryCount", metadataRetryCount)
-						    .detail("ElapsedSec", now() - startTime)
-						    .detail("Message", "DataMove exists but BulkLoadTaskState not yet written");
-					}
-					wait(delay(0.1));
-					tr.reset();
-					tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-					// DD moveShard progressively persists dataMoveMetadata based on range.
-					// Therefore, it is possible that the dataMoveMetadata is persisted at a subrange partially.
-					// Note that DD moveShard only persist the bulkload metadata in the dataMoveMetadata when all
-					// subranges have been persisted. Therefore, if the data move id indicating the move is for bulk
-					// loading, the SS should wait until the bulkLoad metadata is included in the dataMoveMetadata.
-					continue;
+					co_return dataMoveMetaData.bulkLoadTaskState.get();
 				}
+
+				metadataRetryCount++;
+				if (metadataRetryCount % 100 == 0) {
+					TraceEvent(SevWarn, "SSBulkLoadTaskWaitingForMetadata", logId)
+					    .detail("DataMoveID", dataMoveId)
+					    .detail("DataMovePhase", static_cast<int>(dataMoveMetaData.getPhase()))
+					    .detail("MetadataRetryCount", metadataRetryCount)
+					    .detail("ElapsedSec", now() - startTime)
+					    .detail("Message", "DataMove exists but BulkLoadTaskState not yet written");
+				}
+				co_await delay(0.1);
+				tr.reset();
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				// DD moveShard progressively persists dataMoveMetadata based on range.
+				// Therefore, it is possible that the dataMoveMetadata is persisted at a subrange partially.
+				// Note that DD moveShard only persist the bulkload metadata in the dataMoveMetadata when all
+				// subranges have been persisted. Therefore, if the data move id indicating the move is for bulk
+				// loading, the SS should wait until the bulkLoad metadata is included in the dataMoveMetadata.
+				continue;
 			}
 			TraceEvent(SevWarnAlways, "SSBulkLoadTaskDataMoveIdNotExist", logId)
 			    .detail("Message", "This fetchKey is blocked and will be cancelled later")
@@ -175,29 +172,31 @@ ACTOR Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
 			    .detail("ReadVersion", tr.getReadVersion().get())
 			    .detail("AtLeastVersion", atLeastVersion)
 			    .detail("ElapsedSec", now() - startTime);
-			wait(Never());
+			co_await Future<Void>(Never());
 			throw internal_error(); // does not happen
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
 // Return true if generated the byte sampling file. Otherwise, return false.
 // TODO(BulkDump): directly read from special key space.
-ACTOR Future<bool> doBytesSamplingOnDataFile(std::string dataFileFullPath, // input file
-                                             std::string byteSampleFileFullPath, // output file
-                                             UID logId) {
-	state int counter = 0;
-	state bool res = false;
-	state int retryCount = 0;
-	state double startTime = now();
-	loop {
+Future<bool> doBytesSamplingOnDataFile(std::string dataFileFullPath, // input file
+                                       std::string byteSampleFileFullPath, // output file
+                                       UID logId) {
+	int counter = 0;
+	bool res = false;
+	int retryCount = 0;
+	double startTime = now();
+	while (true) {
+		Error err;
 		try {
-			state std::unique_ptr<IRocksDBSstFileWriter> sstWriter = newRocksDBSstFileWriter();
+			std::unique_ptr<IRocksDBSstFileWriter> sstWriter = newRocksDBSstFileWriter();
 			sstWriter->open(abspath(byteSampleFileFullPath));
-			state bool anySampled = false;
-			state std::unique_ptr<IRocksDBSstFileReader> reader = newRocksDBSstFileReader();
+			bool anySampled = false;
+			std::unique_ptr<IRocksDBSstFileReader> reader = newRocksDBSstFileReader();
 			reader->open(abspath(dataFileFullPath));
 			while (reader->hasNext()) {
 				KeyValue kv = reader->next();
@@ -207,7 +206,7 @@ ACTOR Future<bool> doBytesSamplingOnDataFile(std::string dataFileFullPath, // in
 					anySampled = true;
 					counter++;
 					if (counter > SERVER_KNOBS->BULKLOAD_BYTE_SAMPLE_BATCH_KEY_COUNT) {
-						wait(yield());
+						co_await yield();
 						counter = 0;
 					}
 				}
@@ -224,26 +223,27 @@ ACTOR Future<bool> doBytesSamplingOnDataFile(std::string dataFileFullPath, // in
 			}
 			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			TraceEvent(SevWarn, "SSBulkLoadTaskSamplingError", logId)
-			    .errorUnsuppressed(e)
-			    .detail("DataFileFullPath", dataFileFullPath)
-			    .detail("ByteSampleFileFullPath", byteSampleFileFullPath)
-			    .detail("Duration", now() - startTime)
-			    .detail("RetryCount", retryCount);
-			wait(delay(5.0));
-			deleteFile(abspath(byteSampleFileFullPath));
-			retryCount++;
+			err = e;
 		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		TraceEvent(SevWarn, "SSBulkLoadTaskSamplingError", logId)
+		    .errorUnsuppressed(err)
+		    .detail("DataFileFullPath", dataFileFullPath)
+		    .detail("ByteSampleFileFullPath", byteSampleFileFullPath)
+		    .detail("Duration", now() - startTime)
+		    .detail("RetryCount", retryCount);
+		co_await delay(5.0);
+		deleteFile(abspath(byteSampleFileFullPath));
+		retryCount++;
 	}
 	TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskSamplingComplete", logId)
 	    .detail("DataFileFullPath", dataFileFullPath)
 	    .detail("ByteSampleFileFullPath", byteSampleFileFullPath)
 	    .detail("Duration", now() - startTime)
 	    .detail("RetryCount", retryCount);
-	return res;
+	co_return res;
 }
 
 // TODO(BulkLoad): slow task
@@ -271,29 +271,28 @@ void resetFileFolder(const std::string& folderPath) {
 	return;
 }
 
-ACTOR Future<Void> bulkLoadTransportCP_impl(BulkLoadFileSet fromRemoteFileSet,
-                                            BulkLoadFileSet toLocalFileSet,
-                                            size_t fileBytesMax,
-                                            UID logId) {
+Future<Void> bulkLoadTransportCP_impl(BulkLoadFileSet fromRemoteFileSet,
+                                      BulkLoadFileSet toLocalFileSet,
+                                      size_t fileBytesMax,
+                                      UID logId) {
 	// Clear existing local folder
 	resetFileFolder(abspath(toLocalFileSet.getFolder()));
 	// Copy data file
-	wait(copyBulkFile(
-	    abspath(fromRemoteFileSet.getDataFileFullPath()), abspath(toLocalFileSet.getDataFileFullPath()), fileBytesMax));
+	co_await copyBulkFile(
+	    abspath(fromRemoteFileSet.getDataFileFullPath()), abspath(toLocalFileSet.getDataFileFullPath()), fileBytesMax);
 	// Copy byte sample file if exists
 	if (fromRemoteFileSet.hasByteSampleFile()) {
-		wait(copyBulkFile(abspath(fromRemoteFileSet.getBytesSampleFileFullPath()),
-		                  abspath(toLocalFileSet.getBytesSampleFileFullPath()),
-		                  fileBytesMax));
+		co_await copyBulkFile(abspath(fromRemoteFileSet.getBytesSampleFileFullPath()),
+		                      abspath(toLocalFileSet.getBytesSampleFileFullPath()),
+		                      fileBytesMax);
 	}
 	// TODO(BulkLoad): Throw error if the date/bytesample file does not exist while the filename is not empty
-	return Void();
 }
 
-ACTOR Future<Void> bulkLoadTransportBlobstore_impl(BulkLoadFileSet fromRemoteFileSet,
-                                                   BulkLoadFileSet toLocalFileSet,
-                                                   size_t fileBytesMax,
-                                                   UID logId) {
+Future<Void> bulkLoadTransportBlobstore_impl(BulkLoadFileSet fromRemoteFileSet,
+                                             BulkLoadFileSet toLocalFileSet,
+                                             size_t fileBytesMax,
+                                             UID logId) {
 	// Clear existing local folder
 	resetFileFolder(abspath(toLocalFileSet.getFolder()));
 	TraceEvent(SevDebug, "BulkLoadBlobstoreTransportStart", logId)
@@ -307,7 +306,7 @@ ACTOR Future<Void> bulkLoadTransportBlobstore_impl(BulkLoadFileSet fromRemoteFil
 		TraceEvent(SevDebug, "BulkLoadBlobstoreBeforeCopyDataFile", logId)
 		    .detail("FromPath", fromRemoteFileSet.getDataFileFullPath())
 		    .detail("ToPath", abspath(toLocalFileSet.getDataFileFullPath()));
-		wait(copyDownFile(fromRemoteFileSet.getDataFileFullPath(), abspath(toLocalFileSet.getDataFileFullPath())));
+		co_await copyDownFile(fromRemoteFileSet.getDataFileFullPath(), abspath(toLocalFileSet.getDataFileFullPath()));
 		TraceEvent(SevDebug, "BulkLoadBlobstoreAfterCopyDataFile", logId);
 	} else {
 		TraceEvent("BulkLoadBlobstoreSkipEmptyRange", logId).detail("Reason", "No data file for empty range");
@@ -317,38 +316,38 @@ ACTOR Future<Void> bulkLoadTransportBlobstore_impl(BulkLoadFileSet fromRemoteFil
 		TraceEvent(SevDebug, "BulkLoadBlobstoreBeforeCopySampleFile", logId)
 		    .detail("FromPath", fromRemoteFileSet.getBytesSampleFileFullPath())
 		    .detail("ToPath", abspath(toLocalFileSet.getBytesSampleFileFullPath()));
-		wait(copyDownFile(fromRemoteFileSet.getBytesSampleFileFullPath(),
-		                  abspath(toLocalFileSet.getBytesSampleFileFullPath())));
+		co_await copyDownFile(fromRemoteFileSet.getBytesSampleFileFullPath(),
+		                      abspath(toLocalFileSet.getBytesSampleFileFullPath()));
 		TraceEvent(SevDebug, "BulkLoadBlobstoreAfterCopySampleFile", logId);
 	}
 	// TODO(BulkLoad): Throw error if the date/bytesample file does not exist while the filename is not empty
 	TraceEvent(SevDebug, "BulkLoadBlobstoreTransportEnd", logId);
-	return Void();
 }
 
-ACTOR Future<BulkLoadFileSet> bulkLoadDownloadTaskFileSet(BulkLoadTransportMethod transportMethod,
-                                                          BulkLoadFileSet fromRemoteFileSet,
-                                                          std::string toLocalRoot,
-                                                          UID logId) {
-	state int retryCount = 0;
-	state double startTime = now();
+Future<BulkLoadFileSet> bulkLoadDownloadTaskFileSet(BulkLoadTransportMethod transportMethod,
+                                                    BulkLoadFileSet fromRemoteFileSet,
+                                                    std::string toLocalRoot,
+                                                    UID logId) {
+	int retryCount = 0;
+	double startTime = now();
 	ASSERT(transportMethod != BulkLoadTransportMethod::Invalid);
 	TraceEvent(SevDebug, "BulkLoadDownloadTaskFileSetStart", logId)
 	    .detail("FromRemoteFileSet", fromRemoteFileSet.toString())
 	    .detail("ToLocalRoot", toLocalRoot)
 	    .detail("TransportMethod", transportMethod);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			TraceEvent(SevDebug, "BulkLoadDownloadTaskFileSetAttempt", logId)
 			    .detail("RetryCount", retryCount)
 			    .detail("Elapsed", now() - startTime);
 			// Step 1: Generate local file set based on remote file set by replacing the remote root to the local root.
-			state BulkLoadFileSet toLocalFileSet(toLocalRoot,
-			                                     fromRemoteFileSet.getRelativePath(),
-			                                     fromRemoteFileSet.getManifestFileName(),
-			                                     fromRemoteFileSet.getDataFileName(),
-			                                     fromRemoteFileSet.getByteSampleFileName(),
-			                                     BulkLoadChecksum());
+			BulkLoadFileSet toLocalFileSet(toLocalRoot,
+			                               fromRemoteFileSet.getRelativePath(),
+			                               fromRemoteFileSet.getManifestFileName(),
+			                               fromRemoteFileSet.getDataFileName(),
+			                               fromRemoteFileSet.getByteSampleFileName(),
+			                               BulkLoadChecksum());
 
 			// Step 2: Download remote file set to local folder
 			if (transportMethod == BulkLoadTransportMethod::CP) {
@@ -356,15 +355,15 @@ ACTOR Future<BulkLoadFileSet> bulkLoadDownloadTaskFileSet(BulkLoadTransportMetho
 				TraceEvent(SevDebug, "BulkLoadDownloadBeforeCP", logId).detail("Elapsed", now() - startTime);
 				// Copy the data file and the sample file from remote folder to a local folder specified by
 				// fromRemoteFileSet.
-				wait(bulkLoadTransportCP_impl(
-				    fromRemoteFileSet, toLocalFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId));
+				co_await bulkLoadTransportCP_impl(
+				    fromRemoteFileSet, toLocalFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId);
 				TraceEvent(SevDebug, "BulkLoadDownloadAfterCP", logId).detail("Elapsed", now() - startTime);
 			} else if (transportMethod == BulkLoadTransportMethod::BLOBSTORE) {
 				TraceEvent(SevDebug, "BulkLoadDownloadBeforeBlobstore", logId).detail("Elapsed", now() - startTime);
 				// Copy the data file and the sample file from remote folder to a local folder specified by
 				// fromRemoteFileSet.
-				wait(bulkLoadTransportBlobstore_impl(
-				    fromRemoteFileSet, toLocalFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId));
+				co_await bulkLoadTransportBlobstore_impl(
+				    fromRemoteFileSet, toLocalFileSet, SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, logId);
 				TraceEvent(SevDebug, "BulkLoadDownloadAfterBlobstore", logId).detail("Elapsed", now() - startTime);
 			} else {
 				UNREACHABLE();
@@ -374,47 +373,45 @@ ACTOR Future<BulkLoadFileSet> bulkLoadDownloadTaskFileSet(BulkLoadTransportMetho
 			    .detail("ToLocalRoot", toLocalRoot)
 			    .detail("Duration", now() - startTime)
 			    .detail("RetryCount", retryCount);
-
-			return toLocalFileSet;
-
+			co_return toLocalFileSet;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			TraceEvent(SevWarn, "SSBulkLoadTaskDownloadFileSetError", logId)
-			    .errorUnsuppressed(e)
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		TraceEvent(SevWarn, "SSBulkLoadTaskDownloadFileSetError", logId)
+		    .errorUnsuppressed(err)
+		    .detail("FromRemoteFileSet", fromRemoteFileSet.toString())
+		    .detail("ToLocalRoot", toLocalRoot)
+		    .detail("Duration", now() - startTime)
+		    .detail("RetryCount", retryCount);
+		retryCount++;
+		if (retryCount > SERVER_KNOBS->BULKLOAD_DOWNLOAD_MAX_RETRIES) {
+			TraceEvent(SevError, "SSBulkLoadTaskDownloadFileSetMaxRetriesExceeded", logId)
+			    .errorUnsuppressed(err)
 			    .detail("FromRemoteFileSet", fromRemoteFileSet.toString())
 			    .detail("ToLocalRoot", toLocalRoot)
 			    .detail("Duration", now() - startTime)
-			    .detail("RetryCount", retryCount);
-			retryCount++;
-			if (retryCount > SERVER_KNOBS->BULKLOAD_DOWNLOAD_MAX_RETRIES) {
-				TraceEvent(SevError, "SSBulkLoadTaskDownloadFileSetMaxRetriesExceeded", logId)
-				    .errorUnsuppressed(e)
-				    .detail("FromRemoteFileSet", fromRemoteFileSet.toString())
-				    .detail("ToLocalRoot", toLocalRoot)
-				    .detail("Duration", now() - startTime)
-				    .detail("RetryCount", retryCount)
-				    .detail("MaxRetries", SERVER_KNOBS->BULKLOAD_DOWNLOAD_MAX_RETRIES)
-				    .detail("OriginalError", e.code())
-				    .detail("OriginalErrorName", e.name());
-				// Throw bulkload_task_failed to signal fetchKeys that this is a retryable bulk load error
-				// This allows the data movement to be retried at the DD level instead of killing the SS
-				throw bulkload_task_failed();
-			}
-			wait(delay(SERVER_KNOBS->BULKLOAD_DOWNLOAD_RETRY_DELAY));
+			    .detail("RetryCount", retryCount)
+			    .detail("MaxRetries", SERVER_KNOBS->BULKLOAD_DOWNLOAD_MAX_RETRIES)
+			    .detail("OriginalError", err.code())
+			    .detail("OriginalErrorName", err.name());
+			// Throw bulkload_task_failed to signal fetchKeys that this is a retryable bulk load error
+			// This allows the data movement to be retried at the DD level instead of killing the SS
+			throw bulkload_task_failed();
 		}
+		co_await delay(SERVER_KNOBS->BULKLOAD_DOWNLOAD_RETRY_DELAY);
 	}
 }
 
-ACTOR Future<Void> bulkLoadDownloadTaskFileSets(BulkLoadTransportMethod transportMethod,
-                                                std::shared_ptr<BulkLoadFileSetKeyMap> fromRemoteFileSets,
-                                                std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets,
-                                                std::string toLocalRoot,
-                                                UID logId) {
-	state BulkLoadFileSetKeyMap::iterator iter = fromRemoteFileSets->begin();
-	state KeyRange keys;
-	for (; iter != fromRemoteFileSets->end(); iter++) {
+Future<Void> bulkLoadDownloadTaskFileSets(BulkLoadTransportMethod transportMethod,
+                                          std::shared_ptr<BulkLoadFileSetKeyMap> fromRemoteFileSets,
+                                          std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets,
+                                          std::string toLocalRoot,
+                                          UID logId) {
+	KeyRange keys;
+	for (auto iter = fromRemoteFileSets->begin(); iter != fromRemoteFileSets->end(); iter++) {
 		keys = iter->first;
 		if (!iter->second.hasDataFile()) {
 			// For empty ranges (no data file), create an empty local fileSet entry so FetchKeys knows this range was
@@ -433,34 +430,34 @@ ACTOR Future<Void> bulkLoadDownloadTaskFileSets(BulkLoadTransportMethod transpor
 			continue;
 		}
 		BulkLoadFileSet localFileSet =
-		    wait(bulkLoadDownloadTaskFileSet(transportMethod, iter->second, toLocalRoot, logId));
+		    co_await bulkLoadDownloadTaskFileSet(transportMethod, iter->second, toLocalRoot, logId);
 		localFileSets->push_back(std::make_pair(keys, localFileSet));
 	}
-	return Void();
 }
 
-ACTOR Future<Void> downloadManifestFile(BulkLoadTransportMethod transportMethod,
-                                        std::string fromRemotePath,
-                                        std::string toLocalPath,
-                                        UID logId) {
-	state int retryCount = 0;
-	state double startTime = now();
+Future<Void> downloadManifestFile(BulkLoadTransportMethod transportMethod,
+                                  std::string fromRemotePath,
+                                  std::string toLocalPath,
+                                  UID logId) {
+	int retryCount = 0;
+	double startTime = now();
 	TraceEvent(SevDebug, "BulkLoadDownloadManifestStart", logId)
 	    .detail("FromRemotePath", fromRemotePath)
 	    .detail("ToLocalPath", toLocalPath)
 	    .detail("TransportMethod", transportMethod);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			TraceEvent(SevDebug, "BulkLoadDownloadManifestAttempt", logId)
 			    .detail("RetryCount", retryCount)
 			    .detail("Elapsed", now() - startTime);
 			if (transportMethod == BulkLoadTransportMethod::CP) {
-				wait(
-				    copyBulkFile(abspath(fromRemotePath), abspath(toLocalPath), SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX));
-				wait(delay(0.1));
+				co_await copyBulkFile(
+				    abspath(fromRemotePath), abspath(toLocalPath), SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX);
+				co_await delay(0.1);
 			} else if (transportMethod == BulkLoadTransportMethod::BLOBSTORE) {
 				// TODO: Make use of fileBytesMax
-				wait(copyDownFile(fromRemotePath, abspath(toLocalPath)));
+				co_await copyDownFile(fromRemotePath, abspath(toLocalPath));
 			} else {
 				UNREACHABLE();
 			}
@@ -474,34 +471,33 @@ ACTOR Future<Void> downloadManifestFile(BulkLoadTransportMethod transportMethod,
 			    .detail("RetryCount", retryCount);
 			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			TraceEvent(SevWarnAlways, "BulkLoadDownloadManifestFileError", logId)
-			    .errorUnsuppressed(e)
-			    .detail("TransportMethod", transportMethod)
-			    .detail("FromRemotePath", fromRemotePath)
-			    .detail("ToLocalPath", toLocalPath)
-			    .detail("RetryCount", retryCount)
-			    .detail("Duration", now() - startTime);
-			retryCount++;
-			if (retryCount > 10) {
-				throw e;
-			}
-			wait(delay(SERVER_KNOBS->BULKLOAD_DOWNLOAD_RETRY_DELAY));
+			err = e;
 		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		TraceEvent(SevWarnAlways, "BulkLoadDownloadManifestFileError", logId)
+		    .errorUnsuppressed(err)
+		    .detail("TransportMethod", transportMethod)
+		    .detail("FromRemotePath", fromRemotePath)
+		    .detail("ToLocalPath", toLocalPath)
+		    .detail("RetryCount", retryCount)
+		    .detail("Duration", now() - startTime);
+		retryCount++;
+		if (retryCount > 10) {
+			throw err;
+		}
+		co_await delay(SERVER_KNOBS->BULKLOAD_DOWNLOAD_RETRY_DELAY);
 	}
-	return Void();
 }
 
 // Download job manifest file
 // Each job has one manifest file including manifest paths of all tasks
-ACTOR Future<Void> downloadBulkLoadJobManifestFile(BulkLoadTransportMethod transportMethod,
-                                                   std::string localJobManifestFilePath,
-                                                   std::string remoteJobManifestFilePath,
-                                                   UID logId) {
-	wait(downloadManifestFile(transportMethod, remoteJobManifestFilePath, localJobManifestFilePath, logId));
-	return Void();
+Future<Void> downloadBulkLoadJobManifestFile(BulkLoadTransportMethod transportMethod,
+                                             std::string localJobManifestFilePath,
+                                             std::string remoteJobManifestFilePath,
+                                             UID logId) {
+	co_await downloadManifestFile(transportMethod, remoteJobManifestFilePath, localJobManifestFilePath, logId);
 }
 
 // Update manifestEntryMap and expanding mapRange if the input line overlaps with the job range.
@@ -531,22 +527,22 @@ KeyRange updateManifestEntryMap(const std::string& line,
 // Get manifest within the input range.
 // manifestEntryMap is the output.
 // Return value is the range of the manifestEntryMap.
-ACTOR Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
+Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
     std::string localJobManifestFilePath,
     KeyRange range,
     UID logId,
     std::shared_ptr<BulkLoadManifestFileMap> manifestEntryMap) {
-	state KeyRange mapRange;
-	state double startTime = now();
+	KeyRange mapRange;
+	double startTime = now();
 	ASSERT(fileExists(abspath(localJobManifestFilePath)));
 	ASSERT(manifestEntryMap->empty());
 
-	state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
+	Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
 	    abspath(localJobManifestFilePath),
 	    IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED,
-	    0644));
+	    0644);
 
-	state int64_t fileSize = wait(file->size());
+	int64_t fileSize = co_await file->size();
 	if (fileSize > SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX) {
 		TraceEvent(SevError, "ManifestFileTooBig", logId)
 		    .detail("FileSize", fileSize)
@@ -554,18 +550,18 @@ ACTOR Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
 		throw file_too_large();
 	}
 
-	state int64_t chunkSize = 64 * 1024; // 64KB chunks
-	state std::string buffer;
-	state int64_t offset = 0;
-	state std::string leftover;
-	state bool headerProcessed = false;
+	int64_t chunkSize = 64 * 1024; // 64KB chunks
+	std::string buffer;
+	int64_t offset = 0;
+	std::string leftover;
+	bool headerProcessed = false;
 
 	try {
 		while (offset < fileSize) {
-			state int64_t bytesToRead = std::min(chunkSize, fileSize - offset);
+			int64_t bytesToRead = std::min(chunkSize, fileSize - offset);
 			buffer.resize(bytesToRead);
 
-			state int bytesRead = wait(file->read(&buffer[0], bytesToRead, offset));
+			int bytesRead = co_await file->read(&buffer[0], bytesToRead, offset);
 			if (bytesRead != bytesToRead) {
 				TraceEvent(SevError, "ReadFileError", logId)
 				    .detail("BytesRead", bytesRead)
@@ -574,12 +570,12 @@ ACTOR Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
 			}
 
 			// Process the chunk line by line
-			state std::string chunk = leftover + buffer;
-			state size_t pos = 0;
-			state size_t lineStart = 0;
+			std::string chunk = leftover + buffer;
+			size_t pos = 0;
+			size_t lineStart = 0;
 
 			while ((pos = chunk.find(bulkLoadJobManifestLineTerminator, lineStart)) != std::string::npos) {
-				state std::string line = chunk.substr(lineStart, pos - lineStart);
+				std::string line = chunk.substr(lineStart, pos - lineStart);
 				if (!line.empty()) {
 					if (!headerProcessed) {
 						// First line is header
@@ -593,14 +589,14 @@ ACTOR Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
 
 				// Yield every 100 entries
 				if (manifestEntryMap->size() % 100 == 0) {
-					wait(yield());
+					co_await yield();
 				}
 			}
 
 			// Save any partial line for the next chunk
 			leftover = chunk.substr(lineStart);
 			offset += bytesRead;
-			wait(yield());
+			co_await yield();
 		}
 
 		// Process the last line if it didn't end with a newline
@@ -621,29 +617,28 @@ ACTOR Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
 		throw e;
 	}
 
-	return mapRange;
+	co_return mapRange;
 }
 
-ACTOR Future<BulkLoadManifestSet> getBulkLoadManifestMetadataFromEntry(
+Future<BulkLoadManifestSet> getBulkLoadManifestMetadataFromEntry(
     std::vector<BulkLoadJobFileManifestEntry> manifestEntries,
     std::string manifestLocalTempFolder,
     BulkLoadTransportMethod transportMethod,
     std::string jobRoot,
     UID logId) {
-	state BulkLoadManifestSet manifests(SERVER_KNOBS->MANIFEST_COUNT_MAX_PER_BULKLOAD_TASK);
-	state int i = 0;
-	for (; i < manifestEntries.size(); i++) {
-		state std::string remoteManifestFilePath = appendToPath(jobRoot, manifestEntries[i].getManifestRelativePath());
-		state std::string localManifestFilePath = joinPath(manifestLocalTempFolder,
-		                                                   deterministicRandom()->randomUniqueID().toString() + "-" +
-		                                                       basename(getPath(remoteManifestFilePath)));
-		wait(downloadManifestFile(transportMethod, remoteManifestFilePath, localManifestFilePath, logId));
-		state std::shared_ptr<std::string> manifestRawString = std::make_shared<std::string>();
-		wait(readBulkFileBytes(
-		    abspath(localManifestFilePath), SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, manifestRawString));
+	BulkLoadManifestSet manifests(SERVER_KNOBS->MANIFEST_COUNT_MAX_PER_BULKLOAD_TASK);
+	for (int i = 0; i < manifestEntries.size(); ++i) {
+		std::string remoteManifestFilePath = appendToPath(jobRoot, manifestEntries[i].getManifestRelativePath());
+		std::string localManifestFilePath = joinPath(manifestLocalTempFolder,
+		                                             deterministicRandom()->randomUniqueID().toString() + "-" +
+		                                                 basename(getPath(remoteManifestFilePath)));
+		co_await downloadManifestFile(transportMethod, remoteManifestFilePath, localManifestFilePath, logId);
+		auto manifestRawString = std::make_shared<std::string>();
+		co_await readBulkFileBytes(
+		    abspath(localManifestFilePath), SERVER_KNOBS->BULKLOAD_FILE_BYTES_MAX, manifestRawString);
 		ASSERT(!manifestRawString->empty());
 		BulkLoadManifest manifest(*manifestRawString);
 		ASSERT(manifests.addManifest(manifest));
 	}
-	return manifests;
+	co_return manifests;
 }
diff --git a/fdbserver/core/CMakeLists.txt b/fdbserver/core/CMakeLists.txt
new file mode 100644
index 00000000000..6d7d3eac615
--- /dev/null
+++ b/fdbserver/core/CMakeLists.txt
@@ -0,0 +1,34 @@
+fdb_find_sources(FDBSERVER_CORE_LOCAL_SRCS)
+
+set(FDBSERVER_CORE_SRCS ${FDBSERVER_CORE_LOCAL_SRCS})
+if(${COROUTINE_IMPL} STREQUAL libcoro)
+  list(APPEND FDBSERVER_CORE_SRCS ../coroimpl/CoroFlowCoro.actor.cpp)
+else()
+  list(APPEND FDBSERVER_CORE_SRCS ../coroimpl/CoroFlow.actor.cpp)
+endif()
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_core SRCS ${FDBSERVER_CORE_SRCS})
+add_fdbserver_link_test(fdbserver_corelinktest
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_core)
+target_include_directories(fdbserver_core
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+  PRIVATE
+    ${CMAKE_SOURCE_DIR}/fdbserver/include
+    ${CMAKE_BINARY_DIR}/fdbserver/include)
+target_link_libraries(fdbserver_core PUBLIC fdbclient)
+
+if(WITH_ROCKSDB)
+  add_dependencies(fdbserver_core rocksdb)
+  if(WITH_LIBURING)
+    target_include_directories(fdbserver_core PRIVATE ${ROCKSDB_INCLUDE_DIR} ${uring_INCLUDE_DIR})
+    target_link_libraries(fdbserver_core PRIVATE ${ROCKSDB_LIBRARIES} ${uring_LIBRARIES} ${LZ4_LIBRARY})
+  else()
+    target_include_directories(fdbserver_core PRIVATE ${ROCKSDB_INCLUDE_DIR})
+    target_link_libraries(fdbserver_core PRIVATE ${ROCKSDB_LIBRARIES} ${LZ4_LIBRARY})
+  endif()
+  target_compile_definitions(fdbserver_core PUBLIC WITH_ROCKSDB)
+endif()
diff --git a/fdbserver/CoordinatedState.actor.cpp b/fdbserver/core/CoordinatedState.cpp
similarity index 64%
rename from fdbserver/CoordinatedState.actor.cpp
rename to fdbserver/core/CoordinatedState.cpp
index e98822df970..658fbfcbc68 100644
--- a/fdbserver/CoordinatedState.actor.cpp
+++ b/fdbserver/core/CoordinatedState.cpp
@@ -1,5 +1,5 @@
 /*
- * CoordinatedState.actor.cpp
+ * CoordinatedState.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,53 +19,53 @@
  */
 
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbserver/CoordinatedState.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/CoordinatedState.h"
+#include "fdbserver/core/CoordinationInterface.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/ActorCollection.h"
-#include "fdbserver/LeaderElection.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/core/LeaderElection.h"
+#include "flow/CoroUtils.h"
 
-ACTOR Future<GenerationRegReadReply> waitAndSendRead(GenerationRegInterface stateServer, GenerationRegReadRequest req) {
+Future<GenerationRegReadReply> waitAndSendRead(GenerationRegInterface stateServer, GenerationRegReadRequest req) {
 	if (SERVER_KNOBS->BUGGIFY_ALL_COORDINATION || BUGGIFY)
-		wait(delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01()));
-	state GenerationRegReadReply reply;
+		co_await delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01());
+	GenerationRegReadReply reply;
 	if (stateServer.hostname.present()) {
-		wait(store(reply, retryGetReplyFromHostname(req, stateServer.hostname.get(), WLTOKEN_GENERATIONREG_READ)));
+		reply = co_await retryGetReplyFromHostname(req, stateServer.hostname.get(), WLTOKEN_GENERATIONREG_READ);
 	} else {
-		wait(store(reply, retryBrokenPromise(stateServer.read, req)));
+		reply = co_await retryBrokenPromise(stateServer.read, req);
 	}
 	if (SERVER_KNOBS->BUGGIFY_ALL_COORDINATION || BUGGIFY)
-		wait(delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01()));
-	return reply;
+		co_await delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01());
+	co_return reply;
 }
 
-ACTOR Future<UniqueGeneration> waitAndSendWrite(GenerationRegInterface stateServer, GenerationRegWriteRequest req) {
+Future<UniqueGeneration> waitAndSendWrite(GenerationRegInterface stateServer, GenerationRegWriteRequest req) {
 	if (SERVER_KNOBS->BUGGIFY_ALL_COORDINATION || BUGGIFY)
-		wait(delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01()));
-	state UniqueGeneration reply;
+		co_await delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01());
+	UniqueGeneration reply;
 	if (stateServer.hostname.present()) {
-		wait(store(reply, retryGetReplyFromHostname(req, stateServer.hostname.get(), WLTOKEN_GENERATIONREG_WRITE)));
+		reply = co_await retryGetReplyFromHostname(req, stateServer.hostname.get(), WLTOKEN_GENERATIONREG_WRITE);
 	} else {
-		wait(store(reply, retryBrokenPromise(stateServer.write, req)));
+		reply = co_await retryBrokenPromise(stateServer.write, req);
 	}
 	if (SERVER_KNOBS->BUGGIFY_ALL_COORDINATION || BUGGIFY)
-		wait(delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01()));
-	return reply;
+		co_await delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01());
+	co_return reply;
 }
 
-ACTOR Future<GenerationRegReadReply> emptyToNever(Future<GenerationRegReadReply> f) {
-	state GenerationRegReadReply r = wait(f);
+Future<GenerationRegReadReply> emptyToNever(Future<GenerationRegReadReply> f) {
+	GenerationRegReadReply r = co_await f;
 	if (r.gen.generation == 0)
-		wait(Future<Void>(Never()));
-	return r;
+		co_await Future<Void>(Never());
+	co_return r;
 }
 
-ACTOR Future<GenerationRegReadReply> nonemptyToNever(Future<GenerationRegReadReply> f) {
-	state GenerationRegReadReply r = wait(f);
+Future<GenerationRegReadReply> nonemptyToNever(Future<GenerationRegReadReply> f) {
+	GenerationRegReadReply r = co_await f;
 	if (r.gen.generation != 0)
-		wait(Future<Void>(Never()));
-	return r;
+		co_await Future<Void>(Never());
+	co_return r;
 }
 
 struct CoordinatedStateImpl {
@@ -77,7 +77,7 @@ struct CoordinatedStateImpl {
 	ActorCollection ac; // Errors are not reported
 	bool initial;
 
-	CoordinatedStateImpl(ServerCoordinators const& c)
+	explicit CoordinatedStateImpl(ServerCoordinators const& c)
 	  : coordinators(c), stage(0), conflictGen(0), doomed(false), ac(false), initial(false) {}
 	uint64_t getConflict() const { return conflictGen; }
 
@@ -88,13 +88,13 @@ struct CoordinatedStateImpl {
 		// || rep.rgen > gen // setExclusive isn't absolutely doomed, but it may/probably will fail
 	}
 
-	ACTOR static Future<Value> read(CoordinatedStateImpl* self) {
+	static Future<Value> read(CoordinatedStateImpl* self) {
 		ASSERT(self->stage == 0);
 
 		{
 			self->stage = 1;
-			GenerationRegReadReply rep = wait(self->replicatedRead(
-			    self, GenerationRegReadRequest(self->coordinators.clusterKey, UniqueGeneration())));
+			GenerationRegReadReply rep = co_await self->replicatedRead(
+			    self, GenerationRegReadRequest(self->coordinators.clusterKey, UniqueGeneration()));
 			self->conflictGen = std::max(self->conflictGen, std::max(rep.gen.generation, rep.rgen.generation)) + 1;
 			self->gen = UniqueGeneration(self->conflictGen, deterministicRandom()->randomUniqueID());
 		}
@@ -102,7 +102,7 @@ struct CoordinatedStateImpl {
 		{
 			self->stage = 2;
 			GenerationRegReadReply rep =
-			    wait(self->replicatedRead(self, GenerationRegReadRequest(self->coordinators.clusterKey, self->gen)));
+			    co_await self->replicatedRead(self, GenerationRegReadRequest(self->coordinators.clusterKey, self->gen));
 			self->stage = 3;
 			self->conflictGen = std::max(self->conflictGen, std::max(rep.gen.generation, rep.rgen.generation));
 			if (self->isDoomed(rep))
@@ -110,32 +110,31 @@ struct CoordinatedStateImpl {
 			self->initial = rep.gen.generation == 0;
 
 			self->stage = 4;
-			return rep.value.present() ? rep.value.get() : Value();
+			co_return rep.value.present() ? rep.value.get() : Value();
 		}
 	}
-	ACTOR static Future<Void> onConflict(CoordinatedStateImpl* self) {
+	static Future<Void> onConflict(CoordinatedStateImpl* self) {
 		ASSERT(self->stage == 4);
 		if (self->doomed)
-			return Void();
-		loop {
-			wait(delay(SERVER_KNOBS->COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL));
-			GenerationRegReadReply rep = wait(self->replicatedRead(
-			    self, GenerationRegReadRequest(self->coordinators.clusterKey, UniqueGeneration())));
+			co_return;
+		while (true) {
+			co_await delay(SERVER_KNOBS->COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL);
+			GenerationRegReadReply rep = co_await self->replicatedRead(
+			    self, GenerationRegReadRequest(self->coordinators.clusterKey, UniqueGeneration()));
 			if (self->stage > 4)
 				break;
 			self->conflictGen = std::max(self->conflictGen, std::max(rep.gen.generation, rep.rgen.generation));
 			if (self->isDoomed(rep))
-				return Void();
+				co_return;
 		}
-		wait(Future<Void>(Never()));
-		return Void();
+		co_await Future<Void>(Never());
 	}
-	ACTOR static Future<Void> setExclusive(CoordinatedStateImpl* self, Value v) {
+	static Future<Void> setExclusive(CoordinatedStateImpl* self, Value v) {
 		ASSERT(self->stage == 4);
 		self->stage = 5;
 
-		UniqueGeneration wgen = wait(self->replicatedWrite(
-		    self, GenerationRegWriteRequest(KeyValueRef(self->coordinators.clusterKey, v), self->gen)));
+		UniqueGeneration wgen = co_await self->replicatedWrite(
+		    self, GenerationRegWriteRequest(KeyValueRef(self->coordinators.clusterKey, v), self->gen));
 		self->stage = 6;
 
 		TraceEvent("CoordinatedStateSet")
@@ -145,19 +144,16 @@ struct CoordinatedStateImpl {
 		    .detail("Wgenu", wgen.uid)
 		    .detail("Cgen", self->conflictGen);
 
-		if (wgen == self->gen)
-			return Void();
-		else {
+		if (wgen != self->gen) {
 			self->conflictGen = std::max(self->conflictGen, wgen.generation);
 			throw coordinated_state_conflict();
 		}
 	}
 
-	ACTOR static Future<GenerationRegReadReply> replicatedRead(CoordinatedStateImpl* self,
-	                                                           GenerationRegReadRequest req) {
-		state std::vector<GenerationRegInterface>& replicas = self->coordinators.stateServers;
-		state std::vector<Future<GenerationRegReadReply>> rep_empty_reply;
-		state std::vector<Future<GenerationRegReadReply>> rep_reply;
+	static Future<GenerationRegReadReply> replicatedRead(CoordinatedStateImpl* self, GenerationRegReadRequest req) {
+		std::vector<GenerationRegInterface>& replicas = self->coordinators.stateServers;
+		std::vector<Future<GenerationRegReadReply>> rep_empty_reply;
+		std::vector<Future<GenerationRegReadReply>> rep_reply;
 		for (int i = 0; i < replicas.size(); i++) {
 			Future<GenerationRegReadReply> reply =
 			    waitAndSendRead(replicas[i], GenerationRegReadRequest(req.key, req.gen));
@@ -166,10 +162,10 @@ struct CoordinatedStateImpl {
 			self->ac.add(success(reply));
 		}
 
-		state Future<Void> majorityEmpty =
+		Future<Void> majorityEmpty =
 		    quorum(rep_empty_reply,
 		           (replicas.size() + 1) / 2); // enough empty to ensure we cannot achieve a majority non-empty
-		wait(quorum(rep_reply, replicas.size() / 2 + 1) || majorityEmpty);
+		co_await (quorum(rep_reply, replicas.size() / 2 + 1) || majorityEmpty);
 
 		if (majorityEmpty.isReady()) {
 			int best = -1;
@@ -179,8 +175,7 @@ struct CoordinatedStateImpl {
 						best = i;
 				}
 			ASSERT(best >= 0);
-			auto result = rep_empty_reply[best].get();
-			return result;
+			co_return rep_empty_reply[best].get();
 		} else {
 			int best = -1;
 			for (int i = 0; i < rep_reply.size(); i++)
@@ -191,27 +186,26 @@ struct CoordinatedStateImpl {
 						best = i;
 				}
 			ASSERT(best >= 0);
-			auto result = rep_reply[best].get();
-			return result;
+			co_return rep_reply[best].get();
 		}
 	}
 
-	ACTOR static Future<UniqueGeneration> replicatedWrite(CoordinatedStateImpl* self, GenerationRegWriteRequest req) {
-		state std::vector<GenerationRegInterface>& replicas = self->coordinators.stateServers;
-		state std::vector<Future<UniqueGeneration>> wrep_reply;
+	static Future<UniqueGeneration> replicatedWrite(CoordinatedStateImpl* self, GenerationRegWriteRequest req) {
+		std::vector<GenerationRegInterface>& replicas = self->coordinators.stateServers;
+		std::vector<Future<UniqueGeneration>> wrep_reply;
 		for (int i = 0; i < replicas.size(); i++) {
 			Future<UniqueGeneration> reply = waitAndSendWrite(replicas[i], GenerationRegWriteRequest(req.kv, req.gen));
 			wrep_reply.push_back(reply);
 			self->ac.add(success(reply));
 		}
 
-		wait(quorum(wrep_reply, self->initial ? replicas.size() : replicas.size() / 2 + 1));
+		co_await quorum(wrep_reply, self->initial ? replicas.size() : replicas.size() / 2 + 1);
 
 		UniqueGeneration maxGen;
 		for (int i = 0; i < wrep_reply.size(); i++)
 			if (wrep_reply[i].isReady())
 				maxGen = std::max(maxGen, wrep_reply[i].get());
-		return maxGen;
+		co_return maxGen;
 	}
 };
 
@@ -257,12 +251,12 @@ struct MovableCoordinatedStateImpl {
 	Optional<Value> lastValue, // The value passed to setExclusive()
 	    lastCSValue; // The value passed to cs.setExclusive()
 
-	MovableCoordinatedStateImpl(ServerCoordinators const& c) : coordinators(c), cs(c) {}
+	explicit MovableCoordinatedStateImpl(ServerCoordinators const& c) : coordinators(c), cs(c) {}
 
-	ACTOR static Future<Value> read(MovableCoordinatedStateImpl* self) {
-		state MovableValue moveState;
-		Value rawValue = wait(self->cs.read());
-		if (rawValue.size()) {
+	static Future<Value> read(MovableCoordinatedStateImpl* self) {
+		MovableValue moveState;
+		Value rawValue = co_await self->cs.read();
+		if (!rawValue.empty()) {
 			BinaryReader r(rawValue, IncludeVersion());
 			if (!r.protocolVersion().hasMovableCoordinatedState()) {
 				// Old coordinated state, not a MovableValue
@@ -275,10 +269,10 @@ struct MovableCoordinatedStateImpl {
 		if (moveState.mode == MovableValue::MaybeTo) {
 			CODE_PROBE(true, "Maybe moveto state");
 			ASSERT(moveState.other.present());
-			wait(self->moveTo(
-			    self, &self->cs, ClusterConnectionString(moveState.other.get().toString()), moveState.value));
+			co_await moveTo(
+			    self, &self->cs, ClusterConnectionString(moveState.other.get().toString()), moveState.value);
 		}
-		return moveState.value;
+		co_return moveState.value;
 	}
 
 	Future<Void> onConflict() { return cs.onConflict(); }
@@ -290,41 +284,44 @@ struct MovableCoordinatedStateImpl {
 		return cs.setExclusive(lastCSValue.get());
 	}
 
-	ACTOR static Future<Void> move(MovableCoordinatedStateImpl* self, ClusterConnectionString nc) {
+	static Future<Void> move(MovableCoordinatedStateImpl* self, ClusterConnectionString nc) {
 		// Call only after setExclusive returns.  Attempts to move the coordinated state
 		// permanently to the new ServerCoordinators, which must be uninitialized.  Returns when the process has
 		// reached the point where a leader elected by the new coordinators should be doing the rest of the work
 		// (and therefore the caller should die).
-		state CoordinatedState cs(self->coordinators);
-		state CoordinatedState nccs(ServerCoordinators(makeReference<ClusterConnectionMemoryRecord>(nc)));
-		state Future<Void> creationTimeout = delay(30);
+		CoordinatedState cs(self->coordinators);
+		CoordinatedState nccs(ServerCoordinators(makeReference<ClusterConnectionMemoryRecord>(nc)));
+		Future<Void> creationTimeout = delay(30);
 		ASSERT(self->lastValue.present() && self->lastCSValue.present());
 		TraceEvent("StartMove").detail("ConnectionString", nc.toString());
-		choose {
-			when(wait(creationTimeout)) {
+		{
+			auto res = co_await race(creationTimeout, nccs.read());
+			if (res.index() == 0) {
 				throw new_coordinators_timed_out();
 			}
-			when(Value ncInitialValue = wait(nccs.read())) {
-				ASSERT(!ncInitialValue.size()); // The new coordinators must be uninitialized!
-			}
+			ASSERT(res.index() == 1);
+			Value ncInitialValue = std::get<1>(std::move(res));
+			ASSERT(ncInitialValue.empty()); // The new coordinators must be uninitialized!
 		}
 		TraceEvent("FinishedRead").detail("ConnectionString", nc.toString());
 
-		choose {
-			when(wait(creationTimeout)) {
+		{
+			auto res = co_await race(creationTimeout,
+			                         nccs.setExclusive(BinaryWriter::toValue(
+			                             MovableValue(self->lastValue.get(),
+			                                          MovableValue::MovingFrom,
+			                                          self->coordinators.ccr->getConnectionString().toString()),
+			                             IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()))));
+			if (res.index() == 0) {
 				throw new_coordinators_timed_out();
 			}
-			when(wait(nccs.setExclusive(
-			    BinaryWriter::toValue(MovableValue(self->lastValue.get(),
-			                                       MovableValue::MovingFrom,
-			                                       self->coordinators.ccr->getConnectionString().toString()),
-			                          IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()))))) {}
+			ASSERT(res.index() == 1);
 		}
 
 		if (BUGGIFY)
-			wait(delay(5));
+			co_await delay(5);
 
-		Value oldQuorumState = wait(cs.read());
+		Value oldQuorumState = co_await cs.read();
 		if (oldQuorumState != self->lastCSValue.get()) {
 			CODE_PROBE(
 			    true, "Quorum change aborted by concurrent write to old coordination state", probe::decoration::rare);
@@ -332,26 +329,26 @@ struct MovableCoordinatedStateImpl {
 			throw coordinated_state_conflict();
 		}
 
-		wait(self->moveTo(self, &cs, nc, self->lastValue.get()));
+		co_await moveTo(self, &cs, nc, self->lastValue.get());
 
 		throw coordinators_changed();
 	}
 
-	ACTOR static Future<Void> moveTo(MovableCoordinatedStateImpl* self,
-	                                 CoordinatedState* coordinatedState,
-	                                 ClusterConnectionString nc,
-	                                 Value value) {
-		wait(coordinatedState->setExclusive(
+	static Future<Void> moveTo(MovableCoordinatedStateImpl* self,
+	                           CoordinatedState* coordinatedState,
+	                           ClusterConnectionString nc,
+	                           Value value) {
+		co_await coordinatedState->setExclusive(
 		    BinaryWriter::toValue(MovableValue(value, MovableValue::MaybeTo, nc.toString()),
-		                          IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()))));
+		                          IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2())));
 
 		if (BUGGIFY)
-			wait(delay(5));
+			co_await delay(5);
 
 		// SOMEDAY: If we are worried about someone magically getting the new cluster ID and interfering, do a second
 		// cs.setExclusive( encode( ReallyTo, ... ) )
 		TraceEvent("ChangingQuorum").detail("ConnectionString", nc.toString());
-		wait(changeLeaderCoordinators(self->coordinators, StringRef(nc.toString())));
+		co_await changeLeaderCoordinators(self->coordinators, StringRef(nc.toString()));
 		TraceEvent("ChangedQuorum").detail("ConnectionString", nc.toString());
 		throw coordinators_changed();
 	}
@@ -376,7 +373,7 @@ Future<Void> MovableCoordinatedState::move(ClusterConnectionString const& nc) {
 
 Optional<Value> updateCCSInMovableValue(ValueRef movableVal, KeyRef oldClusterKey, KeyRef newClusterKey) {
 	Optional<Value> result;
-	MovableValue moveVal = BinaryReader::fromStringRef<MovableValue>(
+	auto moveVal = BinaryReader::fromStringRef<MovableValue>(
 	    movableVal, IncludeVersion(ProtocolVersion::withMovableCoordinatedStateV2()));
 	if (moveVal.other.present() && moveVal.other.get().startsWith(oldClusterKey)) {
 		TraceEvent(SevDebug, "UpdateCCSInMovableValue").detail("OldConnectionString", moveVal.other.get());
diff --git a/fdbserver/core/CoordinationInterface.cpp b/fdbserver/core/CoordinationInterface.cpp
new file mode 100644
index 00000000000..50f4060ab8b
--- /dev/null
+++ b/fdbserver/core/CoordinationInterface.cpp
@@ -0,0 +1,55 @@
+/*
+ * CoordinationInterface.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/CoordinationInterface.h"
+
+GenerationRegInterface::GenerationRegInterface(NetworkAddress const& remote)
+  : read(Endpoint::wellKnown({ remote }, WLTOKEN_GENERATIONREG_READ)),
+    write(Endpoint::wellKnown({ remote }, WLTOKEN_GENERATIONREG_WRITE)) {}
+
+GenerationRegInterface::GenerationRegInterface(INetwork* local) {
+	read.makeWellKnownEndpoint(WLTOKEN_GENERATIONREG_READ, TaskPriority::Coordination);
+	write.makeWellKnownEndpoint(WLTOKEN_GENERATIONREG_WRITE, TaskPriority::Coordination);
+}
+
+LeaderElectionRegInterface::LeaderElectionRegInterface(NetworkAddress const& remote)
+  : ClientLeaderRegInterface(remote), candidacy(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_CANDIDACY)),
+    electionResult(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT)),
+    leaderHeartbeat(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT)),
+    forward(Endpoint::wellKnown({ remote }, WLTOKEN_LEADERELECTIONREG_FORWARD)) {}
+
+LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : ClientLeaderRegInterface(local) {
+	candidacy.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_CANDIDACY, TaskPriority::Coordination);
+	electionResult.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT, TaskPriority::Coordination);
+	leaderHeartbeat.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_LEADERHEARTBEAT, TaskPriority::Coordination);
+	forward.makeWellKnownEndpoint(WLTOKEN_LEADERELECTIONREG_FORWARD, TaskPriority::Coordination);
+}
+
+ServerCoordinators::ServerCoordinators(Reference<IClusterConnectionRecord> ccr) : ClientCoordinators(ccr) {
+	ClusterConnectionString cs = ccr->getConnectionString();
+	for (const auto& h : cs.hostnames) {
+		leaderElectionServers.emplace_back(h);
+		stateServers.emplace_back(h);
+	}
+	for (auto s : cs.coords) {
+		leaderElectionServers.emplace_back(s);
+		stateServers.emplace_back(s);
+	}
+}
diff --git a/fdbserver/core/DataMovement.cpp b/fdbserver/core/DataMovement.cpp
new file mode 100644
index 00000000000..9d2264773ed
--- /dev/null
+++ b/fdbserver/core/DataMovement.cpp
@@ -0,0 +1,82 @@
+/*
+ * DataMovement.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/DataMovement.h"
+
+#include <map>
+
+#include "fdbserver/core/Knobs.h"
+
+namespace {
+using DmReasonPriorityMapping = std::map<DataMovementReason, int>;
+using PriorityDmReasonMapping = std::map<int, DataMovementReason>;
+
+std::pair<const DmReasonPriorityMapping*, const PriorityDmReasonMapping*> buildPriorityMappings() {
+	static DmReasonPriorityMapping reasonPriority{
+		{ DataMovementReason::INVALID, -1 },
+		{ DataMovementReason::RECOVER_MOVE, SERVER_KNOBS->PRIORITY_RECOVER_MOVE },
+		{ DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM },
+		{ DataMovementReason::REBALANCE_OVERUTILIZED_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM },
+		{ DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_READ_OVERUTIL_TEAM },
+		{ DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM },
+		{ DataMovementReason::PERPETUAL_STORAGE_WIGGLE, SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE },
+		{ DataMovementReason::TEAM_HEALTHY, SERVER_KNOBS->PRIORITY_TEAM_HEALTHY },
+		{ DataMovementReason::TEAM_CONTAINS_UNDESIRED_SERVER, SERVER_KNOBS->PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER },
+		{ DataMovementReason::TEAM_REDUNDANT, SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT },
+		{ DataMovementReason::MERGE_SHARD, SERVER_KNOBS->PRIORITY_MERGE_SHARD },
+		{ DataMovementReason::POPULATE_REGION, SERVER_KNOBS->PRIORITY_POPULATE_REGION },
+		{ DataMovementReason::TEAM_UNHEALTHY, SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY },
+		{ DataMovementReason::TEAM_2_LEFT, SERVER_KNOBS->PRIORITY_TEAM_2_LEFT },
+		{ DataMovementReason::TEAM_1_LEFT, SERVER_KNOBS->PRIORITY_TEAM_1_LEFT },
+		{ DataMovementReason::TEAM_FAILED, SERVER_KNOBS->PRIORITY_TEAM_FAILED },
+		{ DataMovementReason::TEAM_0_LEFT, SERVER_KNOBS->PRIORITY_TEAM_0_LEFT },
+		{ DataMovementReason::SPLIT_SHARD, SERVER_KNOBS->PRIORITY_SPLIT_SHARD },
+		{ DataMovementReason::ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD,
+		  SERVER_KNOBS->PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD },
+		{ DataMovementReason::REBALANCE_STORAGE_QUEUE, SERVER_KNOBS->PRIORITY_REBALANCE_STORAGE_QUEUE },
+		{ DataMovementReason::ASSIGN_EMPTY_RANGE, -2 },
+		{ DataMovementReason::SEED_SHARD_SERVER, -3 },
+		{ DataMovementReason::NUMBER_OF_REASONS, -4 },
+	};
+
+	static PriorityDmReasonMapping priorityReason;
+	if (priorityReason.empty()) {
+		for (const auto& [reason, priority] : reasonPriority) {
+			priorityReason[priority] = reason;
+		}
+		if (priorityReason.size() != reasonPriority.size()) {
+			TraceEvent(SevError, "DuplicateDataMovementPriority").log();
+			ASSERT(false);
+		}
+	}
+
+	return { &reasonPriority, &priorityReason };
+}
+} // namespace
+
+int dataMovementPriority(DataMovementReason reason) {
+	auto [reasonPriority, _] = buildPriorityMappings();
+	return reasonPriority->at(reason);
+}
+
+DataMovementReason priorityToDataMovementReason(int priority) {
+	auto [_, priorityReason] = buildPriorityMappings();
+	return priorityReason->at(priority);
+}
diff --git a/fdbserver/core/FDBSimulationPolicy.cpp b/fdbserver/core/FDBSimulationPolicy.cpp
new file mode 100644
index 00000000000..bab7a0e75ed
--- /dev/null
+++ b/fdbserver/core/FDBSimulationPolicy.cpp
@@ -0,0 +1,401 @@
+/*
+ * FDBSimulationPolicy.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/FDBSimulationPolicy.h"
+
+#include <algorithm>
+#include <set>
+#include <vector>
+
+#include "fdbrpc/Replication.h"
+#include "fdbrpc/ReplicationUtils.h"
+#include "fdbrpc/SimulatorProcessInfo.h"
+#include "fdbrpc/simulator.h"
+
+FDBExtraDatabaseMode stringToFDBExtraDatabaseMode(const std::string& databaseMode) {
+	if (databaseMode == "Disabled") {
+		return FDBExtraDatabaseMode::Disabled;
+	} else if (databaseMode == "LocalOrSingle") {
+		return FDBExtraDatabaseMode::LocalOrSingle;
+	} else if (databaseMode == "Single") {
+		return FDBExtraDatabaseMode::Single;
+	} else if (databaseMode == "Local") {
+		return FDBExtraDatabaseMode::Local;
+	} else if (databaseMode == "Multiple") {
+		return FDBExtraDatabaseMode::Multiple;
+	} else {
+		TraceEvent(SevError, "UnknownExtraDatabaseMode").detail("DatabaseMode", databaseMode);
+		ASSERT(false);
+		throw internal_error();
+	}
+}
+
+namespace {
+
+FDBSimulationPolicyState& policyState() {
+	static auto* state = new FDBSimulationPolicyState;
+	return *state;
+}
+
+class FDBSimulationPolicy final : public ISimulationPolicy {
+public:
+	bool datacenterDead(Optional<Standalone<StringRef>> dcId,
+	                    std::vector<ProcessInfo*> const& allProcesses) const override {
+		if (!dcId.present()) {
+			return false;
+		}
+
+		LocalityGroup primaryProcessesDead;
+		std::vector<LocalityData> primaryLocalitiesLeft;
+
+		for (auto processInfo : allProcesses) {
+			if (!processInfo->isSpawnedKVProcess() && processInfo->isAvailableClass() &&
+			    processInfo->locality.dcId() == dcId) {
+				if (processInfo->isExcluded() || processInfo->isCleared() || !processInfo->isAvailable()) {
+					primaryProcessesDead.add(processInfo->locality);
+				} else {
+					primaryLocalitiesLeft.push_back(processInfo->locality);
+				}
+			}
+		}
+
+		std::vector<LocalityData> badCombo;
+		auto const& state = fdbSimulationPolicyState();
+
+		bool primaryTLogsDead = state.tLogWriteAntiQuorum ? !validateAllCombinations(badCombo,
+		                                                                             primaryProcessesDead,
+		                                                                             state.tLogPolicy,
+		                                                                             primaryLocalitiesLeft,
+		                                                                             state.tLogWriteAntiQuorum,
+		                                                                             false)
+		                                                  : primaryProcessesDead.validate(state.tLogPolicy);
+		if (state.usableRegions > 1 && state.remoteTLogPolicy && !primaryTLogsDead) {
+			primaryTLogsDead = primaryProcessesDead.validate(state.remoteTLogPolicy);
+		}
+		return primaryTLogsDead || primaryProcessesDead.validate(state.storagePolicy);
+	}
+
+	bool shouldRunVersionValidation() const override { return fdbSimulationPolicyState().extraDatabases.empty(); }
+
+	bool canSwapToMachine(Optional<Standalone<StringRef>> const&) const override {
+		return fdbSimulationPolicyState().extraDatabases.empty();
+	}
+
+	bool checkInjectedCorruption(NetworkAddress const& address) const override {
+		auto const& corruptWorkerMap = fdbSimulationPolicyState().corruptWorkerMap;
+		auto iter = corruptWorkerMap.find(address);
+		if (iter != corruptWorkerMap.end()) {
+			return iter->second;
+		}
+		return false;
+	}
+
+	bool hasCapability(Capability capability) const override {
+		switch (capability) {
+		case Capability::WarnOnStorageMismatch:
+			return fdbSimulationPolicyState().tssMode == FDBTSSMode::EnabledDropMutations;
+		case Capability::StorageReplicaFaultInjection:
+			return fdbSimulationPolicyState().tssMode >= FDBTSSMode::EnabledAddDelay;
+		case Capability::StorageReplicaDelay:
+			return fdbSimulationPolicyState().tssMode == FDBTSSMode::EnabledAddDelay;
+		case Capability::StorageReplicaMutationDrop:
+			return fdbSimulationPolicyState().tssMode == FDBTSSMode::EnabledDropMutations;
+		case Capability::LimitStorageServerReadBytes:
+			return fdbSimulationPolicyState().tssMode == FDBTSSMode::Disabled;
+		}
+		UNREACHABLE();
+	}
+
+	bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses,
+	                      std::vector<ProcessInfo*> const& deadProcesses,
+	                      KillType kt,
+	                      KillType* newKillType) const override {
+		auto const& state = fdbSimulationPolicyState();
+		bool canSurvive = true;
+		int nQuorum = ((state.desiredCoordinators + 1) / 2) * 2 - 1;
+
+		KillType newKt = kt;
+		if ((kt == KillType::KillInstantly) || (kt == KillType::InjectFaults) || (kt == KillType::FailDisk) ||
+		    (kt == KillType::RebootAndDelete) || (kt == KillType::RebootProcessAndDelete)) {
+			LocalityGroup primaryProcessesLeft, primaryProcessesDead;
+			LocalityGroup primarySatelliteProcessesLeft, primarySatelliteProcessesDead;
+			LocalityGroup remoteProcessesLeft, remoteProcessesDead;
+			LocalityGroup remoteSatelliteProcessesLeft, remoteSatelliteProcessesDead;
+
+			std::vector<LocalityData> primaryLocalitiesDead, primaryLocalitiesLeft;
+			std::vector<LocalityData> primarySatelliteLocalitiesDead, primarySatelliteLocalitiesLeft;
+			std::vector<LocalityData> remoteLocalitiesDead, remoteLocalitiesLeft;
+			std::vector<LocalityData> remoteSatelliteLocalitiesDead, remoteSatelliteLocalitiesLeft;
+
+			std::vector<LocalityData> badCombo;
+			std::set<Optional<Standalone<StringRef>>> uniqueMachines;
+
+			if (!state.primaryDcId.present() || state.usableRegions == 1) {
+				for (auto processInfo : availableProcesses) {
+					primaryProcessesLeft.add(processInfo->locality);
+					primaryLocalitiesLeft.push_back(processInfo->locality);
+					uniqueMachines.insert(processInfo->locality.zoneId());
+				}
+				for (auto processInfo : deadProcesses) {
+					primaryProcessesDead.add(processInfo->locality);
+					primaryLocalitiesDead.push_back(processInfo->locality);
+				}
+			} else {
+				for (auto processInfo : availableProcesses) {
+					uniqueMachines.insert(processInfo->locality.zoneId());
+					if (processInfo->locality.dcId() == state.primaryDcId) {
+						primaryProcessesLeft.add(processInfo->locality);
+						primaryLocalitiesLeft.push_back(processInfo->locality);
+					}
+					if (processInfo->locality.dcId() == state.remoteDcId) {
+						remoteProcessesLeft.add(processInfo->locality);
+						remoteLocalitiesLeft.push_back(processInfo->locality);
+					}
+					if (std::find(state.primarySatelliteDcIds.begin(),
+					              state.primarySatelliteDcIds.end(),
+					              processInfo->locality.dcId()) != state.primarySatelliteDcIds.end()) {
+						primarySatelliteProcessesLeft.add(processInfo->locality);
+						primarySatelliteLocalitiesLeft.push_back(processInfo->locality);
+					}
+					if (std::find(state.remoteSatelliteDcIds.begin(),
+					              state.remoteSatelliteDcIds.end(),
+					              processInfo->locality.dcId()) != state.remoteSatelliteDcIds.end()) {
+						remoteSatelliteProcessesLeft.add(processInfo->locality);
+						remoteSatelliteLocalitiesLeft.push_back(processInfo->locality);
+					}
+				}
+				for (auto processInfo : deadProcesses) {
+					if (processInfo->locality.dcId() == state.primaryDcId) {
+						primaryProcessesDead.add(processInfo->locality);
+						primaryLocalitiesDead.push_back(processInfo->locality);
+					}
+					if (processInfo->locality.dcId() == state.remoteDcId) {
+						remoteProcessesDead.add(processInfo->locality);
+						remoteLocalitiesDead.push_back(processInfo->locality);
+					}
+					if (std::find(state.primarySatelliteDcIds.begin(),
+					              state.primarySatelliteDcIds.end(),
+					              processInfo->locality.dcId()) != state.primarySatelliteDcIds.end()) {
+						primarySatelliteProcessesDead.add(processInfo->locality);
+						primarySatelliteLocalitiesDead.push_back(processInfo->locality);
+					}
+					if (std::find(state.remoteSatelliteDcIds.begin(),
+					              state.remoteSatelliteDcIds.end(),
+					              processInfo->locality.dcId()) != state.remoteSatelliteDcIds.end()) {
+						remoteSatelliteProcessesDead.add(processInfo->locality);
+						remoteSatelliteLocalitiesDead.push_back(processInfo->locality);
+					}
+				}
+			}
+
+			bool tooManyDead = false;
+			bool notEnoughLeft = false;
+			bool primaryTLogsDead = state.tLogWriteAntiQuorum ? !validateAllCombinations(badCombo,
+			                                                                             primaryProcessesDead,
+			                                                                             state.tLogPolicy,
+			                                                                             primaryLocalitiesLeft,
+			                                                                             state.tLogWriteAntiQuorum,
+			                                                                             false)
+			                                                  : primaryProcessesDead.validate(state.tLogPolicy);
+			if (state.usableRegions > 1 && state.remoteTLogPolicy && !primaryTLogsDead) {
+				primaryTLogsDead = primaryProcessesDead.validate(state.remoteTLogPolicy);
+			}
+
+			if (!state.primaryDcId.present()) {
+				tooManyDead = primaryTLogsDead || primaryProcessesDead.validate(state.storagePolicy);
+				notEnoughLeft = !primaryProcessesLeft.validate(state.tLogPolicy) ||
+				                !primaryProcessesLeft.validate(state.storagePolicy);
+			} else {
+				bool remoteTLogsDead = state.tLogWriteAntiQuorum ? !validateAllCombinations(badCombo,
+				                                                                            remoteProcessesDead,
+				                                                                            state.tLogPolicy,
+				                                                                            remoteLocalitiesLeft,
+				                                                                            state.tLogWriteAntiQuorum,
+				                                                                            false)
+				                                                 : remoteProcessesDead.validate(state.tLogPolicy);
+				if (state.usableRegions > 1 && state.remoteTLogPolicy && !remoteTLogsDead) {
+					remoteTLogsDead = remoteProcessesDead.validate(state.remoteTLogPolicy);
+				}
+
+				if (!state.hasSatelliteReplication) {
+					if (state.usableRegions > 1) {
+						tooManyDead = primaryTLogsDead || remoteTLogsDead ||
+						              (primaryProcessesDead.validate(state.storagePolicy) &&
+						               remoteProcessesDead.validate(state.storagePolicy));
+						notEnoughLeft = !primaryProcessesLeft.validate(state.tLogPolicy) ||
+						                !primaryProcessesLeft.validate(state.remoteTLogPolicy) ||
+						                !primaryProcessesLeft.validate(state.storagePolicy) ||
+						                !remoteProcessesLeft.validate(state.tLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.remoteTLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.storagePolicy);
+					} else {
+						tooManyDead = primaryTLogsDead || remoteTLogsDead ||
+						              primaryProcessesDead.validate(state.storagePolicy) ||
+						              remoteProcessesDead.validate(state.storagePolicy);
+						notEnoughLeft = !primaryProcessesLeft.validate(state.tLogPolicy) ||
+						                !primaryProcessesLeft.validate(state.storagePolicy) ||
+						                !remoteProcessesLeft.validate(state.tLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.storagePolicy);
+					}
+				} else {
+					bool primarySatelliteTLogsDead =
+					    state.satelliteTLogWriteAntiQuorumFallback
+					        ? !validateAllCombinations(badCombo,
+					                                   primarySatelliteProcessesDead,
+					                                   state.satelliteTLogPolicyFallback,
+					                                   primarySatelliteLocalitiesLeft,
+					                                   state.satelliteTLogWriteAntiQuorumFallback,
+					                                   false)
+					        : primarySatelliteProcessesDead.validate(state.satelliteTLogPolicyFallback);
+
+					if (state.usableRegions > 1) {
+						notEnoughLeft = !primaryProcessesLeft.validate(state.tLogPolicy) ||
+						                !primaryProcessesLeft.validate(state.remoteTLogPolicy) ||
+						                !primaryProcessesLeft.validate(state.storagePolicy) ||
+						                !primarySatelliteProcessesLeft.validate(state.satelliteTLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.tLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.remoteTLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.storagePolicy) ||
+						                !remoteSatelliteProcessesLeft.validate(state.satelliteTLogPolicy);
+					} else {
+						notEnoughLeft = !primaryProcessesLeft.validate(state.tLogPolicy) ||
+						                !primaryProcessesLeft.validate(state.storagePolicy) ||
+						                !primarySatelliteProcessesLeft.validate(state.satelliteTLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.tLogPolicy) ||
+						                !remoteProcessesLeft.validate(state.storagePolicy) ||
+						                !remoteSatelliteProcessesLeft.validate(state.satelliteTLogPolicy);
+					}
+
+					if (state.usableRegions > 1 && state.allowLogSetKills) {
+						tooManyDead = (primaryTLogsDead && primarySatelliteTLogsDead) || remoteTLogsDead ||
+						              (primaryTLogsDead && remoteTLogsDead) ||
+						              (primaryProcessesDead.validate(state.storagePolicy) &&
+						               remoteProcessesDead.validate(state.storagePolicy));
+					} else {
+						tooManyDead = primaryTLogsDead || remoteTLogsDead ||
+						              primaryProcessesDead.validate(state.storagePolicy) ||
+						              remoteProcessesDead.validate(state.storagePolicy);
+					}
+				}
+			}
+
+			if (tooManyDead || (state.usableRegions > 1 && notEnoughLeft)) {
+				newKt = KillType::Reboot;
+				canSurvive = false;
+				TraceEvent("KillChanged")
+				    .detail("KillType", kt)
+				    .detail("NewKillType", newKt)
+				    .detail("TLogPolicy", state.tLogPolicy->info())
+				    .detail("Reason", "Too many dead processes that cannot satisfy tLogPolicy.");
+			} else if ((kt < KillType::RebootAndDelete) && notEnoughLeft) {
+				newKt = KillType::RebootAndDelete;
+				canSurvive = false;
+				TraceEvent("KillChanged")
+				    .detail("KillType", kt)
+				    .detail("NewKillType", newKt)
+				    .detail("TLogPolicy", state.tLogPolicy->info())
+				    .detail("Reason", "Not enough tLog left to satisfy tLogPolicy.");
+			} else if ((kt < KillType::RebootAndDelete) && (nQuorum > uniqueMachines.size())) {
+				newKt = KillType::RebootAndDelete;
+				canSurvive = false;
+				TraceEvent("KillChanged")
+				    .detail("KillType", kt)
+				    .detail("NewKillType", newKt)
+				    .detail("StoragePolicy", state.storagePolicy->info())
+				    .detail("Quorum", nQuorum)
+				    .detail("Machines", uniqueMachines.size())
+				    .detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
+			} else {
+				TraceEvent("CanSurviveKills")
+				    .detail("KillType", kt)
+				    .detail("TLogPolicy", state.tLogPolicy->info())
+				    .detail("StoragePolicy", state.storagePolicy->info())
+				    .detail("Quorum", nQuorum)
+				    .detail("Machines", uniqueMachines.size());
+			}
+		}
+		if (newKillType) {
+			*newKillType = newKt;
+		}
+		return canSurvive;
+	}
+};
+
+} // namespace
+
+void installFDBSimulationPolicy() {
+	ASSERT(g_simulator);
+	g_simulator->setSimulationPolicy(makeReference<FDBSimulationPolicy>());
+}
+
+FDBSimulationPolicyState& fdbSimulationPolicyState() {
+	return policyState();
+}
+
+void updateFDBSimulationPolicy(DatabaseConfiguration const& configuration, bool restartingTest) {
+	auto& state = fdbSimulationPolicyState();
+	state.storagePolicy = configuration.storagePolicy;
+	state.tLogPolicy = configuration.tLogPolicy;
+	state.tLogWriteAntiQuorum = configuration.tLogWriteAntiQuorum;
+	state.remoteTLogPolicy = configuration.remoteTLogPolicy;
+	state.usableRegions = configuration.usableRegions;
+	state.primaryDcId = Optional<Standalone<StringRef>>();
+	state.remoteDcId = Optional<Standalone<StringRef>>();
+	state.hasSatelliteReplication = false;
+	state.satelliteTLogPolicy = Reference<IReplicationPolicy>();
+	state.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>();
+	state.satelliteTLogWriteAntiQuorum = 0;
+	state.satelliteTLogWriteAntiQuorumFallback = 0;
+	state.primarySatelliteDcIds.clear();
+	state.remoteSatelliteDcIds.clear();
+
+	if (!configuration.regions.empty()) {
+		state.primaryDcId = configuration.regions[0].dcId;
+		state.hasSatelliteReplication = configuration.regions[0].satelliteTLogReplicationFactor > 0;
+		if (configuration.regions[0].satelliteTLogUsableDcsFallback > 0) {
+			state.satelliteTLogPolicyFallback = configuration.regions[0].satelliteTLogPolicyFallback;
+			state.satelliteTLogWriteAntiQuorumFallback = configuration.regions[0].satelliteTLogWriteAntiQuorumFallback;
+		} else {
+			state.satelliteTLogPolicyFallback = configuration.regions[0].satelliteTLogPolicy;
+			state.satelliteTLogWriteAntiQuorumFallback = configuration.regions[0].satelliteTLogWriteAntiQuorum;
+		}
+		state.satelliteTLogPolicy = configuration.regions[0].satelliteTLogPolicy;
+		state.satelliteTLogWriteAntiQuorum = configuration.regions[0].satelliteTLogWriteAntiQuorum;
+
+		for (const auto& s : configuration.regions[0].satellites) {
+			state.primarySatelliteDcIds.push_back(s.dcId);
+		}
+	}
+
+	if (configuration.regions.size() == 2) {
+		state.remoteDcId = configuration.regions[1].dcId;
+		for (const auto& s : configuration.regions[1].satellites) {
+			state.remoteSatelliteDcIds.push_back(s.dcId);
+		}
+	}
+
+	if (restartingTest || state.usableRegions < 2 || !state.hasSatelliteReplication) {
+		state.allowLogSetKills = false;
+	}
+}
+
+void setFDBSimulationPolicyRemoteTLogPolicy(Reference<IReplicationPolicy> remoteTLogPolicy) {
+	fdbSimulationPolicyState().remoteTLogPolicy = remoteTLogPolicy;
+}
diff --git a/fdbserver/LatencyBandConfig.cpp b/fdbserver/core/LatencyBandConfig.cpp
similarity index 94%
rename from fdbserver/LatencyBandConfig.cpp
rename to fdbserver/core/LatencyBandConfig.cpp
index 974186292b9..8cc4ab0e024 100644
--- a/fdbserver/LatencyBandConfig.cpp
+++ b/fdbserver/core/LatencyBandConfig.cpp
@@ -18,9 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/LatencyBandConfig.h"
+#include "fdbserver/core/LatencyBandConfig.h"
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/Schemas.h"
 
 bool operator==(LatencyBandConfig::RequestConfig const& lhs, LatencyBandConfig::RequestConfig const& rhs) {
@@ -38,7 +38,7 @@ bool LatencyBandConfig::RequestConfig::isEqual(RequestConfig const& r) const {
 void LatencyBandConfig::RequestConfig::fromJson(JSONDoc json) {
 	json_spirit::mArray bandsArray;
 	if (json.get("bands", bandsArray)) {
-		for (auto b : bandsArray) {
+		for (const auto& b : bandsArray) {
 			bands.insert(b.get_real());
 		}
 	}
@@ -72,13 +72,13 @@ void LatencyBandConfig::CommitConfig::fromJson(JSONDoc json) {
 }
 
 bool LatencyBandConfig::CommitConfig::isEqual(RequestConfig const& r) const {
-	CommitConfig const& other = static_cast<CommitConfig const&>(r);
+	auto const& other = static_cast<CommitConfig const&>(r);
 	return RequestConfig::isEqual(r) && maxCommitBytes == other.maxCommitBytes;
 }
 
 Optional<LatencyBandConfig> LatencyBandConfig::parse(ValueRef configurationString) {
 	Optional<LatencyBandConfig> config;
-	if (configurationString.size() == 0) {
+	if (configurationString.empty()) {
 		return config;
 	}
 
diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/core/LeaderElection.actor.cpp
similarity index 87%
rename from fdbserver/LeaderElection.actor.cpp
rename to fdbserver/core/LeaderElection.actor.cpp
index e33aa0e166e..ea3a5fa5959 100644
--- a/fdbserver/LeaderElection.actor.cpp
+++ b/fdbserver/core/LeaderElection.actor.cpp
@@ -20,57 +20,54 @@
 
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/Locality.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/CoordinationInterface.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbclient/MonitorLeader.h"
+#include "flow/CoroUtils.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // Keep trying to become a leader by submitting itself to all coordinators.
 // Monitor the health of all coordinators at the same time.
-ACTOR Future<Void> submitCandidacy(Key key,
-                                   LeaderElectionRegInterface coord,
-                                   LeaderInfo myInfo,
-                                   UID prevChangeID,
-                                   AsyncTrigger* nomineeChange,
-                                   Optional<LeaderInfo>* nominee) {
-	loop {
-		state Optional<LeaderInfo> li;
+Future<Void> submitCandidacy(Key key,
+                             LeaderElectionRegInterface coord,
+                             LeaderInfo myInfo,
+                             UID prevChangeID,
+                             AsyncTrigger* nomineeChange,
+                             Optional<LeaderInfo>* nominee) {
+	while (true) {
+		Optional<LeaderInfo> li;
 		if (coord.hostname.present()) {
-			wait(store(
-			    li,
-			    retryGetReplyFromHostname(
-			        CandidacyRequest(key, myInfo, nominee->present() ? nominee->get().changeID : UID(), prevChangeID),
-			        coord.hostname.get(),
-			        WLTOKEN_LEADERELECTIONREG_CANDIDACY,
-			        TaskPriority::CoordinationReply)));
+			li = co_await retryGetReplyFromHostname(
+			    CandidacyRequest(key, myInfo, nominee->present() ? nominee->get().changeID : UID(), prevChangeID),
+			    coord.hostname.get(),
+			    WLTOKEN_LEADERELECTIONREG_CANDIDACY,
+			    TaskPriority::CoordinationReply);
 		} else {
-			wait(store(
-			    li,
-			    retryBrokenPromise(
-			        coord.candidacy,
-			        CandidacyRequest(key, myInfo, nominee->present() ? nominee->get().changeID : UID(), prevChangeID),
-			        TaskPriority::CoordinationReply)));
+			li = co_await retryBrokenPromise(
+			    coord.candidacy,
+			    CandidacyRequest(key, myInfo, nominee->present() ? nominee->get().changeID : UID(), prevChangeID),
+			    TaskPriority::CoordinationReply);
 		}
 
-		wait(Future<Void>(Void())); // Make sure we weren't cancelled
+		co_await Future<Void>(Void()); // Make sure we weren't cancelled
 
 		if (li != *nominee) {
 			*nominee = li;
 			nomineeChange->trigger();
 
 			if (li.present() && li.get().forward)
-				wait(Future<Void>(Never()));
+				co_await Future<Void>(Never());
 		}
 	}
 }
 
-ACTOR template <class T>
+template <class T>
 Future<Void> buggifyDelayedAsyncVar(Reference<AsyncVar<T>> in, Reference<AsyncVar<T>> out) {
 	try {
-		loop {
-			wait(delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01()));
+		while (true) {
+			co_await delay(SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * deterministicRandom()->random01());
 			out->set(in->get());
-			wait(in->onChange());
+			co_await in->onChange();
 		}
 	} catch (Error& e) {
 		out->set(in->get());
@@ -86,7 +83,9 @@ Future<Void> buggifyDelayedAsyncVar(Reference<AsyncVar<T>>& var) {
 	return f;
 }
 
-ACTOR Future<Void> changeLeaderCoordinators(ServerCoordinators coordinators, Value forwardingInfo) {
+namespace {
+
+Future<Void> changeLeaderCoordinatorsImpl(ServerCoordinators coordinators, Value forwardingInfo) {
 	std::vector<Future<Void>> forwardRequests;
 	forwardRequests.reserve(coordinators.leaderElectionServers.size());
 	for (int i = 0; i < coordinators.leaderElectionServers.size(); i++) {
@@ -100,8 +99,13 @@ ACTOR Future<Void> changeLeaderCoordinators(ServerCoordinators coordinators, Val
 		}
 	}
 	int quorum_size = forwardRequests.size() / 2 + 1;
-	wait(quorum(forwardRequests, quorum_size));
-	return Void();
+	co_await quorum(forwardRequests, quorum_size);
+}
+
+} // namespace
+
+Future<Void> changeLeaderCoordinators(ServerCoordinators const& coordinators, Value const& forwardingInfo) {
+	return changeLeaderCoordinatorsImpl(coordinators, forwardingInfo);
 }
 
 ACTOR Future<Void> tryBecomeLeaderInternal(ServerCoordinators coordinators,
diff --git a/fdbserver/LogSystemConfig.cpp b/fdbserver/core/LogSystemConfig.cpp
similarity index 98%
rename from fdbserver/LogSystemConfig.cpp
rename to fdbserver/core/LogSystemConfig.cpp
index 132385cda35..1efbd1616a2 100644
--- a/fdbserver/LogSystemConfig.cpp
+++ b/fdbserver/core/LogSystemConfig.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/LogSystemConfig.h"
+#include "fdbserver/core/LogSystemConfig.h"
 
 std::string TLogSet::toString() const {
 	return format("anti: %d replication: %d local: %d routers: %d tLogs: %s backupWorkers: %s locality: %d",
@@ -227,7 +227,7 @@ std::pair<int8_t, int8_t> LogSystemConfig::getLocalityForDcId(Optional<Key> dcId
 }
 
 std::vector<std::pair<UID, NetworkAddress>> LogSystemConfig::allSharedLogs() const {
-	typedef std::pair<UID, NetworkAddress> IdAddrPair;
+	using IdAddrPair = std::pair<UID, NetworkAddress>;
 	std::vector<IdAddrPair> results;
 	for (auto& tLogSet : tLogs) {
 		for (auto& tLog : tLogSet.tLogs) {
@@ -271,7 +271,7 @@ bool LogSystemConfig::isEqualIds(LogSystemConfig const& r) const {
 }
 
 bool LogSystemConfig::isNextGenerationOf(LogSystemConfig const& r) const {
-	if (!oldTLogs.size()) {
+	if (oldTLogs.empty()) {
 		return false;
 	}
 
diff --git a/fdbserver/core/MasterInterface.cpp b/fdbserver/core/MasterInterface.cpp
new file mode 100644
index 00000000000..429dea4da06
--- /dev/null
+++ b/fdbserver/core/MasterInterface.cpp
@@ -0,0 +1,4 @@
+#include "fdbserver/core/MasterInterface.h"
+
+template class ReplyPromise<MasterInterface>;
+template struct NetSAV<MasterInterface>;
diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/core/MoveKeys.cpp
similarity index 72%
rename from fdbserver/MoveKeys.actor.cpp
rename to fdbserver/core/MoveKeys.cpp
index 2bf77eed7b1..1c895b84ea9 100644
--- a/fdbserver/MoveKeys.actor.cpp
+++ b/fdbserver/core/MoveKeys.cpp
@@ -1,5 +1,5 @@
 /*
- * MoveKeys.actor.cpp
+ * MoveKeys.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,16 +26,15 @@
 #include "flow/Trace.h"
 #include "flow/Util.h"
 #include "fdbrpc/FailureMonitor.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/BulkLoadUtil.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/TSSMappingUtil.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TSSMappingUtil.h"
+#include "flow/TxnCounters.h"
 
 template <typename... T>
 static inline void dprint(fmt::format_string<T...> fmt, T&&... args) {
@@ -44,6 +43,16 @@ static inline void dprint(fmt::format_string<T...> fmt, T&&... args) {
 }
 
 namespace {
+
+// Exponential backoff with jitter for transaction_too_old retries in finishMoveKeys.
+// Base formula: 0.1 * 2^retries, capped at 5.0s, then jittered to [0.75x, 1.25x].
+// Called with retries >= 1 (retries is incremented before the call), so effective
+// start is ~0.2s. Jitter prevents FlowLock slots from retrying in lockstep.
+double finishMoveKeysBackoff(int retries) {
+	double base = std::min(0.1 * (1 << std::min(retries, 6)), 5.0);
+	return std::min(base * (0.75 + 0.5 * deterministicRandom()->random01()), 5.0); // jitter: [0.75x, 1.25x], capped
+}
+
 struct Shard {
 	Shard() = default;
 	Shard(KeyRangeRef range, const UID& id) : range(range), id(id) {}
@@ -62,28 +71,28 @@ bool shouldCreateCheckpoint(const UID& dataMoveId) {
 
 // Unassigns keyrange `range` from server `ssId`, except ranges in `shards`.
 // Note: krmSetRangeCoalescing() doesn't work in this case since each shard is assigned an ID.
-ACTOR Future<Void> unassignServerKeys(Transaction* tr, UID ssId, KeyRange range, std::vector<Shard> shards, UID logId) {
-	state Key mapPrefix = serverKeysPrefixFor(ssId);
+Future<Void> unassignServerKeys(Transaction* tr, UID ssId, KeyRange range, std::vector<Shard> shards, UID logId) {
+	Key mapPrefix = serverKeysPrefixFor(ssId);
 	if (shards.empty()) {
-		wait(krmSetRangeCoalescing(tr, mapPrefix, range, allKeys, serverKeysFalse));
-		return Void();
+		co_await krmSetRangeCoalescing(tr, mapPrefix, range, allKeys, serverKeysFalse);
+		co_return;
 	}
 
-	state KeyRange withPrefix =
+	KeyRange withPrefix =
 	    KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
-	state KeyRange maxWithPrefix =
+	KeyRange maxWithPrefix =
 	    KeyRangeRef(mapPrefix.toString() + allKeys.begin.toString(), mapPrefix.toString() + allKeys.end.toString());
 
-	state std::vector<Future<RangeResult>> keys;
+	std::vector<Future<RangeResult>> keys;
 	keys.push_back(
 	    tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::True));
 	keys.push_back(
 	    tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, Snapshot::True));
-	wait(waitForAll(keys));
+	co_await waitForAll(keys);
 
 	// Determine how far to extend this range at the beginning
 	auto beginRange = keys[0].get();
-	bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix);
+	bool hasBegin = !beginRange.empty() && beginRange[0].key.startsWith(mapPrefix);
 	Value beginValue = hasBegin ? beginRange[0].value : serverKeysFalse;
 
 	Key beginKey = withPrefix.begin;
@@ -115,7 +124,7 @@ ACTOR Future<Void> unassignServerKeys(Transaction* tr, UID ssId, KeyRange range,
 
 	// Determine how far to extend this range at the end
 	auto endRange = keys[1].get();
-	bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
+	bool hasEnd = !endRange.empty() && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
 	bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
 	               (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
 	Value existingValue = hasEnd ? endRange[0].value : serverKeysFalse;
@@ -168,20 +177,18 @@ ACTOR Future<Void> unassignServerKeys(Transaction* tr, UID ssId, KeyRange range,
 		tr->set(kvs[i].key, kvs[i].value);
 		tr->set(kvs[i + 1].key, kvs[i + 1].value);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> deleteCheckpoints(Transaction* tr, std::set<UID> checkpointIds, UID dataMoveId) {
+Future<Void> deleteCheckpoints(Transaction* tr, std::set<UID> checkpointIds, UID dataMoveId) {
 	if (!shouldCreateCheckpoint(dataMoveId)) {
-		return Void();
+		co_return;
 	}
 	TraceEvent(SevDebug, "DataMoveDeleteCheckpoints", dataMoveId).detail("Checkpoints", describe(checkpointIds));
 	std::vector<Future<Optional<Value>>> checkpointEntries;
 	for (const UID& id : checkpointIds) {
 		checkpointEntries.push_back(tr->get(checkpointKeyFor(id)));
 	}
-	std::vector<Optional<Value>> checkpointValues = wait(getAll(checkpointEntries));
+	std::vector<Optional<Value>> checkpointValues = co_await getAll(checkpointEntries);
 
 	for (int i = 0; i < checkpointIds.size(); ++i) {
 		const auto& value = checkpointValues[i];
@@ -197,8 +204,6 @@ ACTOR Future<Void> deleteCheckpoints(Transaction* tr, std::set<UID> checkpointId
 		tr->clear(singleKeyRange(key));
 		TraceEvent(SevDebug, "DataMoveDeleteCheckpoint", dataMoveId).detail("Checkpoint", checkpoint.toString());
 	}
-
-	return Void();
 }
 } // namespace
 
@@ -253,40 +258,44 @@ bool DDEnabledState::trySetBlobRestorePreparing(UID requesterId) {
 	return true;
 }
 
-ACTOR Future<Void> readMoveKeysLock(Transaction* tr, MoveKeysLock* lock) {
+Future<Void> readMoveKeysLock(Transaction* tr, MoveKeysLock* lock) {
 	{
-		Optional<Value> readVal = wait(tr->get(moveKeysLockOwnerKey));
+		Optional<Value> readVal = co_await tr->get(moveKeysLockOwnerKey);
 		lock->prevOwner = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 	}
 	{
-		Optional<Value> readVal = wait(tr->get(moveKeysLockWriteKey));
+		Optional<Value> readVal = co_await tr->get(moveKeysLockWriteKey);
 		lock->prevWrite = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 	}
-	return Void();
 }
 
-ACTOR Future<MoveKeysLock> readMoveKeysLock(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<MoveKeysLock> readMoveKeysLock(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			state MoveKeysLock lock;
+			MoveKeysLock lock;
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			wait(readMoveKeysLock(&tr, &lock));
-			return lock;
+			co_await readMoveKeysLock(&tr, &lock);
+			co_return lock;
 		} catch (Error& e) {
-			wait(tr.onError(e));
-			CODE_PROBE(true, "readMoveKeysLock retry");
+			err = e;
 		}
+		co_await tr.onError(err);
+		CODE_PROBE(true, "readMoveKeysLock retry");
 	}
 }
 
-ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
-	state Transaction tr(cx);
-	loop {
+Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
+	static auto* counters = makeCounters("/movekeys/takeMoveKeysLock");
+	Transaction tr(cx);
+	while (true) {
+		counters->started->increment(1);
+		Error err;
 		try {
-			state MoveKeysLock lock;
-			state UID txnId;
+			MoveKeysLock lock;
+			UID txnId;
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -294,32 +303,35 @@ ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId) {
 				txnId = deterministicRandom()->randomUniqueID();
 				tr.debugTransaction(txnId);
 			}
-			wait(readMoveKeysLock(&tr, &lock));
+			co_await readMoveKeysLock(&tr, &lock);
 			lock.myOwner = deterministicRandom()->randomUniqueID();
 			tr.set(moveKeysLockOwnerKey, BinaryWriter::toValue(lock.myOwner, Unversioned()));
-			wait(tr.commit());
+			co_await tr.commit();
+			counters->committed->increment(1);
 			TraceEvent("TakeMoveKeysLockTransaction", ddId)
 			    .detail("TransactionUID", txnId)
 			    .detail("PrevOwner", lock.prevOwner.toString())
 			    .detail("PrevWrite", lock.prevWrite.toString())
 			    .detail("MyOwner", lock.myOwner.toString());
-			return lock;
+			co_return lock;
 		} catch (Error& e) {
-			wait(tr.onError(e));
-			CODE_PROBE(true, "takeMoveKeysLock retry");
+			counters->aborted->increment(1);
+			err = e;
 		}
+		co_await tr.onError(err);
+		CODE_PROBE(true, "takeMoveKeysLock retry");
 	}
 }
 
-ACTOR static Future<Void> checkPersistentMoveKeysLock(Transaction* tr, MoveKeysLock lock, bool isWrite = true) {
+static Future<Void> checkPersistentMoveKeysLock(Transaction* tr, MoveKeysLock lock, bool isWrite = true) {
 	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 
-	Optional<Value> readVal = wait(tr->get(moveKeysLockOwnerKey));
-	state UID currentOwner = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
+	Optional<Value> readVal = co_await tr->get(moveKeysLockOwnerKey);
+	UID currentOwner = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 
 	if (currentOwner == lock.prevOwner) {
 		// Check that the previous owner hasn't touched the lock since we took it
-		Optional<Value> readVal = wait(tr->get(moveKeysLockWriteKey));
+		Optional<Value> readVal = co_await tr->get(moveKeysLockWriteKey);
 		UID lastWrite = readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 		if (lastWrite != lock.prevWrite) {
 			CODE_PROBE(true, "checkMoveKeysLock: Conflict with previous owner");
@@ -349,7 +361,7 @@ ACTOR static Future<Void> checkPersistentMoveKeysLock(Transaction* tr, MoveKeysL
 			    .detail("Writer", lastWriter.toString());
 		}
 
-		return Void();
+		co_return;
 	} else if (currentOwner == lock.myOwner) {
 		if (isWrite) {
 			// Touch the lock, preventing overlapping attempts to take it
@@ -360,7 +372,7 @@ ACTOR static Future<Void> checkPersistentMoveKeysLock(Transaction* tr, MoveKeysL
 			tr->makeSelfConflicting();
 		}
 
-		return Void();
+		co_return;
 	} else {
 		CODE_PROBE(true, "checkMoveKeysLock: Conflict with new owner");
 		TraceEvent(SevDebug, "CheckPersistentMoveKeysLockOwnerConflict")
@@ -389,30 +401,30 @@ Future<Void> checkMoveKeysLockReadOnly(Transaction* tr, MoveKeysLock lock, const
 }
 
 namespace {
-ACTOR Future<Optional<UID>> checkReadWrite(Future<ErrorOr<GetShardStateReply>> fReply, UID uid, Version version) {
-	ErrorOr<GetShardStateReply> reply = wait(fReply);
+Future<Optional<UID>> checkReadWrite(Future<ErrorOr<GetShardStateReply>> fReply, UID uid, Version version) {
+	ErrorOr<GetShardStateReply> reply = co_await fReply;
 	if (!reply.present() || reply.get().first < version)
-		return Optional<UID>();
-	return Optional<UID>(uid);
+		co_return Optional<UID>();
+	co_return Optional<UID>(uid);
 }
 
 // Must propagate corruption signal to outside
-ACTOR Future<bool> validateRangeAssignment(Database occ,
-                                           Transaction* tr,
-                                           KeyRange range,
-                                           UID ssid,
-                                           std::string context,
-                                           UID dataMoveId) {
+Future<bool> validateRangeAssignment(Database occ,
+                                     Transaction* tr,
+                                     KeyRange range,
+                                     UID ssid,
+                                     std::string context,
+                                     UID dataMoveId) {
 	ASSERT(!range.empty());
-	state Key toReadRangeBegin = range.begin;
-	state bool allCorrect = true;
+	Key toReadRangeBegin = range.begin;
+	bool allCorrect = true;
 	tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-	loop {
-		RangeResult readResult = wait(krmGetRanges(tr,
-		                                           serverKeysPrefixFor(ssid),
-		                                           KeyRangeRef(toReadRangeBegin, range.end),
-		                                           CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
-		                                           CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES));
+	while (true) {
+		RangeResult readResult = co_await krmGetRanges(tr,
+		                                               serverKeysPrefixFor(ssid),
+		                                               KeyRangeRef(toReadRangeBegin, range.end),
+		                                               CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+		                                               CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
 		// We do not want to reset tr
 		for (int i = 0; i < readResult.size() - 1; i++) {
 			UID shardId;
@@ -454,7 +466,7 @@ ACTOR Future<bool> validateRangeAssignment(Database occ,
 		try {
 			// If corruption detected, enter security mode which
 			// stops using data moves and only allow auditStorage
-			wait(success(setDDMode(occ, 2)));
+			co_await setDDMode(occ, 2);
 			TraceEvent(SevInfo, "ValidateRangeAssignmentCorruptionDetectedAndDDStopped")
 			    .detail("DataMoveID", dataMoveId)
 			    .detail("Range", range)
@@ -467,21 +479,24 @@ ACTOR Future<bool> validateRangeAssignment(Database occ,
 			// We do not want failure of setDDMode hide the corruption signal
 		}
 	}
-	return allCorrect;
+	co_return allCorrect;
 }
 
-ACTOR Future<Void> auditLocationMetadataPreCheck(Database occ,
-                                                 Transaction* tr,
-                                                 KeyRange range,
-                                                 std::vector<UID> servers,
-                                                 std::string context,
-                                                 UID dataMoveId) {
+Future<Void> auditLocationMetadataPreCheck(Database occ,
+                                           Transaction* tr,
+                                           KeyRange range,
+                                           std::vector<UID> servers,
+                                           std::string context,
+                                           UID dataMoveId) {
+	// This code has only been tested with `SHARD_ENCODE_LOCATION_METADATA` enabled.
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+
 	if (range.empty()) {
 		TraceEvent(SevWarn, "CheckLocationMetadataEmptyInputRange").detail("By", "PreCheck").detail("Range", range);
-		return Void();
+		co_return;
 	}
-	state std::vector<Future<Void>> actors;
-	state std::unordered_map<UID, Optional<bool>> results;
+	std::vector<Future<Void>> actors;
+	std::unordered_map<UID, Optional<bool>> results;
 	TraceEvent(SevVerbose, "CheckLocationMetadataStart")
 	    .detail("By", "PreCheck")
 	    .detail("DataMoveID", dataMoveId)
@@ -494,7 +509,7 @@ ACTOR Future<Void> auditLocationMetadataPreCheck(Database occ,
 		for (const auto& ssid : servers) {
 			actors.push_back(store(results[ssid], validateRangeAssignment(occ, tr, range, ssid, context, dataMoveId)));
 		}
-		wait(waitForAllReadyThenThrow(actors));
+		co_await waitForAllReadyThenThrow(actors);
 		for (const auto& [ssid, res] : results) {
 			ASSERT(res.present());
 			if (!res.get()) { // Stop check if corruption detected
@@ -538,32 +553,36 @@ ACTOR Future<Void> auditLocationMetadataPreCheck(Database occ,
 			// If no corruption detected, exit silently
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> auditLocationMetadataPostCheck(Database occ, KeyRange range, std::string context, UID dataMoveId) {
+Future<Void> auditLocationMetadataPostCheck(Database occ, KeyRange range, std::string context, UID dataMoveId) {
+	// This code has only been tested with `SHARD_ENCODE_LOCATION_METADATA` enabled.
+	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
+
 	if (range.empty()) {
 		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "CheckLocationMetadataEmptyInputRange")
 		    .detail("By", "PostCheck")
 		    .detail("Range", range)
 		    .detail("Context", context)
 		    .detail("DataMoveId", dataMoveId.toString());
-		return Void();
-	}
-	state std::vector<Future<Void>> actors;
-	state std::unordered_map<uint64_t, Optional<bool>> results;
-	state Key rangeToReadBegin = range.begin;
-	state RangeResult readResultKS;
-	state RangeResult UIDtoTagMap;
-	state Transaction tr(occ);
-	state int retryCount = 0;
+		co_return;
+	}
+	std::vector<Future<Void>> actors;
+	std::unordered_map<uint64_t, Optional<bool>> results;
+	Key rangeToReadBegin = range.begin;
+	RangeResult readResultKS;
+	RangeResult UIDtoTagMap;
+	Transaction tr(occ);
+	int retryCount = 0;
 	TraceEvent(SevVerbose, "CheckLocationMetadataStart")
 	    .detail("By", "PostCheck")
 	    .detail("Context", context)
 	    .detail("Range", range);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			loop {
+			while (true) {
+				Error err;
 				try {
 					actors.clear();
 					readResultKS.clear();
@@ -579,14 +598,14 @@ ACTOR Future<Void> auditLocationMetadataPostCheck(Database occ, KeyRange range,
 					                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
 					                                    CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES)));
 					actors.push_back(store(UIDtoTagMap, tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY)));
-					wait(waitForAll(actors));
+					co_await waitForAll(actors);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 					TraceEvent(SevVerbose, "CheckLocationMetadataReadDone")
 					    .detail("By", "PostCheck")
 					    .detail("ResultSize", readResultKS.size());
 					// Read serverKeys
 					actors.clear();
-					state uint64_t resIdx = 0;
+					uint64_t resIdx = 0;
 					for (int i = 0; i < readResultKS.size() - 1; ++i) {
 						std::vector<UID> src;
 						std::vector<UID> dest;
@@ -607,7 +626,7 @@ ACTOR Future<Void> auditLocationMetadataPostCheck(Database occ, KeyRange range,
 							++resIdx;
 						}
 					}
-					wait(waitForAllReadyThenThrow(actors));
+					co_await waitForAllReadyThenThrow(actors);
 					for (const auto& [idx, res] : results) {
 						ASSERT(res.present());
 						if (!res.get()) { // Stop check if corruption detected
@@ -617,103 +636,107 @@ ACTOR Future<Void> auditLocationMetadataPostCheck(Database occ, KeyRange range,
 					if (readResultKS.back().key < range.end) {
 						rangeToReadBegin = readResultKS.back().key;
 						continue;
-					} else {
-						TraceEvent(SevVerbose, "CheckLocationMetadataComplete")
-						    .detail("By", "PostCheck")
-						    .detail("DataMoveID", dataMoveId)
-						    .detail("Context", context)
-						    .detail("Range", range);
-						break;
-					}
-				} catch (Error& e) {
-					wait(tr.onError(e));
-					retryCount++;
-				}
-			}
-			break;
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled || e.code() == error_code_location_metadata_corruption) {
-				throw e;
-			} else {
-				// Check corruptions for the current (failed) round
-				for (const auto& [idx, res] : results) {
-					if (res.present() && !res.get()) {
-						TraceEvent(SevError, "CheckLocationMetadataCorruptionDetectedWhenFailed")
-						    .detail("By", "PostCheck")
-						    .detail("DataMoveID", dataMoveId)
-						    .detail("Context", context)
-						    .detail("Range", range);
-						throw location_metadata_corruption();
 					}
-				}
-				if (retryCount > SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK_RETRY_COUNT_MAX) {
-					TraceEvent(SevInfo, "CheckLocationMetadataFailed")
-					    .errorUnsuppressed(e)
+
+					TraceEvent(SevVerbose, "CheckLocationMetadataComplete")
 					    .detail("By", "PostCheck")
 					    .detail("DataMoveID", dataMoveId)
 					    .detail("Context", context)
 					    .detail("Range", range);
-					// If no corruption detected, exit silently
-				} else {
-					wait(delay(0.5));
-					retryCount++;
+					break;
+				} catch (Error& e) {
+					err = e;
 				}
+				co_await tr.onError(err);
+				retryCount++;
+			}
+			break;
+		} catch (Error& e) {
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled || err.code() == error_code_location_metadata_corruption) {
+			throw err;
+		}
+
+		// Check corruptions for the current (failed) round
+		for (const auto& [idx, res] : results) {
+			if (res.present() && !res.get()) {
+				TraceEvent(SevError, "CheckLocationMetadataCorruptionDetectedWhenFailed")
+				    .detail("By", "PostCheck")
+				    .detail("DataMoveID", dataMoveId)
+				    .detail("Context", context)
+				    .detail("Range", range);
+				throw location_metadata_corruption();
 			}
 		}
+		if (retryCount > SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK_RETRY_COUNT_MAX) {
+			TraceEvent(SevInfo, "CheckLocationMetadataFailed")
+			    .errorUnsuppressed(err)
+			    .detail("By", "PostCheck")
+			    .detail("DataMoveID", dataMoveId)
+			    .detail("Context", context)
+			    .detail("Range", range);
+			// If no corruption detected, exit silently
+		} else {
+			co_await delay(0.5);
+			retryCount++;
+		}
 	}
-	return Void();
 }
 
 // Cleans up dest servers of a single shard, and unassigns the keyrange from the dest servers if necessary.
-ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
-                                              KeyRange keys,
-                                              MoveKeysLock lock,
-                                              FlowLock* cleanUpDataMoveParallelismLock,
-                                              UID dataMoveId,
-                                              const DDEnabledState* ddEnabledState) {
+Future<Void> cleanUpSingleShardDataMove(Database occ,
+                                        KeyRange keys,
+                                        MoveKeysLock lock,
+                                        FlowLock* cleanUpDataMoveParallelismLock,
+                                        UID dataMoveId,
+                                        const DDEnabledState* ddEnabledState) {
 	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	TraceEvent(SevInfo, "CleanUpSingleShardDataMoveBegin", dataMoveId).detail("Range", keys);
+	static auto* counters = makeCounters("/movekeys/cleanUpSingleShardDataMove");
 
-	state bool runPreCheck = true;
+	bool runPreCheck = true;
 
-	loop {
-		state Transaction tr(occ);
+	while (true) {
+		counters->started->increment(1);
+		Transaction tr(occ);
 
+		Error err;
 		try {
 			tr.trState->taskID = TaskPriority::MoveKeys;
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-			state RangeResult currentShards = wait(krmGetRanges(&tr,
-			                                                    keyServersPrefix,
-			                                                    keys,
-			                                                    SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
-			                                                    SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT));
+			RangeResult currentShards = co_await krmGetRanges(&tr,
+			                                                  keyServersPrefix,
+			                                                  keys,
+			                                                  SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
+			                                                  SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT);
 			ASSERT(!currentShards.empty() && !currentShards.more);
 
-			state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
 			if (KeyRangeRef(currentShards[0].key, currentShards[1].key) != keys) {
 				throw operation_cancelled();
 			}
 
-			state std::vector<UID> src;
-			state std::vector<UID> dest;
-			state UID srcId;
-			state UID destId;
+			std::vector<UID> src;
+			std::vector<UID> dest;
+			UID srcId;
+			UID destId;
 			decodeKeyServersValue(UIDtoTagMap, currentShards[0].value, src, dest, srcId, destId);
 
 			if (dest.empty() || destId != anonymousShardId) {
-				return Void();
+				co_return;
 			}
 
 			// Pre validate consistency of update of keyServers and serverKeys
 			if (SERVER_KNOBS->AUDIT_DATAMOVE_PRE_CHECK && runPreCheck) {
 				std::vector<UID> servers(src.size() + dest.size());
 				std::merge(src.begin(), src.end(), dest.begin(), dest.end(), servers.begin());
-				wait(auditLocationMetadataPreCheck(
-				    occ, &tr, keys, servers, "cleanUpSingleShardDataMove_precheck", dataMoveId));
+				co_await auditLocationMetadataPreCheck(
+				    occ, &tr, keys, servers, "cleanUpSingleShardDataMove_precheck", dataMoveId);
 			}
 
 			TraceEvent(SevInfo, "CleanUpSingleShardDataMove", dataMoveId)
@@ -735,33 +758,30 @@ ACTOR Future<Void> cleanUpSingleShardDataMove(Database occ,
 				}
 			}
 
-			wait(waitForAll(actors));
+			co_await waitForAll(actors);
 
-			wait(tr.commit());
+			co_await tr.commit();
+			counters->committed->increment(1);
 
 			// Post validate consistency of update of keyServers and serverKeys
 			if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) {
-				wait(auditLocationMetadataPostCheck(occ, keys, "cleanUpSingleShardDataMove_postcheck", dataMoveId));
+				co_await auditLocationMetadataPostCheck(occ, keys, "cleanUpSingleShardDataMove_postcheck", dataMoveId);
 			}
 			break;
 		} catch (Error& e) {
-			state Error err = e;
-			if (err.code() == error_code_location_metadata_corruption) {
-				throw location_metadata_corruption();
-			} else {
-				runPreCheck = false;
-				wait(tr.onError(err));
-			}
-
-			TraceEvent(SevWarn, "CleanUpSingleShardDataMoveRetriableError", dataMoveId)
-			    .error(err)
-			    .detail("Range", keys);
+			counters->aborted->increment(1);
+			err = e;
 		}
+		if (err.code() == error_code_location_metadata_corruption) {
+			throw location_metadata_corruption();
+		}
+		runPreCheck = false;
+		co_await tr.onError(err);
+
+		TraceEvent(SevWarn, "CleanUpSingleShardDataMoveRetriableError", dataMoveId).error(err).detail("Range", keys);
 	}
 
 	TraceEvent(SevInfo, "CleanUpSingleShardDataMoveEnd", dataMoveId).detail("Range", keys);
-
-	return Void();
 }
 
 Future<Void> removeOldDestinations(Reference<ReadYourWritesTransaction> tr,
@@ -786,17 +806,17 @@ Future<Void> removeOldDestinations(Reference<ReadYourWritesTransaction> tr,
 	return waitForAll(actors);
 }
 
-ACTOR Future<std::vector<UID>> addReadWriteDestinations(KeyRangeRef shard,
-                                                        std::vector<StorageServerInterface> srcInterfs,
-                                                        std::vector<StorageServerInterface> destInterfs,
-                                                        Version version,
-                                                        int desiredHealthy,
-                                                        int maxServers) {
+Future<std::vector<UID>> addReadWriteDestinations(KeyRangeRef shard,
+                                                  std::vector<StorageServerInterface> srcInterfs,
+                                                  std::vector<StorageServerInterface> destInterfs,
+                                                  Version version,
+                                                  int desiredHealthy,
+                                                  int maxServers) {
 	if (srcInterfs.size() >= maxServers) {
-		return std::vector<UID>();
+		co_return std::vector<UID>();
 	}
 
-	state std::vector<Future<Optional<UID>>> srcChecks;
+	std::vector<Future<Optional<UID>>> srcChecks;
 	srcChecks.reserve(srcInterfs.size());
 	for (int s = 0; s < srcInterfs.size(); s++) {
 		srcChecks.push_back(checkReadWrite(srcInterfs[s].getShardState.getReplyUnlessFailedFor(
@@ -808,7 +828,7 @@ ACTOR Future<std::vector<UID>> addReadWriteDestinations(KeyRangeRef shard,
 		                                   0));
 	}
 
-	state std::vector<Future<Optional<UID>>> destChecks;
+	std::vector<Future<Optional<UID>>> destChecks;
 	destChecks.reserve(destInterfs.size());
 	for (int s = 0; s < destInterfs.size(); s++) {
 		destChecks.push_back(checkReadWrite(destInterfs[s].getShardState.getReplyUnlessFailedFor(
@@ -820,10 +840,10 @@ ACTOR Future<std::vector<UID>> addReadWriteDestinations(KeyRangeRef shard,
 		                                    version));
 	}
 
-	wait(waitForAll(srcChecks) && waitForAll(destChecks));
+	co_await (waitForAll(srcChecks) && waitForAll(destChecks));
 
 	int healthySrcs = 0;
-	for (auto it : srcChecks) {
+	for (const auto& it : srcChecks) {
 		if (it.get().present()) {
 			healthySrcs++;
 		}
@@ -837,25 +857,25 @@ ACTOR Future<std::vector<UID>> addReadWriteDestinations(KeyRangeRef shard,
 		}
 	}
 
-	return result;
+	co_return result;
 }
 
 // Returns storage servers selected from 'candidates', who is serving a read-write copy of 'range'.
-ACTOR Future<std::vector<UID>> pickReadWriteServers(Transaction* tr, std::vector<UID> candidates, KeyRangeRef range) {
+Future<std::vector<UID>> pickReadWriteServers(Transaction* tr, std::vector<UID> candidates, KeyRangeRef range) {
 	std::vector<Future<Optional<Value>>> serverListEntries;
 
 	for (const UID id : candidates) {
 		serverListEntries.push_back(tr->get(serverListKeyFor(id)));
 	}
 
-	std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+	std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 
 	std::vector<StorageServerInterface> ssis;
 	for (auto& v : serverListValues) {
 		ssis.push_back(decodeServerListValue(v.get()));
 	}
 
-	state std::vector<Future<Optional<UID>>> checks;
+	std::vector<Future<Optional<UID>>> checks;
 	checks.reserve(ssis.size());
 	for (auto& ssi : ssis) {
 		checks.push_back(checkReadWrite(
@@ -867,7 +887,7 @@ ACTOR Future<std::vector<UID>> pickReadWriteServers(Transaction* tr, std::vector
 		    0));
 	}
 
-	wait(waitForAll(checks));
+	co_await waitForAll(checks);
 
 	std::vector<UID> result;
 	for (const auto& it : checks) {
@@ -876,14 +896,14 @@ ACTOR Future<std::vector<UID>> pickReadWriteServers(Transaction* tr, std::vector
 		}
 	}
 
-	return result;
+	co_return result;
 }
 
-ACTOR Future<std::vector<std::vector<UID>>> additionalSources(RangeResult shards,
-                                                              Reference<ReadYourWritesTransaction> tr,
-                                                              int desiredHealthy,
-                                                              int maxServers) {
-	state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+Future<std::vector<std::vector<UID>>> additionalSources(RangeResult shards,
+                                                        Reference<ReadYourWritesTransaction> tr,
+                                                        int desiredHealthy,
+                                                        int maxServers) {
+	RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 	ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 	std::vector<Future<Optional<Value>>> serverListEntries;
 	std::set<UID> fetching;
@@ -908,7 +928,7 @@ ACTOR Future<std::vector<std::vector<UID>>> additionalSources(RangeResult shards
 		}
 	}
 
-	std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+	std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 
 	std::map<UID, StorageServerInterface> ssiMap;
 	for (int s = 0; s < serverListValues.size(); s++) {
@@ -941,14 +961,14 @@ ACTOR Future<std::vector<std::vector<UID>>> additionalSources(RangeResult shards
 		    rangeIntersectKeys, srcInterfs, destInterfs, tr->getReadVersion().get(), desiredHealthy, maxServers));
 	}
 
-	std::vector<std::vector<UID>> result = wait(getAll(allChecks));
-	return result;
+	std::vector<std::vector<UID>> result = co_await getAll(allChecks);
+	co_return result;
 }
 
-ACTOR Future<Void> logWarningAfter(const char* context, double duration, std::vector<UID> servers) {
-	state double startTime = now();
-	loop {
-		wait(delay(duration));
+Future<Void> logWarningAfter(const char* context, double duration, std::vector<UID> servers) {
+	double startTime = now();
+	while (true) {
+		co_await delay(duration);
 		TraceEvent(SevWarnAlways, context).detail("Duration", now() - startTime).detail("Servers", describe(servers));
 	}
 }
@@ -959,29 +979,30 @@ ACTOR Future<Void> logWarningAfter(const char* context, double duration, std::ve
 // subrange of keys that the server did not already have, = complete for each subrange that it already has. Set
 // serverKeys[dest][keys] = "" for the dest servers of each existing shard in keys (unless that destination is a member
 // of servers OR if the source list is sufficiently degraded)
-ACTOR static Future<Void> startMoveKeys(Database occ,
-                                        KeyRange keys,
-                                        std::vector<UID> servers,
-                                        MoveKeysLock lock,
-                                        FlowLock* startMoveKeysLock,
-                                        UID relocationIntervalId,
-                                        std::map<UID, StorageServerInterface>* tssMapping,
-                                        const DDEnabledState* ddEnabledState) {
-	state TraceInterval interval("RelocateShard_StartMoveKeys");
-	state Future<Void> warningLogger = logWarningAfter("StartMoveKeysTooLong", 600, servers);
+static Future<Void> startMoveKeys(Database occ,
+                                  KeyRange keys,
+                                  std::vector<UID> servers,
+                                  MoveKeysLock lock,
+                                  FlowLock* startMoveKeysLock,
+                                  UID relocationIntervalId,
+                                  std::map<UID, StorageServerInterface>* tssMapping,
+                                  const DDEnabledState* ddEnabledState) {
+	TraceInterval interval("RelocateShard_StartMoveKeys");
+	Future<Void> warningLogger = logWarningAfter("StartMoveKeysTooLong", 600, servers);
 	// state TraceInterval waitInterval("");
+	static auto* counters = makeCounters("/movekeys/startMoveKeys");
 
-	wait(startMoveKeysLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser(*startMoveKeysLock);
-	state bool loadedTssMapping = false;
+	co_await startMoveKeysLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser(*startMoveKeysLock);
+	bool loadedTssMapping = false;
 
 	TraceEvent(SevDebug, interval.begin(), relocationIntervalId);
 
 	try {
-		state Key begin = keys.begin;
-		state int batches = 0;
-		state int shards = 0;
-		state int maxRetries = 0;
+		Key begin = keys.begin;
+		int batches = 0;
+		int shards = 0;
+		int maxRetries = 0;
 
 		// If it's multiple transaction, how do we achieve atomicity?
 		// This process can be split up into multiple transactions if there are too many existing overlapping shards
@@ -991,28 +1012,30 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 			batches++;
 
 			// RYW to optimize re-reading the same key ranges
-			state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(occ);
-			state int retries = 0;
+			auto tr = makeReference<ReadYourWritesTransaction>(occ);
+			int retries = 0;
 
-			loop {
+			while (true) {
+				counters->started->increment(1);
+				Error err;
 				try {
 					retries++;
 
 					// Keep track of old dests that may need to have ranges removed from serverKeys
-					state std::set<UID> oldDests;
+					std::set<UID> oldDests;
 
 					// Keep track of shards for all src servers so that we can preserve their values in serverKeys
-					state Map<UID, VectorRef<KeyRangeRef>> shardMap;
+					Map<UID, VectorRef<KeyRangeRef>> shardMap;
 
 					tr->getTransaction().trState->taskID = TaskPriority::MoveKeys;
 					tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-					wait(checkMoveKeysLock(&(tr->getTransaction()), lock, ddEnabledState));
+					co_await checkMoveKeysLock(&(tr->getTransaction()), lock, ddEnabledState);
 
 					if (!loadedTssMapping) {
 						// share transaction for loading tss mapping with the rest of start move keys
-						wait(readTSSMappingRYW(tr, tssMapping));
+						co_await readTSSMappingRYW(tr, tssMapping);
 						loadedTssMapping = true;
 					}
 
@@ -1020,7 +1043,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 					serverListEntries.reserve(servers.size());
 					for (int s = 0; s < servers.size(); s++)
 						serverListEntries.push_back(tr->get(serverListKeyFor(servers[s])));
-					state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+					std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 
 					for (int s = 0; s < serverListValues.size(); s++) {
 						// This can happen if a SS is removed after a shard move. See comments on PR #10110.
@@ -1032,16 +1055,16 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 
 					// Get all existing shards overlapping keys (exclude any that have been processed in a previous
 					// iteration of the outer loop)
-					state KeyRange currentKeys = KeyRangeRef(begin, keys.end);
+					KeyRange currentKeys = KeyRangeRef(begin, keys.end);
 
-					state RangeResult old = wait(krmGetRanges(tr,
-					                                          keyServersPrefix,
-					                                          currentKeys,
-					                                          SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
-					                                          SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
+					RangeResult old = co_await krmGetRanges(tr,
+					                                        keyServersPrefix,
+					                                        currentKeys,
+					                                        SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+					                                        SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
 
 					// Determine the last processed key (which will be the beginning for the next iteration)
-					state Key endKey = old.end()[-1].key;
+					Key endKey = old.end()[-1].key;
 					currentKeys = KeyRangeRef(currentKeys.begin, endKey);
 
 					// TraceEvent("StartMoveKeysBatch", relocationIntervalId)
@@ -1053,10 +1076,10 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 					// 	printf("'%s': '%s'\n", old[i].key.toString().c_str(), old[i].value.toString().c_str());
 
 					// Check that enough servers for each shard are in the correct state
-					state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-					std::vector<std::vector<UID>> addAsSource = wait(additionalSources(
-					    old, tr, servers.size(), SERVER_KNOBS->MAX_ADDED_SOURCES_MULTIPLIER * servers.size()));
+					std::vector<std::vector<UID>> addAsSource = co_await additionalSources(
+					    old, tr, servers.size(), SERVER_KNOBS->MAX_ADDED_SOURCES_MULTIPLIER * servers.size());
 
 					// For each intersecting range, update keyServers[range] dest to be servers and clear existing dest
 					// servers from serverKeys
@@ -1099,7 +1122,7 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 						}
 					}
 
-					state std::set<UID>::iterator oldDest;
+					std::set<UID>::iterator oldDest;
 
 					// Remove old dests from serverKeys.  In order for krmSetRangeCoalescing to work correctly in the
 					// same prefix for a single transaction, we must do most of the coalescing ourselves.  Only the
@@ -1121,9 +1144,10 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 						    tr, serverKeysPrefixFor(servers[i]), currentKeys, allKeys, serverKeysTrue));
 					}
 
-					wait(waitForAll(actors));
+					co_await waitForAll(actors);
 
-					wait(tr->commit());
+					co_await tr->commit();
+					counters->committed->increment(1);
 
 					/*TraceEvent("StartMoveKeysCommitDone", relocationIntervalId)
 					    .detail("CommitVersion", tr.getCommittedVersion())
@@ -1132,19 +1156,19 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 					shards += old.size() - 1;
 					break;
 				} catch (Error& e) {
-					state Error err = e;
-					if (err.code() == error_code_move_to_removed_server)
-						throw;
-					wait(tr->onError(e));
-
-					if (retries % 10 == 0) {
-						TraceEvent(
-						    retries == 50 ? SevWarnAlways : SevWarn, "StartMoveKeysRetrying", relocationIntervalId)
-						    .error(err)
-						    .detail("Keys", keys)
-						    .detail("BeginKey", begin)
-						    .detail("NumTries", retries);
-					}
+					counters->aborted->increment(1);
+					err = e;
+				}
+				if (err.code() == error_code_move_to_removed_server)
+					throw err;
+				co_await tr->onError(err);
+
+				if (retries % 10 == 0) {
+					TraceEvent(retries == 50 ? SevWarnAlways : SevWarn, "StartMoveKeysRetrying", relocationIntervalId)
+					    .error(err)
+					    .detail("Keys", keys)
+					    .detail("BeginKey", begin)
+					    .detail("NumTries", retries);
 				}
 			}
 
@@ -1163,48 +1187,50 @@ ACTOR static Future<Void> startMoveKeys(Database occ,
 		TraceEvent(SevDebug, interval.end(), relocationIntervalId).errorUnsuppressed(e);
 		throw;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> waitForShardReady(StorageServerInterface server,
-                                     KeyRange keys,
-                                     Version minVersion,
-                                     GetShardStateRequest::waitMode mode) {
-	loop {
+Future<Void> waitForShardReady(StorageServerInterface server,
+                               KeyRange keys,
+                               Version minVersion,
+                               GetShardStateRequest::waitMode mode) {
+	while (true) {
+		Error err;
 		try {
 			GetShardStateReply rep =
-			    wait(server.getShardState.getReply(GetShardStateRequest(keys, mode), TaskPriority::MoveKeys));
+			    co_await server.getShardState.getReply(GetShardStateRequest(keys, mode), TaskPriority::MoveKeys);
 			if (rep.first >= minVersion) {
-				return Void();
+				co_return;
 			}
-			wait(delayJittered(SERVER_KNOBS->SHARD_READY_DELAY, TaskPriority::MoveKeys));
+			co_await delayJittered(SERVER_KNOBS->SHARD_READY_DELAY, TaskPriority::MoveKeys);
+			continue;
 		} catch (Error& e) {
-			TraceEvent("GetShardStateReadyError", server.id()).error(e).log();
-			if (e.code() != error_code_timed_out) {
-				if (e.code() != error_code_broken_promise)
-					throw e;
-				wait(Never()); // Never return: A storage server which has failed will never be ready
-				throw internal_error(); // does not happen
-			}
+			err = e;
+		}
+		TraceEvent("GetShardStateReadyError", server.id()).error(err).log();
+		if (err.code() != error_code_timed_out) {
+			if (err.code() != error_code_broken_promise)
+				throw err;
+			co_await Future<Void>(Never()); // Never return: A storage server which has failed will never be ready
+			throw internal_error(); // does not happen
 		}
 	}
 }
 
 // best effort to also wait for TSS on data move
 
-ACTOR Future<Void> checkFetchingState(Database cx,
-                                      std::vector<UID> dest,
-                                      KeyRange keys,
-                                      Promise<Void> dataMovementComplete,
-                                      UID relocationIntervalId,
-                                      std::map<UID, StorageServerInterface> tssMapping) {
-	state Transaction tr(cx);
+Future<Void> checkFetchingState(Database cx,
+                                std::vector<UID> dest,
+                                KeyRange keys,
+                                Promise<Void> dataMovementComplete,
+                                UID relocationIntervalId,
+                                std::map<UID, StorageServerInterface> tssMapping) {
+	Transaction tr(cx);
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			if (BUGGIFY)
-				wait(delay(5));
+				co_await delay(5);
 
 			tr.trState->taskID = TaskPriority::MoveKeys;
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -1214,9 +1240,9 @@ ACTOR Future<Void> checkFetchingState(Database cx,
 			serverListEntries.reserve(dest.size());
 			for (int s = 0; s < dest.size(); s++)
 				serverListEntries.push_back(tr.get(serverListKeyFor(dest[s])));
-			state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+			std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 			std::vector<Future<Void>> requests;
-			state std::vector<Future<Void>> tssRequests;
+			std::vector<Future<Void>> tssRequests;
 			for (int s = 0; s < serverListValues.size(); s++) {
 				// We don't think this condition should be triggered, but we're not sure if there are conditions
 				// that might cause it to trigger. Adding this assertion to find any such cases via testing.
@@ -1238,25 +1264,27 @@ ACTOR Future<Void> checkFetchingState(Database cx,
 				}
 			}
 
-			wait(timeoutError(waitForAll(requests), SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT, TaskPriority::MoveKeys));
+			co_await timeoutError(
+			    waitForAll(requests), SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT, TaskPriority::MoveKeys);
 
 			// If normal servers return normally, give TSS data movement a bit of a chance, but don't block on it, and
 			// ignore errors in tss requests
-			if (tssRequests.size()) {
-				wait(timeout(waitForAllReady(tssRequests),
-				             SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT / 2,
-				             Void(),
-				             TaskPriority::MoveKeys));
+			if (!tssRequests.empty()) {
+				co_await timeout(waitForAllReady(tssRequests),
+				                 SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT / 2,
+				                 Void(),
+				                 TaskPriority::MoveKeys);
 			}
 
 			dataMovementComplete.send(Void());
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			if (e.code() == error_code_timed_out)
-				tr.reset();
-			else
-				wait(tr.onError(e));
+			err = e;
 		}
+		if (err.code() == error_code_timed_out)
+			tr.reset();
+		else
+			co_await tr.onError(err);
 	}
 }
 
@@ -1264,26 +1292,27 @@ ACTOR Future<Void> checkFetchingState(Database cx,
 // keyServers[k].dest must be the same for all k in keys
 // Set serverKeys[dest][keys] = true; serverKeys[src][keys] = false for all src not in dest
 // Should be cancelled and restarted if keyServers[keys].dest changes (?so this is no longer true?)
-ACTOR static Future<Void> finishMoveKeys(Database occ,
-                                         KeyRange keys,
-                                         std::vector<UID> destinationTeam,
-                                         MoveKeysLock lock,
-                                         FlowLock* finishMoveKeysParallelismLock,
-                                         bool hasRemote,
-                                         UID relocationIntervalId,
-                                         std::map<UID, StorageServerInterface> tssMapping,
-                                         const DDEnabledState* ddEnabledState) {
-	state TraceInterval interval("RelocateShard_FinishMoveKeys");
-	state TraceInterval waitInterval("");
-	state Future<Void> warningLogger = logWarningAfter("FinishMoveKeysTooLong", 600, destinationTeam);
-	state Key begin = keys.begin;
-	state Key endKey;
-	state int retries = 0;
-	state FlowLock::Releaser releaser;
-
-	state std::unordered_set<UID> tssToIgnore;
+static Future<Void> finishMoveKeys(Database occ,
+                                   KeyRange keys,
+                                   std::vector<UID> destinationTeam,
+                                   MoveKeysLock lock,
+                                   FlowLock* finishMoveKeysParallelismLock,
+                                   bool hasRemote,
+                                   UID relocationIntervalId,
+                                   std::map<UID, StorageServerInterface> tssMapping,
+                                   const DDEnabledState* ddEnabledState) {
+	TraceInterval interval("RelocateShard_FinishMoveKeys");
+	TraceInterval waitInterval("");
+	Future<Void> warningLogger = logWarningAfter("FinishMoveKeysTooLong", 600, destinationTeam);
+	static auto* counters = makeCounters("/movekeys/finishMoveKeys");
+	Key begin = keys.begin;
+	Key endKey;
+	int retries = 0;
+	FlowLock::Releaser releaser;
+
+	std::unordered_set<UID> tssToIgnore;
 	// try waiting for tss for a 2 loops, give up if they're behind to not affect the rest of the cluster
-	state int waitForTSSCounter = 2;
+	int waitForTSSCounter = 2;
 
 	ASSERT(!destinationTeam.empty());
 
@@ -1297,30 +1326,30 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 		while (begin < keys.end) {
 			CODE_PROBE(begin > keys.begin, "Multi-transactional finishMoveKeys");
 
-			state Transaction tr(occ);
+			Transaction tr(occ);
 
-			// printf("finishMoveKeys( '%s'-'%s' )\n", begin.toString().c_str(), keys.end.toString().c_str());
-			loop {
+			while (true) {
+				counters->started->increment(1);
+				Error err;
 				try {
-
 					tr.trState->taskID = TaskPriority::MoveKeys;
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 					releaser.release();
-					wait(finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
+					co_await finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch);
 					releaser = FlowLock::Releaser(*finishMoveKeysParallelismLock);
 
-					wait(checkMoveKeysLock(&tr, lock, ddEnabledState));
+					co_await checkMoveKeysLock(&tr, lock, ddEnabledState);
 
-					state KeyRange currentKeys = KeyRangeRef(begin, keys.end);
-					state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					KeyRange currentKeys = KeyRangeRef(begin, keys.end);
+					RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-					state RangeResult keyServers = wait(krmGetRanges(&tr,
-					                                                 keyServersPrefix,
-					                                                 currentKeys,
-					                                                 SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
-					                                                 SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
+					RangeResult keyServers = co_await krmGetRanges(&tr,
+					                                               keyServersPrefix,
+					                                               currentKeys,
+					                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+					                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
 
 					// Determine the last processed key (which will be the beginning for the next iteration)
 					endKey = keyServers.end()[-1].key;
@@ -1332,10 +1361,10 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 					// Decode and sanity check the result (dest must be the same for all ranges)
 					bool alreadyMoved = true;
 
-					state std::vector<UID> dest;
-					state std::set<UID> allServers;
-					state std::set<UID> intendedTeam(destinationTeam.begin(), destinationTeam.end());
-					state std::vector<UID> src;
+					std::vector<UID> dest;
+					std::set<UID> allServers;
+					std::set<UID> intendedTeam(destinationTeam.begin(), destinationTeam.end());
+					std::vector<UID> src;
 					std::vector<UID> completeSrc;
 
 					// Iterate through the beginning of keyServers until we find one that hasn't already been processed
@@ -1427,7 +1456,7 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 							ASSERT(false);
 						}
 					}
-					if (!dest.size()) {
+					if (dest.empty()) {
 						CODE_PROBE(true,
 						           "A previous finishMoveKeys for this range committed just as it was cancelled to "
 						           "start this one?");
@@ -1448,10 +1477,10 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 					// Wait for a durable quorum of servers in destServers to have keys available (readWrite)
 					// They must also have at least the transaction read version so they can't "forget" the shard
 					// between now and when this transaction commits.
-					state std::vector<Future<Void>> serverReady; // only for count below
-					state std::vector<Future<Void>> tssReady; // for waiting in parallel with tss
-					state std::vector<StorageServerInterface> tssReadyInterfs;
-					state std::vector<UID> newDestinations;
+					std::vector<Future<Void>> serverReady; // only for count below
+					std::vector<Future<Void>> tssReady; // for waiting in parallel with tss
+					std::vector<StorageServerInterface> tssReadyInterfs;
+					std::vector<UID> newDestinations;
 					std::set<UID> completeSrcSet(completeSrc.begin(), completeSrc.end());
 					for (auto& it : dest) {
 						if (!hasRemote || !completeSrcSet.contains(it)) {
@@ -1460,12 +1489,12 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 					}
 
 					// for smartQuorum
-					state std::vector<StorageServerInterface> storageServerInterfaces;
+					std::vector<StorageServerInterface> storageServerInterfaces;
 					std::vector<Future<Optional<Value>>> serverListEntries;
 					serverListEntries.reserve(newDestinations.size());
 					for (int s = 0; s < newDestinations.size(); s++)
 						serverListEntries.push_back(tr.get(serverListKeyFor(newDestinations[s])));
-					state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+					std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 
 					releaser.release();
 
@@ -1502,15 +1531,15 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 
 					// Wait for all storage server moves, and explicitly swallow errors for tss ones with
 					// waitForAllReady If this takes too long the transaction will time out and retry, which is ok
-					wait(timeout(waitForAll(serverReady) && waitForAllReady(tssReady),
-					             SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT,
-					             Void(),
-					             TaskPriority::MoveKeys));
+					co_await timeout(waitForAll(serverReady) && waitForAllReady(tssReady),
+					                 SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT,
+					                 Void(),
+					                 TaskPriority::MoveKeys);
 
 					// Check to see if we're waiting only on tss. If so, decrement the waiting counter.
 					// If the waiting counter is zero, ignore the slow/non-responsive tss processes before finalizing
 					// the data move.
-					if (tssReady.size()) {
+					if (!tssReady.empty()) {
 						bool allSSDone = true;
 						for (auto& f : serverReady) {
 							allSSDone &= f.isReady() && !f.isError();
@@ -1550,7 +1579,7 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 
 					TraceEvent readyServersEv(SevDebug, waitInterval.end(), relocationIntervalId);
 					readyServersEv.detail("ReadyServers", count).detail("Dests", dest.size());
-					if (tssReady.size()) {
+					if (!tssReady.empty()) {
 						readyServersEv.detail("ReadyTSS", tssCount);
 					}
 
@@ -1558,10 +1587,10 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 						// update keyServers, serverKeys
 						// SOMEDAY: Doing these in parallel is safe because none of them overlap or touch (one per
 						// server)
-						wait(krmSetRangeCoalescing(
-						    &tr, keyServersPrefix, currentKeys, keys, keyServersValue(UIDtoTagMap, dest)));
+						co_await krmSetRangeCoalescing(
+						    &tr, keyServersPrefix, currentKeys, keys, keyServersValue(UIDtoTagMap, dest));
 
-						std::set<UID>::iterator asi = allServers.begin();
+						auto asi = allServers.begin();
 						std::vector<Future<Void>> actors;
 						while (asi != allServers.end()) {
 							bool destHasServer = std::find(dest.begin(), dest.end(), *asi) != dest.end();
@@ -1573,29 +1602,63 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 							++asi;
 						}
 
-						wait(waitForAll(actors));
-						wait(tr.commit());
+						co_await waitForAll(actors);
+
+						// Inject transaction_too_old before commit to exercise the
+						// retry limit and finish_move_keys_too_many_retries path.
+						if (BUGGIFY_WITH_PROB(0.01)) {
+							CODE_PROBE(true, "finishMoveKeys injecting transaction_too_old before commit");
+							throw transaction_too_old();
+						}
+						co_await tr.commit();
+						counters->committed->increment(1);
 
 						begin = endKey;
+						retries = 0;
 						break;
 					}
+					// This leads to a count of transactions starting that exceeds the sum of
+					// committed or aborted, but this is intentional here.
 					tr.reset();
+					continue;
 				} catch (Error& error) {
-					if (error.code() == error_code_actor_cancelled)
-						throw;
-					state Error err = error;
-					wait(tr.onError(error));
-					retries++;
-					if (retries % 10 == 0) {
-						TraceEvent(retries == 20 ? SevWarnAlways : SevWarn,
-						           "RelocateShard_FinishMoveKeysRetrying",
-						           relocationIntervalId)
+					counters->aborted->increment(1);
+					err = error;
+				}
+				if (err.code() == error_code_actor_cancelled)
+					throw err;
+				co_await tr.onError(err);
+				retries++;
+				// tr.onError delays are short for transaction_too_old. With 15
+				// FlowLock slots all retrying, this creates a retry storm. Add
+				// additional exponential backoff capped at 5s.
+				if (err.code() == error_code_transaction_too_old) {
+					double backoff = finishMoveKeysBackoff(retries);
+					CODE_PROBE(true, "finishMoveKeys transaction_too_old backoff");
+					TraceEvent("FinishMoveKeysBackoff", relocationIntervalId)
+					    .suppressFor(1.0)
+					    .detail("Retries", retries)
+					    .detail("BackoffSeconds", backoff);
+					if (retries > SERVER_KNOBS->FINISH_MOVE_KEYS_MAX_RETRIES) {
+						CODE_PROBE(true, "finishMoveKeys giving up after max retries");
+						TraceEvent(SevWarnAlways, "RelocateShard_FinishMoveKeysGivingUp", relocationIntervalId)
 						    .error(err)
 						    .detail("KeyBegin", keys.begin)
 						    .detail("KeyEnd", keys.end)
-						    .detail("IterationBegin", begin)
-						    .detail("IterationEnd", endKey);
+						    .detail("Retries", retries);
+						throw finish_move_keys_too_many_retries();
 					}
+					co_await delay(backoff);
+				}
+				if (retries % 10 == 0) {
+					TraceEvent(retries == 20 ? SevWarnAlways : SevWarn,
+					           "RelocateShard_FinishMoveKeysRetrying",
+					           relocationIntervalId)
+					    .error(err)
+					    .detail("KeyBegin", keys.begin)
+					    .detail("KeyEnd", keys.end)
+					    .detail("IterationBegin", begin)
+					    .detail("IterationEnd", endKey);
 				}
 			}
 		}
@@ -1605,7 +1668,6 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 		TraceEvent(SevDebug, interval.end(), relocationIntervalId).errorUnsuppressed(e);
 		throw;
 	}
-	return Void();
 }
 
 // keyServer: map from keys to destination servers.
@@ -1614,24 +1676,24 @@ ACTOR static Future<Void> finishMoveKeys(Database occ,
 // Set keyServers[keys].dest = servers Set serverKeys[servers][keys] = dataMoveId for each
 // subrange of keys.
 // Set dataMoves[dataMoveId] = DataMoveMetaData.
-ACTOR static Future<Void> startMoveShards(Database occ,
-                                          UID dataMoveId,
-                                          std::vector<KeyRange> ranges,
-                                          std::vector<UID> servers,
-                                          MoveKeysLock lock,
-                                          FlowLock* startMoveKeysLock,
-                                          UID relocationIntervalId,
-                                          std::map<UID, StorageServerInterface>* tssMapping,
-                                          const DDEnabledState* ddEnabledState,
-                                          CancelConflictingDataMoves cancelConflictingDataMoves,
-                                          Optional<BulkLoadTaskState> bulkLoadTaskState) {
-	state Future<Void> warningLogger = logWarningAfter("StartMoveShardsTooLong", 600, servers);
-
-	wait(startMoveKeysLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser(*startMoveKeysLock);
-	state bool loadedTssMapping = false;
-	state DataMoveMetaData dataMove;
-	state Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
+static Future<Void> startMoveShards(Database occ,
+                                    UID dataMoveId,
+                                    std::vector<KeyRange> ranges,
+                                    std::vector<UID> servers,
+                                    MoveKeysLock lock,
+                                    FlowLock* startMoveKeysLock,
+                                    UID relocationIntervalId,
+                                    std::map<UID, StorageServerInterface>* tssMapping,
+                                    const DDEnabledState* ddEnabledState,
+                                    CancelConflictingDataMoves cancelConflictingDataMoves,
+                                    Optional<BulkLoadTaskState> bulkLoadTaskState) {
+	Future<Void> warningLogger = logWarningAfter("StartMoveShardsTooLong", 600, servers);
+	static auto* counters = makeCounters("/movekeys/startMoveShards");
+	co_await startMoveKeysLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser(*startMoveKeysLock);
+	bool loadedTssMapping = false;
+	DataMoveMetaData dataMove;
+	Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
 
 	TraceEvent(SevInfo, "StartMoveShardsBegin", relocationIntervalId)
 	    .detail("DataMoveID", dataMoveId)
@@ -1640,26 +1702,28 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 
 	// TODO: make startMoveShards work with multiple ranges.
 	ASSERT(ranges.size() == 1);
-	state KeyRangeRef keys = ranges[0];
-	state bool cancelDataMove = false;
-	state bool runPreCheck = true;
+	KeyRangeRef keys = ranges[0];
+	bool cancelDataMove = false;
+	bool runPreCheck = true;
 	try {
-		loop {
-			state Key begin = keys.begin;
-			state KeyRange currentKeys = keys;
+		while (true) {
+			counters->started->increment(1);
+			Key begin = keys.begin;
+			KeyRange currentKeys = keys;
 
-			state std::vector<Future<Void>> actors; // Note this clears ongoing actors from the previous iteration
-			state Transaction tr(occ);
+			std::vector<Future<Void>> actors; // Note this clears ongoing actors from the previous iteration
+			Transaction tr(occ);
 
+			Error err;
 			try {
 				tr.trState->taskID = TaskPriority::MoveKeys;
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(checkMoveKeysLock(&tr, lock, ddEnabledState));
+				co_await checkMoveKeysLock(&tr, lock, ddEnabledState);
 
-				Optional<Value> val = wait(tr.get(dataMoveKeyFor(dataMoveId)));
+				Optional<Value> val = co_await tr.get(dataMoveKeyFor(dataMoveId));
 				if (val.present()) {
 					DataMoveMetaData dmv = decodeDataMoveValue(val.get()); // dmv: Data move value.
 					dataMove = dmv;
@@ -1677,14 +1741,14 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 					if (cancelDataMove) {
 						dataMove.setPhase(DataMoveMetaData::Deleting);
 						tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
-						wait(tr.commit());
+						co_await tr.commit();
 						throw movekeys_conflict();
 					}
 					if (dataMove.getPhase() == DataMoveMetaData::Running) {
 						TraceEvent(sevDm, "StartMoveShardsDataMove", relocationIntervalId)
 						    .detail("DataMoveAlreadyCommitted", dataMoveId);
 						ASSERT(keys == dataMove.ranges.front());
-						return Void();
+						co_return;
 					}
 					begin = dataMove.ranges.front().end;
 				} else {
@@ -1700,7 +1764,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 
 				if (!loadedTssMapping) {
 					// share transaction for loading tss mapping with the rest of start move keys
-					wait(readTSSMapping(&tr, tssMapping));
+					co_await readTSSMapping(&tr, tssMapping);
 					loadedTssMapping = true;
 				}
 
@@ -1709,7 +1773,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 				for (int s = 0; s < servers.size(); s++) {
 					serverListEntries.push_back(tr.get(serverListKeyFor(servers[s])));
 				}
-				std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+				std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 
 				for (int s = 0; s < serverListValues.size(); s++) {
 					if (!serverListValues[s].present()) {
@@ -1726,24 +1790,24 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 				if (!currentKeys.empty()) {
 					const int rowLimit = SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT;
 					const int byteLimit = SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT;
-					state RangeResult old = wait(krmGetRanges(&tr, keyServersPrefix, currentKeys, rowLimit, byteLimit));
+					RangeResult old = co_await krmGetRanges(&tr, keyServersPrefix, currentKeys, rowLimit, byteLimit);
 
-					state Key endKey = old.back().key;
+					Key endKey = old.back().key;
 					currentKeys = KeyRangeRef(currentKeys.begin, endKey);
 
 					// Check that enough servers for each shard are in the correct state
-					state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
 					// For each intersecting range, update keyServers[range] dest to be servers and clear existing dest
 					// servers from serverKeys
-					state int oldIndex = 0;
+					int oldIndex = 0;
 					for (; oldIndex < old.size() - 1; ++oldIndex) {
-						state KeyRangeRef rangeIntersectKeys(old[oldIndex].key, old[oldIndex + 1].key);
-						state std::vector<UID> src;
-						state std::vector<UID> dest;
-						state UID srcId;
-						state UID destId;
+						KeyRangeRef rangeIntersectKeys(old[oldIndex].key, old[oldIndex + 1].key);
+						std::vector<UID> src;
+						std::vector<UID> dest;
+						UID srcId;
+						UID destId;
 						decodeKeyServersValue(UIDtoTagMap, old[oldIndex].value, src, dest, srcId, destId);
 						TraceEvent(sevDm, "StartMoveShardsProcessingShard", relocationIntervalId)
 						    .detail("DataMoveID", dataMoveId)
@@ -1755,7 +1819,7 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 						    .detail("ReadVersion", tr.getReadVersion().get());
 
 						if (bulkLoadTaskState.present()) {
-							state std::vector<UID> owners(src.size() + dest.size());
+							std::vector<UID> owners(src.size() + dest.size());
 							std::merge(src.begin(), src.end(), dest.begin(), dest.end(), owners.begin());
 							for (const auto& ssid : servers) {
 								if (std::find(owners.begin(), owners.end(), ssid) != owners.end()) {
@@ -1776,8 +1840,8 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 						if (SERVER_KNOBS->AUDIT_DATAMOVE_PRE_CHECK && runPreCheck) {
 							std::vector<UID> servers(src.size() + dest.size());
 							std::merge(src.begin(), src.end(), dest.begin(), dest.end(), servers.begin());
-							wait(auditLocationMetadataPreCheck(
-							    occ, &tr, rangeIntersectKeys, servers, "startMoveShards_precheck", dataMoveId));
+							co_await auditLocationMetadataPreCheck(
+							    occ, &tr, rangeIntersectKeys, servers, "startMoveShards_precheck", dataMoveId);
 						}
 
 						if (destId.isValid()) {
@@ -1796,8 +1860,8 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 							}
 
 							if (destId == anonymousShardId) {
-								wait(cleanUpSingleShardDataMove(
-								    occ, rangeIntersectKeys, lock, startMoveKeysLock, dataMoveId, ddEnabledState));
+								co_await cleanUpSingleShardDataMove(
+								    occ, rangeIntersectKeys, lock, startMoveKeysLock, dataMoveId, ddEnabledState);
 								throw retry();
 							} else {
 								if (cancelConflictingDataMoves) {
@@ -1806,10 +1870,11 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 									    .detail("Range", rangeIntersectKeys)
 									    .detail("DataMoveID", dataMoveId.toString())
 									    .detail("ExistingDataMoveID", destId.toString());
-									wait(cleanUpDataMove(occ, destId, lock, startMoveKeysLock, keys, ddEnabledState));
+									co_await cleanUpDataMove(
+									    occ, destId, lock, startMoveKeysLock, keys, ddEnabledState);
 									throw retry();
 								} else {
-									Optional<Value> val = wait(tr.get(dataMoveKeyFor(destId)));
+									Optional<Value> val = co_await tr.get(dataMoveKeyFor(destId));
 									ASSERT(val.present());
 									DataMoveMetaData dmv = decodeDataMoveValue(val.get());
 									TraceEvent(
@@ -1870,13 +1935,13 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 
 				if (currentKeys.end == keys.end) {
 					if (bulkLoadTaskState.present()) {
-						state BulkLoadTaskState newBulkLoadTaskState;
+						BulkLoadTaskState newBulkLoadTaskState;
 						try {
-							wait(store(newBulkLoadTaskState,
-							           getBulkLoadTask(&tr,
-							                           bulkLoadTaskState.get().getRange(),
-							                           bulkLoadTaskState.get().getTaskId(),
-							                           { BulkLoadPhase::Triggered, BulkLoadPhase::Running })));
+							newBulkLoadTaskState =
+							    co_await getBulkLoadTask(&tr,
+							                             bulkLoadTaskState.get().getRange(),
+							                             bulkLoadTaskState.get().getTaskId(),
+							                             { BulkLoadPhase::Triggered, BulkLoadPhase::Running });
 							// It is possible that the previous data move is cancelled but has updated the
 							// task phase as running. In this case, we update the phase from Running to Running
 							newBulkLoadTaskState.phase = BulkLoadPhase::Running;
@@ -1889,10 +1954,10 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 						}
 						newBulkLoadTaskState.setDataMoveId(dataMoveId);
 						newBulkLoadTaskState.startTime = now();
-						wait(krmSetRange(&tr,
-						                 bulkLoadTaskPrefix,
-						                 newBulkLoadTaskState.getRange(),
-						                 bulkLoadTaskStateValue(newBulkLoadTaskState)));
+						co_await krmSetRange(&tr,
+						                     bulkLoadTaskPrefix,
+						                     newBulkLoadTaskState.getRange(),
+						                     bulkLoadTaskStateValue(newBulkLoadTaskState));
 						TraceEvent(
 						    bulkLoadVerboseEventSev(), "DDBulkLoadTaskSetRunningStateTransaction", relocationIntervalId)
 						    .detail("DataMoveID", dataMoveId)
@@ -1917,9 +1982,10 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 
 				tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
 
-				wait(waitForAll(actors));
+				co_await waitForAll(actors);
 
-				wait(tr.commit());
+				co_await tr.commit();
+				counters->committed->increment(1);
 
 				if (currentKeys.end == keys.end && bulkLoadTaskState.present()) {
 					Version commitVersion = tr.getCommittedVersion();
@@ -1945,25 +2011,28 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 				if (currentKeys.end == keys.end) {
 					// Post validate consistency of update of keyServers and serverKeys
 					if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) {
-						wait(auditLocationMetadataPostCheck(occ, keys, "startMoveShards_postcheck", dataMoveId));
+						co_await auditLocationMetadataPostCheck(occ, keys, "startMoveShards_postcheck", dataMoveId);
 					}
 					break;
 				}
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_location_metadata_corruption) {
-					throw location_metadata_corruption();
-				} else if (e.code() == error_code_retry) {
-					runPreCheck = false;
-					wait(delay(1));
-				} else {
-					TraceEvent(SevWarn, "StartMoveShardsError", relocationIntervalId)
-					    .errorUnsuppressed(e)
-					    .detail("DataMoveID", dataMoveId)
-					    .detail("DataMoveRange", keys)
-					    .detail("CurrentDataMoveMetaData", dataMove.toString());
-					runPreCheck = false;
-					wait(tr.onError(e));
-				}
+				counters->aborted->increment(1);
+				err = e;
+			}
+			if (err.code() == error_code_location_metadata_corruption) {
+				throw location_metadata_corruption();
+			} else if (err.code() == error_code_retry) {
+				runPreCheck = false;
+				co_await delay(1);
+			} else {
+				TraceEvent(SevWarn, "StartMoveShardsError", relocationIntervalId)
+				    .errorUnsuppressed(err)
+				    .detail("DataMoveID", dataMoveId)
+				    .detail("DataMoveRange", keys)
+				    .detail("CurrentDataMoveMetaData", dataMove.toString());
+				runPreCheck = false;
+				co_await tr.onError(err);
 			}
 		}
 	} catch (Error& e) {
@@ -1974,16 +2043,15 @@ ACTOR static Future<Void> startMoveShards(Database occ,
 	}
 
 	TraceEvent(SevInfo, "StartMoveShardsEnd", relocationIntervalId).detail("DataMoveID", dataMoveId);
-
-	return Void();
 }
 
-ACTOR static Future<Void> checkDataMoveComplete(Database occ, UID dataMoveId, KeyRange keys, UID relocationIntervalId) {
+static Future<Void> checkDataMoveComplete(Database occ, UID dataMoveId, KeyRange keys, UID relocationIntervalId) {
 	try {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(occ);
-		state Key begin = keys.begin;
+		auto tr = makeReference<ReadYourWritesTransaction>(occ);
+		Key begin = keys.begin;
 		while (begin < keys.end) {
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr->getTransaction().trState->taskID = TaskPriority::MoveKeys;
 					tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -1991,20 +2059,20 @@ ACTOR static Future<Void> checkDataMoveComplete(Database occ, UID dataMoveId, Ke
 
 					// Get all existing shards overlapping keys (exclude any that have been processed in a previous
 					// iteration of the outer loop)
-					state KeyRange currentKeys = KeyRangeRef(begin, keys.end);
+					KeyRange currentKeys = KeyRangeRef(begin, keys.end);
 
-					state RangeResult keyServers = wait(krmGetRanges(tr,
-					                                                 keyServersPrefix,
-					                                                 currentKeys,
-					                                                 SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
-					                                                 SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT));
+					RangeResult keyServers = co_await krmGetRanges(tr,
+					                                               keyServersPrefix,
+					                                               currentKeys,
+					                                               SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
+					                                               SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT);
 
 					// Determine the last processed key (which will be the beginning for the next iteration)
-					state Key endKey = keyServers.back().key;
+					Key endKey = keyServers.back().key;
 					currentKeys = KeyRangeRef(currentKeys.begin, endKey);
 
 					// Check that enough servers for each shard are in the correct state
-					state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
 					for (int i = 0; i < keyServers.size() - 1; ++i) {
@@ -2030,47 +2098,47 @@ ACTOR static Future<Void> checkDataMoveComplete(Database occ, UID dataMoveId, Ke
 					begin = endKey;
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 				}
+				co_await tr->onError(err);
 			}
 		}
 	} catch (Error& e) {
 		TraceEvent(SevDebug, "CheckDataMoveCompleteError", relocationIntervalId).errorUnsuppressed(e);
 		throw;
 	}
-
-	return Void();
 }
 
 // Set keyServers[keys].src = keyServers[keys].dest and keyServers[keys].dest=[], return when successful
 // keyServers[k].dest must be the same for all k in keys.
 // Set serverKeys[dest][keys] = dataMoveId; serverKeys[src][keys] = false for all src not in dest.
 // Clear dataMoves[dataMoveId].
-ACTOR static Future<Void> finishMoveShards(Database occ,
-                                           UID dataMoveId,
-                                           std::vector<KeyRange> targetRanges,
-                                           std::vector<UID> destinationTeam,
-                                           MoveKeysLock lock,
-                                           FlowLock* finishMoveKeysParallelismLock,
-                                           bool hasRemote,
-                                           UID relocationIntervalId,
-                                           std::map<UID, StorageServerInterface> tssMapping,
-                                           const DDEnabledState* ddEnabledState,
-                                           Optional<BulkLoadTaskState> bulkLoadTaskState) {
+static Future<Void> finishMoveShards(Database occ,
+                                     UID dataMoveId,
+                                     std::vector<KeyRange> targetRanges,
+                                     std::vector<UID> destinationTeam,
+                                     MoveKeysLock lock,
+                                     FlowLock* finishMoveKeysParallelismLock,
+                                     bool hasRemote,
+                                     UID relocationIntervalId,
+                                     std::map<UID, StorageServerInterface> tssMapping,
+                                     const DDEnabledState* ddEnabledState,
+                                     Optional<BulkLoadTaskState> bulkLoadTaskState) {
 	// TODO: make startMoveShards work with multiple ranges.
 	ASSERT(targetRanges.size() == 1);
-	state KeyRange keys = targetRanges[0];
-	state Future<Void> warningLogger = logWarningAfter("FinishMoveShardsTooLong", 600, destinationTeam);
-	state int retries = 0;
-	state DataMoveMetaData dataMove;
-	state bool cancelDataMove = false;
-	state Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
-
-	wait(finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser = FlowLock::Releaser(*finishMoveKeysParallelismLock);
-	state bool runPreCheck = true;
-	state bool skipTss = false;
-	state double ssReadyTime = std::numeric_limits<double>::max();
+	KeyRange keys = targetRanges[0];
+	Future<Void> warningLogger = logWarningAfter("FinishMoveShardsTooLong", 600, destinationTeam);
+	static auto* counters = makeCounters("/movekeys/finishMoveShards");
+	int retries = 0;
+	DataMoveMetaData dataMove;
+	bool cancelDataMove = false;
+	Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
+
+	co_await finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser = FlowLock::Releaser(*finishMoveKeysParallelismLock);
+	bool runPreCheck = true;
+	bool skipTss = false;
+	double ssReadyTime = std::numeric_limits<double>::max();
 
 	ASSERT(!destinationTeam.empty());
 
@@ -2081,21 +2149,23 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 
 		// This process can be split up into multiple transactions if getRange() doesn't return the entire
 		// target range.
-		loop {
-			state std::vector<UID> completeSrc;
-			state std::vector<UID> destServers;
-			state std::unordered_set<UID> allServers;
-			state KeyRange range;
-			state Transaction tr(occ);
+		while (true) {
+			counters->started->increment(1);
+			std::vector<UID> completeSrc;
+			std::vector<UID> destServers;
+			std::unordered_set<UID> allServers;
+			KeyRange range;
+			Transaction tr(occ);
+			Error err;
 			try {
 				tr.trState->taskID = TaskPriority::MoveKeys;
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				wait(checkMoveKeysLock(&tr, lock, ddEnabledState));
+				co_await checkMoveKeysLock(&tr, lock, ddEnabledState);
 
-				Optional<Value> val = wait(tr.get(dataMoveKeyFor(dataMoveId)));
+				Optional<Value> val = co_await tr.get(dataMoveKeyFor(dataMoveId));
 				if (val.present()) {
 					dataMove = decodeDataMoveValue(val.get());
 					TraceEvent(sevDm, "FinishMoveShardsFoundDataMove", relocationIntervalId)
@@ -2107,7 +2177,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 					if (cancelDataMove) {
 						dataMove.setPhase(DataMoveMetaData::Deleting);
 						tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
-						wait(tr.commit());
+						co_await tr.commit();
 						throw movekeys_conflict();
 					}
 					destServers.insert(destServers.end(), dataMove.dest.begin(), dataMove.dest.end());
@@ -2126,28 +2196,28 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 					    .detail("AtVerison", tr.getReadVersion().get())
 					    .detail("DataMoveID", dataMoveId)
 					    .detail("DataMove", dataMove.toString());
-					wait(checkDataMoveComplete(occ, dataMoveId, keys, relocationIntervalId));
-					return Void();
+					co_await checkDataMoveComplete(occ, dataMoveId, keys, relocationIntervalId);
+					co_return;
 				}
 
-				state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state RangeResult keyServers = wait(krmGetRanges(&tr,
-				                                                 keyServersPrefix,
-				                                                 range,
-				                                                 SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
-				                                                 SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT));
+				RangeResult keyServers = co_await krmGetRanges(&tr,
+				                                               keyServersPrefix,
+				                                               range,
+				                                               SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
+				                                               SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT);
 				ASSERT(!keyServers.empty());
 				range = KeyRangeRef(range.begin, keyServers.back().key);
 				ASSERT(!range.empty());
 
-				state int currentIndex = 0;
+				int currentIndex = 0;
 				for (; currentIndex < keyServers.size() - 1; ++currentIndex) {
-					state std::vector<UID> src;
-					state std::vector<UID> dest;
-					state UID srcId;
-					state UID destId;
+					std::vector<UID> src;
+					std::vector<UID> dest;
+					UID srcId;
+					UID destId;
 					decodeKeyServersValue(UIDtoTagMap, keyServers[currentIndex].value, src, dest, srcId, destId);
 					const KeyRange currentRange =
 					    KeyRangeRef(keyServers[currentIndex].key, keyServers[currentIndex + 1].key);
@@ -2173,8 +2243,8 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 					if (SERVER_KNOBS->AUDIT_DATAMOVE_PRE_CHECK && runPreCheck) {
 						std::vector<UID> servers(src.size() + dest.size());
 						std::merge(src.begin(), src.end(), dest.begin(), dest.end(), servers.begin());
-						wait(auditLocationMetadataPreCheck(
-						    occ, &tr, currentRange, servers, "finishMoveShards_precheck", dataMoveId));
+						co_await auditLocationMetadataPreCheck(
+						    occ, &tr, currentRange, servers, "finishMoveShards_precheck", dataMoveId);
 					}
 
 					std::sort(dest.begin(), dest.end());
@@ -2199,10 +2269,10 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 				// Wait for a durable quorum of servers in destServers to have keys available (readWrite)
 				// They must also have at least the transaction read version so they can't "forget" the shard
 				// between now and when this transaction commits.
-				state std::vector<Future<Void>> serverReady; // only for count below
-				state std::vector<Future<Void>> tssReady; // for waiting in parallel with tss
-				state std::vector<StorageServerInterface> tssReadyInterfs;
-				state std::vector<UID> newDestinations;
+				std::vector<Future<Void>> serverReady; // only for count below
+				std::vector<Future<Void>> tssReady; // for waiting in parallel with tss
+				std::vector<StorageServerInterface> tssReadyInterfs;
+				std::vector<UID> newDestinations;
 				std::set<UID> completeSrcSet(completeSrc.begin(), completeSrc.end());
 				for (const UID& id : destServers) {
 					if (!hasRemote || !completeSrcSet.contains(id)) {
@@ -2210,13 +2280,13 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 					}
 				}
 
-				state std::vector<StorageServerInterface> storageServerInterfaces;
+				std::vector<StorageServerInterface> storageServerInterfaces;
 				std::vector<Future<Optional<Value>>> serverListEntries;
 				serverListEntries.reserve(newDestinations.size());
 				for (const UID& id : newDestinations) {
 					serverListEntries.push_back(tr.get(serverListKeyFor(id)));
 				}
-				state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+				std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 
 				releaser.release();
 
@@ -2257,12 +2327,12 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 
 				// Wait for all storage server moves, and explicitly swallow errors for tss ones with
 				// waitForAllReady If this takes too long the transaction will time out and retry, which is ok
-				wait(timeout(waitForAll(serverReady) && waitForAllReady(tssReady),
-				             SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT,
-				             Void(),
-				             TaskPriority::MoveKeys));
+				co_await timeout(waitForAll(serverReady) && waitForAllReady(tssReady),
+				                 SERVER_KNOBS->SERVER_READY_QUORUM_TIMEOUT,
+				                 Void(),
+				                 TaskPriority::MoveKeys);
 
-				state std::vector<UID> readyServers;
+				std::vector<UID> readyServers;
 				for (int s = 0; s < serverReady.size(); ++s) {
 					if (serverReady[s].isReady() && !serverReady[s].isError()) {
 						readyServers.push_back(storageServerInterfaces[s].uniqueID);
@@ -2319,17 +2389,17 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 						    .detail("DataMove", dataMove.toString());
 					}
 
-					wait(waitForAll(actors));
+					co_await waitForAll(actors);
 
 					if (range.end == dataMove.ranges.front().end) {
 						if (bulkLoadTaskState.present()) {
-							state BulkLoadTaskState newBulkLoadTaskState;
+							BulkLoadTaskState newBulkLoadTaskState;
 							try {
-								wait(store(newBulkLoadTaskState,
-								           getBulkLoadTask(&tr,
-								                           bulkLoadTaskState.get().getRange(),
-								                           bulkLoadTaskState.get().getTaskId(),
-								                           { BulkLoadPhase::Running, BulkLoadPhase::Complete })));
+								newBulkLoadTaskState =
+								    co_await getBulkLoadTask(&tr,
+								                             bulkLoadTaskState.get().getRange(),
+								                             bulkLoadTaskState.get().getTaskId(),
+								                             { BulkLoadPhase::Running, BulkLoadPhase::Complete });
 								newBulkLoadTaskState.phase = BulkLoadPhase::Complete;
 							} catch (Error& e) {
 								if (e.code() == error_code_bulkload_task_outdated) {
@@ -2341,10 +2411,10 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 							ASSERT(newBulkLoadTaskState.getDataMoveId().present() &&
 							       newBulkLoadTaskState.getDataMoveId().get() == dataMoveId);
 							newBulkLoadTaskState.completeTime = now();
-							wait(krmSetRange(&tr,
-							                 bulkLoadTaskPrefix,
-							                 newBulkLoadTaskState.getRange(),
-							                 bulkLoadTaskStateValue(newBulkLoadTaskState)));
+							co_await krmSetRange(&tr,
+							                     bulkLoadTaskPrefix,
+							                     newBulkLoadTaskState.getRange(),
+							                     bulkLoadTaskStateValue(newBulkLoadTaskState));
 							TraceEvent(
 							    bulkLoadVerboseEventSev(), "DDBulkLoadTaskSetCompleteTransaction", relocationIntervalId)
 							    .detail("DataMoveID", dataMoveId)
@@ -2352,7 +2422,7 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 							    .detail("TaskID", newBulkLoadTaskState.getTaskId());
 							dataMove.bulkLoadTaskState = newBulkLoadTaskState;
 						}
-						wait(deleteCheckpoints(&tr, dataMove.checkpoints, dataMoveId));
+						co_await deleteCheckpoints(&tr, dataMove.checkpoints, dataMoveId);
 						tr.clear(dataMoveKeyFor(dataMoveId));
 						TraceEvent(sevDm, "FinishMoveShardsDeleteMetaData", relocationIntervalId)
 						    .detail("DataMove", dataMove.toString());
@@ -2367,7 +2437,8 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 						tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
 					}
 
-					wait(tr.commit());
+					co_await tr.commit();
+					counters->committed->increment(1);
 
 					if (range.end == dataMove.ranges.front().end && bulkLoadTaskState.present()) {
 						Version commitVersion = tr.getCommittedVersion();
@@ -2383,38 +2454,41 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 					if (range.end == dataMove.ranges.front().end) {
 						// Post validate consistency of update of keyServers and serverKeys
 						if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) {
-							wait(auditLocationMetadataPostCheck(
-							    occ, dataMove.ranges.front(), "finishMoveShards_postcheck", relocationIntervalId));
+							co_await auditLocationMetadataPostCheck(
+							    occ, dataMove.ranges.front(), "finishMoveShards_postcheck", relocationIntervalId);
 						}
 						break;
 					}
+					continue;
 				} else {
 					tr.reset();
+					continue;
 				}
 			} catch (Error& error) {
-				TraceEvent(SevWarn, "TryFinishMoveShardsError", relocationIntervalId)
-				    .errorUnsuppressed(error)
-				    .detail("DataMoveID", dataMoveId);
-				if (error.code() == error_code_location_metadata_corruption) {
-					throw location_metadata_corruption();
-				} else if (error.code() == error_code_retry) {
-					runPreCheck = false;
-					++retries;
-					wait(delay(1));
-				} else if (error.code() == error_code_actor_cancelled) {
-					throw;
-				} else {
-					state Error err = error;
-					runPreCheck = false;
-					wait(tr.onError(err));
-					retries++;
-					if (retries % 10 == 0) {
-						TraceEvent(retries == 20 ? SevWarnAlways : SevWarn,
-						           "RelocateShard_FinishMoveShardsRetrying",
-						           relocationIntervalId)
-						    .error(err)
-						    .detail("DataMoveID", dataMoveId);
-					}
+				counters->aborted->increment(1);
+				err = error;
+			}
+			TraceEvent(SevWarn, "TryFinishMoveShardsError", relocationIntervalId)
+			    .errorUnsuppressed(err)
+			    .detail("DataMoveID", dataMoveId);
+			if (err.code() == error_code_location_metadata_corruption) {
+				throw location_metadata_corruption();
+			} else if (err.code() == error_code_retry) {
+				runPreCheck = false;
+				++retries;
+				co_await delay(1);
+			} else if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			} else {
+				runPreCheck = false;
+				co_await tr.onError(err);
+				retries++;
+				if (retries % 10 == 0) {
+					TraceEvent(retries == 20 ? SevWarnAlways : SevWarn,
+					           "RelocateShard_FinishMoveShardsRetrying",
+					           relocationIntervalId)
+					    .error(err)
+					    .detail("DataMoveID", dataMoveId);
 				}
 			}
 		}
@@ -2427,73 +2501,75 @@ ACTOR static Future<Void> finishMoveShards(Database occ,
 	    .detail("DataMoveID", dataMoveId)
 	    .detail("BulkLoadTaskID", bulkLoadTaskState.present() ? bulkLoadTaskState.get().getTaskId().toString() : "")
 	    .detail("DataMove", dataMove.toString());
-	return Void();
 }
 
 }; // anonymous namespace
 
-ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServerInterface server) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
-	state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
-	                                                                                           IncludeVersion());
+Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServerInterface server) {
+	static auto* counters = makeCounters("/movekeys/addStorageServer");
+	auto tr = makeReference<ReadYourWritesTransaction>(cx);
+	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
+	KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
+	                                                                                     IncludeVersion());
 
-	state int maxSkipTags = 1;
+	int maxSkipTags = 1;
 
-	loop {
+	while (true) {
+		counters->started->increment(1);
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
 			// FIXME: don't fetch tag localities, all tags, and history tags if tss. Just fetch pair's tag
-			state Future<RangeResult> fTagLocalities = tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
-			state Future<Optional<Value>> fv = tr->get(serverListKeyFor(server.id()));
+			Future<RangeResult> fTagLocalities = tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
+			Future<Optional<Value>> fv = tr->get(serverListKeyFor(server.id()));
 
-			state Future<Optional<Value>> fExclProc = tr->get(
+			Future<Optional<Value>> fExclProc = tr->get(
 			    StringRef(encodeExcludedServersKey(AddressExclusion(server.address().ip, server.address().port))));
-			state Future<Optional<Value>> fExclIP =
+			Future<Optional<Value>> fExclIP =
 			    tr->get(StringRef(encodeExcludedServersKey(AddressExclusion(server.address().ip))));
-			state Future<Optional<Value>> fFailProc = tr->get(
+			Future<Optional<Value>> fFailProc = tr->get(
 			    StringRef(encodeFailedServersKey(AddressExclusion(server.address().ip, server.address().port))));
-			state Future<Optional<Value>> fFailIP =
+			Future<Optional<Value>> fFailIP =
 			    tr->get(StringRef(encodeFailedServersKey(AddressExclusion(server.address().ip))));
 
-			state Future<Optional<Value>> fExclProc2 =
+			Future<Optional<Value>> fExclProc2 =
 			    server.secondaryAddress().present()
 			        ? tr->get(StringRef(encodeExcludedServersKey(
 			              AddressExclusion(server.secondaryAddress().get().ip, server.secondaryAddress().get().port))))
 			        : Future<Optional<Value>>(Optional<Value>());
-			state Future<Optional<Value>> fExclIP2 =
+			Future<Optional<Value>> fExclIP2 =
 			    server.secondaryAddress().present()
 			        ? tr->get(StringRef(encodeExcludedServersKey(AddressExclusion(server.secondaryAddress().get().ip))))
 			        : Future<Optional<Value>>(Optional<Value>());
-			state Future<Optional<Value>> fFailProc2 =
+			Future<Optional<Value>> fFailProc2 =
 			    server.secondaryAddress().present()
 			        ? tr->get(StringRef(encodeFailedServersKey(
 			              AddressExclusion(server.secondaryAddress().get().ip, server.secondaryAddress().get().port))))
 			        : Future<Optional<Value>>(Optional<Value>());
-			state Future<Optional<Value>> fFailIP2 =
+			Future<Optional<Value>> fFailIP2 =
 			    server.secondaryAddress().present()
 			        ? tr->get(StringRef(encodeFailedServersKey(AddressExclusion(server.secondaryAddress().get().ip))))
 			        : Future<Optional<Value>>(Optional<Value>());
 
-			state std::vector<Future<Optional<Value>>> localityExclusions;
+			std::vector<Future<Optional<Value>>> localityExclusions;
 			std::map<std::string, std::string> localityData = server.locality.getAllData();
 			for (const auto& l : localityData) {
 				localityExclusions.push_back(tr->get(StringRef(encodeExcludedLocalityKey(
 				    LocalityData::ExcludeLocalityPrefix.toString() + l.first + ":" + l.second))));
 			}
 
-			state Future<RangeResult> fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::True);
-			state Future<RangeResult> fHistoryTags =
+			Future<RangeResult> fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::True);
+			Future<RangeResult> fHistoryTags =
 			    tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::True);
 
-			wait(success(fTagLocalities) && success(fv) && success(fTags) && success(fHistoryTags) &&
-			     success(fExclProc) && success(fExclIP) && success(fFailProc) && success(fFailIP) &&
-			     success(fExclProc2) && success(fExclIP2) && success(fFailProc2) && success(fFailIP2));
+			co_await (success(fTagLocalities) && success(fv) && success(fTags) && success(fHistoryTags) &&
+			          success(fExclProc) && success(fExclIP) && success(fFailProc) && success(fFailIP) &&
+			          success(fExclProc2) && success(fExclIP2) && success(fFailProc2) && success(fFailIP2));
 
 			for (const auto& exclusion : localityExclusions) {
-				wait(success(exclusion));
+				co_await exclusion;
 			}
 
 			// If we have been added to the excluded/failed state servers or localities list, we have to fail
@@ -2512,7 +2588,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 			if (fTagLocalities.get().more || fTags.get().more || fHistoryTags.get().more)
 				ASSERT(false);
 
-			state Tag tag;
+			Tag tag;
 			if (server.isTss()) {
 				bool foundTag = false;
 				for (auto& it : fTags.get()) {
@@ -2531,7 +2607,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 
 			} else {
 				int8_t maxTagLocality = 0;
-				state int8_t locality = -1;
+				int8_t locality = -1;
 				for (auto& kv : fTagLocalities.get()) {
 					int8_t loc = decodeTagLocalityListValue(kv.value);
 					if (decodeTagLocalityListKey(kv.key) == server.locality.dcId()) {
@@ -2551,7 +2627,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 
 				int skipTags = deterministicRandom()->randomInt(0, maxSkipTags);
 
-				state uint16_t tagId = 0;
+				uint16_t tagId = 0;
 				std::vector<uint16_t> usedTags;
 				for (auto& it : fTags.get()) {
 					Tag t = decodeServerTagValue(it.value);
@@ -2568,7 +2644,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 				std::sort(usedTags.begin(), usedTags.end());
 
 				int usedIdx = 0;
-				for (; usedTags.size() > 0 && tagId <= usedTags.end()[-1]; tagId++) {
+				for (; !usedTags.empty() && tagId <= usedTags.end()[-1]; tagId++) {
 					if (tagId < usedTags[usedIdx]) {
 						if (skipTags == 0)
 							break;
@@ -2598,28 +2674,31 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
 			tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
 
 			tr->set(serverListKeyFor(server.id()), serverListValue(server));
-			wait(tr->commit());
+			co_await tr->commit();
+			counters->committed->increment(1);
 			TraceEvent("AddedStorageServerSystemKey")
 			    .detail("ServerID", server.id())
 			    .detail("CommitVersion", tr->getCommittedVersion());
 
-			return std::make_pair(tr->getCommittedVersion(), tag);
+			co_return std::make_pair(tr->getCommittedVersion(), tag);
 		} catch (Error& e) {
-			if (e.code() == error_code_commit_unknown_result)
-				throw recruitment_failed(); // There is a remote possibility that we successfully added ourselves and
-				                            // then someone removed us, so we have to fail
-
-			if (e.code() == error_code_not_committed) {
-				maxSkipTags = SERVER_KNOBS->MAX_SKIP_TAGS;
-			}
+			counters->aborted->increment(1);
+			err = e;
+		}
+		if (err.code() == error_code_commit_unknown_result)
+			throw recruitment_failed(); // There is a remote possibility that we successfully added ourselves and
+			                            // then someone removed us, so we have to fail
 
-			wait(tr->onError(e));
+		if (err.code() == error_code_not_committed) {
+			maxSkipTags = SERVER_KNOBS->MAX_SKIP_TAGS;
 		}
+
+		co_await tr->onError(err);
 	}
 }
 // A SS can be removed only if all data (shards) on the SS have been moved away from the SS.
-ACTOR Future<bool> canRemoveStorageServer(Reference<ReadYourWritesTransaction> tr, UID serverID) {
-	RangeResult keys = wait(krmGetRanges(tr, serverKeysPrefixFor(serverID), allKeys, 2));
+Future<bool> canRemoveStorageServer(Reference<ReadYourWritesTransaction> tr, UID serverID) {
+	RangeResult keys = co_await krmGetRanges(tr, serverKeysPrefixFor(serverID), allKeys, 2);
 
 	ASSERT(keys.size() >= 2);
 
@@ -2644,32 +2723,35 @@ ACTOR Future<bool> canRemoveStorageServer(Reference<ReadYourWritesTransaction> t
 	    .detail("Value1", keys[0].value)
 	    .detail("Key2", keys[1].key)
 	    .detail("Value2", keys[1].value);
-	return !assigned && keys[1].key == allKeys.end;
+	co_return !assigned && keys[1].key == allKeys.end;
 }
 
-ACTOR Future<Void> removeStorageServer(Database cx,
-                                       UID serverID,
-                                       Optional<UID> tssPairID,
-                                       MoveKeysLock lock,
-                                       const DDEnabledState* ddEnabledState) {
-	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
-	state KeyBackedObjectMap<UID, StorageMigrationType, decltype(IncludeVersion())> metadataMap(
-	    serverMetadataKeys.begin, IncludeVersion());
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-	state bool retry = false;
-	state int noCanRemoveCount = 0;
-
-	loop {
+Future<Void> removeStorageServer(Database cx,
+                                 UID serverID,
+                                 Optional<UID> tssPairID,
+                                 MoveKeysLock lock,
+                                 const DDEnabledState* ddEnabledState) {
+	static auto* counters = makeCounters("/movekeys/removeStorageServer");
+	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
+	KeyBackedObjectMap<UID, StorageMigrationType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
+	                                                                                      IncludeVersion());
+	auto tr = makeReference<ReadYourWritesTransaction>(cx);
+	bool retry = false;
+	int noCanRemoveCount = 0;
+
+	while (true) {
+		counters->started->increment(1);
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			wait(checkMoveKeysLock(&(tr->getTransaction()), lock, ddEnabledState));
+			co_await checkMoveKeysLock(&(tr->getTransaction()), lock, ddEnabledState);
 			TraceEvent("RemoveStorageServer")
 			    .detail("State", "Locked")
 			    .detail("ServerID", serverID)
 			    .detail("Version", tr->getReadVersion().get());
 
-			state bool canRemove = wait(canRemoveStorageServer(tr, serverID));
+			bool canRemove = co_await canRemoveStorageServer(tr, serverID);
 			if (!canRemove) {
 				CODE_PROBE(true,
 				           "The caller had a transaction in flight that assigned keys to the server.  Wait for it to "
@@ -2678,22 +2760,23 @@ ACTOR Future<Void> removeStorageServer(Database cx,
 				    .detail("State", "CanRemoveFailed")
 				    .detail("ServerID", serverID)
 				    .detail("Count", noCanRemoveCount++);
-				wait(delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskPriority::DataDistributionLaunch));
+				co_await delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskPriority::DataDistributionLaunch);
 				tr->reset();
+				continue;
 			} else {
-				state Future<Optional<Value>> fListKey = tr->get(serverListKeyFor(serverID));
-				state Future<RangeResult> fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fHistoryTags = tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fTagLocalities = tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fTLogDatacenters = tr->getRange(tLogDatacentersKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<Optional<Value>> fListKey = tr->get(serverListKeyFor(serverID));
+				Future<RangeResult> fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> fHistoryTags = tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> fTagLocalities = tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> fTLogDatacenters = tr->getRange(tLogDatacentersKeys, CLIENT_KNOBS->TOO_MANY);
 
-				wait(success(fListKey) && success(fTags) && success(fHistoryTags) && success(fTagLocalities) &&
-				     success(fTLogDatacenters));
+				co_await (success(fListKey) && success(fTags) && success(fHistoryTags) && success(fTagLocalities) &&
+				          success(fTLogDatacenters));
 
 				if (!fListKey.get().present()) {
 					if (retry) {
 						CODE_PROBE(true, "Storage server already removed after retrying transaction");
-						return Void();
+						co_return;
 					}
 					TraceEvent(SevError, "RemoveInvalidServer").detail("ServerID", serverID);
 					ASSERT(false); // Removing an already-removed server?  A never added server?
@@ -2746,7 +2829,7 @@ ACTOR Future<Void> removeStorageServer(Database cx,
 					tssMapDB.erase(tr, tssPairID.get());
 					// remove the TSS from quarantine, if it was in quarantine
 					Key tssQuarantineKey = tssQuarantineKeyFor(serverID);
-					Optional<Value> tssInQuarantine = wait(tr->get(tssQuarantineKey));
+					Optional<Value> tssInQuarantine = co_await tr->get(tssQuarantineKey);
 					if (tssInQuarantine.present()) {
 						tr->clear(tssQuarantineKeyFor(serverID));
 					}
@@ -2756,44 +2839,49 @@ ACTOR Future<Void> removeStorageServer(Database cx,
 				tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
 
 				retry = true;
-				wait(tr->commit());
+				co_await tr->commit();
+				counters->committed->increment(1);
 				TraceEvent("RemoveStorageServer")
 				    .detail("State", "Success")
 				    .detail("ServerID", serverID)
 				    .detail("CommitVersion", tr->getCommittedVersion());
-				return Void();
+				co_return;
 			}
 		} catch (Error& e) {
-			state Error err = e;
-			wait(tr->onError(e));
-			TraceEvent("RemoveStorageServer").error(err).detail("State", "Retry").detail("ServerID", serverID);
+			counters->aborted->increment(1);
+			err = e;
 		}
+		co_await tr->onError(err);
+		TraceEvent("RemoveStorageServer").error(err).detail("State", "Retry").detail("ServerID", serverID);
 	}
 }
 // Remove the server from keyServer list and set serverKeysFalse to the server's serverKeys list.
 // Changes to keyServer and serverKey must happen symmetrically in a transaction.
 // If serverID is the last source server for a shard, the shard will be erased, and then be assigned
 // to teamForDroppedRange.
-ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
-                                              UID serverID,
-                                              std::vector<UID> teamForDroppedRange,
-                                              MoveKeysLock lock,
-                                              const DDEnabledState* ddEnabledState) {
-	state Key begin = allKeys.begin;
-
-	state std::vector<UID> src;
-	state std::vector<UID> dest;
-	state UID srcId;
-	state UID destId;
+Future<Void> removeKeysFromFailedServer(Database cx,
+                                        UID serverID,
+                                        std::vector<UID> teamForDroppedRange,
+                                        MoveKeysLock lock,
+                                        const DDEnabledState* ddEnabledState) {
+	static auto* counters = makeCounters("/movekeys/removeKeysFromFailedServer");
+	Key begin = allKeys.begin;
+
+	std::vector<UID> src;
+	std::vector<UID> dest;
+	UID srcId;
+	UID destId;
 	// Multi-transactional removal in case of large number of shards, concern in violating 5s transaction limit
 	while (begin < allKeys.end) {
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			counters->started->increment(1);
+			Error err;
 			try {
 				tr.trState->taskID = TaskPriority::MoveKeys;
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				wait(checkMoveKeysLock(&tr, lock, ddEnabledState));
+				co_await checkMoveKeysLock(&tr, lock, ddEnabledState);
 				TraceEvent("RemoveKeysFromFailedServerLocked")
 				    .detail("ServerID", serverID)
 				    .detail("Version", tr.getReadVersion().get())
@@ -2801,17 +2889,17 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 				// Get all values of keyServers and remove serverID from every occurrence
 				// Very inefficient going over every entry in keyServers
 				// No shortcut because keyServers and serverKeys are not guaranteed same shard boundaries
-				state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-				state RangeResult keyServers = wait(krmGetRanges(&tr,
-				                                                 keyServersPrefix,
-				                                                 KeyRangeRef(begin, allKeys.end),
-				                                                 SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
-				                                                 SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
-				state KeyRange currentKeys = KeyRangeRef(begin, keyServers.end()[-1].key);
-				state int i = 0;
+				RangeResult keyServers = co_await krmGetRanges(&tr,
+				                                               keyServersPrefix,
+				                                               KeyRangeRef(begin, allKeys.end),
+				                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+				                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
+				KeyRange currentKeys = KeyRangeRef(begin, keyServers.end()[-1].key);
+				int i = 0;
 				for (; i < keyServers.size() - 1; ++i) {
-					state KeyValueRef it = keyServers[i];
+					KeyValueRef it = keyServers[i];
 					decodeKeyServersValue(UIDtoTagMap, it.value, src, dest, srcId, destId);
 
 					// The failed server is not present
@@ -2825,12 +2913,12 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 					src.erase(std::remove(src.begin(), src.end(), serverID), src.end());
 					dest.erase(std::remove(dest.begin(), dest.end(), serverID), dest.end());
 
-					state KeyRangeRef range(it.key, keyServers[i + 1].key);
+					KeyRangeRef range(it.key, keyServers[i + 1].key);
 
 					// If the last src server is to be removed, first check if there are dest servers who is
 					// hosting a read-write copy of the keyrange, and move such dest servers to the src list.
 					if (src.empty() && !dest.empty()) {
-						std::vector<UID> newSources = wait(pickReadWriteServers(&tr, dest, range));
+						std::vector<UID> newSources = co_await pickReadWriteServers(&tr, dest, range);
 						for (const UID& id : newSources) {
 							TraceEvent(SevWarn, "FailedServerAdditionalSourceServer", serverID)
 							    .detail("Range", range)
@@ -2855,9 +2943,9 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 						}
 
 						if (destId.isValid() && destId != anonymousShardId) {
-							Optional<Value> val = wait(tr.get(dataMoveKeyFor(destId)));
+							Optional<Value> val = co_await tr.get(dataMoveKeyFor(destId));
 							if (val.present()) {
-								state DataMoveMetaData dataMove = decodeDataMoveValue(val.get());
+								DataMoveMetaData dataMove = decodeDataMoveValue(val.get());
 								ASSERT(!dataMove.ranges.empty());
 								TraceEvent(SevVerbose, "RemoveRangeFoundDataMove", serverID)
 								    .detail("DataMoveMetaData", dataMove.toString());
@@ -2906,7 +2994,7 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 							}
 						}
 
-						wait(waitForAll(actors));
+						co_await waitForAll(actors);
 
 						TraceEvent trace(SevWarnAlways, "ShardLossAllReplicasDropShard", serverID);
 						trace.detail("Begin", it.key);
@@ -2934,8 +3022,10 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 				TraceEvent(SevDebug, "FailedServerSetRange", serverID)
 				    .detail("Begin", currentKeys.begin)
 				    .detail("End", currentKeys.end);
-				wait(krmSetRangeCoalescing(&tr, serverKeysPrefixFor(serverID), currentKeys, allKeys, serverKeysFalse));
-				wait(tr.commit());
+				co_await krmSetRangeCoalescing(
+				    &tr, serverKeysPrefixFor(serverID), currentKeys, allKeys, serverKeysFalse);
+				co_await tr.commit();
+				counters->committed->increment(1);
 				TraceEvent(SevDebug, "FailedServerCommitSuccess", serverID)
 				    .detail("Begin", currentKeys.begin)
 				    .detail("End", currentKeys.end)
@@ -2944,12 +3034,13 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 				begin = currentKeys.end;
 				break;
 			} catch (Error& e) {
-				TraceEvent("FailedServerError", serverID).error(e);
-				wait(tr.onError(e));
+				counters->aborted->increment(1);
+				err = e;
 			}
+			TraceEvent("FailedServerError", serverID).error(err);
+			co_await tr.onError(err);
 		}
 	}
-	return Void();
 }
 
 // In cleanUpDataMoveCore, to do the actual cleanup, we suppose the target data move already update its
@@ -2971,29 +3062,32 @@ ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
 // the place holder on the metadata put by cleanUpDataMoveCore. Then, startMoveShard gives up and exits.
 // No update to the metadata by the startMoveShard
 // For all three cases, the background cleanup only needs to cleanup the place holder
-ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
-                                             UID dataMoveId,
-                                             MoveKeysLock lock,
-                                             FlowLock* cleanUpDataMoveParallelismLock,
-                                             KeyRange keys,
-                                             const DDEnabledState* ddEnabledState,
-                                             double delaySeconds) {
-	wait(delay(std::max(10.0, delaySeconds)));
+Future<Void> cleanUpDataMoveBackground(Database occ,
+                                       UID dataMoveId,
+                                       MoveKeysLock lock,
+                                       FlowLock* cleanUpDataMoveParallelismLock,
+                                       KeyRange keys,
+                                       const DDEnabledState* ddEnabledState,
+                                       double delaySeconds) {
+	static auto* counters = makeCounters("/movekeys/cleanUpDataMoveBackground");
+	co_await delay(std::max(10.0, delaySeconds));
 	TraceEvent(SevDebug, "CleanUpDataMoveBackgroundBegin", dataMoveId)
 	    .detail("DataMoveID", dataMoveId)
 	    .detail("Range", keys);
-	wait(cleanUpDataMoveParallelismLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser = FlowLock::Releaser(*cleanUpDataMoveParallelismLock);
-	state DataMoveMetaData dataMove;
-	state Transaction tr(occ);
-	loop {
+	co_await cleanUpDataMoveParallelismLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser = FlowLock::Releaser(*cleanUpDataMoveParallelismLock);
+	DataMoveMetaData dataMove;
+	Transaction tr(occ);
+	while (true) {
+		counters->started->increment(1);
+		Error err;
 		try {
 			tr.trState->taskID = TaskPriority::MoveKeys;
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			wait(checkMoveKeysLock(&tr, lock, ddEnabledState));
+			co_await checkMoveKeysLock(&tr, lock, ddEnabledState);
 
-			Optional<Value> val = wait(tr.get(dataMoveKeyFor(dataMoveId)));
+			Optional<Value> val = co_await tr.get(dataMoveKeyFor(dataMoveId));
 			if (!val.present()) {
 				break;
 			}
@@ -3001,53 +3095,57 @@ ACTOR Future<Void> cleanUpDataMoveBackground(Database occ,
 			ASSERT(dataMove.ranges.empty());
 			ASSERT(dataMove.getPhase() == DataMoveMetaData::Deleting);
 			tr.clear(dataMoveKeyFor(dataMoveId));
-			wait(tr.commit());
+			co_await tr.commit();
+			counters->committed->increment(1);
 			break;
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "CleanUpDataMoveBackgroundFail", dataMoveId).errorUnsuppressed(e);
-			wait(tr.onError(e));
+			counters->aborted->increment(1);
+			err = e;
 		}
+		TraceEvent(SevWarn, "CleanUpDataMoveBackgroundFail", dataMoveId).errorUnsuppressed(err);
+		co_await tr.onError(err);
 	}
 
 	TraceEvent(SevDebug, "CleanUpDataMoveBackgroundEnd", dataMoveId)
 	    .detail("DataMoveID", dataMoveId)
 	    .detail("DataMoveRange", keys);
-
-	return Void();
 }
 
-ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
-                                       UID dataMoveId,
-                                       MoveKeysLock lock,
-                                       FlowLock* cleanUpDataMoveParallelismLock,
-                                       KeyRange keys,
-                                       const DDEnabledState* ddEnabledState) {
-	state KeyRange range;
-	state Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
+Future<Void> cleanUpDataMoveCore(Database occ,
+                                 UID dataMoveId,
+                                 MoveKeysLock lock,
+                                 FlowLock* cleanUpDataMoveParallelismLock,
+                                 KeyRange keys,
+                                 const DDEnabledState* ddEnabledState) {
+	static auto* counters = makeCounters("/movekeys/cleanUpDataMoveCore");
+	KeyRange range;
+	Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
 	TraceEvent(SevInfo, "CleanUpDataMoveBegin", dataMoveId).detail("DataMoveID", dataMoveId).detail("Range", keys);
-	state Error lastError;
-	state bool runPreCheck = true;
+	Error lastError;
+	bool runPreCheck = true;
 
-	wait(cleanUpDataMoveParallelismLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser = FlowLock::Releaser(*cleanUpDataMoveParallelismLock);
+	co_await cleanUpDataMoveParallelismLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser = FlowLock::Releaser(*cleanUpDataMoveParallelismLock);
 
 	try {
-		loop {
-			state Transaction tr(occ);
-			state std::unordered_map<UID, std::vector<Shard>> physicalShardMap;
-			state std::set<UID> oldDests;
-			state DataMoveMetaData dataMove;
+		while (true) {
+			counters->started->increment(1);
+			Transaction tr(occ);
+			std::unordered_map<UID, std::vector<Shard>> physicalShardMap;
+			std::set<UID> oldDests;
+			DataMoveMetaData dataMove;
 
 			range = KeyRange();
 
+			Error err;
 			try {
 				tr.trState->taskID = TaskPriority::MoveKeys;
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(checkMoveKeysLock(&tr, lock, ddEnabledState));
+				co_await checkMoveKeysLock(&tr, lock, ddEnabledState);
 
-				Optional<Value> val = wait(tr.get(dataMoveKeyFor(dataMoveId)));
+				Optional<Value> val = co_await tr.get(dataMoveKeyFor(dataMoveId));
 				if (val.present()) {
 					dataMove = decodeDataMoveValue(val.get());
 					if (dataMove.ranges.empty()) {
@@ -3065,47 +3163,47 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
 						// It means the commit was succeed last time
 						// For this case, safely do nothing
 						TraceEvent(sevDm, "CleanUpDataMoveHaveDoneExit", dataMoveId).detail("DataMoveID", dataMoveId);
-						return Void();
+						co_return;
 					}
 					// If a normal cleanup sees nothing, triggers background cleanup
 					dataMove = DataMoveMetaData(dataMoveId);
 					dataMove.setPhase(DataMoveMetaData::Deleting);
 					tr.set(dataMoveKeyFor(dataMoveId), dataMoveValue(dataMove));
-					wait(tr.commit());
+					co_await tr.commit();
 					TraceEvent(sevDm, "CleanUpDataMovePlaceHolder", dataMoveId).detail("DataMoveID", dataMoveId);
 					throw retry_clean_up_datamove_tombstone_added();
 				}
 
 				dataMove.setPhase(DataMoveMetaData::Deleting);
 
-				state RangeResult currentShards = wait(krmGetRanges(&tr,
-				                                                    keyServersPrefix,
-				                                                    range,
-				                                                    SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
-				                                                    SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT));
+				RangeResult currentShards = co_await krmGetRanges(&tr,
+				                                                  keyServersPrefix,
+				                                                  range,
+				                                                  SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
+				                                                  SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT);
 				ASSERT(!currentShards.empty());
 				ASSERT(range.begin == currentShards.front().key);
 				range = KeyRangeRef(range.begin, currentShards.back().key);
 
-				state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
 				// For each intersecting range, clear existing dest servers and checkpoints on all src servers.
-				state int i = 0;
+				int i = 0;
 				for (; i < currentShards.size() - 1; ++i) {
-					state KeyRangeRef rangeIntersectKeys(currentShards[i].key, currentShards[i + 1].key);
-					state std::vector<UID> src;
-					state std::vector<UID> dest;
-					state UID srcId;
-					state UID destId;
+					KeyRangeRef rangeIntersectKeys(currentShards[i].key, currentShards[i + 1].key);
+					std::vector<UID> src;
+					std::vector<UID> dest;
+					UID srcId;
+					UID destId;
 					decodeKeyServersValue(UIDtoTagMap, currentShards[i].value, src, dest, srcId, destId);
 
 					// Pre validate consistency of update of keyServers and serverKeys
 					if (SERVER_KNOBS->AUDIT_DATAMOVE_PRE_CHECK && runPreCheck) {
 						std::vector<UID> servers(src.size() + dest.size());
 						std::merge(src.begin(), src.end(), dest.begin(), dest.end(), servers.begin());
-						wait(auditLocationMetadataPreCheck(
-						    occ, &tr, rangeIntersectKeys, servers, "cleanUpDataMoveCore_precheck", dataMoveId));
+						co_await auditLocationMetadataPreCheck(
+						    occ, &tr, rangeIntersectKeys, servers, "cleanUpDataMoveCore_precheck", dataMoveId);
 					}
 
 					for (const auto& uid : src) {
@@ -3148,7 +3246,7 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
 				}
 
 				if (range.end == dataMove.ranges.front().end) {
-					wait(deleteCheckpoints(&tr, dataMove.checkpoints, dataMoveId));
+					co_await deleteCheckpoints(&tr, dataMove.checkpoints, dataMoveId);
 					tr.clear(dataMoveKeyFor(dataMoveId));
 					TraceEvent(sevDm, "CleanUpDataMoveDeleteMetaData", dataMoveId)
 					    .detail("DataMoveID", dataMove.toString());
@@ -3167,9 +3265,10 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
 				for (const auto& uid : oldDests) {
 					actors.push_back(unassignServerKeys(&tr, uid, range, physicalShardMap[uid], dataMoveId));
 				}
-				wait(waitForAll(actors));
+				co_await waitForAll(actors);
 
-				wait(tr.commit());
+				co_await tr.commit();
+				counters->committed->increment(1);
 
 				TraceEvent(sevDm, "CleanUpDataMoveCommitted", dataMoveId)
 				    .detail("DataMoveID", dataMoveId)
@@ -3178,23 +3277,26 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
 				if (range.end == dataMove.ranges.front().end) {
 					// Post validate consistency of update of keyServers and serverKeys
 					if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) {
-						wait(auditLocationMetadataPostCheck(
-						    occ, dataMove.ranges.front(), "cleanUpDataMoveCore_postcheck", dataMoveId));
+						co_await auditLocationMetadataPostCheck(
+						    occ, dataMove.ranges.front(), "cleanUpDataMoveCore_postcheck", dataMoveId);
 					}
 					break;
 				}
+				continue;
 			} catch (Error& e) {
-				if (e.code() == error_code_location_metadata_corruption) {
-					throw location_metadata_corruption();
-				} else {
-					runPreCheck = false;
-					lastError = e;
-					wait(tr.onError(e)); // throw error if retry_clean_up_datamove_tombstone_added
-					TraceEvent(SevWarn, "CleanUpDataMoveRetriableError", dataMoveId)
-					    .error(lastError)
-					    .detail("DataMoveRange", range.toString());
-				}
+				counters->aborted->increment(1);
+				err = e;
+			}
+			if (err.code() == error_code_location_metadata_corruption) {
+				throw location_metadata_corruption();
 			}
+
+			runPreCheck = false;
+			lastError = err;
+			co_await tr.onError(err); // throw error if retry_clean_up_datamove_tombstone_added
+			TraceEvent(SevWarn, "CleanUpDataMoveRetriableError", dataMoveId)
+			    .error(lastError)
+			    .detail("DataMoveRange", range.toString());
 		}
 	} catch (Error& e) {
 		throw;
@@ -3203,20 +3305,18 @@ ACTOR Future<Void> cleanUpDataMoveCore(Database occ,
 	TraceEvent(SevInfo, "CleanUpDataMoveEnd", dataMoveId)
 	    .detail("DataMoveID", dataMoveId)
 	    .detail("DataMoveRange", range.toString());
-
-	return Void();
 }
 
-ACTOR Future<Void> cleanUpDataMove(Database occ,
-                                   UID dataMoveId,
-                                   MoveKeysLock lock,
-                                   FlowLock* cleanUpDataMoveParallelismLock,
-                                   KeyRange keys,
-                                   const DDEnabledState* ddEnabledState,
-                                   Optional<PromiseStream<Future<Void>>> addCleanUpDataMoveActor) {
-	state Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
+Future<Void> cleanUpDataMove(Database occ,
+                             UID dataMoveId,
+                             MoveKeysLock lock,
+                             FlowLock* cleanUpDataMoveParallelismLock,
+                             KeyRange keys,
+                             const DDEnabledState* ddEnabledState,
+                             Optional<PromiseStream<Future<Void>>> addCleanUpDataMoveActor) {
+	Severity sevDm = static_cast<Severity>(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY);
 	try {
-		wait(cleanUpDataMoveCore(occ, dataMoveId, lock, cleanUpDataMoveParallelismLock, keys, ddEnabledState));
+		co_await cleanUpDataMoveCore(occ, dataMoveId, lock, cleanUpDataMoveParallelismLock, keys, ddEnabledState);
 	} catch (Error& e) {
 		if (e.code() == error_code_retry_clean_up_datamove_tombstone_added) {
 			if (addCleanUpDataMoveActor.present()) {
@@ -3236,8 +3336,6 @@ ACTOR Future<Void> cleanUpDataMove(Database occ,
 			throw e;
 		}
 	}
-
-	return Void();
 }
 
 Future<Void> rawStartMovement(Database occ,
@@ -3320,110 +3418,32 @@ Future<Void> rawFinishMovement(Database occ,
 	                      params.ddEnabledState);
 }
 
-ACTOR Future<Void> moveKeys(Database occ, MoveKeysParams params) {
-	ASSERT(params.destinationTeam.size());
+Future<Void> moveKeys(Database occ, MoveKeysParams params) {
+	ASSERT(!params.destinationTeam.empty());
 	std::sort(params.destinationTeam.begin(), params.destinationTeam.end());
 
-	state std::map<UID, StorageServerInterface> tssMapping;
+	std::map<UID, StorageServerInterface> tssMapping;
 
-	wait(rawStartMovement(occ, params, tssMapping));
+	co_await rawStartMovement(occ, params, tssMapping);
 
-	state Future<Void> completionSignaller = rawCheckFetchingState(occ, params, tssMapping);
+	Future<Void> completionSignaller = rawCheckFetchingState(occ, params, tssMapping);
 
-	wait(rawFinishMovement(occ, params, tssMapping));
+	co_await rawFinishMovement(occ, params, tssMapping);
 
 	// This is defensive, but make sure that we always say that the movement is complete before moveKeys completes
 	completionSignaller.cancel();
 	if (!params.dataMovementComplete.isSet())
 		params.dataMovementComplete.send(Void());
-
-	return Void();
-}
-
-// Called by the master server to write the very first transaction to the database
-// establishing a set of shard servers and all invariants of the systemKeys.
-void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vector<StorageServerInterface> servers) {
-	std::map<Optional<Value>, Tag> dcId_locality;
-	std::map<UID, Tag> server_tag;
-	int8_t nextLocality = 0;
-	for (auto& s : servers) {
-		if (!dcId_locality.contains(s.locality.dcId())) {
-			tr.set(arena, tagLocalityListKeyFor(s.locality.dcId()), tagLocalityListValue(nextLocality));
-			dcId_locality[s.locality.dcId()] = Tag(nextLocality, 0);
-			nextLocality++;
-		}
-		Tag& t = dcId_locality[s.locality.dcId()];
-		server_tag[s.id()] = Tag(t.locality, t.id);
-		t.id++;
-	}
-	std::sort(servers.begin(), servers.end());
-
-	// This isn't strictly necessary, but make sure this is the first transaction
-	tr.read_snapshot = 0;
-	tr.read_conflict_ranges.push_back_deep(arena, allKeys);
-	KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
-	                                                                                     IncludeVersion());
-	StorageMetadataType metadata(StorageMetadataType::currentTime());
-
-	for (auto& s : servers) {
-		tr.set(arena, serverTagKeyFor(s.id()), serverTagValue(server_tag[s.id()]));
-		tr.set(arena, serverListKeyFor(s.id()), serverListValue(s));
-		tr.set(arena, metadataMap.packKey(s.id()), metadataMap.packValue(metadata));
-
-		if (SERVER_KNOBS->TSS_HACK_IDENTITY_MAPPING) {
-			// THIS SHOULD NEVER BE ENABLED IN ANY NON-TESTING ENVIRONMENT
-			TraceEvent(SevError, "TSSIdentityMappingEnabled").log();
-			// hack key-backed map here since we can't really change CommitTransactionRef to a RYW transaction
-			Key uidRef = TupleCodec<UID>::pack(s.id());
-			tr.set(arena, uidRef.withPrefix(tssMappingKeys.begin), uidRef);
-		}
-	}
-	tr.set(arena, serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
-
-	std::vector<Tag> serverTags;
-	std::vector<UID> serverSrcUID;
-	serverTags.reserve(servers.size());
-	for (auto& s : servers) {
-		serverTags.push_back(server_tag[s.id()]);
-		serverSrcUID.push_back(s.id());
-	}
-
-	auto ksValue = CLIENT_KNOBS->TAG_ENCODE_KEY_SERVERS ? keyServersValue(serverTags)
-	                                                    : keyServersValue(RangeResult(), serverSrcUID);
-	// We have to set this range in two blocks, because the master tracking of "keyServersLocations" depends on a change
-	// to a specific
-	//   key (keyServersKeyServersKey)
-	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-		const UID shardId = newDataMoveId(deterministicRandom()->randomUInt64(),
-		                                  AssignEmptyRange(false),
-		                                  DataMoveType::LOGICAL,
-		                                  DataMovementReason::SEED_SHARD_SERVER,
-		                                  UnassignShard(false));
-		ksValue = keyServersValue(serverSrcUID, /*dest=*/std::vector<UID>(), shardId, UID());
-		krmSetPreviouslyEmptyRange(tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), ksValue, Value());
-
-		for (auto& s : servers) {
-			krmSetPreviouslyEmptyRange(
-			    tr, arena, serverKeysPrefixFor(s.id()), allKeys, serverKeysValue(shardId), serverKeysFalse);
-		}
-	} else {
-		krmSetPreviouslyEmptyRange(tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), ksValue, Value());
-
-		for (auto& s : servers) {
-			krmSetPreviouslyEmptyRange(
-			    tr, arena, serverKeysPrefixFor(s.id()), allKeys, serverKeysTrue, serverKeysFalse);
-		}
-	}
 }
 
 // Unassign given key range from its current storage servers
-ACTOR template <typename TrType = Transaction*>
+template <typename TrType = Transaction*>
 Future<Void> unassignServerKeys(UID traceId, TrType tr, KeyRangeRef keys, std::set<UID> ignoreServers) {
-	state RangeResult serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::True));
+	RangeResult serverList = co_await tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::True);
 	ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 	for (auto& server : serverList) {
-		state UID id = decodeServerListValue(server.value).id();
-		Optional<Value> tag = wait(tr->get(serverTagKeyFor(id)));
+		UID id = decodeServerListValue(server.value).id();
+		Optional<Value> tag = co_await tr->get(serverTagKeyFor(id));
 		if (!tag.present()) {
 			dprint("Server {} no tag\n", id.shortString());
 			continue;
@@ -3433,7 +3453,7 @@ Future<Void> unassignServerKeys(UID traceId, TrType tr, KeyRangeRef keys, std::s
 			dprint("Ignore un-assignment from {} .\n", id.toString());
 			continue;
 		}
-		RangeResult ranges = wait(krmGetRanges(tr, serverKeysPrefixFor(id), keys));
+		RangeResult ranges = co_await krmGetRanges(tr, serverKeysPrefixFor(id), keys);
 
 		bool owning = false;
 		for (auto& r : ranges) {
@@ -3443,21 +3463,38 @@ Future<Void> unassignServerKeys(UID traceId, TrType tr, KeyRangeRef keys, std::s
 			}
 		}
 		if (owning) {
-			wait(krmSetRangeCoalescing(tr, serverKeysPrefixFor(id), keys, allKeys, serverKeysFalse));
+			co_await krmSetRangeCoalescing(tr, serverKeysPrefixFor(id), keys, allKeys, serverKeysFalse);
 			dprint("Unassign {} from storage server {}\n", keys.toString(), id.toString());
 			TraceEvent("UnassignKeys", traceId).detail("Keys", keys).detail("SS", id);
 		}
 	}
-	return Void();
 }
 
-// Assign given key range to specified storage server.
-ACTOR template <typename TrType = Transaction*>
-Future<Void> assignKeysToServer(UID traceId, TrType tr, KeyRangeRef keys, UID serverUID) {
-	state Value value = keyServersValue(std::vector<UID>({ serverUID }), std::vector<UID>(), UID(), UID());
-	wait(krmSetRangeCoalescing(tr, keyServersPrefix, keys, allKeys, value));
-	wait(krmSetRangeCoalescing(tr, serverKeysPrefixFor(serverUID), keys, allKeys, serverKeysTrue));
-	dprint("Assign {} to server {}\n", normalKeys.toString(), serverUID.toString());
-	TraceEvent("AssignKeys", traceId).detail("Keys", keys).detail("SS", serverUID);
+// Unit test for the finishMoveKeys backoff formula. A simulation workload can't reliably
+// trigger transaction_too_old in finishMoveKeys because:
+//   1. finishMoveKeys only runs when DD schedules data moves, which depends on cluster
+//      configuration and shard placement — not guaranteed in all simulation configs
+//   2. Even with injection, the injection point must fire before check() runs, but
+//      finishMoveKeys may not be called if no moves are scheduled
+// So we test the backoff arithmetic directly.
+TEST_CASE("/fdbserver/MoveKeys/finishMoveKeysBackoff") {
+	// Verify exponential backoff with jitter: base = 0.1 * 2^retries capped at 5.0s,
+	// then jittered to [0.75x, 1.25x]. In production retries >= 1 at call site.
+	for (int i = 0; i < 100; i++) {
+		double v0 = finishMoveKeysBackoff(0);
+		ASSERT(v0 >= 0.1 * 0.75 && v0 <= 0.1 * 1.25);
+		double v1 = finishMoveKeysBackoff(1);
+		ASSERT(v1 >= 0.2 * 0.75 && v1 <= 0.2 * 1.25);
+		double v3 = finishMoveKeysBackoff(3);
+		ASSERT(v3 >= 0.8 * 0.75 && v3 <= 0.8 * 1.25);
+		double v6 = finishMoveKeysBackoff(6);
+		ASSERT(v6 >= 5.0 * 0.75 && v6 <= 5.0); // capped at 5.0
+		double v100 = finishMoveKeysBackoff(100);
+		ASSERT(v100 >= 5.0 * 0.75 && v100 <= 5.0); // still capped
+	}
+
+	// Verify the retry limit knob exists and is positive
+	ASSERT(SERVER_KNOBS->FINISH_MOVE_KEYS_MAX_RETRIES > 0);
+
 	return Void();
 }
diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/core/MutationTracking.cpp
similarity index 90%
rename from fdbserver/MutationTracking.cpp
rename to fdbserver/core/MutationTracking.cpp
index d8b1560bc64..5cfc95343aa 100644
--- a/fdbserver/MutationTracking.cpp
+++ b/fdbserver/core/MutationTracking.cpp
@@ -21,10 +21,10 @@
 #include <algorithm>
 #include <vector>
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/LogProtocolMessage.h"
-#include "fdbserver/SpanContextMessage.h"
-#include "fdbserver/OTELSpanContextMessage.h"
+#include "fdbserver/core/MutationTracking.h"
+#include "fdbserver/core/LogProtocolMessage.h"
+#include "fdbserver/core/SpanContextMessage.h"
+#include "fdbserver/core/OTELSpanContextMessage.h"
 #include "fdbclient/SystemData.h"
 #if defined(FDB_CLEAN_BUILD) && MUTATION_TRACKING_ENABLED
 #error "You cannot use mutation tracking in a clean/release build."
@@ -40,20 +40,20 @@ std::vector<std::pair<const char*, KeyRangeRef>> debugRanges = { { "Everything",
 TraceEvent debugMutationEnabled(const char* context, Version version, MutationRef const& mutation, UID id) {
 	const char* label = nullptr;
 
-	for (auto& labelKey : debugKeys) {
+	for (const auto& [keyLabel, debugKey] : debugKeys) {
 		if (((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
-		     KeyRangeRef(mutation.param1, mutation.param2).contains(labelKey.second)) ||
-		    mutation.param1 == labelKey.second) {
-			label = labelKey.first;
+		     KeyRangeRef(mutation.param1, mutation.param2).contains(debugKey)) ||
+		    mutation.param1 == debugKey) {
+			label = keyLabel;
 			break;
 		}
 	}
 
-	for (auto& labelRange : debugRanges) {
+	for (const auto& [rangeLabel, debugRange] : debugRanges) {
 		if (((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) &&
-		     KeyRangeRef(mutation.param1, mutation.param2).intersects(labelRange.second)) ||
-		    labelRange.second.contains(mutation.param1)) {
-			label = labelRange.first;
+		     KeyRangeRef(mutation.param1, mutation.param2).intersects(debugRange)) ||
+		    debugRange.contains(mutation.param1)) {
+			label = rangeLabel;
 			break;
 		}
 	}
diff --git a/fdbserver/core/OpenDatabase.cpp b/fdbserver/core/OpenDatabase.cpp
new file mode 100644
index 00000000000..fa83ac42581
--- /dev/null
+++ b/fdbserver/core/OpenDatabase.cpp
@@ -0,0 +1,56 @@
+/*
+ * OpenDatabase.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ActorLineageProfiler.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/MonitorLeader.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+
+static Future<Void> extractClientInfo(Reference<AsyncVar<ServerDBInfo> const> db,
+                                      Reference<AsyncVar<ClientDBInfo>> info) {
+	std::vector<UID> lastCommitProxyUIDs;
+	std::vector<CommitProxyInterface> lastCommitProxies;
+	std::vector<UID> lastGrvProxyUIDs;
+	std::vector<GrvProxyInterface> lastGrvProxies;
+	while (true) {
+		ClientDBInfo ni = db->get().client;
+		shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
+		info->setUnconditional(ni);
+		co_await db->onChange();
+	}
+}
+
+Database openDBOnServer(Reference<AsyncVar<ServerDBInfo> const> const& db,
+                        TaskPriority taskID,
+                        LockAware lockAware,
+                        EnableLocalityLoadBalance enableLocalityLoadBalance) {
+	auto info = makeReference<AsyncVar<ClientDBInfo>>();
+	auto cx = DatabaseContext::create(info,
+	                                  extractClientInfo(db, info),
+	                                  enableLocalityLoadBalance ? db->get().myLocality : LocalityData(),
+	                                  enableLocalityLoadBalance,
+	                                  taskID,
+	                                  lockAware);
+	cx->globalConfig->init(db, std::addressof(db->get().client));
+	cx->globalConfig->trigger(samplingFrequency, samplingProfilerUpdateFrequency);
+	cx->globalConfig->trigger(samplingWindow, samplingProfilerUpdateWindow);
+	return cx;
+}
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/core/QuietDatabase.cpp
similarity index 64%
rename from fdbserver/QuietDatabase.actor.cpp
rename to fdbserver/core/QuietDatabase.cpp
index 6063f079165..a812c23fd73 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/core/QuietDatabase.cpp
@@ -1,5 +1,5 @@
 /*
- * QuietDatabase.actor.cpp
+ * QuietDatabase.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -34,30 +34,42 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR Future<std::vector<WorkerDetails>> getWorkers(Reference<AsyncVar<ServerDBInfo> const> dbInfo, int flags = 0) {
-	loop {
-		choose {
-			when(std::vector<WorkerDetails> w = wait(brokenPromiseToNever(
-			         dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest(flags))))) {
-				return w;
-			}
-			when(wait(dbInfo->onChange())) {}
+#include "fdbclient/RunRYWTransaction.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbclient/ManagementAPI.h"
+#include "flow/CoroUtils.h"
+
+static bool g_ddPipelineControlEnabled = true;
+
+bool isDDPipelineControlEnabled() {
+	return g_ddPipelineControlEnabled;
+}
+
+void disableDDPipelineControl() {
+	TraceEvent("DDPipelineControlDisabled");
+	g_ddPipelineControlEnabled = false;
+}
+
+Future<std::vector<WorkerDetails>> getWorkers(Reference<AsyncVar<ServerDBInfo> const> dbInfo, int flags) {
+	while (true) {
+		auto res = co_await race(
+		    brokenPromiseToNever(dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest(flags))),
+		    dbInfo->onChange());
+		if (res.index() == 0) {
+			co_return std::get<0>(std::move(res));
 		}
 	}
 }
 
 // Gets the WorkerInterface representing the Master server.
-ACTOR Future<WorkerInterface> getMasterWorker(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+Future<WorkerInterface> getMasterWorker(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	TraceEvent("GetMasterWorker").detail("Stage", "GettingWorkers");
 
-	loop {
-		state std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
+	while (true) {
+		auto workers = co_await getWorkers(dbInfo);
 
 		for (int i = 0; i < workers.size(); i++) {
 			if (workers[i].interf.address() == dbInfo->get().master.address()) {
@@ -65,7 +77,7 @@ ACTOR Future<WorkerInterface> getMasterWorker(Database cx, Reference<AsyncVar<Se
 				    .detail("Stage", "GotWorkers")
 				    .detail("MasterId", dbInfo->get().master.id())
 				    .detail("WorkerId", workers[i].interf.id());
-				return workers[i].interf;
+				co_return workers[i].interf;
 			}
 		}
 
@@ -75,16 +87,16 @@ ACTOR Future<WorkerInterface> getMasterWorker(Database cx, Reference<AsyncVar<Se
 		    .detail("MasterAddress", dbInfo->get().master.address())
 		    .detail("WorkerCount", workers.size());
 
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 }
 
 // Gets the WorkerInterface representing the data distributor.
-ACTOR Future<WorkerInterface> getDataDistributorWorker(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+Future<WorkerInterface> getDataDistributorWorker(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	TraceEvent("GetDataDistributorWorker").detail("Stage", "GettingWorkers");
 
-	loop {
-		state std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
+	while (true) {
+		auto workers = co_await getWorkers(dbInfo);
 		if (!dbInfo->get().distributor.present())
 			continue;
 
@@ -94,7 +106,7 @@ ACTOR Future<WorkerInterface> getDataDistributorWorker(Database cx, Reference<As
 				    .detail("Stage", "GotWorkers")
 				    .detail("DataDistributorId", dbInfo->get().distributor.get().id())
 				    .detail("WorkerId", workers[i].interf.id());
-				return workers[i].interf;
+				co_return workers[i].interf;
 			}
 		}
 
@@ -107,13 +119,13 @@ ACTOR Future<WorkerInterface> getDataDistributorWorker(Database cx, Reference<As
 }
 
 // Gets the number of bytes in flight from the data distributor.
-ACTOR Future<int64_t> getDataInFlight(Database cx, WorkerInterface distributorWorker) {
+Future<int64_t> getDataInFlight(Database cx, WorkerInterface distributorWorker) {
 	try {
 		TraceEvent("DataInFlight").detail("Stage", "ContactingDataDistributor");
-		TraceEventFields md = wait(
-		    timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("TotalDataInFlight"_sr)), 1.0));
+		TraceEventFields md = co_await timeoutError(
+		    distributorWorker.eventLogRequest.getReply(EventLogRequest("TotalDataInFlight"_sr)), 1.0);
 		int64_t dataInFlight = boost::lexical_cast<int64_t>(md.getValue("TotalBytes"));
-		return dataInFlight;
+		co_return dataInFlight;
 	} catch (Error& e) {
 		TraceEvent("QuietDatabaseFailure", distributorWorker.id())
 		    .error(e)
@@ -123,10 +135,10 @@ ACTOR Future<int64_t> getDataInFlight(Database cx, WorkerInterface distributorWo
 }
 
 // Gets the number of bytes in flight from the data distributor.
-ACTOR Future<int64_t> getDataInFlight(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	WorkerInterface distributorInterf = wait(getDataDistributorWorker(cx, dbInfo));
-	int64_t dataInFlight = wait(getDataInFlight(cx, distributorInterf));
-	return dataInFlight;
+Future<int64_t> getDataInFlight(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	WorkerInterface distributorInterf = co_await getDataDistributorWorker(cx, dbInfo);
+	int64_t dataInFlight = co_await getDataInFlight(cx, distributorInterf);
+	co_return dataInFlight;
 }
 
 // Computes the queue size for storage servers and tlogs using the bytesInput and bytesDurable attributes
@@ -153,21 +165,20 @@ int64_t getPoppedVersionLag(const TraceEventFields& md) {
 	return persistentDataDurableVersion - queuePoppedVersion;
 }
 
-ACTOR Future<std::vector<WorkerInterface>> getCoordWorkers(Database cx,
-                                                           Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	state std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
+Future<std::vector<WorkerInterface>> getCoordWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	auto workers = co_await getWorkers(dbInfo);
 
 	Optional<Value> coordinators =
-	    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+	    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 		    tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		    tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		    return tr->get(coordinatorsKey);
-	    }));
+	    });
 	if (!coordinators.present()) {
 		throw operation_failed();
 	}
 	ClusterConnectionString ccs(coordinators.get().toString());
-	std::vector<NetworkAddress> coordinatorsAddr = wait(ccs.tryResolveHostnames());
+	std::vector<NetworkAddress> coordinatorsAddr = co_await ccs.tryResolveHostnames();
 	std::set<NetworkAddress> coordinatorsAddrSet;
 	for (const auto& addr : coordinatorsAddr) {
 		TraceEvent(SevDebug, "CoordinatorAddress").detail("Addr", addr);
@@ -183,22 +194,21 @@ ACTOR Future<std::vector<WorkerInterface>> getCoordWorkers(Database cx,
 			result.push_back(worker.interf);
 		}
 	}
-	return result;
+	co_return result;
 }
 
 // This is not robust in the face of a TLog failure
-ACTOR Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database cx,
-                                                           Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	TraceEvent("MaxTLogQueueSize").detail("Stage", "ContactingLogs");
 
-	state std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
+	auto workers = co_await getWorkers(dbInfo);
 	std::map<NetworkAddress, WorkerInterface> workersMap;
-	for (auto worker : workers) {
+	for (const auto& worker : workers) {
 		workersMap[worker.interf.address()] = worker.interf;
 	}
 
-	state std::vector<Future<TraceEventFields>> messages;
-	state std::vector<TLogInterface> tlogs = dbInfo->get().logSystemConfig.allPresentLogs();
+	std::vector<Future<TraceEventFields>> messages;
+	auto tlogs = dbInfo->get().logSystemConfig.allPresentLogs();
 	for (int i = 0; i < tlogs.size(); i++) {
 		auto itr = workersMap.find(tlogs[i].address());
 		if (itr == workersMap.end()) {
@@ -211,14 +221,13 @@ ACTOR Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database cx,
 		    itr->second.eventLogRequest.getReply(EventLogRequest(StringRef(tlogs[i].id().toString() + "/TLogMetrics"))),
 		    1.0));
 	}
-	wait(waitForAll(messages));
+	co_await waitForAll(messages);
 
 	TraceEvent("MaxTLogQueueSize").detail("Stage", "ComputingMax").detail("MessageCount", messages.size());
 
-	state int64_t maxQueueSize = 0;
-	state int64_t maxPoppedVersionLag = 0;
-	state int i = 0;
-	for (; i < messages.size(); i++) {
+	int64_t maxQueueSize = 0;
+	int64_t maxPoppedVersionLag = 0;
+	for (int i = 0; i < messages.size(); i++) {
 		try {
 			maxQueueSize = std::max(maxQueueSize, getQueueSize(messages[i].get()));
 			maxPoppedVersionLag = std::max(maxPoppedVersionLag, getPoppedVersionLag(messages[i].get()));
@@ -230,47 +239,50 @@ ACTOR Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database cx,
 		}
 	}
 
-	return std::make_pair(maxQueueSize, maxPoppedVersionLag);
+	co_return std::make_pair(maxQueueSize, maxPoppedVersionLag);
 }
 
-ACTOR Future<std::vector<StorageServerInterface>> getStorageServers(Database cx, bool use_system_priority = false) {
-	state Transaction tr(cx);
-	loop {
+Future<std::vector<StorageServerInterface>> getStorageServers(Database cx, bool use_system_priority) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		if (use_system_priority) {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		}
 		tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
-			RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
 			std::vector<StorageServerInterface> servers;
 			servers.reserve(serverList.size());
 			for (int i = 0; i < serverList.size(); i++)
 				servers.push_back(decodeServerListValue(serverList[i].value));
-			return servers;
+			co_return servers;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
-ACTOR Future<std::pair<std::vector<WorkerInterface>, int>>
-getStorageWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, bool localOnly) {
-	state std::vector<StorageServerInterface> servers = wait(getStorageServers(cx));
-	state std::map<NetworkAddress, WorkerInterface> workersMap;
-	std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
+Future<std::pair<std::vector<WorkerInterface>, int>> getStorageWorkers(Database cx,
+                                                                       Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                                                       bool localOnly) {
+	auto servers = co_await getStorageServers(cx);
+	std::map<NetworkAddress, WorkerInterface> workersMap;
+	std::vector<WorkerDetails> workers = co_await getWorkers(dbInfo);
 
 	for (const auto& worker : workers) {
 		workersMap[worker.interf.address()] = worker.interf;
 	}
 	Optional<Value> regionsValue =
-	    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+	    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 		    tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		    tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		    return tr->get("usable_regions"_sr.withPrefix(configKeysPrefix));
-	    }));
+	    });
 	int usableRegions = 1;
 	if (regionsValue.present()) {
 		usableRegions = atoi(regionsValue.get().toString().c_str());
@@ -296,7 +308,7 @@ getStorageWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, b
 			}
 		}
 	}
-	return result;
+	co_return result;
 }
 
 // Helper function to extract he maximum SQ size of all provided messages. All futures in the
@@ -333,67 +345,62 @@ int64_t extractMaxQueueSize(const std::vector<Future<TraceEventFields>>& message
 }
 
 // Timeout wrapper when getting the storage metrics. This will do some additional tracing
-ACTOR Future<TraceEventFields> getStorageMetricsTimeout(UID storage, WorkerInterface wi, Version version) {
-	state int retries = 0;
-	loop {
-		++retries;
-		state Future<TraceEventFields> eventLogReply =
+Future<TraceEventFields> getStorageMetricsTimeout(UID storage, WorkerInterface wi, Version version) {
+	int retries = 0;
+	while (true) {
+		Future<TraceEventFields> eventLogReplyFuture =
 		    wi.eventLogRequest.getReply(EventLogRequest(StringRef(storage.toString() + "/StorageMetrics")));
-		state Future<Void> timeout = delay(30.0);
-		state TraceEventFields storageMetrics;
-		choose {
-			when(wait(store(storageMetrics, eventLogReply))) {
-				try {
-					if (version == invalidVersion ||
-					    getDurableVersion(storageMetrics) >= static_cast<int64_t>(version)) {
-						return storageMetrics;
-					}
-				} catch (Error& e) {
-					TraceEvent("QuietDatabaseFailure")
-					    .error(e)
-					    .detail("Reason", "Failed to extract DurableVersion from StorageMetrics")
-					    .detail("SSID", storage)
-					    .detail("StorageMetrics", storageMetrics.toString());
-					throw;
+		auto eventLogReply = co_await timeout(eventLogReplyFuture, 30.0);
+		if (eventLogReply.present()) {
+			TraceEventFields const& storageMetrics = eventLogReply.get();
+			try {
+				if (version == invalidVersion || getDurableVersion(storageMetrics) >= static_cast<int64_t>(version)) {
+					co_return storageMetrics;
 				}
-			}
-			when(wait(timeout)) {
+			} catch (Error& e) {
 				TraceEvent("QuietDatabaseFailure")
-				    .detail("Reason", "Could not fetch StorageMetrics")
-				    .detail("Storage", storage);
-				throw timed_out();
+				    .error(e)
+				    .detail("Reason", "Failed to extract DurableVersion from StorageMetrics")
+				    .detail("SSID", storage)
+				    .detail("StorageMetrics", storageMetrics.toString());
+				throw;
 			}
+		} else {
+			TraceEvent("QuietDatabaseFailure")
+			    .detail("Reason", "Could not fetch StorageMetrics")
+			    .detail("Storage", storage);
+			throw timed_out();
 		}
-		if (retries > 30) {
+		if (++retries > 30) {
 			TraceEvent("QuietDatabaseFailure")
 			    .detail("Reason", "Could not fetch StorageMetrics x30")
 			    .detail("Storage", storage)
 			    .detail("Version", version);
 			throw timed_out();
 		}
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 }
 
 // Gets the maximum size of all the storage server queues
-ACTOR Future<int64_t> getMaxStorageServerQueueSize(Database cx,
-                                                   Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                                                   Version version) {
+Future<int64_t> getMaxStorageServerQueueSize(Database cx,
+                                             Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                             Version version) {
 
 	TraceEvent("MaxStorageServerQueueSize").detail("Stage", "ContactingStorageServers");
 
 	Future<std::vector<StorageServerInterface>> serversFuture = getStorageServers(cx);
-	state Future<std::vector<WorkerDetails>> workersFuture = getWorkers(dbInfo);
+	Future<std::vector<WorkerDetails>> workersFuture = getWorkers(dbInfo);
 
-	state std::vector<StorageServerInterface> servers = wait(serversFuture);
-	state std::vector<WorkerDetails> workers = wait(workersFuture);
+	auto servers = co_await serversFuture;
+	auto workers = co_await workersFuture;
 
 	std::map<NetworkAddress, WorkerInterface> workersMap;
-	for (auto worker : workers) {
+	for (const auto& worker : workers) {
 		workersMap[worker.interf.address()] = worker.interf;
 	}
 
-	state std::vector<Future<TraceEventFields>> messages;
+	std::vector<Future<TraceEventFields>> messages;
 	for (int i = 0; i < servers.size(); i++) {
 		auto itr = workersMap.find(servers[i].address());
 		if (itr == workersMap.end()) {
@@ -405,23 +412,21 @@ ACTOR Future<int64_t> getMaxStorageServerQueueSize(Database cx,
 		messages.push_back(getStorageMetricsTimeout(servers[i].id(), itr->second, version));
 	}
 
-	wait(waitForAll(messages));
+	co_await waitForAll(messages);
 
 	TraceEvent("MaxStorageServerQueueSize").detail("Stage", "ComputingMax").detail("MessageCount", messages.size());
 
-	return extractMaxQueueSize(messages, servers);
+	co_return extractMaxQueueSize(messages, servers);
 }
 
 // Gets the size of the data distribution queue.  If reportInFlight is true, then data in flight is considered part of
 // the queue
-ACTOR Future<int64_t> getDataDistributionQueueSize(Database cx,
-                                                   WorkerInterface distributorWorker,
-                                                   bool reportInFlight) {
+Future<int64_t> getDataDistributionQueueSize(Database cx, WorkerInterface distributorWorker, bool reportInFlight) {
 	try {
 		TraceEvent("DataDistributionQueueSize").detail("Stage", "ContactingDataDistributor");
 
 		TraceEventFields movingDataMessage =
-		    wait(timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0));
+		    co_await timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0);
 
 		TraceEvent("DataDistributionQueueSize").detail("Stage", "GotString");
 
@@ -432,7 +437,7 @@ ACTOR Future<int64_t> getDataDistributionQueueSize(Database cx,
 			inQueue += inFlight;
 		}
 
-		return inQueue;
+		co_return inQueue;
 	} catch (Error& e) {
 		TraceEvent("QuietDatabaseFailure", distributorWorker.id())
 		    .detail("Reason", "Failed to extract DataDistributionQueueSize");
@@ -442,52 +447,52 @@ ACTOR Future<int64_t> getDataDistributionQueueSize(Database cx,
 
 // Gets the size of the data distribution queue.  If reportInFlight is true, then data in flight is considered part of
 // the queue Convenience method that first finds the master worker from a zookeeper interface
-ACTOR Future<int64_t> getDataDistributionQueueSize(Database cx,
-                                                   Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                                                   bool reportInFlight) {
-	WorkerInterface distributorInterf = wait(getDataDistributorWorker(cx, dbInfo));
-	int64_t inQueue = wait(getDataDistributionQueueSize(cx, distributorInterf, reportInFlight));
-	return inQueue;
+Future<int64_t> getDataDistributionQueueSize(Database cx,
+                                             Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                             bool reportInFlight) {
+	WorkerInterface distributorInterf = co_await getDataDistributorWorker(cx, dbInfo);
+	int64_t inQueue = co_await getDataDistributionQueueSize(cx, distributorInterf, reportInFlight);
+	co_return inQueue;
 }
 
 // Gets if the number of process and machine teams does not exceed the maximum allowed number of teams
-ACTOR Future<bool> getTeamCollectionValid(Database cx, WorkerInterface dataDistributorWorker) {
-	state int attempts = 0;
-	state bool ret = false;
-	loop {
+Future<bool> getTeamCollectionValid(Database cx, WorkerInterface dataDistributorWorker) {
+	int attempts = 0;
+	bool ret = false;
+	while (true) {
+		Optional<double> retryDelay;
 		try {
 			if (!g_network->isSimulated()) {
-				return true;
+				co_return true;
 			}
 
 			TraceEvent("GetTeamCollectionValid").detail("Stage", "ContactingMaster");
 
-			TraceEventFields teamCollectionInfoMessage = wait(timeoutError(
-			    dataDistributorWorker.eventLogRequest.getReply(EventLogRequest("TeamCollectionInfo"_sr)), 1.0));
+			TraceEventFields teamCollectionInfoMessage = co_await timeoutError(
+			    dataDistributorWorker.eventLogRequest.getReply(EventLogRequest("TeamCollectionInfo"_sr)), 1.0);
 
 			TraceEvent("GetTeamCollectionValid").detail("Stage", "GotString");
 
-			state int64_t currentTeams =
+			int64_t currentTeams =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("CurrentServerTeams"));
-			state int64_t desiredTeams =
-			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("DesiredTeams"));
-			state int64_t maxTeams = boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MaxTeams"));
-			state int64_t currentMachineTeams =
+			int64_t desiredTeams = boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("DesiredTeams"));
+			int64_t maxTeams = boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MaxTeams"));
+			int64_t currentMachineTeams =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("CurrentMachineTeams"));
-			state int64_t healthyMachineTeams =
+			int64_t healthyMachineTeams =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("CurrentHealthyMachineTeams"));
-			state int64_t desiredMachineTeams =
+			int64_t desiredMachineTeams =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("DesiredMachineTeams"));
-			state int64_t maxMachineTeams =
+			int64_t maxMachineTeams =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MaxMachineTeams"));
 
-			state int64_t minServerTeamsOnServer =
+			int64_t minServerTeamsOnServer =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MinTeamsOnServer"));
-			state int64_t maxServerTeamsOnServer =
+			int64_t maxServerTeamsOnServer =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MaxTeamsOnServer"));
-			state int64_t minMachineTeamsOnMachine =
+			int64_t minMachineTeamsOnMachine =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MinMachineTeamsOnMachine"));
-			state int64_t maxMachineTeamsOnMachine =
+			int64_t maxMachineTeamsOnMachine =
 			    boost::lexical_cast<int64_t>(teamCollectionInfoMessage.getValue("MaxMachineTeamsOnMachine"));
 
 			// The if condition should be consistent with the condition in serverTeamRemover() and
@@ -511,32 +516,32 @@ ACTOR Future<bool> getTeamCollectionValid(Database cx, WorkerInterface dataDistr
 				ret = false;
 
 				if (attempts++ < 10) {
-					wait(delay(60));
-					continue; // We may not receive the most recent TeamCollectionInfo
+					retryDelay = 60.0; // We may not receive the most recent TeamCollectionInfo
+				} else {
+					// When DESIRED_TEAMS_PER_SERVER == 1, we see minMachineTeamOnMachine can be 0 in one out of 30k
+					// test cases. Only check DESIRED_TEAMS_PER_SERVER == 3 for now since it is mostly used
+					// configuration.
+					// TODO: Remove the constraint SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER == 3 to ensure that
+					// the minimum team number per server (and per machine) is always > 0 for any number of replicas
+					TraceEvent("GetTeamCollectionValid")
+					    .detail("CurrentServerTeams", currentTeams)
+					    .detail("DesiredTeams", desiredTeams)
+					    .detail("MaxTeams", maxTeams)
+					    .detail("CurrentHealthyMachineTeams", healthyMachineTeams)
+					    .detail("DesiredMachineTeams", desiredMachineTeams)
+					    .detail("CurrentMachineTeams", currentMachineTeams)
+					    .detail("MaxMachineTeams", maxMachineTeams)
+					    .detail("MinTeamsOnServer", minServerTeamsOnServer)
+					    .detail("MaxTeamsOnServer", maxServerTeamsOnServer)
+					    .detail("MinMachineTeamsOnMachine", minMachineTeamsOnMachine)
+					    .detail("MaxMachineTeamsOnMachine", maxMachineTeamsOnMachine)
+					    .detail("DesiredTeamsPerServer", SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER)
+					    .detail("MaxTeamsPerServer", SERVER_KNOBS->MAX_TEAMS_PER_SERVER)
+					    .detail("RemoveMTWithMostTeams", SERVER_KNOBS->TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS);
+					co_return ret;
 				}
-
-				// When DESIRED_TEAMS_PER_SERVER == 1, we see minMachineTeamOnMachine can be 0 in one out of 30k test
-				// cases. Only check DESIRED_TEAMS_PER_SERVER == 3 for now since it is mostly used configuration.
-				// TODO: Remove the constraint SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER == 3 to ensure that
-				// the minimum team number per server (and per machine) is always > 0 for any number of replicas
-				TraceEvent("GetTeamCollectionValid")
-				    .detail("CurrentServerTeams", currentTeams)
-				    .detail("DesiredTeams", desiredTeams)
-				    .detail("MaxTeams", maxTeams)
-				    .detail("CurrentHealthyMachineTeams", healthyMachineTeams)
-				    .detail("DesiredMachineTeams", desiredMachineTeams)
-				    .detail("CurrentMachineTeams", currentMachineTeams)
-				    .detail("MaxMachineTeams", maxMachineTeams)
-				    .detail("MinTeamsOnServer", minServerTeamsOnServer)
-				    .detail("MaxTeamsOnServer", maxServerTeamsOnServer)
-				    .detail("MinMachineTeamsOnMachine", minMachineTeamsOnMachine)
-				    .detail("MaxMachineTeamsOnMachine", maxMachineTeamsOnMachine)
-				    .detail("DesiredTeamsPerServer", SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER)
-				    .detail("MaxTeamsPerServer", SERVER_KNOBS->MAX_TEAMS_PER_SERVER)
-				    .detail("RemoveMTWithMostTeams", SERVER_KNOBS->TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS);
-				return ret;
 			} else {
-				return true;
+				co_return true;
 			}
 
 		} catch (Error& e) {
@@ -549,31 +554,31 @@ ACTOR Future<bool> getTeamCollectionValid(Database cx, WorkerInterface dataDistr
 			if (attempts > 10) {
 				TraceEvent("QuietDatabaseNoTeamCollectionInfo", dataDistributorWorker.id())
 				    .detail("Reason", "Had never called build team to build any team");
-				return true;
+				co_return true;
 			}
-			// throw;
-			wait(delay(10.0));
+			retryDelay = 10.0;
 		}
-	};
+		co_await delay(retryDelay.get());
+	}
 }
 
 // Gets if the number of process and machine teams does not exceed the maximum allowed number of teams
 // Convenience method that first finds the master worker from a zookeeper interface
-ACTOR Future<bool> getTeamCollectionValid(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	WorkerInterface dataDistributorWorker = wait(getDataDistributorWorker(cx, dbInfo));
-	bool valid = wait(getTeamCollectionValid(cx, dataDistributorWorker));
-	return valid;
+Future<bool> getTeamCollectionValid(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	WorkerInterface dataDistributorWorker = co_await getDataDistributorWorker(cx, dbInfo);
+	bool valid = co_await getTeamCollectionValid(cx, dataDistributorWorker);
+	co_return valid;
 }
 
 // Checks that data distribution is active
-ACTOR Future<bool> getDataDistributionActive(Database cx, WorkerInterface distributorWorker) {
+Future<bool> getDataDistributionActive(Database cx, WorkerInterface distributorWorker) {
 	try {
 		TraceEvent("DataDistributionActive").detail("Stage", "ContactingDataDistributor");
 
-		TraceEventFields activeMessage = wait(
-		    timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest("DDTrackerStarting"_sr)), 1.0));
+		TraceEventFields activeMessage = co_await timeoutError(
+		    distributorWorker.eventLogRequest.getReply(EventLogRequest("DDTrackerStarting"_sr)), 1.0);
 
-		return activeMessage.getValue("State") == "Active";
+		co_return activeMessage.getValue("State") == "Active";
 	} catch (Error& e) {
 		TraceEvent("QuietDatabaseFailure", distributorWorker.id())
 		    .detail("Reason", "Failed to extract DataDistributionActive");
@@ -582,13 +587,13 @@ ACTOR Future<bool> getDataDistributionActive(Database cx, WorkerInterface distri
 }
 
 // Checks to see if any storage servers are being recruited
-ACTOR Future<bool> getStorageServersRecruiting(Database cx, WorkerInterface distributorWorker, UID distributorUID) {
+Future<bool> getStorageServersRecruiting(Database cx, WorkerInterface distributorWorker, UID distributorUID) {
 	try {
 		TraceEvent("StorageServersRecruiting").detail("Stage", "ContactingDataDistributor");
 		TraceEventFields recruitingMessage =
-		    wait(timeoutError(distributorWorker.eventLogRequest.getReply(
-		                          EventLogRequest(StringRef("StorageServerRecruitment_" + distributorUID.toString()))),
-		                      1.0));
+		    co_await timeoutError(distributorWorker.eventLogRequest.getReply(EventLogRequest(
+		                              StringRef("StorageServerRecruitment_" + distributorUID.toString()))),
+		                          1.0);
 
 		TraceEvent("StorageServersRecruiting").detail("Message", recruitingMessage.toString());
 
@@ -596,10 +601,10 @@ ACTOR Future<bool> getStorageServersRecruiting(Database cx, WorkerInterface dist
 			std::string tssValue;
 			// if we're tss recruiting, that's fine because that can block indefinitely if only 1 free storage process
 			if (!recruitingMessage.tryGetValue("IsTSS", tssValue) || tssValue == "False") {
-				return true;
+				co_return true;
 			}
 		}
-		return false;
+		co_return false;
 	} catch (Error& e) {
 		TraceEvent("QuietDatabaseFailure", distributorWorker.id())
 		    .detail("Reason", "Failed to extract StorageServersRecruiting")
@@ -610,38 +615,40 @@ ACTOR Future<bool> getStorageServersRecruiting(Database cx, WorkerInterface dist
 
 // Gets the difference between the expected version (based on the version
 // epoch) and the actual version.
-ACTOR Future<int64_t> getVersionOffset(Database cx,
-                                       WorkerInterface distributorWorker,
-                                       Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	loop {
-		state Transaction tr(cx);
+Future<int64_t> getVersionOffset(Database cx,
+                                 WorkerInterface distributorWorker,
+                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			TraceEvent("GetVersionOffset").detail("Stage", "ReadingVersionEpoch");
 
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			state Version rv = wait(tr.getReadVersion());
-			Optional<Standalone<StringRef>> versionEpochValue = wait(tr.get(versionEpochKey));
+			Version rv = co_await tr.getReadVersion();
+			Optional<Standalone<StringRef>> versionEpochValue = co_await tr.get(versionEpochKey);
 			if (!versionEpochValue.present()) {
-				return 0;
+				co_return 0;
 			}
 			int64_t versionEpoch = BinaryReader::fromStringRef<int64_t>(versionEpochValue.get(), Unversioned());
 			int64_t versionOffset = abs(rv - (g_network->timer() * SERVER_KNOBS->VERSIONS_PER_SECOND - versionEpoch));
-			return versionOffset;
+			co_return versionOffset;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
 // Returns DC lag for simulation runs
-ACTOR Future<Version> getDatacenterLag(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	loop {
-		if (!g_network->isSimulated() || g_simulator->usableRegions == 1) {
-			return 0;
+Future<Version> getDatacenterLag(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	while (true) {
+		if (!g_network->isSimulated() || fdbSimulationPolicyState().usableRegions == 1) {
+			co_return 0;
 		}
 
-		state Optional<TLogInterface> primaryLog;
-		state Optional<TLogInterface> remoteLog;
+		Optional<TLogInterface> primaryLog;
+		Optional<TLogInterface> remoteLog;
 		if (dbInfo->get().recoveryState >= RecoveryState::ALL_LOGS_RECRUITED) {
 			for (const auto& logset : dbInfo->get().logSystemConfig.tLogs) {
 				if (logset.isLocal && logset.locality != tagLocalitySatellite) {
@@ -664,54 +671,53 @@ ACTOR Future<Version> getDatacenterLag(Database cx, Reference<AsyncVar<ServerDBI
 		}
 
 		if (!primaryLog.present() || !remoteLog.present()) {
-			wait(dbInfo->onChange());
+			co_await dbInfo->onChange();
 			continue;
 		}
 
 		ASSERT(primaryLog.present());
 		ASSERT(remoteLog.present());
 
-		state Future<Void> onChange = dbInfo->onChange();
-		loop {
-			state Future<TLogQueuingMetricsReply> primaryMetrics =
+		Future<Void> onChange = dbInfo->onChange();
+		while (true) {
+			Future<TLogQueuingMetricsReply> primaryMetrics =
 			    brokenPromiseToNever(primaryLog.get().getQueuingMetrics.getReply(TLogQueuingMetricsRequest()));
-			state Future<TLogQueuingMetricsReply> remoteMetrics =
+			Future<TLogQueuingMetricsReply> remoteMetrics =
 			    brokenPromiseToNever(remoteLog.get().getQueuingMetrics.getReply(TLogQueuingMetricsRequest()));
 
-			wait((success(primaryMetrics) && success(remoteMetrics)) || onChange);
+			co_await ((success(primaryMetrics) && success(remoteMetrics)) || onChange);
 			if (onChange.isReady()) {
 				break;
 			}
 
 			TraceEvent("DCLag").detail("Primary", primaryMetrics.get().v).detail("Remote", remoteMetrics.get().v);
 			ASSERT(primaryMetrics.get().v >= 0 && remoteMetrics.get().v >= 0);
-			return primaryMetrics.get().v - remoteMetrics.get().v;
+			co_return primaryMetrics.get().v - remoteMetrics.get().v;
 		}
 	}
 }
 
-ACTOR Future<Void> repairDeadDatacenter(Database cx,
-                                        Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                                        std::string context) {
-	if (g_network->isSimulated() && g_simulator->usableRegions > 1 && !g_simulator->quiesced) {
-		state bool primaryDead = g_simulator->datacenterDead(g_simulator->primaryDcId);
-		state bool remoteDead = g_simulator->datacenterDead(g_simulator->remoteDcId);
+Future<Void> repairDeadDatacenter(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, std::string context) {
+	if (g_network->isSimulated() && fdbSimulationPolicyState().usableRegions > 1 &&
+	    !fdbSimulationPolicyState().quiesced) {
+		auto& simPolicy = fdbSimulationPolicyState();
+		bool primaryDead = g_simulator->datacenterDead(simPolicy.primaryDcId);
+		bool remoteDead = g_simulator->datacenterDead(simPolicy.remoteDcId);
 
 		// FIXME: the primary and remote can both be considered dead because excludes are not handled properly by the
 		// datacenterDead function
 		if (primaryDead && remoteDead) {
 			TraceEvent(SevWarnAlways, "CannotDisableFearlessConfiguration").log();
-			return Void();
+			co_return;
 		}
 		if (primaryDead || remoteDead) {
 			if (remoteDead) {
-				std::vector<AddressExclusion> servers =
-				    g_simulator->getAllAddressesInDCToExclude(g_simulator->remoteDcId);
+				std::vector<AddressExclusion> servers = g_simulator->getAllAddressesInDCToExclude(simPolicy.remoteDcId);
 				TraceEvent(SevWarnAlways, "DisablingFearlessConfiguration")
 				    .detail("Location", context)
 				    .detail("Stage", "ExcludeServers")
 				    .detail("Servers", describe(servers));
-				wait(excludeServers(cx, servers, false));
+				co_await excludeServers(cx, servers, false);
 			}
 
 			TraceEvent(SevWarnAlways, "DisablingFearlessConfiguration")
@@ -719,31 +725,30 @@ ACTOR Future<Void> repairDeadDatacenter(Database cx,
 			    .detail("Stage", "Repopulate")
 			    .detail("RemoteDead", remoteDead)
 			    .detail("PrimaryDead", primaryDead);
-			g_simulator->usableRegions = 1;
+			simPolicy.usableRegions = 1;
 
-			wait(success(ManagementAPI::changeConfig(
+			co_await ManagementAPI::changeConfig(
 			    cx.getReference(),
-			    (primaryDead ? g_simulator->disablePrimary : g_simulator->disableRemote) + " repopulate_anti_quorum=1",
-			    true)));
+			    (primaryDead ? fdbSimulationPolicyState().disablePrimary : fdbSimulationPolicyState().disableRemote) +
+			        " repopulate_anti_quorum=1",
+			    true);
 			while (dbInfo->get().recoveryState < RecoveryState::STORAGE_RECOVERED) {
-				wait(dbInfo->onChange());
+				co_await dbInfo->onChange();
 			}
 			TraceEvent(SevWarnAlways, "DisablingFearlessConfiguration")
 			    .detail("Location", context)
 			    .detail("Stage", "Usable_Regions");
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), "usable_regions=1", true)));
+			co_await ManagementAPI::changeConfig(cx.getReference(), "usable_regions=1", true);
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> reconfigureAfter(Database cx,
-                                    double time,
-                                    Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                                    std::string context) {
-	wait(delay(time));
-	wait(uncancellable(repairDeadDatacenter(cx, dbInfo, context)));
-	return Void();
+Future<Void> reconfigureAfter(Database cx,
+                              double time,
+                              Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                              std::string context) {
+	co_await delay(time);
+	co_await uncancellable(repairDeadDatacenter(cx, dbInfo, context));
 }
 
 struct QuietDatabaseChecker {
@@ -755,7 +760,7 @@ struct QuietDatabaseChecker {
 	ProcessEvents::Event timeoutEvent;
 	std::vector<std::string> lastFailReasons;
 
-	QuietDatabaseChecker(double maxDDRunTime)
+	explicit QuietDatabaseChecker(double maxDDRunTime)
 	  : maxDDRunTime(maxDDRunTime), timeoutEvent({ "Timeout"_sr, "TracedTooManyLines"_sr }, timeoutCallback) {}
 
 	void logFailure(StringRef name, StringRef msg, Error const& e) {
@@ -821,35 +826,37 @@ struct QuietDatabaseChecker {
 	}
 };
 
-ACTOR Future<Void> enableConsistencyScanInSim(Database db) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	state ConsistencyScanState cs;
+Future<Void> enableConsistencyScanInSim(Database db) {
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
+	ConsistencyScanState cs;
 	if (!g_network->isSimulated()) {
-		return Void();
+		co_return;
 	}
 	TraceEvent("ConsistencyScan_SimEnable").log();
-	loop {
+	while (true) {
+		Error err;
 		try {
 			SystemDBWriteLockedNow(db.getReference())->setOptions(tr);
-			state ConsistencyScanState::Config config = wait(cs.config().getD(tr));
+			ConsistencyScanState::Config config = co_await cs.config().getD(tr);
 
 			// Enable if disabled, otherwise sometimes update the scan min version to restart it
-			if (!config.enabled && g_simulator->consistencyScanState < ISimulator::SimConsistencyScanState::Enabled) {
-				if (!g_simulator->doInjectConsistencyScanCorruption.present()) {
-					g_simulator->doInjectConsistencyScanCorruption = BUGGIFY_WITH_PROB(0.1);
+			if (!config.enabled &&
+			    fdbSimulationPolicyState().consistencyScanState < FDBSimConsistencyScanState::Enabled) {
+				if (!fdbSimulationPolicyState().doInjectConsistencyScanCorruption.present()) {
+					fdbSimulationPolicyState().doInjectConsistencyScanCorruption = BUGGIFY_WITH_PROB(0.1);
 					TraceEvent("ConsistencyScan_DoInjectCorruption")
-					    .detail("Val", g_simulator->doInjectConsistencyScanCorruption.get());
+					    .detail("Val", fdbSimulationPolicyState().doInjectConsistencyScanCorruption.get());
 				}
 
-				g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::DisabledStart,
-				                                        ISimulator::SimConsistencyScanState::Enabling);
+				fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::DisabledStart,
+				                                                      FDBSimConsistencyScanState::Enabling);
 				config.enabled = true;
 			} else {
-				if (config.enabled && g_simulator->restarted &&
-				    g_simulator->consistencyScanState == ISimulator::SimConsistencyScanState::DisabledStart) {
+				if (config.enabled && fdbSimulationPolicyState().restarted &&
+				    fdbSimulationPolicyState().consistencyScanState == FDBSimConsistencyScanState::DisabledStart) {
 					TraceEvent("ConsistencyScan_SimEnableAlreadyDoneFromRestart").log();
-					g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::DisabledStart,
-					                                        ISimulator::SimConsistencyScanState::Enabling);
+					fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::DisabledStart,
+					                                                      FDBSimConsistencyScanState::Enabling);
 				}
 				if (BUGGIFY_WITH_PROB(0.5)) {
 					config.minStartVersion = tr->getReadVersion().get();
@@ -862,10 +869,10 @@ ACTOR Future<Void> enableConsistencyScanInSim(Database db) {
 
 			if (config.enabled) {
 				cs.config().set(tr, config);
-				wait(tr->commit());
+				co_await tr->commit();
 
-				g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Enabling,
-				                                        ISimulator::SimConsistencyScanState::Enabled);
+				fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::Enabling,
+				                                                      FDBSimConsistencyScanState::Enabled);
 				TraceEvent("ConsistencyScan_SimEnabled")
 				    .detail("MaxReadByteRate", config.maxReadByteRate)
 				    .detail("TargetRoundTimeSeconds", config.targetRoundTimeSeconds)
@@ -875,46 +882,46 @@ ACTOR Future<Void> enableConsistencyScanInSim(Database db) {
 
 			break;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> disableConsistencyScanInSim(Database db, bool waitForCompletion) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(db);
-	state ConsistencyScanState cs;
+Future<Void> disableConsistencyScanInSim(Database db, bool waitForCompletion) {
+	auto tr = makeReference<ReadYourWritesTransaction>(db);
+	ConsistencyScanState cs;
 	if (!g_network->isSimulated()) {
-		return Void();
+		co_return;
 	}
 
 	if (waitForCompletion) {
 		TraceEvent("ConsistencyScan_SimDisableWaiting").log();
 		printf("Waiting for consistency scan to complete...\n");
-		loop {
-			bool waitForCorruption = g_simulator->doInjectConsistencyScanCorruption.present() &&
-			                         g_simulator->doInjectConsistencyScanCorruption.get();
-			if (((waitForCorruption &&
-			      g_simulator->consistencyScanState >= ISimulator::SimConsistencyScanState::Enabled_FoundCorruption) ||
+		while (true) {
+			bool waitForCorruption = fdbSimulationPolicyState().doInjectConsistencyScanCorruption.present() &&
+			                         fdbSimulationPolicyState().doInjectConsistencyScanCorruption.get();
+			if (((waitForCorruption && fdbSimulationPolicyState().consistencyScanState >=
+			                               FDBSimConsistencyScanState::Enabled_FoundCorruption) ||
 			     (!waitForCorruption &&
-			      g_simulator->consistencyScanState >= ISimulator::SimConsistencyScanState::Enabled))) {
+			      fdbSimulationPolicyState().consistencyScanState >= FDBSimConsistencyScanState::Enabled))) {
 				break;
 			}
-			wait(delay(1.0));
+			co_await delay(1.0);
 		}
 	}
 	TraceEvent("ConsistencyScan_SimDisable").log();
-	loop {
+	while (true) {
+		Error err;
 		try {
 			SystemDBWriteLockedNow(db.getReference())->setOptions(tr);
-			state ConsistencyScanState::Config config = wait(cs.config().getD(tr));
-			state bool skipDisable = false;
+			ConsistencyScanState::Config config = co_await cs.config().getD(tr);
+			bool skipDisable = false;
 			// see if any rounds have completed
 			if (waitForCompletion) {
-				state ConsistencyScanState::RoundStats statsCurrentRound = wait(cs.currentRoundStats().getD(tr));
-				state ConsistencyScanState::StatsHistoryMap::RangeResultType olderStats =
-				    wait(cs.roundStatsHistory().getRange(tr, {}, {}, 1, Snapshot::False, Reverse::False));
+				ConsistencyScanState::RoundStats statsCurrentRound = co_await cs.currentRoundStats().getD(tr);
+				ConsistencyScanState::StatsHistoryMap::RangeResultType olderStats =
+				    co_await cs.roundStatsHistory().getRange(tr, {}, {}, 1, Snapshot::False, Reverse::False);
 				if (olderStats.results.empty() && !statsCurrentRound.complete) {
 					TraceEvent("ConsistencyScan_SimDisable_NoRoundsCompleted").log();
 					skipDisable = true;
@@ -924,95 +931,101 @@ ACTOR Future<Void> disableConsistencyScanInSim(Database db, bool waitForCompleti
 			// Enable if disable, else set the scan min version to restart it
 			if (config.enabled) {
 				// state was either enabled or enabled_foundcorruption
-				g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Enabled,
-				                                        ISimulator::SimConsistencyScanState::Complete);
-				g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Enabled_FoundCorruption,
-				                                        ISimulator::SimConsistencyScanState::Complete);
+				fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::Enabled,
+				                                                      FDBSimConsistencyScanState::Complete);
+				fdbSimulationPolicyState().updateConsistencyScanState(
+				    FDBSimConsistencyScanState::Enabled_FoundCorruption, FDBSimConsistencyScanState::Complete);
 				config.enabled = false;
 			} else {
 				TraceEvent("ConsistencyScan_SimDisableAlreadyDisabled").log();
 				printf("Consistency scan already complete.\n");
-				return Void();
+				co_return;
 			}
 
 			if (skipDisable) {
-				wait(delay(2.0));
+				co_await delay(2.0);
 				tr->reset();
+				continue;
 			} else {
 				cs.config().set(tr, config);
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			}
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 
-	g_simulator->updateConsistencyScanState(ISimulator::SimConsistencyScanState::Complete,
-	                                        ISimulator::SimConsistencyScanState::DisabledEnd);
+	fdbSimulationPolicyState().updateConsistencyScanState(FDBSimConsistencyScanState::Complete,
+	                                                      FDBSimConsistencyScanState::DisabledEnd);
 	CODE_PROBE(true, "Consistency Scan disabled in simulation");
 	TraceEvent("ConsistencyScan_SimDisabled").log();
 	printf("Consistency scan complete.\n");
-	return Void();
 }
 
 // Waits until a database quiets down (no data in flight, small tlog queue, low SQ, no active data distribution). This
 // requires the database to be available and healthy in order to succeed.
-ACTOR Future<Void> waitForQuietDatabase(Database cx,
-                                        Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                                        std::string phase,
-                                        int64_t dataInFlightGate = 2e6,
-                                        int64_t maxTLogQueueGate = 5e6,
-                                        int64_t maxStorageServerQueueGate = 5e6,
-                                        int64_t maxDataDistributionQueueSize = 0,
-                                        int64_t maxPoppedVersionLag = 30e6,
-                                        int64_t maxVersionOffset = 1e6,
-                                        double maxDDRunTime = 0) {
+Future<Void> waitForQuietDatabase(Database cx,
+                                  Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                  std::string phase,
+                                  int64_t dataInFlightGate = 2e6,
+                                  int64_t maxTLogQueueGate = 5e6,
+                                  int64_t maxStorageServerQueueGate = 5e6,
+                                  int64_t maxDataDistributionQueueSize = 0,
+                                  int64_t maxPoppedVersionLag = 30e6,
+                                  int64_t maxVersionOffset = 1e6,
+                                  double maxDDRunTime = 0) {
 	// Use provided maxDDRunTime, or fallback to default (1500s or 4000s with buggify)
-	state QuietDatabaseChecker checker(maxDDRunTime > 0 ? maxDDRunTime : (isGeneralBuggifyEnabled() ? 4000.0 : 1500.0));
-	state Future<Void> reconfig =
+	QuietDatabaseChecker checker(maxDDRunTime > 0 ? maxDDRunTime : (isGeneralBuggifyEnabled() ? 4000.0 : 1500.0));
+	[[maybe_unused]] Future<Void> reconfig =
 	    reconfigureAfter(cx, 100 + (deterministicRandom()->random01() * 100), dbInfo, "QuietDatabase");
-	state Future<int64_t> dataInFlight;
-	state Future<std::pair<int64_t, int64_t>> tLogQueueInfo;
-	state Future<int64_t> dataDistributionQueueSize;
-	state Future<bool> teamCollectionValid;
-	state Future<int64_t> storageQueueSize;
-	state Future<bool> dataDistributionActive;
-	state Future<bool> storageServersRecruiting;
-	state Future<int64_t> versionOffset;
-	state Future<Version> dcLag;
-	state Version maxDcLag = 100e6;
-	state std::string traceMessage = "QuietDatabase" + phase + "Begin";
+	Future<int64_t> dataInFlight;
+	Future<std::pair<int64_t, int64_t>> tLogQueueInfo;
+	Future<int64_t> dataDistributionQueueSize;
+	Future<bool> teamCollectionValid;
+	Future<int64_t> storageQueueSize;
+	Future<bool> dataDistributionActive;
+	Future<bool> storageServersRecruiting;
+	Future<int64_t> versionOffset;
+	Future<Version> dcLag;
+	Version maxDcLag = 100e6;
+	std::string traceMessage = "QuietDatabase" + phase + "Begin";
 	TraceEvent(traceMessage.c_str()).log();
 
 	// In a simulated environment, wait 5 seconds so that workers can move to their optimal locations
 	if (g_network->isSimulated())
-		wait(delay(5.0));
+		co_await delay(5.0);
 
 	TraceEvent("QuietDatabaseWaitingOnFullRecovery").detail("Phase", phase).log();
 	while (dbInfo->get().recoveryState != RecoveryState::FULLY_RECOVERED) {
-		wait(dbInfo->onChange());
+		co_await dbInfo->onChange();
 	}
 
 	// The quiet database check (which runs at the end of every test) will always time out due to active data movement.
 	// To get around this, quiet Database will disable the perpetual wiggle in the setup phase.
 
 	printf("Set perpetual_storage_wiggle=0 ...\n");
-	state Version version = wait(setPerpetualStorageWiggle(cx, false, LockAware::True));
+	Version version = co_await setPerpetualStorageWiggle(cx, false, LockAware::True);
 	printf("Set perpetual_storage_wiggle=0 Done.\n");
 
 	printf("Disabling backup worker ...\n");
-	wait(disableBackupWorker(cx));
+	co_await disableBackupWorker(cx);
 	printf("Disabled backup worker.\n");
 
-	wait(disableConsistencyScanInSim(cx, false));
+	co_await disableConsistencyScanInSim(cx, false);
+
+	if (g_network->isSimulated()) {
+		disableDDPipelineControl();
+	}
 
 	// Require 3 consecutive successful quiet database checks spaced 2 second apart
-	state int numSuccesses = 0;
-	loop {
+	int numSuccesses = 0;
+	while (true) {
+		Optional<double> retryDelay;
 		try {
 			TraceEvent("QuietDatabaseWaitingOnDataDistributor").log();
-			WorkerInterface distributorWorker = wait(getDataDistributorWorker(cx, dbInfo));
+			WorkerInterface distributorWorker = co_await getDataDistributorWorker(cx, dbInfo);
 			UID distributorUID = dbInfo->get().distributor.get().id();
 			TraceEvent("QuietDatabaseGotDataDistributor", distributorUID)
 			    .detail("Locality", distributorWorker.locality.toString());
@@ -1027,9 +1040,9 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 			versionOffset = getVersionOffset(cx, distributorWorker, dbInfo);
 			dcLag = getDatacenterLag(cx, dbInfo);
 
-			wait(success(dataInFlight) && success(tLogQueueInfo) && success(dataDistributionQueueSize) &&
-			     success(teamCollectionValid) && success(storageQueueSize) && success(dataDistributionActive) &&
-			     success(storageServersRecruiting) && success(versionOffset) && success(dcLag));
+			co_await (success(dataInFlight) && success(tLogQueueInfo) && success(dataDistributionQueueSize) &&
+			          success(teamCollectionValid) && success(storageQueueSize) && success(dataDistributionActive) &&
+			          success(storageServersRecruiting) && success(versionOffset) && success(dcLag));
 
 			maxVersionOffset += dbInfo->get().recoveryCount * SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT;
 
@@ -1057,10 +1070,10 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 					TraceEvent(msg.c_str()).log();
 					break;
 				} else {
-					wait(delay(g_network->isSimulated() ? 2.0 : 30.0));
+					retryDelay = g_network->isSimulated() ? 2.0 : 30.0;
 				}
 			} else {
-				wait(delay(1.0));
+				retryDelay = 1.0;
 				numSuccesses = 0;
 			}
 		} catch (Error& e) {
@@ -1110,12 +1123,13 @@ ACTOR Future<Void> waitForQuietDatabase(Database cx,
 				auto key = "NotReady" + std::to_string(notReadyCount++);
 				evt.detail(key.c_str(), "versionOffset");
 			}
-			wait(delay(1.0));
+			retryDelay = 1.0;
 			numSuccesses = 0;
 		}
+		if (retryDelay.present()) {
+			co_await delay(retryDelay.get());
+		}
 	}
-
-	return Void();
 }
 
 Future<Void> quietDatabase(Database const& cx,
diff --git a/fdbserver/core/RatekeeperLimitReasons.cpp b/fdbserver/core/RatekeeperLimitReasons.cpp
new file mode 100644
index 00000000000..9b431ef658e
--- /dev/null
+++ b/fdbserver/core/RatekeeperLimitReasons.cpp
@@ -0,0 +1,55 @@
+/*
+ * RatekeeperLimitReasons.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/RatekeeperLimitReasons.h"
+
+const char* limitReasonName[] = {
+	"workload",
+	"storage_server_write_queue_size",
+	"storage_server_write_bandwidth_mvcc",
+	"storage_server_readable_behind",
+	"log_server_mvcc_write_bandwidth",
+	"log_server_write_queue",
+	"storage_server_min_free_space",
+	"storage_server_min_free_space_ratio",
+	"log_server_min_free_space",
+	"log_server_min_free_space_ratio",
+	"storage_server_durability_lag",
+	"storage_server_list_fetch_failed",
+};
+static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonName table size");
+
+int limitReasonEnd = limitReason_t_end;
+
+const char* limitReasonDesc[] = {
+	"Workload or read performance.",
+	"Storage server performance (storage queue).",
+	"Storage server MVCC memory.",
+	"Storage server version falling behind.",
+	"Log server MVCC memory.",
+	"Storage server performance (log queue).",
+	"Storage server running out of space (approaching 100MB limit).",
+	"Storage server running out of space (approaching 5% limit).",
+	"Log server running out of space (approaching 100MB limit).",
+	"Log server running out of space (approaching 5% limit).",
+	"Storage server durable version falling behind.",
+	"Unable to fetch storage server list.",
+};
+static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
diff --git a/fdbserver/RocksDBCheckpointUtils.actor.cpp b/fdbserver/core/RocksDBCheckpointUtils.cpp
similarity index 84%
rename from fdbserver/RocksDBCheckpointUtils.actor.cpp
rename to fdbserver/core/RocksDBCheckpointUtils.cpp
index 82df40ea2fc..585380776c0 100644
--- a/fdbserver/RocksDBCheckpointUtils.actor.cpp
+++ b/fdbserver/core/RocksDBCheckpointUtils.cpp
@@ -1,5 +1,5 @@
 /*
- *RocksDBCheckpointUtils.actor.cpp
+ *RocksDBCheckpointUtils.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
 
 #ifdef WITH_ROCKSDB
 #include <rocksdb/db.h>
@@ -33,18 +33,16 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/MutationTracking.h"
+#include "fdbserver/core/MutationTracking.h"
 #include "fdbclient/StorageCheckpoint.h"
 #include "fdbserver/CoroFlow.h"
-#include "fdbserver/FDBRocksDBVersion.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/FDBRocksDBVersion.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/IThreadPool.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/Trace.h"
 #include "flow/flow.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 #ifdef WITH_ROCKSDB
 
 // Enforcing rocksdb version.
@@ -155,33 +153,36 @@ Error statusToError(const rocksdb::Status& s) {
 }
 
 // Fetch a single sst file from storage server. The progress is checkpointed via cFun.
-ACTOR Future<int64_t> doFetchCheckpointFile(Database cx,
-                                            std::string remoteFile,
-                                            std::string localFile,
-                                            UID ssId,
-                                            UID checkpointId,
-                                            int maxRetries = 3) {
-	state Transaction tr(cx);
-	state StorageServerInterface ssi;
-	loop {
+Future<int64_t> doFetchCheckpointFile(Database cx,
+                                      std::string remoteFile,
+                                      std::string localFile,
+                                      UID ssId,
+                                      UID checkpointId,
+                                      int maxRetries = 3) {
+	Transaction tr(cx);
+	StorageServerInterface ssi;
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			Optional<Value> ss = wait(tr.get(serverListKeyFor(ssId)));
+			Optional<Value> ss = co_await tr.get(serverListKeyFor(ssId));
 			if (!ss.present()) {
 				throw checkpoint_not_found();
 			}
 			ssi = decodeServerListValue(ss.get());
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 
-	state int attempt = 0;
-	state int64_t offset = 0;
-	state Reference<IAsyncFile> asyncFile;
-	loop {
+	int attempt = 0;
+	int64_t offset = 0;
+	Reference<IAsyncFile> asyncFile;
+	while (true) {
 		offset = 0;
+		Error err;
 		try {
 			asyncFile = Reference<IAsyncFile>();
 			++attempt;
@@ -192,12 +193,12 @@ ACTOR Future<int64_t> doFetchCheckpointFile(Database cx,
 			    .detail("CheckpointId", checkpointId)
 			    .detail("Attempt", attempt);
 
-			wait(IAsyncFileSystem::filesystem()->deleteFile(localFile, true));
+			co_await IAsyncFileSystem::filesystem()->deleteFile(localFile, true);
 			const int64_t flags = IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE |
 			                      IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO;
-			wait(store(asyncFile, IAsyncFileSystem::filesystem()->open(localFile, flags, 0666)));
+			asyncFile = co_await IAsyncFileSystem::filesystem()->open(localFile, flags, 0666);
 
-			state ReplyPromiseStream<FetchCheckpointReply> stream =
+			ReplyPromiseStream<FetchCheckpointReply> stream =
 			    ssi.fetchCheckpoint.getReplyStream(FetchCheckpointRequest(checkpointId, remoteFile));
 			TraceEvent(SevDebug, "FetchCheckpointFileReceivingData")
 			    .detail("RemoteFile", remoteFile)
@@ -205,62 +206,62 @@ ACTOR Future<int64_t> doFetchCheckpointFile(Database cx,
 			    .detail("TargetUID", ssId)
 			    .detail("CheckpointId", checkpointId)
 			    .detail("Attempt", attempt);
-			loop {
-				state FetchCheckpointReply rep = waitNext(stream.getFuture());
-				wait(asyncFile->write(rep.data.begin(), rep.data.size(), offset));
-				wait(asyncFile->flush());
+			while (true) {
+				FetchCheckpointReply rep = co_await stream.getFuture();
+				co_await asyncFile->write(rep.data.begin(), rep.data.size(), offset);
+				co_await asyncFile->flush();
 				offset += rep.data.size();
 			}
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			} else if (e.code() != error_code_end_of_stream) {
-				TraceEvent(SevWarnAlways, "FetchCheckpointFileError")
-				    .errorUnsuppressed(e)
-				    .detail("RemoteFile", remoteFile)
-				    .detail("LocalFile", localFile)
-				    .detail("TargetUID", ssId)
-				    .detail("CheckpointId", checkpointId)
-				    .detail("Attempt", attempt);
-				if (attempt >= maxRetries) {
-					throw e;
-				}
-			} else {
-				wait(asyncFile->sync());
-				int64_t fileSize = wait(asyncFile->size());
-				TraceEvent(SevDebug, "FetchCheckpointFileEnd")
-				    .detail("RemoteFile", remoteFile)
-				    .detail("LocalFile", localFile)
-				    .detail("TargetUID", ssId)
-				    .detail("CheckpointId", checkpointId)
-				    .detail("Attempt", attempt)
-				    .detail("FileSize", fileSize);
-				return fileSize;
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		} else if (err.code() != error_code_end_of_stream) {
+			TraceEvent(SevWarnAlways, "FetchCheckpointFileError")
+			    .errorUnsuppressed(err)
+			    .detail("RemoteFile", remoteFile)
+			    .detail("LocalFile", localFile)
+			    .detail("TargetUID", ssId)
+			    .detail("CheckpointId", checkpointId)
+			    .detail("Attempt", attempt);
+			if (attempt >= maxRetries) {
+				throw err;
 			}
+		} else {
+			co_await asyncFile->sync();
+			int64_t fileSize = co_await asyncFile->size();
+			TraceEvent(SevDebug, "FetchCheckpointFileEnd")
+			    .detail("RemoteFile", remoteFile)
+			    .detail("LocalFile", localFile)
+			    .detail("TargetUID", ssId)
+			    .detail("CheckpointId", checkpointId)
+			    .detail("Attempt", attempt)
+			    .detail("FileSize", fileSize);
+			co_return fileSize;
 		}
 	}
 }
 
-ACTOR Future<Void> fetchCheckpointBytesSampleFile(Database cx,
-                                                  std::shared_ptr<CheckpointMetaData> metaData,
-                                                  std::string dir,
-                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun,
-                                                  int maxRetries = 3) {
+Future<Void> fetchCheckpointBytesSampleFile(Database cx,
+                                            std::shared_ptr<CheckpointMetaData> metaData,
+                                            std::string dir,
+                                            std::function<Future<Void>(const CheckpointMetaData&)> cFun,
+                                            int maxRetries = 3) {
 	ASSERT(metaData->bytesSampleFile.present());
-	state std::string localFile = joinPath(dir, metaData->checkpointID.toString() + "_metadata_bytes.sst");
+	std::string localFile = joinPath(dir, metaData->checkpointID.toString() + "_metadata_bytes.sst");
 	TraceEvent(SevDebug, "FetchCheckpointByteSampleBegin")
 	    .detail("Checkpoint", metaData->toString())
 	    .detail("LocalFile", localFile);
 
 	ASSERT(!metaData->src.empty());
-	state UID ssId = metaData->src.front();
+	UID ssId = metaData->src.front();
 
-	wait(success(doFetchCheckpointFile(cx, metaData->bytesSampleFile.get(), localFile, ssId, metaData->checkpointID)));
+	co_await doFetchCheckpointFile(cx, metaData->bytesSampleFile.get(), localFile, ssId, metaData->checkpointID);
 	metaData->bytesSampleFile = localFile;
 	if (cFun) {
-		wait(cFun(*metaData));
+		co_await cFun(*metaData);
 	}
-	return Void();
 }
 
 // RocksDBColumnFamilyReader reads a RocksDB checkpoint, and returns the key-value pairs via nextKeyValues.
@@ -291,7 +292,7 @@ class RocksDBColumnFamilyReader : public ICheckpointReader {
 			iterator->Seek(this->beginSlice);
 		}
 
-		~RocksCfCheckpointIterator() { this->reader->numIter--; }
+		~RocksCfCheckpointIterator() override { this->reader->numIter--; }
 
 		Future<RangeResult> nextBatch(const int rowLimit, const int ByteLimit) override;
 
@@ -310,9 +311,9 @@ class RocksDBColumnFamilyReader : public ICheckpointReader {
 
 	class DummyRocksDBCheckpointIterator : public ICheckpointIterator {
 	public:
-		DummyRocksDBCheckpointIterator(const UID& logId) : logId(logId) {}
+		explicit DummyRocksDBCheckpointIterator(const UID& logId) : logId(logId) {}
 
-		~DummyRocksDBCheckpointIterator() {}
+		~DummyRocksDBCheckpointIterator() override = default;
 
 		Future<RangeResult> nextBatch(const int rowLimit, const int ByteLimit) override { throw end_of_stream(); }
 
@@ -337,7 +338,7 @@ class RocksDBColumnFamilyReader : public ICheckpointReader {
 private:
 	struct Reader : IThreadPoolReceiver {
 		struct OpenAction : TypedAction<Reader, OpenAction> {
-			OpenAction(CheckpointMetaData checkpoint) : checkpoint(std::move(checkpoint)) {}
+			explicit OpenAction(CheckpointMetaData checkpoint) : checkpoint(std::move(checkpoint)) {}
 
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 
@@ -367,7 +368,7 @@ class RocksDBColumnFamilyReader : public ICheckpointReader {
 		};
 
 		explicit Reader(DB& db, CF& cf, std::vector<rocksdb::ColumnFamilyHandle*>& handles, const UID& logId);
-		~Reader() override {}
+		~Reader() override = default;
 
 		void init() override {}
 
@@ -392,7 +393,7 @@ class RocksDBColumnFamilyReader : public ICheckpointReader {
 
 	Future<RangeResult> nextBatch(const int rowLimit, const int byteLimit, RocksCfCheckpointIterator* iterator);
 
-	ACTOR static Future<Void> doClose(RocksDBColumnFamilyReader* self);
+	static Future<Void> doClose(RocksDBColumnFamilyReader* self);
 
 	DB db = nullptr;
 	CF cf = nullptr;
@@ -707,27 +708,21 @@ rocksdb::Status RocksDBColumnFamilyReader::Reader::closeInternal(const std::stri
 	return s;
 }
 
-ACTOR Future<Void> RocksDBColumnFamilyReader::doClose(RocksDBColumnFamilyReader* self) {
-	if (self == nullptr)
-		return Void();
+Future<Void> RocksDBColumnFamilyReader::doClose(RocksDBColumnFamilyReader* self) {
+	if (self == nullptr) {
+		co_return;
+	}
 
 	auto a = new RocksDBColumnFamilyReader::Reader::CloseAction(self->path, false);
 	auto f = a->done.getFuture();
 	self->threads->post(a);
-	wait(f);
-
-	if (self != nullptr) {
-		wait(self->threads->stop());
-	}
+	co_await f;
 
-	if (self != nullptr) {
-		if (self->db != nullptr) {
-			delete self->db;
-		}
-		delete self;
+	co_await self->threads->stop();
+	if (self->db != nullptr) {
+		delete self->db;
 	}
-
-	return Void();
+	delete self;
 }
 
 class RocksDBSstFileWriter : public IRocksDBSstFileWriter {
@@ -798,7 +793,7 @@ class RocksDBSstFileReader : public IRocksDBSstFileReader {
 		endSlice = toSlice(rangeBoundary.end);
 	};
 
-	~RocksDBSstFileReader() {}
+	~RocksDBSstFileReader() override = default;
 
 	void open(const std::string localFile) override;
 
@@ -895,8 +890,8 @@ RangeResult RocksDBSstFileReader::getRange(const KeyRange& range) {
 
 class RocksDBCheckpointByteSampleReader : public ICheckpointByteSampleReader {
 public:
-	RocksDBCheckpointByteSampleReader(const CheckpointMetaData& checkpoint);
-	~RocksDBCheckpointByteSampleReader() {}
+	explicit(false) RocksDBCheckpointByteSampleReader(const CheckpointMetaData& checkpoint);
+	~RocksDBCheckpointByteSampleReader() = default;
 
 	KeyValue next() override;
 
@@ -950,11 +945,11 @@ class RocksDBCFCheckpointReader : public ICheckpointReader {
 	Future<Void> close() override;
 
 private:
-	ACTOR static Future<Void> doInit(RocksDBCFCheckpointReader* self) {
+	static Future<Void> doInit(RocksDBCFCheckpointReader* self) {
 		ASSERT(self != nullptr);
 		try {
-			state Reference<IAsyncFile> _file = wait(IAsyncFileSystem::filesystem()->open(
-			    self->path_, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO, 0));
+			Reference<IAsyncFile> _file = co_await IAsyncFileSystem::filesystem()->open(
+			    self->path_, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO, 0);
 			self->file_ = _file;
 			TraceEvent("RocksDBCheckpointReaderOpenFile").detail("File", self->path_);
 		} catch (Error& e) {
@@ -963,27 +958,25 @@ class RocksDBCFCheckpointReader : public ICheckpointReader {
 			    .detail("File", self->path_);
 			throw e;
 		}
-
-		return Void();
 	}
 
-	ACTOR static UNCANCELLABLE Future<Standalone<StringRef>> getNextChunk(RocksDBCFCheckpointReader* self,
-	                                                                      int byteLimit) {
+	static Future<Standalone<StringRef>> getNextChunk(RocksDBCFCheckpointReader* self,
+	                                                  int byteLimit,
+	                                                  Uncancellable = Uncancellable()) {
 		int blockSize = std::min(64 * 1024, byteLimit); // Block size read from disk.
-		state Standalone<StringRef> buf = makeAlignedString(_PAGE_SIZE, blockSize);
-		int bytesRead = wait(self->file_->read(mutateString(buf), blockSize, self->offset_));
+		Standalone<StringRef> buf = makeAlignedString(_PAGE_SIZE, blockSize);
+		int bytesRead = co_await self->file_->read(mutateString(buf), blockSize, self->offset_);
 		if (bytesRead == 0) {
 			throw end_of_stream();
 		}
 
 		self->offset_ += bytesRead;
-		return buf.substr(0, bytesRead);
+		co_return buf.substr(0, bytesRead);
 	}
 
-	ACTOR static Future<Void> doClose(RocksDBCFCheckpointReader* self) {
-		wait(delay(0, TaskPriority::FetchKeys));
+	static Future<Void> doClose(RocksDBCFCheckpointReader* self) {
+		co_await delay(0, TaskPriority::FetchKeys);
 		delete self;
-		return Void();
 	}
 
 	CheckpointMetaData checkpoint_;
@@ -1028,17 +1021,17 @@ Future<Void> RocksDBCFCheckpointReader::close() {
 }
 
 // Fetch a single sst file from storage server. The progress is checkpointed via cFun.
-ACTOR Future<Void> fetchCheckpointFile(Database cx,
-                                       std::shared_ptr<CheckpointMetaData> metaData,
-                                       int idx,
-                                       std::string dir,
-                                       std::function<Future<Void>(const CheckpointMetaData&)> cFun,
-                                       int maxRetries = 3) {
-	state RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(*metaData);
+Future<Void> fetchCheckpointFile(Database cx,
+                                 std::shared_ptr<CheckpointMetaData> metaData,
+                                 int idx,
+                                 std::string dir,
+                                 std::function<Future<Void>(const CheckpointMetaData&)> cFun,
+                                 int maxRetries = 3) {
+	RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(*metaData);
 
 	// Skip fetched file.
 	if (rocksCF.sstFiles[idx].fetched && rocksCF.sstFiles[idx].db_path == dir) {
-		return Void();
+		co_return;
 	}
 
 	const std::string remoteFile = rocksCF.sstFiles[idx].name;
@@ -1046,25 +1039,24 @@ ACTOR Future<Void> fetchCheckpointFile(Database cx,
 	ASSERT_EQ(metaData->src.size(), 1);
 	const UID ssId = metaData->src.front();
 
-	wait(success(doFetchCheckpointFile(cx, remoteFile, localFile, ssId, metaData->checkpointID)));
+	co_await doFetchCheckpointFile(cx, remoteFile, localFile, ssId, metaData->checkpointID);
 	rocksCF.sstFiles[idx].db_path = dir;
 	rocksCF.sstFiles[idx].fetched = true;
 	metaData->setSerializedCheckpoint(ObjectWriter::toValue(rocksCF, IncludeVersion()));
 	if (cFun) {
-		wait(cFun(*metaData));
+		co_await cFun(*metaData);
 	}
-	return Void();
 }
 
 // TODO: Return when a file exceeds a limit.
-ACTOR Future<Void> fetchCheckpointRange(Database cx,
-                                        std::shared_ptr<CheckpointMetaData> metaData,
-                                        KeyRange range,
-                                        std::string dir,
-                                        std::shared_ptr<rocksdb::SstFileWriter> writer,
-                                        std::function<Future<Void>(const CheckpointMetaData&)> cFun,
-                                        int maxRetries = 3) {
-	state std::string localFile =
+Future<Void> fetchCheckpointRange(Database cx,
+                                  std::shared_ptr<CheckpointMetaData> metaData,
+                                  KeyRange range,
+                                  std::string dir,
+                                  std::shared_ptr<rocksdb::SstFileWriter> writer,
+                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun,
+                                  int maxRetries = 3) {
+	std::string localFile =
 	    joinPath(dir, UID(metaData->checkpointID.first(), deterministicRandom()->randomUInt64()).toString() + ".sst");
 	RocksDBCheckpointKeyValues rkv = getRocksKeyValuesCheckpoint(*metaData);
 	TraceEvent("FetchCheckpointRange", metaData->checkpointID)
@@ -1077,14 +1069,15 @@ ACTOR Future<Void> fetchCheckpointRange(Database cx,
 	}
 
 	ASSERT(!metaData->src.empty());
-	state UID ssID = metaData->src.front();
-	state Transaction tr(cx);
-	state StorageServerInterface ssi;
-	loop {
+	UID ssID = metaData->src.front();
+	Transaction tr(cx);
+	StorageServerInterface ssi;
+	while (true) {
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		Error err;
 		try {
-			Optional<Value> ss = wait(tr.get(serverListKeyFor(ssID)));
+			Optional<Value> ss = co_await tr.get(serverListKeyFor(ssID));
 			if (!ss.present()) {
 				TraceEvent(SevWarnAlways, "FetchCheckpointRangeStorageServerNotFound", metaData->checkpointID)
 				    .detail("SSID", ssID)
@@ -1094,18 +1087,19 @@ ACTOR Future<Void> fetchCheckpointRange(Database cx,
 			ssi = decodeServerListValue(ss.get());
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 
 	ASSERT(ssi.id() == ssID);
 
-	state int attempt = 0;
-	state int64_t totalBytes = 0;
-	state int64_t totalKeys = 0;
-	state rocksdb::Status status;
-	state Optional<Error> error;
-	loop {
+	int attempt = 0;
+	int64_t totalBytes = 0;
+	int64_t totalKeys = 0;
+	rocksdb::Status status;
+	Optional<Error> error;
+	while (true) {
 		totalBytes = 0;
 		totalKeys = 0;
 		++attempt;
@@ -1118,7 +1112,7 @@ ACTOR Future<Void> fetchCheckpointRange(Database cx,
 			    .detail("Attempt", attempt)
 			    .log();
 
-			wait(IAsyncFileSystem::filesystem()->deleteFile(localFile, true));
+			co_await IAsyncFileSystem::filesystem()->deleteFile(localFile, true);
 			status = writer->Open(localFile);
 			if (!status.ok()) {
 				Error e = statusToError(status);
@@ -1128,7 +1122,7 @@ ACTOR Future<Void> fetchCheckpointRange(Database cx,
 				throw e;
 			}
 
-			state ReplyPromiseStream<FetchCheckpointKeyValuesStreamReply> stream =
+			ReplyPromiseStream<FetchCheckpointKeyValuesStreamReply> stream =
 			    ssi.fetchCheckpointKeyValues.getReplyStream(
 			        FetchCheckpointKeyValuesRequest(metaData->checkpointID, range));
 			TraceEvent(SevDebug, "FetchCheckpointKeyValuesReceivingData", metaData->checkpointID)
@@ -1139,8 +1133,8 @@ ACTOR Future<Void> fetchCheckpointRange(Database cx,
 			    .detail("Attempt", attempt)
 			    .log();
 
-			loop {
-				FetchCheckpointKeyValuesStreamReply rep = waitNext(stream.getFuture());
+			while (true) {
+				FetchCheckpointKeyValuesStreamReply rep = co_await stream.getFuture();
 				for (int i = 0; i < rep.data.size(); ++i) {
 					DEBUG_MUTATION("FetchCheckpointData",
 					               metaData->version,
@@ -1215,14 +1209,12 @@ ACTOR Future<Void> fetchCheckpointRange(Database cx,
 	if (error.present()) {
 		throw error.get();
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> fetchCheckpointRanges(Database cx,
-                                         std::shared_ptr<CheckpointMetaData> metaData,
-                                         std::string dir,
-                                         std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+Future<Void> fetchCheckpointRanges(Database cx,
+                                   std::shared_ptr<CheckpointMetaData> metaData,
+                                   std::string dir,
+                                   std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
 	RocksDBCheckpointKeyValues rkv = getRocksKeyValuesCheckpoint(*metaData);
 	TraceEvent("FetchCheckpointRanges")
 	    .detail("InitialState", metaData->toString())
@@ -1246,34 +1238,31 @@ ACTOR Future<Void> fetchCheckpointRanges(Database cx,
 			}
 		}
 	}
-	wait(waitForAll(fs));
+	co_await waitForAll(fs);
 	if (cFun) {
-		wait(cFun(*metaData));
+		co_await cFun(*metaData);
 	}
-
-	return Void();
 }
 } // namespace
 
-ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
-                                                        CheckpointMetaData initialState,
-                                                        std::string dir,
-                                                        std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
+                                                  CheckpointMetaData initialState,
+                                                  std::string dir,
+                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
 	TraceEvent(SevInfo, "FetchRocksCheckpointBegin")
 	    .detail("InitialState", initialState.toString())
 	    .detail("CheckpointDir", dir);
 
 	ASSERT(!initialState.ranges.empty());
 
-	state std::shared_ptr<CheckpointMetaData> metaData = std::make_shared<CheckpointMetaData>(initialState);
-	state std::vector<Future<Void>> futures;
+	std::shared_ptr<CheckpointMetaData> metaData = std::make_shared<CheckpointMetaData>(initialState);
+	std::vector<Future<Void>> futures;
 
 	if (metaData->getFormat() == DataMoveRocksCF) {
-		state RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(initialState);
+		RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(initialState);
 		TraceEvent(SevDebug, "RocksDBCheckpointMetaData").detail("RocksCF", rocksCF.toString());
 
-		state int i = 0;
-		for (; i < rocksCF.sstFiles.size(); ++i) {
+		for (int i = 0; i < rocksCF.sstFiles.size(); ++i) {
 			futures.push_back(fetchCheckpointFile(cx, metaData, i, dir, cFun));
 			TraceEvent(SevDebug, "GetCheckpointFetchingFile")
 			    .detail("FileName", rocksCF.sstFiles[i].name)
@@ -1282,29 +1271,29 @@ ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
 		if (metaData->bytesSampleFile.present()) {
 			futures.push_back(fetchCheckpointBytesSampleFile(cx, metaData, dir, cFun));
 		}
-		wait(waitForAll(futures));
+		co_await waitForAll(futures);
 	} else if (metaData->getFormat() == RocksDBKeyValues) {
 		futures.push_back(fetchCheckpointRanges(cx, metaData, dir, cFun));
 		if (metaData->bytesSampleFile.present()) {
 			futures.push_back(fetchCheckpointBytesSampleFile(cx, metaData, dir, cFun));
 		}
-		wait(waitForAll(futures));
+		co_await waitForAll(futures);
 	} else if (metaData->getFormat() == RocksDB) {
 		throw not_implemented();
 	}
 
-	return *metaData;
+	co_return *metaData;
 }
 
-ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
+Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
 	TraceEvent(SevInfo, "DeleteRocksCheckpointBegin", checkpoint.checkpointID)
 	    .detail("Checkpoint", checkpoint.toString());
 	if (checkpoint.getState() == CheckpointMetaData::Fail) {
-		return Void();
+		co_return;
 	}
 
-	state CheckpointFormat format = checkpoint.getFormat();
-	state std::unordered_set<std::string> dirs;
+	CheckpointFormat format = checkpoint.getFormat();
+	std::unordered_set<std::string> dirs;
 	if (format == DataMoveRocksCF) {
 		RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
 		TraceEvent(SevInfo, "DeleteRocksColumnFamilyCheckpoint", checkpoint.checkpointID)
@@ -1335,30 +1324,25 @@ ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
 		ASSERT(false);
 	}
 
-	state std::unordered_set<std::string>::iterator it = dirs.begin();
-	for (; it != dirs.end(); ++it) {
-		const std::string dir = *it;
+	for (const std::string& dir : dirs) {
 		platform::eraseDirectoryRecursive(dir);
 		TraceEvent(SevInfo, "DeleteCheckpointRemovedDir", checkpoint.checkpointID)
 		    .detail("CheckpointID", checkpoint.checkpointID)
 		    .detail("Dir", dir);
-		wait(delay(0, TaskPriority::FetchKeys));
+		co_await delay(0, TaskPriority::FetchKeys);
 	}
-
-	return Void();
 }
 #else
-ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
-                                                        CheckpointMetaData initialState,
-                                                        std::string dir,
-                                                        std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
-	wait(delay(0));
-	return initialState;
+Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
+                                                  CheckpointMetaData initialState,
+                                                  std::string dir,
+                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+	co_await delay(0);
+	co_return initialState;
 }
 
-ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
-	wait(delay(0));
-	return Void();
+Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint) {
+	co_await delay(0);
 }
 #endif // WITH_ROCKSDB
 
@@ -1453,4 +1437,4 @@ RocksDBCheckpointKeyValues getRocksKeyValuesCheckpoint(const CheckpointMetaData&
 	ObjectReader reader(checkpoint.getSerializedCheckpoint().begin(), IncludeVersion());
 	reader.deserialize(rocksCheckpoint);
 	return rocksCheckpoint;
-}
\ No newline at end of file
+}
diff --git a/fdbserver/core/SeedShardServers.cpp b/fdbserver/core/SeedShardServers.cpp
new file mode 100644
index 00000000000..4bb304bc27f
--- /dev/null
+++ b/fdbserver/core/SeedShardServers.cpp
@@ -0,0 +1,101 @@
+/*
+ * SeedShardServers.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <map>
+#include <vector>
+
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/SeedShardServers.h"
+#include "flow/Trace.h"
+
+void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vector<StorageServerInterface> servers) {
+	std::map<Optional<Value>, Tag> dcIdLocality;
+	std::map<UID, Tag> serverTag;
+	int8_t nextLocality = 0;
+	for (auto& server : servers) {
+		if (!dcIdLocality.contains(server.locality.dcId())) {
+			tr.set(arena, tagLocalityListKeyFor(server.locality.dcId()), tagLocalityListValue(nextLocality));
+			dcIdLocality[server.locality.dcId()] = Tag(nextLocality, 0);
+			nextLocality++;
+		}
+		Tag& tag = dcIdLocality[server.locality.dcId()];
+		serverTag[server.id()] = Tag(tag.locality, tag.id);
+		tag.id++;
+	}
+	std::sort(servers.begin(), servers.end());
+
+	// This isn't strictly necessary, but make sure this is the first transaction.
+	tr.read_snapshot = 0;
+	tr.read_conflict_ranges.push_back_deep(arena, allKeys);
+	KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
+	                                                                                     IncludeVersion());
+	StorageMetadataType metadata(StorageMetadataType::currentTime());
+
+	for (auto& server : servers) {
+		tr.set(arena, serverTagKeyFor(server.id()), serverTagValue(serverTag[server.id()]));
+		tr.set(arena, serverListKeyFor(server.id()), serverListValue(server));
+		tr.set(arena, metadataMap.packKey(server.id()), metadataMap.packValue(metadata));
+
+		if (SERVER_KNOBS->TSS_HACK_IDENTITY_MAPPING) {
+			TraceEvent(SevError, "TSSIdentityMappingEnabled").log();
+			Key uidRef = TupleCodec<UID>::pack(server.id());
+			tr.set(arena, uidRef.withPrefix(tssMappingKeys.begin), uidRef);
+		}
+	}
+	tr.set(arena, serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
+
+	std::vector<Tag> serverTags;
+	std::vector<UID> serverSrcIds;
+	serverTags.reserve(servers.size());
+	for (auto& server : servers) {
+		serverTags.push_back(serverTag[server.id()]);
+		serverSrcIds.push_back(server.id());
+	}
+
+	auto keyServers = CLIENT_KNOBS->TAG_ENCODE_KEY_SERVERS ? keyServersValue(serverTags)
+	                                                       : keyServersValue(RangeResult(), serverSrcIds);
+	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+		const UID shardId = newDataMoveId(deterministicRandom()->randomUInt64(),
+		                                  AssignEmptyRange(false),
+		                                  DataMoveType::LOGICAL,
+		                                  DataMovementReason::SEED_SHARD_SERVER,
+		                                  UnassignShard(false));
+		keyServers = keyServersValue(serverSrcIds, /*dest=*/std::vector<UID>(), shardId, UID());
+		krmSetPreviouslyEmptyRange(
+		    tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), keyServers, Value());
+
+		for (auto& server : servers) {
+			krmSetPreviouslyEmptyRange(
+			    tr, arena, serverKeysPrefixFor(server.id()), allKeys, serverKeysValue(shardId), serverKeysFalse);
+		}
+	} else {
+		krmSetPreviouslyEmptyRange(
+		    tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), keyServers, Value());
+
+		for (auto& server : servers) {
+			krmSetPreviouslyEmptyRange(
+			    tr, arena, serverKeysPrefixFor(server.id()), allKeys, serverKeysTrue, serverKeysFalse);
+		}
+	}
+}
diff --git a/fdbserver/ServerCheckpoint.actor.cpp b/fdbserver/core/ServerCheckpoint.cpp
similarity index 76%
rename from fdbserver/ServerCheckpoint.actor.cpp
rename to fdbserver/core/ServerCheckpoint.cpp
index a34fafab85d..39bf743aa58 100644
--- a/fdbserver/ServerCheckpoint.actor.cpp
+++ b/fdbserver/core/ServerCheckpoint.cpp
@@ -1,5 +1,5 @@
 /*
- *ServerCheckpoint.actor.cpp
+ *ServerCheckpoint.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/ServerCheckpoint.actor.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/core/ServerCheckpoint.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
 
 ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint,
                                        const CheckpointAsKeyValues checkpointAsKeyValues,
@@ -36,8 +34,8 @@ ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint,
 	return nullptr;
 }
 
-ACTOR Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint) {
-	wait(delay(0, TaskPriority::FetchKeys));
+Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint) {
+	co_await delay(0, TaskPriority::FetchKeys);
 	const CheckpointFormat format = checkpoint.getFormat();
 	if (format == DataMoveRocksCF || format == RocksDB || format == RocksDBKeyValues) {
 		if (!checkpoint.dir.empty()) {
@@ -48,40 +46,38 @@ ACTOR Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint) {
 	} else {
 		throw not_implemented();
 	}
-
-	return Void();
 }
 
-ACTOR Future<CheckpointMetaData> fetchCheckpoint(Database cx,
-                                                 CheckpointMetaData initialState,
-                                                 std::string dir,
-                                                 std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+Future<CheckpointMetaData> fetchCheckpoint(Database cx,
+                                           CheckpointMetaData initialState,
+                                           std::string dir,
+                                           std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
 	TraceEvent("FetchCheckpointBegin", initialState.checkpointID).detail("CheckpointMetaData", initialState.toString());
 
-	state CheckpointMetaData result;
+	CheckpointMetaData result;
 	const CheckpointFormat format = initialState.getFormat();
 	ASSERT(format != RocksDBKeyValues);
 	if (format == DataMoveRocksCF || format == RocksDB) {
-		wait(store(result, fetchRocksDBCheckpoint(cx, initialState, dir, cFun)));
+		result = co_await fetchRocksDBCheckpoint(cx, initialState, dir, cFun);
 	} else {
 		throw not_implemented();
 	}
 
 	TraceEvent("FetchCheckpointEnd", initialState.checkpointID).detail("CheckpointMetaData", result.toString());
-	return result;
+	co_return result;
 }
 
-ACTOR Future<CheckpointMetaData> fetchCheckpointRanges(Database cx,
-                                                       CheckpointMetaData initialState,
-                                                       std::string dir,
-                                                       std::vector<KeyRange> ranges,
-                                                       std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
+Future<CheckpointMetaData> fetchCheckpointRanges(Database cx,
+                                                 CheckpointMetaData initialState,
+                                                 std::string dir,
+                                                 std::vector<KeyRange> ranges,
+                                                 std::function<Future<Void>(const CheckpointMetaData&)> cFun) {
 	TraceEvent(SevDebug, "FetchCheckpointRangesBegin", initialState.checkpointID)
 	    .detail("CheckpointMetaData", initialState.toString())
 	    .detail("Ranges", describe(ranges));
 	ASSERT(!ranges.empty());
 
-	state CheckpointMetaData result;
+	CheckpointMetaData result;
 	const CheckpointFormat format = initialState.getFormat();
 	if (format != RocksDBKeyValues) {
 		if (format != DataMoveRocksCF) {
@@ -94,12 +90,12 @@ ACTOR Future<CheckpointMetaData> fetchCheckpointRanges(Database cx,
 		    ObjectWriter::toValue(RocksDBCheckpointKeyValues(ranges), IncludeVersion()));
 	}
 
-	wait(store(result, fetchRocksDBCheckpoint(cx, initialState, dir, cFun)));
+	result = co_await fetchRocksDBCheckpoint(cx, initialState, dir, cFun);
 
 	TraceEvent(SevDebug, "FetchCheckpointRangesEnd", initialState.checkpointID)
 	    .detail("CheckpointMetaData", result.toString())
 	    .detail("Ranges", describe(ranges));
-	return result;
+	co_return result;
 }
 
 std::string serverCheckpointDir(const std::string& baseDir, const UID& checkpointId) {
diff --git a/fdbclient/ServerKnobs.cpp b/fdbserver/core/ServerKnobs.cpp
similarity index 89%
rename from fdbclient/ServerKnobs.cpp
rename to fdbserver/core/ServerKnobs.cpp
index ce3d8ac6977..5c818726651 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbserver/core/ServerKnobs.cpp
@@ -18,15 +18,130 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ServerKnobs.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbclient/Knobs.h"
 #include "flow/IRandom.h"
 
+#include <iostream>
+
 #define init(knob, value) INIT_KNOB(knob, value)
 
 ServerKnobs::ServerKnobs(Randomize randomize, ClientKnobs* clientKnobs, IsSimulated isSimulated) {
 	initialize(randomize, clientKnobs, isSimulated);
 }
 
+namespace {
+FlowKnobs globalFlowKnobs(Randomize::False, IsSimulated::False);
+ClientKnobs globalClientKnobs(Randomize::False, IsSimulated::False);
+ServerKnobs globalServerKnobs(Randomize::False, &globalClientKnobs, IsSimulated::False);
+ClientKnobs bootstrapGlobalClientKnobs(Randomize::False, IsSimulated::False);
+ServerKnobs bootstrapGlobalServerKnobs(Randomize::False, &bootstrapGlobalClientKnobs, IsSimulated::False);
+
+FlowKnobs& mutableFlowKnobs() {
+	return *const_cast<FlowKnobs*>(FLOW_KNOBS);
+}
+
+ClientKnobs& mutableClientKnobs() {
+	return *const_cast<ClientKnobs*>(CLIENT_KNOBS);
+}
+
+ServerKnobs& mutableServerKnobs() {
+	return *const_cast<ServerKnobs*>(SERVER_KNOBS);
+}
+
+Optional<KnobValue> tryParseKnobValueImpl(std::string const& knobName, std::string const& knobValue) {
+	auto parsedKnobValue = FLOW_KNOBS->parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+
+	parsedKnobValue = CLIENT_KNOBS->parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+
+	parsedKnobValue = SERVER_KNOBS->parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+
+	return {};
+}
+} // namespace
+
+ServerKnobs const* SERVER_KNOBS = &bootstrapGlobalServerKnobs;
+
+void resetServerKnobs(Randomize randomize, IsSimulated isSimulated) {
+	FLOW_KNOBS = &globalFlowKnobs;
+	CLIENT_KNOBS = &globalClientKnobs;
+	SERVER_KNOBS = &globalServerKnobs;
+	globalFlowKnobs.reset(randomize, isSimulated);
+	globalClientKnobs.reset(randomize, isSimulated);
+	globalServerKnobs.reset(randomize, &globalClientKnobs, isSimulated);
+}
+
+void initializeServerKnobs(Randomize randomize, IsSimulated isSimulated) {
+	mutableFlowKnobs().initialize(randomize, isSimulated);
+	mutableClientKnobs().initialize(randomize, isSimulated);
+	mutableServerKnobs().initialize(randomize, &mutableClientKnobs(), isSimulated);
+}
+
+Optional<KnobValue> tryParseServerKnobValue(std::string const& knobName, std::string const& knobValue) {
+	return tryParseKnobValueImpl(knobName, knobValue);
+}
+
+KnobValue parseServerKnobValue(std::string const& knobName, std::string const& knobValue) {
+	auto result = tryParseServerKnobValue(knobName, knobValue);
+	if (!result.present()) {
+		throw invalid_option();
+	}
+	return result.get();
+}
+
+bool trySetServerKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	const bool setFlowKnob = knobValue.visitSetKnob(knobName, mutableFlowKnobs());
+	const bool setClientKnob = knobValue.visitSetKnob(knobName, mutableClientKnobs());
+	const bool setServerKnob = knobValue.visitSetKnob(knobName, mutableServerKnobs());
+	return setFlowKnob || setClientKnob || setServerKnob;
+}
+
+void setServerKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	if (trySetServerKnob(knobName, knobValue)) {
+		return;
+	}
+
+	TraceEvent(SevWarnAlways, "FailedToSetKnob").detail("KnobName", knobName).detail("KnobValue", knobValue.toString());
+	throw invalid_option_value();
+}
+
+void setupServerKnobs(std::vector<std::pair<std::string, std::string>> const& knobs) {
+	for (const auto& [knobName, knobValueString] : knobs) {
+		try {
+			setServerKnob(knobName, parseServerKnobValue(knobName, knobValueString));
+		} catch (Error& e) {
+			if (e.code() == error_code_invalid_option_value) {
+				std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName
+				          << "'\n";
+				TraceEvent(SevWarnAlways, "InvalidKnobValue")
+				    .detail("Knob", printable(knobName))
+				    .detail("Value", printable(knobValueString));
+			} else if (e.code() == error_code_invalid_option) {
+				std::cerr << "WARNING: Invalid knob option '" << knobName << "'\n";
+				TraceEvent(SevWarnAlways, "InvalidKnobName")
+				    .detail("Knob", printable(knobName))
+				    .detail("Value", printable(knobValueString));
+			} else {
+				std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n";
+				TraceEvent(SevError, "FailedToSetKnob")
+				    .errorUnsuppressed(e)
+				    .detail("Knob", printable(knobName))
+				    .detail("Value", printable(knobValueString));
+				throw e;
+			}
+		}
+	}
+}
+
 void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSimulated isSimulated) {
 	// clang-format off
 	init( ALLOW_DANGEROUS_KNOBS,                               isSimulated );
@@ -72,7 +187,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MAX_MESSAGE_SIZE,            std::max<int>(LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE, 1e5 + 2e4 + 1) + 8 ); // VALUE_SIZE_LIMIT + SYSTEM_KEY_SIZE_LIMIT + 9 bytes (4 bytes for length, 4 bytes for sequence number, and 1 byte for mutation type)
 	init( TLOG_MESSAGE_BLOCK_BYTES,                             10e6 );
 	init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR,      double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) ); //1.0121466709838096006362758832473
-	init( PEEK_TRACKER_EXPIRATION_TIME,                          600 ); if( randomize && BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = 120; // Cannot be buggified lower without changing the following assert in LogSystemPeekCursor.actor.cpp: ASSERT_WE_THINK(e.code() == error_code_operation_obsolete || SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME < 10);
+	init( PEEK_TRACKER_EXPIRATION_TIME,                          600 ); if( randomize && BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = 120; // Cannot be buggified lower without changing the following assert in LogSystemPeekCursor.cpp: ASSERT_WE_THINK(e.code() == error_code_operation_obsolete || SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME < 10);
 	init( PEEK_USING_STREAMING,                                false ); if( randomize && isSimulated && BUGGIFY ) PEEK_USING_STREAMING = true;
 	init( PARALLEL_GET_MORE_REQUESTS,                             32 ); if( randomize && BUGGIFY ) PARALLEL_GET_MORE_REQUESTS = 2;
 	init( MULTI_CURSOR_PRE_FETCH_LIMIT,                           10 );
@@ -149,6 +264,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MERGE_RELOCATION_PARALLELISM_PER_TEAM,                   6 ); if (randomize && BUGGIFY ) MERGE_RELOCATION_PARALLELISM_PER_TEAM = 1;
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); // Do not buggify
 	init( DD_REBALANCE_PARALLELISM,                               50 );
+	// Hard cap on total relocations DD tracks (queued + in-flight). 1000 corresponds to a 500-server
+	// cluster with two concurrent shard moves per storage server.  We have observed large clusters doing
+	// 25-30GB in flight, or closer to 100 shards at a time, so this has plenty of margin of safety
+	// built in.
+	init( DD_MAX_PIPELINE_MOVES,                                1000 ); if( randomize && BUGGIFY ) DD_MAX_PIPELINE_MOVES = 5;
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
 	init( INFLIGHT_PENALTY_HEALTHY,                              1.0 );
 	init( INFLIGHT_PENALTY_UNHEALTHY,                          500.0 );
@@ -329,9 +449,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DEBOUNCE_RECRUITING_DELAY,                             5.0 );
 	init( DD_FAILURE_TIME,                                       1.0 ); if( randomize && BUGGIFY ) DD_FAILURE_TIME = 10.0;
 	init( DD_ZERO_HEALTHY_TEAM_DELAY,                            1.0 );
-	init( REMOTE_KV_STORE,                                     false );
 	init( REBOOT_KV_STORE_DELAY,                                 0.1 );
-	init( REMOTE_KV_STORE_MAX_INIT_DURATION,                    10.0 );
 	init( REBALANCE_MAX_RETRIES,                                 100 );
 	init( DD_OVERLAP_PENALTY,                                  10000 );
 	init( DD_EXCLUDE_MIN_REPLICAS,                                 1 );
@@ -526,9 +644,12 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME,             30.0 ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME = 0.1;
 	init( ROCKSDB_PROBABILITY_REUSE_ITERATOR_SIM,               0.01 );
 	init( ROCKSDB_READ_RANGE_REUSE_ITERATORS,                  false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom()->coinflip();
-	init( SHARDED_ROCKSDB_REUSE_ITERATORS,                     false ); if (isSimulated) SHARDED_ROCKSDB_REUSE_ITERATORS = deterministicRandom()->coinflip(); 
+	init( SHARDED_ROCKSDB_REUSE_ITERATORS,                     false ); if( isSimulated ) SHARDED_ROCKSDB_REUSE_ITERATORS = deterministicRandom()->coinflip();
 	init( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS,          false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom()->coinflip();
 	init( ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT,        200 );
+	init( ROCKSDB_USE_CACHE_RESULT_OPTION,                     false ); if( isSimulated ) ROCKSDB_USE_CACHE_RESULT_OPTION = deterministicRandom()->coinflip();
+	// Probability that RocksDB can disable block cache in simulation
+	init( ROCKSDB_PROBABILITY_DISABLE_CACHE_SIM,                 0.1 );
 	// Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second.
 	init( ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC,        200000000 );
 	init( ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS,                    10 ); // RocksDB default 10
@@ -781,7 +902,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	// Backup Worker
 	init( BACKUP_TIMEOUT,                                        0.4 );
-	init( BACKUP_NOOP_POP_DELAY,                                 5.0 );
 	init( BACKUP_FILE_BLOCK_BYTES,                       1024 * 1024 );
 	init( BACKUP_WORKER_LOCK_BYTES,                              3e9 ); if(randomize && BUGGIFY) BACKUP_WORKER_LOCK_BYTES = deterministicRandom()->randomInt(2048, 4096) * 4096;
 	init( BACKUP_UPLOAD_DELAY,                                  10.0 ); if(randomize && BUGGIFY) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 60;
@@ -804,7 +924,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( WAIT_FOR_RATEKEEPER_JOIN_DELAY,                        1.0 );
 	init( WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY,                   1.0 );
 	init( WAIT_FOR_BLOB_MANAGER_JOIN_DELAY,                      1.0 );
-	init( WAIT_FOR_ENCRYPT_KEY_PROXY_JOIN_DELAY,                 1.0 );
 	init( WORKER_FAILURE_TIME,                                   1.0 ); if( randomize && BUGGIFY ) WORKER_FAILURE_TIME = 10.0;
 	init( CHECK_OUTSTANDING_INTERVAL,                            0.5 ); if( randomize && BUGGIFY ) CHECK_OUTSTANDING_INTERVAL = 0.001;
 	init( VERSION_LAG_METRIC_INTERVAL,                           0.5 ); if( randomize && BUGGIFY ) VERSION_LAG_METRIC_INTERVAL = 10.0;
@@ -824,7 +943,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CC_WORKER_HEALTH_CHECKING_INTERVAL,                   60.0 );
 	init( CC_DEGRADED_LINK_EXPIRATION_INTERVAL,                300.0 );
 	init( CC_MIN_DEGRADATION_INTERVAL,                         120.0 );
-	init( ENCRYPT_KEY_PROXY_FAILURE_TIME,                        0.1 ); if ( isSimulated ) ENCRYPT_KEY_PROXY_FAILURE_TIME = 1.0 + deterministicRandom()->random01();
 	init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE,                      3 );
 	init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE_MIN,                  1 );
 	init( CC_MAX_EXCLUSION_DUE_TO_HEALTH,                          2 );
@@ -834,6 +952,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CC_HEALTH_TRIGGER_FAILOVER,                          false );
 	init( CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION,               5 );
 	init( CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION,              10 );
+	init( CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION,                 0.0 ); if (randomize && BUGGIFY) CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION = 10.0;
 	init( CC_ENABLE_ENTIRE_SATELLITE_MONITORING,               false );
 	init( CC_SATELLITE_DEGRADATION_MIN_COMPLAINER,                 3 );
 	init( CC_SATELLITE_DEGRADATION_MIN_BAD_SERVER,                 3 );
@@ -850,6 +969,13 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CC_RECOVERY_INIT_REQ_GROWTH_FACTOR,                    2.0 );
 	init( CC_RECOVERY_INIT_REQ_MAX_TIMEOUT,                    300.0 );
 	init( CC_RECOVERY_INIT_REQ_MAX_UNFINISHED_RECOVERIES,        100 );
+	init( CLUSTER_HEALTH_METRIC_ENABLE,                        false ); if( randomize && isSimulated && BUGGIFY ) CLUSTER_HEALTH_METRIC_ENABLE = true;
+	init( CLUSTER_HEALTH_METRIC_POLL_INTERVAL, 5.0 );
+	init( CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD, 0.20 );
+	init( CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD,     0.10 );
+	init( CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD,    0.20 );
+	init( CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD,        0.10 );
+	init( CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD, 1.2 );
 
 	init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL,                   600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
 	init( EXPECTED_MASTER_FITNESS,            ProcessClass::UnsetFit );
@@ -876,6 +1002,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SERVER_READY_QUORUM_TIMEOUT,                          15.0 ); if( randomize && BUGGIFY ) SERVER_READY_QUORUM_TIMEOUT = 1.0;
 	init( REMOVE_RETRY_DELAY,                                    1.0 );
 	init( MOVE_KEYS_KRM_LIMIT,                                  2000 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT = 2;
+	init( FINISH_MOVE_KEYS_MAX_RETRIES,                           50 ); // ~4 min total with exponential backoff (0.2s to 5s cap)
 	init( MOVE_KEYS_KRM_LIMIT_BYTES,                             1e5 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT_BYTES = 5e4; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map
 	init( MOVE_SHARD_KRM_ROW_LIMIT,                            20000 );
  	init( MOVE_SHARD_KRM_BYTE_LIMIT,                             1e6 );
@@ -936,9 +1063,12 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SPRING_BYTES_TLOG,                                   400e6 ); if( smallTlogTarget ) SPRING_BYTES_TLOG = 200e3;
 	init( TARGET_BYTES_PER_TLOG_BATCH,                        1400e6 ); if( smallTlogTarget ) TARGET_BYTES_PER_TLOG_BATCH = 1400e3;
 	init( SPRING_BYTES_TLOG_BATCH,                             300e6 ); if( smallTlogTarget ) SPRING_BYTES_TLOG_BATCH = 150e3;
+	// Match MIN_AVAILABLE_SPACE_RATIO by default; buggified simulations exercise the earlier ramp.
+	init( TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO,            0.05 ); if( randomize && isSimulated && BUGGIFY ) TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO = 0.20;
 	init( TLOG_SPILL_THRESHOLD,                               1500e6 ); if( smallTlogTarget ) TLOG_SPILL_THRESHOLD = 1500e3; if( randomize && BUGGIFY ) TLOG_SPILL_THRESHOLD = 0;
 	init( REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT,            20e6 ); if( (randomize && BUGGIFY) || smallTlogTarget ) REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT = 1e6;
 	init( TLOG_HARD_LIMIT_BYTES,                              3000e6 ); if( smallTlogTarget ) TLOG_HARD_LIMIT_BYTES = 30e6;
+	init( TLOG_MIN_AVAILABLE_SPACE_RATIO,                        0.0 ); if( randomize && BUGGIFY ) TLOG_MIN_AVAILABLE_SPACE_RATIO = 0.1;
 	init( TLOG_RECOVER_MEMORY_LIMIT, TARGET_BYTES_PER_TLOG + SPRING_BYTES_TLOG );
 
 	init( MAX_TRANSACTIONS_PER_BYTE,                            1000 );
@@ -962,20 +1092,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DURABILITY_LAG_REDUCTION_RATE,                      0.9999 );
 	init( DURABILITY_LAG_INCREASE_RATE,                        1.001 );
 	init( STORAGE_SERVER_LIST_FETCH_TIMEOUT,                    20.0 );
-	init( BW_THROTTLING_ENABLED,                                true );
-
-	bool buggifySmallBWLag = randomize && BUGGIFY;
-	init( TARGET_BW_LAG,                                       90.0 ); if(buggifySmallBWLag) TARGET_BW_LAG = 10.0;
-	init( TARGET_BW_LAG_BATCH,                                 60.0 ); if(buggifySmallBWLag) TARGET_BW_LAG_BATCH = 4.0;
-	init( TARGET_BW_LAG_UPDATE,                                  9.0 ); if(buggifySmallBWLag) TARGET_BW_LAG_UPDATE = 1.0;
-	init( MIN_BW_HISTORY,                                         10 );
-	init( BW_ESTIMATION_INTERVAL,                               10.0 ); if(buggifySmallBWLag) BW_ESTIMATION_INTERVAL = 2.0;
-	init( BW_LAG_INCREASE_AMOUNT,                                1.1 );
-	init( BW_LAG_DECREASE_AMOUNT,                                0.9 );
-	init( BW_FETCH_WORKERS_INTERVAL,                             5.0 );
-	init( BW_RW_LOGGING_INTERVAL,                                5.0 );
-	init( BW_MAX_BLOCKED_INTERVAL,                              10.0 ); if(buggifySmallBWLag) BW_MAX_BLOCKED_INTERVAL = 2.0;
-	init( BW_RK_SIM_QUIESCE_DELAY,                             400.0 );
 
 	init( MAX_AUTO_THROTTLED_TRANSACTION_TAGS,                     5 ); if(randomize && BUGGIFY) MAX_AUTO_THROTTLED_TRANSACTION_TAGS = 1;
 	init( MAX_MANUAL_THROTTLED_TRANSACTION_TAGS,                  40 ); if(randomize && BUGGIFY) MAX_MANUAL_THROTTLED_TRANSACTION_TAGS = 1;
@@ -989,22 +1105,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL,                30.0 ); if(randomize && BUGGIFY) TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL = 1.0;
 	init( AUTO_TAG_THROTTLING_ENABLED,                          true ); if(randomize && BUGGIFY) AUTO_TAG_THROTTLING_ENABLED = false;
 	init( SS_THROTTLE_TAGS_TRACKED,                                5 ); if(randomize && BUGGIFY) SS_THROTTLE_TAGS_TRACKED = deterministicRandom()->randomInt(1, 10);
-	init( GLOBAL_TAG_THROTTLING,                                true ); if(isSimulated) GLOBAL_TAG_THROTTLING = deterministicRandom()->coinflip();
-	init( ENFORCE_TAG_THROTTLING_ON_PROXIES,   GLOBAL_TAG_THROTTLING );
-	init( GLOBAL_TAG_THROTTLING_MIN_RATE,                        1.0 );
-	init( GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED,                 10 );
-	init( GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER,              240.0 );
-	init( GLOBAL_TAG_THROTTLING_PROXY_LOGGING_INTERVAL,         60.0 );
-	init( GLOBAL_TAG_THROTTLING_TRACE_INTERVAL,                  5.0 );
-	init( GLOBAL_TAG_THROTTLING_REPORT_ONLY,                   false );
-	init( GLOBAL_TAG_THROTTLING_FORGET_SS_THRESHOLD,            4096 );
-	init( GLOBAL_TAG_THROTTLING_LIMITING_THRESHOLD,         409600.0 );
-
-	init( GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME,        10.0 );
-	init( GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME,   2.0 );
-	init( GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME,   10.0 );
-	init( GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME,               10.0 );
-
 	init( HOT_SHARD_THROTTLING_ENABLED,                         false ); if(randomize && BUGGIFY) HOT_SHARD_THROTTLING_ENABLED = true;
 	init( HOT_SHARD_THROTTLING_EXPIRE_AFTER,                      3.0 );
 	init( HOT_SHARD_THROTTLING_TRACKED,                             1 );
@@ -1044,11 +1144,17 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PERSIST_FINISH_AUDIT_COUNT,                             10 ); if ( isSimulated ) PERSIST_FINISH_AUDIT_COUNT = deterministicRandom()->randomInt(1, PERSIST_FINISH_AUDIT_COUNT+1);
 	init( AUDIT_RETRY_COUNT_MAX,                               10000 ); if ( isSimulated ) AUDIT_RETRY_COUNT_MAX = 10;
 	init( CONCURRENT_AUDIT_TASK_COUNT_MAX,                        20 ); if ( isSimulated ) CONCURRENT_AUDIT_TASK_COUNT_MAX = deterministicRandom()->randomInt(1, CONCURRENT_AUDIT_TASK_COUNT_MAX+1);
+	// Note that the following two checks don't do anything unless SHARD_ENCODE_LOCATION_METADATA is enabled.
 	init( AUDIT_DATAMOVE_PRE_CHECK,                            false ); if ( isSimulated ) AUDIT_DATAMOVE_PRE_CHECK = true;
 	init( AUDIT_DATAMOVE_POST_CHECK,                           false ); if ( isSimulated ) AUDIT_DATAMOVE_POST_CHECK = true;
 	init( AUDIT_DATAMOVE_POST_CHECK_RETRY_COUNT_MAX,              50 );
 	init( AUDIT_STORAGE_RATE_PER_SERVER_MAX,                    50e6 ); // per second
+	init( AUDIT_RESTORE_BATCH_KEY_LIMIT,                      100000 ); // 100K keys per batch (was hardcoded 10K)
+	init( AUDIT_PROGRESS_PERSIST_BYTES_INTERVAL,           100000000 ); // 100MB - only persist progress after this many bytes
 	init( ENABLE_AUDIT_VERBOSE_TRACE,                          false );
+	// Disabled in simulation: audit_storage locationmetadata already runs at controlled times in sim,
+	// and periodic triggering interferes with DD quiescence checks and the test harness audit checks.
+	init( AUDIT_LOCATION_METADATA_INTERVAL,                  3600.0 ); if ( isSimulated ) AUDIT_LOCATION_METADATA_INTERVAL = 0;
 	init( LOGGING_STORAGE_COMMIT_WHEN_IO_TIMEOUT,               true );
 	init( LOGGING_RECENT_STORAGE_COMMIT_SIZE,                     20 );
 	init( LOGGING_COMPLETE_STORAGE_COMMIT_PROBABILITY,         0.001 );
@@ -1124,6 +1230,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( FETCH_SHARD_BUFFER_BYTE_LIMIT,                        20e6 ); if( randomize && BUGGIFY ) FETCH_SHARD_BUFFER_BYTE_LIMIT = 1;
 	init( FETCH_SHARD_UPDATES_BYTE_LIMIT,                    2500000 ); if( randomize && BUGGIFY ) FETCH_SHARD_UPDATES_BYTE_LIMIT = 100;
 	init( TRACK_READ_LATENCIES_PER_TYPE,                       false ); if( randomize && BUGGIFY ) TRACK_READ_LATENCIES_PER_TYPE = true;
+	init( STORAGE_UPDATE_PROCESS_STATS_INTERVAL,                   5 ); if( randomize && BUGGIFY ) STORAGE_UPDATE_PROCESS_STATS_INTERVAL = deterministicRandom()->random01() * 60 + 1;
 
 	//Wait Failure
 	init( MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS,                 250 ); if( randomize && BUGGIFY ) MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS = 2;
@@ -1186,6 +1293,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// Status
 	init( STATUS_MIN_TIME_BETWEEN_REQUESTS,                      0.0 );
 	init( MAX_STATUS_REQUESTS_PER_SECOND,                      256.0 );
+	init( STATUS_TIMEOUT_BUFFER,                                 5.0 );
 	init( CONFIGURATION_ROWS_TO_FETCH,                         20000 );
 	init( DISABLE_DUPLICATE_LOG_WARNING,                       false );
 	init( HISTOGRAM_REPORT_INTERVAL,                           300.0 );
@@ -1195,52 +1303,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( TIME_KEEPER_DELAY,                                      10 );
 	init( TIME_KEEPER_MAX_ENTRIES,                3600 * 24 * 30 * 6 ); if( randomize && BUGGIFY ) { TIME_KEEPER_MAX_ENTRIES = 2; }
 
-	// Fast Restore
-	init( FASTRESTORE_FAILURE_TIMEOUT,                          3600 );
-	init( FASTRESTORE_HEARTBEAT_INTERVAL,                         60 );
-	init( FASTRESTORE_SAMPLING_PERCENT,                          100 ); if( randomize && BUGGIFY ) { FASTRESTORE_SAMPLING_PERCENT = deterministicRandom()->random01() * 100; }
-	init( FASTRESTORE_NUM_LOADERS,                                 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom()->random01() * 10 + 1; }
-	init( FASTRESTORE_NUM_APPLIERS,                                3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom()->random01() * 10 + 1; }
-	init( FASTRESTORE_TXN_BATCH_MAX_BYTES,           1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom()->random01() * 1024.0 * 1024.0 + 1.0; }
-	init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() < 0.2 ? 50 * 1024 : deterministicRandom()->random01() < 0.4 ? 100 * 1024 * 1024 : deterministicRandom()->random01() * 1000.0 * 1024.0 * 1024.0; } // too small value may increase chance of TooManyFile error
-	init( FASTRESTORE_VB_PARALLELISM,                              5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom()->random01() < 0.2 ? 2 : deterministicRandom()->random01() * 10 + 1; }
-	init( FASTRESTORE_VB_MONITOR_DELAY,                           30 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom()->random01() * 20 + 1; }
-	init( FASTRESTORE_VB_LAUNCH_DELAY,                           1.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom()->random01() < 0.2 ? 0.1 : deterministicRandom()->random01() * 10.0 + 1; }
-	init( FASTRESTORE_ROLE_LOGGING_DELAY,                          5 ); if( randomize && BUGGIFY ) { FASTRESTORE_ROLE_LOGGING_DELAY = deterministicRandom()->random01() * 60 + 1; }
-	init( FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL,               5 ); if( randomize && BUGGIFY ) { FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL = deterministicRandom()->random01() * 60 + 1; }
-	init( FASTRESTORE_MONITOR_LEADER_DELAY,                        5 ); if( randomize && BUGGIFY ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom()->random01() * 100; }
-	init( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS,                60 ); if( randomize && BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom()->random01() * 240 + 10; }
-	init( FASTRESTORE_TRACK_REQUEST_LATENCY,              	   false ); if( randomize && BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false; }
-	init( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT,                 6144 ); if( randomize && BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1; }
-	init( FASTRESTORE_WAIT_FOR_MEMORY_LATENCY,                    10 ); if( randomize && BUGGIFY ) { FASTRESTORE_WAIT_FOR_MEMORY_LATENCY = 60; }
-	init( FASTRESTORE_HEARTBEAT_DELAY,                            10 ); if( randomize && BUGGIFY ) { FASTRESTORE_HEARTBEAT_DELAY = deterministicRandom()->random01() * 120 + 2; }
-	init( FASTRESTORE_HEARTBEAT_MAX_DELAY,                        10 ); if( randomize && BUGGIFY ) { FASTRESTORE_HEARTBEAT_MAX_DELAY = FASTRESTORE_HEARTBEAT_DELAY * 10; }
-	init( FASTRESTORE_APPLIER_FETCH_KEYS_SIZE,                   100 ); if( randomize && BUGGIFY ) { FASTRESTORE_APPLIER_FETCH_KEYS_SIZE = deterministicRandom()->random01() * 10240 + 1; }
-	init( FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES, 1.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES = deterministicRandom()->random01() < 0.2 ? 1024 : deterministicRandom()->random01() * 5.0 * 1024.0 * 1024.0 + 1; }
-	init( FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE,            false ); if( randomize && BUGGIFY ) { FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE = deterministicRandom()->random01() < 0.5 ? true : false; }
-	init( FASTRESTORE_REQBATCH_PARALLEL,                          50 ); if( randomize && BUGGIFY ) { FASTRESTORE_REQBATCH_PARALLEL = deterministicRandom()->random01() * 100 + 1; }
-	init( FASTRESTORE_REQBATCH_LOG,                            false ); if( randomize && BUGGIFY ) { FASTRESTORE_REQBATCH_LOG = deterministicRandom()->random01() < 0.2 ? true : false; }
-	init( FASTRESTORE_TXN_CLEAR_MAX,                             100 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_CLEAR_MAX = deterministicRandom()->random01() * 100 + 1; }
-	init( FASTRESTORE_TXN_RETRY_MAX,                              10 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_RETRY_MAX = deterministicRandom()->random01() * 100 + 1; }
-	init( FASTRESTORE_TXN_EXTRA_DELAY,                           0.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_EXTRA_DELAY = deterministicRandom()->random01() * 1 + 0.001;}
-	init( FASTRESTORE_NOT_WRITE_DB,                            false ); // Perf test only: set it to true will cause simulation failure
-	init( FASTRESTORE_USE_RANGE_FILE,                           true ); // Perf test only: set it to false will cause simulation failure
-	init( FASTRESTORE_USE_LOG_FILE,                             true ); // Perf test only: set it to false will cause simulation failure
-	init( FASTRESTORE_SAMPLE_MSG_BYTES,                      1048576 ); if( randomize && BUGGIFY ) { FASTRESTORE_SAMPLE_MSG_BYTES = deterministicRandom()->random01() * 2048;}
-	init( FASTRESTORE_SCHED_UPDATE_DELAY,                        0.1 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_UPDATE_DELAY = deterministicRandom()->random01() * 2;}
-	init( FASTRESTORE_SCHED_TARGET_CPU_PERCENT,                   70 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_TARGET_CPU_PERCENT = deterministicRandom()->random01() * 100 + 50;} // simulate cpu usage can be larger than 100
-	init( FASTRESTORE_SCHED_MAX_CPU_PERCENT,                      90 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_MAX_CPU_PERCENT = FASTRESTORE_SCHED_TARGET_CPU_PERCENT + deterministicRandom()->random01() * 100;}
-	init( FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS,                   50 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 30 + 1;}
-	init( FASTRESTORE_SCHED_INFLIGHT_SEND_REQS,                    3 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_SEND_REQS = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 10 + 1;}
-	init( FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE,                    5 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 10 + 1;}
-	init( FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD,         10 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 15 + 1;}
-	init( FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH,             2 ); if( randomize && BUGGIFY ) { FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 15 + 1;}
-	init( FASTRESTORE_NUM_TRACE_EVENTS,                          100 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_TRACE_EVENTS = deterministicRandom()->random01() < 0.2 ? 1 : deterministicRandom()->random01() * 500 + 1;}
-	init( FASTRESTORE_EXPENSIVE_VALIDATION,                    false ); if( randomize && BUGGIFY ) { FASTRESTORE_EXPENSIVE_VALIDATION = deterministicRandom()->random01() < 0.5 ? true : false;}
-	init( FASTRESTORE_WRITE_BW_MB,                                70 ); if( randomize && BUGGIFY ) { FASTRESTORE_WRITE_BW_MB = deterministicRandom()->random01() < 0.5 ? 2 : 100;}
-	init( FASTRESTORE_RATE_UPDATE_SECONDS,                       1.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_RATE_UPDATE_SECONDS = deterministicRandom()->random01() < 0.5 ? 0.1 : 2;}
-	init( FASTRESTORE_DUMP_INSERT_RANGE_VERSION,               false );
-
 	init( REDWOOD_DEFAULT_PAGE_SIZE,                            8192 );
 	init( REDWOOD_DEFAULT_EXTENT_SIZE,              32 * 1024 * 1024 );
 	init( REDWOOD_DEFAULT_EXTENT_READ_SIZE,              1024 * 1024 );
@@ -1277,41 +1339,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ENCRYPTION_LOGGING_INTERVAL,                           5.0 );
 	init( DISABLED_ENCRYPTION_PROBABILITY_SIM,                  0.95 );
 
-	// KMS connector type
-	init( KMS_CONNECTOR_TYPE,                     "RESTKmsConnector" );
-
 	// Blob Metadata
 	init( BLOB_METADATA_CACHE_TTL,  isSimulated ? 120 : 24 * 60 * 60 );
 	if ( randomize && BUGGIFY) { BLOB_METADATA_CACHE_TTL = deterministicRandom()->randomInt(50, 100); }
 
-	// HTTP KMS Connector
-	init( REST_KMS_CONNECTOR_KMS_DISCOVERY_URL_MODE,           "file");
-	init( REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE,            "file");
-	init( REST_KMS_CONNECTOR_VALIDATION_TOKEN_MAX_SIZE,          1024);
-	init( REST_KMS_CONNECTOR_VALIDATION_TOKENS_MAX_PAYLOAD_SIZE, 10 * 1024);
-	init( REST_KMS_CONNECTOR_REFRESH_KMS_URLS,                   true);
-	init( REST_KMS_CONNECTOR_REFRESH_KMS_URLS_INTERVAL_SEC,       600);
-	// Below KMS configurations are responsible for:
-	// Discovering KMS URLs, fetch encryption keys endpoint and validation token details.
-	// Configurations are expected to be passed as command-line arguments.
-	// NOTE: Care must be taken when attempting to update below configurations for a up/running FDB cluster.
-	init( REST_KMS_CONNECTOR_DISCOVER_KMS_URL_FILE,                "");
-	init( REST_KMS_CONNECTOR_GET_ENCRYPTION_KEYS_ENDPOINT,         "");
-	init( REST_KMS_CONNECTOR_GET_LATEST_ENCRYPTION_KEYS_ENDPOINT,  "");
-	init( REST_KMS_CONNECTOR_GET_BLOB_METADATA_ENDPOINT,           "");
-	// Details to fetch validation token from a localhost file
-	// acceptable format: "<token_name1>$<absolute_file_path1>,<token_name2>$<absolute_file_path2>,.."
-	// NOTE: 'token-name" can NOT contain '$' character
-	init( REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS,             "");
-	init( ENABLE_REST_KMS_COMMUNICATION,                        false); if( randomize && BUGGIFY ) ENABLE_REST_KMS_COMMUNICATION = true;
-	init( REST_KMS_CONNECTOR_REMOVE_TRAILING_NEWLINE,           false);
-	init( REST_KMS_CURRENT_BLOB_METADATA_REQUEST_VERSION,           1);
-	init( REST_KMS_MAX_BLOB_METADATA_REQUEST_VERSION,               1);
-	init( REST_KMS_CURRENT_CIPHER_REQUEST_VERSION,                  1);
-	init( REST_KMS_MAX_CIPHER_REQUEST_VERSION,                      1);
-	init( REST_SIM_KMS_VAULT_DIR,                                  "");
-	init( REST_KMS_STABILITY_CHECK_INTERVAL,                      5.0);
-
 	init( CONSISTENCY_SCAN_ACTIVE_THROTTLE_RATIO,                0.5 ); if( randomize && BUGGIFY ) CONSISTENCY_SCAN_ACTIVE_THROTTLE_RATIO = deterministicRandom()->random01();
 
 
diff --git a/fdbserver/core/ShardSizing.cpp b/fdbserver/core/ShardSizing.cpp
new file mode 100644
index 00000000000..02e51d7b869
--- /dev/null
+++ b/fdbserver/core/ShardSizing.cpp
@@ -0,0 +1,102 @@
+/*
+ * ShardSizing.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/ShardSizing.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "fdbclient/SystemData.h"
+#include "fdbserver/core/Knobs.h"
+#include "flow/Trace.h"
+
+ShardSizeBounds ShardSizeBounds::shardSizeBoundsBeforeTrack() {
+	return ShardSizeBounds{ .max = StorageMetrics{ .bytes = -1,
+		                                           .bytesWrittenPerKSecond = StorageMetrics::infinity,
+		                                           .iosPerKSecond = StorageMetrics::infinity,
+		                                           .bytesReadPerKSecond = StorageMetrics::infinity,
+		                                           .opsReadPerKSecond = StorageMetrics::infinity },
+		                    .min = StorageMetrics{ .bytes = -1,
+		                                           .bytesWrittenPerKSecond = 0,
+		                                           .iosPerKSecond = 0,
+		                                           .bytesReadPerKSecond = 0,
+		                                           .opsReadPerKSecond = 0 },
+		                    .permittedError = StorageMetrics{ .bytes = -1,
+		                                                      .bytesWrittenPerKSecond = StorageMetrics::infinity,
+		                                                      .iosPerKSecond = StorageMetrics::infinity,
+		                                                      .bytesReadPerKSecond = StorageMetrics::infinity,
+		                                                      .opsReadPerKSecond = StorageMetrics::infinity } };
+}
+
+ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
+	ShardSizeBounds bounds;
+
+	if (shard.begin >= keyServersKeys.begin) {
+		bounds.max.bytes = SERVER_KNOBS->KEY_SERVER_SHARD_BYTES;
+	} else {
+		bounds.max.bytes = maxShardSize;
+	}
+
+	bounds.max.bytesWrittenPerKSecond = bounds.max.infinity;
+	bounds.max.iosPerKSecond = bounds.max.infinity;
+	bounds.max.bytesReadPerKSecond = bounds.max.infinity;
+	bounds.max.opsReadPerKSecond = bounds.max.infinity;
+
+	if (shard.begin == allKeys.begin) {
+		bounds.min.bytes = 0;
+	} else {
+		bounds.min.bytes = maxShardSize / SERVER_KNOBS->SHARD_BYTES_RATIO;
+	}
+
+	bounds.min.bytesWrittenPerKSecond = 0;
+	bounds.min.iosPerKSecond = 0;
+	bounds.min.bytesReadPerKSecond = 0;
+	bounds.min.opsReadPerKSecond = 0;
+
+	bounds.permittedError.bytes = bounds.max.bytes / SERVER_KNOBS->SHARD_BYTES_RATIO / 3;
+	bounds.permittedError.bytesWrittenPerKSecond = bounds.permittedError.infinity;
+	bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
+	bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
+	bounds.permittedError.opsReadPerKSecond = bounds.permittedError.infinity;
+
+	return bounds;
+}
+
+int64_t getMaxShardSize(double dbSizeEstimate) {
+	int64_t size = std::min((SERVER_KNOBS->MIN_SHARD_BYTES + (int64_t)std::sqrt(std::max<double>(dbSizeEstimate, 0)) *
+	                                                             SERVER_KNOBS->SHARD_BYTES_PER_SQRT_BYTES) *
+	                            SERVER_KNOBS->SHARD_BYTES_RATIO,
+	                        (int64_t)SERVER_KNOBS->MAX_SHARD_BYTES);
+	if (SERVER_KNOBS->ALLOW_LARGE_SHARD) {
+		size = std::max(size, static_cast<int64_t>(SERVER_KNOBS->MAX_LARGE_SHARD_BYTES));
+	}
+
+	TraceEvent("MaxShardSize")
+	    .suppressFor(60.0)
+	    .detail("Bytes", size)
+	    .detail("EstimatedDbSize", dbSizeEstimate)
+	    .detail("SqrtBytes", SERVER_KNOBS->SHARD_BYTES_PER_SQRT_BYTES)
+	    .detail("AllowLargeShard", SERVER_KNOBS->ALLOW_LARGE_SHARD);
+	return size;
+}
+
+bool ddLargeTeamEnabled() {
+	return SERVER_KNOBS->DD_MAX_SHARDS_ON_LARGE_TEAMS > 0 && !SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA;
+}
diff --git a/fdbserver/core/SimulatorSupport.cpp b/fdbserver/core/SimulatorSupport.cpp
new file mode 100644
index 00000000000..7d1bb302c99
--- /dev/null
+++ b/fdbserver/core/SimulatorSupport.cpp
@@ -0,0 +1,7 @@
+#include "fdbrpc/simulator.h"
+#include "fdbrpc/SimulatorProcessInfo.h"
+#include "flow/network.h"
+
+bool isSimulatorProcessUnreliable() {
+	return g_network->isSimulated() && !g_simulator->getCurrentProcess()->isReliable();
+}
diff --git a/fdbserver/StorageMetrics.actor.cpp b/fdbserver/core/StorageMetrics.cpp
similarity index 85%
rename from fdbserver/StorageMetrics.actor.cpp
rename to fdbserver/core/StorageMetrics.cpp
index 73da5e18abc..5658a54e58d 100644
--- a/fdbserver/StorageMetrics.actor.cpp
+++ b/fdbserver/core/StorageMetrics.cpp
@@ -1,5 +1,5 @@
 /*
- * StorageMetrics.actor.cpp
+ * StorageMetrics.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,9 +18,44 @@
  * limitations under the License.
  */
 
+#include <algorithm>
+
+#include "fdbserver/core/StorageMetrics.h"
+#include "flow/CodeProbe.h"
+#include "flow/Hash3.h"
+#include "flow/IRandom.h"
+#include "flow/Trace.h"
 #include "flow/UnitTest.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
+#include "flow/flow.h"
+
+namespace {
+
+struct WaitMetricsMapStats {
+	int64_t rangeCount = 0;
+	int64_t nonEmptyRangeCount = 0;
+	int64_t watcherCount = 0;
+	int64_t watcherCapacity = 0;
+	int64_t maxWatchersPerRange = 0;
+};
+
+WaitMetricsMapStats summarizeWaitMetricsMap(
+    const KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>>& waitMetricsMap) {
+	WaitMetricsMapStats stats;
+	for (auto it = waitMetricsMap.ranges().begin(); it != waitMetricsMap.ranges().end(); ++it) {
+		++stats.rangeCount;
+		const auto& watchers = it->cvalue();
+		if (!watchers.empty()) {
+			++stats.nonEmptyRangeCount;
+		}
+		stats.watcherCount += watchers.size();
+		stats.watcherCapacity += watchers.capacity();
+		stats.maxWatchersPerRange = std::max<int64_t>(stats.maxWatchersPerRange, watchers.size());
+	}
+	return stats;
+}
+
+} // namespace
 
 CommonStorageCounters::CommonStorageCounters(const std::string& name,
                                              const std::string& id,
@@ -39,6 +74,33 @@ CommonStorageCounters::CommonStorageCounters(const std::string& name,
 	}
 }
 
+ByteSampleInfo isKeyValueInSample(const KeyRef key, int64_t totalKvSize) {
+	ASSERT(totalKvSize >= key.size());
+	ByteSampleInfo info;
+
+	if (totalKvSize == 0) {
+		info.size = 0;
+		info.probability = 0.0;
+		info.inSample = false;
+		info.sampledSize = 0;
+		return info;
+	}
+
+	info.size = totalKvSize;
+
+	uint32_t a = 0;
+	uint32_t b = 0;
+	hashlittle2(key.begin(), key.size(), &a, &b);
+
+	info.probability =
+	    (double)info.size / (key.size() + SERVER_KNOBS->BYTE_SAMPLING_OVERHEAD) / SERVER_KNOBS->BYTE_SAMPLING_FACTOR;
+	info.probability = std::clamp(info.probability, SERVER_KNOBS->MIN_BYTE_SAMPLING_PROBABILITY, 1.0);
+	info.inSample = a / ((1 << 30) * 4.0) < info.probability;
+	info.sampledSize = info.size / info.probability;
+
+	return info;
+}
+
 // TODO: update the cost as bytesReadPerKSecond + opsReadPerKSecond * SERVER_KNOBS->EMPTY_READ_PENALTY. The source of
 // this model is Redwood will have a constant cost of seek of each read ops then read the actual data.
 // As by 71.2.8, bytesReadPerKSecond should be larger than opsReadPerKSecond * SERVER_KNOBS->EMPTY_READ_PENALTY because
@@ -208,7 +270,7 @@ void StorageServerMetrics::notifyNotReadable(KeyRangeRef keys) {
 	auto rs = waitMetricsMap.intersectingRanges(keys);
 	for (auto r = rs.begin(); r != rs.end(); ++r) {
 		auto& v = r->value();
-		CODE_PROBE(v.size(), "notifyNotReadable() sending errors to intersecting ranges");
+		CODE_PROBE(!v.empty(), "notifyNotReadable() sending errors to intersecting ranges");
 		for (int n = 0; n < v.size(); n++)
 			v[n].sendError(wrong_shard_server());
 	}
@@ -602,6 +664,124 @@ void StorageServerMetrics::add(KeyRangeMap<int>& map, KeyRangeRef const& keys, i
 	collapse(map, keys.end);
 }
 
+void StorageServerMetrics::traceWaitMetricsMapIfHighWatermark(const char* reason, KeyRangeRef keys) {
+	auto stats = summarizeWaitMetricsMap(waitMetricsMap);
+	if (stats.rangeCount <= waitMetricsMapHighWatermarks.rangeCount &&
+	    stats.nonEmptyRangeCount <= waitMetricsMapHighWatermarks.nonEmptyRangeCount &&
+	    stats.watcherCount <= waitMetricsMapHighWatermarks.watcherCount &&
+	    stats.watcherCapacity <= waitMetricsMapHighWatermarks.watcherCapacity &&
+	    stats.maxWatchersPerRange <= waitMetricsMapHighWatermarks.maxWatchersPerRange) {
+		return;
+	}
+
+	waitMetricsMapHighWatermarks.rangeCount = std::max(waitMetricsMapHighWatermarks.rangeCount, stats.rangeCount);
+	waitMetricsMapHighWatermarks.nonEmptyRangeCount =
+	    std::max(waitMetricsMapHighWatermarks.nonEmptyRangeCount, stats.nonEmptyRangeCount);
+	waitMetricsMapHighWatermarks.watcherCount = std::max(waitMetricsMapHighWatermarks.watcherCount, stats.watcherCount);
+	waitMetricsMapHighWatermarks.watcherCapacity =
+	    std::max(waitMetricsMapHighWatermarks.watcherCapacity, stats.watcherCapacity);
+	waitMetricsMapHighWatermarks.maxWatchersPerRange =
+	    std::max(waitMetricsMapHighWatermarks.maxWatchersPerRange, stats.maxWatchersPerRange);
+
+	TraceEvent(SevInfo, "WaitMetricsMapStats")
+	    .detail("Reason", reason)
+	    .detail("Keys", keys)
+	    .detail("RangeCount", stats.rangeCount)
+	    .detail("NonEmptyRangeCount", stats.nonEmptyRangeCount)
+	    .detail("WatcherCount", stats.watcherCount)
+	    .detail("WatcherCapacity", stats.watcherCapacity)
+	    .detail("MaxWatchersPerRange", stats.maxWatchersPerRange)
+	    .detail("RangeCountHighWatermark", waitMetricsMapHighWatermarks.rangeCount)
+	    .detail("NonEmptyRangeCountHighWatermark", waitMetricsMapHighWatermarks.nonEmptyRangeCount)
+	    .detail("WatcherCountHighWatermark", waitMetricsMapHighWatermarks.watcherCount)
+	    .detail("WatcherCapacityHighWatermark", waitMetricsMapHighWatermarks.watcherCapacity)
+	    .detail("MaxWatchersPerRangeHighWatermark", waitMetricsMapHighWatermarks.maxWatchersPerRange);
+}
+
+Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest req, Future<Void> timeout) {
+	PromiseStream<StorageMetrics> change;
+	StorageMetrics metrics = self->getMetrics(req.keys);
+	Error error = success();
+	bool timedout = false;
+
+	DisabledTraceEvent(SevDebug, "WaitMetrics", metricReqId)
+	    .detail("Keys", req.keys)
+	    .detail("Metrics", metrics.toString())
+	    .detail("ReqMin", req.min.toString())
+	    .detail("ReqMax", req.max.toString());
+	if (!req.min.allLessOrEqual(metrics) || !metrics.allLessOrEqual(req.max)) {
+		CODE_PROBE(true, "ShardWaitMetrics return case 1 (quickly)");
+		req.reply.send(metrics);
+		co_return;
+	}
+
+	{
+		auto rs = self->waitMetricsMap.modify(req.keys);
+		for (auto r = rs.begin(); r != rs.end(); ++r) {
+			r->value().push_back(change);
+		}
+		self->traceWaitMetricsMapIfHighWatermark("Register", req.keys);
+		while (true) {
+			try {
+				auto res = co_await race(change.getFuture(), timeout);
+				if (res.index() == 0) {
+					metrics += std::get<0>(std::move(res));
+				} else {
+					timedout = true;
+				}
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled) {
+					throw;
+				}
+				error = e;
+				break;
+			}
+
+			if (timedout) {
+				CODE_PROBE(true, "ShardWaitMetrics return on timeout");
+				if (deterministicRandom()->random01() < SERVER_KNOBS->WAIT_METRICS_WRONG_SHARD_CHANCE) {
+					req.reply.sendError(wrong_shard_server());
+				} else {
+					req.reply.send(metrics);
+				}
+				break;
+			}
+
+			if (!req.min.allLessOrEqual(metrics) || !metrics.allLessOrEqual(req.max)) {
+				CODE_PROBE(true, "ShardWaitMetrics return case 2 (delayed)");
+				req.reply.send(metrics);
+				break;
+			}
+		}
+
+		co_await delay(0);
+	}
+	auto rs = self->waitMetricsMap.modify(req.keys);
+	for (auto i = rs.begin(); i != rs.end(); ++i) {
+		auto& x = i->value();
+		for (int j = 0; j < x.size(); j++) {
+			if (x[j] == change) {
+				swapAndPop(&x, j);
+				break;
+			}
+		}
+	}
+	self->waitMetricsMap.coalesce(req.keys);
+	self->traceWaitMetricsMapIfHighWatermark("Cleanup", req.keys);
+
+	if (error.code() != error_code_success) {
+		if (error.code() != error_code_wrong_shard_server) {
+			throw error;
+		}
+		CODE_PROBE(true, "ShardWaitMetrics delayed wrong_shard_server()");
+		req.reply.sendError(error);
+	}
+}
+
+Future<Void> StorageServerMetrics::waitMetrics(WaitMetricsRequest req, Future<Void> delay) {
+	return ::waitMetrics(this, req, delay);
+}
+
 // Returns the sampled metric value (possibly 0, possibly increased by the sampling factor)
 int64_t TransientStorageMetricSample::addAndExpire(const Key& key, int64_t metric, double expiration) {
 	auto x = add(key, metric);
@@ -632,7 +812,7 @@ bool TransientStorageMetricSample::roll(int64_t metric) const {
 void TransientStorageMetricSample::poll(KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>>& waitMap,
                                         StorageMetrics metrics) {
 	double now = ::now();
-	while (queue.size() && queue.front().first <= now) {
+	while (!queue.empty() && queue.front().first <= now) {
 		KeyRef key = queue.front().second.first;
 		int64_t delta = queue.front().second.second;
 		ASSERT(delta != 0);
@@ -654,7 +834,7 @@ void TransientStorageMetricSample::poll(KeyRangeMap<std::vector<PromiseStream<St
 
 void TransientStorageMetricSample::poll() {
 	double now = ::now();
-	while (queue.size() && queue.front().first <= now) {
+	while (!queue.empty() && queue.front().first <= now) {
 		KeyRef key = queue.front().second.first;
 		int64_t delta = queue.front().second.second;
 		ASSERT(delta != 0);
@@ -761,7 +941,7 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/noneSplitable") {
 
 	std::vector<KeyRef> t = ssm.getSplitPoints(KeyRangeRef("A"_sr, "C"_sr), 10000 * sampleUnit, {});
 
-	ASSERT(t.size() == 0);
+	ASSERT(t.empty());
 
 	return Void();
 }
@@ -782,7 +962,7 @@ TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/chunkTooLarge") {
 
 	std::vector<KeyRef> t = ssm.getSplitPoints(KeyRangeRef("A"_sr, "C"_sr), 1000 * sampleUnit, {});
 
-	ASSERT(t.size() == 0);
+	ASSERT(t.empty());
 
 	return Void();
 }
diff --git a/fdbserver/core/TSSMappingUtil.cpp b/fdbserver/core/TSSMappingUtil.cpp
new file mode 100644
index 00000000000..29421fbf064
--- /dev/null
+++ b/fdbserver/core/TSSMappingUtil.cpp
@@ -0,0 +1,48 @@
+/*
+ * TSSMappingUtil.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/SystemData.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbserver/core/TSSMappingUtil.h"
+Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
+                               std::map<UID, StorageServerInterface>* tssMapping) {
+	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
+	KeyBackedMap<UID, UID>::RangeResultType uidMapping =
+	    co_await tssMapDB.getRange(tr, UID(), Optional<UID>(), CLIENT_KNOBS->TOO_MANY);
+	ASSERT(uidMapping.results.size() < CLIENT_KNOBS->TOO_MANY);
+
+	for (auto mapItr = uidMapping.results.begin(); mapItr != uidMapping.results.end(); ++mapItr) {
+		UID ssId = mapItr->first;
+		Optional<Value> v = co_await tr->get(serverListKeyFor(mapItr->second));
+		(*tssMapping)[ssId] = decodeServerListValue(v.get());
+	}
+}
+
+Future<Void> readTSSMapping(Transaction* tr, std::map<UID, StorageServerInterface>* tssMapping) {
+	RangeResult mappingList = co_await tr->getRange(tssMappingKeys, CLIENT_KNOBS->TOO_MANY);
+	ASSERT(!mappingList.more && mappingList.size() < CLIENT_KNOBS->TOO_MANY);
+
+	for (auto& it : mappingList) {
+		UID ssId = TupleCodec<UID>::unpack(it.key.removePrefix(tssMappingKeys.begin));
+		UID tssId = TupleCodec<UID>::unpack(it.value);
+		Optional<Value> v = co_await tr->get(serverListKeyFor(tssId));
+		(*tssMapping)[ssId] = decodeServerListValue(v.get());
+	}
+}
diff --git a/fdbserver/WaitFailure.actor.cpp b/fdbserver/core/WaitFailure.cpp
similarity index 52%
rename from fdbserver/WaitFailure.actor.cpp
rename to fdbserver/core/WaitFailure.cpp
index 1ce7f954182..7290c778b31 100644
--- a/fdbserver/WaitFailure.actor.cpp
+++ b/fdbserver/core/WaitFailure.cpp
@@ -1,5 +1,5 @@
 /*
- * WaitFailure.actor.cpp
+ * WaitFailure.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,15 +21,14 @@
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/fdbrpc.h"
 #include "flow/Deque.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/WaitFailure.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR Future<Void> waitFailureServer(FutureStream<ReplyPromise<Void>> waitFailure) {
+Future<Void> waitFailureServer(FutureStream<ReplyPromise<Void>> waitFailure) {
 	// when this actor is cancelled, the promises in the queue will send broken_promise
-	state Deque<ReplyPromise<Void>> queue;
-	loop {
-		ReplyPromise<Void> P = waitNext(waitFailure);
+	Deque<ReplyPromise<Void>> queue;
+	while (true) {
+		ReplyPromise<Void> P = co_await waitFailure;
 		queue.push_back(P);
 		if (queue.size() > SERVER_KNOBS->MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS) {
 			CODE_PROBE(true, "wait server queue full");
@@ -39,17 +38,17 @@ ACTOR Future<Void> waitFailureServer(FutureStream<ReplyPromise<Void>> waitFailur
 	}
 }
 
-ACTOR Future<Void> waitFailureClient(RequestStream<ReplyPromise<Void>> waitFailure,
-                                     double reactionTime,
-                                     double reactionSlope,
-                                     bool trace,
-                                     Optional<Standalone<StringRef>> traceMsg,
-                                     TaskPriority taskID) {
-	loop {
+Future<Void> waitFailureClient(RequestStream<ReplyPromise<Void>> waitFailure,
+                               double reactionTime,
+                               double reactionSlope,
+                               bool trace,
+                               Optional<Standalone<StringRef>> traceMsg,
+                               TaskPriority taskID) {
+	while (true) {
 		try {
-			state double start = now();
+			double start = now();
 			ErrorOr<Void> x =
-			    wait(waitFailure.getReplyUnlessFailedFor(ReplyPromise<Void>(), reactionTime, reactionSlope, taskID));
+			    co_await waitFailure.getReplyUnlessFailedFor(ReplyPromise<Void>(), reactionTime, reactionSlope, taskID);
 			if (!x.present()) {
 				if (trace) {
 					TraceEvent te("WaitFailureClient");
@@ -59,11 +58,12 @@ ACTOR Future<Void> waitFailureClient(RequestStream<ReplyPromise<Void>> waitFailu
 						te.detail("Context", traceMsg.get());
 					}
 				}
-				return Void();
+				co_return;
 			}
 			double w = start + SERVER_KNOBS->WAIT_FAILURE_DELAY_LIMIT - now();
-			if (w > 0)
-				wait(delay(w, taskID));
+			if (w > 0) {
+				co_await delay(w, taskID);
+			}
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
 				throw;
@@ -73,42 +73,43 @@ ACTOR Future<Void> waitFailureClient(RequestStream<ReplyPromise<Void>> waitFailu
 	}
 }
 
-ACTOR Future<Void> waitFailureClientStrict(RequestStream<ReplyPromise<Void>> waitFailure,
-                                           double failureReactionTime,
-                                           TaskPriority taskID) {
-	loop {
-		wait(waitFailureClient(waitFailure,
-		                       /* failureReactionTime */ 0,
-		                       /* failureReactionSlope */ 0,
-		                       /* trace */ false,
-		                       /* traceMsg */ Optional<Standalone<StringRef>>(),
-		                       taskID));
-		wait(delay(failureReactionTime, taskID) ||
-		     IFailureMonitor::failureMonitor().onStateEqual(waitFailure.getEndpoint(), FailureStatus(false)));
+Future<Void> waitFailureClientStrict(RequestStream<ReplyPromise<Void>> waitFailure,
+                                     double failureReactionTime,
+                                     TaskPriority taskID) {
+	while (true) {
+		co_await waitFailureClient(waitFailure,
+		                           /* failureReactionTime */ 0,
+		                           /* failureReactionSlope */ 0,
+		                           /* trace */ false,
+		                           /* traceMsg */ Optional<Standalone<StringRef>>(),
+		                           taskID);
+		co_await (delay(failureReactionTime, taskID) ||
+		          IFailureMonitor::failureMonitor().onStateEqual(waitFailure.getEndpoint(), FailureStatus(false)));
 		if (IFailureMonitor::failureMonitor().getState(waitFailure.getEndpoint()).isFailed()) {
-			return Void();
+			co_return;
 		}
 	}
 }
 
-ACTOR Future<Void> waitFailureTracker(RequestStream<ReplyPromise<Void>> waitFailure,
-                                      Reference<AsyncVar<bool>> failed,
-                                      double reactionTime,
-                                      double reactionSlope,
-                                      TaskPriority taskID) {
-	loop {
+Future<Void> waitFailureTracker(RequestStream<ReplyPromise<Void>> waitFailure,
+                                Reference<AsyncVar<bool>> failed,
+                                double reactionTime,
+                                double reactionSlope,
+                                TaskPriority taskID) {
+	while (true) {
 		try {
 			failed->set(IFailureMonitor::failureMonitor().getState(waitFailure.getEndpoint()).isFailed());
 			if (failed->get()) {
-				wait(IFailureMonitor::failureMonitor().onStateChanged(waitFailure.getEndpoint()));
+				co_await IFailureMonitor::failureMonitor().onStateChanged(waitFailure.getEndpoint());
 			} else {
-				state double start = now();
-				ErrorOr<Void> x = wait(
-				    waitFailure.getReplyUnlessFailedFor(ReplyPromise<Void>(), reactionTime, reactionSlope, taskID));
+				double start = now();
+				ErrorOr<Void> x = co_await waitFailure.getReplyUnlessFailedFor(
+				    ReplyPromise<Void>(), reactionTime, reactionSlope, taskID);
 				if (x.present()) {
 					double w = start + SERVER_KNOBS->WAIT_FAILURE_DELAY_LIMIT - now();
-					if (w > 0)
-						wait(delay(w, taskID));
+					if (w > 0) {
+						co_await delay(w, taskID);
+					}
 				}
 			}
 		} catch (Error& e) {
diff --git a/fdbserver/core/WorkerEvents.cpp b/fdbserver/core/WorkerEvents.cpp
new file mode 100644
index 00000000000..9b216d9bf1f
--- /dev/null
+++ b/fdbserver/core/WorkerEvents.cpp
@@ -0,0 +1,55 @@
+/*
+ * WorkerEvents.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/WorkerEvents.h"
+
+AsyncResult<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventOnWorkers(
+    std::vector<WorkerDetails> workers,
+    std::string eventName) {
+	try {
+		std::vector<Future<ErrorOr<TraceEventFields>>> eventTraces;
+		for (int c = 0; c < workers.size(); c++) {
+			EventLogRequest req =
+			    !eventName.empty() ? EventLogRequest(Standalone<StringRef>(eventName)) : EventLogRequest();
+			eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0)));
+		}
+
+		co_await waitForAll(eventTraces);
+
+		std::set<std::string> failed;
+		WorkerEvents results;
+
+		for (int i = 0; i < eventTraces.size(); i++) {
+			const ErrorOr<TraceEventFields>& v = eventTraces[i].get();
+			if (v.isError()) {
+				failed.insert(workers[i].interf.address().toString());
+				results[workers[i].interf.address()] = TraceEventFields();
+			} else {
+				results[workers[i].interf.address()] = v.get();
+			}
+		}
+
+		co_return std::make_pair(std::move(results), std::move(failed));
+	} catch (Error& e) {
+		ASSERT(e.code() ==
+		       error_code_actor_cancelled); // All errors should be filtering through the errorOr actor above
+		throw;
+	}
+}
diff --git a/fdbserver/core/WorkerInterface.cpp b/fdbserver/core/WorkerInterface.cpp
new file mode 100644
index 00000000000..29ebd158c55
--- /dev/null
+++ b/fdbserver/core/WorkerInterface.cpp
@@ -0,0 +1,33 @@
+/*
+ * WorkerInterface.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/core/WorkerInterface.actor.h"
+
+Future<Void> extractClusterInterface(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> in,
+                                     Reference<AsyncVar<Optional<ClusterInterface>>> out) {
+	while (true) {
+		if (in->get().present()) {
+			out->set(in->get().get().clientInterface);
+		} else {
+			out->set(Optional<ClusterInterface>());
+		}
+		co_await in->onChange();
+	}
+}
diff --git a/fdbserver/core/WorkerSupport.cpp b/fdbserver/core/WorkerSupport.cpp
new file mode 100644
index 00000000000..0385e1ddd3a
--- /dev/null
+++ b/fdbserver/core/WorkerSupport.cpp
@@ -0,0 +1,243 @@
+#include <algorithm>
+#include <set>
+#include <string>
+
+#include "fdbrpc/simulator.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "flow/TDMetric.h"
+#include "flow/Trace.h"
+#include "flow/genericactors.actor.h"
+#include "flow/network.h"
+
+template class RequestStream<RecruitMasterRequest, false>;
+template struct NetNotifiedQueue<RecruitMasterRequest, false>;
+
+template class RequestStream<InitializeCommitProxyRequest, false>;
+template struct NetNotifiedQueue<InitializeCommitProxyRequest, false>;
+
+template class RequestStream<InitializeGrvProxyRequest, false>;
+template struct NetNotifiedQueue<InitializeGrvProxyRequest, false>;
+
+template class RequestStream<GetServerDBInfoRequest, false>;
+template struct NetNotifiedQueue<GetServerDBInfoRequest, false>;
+
+namespace {
+
+Future<std::vector<Endpoint>> tryDBInfoBroadcast(RequestStream<UpdateServerDBInfoRequest> stream,
+                                                 UpdateServerDBInfoRequest req) {
+	ErrorOr<std::vector<Endpoint>> rep =
+	    co_await stream.getReplyUnlessFailedFor(req, SERVER_KNOBS->DBINFO_FAILED_DELAY, 0);
+	if (rep.present()) {
+		co_return rep.get();
+	}
+	req.broadcastInfo.push_back(stream.getEndpoint());
+	co_return req.broadcastInfo;
+}
+
+std::set<std::pair<std::string, std::string>> g_roles;
+
+Standalone<StringRef> roleString(std::set<std::pair<std::string, std::string>> roles, bool with_ids) {
+	std::string result;
+	for (auto& r : roles) {
+		if (!result.empty()) {
+			result.append(",");
+		}
+		result.append(r.first);
+		if (with_ids) {
+			result.append(":");
+			result.append(r.second);
+		}
+	}
+	return StringRef(result);
+}
+
+} // namespace
+
+Future<std::vector<Endpoint>> broadcastDBInfoRequest(UpdateServerDBInfoRequest req,
+                                                     int sendAmount,
+                                                     Optional<Endpoint> sender,
+                                                     bool sendReply) {
+	std::vector<Future<std::vector<Endpoint>>> replies;
+	ReplyPromise<std::vector<Endpoint>> reply = req.reply;
+	resetReply(req);
+	int currentStream = 0;
+	std::vector<Endpoint> broadcastEndpoints = req.broadcastInfo;
+	for (int i = 0; i < sendAmount && currentStream < broadcastEndpoints.size(); i++) {
+		std::vector<Endpoint> endpoints;
+		RequestStream<UpdateServerDBInfoRequest> cur(broadcastEndpoints[currentStream++]);
+		while (currentStream < broadcastEndpoints.size() * (i + 1) / sendAmount) {
+			endpoints.push_back(broadcastEndpoints[currentStream++]);
+		}
+		req.broadcastInfo = endpoints;
+		replies.push_back(tryDBInfoBroadcast(cur, req));
+		resetReply(req);
+	}
+	co_await waitForAll(replies);
+	std::vector<Endpoint> notUpdated;
+	if (sender.present()) {
+		notUpdated.push_back(sender.get());
+	}
+	for (auto& it : replies) {
+		notUpdated.insert(notUpdated.end(), it.get().begin(), it.get().end());
+	}
+	if (sendReply) {
+		reply.send(notUpdated);
+	}
+	co_return notUpdated;
+}
+
+Future<Void> broadcastTxnRequest(TxnStateRequest req, int sendAmount, bool sendReply) {
+	ReplyPromise<Void> reply = req.reply;
+	resetReply(req);
+	std::vector<Future<Void>> replies;
+	int currentStream = 0;
+	std::vector<Endpoint> broadcastEndpoints = req.broadcastInfo;
+	for (int i = 0; i < sendAmount && currentStream < broadcastEndpoints.size(); i++) {
+		std::vector<Endpoint> endpoints;
+		RequestStream<TxnStateRequest> cur(broadcastEndpoints[currentStream++]);
+		while (currentStream < broadcastEndpoints.size() * (i + 1) / sendAmount) {
+			endpoints.push_back(broadcastEndpoints[currentStream++]);
+		}
+		req.broadcastInfo = endpoints;
+		replies.push_back(brokenPromiseToNever(cur.getReply(req)));
+		resetReply(req);
+	}
+	co_await waitForAll(replies);
+	if (sendReply) {
+		reply.send(Void());
+	}
+}
+
+bool addressInDbAndPrimarySatelliteDc(const NetworkAddress& address, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	for (const auto& logSet : dbInfo->get().logSystemConfig.tLogs) {
+		if (logSet.isLocal && logSet.locality == tagLocalitySatellite) {
+			for (const auto& tlog : logSet.tLogs) {
+				if (tlog.present() && tlog.interf().addresses().contains(address)) {
+					return true;
+				}
+			}
+		}
+	}
+
+	return false;
+}
+
+bool addressInDbAndRemoteDc(const NetworkAddress& address,
+                            Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                            Optional<std::vector<NetworkAddress>> storageServers) {
+	const auto& dbi = dbInfo->get();
+
+	for (const auto& logSet : dbi.logSystemConfig.tLogs) {
+		if (logSet.isLocal || logSet.locality == tagLocalitySatellite) {
+			continue;
+		}
+		for (const auto& tlog : logSet.tLogs) {
+			if (tlog.present() && tlog.interf().addresses().contains(address)) {
+				return true;
+			}
+		}
+
+		for (const auto& logRouter : logSet.logRouters) {
+			if (logRouter.present() && logRouter.interf().addresses().contains(address)) {
+				return true;
+			}
+		}
+	}
+
+	if (storageServers.present() &&
+	    std::find(storageServers.get().begin(), storageServers.get().end(), address) != storageServers.get().end()) {
+		return true;
+	}
+
+	return false;
+}
+
+void startRole(const Role& role,
+               UID roleId,
+               UID workerId,
+               const std::map<std::string, std::string>& details,
+               const std::string& origination) {
+	if (role.includeInTraceRoles) {
+		addTraceRole(role.abbreviation);
+	}
+
+	TraceEvent ev("Role", roleId);
+	ev.detail("As", role.roleName)
+	    .detail("Transition", "Begin")
+	    .detail("Origination", origination)
+	    .detail("OnWorker", workerId);
+	for (auto it = details.begin(); it != details.end(); it++) {
+		ev.detail(it->first.c_str(), it->second);
+	}
+
+	ev.trackLatest(roleId.shortString() + ".Role");
+
+	g_roles.insert({ role.roleName, roleId.shortString() });
+	StringMetricHandle("Roles"_sr) = roleString(g_roles, false);
+	StringMetricHandle("RolesWithIDs"_sr) = roleString(g_roles, true);
+	if (g_network->isSimulated()) {
+		g_simulator->addRole(g_network->getLocalAddress(), role.roleName);
+	}
+}
+
+void endRole(const Role& role, UID id, std::string reason, bool ok, Error e) {
+	{
+		TraceEvent ev("Role", id);
+		if (e.code() != invalid_error_code) {
+			ev.errorUnsuppressed(e);
+		}
+		ev.detail("Transition", "End").detail("As", role.roleName).detail("Reason", reason);
+
+		ev.trackLatest(id.shortString() + ".Role");
+	}
+
+	if (!ok) {
+		std::string type = role.roleName + "Failed";
+
+		TraceEvent err(SevError, type.c_str(), id);
+		if (e.code() != invalid_error_code) {
+			err.errorUnsuppressed(e);
+		}
+		err.detail("Reason", reason);
+	}
+
+	latestEventCache.clear(id.shortString());
+
+	g_roles.erase({ role.roleName, id.shortString() });
+	StringMetricHandle("Roles"_sr) = roleString(g_roles, false);
+	StringMetricHandle("RolesWithIDs"_sr) = roleString(g_roles, true);
+	if (g_network->isSimulated()) {
+		g_simulator->removeRole(g_network->getLocalAddress(), role.roleName);
+	}
+
+	if (role.includeInTraceRoles) {
+		removeTraceRole(role.abbreviation);
+	}
+}
+
+Future<Void> traceRole(Role role, UID roleId) {
+	while (true) {
+		co_await delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL);
+		TraceEvent("Role", roleId).detail("Transition", "Refresh").detail("As", role.roleName);
+	}
+}
+
+const Role Role::WORKER("Worker", "WK", false);
+const Role Role::STORAGE_SERVER("StorageServer", "SS");
+const Role Role::TESTING_STORAGE_SERVER("TestingStorageServer", "ST");
+const Role Role::TRANSACTION_LOG("TLog", "TL");
+const Role Role::SHARED_TRANSACTION_LOG("SharedTLog", "SL", false);
+const Role Role::COMMIT_PROXY("CommitProxyServer", "CP");
+const Role Role::GRV_PROXY("GrvProxyServer", "GP");
+const Role Role::MASTER("MasterServer", "MS");
+const Role Role::RESOLVER("Resolver", "RV");
+const Role Role::CLUSTER_CONTROLLER("ClusterController", "CC");
+const Role Role::TESTER("Tester", "TS");
+const Role Role::LOG_ROUTER("LogRouter", "LR");
+const Role Role::DATA_DISTRIBUTOR("DataDistributor", "DD");
+const Role Role::RATEKEEPER("Ratekeeper", "RK");
+const Role Role::COORDINATOR("Coordinator", "CD");
+const Role Role::BACKUP("Backup", "BK");
+const Role Role::ENCRYPT_KEY_PROXY("EncryptKeyProxy", "EP");
+const Role Role::CONSISTENCYSCAN("ConsistencyScan", "CS");
diff --git a/fdbserver/core/WorkloadKeys.cpp b/fdbserver/core/WorkloadKeys.cpp
new file mode 100644
index 00000000000..3a165908186
--- /dev/null
+++ b/fdbserver/core/WorkloadKeys.cpp
@@ -0,0 +1,23 @@
+#include <cstdio>
+#include <inttypes.h>
+
+#include "fdbserver/core/WorkloadKeys.h"
+#include "flow/flow.h"
+
+Key doubleToTestKey(double p) {
+	return StringRef(format("%016llx", *(uint64_t*)&p));
+}
+
+double testKeyToDouble(const KeyRef& p) {
+	uint64_t x = 0;
+	sscanf(p.toString().c_str(), "%" SCNx64, &x);
+	return *(double*)&x;
+}
+
+Key doubleToTestKey(double p, const KeyRef& prefix) {
+	return doubleToTestKey(p).withPrefix(prefix);
+}
+
+double testKeyToDouble(const KeyRef& p, const KeyRef& prefix) {
+	return testKeyToDouble(p.removePrefix(prefix));
+}
diff --git a/fdbserver/include/fdbserver/AccumulativeChecksumUtil.h b/fdbserver/core/include/fdbserver/core/AccumulativeChecksumUtil.h
similarity index 98%
rename from fdbserver/include/fdbserver/AccumulativeChecksumUtil.h
rename to fdbserver/core/include/fdbserver/core/AccumulativeChecksumUtil.h
index 30c8ad5acde..5adcb909229 100644
--- a/fdbserver/include/fdbserver/AccumulativeChecksumUtil.h
+++ b/fdbserver/core/include/fdbserver/core/AccumulativeChecksumUtil.h
@@ -65,7 +65,7 @@ inline uint32_t aggregateAcs(uint32_t startAcs, Standalone<VectorRef<std::pair<V
 // storage server tags (aka. locality >= 0)
 class AccumulativeChecksumBuilder {
 public:
-	AccumulativeChecksumBuilder(uint16_t acsIndex) : acsIndex(acsIndex), currentVersion(0) {}
+	explicit AccumulativeChecksumBuilder(uint16_t acsIndex) : acsIndex(acsIndex), currentVersion(0) {}
 
 	// Called when commit proxy applies a new tag assignment mutation
 	// At this time, this method erases the corresponding ACS value of the tag
diff --git a/fdbserver/include/fdbserver/BackupInterface.h b/fdbserver/core/include/fdbserver/core/BackupInterface.h
similarity index 100%
rename from fdbserver/include/fdbserver/BackupInterface.h
rename to fdbserver/core/include/fdbserver/core/BackupInterface.h
diff --git a/fdbserver/include/fdbserver/BackupProgress.actor.h b/fdbserver/core/include/fdbserver/core/BackupProgress.h
similarity index 81%
rename from fdbserver/include/fdbserver/BackupProgress.actor.h
rename to fdbserver/core/include/fdbserver/core/BackupProgress.h
index f0ea169b52c..aa4da4c5656 100644
--- a/fdbserver/include/fdbserver/BackupProgress.actor.h
+++ b/fdbserver/core/include/fdbserver/core/BackupProgress.h
@@ -1,5 +1,5 @@
 /*
- * BackupProgress.actor.h
+ * BackupProgress.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,25 +18,20 @@
  * limitations under the License.
  */
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_BACKUPPROGRESS_ACTOR_G_H)
-#define FDBSERVER_BACKUPPROGRESS_ACTOR_G_H
-#include "fdbserver/BackupProgress.actor.g.h"
-#elif !defined(FDBSERVER_BACKUPPROGRESS_ACTOR_H)
-#define FDBSERVER_BACKUPPROGRESS_ACTOR_H
+#pragma once
 
 #include <map>
+#include <set>
 #include <tuple>
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/LogSystem.h"
+#include "fdbserver/core/BackupProgressTypes.h"
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 class BackupProgress : NonCopyable, ReferenceCounted<BackupProgress> {
 public:
-	BackupProgress(UID id, const std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo>& infos)
-	  : dbgid(id), epochInfos(infos) {}
+	BackupProgress(UID id, const std::map<LogEpoch, EpochTagsVersionsInfo>& infos) : dbgid(id), epochInfos(infos) {}
 	~BackupProgress() {}
 
 	// Adds a backup status. If the tag already has an entry, then the max of
@@ -89,7 +84,7 @@ class BackupProgress : NonCopyable, ReferenceCounted<BackupProgress> {
 	const UID dbgid;
 
 	// Note this MUST be iterated in ascending order.
-	const std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> epochInfos;
+	const std::map<LogEpoch, EpochTagsVersionsInfo> epochInfos;
 
 	// Backup progress saved in the system keyspace. Note there can be multiple
 	// progress status for a tag in an epoch due to later epoch trying to fill
@@ -104,7 +99,4 @@ class BackupProgress : NonCopyable, ReferenceCounted<BackupProgress> {
 	Optional<Value> backupStartedValue;
 };
 
-ACTOR Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupProgress> bStatus, bool logging);
-
-#include "flow/unactorcompiler.h"
-#endif
+Future<Void> getBackupProgress(Database cx, UID dbgid, Reference<BackupProgress> bStatus, Severity severity);
diff --git a/fdbserver/core/include/fdbserver/core/BackupProgressTypes.h b/fdbserver/core/include/fdbserver/core/BackupProgressTypes.h
new file mode 100644
index 00000000000..e3a92d5573f
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/BackupProgressTypes.h
@@ -0,0 +1,31 @@
+/*
+ * BackupProgressTypes.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+
+struct EpochTagsVersionsInfo {
+	int32_t logRouterTags;
+	Version epochBegin, epochEnd;
+
+	explicit EpochTagsVersionsInfo(int32_t n, Version begin, Version end)
+	  : logRouterTags(n), epochBegin(begin), epochEnd(end) {}
+};
diff --git a/fdbserver/include/fdbserver/BulkDumpUtil.actor.h b/fdbserver/core/include/fdbserver/core/BulkDumpUtil.h
similarity index 70%
rename from fdbserver/include/fdbserver/BulkDumpUtil.actor.h
rename to fdbserver/core/include/fdbserver/core/BulkDumpUtil.h
index 759feedeef7..88f798f3061 100644
--- a/fdbserver/include/fdbserver/BulkDumpUtil.actor.h
+++ b/fdbserver/core/include/fdbserver/core/BulkDumpUtil.h
@@ -1,5 +1,5 @@
 /*
- * BulkDumpUtil.actor.h
+ * BulkDumpUtil.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,18 +20,13 @@
 
 #include <cstdint>
 #include <string>
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_BULKDUMPUTIL_ACTOR_G_H)
-#define FDBSERVER_BULKDUMPUTIL_ACTOR_G_H
-#include "fdbserver/BulkDumpUtil.actor.g.h"
-#elif !defined(FDBSERVER_BULKDUMPUTIL_ACTOR_H)
-#define FDBSERVER_BULKDUMPUTIL_ACTOR_H
 #pragma once
 
 #include "fdbclient/BulkDumping.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 struct RangeDumpRawData {
+	Arena arena;
 	std::map<Key, Value> kvs;
 	std::map<Key, Value> sampled;
 	Key lastKey;
@@ -83,7 +78,7 @@ std::pair<BulkLoadFileSet, BulkLoadFileSet> getLocalRemoteFileSetSetting(Version
 
 // Persist the complete progress of bulkDump by writing the metadata with Complete phase
 // to the bulk dump system key space.
-ACTOR Future<Void> persistCompleteBulkDumpRange(Database cx, BulkDumpState bulkDumpState);
+Future<Void> persistCompleteBulkDumpRange(Database cx, BulkDumpState bulkDumpState);
 
 // Define bulk dump job folder. Job is set by user. At most one job at a time globally.
 std::string generateBulkDumpJobFolder(const UID& jobId);
@@ -91,43 +86,40 @@ std::string generateBulkDumpJobFolder(const UID& jobId);
 // Define task folder name.
 std::string getBulkDumpJobTaskFolder(const UID& jobId, const UID& taskId);
 
-// Define job root folder.
-std::string getBulkLoadJobRoot(const std::string& root, const UID& jobId);
-
 // Generate key-value data, byte sampling data, and manifest file.
 // Return BulkLoadManifest metadata (equivalent to content of the manifest file).
 // TODO(BulkDump): can cause slow tasks, do the task in a separate thread in the future.
 // The size of sortedData is defined at the place of generating the data (getRangeDataToDump).
 // The size is configured by MOVE_SHARD_KRM_ROW_LIMIT.
-ACTOR Future<BulkLoadManifest> dumpDataFileToLocalDirectory(UID logId,
-                                                            std::shared_ptr<RangeDumpRawData> rangeDumpRawData,
-                                                            BulkLoadFileSet localFileSet,
-                                                            BulkLoadFileSet remoteFileSet,
-                                                            BulkLoadByteSampleSetting byteSampleSetting,
-                                                            Version dumpVersion,
-                                                            KeyRange dumpRange,
-                                                            BulkLoadType dumpType,
-                                                            BulkLoadTransportMethod transportMethod);
+Future<BulkLoadManifest> dumpDataFileToLocalDirectory(UID logId,
+                                                      std::shared_ptr<RangeDumpRawData> rangeDumpRawData,
+                                                      BulkLoadFileSet localFileSet,
+                                                      BulkLoadFileSet remoteFileSet,
+                                                      BulkLoadByteSampleSetting byteSampleSetting,
+                                                      Version dumpVersion,
+                                                      KeyRange dumpRange,
+                                                      BulkLoadType dumpType,
+                                                      BulkLoadTransportMethod transportMethod);
 
 // Upload manifest file for bulkdump job
 // Each job has one manifest file including manifest paths of all tasks.
 // The local file path:	<localRootLocal>/<jobId>-manifest.txt
 // The remote file folder and the name of the file in the remote folder.
-ACTOR Future<Void> uploadBulkDumpJobManifestFile(BulkLoadTransportMethod transportMethod,
-                                                 std::string localJobManifestFilePath,
-                                                 std::string remoteFolder,
-                                                 std::string jobManifestFileName,
-                                                 UID logId);
+Future<Void> uploadBulkDumpJobManifestFile(BulkLoadTransportMethod transportMethod,
+                                           std::string localJobManifestFilePath,
+                                           std::string remoteFolder,
+                                           std::string jobManifestFileName,
+                                           UID logId);
 
 // Upload file for each task. Each task is spawned by bulkdump job according to the shard boundary
-ACTOR Future<Void> uploadBulkDumpFileSet(BulkLoadTransportMethod transportMethod,
-                                         BulkLoadFileSet sourceFileSet,
-                                         BulkLoadFileSet destinationFileSet,
-                                         UID logId);
+Future<Void> uploadBulkDumpFileSet(BulkLoadTransportMethod transportMethod,
+                                   BulkLoadFileSet sourceFileSet,
+                                   BulkLoadFileSet destinationFileSet,
+                                   UID logId);
 
 class ParallelismLimitor {
 public:
-	ParallelismLimitor(int maxParallelism) : maxParallelism(maxParallelism) {}
+	explicit(false) ParallelismLimitor(int maxParallelism) : maxParallelism(maxParallelism) {}
 
 	inline void decrementTaskCounter() {
 		ASSERT(numRunningTasks.get() <= maxParallelism);
@@ -156,6 +148,3 @@ class ParallelismLimitor {
 	AsyncVar<int> numRunningTasks;
 	int maxParallelism;
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/core/include/fdbserver/core/BulkLoadUtil.h b/fdbserver/core/include/fdbserver/core/BulkLoadUtil.h
new file mode 100644
index 00000000000..3805ab09e71
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/BulkLoadUtil.h
@@ -0,0 +1,80 @@
+/*
+ * BulkLoadUtil.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <memory>
+
+#include "fdbclient/BulkLoading.h"
+
+// Erase file folder
+void clearFileFolder(const std::string& folderPath, const UID& logId = UID(), bool ignoreError = false);
+
+// Erase and recreate file folder
+void resetFileFolder(const std::string& folderPath);
+
+// Asynchronously copy file from one path to another.
+Future<Void> copyBulkFile(std::string fromFile, std::string toFile, size_t fileBytesMax);
+
+// Asynchronously read file bytes from local file.
+Future<Void> readBulkFileBytes(std::string path, int64_t maxLength, std::shared_ptr<std::string> output);
+
+// Asynchronously write file bytes to local file.
+Future<Void> writeBulkFileBytes(std::string path, std::shared_ptr<std::string> content);
+
+// Get the bulkLoadTask metadata of the dataMoveMetadata since the atLeastVersion given the dataMoveId
+// This actor is stuck if the actor is failed to read the dataMoveMetadata.
+Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
+                                                           UID dataMoveId,
+                                                           Version atLeastVersion,
+                                                           UID logId);
+
+Future<BulkLoadFileSet> bulkLoadDownloadTaskFileSet(BulkLoadTransportMethod transportMethod,
+                                                    BulkLoadFileSet fromRemoteFileSet,
+                                                    std::string toLocalRoot,
+                                                    UID logId);
+
+Future<Void> bulkLoadDownloadTaskFileSets(BulkLoadTransportMethod transportMethod,
+                                          std::shared_ptr<BulkLoadFileSetKeyMap> fromRemoteFileSets,
+                                          std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets,
+                                          std::string toLocalRoot,
+                                          UID logId);
+
+Future<bool> doBytesSamplingOnDataFile(std::string dataFileFullPath, std::string byteSampleFileFullPath, UID logId);
+
+// Download job manifest file which is generated when dumping the data
+Future<Void> downloadBulkLoadJobManifestFile(BulkLoadTransportMethod transportMethod,
+                                             std::string localJobManifestFilePath,
+                                             std::string remoteJobManifestFilePath,
+                                             UID logId);
+
+// Extract manifest entries from job manifest file with the input range
+Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
+    std::string localJobManifestFilePath,
+    KeyRange range,
+    UID logId,
+    std::shared_ptr<BulkLoadManifestFileMap> manifestMap);
+
+// Get BulkLoad manifest metadata from the entry in the job manifest file
+Future<BulkLoadManifestSet> getBulkLoadManifestMetadataFromEntry(
+    std::vector<BulkLoadJobFileManifestEntry> manifestEntries,
+    std::string manifestLocalTempFolder,
+    BulkLoadTransportMethod transportMethod,
+    std::string jobRoot,
+    UID logId);
diff --git a/fdbserver/core/include/fdbserver/core/ConflictBatch.h b/fdbserver/core/include/fdbserver/core/ConflictBatch.h
new file mode 100644
index 00000000000..56c75a48993
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/ConflictBatch.h
@@ -0,0 +1,37 @@
+/*
+ * ConflictBatch.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_CONFLICTBATCH_H
+#define FDBSERVER_CONFLICTBATCH_H
+#pragma once
+
+#include <cstdint>
+
+struct ConflictBatchStatus {
+	enum TransactionCommitResult : uint8_t {
+		TransactionConflict = 0,
+		TransactionTooOld,
+		TransactionUnusedResultValue1,
+		TransactionCommitted,
+		TransactionLockReject,
+	};
+};
+
+#endif // FDBSERVER_CONFLICTBATCH_H
diff --git a/fdbserver/include/fdbserver/CoordinatedState.h b/fdbserver/core/include/fdbserver/core/CoordinatedState.h
similarity index 96%
rename from fdbserver/include/fdbserver/CoordinatedState.h
rename to fdbserver/core/include/fdbserver/core/CoordinatedState.h
index cc838b05dbe..afaa9b9f027 100644
--- a/fdbserver/include/fdbserver/CoordinatedState.h
+++ b/fdbserver/core/include/fdbserver/core/CoordinatedState.h
@@ -28,7 +28,7 @@
 class CoordinatedState : NonCopyable {
 public:
 	// Callers must ensure that any outstanding operations have been cancelled before destructing *this!
-	CoordinatedState(class ServerCoordinators const&);
+	explicit CoordinatedState(class ServerCoordinators const&);
 	~CoordinatedState();
 
 	Future<Value> read();
@@ -62,7 +62,7 @@ class CoordinatedState : NonCopyable {
 
 class MovableCoordinatedState : NonCopyable {
 public:
-	MovableCoordinatedState(class ServerCoordinators const&);
+	explicit MovableCoordinatedState(class ServerCoordinators const&);
 	MovableCoordinatedState& operator=(MovableCoordinatedState&& av);
 	~MovableCoordinatedState();
 
diff --git a/fdbserver/include/fdbserver/CoordinationInterface.h b/fdbserver/core/include/fdbserver/core/CoordinationInterface.h
similarity index 92%
rename from fdbserver/include/fdbserver/CoordinationInterface.h
rename to fdbserver/core/include/fdbserver/core/CoordinationInterface.h
index 97bd4b2236a..a4490ae2855 100644
--- a/fdbserver/include/fdbserver/CoordinationInterface.h
+++ b/fdbserver/core/include/fdbserver/core/CoordinationInterface.h
@@ -51,9 +51,9 @@ struct GenerationRegInterface {
 	// the v2 of the previous generation is the v1 of the next.
 
 	GenerationRegInterface() {}
-	GenerationRegInterface(NetworkAddress const& remote);
-	GenerationRegInterface(INetwork* local);
-	GenerationRegInterface(Hostname const& hostname) : hostname(hostname) {};
+	explicit GenerationRegInterface(NetworkAddress const& remote);
+	explicit GenerationRegInterface(INetwork* local);
+	explicit GenerationRegInterface(Hostname const& hostname) : hostname(hostname) {};
 };
 
 struct UniqueGeneration {
@@ -126,9 +126,9 @@ struct LeaderElectionRegInterface : ClientLeaderRegInterface {
 	RequestStream<struct ForwardRequest> forward;
 
 	LeaderElectionRegInterface() {}
-	LeaderElectionRegInterface(NetworkAddress const& remote);
-	LeaderElectionRegInterface(INetwork* local);
-	LeaderElectionRegInterface(Hostname const& hostname) : ClientLeaderRegInterface(hostname) {}
+	explicit LeaderElectionRegInterface(NetworkAddress const& remote);
+	explicit LeaderElectionRegInterface(INetwork* local);
+	explicit LeaderElectionRegInterface(Hostname const& hostname) : ClientLeaderRegInterface(hostname) {}
 };
 
 struct CandidacyRequest {
@@ -227,11 +227,7 @@ class ServerCoordinators : public ClientCoordinators {
 	std::vector<GenerationRegInterface> stateServers;
 };
 
-Future<Void> coordinationServer(std::string const& dataFolder, Reference<IClusterConnectionRecord> const& ccf);
-
 // Read a value of MovableValue and if the old cluster key is nested in it, update it to the new key
 Optional<Value> updateCCSInMovableValue(ValueRef movableVal, KeyRef oldClusterKey, KeyRef newClusterKey);
 
-Future<Void> coordChangeClusterKey(std::string dataFolder, KeyRef newClusterKey, KeyRef oldClusterKey);
-
 #endif
diff --git a/fdbserver/include/fdbserver/DBCoreState.h b/fdbserver/core/include/fdbserver/core/DBCoreState.h
similarity index 97%
rename from fdbserver/include/fdbserver/DBCoreState.h
rename to fdbserver/core/include/fdbserver/core/DBCoreState.h
index 3c4512482e8..078caa6d0c7 100644
--- a/fdbserver/include/fdbserver/DBCoreState.h
+++ b/fdbserver/core/include/fdbserver/core/DBCoreState.h
@@ -26,12 +26,11 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/ReplicationPolicy.h"
-#include "fdbserver/LogSystemConfig.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/LogSystemConfig.h"
+#include "fdbserver/core/MasterInterface.h"
 #include "flow/ObjectSerializerTraits.h"
-#include "fdbserver/Knobs.h"
 
-class LogSet;
 struct OldLogData;
 
 // This structure is stored persistently in CoordinatedState and must be versioned carefully!
@@ -60,8 +59,6 @@ struct CoreTLogSet {
 	CoreTLogSet()
 	  : tLogWriteAntiQuorum(0), tLogReplicationFactor(0), isLocal(true), locality(tagLocalityInvalid),
 	    startVersion(invalidVersion) {}
-	explicit CoreTLogSet(const LogSet& logset);
-
 	bool operator==(CoreTLogSet const& rhs) const {
 		return tLogs == rhs.tLogs && tLogWriteAntiQuorum == rhs.tLogWriteAntiQuorum &&
 		       tLogReplicationFactor == rhs.tLogReplicationFactor && isLocal == rhs.isLocal &&
@@ -97,8 +94,6 @@ struct OldTLogCoreData {
 	LogEpoch epoch;
 
 	OldTLogCoreData() : logRouterTags(0), txsTags(0), epochBegin(0), epochEnd(0), recoverAt(0), epoch(0) {}
-	explicit OldTLogCoreData(const OldLogData&);
-
 	bool operator==(const OldTLogCoreData& rhs) const {
 		if (SERVER_KNOBS->RECORD_RECOVER_AT_IN_CSTATE) {
 			return tLogs == rhs.tLogs && logRouterTags == rhs.logRouterTags && txsTags == rhs.txsTags &&
diff --git a/fdbserver/include/fdbserver/DataDistributorInterface.h b/fdbserver/core/include/fdbserver/core/DataDistributorInterface.h
similarity index 93%
rename from fdbserver/include/fdbserver/DataDistributorInterface.h
rename to fdbserver/core/include/fdbserver/core/DataDistributorInterface.h
index 26c1fa9edd3..2cf1a78b12b 100644
--- a/fdbserver/include/fdbserver/DataDistributorInterface.h
+++ b/fdbserver/core/include/fdbserver/core/DataDistributorInterface.h
@@ -25,6 +25,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/fdbrpc.h"
+#include <string>
 #include <unordered_set>
 
 FDB_BOOLEAN_PARAM(IsMocked);
@@ -74,7 +75,7 @@ struct PrepareBlobRestoreReply {
 	int8_t res;
 
 	PrepareBlobRestoreReply() = default;
-	PrepareBlobRestoreReply(Type t) : res(int8_t(t)) {}
+	explicit PrepareBlobRestoreReply(Type t) : res(int8_t(t)) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -158,7 +159,8 @@ struct GetDataDistributorMetricsRequest {
 	}
 };
 
-// FIXME: explain purpose
+// Request sent to the data distributor to coordinate a snapshot across
+// stateful workers for a single snapshot UID.
 struct DistributorSnapRequest {
 	constexpr static FileIdentifier file_identifier = 5427684;
 	Arena arena;
@@ -221,9 +223,24 @@ struct DistributorSplitRangeRequest {
 	}
 };
 
+struct StorageWigglerState {
+	enum Value : uint8_t { INVALID = 0, RUN = 1, PAUSE = 2 };
+
+	static std::string toString(Value state) {
+		switch (state) {
+		case RUN:
+			return "running";
+		case PAUSE:
+			return "paused";
+		default:
+			return "unknown";
+		}
+	}
+};
+
 struct GetStorageWigglerStateReply {
 	constexpr static FileIdentifier file_identifier = 356721;
-	uint8_t primary = 0, remote = 0; // StorageWiggler::State enum
+	uint8_t primary = 0, remote = 0; // StorageWigglerState::Value enum
 	double lastStateChangePrimary = 0.0, lastStateChangeRemote = 0.0;
 
 	GetStorageWigglerStateReply() = default;
diff --git a/fdbserver/core/include/fdbserver/core/DataMovement.h b/fdbserver/core/include/fdbserver/core/DataMovement.h
new file mode 100644
index 00000000000..a4366924e4c
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/DataMovement.h
@@ -0,0 +1,26 @@
+/*
+ * DataMovement.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/SystemData.h"
+
+int dataMovementPriority(DataMovementReason moveReason);
+DataMovementReason priorityToDataMovementReason(int priority);
diff --git a/fdbserver/core/include/fdbserver/core/FDBSimulationPolicy.h b/fdbserver/core/include/fdbserver/core/FDBSimulationPolicy.h
new file mode 100644
index 00000000000..77742bb42e8
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/FDBSimulationPolicy.h
@@ -0,0 +1,117 @@
+/*
+ * FDBSimulationPolicy.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_CORE_FDBSIMULATIONPOLICY_H
+#define FDBSERVER_CORE_FDBSIMULATIONPOLICY_H
+#pragma once
+
+#include "fdbclient/DatabaseConfiguration.h"
+
+#include <limits>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+enum class FDBExtraDatabaseMode { Disabled, LocalOrSingle, Single, Local, Multiple };
+enum class FDBBackupAgentType { NoBackupAgents, WaitForType, BackupToFile, BackupToDB };
+// Order matters: all modes >= EnabledAddDelay are fault injection modes.
+enum class FDBTSSMode { Disabled, EnabledNormal, EnabledAddDelay, EnabledDropMutations };
+enum FDBSimConsistencyScanState {
+	DisabledStart = 0,
+	Enabling = 1,
+	Enabled = 2,
+	Enabled_InjectCorruption = 3,
+	Enabled_FoundCorruption = 4,
+	Complete = 5,
+	DisabledEnd = 6
+};
+enum FDBSimConsistencyScanCorruptionType { FlipMoreFlag = 0, AddToEmpty = 1, RemoveLastRow = 2, ChangeFirstValue = 3 };
+
+struct FDBSimulationPolicyState {
+	int desiredCoordinators = 1;
+	Reference<IReplicationPolicy> storagePolicy;
+	Reference<IReplicationPolicy> tLogPolicy;
+	int32_t tLogWriteAntiQuorum = 0;
+	Optional<Standalone<StringRef>> primaryDcId;
+	Reference<IReplicationPolicy> remoteTLogPolicy;
+	int32_t usableRegions = 1;
+	bool allowLogSetKills = true;
+	Optional<Standalone<StringRef>> remoteDcId;
+	bool hasSatelliteReplication = false;
+	Reference<IReplicationPolicy> satelliteTLogPolicy;
+	Reference<IReplicationPolicy> satelliteTLogPolicyFallback;
+	int32_t satelliteTLogWriteAntiQuorum = 0;
+	int32_t satelliteTLogWriteAntiQuorumFallback = 0;
+	std::vector<Optional<Standalone<StringRef>>> primarySatelliteDcIds;
+	std::vector<Optional<Standalone<StringRef>>> remoteSatelliteDcIds;
+	bool allowStorageMigrationTypeChange = false;
+	FDBBackupAgentType backupAgents = FDBBackupAgentType::WaitForType;
+	FDBBackupAgentType drAgents = FDBBackupAgentType::WaitForType;
+	bool disableTLogRecoveryFinish = false;
+	std::string disablePrimary;
+	std::string disableRemote;
+	std::string originalRegions;
+	std::string startingDisabledConfiguration;
+	int physicalDatacenters = 1;
+	int testerCount = 0;
+	std::string connectionString;
+	bool hasDiffProtocolProcess = false;
+	bool setDiffProtocol = false;
+	bool isConsistencyChecked = false;
+	bool quiesced = false;
+	bool willRestart = false;
+	bool restarted = false;
+	Optional<std::string> dataAtRestPlaintextMarker;
+	std::vector<std::string> extraDatabases;
+	FDBSimConsistencyScanState consistencyScanState = FDBSimConsistencyScanState::DisabledStart;
+	Optional<FDBSimConsistencyScanCorruptionType> consistencyScanInjectedCorruptionType;
+	Optional<UID> consistencyScanInjectedCorruptionDestination;
+	Optional<bool> doInjectConsistencyScanCorruption;
+	Optional<Standalone<StringRef>> consistencyScanCorruptRequestKey;
+	Optional<std::pair<UID, NetworkAddress>> consistencyScanCorruptor;
+	double injectTargetedSSRestartTime = std::numeric_limits<double>::max();
+	double injectSSDelayTime = std::numeric_limits<double>::max();
+	std::map<NetworkAddress, bool> corruptWorkerMap;
+	FDBTSSMode tssMode = FDBTSSMode::Disabled;
+
+	bool updateConsistencyScanState(FDBSimConsistencyScanState expectedCurrent, FDBSimConsistencyScanState desired) {
+		if (consistencyScanState == expectedCurrent && desired > consistencyScanState) {
+			consistencyScanState = desired;
+
+			if (desired == FDBSimConsistencyScanState::Enabled_FoundCorruption) {
+				consistencyScanInjectedCorruptionType = {};
+				consistencyScanCorruptRequestKey = {};
+				consistencyScanCorruptor = {};
+			}
+
+			return true;
+		}
+		return false;
+	}
+};
+
+void installFDBSimulationPolicy();
+FDBSimulationPolicyState& fdbSimulationPolicyState();
+FDBExtraDatabaseMode stringToFDBExtraDatabaseMode(const std::string& databaseMode);
+void updateFDBSimulationPolicy(DatabaseConfiguration const& configuration, bool restartingTest);
+void setFDBSimulationPolicyRemoteTLogPolicy(Reference<IReplicationPolicy> remoteTLogPolicy);
+
+#endif
diff --git a/fdbserver/include/fdbserver/IDiskQueue.h b/fdbserver/core/include/fdbserver/core/IDiskQueue.h
similarity index 93%
rename from fdbserver/include/fdbserver/IDiskQueue.h
rename to fdbserver/core/include/fdbserver/core/IDiskQueue.h
index 3a123d7f84c..ca8447a2cf0 100644
--- a/fdbserver/include/fdbserver/IDiskQueue.h
+++ b/fdbserver/core/include/fdbserver/core/IDiskQueue.h
@@ -23,7 +23,7 @@
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/IKeyValueStore.h"
+#include "fdbclient/IClosable.h"
 #include "flow/BooleanParam.h"
 
 FDB_BOOLEAN_PARAM(CheckHashes);
@@ -34,7 +34,7 @@ class IDiskQueue : public IClosable {
 		// location is same with seq., specifying the index of the virtually infinite queue.
 		int64_t hi, lo; // hi is always 0, lo is always equal to seq.
 		location() : hi(0), lo(0) {}
-		location(int64_t lo) : hi(0), lo(lo) {}
+		explicit(false) location(int64_t lo) : hi(0), lo(lo) {}
 		location(int64_t hi, int64_t lo) : hi(hi), lo(lo) {}
 		operator std::string() const {
 			return format("%lld.%lld", hi, lo);
@@ -126,10 +126,4 @@ enum class DiskQueueVersion : uint16_t {
 	V2 = 2, // Use xxhash3
 };
 
-IDiskQueue* openDiskQueue(std::string basename,
-                          std::string ext,
-                          UID dbgid,
-                          DiskQueueVersion diskQueueVersion,
-                          int64_t fileSizeWarningLimit = -1); // opens basename+"0."+ext and basename+"1."+ext
-
 #endif
diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbserver/core/include/fdbserver/core/Knobs.h
similarity index 89%
rename from fdbclient/include/fdbclient/ServerKnobs.h
rename to fdbserver/core/include/fdbserver/core/Knobs.h
index 172b913cbd1..1324a98bcef 100644
--- a/fdbclient/include/fdbclient/ServerKnobs.h
+++ b/fdbserver/core/include/fdbserver/core/Knobs.h
@@ -1,5 +1,5 @@
 /*
- * ServerKnobs.h
+ * Knobs.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,13 @@
 
 #pragma once
 
+#include <vector>
+
+#include "fdbclient/KnobValue.h"
 #include "flow/Knobs.h"
 #include "flow/swift_support.h"
 #include "fdbrpc/Locality.h"
-#include "fdbclient/ClientKnobs.h"
+#include "fdbclient/Knobs.h"
 
 // Disk queue
 static constexpr int _PAGE_SIZE = 4096;
@@ -148,6 +151,7 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	double MERGE_RELOCATION_PARALLELISM_PER_TEAM;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
+	int DD_MAX_PIPELINE_MOVES; // Hard cap on total relocations DD tracks (queued + in-flight).
 	int DD_REBALANCE_RESET_AMOUNT;
 	double INFLIGHT_PENALTY_HEALTHY;
 	double INFLIGHT_PENALTY_REDUNDANT;
@@ -437,12 +441,8 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	int SS_BULKLOAD_GETRANGE_BATCH_SIZE; // the max number of keys to scan before do context switch
 	int DD_HEALTHY_ZONE_READ_RETRY_COUNT; // numbers of attempts to read \xff\02/healthyZone on startup
 
-	// Run storage engine on a child process on the same machine with storage process
-	bool REMOTE_KV_STORE;
 	// A delay to avoid race on file resources after seeing lock_file_failure
 	double REBOOT_KV_STORE_DELAY;
-	// max waiting time for the remote kv store to initialize
-	double REMOTE_KV_STORE_MAX_INIT_DURATION;
 
 	// KeyValueStore SQLITE
 	int CLEAR_BUFFER_SIZE;
@@ -532,6 +532,10 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	bool SHARDED_ROCKSDB_REUSE_ITERATORS;
 	bool ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS;
 	int ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT;
+	// By default all reads to rocksdb are cached. If this knob is set to true, based on
+	// txn.ReadOptions.CacheResult, rocksdb caching is enabled/disabled accordingly for that read.
+	bool ROCKSDB_USE_CACHE_RESULT_OPTION;
+	double ROCKSDB_PROBABILITY_DISABLE_CACHE_SIM;
 	int64_t ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC;
 	int ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS;
 	bool ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE;
@@ -760,7 +764,6 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 
 	// Backup Worker
 	double BACKUP_TIMEOUT; // master's reaction time for backup failure
-	double BACKUP_NOOP_POP_DELAY;
 	int BACKUP_FILE_BLOCK_BYTES;
 	int64_t BACKUP_WORKER_LOCK_BYTES;
 	double BACKUP_UPLOAD_DELAY;
@@ -783,7 +786,6 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	double WAIT_FOR_RATEKEEPER_JOIN_DELAY;
 	double WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY;
 	double WAIT_FOR_BLOB_MANAGER_JOIN_DELAY;
-	double WAIT_FOR_ENCRYPT_KEY_PROXY_JOIN_DELAY;
 	double WORKER_FAILURE_TIME;
 	double CHECK_OUTSTANDING_INTERVAL;
 	double INCOMPATIBLE_PEERS_LOGGING_INTERVAL;
@@ -808,7 +810,6 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	                                             // degraded server is considered healthy.
 	double CC_MIN_DEGRADATION_INTERVAL; // The minimum interval that a server is reported as degraded to be considered
 	                                    // as degraded by Cluster Controller.
-	double ENCRYPT_KEY_PROXY_FAILURE_TIME;
 	int CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE; // The maximum number of degraded peers when excluding a server. When the
 	                                        // number of degraded peers is more than this value, we will not exclude
 	                                        // this server since it may because of server overload.
@@ -829,6 +830,8 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	                                               // failover.
 	int CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION; // The maximum number of degraded servers that can trigger a
 	                                               // failover.
+	double CC_FAILOVER_DUE_TO_TPS_LIMIT_DURATION; // The duration a zero TPS limit must persist before triggering
+	                                              // failover.
 	bool CC_ENABLE_ENTIRE_SATELLITE_MONITORING; // When enabled, gray failure tries to detect whether the entire
 	                                            // satellite DC is degraded.
 	int CC_SATELLITE_DEGRADATION_MIN_COMPLAINER; // When the network between primary and satellite becomes bad, all the
@@ -878,6 +881,15 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	                                         // applies to initializing_transaction_servers phase.
 	int CC_RECOVERY_INIT_REQ_MAX_UNFINISHED_RECOVERIES; // Maximum unfinished recoveries after which transaction system
 	                                                    // intilization timeouts above do not apply.
+	bool CLUSTER_HEALTH_METRIC_ENABLE; // Enables periodic cluster-health metric evaluation in CC.
+	double CLUSTER_HEALTH_METRIC_POLL_INTERVAL; // Seconds between cluster-health monitor polls.
+	double CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD; // SS free-space ratio below this needs intervention.
+	double CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD; // SS free-space ratio below this is critical.
+	double CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD; // TLog queue-disk free-space ratio below this needs
+	                                                          // intervention.
+	double CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD; // TLog queue-disk free-space ratio below this is critical.
+	double CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD; // tpsLimit/releasedTps ratio below this is
+	                                                                       // critical.
 
 	// Knobs used to select the best policy (via monte carlo)
 	int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)
@@ -908,6 +920,8 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	double SERVER_READY_QUORUM_TIMEOUT;
 	double REMOVE_RETRY_DELAY;
 	int MOVE_KEYS_KRM_LIMIT;
+	int FINISH_MOVE_KEYS_MAX_RETRIES; // Max retries in finishMoveKeys before returning move to queue; set very high to
+	                                  // disable
 	int MOVE_KEYS_KRM_LIMIT_BYTES; // This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT
 	                               // (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an
 	                               // attempt to read a key range map
@@ -965,8 +979,10 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	int64_t SPRING_BYTES_TLOG;
 	int64_t TARGET_BYTES_PER_TLOG_BATCH;
 	int64_t SPRING_BYTES_TLOG_BATCH;
+	double TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO;
 	int64_t TLOG_SPILL_THRESHOLD;
 	int64_t TLOG_HARD_LIMIT_BYTES;
+	double TLOG_MIN_AVAILABLE_SPACE_RATIO;
 	int64_t TLOG_RECOVER_MEMORY_LIMIT;
 	double TLOG_IGNORE_POP_AUTO_ENABLE_DELAY;
 
@@ -986,44 +1002,6 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	// Limit to the number of throttling tags each storage server
 	// will track and send to the ratekeeper
 	int64_t SS_THROTTLE_TAGS_TRACKED;
-	// Use global tag throttling strategy. i.e. throttle based on the cluster-wide
-	// throughput for tags and their associated quotas.
-	bool GLOBAL_TAG_THROTTLING;
-	// Enforce tag throttling on proxies rather than on clients
-	bool ENFORCE_TAG_THROTTLING_ON_PROXIES;
-	// Minimum number of transactions per second that the global tag throttler must allow for each tag.
-	// When the measured tps for a tag gets too low, the denominator in the
-	// average cost calculation gets small, resulting in an unstable calculation.
-	// To protect against this, we do not compute the average cost when the
-	// measured tps drops below this threshold
-	double GLOBAL_TAG_THROTTLING_MIN_RATE;
-	// Maximum number of tags tracked by global tag throttler. Additional tags will be ignored
-	// until some existing tags expire
-	int64_t GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED;
-	// Global tag throttler forgets about throughput from a tag once no new transactions from that
-	// tag have been received for this duration (in seconds):
-	int64_t GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER;
-	// Interval at which latency bands are logged for each tag on grv proxy
-	double GLOBAL_TAG_THROTTLING_PROXY_LOGGING_INTERVAL;
-	// Interval at which ratekeeper logs statistics for each tag:
-	double GLOBAL_TAG_THROTTLING_TRACE_INTERVAL;
-	// If this knob is set to true, the global tag throttler will still
-	// compute rates, but these rates won't be sent to GRV proxies for
-	// enforcement.
-	bool GLOBAL_TAG_THROTTLING_REPORT_ONLY;
-	// Below this throughput threshold (in bytes/second), ratekeeper will forget about the
-	// throughput of a particular tag on a particular storage server
-	int64_t GLOBAL_TAG_THROTTLING_FORGET_SS_THRESHOLD;
-	// If a tag's throughput on a particular storage server exceeds this threshold,
-	// this storage server's throttling ratio will contribute the calculation of the
-	// throttlingId's limiting transaction rate
-	double GLOBAL_TAG_THROTTLING_LIMITING_THRESHOLD;
-
-	double GLOBAL_TAG_THROTTLING_TARGET_RATE_FOLDING_TIME;
-	double GLOBAL_TAG_THROTTLING_TRANSACTION_COUNT_FOLDING_TIME;
-	double GLOBAL_TAG_THROTTLING_TRANSACTION_RATE_FOLDING_TIME;
-	double GLOBAL_TAG_THROTTLING_COST_FOLDING_TIME;
-
 	bool HOT_SHARD_THROTTLING_ENABLED;
 	double HOT_SHARD_THROTTLING_EXPIRE_AFTER;
 	int64_t HOT_SHARD_THROTTLING_TRACKED;
@@ -1051,18 +1029,6 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	double DURABILITY_LAG_REDUCTION_RATE;
 	double DURABILITY_LAG_INCREASE_RATE;
 	double STORAGE_SERVER_LIST_FETCH_TIMEOUT;
-	bool BW_THROTTLING_ENABLED;
-	double TARGET_BW_LAG;
-	double TARGET_BW_LAG_BATCH;
-	double TARGET_BW_LAG_UPDATE;
-	int MIN_BW_HISTORY;
-	double BW_ESTIMATION_INTERVAL;
-	double BW_LAG_INCREASE_AMOUNT;
-	double BW_LAG_DECREASE_AMOUNT;
-	double BW_FETCH_WORKERS_INTERVAL;
-	double BW_RW_LOGGING_INTERVAL;
-	double BW_MAX_BLOCKED_INTERVAL;
-	double BW_RK_SIM_QUIESCE_DELAY;
 
 	// disk snapshot
 	int64_t MAX_FORKED_PROCESS_OUTPUT;
@@ -1121,6 +1087,9 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	int AUDIT_DATAMOVE_POST_CHECK_RETRY_COUNT_MAX;
 	int AUDIT_STORAGE_RATE_PER_SERVER_MAX;
 	bool ENABLE_AUDIT_VERBOSE_TRACE;
+	int AUDIT_RESTORE_BATCH_KEY_LIMIT;
+	int64_t AUDIT_PROGRESS_PERSIST_BYTES_INTERVAL;
+	double AUDIT_LOCATION_METADATA_INTERVAL;
 	bool LOGGING_STORAGE_COMMIT_WHEN_IO_TIMEOUT;
 	double LOGGING_COMPLETE_STORAGE_COMMIT_PROBABILITY;
 	int LOGGING_RECENT_STORAGE_COMMIT_SIZE;
@@ -1184,6 +1153,7 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	int FETCH_SHARD_BUFFER_BYTE_LIMIT;
 	int FETCH_SHARD_UPDATES_BYTE_LIMIT;
 	bool TRACK_READ_LATENCIES_PER_TYPE;
+	int64_t STORAGE_UPDATE_PROCESS_STATS_INTERVAL;
 
 	// Wait Failure
 	int MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS;
@@ -1265,6 +1235,7 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	// Status
 	double STATUS_MIN_TIME_BETWEEN_REQUESTS;
 	double MAX_STATUS_REQUESTS_PER_SECOND;
+	double STATUS_TIMEOUT_BUFFER;
 	int CONFIGURATION_ROWS_TO_FETCH;
 	bool DISABLE_DUPLICATE_LOG_WARNING;
 	double HISTOGRAM_REPORT_INTERVAL;
@@ -1274,59 +1245,6 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	int64_t TIME_KEEPER_DELAY;
 	int64_t TIME_KEEPER_MAX_ENTRIES;
 
-	// Fast Restore
-	// TODO: After 6.3, review FR knobs, remove unneeded ones and change default value
-	// TODO(gglass): revisit the above FR
-	int64_t FASTRESTORE_FAILURE_TIMEOUT;
-	int64_t FASTRESTORE_HEARTBEAT_INTERVAL;
-	double FASTRESTORE_SAMPLING_PERCENT;
-	int64_t FASTRESTORE_NUM_LOADERS;
-	int64_t FASTRESTORE_NUM_APPLIERS;
-	// FASTRESTORE_TXN_BATCH_MAX_BYTES is target txn size used by appliers to apply mutations
-	double FASTRESTORE_TXN_BATCH_MAX_BYTES;
-	// FASTRESTORE_VERSIONBATCH_MAX_BYTES is the maximum data size in each version batch
-	double FASTRESTORE_VERSIONBATCH_MAX_BYTES;
-	// FASTRESTORE_VB_PARALLELISM is the number of concurrently running version batches
-	int64_t FASTRESTORE_VB_PARALLELISM;
-	int64_t FASTRESTORE_VB_MONITOR_DELAY; // How quickly monitor finished version batch
-	double FASTRESTORE_VB_LAUNCH_DELAY;
-	int64_t FASTRESTORE_ROLE_LOGGING_DELAY;
-	int64_t FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL; // How quickly to update process metrics for restore
-	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
-	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
-	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
-	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
-	int64_t FASTRESTORE_WAIT_FOR_MEMORY_LATENCY;
-	int64_t FASTRESTORE_HEARTBEAT_DELAY; // interval for master to ping loaders and appliers
-	int64_t
-	    FASTRESTORE_HEARTBEAT_MAX_DELAY; // master claim a node is down if no heart beat from the node for this delay
-	int64_t FASTRESTORE_APPLIER_FETCH_KEYS_SIZE; // number of keys to fetch in a txn on applier
-	int64_t FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES; // desired size of mutation message sent from loader to appliers
-	bool FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE; // parse each range file to get (range, version) it has?
-	int64_t FASTRESTORE_REQBATCH_PARALLEL; // number of requests to wait on for getBatchReplies()
-	bool FASTRESTORE_REQBATCH_LOG; // verbose log information for getReplyBatches
-	int FASTRESTORE_TXN_CLEAR_MAX; // threshold to start tracking each clear op in a txn
-	int FASTRESTORE_TXN_RETRY_MAX; // threshold to start output error on too many retries
-	double FASTRESTORE_TXN_EXTRA_DELAY; // extra delay to avoid overwhelming fdb
-	bool FASTRESTORE_NOT_WRITE_DB; // do not write result to DB. Only for dev testing
-	bool FASTRESTORE_USE_RANGE_FILE; // use range file in backup
-	bool FASTRESTORE_USE_LOG_FILE; // use log file in backup
-	int64_t FASTRESTORE_SAMPLE_MSG_BYTES; // sample message desired size
-	double FASTRESTORE_SCHED_UPDATE_DELAY; // delay in seconds in updating process metrics
-	int FASTRESTORE_SCHED_TARGET_CPU_PERCENT; // release as many requests as possible when cpu usage is below the knob
-	int FASTRESTORE_SCHED_MAX_CPU_PERCENT; // max cpu percent when scheduler shall not release non-urgent requests
-	int FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS; // number of inflight requests to load backup files
-	int FASTRESTORE_SCHED_INFLIGHT_SEND_REQS; // number of inflight requests for loaders to  send mutations to appliers
-	int FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE; // number of load request to release at once
-	int FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD; // we can send future VB requests if it is less than this knob
-	int FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH; // number of future VB sendLoadingParam requests to process at once
-	int FASTRESTORE_NUM_TRACE_EVENTS;
-	bool FASTRESTORE_EXPENSIVE_VALIDATION; // when set true, performance will be heavily affected
-	double FASTRESTORE_WRITE_BW_MB; // target aggregated write bandwidth from all appliers
-	double FASTRESTORE_RATE_UPDATE_SECONDS; // how long to update appliers target write rate
-	bool FASTRESTORE_DUMP_INSERT_RANGE_VERSION; // Dump all the range version after insertion. This is for debugging
-	                                            // purpose.
-
 	int REDWOOD_DEFAULT_PAGE_SIZE; // Page size for new Redwood files
 	int REDWOOD_DEFAULT_EXTENT_SIZE; // Extent size for new Redwood files
 	int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files
@@ -1373,33 +1291,9 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	double ENCRYPTION_LOGGING_INTERVAL;
 	double DISABLED_ENCRYPTION_PROBABILITY_SIM; // Probability that encryption is forced to be disabled in simulation
 
-	// Key Management Service (KMS) Connector
-	std::string KMS_CONNECTOR_TYPE;
-
 	// Blob metadata
 	int64_t BLOB_METADATA_CACHE_TTL;
 
-	// HTTP KMS Connector
-	std::string REST_KMS_CONNECTOR_KMS_DISCOVERY_URL_MODE;
-	std::string REST_KMS_CONNECTOR_DISCOVER_KMS_URL_FILE;
-	std::string REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE;
-	std::string REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS;
-	bool ENABLE_REST_KMS_COMMUNICATION;
-	bool REST_KMS_CONNECTOR_REMOVE_TRAILING_NEWLINE;
-	int REST_KMS_CONNECTOR_VALIDATION_TOKEN_MAX_SIZE;
-	int REST_KMS_CONNECTOR_VALIDATION_TOKENS_MAX_PAYLOAD_SIZE;
-	bool REST_KMS_CONNECTOR_REFRESH_KMS_URLS;
-	double REST_KMS_CONNECTOR_REFRESH_KMS_URLS_INTERVAL_SEC;
-	std::string REST_KMS_CONNECTOR_GET_ENCRYPTION_KEYS_ENDPOINT;
-	std::string REST_KMS_CONNECTOR_GET_LATEST_ENCRYPTION_KEYS_ENDPOINT;
-	std::string REST_KMS_CONNECTOR_GET_BLOB_METADATA_ENDPOINT;
-	int REST_KMS_CURRENT_BLOB_METADATA_REQUEST_VERSION;
-	int REST_KMS_MAX_BLOB_METADATA_REQUEST_VERSION;
-	int REST_KMS_CURRENT_CIPHER_REQUEST_VERSION;
-	int REST_KMS_MAX_CIPHER_REQUEST_VERSION;
-	std::string REST_SIM_KMS_VAULT_DIR;
-	double REST_KMS_STABILITY_CHECK_INTERVAL;
-
 	double CONSISTENCY_SCAN_ACTIVE_THROTTLE_RATIO;
 
 	// Idempotency ids
@@ -1415,3 +1309,17 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl<ServerKno
 	ServerKnobs(Randomize, ClientKnobs*, IsSimulated);
 	void initialize(Randomize, ClientKnobs*, IsSimulated);
 };
+
+extern ServerKnobs const* SERVER_KNOBS;
+
+inline const ServerKnobs& getServerKnobs() {
+	return *SERVER_KNOBS;
+}
+
+void resetServerKnobs(Randomize randomize, IsSimulated isSimulated);
+void initializeServerKnobs(Randomize randomize, IsSimulated isSimulated);
+Optional<KnobValue> tryParseServerKnobValue(std::string const& knobName, std::string const& knobValue);
+KnobValue parseServerKnobValue(std::string const& knobName, std::string const& knobValue);
+bool trySetServerKnob(std::string const& knobName, KnobValueRef const& knobValue);
+void setServerKnob(std::string const& knobName, KnobValueRef const& knobValue);
+void setupServerKnobs(std::vector<std::pair<std::string, std::string>> const& knobs);
diff --git a/fdbserver/include/fdbserver/LatencyBandConfig.h b/fdbserver/core/include/fdbserver/core/LatencyBandConfig.h
similarity index 100%
rename from fdbserver/include/fdbserver/LatencyBandConfig.h
rename to fdbserver/core/include/fdbserver/core/LatencyBandConfig.h
diff --git a/fdbserver/include/fdbserver/LeaderElection.h b/fdbserver/core/include/fdbserver/core/LeaderElection.h
similarity index 100%
rename from fdbserver/include/fdbserver/LeaderElection.h
rename to fdbserver/core/include/fdbserver/core/LeaderElection.h
diff --git a/fdbserver/include/fdbserver/LogProtocolMessage.h b/fdbserver/core/include/fdbserver/core/LogProtocolMessage.h
similarity index 100%
rename from fdbserver/include/fdbserver/LogProtocolMessage.h
rename to fdbserver/core/include/fdbserver/core/LogProtocolMessage.h
diff --git a/fdbserver/include/fdbserver/LogSystemConfig.h b/fdbserver/core/include/fdbserver/core/LogSystemConfig.h
similarity index 97%
rename from fdbserver/include/fdbserver/LogSystemConfig.h
rename to fdbserver/core/include/fdbserver/core/LogSystemConfig.h
index 7c2ceb3493b..93f9d9472be 100644
--- a/fdbserver/include/fdbserver/LogSystemConfig.h
+++ b/fdbserver/core/include/fdbserver/core/LogSystemConfig.h
@@ -22,8 +22,8 @@
 #define FDBSERVER_LOGSYSTEMCONFIG_H
 #pragma once
 
-#include "fdbserver/BackupInterface.h"
-#include "fdbserver/TLogInterface.h"
+#include "fdbserver/core/BackupInterface.h"
+#include "fdbserver/core/TLogInterface.h"
 #include "fdbrpc/ReplicationPolicy.h"
 #include "fdbclient/DatabaseConfiguration.h"
 
@@ -66,7 +66,6 @@ void OptionalInterface<Interface>::serialize(Ar& ar) {
 		ident = iface.get().id();
 }
 
-class LogSet;
 struct OldLogData;
 
 template <class Interface>
@@ -105,8 +104,6 @@ struct TLogSet {
 	TLogSet()
 	  : tLogWriteAntiQuorum(0), tLogReplicationFactor(0), isLocal(true), locality(tagLocalityInvalid),
 	    startVersion(invalidVersion) {}
-	explicit TLogSet(const LogSet& rhs);
-
 	std::string toString() const;
 
 	bool operator==(const TLogSet& rhs) const;
@@ -146,8 +143,6 @@ struct OldTLogConf {
 	LogEpoch epoch;
 
 	OldTLogConf() : epochBegin(0), epochEnd(0), recoverAt(0), logRouterTags(0), txsTags(0), epoch(0) {}
-	explicit OldTLogConf(const OldLogData&);
-
 	std::string toString() const {
 		return format("end: %d tags: %d %s", epochEnd, logRouterTags, describe(tLogs).c_str());
 	}
@@ -186,7 +181,7 @@ struct LogSystemConfig {
 	LogEpoch oldestBackupEpoch;
 	std::map<uint8_t, std::vector<uint16_t>> knownLockedTLogIds;
 
-	LogSystemConfig(LogEpoch e = 0)
+	explicit LogSystemConfig(LogEpoch e = 0)
 	  : logSystemType(LogSystemType::empty), logRouterTags(0), txsTags(0), expectedLogSets(0), stopped(false), epoch(e),
 	    oldestBackupEpoch(e) {}
 
diff --git a/fdbserver/include/fdbserver/MasterInterface.h b/fdbserver/core/include/fdbserver/core/MasterInterface.h
similarity index 98%
rename from fdbserver/include/fdbserver/MasterInterface.h
rename to fdbserver/core/include/fdbserver/core/MasterInterface.h
index d932e7faf6d..3d22bd76450 100644
--- a/fdbserver/include/fdbserver/MasterInterface.h
+++ b/fdbserver/core/include/fdbserver/core/MasterInterface.h
@@ -26,12 +26,11 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/VersionVector.h"
-#include "fdbserver/TLogInterface.h"
+#include "fdbserver/core/TLogInterface.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbserver/ResolverInterface.h"
-#include "fdbserver/TLogInterface.h"
+#include "fdbserver/core/ResolverInterface.h"
 #include "flow/swift_support.h"
 
 #ifdef WITH_SWIFT
@@ -289,7 +288,7 @@ struct CommitProxyVersionReplies {
 	MAP_UInt64_GetCommitVersionReply replies;
 	NotifiedVersion latestRequestNum;
 
-	CommitProxyVersionReplies(CommitProxyVersionReplies&& r) noexcept
+	explicit(false) CommitProxyVersionReplies(CommitProxyVersionReplies&& r) noexcept
 	  : replies(std::move(r.replies)), latestRequestNum(std::move(r.latestRequestNum)) {}
 
 	void operator=(CommitProxyVersionReplies&& r) noexcept {
diff --git a/fdbserver/include/fdbserver/MoveKeys.actor.h b/fdbserver/core/include/fdbserver/core/MoveKeys.h
similarity index 83%
rename from fdbserver/include/fdbserver/MoveKeys.actor.h
rename to fdbserver/core/include/fdbserver/core/MoveKeys.h
index 27a0d441cda..f9b9c5bf933 100644
--- a/fdbserver/include/fdbserver/MoveKeys.actor.h
+++ b/fdbserver/core/include/fdbserver/core/MoveKeys.h
@@ -1,5 +1,5 @@
 /*
- * MoveKeys.actor.h
+ * MoveKeys.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,19 +18,16 @@
  * limitations under the License.
  */
 
+#ifndef FDBSERVER_CORE_MOVEKEYS_H
+#define FDBSERVER_CORE_MOVEKEYS_H
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_MOVEKEYS_ACTOR_G_H)
-#define FDBSERVER_MOVEKEYS_ACTOR_G_H
-#include "fdbserver/MoveKeys.actor.g.h"
-#elif !defined(FDBSERVER_MOVEKEYS_ACTOR_H)
-#define FDBSERVER_MOVEKEYS_ACTOR_H
 
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/SeedShardServers.h"
 #include "flow/BooleanParam.h"
-#include "flow/actorcompiler.h"
 
 FDB_BOOLEAN_PARAM(CancelConflictingDataMoves);
 
@@ -134,18 +131,16 @@ struct MoveKeysParams {
 };
 
 // read the lock value in system keyspace but do not change anything
-ACTOR Future<MoveKeysLock> readMoveKeysLock(Database cx);
+Future<MoveKeysLock> readMoveKeysLock(Database cx);
 
 // Calling moveKeys, etc with the return value of this actor ensures that no movekeys, etc
 // has been executed by a different locker since takeMoveKeysLock(), as calling
 // takeMoveKeysLock() updates "moveKeysLockOwnerKey" to a random UID.
-ACTOR Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId);
+Future<MoveKeysLock> takeMoveKeysLock(Database cx, UID ddId);
 
 // Checks that the a moveKeysLock has not changed since having taken it
 // This does not modify the moveKeysLock
 Future<Void> checkMoveKeysLockReadOnly(Transaction* tr, MoveKeysLock lock, const DDEnabledState* ddEnabledState);
-
-void seedShardServers(Arena& trArena, CommitTransactionRef& tr, std::vector<StorageServerInterface> servers);
 // Called by the master server to write the very first transaction to the database
 // establishing a set of shard servers and all invariants of the systemKeys.
 
@@ -161,10 +156,10 @@ Future<Void> rawFinishMovement(Database occ,
 // for restarting the remainder, and for not otherwise cancelling it before
 // it returns (since it needs to execute the finishMoveKeys transaction).
 // When dataMoveId.isValid(), the keyrange will be moved to a shard designated as dataMoveId.
-ACTOR Future<Void> moveKeys(Database occ, MoveKeysParams params);
+Future<Void> moveKeys(Database occ, MoveKeysParams params);
 
 // Cancels a data move designated by dataMoveId.
-ACTOR Future<Void> cleanUpDataMove(
+Future<Void> cleanUpDataMove(
     Database occ,
     UID dataMoveId,
     MoveKeysLock lock,
@@ -173,28 +168,28 @@ ACTOR Future<Void> cleanUpDataMove(
     const DDEnabledState* ddEnabledState,
     Optional<PromiseStream<Future<Void>>> addCleanUpDataMoveActor = Optional<PromiseStream<Future<Void>>>());
 
-ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServerInterface server);
+Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServerInterface server);
 // Adds a newly recruited storage server to a database (e.g. adding it to FF/serverList)
 // Returns a Version in which the storage server is in the database
 // This doesn't need to be called for the "seed" storage servers (see seedShardServers above)
 
-ACTOR Future<Void> removeStorageServer(Database cx,
-                                       UID serverID,
-                                       Optional<UID> tssPairID, // if serverID is a tss, set to its ss pair id
-                                       MoveKeysLock lock,
-                                       const DDEnabledState* ddEnabledState);
+Future<Void> removeStorageServer(Database cx,
+                                 UID serverID,
+                                 Optional<UID> tssPairID, // if serverID is a tss, set to its ss pair id
+                                 MoveKeysLock lock,
+                                 const DDEnabledState* ddEnabledState);
 // Removes the given storage server permanently from the database.  It must already
 // have no shards assigned to it.  The storage server MUST NOT be added again after this
 // (though a new storage server with a new unique ID may be recruited from the same fdbserver).
 
-ACTOR Future<bool> canRemoveStorageServer(Reference<ReadYourWritesTransaction> tr, UID serverID);
+Future<bool> canRemoveStorageServer(Reference<ReadYourWritesTransaction> tr, UID serverID);
 // Returns true if the given storage server has no keys assigned to it and may be safely removed
 // Obviously that could change later!
-ACTOR Future<Void> removeKeysFromFailedServer(Database cx,
-                                              UID serverID,
-                                              std::vector<UID> teamForDroppedRange,
-                                              MoveKeysLock lock,
-                                              const DDEnabledState* ddEnabledState);
+Future<Void> removeKeysFromFailedServer(Database cx,
+                                        UID serverID,
+                                        std::vector<UID> teamForDroppedRange,
+                                        MoveKeysLock lock,
+                                        const DDEnabledState* ddEnabledState);
 // Directly removes serverID from serverKeys and keyServers system keyspace.
 // Performed when a storage server is marked as permanently failed.
 
@@ -203,5 +198,4 @@ Future<Void> checkMoveKeysLock(Transaction* tr,
                                const DDEnabledState* ddEnabledState,
                                bool isWrite = true);
 
-#include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbserver/include/fdbserver/MutationTracking.h b/fdbserver/core/include/fdbserver/core/MutationTracking.h
similarity index 100%
rename from fdbserver/include/fdbserver/MutationTracking.h
rename to fdbserver/core/include/fdbserver/core/MutationTracking.h
diff --git a/fdbserver/include/fdbserver/OTELSpanContextMessage.h b/fdbserver/core/include/fdbserver/core/OTELSpanContextMessage.h
similarity index 95%
rename from fdbserver/include/fdbserver/OTELSpanContextMessage.h
rename to fdbserver/core/include/fdbserver/core/OTELSpanContextMessage.h
index bb3283281c5..65bafdbb98b 100644
--- a/fdbserver/include/fdbserver/OTELSpanContextMessage.h
+++ b/fdbserver/core/include/fdbserver/core/OTELSpanContextMessage.h
@@ -40,7 +40,7 @@ struct OTELSpanContextMessage {
 	SpanContext spanContext;
 
 	OTELSpanContextMessage() {}
-	OTELSpanContextMessage(SpanContext const& spanContext) : spanContext(spanContext) {}
+	explicit OTELSpanContextMessage(SpanContext const& spanContext) : spanContext(spanContext) {}
 
 	std::string toString() const {
 		return format("code: %d, span context: %s",
diff --git a/fdbserver/core/include/fdbserver/core/QuietDatabase.h b/fdbserver/core/include/fdbserver/core/QuietDatabase.h
new file mode 100644
index 00000000000..4fa8e4a8125
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/QuietDatabase.h
@@ -0,0 +1,61 @@
+/*
+ * QuietDatabase.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+
+Future<int64_t> getDataInFlight(Database cx, Reference<AsyncVar<struct ServerDBInfo> const> dbInfo);
+Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database cx,
+                                                     Reference<AsyncVar<struct ServerDBInfo> const> dbInfo);
+Future<int64_t> getMaxStorageServerQueueSize(Database cx,
+                                             Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
+                                             Version version);
+Future<int64_t> getDataDistributionQueueSize(Database cx,
+                                             Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
+                                             bool reportInFlight);
+Future<bool> getTeamCollectionValid(Database cx, WorkerInterface distributorWorker);
+Future<bool> getTeamCollectionValid(Database cx, Reference<AsyncVar<struct ServerDBInfo> const> dbInfo);
+Future<std::vector<StorageServerInterface>> getStorageServers(Database cx, bool use_system_priority = false);
+Future<std::vector<WorkerDetails>> getWorkers(Reference<AsyncVar<ServerDBInfo> const> dbInfo, int flags = 0);
+Future<WorkerInterface> getMasterWorker(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
+Future<Void> repairDeadDatacenter(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo, std::string context);
+Future<Void> reconfigureAfter(Database cx,
+                              double time,
+                              Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                              std::string context);
+
+Future<std::pair<std::vector<WorkerInterface>, int>> getStorageWorkers(Database cx,
+                                                                       Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                                                       bool localOnly);
+Future<std::vector<WorkerInterface>> getCoordWorkers(Database cx, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
+
+Future<Void> enableConsistencyScanInSim(Database db);
+Future<Void> disableConsistencyScanInSim(Database db, bool waitForCompletion);
+
+// Permanently disables DD pipeline control so that all blocked relocations pass through.
+// For use by the test harness to allow DD to quiesce after tests complete.
+// Uses a plain boolean (not AsyncVar) to avoid cross-process callback issues in simulation.
+void disableDDPipelineControl();
+bool isDDPipelineControlEnabled();
diff --git a/fdbserver/include/fdbserver/RatekeeperInterface.h b/fdbserver/core/include/fdbserver/core/RatekeeperInterface.h
similarity index 93%
rename from fdbserver/include/fdbserver/RatekeeperInterface.h
rename to fdbserver/core/include/fdbserver/core/RatekeeperInterface.h
index c655ac0b135..a533f215421 100644
--- a/fdbserver/include/fdbserver/RatekeeperInterface.h
+++ b/fdbserver/core/include/fdbserver/core/RatekeeperInterface.h
@@ -80,20 +80,11 @@ struct GetRateInfoReply {
 	double leaseDuration;
 	HealthMetrics healthMetrics;
 
-	// Depending on the value of SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES,
-	// one of these fields may be populated
 	Optional<PrioritizedTransactionTagMap<ClientTagThrottleLimits>> clientThrottledTags;
-	Optional<TransactionTagMap<double>> proxyThrottledTags;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar,
-		           transactionRate,
-		           batchTransactionRate,
-		           leaseDuration,
-		           healthMetrics,
-		           clientThrottledTags,
-		           proxyThrottledTags);
+		serializer(ar, transactionRate, batchTransactionRate, leaseDuration, healthMetrics, clientThrottledTags);
 	}
 };
 
diff --git a/fdbserver/core/include/fdbserver/core/RatekeeperLimitReasons.h b/fdbserver/core/include/fdbserver/core/RatekeeperLimitReasons.h
new file mode 100644
index 00000000000..f44aa963f44
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/RatekeeperLimitReasons.h
@@ -0,0 +1,44 @@
+/*
+ * RatekeeperLimitReasons.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_CORE_RATEKEEPERLIMITREASONS_H
+#define FDBSERVER_CORE_RATEKEEPERLIMITREASONS_H
+
+enum limitReason_t {
+	unlimited,
+	storage_server_write_queue_size,
+	storage_server_write_bandwidth_mvcc,
+	storage_server_readable_behind,
+	log_server_mvcc_write_bandwidth,
+	log_server_write_queue,
+	storage_server_min_free_space,
+	storage_server_min_free_space_ratio,
+	log_server_min_free_space,
+	log_server_min_free_space_ratio,
+	storage_server_durability_lag,
+	storage_server_list_fetch_failed,
+	limitReason_t_end
+};
+
+extern int limitReasonEnd;
+extern const char* limitReasonName[];
+extern const char* limitReasonDesc[];
+
+#endif
diff --git a/fdbserver/include/fdbserver/RecoveryState.h b/fdbserver/core/include/fdbserver/core/RecoveryState.h
similarity index 98%
rename from fdbserver/include/fdbserver/RecoveryState.h
rename to fdbserver/core/include/fdbserver/core/RecoveryState.h
index cede507bcc3..d89a53fd10c 100644
--- a/fdbserver/include/fdbserver/RecoveryState.h
+++ b/fdbserver/core/include/fdbserver/core/RecoveryState.h
@@ -61,7 +61,7 @@ enum RecoveryStatus {
 	END
 };
 
-// in Status.actor.cpp
+// in Status.cpp
 extern const char* names[];
 extern const char* descriptions[];
 }; // namespace RecoveryStatus
diff --git a/fdbserver/include/fdbserver/ResolverInterface.h b/fdbserver/core/include/fdbserver/core/ResolverInterface.h
similarity index 100%
rename from fdbserver/include/fdbserver/ResolverInterface.h
rename to fdbserver/core/include/fdbserver/core/ResolverInterface.h
diff --git a/fdbserver/include/fdbserver/RocksDBCheckpointUtils.actor.h b/fdbserver/core/include/fdbserver/core/RocksDBCheckpointUtils.h
similarity index 93%
rename from fdbserver/include/fdbserver/RocksDBCheckpointUtils.actor.h
rename to fdbserver/core/include/fdbserver/core/RocksDBCheckpointUtils.h
index aaae2c6390f..b7d7c1dfa8c 100644
--- a/fdbserver/include/fdbserver/RocksDBCheckpointUtils.actor.h
+++ b/fdbserver/core/include/fdbserver/core/RocksDBCheckpointUtils.h
@@ -1,5 +1,5 @@
 /*
- *RocksDBCheckpointUtils.actor.h
+ *RocksDBCheckpointUtils.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,18 +19,11 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_ROCKSDB_CHECKPOINT_UTILS_ACTOR_G_H)
-#define FDBSERVER_ROCKSDB_CHECKPOINT_UTILS_ACTOR_G_H
-#include "fdbserver/RocksDBCheckpointUtils.actor.g.h"
-#elif !defined(FDBSERVER_ROCKSDB_CHECKPOINT_UTILS_ACTOR_H)
-#define FDBSERVER_ROCKSDB_CHECKPOINT_UTILS_ACTOR_H
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/ServerCheckpoint.actor.h"
+#include "fdbserver/core/ServerCheckpoint.h"
 #include "flow/flow.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 class ICheckpointByteSampleReader {
 public:
 	virtual ~ICheckpointByteSampleReader() {}
@@ -318,7 +311,7 @@ struct RocksDBCheckpointKeyValues {
 	std::vector<CheckpointFile> fetchedFiles; // Used for fetchCheckpoint, to record the progress.
 	std::vector<KeyRange> ranges; // The ranges we want to fetch.
 
-	RocksDBCheckpointKeyValues(std::vector<KeyRange> ranges) : ranges(ranges) {}
+	explicit RocksDBCheckpointKeyValues(std::vector<KeyRange> ranges) : ranges(ranges) {}
 	RocksDBCheckpointKeyValues() = default;
 
 	CheckpointFormat format() const { return RocksDBKeyValues; }
@@ -340,16 +333,16 @@ struct RocksDBCheckpointKeyValues {
 // Fetch the checkpoint file(s) to local dir, the checkpoint is specified by initialState.
 // If cFun is provided, the fetch progress can be checkpointed, so that next time, the fetch process
 // can be continued, in case of crash.
-ACTOR Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
-                                                        CheckpointMetaData initialState,
-                                                        std::string dir,
-                                                        std::function<Future<Void>(const CheckpointMetaData&)> cFun);
+Future<CheckpointMetaData> fetchRocksDBCheckpoint(Database cx,
+                                                  CheckpointMetaData initialState,
+                                                  std::string dir,
+                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun);
 
 // Returns the total logical bytes of all *fetched* checkpoints.
 int64_t getTotalFetchedBytes(const std::vector<CheckpointMetaData>& checkpoints);
 
 // Clean up on-disk files associated with checkpoint.
-ACTOR Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint);
+Future<Void> deleteRocksCheckpoint(CheckpointMetaData checkpoint);
 
 ICheckpointReader* newRocksDBCheckpointReader(const CheckpointMetaData& checkpoint,
                                               const CheckpointAsKeyValues checkpointAsKeyValues,
@@ -370,7 +363,3 @@ RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint);
 RocksDBCheckpoint getRocksCheckpoint(const CheckpointMetaData& checkpoint);
 
 RocksDBCheckpointKeyValues getRocksKeyValuesCheckpoint(const CheckpointMetaData& checkpoint);
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbserver/core/include/fdbserver/core/SeedShardServers.h b/fdbserver/core/include/fdbserver/core/SeedShardServers.h
new file mode 100644
index 00000000000..b13ae404f38
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/SeedShardServers.h
@@ -0,0 +1,32 @@
+/*
+ * SeedShardServers.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_CORE_SEEDSHARDSERVERS_H
+#define FDBSERVER_CORE_SEEDSHARDSERVERS_H
+#pragma once
+
+#include <vector>
+
+#include "fdbclient/CommitTransaction.h"
+#include "fdbclient/StorageServerInterface.h"
+
+void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vector<StorageServerInterface> servers);
+
+#endif
diff --git a/fdbserver/include/fdbserver/ServerCheckpoint.actor.h b/fdbserver/core/include/fdbserver/core/ServerCheckpoint.h
similarity index 79%
rename from fdbserver/include/fdbserver/ServerCheckpoint.actor.h
rename to fdbserver/core/include/fdbserver/core/ServerCheckpoint.h
index 55d20ad69c4..7ab6bf47e97 100644
--- a/fdbserver/include/fdbserver/ServerCheckpoint.actor.h
+++ b/fdbserver/core/include/fdbserver/core/ServerCheckpoint.h
@@ -1,5 +1,5 @@
 /*
- *ServerCheckpoint.actor.h
+ *ServerCheckpoint.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,18 +19,11 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_SERVER_CHECKPOINT_ACTOR_G_H)
-#define FDBSERVER_SERVER_CHECKPOINT_ACTOR_G_H
-#include "fdbserver/ServerCheckpoint.actor.g.h"
-#elif !defined(FDBSERVER_SERVER_CHECKPOINT_ACTOR_H)
-#define FDBSERVER_SERVER_CHECKPOINT_ACTOR_H
 
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/StorageCheckpoint.h"
 #include "flow/flow.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 FDB_BOOLEAN_PARAM(CheckpointAsKeyValues);
 
 class ICheckpointIterator {
@@ -69,26 +62,22 @@ ICheckpointReader* newCheckpointReader(const CheckpointMetaData& checkpoint,
                                        UID logID);
 
 // Delete a checkpoint.
-ACTOR Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint);
+Future<Void> deleteCheckpoint(CheckpointMetaData checkpoint);
 
 // Fetches checkpoint to a local `dir`, `initialState` provides the checkpoint formats, location, restart point, etc.
 // If cFun is provided, the progress can be checkpointed.
 // Returns a CheckpointMetaData, which could contain KVS-specific results, e.g., the list of fetched checkpoint files.
-ACTOR Future<CheckpointMetaData> fetchCheckpoint(Database cx,
+Future<CheckpointMetaData> fetchCheckpoint(Database cx,
+                                           CheckpointMetaData initialState,
+                                           std::string dir,
+                                           std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
+
+// Same as above, except that the checkpoint is fetched as key-value pairs.
+Future<CheckpointMetaData> fetchCheckpointRanges(Database cx,
                                                  CheckpointMetaData initialState,
                                                  std::string dir,
+                                                 std::vector<KeyRange> ranges,
                                                  std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
 
-// Same as above, except that the checkpoint is fetched as key-value pairs.
-ACTOR Future<CheckpointMetaData> fetchCheckpointRanges(
-    Database cx,
-    CheckpointMetaData initialState,
-    std::string dir,
-    std::vector<KeyRange> ranges,
-    std::function<Future<Void>(const CheckpointMetaData&)> cFun = nullptr);
-
 std::string serverCheckpointDir(const std::string& baseDir, const UID& checkpointId);
 std::string fetchedCheckpointDir(const std::string& baseDir, const UID& checkpointId);
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbserver/include/fdbserver/ServerDBInfo.actor.h b/fdbserver/core/include/fdbserver/core/ServerDBInfo.h
similarity index 78%
rename from fdbserver/include/fdbserver/ServerDBInfo.actor.h
rename to fdbserver/core/include/fdbserver/core/ServerDBInfo.h
index 072dd12b99c..5d94835bb3c 100644
--- a/fdbserver/include/fdbserver/ServerDBInfo.actor.h
+++ b/fdbserver/core/include/fdbserver/core/ServerDBInfo.h
@@ -1,5 +1,5 @@
 /*
- * ServerDBInfo.actor.h
+ * ServerDBInfo.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,23 +18,18 @@
  * limitations under the License.
  */
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_SERVERDBINFO_ACTOR_G_H)
-#define FDBSERVER_SERVERDBINFO_ACTOR_G_H
-#include "fdbserver/ServerDBInfo.actor.g.h"
-#elif !defined(FDBSERVER_SERVERDBINFO_ACTOR_H)
-#define FDBSERVER_SERVERDBINFO_ACTOR_H
-#define FDBSERVER_SERVERDBINFO_H
+#ifndef FDBSERVER_CORE_SERVERDBINFO_H
+#define FDBSERVER_CORE_SERVERDBINFO_H
 #pragma once
 
-#include "fdbclient/ConsistencyScanInterface.actor.h"
-#include "fdbserver/DataDistributorInterface.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/LogSystemConfig.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/LatencyBandConfig.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/ConsistencyScanInterface.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/core/LatencyBandConfig.h"
+#include "fdbserver/core/LogSystemConfig.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 
 struct ServerDBInfo {
 	constexpr static FileIdentifier file_identifier = 13838807;
@@ -120,12 +115,11 @@ struct GetServerDBInfoRequest {
 extern template class RequestStream<GetServerDBInfoRequest, false>;
 extern template struct NetNotifiedQueue<GetServerDBInfoRequest, false>;
 
-ACTOR Future<Void> broadcastTxnRequest(TxnStateRequest req, int sendAmount, bool sendReply);
+Future<Void> broadcastTxnRequest(TxnStateRequest req, int sendAmount, bool sendReply);
 
-ACTOR Future<std::vector<Endpoint>> broadcastDBInfoRequest(UpdateServerDBInfoRequest req,
-                                                           int sendAmount,
-                                                           Optional<Endpoint> sender,
-                                                           bool sendReply);
+Future<std::vector<Endpoint>> broadcastDBInfoRequest(UpdateServerDBInfoRequest req,
+                                                     int sendAmount,
+                                                     Optional<Endpoint> sender,
+                                                     bool sendReply);
 
-#include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbserver/core/include/fdbserver/core/ShardMetrics.h b/fdbserver/core/include/fdbserver/core/ShardMetrics.h
new file mode 100644
index 00000000000..6c8997209b9
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/ShardMetrics.h
@@ -0,0 +1,49 @@
+/*
+ * ShardMetrics.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_CORE_SHARDMETRICS_H
+#define FDBSERVER_CORE_SHARDMETRICS_H
+#pragma once
+
+#include "fdbclient/StorageServerInterface.h"
+#include "flow/flow.h"
+
+struct ShardMetrics {
+	StorageMetrics metrics;
+	double lastLowBandwidthStartTime;
+	int shardCount; // number of smaller shards whose metrics are aggregated in the ShardMetrics
+
+	bool operator==(ShardMetrics const& rhs) const {
+		return metrics == rhs.metrics && lastLowBandwidthStartTime == rhs.lastLowBandwidthStartTime &&
+		       shardCount == rhs.shardCount;
+	}
+
+	ShardMetrics(StorageMetrics const& metrics, double lastLowBandwidthStartTime, int shardCount)
+	  : metrics(metrics), lastLowBandwidthStartTime(lastLowBandwidthStartTime), shardCount(shardCount) {}
+};
+
+struct ShardTrackedData {
+	Future<Void> trackShard;
+	Future<Void> trackBytes;
+	Future<Void> trackUsableRegion;
+	Reference<AsyncVar<Optional<ShardMetrics>>> stats;
+};
+
+#endif // FDBSERVER_CORE_SHARDMETRICS_H
diff --git a/fdbserver/core/include/fdbserver/core/ShardSizing.h b/fdbserver/core/include/fdbserver/core/ShardSizing.h
new file mode 100644
index 00000000000..78b885f2e8c
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/ShardSizing.h
@@ -0,0 +1,41 @@
+/*
+ * ShardSizing.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/StorageServerInterface.h"
+
+// Holds the permitted size and IO bounds for a shard.
+struct ShardSizeBounds {
+	StorageMetrics max;
+	StorageMetrics min;
+	StorageMetrics permittedError;
+
+	bool operator==(ShardSizeBounds const& rhs) const {
+		return max == rhs.max && min == rhs.min && permittedError == rhs.permittedError;
+	}
+
+	static ShardSizeBounds shardSizeBoundsBeforeTrack();
+};
+
+ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize);
+int64_t getMaxShardSize(double dbSizeEstimate);
+bool ddLargeTeamEnabled();
diff --git a/fdbserver/include/fdbserver/SpanContextMessage.h b/fdbserver/core/include/fdbserver/core/SpanContextMessage.h
similarity index 96%
rename from fdbserver/include/fdbserver/SpanContextMessage.h
rename to fdbserver/core/include/fdbserver/core/SpanContextMessage.h
index 17d542c3787..835267e888e 100644
--- a/fdbserver/include/fdbserver/SpanContextMessage.h
+++ b/fdbserver/core/include/fdbserver/core/SpanContextMessage.h
@@ -39,7 +39,7 @@ struct SpanContextMessage {
 	SpanID spanContext;
 
 	SpanContextMessage() {}
-	SpanContextMessage(SpanID const& spanContext) : spanContext(spanContext) {}
+	explicit SpanContextMessage(SpanID const& spanContext) : spanContext(spanContext) {}
 
 	std::string toString() const {
 		return format(
diff --git a/fdbserver/include/fdbserver/StorageMetrics.actor.h b/fdbserver/core/include/fdbserver/core/StorageMetrics.h
similarity index 80%
rename from fdbserver/include/fdbserver/StorageMetrics.actor.h
rename to fdbserver/core/include/fdbserver/core/StorageMetrics.h
index 0faf5f2ddfb..fbc775e2dd8 100644
--- a/fdbserver/include/fdbserver/StorageMetrics.actor.h
+++ b/fdbserver/core/include/fdbserver/core/StorageMetrics.h
@@ -19,18 +19,12 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_STORAGEMETRICS_G_H)
-#define FDBSERVER_STORAGEMETRICS_G_H
-#include "fdbserver/StorageMetrics.actor.g.h"
-#elif !defined(FDBSERVER_STORAGEMETRICS_H)
-#define FDBSERVER_STORAGEMETRICS_H
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/simulator.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbserver/Knobs.h"
-#include "flow/actorcompiler.h"
+#include "fdbserver/core/Knobs.h"
 
 const StringRef STORAGESERVER_HISTOGRAM_GROUP = "StorageServer"_sr;
 const StringRef FETCH_KEYS_LATENCY_HISTOGRAM = "FetchKeysLatency"_sr;
@@ -80,9 +74,18 @@ struct TransientStorageMetricSample : StorageMetricSample {
 	int64_t add(const Key& key, int64_t metric);
 };
 
+struct WaitMetricsMapHighWatermarks {
+	int64_t rangeCount = 0;
+	int64_t nonEmptyRangeCount = 0;
+	int64_t watcherCount = 0;
+	int64_t watcherCapacity = 0;
+	int64_t maxWatchersPerRange = 0;
+};
+
 struct StorageServerMetrics {
 	KeyRangeMap<std::vector<PromiseStream<StorageMetrics>>> waitMetricsMap;
 	StorageMetricSample byteSample;
+	WaitMetricsMapHighWatermarks waitMetricsMapHighWatermarks;
 
 	// FIXME: iops is not effectively tested, and is not used by data distribution
 	TransientStorageMetricSample iopsSample, bytesWriteSample;
@@ -110,6 +113,8 @@ struct StorageServerMetrics {
 
 	void poll();
 
+	void traceWaitMetricsMapIfHighWatermark(const char* reason, KeyRangeRef keys);
+
 	// static void waitMetrics( StorageServerMetrics* const& self, WaitMetricsRequest const& req );
 
 	KeyRef getSplitKey(int64_t remaining,
@@ -253,32 +258,64 @@ class IStorageMetricsService {
 	// void sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double penalty);
 };
 
-ACTOR template <class ServiceType>
-Future<Void> serveStorageMetricsRequests(ServiceType* self, StorageServerInterface ssi) {
-	state Future<Void> doPollMetrics = Void();
-	loop {
-		choose {
-			when(state WaitMetricsRequest req = waitNext(ssi.waitMetrics.getFuture())) {
-				self->addActor(self->waitMetricsForReal(req));
-			}
-			when(SplitMetricsRequest req = waitNext(ssi.splitMetrics.getFuture())) {
-				self->getSplitMetrics(req);
-			}
-			when(GetStorageMetricsRequest req = waitNext(ssi.getStorageMetrics.getFuture())) {
-				self->getStorageMetrics(req);
-			}
-			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) {
-				self->getHotRangeMetrics(req);
-			}
-			when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) {
-				self->getSplitPoints(req);
-			}
-			when(wait(doPollMetrics)) {
-				self->metrics.poll();
-				doPollMetrics = delay(SERVER_KNOBS->STORAGE_SERVER_POLL_METRICS_DELAY);
-			}
-		}
+namespace storagemetrics_impl {
+
+template <class ServiceType>
+Future<Void> serveWaitMetricsReqs(ServiceType* self, FutureStream<WaitMetricsRequest> waitMetrics) {
+	while (true) {
+		auto req = co_await waitMetrics;
+		self->addActor(self->waitMetricsForReal(req));
+	}
+}
+
+template <class ServiceType>
+Future<Void> serveSplitMetricsReqs(ServiceType* self, FutureStream<SplitMetricsRequest> splitMetrics) {
+	while (true) {
+		auto req = co_await splitMetrics;
+		self->getSplitMetrics(req);
+	}
+}
+
+template <class ServiceType>
+Future<Void> serveGetStorageMetricsReqs(ServiceType* self, FutureStream<GetStorageMetricsRequest> getStorageMetrics) {
+	while (true) {
+		auto req = co_await getStorageMetrics;
+		self->getStorageMetrics(req);
+	}
+}
+
+template <class ServiceType>
+Future<Void> serveReadHotRangeMetricsReqs(ServiceType* self, FutureStream<ReadHotSubRangeRequest> readHotRanges) {
+	while (true) {
+		auto req = co_await readHotRanges;
+		self->getHotRangeMetrics(req);
+	}
+}
+
+template <class ServiceType>
+Future<Void> serveSplitPointsReqs(ServiceType* self, FutureStream<SplitRangeRequest> splitPoints) {
+	while (true) {
+		auto req = co_await splitPoints;
+		self->getSplitPoints(req);
+	}
+}
+
+template <class ServiceType>
+Future<Void> pollMetrics(ServiceType* self) {
+	while (true) {
+		self->metrics.poll();
+		co_await delay(SERVER_KNOBS->STORAGE_SERVER_POLL_METRICS_DELAY);
 	}
 }
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_STORAGEMETRICS_H
+
+} // namespace storagemetrics_impl
+
+template <class ServiceType>
+Future<Void> serveStorageMetricsRequests(ServiceType* self, StorageServerInterface ssi) {
+	co_await race(storagemetrics_impl::serveWaitMetricsReqs(self, ssi.waitMetrics.getFuture()),
+	              storagemetrics_impl::serveSplitMetricsReqs(self, ssi.splitMetrics.getFuture()),
+	              storagemetrics_impl::serveGetStorageMetricsReqs(self, ssi.getStorageMetrics.getFuture()),
+	              storagemetrics_impl::serveReadHotRangeMetricsReqs(self, ssi.getReadHotRanges.getFuture()),
+	              storagemetrics_impl::serveSplitPointsReqs(self, ssi.getRangeSplitPoints.getFuture()),
+	              storagemetrics_impl::pollMetrics(self));
+}
diff --git a/fdbserver/include/fdbserver/TLogInterface.h b/fdbserver/core/include/fdbserver/core/TLogInterface.h
similarity index 98%
rename from fdbserver/include/fdbserver/TLogInterface.h
rename to fdbserver/core/include/fdbserver/core/TLogInterface.h
index 3526acf526e..1bb2374e43f 100644
--- a/fdbserver/include/fdbserver/TLogInterface.h
+++ b/fdbserver/core/include/fdbserver/core/TLogInterface.h
@@ -164,7 +164,7 @@ struct TLogConfirmRunningRequest {
 	ReplyPromise<Void> reply;
 
 	TLogConfirmRunningRequest() {}
-	TLogConfirmRunningRequest(Optional<UID> debugID) : debugID(debugID) {}
+	explicit TLogConfirmRunningRequest(Optional<UID> debugID) : debugID(debugID) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -403,7 +403,7 @@ struct TLogDisablePopRequest {
 	Optional<UID> debugID;
 
 	TLogDisablePopRequest() = default;
-	TLogDisablePopRequest(const UID uid) : snapUID(uid) {}
+	explicit TLogDisablePopRequest(const UID uid) : snapUID(uid) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -418,7 +418,7 @@ struct TLogEnablePopRequest {
 	Optional<UID> debugID;
 
 	TLogEnablePopRequest() = default;
-	TLogEnablePopRequest(const UID uid) : snapUID(uid) {}
+	explicit TLogEnablePopRequest(const UID uid) : snapUID(uid) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -468,7 +468,7 @@ struct TrackTLogRecoveryRequest {
 	ReplyPromise<TrackTLogRecoveryReply> reply;
 
 	TrackTLogRecoveryRequest() {}
-	TrackTLogRecoveryRequest(Version oldestGenRecoverAtVersion)
+	explicit TrackTLogRecoveryRequest(Version oldestGenRecoverAtVersion)
 	  : oldestGenRecoverAtVersion(oldestGenRecoverAtVersion) {}
 
 	template <class Ar>
diff --git a/fdbserver/core/include/fdbserver/core/TSSMappingUtil.h b/fdbserver/core/include/fdbserver/core/TSSMappingUtil.h
new file mode 100644
index 00000000000..65a86bcb884
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/TSSMappingUtil.h
@@ -0,0 +1,35 @@
+/*
+ * TSSMappingUtil.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/StorageServerInterface.h"
+
+/*
+ * Collection of utility functions for dealing with the TSS mapping
+ */
+
+// Reads the current cluster TSS mapping as part of the RYW transaction
+Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
+                               std::map<UID, StorageServerInterface>* tssMapping);
+
+// Reads the current cluster TSS mapping as part of the given Transaction
+Future<Void> readTSSMapping(Transaction* tr, std::map<UID, StorageServerInterface>* tssMapping);
diff --git a/fdbserver/include/fdbserver/TesterInterface.actor.h b/fdbserver/core/include/fdbserver/core/TesterInterface.h
similarity index 66%
rename from fdbserver/include/fdbserver/TesterInterface.actor.h
rename to fdbserver/core/include/fdbserver/core/TesterInterface.h
index 8c32f48c557..1c69c49b9bd 100644
--- a/fdbserver/include/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/core/include/fdbserver/core/TesterInterface.h
@@ -1,5 +1,5 @@
 /*
- * TesterInterface.actor.h
+ * TesterInterface.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,17 +20,10 @@
 
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_TESTERINTERFACE_ACTOR_G_H)
-#define FDBSERVER_TESTERINTERFACE_ACTOR_G_H
-#include "fdbserver/TesterInterface.actor.g.h"
-#elif !defined(FDBSERVER_TESTERINTERFACE_ACTOR_H)
-#define FDBSERVER_TESTERINTERFACE_ACTOR_H
-
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/PerfMetric.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // has to be last include
+
 struct CheckReply {
 	constexpr static FileIdentifier file_identifier = 11;
 
@@ -118,30 +111,3 @@ struct TesterInterface {
 		serializer(ar, recruitments);
 	}
 };
-
-ACTOR Future<Void> testerServerCore(TesterInterface interf,
-                                    Reference<IClusterConnectionRecord> ccr,
-                                    Reference<AsyncVar<struct ServerDBInfo> const> serverDBInfo,
-                                    LocalityData locality,
-                                    Optional<std::string> expectedWorkLoad = Optional<std::string>());
-
-enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS };
-enum test_type_t {
-	TEST_TYPE_FROM_FILE,
-	TEST_TYPE_CONSISTENCY_CHECK,
-	TEST_TYPE_UNIT_TESTS,
-	TEST_TYPE_CONSISTENCY_CHECK_URGENT
-};
-
-ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
-                            test_type_t whatToRun,
-                            test_location_t whereToRun,
-                            int minTestersExpected,
-                            std::string fileName = std::string(),
-                            StringRef startingConfiguration = StringRef(),
-                            LocalityData locality = LocalityData(),
-                            UnitTestParameters testOptions = UnitTestParameters(),
-                            bool restartingTest = false);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/core/include/fdbserver/core/WaitFailure.h b/fdbserver/core/include/fdbserver/core/WaitFailure.h
new file mode 100644
index 00000000000..b6b49e4e5d3
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/WaitFailure.h
@@ -0,0 +1,42 @@
+/*
+ * WaitFailure.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include "fdbrpc/fdbrpc.h"
+#include "flow/flow.h"
+
+Future<Void> waitFailureServer(FutureStream<ReplyPromise<Void>> waitFailure);
+
+Future<Void> waitFailureClient(RequestStream<ReplyPromise<Void>> waitFailure,
+                               double failureReactionTime = 0,
+                               double failureReactionSlope = 0,
+                               bool trace = false,
+                               Optional<Standalone<StringRef>> traceMsg = Optional<Standalone<StringRef>>(),
+                               TaskPriority taskID = TaskPriority::DefaultEndpoint);
+
+Future<Void> waitFailureClientStrict(RequestStream<ReplyPromise<Void>> waitFailure,
+                                     double failureReactionTime = 0,
+                                     TaskPriority taskID = TaskPriority::DefaultEndpoint);
+
+Future<Void> waitFailureTracker(RequestStream<ReplyPromise<Void>> waitFailure,
+                                Reference<AsyncVar<bool>> failed,
+                                double failureReactionTime = 0,
+                                double failureReactionSlope = 0,
+                                TaskPriority taskID = TaskPriority::DefaultEndpoint);
diff --git a/fdbserver/core/include/fdbserver/core/WorkerEvents.h b/fdbserver/core/include/fdbserver/core/WorkerEvents.h
new file mode 100644
index 00000000000..e5bc268ac21
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/WorkerEvents.h
@@ -0,0 +1,34 @@
+/*
+ * WorkerEvents.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <set>
+#include <string>
+
+#include "flow/ITrace.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+
+struct WorkerEvents : std::map<NetworkAddress, TraceEventFields> {};
+
+AsyncResult<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventOnWorkers(
+    std::vector<WorkerDetails> workers,
+    std::string eventName);
diff --git a/fdbserver/include/fdbserver/WorkerInterface.actor.h b/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h
similarity index 82%
rename from fdbserver/include/fdbserver/WorkerInterface.actor.h
rename to fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h
index d0c2c450e3c..5366dfd0af5 100644
--- a/fdbserver/include/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h
@@ -21,26 +21,25 @@
 #pragma once
 #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_WORKERINTERFACE_ACTOR_G_H)
 #define FDBSERVER_WORKERINTERFACE_ACTOR_G_H
-#include "fdbserver/WorkerInterface.actor.g.h"
+#include "fdbserver/core/WorkerInterface.actor.g.h"
 #elif !defined(FDBSERVER_WORKERINTERFACE_ACTOR_H)
 #define FDBSERVER_WORKERINTERFACE_ACTOR_H
 
-#include "fdbserver/BackupInterface.h"
-#include "fdbserver/DataDistributorInterface.h"
-#include "fdbclient/EncryptKeyProxyInterface.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbclient/ConsistencyScanInterface.actor.h"
-#include "fdbserver/ResolverInterface.h"
+#include "fdbserver/core/BackupInterface.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbclient/ConsistencyScanInterface.h"
+#include "fdbserver/core/ResolverInterface.h"
 #include "fdbclient/ClientBooleanParams.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/LogSystemConfig.h"
+#include "fdbserver/core/LogSystemConfig.h"
 #include "fdbrpc/MultiInterface.h"
 #include "fdbclient/ClientWorkerInterface.h"
-#include "fdbserver/RecoveryState.h"
+#include "fdbserver/core/RecoveryState.h"
 #include "flow/actorcompiler.h"
 
 struct WorkerInterface {
@@ -58,7 +57,6 @@ struct WorkerInterface {
 	RequestStream<struct InitializeStorageRequest> storage;
 	RequestStream<struct InitializeLogRouterRequest> logRouter;
 	RequestStream<struct InitializeBackupRequest> backup;
-	RequestStream<struct InitializeEncryptKeyProxyRequest> encryptKeyProxy;
 
 	RequestStream<struct LoadedPingRequest> debugPing;
 	RequestStream<struct CoordinationPingMessage> coordinationPing;
@@ -81,7 +79,7 @@ struct WorkerInterface {
 	Optional<NetworkAddress> grpcAddress() const { return clientInterface.grpcAddress; }
 
 	WorkerInterface() {}
-	WorkerInterface(const LocalityData& locality) : locality(locality) {}
+	explicit(false) WorkerInterface(const LocalityData& locality) : locality(locality) {}
 
 	void initEndpoints() {
 		clientInterface.initEndpoints();
@@ -123,7 +121,6 @@ struct WorkerInterface {
 		           execReq,
 		           workerSnapReq,
 		           backup,
-		           encryptKeyProxy,
 		           updateServerDBInfo);
 	}
 };
@@ -394,7 +391,6 @@ struct RegisterWorkerRequest {
 	Generation generation;
 	Optional<DataDistributorInterface> distributorInterf;
 	Optional<RatekeeperInterface> ratekeeperInterf;
-	Optional<EncryptKeyProxyInterface> encryptKeyProxyInterf;
 	Optional<ConsistencyScanInterface> consistencyScanInterf;
 	Standalone<VectorRef<StringRef>> issues;
 	std::vector<NetworkAddress> incompatiblePeers;
@@ -412,15 +408,14 @@ struct RegisterWorkerRequest {
 	                      Generation generation,
 	                      Optional<DataDistributorInterface> ddInterf,
 	                      Optional<RatekeeperInterface> rkInterf,
-	                      Optional<EncryptKeyProxyInterface> ekpInterf,
 	                      Optional<ConsistencyScanInterface> csInterf,
 	                      bool degraded,
 	                      bool recoveredDiskFiles,
 	                      Optional<UID> clusterId)
 	  : wi(wi), initialClass(initialClass), processClass(processClass), priorityInfo(priorityInfo),
 	    generation(generation), distributorInterf(ddInterf), ratekeeperInterf(rkInterf),
-	    encryptKeyProxyInterf(ekpInterf), consistencyScanInterf(csInterf), degraded(degraded),
-	    recoveredDiskFiles(recoveredDiskFiles), clusterId(clusterId) {}
+	    consistencyScanInterf(csInterf), degraded(degraded), recoveredDiskFiles(recoveredDiskFiles),
+	    clusterId(clusterId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -432,7 +427,6 @@ struct RegisterWorkerRequest {
 		           generation,
 		           distributorInterf,
 		           ratekeeperInterf,
-		           encryptKeyProxyInterf,
 		           consistencyScanInterf,
 		           issues,
 		           incompatiblePeers,
@@ -523,7 +517,7 @@ struct GetEncryptionAtRestModeResponse {
 	uint32_t mode;
 
 	GetEncryptionAtRestModeResponse() : mode(EncryptionAtRestModeDeprecated::Mode::DISABLED) {}
-	GetEncryptionAtRestModeResponse(uint32_t m) : mode(m) {}
+	explicit(false) GetEncryptionAtRestModeResponse(uint32_t m) : mode(m) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -537,7 +531,7 @@ struct GetEncryptionAtRestModeRequest {
 	ReplyPromise<GetEncryptionAtRestModeResponse> reply;
 
 	GetEncryptionAtRestModeRequest() {}
-	GetEncryptionAtRestModeRequest(UID tId) : tlogId(tId) {}
+	explicit(false) GetEncryptionAtRestModeRequest(UID tId) : tlogId(tId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -822,22 +816,6 @@ struct InitializeStorageRequest {
 	}
 };
 
-struct InitializeEncryptKeyProxyRequest {
-	constexpr static FileIdentifier file_identifier = 4180191;
-	UID reqId;
-	UID interfaceId;
-	ReplyPromise<EncryptKeyProxyInterface> reply;
-	EncryptionAtRestModeDeprecated encryptMode;
-
-	InitializeEncryptKeyProxyRequest() {}
-	explicit InitializeEncryptKeyProxyRequest(UID uid) : reqId(uid) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reqId, interfaceId, reply, encryptMode);
-	}
-};
-
 struct TraceBatchDumpRequest {
 	constexpr static FileIdentifier file_identifier = 8184121;
 	ReplyPromise<Void> reply;
@@ -855,7 +833,7 @@ struct ExecuteRequest {
 	Arena arena;
 	StringRef execPayload;
 
-	ExecuteRequest(StringRef execPayload) : execPayload(execPayload) {}
+	explicit(false) ExecuteRequest(StringRef execPayload) : execPayload(execPayload) {}
 
 	ExecuteRequest() : execPayload() {}
 
@@ -865,7 +843,7 @@ struct ExecuteRequest {
 	}
 };
 
-// FIXME: describe purpose
+// Request sent to a worker to run the snapshot command for one local role.
 struct WorkerSnapRequest {
 	constexpr static FileIdentifier file_identifier = 8194122;
 	ReplyPromise<Void> reply;
@@ -976,7 +954,7 @@ struct DiskStoreRequest {
 	bool includePartialStores;
 	ReplyPromise<Standalone<VectorRef<UID>>> reply;
 
-	DiskStoreRequest(bool includePartialStores = false) : includePartialStores(includePartialStores) {}
+	explicit(false) DiskStoreRequest(bool includePartialStores = false) : includePartialStores(includePartialStores) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -1062,7 +1040,7 @@ void startRole(const Role& role,
                const std::string& origination = "Recruited");
 void endRole(const Role& role, UID id, std::string reason, bool ok = true, Error e = Error());
 
-ACTOR Future<Void> traceRole(Role role, UID roleId);
+Future<Void> traceRole(Role role, UID roleId);
 
 struct ServerDBInfo;
 
@@ -1070,97 +1048,8 @@ class Database openDBOnServer(Reference<AsyncVar<ServerDBInfo> const> const& db,
                               TaskPriority taskID = TaskPriority::DefaultEndpoint,
                               LockAware = LockAware::False,
                               EnableLocalityLoadBalance = EnableLocalityLoadBalance::True);
-ACTOR Future<Void> extractClusterInterface(
-    Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>> const> in,
-    Reference<AsyncVar<Optional<struct ClusterInterface>>> out);
-
-ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> ccr,
-                        LocalityData localities,
-                        ProcessClass processClass,
-                        std::string dataFolder,
-                        std::string coordFolder,
-                        int64_t memoryLimit,
-                        std::string metricsConnFile,
-                        std::string metricsPrefix,
-                        int64_t memoryProfilingThreshold,
-                        std::string whitelistBinPaths,
-                        bool consistencyCheckUrgentMode);
-
-ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> ccr,
-                                     Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
-                                     Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
-                                     LocalityData locality,
-                                     Reference<AsyncVar<Optional<UID>>> clusterId);
-
-// These servers are started by workerServer
-class IKeyValueStore;
-class ServerCoordinators;
-class IDiskQueue;
-
-ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ei,
-                                         Reference<AsyncVar<ServerDBInfo>> db,
-                                         EncryptionAtRestModeDeprecated encryptMode);
-
-ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
-                                 StorageServerInterface ssi,
-                                 Tag seedTag,
-                                 Version startVersion,
-                                 Version tssSeedVersion,
-                                 ReplyPromise<InitializeStorageReply> recruitReply,
-                                 Reference<AsyncVar<ServerDBInfo> const> db,
-                                 std::string folder);
-
-ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
-                                 StorageServerInterface ssi,
-                                 Reference<AsyncVar<ServerDBInfo> const> db,
-                                 std::string folder,
-                                 Promise<Void> recovered,
-                                 Reference<IClusterConnectionRecord> connRecord);
-
-ACTOR Future<Void> masterServer(MasterInterface mi,
-                                Reference<AsyncVar<ServerDBInfo> const> db,
-                                Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
-                                ServerCoordinators serverCoordinators,
-                                LifetimeToken lifetime,
-                                bool forceRecovery);
-ACTOR Future<Void> commitProxyServer(CommitProxyInterface proxy,
-                                     InitializeCommitProxyRequest req,
-                                     Reference<AsyncVar<ServerDBInfo> const> db,
-                                     std::string whitelistBinPaths);
-ACTOR Future<Void> grvProxyServer(GrvProxyInterface proxy,
-                                  InitializeGrvProxyRequest req,
-                                  Reference<AsyncVar<ServerDBInfo> const> db);
-ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
-                        IDiskQueue* persistentQueue,
-                        Reference<AsyncVar<ServerDBInfo> const> db,
-                        LocalityData locality,
-                        PromiseStream<InitializeTLogRequest> tlogRequests,
-                        UID tlogId,
-                        UID workerID,
-                        bool restoreFromDisk,
-                        Promise<Void> oldLog,
-                        Promise<Void> recovered,
-                        std::string folder,
-                        Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog,
-                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck);
-ACTOR Future<Void> resolver(ResolverInterface resolver,
-                            InitializeResolverRequest initReq,
-                            Reference<AsyncVar<ServerDBInfo> const> db);
-Future<Void> logRouter(TLogInterface interf,
-                       InitializeLogRouterRequest req,
-                       Reference<AsyncVar<ServerDBInfo> const> db);
-Future<Void> dataDistributor(DataDistributorInterface ddi,
-                             Reference<AsyncVar<ServerDBInfo> const> db,
-                             std::string folder);
-ACTOR Future<Void> ratekeeper(RatekeeperInterface rki, Reference<AsyncVar<ServerDBInfo> const> db);
-ACTOR Future<Void> consistencyScan(ConsistencyScanInterface csInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo);
-
-ACTOR Future<Void> backupWorker(BackupInterface bi,
-                                InitializeBackupRequest req,
-                                Reference<AsyncVar<ServerDBInfo> const> db);
-
-void registerThreadForProfiling();
+Future<Void> extractClusterInterface(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>> const> in,
+                                     Reference<AsyncVar<Optional<struct ClusterInterface>>> out);
 
 // Returns true if `address` is used in the db (indicated by `dbInfo`) transaction system and in the db's primary
 // satellite DC.
@@ -1172,10 +1061,6 @@ bool addressInDbAndRemoteDc(
     Reference<AsyncVar<ServerDBInfo> const> dbInfo,
     Optional<std::vector<NetworkAddress>> storageServers = Optional<std::vector<NetworkAddress>>{});
 
-void updateCpuProfiler(ProfilerRequest req);
-
-typedef decltype(&tLog) TLogFn;
-
 extern bool isSimulatorProcessUnreliable();
 
 ACTOR template <class T>
@@ -1208,37 +1093,6 @@ Future<T> ioTimeoutError(Future<T> what, double time, const char* context = null
 	}
 }
 
-ACTOR template <class T>
-Future<T> ioTimeoutErrorIfCleared(Future<T> what,
-                                  double time,
-                                  Reference<AsyncVar<bool>> condition,
-                                  const char* context = nullptr) {
-	// Before simulation is sped up, IO operations can take a very long time so limit timeouts
-	// to not end until at least time after simulation is sped up.
-	if (g_network->isSimulated() && !g_simulator->speedUpSimulation) {
-		time += std::max(0.0, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS - now());
-	}
-	Future<Void> end = lowPriorityDelayAfterCleared(condition, time);
-	choose {
-		when(T t = wait(what)) {
-			return t;
-		}
-		when(wait(end)) {
-			Error err = io_timeout();
-			if (isSimulatorProcessUnreliable()) {
-				err = err.asInjectedFault();
-			}
-			TraceEvent e(SevError, "IoTimeoutError");
-			e.error(err);
-			if (context != nullptr) {
-				e.detail("Context", context);
-			}
-			e.log();
-			throw err;
-		}
-	}
-}
-
 ACTOR template <class T>
 Future<T> ioDegradedOrTimeoutError(Future<T> what,
                                    double errTime,
@@ -1288,6 +1142,6 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
 	}
 }
 
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbserver/core/include/fdbserver/core/WorkloadKeys.h b/fdbserver/core/include/fdbserver/core/WorkloadKeys.h
new file mode 100644
index 00000000000..ba85dcfe042
--- /dev/null
+++ b/fdbserver/core/include/fdbserver/core/WorkloadKeys.h
@@ -0,0 +1,32 @@
+/*
+ * WorkloadKeys.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_CORE_WORKLOADKEYS_H
+#define FDBSERVER_CORE_WORKLOADKEYS_H
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+
+Key doubleToTestKey(double p);
+double testKeyToDouble(const KeyRef& p);
+Key doubleToTestKey(double p, const KeyRef& prefix);
+double testKeyToDouble(const KeyRef& p, const KeyRef& prefix);
+
+#endif
diff --git a/fdbserver/coroimpl/CoroFlow.actor.cpp b/fdbserver/coroimpl/CoroFlow.actor.cpp
index 0b2f966f9c2..789e96c9f67 100644
--- a/fdbserver/coroimpl/CoroFlow.actor.cpp
+++ b/fdbserver/coroimpl/CoroFlow.actor.cpp
@@ -20,7 +20,7 @@
 
 #include "fdbserver/CoroFlow.h"
 #include "flow/ActorCollection.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 #include <boost/coroutine2/all.hpp>
diff --git a/fdbserver/coroimpl/CoroFlowCoro.actor.cpp b/fdbserver/coroimpl/CoroFlowCoro.actor.cpp
index 46b65b115bc..ee3ad055859 100644
--- a/fdbserver/coroimpl/CoroFlowCoro.actor.cpp
+++ b/fdbserver/coroimpl/CoroFlowCoro.actor.cpp
@@ -21,7 +21,7 @@
 #include "fdbserver/CoroFlow.h"
 #include "flow/ActorCollection.h"
 #include "Coro.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 #include "flow/actorcompiler.h" // has to be last include
@@ -260,7 +260,7 @@ class WorkPool final : public IThreadPool, public ReferenceCounted<WorkPool<Thre
 	void delref() override { ReferenceCounted<WorkPool>::delref(); }
 };
 
-typedef WorkPool<Coroutine, ThreadUnsafeSpinLock, true> CoroPool;
+using CoroPool = WorkPool<Coroutine, ThreadUnsafeSpinLock, true>;
 
 ACTOR void coroSwitcher(Future<Void> what, TaskPriority taskID, Coro* coro) {
 	try {
diff --git a/fdbserver/datadistributor/AuditUtilsTests.cpp b/fdbserver/datadistributor/AuditUtilsTests.cpp
new file mode 100644
index 00000000000..30324b94d82
--- /dev/null
+++ b/fdbserver/datadistributor/AuditUtilsTests.cpp
@@ -0,0 +1,407 @@
+/*
+ * AuditUtilsTests.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/AuditUtils.h"
+#include "fdbclient/FDBTypes.h"
+#include "flow/UnitTest.h"
+
+void forceLinkAuditUtilsTests() {}
+
+TEST_CASE("/AuditUtils/coalesceRangeList") {
+	// Empty input
+	{
+		auto result = coalesceRangeList({});
+		ASSERT(result.empty());
+	}
+
+	// Single range unchanged
+	{
+		KeyRange r = Standalone(KeyRangeRef("a"_sr, "c"_sr));
+		auto result = coalesceRangeList({ r });
+		ASSERT(result.size() == 1);
+		ASSERT(result[0] == r);
+	}
+
+	// Non-overlapping ranges preserved
+	{
+		KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "b"_sr));
+		KeyRange r2 = Standalone(KeyRangeRef("d"_sr, "e"_sr));
+		auto result = coalesceRangeList({ r2, r1 });
+		ASSERT(result.size() == 2);
+		ASSERT(result[0] == r1);
+		ASSERT(result[1] == r2);
+	}
+
+	// Overlapping ranges merged
+	{
+		KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "d"_sr));
+		KeyRange r2 = Standalone(KeyRangeRef("c"_sr, "f"_sr));
+		auto result = coalesceRangeList({ r2, r1 });
+		ASSERT(result.size() == 1);
+		ASSERT(result[0].begin == "a"_sr);
+		ASSERT(result[0].end == "f"_sr);
+	}
+
+	// Adjacent ranges merged
+	{
+		KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "c"_sr));
+		KeyRange r2 = Standalone(KeyRangeRef("c"_sr, "f"_sr));
+		auto result = coalesceRangeList({ r1, r2 });
+		ASSERT(result.size() == 1);
+		ASSERT(result[0].begin == "a"_sr);
+		ASSERT(result[0].end == "f"_sr);
+	}
+
+	// Contained range absorbed
+	{
+		KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "f"_sr));
+		KeyRange r2 = Standalone(KeyRangeRef("b"_sr, "d"_sr));
+		auto result = coalesceRangeList({ r1, r2 });
+		ASSERT(result.size() == 1);
+		ASSERT(result[0] == r1);
+	}
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/rangesSame") {
+	// Both empty — no mismatch
+	{
+		auto result = rangesSame({}, {});
+		ASSERT(!result.present());
+	}
+
+	// One empty, one not — mismatch
+	{
+		KeyRange r = Standalone(KeyRangeRef("a"_sr, "c"_sr));
+		auto result = rangesSame({}, { r });
+		ASSERT(result.present());
+		ASSERT(result.get().first.empty());
+		ASSERT(result.get().second == r);
+	}
+
+	// Identical single range — no mismatch
+	{
+		KeyRange r = Standalone(KeyRangeRef("a"_sr, "c"_sr));
+		auto result = rangesSame({ r }, { r });
+		ASSERT(!result.present());
+	}
+
+	// Different split points covering same range — no mismatch
+	{
+		KeyRange whole = Standalone(KeyRangeRef("a"_sr, "f"_sr));
+		KeyRange left = Standalone(KeyRangeRef("a"_sr, "c"_sr));
+		KeyRange right = Standalone(KeyRangeRef("c"_sr, "f"_sr));
+		auto result = rangesSame({ whole }, { left, right });
+		ASSERT(!result.present());
+	}
+
+	// Different begin — mismatch
+	{
+		KeyRange rA = Standalone(KeyRangeRef("a"_sr, "f"_sr));
+		KeyRange rB = Standalone(KeyRangeRef("b"_sr, "f"_sr));
+		auto result = rangesSame({ rA }, { rB });
+		ASSERT(result.present());
+	}
+
+	// Different end — mismatch
+	{
+		KeyRange rA = Standalone(KeyRangeRef("a"_sr, "e"_sr));
+		KeyRange rB = Standalone(KeyRangeRef("a"_sr, "f"_sr));
+		auto result = rangesSame({ rA }, { rB });
+		ASSERT(result.present());
+	}
+
+	// Gap in one list — mismatch
+	{
+		KeyRange rA1 = Standalone(KeyRangeRef("a"_sr, "b"_sr));
+		KeyRange rA2 = Standalone(KeyRangeRef("d"_sr, "f"_sr));
+		KeyRange rB = Standalone(KeyRangeRef("a"_sr, "f"_sr));
+		auto result = rangesSame({ rA1, rA2 }, { rB });
+		ASSERT(result.present());
+	}
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/checkLocationMetadataConsistency/Consistent") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+	KeyRange r2 = Standalone(KeyRangeRef("m"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapKS;
+	mapKS[server1] = { r1 };
+	mapKS[server2] = { r2 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapSK;
+	mapSK[server1] = { r1 };
+	mapSK[server2] = { r2 };
+
+	auto errors = checkLocationMetadataConsistency(mapKS, mapSK, claimRange);
+	ASSERT(errors.empty());
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/checkLocationMetadataConsistency/ServerInKeyServersNotServerKeys") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapKS;
+	mapKS[server1] = { r1 };
+	mapKS[server2] = { r1 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapSK;
+	mapSK[server1] = { r1 };
+	// server2 missing from ServerKeys
+
+	auto errors = checkLocationMetadataConsistency(mapKS, mapSK, claimRange);
+	ASSERT(errors.size() == 1);
+	ASSERT(errors[0].message.find("KeyServers but not ServerKeys") != std::string::npos);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/checkLocationMetadataConsistency/ServerInServerKeysNotKeyServers") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapKS;
+	mapKS[server1] = { r1 };
+	// server2 missing from KeyServers
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapSK;
+	mapSK[server1] = { r1 };
+	mapSK[server2] = { r1 };
+
+	auto errors = checkLocationMetadataConsistency(mapKS, mapSK, claimRange);
+	ASSERT(errors.size() == 1);
+	ASSERT(errors[0].message.find("ServerKeys but not KeyServers") != std::string::npos);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/checkLocationMetadataConsistency/RangeMismatch") {
+	UID server1(1, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapKS;
+	mapKS[server1] = { Standalone(KeyRangeRef("a"_sr, "m"_sr)) };
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapSK;
+	mapSK[server1] = { Standalone(KeyRangeRef("a"_sr, "f"_sr)) };
+
+	auto errors = checkLocationMetadataConsistency(mapKS, mapSK, claimRange);
+	ASSERT(errors.size() == 1);
+	ASSERT(errors[0].message.find("mismatch on Server") != std::string::npos);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/checkLocationMetadataConsistency/MultipleErrors") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	UID server3(3, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange r1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapKS;
+	mapKS[server1] = { r1 };
+	// server2 only in KS
+	mapKS[server2] = { r1 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapSK;
+	mapSK[server1] = { Standalone(KeyRangeRef("a"_sr, "f"_sr)) };
+	// server3 only in SK
+	mapSK[server3] = { r1 };
+
+	auto errors = checkLocationMetadataConsistency(mapKS, mapSK, claimRange);
+	// server1: range mismatch, server2: in KS not SK, server3: in SK not KS
+	ASSERT(errors.size() == 3);
+
+	int ksNotSk = 0, skNotKs = 0, rangeMismatch = 0;
+	for (const auto& e : errors) {
+		if (e.message.find("KeyServers but not ServerKeys") != std::string::npos)
+			ksNotSk++;
+		else if (e.message.find("ServerKeys but not KeyServers") != std::string::npos)
+			skNotKs++;
+		else if (e.message.find("mismatch on Server") != std::string::npos)
+			rangeMismatch++;
+	}
+	ASSERT(ksNotSk == 1);
+	ASSERT(skNotKs == 1);
+	ASSERT(rangeMismatch == 1);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/checkLocationMetadataConsistency/BothEmpty") {
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	std::unordered_map<UID, std::vector<KeyRange>> mapKS;
+	std::unordered_map<UID, std::vector<KeyRange>> mapSK;
+
+	auto errors = checkLocationMetadataConsistency(mapKS, mapSK, claimRange);
+	ASSERT(errors.empty());
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/LocationMetadataProductionPath/Consistent") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange shard1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+	KeyRange shard2 = Standalone(KeyRangeRef("m"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
+	mapFromKeyServersRaw[server1] = { shard1 };
+	mapFromKeyServersRaw[server2] = { shard2 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> serverOwnRangesMap;
+	serverOwnRangesMap[server1] = { shard1 };
+	serverOwnRangesMap[server2] = { shard2 };
+
+	auto builtMaps = buildLocationMetadataMaps(mapFromKeyServersRaw, serverOwnRangesMap, claimRange);
+
+	auto errors = checkLocationMetadataConsistency(builtMaps.fromKeyServers, builtMaps.fromServerKeys, claimRange);
+	ASSERT(errors.empty());
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/LocationMetadataProductionPath/CorruptMissingServerKeys") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange shard1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+	KeyRange shard2 = Standalone(KeyRangeRef("m"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
+	mapFromKeyServersRaw[server1] = { shard1 };
+	mapFromKeyServersRaw[server2] = { shard2 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> serverOwnRangesMap;
+	serverOwnRangesMap[server1] = { shard1 };
+
+	auto builtMaps = buildLocationMetadataMaps(mapFromKeyServersRaw, serverOwnRangesMap, claimRange);
+
+	auto errors = checkLocationMetadataConsistency(builtMaps.fromKeyServers, builtMaps.fromServerKeys, claimRange);
+	ASSERT(!errors.empty());
+	bool found = false;
+	for (const auto& e : errors) {
+		if (e.message.find("KeyServers but not ServerKeys") != std::string::npos)
+			found = true;
+	}
+	ASSERT(found);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/LocationMetadataProductionPath/CorruptPhantomServer") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	UID server3(3, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange shard1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+	KeyRange shard2 = Standalone(KeyRangeRef("m"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
+	mapFromKeyServersRaw[server1] = { shard1 };
+	mapFromKeyServersRaw[server2] = { shard2 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> serverOwnRangesMap;
+	serverOwnRangesMap[server1] = { shard1 };
+	serverOwnRangesMap[server2] = { shard2 };
+	serverOwnRangesMap[server3] = { shard1 };
+
+	auto builtMaps = buildLocationMetadataMaps(mapFromKeyServersRaw, serverOwnRangesMap, claimRange);
+
+	auto errors = checkLocationMetadataConsistency(builtMaps.fromKeyServers, builtMaps.fromServerKeys, claimRange);
+	ASSERT(!errors.empty());
+	bool found = false;
+	for (const auto& e : errors) {
+		if (e.message.find("ServerKeys but not KeyServers") != std::string::npos)
+			found = true;
+	}
+	ASSERT(found);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/LocationMetadataProductionPath/CorruptShiftedBoundary") {
+	UID server1(1, 0);
+	UID server2(2, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange shard1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+	KeyRange shard2 = Standalone(KeyRangeRef("m"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
+	mapFromKeyServersRaw[server1] = { shard1 };
+	mapFromKeyServersRaw[server2] = { shard2 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> serverOwnRangesMap;
+	serverOwnRangesMap[server1] = { Standalone(KeyRangeRef("a"_sr, "g"_sr)) };
+	serverOwnRangesMap[server2] = { shard2 };
+
+	auto builtMaps = buildLocationMetadataMaps(mapFromKeyServersRaw, serverOwnRangesMap, claimRange);
+
+	auto errors = checkLocationMetadataConsistency(builtMaps.fromKeyServers, builtMaps.fromServerKeys, claimRange);
+	ASSERT(!errors.empty());
+	bool found = false;
+	for (const auto& e : errors) {
+		if (e.message.find("mismatch on Server") != std::string::npos)
+			found = true;
+	}
+	ASSERT(found);
+
+	return Void();
+}
+
+TEST_CASE("/AuditUtils/LocationMetadataProductionPath/CorruptPartialOwnership") {
+	UID server1(1, 0);
+	KeyRange claimRange = Standalone(KeyRangeRef("a"_sr, "z"_sr));
+	KeyRange shard1 = Standalone(KeyRangeRef("a"_sr, "m"_sr));
+	KeyRange shard2 = Standalone(KeyRangeRef("m"_sr, "z"_sr));
+
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
+	mapFromKeyServersRaw[server1] = { shard1, shard2 };
+
+	std::unordered_map<UID, std::vector<KeyRange>> serverOwnRangesMap;
+	serverOwnRangesMap[server1] = { shard1 };
+
+	auto builtMaps = buildLocationMetadataMaps(mapFromKeyServersRaw, serverOwnRangesMap, claimRange);
+
+	auto errors = checkLocationMetadataConsistency(builtMaps.fromKeyServers, builtMaps.fromServerKeys, claimRange);
+	ASSERT(!errors.empty());
+	bool found = false;
+	for (const auto& e : errors) {
+		if (e.message.find("mismatch on Server") != std::string::npos)
+			found = true;
+	}
+	ASSERT(found);
+
+	return Void();
+}
diff --git a/fdbserver/datadistributor/CMakeLists.txt b/fdbserver/datadistributor/CMakeLists.txt
new file mode 100644
index 00000000000..c8747fd5362
--- /dev/null
+++ b/fdbserver/datadistributor/CMakeLists.txt
@@ -0,0 +1,14 @@
+fdb_find_sources(FDBSERVER_DATADISTRIBUTOR_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_datadistributor SRCS ${FDBSERVER_DATADISTRIBUTOR_SRCS})
+add_fdbserver_link_test(fdbserver_datadistributorlinktest
+  fdbserver_datadistributor
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_datadistributor)
+target_include_directories(fdbserver_datadistributor
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_datadistributor PRIVATE fdbserver_core)
diff --git a/fdbserver/DDRelocationQueue.actor.cpp b/fdbserver/datadistributor/DDRelocationQueue.actor.cpp
similarity index 85%
rename from fdbserver/DDRelocationQueue.actor.cpp
rename to fdbserver/datadistributor/DDRelocationQueue.actor.cpp
index 3db03189a1e..be0c7322f56 100644
--- a/fdbserver/DDRelocationQueue.actor.cpp
+++ b/fdbserver/datadistributor/DDRelocationQueue.actor.cpp
@@ -18,32 +18,35 @@
  * limitations under the License.
  */
 
+#include <functional>
 #include <limits>
 #include <numeric>
 #include <utility>
 #include <vector>
 
-#include "fdbserver/DataDistributionTeam.h"
+#include "fdbserver/datadistributor/DataDistributionTeam.h"
 #include "flow/ActorCollection.h"
 #include "flow/Buggify.h"
 #include "flow/FastRef.h"
 #include "flow/Trace.h"
 #include "fdbrpc/sim_validation.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/QuietDatabase.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/DDTxnProcessor.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
 #include "flow/DebugTrace.h"
-#include "fdbserver/DDRelocationQueue.h"
+#include "DDRelocationQueue.h"
+#include "flow/CoroUtils.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 #define WORK_FULL_UTILIZATION 10000 // This is not a knob; it is a fixed point scaling factor!
 
-typedef Reference<IDataDistributionTeam> ITeamRef;
-typedef std::pair<ITeamRef, ITeamRef> SrcDestTeamPair;
+using ITeamRef = Reference<IDataDistributionTeam>;
+using SrcDestTeamPair = std::pair<ITeamRef, ITeamRef>;
 
 inline bool isDataMovementForDiskBalancing(DataMovementReason reason) {
 	return reason == DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM ||
@@ -71,62 +74,6 @@ inline bool isDataMovementForValleyFiller(DataMovementReason reason) {
 	       reason == DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM;
 }
 
-typedef std::map<DataMovementReason, int> DmReasonPriorityMapping;
-typedef std::map<int, DataMovementReason> PriorityDmReasonMapping;
-std::pair<const DmReasonPriorityMapping*, const PriorityDmReasonMapping*> buildPriorityMappings() {
-	static DmReasonPriorityMapping reasonPriority{
-		{ DataMovementReason::INVALID, -1 },
-		{ DataMovementReason::RECOVER_MOVE, SERVER_KNOBS->PRIORITY_RECOVER_MOVE },
-		{ DataMovementReason::REBALANCE_UNDERUTILIZED_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM },
-		{ DataMovementReason::REBALANCE_OVERUTILIZED_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM },
-		{ DataMovementReason::REBALANCE_READ_OVERUTIL_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_READ_OVERUTIL_TEAM },
-		{ DataMovementReason::REBALANCE_READ_UNDERUTIL_TEAM, SERVER_KNOBS->PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM },
-		{ DataMovementReason::PERPETUAL_STORAGE_WIGGLE, SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE },
-		{ DataMovementReason::TEAM_HEALTHY, SERVER_KNOBS->PRIORITY_TEAM_HEALTHY },
-		{ DataMovementReason::TEAM_CONTAINS_UNDESIRED_SERVER, SERVER_KNOBS->PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER },
-		{ DataMovementReason::TEAM_REDUNDANT, SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT },
-		{ DataMovementReason::MERGE_SHARD, SERVER_KNOBS->PRIORITY_MERGE_SHARD },
-		{ DataMovementReason::POPULATE_REGION, SERVER_KNOBS->PRIORITY_POPULATE_REGION },
-		{ DataMovementReason::TEAM_UNHEALTHY, SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY },
-		{ DataMovementReason::TEAM_2_LEFT, SERVER_KNOBS->PRIORITY_TEAM_2_LEFT },
-		{ DataMovementReason::TEAM_1_LEFT, SERVER_KNOBS->PRIORITY_TEAM_1_LEFT },
-		{ DataMovementReason::TEAM_FAILED, SERVER_KNOBS->PRIORITY_TEAM_FAILED },
-		{ DataMovementReason::TEAM_0_LEFT, SERVER_KNOBS->PRIORITY_TEAM_0_LEFT },
-		{ DataMovementReason::SPLIT_SHARD, SERVER_KNOBS->PRIORITY_SPLIT_SHARD },
-		{ DataMovementReason::ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD,
-		  SERVER_KNOBS->PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD },
-		{ DataMovementReason::REBALANCE_STORAGE_QUEUE, SERVER_KNOBS->PRIORITY_REBALANCE_STORAGE_QUEUE },
-		{ DataMovementReason::ASSIGN_EMPTY_RANGE, -2 }, // dummy reason, no corresponding actual data move
-		{ DataMovementReason::SEED_SHARD_SERVER, -3 }, // dummy reason, no corresponding actual data move
-		{ DataMovementReason::NUMBER_OF_REASONS, -4 }, // dummy reason, no corresponding actual data move
-	};
-
-	static PriorityDmReasonMapping priorityReason;
-	if (priorityReason.empty()) { // only build once
-		for (const auto& [r, p] : reasonPriority) {
-			priorityReason[p] = r;
-		}
-		// Don't allow 2 priorities value being the same.
-		if (priorityReason.size() != reasonPriority.size()) {
-			TraceEvent(SevError, "DuplicateDataMovementPriority").log();
-			ASSERT(false);
-		}
-	}
-
-	return std::make_pair(&reasonPriority, &priorityReason);
-}
-
-// Return negative priority for invalid or dummy reasons
-int dataMovementPriority(DataMovementReason reason) {
-	auto [reasonPriority, _] = buildPriorityMappings();
-	return reasonPriority->at(reason);
-}
-
-DataMovementReason priorityToDataMovementReason(int priority) {
-	auto [_, priorityReason] = buildPriorityMappings();
-	return priorityReason->at(priority);
-}
-
 RelocateData::RelocateData()
   : priority(-1), boundaryPriority(-1), healthPriority(-1), reason(RelocateReason::OTHER), startTime(-1),
     dataMoveId(anonymousShardId), workFactor(0), wantsNewServers(false), cancellable(false),
@@ -586,33 +533,32 @@ void complete(RelocateData const& relocation, std::map<UID, Busyness>& busymap,
 }
 
 // Cancels in-flight data moves intersecting with range.
-ACTOR Future<Void> cancelDataMove(class DDQueue* self, KeyRange range, const DDEnabledState* ddEnabledState);
+Future<Void> cancelDataMove(class DDQueue* self, KeyRange range, const DDEnabledState* ddEnabledState);
 
-ACTOR Future<Void> dataDistributionRelocator(class DDQueue* self,
-                                             RelocateData rd,
-                                             Future<Void> prevCleanup,
-                                             const DDEnabledState* ddEnabledState);
+Future<Void> dataDistributionRelocator(class DDQueue* self,
+                                       RelocateData rd,
+                                       Future<Void> prevCleanup,
+                                       const DDEnabledState* ddEnabledState);
 
-ACTOR Future<Void> getSourceServersForRange(DDQueue* self,
-                                            RelocateData input,
-                                            PromiseStream<RelocateData> output,
-                                            Reference<FlowLock> fetchLock) {
+Future<Void> getSourceServersForRange(DDQueue* self,
+                                      RelocateData input,
+                                      PromiseStream<RelocateData> output,
+                                      Reference<FlowLock> fetchLock) {
 
 	// FIXME: is the merge case needed
 	if (input.priority == SERVER_KNOBS->PRIORITY_MERGE_SHARD) {
-		wait(delay(0.5, TaskPriority::DataDistributionVeryLow));
+		co_await delay(0.5, TaskPriority::DataDistributionVeryLow);
 	} else {
-		wait(delay(0.0001, TaskPriority::DataDistributionLaunch));
+		co_await delay(0.0001, TaskPriority::DataDistributionLaunch);
 	}
 
-	wait(fetchLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser(*fetchLock);
+	co_await fetchLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser(*fetchLock);
 
-	IDDTxnProcessor::SourceServers res = wait(self->txnProcessor->getSourceServersForRange(input.keys));
+	IDDTxnProcessor::SourceServers res = co_await self->txnProcessor->getSourceServersForRange(input.keys);
 	input.src = std::move(res.srcServers);
 	input.completeSources = std::move(res.completeSources);
 	output.send(input);
-	return Void();
 }
 
 DDQueue::DDQueue(DDQueueInitParams const& params)
@@ -624,7 +570,8 @@ DDQueue::DDQueue(DDQueueInitParams const& params)
     finishMoveKeysParallelismLock(SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM),
     cleanUpDataMoveParallelismLock(SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM),
     fetchSourceLock(new FlowLock(SERVER_KNOBS->DD_FETCH_SOURCE_PARALLELISM)), activeRelocations(0),
-    queuedRelocations(0), bytesWritten(0), teamSize(params.teamSize), singleRegionTeamSize(params.singleRegionTeamSize),
+    queuedRelocations(0), pendingGateRelocations(0), bytesWritten(0), teamSize(params.teamSize),
+    singleRegionTeamSize(params.singleRegionTeamSize), pipelineFull(new AsyncVar<bool>(false)),
     output(params.relocationProducer), input(params.relocationConsumer), getShardMetrics(params.getShardMetrics),
     getTopKMetrics(params.getTopKMetrics), lastInterval(0), suppressIntervals(0),
     rawProcessingUnhealthy(new AsyncVar<bool>(false)), rawProcessingWiggle(new AsyncVar<bool>(false)),
@@ -633,6 +580,24 @@ DDQueue::DDQueue(DDQueueInitParams const& params)
     retryFindDstReasonCount(static_cast<int>(RetryFindDstReason::NumberOfTypes), 0),
     moveBytesRate(SERVER_KNOBS->DD_TRACE_MOVE_BYTES_AVERAGE_INTERVAL) {}
 
+void DDQueue::updatePipelineFull() {
+	if (pipelineSize() >= SERVER_KNOBS->DD_MAX_PIPELINE_MOVES && !pipelineFull->get()) {
+		pipelineFull->set(true);
+		TraceEvent("DDPipelineFullSet", distributorId)
+		    .suppressFor(30.0)
+		    .detail("PipelineSize", pipelineSize())
+		    .detail("PendingGateRelocations", pendingGateRelocations)
+		    .detail("PipelineLimit", SERVER_KNOBS->DD_MAX_PIPELINE_MOVES);
+	} else if (pipelineSize() < SERVER_KNOBS->DD_MAX_PIPELINE_MOVES && pipelineFull->get()) {
+		pipelineFull->set(false);
+		TraceEvent("DDPipelineFullCleared", distributorId)
+		    .suppressFor(30.0)
+		    .detail("PipelineSize", pipelineSize())
+		    .detail("PendingGateRelocations", pendingGateRelocations)
+		    .detail("PipelineLimit", SERVER_KNOBS->DD_MAX_PIPELINE_MOVES);
+	}
+}
+
 void DDQueue::startRelocation(int priority, int healthPriority) {
 	// Although PRIORITY_TEAM_REDUNDANT has lower priority than split and merge shard movement,
 	// we must count it into unhealthyRelocations; because team removers relies on unhealthyRelocations to
@@ -651,6 +616,7 @@ void DDQueue::startRelocation(int priority, int healthPriority) {
 		rawProcessingWiggle->set(true);
 	}
 	priority_relocations[priority]++;
+	updatePipelineFull();
 }
 
 void DDQueue::finishRelocation(int priority, int healthPriority) {
@@ -666,6 +632,7 @@ void DDQueue::finishRelocation(int priority, int healthPriority) {
 		}
 	}
 	priority_relocations[priority]--;
+	updatePipelineFull();
 	if (priority_relocations[SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE] == 0) {
 		rawProcessingWiggle->set(false);
 	}
@@ -1089,8 +1056,7 @@ void DDQueue::launchQueuedWork(std::set<RelocateData, std::greater<RelocateData>
 	[[maybe_unused]] int startedHere = 0;
 	double startTime = now();
 	// kick off relocators from items in the queue as need be
-	auto it = combined.begin();
-	for (; it != combined.end(); it++) {
+	for (auto it = combined.begin(); it != combined.end(); it++) {
 		RelocateData rd(*it);
 
 		// If having a bulk load task overlapping the rd range,
@@ -1328,12 +1294,13 @@ void DDQueue::enqueueCancelledDataMove(UID dataMoveId, KeyRange range, const DDE
 	    .detail("Range", range);
 }
 
+void DDQueue::refreshCounter() {
+	serverCounter.traceAll(distributorId);
+	serverCounter.clear();
+}
+
 Future<Void> DDQueue::periodicalRefreshCounter() {
-	auto f = [this]() {
-		serverCounter.traceAll(distributorId);
-		serverCounter.clear();
-	};
-	return recurring(f, SERVER_KNOBS->DD_QUEUE_COUNTER_REFRESH_INTERVAL);
+	return recurring(std::bind_front(&DDQueue::refreshCounter, this), SERVER_KNOBS->DD_QUEUE_COUNTER_REFRESH_INTERVAL);
 }
 
 int DDQueue::getUnhealthyRelocationCount() const {
@@ -1341,9 +1308,9 @@ int DDQueue::getUnhealthyRelocationCount() const {
 }
 
 // Cancel existing relocation if exists, and serialize all concurrent relocations
-ACTOR Future<Void> cancelDataMove(class DDQueue* self, KeyRange range, const DDEnabledState* ddEnabledState) {
-	state std::vector<Future<Void>> cleanup;
-	state std::vector<std::pair<KeyRange, UID>> lastObservedDataMoves;
+Future<Void> cancelDataMove(class DDQueue* self, KeyRange range, const DDEnabledState* ddEnabledState) {
+	std::vector<Future<Void>> cleanup;
+	std::vector<std::pair<KeyRange, UID>> lastObservedDataMoves;
 
 	try {
 		lastObservedDataMoves.clear();
@@ -1370,7 +1337,7 @@ ACTOR Future<Void> cancelDataMove(class DDQueue* self, KeyRange range, const DDE
 			cleanup.push_back(it->value().cancel);
 		}
 
-		wait(waitForAll(cleanup));
+		co_await waitForAll(cleanup);
 
 		// Since self->dataMoves can only be updated by relocator,
 		// and any old relocator has been cancelled (by overwrite inflightActors)
@@ -1400,8 +1367,6 @@ ACTOR Future<Void> cancelDataMove(class DDQueue* self, KeyRange range, const DDE
 	} catch (Error& e) {
 		throw e;
 	}
-
-	return Void();
 }
 
 static std::string destServersString(std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> const& bestTeams) {
@@ -1499,36 +1464,37 @@ bool getWantTrueBestIfMoveout(int priority) {
 
 // This actor relocates the specified keys to a good place.
 // The inFlightActor key range map stores the actor for each RelocateData
-ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
-                                             RelocateData rd,
-                                             Future<Void> prevCleanup,
-                                             const DDEnabledState* ddEnabledState) {
-	state Promise<Void> errorOut(self->error);
-	state TraceInterval relocateShardInterval("RelocateShard", rd.randomId);
-	state PromiseStream<RelocateData> dataTransferComplete(self->dataTransferComplete);
-	state PromiseStream<RelocateData> relocationComplete(self->relocationComplete);
-	state bool signalledTransferComplete = false;
-	state UID distributorId = self->distributorId;
-	state ParallelTCInfo healthyDestinations;
-
-	state bool anyHealthy = false;
-	state bool allHealthy = true;
-	state bool anyWithSource = false;
-	state bool anyDestOverloaded = false;
-	state int destOverloadedCount = 0;
-	state int stuckCount = 0;
-	state std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> bestTeams;
-	state double startTime = now();
-	state std::vector<UID> destIds;
-	state WantTrueBest wantTrueBest(isValleyFillerPriority(rd.priority));
-	state WantTrueBestIfMoveout wantTrueBestIfMoveout(getWantTrueBestIfMoveout(rd.priority));
-	state uint64_t debugID = deterministicRandom()->randomUInt64();
-	state bool enableShardMove = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD;
+Future<Void> dataDistributionRelocator(DDQueue* self,
+                                       RelocateData rd,
+                                       Future<Void> prevCleanup,
+                                       const DDEnabledState* ddEnabledState) {
+	Promise<Void> errorOut(self->error);
+	TraceInterval relocateShardInterval("RelocateShard", rd.randomId);
+	PromiseStream<RelocateData> dataTransferComplete(self->dataTransferComplete);
+	PromiseStream<RelocateData> relocationComplete(self->relocationComplete);
+	bool signalledTransferComplete = false;
+	UID distributorId = self->distributorId;
+	ParallelTCInfo healthyDestinations;
+
+	bool anyHealthy = false;
+	bool allHealthy = true;
+	bool anyWithSource = false;
+	bool anyDestOverloaded = false;
+	int destOverloadedCount = 0;
+	int stuckCount = 0;
+	std::vector<std::pair<Reference<IDataDistributionTeam>, bool>> bestTeams;
+	double startTime = now();
+	std::vector<UID> destIds;
+	WantTrueBest wantTrueBest(isValleyFillerPriority(rd.priority));
+	WantTrueBestIfMoveout wantTrueBestIfMoveout(getWantTrueBestIfMoveout(rd.priority));
+	uint64_t debugID = deterministicRandom()->randomUInt64();
+	bool enableShardMove = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD;
 
 	// We will decide doBulkLoading after prevCleanup completes.
 	// rd.bulkLoadTask.present() is just the default value.
-	state bool doBulkLoading = rd.bulkLoadTask.present();
+	bool doBulkLoading = rd.bulkLoadTask.present();
 
+	Error err;
 	try {
 		if (now() - self->lastInterval < 1.0) {
 			relocateShardInterval.severity = SevDebug;
@@ -1554,7 +1520,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 			ASSERT(inFlightRange.value().dataMoveId == rd.dataMoveId);
 			inFlightRange.value().cancellable = false;
 
-			wait(prevCleanup);
+			co_await prevCleanup;
 
 			if (doBulkLoading) {
 				// Wait until the overlapped old bulkload relocator get cancelled.
@@ -1562,14 +1528,15 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 				// No more change will be made to the metadata.
 				// At this point, we check if the current rd.bulkLoadTask is outdated.
 				// If yes, do not do bulkload.
-				state Transaction tr(self->txnProcessor->context());
-				loop {
+				Transaction tr(self->txnProcessor->context());
+				while (true) {
+					Error innerErr;
 					try {
 						BulkLoadTaskState currentBulkLoadTaskState =
-						    wait(getBulkLoadTask(&tr,
-						                         rd.bulkLoadTask.get().coreState.getRange(),
-						                         rd.bulkLoadTask.get().coreState.getTaskId(),
-						                         { BulkLoadPhase::Triggered, BulkLoadPhase::Running }));
+						    co_await getBulkLoadTask(&tr,
+						                             rd.bulkLoadTask.get().coreState.getRange(),
+						                             rd.bulkLoadTask.get().coreState.getTaskId(),
+						                             { BulkLoadPhase::Triggered, BulkLoadPhase::Running });
 						TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskDataMoveLaunched", self->distributorId)
 						    .detail("TrackID", rd.randomId)
 						    .detail("DataMovePriority", rd.priority)
@@ -1581,30 +1548,31 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 						    .detail("ExistingTaskRange", currentBulkLoadTaskState.getRange());
 						break;
 					} catch (Error& e) {
-						if (e.code() == error_code_bulkload_task_outdated) {
-							// Notify the bulkload task actor to exit. This avoid bulkload task engine
-							// gets stuck in case a new task overwrite this task on metadata. To schedule
-							// the new task, the old task actor has to exit.
-							if (rd.bulkLoadTask.get().completeAck.canBeSet()) {
-								rd.bulkLoadTask.get().completeAck.sendError(bulkload_task_outdated());
-							}
-							doBulkLoading = false;
-							// we do not want to reset rd.bulkLoadTask here because
-							// the busyness calculation is based on whether rd.bulkLoadTask is set.
-							// When calculating the busyness, we do not count the work for any
-							// rd with bulkLoadTask set.
-							// TODO(BulkLoad): reset rd.bulkLoadTask here for the risk of overloading the source
-							// servers.
-							TraceEvent(SevWarn, "DDBulkLoadTaskFallbackToNormalDataMove", self->distributorId)
-							    .detail("TrackID", rd.randomId)
-							    .detail("DataMovePriority", rd.priority)
-							    .detail("JobID", rd.bulkLoadTask.get().coreState.getJobId())
-							    .detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId())
-							    .detail("TaskRange", rd.bulkLoadTask.get().coreState.getRange());
-							break;
+						innerErr = e;
+					}
+					if (innerErr.code() == error_code_bulkload_task_outdated) {
+						// Notify the bulkload task actor to exit. This avoid bulkload task engine
+						// gets stuck in case a new task overwrite this task on metadata. To schedule
+						// the new task, the old task actor has to exit.
+						if (rd.bulkLoadTask.get().completeAck.canBeSet()) {
+							rd.bulkLoadTask.get().completeAck.sendError(bulkload_task_outdated());
 						}
-						wait(tr.onError(e));
+						doBulkLoading = false;
+						// we do not want to reset rd.bulkLoadTask here because
+						// the busyness calculation is based on whether rd.bulkLoadTask is set.
+						// When calculating the busyness, we do not count the work for any
+						// rd with bulkLoadTask set.
+						// TODO(BulkLoad): reset rd.bulkLoadTask here for the risk of overloading the source
+						// servers.
+						TraceEvent(SevWarn, "DDBulkLoadTaskFallbackToNormalDataMove", self->distributorId)
+						    .detail("TrackID", rd.randomId)
+						    .detail("DataMovePriority", rd.priority)
+						    .detail("JobID", rd.bulkLoadTask.get().coreState.getJobId())
+						    .detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId())
+						    .detail("TaskRange", rd.bulkLoadTask.get().coreState.getRange());
+						break;
 					}
+					co_await tr.onError(innerErr);
 				}
 				DataMoveType dataMoveType = newDataMoveType(doBulkLoading);
 				rd.dataMoveId = newDataMoveId(
@@ -1643,31 +1611,31 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 			// are all invalid. i.e. the range of new relocators is match to the range of prevCleanup.
 		}
 
-		state Optional<StorageMetrics> parentMetrics;
-		state StorageMetrics metrics;
+		Optional<StorageMetrics> parentMetrics;
+		StorageMetrics metrics;
 
 		Future<StorageMetrics> metricsF =
 		    brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(rd.keys)));
 		if (rd.getParentRange().present()) {
 			Future<StorageMetrics> parentMetricsF =
 			    brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(rd.getParentRange().get())));
-			wait(store(metrics, metricsF) && store(parentMetrics, parentMetricsF));
+			co_await (store(metrics, metricsF) && store(parentMetrics, parentMetricsF));
 		} else {
-			wait(store(metrics, metricsF));
+			metrics = co_await metricsF;
 		}
 
-		state std::unordered_set<uint64_t> excludedDstPhysicalShards;
+		std::unordered_set<uint64_t> excludedDstPhysicalShards;
 
 		ASSERT(rd.src.size());
-		loop {
+		while (true) {
 			destOverloadedCount = 0;
 			stuckCount = 0;
-			state uint64_t physicalShardIDCandidate = UID().first();
-			state bool forceToUseNewPhysicalShard = false;
-			loop {
-				state int tciIndex = 0;
-				state bool foundTeams = true;
-				state bool bestTeamReady = false;
+			uint64_t physicalShardIDCandidate = UID().first();
+			bool forceToUseNewPhysicalShard = false;
+			while (true) {
+				int tciIndex = 0;
+				bool foundTeams = true;
+				bool bestTeamReady = false;
 				anyHealthy = false;
 				allHealthy = true;
 				anyWithSource = false;
@@ -1681,7 +1649,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 						Future<std::pair<Optional<Reference<IDataDistributionTeam>>, bool>> fbestTeam =
 						    brokenPromiseToNever(self->teamCollections[tciIndex].getTeam.getReply(req));
 						bestTeamReady = fbestTeam.isReady();
-						std::pair<Optional<Reference<IDataDistributionTeam>>, bool> bestTeam = wait(fbestTeam);
+						std::pair<Optional<Reference<IDataDistributionTeam>>, bool> bestTeam = co_await fbestTeam;
 						if (tciIndex > 0 && !bestTeamReady) {
 							// self->shardsAffectedByTeamFailure->moveShard must be called without any waits after
 							// getting the destination team or we could miss failure notifications for the storage
@@ -1781,7 +1749,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 						Future<std::pair<Optional<Reference<IDataDistributionTeam>>, bool>> fbestTeam =
 						    brokenPromiseToNever(self->teamCollections[tciIndex].getTeam.getReply(req));
 						bestTeamReady = fbestTeam.isReady();
-						std::pair<Optional<Reference<IDataDistributionTeam>>, bool> bestTeam = wait(fbestTeam);
+						std::pair<Optional<Reference<IDataDistributionTeam>>, bool> bestTeam = co_await fbestTeam;
 						if (doBulkLoading) {
 							TraceEvent(bulkLoadVerboseEventSev(),
 							           "DDBulkLoadTaskRelocatorBestTeamReceived",
@@ -1945,7 +1913,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 							throw data_move_dest_team_not_found();
 						}
 					}
-					wait(delay(SERVER_KNOBS->DEST_OVERLOADED_DELAY, TaskPriority::DataDistributionLaunch));
+					co_await delay(SERVER_KNOBS->DEST_OVERLOADED_DELAY, TaskPriority::DataDistributionLaunch);
 				} else {
 					CODE_PROBE(true, "did not find a healthy destination team on the first attempt");
 					stuckCount++;
@@ -1984,7 +1952,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 					if (rd.isRestore() && stuckCount > 50) {
 						throw data_move_dest_team_not_found();
 					}
-					wait(delay(SERVER_KNOBS->BEST_TEAM_STUCK_DELAY, TaskPriority::DataDistributionLaunch));
+					co_await delay(SERVER_KNOBS->BEST_TEAM_STUCK_DELAY, TaskPriority::DataDistributionLaunch);
 				}
 				// When forceToUseNewPhysicalShard = false, we get paired primary team and remote team
 				// However, this may be failed
@@ -2044,9 +2012,9 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 			inFlightRange.value().cancellable = false;
 
 			destIds.clear();
-			state std::vector<UID> healthyIds;
-			state std::vector<UID> extraIds;
-			state std::vector<ShardsAffectedByTeamFailure::Team> destinationTeams;
+			std::vector<UID> healthyIds;
+			std::vector<UID> extraIds;
+			std::vector<ShardsAffectedByTeamFailure::Team> destinationTeams;
 
 			for (int i = 0; i < bestTeams.size(); i++) {
 				auto& serverIds = bestTeams[i].first->getServerIDs();
@@ -2094,7 +2062,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 			    .detail("BulkLoadTaskID", doBulkLoading ? rd.bulkLoadTask.get().coreState.getTaskId().toString() : "");
 
 			// Sanity check
-			state int totalIds = 0;
+			int totalIds = 0;
 			for (auto& destTeam : destinationTeams) {
 				totalIds += destTeam.servers.size();
 			}
@@ -2132,8 +2100,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 			self->serverCounter.increaseForTeam(destIds, rd.reason, DDQueue::ServerCounter::LaunchedDest);
 			self->serverCounter.increaseForTeam(extraIds, rd.reason, DDQueue::ServerCounter::LaunchedDest);
 
-			state Error error = success();
-			state Promise<Void> dataMovementComplete;
+			Error error = success();
+			Promise<Void> dataMovementComplete;
 			// Move keys from source to destination by changing the serverKeyList and keyServerList system keys
 			std::unique_ptr<MoveKeysParams> params;
 			if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
@@ -2167,87 +2135,88 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 				                                          doBulkLoading ? rd.bulkLoadTask.get().coreState
 				                                                        : Optional<BulkLoadTaskState>());
 			}
-			state Future<Void> doMoveKeys = self->txnProcessor->moveKeys(*params);
-			state Future<Void> pollHealth =
-			    signalledTransferComplete ? Never()
+			Future<Void> doMoveKeys = self->txnProcessor->moveKeys(*params);
+			Future<Void> pollHealth = signalledTransferComplete
+			                              ? Never()
 			                              : delay(SERVER_KNOBS->HEALTH_POLL_TIME, TaskPriority::DataDistributionLaunch);
 			try {
-				loop {
-					choose {
-						when(wait(doMoveKeys)) {
-							if (extraIds.size()) {
-								destIds.insert(destIds.end(), extraIds.begin(), extraIds.end());
-								healthyIds.insert(healthyIds.end(), extraIds.begin(), extraIds.end());
-								extraIds.clear();
-								ASSERT(totalIds == destIds.size()); // Sanity check the destIDs before we move keys
-								std::unique_ptr<MoveKeysParams> params;
-								if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-									params = std::make_unique<MoveKeysParams>(rd.dataMoveId,
-									                                          std::vector<KeyRange>{ rd.keys },
-									                                          destIds,
-									                                          healthyIds,
-									                                          self->lock,
-									                                          Promise<Void>(),
-									                                          &self->startMoveKeysParallelismLock,
-									                                          &self->finishMoveKeysParallelismLock,
-									                                          self->teamCollections.size() > 1,
-									                                          relocateShardInterval.pairID,
-									                                          ddEnabledState,
-									                                          CancelConflictingDataMoves::False,
-									                                          rd.bulkLoadTask.present()
-									                                              ? rd.bulkLoadTask.get().coreState
-									                                              : Optional<BulkLoadTaskState>());
-								} else {
-									params = std::make_unique<MoveKeysParams>(rd.dataMoveId,
-									                                          rd.keys,
-									                                          destIds,
-									                                          healthyIds,
-									                                          self->lock,
-									                                          Promise<Void>(),
-									                                          &self->startMoveKeysParallelismLock,
-									                                          &self->finishMoveKeysParallelismLock,
-									                                          self->teamCollections.size() > 1,
-									                                          relocateShardInterval.pairID,
-									                                          ddEnabledState,
-									                                          CancelConflictingDataMoves::False,
-									                                          rd.bulkLoadTask.present()
-									                                              ? rd.bulkLoadTask.get().coreState
-									                                              : Optional<BulkLoadTaskState>());
-								}
-								doMoveKeys = self->txnProcessor->moveKeys(*params);
+				while (true) {
+					Future<Void> dataMovementFuture =
+					    signalledTransferComplete ? Never() : dataMovementComplete.getFuture();
+					auto res = co_await race(doMoveKeys, pollHealth, dataMovementFuture);
+					if (res.index() == 0) {
+						if (extraIds.size()) {
+							destIds.insert(destIds.end(), extraIds.begin(), extraIds.end());
+							healthyIds.insert(healthyIds.end(), extraIds.begin(), extraIds.end());
+							extraIds.clear();
+							ASSERT(totalIds == destIds.size()); // Sanity check the destIDs before we move keys
+							std::unique_ptr<MoveKeysParams> params;
+							if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+								params = std::make_unique<MoveKeysParams>(rd.dataMoveId,
+								                                          std::vector<KeyRange>{ rd.keys },
+								                                          destIds,
+								                                          healthyIds,
+								                                          self->lock,
+								                                          Promise<Void>(),
+								                                          &self->startMoveKeysParallelismLock,
+								                                          &self->finishMoveKeysParallelismLock,
+								                                          self->teamCollections.size() > 1,
+								                                          relocateShardInterval.pairID,
+								                                          ddEnabledState,
+								                                          CancelConflictingDataMoves::False,
+								                                          rd.bulkLoadTask.present()
+								                                              ? rd.bulkLoadTask.get().coreState
+								                                              : Optional<BulkLoadTaskState>());
 							} else {
-								self->fetchKeysComplete.insert(rd);
-								if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-									auto ranges = self->dataMoves.getAffectedRangesAfterInsertion(rd.keys);
-									if (ranges.size() == 1 && static_cast<KeyRange>(ranges[0]) == rd.keys &&
-									    ranges[0].value.id == rd.dataMoveId && !ranges[0].value.cancel.isValid()) {
-										self->dataMoves.insert(rd.keys, DDQueue::DDDataMove());
-										TraceEvent(SevVerbose, "DequeueDataMoveOnSuccess", self->distributorId)
-										    .detail("DataMoveID", rd.dataMoveId)
-										    .detail("DataMoveRange", rd.keys);
-									}
-								}
-								break;
+								params = std::make_unique<MoveKeysParams>(rd.dataMoveId,
+								                                          rd.keys,
+								                                          destIds,
+								                                          healthyIds,
+								                                          self->lock,
+								                                          Promise<Void>(),
+								                                          &self->startMoveKeysParallelismLock,
+								                                          &self->finishMoveKeysParallelismLock,
+								                                          self->teamCollections.size() > 1,
+								                                          relocateShardInterval.pairID,
+								                                          ddEnabledState,
+								                                          CancelConflictingDataMoves::False,
+								                                          rd.bulkLoadTask.present()
+								                                              ? rd.bulkLoadTask.get().coreState
+								                                              : Optional<BulkLoadTaskState>());
 							}
-						}
-						when(wait(pollHealth)) {
-							if (!healthyDestinations.isHealthy()) {
-								if (!signalledTransferComplete) {
-									signalledTransferComplete = true;
-									self->dataTransferComplete.send(rd);
+							doMoveKeys = self->txnProcessor->moveKeys(*params);
+						} else {
+							self->fetchKeysComplete.insert(rd);
+							if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+								auto ranges = self->dataMoves.getAffectedRangesAfterInsertion(rd.keys);
+								if (ranges.size() == 1 && static_cast<KeyRange>(ranges[0]) == rd.keys &&
+								    ranges[0].value.id == rd.dataMoveId && !ranges[0].value.cancel.isValid()) {
+									self->dataMoves.insert(rd.keys, DDQueue::DDDataMove());
+									TraceEvent(SevVerbose, "DequeueDataMoveOnSuccess", self->distributorId)
+									    .detail("DataMoveID", rd.dataMoveId)
+									    .detail("DataMoveRange", rd.keys);
 								}
 							}
-							pollHealth = signalledTransferComplete ? Never()
-							                                       : delay(SERVER_KNOBS->HEALTH_POLL_TIME,
-							                                               TaskPriority::DataDistributionLaunch);
+							break;
 						}
-						when(wait(signalledTransferComplete ? Never() : dataMovementComplete.getFuture())) {
-							self->fetchKeysComplete.insert(rd);
+					} else if (res.index() == 1) {
+						if (!healthyDestinations.isHealthy()) {
 							if (!signalledTransferComplete) {
 								signalledTransferComplete = true;
 								self->dataTransferComplete.send(rd);
 							}
 						}
+						pollHealth = signalledTransferComplete
+						                 ? Never()
+						                 : delay(SERVER_KNOBS->HEALTH_POLL_TIME, TaskPriority::DataDistributionLaunch);
+					} else if (res.index() == 2) {
+						self->fetchKeysComplete.insert(rd);
+						if (!signalledTransferComplete) {
+							signalledTransferComplete = true;
+							self->dataTransferComplete.send(rd);
+						}
+					} else {
+						UNREACHABLE();
 					}
 				}
 			} catch (Error& e) {
@@ -2256,12 +2225,13 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 
 			//TraceEvent("RelocateShardFinished", distributorId).detail("RelocateId", relocateShardInterval.pairID);
 
-			if (error.code() != error_code_move_to_removed_server) {
+			if (error.code() != error_code_move_to_removed_server &&
+			    error.code() != error_code_finish_move_keys_too_many_retries) {
 				if (!error.code()) {
 					try {
-						wait(healthyDestinations
-						         .updateStorageMetrics()); // prevent a gap between the polling for an increase in
-						                                   // storage metrics and decrementing data in flight
+						co_await healthyDestinations
+						    .updateStorageMetrics(); // prevent a gap between the polling for an increase in
+						                             // storage metrics and decrementing data in flight
 					} catch (Error& e) {
 						error = e;
 					}
@@ -2356,7 +2326,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 							    .detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId());
 						}
 					}
-					return Void();
+					co_return;
 				} else {
 					if (doBulkLoading) {
 						TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskRelocatorError")
@@ -2399,46 +2369,45 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueue* self,
 					}
 				}
 
-				wait(delay(SERVER_KNOBS->RETRY_RELOCATESHARD_DELAY, TaskPriority::DataDistributionLaunch));
+				co_await delay(SERVER_KNOBS->RETRY_RELOCATESHARD_DELAY, TaskPriority::DataDistributionLaunch);
 			}
 		}
 	} catch (Error& e) {
-		state Error err = e;
-		TraceEvent(relocateShardInterval.end(), distributorId)
+		err = e;
+	}
+
+	TraceEvent(relocateShardInterval.end(), distributorId).errorUnsuppressed(err).detail("Duration", now() - startTime);
+	if (now() - startTime > 600) {
+		TraceEvent(SevWarnAlways, "RelocateShardTooLong")
 		    .errorUnsuppressed(err)
-		    .detail("Duration", now() - startTime);
-		if (now() - startTime > 600) {
-			TraceEvent(SevWarnAlways, "RelocateShardTooLong")
-			    .errorUnsuppressed(err)
-			    .detail("Duration", now() - startTime)
-			    .detail("Dest", describe(destIds))
-			    .detail("Src", describe(rd.src));
-		}
-		if (!signalledTransferComplete)
-			dataTransferComplete.send(rd);
-
-		relocationComplete.send(rd);
-
-		if (doBulkLoading && e.code() != error_code_actor_cancelled && e.code() != error_code_movekeys_conflict) {
-			TraceEvent(SevWarnAlways, "DDBulkLoadTaskRelocatorFailed", self->distributorId)
-			    .errorUnsuppressed(e)
-			    .detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId())
-			    .detail("JobID", rd.bulkLoadTask.get().coreState.getJobId())
-			    .detail("TaskRange", rd.bulkLoadTask.get().coreState.getRange());
-		}
+		    .detail("Duration", now() - startTime)
+		    .detail("Dest", describe(destIds))
+		    .detail("Src", describe(rd.src));
+	}
+	if (!signalledTransferComplete)
+		dataTransferComplete.send(rd);
 
-		if (err.code() == error_code_data_move_dest_team_not_found) {
-			wait(cancelDataMove(self, rd.keys, ddEnabledState));
-			TraceEvent(SevWarnAlways, "RelocateShardCancelDataMoveTeamNotFound")
-			    .detail("Src", describe(rd.src))
-			    .detail("DataMoveMetaData", rd.dataMove != nullptr ? rd.dataMove->meta.toString() : "Empty");
-		} else if (err.code() != error_code_actor_cancelled && err.code() != error_code_data_move_cancelled) {
-			if (errorOut.canBeSet()) {
-				errorOut.sendError(err);
-			}
+	relocationComplete.send(rd);
+
+	if (doBulkLoading && err.code() != error_code_actor_cancelled && err.code() != error_code_movekeys_conflict) {
+		TraceEvent(SevWarnAlways, "DDBulkLoadTaskRelocatorFailed", self->distributorId)
+		    .errorUnsuppressed(err)
+		    .detail("TaskID", rd.bulkLoadTask.get().coreState.getTaskId())
+		    .detail("JobID", rd.bulkLoadTask.get().coreState.getJobId())
+		    .detail("TaskRange", rd.bulkLoadTask.get().coreState.getRange());
+	}
+
+	if (err.code() == error_code_data_move_dest_team_not_found) {
+		co_await cancelDataMove(self, rd.keys, ddEnabledState);
+		TraceEvent(SevWarnAlways, "RelocateShardCancelDataMoveTeamNotFound")
+		    .detail("Src", describe(rd.src))
+		    .detail("DataMoveMetaData", rd.dataMove != nullptr ? rd.dataMove->meta.toString() : "Empty");
+	} else if (err.code() != error_code_actor_cancelled && err.code() != error_code_data_move_cancelled) {
+		if (errorOut.canBeSet()) {
+			errorOut.sendError(err);
 		}
-		throw err;
 	}
+	throw err;
 }
 
 inline double getWorstCpu(const HealthMetrics& metrics, const std::vector<UID>& ids) {
@@ -2457,18 +2426,18 @@ inline double getWorstCpu(const HealthMetrics& metrics, const std::vector<UID>&
 
 // Move the shard with the top K highest read density of sourceTeam's to destTeam if sourceTeam has much more read
 // load than destTeam
-ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
-                                     DataMovementReason moveReason,
-                                     Reference<IDataDistributionTeam> sourceTeam,
-                                     Reference<IDataDistributionTeam> destTeam,
-                                     bool primary,
-                                     TraceEvent* traceEvent) {
+Future<bool> rebalanceReadLoad(DDQueue* self,
+                               DataMovementReason moveReason,
+                               Reference<IDataDistributionTeam> sourceTeam,
+                               Reference<IDataDistributionTeam> destTeam,
+                               bool primary,
+                               TraceEvent* traceEvent) {
 	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 		traceEvent->detail("CancelingDueToSimulationSpeedup", true);
-		return false;
+		co_return false;
 	}
 
-	state std::vector<KeyRange> shards = self->shardsAffectedByTeamFailure->getShardsFor(
+	std::vector<KeyRange> shards = self->shardsAffectedByTeamFailure->getShardsFor(
 	    ShardsAffectedByTeamFailure::Team(sourceTeam->getServerIDs(), primary));
 	traceEvent->detail("ShardsInSource", shards.size());
 	// For read rebalance if there is just 1 hot shard remained, move this shard to another server won't solve the
@@ -2476,7 +2445,7 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
 	// TODO: This situation should be solved by split and merge
 	if (shards.size() <= 1) {
 		traceEvent->detail("SkipReason", "NoShardOnSource");
-		return false;
+		co_return false;
 	}
 
 	// Check lastAsSource, at most SERVER_KNOBS->READ_REBALANCE_SRC_PARALLELISM shards can be moved within a sample
@@ -2485,7 +2454,7 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
 	// metrics.
 	if (self->timeThrottle(sourceTeam->getServerIDs())) {
 		traceEvent->detail("SkipReason", "SourceTeamThrottle");
-		return false;
+		co_return false;
 	}
 	// check team difference
 	auto srcLoad = sourceTeam->getReadLoad(false), destLoad = destTeam->getReadLoad();
@@ -2494,21 +2463,21 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
 	// read bandwidth difference is less than 30% of src load
 	if ((1.0 - SERVER_KNOBS->READ_REBALANCE_DIFF_FRAC) * srcLoad <= destLoad) {
 		traceEvent->detail("SkipReason", "TeamTooSimilar");
-		return false;
+		co_return false;
 	}
 	// randomly choose topK shards
 	int topK = std::max(1, std::min(int(0.1 * shards.size()), SERVER_KNOBS->READ_REBALANCE_SHARD_TOPK));
-	state Future<HealthMetrics> healthMetrics = self->txnProcessor->getHealthMetrics(true);
-	state GetTopKMetricsRequest req(shards,
-	                                topK,
-	                                (srcLoad - destLoad) * SERVER_KNOBS->READ_REBALANCE_MAX_SHARD_FRAC,
-	                                std::min(srcLoad / shards.size(), SERVER_KNOBS->READ_REBALANCE_MIN_READ_BYTES_KS));
-	state GetTopKMetricsReply reply = wait(brokenPromiseToNever(self->getTopKMetrics.getReply(req)));
-	wait(ready(healthMetrics));
+	Future<HealthMetrics> healthMetrics = self->txnProcessor->getHealthMetrics(true);
+	GetTopKMetricsRequest req(shards,
+	                          topK,
+	                          (srcLoad - destLoad) * SERVER_KNOBS->READ_REBALANCE_MAX_SHARD_FRAC,
+	                          std::min(srcLoad / shards.size(), SERVER_KNOBS->READ_REBALANCE_MIN_READ_BYTES_KS));
+	GetTopKMetricsReply reply = co_await brokenPromiseToNever(self->getTopKMetrics.getReply(req));
+	co_await ready(healthMetrics);
 	auto cpu = getWorstCpu(healthMetrics.get(), sourceTeam->getServerIDs());
 	if (cpu < SERVER_KNOBS->READ_REBALANCE_CPU_THRESHOLD) { // 15.0 +- (0.3 * 15) < 20.0
 		traceEvent->detail("SkipReason", "LowReadLoad").detail("WorstSrcCpu", cpu);
-		return false;
+		co_return false;
 	}
 
 	auto& metricsList = reply.shardMetrics;
@@ -2518,7 +2487,7 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
 
 	if (metricsList.empty()) {
 		traceEvent->detail("SkipReason", "NoEligibleShards");
-		return false;
+		co_return false;
 	}
 
 	auto& [shard, metrics] = metricsList[0];
@@ -2539,46 +2508,46 @@ ACTOR Future<bool> rebalanceReadLoad(DDQueue* self,
 
 			self->serverCounter.increaseForTeam(
 			    serverIds, RelocateReason::REBALANCE_READ, DDQueue::ServerCounter::ProposedSource);
-			return true;
+			co_return true;
 		}
 	}
 	traceEvent->detail("SkipReason", "ShardNotPresent");
-	return false;
+	co_return false;
 }
 
 // Move a random shard from sourceTeam if sourceTeam has much more data than provided destTeam
-ACTOR static Future<bool> rebalanceTeams(DDQueue* self,
-                                         DataMovementReason moveReason,
-                                         Reference<IDataDistributionTeam const> sourceTeam,
-                                         Reference<IDataDistributionTeam const> destTeam,
-                                         bool primary,
-                                         TraceEvent* traceEvent) {
+static Future<bool> rebalanceTeams(DDQueue* self,
+                                   DataMovementReason moveReason,
+                                   Reference<IDataDistributionTeam const> sourceTeam,
+                                   Reference<IDataDistributionTeam const> destTeam,
+                                   bool primary,
+                                   TraceEvent* traceEvent) {
 	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
 		traceEvent->detail("CancelingDueToSimulationSpeedup", true);
-		return false;
+		co_return false;
 	}
 
 	Promise<int64_t> req;
 	self->getAverageShardBytes.send(req);
 
-	state int64_t averageShardBytes = wait(req.getFuture());
-	state std::vector<KeyRange> shards = self->shardsAffectedByTeamFailure->getShardsFor(
+	int64_t averageShardBytes = co_await req.getFuture();
+	std::vector<KeyRange> shards = self->shardsAffectedByTeamFailure->getShardsFor(
 	    ShardsAffectedByTeamFailure::Team(sourceTeam->getServerIDs(), primary));
 
 	traceEvent->detail("AverageShardBytes", averageShardBytes).detail("ShardsInSource", shards.size());
 
 	if (!shards.size()) {
 		traceEvent->detail("SkipReason", "NoShardOnSource");
-		return false;
+		co_return false;
 	}
 
-	state KeyRange moveShard;
-	state StorageMetrics metrics;
-	state int retries = 0;
+	KeyRange moveShard;
+	StorageMetrics metrics;
+	int retries = 0;
 	while (retries < SERVER_KNOBS->REBALANCE_MAX_RETRIES) {
-		state KeyRange testShard = deterministicRandom()->randomChoice(shards);
+		KeyRange testShard = deterministicRandom()->randomChoice(shards);
 		StorageMetrics testMetrics =
-		    wait(brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(testShard))));
+		    co_await brokenPromiseToNever(self->getShardMetrics.getReply(GetMetricsRequest(testShard)));
 		if (testMetrics.bytes > metrics.bytes) {
 			moveShard = testShard;
 			metrics = testMetrics;
@@ -2601,7 +2570,7 @@ ACTOR static Future<bool> rebalanceTeams(DDQueue* self,
 
 	if (sourceAndDestTooSimilar || metrics.bytes == 0) {
 		traceEvent->detail("SkipReason", sourceAndDestTooSimilar ? "TeamTooSimilar" : "ShardZeroSize");
-		return false;
+		co_return false;
 	}
 
 	// Verify the shard is still in ShardsAffectedByTeamFailure
@@ -2615,36 +2584,36 @@ ACTOR static Future<bool> rebalanceTeams(DDQueue* self,
 
 			self->serverCounter.increaseForTeam(
 			    sourceTeam->getServerIDs(), RelocateReason::REBALANCE_DISK, DDQueue::ServerCounter::ProposedSource);
-			return true;
+			co_return true;
 		}
 	}
 
 	traceEvent->detail("SkipReason", "ShardNotPresent");
-	return false;
+	co_return false;
 }
 
-ACTOR Future<SrcDestTeamPair> getSrcDestTeams(DDQueue* self,
-                                              int teamCollectionIndex,
-                                              GetTeamRequest srcReq,
-                                              GetTeamRequest destReq,
-                                              int priority,
-                                              TraceEvent* traceEvent) {
+Future<SrcDestTeamPair> getSrcDestTeams(DDQueue* self,
+                                        int teamCollectionIndex,
+                                        GetTeamRequest srcReq,
+                                        GetTeamRequest destReq,
+                                        int priority,
+                                        TraceEvent* traceEvent) {
 
-	state std::pair<Optional<ITeamRef>, bool> randomTeam =
-	    wait(brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(destReq)));
+	std::pair<Optional<ITeamRef>, bool> randomTeam =
+	    co_await brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(destReq));
 	traceEvent->detail("DestTeam", printable(randomTeam.first.mapRef(&IDataDistributionTeam::getDesc)));
 
 	if (randomTeam.first.present()) {
-		state std::pair<Optional<ITeamRef>, bool> loadedTeam =
-		    wait(brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(srcReq)));
+		std::pair<Optional<ITeamRef>, bool> loadedTeam =
+		    co_await brokenPromiseToNever(self->teamCollections[teamCollectionIndex].getTeam.getReply(srcReq));
 
 		traceEvent->detail("SourceTeam", printable(loadedTeam.first.mapRef(&IDataDistributionTeam::getDesc)));
 
 		if (loadedTeam.first.present()) {
-			return std::make_pair(loadedTeam.first.get(), randomTeam.first.get());
+			co_return std::make_pair(loadedTeam.first.get(), randomTeam.first.get());
 		}
 	}
-	return {};
+	co_return SrcDestTeamPair{};
 }
 
 Future<SrcDestTeamPair> DDQueue::getSrcDestTeams(const int& teamCollectionIndex,
@@ -2670,11 +2639,11 @@ Future<bool> DDQueue::rebalanceTeams(DataMovementReason moveReason,
 	return ::rebalanceTeams(this, moveReason, sourceTeam, destTeam, primary, traceEvent);
 }
 
-ACTOR Future<bool> getSkipRebalanceValue(Reference<IDDTxnProcessor> txnProcessor, bool readRebalance) {
-	Optional<Value> val = wait(txnProcessor->readRebalanceDDIgnoreKey());
+Future<bool> getSkipRebalanceValue(Reference<IDDTxnProcessor> txnProcessor, bool readRebalance) {
+	Optional<Value> val = co_await txnProcessor->readRebalanceDDIgnoreKey();
 
 	if (!val.present())
-		return false;
+		co_return false;
 
 	bool skipCurrentLoop = false;
 	// NOTE: check special value "" and "on" might written in old version < 7.2
@@ -2689,33 +2658,32 @@ ACTOR Future<bool> getSkipRebalanceValue(Reference<IDDTxnProcessor> txnProcessor
 		skipCurrentLoop = true;
 	}
 
-	return skipCurrentLoop;
+	co_return skipCurrentLoop;
 }
 
-ACTOR Future<Void> BgDDLoadRebalance(DDQueue* self, int teamCollectionIndex, DataMovementReason reason) {
-	state int resetCount = 0;
-	state double lastRead = 0;
-	state bool skipCurrentLoop = false;
-	state const bool readRebalance = isDataMovementForReadBalancing(reason);
-	state const std::string moveType =
-	    isDataMovementForMountainChopper(reason) ? "BgDDMountainChopper" : "BgDDValleyFiller";
-	state int ddPriority = dataMovementPriority(reason);
-	state bool mcMove = isDataMovementForMountainChopper(reason);
-	state PreferLowerReadUtil preferLowerReadTeam = readRebalance || SERVER_KNOBS->DD_PREFER_LOW_READ_UTIL_TEAM
-	                                                    ? PreferLowerReadUtil::True
-	                                                    : PreferLowerReadUtil::False;
-	state double rebalancePollingInterval = 0;
-
-	loop {
-		state bool moved = false;
-		state Reference<IDataDistributionTeam> sourceTeam;
-		state Reference<IDataDistributionTeam> destTeam;
+Future<Void> BgDDLoadRebalance(DDQueue* self, int teamCollectionIndex, DataMovementReason reason) {
+	int resetCount = 0;
+	double lastRead = 0;
+	bool skipCurrentLoop = false;
+	const bool readRebalance = isDataMovementForReadBalancing(reason);
+	const std::string moveType = isDataMovementForMountainChopper(reason) ? "BgDDMountainChopper" : "BgDDValleyFiller";
+	int ddPriority = dataMovementPriority(reason);
+	bool mcMove = isDataMovementForMountainChopper(reason);
+	PreferLowerReadUtil preferLowerReadTeam = readRebalance || SERVER_KNOBS->DD_PREFER_LOW_READ_UTIL_TEAM
+	                                              ? PreferLowerReadUtil::True
+	                                              : PreferLowerReadUtil::False;
+	double rebalancePollingInterval = 0;
+
+	while (true) {
+		bool moved = false;
+		Reference<IDataDistributionTeam> sourceTeam;
+		Reference<IDataDistributionTeam> destTeam;
 
 		// NOTE: the DD throttling relies on DDQueue, so here just trigger the balancer periodically
-		wait(delay(rebalancePollingInterval, TaskPriority::DataDistributionLaunch));
+		co_await delay(rebalancePollingInterval, TaskPriority::DataDistributionLaunch);
 		try {
 			if ((now() - lastRead) > SERVER_KNOBS->BG_REBALANCE_SWITCH_CHECK_INTERVAL) {
-				wait(store(skipCurrentLoop, getSkipRebalanceValue(self->txnProcessor, readRebalance)));
+				skipCurrentLoop = co_await getSkipRebalanceValue(self->txnProcessor, readRebalance);
 				lastRead = now();
 			}
 
@@ -2744,7 +2712,7 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueue* self, int teamCollectionIndex, Dat
 			rebalancePollingInterval =
 			    std::max(rebalancePollingInterval / 2, SERVER_KNOBS->BG_REBALANCE_POLLING_INTERVAL);
 
-			state TraceEvent traceEvent(mcMove ? "MountainChopperSample" : "ValleyFillerSample", self->distributorId);
+			TraceEvent traceEvent(mcMove ? "MountainChopperSample" : "ValleyFillerSample", self->distributorId);
 			traceEvent.suppressFor(5.0);
 			GetTeamRequest srcReq = GetTeamRequest(mcMove ? TeamSelect::WANT_TRUE_BEST : TeamSelect::ANY,
 			                                       PreferLowerDiskUtil::False,
@@ -2759,18 +2727,18 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueue* self, int teamCollectionIndex, Dat
 			                                        preferLowerReadTeam,
 			                                        PreferWithinShardLimit::False,
 			                                        ForReadBalance(readRebalance));
-			state Future<SrcDestTeamPair> getTeamFuture =
+			Future<SrcDestTeamPair> getTeamFuture =
 			    self->getSrcDestTeams(teamCollectionIndex, srcReq, destReq, ddPriority, &traceEvent);
-			wait(ready(getTeamFuture));
+			co_await ready(getTeamFuture);
 			sourceTeam = getTeamFuture.get().first;
 			destTeam = getTeamFuture.get().second;
 
 			// clang-format off
 			if (sourceTeam.isValid() && destTeam.isValid()) {
 				if (readRebalance) {
-					wait(store(moved,self->rebalanceReadLoad( reason, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent)));
+					moved = co_await self->rebalanceReadLoad( reason, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent);
 				} else {
-					wait(store(moved,self->rebalanceTeams( reason, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent)));
+					moved = co_await self->rebalanceTeams( reason, sourceTeam, destTeam, teamCollectionIndex == 0, &traceEvent);
 				}
 			}
 			// clang-format on
@@ -2802,6 +2770,32 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueue* self, int teamCollectionIndex, Dat
 	}
 }
 
+// Gates the relocation input stream by the pipeline limit. Cancellations and high-priority
+// moves (>= PRIORITY_TEAM_UNHEALTHY) always pass through immediately so that failure recovery
+// is never blocked by stuck or zombie moves holding pipeline slots. All other relocations are
+// held when the pipeline is full, waiting for pipelineFull to become false before forwarding.
+// The global isDDPipelineControlEnabled() flag (cleared by disableDDPipelineControl()) also
+// bypasses the gate, allowing the test harness to open up the pipeline so DD can quiesce.
+// We poll it via delay() rather than AsyncVar to avoid cross-process callbacks in simulation.
+ACTOR Future<Void> pipelineGateActor(Reference<DDQueue> self,
+                                     FutureStream<RelocateShard> input,
+                                     PromiseStream<RelocateShard> output) {
+	loop {
+		state RelocateShard rs = waitNext(input);
+		if (!rs.cancelled && rs.priority < SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY) {
+			while (self->pipelineFull->get() && isDDPipelineControlEnabled()) {
+				TraceEvent("DDPipelineFull", self->distributorId)
+				    .suppressFor(30.0)
+				    .detail("PipelineFull", self->pipelineFull->get());
+				wait(self->pipelineFull->onChange() || delay(1.0));
+			}
+		}
+		self->pendingGateRelocations++;
+		self->updatePipelineFull();
+		output.send(rs);
+	}
+}
+
 struct DDQueueImpl {
 	ACTOR static Future<Void> run(Reference<DDQueue> self,
 	                              Reference<AsyncVar<bool>> processingUnhealthy,
@@ -2820,6 +2814,11 @@ struct DDQueueImpl {
 		state Future<Void> onCleanUpDataMoveActorError =
 		    actorCollection(self->addBackgroundCleanUpDataMoveActor.getFuture());
 
+		// Gate the input stream by the pipeline limit so that DD never tracks more
+		// than DD_MAX_PIPELINE_MOVES relocations at once (queued + in-flight).
+		state PromiseStream<RelocateShard> gatedRelocationStream;
+		state Future<Void> pipelineGate = pipelineGateActor(self, self->input, gatedRelocationStream);
+
 		for (int i = 0; i < self->teamCollections.size(); i++) {
 			ddQueueFutures.push_back(
 			    BgDDLoadRebalance(self.getPtr(), i, DataMovementReason::REBALANCE_OVERUTILIZED_TEAM));
@@ -2854,7 +2853,9 @@ struct DDQueueImpl {
 				ASSERT(launchData.startTime == -1 && keysToLaunchFrom.empty());
 
 				choose {
-					when(RelocateShard rs = waitNext(self->input)) {
+					when(RelocateShard rs = waitNext(gatedRelocationStream.getFuture())) {
+						self->pendingGateRelocations--;
+						self->updatePipelineFull();
 						if (rs.isRestore()) {
 							ASSERT(rs.dataMove != nullptr);
 							ASSERT(rs.dataMoveId.isValid());
@@ -2921,6 +2922,9 @@ struct DDQueueImpl {
 						    .detail("HighestPriority", highestPriorityRelocation)
 						    .detail("BytesWritten", self->moveBytesRate.getTotal())
 						    .detail("BytesWrittenAverageRate", self->moveBytesRate.getAverage())
+						    .detail("PipelineSize", self->pipelineSize())
+						    .detail("PipelineLimit", SERVER_KNOBS->DD_MAX_PIPELINE_MOVES)
+						    .detail("PendingGateRelocations", self->pendingGateRelocations)
 						    .detail("PriorityRecoverMove",
 						            self->priority_relocations[SERVER_KNOBS->PRIORITY_RECOVER_MOVE])
 						    .detail("PriorityRebalanceUnderutilizedTeam",
@@ -2991,6 +2995,7 @@ struct DDQueueImpl {
 					}
 					when(wait(self->error.getFuture())) {} // Propagate errors from dataDistributionRelocator
 					when(wait(waitForAll(ddQueueFutures))) {}
+					when(wait(pipelineGate)) {} // Propagate errors from pipelineGateActor
 					when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) {
 						r.send(self->getUnhealthyRelocationCount());
 					}
diff --git a/fdbserver/include/fdbserver/DDRelocationQueue.h b/fdbserver/datadistributor/DDRelocationQueue.h
similarity index 96%
rename from fdbserver/include/fdbserver/DDRelocationQueue.h
rename to fdbserver/datadistributor/DDRelocationQueue.h
index 12b207e0985..c07f016d9cf 100644
--- a/fdbserver/include/fdbserver/DDRelocationQueue.h
+++ b/fdbserver/datadistributor/DDRelocationQueue.h
@@ -22,8 +22,8 @@
 
 #include <numeric>
 
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/MovingWindow.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "MovingWindow.h"
 
 // send request/signal to DDRelocationQueue through interface
 // call synchronous method from components outside DDRelocationQueue
@@ -210,8 +210,7 @@ class DDQueue : public IDDRelocationQueue, public ReferenceCounted<DDQueue> {
 
 		void traceAll(const UID& debugId = UID()) const {
 			auto it = counter.cbegin();
-			int count = 0;
-			for (; count < SERVER_KNOBS->DD_QUEUE_COUNTER_MAX_LOG && it != counter.cend(); ++count, ++it) {
+			for (int count = 0; count < SERVER_KNOBS->DD_QUEUE_COUNTER_MAX_LOG && it != counter.cend(); ++count, ++it) {
 				TraceEvent event("DDQueueServerCounter", debugId);
 				event.detail("ServerId", it->first);
 				traceReasonItem(&event, it->second);
@@ -255,10 +254,17 @@ class DDQueue : public IDDRelocationQueue, public ReferenceCounted<DDQueue> {
 
 	int activeRelocations;
 	int queuedRelocations;
+	int pendingGateRelocations; // forwarded by pipelineGateActor but not yet consumed by DDQueue
 	int64_t bytesWritten;
 	int teamSize;
 	int singleRegionTeamSize;
 
+	int pipelineSize() const { return pendingGateRelocations + activeRelocations + queuedRelocations; }
+
+	void updatePipelineFull();
+
+	Reference<AsyncVar<bool>> pipelineFull;
+
 	std::map<UID, Busyness> busymap; // UID is serverID
 	std::map<UID, Busyness> destBusymap; // UID is serverID
 
@@ -358,6 +364,8 @@ class DDQueue : public IDDRelocationQueue, public ReferenceCounted<DDQueue> {
 	// Schedules cancellation of a data move.
 	void enqueueCancelledDataMove(UID dataMoveId, KeyRange range, const DDEnabledState* ddEnabledState);
 
+	void refreshCounter();
+
 	Future<Void> periodicalRefreshCounter();
 
 	int getUnhealthyRelocationCount() const override;
diff --git a/fdbserver/DDShardTracker.actor.cpp b/fdbserver/datadistributor/DDShardTracker.cpp
similarity index 85%
rename from fdbserver/DDShardTracker.actor.cpp
rename to fdbserver/datadistributor/DDShardTracker.cpp
index 60d090d871f..e50e955098e 100644
--- a/fdbserver/DDShardTracker.actor.cpp
+++ b/fdbserver/datadistributor/DDShardTracker.cpp
@@ -1,5 +1,5 @@
 /*
- * DataDistributionTracker.actor.cpp
+ * DDShardTracker.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,16 +22,17 @@
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/ShardSizing.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/CodeProbe.h"
 #include "flow/FastRef.h"
 #include "flow/Trace.h"
-#include "fdbserver/DDShardTracker.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/datadistributor/DDShardTracker.h"
+#include "flow/CoroUtils.h"
 
 // The used bandwidth of a shard. The higher the value is, the busier the shard is.
 enum BandwidthStatus { BandwidthStatusLow, BandwidthStatusNormal, BandwidthStatusHigh };
@@ -57,19 +58,19 @@ ReadBandwidthStatus getReadBandwidthStatus(StorageMetrics const& metrics) {
 	}
 }
 
-ACTOR Future<Void> updateMaxShardSize(Reference<AsyncVar<int64_t>> dbSizeEstimate,
-                                      Reference<AsyncVar<Optional<int64_t>>> maxShardSize) {
-	state int64_t lastDbSize = 0;
-	state int64_t granularity = g_network->isSimulated() ? SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY_SIM
-	                                                     : SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY;
-	loop {
+Future<Void> updateMaxShardSize(Reference<AsyncVar<int64_t>> dbSizeEstimate,
+                                Reference<AsyncVar<Optional<int64_t>>> maxShardSize) {
+	int64_t lastDbSize = 0;
+	int64_t granularity = g_network->isSimulated() ? SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY_SIM
+	                                               : SERVER_KNOBS->DD_SHARD_SIZE_GRANULARITY;
+	while (true) {
 		auto sizeDelta = std::abs(dbSizeEstimate->get() - lastDbSize);
 		if (sizeDelta > granularity || !maxShardSize->get().present()) {
 			auto v = getMaxShardSize(dbSizeEstimate->get());
 			maxShardSize->set(v);
 			lastDbSize = dbSizeEstimate->get();
 		}
-		wait(dbSizeEstimate->onChange());
+		co_await dbSizeEstimate->onChange();
 	}
 }
 
@@ -80,65 +81,6 @@ void restartShardTrackers(DataDistributionTracker* self,
 
 // Gets the permitted size and IO bounds for a shard. A shard that starts at allKeys.begin
 //  (i.e. '') will have a permitted size of 0, since the database can contain no data.
-ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize) {
-	ShardSizeBounds bounds;
-
-	if (shard.begin >= keyServersKeys.begin) {
-		bounds.max.bytes = SERVER_KNOBS->KEY_SERVER_SHARD_BYTES;
-	} else {
-		bounds.max.bytes = maxShardSize;
-	}
-
-	bounds.max.bytesWrittenPerKSecond = bounds.max.infinity;
-	bounds.max.iosPerKSecond = bounds.max.infinity;
-	bounds.max.bytesReadPerKSecond = bounds.max.infinity;
-	bounds.max.opsReadPerKSecond = bounds.max.infinity;
-
-	// The first shard can have arbitrarily small size
-	if (shard.begin == allKeys.begin) {
-		bounds.min.bytes = 0;
-	} else {
-		bounds.min.bytes = maxShardSize / SERVER_KNOBS->SHARD_BYTES_RATIO;
-	}
-
-	bounds.min.bytesWrittenPerKSecond = 0;
-	bounds.min.iosPerKSecond = 0;
-	bounds.min.bytesReadPerKSecond = 0;
-	bounds.min.opsReadPerKSecond = 0;
-
-	// The permitted error is 1/3 of the general-case minimum bytes (even in the special case where this is the last
-	// shard)
-	bounds.permittedError.bytes = bounds.max.bytes / SERVER_KNOBS->SHARD_BYTES_RATIO / 3;
-	bounds.permittedError.bytesWrittenPerKSecond = bounds.permittedError.infinity;
-	bounds.permittedError.iosPerKSecond = bounds.permittedError.infinity;
-	bounds.permittedError.bytesReadPerKSecond = bounds.permittedError.infinity;
-	bounds.permittedError.opsReadPerKSecond = bounds.permittedError.infinity;
-
-	return bounds;
-}
-
-int64_t getMaxShardSize(double dbSizeEstimate) {
-	int64_t size = std::min((SERVER_KNOBS->MIN_SHARD_BYTES + (int64_t)std::sqrt(std::max<double>(dbSizeEstimate, 0)) *
-	                                                             SERVER_KNOBS->SHARD_BYTES_PER_SQRT_BYTES) *
-	                            SERVER_KNOBS->SHARD_BYTES_RATIO,
-	                        (int64_t)SERVER_KNOBS->MAX_SHARD_BYTES);
-	if (SERVER_KNOBS->ALLOW_LARGE_SHARD) {
-		size = std::max(size, static_cast<int64_t>(SERVER_KNOBS->MAX_LARGE_SHARD_BYTES));
-	}
-
-	TraceEvent("MaxShardSize")
-	    .suppressFor(60.0)
-	    .detail("Bytes", size)
-	    .detail("EstimatedDbSize", dbSizeEstimate)
-	    .detail("SqrtBytes", SERVER_KNOBS->SHARD_BYTES_PER_SQRT_BYTES)
-	    .detail("AllowLargeShard", SERVER_KNOBS->ALLOW_LARGE_SHARD);
-	return size;
-}
-
-bool ddLargeTeamEnabled() {
-	return SERVER_KNOBS->DD_MAX_SHARDS_ON_LARGE_TEAMS > 0 && !SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA;
-}
-
 // Returns the shard size bounds as well as whether `keys` a read hot shard.
 std::pair<ShardSizeBounds, bool> calculateShardSizeBounds(
     const KeyRange& keys,
@@ -209,13 +151,13 @@ std::pair<ShardSizeBounds, bool> calculateShardSizeBounds(
 	return { bounds, readHotShard };
 }
 
-ACTOR Future<Void> shardUsableRegions(DataDistributionTracker::SafeAccessor self, KeyRange keys) {
+Future<Void> shardUsableRegions(DataDistributionTracker::SafeAccessor self, KeyRange keys) {
 	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	ASSERT(SERVER_KNOBS->DD_SHARD_USABLE_REGION_CHECK_RATE > 0);
-	wait(yieldedFuture(self()->readyToStart.getFuture()));
+	co_await yieldedFuture(self()->readyToStart.getFuture());
 	double expectedCompletionSeconds = self()->shards->size() * 1.0 / SERVER_KNOBS->DD_SHARD_USABLE_REGION_CHECK_RATE;
 	double delayTime = deterministicRandom()->random01() * expectedCompletionSeconds;
-	wait(delayJittered(delayTime));
+	co_await delayJittered(delayTime);
 	auto [newTeam, previousTeam] = self()->shardsAffectedByTeamFailure->getTeamsForFirstShard(keys);
 	if (newTeam.size() < self()->usableRegions) {
 		TraceEvent(SevWarn, "ShardUsableRegionMismatch", self()->distributorId)
@@ -229,20 +171,19 @@ ACTOR Future<Void> shardUsableRegions(DataDistributionTracker::SafeAccessor self
 		RelocateShard rs(keys, DataMovementReason::POPULATE_REGION, RelocateReason::OTHER);
 		self()->output.send(rs);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
-                                     KeyRange keys,
-                                     Reference<AsyncVar<Optional<ShardMetrics>>> shardMetrics,
-                                     bool whenDDInit) {
-	state BandwidthStatus bandwidthStatus =
+Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
+                               KeyRange keys,
+                               Reference<AsyncVar<Optional<ShardMetrics>>> shardMetrics,
+                               bool whenDDInit) {
+	BandwidthStatus bandwidthStatus =
 	    shardMetrics->get().present() ? getBandwidthStatus(shardMetrics->get().get().metrics) : BandwidthStatusNormal;
-	state double lastLowBandwidthStartTime =
+	double lastLowBandwidthStartTime =
 	    shardMetrics->get().present() ? shardMetrics->get().get().lastLowBandwidthStartTime : now();
-	state int shardCount = shardMetrics->get().present() ? shardMetrics->get().get().shardCount : 1;
-	state bool initWithNewMetrics = whenDDInit;
-	wait(delay(0, TaskPriority::DataDistribution));
+	int shardCount = shardMetrics->get().present() ? shardMetrics->get().get().shardCount : 1;
+	bool initWithNewMetrics = whenDDInit;
+	co_await delay(0, TaskPriority::DataDistribution);
 
 	DisabledTraceEvent(SevDebug, "TrackShardMetricsStarting", self()->distributorId)
 	    .detail("Keys", keys)
@@ -250,8 +191,8 @@ ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
 	    .detail("StartingMetrics", shardMetrics->get().present() ? shardMetrics->get().get().metrics.bytes : 0);
 
 	try {
-		loop {
-			state ShardSizeBounds bounds;
+		while (true) {
+			ShardSizeBounds bounds;
 			bool readHotShard;
 			std::tie(bounds, readHotShard) = calculateShardSizeBounds(keys, shardMetrics, bandwidthStatus);
 
@@ -262,15 +203,15 @@ ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
 				self()->readHotShard.send(keys);
 			}
 
-			loop {
+			while (true) {
 				// metrics.second is the number of key-ranges (i.e., shards) in the 'keys' key-range
 				std::pair<Optional<StorageMetrics>, int> metrics =
-				    wait(self()->db->waitStorageMetrics(keys,
-				                                        bounds.min,
-				                                        bounds.max,
-				                                        bounds.permittedError,
-				                                        CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
-				                                        shardCount));
+				    co_await self()->db->waitStorageMetrics(keys,
+				                                            bounds.min,
+				                                            bounds.max,
+				                                            bounds.permittedError,
+				                                            CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
+				                                            shardCount);
 				if (metrics.first.present()) {
 					BandwidthStatus newBandwidthStatus = getBandwidthStatus(metrics.first.get());
 					if (newBandwidthStatus == BandwidthStatusLow && bandwidthStatus != BandwidthStatusLow) {
@@ -349,11 +290,11 @@ ACTOR Future<Void> trackShardMetrics(DataDistributionTracker::SafeAccessor self,
 	}
 }
 
-ACTOR Future<Void> readHotDetector(DataDistributionTracker* self) {
+Future<Void> readHotDetector(DataDistributionTracker* self) {
 	try {
-		loop {
-			state KeyRange keys = waitNext(self->readHotShard.getFuture());
-			Standalone<VectorRef<ReadHotRangeWithMetrics>> readHotRanges = wait(self->db->getReadHotRanges(keys));
+		while (true) {
+			KeyRange keys = co_await self->readHotShard.getFuture();
+			Standalone<VectorRef<ReadHotRangeWithMetrics>> readHotRanges = co_await self->db->getReadHotRanges(keys);
 
 			for (const auto& keyRange : readHotRanges) {
 				TraceEvent("ReadHotRangeLog")
@@ -374,33 +315,20 @@ ACTOR Future<Void> readHotDetector(DataDistributionTracker* self) {
 	}
 }
 
-/*
-ACTOR Future<Void> extrapolateShardBytes( Reference<AsyncVar<Optional<int64_t>>> inBytes,
-Reference<AsyncVar<Optional<int64_t>>> outBytes ) { state std::deque< std::pair<double,int64_t> > past; loop { wait(
-inBytes->onChange() ); if( inBytes->get().present() ) { past.emplace_back(now(),inBytes->get().get()); if
-(past.size() < 2) outBytes->set( inBytes->get() ); else { while (past.size() > 1 && past.end()[-1].first -
-past.begin()[1].first > 1.0) past.pop_front(); double rate = std::max(0.0,
-double(past.end()[-1].second-past.begin()[0].second)/(past.end()[-1].first - past.begin()[0].first)); outBytes->set(
-inBytes->get().get() + rate * 10.0 );
-            }
-        }
-    }
-}*/
-
-ACTOR Future<int64_t> getFirstSize(Reference<AsyncVar<Optional<ShardMetrics>>> stats) {
-	loop {
+Future<int64_t> getFirstSize(Reference<AsyncVar<Optional<ShardMetrics>>> stats) {
+	while (true) {
 		if (stats->get().present())
-			return stats->get().get().metrics.bytes;
-		wait(stats->onChange());
+			co_return stats->get().get().metrics.bytes;
+		co_await stats->onChange();
 	}
 }
 
-ACTOR Future<Void> changeSizes(DataDistributionTracker* self,
-                               KeyRange keys,
-                               int64_t oldShardsEndingSize,
-                               std::string context) {
-	state std::vector<Future<int64_t>> sizes;
-	state std::vector<Future<int64_t>> systemSizes;
+Future<Void> changeSizes(DataDistributionTracker* self,
+                         KeyRange keys,
+                         int64_t oldShardsEndingSize,
+                         std::string context) {
+	std::vector<Future<int64_t>> sizes;
+	std::vector<Future<int64_t>> systemSizes;
 	for (auto it : self->shards->intersectingRanges(keys)) {
 		Future<int64_t> thisSize = getFirstSize(it->value().stats);
 		sizes.push_back(thisSize);
@@ -409,8 +337,8 @@ ACTOR Future<Void> changeSizes(DataDistributionTracker* self,
 		}
 	}
 
-	wait(waitForAll(sizes));
-	wait(yield(TaskPriority::DataDistribution));
+	co_await waitForAll(sizes);
+	co_await yield(TaskPriority::DataDistribution);
 
 	int64_t newShardsStartingSize = 0;
 	for (const auto& size : sizes) {
@@ -434,7 +362,6 @@ ACTOR Future<Void> changeSizes(DataDistributionTracker* self,
 	if (keys.begin >= systemKeys.begin) {
 		self->systemSizeEstimate -= oldShardsEndingSize;
 	}
-	return Void();
 }
 
 struct HasBeenTrueFor : ReferenceCounted<HasBeenTrueFor> {
@@ -571,13 +498,13 @@ bool faultLinesMatch(std::vector<RangeToSplit>& ranges, std::vector<std::vector<
 	return true;
 }
 
-ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
-                                 KeyRange keys,
-                                 Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                 ShardSizeBounds shardBounds,
-                                 RelocateReason reason) {
-	state StorageMetrics metrics = shardSize->get().get().metrics;
-	state BandwidthStatus bandwidthStatus = getBandwidthStatus(metrics);
+Future<Void> shardSplitter(DataDistributionTracker* self,
+                           KeyRange keys,
+                           Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
+                           ShardSizeBounds shardBounds,
+                           RelocateReason reason) {
+	StorageMetrics metrics = shardSize->get().get().metrics;
+	BandwidthStatus bandwidthStatus = getBandwidthStatus(metrics);
 
 	// Split
 	CODE_PROBE(true, "shard to be split");
@@ -589,8 +516,8 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
 	splitMetrics.iosPerKSecond = splitMetrics.infinity;
 	splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidthSec
 
-	state Standalone<VectorRef<KeyRef>> splitKeys =
-	    wait(self->db->splitStorageMetrics(keys, splitMetrics, metrics, SERVER_KNOBS->MIN_SHARD_BYTES));
+	Standalone<VectorRef<KeyRef>> splitKeys =
+	    co_await self->db->splitStorageMetrics(keys, splitMetrics, metrics, SERVER_KNOBS->MIN_SHARD_BYTES);
 	// fprintf(stderr, "split keys:\n");
 	// for( int i = 0; i < splitKeys.size(); i++ ) {
 	//	fprintf(stderr, "   %s\n", printable(splitKeys[i]).c_str());
@@ -613,21 +540,24 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
 	if (numShards > 1) {
 		executeShardSplit(self, keys, splitKeys, shardSize, true, reason);
 	} else {
-		wait(delay(1.0, TaskPriority::DataDistribution)); // In case the reason the split point was off was due to a
-		                                                  // discrepancy between storage servers
+		co_await delay(1.0, TaskPriority::DataDistribution); // In case the reason the split point was off was due to a
+		                                                     // discrepancy between storage servers
 	}
-	return Void();
 }
 
-ACTOR Future<Void> brokenPromiseToReady(Future<Void> f) {
+Future<Void> brokenPromiseToReady(Future<Void> f) {
 	try {
-		wait(f);
+		co_await f;
 	} catch (Error& e) {
 		if (e.code() != error_code_broken_promise) {
 			throw;
 		}
 	}
-	return Void();
+}
+
+// True if [begin, end) spans across the system keyspace boundary.
+static bool crossesCriticalSystemBoundary(KeyRef begin, KeyRef end) {
+	return begin < systemKeys.begin && end > systemKeys.begin;
 }
 
 static bool shardForwardMergeFeasible(DataDistributionTracker* self, KeyRange const& keys, KeyRangeRef nextRange) {
@@ -635,6 +565,11 @@ static bool shardForwardMergeFeasible(DataDistributionTracker* self, KeyRange co
 		return false;
 	}
 
+	// Don't merge across critical system metadata boundaries
+	if (crossesCriticalSystemBoundary(keys.begin, nextRange.end)) {
+		return false;
+	}
+
 	if (self->userRangeConfig->rangeContaining(keys.begin)->range().end < nextRange.end) {
 		return false;
 	}
@@ -655,6 +590,11 @@ static bool shardBackwardMergeFeasible(DataDistributionTracker* self, KeyRange c
 		return false;
 	}
 
+	// Don't merge across critical system metadata boundaries
+	if (crossesCriticalSystemBoundary(prevRange.begin, keys.end)) {
+		return false;
+	}
+
 	if (self->userRangeConfig->rangeContaining(keys.begin)->range().begin > prevRange.begin) {
 		return false;
 	}
@@ -776,7 +716,7 @@ Future<Void> shardMerger(DataDistributionTracker* self,
 
 	int64_t systemBytes = keys.begin >= systemKeys.begin ? shardSize->get().get().metrics.bytes : 0;
 
-	loop {
+	while (true) {
 		Optional<ShardMetrics> newMetrics;
 		if (!forwardComplete) {
 			if (nextIter->range().end == allKeys.end) {
@@ -927,10 +867,10 @@ Future<Void> shardMerger(DataDistributionTracker* self,
 	return Void();
 }
 
-ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
-                                  KeyRange keys,
-                                  Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                  Reference<HasBeenTrueFor> wantsToMerge) {
+Future<Void> shardEvaluator(DataDistributionTracker* self,
+                            KeyRange keys,
+                            Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
+                            Reference<HasBeenTrueFor> wantsToMerge) {
 	Future<Void> onChange = shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange());
 
 	// There are the bounds inside of which we are happy with the shard size.
@@ -989,26 +929,25 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds, reason);
 	}
 
-	wait(onChange);
-	return Void();
+	co_await onChange;
 }
 
-ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
-                                KeyRange keys,
-                                Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
-	wait(yieldedFuture(self()->readyToStart.getFuture()));
+Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
+                          KeyRange keys,
+                          Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
+	co_await yieldedFuture(self()->readyToStart.getFuture());
 
 	if (!shardSize->get().present())
-		wait(shardSize->onChange());
+		co_await shardSize->onChange();
 
 	if (!self()->maxShardSize->get().present())
-		wait(yieldedFuture(self()->maxShardSize->onChange()));
+		co_await yieldedFuture(self()->maxShardSize->onChange());
 
 	// Since maxShardSize will become present for all shards at once, avoid slow tasks with a short delay
-	wait(delay(0, TaskPriority::DataDistribution));
+	co_await delay(0, TaskPriority::DataDistribution);
 
 	// Survives multiple calls to shardEvaluator and keeps merges from happening too quickly.
-	state Reference<HasBeenTrueFor> wantsToMerge(new HasBeenTrueFor(shardSize->get()));
+	Reference<HasBeenTrueFor> wantsToMerge(new HasBeenTrueFor(shardSize->get()));
 
 	/*TraceEvent("ShardTracker", self()->distributorId)
 	    .detail("Begin", keys.begin)
@@ -1019,21 +958,20 @@ ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
 	    .detail("BytesPerKSec", shardSize->get().get().bytesWrittenPerKSecond);*/
 
 	try {
-		loop {
+		while (true) {
 			while (self()->bulkLoadEnabled && self()->bulkLoadTaskCollection->bulkLoading(keys)) {
 				TraceEvent(SevWarn, "ShardBoundaryChangeDisabledForBulkLoad", self()->distributorId)
 				    .suppressFor(60.0)
 				    .detail("KeyRange", keys);
-				wait(delay(SERVER_KNOBS->DD_BULKLOAD_SHARD_BOUNDARY_CHANGE_DELAY_SEC *
-				               deterministicRandom()->random01() +
-				           60));
+				co_await delay(
+				    SERVER_KNOBS->DD_BULKLOAD_SHARD_BOUNDARY_CHANGE_DELAY_SEC * deterministicRandom()->random01() + 60);
 			}
 			// Use the current known size to check for (and start) splits and merges.
-			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge));
+			co_await shardEvaluator(self(), keys, shardSize, wantsToMerge);
 
 			// We could have a lot of actors being released from the previous wait at the same time. Immediately
 			// calling delay(0) mitigates the resulting SlowTask
-			wait(delay(0, TaskPriority::DataDistribution));
+			co_await delay(0, TaskPriority::DataDistribution);
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled && e.code() != error_code_dd_tracker_cancelled) {
@@ -1083,20 +1021,20 @@ void restartShardTrackers(DataDistributionTracker* self,
 	}
 }
 
-ACTOR Future<Void> trackInitialShards(DataDistributionTracker* self, Reference<InitialDataDistribution> initData) {
+Future<Void> trackInitialShards(DataDistributionTracker* self, Reference<InitialDataDistribution> initData) {
 	TraceEvent("TrackInitialShards", self->distributorId).detail("InitialShardCount", initData->shards.size());
 
 	// This line reduces the priority of shard initialization to prevent interference with failure monitoring.
 	// SOMEDAY: Figure out what this priority should actually be
-	wait(delay(0.0, TaskPriority::DataDistribution));
+	co_await delay(0.0, TaskPriority::DataDistribution);
 
-	state std::vector<Key> customBoundaries;
+	std::vector<Key> customBoundaries;
 	for (auto it : self->userRangeConfig->ranges()) {
 		customBoundaries.push_back(it->range().begin);
 	}
 
-	state int s;
-	state int customBoundary = 0;
+	int s{ 0 };
+	int customBoundary = 0;
 	for (s = 0; s < initData->shards.size() - 1; s++) {
 		Key beginKey = initData->shards[s].key;
 		Key endKey = initData->shards[s + 1].key;
@@ -1110,38 +1048,46 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker* self, Reference<I
 			customBoundary++;
 		}
 		restartShardTrackers(self, KeyRangeRef(beginKey, endKey), Optional<ShardMetrics>(), true);
-		wait(yield(TaskPriority::DataDistribution));
+		co_await yield(TaskPriority::DataDistribution);
 	}
 
+	TraceEvent("TrackInitialShardsComplete", self->distributorId).detail("ShardsTracked", s);
+
+	double changeSizesStart = now();
 	Future<Void> initialSize = changeSizes(self, KeyRangeRef(allKeys.begin, allKeys.end), 0, "ShardInit");
 	self->readyToStart.send(Void());
-	wait(initialSize);
-	self->maxShardSizeUpdater = updateMaxShardSize(self->dbSizeEstimate, self->maxShardSize);
+	co_await initialSize;
 
-	return Void();
+	TraceEvent("TrackInitialShardsMetricsComplete", self->distributorId)
+	    .detail("ElapsedSeconds", now() - changeSizesStart);
+
+	// DDInitDone bookends DDInitRunning — marks DD fully operational. Uses DD* prefix so
+	// the full startup sequence can be queried with Type="DDInit*" in trace logs.
+	TraceEvent("DDInitDone", self->distributorId);
+	self->maxShardSizeUpdater = updateMaxShardSize(self->dbSizeEstimate, self->maxShardSize);
 }
 
-ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, GetTopKMetricsRequest req) {
-	state Future<Void> onChange;
-	state std::vector<GetTopKMetricsReply::KeyRangeStorageMetrics> returnMetrics;
+Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, GetTopKMetricsRequest req) {
+	Future<Void> onChange;
+	std::vector<GetTopKMetricsReply::KeyRangeStorageMetrics> returnMetrics;
 	// random pick a portion of shard
 	if (req.keys.size() > SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT) {
 		deterministicRandom()->randomShuffle(req.keys, 0, SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT);
 	}
 	try {
-		loop {
+		while (true) {
 			onChange = Future<Void>();
 			returnMetrics.clear();
-			state int64_t minReadLoad = -1;
-			state int64_t maxReadLoad = -1;
-			state int i;
+			int64_t minReadLoad = -1;
+			int64_t maxReadLoad = -1;
+			int i{ 0 };
 			for (i = 0; i < SERVER_KNOBS->DD_SHARD_COMPARE_LIMIT && i < req.keys.size(); ++i) {
 				auto range = req.keys[i];
 				StorageMetrics metrics;
 				for (auto t : self->shards->intersectingRanges(range)) {
 					auto& stats = t.value().stats;
 					if (!stats->get().present()) {
-						onChange = stats->onChange();
+						onChange = brokenPromiseToReady(stats->onChange());
 						break;
 					}
 					metrics += t.value().stats->get().get().metrics;
@@ -1161,7 +1107,7 @@ ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, Get
 					}
 				}
 
-				wait(yield());
+				co_await yield();
 			}
 			// FIXME(xwang): Do we need to track slow task here?
 			if (!onChange.isValid()) {
@@ -1178,9 +1124,9 @@ ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, Get
 					                        minReadLoad,
 					                        maxReadLoad));
 				}
-				return Void();
+				co_return;
 			}
-			wait(onChange);
+			co_await onChange;
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
@@ -1189,28 +1135,25 @@ ACTOR Future<Void> fetchTopKShardMetrics_impl(DataDistributionTracker* self, Get
 	}
 }
 
-ACTOR Future<Void> fetchTopKShardMetrics(DataDistributionTracker* self, GetTopKMetricsRequest req) {
-	choose {
-		// simulate time_out
-		when(wait(g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01) ? Never()
-		                                                              : fetchTopKShardMetrics_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) {
-			CODE_PROBE(true, "TopK DD_SHARD_METRICS_TIMEOUT");
-			req.reply.send(GetTopKMetricsReply());
-		}
+Future<Void> fetchTopKShardMetrics(DataDistributionTracker* self, GetTopKMetricsRequest req) {
+	// simulate time_out
+	Future<Void> f =
+	    g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01) ? Never() : fetchTopKShardMetrics_impl(self, req);
+	if (auto const res = co_await timeout(f, SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT); !res.present()) {
+		CODE_PROBE(true, "TopK DD_SHARD_METRICS_TIMEOUT");
+		req.reply.send(GetTopKMetricsReply());
 	}
-	return Void();
 }
 
-ACTOR Future<Void> fetchShardMetrics_impl(DataDistributionTracker* self, GetMetricsRequest req) {
+Future<Void> fetchShardMetrics_impl(DataDistributionTracker* self, GetMetricsRequest req) {
 	try {
-		loop {
+		while (true) {
 			Future<Void> onChange;
 			StorageMetrics returnMetrics;
 			for (auto t : self->shards->intersectingRanges(req.keys)) {
 				auto& stats = t.value().stats;
 				if (!stats->get().present()) {
-					onChange = stats->onChange();
+					onChange = brokenPromiseToReady(stats->onChange());
 					break;
 				}
 				returnMetrics += t.value().stats->get().get().metrics;
@@ -1218,10 +1161,10 @@ ACTOR Future<Void> fetchShardMetrics_impl(DataDistributionTracker* self, GetMetr
 
 			if (!onChange.isValid()) {
 				req.reply.send(returnMetrics);
-				return Void();
+				co_return;
 			}
 
-			wait(onChange);
+			co_await onChange;
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
@@ -1230,22 +1173,20 @@ ACTOR Future<Void> fetchShardMetrics_impl(DataDistributionTracker* self, GetMetr
 	}
 }
 
-ACTOR Future<Void> fetchShardMetrics(DataDistributionTracker* self, GetMetricsRequest req) {
-	choose {
-		when(wait(fetchShardMetrics_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT, TaskPriority::DataDistribution))) {
-			CODE_PROBE(true, "DD_SHARD_METRICS_TIMEOUT");
-			StorageMetrics largeMetrics;
-			largeMetrics.bytes = getMaxShardSize(self->dbSizeEstimate->get());
-			req.reply.send(largeMetrics);
-		}
+Future<Void> fetchShardMetrics(DataDistributionTracker* self, GetMetricsRequest req) {
+	if (auto const res = co_await timeout(
+	        fetchShardMetrics_impl(self, req), SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT, TaskPriority::DataDistribution);
+	    !res.present()) {
+		CODE_PROBE(true, "DD_SHARD_METRICS_TIMEOUT");
+		StorageMetrics largeMetrics;
+		largeMetrics.bytes = getMaxShardSize(self->dbSizeEstimate->get());
+		req.reply.send(largeMetrics);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, GetMetricsListRequest req) {
+Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, GetMetricsListRequest req) {
 	try {
-		loop {
+		while (true) {
 			// used to control shard limit
 			int shardNum = 0;
 			// list of metrics, regenerate on loop when full range unsuccessful
@@ -1256,7 +1197,7 @@ ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, Get
 			for (auto t = beginIter; t != endIter; ++t) {
 				auto& stats = t.value().stats;
 				if (!stats->get().present()) {
-					onChange = stats->onChange();
+					onChange = brokenPromiseToReady(stats->onChange());
 					break;
 				}
 				result.push_back_deep(result.arena(),
@@ -1271,10 +1212,10 @@ ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, Get
 
 			if (!onChange.isValid()) {
 				req.reply.send(result);
-				return Void();
+				co_return;
 			}
 
-			wait(onChange);
+			co_await onChange;
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled && !req.reply.isSet())
@@ -1283,14 +1224,12 @@ ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, Get
 	}
 }
 
-ACTOR Future<Void> fetchShardMetricsList(DataDistributionTracker* self, GetMetricsListRequest req) {
-	choose {
-		when(wait(fetchShardMetricsList_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) {
-			req.reply.sendError(timed_out());
-		}
+Future<Void> fetchShardMetricsList(DataDistributionTracker* self, GetMetricsListRequest req) {
+	if (auto const res =
+	        co_await timeout(fetchShardMetricsList_impl(self, req), SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT);
+	    !res.present()) {
+		req.reply.sendError(timed_out());
 	}
-	return Void();
 }
 
 void triggerStorageQueueRebalance(DataDistributionTracker* self, RebalanceStorageQueueRequest req) {
@@ -1348,48 +1287,95 @@ DataDistributionTracker::~DataDistributionTracker() {
 }
 
 struct DataDistributionTrackerImpl {
-	ACTOR static Future<Void> run(DataDistributionTracker* self, Reference<InitialDataDistribution> initData) {
-		state Future<Void> loggingTrigger = Void();
-		state Future<Void> readHotDetect = readHotDetector(self);
-		state Reference<EventCacheHolder> ddTrackerStatsEventHolder = makeReference<EventCacheHolder>("DDTrackerStats");
+	static Future<Void> serveAverageShardBytesReqs(DataDistributionTracker* self,
+	                                               FutureStream<Promise<int64_t>> averageShardBytes) {
+		while (true) {
+			Promise<int64_t> req = co_await averageShardBytes;
+			req.send(self->getAverageShardBytes());
+		}
+	}
+
+	static Future<Void> logDDTrackerStats(DataDistributionTracker* self) {
+		auto ddTrackerStatsEventHolder = makeReference<EventCacheHolder>("DDTrackerStats");
+		while (true) {
+			TraceEvent("DDTrackerStats", self->distributorId)
+			    .detail("Shards", self->shards->size())
+			    .detail("TotalSizeBytes", self->dbSizeEstimate->get())
+			    .detail("SystemSizeBytes", self->systemSizeEstimate)
+			    .trackLatest(ddTrackerStatsEventHolder->trackingKey);
+			co_await delay(SERVER_KNOBS->DATA_DISTRIBUTION_LOGGING_INTERVAL, TaskPriority::FlushTrace);
+		}
+	}
+
+	static Future<Void> serveGetMetricsReqs(DataDistributionTracker* self,
+	                                        FutureStream<GetMetricsRequest> getShardMetrics) {
+		while (true) {
+			GetMetricsRequest req = co_await getShardMetrics;
+			self->actors.add(fetchShardMetrics(self, req));
+		}
+	}
+
+	static Future<Void> serveGetTopKMetricsReqs(DataDistributionTracker* self,
+	                                            FutureStream<GetTopKMetricsRequest> getTopKMetrics) {
+		while (true) {
+			GetTopKMetricsRequest req = co_await getTopKMetrics;
+			self->actors.add(fetchTopKShardMetrics(self, req));
+		}
+	}
+
+	static Future<Void> serveGetMetricsListReqs(DataDistributionTracker* self,
+	                                            FutureStream<GetMetricsListRequest> getShardMetricsList) {
+		while (true) {
+			GetMetricsListRequest req = co_await getShardMetricsList;
+			self->actors.add(fetchShardMetricsList(self, req));
+		}
+	}
+
+	static Future<Void> serveTriggerStorageQueueRebalanceReqs(
+	    DataDistributionTracker* self,
+	    FutureStream<RebalanceStorageQueueRequest> triggerStorageQueueRebalanceReqs) {
+		while (true) {
+			RebalanceStorageQueueRequest req = co_await triggerStorageQueueRebalanceReqs;
+			triggerStorageQueueRebalance(self, req);
+		}
+	}
 
+	static Future<Void> serveTriggerShardBulkLoadingReqs(
+	    DataDistributionTracker* self,
+	    FutureStream<BulkLoadShardRequest> triggerShardBulkLoadingReqs) {
+		while (true) {
+			BulkLoadShardRequest req = co_await triggerShardBulkLoadingReqs;
+			createShardToBulkLoad(self, req.bulkLoadTaskState, req.cancelledDataMovePriority);
+		}
+	}
+
+	static Future<Void> serveRestartShardTrackerReqs(DataDistributionTracker* self,
+	                                                 FutureStream<KeyRange> restartShardTrackerReqs) {
+		while (true) {
+			KeyRange req = co_await restartShardTrackerReqs;
+			restartShardTrackers(self, req);
+		}
+	}
+
+	static Future<Void> run(DataDistributionTracker* self, Reference<InitialDataDistribution> initData) {
 		try {
-			wait(trackInitialShards(self, initData));
+			co_await trackInitialShards(self, initData);
+			if (*self->trackerCancelled) {
+				co_return;
+			}
 			initData.clear(); // Release reference count.
 
-			loop choose {
-				when(Promise<int64_t> req = waitNext(self->averageShardBytes)) {
-					req.send(self->getAverageShardBytes());
-				}
-				when(wait(loggingTrigger)) {
-					TraceEvent("DDTrackerStats", self->distributorId)
-					    .detail("Shards", self->shards->size())
-					    .detail("TotalSizeBytes", self->dbSizeEstimate->get())
-					    .detail("SystemSizeBytes", self->systemSizeEstimate)
-					    .trackLatest(ddTrackerStatsEventHolder->trackingKey);
-
-					loggingTrigger = delay(SERVER_KNOBS->DATA_DISTRIBUTION_LOGGING_INTERVAL, TaskPriority::FlushTrace);
-				}
-				when(GetMetricsRequest req = waitNext(self->getShardMetrics)) {
-					self->actors.add(fetchShardMetrics(self, req));
-				}
-				when(GetTopKMetricsRequest req = waitNext(self->getTopKMetrics)) {
-					self->actors.add(fetchTopKShardMetrics(self, req));
-				}
-				when(GetMetricsListRequest req = waitNext(self->getShardMetricsList)) {
-					self->actors.add(fetchShardMetricsList(self, req));
-				}
-				when(RebalanceStorageQueueRequest req = waitNext(self->triggerStorageQueueRebalance)) {
-					triggerStorageQueueRebalance(self, req);
-				}
-				when(BulkLoadShardRequest req = waitNext(self->triggerShardBulkLoading)) {
-					createShardToBulkLoad(self, req.bulkLoadTaskState, req.cancelledDataMovePriority);
-				}
-				when(wait(self->actors.getResult())) {}
-				when(KeyRange req = waitNext(self->shardsAffectedByTeamFailure->restartShardTracker.getFuture())) {
-					restartShardTrackers(self, req);
-				}
-			}
+			co_await race(
+			    logDDTrackerStats(self),
+			    readHotDetector(self),
+			    serveAverageShardBytesReqs(self, self->averageShardBytes),
+			    serveGetMetricsReqs(self, self->getShardMetrics),
+			    serveGetTopKMetricsReqs(self, self->getTopKMetrics),
+			    serveGetMetricsListReqs(self, self->getShardMetricsList),
+			    serveTriggerStorageQueueRebalanceReqs(self, self->triggerStorageQueueRebalance),
+			    serveTriggerShardBulkLoadingReqs(self, self->triggerShardBulkLoading),
+			    serveRestartShardTrackerReqs(self, self->shardsAffectedByTeamFailure->restartShardTracker.getFuture()),
+			    self->actors.getResult());
 		} catch (Error& e) {
 			if (e.code() != error_code_broken_promise) {
 				TraceEvent(SevError, "DataDistributionTrackerError", self->distributorId)
@@ -1397,7 +1383,7 @@ struct DataDistributionTrackerImpl {
 			} else {
 				TraceEvent(SevWarn, "DataDistributionTrackerError", self->distributorId).error(e);
 			}
-			throw e;
+			throw;
 		}
 	}
 };
@@ -1425,17 +1411,16 @@ Future<Void> DataDistributionTracker::run(
 // Tracks storage metrics for `keys` and updates `physicalShardStats` which is the stats for the physical shard owning
 // this key range. This function is similar to `trackShardMetrics()` and altered for physical shard. This meant to be
 // temporary. Eventually, we want a new interface to track physical shard metrics more efficiently.
-ACTOR Future<Void> trackKeyRangeInPhysicalShardMetrics(
-    Reference<IDDTxnProcessor> db,
-    KeyRange keys,
-    Reference<AsyncVar<Optional<ShardMetrics>>> shardMetrics,
-    Reference<AsyncVar<Optional<StorageMetrics>>> physicalShardStats) {
-	state BandwidthStatus bandwidthStatus =
+Future<Void> trackKeyRangeInPhysicalShardMetrics(Reference<IDDTxnProcessor> db,
+                                                 KeyRange keys,
+                                                 Reference<AsyncVar<Optional<ShardMetrics>>> shardMetrics,
+                                                 Reference<AsyncVar<Optional<StorageMetrics>>> physicalShardStats) {
+	BandwidthStatus bandwidthStatus =
 	    shardMetrics->get().present() ? getBandwidthStatus(shardMetrics->get().get().metrics) : BandwidthStatusNormal;
-	state double lastLowBandwidthStartTime =
+	double lastLowBandwidthStartTime =
 	    shardMetrics->get().present() ? shardMetrics->get().get().lastLowBandwidthStartTime : now();
-	state int shardCount = shardMetrics->get().present() ? shardMetrics->get().get().shardCount : 1;
-	wait(delay(0, TaskPriority::DataDistribution));
+	int shardCount = shardMetrics->get().present() ? shardMetrics->get().get().shardCount : 1;
+	co_await delay(0, TaskPriority::DataDistribution);
 
 	/*TraceEvent("trackKeyRangeInPhysicalShardMetricsStarting")
 	    .detail("Keys", keys)
@@ -1443,20 +1428,20 @@ ACTOR Future<Void> trackKeyRangeInPhysicalShardMetrics(
 	    .detail("StartingMetrics", shardMetrics->get().present() ? shardMetrics->get().get().metrics.bytes : 0)
 	    .detail("StartingMerges", shardMetrics->get().present() ? shardMetrics->get().get().merges : 0);*/
 
-	loop {
-		state ShardSizeBounds bounds;
+	while (true) {
+		ShardSizeBounds bounds;
 		bool readHotShard;
 		std::tie(bounds, readHotShard) = calculateShardSizeBounds(keys, shardMetrics, bandwidthStatus);
 
-		loop {
+		while (true) {
 			// metrics.second is the number of key-ranges (i.e., shards) in the 'keys' key-range
 			std::pair<Optional<StorageMetrics>, int> metrics =
-			    wait(db->waitStorageMetrics(keys,
-			                                bounds.min,
-			                                bounds.max,
-			                                bounds.permittedError,
-			                                CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
-			                                shardCount));
+			    co_await db->waitStorageMetrics(keys,
+			                                    bounds.min,
+			                                    bounds.max,
+			                                    bounds.permittedError,
+			                                    CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
+			                                    shardCount);
 			if (metrics.first.present()) {
 				BandwidthStatus newBandwidthStatus = getBandwidthStatus(metrics.first.get());
 				if (newBandwidthStatus == BandwidthStatusLow && bandwidthStatus != BandwidthStatusLow) {
@@ -1554,7 +1539,7 @@ void PhysicalShardCollection::updateTeamPhysicalShardIDsMap(uint64_t inputPhysic
                                                             uint64_t debugID) {
 	ASSERT(inputTeams.size() <= 2);
 	ASSERT(inputPhysicalShardID != anonymousShardId.first() && inputPhysicalShardID != UID().first());
-	for (auto inputTeam : inputTeams) {
+	for (const auto& inputTeam : inputTeams) {
 		if (!teamPhysicalShardIDs.contains(inputTeam)) {
 			std::set<uint64_t> physicalShardIDSet;
 			physicalShardIDSet.insert(inputPhysicalShardID);
@@ -1647,12 +1632,12 @@ Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(
     StorageMetrics const& moveInMetrics,
     const std::unordered_set<uint64_t>& excludedPhysicalShards,
     uint64_t debugID) {
-	ASSERT(team.servers.size() > 0);
+	ASSERT(!team.servers.empty());
 	// Case: The team is not tracked in the mapping (teamPhysicalShardIDs)
 	if (!teamPhysicalShardIDs.contains(team)) {
 		return Optional<uint64_t>();
 	}
-	ASSERT(teamPhysicalShardIDs[team].size() >= 1);
+	ASSERT(!teamPhysicalShardIDs[team].empty());
 	// Case: The team is tracked in the mapping and the system already has physical shard notion
 	// 		and the number of physicalShard is large
 	std::vector<uint64_t> availablePhysicalShardIDs;
@@ -1674,7 +1659,7 @@ Optional<uint64_t> PhysicalShardCollection::trySelectAvailablePhysicalShardFor(
 		}
 		availablePhysicalShardIDs.push_back(physicalShardID);
 	}
-	if (availablePhysicalShardIDs.size() == 0) {
+	if (availablePhysicalShardIDs.empty()) {
 		/*TraceEvent("TryGetPhysicalShardIDResultFailed")
 		    .detail("Reason", "no valid physicalShard")
 		    .detail("MoveInBytes", moveInMetrics.bytes)
@@ -1811,7 +1796,7 @@ std::pair<Optional<ShardsAffectedByTeamFailure::Team>, bool> PhysicalShardCollec
 	if (!checkPhysicalShardAvailable(inputPhysicalShardID, moveInMetrics)) {
 		return { Optional<ShardsAffectedByTeamFailure::Team>(), false };
 	}
-	for (auto team : physicalShardInstances[inputPhysicalShardID].teams) {
+	for (const auto& team : physicalShardInstances[inputPhysicalShardID].teams) {
 		if (team.primary == false) {
 			/*TraceEvent("TryGetRemoteTeamWith")
 			    .detail("PhysicalShardID", inputPhysicalShardID)
@@ -1975,7 +1960,7 @@ void PhysicalShardCollection::cleanUpPhysicalShardCollection() {
 	}
 	// Step 2: Clean up teamPhysicalShardIDs
 	std::set<ShardsAffectedByTeamFailure::Team> toRemoveTeams;
-	for (auto [team, _] : teamPhysicalShardIDs) {
+	for (const auto& [team, _] : teamPhysicalShardIDs) {
 		for (auto it = teamPhysicalShardIDs[team].begin(); it != teamPhysicalShardIDs[team].end();) {
 			uint64_t physicalShardID = *it;
 			if (!physicalShardInstances.contains(physicalShardID)) {
@@ -1986,12 +1971,12 @@ void PhysicalShardCollection::cleanUpPhysicalShardCollection() {
 				it++;
 			}
 		}
-		if (teamPhysicalShardIDs[team].size() == 0) {
+		if (teamPhysicalShardIDs[team].empty()) {
 			// If a team has no physicalShard, remove the team from teamPhysicalShardID
 			toRemoveTeams.insert(team);
 		}
 	}
-	for (auto team : toRemoveTeams) {
+	for (const auto& team : toRemoveTeams) {
 		teamPhysicalShardIDs.erase(team);
 	}
 }
@@ -2000,14 +1985,14 @@ void PhysicalShardCollection::logPhysicalShardCollection() {
 	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
 	// Step 1: Logging non-empty physicalShard
-	for (auto [physicalShardID, physicalShard] : physicalShardInstances) {
+	for (const auto& [physicalShardID, physicalShard] : physicalShardInstances) {
 		ASSERT(physicalShardID == physicalShard.id);
 		TraceEvent e("PhysicalShardStatus");
 		e.detail("PhysicalShardID", physicalShardID);
 		e.detail("TotalBytes", physicalShard.metrics.bytes);
 	}
 	// Step 2: Logging TeamPhysicalShardStatus
-	for (auto [team, physicalShardIDs] : teamPhysicalShardIDs) {
+	for (const auto& [team, physicalShardIDs] : teamPhysicalShardIDs) {
 		TraceEvent e("TeamPhysicalShardStatus");
 		e.detail("Team", team.toString());
 		// std::string metricsStr = "";
@@ -2044,7 +2029,7 @@ void PhysicalShardCollection::logPhysicalShardCollection() {
 	}
 	// Step 3: Logging StorageServerPhysicalShardStatus
 	std::map<UID, std::map<uint64_t, int64_t>> storageServerPhysicalShardStatus;
-	for (auto [team, _] : teamPhysicalShardIDs) {
+	for (const auto& [team, _] : teamPhysicalShardIDs) {
 		for (auto ssid : team.servers) {
 			for (auto it = teamPhysicalShardIDs[team].begin(); it != teamPhysicalShardIDs[team].end();) {
 				uint64_t physicalShardID = *it;
@@ -2064,7 +2049,7 @@ void PhysicalShardCollection::logPhysicalShardCollection() {
 			}
 		}
 	}
-	for (auto [serverID, physicalShardMetrics] : storageServerPhysicalShardStatus) {
+	for (const auto& [serverID, physicalShardMetrics] : storageServerPhysicalShardStatus) {
 		TraceEvent e("ServerPhysicalShardStatus");
 		e.detail("Server", serverID);
 		e.detail("NumPhysicalShards", physicalShardMetrics.size());
@@ -2103,21 +2088,28 @@ bool PhysicalShardCollection::physicalShardExists(uint64_t physicalShardID) {
 
 // FIXME: complete this test with non-empty range
 TEST_CASE("/DataDistributor/Tracker/FetchTopK") {
-	state DataDistributionTracker self;
-	state std::vector<KeyRange> ranges;
+	DataDistributionTracker self;
+	std::vector<KeyRange> ranges;
 	// for (int i = 1; i <= 10; i += 2) {
 	//     ranges.emplace_back(KeyRangeRef(doubleToTestKey(i), doubleToTestKey(i + 2)));
 	//     std::cout << "add range: " << ranges.back().begin.toString() << "\n";
 	// }
-	state GetTopKMetricsRequest req(ranges, 3, 1000, 100000);
+	GetTopKMetricsRequest req(ranges, 3, 1000, 100000);
 
 	// double targetDensities[10] = { 2, 1, 3, 5, 4, 10, 6, 8, 7, 0 };
 
-	wait(fetchTopKShardMetrics(&self, req));
+	co_await fetchTopKShardMetrics(&self, req);
 	auto& reply = req.reply.getFuture().get();
 	ASSERT(reply.shardMetrics.empty());
 	ASSERT(reply.maxReadLoad == -1);
 	ASSERT(reply.minReadLoad == -1);
+}
 
+TEST_CASE("/DataDistributor/Tracker/CrossesCriticalSystemBoundary") {
+	ASSERT(!crossesCriticalSystemBoundary("a"_sr, "z"_sr));
+	ASSERT(!crossesCriticalSystemBoundary(systemKeys.begin, systemKeys.end));
+	ASSERT(!crossesCriticalSystemBoundary("a"_sr, systemKeys.begin));
+	ASSERT(crossesCriticalSystemBoundary("a"_sr, "\xff/serverList/"_sr));
+	ASSERT(crossesCriticalSystemBoundary(""_sr, allKeys.end));
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbserver/datadistributor/DDSharedContext.cpp b/fdbserver/datadistributor/DDSharedContext.cpp
new file mode 100644
index 00000000000..8c5fa00af1b
--- /dev/null
+++ b/fdbserver/datadistributor/DDSharedContext.cpp
@@ -0,0 +1,34 @@
+/*
+ * DDSharedContext.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/datadistributor/DDSharedContext.h"
+
+#include "DDRelocationQueue.h"
+
+DDSharedContext::DDSharedContext() = default;
+
+DDSharedContext::DDSharedContext(const DataDistributorInterface& iface) : DDSharedContext(iface.id()) {
+	interface = iface;
+}
+
+DDSharedContext::DDSharedContext(UID id)
+  : ddEnabledState(new DDEnabledState), ddId(id), shardsAffectedByTeamFailure(new ShardsAffectedByTeamFailure) {}
+
+DDSharedContext::~DDSharedContext() = default;
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/datadistributor/DDTeamCollection.actor.cpp
similarity index 91%
rename from fdbserver/DDTeamCollection.actor.cpp
rename to fdbserver/datadistributor/DDTeamCollection.actor.cpp
index ba9c0ce7566..0156fbe457b 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/datadistributor/DDTeamCollection.actor.cpp
@@ -22,14 +22,17 @@
 
 #include "fdbclient/SystemData.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/DDTeamCollection.h"
-#include "fdbserver/ExclusionTracker.actor.h"
-#include "fdbserver/DataDistributionTeam.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/datadistributor/DDTeamCollection.h"
+#include "fdbserver/datadistributor/DataDistributionTeam.h"
+#include "ExclusionTracker.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "flow/network.h"
+#include "flow/TxnCounters.h"
 
+#include "flow/CoroUtils.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
@@ -73,20 +76,37 @@ int EligibilityCounter::getCount(int combinedType) const {
 
 } // namespace data_distribution
 
+static TxnCounters* updateNextWigglingStorageIDCounters() {
+	static auto* c = makeCounters("/dd/updateNextWigglingStorageID");
+	return c;
+}
+static TxnCounters* perpetualStorageWigglerCounters() {
+	static auto* c = makeCounters("/dd/perpetualStorageWiggler");
+	return c;
+}
+static TxnCounters* waitHealthyZoneChangeCounters() {
+	static auto* c = makeCounters("/dd/waitHealthyZoneChange");
+	return c;
+}
+static TxnCounters* updateStorageMetadataCounters() {
+	static auto* c = makeCounters("/dd/updateStorageMetadata");
+	return c;
+}
+
 class DDTeamCollectionImpl {
-	ACTOR static Future<Void> checkAndRemoveInvalidLocalityAddr(DDTeamCollection* self) {
-		state double start = now();
-		state bool hasCorrectedLocality = false;
+	static Future<Void> checkAndRemoveInvalidLocalityAddr(DDTeamCollection* self) {
+		double start = now();
+		bool hasCorrectedLocality = false;
 
-		loop {
+		while (true) {
 			try {
-				wait(delay(SERVER_KNOBS->DD_CHECK_INVALID_LOCALITY_DELAY, TaskPriority::DataDistribution));
+				co_await delay(SERVER_KNOBS->DD_CHECK_INVALID_LOCALITY_DELAY, TaskPriority::DataDistribution);
 
 				// Because worker's processId can be changed when its locality is changed, we cannot watch on the old
 				// processId; This actor is inactive most time, so iterating all workers incurs little performance
 				// overhead.
-				state std::vector<ProcessData> workers = wait(self->db->getWorkers());
-				state std::set<AddressExclusion> existingAddrs;
+				std::vector<ProcessData> workers = co_await self->db->getWorkers();
+				std::set<AddressExclusion> existingAddrs;
 				for (int i = 0; i < workers.size(); i++) {
 					const ProcessData& workerData = workers[i];
 					AddressExclusion addr(workerData.address.ip, workerData.address.port);
@@ -100,7 +120,7 @@ class DDTeamCollectionImpl {
 					}
 				}
 
-				wait(yield(TaskPriority::DataDistribution));
+				co_await yield(TaskPriority::DataDistribution);
 
 				// In case system operator permanently excludes workers on the address with invalid locality
 				for (auto addr = self->invalidLocalityAddr.begin(); addr != self->invalidLocalityAddr.end();) {
@@ -134,52 +154,44 @@ class DDTeamCollectionImpl {
 				TraceEvent("CheckAndRemoveInvalidLocalityAddrRetry", self->distributorId).detail("Error", e.what());
 			}
 		}
-
-		return Void();
 	}
 
 public:
-	ACTOR static Future<Void> logOnCompletion(DDTeamCollection* self, Future<Void> signal) {
-		wait(signal);
-		wait(delay(SERVER_KNOBS->LOG_ON_COMPLETION_DELAY, TaskPriority::DataDistribution));
+	static Future<Void> logOnCompletion(DDTeamCollection* self, Future<Void> signal) {
+		co_await signal;
+		co_await delay(SERVER_KNOBS->LOG_ON_COMPLETION_DELAY, TaskPriority::DataDistribution);
 
 		if (!self->primary || self->configuration.usableRegions == 1) {
 			TraceEvent("DDTrackerStarting", self->distributorId)
 			    .detail("State", "Active")
 			    .trackLatest(self->ddTrackerStartingEventHolder->trackingKey);
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> interruptableBuildTeams(DDTeamCollection* self) {
+	static Future<Void> interruptableBuildTeams(DDTeamCollection* self) {
 		if (!self->addSubsetComplete.isSet()) {
-			wait(addSubsetOfEmergencyTeams(self));
+			co_await addSubsetOfEmergencyTeams(self);
 			self->addSubsetComplete.send(Void());
 		}
 
-		loop {
-			choose {
-				when(wait(self->buildTeams())) {
-					return Void();
-				}
-				when(wait(self->restartTeamBuilder.onTrigger())) {}
+		while (true) {
+			auto res = co_await race(self->buildTeams(), self->restartTeamBuilder.onTrigger());
+			if (res.index() == 0) {
+				co_return;
 			}
 		}
 	}
 
-	ACTOR static Future<Void> checkBuildTeams(DDTeamCollection* self) {
-		wait(self->checkTeamDelay);
+	static Future<Void> checkBuildTeams(DDTeamCollection* self) {
+		co_await self->checkTeamDelay;
 		while (!self->teamBuilder.isReady())
-			wait(self->teamBuilder);
+			co_await self->teamBuilder;
 
 		if (self->doBuildTeams && self->readyToStart.isReady()) {
 			self->doBuildTeams = false;
 			self->teamBuilder = self->interruptableBuildTeams();
-			wait(self->teamBuilder);
+			co_await self->teamBuilder;
 		}
-
-		return Void();
 	}
 
 	// Find the team with the exact storage servers as req.src.
@@ -196,7 +208,7 @@ class DDTeamCollectionImpl {
 	}
 
 	// Random selection for load balance
-	ACTOR static Future<Void> getTeamForBulkLoad(DDTeamCollection* self, GetTeamRequest req) {
+	static Future<Void> getTeamForBulkLoad(DDTeamCollection* self, GetTeamRequest req) {
 		try {
 			TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadEngineTaskGetTeamReqReceived", self->distributorId)
 			    .detail("TCReady", self->readyToStart.isReady())
@@ -205,7 +217,7 @@ class DDTeamCollectionImpl {
 			    .detail("SrcIds", describe(req.src))
 			    .detail("Primary", self->isPrimary())
 			    .detail("TeamSize", self->teams.size());
-			wait(self->checkBuildTeams());
+			co_await self->checkBuildTeams();
 
 			TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadEngineTaskGetTeamCheckBuildTeamDone", self->distributorId)
 			    .detail("TCReady", self->readyToStart.isReady())
@@ -225,7 +237,7 @@ class DDTeamCollectionImpl {
 				    .detail("Primary", self->primary)
 				    .detail("Team", team.present() ? describe(team.get()->getServerIDs()) : "");
 				req.reply.send(std::make_pair(team, true));
-				return Void();
+				co_return;
 			}
 
 			self->updateTeamPivotValues();
@@ -322,7 +334,6 @@ class DDTeamCollectionImpl {
 				    .detail("NotEligibileTeamCount", notEligibileTeamCount);
 			}
 			req.reply.send(std::make_pair(res, false));
-			return Void();
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled && req.reply.canBeSet())
 				req.reply.sendError(e);
@@ -462,9 +473,9 @@ class DDTeamCollectionImpl {
 
 	// SOMEDAY: Make bestTeam better about deciding to leave a shard where it is (e.g. in PRIORITY_TEAM_HEALTHY case)
 	//		    use keys, src, dest, metrics, priority, system load, etc.. to decide...
-	ACTOR static Future<Void> getTeam(DDTeamCollection* self, GetTeamRequest req) {
+	static Future<Void> getTeam(DDTeamCollection* self, GetTeamRequest req) {
 		try {
-			wait(self->checkBuildTeams());
+			co_await self->checkBuildTeams();
 
 			if (!self->primary && !self->readyToStart.isReady()) {
 				// When remote DC is not ready, DD shouldn't reply with a new team because
@@ -476,7 +487,7 @@ class DDTeamCollectionImpl {
 				    .detail("Primary", self->primary)
 				    .detail("Team", team.present() ? describe(team.get()->getServerIDs()) : "");
 				req.reply.send(std::make_pair(team, true));
-				return Void();
+				co_return;
 			}
 
 			// report the pivot values
@@ -501,12 +512,12 @@ class DDTeamCollectionImpl {
 			// The situation happens rarely. We may want to eliminate this situation someday
 			if (!self->teams.size()) {
 				req.reply.send(std::make_pair(Optional<Reference<IDataDistributionTeam>>(), foundSrc));
-				return Void();
+				co_return;
 			}
 
 			Optional<Reference<IDataDistributionTeam>> bestOption;
-			state int numSkippedSSFailedGetQueueLength = 0;
-			state int numSkippedSSQueueTooLong = 0;
+			int numSkippedSSFailedGetQueueLength = 0;
+			int numSkippedSSQueueTooLong = 0;
 
 			if (ddLargeTeamEnabled() && req.keys.present()) {
 				int customReplicas = self->configuration.storageTeamSize;
@@ -523,7 +534,7 @@ class DDTeamCollectionImpl {
 							}
 							if (now() - firstFailureTime.get() < SERVER_KNOBS->DD_LARGE_TEAM_DELAY) {
 								req.reply.send(std::make_pair(Optional<Reference<IDataDistributionTeam>>(), foundSrc));
-								return Void();
+								co_return;
 							}
 							self->underReplication.insert(req.keys.get(), true);
 						} else {
@@ -535,14 +546,14 @@ class DDTeamCollectionImpl {
 						    .detail("DesiredReplicas", customReplicas)
 						    .detail("UnderReplicated", newTeam->size() < customReplicas);
 						req.reply.send(std::make_pair(newTeam, foundSrc));
-						return Void();
+						co_return;
 					} else {
 						if (!firstFailureTime.present()) {
 							firstFailureTime = now();
 						}
 						if (now() - firstFailureTime.get() < SERVER_KNOBS->DD_LARGE_TEAM_DELAY) {
 							req.reply.send(std::make_pair(Optional<Reference<IDataDistributionTeam>>(), foundSrc));
-							return Void();
+							co_return;
 						}
 						TraceEvent(SevWarnAlways, "LargeTeamNotFound", self->distributorId)
 						    .suppressFor(1.0)
@@ -559,7 +570,7 @@ class DDTeamCollectionImpl {
 				auto healthyTeam = self->findTeamFromServers(req.completeSources, /* wantHealthy=*/true);
 				if (healthyTeam.present()) {
 					req.reply.send(std::make_pair(healthyTeam, foundSrc));
-					return Void();
+					co_return;
 				}
 			}
 
@@ -674,7 +685,7 @@ class DDTeamCollectionImpl {
 				auto healthyTeam = self->findTeamFromServers(req.completeSources, /* wantHealthy=*/false);
 				if (healthyTeam.present()) {
 					req.reply.send(std::make_pair(healthyTeam, foundSrc));
-					return Void();
+					co_return;
 				}
 			}
 			if (g_network->isSimulated() && !bestOption.present()) {
@@ -696,12 +707,10 @@ class DDTeamCollectionImpl {
 				TraceEvent(SevWarn, "GetTeamRetry", self->distributorId)
 				    .detail("OldStorageQueueAware", req.storageQueueAware)
 				    .detail("OldWantTrueBestIfMoveout", req.wantTrueBestIfMoveout);
-				wait(getTeam(self, req));
+				co_await getTeam(self, req);
 			} else {
 				req.reply.send(std::make_pair(bestOption, foundSrc));
 			}
-
-			return Void();
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled && req.reply.canBeSet())
 				req.reply.sendError(e);
@@ -710,14 +719,15 @@ class DDTeamCollectionImpl {
 	}
 
 	// FIXME: describe purpose
-	ACTOR static Future<Void> addSubsetOfEmergencyTeams(DDTeamCollection* self) {
-		state int idx = 0;
-		state std::vector<Reference<TCServerInfo>> servers;
-		state std::vector<UID> serverIds;
-		state Reference<LocalitySet> tempSet = Reference<LocalitySet>(new LocalityMap<UID>());
-		state LocalityMap<UID>* tempMap = (LocalityMap<UID>*)tempSet.getPtr();
-		state std::vector<Reference<TCTeamInfo>> largeOrBadTeams = self->badTeams;
+	static Future<Void> addSubsetOfEmergencyTeams(DDTeamCollection* self) {
+		int idx = 0;
+		std::vector<Reference<TCServerInfo>> servers;
+		std::vector<UID> serverIds;
+		Reference<LocalitySet> tempSet = makeReference<LocalityMap<UID>>();
+		LocalityMap<UID>* tempMap = nullptr;
+		std::vector<Reference<TCTeamInfo>> largeOrBadTeams = self->badTeams;
 		largeOrBadTeams.insert(largeOrBadTeams.end(), self->largeTeams.begin(), self->largeTeams.end());
+		tempMap = static_cast<LocalityMap<UID>*>(tempSet.getPtr());
 
 		for (; idx < largeOrBadTeams.size(); idx++) {
 			servers.clear();
@@ -791,18 +801,16 @@ class DDTeamCollectionImpl {
 					}
 				}
 			}
-			wait(yield());
+			co_await yield();
 		}
 
 		// Trace and record the current number of teams for correctness test
 		self->traceTeamCollectionInfo();
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> init(DDTeamCollection* self,
-	                               Reference<InitialDataDistribution> initTeams,
-	                               const DDEnabledState* ddEnabledState) {
+	static Future<Void> init(DDTeamCollection* self,
+	                         Reference<InitialDataDistribution> initTeams,
+	                         const DDEnabledState* ddEnabledState) {
 
 		self->userRangeConfig = initTeams->userRangeConfig;
 		self->healthyZone.set(initTeams->initHealthyZoneValue);
@@ -825,33 +833,32 @@ class DDTeamCollectionImpl {
 			}
 		}
 
-		state std::set<std::vector<UID>>::iterator teamIter =
+		std::set<std::vector<UID>>::iterator teamIter =
 		    self->primary ? initTeams->primaryTeams.begin() : initTeams->remoteTeams.begin();
-		state std::set<std::vector<UID>>::iterator teamIterEnd =
+		std::set<std::vector<UID>>::iterator teamIterEnd =
 		    self->primary ? initTeams->primaryTeams.end() : initTeams->remoteTeams.end();
 		for (; teamIter != teamIterEnd; ++teamIter) {
 			self->addTeam(teamIter->begin(), teamIter->end(), IsInitialTeam::True);
-			wait(yield());
+			co_await yield();
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> buildTeams(DDTeamCollection* self) {
-		state int desiredTeams;
-		state int serverCount = 0;
-		state std::set<Optional<Standalone<StringRef>>> machines;
+	static Future<Void> buildTeams(DDTeamCollection* self) {
+		int desiredTeams{ 0 };
+		int serverCount = 0;
+		std::set<Optional<Standalone<StringRef>>> machines;
 
 		// wait to see whether restartTeamBuilder is triggered
-		wait(delay(0, g_network->getCurrentTask()));
+		co_await delay(0, g_network->getCurrentTask());
 		// make team builder don't build team during the interval between excluding the wiggled process and recruited a
 		// new SS to avoid redundant teams
 		while (self->pauseWiggle && !self->pauseWiggle->get() && self->waitUntilRecruited.get()) {
-			choose {
-				when(wait(self->waitUntilRecruited.onChange() || self->pauseWiggle->onChange())) {}
-				when(wait(delay(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY, g_network->getCurrentTask()))) {
-					break;
-				}
+			auto const recruitedOrPauseChanged =
+			    co_await timeout(self->waitUntilRecruited.onChange() || self->pauseWiggle->onChange(),
+			                     SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY,
+			                     g_network->getCurrentTask());
+			if (!recruitedOrPauseChanged.present()) {
+				break;
 			}
 		}
 
@@ -892,8 +899,7 @@ class DDTeamCollectionImpl {
 
 			// teamsToBuild is calculated such that we will not build too many teams in the situation
 			// when all (or most of) teams become unhealthy temporarily and then healthy again
-			state int teamsToBuild;
-			teamsToBuild = std::max(0, std::min(desiredTeams - teamCount, maxTeams - totalTeamCount));
+			int teamsToBuild = std::max(0, std::min(desiredTeams - teamCount, maxTeams - totalTeamCount));
 
 			if (teamCount == 0 && teamsToBuild == 0 && SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE > 0) {
 				// Use DD_BUILD_EXTRA_TEAMS_OVERRIDE > 0 as the feature flag: Set to 0 to disable it
@@ -923,8 +929,6 @@ class DDTeamCollectionImpl {
 
 			self->lastBuildTeamsFailed = false;
 			if (teamsToBuild > 0 || self->notEnoughTeamsForAServer()) {
-				state std::vector<std::vector<UID>> builtTeams;
-
 				// addTeamsBestOf() will not add more teams than needed.
 				// If the team number is more than the desired, the extra teams are added in the code path when
 				// a team is added as an initial team
@@ -985,31 +989,29 @@ class DDTeamCollectionImpl {
 
 		// Building teams can cause servers to become undesired, which can make teams unhealthy.
 		// Let all of these changes get worked out before responding to the get team request
-		wait(delay(0, TaskPriority::DataDistributionLaunch));
-
-		return Void();
+		co_await delay(0, TaskPriority::DataDistributionLaunch);
 	}
 
 	// Track a team and issue RelocateShards when the level of degradation changes
 	// A badTeam can be unhealthy or just a redundantTeam removed by machineTeamRemover() or serverTeamRemover()
-	ACTOR static Future<Void> teamTracker(DDTeamCollection* self,
-	                                      Reference<TCTeamInfo> team,
-	                                      IsBadTeam badTeam,
-	                                      IsRedundantTeam redundantTeam) {
-		state int lastServersLeft = team->size();
-		state bool lastAnyUndesired = false;
-		state bool lastAnyWigglingServer = false;
-		state bool logTeamEvents =
+	static Future<Void> teamTracker(DDTeamCollection* self,
+	                                Reference<TCTeamInfo> team,
+	                                IsBadTeam badTeam,
+	                                IsRedundantTeam redundantTeam) {
+		int lastServersLeft = team->size();
+		bool lastAnyUndesired = false;
+		bool lastAnyWigglingServer = false;
+		bool logTeamEvents =
 		    g_network->isSimulated() || !badTeam || team->size() <= self->configuration.storageTeamSize;
-		state bool lastReady = false;
-		state bool lastHealthy;
-		state bool lastOptimal;
-		state bool lastWrongConfiguration = team->isWrongConfiguration();
-		state bool trackHealthyTeam = team->size() == self->configuration.storageTeamSize;
-		state bool lastZeroHealthy = self->zeroHealthyTeams->get();
-		state bool firstCheck = true;
+		bool lastReady = false;
+		bool lastHealthy{ false };
+		bool lastOptimal{ false };
+		bool lastWrongConfiguration = team->isWrongConfiguration();
+		bool trackHealthyTeam = team->size() == self->configuration.storageTeamSize;
+		bool lastZeroHealthy = self->zeroHealthyTeams->get();
+		bool firstCheck = true;
 
-		state Future<Void> zeroServerLeftLogger;
+		Future<Void> zeroServerLeftLogger;
 
 		if (logTeamEvents) {
 			TraceEvent("ServerTeamTrackerStarting", self->distributorId)
@@ -1019,7 +1021,7 @@ class DDTeamCollectionImpl {
 		self->priority_teams[team->getPriority()]++;
 
 		try {
-			loop {
+			while (true) {
 				if (logTeamEvents) {
 					TraceEvent("ServerTeamHealthChangeDetected", self->distributorId)
 					    .detail("ServerTeam", team->getDesc())
@@ -1029,7 +1031,7 @@ class DDTeamCollectionImpl {
 				}
 
 				// Check if the number of degraded machines has changed
-				state std::vector<Future<Void>> change;
+				std::vector<Future<Void>> change;
 				bool anyUndesired = false;
 				bool anyWrongConfiguration = false;
 				bool anyWigglingServer = false;
@@ -1115,6 +1117,7 @@ class DDTeamCollectionImpl {
 				    recheck) { // NOTE: do not check wrongSize
 					if (logTeamEvents) {
 						TraceEvent("ServerTeamHealthChanged", self->distributorId)
+						    .suppressFor(1.0)
 						    .detail("ServerTeam", team->getDesc())
 						    .detail("ServersLeft", serversLeft)
 						    .detail("LastServersLeft", lastServersLeft)
@@ -1145,6 +1148,7 @@ class DDTeamCollectionImpl {
 
 							if (self->healthyTeamCount == 0) {
 								TraceEvent(SevWarn, "ZeroServerTeamsHealthySignalling", self->distributorId)
+								    .suppressFor(1.0)
 								    .detail("SignallingTeam", team->getDesc())
 								    .detail("Primary", self->primary);
 							}
@@ -1165,7 +1169,7 @@ class DDTeamCollectionImpl {
 					lastWrongConfiguration = anyWrongConfiguration;
 					lastAnyWigglingServer = anyWigglingServer;
 
-					state int lastPriority = team->getPriority();
+					int lastPriority = team->getPriority();
 					if (team->size() == 0) {
 						team->setPriority(SERVER_KNOBS->PRIORITY_POPULATE_REGION);
 					} else if (serversLeft < team->size()) {
@@ -1319,8 +1323,8 @@ class DDTeamCollectionImpl {
 				}
 
 				// Wait for any of the machines to change status
-				wait(quorum(change, 1));
-				wait(yield());
+				co_await quorum(change, 1);
+				co_await yield();
 			}
 		} catch (Error& e) {
 			if (logTeamEvents) {
@@ -1338,6 +1342,7 @@ class DDTeamCollectionImpl {
 
 					if (self->healthyTeamCount == 0) {
 						TraceEvent(SevWarn, "ZeroTeamsHealthySignalling", self->distributorId)
+						    .suppressFor(1.0)
 						    .detail("ServerPrimary", self->primary)
 						    .detail("SignallingServerTeam", team->getDesc());
 						self->zeroHealthyTeams->set(true);
@@ -1374,7 +1379,17 @@ class DDTeamCollectionImpl {
 		state Future<Void> storageMetadataTracker = self->updateStorageMetadata(server);
 		try {
 			loop {
-				status.isUndesired = (!self->disableFailingLaggingServers.get() && server->ssVersionTooFarBehind.get());
+				{
+					bool versionLagUndesired =
+					    !self->disableFailingLaggingServers.get() && server->ssVersionTooFarBehind.get();
+					if (versionLagUndesired && !status.isUndesired) {
+						TraceEvent(SevWarn, "UndesiredStorageServer", self->distributorId)
+						    .detail("Server", server->getId())
+						    .detail("Address", server->getLastKnownInterface().address())
+						    .detail("Reason", "VersionLag");
+					}
+					status.isUndesired = versionLagUndesired;
+				}
 				status.isWrongConfiguration = false;
 				status.isWiggling = false;
 				hasWrongDC = !self->isCorrectDC(*server);
@@ -1410,6 +1425,7 @@ class DDTeamCollectionImpl {
 								TraceEvent(SevWarn, "UndesiredStorageServer", self->distributorId)
 								    .detail("Server", server->getId())
 								    .detail("Address", server->getLastKnownInterface().address())
+								    .detail("Reason", "SameAddress")
 								    .detail("OtherServer", i.second->getId())
 								    .detail("NumShards",
 								            self->shardsAffectedByTeamFailure->getNumberOfShards(server->getId()))
@@ -1435,6 +1451,8 @@ class DDTeamCollectionImpl {
 					if (self->optimalTeamCount > 0) {
 						TraceEvent(SevWarn, "UndesiredStorageServer", self->distributorId)
 						    .detail("Server", server->getId())
+						    .detail("Address", server->getLastKnownInterface().address())
+						    .detail("Reason", "WrongMachineClass")
 						    .detail("OptimalTeamCount", self->optimalTeamCount)
 						    .detail("Fitness", server->getLastKnownClass().machineClassFitness(ProcessClass::Storage));
 						status.isUndesired = true;
@@ -1480,7 +1498,7 @@ class DDTeamCollectionImpl {
 				}
 				otherChanges.push_back(self->excludedServers.onChange(worstAddr));
 
-				// FIXME: explain the 3 here
+				// Check the remaining exclusion forms: primary IP, secondary address and port, and secondary IP.
 				for (int i = 0; i < 3; i++) {
 					if (i > 0 && !server->getLastKnownInterface().secondaryAddress().present()) {
 						break;
@@ -1512,9 +1530,16 @@ class DDTeamCollectionImpl {
 				}
 
 				if (worstStatus != DDTeamCollection::Status::NONE) {
+					const char* exclusionType = worstStatus == DDTeamCollection::Status::WIGGLING   ? "Wiggling"
+					                            : worstStatus == DDTeamCollection::Status::FAILED   ? "Failed"
+					                            : worstStatus == DDTeamCollection::Status::EXCLUDED ? "Excluded"
+					                                                                                : "Unknown";
 					TraceEvent(SevWarn, "UndesiredStorageServer", self->distributorId)
 					    .detail("Server", server->getId())
-					    .detail("Excluded", worstAddr.toString());
+					    .detail("Address", server->getLastKnownInterface().address())
+					    .detail("Reason", "Excluded")
+					    .detail("ExclusionType", exclusionType)
+					    .detail("ExcludedAddress", worstAddr.toString());
 					status.isUndesired = true;
 					status.isWrongConfiguration = true;
 
@@ -1589,33 +1614,33 @@ class DDTeamCollectionImpl {
 						bool restartRecruiting = newInterface.first.waitFailure.getEndpoint().getPrimaryAddress() !=
 						                         lastKnownInterface.waitFailure.getEndpoint().getPrimaryAddress();
 						bool localityChanged = lastKnownInterface.locality != newInterface.first.locality;
-						bool machineLocalityChanged =
+						bool zoneLocalityChanged =
 						    lastKnownInterface.locality.zoneId().get() != newInterface.first.locality.zoneId().get();
 						TraceEvent("StorageServerInterfaceChanged", self->distributorId)
 						    .detail("ServerID", server->getId())
 						    .detail("NewWaitFailureToken", newInterface.first.waitFailure.getEndpoint().token)
 						    .detail("OldWaitFailureToken", lastKnownInterface.waitFailure.getEndpoint().token)
 						    .detail("LocalityChanged", localityChanged)
-						    .detail("MachineLocalityChanged", machineLocalityChanged);
+						    .detail("ZoneLocalityChanged", zoneLocalityChanged);
 
 						server->updateLastKnown(newInterface.first, newInterface.second);
 						if (localityChanged && !isTss) {
 							CODE_PROBE(true, "Server locality changed");
 
-							// The locality change of a server will affect machine teams related to the server if
-							// the server's machine locality is changed
-							if (machineLocalityChanged) {
-								// First handle the impact on the machine of the server on the old locality
+							// Machine teams are keyed by zone ID, so zone locality changes require updating
+							// machine-team membership for this server.
+							if (zoneLocalityChanged) {
+								// First handle the impact on the server's old zone-keyed machine.
 								Reference<TCMachineInfo> machine = server->machine;
 								ASSERT_GE(machine->serversOnMachine.size(), 1);
 								if (machine->serversOnMachine.size() == 1) {
-									// When server is the last server on the machine,
-									// remove the machine and the related machine team
+									// When this is the last server on the zone-keyed machine,
+									// remove the machine and related machine teams.
 									self->removeMachine(machine);
 									server->machine = Reference<TCMachineInfo>();
 								} else {
-									// we remove the server from the machine, and
-									// update locality entry for the machine and the global machineLocalityMap
+									// Remove the server from the zone-keyed machine. The representative locality
+									// entry is left stale until machineLocalityMap is rebuilt.
 									int serverIndex = -1;
 									for (int i = 0; i < machine->serversOnMachine.size(); ++i) {
 										if (machine->serversOnMachine[i].getPtr() == server) {
@@ -1632,9 +1657,8 @@ class DDTeamCollectionImpl {
 									// its representative server is changed.
 								}
 
-								// Second handle the impact on the destination machine where the server's new locality
-								// is; If the destination machine is new, create one; otherwise, add server to an
-								// existing one Update server's machine reference to the destination machine
+								// Next handle the destination zone-keyed machine. Create it if necessary;
+								// otherwise add the server and update its machine reference.
 								Reference<TCMachineInfo> destMachine =
 								    self->checkAndCreateMachine(self->server_info[server->getId()]);
 								ASSERT(destMachine.isValid());
@@ -1648,7 +1672,7 @@ class DDTeamCollectionImpl {
 									newBadTeams.push_back(serverTeam);
 									continue;
 								}
-								if (machineLocalityChanged) {
+								if (zoneLocalityChanged) {
 									Reference<TCMachineTeamInfo> machineTeam =
 									    self->checkAndCreateMachineTeam(serverTeam);
 									ASSERT(machineTeam.isValid());
@@ -1685,9 +1709,9 @@ class DDTeamCollectionImpl {
 						interfaceChanged = server->onInterfaceChanged;
 						// Old failureTracker for the old interface will be actorCancelled since the handler of the old
 						// actor now points to the new failure monitor actor.
-						status = ServerStatus(status.isFailed,
-						                      status.isUndesired,
-						                      status.isWiggling,
+						status = ServerStatus(IsFailed(status.isFailed),
+						                      IsUndesired(status.isUndesired),
+						                      IsWiggling(status.isWiggling),
 						                      server->getLastKnownInterface().locality);
 
 						// self->traceTeamCollectionInfo();
@@ -1762,18 +1786,18 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> removeWrongStoreType(DDTeamCollection* self) {
+	static Future<Void> removeWrongStoreType(DDTeamCollection* self) {
 		// Wait for storage servers to initialize its storeType
-		wait(delay(SERVER_KNOBS->DD_REMOVE_STORE_ENGINE_DELAY));
+		co_await delay(SERVER_KNOBS->DD_REMOVE_STORE_ENGINE_DELAY);
 
-		state Future<Void> fisServerRemoved = Never();
+		Future<Void> fisServerRemoved = Never();
 
 		TraceEvent("WrongStoreTypeRemoverStart", self->distributorId).detail("Servers", self->server_info.size());
-		loop {
+		while (true) {
 			// Removing a server here when DD is not healthy may lead to rare failure scenarios, for example,
 			// the server with wrong storeType is shutting down while this actor marks it as to-be-removed.
 			// In addition, removing servers cause extra data movement, which should be done while a cluster is healthy
-			wait(self->waitUntilHealthy());
+			co_await self->waitUntilHealthy();
 
 			bool foundSSToRemove = false;
 
@@ -1808,15 +1832,13 @@ class DDTeamCollectionImpl {
 				break;
 			}
 		}
-
-		return Void();
 	}
 
 	// NOTE: this actor returns when the cluster is healthy and stable (no server is expected to be removed in a period)
 	// processingWiggle and processingUnhealthy indicate that some servers are going to be removed.
-	ACTOR static Future<Void> waitUntilHealthy(DDTeamCollection const* self, double extraDelay, WaitWiggle waitWiggle) {
-		state int waitCount = 0;
-		loop {
+	static Future<Void> waitUntilHealthy(DDTeamCollection const* self, double extraDelay, WaitWiggle waitWiggle) {
+		int waitCount = 0;
+		while (true) {
 			while (self->zeroHealthyTeams->get() || self->processingUnhealthy->get() ||
 			       (waitWiggle && self->processingWiggle->get())) {
 				// processingUnhealthy: true when there exists data movement
@@ -1826,45 +1848,45 @@ class DDTeamCollectionImpl {
 				    .detail("ZeroHealthy", self->zeroHealthyTeams->get())
 				    .detail("ProcessingUnhealthy", self->processingUnhealthy->get())
 				    .detail("ProcessingPerpetualWiggle", self->processingWiggle->get());
-				wait(self->zeroHealthyTeams->onChange() || self->processingUnhealthy->onChange() ||
-				     self->processingWiggle->onChange());
+				co_await race(self->zeroHealthyTeams->onChange(),
+				              self->processingUnhealthy->onChange(),
+				              self->processingWiggle->onChange());
 				waitCount = 0;
 			}
-			wait(delay(SERVER_KNOBS->DD_STALL_CHECK_DELAY,
-			           TaskPriority::Low)); // After the team trackers wait on the initial failure reaction delay, they
-			                                // yield. We want to make sure every tracker has had the opportunity to send
-			                                // their relocations to the queue.
+			co_await delay(SERVER_KNOBS->DD_STALL_CHECK_DELAY,
+			               TaskPriority::Low); // After the team trackers wait on the initial failure reaction delay,
+			                                   // they yield. We want to make sure every tracker has had the opportunity
+			                                   // to send their relocations to the queue.
 			if (!self->zeroHealthyTeams->get() && !self->processingUnhealthy->get() &&
 			    (!waitWiggle || !self->processingWiggle->get())) {
 				if (extraDelay <= 0.01 || waitCount >= 1) {
 					// Return healthy if we do not need extraDelay or when DD are healthy in at least two consecutive
 					// check
-					return Void();
+					co_return;
 				} else {
-					wait(delay(extraDelay, TaskPriority::Low));
+					co_await delay(extraDelay, TaskPriority::Low);
 					waitCount++;
 				}
 			}
 		}
 	}
 
-	ACTOR static Future<Void> removeBadTeams(DDTeamCollection* self) {
-		wait(self->initialFailureReactionDelay);
-		wait(self->waitUntilHealthy());
-		wait(self->addSubsetComplete.getFuture());
+	static Future<Void> removeBadTeams(DDTeamCollection* self) {
+		co_await self->initialFailureReactionDelay;
+		co_await self->waitUntilHealthy();
+		co_await self->addSubsetComplete.getFuture();
 		TraceEvent("DDRemovingBadServerTeams", self->distributorId).detail("Primary", self->primary);
 		for (auto it : self->badTeams) {
 			it->tracker.cancel();
 		}
 		self->badTeams.clear();
-		return Void();
 	}
 
-	ACTOR static Future<Void> zeroServerLeftLoggerActor(DDTeamCollection* self, Reference<TCTeamInfo> team) {
-		wait(delay(SERVER_KNOBS->DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY));
-		state std::vector<KeyRange> shards = self->shardsAffectedByTeamFailure->getShardsFor(
+	static Future<Void> zeroServerLeftLoggerActor(DDTeamCollection* self, Reference<TCTeamInfo> team) {
+		co_await delay(SERVER_KNOBS->DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY);
+		std::vector<KeyRange> shards = self->shardsAffectedByTeamFailure->getShardsFor(
 		    ShardsAffectedByTeamFailure::Team(team->getServerIDs(), self->primary));
-		state std::vector<Future<StorageMetrics>> sizes;
+		std::vector<Future<StorageMetrics>> sizes;
 		sizes.reserve(shards.size());
 
 		for (auto const& shard : shards) {
@@ -1875,7 +1897,7 @@ class DDTeamCollectionImpl {
 			    .detail("ShardEnd", shard.end);
 		}
 
-		wait(waitForAll(sizes));
+		co_await waitForAll(sizes);
 
 		int64_t bytesLost = 0;
 		for (auto const& size : sizes) {
@@ -1885,8 +1907,6 @@ class DDTeamCollectionImpl {
 		TraceEvent(SevWarnAlways, "DDZeroServerLeftInTeam", self->distributorId)
 		    .detail("Team", team->getDesc())
 		    .detail("TotalBytesLost", bytesLost);
-
-		return Void();
 	}
 
 	ACTOR static Future<Void> storageServerFailureTracker(DDTeamCollection* self,
@@ -1995,22 +2015,22 @@ class DDTeamCollectionImpl {
 		return Void(); // Don't ignore failures
 	}
 
-	ACTOR static Future<Void> machineTeamRemover(DDTeamCollection* self) {
-		state int numMachineTeamRemoved = 0;
-		loop {
+	static Future<Void> machineTeamRemover(DDTeamCollection* self) {
+		int numMachineTeamRemoved = 0;
+		while (true) {
 			// In case the machineTeamRemover cause problems in production, we can disable it
 			if (SERVER_KNOBS->TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER) {
-				return Void();
+				co_return;
 			}
 
 			// To avoid removing machine teams too fast, which is unlikely happen though
-			wait(delay(SERVER_KNOBS->TR_REMOVE_MACHINE_TEAM_DELAY, TaskPriority::DataDistribution));
+			co_await delay(SERVER_KNOBS->TR_REMOVE_MACHINE_TEAM_DELAY, TaskPriority::DataDistribution);
 
-			wait(self->waitUntilHealthy(SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY));
+			co_await self->waitUntilHealthy(SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY);
 
 			// Wait for the badTeamRemover() to avoid the potential race between adding the bad team (add the team
 			// tracker) and remove bad team (cancel the team tracker).
-			wait(self->badTeamRemover);
+			co_await self->badTeamRemover;
 
 			if (SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC > 0 &&
 			    self->teamPivots.pivotAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) {
@@ -2018,11 +2038,11 @@ class DDTeamCollectionImpl {
 				    .detail("IsPrimary", self->primary)
 				    .detail("CurrentSpacePivot", self->teamPivots.pivotAvailableSpaceRatio)
 				    .detail("TargetSpacePivot", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO);
-				wait(delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC));
+				co_await delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC);
 				continue;
 			}
 
-			state int healthyMachineCount = self->calculateHealthyMachineCount();
+			int healthyMachineCount = self->calculateHealthyMachineCount();
 			// Check if all machines are healthy, if not, we wait for 1 second and loop back.
 			// Eventually, all machines will become healthy.
 			if (healthyMachineCount != self->machine_info.size()) {
@@ -2080,7 +2100,7 @@ class DDTeamCollectionImpl {
 
 				TraceEvent("MachineTeamRemover", self->distributorId)
 				    .detail("MachineTeamIDToRemove", mt->id().shortString())
-				    .detail("MachineTeamToRemove", mt->getMachineIDsStr())
+				    .detail("ZoneIDsToRemove", mt->getMachineIDsStr())
 				    .detail("NumProcessTeamsOnTheMachineTeam", minNumProcessTeams)
 				    .detail("CurrentMachineTeams", self->machineTeams.size())
 				    .detail("DesiredMachineTeams", desiredMachineTeams);
@@ -2108,12 +2128,12 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> serverTeamRemover(DDTeamCollection* self) {
-		state int numServerTeamRemoved = 0;
-		loop {
+	static Future<Void> serverTeamRemover(DDTeamCollection* self) {
+		int numServerTeamRemoved = 0;
+		while (true) {
 			// In case the serverTeamRemover cause problems in production, we can disable it
 			if (SERVER_KNOBS->TR_FLAG_DISABLE_SERVER_TEAM_REMOVER) {
-				return Void();
+				co_return;
 			}
 
 			double removeServerTeamDelay = SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_DELAY;
@@ -2123,18 +2143,18 @@ class DDTeamCollectionImpl {
 				removeServerTeamDelay = removeServerTeamDelay / 100;
 			}
 			// To avoid removing server teams too fast, which is unlikely happen though
-			wait(delay(removeServerTeamDelay, TaskPriority::DataDistribution));
+			co_await delay(removeServerTeamDelay, TaskPriority::DataDistribution);
 
 			if (SERVER_KNOBS->PERPETUAL_WIGGLE_DISABLE_REMOVER && self->pauseWiggle) {
 				while (!self->pauseWiggle->get()) {
-					wait(self->pauseWiggle->onChange());
+					co_await self->pauseWiggle->onChange();
 				}
 			} else {
-				wait(self->waitUntilHealthy(SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY));
+				co_await self->waitUntilHealthy(SERVER_KNOBS->TR_REMOVE_SERVER_TEAM_EXTRA_DELAY);
 			}
 			// Wait for the badTeamRemover() to avoid the potential race between
 			// adding the bad team (add the team tracker) and remove bad team (cancel the team tracker).
-			wait(self->badTeamRemover);
+			co_await self->badTeamRemover;
 
 			if (SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC > 0 &&
 			    self->teamPivots.pivotAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) {
@@ -2142,7 +2162,7 @@ class DDTeamCollectionImpl {
 				    .detail("IsPrimary", self->primary)
 				    .detail("CurrentSpacePivot", self->teamPivots.pivotAvailableSpaceRatio)
 				    .detail("TargetSpacePivot", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO);
-				wait(delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC));
+				co_await delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC);
 				continue;
 			}
 
@@ -2196,19 +2216,19 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> fixUnderReplicationLoop(DDTeamCollection* self) {
-		loop {
-			wait(delay(SERVER_KNOBS->DD_FIX_WRONG_REPLICAS_DELAY));
+	static Future<Void> fixUnderReplicationLoop(DDTeamCollection* self) {
+		while (true) {
+			co_await delay(SERVER_KNOBS->DD_FIX_WRONG_REPLICAS_DELAY);
 			self->cleanupLargeTeams();
 			self->fixUnderReplication();
 		}
 	}
 
-	ACTOR static Future<Void> trackExcludedServers(DDTeamCollection* self) {
-		state ExclusionTracker exclusionTracker(self->dbContext());
-		loop {
+	static Future<Void> trackExcludedServers(DDTeamCollection* self) {
+		ExclusionTracker exclusionTracker(self->dbContext());
+		while (true) {
 			// wait for new set of excluded servers
-			wait(exclusionTracker.changed.onTrigger());
+			co_await exclusionTracker.changed.onTrigger();
 
 			// Reset and reassign self->excludedServers based on excluded, but we only
 			// want to trigger entries that are different
@@ -2240,70 +2260,74 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> updateNextWigglingStorageID(DDTeamCollection* self) {
-		state StorageWiggleData wiggleState;
-		state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap =
+	static Future<Void> updateNextWigglingStorageID(DDTeamCollection* self) {
+		auto* counters = updateNextWigglingStorageIDCounters();
+		StorageWiggleData wiggleState;
+		KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap =
 		    wiggleState.wigglingStorageServer(PrimaryRegion(self->primary));
 
-		state UID nextId = wait(self->getNextWigglingServerID());
-		state StorageWiggleValue value(nextId);
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->dbContext()));
-		loop {
+		UID nextId = co_await self->getNextWigglingServerID();
+		StorageWiggleValue value(nextId);
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->dbContext()));
+		while (true) {
+			counters->started->increment(1);
 			// write the next server id
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				metadataMap.set(tr, nextId, value);
-				wait(tr->commit());
+				co_await tr->commit();
+				counters->committed->increment(1);
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				counters->aborted->increment(1);
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
 		self->nextWiggleInfo.send(value);
 		TraceEvent(SevDebug, "PerpetualStorageWiggleNextID", self->distributorId)
 		    .detail("Primary", self->primary)
 		    .detail("WriteID", nextId);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> waitPerpetualWiggleDelay(DDTeamCollection* self) {
-		if (g_network->isSimulated() && g_simulator->isConsistencyChecked) {
+	static Future<Void> waitPerpetualWiggleDelay(DDTeamCollection* self) {
+		if (g_network->isSimulated() && fdbSimulationPolicyState().isConsistencyChecked) {
 			// Wiggle can cause consistency check to repeatedly restart. So we want to
 			// slow it down to avoid consistency check timeout.
-			wait(delay(300));
-			return Void();
+			co_await delay(300);
+			co_return;
 		} else if (SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY <= 60.0) {
-			wait(delay(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY));
-			return Void();
+			co_await delay(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY);
+			co_return;
 		}
-		state double nextDelay = 60.0;
+		double nextDelay = 60.0;
 		while (true) {
-			wait(delay(nextDelay));
+			co_await delay(nextDelay);
 
-			double totalDelay = wait(self->storageWiggler->wiggleData.addPerpetualWiggleDelay(
-			    self->dbContext().getReference(), PrimaryRegion(self->primary), nextDelay));
+			double totalDelay = co_await self->storageWiggler->wiggleData.addPerpetualWiggleDelay(
+			    self->dbContext().getReference(), PrimaryRegion(self->primary), nextDelay);
 			nextDelay = std::min(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY - totalDelay, 60.0);
 
 			if (totalDelay >= SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY) {
-				wait(self->storageWiggler->wiggleData.clearPerpetualWiggleDelay(self->dbContext().getReference(),
-				                                                                PrimaryRegion(self->primary)));
-				return Void();
+				co_await self->storageWiggler->wiggleData.clearPerpetualWiggleDelay(self->dbContext().getReference(),
+				                                                                    PrimaryRegion(self->primary));
+				co_return;
 			}
 		}
 	}
 
-	ACTOR static Future<Void> perpetualStorageWiggleRest(DDTeamCollection* self) {
-		state bool takeRest = true;
-		state Promise<int64_t> avgShardBytes;
+	static Future<Void> perpetualStorageWiggleRest(DDTeamCollection* self) {
+		bool takeRest = true;
+		Promise<int64_t> avgShardBytes;
 		while (takeRest) {
 			// a minimal delay to avoid excluding and including SS too fast
-			wait(waitPerpetualWiggleDelay(self));
+			co_await waitPerpetualWiggleDelay(self);
 
 			avgShardBytes.reset();
 			self->getAverageShardBytes.send(avgShardBytes);
-			int64_t avgBytes = wait(avgShardBytes.getFuture());
+			int64_t avgBytes = co_await avgShardBytes.getFuture();
 			double ratio;
 			bool imbalance, noMinAvailSpace;
 			int numSSToBeLoadBytesBalanced;
@@ -2357,36 +2381,31 @@ class DDTeamCollectionImpl {
 				    .detail("MigrationType", self->configuration.storageMigrationType.toString());
 			}
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> perpetualStorageWiggleIterator(DDTeamCollection* teamCollection,
-	                                                         AsyncVar<bool>* stopSignal,
-	                                                         FutureStream<Void> finishStorageWiggleSignal) {
-		loop {
-			choose {
-				when(wait(stopSignal->onChange())) {}
-				when(waitNext(finishStorageWiggleSignal)) {
-					// delay to avoid delete and update ServerList too frequently, which could result busy loop or over
-					// utilize the disk of other active SS
-					wait(perpetualStorageWiggleRest(teamCollection));
-					wait(updateNextWigglingStorageID(teamCollection));
-				}
+	static Future<Void> perpetualStorageWiggleIterator(DDTeamCollection* teamCollection,
+	                                                   AsyncVar<bool>* stopSignal,
+	                                                   FutureStream<Void> finishStorageWiggleSignal) {
+		while (true) {
+			auto res = co_await race(stopSignal->onChange(), finishStorageWiggleSignal);
+			if (res.index() == 1) {
+				// delay to avoid delete and update ServerList too frequently, which could result busy loop or over
+				// utilize the disk of other active SS
+				co_await perpetualStorageWiggleRest(teamCollection);
+				co_await updateNextWigglingStorageID(teamCollection);
 			}
 			if (stopSignal->get()) {
 				break;
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> clusterHealthCheckForPerpetualWiggle(DDTeamCollection* self, int* extraTeamCount) {
-		state int pausePenalty = 1;
-		loop {
+	static Future<Void> clusterHealthCheckForPerpetualWiggle(DDTeamCollection* self, int* extraTeamCount) {
+		int pausePenalty = 1;
+		while (true) {
 			Promise<int> countp;
 			self->getUnhealthyRelocationCount.send(countp);
-			int count = wait(countp.getFuture());
+			int count = co_await countp.getFuture();
 			// pause wiggle when
 			// a. DDQueue is busy with unhealthy relocation request
 			// b. healthy teams are not enough
@@ -2411,7 +2430,7 @@ class DDTeamCollectionImpl {
 			} else {
 				self->pauseWiggle->set(false);
 			}
-			wait(delay(SERVER_KNOBS->CHECK_TEAM_DELAY, TaskPriority::DataDistributionLow));
+			co_await delay(SERVER_KNOBS->CHECK_TEAM_DELAY, TaskPriority::DataDistributionLow);
 		}
 	}
 
@@ -2524,30 +2543,34 @@ class DDTeamCollectionImpl {
 	// This coroutine sets a watch to monitor the value change of `perpetualStorageWiggleKey` which is controlled by
 	// command `configure perpetual_storage_wiggle=$value` if the value is 1, this actor start 2 actors,
 	// `perpetualStorageWiggleIterator` and `perpetualStorageWiggler`. Otherwise, it sends stop signal to them.
-	ACTOR static Future<Void> monitorPerpetualStorageWiggle(DDTeamCollection* self) {
-		state int speed = 0;
-		state PromiseStream<Void> finishStorageWiggleSignal;
-		state SignalableActorCollection collection;
+	static Future<Void> monitorPerpetualStorageWiggle(DDTeamCollection* self) {
+		auto* counters = perpetualStorageWigglerCounters();
+		int speed = 0;
+		PromiseStream<Void> finishStorageWiggleSignal;
+		SignalableActorCollection collection;
 		self->pauseWiggle = makeReference<AsyncVar<bool>>(true);
 		ASSERT(self->storageWiggler->isStopped());
 
-		loop {
-			state ReadYourWritesTransaction tr(self->dbContext());
-			loop {
+		while (true) {
+			ReadYourWritesTransaction tr(self->dbContext());
+			while (true) {
+				counters->started->increment(1);
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					Optional<Standalone<StringRef>> value = wait(tr.get(perpetualStorageWiggleKey));
+					Optional<Standalone<StringRef>> value = co_await tr.get(perpetualStorageWiggleKey);
 
 					if (value.present()) {
 						speed = std::stoi(value.get().toString());
 					}
-					state Future<Void> watchFuture = tr.watch(perpetualStorageWiggleKey);
-					wait(tr.commit());
+					Future<Void> watchFuture = tr.watch(perpetualStorageWiggleKey);
+					co_await tr.commit();
+					counters->committed->increment(1);
 
 					ASSERT(speed == 1 || speed == 0);
 					if (speed == 1 && self->storageWiggler->isStopped()) { // avoid duplicated start
 						self->storageWiggler->setStopSignal(false);
-						wait(self->storageWiggler->restoreStats());
+						co_await self->storageWiggler->restoreStats();
 						collection.add(self->perpetualStorageWiggleIterator(self->storageWiggler->stopWiggleSignal,
 						                                                    finishStorageWiggleSignal.getFuture()));
 						collection.add(self->perpetualStorageWiggler(self->storageWiggler->stopWiggleSignal,
@@ -2556,29 +2579,35 @@ class DDTeamCollectionImpl {
 					} else if (speed == 0) {
 						if (!self->storageWiggler->isStopped()) {
 							self->storageWiggler->setStopSignal(true);
-							wait(collection.signalAndReset());
+							co_await collection.signalAndReset();
 							self->pauseWiggle->set(true);
 						}
-						wait(self->storageWiggler->resetStats());
+						co_await self->storageWiggler->resetStats();
 						TraceEvent("PerpetualStorageWiggleClose", self->distributorId).detail("Primary", self->primary);
 					}
-					wait(watchFuture);
+					co_await watchFuture;
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					counters->aborted->increment(1);
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Void> waitHealthyZoneChange(DDTeamCollection* self) {
-		state ReadYourWritesTransaction tr(self->dbContext());
-		loop {
+	static Future<Void> waitHealthyZoneChange(DDTeamCollection* self) {
+		auto* counters = waitHealthyZoneChangeCounters();
+		ReadYourWritesTransaction tr(self->dbContext());
+		while (true) {
+			counters->started->increment(1);
+			Error err;
+			bool hasErr = false;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> val = wait(tr.get(healthyZoneKey));
-				state Future<Void> healthyZoneTimeout = Never();
+				Optional<Value> val = co_await tr.get(healthyZoneKey);
+				Future<Void> healthyZoneTimeout = Never();
 				if (val.present()) {
 					auto p = decodeHealthyZoneValue(val.get());
 					if (p.first == ignoreSSFailuresZoneString) {
@@ -2612,12 +2641,18 @@ class DDTeamCollectionImpl {
 					self->healthyZone.set(Optional<Key>());
 				}
 
-				state Future<Void> watchFuture = tr.watch(healthyZoneKey);
-				wait(tr.commit());
-				wait(watchFuture || healthyZoneTimeout);
+				Future<Void> watchFuture = tr.watch(healthyZoneKey);
+				co_await tr.commit();
+				counters->committed->increment(1);
+				co_await (watchFuture || healthyZoneTimeout);
 				tr.reset();
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				counters->aborted->increment(1);
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr.onError(err);
 			}
 		}
 	}
@@ -2670,11 +2705,11 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> initializeStorage(DDTeamCollection* self,
-	                                            RecruitStorageReply candidateWorker,
-	                                            const DDEnabledState* ddEnabledState,
-	                                            bool recruitTss,
-	                                            Reference<TSSPairState> tssState) {
+	static Future<Void> initializeStorage(DDTeamCollection* self,
+	                                      RecruitStorageReply candidateWorker,
+	                                      const DDEnabledState* ddEnabledState,
+	                                      bool recruitTss,
+	                                      Reference<TSSPairState> tssState) {
 		// SOMEDAY: Cluster controller waits for availability, retry quickly if a server's Locality changes
 		self->recruitingStream.set(self->recruitingStream.get() + 1);
 
@@ -2686,14 +2721,14 @@ class DDTeamCollectionImpl {
 			// Only allow at most 2 storage servers on an address, because
 			// too many storage server on the same address (i.e., process) can cause OOM.
 			// Ask the candidateWorker to initialize a SS only if the worker does not have a pending request
-			state UID interfaceId = deterministicRandom()->randomUniqueID();
+			UID interfaceId = deterministicRandom()->randomUniqueID();
 
 			// insert recruiting localities BEFORE actor waits, to ensure we don't send many recruitment requests to the
 			// same storage
 			self->recruitingIds.insert(interfaceId);
 			self->recruitingLocalities.insert(candidateWorker.worker.stableAddress());
 
-			state InitializeStorageRequest isr;
+			InitializeStorageRequest isr{};
 			isr.storeType = recruitTss ? self->configuration.testingStorageServerStoreType
 			                           : self->configuration.storageServerStoreType;
 
@@ -2717,7 +2752,7 @@ class DDTeamCollectionImpl {
 			isr.interfaceId = interfaceId;
 
 			// if tss, wait for pair ss to finish and add its id to isr. If pair fails, don't recruit tss
-			state bool doRecruit = true;
+			bool doRecruit = true;
 			if (recruitTss) {
 				TraceEvent("TSS_Recruit", self->distributorId)
 				    .detail("ReqID", isr.reqId)
@@ -2727,7 +2762,7 @@ class DDTeamCollectionImpl {
 				    .detail("TSSLocality", candidateWorker.worker.locality.toString())
 				    .detail("Primary", self->primary);
 
-				Optional<std::pair<UID, Version>> ssPairInfoResult = wait(tssState->waitOnSS());
+				Optional<std::pair<UID, Version>> ssPairInfoResult = co_await tssState->waitOnSS();
 				if (ssPairInfoResult.present()) {
 					isr.tssPairIDAndVersion = ssPairInfoResult.get();
 
@@ -2770,7 +2805,7 @@ class DDTeamCollectionImpl {
 			        ? candidateWorker.worker.storage.tryGetReply(isr, TaskPriority::DataDistribution)
 			        : Future<ErrorOr<InitializeStorageReply>>(ErrorOr<InitializeStorageReply>(recruitment_failed()));
 
-			state ErrorOr<InitializeStorageReply> newServer = wait(fRecruit);
+			ErrorOr<InitializeStorageReply> newServer = co_await fRecruit;
 
 			if (doRecruit && newServer.isError()) {
 				TraceEvent(SevWarn, "DDRecruitmentError").error(newServer.getError());
@@ -2779,7 +2814,7 @@ class DDTeamCollectionImpl {
 					tssState->markComplete();
 					throw newServer.getError();
 				}
-				wait(delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY, TaskPriority::DataDistribution));
+				co_await delay(SERVER_KNOBS->STORAGE_RECRUITMENT_DELAY, TaskPriority::DataDistribution);
 			}
 
 			if (!recruitTss && newServer.present() &&
@@ -2796,7 +2831,7 @@ class DDTeamCollectionImpl {
 
 				// wait for timeout, but eventually move on if no TSS pair recruited
 				Optional<bool> tssSuccessful =
-				    wait(timeout(tssState->waitOnTSS(), SERVER_KNOBS->TSS_RECRUITMENT_TIMEOUT));
+				    co_await timeout(tssState->waitOnTSS(), SERVER_KNOBS->TSS_RECRUITMENT_TIMEOUT);
 
 				if (tssSuccessful.present() && tssSuccessful.get()) {
 					TraceEvent("TSS_Recruit", self->distributorId)
@@ -2879,8 +2914,6 @@ class DDTeamCollectionImpl {
 
 		self->recruitingStream.set(self->recruitingStream.get() - 1);
 		self->restartRecruiting.trigger();
-
-		return Void();
 	}
 
 	ACTOR static Future<Void> storageRecruiter(
@@ -3144,21 +3177,21 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> updateReplicasKey(DDTeamCollection* self, Optional<Key> dcId) {
+	static Future<Void> updateReplicasKey(DDTeamCollection* self, Optional<Key> dcId) {
 		std::vector<Future<Void>> serverUpdates;
 
 		for (auto& it : self->server_info) {
 			serverUpdates.push_back(it.second->updated.getFuture());
 		}
 
-		wait(self->initialFailureReactionDelay && waitForAll(serverUpdates));
-		wait(self->waitUntilHealthy());
+		co_await (self->initialFailureReactionDelay && waitForAll(serverUpdates));
+		co_await self->waitUntilHealthy();
 		TraceEvent("DDUpdatingReplicas", self->distributorId)
 		    .detail("Primary", self->primary)
 		    .detail("DcId", dcId)
 		    .detail("Replicas", self->configuration.storageTeamSize);
 
-		int oldReplicas = wait(self->db->tryUpdateReplicasKeyForDc(dcId, self->configuration.storageTeamSize));
+		int oldReplicas = co_await self->db->tryUpdateReplicasKeyForDc(dcId, self->configuration.storageTeamSize);
 		if (oldReplicas == self->configuration.storageTeamSize) {
 			TraceEvent("DDUpdatedAlready", self->distributorId)
 			    .detail("Primary", self->primary)
@@ -3171,12 +3204,11 @@ class DDTeamCollectionImpl {
 			    .detail("Replicas", self->configuration.storageTeamSize)
 			    .detail("OldReplicas", oldReplicas);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> serverGetTeamRequests(DDTeamCollection* self, TeamCollectionInterface tci) {
-		loop {
-			GetTeamRequest req = waitNext(tci.getTeam.getFuture());
+	static Future<Void> serverGetTeamRequests(DDTeamCollection* self, TeamCollectionInterface tci) {
+		while (true) {
+			GetTeamRequest req = co_await tci.getTeam.getFuture();
 			if (req.findTeamByServers) {
 				getTeamByServers(self, req);
 			} else if (req.findTeamForBulkLoad) {
@@ -3187,16 +3219,20 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<Void> monitorHealthyTeams(DDTeamCollection* self) {
+	static Future<Void> monitorHealthyTeams(DDTeamCollection* self) {
 		TraceEvent("DDMonitorHealthyTeamsStart").detail("ZeroHealthyTeams", self->zeroHealthyTeams->get());
-		loop choose {
-			when(wait(self->zeroHealthyTeams->get()
-			              ? delay(SERVER_KNOBS->DD_ZERO_HEALTHY_TEAM_DELAY, TaskPriority::DataDistribution)
-			              : Never())) {
-				self->doBuildTeams = true;
-				wait(self->checkBuildTeams());
+		while (true) {
+			if (self->zeroHealthyTeams->get()) {
+				auto const zeroHealthyTeamsChanged = co_await timeout(self->zeroHealthyTeams->onChange(),
+				                                                      SERVER_KNOBS->DD_ZERO_HEALTHY_TEAM_DELAY,
+				                                                      TaskPriority::DataDistribution);
+				if (!zeroHealthyTeamsChanged.present()) {
+					self->doBuildTeams = true;
+					co_await self->checkBuildTeams();
+				}
+			} else {
+				co_await self->zeroHealthyTeams->onChange();
 			}
-			when(wait(self->zeroHealthyTeams->onChange())) {}
 		}
 	}
 
@@ -3258,18 +3294,18 @@ class DDTeamCollectionImpl {
 		}
 	}
 
-	ACTOR static Future<UID> getNextWigglingServerID(
+	static Future<UID> getNextWigglingServerID(
 	    Reference<StorageWiggler> wiggler,
 	    std::vector<std::pair<Optional<Value>, Optional<Value>>> localityKeyValues =
 	        std::vector<std::pair<Optional<Value>, Optional<Value>>>(),
 	    DDTeamCollection* teamCollection = nullptr) {
 		ASSERT(wiggler->teamCollection == teamCollection);
-		loop {
+		while (true) {
 			// when the DC need more
-			state Optional<UID> id =
+			Optional<UID> id =
 			    wiggler->getNextServerId(teamCollection == nullptr || teamCollection->reachTSSPairTarget());
 			if (!id.present()) {
-				wait(wiggler->onCheck());
+				co_await wiggler->onCheck();
 				continue;
 			}
 
@@ -3279,7 +3315,7 @@ class DDTeamCollectionImpl {
 				auto server = teamCollection->server_info.at(id.get());
 				// TraceEvent("PerpetualLocality").detail("Server", server->getLastKnownInterface().locality.get(localityKey)).detail("Desire", localityValue);
 				if (localityMatchInList(localityKeyValues, server->getLastKnownInterface().locality)) {
-					return id.get();
+					co_return id.get();
 				}
 
 				if (wiggler->empty()) {
@@ -3292,17 +3328,17 @@ class DDTeamCollectionImpl {
 				}
 				continue;
 			}
-			return id.get();
+			co_return id.get();
 		}
 	}
 
 	// read the current map of `perpetualStorageWiggleIDPrefix`, then restore wigglingId.
-	ACTOR static Future<Void> readStorageWiggleMap(DDTeamCollection* self) {
-		state StorageWiggleData wiggleState;
-		state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap =
+	static Future<Void> readStorageWiggleMap(DDTeamCollection* self) {
+		StorageWiggleData wiggleState;
+		KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap =
 		    wiggleState.wigglingStorageServer(PrimaryRegion(self->primary));
-		state std::vector<std::pair<UID, StorageWiggleValue>> res =
-		    wait(readStorageWiggleValues(self->dbContext(), self->primary, false));
+		std::vector<std::pair<UID, StorageWiggleValue>> res =
+		    co_await readStorageWiggleValues(self->dbContext(), self->primary, false);
 
 		if (res.size() > 0) {
 			// SOMEDAY: support wiggle multiple SS at once
@@ -3327,17 +3363,17 @@ class DDTeamCollectionImpl {
 					}
 
 					if (!self->wigglingId.present()) {
-						wait(self->eraseStorageWiggleMap(&metadataMap, res.begin()->first));
+						co_await self->eraseStorageWiggleMap(&metadataMap, res.begin()->first);
 					}
 				}
 			} else {
 				self->wigglingId = res.begin()->first;
 			}
 		}
-		return Void();
 	}
 
 	ACTOR static Future<Void> updateStorageMetadata(DDTeamCollection* self, TCServerInfo* server) {
+		state TxnCounters* counters = updateStorageMetadataCounters();
 		state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(
 		    serverMetadataKeys.begin, IncludeVersion());
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->dbContext());
@@ -3361,6 +3397,7 @@ class DDTeamCollectionImpl {
 
 		// read storage metadata
 		loop {
+			counters->started->increment(1);
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				Optional<Value> serverInterfaceValue = wait(tr->get(serverListKeyFor(server->getId())));
@@ -3381,8 +3418,10 @@ class DDTeamCollectionImpl {
 				metadataMap.set(tr, server->getId(), data);
 				tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
 				wait(tr->commit());
+				counters->committed->increment(1);
 				break;
 			} catch (Error& e) {
+				counters->aborted->increment(1);
 				wait(tr->onError(e));
 			}
 		}
@@ -3527,22 +3566,22 @@ class DDTeamCollectionImpl {
 
 	// Take a snapshot of necessary data structures from `DDTeamCollection` and print them out with yields to avoid slow
 	// task on the run loop.
-	ACTOR static Future<Void> printSnapshotTeamsInfo(Reference<DDTeamCollection> self) {
-		state DatabaseConfiguration configuration;
-		state std::map<UID, Reference<TCServerInfo>> server_info;
-		state std::map<UID, ServerStatus> server_status;
-		state std::vector<Reference<TCTeamInfo>> teams;
-		state std::map<Standalone<StringRef>, Reference<TCMachineInfo>> machine_info;
-		state std::vector<Reference<TCMachineTeamInfo>> machineTeams;
+	static Future<Void> printSnapshotTeamsInfo(Reference<DDTeamCollection> self) {
+		DatabaseConfiguration configuration;
+		std::map<UID, Reference<TCServerInfo>> server_info;
+		std::map<UID, ServerStatus> server_status;
+		std::vector<Reference<TCTeamInfo>> teams;
+		std::map<Standalone<StringRef>, Reference<TCMachineInfo>> machine_info;
+		std::vector<Reference<TCMachineTeamInfo>> machineTeams;
 		// state std::vector<std::string> internedLocalityRecordKeyNameStrings;
 		// state int machineLocalityMapEntryArraySize;
 		// state std::vector<Reference<LocalityRecord>> machineLocalityMapRecordArray;
-		state int traceEventsPrinted = 0;
-		state std::vector<const UID*> serverIDs;
-		state double lastPrintTime = 0;
-		loop {
+		int traceEventsPrinted = 0;
+		std::vector<const UID*> serverIDs;
+		double lastPrintTime = 0;
+		while (true) {
 
-			wait(self->printDetailedTeamsInfo.onTrigger() || self->db->waitDDTeamInfoPrintSignal());
+			co_await (self->printDetailedTeamsInfo.onTrigger() || self->db->waitDDTeamInfoPrintSignal());
 
 			if (now() - lastPrintTime < SERVER_KNOBS->DD_TEAMS_INFO_PRINT_INTERVAL) {
 				continue;
@@ -3588,15 +3627,15 @@ class DDTeamCollectionImpl {
 			    .detail("Size", server_info.size())
 			    .detail("Primary", self->isPrimary());
 
-			state int i;
-			state std::map<UID, Reference<TCServerInfo>>::iterator server = server_info.begin();
+			int i{ 0 };
+			std::map<UID, Reference<TCServerInfo>>::iterator server = server_info.begin();
 			for (i = 0; i < server_info.size(); i++) {
 				auto serverStats = server->second->getStorageStats();
 				TraceEvent("ServerInfo", self->getDistributorId())
 				    .detail("ServerInfoIndex", i)
 				    .detail("ServerID", server->first.toString())
 				    .detail("ServerTeamOwned", server->second->getTeams().size())
-				    .detail("MachineID", server->second->machine->machineID.contents().toString())
+				    .detail("ZoneID", server->second->machine->machineID.contents().toString())
 				    .detail("Primary", self->isPrimary())
 				    .detail("IsInDesiredDC", server->second->isInDesiredDC())
 				    .detail("DataInFlightToServer", server->second->getDataInFlightToServer())
@@ -3612,7 +3651,7 @@ class DDTeamCollectionImpl {
 				// return 100.0;
 				server++;
 				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-					wait(yield());
+					co_await yield();
 				}
 			}
 
@@ -3646,7 +3685,7 @@ class DDTeamCollectionImpl {
 
 				server++;
 				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-					wait(yield());
+					co_await yield();
 				}
 			}
 
@@ -3681,20 +3720,20 @@ class DDTeamCollectionImpl {
 				                ->getShardsFor(ShardsAffectedByTeamFailure::Team(team->getServerIDs(), self->primary))
 				                .size());
 				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-					wait(yield());
+					co_await yield();
 				}
 			}
 
 			TraceEvent("MachineInfo", self->getDistributorId())
 			    .detail("Size", machine_info.size())
 			    .detail("Primary", self->isPrimary());
-			state std::map<Standalone<StringRef>, Reference<TCMachineInfo>>::iterator machine = machine_info.begin();
-			state bool isMachineHealthy = false;
+			std::map<Standalone<StringRef>, Reference<TCMachineInfo>>::iterator machine = machine_info.begin();
+			bool isMachineHealthy = false;
 			for (i = 0; i < machine_info.size(); i++) {
 				Reference<TCMachineInfo> _machine = machine->second;
-				bool machineIDFound = machine_info.find(_machine->machineID) != machine_info.end();
+				bool zoneIDFound = machine_info.find(_machine->machineID) != machine_info.end();
 				bool zeroHealthyServersOnMachine = true;
-				if (!_machine.isValid() || !machineIDFound || _machine->serversOnMachine.empty()) {
+				if (!_machine.isValid() || !zoneIDFound || _machine->serversOnMachine.empty()) {
 					isMachineHealthy = false;
 				}
 
@@ -3714,17 +3753,17 @@ class DDTeamCollectionImpl {
 				TraceEvent("MachineInfo", self->getDistributorId())
 				    .detail("MachineInfoIndex", i)
 				    .detail("Healthy", isMachineHealthy)
-				    .detail("MachineIDFound", machineIDFound)
+				    .detail("ZoneIDFound", zoneIDFound)
 				    .detail("ZeroServersOnMachine", _machine->serversOnMachine.empty())
 				    .detail("ZeroHealthyServersOnMachine", zeroHealthyServersOnMachine)
-				    .detail("MachineID", machine->first.contents().toString())
+				    .detail("ZoneID", machine->first.contents().toString())
 				    .detail("MachineTeamOwned", machine->second->machineTeams.size())
 				    .detail("ServerNumOnMachine", machine->second->serversOnMachine.size())
 				    .detail("ServersID", machine->second->getServersIDStr())
 				    .detail("Primary", self->isPrimary());
 				machine++;
 				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-					wait(yield());
+					co_await yield();
 				}
 			}
 
@@ -3736,11 +3775,11 @@ class DDTeamCollectionImpl {
 				const auto& team = machineTeams[i];
 				TraceEvent(g_network->isSimulated() ? SevVerbose : SevInfo, "MachineTeamInfo", self->getDistributorId())
 				    .detail("TeamIndex", i)
-				    .detail("MachineIDs", team->getMachineIDsStr())
+				    .detail("ZoneIDs", team->getMachineIDsStr())
 				    .detail("ServerTeams", team->getServerTeams().size())
 				    .detail("Primary", self->isPrimary());
 				if (++traceEventsPrinted % SERVER_KNOBS->DD_TEAMS_INFO_PRINT_YIELD_COUNT == 0) {
-					wait(yield());
+					co_await yield();
 				}
 			}
 
@@ -3940,7 +3979,7 @@ void DDTeamCollection::traceServerInfo() const {
 		    .detail("ServerInfoIndex", i++)
 		    .detail("ServerID", serverID.toString())
 		    .detail("ServerTeamOwned", server->getTeams().size())
-		    .detail("MachineID", server->machine->machineID.contents().toString())
+		    .detail("ZoneID", server->machine->machineID.contents().toString())
 		    .detail("StoreType", server->getStoreType().toString())
 		    .detail("InDesiredDC", server->isInDesiredDC());
 	}
@@ -4376,8 +4415,8 @@ Future<Void> DDTeamCollection::updateStorageMetadata(TCServerInfo* server) {
 }
 
 void DDTeamCollection::resetLocalitySet() {
-	storageServerSet = Reference<LocalitySet>(new LocalityMap<UID>());
-	LocalityMap<UID>* storageServerMap = (LocalityMap<UID>*)storageServerSet.getPtr();
+	storageServerSet = makeReference<LocalityMap<UID>>();
+	auto* storageServerMap = static_cast<LocalityMap<UID>*>(storageServerSet.getPtr());
 
 	for (auto& it : server_info) {
 		it.second->localityEntry =
@@ -4590,7 +4629,6 @@ void DDTeamCollection::evaluateTeamQuality() const {
 			minTeams = std::min(minTeams, stc);
 			maxTeams = std::max(maxTeams, stc);
 			varTeams += (stc - teamsPerServer) * (stc - teamsPerServer);
-			// Use zoneId as server's machine id
 			machineTeams[info->getLastKnownInterface().locality.zoneId()] += stc;
 			// Check invariant: if latest buildTeam succeeds, then each server must have at least
 			// targetTeamNumPerServer serverTeams
@@ -4727,7 +4765,7 @@ struct ServerPriority {
 	UID id;
 	Reference<TCServerInfo> info;
 
-	ServerPriority() {}
+	ServerPriority() = default;
 	ServerPriority(int healthyShards, int unhealthyShards, int64_t loadBytes, UID id, Reference<TCServerInfo> info)
 	  : healthyShards(healthyShards), unhealthyShards(unhealthyShards), loadBytes(loadBytes), id(id), info(info) {}
 
@@ -4815,8 +4853,8 @@ Reference<TCTeamInfo> DDTeamCollection::buildLargeTeam(int teamSize) {
 		    .detail("SatisfiesPolicy", satisfiesPolicy(candidateTeam));
 		return Reference<TCTeamInfo>();
 	} else if (candidateTeam.size() > teamSize) {
-		Reference<LocalitySet> tempSet = Reference<LocalitySet>(new LocalityMap<UID>());
-		LocalityMap<UID>* tempMap = (LocalityMap<UID>*)tempSet.getPtr();
+		Reference<LocalitySet> tempSet = makeReference<LocalityMap<UID>>();
+		auto* tempMap = static_cast<LocalityMap<UID>*>(tempSet.getPtr());
 		tempSet->clear();
 		for (auto& it : candidateTeam) {
 			tempMap->add(it->getLastKnownInterface().locality, &it->getId());
@@ -4941,7 +4979,7 @@ Reference<TCMachineTeamInfo> DDTeamCollection::addMachineTeam(std::vector<Standa
 		if (machine_info.find(*i) != machine_info.end()) {
 			machines.push_back(machine_info[*i]);
 		} else {
-			TraceEvent(SevWarn, "AddMachineTeamError").detail("MachineIDNotExist", i->contents().toString());
+			TraceEvent(SevWarn, "AddMachineTeamError").detail("ZoneIDNotExist", i->contents().toString());
 		}
 	}
 
@@ -4990,7 +5028,7 @@ void DDTeamCollection::traceMachineInfo() const {
 		TraceEvent("MachineInfo", distributorId)
 		    .detail("MachineInfoIndex", i++)
 		    .detail("Healthy", isMachineHealthy(machineInfo))
-		    .detail("MachineID", machineName.contents().toString())
+		    .detail("ZoneID", machineName.contents().toString())
 		    .detail("MachineTeamOwned", machineInfo->machineTeams.size())
 		    .detail("ServerNumOnMachine", machineInfo->serversOnMachine.size())
 		    .detail("ServersID", machineInfo->getServersIDStr());
@@ -5004,7 +5042,7 @@ void DDTeamCollection::traceMachineTeamInfo() const {
 	for (auto& team : machineTeams) {
 		TraceEvent("MachineTeamInfo", distributorId)
 		    .detail("TeamIndex", i++)
-		    .detail("MachineIDs", team->getMachineIDsStr())
+		    .detail("ZoneIDs", team->getMachineIDsStr())
 		    .detail("ServerTeams", team->getServerTeams().size());
 	}
 }
@@ -5148,7 +5186,7 @@ int DDTeamCollection::addBestMachineTeams(int machineTeamsToBuild) {
 				forcedAttributes.push_back(process);
 				TraceEvent("ChosenMachine")
 				    .suppressFor(30.0)
-				    .detail("MachineInfo", tcMachineInfo->machineID)
+				    .detail("ZoneID", tcMachineInfo->machineID)
 				    .detail("LeaseUsedMachinesSize", leastUsedMachines.size())
 				    .detail("ForcedAttributesSize", forcedAttributes.size());
 			} else {
@@ -5183,8 +5221,8 @@ int DDTeamCollection::addBestMachineTeams(int machineTeamsToBuild) {
 			for (auto process = team.begin(); process != team.end(); process++) {
 				Reference<TCServerInfo> server = server_info[**process];
 				score += server->machine->machineTeams.size();
-				Standalone<StringRef> machine_id = server->getLastKnownInterface().locality.zoneId().get();
-				machineIDs.push_back(machine_id);
+				Standalone<StringRef> zone_id = server->getLastKnownInterface().locality.zoneId().get();
+				machineIDs.push_back(zone_id);
 			}
 
 			// Only choose healthy machines into machine team
@@ -5375,7 +5413,7 @@ bool DDTeamCollection::isServerTeamCountCorrect(Reference<TCMachineTeamInfo> con
 	if (num != mt->getServerTeams().size()) {
 		ret = false;
 		TraceEvent(SevError, "ServerTeamCountOnMachineIncorrect")
-		    .detail("MachineTeam", mt->getMachineIDsStr())
+		    .detail("ZoneIDs", mt->getMachineIDsStr())
 		    .detail("ServerTeamsSize", mt->getServerTeams().size())
 		    .detail("CountedServerTeams", num);
 	}
@@ -5732,6 +5770,7 @@ void DDTeamCollection::noHealthyTeams() const {
 	}
 
 	TraceEvent(SevWarn, "NoHealthyTeams", distributorId)
+	    .suppressFor(5.0)
 	    .detail("CurrentServerTeamCount", teams.size())
 	    .detail("ServerCount", server_info.size())
 	    .detail("NonFailedServerCount", desiredServerSet.size());
@@ -5836,22 +5875,22 @@ bool DDTeamCollection::removeTeam(Reference<TCTeamInfo> team) {
 Reference<TCMachineInfo> DDTeamCollection::checkAndCreateMachine(Reference<TCServerInfo> server) {
 	ASSERT(server.isValid() && server_info.find(server->getId()) != server_info.end());
 	auto const& locality = server->getLastKnownInterface().locality;
-	Standalone<StringRef> machine_id = locality.zoneId().get(); // locality to machine_id with std::string type
+	Standalone<StringRef> zone_id = locality.zoneId().get();
 
 	Reference<TCMachineInfo> machineInfo;
-	if (machine_info.find(machine_id) == machine_info.end()) {
-		// uid is the first storage server process on the machine
-		CODE_PROBE(true, "First storage server in process on the machine");
-		// For each machine, store the first server's localityEntry into machineInfo for later use.
+	if (machine_info.find(zone_id) == machine_info.end()) {
+		// uid is the first storage server process on the zone-keyed machine.
+		CODE_PROBE(true, "First storage server in process on the zone-keyed machine");
+		// Store the first server's localityEntry for this zone-keyed machine for later use.
 		LocalityEntry localityEntry = machineLocalityMap.add(locality, &server->getId());
 		machineInfo = makeReference<TCMachineInfo>(server, localityEntry);
-		machine_info.insert(std::make_pair(machine_id, machineInfo));
+		machine_info.insert(std::make_pair(zone_id, machineInfo));
 	} else {
-		machineInfo = machine_info.find(machine_id)->second;
+		machineInfo = machine_info.find(zone_id)->second;
 		machineInfo->serversOnMachine.push_back(server);
 	}
 	server->machine = machineInfo;
-	ASSERT(machineInfo->machineID == machine_id); // invariant for TC to work
+	ASSERT(machineInfo->machineID == zone_id); // invariant for TC to work
 
 	return machineInfo;
 }
@@ -5908,9 +5947,9 @@ void DDTeamCollection::removeMachine(Reference<TCMachineInfo> removedMachineInfo
 
 	// Remove removedMachineInfo from machine's global info
 	machine_info.erase(removedMachineInfo->machineID);
-	TraceEvent("MachineLocalityMapUpdate").detail("MachineUIDRemoved", removedMachineInfo->machineID.toString());
+	TraceEvent("MachineLocalityMapUpdate").detail("ZoneIDRemoved", removedMachineInfo->machineID.toString());
 
-	// We do not update macineLocalityMap when a machine is removed because we will do so when we use it in
+	// We do not update machineLocalityMap when a machine is removed because we will do so when we use it in
 	// addBestMachineTeams()
 	// rebuildMachineLocalityMap();
 }
@@ -6181,7 +6220,7 @@ class DDTeamCollectionUnitTest {
 	    Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure) {
 		Database database = DatabaseContext::create(
 		    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
-		auto txnProcessor = Reference<IDDTxnProcessor>(new DDTxnProcessor(database));
+		auto txnProcessor = makeReference<DDTxnProcessor>(database);
 		DatabaseConfiguration conf;
 		conf.storageTeamSize = teamSize;
 		conf.storagePolicy = policy;
@@ -6216,7 +6255,8 @@ class DDTeamCollectionUnitTest {
 			interface.locality.set("data_hall"_sr, Standalone<StringRef>(std::to_string(id % 3)));
 			collection->server_info[uid] = makeReference<TCServerInfo>(
 			    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
-			collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
+			collection->server_status.set(
+			    uid, ServerStatus(IsFailed::False, IsUndesired::False, IsWiggling::False, interface.locality));
 			collection->checkAndCreateMachine(collection->server_info[uid]);
 		}
 
@@ -6226,8 +6266,7 @@ class DDTeamCollectionUnitTest {
 	static std::unique_ptr<DDTeamCollection> testTeamCollection(int teamSize,
 	                                                            Reference<IReplicationPolicy> policy,
 	                                                            int processCount) {
-		Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure =
-		    makeReference<ShardsAffectedByTeamFailure>();
+		auto shardsAffectedByTeamFailure = makeReference<ShardsAffectedByTeamFailure>();
 		return testTeamCollection(teamSize, policy, processCount, shardsAffectedByTeamFailure);
 	}
 
@@ -6236,7 +6275,7 @@ class DDTeamCollectionUnitTest {
 	                                                                   int processCount) {
 		Database database = DatabaseContext::create(
 		    makeReference<AsyncVar<ClientDBInfo>>(), Never(), LocalityData(), EnableLocalityLoadBalance::False);
-		auto txnProcessor = Reference<IDDTxnProcessor>(new DDTxnProcessor(database));
+		auto txnProcessor = makeReference<DDTxnProcessor>(database);
 		DatabaseConfiguration conf;
 		conf.storageTeamSize = teamSize;
 		conf.storagePolicy = policy;
@@ -6285,7 +6324,8 @@ class DDTeamCollectionUnitTest {
 			collection->server_info[uid] = makeReference<TCServerInfo>(
 			    interface, collection.get(), ProcessClass(), true, collection->storageServerSet);
 
-			collection->server_status.set(uid, ServerStatus(false, false, false, interface.locality));
+			collection->server_status.set(
+			    uid, ServerStatus(IsFailed::False, IsUndesired::False, IsWiggling::False, interface.locality));
 		}
 
 		int totalServerIndex = collection->constructMachinesFromServers();
@@ -6294,47 +6334,43 @@ class DDTeamCollectionUnitTest {
 		return collection;
 	}
 
-	ACTOR static Future<Void> AddTeamsBestOf_UseMachineID() {
-		wait(Future<Void>(Void()));
+	static Future<Void> AddTeamsBestOf_UseMachineID() {
+		co_await Future<Void>(Void());
 
 		int teamSize = 3; // replication size
 		int processSize = 60;
 		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
 		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
 
-		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-		state std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
+		Reference<IReplicationPolicy> policy =
+		    makeReference<PolicyAcross>(teamSize, "zoneid", makeReference<PolicyOne>());
+		std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
 
 		collection->addTeamsBestOf(30, desiredTeams, maxTeams);
 
 		ASSERT(collection->sanityCheckTeams() == true);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> AddTeamsBestOf_NotUseMachineID() {
-		wait(Future<Void>(Void()));
+	static Future<Void> AddTeamsBestOf_NotUseMachineID() {
+		co_await Future<Void>(Void());
 
 		int teamSize = 3; // replication size
 		int processSize = 60;
 		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
 		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
 
-		Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(
-		    new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-		state std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
+		Reference<IReplicationPolicy> policy =
+		    makeReference<PolicyAcross>(teamSize, "zoneid", makeReference<PolicyOne>());
+		std::unique_ptr<DDTeamCollection> collection = testMachineTeamCollection(teamSize, policy, processSize);
 
 		if (collection == nullptr) {
 			fprintf(stderr, "collection is null\n");
-			return Void();
+			co_return;
 		}
 
 		collection->addBestMachineTeams(30); // Create machine teams to help debug
 		collection->addTeamsBestOf(30, desiredTeams, maxTeams);
 		collection->sanityCheckTeams(); // Server team may happen to be on the same machine team, although unlikely
-
-		return Void();
 	}
 
 	static void AddAllTeams_isExhaustive() {
@@ -6346,9 +6382,9 @@ class DDTeamCollectionUnitTest {
 
 		int result = collection->addTeamsBestOf(200, desiredTeams, maxTeams);
 
-		// The maximum number of available server teams without considering machine locality is 120
-		// The maximum number of available server teams with machine locality constraint is 120 - 40, because
-		// the 40 (5*4*2) server teams whose servers come from the same machine are invalid.
+		// The maximum number of available server teams without considering zone locality is 120
+		// The maximum number of available server teams with zone locality constraint is 120 - 40, because
+		// the 40 (5*4*2) server teams whose servers come from the same zone are invalid.
 		ASSERT(result == 80);
 	}
 
@@ -6365,20 +6401,20 @@ class DDTeamCollectionUnitTest {
 		ASSERT(result >= 10);
 	}
 
-	ACTOR static Future<Void> AddTeamsBestOf_SkippingBusyServers() {
-		wait(Future<Void>(Void()));
+	static Future<Void> AddTeamsBestOf_SkippingBusyServers() {
+		co_await Future<Void>(Void());
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 10;
-		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-		state int teamSize = 3;
+		int processSize = 10;
+		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		int teamSize = 3;
 		// state int targetTeamsPerServer = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (teamSize + 1) / 2;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
 		collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
 
-		state int result = collection->addTeamsBestOf(8, desiredTeams, maxTeams);
+		int result = collection->addTeamsBestOf(8, desiredTeams, maxTeams);
 
 		ASSERT(result >= 8);
 
@@ -6387,21 +6423,19 @@ class DDTeamCollectionUnitTest {
 			ASSERT(teamCount >= 1);
 			// ASSERT(teamCount <= targetTeamsPerServer);
 		}
-
-		return Void();
 	}
 
 	// Due to the randomness in choosing the machine team and the server team from the machine team, it is possible that
 	// we may not find the remaining several (e.g., 1 or 2) available teams.
 	// It is hard to conclude what is the minimum number of  teams the addTeamsBestOf() should create in this situation.
-	ACTOR static Future<Void> AddTeamsBestOf_NotEnoughServers() {
-		wait(Future<Void>(Void()));
+	static Future<Void> AddTeamsBestOf_NotEnoughServers() {
+		co_await Future<Void>(Void());
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
-		state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
+		int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
 		collection->addTeam(std::set<UID>({ UID(1, 0), UID(3, 0), UID(4, 0) }), IsInitialTeam::True);
@@ -6427,15 +6461,13 @@ class DDTeamCollectionUnitTest {
 
 		// If we find all available teams, result will be 8 because we prebuild 2 teams
 		ASSERT(result == 8);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_NewServersNotNeeded() {
+	static Future<Void> GetTeam_NewServersNotNeeded() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		GetStorageMetricsReply mid_avail;
 		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
@@ -6465,14 +6497,14 @@ class DDTeamCollectionUnitTest {
 		 */
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_COMPLETE_SRCS,
-		                         PreferLowerDiskUtil::True,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_COMPLETE_SRCS,
+		                   PreferLowerDiskUtil::True,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
@@ -6481,15 +6513,13 @@ class DDTeamCollectionUnitTest {
 		auto servers = resTeam.get()->getServerIDs();
 		const std::set<UID> selectedServers(servers.begin(), servers.end());
 		ASSERT(expectedServers == selectedServers);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_HealthyCompleteSource() {
+	static Future<Void> GetTeam_HealthyCompleteSource() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		GetStorageMetricsReply mid_avail;
 		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
@@ -6520,14 +6550,14 @@ class DDTeamCollectionUnitTest {
 		 */
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0), UID(4, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_COMPLETE_SRCS,
-		                         PreferLowerDiskUtil::True,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_COMPLETE_SRCS,
+		                   PreferLowerDiskUtil::True,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
@@ -6537,14 +6567,13 @@ class DDTeamCollectionUnitTest {
 		const std::set<UID> selectedServers(servers.begin(), servers.end());
 
 		ASSERT(expectedServers == selectedServers);
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_TrueBestLeastUtilized() {
+	static Future<Void> GetTeam_TrueBestLeastUtilized() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		GetStorageMetricsReply mid_avail;
 		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
@@ -6573,14 +6602,14 @@ class DDTeamCollectionUnitTest {
 
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
-		                         PreferLowerDiskUtil::True,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
+		                   PreferLowerDiskUtil::True,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
@@ -6589,15 +6618,13 @@ class DDTeamCollectionUnitTest {
 		auto servers = resTeam.get()->getServerIDs();
 		const std::set<UID> selectedServers(servers.begin(), servers.end());
 		ASSERT(expectedServers == selectedServers);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_TrueBestMostUtilized() {
+	static Future<Void> GetTeam_TrueBestMostUtilized() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		GetStorageMetricsReply mid_avail;
 		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
@@ -6625,14 +6652,14 @@ class DDTeamCollectionUnitTest {
 		 */
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
-		                         PreferLowerDiskUtil::False,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
+		                   PreferLowerDiskUtil::False,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
@@ -6641,15 +6668,13 @@ class DDTeamCollectionUnitTest {
 		auto servers = resTeam.get()->getServerIDs();
 		const std::set<UID> selectedServers(servers.begin(), servers.end());
 		ASSERT(expectedServers == selectedServers);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_ServerUtilizationBelowCutoff() {
+	static Future<Void> GetTeam_ServerUtilizationBelowCutoff() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		GetStorageMetricsReply low_avail;
 		low_avail.capacity.bytes = SERVER_KNOBS->MIN_AVAILABLE_SPACE * 20;
@@ -6679,27 +6704,25 @@ class DDTeamCollectionUnitTest {
 		 */
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
-		                         PreferLowerDiskUtil::True,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
+		                   PreferLowerDiskUtil::True,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
 		ASSERT(!resTeam.present());
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_ServerUtilizationNearCutoff() {
+	static Future<Void> GetTeam_ServerUtilizationNearCutoff() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		GetStorageMetricsReply low_avail;
 		if (SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO > 0) {
@@ -6738,27 +6761,25 @@ class DDTeamCollectionUnitTest {
 		 */
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
-		                         PreferLowerDiskUtil::True,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
+		                   PreferLowerDiskUtil::True,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto& [resTeam, srcTeamFound] = req.reply.getFuture().get();
 
 		ASSERT(!resTeam.present());
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_TrueBestLeastReadBandwidth() {
+	static Future<Void> GetTeam_TrueBestLeastReadBandwidth() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(1, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 1;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 1;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 
 		int64_t capacity = 1000 * 1024 * 1024, available = 800 * 1024 * 1024;
 		std::vector<int64_t> read_bandwidths{
@@ -6785,22 +6806,22 @@ class DDTeamCollectionUnitTest {
 		auto forReadBalance = ForReadBalance::True;
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
-		                         preferLowerDiskUtil,
-		                         teamMustHaveShards,
-		                         PreferLowerReadUtil::True,
-		                         PreferWithinShardLimit::False,
-		                         forReadBalance);
+		GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
+		                   preferLowerDiskUtil,
+		                   teamMustHaveShards,
+		                   PreferLowerReadUtil::True,
+		                   PreferWithinShardLimit::False,
+		                   forReadBalance);
 		req.completeSources = completeSources;
 
-		state GetTeamRequest reqHigh(TeamSelect::WANT_TRUE_BEST,
-		                             PreferLowerDiskUtil::False,
-		                             teamMustHaveShards,
-		                             PreferLowerReadUtil::False,
-		                             PreferWithinShardLimit::False,
-		                             forReadBalance);
+		GetTeamRequest reqHigh(TeamSelect::WANT_TRUE_BEST,
+		                       PreferLowerDiskUtil::False,
+		                       teamMustHaveShards,
+		                       PreferLowerReadUtil::False,
+		                       PreferWithinShardLimit::False,
+		                       forReadBalance);
 
-		wait(collection->getTeam(req) && collection->getTeam(reqHigh));
+		co_await (collection->getTeam(req) && collection->getTeam(reqHigh));
 		auto [resTeam, resTeamSrcFound] = req.reply.getFuture().get();
 		auto [resTeamHigh, resTeamHighSrcFound] = reqHigh.reply.getFuture().get();
 
@@ -6819,21 +6840,19 @@ class DDTeamCollectionUnitTest {
 
 		resTeam.get()->addReadInFlightToTeam(50);
 		req.reply.reset();
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 		auto [resTeam1, resTeam1Found] = req.reply.getFuture().get();
 		std::set<UID> expectedServers1{ UID(2, 0) };
 		auto servers1 = resTeam1.get()->getServerIDs();
 		const std::set<UID> selectedServers1(servers1.begin(), servers1.end());
 		ASSERT(expectedServers1 == selectedServers1);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_DeprioritizeWigglePausedTeam() {
+	static Future<Void> GetTeam_DeprioritizeWigglePausedTeam() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		int processSize = 5;
+		int teamSize = 3;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
 		GetStorageMetricsReply mid_avail;
 		mid_avail.capacity.bytes = 1000 * 1024 * 1024;
 		mid_avail.available.bytes = 400 * 1024 * 1024;
@@ -6859,14 +6878,14 @@ class DDTeamCollectionUnitTest {
 
 		std::vector<UID> completeSources{ UID(1, 0), UID(2, 0), UID(3, 0) };
 
-		state GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
-		                         PreferLowerDiskUtil::True,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::False);
+		GetTeamRequest req(TeamSelect::WANT_TRUE_BEST,
+		                   PreferLowerDiskUtil::True,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::False);
 		req.completeSources = completeSources;
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
@@ -6875,28 +6894,26 @@ class DDTeamCollectionUnitTest {
 		auto servers = resTeam.get()->getServerIDs();
 		const std::set<UID> selectedServers(servers.begin(), servers.end());
 		ASSERT(expectedServers == selectedServers);
-
-		return Void();
 	}
 
 	// Cut off high Cpu teams
-	ACTOR static Future<Void> GetTeam_CutOffByCpu() {
+	static Future<Void> GetTeam_CutOffByCpu() {
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(1, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 4;
-		state int teamSize = 1;
-		state std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
-		state GetTeamRequest bestReq(TeamSelect::WANT_TRUE_BEST,
-		                             PreferLowerDiskUtil::True,
-		                             TeamMustHaveShards::False,
-		                             PreferLowerReadUtil::True,
-		                             PreferWithinShardLimit::False,
-		                             ForReadBalance::True);
-		state GetTeamRequest randomReq(TeamSelect::ANY,
-		                               PreferLowerDiskUtil::True,
-		                               TeamMustHaveShards::False,
-		                               PreferLowerReadUtil::True,
-		                               PreferWithinShardLimit::False,
-		                               ForReadBalance::True);
+		int processSize = 4;
+		int teamSize = 1;
+		std::unique_ptr<DDTeamCollection> collection = testTeamCollection(teamSize, policy, processSize);
+		GetTeamRequest bestReq(TeamSelect::WANT_TRUE_BEST,
+		                       PreferLowerDiskUtil::True,
+		                       TeamMustHaveShards::False,
+		                       PreferLowerReadUtil::True,
+		                       PreferWithinShardLimit::False,
+		                       ForReadBalance::True);
+		GetTeamRequest randomReq(TeamSelect::ANY,
+		                         PreferLowerDiskUtil::True,
+		                         TeamMustHaveShards::False,
+		                         PreferLowerReadUtil::True,
+		                         PreferWithinShardLimit::False,
+		                         ForReadBalance::True);
 		collection->teamPivots.lastPivotValuesUpdate = -100;
 
 		int64_t capacity = SERVER_KNOBS->MIN_AVAILABLE_SPACE * 20, loadBytes = 90 * 1024 * 1024;
@@ -6924,7 +6941,7 @@ class DDTeamCollectionUnitTest {
 		if (!maxCutoff) {
 			// use pivot value as cutoff
 			auto ratio = KnobValueRef::create(double{ 0.7 });
-			IKnobCollection::getMutableGlobalKnobCollection().setKnob("cpu_pivot_ratio", ratio);
+			setServerKnob("cpu_pivot_ratio", ratio);
 		}
 		low_cpu.cpuUsage = SERVER_KNOBS->MAX_DEST_CPU_PERCENT - 60;
 		mid_cpu.cpuUsage = SERVER_KNOBS->MAX_DEST_CPU_PERCENT - 40;
@@ -6950,7 +6967,7 @@ class DDTeamCollectionUnitTest {
 		collection->disableBuildingTeams();
 		collection->setCheckTeamDelay();
 
-		wait(collection->getTeam(bestReq));
+		co_await collection->getTeam(bestReq);
 		const auto [bestTeam, found1] = bestReq.reply.getFuture().get();
 		fmt::print("{} {} {}\n",
 		           SERVER_KNOBS->CPU_PIVOT_RATIO,
@@ -6959,24 +6976,22 @@ class DDTeamCollectionUnitTest {
 		ASSERT(bestTeam.present());
 		ASSERT_EQ(bestTeam.get()->getServerIDs(), std::vector<UID>{ UID(2, 0) });
 
-		wait(collection->getTeam(randomReq));
+		co_await collection->getTeam(randomReq);
 		const auto [randomTeam, found2] = randomReq.reply.getFuture().get();
 		if (randomTeam.present()) {
 			CODE_PROBE(true, "Unit Test Random Team Return Candidate.");
 			ASSERT_NE(randomTeam.get()->getServerIDs(), std::vector<UID>{ UID(3, 0) });
 			ASSERT_NE(randomTeam.get()->getServerIDs(), std::vector<UID>{ UID(4, 0) });
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> GetTeam_PreferShardsWithinLimit() {
+	static Future<Void> GetTeam_PreferShardsWithinLimit() {
 		ASSERT(SERVER_KNOBS->ENFORCE_SHARD_COUNT_PER_TEAM);
 		Reference<IReplicationPolicy> policy = makeReference<PolicyAcross>(3, "zoneid", makeReference<PolicyOne>());
-		state int processSize = 5;
-		state int teamSize = 3;
-		Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure =
-		    makeReference<ShardsAffectedByTeamFailure>();
-		state std::unique_ptr<DDTeamCollection> collection =
+		int processSize = 5;
+		int teamSize = 3;
+		auto shardsAffectedByTeamFailure = makeReference<ShardsAffectedByTeamFailure>();
+		std::unique_ptr<DDTeamCollection> collection =
 		    testTeamCollection(teamSize, policy, processSize, shardsAffectedByTeamFailure);
 
 		collection->addTeam(std::set<UID>({ UID(1, 0), UID(2, 0), UID(3, 0) }), IsInitialTeam::True);
@@ -6999,13 +7014,13 @@ class DDTeamCollectionUnitTest {
 		}
 
 		// Call getTeam and check that the second team should always be the selected team.
-		state GetTeamRequest req(deterministicRandom()->coinflip() ? TeamSelect::WANT_TRUE_BEST : TeamSelect::ANY,
-		                         PreferLowerDiskUtil::False,
-		                         TeamMustHaveShards::False,
-		                         PreferLowerReadUtil::False,
-		                         PreferWithinShardLimit::True);
+		GetTeamRequest req(deterministicRandom()->coinflip() ? TeamSelect::WANT_TRUE_BEST : TeamSelect::ANY,
+		                   PreferLowerDiskUtil::False,
+		                   TeamMustHaveShards::False,
+		                   PreferLowerReadUtil::False,
+		                   PreferWithinShardLimit::True);
 
-		wait(collection->getTeam(req));
+		co_await collection->getTeam(req);
 
 		const auto [resTeam, srcFound] = req.reply.getFuture().get();
 
@@ -7014,8 +7029,6 @@ class DDTeamCollectionUnitTest {
 		auto servers = resTeam.get()->getServerIDs();
 		const std::set<UID> selectedServers(servers.begin(), servers.end());
 		ASSERT(expectedServers == selectedServers);
-
-		return Void();
 	}
 };
 
@@ -7142,7 +7155,7 @@ TEST_CASE("/DataDistribution/StorageWiggler/NextIdWithMinAge") {
 
 TEST_CASE("/DataDistribution/StorageWiggler/NextIdWithTSS") {
 	state std::unique_ptr<DDTeamCollection> collection =
-	    DDTeamCollectionUnitTest::testMachineTeamCollection(1, Reference<IReplicationPolicy>(new PolicyOne()), 5);
+	    DDTeamCollectionUnitTest::testMachineTeamCollection(1, makeReference<PolicyOne>(), 5);
 	state Reference<StorageWiggler> wiggler = makeReference<StorageWiggler>(collection.get());
 
 	std::cout << "Test when need TSS ... \n";
diff --git a/fdbserver/DDTxnProcessor.actor.cpp b/fdbserver/datadistributor/DDTxnProcessor.cpp
similarity index 74%
rename from fdbserver/DDTxnProcessor.actor.cpp
rename to fdbserver/datadistributor/DDTxnProcessor.cpp
index 3b3f59a7434..deb4aaf43e7 100644
--- a/fdbserver/DDTxnProcessor.actor.cpp
+++ b/fdbserver/datadistributor/DDTxnProcessor.cpp
@@ -1,5 +1,5 @@
 /*
- * DDTxnProcessor.actor.cpp
+ * DDTxnProcessor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,14 @@
  * limitations under the License.
  */
 
-#include "fdbserver/DDTxnProcessor.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/DataDistribution.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
 #include "fdbclient/DatabaseContext.h"
+#include "flow/TxnCounters.h"
+#include "flow/CoroUtils.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 static void updateServersAndCompleteSources(std::set<UID>& servers,
                                             std::vector<UID>& completeSources,
@@ -45,52 +46,55 @@ static void updateServersAndCompleteSources(std::set<UID>& servers,
 class DDTxnProcessorImpl {
 	friend class DDTxnProcessor;
 
-	ACTOR static Future<ServerWorkerInfos> getServerListAndProcessClasses(Database cx) {
-		state Transaction tr(cx);
-		state ServerWorkerInfos res;
-		loop {
+	static Future<ServerWorkerInfos> getServerListAndProcessClasses(Database cx) {
+		Transaction tr(cx);
+		ServerWorkerInfos res;
+		while (true) {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			Error err;
 			try {
-				wait(store(res.servers, NativeAPI::getServerListAndProcessClasses(&tr)));
+				res.servers = co_await NativeAPI::getServerListAndProcessClasses(&tr);
 				res.readVersion = tr.getReadVersion().get();
-				return res;
+				co_return res;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
 	// return {sourceServers, completeSources}
-	ACTOR static Future<IDDTxnProcessor::SourceServers> getSourceServersForRange(Database cx, KeyRangeRef keys) {
-		state std::set<UID> servers;
-		state std::vector<UID> completeSources;
-		state Transaction tr(cx);
+	static Future<IDDTxnProcessor::SourceServers> getSourceServersForRange(Database cx, KeyRangeRef keys) {
+		std::set<UID> servers;
+		std::vector<UID> completeSources;
+		Transaction tr(cx);
 
-		loop {
+		while (true) {
 			servers.clear();
 			completeSources.clear();
 
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			Error err;
 			try {
-				state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-				RangeResult keyServersEntries = wait(tr.getRange(lastLessOrEqual(keyServersKey(keys.begin)),
-				                                                 firstGreaterOrEqual(keyServersKey(keys.end)),
-				                                                 SERVER_KNOBS->DD_QUEUE_MAX_KEY_SERVERS));
+				RangeResult keyServersEntries = co_await tr.getRange(lastLessOrEqual(keyServersKey(keys.begin)),
+				                                                     firstGreaterOrEqual(keyServersKey(keys.end)),
+				                                                     SERVER_KNOBS->DD_QUEUE_MAX_KEY_SERVERS);
 
 				if (keyServersEntries.size() < SERVER_KNOBS->DD_QUEUE_MAX_KEY_SERVERS) {
 					for (int shard = 0; shard < keyServersEntries.size(); shard++) {
 						std::vector<UID> src, dest;
 						decodeKeyServersValue(UIDtoTagMap, keyServersEntries[shard].value, src, dest);
-						ASSERT(src.size());
+						ASSERT(!src.empty());
 						updateServersAndCompleteSources(servers, completeSources, shard, src);
 					}
 
-					ASSERT(servers.size() > 0);
+					ASSERT(!servers.empty());
 				}
 
 				// If the size of keyServerEntries is large, then just assume we are using all storage servers
@@ -98,49 +102,50 @@ class DDTxnProcessorImpl {
 				// When a shard is inflight and DD crashes, some destination servers may have already got the data.
 				// The new DD will treat the destination servers as source servers. So the size can be large.
 				else {
-					RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+					RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
 					for (auto s = serverList.begin(); s != serverList.end(); ++s)
 						servers.insert(decodeServerListValue(s->value).id());
 
-					ASSERT(servers.size() > 0);
+					ASSERT(!servers.empty());
 				}
 
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
-		return IDDTxnProcessor::SourceServers{ std::vector<UID>(servers.begin(), servers.end()), completeSources };
+		co_return IDDTxnProcessor::SourceServers{ std::vector<UID>(servers.begin(), servers.end()), completeSources };
 	}
 
-	ACTOR static Future<std::vector<IDDTxnProcessor::DDRangeLocations>> getSourceServerInterfacesForRange(
-	    Database cx,
-	    KeyRangeRef range) {
-		state std::vector<IDDTxnProcessor::DDRangeLocations> res;
-		state Transaction tr(cx);
+	static Future<std::vector<IDDTxnProcessor::DDRangeLocations>> getSourceServerInterfacesForRange(Database cx,
+	                                                                                                KeyRangeRef range) {
+		std::vector<IDDTxnProcessor::DDRangeLocations> res;
+		Transaction tr(cx);
 		tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-		loop {
+		while (true) {
 			res.clear();
+			Error err;
 			try {
-				state RangeResult shards = wait(krmGetRanges(&tr,
-				                                             keyServersPrefix,
-				                                             range,
-				                                             SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
-				                                             SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT));
+				RangeResult shards = co_await krmGetRanges(&tr,
+				                                           keyServersPrefix,
+				                                           range,
+				                                           SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT,
+				                                           SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT);
 				ASSERT(!shards.empty());
 
-				state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state int i = 0;
+				int i = 0;
 				for (i = 0; i < shards.size() - 1; ++i) {
-					state std::vector<UID> src;
+					std::vector<UID> src;
 					std::vector<UID> dest;
 					UID srcId, destId;
 					decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId);
@@ -149,7 +154,7 @@ class DDTxnProcessorImpl {
 					for (int j = 0; j < src.size(); ++j) {
 						serverListEntries.push_back(tr.get(serverListKeyFor(src[j])));
 					}
-					std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+					std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 					IDDTxnProcessor::DDRangeLocations current(KeyRangeRef(shards[i].key, shards[i + 1].key));
 					for (int j = 0; j < serverListValues.size(); ++j) {
 						if (!serverListValues[j].present()) {
@@ -165,32 +170,38 @@ class DDTxnProcessorImpl {
 				}
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevWarnAlways, "GetSourceServerInterfacesError").errorUnsuppressed(e).detail("Range", range);
-				wait(tr.onError(e));
+				err = e;
+				TraceEvent(SevWarnAlways, "GetSourceServerInterfacesError")
+				    .errorUnsuppressed(err)
+				    .detail("Range", range);
 			}
+			co_await tr.onError(err);
 		}
 
-		return res;
+		co_return res;
 	}
 
 	// set the system key space
-	ACTOR static Future<Void> updateReplicaKeys(Database cx,
-	                                            std::vector<Optional<Key>> primaryDcId,
-	                                            std::vector<Optional<Key>> remoteDcIds,
-	                                            DatabaseConfiguration configuration) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> updateReplicaKeys(Database cx,
+	                                      std::vector<Optional<Key>> primaryDcId,
+	                                      std::vector<Optional<Key>> remoteDcIds,
+	                                      DatabaseConfiguration configuration) {
+		static auto* counters = makeCounters("/dd/updateReplicaKeys");
+		Transaction tr(cx);
+		while (true) {
+			counters->started->increment(1);
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-				RangeResult replicaKeys = wait(tr.getRange(datacenterReplicasKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult replicaKeys = co_await tr.getRange(datacenterReplicasKeys, CLIENT_KNOBS->TOO_MANY);
 
 				for (auto& kv : replicaKeys) {
 					auto dcId = decodeDatacenterReplicasKey(kv.key);
 					auto replicas = decodeDatacenterReplicasValue(kv.value);
-					if ((primaryDcId.size() && primaryDcId.at(0) == dcId) ||
-					    (remoteDcIds.size() && remoteDcIds.at(0) == dcId && configuration.usableRegions > 1)) {
+					if ((!primaryDcId.empty() && primaryDcId.at(0) == dcId) ||
+					    (!remoteDcIds.empty() && remoteDcIds.at(0) == dcId && configuration.usableRegions > 1)) {
 						if (replicas > configuration.storageTeamSize) {
 							tr.set(kv.key, datacenterReplicasValue(configuration.storageTeamSize));
 						}
@@ -199,71 +210,81 @@ class DDTxnProcessorImpl {
 					}
 				}
 
-				wait(tr.commit());
+				co_await tr.commit();
+				counters->committed->increment(1);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				counters->aborted->increment(1);
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<int> tryUpdateReplicasKeyForDc(Database cx, Optional<Key> dcId, int storageTeamSize) {
-		state Transaction tr(cx);
-		loop {
+	static Future<int> tryUpdateReplicasKeyForDc(Database cx, Optional<Key> dcId, int storageTeamSize) {
+		static auto* counters = makeCounters("/dd/tryUpdateReplicasKeyForDc");
+		Transaction tr(cx);
+		while (true) {
+			counters->started->increment(1);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
+			Error err;
 			try {
-				Optional<Value> val = wait(tr.get(datacenterReplicasKeyFor(dcId)));
-				state int oldReplicas = val.present() ? decodeDatacenterReplicasValue(val.get()) : 0;
+				Optional<Value> val = co_await tr.get(datacenterReplicasKeyFor(dcId));
+				int oldReplicas = val.present() ? decodeDatacenterReplicasValue(val.get()) : 0;
 				if (oldReplicas == storageTeamSize) {
-					return oldReplicas;
+					co_return oldReplicas;
 				}
 				if (oldReplicas < storageTeamSize) {
 					tr.set(rebootWhenDurableKey, StringRef());
 				}
 				tr.set(datacenterReplicasKeyFor(dcId), datacenterReplicasValue(storageTeamSize));
-				wait(tr.commit());
+				co_await tr.commit();
+				counters->committed->increment(1);
 
-				return oldReplicas;
+				co_return oldReplicas;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				counters->aborted->increment(1);
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Optional<Key>> getHealthyZone(Database cx, UID distributorId) {
+	static Future<Optional<Key>> getHealthyZone(Database cx, UID distributorId) {
 		// Read healthyZone value which is later used to determine on/off of failure triggered DD.
 		// Occasionally this can be slow to read.  This is due to hotspots on the \xff\x02 shard
 		// due to over-aggressive use of this feature:
 		// https://apple.github.io/foundationdb/transaction-profiler-analyzer.html
 		// All in all, it's better to succeed in starting up with a risk to a
 		// non-default feature (maintenance mode) than to block DD startup indefinitely.
-		state Transaction tr(cx);
-		state int maxRetries = SERVER_KNOBS->DD_HEALTHY_ZONE_READ_RETRY_COUNT;
-		state bool healthyZoneRead = false;
-		state Optional<Value> healthyZoneVal;
-		state int i = 0;
+		Transaction tr(cx);
+		int maxRetries = SERVER_KNOBS->DD_HEALTHY_ZONE_READ_RETRY_COUNT;
+		bool healthyZoneRead = false;
+		Optional<Value> healthyZoneVal;
+		int i = 0;
 		for (; i < maxRetries; i++) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				Optional<Value> _hz = wait(tr.get(healthyZoneKey));
+				Optional<Value> _hz = co_await tr.get(healthyZoneKey);
 				healthyZoneVal = _hz;
 				healthyZoneRead = true;
 				break;
 			} catch (Error& e) {
-				TraceEvent("ReadHealthyZone", distributorId).error(e);
-				wait(tr.onError(e));
+				err = e;
+				TraceEvent("ReadHealthyZone", distributorId).error(err);
 			}
+			co_await tr.onError(err);
 		}
 		if (healthyZoneRead) {
 			if (healthyZoneVal.present()) {
 				auto p = decodeHealthyZoneValue(healthyZoneVal.get());
 				if (p.second > tr.getReadVersion().get() || p.first == ignoreSSFailuresZoneString) {
-					return Optional<Key>(p.first);
+					co_return Optional<Key>(p.first);
 				}
 			}
 		} else {
@@ -276,41 +297,41 @@ class DDTxnProcessorImpl {
 			            " appear to be down. Data distributor/cluster restart can be used to reattempt to read"
 			            " the maintenance mode settings.");
 		}
-		return Optional<Key>();
+		co_return Optional<Key>();
 	}
 
 	// Read keyservers, return unique set of teams
-	ACTOR static Future<Reference<InitialDataDistribution>> getInitialDataDistribution(
-	    Database cx,
-	    UID distributorId,
-	    MoveKeysLock moveKeysLock,
-	    std::vector<Optional<Key>> remoteDcIds,
-	    const DDEnabledState* ddEnabledState,
-	    SkipDDModeCheck skipDDModeCheck) {
-		state Reference<InitialDataDistribution> result = makeReference<InitialDataDistribution>();
-		state Key beginKey = allKeys.begin;
+	static Future<Reference<InitialDataDistribution>> getInitialDataDistribution(Database cx,
+	                                                                             UID distributorId,
+	                                                                             MoveKeysLock moveKeysLock,
+	                                                                             std::vector<Optional<Key>> remoteDcIds,
+	                                                                             const DDEnabledState* ddEnabledState,
+	                                                                             SkipDDModeCheck skipDDModeCheck) {
+		auto result = makeReference<InitialDataDistribution>();
+		Key beginKey = allKeys.begin;
 
-		state bool succeeded;
+		bool succeeded{ false };
 
-		state Transaction tr(cx);
+		Transaction tr(cx);
 
 		if (ddLargeTeamEnabled()) {
-			wait(store(result->userRangeConfig,
-			           DDConfiguration().userRangeConfig().getSnapshot(
-			               SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end)));
+			result->userRangeConfig = co_await DDConfiguration().userRangeConfig().getSnapshot(
+			    SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end);
 		}
-		state std::map<UID, Optional<Key>> server_dc;
-		state std::map<std::vector<UID>, std::pair<std::vector<UID>, std::vector<UID>>> team_cache;
-		state std::vector<std::pair<StorageServerInterface, ProcessClass>> tss_servers;
-		state int numDataMoves = 0;
+		std::map<UID, Optional<Key>> server_dc;
+		std::map<std::vector<UID>, std::pair<std::vector<UID>, std::vector<UID>>> team_cache;
+		std::vector<std::pair<StorageServerInterface, ProcessClass>> tss_servers;
+		int numDataMoves = 0;
 
-		Optional<Key> healthyZone = wait(getHealthyZone(cx, distributorId));
+		Optional<Key> healthyZone = co_await getHealthyZone(cx, distributorId);
 		result->initHealthyZoneValue = healthyZone;
 
 		CODE_PROBE((bool)skipDDModeCheck, "DD Mode won't prevent read initial data distribution.");
 		// Get the server list in its own try/catch block since it modifies result.  We don't want a subsequent failure
 		// causing entries to be duplicated
-		loop {
+		// Phase 1: Single transaction to read server list and all persisted data moves
+		double serverListAndDataMoveReadStart = now();
+		while (true) {
 			numDataMoves = 0;
 			server_dc.clear();
 			result->allServers.clear();
@@ -319,12 +340,13 @@ class DDTxnProcessorImpl {
 			tss_servers.clear();
 			team_cache.clear();
 			succeeded = false;
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				result->mode = 1;
-				Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
+				Optional<Value> mode = co_await tr.get(dataDistributionModeKey);
 				if (mode.present()) {
 					BinaryReader rd(mode.get(), Unversioned());
 					rd >> result->mode;
@@ -332,11 +354,11 @@ class DDTxnProcessorImpl {
 				if ((!skipDDModeCheck && !result->mode) || !ddEnabledState->isEnabled()) {
 					// DD can be disabled persistently (result->mode = 0) or transiently (isEnabled() = 0)
 					TraceEvent(SevDebug, "GetInitialDataDistribution_DisabledDD").log();
-					return result;
+					co_return result;
 				}
 
 				result->bulkLoadMode = 0;
-				Optional<Value> bulkLoadMode = wait(tr.get(bulkLoadModeKey));
+				Optional<Value> bulkLoadMode = co_await tr.get(bulkLoadModeKey);
 				if (bulkLoadMode.present()) {
 					BinaryReader rd(bulkLoadMode.get(), Unversioned());
 					rd >> result->bulkLoadMode;
@@ -344,16 +366,16 @@ class DDTxnProcessorImpl {
 				TraceEvent(SevInfo, "DDBulkLoadEngineInitMode").detail("Mode", result->bulkLoadMode);
 
 				result->bulkDumpMode = 0;
-				Optional<Value> bulkDumpMode = wait(tr.get(bulkDumpModeKey));
+				Optional<Value> bulkDumpMode = co_await tr.get(bulkDumpModeKey);
 				if (bulkDumpMode.present()) {
 					BinaryReader rd(bulkDumpMode.get(), Unversioned());
 					rd >> result->bulkDumpMode;
 				}
 				TraceEvent(SevInfo, "DDBulkDumpInitMode").detail("Mode", result->bulkDumpMode);
 
-				state Future<std::vector<ProcessData>> workers = getWorkers(&tr);
-				state Future<RangeResult> serverList = tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-				wait(success(workers) && success(serverList));
+				Future<std::vector<ProcessData>> workers = getWorkers(&tr);
+				Future<RangeResult> serverList = tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+				co_await (success(workers) && success(serverList));
 				ASSERT(!serverList.get().more && serverList.get().size() < CLIENT_KNOBS->TOO_MANY);
 
 				std::map<Optional<Standalone<StringRef>>, ProcessData> id_data;
@@ -370,7 +392,12 @@ class DDTxnProcessorImpl {
 					}
 				}
 
-				RangeResult dms = wait(tr.getRange(dataMoveKeys, CLIENT_KNOBS->TOO_MANY));
+				double dataMoveReadStart = now();
+				RangeResult dms = co_await tr.getRange(dataMoveKeys, CLIENT_KNOBS->TOO_MANY);
+				if (now() - dataMoveReadStart > 5.0) {
+					TraceEvent(SevWarn, "DDInitSlowDataMoveRead", distributorId)
+					    .detail("ElapsedSeconds", now() - dataMoveReadStart);
+				}
 				ASSERT(!dms.more && dms.size() < CLIENT_KNOBS->TOO_MANY);
 				// For each data move, find out the src or dst servers are in primary or remote DC.
 				for (int i = 0; i < dms.size(); ++i) {
@@ -418,33 +445,44 @@ class DDTxnProcessorImpl {
 
 				succeeded = true;
 
+				TraceEvent("DDInitServerListAndDataMoveReadComplete", distributorId)
+				    .detail("NumDataMoves", numDataMoves)
+				    .detail("NumServers", result->allServers.size())
+				    .detail("ElapsedSeconds", now() - serverListAndDataMoveReadStart);
+
 				break;
 			} catch (Error& e) {
-				TraceEvent("GetInitialTeamsRetry", distributorId).error(e);
-				wait(tr.onError(e));
-
-				ASSERT(!succeeded); // We shouldn't be retrying if we have already started modifying result in this loop
+				err = e;
+				TraceEvent("GetInitialTeamsRetry", distributorId).error(err);
 			}
+			co_await tr.onError(err);
+			ASSERT(!succeeded); // We shouldn't be retrying if we have already started modifying result in this
+			                    // loop
 		}
 
 		// If keyServers is too large to read in a single transaction, then we will have to break this process up into
 		// multiple transactions. In that case, each iteration should begin where the previous left off
+		// Scan keyServers in batches to build the shard map
+		double keyServerScanStart = now();
+		double lastScanLogTime = now();
+		int scanBatchCount = 0;
 		while (beginKey < allKeys.end) {
 			CODE_PROBE(beginKey > allKeys.begin, "Multi-transactional getInitialDataDistribution");
-			loop {
+			while (true) {
 				succeeded = false;
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-					wait(checkMoveKeysLockReadOnly(&tr, moveKeysLock, ddEnabledState));
-					state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					co_await checkMoveKeysLockReadOnly(&tr, moveKeysLock, ddEnabledState);
+					RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-					RangeResult keyServers = wait(krmGetRanges(&tr,
-					                                           keyServersPrefix,
-					                                           KeyRangeRef(beginKey, allKeys.end),
-					                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
-					                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
+					RangeResult keyServers = co_await krmGetRanges(&tr,
+					                                               keyServersPrefix,
+					                                               KeyRangeRef(beginKey, allKeys.end),
+					                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+					                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
 					succeeded = true;
 
 					std::vector<UID> src, dest, last;
@@ -454,7 +492,7 @@ class DDTxnProcessorImpl {
 					for (int i = 0; i < keyServers.size() - 1; i++) {
 						decodeKeyServersValue(UIDtoTagMap, keyServers[i].value, src, dest, srcId, destId);
 						DDShardInfo info(keyServers[i].key, srcId, destId);
-						if (remoteDcIds.size()) {
+						if (!remoteDcIds.empty()) {
 							auto srcIter = team_cache.find(src);
 							if (srcIter == team_cache.end()) {
 								for (auto& id : src) {
@@ -472,7 +510,7 @@ class DDTxnProcessorImpl {
 								info.primarySrc = srcIter->second.first;
 								info.remoteSrc = srcIter->second.second;
 							}
-							if (dest.size()) {
+							if (!dest.empty()) {
 								info.hasDest = true;
 								auto destIter = team_cache.find(dest);
 								if (destIter == team_cache.end()) {
@@ -500,7 +538,7 @@ class DDTxnProcessorImpl {
 								result->primaryTeams.insert(src);
 								team_cache[src] = std::pair<std::vector<UID>, std::vector<UID>>();
 							}
-							if (dest.size()) {
+							if (!dest.empty()) {
 								info.hasDest = true;
 								info.primaryDest = dest;
 								auto destIter = team_cache.find(dest);
@@ -515,14 +553,23 @@ class DDTxnProcessorImpl {
 
 					ASSERT_GT(keyServers.size(), 0);
 					beginKey = keyServers.end()[-1].key;
+					scanBatchCount++;
+					if (now() - lastScanLogTime >= 30.0) {
+						lastScanLogTime = now();
+						TraceEvent("DDInitKeyServerScanProgress", distributorId)
+						    .detail("BeginKey", beginKey)
+						    .detail("Batches", scanBatchCount)
+						    .detail("ShardsScanned", result->shards.size())
+						    .detail("ElapsedSeconds", now() - keyServerScanStart);
+					}
 					break;
 				} catch (Error& e) {
-					TraceEvent("GetInitialTeamsKeyServersRetry", distributorId).error(e);
-
-					wait(tr.onError(e));
-					ASSERT(!succeeded); // We shouldn't be retrying if we have already started modifying result in this
-					                    // loop
+					err = e;
+					TraceEvent("GetInitialTeamsKeyServersRetry", distributorId).error(err);
 				}
+				co_await tr.onError(err);
+				ASSERT(!succeeded); // We shouldn't be retrying if we have already started modifying result in
+				                    // this loop
 			}
 
 			tr.reset();
@@ -531,6 +578,10 @@ class DDTxnProcessorImpl {
 		// a dummy shard at the end with no keys or servers makes life easier for trackInitialShards()
 		result->shards.push_back(DDShardInfo(allKeys.end));
 
+		TraceEvent("DDInitKeyServerScanComplete", distributorId)
+		    .detail("NumShards", result->shards.size())
+		    .detail("ElapsedSeconds", now() - keyServerScanStart);
+
 		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && numDataMoves > 0) {
 			for (int shard = 0; shard < result->shards.size() - 1; ++shard) {
 				const DDShardInfo& iShard = result->shards[shard];
@@ -544,23 +595,25 @@ class DDTxnProcessorImpl {
 			result->allServers.push_back(it);
 		}
 
-		return result;
+		co_return result;
 	}
 
-	ACTOR static Future<Void> waitForDataDistributionEnabled(Database cx, const DDEnabledState* ddEnabledState) {
-		state Transaction tr(cx);
-		loop {
-			wait(delay(SERVER_KNOBS->DD_ENABLED_CHECK_DELAY, TaskPriority::DataDistribution));
+	static Future<Void> waitForDataDistributionEnabled(Database cx, const DDEnabledState* ddEnabledState) {
+		Transaction tr(cx);
+		while (true) {
+			co_await delay(SERVER_KNOBS->DD_ENABLED_CHECK_DELAY, TaskPriority::DataDistribution);
 
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
+			Error err;
+			bool hasErr = false;
 			try {
-				Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
+				Optional<Value> mode = co_await tr.get(dataDistributionModeKey);
 				if (!mode.present() && ddEnabledState->isEnabled()) {
 					TraceEvent("WaitForDDEnabledSucceeded").log();
-					return Void();
+					co_return;
 				}
 				if (mode.present()) {
 					BinaryReader rd(mode.get(), Unversioned());
@@ -571,28 +624,33 @@ class DDTxnProcessorImpl {
 					    .detail("IsDDEnabled", ddEnabledState->isEnabled());
 					if (m && ddEnabledState->isEnabled()) {
 						TraceEvent("WaitForDDEnabledSucceeded").log();
-						return Void();
+						co_return;
 					}
 				}
 
 				tr.reset();
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<bool> isDataDistributionEnabled(Database cx, const DDEnabledState* ddEnabledState) {
-		state Transaction tr(cx);
-		loop {
+	static Future<bool> isDataDistributionEnabled(Database cx, const DDEnabledState* ddEnabledState) {
+		Transaction tr(cx);
+		while (true) {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 
+			Error err;
 			try {
-				Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
+				Optional<Value> mode = co_await tr.get(dataDistributionModeKey);
 				if (!mode.present() && ddEnabledState->isEnabled())
-					return true;
+					co_return true;
 				if (mode.present()) {
 					BinaryReader rd(mode.get(), Unversioned());
 					int m;
@@ -601,11 +659,11 @@ class DDTxnProcessorImpl {
 						TraceEvent(SevDebug, "IsDDEnabledSucceeded")
 						    .detail("Mode", m)
 						    .detail("IsDDEnabled", ddEnabledState->isEnabled());
-						return true;
+						co_return true;
 					}
 				}
 				// SOMEDAY: Write a wrapper in MoveKeys.actor.h
-				Optional<Value> readVal = wait(tr.get(moveKeysLockOwnerKey));
+				Optional<Value> readVal = co_await tr.get(moveKeysLockOwnerKey);
 				UID currentOwner =
 				    readVal.present() ? BinaryReader::fromStringRef<UID>(readVal.get(), Unversioned()) : UID();
 				if (ddEnabledState->isEnabled() && (currentOwner != dataDistributionModeLock)) {
@@ -613,84 +671,96 @@ class DDTxnProcessorImpl {
 					    .detail("CurrentOwner", currentOwner)
 					    .detail("DDModeLock", dataDistributionModeLock)
 					    .detail("IsDDEnabled", ddEnabledState->isEnabled());
-					return true;
+					co_return true;
 				}
 				TraceEvent(SevDebug, "IsDDEnabledFailed")
 				    .detail("CurrentOwner", currentOwner)
 				    .detail("DDModeLock", dataDistributionModeLock)
 				    .detail("IsDDEnabled", ddEnabledState->isEnabled());
-				return false;
+				co_return false;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> pollMoveKeysLock(Database cx, MoveKeysLock lock, const DDEnabledState* ddEnabledState) {
-		loop {
-			wait(delay(SERVER_KNOBS->MOVEKEYS_LOCK_POLLING_DELAY));
-			state Transaction tr(cx);
-			loop {
+	static Future<Void> pollMoveKeysLock(Database cx, MoveKeysLock lock, const DDEnabledState* ddEnabledState) {
+		while (true) {
+			co_await delay(SERVER_KNOBS->MOVEKEYS_LOCK_POLLING_DELAY);
+			Transaction tr(cx);
+			while (true) {
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+				Error err;
 				try {
-					wait(checkMoveKeysLockReadOnly(&tr, lock, ddEnabledState));
+					co_await checkMoveKeysLockReadOnly(&tr, lock, ddEnabledState);
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Optional<Value>> readRebalanceDDIgnoreKey(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Optional<Value>> readRebalanceDDIgnoreKey(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-				Optional<Value> res = wait(tr.get(rebalanceDDIgnoreKey));
-				return res;
+				Optional<Value> res = co_await tr.get(rebalanceDDIgnoreKey);
+				co_return res;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> waitDDTeamInfoPrintSignal(Database cx) {
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+	static Future<Void> waitDDTeamInfoPrintSignal(Database cx) {
+		static auto* counters = makeCounters("/dd/waitDDTeamInfoPrintSignal");
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			counters->started->increment(1);
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				state Future<Void> watchFuture = tr.watch(triggerDDTeamInfoPrintKey);
-				wait(tr.commit());
-				wait(watchFuture);
-				return Void();
+				Future<Void> watchFuture = tr.watch(triggerDDTeamInfoPrintKey);
+				co_await tr.commit();
+				counters->committed->increment(1);
+				co_await watchFuture;
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				counters->aborted->increment(1);
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> waitForAllDataRemoved(
-	    Database cx,
-	    UID serverID,
-	    Version addedVersion,
-	    Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure) {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		loop {
+	static Future<Void> waitForAllDataRemoved(Database cx,
+	                                          UID serverID,
+	                                          Version addedVersion,
+	                                          Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure) {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				Version ver = wait(tr->getReadVersion());
+				Version ver = co_await tr->getReadVersion();
 
 				// we cannot remove a server immediately after adding it, because a perfectly timed cluster recovery
 				// could cause us to not store the mutations sent to the short lived storage server.
 				if (ver > addedVersion + SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) {
-					bool canRemove = wait(canRemoveStorageServer(tr, serverID));
+					bool canRemove = co_await canRemoveStorageServer(tr, serverID);
 					auto shards = shardsAffectedByTeamFailure->getNumberOfShards(serverID);
 					TraceEvent(SevVerbose, "WaitForAllDataRemoved")
 					    .detail("Server", serverID)
@@ -698,15 +768,19 @@ class DDTxnProcessorImpl {
 					    .detail("Shards", shards);
 					ASSERT_GE(shards, 0);
 					if (canRemove && shards == 0) {
-						return Void();
+						co_return;
 					}
 				}
 
 				// Wait for any change to the serverKeys for this server
-				wait(delay(SERVER_KNOBS->ALL_DATA_REMOVED_DELAY, TaskPriority::DataDistribution));
+				co_await delay(SERVER_KNOBS->ALL_DATA_REMOVED_DELAY, TaskPriority::DataDistribution);
 				tr->reset();
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr->onError(err);
 			}
 		}
 	}
@@ -821,11 +895,11 @@ Future<Void> DDTxnProcessor::rawFinishMovement(const MoveKeysParams& params,
 
 struct DDMockTxnProcessorImpl {
 	// return when all status become FETCHED
-	ACTOR static Future<Void> checkFetchingState(DDMockTxnProcessor* self, std::vector<UID> ids, KeyRangeRef range) {
-		state TraceInterval interval("MockCheckFetchingState");
+	static Future<Void> checkFetchingState(DDMockTxnProcessor* self, std::vector<UID> ids, KeyRangeRef range) {
+		TraceInterval interval("MockCheckFetchingState");
 		TraceEvent(SevDebug, interval.begin()).detail("Range", range);
-		loop {
-			wait(delayJittered(1.0, TaskPriority::FetchKeys));
+		while (true) {
+			co_await delayJittered(1.0, TaskPriority::FetchKeys);
 			bool done = true;
 			for (auto& id : ids) {
 				auto& server = self->mgs->allServers.at(id);
@@ -839,7 +913,6 @@ struct DDMockTxnProcessorImpl {
 			}
 		}
 		TraceEvent(SevDebug, interval.end()).log();
-		return Void();
 	}
 
 	static Future<Void> rawCheckFetchingState(DDMockTxnProcessor* self, const MoveKeysParams& params) {
@@ -853,21 +926,20 @@ struct DDMockTxnProcessorImpl {
 		return checkFetchingState(self, params.destinationTeam, params.keys.get());
 	}
 
-	ACTOR static Future<Void> moveKeys(DDMockTxnProcessor* self, MoveKeysParams params) {
-		state std::map<UID, StorageServerInterface> tssMapping;
+	static Future<Void> moveKeys(DDMockTxnProcessor* self, MoveKeysParams params) {
+		std::map<UID, StorageServerInterface> tssMapping;
 		// Because SFBTF::Team requires the ID is ordered
 		std::sort(params.destinationTeam.begin(), params.destinationTeam.end());
 		std::sort(params.healthyDestinations.begin(), params.healthyDestinations.end());
 
-		wait(self->rawStartMovement(params, tssMapping));
+		co_await self->rawStartMovement(params, tssMapping);
 		ASSERT(tssMapping.empty());
 
-		wait(rawCheckFetchingState(self, params));
+		co_await rawCheckFetchingState(self, params);
 
-		wait(self->rawFinishMovement(params, tssMapping));
+		co_await self->rawFinishMovement(params, tssMapping);
 		if (!params.dataMovementComplete.isSet())
 			params.dataMovementComplete.send(Void());
-		return Void();
 	}
 };
 
@@ -951,7 +1023,7 @@ Future<Reference<InitialDataDistribution>> DDMockTxnProcessor::getInitialDataDis
     SkipDDModeCheck skipDDModeCheck) {
 
 	// FIXME: now we just ignore ddEnabledState and moveKeysLock, will fix it in the future
-	Reference<InitialDataDistribution> res = makeReference<InitialDataDistribution>();
+	auto res = makeReference<InitialDataDistribution>();
 	res->mode = 1;
 	res->allServers = getServerListAndProcessClasses().get().servers;
 	res->shards = getDDShardInfos();
@@ -1042,11 +1114,11 @@ Future<std::vector<ProcessData>> DDMockTxnProcessor::getWorkers() const {
 	return Future<std::vector<ProcessData>>();
 }
 
-ACTOR Future<Void> rawStartMovement(std::shared_ptr<MockGlobalState> mgs,
-                                    MoveKeysParams params,
-                                    std::map<UID, StorageServerInterface> tssMapping) {
-	state TraceInterval interval("RelocateShard_MockStartMoveKeys");
-	state KeyRange keys;
+Future<Void> rawStartMovement(std::shared_ptr<MockGlobalState> mgs,
+                              MoveKeysParams params,
+                              std::map<UID, StorageServerInterface> tssMapping) {
+	TraceInterval interval("RelocateShard_MockStartMoveKeys");
+	KeyRange keys;
 	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 		ASSERT(params.ranges.present());
 		// TODO: make startMoveShards work with multiple ranges.
@@ -1057,8 +1129,8 @@ ACTOR Future<Void> rawStartMovement(std::shared_ptr<MockGlobalState> mgs,
 		keys = params.keys.get();
 	}
 	TraceEvent(SevDebug, interval.begin()).detail("Keys", keys);
-	wait(params.startMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser(*params.startMoveKeysParallelismLock);
+	co_await params.startMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser(*params.startMoveKeysParallelismLock);
 
 	std::vector<ShardsAffectedByTeamFailure::Team> destTeams;
 	destTeams.emplace_back(params.destinationTeam, true);
@@ -1093,7 +1165,6 @@ ACTOR Future<Void> rawStartMovement(std::shared_ptr<MockGlobalState> mgs,
 		server->signalFetchKeys(keys, totalRangeSize);
 	}
 	TraceEvent(SevDebug, interval.end());
-	return Void();
 }
 
 Future<Void> DDMockTxnProcessor::rawStartMovement(const MoveKeysParams& params,
@@ -1101,11 +1172,11 @@ Future<Void> DDMockTxnProcessor::rawStartMovement(const MoveKeysParams& params,
 	return ::rawStartMovement(mgs, params, tssMapping);
 }
 
-ACTOR Future<Void> rawFinishMovement(std::shared_ptr<MockGlobalState> mgs,
-                                     MoveKeysParams params,
-                                     std::map<UID, StorageServerInterface> tssMapping) {
-	state TraceInterval interval("RelocateShard_MockFinishMoveKeys");
-	state KeyRange keys;
+Future<Void> rawFinishMovement(std::shared_ptr<MockGlobalState> mgs,
+                               MoveKeysParams params,
+                               std::map<UID, StorageServerInterface> tssMapping) {
+	TraceInterval interval("RelocateShard_MockFinishMoveKeys");
+	KeyRange keys;
 	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 		ASSERT(params.ranges.present());
 		// TODO: make startMoveShards work with multiple ranges.
@@ -1118,8 +1189,8 @@ ACTOR Future<Void> rawFinishMovement(std::shared_ptr<MockGlobalState> mgs,
 
 	TraceEvent(SevDebug, interval.begin()).detail("Keys", keys);
 
-	wait(params.finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch));
-	state FlowLock::Releaser releaser(*params.finishMoveKeysParallelismLock);
+	co_await params.finishMoveKeysParallelismLock->take(TaskPriority::DataDistributionLaunch);
+	FlowLock::Releaser releaser(*params.finishMoveKeysParallelismLock);
 
 	// get source and dest teams
 	auto [destTeams, srcTeams] = mgs->shardMapping->getTeamsForFirstShard(keys);
@@ -1150,7 +1221,6 @@ ACTOR Future<Void> rawFinishMovement(std::shared_ptr<MockGlobalState> mgs,
 	mgs->shardMapping->finishMove(keys);
 	mgs->shardMapping->defineShard(keys); // coalesce for merge
 	TraceEvent(SevDebug, interval.end());
-	return Void();
 }
 
 Future<Void> DDMockTxnProcessor::rawFinishMovement(const MoveKeysParams& params,
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/datadistributor/DataDistribution.cpp
similarity index 74%
rename from fdbserver/DataDistribution.actor.cpp
rename to fdbserver/datadistributor/DataDistribution.cpp
index 001e877a829..68756ebbfa5 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/datadistributor/DataDistribution.cpp
@@ -1,5 +1,5 @@
 /*
- * DataDistribution.actor.cpp
+ * DataDistribution.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,41 +19,43 @@
  */
 
 #include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/BulkDumping.h"
 #include "fdbclient/BulkLoading.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Knobs.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/BulkDumpUtil.actor.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/DDTeamCollection.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/MockDataDistributor.h"
+#include "fdbserver/core/BulkDumpUtil.h"
+#include "fdbserver/core/BulkLoadUtil.h"
+#include "fdbserver/datadistributor/DataDistributor.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/datadistributor/DDTeamCollection.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "DDRelocationQueue.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/core/WorkloadKeys.h"
+#include "fdbserver/datadistributor/MockDataDistributor.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/Buggify.h"
 #include "flow/Error.h"
 #include "flow/Platform.h"
+#include "flow/TxnCounters.h"
 #include "flow/Trace.h"
 #include "flow/UnitTest.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 #include "flow/serialize.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 static const std::string ddServerBulkDumpFolder = "ddBulkDumpFiles";
 static const std::string ddServerBulkLoadFolder = "ddBulkLoadFiles";
@@ -75,31 +77,14 @@ Optional<KeyRange> RelocateShard::getParentRange() const {
 	return parent_range;
 }
 
-ShardSizeBounds ShardSizeBounds::shardSizeBoundsBeforeTrack() {
-	return ShardSizeBounds{ .max = StorageMetrics{ .bytes = -1,
-		                                           .bytesWrittenPerKSecond = StorageMetrics::infinity,
-		                                           .iosPerKSecond = StorageMetrics::infinity,
-		                                           .bytesReadPerKSecond = StorageMetrics::infinity,
-		                                           .opsReadPerKSecond = StorageMetrics::infinity },
-		                    .min = StorageMetrics{ .bytes = -1,
-		                                           .bytesWrittenPerKSecond = 0,
-		                                           .iosPerKSecond = 0,
-		                                           .bytesReadPerKSecond = 0,
-		                                           .opsReadPerKSecond = 0 },
-		                    .permittedError = StorageMetrics{ .bytes = -1,
-		                                                      .bytesWrittenPerKSecond = StorageMetrics::infinity,
-		                                                      .iosPerKSecond = StorageMetrics::infinity,
-		                                                      .bytesReadPerKSecond = StorageMetrics::infinity,
-		                                                      .opsReadPerKSecond = StorageMetrics::infinity } };
-}
-
 namespace {
 
 std::set<int> const& normalDDQueueErrors() {
 	static std::set<int> s{ error_code_movekeys_conflict,
 		                    error_code_broken_promise,
 		                    error_code_data_move_cancelled,
-		                    error_code_data_move_dest_team_not_found };
+		                    error_code_data_move_dest_team_not_found,
+		                    error_code_finish_move_keys_too_many_retries };
 	return s;
 }
 
@@ -113,7 +98,7 @@ enum class DDAuditContext : uint8_t {
 };
 
 struct DDAudit {
-	DDAudit(AuditStorageState coreState)
+	explicit(false) DDAudit(AuditStorageState coreState)
 	  : coreState(coreState), actors(true), foundError(false), auditStorageAnyChildFailed(false), retryCount(0),
 	    cancelled(false), overallCompleteDoAuditCount(0), overallIssuedDoAuditCount(0), overallSkippedDoAuditCount(0),
 	    remainingBudgetForAuditTasks(SERVER_KNOBS->CONCURRENT_AUDIT_TASK_COUNT_MAX), context(DDAuditContext::INVALID) {}
@@ -313,28 +298,28 @@ Future<Void> StorageWiggler::finishWiggle() {
 	    });
 }
 
-ACTOR Future<Void> remoteRecovered(Reference<AsyncVar<ServerDBInfo> const> db) {
+Future<Void> remoteRecovered(Reference<AsyncVar<ServerDBInfo> const> db) {
 	TraceEvent("DDTrackerStarting").log();
 	while (db->get().recoveryState < RecoveryState::ALL_LOGS_RECRUITED) {
 		TraceEvent("DDTrackerStarting").detail("RecoveryState", (int)db->get().recoveryState);
-		wait(db->onChange());
+		co_await db->onChange();
 	}
-	return Void();
 }
 
 // Ensures that the serverKeys key space is properly coalesced
 // This method is only used for testing and is not implemented in a manner that is safe for large databases
-ACTOR Future<Void> debugCheckCoalescing(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<Void> debugCheckCoalescing(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			state RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
-			state int i;
+			int i{ 0 };
 			for (i = 0; i < serverList.size(); i++) {
-				state UID id = decodeServerListValue(serverList[i].value).id();
-				RangeResult ranges = wait(krmGetRanges(&tr, serverKeysPrefixFor(id), allKeys));
+				UID id = decodeServerListValue(serverList[i].value).id();
+				RangeResult ranges = co_await krmGetRanges(&tr, serverKeysPrefixFor(id), allKeys);
 				ASSERT(ranges.end()[-1].key == allKeys.end);
 
 				for (int j = 0; j < ranges.size() - 2; j++)
@@ -346,10 +331,11 @@ ACTOR Future<Void> debugCheckCoalescing(Database cx) {
 			}
 
 			TraceEvent("DoneCheckingCoalescing").log();
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -360,40 +346,34 @@ void runAuditStorage(
     int retryCount,
     DDAuditContext context,
     Optional<std::unordered_set<UID>> serversFinishedSSShardAudit = Optional<std::unordered_set<UID>>());
-ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
-                                    UID auditID,
-                                    AuditType auditType,
-                                    int currentRetryCount);
-ACTOR Future<UID> launchAudit(Reference<DataDistributor> self,
-                              KeyRange auditRange,
-                              AuditType auditType,
-                              KeyValueStoreType auditStorageEngineType);
-ACTOR Future<Void> auditStorage(Reference<DataDistributor> self, TriggerAuditRequest req);
+Future<Void> auditStorageCore(Reference<DataDistributor> self, UID auditID, AuditType auditType, int currentRetryCount);
+Future<UID> launchAudit(Reference<DataDistributor> self,
+                        KeyRange auditRange,
+                        AuditType auditType,
+                        KeyValueStoreType auditStorageEngineType);
+Future<Void> auditStorage(Reference<DataDistributor> self, TriggerAuditRequest req);
+Future<Void> periodicAuditLocationMetadata(Reference<DataDistributor> self);
 void loadAndDispatchAudit(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit);
-ACTOR Future<Void> dispatchAuditStorageServerShard(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit);
-ACTOR Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor> self,
-                                                     std::shared_ptr<DDAudit> audit,
-                                                     StorageServerInterface ssi);
-ACTOR Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit);
-ACTOR Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self,
-                                                 std::shared_ptr<DDAudit> audit,
-                                                 KeyRange range);
-ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
+Future<Void> dispatchAuditStorageServerShard(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit);
+Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor> self,
+                                               std::shared_ptr<DDAudit> audit,
+                                               StorageServerInterface ssi);
+Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit);
+Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self,
                                            std::shared_ptr<DDAudit> audit,
-                                           KeyRange auditRange);
-ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
-                                        std::shared_ptr<DDAudit> audit,
-                                        KeyRange range);
-ACTOR Future<Void> doAuditOnStorageServer(Reference<DataDistributor> self,
-                                          std::shared_ptr<DDAudit> audit,
-                                          StorageServerInterface ssi,
-                                          AuditStorageRequest req);
-ACTOR Future<Void> skipAuditOnRange(Reference<DataDistributor> self,
+                                           KeyRange range);
+Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
+                                     std::shared_ptr<DDAudit> audit,
+                                     KeyRange auditRange);
+Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit, KeyRange range);
+Future<Void> doAuditOnStorageServer(Reference<DataDistributor> self,
                                     std::shared_ptr<DDAudit> audit,
-                                    KeyRange rangeToSkip);
+                                    StorageServerInterface ssi,
+                                    AuditStorageRequest req);
+Future<Void> skipAuditOnRange(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit, KeyRange rangeToSkip);
 
 void runBulkLoadTaskAsync(Reference<DataDistributor> self, KeyRange range, UID taskId, bool restart);
-ACTOR Future<Void> scheduleBulkLoadTasks(Reference<DataDistributor> self);
+Future<Void> scheduleBulkLoadTasks(Reference<DataDistributor> self);
 
 struct DDBulkLoadJobManager {
 	BulkLoadJobState jobState;
@@ -421,7 +401,7 @@ struct DDBulkDumpJobManager {
 	std::map<Key, BulkLoadManifest> taskManifestMap;
 
 	DDBulkDumpJobManager() = default;
-	DDBulkDumpJobManager(const BulkDumpState& jobState) : jobState(jobState) {}
+	explicit DDBulkDumpJobManager(const BulkDumpState& jobState) : jobState(jobState) {}
 
 	bool isValid() const { return jobState.isValid(); }
 };
@@ -534,7 +514,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 		primaryDcId.clear();
 		remoteDcIds.clear();
 		const std::vector<RegionInfo>& regions = configuration.regions;
-		if (configuration.regions.size() > 0) {
+		if (!configuration.regions.empty()) {
 			primaryDcId.push_back(regions[0].dcId);
 		}
 		if (configuration.regions.size() > 1) {
@@ -569,59 +549,86 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 		return;
 	}
 
-	ACTOR static Future<Void> initAuditStorage(Reference<DataDistributor> self) {
+	static Future<Void> initAuditStorage(Reference<DataDistributor> self) {
 		self->auditStorageInitStarted = true;
 		MoveKeyLockInfo lockInfo;
 		lockInfo.myOwner = self->lock.myOwner;
 		lockInfo.prevOwner = self->lock.prevOwner;
 		lockInfo.prevWrite = self->lock.prevWrite;
 		std::vector<AuditStorageState> auditStatesToResume =
-		    wait(initAuditMetadata(self->txnProcessor->context(),
-		                           lockInfo,
-		                           self->context->isDDEnabled(),
-		                           self->ddId,
-		                           SERVER_KNOBS->PERSIST_FINISH_AUDIT_COUNT));
+		    co_await initAuditMetadata(self->txnProcessor->context(),
+		                               lockInfo,
+		                               self->context->isDDEnabled(),
+		                               self->ddId,
+		                               SERVER_KNOBS->PERSIST_FINISH_AUDIT_COUNT);
 		self->resumeAuditStorage(self, auditStatesToResume);
 		self->auditStorageInitialized.send(Void());
-		return Void();
 	}
 
-	ACTOR static Future<Void> waitUntilDataDistributorExitSecurityMode(Reference<DataDistributor> self) {
-		state Transaction tr(self->txnProcessor->context());
-		loop {
-			wait(delay(SERVER_KNOBS->DD_ENABLED_CHECK_DELAY, TaskPriority::DataDistribution));
+	static Future<Void> waitUntilDataDistributorExitSecurityMode(Reference<DataDistributor> self) {
+		Transaction tr(self->txnProcessor->context());
+		while (true) {
+			co_await delay(SERVER_KNOBS->DD_ENABLED_CHECK_DELAY, TaskPriority::DataDistribution);
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			Error err;
 			try {
-				Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
+				Optional<Value> mode = co_await tr.get(dataDistributionModeKey);
 				if (!mode.present()) {
-					return Void();
+					co_return;
 				}
 				BinaryReader rd(mode.get(), Unversioned());
 				int ddMode = 1;
 				rd >> ddMode;
 				if (ddMode != 2) {
-					return Void();
+					co_return;
 				}
-				wait(checkMoveKeysLockReadOnly(&tr, self->context->lock, self->context->ddEnabledState.get()));
+				co_await checkMoveKeysLockReadOnly(&tr, self->context->lock, self->context->ddEnabledState.get());
 				tr.reset();
+				continue;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
 	// Initialize the required internal states of DataDistributor from system metadata. It's necessary before
 	// DataDistributor start working. Doesn't include initialization of optional components, like DDQueue,
 	// Tracker, TeamCollection. The components should call its own ::init methods.
-	ACTOR static Future<Void> init(Reference<DataDistributor> self) {
-		loop {
-			wait(self->waitDataDistributorEnabled());
+	//
+	// DD Startup Progress (trace events in order):
+	//   DDInitRunning                     - DD process recruited and starting init
+	//   DDInitTakingMoveKeysLock          - Acquiring move keys lock
+	//   DDInitTookMoveKeysLock            - Lock acquired
+	//   DDInitGotConfiguration            - Database configuration loaded
+	//   DDInitUpdatedReplicaKeys          - Replica keys updated
+	//   DDInitSlowDataMoveRead            - (SevWarn) dataMoveKeys read taking >5s
+	//   DDInitServerListAndDataMoveReadComplete - Server list + data moves read: NumDataMoves, NumServers,
+	//   ElapsedSeconds
+	//   DDInitKeyServerScanProgress       - (every 30s) keyServer scan: BeginKey, Batches, ShardsScanned
+	//   DDInitKeyServerScanComplete       - keyServer scan done: NumShards, ElapsedSeconds
+	//   DDInitGotInitialDD                - Init data loaded: NumShards, NumServers
+	//   DDInitDataLoaded                  - Init data loaded, ElapsedSeconds (does NOT mean DD is fully operational)
+	//
+	// After init(), the following startup events fire from other components:
+	//   DDInitResumeDataMovesProgress     - (every 30s) data move resume: ValidMoves, CancelledMoves, EmptyMoves
+	//   DDInitResumedDataMoves            - Data move resume complete with counts
+	//   TrackInitialShards                - Shard tracker setup started with InitialShardCount
+	//   TrackInitialShardsComplete        - Shard trackers created: ShardsTracked
+	//   DDTrackerStarting                 - Teams ready (fires from DDTeamCollection after readyToStart + delay)
+	//   TrackInitialShardsMetricsComplete - All shard metrics received: ElapsedSeconds
+	//                                       WaitStorageMetricsHandleError may fire (SevWarn after 60s) if a
+	//                                       shard's metrics read is stuck retrying: Keys, Retries
+	//   DDInitDone                        - DD is fully operational with all shard sizes loaded
+	static Future<Void> init(Reference<DataDistributor> self) {
+		while (true) {
+			co_await self->waitDataDistributorEnabled();
 			TraceEvent("DataDistributionEnabled").log();
 
 			TraceEvent("DDInitTakingMoveKeysLock", self->ddId).log();
-			wait(self->takeMoveKeysLock());
+			co_await self->takeMoveKeysLock();
 			TraceEvent("DDInitTookMoveKeysLock", self->ddId).log();
 
 			// AuditStorage does not rely on DatabaseConfiguration
@@ -639,7 +646,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			// So, we need to check MoveKeyLock when waitUntilDataDistributorExitSecurityMode
 			if (!self->txnProcessor->isMocked()) {
 				// AuditStorage currently does not support DDMockTxnProcessor
-				wait(waitUntilDataDistributorExitSecurityMode(self)); // Trap DDMode == 2
+				co_await waitUntilDataDistributorExitSecurityMode(self); // Trap DDMode == 2
 			}
 			// It is possible DDMode begins with 2 and passes
 			// waitDataDistributorEnabled and then set to 0 before
@@ -649,7 +656,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			// waitDataDistributorEnabled in the next iteration.
 			TraceEvent("DataDistributorExitSecurityMode").log();
 
-			wait(self->loadDatabaseConfiguration());
+			co_await self->loadDatabaseConfiguration();
 			self->initDcInfo();
 			TraceEvent("DDInitGotConfiguration", self->ddId)
 			    .setMaxFieldLength(-1)
@@ -662,10 +669,10 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				throw internal_error();
 			}
 
-			wait(self->updateReplicaKeys());
+			co_await self->updateReplicaKeys();
 			TraceEvent("DDInitUpdatedReplicaKeys", self->ddId).log();
 
-			wait(self->loadInitialDataDistribution());
+			co_await self->loadInitialDataDistribution();
 
 			if (self->initData->shards.size() > 1) {
 				TraceEvent("DDInitGotInitialDD", self->ddId)
@@ -673,6 +680,8 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				    .detail("E", self->initData->shards.end()[-1].key)
 				    .detail("Src", describe(self->initData->shards.end()[-2].primarySrc))
 				    .detail("Dest", describe(self->initData->shards.end()[-2].primaryDest))
+				    .detail("NumShards", self->initData->shards.size())
+				    .detail("NumServers", self->initData->allServers.size())
 				    .trackLatest(self->initialDDEventHolder->trackingKey);
 			} else {
 				TraceEvent("DDInitGotInitialDD", self->ddId)
@@ -680,6 +689,8 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				    .detail("E", "")
 				    .detail("Src", "[no items]")
 				    .detail("Dest", "[no items]")
+				    .detail("NumShards", self->initData->shards.size())
+				    .detail("NumServers", self->initData->allServers.size())
 				    .trackLatest(self->initialDDEventHolder->trackingKey);
 			}
 
@@ -728,15 +739,17 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			    .detail("HighestPriority", self->configuration.usableRegions > 1 ? 0 : -1)
 			    .trackLatest(self->totalDataInFlightRemoteEventHolder->trackingKey);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> removeDataMoveTombstoneBackground(Reference<DataDistributor> self) {
-		state UID currentID;
+	static Future<Void> removeDataMoveTombstoneBackground(Reference<DataDistributor> self) {
+		static auto* counters = makeCounters("/dd/removeDataMoveTombstone");
+		UID currentID;
 		try {
-			state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
-			state Transaction tr(cx);
-			loop {
+			Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
+			Transaction tr(cx);
+			while (true) {
+				counters->started->increment(1);
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -745,11 +758,14 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 						tr.clear(dataMoveKeyFor(currentID));
 						TraceEvent(SevDebug, "RemoveDataMoveTombstone", self->ddId).detail("DataMoveID", currentID);
 					}
-					wait(tr.commit());
+					co_await tr.commit();
+					counters->committed->increment(1);
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					counters->aborted->increment(1);
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
@@ -761,10 +777,9 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			// DD needs not restart when removing tombstone gets failed unless this actor gets cancelled
 			// So, do not throw error
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> resumeFromShards(Reference<DataDistributor> self, bool traceShard) {
+	static Future<Void> resumeFromShards(Reference<DataDistributor> self, bool traceShard) {
 		// All physicalShard init must be completed before issuing data move
 		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
 			for (int i = 0; i < self->initData->shards.size() - 1; i++) {
@@ -779,7 +794,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			}
 		}
 
-		state std::vector<Key> customBoundaries;
+		std::vector<Key> customBoundaries;
 		if (bulkLoadIsEnabled(self->initData->bulkLoadMode)) {
 			// Bulk load does not allow boundary change
 			TraceEvent(SevInfo, "DDInitCustomRangeConfigDisabledByBulkLoadMode", self->ddId);
@@ -793,9 +808,9 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 			}
 		}
 
-		state int shard = 0;
-		state int customBoundary = 0;
-		state int overreplicatedCount = 0;
+		int shard = 0;
+		int customBoundary = 0;
+		int overreplicatedCount = 0;
 		for (; shard < self->initData->shards.size() - 1; shard++) {
 			const DDShardInfo& iShard = self->initData->shards[shard];
 			std::vector<KeyRangeRef> ranges;
@@ -868,22 +883,27 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				}
 			}
 
-			wait(yield(TaskPriority::DataDistribution));
+			co_await yield(TaskPriority::DataDistribution);
 		}
-		return Void();
 	}
 
 	// TODO: unit test needed
-	ACTOR static Future<Void> resumeFromDataMoves(Reference<DataDistributor> self, Future<Void> readyToStart) {
-		state KeyRangeMap<std::shared_ptr<DataMove>>::iterator it = self->initData->dataMoveMap.ranges().begin();
+	static Future<Void> resumeFromDataMoves(Reference<DataDistributor> self, Future<Void> readyToStart) {
+		KeyRangeMap<std::shared_ptr<DataMove>>::iterator it = self->initData->dataMoveMap.ranges().begin();
+		int validMoves = 0;
+		int cancelledMoves = 0;
+		int emptyMoves = 0;
+		double resumeStart = now();
+		double lastLogTime = now();
 
-		wait(readyToStart);
+		co_await readyToStart;
 
 		for (; it != self->initData->dataMoveMap.ranges().end(); ++it) {
 			const DataMoveMetaData& meta = it.value()->meta;
 			DataMoveType dataMoveType = getDataMoveTypeFromDataMoveId(meta.id);
 			if (meta.ranges.empty()) {
 				TraceEvent(SevInfo, "EmptyDataMoveRange", self->ddId).detail("DataMoveMetaData", meta.toString());
+				emptyMoves++;
 				continue;
 			}
 			if (meta.bulkLoadTaskState.present()) {
@@ -896,6 +916,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				TraceEvent(SevWarnAlways, "DDBulkLoadTaskCancelDataMove", self->ddId)
 				    .detail("Reason", "DDInit")
 				    .detail("DataMove", meta.toString());
+				cancelledMoves++;
 			} else if (dataMoveType == DataMoveType::LOGICAL_BULKLOAD ||
 			           dataMoveType == DataMoveType::PHYSICAL_BULKLOAD) {
 				// The metadata is from the old system
@@ -907,6 +928,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				    .detail("Reason", "WrongTypeWhenDDInit")
 				    .detail("DataMoveType", dataMoveType)
 				    .detail("DataMove", meta.toString());
+				cancelledMoves++;
 			} else if (it.value()->isCancelled() ||
 			           (it.value()->valid && !SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA)) {
 				RelocateShard rs(meta.ranges.front(), DataMovementReason::RECOVER_MOVE, RelocateReason::OTHER);
@@ -914,6 +936,7 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				rs.cancelled = true;
 				self->relocationProducer.send(rs);
 				TraceEvent("DDInitScheduledCancelDataMove", self->ddId).detail("DataMove", meta.toString());
+				cancelledMoves++;
 			} else if (it.value()->valid) {
 				TraceEvent(SevDebug, "DDInitFoundDataMove", self->ddId).detail("DataMove", meta.toString());
 				ASSERT(meta.ranges.front() == it.range());
@@ -936,15 +959,29 @@ struct DataDistributor : NonCopyable, ReferenceCounted<DataDistributor> {
 				// the data move now, so that team failures can be captured.
 				self->shardsAffectedByTeamFailure->moveShard(rs.keys, teams);
 				self->relocationProducer.send(rs);
-				wait(yield(TaskPriority::DataDistribution));
+				co_await yield(TaskPriority::DataDistribution);
+				validMoves++;
+			}
+			if (now() - lastLogTime >= 30.0) {
+				lastLogTime = now();
+				TraceEvent("DDInitResumeDataMovesProgress", self->ddId)
+				    .detail("ValidMoves", validMoves)
+				    .detail("CancelledMoves", cancelledMoves)
+				    .detail("EmptyMoves", emptyMoves)
+				    .detail("ElapsedSeconds", now() - resumeStart);
 			}
 		}
 
+		TraceEvent("DDInitResumedDataMoves", self->ddId)
+		    .detail("ValidMoves", validMoves)
+		    .detail("CancelledMoves", cancelledMoves)
+		    .detail("EmptyMoves", emptyMoves)
+		    .detail("ElapsedSeconds", now() - resumeStart);
+
 		// Trigger background cleanup for datamove tombstones
 		if (!self->txnProcessor->isMocked()) {
 			self->addActor.send(self->removeDataMoveTombstoneBackground(self));
 		}
-		return Void();
 	}
 
 	// Resume inflight relocations from the previous DD
@@ -1044,86 +1081,82 @@ inline std::unordered_map<UID, std::shared_ptr<DDAudit>> getAuditsForType(Refere
 }
 
 // Periodically check and log the physicalShard status; clean up empty physicalShard;
-ACTOR Future<Void> monitorPhysicalShardStatus(Reference<PhysicalShardCollection> self) {
+Future<Void> monitorPhysicalShardStatus(Reference<PhysicalShardCollection> self) {
 	ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 	ASSERT(SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD);
-	loop {
+	while (true) {
 		self->cleanUpPhysicalShardCollection();
 		self->logPhysicalShardCollection();
-		wait(delay(SERVER_KNOBS->PHYSICAL_SHARD_METRICS_DELAY));
+		co_await delay(SERVER_KNOBS->PHYSICAL_SHARD_METRICS_DELAY);
 	}
 }
 
 // Trigger a task on range based on the current bulk load task metadata
-ACTOR Future<std::pair<BulkLoadTaskState, Version>> triggerBulkLoadTask(Reference<DataDistributor> self,
-                                                                        KeyRange taskRange,
-                                                                        UID taskId) {
-	loop {
+Future<std::pair<BulkLoadTaskState, Version>> triggerBulkLoadTask(Reference<DataDistributor> self,
+                                                                  KeyRange taskRange,
+                                                                  UID taskId) {
+	while (true) {
 		Database cx = self->txnProcessor->context();
-		state Transaction tr(cx);
-		state BulkLoadTaskState newBulkLoadTaskState;
+		Transaction tr(cx);
+		BulkLoadTaskState newBulkLoadTaskState;
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			wait(checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get()));
+			co_await checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get());
 			std::vector<BulkLoadPhase> phase;
-			wait(
-			    store(newBulkLoadTaskState,
-			          getBulkLoadTask(&tr,
-			                          taskRange,
-			                          taskId,
-			                          { BulkLoadPhase::Submitted, BulkLoadPhase::Triggered, BulkLoadPhase::Running })));
+			newBulkLoadTaskState = co_await getBulkLoadTask(
+			    &tr, taskRange, taskId, { BulkLoadPhase::Submitted, BulkLoadPhase::Triggered, BulkLoadPhase::Running });
 			newBulkLoadTaskState.phase = BulkLoadPhase::Triggered;
 			newBulkLoadTaskState.clearDataMoveId();
 			newBulkLoadTaskState.restartCount = newBulkLoadTaskState.restartCount + 1;
 			newBulkLoadTaskState.triggerTime = now();
-			wait(krmSetRange(&tr,
-			                 bulkLoadTaskPrefix,
-			                 newBulkLoadTaskState.getRange(),
-			                 bulkLoadTaskStateValue(newBulkLoadTaskState)));
-			wait(tr.commit());
+			co_await krmSetRange(
+			    &tr, bulkLoadTaskPrefix, newBulkLoadTaskState.getRange(), bulkLoadTaskStateValue(newBulkLoadTaskState));
+			co_await tr.commit();
 			Version commitVersion = tr.getCommittedVersion();
 			TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskPersistTriggerState", self->ddId)
 			    .detail("CommitVersion", commitVersion)
 			    .detail("TaskID", newBulkLoadTaskState.getTaskId())
 			    .detail("JobID", newBulkLoadTaskState.getJobId());
 			ASSERT(commitVersion != invalidVersion);
-			return std::make_pair(newBulkLoadTaskState, commitVersion);
-
+			co_return std::make_pair(newBulkLoadTaskState, commitVersion);
 		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevWarn, "DDBulkLoadTaskPersistTriggerStateError", self->ddId)
-				    .errorUnsuppressed(e)
-				    .detail("TaskID", newBulkLoadTaskState.getTaskId())
-				    .detail("JobID", newBulkLoadTaskState.getJobId());
-			}
-			wait(tr.onError(e));
+			err = e;
 		}
+		if (err.code() != error_code_actor_cancelled) {
+			TraceEvent(SevWarn, "DDBulkLoadTaskPersistTriggerStateError", self->ddId)
+			    .errorUnsuppressed(err)
+			    .detail("TaskID", newBulkLoadTaskState.getTaskId())
+			    .detail("JobID", newBulkLoadTaskState.getJobId());
+		}
+		co_await tr.onError(err);
 	}
 }
 
 // TODO(BulkLoad): add reason to persist
-ACTOR Future<Void> failBulkLoadTask(Reference<DataDistributor> self,
-                                    KeyRange taskRange,
-                                    UID taskId,
-                                    int cancelledDataMovePriority) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
-	state BulkLoadTaskState bulkLoadTaskState;
-	loop {
+Future<Void> failBulkLoadTask(Reference<DataDistributor> self,
+                              KeyRange taskRange,
+                              UID taskId,
+                              int cancelledDataMovePriority) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+	BulkLoadTaskState bulkLoadTaskState;
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			wait(checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get()));
-			wait(store(bulkLoadTaskState,
-			           getBulkLoadTask(&tr, taskRange, taskId, { BulkLoadPhase::Triggered, BulkLoadPhase::Running })));
+			co_await checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get());
+			bulkLoadTaskState =
+			    co_await getBulkLoadTask(&tr, taskRange, taskId, { BulkLoadPhase::Triggered, BulkLoadPhase::Running });
 			bulkLoadTaskState.phase = BulkLoadPhase::Error;
 			bulkLoadTaskState.setCancelledDataMovePriority(cancelledDataMovePriority);
 			ASSERT(taskRange == bulkLoadTaskState.getRange() && taskId == bulkLoadTaskState.getTaskId());
 			ASSERT(normalKeys.contains(taskRange));
-			wait(krmSetRange(
-			    &tr, bulkLoadTaskPrefix, bulkLoadTaskState.getRange(), bulkLoadTaskStateValue(bulkLoadTaskState)));
-			wait(tr.commit());
+			co_await krmSetRange(
+			    &tr, bulkLoadTaskPrefix, bulkLoadTaskState.getRange(), bulkLoadTaskStateValue(bulkLoadTaskState));
+			co_await tr.commit();
 			Version commitVersion = tr.getCommittedVersion();
 			TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskPersistErrorState", self->ddId)
 			    .detail("CommitVersion", commitVersion)
@@ -1131,29 +1164,30 @@ ACTOR Future<Void> failBulkLoadTask(Reference<DataDistributor> self,
 			    .detail("JobID", bulkLoadTaskState.getJobId());
 			break;
 		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevWarn, "DDBulkLoadTaskPersistErrorStateError", self->ddId)
-				    .errorUnsuppressed(e)
-				    .detail("TaskID", bulkLoadTaskState.getTaskId())
-				    .detail("JobID", bulkLoadTaskState.getJobId());
-			}
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.code() != error_code_actor_cancelled) {
+			TraceEvent(SevWarn, "DDBulkLoadTaskPersistErrorStateError", self->ddId)
+			    .errorUnsuppressed(err)
+			    .detail("TaskID", bulkLoadTaskState.getTaskId())
+			    .detail("JobID", bulkLoadTaskState.getJobId());
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
 // A bulk load task is guaranteed to be either complete or overwritten by another task
-ACTOR Future<Void> doBulkLoadTask(Reference<DataDistributor> self, KeyRange range, UID taskId) {
-	state Promise<BulkLoadAck> completeAck;
-	state BulkLoadTaskState triggeredBulkLoadTask;
-	state Version commitVersion = invalidVersion;
-	state double beginTime = now();
+Future<Void> doBulkLoadTask(Reference<DataDistributor> self, KeyRange range, UID taskId) {
+	Promise<BulkLoadAck> completeAck;
+	BulkLoadTaskState triggeredBulkLoadTask;
+	Version commitVersion = invalidVersion;
+	double beginTime = now();
 	ASSERT(self->bulkLoadEngineParallelismLimitor.canStart());
 	self->bulkLoadEngineParallelismLimitor.incrementTaskCounter();
 	try {
 		// Step 1: persist bulk load task phase as triggered
-		std::pair<BulkLoadTaskState, Version> triggeredBulkLoadTask_ = wait(triggerBulkLoadTask(self, range, taskId));
+		std::pair<BulkLoadTaskState, Version> triggeredBulkLoadTask_ =
+		    co_await triggerBulkLoadTask(self, range, taskId);
 		triggeredBulkLoadTask = triggeredBulkLoadTask_.first;
 		commitVersion = triggeredBulkLoadTask_.second;
 		TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskDoTask", self->ddId)
@@ -1175,7 +1209,7 @@ ACTOR Future<Void> doBulkLoadTask(Reference<DataDistributor> self, KeyRange rang
 		// The completion of the task relies on the fact that a data move on a range is either
 		// completed by itself or replaced by a data move on the overlapping range
 		self->triggerShardBulkLoading.send(BulkLoadShardRequest(triggeredBulkLoadTask));
-		state BulkLoadAck ack = wait(completeAck.getFuture()); // proceed when a data move completes with this task
+		BulkLoadAck ack = co_await completeAck.getFuture(); // proceed when a data move completes with this task
 		if (ack.unretryableError) {
 			TraceEvent(SevWarnAlways, "DDBulkLoadTaskDoTask", self->ddId)
 			    .detail("Phase", "See unretryable error")
@@ -1185,7 +1219,7 @@ ACTOR Future<Void> doBulkLoadTask(Reference<DataDistributor> self, KeyRange rang
 			    .detail("Duration", now() - beginTime);
 			try {
 				// Mark this task failed in system metadata
-				wait(failBulkLoadTask(self, range, taskId, ack.dataMovePriority));
+				co_await failBulkLoadTask(self, range, taskId, ack.dataMovePriority);
 				TraceEvent(SevWarnAlways, "DDBulkLoadTaskDoTask", self->ddId)
 				    .detail("Phase", "Marked as error")
 				    .detail("CancelledDataMovePriority", ack.dataMovePriority)
@@ -1232,21 +1266,21 @@ ACTOR Future<Void> doBulkLoadTask(Reference<DataDistributor> self, KeyRange rang
 		// sliently exits
 	}
 	self->bulkLoadEngineParallelismLimitor.decrementTaskCounter();
-	return Void();
 }
 
-ACTOR Future<Void> eraseBulkLoadTask(Reference<DataDistributor> self, KeyRange taskRange, UID taskId) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
-	state BulkLoadTaskState bulkLoadTask;
-	loop {
+Future<Void> eraseBulkLoadTask(Reference<DataDistributor> self, KeyRange taskRange, UID taskId) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+	BulkLoadTaskState bulkLoadTask;
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			wait(store(bulkLoadTask, getBulkLoadTask(&tr, taskRange, taskId, { BulkLoadPhase::Acknowledged })));
-			wait(krmSetRangeCoalescing(
-			    &tr, bulkLoadTaskPrefix, taskRange, normalKeys, bulkLoadTaskStateValue(BulkLoadTaskState())));
-			wait(tr.commit());
+			bulkLoadTask = co_await getBulkLoadTask(&tr, taskRange, taskId, { BulkLoadPhase::Acknowledged });
+			co_await krmSetRangeCoalescing(
+			    &tr, bulkLoadTaskPrefix, taskRange, normalKeys, bulkLoadTaskStateValue(BulkLoadTaskState()));
+			co_await tr.commit();
 			Version commitVersion = tr.getCommittedVersion();
 			TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskEraseState", self->ddId)
 			    .detail("CommitVersion", commitVersion)
@@ -1266,65 +1300,66 @@ ACTOR Future<Void> eraseBulkLoadTask(Reference<DataDistributor> self, KeyRange t
 			}
 			break;
 		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevWarn, "DDBulkLoadTaskEraseStateError", self->ddId)
-				    .errorUnsuppressed(e)
-				    .detail("TaskRange", taskRange)
-				    .detail("TaskID", taskId);
-			}
-			if (e.code() == error_code_bulkload_task_outdated) {
-				// Silently exit
-				break;
-			}
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.code() != error_code_actor_cancelled) {
+			TraceEvent(SevWarn, "DDBulkLoadTaskEraseStateError", self->ddId)
+			    .errorUnsuppressed(err)
+			    .detail("TaskRange", taskRange)
+			    .detail("TaskID", taskId);
 		}
+		if (err.code() == error_code_bulkload_task_outdated) {
+			// Silently exit
+			break;
+		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> scheduleBulkLoadTasks(Reference<DataDistributor> self) {
-	state Key beginKey = normalKeys.begin;
-	state Key endKey = normalKeys.end;
-	state KeyRange rangeToRead;
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
-	state int i = 0;
-	state BulkLoadTaskState bulkLoadTaskState;
-	state RangeResult result;
-	state std::vector<Future<Void>> bulkLoadActors;
-	state KeyRange range;
+Future<Void> scheduleBulkLoadTasks(Reference<DataDistributor> self) {
+	Key beginKey = normalKeys.begin;
+	Key endKey = normalKeys.end;
+	KeyRange rangeToRead;
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+	BulkLoadTaskState bulkLoadTaskState;
+	RangeResult result;
+	std::vector<Future<Void>> bulkLoadActors;
+	KeyRange range;
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
 			result.clear();
-			wait(store(
-			    result,
-			    krmGetRanges(&tr, bulkLoadTaskPrefix, rangeToRead, SERVER_KNOBS->DD_BULKLOAD_TASK_METADATA_READ_SIZE)));
-			i = 0;
-			for (; i < result.size() - 1; i++) {
+			result = co_await krmGetRanges(
+			    &tr, bulkLoadTaskPrefix, rangeToRead, SERVER_KNOBS->DD_BULKLOAD_TASK_METADATA_READ_SIZE);
+			if (result.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(result.size()) - 1; i++) {
 				if (result[i].value.empty()) {
 					continue;
 				}
 				range = Standalone(KeyRangeRef(result[i].key, result[i + 1].key));
 				bulkLoadTaskState = decodeBulkLoadTaskState(result[i].value);
 				if (!bulkLoadTaskState.isValid()) {
-					wait(delay(0.1));
+					co_await delay(0.1);
 					continue;
 				}
 				if (range != bulkLoadTaskState.getRange()) {
 					// This task is outdated
-					wait(delay(0.1));
+					co_await delay(0.1);
 					continue;
 				}
 				if (bulkLoadTaskState.phase == BulkLoadPhase::Submitted ||
 				    bulkLoadTaskState.phase == BulkLoadPhase::Triggered ||
 				    bulkLoadTaskState.phase == BulkLoadPhase::Running) {
 					// Limit parallelism
-					loop {
+					while (true) {
 						if (self->bulkLoadEngineParallelismLimitor.canStart()) {
 							break;
 						}
-						wait(self->bulkLoadEngineParallelismLimitor.waitUntilCounterChanged());
+						co_await self->bulkLoadEngineParallelismLimitor.waitUntilCounterChanged();
 					}
 					TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadTaskSchedule", self->ddId)
 					    .detail("Range", bulkLoadTaskState.getRange())
@@ -1337,7 +1372,7 @@ ACTOR Future<Void> scheduleBulkLoadTasks(Reference<DataDistributor> self) {
 					    .detail("Range", bulkLoadTaskState.getRange())
 					    .detail("TaskID", bulkLoadTaskState.getTaskId());
 					// We do one metadata erase at a time to aviod unnecessary transaction conflicts
-					wait(eraseBulkLoadTask(self, bulkLoadTaskState.getRange(), bulkLoadTaskState.getTaskId()));
+					co_await eraseBulkLoadTask(self, bulkLoadTaskState.getRange(), bulkLoadTaskState.getTaskId());
 				} else if (bulkLoadTaskState.phase == BulkLoadPhase::Error) {
 					TraceEvent(SevWarnAlways, "DDBulkLoadTaskUnretriableError", self->ddId)
 					    .detail("Range", bulkLoadTaskState.getRange())
@@ -1347,25 +1382,26 @@ ACTOR Future<Void> scheduleBulkLoadTasks(Reference<DataDistributor> self) {
 				}
 			}
 			beginKey = result.back().key;
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
 		}
+		co_await tr.onError(err);
 	}
-	wait(waitForAll(bulkLoadActors));
-	return Void();
+	co_await waitForAll(bulkLoadActors);
 }
 
-ACTOR Future<Void> bulkLoadTaskCore(Reference<DataDistributor> self, Future<Void> readyToStart) {
-	wait(readyToStart);
-	state Database cx = self->txnProcessor->context();
-	wait(registerRangeLockOwner(cx, rangeLockNameForBulkLoad, rangeLockNameForBulkLoad));
+Future<Void> bulkLoadTaskCore(Reference<DataDistributor> self, Future<Void> readyToStart) {
+	co_await readyToStart;
+	Database cx = self->txnProcessor->context();
+	co_await registerRangeLockOwner(cx, rangeLockNameForBulkLoad, rangeLockNameForBulkLoad);
 	TraceEvent(SevInfo, "DDBulkLoadTaskCoreStart", self->ddId);
-	loop {
+	while (true) {
 		try {
-			wait(scheduleBulkLoadTasks(self));
+			co_await scheduleBulkLoadTasks(self);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
@@ -1375,7 +1411,7 @@ ACTOR Future<Void> bulkLoadTaskCore(Reference<DataDistributor> self, Future<Void
 				throw e;
 			}
 		}
-		wait(delay(SERVER_KNOBS->DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC));
+		co_await delay(SERVER_KNOBS->DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC);
 	}
 }
 
@@ -1395,11 +1431,11 @@ void explainBulkLoadJobGetRangeResult(const RangeResult& rangeResult) {
 
 // Return the current bulk load job state for the given jobId and jobRange
 // If the job is not found or outdated, throw bulkload_task_outdated error
-ACTOR Future<BulkLoadJobState> getBulkLoadJob(Transaction* tr, UID jobId, KeyRange jobRange) {
-	state RangeResult rangeResult;
-	state std::string errorMessage;
+Future<BulkLoadJobState> getBulkLoadJob(Transaction* tr, UID jobId, KeyRange jobRange) {
+	RangeResult rangeResult;
+	std::string errorMessage;
 	try {
-		wait(store(rangeResult, krmGetRanges(tr, bulkLoadJobPrefix, jobRange)));
+		rangeResult = co_await krmGetRanges(tr, bulkLoadJobPrefix, jobRange);
 		if (rangeResult.size() != 2 && rangeResult.size() != 3) {
 			// In the middle of finalizing a bulk load job, the size is 3
 			errorMessage = fmt::format("Expected 2 or 3 results for bulk load job {} in range {}, but got {} results",
@@ -1424,7 +1460,7 @@ ACTOR Future<BulkLoadJobState> getBulkLoadJob(Transaction* tr, UID jobId, KeyRan
 			                           jobId.toString());
 			throw bulkload_task_outdated();
 		}
-		return currentJobState;
+		co_return currentJobState;
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
 			throw e;
@@ -1438,26 +1474,27 @@ ACTOR Future<BulkLoadJobState> getBulkLoadJob(Transaction* tr, UID jobId, KeyRan
 }
 
 // Find task metadata for a bulk load job with jobId and input range
-ACTOR Future<Optional<BulkLoadTaskState>> bulkLoadJobFindTask(Reference<DataDistributor> self,
-                                                              KeyRange range,
-                                                              UID jobId,
-                                                              KeyRange jobRange,
-                                                              UID logId) {
-	state BulkLoadTaskState bulkLoadTaskState;
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
-	loop {
+Future<Optional<BulkLoadTaskState>> bulkLoadJobFindTask(Reference<DataDistributor> self,
+                                                        KeyRange range,
+                                                        UID jobId,
+                                                        KeyRange jobRange,
+                                                        UID logId) {
+	BulkLoadTaskState bulkLoadTaskState;
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			BulkLoadJobState checkJobState =
-			    wait(getBulkLoadJob(&tr, jobId, jobRange)); // Make sure the current job is the input one
+			    co_await getBulkLoadJob(&tr, jobId, jobRange); // Make sure the current job is the input one
 			ASSERT(!range.empty());
-			RangeResult result = wait(krmGetRanges(&tr, bulkLoadTaskPrefix, range));
+			RangeResult result = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, range);
 			// The task map has been initialized when submitBulkLoadJob, so we check the invariant here.
 			ASSERT(!result[0].value.empty() && result.size() == 2);
 			bulkLoadTaskState = decodeBulkLoadTaskState(result[0].value);
 			if (!bulkLoadTaskState.isValid()) {
-				return Optional<BulkLoadTaskState>();
+				co_return Optional<BulkLoadTaskState>();
 			}
 			KeyRange currentRange = Standalone(KeyRangeRef(result[0].key, result[1].key));
 			ASSERT(result[0].key != result[1].key);
@@ -1473,28 +1510,30 @@ ACTOR Future<Optional<BulkLoadTaskState>> bulkLoadJobFindTask(Reference<DataDist
 			}
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return bulkLoadTaskState;
+	co_return bulkLoadTaskState;
 }
 
 // Submit a bulkload task for the given jobId
-ACTOR Future<BulkLoadTaskState> bulkLoadJobSubmitTask(Reference<DataDistributor> self,
-                                                      UID jobId,
-                                                      BulkLoadManifestSet manifests,
-                                                      KeyRange taskRange) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
+Future<BulkLoadTaskState> bulkLoadJobSubmitTask(Reference<DataDistributor> self,
+                                                UID jobId,
+                                                BulkLoadManifestSet manifests,
+                                                KeyRange taskRange) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
 	// We define the task range is the range of the min begin key and the max end key among all input manifests
-	state BulkLoadTaskState bulkLoadTask(jobId, manifests, taskRange);
-	loop {
+	BulkLoadTaskState bulkLoadTask(jobId, manifests, taskRange);
+	while (true) {
+		Error err;
 		try {
 			// At any time, there must be at most one bulkload job
-			wait(checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get()));
-			wait(setBulkLoadSubmissionTransaction(&tr, bulkLoadTask));
+			co_await checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get());
+			co_await setBulkLoadSubmissionTransaction(&tr, bulkLoadTask);
 			// setBulkLoadSubmissionTransaction shuts down traffic to the range
-			wait(tr.commit());
+			co_await tr.commit();
 			Version commitVersion = tr.getCommittedVersion();
 			TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadJobExecutorSubmitTask", self->ddId)
 			    .detail("InputJobID", jobId)
@@ -1505,31 +1544,33 @@ ACTOR Future<BulkLoadTaskState> bulkLoadJobSubmitTask(Reference<DataDistributor>
 			    .detail("CommitVersion", commitVersion);
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return bulkLoadTask;
+	co_return bulkLoadTask;
 }
 
-ACTOR Future<Void> bulkLoadJobWaitUntilTaskCompleteOrError(Reference<DataDistributor> self,
-                                                           UID jobId,
-                                                           BulkLoadTaskState bulkLoadTask) {
+Future<Void> bulkLoadJobWaitUntilTaskCompleteOrError(Reference<DataDistributor> self,
+                                                     UID jobId,
+                                                     BulkLoadTaskState bulkLoadTask) {
 	ASSERT(bulkLoadTask.isValid());
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
-	state BulkLoadTaskState currentTask;
-	loop {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+	BulkLoadTaskState currentTask;
+	while (true) {
+		Error err;
+		bool hasErr = false;
 		try {
-			wait(store(currentTask,
-			           getBulkLoadTask(&tr,
-			                           bulkLoadTask.getRange(),
-			                           bulkLoadTask.getTaskId(),
-			                           { BulkLoadPhase::Submitted,
-			                             BulkLoadPhase::Triggered,
-			                             BulkLoadPhase::Running,
-			                             BulkLoadPhase::Complete,
-			                             BulkLoadPhase::Acknowledged,
-			                             BulkLoadPhase::Error })));
+			currentTask = co_await getBulkLoadTask(&tr,
+			                                       bulkLoadTask.getRange(),
+			                                       bulkLoadTask.getTaskId(),
+			                                       { BulkLoadPhase::Submitted,
+			                                         BulkLoadPhase::Triggered,
+			                                         BulkLoadPhase::Running,
+			                                         BulkLoadPhase::Complete,
+			                                         BulkLoadPhase::Acknowledged,
+			                                         BulkLoadPhase::Error });
 			if (currentTask.getJobId() != bulkLoadTask.getJobId()) {
 				throw bulkload_task_outdated();
 			}
@@ -1539,15 +1580,19 @@ ACTOR Future<Void> bulkLoadJobWaitUntilTaskCompleteOrError(Reference<DataDistrib
 				    .detail("TaskJobID", currentTask.getJobId())
 				    .detail("TaskRange", currentTask.getRange())
 				    .detail("TaskID", currentTask.getTaskId());
-				return Void();
+				co_return;
 			}
 			if (currentTask.phase == BulkLoadPhase::Complete || currentTask.phase == BulkLoadPhase::Acknowledged) {
-				return Void();
+				co_return;
 			}
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
+			hasErr = true;
+		}
+		if (hasErr) {
+			co_await tr.onError(err);
 		}
-		wait(delay(SERVER_KNOBS->DD_BULKLOAD_JOB_MONITOR_PERIOD_SEC));
+		co_await delay(SERVER_KNOBS->DD_BULKLOAD_JOB_MONITOR_PERIOD_SEC);
 	}
 }
 
@@ -1562,34 +1607,33 @@ KeyRange generateBulkLoadTaskRange(const BulkLoadManifestSet& manifests, const K
 // Given a list of manifestEntry, create a bulkload task.
 // There is an invariant by bulkload engine: if a task metadata is persisted, the task
 // is guaranteed to be eventually marked as complete or error.
-ACTOR Future<Void> bulkLoadJobNewTask(Reference<DataDistributor> self,
-                                      UID jobId,
-                                      std::string jobRoot,
-                                      KeyRange jobRange,
-                                      BulkLoadTransportMethod jobTransportMethod,
-                                      std::string manifestLocalTempFolder,
-                                      std::vector<BulkLoadJobFileManifestEntry> manifestEntries,
-                                      Promise<Void> errorOut) {
-	state Database cx = self->txnProcessor->context();
-	state BulkLoadTaskState bulkLoadTask;
-	state BulkLoadManifestSet manifests;
-	state double beginTime = now();
-	state KeyRange taskRange;
+Future<Void> bulkLoadJobNewTask(Reference<DataDistributor> self,
+                                UID jobId,
+                                std::string jobRoot,
+                                KeyRange jobRange,
+                                BulkLoadTransportMethod jobTransportMethod,
+                                std::string manifestLocalTempFolder,
+                                std::vector<BulkLoadJobFileManifestEntry> manifestEntries,
+                                Promise<Void> errorOut) {
+	Database cx = self->txnProcessor->context();
+	BulkLoadTaskState bulkLoadTask;
+	BulkLoadManifestSet manifests;
+	double beginTime = now();
+	KeyRange taskRange;
 	ASSERT(!manifestEntries.empty());
 	try {
 		// Step 1: Get manifest metadata by downloading the manifest file
-		wait(store(manifests,
-		           getBulkLoadManifestMetadataFromEntry(
-		               manifestEntries, manifestLocalTempFolder, jobTransportMethod, jobRoot, self->ddId)));
+		manifests = co_await getBulkLoadManifestMetadataFromEntry(
+		    manifestEntries, manifestLocalTempFolder, jobTransportMethod, jobRoot, self->ddId);
 		taskRange = generateBulkLoadTaskRange(manifests, jobRange);
 
 		// Step 2: Check if the task has been created
 		// We define the task range as the range between the min begin key and the max end key of all manifests
 		Optional<BulkLoadTaskState> bulkLoadTask_ =
-		    wait(bulkLoadJobFindTask(self, taskRange, jobId, jobRange, self->ddId));
+		    co_await bulkLoadJobFindTask(self, taskRange, jobId, jobRange, self->ddId);
 		if (bulkLoadTask_.present()) {
 			// The task was not existing in the metadata but existing now. So, we need not create the task.
-			return Void();
+			co_return;
 		}
 
 		// Step 3: Trigger bulkload task which is handled by bulkload task engine
@@ -1603,7 +1647,7 @@ ACTOR Future<Void> bulkLoadJobNewTask(Reference<DataDistributor> self,
 		// the manifests can contain more data outside the task range.
 		// The task range is the source of truth for the data that the task will cover.
 		// The task range is used to filter out data outside the task range when the SS loading the data.
-		wait(store(bulkLoadTask, bulkLoadJobSubmitTask(self, jobId, manifests, taskRange)));
+		bulkLoadTask = co_await bulkLoadJobSubmitTask(self, jobId, manifests, taskRange);
 
 		TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadJobExecutorTask", self->ddId)
 		    .detail("Phase", "Task submitted")
@@ -1635,30 +1679,29 @@ ACTOR Future<Void> bulkLoadJobNewTask(Reference<DataDistributor> self,
 		// Currently, all errors here come from the bulkload job mechanism.
 		// BulkLoad task is guaranteed to be completed by the engine given a task metadata is persisted.
 	}
-	return Void();
 }
 
 // Given a bulkload task range, find the task and wait until the task is complete or error.
 // There is an invariant by bulkload engine: if a task metadata is persisted, the task
 // is guaranteed to be eventually marked as complete or error.
-ACTOR Future<Void> bulkLoadJobMonitorTask(Reference<DataDistributor> self,
-                                          UID jobId,
-                                          KeyRange jobRange,
-                                          KeyRange taskRange,
-                                          Promise<Void> errorOut) {
-	state Database cx = self->txnProcessor->context();
-	state BulkLoadTaskState bulkLoadTask;
-	state double beginTime = now();
+Future<Void> bulkLoadJobMonitorTask(Reference<DataDistributor> self,
+                                    UID jobId,
+                                    KeyRange jobRange,
+                                    KeyRange taskRange,
+                                    Promise<Void> errorOut) {
+	Database cx = self->txnProcessor->context();
+	BulkLoadTaskState bulkLoadTask;
+	double beginTime = now();
 	ASSERT(self->bulkLoadParallelismLimitor.canStart());
 	self->bulkLoadParallelismLimitor.incrementTaskCounter();
 	try {
 		// Step 1: Check if the task has been created
 		Optional<BulkLoadTaskState> bulkLoadTask_ =
-		    wait(bulkLoadJobFindTask(self, taskRange, jobId, jobRange, self->ddId));
+		    co_await bulkLoadJobFindTask(self, taskRange, jobId, jobRange, self->ddId);
 		if (!bulkLoadTask_.present()) {
 			// The task was existing in the metadata but now disappear. So, we need not monitor the task.
 			self->bulkLoadParallelismLimitor.decrementTaskCounter();
-			return Void();
+			co_return;
 		}
 		bulkLoadTask = bulkLoadTask_.get();
 		TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadJobExecutorTask", self->ddId)
@@ -1675,7 +1718,7 @@ ACTOR Future<Void> bulkLoadJobMonitorTask(Reference<DataDistributor> self,
 		}
 
 		// Step 2: Monitor the bulkload completion
-		wait(bulkLoadJobWaitUntilTaskCompleteOrError(self, jobId, bulkLoadTask));
+		co_await bulkLoadJobWaitUntilTaskCompleteOrError(self, jobId, bulkLoadTask);
 		TraceEvent(bulkLoadPerfEventSev(), "DDBulkLoadJobExecutorTask", self->ddId)
 		    .detail("Phase", "Found task complete")
 		    .detail("JobID", jobId)
@@ -1698,22 +1741,22 @@ ACTOR Future<Void> bulkLoadJobMonitorTask(Reference<DataDistributor> self,
 		// Currently, all errors here come from the bulkload job mechanism.
 		// BulkLoad task is guaranteed to be completed by the engine given a task metadata is persisted.
 	}
-	return Void();
 }
 
-ACTOR Future<Void> persistBulkLoadJobTaskCount(Reference<DataDistributor> self) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
+Future<Void> persistBulkLoadJobTaskCount(Reference<DataDistributor> self) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
 	ASSERT(self->bulkLoadJobManager.present() && self->bulkLoadJobManager.get().isValid());
-	state BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
-	state UID jobId = jobState.getJobId();
-	state KeyRange jobRange = jobState.getJobRange();
+	BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
+	UID jobId = jobState.getJobId();
+	KeyRange jobRange = jobState.getJobRange();
 	ASSERT(jobState.getTaskCount().present());
-	state uint64_t taskCount = jobState.getTaskCount().get();
-	state BulkLoadJobState currentJobState;
-	loop {
+	uint64_t taskCount = jobState.getTaskCount().get();
+	BulkLoadJobState currentJobState;
+	while (true) {
+		Error err;
 		try {
-			wait(store(currentJobState, getBulkLoadJob(&tr, jobId, jobRange)));
+			currentJobState = co_await getBulkLoadJob(&tr, jobId, jobRange);
 			if (currentJobState.getTaskCount().present()) {
 				if (currentJobState.getTaskCount().get() != taskCount) {
 					TraceEvent(SevError, "DDBulkLoadJobManagerFindTaskCountMismatch", self->ddId)
@@ -1724,11 +1767,11 @@ ACTOR Future<Void> persistBulkLoadJobTaskCount(Reference<DataDistributor> self)
 					    .detail("CurrentJobRange", currentJobState.getJobRange());
 					ASSERT(false);
 				}
-				return Void();
+				co_return;
 			}
 			currentJobState.setTaskCount(taskCount);
-			wait(krmSetRange(&tr, bulkLoadJobPrefix, jobRange, bulkLoadJobValue(currentJobState)));
-			wait(tr.commit());
+			co_await krmSetRange(&tr, bulkLoadJobPrefix, jobRange, bulkLoadJobValue(currentJobState));
+			co_await tr.commit();
 			Version commitVersion = tr.getCommittedVersion();
 			TraceEvent(SevInfo, "DDBulkLoadJobManagerPersistTaskCountToJobMetadata", self->ddId)
 			    .detail("JobID", jobId)
@@ -1737,60 +1780,63 @@ ACTOR Future<Void> persistBulkLoadJobTaskCount(Reference<DataDistributor> self)
 			    .detail("TaskCount", taskCount);
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
 // Remove the bulkload job metadata from the range map metadata.
 // Add the job metadata to the history metadata with the error message.
-ACTOR Future<Void> moveErrorBulkLoadJobToHistory(Reference<DataDistributor> self, std::string errorMessage) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
-	state BulkLoadJobState currentJobState;
+Future<Void> moveErrorBulkLoadJobToHistory(Reference<DataDistributor> self, std::string errorMessage) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+	BulkLoadJobState currentJobState;
 	ASSERT(self->bulkLoadJobManager.present() && self->bulkLoadJobManager.get().isValid());
-	state UID jobId = self->bulkLoadJobManager.get().jobState.getJobId();
-	state KeyRange jobRange = self->bulkLoadJobManager.get().jobState.getJobRange();
-	loop {
+	UID jobId = self->bulkLoadJobManager.get().jobState.getJobId();
+	KeyRange jobRange = self->bulkLoadJobManager.get().jobState.getJobRange();
+	while (true) {
+		Error err;
 		try {
-			wait(checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get()));
-			wait(store(currentJobState, getBulkLoadJob(&tr, jobId, jobRange)));
-			wait(krmSetRange(
-			    &tr, bulkLoadJobPrefix, currentJobState.getJobRange(), bulkLoadJobValue(BulkLoadJobState())));
+			co_await checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get());
+			currentJobState = co_await getBulkLoadJob(&tr, jobId, jobRange);
+			co_await krmSetRange(
+			    &tr, bulkLoadJobPrefix, currentJobState.getJobRange(), bulkLoadJobValue(BulkLoadJobState()));
 			currentJobState.setErrorPhase(errorMessage);
 			currentJobState.setEndTime(now());
-			wait(addBulkLoadJobToHistory(&tr, currentJobState));
-			wait(releaseExclusiveReadLockOnRange(&tr, jobRange, rangeLockNameForBulkLoad));
-			wait(tr.commit());
+			co_await addBulkLoadJobToHistory(&tr, currentJobState);
+			co_await releaseExclusiveReadLockOnRange(&tr, jobRange, rangeLockNameForBulkLoad);
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
 // Download the job manifest file from the remoteJobManifestFilePath to the localJobManifestFilePath.
 // Build the bulkload manifest range map based on the localJobManifestFilePath file content.
 // For any failure, we mark the job metadata as error and move the metadata to the history.
-ACTOR Future<Void> fetchBulkLoadTaskManifestEntryMap(Reference<DataDistributor> self,
-                                                     BulkLoadTransportMethod jobTransportMethod,
-                                                     std::string localJobManifestFilePath,
-                                                     std::string remoteJobManifestFilePath) {
+Future<Void> fetchBulkLoadTaskManifestEntryMap(Reference<DataDistributor> self,
+                                               BulkLoadTransportMethod jobTransportMethod,
+                                               std::string localJobManifestFilePath,
+                                               std::string remoteJobManifestFilePath) {
 	ASSERT(self->bulkLoadJobManager.present() && self->bulkLoadJobManager.get().isValid() &&
 	       self->bulkLoadJobManager.get().manifestEntryMap->empty());
-	state double beginTime = now();
-	state KeyRange jobRange = self->bulkLoadJobManager.get().jobState.getJobRange();
-	state KeyRange manifestMapRange;
+	double beginTime = now();
+	KeyRange jobRange = self->bulkLoadJobManager.get().jobState.getJobRange();
+	KeyRange manifestMapRange;
+	Error caughtErr;
+	bool hasCaughtErr = false;
 	try {
 		if (!fileExists(abspath(localJobManifestFilePath))) {
 			TraceEvent(SevDebug, "DDBulkLoadJobManagerDownloadJobManifest", self->ddId)
 			    .detail("JobTransportMethod", jobTransportMethod)
 			    .detail("LocalJobManifestFilePath", localJobManifestFilePath)
 			    .detail("RemoteJobManifestFilePath", remoteJobManifestFilePath);
-			wait(downloadBulkLoadJobManifestFile(
-			    jobTransportMethod, localJobManifestFilePath, remoteJobManifestFilePath, self->ddId));
+			co_await downloadBulkLoadJobManifestFile(
+			    jobTransportMethod, localJobManifestFilePath, remoteJobManifestFilePath, self->ddId);
 			TraceEvent(SevInfo, "DDBulkLoadJobManagerManifestDownloaded", self->ddId)
 			    .detail("JobTransportMethod", jobTransportMethod)
 			    .detail("LocalJobManifestFilePath", localJobManifestFilePath)
@@ -1800,12 +1846,11 @@ ACTOR Future<Void> fetchBulkLoadTaskManifestEntryMap(Reference<DataDistributor>
 		// At this point, we have the global job manifest file stored locally at localJobManifestFilePath.
 		// This job manifest file stores all remote manifest filepath per range.
 		// Here, we want to get all manifest entries of the file with in the range specified by jobRange.
-		wait(store(manifestMapRange,
-		           getBulkLoadJobFileManifestEntryFromJobManifestFile(
-		               localJobManifestFilePath,
-		               jobRange,
-		               self->ddId,
-		               /*output=*/self->bulkLoadJobManager.get().manifestEntryMap)));
+		manifestMapRange = co_await getBulkLoadJobFileManifestEntryFromJobManifestFile(
+		    localJobManifestFilePath,
+		    jobRange,
+		    self->ddId,
+		    /*output=*/self->bulkLoadJobManager.get().manifestEntryMap);
 		// It is possible that the bulkload job is using a data set that does not entirely contain the bulkload job
 		// range. In this case, we give up the bulkload job immediately without loading any range..
 		if (self->bulkLoadJobManager.get().jobState.getJobRange() != manifestMapRange) {
@@ -1822,10 +1867,14 @@ ACTOR Future<Void> fetchBulkLoadTaskManifestEntryMap(Reference<DataDistributor>
 		    .detail("TaskCount", self->bulkLoadJobManager.get().manifestEntryMap->size())
 		    .detail("Duration", now() - beginTime);
 	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			throw e;
+		caughtErr = e;
+		hasCaughtErr = true;
+	}
+	if (hasCaughtErr) {
+		if (caughtErr.code() == error_code_actor_cancelled) {
+			throw caughtErr;
 		}
-		state Error err = e;
+		Error err = caughtErr;
 		TraceEvent(SevWarnAlways, "DDBulkLoadJobManagerFindUnretryableError", self->ddId)
 		    .errorUnsuppressed(err)
 		    .detail("JobTransportMethod", jobTransportMethod)
@@ -1837,7 +1886,7 @@ ACTOR Future<Void> fetchBulkLoadTaskManifestEntryMap(Reference<DataDistributor>
 		                           ". The local file path is " + localJobManifestFilePath +
 		                           ". The transport method is " +
 		                           convertBulkLoadTransportMethodToString(jobTransportMethod) + ".";
-		wait(moveErrorBulkLoadJobToHistory(self, errorMessage));
+		co_await moveErrorBulkLoadJobToHistory(self, errorMessage);
 		TraceEvent(SevWarnAlways, "DDBulkLoadJobManagerPersistUnretryableError", self->ddId)
 		    .errorUnsuppressed(err)
 		    .detail("JobTransportMethod", jobTransportMethod)
@@ -1846,31 +1895,30 @@ ACTOR Future<Void> fetchBulkLoadTaskManifestEntryMap(Reference<DataDistributor>
 		    .detail("Duration", now() - beginTime);
 		throw err;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> scheduleBulkLoadJob(Reference<DataDistributor> self, Promise<Void> errorOut) {
-	state BulkLoadJobFileManifestEntry manifestEntry;
+Future<Void> scheduleBulkLoadJob(Reference<DataDistributor> self, Promise<Void> errorOut) {
+	BulkLoadJobFileManifestEntry manifestEntry;
 	ASSERT(self->bulkLoadJobManager.present() && self->bulkLoadJobManager.get().isValid());
-	state BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
-	state Key beginKey = jobState.getJobRange().begin;
-	state std::vector<Future<Void>> actors;
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
+	BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
+	Key beginKey = jobState.getJobRange().begin;
+	std::vector<Future<Void>> actors;
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
 	// We load the bulkload task from the job manifest.
 	// The job manifest is organized in a sorted map. The key is the beginKey of the manifest.
 	// The value is the manifest. For details, please see comments in getBulkLoadJobManifestData.
-	loop {
+	while (true) {
+		Error err;
 		try {
-			state RangeResult res =
-			    wait(krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, jobState.getJobRange().end)));
-			state int i = 0;
-			for (; i < res.size() - 1; i++) {
+			RangeResult res =
+			    co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, jobState.getJobRange().end));
+			for (int i = 0; i < res.size() - 1; i++) {
 				// Proceeding the beginKey for submitted/completed tasks.
 				// Bypass completed tasks.
 				// Start monitoring for submitted tasks.
 				if (!res[i].value.empty()) {
-					state BulkLoadTaskState task = decodeBulkLoadTaskState(res[i].value);
+					BulkLoadTaskState task = decodeBulkLoadTaskState(res[i].value);
 					if (task.isValid()) {
 						if (task.getJobId() != self->bulkLoadJobManager.get().jobState.getJobId()) {
 							throw bulkload_task_outdated();
@@ -1894,11 +1942,11 @@ ACTOR Future<Void> scheduleBulkLoadJob(Reference<DataDistributor> self, Promise<
 							// Otherwise, the parallelism limitor will slow down the task submission.
 							if (self->bulkLoadJobManager.get().allTaskSubmitted) {
 								// Limit parallelism
-								loop {
+								while (true) {
 									if (self->bulkLoadParallelismLimitor.canStart()) {
 										break;
 									}
-									wait(self->bulkLoadParallelismLimitor.waitUntilCounterChanged());
+									co_await self->bulkLoadParallelismLimitor.waitUntilCounterChanged();
 								}
 								// Monitor submitted tasks
 								actors.push_back(bulkLoadJobMonitorTask(
@@ -1936,7 +1984,7 @@ ACTOR Future<Void> scheduleBulkLoadJob(Reference<DataDistributor> self, Promise<
 					                                    self->bulkLoadJobManager.get().manifestLocalTempFolder,
 					                                    manifestEntries,
 					                                    errorOut));
-					wait(delay(SERVER_KNOBS->DD_BULKLOAD_TASK_SUBMISSION_INTERVAL_SEC)); // Avoid busy loop
+					co_await delay(SERVER_KNOBS->DD_BULKLOAD_TASK_SUBMISSION_INTERVAL_SEC); // Avoid busy loop
 				}
 				ASSERT(beginKey == res[i + 1].key || beginKey > jobState.getJobRange().end);
 			}
@@ -1945,36 +1993,41 @@ ACTOR Future<Void> scheduleBulkLoadJob(Reference<DataDistributor> self, Promise<
 				self->bulkLoadJobManager.get().allTaskSubmitted = true;
 				break;
 			}
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	wait(waitForAll(actors));
-	return Void();
+	co_await waitForAll(actors);
 }
 
-ACTOR Future<bool> checkBulkLoadTaskCompleteOrError(Reference<DataDistributor> self) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
+Future<bool> checkBulkLoadTaskCompleteOrError(Reference<DataDistributor> self) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
 	ASSERT(self->bulkLoadJobManager.present() && self->bulkLoadJobManager.get().isValid());
-	state BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
-	state Key beginKey = jobState.getJobRange().begin;
-	state Key endKey = jobState.getJobRange().end;
-	state BulkLoadTaskState existTask;
-	state KeyRange rangeToRead;
-	state RangeResult bulkLoadTaskResult;
+	BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
+	Key beginKey = jobState.getJobRange().begin;
+	Key endKey = jobState.getJobRange().end;
+	BulkLoadTaskState existTask;
+	KeyRange rangeToRead;
+	RangeResult bulkLoadTaskResult;
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			bulkLoadTaskResult.clear();
 			rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
-			BulkLoadJobState checkJobState = wait(getBulkLoadJob(&tr, jobState.getJobId(), jobState.getJobRange()));
-			wait(store(bulkLoadTaskResult, krmGetRanges(&tr, bulkLoadTaskPrefix, rangeToRead)));
-			for (int i = 0; i < bulkLoadTaskResult.size() - 1; i++) {
+			BulkLoadJobState checkJobState = co_await getBulkLoadJob(&tr, jobState.getJobId(), jobState.getJobRange());
+			bulkLoadTaskResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, rangeToRead);
+			if (bulkLoadTaskResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(bulkLoadTaskResult.size()) - 1; i++) {
 				ASSERT(!bulkLoadTaskResult[i].value.empty());
 				existTask = decodeBulkLoadTaskState(bulkLoadTaskResult[i].value);
 				if (!existTask.isValid()) {
 					// At this time, the task metadata must be existing since no one acknowledges this task.
-					return false;
+					co_return false;
 				}
 				// When start loading a job, the old job metadata must be cleared at first.
 				// So, any existing bulkload job id must match the running job id.
@@ -2000,40 +2053,45 @@ ACTOR Future<bool> checkBulkLoadTaskCompleteOrError(Reference<DataDistributor> s
 					    .detail("TaskID", existTask.getTaskId())
 					    .detail("TaskRange", existTask.getRange())
 					    .detail("InputJobID", jobState.getJobId());
-					return false;
+					co_return false;
 				}
 			}
 			beginKey = bulkLoadTaskResult.back().key;
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return true;
+	co_return true;
 }
 
-ACTOR Future<Void> finalizeBulkLoadJob(Reference<DataDistributor> self) {
-	state Database cx = self->txnProcessor->context();
-	state Transaction tr(cx);
+Future<Void> finalizeBulkLoadJob(Reference<DataDistributor> self) {
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
 	ASSERT(self->bulkLoadJobManager.present() && self->bulkLoadJobManager.get().isValid());
-	state BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
-	state Key beginKey = jobState.getJobRange().begin;
-	state Key endKey = jobState.getJobRange().end;
-	state Optional<Key> lastKey;
-	state BulkLoadTaskState existTask;
-	state RangeResult bulkLoadTaskResult;
-	state KeyRange jobCompleteRange;
-	state bool hasError = false;
-	state int i = 0;
-	state bool allFinish = false;
+	BulkLoadJobState jobState = self->bulkLoadJobManager.get().jobState;
+	Key beginKey = jobState.getJobRange().begin;
+	Key endKey = jobState.getJobRange().end;
+	Optional<Key> lastKey;
+	BulkLoadTaskState existTask;
+	RangeResult bulkLoadTaskResult;
+	KeyRange jobCompleteRange;
+	bool hasError = false;
+	bool allFinish = false;
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			tr.reset();
 			bulkLoadTaskResult.clear();
-			BulkLoadJobState currentJobState = wait(getBulkLoadJob(&tr, jobState.getJobId(), jobState.getJobRange()));
+			BulkLoadJobState currentJobState =
+			    co_await getBulkLoadJob(&tr, jobState.getJobId(), jobState.getJobRange());
 			hasError = hasError && (currentJobState.getPhase() == BulkLoadJobPhase::Error);
-			wait(store(bulkLoadTaskResult, krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey))));
-			i = 0;
-			for (; i < bulkLoadTaskResult.size() - 1; i++) {
+			bulkLoadTaskResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey));
+			if (bulkLoadTaskResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(bulkLoadTaskResult.size()) - 1; i++) {
 				ASSERT(!bulkLoadTaskResult[i].value.empty());
 				existTask = decodeBulkLoadTaskState(bulkLoadTaskResult[i].value);
 				if (!existTask.isValid()) {
@@ -2066,7 +2124,7 @@ ACTOR Future<Void> finalizeBulkLoadJob(Reference<DataDistributor> self) {
 						ASSERT(false);
 					}
 					// Persist metadata and turn on traffic
-					wait(setBulkLoadFinalizeTransaction(&tr, existTask.getRange(), existTask.getTaskId()));
+					co_await setBulkLoadFinalizeTransaction(&tr, existTask.getRange(), existTask.getTaskId());
 				}
 				lastKey = bulkLoadTaskResult[i + 1].key;
 				break; // We actively break because we do not want transaction large
@@ -2078,18 +2136,19 @@ ACTOR Future<Void> finalizeBulkLoadJob(Reference<DataDistributor> self) {
 			}
 			ASSERT(lastKey.present());
 			jobCompleteRange = KeyRangeRef(jobState.getJobRange().begin, lastKey.get());
-			wait(checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get()));
+			co_await checkMoveKeysLock(&tr, self->context->lock, self->context->ddEnabledState.get());
 			allFinish = jobCompleteRange == jobState.getJobRange();
 			if (allFinish) {
 				// Move the complete job from job range map to the history map
 				jobState.setEndTime(now());
-				wait(krmSetRange(&tr, bulkLoadJobPrefix, jobState.getJobRange(), bulkLoadJobValue(BulkLoadJobState())));
-				wait(addBulkLoadJobToHistory(&tr, jobState));
-				wait(releaseExclusiveReadLockOnRange(&tr, jobState.getJobRange(), rangeLockNameForBulkLoad));
+				co_await krmSetRange(
+				    &tr, bulkLoadJobPrefix, jobState.getJobRange(), bulkLoadJobValue(BulkLoadJobState()));
+				co_await addBulkLoadJobToHistory(&tr, jobState);
+				co_await releaseExclusiveReadLockOnRange(&tr, jobState.getJobRange(), rangeLockNameForBulkLoad);
 			} else {
-				wait(krmSetRange(&tr, bulkLoadJobPrefix, jobCompleteRange, bulkLoadJobValue(jobState)));
+				co_await krmSetRange(&tr, bulkLoadJobPrefix, jobCompleteRange, bulkLoadJobValue(jobState));
 			}
-			wait(tr.commit());
+			co_await tr.commit();
 			Version commitVersion = tr.getCommittedVersion();
 			TraceEvent(SevInfo, "DDBulkLoadJobManagerFinalizeRange", self->ddId)
 			    .detail("JobCompleteRange", jobCompleteRange)
@@ -2101,30 +2160,31 @@ ACTOR Future<Void> finalizeBulkLoadJob(Reference<DataDistributor> self) {
 			    .detail("ExistTaskID", existTask.getTaskId())
 			    .detail("ExistTaskRange", existTask.getRange());
 			beginKey = lastKey.get();
+			continue;
 		} catch (Error& e) {
-			// Currently, only bulkload job uses the range lock, and one job exists at a time.
-			// TODO(BulkLoad): support multiple jobs at a time
-			ASSERT(e.code() != error_code_range_unlock_reject);
-			wait(tr.onError(e));
+			err = e;
 		}
+		// Currently, only bulkload job uses the range lock, and one job exists at a time.
+		// TODO(BulkLoad): support multiple jobs at a time
+		ASSERT(err.code() != error_code_range_unlock_reject);
+		co_await tr.onError(err);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> bulkLoadJobManager(Reference<DataDistributor> self) {
+Future<Void> bulkLoadJobManager(Reference<DataDistributor> self) {
 	// Find any existing bulkload job metadata. If not existing, exit.
-	state Database cx = self->txnProcessor->context();
-	state Optional<BulkLoadJobState> job = wait(getRunningBulkLoadJob(cx));
+	Database cx = self->txnProcessor->context();
+	Optional<BulkLoadJobState> job = co_await getRunningBulkLoadJob(cx);
 	if (!job.present()) {
 		TraceEvent(bulkLoadVerboseEventSev(), "DDBulkLoadJobManagerNoJobExist", self->ddId);
 		self->bulkLoadJobManager.reset(); // set to empty
 		self->bulkLoadTaskCollection->removeBulkLoadJobRange();
-		return Void();
+		co_return;
 	}
-	state UID jobId = job.get().getJobId();
-	state KeyRange jobRange = job.get().getJobRange();
-	state std::string jobRoot = job.get().getJobRoot();
-	state BulkLoadTransportMethod jobTransportMethod = job.get().getTransportMethod();
+	UID jobId = job.get().getJobId();
+	KeyRange jobRange = job.get().getJobRange();
+	std::string jobRoot = job.get().getJobRoot();
+	BulkLoadTransportMethod jobTransportMethod = job.get().getTransportMethod();
 	self->bulkLoadTaskCollection->setBulkLoadJobRange(jobRange);
 
 	// Build up bulkLoadJobManager if a new job starts or the bulkLoadJobManager has not been set up.
@@ -2146,9 +2206,9 @@ ACTOR Future<Void> bulkLoadJobManager(Reference<DataDistributor> self) {
 		std::string localJobManifestFilePath = joinPath(localFolder, jobManifestFileName);
 		std::string remoteJobManifestFilePath = appendToPath(remoteFolder, jobManifestFileName);
 		self->bulkLoadJobManager = DDBulkLoadJobManager(job.get(), manifestLocalTempFolder);
-		wait(fetchBulkLoadTaskManifestEntryMap(
-		    self, jobTransportMethod, localJobManifestFilePath, remoteJobManifestFilePath));
-		wait(persistBulkLoadJobTaskCount(self));
+		co_await fetchBulkLoadTaskManifestEntryMap(
+		    self, jobTransportMethod, localJobManifestFilePath, remoteJobManifestFilePath);
+		co_await persistBulkLoadJobTaskCount(self);
 	} else {
 		TraceEvent(SevInfo, "DDBulkLoadJobManagerExist", self->ddId)
 		    .detail("JobID", self->bulkLoadJobManager.get().jobState.getJobId())
@@ -2162,20 +2222,20 @@ ACTOR Future<Void> bulkLoadJobManager(Reference<DataDistributor> self) {
 	// If yes, acknowledge complete tasks and leave error tasks there
 	// We turn on the traffic and shard boundary change only for completed range when bulkload
 	// on all ranges have been completed or error
-	state Promise<Void> errorOut; // Capture errors from bulkLoadJobExecuteTask
-	loop {
-		bool complete = wait(checkBulkLoadTaskCompleteOrError(self));
+	Promise<Void> errorOut; // Capture errors from bulkLoadJobExecuteTask
+	while (true) {
+		bool complete = co_await checkBulkLoadTaskCompleteOrError(self);
 		if (complete) {
 			TraceEvent(SevInfo, "DDBulkLoadJobManagerAllTaskComplete", self->ddId)
 			    .detail("JobID", self->bulkLoadJobManager.get().jobState.getJobId())
 			    .detail("JobRange", self->bulkLoadJobManager.get().jobState.getJobRange());
-			wait(finalizeBulkLoadJob(self));
+			co_await finalizeBulkLoadJob(self);
 			break; // end
 		} else {
 			std::vector<Future<Void>> actors;
 			actors.push_back(errorOut.getFuture());
 			actors.push_back(scheduleBulkLoadJob(self, errorOut));
-			wait(waitForAny(actors));
+			co_await waitForAny(actors);
 			// Any error in bulkLoadJobExecuteTask will cause cancellation of scheduleBulkLoadJob
 			// Note that bulkLoadJobExecuteTask simply does transaction to create and monitor
 			// the bulkload task. The error is expected to be bulkload_task_outdated error.
@@ -2183,16 +2243,15 @@ ACTOR Future<Void> bulkLoadJobManager(Reference<DataDistributor> self) {
 			    .detail("JobID", self->bulkLoadJobManager.get().jobState.getJobId())
 			    .detail("JobRange", self->bulkLoadJobManager.get().jobState.getJobRange());
 		}
-		wait(delay(SERVER_KNOBS->DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC));
+		co_await delay(SERVER_KNOBS->DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> bulkLoadJobCore(Reference<DataDistributor> self, Future<Void> readyToStart) {
-	wait(readyToStart);
-	loop {
+Future<Void> bulkLoadJobCore(Reference<DataDistributor> self, Future<Void> readyToStart) {
+	co_await readyToStart;
+	while (true) {
 		try {
-			wait(bulkLoadJobManager(self));
+			co_await bulkLoadJobManager(self);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
@@ -2202,19 +2261,95 @@ ACTOR Future<Void> bulkLoadJobCore(Reference<DataDistributor> self, Future<Void>
 				throw e;
 			}
 		}
-		wait(delay(SERVER_KNOBS->DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC));
+		co_await delay(SERVER_KNOBS->DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC);
+	}
+}
+
+// Monitor bulkLoadMode changes and dynamically spawn bulkload actors when mode becomes enabled.
+// This is necessary because the restore agent sets bulkLoadMode=1 AFTER DD has already initialized.
+// Without dynamic monitoring, bulkload jobs submitted by restore would never be processed.
+Future<Void> monitorBulkLoadModeAndSpawnActors(Reference<DataDistributor> self, Future<Void> readyToStart) {
+	// If bulkload is already enabled at startup, don't need to monitor
+	if (self->bulkLoadEnabled) {
+		co_return;
+	}
+
+	// Only monitor if SHARD_ENCODE_LOCATION_METADATA is enabled (required for bulkload)
+	if (!SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
+		TraceEvent(SevInfo, "DDBulkLoadModeMonitorSkipped", self->ddId)
+		    .detail("Reason", "SHARD_ENCODE_LOCATION_METADATA is disabled");
+		co_return;
+	}
+
+	Database cx = self->txnProcessor->context();
+	Transaction tr(cx);
+
+	TraceEvent(SevInfo, "DDBulkLoadModeMonitorStarted", self->ddId);
+
+	// Wait for mode to be enabled
+	while (true) {
+		bool hadError = false;
+		Error err;
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+
+			Optional<Value> modeValue = co_await tr.get(bulkLoadModeKey);
+			int mode = 0;
+			if (modeValue.present()) {
+				BinaryReader rd(modeValue.get(), Unversioned());
+				rd >> mode;
+			}
+
+			if (bulkLoadIsEnabled(mode) && !self->bulkLoadEnabled) {
+				TraceEvent(SevInfo, "DDBulkLoadModeDynamicallyEnabled", self->ddId)
+				    .detail("UsableRegions", self->configuration.usableRegions);
+				self->bulkLoadEnabled = true;
+				break;
+			}
+
+			tr.reset();
+			co_await delay(1.0); // Check every second
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
+			err = e;
+			hadError = true;
+		}
+		if (hadError) {
+			co_await tr.onError(err);
+		}
 	}
+
+	// Mode is now enabled - run the bulkload actors directly
+	TraceEvent(SevInfo, "DDBulkLoadModeActorsSpawning", self->ddId);
+
+	std::vector<Future<Void>> bulkLoadActors;
+	if (self->configuration.usableRegions > 1) {
+		bulkLoadActors.push_back(bulkLoadTaskCore(self, readyToStart && remoteRecovered(self->dbInfo)));
+		bulkLoadActors.push_back(bulkLoadJobCore(self, readyToStart && remoteRecovered(self->dbInfo)));
+	} else {
+		bulkLoadActors.push_back(bulkLoadTaskCore(self, readyToStart));
+		bulkLoadActors.push_back(bulkLoadJobCore(self, readyToStart));
+	}
+
+	TraceEvent(SevInfo, "DDBulkLoadModeActorsSpawned", self->ddId);
+
+	// Wait for all bulkload actors (they run forever unless cancelled)
+	co_await waitForAll(bulkLoadActors);
 }
 
 // The actor spawned by DD dedicated to listen on a SS bulkdump task and holding a budget of parallelismLimitor.
 // The parallelismLimitor is used to limit the maximum concurrent bulkloading tasks spawned by DD.
 // Each DD spawned task corresponds to an actual alive SS bulk dumping task.
 // This actor silently exit if SS suceeds or fails to handle a task.
-ACTOR Future<Void> doBulkDumpTask(Reference<DataDistributor> self,
-                                  StorageServerInterface ssi,
-                                  BulkDumpState bulkDumpState,
-                                  std::vector<UID> checksumServers) {
-	state double beginTime = now();
+Future<Void> doBulkDumpTask(Reference<DataDistributor> self,
+                            StorageServerInterface ssi,
+                            BulkDumpState bulkDumpState,
+                            std::vector<UID> checksumServers) {
+	double beginTime = now();
 	ASSERT(self->bulkDumpParallelismLimitor.canStart());
 	self->bulkDumpParallelismLimitor.incrementTaskCounter();
 	TraceEvent(bulkLoadVerboseEventSev(), "DDBulkDumpDoTaskStart", self->ddId)
@@ -2224,7 +2359,7 @@ ACTOR Future<Void> doBulkDumpTask(Reference<DataDistributor> self,
 	    .detail("JobID", bulkDumpState.getJobRange());
 	try {
 		ErrorOr<BulkDumpState> vResult =
-		    wait(ssi.bulkdump.tryGetReply(BulkDumpRequest(checksumServers, bulkDumpState)));
+		    co_await ssi.bulkdump.tryGetReply(BulkDumpRequest(checksumServers, bulkDumpState));
 		if (vResult.isError()) {
 			throw vResult.getError();
 		}
@@ -2251,35 +2386,38 @@ ACTOR Future<Void> doBulkDumpTask(Reference<DataDistributor> self,
 		// Sliently exit for other errors
 	}
 	self->bulkDumpParallelismLimitor.decrementTaskCounter();
-	return Void();
 }
 
-ACTOR Future<Void> scheduleBulkDumpJob(Reference<DataDistributor> self) {
-	state UID jobId = self->bulkDumpJobManager.jobState.getJobId();
-	state KeyRange jobRange = self->bulkDumpJobManager.jobState.getJobRange();
+Future<Void> scheduleBulkDumpJob(Reference<DataDistributor> self) {
+	UID jobId = self->bulkDumpJobManager.jobState.getJobId();
+	KeyRange jobRange = self->bulkDumpJobManager.jobState.getJobRange();
 	TraceEvent(SevInfo, "DDBulkDumpJobScheduleStart", self->ddId).detail("JobId", jobId).detail("JobRange", jobRange);
-	state Database cx = self->txnProcessor->context();
-	state Key beginKey = jobRange.begin;
-	state Key endKey = jobRange.end;
-	state KeyRange rangeToRead;
-	state int bulkDumpResultIndex = 0;
-	state BulkDumpState bulkDumpState;
-	state KeyRange bulkDumpRange;
-	state RangeResult bulkDumpResult;
-	state int rangeLocationIndex = 0;
-	state std::vector<IDDTxnProcessor::DDRangeLocations> rangeLocations;
-	state KeyRange taskRange;
-	state std::vector<Future<Void>> actors;
-	state Transaction tr(cx);
+	Database cx = self->txnProcessor->context();
+	Key beginKey = jobRange.begin;
+	Key endKey = jobRange.end;
+	KeyRange rangeToRead;
+	int bulkDumpResultIndex = 0;
+	BulkDumpState bulkDumpState;
+	KeyRange bulkDumpRange;
+	RangeResult bulkDumpResult;
+	int rangeLocationIndex = 0;
+	std::vector<IDDTxnProcessor::DDRangeLocations> rangeLocations;
+	KeyRange taskRange;
+	std::vector<Future<Void>> actors;
+	Transaction tr(cx);
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
 			bulkDumpResult.clear();
-			wait(store(bulkDumpResult, krmGetRanges(&tr, bulkDumpPrefix, rangeToRead)));
+			bulkDumpResult = co_await krmGetRanges(&tr, bulkDumpPrefix, rangeToRead);
+			if (bulkDumpResult.empty()) {
+				break;
+			}
 			// We rely on random assignment of shards for the load balancing. Also, there is a flow lock to prevent SS
 			// from being overloaded.
 			bulkDumpResultIndex = 0;
-			for (; bulkDumpResultIndex < bulkDumpResult.size() - 1; bulkDumpResultIndex++) {
+			for (; bulkDumpResultIndex < static_cast<int>(bulkDumpResult.size()) - 1; bulkDumpResultIndex++) {
 				if (bulkDumpResult[bulkDumpResultIndex].value.empty()) {
 					beginKey = bulkDumpResult[bulkDumpResultIndex + 1].key;
 					continue;
@@ -2308,7 +2446,7 @@ ACTOR Future<Void> scheduleBulkDumpJob(Reference<DataDistributor> self) {
 				}
 				ASSERT(bulkDumpState.getPhase() == BulkDumpPhase::Submitted);
 				// Partition the job in the unit of shard
-				wait(store(rangeLocations, self->txnProcessor->getSourceServerInterfacesForRange(bulkDumpRange)));
+				rangeLocations = co_await self->txnProcessor->getSourceServerInterfacesForRange(bulkDumpRange);
 				TraceEvent(bulkLoadVerboseEventSev(), "DDBulkDumpJobScheduleJobPartition", self->ddId)
 				    .detail("JobId", jobId)
 				    .detail("NumShard", rangeLocations.size())
@@ -2322,11 +2460,11 @@ ACTOR Future<Void> scheduleBulkDumpJob(Reference<DataDistributor> self) {
 					taskRange = rangeLocations[rangeLocationIndex].range;
 					ASSERT(!taskRange.empty());
 					// Limit parallelism
-					loop {
+					while (true) {
 						if (self->bulkDumpParallelismLimitor.canStart()) {
 							break;
 						}
-						wait(self->bulkDumpParallelismLimitor.waitUntilCounterChanged());
+						co_await self->bulkDumpParallelismLimitor.waitUntilCounterChanged();
 					}
 					SSBulkDumpTask task = getSSBulkDumpTask(rangeLocations[rangeLocationIndex].servers,
 					                                        bulkDumpState.generateRangeTask(taskRange));
@@ -2342,34 +2480,39 @@ ACTOR Future<Void> scheduleBulkDumpJob(Reference<DataDistributor> self) {
 				beginKey = rangeLocations.back().range.end;
 				break;
 			}
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			wait(tr.onError(e));
+			err = e;
 		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		co_await tr.onError(err);
 	}
-	wait(waitForAll(actors));
+	co_await waitForAll(actors);
 	TraceEvent(SevInfo, "DDBulkDumpJobScheduleEnd", self->ddId).detail("JobId", jobId).detail("JobRange", jobRange);
-	return Void();
 }
 
-ACTOR Future<bool> checkBulkDumpJobComplete(Reference<DataDistributor> self) {
-	state UID jobId = self->bulkDumpJobManager.jobState.getJobId();
-	state KeyRange jobRange = self->bulkDumpJobManager.jobState.getJobRange();
-	state Database cx = self->txnProcessor->context();
-	state Key beginKey = jobRange.begin;
-	state Key endKey = jobRange.end;
-	state RangeResult bulkDumpResult;
-	state BulkDumpState bulkDumpState;
-	state KeyRange rangeToRead;
-	state Transaction tr(cx);
+Future<bool> checkBulkDumpJobComplete(Reference<DataDistributor> self) {
+	UID jobId = self->bulkDumpJobManager.jobState.getJobId();
+	KeyRange jobRange = self->bulkDumpJobManager.jobState.getJobRange();
+	Database cx = self->txnProcessor->context();
+	Key beginKey = jobRange.begin;
+	Key endKey = jobRange.end;
+	RangeResult bulkDumpResult;
+	BulkDumpState bulkDumpState;
+	KeyRange rangeToRead;
+	Transaction tr(cx);
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
 			bulkDumpResult.clear();
-			wait(store(bulkDumpResult, krmGetRanges(&tr, bulkDumpPrefix, rangeToRead)));
-			for (int i = 0; i < bulkDumpResult.size() - 1; i++) {
+			bulkDumpResult = co_await krmGetRanges(&tr, bulkDumpPrefix, rangeToRead);
+			if (bulkDumpResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(bulkDumpResult.size()) - 1; i++) {
 				ASSERT(!bulkDumpResult[i].value.empty());
 				bulkDumpState = decodeBulkDumpState(bulkDumpResult[i].value);
 				if (!bulkDumpState.isValid() || bulkDumpState.getJobId() != jobId) {
@@ -2377,15 +2520,17 @@ ACTOR Future<bool> checkBulkDumpJobComplete(Reference<DataDistributor> self) {
 				}
 				if (bulkDumpState.getPhase() != BulkDumpPhase::Complete) {
 					ASSERT(bulkDumpState.getPhase() == BulkDumpPhase::Submitted);
-					return false;
+					co_return false;
 				}
 			}
 			beginKey = bulkDumpResult.back().key;
+			continue;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return true;
+	co_return true;
 }
 
 // Generate the bulkload job manifest file. Here is an example.
@@ -2402,49 +2547,48 @@ ACTOR Future<bool> checkBulkDumpJobComplete(Reference<DataDistributor> self) {
 // 300, the data size is 8.1KB, the manifest file path is "/tmp/range1/manifest3.txt".
 // Note that job-manifest.txt grows with the amount of data we dump.
 // TODO(BulkDump): revisit this later. May need to add some warning if the job-manifest.txt file size is too large.
-ACTOR Future<Void> generateLocalBulkDumpJobManifestFile(Reference<DataDistributor> self,
-                                                        std::string workFolder,
-                                                        std::string localJobManifestFilePath) {
-	state uint64_t counter = 0;
-	state std::shared_ptr<std::string> content = std::make_shared<std::string>();
+Future<Void> generateLocalBulkDumpJobManifestFile(Reference<DataDistributor> self,
+                                                  std::string workFolder,
+                                                  std::string localJobManifestFilePath) {
+	uint64_t counter = 0;
+	std::shared_ptr<std::string> content = std::make_shared<std::string>();
 	content->append(
 	    BulkLoadJobManifestFileHeader(bulkLoadManifestFormatVersion, self->bulkDumpJobManager.taskManifestMap.size())
 	        .toString());
 	content->append(bulkLoadJobManifestLineTerminator);
-	state std::map<Key, BulkLoadManifest>::iterator iter = self->bulkDumpJobManager.taskManifestMap.begin();
+	auto iter = self->bulkDumpJobManager.taskManifestMap.begin();
 	for (; iter != self->bulkDumpJobManager.taskManifestMap.end(); iter++) {
 		content->append(BulkLoadJobFileManifestEntry(iter->second).toString());
 		content->append(bulkLoadJobManifestLineTerminator);
 		counter++;
 		if (counter % SERVER_KNOBS->DD_BULKDUMP_BUILD_JOB_MANIFEST_BATCH_SIZE) {
-			wait(yield());
+			co_await yield();
 		}
 	}
 	ASSERT(!content->empty());
 	resetFileFolder(workFolder);
-	wait(writeBulkFileBytes(localJobManifestFilePath, content));
+	co_await writeBulkFileBytes(localJobManifestFilePath, content);
 	TraceEvent(SevInfo, "GenerateBulkDumpJobManifestWriteLocal", self->ddId)
 	    .detail("LocalJobManifestFilePath", localJobManifestFilePath)
 	    .detail("ContentSize", content->size());
-	return Void();
 }
 
-ACTOR Future<Void> bulkDumpUploadJobManifestFile(Reference<DataDistributor> self) {
+Future<Void> bulkDumpUploadJobManifestFile(Reference<DataDistributor> self) {
 	if (self->folder.empty()) {
-		return Void();
+		co_return;
 	}
-	state UID jobId = self->bulkDumpJobManager.jobState.getJobId();
-	state std::string jobRoot = self->bulkDumpJobManager.jobState.getJobRoot();
-	state BulkLoadTransportMethod transportMethod = self->bulkDumpJobManager.jobState.getTransportMethod();
+	UID jobId = self->bulkDumpJobManager.jobState.getJobId();
+	std::string jobRoot = self->bulkDumpJobManager.jobState.getJobRoot();
+	BulkLoadTransportMethod transportMethod = self->bulkDumpJobManager.jobState.getTransportMethod();
 	// Upload job manifest file
 	ASSERT(!self->bulkDumpFolder.empty());
-	state std::string localFolder = getBulkLoadJobRoot(self->bulkDumpFolder, jobId);
-	state std::string remoteFolder = getBulkLoadJobRoot(jobRoot, jobId);
-	state std::string jobManifestFileName = getBulkLoadJobManifestFileName();
-	state std::string localJobManifestFilePath = joinPath(localFolder, jobManifestFileName);
-	wait(generateLocalBulkDumpJobManifestFile(self, localFolder, localJobManifestFilePath));
-	wait(uploadBulkDumpJobManifestFile(
-	    transportMethod, localJobManifestFilePath, remoteFolder, jobManifestFileName, self->ddId));
+	std::string localFolder = getBulkLoadJobRoot(self->bulkDumpFolder, jobId);
+	std::string remoteFolder = getBulkLoadJobRoot(jobRoot, jobId);
+	std::string jobManifestFileName = getBulkLoadJobManifestFileName();
+	std::string localJobManifestFilePath = joinPath(localFolder, jobManifestFileName);
+	co_await generateLocalBulkDumpJobManifestFile(self, localFolder, localJobManifestFilePath);
+	co_await uploadBulkDumpJobManifestFile(
+	    transportMethod, localJobManifestFilePath, remoteFolder, jobManifestFileName, self->ddId);
 	clearFileFolder(localFolder, self->ddId, /*ignoreError=*/true); // best effort to clear the local folder
 	TraceEvent(SevInfo, "DDBulkDumpJobManifestUploaded", self->ddId)
 	    .detail("JobId", jobId)
@@ -2452,27 +2596,30 @@ ACTOR Future<Void> bulkDumpUploadJobManifestFile(Reference<DataDistributor> self
 	    .detail("RemoteFolder", remoteFolder)
 	    .detail("JobManifestFileName", jobManifestFileName)
 	    .detail("TaskCount", self->bulkDumpJobManager.taskManifestMap.size());
-	return Void();
 }
 
 // Setup self->bulkDumpJobManager.jobManifest by scanning the entire bulkDump key space
-ACTOR Future<Void> getBulkLoadJobManifestData(Reference<DataDistributor> self) {
-	state UID jobId = self->bulkDumpJobManager.jobState.getJobId();
-	state KeyRange jobRange = self->bulkDumpJobManager.jobState.getJobRange();
-	state std::map<Key, BulkLoadManifest> manifests;
-	state Database cx = self->txnProcessor->context();
-	state RangeResult bulkDumpResult;
-	state Key beginKey = jobRange.begin;
-	state Key endKey = jobRange.end;
-	state KeyRange rangeToRead;
-	state Transaction tr(cx);
+Future<Void> getBulkLoadJobManifestData(Reference<DataDistributor> self) {
+	UID jobId = self->bulkDumpJobManager.jobState.getJobId();
+	KeyRange jobRange = self->bulkDumpJobManager.jobState.getJobRange();
+	std::map<Key, BulkLoadManifest> manifests;
+	Database cx = self->txnProcessor->context();
+	RangeResult bulkDumpResult;
+	Key beginKey = jobRange.begin;
+	Key endKey = jobRange.end;
+	KeyRange rangeToRead;
+	Transaction tr(cx);
 	self->bulkDumpJobManager.taskManifestMap.clear();
 	while (beginKey < endKey) {
+		Error err;
 		try {
 			rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
 			bulkDumpResult.clear();
-			wait(store(bulkDumpResult, krmGetRanges(&tr, bulkDumpPrefix, rangeToRead)));
-			for (int i = 0; i < bulkDumpResult.size() - 1; i++) {
+			bulkDumpResult = co_await krmGetRanges(&tr, bulkDumpPrefix, rangeToRead);
+			if (bulkDumpResult.empty()) {
+				break;
+			}
+			for (int i = 0; i < static_cast<int>(bulkDumpResult.size()) - 1; i++) {
 				ASSERT(!bulkDumpResult[i].value.empty());
 				BulkDumpState bulkDumpState = decodeBulkDumpState(bulkDumpResult[i].value);
 				if (bulkDumpState.getJobId() != jobId) {
@@ -2482,97 +2629,98 @@ ACTOR Future<Void> getBulkLoadJobManifestData(Reference<DataDistributor> self) {
 				       bulkDumpState.getManifest().getRange() ==
 				           KeyRangeRef(bulkDumpResult[i].key, bulkDumpResult[i + 1].key));
 				// Important! This is how to build job manifest file.
-				// The taskManifestMap is a sorted map. Each item is a manifest. The key of the item is the beginkey of
-				// the manifest. The endkey of the manifest must be the map key of the next item.
-				// When doing bulkload job, we decode the map in the same way. Please check scheduleBulkLoadJob where
-				// we decode the job manifest file.
+				// The taskManifestMap is a sorted map. Each item is a manifest. The key of the item is the beginkey
+				// of the manifest. The endkey of the manifest must be the map key of the next item. When doing
+				// bulkload job, we decode the map in the same way. Please check scheduleBulkLoadJob where we decode
+				// the job manifest file.
 				auto res = self->bulkDumpJobManager.taskManifestMap.insert(
 				    { bulkDumpState.getManifest().getBeginKey(), bulkDumpState.getManifest() });
 				ASSERT(res.second);
 			}
 			beginKey = bulkDumpResult.back().key;
+			continue;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
 		}
+		co_await tr.onError(err);
 	}
 	ASSERT(!self->bulkDumpJobManager.taskManifestMap.empty());
-	return Void();
 }
 
-ACTOR Future<Optional<BulkDumpState>> getAliveBulkDumpJob(Database cx) {
-	state Transaction tr(cx);
-	state Optional<BulkDumpState> res;
-	loop {
+Future<Optional<BulkDumpState>> getAliveBulkDumpJob(Database cx) {
+	Transaction tr(cx);
+	Optional<BulkDumpState> res;
+	while (true) {
+		Error err;
 		try {
-			wait(store(res, getSubmittedBulkDumpJob(&tr)));
+			res = co_await getSubmittedBulkDumpJob(&tr);
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return res;
+	co_return res;
 }
 
-ACTOR Future<Void> bulkDumpManager(Reference<DataDistributor> self) {
-	state Database cx = self->txnProcessor->context();
-	state Optional<BulkDumpState> job;
-	wait(store(job, getAliveBulkDumpJob(cx)));
+Future<Void> bulkDumpManager(Reference<DataDistributor> self) {
+	Database cx = self->txnProcessor->context();
+	Optional<BulkDumpState> job = co_await getAliveBulkDumpJob(cx);
 	if (!job.present()) {
-		return Void();
+		co_return;
 	}
-	state UID jobId = job.get().getJobId();
+	UID jobId = job.get().getJobId();
 	TraceEvent(SevInfo, "DDBulkDumpManagerGotJob", self->ddId)
 	    .setMaxEventLength(-1)
 	    .setMaxFieldLength(-1)
 	    .detail("Job", job.get().toString());
 	self->bulkDumpJobManager = DDBulkDumpJobManager(job.get());
-	loop {
-		bool allComplete = wait(checkBulkDumpJobComplete(self));
+	while (true) {
+		bool allComplete = co_await checkBulkDumpJobComplete(self);
 		if (allComplete) {
 			TraceEvent(SevInfo, "DDBulkDumpManagerJobAllTaskComplete", self->ddId).detail("JobId", jobId);
 			// Generate the job manifest file for bulkload.
 			// The job manifest file is the global map between ranges and their corresponding manifest file.
 			// When bulkload job loads a range, the job relies on this map to find the correct
 			// manifest file set to get the necessary information to perform the bulk load.
-			wait(getBulkLoadJobManifestData(self));
+			co_await getBulkLoadJobManifestData(self);
 			TraceEvent(SevInfo, "DDBulkDumpManagerJobManifestGenerated", self->ddId)
 			    .detail("JobId", jobId)
 			    .detail("NumManifest", self->bulkDumpJobManager.taskManifestMap.size());
 			// At this point, we have all manifest data to generate the job manifest file.
 			// Generate the file at a local folder at first and then upload the file to the remote.
-			wait(bulkDumpUploadJobManifestFile(self));
+			co_await bulkDumpUploadJobManifestFile(self);
 			TraceEvent(SevInfo, "DDBulkDumpManagerJobManifestUpload", self->ddId).detail("JobId", jobId);
 			// Finally, clear all bulkdump metadata
-			wait(cancelBulkDumpJob(cx, jobId));
+			co_await cancelBulkDumpJob(cx, jobId);
 			TraceEvent(SevInfo, "DDBulkDumpManagerMetadataCleared", self->ddId).detail("JobId", jobId);
 			break; // end
 		} else {
 			// Schedule the job
-			wait(scheduleBulkDumpJob(self));
+			co_await scheduleBulkDumpJob(self);
 		}
-		wait(delay(SERVER_KNOBS->DD_BULKDUMP_SCHEDULE_MIN_INTERVAL_SEC));
+		co_await delay(SERVER_KNOBS->DD_BULKDUMP_SCHEDULE_MIN_INTERVAL_SEC);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> bulkDumpCore(Reference<DataDistributor> self, Future<Void> readyToStart) {
-	wait(readyToStart);
-	state Database cx = self->txnProcessor->context();
+Future<Void> bulkDumpCore(Reference<DataDistributor> self, Future<Void> readyToStart) {
+	co_await readyToStart;
+	Database cx = self->txnProcessor->context();
 	TraceEvent(SevInfo, "DDBulkDumpCoreStart", self->ddId);
-	loop {
+	while (true) {
 		// Dynamically check if BulkDump mode is enabled
-		state int currentMode = wait(getBulkDumpMode(cx));
+		int currentMode = co_await getBulkDumpMode(cx);
 		if (!bulkDumpIsEnabled(currentMode)) {
 			// Mode is disabled - use a longer polling interval to avoid keeping DD "active"
 			// during QuietDatabase checks. BulkDumpTaskFunc will eventually enable the mode.
-			wait(delay(60.0));
+			co_await delay(60.0);
 			continue;
 		}
 		try {
-			wait(bulkDumpManager(self));
+			co_await bulkDumpManager(self);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
@@ -2582,19 +2730,19 @@ ACTOR Future<Void> bulkDumpCore(Reference<DataDistributor> self, Future<Void> re
 				throw e;
 			}
 		}
-		wait(delay(SERVER_KNOBS->DD_BULKDUMP_SCHEDULE_MIN_INTERVAL_SEC));
+		co_await delay(SERVER_KNOBS->DD_BULKDUMP_SCHEDULE_MIN_INTERVAL_SEC);
 	}
 }
 
 // Runs the data distribution algorithm for FDB, including the DD Queue, DD tracker, and DD team collection
-ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
-                                    PromiseStream<GetMetricsListRequest> getShardMetricsList,
-                                    IsMocked isMocked) {
+Future<Void> dataDistribution(Reference<DataDistributor> self,
+                              PromiseStream<GetMetricsListRequest> getShardMetricsList,
+                              IsMocked isMocked) {
 
 	if (!isMocked) {
 		Database cx = openDBOnServer(self->dbInfo, TaskPriority::DataDistributionLaunch, LockAware::True);
 		cx->locationCacheSize = SERVER_KNOBS->DD_LOCATION_CACHE_SIZE;
-		self->txnProcessor = Reference<IDDTxnProcessor>(new DDTxnProcessor(cx));
+		self->txnProcessor = makeReference<DDTxnProcessor>(cx);
 	} else {
 		ASSERT(self->txnProcessor.isValid() && self->txnProcessor->isMocked());
 	}
@@ -2603,37 +2751,41 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 
 	// Make sure that the watcher has established a baseline before init() below so the watcher will
 	// see any changes that occur after init() has read the config state.
-	wait(self->initDDConfigWatch());
+	co_await self->initDDConfigWatch();
 
 	TraceEvent(SevInfo, "DataDistributionInitProgress", self->ddId).detail("Phase", "DDConfigWatch Initialized");
 
-	loop {
+	while (true) {
+		double ddStartTime = now();
 		self->context->trackerCancelled = false;
 		// whether all initial shard are tracked
 		self->initialized = Promise<Void>();
 
 		// Stored outside of data distribution tracker to avoid slow tasks
 		// when tracker is cancelled
-		state KeyRangeMap<ShardTrackedData> shards;
-		state Promise<UID> removeFailedServer;
+		KeyRangeMap<ShardTrackedData> shards;
+		Promise<UID> removeFailedServer;
+		Error caughtErr;
+		bool hasCaughtErr = false;
 		try {
+			co_await DataDistributor::init(self);
 
-			wait(DataDistributor::init(self));
+			TraceEvent("DDInitDataLoaded", self->ddId).detail("ElapsedSeconds", now() - ddStartTime);
 
 			TraceEvent(SevInfo, "DataDistributionInitProgress", self->ddId).detail("Phase", "Metadata Initialized");
 
-			state PromiseStream<Promise<int64_t>> getAverageShardBytes;
-			state PromiseStream<RebalanceStorageQueueRequest> triggerStorageQueueRebalance;
-			state PromiseStream<Promise<int>> getUnhealthyRelocationCount;
-			state PromiseStream<GetMetricsRequest> getShardMetrics;
-			state PromiseStream<GetTopKMetricsRequest> getTopKShardMetrics;
-			state Reference<AsyncVar<bool>> processingUnhealthy(new AsyncVar<bool>(false));
-			state Reference<AsyncVar<bool>> processingWiggle(new AsyncVar<bool>(false));
+			PromiseStream<Promise<int64_t>> getAverageShardBytes;
+			PromiseStream<RebalanceStorageQueueRequest> triggerStorageQueueRebalance;
+			PromiseStream<Promise<int>> getUnhealthyRelocationCount;
+			PromiseStream<GetMetricsRequest> getShardMetrics;
+			PromiseStream<GetTopKMetricsRequest> getTopKShardMetrics;
+			Reference<AsyncVar<bool>> processingUnhealthy(new AsyncVar<bool>(false));
+			Reference<AsyncVar<bool>> processingWiggle(new AsyncVar<bool>(false));
 
 			self->shardsAffectedByTeamFailure = makeReference<ShardsAffectedByTeamFailure>();
 			self->physicalShardCollection = makeReference<PhysicalShardCollection>(self->txnProcessor);
 			self->bulkLoadTaskCollection = makeReference<BulkLoadTaskCollection>(self->ddId);
-			wait(self->resumeRelocations());
+			co_await self->resumeRelocations();
 
 			TraceEvent(SevInfo, "DataDistributionInitProgress", self->ddId).detail("Phase", "Relocation Resumed");
 
@@ -2804,6 +2956,9 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 				}
 			} else {
 				self->bulkLoadTaskCollection->removeBulkLoadJobRange();
+				// Monitor for bulkLoadMode changes and spawn actors dynamically.
+				// This is critical for restore operations which set bulkLoadMode=1 after DD starts.
+				actors.push_back(monitorBulkLoadModeAndSpawnActors(self, self->initialized.getFuture()));
 			}
 
 			// Always spawn bulkDumpCore - it will dynamically check the mode
@@ -2815,16 +2970,23 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 			    .detail("InitialMode", self->initData->bulkDumpMode);
 			actors.push_back(bulkDumpCore(self, self->initialized.getFuture()));
 
-			wait(waitForAll(actors));
+			actors.push_back(periodicAuditLocationMetadata(self));
+
+			co_await waitForAll(actors);
 			ASSERT_WE_THINK(false);
-			return Void();
+			co_return;
 		} catch (Error& e) {
+			caughtErr = e;
+			hasCaughtErr = true;
+		}
+		if (hasCaughtErr) {
 			self->context->tracker.clear();
 			self->context->ddQueue.clear();
 			self->context->markTrackerCancelled();
-			state Error err = e;
-			TraceEvent("DataDistributorDestroyTeamCollections", self->ddId).error(e);
-			state std::vector<UID> teamForDroppedRange;
+			Error err = caughtErr;
+			TraceEvent("DataDistributorDestroyTeamCollections", self->ddId).error(caughtErr);
+			TraceEvent(SevWarn, "DDExiting", self->ddId).error(caughtErr);
+			std::vector<UID> teamForDroppedRange;
 			if (removeFailedServer.getFuture().isReady() && !removeFailedServer.getFuture().isError()) {
 				// Choose a random healthy team to host the to-be-dropped range.
 				const UID serverID = removeFailedServer.getFuture().get();
@@ -2846,21 +3008,21 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributor> self,
 					TraceEvent(SevWarn, "DataDistributorCancelled");
 				}
 				shards.clear();
-				throw e;
+				throw caughtErr;
 			} else {
-				wait(shards.clearAsync());
+				co_await shards.clearAsync();
 			}
 			TraceEvent("DataDistributorTeamCollectionsDestroyed", self->ddId).error(err);
 			if (removeFailedServer.getFuture().isReady() && !removeFailedServer.getFuture().isError()) {
 				TraceEvent("RemoveFailedServer", removeFailedServer.getFuture().get()).error(err);
-				wait(self->removeKeysFromFailedServer(removeFailedServer.getFuture().get(), teamForDroppedRange));
-				wait(self->removeStorageServer(removeFailedServer.getFuture().get()));
+				co_await self->removeKeysFromFailedServer(removeFailedServer.getFuture().get(), teamForDroppedRange);
+				co_await self->removeStorageServer(removeFailedServer.getFuture().get());
 			} else {
 				if (err.code() != error_code_movekeys_conflict && err.code() != error_code_dd_config_changed) {
 					throw err;
 				}
 
-				bool ddEnabled = wait(self->isDataDistributionEnabled());
+				bool ddEnabled = co_await self->isDataDistributionEnabled();
 				TraceEvent("DataDistributionError", self->ddId).error(err).detail("DataDistributionEnabled", ddEnabled);
 				if (ddEnabled) {
 					throw err;
@@ -2886,9 +3048,9 @@ static std::set<int> const& normalDataDistributorErrors() {
 	return s;
 }
 
-ACTOR template <class Req>
+template <class Req>
 Future<Void> sendSnapReq(RequestStream<Req> stream, Req req, Error e) {
-	ErrorOr<REPLY_TYPE(Req)> reply = wait(stream.tryGetReply(req));
+	ErrorOr<REPLY_TYPE(Req)> reply = co_await stream.tryGetReply(req);
 	if (reply.isError()) {
 		TraceEvent("SnapDataDistributor_ReqError")
 		    .errorUnsuppressed(reply.getError())
@@ -2897,14 +3059,13 @@ Future<Void> sendSnapReq(RequestStream<Req> stream, Req req, Error e) {
 		    .detail("PeerAddress", stream.getEndpoint().getPrimaryAddress());
 		throw e;
 	}
-	return Void();
 }
 
-ACTOR Future<ErrorOr<Void>> trySendSnapReq(RequestStream<WorkerSnapRequest> stream, WorkerSnapRequest req) {
-	state int snapReqRetry = 0;
-	state double snapRetryBackoff = FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY;
-	loop {
-		ErrorOr<REPLY_TYPE(WorkerSnapRequest)> reply = wait(stream.tryGetReply(req));
+Future<ErrorOr<Void>> trySendSnapReq(RequestStream<WorkerSnapRequest> stream, WorkerSnapRequest req) {
+	int snapReqRetry = 0;
+	double snapRetryBackoff = FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY;
+	while (true) {
+		ErrorOr<REPLY_TYPE(WorkerSnapRequest)> reply = co_await stream.tryGetReply(req);
 		if (reply.isError()) {
 			TraceEvent("SnapDataDistributor_ReqError")
 			    .errorUnsuppressed(reply.getError())
@@ -2913,53 +3074,54 @@ ACTOR Future<ErrorOr<Void>> trySendSnapReq(RequestStream<WorkerSnapRequest> stre
 			    .detail("Retry", snapReqRetry);
 			if (reply.getError().code() != error_code_request_maybe_delivered ||
 			    ++snapReqRetry > SERVER_KNOBS->SNAP_NETWORK_FAILURE_RETRY_LIMIT)
-				return ErrorOr<Void>(reply.getError());
+				co_return ErrorOr<Void>(reply.getError());
 			else {
 				// retry for network failures with same snap UID to avoid snapshot twice
 				req = WorkerSnapRequest(req.snapPayload, req.snapUID, req.role);
-				wait(delay(snapRetryBackoff));
+				co_await delay(snapRetryBackoff);
 				snapRetryBackoff = snapRetryBackoff * 2;
 			}
 		} else
 			break;
 	}
-	return ErrorOr<Void>(Void());
+	co_return ErrorOr<Void>(Void());
 }
 
-ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>> getStatefulWorkers(
+Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>> getStatefulWorkers(
     Database cx,
     Reference<AsyncVar<ServerDBInfo> const> dbInfo,
     std::vector<TLogInterface>* tlogs,
     int* storageFaultTolerance) {
-	state std::map<NetworkAddress, std::pair<WorkerInterface, std::string>> result;
-	state std::map<NetworkAddress, WorkerInterface> workersMap;
-	state Transaction tr(cx);
-	state DatabaseConfiguration configuration;
-	loop {
+	std::map<NetworkAddress, std::pair<WorkerInterface, std::string>> result;
+	std::map<NetworkAddress, WorkerInterface> workersMap;
+	Transaction tr(cx);
+	DatabaseConfiguration configuration;
+	while (true) {
+		Error err;
 		try {
 			// necessary options
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 
 			// get database configuration
-			DatabaseConfiguration _configuration = wait(getDatabaseConfiguration(&tr));
+			DatabaseConfiguration _configuration = co_await getDatabaseConfiguration(&tr);
 			configuration = _configuration;
 
 			// get storages
-			RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
+			RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
-			state std::vector<StorageServerInterface> storageServers;
+			std::vector<StorageServerInterface> storageServers;
 			storageServers.reserve(serverList.size());
 			for (int i = 0; i < serverList.size(); i++)
 				storageServers.push_back(decodeServerListValue(serverList[i].value));
 
 			// get workers
-			state std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
+			std::vector<WorkerDetails> workers = co_await getWorkers(dbInfo);
 			for (const auto& worker : workers) {
 				workersMap[worker.interf.address()] = worker.interf;
 			}
 
-			Optional<Value> regionsValue = wait(tr.get("usable_regions"_sr.withPrefix(configKeysPrefix)));
+			Optional<Value> regionsValue = co_await tr.get("usable_regions"_sr.withPrefix(configKeysPrefix));
 			int usableRegions = 1;
 			if (regionsValue.present()) {
 				usableRegions = atoi(regionsValue.get().toString().c_str());
@@ -3012,13 +3174,13 @@ ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>>
 				}
 			}
 
-			Optional<Value> coordinators = wait(tr.get(coordinatorsKey));
+			Optional<Value> coordinators = co_await tr.get(coordinatorsKey);
 			if (!coordinators.present()) {
 				CODE_PROBE(true, "Failed to read the coordinatorsKey", probe::decoration::rare);
 				throw operation_failed();
 			}
 			ClusterConnectionString ccs(coordinators.get().toString());
-			std::vector<NetworkAddress> coordinatorsAddr = wait(ccs.tryResolveHostnames());
+			std::vector<NetworkAddress> coordinatorsAddr = co_await ccs.tryResolveHostnames();
 			std::set<NetworkAddress> coordinatorsAddrSet(coordinatorsAddr.begin(), coordinatorsAddr.end());
 			for (const auto& worker : workers) {
 				// Note : only considers second address for coordinators,
@@ -3053,20 +3215,27 @@ ACTOR Future<std::map<NetworkAddress, std::pair<WorkerInterface, std::string>>>
 					}
 				}
 			}
-			return result;
+			co_return result;
 		} catch (Error& e) {
-			wait(tr.onError(e));
-			result.clear();
+			err = e;
 		}
+		co_await tr.onError(err);
+		result.clear();
 	}
 }
 
-// FIXME: explain what this is trying to accomplish
-ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<AsyncVar<ServerDBInfo> const> db) {
-	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
-
-	state ReadYourWritesTransaction tr(cx);
-	loop {
+// Coordinates the data-distributor side of snapshot creation by preventing
+// recovery, pausing TLog pops, snapshotting stateful workers, and resuming
+// TLog pops before reporting completion.
+Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<AsyncVar<ServerDBInfo> const> db) {
+	Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
+	static auto* setRecoveryCounters = makeCounters("/dd/ddSnapSetRecovery");
+	static auto* clearRecoveryCounters = makeCounters("/dd/ddSnapClearRecovery");
+
+	ReadYourWritesTransaction tr(cx);
+	while (true) {
+		setRecoveryCounters->started->increment(1);
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -3074,41 +3243,46 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 			    .detail("SnapPayload", snapReq.snapPayload)
 			    .detail("SnapUID", snapReq.snapUID);
 			tr.set(writeRecoveryKey, writeRecoveryKeyTrue);
-			wait(tr.commit());
+			co_await tr.commit();
+			setRecoveryCounters->committed->increment(1);
 			break;
 		} catch (Error& e) {
-			TraceEvent("SnapDataDistributor_WriteFlagError").error(e);
-			wait(tr.onError(e));
+			setRecoveryCounters->aborted->increment(1);
+			err = e;
 		}
+		TraceEvent("SnapDataDistributor_WriteFlagError").error(err);
+		co_await tr.onError(err);
 	}
 	TraceEvent("SnapDataDistributor_SnapReqEnter")
 	    .detail("SnapPayload", snapReq.snapPayload)
 	    .detail("SnapUID", snapReq.snapUID);
+	Error caughtErr;
+	bool hasCaughtErr = false;
 	try {
-		state std::vector<TLogInterface> tlogs = db->get().logSystemConfig.allLocalLogs(false);
+		std::vector<TLogInterface> tlogs = db->get().logSystemConfig.allLocalLogs(false);
 		std::vector<Future<Void>> disablePops;
 		disablePops.reserve(tlogs.size());
 		for (const auto& tlog : tlogs) {
 			disablePops.push_back(sendSnapReq(
 			    tlog.disablePopRequest, TLogDisablePopRequest{ snapReq.snapUID }, snap_disable_tlog_pop_failed()));
 		}
-		wait(waitForAll(disablePops));
+		co_await waitForAll(disablePops);
 
 		TraceEvent("SnapDataDistributor_AfterDisableTLogPop")
 		    .detail("SnapPayload", snapReq.snapPayload)
 		    .detail("SnapUID", snapReq.snapUID);
 
-		state int storageFaultTolerance;
-		state std::map<NetworkAddress, std::pair<WorkerInterface, std::string>> statefulWorkers =
-		    wait(transformErrors(getStatefulWorkers(cx, db, &tlogs, &storageFaultTolerance), snap_storage_failed()));
+		int storageFaultTolerance{ 0 };
+		std::map<NetworkAddress, std::pair<WorkerInterface, std::string>> statefulWorkers =
+		    co_await transformErrors(getStatefulWorkers(cx, db, &tlogs, &storageFaultTolerance), snap_storage_failed());
 
 		TraceEvent("SnapDataDistributor_GotStatefulWorkers")
 		    .detail("SnapPayload", snapReq.snapPayload)
 		    .detail("SnapUID", snapReq.snapUID)
 		    .detail("StorageFaultTolerance", storageFaultTolerance);
 
-		// we need to snapshot storage nodes before snapshot any tlogs
-		// FIXME: if it's non-obvious enough to comment about, then also explain why
+		// Snapshot storage nodes before TLogs so storage files are captured
+		// while TLogs still retain the mutations needed to recover them.
 		std::vector<Future<ErrorOr<Void>>> storageSnapReqs;
 		for (const auto& [addr, entry] : statefulWorkers) {
 			auto& [interf, role] = entry;
@@ -3116,7 +3290,7 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 				storageSnapReqs.push_back(trySendSnapReq(
 				    interf.workerSnapReq, WorkerSnapRequest(snapReq.snapPayload, snapReq.snapUID, "storage"_sr)));
 		}
-		wait(waitForMost(storageSnapReqs, storageFaultTolerance, snap_storage_failed()));
+		co_await waitForMost(storageSnapReqs, storageFaultTolerance, snap_storage_failed());
 		TraceEvent("SnapDataDistributor_AfterSnapStorage")
 		    .detail("SnapPayload", snapReq.snapPayload)
 		    .detail("SnapUID", snapReq.snapUID);
@@ -3129,7 +3303,7 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 				tLogSnapReqs.push_back(trySendSnapReq(
 				    interf.workerSnapReq, WorkerSnapRequest(snapReq.snapPayload, snapReq.snapUID, "tlog"_sr)));
 		}
-		wait(waitForMost(tLogSnapReqs, 0, snap_tlog_failed()));
+		co_await waitForMost(tLogSnapReqs, 0, snap_tlog_failed());
 
 		TraceEvent("SnapDataDistributor_AfterTLogStorage")
 		    .detail("SnapPayload", snapReq.snapPayload)
@@ -3141,7 +3315,7 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 			enablePops.push_back(sendSnapReq(
 			    tlog.enablePopRequest, TLogEnablePopRequest{ snapReq.snapUID }, snap_enable_tlog_pop_failed()));
 		}
-		wait(waitForAll(enablePops));
+		co_await waitForAll(enablePops);
 
 		TraceEvent("SnapDataDistributor_AfterEnableTLogPops")
 		    .detail("SnapPayload", snapReq.snapPayload)
@@ -3159,13 +3333,15 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 		auto const coordFaultTolerance = std::min<int>(
 		    std::max<int>(0, (coordSnapReqs.size() - 1) / 2),
 		    g_network->isSimulated() ? coordSnapReqs.size() : SERVER_KNOBS->MAX_COORDINATOR_SNAPSHOT_FAULT_TOLERANCE);
-		wait(waitForMost(coordSnapReqs, coordFaultTolerance, snap_coord_failed()));
+		co_await waitForMost(coordSnapReqs, coordFaultTolerance, snap_coord_failed());
 
 		TraceEvent("SnapDataDistributor_AfterSnapCoords")
 		    .detail("SnapPayload", snapReq.snapPayload)
 		    .detail("SnapUID", snapReq.snapUID);
 		tr.reset();
-		loop {
+		while (true) {
+			clearRecoveryCounters->started->increment(1);
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -3173,15 +3349,22 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 				    .detail("SnapPayload", snapReq.snapPayload)
 				    .detail("SnapUID", snapReq.snapUID);
 				tr.clear(writeRecoveryKey);
-				wait(tr.commit());
+				co_await tr.commit();
+				clearRecoveryCounters->committed->increment(1);
 				break;
 			} catch (Error& e) {
-				TraceEvent("SnapDataDistributor_ClearFlagError").error(e);
-				wait(tr.onError(e));
+				clearRecoveryCounters->aborted->increment(1);
+				err = e;
 			}
+			TraceEvent("SnapDataDistributor_ClearFlagError").error(err);
+			co_await tr.onError(err);
 		}
 	} catch (Error& err) {
-		state Error e = err;
+		caughtErr = err;
+		hasCaughtErr = true;
+	}
+	if (hasCaughtErr) {
+		Error e = caughtErr;
 		TraceEvent("SnapDataDistributor_SnapReqExit")
 		    .errorUnsuppressed(e)
 		    .detail("SnapPayload", snapReq.snapPayload)
@@ -3198,24 +3381,23 @@ ACTOR Future<Void> ddSnapCreateCore(DistributorSnapRequest snapReq, Reference<As
 					    throwErrorOr(tlog.enablePopRequest.tryGetReply(TLogEnablePopRequest(snapReq.snapUID))),
 					    snap_enable_tlog_pop_failed()));
 				}
-				wait(waitForAll(enablePops));
+				co_await waitForAll(enablePops);
 			} catch (Error& error) {
 				TraceEvent(SevDebug, "IgnoreEnableTLogPopFailure").log();
 			}
 		}
 		throw e;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> ddSnapCreate(
+Future<Void> ddSnapCreate(
     DistributorSnapRequest snapReq,
     Reference<AsyncVar<ServerDBInfo> const> db,
     DDEnabledState* ddEnabledState,
     std::map<UID, DistributorSnapRequest>* ddSnapMap /* ongoing snapshot requests */,
     std::map<UID, ErrorOr<Void>>*
         ddSnapResultMap /* finished snapshot requests, expired in SNAP_MINIMUM_TIME_GAP seconds */) {
-	state Future<Void> dbInfoChange = db->onChange();
+	Future<Void> dbInfoChange = db->onChange();
 	if (!ddEnabledState->trySetSnapshot(snapReq.snapUID)) {
 		// disable DD before doing snapCreate, if previous snap req has already disabled DD then this operation
 		// fails here
@@ -3223,34 +3405,32 @@ ACTOR Future<Void> ddSnapCreate(
 		ddSnapMap->at(snapReq.snapUID).reply.sendError(operation_failed());
 		ddSnapMap->erase(snapReq.snapUID);
 		(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(operation_failed());
-		return Void();
+		co_return;
 	}
 	try {
-		choose {
-			when(wait(dbInfoChange)) {
-				TraceEvent("SnapDDCreateDBInfoChanged")
-				    .detail("SnapPayload", snapReq.snapPayload)
-				    .detail("SnapUID", snapReq.snapUID);
-				ddSnapMap->at(snapReq.snapUID).reply.sendError(snap_with_recovery_unsupported());
-				ddSnapMap->erase(snapReq.snapUID);
-				(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(snap_with_recovery_unsupported());
-			}
-			when(wait(ddSnapCreateCore(snapReq, db))) {
-				TraceEvent("SnapDDCreateSuccess")
-				    .detail("SnapPayload", snapReq.snapPayload)
-				    .detail("SnapUID", snapReq.snapUID);
-				ddSnapMap->at(snapReq.snapUID).reply.send(Void());
-				ddSnapMap->erase(snapReq.snapUID);
-				(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(Void());
-			}
-			when(wait(delay(SERVER_KNOBS->SNAP_CREATE_MAX_TIMEOUT))) {
-				TraceEvent("SnapDDCreateTimedOut")
-				    .detail("SnapPayload", snapReq.snapPayload)
-				    .detail("SnapUID", snapReq.snapUID);
-				ddSnapMap->at(snapReq.snapUID).reply.sendError(timed_out());
-				ddSnapMap->erase(snapReq.snapUID);
-				(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(timed_out());
-			}
+		auto res =
+		    co_await race(dbInfoChange, ddSnapCreateCore(snapReq, db), delay(SERVER_KNOBS->SNAP_CREATE_MAX_TIMEOUT));
+		if (res.index() == 0) {
+			TraceEvent("SnapDDCreateDBInfoChanged")
+			    .detail("SnapPayload", snapReq.snapPayload)
+			    .detail("SnapUID", snapReq.snapUID);
+			ddSnapMap->at(snapReq.snapUID).reply.sendError(snap_with_recovery_unsupported());
+			ddSnapMap->erase(snapReq.snapUID);
+			(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(snap_with_recovery_unsupported());
+		} else if (res.index() == 1) {
+			TraceEvent("SnapDDCreateSuccess")
+			    .detail("SnapPayload", snapReq.snapPayload)
+			    .detail("SnapUID", snapReq.snapUID);
+			ddSnapMap->at(snapReq.snapUID).reply.send(Void());
+			ddSnapMap->erase(snapReq.snapUID);
+			(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(Void());
+		} else if (res.index() == 2) {
+			TraceEvent("SnapDDCreateTimedOut")
+			    .detail("SnapPayload", snapReq.snapPayload)
+			    .detail("SnapUID", snapReq.snapUID);
+			ddSnapMap->at(snapReq.snapUID).reply.sendError(timed_out());
+			ddSnapMap->erase(snapReq.snapUID);
+			(*ddSnapResultMap)[snapReq.snapUID] = ErrorOr<Void>(timed_out());
 		}
 	} catch (Error& e) {
 		TraceEvent("SnapDDCreateError")
@@ -3269,28 +3449,27 @@ ACTOR Future<Void> ddSnapCreate(
 	}
 	bool success = ddEnabledState->trySetEnabled(snapReq.snapUID);
 	ASSERT(success);
-	return Void();
 }
 
 // FIXME: explain purpose
-ACTOR Future<Void> ddExclusionSafetyCheck(DistributorExclusionSafetyCheckRequest req,
-                                          Reference<DataDistributor> self,
-                                          Database cx) {
+Future<Void> ddExclusionSafetyCheck(DistributorExclusionSafetyCheckRequest req,
+                                    Reference<DataDistributor> self,
+                                    Database cx) {
 	TraceEvent("DDExclusionSafetyCheckBegin", self->ddId).log();
-	std::vector<StorageServerInterface> ssis = wait(getStorageServers(cx));
+	std::vector<StorageServerInterface> ssis = co_await getStorageServers(cx);
 	DistributorExclusionSafetyCheckReply reply(true);
 	if (!self->teamCollection) {
 		TraceEvent("DDExclusionSafetyCheckTeamCollectionInvalid", self->ddId).log();
 		reply.safe = false;
 		req.reply.send(reply);
-		return Void();
+		co_return;
 	}
 	// If there is only 1 team, unsafe to mark failed: team building can get stuck due to lack of servers left
 	if (self->teamCollection->teams.size() <= 1) {
 		TraceEvent("DDExclusionSafetyCheckNotEnoughTeams", self->ddId).log();
 		reply.safe = false;
 		req.reply.send(reply);
-		return Void();
+		co_return;
 	}
 	std::vector<UID> excludeServerIDs;
 	// Go through storage server interfaces and translate Address -> server ID (UID)
@@ -3305,7 +3484,6 @@ ACTOR Future<Void> ddExclusionSafetyCheck(DistributorExclusionSafetyCheckRequest
 	reply.safe = self->teamCollection->exclusionSafetyCheck(excludeServerIDs);
 	TraceEvent("DDExclusionSafetyCheckFinish", self->ddId).log();
 	req.reply.send(reply);
-	return Void();
 }
 
 static int64_t getMedianShardSize(VectorRef<DDMetricsRef> metricVec) {
@@ -3328,10 +3506,10 @@ GetStorageWigglerStateReply getStorageWigglerStates(Reference<DataDistributor> s
 	return reply;
 }
 
-ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
-                                PromiseStream<GetMetricsListRequest> getShardMetricsList) {
-	ErrorOr<Standalone<VectorRef<DDMetricsRef>>> result = wait(
-	    errorOr(brokenPromiseToNever(getShardMetricsList.getReply(GetMetricsListRequest(req.keys, req.shardLimit)))));
+Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
+                          PromiseStream<GetMetricsListRequest> getShardMetricsList) {
+	ErrorOr<Standalone<VectorRef<DDMetricsRef>>> result = co_await errorOr(
+	    brokenPromiseToNever(getShardMetricsList.getReply(GetMetricsListRequest(req.keys, req.shardLimit))));
 
 	if (result.isError()) {
 		req.reply.sendError(result.getError());
@@ -3349,24 +3527,21 @@ ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
 		}
 		req.reply.send(rep);
 	}
-
-	return Void();
 }
 
-ACTOR Future<bool> checkAuditProgressCompleteForSSShard(Database cx, std::shared_ptr<DDAudit> audit) {
+Future<bool> checkAuditProgressCompleteForSSShard(Database cx, std::shared_ptr<DDAudit> audit) {
 	ASSERT(audit->coreState.getType() == AuditType::ValidateStorageServerShard);
-	state ActorCollection actors(true);
-	state std::unordered_map<UID, bool> res;
-	state std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
-	state std::shared_ptr<AsyncVar<int>> remainingBudget =
+	ActorCollection actors(true);
+	std::unordered_map<UID, bool> res;
+	std::vector<StorageServerInterface> interfs = co_await getStorageServers(cx);
+	std::shared_ptr<AsyncVar<int>> remainingBudget =
 	    std::make_shared<AsyncVar<int>>(SERVER_KNOBS->CONCURRENT_AUDIT_TASK_COUNT_MAX);
-	state UID serverId;
-	state bool allFinish = true;
-	state int i = 0;
+	UID serverId;
+	bool allFinish = true;
 	TraceEvent(SevDebug, "CheckAuditProgressCompleteForSSShardStart")
 	    .detail("TotalSS", interfs.size())
 	    .detail("InitBudget", remainingBudget->get());
-	for (; i < interfs.size(); i++) {
+	for (int i = 0; i < interfs.size(); i++) {
 		serverId = interfs[i].uniqueID;
 		if (audit->serversFinishedSSShardAudit.contains(serverId)) {
 			TraceEvent(SevDebug, "CheckAuditProgressCompleteForSSShardSkipCheck").detail("ServerId", serverId);
@@ -3377,7 +3552,7 @@ ACTOR Future<bool> checkAuditProgressCompleteForSSShard(Database cx, std::shared
 		}
 		ASSERT(remainingBudget->get() >= 0);
 		while (remainingBudget->get() == 0) {
-			wait(remainingBudget->onChange());
+			co_await remainingBudget->onChange();
 			ASSERT(remainingBudget->get() >= 0);
 		}
 		remainingBudget->set(remainingBudget->get() - 1);
@@ -3392,7 +3567,7 @@ ACTOR Future<bool> checkAuditProgressCompleteForSSShard(Database cx, std::shared
 		                 checkAuditProgressCompleteByServer(
 		                     cx, audit->coreState.getType(), audit->coreState.id, allKeys, serverId, remainingBudget)));
 	}
-	wait(actors.getResult());
+	co_await actors.getResult();
 	for (const auto& [serverId, finish] : res) {
 		TraceEvent(SevDebug, "CheckAuditProgressCompleteForSSShardRes")
 		    .detail("AuditState", audit->coreState.toString())
@@ -3404,25 +3579,26 @@ ACTOR Future<bool> checkAuditProgressCompleteForSSShard(Database cx, std::shared
 			allFinish = false;
 		}
 	}
-	return allFinish;
+	co_return allFinish;
 }
 
 // Maintain an alive state of an audit until the audit completes
 // Automatically retry until if errors of the auditing process happen
 // Return if (1) audit completes; (2) retry times exceed the maximum retry times
 // Throw error if this actor gets cancelled
-ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
-                                    UID auditID,
-                                    AuditType auditType,
-                                    int currentRetryCount) {
+Future<Void> auditStorageCore(Reference<DataDistributor> self,
+                              UID auditID,
+                              AuditType auditType,
+                              int currentRetryCount) {
 	ASSERT(auditID.isValid());
-	state std::shared_ptr<DDAudit> audit = getAuditFromAuditMap(self, auditType, auditID);
+	std::shared_ptr<DDAudit> audit = getAuditFromAuditMap(self, auditType, auditID);
 
-	state MoveKeyLockInfo lockInfo;
+	MoveKeyLockInfo lockInfo;
 	lockInfo.myOwner = self->lock.myOwner;
 	lockInfo.prevOwner = self->lock.prevOwner;
 	lockInfo.prevWrite = self->lock.prevWrite;
 
+	Error err;
 	try {
 		ASSERT(audit != nullptr);
 		ASSERT(audit->coreState.ddId == self->ddId);
@@ -3434,7 +3610,7 @@ ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
 		    .detail("AuditType", audit->coreState.getType())
 		    .detail("AuditStorageCoreGeneration", currentRetryCount)
 		    .detail("RetryCount", audit->retryCount);
-		wait(audit->actors.getResult()); // goto exception handler if any actor is failed
+		co_await audit->actors.getResult(); // goto exception handler if any actor is failed
 		TraceEvent(SevInfo, "DDAuditStorageCoreAllActorsComplete", self->ddId)
 		    .detail("AuditID", audit->coreState.id)
 		    .detail("Range", audit->coreState.range)
@@ -3460,10 +3636,10 @@ ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
 			if (audit->coreState.getType() == AuditType::ValidateHA ||
 			    audit->coreState.getType() == AuditType::ValidateReplica ||
 			    audit->coreState.getType() == AuditType::ValidateRestore) {
-				bool allFinish = wait(checkAuditProgressCompleteByRange(self->txnProcessor->context(),
-				                                                        audit->coreState.getType(),
-				                                                        audit->coreState.id,
-				                                                        audit->coreState.range));
+				bool allFinish = co_await checkAuditProgressCompleteByRange(self->txnProcessor->context(),
+				                                                            audit->coreState.getType(),
+				                                                            audit->coreState.id,
+				                                                            audit->coreState.range);
 				if (!allFinish) {
 					TraceEvent(SevInfo, "DDAuditStorageCoreRetry", self->ddId)
 					    .detail("Reason", "AuditReplicaNotFinish")
@@ -3473,8 +3649,8 @@ ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
 					throw retry();
 				}
 			} else if (audit->coreState.getType() == AuditType::ValidateLocationMetadata) {
-				bool allFinish = wait(checkAuditProgressCompleteByRange(
-				    self->txnProcessor->context(), audit->coreState.getType(), audit->coreState.id, allKeys));
+				bool allFinish = co_await checkAuditProgressCompleteByRange(
+				    self->txnProcessor->context(), audit->coreState.getType(), audit->coreState.id, allKeys);
 				if (!allFinish) {
 					TraceEvent(SevInfo, "DDAuditStorageCoreRetry", self->ddId)
 					    .detail("Reason", "AuditLocationMetadataNotFinish")
@@ -3484,7 +3660,7 @@ ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
 					throw retry();
 				}
 			} else {
-				bool allFinish = wait(checkAuditProgressCompleteForSSShard(self->txnProcessor->context(), audit));
+				bool allFinish = co_await checkAuditProgressCompleteForSSShard(self->txnProcessor->context(), audit);
 				if (!allFinish) {
 					TraceEvent(SevInfo, "DDAuditStorageCoreRetry", self->ddId)
 					    .detail("Reason", "AuditSSShardNotFinish")
@@ -3505,11 +3681,11 @@ ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
 		    .detail("AuditState", audit->coreState.toString())
 		    .detail("AuditStorageCoreGeneration", currentRetryCount)
 		    .detail("RetryCount", audit->retryCount);
-		wait(persistAuditState(self->txnProcessor->context(),
-		                       audit->coreState,
-		                       "AuditStorageCore",
-		                       lockInfo,
-		                       self->context->isDDEnabled()));
+		co_await persistAuditState(self->txnProcessor->context(),
+		                           audit->coreState,
+		                           "AuditStorageCore",
+		                           lockInfo,
+		                           self->context->isDDEnabled());
 		TraceEvent(SevVerbose, "DDAuditStorageCoreSetResult", self->ddId)
 		    .detail("Context", audit->getDDAuditContext())
 		    .detail("AuditState", audit->coreState.toString())
@@ -3525,99 +3701,94 @@ ACTOR Future<Void> auditStorageCore(Reference<DataDistributor> self,
 		    .detail("Range", audit->coreState.range)
 		    .detail("AuditStorageCoreGeneration", currentRetryCount)
 		    .detail("RetryCount", audit->retryCount);
+		co_return;
 	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			// If this audit is cancelled, the place where cancelling
-			// this audit does removeAuditFromAuditMap
-			throw e;
-		}
-		TraceEvent(SevInfo, "DDAuditStorageCoreError", self->ddId)
-		    .errorUnsuppressed(e)
-		    .detail("Context", audit->getDDAuditContext())
+		err = e;
+	}
+	if (err.code() == error_code_actor_cancelled) {
+		// If this audit is cancelled, the place where cancelling
+		// this audit does removeAuditFromAuditMap
+		throw err;
+	}
+	TraceEvent(SevInfo, "DDAuditStorageCoreError", self->ddId)
+	    .errorUnsuppressed(err)
+	    .detail("Context", audit->getDDAuditContext())
+	    .detail("AuditID", auditID)
+	    .detail("AuditStorageCoreGeneration", currentRetryCount)
+	    .detail("RetryCount", audit->retryCount)
+	    .detail("AuditType", auditType)
+	    .detail("Range", audit->coreState.range);
+	if (err.code() == error_code_movekeys_conflict) {
+		removeAuditFromAuditMap(self, audit->coreState.getType(), audit->coreState.id); // remove audit
+		// Silently exit
+	} else if (err.code() == error_code_audit_storage_task_outdated) {
+		// DD failover occurred - storage server completed audit with old DD ID
+		// Remove from map so it can be properly resumed/retried
+		removeAuditFromAuditMap(self, audit->coreState.getType(), audit->coreState.id);
+		// Silently exit
+	} else if (err.code() == error_code_audit_storage_cancelled) {
+		// If this audit is cancelled, the place where cancelling
+		// this audit does removeAuditFromAuditMap
+	} else if (audit->retryCount < SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX && err.code() != error_code_not_implemented) {
+		audit->retryCount++;
+		audit->actors.clear(true);
+		TraceEvent(SevVerbose, "DDAuditStorageCoreRetry", self->ddId)
 		    .detail("AuditID", auditID)
+		    .detail("AuditType", auditType)
 		    .detail("AuditStorageCoreGeneration", currentRetryCount)
 		    .detail("RetryCount", audit->retryCount)
+		    .detail("Contains", self->audits.contains(auditType) && self->audits[auditType].contains(auditID));
+		co_await delay(0.1);
+		TraceEvent(SevVerbose, "DDAuditStorageCoreRetryAfterWait", self->ddId)
+		    .detail("AuditID", auditID)
 		    .detail("AuditType", auditType)
-		    .detail("Range", audit->coreState.range);
-		if (e.code() == error_code_movekeys_conflict) {
-			removeAuditFromAuditMap(self, audit->coreState.getType(),
-			                        audit->coreState.id); // remove audit
-			// Silently exit
-		} else if (e.code() == error_code_audit_storage_task_outdated) {
-			// DD failover occurred - storage server completed audit with old DD ID
-			// Remove from map so it can be properly resumed/retried
-			removeAuditFromAuditMap(self, audit->coreState.getType(), audit->coreState.id);
-			// Silently exit
-		} else if (e.code() == error_code_audit_storage_cancelled) {
-			// If this audit is cancelled, the place where cancelling
-			// this audit does removeAuditFromAuditMap
-		} else if (audit->retryCount < SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX && e.code() != error_code_not_implemented) {
-			audit->retryCount++;
-			audit->actors.clear(true);
-			TraceEvent(SevVerbose, "DDAuditStorageCoreRetry", self->ddId)
+		    .detail("AuditStorageCoreGeneration", currentRetryCount)
+		    .detail("RetryCount", audit->retryCount)
+		    .detail("Contains", self->audits.contains(auditType) && self->audits[auditType].contains(auditID));
+		// Erase the old audit from map and spawn a new audit inherit from the old audit
+		removeAuditFromAuditMap(self, audit->coreState.getType(), audit->coreState.id); // remove audit
+		if (audit->coreState.getType() == AuditType::ValidateStorageServerShard) {
+			runAuditStorage(
+			    self, audit->coreState, audit->retryCount, DDAuditContext::RETRY, audit->serversFinishedSSShardAudit);
+		} else {
+			runAuditStorage(self, audit->coreState, audit->retryCount, DDAuditContext::RETRY);
+		}
+	} else {
+		try {
+			audit->coreState.setPhase(AuditPhase::Failed);
+			co_await persistAuditState(self->txnProcessor->context(),
+			                           audit->coreState,
+			                           "AuditStorageCoreError",
+			                           lockInfo,
+			                           self->context->isDDEnabled());
+			TraceEvent(SevWarn, "DDAuditStorageCoreSetAuditFailed", self->ddId)
+			    .detail("Context", audit->getDDAuditContext())
 			    .detail("AuditID", auditID)
 			    .detail("AuditType", auditType)
 			    .detail("AuditStorageCoreGeneration", currentRetryCount)
 			    .detail("RetryCount", audit->retryCount)
-			    .detail("Contains", self->audits.contains(auditType) && self->audits[auditType].contains(auditID));
-			wait(delay(0.1));
-			TraceEvent(SevVerbose, "DDAuditStorageCoreRetryAfterWait", self->ddId)
+			    .detail("AuditState", audit->coreState.toString());
+		} catch (Error& err) {
+			TraceEvent(SevWarn, "DDAuditStorageCoreErrorWhenSetAuditFailed", self->ddId)
+			    .errorUnsuppressed(err)
+			    .detail("Context", audit->getDDAuditContext())
 			    .detail("AuditID", auditID)
 			    .detail("AuditType", auditType)
 			    .detail("AuditStorageCoreGeneration", currentRetryCount)
 			    .detail("RetryCount", audit->retryCount)
-			    .detail("Contains", self->audits.contains(auditType) && self->audits[auditType].contains(auditID));
-			// Erase the old audit from map and spawn a new audit inherit from the old audit
-			removeAuditFromAuditMap(self, audit->coreState.getType(),
-			                        audit->coreState.id); // remove audit
-			if (audit->coreState.getType() == AuditType::ValidateStorageServerShard) {
-				runAuditStorage(self,
-				                audit->coreState,
-				                audit->retryCount,
-				                DDAuditContext::RETRY,
-				                audit->serversFinishedSSShardAudit);
-			} else {
-				runAuditStorage(self, audit->coreState, audit->retryCount, DDAuditContext::RETRY);
-			}
-		} else {
-			try {
-				audit->coreState.setPhase(AuditPhase::Failed);
-				wait(persistAuditState(self->txnProcessor->context(),
-				                       audit->coreState,
-				                       "AuditStorageCoreError",
-				                       lockInfo,
-				                       self->context->isDDEnabled()));
-				TraceEvent(SevWarn, "DDAuditStorageCoreSetAuditFailed", self->ddId)
-				    .detail("Context", audit->getDDAuditContext())
-				    .detail("AuditID", auditID)
-				    .detail("AuditType", auditType)
-				    .detail("AuditStorageCoreGeneration", currentRetryCount)
-				    .detail("RetryCount", audit->retryCount)
-				    .detail("AuditState", audit->coreState.toString());
-			} catch (Error& e) {
-				TraceEvent(SevWarn, "DDAuditStorageCoreErrorWhenSetAuditFailed", self->ddId)
-				    .errorUnsuppressed(e)
-				    .detail("Context", audit->getDDAuditContext())
-				    .detail("AuditID", auditID)
-				    .detail("AuditType", auditType)
-				    .detail("AuditStorageCoreGeneration", currentRetryCount)
-				    .detail("RetryCount", audit->retryCount)
-				    .detail("AuditState", audit->coreState.toString());
-				// unexpected error when persistAuditState
-				// However, we do not want any audit error kills the DD
-				// So, we silently remove audit from auditMap
-				// As a result, this audit can be in RUNNING state on disk but not alive
-				// We call this audit a zombie audit
-				// Note that a client may wait for the state on disk to proceed to "complete"
-				// However, this progress can never happen to a zombie audit
-				// For this case, the client should be able to be timed out
-				// A zombie audit will be either: (1) resumed by the next DD; (2) removed by client
-			}
-			removeAuditFromAuditMap(self, audit->coreState.getType(),
-			                        audit->coreState.id); // remove audit
+			    .detail("AuditState", audit->coreState.toString());
+			// unexpected error when persistAuditState
+			// However, we do not want any audit error kills the DD
+			// So, we silently remove audit from auditMap
+			// As a result, this audit can be in RUNNING state on disk but not alive
+			// We call this audit a zombie audit
+			// Note that a client may wait for the state on disk to proceed to "complete"
+			// However, this progress can never happen to a zombie audit
+			// For this case, the client should be able to be timed out
+			// A zombie audit will be either: (1) resumed by the next DD; (2) removed by client
 		}
+		removeAuditFromAuditMap(self, audit->coreState.getType(), audit->coreState.id); // remove audit
 	}
-	return Void();
 }
 
 // runAuditStorage is the only entry to start an Audit entity
@@ -3660,22 +3831,22 @@ void runAuditStorage(Reference<DataDistributor> self,
 }
 
 // Get audit for auditRange and auditType, if not exist, launch a new one
-ACTOR Future<UID> launchAudit(Reference<DataDistributor> self,
-                              KeyRange auditRange,
-                              AuditType auditType,
-                              KeyValueStoreType auditStorageEngineType) {
-	state MoveKeyLockInfo lockInfo;
+Future<UID> launchAudit(Reference<DataDistributor> self,
+                        KeyRange auditRange,
+                        AuditType auditType,
+                        KeyValueStoreType auditStorageEngineType) {
+	MoveKeyLockInfo lockInfo;
 	lockInfo.myOwner = self->lock.myOwner;
 	lockInfo.prevOwner = self->lock.prevOwner;
 	lockInfo.prevWrite = self->lock.prevWrite;
 
-	state UID auditID;
+	UID auditID;
 	try {
 		TraceEvent(SevInfo, "DDAuditStorageLaunchStarts", self->ddId)
 		    .detail("AuditType", auditType)
 		    .detail("KeyValueStoreType", auditStorageEngineType)
 		    .detail("RequestedRange", auditRange);
-		wait(self->auditStorageInitialized.getFuture());
+		co_await self->auditStorageInitialized.getFuture();
 		// Start an audit if no audit exists
 		// If existing an audit for a different purpose, send error to client
 		// aka, we only allow one audit at a time for all purposes
@@ -3713,7 +3884,7 @@ ACTOR Future<UID> launchAudit(Reference<DataDistributor> self,
 			    .detail("RequestedRange", auditRange)
 			    .detail("ExistingState", audit->coreState.toString());
 		} else {
-			state AuditStorageState auditState;
+			AuditStorageState auditState;
 			auditState.setType(auditType);
 			auditState.engineType = auditStorageEngineType;
 			auditState.range = auditRange;
@@ -3723,8 +3894,8 @@ ACTOR Future<UID> launchAudit(Reference<DataDistributor> self,
 			    .detail("AuditType", auditType)
 			    .detail("KeyValueStoreType", auditStorageEngineType)
 			    .detail("Range", auditRange);
-			UID auditID_ = wait(persistNewAuditState(
-			    self->txnProcessor->context(), auditState, lockInfo, self->context->isDDEnabled()));
+			UID auditID_ = co_await persistNewAuditState(
+			    self->txnProcessor->context(), auditState, lockInfo, self->context->isDDEnabled());
 			self->addActor.send(clearAuditMetadataForType(self->txnProcessor->context(),
 			                                              auditState.getType(),
 			                                              auditID_,
@@ -3758,7 +3929,7 @@ ACTOR Future<UID> launchAudit(Reference<DataDistributor> self,
 				// Suppose DDinit re-runs right after a new audit is persisted
 				// For this case, auditResume sees the new audit and resumes it
 				// At this point, the new audit is already in the audit map
-				return auditID;
+				co_return auditID;
 			}
 			runAuditStorage(self, auditState, 0, DDAuditContext::LAUNCH);
 		}
@@ -3773,29 +3944,29 @@ ACTOR Future<UID> launchAudit(Reference<DataDistributor> self,
 		    .detail("Range", auditRange);
 		throw e;
 	}
-	return auditID;
+	co_return auditID;
 }
 
-ACTOR Future<Void> cancelAuditStorage(Reference<DataDistributor> self, TriggerAuditRequest req) {
-	state FlowLock::Releaser holder;
+Future<Void> cancelAuditStorage(Reference<DataDistributor> self, TriggerAuditRequest req) {
+	FlowLock::Releaser holder;
 	if (req.getType() == AuditType::ValidateHA) {
-		wait(self->auditStorageHaLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageHaLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageHaLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateReplica) {
-		wait(self->auditStorageReplicaLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageReplicaLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageReplicaLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateLocationMetadata) {
-		wait(self->auditStorageLocationMetadataLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageLocationMetadataLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageLocationMetadataLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateStorageServerShard) {
-		wait(self->auditStorageSsShardLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageSsShardLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageSsShardLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateRestore) {
-		wait(self->auditStorageRestoreLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageRestoreLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageRestoreLaunchingLock);
 	} else {
 		req.reply.sendError(not_implemented());
-		return Void();
+		co_return;
 	}
 
 	try {
@@ -3804,7 +3975,7 @@ ACTOR Future<Void> cancelAuditStorage(Reference<DataDistributor> self, TriggerAu
 		TraceEvent(SevDebug, "DDCancelAuditStorageStart", self->ddId)
 		    .detail("AuditType", req.getType())
 		    .detail("AuditID", req.id);
-		wait(cancelAuditMetadata(self->txnProcessor->context(), req.getType(), req.id));
+		co_await cancelAuditMetadata(self->txnProcessor->context(), req.getType(), req.id);
 		// Once auditMetadata cancelled, any ongoing audit will stop
 		// Then clear ongoing audit D/S
 		if (auditExistInAuditMap(self, req.getType(), req.id)) {
@@ -3824,7 +3995,30 @@ ACTOR Future<Void> cancelAuditStorage(Reference<DataDistributor> self, TriggerAu
 		    .detail("AuditType", req.getType());
 		req.reply.sendError(cancel_audit_storage_failed());
 	}
-	return Void();
+}
+
+Future<Void> periodicAuditLocationMetadata(Reference<DataDistributor> self) {
+	if (SERVER_KNOBS->AUDIT_LOCATION_METADATA_INTERVAL <= 0) {
+		co_return;
+	}
+	co_await self->auditStorageInitialized.getFuture();
+	TraceEvent("PeriodicAuditLocationMetadataEnabled", self->ddId)
+	    .detail("IntervalSeconds", SERVER_KNOBS->AUDIT_LOCATION_METADATA_INTERVAL);
+	while (true) {
+		co_await delay(SERVER_KNOBS->AUDIT_LOCATION_METADATA_INTERVAL);
+		try {
+			co_await self->auditStorageLocationMetadataLaunchingLock.take(TaskPriority::DefaultYield);
+			FlowLock::Releaser holder(self->auditStorageLocationMetadataLaunchingLock);
+			UID auditID =
+			    co_await launchAudit(self, allKeys, AuditType::ValidateLocationMetadata, KeyValueStoreType::END);
+			TraceEvent("PeriodicAuditLocationMetadataLaunched", self->ddId).detail("AuditID", auditID);
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
+			TraceEvent(SevWarn, "PeriodicAuditLocationMetadataError", self->ddId).error(e);
+		}
+	}
 }
 
 // Handling audit requests
@@ -3836,42 +4030,43 @@ ACTOR Future<Void> cancelAuditStorage(Reference<DataDistributor> self, TriggerAu
 // 2. failed to persist new audit state; 3. DD is cancelled during persisting new audit state
 // For 1 and 2, we believe no new audit is persisted; For 3, we do not know whether a new
 // audit is persisted.
-ACTOR Future<Void> auditStorage(Reference<DataDistributor> self, TriggerAuditRequest req) {
-	state FlowLock::Releaser holder;
+Future<Void> auditStorage(Reference<DataDistributor> self, TriggerAuditRequest req) {
+	FlowLock::Releaser holder;
 	if (req.getType() == AuditType::ValidateHA) {
-		wait(self->auditStorageHaLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageHaLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageHaLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateReplica) {
-		wait(self->auditStorageReplicaLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageReplicaLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageReplicaLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateLocationMetadata) {
-		wait(self->auditStorageLocationMetadataLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageLocationMetadataLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageLocationMetadataLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateStorageServerShard) {
-		wait(self->auditStorageSsShardLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageSsShardLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageSsShardLaunchingLock);
 	} else if (req.getType() == AuditType::ValidateRestore) {
-		wait(self->auditStorageRestoreLaunchingLock.take(TaskPriority::DefaultYield));
+		co_await self->auditStorageRestoreLaunchingLock.take(TaskPriority::DefaultYield);
 		holder = FlowLock::Releaser(self->auditStorageRestoreLaunchingLock);
 	} else {
 		req.reply.sendError(not_implemented());
-		return Void();
+		co_return;
 	}
 
 	if (req.range.empty()) {
 		req.reply.sendError(audit_storage_failed());
-		return Void();
+		co_return;
 	}
 
-	state int retryCount = 0;
-	loop {
+	int retryCount = 0;
+	while (true) {
+		Error err;
 		try {
 			TraceEvent(SevDebug, "DDAuditStorageStart", self->ddId)
 			    .detail("RetryCount", retryCount)
 			    .detail("AuditType", req.getType())
 			    .detail("KeyValueStoreType", req.engineType.toString())
 			    .detail("Range", req.range);
-			UID auditID = wait(launchAudit(self, req.range, req.getType(), req.engineType));
+			UID auditID = co_await launchAudit(self, req.range, req.getType(), req.engineType);
 			req.reply.send(auditID);
 			TraceEvent(SevVerbose, "DDAuditStorageReply", self->ddId)
 			    .detail("RetryCount", retryCount)
@@ -3879,33 +4074,34 @@ ACTOR Future<Void> auditStorage(Reference<DataDistributor> self, TriggerAuditReq
 			    .detail("KeyValueStoreType", req.engineType.toString())
 			    .detail("Range", req.range)
 			    .detail("AuditID", auditID);
+			break;
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			}
-			TraceEvent(SevInfo, "DDAuditStorageError", self->ddId)
-			    .errorUnsuppressed(e)
-			    .detail("RetryCount", retryCount)
-			    .detail("AuditType", req.getType())
-			    .detail("KeyValueStoreType", req.engineType.toString())
-			    .detail("Range", req.range);
-			if (e.code() == error_code_operation_failed && g_network->isSimulated()) {
-				throw audit_storage_failed(); // to trigger dd restart
-			} else if (e.code() == error_code_audit_storage_exceeded_request_limit) {
-				req.reply.sendError(audit_storage_exceeded_request_limit());
-			} else if (e.code() == error_code_persist_new_audit_metadata_error) {
-				req.reply.sendError(audit_storage_failed());
-			} else if (retryCount < SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX) {
-				retryCount++;
-				wait(delay(0.1));
-				continue;
-			} else {
-				req.reply.sendError(audit_storage_failed());
-			}
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled) {
+			throw err;
+		}
+		TraceEvent(SevInfo, "DDAuditStorageError", self->ddId)
+		    .errorUnsuppressed(err)
+		    .detail("RetryCount", retryCount)
+		    .detail("AuditType", req.getType())
+		    .detail("KeyValueStoreType", req.engineType.toString())
+		    .detail("Range", req.range);
+		if (err.code() == error_code_operation_failed && g_network->isSimulated()) {
+			throw audit_storage_failed(); // to trigger dd restart
+		} else if (err.code() == error_code_audit_storage_exceeded_request_limit) {
+			req.reply.sendError(audit_storage_exceeded_request_limit());
+		} else if (err.code() == error_code_persist_new_audit_metadata_error) {
+			req.reply.sendError(audit_storage_failed());
+		} else if (retryCount < SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX) {
+			retryCount++;
+			co_await delay(0.1);
+			continue;
+		} else {
+			req.reply.sendError(audit_storage_failed());
 		}
 		break;
 	}
-	return Void();
 }
 
 // The entry of starting a series of audit workers
@@ -3934,25 +4130,24 @@ void loadAndDispatchAudit(Reference<DataDistributor> self, std::shared_ptr<DDAud
 
 // This function is for locationmetadata audits
 // Schedule audit task on input range
-ACTOR Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self,
-                                                 std::shared_ptr<DDAudit> audit,
-                                                 KeyRange range) {
-	state const AuditType auditType = audit->coreState.getType();
+Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self,
+                                           std::shared_ptr<DDAudit> audit,
+                                           KeyRange range) {
+	const AuditType auditType = audit->coreState.getType();
 	ASSERT(auditType == AuditType::ValidateLocationMetadata);
 	TraceEvent(SevInfo, "DDdispatchAuditLocationMetadataBegin", self->ddId)
 	    .detail("AuditID", audit->coreState.id)
 	    .detail("AuditType", auditType);
-	state Key begin = range.begin;
-	state KeyRange currentRange = range;
-	state std::vector<AuditStorageState> auditStates;
-	state int64_t issueDoAuditCount = 0;
+	Key begin = range.begin;
+	KeyRange currentRange = range;
+	std::vector<AuditStorageState> auditStates;
+	int64_t issueDoAuditCount = 0;
 
 	try {
 		while (begin < range.end) {
 			currentRange = KeyRangeRef(begin, range.end);
-			wait(store(
-			    auditStates,
-			    getAuditStateByRange(self->txnProcessor->context(), auditType, audit->coreState.id, currentRange)));
+			auditStates = co_await getAuditStateByRange(
+			    self->txnProcessor->context(), auditType, audit->coreState.id, currentRange);
 			ASSERT(!auditStates.empty());
 			begin = auditStates.back().range.end;
 			TraceEvent(SevInfo, "DDdispatchAuditLocationMetadataDispatch", self->ddId)
@@ -3961,9 +4156,8 @@ ACTOR Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self
 			    .detail("AuditType", auditType)
 			    .detail("NextBegin", begin)
 			    .detail("RangeEnd", range.end);
-			state int i = 0;
-			for (; i < auditStates.size(); i++) {
-				state AuditPhase phase = auditStates[i].getPhase();
+			for (int i = 0; i < auditStates.size(); i++) {
+				AuditPhase phase = auditStates[i].getPhase();
 				ASSERT(phase != AuditPhase::Running && phase != AuditPhase::Failed);
 				if (phase == AuditPhase::Complete) {
 					continue; // pass
@@ -3974,7 +4168,7 @@ ACTOR Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self
 					// Set doAuditOnStorageServer
 					ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 					while (audit->remainingBudgetForAuditTasks.get() == 0) {
-						wait(audit->remainingBudgetForAuditTasks.onChange());
+						co_await audit->remainingBudgetForAuditTasks.onChange();
 						ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 					}
 					audit->remainingBudgetForAuditTasks.set(audit->remainingBudgetForAuditTasks.get() - 1);
@@ -3988,7 +4182,7 @@ ACTOR Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self
 					audit->actors.add(doAuditLocationMetadata(self, audit, auditStates[i].range));
 				}
 			}
-			wait(delay(0.1));
+			co_await delay(0.1);
 		}
 		TraceEvent(SevInfo, "DDdispatchAuditLocationMetadataEnd", self->ddId)
 		    .detail("AuditID", audit->coreState.id)
@@ -4005,23 +4199,20 @@ ACTOR Future<Void> dispatchAuditLocationMetadata(Reference<DataDistributor> self
 		    .detail("AuditType", auditType);
 		audit->auditStorageAnyChildFailed = true;
 	}
-
-	return Void();
 }
 
 // This function dedicates to audit ssshard
 // For each of storage servers, audits allKeys
-ACTOR Future<Void> dispatchAuditStorageServerShard(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit) {
-	state const AuditType auditType = audit->coreState.getType();
+Future<Void> dispatchAuditStorageServerShard(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit) {
+	const AuditType auditType = audit->coreState.getType();
 	ASSERT(auditType == AuditType::ValidateStorageServerShard);
 	TraceEvent(SevInfo, "DDDispatchAuditStorageServerShardBegin", self->ddId)
 	    .detail("AuditID", audit->coreState.id)
 	    .detail("AuditType", auditType);
 	try {
-		state std::vector<StorageServerInterface> interfs = wait(getStorageServers(self->txnProcessor->context()));
-		state int i = 0;
-		for (; i < interfs.size(); ++i) {
-			state StorageServerInterface targetServer = interfs[i];
+		std::vector<StorageServerInterface> interfs = co_await getStorageServers(self->txnProcessor->context());
+		for (int i = 0; i < interfs.size(); ++i) {
+			StorageServerInterface targetServer = interfs[i];
 			// Currently, Tss server may not follow the auit consistency rule
 			// Thus, skip if the server is tss
 			if (targetServer.isTss()) {
@@ -4029,11 +4220,11 @@ ACTOR Future<Void> dispatchAuditStorageServerShard(Reference<DataDistributor> se
 			}
 			ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 			while (audit->remainingBudgetForAuditTasks.get() == 0) {
-				wait(audit->remainingBudgetForAuditTasks.onChange());
+				co_await audit->remainingBudgetForAuditTasks.onChange();
 				ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 			}
 			audit->actors.add(scheduleAuditStorageShardOnServer(self, audit, targetServer));
-			wait(delay(0.1));
+			co_await delay(0.1);
 		}
 		TraceEvent(SevInfo, "DDDispatchAuditStorageServerShardEnd", self->ddId)
 		    .detail("AuditID", audit->coreState.id)
@@ -4049,34 +4240,31 @@ ACTOR Future<Void> dispatchAuditStorageServerShard(Reference<DataDistributor> se
 		    .detail("AuditType", auditType);
 		audit->auditStorageAnyChildFailed = true;
 	}
-
-	return Void();
 }
 
 // Schedule audit ssshard task on the input storage server (ssi)
 // Do audit on allKeys
 // Automatically retry until complete or timed out
-ACTOR Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor> self,
-                                                     std::shared_ptr<DDAudit> audit,
-                                                     StorageServerInterface ssi) {
-	state UID serverId = ssi.uniqueID;
-	state const AuditType auditType = audit->coreState.getType();
+Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor> self,
+                                               std::shared_ptr<DDAudit> audit,
+                                               StorageServerInterface ssi) {
+	UID serverId = ssi.uniqueID;
+	const AuditType auditType = audit->coreState.getType();
 	ASSERT(auditType == AuditType::ValidateStorageServerShard);
 	TraceEvent(SevInfo, "DDScheduleAuditStorageShardOnServerBegin", self->ddId)
 	    .detail("ServerID", serverId)
 	    .detail("AuditID", audit->coreState.id)
 	    .detail("AuditType", auditType);
-	state Key begin = allKeys.begin;
-	state KeyRange currentRange = allKeys;
-	state std::vector<AuditStorageState> auditStates;
-	state int64_t issueDoAuditCount = 0;
-
+	Key begin = allKeys.begin;
+	KeyRange currentRange = allKeys;
+	std::vector<AuditStorageState> auditStates;
+	int64_t issueDoAuditCount = 0;
+	Error err;
 	try {
 		while (begin < allKeys.end) {
 			currentRange = KeyRangeRef(begin, allKeys.end);
-			wait(store(auditStates,
-			           getAuditStateByServer(
-			               self->txnProcessor->context(), auditType, audit->coreState.id, serverId, currentRange)));
+			auditStates = co_await getAuditStateByServer(
+			    self->txnProcessor->context(), auditType, audit->coreState.id, serverId, currentRange);
 			ASSERT(!auditStates.empty());
 			begin = auditStates.back().range.end;
 			TraceEvent(SevInfo, "DDScheduleAuditStorageShardOnServerDispatch", self->ddId)
@@ -4086,9 +4274,8 @@ ACTOR Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor>
 			    .detail("AuditType", auditType)
 			    .detail("NextBegin", begin)
 			    .detail("RangeEnd", allKeys.end);
-			state int i = 0;
-			for (; i < auditStates.size(); i++) {
-				state AuditPhase phase = auditStates[i].getPhase();
+			for (int i = 0; i < auditStates.size(); i++) {
+				AuditPhase phase = auditStates[i].getPhase();
 				ASSERT(phase != AuditPhase::Running && phase != AuditPhase::Failed);
 				if (phase == AuditPhase::Complete) {
 					continue; // pass
@@ -4099,7 +4286,7 @@ ACTOR Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor>
 					// Set doAuditOnStorageServer
 					ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 					while (audit->remainingBudgetForAuditTasks.get() == 0) {
-						wait(audit->remainingBudgetForAuditTasks.onChange());
+						co_await audit->remainingBudgetForAuditTasks.onChange();
 						ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 					}
 					audit->remainingBudgetForAuditTasks.set(audit->remainingBudgetForAuditTasks.get() - 1);
@@ -4112,10 +4299,10 @@ ACTOR Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor>
 					AuditStorageRequest req(audit->coreState.id, auditStates[i].range, auditType);
 					issueDoAuditCount++;
 					req.ddId = self->ddId; // send this ddid to SS
-					wait(doAuditOnStorageServer(self, audit, ssi, req)); // do audit one by one
+					co_await doAuditOnStorageServer(self, audit, ssi, req); // do audit one by one
 				}
 			}
-			wait(delay(0.1));
+			co_await delay(0.1);
 		}
 
 		TraceEvent(SevInfo, "DDScheduleAuditStorageShardOnServerEnd", self->ddId)
@@ -4123,63 +4310,62 @@ ACTOR Future<Void> scheduleAuditStorageShardOnServer(Reference<DataDistributor>
 		    .detail("AuditID", audit->coreState.id)
 		    .detail("AuditType", auditType)
 		    .detail("IssuedDoAuditCount", issueDoAuditCount);
-
+		co_return;
 	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			throw e;
-		}
-		TraceEvent(SevInfo, "DDScheduleAuditStorageShardOnServerError", self->ddId)
-		    .errorUnsuppressed(e)
-		    .detail("AuditID", audit->coreState.id)
-		    .detail("AuditType", auditType)
-		    .detail("IssuedDoAuditCount", issueDoAuditCount);
+		err = e;
+	}
+	if (err.code() == error_code_actor_cancelled) {
+		throw err;
+	}
+	TraceEvent(SevInfo, "DDScheduleAuditStorageShardOnServerError", self->ddId)
+	    .errorUnsuppressed(err)
+	    .detail("AuditID", audit->coreState.id)
+	    .detail("AuditType", auditType)
+	    .detail("IssuedDoAuditCount", issueDoAuditCount);
 
-		if (e.code() == error_code_not_implemented || e.code() == error_code_audit_storage_cancelled) {
-			throw e;
-		} else if (e.code() == error_code_audit_storage_error) {
-			audit->foundError = true;
-		} else if (audit->retryCount >= SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX) {
-			throw audit_storage_failed();
-		} else {
-			if (e.code() != error_code_audit_storage_failed) {
-				try {
-					bool ssRemoved = wait(checkStorageServerRemoved(self->txnProcessor->context(), ssi.uniqueID));
-					if (ssRemoved) {
-						// It is possible that the input ss has been removed, then silently exit
-						return Void();
-					}
-				} catch (Error& e) {
-					// retry
+	if (err.code() == error_code_not_implemented || err.code() == error_code_audit_storage_cancelled) {
+		throw err;
+	} else if (err.code() == error_code_audit_storage_error) {
+		audit->foundError = true;
+	} else if (audit->retryCount >= SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX) {
+		throw audit_storage_failed();
+	} else {
+		if (err.code() != error_code_audit_storage_failed) {
+			try {
+				bool ssRemoved = co_await checkStorageServerRemoved(self->txnProcessor->context(), ssi.uniqueID);
+				if (ssRemoved) {
+					// It is possible that the input ss has been removed, then silently exit
+					co_return;
 				}
+			} catch (Error& err) {
+				// retry
 			}
-			audit->retryCount++;
-			audit->actors.add(scheduleAuditStorageShardOnServer(self, audit, ssi));
 		}
+		audit->retryCount++;
+		audit->actors.add(scheduleAuditStorageShardOnServer(self, audit, ssi));
 	}
-
-	return Void();
 }
 
 // This function is for ha/replica/restore audits
 // Schedule audit task on the input range
-ACTOR Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit) {
-	state const AuditType auditType = audit->coreState.getType();
-	state const KeyRange range = audit->coreState.range;
+Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit) {
+	const AuditType auditType = audit->coreState.getType();
+	const KeyRange range = audit->coreState.range;
 	ASSERT(auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica ||
 	       auditType == AuditType::ValidateRestore);
 	TraceEvent(SevInfo, "DDDispatchAuditStorageBegin", self->ddId)
 	    .detail("AuditID", audit->coreState.id)
 	    .detail("Range", range)
 	    .detail("AuditType", auditType);
-	state Key begin = range.begin;
-	state KeyRange currentRange = range;
-	state int64_t completedCount = 0;
-	state int64_t totalCount = 0;
+	Key begin = range.begin;
+	KeyRange currentRange = range;
+	int64_t completedCount = 0;
+	int64_t totalCount = 0;
 	try {
 		while (begin < range.end) {
 			currentRange = KeyRangeRef(begin, range.end);
-			state std::vector<AuditStorageState> auditStates =
-			    wait(getAuditStateByRange(self->txnProcessor->context(), auditType, audit->coreState.id, currentRange));
+			std::vector<AuditStorageState> auditStates = co_await getAuditStateByRange(
+			    self->txnProcessor->context(), auditType, audit->coreState.id, currentRange);
 			ASSERT(!auditStates.empty());
 			begin = auditStates.back().range.end;
 			TraceEvent(SevInfo, "DDDispatchAuditStorageDispatch", self->ddId)
@@ -4189,9 +4375,8 @@ ACTOR Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::sh
 			    .detail("AuditType", auditType)
 			    .detail("NextBegin", begin)
 			    .detail("NumAuditStates", auditStates.size());
-			state int i = 0;
-			for (; i < auditStates.size(); i++) {
-				state AuditPhase phase = auditStates[i].getPhase();
+			for (int i = 0; i < auditStates.size(); i++) {
+				AuditPhase phase = auditStates[i].getPhase();
 				// Skip Running/Failed states during retries (they will be updated on retry)
 				if (phase == AuditPhase::Running || phase == AuditPhase::Failed) {
 					continue;
@@ -4210,13 +4395,13 @@ ACTOR Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::sh
 					ASSERT(phase == AuditPhase::Invalid);
 					ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 					while (audit->remainingBudgetForAuditTasks.get() == 0) {
-						wait(audit->remainingBudgetForAuditTasks.onChange());
+						co_await audit->remainingBudgetForAuditTasks.onChange();
 						ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 					}
 					audit->actors.add(scheduleAuditOnRange(self, audit, auditStates[i].range));
 				}
 			}
-			wait(delay(0.1));
+			co_await delay(0.1);
 		}
 		TraceEvent(SevInfo, "DDDispatchAuditStorageEnd", self->ddId)
 		    .detail("AuditID", audit->coreState.id)
@@ -4236,14 +4421,11 @@ ACTOR Future<Void> dispatchAuditStorage(Reference<DataDistributor> self, std::sh
 		    .detail("AuditType", auditType);
 		audit->auditStorageAnyChildFailed = true;
 	}
-
-	return Void();
 }
 
-ACTOR Future<std::unordered_map<UID, KeyValueStoreType>> getStorageType(
-    std::vector<StorageServerInterface> storageServers) {
-	state std::vector<Future<ErrorOr<KeyValueStoreType>>> storageTypeFutures;
-	state std::unordered_map<UID, KeyValueStoreType> res;
+Future<std::unordered_map<UID, KeyValueStoreType>> getStorageType(std::vector<StorageServerInterface> storageServers) {
+	std::vector<Future<ErrorOr<KeyValueStoreType>>> storageTypeFutures;
+	std::unordered_map<UID, KeyValueStoreType> res;
 
 	try {
 		for (int i = 0; i < storageServers.size(); i++) {
@@ -4251,7 +4433,7 @@ ACTOR Future<std::unordered_map<UID, KeyValueStoreType>> getStorageType(
 			storageTypeFutures.push_back(
 			    storageServers[i].getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0));
 		}
-		wait(waitForAll(storageTypeFutures));
+		co_await waitForAll(storageTypeFutures);
 
 		for (int i = 0; i < storageServers.size(); i++) {
 			ErrorOr<KeyValueStoreType> reply = storageTypeFutures[i].get();
@@ -4272,16 +4454,16 @@ ACTOR Future<std::unordered_map<UID, KeyValueStoreType>> getStorageType(
 		res.clear();
 	}
 
-	return res;
+	co_return res;
 }
 
 // Partition the input range into multiple subranges according to the range ownership, and
 // schedule ha/replica/restore audit tasks of each subrange on the server which owns the subrange
 // Automatically retry until complete or timed out
-ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
-                                        std::shared_ptr<DDAudit> audit,
-                                        KeyRange rangeToSchedule) {
-	state const AuditType auditType = audit->coreState.getType();
+Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
+                                  std::shared_ptr<DDAudit> audit,
+                                  KeyRange rangeToSchedule) {
+	const AuditType auditType = audit->coreState.getType();
 	ASSERT(auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica ||
 	       auditType == AuditType::ValidateRestore);
 	TraceEvent(SevInfo, "DDScheduleAuditOnRangeBegin", self->ddId)
@@ -4291,16 +4473,16 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 	    .detail("AuditType", auditType)
 	    .detail("RemainingBudget", audit->remainingBudgetForAuditTasks.get());
 
-	state Key currentRangeToScheduleBegin = rangeToSchedule.begin;
-	state KeyRange currentRangeToSchedule;
-	state int64_t issueDoAuditCount = 0;
-	state int64_t numSkippedShards = 0;
+	Key currentRangeToScheduleBegin = rangeToSchedule.begin;
+	KeyRange currentRangeToSchedule;
+	int64_t issueDoAuditCount = 0;
+	int64_t numSkippedShards = 0;
 
 	try {
 		while (currentRangeToScheduleBegin < rangeToSchedule.end) {
 			currentRangeToSchedule = Standalone(KeyRangeRef(currentRangeToScheduleBegin, rangeToSchedule.end));
-			state std::vector<IDDTxnProcessor::DDRangeLocations> rangeLocations =
-			    wait(self->txnProcessor->getSourceServerInterfacesForRange(currentRangeToSchedule));
+			std::vector<IDDTxnProcessor::DDRangeLocations> rangeLocations =
+			    co_await self->txnProcessor->getSourceServerInterfacesForRange(currentRangeToSchedule);
 			if (SERVER_KNOBS->ENABLE_AUDIT_VERBOSE_TRACE) {
 				TraceEvent(SevInfo, "DDScheduleAuditOnCurrentRange", self->ddId)
 				    .detail("AuditID", audit->coreState.id)
@@ -4312,11 +4494,11 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 			}
 
 			// Divide the audit job in to tasks according to KeyServers system mapping
-			state int assignedRangeTasks = 0;
-			state int rangeLocationIndex = 0;
+			int assignedRangeTasks = 0;
+			int rangeLocationIndex = 0;
 			for (; rangeLocationIndex < rangeLocations.size(); ++rangeLocationIndex) {
 				// For each task, check the progress, and create task request for the unfinished range
-				state KeyRange taskRange = rangeLocations[rangeLocationIndex].range;
+				KeyRange taskRange = rangeLocations[rangeLocationIndex].range;
 				if (SERVER_KNOBS->ENABLE_AUDIT_VERBOSE_TRACE) {
 					TraceEvent(SevInfo, "DDScheduleAuditOnCurrentRangeTask", self->ddId)
 					    .detail("AuditID", audit->coreState.id)
@@ -4326,13 +4508,13 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 					    .detail("TaskRange", taskRange);
 				}
 
-				state Key taskRangeBegin = taskRange.begin;
+				Key taskRangeBegin = taskRange.begin;
 				while (taskRangeBegin < taskRange.end) {
-					state std::vector<AuditStorageState> auditStates =
-					    wait(getAuditStateByRange(self->txnProcessor->context(),
-					                              auditType,
-					                              audit->coreState.id,
-					                              KeyRangeRef(taskRangeBegin, taskRange.end)));
+					std::vector<AuditStorageState> auditStates =
+					    co_await getAuditStateByRange(self->txnProcessor->context(),
+					                                  auditType,
+					                                  audit->coreState.id,
+					                                  KeyRangeRef(taskRangeBegin, taskRange.end));
 					if (SERVER_KNOBS->ENABLE_AUDIT_VERBOSE_TRACE) {
 						TraceEvent(SevInfo, "DDScheduleAuditOnRangeSubTask", self->ddId)
 						    .detail("AuditID", audit->coreState.id)
@@ -4347,9 +4529,9 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 					}
 					ASSERT(!auditStates.empty());
 
-					state int auditStateIndex = 0;
+					int auditStateIndex = 0;
 					for (; auditStateIndex < auditStates.size(); ++auditStateIndex) {
-						state AuditPhase phase = auditStates[auditStateIndex].getPhase();
+						AuditPhase phase = auditStates[auditStateIndex].getPhase();
 						// Skip Running/Failed states during retries (they will be updated on retry)
 						if (phase == AuditPhase::Running || phase == AuditPhase::Failed) {
 							continue;
@@ -4363,10 +4545,9 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 						// Create audit task for the range where the phase is Invalid which indicates
 						// this range has not been audited
 						ASSERT(phase == AuditPhase::Invalid);
-						state AuditStorageRequest req(
-						    audit->coreState.id, auditStates[auditStateIndex].range, auditType);
-						state StorageServerInterface targetServer;
-						state std::vector<StorageServerInterface> storageServersToCheck;
+						AuditStorageRequest req(audit->coreState.id, auditStates[auditStateIndex].range, auditType);
+						StorageServerInterface targetServer;
+						std::vector<StorageServerInterface> storageServersToCheck;
 						// Set req.targetServers and targetServer, which will be
 						// used to doAuditOnStorageServer
 						// Different audit types have different settings
@@ -4377,7 +4558,7 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 								    .detail("AuditID", audit->coreState.id)
 								    .detail("AuditRange", audit->coreState.range)
 								    .detail("AuditType", auditType);
-								return Void();
+								co_return;
 							}
 							// pick a server from primary DC
 							auto it = rangeLocations[rangeLocationIndex].servers.begin();
@@ -4418,7 +4599,7 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 								    .detail("AuditID", audit->coreState.id)
 								    .detail("AuditRange", audit->coreState.range)
 								    .detail("AuditType", auditType);
-								return Void();
+								co_return;
 							}
 						} else if (auditType == AuditType::ValidateRestore) {
 							// select a server from primary DC to do audit
@@ -4477,7 +4658,7 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 						// Set doAuditOnStorageServer
 						ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 						while (audit->remainingBudgetForAuditTasks.get() == 0) {
-							wait(audit->remainingBudgetForAuditTasks.onChange());
+							co_await audit->remainingBudgetForAuditTasks.onChange();
 							ASSERT(audit->remainingBudgetForAuditTasks.get() >= 0);
 						}
 						audit->remainingBudgetForAuditTasks.set(audit->remainingBudgetForAuditTasks.get() - 1);
@@ -4492,14 +4673,14 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 						// Check if the shard is in any specified storage engine
 						// If yes, issue doAuditOnStorageServer
 						// Otherwise, persist progress complete
-						state bool anySpecifiedEngine = false;
+						bool anySpecifiedEngine = false;
 						if (audit->coreState.engineType == KeyValueStoreType::END) {
 							// Do not specify any storage engine, so check for all engine types
 							anySpecifiedEngine = true;
 						} else {
 							try {
 								std::unordered_map<UID, KeyValueStoreType> storageTypeMapping =
-								    wait(getStorageType(storageServersToCheck));
+								    co_await getStorageType(storageServersToCheck);
 								for (int j = 0; j < storageServersToCheck.size(); j++) {
 									auto ssStorageType = storageTypeMapping.find(storageServersToCheck[j].id());
 									if (ssStorageType != storageTypeMapping.end()) {
@@ -4542,7 +4723,7 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 					TraceEvent(SevInfo, "DDScheduleAuditOnCurrentRangeTaskAssigned", self->ddId);
 				}
 				++assignedRangeTasks;
-				wait(delay(0.1));
+				co_await delay(0.1);
 			}
 			// Proceed to the next range if getSourceServerInterfacesForRange is partially read
 			currentRangeToScheduleBegin = rangeLocations.back().range.end;
@@ -4578,14 +4759,10 @@ ACTOR Future<Void> scheduleAuditOnRange(Reference<DataDistributor> self,
 		    .detail("IssuedDoAuditCountInThisSchedule", issueDoAuditCount);
 		audit->auditStorageAnyChildFailed = true;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> skipAuditOnRange(Reference<DataDistributor> self,
-                                    std::shared_ptr<DDAudit> audit,
-                                    KeyRange rangeToSkip) {
-	state AuditType auditType = audit->coreState.getType();
+Future<Void> skipAuditOnRange(Reference<DataDistributor> self, std::shared_ptr<DDAudit> audit, KeyRange rangeToSkip) {
+	AuditType auditType = audit->coreState.getType();
 	ASSERT(auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica ||
 	       auditType == AuditType::ValidateRestore);
 	try {
@@ -4593,7 +4770,7 @@ ACTOR Future<Void> skipAuditOnRange(Reference<DataDistributor> self,
 		AuditStorageState res(audit->coreState.id, rangeToSkip, auditType);
 		res.setPhase(AuditPhase::Complete);
 		res.ddId = self->ddId;
-		wait(persistAuditStateByRange(self->txnProcessor->context(), res));
+		co_await persistAuditStateByRange(self->txnProcessor->context(), res);
 		audit->overallSkippedDoAuditCount++;
 		TraceEvent(SevInfo, "DDSkipAuditOnRangeComplete", self->ddId)
 		    .detail("AuditID", audit->coreState.id)
@@ -4638,17 +4815,16 @@ ACTOR Future<Void> skipAuditOnRange(Reference<DataDistributor> self,
 			audit->actors.add(scheduleAuditOnRange(self, audit, rangeToSkip));
 		}
 	}
-	return Void();
 }
 
 // Request SS to do the audit
 // This actor is the only interface to SS to do the audit for
 // all audit types
-ACTOR Future<Void> doAuditOnStorageServer(Reference<DataDistributor> self,
-                                          std::shared_ptr<DDAudit> audit,
-                                          StorageServerInterface ssi,
-                                          AuditStorageRequest req) {
-	state AuditType auditType = req.getType();
+Future<Void> doAuditOnStorageServer(Reference<DataDistributor> self,
+                                    std::shared_ptr<DDAudit> audit,
+                                    StorageServerInterface ssi,
+                                    AuditStorageRequest req) {
+	AuditType auditType = req.getType();
 	ASSERT(auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica ||
 	       auditType == AuditType::ValidateStorageServerShard || auditType == AuditType::ValidateRestore);
 	TraceEvent(SevInfo, "DDDoAuditOnStorageServerBegin", self->ddId)
@@ -4665,7 +4841,7 @@ ACTOR Future<Void> doAuditOnStorageServer(Reference<DataDistributor> self,
 	try {
 		audit->overallIssuedDoAuditCount++;
 		ASSERT(req.ddId.isValid());
-		ErrorOr<AuditStorageState> vResult = wait(ssi.auditStorage.tryGetReply(req));
+		ErrorOr<AuditStorageState> vResult = co_await ssi.auditStorage.tryGetReply(req);
 		if (vResult.isError()) {
 			throw vResult.getError();
 		}
@@ -4738,50 +4914,50 @@ ACTOR Future<Void> doAuditOnStorageServer(Reference<DataDistributor> self,
 			audit->actors.add(scheduleAuditOnRange(self, audit, req.range));
 		}
 	}
-	return Void();
 }
 
 // Check consistency between KeyServers and ServerKeys system key space
-ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
-                                           std::shared_ptr<DDAudit> audit,
-                                           KeyRange auditRange) {
+Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
+                                     std::shared_ptr<DDAudit> audit,
+                                     KeyRange auditRange) {
 	ASSERT(audit->coreState.getType() == AuditType::ValidateLocationMetadata);
 	TraceEvent(SevInfo, "DDDoAuditLocationMetadataBegin", self->ddId)
 	    .detail("AuditId", audit->coreState.id)
 	    .detail("AuditRange", auditRange);
-	state AuditStorageState res(audit->coreState.id,
-	                            audit->coreState.getType()); // we will set range of audit later
-	state std::vector<Future<Void>> actors;
-	state std::vector<std::string> errors;
-	state AuditGetKeyServersRes keyServerRes;
-	state std::unordered_map<UID, AuditGetServerKeysRes> serverKeyResMap;
-	state Version readAtVersion;
-	state std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
+	AuditStorageState res(audit->coreState.id,
+	                      audit->coreState.getType()); // we will set range of audit later
+	std::vector<Future<Void>> actors;
+	std::vector<LocationMetadataError> errors;
+	AuditGetKeyServersRes keyServerRes;
+	std::unordered_map<UID, AuditGetServerKeysRes> serverKeyResMap;
+	Version readAtVersion{ 0 };
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServersRaw;
 	// Note that since krmReadRange may not return the value of the entire range at a time
 	// Given auditRange, a part of the range is returned, thus, only a part of the range is
 	// able to be compared --- claimRange
 	// Given claimRange, rangeToRead is decided for reading the remaining range
 	// At beginning, rangeToRead is auditRange
-	state KeyRange claimRange;
-	state KeyRange completeRangeByKeyServer;
+	KeyRange claimRange;
+	KeyRange completeRangeByKeyServer;
 	// To compare
-	state std::unordered_map<UID, std::vector<KeyRange>> mapFromServerKeys;
-	state std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServers;
-
-	state Transaction tr(self->txnProcessor->context());
-	state Key rangeToReadBegin = auditRange.begin;
-	state KeyRangeRef rangeToRead;
-	state int64_t cumulatedValidatedServerKeysNum = 0;
-	state int64_t cumulatedValidatedKeyServersNum = 0;
-	state Reference<IRateControl> rateLimiter =
-	    Reference<IRateControl>(new SpeedLimit(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1));
-	state int64_t remoteReadBytes = 0;
-	state double lastRateLimiterWaitTime = 0;
-	state double rateLimiterBeforeWaitTime = 0;
-	state double rateLimiterTotalWaitTime = 0;
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromServerKeys;
+	std::unordered_map<UID, std::vector<KeyRange>> mapFromKeyServers;
+
+	Transaction tr(self->txnProcessor->context());
+	Key rangeToReadBegin = auditRange.begin;
+	KeyRangeRef rangeToRead;
+	int64_t cumulatedValidatedServerKeysNum = 0;
+	int64_t cumulatedValidatedKeyServersNum = 0;
+	Reference<IRateControl> rateLimiter = makeReference<SpeedLimit>(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1);
+	int64_t remoteReadBytes = 0;
+	double lastRateLimiterWaitTime = 0;
+	double rateLimiterBeforeWaitTime = 0;
+	double rateLimiterTotalWaitTime = 0;
 
 	try {
-		loop {
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				// Read
 				actors.clear();
@@ -4798,7 +4974,7 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				// Read KeyServers
-				wait(store(keyServerRes, getShardMapFromKeyServers(self->ddId, &tr, rangeToRead)));
+				keyServerRes = co_await getShardMapFromKeyServers(self->ddId, &tr, rangeToRead);
 				completeRangeByKeyServer = keyServerRes.completeRange;
 				readAtVersion = keyServerRes.readAtVersion;
 				mapFromKeyServersRaw = keyServerRes.rangeOwnershipMap;
@@ -4808,7 +4984,7 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 					actors.push_back(
 					    store(serverKeyResMap[ssid], getThisServerKeysFromServerKeys(ssid, &tr, rangeToRead)));
 				}
-				wait(waitForAll(actors));
+				co_await waitForAll(actors);
 
 				// Decide claimRange and check readAtVersion
 				claimRange = completeRangeByKeyServer;
@@ -4835,101 +5011,33 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 					remoteReadBytes += serverKeyRes.readBytes;
 				}
 				// Use claimRange to get mapFromServerKeys and mapFromKeyServers to compare
-				int64_t numValidatedServerKeys = 0;
+				std::unordered_map<UID, std::vector<KeyRange>> serverOwnRangesMap;
 				for (auto& [ssid, serverKeyRes] : serverKeyResMap) {
-					for (auto& range : serverKeyRes.ownRanges) {
-						KeyRange overlappingRange = range & claimRange;
-						if (overlappingRange.empty()) {
-							continue;
-						}
-						TraceEvent(SevVerbose, "DDDoAuditLocationMetadataAddToServerKeyMap", self->ddId)
-						    .detail("RawRange", range)
-						    .detail("ClaimRange", claimRange)
-						    .detail("Range", overlappingRange)
-						    .detail("SSID", ssid);
-						mapFromServerKeys[ssid].push_back(overlappingRange);
-						numValidatedServerKeys++;
-					}
-				}
-				cumulatedValidatedServerKeysNum = cumulatedValidatedServerKeysNum + numValidatedServerKeys;
-
-				int64_t numValidatedKeyServers = 0;
-				for (auto& [ssid, ranges] : mapFromKeyServersRaw) {
-					std::vector mergedRanges = coalesceRangeList(ranges);
-					for (auto& range : mergedRanges) {
-						KeyRange overlappingRange = range & claimRange;
-						if (overlappingRange.empty()) {
-							continue;
-						}
-						TraceEvent(SevVerbose, "DDDoAuditLocationMetadataAddToKeyServerMap", self->ddId)
-						    .detail("RawRange", range)
-						    .detail("ClaimRange", claimRange)
-						    .detail("Range", overlappingRange)
-						    .detail("SSID", ssid);
-						mapFromKeyServers[ssid].push_back(overlappingRange);
-						numValidatedKeyServers++;
-					}
+					serverOwnRangesMap[ssid] = serverKeyRes.ownRanges;
 				}
-				cumulatedValidatedKeyServersNum = cumulatedValidatedKeyServersNum + numValidatedKeyServers;
+				LocationMetadataMaps builtMaps =
+				    buildLocationMetadataMaps(mapFromKeyServersRaw, serverOwnRangesMap, claimRange, self->ddId);
+				mapFromServerKeys = std::move(builtMaps.fromServerKeys);
+				mapFromKeyServers = std::move(builtMaps.fromKeyServers);
+				cumulatedValidatedServerKeysNum += builtMaps.numValidatedServerKeys;
+				cumulatedValidatedKeyServersNum += builtMaps.numValidatedKeyServers;
 
 				// Compare: check if mapFromKeyServers === mapFromServerKeys
-				// 1. check mapFromKeyServers => mapFromServerKeys
-				for (auto& [ssid, keyServerRanges] : mapFromKeyServers) {
-					if (!mapFromServerKeys.contains(ssid)) {
-						std::string error =
-						    format("KeyServers and serverKeys mismatch: Some key in range(%s, %s) exists "
-						           "on Server(%s) in KeyServers but not ServerKeys",
-						           claimRange.toString().c_str(),
-						           claimRange.toString().c_str(),
-						           ssid.toString().c_str());
-						errors.push_back(error);
-						TraceEvent(SevError, "DDDoAuditLocationMetadataError", self->ddId)
-						    .setMaxFieldLength(-1)
-						    .setMaxEventLength(-1)
-						    .detail("AuditId", audit->coreState.id)
-						    .detail("AuditRange", auditRange)
-						    .detail("ClaimRange", claimRange)
-						    .detail("ErrorMessage", error);
-					}
-					std::vector<KeyRange> serverKeyRanges = mapFromServerKeys[ssid];
-					Optional<std::pair<KeyRange, KeyRange>> anyMismatch = rangesSame(keyServerRanges, serverKeyRanges);
-					if (anyMismatch.present()) { // mismatch detected
-						KeyRange mismatchedRangeByKeyServer = anyMismatch.get().first;
-						KeyRange mismatchedRangeByServerKey = anyMismatch.get().second;
-						std::string error =
-						    format("KeyServers and serverKeys mismatch on Server(%s): KeyServer: %s; ServerKey: %s",
-						           ssid.toString().c_str(),
-						           mismatchedRangeByKeyServer.toString().c_str(),
-						           mismatchedRangeByServerKey.toString().c_str());
-						errors.push_back(error);
-						TraceEvent(SevError, "DDDoAuditLocationMetadataError", self->ddId)
-						    .setMaxFieldLength(-1)
-						    .setMaxEventLength(-1)
-						    .detail("AuditId", audit->coreState.id)
-						    .detail("AuditRange", auditRange)
-						    .detail("ClaimRange", claimRange)
-						    .detail("ErrorMessage", error)
-						    .detail("MismatchedRangeByKeyServer", mismatchedRangeByKeyServer)
-						    .detail("MismatchedRangeByServerKey", mismatchedRangeByServerKey);
+				errors = checkLocationMetadataConsistency(mapFromKeyServers, mapFromServerKeys, claimRange);
+				for (const auto& error : errors) {
+					auto te = TraceEvent(SevError, "DDDoAuditLocationMetadataError", self->ddId);
+					te.setMaxFieldLength(-1)
+					    .setMaxEventLength(-1)
+					    .detail("AuditId", audit->coreState.id)
+					    .detail("AuditRange", auditRange)
+					    .detail("ClaimRange", claimRange)
+					    .detail("ErrorMessage", error.message)
+					    .detail("ServerId", error.serverId);
+					if (error.mismatchedRangeByKeyServer.present()) {
+						te.detail("MismatchedRangeByKeyServer", error.mismatchedRangeByKeyServer.get());
 					}
-				}
-				// 2. check mapFromServerKeys => mapFromKeyServers
-				for (auto& [ssid, serverKeyRanges] : mapFromServerKeys) {
-					if (!mapFromKeyServers.contains(ssid)) {
-						std::string error =
-						    format("KeyServers and serverKeys mismatch: Some key of range(%s, %s) exists "
-						           "on Server(%s) in ServerKeys but not KeyServers",
-						           claimRange.toString().c_str(),
-						           claimRange.toString().c_str(),
-						           ssid.toString().c_str());
-						errors.push_back(error);
-						TraceEvent(SevError, "DDDoAuditLocationMetadataError", self->ddId)
-						    .setMaxFieldLength(-1)
-						    .setMaxEventLength(-1)
-						    .detail("AuditId", audit->coreState.id)
-						    .detail("AuditRange", auditRange)
-						    .detail("ClaimRange", claimRange)
-						    .detail("ErrorMessage", error);
+					if (error.mismatchedRangeByServerKey.present()) {
+						te.detail("MismatchedRangeByServerKey", error.mismatchedRangeByServerKey.get());
 					}
 				}
 
@@ -4939,8 +5047,8 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 				    .detail("AuditType", audit->coreState.getType())
 				    .detail("AuditId", audit->coreState.id)
 				    .detail("AuditRange", auditRange)
-				    .detail("CurrentValidatedServerKeysNum", numValidatedServerKeys)
-				    .detail("CurrentValidatedKeyServersNum", numValidatedServerKeys)
+				    .detail("CurrentValidatedServerKeysNum", builtMaps.numValidatedServerKeys)
+				    .detail("CurrentValidatedKeyServersNum", builtMaps.numValidatedKeyServers)
 				    .detail("CurrentValidatedInclusiveRange", claimRange)
 				    .detail("CumulatedValidatedServerKeysNum", cumulatedValidatedServerKeysNum)
 				    .detail("CumulatedValidatedKeyServersNum", cumulatedValidatedKeyServersNum)
@@ -4959,14 +5067,14 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 					res.range = claimRange;
 					res.setPhase(AuditPhase::Error);
 					res.ddId = self->ddId; // used to compare self->ddId with existing persisted ddId
-					wait(persistAuditStateByRange(self->txnProcessor->context(), res));
+					co_await persistAuditStateByRange(self->txnProcessor->context(), res);
 					throw audit_storage_error();
 				} else {
 					// Expand persisted complete range
 					res.range = Standalone(KeyRangeRef(auditRange.begin, claimRange.end));
 					res.setPhase(AuditPhase::Complete);
 					res.ddId = self->ddId; // used to compare self->ddId with existing persisted ddId
-					wait(persistAuditStateByRange(self->txnProcessor->context(), res));
+					co_await persistAuditStateByRange(self->txnProcessor->context(), res);
 					if (res.range.end < auditRange.end) {
 						TraceEvent(SevInfo, "DDDoAuditLocationMetadataPartialDone", self->ddId)
 						    .suppressFor(10.0)
@@ -4989,11 +5097,15 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 					}
 				}
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await tr.onError(err);
 			}
 
 			rateLimiterBeforeWaitTime = now();
-			wait(rateLimiter->getAllowance(remoteReadBytes)); // Rate Keeping
+			co_await rateLimiter->getAllowance(remoteReadBytes); // Rate Keeping
 			lastRateLimiterWaitTime = now() - rateLimiterBeforeWaitTime;
 			rateLimiterTotalWaitTime = rateLimiterTotalWaitTime + lastRateLimiterWaitTime;
 		}
@@ -5030,21 +5142,20 @@ ACTOR Future<Void> doAuditLocationMetadata(Reference<DataDistributor> self,
 			audit->actors.add(dispatchAuditLocationMetadata(self, audit, auditRange));
 		}
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> dataDistributor_impl(DataDistributorInterface di,
-                                        Reference<DataDistributor> self,
-                                        IsMocked isMocked) {
-	state Future<Void> collection = actorCollection(self->addActor.getFuture());
-	state PromiseStream<GetMetricsListRequest> getShardMetricsList;
-	state Database cx;
-	state ActorCollection actors(false);
-	state std::map<UID, DistributorSnapRequest> ddSnapReqMap;
-	state std::map<UID, ErrorOr<Void>> ddSnapReqResultMap;
+Future<Void> dataDistributor_impl(DataDistributorInterface di, Reference<DataDistributor> self, IsMocked isMocked) {
+	Future<Void> collection = actorCollection(self->addActor.getFuture());
+	PromiseStream<GetMetricsListRequest> getShardMetricsList;
+	Database cx;
+	ActorCollection actors(false);
+	std::map<UID, DistributorSnapRequest> ddSnapReqMap;
+	std::map<UID, ErrorOr<Void>> ddSnapReqResultMap;
 
 	TraceEvent("DataDistributorRunning", di.id()).detail("IsMocked", isMocked);
+	// DDInitRunning duplicates the above with DDInit* prefix so the full startup sequence
+	// can be queried with Type="DDInit*" in trace logs
+	TraceEvent("DDInitRunning", di.id());
 	self->addActor.send(actors.getResult());
 	self->addActor.send(traceRole(Role::DATA_DISTRIBUTOR, di.id()));
 	self->addActor.send(waitFailureServer(di.waitFailure.getFuture()));
@@ -5052,26 +5163,34 @@ ACTOR Future<Void> dataDistributor_impl(DataDistributorInterface di,
 		cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultDelay, LockAware::True);
 	}
 
-	state Future<Void> distributor = reportErrorsExcept(dataDistribution(self, getShardMetricsList, isMocked),
-	                                                    "DataDistribution",
-	                                                    di.id(),
-	                                                    &normalDataDistributorErrors());
+	Future<Void> distributor = reportErrorsExcept(dataDistribution(self, getShardMetricsList, isMocked),
+	                                              "DataDistribution",
+	                                              di.id(),
+	                                              &normalDataDistributorErrors());
 
 	try {
-		loop choose {
-			when(wait(distributor || collection)) {
+		while (true) {
+			auto res = co_await race(distributor || collection,
+			                         di.haltDataDistributor.getFuture(),
+			                         di.dataDistributorMetrics.getFuture(),
+			                         di.distributorSnapReq.getFuture(),
+			                         di.distributorExclCheckReq.getFuture(),
+			                         di.storageWigglerState.getFuture(),
+			                         di.triggerAudit.getFuture());
+			if (res.index() == 0) {
+				// distributor or collection should never return
 				ASSERT(false);
 				throw internal_error();
-			}
-			when(HaltDataDistributorRequest req = waitNext(di.haltDataDistributor.getFuture())) {
+			} else if (res.index() == 1) {
+				HaltDataDistributorRequest req = std::get<1>(std::move(res));
 				req.reply.send(Void());
 				TraceEvent("DataDistributorHalted", di.id()).detail("ReqID", req.requesterID);
-				break;
-			}
-			when(GetDataDistributorMetricsRequest req = waitNext(di.dataDistributorMetrics.getFuture())) {
+				co_return;
+			} else if (res.index() == 2) {
+				GetDataDistributorMetricsRequest req = std::get<2>(std::move(res));
 				actors.add(ddGetMetrics(req, getShardMetricsList));
-			}
-			when(DistributorSnapRequest snapReq = waitNext(di.distributorSnapReq.getFuture())) {
+			} else if (res.index() == 3) {
+				DistributorSnapRequest snapReq = std::get<3>(std::move(res));
 				auto& snapUID = snapReq.snapUID;
 				if (ddSnapReqResultMap.contains(snapUID)) {
 					CODE_PROBE(true,
@@ -5104,21 +5223,22 @@ ACTOR Future<Void> dataDistributor_impl(DataDistributorInterface di,
 					                         &ddSnapReqResultMap),
 					            SERVER_KNOBS->SNAP_MINIMUM_TIME_GAP)));
 				}
-			}
-			when(DistributorExclusionSafetyCheckRequest exclCheckReq =
-			         waitNext(di.distributorExclCheckReq.getFuture())) {
+			} else if (res.index() == 4) {
+				DistributorExclusionSafetyCheckRequest exclCheckReq = std::get<4>(std::move(res));
 				actors.add(ddExclusionSafetyCheck(exclCheckReq, self, cx));
-			}
-			when(GetStorageWigglerStateRequest req = waitNext(di.storageWigglerState.getFuture())) {
+			} else if (res.index() == 5) {
+				GetStorageWigglerStateRequest req = std::get<5>(std::move(res));
 				req.reply.send(getStorageWigglerStates(self));
-			}
-			when(TriggerAuditRequest req = waitNext(di.triggerAudit.getFuture())) {
+			} else if (res.index() == 6) {
+				TriggerAuditRequest req = std::get<6>(std::move(res));
 				if (req.cancel) {
 					ASSERT(req.id.isValid());
 					actors.add(cancelAuditStorage(self, req));
-					continue;
+				} else {
+					actors.add(auditStorage(self, req));
 				}
-				actors.add(auditStorage(self, req));
+			} else {
+				UNREACHABLE();
 			}
 		}
 	} catch (Error& err) {
@@ -5128,12 +5248,10 @@ ACTOR Future<Void> dataDistributor_impl(DataDistributorInterface di,
 		}
 		TraceEvent("DataDistributorDied", di.id()).errorUnsuppressed(err);
 	}
-
-	return Void();
 }
 
 Future<Void> MockDataDistributor::run(Reference<DDSharedContext> context, Reference<DDMockTxnProcessor> txnProcessor) {
-	Reference<DataDistributor> dd =
+	auto dd =
 	    makeReference<DataDistributor>(Reference<AsyncVar<ServerDBInfo> const>(nullptr), context->ddId, context, "");
 	dd->txnProcessor = txnProcessor;
 	return dataDistributor_impl(context->interface, dd, IsMocked::True);
@@ -5187,9 +5305,9 @@ TEST_CASE("/DataDistribution/StorageWiggler/Order") {
 }
 
 TEST_CASE("/DataDistribution/Initialization/ResumeFromShard") {
-	state Reference<DDSharedContext> context(new DDSharedContext(UID()));
-	state Reference<AsyncVar<ServerDBInfo> const> dbInfo;
-	state Reference<DataDistributor> self(new DataDistributor(dbInfo, UID(), context, ""));
+	Reference<DDSharedContext> context(new DDSharedContext(UID()));
+	Reference<AsyncVar<ServerDBInfo> const> dbInfo;
+	Reference<DataDistributor> self(new DataDistributor(dbInfo, UID(), context, ""));
 
 	self->shardsAffectedByTeamFailure = makeReference<ShardsAffectedByTeamFailure>();
 	if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD) {
@@ -5212,7 +5330,7 @@ TEST_CASE("/DataDistribution/Initialization/ResumeFromShard") {
 	}
 	self->initData->shards.emplace_back(DDShardInfo(allKeys.end));
 	std::cout << "Start resuming...\n";
-	wait(DataDistributor::resumeFromShards(self, false));
+	co_await DataDistributor::resumeFromShards(self, false);
 	std::cout << "Start validation...\n";
 	auto relocateFuture = self->relocationProducer.getFuture();
 	for (int i = 0; i < SERVER_KNOBS->DD_MOVE_KEYS_PARALLELISM; ++i) {
@@ -5228,5 +5346,5 @@ TEST_CASE("/DataDistribution/Initialization/ResumeFromShard") {
 	}
 	self->shardsAffectedByTeamFailure->setCheckMode(ShardsAffectedByTeamFailure::CheckMode::ForceCheck);
 	self->shardsAffectedByTeamFailure->check();
-	return Void();
+	co_return;
 }
diff --git a/fdbserver/include/fdbserver/ExclusionTracker.actor.h b/fdbserver/datadistributor/ExclusionTracker.h
similarity index 68%
rename from fdbserver/include/fdbserver/ExclusionTracker.actor.h
rename to fdbserver/datadistributor/ExclusionTracker.h
index 13ed5c27855..c2dd0949850 100644
--- a/fdbserver/include/fdbserver/ExclusionTracker.actor.h
+++ b/fdbserver/datadistributor/ExclusionTracker.h
@@ -1,5 +1,5 @@
 /*
- * ExclusionTracker.actor.h
+ * ExclusionTracker.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,20 +20,13 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(EXCLUSION_TRACKER_ACTOR_G_H)
-#define EXCLUSION_TRACKER_ACTOR_G_H
-#include "fdbserver/ExclusionTracker.actor.g.h"
-#elif !defined(EXCLUSION_TRACKER_ACTOR_H)
-#define EXCLUSION_TRACKER_ACTOR_H
-
 #include <set>
+
+#include "flow/CoroUtils.h"
 #include "flow/flow.h"
 #include "flow/Trace.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/ManagementAPI.h"
 
 struct ExclusionTracker {
 	std::set<AddressExclusion> excluded;
@@ -45,7 +38,7 @@ struct ExclusionTracker {
 	Future<Void> trackerFuture;
 
 	ExclusionTracker() {}
-	ExclusionTracker(Database db) : db(db) { trackerFuture = tracker(this); }
+	explicit ExclusionTracker(Database db) : db(db) { trackerFuture = tracker(this); }
 
 	bool isFailedOrExcluded(NetworkAddress addr) {
 		AddressExclusion addrExclusion(addr.ip, addr.port);
@@ -54,38 +47,39 @@ struct ExclusionTracker {
 
 	// Note the tracker is intended to be used by the Data Distributor. The tracker will check for excluded localities
 	// based on the server list, the server list only includes storage processes.
-	ACTOR static Future<Void> tracker(ExclusionTracker* self) {
+	static Future<Void> tracker(ExclusionTracker* self) {
 		// Fetch the list of excluded servers
-		state ReadYourWritesTransaction tr(self->db);
-		loop {
+		ReadYourWritesTransaction tr(self->db);
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				state Future<RangeResult> fresultsExclude = tr.getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fresultsFailed = tr.getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> flocalitiesExclude =
-				    tr.getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> flocalitiesFailed = tr.getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fServerList = tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> fresultsExclude = tr.getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> fresultsFailed = tr.getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> flocalitiesExclude = tr.getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> flocalitiesFailed = tr.getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> fServerList = tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
 
-				wait(success(fresultsExclude) && success(fresultsFailed) && success(flocalitiesExclude) &&
-				     success(flocalitiesFailed));
+				co_await (success(fresultsExclude) && success(fresultsFailed) && success(flocalitiesExclude) &&
+				          success(flocalitiesFailed));
 
-				state RangeResult excludedResults = fresultsExclude.get();
+				RangeResult excludedResults = fresultsExclude.get();
 				ASSERT(!excludedResults.more && excludedResults.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state RangeResult failedResults = fresultsFailed.get();
+				RangeResult failedResults = fresultsFailed.get();
 				ASSERT(!failedResults.more && failedResults.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state RangeResult excludedLocalityResults = flocalitiesExclude.get();
+				RangeResult excludedLocalityResults = flocalitiesExclude.get();
 				ASSERT(!excludedLocalityResults.more && excludedLocalityResults.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state RangeResult failedLocalityResults = flocalitiesFailed.get();
+				RangeResult failedLocalityResults = flocalitiesFailed.get();
 				ASSERT(!failedLocalityResults.more && failedLocalityResults.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state std::set<AddressExclusion> newExcluded;
-				state std::set<AddressExclusion> newFailed;
+				std::set<AddressExclusion> newExcluded;
+				std::set<AddressExclusion> newFailed;
 				for (const auto& r : excludedResults) {
 					AddressExclusion addr = decodeExcludedServersKey(r.key);
 					if (addr.isValid()) {
@@ -99,23 +93,23 @@ struct ExclusionTracker {
 					}
 				}
 
-				wait(success(fServerList));
+				co_await fServerList;
 				// In some cases it can happen that the process is not running, e.g. because the process is down
 				// for maintenance. In this case the process will not be part of the worker list, but the process
 				// might be a storage server and could be part of the server list.
 				// See: https://github.com/apple/foundationdb/issues/12168
-				state std::vector<std::pair<std::string, std::string>> decodedExcludedLocalities;
+				std::vector<std::pair<std::string, std::string>> decodedExcludedLocalities;
 				for (auto& excludedLocality : excludedLocalityResults) {
 					decodedExcludedLocalities.push_back(
 					    decodeLocality(decodeExcludedLocalityKey(excludedLocality.key)));
 				}
 
-				state std::vector<std::pair<std::string, std::string>> decodedFailedLocalities;
+				std::vector<std::pair<std::string, std::string>> decodedFailedLocalities;
 				for (auto& failedLocality : failedLocalityResults) {
 					decodedFailedLocalities.push_back(decodeLocality(decodeFailedLocalityKey(failedLocality.key)));
 				}
 
-				state RangeResult serverList = fServerList.get();
+				RangeResult serverList = fServerList.get();
 				for (auto& s : serverList) {
 					auto decodedServer = decodeServerListValue(s.value);
 					// Check if the server is excluded based on a locality.
@@ -169,24 +163,24 @@ struct ExclusionTracker {
 					self->changed.trigger();
 				}
 
-				state Future<Void> watchFuture =
-				    tr.watch(excludedServersVersionKey) || tr.watch(failedServersVersionKey) ||
-				    tr.watch(excludedLocalityVersionKey) || tr.watch(failedLocalityVersionKey);
-				wait(tr.commit());
+				Future<Void> watchFuture = tr.watch(excludedServersVersionKey) || tr.watch(failedServersVersionKey) ||
+				                           tr.watch(excludedLocalityVersionKey) || tr.watch(failedLocalityVersionKey);
+				co_await tr.commit();
 				if (excludedLocalityResults.size() > 0 || failedLocalityResults.size() > 0) {
 					// when there are excluded localities we need to monitor for when the worker list changes, so we
 					// must poll
 					watchFuture = watchFuture || delay(10.0);
 				}
-				wait(watchFuture);
+				co_await watchFuture;
 				tr.reset();
 			} catch (Error& e) {
-				TraceEvent("ExclusionTrackerError").error(e);
-				wait(tr.onError(e));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				TraceEvent("ExclusionTrackerError").error(err);
+				co_await tr.onError(err);
 			}
 		}
 	}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/MockGlobalState.actor.cpp b/fdbserver/datadistributor/MockGlobalState.cpp
similarity index 93%
rename from fdbserver/MockGlobalState.actor.cpp
rename to fdbserver/datadistributor/MockGlobalState.cpp
index f86e5caa719..9ab23d0a78e 100644
--- a/fdbserver/MockGlobalState.actor.cpp
+++ b/fdbserver/datadistributor/MockGlobalState.cpp
@@ -1,5 +1,5 @@
 /*
- * MockGlobalState.actor.cpp
+ * MockGlobalState.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,23 +18,23 @@
  * limitations under the License.
  */
 
-#include "fdbserver/MockGlobalState.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/datadistributor/MockGlobalState.h"
+#include "fdbserver/core/WorkloadKeys.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
 #include "fdbclient/FDBTypes.h"
-#include "flow/actorcompiler.h"
+#include "flow/CoroUtils.h"
 
 class MockGlobalStateImpl {
 public:
-	ACTOR static Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(MockGlobalState* mgs,
-	                                                                                 KeyRange keys,
-	                                                                                 StorageMetrics min,
-	                                                                                 StorageMetrics max,
-	                                                                                 StorageMetrics permittedError,
-	                                                                                 int shardLimit,
-	                                                                                 int expectedShardCount) {
-		state Version version = 0;
-		loop {
+	static Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(MockGlobalState* mgs,
+	                                                                           KeyRange keys,
+	                                                                           StorageMetrics min,
+	                                                                           StorageMetrics max,
+	                                                                           StorageMetrics permittedError,
+	                                                                           int shardLimit,
+	                                                                           int expectedShardCount) {
+		Version version = 0;
+		while (true) {
 			auto locations = mgs->getKeyRangeLocations(keys,
 			                                           shardLimit,
 			                                           Reverse::False,
@@ -55,9 +55,11 @@ class MockGlobalStateImpl {
 			           "Some shard is in the same location.",
 			           probe::decoration::rare);
 
+			Error err;
+			bool hasErr = false;
 			try {
 				Optional<StorageMetrics> res =
-				    wait(::waitStorageMetricsWithLocation(version, keys, locations, min, max, permittedError));
+				    co_await ::waitStorageMetricsWithLocation(version, keys, locations, min, max, permittedError);
 
 				TraceEvent(SevDebug, "MGSWaitStorageMetrics")
 				    .detail("Phase", "GetStorageMetrics")
@@ -65,32 +67,36 @@ class MockGlobalStateImpl {
 				    .detail("Present", res.present());
 
 				if (res.present()) {
-					return std::make_pair(res, -1);
+					co_return std::make_pair(res, -1);
 				}
 			} catch (Error& e) {
-				TraceEvent(SevDebug, "MGSWaitStorageMetricsHandleError").error(e);
-				if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
-					wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
-				} else if (e.code() == error_code_future_version) {
-					wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				TraceEvent(SevDebug, "MGSWaitStorageMetricsHandleError").error(err);
+				if (err.code() == error_code_wrong_shard_server || err.code() == error_code_all_alternatives_failed) {
+					co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
+				} else if (err.code() == error_code_future_version) {
+					co_await delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution);
 				} else {
-					TraceEvent(SevError, "MGSWaitStorageMetricsError").error(e);
-					throw;
+					TraceEvent(SevError, "MGSWaitStorageMetricsError").error(err);
+					throw err;
 				}
 			}
 			// Avoid busy spin
-			wait(delay(0.1, TaskPriority::DataDistribution));
+			co_await delay(0.1, TaskPriority::DataDistribution);
 		}
 	}
 
 	// SOMEDAY: reuse the NativeAPI implementation
-	ACTOR static Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(MockGlobalState* mgs,
-	                                                                       KeyRange keys,
-	                                                                       StorageMetrics limit,
-	                                                                       StorageMetrics estimated,
-	                                                                       Optional<int> minSplitBytes) {
-		loop {
-			state std::vector<KeyRangeLocationInfo> locations =
+	static Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(MockGlobalState* mgs,
+	                                                                 KeyRange keys,
+	                                                                 StorageMetrics limit,
+	                                                                 StorageMetrics estimated,
+	                                                                 Optional<int> minSplitBytes) {
+		while (true) {
+			std::vector<KeyRangeLocationInfo> locations =
 			    mgs->getKeyRangeLocations(keys,
 			                              CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
 			                              Reverse::False,
@@ -102,50 +108,48 @@ class MockGlobalStateImpl {
 
 			// Same solution to NativeAPI::splitStorageMetrics, wait some merge finished
 			if (locations.size() == CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT) {
-				wait(delay(CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, TaskPriority::DataDistribution));
+				co_await delay(CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, TaskPriority::DataDistribution);
 			}
 
 			Optional<Standalone<VectorRef<KeyRef>>> results =
-			    wait(splitStorageMetricsWithLocations(locations, keys, limit, estimated, minSplitBytes));
+			    co_await splitStorageMetricsWithLocations(locations, keys, limit, estimated, minSplitBytes);
 
 			if (results.present()) {
-				return results.get();
+				co_return results.get();
 			}
 
-			wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
+			co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
 		}
 	}
 };
 
 class MockStorageServerImpl {
 public:
-	ACTOR static Future<Void> waitMetricsForReal(MockStorageServer* self, WaitMetricsRequest req) {
+	static Future<Void> waitMetricsForReal(MockStorageServer* self, WaitMetricsRequest req) {
 		if (!self->isReadable(req.keys)) {
 			self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 		} else {
-			wait(self->metrics.waitMetrics(req, delayJittered(SERVER_KNOBS->STORAGE_METRIC_TIMEOUT)));
+			co_await self->metrics.waitMetrics(req, delayJittered(SERVER_KNOBS->STORAGE_METRIC_TIMEOUT));
 		}
-		return Void();
 	}
 
 	// Randomly generate keys and kv size between the fetch range, updating the byte sample.
 	// Once the fetchKeys return, the shard status will become FETCHED.
-	ACTOR static Future<Void> waitFetchKeysFinish(MockStorageServer* self, MockStorageServer::FetchKeysParams params) {
-		state TraceInterval interval("MockFetchKeys");
+	static Future<Void> waitFetchKeysFinish(MockStorageServer* self, MockStorageServer::FetchKeysParams params) {
+		TraceInterval interval("MockFetchKeys");
 		// between each chunk delay for random time, and finally set the fetchComplete signal.
 		ASSERT(params.totalRangeBytes > 0);
-		state int chunkCount = std::ceil(params.totalRangeBytes * 1.0 / SERVER_KNOBS->FETCH_BLOCK_BYTES);
-		state int64_t currentTotal = 0;
-		state Key lastKey = params.keys.begin;
+		int chunkCount = std::ceil(params.totalRangeBytes * 1.0 / SERVER_KNOBS->FETCH_BLOCK_BYTES);
+		int64_t currentTotal = 0;
+		Key lastKey = params.keys.begin;
 
 		TraceEvent(SevDebug, interval.begin(), self->id)
 		    .detail("Range", params.keys)
 		    .detail("ChunkCount", chunkCount)
 		    .detail("TotalBytes", params.totalRangeBytes);
 
-		state int i = 0;
-		for (; i < chunkCount && currentTotal < params.totalRangeBytes; ++i) {
-			wait(delayJittered(0.1, TaskPriority::FetchKeys));
+		for (int i = 0; i < chunkCount && currentTotal < params.totalRangeBytes; ++i) {
+			co_await delayJittered(0.1, TaskPriority::FetchKeys);
 
 			int remainedBytes = (chunkCount == 1 ? params.totalRangeBytes : SERVER_KNOBS->FETCH_BLOCK_BYTES);
 
@@ -191,7 +195,6 @@ class MockStorageServerImpl {
 
 		self->setShardStatus(params.keys, MockShardStatus::FETCHED);
 		TraceEvent(SevDebug, interval.end(), self->id).log();
-		return Void();
 	}
 };
 
@@ -654,7 +657,7 @@ void MockGlobalState::addStorageServer(StorageServerInterface server, uint64_t d
 }
 
 void MockGlobalState::addStoragePerProcess(uint64_t defaultDiskSpace) {
-	for (auto p : processes) {
+	for (const auto& p : processes) {
 		if (p->ssInterfaces.empty()) {
 			p->ssInterfaces.emplace_back(deterministicRandom()->randomUniqueID());
 			p->ssInterfaces.back().locality = p->locality;
@@ -915,7 +918,7 @@ TEST_CASE("/MockGlobalState/initializeAsEmptyDatabaseMGS/SimpleThree") {
 
 	BasicSimulationConfig dbConfig = generateBasicSimulationConfig(testConfig);
 	TraceEvent("UnitTestDBConfig").detail("Config", dbConfig.db.toString());
-	std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
+	auto mgs = std::make_shared<MockGlobalState>();
 	mgs->initializeClusterLayout(dbConfig);
 	mgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 
@@ -1001,7 +1004,7 @@ TEST_CASE("/MockGlobalState/MockStorageServer/SplittingFunctions") {
 	testConfig.logAntiQuorum = 0;
 	BasicSimulationConfig dbConfig = generateBasicSimulationConfig(testConfig);
 	TraceEvent("UnitTestDBConfig").detail("Config", dbConfig.db.toString());
-	std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
+	auto mgs = std::make_shared<MockGlobalState>();
 	mgs->initializeClusterLayout(dbConfig);
 	mgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 
@@ -1023,7 +1026,7 @@ TEST_CASE("/MockGlobalState/MockStorageServer/SetShardStatus") {
 	testConfig.logAntiQuorum = 0;
 	BasicSimulationConfig dbConfig = generateBasicSimulationConfig(testConfig);
 	TraceEvent("UnitTestDBConfig").detail("Config", dbConfig.db.toString());
-	state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
+	std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
 	mgs->initializeClusterLayout(dbConfig);
 	mgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 
@@ -1089,7 +1092,7 @@ TEST_CASE("/MockGlobalState/MockStorageServer/GetKeyLocations") {
 	testConfig.logAntiQuorum = 0;
 	BasicSimulationConfig dbConfig = generateBasicSimulationConfig(testConfig);
 	TraceEvent("UnitTestDBConfig").detail("Config", dbConfig.db.toString());
-	state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
+	std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
 	mgs->initializeClusterLayout(dbConfig);
 	mgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 	// add one empty server
@@ -1156,7 +1159,7 @@ TEST_CASE("/MockGlobalState/MockStorageServer/WaitStorageMetricsRequest") {
 
 	BasicSimulationConfig dbConfig = generateBasicSimulationConfig(testConfig);
 	TraceEvent("UnitTestDBConfig").detail("Config", dbConfig.db.toString());
-	state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
+	std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
 	mgs->initializeClusterLayout(dbConfig);
 	mgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 
@@ -1164,17 +1167,17 @@ TEST_CASE("/MockGlobalState/MockStorageServer/WaitStorageMetricsRequest") {
 		server.second->metrics.byteSample.sample.insert("something"_sr, 500000);
 	});
 
-	state Future<Void> allServerFutures = waitForAll(mgs->runAllMockServers());
+	auto allServerFutures = waitForAll(mgs->runAllMockServers());
+	(void)allServerFutures;
 
 	KeyRange testRange = allKeys;
 	ShardSizeBounds bounds = ShardSizeBounds::shardSizeBoundsBeforeTrack();
 	std::pair<Optional<StorageMetrics>, int> res =
-	    wait(mgs->waitStorageMetrics(testRange, bounds.min, bounds.max, bounds.permittedError, 1, 1));
+	    co_await mgs->waitStorageMetrics(testRange, bounds.min, bounds.max, bounds.permittedError, 1, 1);
 	// std::cout << "get result " << res.second << "\n";
 	// std::cout << "get byte "<< res.first.get().bytes << "\n";
 	ASSERT_EQ(res.second, -1); // the valid result always return -1, strange contraction though.
 	ASSERT_EQ(res.first.get().bytes, 500000);
-	return Void();
 }
 
 TEST_CASE("/MockGlobalState/MockStorageServer/DataOpsSet") {
@@ -1185,11 +1188,12 @@ TEST_CASE("/MockGlobalState/MockStorageServer/DataOpsSet") {
 
 	BasicSimulationConfig dbConfig = generateBasicSimulationConfig(testConfig);
 	TraceEvent("UnitTestDBConfig").detail("Config", dbConfig.db.toString());
-	state std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
+	std::shared_ptr<MockGlobalState> mgs = std::make_shared<MockGlobalState>();
 	mgs->initializeClusterLayout(dbConfig);
 	mgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 
-	state Future<Void> allServerFutures = waitForAll(mgs->runAllMockServers());
+	auto allServerFutures = waitForAll(mgs->runAllMockServers());
+	(void)allServerFutures;
 
 	// insert
 	{
@@ -1201,8 +1205,8 @@ TEST_CASE("/MockGlobalState/MockStorageServer/DataOpsSet") {
 			ASSERT_EQ(server.second->serverKeys[""_sr].shardSize, 3 + 6 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE);
 		}
 		ShardSizeBounds bounds = ShardSizeBounds::shardSizeBoundsBeforeTrack();
-		std::pair<Optional<StorageMetrics>, int> res = wait(
-		    mgs->waitStorageMetrics(KeyRangeRef("a"_sr, "bc"_sr), bounds.min, bounds.max, bounds.permittedError, 1, 1));
+		std::pair<Optional<StorageMetrics>, int> res = co_await mgs->waitStorageMetrics(
+		    KeyRangeRef("a"_sr, "bc"_sr), bounds.min, bounds.max, bounds.permittedError, 1, 1);
 
 		int64_t testSize = 2 + 3 * SERVER_KNOBS->BYTES_WRITTEN_UNITS_PER_SAMPLE;
 		// SOMEDAY: how to integrate with isKeyValueInSample() better?
@@ -1212,5 +1216,4 @@ TEST_CASE("/MockGlobalState/MockStorageServer/DataOpsSet") {
 			ASSERT_GT(res.first.get().bytesWrittenPerKSecond, 0);
 		}
 	}
-	return Void();
 }
diff --git a/fdbserver/include/fdbserver/MovingWindow.h b/fdbserver/datadistributor/MovingWindow.h
similarity index 97%
rename from fdbserver/include/fdbserver/MovingWindow.h
rename to fdbserver/datadistributor/MovingWindow.h
index 48d3c33bd64..3c50df64843 100644
--- a/fdbserver/include/fdbserver/MovingWindow.h
+++ b/fdbserver/datadistributor/MovingWindow.h
@@ -25,7 +25,7 @@
 
 #include <limits.h>
 #include "flow/Deque.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 
 // MovingWindow::addSample() will sum up all the samples, and getAverage() method will return the average
 // sampling rate in the past <timeWindow>, where every sample weights evenly.
@@ -84,4 +84,4 @@ class MovingWindow {
 	}
 };
 
-#endif // FOUNDATIONDB_MOVINGWINDOW_H
\ No newline at end of file
+#endif // FOUNDATIONDB_MOVINGWINDOW_H
diff --git a/fdbserver/ShardsAffectedByTeamFailure.cpp b/fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp
similarity index 97%
rename from fdbserver/ShardsAffectedByTeamFailure.cpp
rename to fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp
index 013b0f2bf20..7ec821b60ef 100644
--- a/fdbserver/ShardsAffectedByTeamFailure.cpp
+++ b/fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/ShardsAffectedByTeamFailure.h"
+#include "fdbserver/datadistributor/ShardsAffectedByTeamFailure.h"
 
 std::vector<KeyRange> ShardsAffectedByTeamFailure::getShardsFor(Team team) const {
 	std::vector<KeyRange> r;
@@ -260,11 +260,11 @@ void ShardsAffectedByTeamFailure::removeFailedServerForRange(KeyRangeRef keys, c
 	auto rs = shard_teams.intersectingRanges(keys);
 	for (auto it = rs.begin(); it != rs.end(); ++it) {
 		// first team vector
-		for (std::vector<Team>::iterator t = it->value().first.begin(); t != it->value().first.end(); ++t) {
+		for (auto t = it->value().first.begin(); t != it->value().first.end(); ++t) {
 			removeFailedServerForSingleRange(*t, serverID, it->range());
 		}
 		// second team vector
-		for (std::vector<Team>::iterator t = it->value().second.begin(); t != it->value().second.end(); ++t) {
+		for (auto t = it->value().second.begin(); t != it->value().second.end(); ++t) {
 			removeFailedServerForSingleRange(*t, serverID, it->range());
 		}
 	}
diff --git a/fdbserver/datadistributor/SimulatedCluster.cpp b/fdbserver/datadistributor/SimulatedCluster.cpp
new file mode 100644
index 00000000000..656abbd3091
--- /dev/null
+++ b/fdbserver/datadistributor/SimulatedCluster.cpp
@@ -0,0 +1,96 @@
+/*
+ * SimulatedCluster.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/datadistributor/SimulatedCluster.h"
+
+#include <algorithm>
+#include <map>
+#include <string>
+
+#include "fdbclient/GenericManagementAPI.h"
+
+namespace {
+
+std::string getRedundancyMode(const BasicTestConfig& testConfig) {
+	if (testConfig.minimumReplication >= 3) {
+		return "triple";
+	}
+	if (testConfig.minimumReplication == 2) {
+		return "double";
+	}
+	return "single";
+}
+
+void applyConfigurationString(DatabaseConfiguration& db, const std::string& configMode) {
+	std::map<std::string, std::string> configEntries;
+	const auto result = buildConfiguration(configMode, configEntries);
+	ASSERT(result != ConfigurationResult::NO_OPTIONS_PROVIDED);
+	for (const auto& [configKey, configValue] : configEntries) {
+		db.set(configKey, configValue);
+	}
+}
+
+} // namespace
+
+BasicSimulationConfig generateBasicSimulationConfig(const BasicTestConfig& testConfig) {
+	BasicSimulationConfig config;
+
+	config.datacenters =
+	    testConfig.singleRegion || testConfig.simpleConfig ? 1 : std::max(1, testConfig.minimumReplication);
+	config.replication_type = std::max(1, testConfig.minimumReplication);
+	config.processes_per_machine = 1;
+
+	applyConfigurationString(config.db, getRedundancyMode(testConfig));
+
+	if (testConfig.simpleConfig) {
+		config.db.desiredTLogCount = 1;
+		config.db.commitProxyCount = 1;
+		config.db.grvProxyCount = 1;
+		config.db.resolverCount = 1;
+	}
+
+	if (testConfig.desiredTLogCount.present()) {
+		config.db.desiredTLogCount = testConfig.desiredTLogCount.get();
+	}
+	if (testConfig.commitProxyCount.present()) {
+		config.db.commitProxyCount = testConfig.commitProxyCount.get();
+	}
+	if (testConfig.grvProxyCount.present()) {
+		config.db.grvProxyCount = testConfig.grvProxyCount.get();
+	}
+	if (testConfig.resolverCount.present()) {
+		config.db.resolverCount = testConfig.resolverCount.get();
+	}
+
+	config.db.storageTeamSize = std::max(1, testConfig.minimumReplication);
+	if (testConfig.logAntiQuorum != -1) {
+		config.db.tLogWriteAntiQuorum = testConfig.logAntiQuorum;
+	}
+
+	if (config.datacenters == 1) {
+		config.db.usableRegions = 1;
+	}
+
+	const int defaultMachineCount = std::max(config.db.storageTeamSize, 3) * config.datacenters;
+	config.machine_count = testConfig.machineCount.present() ? testConfig.machineCount.get() : defaultMachineCount;
+	config.machine_count = std::max(config.machine_count, config.db.storageTeamSize);
+
+	return config;
+}
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/datadistributor/TCInfo.cpp
similarity index 86%
rename from fdbserver/TCInfo.actor.cpp
rename to fdbserver/datadistributor/TCInfo.cpp
index 24c4703d6dd..5b2614f7937 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/datadistributor/TCInfo.cpp
@@ -1,5 +1,5 @@
 /*
- * TCInfo.actor.cpp
+ * TCInfo.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,51 +18,53 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ServerKnobs.h"
-#include "fdbserver/DDTeamCollection.h"
-#include "fdbserver/TCInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/datadistributor/DDTeamCollection.h"
+#include "fdbserver/datadistributor/TCInfo.h"
+#include "flow/CoroUtils.h"
 
 class TCServerInfoImpl {
 public:
-	ACTOR static Future<Void> updateServerMetrics(TCServerInfo* server) {
-		state StorageServerInterface ssi = server->lastKnownInterface;
-		state Future<ErrorOr<GetStorageMetricsReply>> metricsRequest =
+	static Future<Void> updateServerMetrics(TCServerInfo* server) {
+		StorageServerInterface ssi = server->lastKnownInterface;
+		Future<ErrorOr<GetStorageMetricsReply>> metricsRequest =
 		    ssi.getStorageMetrics.tryGetReply(GetStorageMetricsRequest(), TaskPriority::DataDistributionLaunch);
-		state Future<Void> resetRequest = Never();
-		state Future<std::pair<StorageServerInterface, ProcessClass>> interfaceChanged(server->onInterfaceChanged);
-		state Future<Void> serverRemoved(server->onRemoved);
-
-		loop {
-			choose {
-				when(ErrorOr<GetStorageMetricsReply> rep = wait(metricsRequest)) {
-					if (rep.present()) {
-						server->metrics = rep;
-						if (server->updated.canBeSet()) {
-							server->updated.send(Void());
-						}
-						break;
+		Future<Void> resetRequest = Never();
+		Future<std::pair<StorageServerInterface, ProcessClass>> interfaceChanged(server->onInterfaceChanged);
+		Future<Void> serverRemoved(server->onRemoved);
+
+		while (true) {
+			auto res = co_await race(metricsRequest, interfaceChanged, serverRemoved, resetRequest);
+			if (res.index() == 0) {
+				ErrorOr<GetStorageMetricsReply> rep = std::get<0>(std::move(res));
+				if (rep.present()) {
+					server->metrics = rep;
+					if (server->updated.canBeSet()) {
+						server->updated.send(Void());
 					}
-					metricsRequest = Never();
-					resetRequest = delay(SERVER_KNOBS->METRIC_DELAY, TaskPriority::DataDistributionLaunch);
+					break;
 				}
-				when(std::pair<StorageServerInterface, ProcessClass> _ssi = wait(interfaceChanged)) {
-					ssi = _ssi.first;
-					interfaceChanged = server->onInterfaceChanged;
-					resetRequest = Void();
-				}
-				when(wait(serverRemoved)) {
-					return Void();
-				}
-				when(wait(resetRequest)) { // To prevent a tight spin loop
-					if (IFailureMonitor::failureMonitor().getState(ssi.getStorageMetrics.getEndpoint()).isFailed()) {
-						resetRequest = IFailureMonitor::failureMonitor().onStateEqual(
-						    ssi.getStorageMetrics.getEndpoint(), FailureStatus(false));
-					} else {
-						resetRequest = Never();
-						metricsRequest = ssi.getStorageMetrics.tryGetReply(GetStorageMetricsRequest(),
-						                                                   TaskPriority::DataDistributionLaunch);
-					}
+				metricsRequest = Never();
+				resetRequest = delay(SERVER_KNOBS->METRIC_DELAY, TaskPriority::DataDistributionLaunch);
+			} else if (res.index() == 1) {
+				// interfaceChanged
+				auto updatedInterface = std::get<1>(std::move(res));
+				ssi = updatedInterface.first;
+				interfaceChanged = server->onInterfaceChanged;
+				resetRequest = Void();
+			} else if (res.index() == 2) {
+				// serverRemoved
+				co_return;
+			} else if (res.index() == 3) {
+				// resetRequest
+				// To prevent a tight spin loop
+				if (IFailureMonitor::failureMonitor().getState(ssi.getStorageMetrics.getEndpoint()).isFailed()) {
+					resetRequest = IFailureMonitor::failureMonitor().onStateEqual(ssi.getStorageMetrics.getEndpoint(),
+					                                                              FailureStatus(false));
+				} else {
+					resetRequest = Never();
+					metricsRequest = ssi.getStorageMetrics.tryGetReply(GetStorageMetricsRequest(),
+					                                                   TaskPriority::DataDistributionLaunch);
 				}
 			}
 		}
@@ -110,24 +112,21 @@ class TCServerInfoImpl {
 				}
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> updateServerMetrics(Reference<TCServerInfo> server) {
-		wait(updateServerMetrics(server.getPtr()));
-		return Void();
+	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server) {
+		co_await updateServerMetrics(server.getPtr());
 	}
 
-	ACTOR static Future<Void> serverMetricsPolling(TCServerInfo* server, Reference<IDDTxnProcessor> txnProcessor) {
-		state double lastUpdate = now();
-		loop {
-			wait(server->updateServerMetrics() &&
-			     store(server->storageStats,
-			           txnProcessor->getStorageStats(server->getId(), SERVER_KNOBS->DETAILED_METRIC_UPDATE_RATE)));
-			wait(delayUntil(lastUpdate + SERVER_KNOBS->STORAGE_METRICS_POLLING_DELAY +
-			                    SERVER_KNOBS->STORAGE_METRICS_RANDOM_DELAY * deterministicRandom()->random01(),
-			                TaskPriority::DataDistributionLaunch));
+	static Future<Void> serverMetricsPolling(TCServerInfo* server, Reference<IDDTxnProcessor> txnProcessor) {
+		double lastUpdate = now();
+		while (true) {
+			co_await (server->updateServerMetrics() &&
+			          store(server->storageStats,
+			                txnProcessor->getStorageStats(server->getId(), SERVER_KNOBS->DETAILED_METRIC_UPDATE_RATE)));
+			co_await delayUntil(lastUpdate + SERVER_KNOBS->STORAGE_METRICS_POLLING_DELAY +
+			                        SERVER_KNOBS->STORAGE_METRICS_RANDOM_DELAY * deterministicRandom()->random01(),
+			                    TaskPriority::DataDistributionLaunch);
 			lastUpdate = now();
 		}
 	}
@@ -135,13 +134,12 @@ class TCServerInfoImpl {
 
 class TCTeamInfoImpl {
 public:
-	ACTOR static Future<Void> updateStorageMetrics(TCTeamInfo* self) {
+	static Future<Void> updateStorageMetrics(TCTeamInfo* self) {
 		std::vector<Future<Void>> updates;
 		updates.reserve(self->servers.size());
 		for (int i = 0; i < self->servers.size(); i++)
 			updates.push_back(TCServerInfo::updateServerMetrics(self->servers[i]));
-		wait(waitForAll(updates));
-		return Void();
+		co_await waitForAll(updates);
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/DDShardTracker.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDShardTracker.h
similarity index 97%
rename from fdbserver/include/fdbserver/DDShardTracker.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/DDShardTracker.h
index 2a42d861dee..582cabbcca0 100644
--- a/fdbserver/include/fdbserver/DDShardTracker.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDShardTracker.h
@@ -19,7 +19,7 @@
  */
 #ifndef FOUNDATIONDB_DDSHARDTRACKER_H
 #define FOUNDATIONDB_DDSHARDTRACKER_H
-#include "fdbserver/DataDistribution.actor.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
 
 // send request/signal to DDTracker through interface
 // call synchronous method from components outside DDShardTracker
@@ -100,7 +100,7 @@ class DataDistributionTracker : public IDDShardTracker, public ReferenceCounted<
 		DataDistributionTracker& tracker;
 
 	public:
-		SafeAccessor(DataDistributionTracker* tracker)
+		explicit SafeAccessor(DataDistributionTracker* tracker)
 		  : trackerCancelled(*tracker->trackerCancelled), tracker(*tracker) {
 			ASSERT(!trackerCancelled);
 		}
diff --git a/fdbserver/include/fdbserver/DDSharedContext.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDSharedContext.h
similarity index 80%
rename from fdbserver/include/fdbserver/DDSharedContext.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/DDSharedContext.h
index e1359a5a54e..2c8ca909e96 100644
--- a/fdbserver/include/fdbserver/DDSharedContext.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDSharedContext.h
@@ -19,12 +19,13 @@
  */
 #ifndef FOUNDATIONDB_DDSHAREDCONTEXT_H
 #define FOUNDATIONDB_DDSHAREDCONTEXT_H
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/ShardsAffectedByTeamFailure.h"
-#include "fdbserver/DDShardTracker.h"
-#include "fdbserver/DDRelocationQueue.h"
-#include "fdbserver/DDTeamCollection.h"
-#include "fdbserver/DataDistributorInterface.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/datadistributor/DDShardTracker.h"
+#include "fdbserver/datadistributor/ShardsAffectedByTeamFailure.h"
+#include "fdbserver/datadistributor/DDTeamCollection.h"
+
+class DDQueue;
 
 // The common info shared by all DD components. Normally the DD components should share the reference to the same
 // context.
@@ -46,12 +47,10 @@ class DDSharedContext : public ReferenceCounted<DDSharedContext> {
 	Reference<DDQueue> ddQueue;
 	Reference<DDTeamCollection> primaryTeamCollection, remoteTeamCollection;
 
-	DDSharedContext() = default;
-
-	explicit DDSharedContext(const DataDistributorInterface& iface) : DDSharedContext(iface.id()) { interface = iface; }
-
-	explicit DDSharedContext(UID id)
-	  : ddEnabledState(new DDEnabledState), ddId(id), shardsAffectedByTeamFailure(new ShardsAffectedByTeamFailure) {}
+	DDSharedContext();
+	~DDSharedContext();
+	explicit DDSharedContext(const DataDistributorInterface& iface);
+	explicit DDSharedContext(UID id);
 
 	UID id() const { return ddId; }
 
diff --git a/fdbserver/include/fdbserver/DDTeamCollection.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDTeamCollection.h
similarity index 97%
rename from fdbserver/include/fdbserver/DDTeamCollection.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/DDTeamCollection.h
index ac2cc6a3960..3c7d343c689 100644
--- a/fdbserver/include/fdbserver/DDTeamCollection.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDTeamCollection.h
@@ -24,24 +24,22 @@
 #include <sstream>
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbrpc/Replication.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/TCInfo.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/WaitFailure.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/datadistributor/TCInfo.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/BooleanParam.h"
@@ -66,7 +64,7 @@ struct TSSPairState : ReferenceCounted<TSSPairState>, NonCopyable {
 
 	TSSPairState() : active(false) {}
 
-	TSSPairState(const LocalityData& locality)
+	explicit TSSPairState(const LocalityData& locality)
 	  : dcId(locality.dcId()), dataHallId(locality.dataHallId()), active(true) {}
 
 	bool inDataZone(const LocalityData& locality) const {
@@ -135,6 +133,10 @@ struct TSSPairState : ReferenceCounted<TSSPairState>, NonCopyable {
 	Future<Void> waitComplete() const { return complete.getFuture(); }
 };
 
+FDB_BOOLEAN_PARAM(IsFailed);
+FDB_BOOLEAN_PARAM(IsUndesired);
+FDB_BOOLEAN_PARAM(IsWiggling);
+
 class ServerStatus {
 public:
 	bool isWiggling;
@@ -145,8 +147,9 @@ class ServerStatus {
 	LocalityData locality;
 	ServerStatus()
 	  : isWiggling(false), isFailed(true), isUndesired(false), isWrongConfiguration(false), initialized(false) {}
-	ServerStatus(LocalityData const& locality) : ServerStatus(false, false, false, locality) {}
-	ServerStatus(bool isFailed, bool isUndesired, bool isWiggling, LocalityData const& locality)
+	explicit ServerStatus(LocalityData const& locality)
+	  : ServerStatus(IsFailed::False, IsUndesired::False, IsWiggling::False, locality) {}
+	ServerStatus(IsFailed isFailed, IsUndesired isUndesired, IsWiggling isWiggling, LocalityData const& locality)
 	  : isWiggling(isWiggling), isFailed(isFailed), isUndesired(isUndesired), isWrongConfiguration(false),
 	    initialized(true), locality(locality) {}
 	bool isUnhealthy() const { return isFailed || isUndesired; }
@@ -689,7 +692,7 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	AsyncTrigger printDetailedTeamsInfo;
 	Reference<LocalitySet> storageServerSet;
 
-	DDTeamCollection(DDTeamCollectionInitParams const& params);
+	explicit DDTeamCollection(DDTeamCollectionInitParams const& params);
 
 	~DDTeamCollection();
 
diff --git a/fdbserver/include/fdbserver/DDTxnProcessor.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDTxnProcessor.h
similarity index 98%
rename from fdbserver/include/fdbserver/DDTxnProcessor.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/DDTxnProcessor.h
index 6aea9a1def5..d8e114cd408 100644
--- a/fdbserver/include/fdbserver/DDTxnProcessor.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DDTxnProcessor.h
@@ -21,10 +21,10 @@
 #ifndef FOUNDATIONDB_DDTXNPROCESSOR_H
 #define FOUNDATIONDB_DDTXNPROCESSOR_H
 
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
 #include "flow/FastRef.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/MockGlobalState.h"
+#include "fdbserver/datadistributor/MockGlobalState.h"
 
 FDB_BOOLEAN_PARAM(SkipDDModeCheck);
 
@@ -49,7 +49,7 @@ class IDDTxnProcessor : public ReferenceCounted<IDDTxnProcessor> {
 
 	struct DDRangeLocations {
 		DDRangeLocations() = default;
-		DDRangeLocations(KeyRangeRef range) : range(range) {}
+		explicit DDRangeLocations(KeyRangeRef range) : range(range) {}
 
 		// A map of dcId : list of servers
 		std::map<std::string, std::vector<StorageServerInterface>> servers;
diff --git a/fdbserver/include/fdbserver/DataDistribution.actor.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistribution.h
similarity index 90%
rename from fdbserver/include/fdbserver/DataDistribution.actor.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistribution.h
index 2c0e5deda9d..4c8eb66509d 100644
--- a/fdbserver/include/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistribution.h
@@ -1,5 +1,5 @@
 /*
- * DataDistribution.actor.h
+ * DataDistribution.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,28 +18,26 @@
  * limitations under the License.
  */
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_DATA_DISTRIBUTION_ACTOR_G_H)
-#define FDBSERVER_DATA_DISTRIBUTION_ACTOR_G_H
-#include "fdbserver/DataDistribution.actor.g.h"
-#elif !defined(FDBSERVER_DATA_DISTRIBUTION_ACTOR_H)
-#define FDBSERVER_DATA_DISTRIBUTION_ACTOR_H
+#pragma once
+#ifndef FDBSERVER_DATADISTRIBUTOR_DATA_DISTRIBUTION_H
+#define FDBSERVER_DATADISTRIBUTOR_DATA_DISTRIBUTION_H
 
 #include "fdbclient/BulkLoading.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/TCInfo.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbserver/DDTxnProcessor.h"
-#include "fdbserver/ShardsAffectedByTeamFailure.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/ShardsAffectedByTeamFailure.h"
-#include "fdbclient/StorageWiggleMetrics.actor.h"
-#include "fdbclient/DataDistributionConfig.actor.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/DataMovement.h"
+#include "fdbserver/core/ShardMetrics.h"
+#include "fdbserver/core/ShardSizing.h"
+#include "fdbclient/RunRYWTransaction.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
+#include "fdbserver/datadistributor/ShardsAffectedByTeamFailure.h"
+#include "fdbserver/datadistributor/TCInfo.h"
+#include "fdbclient/StorageWiggleMetrics.h"
+#include "fdbclient/DataDistributionConfig.h"
 #include <boost/heap/policies.hpp>
 #include <boost/heap/skew_heap.hpp>
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 /////////////////////////////// Data //////////////////////////////////////
 #ifndef __INTEL_COMPILER
@@ -60,7 +58,7 @@ class RelocateReason {
 		WRITE_SPLIT,
 		__COUNT
 	};
-	RelocateReason(Value v) : value(v) { ASSERT(value != __COUNT); }
+	explicit(false) RelocateReason(Value v) : value(v) { ASSERT(value != __COUNT); }
 	explicit RelocateReason(int v) : value((Value)v) { ASSERT(value != __COUNT); }
 	std::string toString() const {
 		switch (value) {
@@ -91,9 +89,6 @@ class RelocateReason {
 	Value value;
 };
 
-extern int dataMovementPriority(DataMovementReason moveReason);
-extern DataMovementReason priorityToDataMovementReason(int priority);
-
 DataMoveType getDataMoveTypeFromDataMoveId(const UID& dataMoveId);
 
 struct DDShardInfo;
@@ -147,20 +142,20 @@ struct RelocateShard {
 	void setParentRange(KeyRange const& parent);
 	Optional<KeyRange> getParentRange() const;
 
-private:
-	// If this rs comes from a splitting, parent range is the original range.
-	Optional<KeyRange> parent_range;
-
 	RelocateShard()
 	  : priority(0), cancelled(false), dataMoveId(anonymousShardId), reason(RelocateReason::OTHER),
 	    moveReason(DataMovementReason::INVALID) {}
+
+private:
+	// If this rs comes from a splitting, parent range is the original range.
+	Optional<KeyRange> parent_range;
 };
 
 struct GetMetricsRequest {
 	KeyRange keys;
 	Promise<StorageMetrics> reply;
 	GetMetricsRequest() {}
-	GetMetricsRequest(KeyRange const& keys) : keys(keys) {}
+	explicit(false) GetMetricsRequest(KeyRange const& keys) : keys(keys) {}
 };
 
 struct GetTopKMetricsReply {
@@ -186,10 +181,10 @@ struct GetTopKMetricsRequest {
 	double maxReadLoadPerKSecond = 0, minReadLoadPerKSecond = 0; // all returned shards won't exceed this read load
 
 	GetTopKMetricsRequest() {}
-	GetTopKMetricsRequest(std::vector<KeyRange> const& keys,
-	                      int topK = 1,
-	                      double maxReadLoadPerKSecond = std::numeric_limits<double>::max(),
-	                      double minReadLoadPerKSecond = 0)
+	explicit(false) GetTopKMetricsRequest(std::vector<KeyRange> const& keys,
+	                                      int topK = 1,
+	                                      double maxReadLoadPerKSecond = std::numeric_limits<double>::max(),
+	                                      double minReadLoadPerKSecond = 0)
 	  : topK(topK), keys(keys), maxReadLoadPerKSecond(maxReadLoadPerKSecond),
 	    minReadLoadPerKSecond(minReadLoadPerKSecond) {
 		ASSERT_GE(topK, 1);
@@ -227,7 +222,7 @@ struct BulkLoadShardRequest {
 	                                         // unretryable error.
 	BulkLoadShardRequest() = default;
 
-	BulkLoadShardRequest(BulkLoadTaskState const& bulkLoadTaskState) : bulkLoadTaskState(bulkLoadTaskState) {}
+	explicit BulkLoadShardRequest(BulkLoadTaskState const& bulkLoadTaskState) : bulkLoadTaskState(bulkLoadTaskState) {}
 
 	BulkLoadShardRequest(BulkLoadTaskState const& bulkLoadTaskState, int cancelledDataMovePriority)
 	  : bulkLoadTaskState(bulkLoadTaskState), cancelledDataMovePriority(cancelledDataMovePriority) {}
@@ -247,31 +242,10 @@ FDB_BOOLEAN_PARAM(InOverSizePhysicalShard);
 FDB_BOOLEAN_PARAM(PhysicalShardAvailable);
 FDB_BOOLEAN_PARAM(MoveKeyRangeOutPhysicalShard);
 
-struct ShardMetrics {
-	StorageMetrics metrics;
-	double lastLowBandwidthStartTime;
-	int shardCount; // number of smaller shards whose metrics are aggregated in the ShardMetrics
-
-	bool operator==(ShardMetrics const& rhs) const {
-		return metrics == rhs.metrics && lastLowBandwidthStartTime == rhs.lastLowBandwidthStartTime &&
-		       shardCount == rhs.shardCount;
-	}
-
-	ShardMetrics(StorageMetrics const& metrics, double lastLowBandwidthStartTime, int shardCount)
-	  : metrics(metrics), lastLowBandwidthStartTime(lastLowBandwidthStartTime), shardCount(shardCount) {}
-};
-
-struct ShardTrackedData {
-	Future<Void> trackShard;
-	Future<Void> trackBytes;
-	Future<Void> trackUsableRegion;
-	Reference<AsyncVar<Optional<ShardMetrics>>> stats;
-};
-
 class PhysicalShardCollection : public ReferenceCounted<PhysicalShardCollection> {
 public:
 	PhysicalShardCollection() : lastTransitionStartTime(now()), requireTransition(false) {}
-	PhysicalShardCollection(Reference<IDDTxnProcessor> db)
+	explicit(false) PhysicalShardCollection(Reference<IDDTxnProcessor> db)
 	  : txnProcessor(db), lastTransitionStartTime(now()), requireTransition(false) {}
 
 	enum class PhysicalShardCreationTime { DDInit, DDRelocator };
@@ -510,27 +484,6 @@ struct InitialDataDistribution : ReferenceCounted<InitialDataDistribution> {
 	Reference<DDConfiguration::RangeConfigMapSnapshot> userRangeConfig;
 };
 
-// Holds the permitted size and IO Bounds for a shard
-struct ShardSizeBounds {
-	StorageMetrics max;
-	StorageMetrics min;
-	StorageMetrics permittedError;
-
-	bool operator==(ShardSizeBounds const& rhs) const {
-		return max == rhs.max && min == rhs.min && permittedError == rhs.permittedError;
-	}
-
-	static ShardSizeBounds shardSizeBoundsBeforeTrack();
-};
-
-// Gets the permitted size and IO bounds for a shard
-ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize);
-
-// Determines the maximum shard size based on the size of the database
-int64_t getMaxShardSize(double dbSizeEstimate);
-
-bool ddLargeTeamEnabled();
-
 struct TeamCollectionInterface {
 	PromiseStream<GetTeamRequest> getTeam;
 };
@@ -604,7 +557,9 @@ inline bool bulkDumpIsEnabled(int bulkDumpModeValue) {
 
 class BulkLoadTaskCollection : public ReferenceCounted<BulkLoadTaskCollection> {
 public:
-	BulkLoadTaskCollection(UID ddId) : ddId(ddId) { bulkLoadTaskMap.insert(allKeys, Optional<DDBulkLoadEngineTask>()); }
+	explicit(false) BulkLoadTaskCollection(UID ddId) : ddId(ddId) {
+		bulkLoadTaskMap.insert(allKeys, Optional<DDBulkLoadEngineTask>());
+	}
 
 	// Return true if there exists a bulk load job/task or the collection has not been initialized.
 	// This takes effect only when DDBulkLoad Mode is enabled.
@@ -794,7 +749,10 @@ class DDTeamCollection;
 
 struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 	static constexpr double MIN_ON_CHECK_DELAY_SEC = 5.0;
-	enum State : uint8_t { INVALID = 0, RUN = 1, PAUSE = 2 };
+	using State = StorageWigglerState::Value;
+	static constexpr State INVALID = StorageWigglerState::INVALID;
+	static constexpr State RUN = StorageWigglerState::RUN;
+	static constexpr State PAUSE = StorageWigglerState::PAUSE;
 
 	DDTeamCollection const* teamCollection;
 	StorageWiggleData wiggleData; // the wiggle related data persistent in database
@@ -808,7 +766,7 @@ struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 	    wiggle_pq;
 	std::unordered_map<UID, decltype(wiggle_pq)::handle_type> pq_handles;
 
-	State wiggleState = State::INVALID;
+	State wiggleState = INVALID;
 	double lastStateChangeTs = 0.0; // timestamp describes when did the state change
 
 	explicit StorageWiggler(DDTeamCollection* collection) : teamCollection(collection), stopWiggleSignal(true) {};
@@ -838,16 +796,7 @@ struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 			lastStateChangeTs = g_network->now();
 		}
 	}
-	static std::string getWiggleStateStr(State s) {
-		switch (s) {
-		case State::RUN:
-			return "running";
-		case State::PAUSE:
-			return "paused";
-		default:
-			return "unknown";
-		}
-	}
+	static std::string getWiggleStateStr(State s) { return StorageWigglerState::toString(s); }
 
 	// -- statistic update
 
@@ -870,5 +819,4 @@ struct StorageWiggler : ReferenceCounted<StorageWiggler> {
 #pragma endregion
 #endif
 
-#include "flow/unactorcompiler.h"
 #endif
diff --git a/fdbserver/include/fdbserver/DataDistributionTeam.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributionTeam.h
similarity index 98%
rename from fdbserver/include/fdbserver/DataDistributionTeam.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributionTeam.h
index a35f8216185..338853e6b47 100644
--- a/fdbserver/include/fdbserver/DataDistributionTeam.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributionTeam.h
@@ -116,7 +116,7 @@ class TeamSelect {
 		WANT_TRUE_BEST, // Ask for the most or least utilized team in the cluster
 	};
 	TeamSelect() : value(ANY) {}
-	TeamSelect(Value v) : value(v) {}
+	explicit(false) TeamSelect(Value v) : value(v) {}
 	std::string toString() const {
 		switch (value) {
 		case WANT_COMPLETE_SRCS:
@@ -177,7 +177,7 @@ struct GetTeamRequest {
 	    preferLowerReadUtil(preferLowerReadUtil), preferWithinShardLimit(preferWithinShardLimit),
 	    inflightPenalty(inflightPenalty), findTeamByServers(FindTeamByServers::False),
 	    findTeamForBulkLoad(FindTeamForBulkLoad::False), keys(keys), wantTrueBestIfMoveout(false) {}
-	GetTeamRequest(std::vector<UID> servers)
+	explicit GetTeamRequest(std::vector<UID> servers)
 	  : teamSelect(TeamSelect::WANT_COMPLETE_SRCS), storageQueueAware(false),
 	    preferLowerDiskUtil(PreferLowerDiskUtil::False), teamMustHaveShards(TeamMustHaveShards::False),
 	    forReadBalance(ForReadBalance::False), preferLowerReadUtil(PreferLowerReadUtil::False),
diff --git a/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributor.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributor.h
new file mode 100644
index 00000000000..e1117553db6
--- /dev/null
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/DataDistributor.h
@@ -0,0 +1,30 @@
+/*
+ * DataDistributor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "flow/flow.h"
+
+struct ServerDBInfo;
+
+Future<Void> dataDistributor(DataDistributorInterface ddi,
+                             Reference<AsyncVar<ServerDBInfo> const> db,
+                             std::string folder);
diff --git a/fdbserver/include/fdbserver/MockDataDistributor.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/MockDataDistributor.h
similarity index 85%
rename from fdbserver/include/fdbserver/MockDataDistributor.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/MockDataDistributor.h
index a489f9b8202..c7d9a09338c 100644
--- a/fdbserver/include/fdbserver/MockDataDistributor.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/MockDataDistributor.h
@@ -20,9 +20,9 @@
 #ifndef FOUNDATIONDB_MOCKDATADISTRIBUTOR_H
 #define FOUNDATIONDB_MOCKDATADISTRIBUTOR_H
 
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/MockGlobalState.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/datadistributor/MockGlobalState.h"
 
 class MockDataDistributor {
 public:
diff --git a/fdbserver/include/fdbserver/MockGlobalState.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/MockGlobalState.h
similarity index 96%
rename from fdbserver/include/fdbserver/MockGlobalState.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/MockGlobalState.h
index ca1adff03f2..53c5d6fcd36 100644
--- a/fdbserver/include/fdbserver/MockGlobalState.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/MockGlobalState.h
@@ -21,13 +21,13 @@
 #ifndef FOUNDATIONDB_MOCKGLOBALSTATE_H
 #define FOUNDATIONDB_MOCKGLOBALSTATE_H
 
-#include "fdbserver/StorageMetrics.actor.h"
+#include "fdbserver/core/StorageMetrics.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/KeyLocationService.h"
-#include "fdbserver/SimulatedCluster.h"
-#include "fdbserver/ShardsAffectedByTeamFailure.h"
+#include "fdbserver/datadistributor/ShardsAffectedByTeamFailure.h"
+#include "fdbserver/datadistributor/SimulatedCluster.h"
 
 constexpr const char* MOCK_DD_TEST_CLASS = "MockDD";
 
@@ -161,9 +161,8 @@ class MockStorageServer : public IStorageMetricsService, public ReferenceCounted
 	static constexpr bool isLoadBalancedReply = std::is_base_of_v<LoadBalancedReply, Reply>;
 
 	template <class Reply>
-	typename std::enable_if_t<isLoadBalancedReply<Reply>, void> sendErrorWithPenalty(const ReplyPromise<Reply>& promise,
-	                                                                                 const Error& err,
-	                                                                                 double penalty) {
+	    requires(isLoadBalancedReply<Reply>)
+	void sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double penalty) {
 		Reply reply;
 		reply.error = err;
 		reply.penalty = penalty;
@@ -171,8 +170,8 @@ class MockStorageServer : public IStorageMetricsService, public ReferenceCounted
 	}
 
 	template <class Reply>
-	typename std::enable_if_t<!isLoadBalancedReply<Reply>, void>
-	sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double) {
+	    requires(!isLoadBalancedReply<Reply>)
+	void sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double) {
 		promise.sendError(err);
 	}
 
diff --git a/fdbserver/include/fdbserver/ShardsAffectedByTeamFailure.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/ShardsAffectedByTeamFailure.h
similarity index 100%
rename from fdbserver/include/fdbserver/ShardsAffectedByTeamFailure.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/ShardsAffectedByTeamFailure.h
diff --git a/fdbserver/include/fdbserver/SimulatedCluster.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/SimulatedCluster.h
similarity index 100%
rename from fdbserver/include/fdbserver/SimulatedCluster.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/SimulatedCluster.h
diff --git a/fdbserver/include/fdbserver/TCInfo.h b/fdbserver/datadistributor/include/fdbserver/datadistributor/TCInfo.h
similarity index 98%
rename from fdbserver/include/fdbserver/TCInfo.h
rename to fdbserver/datadistributor/include/fdbserver/datadistributor/TCInfo.h
index 29d6639255d..63d121f28b7 100644
--- a/fdbserver/include/fdbserver/TCInfo.h
+++ b/fdbserver/datadistributor/include/fdbserver/datadistributor/TCInfo.h
@@ -22,8 +22,8 @@
 
 #include "fdbclient/SystemData.h"
 #include "fdbrpc/ReplicationTypes.h"
-#include "fdbserver/DataDistributionTeam.h"
-#include "fdbserver/DDTxnProcessor.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
+#include "fdbserver/datadistributor/DataDistributionTeam.h"
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
 
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.cpp
similarity index 88%
rename from fdbserver/fdbserver.actor.cpp
rename to fdbserver/fdbserver.cpp
index 02185dccfc0..76a7af031b9 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.cpp
@@ -1,5 +1,5 @@
 /*
- * fdbserver.actor.cpp
+ * fdbserver.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -40,7 +40,6 @@
 
 #include "fdbclient/ActorLineageProfiler.h"
 #include "fdbclient/ClusterConnectionFile.h"
-#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/versions.h"
@@ -48,35 +47,30 @@
 #include "fdbrpc/WellKnownEndpoints.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 #include "fdbclient/SimpleIni.h"
-#include "fdbrpc/AsyncFileCached.actor.h"
+#include "fdbrpc/AsyncFileCached.h"
 #include "fdbrpc/IPAllowList.h"
-#include "fdbrpc/FlowProcess.actor.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/PerfMetric.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/FlowGrpc.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/ConflictSet.h"
-#include "fdbserver/CoordinationInterface.h"
+#include "fdbserver/coordinator/CoordinationServer.h"
 #include "fdbserver/CoroFlow.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbserver/NetworkTest.h"
-#include "fdbserver/RemoteIKeyValueStore.actor.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/SimulatedCluster.h"
-#include "fdbserver/Status.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/pubsub.h"
-#include "fdbserver/OnDemandStore.h"
-#include "fdbserver/MockS3Server.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/kvstore/KVFileUtils.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/datadistributor/SimulatedCluster.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
+#include "fdbserver/tester/tester.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/worker/Worker.h"
+#include "fdbserver/mocks3/MockS3Server.h"
 #ifdef WITH_ROCKSDB
-#include "fdbserver/FDBRocksDBVersion.h"
+#include "fdbserver/core/FDBRocksDBVersion.h"
 #endif
 #include "flow/ArgParseUtil.h"
 #include "flow/DeterministicRandom.h"
@@ -84,7 +78,7 @@
 #include "flow/ProtocolVersion.h"
 #include "SimpleOpt/SimpleOpt.h"
 #include "flow/SystemMonitor.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "fdbclient/Tracing.h"
 #include "flow/WriteOnlySet.h"
 #include "flow/UnitTest.h"
@@ -92,7 +86,7 @@
 #include "flow/flow.h"
 #include "flow/network.h"
 #include "flow/SimpleCounter.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 
 #include "flow/swift.h"
 #include "flow/swift_concurrency_hooks.h"
@@ -126,8 +120,6 @@ class MasterData;
 #include "SwiftModules/Flow"
 #endif
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 // FIXME(swift): remove those
 extern "C" void swiftCallMeFuture(void* _Nonnull opaqueResultPromisePtr) noexcept;
 
@@ -141,7 +133,7 @@ enum {
 	OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_PRINT_CODE_PROBES, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
 	OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
 	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIALS, OPT_PROXY, OPT_DEPRECATED_CONFIG_PATH, OPT_DEPRECATED_USE_TEST_CONFIG_DB, OPT_DEPRECATED_NO_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME,
-	OPT_FLOW_PROCESS_NAME, OPT_FLOW_PROCESS_ENDPOINT, OPT_IP_TRUSTED_MASK, 
+	OPT_IP_TRUSTED_MASK,
 	OPT_NEW_CLUSTER_KEY, OPT_AUTHZ_PUBLIC_KEY_FILE, OPT_USE_FUTURE_PROTOCOL_VERSION, OPT_CONSISTENCY_CHECK_URGENT_MODE,
 	OPT_MOCKS3_PERSISTENCE_DIR
 };
@@ -238,8 +230,6 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_FAULT_INJECTION,       "--fault-injection",           SO_REQ_SEP },
 	{ OPT_PROFILER,	             "--profiler-",                 SO_REQ_SEP },
 	{ OPT_PRINT_SIMTIME,         "--print-sim-time",             SO_NONE },
-	{ OPT_FLOW_PROCESS_NAME,     "--process-name",              SO_REQ_SEP },
-	{ OPT_FLOW_PROCESS_ENDPOINT, "--process-endpoint",          SO_REQ_SEP },
 	{ OPT_IP_TRUSTED_MASK,       "--trusted-subnet-",           SO_REQ_SEP },
 	{ OPT_NEW_CLUSTER_KEY,       "--new-cluster-key",           SO_REQ_SEP },
 	{ OPT_AUTHZ_PUBLIC_KEY_FILE, "--authorization-public-key-file", SO_REQ_SEP },
@@ -324,7 +314,7 @@ class WorldReadablePermissions {
 	boost::interprocess::permissions permission;
 
 private:
-	WorldReadablePermissions(const WorldReadablePermissions& rhs) {}
+	explicit(false) WorldReadablePermissions(const WorldReadablePermissions& rhs) {}
 #ifdef _WIN32
 	SECURITY_ATTRIBUTES sa;
 #endif
@@ -345,7 +335,7 @@ UID getSharedMemoryMachineId() {
 	// On windows, this means that we have to create an elaborate workaround for DACLs
 	WorldReadablePermissions p;
 	std::string sharedMemoryIdentifier = "fdbserver_shared_memory_id";
-	loop {
+	while (true) {
 		try {
 			// "0" is the default netPrefix "addr"
 			boost::interprocess::managed_shared_memory segment(
@@ -377,8 +367,10 @@ UID getSharedMemoryMachineId() {
 #endif
 }
 
-ACTOR void failAfter(Future<Void> trigger, ISimulator::ProcessInfo* m = g_simulator->getCurrentProcess()) {
-	wait(trigger);
+Future<Void> failAfter(Uncancellable,
+                       Future<Void> trigger,
+                       ISimulator::ProcessInfo* m = g_simulator->getCurrentProcess()) {
+	co_await trigger;
 	if (enableFailures) {
 		printf("Killing machine: %s at %f\n", m->address.toString().c_str(), now());
 		g_simulator->killProcess(m, ISimulator::KillType::KillInstantly);
@@ -387,30 +379,30 @@ ACTOR void failAfter(Future<Void> trigger, ISimulator::ProcessInfo* m = g_simula
 
 void failAfter(Future<Void> trigger, Endpoint e) {
 	if (g_network == g_simulator)
-		failAfter(trigger, g_simulator->getProcess(e));
+		failAfter(Uncancellable(), trigger, g_simulator->getProcess(e));
 }
 
 #ifdef WITH_SWIFT
-ACTOR void swiftTestRunner() {
+Future<Void> swiftTestRunner(Uncancellable) {
 	auto p = PromiseVoid();
 	fdbserver_swift::swiftyTestRunner(p);
-	wait(p.getFuture());
+	co_await p.getFuture();
 
 	flushAndExit(0);
 }
 #endif
 
-ACTOR Future<Void> histogramReport() {
-	loop {
-		wait(delay(SERVER_KNOBS->HISTOGRAM_REPORT_INTERVAL));
+Future<Void> histogramReport() {
+	while (true) {
+		co_await delay(SERVER_KNOBS->HISTOGRAM_REPORT_INTERVAL);
 
 		GetHistogramRegistry().logReport();
 	}
 }
 
-ACTOR Future<Void> metricsReport() {
-	loop {
-		wait(delay(SERVER_KNOBS->GENERIC_METRICS_REPORT_INTERVAL));
+Future<Void> metricsReport() {
+	while (true) {
+		co_await delay(SERVER_KNOBS->GENERIC_METRICS_REPORT_INTERVAL);
 
 		simpleCounterReport();
 	}
@@ -478,67 +470,6 @@ void testSerializationSpeed() {
 	printf("\n");
 }
 
-std::string toHTML(const StringRef& binaryString) {
-	std::string s;
-
-	for (int i = 0; i < binaryString.size(); i++) {
-		uint8_t c = binaryString[i];
-		if (c == '<')
-			s += "&lt;";
-		else if (c == '>')
-			s += "&gt;";
-		else if (c == '&')
-			s += "&amp;";
-		else if (c == '"')
-			s += "&quot;";
-		else if (c == ' ')
-			s += "&nbsp;";
-		else if (c > 32 && c < 127)
-			s += c;
-		else
-			s += format("<span class=\"binary\">[%02x]</span>", c);
-	}
-
-	return s;
-}
-
-ACTOR Future<Void> dumpDatabase(Database cx, std::string outputFilename, KeyRange range = allKeys) {
-	try {
-		state Transaction tr(cx);
-		loop {
-			state FILE* output = fopen(outputFilename.c_str(), "wt");
-			try {
-				state KeySelectorRef iter = firstGreaterOrEqual(range.begin);
-				state Arena arena;
-				fprintf(output, "<html><head><style type=\"text/css\">.binary {color:red}</style></head><body>\n");
-				Version ver = wait(tr.getReadVersion());
-				fprintf(output, "<h3>Database version: %" PRId64 "</h3>", ver);
-
-				loop {
-					RangeResult results = wait(tr.getRange(iter, firstGreaterOrEqual(range.end), 1000));
-					for (int r = 0; r < results.size(); r++) {
-						std::string key = toHTML(results[r].key), value = toHTML(results[r].value);
-						fprintf(output, "<p>%s <b>:=</b> %s</p>\n", key.c_str(), value.c_str());
-					}
-					if (results.size() < 1000)
-						break;
-					iter = firstGreaterThan(KeyRef(arena, results[results.size() - 1].key));
-				}
-				fprintf(output, "</body></html>");
-				fclose(output);
-				TraceEvent("DatabaseDumped").detail("Filename", outputFilename);
-				return Void();
-			} catch (Error& e) {
-				fclose(output);
-				wait(tr.onError(e));
-			}
-		}
-	} catch (Error& e) {
-		TraceEvent(SevError, "DumpDatabaseError").error(e).detail("Filename", outputFilename);
-		throw;
-	}
-}
-
 void memoryTest();
 void skipListTest();
 
@@ -1109,7 +1040,6 @@ enum class ServerRole {
 	CreateTemplateDatabase,
 	DSLTest,
 	FDBD,
-	FlowProcess,
 	KVFileGenerateIOLogChecksums,
 	KVFileIntegrityCheck,
 	KVFileDump,
@@ -1138,7 +1068,7 @@ struct CLIOptions {
 	bool maxLogsSet = false;
 
 	ServerRole role = ServerRole::FDBD;
-	uint32_t randomSeed = platform::getRandomSeed();
+	uint64_t randomSeed = platform::getRandomSeed();
 	double reseedTime = -1.0; // Time in seconds when to reset random seed in simulation (-1 = disabled)
 
 	const char* testFile = "tests/default.txt";
@@ -1182,8 +1112,6 @@ struct CLIOptions {
 	UnitTestParameters testParams;
 
 	std::map<std::string, std::string> profilerConfig;
-	std::string flowProcessName;
-	Endpoint flowProcessEndpoint;
 	bool printSimTime = false;
 	IPAllowList allowList;
 
@@ -1406,8 +1334,6 @@ struct CLIOptions {
 					role = ServerRole::NetworkTestClient;
 				else if (!strcmp(sRole, "networktestserver"))
 					role = ServerRole::NetworkTestServer;
-				else if (!strcmp(sRole, "restore"))
-					role = ServerRole::Restore;
 				else if (!strcmp(sRole, "kvfileintegritycheck"))
 					role = ServerRole::KVFileIntegrityCheck;
 				else if (!strcmp(sRole, "kvfilegeneratesums"))
@@ -1420,8 +1346,6 @@ struct CLIOptions {
 					role = ServerRole::ConsistencyCheckUrgent;
 				else if (!strcmp(sRole, "unittests"))
 					role = ServerRole::UnitTests;
-				else if (!strcmp(sRole, "flowprocess"))
-					role = ServerRole::FlowProcess;
 				else if (!strcmp(sRole, "changeclusterkey"))
 					role = ServerRole::ChangeClusterKey;
 				else if (!strcmp(sRole, "mocks3server"))
@@ -1615,7 +1539,7 @@ struct CLIOptions {
 				break;
 			case OPT_RANDOMSEED: {
 				char* end;
-				randomSeed = (uint32_t)strtoul(args.OptionArg(), &end, 0);
+				randomSeed = strtoull(args.OptionArg(), &end, 0);
 				if (*end) {
 					fprintf(stderr, "ERROR: Could not parse random seed `%s'\n", args.OptionArg());
 					printHelpTeaser(argv[0]);
@@ -1774,42 +1698,6 @@ struct CLIOptions {
 			case OPT_DEPRECATED_NO_CONFIG_DB:
 				warnDeprecatedOption(args.OptionText());
 				break;
-			case OPT_FLOW_PROCESS_NAME:
-				flowProcessName = args.OptionArg();
-				std::cout << flowProcessName << std::endl;
-				break;
-			case OPT_FLOW_PROCESS_ENDPOINT: {
-				std::vector<std::string> strings;
-				std::cout << args.OptionArg() << std::endl;
-				boost::split(strings, args.OptionArg(), [](char c) { return c == ','; });
-				for (auto& str : strings) {
-					std::cout << str << " ";
-				}
-				std::cout << "\n";
-				if (strings.size() != 3) {
-					std::cerr << "Invalid argument, expected 3 elements in --process-endpoint got " << strings.size()
-					          << std::endl;
-					flushAndExit(FDB_EXIT_ERROR);
-				}
-				try {
-					auto addr = NetworkAddress::parse(strings[0]);
-					uint64_t fst = std::stoul(strings[1]);
-					uint64_t snd = std::stoul(strings[2]);
-					UID token(fst, snd);
-					NetworkAddressList l;
-					l.address = addr;
-					flowProcessEndpoint = Endpoint(l, token);
-					std::cout << "flowProcessEndpoint: " << flowProcessEndpoint.getPrimaryAddress().toString()
-					          << ", token: " << flowProcessEndpoint.token.toString() << "\n";
-				} catch (Error& e) {
-					std::cerr << "Could not parse network address " << strings[0] << std::endl;
-					flushAndExit(FDB_EXIT_ERROR);
-				} catch (std::exception& e) {
-					std::cerr << "Could not parse token " << strings[1] << "," << strings[2] << std::endl;
-					flushAndExit(FDB_EXIT_ERROR);
-				}
-				break;
-			}
 			case OPT_PRINT_SIMTIME:
 				printSimTime = true;
 				break;
@@ -2087,7 +1975,7 @@ int main(int argc, char* argv[]) {
 		const auto role = opts.role;
 
 		if (role == ServerRole::Simulation) {
-			printf("Random seed is %u...\n", opts.randomSeed);
+			printf("Random seed is %llu...\n", opts.randomSeed);
 			bindDeterministicRandomToOpenssl();
 		}
 
@@ -2101,31 +1989,18 @@ int main(int argc, char* argv[]) {
 		}
 		enableFaultInjection(opts.faultInjectionEnabled);
 
-		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::SERVER,
-		                                         Randomize::True,
-		                                         role == ServerRole::Simulation ? IsSimulated::True
-		                                                                        : IsSimulated::False);
-		auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
-		g_knobs.setKnob("log_directory", KnobValueRef::create(opts.logFolder));
-		g_knobs.setKnob("conn_file", KnobValueRef::create(opts.connFile));
+		resetServerKnobs(Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
+		setServerKnob("log_directory", KnobValueRef::create(opts.logFolder));
+		setServerKnob("conn_file", KnobValueRef::create(opts.connFile));
 		if (role != ServerRole::Simulation && opts.memLimit > 0) {
-			g_knobs.setKnob("commit_batches_mem_bytes_hard_limit",
-			                KnobValueRef::create(static_cast<int64_t>(opts.memLimit)));
+			setServerKnob("commit_batches_mem_bytes_hard_limit",
+			              KnobValueRef::create(static_cast<int64_t>(opts.memLimit)));
 		}
 
-		IKnobCollection::setupKnobs(opts.knobs);
-		g_knobs.setKnob("server_mem_limit", KnobValueRef::create(static_cast<int64_t>(opts.memLimit)));
+		setupServerKnobs(opts.knobs);
+		setServerKnob("server_mem_limit", KnobValueRef::create(static_cast<int64_t>(opts.memLimit)));
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		g_knobs.initialize(Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
-
-		if (!SERVER_KNOBS->ALLOW_DANGEROUS_KNOBS) {
-			if (SERVER_KNOBS->REMOTE_KV_STORE) {
-				fprintf(stderr,
-				        "ERROR : explicitly setting REMOTE_KV_STORE is dangerous! set ALLOW_DANGEROUS_KNOBS to "
-				        "proceed anyways\n");
-				flushAndExit(FDB_EXIT_ERROR);
-			}
-		}
+		initializeServerKnobs(Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
 
 		// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
 		EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
@@ -2167,6 +2042,7 @@ int main(int argc, char* argv[]) {
 			// startOldSimulator();
 			opts.buildNetwork(argv[0]);
 			startNewSimulator(opts.printSimTime);
+			installFDBSimulationPolicy();
 
 			if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
 				// TODO (Swift): Make it TraceEvent
@@ -2195,7 +2071,7 @@ int main(int argc, char* argv[]) {
 #if WITH_SWIFT
 			// Set FDBSWIFTTEST env variable to execute some simple Swift/Flow interop tests.
 			if (SERVER_KNOBS->FLOW_WITH_SWIFT && getenv("FDBSWIFTTEST")) {
-				swiftTestRunner(); // spawns actor that will call Swift functions
+				swiftTestRunner(Uncancellable()); // spawns coroutine that will call Swift functions
 			}
 #endif
 
@@ -2204,8 +2080,7 @@ int main(int argc, char* argv[]) {
 			opts.buildNetwork(argv[0]);
 
 			const bool expectsPublicAddress =
-			    (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::Restore ||
-			     role == ServerRole::FlowProcess || role == ServerRole::MockS3Server);
+			    (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::MockS3Server);
 			if (opts.publicAddressStrs.empty()) {
 				if (expectsPublicAddress) {
 					fprintf(stderr, "ERROR: The -p or --public-address option is required\n");
@@ -2457,13 +2332,13 @@ int main(int argc, char* argv[]) {
 						}
 					}
 				}
-				g_knobs.setKnob("encrypt_header_auth_token_enabled",
-				                KnobValueRef::create(ini.GetBoolValue("META", "encryptHeaderAuthTokenEnabled", false)));
-				g_knobs.setKnob("encrypt_header_auth_token_algo",
-				                KnobValueRef::create((int)ini.GetLongValue(
-				                    "META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO)));
+				setServerKnob("encrypt_header_auth_token_enabled",
+				              KnobValueRef::create(ini.GetBoolValue("META", "encryptHeaderAuthTokenEnabled", false)));
+				setServerKnob("encrypt_header_auth_token_algo",
+				              KnobValueRef::create((int)ini.GetLongValue(
+				                  "META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO)));
 
-				g_knobs.setKnob(
+				setServerKnob(
 				    "shard_encode_location_metadata",
 				    KnobValueRef::create(ini.GetBoolValue("META", "enableShardEncodeLocationMetadata", false)));
 			}
@@ -2477,8 +2352,7 @@ int main(int argc, char* argv[]) {
 		} else if (role == ServerRole::FDBD) {
 			// Update the global blob credential files list so that both fast
 			// restore workers and backup workers can access blob storage.
-			std::vector<std::string>* pFiles =
-			    (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
+			auto* pFiles = static_cast<std::vector<std::string>*>(g_network->global(INetwork::enBlobCredentialFiles));
 			if (pFiles != nullptr) {
 				for (auto& f : opts.blobCredentials) {
 					pFiles->push_back(f);
@@ -2486,57 +2360,43 @@ int main(int argc, char* argv[]) {
 			}
 
 			// Update proxy string
-			Optional<std::string>* pProxy = (Optional<std::string>*)g_network->global(INetwork::enProxy);
+			auto* pProxy = static_cast<Optional<std::string>*>(g_network->global(INetwork::enProxy));
 			*pProxy = opts.proxy;
 
-			// Call fast restore for the class FastRestoreClass. This is a short-cut to run fast restore in circus
-			if (opts.processClass == ProcessClass::FastRestoreClass) {
-				printf("Run as fast restore worker\n");
-				ASSERT(opts.connectionFile);
-				auto dataFolder = opts.dataFolder;
-				if (!dataFolder.size())
-					dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
+			ASSERT(opts.connectionFile);
 
-				std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
-				actors.push_back(restoreWorker(opts.connectionFile, opts.localities, dataFolder));
-				f = stopAfter(waitForAll(actors));
-				printf("Fast restore worker started\n");
-				g_network->run();
-				printf("g_network->run() done\n");
-			} else { // Call fdbd roles in conventional way
-				ASSERT(opts.connectionFile);
-
-				setupRunLoopProfiler();
-
-				auto dataFolder = opts.dataFolder;
-				if (!dataFolder.size())
-					dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
+			setupRunLoopProfiler();
 
-				std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
+			auto dataFolder = opts.dataFolder;
+			if (!dataFolder.size())
+				dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
+
+			std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
+
+			actors.push_back(fdbd(opts.connectionFile,
+			                      opts.localities,
+			                      opts.processClass,
+			                      dataFolder,
+			                      dataFolder,
+			                      opts.storageMemLimit,
+			                      opts.metricsConnFile,
+			                      opts.metricsPrefix,
+			                      opts.rsssize,
+			                      opts.whitelistBinPaths,
+			                      opts.consistencyCheckUrgentMode));
+			actors.push_back(histogramReport());
+			actors.push_back(metricsReport());
 
 #ifdef FLOW_GRPC_ENABLED
-				if (opts.grpcAddressStrs.size() > 0) {
-					FlowGrpc::init(&opts.tlsConfig, NetworkAddress::parse(opts.grpcAddressStrs[0]));
-					actors.push_back(GrpcServer::instance()->run());
-				}
-#endif
-				actors.push_back(fdbd(opts.connectionFile,
-				                      opts.localities,
-				                      opts.processClass,
-				                      dataFolder,
-				                      dataFolder,
-				                      opts.storageMemLimit,
-				                      opts.metricsConnFile,
-				                      opts.metricsPrefix,
-				                      opts.rsssize,
-				                      opts.whitelistBinPaths,
-				                      opts.consistencyCheckUrgentMode));
-				actors.push_back(histogramReport());
-				actors.push_back(metricsReport());
-
-				f = stopAfter(waitForAll(actors));
-				g_network->run();
+			if (opts.grpcAddressStrs.size() > 0) {
+				FlowGrpc::init(&opts.tlsConfig, NetworkAddress::parse(opts.grpcAddressStrs[0]));
+				actors.push_back(GrpcServer::instance()->run());
 			}
+#endif
+
+			f = stopAfter(waitForAll(actors));
+			g_network->run();
+
 		} else if (role == ServerRole::MultiTester) {
 			setupRunLoopProfiler();
 			f = stopAfter(runTests(opts.connectionFile,
@@ -2602,9 +2462,6 @@ int main(int argc, char* argv[]) {
 		} else if (role == ServerRole::NetworkTestServer) {
 			f = stopAfter(networkTestServer());
 			g_network->run();
-		} else if (role == ServerRole::Restore) {
-			f = stopAfter(restoreWorker(opts.connectionFile, opts.localities, opts.dataFolder));
-			g_network->run();
 		} else if (role == ServerRole::KVFileIntegrityCheck) {
 			f = stopAfter(KVFileCheck(opts.kvFile, true));
 			g_network->run();
@@ -2618,36 +2475,6 @@ int main(int argc, char* argv[]) {
 			}
 
 			f = result;
-		} else if (role == ServerRole::FlowProcess) {
-			std::string traceFormat = getTraceFormatExtension();
-			// close and reopen trace file with the correct process listen address to name the file
-			closeTraceFile();
-			// writer is not shutdown immediately, addref on it
-			disposeTraceFileWriter();
-			// use the same trace format as before
-			selectTraceFormatter(traceFormat);
-			// create the trace file with the correct process address
-			openTraceFile(
-			    g_network->getLocalAddress(), opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
-			auto m =
-			    startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId, opts.localities.dataHallId());
-			TraceEvent(SevDebug, "StartingFlowProcess").detail("FlowProcessName", opts.flowProcessName);
-#if defined(__linux__)
-			prctl(PR_SET_PDEATHSIG, SIGTERM);
-			if (getppid() == 1) /* parent already died before prctl */
-				flushAndExit(FDB_EXIT_SUCCESS);
-#elif defined(__FreeBSD__)
-			const int sig = SIGTERM;
-			procctl(P_PID, 0, PROC_PDEATHSIG_CTL, (void*)&sig);
-			if (getppid() == 1) /* parent already died before procctl */
-				flushAndExit(FDB_EXIT_SUCCESS);
-#endif
-
-			if (opts.flowProcessName == "KeyValueStoreProcess") {
-				ProcessFactory<KeyValueStoreProcess>(opts.flowProcessName.c_str());
-			}
-			f = stopAfter(runFlowProcess(opts.flowProcessName, opts.flowProcessEndpoint));
-			g_network->run();
 		} else if (role == ServerRole::KVFileDump) {
 			f = stopAfter(KVFileDump(opts.kvFile));
 			g_network->run();
@@ -2676,41 +2503,6 @@ int main(int argc, char* argv[]) {
 		if (role == ServerRole::Simulation) {
 			printf("Unseed: %d\n", unseed);
 			printf("Elapsed: %f simsec, %f real seconds\n", now() - startNow, timer() - start);
-		}
-
-		// IFailureMonitor::failureMonitor().address_info.clear();
-
-		// we should have shut down ALL actors associated with this machine; let's list all of the ones still live
-		/*{
-		    auto living = Actor::all;
-		    printf("%d surviving actors:\n", living.size());
-		    for(auto a = living.begin(); a != living.end(); ++a)
-		        printf("  #%lld %s %p\n", (*a)->creationIndex, (*a)->getName(), (*a));
-		}
-
-		{
-		    auto living = DatabaseContext::all;
-		    printf("%d surviving DatabaseContexts:\n", living.size());
-		    for(auto a = living.begin(); a != living.end(); ++a)
-		        printf("  #%lld %p\n", (*a)->creationIndex, (*a));
-		}
-
-		{
-		    auto living = TransactionData::all;
-		    printf("%d surviving TransactionData(s):\n", living.size());
-		    for(auto a = living.begin(); a != living.end(); ++a)
-		        printf("  #%lld %p\n", (*a)->creationIndex, (*a));
-		}*/
-
-		/*cout << Actor::allActors.size() << " surviving actors:" << std::endl;
-		std::map<std::string,int> actorCount;
-		for(int i=0; i<Actor::allActors.size(); i++)
-		    ++actorCount[Actor::allActors[i]->getName()];
-		for(auto i = actorCount.rbegin(); !(i == actorCount.rend()); ++i)
-		    std::cout << "  " << i->second << " " << i->first << std::endl;*/
-		//	std::cout << "  " << Actor::allActors[i]->getName() << std::endl;
-
-		if (role == ServerRole::Simulation) {
 			unsigned long sevErrorEventsLogged = TraceEvent::CountEventsLoggedAt(SevError);
 			if (sevErrorEventsLogged > 0) {
 				printf("%lu SevError events logged\n", sevErrorEventsLogged);
@@ -2718,8 +2510,6 @@ int main(int argc, char* argv[]) {
 			}
 		}
 
-		// g_simulator->run();
-
 #ifdef ALLOC_INSTRUMENTATION
 		{
 			std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " "
@@ -2770,23 +2560,19 @@ int main(int argc, char* argv[]) {
 			memSample_entered = true;
 		}
 #endif
-		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
 		flushAndExit(rc);
 	} catch (Error& e) {
 		fprintf(stderr, "Error: %s\n", e.what());
 		TraceEvent(SevError, "MainError").error(e);
-		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
 		flushAndExit(FDB_EXIT_MAIN_ERROR);
 	} catch (boost::system::system_error& e) {
 		ASSERT_WE_THINK(false); // boost errors shouldn't leak
 		fprintf(stderr, "boost::system::system_error: %s (%d)", e.what(), e.code().value());
 		TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
-		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
 		flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
 	} catch (std::exception& e) {
 		fprintf(stderr, "std::exception: %s\n", e.what());
 		TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
-		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
 		flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
 	}
 
diff --git a/fdbserver/grvproxy/CMakeLists.txt b/fdbserver/grvproxy/CMakeLists.txt
new file mode 100644
index 00000000000..55b73b9b7de
--- /dev/null
+++ b/fdbserver/grvproxy/CMakeLists.txt
@@ -0,0 +1,15 @@
+fdb_find_sources(FDBSERVER_GRVPROXY_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_grvproxy SRCS ${FDBSERVER_GRVPROXY_SRCS})
+add_fdbserver_link_test(fdbserver_grvproxylinktest
+  fdbserver_grvproxy
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_grvproxy)
+target_include_directories(fdbserver_grvproxy
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_grvproxy PRIVATE fdbserver_core fdbserver_logsystem)
diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/grvproxy/GrvProxyServer.cpp
similarity index 69%
rename from fdbserver/GrvProxyServer.actor.cpp
rename to fdbserver/grvproxy/GrvProxyServer.cpp
index e680b874a24..69fdb94a0a6 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/grvproxy/GrvProxyServer.cpp
@@ -1,5 +1,5 @@
 /*
- * GrvProxyServer.actor.cpp
+ * GrvProxyServer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,6 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ClientKnobs.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/Notified.h"
@@ -27,18 +26,20 @@
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/GrvProxyInterface.h"
 #include "fdbclient/VersionVector.h"
-#include "fdbserver/GrvProxyTagThrottler.h"
-#include "fdbserver/GrvTransactionRateInfo.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/grvproxy/GrvProxyServer.h"
+#include "GrvQueueDelay.h"
+#include "GrvTransactionRateInfo.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "fdbrpc/sim_validation.h"
 #include "flow/Buggify.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 #include "flow/genericactors.actor.h"
 
 struct GrvProxyStats {
@@ -55,15 +56,12 @@ struct GrvProxyStats {
 	Counter txnBatchPriorityStartOut;
 	Counter txnDefaultPriorityStartIn;
 	Counter txnDefaultPriorityStartOut;
-	Counter txnTagThrottlerIn;
-	Counter txnTagThrottlerOut;
 	Counter txnThrottled;
 	Counter updatesFromRatekeeper;
 	Counter leaseTimeouts;
 	int systemGRVQueueSize;
 	int defaultGRVQueueSize;
 	int batchGRVQueueSize;
-	int tagThrottlerGRVQueueSize;
 	double transactionRateAllowed;
 	double batchTransactionRateAllowed;
 	double transactionLimit;
@@ -80,7 +78,6 @@ struct GrvProxyStats {
 	LatencySample defaultTxnGRVTimeInQueue;
 	LatencySample batchTxnGRVTimeInQueue;
 
-	// These latency bands and samples ignore latency injected by the GrvProxyTagThrottler
 	LatencyBands grvLatencyBands;
 	LatencySample grvLatencySample; // GRV latency metric sample of default priority
 	LatencySample grvBatchLatencySample; // GRV latency metric sample of batched priority
@@ -115,23 +112,6 @@ struct GrvProxyStats {
 		       (FLOW_KNOBS->BASIC_LOAD_BALANCE_UPDATE_RATE - (lastBucketBegin + bucketInterval - now()));
 	}
 
-	void update(GrvProxyTagThrottler::ReleaseTransactionsResult const& releaseStats) {
-		auto const totalReleasedRequests =
-		    releaseStats.batchPriorityRequestsReleased + releaseStats.defaultPriorityRequestsReleased;
-		auto const totalReleasedTransactions =
-		    releaseStats.batchPriorityTransactionsReleased + releaseStats.defaultPriorityTransactionsReleased;
-
-		txnRequestIn += totalReleasedRequests;
-		txnStartIn += totalReleasedTransactions;
-		txnBatchPriorityStartIn += releaseStats.batchPriorityTransactionsReleased;
-		txnDefaultPriorityStartIn += releaseStats.defaultPriorityTransactionsReleased;
-		batchGRVQueueSize += releaseStats.batchPriorityRequestsReleased;
-		defaultGRVQueueSize += releaseStats.defaultPriorityRequestsReleased;
-		txnRequestErrors += releaseStats.rejectedRequests;
-		txnTagThrottlerOut += totalReleasedTransactions;
-		tagThrottlerGRVQueueSize -= totalReleasedRequests;
-	}
-
 	// Current stats maintained for a given grv proxy server
 	explicit GrvProxyStats(UID id)
 	  : cc("GrvProxyStats", id.toString()),
@@ -143,13 +123,12 @@ struct GrvProxyStats {
 	    txnBatchPriorityStartIn("TxnBatchPriorityStartIn", cc),
 	    txnBatchPriorityStartOut("TxnBatchPriorityStartOut", cc),
 	    txnDefaultPriorityStartIn("TxnDefaultPriorityStartIn", cc),
-	    txnDefaultPriorityStartOut("TxnDefaultPriorityStartOut", cc), txnTagThrottlerIn("TxnTagThrottlerIn", cc),
-	    txnTagThrottlerOut("TxnTagThrottlerOut", cc), txnThrottled("TxnThrottled", cc),
+	    txnDefaultPriorityStartOut("TxnDefaultPriorityStartOut", cc), txnThrottled("TxnThrottled", cc),
 	    updatesFromRatekeeper("UpdatesFromRatekeeper", cc), leaseTimeouts("LeaseTimeouts", cc), systemGRVQueueSize(0),
-	    defaultGRVQueueSize(0), batchGRVQueueSize(0), tagThrottlerGRVQueueSize(0), transactionRateAllowed(0),
-	    batchTransactionRateAllowed(0), transactionLimit(0), batchTransactionLimit(0),
-	    percentageOfDefaultGRVQueueProcessed(0), percentageOfBatchGRVQueueProcessed(0), lastBatchQueueThrottled(false),
-	    lastDefaultQueueThrottled(false), batchThrottleStartTime(0.0), defaultThrottleStartTime(0.0),
+	    defaultGRVQueueSize(0), batchGRVQueueSize(0), transactionRateAllowed(0), batchTransactionRateAllowed(0),
+	    transactionLimit(0), batchTransactionLimit(0), percentageOfDefaultGRVQueueProcessed(0),
+	    percentageOfBatchGRVQueueProcessed(0), lastBatchQueueThrottled(false), lastDefaultQueueThrottled(false),
+	    batchThrottleStartTime(0.0), defaultThrottleStartTime(0.0),
 	    defaultTxnGRVTimeInQueue("DefaultTxnGRVTimeInQueue",
 	                             id,
 	                             SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
@@ -177,7 +156,6 @@ struct GrvProxyStats {
 		specialCounter(cc, "SystemGRVQueueSize", [this]() { return this->systemGRVQueueSize; });
 		specialCounter(cc, "DefaultGRVQueueSize", [this]() { return this->defaultGRVQueueSize; });
 		specialCounter(cc, "BatchGRVQueueSize", [this]() { return this->batchGRVQueueSize; });
-		specialCounter(cc, "TagThrottlerGRVQueueSize", [this]() { return this->tagThrottlerGRVQueueSize; });
 		specialCounter(
 		    cc, "SystemAndDefaultTxnRateAllowed", [this]() { return int64_t(this->transactionRateAllowed); });
 		specialCounter(
@@ -205,7 +183,7 @@ struct GrvProxyData {
 	GrvProxyStats stats;
 	MasterInterface master;
 	PublicRequestStream<GetReadVersionRequest> getConsistentReadVersion;
-	Reference<ILogSystem> logSystem;
+	Reference<LogSystem> logSystem;
 
 	Database cx;
 	Reference<AsyncVar<ServerDBInfo> const> db;
@@ -220,8 +198,6 @@ struct GrvProxyData {
 	Version version;
 	Version minKnownCommittedVersion; // we should ask master for this version.
 
-	GrvProxyTagThrottler tagThrottler;
-
 	// Cache of the latest commit versions of storage servers.
 	VersionVector ssVersionVectorCache;
 
@@ -234,7 +210,6 @@ struct GrvProxyData {
 			if (newLatencyBandConfig.present()) {
 				for (auto band : newLatencyBandConfig.get().grvConfig.bands) {
 					stats.grvLatencyBands.addThreshold(band);
-					tagThrottler.addLatencyBandThreshold(band);
 				}
 			}
 		}
@@ -249,8 +224,7 @@ struct GrvProxyData {
 	  : dbgid(dbgid), stats(dbgid), master(master), getConsistentReadVersion(getConsistentReadVersion),
 	    cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::True)), db(db), lastStartCommit(0),
 	    lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), updateCommitRequests(0), lastCommitTime(0),
-	    version(0), minKnownCommittedVersion(invalidVersion),
-	    tagThrottler(CLIENT_KNOBS->PROXY_MAX_TAG_THROTTLE_DURATION) {
+	    version(0), minKnownCommittedVersion(invalidVersion) {
 		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
 			versionVectorSizeOnGRVReply =
 			    std::make_unique<LatencySample>("VersionVectorSizeOnGRVReply",
@@ -261,17 +235,15 @@ struct GrvProxyData {
 	}
 };
 
-ACTOR Future<Void> healthMetricsRequestServer(GrvProxyInterface grvProxy,
-                                              GetHealthMetricsReply* healthMetricsReply,
-                                              GetHealthMetricsReply* detailedHealthMetricsReply) {
-	loop {
-		choose {
-			when(GetHealthMetricsRequest req = waitNext(grvProxy.getHealthMetrics.getFuture())) {
-				if (req.detailed)
-					req.reply.send(*detailedHealthMetricsReply);
-				else
-					req.reply.send(*healthMetricsReply);
-			}
+Future<Void> healthMetricsRequestServer(GrvProxyInterface grvProxy,
+                                        GetHealthMetricsReply* healthMetricsReply,
+                                        GetHealthMetricsReply* detailedHealthMetricsReply) {
+	while (true) {
+		GetHealthMetricsRequest req = co_await grvProxy.getHealthMetrics.getFuture();
+		if (req.detailed) {
+			req.reply.send(*detailedHealthMetricsReply);
+		} else {
+			req.reply.send(*healthMetricsReply);
 		}
 	}
 }
@@ -279,24 +251,23 @@ ACTOR Future<Void> healthMetricsRequestServer(GrvProxyInterface grvProxy,
 // Older FDB versions used different keys for client profiling data. This
 // function performs a one-time migration of data in these keys to the new
 // global configuration key space.
-ACTOR Future<Void> globalConfigMigrate(GrvProxyData* grvProxyData) {
-	state Key migratedKey("\xff\x02/fdbClientInfo/migrated/"_sr);
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(grvProxyData->cx);
+Future<Void> globalConfigMigrate(GrvProxyData* grvProxyData) {
+	Key migratedKey("\xff\x02/fdbClientInfo/migrated/"_sr);
+	auto tr = makeReference<ReadYourWritesTransaction>(grvProxyData->cx);
 	try {
-		loop {
+		while (true) {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
+			Error err;
 			try {
-				state Optional<Value> migrated = wait(tr->get(migratedKey));
+				Optional<Value> migrated = co_await tr->get(migratedKey);
 				if (migrated.present()) {
 					// Already performed migration.
 					break;
 				}
 
-				state Optional<Value> sampleRate =
-				    wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_sample_rate/"_sr)));
-				state Optional<Value> sizeLimit =
-				    wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_size_limit/"_sr)));
+				Optional<Value> sampleRate = co_await tr->get(Key("\xff\x02/fdbClientInfo/client_txn_sample_rate/"_sr));
+				Optional<Value> sizeLimit = co_await tr->get(Key("\xff\x02/fdbClientInfo/client_txn_size_limit/"_sr));
 
 				tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				// The value doesn't matter too much, as long as the key is set.
@@ -314,53 +285,55 @@ ACTOR Future<Void> globalConfigMigrate(GrvProxyData* grvProxyData) {
 					tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
 				}
 
-				wait(tr->commit());
+				co_await tr->commit();
 				break;
 			} catch (Error& e) {
 				// Multiple GRV proxies may attempt this migration at the same
 				// time, sometimes resulting in aborts due to conflicts.
 				TraceEvent(SevInfo, "GlobalConfigRetryableMigrationError").errorUnsuppressed(e).suppressFor(1.0);
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	} catch (Error& e) {
 		// Catch non-retryable errors (and do nothing).
 		TraceEvent(SevWarnAlways, "GlobalConfigMigrationError").error(e);
 	}
-	return Void();
 }
 
 // Periodically refresh local copy of global configuration.
-ACTOR Future<Void> globalConfigRefresh(GrvProxyData* grvProxyData, Version* cachedVersion, RangeResult* cachedData) {
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(grvProxyData->cx);
-	loop {
+Future<Void> globalConfigRefresh(GrvProxyData* grvProxyData, Version* cachedVersion, RangeResult* cachedData) {
+	auto tr = makeReference<ReadYourWritesTransaction>(grvProxyData->cx);
+	while (true) {
+		Error err;
 		try {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			state Future<Optional<Value>> globalConfigVersionFuture = tr->get(globalConfigVersionKey);
-			state Future<RangeResult> tmpCachedDataFuture = tr->getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY);
-			state Optional<Value> globalConfigVersion = wait(globalConfigVersionFuture);
-			RangeResult tmpCachedData = wait(tmpCachedDataFuture);
+			Future<Optional<Value>> globalConfigVersionFuture = tr->get(globalConfigVersionKey);
+			Future<RangeResult> tmpCachedDataFuture = tr->getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY);
+			Optional<Value> globalConfigVersion = co_await globalConfigVersionFuture;
+			RangeResult tmpCachedData = co_await tmpCachedDataFuture;
 			*cachedData = tmpCachedData;
 			if (globalConfigVersion.present()) {
 				Version parsedVersion;
 				memcpy(&parsedVersion, globalConfigVersion.get().begin(), sizeof(Version));
 				*cachedVersion = bigEndian64(parsedVersion);
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
 }
 
 // Handle common GlobalConfig transactions on the server side, because not all
 // clients are allowed to read system keys. Eventually, this could become its
 // own role.
-ACTOR Future<Void> globalConfigRequestServer(GrvProxyData* grvProxyData, GrvProxyInterface grvProxy) {
-	state ActorCollection actors(false);
-	state Future<Void> refreshFuture; // so there is only one running attempt
-	state Version cachedVersion = 0;
-	state RangeResult cachedData;
+Future<Void> globalConfigRequestServer(GrvProxyData* grvProxyData, GrvProxyInterface grvProxy) {
+	ActorCollection actors(false);
+	Future<Void> refreshFuture; // so there is only one running attempt
+	Version cachedVersion = 0;
+	RangeResult cachedData;
 
 	// Attempt to refresh the global configuration while the migration is
 	// ongoing. This is a small optimization to avoid waiting for the migration
@@ -371,59 +344,62 @@ ACTOR Future<Void> globalConfigRequestServer(GrvProxyData* grvProxyData, GrvProx
 	                delay(SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_INTERVAL);
 
 	// Run one-time migration to support upgrades.
-	wait(success(timeout(globalConfigMigrate(grvProxyData), SERVER_KNOBS->GLOBAL_CONFIG_MIGRATE_TIMEOUT)));
-
-	loop {
-		choose {
-			when(GlobalConfigRefreshRequest refresh = waitNext(grvProxy.refreshGlobalConfig.getFuture())) {
-				// Must have an up to date copy of global configuration in
-				// order to serve it to the client (up to date from the clients
-				// point of view. The client learns the version through a
-				// ClientDBInfo update).
-				if (refresh.lastKnown <= cachedVersion) {
-					refresh.reply.send(GlobalConfigRefreshReply{ cachedData.arena(), cachedVersion, cachedData });
-				} else {
-					refresh.reply.sendError(future_version());
-				}
-			}
-			when(wait(refreshFuture)) {
-				refreshFuture = timeout(globalConfigRefresh(grvProxyData, &cachedVersion, &cachedData),
-				                        SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT,
-				                        Void()) &&
-				                delay(SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_INTERVAL);
-			}
-			when(wait(actors.getResult())) {
-				ASSERT(false);
+	co_await timeout(globalConfigMigrate(grvProxyData), SERVER_KNOBS->GLOBAL_CONFIG_MIGRATE_TIMEOUT);
+
+	while (true) {
+		auto res = co_await race(grvProxy.refreshGlobalConfig.getFuture(), refreshFuture, actors.getResult());
+		if (res.index() == 0) {
+			GlobalConfigRefreshRequest refresh = std::get<0>(std::move(res));
+
+			// Must have an up to date copy of global configuration in
+			// order to serve it to the client (up to date from the clients
+			// point of view. The client learns the version through a
+			// ClientDBInfo update).
+			if (refresh.lastKnown <= cachedVersion) {
+				refresh.reply.send(GlobalConfigRefreshReply{ cachedData.arena(), cachedVersion, cachedData });
+			} else {
+				refresh.reply.sendError(future_version());
 			}
+		} else if (res.index() == 1) {
+			// refreshFuture fired
+			refreshFuture = timeout(globalConfigRefresh(grvProxyData, &cachedVersion, &cachedData),
+			                        SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_TIMEOUT,
+			                        Void()) &&
+			                delay(SERVER_KNOBS->GLOBAL_CONFIG_REFRESH_INTERVAL);
+		} else if (res.index() == 2) {
+			// actors.getResult() should never return
+			ASSERT(false);
 		}
 	}
 }
 
 // Get transaction rate info from RateKeeper.
-ACTOR Future<Void> getRate(UID myID,
-                           Reference<AsyncVar<ServerDBInfo> const> db,
-                           int64_t* inTransactionCount,
-                           int64_t* inBatchTransactionCount,
-                           GrvTransactionRateInfo* transactionRateInfo,
-                           GrvTransactionRateInfo* batchTransactionRateInfo,
-                           GetHealthMetricsReply* healthMetricsReply,
-                           GetHealthMetricsReply* detailedHealthMetricsReply,
-                           TransactionTagMap<uint64_t>* transactionTagCounter,
-                           PrioritizedTransactionTagMap<ClientTagThrottleLimits>* clientThrottledTags,
-                           GrvProxyStats* stats,
-                           GrvProxyData* proxyData) {
-	state Future<Void> nextRequestTimer = Never();
-	state Future<Void> leaseTimeout = Never();
-	state Future<GetRateInfoReply> reply = Never();
-	state double lastDetailedReply = 0.0; // request detailed metrics immediately
-	state bool expectingDetailedReply = false;
+Future<Void> getRate(UID myID,
+                     Reference<AsyncVar<ServerDBInfo> const> db,
+                     int64_t* inTransactionCount,
+                     int64_t* inBatchTransactionCount,
+                     GrvTransactionRateInfo* transactionRateInfo,
+                     GrvTransactionRateInfo* batchTransactionRateInfo,
+                     GetHealthMetricsReply* healthMetricsReply,
+                     GetHealthMetricsReply* detailedHealthMetricsReply,
+                     TransactionTagMap<uint64_t>* transactionTagCounter,
+                     PrioritizedTransactionTagMap<ClientTagThrottleLimits>* clientThrottledTags,
+                     GrvProxyStats* stats,
+                     GrvProxyData* proxyData) {
+	Future<Void> nextRequestTimer = Never();
+	Future<Void> leaseTimeout = Never();
+	Future<GetRateInfoReply> reply = Never();
+	double lastDetailedReply = 0.0; // request detailed metrics immediately
+	bool expectingDetailedReply = false;
 	// state int64_t lastTC = 0;
 
 	if (db->get().ratekeeper.present()) {
 		nextRequestTimer = Void();
 	}
-	loop choose {
-		when(wait(db->onChange())) {
+	while (true) {
+		auto res = co_await race(db->onChange(), nextRequestTimer, reply, leaseTimeout);
+		if (res.index() == 0) {
+
 			if (db->get().ratekeeper.present()) {
 				TraceEvent("ProxyRatekeeperChanged", myID).detail("RKID", db->get().ratekeeper.get().id());
 				nextRequestTimer = Void(); // trigger GetRate request
@@ -432,8 +408,8 @@ ACTOR Future<Void> getRate(UID myID,
 				nextRequestTimer = Never();
 				reply = Never();
 			}
-		}
-		when(wait(nextRequestTimer)) {
+		} else if (res.index() == 1) {
+
 			nextRequestTimer = Never();
 			bool detailed = now() - lastDetailedReply > SERVER_KNOBS->DETAILED_METRIC_UPDATE_RATE;
 
@@ -446,8 +422,9 @@ ACTOR Future<Void> getRate(UID myID,
 			                                                                       detailed)));
 			transactionTagCounter->clear();
 			expectingDetailedReply = detailed;
-		}
-		when(GetRateInfoReply rep = wait(reply)) {
+		} else if (res.index() == 2) {
+			GetRateInfoReply rep = std::get<2>(std::move(res));
+
 			reply = Never();
 
 			transactionRateInfo->setRate(rep.transactionRate);
@@ -470,17 +447,16 @@ ACTOR Future<Void> getRate(UID myID,
 			if (rep.clientThrottledTags.present()) {
 				*clientThrottledTags = std::move(rep.clientThrottledTags.get());
 			}
-			if (rep.proxyThrottledTags.present()) {
-				proxyData->tagThrottler.updateRates(rep.proxyThrottledTags.get());
-			}
-		}
-		when(wait(leaseTimeout)) {
+		} else if (res.index() == 3) {
+
 			transactionRateInfo->disable();
 			batchTransactionRateInfo->disable();
 			++stats->leaseTimeouts;
 			TraceEvent(SevWarn, "GrvProxyRateLeaseExpired", myID).suppressFor(5.0);
 			//TraceEvent("GrvProxyRate", myID).detail("Rate", 0.0).detail("BatchRate", 0.0).detail("Lease", 0);
 			leaseTimeout = Never();
+		} else {
+			UNREACHABLE();
 		}
 	}
 }
@@ -502,29 +478,82 @@ void proxyGRVThresholdExceeded(const GetReadVersionRequest* req, GrvProxyStats*
 }
 
 // Drop a GetReadVersion request from a queue, by responding an error to the request.
-void dropRequestFromQueue(Deque<GetReadVersionRequest>* queue, GrvProxyStats* stats) {
+void dropRequestFromQueue(Deque<GetReadVersionRequest>* queue,
+                          GrvProxyStats* stats,
+                          GrvQueueTransactionCounts* queueTransactionCounts) {
 	proxyGRVThresholdExceeded(&queue->front(), stats);
 	++stats->txnRequestOut;
+	queueTransactionCounts->remove(queue->front());
 	queue->pop_front();
 }
 
+void rejectForMaxGrvQueueDelay(GetReadVersionRequest const& req, GrvProxyStats* stats, double estimatedRemainingDelay) {
+	double elapsedQueueDelay = std::max(0.0, now() - req.requestTime());
+	TraceEvent("ProxyGRVQueueDelayExceeded")
+	    .suppressFor(5.0)
+	    .detail("Priority", static_cast<int>(req.priority))
+	    .detail("TransactionCount", req.transactionCount)
+	    .detail("ElapsedQueueDelay", elapsedQueueDelay)
+	    .detail("EstimatedRemainingDelay", estimatedRemainingDelay)
+	    .detail("MaxGrvQueueDelayMS", req.maxGrvQueueDelayMS.orDefault(-1));
+	req.reply.sendError(transaction_grv_queue_rejected());
+	stats->txnThrottled += req.transactionCount;
+	++stats->txnRequestErrors;
+}
+
+bool rejectIncomingForMaxGrvQueueDelay(GetReadVersionRequest const& req,
+                                       GrvQueueTransactionCounts const& queueTransactionCounts,
+                                       GrvProxyStats* stats,
+                                       GrvTransactionRateInfo const* normalRateInfo,
+                                       GrvTransactionRateInfo const* batchRateInfo) {
+	// Requests with this option should be rejected before normal queue accounting
+	// or insertion. The estimate uses normal-rate work for every request and adds
+	// batch-rate work for batch-priority requests.
+	if (!req.maxGrvQueueDelayMS.present()) {
+		return false;
+	}
+	if (req.priority >= TransactionPriority::IMMEDIATE) {
+		return false;
+	}
+
+	auto estimate = estimateRemainingGrvQueueDelay(
+	    req.priority, req.transactionCount, queueTransactionCounts, normalRateInfo, batchRateInfo);
+	if (estimate.batchRateDelay.present()) {
+		if (shouldRejectForMaxGrvQueueDelay(req, estimate.batchRateDelay.get())) {
+			rejectForMaxGrvQueueDelay(req, stats, estimate.batchRateDelay.get());
+			return true;
+		}
+	}
+
+	if (shouldRejectForMaxGrvQueueDelay(req, estimate.normalRateDelay)) {
+		rejectForMaxGrvQueueDelay(req, stats, estimate.normalRateDelay);
+		return true;
+	}
+
+	return false;
+}
+
 // Put a GetReadVersion request into the queue corresponding to its priority.
-ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo> const> db,
-                                               Deque<GetReadVersionRequest>* systemQueue,
-                                               Deque<GetReadVersionRequest>* defaultQueue,
-                                               Deque<GetReadVersionRequest>* batchQueue,
-                                               FutureStream<GetReadVersionRequest> readVersionRequests,
-                                               PromiseStream<Void> GRVTimer,
-                                               double* lastGRVTime,
-                                               double* GRVBatchTime,
-                                               FutureStream<double> normalGRVLatency,
-                                               GrvProxyStats* stats,
-                                               GrvTransactionRateInfo* batchRateInfo,
-                                               GrvProxyTagThrottler* tagThrottler) {
+Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo> const> db,
+                                         Deque<GetReadVersionRequest>* systemQueue,
+                                         Deque<GetReadVersionRequest>* defaultQueue,
+                                         Deque<GetReadVersionRequest>* batchQueue,
+                                         FutureStream<GetReadVersionRequest> readVersionRequests,
+                                         PromiseStream<Void> GRVTimer,
+                                         double* lastGRVTime,
+                                         double* GRVBatchTime,
+                                         FutureStream<double> normalGRVLatency,
+                                         GrvProxyStats* stats,
+                                         GrvTransactionRateInfo* normalRateInfo,
+                                         GrvTransactionRateInfo* batchRateInfo,
+                                         GrvQueueTransactionCounts* queueTransactionCounts) {
 	getCurrentLineage()->modify(&TransactionLineage::operation) =
 	    TransactionLineage::Operation::GetConsistentReadVersion;
-	loop choose {
-		when(GetReadVersionRequest req = waitNext(readVersionRequests)) {
+	while (true) {
+		auto res = co_await race(readVersionRequests, normalGRVLatency);
+		if (res.index() == 0) {
+			GetReadVersionRequest req = std::get<0>(std::move(res));
+
 			// auto lineage = make_scoped_lineage(&TransactionLineage::txID, req.spanContext.first());
 			// getCurrentLineage()->modify(&TransactionLineage::txID) =
 			// WARNING: this code is run at a high priority, so it needs to do as little work as possible
@@ -538,17 +567,17 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
 					canBeQueued = false;
 				} else if (req.priority == TransactionPriority::DEFAULT) {
 					if (!batchQueue->empty()) {
-						dropRequestFromQueue(batchQueue, stats);
+						dropRequestFromQueue(batchQueue, stats, queueTransactionCounts);
 						--stats->batchGRVQueueSize;
 					} else {
 						canBeQueued = false;
 					}
 				} else {
 					if (!batchQueue->empty()) {
-						dropRequestFromQueue(batchQueue, stats);
+						dropRequestFromQueue(batchQueue, stats, queueTransactionCounts);
 						--stats->batchGRVQueueSize;
 					} else if (!defaultQueue->empty()) {
-						dropRequestFromQueue(defaultQueue, stats);
+						dropRequestFromQueue(defaultQueue, stats, queueTransactionCounts);
 						--stats->defaultGRVQueueSize;
 					} else {
 						canBeQueued = false;
@@ -558,6 +587,11 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
 			if (!canBeQueued) {
 				proxyGRVThresholdExceeded(&req, stats);
 			} else {
+				if (rejectIncomingForMaxGrvQueueDelay(
+				        req, *queueTransactionCounts, stats, normalRateInfo, batchRateInfo)) {
+					continue;
+				}
+
 				stats->addRequest(req.transactionCount);
 
 				if (req.debugID.present())
@@ -566,7 +600,8 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
 					                      "GrvProxyServer.queueTransactionStartRequests.Before");
 
 				if (systemQueue->empty() && defaultQueue->empty() && batchQueue->empty()) {
-					forwardPromise(GRVTimer,
+					forwardPromise(Uncancellable{},
+					               GRVTimer,
 					               delayJittered(std::max(0.0, *GRVBatchTime - (now() - *lastGRVTime)),
 					                             TaskPriority::ProxyGRVTimer));
 				}
@@ -576,19 +611,15 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
 					stats->txnStartIn += req.transactionCount;
 					stats->txnSystemPriorityStartIn += req.transactionCount;
 					++stats->systemGRVQueueSize;
+					queueTransactionCounts->add(req);
 					systemQueue->push_back(req);
 				} else if (req.priority >= TransactionPriority::DEFAULT) {
-					if (SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES && req.isTagged()) {
-						++stats->tagThrottlerGRVQueueSize;
-						stats->txnTagThrottlerIn += req.transactionCount;
-						tagThrottler->addRequest(req);
-					} else {
-						++stats->txnRequestIn;
-						stats->txnStartIn += req.transactionCount;
-						stats->txnDefaultPriorityStartIn += req.transactionCount;
-						++stats->defaultGRVQueueSize;
-						defaultQueue->push_back(req);
-					}
+					++stats->txnRequestIn;
+					stats->txnStartIn += req.transactionCount;
+					stats->txnDefaultPriorityStartIn += req.transactionCount;
+					++stats->defaultGRVQueueSize;
+					queueTransactionCounts->add(req);
+					defaultQueue->push_back(req);
 				} else {
 					// Return error for batch_priority GRV requests
 					int64_t proxiesCount = std::max((int)db->get().client.grvProxies.size(), 1);
@@ -596,51 +627,49 @@ ACTOR Future<Void> queueGetReadVersionRequests(Reference<AsyncVar<ServerDBInfo>
 						req.reply.sendError(batch_transaction_throttled());
 						stats->txnThrottled += req.transactionCount;
 					} else {
-						if (SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES && req.isTagged()) {
-							++stats->tagThrottlerGRVQueueSize;
-							stats->txnTagThrottlerIn += req.transactionCount;
-							tagThrottler->addRequest(req);
-						} else {
-							++stats->txnRequestIn;
-							stats->txnStartIn += req.transactionCount;
-							stats->txnBatchPriorityStartIn += req.transactionCount;
-							++stats->batchGRVQueueSize;
-							batchQueue->push_back(req);
-						}
+						++stats->txnRequestIn;
+						stats->txnStartIn += req.transactionCount;
+						stats->txnBatchPriorityStartIn += req.transactionCount;
+						++stats->batchGRVQueueSize;
+						queueTransactionCounts->add(req);
+						batchQueue->push_back(req);
 					}
 				}
 			}
 		}
-		// dynamic batching monitors reply latencies
-		when(double reply_latency = waitNext(normalGRVLatency)) {
+		// Dynamic batching monitors reply latencies.
+		else if (res.index() == 1) {
+			double reply_latency = std::get<1>(std::move(res));
+
 			double target_latency = reply_latency * SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION;
 			*GRVBatchTime = std::max(
 			    SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_MIN,
 			    std::min(SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_MAX,
 			             target_latency * SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA +
 			                 *GRVBatchTime * (1 - SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA)));
+		} else {
+			UNREACHABLE();
 		}
 	}
 }
 
-ACTOR Future<Void> updateLastCommit(GrvProxyData* self, Optional<UID> debugID = Optional<UID>()) {
-	state double confirmStart = now();
+Future<Void> updateLastCommit(GrvProxyData* self, Optional<UID> debugID = Optional<UID>()) {
+	double confirmStart = now();
 	self->lastStartCommit = confirmStart;
 	self->updateCommitRequests++;
-	wait(self->logSystem->confirmEpochLive(debugID));
+	co_await self->logSystem->confirmEpochLive(debugID);
 	self->updateCommitRequests--;
 	self->lastCommitLatency = now() - confirmStart;
 	self->lastCommitTime = std::max(self->lastCommitTime.get(), confirmStart);
-	return Void();
 }
 
-ACTOR Future<Void> lastCommitUpdater(GrvProxyData* self, PromiseStream<Future<Void>> addActor) {
-	loop {
+Future<Void> lastCommitUpdater(GrvProxyData* self, PromiseStream<Future<Void>> addActor) {
+	while (true) {
 		double interval = std::max(SERVER_KNOBS->MIN_CONFIRM_INTERVAL,
 		                           (SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION - self->lastCommitLatency) / 2.0);
 		double elapsed = now() - self->lastStartCommit;
 		if (elapsed < interval) {
-			wait(delay(interval + 0.0001 - elapsed));
+			co_await delay(interval + 0.0001 - elapsed);
 		} else {
 			// May want to change the default value of MAX_COMMIT_UPDATES since we don't have
 			if (self->updateCommitRequests < SERVER_KNOBS->MAX_COMMIT_UPDATES) {
@@ -654,39 +683,39 @@ ACTOR Future<Void> lastCommitUpdater(GrvProxyData* self, PromiseStream<Future<Vo
 	}
 }
 
-ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(std::vector<SpanContext> spanContexts,
-                                                          GrvProxyData* grvProxyData,
-                                                          uint32_t flags,
-                                                          Optional<UID> debugID,
-                                                          int transactionCount,
-                                                          int systemTransactionCount,
-                                                          int defaultPriTransactionCount,
-                                                          int batchPriTransactionCount) {
+Future<GetReadVersionReply> getLiveCommittedVersion(std::vector<SpanContext> spanContexts,
+                                                    GrvProxyData* grvProxyData,
+                                                    uint32_t flags,
+                                                    Optional<UID> debugID,
+                                                    int transactionCount,
+                                                    int systemTransactionCount,
+                                                    int defaultPriTransactionCount,
+                                                    int batchPriTransactionCount) {
 	// Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit
 	// response) when this request was sent (1) The version returned is the committedVersion of some proxy at some point
 	// before the request returns, so it is committed. (2) No proxy on our list reported committed a higher version
 	// before this request was received, because then its committedVersion would have been higher,
 	//     and no other proxy could have already committed anything without first ending the epoch
-	state Span span("GP:getLiveCommittedVersion"_loc);
+	Span span("GP:getLiveCommittedVersion"_loc);
 	for (const SpanContext& spanContext : spanContexts) {
 		span.addLink(spanContext);
 	}
 	++grvProxyData->stats.txnStartBatch;
 
-	state double grvStart = now();
-	state Future<GetRawCommittedVersionReply> replyFromMasterFuture;
+	double grvStart = now();
+	Future<GetRawCommittedVersionReply> replyFromMasterFuture;
 	replyFromMasterFuture = grvProxyData->master.getLiveCommittedVersion.getReply(
 	    GetRawCommittedVersionRequest(span.context, debugID, grvProxyData->ssVersionVectorCache.getMaxVersion()),
 	    TaskPriority::GetLiveCommittedVersionReply);
 
 	if (!SERVER_KNOBS->ALWAYS_CAUSAL_READ_RISKY && !(flags & GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY)) {
-		wait(transformError(updateLastCommit(grvProxyData, debugID), broken_promise(), tlog_failed()));
+		co_await transformError(updateLastCommit(grvProxyData, debugID), broken_promise(), tlog_failed());
 	} else if (SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION > 0 &&
 	           now() - SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION > grvProxyData->lastCommitTime.get()) {
-		wait(grvProxyData->lastCommitTime.whenAtLeast(now() - SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION));
+		co_await grvProxyData->lastCommitTime.whenAtLeast(now() - SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION);
 	}
 
-	state double grvConfirmEpochLive = now();
+	double grvConfirmEpochLive = now();
 	grvProxyData->stats.grvConfirmEpochLiveDist->sampleSeconds(grvConfirmEpochLive - grvStart);
 	if (debugID.present()) {
 		g_traceBatch.addEvent(
@@ -694,7 +723,7 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(std::vector<SpanContex
 	}
 
 	GetRawCommittedVersionReply repFromMaster =
-	    wait(transformError(replyFromMasterFuture, broken_promise(), master_failed()));
+	    co_await transformError(replyFromMasterFuture, broken_promise(), master_failed());
 	grvProxyData->version = std::max(grvProxyData->version, repFromMaster.version);
 	grvProxyData->minKnownCommittedVersion =
 	    std::max(grvProxyData->minKnownCommittedVersion, repFromMaster.minKnownCommittedVersion);
@@ -725,26 +754,25 @@ ACTOR Future<GetReadVersionReply> getLiveCommittedVersion(std::vector<SpanContex
 	grvProxyData->stats.txnDefaultPriorityStartOut += defaultPriTransactionCount;
 	grvProxyData->stats.txnBatchPriorityStartOut += batchPriTransactionCount;
 
-	return rep;
+	co_return rep;
 }
 
 // Returns the current read version (or minimum known committed version if requested),
 // to each request in the provided list. Also check if the request should be throttled.
 // Update GRV statistics according to the request's priority.
-ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
-                                  std::vector<GetReadVersionRequest> requests,
-                                  GrvProxyData* grvProxyData,
-                                  GrvProxyStats* stats,
-                                  Version minKnownCommittedVersion,
-                                  PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientThrottledTags,
-                                  int64_t midShardSize = 0) {
-	GetReadVersionReply _reply = wait(replyFuture);
-	GetReadVersionReply reply = _reply;
+Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
+                            std::vector<GetReadVersionRequest> requests,
+                            GrvProxyData* grvProxyData,
+                            GrvProxyStats* stats,
+                            Version minKnownCommittedVersion,
+                            PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientThrottledTags,
+                            int64_t midShardSize = 0) {
+	GetReadVersionReply reply = co_await replyFuture;
 	Version replyVersion = reply.version;
 
 	double end = g_network->timer();
 	for (GetReadVersionRequest const& request : requests) {
-		double duration = end - request.requestTime() - request.proxyTagThrottledDuration;
+		double duration = end - request.requestTime();
 		if (request.priority == TransactionPriority::BATCH) {
 			stats->grvBatchLatencySample.addMeasurement(duration);
 		}
@@ -770,11 +798,10 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 			grvProxyData->versionVectorSizeOnGRVReply->addMeasurement(reply.ssVersionVectorDelta.size());
 		}
 		reply.proxyId = grvProxyData->dbgid;
-		reply.proxyTagThrottledDuration = request.proxyTagThrottledDuration;
 
 		if (request.isTagged()) {
 			auto& priorityThrottledTags = clientThrottledTags[request.priority];
-			for (auto tag : request.tags) {
+			for (const auto& tag : request.tags) {
 				auto tagItr = priorityThrottledTags.find(tag.first);
 				if (tagItr != priorityThrottledTags.end()) {
 					if (tagItr->second.expiration > now()) {
@@ -812,45 +839,44 @@ ACTOR Future<Void> sendGrvReplies(Future<GetReadVersionReply> replyFuture,
 		request.reply.send(reply);
 		++stats->txnRequestOut;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> monitorDDMetricsChanges(int64_t* midShardSize, Reference<AsyncVar<ServerDBInfo> const> db) {
-	state Future<Void> nextRequestTimer = Never();
-	state Future<GetDataDistributorMetricsReply> nextReply = Never();
+Future<Void> monitorDDMetricsChanges(int64_t* midShardSize, Reference<AsyncVar<ServerDBInfo> const> db) {
+	Future<Void> nextRequestTimer = Never();
+	Future<GetDataDistributorMetricsReply> nextReply = Never();
 
 	if (db->get().distributor.present()) {
 		nextRequestTimer = Void();
 	}
-	loop {
+	while (true) {
 		try {
-			choose {
-				when(wait(db->onChange())) {
-					if (db->get().distributor.present()) {
-						TraceEvent("DataDistributorChanged", db->get().id)
-						    .detail("DDID", db->get().distributor.get().id());
-						nextRequestTimer = Void();
-					} else {
-						TraceEvent("DataDistributorDied", db->get().id);
-						nextRequestTimer = Never();
-					}
-					nextReply = Never();
-				}
-				when(wait(nextRequestTimer)) {
+			auto res = co_await race(db->onChange(), nextRequestTimer, nextReply);
+			if (res.index() == 0) {
+				if (db->get().distributor.present()) {
+					TraceEvent("DataDistributorChanged", db->get().id).detail("DDID", db->get().distributor.get().id());
+					nextRequestTimer = Void();
+				} else {
+					TraceEvent("DataDistributorDied", db->get().id);
 					nextRequestTimer = Never();
-					if (db->get().distributor.present()) {
-						nextReply = brokenPromiseToNever(db->get().distributor.get().dataDistributorMetrics.getReply(
-						    GetDataDistributorMetricsRequest(normalKeys, CLIENT_KNOBS->TOO_MANY, true)));
-					} else
-						nextReply = Never();
 				}
-				when(GetDataDistributorMetricsReply reply = wait(nextReply)) {
+				nextReply = Never();
+			} else if (res.index() == 1) {
+				nextRequestTimer = Never();
+				if (db->get().distributor.present()) {
+					nextReply = brokenPromiseToNever(db->get().distributor.get().dataDistributorMetrics.getReply(
+					    GetDataDistributorMetricsRequest(normalKeys, CLIENT_KNOBS->TOO_MANY, true)));
+				} else {
 					nextReply = Never();
-					ASSERT(reply.midShardSize.present());
-					*midShardSize = reply.midShardSize.get();
-					nextRequestTimer = delay(CLIENT_KNOBS->MID_SHARD_SIZE_MAX_STALENESS);
 				}
+			} else if (res.index() == 2) {
+				GetDataDistributorMetricsReply reply = std::get<2>(std::move(res));
+
+				nextReply = Never();
+				ASSERT(reply.midShardSize.present());
+				*midShardSize = reply.midShardSize.get();
+				nextRequestTimer = delay(CLIENT_KNOBS->MID_SHARD_SIZE_MAX_STALENESS);
+			} else {
+				UNREACHABLE();
 			}
 		} catch (Error& e) {
 			TraceEvent("DDMidShardSizeUpdateFail").error(e);
@@ -863,35 +889,36 @@ ACTOR Future<Void> monitorDDMetricsChanges(int64_t* midShardSize, Reference<Asyn
 	}
 }
 
-ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
-                                             Reference<AsyncVar<ServerDBInfo> const> db,
-                                             PromiseStream<Future<Void>> addActor,
-                                             GrvProxyData* grvProxyData,
-                                             GetHealthMetricsReply* healthMetricsReply,
-                                             GetHealthMetricsReply* detailedHealthMetricsReply) {
-	state double lastGRVTime = 0;
-	state PromiseStream<Void> GRVTimer;
-	state double GRVBatchTime = SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_MIN;
-
-	state int64_t transactionCount = 0;
-	state int64_t batchTransactionCount = 0;
-	state GrvTransactionRateInfo normalRateInfo(SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW,
-	                                            SERVER_KNOBS->START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET,
-	                                            /*rate=*/10);
-	state GrvTransactionRateInfo batchRateInfo(SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW,
-	                                           SERVER_KNOBS->START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET,
-	                                           /*rate=*/0);
-
-	state Deque<GetReadVersionRequest> systemQueue;
-	state Deque<GetReadVersionRequest> defaultQueue;
-	state Deque<GetReadVersionRequest> batchQueue;
-
-	state TransactionTagMap<uint64_t> transactionTagCounter;
-	state PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientThrottledTags;
-
-	state PromiseStream<double> normalGRVLatency;
-
-	state int64_t midShardSize = SERVER_KNOBS->MIN_SHARD_BYTES;
+static Future<Void> transactionStarter(GrvProxyInterface proxy,
+                                       Reference<AsyncVar<ServerDBInfo> const> db,
+                                       PromiseStream<Future<Void>> addActor,
+                                       GrvProxyData* grvProxyData,
+                                       GetHealthMetricsReply* healthMetricsReply,
+                                       GetHealthMetricsReply* detailedHealthMetricsReply) {
+	double lastGRVTime = 0;
+	PromiseStream<Void> GRVTimer;
+	double GRVBatchTime = SERVER_KNOBS->START_TRANSACTION_BATCH_INTERVAL_MIN;
+
+	int64_t transactionCount = 0;
+	int64_t batchTransactionCount = 0;
+	GrvTransactionRateInfo normalRateInfo(SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW,
+	                                      SERVER_KNOBS->START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET,
+	                                      /*rate=*/10);
+	GrvTransactionRateInfo batchRateInfo(SERVER_KNOBS->START_TRANSACTION_RATE_WINDOW,
+	                                     SERVER_KNOBS->START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET,
+	                                     /*rate=*/0);
+
+	Deque<GetReadVersionRequest> systemQueue;
+	Deque<GetReadVersionRequest> defaultQueue;
+	Deque<GetReadVersionRequest> batchQueue;
+	GrvQueueTransactionCounts queueTransactionCounts;
+
+	TransactionTagMap<uint64_t> transactionTagCounter;
+	PrioritizedTransactionTagMap<ClientTagThrottleLimits> clientThrottledTags;
+
+	PromiseStream<double> normalGRVLatency;
+
+	int64_t midShardSize = SERVER_KNOBS->MIN_SHARD_BYTES;
 	getCurrentLineage()->modify(&TransactionLineage::operation) =
 	    TransactionLineage::Operation::GetConsistentReadVersion;
 	addActor.send(monitorDDMetricsChanges(&midShardSize, db));
@@ -918,20 +945,21 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 	                                          &GRVBatchTime,
 	                                          normalGRVLatency.getFuture(),
 	                                          &grvProxyData->stats,
+	                                          &normalRateInfo,
 	                                          &batchRateInfo,
-	                                          &grvProxyData->tagThrottler));
+	                                          &queueTransactionCounts));
 
 	while (std::find(db->get().client.grvProxies.begin(), db->get().client.grvProxies.end(), proxy) ==
 	       db->get().client.grvProxies.end()) {
-		wait(db->onChange());
+		co_await db->onChange();
 	}
 
 	ASSERT(db->get().recoveryState >=
 	       RecoveryState::ACCEPTING_COMMITS); // else potentially we could return uncommitted read versions from master.
 	TraceEvent("GrvProxyReadyForTxnStarts", proxy.id());
 
-	loop {
-		waitNext(GRVTimer.getFuture());
+	while (true) {
+		co_await GRVTimer.getFuture();
 		// Select zero or more transactions to start
 		double t = now();
 		double elapsed = now() - lastGRVTime;
@@ -942,8 +970,6 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 			elapsed = 1e-15;
 		}
 
-		grvProxyData->stats.update(grvProxyData->tagThrottler.releaseTransactions(elapsed, batchQueue, defaultQueue));
-
 		normalRateInfo.startReleaseWindow();
 		batchRateInfo.startReleaseWindow();
 
@@ -1008,9 +1034,10 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 				grvProxyData->stats.batchTxnGRVTimeInQueue.addMeasurement(currentTime - req.requestTime());
 				--grvProxyData->stats.batchGRVQueueSize;
 			}
-			for (auto tag : req.tags) {
-				transactionTagCounter[tag.first] += tag.second;
+			for (const auto& [tag, count] : req.tags) {
+				transactionTagCounter[tag] += count;
 			}
+			queueTransactionCounts.remove(req);
 			start[req.flags & 1].push_back(std::move(req));
 			static_assert(GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY == 1, "Implementation dependent on flag value");
 			transactionQueue->pop_front();
@@ -1035,6 +1062,7 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 
 		if (!systemQueue.empty() || !defaultQueue.empty() || !batchQueue.empty()) {
 			forwardPromise(
+			    Uncancellable{},
 			    GRVTimer,
 			    delayJittered(SERVER_KNOBS->START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL, TaskPriority::ProxyGRVTimer));
 		}
@@ -1073,7 +1101,7 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 		int defaultGRVProcessed = 0;
 		int batchGRVProcessed = 0;
 		for (int i = 0; i < start.size(); i++) {
-			if (start[i].size()) {
+			if (!start[i].empty()) {
 				std::vector<SpanContext> spanContexts;
 				spanContexts.reserve(start[i].size());
 				for (const GetReadVersionRequest& request : start[i]) {
@@ -1112,17 +1140,17 @@ ACTOR static Future<Void> transactionStarter(GrvProxyInterface proxy,
 	}
 }
 
-ACTOR Future<Void> grvProxyServerCore(GrvProxyInterface proxy,
-                                      MasterInterface master,
-                                      LifetimeToken masterLifetime,
-                                      Reference<AsyncVar<ServerDBInfo> const> db) {
-	state GrvProxyData grvProxyData(proxy.id(), master, proxy.getConsistentReadVersion, db);
+Future<Void> grvProxyServerCore(GrvProxyInterface proxy,
+                                MasterInterface master,
+                                LifetimeToken masterLifetime,
+                                Reference<AsyncVar<ServerDBInfo> const> db) {
+	GrvProxyData grvProxyData(proxy.id(), master, proxy.getConsistentReadVersion, db);
 
-	state PromiseStream<Future<Void>> addActor;
-	state Future<Void> onError = actorCollection(addActor.getFuture());
+	PromiseStream<Future<Void>> addActor;
+	Future<Void> onError = actorCollection(addActor.getFuture());
 
-	state GetHealthMetricsReply healthMetricsReply;
-	state GetHealthMetricsReply detailedHealthMetricsReply;
+	GetHealthMetricsReply healthMetricsReply;
+	GetHealthMetricsReply detailedHealthMetricsReply;
 
 	addActor.send(waitFailureServer(proxy.waitFailure.getFuture()));
 	addActor.send(traceRole(Role::GRV_PROXY, proxy.id()));
@@ -1135,11 +1163,11 @@ ACTOR Future<Void> grvProxyServerCore(GrvProxyInterface proxy,
 	// Wait until we can load the "real" logsystem, since we don't support switching them currently
 	while (!(masterLifetime.isEqual(grvProxyData.db->get().masterLifetime) &&
 	         grvProxyData.db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS)) {
-		wait(grvProxyData.db->onChange());
+		co_await grvProxyData.db->onChange();
 	}
 	// Do we need to wait for any db info change? Yes. To update latency band.
-	state Future<Void> dbInfoChange = grvProxyData.db->onChange();
-	grvProxyData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), grvProxyData.db->get(), false, addActor);
+	Future<Void> dbInfoChange = grvProxyData.db->onChange();
+	grvProxyData.logSystem = makeLogSystemFromServerDBInfo(proxy.id(), grvProxyData.db->get(), false, addActor);
 
 	grvProxyData.updateLatencyBandConfig(grvProxyData.db->get().latencyBandConfig);
 
@@ -1152,40 +1180,37 @@ ACTOR Future<Void> grvProxyServerCore(GrvProxyInterface proxy,
 		addActor.send(lastCommitUpdater(&grvProxyData, addActor));
 	}
 
-	loop choose {
-		when(wait(dbInfoChange)) {
-			dbInfoChange = grvProxyData.db->onChange();
+	while (true) {
+		co_await waitOrError(dbInfoChange, onError);
+		dbInfoChange = grvProxyData.db->onChange();
 
-			if (masterLifetime.isEqual(grvProxyData.db->get().masterLifetime) &&
-			    grvProxyData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION) {
-				grvProxyData.logSystem =
-				    ILogSystem::fromServerDBInfo(proxy.id(), grvProxyData.db->get(), false, addActor);
-			}
-			grvProxyData.updateLatencyBandConfig(grvProxyData.db->get().latencyBandConfig);
+		if (masterLifetime.isEqual(grvProxyData.db->get().masterLifetime) &&
+		    grvProxyData.db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION) {
+			grvProxyData.logSystem = makeLogSystemFromServerDBInfo(proxy.id(), grvProxyData.db->get(), false, addActor);
 		}
-		when(wait(onError)) {}
+		grvProxyData.updateLatencyBandConfig(grvProxyData.db->get().latencyBandConfig);
 	}
 }
 
-ACTOR Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db,
-                                uint64_t recoveryCount,
-                                GrvProxyInterface myInterface) {
-	loop {
+Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db,
+                          uint64_t recoveryCount,
+                          GrvProxyInterface myInterface) {
+	while (true) {
 		if (db->get().recoveryCount >= recoveryCount &&
 		    std::find(db->get().client.grvProxies.begin(), db->get().client.grvProxies.end(), myInterface) ==
 		        db->get().client.grvProxies.end()) {
 			throw worker_removed();
 		}
-		wait(db->onChange());
+		co_await db->onChange();
 	}
 }
 
-ACTOR Future<Void> grvProxyServer(GrvProxyInterface proxy,
-                                  InitializeGrvProxyRequest req,
-                                  Reference<AsyncVar<ServerDBInfo> const> db) {
+Future<Void> grvProxyServer(GrvProxyInterface proxy,
+                            InitializeGrvProxyRequest req,
+                            Reference<AsyncVar<ServerDBInfo> const> db) {
 	try {
-		state Future<Void> core = grvProxyServerCore(proxy, req.master, req.masterLifetime, db);
-		wait(core || checkRemoved(db, req.recoveryCount, proxy));
+		Future<Void> core = grvProxyServerCore(proxy, req.master, req.masterLifetime, db);
+		co_await (core || checkRemoved(db, req.recoveryCount, proxy));
 	} catch (Error& e) {
 		TraceEvent("GrvProxyTerminated", proxy.id()).errorUnsuppressed(e);
 		ASSERT(e.code() !=
@@ -1199,5 +1224,4 @@ ACTOR Future<Void> grvProxyServer(GrvProxyInterface proxy,
 			throw;
 		}
 	}
-	return Void();
 }
diff --git a/fdbserver/grvproxy/GrvQueueDelay.cpp b/fdbserver/grvproxy/GrvQueueDelay.cpp
new file mode 100644
index 00000000000..9fd1252d3dc
--- /dev/null
+++ b/fdbserver/grvproxy/GrvQueueDelay.cpp
@@ -0,0 +1,91 @@
+/*
+ * GrvQueueDelay.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "GrvQueueDelay.h"
+
+#include <algorithm>
+
+void GrvQueueTransactionCounts::add(TransactionPriority priority, int64_t transactionCount) {
+	ASSERT_GE(transactionCount, 0);
+	if (priority >= TransactionPriority::IMMEDIATE) {
+		systemPriority += transactionCount;
+	} else if (priority >= TransactionPriority::DEFAULT) {
+		defaultPriority += transactionCount;
+	} else {
+		batchPriority += transactionCount;
+	}
+}
+
+void GrvQueueTransactionCounts::add(GetReadVersionRequest const& req) {
+	add(req.priority, req.transactionCount);
+}
+
+void GrvQueueTransactionCounts::remove(TransactionPriority priority, int64_t transactionCount) {
+	ASSERT_GE(transactionCount, 0);
+	if (priority >= TransactionPriority::IMMEDIATE) {
+		systemPriority -= transactionCount;
+		ASSERT_GE(systemPriority, 0);
+	} else if (priority >= TransactionPriority::DEFAULT) {
+		defaultPriority -= transactionCount;
+		ASSERT_GE(defaultPriority, 0);
+	} else {
+		batchPriority -= transactionCount;
+		ASSERT_GE(batchPriority, 0);
+	}
+}
+
+void GrvQueueTransactionCounts::remove(GetReadVersionRequest const& req) {
+	remove(req.priority, req.transactionCount);
+}
+
+int64_t GrvQueueTransactionCounts::normalRateQueuedTransactions() const {
+	return systemPriority + defaultPriority;
+}
+
+int64_t GrvQueueTransactionCounts::batchRateQueuedTransactions() const {
+	return normalRateQueuedTransactions() + batchPriority;
+}
+
+GrvQueueDelayEstimate estimateRemainingGrvQueueDelay(TransactionPriority priority,
+                                                     int64_t transactionCount,
+                                                     GrvQueueTransactionCounts const& queueTransactionCounts,
+                                                     GrvTransactionRateInfo const* normalRateInfo,
+                                                     GrvTransactionRateInfo const* batchRateInfo) {
+	double normalRateDelay =
+	    normalRateInfo->estimateDelay(queueTransactionCounts.normalRateQueuedTransactions(), transactionCount);
+	GrvQueueDelayEstimate estimate{ normalRateDelay, Optional<double>() };
+
+	if (priority < TransactionPriority::DEFAULT) {
+		estimate.batchRateDelay =
+		    batchRateInfo->estimateDelay(queueTransactionCounts.batchRateQueuedTransactions(), transactionCount);
+	}
+
+	return estimate;
+}
+
+bool shouldRejectForMaxGrvQueueDelay(GetReadVersionRequest const& req, double remainingDelay) {
+	if (!req.maxGrvQueueDelayMS.present()) {
+		return false;
+	}
+
+	double elapsedQueueDelay = now() - req.requestTime();
+	double estimatedQueueDelay = std::max(0.0, elapsedQueueDelay) + remainingDelay;
+	return estimatedQueueDelay > req.maxGrvQueueDelayMS.get() / 1000.0;
+}
diff --git a/fdbserver/grvproxy/GrvQueueDelay.h b/fdbserver/grvproxy/GrvQueueDelay.h
new file mode 100644
index 00000000000..4fb2652cc32
--- /dev/null
+++ b/fdbserver/grvproxy/GrvQueueDelay.h
@@ -0,0 +1,54 @@
+/*
+ * GrvQueueDelay.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/GrvProxyInterface.h"
+#include "GrvTransactionRateInfo.h"
+#include "flow/flow.h"
+
+struct GrvQueueTransactionCounts {
+	int64_t systemPriority{ 0 };
+	int64_t defaultPriority{ 0 };
+	int64_t batchPriority{ 0 };
+
+	void add(TransactionPriority priority, int64_t transactionCount);
+	void add(GetReadVersionRequest const& req);
+
+	void remove(TransactionPriority priority, int64_t transactionCount);
+	void remove(GetReadVersionRequest const& req);
+
+	int64_t normalRateQueuedTransactions() const;
+	int64_t batchRateQueuedTransactions() const;
+};
+
+struct GrvQueueDelayEstimate {
+	double normalRateDelay;
+	Optional<double> batchRateDelay;
+};
+
+GrvQueueDelayEstimate estimateRemainingGrvQueueDelay(TransactionPriority priority,
+                                                     int64_t transactionCount,
+                                                     GrvQueueTransactionCounts const& queueTransactionCounts,
+                                                     GrvTransactionRateInfo const* normalRateInfo,
+                                                     GrvTransactionRateInfo const* batchRateInfo);
+
+bool shouldRejectForMaxGrvQueueDelay(GetReadVersionRequest const& req, double remainingDelay);
diff --git a/fdbserver/grvproxy/GrvQueueDelayTests.cpp b/fdbserver/grvproxy/GrvQueueDelayTests.cpp
new file mode 100644
index 00000000000..148b4d1cd0d
--- /dev/null
+++ b/fdbserver/grvproxy/GrvQueueDelayTests.cpp
@@ -0,0 +1,181 @@
+/*
+ * GrvQueueDelayTests.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "GrvQueueDelay.h"
+
+#include "fdbserver/core/Knobs.h"
+#include "flow/UnitTest.h"
+
+#include <array>
+#include <cmath>
+#include <limits>
+#include <string_view>
+
+void forceLinkGrvQueueDelayTests() {}
+
+namespace {
+
+constexpr double EXPECTED_RATE = 10.0;
+
+void assertNear(double expected, double actual) {
+	ASSERT_LT(std::abs(expected - actual), 1e-9);
+}
+
+GrvTransactionRateInfo makeRateInfo() {
+	GrvTransactionRateInfo rateInfo(/*rateWindow=*/1.0, /*maxEmptyQueueBudget=*/0.0, EXPECTED_RATE);
+	rateInfo.setRate(EXPECTED_RATE);
+	rateInfo.startReleaseWindow();
+	return rateInfo;
+}
+
+double expectedEstimateDelay(int64_t queuedTransactions, int64_t transactionCount) {
+	double capacity = std::min(EXPECTED_RATE, SERVER_KNOBS->START_TRANSACTION_MAX_TRANSACTIONS_TO_START);
+	double deficit = queuedTransactions + transactionCount - capacity;
+	if (deficit <= 0.0) {
+		return 0.0;
+	}
+	return deficit / EXPECTED_RATE;
+}
+
+bool expectedShouldReject(Optional<int64_t> maxDelayMS, double elapsedQueueDelay, double remainingDelay) {
+	if (!maxDelayMS.present()) {
+		return false;
+	}
+	double estimatedQueueDelay = std::max(0.0, elapsedQueueDelay) + remainingDelay;
+	return estimatedQueueDelay > maxDelayMS.get() / 1000.0;
+}
+
+} // namespace
+
+TEST_CASE("/fdbserver/grvproxy/maxGrvQueueDelay/queueTransactionCounts") {
+	GrvQueueTransactionCounts counts;
+
+	GetReadVersionRequest systemReq;
+	systemReq.priority = TransactionPriority::IMMEDIATE;
+	systemReq.transactionCount = 2;
+	counts.add(systemReq);
+
+	GetReadVersionRequest defaultReq;
+	defaultReq.priority = TransactionPriority::DEFAULT;
+	defaultReq.transactionCount = 3;
+	counts.add(defaultReq);
+
+	GetReadVersionRequest batchReq;
+	batchReq.priority = TransactionPriority::BATCH;
+	batchReq.transactionCount = 5;
+	counts.add(batchReq);
+
+	ASSERT_EQ(counts.systemPriority, 2);
+	ASSERT_EQ(counts.defaultPriority, 3);
+	ASSERT_EQ(counts.batchPriority, 5);
+	ASSERT_EQ(counts.normalRateQueuedTransactions(), 5);
+	ASSERT_EQ(counts.batchRateQueuedTransactions(), 10);
+
+	counts.remove(defaultReq);
+	ASSERT_EQ(counts.defaultPriority, 0);
+	ASSERT_EQ(counts.normalRateQueuedTransactions(), 2);
+
+	counts.add(TransactionPriority::DEFAULT, 7);
+	counts.add(TransactionPriority::BATCH, 11);
+
+	ASSERT_EQ(counts.defaultPriority, 7);
+	ASSERT_EQ(counts.batchPriority, 16);
+	ASSERT_EQ(counts.normalRateQueuedTransactions(), 9);
+	ASSERT_EQ(counts.batchRateQueuedTransactions(), 25);
+
+	return Void();
+}
+
+TEST_CASE("/fdbserver/grvproxy/maxGrvQueueDelay/remainingDelayEstimate/table") {
+	struct Case {
+		std::string_view name;
+		TransactionPriority priority;
+		int64_t systemPriority;
+		int64_t defaultPriority;
+		int64_t batchPriority;
+		int64_t transactionCount;
+		bool expectBatchDelay;
+	};
+
+	std::array<Case, 6> cases{ {
+		{ "default under limit", TransactionPriority::DEFAULT, 4, 5, 50, 1, false },
+		{ "default exactly at limit", TransactionPriority::DEFAULT, 5, 4, 50, 1, false },
+		{ "default over limit", TransactionPriority::DEFAULT, 5, 5, 50, 1, false },
+		{ "batch under both limits", TransactionPriority::BATCH, 4, 5, 0, 1, true },
+		{ "batch over batch limit only", TransactionPriority::BATCH, 4, 5, 20, 1, true },
+		{ "batch over both limits", TransactionPriority::BATCH, 5, 5, 20, 1, true },
+	} };
+
+	for (auto const& c : cases) {
+		GrvTransactionRateInfo normalRateInfo = makeRateInfo();
+		GrvTransactionRateInfo batchRateInfo = makeRateInfo();
+		GrvQueueTransactionCounts counts;
+		counts.systemPriority = c.systemPriority;
+		counts.defaultPriority = c.defaultPriority;
+		counts.batchPriority = c.batchPriority;
+
+		auto estimate =
+		    estimateRemainingGrvQueueDelay(c.priority, c.transactionCount, counts, &normalRateInfo, &batchRateInfo);
+
+		assertNear(expectedEstimateDelay(c.systemPriority + c.defaultPriority, c.transactionCount),
+		           estimate.normalRateDelay);
+		ASSERT_EQ(estimate.batchRateDelay.present(), c.expectBatchDelay);
+		if (c.expectBatchDelay) {
+			assertNear(
+			    expectedEstimateDelay(c.systemPriority + c.defaultPriority + c.batchPriority, c.transactionCount),
+			    estimate.batchRateDelay.get());
+		}
+	}
+
+	return Void();
+}
+
+TEST_CASE("/fdbserver/grvproxy/maxGrvQueueDelay/rejectDecision/table") {
+	struct Case {
+		std::string_view name;
+		Optional<int64_t> maxDelayMS;
+		double requestTimeOffset;
+		double remainingDelay;
+		bool expectedReject;
+	};
+
+	std::array<Case, 7> cases{ {
+		{ "option absent", Optional<int64_t>(), 3600.0, std::numeric_limits<double>::infinity(), false },
+		{ "under threshold", Optional<int64_t>(100), 3600.0, 0.099, false },
+		{ "exactly at threshold", Optional<int64_t>(100), 3600.0, 0.100, false },
+		{ "over threshold", Optional<int64_t>(100), 3600.0, 0.101, true },
+		{ "negative elapsed is clamped", Optional<int64_t>(100), 3600.0, 0.050, false },
+		{ "elapsed queue time contributes", Optional<int64_t>(49), -0.010, 0.040, true },
+		{ "elapsed queue time under threshold", Optional<int64_t>(100), -0.010, 0.040, false },
+	} };
+
+	for (auto const& c : cases) {
+		GetReadVersionRequest req;
+		req.setRequestTime(now() + c.requestTimeOffset);
+		req.maxGrvQueueDelayMS = c.maxDelayMS;
+
+		ASSERT_EQ(shouldRejectForMaxGrvQueueDelay(req, c.remainingDelay), c.expectedReject);
+
+		double elapsedLowerBound = c.requestTimeOffset < 0.0 ? -c.requestTimeOffset : 0.0;
+		ASSERT_EQ(expectedShouldReject(c.maxDelayMS, elapsedLowerBound, c.remainingDelay), c.expectedReject);
+	}
+
+	return Void();
+}
diff --git a/fdbserver/GrvTransactionRateInfo.actor.cpp b/fdbserver/grvproxy/GrvTransactionRateInfo.cpp
similarity index 83%
rename from fdbserver/GrvTransactionRateInfo.actor.cpp
rename to fdbserver/grvproxy/GrvTransactionRateInfo.cpp
index 07ba1eaf50e..03df4da1b15 100644
--- a/fdbserver/GrvTransactionRateInfo.actor.cpp
+++ b/fdbserver/grvproxy/GrvTransactionRateInfo.cpp
@@ -1,5 +1,5 @@
 /*
- * GrvTransactionRateInfo.actor.cpp
+ * GrvTransactionRateInfo.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbserver/GrvTransactionRateInfo.h"
+#include "GrvTransactionRateInfo.h"
 
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // must be last include
 
 GrvTransactionRateInfo::GrvTransactionRateInfo(double rateWindow, double maxEmptyQueueBudget, double rate)
   : rateWindow(rateWindow), maxEmptyQueueBudget(maxEmptyQueueBudget), rate(rate), smoothRate(rateWindow),
@@ -35,6 +34,18 @@ bool GrvTransactionRateInfo::canStart(int64_t numAlreadyStarted, int64_t count)
 	       std::min(limit + budget, SERVER_KNOBS->START_TRANSACTION_MAX_TRANSACTIONS_TO_START);
 }
 
+double GrvTransactionRateInfo::estimateDelay(int64_t numAlreadyStarted, int64_t count) const {
+	if (canStart(numAlreadyStarted, count) || disabled) {
+		return 0.0;
+	}
+	if (rate <= 0.0) {
+		return std::numeric_limits<double>::infinity();
+	}
+	double capacity = std::min(limit + budget, double(SERVER_KNOBS->START_TRANSACTION_MAX_TRANSACTIONS_TO_START));
+	double deficit = numAlreadyStarted + count - capacity;
+	return std::max(0.0, deficit / rate);
+}
+
 void GrvTransactionRateInfo::endReleaseWindow(int64_t numStarted, bool queueEmpty, double elapsed) {
 	// Update the budget to accumulate any extra capacity available or remove any excess that was used.
 	// The actual delta is the portion of the limit we didn't use multiplied by the fraction of the rate window that
@@ -98,10 +109,10 @@ static bool isNear(double desired, int64_t actual) {
 	return std::abs(desired - actual) * 10 < desired;
 }
 
-ACTOR static Future<Void> mockClient(GrvTransactionRateInfo* rateInfo, double desiredRate, int64_t* counter) {
-	loop {
-		state double elapsed = (0.9 + 0.2 * deterministicRandom()->random01()) / desiredRate;
-		wait(delay(elapsed));
+static Future<Void> mockClient(GrvTransactionRateInfo* rateInfo, double desiredRate, int64_t* counter) {
+	while (true) {
+		double elapsed = (0.9 + 0.2 * deterministicRandom()->random01()) / desiredRate;
+		co_await delay(elapsed);
 		rateInfo->startReleaseWindow();
 		int started = rateInfo->canStart(0, 1) ? 1 : 0;
 		*counter += started;
@@ -112,11 +123,10 @@ ACTOR static Future<Void> mockClient(GrvTransactionRateInfo* rateInfo, double de
 // Rate limit set at 10, but client attempts 20 transactions per second.
 // Client should be throttled to only 10 transactions per second.
 TEST_CASE("/GrvTransactionRateInfo/Simple") {
-	state GrvTransactionRateInfo rateInfo(/*rateWindow=*/2.0, /*maxEmptyQueueBudget=*/100, /*rate=*/10);
-	state int64_t counter;
+	GrvTransactionRateInfo rateInfo(/*rateWindow=*/2.0, /*maxEmptyQueueBudget=*/100, /*rate=*/10);
+	int64_t counter = 0;
 	rateInfo.setRate(10.0);
-	wait(timeout(mockClient(&rateInfo, 20.0, &counter), 60.0, Void()));
+	co_await timeout(mockClient(&rateInfo, 20.0, &counter), 60.0, Void());
 	TraceEvent("GrvTransactionRateInfoTest").detail("Counter", counter);
 	ASSERT(isNear(60.0 * 10.0, counter));
-	return Void();
 }
diff --git a/fdbserver/include/fdbserver/GrvTransactionRateInfo.h b/fdbserver/grvproxy/GrvTransactionRateInfo.h
similarity index 92%
rename from fdbserver/include/fdbserver/GrvTransactionRateInfo.h
rename to fdbserver/grvproxy/GrvTransactionRateInfo.h
index 5a4dfdb70d0..b69c6f2d0e7 100644
--- a/fdbserver/include/fdbserver/GrvTransactionRateInfo.h
+++ b/fdbserver/grvproxy/GrvTransactionRateInfo.h
@@ -55,6 +55,10 @@ class GrvTransactionRateInfo {
 	// current release window.
 	bool canStart(int64_t numAlreadyStarted, int64_t count) const;
 
+	// Estimates how long it will take before canStart returns true for the
+	// same arguments. Returns zero when the request can start now.
+	double estimateDelay(int64_t numAlreadyStarted, int64_t count) const;
+
 	// Updates the budget to accumulate any extra capacity available or remove any excess that was used.
 	// Call at the end of a release window.
 	void endReleaseWindow(int64_t numStarted, bool queueEmpty, double elapsed);
diff --git a/fdbserver/grvproxy/include/fdbserver/grvproxy/GrvProxyServer.h b/fdbserver/grvproxy/include/fdbserver/grvproxy/GrvProxyServer.h
new file mode 100644
index 00000000000..0b6d2260307
--- /dev/null
+++ b/fdbserver/grvproxy/include/fdbserver/grvproxy/GrvProxyServer.h
@@ -0,0 +1,31 @@
+/*
+ * GrvProxyServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/GrvProxyInterface.h"
+#include "flow/flow.h"
+
+struct InitializeGrvProxyRequest;
+struct ServerDBInfo;
+
+Future<Void> grvProxyServer(GrvProxyInterface proxy,
+                            InitializeGrvProxyRequest req,
+                            Reference<AsyncVar<ServerDBInfo> const> db);
diff --git a/fdbserver/include/fdbserver/BulkLoadUtil.actor.h b/fdbserver/include/fdbserver/BulkLoadUtil.actor.h
deleted file mode 100644
index 4e64bbf8227..00000000000
--- a/fdbserver/include/fdbserver/BulkLoadUtil.actor.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * BulkLoadUtil.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <memory>
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_BULKLOADUTIL_ACTOR_G_H)
-#define FDBSERVER_BULKLOADUTIL_ACTOR_G_H
-#include "fdbserver/BulkLoadUtil.actor.g.h"
-#elif !defined(FDBSERVER_BULKLOADUTIL_ACTOR_H)
-#define FDBSERVER_BULKLOADUTIL_ACTOR_H
-#pragma once
-
-#include "fdbclient/BulkLoading.h"
-#include "flow/actorcompiler.h" // has to be last include
-
-// Erase file folder
-void clearFileFolder(const std::string& folderPath, const UID& logId = UID(), bool ignoreError = false);
-
-// Erase and recreate file folder
-void resetFileFolder(const std::string& folderPath);
-
-// Asynchronously copy file from one path to another.
-ACTOR Future<Void> copyBulkFile(std::string fromFile, std::string toFile, size_t fileBytesMax);
-
-// Asynchronously read file bytes from local file.
-ACTOR Future<Void> readBulkFileBytes(std::string path, int64_t maxLength, std::shared_ptr<std::string> output);
-
-// Asynchronously write file bytes to local file.
-ACTOR Future<Void> writeBulkFileBytes(std::string path, std::shared_ptr<std::string> content);
-
-// Get the bulkLoadTask metadata of the dataMoveMetadata since the atLeastVersion given the dataMoveId
-// This actor is stuck if the actor is failed to read the dataMoveMetadata.
-ACTOR Future<BulkLoadTaskState> getBulkLoadTaskStateFromDataMove(Database cx,
-                                                                 UID dataMoveId,
-                                                                 Version atLeastVersion,
-                                                                 UID logId);
-
-ACTOR Future<BulkLoadFileSet> bulkLoadDownloadTaskFileSet(BulkLoadTransportMethod transportMethod,
-                                                          BulkLoadFileSet fromRemoteFileSet,
-                                                          std::string toLocalRoot,
-                                                          UID logId);
-
-ACTOR Future<Void> bulkLoadDownloadTaskFileSets(BulkLoadTransportMethod transportMethod,
-                                                std::shared_ptr<BulkLoadFileSetKeyMap> fromRemoteFileSets,
-                                                std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets,
-                                                std::string toLocalRoot,
-                                                UID logId);
-
-ACTOR Future<bool> doBytesSamplingOnDataFile(std::string dataFileFullPath,
-                                             std::string byteSampleFileFullPath,
-                                             UID logId);
-
-// Download job manifest file which is generated when dumping the data
-ACTOR Future<Void> downloadBulkLoadJobManifestFile(BulkLoadTransportMethod transportMethod,
-                                                   std::string localJobManifestFilePath,
-                                                   std::string remoteJobManifestFilePath,
-                                                   UID logId);
-
-// Extract manifest entries from job manifest file with the input range
-ACTOR Future<KeyRange> getBulkLoadJobFileManifestEntryFromJobManifestFile(
-    std::string localJobManifestFilePath,
-    KeyRange range,
-    UID logId,
-    std::shared_ptr<BulkLoadManifestFileMap> manifestMap);
-
-// Get BulkLoad manifest metadata from the entry in the job manifest file
-ACTOR Future<BulkLoadManifestSet> getBulkLoadManifestMetadataFromEntry(
-    std::vector<BulkLoadJobFileManifestEntry> manifestEntries,
-    std::string manifestLocalTempFolder,
-    BulkLoadTransportMethod transportMethod,
-    std::string jobRoot,
-    UID logId);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/FDBExecHelper.actor.h b/fdbserver/include/fdbserver/FDBExecHelper.actor.h
deleted file mode 100644
index 4d21a840a37..00000000000
--- a/fdbserver/include/fdbserver/FDBExecHelper.actor.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * FDBExecHelper.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_EXEC_HELPER_ACTOR_G_H)
-#define FDBSERVER_EXEC_HELPER_ACTOR_G_H
-#include "fdbserver/FDBExecHelper.actor.g.h"
-#elif !defined(FDBSERVER_EXEC_HELPER_ACTOR_H)
-#define FDBSERVER_EXEC_HELPER_ACTOR_H
-
-#include <string>
-#include <vector>
-#include <map>
-#include "flow/Arena.h"
-#include "flow/flow.h"
-#include "fdbclient/FDBTypes.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// execute/snapshot command takes two arguments: <param1> <param2>
-// param1 - represents the command type/name
-// param2 - takes a binary path followed by a set of arguments in the following
-// format <binary-path>:<key1=val1>,<key2=val2>...
-// this class will abstract the format and give functions to get various pieces
-// of information
-class ExecCmdValueString {
-public: // ctor & dtor
-	ExecCmdValueString() {}
-	explicit ExecCmdValueString(StringRef cmdValueString);
-
-public: // interfaces
-	StringRef getBinaryPath() const;
-	VectorRef<StringRef> getBinaryArgs() const;
-	void setCmdValueString(StringRef cmdValueString);
-	StringRef getCmdValueString(void) const;
-
-public: // helper functions
-	void dbgPrint() const;
-
-private: // functions
-	void parseCmdValue();
-
-private: // data
-	Standalone<StringRef> cmdValueString;
-	Standalone<VectorRef<StringRef>> binaryArgs;
-	StringRef binaryPath;
-};
-
-class IClosable; // Forward declaration
-
-// FIXME: move this function to a common location
-// spawns a process pointed by `binPath` and the arguments provided at `paramList`,
-// if the process spawned takes more than `maxWaitTime` then it will be killed, if `maxWaitTime` < 0, then there won't
-// be timeout if isSync is set to true then the process will be synchronously executed if async and in simulator then
-// delay spawning the process to max of maxSimDelayTime
-ACTOR Future<int> spawnProcess(std::string binPath,
-                               std::vector<std::string> paramList,
-                               double maxWaitTime,
-                               bool isSync,
-                               double maxSimDelayTime,
-                               IClosable* parent = nullptr);
-
-// helper to run all the work related to running the exec command
-ACTOR Future<int> execHelper(ExecCmdValueString* execArg, UID snapUID, std::string folder, std::string role);
-
-// set the data version for the specified storage server UID
-void setDataVersion(UID uid, Version version);
-// set the data durable version for the specified storage server UID
-void setDataDurableVersion(UID uid, Version version);
-// print the version info all the storages servers on this node
-void printStorageVersionInfo();
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/GrvProxyTagThrottler.h b/fdbserver/include/fdbserver/GrvProxyTagThrottler.h
deleted file mode 100644
index 7b5f4a7f4de..00000000000
--- a/fdbserver/include/fdbserver/GrvProxyTagThrottler.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * GrvProxyTagThrottler.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "fdbclient/CommitProxyInterface.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbserver/GrvTransactionRateInfo.h"
-#include "fdbserver/LatencyBandsMap.h"
-
-// GrvProxyTagThrottler is used to throttle GetReadVersionRequests based on tag quotas
-// before they're pushed into priority-partitioned queues.
-//
-// A GrvTransactionRateInfo object and a request queue are maintained for each tag.
-// The GrvTransactionRateInfo object is used to determine when a request can be released.
-//
-// Between each set of waits, releaseTransactions is run, releasing queued transactions
-// that have passed the tag throttling stage. Transactions that are not yet ready
-// are requeued during releaseTransactions.
-class GrvProxyTagThrottler {
-	class DelayedRequest {
-		static uint64_t lastSequenceNumber;
-		double startTime;
-
-	public:
-		GetReadVersionRequest req;
-		uint64_t sequenceNumber;
-
-		explicit DelayedRequest(GetReadVersionRequest const& req)
-		  : req(req), startTime(now()), sequenceNumber(++lastSequenceNumber) {}
-
-		void updateProxyTagThrottledDuration(LatencyBandsMap&);
-		bool isMaxThrottled(double maxThrottleDuration) const;
-	};
-
-	struct TagQueue {
-		Optional<GrvTransactionRateInfo> rateInfo;
-		Deque<DelayedRequest> requests;
-
-		TagQueue() = default;
-		explicit TagQueue(double rate)
-		  : rateInfo(GrvTransactionRateInfo(SERVER_KNOBS->TAG_THROTTLE_RATE_WINDOW,
-		                                    SERVER_KNOBS->TAG_THROTTLE_MAX_EMPTY_QUEUE_BUDGET,
-		                                    rate)) {}
-
-		void setRate(double rate);
-		bool isMaxThrottled(double maxThrottleDuration) const;
-		void rejectRequests(LatencyBandsMap&);
-		void endReleaseWindow(int64_t numStarted, double elapsed);
-	};
-
-	// Track the budgets for each tag
-	TransactionTagMap<TagQueue> queues;
-	double maxThrottleDuration;
-
-	// Track latency bands for each tag
-	LatencyBandsMap latencyBandsMap;
-
-public:
-	explicit GrvProxyTagThrottler(double maxThrottleDuration);
-
-	// Called with rates received from ratekeeper
-	void updateRates(TransactionTagMap<double> const& newRates);
-
-	struct ReleaseTransactionsResult {
-		uint64_t batchPriorityTransactionsReleased{ 0 };
-		uint64_t batchPriorityRequestsReleased{ 0 };
-		uint64_t defaultPriorityTransactionsReleased{ 0 };
-		uint64_t defaultPriorityRequestsReleased{ 0 };
-		uint64_t rejectedRequests{ 0 };
-	};
-
-	// elapsed indicates the amount of time since the last epoch was run.
-	// If a request is ready to be executed, it is sent to the deque
-	// corresponding to its priority. If not, the request remains queued.
-	ReleaseTransactionsResult releaseTransactions(double elapsed,
-	                                              Deque<GetReadVersionRequest>& outBatchPriority,
-	                                              Deque<GetReadVersionRequest>& outDefaultPriority);
-
-	void addRequest(GetReadVersionRequest const&);
-
-	void addLatencyBandThreshold(double value);
-
-public: // testing
-	// Returns number of tags tracked
-	uint32_t size() const;
-};
diff --git a/fdbserver/include/fdbserver/IKeyValueStore.h b/fdbserver/include/fdbserver/IKeyValueStore.h
deleted file mode 100644
index bbec115e44e..00000000000
--- a/fdbserver/include/fdbserver/IKeyValueStore.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * IKeyValueStore.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_IKEYVALUESTORE_H
-#define FDBSERVER_IKEYVALUESTORE_H
-#pragma once
-
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/IKeyValueStore.actor.h"
-#include "flow/BooleanParam.h"
-
-extern IKeyValueStore* keyValueStoreSQLite(std::string const& filename,
-                                           UID logID,
-                                           KeyValueStoreType storeType,
-                                           bool checkChecksums = false,
-                                           bool checkIntegrity = false);
-extern IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename,
-                                              UID logID,
-                                              Reference<AsyncVar<struct ServerDBInfo> const> db = {},
-                                              int64_t pageCacheBytes = 0);
-extern IKeyValueStore* keyValueStoreRocksDB(std::string const& path,
-                                            UID logID,
-                                            KeyValueStoreType storeType,
-                                            bool checkChecksums = false,
-                                            bool checkIntegrity = false);
-extern IKeyValueStore* keyValueStoreShardedRocksDB(std::string const& path,
-                                                   UID logID,
-                                                   KeyValueStoreType storeType,
-                                                   bool checkChecksums = false,
-                                                   bool checkIntegrity = false);
-extern IKeyValueStore* keyValueStoreMemory(std::string const& basename,
-                                           UID logID,
-                                           int64_t memoryLimit,
-                                           std::string ext = "fdq",
-                                           KeyValueStoreType storeType = KeyValueStoreType::MEMORY);
-extern IKeyValueStore* keyValueStoreLogSystem(class IDiskQueue* queue,
-                                              Reference<AsyncVar<struct ServerDBInfo> const> db,
-                                              UID logID,
-                                              int64_t memoryLimit,
-                                              bool disableSnapshot,
-                                              bool replaceContent,
-                                              bool exactRecovery);
-
-extern IKeyValueStore* openRemoteKVStore(KeyValueStoreType storeType,
-                                         std::string const& filename,
-                                         UID logID,
-                                         int64_t memoryLimit,
-                                         bool checkChecksums = false,
-                                         bool checkIntegrity = false);
-
-IKeyValueStore* openKVStore(KeyValueStoreType storeType,
-                            std::string const& filename,
-                            UID logID,
-                            int64_t memoryLimit,
-                            bool checkChecksums = false,
-                            bool checkIntegrity = false,
-                            bool openRemotely = false,
-                            Reference<AsyncVar<struct ServerDBInfo> const> db = {},
-                            int64_t pageCacheBytes = 0);
-
-void GenerateIOLogChecksumFile(std::string filename);
-Future<Void> KVFileCheck(std::string const& filename, bool const& integrity);
-Future<Void> KVFileDump(std::string const& filename);
-
-#endif
diff --git a/fdbserver/include/fdbserver/KmsConnector.h b/fdbserver/include/fdbserver/KmsConnector.h
deleted file mode 100644
index 931924fa127..00000000000
--- a/fdbserver/include/fdbserver/KmsConnector.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * KmsConnector.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef KMS_CONNECTOR_H
-#define KMS_CONNECTOR_H
-#pragma once
-
-#include "fdbserver/KmsConnectorInterface.h"
-#include "flow/Arena.h"
-#include "flow/EncryptUtils.h"
-
-// FDB encryption needs to interact with external Key Management Services (KMS) solutions to lookup/refresh encryption
-// keys. KmsConnector interface is an abstract interface enabling implementing specialized KMS connector
-// implementations.
-// FDB KMSConnector implementation should inherit from KmsConnector and implement pure virtual function,
-// EncryptKeyProxyServer instantiates desired implementation object based on SERVER_KNOB->KMS_CONNECTOR_TYPE knob.
-
-class KmsConnector : public NonCopyable {
-public:
-	KmsConnector(const std::string& conStr) : connectorStr(conStr) {}
-	virtual ~KmsConnector() {}
-
-	virtual Future<Void> connectorCore(struct KmsConnectorInterface interf) = 0;
-
-	std::string getConnectorStr() const { return connectorStr; }
-
-protected:
-	std::string connectorStr;
-};
-
-#endif
diff --git a/fdbserver/include/fdbserver/LatencyBandsMap.h b/fdbserver/include/fdbserver/LatencyBandsMap.h
deleted file mode 100644
index acfaa5b4c32..00000000000
--- a/fdbserver/include/fdbserver/LatencyBandsMap.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * LatencyBandsMap.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbrpc/Stats.h"
-#include "fdbserver/Knobs.h"
-
-class LatencyBandsMap {
-	friend class LatencyBandsMapImpl;
-
-	std::string name;
-	UID id;
-	double loggingInterval;
-	int maxSize;
-	Future<Void> expireOldTags;
-
-	struct ExpirableBands {
-		LatencyBands latencyBands;
-		double lastUpdated;
-
-		explicit ExpirableBands(LatencyBands&&);
-	};
-
-	TransactionTagMap<ExpirableBands> map;
-	// Manually added thresholds (does not include "infinite" threshold automatically
-	// added by LatencyBands)
-	std::vector<double> thresholds;
-
-	// Get or create an LatencyBands object stored in map.
-	// Updates the lastUpdated field corresponding to this LatencyBands object.
-	// Returns pointer to this object, or an empty optional if object
-	// cannot be created.
-	Optional<LatencyBands*> getLatencyBands(TransactionTag tag);
-
-public:
-	LatencyBandsMap(std::string const& name, UID id, double loggingInterval, int maxSize);
-
-	void addMeasurement(TransactionTag tag, double measurement, int count = 1);
-	void addThreshold(double value);
-	int size() const { return map.size(); }
-};
diff --git a/fdbserver/include/fdbserver/LogSystem.h b/fdbserver/include/fdbserver/LogSystem.h
deleted file mode 100644
index 7eb6d6a6a83..00000000000
--- a/fdbserver/include/fdbserver/LogSystem.h
+++ /dev/null
@@ -1,943 +0,0 @@
-/*
- * LogSystem.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_LOGSYSTEM_H
-#define FDBSERVER_LOGSYSTEM_H
-
-#include <cstdint>
-#include <set>
-#include <vector>
-
-#include "fdbclient/DatabaseConfiguration.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/Replication.h"
-#include "fdbrpc/ReplicationPolicy.h"
-#include "fdbserver/LogSystemConfig.h"
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/OTELSpanContextMessage.h"
-#include "fdbserver/SpanContextMessage.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "flow/Arena.h"
-#include "flow/Error.h"
-#include "flow/Histogram.h"
-#include "flow/IndexedSet.h"
-#include "flow/Knobs.h"
-
-struct DBCoreState;
-struct TLogSet;
-struct CoreTLogSet;
-struct LogPushData;
-struct LocalityData;
-
-struct ConnectionResetInfo : public ReferenceCounted<ConnectionResetInfo> {
-	double lastReset;
-	Future<Void> resetCheck;
-	int slowReplies;
-	int fastReplies;
-
-	ConnectionResetInfo() : lastReset(now()), resetCheck(Void()), slowReplies(0), fastReplies(0) {}
-};
-
-// The set of tLog servers, logRouters and backupWorkers for a log tag
-class LogSet : NonCopyable, public ReferenceCounted<LogSet> {
-public:
-	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> logServers;
-	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> logRouters;
-	std::vector<Reference<AsyncVar<OptionalInterface<BackupInterface>>>> backupWorkers;
-	std::vector<Reference<ConnectionResetInfo>> connectionResetTrackers;
-	std::vector<Reference<Histogram>> tlogPushDistTrackers;
-	int32_t tLogWriteAntiQuorum;
-	int32_t tLogReplicationFactor;
-	std::vector<LocalityData> tLogLocalities; // Stores the localities of the log servers
-	TLogVersion tLogVersion;
-	Reference<IReplicationPolicy> tLogPolicy;
-	Reference<LocalitySet> logServerSet;
-	std::vector<int> logIndexArray;
-	std::vector<LocalityEntry> logEntryArray;
-	bool isLocal; // true if the LogSet is in primary DC or primary DC's satellite
-	int8_t locality;
-	Version startVersion;
-	std::vector<Future<TLogLockResult>> replies;
-	std::vector<std::vector<int>> satelliteTagLocations;
-
-	LogSet()
-	  : tLogWriteAntiQuorum(0), tLogReplicationFactor(0), isLocal(true), locality(tagLocalityInvalid),
-	    startVersion(invalidVersion) {}
-	LogSet(const TLogSet& tlogSet);
-	LogSet(const CoreTLogSet& coreSet);
-
-	std::string logRouterString();
-
-	bool hasLogRouter(UID id) const;
-
-	bool hasBackupWorker(UID id) const;
-
-	std::string logServerString();
-
-	void populateSatelliteTagLocations(int logRouterTags, int oldLogRouterTags, int txsTags, int oldTxsTags);
-
-	void checkSatelliteTagLocations();
-
-	int bestLocationFor(Tag tag);
-
-	void updateLocalitySet(std::vector<LocalityData> const& localities);
-
-	bool satisfiesPolicy(const std::vector<LocalityEntry>& locations);
-
-	void getPushLocations(
-	    VectorRef<Tag> tags,
-	    std::vector<int>& locations,
-	    int locationOffset,
-	    bool allLocations = false,
-	    const Optional<Reference<LocalitySet>>& restrictedLogSet = Optional<Reference<LocalitySet>>());
-
-private:
-	std::vector<LocalityEntry> alsoServers, resultEntries;
-	std::vector<int> newLocations;
-};
-
-struct ILogSystem {
-	// Represents a particular (possibly provisional) epoch of the log subsystem
-
-	struct IPeekCursor {
-		// clones the peek cursor, however you cannot call getMore() on the cloned cursor.
-		virtual Reference<IPeekCursor> cloneNoMore() = 0;
-
-		virtual void setProtocolVersion(ProtocolVersion version) = 0;
-
-		// if hasMessage() returns true, getMessage(), getMessageWithTags(), or reader() can be called.
-		// does not modify the cursor
-		virtual bool hasMessage() const = 0;
-
-		// pre: only callable if hasMessage() returns true
-		// return the tags associated with the message for the current sequence
-		virtual VectorRef<Tag> getTags() const = 0;
-
-		// pre: only callable if hasMessage() returns true
-		// returns the arena containing the contents of getMessage(), getMessageWithTags(), and reader()
-		virtual Arena& arena() = 0;
-
-		// pre: only callable if hasMessage() returns true
-		// returns an arena reader for the next message
-		// caller cannot call getMessage(), getMessageWithTags(), and reader()
-		// the caller must advance the reader before calling nextMessage()
-		virtual ArenaReader* reader() = 0;
-
-		// pre: only callable if hasMessage() returns true
-		// caller cannot call getMessage(), getMessageWithTags(), and reader()
-		// return the contents of the message for the current sequence
-		virtual StringRef getMessage() = 0;
-
-		// pre: only callable if hasMessage() returns true
-		// caller cannot call getMessage(), getMessageWithTags(), and reader()
-		// return the contents of the message for the current sequence
-		virtual StringRef getMessageWithTags() = 0;
-
-		// pre: only callable after getMessage(), getMessageWithTags(), or reader()
-		// post: hasMessage() and version() have been updated
-		// hasMessage() will never return false "in the middle" of a version (that is, if it does return false,
-		// version().subsequence will be zero)  < FIXME: Can we lose this property?
-		virtual void nextMessage() = 0;
-
-		// advances the cursor to the supplied LogMessageVersion, and updates hasMessage
-		virtual void advanceTo(LogMessageVersion n) = 0;
-
-		// returns immediately if hasMessage() returns true.
-		// returns when either the result of hasMessage() or version() has changed, or a cursor has internally been
-		// exhausted.
-		virtual Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) = 0;
-
-		// returns when the failure monitor detects that the servers associated with the cursor are failed
-		virtual Future<Void> onFailed() const = 0;
-
-		// returns false if:
-		// (1) the failure monitor detects that the servers associated with the cursor is failed
-		// (2) the interface is not present
-		// (3) the cursor cannot return any more results
-		virtual bool isActive() const = 0;
-
-		// returns true if the cursor cannot return any more results
-		virtual bool isExhausted() const = 0;
-
-		// Returns the smallest possible message version which the current message (if any) or a subsequent message
-		// might have (If hasMessage(), this is therefore the message version of the current message)
-		virtual const LogMessageVersion& version() const = 0;
-
-		// So far, the cursor has returned all messages which both satisfy the criteria passed to peek() to create the
-		// cursor AND have (popped(),0) <= message version number <= version() Other messages might have been skipped
-		virtual Version popped() const = 0;
-
-		// Returns the maximum version known to have been pushed (not necessarily durably) into the log system (0 is
-		// always a possible result!)
-		virtual Version getMaxKnownVersion() const { return 0; }
-
-		virtual Version getMinKnownCommittedVersion() const = 0;
-
-		virtual Optional<UID> getPrimaryPeekLocation() const = 0;
-
-		virtual Optional<UID> getCurrentPeekLocation() const = 0;
-
-		virtual void addref() = 0;
-
-		virtual void delref() = 0;
-	};
-
-	struct ServerPeekCursor final : IPeekCursor, ReferenceCounted<ServerPeekCursor> {
-		Reference<AsyncVar<OptionalInterface<TLogInterface>>> interf;
-		const Tag tag;
-
-		TLogPeekReply results;
-		ArenaReader rd;
-		LogMessageVersion messageVersion,
-		    end; // the version of current message; the intended end version of current cursor
-		Version poppedVersion;
-		TagsAndMessage messageAndTags;
-		bool hasMsg;
-		Future<Void> more;
-		UID randomID;
-		bool returnIfBlocked;
-
-		bool onlySpilled;
-		bool parallelGetMore;
-		bool usePeekStream;
-		int sequence;
-		Deque<Future<TLogPeekReply>> futureResults;
-		Future<Void> interfaceChanged;
-		Optional<ReplyPromiseStream<TLogPeekStreamReply>> peekReplyStream;
-
-		double lastReset;
-		Future<Void> resetCheck;
-		int slowReplies;
-		int fastReplies;
-		int unknownReplies;
-
-		bool returnEmptyIfStopped; // the source log server can return an empty version range if this flag is set; valid
-		                           // only when version vector/unicast is enabled
-
-		ServerPeekCursor(Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf,
-		                 Tag tag,
-		                 Version begin,
-		                 Version end,
-		                 bool returnIfBlocked,
-		                 bool parallelGetMore,
-		                 bool returnEmtpyIfStopped = false);
-		ServerPeekCursor(TLogPeekReply const& results,
-		                 LogMessageVersion const& messageVersion,
-		                 LogMessageVersion const& end,
-		                 TagsAndMessage const& message,
-		                 bool hasMsg,
-		                 Version poppedVersion,
-		                 Tag tag);
-
-		Reference<IPeekCursor> cloneNoMore() override;
-		void setProtocolVersion(ProtocolVersion version) override;
-		Arena& arena() override;
-		ArenaReader* reader() override;
-		bool hasMessage() const override;
-		void nextMessage() override;
-		StringRef getMessage() override;
-		StringRef getMessageWithTags() override;
-		VectorRef<Tag> getTags() const override;
-		void advanceTo(LogMessageVersion n) override;
-		Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
-		Future<Void> onFailed() const override;
-		bool isActive() const override;
-		bool isExhausted() const override;
-		const LogMessageVersion& version() const override;
-		Version popped() const override;
-		Version getMinKnownCommittedVersion() const override;
-		Optional<UID> getPrimaryPeekLocation() const override;
-		Optional<UID> getCurrentPeekLocation() const override;
-
-		void addref() override { ReferenceCounted<ServerPeekCursor>::addref(); }
-
-		void delref() override { ReferenceCounted<ServerPeekCursor>::delref(); }
-
-		Version getMaxKnownVersion() const override { return results.maxKnownVersion; }
-	};
-
-	struct MergedPeekCursor final : IPeekCursor, ReferenceCounted<MergedPeekCursor> {
-		Reference<LogSet> logSet;
-		std::vector<Reference<IPeekCursor>> serverCursors;
-		std::vector<LocalityEntry> locations;
-		std::vector<std::pair<LogMessageVersion, int>> sortedVersions;
-		Tag tag;
-		int bestServer, currentCursor, readQuorum;
-		Optional<LogMessageVersion> nextVersion;
-		LogMessageVersion messageVersion;
-		bool hasNextMessage;
-		UID randomID;
-		int tLogReplicationFactor;
-		Future<Void> more;
-
-		MergedPeekCursor(std::vector<Reference<ILogSystem::IPeekCursor>> const& serverCursors, Version begin);
-		MergedPeekCursor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
-		                 int bestServer,
-		                 int readQuorum,
-		                 Tag tag,
-		                 Version begin,
-		                 Version end,
-		                 bool parallelGetMore,
-		                 std::vector<LocalityData> const& tLogLocalities,
-		                 Reference<IReplicationPolicy> const tLogPolicy,
-		                 int tLogReplicationFactor,
-		                 const Optional<std::vector<uint16_t>>& knownLockedTLogIds = Optional<std::vector<uint16_t>>());
-		MergedPeekCursor(std::vector<Reference<IPeekCursor>> const& serverCursors,
-		                 LogMessageVersion const& messageVersion,
-		                 int bestServer,
-		                 int readQuorum,
-		                 Optional<LogMessageVersion> nextVersion,
-		                 Reference<LogSet> logSet,
-		                 int tLogReplicationFactor);
-
-		Reference<IPeekCursor> cloneNoMore() override;
-		void setProtocolVersion(ProtocolVersion version) override;
-		Arena& arena() override;
-		ArenaReader* reader() override;
-		void calcHasMessage();
-		void updateMessage(bool usePolicy);
-		bool hasMessage() const override;
-		void nextMessage() override;
-		StringRef getMessage() override;
-		StringRef getMessageWithTags() override;
-		VectorRef<Tag> getTags() const override;
-		void advanceTo(LogMessageVersion n) override;
-		Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
-		Future<Void> onFailed() const override;
-		bool isActive() const override;
-		bool isExhausted() const override;
-		const LogMessageVersion& version() const override;
-		Version popped() const override;
-		Version getMinKnownCommittedVersion() const override;
-		Optional<UID> getPrimaryPeekLocation() const override;
-		Optional<UID> getCurrentPeekLocation() const override;
-
-		void addref() override { ReferenceCounted<MergedPeekCursor>::addref(); }
-
-		void delref() override { ReferenceCounted<MergedPeekCursor>::delref(); }
-	};
-
-	struct SetPeekCursor final : IPeekCursor, ReferenceCounted<SetPeekCursor> {
-		std::vector<Reference<LogSet>> logSets;
-		std::vector<std::vector<Reference<IPeekCursor>>> serverCursors;
-		Tag tag;
-		int bestSet, bestServer, currentSet, currentCursor;
-		std::vector<LocalityEntry> locations;
-		std::vector<std::pair<LogMessageVersion, int>> sortedVersions;
-		Optional<LogMessageVersion> nextVersion;
-		LogMessageVersion messageVersion;
-		bool hasNextMessage;
-		bool useBestSet;
-		UID randomID;
-		Future<Void> more;
-		Optional<Version> end;
-
-		SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
-		              int bestSet,
-		              int bestServer,
-		              Tag tag,
-		              Version begin,
-		              Version end,
-		              bool parallelGetMore,
-		              const Optional<std::vector<uint16_t>>& knownLockedTLogIds = Optional<std::vector<uint16_t>>());
-		SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
-		              std::vector<std::vector<Reference<IPeekCursor>>> const& serverCursors,
-		              LogMessageVersion const& messageVersion,
-		              int bestSet,
-		              int bestServer,
-		              Optional<LogMessageVersion> nextVersion,
-		              bool useBestSet);
-
-		Reference<IPeekCursor> cloneNoMore() override;
-		void setProtocolVersion(ProtocolVersion version) override;
-		Arena& arena() override;
-		ArenaReader* reader() override;
-		void calcHasMessage();
-		void updateMessage(int logIdx, bool usePolicy);
-		bool hasMessage() const override;
-		void nextMessage() override;
-		StringRef getMessage() override;
-		StringRef getMessageWithTags() override;
-		VectorRef<Tag> getTags() const override;
-		void advanceTo(LogMessageVersion n) override;
-		Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
-		Future<Void> onFailed() const override;
-		bool isActive() const override;
-		bool isExhausted() const override;
-		const LogMessageVersion& version() const override;
-		Version popped() const override;
-		Version getMinKnownCommittedVersion() const override;
-		Optional<UID> getPrimaryPeekLocation() const override;
-		Optional<UID> getCurrentPeekLocation() const override;
-
-		void addref() override { ReferenceCounted<SetPeekCursor>::addref(); }
-
-		void delref() override { ReferenceCounted<SetPeekCursor>::delref(); }
-	};
-
-	struct MultiCursor final : IPeekCursor, ReferenceCounted<MultiCursor> {
-		std::vector<Reference<IPeekCursor>> cursors;
-		std::vector<LogMessageVersion> epochEnds;
-		Version poppedVersion;
-
-		MultiCursor(std::vector<Reference<IPeekCursor>> cursors, std::vector<LogMessageVersion> epochEnds);
-
-		Reference<IPeekCursor> cloneNoMore() override;
-		void setProtocolVersion(ProtocolVersion version) override;
-		Arena& arena() override;
-		ArenaReader* reader() override;
-		bool hasMessage() const override;
-		void nextMessage() override;
-		StringRef getMessage() override;
-		StringRef getMessageWithTags() override;
-		VectorRef<Tag> getTags() const override;
-		void advanceTo(LogMessageVersion n) override;
-		Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
-		Future<Void> onFailed() const override;
-		bool isActive() const override;
-		bool isExhausted() const override;
-		const LogMessageVersion& version() const override;
-		Version popped() const override;
-		Version getMinKnownCommittedVersion() const override;
-		Optional<UID> getPrimaryPeekLocation() const override;
-		Optional<UID> getCurrentPeekLocation() const override;
-
-		void addref() override { ReferenceCounted<MultiCursor>::addref(); }
-
-		void delref() override { ReferenceCounted<MultiCursor>::delref(); }
-	};
-
-	struct BufferedCursor final : IPeekCursor, ReferenceCounted<BufferedCursor> {
-		struct BufferedMessage {
-			Arena arena;
-			StringRef message;
-			VectorRef<Tag> tags;
-			LogMessageVersion version;
-
-			BufferedMessage() {}
-			explicit BufferedMessage(Version version) : version(version) {}
-			BufferedMessage(Arena arena,
-			                StringRef message,
-			                const VectorRef<Tag>& tags,
-			                const LogMessageVersion& version)
-			  : arena(arena), message(message), tags(tags), version(version) {}
-
-			bool operator<(BufferedMessage const& r) const { return version < r.version; }
-
-			bool operator==(BufferedMessage const& r) const { return version == r.version; }
-		};
-
-		std::vector<Reference<IPeekCursor>> cursors;
-		std::vector<Deque<BufferedMessage>> cursorMessages;
-		std::vector<BufferedMessage> messages;
-		int messageIndex;
-		LogMessageVersion messageVersion;
-		Version end;
-		bool hasNextMessage;
-		bool withTags;
-		bool knownUnique;
-		Version minKnownCommittedVersion;
-		Version poppedVersion;
-		Version initialPoppedVersion;
-		bool canDiscardPopped;
-		Future<Void> more;
-		int targetQueueSize;
-		UID randomID;
-
-		BufferedCursor(std::vector<Reference<IPeekCursor>> cursors,
-		               Version begin,
-		               Version end,
-		               bool withTags,
-		               bool canDiscardPopped);
-		BufferedCursor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
-		               Tag tag,
-		               Version begin,
-		               Version end,
-		               bool parallelGetMore);
-
-		Reference<IPeekCursor> cloneNoMore() override;
-		void setProtocolVersion(ProtocolVersion version) override;
-		Arena& arena() override;
-		ArenaReader* reader() override;
-		bool hasMessage() const override;
-		void nextMessage() override;
-		StringRef getMessage() override;
-		StringRef getMessageWithTags() override;
-		VectorRef<Tag> getTags() const override;
-		void advanceTo(LogMessageVersion n) override;
-		Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
-		Future<Void> onFailed() const override;
-		bool isActive() const override;
-		bool isExhausted() const override;
-		const LogMessageVersion& version() const override;
-		Version popped() const override;
-		Version getMinKnownCommittedVersion() const override;
-		Optional<UID> getPrimaryPeekLocation() const override;
-		Optional<UID> getCurrentPeekLocation() const override;
-
-		void addref() override { ReferenceCounted<BufferedCursor>::addref(); }
-
-		void delref() override { ReferenceCounted<BufferedCursor>::delref(); }
-	};
-
-	virtual void addref() = 0;
-	virtual void delref() = 0;
-
-	virtual std::string describe() const = 0;
-	virtual UID getDebugID() const = 0;
-
-	virtual void toCoreState(DBCoreState&) const = 0;
-
-	virtual bool remoteStorageRecovered() const = 0;
-
-	virtual void purgeOldRecoveredGenerationsCoreState(DBCoreState&) = 0;
-	virtual void purgeOldRecoveredGenerationsInMemory(const DBCoreState&) = 0;
-
-	virtual Future<Void> onCoreStateChanged() const = 0;
-	// Returns if and when the output of toCoreState() would change (for example, when older logs can be discarded from
-	// the state)
-
-	virtual void coreStateWritten(DBCoreState const& newState) = 0;
-	// Called when a core state has been written to the coordinators
-
-	virtual Future<Void> onError() const = 0;
-	// Never returns normally, but throws an error if the subsystem stops working
-
-	struct PushVersionSet {
-		Version prevVersion;
-		Version version;
-		Version knownCommittedVersion;
-		Version minKnownCommittedVersion;
-	};
-
-	virtual Future<Version> push(const PushVersionSet& verisonSet,
-	                             LogPushData& data,
-	                             SpanContext const& spanContext,
-	                             Optional<UID> debugID = Optional<UID>(),
-	                             Optional<std::unordered_map<uint16_t, Version>> tpcvMap =
-	                                 Optional<std::unordered_map<uint16_t, Version>>()) = 0;
-	// Waits for the version number of the bundle (in this epoch) to be prevVersion (i.e. for all pushes ordered
-	// earlier) Puts the given messages into the bundle, each with the given tags, and with message versions (version,
-	// 0) - (version, N) Changes the version number of the bundle to be version (unblocking the next push) Returns when
-	// the preceding changes are durable.  (Later we will need multiple return signals for different durability levels)
-	// If the current epoch has ended, push will not return, and the pushed messages will not be visible in any
-	// subsequent epoch (but may become visible in this epoch)
-
-	virtual Reference<IPeekCursor> peek(UID dbgid,
-	                                    Version begin,
-	                                    Optional<Version> end,
-	                                    Tag tag,
-	                                    bool parallelGetMore = false) = 0;
-	// Returns (via cursor interface) a stream of messages with the given tag and message versions >= (begin, 0),
-	// ordered by message version If pop was previously or concurrently called with upTo > begin, the cursor may not
-	// return all such messages.  In that case cursor->popped() will be greater than begin to reflect that.
-
-	virtual Reference<IPeekCursor> peek(UID dbgid,
-	                                    Version begin,
-	                                    Optional<Version> end,
-	                                    std::vector<Tag> tags,
-	                                    bool parallelGetMore = false) = 0;
-	// Same contract as peek(), but for a set of tags
-
-	virtual Reference<IPeekCursor> peekSingle(
-	    UID dbgid,
-	    Version begin,
-	    Tag tag,
-	    std::vector<std::pair<Version, Tag>> history = std::vector<std::pair<Version, Tag>>()) = 0;
-	// Same contract as peek(), but blocks until the preferred log server(s) for the given tag are available (and is
-	// correspondingly less expensive)
-
-	virtual Reference<IPeekCursor> peekLogRouter(
-	    UID dbgid,
-	    Version begin,
-	    Tag tag,
-	    bool useSatellite,
-	    Optional<Version> end = Optional<Version>(),
-	    const Optional<std::map<uint8_t, std::vector<uint16_t>>>& knownStoppedTLogIds =
-	        Optional<std::map<uint8_t, std::vector<uint16_t>>>()) = 0;
-	// Same contract as peek(), but can only peek from the logs elected in the same generation.
-	// If the preferred log server is down, a different log from the same generation will merge results locally before
-	// sending them to the log router.
-
-	virtual Reference<IPeekCursor> peekTxs(UID dbgid,
-	                                       Version begin,
-	                                       int8_t peekLocality,
-	                                       Version localEnd,
-	                                       bool canDiscardPopped) = 0;
-	// Same contract as peek(), but only for peeking the txsLocality. It allows specifying a preferred peek locality.
-
-	virtual Future<Version> getTxsPoppedVersion() = 0;
-
-	virtual Version getKnownCommittedVersion() = 0;
-
-	virtual Future<Void> onKnownCommittedVersionChange() = 0;
-
-	virtual void popTxs(Version upTo, int8_t popLocality = tagLocalityInvalid) = 0;
-
-	virtual void pop(Version upTo,
-	                 Tag tag,
-	                 Version knownCommittedVersion = 0,
-	                 int8_t popLocality = tagLocalityInvalid) = 0;
-	// Permits, but does not require, the log subsystem to strip `tag` from any or all messages with message versions <
-	// (upTo,0) The popping of any given message may be arbitrarily delayed.
-
-	virtual Future<Void> confirmEpochLive(Optional<UID> debugID = Optional<UID>()) = 0;
-	// Returns success after confirming that pushes in the current epoch are still possible
-
-	virtual Future<Void> endEpoch() = 0;
-	// Ends the current epoch without starting a new one
-
-	static Reference<ILogSystem> fromServerDBInfo(
-	    UID const& dbgid,
-	    ServerDBInfo const& db,
-	    bool useRecoveredAt = false,
-	    Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>());
-	static Reference<ILogSystem> fromLogSystemConfig(
-	    UID const& dbgid,
-	    LocalityData const&,
-	    LogSystemConfig const&,
-	    bool excludeRemote = false,
-	    bool useRecoveredAt = false,
-	    Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>());
-	// Constructs a new ILogSystem implementation from the given ServerDBInfo/LogSystemConfig.  Might return a null
-	// reference if there isn't a fully recovered log system available. The caller can peek() the returned log system
-	// and can push() if it has version numbers reserved for it and prevVersions
-
-	static Reference<ILogSystem> fromOldLogSystemConfig(UID const& dbgid, LocalityData const&, LogSystemConfig const&);
-	// Constructs a new ILogSystem implementation from the old log data within a ServerDBInfo/LogSystemConfig.  Might
-	// return a null reference if there isn't a fully recovered log system available.
-
-	static Future<Void> recoverAndEndEpoch(Reference<AsyncVar<Reference<ILogSystem>>> const& outLogSystem,
-	                                       UID const& dbgid,
-	                                       DBCoreState const& oldState,
-	                                       FutureStream<TLogRejoinRequest> const& rejoins,
-	                                       LocalityData const& locality,
-	                                       bool* forceRecovery);
-	// Constructs a new ILogSystem implementation based on the given oldState and rejoining log servers
-	// Ensures that any calls to push or confirmEpochLive in the current epoch but strictly later than change_epoch will
-	// not return Whenever changes in the set of available log servers require restarting recovery with a different end
-	// sequence, outLogSystem will be changed to a new ILogSystem
-
-	virtual Version getEnd() const = 0;
-	// Call only on an ILogSystem obtained from recoverAndEndEpoch()
-	// Returns the first unreadable version number of the recovered epoch (i.e. message version numbers < (get_end(), 0)
-	// will be readable)
-
-	// Returns the start version of current epoch for backup workers.
-	virtual Version getBackupStartVersion() const = 0;
-
-	struct EpochTagsVersionsInfo {
-		int32_t logRouterTags; // Number of log router tags.
-		Version epochBegin, epochEnd;
-
-		explicit EpochTagsVersionsInfo(int32_t n, Version begin, Version end)
-		  : logRouterTags(n), epochBegin(begin), epochEnd(end) {}
-	};
-
-	// Returns EpochTagVersionsInfo for old epochs that this log system is aware of, excluding the current epoch.
-	virtual std::map<LogEpoch, EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const = 0;
-
-	virtual Future<Reference<ILogSystem>> newEpoch(
-	    RecruitFromConfigurationReply const& recr,
-	    Future<struct RecruitRemoteFromConfigurationReply> const& fRemoteWorkers,
-	    DatabaseConfiguration const& config,
-	    LogEpoch recoveryCount,
-	    Version recoveryTransactionVersion,
-	    int8_t primaryLocality,
-	    int8_t remoteLocality,
-	    std::vector<Tag> const& allTags,
-	    Reference<AsyncVar<bool>> const& recruitmentStalled) = 0;
-	// Call only on an ILogSystem obtained from recoverAndEndEpoch()
-	// Returns an ILogSystem representing a new epoch immediately following this one.  The new epoch is only provisional
-	// until the caller updates the coordinated DBCoreState
-
-	// Returns the physical configuration of this LogSystem, that could be used to construct an equivalent LogSystem
-	// using fromLogSystemConfig()
-	virtual LogSystemConfig getLogSystemConfig() const = 0;
-
-	// Returns the type of LogSystem, this should be faster than using RTTI
-	virtual LogSystemType getLogSystemType() const = 0;
-
-	virtual Standalone<StringRef> getLogsValue() const = 0;
-
-	// Returns when the log system configuration has changed due to a tlog rejoin.
-	virtual Future<Void> onLogSystemConfigChange() = 0;
-
-	// Update a specific log router in the log system configuration
-	virtual void updateLogRouter(int logSetIndex, int tagId, TLogInterface const& newLogRouter) = 0;
-
-	virtual void getPushLocations(VectorRef<Tag> tags,
-	                              std::vector<int>& locations,
-	                              bool allLocations = false,
-	                              Optional<std::vector<Reference<LocalitySet>>> fromLocations =
-	                                  Optional<std::vector<Reference<LocalitySet>>>()) const = 0;
-
-	void getPushLocations(
-	    std::vector<Tag> const& tags,
-	    std::vector<int>& locations,
-	    bool allLocations = false,
-	    Optional<std::vector<Reference<LocalitySet>>> fromLocations = Optional<std::vector<Reference<LocalitySet>>>()) {
-		getPushLocations(VectorRef<Tag>((Tag*)&tags.front(), tags.size()), locations, allLocations, fromLocations);
-	}
-
-	virtual std::vector<Reference<LocalitySet>> getPushLocationsForTags(std::vector<int>& fromLocations) const = 0;
-
-	virtual bool hasRemoteLogs() const = 0;
-
-	virtual Tag getRandomRouterTag() const = 0;
-	virtual int getLogRouterTags() const = 0; // Returns the number of router tags.
-
-	virtual Tag getRandomTxsTag() const = 0;
-
-	// Returns the TLogVersion of the current generation of TLogs.
-	// (This only exists because getLogSystemConfig is a significantly more expensive call.)
-	virtual TLogVersion getTLogVersion() const = 0;
-
-	virtual void stopRejoins() = 0;
-
-	// XXX: Should Tag related functions stay inside TagPartitionedLogSystem??
-	// Returns the pseudo tag to be popped for the given process class. If the
-	// process class doesn't use pseudo tag, return the same tag.
-	virtual Tag getPseudoPopTag(Tag tag, ProcessClass::ClassType type) const = 0;
-
-	virtual bool hasPseudoLocality(int8_t locality) const = 0;
-
-	// Returns the actual version to be popped from the log router tag for the given pseudo tag.
-	// For instance, a pseudo tag (-8, 2) means the actual popping tag is (-2, 2). Assuming there
-	// are multiple pseudo tags, the returned version is the min(all pseudo tags' "upTo" versions).
-	virtual Version popPseudoLocalityTag(Tag tag, Version upTo) = 0;
-
-	virtual void setBackupWorkers(const std::vector<InitializeBackupReply>& replies) = 0;
-
-	// Removes a finished backup worker from log system and returns true. Returns false
-	// if the worker is not found.
-	virtual bool removeBackupWorker(const BackupWorkerDoneRequest& req) = 0;
-
-	virtual LogEpoch getOldestBackupEpoch() const = 0;
-	virtual void setOldestBackupEpoch(LogEpoch epoch) = 0;
-};
-
-struct LengthPrefixedStringRef {
-	// Represents a pointer to a string which is prefixed by a 4-byte length
-	// A LengthPrefixedStringRef is only pointer-sized (8 bytes vs 12 bytes for StringRef), but the corresponding string
-	// is 4 bytes bigger, and substring operations aren't efficient as they are with StringRef.  It's a good choice when
-	// there might be lots of references to the same exact string.
-
-	uint32_t* length;
-
-	StringRef toStringRef() const {
-		ASSERT(length);
-		return StringRef((uint8_t*)(length + 1), *length);
-	}
-	int expectedSize() const {
-		ASSERT(length);
-		return *length;
-	}
-	uint32_t* getLengthPtr() const { return length; }
-
-	LengthPrefixedStringRef() : length(nullptr) {}
-	LengthPrefixedStringRef(uint32_t* length) : length(length) {}
-};
-
-// Structure to store serialized mutations sent from the proxy to the
-// transaction logs. The serialization repeats with the following format:
-//
-// +----------------------+ +----------------------+ +----------+ +----------------+         +----------------------+
-// |     Message size     | |      Subsequence     | | # of tags| |      Tag       | . . . . |       Mutation       |
-// +----------------------+ +----------------------+ +----------+ +----------------+         +----------------------+
-// <------- 32 bits ------> <------- 32 bits ------> <- 16 bits-> <---- 24 bits --->         <---- variable bits --->
-//
-// `Mutation` can be a serialized MutationRef or a special metadata message
-// such as LogProtocolMessage or SpanContextMessage. The type of `Mutation` is
-// uniquely identified by its first byte -- a value from MutationRef::Type.
-//
-struct LogPushData : NonCopyable {
-	// Log subsequences have to start at 1 (the MergedPeekCursor relies on this to make sure we never have !hasMessage()
-	// in the middle of data for a version
-
-	explicit LogPushData(Reference<ILogSystem> logSystem, int tlogCount);
-
-	void addTxsTag();
-
-	// addTag() adds a tag for the *next* message to be added
-	void addTag(Tag tag) { next_message_tags.push_back(tag); }
-
-	template <class T>
-	void addTags(T tags) {
-		next_message_tags.insert(next_message_tags.end(), tags.begin(), tags.end());
-	}
-
-	// Add transaction info to be written before the first mutation in the transaction.
-	void addTransactionInfo(SpanContext const& context);
-
-	// copy written_tags, after filtering, into given set
-	void saveTags(std::set<Tag>& filteredTags) const {
-		for (const auto& tag : written_tags) {
-			filteredTags.insert(tag);
-		}
-	}
-
-	void addWrittenTags(const std::set<Tag>& tags) { written_tags.insert(tags.begin(), tags.end()); }
-
-	void getLocations(const std::set<Tag>& tags, std::set<uint16_t>& writtenTLogs) {
-		std::vector<Tag> vtags(tags.begin(), tags.end());
-		std::vector<int> msg_locations;
-		logSystem->getPushLocations(vtags, msg_locations, false /*allLocations*/);
-		writtenTLogs.insert(msg_locations.begin(), msg_locations.end());
-	}
-
-	// store tlogs as represented by index
-	void saveLocations(std::set<uint16_t>& writtenTLogs) {
-		writtenTLogs.insert(msg_locations.begin(), msg_locations.end());
-	}
-
-	void setLogsChanged() { logsChanged = true; }
-	bool haveLogsChanged() const { return logsChanged; }
-
-	void writeMessage(StringRef rawMessageWithoutLength, bool usePreviousLocations);
-
-	void setPushLocationsForTags(std::vector<int> fromLocationsVec) {
-		fromLocations = logSystem->getPushLocationsForTags(fromLocationsVec);
-	}
-
-	template <class T>
-	void writeTypedMessage(T const& item, bool metadataMessage = false, bool allLocations = false);
-
-	Standalone<StringRef> getMessages(int loc) const { return messagesWriter[loc].toValue(); }
-
-	// Returns all locations' messages, including empty ones.
-	std::vector<Standalone<StringRef>> getAllMessages() const;
-
-	// Records if a tlog (specified by "loc") will receive an empty version batch message.
-	// "value" is the message returned by getMessages() call.
-	void recordEmptyMessage(int loc, const Standalone<StringRef>& value);
-
-	// Returns the ratio of empty messages in this version batch.
-	// MUST be called after getMessages() and recordEmptyMessage().
-	float getEmptyMessageRatio() const;
-
-	// Returns the total number of mutations. Subsequence is initialized to 1, so subtract 1 to get count.
-	uint32_t getMutationCount() const { return subsequence - 1; }
-
-	// Sets mutations for all internal writers. "mutations" is the output from
-	// getAllMessages() and is used before writing any other mutations.
-	void setMutations(uint32_t totalMutations, VectorRef<StringRef> mutations);
-
-	Optional<Tag> savedRandomRouterTag;
-	void storeRandomRouterTag() { savedRandomRouterTag = logSystem->getRandomRouterTag(); }
-	int getLogRouterTags() { return logSystem->getLogRouterTags(); }
-
-private:
-	Reference<ILogSystem> logSystem;
-	std::vector<Tag> next_message_tags;
-	std::vector<Tag> prev_tags;
-	std::set<Tag> written_tags;
-	std::vector<BinaryWriter> messagesWriter;
-	std::vector<bool> messagesWritten; // if messagesWriter has written anything
-	std::vector<int> msg_locations;
-	Optional<std::vector<Reference<LocalitySet>>> fromLocations;
-	// Stores message locations that have had span information written to them
-	// for the current transaction. Adding transaction info will reset this
-	// field.
-	std::unordered_set<int> writtenLocations;
-	uint32_t subsequence;
-	SpanContext spanContext;
-	bool logsChanged = false; // if keyServers has any changes, i.e., shard boundary modifications.
-
-	// Writes transaction info to the message stream at the given location if
-	// it has not already been written (for the current transaction). Returns
-	// true on a successful write, and false if the location has already been
-	// written.
-	bool writeTransactionInfo(int location, uint32_t subseq);
-
-	Tag chooseRouterTag() {
-		return savedRandomRouterTag.present() ? savedRandomRouterTag.get() : logSystem->getRandomRouterTag();
-	}
-};
-
-template <class T>
-void LogPushData::writeTypedMessage(T const& item, bool metadataMessage, bool allLocations) {
-	prev_tags.clear();
-	if (logSystem->hasRemoteLogs()) {
-		prev_tags.push_back(chooseRouterTag());
-	}
-	for (auto& tag : next_message_tags) {
-		prev_tags.push_back(tag);
-	}
-	msg_locations.clear();
-	logSystem->getPushLocations(prev_tags, msg_locations, allLocations, fromLocations);
-
-	BinaryWriter bw(AssumeVersion(g_network->protocolVersion()));
-
-	// Metadata messages (currently LogProtocolMessage is the only metadata
-	// message) should be written before span information. If this isn't a
-	// metadata message, make sure all locations have had transaction info
-	// written to them.  Mutations may have different sets of tags, so it
-	// is necessary to check all tag locations each time a mutation is
-	// written.
-	if (!metadataMessage) {
-		uint32_t subseq = this->subsequence++;
-		bool updatedLocation = false;
-		for (int loc : msg_locations) {
-			updatedLocation = writeTransactionInfo(loc, subseq) || updatedLocation;
-		}
-		// If this message doesn't write to any new locations, the
-		// subsequence wasn't actually used and can be decremented.
-		if (!updatedLocation) {
-			this->subsequence--;
-			CODE_PROBE(true, "No new SpanContextMessage written to transaction logs");
-			ASSERT(this->subsequence > 0);
-		}
-	} else {
-		// When writing a metadata message, make sure transaction state has
-		// been reset. If you are running into this assertion, make sure
-		// you are calling addTransactionInfo before each transaction.
-		ASSERT(writtenLocations.size() == 0);
-	}
-
-	uint32_t subseq = this->subsequence++;
-	bool first = true;
-	int firstOffset = -1, firstLength = -1;
-	for (int loc : msg_locations) {
-		BinaryWriter& wr = messagesWriter[loc];
-
-		if (first) {
-			firstOffset = wr.getLength();
-			wr << uint32_t(0) << subseq << uint16_t(prev_tags.size());
-			for (auto& tag : prev_tags)
-				wr << tag;
-			wr << item;
-			firstLength = wr.getLength() - firstOffset;
-			*(uint32_t*)((uint8_t*)wr.getData() + firstOffset) = firstLength - sizeof(uint32_t);
-			DEBUG_TAGS_AND_MESSAGE(
-			    "ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength))
-			    .detail("PushLocations", msg_locations);
-			first = false;
-		} else {
-			BinaryWriter& from = messagesWriter[msg_locations[0]];
-			wr.serializeBytes((uint8_t*)from.getData() + firstOffset, firstLength);
-		}
-	}
-	written_tags.insert(next_message_tags.begin(), next_message_tags.end());
-	next_message_tags.clear();
-}
-
-#endif // FDBSERVER_LOGSYSTEM_H
diff --git a/fdbserver/include/fdbserver/NetworkTest.h b/fdbserver/include/fdbserver/NetworkTest.h
index 1725fe54b49..7678e5c8ec8 100644
--- a/fdbserver/include/fdbserver/NetworkTest.h
+++ b/fdbserver/include/fdbserver/NetworkTest.h
@@ -30,15 +30,15 @@ struct NetworkTestInterface {
 	RequestStream<struct NetworkTestRequest> test;
 	RequestStream<struct NetworkTestStreamingRequest> testStream;
 	NetworkTestInterface() {}
-	NetworkTestInterface(NetworkAddress remote);
-	NetworkTestInterface(INetwork* local);
+	explicit NetworkTestInterface(NetworkAddress remote);
+	explicit NetworkTestInterface(INetwork* local);
 };
 
 struct NetworkTestReply {
 	constexpr static FileIdentifier file_identifier = 14465374;
 	Value value;
 	NetworkTestReply() {}
-	NetworkTestReply(Value value) : value(value) {}
+	explicit NetworkTestReply(Value value) : value(value) {}
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, value);
diff --git a/fdbserver/include/fdbserver/QuietDatabase.h b/fdbserver/include/fdbserver/QuietDatabase.h
deleted file mode 100644
index 53ba2ef98e1..00000000000
--- a/fdbserver/include/fdbserver/QuietDatabase.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * QuietDatabase.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_QUIETDATABASE_H
-#define FDBSERVER_QUIETDATABASE_H
-#pragma once
-
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/DatabaseContext.h" // for clone()
-#include "fdbclient/FDBTypes.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-
-Future<int64_t> getDataInFlight(Database const& cx, Reference<AsyncVar<struct ServerDBInfo> const> const&);
-Future<std::pair<int64_t, int64_t>> getTLogQueueInfo(Database const& cx,
-                                                     Reference<AsyncVar<struct ServerDBInfo> const> const&);
-Future<int64_t> getMaxStorageServerQueueSize(Database const& cx,
-                                             Reference<AsyncVar<struct ServerDBInfo> const> const&,
-                                             Version const& version);
-Future<int64_t> getDataDistributionQueueSize(Database const& cx,
-                                             Reference<AsyncVar<struct ServerDBInfo> const> const&,
-                                             bool const& reportInFlight);
-Future<bool> getTeamCollectionValid(Database const& cx, WorkerInterface const&);
-Future<bool> getTeamCollectionValid(Database const& cx, Reference<AsyncVar<struct ServerDBInfo> const> const&);
-Future<std::vector<StorageServerInterface>> getStorageServers(Database const& cx,
-                                                              bool const& use_system_priority = false);
-Future<std::vector<WorkerDetails>> getWorkers(Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
-                                              int const& flags = 0);
-Future<WorkerInterface> getMasterWorker(Database const& cx, Reference<AsyncVar<ServerDBInfo> const> const& dbInfo);
-Future<Void> repairDeadDatacenter(Database const& cx,
-                                  Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
-                                  std::string const& context);
-Future<Void> reconfigureAfter(Database const& cx,
-                              double const& time,
-                              Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
-                              std::string const& context);
-
-// Returns list of worker interfaces for available storage servers and the number of unavailable
-// storage servers
-Future<std::pair<std::vector<WorkerInterface>, int>>
-getStorageWorkers(Database const& cx, Reference<AsyncVar<ServerDBInfo> const> const& dbInfo, bool const& localOnly);
-Future<std::vector<WorkerInterface>> getCoordWorkers(Database const& cx,
-                                                     Reference<AsyncVar<ServerDBInfo> const> const& dbInfo);
-
-Future<Void> enableConsistencyScanInSim(Database const& db);
-Future<Void> disableConsistencyScanInSim(Database const& db, bool const& waitForCompletion);
-
-#endif
diff --git a/fdbserver/include/fdbserver/RESTKmsConnectorUtils.h b/fdbserver/include/fdbserver/RESTKmsConnectorUtils.h
deleted file mode 100644
index e589086fdcb..00000000000
--- a/fdbserver/include/fdbserver/RESTKmsConnectorUtils.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * RESTKmsConnectorUtils.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_REST_KMS_CONNECTOR_UTILS_H
-#define FDBSERVER_REST_KMS_CONNECTOR_UTILS_H
-#pragma once
-
-#include "fdbrpc/HTTP.h"
-#include "flow/EncryptUtils.h"
-#include "flow/flow.h"
-
-#include <rapidjson/rapidjson.h>
-#include <rapidjson/document.h>
-#include <rapidjson/stringbuffer.h>
-#include <rapidjson/writer.h>
-
-namespace RESTKmsConnectorUtils {
-
-extern const char* BASE_CIPHER_ID_TAG;
-extern const char* BASE_CIPHER_TAG;
-extern const char* CIPHER_KEY_DETAILS_TAG;
-extern const char* ENCRYPT_DOMAIN_ID_TAG;
-extern const char* REFRESH_AFTER_SEC;
-extern const char* EXPIRE_AFTER_SEC;
-extern const char* ERROR_TAG;
-extern const char* ERROR_MSG_TAG;
-extern const char* ERROR_CODE_TAG;
-extern const char* KMS_URLS_TAG;
-extern const char* REFRESH_KMS_URLS_TAG;
-extern const char* REQUEST_VERSION_TAG;
-extern const char* VALIDATION_TOKENS_TAG;
-extern const char* VALIDATION_TOKEN_NAME_TAG;
-extern const char* VALIDATION_TOKEN_VALUE_TAG;
-extern const char* DEBUG_UID_TAG;
-
-extern const char* TOKEN_NAME_FILE_SEP;
-extern const char* TOKEN_TUPLE_SEP;
-extern const char DISCOVER_URL_FILE_URL_SEP;
-
-extern const char* BLOB_METADATA_DETAILS_TAG;
-extern const char* BLOB_METADATA_DOMAIN_ID_TAG;
-extern const char* BLOB_METADATA_LOCATIONS_TAG;
-extern const char* BLOB_METADATA_LOCATION_ID_TAG;
-extern const char* BLOB_METADATA_LOCATION_PATH_TAG;
-
-extern const int INVALID_REQUEST_VERSION;
-
-enum class ValidationTokenSource {
-	VALIDATION_TOKEN_SOURCE_FILE = 1,
-	VALIDATION_TOKEN_SOURCE_LAST // Always the last element
-};
-
-struct ErrorDetail {
-	std::string errorCode;
-	std::string errorMsg;
-
-	ErrorDetail() {}
-	ErrorDetail(const std::string& code, const std::string& msg) : errorCode(code), errorMsg(msg) {}
-
-	bool isEqual(const ErrorDetail& toCompare) const {
-		return errorCode.compare(toCompare.errorCode) == 0 && errorMsg.compare(toCompare.errorMsg) == 0;
-	}
-};
-
-struct ValidationTokenCtx {
-	std::string name;
-	std::string value;
-	ValidationTokenSource source;
-	Optional<std::string> filePath;
-
-	explicit ValidationTokenCtx(const std::string& n, ValidationTokenSource s)
-	  : name(n), value(""), source(s), filePath(Optional<std::string>()), readTS(now()) {}
-	double getReadTS() const { return readTS; }
-
-private:
-	double readTS; // Approach assists refreshing token based on time of creation
-};
-using ValidationTokenMap = std::unordered_map<std::string, ValidationTokenCtx>;
-
-HTTP::Headers getHTTPHeaders();
-
-void addVersionToDoc(rapidjson::Document& doc, const int requestVersion);
-void addLatestDomainDetailsToDoc(rapidjson::Document& doc,
-                                 const char* rootTagName,
-                                 const char* idTagName,
-                                 const EncryptCipherDomainIdVec& domainIds);
-void addBaseCipherIdDomIdToDoc(rapidjson::Document& doc,
-                               rapidjson::Value& keyIdDetails,
-                               const EncryptCipherBaseKeyId baseCipherId,
-                               const Optional<EncryptCipherDomainId> domainId);
-void addValidationTokensSectionToJsonDoc(rapidjson::Document& doc, const ValidationTokenMap& tokenMap);
-void addRefreshKmsUrlsSectionToJsonDoc(rapidjson::Document& doc, const bool refreshKmsUrls);
-void addDebugUidSectionToJsonDoc(rapidjson::Document& doc, Optional<UID> dbgId);
-
-Optional<ErrorDetail> getError(const rapidjson::Document& doc);
-
-} // namespace RESTKmsConnectorUtils
-
-#endif
diff --git a/fdbserver/include/fdbserver/RESTSimKmsVault.h b/fdbserver/include/fdbserver/RESTSimKmsVault.h
deleted file mode 100644
index 3a6c5d1bd41..00000000000
--- a/fdbserver/include/fdbserver/RESTSimKmsVault.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * RESTSimKmsVault.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef FDBSERVER_REST_SIM_KMS_VAULT_H
-#define FDBSERVER_REST_SIM_KMS_VAULT_H
-#pragma once
-
-#include "fdbrpc/HTTP.h"
-#include "fdbrpc/simulator.h"
-
-const std::string REST_SIM_KMS_VAULT_GET_ENCRYPTION_KEYS_BY_KEY_IDS_RESOURCE = "/get-encryption-keys-by-key-ids";
-const std::string REST_SIM_KMS_VAULT_GET_ENCRYPTION_KEYS_BY_DOMAIN_IDS_RESOURCE = "/get-encryption-keys-by-domain-ids";
-const std::string REST_SIM_KMS_VAULT_GET_BLOB_METADATA_RESOURCE = "/get-blob-metadata";
-
-struct RESTSimKmsVaultRequestHandler : HTTP::IRequestHandler, ReferenceCounted<RESTSimKmsVaultRequestHandler> {
-	Future<Void> handleRequest(Reference<HTTP::IncomingRequest> req,
-	                           Reference<HTTP::OutgoingResponse> response) override;
-	Reference<HTTP::IRequestHandler> clone() override { return makeReference<RESTSimKmsVaultRequestHandler>(); }
-
-	void addref() override { ReferenceCounted<RESTSimKmsVaultRequestHandler>::addref(); }
-	void delref() override { ReferenceCounted<RESTSimKmsVaultRequestHandler>::delref(); }
-};
-
-#endif
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h b/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h
deleted file mode 100644
index 701ef06dcac..00000000000
--- a/fdbserver/include/fdbserver/RemoteIKeyValueStore.actor.h
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * RemoteIKeyValueStore.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_G_H)
-#define FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_G_H
-#include "fdbserver/RemoteIKeyValueStore.actor.g.h"
-#elif !defined(FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_H)
-#define FDBSERVER_REMOTE_IKEYVALUESTORE_ACTOR_H
-
-#include "flow/ActorCollection.h"
-#include "flow/IRandom.h"
-#include "flow/Knobs.h"
-#include "flow/Trace.h"
-#include "flow/flow.h"
-#include "flow/network.h"
-#include "fdbrpc/FlowProcess.actor.h"
-#include "fdbrpc/FlowTransport.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-struct IKVSCommitReply {
-	constexpr static FileIdentifier file_identifier = 3958189;
-	StorageBytes storeBytes;
-
-	IKVSCommitReply() : storeBytes(0, 0, 0, 0) {}
-	IKVSCommitReply(const StorageBytes& sb) : storeBytes(sb) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, storeBytes);
-	}
-};
-
-struct RemoteKVSProcessInterface {
-
-	constexpr static FileIdentifier file_identifier = 3491838;
-	RequestStream<struct GetRemoteKVSProcessInterfaceRequest> getProcessInterface;
-	RequestStream<struct OpenKVStoreRequest> openKVStore;
-
-	UID uniqueID = deterministicRandom()->randomUniqueID();
-
-	UID id() const { return uniqueID; }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, getProcessInterface, openKVStore);
-	}
-};
-
-struct IKVSInterface {
-	constexpr static FileIdentifier file_identifier = 4929113;
-	RequestStream<struct IKVSGetValueRequest> getValue;
-	RequestStream<struct IKVSSetRequest> set;
-	RequestStream<struct IKVSClearRequest> clear;
-	RequestStream<struct IKVSCommitRequest> commit;
-	RequestStream<struct IKVSReadValuePrefixRequest> readValuePrefix;
-	RequestStream<struct IKVSReadRangeRequest> readRange;
-	RequestStream<struct IKVSGetStorageByteRequest> getStorageBytes;
-	RequestStream<struct IKVSGetErrorRequest> getError;
-	RequestStream<struct IKVSOnClosedRequest> onClosed;
-	RequestStream<struct IKVSDisposeRequest> dispose;
-	RequestStream<struct IKVSCloseRequest> close;
-
-	UID uniqueID;
-
-	UID id() const { return uniqueID; }
-
-	KeyValueStoreType storeType;
-
-	KeyValueStoreType type() const { return storeType; }
-
-	IKVSInterface() {}
-
-	IKVSInterface(KeyValueStoreType type) : uniqueID(deterministicRandom()->randomUniqueID()), storeType(type) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar,
-		           getValue,
-		           set,
-		           clear,
-		           commit,
-		           readValuePrefix,
-		           readRange,
-		           getStorageBytes,
-		           getError,
-		           onClosed,
-		           dispose,
-		           close,
-		           uniqueID);
-	}
-};
-
-struct GetRemoteKVSProcessInterfaceRequest {
-	constexpr static FileIdentifier file_identifier = 8382983;
-	ReplyPromise<struct RemoteKVSProcessInterface> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
-struct OpenKVStoreRequest {
-	constexpr static FileIdentifier file_identifier = 5918682;
-	KeyValueStoreType storeType;
-	std::string filename;
-	UID logID;
-	int64_t memoryLimit;
-	bool checkChecksums;
-	bool checkIntegrity;
-	ReplyPromise<struct IKVSInterface> reply;
-
-	OpenKVStoreRequest() {};
-
-	OpenKVStoreRequest(KeyValueStoreType storeType,
-	                   std::string filename,
-	                   UID logID,
-	                   int64_t memoryLimit,
-	                   bool checkChecksums = false,
-	                   bool checkIntegrity = false)
-	  : storeType(storeType), filename(filename), logID(logID), memoryLimit(memoryLimit),
-	    checkChecksums(checkChecksums), checkIntegrity(checkIntegrity) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity, reply);
-	}
-};
-
-struct IKVSGetValueRequest {
-	constexpr static FileIdentifier file_identifier = 1029439;
-	KeyRef key;
-	Optional<ReadOptions> options;
-	ReplyPromise<Optional<Value>> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, key, options, reply);
-	}
-};
-
-struct IKVSSetRequest {
-	constexpr static FileIdentifier file_identifier = 7283948;
-	KeyValueRef keyValue;
-	ReplyPromise<Void> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, keyValue, reply);
-	}
-};
-
-struct IKVSClearRequest {
-	constexpr static FileIdentifier file_identifier = 2838575;
-	KeyRangeRef range;
-	ReplyPromise<Void> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, range, reply);
-	}
-};
-
-struct IKVSCommitRequest {
-	constexpr static FileIdentifier file_identifier = 2985129;
-	bool sequential;
-	ReplyPromise<IKVSCommitReply> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, sequential, reply);
-	}
-};
-
-struct IKVSReadValuePrefixRequest {
-	constexpr static FileIdentifier file_identifier = 1928374;
-	KeyRef key;
-	int maxLength;
-	Optional<ReadOptions> options;
-	ReplyPromise<Optional<Value>> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, key, maxLength, options, reply);
-	}
-};
-
-// Use this instead of RangeResult as reply for better serialization performance
-struct IKVSReadRangeReply {
-	constexpr static FileIdentifier file_identifier = 6682449;
-	Arena arena;
-	VectorRef<KeyValueRef, VecSerStrategy::String> data;
-	bool more;
-	Optional<KeyRef> readThrough;
-	bool readToBegin;
-	bool readThroughEnd;
-
-	IKVSReadRangeReply() = default;
-
-	explicit IKVSReadRangeReply(const RangeResult& res)
-	  : arena(res.arena()), data(static_cast<const VectorRef<KeyValueRef>&>(res)), more(res.more),
-	    readThrough(res.readThrough), readToBegin(res.readToBegin), readThroughEnd(res.readThroughEnd) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, data, more, readThrough, readToBegin, readThroughEnd, arena);
-	}
-
-	RangeResult toRangeResult() const {
-		RangeResult r(RangeResultRef(data, more, readThrough), arena);
-		r.readToBegin = readToBegin;
-		r.readThroughEnd = readThroughEnd;
-		return r;
-	}
-};
-
-struct IKVSReadRangeRequest {
-	constexpr static FileIdentifier file_identifier = 5918394;
-	KeyRangeRef keys;
-	int rowLimit;
-	int byteLimit;
-	Optional<ReadOptions> options;
-	ReplyPromise<IKVSReadRangeReply> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, keys, rowLimit, byteLimit, options, reply);
-	}
-};
-
-struct IKVSGetStorageByteRequest {
-	constexpr static FileIdentifier file_identifier = 3512344;
-	ReplyPromise<StorageBytes> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
-struct IKVSGetErrorRequest {
-	constexpr static FileIdentifier file_identifier = 3942891;
-	ReplyPromise<Void> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
-struct IKVSOnClosedRequest {
-	constexpr static FileIdentifier file_identifier = 1923894;
-	ReplyPromise<Void> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
-struct IKVSDisposeRequest {
-	constexpr static FileIdentifier file_identifier = 1235952;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar);
-	}
-};
-
-struct IKVSCloseRequest {
-	constexpr static FileIdentifier file_identifier = 13859172;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar);
-	}
-};
-
-ACTOR Future<Void> runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterface);
-
-struct KeyValueStoreProcess : FlowProcess {
-	RemoteKVSProcessInterface kvsIf;
-	Standalone<StringRef> serializedIf;
-
-	Endpoint ssProcess; // endpoint for the storage process
-	RequestStream<FlowProcessRegistrationRequest> ssRequestStream;
-
-	KeyValueStoreProcess() {
-		TraceEvent(SevDebug, "InitKeyValueStoreProcess").log();
-		ObjectWriter writer(IncludeVersion());
-		writer.serialize(kvsIf);
-		serializedIf = writer.toString();
-	}
-
-	void registerEndpoint(Endpoint p) override {
-		ssProcess = p;
-		ssRequestStream = RequestStream<FlowProcessRegistrationRequest>(p);
-	}
-
-	StringRef name() const override { return _name; }
-	StringRef serializedInterface() const override { return serializedIf; }
-
-	ACTOR static Future<Void> _run(KeyValueStoreProcess* self) {
-		state ActorCollection actors(true);
-		TraceEvent("WaitingForOpenKVStoreRequest").log();
-		loop {
-			choose {
-				when(OpenKVStoreRequest req = waitNext(self->kvsIf.openKVStore.getFuture())) {
-					TraceEvent("OpenKVStoreRequestReceived").log();
-					IKVSInterface interf;
-					actors.add(runIKVS(req, interf));
-				}
-				when(ErrorOr<Void> e = wait(errorOr(actors.getResult()))) {
-					if (e.isError()) {
-						TraceEvent("KeyValueStoreProcessRunActorError").errorUnsuppressed(e.getError());
-						throw e.getError();
-					} else {
-						TraceEvent("KeyValueStoreProcessFinished").log();
-						return e.get();
-					}
-				}
-			}
-		}
-	}
-
-	Future<Void> run() override { return _run(this); }
-
-	static StringRef _name;
-};
-
-struct RemoteIKeyValueStore : public IKeyValueStore {
-	RemoteKVSProcessInterface kvsProcess;
-	IKVSInterface interf;
-	Future<Void> initialized;
-	Future<int> returnCode;
-	StorageBytes storageBytes;
-
-	RemoteIKeyValueStore() : storageBytes(0, 0, 0, 0) {}
-
-	Future<Void> init() override {
-		TraceEvent(SevInfo, "RemoteIKeyValueStoreInit").log();
-		return initialized;
-	}
-
-	Future<Void> getError() const override { return getErrorImpl(this, returnCode); }
-	Future<Void> onClosed() const override { return onCloseImpl(this); }
-
-	void dispose() override {
-		TraceEvent(SevDebug, "RemoteIKVSDisposeRequest").backtrace();
-		interf.dispose.send(IKVSDisposeRequest{});
-		// hold the future to not cancel the spawned process
-		uncancellable(returnCode);
-		delete this;
-	}
-	void close() override {
-		TraceEvent(SevDebug, "RemoteIKVSCloseRequest").backtrace();
-		interf.close.send(IKVSCloseRequest{});
-		// hold the future to not cancel the spawned process
-		uncancellable(returnCode);
-		delete this;
-	}
-
-	KeyValueStoreType getType() const override { return interf.type(); }
-
-	void set(KeyValueRef keyValue, const Arena* arena = nullptr) override {
-		interf.set.send(IKVSSetRequest{ keyValue, ReplyPromise<Void>() });
-	}
-	void clear(KeyRangeRef range, const Arena* arena = nullptr) override {
-		interf.clear.send(IKVSClearRequest{ range, ReplyPromise<Void>() });
-	}
-
-	Future<Void> commit(bool sequential = false) override {
-		Future<IKVSCommitReply> commitReply =
-		    interf.commit.getReply(IKVSCommitRequest{ sequential, ReplyPromise<IKVSCommitReply>() });
-		return commitAndGetStorageBytes(this, commitReply);
-	}
-
-	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options = Optional<ReadOptions>()) override {
-		return readValueImpl(this, IKVSGetValueRequest{ key, options, ReplyPromise<Optional<Value>>() });
-	}
-
-	Future<Optional<Value>> readValuePrefix(KeyRef key,
-	                                        int maxLength,
-	                                        Optional<ReadOptions> options = Optional<ReadOptions>()) override {
-		return interf.readValuePrefix.getReply(
-		    IKVSReadValuePrefixRequest{ key, maxLength, options, ReplyPromise<Optional<Value>>() });
-	}
-
-	Future<RangeResult> readRange(KeyRangeRef keys,
-	                              int rowLimit = 1 << 30,
-	                              int byteLimit = 1 << 30,
-	                              Optional<ReadOptions> options = Optional<ReadOptions>()) override {
-		IKVSReadRangeRequest req{ keys, rowLimit, byteLimit, options, ReplyPromise<IKVSReadRangeReply>() };
-		return fmap([](const IKVSReadRangeReply& reply) { return reply.toRangeResult(); },
-		            interf.readRange.getReply(req));
-	}
-
-	StorageBytes getStorageBytes() const override { return storageBytes; }
-
-	void consumeInterface(StringRef intf) {
-		kvsProcess = ObjectReader::fromStringRef<RemoteKVSProcessInterface>(intf, IncludeVersion());
-	}
-
-	ACTOR static Future<Void> commitAndGetStorageBytes(RemoteIKeyValueStore* self,
-	                                                   Future<IKVSCommitReply> commitReplyFuture) {
-		IKVSCommitReply commitReply = wait(commitReplyFuture);
-		self->storageBytes = commitReply.storeBytes;
-		return Void();
-	}
-
-	ACTOR static Future<Optional<Value>> readValueImpl(RemoteIKeyValueStore* self, IKVSGetValueRequest req) {
-		Optional<Value> val = wait(self->interf.getValue.getReply(req));
-		return val;
-	}
-
-	ACTOR static Future<Void> getErrorImpl(const RemoteIKeyValueStore* self, Future<int> returnCode) {
-		choose {
-			when(wait(self->initialized)) {}
-			when(wait(delay(SERVER_KNOBS->REMOTE_KV_STORE_MAX_INIT_DURATION))) {
-				TraceEvent(SevError, "RemoteIKVSInitTooLong")
-				    .detail("TimeLimit", SERVER_KNOBS->REMOTE_KV_STORE_MAX_INIT_DURATION);
-				throw please_reboot_kv_store(); // this will reboot the kv store
-			}
-		}
-		state Future<Void> connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY);
-		state Future<ErrorOr<Void>> storeError = errorOr(self->interf.getError.getReply(IKVSGetErrorRequest{}));
-		state NetworkAddress childAddr = self->interf.getError.getEndpoint().getPrimaryAddress();
-		loop choose {
-			when(ErrorOr<Void> e = wait(storeError)) {
-				TraceEvent(SevDebug, "RemoteIKVSGetError")
-				    .errorUnsuppressed(e.isError() ? e.getError() : success())
-				    .backtrace();
-				if (e.isError())
-					throw e.getError();
-				else
-					return Never();
-			}
-			when(int res = wait(returnCode)) {
-				TraceEvent(res != 0 ? SevError : SevInfo, "SpawnedProcessDied").detail("Res", res);
-				if (res)
-					throw please_reboot_kv_store(); // this will reboot the kv store
-				else
-					return Never();
-			}
-			when(wait(connectionCheckingDelay)) {
-				// for the corner case where the child process stuck and waitpid also does not give update on it
-				// In this scenario, we need to manually reboot the storage engine process
-				if (IFailureMonitor::failureMonitor().getState(childAddr).isFailed()) {
-					TraceEvent(SevError, "RemoteKVStoreConnectionStuck").log();
-					throw please_reboot_kv_store(); // this will reboot the kv store
-				}
-				connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY);
-			}
-		}
-	}
-
-	ACTOR static Future<Void> onCloseImpl(const RemoteIKeyValueStore* self) {
-		try {
-			wait(self->initialized);
-			wait(self->interf.onClosed.getReply(IKVSOnClosedRequest{}));
-			TraceEvent(SevDebug, "RemoteIKVSOnCloseImplOnClosedFinished");
-		} catch (Error& e) {
-			TraceEvent(SevInfo, "RemoteIKVSOnCloseImplError").errorUnsuppressed(e).backtrace();
-			throw;
-		}
-		return Void();
-	}
-};
-
-Future<Void> runFlowProcess(std::string const& name, Endpoint endpoint);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/RestoreApplier.actor.h b/fdbserver/include/fdbserver/RestoreApplier.actor.h
deleted file mode 100644
index 173e87569c3..00000000000
--- a/fdbserver/include/fdbserver/RestoreApplier.actor.h
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * RestoreApplier.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file declears RestoreApplier interface and actors
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_APPLIER_G_H)
-#define FDBSERVER_RESTORE_APPLIER_G_H
-#include "fdbserver/RestoreApplier.actor.g.h"
-#elif !defined(FDBSERVER_RESTORE_APPLIER_H)
-#define FDBSERVER_RESTORE_APPLIER_H
-
-#include <sstream>
-#include "fdbclient/Atomic.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/CommitTransaction.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/Stats.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-Value applyAtomicOp(Optional<StringRef> existingValue, Value value, MutationRef::Type type);
-
-// Key whose mutations are buffered on applier.
-// key, value, type and version defines the parsed mutation at version.
-// pendingMutations has all versioned mutations to be applied.
-// Mutations in pendingMutations whose version is below the version in StagingKey can be ignored in applying phase.
-struct StagingKey {
-	Key key; // TODO: Maybe not needed?
-	Value val;
-	MutationRef::Type type; // set or clear
-	LogMessageVersion version; // largest version of set or clear for the key
-	std::map<LogMessageVersion, Standalone<MutationRef>> pendingMutations; // mutations not set or clear type
-
-	explicit StagingKey(Key key) : key(key), type(MutationRef::MAX_ATOMIC_OP), version(0) {}
-
-	// Add mutation m at newVersion to stagingKey
-	// Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set
-	void add(const MutationRef& m, LogMessageVersion newVersion) {
-		ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue);
-		DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m)
-		    .detail("SubVersion", version.toString())
-		    .detail("NewSubVersion", newVersion.toString());
-		if (version == newVersion) {
-			// This could happen because the same mutation can be present in
-			// overlapping mutation logs, because new TLogs can copy mutations
-			// from old generation TLogs (or backup worker is recruited without
-			// knowning previously saved progress).
-			ASSERT(type == m.type && key == m.param1 && val == m.param2);
-			TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m);
-			return;
-		}
-
-		// newVersion can be smaller than version as different loaders can send
-		// mutations out of order.
-		if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
-			if (m.type == MutationRef::ClearRange) {
-				// We should only clear this key! Otherwise, it causes side effect to other keys
-				ASSERT(m.param1 == m.param2);
-			}
-			if (version < newVersion) {
-				DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m)
-				    .detail("SubVersion", version.toString())
-				    .detail("NewSubVersion", newVersion.toString())
-				    .detail("MType", getTypeString(type))
-				    .detail("Key", key)
-				    .detail("Val", val)
-				    .detail("NewMutation", m.toString());
-				key = m.param1;
-				val = m.param2;
-				type = (MutationRef::Type)m.type;
-				version = newVersion;
-			}
-		} else {
-			auto it = pendingMutations.find(newVersion);
-			if (it == pendingMutations.end()) {
-				pendingMutations.emplace(newVersion, m);
-			} else {
-				// Duplicated mutation ignored.
-				// TODO: Add SevError here
-				TraceEvent("SameVersion")
-				    .detail("Version", version.toString())
-				    .detail("NewVersion", newVersion.toString())
-				    .detail("OldMutation", it->second)
-				    .detail("NewMutation", m);
-				ASSERT(it->second.type == m.type && it->second.param1 == m.param1 && it->second.param2 == m.param2);
-			}
-		}
-	}
-
-	// Precompute the final value of the key.
-	// TODO: Look at the last LogMessageVersion, if it set or clear, we can ignore the rest of versions.
-	void precomputeResult(const char* context, UID applierID, int batchIndex) {
-		TraceEvent(SevFRMutationInfo, "FastRestoreApplierPrecomputeResult", applierID)
-		    .detail("BatchIndex", batchIndex)
-		    .detail("Context", context)
-		    .detail("Version", version.toString())
-		    .detail("Key", key)
-		    .detail("Value", val)
-		    .detail("MType", type < MutationRef::MAX_ATOMIC_OP ? getTypeString(type) : "[Unset]")
-		    .detail("LargestPendingVersion",
-		            (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString()))
-		    .detail("PendingMutations", pendingMutations.size());
-		std::map<LogMessageVersion, Standalone<MutationRef>>::iterator lb = pendingMutations.lower_bound(version);
-		if (lb == pendingMutations.end()) {
-			return;
-		}
-		ASSERT(!pendingMutations.empty());
-		if (lb->first == version) {
-			// Sanity check mutations at version are either atomicOps which can be ignored or the same value as buffered
-			MutationRef m = lb->second;
-			if (m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) {
-				if (std::tie(type, key, val) != std::tie(m.type, m.param1, m.param2)) {
-					TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnhandledSituation", applierID)
-					    .detail("BatchIndex", batchIndex)
-					    .detail("Context", context)
-					    .detail("BufferedType", getTypeString(type))
-					    .detail("PendingType", getTypeString(m.type))
-					    .detail("BufferedVal", val.toString())
-					    .detail("PendingVal", m.param2.toString());
-				}
-			}
-			lb++;
-		}
-		for (; lb != pendingMutations.end(); lb++) {
-			MutationRef mutation = lb->second;
-			if (mutation.type == MutationRef::CompareAndClear) { // Special atomicOp
-				Arena arena;
-				Optional<StringRef> inputVal;
-				if (hasBaseValue()) {
-					inputVal = val;
-				}
-				Optional<ValueRef> retVal = doCompareAndClear(inputVal, mutation.param2, arena);
-				if (!retVal.present()) {
-					val = key;
-					type = MutationRef::ClearRange;
-				} // else no-op
-			} else if (isAtomicOp((MutationRef::Type)mutation.type)) {
-				Optional<StringRef> inputVal;
-				if (hasBaseValue()) {
-					inputVal = val;
-				}
-				val = applyAtomicOp(inputVal, mutation.param2, (MutationRef::Type)mutation.type);
-				type = MutationRef::SetValue; // Precomputed result should be set to DB.
-			} else if (mutation.type == MutationRef::SetValue || mutation.type == MutationRef::ClearRange) {
-				type = MutationRef::SetValue;
-				TraceEvent(SevError, "FastRestoreApplierPrecomputeResultUnexpectedSet", applierID)
-				    .detail("BatchIndex", batchIndex)
-				    .detail("Context", context)
-				    .detail("MutationType", getTypeString(mutation.type))
-				    .detail("Version", lb->first.toString());
-			} else {
-				TraceEvent(SevError, "FastRestoreApplierPrecomputeResultSkipUnexpectedBackupMutation", applierID)
-				    .detail("BatchIndex", batchIndex)
-				    .detail("Context", context)
-				    .detail("MutationType", getTypeString(mutation.type))
-				    .detail("Version", lb->first.toString());
-			}
-			ASSERT(lb->first > version);
-			version = lb->first;
-		}
-	}
-
-	// Does the key has at least 1 set or clear mutation to get the base value
-	bool hasBaseValue() const {
-		if (version.version > 0) {
-			ASSERT(type == MutationRef::SetValue || type == MutationRef::ClearRange);
-		}
-		return version.version > 0;
-	}
-
-	// Has all pendingMutations been pre-applied to the val?
-	bool hasPrecomputed() const {
-		ASSERT(pendingMutations.empty() || pendingMutations.rbegin()->first >= pendingMutations.begin()->first);
-		return pendingMutations.empty() || version >= pendingMutations.rbegin()->first;
-	}
-
-	int totalSize() const { return MutationRef::OVERHEAD_BYTES + key.size() + val.size(); }
-};
-
-// The range mutation received on applier.
-// Range mutations should be applied both to the destination DB and to the StagingKeys
-struct StagingKeyRange {
-	Standalone<MutationRef> mutation;
-	LogMessageVersion version;
-
-	explicit StagingKeyRange(MutationRef m, LogMessageVersion newVersion) : mutation(m), version(newVersion) {}
-
-	bool operator<(const StagingKeyRange& rhs) const {
-		return std::tie(version, mutation.type, mutation.param1, mutation.param2) <
-		       std::tie(rhs.version, rhs.mutation.type, rhs.mutation.param1, rhs.mutation.param2);
-	}
-};
-
-// Applier state in each version batch
-class ApplierVersionBatchState : RoleVersionBatchState {
-public:
-	static const int NOT_INIT = 0;
-	static const int INIT = 1;
-	static const int RECEIVE_MUTATIONS = 2;
-	static const int WRITE_TO_DB = 3;
-	static const int DONE = 4;
-	static const int INVALID = 5;
-
-	explicit ApplierVersionBatchState(int newState) { vbState = newState; }
-
-	~ApplierVersionBatchState() override = default;
-
-	void operator=(int newState) override { vbState = newState; }
-
-	int get() const override { return vbState; }
-};
-
-struct ApplierBatchData : public ReferenceCounted<ApplierBatchData> {
-	// processedFileState: key: RestoreAsset; value: largest version of mutation received on the applier
-	std::map<RestoreAsset, NotifiedVersion> processedFileState;
-	Optional<Future<Void>> dbApplier;
-	VersionedMutationsMap kvOps; // Mutations at each version
-	std::map<Key, StagingKey> stagingKeys;
-	std::set<StagingKeyRange> stagingKeyRanges;
-
-	Future<Void> pollMetrics;
-
-	RoleVersionBatchState vbState;
-
-	long receiveMutationReqs;
-
-	// Stats
-	double receivedBytes; // received mutation size
-	double appliedBytes; // after coalesce, how many bytes to write to DB
-	double targetWriteRateMB; // target amount of data outstanding for DB;
-	double totalBytesToWrite; // total amount of data in bytes to write
-	double applyingDataBytes; // amount of data in flight of committing
-	AsyncTrigger releaseTxnTrigger; // trigger to release more txns
-	Future<Void> rateTracer; // trace transaction rate control info
-
-	// Status counters
-	struct Counters {
-		CounterCollection cc;
-		Counter receivedBytes, receivedWeightedBytes, receivedMutations, receivedAtomicOps;
-		Counter appliedBytes, appliedWeightedBytes, appliedMutations, appliedAtomicOps;
-		Counter appliedTxns, appliedTxnRetries;
-		Counter fetchKeys, fetchTxns, fetchTxnRetries; // number of keys to fetch from dest. FDB cluster.
-		Counter clearOps, clearTxns;
-
-		Counters(ApplierBatchData* self, UID applierInterfID, int batchIndex)
-		  : cc("ApplierBatch", applierInterfID.toString() + ":" + std::to_string(batchIndex)),
-		    receivedBytes("ReceivedBytes", cc), receivedWeightedBytes("ReceivedWeightedMutations", cc),
-		    receivedMutations("ReceivedMutations", cc), receivedAtomicOps("ReceivedAtomicOps", cc),
-		    appliedBytes("AppliedBytes", cc), appliedWeightedBytes("AppliedWeightedBytes", cc),
-		    appliedMutations("AppliedMutations", cc), appliedAtomicOps("AppliedAtomicOps", cc),
-		    appliedTxns("AppliedTxns", cc), appliedTxnRetries("AppliedTxnRetries", cc), fetchKeys("FetchKeys", cc),
-		    fetchTxns("FetchTxns", cc), fetchTxnRetries("FetchTxnRetries", cc), clearOps("ClearOps", cc),
-		    clearTxns("ClearTxns", cc) {}
-	} counters;
-
-	void addref() { return ReferenceCounted<ApplierBatchData>::addref(); }
-	void delref() { return ReferenceCounted<ApplierBatchData>::delref(); }
-
-	explicit ApplierBatchData(UID nodeID, int batchIndex)
-	  : vbState(ApplierVersionBatchState::NOT_INIT), receiveMutationReqs(0), receivedBytes(0), appliedBytes(0),
-	    targetWriteRateMB(SERVER_KNOBS->FASTRESTORE_WRITE_BW_MB / SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS),
-	    totalBytesToWrite(-1), applyingDataBytes(0), counters(this, nodeID, batchIndex) {
-		pollMetrics =
-		    counters.cc.traceCounters(format("FastRestoreApplierMetrics%d", batchIndex),
-		                              nodeID,
-		                              SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY,
-		                              nodeID.toString() + "/RestoreApplierMetrics/" + std::to_string(batchIndex));
-		TraceEvent("FastRestoreApplierMetricsCreated").detail("Node", nodeID);
-	}
-	~ApplierBatchData() {
-		rateTracer = Void(); // cancel actor
-	}
-
-	void addMutation(MutationRef m, LogMessageVersion ver) {
-		if (!isRangeMutation(m)) {
-			auto item = stagingKeys.emplace(m.param1, StagingKey(m.param1));
-			item.first->second.add(m, ver);
-		} else {
-			stagingKeyRanges.insert(StagingKeyRange(m, ver));
-		}
-	}
-
-	// Return true if all staging keys have been precomputed
-	bool allKeysPrecomputed() {
-		for (auto& stagingKey : stagingKeys) {
-			if (!stagingKey.second.hasPrecomputed()) {
-				TraceEvent("FastRestoreApplierAllKeysPrecomputedFalse")
-				    .detail("Key", stagingKey.first)
-				    .detail("BufferedVersion", stagingKey.second.version.toString())
-				    .detail("MaxPendingVersion", stagingKey.second.pendingMutations.rbegin()->first.toString());
-				return false;
-			}
-		}
-		TraceEvent("FastRestoreApplierAllKeysPrecomputed").log();
-		return true;
-	}
-
-	void reset() {
-		kvOps.clear();
-		dbApplier = Optional<Future<Void>>();
-	}
-
-	void sanityCheckMutationOps() const {
-		if (kvOps.empty())
-			return;
-
-		ASSERT_WE_THINK(isKVOpsSorted());
-		ASSERT_WE_THINK(allOpsAreKnown());
-	}
-
-	bool isKVOpsSorted() const {
-		auto prev = kvOps.begin();
-		for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
-			if (prev->first > it->first) {
-				return false;
-			}
-			prev = it;
-		}
-		return true;
-	}
-
-	bool allOpsAreKnown() const {
-		for (auto it = kvOps.begin(); it != kvOps.end(); ++it) {
-			for (auto m = it->second.begin(); m != it->second.end(); ++m) {
-				if (m->type == MutationRef::SetValue || m->type == MutationRef::ClearRange ||
-				    isAtomicOp((MutationRef::Type)m->type))
-					continue;
-				else {
-					TraceEvent(SevError, "FastRestoreApplier").detail("UnknownMutationType", m->type);
-					return false;
-				}
-			}
-		}
-		return true;
-	}
-};
-
-struct RestoreApplierData : RestoreRoleData, public ReferenceCounted<RestoreApplierData> {
-	// Buffer for uncommitted data at ongoing version batches
-	std::map<int, Reference<ApplierBatchData>> batch;
-
-	void addref() { return ReferenceCounted<RestoreApplierData>::addref(); }
-	void delref() { return ReferenceCounted<RestoreApplierData>::delref(); }
-
-	explicit RestoreApplierData(UID applierInterfID, int assignedIndex) {
-		nodeID = applierInterfID;
-		nodeIndex = assignedIndex;
-
-		// Q: Why do we need to initMetric?
-		// version.initMetric("RestoreApplier.Version"_sr, cc.id);
-
-		role = RestoreRole::Applier;
-	}
-
-	~RestoreApplierData() override = default;
-
-	// getVersionBatchState may be called periodically to dump version batch state,
-	// even when no version batch has been started.
-	int getVersionBatchState(int batchIndex) const final {
-		auto item = batch.find(batchIndex);
-		if (item == batch.end()) { // Batch has not been initialized when we blindly profile the state
-			return ApplierVersionBatchState::INVALID;
-		} else {
-			return item->second->vbState.get();
-		}
-	}
-	void setVersionBatchState(int batchIndex, int vbState) final {
-		std::map<int, Reference<ApplierBatchData>>::iterator item = batch.find(batchIndex);
-		ASSERT(item != batch.end());
-		item->second->vbState = vbState;
-	}
-
-	void initVersionBatch(int batchIndex) override {
-		TraceEvent("FastRestoreApplierInitVersionBatch", id()).detail("BatchIndex", batchIndex);
-		batch[batchIndex] = Reference<ApplierBatchData>(new ApplierBatchData(nodeID, batchIndex));
-	}
-
-	void resetPerRestoreRequest() override {
-		batch.clear();
-		finishedBatch = NotifiedVersion(0);
-	}
-
-	std::string describeNode() const override {
-		std::stringstream ss;
-		ss << "NodeID:" << nodeID.toString() << " nodeIndex:" << nodeIndex;
-		return ss.str();
-	}
-};
-
-ACTOR Future<Void> restoreApplierCore(RestoreApplierInterface applierInterf, int nodeIndex, Database cx);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/RestoreCommon.actor.h b/fdbserver/include/fdbserver/RestoreCommon.actor.h
deleted file mode 100644
index 4cc7fc35f1c..00000000000
--- a/fdbserver/include/fdbserver/RestoreCommon.actor.h
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
- * RestoreCommon.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file includes the code copied from the old restore in FDB 5.2
-// The functions and structure declared in this file can be shared by
-// the old restore and the new performant restore systems
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORECOMMON_ACTOR_G_H)
-#define FDBSERVER_RESTORECOMMON_ACTOR_G_H
-#include "fdbserver/RestoreCommon.actor.g.h"
-#elif !defined(FDBSERVER_RESTORECOMMON_ACTOR_H)
-#define FDBSERVER_RESTORECOMMON_ACTOR_H
-
-#include "flow/flow.h"
-#include "flow/genericactors.actor.h"
-#include "fdbclient/Tuple.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "flow/IAsyncFile.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbserver/Knobs.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-// RestoreConfig copied from FileBackupAgent.actor.cpp
-// We copy RestoreConfig instead of using (and potentially changing) it in place
-// to avoid conflict with the existing code.
-// We also made minor changes to allow RestoreConfig to be ReferenceCounted
-// TODO: Merge this RestoreConfig with the original RestoreConfig in FileBackupAgent.actor.cpp
-// For convenience
-typedef FileBackupAgent::ERestoreState ERestoreState;
-
-struct RestoreFileFR;
-
-// We copy RestoreConfig copied from FileBackupAgent.actor.cpp instead of using (and potentially changing) it in place
-// to avoid conflict with the existing code Split RestoreConfig defined in FileBackupAgent.actor.cpp to declaration in
-// Restore.actor.h and implementation in RestoreCommon.actor.cpp, so that we can use in both the existing restore and
-// the new fast restore subsystems. We use RestoreConfig as a Reference<RestoreConfig>, which leads to some
-// non-functional changes in RestoreConfig
-class RestoreConfigFR : public KeyBackedTaskConfig, public ReferenceCounted<RestoreConfigFR> {
-public:
-	RestoreConfigFR(UID uid = UID()) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, uid) {}
-	RestoreConfigFR(Reference<Task> task) : KeyBackedTaskConfig(fileRestorePrefixRange.begin, task) {}
-
-	KeyBackedProperty<ERestoreState> stateEnum();
-
-	Future<StringRef> stateText(Reference<ReadYourWritesTransaction> tr);
-
-	KeyBackedProperty<Key> addPrefix();
-
-	KeyBackedProperty<Key> removePrefix();
-
-	// XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges
-	KeyBackedProperty<KeyRange> restoreRange();
-
-	KeyBackedProperty<std::vector<KeyRange>> restoreRanges();
-
-	KeyBackedProperty<Key> batchFuture();
-
-	KeyBackedProperty<Version> restoreVersion();
-
-	KeyBackedProperty<Reference<IBackupContainer>> sourceContainer();
-
-	// Get the source container as a bare URL, without creating a container instance
-	KeyBackedProperty<Value> sourceContainerURL();
-
-	// Total bytes written by all log and range restore tasks.
-	KeyBackedBinaryValue<int64_t> bytesWritten();
-
-	// File blocks that have had tasks created for them by the Dispatch task
-	KeyBackedBinaryValue<int64_t> filesBlocksDispatched();
-
-	// File blocks whose tasks have finished
-	KeyBackedBinaryValue<int64_t> fileBlocksFinished();
-
-	// Total number of files in the fileMap
-	KeyBackedBinaryValue<int64_t> fileCount();
-
-	// Total number of file blocks in the fileMap
-	KeyBackedBinaryValue<int64_t> fileBlockCount();
-
-	Future<std::vector<KeyRange>> getRestoreRangesOrDefault(Reference<ReadYourWritesTransaction> tr);
-	ACTOR static Future<std::vector<KeyRange>> getRestoreRangesOrDefault_impl(RestoreConfigFR* self,
-	                                                                          Reference<ReadYourWritesTransaction> tr);
-
-	// Describes a file to load blocks from during restore.  Ordered by version and then fileName to enable
-	// incrementally advancing through the map, saving the version and path of the next starting point.
-	struct RestoreFile {
-		Version version;
-		std::string fileName;
-		bool isRange; // false for log file
-		int64_t blockSize;
-		int64_t fileSize;
-		Version endVersion; // not meaningful for range files
-
-		Tuple pack() const {
-			// fprintf(stderr, "Filename:%s\n", fileName.c_str());
-			return Tuple::makeTuple(version, fileName, (int)isRange, fileSize, blockSize, endVersion);
-		}
-		static RestoreFile unpack(Tuple const& t) {
-			RestoreFile r;
-			int i = 0;
-			r.version = t.getInt(i++);
-			r.fileName = t.getString(i++).toString();
-			r.isRange = t.getInt(i++) != 0;
-			r.fileSize = t.getInt(i++);
-			r.blockSize = t.getInt(i++);
-			r.endVersion = t.getInt(i++);
-			return r;
-		}
-	};
-
-	// typedef KeyBackedSet<RestoreFile> FileSetT;
-	KeyBackedSet<RestoreFile> fileSet();
-
-	Future<bool> isRunnable(Reference<ReadYourWritesTransaction> tr);
-
-	Future<Void> logError(Database cx, Error e, std::string const& details, void* taskInstance = nullptr);
-
-	Key mutationLogPrefix();
-
-	Key applyMutationsMapPrefix();
-
-	ACTOR Future<int64_t> getApplyVersionLag_impl(Reference<ReadYourWritesTransaction> tr, UID uid);
-
-	Future<int64_t> getApplyVersionLag(Reference<ReadYourWritesTransaction> tr);
-
-	void initApplyMutations(Reference<ReadYourWritesTransaction> tr, Key addPrefix, Key removePrefix);
-
-	void clearApplyMutationsKeys(Reference<ReadYourWritesTransaction> tr);
-
-	void setApplyBeginVersion(Reference<ReadYourWritesTransaction> tr, Version ver);
-
-	void setApplyEndVersion(Reference<ReadYourWritesTransaction> tr, Version ver);
-
-	Future<Version> getApplyEndVersion(Reference<ReadYourWritesTransaction> tr);
-
-	ACTOR static Future<std::string> getProgress_impl(Reference<RestoreConfigFR> restore,
-	                                                  Reference<ReadYourWritesTransaction> tr);
-	Future<std::string> getProgress(Reference<ReadYourWritesTransaction> tr);
-
-	ACTOR static Future<std::string> getFullStatus_impl(Reference<RestoreConfigFR> restore,
-	                                                    Reference<ReadYourWritesTransaction> tr);
-	Future<std::string> getFullStatus(Reference<ReadYourWritesTransaction> tr);
-
-	std::string toString(); // Added by Meng
-};
-
-// typedef RestoreConfigFR::RestoreFile RestoreFile;
-
-// Describes a file to load blocks from during restore.  Ordered by version and then fileName to enable
-// incrementally advancing through the map, saving the version and path of the next starting point.
-// NOTE: The struct RestoreFileFR can NOT be named RestoreFile, because compiler will get confused in linking which
-// RestoreFile should be used. If we use RestoreFile, compilation succeeds, but weird segmentation fault will happen.
-struct RestoreFileFR {
-	Version version;
-	std::string fileName;
-	bool isRange; // false for log file
-	int64_t blockSize;
-	int64_t fileSize;
-	Version endVersion; // not meaningful for range files
-	Version beginVersion; // range file's beginVersion == endVersion; log file contains mutations in version
-	                      // [beginVersion, endVersion)
-	int64_t cursor; // The start block location to be restored. All blocks before cursor have been scheduled to load and
-	                // restore
-	int fileIndex; // index of backup file. Must be identical per file.
-	int partitionId = -1; // Partition ID (Log Router Tag ID) for mutation files.
-
-	Tuple pack() const {
-		return Tuple::makeTuple(version,
-		                        fileName,
-		                        (int)isRange,
-		                        fileSize,
-		                        blockSize,
-		                        endVersion,
-		                        beginVersion,
-		                        cursor,
-		                        fileIndex,
-		                        partitionId);
-	}
-	static RestoreFileFR unpack(Tuple const& t) {
-		RestoreFileFR r;
-		int i = 0;
-		r.version = t.getInt(i++);
-		r.fileName = t.getString(i++).toString();
-		r.isRange = t.getInt(i++) != 0;
-		r.fileSize = t.getInt(i++);
-		r.blockSize = t.getInt(i++);
-		r.endVersion = t.getInt(i++);
-		r.beginVersion = t.getInt(i++);
-		r.cursor = t.getInt(i++);
-		r.fileIndex = t.getInt(i++);
-		r.partitionId = t.getInt(i++);
-		return r;
-	}
-
-	bool operator<(const RestoreFileFR& rhs) const {
-		return std::tie(beginVersion, endVersion, fileIndex, fileName) <
-		       std::tie(rhs.beginVersion, rhs.endVersion, rhs.fileIndex, rhs.fileName);
-	}
-
-	RestoreFileFR()
-	  : version(invalidVersion), isRange(false), blockSize(0), fileSize(0), endVersion(invalidVersion),
-	    beginVersion(invalidVersion), cursor(0), fileIndex(0) {}
-
-	explicit RestoreFileFR(const RangeFile& f)
-	  : version(f.version), fileName(f.fileName), isRange(true), blockSize(f.blockSize), fileSize(f.fileSize),
-	    endVersion(f.version), beginVersion(f.version), cursor(0), fileIndex(0) {}
-
-	explicit RestoreFileFR(const LogFile& f)
-	  : version(f.beginVersion), fileName(f.fileName), isRange(false), blockSize(f.blockSize), fileSize(f.fileSize),
-	    endVersion(f.endVersion), beginVersion(f.beginVersion), cursor(0), fileIndex(0), partitionId(f.tagId) {}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "version:" << version << " fileName:" << fileName << " isRange:" << isRange << " blockSize:" << blockSize
-		   << " fileSize:" << fileSize << " endVersion:" << endVersion << " beginVersion:" << beginVersion
-		   << " cursor:" << cursor << " fileIndex:" << fileIndex << " partitionId:" << partitionId;
-		return ss.str();
-	}
-};
-
-namespace parallelFileRestore {
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeLogFileBlock(Reference<IAsyncFile> file,
-                                                                    int64_t offset,
-                                                                    int len);
-} // namespace parallelFileRestore
-
-// Send each request in requests via channel of the request's interface.
-// Save replies to replies if replies != nullptr
-// The UID in a request is the UID of the interface to handle the request
-ACTOR template <class Interface, class Request>
-Future<Void> getBatchReplies(RequestStream<Request> Interface::* channel,
-                             std::map<UID, Interface> interfaces,
-                             std::vector<std::pair<UID, Request>> requests,
-                             std::vector<REPLY_TYPE(Request)>* replies,
-                             TaskPriority taskID = TaskPriority::Low,
-                             bool trackRequestLatency = true) {
-	if (requests.empty()) {
-		return Void();
-	}
-
-	state double start = now();
-	state int oustandingReplies = requests.size();
-	loop {
-		try {
-			state std::vector<Future<REPLY_TYPE(Request)>> cmdReplies;
-			state std::vector<std::tuple<UID, Request, double>> replyDurations; // double is end time of the request
-			for (auto& request : requests) {
-				RequestStream<Request> const* stream = &(interfaces[request.first].*channel);
-				cmdReplies.push_back(stream->getReply(request.second, taskID));
-				replyDurations.emplace_back(request.first, request.second, 0);
-			}
-
-			state std::vector<Future<REPLY_TYPE(Request)>> ongoingReplies;
-			state std::vector<int> ongoingRepliesIndex;
-			loop {
-				ongoingReplies.clear();
-				ongoingRepliesIndex.clear();
-				for (int i = 0; i < cmdReplies.size(); ++i) {
-					if (SERVER_KNOBS->FASTRESTORE_REQBATCH_LOG) {
-						TraceEvent(SevInfo, "FastRestoreGetBatchReplies")
-						    .suppressFor(1.0)
-						    .detail("Requests", requests.size())
-						    .detail("OutstandingReplies", oustandingReplies)
-						    .detail("ReplyIndex", i)
-						    .detail("ReplyIsReady", cmdReplies[i].isReady())
-						    .detail("ReplyIsError", cmdReplies[i].isError())
-						    .detail("RequestNode", requests[i].first)
-						    .detail("Request", requests[i].second.toString());
-					}
-					if (!cmdReplies[i].isReady()) { // still wait for reply
-						ongoingReplies.push_back(cmdReplies[i]);
-						ongoingRepliesIndex.push_back(i);
-					}
-				}
-				ASSERT(ongoingReplies.size() == oustandingReplies);
-				if (ongoingReplies.empty()) {
-					break;
-				} else {
-					wait(
-					    quorum(ongoingReplies,
-					           std::min((int)SERVER_KNOBS->FASTRESTORE_REQBATCH_PARALLEL, (int)ongoingReplies.size())));
-				}
-				// At least one reply is received; Calculate the reply duration
-				for (int j = 0; j < ongoingReplies.size(); ++j) {
-					if (ongoingReplies[j].isReady()) {
-						std::get<2>(replyDurations[ongoingRepliesIndex[j]]) = now();
-						--oustandingReplies;
-					} else if (ongoingReplies[j].isError()) {
-						// When this happens,
-						// the above assertion ASSERT(ongoingReplies.size() == oustandingReplies) will fail
-						TraceEvent(SevError, "FastRestoreGetBatchRepliesReplyError")
-						    .detail("OngoingReplyIndex", j)
-						    .detail("FutureError", ongoingReplies[j].getError().what());
-					}
-				}
-			}
-			ASSERT(oustandingReplies == 0);
-			if (trackRequestLatency && SERVER_KNOBS->FASTRESTORE_TRACK_REQUEST_LATENCY) {
-				// Calculate the latest end time for each interface
-				std::map<UID, double> maxEndTime;
-				UID bathcID = deterministicRandom()->randomUniqueID();
-				for (int i = 0; i < replyDurations.size(); ++i) {
-					double endTime = std::get<2>(replyDurations[i]);
-					TraceEvent(SevInfo, "ProfileSendRequestBatchLatency", bathcID)
-					    .detail("Node", std::get<0>(replyDurations[i]))
-					    .detail("Request", std::get<1>(replyDurations[i]).toString())
-					    .detail("Duration", endTime - start);
-					auto item = maxEndTime.emplace(std::get<0>(replyDurations[i]), endTime);
-					item.first->second = std::max(item.first->second, endTime);
-				}
-				// Check the time gap between the earliest and latest node
-				double earliest = std::numeric_limits<double>::max();
-				double latest = std::numeric_limits<double>::min();
-				UID earliestNode, latestNode;
-
-				for (auto& endTime : maxEndTime) {
-					if (earliest > endTime.second) {
-						earliest = endTime.second;
-						earliestNode = endTime.first;
-					}
-					if (latest < endTime.second) {
-						latest = endTime.second;
-						latestNode = endTime.first;
-					}
-				}
-				if (latest - earliest > SERVER_KNOBS->FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS) {
-					TraceEvent(SevWarn, "ProfileSendRequestBatchLatencyFoundStraggler", bathcID)
-					    .detail("SlowestNode", latestNode)
-					    .detail("FatestNode", earliestNode)
-					    .detail("EarliestEndtime", earliest)
-					    .detail("LagTime", latest - earliest);
-				}
-			}
-			// Update replies
-			if (replies != nullptr) {
-				for (int i = 0; i < cmdReplies.size(); ++i) {
-					replies->emplace_back(cmdReplies[i].get());
-				}
-			}
-			break;
-		} catch (Error& e) {
-			if (e.code() == error_code_operation_cancelled)
-				break;
-			// fprintf(stdout, "sendBatchRequests Error code:%d, error message:%s\n", e.code(), e.what());
-			TraceEvent(SevWarn, "FastRestoreSendBatchRequests").error(e);
-			for (auto& request : requests) {
-				TraceEvent(SevWarn, "FastRestoreSendBatchRequests")
-				    .detail("SendBatchRequests", requests.size())
-				    .detail("RequestID", request.first)
-				    .detail("Request", request.second.toString());
-				resetReply(request.second);
-			}
-		}
-	}
-
-	return Void();
-}
-
-// Similar to getBatchReplies except that the caller does not expect to process the reply info.
-ACTOR template <class Interface, class Request>
-Future<Void> sendBatchRequests(RequestStream<Request> Interface::* channel,
-                               std::map<UID, Interface> interfaces,
-                               std::vector<std::pair<UID, Request>> requests,
-                               TaskPriority taskID = TaskPriority::Low,
-                               bool trackRequestLatency = true) {
-	wait(getBatchReplies(channel, interfaces, requests, nullptr, taskID, trackRequestLatency));
-
-	return Void();
-}
-
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_RESTORECOMMON_ACTOR_H
diff --git a/fdbserver/include/fdbserver/RestoreController.actor.h b/fdbserver/include/fdbserver/RestoreController.actor.h
deleted file mode 100644
index 22a40de7a95..00000000000
--- a/fdbserver/include/fdbserver/RestoreController.actor.h
+++ /dev/null
@@ -1,464 +0,0 @@
-/*
- * RestoreController.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file declear RestoreController interface and actors
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_CONTROLLER_G_H)
-#define FDBSERVER_RESTORE_CONTROLLER_G_H
-#include "fdbserver/RestoreController.actor.g.h"
-#elif !defined(FDBSERVER_RESTORE_CONTROLLER_H)
-#define FDBSERVER_RESTORE_CONTROLLER_H
-
-#include <sstream>
-#include "flow/Platform.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/CommitTransaction.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/Stats.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreWorker.actor.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-struct VersionBatch {
-	Version beginVersion; // Inclusive
-	Version endVersion; // exclusive
-	std::set<RestoreFileFR> logFiles;
-	std::set<RestoreFileFR> rangeFiles;
-	double size; // size of data in range and log files
-	int batchIndex; // Never reset
-
-	VersionBatch() : beginVersion(0), endVersion(0), size(0) {};
-
-	bool operator<(const VersionBatch& rhs) const {
-		return std::tie(batchIndex, beginVersion, endVersion, logFiles, rangeFiles, size) <
-		       std::tie(rhs.batchIndex, rhs.beginVersion, rhs.endVersion, rhs.logFiles, rhs.rangeFiles, rhs.size);
-	}
-
-	bool isEmpty() const { return logFiles.empty() && rangeFiles.empty(); }
-	void reset() {
-		beginVersion = 0;
-		endVersion = 0;
-		logFiles.clear();
-		rangeFiles.clear();
-		size = 0;
-	}
-
-	// RestoreAsset and VersionBatch both use endVersion as exclusive in version range
-	bool isInVersionRange(Version version) const { return version >= beginVersion && version < endVersion; }
-};
-
-struct ControllerBatchData : public ReferenceCounted<ControllerBatchData> {
-	// rangeToApplier is in controller and loader node. Loader uses this to determine which applier a mutation should be
-	// sent.
-	//   KeyRef is the inclusive lower bound of the key range the applier (UID) is responsible for
-	std::map<Key, UID> rangeToApplier;
-	Optional<Future<Void>> applyToDB;
-
-	IndexedSet<Key, int64_t> samples; // sample of range and log files
-	double samplesSize; // sum of the metric of all samples
-	std::set<UID> sampleMsgs; // deduplicate sample messages
-
-	ControllerBatchData() = default;
-	~ControllerBatchData() = default;
-
-	// Return true if pass the sanity check
-	bool sanityCheckApplierKeyRange() {
-		bool ret = true;
-		// An applier should only appear once in rangeToApplier
-		std::map<UID, Key> applierToRange;
-		for (auto& applier : rangeToApplier) {
-			if (applierToRange.find(applier.second) == applierToRange.end()) {
-				applierToRange[applier.second] = applier.first;
-			} else {
-				TraceEvent(SevError, "FastRestoreController")
-				    .detail("SanityCheckApplierKeyRange", applierToRange.size())
-				    .detail("ApplierID", applier.second)
-				    .detail("Key1", applierToRange[applier.second])
-				    .detail("Key2", applier.first);
-				ret = false;
-			}
-		}
-		return ret;
-	}
-
-	void logApplierKeyRange(int batchIndex) {
-		TraceEvent("FastRestoreLogApplierKeyRange")
-		    .detail("BatchIndex", batchIndex)
-		    .detail("ApplierKeyRangeNum", rangeToApplier.size());
-		for (auto& applier : rangeToApplier) {
-			TraceEvent("FastRestoreLogApplierKeyRange")
-			    .detail("BatchIndex", batchIndex)
-			    .detail("KeyRangeLowerBound", applier.first)
-			    .detail("Applier", applier.second);
-		}
-	}
-};
-
-enum class RestoreAssetStatus { Loading, Loaded };
-
-enum class RestoreSendStatus { SendingLogs, SendedLogs, SendingRanges, SendedRanges };
-
-enum class RestoreApplyStatus { Applying, Applied };
-
-// Track restore progress of each RestoreAsset (RA) and
-// Use status to sanity check restore property, e.g., each RA should be processed exactly once.
-struct ControllerBatchStatus : public ReferenceCounted<ControllerBatchStatus> {
-	std::map<RestoreAsset, RestoreAssetStatus> raStatus;
-	std::map<UID, RestoreSendStatus> loadStatus;
-	std::map<UID, RestoreApplyStatus> applyStatus;
-
-	void addref() { return ReferenceCounted<ControllerBatchStatus>::addref(); }
-	void delref() { return ReferenceCounted<ControllerBatchStatus>::delref(); }
-
-	ControllerBatchStatus() = default;
-	~ControllerBatchStatus() = default;
-};
-
-struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreControllerData> {
-	std::map<Version, VersionBatch> versionBatches; // key is the beginVersion of the version batch
-
-	Reference<IBackupContainer> bc; // Backup container is used to read backup files
-	Key bcUrl; // The url used to get the bc
-
-	std::map<int, Reference<ControllerBatchData>> batch;
-	std::map<int, Reference<ControllerBatchStatus>> batchStatus;
-
-	AsyncVar<int> runningVersionBatches; // Currently running version batches
-
-	std::map<UID, double> rolesHeartBeatTime; // Key: role id; Value: most recent time controller receives heart beat
-
-	// addActor: add to actorCollection so that when an actor has error, the ActorCollection can catch the error.
-	// addActor is used to create the actorCollection when the RestoreController is created
-	PromiseStream<Future<Void>> addActor;
-
-	void addref() { return ReferenceCounted<RestoreControllerData>::addref(); }
-	void delref() { return ReferenceCounted<RestoreControllerData>::delref(); }
-
-	RestoreControllerData(UID interfId) {
-		role = RestoreRole::Controller;
-		nodeID = interfId;
-		runningVersionBatches.set(0);
-	}
-
-	~RestoreControllerData() override = default;
-
-	int getVersionBatchState(int batchIndex) const final { return RoleVersionBatchState::INVALID; }
-	void setVersionBatchState(int batchIndex, int vbState) final {}
-
-	void initVersionBatch(int batchIndex) override {
-		TraceEvent("FastRestoreControllerInitVersionBatch", id()).detail("VersionBatchIndex", batchIndex);
-	}
-
-	// Reset controller data at the beginning of each restore request
-	void resetPerRestoreRequest() override {
-		TraceEvent("FastRestoreControllerReset").detail("OldVersionBatches", versionBatches.size());
-		versionBatches.clear();
-		batch.clear();
-		batchStatus.clear();
-		finishedBatch = NotifiedVersion(0);
-		versionBatchId = NotifiedVersion(0);
-		ASSERT(runningVersionBatches.get() == 0);
-	}
-
-	std::string describeNode() const override {
-		std::stringstream ss;
-		ss << "Controller";
-		return ss.str();
-	}
-
-	void dumpVersionBatches(const std::map<Version, VersionBatch>& versionBatches) const {
-		int i = 1;
-		double rangeFiles = 0;
-		double rangeSize = 0;
-		double logFiles = 0;
-		double logSize = 0;
-		for (auto& vb : versionBatches) {
-			TraceEvent("FastRestoreVersionBatches")
-			    .detail("BatchIndex", vb.second.batchIndex)
-			    .detail("ExpectedBatchIndex", i)
-			    .detail("BeginVersion", vb.second.beginVersion)
-			    .detail("EndVersion", vb.second.endVersion)
-			    .detail("Size", vb.second.size);
-			for (auto& f : vb.second.rangeFiles) {
-				bool invalidVersion = (f.beginVersion != f.endVersion) || (f.beginVersion >= vb.second.endVersion ||
-				                                                           f.beginVersion < vb.second.beginVersion);
-				TraceEvent(invalidVersion ? SevError : SevInfo, "FastRestoreVersionBatches")
-				    .detail("BatchIndex", i)
-				    .detail("RangeFile", f.toString());
-				rangeSize += f.fileSize;
-				rangeFiles++;
-			}
-			for (auto& f : vb.second.logFiles) {
-				bool outOfRange = (f.beginVersion >= vb.second.endVersion || f.endVersion <= vb.second.beginVersion);
-				TraceEvent(outOfRange ? SevError : SevInfo, "FastRestoreVersionBatches")
-				    .detail("BatchIndex", i)
-				    .detail("LogFile", f.toString());
-				logSize += f.fileSize;
-				logFiles++;
-			}
-			++i;
-		}
-
-		TraceEvent("FastRestoreVersionBatchesSummary")
-		    .detail("VersionBatches", versionBatches.size())
-		    .detail("LogFiles", logFiles)
-		    .detail("RangeFiles", rangeFiles)
-		    .detail("LogBytes", logSize)
-		    .detail("RangeBytes", rangeSize);
-	}
-
-	// Input: Get the size of data in backup files in version range [prevVersion, nextVersion)
-	// Return: param1: the size of data at nextVersion, param2: the minimum range file index whose version >
-	// nextVersion, param3: log files with data in [prevVersion, nextVersion)
-	std::tuple<double, int, std::vector<RestoreFileFR>> getVersionSize(Version prevVersion,
-	                                                                   Version nextVersion,
-	                                                                   const std::vector<RestoreFileFR>& rangeFiles,
-	                                                                   int rangeIdx,
-	                                                                   const std::vector<RestoreFileFR>& logFiles) {
-		double size = 0;
-		TraceEvent(SevVerbose, "FastRestoreGetVersionSize")
-		    .detail("PreviousVersion", prevVersion)
-		    .detail("NextVersion", nextVersion)
-		    .detail("RangeFiles", rangeFiles.size())
-		    .detail("RangeIndex", rangeIdx)
-		    .detail("LogFiles", logFiles.size());
-		ASSERT(prevVersion <= nextVersion);
-		while (rangeIdx < rangeFiles.size()) {
-			TraceEvent(SevVerbose, "FastRestoreGetVersionSize").detail("RangeFile", rangeFiles[rangeIdx].toString());
-			if (rangeFiles[rangeIdx].version < nextVersion) {
-				ASSERT(rangeFiles[rangeIdx].version >= prevVersion);
-				size += rangeFiles[rangeIdx].fileSize;
-			} else {
-				break;
-			}
-			++rangeIdx;
-		}
-		std::vector<RestoreFileFR> retLogs;
-		// Scan all logFiles every time to avoid assumption on log files' version ranges.
-		// For example, we do not assume each version range only exists in one log file
-		for (const auto& file : logFiles) {
-			Version begin = std::max(prevVersion, file.beginVersion);
-			Version end = std::min(nextVersion, file.endVersion);
-			if (begin < end) { // logIdx file overlap in [prevVersion, nextVersion)
-				double ratio = (end - begin) * 1.0 / (file.endVersion - file.beginVersion);
-				size += file.fileSize * ratio;
-				retLogs.push_back(file);
-			}
-		}
-		return std::make_tuple(size, rangeIdx, retLogs);
-	}
-
-	// Split backup files into version batches, each of which has similar data size
-	// Input: sorted range files, sorted log files;
-	// Output: a set of version batches whose size is less than SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES
-	//         and each mutation in backup files is included in the version batches exactly once.
-	// Assumption 1: input files has no empty files;
-	// Assumption 2: range files at one version <= FASTRESTORE_VERSIONBATCH_MAX_BYTES.
-	// Note: We do not allow a versionBatch size larger than the FASTRESTORE_VERSIONBATCH_MAX_BYTES because the range
-	// file size at a version depends on the number of backupAgents and its upper bound is hard to get.
-	void buildVersionBatches(const std::vector<RestoreFileFR>& rangeFiles,
-	                         const std::vector<RestoreFileFR>& logFiles,
-	                         std::map<Version, VersionBatch>* versionBatches,
-	                         Version targetVersion) {
-		bool rewriteNextVersion = false;
-		int rangeIdx = 0;
-		int logIdx = 0; // Ensure each log file is included in version batch
-		Version prevEndVersion = 0;
-		Version nextVersion = 0; // Used to calculate the batch's endVersion
-		VersionBatch vb;
-		Version maxVBVersion = 0;
-		bool lastLogFile = false;
-		vb.beginVersion = 0; // Version batch range [beginVersion, endVersion)
-		vb.batchIndex = 1;
-
-		while (rangeIdx < rangeFiles.size() || logIdx < logFiles.size()) {
-			if (!rewriteNextVersion) {
-				if (rangeIdx < rangeFiles.size() && logIdx < logFiles.size()) {
-					// nextVersion as endVersion is exclusive in the version range
-					nextVersion = std::max(rangeFiles[rangeIdx].version + 1, nextVersion);
-				} else if (rangeIdx < rangeFiles.size()) { // i.e., logIdx >= logFiles.size()
-					nextVersion = rangeFiles[rangeIdx].version + 1;
-				} else if (logIdx < logFiles.size()) {
-					while (logIdx < logFiles.size() && logFiles[logIdx].endVersion <= nextVersion) {
-						logIdx++;
-					}
-					if (logIdx < logFiles.size()) {
-						nextVersion = logFiles[logIdx].endVersion;
-					} else {
-						TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
-						    .detail("FinishAllLogFiles", logIdx)
-						    .detail("CurBatchIndex", vb.batchIndex)
-						    .detail("CurBatchSize", vb.size);
-						if (prevEndVersion < nextVersion) {
-							// Ensure the last log file is included in version batch
-							lastLogFile = true;
-						} else {
-							break; // Finished all log files
-						}
-					}
-				} else {
-					// TODO: Check why this may happen?!
-					TraceEvent(SevError, "FastRestoreBuildVersionBatch")
-					    .detail("RangeIndex", rangeIdx)
-					    .detail("RangeFiles", rangeFiles.size())
-					    .detail("LogIndex", logIdx)
-					    .detail("LogFiles", logFiles.size());
-				}
-			} else {
-				rewriteNextVersion = false;
-			}
-
-			double nextVersionSize;
-			int nextRangeIdx;
-			std::vector<RestoreFileFR> curLogFiles;
-			std::tie(nextVersionSize, nextRangeIdx, curLogFiles) =
-			    getVersionSize(prevEndVersion, nextVersion, rangeFiles, rangeIdx, logFiles);
-
-			TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
-			    .detail("BatchIndex", vb.batchIndex)
-			    .detail("VersionBatchBeginVersion", vb.beginVersion)
-			    .detail("PreviousEndVersion", prevEndVersion)
-			    .detail("NextVersion", nextVersion)
-			    .detail("TargetVersion", targetVersion)
-			    .detail("RangeIndex", rangeIdx)
-			    .detail("RangeFiles", rangeFiles.size())
-			    .detail("LogIndex", logIdx)
-			    .detail("LogFiles", logFiles.size())
-			    .detail("VersionBatchSizeThreshold", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES)
-			    .detail("CurrentBatchSize", vb.size)
-			    .detail("NextVersionIntervalSize", nextVersionSize)
-			    .detail("NextRangeIndex", nextRangeIdx)
-			    .detail("UsedLogFiles", curLogFiles.size())
-			    .detail("VersionBatchCurRangeFiles", vb.rangeFiles.size())
-			    .detail("VersionBatchCurLogFiles", vb.logFiles.size())
-			    .detail("LastLogFile", lastLogFile);
-
-			ASSERT(prevEndVersion < nextVersion); // Ensure progress
-			if (vb.size + nextVersionSize <= SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES ||
-			    (vb.size < 1 && prevEndVersion + 1 == nextVersion) || lastLogFile) {
-				// In case the batch size at a single version > FASTRESTORE_VERSIONBATCH_MAX_BYTES,
-				// the version batch should include the single version to avoid false positive in simulation.
-				if (vb.size + nextVersionSize > SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES) {
-					TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "FastRestoreBuildVersionBatch")
-					    .detail("NextVersion", nextVersion)
-					    .detail("PreviousEndVersion", prevEndVersion)
-					    .detail("NextVersionIntervalSize", nextVersionSize)
-					    .detail("VersionBatchSizeThreshold", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES)
-					    .detail("SuggestedMinimumVersionBatchSizeThreshold", nextVersionSize * 2);
-				}
-				// nextVersion should be included in this batch
-				vb.size += nextVersionSize;
-				while (rangeIdx < nextRangeIdx && rangeIdx < rangeFiles.size()) {
-					ASSERT(rangeFiles[rangeIdx].fileSize > 0);
-					vb.rangeFiles.insert(rangeFiles[rangeIdx]);
-					++rangeIdx;
-				}
-
-				for (auto& log : curLogFiles) {
-					ASSERT(log.beginVersion < nextVersion);
-					ASSERT(log.endVersion > prevEndVersion);
-					ASSERT(log.fileSize > 0);
-					vb.logFiles.insert(log);
-				}
-
-				vb.endVersion = std::min(nextVersion, targetVersion + 1);
-				maxVBVersion = std::max(maxVBVersion, vb.endVersion);
-				prevEndVersion = vb.endVersion;
-			} else {
-				if (vb.size < 1) {
-					// [vb.endVersion, nextVersion) > SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES. We should split
-					// the version range
-					if (prevEndVersion >= nextVersion) {
-						// If range files at one version > FASTRESTORE_VERSIONBATCH_MAX_BYTES, DBA should increase
-						// FASTRESTORE_VERSIONBATCH_MAX_BYTES to some value larger than nextVersion
-						TraceEvent(SevError, "FastRestoreBuildVersionBatch")
-						    .detail("NextVersion", nextVersion)
-						    .detail("PreviousEndVersion", prevEndVersion)
-						    .detail("NextVersionIntervalSize", nextVersionSize)
-						    .detail("VersionBatchSizeThreshold", SERVER_KNOBS->FASTRESTORE_VERSIONBATCH_MAX_BYTES)
-						    .detail("SuggestedMinimumVersionBatchSizeThreshold", nextVersionSize * 2);
-						// Exit restore early if it won't succeed
-						flushAndExit(FDB_EXIT_ERROR);
-					}
-					ASSERT(prevEndVersion < nextVersion); // Ensure progress
-					nextVersion = (prevEndVersion + nextVersion) / 2;
-					rewriteNextVersion = true;
-					TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
-					    .detail("NextVersionIntervalSize", nextVersionSize); // Duplicate Trace
-					continue;
-				}
-				// Finalize the current version batch
-				versionBatches->emplace(vb.beginVersion, vb); // copy vb to versionBatch
-				TraceEvent(SevFRDebugInfo, "FastRestoreBuildVersionBatch")
-				    .detail("FinishBatchIndex", vb.batchIndex)
-				    .detail("VersionBatchBeginVersion", vb.beginVersion)
-				    .detail("VersionBatchEndVersion", vb.endVersion)
-				    .detail("VersionBatchLogFiles", vb.logFiles.size())
-				    .detail("VersionBatchRangeFiles", vb.rangeFiles.size())
-				    .detail("VersionBatchSize", vb.size)
-				    .detail("RangeIndex", rangeIdx)
-				    .detail("LogIndex", logIdx)
-				    .detail("NewVersionBatchBeginVersion", prevEndVersion)
-				    .detail("RewriteNextVersion", rewriteNextVersion);
-
-				// start finding the next version batch
-				vb.reset();
-				vb.size = 0;
-				vb.beginVersion = prevEndVersion;
-				vb.batchIndex++;
-			}
-		}
-		// The last wip version batch has some files
-		if (vb.size > 0) {
-			vb.endVersion = std::min(nextVersion, targetVersion + 1);
-			maxVBVersion = std::max(maxVBVersion, vb.endVersion);
-			versionBatches->emplace(vb.beginVersion, vb);
-		}
-		// Invariant: The last vb endverion should be no smaller than targetVersion
-		if (maxVBVersion < targetVersion) {
-			// Q: Is the restorable version always less than the maximum version from all backup filenames?
-			// A: This is true for the raw backup files returned by backup container before we remove the empty files.
-			TraceEvent(SevWarnAlways, "FastRestoreBuildVersionBatch")
-			    .detail("TargetVersion", targetVersion)
-			    .detail("MaxVersionBatchVersion", maxVBVersion);
-		}
-	}
-
-	void initBackupContainer(Key url, Optional<std::string> proxy) {
-		if (bcUrl == url && bc.isValid()) {
-			return;
-		}
-		TraceEvent("FastRestoreControllerInitBackupContainer")
-		    .detail("URL", url)
-		    .detail("Proxy", proxy.present() ? proxy.get() : "");
-		bcUrl = url;
-		bc = IBackupContainer::openContainer(url.toString(), proxy, {});
-	}
-};
-
-ACTOR Future<Void> startRestoreController(Reference<RestoreWorkerData> controllerWorker, Database cx);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/RestoreLoader.actor.h b/fdbserver/include/fdbserver/RestoreLoader.actor.h
deleted file mode 100644
index c876b40fa86..00000000000
--- a/fdbserver/include/fdbserver/RestoreLoader.actor.h
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * RestoreLoader.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file declares the actors used by the RestoreLoader role
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_LOADER_G_H)
-#define FDBSERVER_RESTORE_LOADER_G_H
-#include "fdbserver/RestoreLoader.actor.g.h"
-#elif !defined(FDBSERVER_RESTORE_LOADER_H)
-#define FDBSERVER_RESTORE_LOADER_H
-
-#include <sstream>
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/CommitTransaction.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbrpc/Stats.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbrpc/Locality.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreCommon.actor.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-#include "fdbclient/BackupContainer.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-class LoaderVersionBatchState : RoleVersionBatchState {
-public:
-	static const int NOT_INIT = 0;
-	static const int INIT = 1;
-	static const int LOAD_FILE = 2;
-	static const int SEND_MUTATIONS = 3;
-	static const int INVALID = 4;
-
-	explicit LoaderVersionBatchState(int newState) { vbState = newState; }
-
-	~LoaderVersionBatchState() override = default;
-
-	void operator=(int newState) override { vbState = newState; }
-
-	int get() const override { return vbState; }
-};
-
-struct LoaderBatchData : public ReferenceCounted<LoaderBatchData> {
-	std::map<LoadingParam, Future<Void>> processedFileParams;
-	std::map<LoadingParam, VersionedMutationsMap> kvOpsPerLP; // Buffered kvOps for each loading param
-
-	// rangeToApplier is in controller and loader. Loader uses this to determine which applier a mutation should be sent
-	//   Key is the inclusive lower bound of the key range the applier (UID) is responsible for
-	std::map<Key, UID> rangeToApplier;
-
-	// Sampled mutations to be sent back to restore controller
-	std::map<LoadingParam, SampledMutationsVec> sampleMutations;
-	int numSampledMutations; // The total number of mutations received from sampled data.
-
-	Future<Void> pollMetrics;
-
-	LoaderVersionBatchState vbState;
-
-	long loadFileReqs;
-
-	// Status counters
-	struct Counters {
-		CounterCollection cc;
-		Counter loadedRangeBytes, loadedLogBytes, sentBytes;
-		Counter sampledRangeBytes, sampledLogBytes;
-		Counter oldLogMutations;
-
-		Counters(LoaderBatchData* self, UID loaderInterfID, int batchIndex)
-		  : cc("LoaderBatch", loaderInterfID.toString() + ":" + std::to_string(batchIndex)),
-		    loadedRangeBytes("LoadedRangeBytes", cc), loadedLogBytes("LoadedLogBytes", cc), sentBytes("SentBytes", cc),
-		    sampledRangeBytes("SampledRangeBytes", cc), sampledLogBytes("SampledLogBytes", cc),
-		    oldLogMutations("OldLogMutations", cc) {}
-	} counters;
-
-	explicit LoaderBatchData(UID nodeID, int batchIndex)
-	  : vbState(LoaderVersionBatchState::NOT_INIT), loadFileReqs(0), counters(this, nodeID, batchIndex) {
-		pollMetrics =
-		    counters.cc.traceCounters(format("FastRestoreLoaderMetrics%d", batchIndex),
-		                              nodeID,
-		                              SERVER_KNOBS->FASTRESTORE_ROLE_LOGGING_DELAY,
-		                              nodeID.toString() + "/RestoreLoaderMetrics/" + std::to_string(batchIndex));
-		TraceEvent("FastRestoreLoaderMetricsCreated").detail("Node", nodeID);
-	}
-
-	void reset() {
-		processedFileParams.clear();
-		kvOpsPerLP.clear();
-		sampleMutations.clear();
-		numSampledMutations = 0;
-		rangeToApplier.clear();
-	}
-};
-
-using LoaderCounters = LoaderBatchData::Counters;
-
-struct LoaderBatchStatus : public ReferenceCounted<LoaderBatchStatus> {
-	Optional<Future<Void>> sendAllRanges;
-	Optional<Future<Void>> sendAllLogs;
-
-	void addref() { return ReferenceCounted<LoaderBatchStatus>::addref(); }
-	void delref() { return ReferenceCounted<LoaderBatchStatus>::delref(); }
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "sendAllRanges: "
-		   << (!sendAllRanges.present() ? "invalid" : (sendAllRanges.get().isReady() ? "ready" : "notReady"))
-		   << " sendAllLogs: "
-		   << (!sendAllLogs.present() ? "invalid" : (sendAllLogs.get().isReady() ? "ready" : "notReady"));
-		return ss.str();
-	}
-};
-
-// Each request for each loadingParam, so that scheduler can control which requests in which version batch to send first
-struct RestoreLoaderSchedSendLoadParamRequest {
-	int batchIndex;
-	Promise<Void> toSched;
-	double start;
-
-	explicit RestoreLoaderSchedSendLoadParamRequest(int batchIndex, Promise<Void> toSched, double start)
-	  : batchIndex(batchIndex), toSched(toSched), start(start) {};
-	RestoreLoaderSchedSendLoadParamRequest() = default;
-
-	bool operator<(RestoreLoaderSchedSendLoadParamRequest const& rhs) const {
-		return batchIndex > rhs.batchIndex || (batchIndex == rhs.batchIndex && start > rhs.start);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreLoaderSchedSendLoadParamRequest: "
-		   << " batchIndex:" << batchIndex << " toSchedFutureIsReady:" << toSched.getFuture().isReady()
-		   << " start:" << start;
-		return ss.str();
-	}
-};
-
-struct RestoreLoaderData : RestoreRoleData, public ReferenceCounted<RestoreLoaderData> {
-	// buffered data per version batch
-	std::map<int, Reference<LoaderBatchData>> batch;
-	std::map<int, Reference<LoaderBatchStatus>> status;
-	RestoreControllerInterface ci;
-
-	KeyRangeMap<Version> rangeVersions;
-
-	Reference<IBackupContainer> bc; // Backup container is used to read backup files
-	Key bcUrl; // The url used to get the bc
-
-	// Request scheduler
-	std::priority_queue<RestoreLoadFileRequest> loadingQueue; // request queue of loading files
-	std::priority_queue<RestoreSendMutationsToAppliersRequest>
-	    sendingQueue; // request queue of sending mutations to appliers
-	std::priority_queue<RestoreLoaderSchedSendLoadParamRequest> sendLoadParamQueue;
-	int finishedLoadingVB; // the max version batch index that finished loading file phase
-	int finishedSendingVB; // the max version batch index that finished sending mutations phase
-	int inflightSendingReqs; // number of sendingMutations requests released
-	int inflightLoadingReqs; // number of load backup file requests released
-	std::map<int, int> inflightSendLoadParamReqs; // key: batchIndex, value: inflightSendLoadParamReqs
-
-	Reference<AsyncVar<bool>> hasPendingRequests; // are there pending requests for loader
-
-	// addActor: add to actorCollection so that when an actor has error, the ActorCollection can catch the error.
-	// addActor is used to create the actorCollection when the RestoreController is created
-	PromiseStream<Future<Void>> addActor;
-
-	void addref() { return ReferenceCounted<RestoreLoaderData>::addref(); }
-	void delref() { return ReferenceCounted<RestoreLoaderData>::delref(); }
-
-	explicit RestoreLoaderData(UID loaderInterfID, int assignedIndex, RestoreControllerInterface ci)
-	  : ci(ci), finishedLoadingVB(0), finishedSendingVB(0), inflightSendingReqs(0), inflightLoadingReqs(0) {
-		nodeID = loaderInterfID;
-		nodeIndex = assignedIndex;
-		role = RestoreRole::Loader;
-		hasPendingRequests = makeReference<AsyncVar<bool>>(false);
-	}
-
-	~RestoreLoaderData() override = default;
-
-	std::string describeNode() const override {
-		std::stringstream ss;
-		ss << "[Role: Loader] [NodeID:" << nodeID.toString().c_str() << "] [NodeIndex:" << std::to_string(nodeIndex)
-		   << "]";
-		return ss.str();
-	}
-
-	int getVersionBatchState(int batchIndex) const final {
-		auto item = batch.find(batchIndex);
-		if (item == batch.end()) { // Batch has not been initialized when we blindly profile the state
-			return LoaderVersionBatchState::INVALID;
-		} else {
-			return item->second->vbState.get();
-		}
-	}
-	void setVersionBatchState(int batchIndex, int vbState) final {
-		std::map<int, Reference<LoaderBatchData>>::iterator item = batch.find(batchIndex);
-		ASSERT(item != batch.end());
-		item->second->vbState = vbState;
-	}
-
-	void initVersionBatch(int batchIndex) override {
-		TraceEvent("FastRestoreLoaderInitVersionBatch", nodeID).detail("BatchIndex", batchIndex);
-		batch[batchIndex] = makeReference<LoaderBatchData>(nodeID, batchIndex);
-		status[batchIndex] = makeReference<LoaderBatchStatus>();
-	}
-
-	void resetPerRestoreRequest() override {
-		batch.clear();
-		status.clear();
-		finishedBatch = NotifiedVersion(0);
-	}
-
-	void initBackupContainer(Key url, Optional<std::string> proxy) {
-		if (bcUrl == url && bc.isValid()) {
-			return;
-		}
-		bcUrl = url;
-		bc = IBackupContainer::openContainer(url.toString(), proxy, {});
-	}
-};
-
-ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf,
-                                     int nodeIndex,
-                                     Database cx,
-                                     RestoreControllerInterface ci);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/RestoreRoleCommon.actor.h b/fdbserver/include/fdbserver/RestoreRoleCommon.actor.h
deleted file mode 100644
index 7fc8261425d..00000000000
--- a/fdbserver/include/fdbserver/RestoreRoleCommon.actor.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * RestoreRoleCommon.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file declares common struct and functions shared by restore roles, i.e.,
-// RestoreController, RestoreLoader, RestoreApplier
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RestoreRoleCommon_G_H)
-#define FDBSERVER_RestoreRoleCommon_G_H
-#include "fdbserver/RestoreRoleCommon.actor.g.h"
-#elif !defined(FDBSERVER_RestoreRoleCommon_H)
-#define FDBSERVER_RestoreRoleCommon_H
-
-#include <sstream>
-#include "flow/SystemMonitor.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/CommitTransaction.h"
-#include "fdbclient/Notified.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/Stats.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-#include "fdbserver/RestoreUtil.h"
-
-#include "flow/actorcompiler.h" // has to be last include
-
-struct RestoreRoleInterface;
-struct RestoreLoaderInterface;
-struct RestoreApplierInterface;
-
-struct RestoreRoleData;
-struct RestoreControllerData;
-
-struct RestoreSimpleRequest;
-
-// Key is the (version, subsequence) of parsed backup mutations.
-// Value MutationsVec is the vector of parsed backup mutations.
-// For old mutation logs, the subsequence number is always 0.
-// For partitioned mutation logs, each mutation has a unique LogMessageVersion.
-// Note for partitioned logs, one LogMessageVersion can have multiple mutations,
-// because a clear mutation may be split into several smaller clear mutations by
-// backup workers.
-using VersionedMutationsMap = std::map<LogMessageVersion, MutationsVec>;
-
-ACTOR Future<Void> isSchedulable(Reference<RestoreRoleData> self, int actorBatchIndex, std::string name);
-ACTOR Future<Void> handleHeartbeat(RestoreSimpleRequest req, UID id);
-ACTOR Future<Void> handleInitVersionBatchRequest(RestoreVersionBatchRequest req, Reference<RestoreRoleData> self);
-void handleFinishRestoreRequest(const RestoreFinishRequest& req, Reference<RestoreRoleData> self);
-
-class RoleVersionBatchState {
-public:
-	static const int INVALID = -1;
-
-	virtual int get() const { return vbState; }
-
-	virtual void operator=(int newState) { vbState = newState; }
-
-	explicit RoleVersionBatchState() : vbState(INVALID) {}
-	explicit RoleVersionBatchState(int newState) : vbState(newState) {}
-
-	virtual ~RoleVersionBatchState() = default;
-
-	int vbState;
-};
-
-struct RestoreRoleData : NonCopyable, public ReferenceCounted<RestoreRoleData> {
-public:
-	RestoreRole role;
-	UID nodeID;
-	int nodeIndex;
-
-	double cpuUsage;
-	double memory;
-	double residentMemory;
-
-	AsyncTrigger checkMemory;
-	int delayedActors; // actors that are delayed to release because of low memory
-
-	std::map<UID, RestoreLoaderInterface> loadersInterf; // UID: loaderInterf's id
-	std::map<UID, RestoreApplierInterface> appliersInterf; // UID: applierInterf's id
-	Promise<Void> recruitedRoles; // sent when loaders and appliers are recruited
-
-	NotifiedVersion versionBatchId; // The index of the version batch that has been initialized and put into pipeline
-	NotifiedVersion finishedBatch; // The highest batch index all appliers have applied mutations
-
-	RestoreRoleData()
-	  : role(RestoreRole::Invalid), cpuUsage(0.0), memory(0.0), residentMemory(0.0), delayedActors(0) {};
-
-	virtual ~RestoreRoleData() = default;
-
-	UID id() const { return nodeID; }
-
-	virtual void initVersionBatch(int batchIndex) = 0;
-	virtual void resetPerRestoreRequest() = 0;
-	virtual int getVersionBatchState(int batchIndex) const = 0;
-	virtual void setVersionBatchState(int batchIndex, int vbState) = 0;
-
-	void clearInterfaces() {
-		loadersInterf.clear();
-		appliersInterf.clear();
-	}
-
-	virtual std::string describeNode() const = 0;
-};
-
-void updateProcessStats(Reference<RestoreRoleData> self);
-ACTOR Future<Void> updateProcessMetrics(Reference<RestoreRoleData> self);
-ACTOR Future<Void> traceProcessMetrics(Reference<RestoreRoleData> self, std::string role);
-ACTOR Future<Void> traceRoleVersionBatchProgress(Reference<RestoreRoleData> self, std::string role);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/RestoreUtil.h b/fdbserver/include/fdbserver/RestoreUtil.h
deleted file mode 100644
index 6e693f2d0cc..00000000000
--- a/fdbserver/include/fdbserver/RestoreUtil.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * RestoreUtil.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file defines the commonly used data structure and functions
-// that are used by both RestoreWorker and RestoreRoles(Controller, Loader, and Applier)
-
-#ifndef FDBSERVER_RESTOREUTIL_H
-#define FDBSERVER_RESTOREUTIL_H
-
-#pragma once
-
-#include "fdbclient/Tuple.h"
-#include "fdbclient/CommitTransaction.h"
-#include "fdbclient/RestoreInterface.h"
-#include "flow/flow.h"
-#include "fdbrpc/TimedRequest.h"
-#include "fdbrpc/fdbrpc.h"
-#include "flow/IAsyncFile.h"
-#include "fdbrpc/Stats.h"
-#include <cstdint>
-#include <cstdarg>
-
-#define SevFRMutationInfo SevVerbose
-// #define SevFRMutationInfo SevInfo
-
-#define SevFRDebugInfo SevVerbose
-// #define SevFRDebugInfo SevInfo
-
-struct VersionedMutationSerialized {
-	MutationRef mutation;
-	LogMessageVersion version;
-
-	VersionedMutationSerialized() = default;
-	explicit VersionedMutationSerialized(MutationRef mutation, LogMessageVersion version)
-	  : mutation(mutation), version(version) {}
-	explicit VersionedMutationSerialized(Arena& arena, const VersionedMutationSerialized& vm)
-	  : mutation(arena, vm.mutation), version(vm.version) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, mutation, version);
-	}
-};
-
-struct SampledMutation {
-	KeyRef key;
-	long size;
-
-	explicit SampledMutation(KeyRef key, long size) : key(key), size(size) {}
-	explicit SampledMutation(Arena& arena, const SampledMutation& sm) : key(arena, sm.key), size(sm.size) {}
-	SampledMutation() = default;
-
-	int totalSize() { return key.size() + sizeof(size); }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, key, size);
-	}
-};
-
-using MutationsVec = Standalone<VectorRef<MutationRef>>;
-using LogMessageVersionVec = Standalone<VectorRef<LogMessageVersion>>;
-using VersionedMutationsVec = Standalone<VectorRef<VersionedMutationSerialized>>;
-using SampledMutationsVec = Standalone<VectorRef<SampledMutation>>;
-
-enum class RestoreRole { Invalid = 0, Controller = 1, Loader, Applier };
-std::string getRoleStr(RestoreRole role);
-extern const std::vector<std::string> RestoreRoleStr;
-extern int numRoles;
-
-std::string getHexString(StringRef input);
-
-bool debugFRMutation(const char* context, Version version, MutationRef const& mutation);
-
-struct RestoreSimpleRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 16448937;
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreSimpleRequest() = default;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreSimpleRequest";
-		return ss.str();
-	}
-};
-
-bool isRangeMutation(MutationRef m);
-
-#endif // FDBSERVER_RESTOREUTIL_H
diff --git a/fdbserver/include/fdbserver/RestoreWorker.actor.h b/fdbserver/include/fdbserver/RestoreWorker.actor.h
deleted file mode 100644
index fb3196015cd..00000000000
--- a/fdbserver/include/fdbserver/RestoreWorker.actor.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * RestoreWorker.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTOREWORKER_G_H)
-#define FDBSERVER_RESTOREWORKER_G_H
-#include "fdbserver/RestoreWorker.actor.g.h"
-#elif !defined(FDBSERVER_RESTOREWORKER_H)
-#define FDBSERVER_RESTOREWORKER_H
-
-#include "fdbclient/Tuple.h"
-#include "flow/flow.h"
-#include "fdbrpc/fdbrpc.h"
-#include "flow/IAsyncFile.h"
-#include "fdbrpc/Stats.h"
-#include <cstdint>
-#include <cstdarg>
-
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/RestoreCommon.actor.h"
-#include "fdbserver/RestoreRoleCommon.actor.h"
-#include "fdbserver/RestoreLoader.actor.h"
-#include "fdbserver/RestoreApplier.actor.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-
-#include "flow/actorcompiler.h" // must be last include
-
-// Each restore worker (a process) is assigned for a role.
-// MAYBE Later: We will support multiple restore roles on a worker
-struct RestoreWorkerData : NonCopyable, public ReferenceCounted<RestoreWorkerData> {
-	UID workerID;
-	std::map<UID, RestoreWorkerInterface>
-	    workerInterfaces; // UID is worker's node id, RestoreWorkerInterface is worker's communication workerInterface
-
-	// Restore Roles
-	Optional<RestoreControllerInterface> controllerInterf;
-	Optional<RestoreLoaderInterface> loaderInterf;
-	Optional<RestoreApplierInterface> applierInterf;
-
-	UID id() const { return workerID; };
-
-	RestoreWorkerData() = default;
-
-	~RestoreWorkerData() {
-		TraceEvent("RestoreWorkerDataDeleted").detail("WorkerID", workerID.toString());
-		printf("[Exit] Worker:%s RestoreWorkerData is deleted\n", workerID.toString().c_str());
-	}
-
-	std::string describeNode() {
-		std::stringstream ss;
-		ss << "RestoreWorker workerID:" << workerID.toString();
-		return ss.str();
-	}
-};
-
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_RESTOREWORKER_H
diff --git a/fdbserver/include/fdbserver/RestoreWorkerInterface.actor.h b/fdbserver/include/fdbserver/RestoreWorkerInterface.actor.h
deleted file mode 100644
index e6ce8b7a510..00000000000
--- a/fdbserver/include/fdbserver/RestoreWorkerInterface.actor.h
+++ /dev/null
@@ -1,739 +0,0 @@
-/*
- * RestoreWorkerInterface.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file declare and define the interface for RestoreWorker and restore roles
-// which are RestoreController, RestoreLoader, and RestoreApplier
-
-#pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_G_H)
-#define FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_G_H
-#include "fdbserver/RestoreWorkerInterface.actor.g.h"
-#elif !defined(FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_H)
-#define FDBSERVER_RESTORE_WORKER_INTERFACE_ACTOR_H
-
-#include <sstream>
-#include <string>
-#include "flow/flow.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/Stats.h"
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/CommitTransaction.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RestoreUtil.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-class RestoreConfigFR;
-
-struct RestoreCommonReply;
-struct RestoreRecruitRoleRequest;
-struct RestoreSysInfoRequest;
-struct RestoreLoadFileRequest;
-struct RestoreVersionBatchRequest;
-struct RestoreSendMutationsToAppliersRequest;
-struct RestoreSendVersionedMutationsRequest;
-struct RestoreSysInfo;
-struct RestoreApplierInterface;
-struct RestoreFinishRequest;
-struct RestoreSamplesRequest;
-struct RestoreUpdateRateRequest;
-
-// RestoreSysInfo includes information each (type of) restore roles should know.
-// At this moment, it only include appliers. We keep the name for future extension.
-// TODO: If it turns out this struct only has appliers in the final version, we will rename it to a more specific name,
-// e.g., AppliersMap
-struct RestoreSysInfo {
-	constexpr static FileIdentifier file_identifier = 68098739;
-	std::map<UID, RestoreApplierInterface> appliers;
-
-	RestoreSysInfo() = default;
-	explicit RestoreSysInfo(const std::map<UID, RestoreApplierInterface> appliers) : appliers(appliers) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, appliers);
-	}
-};
-
-struct RestoreWorkerInterface {
-	constexpr static FileIdentifier file_identifier = 15715718;
-	UID interfID;
-
-	RequestStream<RestoreSimpleRequest> heartbeat;
-	RequestStream<RestoreRecruitRoleRequest> recruitRole;
-	RequestStream<RestoreSimpleRequest> terminateWorker;
-
-	bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
-	bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
-
-	UID id() const { return interfID; } // cmd.getEndpoint().token;
-
-	NetworkAddress address() const { return recruitRole.getEndpoint().addresses.address; }
-
-	void initEndpoints() {
-		heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		recruitRole.getEndpoint(TaskPriority::LoadBalancedEndpoint); // Q: Why do we need this?
-		terminateWorker.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-
-		interfID = deterministicRandom()->randomUniqueID();
-	}
-
-	// To change this serialization, ProtocolVersion::RestoreWorkerInterfaceValue must be updated, and downgrades need
-	// to be considered
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, interfID, heartbeat, recruitRole, terminateWorker);
-	}
-};
-
-struct RestoreRoleInterface {
-	constexpr static FileIdentifier file_identifier = 12199691;
-	UID nodeID;
-	RestoreRole role;
-
-	RestoreRoleInterface() { role = RestoreRole::Invalid; }
-
-	explicit RestoreRoleInterface(RestoreRoleInterface const& interf) : nodeID(interf.nodeID), role(interf.role) {};
-
-	UID id() const { return nodeID; }
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "Role:" << getRoleStr(role) << " interfID:" << nodeID.toString();
-		return ss.str();
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, nodeID, role);
-	}
-};
-
-struct RestoreLoaderInterface : RestoreRoleInterface {
-	constexpr static FileIdentifier file_identifier = 358571;
-
-	RequestStream<RestoreSimpleRequest> heartbeat;
-	RequestStream<RestoreSysInfoRequest> updateRestoreSysInfo;
-	RequestStream<RestoreLoadFileRequest> loadFile;
-	RequestStream<RestoreSendMutationsToAppliersRequest> sendMutations;
-	RequestStream<RestoreVersionBatchRequest> initVersionBatch;
-	RequestStream<RestoreVersionBatchRequest> finishVersionBatch;
-	RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
-	RequestStream<RestoreFinishRequest> finishRestore;
-
-	bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
-	bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
-
-	RestoreLoaderInterface() {
-		role = RestoreRole::Loader;
-		nodeID = deterministicRandom()->randomUniqueID();
-	}
-
-	NetworkAddress address() const { return heartbeat.getEndpoint().addresses.address; }
-
-	void initEndpoints() {
-		// Endpoint in a later restore phase has higher priority
-		heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		updateRestoreSysInfo.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		loadFile.getEndpoint(TaskPriority::RestoreLoaderLoadFiles);
-		sendMutations.getEndpoint(TaskPriority::RestoreLoaderSendMutations);
-		finishVersionBatch.getEndpoint(TaskPriority::RestoreLoaderFinishVersionBatch);
-		collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar,
-		           *(RestoreRoleInterface*)this,
-		           heartbeat,
-		           updateRestoreSysInfo,
-		           loadFile,
-		           sendMutations,
-		           initVersionBatch,
-		           finishVersionBatch,
-		           collectRestoreRoleInterfaces,
-		           finishRestore);
-	}
-};
-
-struct RestoreApplierInterface : RestoreRoleInterface {
-	constexpr static FileIdentifier file_identifier = 3921400;
-
-	RequestStream<RestoreSimpleRequest> heartbeat;
-	RequestStream<RestoreSendVersionedMutationsRequest> sendMutationVector;
-	RequestStream<RestoreVersionBatchRequest> applyToDB;
-	RequestStream<RestoreVersionBatchRequest> initVersionBatch;
-	RequestStream<RestoreSimpleRequest> collectRestoreRoleInterfaces;
-	RequestStream<RestoreFinishRequest> finishRestore;
-	RequestStream<RestoreUpdateRateRequest> updateRate;
-
-	bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
-	bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
-
-	RestoreApplierInterface() {
-		role = RestoreRole::Applier;
-		nodeID = deterministicRandom()->randomUniqueID();
-	}
-
-	NetworkAddress address() const { return heartbeat.getEndpoint().addresses.address; }
-
-	void initEndpoints() {
-		// Endpoint in a later restore phase has higher priority
-		heartbeat.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		sendMutationVector.getEndpoint(TaskPriority::RestoreApplierReceiveMutations);
-		applyToDB.getEndpoint(TaskPriority::RestoreApplierWriteDB);
-		initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-		updateRate.getEndpoint(TaskPriority::LoadBalancedEndpoint);
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar,
-		           *(RestoreRoleInterface*)this,
-		           heartbeat,
-		           sendMutationVector,
-		           applyToDB,
-		           initVersionBatch,
-		           collectRestoreRoleInterfaces,
-		           finishRestore,
-		           updateRate);
-	}
-
-	std::string toString() const { return nodeID.toString(); }
-};
-
-struct RestoreControllerInterface : RestoreRoleInterface {
-	constexpr static FileIdentifier file_identifier = 11642024;
-
-	RequestStream<RestoreSamplesRequest> samples;
-
-	bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
-	bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
-
-	RestoreControllerInterface() {
-		role = RestoreRole::Controller;
-		nodeID = deterministicRandom()->randomUniqueID();
-	}
-
-	NetworkAddress address() const { return samples.getEndpoint().addresses.address; }
-
-	void initEndpoints() { samples.getEndpoint(TaskPriority::LoadBalancedEndpoint); }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, *(RestoreRoleInterface*)this, samples);
-	}
-
-	std::string toString() const { return nodeID.toString(); }
-};
-
-// RestoreAsset uniquely identifies the work unit done by restore roles;
-// It is used to ensure exact-once processing on restore loader and applier;
-// By combining all RestoreAssets across all version batches, restore should process all mutations in
-// backup range and log files up to the target restore version.
-struct RestoreAsset {
-	UID uid;
-
-	Version beginVersion, endVersion; // Only use mutation in [begin, end) versions;
-	KeyRange range; // Only use mutations in range
-
-	int fileIndex;
-	// Partition ID for mutation log files, which is also encoded in the filename of mutation logs.
-	int partitionId = -1;
-	std::string filename;
-	int64_t offset;
-	int64_t len;
-
-	Key addPrefix;
-	Key removePrefix;
-
-	int batchIndex; // for progress tracking and performance investigation
-
-	RestoreAsset() = default;
-
-	// Q: Can we simply use uid for == and use different comparison rule for less than operator.
-	// The ordering of RestoreAsset may change, will that affect correctness or performance?
-	bool operator==(const RestoreAsset& r) const {
-		return batchIndex == r.batchIndex && beginVersion == r.beginVersion && endVersion == r.endVersion &&
-		       range == r.range && fileIndex == r.fileIndex && partitionId == r.partitionId && filename == r.filename &&
-		       offset == r.offset && len == r.len && addPrefix == r.addPrefix && removePrefix == r.removePrefix;
-	}
-	bool operator!=(const RestoreAsset& r) const { return !(*this == r); }
-	bool operator<(const RestoreAsset& r) const {
-		return std::make_tuple(batchIndex,
-		                       fileIndex,
-		                       filename,
-		                       offset,
-		                       len,
-		                       beginVersion,
-		                       endVersion,
-		                       range.begin,
-		                       range.end,
-		                       addPrefix,
-		                       removePrefix) < std::make_tuple(r.batchIndex,
-		                                                       r.fileIndex,
-		                                                       r.filename,
-		                                                       r.offset,
-		                                                       r.len,
-		                                                       r.beginVersion,
-		                                                       r.endVersion,
-		                                                       r.range.begin,
-		                                                       r.range.end,
-		                                                       r.addPrefix,
-		                                                       r.removePrefix);
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar,
-		           uid,
-		           beginVersion,
-		           endVersion,
-		           range,
-		           filename,
-		           fileIndex,
-		           partitionId,
-		           offset,
-		           len,
-		           addPrefix,
-		           removePrefix,
-		           batchIndex);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion
-		   << " range:" << range.toString() << " filename:" << filename << " fileIndex:" << fileIndex
-		   << " partitionId:" << partitionId << " offset:" << offset << " len:" << len
-		   << " addPrefix:" << addPrefix.toString() << " removePrefix:" << removePrefix.toString()
-		   << " BatchIndex:" << batchIndex;
-		return ss.str();
-	}
-
-	bool hasPrefix() const { return addPrefix.size() > 0 || removePrefix.size() > 0; }
-
-	// RestoreAsset and VersionBatch both use endVersion as exclusive in version range
-	bool isInVersionRange(Version commitVersion) const {
-		return commitVersion >= beginVersion && commitVersion < endVersion;
-	}
-
-	// Is mutation's begin and end keys are in RestoreAsset's range
-	bool isInKeyRange(MutationRef mutation) const {
-		if (hasPrefix()) {
-			Key begin = range.begin; // Avoid creating new keys if we do not have addPrefix or removePrefix
-			Key end = range.end;
-			begin = begin.removePrefix(removePrefix).withPrefix(addPrefix);
-			end = end.removePrefix(removePrefix).withPrefix(addPrefix);
-			if (isRangeMutation(mutation)) {
-				// Range mutation's right side is exclusive
-				return mutation.param1 >= begin && mutation.param2 <= end;
-			} else {
-				return mutation.param1 >= begin && mutation.param1 < end;
-			}
-		} else {
-			if (isRangeMutation(mutation)) {
-				// Range mutation's right side is exclusive
-				return mutation.param1 >= range.begin && mutation.param2 <= range.end;
-			} else {
-				return mutation.param1 >= range.begin && mutation.param1 < range.end;
-			}
-		}
-	}
-};
-
-struct LoadingParam {
-	constexpr static FileIdentifier file_identifier = 246621;
-
-	bool isRangeFile;
-	Key url;
-	Optional<std::string> proxy;
-	Optional<Version> rangeVersion; // range file's version
-
-	int64_t blockSize;
-	RestoreAsset asset;
-
-	LoadingParam() = default;
-
-	// TODO: Compare all fields for loadingParam
-	bool operator==(const LoadingParam& r) const { return isRangeFile == r.isRangeFile && asset == r.asset; }
-	bool operator!=(const LoadingParam& r) const { return isRangeFile != r.isRangeFile || asset != r.asset; }
-	bool operator<(const LoadingParam& r) const {
-		return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && asset < r.asset);
-	}
-
-	bool isPartitionedLog() const { return !isRangeFile && asset.partitionId >= 0; }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, isRangeFile, url, proxy, rangeVersion, blockSize, asset);
-	}
-
-	std::string toString() const {
-		std::stringstream str;
-		str << "isRangeFile:" << isRangeFile << " url:" << url.toString()
-		    << " proxy:" << (proxy.present() ? proxy.get() : "")
-		    << " rangeVersion:" << (rangeVersion.present() ? rangeVersion.get() : -1) << " blockSize:" << blockSize
-		    << " RestoreAsset:" << asset.toString();
-		return str.str();
-	}
-};
-
-struct RestoreRecruitRoleReply : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 13532876;
-
-	UID id;
-	RestoreRole role;
-	Optional<RestoreLoaderInterface> loader;
-	Optional<RestoreApplierInterface> applier;
-
-	RestoreRecruitRoleReply() = default;
-	explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreLoaderInterface const& loader)
-	  : id(id), role(role), loader(loader) {}
-	explicit RestoreRecruitRoleReply(UID id, RestoreRole role, RestoreApplierInterface const& applier)
-	  : id(id), role(role), applier(applier) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, id, role, loader, applier);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "roleInterf role:" << getRoleStr(role) << " replyID:" << id.toString();
-		if (loader.present()) {
-			ss << "loader:" << loader.get().toString();
-		}
-		if (applier.present()) {
-			ss << "applier:" << applier.get().toString();
-		}
-
-		return ss.str();
-	}
-};
-
-struct RestoreRecruitRoleRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 3136280;
-
-	RestoreControllerInterface ci;
-	RestoreRole role;
-	int nodeIndex; // Each role is a node
-
-	ReplyPromise<RestoreRecruitRoleReply> reply;
-
-	RestoreRecruitRoleRequest() : role(RestoreRole::Invalid) {}
-	explicit RestoreRecruitRoleRequest(RestoreControllerInterface ci, RestoreRole role, int nodeIndex)
-	  : ci(ci), role(role), nodeIndex(nodeIndex) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, ci, role, nodeIndex, reply);
-	}
-
-	std::string printable() const {
-		std::stringstream ss;
-		ss << "RestoreRecruitRoleRequest Role:" << getRoleStr(role) << " NodeIndex:" << nodeIndex
-		   << " RestoreController:" << ci.id().toString();
-		return ss.str();
-	}
-
-	std::string toString() const { return printable(); }
-};
-
-// Static info. across version batches
-struct RestoreSysInfoRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 8851877;
-
-	RestoreSysInfo sysInfo;
-	Standalone<VectorRef<std::pair<KeyRangeRef, Version>>> rangeVersions;
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreSysInfoRequest() = default;
-	explicit RestoreSysInfoRequest(RestoreSysInfo sysInfo,
-	                               Standalone<VectorRef<std::pair<KeyRangeRef, Version>>> rangeVersions)
-	  : sysInfo(sysInfo), rangeVersions(rangeVersions) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, sysInfo, rangeVersions, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreSysInfoRequest "
-		   << "rangeVersions.size:" << rangeVersions.size();
-		return ss.str();
-	}
-};
-
-struct RestoreSamplesRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 10751035;
-	UID id; // deduplicate data
-	int batchIndex;
-	SampledMutationsVec samples; // sampled mutations
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreSamplesRequest() = default;
-	explicit RestoreSamplesRequest(UID id, int batchIndex, SampledMutationsVec samples)
-	  : id(id), batchIndex(batchIndex), samples(samples) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, id, batchIndex, samples, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "ID:" << id.toString() << " BatchIndex:" << batchIndex << " samples:" << samples.size();
-		return ss.str();
-	}
-};
-
-struct RestoreLoadFileReply : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 523470;
-
-	LoadingParam param;
-	bool isDuplicated; // true if loader thinks the request is a duplicated one
-
-	RestoreLoadFileReply() = default;
-	explicit RestoreLoadFileReply(LoadingParam param, bool isDuplicated) : param(param), isDuplicated(isDuplicated) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, param, isDuplicated);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "LoadingParam:" << param.toString() << " isDuplicated:" << isDuplicated;
-		return ss.str();
-	}
-};
-
-// Sample_Range_File and Assign_Loader_Range_File, Assign_Loader_Log_File
-struct RestoreLoadFileRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 9780148;
-
-	int batchIndex;
-	LoadingParam param;
-
-	ReplyPromise<RestoreLoadFileReply> reply;
-
-	RestoreLoadFileRequest() = default;
-	explicit RestoreLoadFileRequest(int batchIndex, LoadingParam& param) : batchIndex(batchIndex), param(param) {};
-
-	bool operator<(RestoreLoadFileRequest const& rhs) const { return batchIndex > rhs.batchIndex; }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, param, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreLoadFileRequest batchIndex:" << batchIndex << " param:" << param.toString();
-		return ss.str();
-	}
-};
-
-struct RestoreSendMutationsToAppliersRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 1718441;
-
-	int batchIndex; // version batch index
-	std::map<Key, UID> rangeToApplier;
-	bool useRangeFile; // Send mutations parsed from range file?
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreSendMutationsToAppliersRequest() = default;
-	explicit RestoreSendMutationsToAppliersRequest(int batchIndex, std::map<Key, UID> rangeToApplier, bool useRangeFile)
-	  : batchIndex(batchIndex), rangeToApplier(rangeToApplier), useRangeFile(useRangeFile) {}
-
-	bool operator<(RestoreSendMutationsToAppliersRequest const& rhs) const { return batchIndex > rhs.batchIndex; }
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, rangeToApplier, useRangeFile, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreSendMutationsToAppliersRequest batchIndex:" << batchIndex
-		   << " keyToAppliers.size:" << rangeToApplier.size() << " useRangeFile:" << useRangeFile;
-		return ss.str();
-	}
-};
-
-struct RestoreSendVersionedMutationsRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 2655701;
-
-	int batchIndex; // version batch index
-	RestoreAsset asset; // Unique identifier for the current restore asset
-
-	Version msgIndex; // Monitonically increasing index of mutation messages
-	bool isRangeFile;
-	VersionedMutationsVec versionedMutations; // Versioned mutations may be at different versions parsed by one loader
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreSendVersionedMutationsRequest() = default;
-	explicit RestoreSendVersionedMutationsRequest(int batchIndex,
-	                                              const RestoreAsset& asset,
-	                                              Version msgIndex,
-	                                              bool isRangeFile,
-	                                              VersionedMutationsVec versionedMutations)
-	  : batchIndex(batchIndex), asset(asset), msgIndex(msgIndex), isRangeFile(isRangeFile),
-	    versionedMutations(versionedMutations) {}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "VersionBatchIndex:" << batchIndex << " msgIndex:" << msgIndex << " isRangeFile:" << isRangeFile
-		   << " versionedMutations.size:" << versionedMutations.size() << " RestoreAsset:" << asset.toString();
-		return ss.str();
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, asset, msgIndex, isRangeFile, versionedMutations, reply);
-	}
-};
-
-struct RestoreVersionBatchRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 13337457;
-
-	int batchIndex;
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreVersionBatchRequest() = default;
-	explicit RestoreVersionBatchRequest(int batchIndex) : batchIndex(batchIndex) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreVersionBatchRequest batchIndex:" << batchIndex;
-		return ss.str();
-	}
-};
-
-struct RestoreFinishRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 13018413;
-
-	bool terminate; // role exits if terminate = true
-
-	ReplyPromise<RestoreCommonReply> reply;
-
-	RestoreFinishRequest() = default;
-	explicit RestoreFinishRequest(bool terminate) : terminate(terminate) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, terminate, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreFinishRequest terminate:" << terminate;
-		return ss.str();
-	}
-};
-
-struct RestoreUpdateRateReply : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 13018414;
-
-	UID id;
-	double remainMB; // remaining data in MB to write to DB;
-
-	RestoreUpdateRateReply() = default;
-	explicit RestoreUpdateRateReply(UID id, double remainMB) : id(id), remainMB(remainMB) {}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreUpdateRateReply NodeID:" << id.toString() << " remainMB:" << remainMB;
-		return ss.str();
-	}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, id, remainMB);
-	}
-};
-
-struct RestoreUpdateRateRequest : TimedRequest {
-	constexpr static FileIdentifier file_identifier = 13018415;
-
-	int batchIndex;
-	double writeMB;
-
-	ReplyPromise<RestoreUpdateRateReply> reply;
-
-	RestoreUpdateRateRequest() = default;
-	explicit RestoreUpdateRateRequest(int batchIndex, double writeMB) : batchIndex(batchIndex), writeMB(writeMB) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, batchIndex, writeMB, reply);
-	}
-
-	std::string toString() const {
-		std::stringstream ss;
-		ss << "RestoreUpdateRateRequest batchIndex:" << batchIndex << " writeMB:" << writeMB;
-		return ss.str();
-	}
-};
-
-std::string getRoleStr(RestoreRole role);
-
-////--- Interface functions
-ACTOR Future<Void> _restoreWorker(Database cx, LocalityData locality);
-ACTOR Future<Void> restoreWorker(Reference<IClusterConnectionRecord> ccr,
-                                 LocalityData locality,
-                                 std::string coordFolder);
-
-extern const KeyRef restoreLeaderKey;
-extern const KeyRangeRef restoreWorkersKeys;
-extern const KeyRef restoreStatusKey; // To be used when we measure fast restore performance
-extern const KeyRangeRef restoreRequestKeys;
-extern const KeyRangeRef restoreApplierKeys;
-extern const KeyRef restoreApplierTxnValue;
-
-const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version);
-std::tuple<UID, int64_t, Version> decodeRestoreApplierKey(ValueRef const& key);
-const Key restoreWorkerKeyFor(UID const& workerID);
-const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& server);
-RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value);
-Version decodeRestoreRequestDoneVersionValue(ValueRef const& value);
-RestoreRequest decodeRestoreRequestValue(ValueRef const& value);
-const Key restoreStatusKeyFor(StringRef statusType);
-const Value restoreStatusValue(double val);
-Value restoreRequestDoneVersionValue(Version readVersion);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/ServerThroughputTracker.h b/fdbserver/include/fdbserver/ServerThroughputTracker.h
deleted file mode 100644
index 578d3db55b3..00000000000
--- a/fdbserver/include/fdbserver/ServerThroughputTracker.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * ServerThroughputTracker.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbrpc/Smoother.h"
-#include "fdbserver/Ratekeeper.h"
-
-// The ServerThroughputTracker class is responsible for tracking
-// every tag's reported throughput across all storage servers
-class ServerThroughputTracker {
-	enum class OpType {
-		READ,
-		WRITE,
-	};
-
-	class ThroughputCounters {
-		Smoother readThroughput;
-		Smoother writeThroughput;
-
-	public:
-		ThroughputCounters();
-		void updateThroughput(double newThroughput, OpType);
-		double getThroughput() const;
-	};
-
-	std::unordered_map<UID, TransactionTagMap<ThroughputCounters>> throughput;
-
-	void cleanupUnseenStorageServers(std::unordered_set<UID> const& seen);
-
-	static void cleanupUnseenTags(TransactionTagMap<ThroughputCounters>&,
-	                              std::unordered_set<TransactionTag> const& seenReadTags,
-	                              std::unordered_set<TransactionTag> const& seenWriteTags);
-
-public:
-	~ServerThroughputTracker();
-
-	// Returns all tags running significant workload on the specified storage server.
-	std::vector<TransactionTag> getTagsAffectingStorageServer(UID storageServerId) const;
-
-	// Updates throughput statistics based on new storage queue info
-	void update(Map<UID, StorageQueueInfo> const&);
-
-	// Returns the current throughput for the provided tag on the
-	// provided storage server
-	Optional<double> getThroughput(UID storageServerId, TransactionTag const&) const;
-
-	// Returns the current cluster-wide throughput for the provided tag
-	double getThroughput(TransactionTag const&) const;
-
-	// Returns the current throughput on the provided storage server, summed
-	// across all throttling IDs
-	Optional<double> getThroughput(UID storageServerId) const;
-
-	// Used to remove a tag which has expired
-	void removeTag(TransactionTag const&);
-
-	// Returns the number of storage servers currently being tracked
-	int storageServersTracked() const;
-};
diff --git a/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h b/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h
deleted file mode 100644
index 894ffa211e0..00000000000
--- a/fdbserver/include/fdbserver/SimEncryptKmsProxy.actor.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * SimEncryptKmsProxy.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include "flow/Arena.h"
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_SIMENCRYPTKMSPROXY_ACTOR_G_H)
-#define FDBSERVER_SIMENCRYPTKMSPROXY_ACTOR_G_H
-#include "fdbserver/SimEncryptKmsProxy.actor.g.h"
-#elif !defined(FDBSERVER_SIMENCRYPTKMSPROXY_ACTOR_H)
-#define FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_H
-
-#include "fdbclient/FDBTypes.h"
-#include "fdbrpc/fdbrpc.h"
-#include "flow/EncryptUtils.h"
-#include "flow/FileIdentifier.h"
-#include "flow/Trace.h"
-#include "flow/flow.h"
-#include "flow/network.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-using SimEncryptKey = std::string;
-
-struct SimKmsProxyInterface {
-	constexpr static FileIdentifier file_identifier = 2416711;
-	RequestStream<ReplyPromise<Void>> waitFailure;
-	RequestStream<struct SimGetEncryptKeysByKeyIdsRequest> encryptKeyLookupByKeyIds;
-	RequestStream<struct SimGetEncryptKeysByDomainIdsRequest> encryptKeyLookupByDomainId;
-
-	SimKmsProxyInterface() {}
-
-	UID id() const { return encryptKeyLookupByKeyIds.getEndpoint().token; }
-	template <class Archive>
-	void serialize(Archive& ar) {
-		if constexpr (!is_fb_function<Archive>) {
-			ASSERT(ar.protocolVersion().isValid());
-		}
-		serializer(ar, waitFailure);
-		if (Archive::isDeserializing) {
-			encryptKeyLookupByKeyIds = RequestStream<struct SimGetEncryptKeysByKeyIdsRequest>(
-			    waitFailure.getEndpoint().getAdjustedEndpoint(1));
-			encryptKeyLookupByDomainId = RequestStream<struct SimGetEncryptKeysByDomainIdsRequest>(
-			    waitFailure.getEndpoint().getAdjustedEndpoint(2));
-		}
-	}
-
-	void initEndpoints() {
-		std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
-		streams.push_back(waitFailure.getReceiver());
-		streams.push_back(encryptKeyLookupByKeyIds.getReceiver(TaskPriority::DefaultPromiseEndpoint));
-		streams.push_back(encryptKeyLookupByDomainId.getReceiver(TaskPriority::DefaultPromiseEndpoint));
-		FlowTransport::transport().addEndpoints(streams);
-	}
-};
-
-struct SimEncryptKeyDetails {
-	constexpr static FileIdentifier file_identifier = 1227025;
-	EncryptCipherDomainId encryptDomainId;
-	EncryptCipherBaseKeyId encryptKeyId;
-	StringRef encryptKey;
-
-	SimEncryptKeyDetails() {}
-	explicit SimEncryptKeyDetails(EncryptCipherDomainId domainId,
-	                              EncryptCipherBaseKeyId keyId,
-	                              StringRef key,
-	                              Arena& arena)
-	  : encryptDomainId(domainId), encryptKeyId(keyId), encryptKey(StringRef(arena, key)) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptDomainId, encryptKeyId, encryptKey);
-	}
-};
-
-struct SimGetEncryptKeysByKeyIdsReply {
-	constexpr static FileIdentifier file_identifier = 2313778;
-	Arena arena;
-	std::vector<SimEncryptKeyDetails> encryptKeyDetails;
-
-	SimGetEncryptKeysByKeyIdsReply() {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptKeyDetails, arena);
-	}
-};
-
-struct SimGetEncryptKeysByKeyIdsRequest {
-	constexpr static FileIdentifier file_identifier = 6913396;
-	std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>> encryptKeyIds;
-	ReplyPromise<SimGetEncryptKeysByKeyIdsReply> reply;
-
-	SimGetEncryptKeysByKeyIdsRequest() {}
-	explicit SimGetEncryptKeysByKeyIdsRequest(
-	    const std::vector<std::pair<EncryptCipherBaseKeyId, EncryptCipherDomainId>>& keyIds)
-	  : encryptKeyIds(keyIds) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptKeyIds, reply);
-	}
-};
-
-struct SimGetEncryptKeyByDomainIdReply {
-	constexpr static FileIdentifier file_identifier = 3009025;
-	Arena arena;
-	std::vector<SimEncryptKeyDetails> encryptKeyDetails;
-
-	SimGetEncryptKeyByDomainIdReply() {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptKeyDetails, arena);
-	}
-};
-
-struct SimGetEncryptKeysByDomainIdsRequest {
-	constexpr static FileIdentifier file_identifier = 9918682;
-	std::vector<EncryptCipherDomainId> encryptDomainIds;
-	ReplyPromise<SimGetEncryptKeyByDomainIdReply> reply;
-
-	SimGetEncryptKeysByDomainIdsRequest() {}
-	explicit SimGetEncryptKeysByDomainIdsRequest(const std::vector<EncryptCipherDomainId>& ids)
-	  : encryptDomainIds(ids) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, encryptDomainIds, reply);
-	}
-};
-
-ACTOR Future<Void> simEncryptKmsProxyCore(struct SimKmsProxyInterface interf);
-
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_H
diff --git a/fdbserver/include/fdbserver/TSSMappingUtil.actor.h b/fdbserver/include/fdbserver/TSSMappingUtil.actor.h
deleted file mode 100644
index 424411e964d..00000000000
--- a/fdbserver/include/fdbserver/TSSMappingUtil.actor.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * TSSMappingUtil.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(TSS_MAPPING_UTIL_SERVER_G_H)
-#define TSS_MAPPING_UTIL_SERVER_G_H
-#include "fdbserver/TSSMappingUtil.actor.g.h"
-#elif !defined(TSS_MAPPING_UTIL_SERVER_H)
-#define TSS_MAPPING_UTIL_SERVER_H
-
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/StorageServerInterface.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-/*
- * Collection of utility functions for dealing with the TSS mapping
- */
-
-// Reads the current cluster TSS mapping as part of the RYW transaction
-ACTOR Future<Void> readTSSMappingRYW(Reference<ReadYourWritesTransaction> tr,
-                                     std::map<UID, StorageServerInterface>* tssMapping);
-
-// Reads the current cluster TSS mapping as part of the given Transaction
-ACTOR Future<Void> readTSSMapping(Transaction* tr, std::map<UID, StorageServerInterface>* tssMapping);
-
-// Removes the TSS pairs from the cluster
-ACTOR Future<Void> removeTSSPairsFromCluster(Database cx, std::vector<std::pair<UID, UID>> pairsToRemove);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h b/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h
deleted file mode 100644
index 0fcfb31a658..00000000000
--- a/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * TagPartitionedLogSystem.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_TAGPARTITIONEDLOGSYSTEM_ACTOR_G_H)
-#define FDBSERVER_TAGPARTITIONEDLOGSYSTEM_ACTOR_G_H
-#include "fdbserver/TagPartitionedLogSystem.actor.g.h"
-#elif !defined(FDBSERVER_TAGPARTITIONEDLOGSYSTEM_ACTOR_H)
-#define FDBSERVER_TAGPARTITIONEDLOGSYSTEM_ACTOR_H
-
-#pragma once
-
-#include "fdbrpc/Replication.h"
-#include "fdbserver/DBCoreState.h"
-#include "fdbserver/LogSystem.h"
-#include "flow/ActorCollection.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// TagPartitionedLogSystem info in old epoch
-struct OldLogData {
-	std::vector<Reference<LogSet>> tLogs;
-	int32_t logRouterTags;
-	int32_t txsTags; // The number of txsTags, which may change across generations.
-	Version epochBegin, epochEnd;
-	Version recoverAt;
-	std::set<int8_t> pseudoLocalities;
-	LogEpoch epoch;
-
-	OldLogData() : logRouterTags(0), txsTags(0), epochBegin(0), epochEnd(0), recoverAt(0), epoch(0) {}
-
-	// Constructor for T of OldTLogConf and OldTLogCoreData
-	template <class T>
-	explicit OldLogData(const T& conf)
-	  : logRouterTags(conf.logRouterTags), txsTags(conf.txsTags), epochBegin(conf.epochBegin), epochEnd(conf.epochEnd),
-	    recoverAt(conf.recoverAt), pseudoLocalities(conf.pseudoLocalities), epoch(conf.epoch) {
-		tLogs.resize(conf.tLogs.size());
-		for (int j = 0; j < conf.tLogs.size(); j++) {
-			auto logSet = makeReference<LogSet>(conf.tLogs[j]);
-			tLogs[j] = logSet;
-		}
-	}
-};
-
-struct IdToInterf : ReferenceCounted<IdToInterf> {
-	Optional<Version> recoverAt = Optional<Version>();
-	std::map<UID, TLogInterface> lockInterf;
-};
-
-struct LogLockInfo {
-	Version epochEnd;
-	bool isCurrent;
-	Reference<LogSet> logSet;
-	std::vector<Future<TLogLockResult>> replies;
-
-	LogLockInfo() : epochEnd(std::numeric_limits<Version>::max()), isCurrent(false) {}
-};
-
-struct DurableVersionInfo {
-	Version knownCommittedVersion; // maximum of the known committed versions of available tLogs
-	Version minimumDurableVersion; // mimimum of the durable versions of available tLogs
-	std::vector<TLogLockResult> lockResults; // replies from various tLogs
-	bool policyResult; // unavailable tLogs meet the replication policy or not
-	std::vector<uint16_t> knownLockedTLogIds; // tLogs that are known to have been locked
-
-	DurableVersionInfo(Version kcv,
-	                   Version dv,
-	                   std::vector<TLogLockResult>& replies,
-	                   bool meetsPolicy,
-	                   std::vector<uint16_t>& lockedTLogIds)
-	  : knownCommittedVersion(kcv), minimumDurableVersion(dv), lockResults(std::move(replies)),
-	    policyResult(meetsPolicy), knownLockedTLogIds(std::move(lockedTLogIds)) {}
-};
-
-struct TagPartitionedLogSystem final : ILogSystem, ReferenceCounted<TagPartitionedLogSystem> {
-	const UID dbgid;
-	LogSystemType logSystemType;
-	std::vector<Reference<LogSet>> tLogs; // LogSets in different locations: primary, satellite, or remote
-	int expectedLogSets;
-	int logRouterTags;
-	int txsTags;
-	UID recruitmentID;
-	int repopulateRegionAntiQuorum;
-	bool stopped;
-	std::set<int8_t> pseudoLocalities; // Represent special localities that will be mapped to tagLocalityLogRouter
-	const LogEpoch epoch;
-	LogEpoch oldestBackupEpoch;
-
-	// new members
-	std::map<Tag, Version> pseudoLocalityPopVersion;
-	Future<Void> rejoins;
-	Future<Void> recoveryComplete;
-	Future<Void> remoteRecovery;
-	Future<Void> remoteRecoveryComplete;
-	Future<Void> trackTLogRecovery;
-	Future<Void> remoteTrackTLogRecovery;
-	Reference<AsyncVar<Version>> recoveredVersion;
-	Reference<AsyncVar<Version>> remoteRecoveredVersion;
-	std::vector<LogLockInfo> lockResults;
-	AsyncVar<bool> recoveryCompleteWrittenToCoreState;
-	bool remoteLogsWrittenToCoreState;
-	bool hasRemoteServers;
-	AsyncTrigger backupWorkerChanged;
-	std::set<UID> removedBackupWorkers; // Workers that are removed before setting them.
-	std::map<uint8_t, std::vector<uint16_t>> knownLockedTLogIds;
-
-	Optional<Version> recoverAt;
-	Optional<Version> recoveredAt;
-	Version knownCommittedVersion;
-	Version backupStartVersion = invalidVersion; // max(tLogs[0].startVersion, previous epochEnd).
-	std::map<UID, Version> rvLogs; // recovery versions per tlog
-	LocalityData locality;
-	// For each currently running popFromLog actor, outstandingPops is
-	// (logID, tag)->(max popped version, durableKnownCommittedVersion).
-	// Why do we need durableKnownCommittedVersion? knownCommittedVersion gives the lower bound of what data
-	// will need to be copied into the next generation to restore the replication factor.
-	// Guess: It probably serves as a minimum version of what data should be on a TLog in the next generation and
-	// sending a pop for anything less than durableKnownCommittedVersion for the TLog will be absurd.
-	std::map<std::pair<UID, Tag>, std::pair<Version, Version>> outstandingPops;
-
-	// Stores each <log router, tag> pair's last popped version. This is used to determine whether we need to pop an old
-	// generation log router.
-	std::map<std::pair<UID, Tag>, Version> logRouterLastPops;
-
-	Optional<PromiseStream<Future<Void>>> addActor;
-	ActorCollection popActors;
-	std::vector<OldLogData> oldLogData; // each element has the log info. in one old epoch.
-	AsyncTrigger logSystemConfigChanged;
-
-	TagPartitionedLogSystem(UID dbgid,
-	                        LocalityData locality,
-	                        LogEpoch e,
-	                        Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>())
-	  : dbgid(dbgid), logSystemType(LogSystemType::empty), expectedLogSets(0), logRouterTags(0), txsTags(0),
-	    repopulateRegionAntiQuorum(0), stopped(false), epoch(e), oldestBackupEpoch(0),
-	    recoveredVersion(makeReference<AsyncVar<Version>>(invalidVersion)),
-	    remoteRecoveredVersion(makeReference<AsyncVar<Version>>(invalidVersion)),
-	    recoveryCompleteWrittenToCoreState(false), remoteLogsWrittenToCoreState(false), hasRemoteServers(false),
-	    locality(locality), addActor(addActor), popActors(false) {}
-
-	void stopRejoins() final;
-
-	void addref() final;
-
-	void delref() final;
-
-	std::string describe() const final;
-
-	UID getDebugID() const final;
-
-	LogSystemType getLogSystemType() const final;
-
-	void addPseudoLocality(int8_t locality);
-
-	Tag getPseudoPopTag(Tag tag, ProcessClass::ClassType type) const final;
-
-	bool hasPseudoLocality(int8_t locality) const final;
-
-	// Return the min version of all pseudoLocalities, i.e., logRouter and backupTag
-	Version popPseudoLocalityTag(Tag tag, Version upTo) final;
-
-	static Future<Void> recoverAndEndEpoch(Reference<AsyncVar<Reference<ILogSystem>>> const& outLogSystem,
-	                                       UID const& dbgid,
-	                                       DBCoreState const& oldState,
-	                                       FutureStream<TLogRejoinRequest> const& rejoins,
-	                                       LocalityData const& locality,
-	                                       bool* forceRecovery);
-
-	static Reference<ILogSystem> fromLogSystemConfig(UID const& dbgid,
-	                                                 LocalityData const& locality,
-	                                                 LogSystemConfig const& lsConf,
-	                                                 bool excludeRemote,
-	                                                 bool useRecoveredAt,
-	                                                 Optional<PromiseStream<Future<Void>>> addActor);
-
-	static Reference<ILogSystem> fromOldLogSystemConfig(UID const& dbgid,
-	                                                    LocalityData const& locality,
-	                                                    LogSystemConfig const& lsConf);
-
-	// Convert TagPartitionedLogSystem to DBCoreState and override input newState as return value
-	void toCoreState(DBCoreState& newState) const final;
-
-	bool remoteStorageRecovered() const final;
-
-	// Checks older TLog generations and remove no longer needed generations from the log system.
-	void purgeOldRecoveredGenerationsCoreState(DBCoreState&) final;
-	void purgeOldRecoveredGenerationsInMemory(const DBCoreState&) final;
-
-	Future<Void> onCoreStateChanged() const final;
-
-	void coreStateWritten(DBCoreState const& newState) final;
-
-	Future<Void> onError() const final;
-
-	ACTOR static Future<Void> onError_internal(TagPartitionedLogSystem const* self);
-
-	ACTOR static Future<Void> pushResetChecker(Reference<ConnectionResetInfo> self, NetworkAddress addr);
-
-	ACTOR static Future<TLogCommitReply> recordPushMetrics(Reference<ConnectionResetInfo> self,
-	                                                       Reference<Histogram> dist,
-	                                                       NetworkAddress addr,
-	                                                       Future<TLogCommitReply> in);
-
-	Future<Version> push(const ILogSystem::PushVersionSet& versionSet,
-	                     LogPushData& data,
-	                     SpanContext const& spanContext,
-	                     Optional<UID> debugID,
-	                     Optional<std::unordered_map<uint16_t, Version>> tpcvMap) final;
-
-	// Version vector/Unicast specific: reset best server if it is not known to have been locked/stopped.
-	void resetBestServerIfNotLocked(int bestSet,
-	                                int& bestServer,
-	                                Optional<Version> end,
-	                                const Optional<std::map<uint8_t, std::vector<uint16_t>>>& knownLockedTLogIds);
-
-	Reference<IPeekCursor> peekAll(UID dbgid, Version begin, Version end, Tag tag, bool parallelGetMore);
-
-	Reference<IPeekCursor> peekRemote(UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore);
-
-	Reference<IPeekCursor> peek(UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore) final;
-
-	Reference<IPeekCursor> peek(UID dbgid,
-	                            Version begin,
-	                            Optional<Version> end,
-	                            std::vector<Tag> tags,
-	                            bool parallelGetMore) final;
-
-	Reference<IPeekCursor> peekLocal(UID dbgid,
-	                                 Tag tag,
-	                                 Version begin,
-	                                 Version end,
-	                                 bool useMergePeekCursors,
-	                                 int8_t peekLocality = tagLocalityInvalid);
-
-	Reference<IPeekCursor> peekTxs(UID dbgid,
-	                               Version begin,
-	                               int8_t peekLocality,
-	                               Version localEnd,
-	                               bool canDiscardPopped) final;
-
-	Reference<IPeekCursor> peekSingle(UID dbgid,
-	                                  Version begin,
-	                                  Tag tag,
-	                                  std::vector<std::pair<Version, Tag>> history) final;
-
-	// LogRouter or BackupWorker use this function to obtain a cursor for peeking tlogs of a generation (i.e., epoch).
-	// Specifically, the epoch is determined by looking up "dbgid" in tlog sets of generations.
-	// The returned cursor can peek data at the "tag" from the given "begin" version to that epoch's end version or
-	// the recovery version for the latest old epoch. For the current epoch, the cursor has no end version.
-	// For the old epoch, the cursor is provided an end version.
-	Reference<IPeekCursor> peekLogRouter(
-	    UID dbgid,
-	    Version begin,
-	    Tag tag,
-	    bool useSatellite,
-	    Optional<Version> end,
-	    const Optional<std::map<uint8_t, std::vector<uint16_t>>>& knownStoppedTLogIds) final;
-
-	Version getKnownCommittedVersion() final;
-
-	Future<Void> onKnownCommittedVersionChange() final;
-
-	void popLogRouter(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality);
-
-	void popTxs(Version upTo, int8_t popLocality) final;
-
-	// pop 'tag.locality' type data up to the 'upTo' version
-	void pop(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) final;
-
-	// pop tag from log up to the version defined in self->outstandingPops[].first
-	ACTOR static Future<Void> popFromLog(TagPartitionedLogSystem* self,
-	                                     Reference<AsyncVar<OptionalInterface<TLogInterface>>> log,
-	                                     Tag tag,
-	                                     double delayBeforePop,
-	                                     bool popLogRouter);
-
-	ACTOR static Future<Version> getPoppedFromTLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> log, Tag tag);
-
-	ACTOR static Future<Version> getPoppedTxs(TagPartitionedLogSystem* self);
-
-	Future<Version> getTxsPoppedVersion() final;
-
-	ACTOR static Future<Void> confirmEpochLive_internal(Reference<LogSet> logSet, Optional<UID> debugID);
-
-	// Returns success after confirming that pushes in the current epoch are still possible
-	Future<Void> confirmEpochLive(Optional<UID> debugID) final;
-
-	Future<Void> endEpoch() final;
-
-	// Call only after end_epoch() has successfully completed.  Returns a new epoch immediately following this one.
-	// The new epoch is only provisional until the caller updates the coordinated DBCoreState.
-	Future<Reference<ILogSystem>> newEpoch(RecruitFromConfigurationReply const& recr,
-	                                       Future<RecruitRemoteFromConfigurationReply> const& fRemoteWorkers,
-	                                       DatabaseConfiguration const& config,
-	                                       LogEpoch recoveryCount,
-	                                       Version recoveryTransactionVersion,
-	                                       int8_t primaryLocality,
-	                                       int8_t remoteLocality,
-	                                       std::vector<Tag> const& allTags,
-	                                       Reference<AsyncVar<bool>> const& recruitmentStalled) final;
-
-	LogSystemConfig getLogSystemConfig() const final;
-
-	Standalone<StringRef> getLogsValue() const final;
-
-	Future<Void> onLogSystemConfigChange() final;
-
-	void updateLogRouter(int logSetIndex, int tagId, TLogInterface const& newLogRouter) final;
-
-	Version getEnd() const final;
-
-	Version getPeekEnd() const;
-
-	void getPushLocations(VectorRef<Tag> tags,
-	                      std::vector<int>& locations,
-	                      bool allLocations,
-	                      Optional<std::vector<Reference<LocalitySet>>> fromLocations) const final;
-
-	std::vector<Reference<LocalitySet>> getPushLocationsForTags(std::vector<int>& fromLocations) const final;
-
-	bool hasRemoteLogs() const final;
-
-	Tag getRandomRouterTag() const final;
-
-	Tag getRandomTxsTag() const final;
-
-	TLogVersion getTLogVersion() const final;
-
-	int getLogRouterTags() const final;
-
-	Version getBackupStartVersion() const final;
-
-	std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const final;
-
-	inline Reference<LogSet> getEpochLogSet(LogEpoch epoch) const;
-
-	void setBackupWorkers(const std::vector<InitializeBackupReply>& replies) final;
-
-	bool removeBackupWorker(const BackupWorkerDoneRequest& req) final;
-
-	LogEpoch getOldestBackupEpoch() const final;
-
-	void setOldestBackupEpoch(LogEpoch epoch) final;
-	ACTOR static Future<Void> monitorLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> logServer,
-	                                     Reference<AsyncVar<bool>> failed);
-
-	// returns the log group's knownComittedVersion, DV, a vector of TLogLockResults for
-	// each tLog in the group, and the result of applying the replication policy check over
-	// unavailable tLogs from the group
-	Optional<DurableVersionInfo> static getDurableVersion(
-	    UID dbgid,
-	    LogLockInfo lockInfo,
-	    std::vector<Reference<AsyncVar<bool>>> failed = std::vector<Reference<AsyncVar<bool>>>(),
-	    Optional<Version> lastEnd = Optional<Version>());
-
-	ACTOR static Future<Void> getDurableVersionChanged(
-	    LogLockInfo lockInfo,
-	    std::vector<Reference<AsyncVar<bool>>> failed = std::vector<Reference<AsyncVar<bool>>>());
-
-	ACTOR static Future<Void> epochEnd(Reference<AsyncVar<Reference<ILogSystem>>> outLogSystem,
-	                                   UID dbgid,
-	                                   DBCoreState prevState,
-	                                   FutureStream<TLogRejoinRequest> rejoinRequests,
-	                                   LocalityData locality,
-	                                   bool* forceRecovery);
-
-	ACTOR static Future<Void> recruitOldLogRouters(TagPartitionedLogSystem* self,
-	                                               std::vector<WorkerInterface> workers,
-	                                               LogEpoch recoveryCount,
-	                                               int8_t locality,
-	                                               Version startVersion,
-	                                               std::vector<LocalityData> tLogLocalities,
-	                                               Reference<IReplicationPolicy> tLogPolicy,
-	                                               bool forRemote);
-
-	static Version getMaxLocalStartVersion(const std::vector<Reference<LogSet>>& tLogs);
-
-	static std::vector<Tag> getLocalTags(int8_t locality, const std::vector<Tag>& allTags);
-
-	ACTOR static Future<Void> newRemoteEpoch(TagPartitionedLogSystem* self,
-	                                         Reference<TagPartitionedLogSystem> oldLogSystem,
-	                                         Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
-	                                         DatabaseConfiguration configuration,
-	                                         LogEpoch recoveryCount,
-	                                         Version recoveryTransactionVersion,
-	                                         int8_t remoteLocality,
-	                                         std::vector<Tag> allTags,
-	                                         std::vector<Version> oldGenerationRecoverAtVersions);
-
-	ACTOR static Future<Reference<ILogSystem>> newEpoch(Reference<TagPartitionedLogSystem> oldLogSystem,
-	                                                    RecruitFromConfigurationReply recr,
-	                                                    Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
-	                                                    DatabaseConfiguration configuration,
-	                                                    LogEpoch recoveryCount,
-	                                                    Version recoveryTransactionVersion,
-	                                                    int8_t primaryLocality,
-	                                                    int8_t remoteLocality,
-	                                                    std::vector<Tag> allTags,
-	                                                    Reference<AsyncVar<bool>> recruitmentStalled);
-
-	ACTOR static Future<Void> trackRejoins(
-	    UID dbgid,
-	    std::vector<std::pair<Reference<AsyncVar<OptionalInterface<TLogInterface>>>, Reference<IReplicationPolicy>>>
-	        logServers,
-	    FutureStream<struct TLogRejoinRequest> rejoinRequests);
-
-	// Keeps track of the recovered generations in all the TLogs in `tlogs` list and updates the latest
-	// `recoveredVersion`.
-	ACTOR static Future<Void> trackTLogRecoveryActor(
-	    std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> tlogs,
-	    Reference<AsyncVar<Version>> recoveredVersion);
-
-	ACTOR static Future<TLogLockResult> lockTLog(UID myID, Reference<AsyncVar<OptionalInterface<TLogInterface>>> tlog);
-	template <class T>
-	static std::vector<T> getReadyNonError(std::vector<Future<T>> const& futures);
-};
-
-// Recovery version calculation for version vector unicast
-Optional<std::tuple<Version, Version>> getRecoverVersionUnicast(
-    const std::vector<Reference<LogSet>>& logServers,
-    const std::tuple<int, std::vector<TLogLockResult>, bool>& logGroupResults,
-    Version minDV);
-
-template <class T>
-std::vector<T> TagPartitionedLogSystem::getReadyNonError(std::vector<Future<T>> const& futures) {
-	// Return the values of those futures which have (non-error) values ready
-	std::vector<T> result;
-	for (auto& f : futures)
-		if (f.isReady() && !f.isError())
-			result.push_back(f.get());
-	return result;
-}
-
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_TAGPARTITIONEDLOGSYSTEM_ACTOR_H
diff --git a/fdbserver/include/fdbserver/WaitFailure.h b/fdbserver/include/fdbserver/WaitFailure.h
deleted file mode 100644
index 3aef559c884..00000000000
--- a/fdbserver/include/fdbserver/WaitFailure.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * WaitFailure.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef WAIT_FAILURE_SERVER_H
-#define WAIT_FAILURE_SERVER_H
-#pragma once
-
-#include "flow/flow.h"
-#include "fdbrpc/fdbrpc.h"
-
-Future<Void> waitFailureServer(const FutureStream<ReplyPromise<Void>>& waitFailure);
-
-// talks to a wait failure server, returns Void on failure
-Future<Void> waitFailureClient(const RequestStream<ReplyPromise<Void>>& waitFailure,
-                               double const& failureReactionTime = 0,
-                               double const& failureReactionSlope = 0,
-                               bool const& trace = false,
-                               Optional<Standalone<StringRef>> const& traceMsg = Optional<Standalone<StringRef>>(),
-                               TaskPriority const& taskID = TaskPriority::DefaultEndpoint);
-
-// talks to a wait failure server, returns Void on failure, reaction time is always waited
-Future<Void> waitFailureClientStrict(const RequestStream<ReplyPromise<Void>>& waitFailure,
-                                     double const& failureReactionTime = 0,
-                                     TaskPriority const& taskID = TaskPriority::DefaultEndpoint);
-
-// talks to a wait failure server, updates failed to be true or false based on failure status.
-Future<Void> waitFailureTracker(const RequestStream<ReplyPromise<Void>>& waitFailure,
-                                Reference<AsyncVar<bool>> const& failed,
-                                double const& failureReactionTime = 0,
-                                double const& failureReactionSlope = 0,
-                                TaskPriority const& taskID = TaskPriority::DefaultEndpoint);
-
-#endif
diff --git a/fdbserver/include/fdbserver/ArtMutationBuffer.h b/fdbserver/kvstore/ArtMutationBuffer.h
similarity index 92%
rename from fdbserver/include/fdbserver/ArtMutationBuffer.h
rename to fdbserver/kvstore/ArtMutationBuffer.h
index 4a125d888ab..cc4a7c3b5fd 100644
--- a/fdbserver/include/fdbserver/ArtMutationBuffer.h
+++ b/fdbserver/kvstore/ArtMutationBuffer.h
@@ -20,7 +20,7 @@
 #ifndef ART_MUTATION_BUFFER
 #define ART_MUTATION_BUFFER
 
-#include "fdbserver/art.h"
+#include "art.h"
 #include "flow/Arena.h"
 
 struct MutationBufferART {
@@ -35,9 +35,9 @@ struct MutationBufferART {
 
 		const_iterator() = default;
 
-		const_iterator(art_iterator i) { artIterator = i; }
+		explicit const_iterator(art_iterator i) { artIterator = i; }
 
-		const_iterator(const const_iterator& i) { artIterator = i.artIterator; }
+		explicit(false) const_iterator(const const_iterator& i) { artIterator = i.artIterator; }
 
 		const KeyRef& key() { return artIterator.key(); }
 
@@ -63,9 +63,9 @@ struct MutationBufferART {
 
 		iterator() = default;
 
-		iterator(art_iterator i) { artIterator = i; }
+		explicit iterator(art_iterator i) { artIterator = i; }
 
-		iterator(const iterator& i) { artIterator = i.artIterator; }
+		explicit(false) iterator(const iterator& i) { artIterator = i.artIterator; }
 
 		const KeyRef& key() { return artIterator.key(); }
 
diff --git a/fdbserver/kvstore/CMakeLists.txt b/fdbserver/kvstore/CMakeLists.txt
new file mode 100644
index 00000000000..382f4e54edc
--- /dev/null
+++ b/fdbserver/kvstore/CMakeLists.txt
@@ -0,0 +1,29 @@
+fdb_find_sources(FDBSERVER_KVSTORE_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_kvstore SRCS ${FDBSERVER_KVSTORE_SRCS})
+add_fdbserver_link_test(fdbserver_kvstorelinktest
+  fdbserver_kvstore
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_kvstore)
+target_include_directories(fdbserver_kvstore
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_SOURCE_DIR}/contrib/sqlite
+    ${CMAKE_SOURCE_DIR}/fdbserver/include
+    ${CMAKE_BINARY_DIR}/fdbserver/include)
+target_link_libraries(fdbserver_kvstore PUBLIC fdbserver_core sqlite)
+
+if(WITH_ROCKSDB)
+  add_dependencies(fdbserver_kvstore rocksdb)
+  if(WITH_LIBURING)
+    target_include_directories(fdbserver_kvstore PRIVATE ${ROCKSDB_INCLUDE_DIR} ${uring_INCLUDE_DIR})
+    target_link_libraries(fdbserver_kvstore PUBLIC ${ROCKSDB_LIBRARIES} ${uring_LIBRARIES} ${LZ4_LIBRARY})
+  else()
+    target_include_directories(fdbserver_kvstore PRIVATE ${ROCKSDB_INCLUDE_DIR})
+    target_link_libraries(fdbserver_kvstore PUBLIC ${ROCKSDB_LIBRARIES} ${LZ4_LIBRARY})
+  endif()
+  target_compile_definitions(fdbserver_kvstore PRIVATE WITH_ROCKSDB)
+endif()
diff --git a/fdbserver/include/fdbserver/DeltaTree.h b/fdbserver/kvstore/DeltaTree.h
similarity index 99%
rename from fdbserver/include/fdbserver/DeltaTree.h
rename to fdbserver/kvstore/DeltaTree.h
index 82f58b22b95..0b69a29e163 100644
--- a/fdbserver/include/fdbserver/DeltaTree.h
+++ b/fdbserver/kvstore/DeltaTree.h
@@ -23,7 +23,7 @@
 #include "flow/flow.h"
 #include "flow/Arena.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include <string.h>
 
 #define DELTATREE_DEBUG 0
@@ -324,7 +324,7 @@ struct DeltaTree {
 	struct Mirror : FastAllocated<Mirror> {
 		friend class Cursor;
 
-		Mirror(const void* treePtr = nullptr, const T* lowerBound = nullptr, const T* upperBound = nullptr)
+		explicit Mirror(const void* treePtr = nullptr, const T* lowerBound = nullptr, const T* upperBound = nullptr)
 		  : tree((DeltaTree*)treePtr), lower(lowerBound), upper(upperBound) {
 			lower = new (arena) T(arena, *lower);
 			upper = new (arena) T(arena, *upper);
@@ -485,7 +485,7 @@ struct DeltaTree {
 	public:
 		Cursor() : mirror(nullptr), node(nullptr) {}
 
-		Cursor(Mirror* r) : mirror(r), node(mirror->root) {}
+		explicit Cursor(Mirror* r) : mirror(r), node(mirror->root) {}
 
 		Mirror* mirror;
 		DecodedNode* node;
@@ -1061,7 +1061,7 @@ struct DeltaTree2 {
 	// DecodedNodes are stored in a contiguous vector, which sometimes must be expanded, so care
 	// must be taken to resolve DecodedNode pointers again after the DecodeCache has new entries added.
 	struct DecodeCache : FastAllocated<DecodeCache>, ReferenceCounted<DecodeCache> {
-		DecodeCache(const T& lowerBound = T(), const T& upperBound = T(), int64_t* pMemoryTracker = nullptr)
+		explicit DecodeCache(const T& lowerBound = T(), const T& upperBound = T(), int64_t* pMemoryTracker = nullptr)
 		  : lowerBound(arena, lowerBound), upperBound(arena, upperBound), lastKnownUsedMemory(0),
 		    pMemoryTracker(pMemoryTracker) {
 			decodedNodes.reserve(10);
@@ -1149,7 +1149,7 @@ struct DeltaTree2 {
 		  : tree(tree), cache(cache), nodeIndex(nodeIndex) {}
 
 		// Copy constructor does not copy item because normally a copied cursor will be immediately moved.
-		Cursor(const Cursor& c) : tree(c.tree), cache(c.cache), nodeIndex(c.nodeIndex) {}
+		explicit(false) Cursor(const Cursor& c) : tree(c.tree), cache(c.cache), nodeIndex(c.nodeIndex) {}
 
 		~Cursor() {
 			if (cache.isValid()) {
diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/kvstore/DiskQueue.cpp
similarity index 84%
rename from fdbserver/DiskQueue.actor.cpp
rename to fdbserver/kvstore/DiskQueue.cpp
index adc31f16997..f7ab0fb40cc 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/kvstore/DiskQueue.cpp
@@ -1,5 +1,5 @@
 /*
- * DiskQueue.actor.cpp
+ * DiskQueue.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,18 +18,17 @@
  * limitations under the License.
  */
 
-#include "fdbserver/IDiskQueue.h"
+#include "fdbserver/kvstore/IDiskQueue.h"
 #include "flow/IAsyncFile.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbrpc/simulator.h"
 #include "crc32/crc32c.h"
 #include "flow/genericactors.actor.h"
+#include "flow/UnitTest.h"
 #include "flow/xxhash.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-typedef bool (*compare_pages)(void*, void*);
-typedef int64_t loc_t;
+using compare_pages = bool (*)(void*, void*);
+using loc_t = int64_t;
 
 // 0 -> 0
 // 1 -> 4k
@@ -50,7 +49,7 @@ struct StringBuffer {
 	int reserved;
 	UID id;
 
-	StringBuffer(UID fromFileID) : reserved(0), id(fromFileID) {}
+	explicit StringBuffer(UID fromFileID) : reserved(0), id(fromFileID) {}
 
 	int size() const { return str.size(); }
 	Standalone<StringRef> get() { return str; }
@@ -121,11 +120,10 @@ struct SyncQueue : ReferenceCounted<SyncQueue> {
 	Deque<Future<Void>> outstanding;
 	Reference<IAsyncFile> file;
 
-	ACTOR static Future<Void> waitAndSync(SyncQueue* self) {
-		wait(self->outstanding.front());
+	static Future<Void> waitAndSync(SyncQueue* self) {
+		co_await self->outstanding.front();
 		self->outstanding.pop_front();
-		wait(self->file->sync());
-		return Void();
+		co_await self->file->sync();
 	}
 };
 
@@ -195,7 +193,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	Future<Void> pushAndCommit(Standalone<StringRef> pageData, StringBuffer* pageMem, uint64_t poppedPages) {
-		return pushAndCommit(this, pageData, pageMem, poppedPages);
+		return pushAndCommit(Uncancellable(), this, pageData, pageMem, poppedPages);
 	}
 
 	void stall() {
@@ -204,7 +202,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	Future<Standalone<StringRef>> readFirstAndLastPages(compare_pages compare) {
-		return readFirstAndLastPages(this, compare);
+		return readFirstAndLastPages(Uncancellable(), this, compare);
 	}
 
 	void setStartPage(int file, int64_t page) {
@@ -221,14 +219,16 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	// FIXME: let the caller pass in where to write the data.
-	Future<Standalone<StringRef>> read(int file, int page, int nPages) { return read(this, file, page, nPages); }
-	Future<Standalone<StringRef>> readNextPage() { return readNextPage(this); }
+	Future<Standalone<StringRef>> read(int file, int page, int nPages) {
+		return read(Uncancellable(), this, file, page, nPages);
+	}
+	Future<Standalone<StringRef>> readNextPage() { return readNextPage(Uncancellable(), this); }
 	Future<Void> truncateBeforeLastReadPage() { return truncateBeforeLastReadPage(this); }
 
 	Future<Void> getError() { return onError; }
 	Future<Void> onClosed() { return onStopped; }
-	void dispose() { shutdown(this, true); }
-	void close() { shutdown(this, false); }
+	void dispose() { shutdown(Uncancellable(), this, true); }
+	void close() { shutdown(Uncancellable(), this, false); }
 
 	StorageBytes getStorageBytes() const {
 		int64_t free;
@@ -293,50 +293,51 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 
 	Int64MetricHandle stallCount;
 
-	Future<Void> truncateFile(int file, int64_t pos) { return truncateFile(this, file, pos); }
+	Future<Void> truncateFile(int file, int64_t pos) { return truncateFile(Uncancellable(), this, file, pos); }
 
 	// FIXME: Merge this function with IAsyncFileSystem::incrementalDeleteFile().
-	ACTOR static void incrementalTruncate(Reference<IAsyncFile> file) {
-		state int64_t remainingFileSize = wait(file->size());
+	static Future<Void> incrementalTruncate(Uncancellable, Reference<IAsyncFile> file) {
+		int64_t remainingFileSize = co_await file->size();
 
 		for (; remainingFileSize > 0; remainingFileSize -= FLOW_KNOBS->INCREMENTAL_DELETE_TRUNCATE_AMOUNT) {
-			wait(file->truncate(remainingFileSize));
-			wait(file->sync());
-			wait(delay(FLOW_KNOBS->INCREMENTAL_DELETE_INTERVAL));
+			co_await file->truncate(remainingFileSize);
+			co_await file->sync();
+			co_await delay(FLOW_KNOBS->INCREMENTAL_DELETE_INTERVAL);
 		}
 
 		TraceEvent("DiskQueueReplaceTruncateEnded").detail("Filename", file->getFilename());
 	}
 
 #if defined(_WIN32)
-	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
+	static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
 		// Windows doesn't support a rename over an open file.
-		wait(toReplace->truncate(4 << 10));
-		return toReplace;
+		co_await toReplace->truncate(4 << 10);
+		co_return toReplace;
 	}
 #else
-	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
-		incrementalTruncate(toReplace);
+	static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
+		incrementalTruncate(Uncancellable(), toReplace);
 
-		Reference<IAsyncFile> _replacement = wait(IAsyncFileSystem::filesystem()->open(
+		Reference<IAsyncFile> _replacement = co_await IAsyncFileSystem::filesystem()->open(
 		    toReplace->getFilename(),
 		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE |
 		        IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK,
-		    0600));
-		state Reference<IAsyncFile> replacement = _replacement;
-		wait(replacement->sync());
+		    0600);
+		Reference<IAsyncFile> replacement = _replacement;
+		co_await replacement->sync();
 
-		return replacement;
+		co_return replacement;
 	}
 #endif
 
 	Future<Future<Void>> push(Standalone<StringRef> pageData, std::vector<Reference<SyncQueue>>* toSync) {
-		return push(this, pageData, toSync);
+		return push(Uncancellable(), this, pageData, toSync);
 	}
 
-	ACTOR static UNCANCELLABLE Future<Future<Void>> push(RawDiskQueue_TwoFiles* self,
-	                                                     Standalone<StringRef> pageData,
-	                                                     std::vector<Reference<SyncQueue>>* toSync) {
+	static Future<Future<Void>> push(Uncancellable,
+	                                 RawDiskQueue_TwoFiles* self,
+	                                 Standalone<StringRef> pageData,
+	                                 std::vector<Reference<SyncQueue>>* toSync) {
 		// Write the given data (pageData) to the queue files, swapping or extending them if necessary.
 		// Don't do any syncs, but push the modified file(s) onto toSync.
 		ASSERT(self->readingFile == 2);
@@ -345,7 +346,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 		ASSERT(self->writingPos % _PAGE_SIZE == 0);
 		ASSERT(self->files[0].size % _PAGE_SIZE == 0 && self->files[1].size % _PAGE_SIZE == 0);
 
-		state std::vector<Future<Void>> waitfor;
+		std::vector<Future<Void>> waitfor;
 
 		if (pageData.size() + self->writingPos > self->files[1].size) {
 			if (self->files[0].popped == self->files[0].size) {
@@ -383,7 +384,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 						    .detail("Filename", self->files[1].f->getFilename())
 						    .detail("OldFileSize", self->files[1].size)
 						    .detail("ElidedTruncateSize", maxShrink);
-						Reference<IAsyncFile> newFile = wait(replaceFile(self->files[1].f));
+						Reference<IAsyncFile> newFile = co_await replaceFile(self->files[1].f);
 						self->files[1].setFile(newFile);
 						waitfor.push_back(self->files[1].f->truncate(self->fileExtensionBytes));
 						self->files[1].size = self->fileExtensionBytes;
@@ -429,21 +430,23 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 		    holdWhile(pageData, self->files[1].f->write(pageData.begin(), pageData.size(), self->writingPos))));
 		self->writingPos += pageData.size();
 
-		return waitForAllReadyThenThrow(waitfor);
+		co_return waitForAllReadyThenThrow(waitfor);
 	}
 
 	// Write the given data (pageData) to the queue files of self, sync data to disk, and delete the memory (pageMem)
 	// that hold the pageData
-	ACTOR static UNCANCELLABLE Future<Void> pushAndCommit(RawDiskQueue_TwoFiles* self,
-	                                                      Standalone<StringRef> pageData,
-	                                                      StringBuffer* pageMem,
-	                                                      uint64_t poppedPages) {
-		state Promise<Void> pushing, committed;
-		state Promise<Void> errorPromise = self->error;
-		state std::string filename = self->files[0].dbgFilename;
-		state UID dbgid = self->dbgid;
-		state std::vector<Reference<SyncQueue>> syncFiles;
-		state Future<Void> lastCommit = self->lastCommit;
+	static Future<Void> pushAndCommit(Uncancellable,
+	                                  RawDiskQueue_TwoFiles* self,
+	                                  Standalone<StringRef> pageData,
+	                                  StringBuffer* pageMem,
+	                                  uint64_t poppedPages) {
+		Promise<Void> pushing;
+		Promise<Void> committed;
+		Promise<Void> errorPromise = self->error;
+		std::string filename = self->files[0].dbgFilename;
+		UID dbgid = self->dbgid;
+		std::vector<Reference<SyncQueue>> syncFiles;
+		Future<Void> lastCommit = self->lastCommit;
 		try {
 			// pushing might need to wait for previous pushes to start (to maintain order) or for
 			// a previous commit to finish if stall() was called
@@ -458,15 +461,15 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 				self->readyToPush = self->lastCommit;
 			}
 
-			wait(ready);
+			co_await ready;
 
 			CODE_PROBE(pageData.size() > sizeof(Page), "push more than one page of data");
 
-			Future<Void> pushed = wait(self->push(pageData, &syncFiles));
+			Future<Void> pushed = co_await self->push(pageData, &syncFiles);
 			pushing.send(Void());
 			ASSERT(syncFiles.size() >= 1 && syncFiles.size() <= 2);
 			CODE_PROBE(2 == syncFiles.size(), "push spans both files");
-			wait(pushed);
+			co_await pushed;
 
 			delete pageMem;
 			pageMem = 0;
@@ -474,12 +477,12 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 			Future<Void> sync = syncFiles[0]->onSync();
 			for (int i = 1; i < syncFiles.size(); i++)
 				sync = sync && syncFiles[i]->onSync();
-			wait(sync);
-			wait(lastCommit);
+			co_await sync;
+			co_await lastCommit;
 
 			// Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
 			if (g_network->check_yield(g_network->getCurrentTask())) {
-				wait(delay(0, g_network->getCurrentTask()));
+				co_await delay(0, g_network->getCurrentTask());
 			}
 
 			self->updatePopped(poppedPages * sizeof(Page));
@@ -506,7 +509,6 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 
 			throw e;
 		}
-		return Void();
 	}
 
 	void updatePopped(int64_t popped) {
@@ -516,7 +518,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	// Set the starting point of the ring buffer, i.e., the first useful page to be read (and poped)
-	ACTOR static Future<Void> setPoppedPage(RawDiskQueue_TwoFiles* self, int file, int64_t page, int64_t debugSeq) {
+	static Future<Void> setPoppedPage(RawDiskQueue_TwoFiles* self, int file, int64_t page, int64_t debugSeq) {
 		self->files[file].popped = page * sizeof(Page);
 		if (file)
 			self->files[0].popped = self->files[0].size;
@@ -529,20 +531,18 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 		//  be sequenced before file 1, when in fact it contains many pages that follow file 1.  These ok pages may be
 		//  incorrectly read if the machine dies after overwritting the first page of file 0 and is then recovered
 		if (file == 1)
-			wait(self->truncateFile(self, 0, 0));
-
-		return Void();
+			co_await truncateFile(Uncancellable(), self, 0, 0);
 	}
 
-	ACTOR static Future<Void> openFiles(RawDiskQueue_TwoFiles* self) {
-		state std::vector<Future<Reference<IAsyncFile>>> fs;
+	static Future<Void> openFiles(RawDiskQueue_TwoFiles* self) {
+		std::vector<Future<Reference<IAsyncFile>>> fs;
 		fs.reserve(2);
 		for (int i = 0; i < 2; i++)
 			fs.push_back(IAsyncFileSystem::filesystem()->open(self->filename(i),
 			                                                  IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED |
 			                                                      IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK,
 			                                                  0));
-		wait(waitForAllReady(fs));
+		co_await waitForAllReady(fs);
 
 		// Treatment of errors here is important.  If only one of the two files is present
 		// (due to a power failure during creation or deletion, or administrative error) we don't want to
@@ -563,7 +563,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 				    0600);
 
 			// Any error here is fatal
-			wait(waitForAll(fs));
+			co_await waitForAll(fs);
 
 			// sync on each file to actually create it will be done below
 		} else {
@@ -582,23 +582,21 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 		syncs.reserve(fs.size());
 		for (int i = 0; i < fs.size(); i++)
 			syncs.push_back(fs[i].get()->sync());
-		wait(waitForAll(syncs));
+		co_await waitForAll(syncs);
 
 		// Successfully opened or created; fill in self->files[]
 		for (int i = 0; i < 2; i++)
 			self->files[i].setFile(fs[i].get());
-
-		return Void();
 	}
 
-	ACTOR static void shutdown(RawDiskQueue_TwoFiles* self, bool deleteFiles) {
+	static Future<Void> shutdown(Uncancellable, RawDiskQueue_TwoFiles* self, bool deleteFiles) {
 		// Wait for all reads and writes on the file, and all actors referencing self, to be finished
-		state Error error = success();
+		Error error = success();
 		try {
-			wait(success(errorOr(self->lastCommit)));
+			co_await errorOr(self->lastCommit);
 			// Wait for the pending operations (e.g., read) to finish before we destroy the DiskQueue, because
 			// tLog, instead of DiskQueue, hold the future of the pending operations.
-			wait(self->onSafeToDestruct());
+			co_await self->onSafeToDestruct();
 
 			for (int i = 0; i < 2; i++)
 				self->files[i].f.clear();
@@ -607,8 +605,8 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 				TraceEvent("DiskQueueShutdownDeleting", self->dbgid)
 				    .detail("File0", self->filename(0))
 				    .detail("File1", self->filename(1));
-				wait(IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename(0), false));
-				wait(IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename(1), true));
+				co_await IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename(0), false);
+				co_await IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename(1), true);
 			}
 			TraceEvent("DiskQueueShutdownComplete", self->dbgid)
 			    .detail("DeleteFiles", deleteFiles)
@@ -630,20 +628,21 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	// Return the most recently written page, the page with largest seq number
-	ACTOR static UNCANCELLABLE Future<Standalone<StringRef>> readFirstAndLastPages(RawDiskQueue_TwoFiles* self,
-	                                                                               compare_pages compare) {
-		state TrackMe trackMe(self);
+	static Future<Standalone<StringRef>> readFirstAndLastPages(Uncancellable,
+	                                                           RawDiskQueue_TwoFiles* self,
+	                                                           compare_pages compare) {
+		TrackMe trackMe(self);
 
 		try {
 			// Open both files or create both files
-			wait(openFiles(self));
+			co_await openFiles(self);
 
 			// Get the file sizes
 			std::vector<Future<int64_t>> fsize;
 			fsize.reserve(2);
 			for (int i = 0; i < 2; i++)
 				fsize.push_back(self->files[i].f->size());
-			std::vector<int64_t> file_sizes = wait(getAll(fsize));
+			std::vector<int64_t> file_sizes = co_await getAll(fsize);
 			for (int i = 0; i < 2; i++) {
 				// SOMEDAY: If the file size is not a multiple of page size, it may never be shortened.  Change this?
 				self->files[i].size = file_sizes[i] - file_sizes[i] % sizeof(Page);
@@ -655,7 +654,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 			for (int i = 0; i < 2; i++)
 				if (self->files[i].size > 0)
 					reads.push_back(self->files[i].f->read(self->firstPages[i], sizeof(Page), 0));
-			wait(waitForAllReadyThenThrow(reads));
+			co_await waitForAllReadyThenThrow(reads);
 
 			// Determine which file comes first
 			if (compare(self->firstPages[1], self->firstPages[0])) {
@@ -676,30 +675,30 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 				std::vector<Future<Void>> truncates;
 				for (int i = 0; i < 2; ++i)
 					if (self->files[i].size > 0)
-						truncates.push_back(self->truncateFile(self, i, 0));
+						truncates.push_back(truncateFile(Uncancellable(), self, i, 0));
 
-				wait(waitForAll(truncates));
+				co_await waitForAll(truncates);
 
 				self->files[0].popped = self->files[0].size;
 				self->files[1].popped = 0;
 				memset(self->firstPages[1], 0xFF, sizeof(Page));
 				self->writingPos = 0;
 				self->readingFile = 2;
-				return Standalone<StringRef>();
+				co_return Standalone<StringRef>();
 			}
 
 			// A page in files[1] is "valid" iff compare(self->firstPages[1], page)
 			// Binary search to find a page in files[1] that is "valid" but the next page is not valid
 			// Invariant: the page at begin is valid, and the page at end is invalid
-			state int64_t begin = 0;
-			state int64_t end = self->files[1].size / sizeof(Page);
-			state Standalone<StringRef> middlePageAllocation = makeAlignedString(sizeof(Page), sizeof(Page));
-			state Page* middlePage = (Page*)middlePageAllocation.begin();
+			int64_t begin = 0;
+			int64_t end = self->files[1].size / sizeof(Page);
+			Standalone<StringRef> middlePageAllocation = makeAlignedString(sizeof(Page), sizeof(Page));
+			Page* middlePage = (Page*)middlePageAllocation.begin();
 			while (begin + 1 != end) {
-				state int64_t middle = (begin + end) / 2;
+				int64_t middle = (begin + end) / 2;
 				ASSERT(middle > begin && middle < end); // So the loop always changes begin or end
 
-				int len = wait(self->files[1].f->read(middlePage, sizeof(Page), middle * sizeof(Page)));
+				int len = co_await self->files[1].f->read(middlePage, sizeof(Page), middle * sizeof(Page));
 				ASSERT(len == sizeof(Page));
 
 				bool middleValid = compare(self->firstPages[1], middlePage);
@@ -718,11 +717,11 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 			}
 			// Now by the invariant and the loop condition, begin is a valid page and begin+1 is an invalid page
 			// Check that begin+1 is invalid
-			int len1 = wait(self->files[1].f->read(middlePage, sizeof(Page), (begin + 1) * sizeof(Page)));
+			int len1 = co_await self->files[1].f->read(middlePage, sizeof(Page), (begin + 1) * sizeof(Page));
 			ASSERT(!(len1 == sizeof(Page) && compare(self->firstPages[1], middlePage)));
 
 			// Read it
-			int len2 = wait(self->files[1].f->read(middlePage, sizeof(Page), begin * sizeof(Page)));
+			int len2 = co_await self->files[1].f->read(middlePage, sizeof(Page), begin * sizeof(Page));
 			ASSERT(len2 == sizeof(Page) && compare(self->firstPages[1], middlePage));
 
 			TraceEvent("RDQEndFound", self->dbgid)
@@ -730,7 +729,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 			    .detail("Pos", begin)
 			    .detail("FileSize", self->files[1].size);
 
-			return middlePageAllocation;
+			co_return middlePageAllocation;
 		} catch (Error& e) {
 			bool ok = e.code() == error_code_file_not_found;
 			TraceEvent(ok ? SevInfo : SevError, "RDQReadFirstAndLastPagesError", self->dbgid)
@@ -743,19 +742,20 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	// Read nPages from pageOffset*sizeof(Page) offset in file self->files[file]
-	ACTOR static UNCANCELLABLE Future<Standalone<StringRef>> read(RawDiskQueue_TwoFiles* self,
-	                                                              int file,
-	                                                              int pageOffset,
-	                                                              int nPages) {
-		state TrackMe trackMe(self);
-		state const size_t bytesRequested = nPages * sizeof(Page);
-		state Standalone<StringRef> result = makeAlignedString(sizeof(Page), bytesRequested);
+	static Future<Standalone<StringRef>> read(Uncancellable,
+	                                          RawDiskQueue_TwoFiles* self,
+	                                          int file,
+	                                          int pageOffset,
+	                                          int nPages) {
+		TrackMe trackMe(self);
+		const size_t bytesRequested = nPages * sizeof(Page);
+		Standalone<StringRef> result = makeAlignedString(sizeof(Page), bytesRequested);
 		if (file == 1)
 			ASSERT_WE_THINK(pageOffset * sizeof(Page) + bytesRequested <= self->writingPos);
 		int bytesRead =
-		    wait(self->files[file].f->read(mutateString(result), bytesRequested, pageOffset * sizeof(Page)));
+		    co_await self->files[file].f->read(mutateString(result), bytesRequested, pageOffset * sizeof(Page));
 		ASSERT_WE_THINK(bytesRead == bytesRequested);
-		return result;
+		co_return result;
 	}
 
 	Future<int> fillReadingBuffer() {
@@ -785,28 +785,28 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 		return files[readingFile].f->read(p, len, pos);
 	}
 
-	ACTOR static UNCANCELLABLE Future<Standalone<StringRef>> readNextPage(RawDiskQueue_TwoFiles* self) {
-		state TrackMe trackMe(self);
+	static Future<Standalone<StringRef>> readNextPage(Uncancellable, RawDiskQueue_TwoFiles* self) {
+		TrackMe trackMe(self);
 
 		try {
 			ASSERT(self->readingFile < 2);
 			ASSERT(self->files[0].f && self->files[1].f);
 
 			if (!self->readingBuffer.size()) {
-				state Future<Void> f = Void();
+				Future<Void> f = Void();
 				// if (BUGGIFY) f = delay( deterministicRandom()->random01() * 0.1 );
 
-				int read = wait(self->fillReadingBuffer());
+				int read = co_await self->fillReadingBuffer();
 				ASSERT(read == self->readingBuffer.size());
 
-				wait(f);
+				co_await f;
 			}
 			if (!self->readingBuffer.size())
-				return Standalone<StringRef>();
+				co_return Standalone<StringRef>();
 
 			ASSERT(self->readingBuffer.size() >= sizeof(Page));
 			Standalone<StringRef> result = self->readingBuffer.pop_front(sizeof(Page));
-			return result;
+			co_return result;
 		} catch (Error& e) {
 			CODE_PROBE(true, "Read next page error");
 			TraceEvent(SevError, "RDQReadNextPageError", self->dbgid)
@@ -819,29 +819,28 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 	}
 
 	// Set zero and free the memory from pos to the end of file self->files[file].
-	ACTOR static UNCANCELLABLE Future<Void> truncateFile(RawDiskQueue_TwoFiles* self, int file, int64_t pos) {
-		state TrackMe trackMe(self);
+	static Future<Void> truncateFile(Uncancellable, RawDiskQueue_TwoFiles* self, int file, int64_t pos) {
+		TrackMe trackMe(self);
 		TraceEvent("DQTruncateFile", self->dbgid)
 		    .detail("File", file)
 		    .detail("Pos", pos)
 		    .detail("File0Name", self->files[0].dbgFilename);
-		state Reference<IAsyncFile> f =
-		    self->files[file].f; // Hold onto a reference in the off-chance that the DQ is removed from underneath us.
+		// Hold onto a reference in the off-chance that the DQ is removed from underneath us.
+		Reference<IAsyncFile> f = self->files[file].f;
 		if (pos == 0) {
 			memset(self->firstPages[file], 0xFF, _PAGE_SIZE);
 		}
-		wait(f->zeroRange(pos, self->files[file].size - pos));
-		wait(self->files[file].syncQueue->onSync());
+		co_await f->zeroRange(pos, self->files[file].size - pos);
+		co_await self->files[file].syncQueue->onSync();
 		// We intentionally don't return the f->zero future, so that TrackMe is destructed after f->zero finishes.
-		return Void();
 	}
 
-	ACTOR static Future<Void> truncateBeforeLastReadPage(RawDiskQueue_TwoFiles* self) {
+	static Future<Void> truncateBeforeLastReadPage(RawDiskQueue_TwoFiles* self) {
 		try {
-			state int file = self->readingFile;
-			state int64_t pos = (self->readingPage - self->readingBuffer.size() / sizeof(Page) - 1) * sizeof(Page);
-			state std::vector<Future<Void>> commits;
-			state bool swap = file == 0;
+			int file = self->readingFile;
+			int64_t pos = (self->readingPage - self->readingBuffer.size() / sizeof(Page) - 1) * sizeof(Page);
+			std::vector<Future<Void>> commits;
+			bool swap = file == 0;
 
 			CODE_PROBE(file == 0, "truncate before last read page on file 0");
 			CODE_PROBE(file == 1 && pos != self->files[1].size, "truncate before last read page on file 1");
@@ -851,12 +850,12 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 			self->writingPos = pos;
 
 			while (file < 2) {
-				commits.push_back(self->truncateFile(self, file, pos));
+				commits.push_back(truncateFile(Uncancellable(), self, file, pos));
 				file++;
 				pos = 0;
 			}
 
-			wait(waitForAll(commits));
+			co_await waitForAll(commits);
 
 			if (swap) {
 				std::swap(self->files[0], self->files[1]);
@@ -864,7 +863,6 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 				self->files[0].popped = self->files[0].size;
 			}
 
-			return Void();
 		} catch (Error& e) {
 			TraceEvent(SevError, "RDQTruncateBeforeLastReadPageError", self->dbgid)
 			    .error(e)
@@ -1012,7 +1010,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		    .detail("PoppedSeq", poppedSeq)
 		    .detail("NextPageSeq", nextPageSeq)
 		    .detail("File0Name", rawQueue->files[0].dbgFilename);
-		dispose(this);
+		dispose(Uncancellable(), this);
 	}
 
 	void close() override {
@@ -1023,21 +1021,21 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		    .detail("PoppedCommitted",
 		            rawQueue->dbg_file0BeginSeq + rawQueue->files[0].popped + rawQueue->files[1].popped)
 		    .detail("File0Name", rawQueue->files[0].dbgFilename);
-		close(this);
+		close(Uncancellable(), this);
 	}
 
 	StorageBytes getStorageBytes() const override { return rawQueue->getStorageBytes(); }
 
 private:
-	ACTOR static void dispose(DiskQueue* self) {
-		wait(self->onSafeToDestruct());
+	static Future<Void> dispose(Uncancellable, DiskQueue* self) {
+		co_await self->onSafeToDestruct();
 		TraceEvent("DQDestroyDone", self->dbgid).detail("File0Name", self->rawQueue->files[0].dbgFilename);
 		self->rawQueue->dispose();
 		delete self;
 	}
 
-	ACTOR static void close(DiskQueue* self) {
-		wait(self->onSafeToDestruct());
+	static Future<Void> close(Uncancellable, DiskQueue* self) {
+		co_await self->onSafeToDestruct();
 		TraceEvent("DQCloseDone", self->dbgid).detail("File0Name", self->rawQueue->files[0].dbgFilename);
 		self->rawQueue->close();
 		delete self;
@@ -1165,35 +1163,14 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		}
 	}
 
-	ACTOR static void verifyCommit(DiskQueue* self,
-	                               Future<Void> commitSynced,
-	                               StringBuffer* buffer,
-	                               loc_t start,
-	                               loc_t end) {
-		state TrackMe trackme(self);
-		try {
-			wait(commitSynced);
-			Standalone<StringRef> pagedData = wait(readPages(self, start, end));
-			const int startOffset = start % _PAGE_SIZE;
-			const int dataLen = end - start;
-			ASSERT(pagedData.substr(startOffset, dataLen).compare(buffer->get().substr(0, dataLen)) == 0);
-		} catch (Error& e) {
-			if (e.code() != error_code_io_error) {
-				delete buffer;
-				throw;
-			}
-		}
-		delete buffer;
-	}
-
 	// Read pages from [start, end) bytes
-	ACTOR static Future<Standalone<StringRef>> readPages(DiskQueue* self, location start, location end) {
-		state TrackMe trackme(self);
-		state int fromFile;
-		state int toFile;
-		state int64_t fromPage;
-		state int64_t toPage;
-		state uint64_t file0size =
+	static Future<Standalone<StringRef>> readPages(DiskQueue* self, location start, location end) {
+		TrackMe trackme(self);
+		int fromFile{ 0 };
+		int toFile{ 0 };
+		int64_t fromPage{ 0 };
+		int64_t toPage{ 0 };
+		uint64_t file0size =
 		    self->rawQueue->files[0].size ? self->firstPages(1).seq - self->firstPages(0).seq : self->firstPages(1).seq;
 		ASSERT(end > start);
 		ASSERT(start.lo >= self->firstPages(0).seq || start.lo >= self->firstPages(1).seq);
@@ -1214,7 +1191,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		}
 		if (fromFile == toFile) {
 			ASSERT(toPage >= fromPage);
-			Standalone<StringRef> pagedData = wait(self->rawQueue->read(fromFile, fromPage, toPage - fromPage + 1));
+			Standalone<StringRef> pagedData = co_await self->rawQueue->read(fromFile, fromPage, toPage - fromPage + 1);
 			if (std::min(self->firstPages(0).seq, self->firstPages(1).seq) > start.lo) {
 				// Simulation allows for reads to be delayed and executed after overlapping subsequent
 				// write operations.  This means that by the time our read was executed, it's possible
@@ -1227,13 +1204,14 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 			ASSERT(pagedData.size() == (toPage - fromPage + 1) * _PAGE_SIZE);
 
 			ASSERT(((Page*)pagedData.end() - 1)->seq == pageFloor(end.lo - 1));
-			return pagedData;
+			co_return pagedData;
 		} else {
 			ASSERT(fromFile == 0);
-			state Standalone<StringRef> firstChunk;
-			state Standalone<StringRef> secondChunk;
-			wait(store(firstChunk, self->rawQueue->read(fromFile, fromPage, (file0size / sizeof(Page)) - fromPage)) &&
-			     store(secondChunk, self->rawQueue->read(toFile, 0, toPage + 1)));
+			Standalone<StringRef> firstChunk;
+			Standalone<StringRef> secondChunk;
+			co_await (
+			    store(firstChunk, self->rawQueue->read(fromFile, fromPage, (file0size / sizeof(Page)) - fromPage)) &&
+			    store(secondChunk, self->rawQueue->read(toFile, 0, toPage + 1)));
 			if (std::min(self->firstPages(0).seq, self->firstPages(1).seq) > start.lo) {
 				// See above.
 				throw io_error();
@@ -1242,14 +1220,12 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 			ASSERT(((Page*)firstChunk.begin())->seq == pageFloor(start.lo));
 			ASSERT(secondChunk.size() == (toPage + 1) * _PAGE_SIZE);
 			ASSERT(((Page*)secondChunk.end() - 1)->seq == pageFloor(end.lo - 1));
-			return firstChunk.withSuffix(secondChunk);
+			co_return firstChunk.withSuffix(secondChunk);
 		}
 	}
 
-	ACTOR static Future<Standalone<StringRef>> read(DiskQueue* self, location start, location end, CheckHashes ch) {
-		// This `state` is unnecessary, but works around pagedData wrongly becoming const
-		// due to the actor compiler.
-		state Standalone<StringRef> pagedData = wait(readPages(self, start, end));
+	static Future<Standalone<StringRef>> read(DiskQueue* self, location start, location end, CheckHashes ch) {
+		Standalone<StringRef> pagedData = co_await readPages(self, start, end);
 		ASSERT(start.lo % sizeof(Page) == 0 || start.lo % sizeof(Page) >= sizeof(PageHeader));
 		int startingOffset = start.lo % sizeof(Page);
 		if (startingOffset > 0)
@@ -1270,7 +1246,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 			if (!ch && data->payloadSize > Page::maxPayload)
 				throw io_error();
 			pagedData.contents() = pagedData.substr(sizeof(PageHeader) + startingOffset, endingOffset - startingOffset);
-			return pagedData;
+			co_return pagedData;
 		} else {
 			// Reusing pagedData wastes # of pages * sizeof(PageHeader) bytes, but means
 			// we don't have to double allocate in a hot, memory hungry call.
@@ -1314,7 +1290,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 
 			memset(buf, 0, pagedData.size() - (buf - pagedData.begin()));
 			Standalone<StringRef> unpagedData = pagedData.substr(0, buf - pagedData.begin());
-			return unpagedData;
+			co_return unpagedData;
 		}
 	}
 
@@ -1331,36 +1307,36 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		nextReadLocation += len;
 	}
 
-	ACTOR static Future<Standalone<StringRef>> readNext(DiskQueue* self, int bytes) {
-		state StringBuffer result(self->dbgid);
+	static Future<Standalone<StringRef>> readNext(DiskQueue* self, int bytes) {
+		StringBuffer result(self->dbgid);
 		ASSERT(bytes >= 0);
 		result.clearReserve(bytes);
 
 		ASSERT(!self->recovered);
 
 		if (!self->initialized) {
-			bool recoveryComplete = wait(initializeRecovery(self, 0));
+			bool recoveryComplete = co_await initializeRecovery(self, 0);
 
 			if (recoveryComplete) {
 				ASSERT(self->poppedSeq <= self->endLocation());
 
-				return Standalone<StringRef>();
+				co_return Standalone<StringRef>();
 			}
 		}
 
-		loop {
+		while (true) {
 			if (self->readBufPage) {
 				self->readFromBuffer(&result, &bytes);
 				// if done, return
 				if (!bytes)
-					return result.str;
+					co_return result.str;
 				ASSERT(self->readBufPos == self->readBufPage->payloadSize);
 				self->readBufPage = 0;
 				self->nextReadLocation += sizeof(Page) - self->readBufPos;
 				self->readBufPos = 0;
 			}
 
-			Standalone<StringRef> page = wait(self->rawQueue->readNextPage());
+			Standalone<StringRef> page = co_await self->rawQueue->readNextPage();
 			if (!page.size()) {
 				TraceEvent("DQRecEOF", self->dbgid)
 				    .detail("NextReadLocation", self->nextReadLocation)
@@ -1378,7 +1354,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 				    .detail("Seq", self->readBufPage->seq)
 				    .detail("Expect", pageFloor(self->nextReadLocation))
 				    .detail("File0Name", self->rawQueue->files[0].dbgFilename);
-				wait(self->rawQueue->truncateBeforeLastReadPage());
+				co_await self->rawQueue->truncateBeforeLastReadPage();
 				break;
 			}
 			//TraceEvent("DQRecPage", self->dbgid).detail("NextReadLoc", self->nextReadLocation).detail("Seq", self->readBufPage->seq).detail("Pop", self->readBufPage->popped).detail("Payload", self->readBufPage->payloadSize).detail("File0Name", self->rawQueue->files[0].dbgFilename);
@@ -1393,7 +1369,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		bool poppedNotDurable = self->lastPoppedSeq / sizeof(Page) != self->poppedSeq / sizeof(Page);
 		CODE_PROBE(poppedNotDurable, "DiskQueue: Recovery popped position not fully durable");
 		self->findPhysicalLocation(self->lastPoppedSeq, &f, &p, "lastPoppedSeq");
-		wait(self->rawQueue->setPoppedPage(f, p, pageFloor(self->lastPoppedSeq)));
+		co_await self->rawQueue->setPoppedPage(f, p, pageFloor(self->lastPoppedSeq));
 
 		// Writes go at the end of our reads (but on the next page)
 		self->nextPageSeq = pageFloor(self->nextReadLocation);
@@ -1415,17 +1391,17 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		// check an ASSERTion
 		self->lastCommittedSeq = self->nextReadLocation;
 
-		return result.str;
+		co_return result.str;
 	}
 
 	// recoverAt is the minimum position in the disk queue file that needs to be read to restore log's states.
 	// This allows log to read only a small portion of the most recent data from a large (e.g., 10GB) disk file.
 	// This is particularly useful for logSpilling feature.
-	ACTOR static Future<bool> initializeRecovery(DiskQueue* self, location recoverAt) {
+	static Future<bool> initializeRecovery(DiskQueue* self, location recoverAt) {
 		if (self->initialized) {
-			return self->recovered;
+			co_return self->recovered;
 		}
-		Standalone<StringRef> lastPageData = wait(self->rawQueue->readFirstAndLastPages(&comparePages));
+		Standalone<StringRef> lastPageData = co_await self->rawQueue->readFirstAndLastPages(&comparePages);
 		self->initialized = true;
 
 		if (!lastPageData.size()) {
@@ -1433,7 +1409,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 			self->nextReadLocation = 0;
 			self->lastCommittedSeq = 0;
 			self->recovered = true;
-			return true;
+			co_return true;
 		}
 
 		Page* lastPage = (Page*)lastPageData.begin();
@@ -1469,7 +1445,7 @@ class DiskQueue final : public IDiskQueue, public Tracked<DiskQueue> {
 		    .detail("MinRecoverAt", recoverAt)
 		    .detail("File0Name", self->rawQueue->files[0].dbgFilename);
 
-		return false;
+		co_return false;
 	}
 
 	Page& firstPages(int i) {
@@ -1650,12 +1626,12 @@ class DiskQueue_PopUncommitted final : public IDiskQueue {
 	location popped;
 	location committed;
 
-	ACTOR static Future<Standalone<StringRef>> readNext(DiskQueue_PopUncommitted* self, int bytes) {
-		Standalone<StringRef> str = wait(self->queue->readNext(bytes));
+	static Future<Standalone<StringRef>> readNext(DiskQueue_PopUncommitted* self, int bytes) {
+		Standalone<StringRef> str = co_await self->queue->readNext(bytes);
 		if (str.size() < bytes)
 			self->pushed = self->getNextReadLocation();
 
-		return str;
+		co_return str;
 	}
 };
 
@@ -1668,17 +1644,16 @@ IDiskQueue* openDiskQueue(std::string basename,
 }
 
 TEST_CASE("performance/fdbserver/DiskQueue") {
-	state IDiskQueue* queue =
-	    openDiskQueue("test-", "fdq", deterministicRandom()->randomUniqueID(), DiskQueueVersion::V2);
-	state std::string valueString = std::string(10e6, '.');
-	state StringRef valueStr((uint8_t*)valueString.c_str(), 10e6);
-	state std::deque<IDiskQueue::location> locations;
-	state int loopCount = 0;
-	state Future<Void> lastCommit = Void();
-	bool fullyRecovered = wait(queue->initializeRecovery(0));
+	IDiskQueue* queue = openDiskQueue("test-", "fdq", deterministicRandom()->randomUniqueID(), DiskQueueVersion::V2);
+	std::string valueString = std::string(10e6, '.');
+	StringRef valueStr((uint8_t*)valueString.c_str(), 10e6);
+	std::deque<IDiskQueue::location> locations;
+	int loopCount = 0;
+	Future<Void> lastCommit = Void();
+	bool fullyRecovered = co_await queue->initializeRecovery(0);
 	if (!fullyRecovered) {
-		loop {
-			Standalone<StringRef> h = wait(queue->readNext(1e6));
+		while (true) {
+			Standalone<StringRef> h = co_await queue->readNext(1e6);
 			if (h.size() < 1e6) {
 				break;
 			}
@@ -1689,20 +1664,19 @@ TEST_CASE("performance/fdbserver/DiskQueue") {
 			printf("loop count: %d\n", loopCount);
 		}
 		if (++loopCount % 2 == 0) {
-			state IDiskQueue::location frontLocation = locations.front();
+			IDiskQueue::location frontLocation = locations.front();
 			locations.pop_front();
 			if (locations.size() > 10) {
-				Standalone<StringRef> r = wait(queue->read(frontLocation, locations.front(), CheckHashes::True));
+				Standalone<StringRef> r = co_await queue->read(frontLocation, locations.front(), CheckHashes::True);
 			}
 			queue->pop(frontLocation);
 		}
-		wait(delay(0.001));
+		co_await delay(0.001);
 		locations.push_back(queue->push(valueStr));
 		Future<Void> prevCommit = lastCommit;
 		lastCommit = queue->commit();
-		wait(prevCommit);
+		co_await prevCommit;
 	}
 	queue->dispose();
-	wait(queue->onClosed());
-	return Void();
+	co_await queue->onClosed();
 }
diff --git a/fdbserver/FDBExecHelper.actor.cpp b/fdbserver/kvstore/FDBExecHelper.cpp
similarity index 58%
rename from fdbserver/FDBExecHelper.actor.cpp
rename to fdbserver/kvstore/FDBExecHelper.cpp
index 92ce3cabc48..a2bdc537b96 100644
--- a/fdbserver/FDBExecHelper.actor.cpp
+++ b/fdbserver/kvstore/FDBExecHelper.cpp
@@ -1,5 +1,5 @@
 /*
- * FDBExecHelper.actor.cpp
+ * FDBExecHelper.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -32,24 +32,19 @@
 #endif
 #include <boost/algorithm/string.hpp>
 
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/Trace.h"
 #include "flow/Platform.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 #include "flow/network.h"
-#include "fdbrpc/FlowProcess.actor.h"
-#include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbrpc/WellKnownEndpoints.h"
+#include "fdbclient/IClosable.h"
 #include "fdbclient/versions.h"
 #include "fdbserver/CoroFlow.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RemoteIKeyValueStore.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/kvstore/FDBExecHelper.h"
+#include "fdbserver/core/Knobs.h"
 
 ExecCmdValueString::ExecCmdValueString(StringRef pCmdValueString) {
 	cmdValueString = pCmdValueString;
@@ -85,12 +80,12 @@ void ExecCmdValueString::parseCmdValue() {
 	this->binaryPath = param.eat(" "_sr);
 
 	// no arguments provided
-	if (param == StringRef()) {
+	if (param.empty()) {
 		return;
 	}
 
 	// extract the arguments
-	while (param != StringRef()) {
+	while (!param.empty()) {
 		StringRef token = param.eat(" "_sr);
 		this->binaryArgs.push_back(this->binaryArgs.arena(), token);
 	}
@@ -110,9 +105,12 @@ void ExecCmdValueString::dbgPrint() const {
 	return;
 }
 
-ACTOR void destroyChildProcess(Future<Void> parentSSClosed, ISimulator::ProcessInfo* childInfo, std::string message) {
+Future<Void> destroyChildProcess(Uncancellable,
+                                 Future<Void> parentSSClosed,
+                                 ISimulator::ProcessInfo* childInfo,
+                                 std::string message) {
 	// This code path should be bug free
-	wait(parentSSClosed);
+	co_await parentSSClosed;
 	TraceEvent(SevDebug, message.c_str()).log();
 	// This one is root cause for most failures, make sure it's okay to destroy
 	g_simulator->destroyProcess(childInfo);
@@ -120,132 +118,14 @@ ACTOR void destroyChildProcess(Future<Void> parentSSClosed, ISimulator::ProcessI
 	FlowTransport::transport().resetConnection(childInfo->address);
 }
 
-ACTOR Future<int> spawnSimulated(std::vector<std::string> paramList,
-                                 double maxWaitTime,
-                                 bool isSync,
-                                 double maxSimDelayTime,
-                                 IClosable* parent) {
-	state ISimulator::ProcessInfo* self = g_simulator->getCurrentProcess();
-	state ISimulator::ProcessInfo* child;
-
-	state std::string role;
-	state std::string addr;
-	state std::string flowProcessName;
-	state Endpoint parentProcessEndpoint;
-	state int i = 0;
-	// fdbserver -r flowprocess --process-name ikvs --process-endpoint ip:port,token,id
-	for (; i < paramList.size(); i++) {
-		if (paramList.size() > i + 1) {
-			// temporary args parser that only supports the flowprocess role
-			if (paramList[i] == "-r") {
-				role = paramList[i + 1];
-			} else if (paramList[i] == "-p" || paramList[i] == "--public_address") {
-				addr = paramList[i + 1];
-			} else if (paramList[i] == "--process-name") {
-				flowProcessName = paramList[i + 1];
-			} else if (paramList[i] == "--process-endpoint") {
-				state std::vector<std::string> addressArray;
-				boost::split(addressArray, paramList[i + 1], [](char c) { return c == ','; });
-				if (addressArray.size() != 3) {
-					std::cerr << "Invalid argument, expected 3 elements in --process-endpoint got "
-					          << addressArray.size() << std::endl;
-					flushAndExit(FDB_EXIT_ERROR);
-				}
-				try {
-					auto addr = NetworkAddress::parse(addressArray[0]);
-					uint64_t fst = std::stoul(addressArray[1]);
-					uint64_t snd = std::stoul(addressArray[2]);
-					UID token(fst, snd);
-					NetworkAddressList l;
-					l.address = addr;
-					parentProcessEndpoint = Endpoint(l, token);
-				} catch (Error& e) {
-					std::cerr << "Could not parse network address " << addressArray[0] << std::endl;
-					flushAndExit(FDB_EXIT_ERROR);
-				}
-			}
-		}
-	}
-	state int result = 0;
-	child = g_simulator->newProcess(
-	    "remote flow process",
-	    self->address.ip,
-	    0,
-	    self->address.isTLS(),
-	    self->addresses.secondaryAddress.present() ? 2 : 1,
-	    self->locality,
-	    ProcessClass(ProcessClass::UnsetClass, ProcessClass::AutoSource),
-	    self->dataFolder.c_str(),
-	    self->coordinationFolder.c_str(), // do we need to customize this coordination folder path?
-	    self->protocolVersion,
-	    false);
-	wait(g_simulator->onProcess(child));
-	state Future<ISimulator::KillType> onShutdown = child->onShutdown();
-	state Future<ISimulator::KillType> parentShutdown = self->onShutdown();
-	state Future<Void> flowProcessF;
-
-	try {
-		TraceEvent(SevDebug, "SpawnedChildProcess")
-		    .detail("Child", child->toString())
-		    .detail("Parent", self->toString());
-		std::string role = "";
-		std::string addr = "";
-		for (int i = 0; i < paramList.size(); i++) {
-			if (paramList.size() > i + 1 && paramList[i] == "-r") {
-				role = paramList[i + 1];
-			}
-		}
-		if (role == "flowprocess" && !parentShutdown.isReady()) {
-			self->childs.push_back(child);
-			state Future<Void> parentSSClosed = parent->onClosed();
-			FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT);
-			FlowTransport::transport().bind(child->address, child->address);
-			Sim2FileSystem::newFileSystem();
-			ProcessFactory<KeyValueStoreProcess>(flowProcessName.c_str());
-			flowProcessF = runFlowProcess(flowProcessName, parentProcessEndpoint);
-
-			choose {
-				when(wait(flowProcessF)) {
-					TraceEvent(SevDebug, "ChildProcessKilled").log();
-					wait(g_simulator->onProcess(self));
-					TraceEvent(SevDebug, "BackOnParentProcess").detail("Result", std::to_string(result));
-					destroyChildProcess(parentSSClosed, child, "StorageServerReceivedClosedMessage");
-				}
-				when(wait(success(onShutdown))) {
-					ASSERT(false);
-					// In prod, we use prctl to bind parent and child processes to die together
-					// In simulation, we simply disable killing parent or child processes as we cannot use the same
-					// mechanism here
-				}
-				when(wait(success(parentShutdown))) {
-					ASSERT(false);
-					// Parent process is not killed, see above
-				}
-			}
-		} else {
-			ASSERT(false);
-		}
-	} catch (Error& e) {
-		TraceEvent(SevError, "RemoteIKVSDied").errorUnsuppressed(e);
-		result = -1;
-	}
-
-	return result;
-}
-
 #if defined(_WIN32) || defined(__APPLE__) || defined(__INTEL_COMPILER)
-ACTOR Future<int> spawnProcess(std::string binPath,
-                               std::vector<std::string> paramList,
-                               double maxWaitTime,
-                               bool isSync,
-                               double maxSimDelayTime,
-                               IClosable* parent) {
-	if (g_network->isSimulated() && getExecPath() == binPath) {
-		int res = wait(spawnSimulated(paramList, maxWaitTime, isSync, maxSimDelayTime, parent));
-		return res;
-	}
-	wait(delay(0.0));
-	return 0;
+Future<int> spawnProcess(std::string binPath,
+                         std::vector<std::string> paramList,
+                         double maxWaitTime,
+                         bool isSync,
+                         double maxSimDelayTime) {
+	co_await delay(0.0);
+	co_return 0;
 }
 #else
 
@@ -288,16 +168,11 @@ static void setupTraceWithOutput(TraceEvent& event, size_t bytesRead, char* outp
 	event.detail("Output", std::string(outputBuffer));
 }
 
-ACTOR Future<int> spawnProcess(std::string path,
-                               std::vector<std::string> args,
-                               double maxWaitTime,
-                               bool isSync,
-                               double maxSimDelayTime,
-                               IClosable* parent) {
-	if (g_network->isSimulated() && getExecPath() == path) {
-		int res = wait(spawnSimulated(args, maxWaitTime, isSync, maxSimDelayTime, parent));
-		return res;
-	}
+Future<int> spawnProcess(std::string path,
+                         std::vector<std::string> args,
+                         double maxWaitTime,
+                         bool isSync,
+                         double maxSimDelayTime) {
 	// for async calls in simulator, always delay by a deterministic amount of time and then
 	// do the call synchronously, otherwise the predictability of the simulator breaks
 	if (!isSync && g_network->isSimulated()) {
@@ -305,7 +180,7 @@ ACTOR Future<int> spawnProcess(std::string path,
 		// add some randomness
 		snapDelay += deterministicRandom()->random01();
 		TraceEvent("SnapDelaySpawnProcess").detail("SnapDelay", snapDelay);
-		wait(delay(snapDelay));
+		co_await delay(snapDelay);
 	}
 
 	std::vector<char*> paramList;
@@ -315,28 +190,28 @@ ACTOR Future<int> spawnProcess(std::string path,
 	}
 	paramList.push_back(nullptr);
 
-	state std::string allArgs;
+	std::string allArgs;
 	for (int i = 0; i < args.size(); i++) {
 		if (i > 0)
 			allArgs += " ";
 		allArgs += args[i];
 	}
 
-	state std::pair<pid_t, Optional<int>> pidAndReadFD = fork_child(path, paramList);
-	state pid_t pid = pidAndReadFD.first;
-	state Optional<int> readFD = pidAndReadFD.second;
+	std::pair<pid_t, Optional<int>> pidAndReadFD = fork_child(path, paramList);
+	pid_t pid = pidAndReadFD.first;
+	Optional<int> readFD = pidAndReadFD.second;
 	if (pid == -1) {
 		TraceEvent(SevWarnAlways, "SpawnProcessFailure")
 		    .detail("Reason", "Command failed to spawn")
 		    .detail("Cmd", path)
 		    .detail("Args", allArgs);
-		return -1;
+		co_return -1;
 	} else if (pid > 0) {
-		state int status = -1;
-		state double runTime = 0;
-		state Arena arena;
-		state char* outputBuffer = new (arena) char[SERVER_KNOBS->MAX_FORKED_PROCESS_OUTPUT];
-		state size_t bytesRead = 0;
+		int status = -1;
+		double runTime = 0;
+		Arena arena;
+		char* outputBuffer = new (arena) char[SERVER_KNOBS->MAX_FORKED_PROCESS_OUTPUT];
+		size_t bytesRead = 0;
 		int flags = fcntl(readFD.get(), F_GETFL, 0);
 		fcntl(readFD.get(), F_SETFL, flags | O_NONBLOCK);
 		while (true) {
@@ -347,10 +222,10 @@ ACTOR Future<int> spawnProcess(std::string path,
 				    .detail("Reason", "Command failed, timeout")
 				    .detail("Cmd", path)
 				    .detail("Args", allArgs);
-				return -1;
+				co_return -1;
 			}
 			int err = waitpid(pid, &status, WNOHANG);
-			loop {
+			while (true) {
 				int bytes =
 				    read(readFD.get(), &outputBuffer[bytesRead], SERVER_KNOBS->MAX_FORKED_PROCESS_OUTPUT - bytesRead);
 				if (bytes < 0 && errno == EAGAIN)
@@ -368,7 +243,7 @@ ACTOR Future<int> spawnProcess(std::string path,
 				    .detail("Cmd", path)
 				    .detail("Args", allArgs)
 				    .detail("Errno", WIFEXITED(status) ? WEXITSTATUS(status) : -1);
-				return -1;
+				co_return -1;
 			} else if (err == 0) {
 				// child process has not completed yet
 				if (isSync || g_network->isSimulated()) {
@@ -376,7 +251,7 @@ ACTOR Future<int> spawnProcess(std::string path,
 					threadSleep(0.1);
 				} else {
 					// yield for other actors to run
-					wait(delay(0.1));
+					co_await delay(0.1);
 				}
 				runTime += 0.1;
 			} else {
@@ -388,26 +263,26 @@ ACTOR Future<int> spawnProcess(std::string path,
 					    .detail("Cmd", path)
 					    .detail("Args", allArgs)
 					    .detail("Errno", WIFEXITED(status) ? WEXITSTATUS(status) : -1);
-					return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
+					co_return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
 				}
 				TraceEvent event("SpawnProcessCommandStatus");
 				setupTraceWithOutput(event, bytesRead, outputBuffer);
 				event.detail("Cmd", path)
 				    .detail("Args", allArgs)
 				    .detail("Errno", WIFEXITED(status) ? WEXITSTATUS(status) : 0);
-				return 0;
+				co_return 0;
 			}
 		}
 	}
-	return -1;
+	co_return -1;
 }
 #endif
 
-ACTOR Future<int> execHelper(ExecCmdValueString* execArg, UID snapUID, std::string folder, std::string role) {
-	state Standalone<StringRef> uidStr(snapUID.toString());
-	state int err = 0;
-	state Future<int> cmdErr;
-	state double maxWaitTime = SERVER_KNOBS->SNAP_CREATE_MAX_TIMEOUT;
+static Future<int> execHelperImpl(ExecCmdValueString* execArg, UID snapUID, std::string folder, std::string role) {
+	Standalone<StringRef> uidStr(snapUID.toString());
+	int err = 0;
+	Future<int> cmdErr;
+	double maxWaitTime = SERVER_KNOBS->SNAP_CREATE_MAX_TIMEOUT;
 	if (!g_network->isSimulated()) {
 		// get bin path
 		auto snapBin = execArg->getBinaryPath();
@@ -429,18 +304,18 @@ ACTOR Future<int> execHelper(ExecCmdValueString* execArg, UID snapUID, std::stri
 		paramList.push_back("--uid");
 		paramList.push_back(uidStr.toString());
 		cmdErr = spawnProcess(snapBin.toString(), paramList, maxWaitTime, false /*isSync*/, 0);
-		wait(success(cmdErr));
+		co_await cmdErr;
 		err = cmdErr.get();
 	} else {
 		// copy the files
-		state std::string folderFrom = folder + "/.";
-		state std::string folderTo = folder + "-snap-" + uidStr.toString() + "-" + role;
+		std::string folderFrom = folder + "/.";
+		std::string folderTo = folder + "-snap-" + uidStr.toString() + "-" + role;
 		std::vector<std::string> paramList;
 		std::string mkdirBin = "/bin/mkdir";
 		paramList.push_back(mkdirBin);
 		paramList.push_back(folderTo);
 		cmdErr = spawnProcess(mkdirBin, paramList, maxWaitTime, false /*isSync*/, 10.0);
-		wait(success(cmdErr));
+		co_await cmdErr;
 		err = cmdErr.get();
 		if (err == 0) {
 			std::vector<std::string> paramList;
@@ -450,11 +325,15 @@ ACTOR Future<int> execHelper(ExecCmdValueString* execArg, UID snapUID, std::stri
 			paramList.push_back(folderFrom);
 			paramList.push_back(folderTo);
 			cmdErr = spawnProcess(cpBin, paramList, maxWaitTime, true /*isSync*/, 1.0);
-			wait(success(cmdErr));
+			co_await cmdErr;
 			err = cmdErr.get();
 		}
 	}
-	return err;
+	co_return err;
+}
+
+Future<int> execHelper(ExecCmdValueString* execArg, UID snapUID, std::string folder, std::string role) {
+	return execHelperImpl(execArg, snapUID, folder, role);
 }
 
 struct StorageVersionInfo {
@@ -464,7 +343,7 @@ struct StorageVersionInfo {
 
 // storage nodes get snapshotted through the worker interface which does not have context about version information,
 // following info is gathered at worker level to facilitate printing of version info during storage snapshots.
-typedef std::map<UID, StorageVersionInfo> UidStorageVersionInfo;
+using UidStorageVersionInfo = std::map<UID, StorageVersionInfo>;
 
 std::map<NetworkAddress, UidStorageVersionInfo> workerStorageVersionInfo;
 
diff --git a/fdbserver/include/fdbserver/IKeyValueContainer.h b/fdbserver/kvstore/IKeyValueContainer.h
similarity index 95%
rename from fdbserver/include/fdbserver/IKeyValueContainer.h
rename to fdbserver/kvstore/IKeyValueContainer.h
index 6209adc5376..52746fcf275 100644
--- a/fdbserver/include/fdbserver/IKeyValueContainer.h
+++ b/fdbserver/kvstore/IKeyValueContainer.h
@@ -36,7 +36,7 @@ struct KeyValueMapPair {
 		key = rhs.key;
 		value = rhs.value;
 	}
-	KeyValueMapPair(KeyValueMapPair const& rhs) : arena(rhs.arena), key(rhs.key), value(rhs.value) {}
+	explicit(false) KeyValueMapPair(KeyValueMapPair const& rhs) : arena(rhs.arena), key(rhs.key), value(rhs.value) {}
 
 	KeyValueMapPair(KeyRef key, ValueRef value)
 	  : arena(key.expectedSize() + value.expectedSize()), key(arena, key), value(arena, value) {}
@@ -115,7 +115,7 @@ class IKeyValueContainer {
 	static constexpr int getElementBytes() { return IndexedSet<KeyValueMapPair, uint64_t>::getElementBytes(); }
 
 private:
-	IKeyValueContainer(IKeyValueContainer const&); // unimplemented
+	explicit(false) IKeyValueContainer(IKeyValueContainer const&); // unimplemented
 	void operator=(IKeyValueContainer const&); // unimplemented
 	IndexedSet<KeyValueMapPair, uint64_t> data;
 };
diff --git a/fdbserver/IKeyValueStore.cpp b/fdbserver/kvstore/IKeyValueStore.cpp
similarity index 76%
rename from fdbserver/IKeyValueStore.cpp
rename to fdbserver/kvstore/IKeyValueStore.cpp
index 6e589f0c4d7..2f79dd8a337 100644
--- a/fdbserver/IKeyValueStore.cpp
+++ b/fdbserver/kvstore/IKeyValueStore.cpp
@@ -18,8 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
 #include "flow/flow.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -29,17 +29,18 @@ IKeyValueStore* openKVStore(KeyValueStoreType storeType,
                             int64_t memoryLimit,
                             bool checkChecksums,
                             bool checkIntegrity,
-                            bool openRemotely,
                             Reference<AsyncVar<ServerDBInfo> const> db,
                             int64_t pageCacheBytes) {
-	if (openRemotely) {
-		return openRemoteKVStore(storeType, filename, logID, memoryLimit, checkChecksums, checkIntegrity);
-	}
 	switch (storeType) {
 	case KeyValueStoreType::SSD_BTREE_V1:
-		return keyValueStoreSQLite(filename, logID, KeyValueStoreType::SSD_BTREE_V1, false, checkIntegrity);
+		return keyValueStoreSQLite(
+		    filename, logID, KeyValueStoreType::SSD_BTREE_V1, CheckChecksums::False, CheckIntegrity(checkIntegrity));
 	case KeyValueStoreType::SSD_BTREE_V2:
-		return keyValueStoreSQLite(filename, logID, KeyValueStoreType::SSD_BTREE_V2, checkChecksums, checkIntegrity);
+		return keyValueStoreSQLite(filename,
+		                           logID,
+		                           KeyValueStoreType::SSD_BTREE_V2,
+		                           CheckChecksums(checkChecksums),
+		                           CheckIntegrity(checkIntegrity));
 	case KeyValueStoreType::MEMORY:
 		return keyValueStoreMemory(filename, logID, memoryLimit);
 	case KeyValueStoreType::SSD_REDWOOD_V1:
@@ -47,7 +48,8 @@ IKeyValueStore* openKVStore(KeyValueStoreType storeType,
 	case KeyValueStoreType::SSD_ROCKSDB_V1:
 		return keyValueStoreRocksDB(filename, logID, storeType);
 	case KeyValueStoreType::SSD_SHARDED_ROCKSDB:
-		return keyValueStoreShardedRocksDB(filename, logID, storeType, checkChecksums, checkIntegrity);
+		return keyValueStoreShardedRocksDB(
+		    filename, logID, storeType, CheckChecksums(checkChecksums), CheckIntegrity(checkIntegrity));
 	case KeyValueStoreType::MEMORY_RADIXTREE:
 		return keyValueStoreMemory(filename,
 		                           logID,
diff --git a/fdbserver/IPager.cpp b/fdbserver/kvstore/IPager.cpp
similarity index 93%
rename from fdbserver/IPager.cpp
rename to fdbserver/kvstore/IPager.cpp
index afeaeaa9c28..d29853dd092 100644
--- a/fdbserver/IPager.cpp
+++ b/fdbserver/kvstore/IPager.cpp
@@ -17,7 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "fdbserver/IPager.h"
+#include "IPager.h"
 
 #include "flow/EncryptUtils.h"
 #include "flow/IRandom.h"
@@ -29,7 +29,7 @@ TEST_CASE("/fdbserver/IPager/ArenaPage/PageContentChecksum") {
 	// TODO: it should not be necessary to define this constant here.  ArenaPage or something
 	// should export one.
 	constexpr int _PAGE_SIZE = 8 * 1024;
-	Reference<ArenaPage> page = makeReference<ArenaPage>(_PAGE_SIZE, _PAGE_SIZE);
+	auto page = makeReference<ArenaPage>(_PAGE_SIZE, _PAGE_SIZE);
 	page->init(encodingType, PageType::BTreeNode, 1);
 	deterministicRandom()->randomBytes(page->mutateData(), page->dataSize());
 	PhysicalPageID pageID = deterministicRandom()->randomUInt32();
@@ -47,3 +47,5 @@ TEST_CASE("/fdbserver/IPager/ArenaPage/PageContentChecksum") {
 	}
 	return Void();
 }
+
+void forceLinkIPagerTests() {}
diff --git a/fdbserver/include/fdbserver/IPager.h b/fdbserver/kvstore/IPager.h
similarity index 92%
rename from fdbserver/include/fdbserver/IPager.h
rename to fdbserver/kvstore/IPager.h
index 67afcea74d3..169c232daa6 100644
--- a/fdbserver/include/fdbserver/IPager.h
+++ b/fdbserver/kvstore/IPager.h
@@ -333,6 +333,37 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 		}
 	};
 
+	// Legacy test-only encoding retained to read and continue writing simulation-created Redwood files from
+	// older releases. The xorWith byte is derived from the pager filename and provided by the caller.
+	struct XOREncryptionEncoder {
+		struct Header {
+			XXH64_hash_t checksum;
+			uint8_t xorKey;
+		};
+
+		static void encode(void* header, uint8_t xorWith, uint8_t* payload, int len, PhysicalPageID seed) {
+			Header* h = reinterpret_cast<Header*>(header);
+			h->checksum = XXH3_64bits_withSeed(payload, len, seed);
+			h->xorKey =
+			    static_cast<uint8_t>(deterministicRandom()->randomInt(0, std::numeric_limits<uint8_t>::max() + 1));
+			uint8_t xorMask = static_cast<uint8_t>(~h->xorKey) ^ xorWith;
+			for (int i = 0; i < len; ++i) {
+				payload[i] ^= xorMask;
+			}
+		}
+
+		static void decode(void* header, uint8_t xorWith, uint8_t* payload, int len, PhysicalPageID seed) {
+			const Header* h = reinterpret_cast<const Header*>(header);
+			uint8_t xorMask = static_cast<uint8_t>(~h->xorKey) ^ xorWith;
+			for (int i = 0; i < len; ++i) {
+				payload[i] ^= xorMask;
+			}
+			if (h->checksum != XXH3_64bits_withSeed(payload, len, seed)) {
+				throw page_decoding_failed();
+			}
+		}
+	};
+
 #pragma pack(pop)
 
 	// Get the size of the encoding header based on type
@@ -341,6 +372,9 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 	static int encodingHeaderSize(EncodingType t) {
 		if (t == EncodingType::XXHash64) {
 			return sizeof(XXHashEncoder::Header);
+		} else if (t == EncodingType::XOREncryption_TestOnly_DEPRECATED) {
+			ASSERT(g_network->isSimulated());
+			return sizeof(XOREncryptionEncoder::Header);
 		} else {
 			TraceEvent(SevWarnAlways, "InvalidPageEncoding").detail("EncodingType", t);
 			throw page_encoding_not_supported();
@@ -391,6 +425,7 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 
 		// Non-verifying header parse just to initialize members
 		p->postReadHeader(invalidPhysicalPageID, false);
+		p->legacyXorWith = legacyXorWith;
 
 		return Reference<ArenaPage>(p);
 	}
@@ -404,10 +439,13 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 
 		// Non-verifying header parse just to initialize component pointers
 		p->postReadHeader(invalidPhysicalPageID, false);
+		p->legacyXorWith = legacyXorWith;
 
 		return Reference<ArenaPage>(p);
 	}
 
+	void setLegacyXorWith(uint8_t xorWith) { legacyXorWith = xorWith; }
+
 	// The next two functions set mostly forensic info that may help in an investigation to identify data on disk.  The
 	// exception is pageID which must be set to the physical page ID on disk where the page is written or post-read
 	// verification will fail.
@@ -444,6 +482,12 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 
 		if (page->encodingType == EncodingType::XXHash64) {
 			XXHashEncoder::encode(page->getEncodingHeader(), pPayload, payloadSize, pageID);
+		} else if (page->encodingType == EncodingType::XOREncryption_TestOnly_DEPRECATED) {
+			if (!legacyXorWith.present()) {
+				TraceEvent(SevWarnAlways, "LegacyXorCompatibilityNotInitialized");
+				throw page_encoding_not_supported();
+			}
+			XOREncryptionEncoder::encode(page->getEncodingHeader(), legacyXorWith.get(), pPayload, payloadSize, pageID);
 		} else {
 			TraceEvent(SevWarnAlways, "InvalidPageEncoding").detail("EncodingType", page->encodingType);
 			throw page_encoding_not_supported();
@@ -489,6 +533,12 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 	void postReadPayload(PhysicalPageID pageID, double* decryptTime = nullptr) {
 		if (page->encodingType == EncodingType::XXHash64) {
 			XXHashEncoder::decode(page->getEncodingHeader(), pPayload, payloadSize, pageID);
+		} else if (page->encodingType == EncodingType::XOREncryption_TestOnly_DEPRECATED) {
+			if (!legacyXorWith.present()) {
+				TraceEvent(SevWarnAlways, "LegacyXorCompatibilityNotInitialized");
+				throw page_encoding_not_supported();
+			}
+			XOREncryptionEncoder::decode(page->getEncodingHeader(), legacyXorWith.get(), pPayload, payloadSize, pageID);
 		} else {
 			TraceEvent(SevWarnAlways, "InvalidPageEncoding").detail("EncodingType", page->encodingType);
 			throw page_encoding_not_supported();
@@ -523,6 +573,7 @@ class ArenaPage : public ReferenceCounted<ArenaPage>, public FastAllocated<Arena
 	// These are accessed very often so they are stored directly
 	uint8_t* pPayload;
 	int payloadSize;
+	Optional<uint8_t> legacyXorWith;
 
 public:
 	EncodingType getEncodingType() const { return page->encodingType; }
diff --git a/fdbserver/KeyValueStoreCompressTestData.actor.cpp b/fdbserver/kvstore/KeyValueStoreCompressTestData.cpp
similarity index 78%
rename from fdbserver/KeyValueStoreCompressTestData.actor.cpp
rename to fdbserver/kvstore/KeyValueStoreCompressTestData.cpp
index 63f147789c0..920939962a9 100644
--- a/fdbserver/KeyValueStoreCompressTestData.actor.cpp
+++ b/fdbserver/kvstore/KeyValueStoreCompressTestData.cpp
@@ -1,5 +1,5 @@
 /*
- * KeyValueStoreCompressTestData.actor.cpp
+ * KeyValueStoreCompressTestData.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,8 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/IKeyValueStore.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/kvstore/IKeyValueStore.h"
 
 // KeyValueStoreCompressTestData wraps an existing IKeyValueStore and
 // implements the following rudimentary compression scheme:
@@ -33,7 +32,7 @@
 struct KeyValueStoreCompressTestData final : IKeyValueStore {
 	IKeyValueStore* store;
 
-	KeyValueStoreCompressTestData(IKeyValueStore* store) : store(store) {}
+	explicit KeyValueStoreCompressTestData(IKeyValueStore* store) : store(store) {}
 
 	Future<Void> getError() const override { return store->getError(); }
 	Future<Void> onClosed() const override { return store->onClosed(); }
@@ -78,42 +77,42 @@ struct KeyValueStoreCompressTestData final : IKeyValueStore {
 	}
 
 private:
-	ACTOR static Future<Optional<Value>> doReadValue(IKeyValueStore* store, Key key, Optional<ReadOptions> options) {
-		Optional<Value> v = wait(store->readValue(key, options));
+	static Future<Optional<Value>> doReadValue(IKeyValueStore* store, Key key, Optional<ReadOptions> options) {
+		Optional<Value> v = co_await store->readValue(key, options);
 		if (!v.present())
-			return v;
-		return unpack(v.get());
+			co_return v;
+		co_return unpack(v.get());
 	}
 
-	ACTOR static Future<Optional<Value>> doReadValuePrefix(IKeyValueStore* store,
-	                                                       Key key,
-	                                                       int maxLength,
-	                                                       Optional<ReadOptions> options) {
-		Optional<Value> v = wait(doReadValue(store, key, options));
+	static Future<Optional<Value>> doReadValuePrefix(IKeyValueStore* store,
+	                                                 Key key,
+	                                                 int maxLength,
+	                                                 Optional<ReadOptions> options) {
+		Optional<Value> v = co_await doReadValue(store, key, options);
 		if (!v.present())
-			return v;
+			co_return v;
 		if (maxLength < v.get().size()) {
-			return v.get().substr(0, maxLength);
+			co_return v.get().substr(0, maxLength);
 		} else {
-			return v;
+			co_return v;
 		}
 	}
-	ACTOR Future<RangeResult> doReadRange(IKeyValueStore* store,
-	                                      KeyRangeRef keys,
-	                                      int rowLimit,
-	                                      int byteLimit,
-	                                      Optional<ReadOptions> options) {
-		RangeResult _vs = wait(store->readRange(keys, rowLimit, byteLimit, options));
+	Future<RangeResult> doReadRange(IKeyValueStore* store,
+	                                KeyRangeRef keys,
+	                                int rowLimit,
+	                                int byteLimit,
+	                                Optional<ReadOptions> options) {
+		RangeResult _vs = co_await store->readRange(keys, rowLimit, byteLimit, options);
 		RangeResult vs = _vs; // Get rid of implicit const& from wait statement
 		Arena& a = vs.arena();
 		for (int i = 0; i < vs.size(); i++)
 			vs[i].value = ValueRef(a, (ValueRef const&)unpack(vs[i].value));
-		return vs;
+		co_return vs;
 	}
 
 	// These implement the actual "compression" scheme
 	static Value pack(Value val) {
-		if (!val.size())
+		if (val.empty())
 			return val;
 		uint8_t c = val[0];
 
@@ -136,7 +135,7 @@ struct KeyValueStoreCompressTestData final : IKeyValueStore {
 		return val;
 	}
 	static Value unpack(Value val) {
-		if (!val.size())
+		if (val.empty())
 			return val;
 		if (val[0] == 0)
 			return val.substr(1); // Uncompressed value
diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/kvstore/KeyValueStoreMemory.cpp
similarity index 86%
rename from fdbserver/KeyValueStoreMemory.actor.cpp
rename to fdbserver/kvstore/KeyValueStoreMemory.cpp
index d91f9d92b4f..203206da360 100644
--- a/fdbserver/KeyValueStoreMemory.actor.cpp
+++ b/fdbserver/kvstore/KeyValueStoreMemory.cpp
@@ -1,5 +1,5 @@
 /*
- * KeyValueStoreMemory.actor.cpp
+ * KeyValueStoreMemory.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,17 +22,16 @@
 #include "fdbclient/Knobs.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/DeltaTree.h"
-#include "fdbserver/IDiskQueue.h"
-#include "fdbserver/IKeyValueContainer.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/RadixTree.h"
-#include "fdbserver/TransactionStoreMutationTracking.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "DeltaTree.h"
+#include "fdbserver/kvstore/IDiskQueue.h"
+#include "IKeyValueContainer.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "RadixTree.h"
+#include "TransactionStoreMutationTracking.h"
 #include "flow/ActorCollection.h"
 #include "flow/EncryptUtils.h"
 #include "flow/Knobs.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 #define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1)
 static_assert(sizeof(uint32_t) == 4);
@@ -45,9 +44,9 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 	                    UID id,
 	                    int64_t memoryLimit,
 	                    KeyValueStoreType storeType,
-	                    bool disableSnapshot,
-	                    bool replaceContent,
-	                    bool exactRecovery);
+	                    DisableSnapshot,
+	                    ReplaceContent,
+	                    ExactRecovery);
 
 	bool getReplaceContent() const override { return replaceContent; }
 	// IClosable
@@ -305,7 +304,7 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 	struct OpRef {
 		OpType op;
 		StringRef p1, p2;
-		OpRef() {}
+		OpRef() = default;
 		OpRef(Arena& a, OpRef const& o) : op(o.op), p1(a, o.p1), p2(a, o.p2) {}
 		size_t expectedSize() const { return p1.expectedSize() + p2.expectedSize(); }
 	};
@@ -465,52 +464,52 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 		return loc;
 	}
 
-	ACTOR static Future<Standalone<StringRef>> readOpData(KeyValueStoreMemory* self,
-	                                                      OpHeader h,
-	                                                      bool* isZeroFilled,
-	                                                      int* zeroFillSize) {
-		state int remainingBytes = h.len1 + h.len2 + 1;
-		state Standalone<StringRef> data = wait(self->log->readNext(remainingBytes));
+	static Future<Standalone<StringRef>> readOpData(KeyValueStoreMemory* self,
+	                                                OpHeader h,
+	                                                bool* isZeroFilled,
+	                                                int* zeroFillSize) {
+		int remainingBytes = h.len1 + h.len2 + 1;
+		Standalone<StringRef> data = co_await self->log->readNext(remainingBytes);
 		ASSERT(data.size() <= remainingBytes);
 		*zeroFillSize = remainingBytes - data.size();
 		if (*zeroFillSize == 0) {
 			*isZeroFilled = (data[data.size() - 1] == 0);
 		}
-		return data;
+		co_return data;
 	}
 
-	ACTOR static Future<Void> recover(KeyValueStoreMemory* self, bool exactRecovery) {
-		loop {
+	static Future<Void> recover(KeyValueStoreMemory* self, bool exactRecovery) {
+		while (true) {
 			// 'uncommitted' variables track something that might be rolled back by an OpRollback, and are copied into
 			// permanent variables (in self) in OpCommit.  OpRollback does the reverse (copying the permanent versions
 			// over the uncommitted versions) the uncommitted and committed variables should be equal initially (to
 			// whatever makes sense if there are no committed transactions recovered)
-			state Key uncommittedNextKey = self->recoveredSnapshotKey;
-			state IDiskQueue::location uncommittedPrevSnapshotEnd = self->previousSnapshotEnd =
+			Key uncommittedNextKey = self->recoveredSnapshotKey;
+			IDiskQueue::location uncommittedPrevSnapshotEnd = self->previousSnapshotEnd =
 			    self->log->getNextReadLocation(); // not really, but popping up to here does nothing
-			state IDiskQueue::location uncommittedSnapshotEnd = self->currentSnapshotEnd = uncommittedPrevSnapshotEnd;
+			IDiskQueue::location uncommittedSnapshotEnd = self->currentSnapshotEnd = uncommittedPrevSnapshotEnd;
 
-			state int zeroFillSize = 0;
-			state int dbgSnapshotItemCount = 0;
-			state int dbgSnapshotEndCount = 0;
-			state int dbgMutationCount = 0;
-			state int dbgCommitCount = 0;
-			state double startt = now();
-			state UID dbgid = self->id;
+			int zeroFillSize = 0;
+			int dbgSnapshotItemCount = 0;
+			int dbgSnapshotEndCount = 0;
+			int dbgMutationCount = 0;
+			int dbgCommitCount = 0;
+			double startt = now();
+			UID dbgid = self->id;
 
-			state Future<Void> loggingDelay = delay(1.0);
+			Future<Void> loggingDelay = delay(1.0);
 
-			state OpQueue recoveryQueue;
-			state OpHeader h;
-			state Standalone<StringRef> lastSnapshotKey;
-			state bool isZeroFilled;
+			OpQueue recoveryQueue;
+			OpHeader h;
+			Standalone<StringRef> lastSnapshotKey;
+			bool isZeroFilled{ false };
 
 			TraceEvent("KVSMemRecoveryStarted", self->id).detail("SnapshotEndLocation", uncommittedSnapshotEnd);
 
 			try {
-				loop {
+				while (true) {
 					{
-						Standalone<StringRef> data = wait(self->log->readNext(sizeof(OpHeader)));
+						Standalone<StringRef> data = co_await self->log->readNext(sizeof(OpHeader));
 						if (data.size() != sizeof(OpHeader)) {
 							if (data.size()) {
 								CODE_PROBE(
@@ -530,7 +529,7 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 						h = *(OpHeader*)data.begin();
 						ASSERT(h.op != OpEncrypted_Deprecated);
 					}
-					state Standalone<StringRef> data = wait(readOpData(self, h, &isZeroFilled, &zeroFillSize));
+					Standalone<StringRef> data = co_await readOpData(self, h, &isZeroFilled, &zeroFillSize);
 					if (zeroFillSize > 0) {
 						TraceEvent("KVSMemRecoveryComplete", self->id)
 						    .detail("Reason", "data specified by header does not exist")
@@ -635,7 +634,7 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 						loggingDelay = delay(1.0);
 					}
 
-					wait(yield());
+					co_await yield();
 				}
 
 				if (zeroFillSize) {
@@ -665,7 +664,7 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 
 				self->semiCommit();
 
-				return Void();
+				co_return;
 			} catch (Error& e) {
 				bool ok = e.code() == error_code_operation_cancelled || e.code() == error_code_file_not_found ||
 				          e.code() == error_code_disk_adapter_reset;
@@ -705,27 +704,27 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 		currentSnapshotEnd = log_op(OpSnapshotEnd, StringRef(), StringRef());
 	}
 
-	ACTOR static Future<Void> snapshot(KeyValueStoreMemory* self) {
-		wait(self->recovering);
+	static Future<Void> snapshot(KeyValueStoreMemory* self) {
+		co_await self->recovering;
 
-		state Key nextKey = self->recoveredSnapshotKey;
-		state bool nextKeyAfter = false; // setting this to true is equilvent to setting nextKey = keyAfter(nextKey)
-		state uint64_t snapshotTotalWrittenBytes = 0;
-		state int lastDiff = 0;
-		state int snapItems = 0;
-		state uint64_t snapshotBytes = 0;
+		Key nextKey = self->recoveredSnapshotKey;
+		bool nextKeyAfter = false; // setting this to true is equilvent to setting nextKey = keyAfter(nextKey)
+		uint64_t snapshotTotalWrittenBytes = 0;
+		int lastDiff = 0;
+		int snapItems = 0;
+		uint64_t snapshotBytes = 0;
 
 		// Snapshot keys will be alternately written to two preallocated buffers.
 		// This allows consecutive snapshot keys to be compared for delta compression while only copying each key's
 		// bytes once.
-		state Key lastSnapshotKeyA = makeString(CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
-		state Key lastSnapshotKeyB = makeString(CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
-		state bool lastSnapshotKeyUsingA = true;
+		Key lastSnapshotKeyA = makeString(CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
+		Key lastSnapshotKeyB = makeString(CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
+		bool lastSnapshotKeyUsingA = true;
 
 		TraceEvent("KVSMemStartingSnapshot", self->id).detail("StartKey", nextKey);
 
-		loop {
-			wait(self->notifiedCommittedWriteBytes.whenAtLeast(snapshotTotalWrittenBytes + 1));
+		while (true) {
+			co_await self->notifiedCommittedWriteBytes.whenAtLeast(snapshotTotalWrittenBytes + 1);
 
 			if (self->resetSnapshot) {
 				nextKey = Key();
@@ -751,7 +750,7 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 			bool useDelta = false;
 
 			// Write snapshot items until the wait above would block because we've used up all of the byte budget
-			loop {
+			while (true) {
 
 				if (next == self->data.end()) {
 					// After a snapshot end is logged, recovery may not see the last snapshot item logged before it so
@@ -862,38 +861,34 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 		}
 	}
 
-	ACTOR static Future<Optional<Value>> waitAndReadValue(KeyValueStoreMemory* self,
+	static Future<Optional<Value>> waitAndReadValue(KeyValueStoreMemory* self, Key key, Optional<ReadOptions> options) {
+		co_await self->recovering;
+		co_return static_cast<IKeyValueStore*>(self)->readValue(key, options).get();
+	}
+	static Future<Optional<Value>> waitAndReadValuePrefix(KeyValueStoreMemory* self,
 	                                                      Key key,
+	                                                      int maxLength,
 	                                                      Optional<ReadOptions> options) {
-		wait(self->recovering);
-		return static_cast<IKeyValueStore*>(self)->readValue(key, options).get();
-	}
-	ACTOR static Future<Optional<Value>> waitAndReadValuePrefix(KeyValueStoreMemory* self,
-	                                                            Key key,
-	                                                            int maxLength,
-	                                                            Optional<ReadOptions> options) {
-		wait(self->recovering);
-		return static_cast<IKeyValueStore*>(self)->readValuePrefix(key, maxLength, options).get();
+		co_await self->recovering;
+		co_return static_cast<IKeyValueStore*>(self)->readValuePrefix(key, maxLength, options).get();
 	}
-	ACTOR static Future<RangeResult> waitAndReadRange(KeyValueStoreMemory* self,
-	                                                  KeyRange keys,
-	                                                  int rowLimit,
-	                                                  int byteLimit,
-	                                                  Optional<ReadOptions> options) {
-		wait(self->recovering);
-		return static_cast<IKeyValueStore*>(self)->readRange(keys, rowLimit, byteLimit, options).get();
+	static Future<RangeResult> waitAndReadRange(KeyValueStoreMemory* self,
+	                                            KeyRange keys,
+	                                            int rowLimit,
+	                                            int byteLimit,
+	                                            Optional<ReadOptions> options) {
+		co_await self->recovering;
+		co_return static_cast<IKeyValueStore*>(self)->readRange(keys, rowLimit, byteLimit, options).get();
 	}
-	ACTOR static Future<Void> waitAndCommit(KeyValueStoreMemory* self, bool sequential) {
-		wait(self->recovering);
-		wait(self->commit(sequential));
-		return Void();
+	static Future<Void> waitAndCommit(KeyValueStoreMemory* self, bool sequential) {
+		co_await self->recovering;
+		co_await self->commit(sequential);
 	}
-	ACTOR static Future<Void> commitAndUpdateVersions(KeyValueStoreMemory* self,
-	                                                  Future<Void> commit,
-	                                                  IDiskQueue::location location) {
-		wait(commit);
+	static Future<Void> commitAndUpdateVersions(KeyValueStoreMemory* self,
+	                                            Future<Void> commit,
+	                                            IDiskQueue::location location) {
+		co_await commit;
 		self->log->pop(location);
-		return Void();
 	}
 };
 
@@ -903,9 +898,9 @@ KeyValueStoreMemory<Container>::KeyValueStoreMemory(IDiskQueue* log,
                                                     UID id,
                                                     int64_t memoryLimit,
                                                     KeyValueStoreType storeType,
-                                                    bool disableSnapshot,
-                                                    bool replaceContent,
-                                                    bool exactRecovery)
+                                                    DisableSnapshot disableSnapshot,
+                                                    ReplaceContent replaceContent,
+                                                    ExactRecovery exactRecovery)
   : type(storeType), id(id), log(log), db(db), committedWriteBytes(0), overheadWriteBytes(0), currentSnapshotEnd(-1),
     previousSnapshotEnd(-1), committedDataSize(0), transactionSize(0), transactionIsLarge(false), resetSnapshot(false),
     disableSnapshot(disableSnapshot), replaceContent(replaceContent), firstCommitWithSnapshot(true), snapshotCount(0),
@@ -939,18 +934,18 @@ IKeyValueStore* keyValueStoreMemory(std::string const& basename,
 		                                           logID,
 		                                           memoryLimit,
 		                                           storeType,
-		                                           /*doc*/ false,
-		                                           /*ument*/ false,
-		                                           /*thisstuff FFS*/ false);
+		                                           DisableSnapshot::False,
+		                                           ReplaceContent::False,
+		                                           ExactRecovery::False);
 	} else {
 		return new KeyValueStoreMemory<IKeyValueContainer>(log,
 		                                                   Reference<AsyncVar<ServerDBInfo> const>(),
 		                                                   logID,
 		                                                   memoryLimit,
 		                                                   storeType,
-		                                                   /* name */ false,
-		                                                   /*the */ false,
-		                                                   /* effing parameter*/ false);
+		                                                   DisableSnapshot::False,
+		                                                   ReplaceContent::False,
+		                                                   ExactRecovery::False);
 	}
 }
 
@@ -958,9 +953,9 @@ IKeyValueStore* keyValueStoreLogSystem(class IDiskQueue* queue,
                                        Reference<AsyncVar<ServerDBInfo> const> db,
                                        UID logID,
                                        int64_t memoryLimit,
-                                       bool disableSnapshot,
-                                       bool replaceContent,
-                                       bool exactRecovery) {
+                                       DisableSnapshot disableSnapshot,
+                                       ReplaceContent replaceContent,
+                                       ExactRecovery exactRecovery) {
 	return new KeyValueStoreMemory<IKeyValueContainer>(
 	    queue, db, logID, memoryLimit, KeyValueStoreType::MEMORY, disableSnapshot, replaceContent, exactRecovery);
 }
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp
similarity index 95%
rename from fdbserver/KeyValueStoreRocksDB.actor.cpp
rename to fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp
index 24b2272834a..43619e1f456 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp
@@ -50,8 +50,8 @@
 #endif
 #include "fdbclient/SystemData.h"
 #include "fdbserver/CoroFlow.h"
-#include "fdbserver/FDBRocksDBVersion.h"
-#include "fdbserver/RocksDBLogForwarder.h"
+#include "fdbserver/core/FDBRocksDBVersion.h"
+#include "RocksDBLogForwarder.h"
 #include "flow/ActorCollection.h"
 #include "flow/flow.h"
 #include "flow/IThreadPool.h"
@@ -65,10 +65,10 @@
 
 #endif // WITH_ROCKSDB
 
-#include "fdbserver/Knobs.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/RocksDBCommon.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
+#include "RocksDBCommon.h"
 
 #include "flow/actorcompiler.h" // has to be last include
 
@@ -94,7 +94,7 @@ rocksdb::BlockBasedTableOptions::IndexType getIndexType() {
 
 class SharedRocksDBState {
 public:
-	SharedRocksDBState(UID id);
+	explicit SharedRocksDBState(UID id);
 
 	void setClosing() { this->closing = true; }
 	bool isClosing() const { return this->closing; }
@@ -327,7 +327,7 @@ using RocksDBCommon::getErrorReason;
 // could potentially cause segmentation fault.
 class RocksDBErrorListener : public rocksdb::EventListener {
 public:
-	RocksDBErrorListener(UID id) : id(id) {};
+	explicit RocksDBErrorListener(UID id) : id(id) {};
 	void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* bg_error) override {
 		TraceEvent(SevError, "RocksDBBGError", id)
 		    .detail("Reason", getErrorReason(reason))
@@ -352,7 +352,7 @@ class RocksDBErrorListener : public rocksdb::EventListener {
 		std::unique_lock<std::mutex> lock(mutex);
 		return errorPromise.getFuture();
 	}
-	~RocksDBErrorListener() {
+	~RocksDBErrorListener() override {
 		std::unique_lock<std::mutex> lock(mutex);
 		if (!errorPromise.isValid())
 			return;
@@ -367,7 +367,7 @@ class RocksDBErrorListener : public rocksdb::EventListener {
 
 class RocksDBEventListener : public rocksdb::EventListener {
 public:
-	RocksDBEventListener(std::shared_ptr<SharedRocksDBState> sharedState) : sharedState(sharedState) {};
+	explicit RocksDBEventListener(std::shared_ptr<SharedRocksDBState> sharedState) : sharedState(sharedState) {};
 
 	void OnFlushCompleted(rocksdb::DB* db, const rocksdb::FlushJobInfo& info) override {
 		sharedState->setLastFlushTime(now());
@@ -512,9 +512,25 @@ struct ReadIterator {
 	std::shared_ptr<rocksdb::Slice> beginSlice, endSlice;
 	ReadIterator(CF& cf, uint64_t index, DB& db, std::shared_ptr<SharedRocksDBState> sharedState)
 	  : index(index), inUse(true), creationTime(now()), iter(db->NewIterator(sharedState->getReadOptions(), cf)) {}
-	ReadIterator(CF& cf, uint64_t index, DB& db, std::shared_ptr<SharedRocksDBState> sharedState, KeyRange keyRange)
+	ReadIterator(CF& cf,
+	             uint64_t index,
+	             DB& db,
+	             std::shared_ptr<SharedRocksDBState> sharedState,
+	             KeyRange keyRange,
+	             bool cacheResult = true)
 	  : index(index), inUse(true), creationTime(now()), keyRange(keyRange) {
 		rocksdb::ReadOptions readOptions = sharedState->getReadOptions();
+		// Honor the txn.ReadOptions.CacheResult setting when this ROCKSDB_USE_CACHE_RESULT_OPTION is enabled.
+		if (g_network->isSimulated()) {
+			// Disabling rocksdb cache in simulation can cause slow down, so honoring txn.ReadOptions.CacheResult
+			// option only for few requests, controlled by SERVER_KNOBS->ROCKSDB_PROBABILITY_DISABLE_CACHE_SIM.
+			if (SERVER_KNOBS->ROCKSDB_USE_CACHE_RESULT_OPTION &&
+			    deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PROBABILITY_DISABLE_CACHE_SIM)
+				readOptions.fill_cache = cacheResult;
+		} else {
+			if (SERVER_KNOBS->ROCKSDB_USE_CACHE_RESULT_OPTION)
+				readOptions.fill_cache = cacheResult;
+		}
 		beginSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.begin)));
 		readOptions.iterate_lower_bound = beginSlice.get();
 		endSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.end)));
@@ -567,16 +583,18 @@ class ReadIteratorPool {
 	}
 
 	// Called on every read operation.
-	ReadIterator getIterator(KeyRange keyRange) {
+	ReadIterator getIterator(KeyRange keyRange, bool cacheResult = true) {
 		// Reusing iterator in simulation can cause slow down
 		// We avoid to always reuse iterator in simulation to speed up the simulation
 		if (g_network->isSimulated() &&
 		    deterministicRandom()->random01() > SERVER_KNOBS->ROCKSDB_PROBABILITY_REUSE_ITERATOR_SIM) {
 			index++;
-			ReadIterator iter(cf, index, db, sharedState, keyRange);
+			ReadIterator iter(cf, index, db, sharedState, keyRange, cacheResult);
 			return iter;
 		}
 
+		// When ROCKSDB_READ_RANGE_REUSE_ITERATORS is true, the iterator is not bounded
+		// and reused, so txn.ReadOptions.CacheResult is ignored and always reads are cached.
 		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
 			mutex.lock();
 			for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) {
@@ -614,8 +632,10 @@ class ReadIteratorPool {
 			uint64_t readIteratorIndex = index;
 			mutex.unlock();
 
-			ReadIterator iter(cf, readIteratorIndex, db, sharedState, keyRange);
-			if (iteratorsMap.size() < SERVER_KNOBS->ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT) {
+			ReadIterator iter(cf, readIteratorIndex, db, sharedState, keyRange, cacheResult);
+			// If txn.ReadOptions.CacheResult is false, less probability to get the read in
+			// the same range, so don't save the iterator for reuse.
+			if (iteratorsMap.size() < SERVER_KNOBS->ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT && cacheResult) {
 				// Not storing more than ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT of iterators
 				// to avoid 'out of memory' issues.
 				mutex.lock();
@@ -625,7 +645,7 @@ class ReadIteratorPool {
 			return iter;
 		} else {
 			index++;
-			ReadIterator iter(cf, index, db, sharedState, keyRange);
+			ReadIterator iter(cf, index, db, sharedState, keyRange, cacheResult);
 			return iter;
 		}
 	}
@@ -965,9 +985,9 @@ ACTOR Future<Void> refreshReadIteratorPool(std::shared_ptr<ReadIteratorPool> rea
 	return Void();
 }
 
-ACTOR Future<Void> flowLockLogger(UID id, const FlowLock* readLock, const FlowLock* fetchLock) {
-	loop {
-		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+Future<Void> flowLockLogger(UID id, const FlowLock* readLock, const FlowLock* fetchLock) {
+	while (true) {
+		co_await delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY);
 		TraceEvent e("RocksDBFlowLock", id);
 		e.detail("ReadAvailable", readLock->available());
 		e.detail("ReadActivePermits", readLock->activePermits());
@@ -978,17 +998,15 @@ ACTOR Future<Void> flowLockLogger(UID id, const FlowLock* readLock, const FlowLo
 	}
 }
 
-ACTOR Future<Void> manualFlush(UID id, rocksdb::DB* db, std::shared_ptr<SharedRocksDBState> sharedState, CF cf) {
+Future<Void> manualFlush(UID id, rocksdb::DB* db, std::shared_ptr<SharedRocksDBState> sharedState, CF cf) {
 	if (SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL) {
-		state rocksdb::FlushOptions fOptions = sharedState->getFlushOptions();
-		state double waitTime = SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL;
-		state double currTime = 0;
-		state int timeElapsedAfterLastFlush = 0;
-		loop {
-			wait(delay(waitTime));
+		rocksdb::FlushOptions fOptions = sharedState->getFlushOptions();
+		double waitTime = SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL;
+		while (true) {
+			co_await delay(waitTime);
 
-			currTime = now();
-			timeElapsedAfterLastFlush = currTime - sharedState->getLastFlushTime();
+			double currTime = now();
+			int timeElapsedAfterLastFlush = currTime - sharedState->getLastFlushTime();
 			if (timeElapsedAfterLastFlush >= SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL) {
 				db->Flush(fOptions, cf);
 				waitTime = SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL;
@@ -1005,18 +1023,17 @@ ACTOR Future<Void> manualFlush(UID id, rocksdb::DB* db, std::shared_ptr<SharedRo
 			}
 		}
 	}
-	return Void();
 }
 
-ACTOR Future<Void> rocksDBMetricLogger(UID id,
-                                       std::shared_ptr<SharedRocksDBState> sharedState,
-                                       std::shared_ptr<rocksdb::Statistics> statistics,
-                                       std::shared_ptr<PerfContextMetrics> perfContextMetrics,
-                                       rocksdb::DB* db,
-                                       std::shared_ptr<ReadIteratorPool> readIterPool,
-                                       Counters* counters,
-                                       CF cf) {
-	state std::vector<std::tuple<const char*, uint32_t, uint64_t>> tickerStats = {
+Future<Void> rocksDBMetricLogger(UID id,
+                                 std::shared_ptr<SharedRocksDBState> sharedState,
+                                 std::shared_ptr<rocksdb::Statistics> statistics,
+                                 std::shared_ptr<PerfContextMetrics> perfContextMetrics,
+                                 rocksdb::DB* db,
+                                 std::shared_ptr<ReadIteratorPool> readIterPool,
+                                 Counters* counters,
+                                 CF cf) {
+	std::vector<std::tuple<const char*, uint32_t, uint64_t>> tickerStats = {
 		{ "StallMicros", rocksdb::STALL_MICROS, 0 },
 		{ "BytesRead", rocksdb::BYTES_READ, 0 },
 		{ "IterBytesRead", rocksdb::ITER_BYTES_READ, 0 },
@@ -1073,7 +1090,7 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 
 	// To control the rocksdb::StatsLevel, use ROCKSDB_STATS_LEVEL knob.
 	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
-	state std::vector<std::pair<const char*, uint32_t>> histogramStats = {
+	std::vector<std::pair<const char*, uint32_t>> histogramStats = {
 		{ "CompactionTime", rocksdb::COMPACTION_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
 		{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
 		{ "CompressionTimeNanos",
@@ -1088,7 +1105,7 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "WriteStall", rocksdb::WRITE_STALL }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
 	};
 
-	state std::vector<std::pair<const char*, std::string>> intPropertyStats = {
+	std::vector<std::pair<const char*, std::string>> intPropertyStats = {
 		{ "NumImmutableMemtables", rocksdb::DB::Properties::kNumImmutableMemTable },
 		{ "NumImmutableMemtablesFlushed", rocksdb::DB::Properties::kNumImmutableMemTableFlushed },
 		{ "IsMemtableFlushPending", rocksdb::DB::Properties::kMemTableFlushPending },
@@ -1117,23 +1134,23 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "ObsoleteSstFilesSize", rocksdb::DB::Properties::kObsoleteSstFilesSize },
 	};
 
-	state std::vector<std::pair<const char*, std::string>> strPropertyStats = {
+	std::vector<std::pair<const char*, std::string>> strPropertyStats = {
 		{ "LevelStats", rocksdb::DB::Properties::kLevelStats },
 		{ "BlockCacheEntryStats", rocksdb::DB::Properties::kBlockCacheEntryStats },
 	};
 
-	state std::vector<std::pair<const char*, std::string>> levelStrPropertyStats = {
+	std::vector<std::pair<const char*, std::string>> levelStrPropertyStats = {
 		{ "CompressionRatioAtLevel", rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix },
 	};
 
-	state std::unordered_map<std::string, uint64_t> readIteratorPoolStats = {
+	std::unordered_map<std::string, uint64_t> readIteratorPoolStats = {
 		{ "NumReadIteratorsCreated", 0 },
 		{ "NumTimesReadIteratorsReused", 0 },
 	};
 
-	state std::string rocksdbMetricsTrackingKey = id.toString() + "/RocksDBMetrics";
-	loop {
-		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+	std::string rocksdbMetricsTrackingKey = id.toString() + "/RocksDBMetrics";
+	while (true) {
+		co_await delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY);
 		if (sharedState->isClosing()) {
 			break;
 		}
@@ -1202,8 +1219,6 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 			perfContextMetrics->log(true);
 		}
 	}
-
-	return Void();
 }
 
 void logRocksDBError(UID id,
@@ -1234,7 +1249,7 @@ Error statusToError(const rocksdb::Status& s) {
 struct RocksDBKeyValueStore : IKeyValueStore {
 	struct Writer : IThreadPoolReceiver {
 		struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
-			CheckpointAction(const CheckpointRequest& request) : request(request) {}
+			explicit CheckpointAction(const CheckpointRequest& request) : request(request) {}
 
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 
@@ -1281,7 +1296,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		void init() override {}
 
 		struct IngestSSTFilesAction : TypedAction<Writer, IngestSSTFilesAction> {
-			IngestSSTFilesAction(std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) : localFileSets(localFileSets) {}
+			explicit IngestSSTFilesAction(std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets)
+			  : localFileSets(localFileSets) {}
 
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 
@@ -1326,7 +1342,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 
 		struct CompactRangeAction : TypedAction<Writer, CompactRangeAction> {
-			CompactRangeAction(KeyRangeRef range) : range(range) {}
+			explicit CompactRangeAction(KeyRangeRef range) : range(range) {}
 
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 
@@ -1681,12 +1697,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		struct ReadValueAction : TypedAction<Reader, ReadValueAction> {
 			Key key;
 			ReadType type;
+			bool cacheResult;
 			Optional<UID> debugID;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<Optional<Value>> result;
-			ReadValueAction(KeyRef key, ReadType type, Optional<UID> debugID)
-			  : key(key), type(type), debugID(debugID), startTime(timer_monotonic()),
+			ReadValueAction(KeyRef key, ReadType type, bool cacheResult, Optional<UID> debugID)
+			  : key(key), type(type), cacheResult(cacheResult), debugID(debugID), startTime(timer_monotonic()),
 			    getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
 		};
@@ -1720,6 +1737,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			rocksdb::PinnableSlice value;
 			rocksdb::ReadOptions readOptions = sharedState->getReadOptions();
+			if (SERVER_KNOBS->ROCKSDB_USE_CACHE_RESULT_OPTION)
+				readOptions.fill_cache = a.cacheResult;
 			if (shouldThrottle(a.type, a.key) && SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT) {
 				uint64_t deadlineMircos =
 				    db->GetEnv()->NowMicros() + (readValueTimeout - (readBeginTime - a.startTime)) * 1000000;
@@ -1763,12 +1782,14 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			Key key;
 			int maxLength;
 			ReadType type;
+			bool cacheResult;
 			Optional<UID> debugID;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<Optional<Value>> result;
-			ReadValuePrefixAction(Key key, int maxLength, ReadType type, Optional<UID> debugID)
-			  : key(key), maxLength(maxLength), type(type), debugID(debugID), startTime(timer_monotonic()),
+			ReadValuePrefixAction(Key key, int maxLength, ReadType type, bool cacheResult, Optional<UID> debugID)
+			  : key(key), maxLength(maxLength), type(type), cacheResult(cacheResult), debugID(debugID),
+			    startTime(timer_monotonic()),
 			    getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
 		};
@@ -1803,6 +1824,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			rocksdb::PinnableSlice value;
 			rocksdb::ReadOptions readOptions = sharedState->getReadOptions();
+			if (SERVER_KNOBS->ROCKSDB_USE_CACHE_RESULT_OPTION)
+				readOptions.fill_cache = a.cacheResult;
 			if (shouldThrottle(a.type, a.key) && SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT) {
 				uint64_t deadlineMircos =
 				    db->GetEnv()->NowMicros() + (readValuePrefixTimeout - (readBeginTime - a.startTime)) * 1000000;
@@ -1843,11 +1866,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			KeyRange keys;
 			int rowLimit, byteLimit;
 			ReadType type;
+			bool cacheResult;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<RangeResult> result;
-			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, ReadType type)
-			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), type(type), startTime(timer_monotonic()),
+			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, ReadType type, bool cacheResult)
+			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), type(type), cacheResult(cacheResult),
+			    startTime(timer_monotonic()),
 			    getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
 		};
@@ -1883,7 +1908,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			rocksdb::Status s;
 			if (a.rowLimit >= 0) {
 				double iterCreationBeginTime = a.getHistograms ? timer_monotonic() : 0;
-				ReadIterator readIter = readIterPool->getIterator(a.keys);
+				ReadIterator readIter = readIterPool->getIterator(a.keys, a.cacheResult);
 				if (a.getHistograms) {
 					metricPromiseStream->send(std::make_pair(ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
 					                                         timer_monotonic() - iterCreationBeginTime));
@@ -1913,7 +1938,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				readIterPool->returnIterator(readIter);
 			} else {
 				double iterCreationBeginTime = a.getHistograms ? timer_monotonic() : 0;
-				ReadIterator readIter = readIterPool->getIterator(a.keys);
+				ReadIterator readIter = readIterPool->getIterator(a.keys, a.cacheResult);
 				if (a.getHistograms) {
 					metricPromiseStream->send(std::make_pair(ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
 					                                         timer_monotonic() - iterCreationBeginTime));
@@ -2048,16 +2073,6 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 	}
 
-	ACTOR Future<Void> errorListenActor(Future<Void> collection) {
-		try {
-			wait(collection);
-			ASSERT(false);
-			throw internal_error();
-		} catch (Error& e) {
-			throw e;
-		}
-	}
-
 	ACTOR Future<Void> updateHistogram(ThreadFutureStream<std::pair<std::string, double>> metricFutureStream) {
 		state Reference<Histogram> commitLatencyHistogram = Histogram::getHistogram(
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds);
@@ -2240,6 +2255,17 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			    !SERVER_KNOBS->ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE && maxDeletes > 0) {
 				++counters.convertedDeleteRangeReqs;
 				rocksdb::ReadOptions readOptions = sharedState->getReadOptions();
+				// Below does a read operation for the deleteRange request, so disable rocksdb cache.
+				if (g_network->isSimulated()) {
+					// Disabling rocksdb cache in simulation can cause slow down, so disabling only for
+					// few requests controlled by SERVER_KNOBS->ROCKSDB_PROBABILITY_DISABLE_CACHE_SIM.
+					if (SERVER_KNOBS->ROCKSDB_USE_CACHE_RESULT_OPTION &&
+					    deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PROBABILITY_DISABLE_CACHE_SIM)
+						readOptions.fill_cache = false;
+				} else {
+					if (SERVER_KNOBS->ROCKSDB_USE_CACHE_RESULT_OPTION)
+						readOptions.fill_cache = false;
+				}
 				auto beginSlice = toSlice(keyRange.begin);
 				auto endSlice = toSlice(keyRange.end);
 				readOptions.iterate_lower_bound = &beginSlice;
@@ -2378,15 +2404,17 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
 		ReadType type = ReadType::NORMAL;
+		bool cacheResult = true;
 		Optional<UID> debugID;
 
 		if (options.present()) {
 			type = options.get().type;
+			cacheResult = options.get().cacheResult;
 			debugID = options.get().debugID;
 		}
 
 		if (!shouldThrottle(type, key)) {
-			auto a = new Reader::ReadValueAction(key, type, debugID);
+			auto a = new Reader::ReadValueAction(key, type, cacheResult, debugID);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2396,21 +2424,23 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadValueAction>(key, type, debugID);
+		auto a = std::make_unique<Reader::ReadValueAction>(key, type, cacheResult, debugID);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
 	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
 		ReadType type = ReadType::NORMAL;
+		bool cacheResult = true;
 		Optional<UID> debugID;
 
 		if (options.present()) {
 			type = options.get().type;
+			cacheResult = options.get().cacheResult;
 			debugID = options.get().debugID;
 		}
 
 		if (!shouldThrottle(type, key)) {
-			auto a = new Reader::ReadValuePrefixAction(key, maxLength, type, debugID);
+			auto a = new Reader::ReadValuePrefixAction(key, maxLength, type, cacheResult, debugID);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2420,7 +2450,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, type, debugID);
+		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, type, cacheResult, debugID);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
@@ -2449,14 +2479,16 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	                              int byteLimit,
 	                              Optional<ReadOptions> options) override {
 		ReadType type = ReadType::NORMAL;
+		bool cacheResult = true;
 
 		if (options.present()) {
 			type = options.get().type;
+			cacheResult = options.get().cacheResult;
 		}
 
 		if (!shouldThrottle(type, keys.begin)) {
 			++counters.rocksdbReadRangeQueries;
-			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, type);
+			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, type, cacheResult);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2467,7 +2499,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 		checkWaiters(semaphore, maxWaiters);
 		++counters.rocksdbReadRangeQueries;
-		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit, type);
+		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit, type, cacheResult);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
@@ -2771,8 +2803,8 @@ void RocksDBKeyValueStore::Writer::action(RestoreAction& a) {
 IKeyValueStore* keyValueStoreRocksDB(std::string const& path,
                                      UID logID,
                                      KeyValueStoreType storeType,
-                                     bool checkChecksums,
-                                     bool checkIntegrity) {
+                                     CheckChecksums checkChecksums,
+                                     CheckIntegrity checkIntegrity) {
 #ifdef WITH_ROCKSDB
 	return new RocksDBKeyValueStore(path, logID);
 #else
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/kvstore/KeyValueStoreSQLite.cpp
similarity index 96%
rename from fdbserver/KeyValueStoreSQLite.actor.cpp
rename to fdbserver/kvstore/KeyValueStoreSQLite.cpp
index d9132e0f2f1..c157cf43611 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/kvstore/KeyValueStoreSQLite.cpp
@@ -1,5 +1,5 @@
 /*
- * KeyValueStoreSQLite.actor.cpp
+ * KeyValueStoreSQLite.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,9 +22,10 @@
 #define SQLITE_THREADSAFE 0 // also in sqlite3.amalgamation.c!
 #include "fmt/format.h"
 #include "crc32/crc32c.h"
-#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/kvstore/KVFileUtils.h"
 #include "fdbserver/CoroFlow.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/Hash3.h"
 #include "flow/xxhash.h"
 
@@ -36,8 +37,8 @@ extern "C" {
 u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*);
 }
 #include "flow/ThreadPrimitives.h"
-#include "fdbserver/VFSAsync.h"
-#include "fdbserver/template_fdb.h"
+#include "VFSAsync.h"
+#include "template_fdb.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -73,7 +74,8 @@ struct SpringCleaningStats {
 };
 
 struct PageChecksumCodec {
-	PageChecksumCodec(std::string const& filename) : pageSize(0), reserveSize(0), filename(filename), silent(false) {}
+	explicit PageChecksumCodec(std::string const& filename)
+	  : pageSize(0), reserveSize(0), filename(filename), silent(false) {}
 
 	int pageSize;
 	int reserveSize;
@@ -197,7 +199,7 @@ struct PageChecksumCodec {
 	}
 
 	static void* codec(void* vpSelf, void* data, Pgno pageNumber, int op) {
-		PageChecksumCodec* self = (PageChecksumCodec*)vpSelf;
+		auto* self = (PageChecksumCodec*)vpSelf;
 
 		// Page write operations are 6 for DB page and 7 for journal page
 		bool write = (op == 6 || op == 7);
@@ -239,13 +241,13 @@ struct PageChecksumCodec {
 	}
 
 	static void sizeChange(void* vpSelf, int new_pageSize, int new_reserveSize) {
-		PageChecksumCodec* self = (PageChecksumCodec*)vpSelf;
+		auto* self = (PageChecksumCodec*)vpSelf;
 		self->pageSize = new_pageSize;
 		self->reserveSize = new_reserveSize;
 	}
 
 	static void free(void* vpSelf) {
-		PageChecksumCodec* self = (PageChecksumCodec*)vpSelf;
+		auto* self = (PageChecksumCodec*)vpSelf;
 		delete self;
 	}
 };
@@ -1610,8 +1612,8 @@ struct ThreadSafeCounter {
 
 class KeyValueStoreSQLite final : public IKeyValueStore {
 public:
-	void dispose() override { doClose(this, true); }
-	void close() override { doClose(this, false); }
+	void dispose() override { doClose(Uncancellable(), this, true); }
+	void close() override { doClose(Uncancellable(), this, false); }
 
 	Future<Void> getError() const override { return delayed(readThreads->getError() || writeThread->getError()); }
 	Future<Void> onClosed() const override { return stopped.getFuture(); }
@@ -1835,7 +1837,7 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 
 		struct SetAction final : TypedAction<Writer, SetAction>, FastAllocated<SetAction> {
 			KeyValue kv;
-			SetAction(KeyValue kv) : kv(kv) {}
+			explicit SetAction(KeyValue kv) : kv(kv) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->SET_TIME_ESTIMATE; }
 		};
 		void action(SetAction& a) {
@@ -1850,7 +1852,7 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 
 		struct ClearAction final : TypedAction<Writer, ClearAction>, FastAllocated<ClearAction> {
 			KeyRange range;
-			ClearAction(KeyRange range) : range(range) {}
+			explicit ClearAction(KeyRange range) : range(range) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->CLEAR_TIME_ESTIMATE; }
 		};
 		void action(ClearAction& a) {
@@ -1956,7 +1958,7 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 			                   std::max(1, SERVER_KNOBS->SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE));
 			bool vacuumFinished = false;
 
-			loop {
+			while (true) {
 				double begin = now();
 				bool canDelete = !freeTableEmpty &&
 				                 (now() < lazyDeleteEnd || workPerformed.lazyDeletePages <
@@ -2023,11 +2025,11 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 		}
 	};
 
-	ACTOR static Future<Void> logPeriodically(KeyValueStoreSQLite* self) {
-		state int64_t lastReadsComplete = 0;
-		state int64_t lastWritesComplete = 0;
-		loop {
-			wait(delay(FLOW_KNOBS->SQLITE_DISK_METRIC_LOGGING_INTERVAL));
+	static Future<Void> logPeriodically(KeyValueStoreSQLite* self) {
+		int64_t lastReadsComplete = 0;
+		int64_t lastWritesComplete = 0;
+		while (true) {
+			co_await delay(FLOW_KNOBS->SQLITE_DISK_METRIC_LOGGING_INTERVAL);
 
 			int64_t rc = self->readsComplete, wc = self->writesComplete;
 			TraceEvent("DiskMetrics", self->logID)
@@ -2068,9 +2070,9 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 		walFile.clear();
 	}
 
-	ACTOR static Future<Void> stopOnError(KeyValueStoreSQLite* self) {
+	static Future<Void> stopOnError(KeyValueStoreSQLite* self) {
 		try {
-			wait(self->readThreads->getError() || self->writeThread->getError());
+			co_await (self->readThreads->getError() || self->writeThread->getError());
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -2080,12 +2082,10 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 			self->readThreads->stop(e).isReady();
 			self->writeThread->stop(e).isReady();
 		}
-
-		return Void();
 	}
 
-	ACTOR static void doClose(KeyValueStoreSQLite* self, bool deleteOnClose) {
-		state Error error = success();
+	static Future<Void> doClose(Uncancellable, KeyValueStoreSQLite* self, bool deleteOnClose) {
+		Error error = success();
 
 		self->disableRateControl();
 
@@ -2094,10 +2094,10 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 			self->starting.cancel();
 			self->cleaning.cancel();
 			self->logging.cancel();
-			wait(self->readThreads->stop() && self->writeThread->stop());
+			co_await (self->readThreads->stop() && self->writeThread->stop());
 			if (deleteOnClose) {
-				wait(IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename, true));
-				wait(IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename + "-wal", false));
+				co_await IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename, true);
+				co_await IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename + "-wal", false);
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "KVDoCloseError", self->logID)
@@ -2117,15 +2117,15 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 IKeyValueStore* keyValueStoreSQLite(std::string const& filename,
                                     UID logID,
                                     KeyValueStoreType storeType,
-                                    bool checkChecksums,
-                                    bool checkIntegrity) {
+                                    CheckChecksums checkChecksums,
+                                    CheckIntegrity checkIntegrity) {
 	return new KeyValueStoreSQLite(filename, logID, storeType, checkChecksums, checkIntegrity);
 }
 
-ACTOR Future<Void> cleanPeriodically(KeyValueStoreSQLite* self) {
-	wait(delayJittered(SERVER_KNOBS->SPRING_CLEANING_NO_ACTION_INTERVAL));
-	loop {
-		KeyValueStoreSQLite::SpringCleaningWorkPerformed workPerformed = wait(self->doClean());
+Future<Void> cleanPeriodically(KeyValueStoreSQLite* self) {
+	co_await delayJittered(SERVER_KNOBS->SPRING_CLEANING_NO_ACTION_INTERVAL);
+	while (true) {
+		KeyValueStoreSQLite::SpringCleaningWorkPerformed workPerformed = co_await self->doClean();
 
 		double duration = std::numeric_limits<double>::max();
 		if (workPerformed.lazyDeletePages >= SERVER_KNOBS->SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE) {
@@ -2138,14 +2138,13 @@ ACTOR Future<Void> cleanPeriodically(KeyValueStoreSQLite* self) {
 			duration = SERVER_KNOBS->SPRING_CLEANING_NO_ACTION_INTERVAL;
 		}
 
-		wait(delayJittered(duration));
+		co_await delayJittered(duration);
 	}
 }
 
-ACTOR static Future<Void> startReadThreadsWhen(KeyValueStoreSQLite* kv, Future<Void> onReady, UID id) {
-	wait(onReady);
+static Future<Void> startReadThreadsWhen(KeyValueStoreSQLite* kv, Future<Void> onReady) {
+	co_await onReady;
 	kv->startReadThreads();
-	return Void();
 }
 
 sqlite3_vfs* vfsAsync();
@@ -2197,14 +2196,11 @@ KeyValueStoreSQLite::KeyValueStoreSQLite(std::string const& filename,
 	auto p = new Writer::InitAction();
 	auto f = p->result.getFuture();
 	writeThread->post(p);
-	starting = startReadThreadsWhen(this, f, logID);
+	starting = startReadThreadsWhen(this, f);
 	cleaning = cleanPeriodically(this);
 	logging = logPeriodically(this);
 }
-KeyValueStoreSQLite::~KeyValueStoreSQLite() {
-	// printf("dbf=%lld bytes, wal=%lld bytes\n", getFileSize((filename+".fdb").c_str()),
-	// getFileSize((filename+".fdb-wal").c_str()));
-}
+KeyValueStoreSQLite::~KeyValueStoreSQLite() = default;
 
 StorageBytes KeyValueStoreSQLite::getStorageBytes() const {
 	int64_t free;
@@ -2312,7 +2308,7 @@ void GenerateIOLogChecksumFile(std::string filename) {
 
 // If integrity is true, a full btree integrity check is done.
 // If integrity is false, only a scan of all pages to validate their checksums is done.
-ACTOR Future<Void> KVFileCheck(std::string filename, bool integrity) {
+Future<Void> KVFileCheck(std::string filename, bool integrity) {
 	if (!fileExists(filename))
 		throw file_not_found();
 
@@ -2324,22 +2320,21 @@ ACTOR Future<Void> KVFileCheck(std::string filename, bool integrity) {
 		type = KeyValueStoreType::SSD_BTREE_V2;
 	ASSERT(type != KeyValueStoreType::END);
 
-	state IKeyValueStore* store = keyValueStoreSQLite(filename, UID(0, 0), type, !integrity, integrity);
+	IKeyValueStore* store =
+	    keyValueStoreSQLite(filename, UID(0, 0), type, CheckChecksums(!integrity), CheckIntegrity(integrity));
 	ASSERT(store != nullptr);
 
 	// Wait for integry check to finish
-	wait(success(store->readValue(StringRef())));
+	co_await store->readValue(StringRef());
 
 	if (store->getError().isError())
-		wait(store->getError());
+		co_await store->getError();
 	Future<Void> c = store->onClosed();
 	store->close();
-	wait(c);
-
-	return Void();
+	co_await c;
 }
 
-ACTOR Future<Void> KVFileDump(std::string filename) {
+Future<Void> KVFileDump(std::string filename) {
 	if (!fileExists(filename))
 		throw file_not_found();
 
@@ -2351,23 +2346,23 @@ ACTOR Future<Void> KVFileDump(std::string filename) {
 		type = KeyValueStoreType::SSD_BTREE_V2;
 	ASSERT(type != KeyValueStoreType::END);
 
-	state IKeyValueStore* store = keyValueStoreSQLite(filename, UID(0, 0), type);
+	IKeyValueStore* store = keyValueStoreSQLite(filename, UID(0, 0), type);
 	ASSERT(store != nullptr);
 
 	// dump
-	state int64_t count = 0;
-	state Key k;
-	state Key endk = allKeys.end;
-	state bool debug = false;
+	int64_t count = 0;
+	Key k;
+	Key endk = allKeys.end;
+	bool debug = false;
 
 	const char* startKey = getenv("FDB_DUMP_STARTKEY");
 	const char* endKey = getenv("FDB_DUMP_ENDKEY");
 	const char* debugS = getenv("FDB_DUMP_DEBUG");
-	if (startKey != NULL)
+	if (startKey != nullptr)
 		k = StringRef(unprintable(std::string(startKey)));
-	if (endKey != NULL)
+	if (endKey != nullptr)
 		endk = StringRef(unprintable(std::string(endKey)));
-	if (debugS != NULL)
+	if (debugS != nullptr)
 		debug = true;
 
 	fprintf(stderr,
@@ -2377,10 +2372,10 @@ ACTOR Future<Void> KVFileDump(std::string filename) {
 	        debug ? "true" : "false");
 
 	while (true) {
-		RangeResult kv = wait(store->readRange(KeyRangeRef(k, endk), 1000));
+		RangeResult kv = co_await store->readRange(KeyRangeRef(k, endk), 1000);
 		for (auto& one : kv) {
 			int size = 0;
-			const uint8_t* data = NULL;
+			const uint8_t* data = nullptr;
 
 			size = one.key.size();
 			data = one.key.begin();
@@ -2407,10 +2402,8 @@ ACTOR Future<Void> KVFileDump(std::string filename) {
 	fmt::print(stderr, "Counted: {}\n", count);
 
 	if (store->getError().isError())
-		wait(store->getError());
+		co_await store->getError();
 	Future<Void> c = store->onClosed();
 	store->close();
-	wait(c);
-
-	return Void();
+	co_await c;
 }
diff --git a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp b/fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp
similarity index 97%
rename from fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
rename to fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp
index 2de4aaf8979..4e0e98fdaf2 100644
--- a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
+++ b/fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp
@@ -31,7 +31,7 @@
 #endif
 #include "fdbclient/SystemData.h"
 #include "fdbserver/CoroFlow.h"
-#include "fdbserver/FDBRocksDBVersion.h"
+#include "fdbserver/core/FDBRocksDBVersion.h"
 #include "flow/flow.h"
 #include "flow/IThreadPool.h"
 #include "flow/ThreadHelper.actor.h"
@@ -44,10 +44,10 @@
 
 #endif // WITH_ROCKSDB
 
-#include "fdbserver/Knobs.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/RocksDBCommon.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
+#include "RocksDBCommon.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 #ifdef WITH_ROCKSDB
@@ -160,7 +160,7 @@ const char* getFlushReasonString(FlushReason flush_reason) {
 
 class RocksDBEventListener : public rocksdb::EventListener {
 public:
-	RocksDBEventListener(UID id)
+	explicit RocksDBEventListener(UID id)
 	  : logId(id), compactionReasons((int)CompactionReason::kNumOfReasons), flushReasons(ROCKSDB_NUM_FLUSH_REASONS),
 	    numRangeDeletionsInTableFile(Histogram::getHistogram(ROCKSDB_STATS_HISTOGRAM_GROUP,
 	                                                         "NumRangeDeletionsInTableFile"_sr,
@@ -279,7 +279,7 @@ class RocksDBEventListener : public rocksdb::EventListener {
 // could potentially cause segmentation fault.
 class RocksDBErrorListener : public rocksdb::EventListener {
 public:
-	RocksDBErrorListener() {}
+	RocksDBErrorListener() = default;
 	void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* bg_error) override {
 		if (!bg_error)
 			return;
@@ -307,7 +307,7 @@ class RocksDBErrorListener : public rocksdb::EventListener {
 		std::unique_lock<std::mutex> lock(mutex);
 		return errorPromise.getFuture();
 	}
-	~RocksDBErrorListener() {
+	~RocksDBErrorListener() override {
 		std::unique_lock<std::mutex> lock(mutex);
 		if (!errorPromise.isValid())
 			return;
@@ -380,10 +380,10 @@ class CompactOnRangeDeletionCollectorFactory : public rocksdb::TablePropertiesCo
 	// A factory of a table property collector that marks a SST file as need-compaction when the number of range
 	// deletions exceeds the threshold.
 	// @param numRangeDeletionsAllowed,  triggers compaction range deletion count exceeds numRangeDeletionsAllowed.
-	CompactOnRangeDeletionCollectorFactory(uint64_t numRangeDeletionsAllowed)
+	explicit(false) CompactOnRangeDeletionCollectorFactory(uint64_t numRangeDeletionsAllowed)
 	  : threshold(numRangeDeletionsAllowed), numFilesMarkedForCompaction(0) {}
 
-	~CompactOnRangeDeletionCollectorFactory() {}
+	~CompactOnRangeDeletionCollectorFactory() override = default;
 
 	rocksdb::TablePropertiesCollector* CreateTablePropertiesCollector(
 	    rocksdb::TablePropertiesCollectorFactory::Context context) override {
@@ -820,7 +820,7 @@ struct ReadIterator {
 // Stores iterators for all shards for future reuse. One iterator is stored per shard.
 class IteratorPool {
 public:
-	IteratorPool() {}
+	IteratorPool() = default;
 
 	std::shared_ptr<ReadIterator> getIterator(const std::string& id) {
 		std::unique_lock<std::mutex> lock(mu);
@@ -899,9 +899,9 @@ class IteratorPool {
 	uint64_t numNewIterators = 0;
 };
 
-ACTOR Future<Void> flowLockLogger(const FlowLock* readLock, const FlowLock* fetchLock) {
-	loop {
-		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+Future<Void> flowLockLogger(const FlowLock* readLock, const FlowLock* fetchLock) {
+	while (true) {
+		co_await delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY);
 		TraceEvent e("ShardedRocksDBFlowLock");
 		e.detail("ReadAvailable", readLock->available());
 		e.detail("ReadActivePermits", readLock->activePermits());
@@ -1176,16 +1176,15 @@ class ShardManager {
 		}
 	}
 
-	ACTOR static Future<Void> shardMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
-	                                             Future<Void> openFuture,
-	                                             ShardManager* shardManager) {
-		state std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards =
-		    shardManager->getAllShards();
+	static Future<Void> shardMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
+	                                       Future<Void> openFuture,
+	                                       ShardManager* shardManager) {
+		std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards = shardManager->getAllShards();
 
 		try {
-			wait(openFuture);
-			loop {
-				wait(delay(SERVER_KNOBS->ROCKSDB_CF_METRICS_DELAY));
+			co_await openFuture;
+			while (true) {
+				co_await delay(SERVER_KNOBS->ROCKSDB_CF_METRICS_DELAY);
 				if (rState->closing) {
 					break;
 				}
@@ -1233,7 +1232,6 @@ class ShardManager {
 				TraceEvent(SevError, "ShardedRocksShardMetricsLoggerError").errorUnsuppressed(e);
 			}
 		}
-		return Void();
 	}
 
 	rocksdb::Status init() {
@@ -2215,16 +2213,16 @@ struct RocksDBMetrics {
 	}
 };
 
-ACTOR Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
-                                                  Future<Void> openFuture,
-                                                  std::shared_ptr<RocksDBMetrics> rocksDBMetrics,
-                                                  ShardManager* shardManager,
-                                                  std::string manifestDirectory) {
+Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
+                                            Future<Void> openFuture,
+                                            std::shared_ptr<RocksDBMetrics> rocksDBMetrics,
+                                            ShardManager* shardManager,
+                                            std::string manifestDirectory) {
 	try {
-		wait(openFuture);
-		state rocksdb::DB* db = shardManager->getDb();
-		loop {
-			wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+		co_await openFuture;
+		rocksdb::DB* db = shardManager->getDb();
+		while (true) {
+			co_await delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY);
 			if (rState->closing) {
 				break;
 			}
@@ -2238,25 +2236,24 @@ ACTOR Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<ShardedRocksDB
 			TraceEvent(SevError, "ShardedRocksDBMetricsError").errorUnsuppressed(e);
 		}
 	}
-	return Void();
 }
 
 struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 	using CF = rocksdb::ColumnFamilyHandle*;
 
-	ACTOR static Future<Void> refreshIteratorPool(std::shared_ptr<ShardedRocksDBState> rState,
-	                                              std::shared_ptr<IteratorPool> iteratorPool,
-	                                              Future<Void> readyToStart) {
+	static Future<Void> refreshIteratorPool(std::shared_ptr<ShardedRocksDBState> rState,
+	                                        std::shared_ptr<IteratorPool> iteratorPool,
+	                                        Future<Void> readyToStart) {
 		if (!SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) {
-			return Void();
+			co_return;
 		}
-		state Reference<Histogram> histogram = Histogram::getHistogram(
+		Reference<Histogram> histogram = Histogram::getHistogram(
 		    SHARDED_ROCKSDB_HISTOGRAM_GROUP, "TimeSpentRefreshIterators"_sr, Histogram::Unit::milliseconds);
 
 		try {
-			wait(readyToStart);
-			loop {
-				wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME));
+			co_await readyToStart;
+			while (true) {
+				co_await delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME);
 				if (rState->closing) {
 					break;
 				}
@@ -2270,20 +2267,17 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				TraceEvent(SevError, "RefreshReadIteratorPoolError").errorUnsuppressed(e);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> refreshRocksDBBackgroundEventCounter(
-	    UID id,
-	    std::shared_ptr<RocksDBEventListener> eventListener) {
+	static Future<Void> refreshRocksDBBackgroundEventCounter(UID id,
+	                                                         std::shared_ptr<RocksDBEventListener> eventListener) {
 		if (!SERVER_KNOBS->LOGGING_ROCKSDB_BG_WORK_WHEN_IO_TIMEOUT &&
 		    SERVER_KNOBS->LOGGING_ROCKSDB_BG_WORK_PROBABILITY <= 0.0) {
-			return Void();
+			co_return;
 		}
 		try {
-			loop {
-				wait(delay(SERVER_KNOBS->LOGGING_ROCKSDB_BG_WORK_PERIOD_SEC));
+			while (true) {
+				co_await delay(SERVER_KNOBS->LOGGING_ROCKSDB_BG_WORK_PERIOD_SEC);
 				eventListener->logRecentRocksDBBackgroundWorkStats(id);
 				eventListener->resetCounters();
 			}
@@ -2292,13 +2286,12 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				TraceEvent(SevError, "RefreshRocksDBBackgroundEventCounter").errorUnsuppressed(e);
 			}
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> doRestore(ShardedRocksDBKeyValueStore* self,
-	                                    std::string shardId,
-	                                    std::vector<KeyRange> ranges,
-	                                    std::vector<CheckpointMetaData> checkpoints) {
+	static Future<Void> doRestore(ShardedRocksDBKeyValueStore* self,
+	                              std::string shardId,
+	                              std::vector<KeyRange> ranges,
+	                              std::vector<CheckpointMetaData> checkpoints) {
 		for (const KeyRange& range : ranges) {
 			std::vector<DataShard*> shards = self->shardManager.getDataShardsByRange(range);
 			if (!shards.empty()) {
@@ -2327,7 +2320,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		self->writeThread->post(a);
 
 		try {
-			wait(res);
+			co_await res;
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw;
@@ -2337,8 +2330,6 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			}
 			throw;
 		}
-
-		return Void();
 	}
 
 	struct CompactionWorker : IThreadPoolReceiver {
@@ -2346,7 +2337,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		explicit CompactionWorker(UID logId) : logId(logId) {}
 
 		void init() override {}
-		~CompactionWorker() override {}
+		~CompactionWorker() override = default;
 
 		struct CompactShardsAction : TypedAction<CompactionWorker, CompactShardsAction> {
 			std::vector<std::shared_ptr<PhysicalShard>> shards;
@@ -2427,7 +2418,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			ASSERT(iteratorPool);
 		}
 
-		~Writer() override {}
+		~Writer() override = default;
 
 		void init() override {}
 
@@ -2464,7 +2455,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 			PhysicalShard* shard;
 			ThreadReturnPromise<Void> done;
 
-			AddShardAction(PhysicalShard* shard) : shard(shard) { ASSERT(shard); }
+			explicit AddShardAction(PhysicalShard* shard) : shard(shard) { ASSERT(shard); }
 			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
 		};
 
@@ -2528,7 +2519,9 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		struct DeleteVisitor : public rocksdb::WriteBatch::Handler {
 			std::vector<std::pair<uint32_t, KeyRange>>* deletes;
 
-			DeleteVisitor(std::vector<std::pair<uint32_t, KeyRange>>* deletes) : deletes(deletes) { ASSERT(deletes); }
+			explicit(false) DeleteVisitor(std::vector<std::pair<uint32_t, KeyRange>>* deletes) : deletes(deletes) {
+				ASSERT(deletes);
+			}
 
 			rocksdb::Status DeleteRangeCF(uint32_t column_family_id,
 			                              const rocksdb::Slice& begin,
@@ -3704,19 +3697,19 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 		return true;
 	}
 
-	ACTOR static Future<Void> compactShards(std::shared_ptr<ShardedRocksDBState> rState,
-	                                        Future<Void> openFuture,
-	                                        ShardManager* shardManager,
-	                                        Reference<IThreadPool> thread) {
+	static Future<Void> compactShards(std::shared_ptr<ShardedRocksDBState> rState,
+	                                  Future<Void> openFuture,
+	                                  ShardManager* shardManager,
+	                                  Reference<IThreadPool> thread) {
 		try {
-			wait(openFuture);
-			state std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards =
+			co_await openFuture;
+			std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards =
 			    shardManager->getAllShards();
-			loop {
+			while (true) {
 				if (rState->closing) {
 					break;
 				}
-				wait(delay(SERVER_KNOBS->SHARDED_ROCKSDB_COMPACTION_ACTOR_DELAY));
+				co_await delay(SERVER_KNOBS->SHARDED_ROCKSDB_COMPACTION_ACTOR_DELAY);
 				int count = 0;
 				double start = now();
 				std::vector<std::shared_ptr<PhysicalShard>> shards;
@@ -3755,7 +3748,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 					auto a = new CompactionWorker::CompactShardsAction(shards, shardManager->getMetaDataShard());
 					auto res = a->done.getFuture();
 					thread->post(a);
-					wait(res);
+					co_await res;
 				} else {
 					TraceEvent("CompactionSkipped").detail("Reason", "NoCandidate");
 				}
@@ -3765,18 +3758,17 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				TraceEvent(SevError, "ShardedRocksDBCompactionActorError").errorUnsuppressed(e);
 			}
 		}
-		return Void();
 	}
-	ACTOR static Future<Void> emptyShardCleaner(std::shared_ptr<ShardedRocksDBState> rState,
-	                                            Future<Void> openFuture,
-	                                            ShardManager* shardManager,
-	                                            Reference<IThreadPool> writeThread) {
-		state double cleanUpDelay = SERVER_KNOBS->ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY;
-		state double cleanUpPeriod = cleanUpDelay * 2;
+	static Future<Void> emptyShardCleaner(std::shared_ptr<ShardedRocksDBState> rState,
+	                                      Future<Void> openFuture,
+	                                      ShardManager* shardManager,
+	                                      Reference<IThreadPool> writeThread) {
+		double cleanUpDelay = SERVER_KNOBS->ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY;
+		double cleanUpPeriod = cleanUpDelay * 2;
 		try {
-			wait(openFuture);
-			loop {
-				wait(delay(cleanUpPeriod));
+			co_await openFuture;
+			while (true) {
+				co_await delay(cleanUpPeriod);
 				if (rState->closing) {
 					break;
 				}
@@ -3786,7 +3778,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 					Future<Void> f = a->done.getFuture();
 					writeThread->post(a);
 					TraceEvent(SevInfo, "ShardedRocksDB").detail("DeleteEmptyShards", shards.size());
-					wait(f);
+					co_await f;
 				}
 			}
 		} catch (Error& e) {
@@ -3794,7 +3786,6 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore {
 				TraceEvent(SevError, "DeleteEmptyShardsError").errorUnsuppressed(e);
 			}
 		}
-		return Void();
 	}
 
 	StorageBytes getStorageBytes() const override {
@@ -3929,8 +3920,8 @@ ACTOR Future<Void> testCheckpointRestore(IKeyValueStore* kvStore, std::vector<Ke
 IKeyValueStore* keyValueStoreShardedRocksDB(std::string const& path,
                                             UID logID,
                                             KeyValueStoreType storeType,
-                                            bool checkChecksums,
-                                            bool checkIntegrity) {
+                                            CheckChecksums checkChecksums,
+                                            CheckIntegrity checkIntegrity) {
 #ifdef WITH_ROCKSDB
 	return new ShardedRocksDBKeyValueStore(path, logID);
 #else
diff --git a/fdbserver/include/fdbserver/RadixTree.h b/fdbserver/kvstore/RadixTree.h
similarity index 99%
rename from fdbserver/include/fdbserver/RadixTree.h
rename to fdbserver/kvstore/RadixTree.h
index 4a1ed2507dc..fc14631915b 100644
--- a/fdbserver/include/fdbserver/RadixTree.h
+++ b/fdbserver/kvstore/RadixTree.h
@@ -31,7 +31,7 @@
 #include <map>
 #include <stdexcept>
 
-#include "fdbserver/IKeyValueContainer.h"
+#include "IKeyValueContainer.h"
 #include "flow/Arena.h"
 
 // forward declaration
@@ -151,7 +151,7 @@ class radix_tree {
 	};
 
 	struct leafNode : FastAllocated<leafNode> {
-		leafNode(const StringRef& content) : base(), is_inline(0), inline_length(0), arena() {
+		explicit leafNode(const StringRef& content) : base(), is_inline(0), inline_length(0), arena() {
 			base.m_is_leaf = 1;
 			setValue(content);
 		}
@@ -235,8 +235,8 @@ class radix_tree {
 		node* m_pointee;
 
 		iterator() : m_pointee(nullptr) {}
-		iterator(const iterator& r) : m_pointee(r.m_pointee) {}
-		iterator(node* p) : m_pointee(p) {}
+		explicit(false) iterator(const iterator& r) : m_pointee(r.m_pointee) {}
+		explicit(false) iterator(node* p) : m_pointee(p) {}
 		iterator& operator=(const iterator& r) {
 			m_pointee = r.m_pointee;
 			return *this;
diff --git a/fdbserver/RocksDBCommon.cpp b/fdbserver/kvstore/RocksDBCommon.cpp
similarity index 98%
rename from fdbserver/RocksDBCommon.cpp
rename to fdbserver/kvstore/RocksDBCommon.cpp
index 9b86ac07304..d59f3c12a03 100644
--- a/fdbserver/RocksDBCommon.cpp
+++ b/fdbserver/kvstore/RocksDBCommon.cpp
@@ -18,11 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbserver/RocksDBCommon.h"
+#include "RocksDBCommon.h"
 
 #ifdef WITH_ROCKSDB
 
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/Trace.h"
 
 namespace RocksDBCommon {
diff --git a/fdbserver/include/fdbserver/RocksDBCommon.h b/fdbserver/kvstore/RocksDBCommon.h
similarity index 100%
rename from fdbserver/include/fdbserver/RocksDBCommon.h
rename to fdbserver/kvstore/RocksDBCommon.h
diff --git a/fdbserver/RocksDBLogForwarder.actor.cpp b/fdbserver/kvstore/RocksDBLogForwarder.cpp
similarity index 93%
rename from fdbserver/RocksDBLogForwarder.actor.cpp
rename to fdbserver/kvstore/RocksDBLogForwarder.cpp
index 3679baebcd3..16ac1a56d6b 100644
--- a/fdbserver/RocksDBLogForwarder.actor.cpp
+++ b/fdbserver/kvstore/RocksDBLogForwarder.cpp
@@ -1,5 +1,5 @@
 /*
- * RocksDBLogForwarder.actor.cpp
+ * RocksDBLogForwarder.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,14 +19,12 @@
  */
 
 #ifdef WITH_ROCKSDB
-#include "fdbserver/RocksDBLogForwarder.h"
+#include "RocksDBLogForwarder.h"
 
 #include "flow/network.h"
 #include "flow/Trace.h"
 #include "fdbrpc/simulator.h"
 
-#include "flow/actorcompiler.h" // This must be the last include file
-
 using InfoLogLevel = rocksdb::InfoLogLevel;
 
 namespace {
@@ -70,11 +68,10 @@ void logTraceEvent(const RocksDBLogRecord& record) {
 	}
 }
 
-ACTOR Future<Void> rocksDBPeriodicallyLogger(RocksDBLogger* pRecords) {
-	loop choose {
-		when(wait(delay(0.1))) {
-			pRecords->consume();
-		}
+Future<Void> rocksDBPeriodicallyLogger(RocksDBLogger* pRecords) {
+	while (true) {
+		co_await delay(0.1);
+		pRecords->consume();
 	}
 }
 
diff --git a/fdbserver/include/fdbserver/RocksDBLogForwarder.h b/fdbserver/kvstore/RocksDBLogForwarder.h
similarity index 100%
rename from fdbserver/include/fdbserver/RocksDBLogForwarder.h
rename to fdbserver/kvstore/RocksDBLogForwarder.h
diff --git a/fdbserver/TransactionStoreMutationTracking.cpp b/fdbserver/kvstore/TransactionStoreMutationTracking.cpp
similarity index 94%
rename from fdbserver/TransactionStoreMutationTracking.cpp
rename to fdbserver/kvstore/TransactionStoreMutationTracking.cpp
index c4fb0fdb5a3..101f1d1f963 100644
--- a/fdbserver/TransactionStoreMutationTracking.cpp
+++ b/fdbserver/kvstore/TransactionStoreMutationTracking.cpp
@@ -20,7 +20,7 @@
 
 #include "fdbclient/SystemData.h"
 
-#include "fdbserver/TransactionStoreMutationTracking.h"
+#include "TransactionStoreMutationTracking.h"
 #if defined(FDB_CLEAN_BUILD) && DEBUG_TRANSACTION_STATE_STORE_ENABLED
 #error "You cannot use transaction store mutation tracking in a clean/release build."
 #endif
@@ -56,9 +56,9 @@ TraceEvent transactionStoreDebugMutationEnabled(const char* context,
 		label = DEBUG_KEY.label;
 	}
 
-	for (auto& labelRange : debugRanges) {
-		if (labelRange.second.contains(mutation)) {
-			label = labelRange.first;
+	for (const auto& [rangeLabel, debugRange] : debugRanges) {
+		if (debugRange.contains(mutation)) {
+			label = rangeLabel;
 			break;
 		}
 	}
diff --git a/fdbserver/include/fdbserver/TransactionStoreMutationTracking.h b/fdbserver/kvstore/TransactionStoreMutationTracking.h
similarity index 100%
rename from fdbserver/include/fdbserver/TransactionStoreMutationTracking.h
rename to fdbserver/kvstore/TransactionStoreMutationTracking.h
diff --git a/fdbserver/VFSAsync.cpp b/fdbserver/kvstore/VFSAsync.cpp
similarity index 97%
rename from fdbserver/VFSAsync.cpp
rename to fdbserver/kvstore/VFSAsync.cpp
index 503fdec9d61..c9299e1f0c0 100644
--- a/fdbserver/VFSAsync.cpp
+++ b/fdbserver/kvstore/VFSAsync.cpp
@@ -27,7 +27,7 @@
 #include "fdbserver/CoroFlow.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbrpc/AsyncFileReadAhead.actor.h"
+#include "fdbrpc/AsyncFileReadAhead.h"
 
 #include <assert.h>
 #include <string.h>
@@ -47,7 +47,7 @@
 #include <fcntl.h>
 #endif
 
-#include "fdbserver/VFSAsync.h"
+#include "VFSAsync.h"
 
 /*
 ** The maximum pathname length supported by this VFS.
@@ -76,7 +76,7 @@ VFSAsyncFile::VFSAsyncFile(std::string const& filename, int flags)
 std::map<std::string, std::pair<uint32_t, int>> VFSAsyncFile::filename_lockCount_openCount;
 
 static int asyncClose(sqlite3_file* pFile) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 
 	TraceEvent(SevDebug, "VFSAsyncFileDestroy").detail("Filename", p->filename).backtrace();
 
@@ -89,7 +89,7 @@ static int asyncClose(sqlite3_file* pFile) {
 }
 
 static int asyncRead(sqlite3_file* pFile, void* zBuf, int iAmt, sqlite_int64 iOfst) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		++p->debug_reads;
 		int readBytes = waitForAndGet(p->file->read(zBuf, iAmt, iOfst));
@@ -109,7 +109,7 @@ static int asyncRead(sqlite3_file* pFile, void* zBuf, int iAmt, sqlite_int64 iOf
 
 #if 1
 static int asyncReleaseZeroCopy(sqlite3_file* pFile, void* data, int iAmt, sqlite_int64 iOfst) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		--p->debug_zcrefs;
 		p->file->releaseZeroCopy(data, iAmt, iOfst);
@@ -123,7 +123,7 @@ static int asyncReleaseZeroCopy(sqlite3_file* pFile, void* data, int iAmt, sqlit
 }
 
 static int asyncReadZeroCopy(sqlite3_file* pFile, void** data, int iAmt, sqlite_int64 iOfst, int* pDataWasCached) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		int readBytes = iAmt;
 		Future<Void> readFuture = p->file->readZeroCopy(data, &readBytes, iOfst);
@@ -149,7 +149,7 @@ static int asyncReadZeroCopy(sqlite3_file* pFile, void** data, int iAmt, sqlite_
 
 #else
 static int asyncReadZeroCopy(sqlite3_file* pFile, void** data, int iAmt, sqlite_int64 iOfst) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		*data = new char[iAmt];
 		int readBytes = waitForAndGet(p->file->read(*data, iAmt, iOfst));
@@ -175,7 +175,7 @@ static int asyncReleaseZeroCopy(sqlite3_file* pFile, void* data, int iAmt, sqlit
 #endif
 
 static int asyncWrite(sqlite3_file* pFile, const void* zBuf, int iAmt, sqlite_int64 iOfst) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		waitFor(p->file->write(zBuf, iAmt, iOfst));
 		return SQLITE_OK;
@@ -188,7 +188,7 @@ static int asyncWrite(sqlite3_file* pFile, const void* zBuf, int iAmt, sqlite_in
 }
 
 static int asyncTruncate(sqlite3_file* pFile, sqlite_int64 size) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 
 	// Adjust size to a multiple of chunkSize if set
 	if (p->chunkSize != 0) {
@@ -207,7 +207,7 @@ static int asyncTruncate(sqlite3_file* pFile, sqlite_int64 size) {
 }
 
 static int asyncSync(sqlite3_file* pFile, int flags) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		waitFor(p->file->sync());
 		return SQLITE_OK;
@@ -230,7 +230,7 @@ static int asyncSync(sqlite3_file* pFile, int flags) {
 ** Write the size of the file in bytes to *pSize.
 */
 static int VFSAsyncFileSize(sqlite3_file* pFile, sqlite_int64* pSize) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	try {
 		*pSize = waitForAndGet(p->file->size());
 		return SQLITE_OK;
@@ -255,7 +255,7 @@ static int asyncUnlock(sqlite3_file* pFile, int eLock) {
 	return SQLITE_OK;
 }
 static int asyncCheckReservedLock(sqlite3_file* pFile, int* pResOut) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	*pResOut = *p->pLockCount >= RESERVED_COUNT;
 	return SQLITE_OK;
 }
@@ -264,7 +264,7 @@ static int asyncCheckReservedLock(sqlite3_file* pFile, int* pResOut) {
 ** No xFileControl() verbs are implemented by this VFS.
 */
 static int VFSAsyncFileControl(sqlite3_file* pFile, int op, void* pArg) {
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile;
+	auto* p = (VFSAsyncFile*)pFile;
 	switch (op) {
 	case SQLITE_FCNTL_CHUNK_SIZE:
 		p->chunkSize = *(int*)pArg;
@@ -337,7 +337,7 @@ static int asyncShmMap(sqlite3_file* fd, /* Handle open on database file */
 ) {
 	MutexHolder hold(SharedMemoryInfo::mutex);
 
-	VFSAsyncFile* pDbFd = (VFSAsyncFile*)fd;
+	auto* pDbFd = (VFSAsyncFile*)fd;
 	SharedMemoryInfo* memInfo = pDbFd->sharedMemory;
 	if (!memInfo) {
 		std::string filename = pDbFd->filename;
@@ -388,7 +388,7 @@ static int asyncShmLock(sqlite3_file* fd, /* Database file holding the shared me
 
 	MutexHolder hold(SharedMemoryInfo::mutex);
 
-	VFSAsyncFile* pDbFd = (VFSAsyncFile*)fd;
+	auto* pDbFd = (VFSAsyncFile*)fd;
 	SharedMemoryInfo* memInfo = pDbFd->sharedMemory;
 
 	if (flags & SQLITE_SHM_UNLOCK) {
@@ -455,7 +455,7 @@ static int asyncShmUnmap(sqlite3_file* fd, /* The underlying database file */
 ) {
 	MutexHolder hold(SharedMemoryInfo::mutex);
 
-	VFSAsyncFile* pDbFd = (VFSAsyncFile*)fd;
+	auto* pDbFd = (VFSAsyncFile*)fd;
 	SharedMemoryInfo* memInfo = pDbFd->sharedMemory;
 	if (!memInfo)
 		return SQLITE_OK;
@@ -538,7 +538,7 @@ static int asyncOpen(sqlite3_vfs* pVfs, /* VFS */
 		                                        asyncReadZeroCopy,
 		                                        asyncReleaseZeroCopy };
 
-	VFSAsyncFile* p = (VFSAsyncFile*)pFile; /* Populate this structure */
+	auto* p = (VFSAsyncFile*)pFile; /* Populate this structure */
 
 	if (zName == 0)
 		return SQLITE_IOERR;
diff --git a/fdbserver/include/fdbserver/VFSAsync.h b/fdbserver/kvstore/VFSAsync.h
similarity index 100%
rename from fdbserver/include/fdbserver/VFSAsync.h
rename to fdbserver/kvstore/VFSAsync.h
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/kvstore/VersionedBTree.actor.cpp
similarity index 90%
rename from fdbserver/VersionedBTree.actor.cpp
rename to fdbserver/kvstore/VersionedBTree.actor.cpp
index d0e32e682a1..2edfbedaba3 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/kvstore/VersionedBTree.actor.cpp
@@ -24,13 +24,15 @@
 #include "fdbclient/Tuple.h"
 #include "fdbrpc/DDSketch.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/DeltaTree.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/IPager.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/VersionedBTreeDebug.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "DeltaTree.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "IPager.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "VersionedBTreeDebug.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
 #include "flow/Error.h"
 #include "flow/FastRef.h"
 #include "flow/flow.h"
@@ -40,7 +42,7 @@
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
 #include "flow/ObjectSerializer.h"
-#include "flow/PriorityMultiLock.actor.h"
+#include "flow/PriorityMultiLock.h"
 #include "flow/network.h"
 #include "flow/serialize.h"
 #include "flow/Trace.h"
@@ -49,6 +51,7 @@
 
 #include <boost/intrusive/list.hpp>
 #include <cinttypes>
+#include <functional>
 #include <limits>
 #include <map>
 #include <random>
@@ -60,6 +63,27 @@
 
 using namespace std::string_view_literals;
 
+// Older simulated Redwood files derived the test-only XOR secret from the pager filename after erasing the directory
+// prefix with `erase(0, lastSlash)`, which intentionally leaves the final slash in the hashed suffix. Keep that exact
+// mapping stable so restarted upgrade tests can still read and continue writing those files.
+static uint8_t legacyXorWithForPagerName(std::string filename) {
+	size_t lastSlash = filename.find_last_of("\\/");
+	if (lastSlash != filename.npos) {
+		filename.erase(0, lastSlash);
+	}
+	return filename.empty()
+	           ? 0x5e
+	           : static_cast<uint8_t>(filename[XXH3_64bits(filename.data(), filename.size()) % filename.size()]);
+}
+
+// Legacy XOR pages need the filename-derived byte installed on the ArenaPage before encode/decode.
+// XXHash64 pages ignore this state.
+static void prepareLegacyXorCompatibility(const Reference<ArenaPage>& page, const std::string& pagerName) {
+	if (page->getEncodingType() == EncodingType::XOREncryption_TestOnly_DEPRECATED) {
+		page->setLegacyXorWith(legacyXorWithForPagerName(pagerName));
+	}
+}
+
 // Returns a string where every line in lines is prefixed with prefix
 std::string addPrefix(std::string prefix, std::string lines) {
 	StringRef m = lines;
@@ -250,7 +274,8 @@ struct FIFOQueueCodec {
 };
 
 template <typename T>
-struct FIFOQueueCodec<T, typename std::enable_if<std::is_trivially_copyable<T>::value>::type> {
+    requires(std::is_trivially_copyable_v<T>)
+struct FIFOQueueCodec<T> {
 	static_assert(std::is_trivially_copyable<T>::value);
 	static T readFromBytes(const uint8_t* src, int& bytesRead) {
 		bytesRead = sizeof(T);
@@ -468,12 +493,10 @@ class FIFOQueue {
 		// Wait for all operations started before now to be ready, which is done by
 		// obtaining and releasing the mutex.
 		Future<Void> notBusy() {
-			return isBusy() ? map(mutex.take(),
-			                      [&](FlowMutex::Lock lock) {
-				                      lock.release();
-				                      return Void();
-			                      })
-			                : Void();
+			if (isBusy()) {
+				FlowMutex::Lock lock = co_await mutex.take();
+				lock.release();
+			}
 		}
 
 		// Returns true if any items have been written to the last page
@@ -502,11 +525,8 @@ class FIFOQueue {
 		Future<Void> loadExtent() {
 			ASSERT(mode == POP | mode == READONLY);
 			debug_printf("FIFOQueue::Cursor(%s) loadExtent\n", toString().c_str());
-			return map(queue->pager->readExtent(pageID), [=](Reference<ArenaPage> p) {
-				page = p;
-				debug_printf("FIFOQueue::Cursor(%s) loadExtent done. Page: %p\n", toString().c_str(), page->rawData());
-				return Void();
-			});
+			page = co_await queue->pager->readExtent(pageID);
+			debug_printf("FIFOQueue::Cursor(%s) loadExtent done. Page: %p\n", toString().c_str(), page->rawData());
 		}
 
 		void writePage() {
@@ -627,13 +647,13 @@ class FIFOQueue {
 		}
 
 		// Write item to the next position in the current page or, if it won't fit, add a new page and write it there.
-		ACTOR static Future<Void> write_impl(Cursor* self, T item) {
+		static Future<Void> write_impl(Cursor* self, T item) {
 			ASSERT(self->mode == WRITE);
 
-			state FlowMutex::Lock lock;
-			state bool mustWait = self->isBusy();
-			state int bytesNeeded = Codec::bytesNeeded(item);
-			state bool needNewPage =
+			FlowMutex::Lock lock;
+			bool mustWait = self->isBusy();
+			int bytesNeeded = Codec::bytesNeeded(item);
+			bool needNewPage =
 			    self->pageID == invalidPhysicalPageID || self->offset + bytesNeeded > self->header()->itemSpace;
 
 			if (g_network->isSimulated()) {
@@ -652,8 +672,7 @@ class FIFOQueue {
 
 			// If we have to wait for the mutex because it's busy, or we need a new page, then wait for the mutex.
 			if (mustWait || needNewPage) {
-				FlowMutex::Lock _lock = wait(self->mutex.take());
-				lock = _lock;
+				lock = co_await self->mutex.take();
 
 				// If we had to wait because the mutex was busy, then update needNewPage as another writer
 				// would have changed the cursor state
@@ -676,7 +695,7 @@ class FIFOQueue {
 				debug_printf("FIFOQueue::Cursor(%s) write(%s) page is full, adding new page\n",
 				             self->toString().c_str(),
 				             ::toString(item).c_str());
-				state PhysicalPageID newPageID;
+				PhysicalPageID newPageID;
 				// If this is an extent based queue, check if there is an available page in current extent
 				if (self->queue->usesExtents) {
 					bool allocateNewExtent = false;
@@ -690,11 +709,11 @@ class FIFOQueue {
 					} else
 						allocateNewExtent = true;
 					if (allocateNewExtent) {
-						PhysicalPageID newPID = wait(self->queue->pager->newExtentPageID(self->queue->queueID));
+						PhysicalPageID newPID = co_await self->queue->pager->newExtentPageID(self->queue->queueID);
 						newPageID = newPID;
 					}
 				} else {
-					PhysicalPageID newPID = wait(self->queue->pager->newPageID());
+					PhysicalPageID newPID = co_await self->queue->pager->newPageID();
 					newPageID = newPID;
 				}
 				self->addNewPage(newPageID, 0, true, true);
@@ -716,13 +735,11 @@ class FIFOQueue {
 				static int sinceYield = 0;
 				if (++sinceYield == 1000) {
 					sinceYield = 0;
-					wait(delay(0));
+					co_await delay(0);
 				}
 
 				lock.release();
 			}
-
-			return Void();
 		}
 
 		void write(const T& item) {
@@ -741,26 +758,26 @@ class FIFOQueue {
 		// The mutex will be taken if locked is false.
 		// The next page will be waited for if load is true.
 		// Only mutex holders will wait on the page read.
-		ACTOR static Future<Optional<T>> waitThenReadNext(Cursor* self,
-		                                                  Optional<T> inclusiveMaximum,
-		                                                  FlowMutex::Lock* lock,
-		                                                  bool load) {
-			state FlowMutex::Lock localLock;
+		static Future<Optional<T>> waitThenReadNext(Cursor* self,
+		                                            Optional<T> inclusiveMaximum,
+		                                            FlowMutex::Lock* lock,
+		                                            bool load) {
+			FlowMutex::Lock localLock;
 
 			// Lock the mutex if it wasn't already locked, so we didn't get a lock pointer
 			if (lock == nullptr) {
 				debug_printf("FIFOQueue::Cursor(%s) waitThenReadNext locking mutex\n", self->toString().c_str());
-				FlowMutex::Lock newLock = wait(self->mutex.take());
+				FlowMutex::Lock newLock = co_await self->mutex.take();
 				localLock = newLock;
 			}
 
 			if (load) {
 				debug_printf("FIFOQueue::Cursor(%s) waitThenReadNext waiting for page load\n",
 				             self->toString().c_str());
-				wait(success(self->nextPageReader));
+				co_await self->nextPageReader;
 			}
 
-			state Optional<T> result = wait(self->readNext(inclusiveMaximum, &localLock));
+			Optional<T> result = co_await self->readNext(inclusiveMaximum, &localLock);
 
 			// If a lock was not passed in, so this actor locked the mutex above, then unlock it
 			if (lock == nullptr) {
@@ -769,14 +786,14 @@ class FIFOQueue {
 				static int sinceYield = 0;
 				if (++sinceYield == 1000) {
 					sinceYield = 0;
-					wait(delay(0));
+					co_await delay(0);
 				}
 
 				debug_printf("FIFOQueue::Cursor(%s) waitThenReadNext unlocking mutex\n", self->toString().c_str());
 				localLock.release();
 			}
 
-			return result;
+			co_return result;
 		}
 
 		// Read the next item from the cursor, possibly moving to and waiting for a new page if the prior page was
@@ -966,38 +983,38 @@ class FIFOQueue {
 	// Fast path extent peekAll (this zooms through the queue reading extents at a time)
 	// Output interface is a promise stream and one vector of results per extent found is sent to the promise stream
 	// Once we are finished reading all the extents of the queue, end_of_stream() is sent to mark completion
-	ACTOR static Future<Void> peekAll_ext(FIFOQueue* self, PromiseStream<Standalone<VectorRef<T>>> res) {
-		state Cursor c;
+	static Future<Void> peekAll_ext(FIFOQueue* self, PromiseStream<Standalone<VectorRef<T>>> res) {
+		Cursor c;
 		c.initReadOnly(self->headReader, true);
 
 		debug_printf("FIFOQueue::Cursor(%s) peekAllExt begin\n", c.toString().c_str());
 		if (c.pageID == invalidPhysicalPageID || c.pageID == c.endPageID) {
 			debug_printf("FIFOQueue::Cursor(%s) peekAllExt returning nothing\n", c.toString().c_str());
 			res.sendError(end_of_stream());
-			return Void();
+			co_return;
 		}
 
-		state int entriesRead = 0;
+		int entriesRead = 0;
 		// Loop over all the extents in this queue
-		loop {
+		while (true) {
 			// We now know we are pointing to PageID and it should be read and used, but it may not be loaded yet.
 			if (!c.page) {
 				debug_printf("FIFOQueue::Cursor(%s) peekAllExt going to Load Extent %s.\n",
 				             c.toString().c_str(),
 				             ::toString(c.pageID).c_str());
-				wait(c.loadExtent());
-				wait(yield());
+				co_await c.loadExtent();
+				co_await yield();
 			}
 
-			state Standalone<VectorRef<T>> results;
+			Standalone<VectorRef<T>> results;
 			results.reserve(results.arena(), self->pagesPerExtent * self->pager->getPhysicalPageSize() / sizeof(T));
 
 			// Loop over all the pages in this extent
-			state int pageIdx = 0;
-			loop {
+			int pageIdx = 0;
+			while (true) {
 				// Position the page pointer to current page in the extent
-				state Reference<ArenaPage> page = c.page->getSubPage(pageIdx++ * self->pager->getPhysicalPageSize(),
-				                                                     self->pager->getLogicalPageSize());
+				Reference<ArenaPage> page = c.page->getSubPage(pageIdx++ * self->pager->getPhysicalPageSize(),
+				                                               self->pager->getLogicalPageSize());
 				debug_printf("FIFOQueue::Cursor(%s) peekALLExt %s. Offset %d\n",
 				             c.toString().c_str(),
 				             toString(c.pageID).c_str(),
@@ -1039,7 +1056,7 @@ class FIFOQueue {
 				int bytesRead;
 
 				// Now loop over all entries inside the current page
-				loop {
+				while (true) {
 					ASSERT(c.offset < p->endOffset);
 					T result = Codec::readFromBytes(p->begin() + c.offset, bytesRead);
 					debug_printf(
@@ -1079,7 +1096,7 @@ class FIFOQueue {
 						throw e;
 					}
 					res.sendError(end_of_stream());
-					return Void();
+					co_return;
 				}
 
 				// Check if we have reached the end of current extent
@@ -1099,15 +1116,15 @@ class FIFOQueue {
 
 	Future<Void> peekAllExt(PromiseStream<Standalone<VectorRef<T>>> resStream) { return peekAll_ext(this, resStream); }
 
-	ACTOR static Future<Standalone<VectorRef<T>>> peekAll_impl(FIFOQueue* self) {
-		state Standalone<VectorRef<T>> results;
-		state Cursor c;
+	static Future<Standalone<VectorRef<T>>> peekAll_impl(FIFOQueue* self) {
+		Standalone<VectorRef<T>> results;
+		Cursor c;
 		c.initReadOnly(self->headReader);
 		results.reserve(results.arena(), self->numEntries);
 
-		state int sinceYield = 0;
-		loop {
-			Optional<T> x = wait(c.readNext());
+		int sinceYield = 0;
+		while (true) {
+			Optional<T> x = co_await c.readNext();
 			if (!x.present()) {
 				break;
 			}
@@ -1116,20 +1133,20 @@ class FIFOQueue {
 			// yield periodically to avoid overflowing the stack
 			if (++sinceYield >= 100) {
 				sinceYield = 0;
-				wait(yield());
+				co_await yield();
 			}
 		}
 
-		return results;
+		co_return results;
 	}
 
 	Future<Standalone<VectorRef<T>>> peekAll() { return peekAll_impl(this); }
 
-	ACTOR static Future<Optional<T>> peek_impl(FIFOQueue* self) {
-		state Cursor c;
+	static Future<Optional<T>> peek_impl(FIFOQueue* self) {
+		Cursor c;
 		c.initReadOnly(self->headReader);
-		Optional<T> x = wait(c.readNext());
-		return x;
+		Optional<T> x = co_await c.readNext();
+		co_return x;
 	}
 
 	Future<Optional<T>> peek() { return peek_impl(this); }
@@ -1187,9 +1204,9 @@ class FIFOQueue {
 	//   - queue push() can call pager->newPageID() which can call pop() on the same or another queue
 	// This creates a circular dependency with 1 or more queues when those queues are used by the pager
 	// to manage free page IDs.
-	ACTOR static Future<bool> preFlush_impl(FIFOQueue* self) {
+	static Future<bool> preFlush_impl(FIFOQueue* self) {
 		debug_printf("FIFOQueue(%s) preFlush begin\n", self->name.c_str());
-		wait(self->notBusy());
+		co_await self->notBusy();
 
 		// Completion of the pending operations as of the start of notBusy() could have began new operations,
 		// so see if any work is pending now.
@@ -1257,7 +1274,7 @@ class FIFOQueue {
 		}
 
 		debug_printf("FIFOQueue(%s) preFlush returning %d\n", self->name.c_str(), workPending);
-		return workPending;
+		co_return workPending;
 	}
 
 	Future<bool> preFlush() { return preFlush_impl(this); }
@@ -1307,15 +1324,14 @@ class FIFOQueue {
 		debug_printf("FIFOQueue(%s) finishFlush end\n", name.c_str());
 	}
 
-	ACTOR static Future<Void> flush_impl(FIFOQueue* self) {
-		loop {
-			bool notDone = wait(self->preFlush());
+	static Future<Void> flush_impl(FIFOQueue* self) {
+		while (true) {
+			bool notDone = co_await self->preFlush();
 			if (!notDone) {
 				break;
 			}
 		}
 		self->finishFlush();
-		return Void();
 	}
 
 	Future<Void> flush() { return flush_impl(this); }
@@ -1462,12 +1478,12 @@ struct RedwoodMetrics {
 
 	RedwoodMetrics() {
 		// All histograms have reset their buckets to 0 in the constructor.
-		kvSizeWritten = Reference<Histogram>(
-		    new Histogram(Reference<HistogramRegistry>(), "kvSize", "Written", Histogram::Unit::bytes));
-		kvSizeReadByGet = Reference<Histogram>(
-		    new Histogram(Reference<HistogramRegistry>(), "kvSize", "ReadByGet", Histogram::Unit::bytes));
-		kvSizeReadByGetRange = Reference<Histogram>(
-		    new Histogram(Reference<HistogramRegistry>(), "kvSize", "ReadByGetRange", Histogram::Unit::bytes));
+		kvSizeWritten =
+		    makeReference<Histogram>(Reference<HistogramRegistry>(), "kvSize", "Written", Histogram::Unit::bytes);
+		kvSizeReadByGet =
+		    makeReference<Histogram>(Reference<HistogramRegistry>(), "kvSize", "ReadByGet", Histogram::Unit::bytes);
+		kvSizeReadByGetRange = makeReference<Histogram>(
+		    Reference<HistogramRegistry>(), "kvSize", "ReadByGetRange", Histogram::Unit::bytes);
 
 		ioLock = nullptr;
 
@@ -1476,26 +1492,26 @@ struct RedwoodMetrics {
 		for (RedwoodMetrics::Level& level : levels) {
 			if (levelCounter > 0) {
 				std::string levelString = "L" + std::to_string(levelCounter);
-				level.buildFillPctSketch = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "buildFillPct", levelString, Histogram::Unit::percentageLinear));
-				level.modifyFillPctSketch = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "modifyFillPct", levelString, Histogram::Unit::percentageLinear));
-				level.buildStoredPctSketch = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "buildStoredPct", levelString, Histogram::Unit::percentageLinear));
-				level.modifyStoredPctSketch = Reference<Histogram>(new Histogram(
-				    Reference<HistogramRegistry>(), "modifyStoredPct", levelString, Histogram::Unit::percentageLinear));
-				level.buildItemCountSketch = Reference<Histogram>(new Histogram(Reference<HistogramRegistry>(),
-				                                                                "buildItemCount",
-				                                                                levelString,
-				                                                                Histogram::Unit::countLinear,
-				                                                                0,
-				                                                                maxRecordCount));
-				level.modifyItemCountSketch = Reference<Histogram>(new Histogram(Reference<HistogramRegistry>(),
-				                                                                 "modifyItemCount",
-				                                                                 levelString,
-				                                                                 Histogram::Unit::countLinear,
-				                                                                 0,
-				                                                                 maxRecordCount));
+				level.buildFillPctSketch = makeReference<Histogram>(
+				    Reference<HistogramRegistry>(), "buildFillPct", levelString, Histogram::Unit::percentageLinear);
+				level.modifyFillPctSketch = makeReference<Histogram>(
+				    Reference<HistogramRegistry>(), "modifyFillPct", levelString, Histogram::Unit::percentageLinear);
+				level.buildStoredPctSketch = makeReference<Histogram>(
+				    Reference<HistogramRegistry>(), "buildStoredPct", levelString, Histogram::Unit::percentageLinear);
+				level.modifyStoredPctSketch = makeReference<Histogram>(
+				    Reference<HistogramRegistry>(), "modifyStoredPct", levelString, Histogram::Unit::percentageLinear);
+				level.buildItemCountSketch = makeReference<Histogram>(Reference<HistogramRegistry>(),
+				                                                      "buildItemCount",
+				                                                      levelString,
+				                                                      Histogram::Unit::countLinear,
+				                                                      0,
+				                                                      maxRecordCount);
+				level.modifyItemCountSketch = makeReference<Histogram>(Reference<HistogramRegistry>(),
+				                                                       "modifyItemCount",
+				                                                       levelString,
+				                                                       Histogram::Unit::countLinear,
+				                                                       0,
+				                                                       maxRecordCount);
 			}
 			++levelCounter;
 		}
@@ -1589,21 +1605,25 @@ int RedwoodMetrics::maxRecordCount = 315;
 RedwoodMetrics g_redwoodMetrics = {};
 Future<Void> g_redwoodMetricsActor;
 
-ACTOR Future<Void> redwoodHistogramsLogger(double interval) {
-	state double currTime;
-	loop {
+static void printRedwoodStats() {
+	printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str());
+}
+
+Future<Void> redwoodHistogramsLogger(double interval) {
+	double currTime{ 0 };
+	while (true) {
 		currTime = now();
-		wait(delay(interval));
+		co_await delay(interval);
 		double elapsed = now() - currTime;
 		g_redwoodMetrics.logHistograms(elapsed);
 	}
 }
 
-ACTOR Future<Void> redwoodMetricsLogger() {
+Future<Void> redwoodMetricsLogger() {
 	g_redwoodMetrics.clear();
-	state Future<Void> loggingFuture = redwoodHistogramsLogger(SERVER_KNOBS->REDWOOD_HISTOGRAM_INTERVAL);
-	loop {
-		wait(delay(SERVER_KNOBS->REDWOOD_METRICS_INTERVAL));
+	[[maybe_unused]] Future<Void> loggingFuture = redwoodHistogramsLogger(SERVER_KNOBS->REDWOOD_HISTOGRAM_INTERVAL);
+	while (true) {
+		co_await delay(SERVER_KNOBS->REDWOOD_METRICS_INTERVAL);
 
 		TraceEvent e("RedwoodMetrics");
 		double elapsed = now() - g_redwoodMetrics.startTime;
@@ -1629,7 +1649,7 @@ ACTOR Future<Void> redwoodMetricsLogger() {
 template <class IndexType, class ObjectType>
 class ObjectCache : NonCopyable {
 	struct Entry;
-	typedef std::unordered_map<IndexType, Entry> CacheT;
+	using CacheT = std::unordered_map<IndexType, Entry>;
 
 	struct Entry : public boost::intrusive::list_base_hook<> {
 		Entry() : hits(0), size(0) {}
@@ -1641,7 +1661,7 @@ class ObjectCache : NonCopyable {
 		CacheT* pCache;
 	};
 
-	typedef boost::intrusive::list<Entry> EvictionOrderT;
+	using EvictionOrderT = boost::intrusive::list<Entry>;
 
 public:
 	// Object evictor, manages the eviction order for one or more ObjectCaches
@@ -1650,7 +1670,7 @@ class ObjectCache : NonCopyable {
 	// must eventually give them back with moveIn() or remove them with reclaim().
 	class Evictor : NonCopyable {
 	public:
-		Evictor(int64_t sizeLimit = 0) : sizeLimit(sizeLimit) {}
+		explicit(false) Evictor(int64_t sizeLimit = 0) : sizeLimit(sizeLimit) {}
 
 		// Evictors are normally singletons, either one per real process or one per virtual process in simulation
 		static Evictor* getEvictor() {
@@ -1783,7 +1803,7 @@ class ObjectCache : NonCopyable {
 		int64_t movedOutCount = 0;
 	};
 
-	ObjectCache(Evictor* evictor = nullptr) : pEvictor(evictor) {
+	explicit(false) ObjectCache(Evictor* evictor = nullptr) : pEvictor(evictor) {
 		if (pEvictor == nullptr) {
 			pEvictor = Evictor::getEvictor();
 		}
@@ -1848,7 +1868,7 @@ class ObjectCache : NonCopyable {
 	}
 
 	// Clears the cache, saving the entries to second cache, then waits for each item to be evictable and evicts it.
-	ACTOR static Future<Void> clear_impl(ObjectCache* self, bool waitForSafeEviction) {
+	static Future<Void> clear_impl(ObjectCache* self, bool waitForSafeEviction) {
 		// Claim ownership of all of our cached items, removing them from the evictor's control and quota.
 		for (auto& ie : self->cache) {
 			self->pEvictor->reclaim(ie.second);
@@ -1858,14 +1878,12 @@ class ObjectCache : NonCopyable {
 		// to be destroyed so the prioritizedEvictions head/tail will become invalid.
 		self->prioritizedEvictions.clear();
 
-		state typename CacheT::iterator i = self->cache.begin();
+		typename CacheT::iterator i = self->cache.begin();
 		while (i != self->cache.end()) {
-			wait(waitForSafeEviction ? i->second.item.onEvictable() : i->second.item.cancel());
+			co_await (waitForSafeEviction ? i->second.item.onEvictable() : i->second.item.cancel());
 			++i;
 		}
 		self->cache.clear();
-
-		return Void();
 	}
 
 	Future<Void> clear(bool waitForSafeEviction = false) { return clear_impl(this, waitForSafeEviction); }
@@ -1879,11 +1897,11 @@ class ObjectCache : NonCopyable {
 	EvictionOrderT prioritizedEvictions;
 };
 
-ACTOR template <class T>
-Future<T> forwardError(Future<T> f, Promise<Void> target) {
+template <class T>
+Future<T> forwardError(Future<T> f, Promise<Void> target, ExplicitVoid = {}) {
 	try {
-		T x = wait(f);
-		return x;
+		T x = co_await f;
+		co_return x;
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled && target.canBeSet()) {
 			target.sendError(e);
@@ -1907,9 +1925,9 @@ class DWALPagerSnapshot;
 // back to their original location once the original version is no longer needed.
 class DWALPager final : public IPager2 {
 public:
-	typedef FIFOQueue<LogicalPageID> LogicalPageQueueT;
-	typedef std::map<Version, LogicalPageID> VersionToPageMapT;
-	typedef std::unordered_map<LogicalPageID, VersionToPageMapT> PageToVersionedMapT;
+	using LogicalPageQueueT = FIFOQueue<LogicalPageID>;
+	using VersionToPageMapT = std::map<Version, LogicalPageID>;
+	using PageToVersionedMapT = std::unordered_map<LogicalPageID, VersionToPageMapT>;
 	struct PageCacheEntry {
 		Future<Reference<ArenaPage>> readFuture;
 		Future<Void> writeFuture;
@@ -1936,7 +1954,7 @@ class DWALPager final : public IPager2 {
 			return Void();
 		}
 	};
-	typedef ObjectCache<LogicalPageID, PageCacheEntry> PageCacheT;
+	using PageCacheT = ObjectCache<LogicalPageID, PageCacheEntry>;
 
 	int64_t* getPageCachePenaltySource() override { return &pageCache.evictor().reservedSize; }
 
@@ -1957,9 +1975,9 @@ class DWALPager final : public IPager2 {
 
 	struct RemappedPage {
 		enum Type { NONE = 'N', REMAP = 'R', FREE = 'F', DETACH = 'D' };
-		RemappedPage(Version v = invalidVersion,
-		             LogicalPageID o = invalidLogicalPageID,
-		             LogicalPageID n = invalidLogicalPageID)
+		explicit(false) RemappedPage(Version v = invalidVersion,
+		                             LogicalPageID o = invalidLogicalPageID,
+		                             LogicalPageID n = invalidLogicalPageID)
 		  : version(v), originalPageID(o), newPageID(n) {}
 
 		Version version;
@@ -2002,9 +2020,9 @@ class DWALPager final : public IPager2 {
 
 #pragma pack(pop)
 
-	typedef FIFOQueue<DelayedFreePage> DelayedFreePageQueueT;
-	typedef FIFOQueue<RemappedPage> RemapQueueT;
-	typedef FIFOQueue<ExtentUsedListEntry> ExtentUsedListQueueT;
+	using DelayedFreePageQueueT = FIFOQueue<DelayedFreePage>;
+	using RemapQueueT = FIFOQueue<RemappedPage>;
+	using ExtentUsedListQueueT = FIFOQueue<ExtentUsedListEntry>;
 
 	// If the file already exists, pageSize might be different than desiredPageSize
 	// Use pageCacheSizeBytes == 0 to use default from flow knobs
@@ -2159,7 +2177,7 @@ class DWALPager final : public IPager2 {
 					// If this is a restarted sim test, assume this situation was created by the first phase being
 					// killed during initialization so the second phase found the file on disk but it is not
 					// recoverable.
-					if (g_network->isSimulated() && g_simulator->restarted) {
+					if (g_network->isSimulated() && fdbSimulationPolicyState().restarted) {
 						throw e.asInjectedFault();
 					}
 
@@ -2369,17 +2387,17 @@ class DWALPager final : public IPager2 {
 		getStorageBytes().toTraceEvent(e);
 	}
 
-	ACTOR static void extentCacheClear_impl(DWALPager* self) { wait(self->extentCache.clear()); }
+	static Future<Void> extentCacheClear_impl(Uncancellable, DWALPager* self) { co_await self->extentCache.clear(); }
 
-	void extentCacheClear() override { extentCacheClear_impl(this); }
+	void extentCacheClear() override { extentCacheClear_impl(Uncancellable(), this); }
 
 	// get a list of used extents for a given extent based queue (for testing purpose)
-	ACTOR static Future<Standalone<VectorRef<LogicalPageID>>> getUsedExtents_impl(DWALPager* self, QueueID queueID) {
-		state Standalone<VectorRef<LogicalPageID>> extentIDs;
+	static Future<Standalone<VectorRef<LogicalPageID>>> getUsedExtents_impl(DWALPager* self, QueueID queueID) {
+		Standalone<VectorRef<LogicalPageID>> extentIDs;
 		extentIDs.reserve(extentIDs.arena(),
 		                  self->extentUsedList.numEntries); // TODO this is overreserving. is that a problem?
 
-		Standalone<VectorRef<ExtentUsedListEntry>> extents = wait(self->extentUsedList.peekAll());
+		Standalone<VectorRef<ExtentUsedListEntry>> extents = co_await self->extentUsedList.peekAll();
 		debug_printf("DWALPager(%s) ExtentUsedList size: %d.\n", self->filename.c_str(), extents.size());
 		if (extents.size() > 1) {
 			for (int i = 1; i < extents.size() - 1; i++) {
@@ -2390,7 +2408,7 @@ class DWALPager final : public IPager2 {
 				}
 			}
 		}
-		return extentIDs;
+		co_return extentIDs;
 	}
 
 	Future<Standalone<VectorRef<LogicalPageID>>> getUsedExtents(QueueID queueID) override {
@@ -2414,32 +2432,32 @@ class DWALPager final : public IPager2 {
 
 	// Get a new, previously available page ID.  The page will be considered in-use after the next commit
 	// regardless of whether or not it was written to, until it is returned to the pager via freePage()
-	ACTOR static Future<LogicalPageID> newPageID_impl(DWALPager* self) {
+	static Future<LogicalPageID> newPageID_impl(DWALPager* self) {
 		// First try the free list
-		Optional<LogicalPageID> freePageID = wait(self->freeList.pop());
+		Optional<LogicalPageID> freePageID = co_await self->freeList.pop();
 		if (freePageID.present()) {
 			debug_printf("DWALPager(%s) newPageID() returning %s from free list\n",
 			             self->filename.c_str(),
 			             toString(freePageID.get()).c_str());
-			return freePageID.get();
+			co_return freePageID.get();
 		}
 
 		// Try to reuse pages up to the earlier of the oldest version set by the user or the oldest snapshot still in
 		// the snapshots list
 		Optional<DelayedFreePage> delayedFreePageID =
-		    wait(self->delayedFreeList.pop(DelayedFreePage{ self->effectiveOldestVersion(), 0 }));
+		    co_await self->delayedFreeList.pop(DelayedFreePage{ self->effectiveOldestVersion(), 0 });
 		if (delayedFreePageID.present()) {
 			debug_printf("DWALPager(%s) newPageID() returning %s from delayed free list\n",
 			             self->filename.c_str(),
 			             toString(delayedFreePageID.get()).c_str());
-			return delayedFreePageID.get().pageID;
+			co_return delayedFreePageID.get().pageID;
 		}
 
 		// Lastly, add a new page to the pager
 		LogicalPageID id = self->newLastPageID();
 		debug_printf(
 		    "DWALPager(%s) newPageID() returning %s at end of file\n", self->filename.c_str(), toString(id).c_str());
-		return id;
+		co_return id;
 	};
 
 	// Grow the pager file by one page and return it
@@ -2455,16 +2473,16 @@ class DWALPager final : public IPager2 {
 
 	// Get a new, previously available extent and it's first page ID.  The page will be considered in-use after the next
 	// commit regardless of whether or not it was written to, until it is returned to the pager via freePage()
-	ACTOR static Future<LogicalPageID> newExtentPageID_impl(DWALPager* self, QueueID queueID) {
+	static Future<LogicalPageID> newExtentPageID_impl(DWALPager* self, QueueID queueID) {
 		// First try the free list
-		Optional<LogicalPageID> freeExtentID = wait(self->extentFreeList.pop());
+		Optional<LogicalPageID> freeExtentID = co_await self->extentFreeList.pop();
 		if (freeExtentID.present()) {
 			debug_printf("DWALPager(%s) remapQueue newExtentPageID() returning %s from free list\n",
 			             self->filename.c_str(),
 			             toString(freeExtentID.get()).c_str());
 			self->extentUsedList.pushBack({ queueID, freeExtentID.get() });
 			self->extentUsedList.getState();
-			return freeExtentID.get();
+			co_return freeExtentID.get();
 		}
 
 		// Lastly, add a new extent to the pager
@@ -2474,7 +2492,7 @@ class DWALPager final : public IPager2 {
 		             toString(id).c_str());
 		self->extentUsedList.pushBack({ queueID, id });
 		self->extentUsedList.getState();
-		return id;
+		co_return id;
 	}
 
 	// Grow the pager file by one extent and return it
@@ -2489,20 +2507,21 @@ class DWALPager final : public IPager2 {
 	Future<LogicalPageID> newExtentPageID(QueueID queueID) override { return newExtentPageID_impl(this, queueID); }
 
 	// Write one block of a page of a physical page in the page file.  Futures returned must be allowed to complete.
-	ACTOR static UNCANCELLABLE Future<Void> writePhysicalBlock(DWALPager* self,
-	                                                           Reference<ArenaPage> page,
-	                                                           int blockNum,
-	                                                           int blockSize,
-	                                                           PhysicalPageID pageID,
-	                                                           PagerEventReasons reason,
-	                                                           unsigned int level,
-	                                                           bool header) {
-
-		state PriorityMultiLock::Lock lock = wait(self->ioLock->lock(header ? ioMaxPriority : ioMinPriority));
+	static Future<Void> writePhysicalBlock(DWALPager* self,
+	                                       Reference<ArenaPage> page,
+	                                       int blockNum,
+	                                       int blockSize,
+	                                       PhysicalPageID pageID,
+	                                       PagerEventReasons reason,
+	                                       unsigned int level,
+	                                       bool header,
+	                                       Uncancellable = Uncancellable()) {
+
+		PriorityMultiLock::Lock lock = co_await self->ioLock->lock(header ? ioMaxPriority : ioMinPriority);
 		++g_redwoodMetrics.metric.pagerDiskWrite;
 		g_redwoodMetrics.level(level).metrics.events.addEventReason(PagerEvents::PageWrite, reason);
 		if (self->memoryOnly) {
-			return Void();
+			co_return;
 		}
 
 		// If a truncation up to include pageID has not yet been completed
@@ -2514,38 +2533,35 @@ class DWALPager final : public IPager2 {
 			}
 
 			// Wait for extension that covers pageID to complete;
-			wait(self->fileExtension);
+			co_await self->fileExtension;
 		}
 
 		// Note:  Not using forwardError here so a write error won't be discovered until commit time.
 		debug_printf("DWALPager(%s) op=writeBlock %s\n", self->filename.c_str(), toString(pageID).c_str());
-		wait(self->pageFile->write(page->rawData() + (blockNum * blockSize), blockSize, (int64_t)pageID * blockSize));
+		co_await self->pageFile->write(
+		    page->rawData() + (blockNum * blockSize), blockSize, (int64_t)pageID * blockSize);
 
 		// This next line could crash on shutdown because this actor can't be cancelled so self could be destroyed after
 		// write, so enable this line with caution when debugging.
 		// debug_printf("DWALPager(%s) op=writeBlockDone %s\n", self->filename.c_str(), toString(pageID).c_str());
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> extendToCover(DWALPager* self, uint64_t pageID, Future<Void> previousExtension) {
+	static Future<Void> extendToCover(DWALPager* self, uint64_t pageID, Future<Void> previousExtension) {
 		// Calculate new page count, round up to nearest multiple of growth size > pageID
-		state int64_t newPageCount = pageID + SERVER_KNOBS->REDWOOD_PAGEFILE_GROWTH_SIZE_PAGES -
-		                             (pageID % SERVER_KNOBS->REDWOOD_PAGEFILE_GROWTH_SIZE_PAGES);
+		int64_t newPageCount = pageID + SERVER_KNOBS->REDWOOD_PAGEFILE_GROWTH_SIZE_PAGES -
+		                       (pageID % SERVER_KNOBS->REDWOOD_PAGEFILE_GROWTH_SIZE_PAGES);
 
 		// Indicate that extension to this new count has been started
 		self->filePageCountPending = newPageCount;
 
 		// Wait for any previous extensions to complete
-		wait(previousExtension);
+		co_await previousExtension;
 
 		// Grow the file
-		wait(self->pageFile->truncate(newPageCount * self->physicalPageSize));
+		co_await self->pageFile->truncate(newPageCount * self->physicalPageSize);
 
 		// Indicate that extension to the new count has been completed
 		self->filePageCount = newPageCount;
-
-		return Void();
 	}
 
 	// All returned futures are added to the operations vector
@@ -2565,6 +2581,11 @@ class DWALPager final : public IPager2 {
 		// last committed version + 1
 		page->setWriteInfo(pageIDs.front(), this->getLastCommittedVersion() + 1);
 
+		if (page->getEncodingType() == EncodingType::XOREncryption_TestOnly_DEPRECATED) {
+			page = page->clone();
+			prepareLegacyXorCompatibility(page, filename);
+		}
+
 		page->preWrite(pageIDs.front());
 
 		int blockSize = header ? smallestPhysicalBlock : physicalPageSize;
@@ -2589,6 +2610,16 @@ class DWALPager final : public IPager2 {
 		    PagerEventReasons::MetaData, nonBtreeLevel, VectorRef<PhysicalPageID>(&pageID, 1), page, true);
 	}
 
+	static Future<Void> writePhysicalPageAfter(DWALPager* self,
+	                                           Future<Void> after,
+	                                           PagerEventReasons reason,
+	                                           unsigned int level,
+	                                           Standalone<VectorRef<LogicalPageID>> pageIDs,
+	                                           Reference<ArenaPage> data) {
+		co_await after;
+		co_await self->writePhysicalPage(reason, level, pageIDs, data);
+	}
+
 	void updatePage(PagerEventReasons reason,
 	                unsigned int level,
 	                Standalone<VectorRef<LogicalPageID>> pageIDs,
@@ -2617,8 +2648,8 @@ class DWALPager final : public IPager2 {
 		} else if (cacheEntry.reading()) {
 			// This is very unlikely, maybe impossible in the current pager use cases
 			// Wait for the outstanding read to finish, then start the write
-			cacheEntry.writeFuture = mapAsync(success(cacheEntry.readFuture),
-			                                  [=](Void) { return writePhysicalPage(reason, level, pageIDs, data); });
+			cacheEntry.writeFuture =
+			    writePhysicalPageAfter(this, success(cacheEntry.readFuture), reason, level, pageIDs, data);
 		}
 
 		// If the page is being written, wait for this write before issuing the new write to ensure the
@@ -2626,8 +2657,7 @@ class DWALPager final : public IPager2 {
 		else if (cacheEntry.writing()) {
 			// This is very unlikely, maybe impossible in the current pager use cases
 			// Wait for the previous write to finish, then start new write
-			cacheEntry.writeFuture =
-			    mapAsync(cacheEntry.writeFuture, [=](Void) { return writePhysicalPage(reason, level, pageIDs, data); });
+			cacheEntry.writeFuture = writePhysicalPageAfter(this, cacheEntry.writeFuture, reason, level, pageIDs, data);
 		} else {
 			cacheEntry.writeFuture = detach(writePhysicalPage(reason, level, pageIDs, data));
 		}
@@ -2642,26 +2672,23 @@ class DWALPager final : public IPager2 {
 	                                       Reference<ArenaPage> data,
 	                                       Version v) override {
 		debug_printf("DWALPager(%s) op=writeAtomic %s @%" PRId64 "\n", filename.c_str(), toString(pageID).c_str(), v);
-		Future<LogicalPageID> f = map(newPageID(), [=](LogicalPageID newPageID) {
-			updatePage(reason, level, VectorRef<LogicalPageID>(&newPageID, 1), data);
-			// TODO:  Possibly limit size of remap queue since it must be recovered on cold start
-			RemappedPage r{ v, pageID, newPageID };
-			remapQueue.pushBack(r);
-			auto& versionedMap = remappedPages[pageID];
+		LogicalPageID newID = co_await newPageID();
+		updatePage(reason, level, VectorRef<LogicalPageID>(&newID, 1), data);
+		// TODO:  Possibly limit size of remap queue since it must be recovered on cold start
+		RemappedPage r{ v, pageID, newID };
+		remapQueue.pushBack(r);
+		auto& versionedMap = remappedPages[pageID];
 
-			if (SERVER_KNOBS->REDWOOD_EVICT_UPDATED_PAGES) {
-				// An update page is unlikely to have its old version read again soon, so prioritize its cache eviction
-				// If the versioned map is empty for this page then the prior version of the page is at stored at the
-				// PhysicalPageID pageID, otherwise it is the last mapped value in the version-ordered map.
-				pageCache.prioritizeEviction(versionedMap.empty() ? pageID : versionedMap.rbegin()->second);
-			}
-			versionedMap[v] = newPageID;
-
-			debug_printf("DWALPager(%s) pushed %s\n", filename.c_str(), RemappedPage(r).toString().c_str());
-			return pageID;
-		});
+		if (SERVER_KNOBS->REDWOOD_EVICT_UPDATED_PAGES) {
+			// An update page is unlikely to have its old version read again soon, so prioritize its cache eviction
+			// If the versioned map is empty for this page then the prior version of the page is at stored at the
+			// PhysicalPageID pageID, otherwise it is the last mapped value in the version-ordered map.
+			pageCache.prioritizeEviction(versionedMap.empty() ? pageID : versionedMap.rbegin()->second);
+		}
+		versionedMap[v] = newID;
 
-		return f;
+		debug_printf("DWALPager(%s) pushed %s\n", filename.c_str(), RemappedPage(r).toString().c_str());
+		co_return pageID;
 	}
 
 	// Free pageID as of version v.  This means that once the oldest readable pager snapshot is at version v, pageID is
@@ -2754,9 +2781,9 @@ class DWALPager final : public IPager2 {
 		freeUnmappedPage(pageID, v);
 	};
 
-	ACTOR static void freeExtent_impl(DWALPager* self, LogicalPageID pageID) {
+	static Future<Void> freeExtent_impl(Uncancellable, DWALPager* self, LogicalPageID pageID) {
 		self->extentFreeList.pushBack(pageID);
-		Optional<ExtentUsedListEntry> freeExtent = wait(self->extentUsedList.pop());
+		Optional<ExtentUsedListEntry> freeExtent = co_await self->extentUsedList.pop();
 		// Optional<LogicalPageID> freeExtentPageID = wait(self->extentUsedList.pop());
 		if (freeExtent.present()) {
 			debug_printf("DWALPager(%s) freeExtentPageID() popped %s from used list\n",
@@ -2764,31 +2791,32 @@ class DWALPager final : public IPager2 {
 			             toString(freeExtent.get().extentID).c_str());
 		}
 	}
-	void freeExtent(LogicalPageID pageID) override { freeExtent_impl(this, pageID); }
+	void freeExtent(LogicalPageID pageID) override { freeExtent_impl(Uncancellable(), this, pageID); }
 
-	ACTOR static UNCANCELLABLE Future<int> readPhysicalBlock(DWALPager* self,
-	                                                         Reference<ArenaPage> pageBuffer,
-	                                                         int pageOffset,
-	                                                         int blockSize,
-	                                                         int64_t offset,
-	                                                         int priority) {
-		state PriorityMultiLock::Lock lock = wait(self->ioLock->lock(std::min(priority, ioMaxPriority)));
+	static Future<int> readPhysicalBlock(DWALPager* self,
+	                                     Reference<ArenaPage> pageBuffer,
+	                                     int pageOffset,
+	                                     int blockSize,
+	                                     int64_t offset,
+	                                     int priority,
+	                                     Uncancellable = Uncancellable()) {
+		PriorityMultiLock::Lock lock = co_await self->ioLock->lock(std::min(priority, ioMaxPriority));
 		++g_redwoodMetrics.metric.pagerDiskRead;
-		int bytes = wait(self->pageFile->read(pageBuffer->rawData() + pageOffset, blockSize, offset));
-		return bytes;
+		int bytes = co_await self->pageFile->read(pageBuffer->rawData() + pageOffset, blockSize, offset);
+		co_return bytes;
 	}
 
 	// Read a physical page from the page file.  Note that header pages use a page size of smallestPhysicalBlock.
 	// If the user chosen physical page size is larger, then there will be a gap of unused space after the header pages
 	// and before the user-chosen sized pages.
-	ACTOR static Future<Reference<ArenaPage>> readPhysicalPage(DWALPager* self,
-	                                                           PhysicalPageID pageID,
-	                                                           int priority,
-	                                                           bool header,
-	                                                           PagerEventReasons reason) {
+	static Future<Reference<ArenaPage>> readPhysicalPage(DWALPager* self,
+	                                                     PhysicalPageID pageID,
+	                                                     int priority,
+	                                                     bool header,
+	                                                     PagerEventReasons reason) {
 		ASSERT(!self->memoryOnly);
 
-		state Reference<ArenaPage> page =
+		Reference<ArenaPage> page =
 		    header ? makeReference<ArenaPage>(smallestPhysicalBlock, smallestPhysicalBlock) : self->newPageBuffer();
 		debug_printf("DWALPager(%s) op=readPhysicalStart %s ptr=%p header=%d\n",
 		             self->filename.c_str(),
@@ -2796,8 +2824,8 @@ class DWALPager final : public IPager2 {
 		             page->rawData(),
 		             header);
 
-		int readBytes =
-		    wait(readPhysicalBlock(self, page, 0, page->rawSize(), (int64_t)pageID * page->rawSize(), priority));
+		[[maybe_unused]] int readBytes =
+		    co_await readPhysicalBlock(self, page, 0, page->rawSize(), (int64_t)pageID * page->rawSize(), priority);
 		debug_printf("DWALPager(%s) op=readPhysicalDiskReadComplete %s ptr=%p bytes=%d\n",
 		             self->filename.c_str(),
 		             toString(pageID).c_str(),
@@ -2806,6 +2834,7 @@ class DWALPager final : public IPager2 {
 
 		try {
 			page->postReadHeader(pageID);
+			prepareLegacyXorCompatibility(page, self->filename);
 			page->postReadPayload(pageID);
 			debug_printf("DWALPager(%s) op=readPhysicalVerified %s ptr=%p\n",
 			             self->filename.c_str(),
@@ -2834,34 +2863,34 @@ class DWALPager final : public IPager2 {
 			throw err;
 		}
 
-		return page;
+		co_return page;
 	}
 
-	ACTOR static Future<Reference<ArenaPage>> readPhysicalMultiPage(DWALPager* self,
-	                                                                Standalone<VectorRef<PhysicalPageID>> pageIDs,
-	                                                                int priority,
-	                                                                Optional<PagerEventReasons> reason) {
+	static Future<Reference<ArenaPage>> readPhysicalMultiPage(DWALPager* self,
+	                                                          Standalone<VectorRef<PhysicalPageID>> pageIDs,
+	                                                          int priority,
+	                                                          Optional<PagerEventReasons> reason) {
 		ASSERT(!self->memoryOnly);
 
 		// if (g_network->getCurrentTask() > TaskPriority::DiskRead) {
 		// 	wait(delay(0, TaskPriority::DiskRead));
 		// }
 
-		state Reference<ArenaPage> page = self->newPageBuffer(pageIDs.size());
+		Reference<ArenaPage> page = self->newPageBuffer(pageIDs.size());
 		debug_printf("DWALPager(%s) op=readPhysicalMultiStart %s ptr=%p\n",
 		             self->filename.c_str(),
 		             toString(pageIDs).c_str(),
 		             page->rawData());
 
 		// TODO:  Could a dispatched read try to write to page after it has been destroyed if this actor is cancelled?
-		state int blockSize = self->physicalPageSize;
+		int blockSize = self->physicalPageSize;
 		std::vector<Future<int>> reads;
 		for (int i = 0; i < pageIDs.size(); ++i) {
 			reads.push_back(
 			    readPhysicalBlock(self, page, i * blockSize, blockSize, ((int64_t)pageIDs[i]) * blockSize, priority));
 		}
 		// wait for all the parallel read futures
-		wait(waitForAll(reads));
+		co_await waitForAll(reads);
 
 		debug_printf("DWALPager(%s) op=readPhysicalMultiDiskReadsComplete %s ptr=%p bytes=%d\n",
 		             self->filename.c_str(),
@@ -2871,6 +2900,7 @@ class DWALPager final : public IPager2 {
 
 		try {
 			page->postReadHeader(pageIDs.front());
+			prepareLegacyXorCompatibility(page, self->filename);
 			page->postReadPayload(pageIDs.front());
 			debug_printf("DWALPager(%s) op=readPhysicalVerified %s ptr=%p bytes=%d\n",
 			             self->filename.c_str(),
@@ -2893,7 +2923,7 @@ class DWALPager final : public IPager2 {
 			throw;
 		}
 
-		return page;
+		co_return page;
 	}
 
 	Future<Reference<ArenaPage>> readHeaderPage(PhysicalPageID pageID) {
@@ -3451,21 +3481,21 @@ class DWALPager final : public IPager2 {
 	}
 
 	// Flush all queues so they have no operations pending.
-	ACTOR static Future<Void> flushQueues(DWALPager* self) {
+	static Future<Void> flushQueues(DWALPager* self) {
 		ASSERT(self->remapCleanupFuture.isReady());
 
 		// Flush remap queue and related queues separately, they are not involved in free page management
-		wait(self->remapQueue.flush());
-		wait(self->extentFreeList.flush());
-		wait(self->extentUsedList.flush());
+		co_await self->remapQueue.flush();
+		co_await self->extentFreeList.flush();
+		co_await self->extentUsedList.flush();
 
 		// Flush the free list and delayed free list queues together as they are used by freePage() and newPageID()
 		// Since each queue's preFlush can create work for the other, we must see preflush return false for both
 		// twice in row.
-		state int clear = 0;
-		loop {
-			state bool freeBusy = wait(self->freeList.preFlush());
-			state bool delayedFreeBusy = wait(self->delayedFreeList.preFlush());
+		int clear = 0;
+		while (true) {
+			bool freeBusy = co_await self->freeList.preFlush();
+			bool delayedFreeBusy = co_await self->delayedFreeList.preFlush();
 			debug_printf("DWALPager(%s) flushQueues freeBusy=%d delayedFreeBusy=%d\n",
 			             self->filename.c_str(),
 			             freeBusy,
@@ -3484,8 +3514,6 @@ class DWALPager final : public IPager2 {
 		}
 		self->freeList.finishFlush();
 		self->delayedFreeList.finishFlush();
-
-		return Void();
 	}
 
 	ACTOR static Future<Void> commit_impl(DWALPager* self, Version v, Value commitRecord) {
@@ -3577,14 +3605,14 @@ class DWALPager final : public IPager2 {
 
 	Value getCommitRecord() const override { return lastCommittedHeader.userCommitRecord; }
 
-	ACTOR void shutdown(DWALPager* self, bool dispose) {
+	Future<Void> shutdown(Uncancellable, DWALPager* self, bool dispose) {
 		// Send to the error promise first and then delay(0) to give users a chance to cancel
 		// any outstanding operations
 		if (self->errorPromise.canBeSet()) {
 			debug_printf("DWALPager(%s) shutdown sending error\n", self->filename.c_str());
 			self->errorPromise.sendError(actor_cancelled()); // Ideally this should be shutdown_in_progress
 		}
-		wait(delay(0));
+		co_await delay(0);
 
 		// The next section explicitly cancels all pending operations held in the pager
 		debug_printf("DWALPager(%s) shutdown kill ioLock\n", self->filename.c_str());
@@ -3593,7 +3621,9 @@ class DWALPager final : public IPager2 {
 		debug_printf("DWALPager(%s) shutdown cancel recovery\n", self->filename.c_str());
 		self->recoverFuture.cancel();
 		debug_printf("DWALPager(%s) shutdown cancel commit\n", self->filename.c_str());
-		self->commitFuture.cancel();
+		// Cancelling the future does not destroy the state held by the commit coroutine. Reset the future entirely so
+		// shutdown releases that coroutine state before tearing down the pager.
+		self->commitFuture = Future<Void>();
 		debug_printf("DWALPager(%s) shutdown cancel remap\n", self->filename.c_str());
 		self->remapCleanupFuture.cancel();
 		debug_printf("DWALPager(%s) shutdown kill file extension\n", self->filename.c_str());
@@ -3613,8 +3643,8 @@ class DWALPager final : public IPager2 {
 		self->extentUsedList.cancel();
 
 		debug_printf("DWALPager(%s) shutdown destroy page cache\n", self->filename.c_str());
-		wait(self->extentCache.clear());
-		wait(self->pageCache.clear());
+		co_await self->extentCache.clear();
+		co_await self->pageCache.clear();
 
 		debug_printf("DWALPager(%s) shutdown remappedPagesMap: %s\n",
 		             self->filename.c_str(),
@@ -3628,7 +3658,7 @@ class DWALPager final : public IPager2 {
 				// We wrap this with ready() because we don't care if incrementalDeleteFile throws an error because:
 				// - if the file was unlinked, the file will be cleaned up by the filesystem after the process exits
 				// - if the file was not unlinked, the worker will restart and pick it up and it'll be removed later
-				wait(ready(IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename, true)));
+				co_await ready(IAsyncFileSystem::filesystem()->incrementalDeleteFile(self->filename, true));
 			}
 		}
 
@@ -3637,9 +3667,9 @@ class DWALPager final : public IPager2 {
 		closedPromise.send(Void());
 	}
 
-	void dispose() override { shutdown(this, true); }
+	void dispose() override { shutdown(Uncancellable(), this, true); }
 
-	void close() override { shutdown(this, false); }
+	void close() override { shutdown(Uncancellable(), this, false); }
 
 	Future<Void> getError() const override { return errorPromise.getFuture(); }
 
@@ -3689,12 +3719,12 @@ class DWALPager final : public IPager2 {
 	int64_t getPageCount() override { return header.pageCount; }
 	int64_t getExtentCacheCount() override { return extentCache.getCount(); }
 
-	ACTOR static Future<Void> getUserPageCount_cleanup(DWALPager* self) {
+	static Future<Void> getUserPageCount_cleanup(DWALPager* self) {
 		// Wait for the remap eraser to finish all of its work (not triggering stop)
-		wait(self->remapCleanupFuture);
+		co_await self->remapCleanupFuture;
 
 		// Flush queues so there are no pending freelist operations
-		wait(flushQueues(self));
+		co_await flushQueues(self);
 
 		debug_printf("DWALPager getUserPageCount_cleanup\n");
 		self->freeList.getState();
@@ -3702,31 +3732,29 @@ class DWALPager final : public IPager2 {
 		self->extentFreeList.getState();
 		self->extentUsedList.getState();
 		self->remapQueue.getState();
-		return Void();
 	}
 
 	// Get the number of pages in use by the pager's user
 	Future<int64_t> getUserPageCount() override {
-		return map(getUserPageCount_cleanup(this), [=](Void) {
-			int64_t userPages =
-			    header.pageCount - 2 - freeList.numPages - freeList.numEntries - delayedFreeList.numPages -
-			    delayedFreeList.numEntries - ((((remapQueue.numPages - 1) / pagesPerExtent) + 1) * pagesPerExtent) -
-			    extentFreeList.numPages - (pagesPerExtent * extentFreeList.numEntries) - extentUsedList.numPages;
-
-			debug_printf("DWALPager(%s) userPages=%" PRId64 " totalPageCount=%" PRId64 " freeQueuePages=%" PRId64
-			             " freeQueueCount=%" PRId64 " delayedFreeQueuePages=%" PRId64 " delayedFreeQueueCount=%" PRId64
-			             " remapQueuePages=%" PRId64 " remapQueueCount=%" PRId64 "\n",
-			             filename.c_str(),
-			             userPages,
-			             header.pageCount,
-			             freeList.numPages,
-			             freeList.numEntries,
-			             delayedFreeList.numPages,
-			             delayedFreeList.numEntries,
-			             remapQueue.numPages,
-			             remapQueue.numEntries);
-			return userPages;
-		});
+		co_await getUserPageCount_cleanup(this);
+		int64_t userPages =
+		    header.pageCount - 2 - freeList.numPages - freeList.numEntries - delayedFreeList.numPages -
+		    delayedFreeList.numEntries - ((((remapQueue.numPages - 1) / pagesPerExtent) + 1) * pagesPerExtent) -
+		    extentFreeList.numPages - (pagesPerExtent * extentFreeList.numEntries) - extentUsedList.numPages;
+
+		debug_printf("DWALPager(%s) userPages=%" PRId64 " totalPageCount=%" PRId64 " freeQueuePages=%" PRId64
+		             " freeQueueCount=%" PRId64 " delayedFreeQueuePages=%" PRId64 " delayedFreeQueueCount=%" PRId64
+		             " remapQueuePages=%" PRId64 " remapQueueCount=%" PRId64 "\n",
+		             filename.c_str(),
+		             userPages,
+		             header.pageCount,
+		             freeList.numPages,
+		             freeList.numEntries,
+		             delayedFreeList.numPages,
+		             delayedFreeList.numEntries,
+		             remapQueue.numPages,
+		             remapQueue.numEntries);
+		co_return userPages;
 	}
 
 	Future<Void> init() override { return recoverFuture; }
@@ -3734,7 +3762,7 @@ class DWALPager final : public IPager2 {
 	Version getLastCommittedVersion() const override { return lastCommittedHeader.committedVersion; }
 
 private:
-	~DWALPager() {}
+	~DWALPager() = default;
 
 	// Try to expire snapshots up to but not including v, but do not expire any snapshots that are in use.
 	void expireSnapshots(Version v);
@@ -3777,20 +3805,20 @@ class DWALPager final : public IPager2 {
 		}
 	};
 
-	ACTOR static Future<Void> clearRemapQueue_impl(DWALPager* self) {
+	static Future<Void> clearRemapQueue_impl(DWALPager* self) {
 		// Wait for outstanding commit.
-		wait(self->commitFuture);
+		co_await self->commitFuture;
 
 		// Set remap cleanup window to 0 to allow the remap queue to drain.
-		state int64_t remapCleanupWindowBytes = self->remapCleanupWindowBytes;
+		int64_t remapCleanupWindowBytes = self->remapCleanupWindowBytes;
 		self->remapCleanupWindowBytes = 0;
 
 		// Try twice to commit and advance version. The first commit should trigger a remap cleanup actor, which picks
 		// up the new remap cleanup window being 0. The second commit waits for the remap cleanup actor to finish.
-		state int attempt = 0;
+		int attempt = 0;
 		for (attempt = 0; attempt < 2; attempt++) {
 			self->setOldestReadableVersion(self->getLastCommittedVersion());
-			wait(self->commit(self->getLastCommittedVersion() + 1, self->getCommitRecord()));
+			co_await self->commit(self->getLastCommittedVersion() + 1, self->getCommitRecord());
 		}
 		ASSERT(self->remapQueue.numEntries == 0);
 
@@ -3801,7 +3829,6 @@ class DWALPager final : public IPager2 {
 		TraceEvent e("RedwoodClearRemapQueue");
 		self->toTraceEvent(e);
 		e.log();
-		return Void();
 	}
 
 	Future<Void> clearRemapQueue() override { return clearRemapQueue_impl(this); }
@@ -3901,7 +3928,7 @@ class DWALPager final : public IPager2 {
 class DWALPagerSnapshot : public IPagerSnapshot, public ReferenceCounted<DWALPagerSnapshot> {
 public:
 	DWALPagerSnapshot(DWALPager* pager, Key meta, Version version) : pager(pager), version(version), metaKey(meta) {}
-	~DWALPagerSnapshot() override {}
+	~DWALPagerSnapshot() override = default;
 
 	Future<Reference<const ArenaPage>> getPhysicalPage(PagerEventReasons reason,
 	                                                   unsigned int level,
@@ -3910,8 +3937,9 @@ class DWALPagerSnapshot : public IPagerSnapshot, public ReferenceCounted<DWALPag
 	                                                   bool cacheable,
 	                                                   bool noHit) override {
 
-		return map(pager->readPageAtVersion(reason, level, pageID, priority, version, cacheable, noHit),
-		           [=](Reference<ArenaPage> p) { return Reference<const ArenaPage>(std::move(p)); });
+		Reference<ArenaPage> page =
+		    co_await pager->readPageAtVersion(reason, level, pageID, priority, version, cacheable, noHit);
+		co_return Reference<const ArenaPage>(std::move(page));
 	}
 
 	Future<Reference<const ArenaPage>> getMultiPhysicalPage(PagerEventReasons reason,
@@ -3921,8 +3949,8 @@ class DWALPagerSnapshot : public IPagerSnapshot, public ReferenceCounted<DWALPag
 	                                                        bool cacheable,
 	                                                        bool noHit) override {
 
-		return map(pager->readMultiPage(reason, level, pageIDs, priority, cacheable, noHit),
-		           [=](Reference<ArenaPage> p) { return Reference<const ArenaPage>(std::move(p)); });
+		Reference<ArenaPage> page = co_await pager->readMultiPage(reason, level, pageIDs, priority, cacheable, noHit);
+		co_return Reference<const ArenaPage>(std::move(page));
 	}
 
 	Key getMetaKey() const override { return metaKey; }
@@ -3971,7 +3999,7 @@ struct SplitStringRef {
 	StringRef a;
 	StringRef b;
 
-	SplitStringRef(StringRef a = StringRef(), StringRef b = StringRef()) : a(a), b(b) {}
+	explicit SplitStringRef(StringRef a = StringRef(), StringRef b = StringRef()) : a(a), b(b) {}
 
 	SplitStringRef(Arena& arena, const SplitStringRef& toCopy) : a(toStringRef(arena)), b() {}
 
@@ -4059,8 +4087,8 @@ struct SplitStringRef {
 };
 
 // A BTree node link is a list of LogicalPageID's whose contents should be concatenated together.
-typedef VectorRef<LogicalPageID> BTreeNodeLinkRef;
-typedef Standalone<BTreeNodeLinkRef> BTreeNodeLink;
+using BTreeNodeLinkRef = VectorRef<LogicalPageID>;
+using BTreeNodeLink = Standalone<BTreeNodeLinkRef>;
 
 constexpr LogicalPageID maxPageID = (LogicalPageID)-1;
 
@@ -4069,9 +4097,9 @@ std::string toString(BTreeNodeLinkRef id) {
 }
 
 struct RedwoodRecordRef {
-	typedef uint8_t byte;
+	using byte = uint8_t;
 
-	RedwoodRecordRef(KeyRef key = KeyRef(), Optional<ValueRef> value = {}) : key(key), value(value) {}
+	explicit(false) RedwoodRecordRef(KeyRef key = KeyRef(), Optional<ValueRef> value = {}) : key(key), value(value) {}
 
 	RedwoodRecordRef(Arena& arena, const RedwoodRecordRef& toCopy) : key(arena, toCopy.key) {
 		if (toCopy.value.present()) {
@@ -4079,7 +4107,7 @@ struct RedwoodRecordRef {
 		}
 	}
 
-	typedef KeyRef Partial;
+	using Partial = KeyRef;
 
 	void updateCache(Optional<Partial>& cache, Arena& arena) const { cache = KeyRef(arena, key); }
 
@@ -4529,8 +4557,8 @@ struct RedwoodRecordRef {
 };
 
 struct BTreePage {
-	typedef DeltaTree2<RedwoodRecordRef> BinaryTree;
-	typedef DeltaTree2<RedwoodRecordRef, RedwoodRecordRef::DeltaValueOnly> ValueTree;
+	using BinaryTree = DeltaTree2<RedwoodRecordRef>;
+	using ValueTree = DeltaTree2<RedwoodRecordRef, RedwoodRecordRef::DeltaValueOnly>;
 
 #pragma pack(push, 1)
 	struct {
@@ -4639,7 +4667,7 @@ class DecodeBoundaryVerifier {
 		bool empty() const { return lower.empty() && upper.empty(); }
 	};
 
-	typedef std::map<Version, DecodeBoundaries> BoundariesByVersion;
+	using BoundariesByVersion = std::map<Version, DecodeBoundaries>;
 	std::unordered_map<LogicalPageID, BoundariesByVersion> boundariesByPageID;
 	int boundarySampleSize = 1000;
 	int boundaryPopulation = 0;
@@ -4654,7 +4682,7 @@ class DecodeBoundaryVerifier {
 	static DecodeBoundaryVerifier* getVerifier(std::string name) {
 		static std::map<std::string, DecodeBoundaryVerifier> verifiers;
 		// Only use verifier in a non-restarted simulation so that all page writes are captured
-		if (g_network->isSimulated() && !g_simulator->restarted) {
+		if (g_network->isSimulated() && !fdbSimulationPolicyState().restarted) {
 			return &verifiers[name];
 		}
 		return nullptr;
@@ -4794,7 +4822,7 @@ class VersionedBTree {
 		std::string toString() const { return format("{%s @%" PRId64 "}", ::toString(pageID).c_str(), version); }
 	};
 
-	typedef FIFOQueue<LazyClearQueueEntry> LazyClearQueueT;
+	using LazyClearQueueT = FIFOQueue<LazyClearQueueEntry>;
 
 	struct ParentInfo {
 		ParentInfo() {
@@ -4822,7 +4850,7 @@ class VersionedBTree {
 		int count;
 	};
 
-	typedef std::unordered_map<LogicalPageID, ParentInfo> ParentInfoMapT;
+	using ParentInfoMapT = std::unordered_map<LogicalPageID, ParentInfo>;
 
 	struct BTreeCommitHeader {
 		constexpr static FileIdentifier file_identifier = 10847329;
@@ -4921,8 +4949,8 @@ class VersionedBTree {
 		m_latestCommit = m_init;
 	}
 
-	ACTOR static Future<Reference<ArenaPage>> makeEmptyRoot(VersionedBTree* self) {
-		state Reference<ArenaPage> page = self->m_pager->newPageBuffer();
+	static Future<Reference<ArenaPage>> makeEmptyRoot(VersionedBTree* self) {
+		Reference<ArenaPage> page = self->m_pager->newPageBuffer();
 		page->init(self->m_encodingType, PageType::BTreeNode, 1);
 
 		BTreePage* btpage = (BTreePage*)page->mutateData();
@@ -5091,11 +5119,16 @@ class VersionedBTree {
 			}
 
 			if (self->m_encodingType != self->m_header.encodingType) {
-				TraceEvent(SevWarn, "RedwoodBTreeUnexpectedEncodingType")
-				    .detail("InstanceName", self->m_pager->getName())
-				    .detail("UsingEncodingType", self->m_encodingType)
-				    .detail("ExistingEncodingType", self->m_header.encodingType);
-				throw unexpected_encoding_type();
+				if (g_network->isSimulated() &&
+				    self->m_header.encodingType == EncodingType::XOREncryption_TestOnly_DEPRECATED) {
+					self->m_encodingType = self->m_header.encodingType;
+				} else {
+					TraceEvent(SevWarn, "RedwoodBTreeUnexpectedEncodingType")
+					    .detail("InstanceName", self->m_pager->getName())
+					    .detail("UsingEncodingType", self->m_encodingType)
+					    .detail("ExistingEncodingType", self->m_header.encodingType);
+					throw unexpected_encoding_type();
+				}
 			}
 
 			self->m_lazyClearQueue.recover(self->m_pager, self->m_header.lazyDeleteQueue, "LazyClearQueueRecovered");
@@ -5185,7 +5218,7 @@ class VersionedBTree {
 		// Clear
 		SingleKeyMutation() : op(MutationRef::ClearRange) {}
 		// Set
-		SingleKeyMutation(Value val) : op(MutationRef::SetValue), value(val) {}
+		explicit(false) SingleKeyMutation(Value val) : op(MutationRef::SetValue), value(val) {}
 		// Atomic Op
 		SingleKeyMutation(MutationRef::Type op, Value val) : op(op), value(val) {}
 
@@ -5248,7 +5281,7 @@ class VersionedBTree {
 	};
 
 public:
-#include "fdbserver/ArtMutationBuffer.h"
+#include "ArtMutationBuffer.h"
 	struct MutationBufferStdMap {
 		MutationBufferStdMap() {
 			// Create range representing the entire keyspace.  This reduces edge cases to applying mutations
@@ -5260,15 +5293,15 @@ class VersionedBTree {
 		}
 
 	private:
-		typedef std::map<KeyRef, RangeMutation> MutationsT;
+		using MutationsT = std::map<KeyRef, RangeMutation>;
 		Arena arena;
 		MutationsT mutations;
 
 	public:
 		struct iterator : public MutationsT::iterator {
-			typedef MutationsT::iterator Base;
+			using Base = MutationsT::iterator;
 			iterator() = default;
-			iterator(const MutationsT::iterator& i) : Base(i) {}
+			explicit(false) iterator(const MutationsT::iterator& i) : Base(i) {}
 
 			const KeyRef& key() { return (*this)->first; }
 
@@ -5276,10 +5309,10 @@ class VersionedBTree {
 		};
 
 		struct const_iterator : public MutationsT::const_iterator {
-			typedef MutationsT::const_iterator Base;
+			using Base = MutationsT::const_iterator;
 			const_iterator() = default;
-			const_iterator(const MutationsT::const_iterator& i) : Base(i) {}
-			const_iterator(const MutationsT::iterator& i) : Base(i) {}
+			explicit(false) const_iterator(const MutationsT::const_iterator& i) : Base(i) {}
+			explicit(false) const_iterator(const MutationsT::iterator& i) : Base(i) {}
 
 			const KeyRef& key() { return (*this)->first; }
 
@@ -5333,9 +5366,9 @@ class VersionedBTree {
 #define USE_ART_MUTATION_BUFFER 1
 
 #ifdef USE_ART_MUTATION_BUFFER
-	typedef struct MutationBufferART MutationBuffer;
+	using MutationBuffer = MutationBufferART;
 #else
-	typedef struct MutationBufferStdMap MutationBuffer;
+	using MutationBuffer = MutationBufferStdMap;
 #endif
 
 private:
@@ -5639,36 +5672,32 @@ class VersionedBTree {
 	}
 
 	// Writes entries to 1 or more pages and return a vector of boundary keys with their ArenaPage(s)
-	ACTOR static Future<Standalone<VectorRef<RedwoodRecordRef>>> writePages(VersionedBTree* self,
-	                                                                        const RedwoodRecordRef* lowerBound,
-	                                                                        const RedwoodRecordRef* upperBound,
-	                                                                        VectorRef<RedwoodRecordRef> entries,
-	                                                                        unsigned int height,
-	                                                                        Version v,
-	                                                                        BTreeNodeLinkRef previousID,
-	                                                                        LogicalPageID parentID) {
+	static Future<Standalone<VectorRef<RedwoodRecordRef>>> writePages(VersionedBTree* self,
+	                                                                  const RedwoodRecordRef* lowerBound,
+	                                                                  const RedwoodRecordRef* upperBound,
+	                                                                  VectorRef<RedwoodRecordRef> entries,
+	                                                                  unsigned int height,
+	                                                                  Version v,
+	                                                                  BTreeNodeLinkRef previousID,
+	                                                                  LogicalPageID parentID) {
 		ASSERT(entries.size() > 0);
 
-		state Standalone<VectorRef<RedwoodRecordRef>> records;
+		Standalone<VectorRef<RedwoodRecordRef>> records;
 
 		// All records share the prefix shared by the lower and upper boundaries
-		state int prefixLen = lowerBound->getCommonPrefixLen(*upperBound);
+		int prefixLen = lowerBound->getCommonPrefixLen(*upperBound);
 
-		state std::vector<PageToBuild> pagesToBuild =
-		    self->splitPages(lowerBound, upperBound, prefixLen, entries, height);
+		std::vector<PageToBuild> pagesToBuild = self->splitPages(lowerBound, upperBound, prefixLen, entries, height);
 		ASSERT(pagesToBuild.size() > 0);
 		debug_printf("splitPages returning %s\n", toString(pagesToBuild).c_str());
 
 		// Lower bound of the page being added to
-		state RedwoodRecordRef pageLowerBound = lowerBound->withoutValue();
-		state RedwoodRecordRef pageUpperBound;
-		state int sinceYield = 0;
-
-		state int pageIndex;
+		RedwoodRecordRef pageLowerBound = lowerBound->withoutValue();
+		RedwoodRecordRef pageUpperBound;
+		int sinceYield = 0;
 
-		state PageToBuild* p = nullptr;
-		for (pageIndex = 0; pageIndex < pagesToBuild.size(); ++pageIndex) {
-			p = &pagesToBuild[pageIndex];
+		for (int pageIndex = 0; pageIndex < pagesToBuild.size(); ++pageIndex) {
+			PageToBuild* p = &pagesToBuild[pageIndex];
 			debug_printf("building page %d of %zu %s\n", pageIndex + 1, pagesToBuild.size(), p->toString().c_str());
 			ASSERT(p->count != 0);
 
@@ -5713,7 +5742,7 @@ class VersionedBTree {
 			}
 
 			// Create and init page here otherwise many variables must become state vars
-			state Reference<ArenaPage> page = self->m_pager->newPageBuffer(p->blockCount);
+			Reference<ArenaPage> page = self->m_pager->newPageBuffer(p->blockCount);
 			page->init(
 			    self->m_encodingType, (p->blockCount == 1) ? PageType::BTreeNode : PageType::BTreeSuperNode, height);
 
@@ -5731,7 +5760,7 @@ class VersionedBTree {
 			             btPage->tree(),
 			             deltaTreeSpace,
 			             p->usedBytes());
-			state int written = btPage->tree()->build(
+			int written = btPage->tree()->build(
 			    deltaTreeSpace, &entries[p->startIndex], &entries[p->endIndex()], &pageLowerBound, &pageUpperBound);
 
 			if (written > deltaTreeSpace) {
@@ -5753,14 +5782,14 @@ class VersionedBTree {
 			metrics.buildItemCountSketch->sampleRecordCounter(p->count);
 
 			// Write this btree page, which is made of 1 or more pager pages.
-			state BTreeNodeLinkRef childPageID;
+			BTreeNodeLinkRef childPageID;
 
 			// If we are only writing 1 BTree node and its block count is 1 and the original node also had 1 block
 			// then try to update the page atomically so its logical page ID does not change
 			if (pagesToBuild.size() == 1 && p->blockCount == 1 && previousID.size() == 1) {
 				page->setLogicalPageInfo(previousID.front(), parentID);
-				LogicalPageID id = wait(
-				    self->m_pager->atomicUpdatePage(PagerEventReasons::Commit, height, previousID.front(), page, v));
+				LogicalPageID id = co_await self->m_pager->atomicUpdatePage(
+				    PagerEventReasons::Commit, height, previousID.front(), page, v);
 				childPageID.push_back(records.arena(), id);
 			} else {
 				// Either the original node is being split, or only a single node was produced but
@@ -5792,9 +5821,8 @@ class VersionedBTree {
 				}
 
 				childPageID.resize(records.arena(), p->blockCount);
-				state int i = 0;
-				for (i = 0; i < childPageID.size(); ++i) {
-					LogicalPageID id = wait(self->m_pager->newPageID());
+				for (int i = 0; i < childPageID.size(); ++i) {
+					LogicalPageID id = co_await self->m_pager->newPageID();
 					childPageID[i] = id;
 				}
 				debug_printf("writePages: newPages %s", toString(childPageID).c_str());
@@ -5811,7 +5839,7 @@ class VersionedBTree {
 
 			if (++sinceYield > 100) {
 				sinceYield = 0;
-				wait(yield());
+				co_await yield();
 			}
 
 			if (REDWOOD_DEBUG) {
@@ -5838,7 +5866,7 @@ class VersionedBTree {
 			pageLowerBound = pageUpperBound;
 		}
 
-		return records;
+		co_return records;
 	}
 
 	// Takes a list of records commitSubtree() on the root and builds new root pages
@@ -5978,18 +6006,18 @@ class VersionedBTree {
 	// If oldID size is 1, attempts to keep logical page ID via an atomic page update.
 	// Returns resulting BTreePageID which might be the same as the input
 	// updateBTreePage is only called from commitSubTree function so write reason is always btree commit
-	ACTOR static Future<BTreeNodeLinkRef> updateBTreePage(VersionedBTree* self,
-	                                                      BTreeNodeLinkRef oldID,
-	                                                      LogicalPageID parentID,
-	                                                      Arena* arena,
-	                                                      Reference<ArenaPage> page,
-	                                                      Version writeVersion) {
-		state BTreeNodeLinkRef newID;
+	static Future<BTreeNodeLinkRef> updateBTreePage(VersionedBTree* self,
+	                                                BTreeNodeLinkRef oldID,
+	                                                LogicalPageID parentID,
+	                                                Arena* arena,
+	                                                Reference<ArenaPage> page,
+	                                                Version writeVersion) {
+		BTreeNodeLinkRef newID;
 		newID.resize(*arena, oldID.size());
 
 		if (REDWOOD_DEBUG) {
 			const BTreePage* btPage = (const BTreePage*)page->mutateData();
-			BTreePage::BinaryTree::DecodeCache* cache = page->extra.getPtr<BTreePage::BinaryTree::DecodeCache>();
+			auto* cache = page->extra.getPtr<BTreePage::BinaryTree::DecodeCache>();
 
 			debug_printf_always(
 			    "updateBTreePage(%s, %s) start, page:\n%s\n",
@@ -6001,19 +6029,19 @@ class VersionedBTree {
 			              .c_str());
 		}
 
-		state unsigned int height = (unsigned int)((const BTreePage*)page->data())->height;
+		unsigned int height = (unsigned int)((const BTreePage*)page->data())->height;
 		ASSERT(height < 0xf0);
 		if (oldID.size() == 1) {
 			page->setLogicalPageInfo(oldID.front(), parentID);
-			LogicalPageID id = wait(
-			    self->m_pager->atomicUpdatePage(PagerEventReasons::Commit, height, oldID.front(), page, writeVersion));
+			LogicalPageID id = co_await self->m_pager->atomicUpdatePage(
+			    PagerEventReasons::Commit, height, oldID.front(), page, writeVersion);
 			newID.front() = id;
-			return newID;
+			co_return newID;
 		}
 
-		state int i = 0;
+		int i = 0;
 		for (i = 0; i < oldID.size(); ++i) {
-			LogicalPageID id = wait(self->m_pager->newPageID());
+			LogicalPageID id = co_await self->m_pager->newPageID();
 			newID[i] = id;
 		}
 		debug_printf("updateBTreePage(%s, %s): newPages %s",
@@ -6030,7 +6058,7 @@ class VersionedBTree {
 		}
 
 		self->freeBTreePage(height, oldID, writeVersion);
-		return newID;
+		co_return newID;
 	}
 
 	// Copy page to a new page which shares the same DecodeCache with the old page
@@ -6187,7 +6215,7 @@ class VersionedBTree {
 	};
 
 	struct InternalPageModifier {
-		InternalPageModifier() {}
+		InternalPageModifier() = default;
 		InternalPageModifier(Reference<const ArenaPage> p,
 		                     bool alreadyCloned,
 		                     bool updating,
@@ -6357,7 +6385,7 @@ class VersionedBTree {
 		}
 	};
 
-	ACTOR static Future<Void> commitSubtree(
+	static Future<Void> commitSubtree(
 	    VersionedBTree* self,
 	    CommitBatch* batch,
 	    BTreeNodeLinkRef rootID,
@@ -6367,7 +6395,7 @@ class VersionedBTree {
 	    MutationBuffer::const_iterator mEnd, // least boundary >= subtreeUpperBound->key
 	    InternalPageSliceUpdate* update) {
 
-		state std::string context;
+		std::string context;
 		if (REDWOOD_DEBUG) {
 			context = format("CommitSubtree(root=%s+%d %s): ",
 			                 toString(rootID.front()).c_str(),
@@ -6394,16 +6422,16 @@ class VersionedBTree {
 			}
 		}
 
-		state Reference<const ArenaPage> page = wait(
-		    readPage(self, PagerEventReasons::Commit, height, batch->snapshot.getPtr(), rootID, height, false, true));
+		Reference<const ArenaPage> page = co_await readPage(
+		    self, PagerEventReasons::Commit, height, batch->snapshot.getPtr(), rootID, height, false, true);
 
 		// If the page exists in the cache, it must be copied before modification.
 		// That copy will be referenced by pageCopy, as page must stay in scope in case anything references its
 		// memory and it gets evicted from the cache.
 		// If the page is not in the cache, then no copy is needed so we will initialize pageCopy to page
-		state Reference<const ArenaPage> pageCopy;
+		Reference<const ArenaPage> pageCopy;
 
-		state BTreePage* btPage = (BTreePage*)page->mutateData();
+		BTreePage* btPage = (BTreePage*)page->mutateData();
 		if (height != btPage->height) {
 			fprintf(stderr, "height [%d] != btPage->height [%d]\n", height, btPage->height);
 			ASSERT(false);
@@ -6415,7 +6443,7 @@ class VersionedBTree {
 		// records in a DeltaTree being outside its decode boundary range, which isn't actually invalid
 		// though it is awkward to reason about.
 		// TryToUpdate indicates insert and erase operations should be tried on the existing page first
-		state bool tryToUpdate = btPage->tree()->numItems > 0 && update->boundariesNormal();
+		bool tryToUpdate = btPage->tree()->numItems > 0 && update->boundariesNormal();
 
 		debug_printf("%s tryToUpdate=%d\n", context.c_str(), tryToUpdate);
 		debug_print(addPrefix(context,
@@ -6425,9 +6453,8 @@ class VersionedBTree {
 		                                       update->decodeLowerBound,
 		                                       update->decodeUpperBound)));
 
-		state BTreePage::BinaryTree::Cursor cursor = update->cBegin.valid()
-		                                                 ? self->getCursor(page.getPtr(), update->cBegin)
-		                                                 : self->getCursor(page.getPtr(), dbBegin, dbEnd);
+		BTreePage::BinaryTree::Cursor cursor = update->cBegin.valid() ? self->getCursor(page.getPtr(), update->cBegin)
+		                                                              : self->getCursor(page.getPtr(), dbBegin, dbEnd);
 
 		if (self->m_pBoundaryVerifier != nullptr) {
 			if (update->cBegin.valid()) {
@@ -6439,7 +6466,7 @@ class VersionedBTree {
 			}
 		}
 
-		state int maxHeightAllowed = btPage->tree()->initialHeight + SERVER_KNOBS->REDWOOD_NODE_MAX_UNBALANCE;
+		int maxHeightAllowed = btPage->tree()->initialHeight + SERVER_KNOBS->REDWOOD_NODE_MAX_UNBALANCE;
 
 		// Leaf Page
 		if (btPage->isLeaf()) {
@@ -6457,7 +6484,7 @@ class VersionedBTree {
 				}
 			};
 
-			state Standalone<VectorRef<RedwoodRecordRef>> merged;
+			Standalone<VectorRef<RedwoodRecordRef>> merged;
 
 			// The first mutation buffer boundary has a key <= the first key in the page.
 
@@ -6681,7 +6708,7 @@ class VersionedBTree {
 			if (!changesMade) {
 				debug_printf("%s No changes were made during mutation merge, returning slice:\n", context.c_str());
 				debug_print(addPrefix(context, update->toString()));
-				return Void();
+				co_return;
 			} else {
 				debug_printf("%s Changes were made, writing, but subtree may still be unchanged from parent's "
 				             "perspective.\n",
@@ -6697,12 +6724,12 @@ class VersionedBTree {
 					debug_print(addPrefix(context, update->toString()));
 				} else {
 					// Otherwise update it.
-					BTreeNodeLinkRef newID = wait(self->updateBTreePage(self,
-					                                                    rootID,
-					                                                    parentID,
-					                                                    &update->newLinks.arena(),
-					                                                    pageCopy.castTo<ArenaPage>(),
-					                                                    batch->writeVersion));
+					BTreeNodeLinkRef newID = co_await self->updateBTreePage(self,
+					                                                        rootID,
+					                                                        parentID,
+					                                                        &update->newLinks.arena(),
+					                                                        pageCopy.castTo<ArenaPage>(),
+					                                                        batch->writeVersion);
 
 					debug_printf("%s Leaf node updated in-place at version %s, new contents:\n",
 					             context.c_str(),
@@ -6718,7 +6745,7 @@ class VersionedBTree {
 					debug_printf("%s Leaf node updated in-place, returning slice:\n", context.c_str());
 					debug_print(addPrefix(context, update->toString()));
 				}
-				return Void();
+				co_return;
 			}
 
 			// If everything in the page was deleted then this page should be deleted as of the new version
@@ -6728,29 +6755,29 @@ class VersionedBTree {
 
 				debug_printf("%s All leaf page contents were cleared, returning slice:\n", context.c_str());
 				debug_print(addPrefix(context, update->toString()));
-				return Void();
+				co_return;
 			}
 
 			// Rebuild new page(s).
-			state Standalone<VectorRef<RedwoodRecordRef>> entries = wait(writePages(self,
-			                                                                        &update->subtreeLowerBound,
-			                                                                        &update->subtreeUpperBound,
-			                                                                        merged,
-			                                                                        height,
-			                                                                        batch->writeVersion,
-			                                                                        rootID,
-			                                                                        parentID));
+			Standalone<VectorRef<RedwoodRecordRef>> entries = co_await writePages(self,
+			                                                                      &update->subtreeLowerBound,
+			                                                                      &update->subtreeUpperBound,
+			                                                                      merged,
+			                                                                      height,
+			                                                                      batch->writeVersion,
+			                                                                      rootID,
+			                                                                      parentID);
 
 			// Put new links into update and tell update that pages were rebuilt
 			update->rebuilt(entries);
 
 			debug_printf("%s Merge complete, returning slice:\n", context.c_str());
 			debug_print(addPrefix(context, update->toString()));
-			return Void();
+			co_return;
 		} else {
 			// Internal Page
 			std::vector<Future<Void>> recursions;
-			state std::vector<std::unique_ptr<InternalPageSliceUpdate>> slices;
+			std::vector<std::unique_ptr<InternalPageSliceUpdate>> slices;
 
 			cursor.moveFirst();
 
@@ -6926,19 +6953,19 @@ class VersionedBTree {
 			             slices.size(),
 			             recursions.size());
 
-			wait(waitForAll(recursions));
+			co_await waitForAll(recursions);
 			debug_printf("%s Recursions done, processing slice updates.\n", context.c_str());
 
 			// ParentInfo could be invalid after a wait and must be re-initialized.
 			// All uses below occur before waits so no reinitialization is done.
-			state ParentInfo* parentInfo = &self->childUpdateTracker[rootID.front()];
+			ParentInfo* parentInfo = &self->childUpdateTracker[rootID.front()];
 
 			// InternalPageModifier takes the results of the recursive commitSubtree() calls in order
 			// and makes changes to page as needed, copying as needed, and generating an array from
 			// which to build new page(s) if modification is not possible or not allowed.
 			// If pageCopy is already set it was initialized to page above so the modifier doesn't need
 			// to copy it
-			state InternalPageModifier modifier(page, pageCopy.isValid(), tryToUpdate, parentInfo, maxHeightAllowed);
+			InternalPageModifier modifier(page, pageCopy.isValid(), tryToUpdate, parentInfo, maxHeightAllowed);
 
 			// Apply the possible changes for each subtree range recursed to, except the last one.
 			// For each range, the expected next record, if any, is checked against the first boundary
@@ -6963,8 +6990,8 @@ class VersionedBTree {
 			                     modifier.changesMade || slices.back()->childrenChanged ? &update->subtreeUpperBound
 			                                                                            : &update->decodeUpperBound);
 
-			state bool detachChildren = (parentInfo->count > 2);
-			state bool forceUpdate = false;
+			bool detachChildren = (parentInfo->count > 2);
+			bool forceUpdate = false;
 
 			// If no changes were made, but we should rewrite it to point directly to remapped child pages
 			if (!modifier.changesMade && detachChildren) {
@@ -7042,16 +7069,16 @@ class VersionedBTree {
 								             context.c_str());
 								debug_print(addPrefix(context, update->toString()));
 
-								return Void();
+								co_return;
 							}
 						}
 
-						BTreeNodeLinkRef newID = wait(self->updateBTreePage(self,
-						                                                    rootID,
-						                                                    parentID,
-						                                                    &update->newLinks.arena(),
-						                                                    pageCopy.castTo<ArenaPage>(),
-						                                                    batch->writeVersion));
+						BTreeNodeLinkRef newID = co_await self->updateBTreePage(self,
+						                                                        rootID,
+						                                                        parentID,
+						                                                        &update->newLinks.arena(),
+						                                                        pageCopy.castTo<ArenaPage>(),
+						                                                        batch->writeVersion);
 						debug_printf(
 						    "%s commitSubtree(): Internal node updated in-place at version %s, new contents:\n",
 						    context.c_str(),
@@ -7103,14 +7130,14 @@ class VersionedBTree {
 							parentInfo->clear();
 						}
 						Standalone<VectorRef<RedwoodRecordRef>> newChildEntries =
-						    wait(writePages(self,
-						                    &update->subtreeLowerBound,
-						                    &update->subtreeUpperBound,
-						                    modifier.rebuild,
-						                    height,
-						                    batch->writeVersion,
-						                    rootID,
-						                    parentID));
+						    co_await writePages(self,
+						                        &update->subtreeLowerBound,
+						                        &update->subtreeUpperBound,
+						                        modifier.rebuild,
+						                        height,
+						                        batch->writeVersion,
+						                        rootID,
+						                        parentID);
 						update->rebuilt(newChildEntries);
 
 						debug_printf("%s Internal page rebuilt, returning slice:\n", context.c_str());
@@ -7121,7 +7148,6 @@ class VersionedBTree {
 				debug_printf("%s Page has no changes, returning slice:\n", context.c_str());
 				debug_print(addPrefix(context, update->toString()));
 			}
-			return Void();
 		}
 	}
 
@@ -7294,44 +7320,41 @@ class VersionedBTree {
 
 		Future<Void> pushPage(const BTreePage::BinaryTree::Cursor& link) {
 			debug_printf("pushPage(link=%s)\n", link.get().toString(false).c_str());
-			return map(readPage(btree,
-			                    reason,
-			                    path.back().btPage()->height - 1,
-			                    pager.getPtr(),
-			                    link.get().getChildPage(),
-			                    ioMaxPriority,
-			                    false,
-			                    !options.present() || options.get().cacheResult || path.back().btPage()->height != 2),
-			           [=](Reference<const ArenaPage> p) {
-				           BTreePage::BinaryTree::Cursor cursor = btree->getCursor(p.getPtr(), link);
+			BTreePage::BinaryTree::Cursor linkCopy = link;
+			Reference<const ArenaPage> p =
+			    co_await readPage(btree,
+			                      reason,
+			                      path.back().btPage()->height - 1,
+			                      pager.getPtr(),
+			                      linkCopy.get().getChildPage(),
+			                      ioMaxPriority,
+			                      false,
+			                      !options.present() || options.get().cacheResult || path.back().btPage()->height != 2);
+			BTreePage::BinaryTree::Cursor cursor = btree->getCursor(p.getPtr(), linkCopy);
 #if REDWOOD_DEBUG
-				           path.push_back({ p, cursor, link.get().getChildPage() });
+			path.push_back({ p, cursor, linkCopy.get().getChildPage() });
 #else
-					    path.push_back({ p, cursor });
+			path.push_back({ p, cursor });
 #endif
 
-				           if (btree->m_pBoundaryVerifier != nullptr) {
-					           ASSERT(btree->m_pBoundaryVerifier->verify(link.get().getChildPage().front(),
-					                                                     pager->getVersion(),
-					                                                     link.get().key,
-					                                                     link.next().getOrUpperBound().key,
-					                                                     cursor));
-				           }
-				           return Void();
-			           });
+			if (btree->m_pBoundaryVerifier != nullptr) {
+				ASSERT(btree->m_pBoundaryVerifier->verify(linkCopy.get().getChildPage().front(),
+				                                          pager->getVersion(),
+				                                          linkCopy.get().key,
+				                                          linkCopy.next().getOrUpperBound().key,
+				                                          cursor));
+			}
 		}
 
 		Future<Void> pushPage(BTreeNodeLinkRef id) {
 			debug_printf("pushPage(root=%s)\n", ::toString(id).c_str());
-			return map(readPage(btree, reason, btree->m_header.height, pager.getPtr(), id, ioMaxPriority, false, true),
-			           [=](Reference<const ArenaPage> p) {
+			Reference<const ArenaPage> p = co_await readPage(
+			    btree, reason, btree->m_header.height, pager.getPtr(), id, ioMaxPriority, false, true);
 #if REDWOOD_DEBUG
-				           path.push_back({ p, btree->getCursor(p.getPtr(), dbBegin, dbEnd), id });
+			path.push_back({ p, btree->getCursor(p.getPtr(), dbBegin, dbEnd), id });
 #else
-					    path.push_back({ p, btree->getCursor(p.getPtr(), dbBegin, dbEnd) });
+			path.push_back({ p, btree->getCursor(p.getPtr(), dbBegin, dbEnd) });
 #endif
-				           return Void();
-			           });
 		}
 
 		// Initialize or reinitialize cursor
@@ -7359,12 +7382,12 @@ class VersionedBTree {
 		//     If there is a record in the tree > query then moveNext() will move to it.
 		// If non-zero is returned then the cursor is valid and the return value is logically equivalent
 		// to query.compare(cursor.get())
-		ACTOR Future<int> seek_impl(BTreeCursor* self, RedwoodRecordRef query) {
-			state RedwoodRecordRef internalPageQuery = query.withMaxPageID();
+		Future<int> seek_impl(BTreeCursor* self, RedwoodRecordRef query) {
+			RedwoodRecordRef internalPageQuery = query.withMaxPageID();
 			self->path.resize(1);
 			debug_printf("seek(%s) start cursor = %s\n", query.toString().c_str(), self->toString().c_str());
 
-			loop {
+			while (true) {
 				auto& entry = self->path.back();
 				if (entry.btPage()->isLeaf()) {
 					int cmp = entry.cursor.seek(query);
@@ -7373,7 +7396,7 @@ class VersionedBTree {
 					             query.toString().c_str(),
 					             cmp,
 					             self->toString().c_str());
-					return self->valid ? cmp : 0;
+					co_return self->valid ? cmp : 0;
 				}
 
 				// Internal page, so seek to the branch where query must be
@@ -7384,25 +7407,24 @@ class VersionedBTree {
 					debug_printf(
 					    "seek(%s) loop seek success cursor=%s\n", query.toString().c_str(), self->toString().c_str());
 					Future<Void> f = self->pushPage(entry.cursor);
-					wait(f);
+					co_await f;
 				} else {
 					self->valid = false;
 					debug_printf(
 					    "seek(%s) loop exit cmp=0 cursor=%s\n", query.toString().c_str(), self->toString().c_str());
-					return 0;
+					co_return 0;
 				}
 			}
 		}
 
 		Future<int> seek(RedwoodRecordRef query) { return path.empty() ? 0 : seek_impl(this, query); }
 
-		ACTOR Future<Void> seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query) {
+		Future<Void> seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query) {
 			debug_printf("seekGTE(%s) start\n", query.toString().c_str());
-			int cmp = wait(self->seek(query));
+			int cmp = co_await self->seek(query);
 			if (cmp > 0 || (cmp == 0 && !self->isValid())) {
-				wait(self->moveNext());
+				co_await self->moveNext();
 			}
-			return Void();
 		}
 
 		Future<Void> seekGTE(RedwoodRecordRef query) { return seekGTE_impl(this, query); }
@@ -7463,18 +7485,17 @@ class VersionedBTree {
 			}
 		}
 
-		ACTOR Future<Void> seekLT_impl(BTreeCursor* self, RedwoodRecordRef query) {
+		Future<Void> seekLT_impl(BTreeCursor* self, RedwoodRecordRef query) {
 			debug_printf("seekLT(%s) start\n", query.toString().c_str());
-			int cmp = wait(self->seek(query));
+			int cmp = co_await self->seek(query);
 			if (cmp <= 0) {
-				wait(self->movePrev());
+				co_await self->movePrev();
 			}
-			return Void();
 		}
 
 		Future<Void> seekLT(RedwoodRecordRef query) { return seekLT_impl(this, query); }
 
-		ACTOR Future<Void> move_impl(BTreeCursor* self, bool forward) {
+		Future<Void> move_impl(BTreeCursor* self, bool forward) {
 			// Try to the move cursor at the end of the path in the correct direction
 			debug_printf("move%s() start cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str());
 			while (1) {
@@ -7501,7 +7522,7 @@ class VersionedBTree {
 				if (self->path.size() == 1) {
 					self->valid = false;
 					debug_printf("move%s() exit cursor=%s\n", forward ? "Next" : "Prev", self->toString(1).c_str());
-					return Void();
+					co_return;
 				}
 
 				// Move to parent
@@ -7522,7 +7543,7 @@ class VersionedBTree {
 					UNSTOPPABLE_ASSERT(entry.cursor.get().value.present());
 				}
 
-				wait(self->pushPage(entry.cursor));
+				co_await self->pushPage(entry.cursor);
 				auto& newEntry = self->path.back();
 				UNSTOPPABLE_ASSERT(forward ? newEntry.cursor.moveFirst() : newEntry.cursor.moveLast());
 			}
@@ -7530,7 +7551,6 @@ class VersionedBTree {
 			self->valid = true;
 
 			debug_printf("move%s() exit cursor=%s\n", forward ? "Next" : "Prev", self->toString(1).c_str());
-			return Void();
 		}
 
 		Future<Void> moveNext() { return path.empty() ? Void() : move_impl(this, true); }
@@ -7548,7 +7568,7 @@ class VersionedBTree {
 		if (!snapshot->extra.valid()) {
 			KeyRef m = snapshot->getMetaKey();
 			if (!m.empty()) {
-				BTreeCommitHeader h = ObjectReader::fromStringRef<VersionedBTree::BTreeCommitHeader>(m, Unversioned());
+				auto h = ObjectReader::fromStringRef<VersionedBTree::BTreeCommitHeader>(m, Unversioned());
 				root = h.root;
 				// Copy the BTreeNodeLink but keep the same arena and BTreeNodeLinkRef
 				snapshot->extra = new BTreeNodeLink(h.root, h.root.arena());
@@ -7561,7 +7581,7 @@ class VersionedBTree {
 	}
 };
 
-#include "fdbserver/art_impl.h"
+#include "art_impl.h"
 
 RedwoodRecordRef VersionedBTree::dbBegin(""_sr);
 RedwoodRecordRef VersionedBTree::dbEnd("\xff\xff\xff\xff\xff"_sr);
@@ -7606,17 +7626,16 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 
 	Future<Void> init() override { return m_init; }
 
-	ACTOR Future<Void> init_impl(KeyValueStoreRedwood* self) {
+	Future<Void> init_impl(KeyValueStoreRedwood* self) {
 		TraceEvent(SevInfo, "RedwoodInit").detail("FilePrefix", self->m_filename);
-		wait(self->m_tree->init());
+		co_await self->m_tree->init();
 		TraceEvent(SevInfo, "RedwoodInitComplete")
 		    .detail("Filename", self->m_filename)
 		    .detail("Version", self->m_tree->getLastCommittedVersion());
 		self->m_nextCommitVersion = self->m_tree->getLastCommittedVersion() + 1;
-		return Void();
 	}
 
-	ACTOR void shutdown(KeyValueStoreRedwood* self, bool dispose) {
+	Future<Void> shutdown(Uncancellable, KeyValueStoreRedwood* self, bool dispose) {
 		TraceEvent(SevInfo, "RedwoodShutdown").detail("Filename", self->m_filename).detail("Dispose", dispose);
 
 		g_redwoodMetrics.ioLock = nullptr;
@@ -7625,10 +7644,10 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 		if (g_network->isSimulated() && dispose && BUGGIFY) {
 			// Only proceed if the last commit is a success, but don't throw if it's not because shutdown
 			// should not throw.
-			wait(ready(self->m_lastCommit));
+			co_await ready(self->m_lastCommit);
 			if (!self->getErrorNoDelay().isReady()) {
 				// Run the destructive sanity check, but don't throw.
-				ErrorOr<Void> err = wait(errorOr(self->m_tree->clearAllAndCheckSanity()));
+				ErrorOr<Void> err = co_await errorOr(self->m_tree->clearAllAndCheckSanity());
 				// If the test threw an error, it must be an injected fault or something has gone wrong.
 				ASSERT(!err.isError() || err.getError().isInjectedFault() ||
 				       err.getError().code() == error_code_unexpected_encoding_type);
@@ -7647,7 +7666,7 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 			self->m_tree->dispose();
 		else
 			self->m_tree->close();
-		wait(closedFuture);
+		co_await closedFuture;
 
 		Promise<Void> closedPromise = self->m_closed;
 		TraceEvent(SevInfo, "RedwoodShutdownComplete").detail("Filename", self->m_filename).detail("Dispose", dispose);
@@ -7655,9 +7674,9 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 		closedPromise.send(Void());
 	}
 
-	void close() override { shutdown(this, false); }
+	void close() override { shutdown(Uncancellable(), this, false); }
 
-	void dispose() override { shutdown(this, true); }
+	void dispose() override { shutdown(Uncancellable(), this, true); }
 
 	Future<Void> onClosed() const override { return m_closed.getFuture(); }
 
@@ -7695,28 +7714,28 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 		return catchError(readRange_impl(this, keys, rowLimit, byteLimit, options));
 	}
 
-	ACTOR static Future<RangeResult> readRange_impl(KeyValueStoreRedwood* self,
-	                                                KeyRange keys,
-	                                                int rowLimit,
-	                                                int byteLimit,
-	                                                Optional<ReadOptions> options) {
-		state PagerEventReasons reason = PagerEventReasons::RangeRead;
-		state VersionedBTree::BTreeCursor cur;
+	static Future<RangeResult> readRange_impl(KeyValueStoreRedwood* self,
+	                                          KeyRange keys,
+	                                          int rowLimit,
+	                                          int byteLimit,
+	                                          Optional<ReadOptions> options) {
+		PagerEventReasons reason = PagerEventReasons::RangeRead;
+		VersionedBTree::BTreeCursor cur;
 		if (options.present() && options.get().type == ReadType::FETCH) {
 			reason = PagerEventReasons::FetchRange;
 		}
-		wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion(), reason, options));
+		co_await self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion(), reason, options);
 
-		state PriorityMultiLock::Lock lock;
-		state Future<Void> f;
+		PriorityMultiLock::Lock lock;
+		Future<Void> f;
 		++g_redwoodMetrics.metric.opGetRange;
 
-		state RangeResult result;
-		state int accumulatedBytes = 0;
+		RangeResult result;
+		int accumulatedBytes = 0;
 		ASSERT(byteLimit > 0);
 
 		if (rowLimit == 0) {
-			return result;
+			co_return result;
 		}
 
 		if (rowLimit > 0) {
@@ -7726,7 +7745,7 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 				f.get();
 			} else {
 				CODE_PROBE(true, "Uncached forward range read seek");
-				wait(f);
+				co_await f;
 			}
 
 			if (self->prefetch) {
@@ -7772,7 +7791,7 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 					break;
 				}
 				cur.popPath();
-				wait(cur.moveNext());
+				co_await cur.moveNext();
 			}
 		} else {
 			f = cur.seekLT(keys.end);
@@ -7781,7 +7800,7 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 				f.get();
 			} else {
 				CODE_PROBE(true, "Uncached reverse range read seek");
-				wait(f);
+				co_await f;
 			}
 
 			if (self->prefetch) {
@@ -7827,50 +7846,54 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 					break;
 				}
 				cur.popPath();
-				wait(cur.movePrev());
+				co_await cur.movePrev();
 			}
 		}
 
 		result.more = rowLimit == 0 || accumulatedBytes >= byteLimit;
 		g_redwoodMetrics.kvSizeReadByGetRange->sample(accumulatedBytes);
-		return result;
+		co_return result;
 	}
 
-	ACTOR static Future<Optional<Value>> readValue_impl(KeyValueStoreRedwood* self,
-	                                                    Key key,
-	                                                    Optional<ReadOptions> options) {
-		state VersionedBTree::BTreeCursor cur;
-		wait(self->m_tree->initBTreeCursor(
-		    &cur, self->m_tree->getLastCommittedVersion(), PagerEventReasons::PointRead, options));
+	static Future<Optional<Value>> readValue_impl(KeyValueStoreRedwood* self, Key key, Optional<ReadOptions> options) {
+		VersionedBTree::BTreeCursor cur;
+		co_await self->m_tree->initBTreeCursor(
+		    &cur, self->m_tree->getLastCommittedVersion(), PagerEventReasons::PointRead, options);
 
 		++g_redwoodMetrics.metric.opGet;
-		wait(cur.seekGTE(key));
+		co_await cur.seekGTE(key);
 		if (cur.isValid() && cur.get().key == key) {
 			// Return a Value whose arena depends on the source page arena
 			Value v;
 			v.arena().dependsOn(cur.back().page->getArena());
 			v.contents() = cur.get().value.get();
 			g_redwoodMetrics.kvSizeReadByGet->sample(cur.get().kvBytes());
-			return v;
+			co_return v;
 		}
 
-		return Optional<Value>();
+		co_return Optional<Value>();
 	}
 
 	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
 		return catchError(readValue_impl(this, key, options));
 	}
 
+	static Future<Optional<Value>> readValuePrefix_impl(KeyValueStoreRedwood* self,
+	                                                    Key key,
+	                                                    int maxLength,
+	                                                    Optional<ReadOptions> options) {
+		Optional<Value> v = co_await readValue_impl(self, key, options);
+		if (v.present() && v.get().size() > maxLength) {
+			v.get().contents() = v.get().substr(0, maxLength);
+		}
+		co_return v;
+	}
+
 	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
-		return catchError(map(readValue_impl(this, key, options), [maxLength](Optional<Value> v) {
-			if (v.present() && v.get().size() > maxLength) {
-				v.get().contents() = v.get().substr(0, maxLength);
-			}
-			return v;
-		}));
+		return catchError(readValuePrefix_impl(this, key, maxLength, options));
 	}
 
-	~KeyValueStoreRedwood() override {};
+	~KeyValueStoreRedwood() override = default;
 
 private:
 	std::string m_filename;
@@ -7936,42 +7959,42 @@ KeyValue randomKV(int maxKeySize = 10, int maxValueSize = 5) {
 
 // Verify a range using a BTreeCursor.
 // Assumes that the BTree holds a single data version and the version is 0.
-ACTOR Future<Void> verifyRangeBTreeCursor(VersionedBTree* btree,
-                                          Key start,
-                                          Key end,
-                                          Version v,
-                                          std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
-                                          int64_t* pRecordsRead) {
+Future<Void> verifyRangeBTreeCursor(VersionedBTree* btree,
+                                    Key start,
+                                    Key end,
+                                    Version v,
+                                    std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
+                                    int64_t* pRecordsRead) {
 	if (end <= start)
 		end = keyAfter(start);
 
-	state std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator i =
+	std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator i =
 	    written->lower_bound(std::make_pair(start.toString(), 0));
-	state std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator iEnd =
+	std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator iEnd =
 	    written->upper_bound(std::make_pair(end.toString(), initialVersion));
-	state std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator iLast;
+	std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator iLast;
 
-	state VersionedBTree::BTreeCursor cur;
-	wait(btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead));
+	VersionedBTree::BTreeCursor cur;
+	co_await btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead);
 	debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Start\n", v, start.printable().c_str(), end.printable().c_str());
 
 	// Randomly use the cursor for something else first.
 	if (deterministicRandom()->coinflip()) {
-		state Key randomKey = randomKV().key;
+		Key randomKey = randomKV().key;
 		debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Dummy seek to '%s'\n",
 		             v,
 		             start.printable().c_str(),
 		             end.printable().c_str(),
 		             randomKey.toString().c_str());
-		wait(success(cur.seek(randomKey)));
+		co_await cur.seek(randomKey);
 	}
 
 	debug_printf(
 	    "VerifyRange(@%" PRId64 ", %s, %s): Actual seek\n", v, start.printable().c_str(), end.printable().c_str());
 
-	wait(cur.seekGTE(start));
+	co_await cur.seekGTE(start);
 
-	state Standalone<VectorRef<KeyValueRef>> results;
+	Standalone<VectorRef<KeyValueRef>> results;
 
 	while (cur.isValid() && cur.get().key < end) {
 		// Find the next written kv pair that would be present at this version
@@ -8028,7 +8051,7 @@ ACTOR Future<Void> verifyRangeBTreeCursor(VersionedBTree* btree,
 		results.arena().dependsOn(cur.back().cursor.cache->arena);
 		results.arena().dependsOn(cur.back().page->getArena());
 
-		wait(cur.moveNext());
+		co_await cur.moveNext();
 	}
 
 	// Make sure there are no further written kv pairs that would be present at this version.
@@ -8059,13 +8082,13 @@ ACTOR Future<Void> verifyRangeBTreeCursor(VersionedBTree* btree,
 	// opening new cursors
 	if (v >= btree->getOldestReadableVersion() && deterministicRandom()->coinflip()) {
 		cur = VersionedBTree::BTreeCursor();
-		wait(btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead));
+		co_await btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead);
 	}
 
 	// Now read the range from the tree in reverse order and compare to the saved results
-	wait(cur.seekLT(end));
+	co_await cur.seekLT(end);
 
-	state std::reverse_iterator<const KeyValueRef*> r = results.rbegin();
+	std::reverse_iterator<const KeyValueRef*> r = results.rbegin();
 
 	while (cur.isValid() && cur.get().key >= start) {
 		++*pRecordsRead;
@@ -8100,7 +8123,7 @@ ACTOR Future<Void> verifyRangeBTreeCursor(VersionedBTree* btree,
 		}
 
 		++r;
-		wait(cur.movePrev());
+		co_await cur.movePrev();
 	}
 
 	if (r != results.rend()) {
@@ -8111,31 +8134,29 @@ ACTOR Future<Void> verifyRangeBTreeCursor(VersionedBTree* btree,
 		       r->key.toString().c_str());
 		ASSERT(false);
 	}
-
-	return Void();
 }
 
 // Verify the result of point reads for every set or cleared key change made at exactly v
-ACTOR Future<Void> seekAllBTreeCursor(VersionedBTree* btree,
-                                      Version v,
-                                      std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
-                                      int64_t* pRecordsRead) {
-	state std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator i = written->cbegin();
-	state std::map<std::pair<std::string, Version>, Optional<std::string>>::const_iterator iEnd = written->cend();
-	state VersionedBTree::BTreeCursor cur;
+Future<Void> seekAllBTreeCursor(VersionedBTree* btree,
+                                Version v,
+                                std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
+                                int64_t* pRecordsRead) {
+	auto i = written->cbegin();
+	auto iEnd = written->cend();
+	VersionedBTree::BTreeCursor cur;
 
-	wait(btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead));
+	co_await btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead);
 
 	while (i != iEnd) {
 		// Since the written map gets larger and takes longer to scan each time, count visits to all written recs
 		++*pRecordsRead;
-		state std::string key = i->first.first;
-		state Version ver = i->first.second;
+		std::string key = i->first.first;
+		Version ver = i->first.second;
 		if (ver == v) {
-			state Optional<std::string> val = i->second;
+			Optional<std::string> val = i->second;
 			debug_printf("Verifying @%" PRId64 " '%s'\n", ver, key.c_str());
-			state Arena arena;
-			wait(cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key))));
+			Arena arena;
+			co_await cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key)));
 			bool foundKey = cur.isValid() && cur.get().key == key;
 			bool hasValue = foundKey && cur.get().value.present();
 
@@ -8171,21 +8192,20 @@ ACTOR Future<Void> seekAllBTreeCursor(VersionedBTree* btree,
 		}
 		++i;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> verify(VersionedBTree* btree,
-                          FutureStream<Version> vStream,
-                          std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
-                          int64_t* pRecordsRead,
-                          bool serial) {
+Future<Void> verify(VersionedBTree* btree,
+                    FutureStream<Version> vStream,
+                    std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
+                    int64_t* pRecordsRead,
+                    bool serial) {
 
 	// Queue of committed versions still readable from btree
-	state std::deque<Version> committedVersions;
+	std::deque<Version> committedVersions;
 
 	try {
-		loop {
-			state Version v = waitNext(vStream);
+		while (true) {
+			Version v = co_await vStream;
 			committedVersions.push_back(v);
 
 			// Remove expired versions
@@ -8205,14 +8225,13 @@ ACTOR Future<Void> verify(VersionedBTree* btree,
 			debug_printf("Using committed version %" PRId64 "\n", v);
 
 			// Get a cursor at v so that v doesn't get expired between the possibly serial steps below.
-			state VersionedBTree::BTreeCursor cur;
-			wait(btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead));
+			VersionedBTree::BTreeCursor cur;
+			co_await btree->initBTreeCursor(&cur, v, PagerEventReasons::RangeRead);
 
 			debug_printf("Verifying entire key range at version %" PRId64 "\n", v);
-			state Future<Void> fRangeAll =
-			    verifyRangeBTreeCursor(btree, ""_sr, "\xff\xff"_sr, v, written, pRecordsRead);
+			Future<Void> fRangeAll = verifyRangeBTreeCursor(btree, ""_sr, "\xff\xff"_sr, v, written, pRecordsRead);
 			if (serial) {
-				wait(fRangeAll);
+				co_await fRangeAll;
 			}
 
 			Key begin = randomKV().key;
@@ -8220,18 +8239,18 @@ ACTOR Future<Void> verify(VersionedBTree* btree,
 
 			debug_printf(
 			    "Verifying range (%s, %s) at version %" PRId64 "\n", toString(begin).c_str(), toString(end).c_str(), v);
-			state Future<Void> fRangeRandom = verifyRangeBTreeCursor(btree, begin, end, v, written, pRecordsRead);
+			Future<Void> fRangeRandom = verifyRangeBTreeCursor(btree, begin, end, v, written, pRecordsRead);
 			if (serial) {
-				wait(fRangeRandom);
+				co_await fRangeRandom;
 			}
 
 			debug_printf("Verifying seeks to each changed key at version %" PRId64 "\n", v);
-			state Future<Void> fSeekAll = seekAllBTreeCursor(btree, v, written, pRecordsRead);
+			Future<Void> fSeekAll = seekAllBTreeCursor(btree, v, written, pRecordsRead);
 			if (serial) {
-				wait(fSeekAll);
+				co_await fSeekAll;
 			}
 
-			wait(fRangeAll && fRangeRandom && fSeekAll);
+			co_await (fRangeAll && fRangeRandom && fSeekAll);
 
 			printf("Verified version %" PRId64 "\n", v);
 		}
@@ -8240,37 +8259,40 @@ ACTOR Future<Void> verify(VersionedBTree* btree,
 			throw;
 		}
 	}
-	return Void();
 }
 
 // Does a random range read, doesn't trap/report errors
-ACTOR Future<Void> randomReader(VersionedBTree* btree, int64_t* pRecordsRead) {
-	state VersionedBTree::BTreeCursor cur;
+Future<Void> randomReader(VersionedBTree* btree, int64_t* pRecordsRead) {
+	VersionedBTree::BTreeCursor cur;
 
-	loop {
-		wait(yield());
+	while (true) {
+		co_await yield();
 		if (!cur.initialized() || deterministicRandom()->random01() > .01) {
-			wait(btree->initBTreeCursor(&cur, btree->getLastCommittedVersion(), PagerEventReasons::RangeRead));
+			co_await btree->initBTreeCursor(&cur, btree->getLastCommittedVersion(), PagerEventReasons::RangeRead);
 		}
 
-		state KeyValue kv = randomKV(10, 0);
-		wait(cur.seekGTE(kv.key));
-		state int c = deterministicRandom()->randomInt(0, 100);
-		state bool direction = deterministicRandom()->coinflip();
+		KeyValue kv = randomKV(10, 0);
+		co_await cur.seekGTE(kv.key);
+		int c = deterministicRandom()->randomInt(0, 100);
+		bool direction = deterministicRandom()->coinflip();
 		while (cur.isValid() && c-- > 0) {
 			++*pRecordsRead;
-			wait(success(direction ? cur.moveNext() : cur.movePrev()));
-			wait(yield());
+			if (direction) {
+				co_await cur.moveNext();
+			} else {
+				co_await cur.movePrev();
+			}
+			co_await yield();
 		}
 	}
 }
 
 struct IntIntPair {
-	IntIntPair() {}
+	IntIntPair() = default;
 	IntIntPair(int k, int v) : k(k), v(v) {}
 	IntIntPair(Arena& arena, const IntIntPair& toCopy) { *this = toCopy; }
 
-	typedef IntIntPair Partial;
+	using Partial = IntIntPair;
 
 	void updateCache(Optional<Partial> cache, Arena& arena) const {}
 	struct Delta {
@@ -8713,7 +8735,7 @@ TEST_CASE("Lredwood/correctness/unit/deltaTree/RedwoodRecordRef") {
 
 	int bufferSize = N * 100;
 	bool largeTree = bufferSize > DeltaTree<RedwoodRecordRef>::SmallSizeLimit;
-	DeltaTree<RedwoodRecordRef>* tree = (DeltaTree<RedwoodRecordRef>*)new uint8_t[bufferSize];
+	auto* tree = (DeltaTree<RedwoodRecordRef>*)new uint8_t[bufferSize];
 
 	tree->build(bufferSize, &items[0], &items[items.size()], &prev, &next);
 
@@ -8890,7 +8912,7 @@ TEST_CASE("Lredwood/correctness/unit/deltaTree/RedwoodRecordRef2") {
 
 	int bufferSize = N * 100;
 	bool largeTree = bufferSize > DeltaTree2<RedwoodRecordRef>::SmallSizeLimit;
-	DeltaTree2<RedwoodRecordRef>* tree = (DeltaTree2<RedwoodRecordRef>*)new uint8_t[bufferSize];
+	auto* tree = (DeltaTree2<RedwoodRecordRef>*)new uint8_t[bufferSize];
 
 	tree->build(bufferSize, &items[0], &items[items.size()], &prev, &next);
 
@@ -9072,12 +9094,12 @@ TEST_CASE("Lredwood/correctness/unit/deltaTree/IntIntPair") {
 	std::vector<IntIntPair> items(uniqueItems.begin(), uniqueItems.end());
 	int bufferSize = N * 2 * 30;
 
-	DeltaTree<IntIntPair>* tree = (DeltaTree<IntIntPair>*)new uint8_t[bufferSize];
+	auto* tree = (DeltaTree<IntIntPair>*)new uint8_t[bufferSize];
 	int builtSize = tree->build(bufferSize, &items[0], &items[items.size()], &lowerBound, &upperBound);
 	ASSERT(builtSize <= bufferSize);
 	DeltaTree<IntIntPair>::Mirror r(tree, &lowerBound, &upperBound);
 
-	DeltaTree2<IntIntPair>* tree2 = (DeltaTree2<IntIntPair>*)new uint8_t[bufferSize];
+	auto* tree2 = (DeltaTree2<IntIntPair>*)new uint8_t[bufferSize];
 	int builtSize2 = tree2->build(bufferSize, &items[0], &items[0] + items.size(), &lowerBound, &upperBound);
 	ASSERT(builtSize2 <= bufferSize);
 	auto cache = makeReference<DeltaTree2<IntIntPair>::DecodeCache>(lowerBound, upperBound);
@@ -9619,7 +9641,7 @@ TEST_CASE(":/redwood/performance/mutationBuffer") {
 TEST_CASE(":/redwood/pager/ArenaPage") {
 	Arena x;
 	printf("Making p\n");
-	Reference<ArenaPage> p = makeReference<ArenaPage>(4096, 4096);
+	auto p = makeReference<ArenaPage>(4096, 4096);
 	printf("Made p=%p\n", p->data());
 	printf("Clearing p\n");
 	p.clear();
@@ -9674,6 +9696,57 @@ double getExternalTimeoutThreshold(const UnitTestParameters& params) {
 	return ret;
 }
 
+Future<Void> commitAndReportCorrectnessProgress(
+    VersionedBTree* btree,
+    Version version,
+    int64_t* totalPageOps,
+    int64_t maxPageOps,
+    std::map<std::pair<std::string, Version>, Optional<std::string>>* written,
+    int maxVerificationMapEntries,
+    int64_t* totalRecordsRead,
+    int64_t maxRecordsRead,
+    PromiseStream<Version> committedVersions) {
+	co_await btree->commit(version);
+
+	// Update pager ops before clearing metrics
+	*totalPageOps += g_redwoodMetrics.pageOps();
+	fmt::print("Committed {0} PageOps {1}/{2} ({3:.2f}%)  VerificationMapEntries {4}/{5} ({6:.2f}%)  "
+	           "RecordsRead {7}/{8} ({9:.2f}%)\n",
+	           toString(version).c_str(),
+	           *totalPageOps,
+	           maxPageOps,
+	           *totalPageOps * 100.0 / maxPageOps,
+	           written->size(),
+	           maxVerificationMapEntries,
+	           written->size() * 100.0 / maxVerificationMapEntries,
+	           *totalRecordsRead,
+	           maxRecordsRead,
+	           *totalRecordsRead * 100.0 / maxRecordsRead);
+	printf("Committed:\n%s\n", g_redwoodMetrics.toString(true).c_str());
+
+	// Notify the background verifier that version is committed and therefore readable
+	committedVersions.send(version);
+}
+
+Future<Void> commitAndReportLoadProgress(VersionedBTree* btree,
+                                         Version version,
+                                         bool traceMetrics,
+                                         int64_t keyValueBytes,
+                                         int records,
+                                         double* intervalStart) {
+	co_await btree->commit(version);
+	if (!traceMetrics) {
+		printf("%s\n", g_redwoodMetrics.toString(true).c_str());
+	}
+	double elapsed = timer() - *intervalStart;
+	printf("Committed %" PRId64 " keyValueBytes in %d records in %f seconds, %.2f MB/s\n",
+	       keyValueBytes,
+	       records,
+	       elapsed,
+	       keyValueBytes / elapsed / 1e6);
+	*intervalStart = timer();
+}
+
 } // namespace
 
 TEST_CASE("Lredwood/correctness/btree") {
@@ -9983,27 +10056,15 @@ TEST_CASE("Lredwood/correctness/btree") {
 				        0, btree->getLastCommittedVersion() - btree->getOldestReadableVersion() + 1));
 			}
 
-			commit = map(btree->commit(version), [&, v = version](Void) {
-				// Update pager ops before clearing metrics
-				totalPageOps += g_redwoodMetrics.pageOps();
-				fmt::print("Committed {0} PageOps {1}/{2} ({3:.2f}%)  VerificationMapEntries {4}/{5} ({6:.2f}%)  "
-				           "RecordsRead {7}/{8} ({9:.2f}%)\n",
-				           toString(v).c_str(),
-				           totalPageOps,
-				           maxPageOps,
-				           totalPageOps * 100.0 / maxPageOps,
-				           written.size(),
-				           maxVerificationMapEntries,
-				           written.size() * 100.0 / maxVerificationMapEntries,
-				           totalRecordsRead,
-				           maxRecordsRead,
-				           totalRecordsRead * 100.0 / maxRecordsRead);
-				printf("Committed:\n%s\n", g_redwoodMetrics.toString(true).c_str());
-
-				// Notify the background verifier that version is committed and therefore readable
-				committedVersions.send(v);
-				return Void();
-			});
+			commit = commitAndReportCorrectnessProgress(btree,
+			                                            version,
+			                                            &totalPageOps,
+			                                            maxPageOps,
+			                                            &written,
+			                                            maxVerificationMapEntries,
+			                                            &totalRecordsRead,
+			                                            maxRecordsRead,
+			                                            committedVersions);
 			++commitOps;
 
 			if (serialTest) {
@@ -10110,54 +10171,53 @@ TEST_CASE("Lredwood/correctness/btree") {
 	return Void();
 }
 
-ACTOR Future<Void> randomSeeks(VersionedBTree* btree,
-                               Optional<Version> v,
-                               bool reInitCursor,
-                               int count,
-                               char firstChar,
-                               char lastChar,
-                               int keyLen) {
-	state int c = 0;
-	state double readStart = timer();
-	state VersionedBTree::BTreeCursor cur;
-	state Version readVer = v.orDefault(btree->getLastCommittedVersion());
-	wait(btree->initBTreeCursor(&cur, readVer, PagerEventReasons::PointRead));
+Future<Void> randomSeeks(VersionedBTree* btree,
+                         Optional<Version> v,
+                         bool reInitCursor,
+                         int count,
+                         char firstChar,
+                         char lastChar,
+                         int keyLen) {
+	int c = 0;
+	double readStart = timer();
+	VersionedBTree::BTreeCursor cur;
+	Version readVer = v.orDefault(btree->getLastCommittedVersion());
+	co_await btree->initBTreeCursor(&cur, readVer, PagerEventReasons::PointRead);
 
 	while (c < count) {
 		if (!v.present()) {
 			readVer = btree->getLastCommittedVersion();
 		}
 		if (reInitCursor) {
-			wait(btree->initBTreeCursor(&cur, readVer, PagerEventReasons::PointRead));
+			co_await btree->initBTreeCursor(&cur, readVer, PagerEventReasons::PointRead);
 		}
 
-		state Key k = randomString(keyLen, firstChar, lastChar);
-		wait(cur.seekGTE(k));
+		Key k = randomString(keyLen, firstChar, lastChar);
+		co_await cur.seekGTE(k);
 		++c;
 	}
 	double elapsed = timer() - readStart;
 	printf("Random seek speed %d/s\n", int(count / elapsed));
-	return Void();
 }
 
-ACTOR Future<Void> randomScans(VersionedBTree* btree,
-                               int count,
-                               int width,
-                               int prefetchBytes,
-                               char firstChar,
-                               char lastChar) {
-	state Version readVer = btree->getLastCommittedVersion();
-	state int c = 0;
-	state double readStart = timer();
-	state VersionedBTree::BTreeCursor cur;
-	wait(btree->initBTreeCursor(&cur, readVer, PagerEventReasons::RangeRead));
-
-	state int totalScanBytes = 0;
+Future<Void> randomScans(VersionedBTree* btree,
+                         int count,
+                         int width,
+                         int prefetchBytes,
+                         char firstChar,
+                         char lastChar) {
+	Version readVer = btree->getLastCommittedVersion();
+	int c = 0;
+	double readStart = timer();
+	VersionedBTree::BTreeCursor cur;
+	co_await btree->initBTreeCursor(&cur, readVer, PagerEventReasons::RangeRead);
+
+	int totalScanBytes = 0;
 	while (c++ < count) {
-		state Key k = randomString(20, firstChar, lastChar);
-		wait(cur.seekGTE(k));
-		state int w = width;
-		state bool directionFwd = deterministicRandom()->coinflip();
+		Key k = randomString(20, firstChar, lastChar);
+		co_await cur.seekGTE(k);
+		int w = width;
+		bool directionFwd = deterministicRandom()->coinflip();
 
 		if (prefetchBytes > 0) {
 			cur.prefetch(directionFwd ? VersionedBTree::dbEnd.key : VersionedBTree::dbBegin.key,
@@ -10168,7 +10228,11 @@ ACTOR Future<Void> randomScans(VersionedBTree* btree,
 
 		while (w > 0 && cur.isValid()) {
 			totalScanBytes += cur.get().expectedSize();
-			wait(success(directionFwd ? cur.moveNext() : cur.movePrev()));
+			if (directionFwd) {
+				co_await cur.moveNext();
+			} else {
+				co_await cur.movePrev();
+			}
 			--w;
 		}
 	}
@@ -10181,7 +10245,6 @@ ACTOR Future<Void> randomScans(VersionedBTree* btree,
 	    prefetchBytes,
 	    int(count / elapsed),
 	    double(totalScanBytes) / 1e6 / elapsed);
-	return Void();
 }
 
 template <int size>
@@ -10195,7 +10258,7 @@ struct ExtentQueueEntry {
 	}
 };
 
-typedef FIFOQueue<ExtentQueueEntry<16>> ExtentQueueT;
+using ExtentQueueT = FIFOQueue<ExtentQueueEntry<16>>;
 TEST_CASE(":/redwood/performance/extentQueue") {
 	state ExtentQueueT m_extentQueue;
 	state ExtentQueueT::QueueState extentQueueState;
@@ -10485,27 +10548,11 @@ TEST_CASE(":/redwood/performance/set") {
 				       kvBytesTotal / 1e6,
 				       kvBytesTotal / (timer() - start) / 1e6);
 
-				// Avoid capturing via this to freeze counter values
-				int recs = recordsThisCommit;
-				int kvb = kvBytesThisCommit;
-
-				// Capturing invervalStart via this->intervalStart makes IDE's unhappy as they do not know about the
-				// actor state object
+				// Pass intervalStart by pointer to avoid capturing actor state in the commit coroutine.
 				double* pIntervalStart = &intervalStart;
 
-				commit = map(btree->commit(++version), [=](Void result) {
-					if (!traceMetrics) {
-						printf("%s\n", g_redwoodMetrics.toString(true).c_str());
-					}
-					double elapsed = timer() - *pIntervalStart;
-					printf("Committed %d keyValueBytes in %d records in %f seconds, %.2f MB/s\n",
-					       kvb,
-					       recs,
-					       elapsed,
-					       kvb / elapsed / 1e6);
-					*pIntervalStart = timer();
-					return Void();
-				});
+				commit = commitAndReportLoadProgress(
+				    btree, ++version, traceMetrics, kvBytesThisCommit, recordsThisCommit, pIntervalStart);
 
 				kvBytesTotal += kvBytesThisCommit;
 				kvBytesThisCommit = 0;
@@ -10520,9 +10567,7 @@ TEST_CASE(":/redwood/performance/set") {
 		printf("StorageBytes=%s\n", btree->getStorageBytes().toString().c_str());
 	}
 
-	state Future<Void> stats =
-	    traceMetrics ? Void()
-	                 : recurring([&]() { printf("Stats:\n%s\n", g_redwoodMetrics.toString(true).c_str()); }, 1.0);
+	state Future<Void> stats = traceMetrics ? Void() : recurring(std::bind_front(&printRedwoodStats), 1.0);
 
 	if (scans > 0) {
 		printf("Parallel scans, concurrency=%d, scans=%d, scanWidth=%d, scanPreftchBytes=%d ...\n",
@@ -10572,10 +10617,10 @@ struct PrefixSegment {
 // Utility class for generating kv pairs under a prefix pattern
 // It currently uses std::string in an abstraction breaking way.
 struct KVSource {
-	KVSource() {}
+	KVSource() = default;
 
-	typedef VectorRef<uint8_t> PrefixRef;
-	typedef Standalone<PrefixRef> Prefix;
+	using PrefixRef = VectorRef<uint8_t>;
+	using Prefix = Standalone<PrefixRef>;
 
 	std::vector<PrefixSegment> desc;
 	std::vector<std::vector<std::string>> segments;
@@ -10587,7 +10632,7 @@ struct KVSource {
 	// TODO there is probably a better way to do this
 	Prefix extraRangePrefix;
 
-	KVSource(const std::vector<PrefixSegment>& desc, int numPrefixes = 0) : desc(desc) {
+	explicit(false) KVSource(const std::vector<PrefixSegment>& desc, int numPrefixes = 0) : desc(desc) {
 		if (numPrefixes == 0) {
 			numPrefixes = 1;
 			for (auto& p : desc) {
@@ -10690,12 +10735,12 @@ struct KVSource {
 	}
 };
 
-ACTOR Future<StorageBytes> getStableStorageBytes(IKeyValueStore* kvs) {
-	state StorageBytes sb = kvs->getStorageBytes();
+Future<StorageBytes> getStableStorageBytes(IKeyValueStore* kvs) {
+	StorageBytes sb = kvs->getStorageBytes();
 
 	// Wait for StorageBytes used metric to stabilize
-	loop {
-		wait(kvs->commit());
+	while (true) {
+		co_await kvs->commit();
 		StorageBytes sb2 = kvs->getStorageBytes();
 		bool stable = sb2.used == sb.used;
 		sb = sb2;
@@ -10704,21 +10749,21 @@ ACTOR Future<StorageBytes> getStableStorageBytes(IKeyValueStore* kvs) {
 		}
 	}
 
-	return sb;
+	co_return sb;
 }
 
-ACTOR Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
-                                         int suffixSize,
-                                         int valueSize,
-                                         KVSource source,
-                                         int recordCountTarget,
-                                         bool usePrefixesInOrder,
-                                         bool clearAfter) {
-	state int commitTarget = 5e6;
+Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
+                                   int suffixSize,
+                                   int valueSize,
+                                   KVSource source,
+                                   int recordCountTarget,
+                                   bool usePrefixesInOrder,
+                                   bool clearAfter) {
+	int commitTarget = 5e6;
 
-	state int recordSize = source.prefixLen + suffixSize + valueSize;
-	state int64_t kvBytesTarget = (int64_t)recordCountTarget * recordSize;
-	state int recordsPerPrefix = recordCountTarget / source.numPrefixes();
+	int recordSize = source.prefixLen + suffixSize + valueSize;
+	int64_t kvBytesTarget = (int64_t)recordCountTarget * recordSize;
+	int recordsPerPrefix = recordCountTarget / source.numPrefixes();
 
 	fmt::print("\nstoreType: {}\n", static_cast<int>(kvs->getType()));
 	fmt::print("commitTarget: {}\n", commitTarget);
@@ -10731,18 +10776,18 @@ ACTOR Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
 	fmt::print("recordCountTarget: {}\n", recordCountTarget);
 	fmt::print("kvBytesTarget: {}\n", kvBytesTarget);
 
-	state int64_t kvBytes = 0;
-	state int64_t kvBytesTotal = 0;
-	state int records = 0;
-	state Future<Void> commit = Void();
-	state std::string value = deterministicRandom()->randomAlphaNumeric(1e6);
+	int64_t kvBytes = 0;
+	int64_t kvBytesTotal = 0;
+	int records = 0;
+	Future<Void> commit = Void();
+	std::string value = deterministicRandom()->randomAlphaNumeric(1e6);
 
-	wait(kvs->init());
+	co_await kvs->init();
 
-	state double intervalStart = timer();
-	state double start = intervalStart;
+	double intervalStart = timer();
+	double start = intervalStart;
 
-	state std::function<void()> stats = [&]() {
+	std::function<void()> stats = [&]() {
 		double elapsed = timer() - start;
 		printf("Cumulative stats: %.2f seconds  %.2f MB keyValue bytes  %d records  %.2f MB/s  %.2f rec/s\r",
 		       elapsed,
@@ -10754,17 +10799,16 @@ ACTOR Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
 	};
 
 	while (kvBytesTotal < kvBytesTarget) {
-		wait(yield());
+		co_await yield();
 
-		state int i;
-		for (i = 0; i < recordsPerPrefix; ++i) {
+		for (int i = 0; i < recordsPerPrefix; ++i) {
 			KeyValueRef kv(source.getAnotherKeyRef(suffixSize, usePrefixesInOrder), source.getValue(valueSize));
 			kvs->set(kv);
 			kvBytes += kv.expectedSize();
 			++records;
 
 			if (kvBytes >= commitTarget) {
-				wait(commit);
+				co_await commit;
 				stats();
 				commit = kvs->commit();
 				kvBytesTotal += kvBytes;
@@ -10779,7 +10823,7 @@ ACTOR Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
 		source.nextPrefix();
 	}
 
-	wait(commit);
+	co_await commit;
 	// TODO is it desired that not all records are committed? This could commit again to ensure any records set()
 	// since the last commit are persisted. For the purposes of how this is used currently, I don't think it matters
 	// though
@@ -10787,28 +10831,26 @@ ACTOR Future<Void> prefixClusteredInsert(IKeyValueStore* kvs,
 	printf("\n");
 
 	intervalStart = timer();
-	StorageBytes sb = wait(getStableStorageBytes(kvs));
+	StorageBytes sb = co_await getStableStorageBytes(kvs);
 	printf("storageBytes: %s (stable after %.2f seconds)\n", toString(sb).c_str(), timer() - intervalStart);
 
 	if (clearAfter) {
 		printf("Clearing all keys\n");
 		intervalStart = timer();
 		kvs->clear(KeyRangeRef(""_sr, "\xff"_sr));
-		state StorageBytes sbClear = wait(getStableStorageBytes(kvs));
+		StorageBytes sbClear = co_await getStableStorageBytes(kvs);
 		printf("Cleared all keys in %.2f seconds, final storageByte: %s\n",
 		       timer() - intervalStart,
 		       toString(sbClear).c_str());
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valueSize, int recordCountTarget) {
-	state int commitTarget = 5e6;
+Future<Void> sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valueSize, int recordCountTarget) {
+	int commitTarget = 5e6;
 
-	state KVSource source({ { prefixLen, 1 } });
-	state int recordSize = source.prefixLen + sizeof(uint64_t) + valueSize;
-	state int64_t kvBytesTarget = (int64_t)recordCountTarget * recordSize;
+	KVSource source({ { prefixLen, 1 } });
+	int recordSize = source.prefixLen + sizeof(uint64_t) + valueSize;
+	int64_t kvBytesTarget = (int64_t)recordCountTarget * recordSize;
 
 	fmt::print("\nstoreType: {}\n", static_cast<int>(kvs->getType()));
 	fmt::print("commitTarget: {}\n", commitTarget);
@@ -10817,18 +10859,18 @@ ACTOR Future<Void> sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valu
 	fmt::print("recordCountTarget: {}\n", recordCountTarget);
 	fmt::print("kvBytesTarget: {}\n", kvBytesTarget);
 
-	state int64_t kvBytes = 0;
-	state int64_t kvBytesTotal = 0;
-	state int records = 0;
-	state Future<Void> commit = Void();
-	state std::string value = deterministicRandom()->randomAlphaNumeric(1e6);
+	int64_t kvBytes = 0;
+	int64_t kvBytesTotal = 0;
+	int records = 0;
+	Future<Void> commit = Void();
+	std::string value = deterministicRandom()->randomAlphaNumeric(1e6);
 
-	wait(kvs->init());
+	co_await kvs->init();
 
-	state double intervalStart = timer();
-	state double start = intervalStart;
+	double intervalStart = timer();
+	double start = intervalStart;
 
-	state std::function<void()> stats = [&]() {
+	std::function<void()> stats = [&]() {
 		double elapsed = timer() - start;
 		printf("Cumulative stats: %.2f seconds  %.2f MB keyValue bytes  %d records  %.2f MB/s  %.2f rec/s\r",
 		       elapsed,
@@ -10839,11 +10881,11 @@ ACTOR Future<Void> sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valu
 		fflush(stdout);
 	};
 
-	state uint64_t c = 0;
-	state Key key = source.getKeyRef(sizeof(uint64_t));
+	uint64_t c = 0;
+	Key key = source.getKeyRef(sizeof(uint64_t));
 
 	while (kvBytesTotal < kvBytesTarget) {
-		wait(yield());
+		co_await yield();
 		*(uint64_t*)(key.end() - sizeof(uint64_t)) = bigEndian64(c);
 		KeyValueRef kv(key, source.getValue(valueSize));
 		kvs->set(kv);
@@ -10851,7 +10893,7 @@ ACTOR Future<Void> sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valu
 		++records;
 
 		if (kvBytes >= commitTarget) {
-			wait(commit);
+			co_await commit;
 			stats();
 			commit = kvs->commit();
 			kvBytesTotal += kvBytes;
@@ -10863,11 +10905,9 @@ ACTOR Future<Void> sequentialInsert(IKeyValueStore* kvs, int prefixLen, int valu
 		++c;
 	}
 
-	wait(commit);
+	co_await commit;
 	stats();
 	printf("\n");
-
-	return Void();
 }
 
 Future<Void> closeKVS(IKeyValueStore* kvs, bool dispose = false) {
@@ -10880,28 +10920,26 @@ Future<Void> closeKVS(IKeyValueStore* kvs, bool dispose = false) {
 	return closed;
 }
 
-ACTOR Future<Void> doPrefixInsertComparison(int suffixSize,
-                                            int valueSize,
-                                            int recordCountTarget,
-                                            bool usePrefixesInOrder,
-                                            KVSource source) {
+Future<Void> doPrefixInsertComparison(int suffixSize,
+                                      int valueSize,
+                                      int recordCountTarget,
+                                      bool usePrefixesInOrder,
+                                      KVSource source) {
 
 	deleteFile("test.redwood-v1");
-	wait(delay(5));
-	state IKeyValueStore* redwood = openKVStore(KeyValueStoreType::SSD_REDWOOD_V1, "test.redwood-v1", UID(), 0);
-	wait(prefixClusteredInsert(redwood, suffixSize, valueSize, source, recordCountTarget, usePrefixesInOrder, true));
-	wait(closeKVS(redwood));
+	co_await delay(5);
+	IKeyValueStore* redwood = openKVStore(KeyValueStoreType::SSD_REDWOOD_V1, "test.redwood-v1", UID(), 0);
+	co_await prefixClusteredInsert(redwood, suffixSize, valueSize, source, recordCountTarget, usePrefixesInOrder, true);
+	co_await closeKVS(redwood);
 	printf("\n");
 
 	deleteFile("test.sqlite");
 	deleteFile("test.sqlite-wal");
-	wait(delay(5));
-	state IKeyValueStore* sqlite = openKVStore(KeyValueStoreType::SSD_BTREE_V2, "test.sqlite", UID(), 0);
-	wait(prefixClusteredInsert(sqlite, suffixSize, valueSize, source, recordCountTarget, usePrefixesInOrder, true));
-	wait(closeKVS(sqlite));
+	co_await delay(5);
+	IKeyValueStore* sqlite = openKVStore(KeyValueStoreType::SSD_BTREE_V2, "test.sqlite", UID(), 0);
+	co_await prefixClusteredInsert(sqlite, suffixSize, valueSize, source, recordCountTarget, usePrefixesInOrder, true);
+	co_await closeKVS(sqlite);
 	printf("\n");
-
-	return Void();
 }
 
 TEST_CASE(":/redwood/performance/prefixSizeComparison") {
@@ -10941,15 +10979,15 @@ TEST_CASE(":/redwood/performance/sequentialInsert") {
 }
 
 // singlePrefix forces the range read to have the start and end key with the same prefix
-ACTOR Future<Void> randomRangeScans(IKeyValueStore* kvs,
-                                    int suffixSize,
-                                    KVSource source,
-                                    int valueSize,
-                                    int recordCountTarget,
-                                    bool singlePrefix,
-                                    int rowLimit,
-                                    int byteLimit,
-                                    Optional<ReadOptions> options = Optional<ReadOptions>()) {
+Future<Void> randomRangeScans(IKeyValueStore* kvs,
+                              int suffixSize,
+                              KVSource source,
+                              int valueSize,
+                              int recordCountTarget,
+                              bool singlePrefix,
+                              int rowLimit,
+                              int byteLimit,
+                              Optional<ReadOptions> options = Optional<ReadOptions>()) {
 	fmt::print("\nstoreType: {}\n", static_cast<int>(kvs->getType()));
 	fmt::print("prefixSource: {}\n", source.toString());
 	fmt::print("suffixSize: {}\n", suffixSize);
@@ -10957,14 +10995,14 @@ ACTOR Future<Void> randomRangeScans(IKeyValueStore* kvs,
 	fmt::print("singlePrefix: {}\n", singlePrefix);
 	fmt::print("rowLimit: {}\n", rowLimit);
 
-	state int64_t recordSize = source.prefixLen + suffixSize + valueSize;
-	state int64_t bytesRead = 0;
-	state int64_t recordsRead = 0;
-	state int queries = 0;
-	state int64_t nextPrintRecords = 1e5;
+	int64_t recordSize = source.prefixLen + suffixSize + valueSize;
+	int64_t bytesRead = 0;
+	int64_t recordsRead = 0;
+	int queries = 0;
+	int64_t nextPrintRecords = 1e5;
 
-	state double start = timer();
-	state std::function<void()> stats = [&]() {
+	double start = timer();
+	std::function<void()> stats = [&]() {
 		double elapsed = timer() - start;
 		fmt::print("Cumulative stats: {0:.2f} seconds  {1} queries {2:.2f} MB {3} records  {4:.2f} qps {5:.2f} MB/s  "
 		           "{6:.2f} rec/s\r\n",
@@ -10982,7 +11020,7 @@ ACTOR Future<Void> randomRangeScans(IKeyValueStore* kvs,
 		KeyRangeRef range = source.getKeyRangeRef(singlePrefix, suffixSize);
 		int rowLim = (deterministicRandom()->randomInt(0, 2) != 0) ? rowLimit : -rowLimit;
 
-		RangeResult result = wait(kvs->readRange(range, rowLim, byteLimit, options));
+		RangeResult result = co_await kvs->readRange(range, rowLim, byteLimit, options);
 
 		recordsRead += result.size();
 		bytesRead += result.size() * recordSize;
@@ -10997,8 +11035,6 @@ ACTOR Future<Void> randomRangeScans(IKeyValueStore* kvs,
 
 	stats();
 	printf("\n");
-
-	return Void();
 }
 
 TEST_CASE(":/redwood/performance/randomRangeScans") {
diff --git a/fdbserver/VersionedBTreeDebug.cpp b/fdbserver/kvstore/VersionedBTreeDebug.cpp
similarity index 96%
rename from fdbserver/VersionedBTreeDebug.cpp
rename to fdbserver/kvstore/VersionedBTreeDebug.cpp
index 16793f9884f..4184d234eb9 100644
--- a/fdbserver/VersionedBTreeDebug.cpp
+++ b/fdbserver/kvstore/VersionedBTreeDebug.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/VersionedBTreeDebug.h"
+#include "VersionedBTreeDebug.h"
 
 // Whether to enable debug log
 static bool g_debugEnabled = true;
@@ -35,4 +35,4 @@ bool enableRedwoodDebug() {
 	return g_debugEnabled && now() >= g_debugStart && now() < g_debugEnd &&
 	       (!g_network->getLocalAddress().isValid() || !g_debugAddress.isValid() ||
 	        g_network->getLocalAddress() == g_debugAddress);
-}
\ No newline at end of file
+}
diff --git a/fdbserver/include/fdbserver/VersionedBTreeDebug.h b/fdbserver/kvstore/VersionedBTreeDebug.h
similarity index 100%
rename from fdbserver/include/fdbserver/VersionedBTreeDebug.h
rename to fdbserver/kvstore/VersionedBTreeDebug.h
diff --git a/fdbserver/include/fdbserver/art.h b/fdbserver/kvstore/art.h
similarity index 98%
rename from fdbserver/include/fdbserver/art.h
rename to fdbserver/kvstore/art.h
index 308f98c8fec..548cbde983a 100644
--- a/fdbserver/include/fdbserver/art.h
+++ b/fdbserver/kvstore/art.h
@@ -299,7 +299,7 @@ struct art_tree {
 	void add_child4(art_node4* n, art_node** ref, unsigned char c, void* child);
 
 public:
-	art_tree(Arena& a) {
+	explicit art_tree(Arena& a) {
 		this->arena = &a;
 		this->root = alloc_node(ART_NODE4); // Sentinel node. Also needed for empty key
 		this->size = 0;
@@ -322,9 +322,9 @@ struct art_iterator {
 	art_tree::art_leaf* leaf;
 
 public:
-	art_iterator(art_tree::art_leaf* l) { leaf = l; }
+	explicit art_iterator(art_tree::art_leaf* l) { leaf = l; }
 
-	art_iterator(const art_iterator& i) { leaf = i.leaf; }
+	explicit(false) art_iterator(const art_iterator& i) { leaf = i.leaf; }
 
 	art_iterator() { leaf = nullptr; }
 
diff --git a/fdbserver/include/fdbserver/art_impl.h b/fdbserver/kvstore/art_impl.h
similarity index 100%
rename from fdbserver/include/fdbserver/art_impl.h
rename to fdbserver/kvstore/art_impl.h
diff --git a/fdbserver/kvstore/include/fdbserver/kvstore/FDBExecHelper.h b/fdbserver/kvstore/include/fdbserver/kvstore/FDBExecHelper.h
new file mode 100644
index 00000000000..3c2df8712bc
--- /dev/null
+++ b/fdbserver/kvstore/include/fdbserver/kvstore/FDBExecHelper.h
@@ -0,0 +1,57 @@
+/*
+ * FDBExecHelper.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_KVSTORE_FDBEXECHELPER_H
+#define FDBSERVER_KVSTORE_FDBEXECHELPER_H
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "fdbclient/FDBTypes.h"
+#include "flow/Arena.h"
+#include "flow/flow.h"
+
+class ExecCmdValueString {
+public:
+	ExecCmdValueString() {}
+	explicit ExecCmdValueString(StringRef cmdValueString);
+
+	StringRef getBinaryPath() const;
+	VectorRef<StringRef> getBinaryArgs() const;
+	void setCmdValueString(StringRef cmdValueString);
+	StringRef getCmdValueString() const;
+	void dbgPrint() const;
+
+private:
+	void parseCmdValue();
+
+	Standalone<StringRef> cmdValueString;
+	Standalone<VectorRef<StringRef>> binaryArgs;
+	StringRef binaryPath;
+};
+
+Future<int> execHelper(ExecCmdValueString* execArg, UID snapUID, std::string folder, std::string role);
+void setDataVersion(UID uid, Version version);
+void setDataDurableVersion(UID uid, Version version);
+void printStorageVersionInfo();
+
+#endif
diff --git a/fdbserver/kvstore/include/fdbserver/kvstore/IDiskQueue.h b/fdbserver/kvstore/include/fdbserver/kvstore/IDiskQueue.h
new file mode 100644
index 00000000000..a95487fe27e
--- /dev/null
+++ b/fdbserver/kvstore/include/fdbserver/kvstore/IDiskQueue.h
@@ -0,0 +1,33 @@
+/*
+ * IDiskQueue.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_KVSTORE_IDISKQUEUE_H
+#define FDBSERVER_KVSTORE_IDISKQUEUE_H
+#pragma once
+
+#include "fdbserver/core/IDiskQueue.h"
+
+IDiskQueue* openDiskQueue(std::string basename,
+                          std::string ext,
+                          UID dbgid,
+                          DiskQueueVersion diskQueueVersion,
+                          int64_t fileSizeWarningLimit = -1);
+
+#endif
diff --git a/fdbclient/include/fdbclient/IKeyValueStore.actor.h b/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h
similarity index 67%
rename from fdbclient/include/fdbclient/IKeyValueStore.actor.h
rename to fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h
index 5ee0c4fd696..81b2eb736d8 100644
--- a/fdbclient/include/fdbclient/IKeyValueStore.actor.h
+++ b/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h
@@ -1,5 +1,5 @@
 /*
- * IKeyValueStore.actor.h
+ * IKeyValueStore.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,14 +18,10 @@
  * limitations under the License.
  */
 
+#ifndef FDBSERVER_KVSTORE_IKEYVALUESTORE_H
+#define FDBSERVER_KVSTORE_IKEYVALUESTORE_H
 #pragma once
 
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_IKEYVALUESTORE_ACTOR_G_H)
-#define FDBCLIENT_IKEYVALUESTORE_ACTOR_G_H
-#include "fdbclient/IKeyValueStore.actor.g.h"
-#elif !defined(FDBCLIENT_IKEYVALUESTORE_ACTOR_H)
-#define FDBCLIENT_IKEYVALUESTORE_ACTOR_H
-
 #include "flow/Trace.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/StorageCheckpoint.h"
@@ -33,8 +29,6 @@
 #include "fdbclient/KeyRangeMap.h"
 #include "flow/flow.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 struct CheckpointRequest {
 	const Version version; // The FDB version at which the checkpoint is created.
 	const std::vector<KeyRange> ranges; // Keyranges this checkpoint must contain.
@@ -50,10 +44,6 @@ struct CheckpointRequest {
 	  : version(version), ranges(ranges), format(format), checkpointID(id), checkpointDir(checkpointDir) {}
 };
 
-ACTOR static Future<Void> replaceRange_impl(class IKeyValueStore* self,
-                                            KeyRange range,
-                                            Standalone<VectorRef<KeyValueRef>> data);
-
 class IKeyValueStore : public IClosable {
 public:
 	virtual KeyValueStoreType getType() const = 0;
@@ -92,7 +82,19 @@ class IKeyValueStore : public IClosable {
 
 	// Replace the specified range, the default implementation ignores `blockRange` and writes the key one by one.
 	virtual Future<Void> replaceRange(KeyRange range, Standalone<VectorRef<KeyValueRef>> data) {
-		return replaceRange_impl(this, range, data);
+		int sinceYield = 0;
+		KeyRangeRef rangeRef = range;
+		if (rangeRef.empty()) {
+			co_return;
+		}
+		clear(rangeRef);
+		for (KeyValueRef const* kvItr = data.begin(); kvItr != data.end(); kvItr++) {
+			set(*kvItr);
+			if (++sinceYield > 1000) {
+				co_await yield();
+				sinceYield = 0;
+			}
+		}
 	}
 
 	// Marks a key range as active and prepares it for future read.
@@ -169,25 +171,50 @@ class IKeyValueStore : public IClosable {
 	virtual ~IKeyValueStore() {}
 };
 
-ACTOR static Future<Void> replaceRange_impl(IKeyValueStore* self,
-                                            KeyRange range,
-                                            Standalone<VectorRef<KeyValueRef>> data) {
-	state int sinceYield = 0;
-	state const KeyValueRef* kvItr = data.begin();
-	state KeyRangeRef rangeRef = range;
-	if (rangeRef.empty()) {
-		return Void();
-	}
-	self->clear(rangeRef);
-	for (; kvItr != data.end(); kvItr++) {
-		self->set(*kvItr);
-		if (++sinceYield > 1000) {
-			wait(yield());
-			sinceYield = 0;
-		}
-	}
-	return Void();
-}
-
-#include "flow/unactorcompiler.h"
+FDB_BOOLEAN_PARAM(CheckChecksums);
+FDB_BOOLEAN_PARAM(CheckIntegrity);
+FDB_BOOLEAN_PARAM(DisableSnapshot);
+FDB_BOOLEAN_PARAM(ReplaceContent);
+FDB_BOOLEAN_PARAM(ExactRecovery);
+
+extern IKeyValueStore* keyValueStoreSQLite(std::string const& filename,
+                                           UID logID,
+                                           KeyValueStoreType storeType,
+                                           CheckChecksums = CheckChecksums::False,
+                                           CheckIntegrity = CheckIntegrity::False);
+extern IKeyValueStore* keyValueStoreRedwoodV1(std::string const& filename,
+                                              UID logID,
+                                              Reference<AsyncVar<struct ServerDBInfo> const> db = {},
+                                              int64_t pageCacheBytes = 0);
+extern IKeyValueStore* keyValueStoreRocksDB(std::string const& path,
+                                            UID logID,
+                                            KeyValueStoreType storeType,
+                                            CheckChecksums = CheckChecksums::False,
+                                            CheckIntegrity = CheckIntegrity::False);
+extern IKeyValueStore* keyValueStoreShardedRocksDB(std::string const& path,
+                                                   UID logID,
+                                                   KeyValueStoreType storeType,
+                                                   CheckChecksums = CheckChecksums::False,
+                                                   CheckIntegrity = CheckIntegrity::False);
+extern IKeyValueStore* keyValueStoreMemory(std::string const& basename,
+                                           UID logID,
+                                           int64_t memoryLimit,
+                                           std::string ext = "fdq",
+                                           KeyValueStoreType storeType = KeyValueStoreType::MEMORY);
+extern IKeyValueStore* keyValueStoreLogSystem(class IDiskQueue* queue,
+                                              Reference<AsyncVar<struct ServerDBInfo> const> db,
+                                              UID logID,
+                                              int64_t memoryLimit,
+                                              DisableSnapshot,
+                                              ReplaceContent,
+                                              ExactRecovery);
+
+IKeyValueStore* openKVStore(KeyValueStoreType storeType,
+                            std::string const& filename,
+                            UID logID,
+                            int64_t memoryLimit,
+                            bool checkChecksums = false,
+                            bool checkIntegrity = false,
+                            Reference<AsyncVar<struct ServerDBInfo> const> db = {},
+                            int64_t pageCacheBytes = 0);
 #endif
diff --git a/fdbserver/include/fdbserver/RESTKmsConnector.h b/fdbserver/kvstore/include/fdbserver/kvstore/KVFileUtils.h
similarity index 70%
rename from fdbserver/include/fdbserver/RESTKmsConnector.h
rename to fdbserver/kvstore/include/fdbserver/kvstore/KVFileUtils.h
index b20a5e4482b..8d1014164f5 100644
--- a/fdbserver/include/fdbserver/RESTKmsConnector.h
+++ b/fdbserver/kvstore/include/fdbserver/kvstore/KVFileUtils.h
@@ -1,5 +1,5 @@
 /*
- * RESTKmsConnector.actor.h
+ * KVFileUtils.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,16 +18,16 @@
  * limitations under the License.
  */
 
-#ifndef REST_KMS_CONNECTOR_H
-#define REST_KMS_CONNECTOR_H
+#ifndef FDBSERVER_KVSTORE_KVFILEUTILS_H
+#define FDBSERVER_KVSTORE_KVFILEUTILS_H
 #pragma once
 
-#include "fdbserver/KmsConnector.h"
+#include <string>
 
-class RESTKmsConnector : public KmsConnector {
-public:
-	RESTKmsConnector(const std::string& conStr) : KmsConnector(conStr) {}
-	Future<Void> connectorCore(KmsConnectorInterface interf);
-};
+#include "flow/flow.h"
+
+void GenerateIOLogChecksumFile(std::string filename);
+Future<Void> KVFileCheck(std::string filename, bool integrity);
+Future<Void> KVFileDump(std::string filename);
 
 #endif
diff --git a/fdbserver/include/fdbserver/template_fdb.h b/fdbserver/kvstore/template_fdb.h
similarity index 100%
rename from fdbserver/include/fdbserver/template_fdb.h
rename to fdbserver/kvstore/template_fdb.h
diff --git a/fdbserver/logrouter/CMakeLists.txt b/fdbserver/logrouter/CMakeLists.txt
new file mode 100644
index 00000000000..70267414279
--- /dev/null
+++ b/fdbserver/logrouter/CMakeLists.txt
@@ -0,0 +1,13 @@
+fdb_find_sources(FDBSERVER_LOGROUTER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_logrouter SRCS ${FDBSERVER_LOGROUTER_SRCS})
+add_fdbserver_link_test(fdbserver_logrouterlinktest
+  fdbserver_logrouter
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_logrouter)
+target_include_directories(fdbserver_logrouter
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(fdbserver_logrouter PRIVATE fdbserver_core fdbserver_logsystem)
diff --git a/fdbserver/LogRouter.cpp b/fdbserver/logrouter/LogRouter.cpp
similarity index 96%
rename from fdbserver/LogRouter.cpp
rename to fdbserver/logrouter/LogRouter.cpp
index 2385e68ae29..ad64ac8cb50 100644
--- a/fdbserver/LogRouter.cpp
+++ b/fdbserver/logrouter/LogRouter.cpp
@@ -19,11 +19,13 @@
  */
 
 #include "fdbrpc/Stats.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/TLogInterface.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logrouter/LogRouter.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/TLogInterface.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/CodeProbe.h"
@@ -44,7 +46,7 @@ struct LogRouterData {
 		TagData(Tag tag, Version popped, Version durableKnownCommittedVersion)
 		  : popped(popped), durableKnownCommittedVersion(durableKnownCommittedVersion), tag(tag) {}
 
-		TagData(TagData&& r) noexcept
+		explicit(false) TagData(TagData&& r) noexcept
 		  : version_messages(std::move(r.version_messages)), popped(r.popped),
 		    durableKnownCommittedVersion(r.durableKnownCommittedVersion), tag(r.tag) {}
 		void operator=(TagData&& r) noexcept {
@@ -69,7 +71,7 @@ struct LogRouterData {
 	};
 
 	const UID dbgid;
-	Reference<AsyncVar<Reference<ILogSystem>>> logSystem;
+	Reference<AsyncVar<Reference<LogSystem>>> logSystem;
 	Future<Void> logSystemChanged = Void();
 	Optional<UID> primaryPeekLocation;
 	NotifiedVersion version; // The largest version at which the log router has peeked mutations
@@ -128,7 +130,7 @@ struct LogRouterData {
 	}
 
 	LogRouterData(UID dbgid, const InitializeLogRouterRequest& req)
-	  : dbgid(dbgid), logSystem(new AsyncVar<Reference<ILogSystem>>()), version(req.startVersion - 1), minPopped(0),
+	  : dbgid(dbgid), logSystem(new AsyncVar<Reference<LogSystem>>()), version(req.startVersion - 1), minPopped(0),
 	    startVersion(req.startVersion), minKnownCommittedVersion(0), poppedVersion(0), routerTag(req.routerTag),
 	    allowPops(false), foundEpochEnd(false), generation(req.recoveryCount),
 	    peekLatencyDist(Histogram::getHistogram("LogRouter"_sr, "PeekTLogLatency"_sr, Histogram::Unit::milliseconds)),
@@ -238,15 +240,14 @@ struct LogRouterData {
 	// recovery version.
 	Future<Void> pullAsyncData();
 
-	Future<Reference<ILogSystem::IPeekCursor>> getPeekCursorData(Reference<ILogSystem::IPeekCursor> r,
-	                                                             Version beginVersion);
+	Future<Reference<IPeekCursor>> getPeekCursorData(Reference<IPeekCursor> r, Version beginVersion);
 
 	// Future<Void> logRouterPop(const TLogPopRequest& req);
 	Future<Void> cleanupPeekTrackers();
 };
 
 void LogRouterData::commitMessages(Version version, const std::vector<TagsAndMessage>& taggedMessages) {
-	if (!taggedMessages.size()) {
+	if (taggedMessages.empty()) {
 		return;
 	}
 
@@ -363,9 +364,8 @@ Future<Void> LogRouterData::waitForVersionAndLog(Version ver) {
 	}
 }
 
-Future<Reference<ILogSystem::IPeekCursor>> LogRouterData::getPeekCursorData(Reference<ILogSystem::IPeekCursor> r,
-                                                                            Version beginVersion) {
-	Reference<ILogSystem::IPeekCursor> result = r;
+Future<Reference<IPeekCursor>> LogRouterData::getPeekCursorData(Reference<IPeekCursor> r, Version beginVersion) {
+	Reference<IPeekCursor> result = r;
 	bool useSatellite = SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED;
 	uint32_t noPrimaryPeekLocation = 0;
 
@@ -402,7 +402,7 @@ Future<Reference<ILogSystem::IPeekCursor>> LogRouterData::getPeekCursorData(Refe
 				              .detail("LogID", result->getPrimaryPeekLocation())
 				              .trackLatest(eventCacheHolder->trackingKey);
 			          } else {
-				          result = Reference<ILogSystem::IPeekCursor>();
+				          result = Reference<IPeekCursor>();
 			          }
 			          logSystemChanged = logSystem->onChange();
 		          })
@@ -436,7 +436,7 @@ Future<Reference<ILogSystem::IPeekCursor>> LogRouterData::getPeekCursorData(Refe
 }
 
 Future<Void> LogRouterData::pullAsyncData() {
-	Reference<ILogSystem::IPeekCursor> r;
+	Reference<IPeekCursor> r;
 	Version tagAt = version.get() + 1;
 	Version lastVer = 0;
 	std::vector<int> tags; // an optimization to avoid reallocating vector memory in every loop
@@ -579,7 +579,7 @@ Future<Void> LogRouterData::logRouterPeekMessages(PromiseType replyPromise,
 			}
 			auto seqBegin = trackerData.sequence_version.begin();
 			// The peek cursor and this comparison need to agree about the maximum number of in-flight requests.
-			while (trackerData.sequence_version.size() &&
+			while (!trackerData.sequence_version.empty() &&
 			       seqBegin->first <= sequence - SERVER_KNOBS->PARALLEL_GET_MORE_REQUESTS) {
 				if (seqBegin->second.canBeSet()) {
 					seqBegin->second.sendError(operation_obsolete());
@@ -588,7 +588,7 @@ Future<Void> LogRouterData::logRouterPeekMessages(PromiseType replyPromise,
 				seqBegin = trackerData.sequence_version.begin();
 			}
 
-			if (trackerData.sequence_version.size() && sequence < seqBegin->first) {
+			if (!trackerData.sequence_version.empty() && sequence < seqBegin->first) {
 				throw operation_obsolete();
 			}
 
@@ -706,7 +706,7 @@ Future<Void> LogRouterData::logRouterPeekMessages(PromiseType replyPromise,
 		auto& trackerData = peekTracker[peekId];
 		trackerData.lastUpdate = now();
 		auto& sequenceData = trackerData.sequence_version[sequence + 1];
-		if (trackerData.sequence_version.size() && sequence + 1 < trackerData.sequence_version.begin()->first) {
+		if (!trackerData.sequence_version.empty() && sequence + 1 < trackerData.sequence_version.begin()->first) {
 			replyPromise.sendError(operation_obsolete());
 			if (!sequenceData.isSet())
 				sequenceData.sendError(operation_obsolete());
@@ -782,7 +782,7 @@ Future<Void> LogRouterData::cleanupPeekTrackers() {
 		while (it != peekTracker.end()) {
 			double timeUntilExpiration = it->second.lastUpdate + SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME - now();
 			if (timeUntilExpiration < 1.0e-6) {
-				for (auto seq : it->second.sequence_version) {
+				for (const auto& seq : it->second.sequence_version) {
 					if (!seq.second.isSet()) {
 						seq.second.sendError(timed_out());
 					}
@@ -811,7 +811,7 @@ Future<Void> logRouterPop(LogRouterData* self, TLogPopRequest req) {
 
 	Version minPopped = std::numeric_limits<Version>::max();
 	Version minKnownCommittedVersion = std::numeric_limits<Version>::max();
-	for (auto it : self->tag_data) {
+	for (const auto& it : self->tag_data) {
 		if (it) {
 			minPopped = std::min(it->popped, minPopped);
 			minKnownCommittedVersion = std::min(it->durableKnownCommittedVersion, minKnownCommittedVersion);
@@ -851,7 +851,7 @@ Future<Void> logRouterCore(TLogInterface interf,
 			          dbInfoChange = db->onChange();
 			          logRouterData.allowPops = db->get().recoveryState == RecoveryState::FULLY_RECOVERED &&
 			                                    db->get().recoveryCount >= req.recoveryCount;
-			          logRouterData.logSystem->set(ILogSystem::fromServerDBInfo(logRouterData.dbgid, db->get(), true));
+			          logRouterData.logSystem->set(makeLogSystemFromServerDBInfo(logRouterData.dbgid, db->get(), true));
 		          })
 		    .When(interf.peekMessages.getFuture(),
 		          [&](const TLogPeekRequest& req) {
diff --git a/fdbserver/logrouter/include/fdbserver/logrouter/LogRouter.h b/fdbserver/logrouter/include/fdbserver/logrouter/LogRouter.h
new file mode 100644
index 00000000000..b14771ffb27
--- /dev/null
+++ b/fdbserver/logrouter/include/fdbserver/logrouter/LogRouter.h
@@ -0,0 +1,31 @@
+/*
+ * LogRouter.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/TLogInterface.h"
+#include "flow/flow.h"
+
+struct InitializeLogRouterRequest;
+struct ServerDBInfo;
+
+Future<Void> logRouter(TLogInterface interf,
+                       InitializeLogRouterRequest req,
+                       Reference<AsyncVar<ServerDBInfo> const> db);
diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/logsystem/ApplyMetadataMutation.cpp
similarity index 95%
rename from fdbserver/ApplyMetadataMutation.cpp
rename to fdbserver/logsystem/ApplyMetadataMutation.cpp
index 7e4b0fdaa81..8b023cda062 100644
--- a/fdbserver/ApplyMetadataMutation.cpp
+++ b/fdbserver/logsystem/ApplyMetadataMutation.cpp
@@ -18,17 +18,17 @@
  * limitations under the License.
  */
 
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/KeyBackedTypes.actor.h" // for key backed map codecs for tss mapping
+#include "fdbclient/BackupAgent.h"
+#include "fdbclient/KeyBackedTypes.h" // for key backed map codecs for tss mapping
 #include "fdbclient/MutationList.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/AccumulativeChecksumUtil.h"
-#include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogProtocolMessage.h"
-#include "fdbserver/LogSystem.h"
+#include "fdbserver/core/AccumulativeChecksumUtil.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/LogProtocolMessage.h"
+#include "fdbserver/logsystem/ApplyMetadataMutation.h"
+#include "fdbserver/logsystem/LogSystem.h"
 #include "flow/Error.h"
 #include "flow/Trace.h"
 
@@ -66,25 +66,24 @@ class ApplyMetadataMutationsImpl {
 	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           Arena& arena_,
 	                           const VectorRef<MutationRef>& mutations_,
-	                           ProxyCommitData& proxyCommitData_,
-	                           Reference<ILogSystem> logSystem_,
+	                           const ApplyMetadataProxyContext& proxyMetadata_,
+	                           Reference<LogSystem> logSystem_,
 	                           LogPushData* toCommit_,
 	                           bool& confChange_,
 	                           Version version,
 	                           Version popVersion_,
 	                           bool initialCommit_,
 	                           bool provisionalCommitProxy_)
-	  : spanContext(spanContext_), dbgid(proxyCommitData_.dbgid), arena(arena_), mutations(mutations_),
-	    txnStateStore(proxyCommitData_.txnStateStore), toCommit(toCommit_), confChange(confChange_),
-	    logSystem(logSystem_), version(version), popVersion(popVersion_),
-	    vecBackupKeys(&proxyCommitData_.vecBackupKeys), keyInfo(&proxyCommitData_.keyInfo),
-	    uid_applyMutationsData(proxyCommitData_.firstProxy ? &proxyCommitData_.uid_applyMutationsData : nullptr),
-	    commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion),
-	    storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped),
-	    tssMapping(&proxyCommitData_.tssMapping), initialCommit(initialCommit_),
+	  : spanContext(spanContext_), dbgid(proxyMetadata_.dbgid), arena(arena_), mutations(mutations_),
+	    txnStateStore(proxyMetadata_.txnStateStore), toCommit(toCommit_), confChange(confChange_),
+	    logSystem(logSystem_), version(version), popVersion(popVersion_), vecBackupKeys(proxyMetadata_.vecBackupKeys),
+	    keyInfo(proxyMetadata_.keyInfo), uid_applyMutationsData(proxyMetadata_.uid_applyMutationsData),
+	    commit(proxyMetadata_.commit), cx(proxyMetadata_.cx), committedVersion(proxyMetadata_.committedVersion),
+	    storageCache(proxyMetadata_.storageCache), tag_popped(proxyMetadata_.tag_popped),
+	    tssMapping(proxyMetadata_.tssMapping), initialCommit(initialCommit_),
 	    provisionalCommitProxy(provisionalCommitProxy_),
-	    accumulativeChecksumIndex(getCommitProxyAccumulativeChecksumIndex(proxyCommitData_.commitProxyIndex)),
-	    acsBuilder(proxyCommitData_.acsBuilder), epoch(proxyCommitData_.epoch), rangeLock(proxyCommitData_.rangeLock) {
+	    accumulativeChecksumIndex(getCommitProxyAccumulativeChecksumIndex(proxyMetadata_.commitProxyIndex)),
+	    acsBuilder(proxyMetadata_.acsBuilder), epoch(proxyMetadata_.epoch), rangeLock(proxyMetadata_.rangeLock) {
 
 		// If commit proxy, epoch must be set
 		ASSERT(toCommit == nullptr || epoch.present());
@@ -120,7 +119,7 @@ class ApplyMetadataMutationsImpl {
 	// Flag indicates if the configure is changed
 	bool& confChange;
 
-	Reference<ILogSystem> logSystem = Reference<ILogSystem>();
+	Reference<LogSystem> logSystem = Reference<LogSystem>();
 	Version version = invalidVersion;
 	Version popVersion = 0;
 	KeyRangeMap<std::set<Key>>* vecBackupKeys = nullptr;
@@ -162,7 +161,7 @@ class ApplyMetadataMutationsImpl {
 	// commit
 	std::vector<std::pair<UID, UID>> tssMappingToAdd;
 
-	std::shared_ptr<RangeLock> rangeLock = nullptr;
+	ApplyMetadataRangeLock* rangeLock = nullptr;
 
 private:
 	bool dummyConfChange = false;
@@ -309,7 +308,7 @@ class ApplyMetadataMutationsImpl {
 			if (storageCache) {
 				auto cacheItr = storageCache->find(id);
 				if (cacheItr == storageCache->end()) {
-					Reference<StorageInfo> storageInfo = makeReference<StorageInfo>();
+					auto storageInfo = makeReference<StorageInfo>();
 					storageInfo->tag = tag;
 					Optional<Key> interfKey = txnStateStore->readValue(serverListKeyFor(id)).get();
 					if (interfKey.present()) {
@@ -366,7 +365,7 @@ class ApplyMetadataMutationsImpl {
 
 				auto cacheItr = storageCache->find(id);
 				if (cacheItr == storageCache->end()) {
-					Reference<StorageInfo> storageInfo = makeReference<StorageInfo>();
+					auto storageInfo = makeReference<StorageInfo>();
 					storageInfo->interf = interf;
 					Optional<Key> tagKey = txnStateStore->readValue(serverTagKeyFor(id)).get();
 					if (tagKey.present()) {
@@ -700,9 +699,8 @@ class ApplyMetadataMutationsImpl {
 		if (keyInfo) {
 			KeyRangeRef clearRange(r.begin.removePrefix(keyServersPrefix), r.end.removePrefix(keyServersPrefix));
 			keyInfo->insert(clearRange,
-			                clearRange.begin == StringRef()
-			                    ? ServerCacheInfo()
-			                    : keyInfo->rangeContainingKeyBefore(clearRange.begin).value());
+			                clearRange.begin.empty() ? ServerCacheInfo()
+			                                         : keyInfo->rangeContainingKeyBefore(clearRange.begin).value());
 			if (toCommit && SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 				toCommit->setLogsChanged();
 			}
@@ -794,7 +792,7 @@ class ApplyMetadataMutationsImpl {
 			}
 			// Might be a tss removal, which doesn't store a tag there.
 			// Chained if is a little verbose, but avoids unnecessary work
-			if (toCommit && !initialCommit && !serverKeysCleared.size()) {
+			if (toCommit && !initialCommit && serverKeysCleared.empty()) {
 				KeyRangeRef maybeTssRange = range & serverTagKeys;
 				if (maybeTssRange.singleKeyRange()) {
 					UID id = decodeServerTagKey(maybeTssRange.begin);
@@ -1173,9 +1171,9 @@ class ApplyMetadataMutationsImpl {
 } // anonymous namespace
 
 void applyMetadataMutations(SpanContext const& spanContext,
-                            ProxyCommitData& proxyCommitData,
+                            const ApplyMetadataProxyContext& proxyMetadata,
                             Arena& arena,
-                            Reference<ILogSystem> logSystem,
+                            Reference<LogSystem> logSystem,
                             const VectorRef<MutationRef>& mutations,
                             LogPushData* toCommit,
                             bool& confChange,
@@ -1186,7 +1184,7 @@ void applyMetadataMutations(SpanContext const& spanContext,
 	ApplyMetadataMutationsImpl(spanContext,
 	                           arena,
 	                           mutations,
-	                           proxyCommitData,
+	                           proxyMetadata,
 	                           logSystem,
 	                           toCommit,
 	                           confChange,
diff --git a/fdbserver/logsystem/CMakeLists.txt b/fdbserver/logsystem/CMakeLists.txt
new file mode 100644
index 00000000000..f36fde984c0
--- /dev/null
+++ b/fdbserver/logsystem/CMakeLists.txt
@@ -0,0 +1,16 @@
+fdb_find_sources(FDBSERVER_LOGSYSTEM_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_logsystem SRCS ${FDBSERVER_LOGSYSTEM_SRCS})
+add_fdbserver_link_test(fdbserver_logsystemlinktest
+  fdbserver_logsystem
+  fdbserver_kvstore
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_logsystem)
+target_include_directories(fdbserver_logsystem
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(fdbserver_logsystem PUBLIC fdbserver_core fdbserver_kvstore)
diff --git a/fdbserver/LogSystem.cpp b/fdbserver/logsystem/LogSet.cpp
similarity index 54%
rename from fdbserver/LogSystem.cpp
rename to fdbserver/logsystem/LogSet.cpp
index 3bab086acae..e8cff87116f 100644
--- a/fdbserver/LogSystem.cpp
+++ b/fdbserver/logsystem/LogSet.cpp
@@ -1,5 +1,5 @@
 /*
- * LogSystem.cpp
+ * LogSet.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/LogSystem.h"
+#include "fdbserver/logsystem/LogSystem.h"
+
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/OTELSpanContextMessage.h"
-#include "fdbserver/SpanContextMessage.h"
-#include "flow/serialize.h"
 
 std::string LogSet::logRouterString() {
 	std::string result;
@@ -74,8 +72,8 @@ void LogSet::populateSatelliteTagLocations(int logRouterTags, int oldLogRouterTa
 		used_servers.insert(std::make_pair(0, i));
 	}
 
-	Reference<LocalitySet> serverSet = Reference<LocalitySet>(new LocalityMap<std::pair<int, int>>());
-	LocalityMap<std::pair<int, int>>* serverMap = (LocalityMap<std::pair<int, int>>*)serverSet.getPtr();
+	Reference<LocalitySet> serverSet = makeReference<LocalityMap<std::pair<int, int>>>();
+	auto* serverMap = (LocalityMap<std::pair<int, int>>*)serverSet.getPtr();
 	std::vector<std::pair<int, int>> resultPairs;
 	for (int loc = 0; loc < satelliteTagLocations.size(); loc++) {
 		int team = loc;
@@ -91,7 +89,7 @@ void LogSet::populateSatelliteTagLocations(int logRouterTags, int oldLogRouterTa
 		resultPairs.clear();
 		for (auto& used_idx : used_servers) {
 			auto entry = serverMap->add(tLogLocalities[used_idx.second], &used_idx);
-			if (!resultPairs.size()) {
+			if (resultPairs.empty()) {
 				resultPairs.push_back(used_idx);
 				alsoServers[0] = entry;
 			}
@@ -164,7 +162,7 @@ void LogSet::checkSatelliteTagLocations() {
 		zones.insert(loc.zoneId());
 		dcs.insert(loc.dcId());
 	}
-	bool moreThanOneDC = dcs.size() > 1 ? true : false;
+	bool moreThanOneDC = dcs.size() > 1;
 
 	TraceEvent(((maxUsed - minUsed > 1) || (maxUsedBest - minUsedBest > 1))
 	               ? (g_network->isSimulated() && !foundDuplicate && !moreThanOneDC ? SevError : SevWarnAlways)
@@ -189,7 +187,7 @@ int LogSet::bestLocationFor(Tag tag) {
 void LogSet::updateLocalitySet(std::vector<LocalityData> const& localities) {
 	LocalityMap<int>* logServerMap;
 
-	logServerSet = Reference<LocalitySet>(new LocalityMap<int>());
+	logServerSet = makeReference<LocalityMap<int>>();
 	logServerMap = (LocalityMap<int>*)logServerSet.getPtr();
 
 	logEntryArray.clear();
@@ -206,11 +204,10 @@ void LogSet::updateLocalitySet(std::vector<LocalityData> const& localities) {
 bool LogSet::satisfiesPolicy(const std::vector<LocalityEntry>& locations) {
 	resultEntries.clear();
 
-	// Run the policy, assert if unable to satisfy
 	bool result = logServerSet->selectReplicas(tLogPolicy, locations, resultEntries);
 	ASSERT(result);
 
-	return resultEntries.size() == 0;
+	return resultEntries.empty();
 }
 
 void LogSet::getPushLocations(VectorRef<Tag> tags,
@@ -235,7 +232,6 @@ void LogSet::getPushLocations(VectorRef<Tag> tags,
 	resultEntries.clear();
 
 	if (allLocations) {
-		// special handling for allLocations
 		TraceEvent("AllLocationsSet").log();
 		for (int i = 0; i < logServers.size(); i++) {
 			newLocations.push_back(i);
@@ -250,16 +246,15 @@ void LogSet::getPushLocations(VectorRef<Tag> tags,
 
 	uniquify(newLocations);
 
-	if (newLocations.size())
+	if (!newLocations.empty()) {
 		alsoServers.reserve(newLocations.size());
+	}
 
-	// Convert locations to the also servers
 	for (auto location : newLocations) {
 		locations.push_back(locationOffset + location);
 		alsoServers.push_back(logEntryArray[location]);
 	}
 
-	// Run the policy, assert if unable to satisfy
 	bool result;
 	if (restrictedLogSet.present()) {
 		result = restrictedLogSet.get()->selectReplicas(tLogPolicy, alsoServers, resultEntries);
@@ -268,138 +263,8 @@ void LogSet::getPushLocations(VectorRef<Tag> tags,
 	}
 	ASSERT(result);
 
-	// Add the new servers to the location array
-	LocalityMap<int>* logServerMap = (LocalityMap<int>*)logServerSet.getPtr();
+	auto* logServerMap = (LocalityMap<int>*)logServerSet.getPtr();
 	for (auto entry : resultEntries) {
 		locations.push_back(locationOffset + *logServerMap->getObject(entry));
 	}
-	//TraceEvent("GetPushLocations").detail("Policy", tLogPolicy->info())
-	//	.detail("Results", locations.size()).detail("Selection", logServerSet->size())
-	//	.detail("Included", alsoServers.size()).detail("Duration", timer() - t);
-}
-
-LogPushData::LogPushData(Reference<ILogSystem> logSystem, int tlogCount) : logSystem(logSystem), subsequence(1) {
-	ASSERT(tlogCount > 0);
-	messagesWriter.reserve(tlogCount);
-	for (int i = 0; i < tlogCount; i++) {
-		messagesWriter.emplace_back(AssumeVersion(g_network->protocolVersion()));
-	}
-	messagesWritten = std::vector<bool>(tlogCount, false);
-}
-
-void LogPushData::addTxsTag() {
-	next_message_tags.push_back(logSystem->getRandomTxsTag());
-}
-
-void LogPushData::addTransactionInfo(SpanContext const& context) {
-	CODE_PROBE(!spanContext.isValid(), "addTransactionInfo with invalid SpanContext");
-	spanContext = context;
-	writtenLocations.clear();
-}
-
-void LogPushData::writeMessage(StringRef rawMessageWithoutLength, bool usePreviousLocations) {
-	if (!usePreviousLocations) {
-		prev_tags.clear();
-		if (logSystem->hasRemoteLogs()) {
-			prev_tags.push_back(chooseRouterTag());
-		}
-		for (auto& tag : next_message_tags) {
-			prev_tags.push_back(tag);
-		}
-		msg_locations.clear();
-		logSystem->getPushLocations(prev_tags, msg_locations);
-		written_tags.insert(next_message_tags.begin(), next_message_tags.end());
-		next_message_tags.clear();
-	}
-	uint32_t subseq = this->subsequence++;
-	uint32_t msgsize =
-	    rawMessageWithoutLength.size() + sizeof(subseq) + sizeof(uint16_t) + sizeof(Tag) * prev_tags.size();
-	for (int loc : msg_locations) {
-		BinaryWriter& wr = messagesWriter[loc];
-		wr << msgsize << subseq << uint16_t(prev_tags.size());
-		for (auto& tag : prev_tags)
-			wr << tag;
-		wr.serializeBytes(rawMessageWithoutLength);
-	}
-}
-
-std::vector<Standalone<StringRef>> LogPushData::getAllMessages() const {
-	std::vector<Standalone<StringRef>> results;
-	results.reserve(messagesWriter.size());
-	for (int loc = 0; loc < messagesWriter.size(); loc++) {
-		results.push_back(getMessages(loc));
-	}
-	return results;
-}
-
-void LogPushData::recordEmptyMessage(int loc, const Standalone<StringRef>& value) {
-	if (!messagesWritten[loc]) {
-		BinaryWriter w(AssumeVersion(g_network->protocolVersion()));
-		Standalone<StringRef> v = w.toValue();
-		if (value.size() > v.size()) {
-			messagesWritten[loc] = true;
-		}
-	}
-}
-
-float LogPushData::getEmptyMessageRatio() const {
-	auto count = std::count(messagesWritten.begin(), messagesWritten.end(), false);
-	ASSERT_WE_THINK(messagesWritten.size() > 0);
-	return 1.0 * count / messagesWritten.size();
-}
-
-bool LogPushData::writeTransactionInfo(int location, uint32_t subseq) {
-	if (!FLOW_KNOBS->WRITE_TRACING_ENABLED || logSystem->getTLogVersion() < TLogVersion::V6 ||
-	    writtenLocations.contains(location)) {
-		return false;
-	}
-
-	CODE_PROBE(true, "Wrote SpanContextMessage to a transaction log");
-	writtenLocations.insert(location);
-
-	BinaryWriter& wr = messagesWriter[location];
-	int offset = wr.getLength();
-	wr << uint32_t(0) << subseq << uint16_t(prev_tags.size());
-	for (auto& tag : prev_tags)
-		wr << tag;
-	if (logSystem->getTLogVersion() >= TLogVersion::V7) {
-		OTELSpanContextMessage contextMessage(spanContext);
-		wr << contextMessage;
-	} else {
-		// When we're on a TLog version below 7, but the front end of the system (i.e. proxy, sequencer, resolver)
-		// is using OpenTelemetry tracing (i.e on or above 7.2), we need to convert the OpenTelemetry Span data model
-		// i.e. 16 bytes for traceId, 8 bytes for spanId, to the OpenTracing spec, which is 8 bytes for traceId
-		// and 8 bytes for spanId. That means we need to drop some data.
-		//
-		// As a workaround for this special case we've decided to drop is the 8 bytes
-		// for spanId. Therefore we're passing along the full 16 byte traceId to the storage server with 0 for spanID.
-		// This will result in a follows from relationship for the storage span within the trace rather than a
-		// parent->child.
-		SpanContextMessage contextMessage;
-		if (spanContext.isSampled()) {
-			CODE_PROBE(true, "Converting OTELSpanContextMessage to traced SpanContextMessage");
-			contextMessage = SpanContextMessage(UID(spanContext.traceID.first(), spanContext.traceID.second()));
-		} else {
-			CODE_PROBE(true, "Converting OTELSpanContextMessage to untraced SpanContextMessage");
-			contextMessage = SpanContextMessage(UID(0, 0));
-		}
-		wr << contextMessage;
-	}
-	int length = wr.getLength() - offset;
-	*(uint32_t*)((uint8_t*)wr.getData() + offset) = length - sizeof(uint32_t);
-	return true;
-}
-
-void LogPushData::setMutations(uint32_t totalMutations, VectorRef<StringRef> mutations) {
-	ASSERT_EQ(subsequence, 1);
-	subsequence = totalMutations + 1; // set to next mutation number
-
-	ASSERT_EQ(messagesWriter.size(), mutations.size());
-	BinaryWriter w(AssumeVersion(g_network->protocolVersion()));
-	Standalone<StringRef> v = w.toValue();
-	const int header = v.size();
-	for (int i = 0; i < mutations.size(); i++) {
-		BinaryWriter& wr = messagesWriter[i];
-		wr.serializeBytes(mutations[i].substr(header));
-	}
 }
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/logsystem/LogSystem.cpp
similarity index 73%
rename from fdbserver/TagPartitionedLogSystem.actor.cpp
rename to fdbserver/logsystem/LogSystem.cpp
index 736ec7a6f92..98a91c88dea 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/logsystem/LogSystem.cpp
@@ -1,5 +1,5 @@
 /*
- * TagPartitionedLogSystem.actor.cpp
+ * LogSystem.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,27 +18,256 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TagPartitionedLogSystem.actor.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbserver/core/OTELSpanContextMessage.h"
+#include "fdbserver/core/SpanContextMessage.h"
+#include "flow/serialize.h"
+
+bool logSystemHasRemoteLogs(LogSystem const& logSystem) {
+	return logSystem.hasRemoteLogs();
+}
+
+void logSystemGetPushLocations(LogSystem const& logSystem,
+                               VectorRef<Tag> tags,
+                               std::vector<int>& locations,
+                               bool allLocations,
+                               Optional<std::vector<Reference<LocalitySet>>> fromLocations) {
+	logSystem.getPushLocations(tags, locations, allLocations, fromLocations);
+}
+
+std::vector<Reference<LocalitySet>> logSystemGetPushLocationsForTags(LogSystem const& logSystem,
+                                                                     std::vector<int>& fromLocations) {
+	return logSystem.getPushLocationsForTags(fromLocations);
+}
+
+Tag logSystemGetRandomRouterTag(LogSystem const& logSystem) {
+	return logSystem.getRandomRouterTag();
+}
+
+int logSystemGetLogRouterTags(LogSystem const& logSystem) {
+	return logSystem.getLogRouterTags();
+}
+
+Tag logSystemGetRandomTxsTag(LogSystem const& logSystem) {
+	return logSystem.getRandomTxsTag();
+}
+
+TLogVersion logSystemGetTLogVersion(LogSystem const& logSystem) {
+	return logSystem.getTLogVersion();
+}
+
+LogPushData::LogPushData(Reference<LogSystem> logSystem, int tlogCount) : logSystem(logSystem), subsequence(1) {
+	ASSERT(tlogCount > 0);
+	messagesWriter.reserve(tlogCount);
+	for (int i = 0; i < tlogCount; i++) {
+		messagesWriter.emplace_back(AssumeVersion(g_network->protocolVersion()));
+	}
+	messagesWritten = std::vector<bool>(tlogCount, false);
+}
+
+void LogPushData::addTxsTag() {
+	next_message_tags.push_back(logSystemGetRandomTxsTag(*logSystem));
+}
+
+void LogPushData::addTransactionInfo(SpanContext const& context) {
+	CODE_PROBE(!spanContext.isValid(), "addTransactionInfo with invalid SpanContext");
+	spanContext = context;
+	writtenLocations.clear();
+}
+
+void LogPushData::writeMessage(StringRef rawMessageWithoutLength, bool usePreviousLocations) {
+	if (!usePreviousLocations) {
+		prev_tags.clear();
+		if (logSystemHasRemoteLogs(*logSystem)) {
+			prev_tags.push_back(chooseRouterTag());
+		}
+		for (auto& tag : next_message_tags) {
+			prev_tags.push_back(tag);
+		}
+		msg_locations.clear();
+		logSystemGetPushLocations(*logSystem, VectorRef<Tag>((Tag*)prev_tags.data(), prev_tags.size()), msg_locations);
+		written_tags.insert(next_message_tags.begin(), next_message_tags.end());
+		next_message_tags.clear();
+	}
+	uint32_t subseq = this->subsequence++;
+	uint32_t msgsize =
+	    rawMessageWithoutLength.size() + sizeof(subseq) + sizeof(uint16_t) + sizeof(Tag) * prev_tags.size();
+	for (int loc : msg_locations) {
+		BinaryWriter& wr = messagesWriter[loc];
+		wr << msgsize << subseq << uint16_t(prev_tags.size());
+		for (auto& tag : prev_tags)
+			wr << tag;
+		wr.serializeBytes(rawMessageWithoutLength);
+	}
+}
+
+std::vector<Standalone<StringRef>> LogPushData::getAllMessages() const {
+	std::vector<Standalone<StringRef>> results;
+	results.reserve(messagesWriter.size());
+	for (int loc = 0; loc < messagesWriter.size(); loc++) {
+		results.push_back(getMessages(loc));
+	}
+	return results;
+}
+
+void LogPushData::recordEmptyMessage(int loc, const Standalone<StringRef>& value) {
+	if (!messagesWritten[loc]) {
+		BinaryWriter w(AssumeVersion(g_network->protocolVersion()));
+		Standalone<StringRef> v = w.toValue();
+		if (value.size() > v.size()) {
+			messagesWritten[loc] = true;
+		}
+	}
+}
+
+float LogPushData::getEmptyMessageRatio() const {
+	auto count = std::count(messagesWritten.begin(), messagesWritten.end(), false);
+	ASSERT_WE_THINK(!messagesWritten.empty());
+	return 1.0 * count / messagesWritten.size();
+}
+
+bool LogPushData::writeTransactionInfo(int location, uint32_t subseq) {
+	if (!FLOW_KNOBS->WRITE_TRACING_ENABLED || logSystemGetTLogVersion(*logSystem) < TLogVersion::V6 ||
+	    writtenLocations.contains(location)) {
+		return false;
+	}
+
+	CODE_PROBE(true, "Wrote SpanContextMessage to a transaction log");
+	writtenLocations.insert(location);
+
+	BinaryWriter& wr = messagesWriter[location];
+	int offset = wr.getLength();
+	wr << uint32_t(0) << subseq << uint16_t(prev_tags.size());
+	for (auto& tag : prev_tags)
+		wr << tag;
+	if (logSystemGetTLogVersion(*logSystem) >= TLogVersion::V7) {
+		OTELSpanContextMessage contextMessage(spanContext);
+		wr << contextMessage;
+	} else {
+		SpanContextMessage contextMessage;
+		if (spanContext.isSampled()) {
+			CODE_PROBE(true, "Converting OTELSpanContextMessage to traced SpanContextMessage");
+			contextMessage = SpanContextMessage(UID(spanContext.traceID.first(), spanContext.traceID.second()));
+		} else {
+			CODE_PROBE(true, "Converting OTELSpanContextMessage to untraced SpanContextMessage");
+			contextMessage = SpanContextMessage(UID(0, 0));
+		}
+		wr << contextMessage;
+	}
+	int length = wr.getLength() - offset;
+	*(uint32_t*)((uint8_t*)wr.getData() + offset) = length - sizeof(uint32_t);
+	return true;
+}
+
+void LogPushData::setMutations(uint32_t totalMutations, VectorRef<StringRef> mutations) {
+	ASSERT_EQ(subsequence, 1);
+	subsequence = totalMutations + 1;
+
+	ASSERT_EQ(messagesWriter.size(), mutations.size());
+	BinaryWriter w(AssumeVersion(g_network->protocolVersion()));
+	Standalone<StringRef> v = w.toValue();
+	const int header = v.size();
+	for (int i = 0; i < mutations.size(); i++) {
+		BinaryWriter& wr = messagesWriter[i];
+		wr.serializeBytes(mutations[i].substr(header));
+	}
+}
 
 #include <boost/dynamic_bitset.hpp>
 #include <utility>
 
 #include "fdbrpc/ReplicationUtils.h"
-#include "fdbserver/WaitFailure.h"
+#include "fdbserver/core/WaitFailure.h"
+
+#include "flow/CoroUtils.h"
+
+namespace {
+
+TLogSet toTLogSet(const LogSet& rhs) {
+	TLogSet result;
+	result.tLogWriteAntiQuorum = rhs.tLogWriteAntiQuorum;
+	result.tLogReplicationFactor = rhs.tLogReplicationFactor;
+	result.tLogLocalities = rhs.tLogLocalities;
+	result.tLogVersion = rhs.tLogVersion;
+	result.tLogPolicy = rhs.tLogPolicy;
+	result.isLocal = rhs.isLocal;
+	result.locality = rhs.locality;
+	result.startVersion = rhs.startVersion;
+	result.satelliteTagLocations = rhs.satelliteTagLocations;
+	for (const auto& tlog : rhs.logServers) {
+		result.tLogs.push_back(tlog->get());
+	}
+	for (const auto& logRouter : rhs.logRouters) {
+		result.logRouters.push_back(logRouter->get());
+	}
+	for (const auto& worker : rhs.backupWorkers) {
+		result.backupWorkers.push_back(worker->get());
+	}
+	return result;
+}
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+CoreTLogSet toCoreTLogSet(const LogSet& logset) {
+	CoreTLogSet result;
+	result.tLogWriteAntiQuorum = logset.tLogWriteAntiQuorum;
+	result.tLogReplicationFactor = logset.tLogReplicationFactor;
+	result.tLogLocalities = logset.tLogLocalities;
+	result.tLogPolicy = logset.tLogPolicy;
+	result.isLocal = logset.isLocal;
+	result.locality = logset.locality;
+	result.startVersion = logset.startVersion;
+	result.satelliteTagLocations = logset.satelliteTagLocations;
+	result.tLogVersion = logset.tLogVersion;
+	for (const auto& log : logset.logServers) {
+		result.tLogs.push_back(log->get().id());
+	}
+	return result;
+}
 
-ACTOR Future<Version> minVersionWhenReady(Future<Void> f,
-                                          std::vector<std::pair<UID, Future<TLogCommitReply>>> replies) {
+OldTLogConf toOldTLogConf(const OldLogData& oldLogData) {
+	OldTLogConf result;
+	result.epochBegin = oldLogData.epochBegin;
+	result.epochEnd = oldLogData.epochEnd;
+	result.recoverAt = oldLogData.recoverAt;
+	result.logRouterTags = oldLogData.logRouterTags;
+	result.txsTags = oldLogData.txsTags;
+	result.pseudoLocalities = oldLogData.pseudoLocalities;
+	result.epoch = oldLogData.epoch;
+	for (const Reference<LogSet>& logSet : oldLogData.tLogs) {
+		result.tLogs.push_back(toTLogSet(*logSet));
+	}
+	return result;
+}
+
+OldTLogCoreData toOldTLogCoreData(const OldLogData& oldData) {
+	OldTLogCoreData result;
+	result.logRouterTags = oldData.logRouterTags;
+	result.txsTags = oldData.txsTags;
+	result.epochBegin = oldData.epochBegin;
+	result.epochEnd = oldData.epochEnd;
+	result.recoverAt = oldData.recoverAt;
+	result.pseudoLocalities = oldData.pseudoLocalities;
+	result.epoch = oldData.epoch;
+	for (const Reference<LogSet>& logSet : oldData.tLogs) {
+		if (!logSet->logServers.empty()) {
+			result.tLogs.push_back(toCoreTLogSet(*logSet));
+		}
+	}
+	return result;
+}
+
+} // namespace
+
+Future<Version> minVersionWhenReady(Future<Void> f, std::vector<std::pair<UID, Future<TLogCommitReply>>> replies) {
 	try {
-		wait(f);
+		co_await f;
 		Version minVersion = std::numeric_limits<Version>::max();
 		for (const auto& [_tlogID, reply] : replies) {
 			if (reply.isReady() && !reply.isError()) {
 				minVersion = std::min(minVersion, reply.get().version);
 			}
 		}
-		return minVersion;
+		co_return minVersion;
 	} catch (Error& err) {
 		if (err.code() == error_code_operation_cancelled) {
 			TraceEvent(g_network->isSimulated() ? SevInfo : SevWarnAlways, "TLogPushCancelled");
@@ -94,107 +323,19 @@ LogSet::LogSet(const CoreTLogSet& coreSet)
 	updateLocalitySet(tLogLocalities);
 }
 
-TLogSet::TLogSet(const LogSet& rhs)
-  : tLogWriteAntiQuorum(rhs.tLogWriteAntiQuorum), tLogReplicationFactor(rhs.tLogReplicationFactor),
-    tLogLocalities(rhs.tLogLocalities), tLogVersion(rhs.tLogVersion), tLogPolicy(rhs.tLogPolicy), isLocal(rhs.isLocal),
-    locality(rhs.locality), startVersion(rhs.startVersion), satelliteTagLocations(rhs.satelliteTagLocations) {
-	for (const auto& tlog : rhs.logServers) {
-		tLogs.push_back(tlog->get());
-	}
-	for (const auto& logRouter : rhs.logRouters) {
-		logRouters.push_back(logRouter->get());
-	}
-	for (const auto& worker : rhs.backupWorkers) {
-		backupWorkers.push_back(worker->get());
-	}
-}
-
-OldTLogConf::OldTLogConf(const OldLogData& oldLogData)
-  : epochBegin(oldLogData.epochBegin), epochEnd(oldLogData.epochEnd), recoverAt(oldLogData.recoverAt),
-    logRouterTags(oldLogData.logRouterTags), txsTags(oldLogData.txsTags), pseudoLocalities(oldLogData.pseudoLocalities),
-    epoch(oldLogData.epoch) {
-	for (const Reference<LogSet>& logSet : oldLogData.tLogs) {
-		tLogs.emplace_back(*logSet);
-	}
-}
-
-CoreTLogSet::CoreTLogSet(const LogSet& logset)
-  : tLogWriteAntiQuorum(logset.tLogWriteAntiQuorum), tLogReplicationFactor(logset.tLogReplicationFactor),
-    tLogLocalities(logset.tLogLocalities), tLogPolicy(logset.tLogPolicy), isLocal(logset.isLocal),
-    locality(logset.locality), startVersion(logset.startVersion), satelliteTagLocations(logset.satelliteTagLocations),
-    tLogVersion(logset.tLogVersion) {
-	for (const auto& log : logset.logServers) {
-		tLogs.push_back(log->get().id());
-	}
-	// Do NOT store logset.backupWorkers, because master will recruit new ones.
-}
-
-OldTLogCoreData::OldTLogCoreData(const OldLogData& oldData)
-  : logRouterTags(oldData.logRouterTags), txsTags(oldData.txsTags), epochBegin(oldData.epochBegin),
-    epochEnd(oldData.epochEnd), recoverAt(oldData.recoverAt), pseudoLocalities(oldData.pseudoLocalities),
-    epoch(oldData.epoch) {
-	for (const Reference<LogSet>& logSet : oldData.tLogs) {
-		if (logSet->logServers.size()) {
-			tLogs.emplace_back(*logSet);
-		}
-	}
-}
-
-Future<Void> ILogSystem::recoverAndEndEpoch(Reference<AsyncVar<Reference<ILogSystem>>> const& outLogSystem,
-                                            UID const& dbgid,
-                                            DBCoreState const& oldState,
-                                            FutureStream<TLogRejoinRequest> const& rejoins,
-                                            LocalityData const& locality,
-                                            bool* forceRecovery) {
-	return TagPartitionedLogSystem::recoverAndEndEpoch(outLogSystem, dbgid, oldState, rejoins, locality, forceRecovery);
-}
-
-Reference<ILogSystem> ILogSystem::fromLogSystemConfig(UID const& dbgid,
-                                                      struct LocalityData const& locality,
-                                                      struct LogSystemConfig const& conf,
-                                                      bool excludeRemote,
-                                                      bool useRecoveredAt,
-                                                      Optional<PromiseStream<Future<Void>>> addActor) {
-	if (conf.logSystemType == LogSystemType::empty)
-		return Reference<ILogSystem>();
-	else if (conf.logSystemType == LogSystemType::tagPartitioned)
-		return TagPartitionedLogSystem::fromLogSystemConfig(
-		    dbgid, locality, conf, excludeRemote, useRecoveredAt, addActor);
-	else
-		throw internal_error();
-}
-
-Reference<ILogSystem> ILogSystem::fromOldLogSystemConfig(UID const& dbgid,
-                                                         struct LocalityData const& locality,
-                                                         struct LogSystemConfig const& conf) {
-	if (conf.logSystemType == LogSystemType::empty)
-		return Reference<ILogSystem>();
-	else if (conf.logSystemType == LogSystemType::tagPartitioned)
-		return TagPartitionedLogSystem::fromOldLogSystemConfig(dbgid, locality, conf);
-	else
-		throw internal_error();
-}
-
-Reference<ILogSystem> ILogSystem::fromServerDBInfo(UID const& dbgid,
-                                                   ServerDBInfo const& dbInfo,
-                                                   bool useRecoveredAt,
-                                                   Optional<PromiseStream<Future<Void>>> addActor) {
-	return fromLogSystemConfig(dbgid, dbInfo.myLocality, dbInfo.logSystemConfig, false, useRecoveredAt, addActor);
-}
-
-void TagPartitionedLogSystem::stopRejoins() {
+void LogSystem::stopRejoins() {
 	rejoins = Future<Void>();
 }
 
-void TagPartitionedLogSystem::addref() {
-	ReferenceCounted<TagPartitionedLogSystem>::addref();
+void LogSystem::addref() {
+	ReferenceCounted<LogSystem>::addref();
 }
 
-void TagPartitionedLogSystem::delref() {
-	ReferenceCounted<TagPartitionedLogSystem>::delref();
+void LogSystem::delref() {
+	ReferenceCounted<LogSystem>::delref();
 }
 
-std::string TagPartitionedLogSystem::describe() const {
+std::string LogSystem::describe() const {
 	std::string result;
 	for (int i = 0; i < tLogs.size(); i++) {
 		result += format("%d: ", i);
@@ -206,11 +347,11 @@ std::string TagPartitionedLogSystem::describe() const {
 	return result;
 }
 
-UID TagPartitionedLogSystem::getDebugID() const {
+UID LogSystem::getDebugID() const {
 	return dbgid;
 }
 
-void TagPartitionedLogSystem::addPseudoLocality(int8_t locality) {
+void LogSystem::addPseudoLocality(int8_t locality) {
 	ASSERT(locality < 0);
 	pseudoLocalities.insert(locality);
 	for (uint16_t i = 0; i < logRouterTags; i++) {
@@ -218,7 +359,7 @@ void TagPartitionedLogSystem::addPseudoLocality(int8_t locality) {
 	}
 }
 
-Tag TagPartitionedLogSystem::getPseudoPopTag(Tag tag, ProcessClass::ClassType type) const {
+Tag LogSystem::getPseudoPopTag(Tag tag, ProcessClass::ClassType type) const {
 	switch (type) {
 	case ProcessClass::LogRouterClass:
 		if (tag.locality == tagLocalityLogRouter) {
@@ -240,11 +381,11 @@ Tag TagPartitionedLogSystem::getPseudoPopTag(Tag tag, ProcessClass::ClassType ty
 	return tag;
 }
 
-bool TagPartitionedLogSystem::hasPseudoLocality(int8_t locality) const {
+bool LogSystem::hasPseudoLocality(int8_t locality) const {
 	return pseudoLocalities.contains(locality);
 }
 
-Version TagPartitionedLogSystem::popPseudoLocalityTag(Tag tag, Version upTo) {
+Version LogSystem::popPseudoLocalityTag(Tag tag, Version upTo) {
 	ASSERT(isPseudoLocality(tag.locality) && hasPseudoLocality(tag.locality));
 
 	Version& localityVersion = pseudoLocalityPopVersion[tag];
@@ -260,25 +401,25 @@ Version TagPartitionedLogSystem::popPseudoLocalityTag(Tag tag, Version upTo) {
 	return minVersion;
 }
 
-Future<Void> TagPartitionedLogSystem::recoverAndEndEpoch(Reference<AsyncVar<Reference<ILogSystem>>> const& outLogSystem,
-                                                         UID const& dbgid,
-                                                         DBCoreState const& oldState,
-                                                         FutureStream<TLogRejoinRequest> const& rejoins,
-                                                         LocalityData const& locality,
-                                                         bool* forceRecovery) {
+Future<Void> LogSystem::recoverAndEndEpoch(Reference<AsyncVar<Reference<LogSystem>>> const& outLogSystem,
+                                           UID const& dbgid,
+                                           DBCoreState const& oldState,
+                                           FutureStream<TLogRejoinRequest> const& rejoins,
+                                           LocalityData const& locality,
+                                           bool* forceRecovery) {
 	return epochEnd(outLogSystem, dbgid, oldState, rejoins, locality, forceRecovery);
 }
 
-Reference<ILogSystem> TagPartitionedLogSystem::fromLogSystemConfig(UID const& dbgid,
-                                                                   LocalityData const& locality,
-                                                                   LogSystemConfig const& lsConf,
-                                                                   bool excludeRemote,
-                                                                   bool useRecoveredAt,
-                                                                   Optional<PromiseStream<Future<Void>>> addActor) {
+Reference<LogSystem> LogSystem::fromLogSystemConfig(UID const& dbgid,
+                                                    LocalityData const& locality,
+                                                    LogSystemConfig const& lsConf,
+                                                    bool excludeRemote,
+                                                    bool useRecoveredAt,
+                                                    Optional<PromiseStream<Future<Void>>> addActor) {
 	ASSERT(lsConf.logSystemType == LogSystemType::tagPartitioned ||
-	       (lsConf.logSystemType == LogSystemType::empty && !lsConf.tLogs.size()));
+	       (lsConf.logSystemType == LogSystemType::empty && lsConf.tLogs.empty()));
 	// ASSERT(lsConf.epoch == epoch);  //< FIXME
-	auto logSystem = makeReference<TagPartitionedLogSystem>(dbgid, locality, lsConf.epoch, addActor);
+	auto logSystem = makeReference<LogSystem>(dbgid, locality, lsConf.epoch, addActor);
 
 	logSystem->tLogs.reserve(lsConf.tLogs.size());
 	logSystem->expectedLogSets = lsConf.expectedLogSets;
@@ -309,16 +450,16 @@ Reference<ILogSystem> TagPartitionedLogSystem::fromLogSystemConfig(UID const& db
 	return logSystem;
 }
 
-Reference<ILogSystem> TagPartitionedLogSystem::fromOldLogSystemConfig(UID const& dbgid,
-                                                                      LocalityData const& locality,
-                                                                      LogSystemConfig const& lsConf) {
+Reference<LogSystem> LogSystem::fromOldLogSystemConfig(UID const& dbgid,
+                                                       LocalityData const& locality,
+                                                       LogSystemConfig const& lsConf) {
 	ASSERT(lsConf.logSystemType == LogSystemType::tagPartitioned ||
-	       (lsConf.logSystemType == LogSystemType::empty && !lsConf.tLogs.size()));
+	       (lsConf.logSystemType == LogSystemType::empty && lsConf.tLogs.empty()));
 	// ASSERT(lsConf.epoch == epoch);  //< FIXME
-	const LogEpoch e = lsConf.oldTLogs.size() > 0 ? lsConf.oldTLogs[0].epoch : 0;
-	auto logSystem = makeReference<TagPartitionedLogSystem>(dbgid, locality, e);
+	const LogEpoch e = !lsConf.oldTLogs.empty() ? lsConf.oldTLogs[0].epoch : 0;
+	auto logSystem = makeReference<LogSystem>(dbgid, locality, e);
 
-	if (lsConf.oldTLogs.size()) {
+	if (!lsConf.oldTLogs.empty()) {
 		for (const TLogSet& tLogSet : lsConf.oldTLogs[0].tLogs) {
 			logSystem->tLogs.push_back(makeReference<LogSet>(tLogSet));
 		}
@@ -337,7 +478,7 @@ Reference<ILogSystem> TagPartitionedLogSystem::fromOldLogSystemConfig(UID const&
 	return logSystem;
 }
 
-void TagPartitionedLogSystem::purgeOldRecoveredGenerationsCoreState(DBCoreState& newState) {
+void LogSystem::purgeOldRecoveredGenerationsCoreState(DBCoreState& newState) {
 	Version oldestGenerationRecoverAtVersion = std::min(recoveredVersion->get(), remoteRecoveredVersion->get());
 	TraceEvent("ToCoreStateOldestGenerationRecoverAtVersion")
 	    .detail("RecoveredVersion", recoveredVersion->get())
@@ -360,7 +501,7 @@ void TagPartitionedLogSystem::purgeOldRecoveredGenerationsCoreState(DBCoreState&
 				}
 				for (int j = i + 1; j < newState.oldTLogData.size(); ++j) {
 					ASSERT(newState.oldTLogData[j].recoverAt < oldestGenerationRecoverAtVersion);
-					ASSERT(oldLogData[i].tLogs[0]->backupWorkers.size() == 0 ||
+					ASSERT(oldLogData[i].tLogs[0]->backupWorkers.empty() ||
 					       newState.oldTLogData[j].epoch < oldestBackupEpoch);
 				}
 			}
@@ -378,7 +519,7 @@ void TagPartitionedLogSystem::purgeOldRecoveredGenerationsCoreState(DBCoreState&
 	}
 }
 
-void TagPartitionedLogSystem::purgeOldRecoveredGenerationsInMemory(const DBCoreState& newState) {
+void LogSystem::purgeOldRecoveredGenerationsInMemory(const DBCoreState& newState) {
 	auto generations = newState.oldTLogData.size();
 	if (generations < oldLogData.size()) {
 		TraceEvent("PurgeOldTLogGenerationsInMemory", dbgid)
@@ -388,7 +529,7 @@ void TagPartitionedLogSystem::purgeOldRecoveredGenerationsInMemory(const DBCoreS
 	}
 }
 
-void TagPartitionedLogSystem::toCoreState(DBCoreState& newState) const {
+void LogSystem::toCoreState(DBCoreState& newState) const {
 	if (recoveryComplete.isValid() && recoveryComplete.isError())
 		throw recoveryComplete.getError();
 
@@ -400,8 +541,8 @@ void TagPartitionedLogSystem::toCoreState(DBCoreState& newState) const {
 	newState.txsTags = txsTags;
 	newState.pseudoLocalities = pseudoLocalities;
 	for (const auto& t : tLogs) {
-		if (t->logServers.size()) {
-			newState.tLogs.emplace_back(*t);
+		if (!t->logServers.empty()) {
+			newState.tLogs.push_back(toCoreTLogSet(*t));
 			newState.tLogs.back().tLogLocalities.clear();
 			for (const auto& log : t->logServers) {
 				newState.tLogs.back().tLogLocalities.push_back(log->get().interf().filteredLocality);
@@ -414,7 +555,7 @@ void TagPartitionedLogSystem::toCoreState(DBCoreState& newState) const {
 	    (repopulateRegionAntiQuorum == 0 && (!remoteRecoveryComplete.isValid() || !remoteRecoveryComplete.isReady())) ||
 	    epoch != oldestBackupEpoch) {
 		for (const auto& oldData : oldLogData) {
-			newState.oldTLogData.emplace_back(oldData);
+			newState.oldTLogData.push_back(toOldTLogCoreData(oldData));
 			TraceEvent("BWToCore")
 			    .detail("Epoch", newState.oldTLogData.back().epoch)
 			    .detail("TotalTags", newState.oldTLogData.back().logRouterTags)
@@ -426,11 +567,11 @@ void TagPartitionedLogSystem::toCoreState(DBCoreState& newState) const {
 	newState.logSystemType = logSystemType;
 }
 
-bool TagPartitionedLogSystem::remoteStorageRecovered() const {
+bool LogSystem::remoteStorageRecovered() const {
 	return remoteRecoveryComplete.isValid() && remoteRecoveryComplete.isReady();
 }
 
-Future<Void> TagPartitionedLogSystem::onCoreStateChanged() const {
+Future<Void> LogSystem::onCoreStateChanged() const {
 	std::vector<Future<Void>> changes;
 	changes.push_back(Never());
 	if (recoveryComplete.isValid() && !recoveryComplete.isReady()) {
@@ -448,8 +589,8 @@ Future<Void> TagPartitionedLogSystem::onCoreStateChanged() const {
 	return waitForAny(changes);
 }
 
-void TagPartitionedLogSystem::coreStateWritten(DBCoreState const& newState) {
-	if (!newState.oldTLogData.size()) {
+void LogSystem::coreStateWritten(DBCoreState const& newState) {
+	if (newState.oldTLogData.empty()) {
 		recoveryCompleteWrittenToCoreState.set(true);
 	}
 	for (auto& t : newState.tLogs) {
@@ -461,13 +602,13 @@ void TagPartitionedLogSystem::coreStateWritten(DBCoreState const& newState) {
 	}
 }
 
-Future<Void> TagPartitionedLogSystem::onError() const {
+Future<Void> LogSystem::onError() const {
 	return onError_internal(this);
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::onError_internal(TagPartitionedLogSystem const* self) {
+Future<Void> LogSystem::onError_internal(LogSystem const* self) {
 	// Never returns normally, but throws an error if the subsystem stops working
-	loop {
+	while (true) {
 		std::vector<Future<Void>> failed;
 		std::vector<Future<Void>> routerFailed;
 		std::vector<Future<Void>> backupFailed(1, Never());
@@ -562,17 +703,18 @@ ACTOR Future<Void> TagPartitionedLogSystem::onError_internal(TagPartitionedLogSy
 		changes.push_back(self->recoveryCompleteWrittenToCoreState.onChange());
 		changes.push_back(self->backupWorkerChanged.onTrigger());
 
-		ASSERT(failed.size() >= 1);
-		wait(quorum(changes, 1) ||
-		     tagError<Void>(traceAfter(quorum(failed, 1), "TPLSOnErrorLogSystemFailed"), tlog_failed()) ||
-		     tagError<Void>(traceAfter(quorum(backupFailed, 1), "TPLSOnErrorBackupFailed"), backup_worker_failed()));
+		ASSERT(!failed.empty());
+		co_await (
+		    quorum(changes, 1) ||
+		    tagError<Void>(traceAfter(quorum(failed, 1), "TPLSOnErrorLogSystemFailed"), tlog_failed()) ||
+		    tagError<Void>(traceAfter(quorum(backupFailed, 1), "TPLSOnErrorBackupFailed"), backup_worker_failed()));
 	}
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::pushResetChecker(Reference<ConnectionResetInfo> self, NetworkAddress addr) {
+Future<Void> LogSystem::pushResetChecker(Reference<ConnectionResetInfo> self, NetworkAddress addr) {
 	self->slowReplies = 0;
 	self->fastReplies = 0;
-	wait(delay(SERVER_KNOBS->PUSH_STATS_INTERVAL));
+	co_await delay(SERVER_KNOBS->PUSH_STATS_INTERVAL);
 	TraceEvent("SlowPushStats")
 	    .detail("PeerAddress", addr)
 	    .detail("SlowReplies", self->slowReplies)
@@ -582,19 +724,18 @@ ACTOR Future<Void> TagPartitionedLogSystem::pushResetChecker(Reference<Connectio
 		FlowTransport::transport().resetConnection(addr);
 		self->lastReset = now();
 	}
-	return Void();
 }
 
-ACTOR Future<TLogCommitReply> TagPartitionedLogSystem::recordPushMetrics(Reference<ConnectionResetInfo> self,
-                                                                         Reference<Histogram> dist,
-                                                                         NetworkAddress addr,
-                                                                         Future<TLogCommitReply> in) {
-	state double startTime = now();
-	TLogCommitReply t = wait(in);
+Future<TLogCommitReply> LogSystem::recordPushMetrics(Reference<ConnectionResetInfo> self,
+                                                     Reference<Histogram> dist,
+                                                     NetworkAddress addr,
+                                                     Future<TLogCommitReply> in) {
+	double startTime = now();
+	TLogCommitReply t = co_await in;
 	if (now() - self->lastReset > SERVER_KNOBS->PUSH_RESET_INTERVAL) {
 		if (now() - startTime > SERVER_KNOBS->PUSH_MAX_LATENCY) {
 			if (self->resetCheck.isReady()) {
-				self->resetCheck = TagPartitionedLogSystem::pushResetChecker(self, addr);
+				self->resetCheck = LogSystem::pushResetChecker(self, addr);
 			}
 			self->slowReplies++;
 		} else {
@@ -602,14 +743,14 @@ ACTOR Future<TLogCommitReply> TagPartitionedLogSystem::recordPushMetrics(Referen
 		}
 	}
 	dist->sampleSeconds(now() - startTime);
-	return t;
+	co_return t;
 }
 
-Future<Version> TagPartitionedLogSystem::push(const ILogSystem::PushVersionSet& versionSet,
-                                              LogPushData& data,
-                                              SpanContext const& spanContext,
-                                              Optional<UID> debugID,
-                                              Optional<std::unordered_map<uint16_t, Version>> tpcvMap) {
+Future<Version> LogSystem::push(const LogPushVersionSet& versionSet,
+                                LogPushData& data,
+                                SpanContext const& spanContext,
+                                Optional<UID> debugID,
+                                Optional<std::unordered_map<uint16_t, Version>> tpcvMap) {
 	// FIXME: Randomize request order as in LegacyLogSystem?
 	Version prevVersion = versionSet.prevVersion; // this might be updated when version vector unicast is enabled
 	Version seqPrevVersion = versionSet.prevVersion; // a copy of the prevVersion provided by the sequencer
@@ -624,7 +765,7 @@ Future<Version> TagPartitionedLogSystem::push(const ILogSystem::PushVersionSet&
 			if (!it->isLocal) {
 				continue;
 			}
-			if (it->logServers.size() == 0) {
+			if (it->logServers.empty()) {
 				continue;
 			}
 			for (size_t loc = 0; loc < it->logServers.size(); loc++) {
@@ -648,12 +789,12 @@ Future<Version> TagPartitionedLogSystem::push(const ILogSystem::PushVersionSet&
 			// Remote TLogs should read from LogRouter
 			continue;
 		}
-		if (it->logServers.size() == 0) {
+		if (it->logServers.empty()) {
 			// Empty TLog set
 			continue;
 		}
 
-		if (it->connectionResetTrackers.size() == 0) {
+		if (it->connectionResetTrackers.empty()) {
 			for (int i = 0; i < it->logServers.size(); i++) {
 				it->connectionResetTrackers.push_back(makeReference<ConnectionResetInfo>());
 			}
@@ -675,7 +816,7 @@ Future<Version> TagPartitionedLogSystem::push(const ILogSystem::PushVersionSet&
 				if (tpcvMap.get().contains(location)) {
 					prevVersion = tpcvMap.get()[location];
 				} else {
-					ASSERT(!msg.size());
+					ASSERT(msg.empty());
 					location++;
 					continue;
 				}
@@ -716,7 +857,7 @@ Future<Version> TagPartitionedLogSystem::push(const ILogSystem::PushVersionSet&
 // buddy of, hence do not treat such a server as the best server. Note that this reset logic gets
 // invoked only in the context of peeks that get done during recovery, and the best server should
 // always be available for peeking after recovery is done.
-void TagPartitionedLogSystem::resetBestServerIfNotLocked(
+void LogSystem::resetBestServerIfNotLocked(
     int bestSet,
     int& bestServer,
     Optional<Version> end,
@@ -734,11 +875,7 @@ void TagPartitionedLogSystem::resetBestServerIfNotLocked(
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
-                                                                    Version begin,
-                                                                    Version end,
-                                                                    Tag tag,
-                                                                    bool parallelGetMore) {
+Reference<IPeekCursor> LogSystem::peekAll(UID dbgid, Version begin, Version end, Tag tag, bool parallelGetMore) {
 	int bestSet = 0;
 	std::vector<Reference<LogSet>> localSets;
 	Version lastBegin = 0;
@@ -749,7 +886,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 		if (log->locality == tagLocalitySpecial) {
 			foundSpecial = true;
 		}
-		if (log->isLocal && log->logServers.size() &&
+		if (log->isLocal && !log->logServers.empty() &&
 		    (log->locality == tagLocalitySpecial || log->locality == tag.locality || tag.locality == tagLocalityTxs ||
 		     tag.locality == tagLocalityLogRouter)) {
 			lastBegin = std::max(lastBegin, log->startVersion);
@@ -762,11 +899,11 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 		logIdx++;
 	}
 
-	if (!localSets.size()) {
+	if (localSets.empty()) {
 		lastBegin = end;
 	}
 
-	if (begin >= lastBegin && localSets.size()) {
+	if (begin >= lastBegin && !localSets.empty()) {
 		TraceEvent("TLogPeekAllCurrentOnly", dbgid)
 		    .detail("Tag", tag.toString())
 		    .detail("Begin", begin)
@@ -779,13 +916,13 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 			ASSERT_WE_THINK(knownLockedTLogIds.contains(bestSetIdx));
 			bestKnownLockedTLogIds = knownLockedTLogIds[bestSetIdx];
 		}
-		return makeReference<ILogSystem::SetPeekCursor>(
+		return makeReference<SetPeekCursor>(
 		    localSets, bestSet, bestServer, tag, begin, end, parallelGetMore, bestKnownLockedTLogIds);
 	} else {
-		std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+		std::vector<Reference<IPeekCursor>> cursors;
 		std::vector<LogMessageVersion> epochEnds;
 
-		if (lastBegin < end && localSets.size()) {
+		if (lastBegin < end && !localSets.empty()) {
 			TraceEvent("TLogPeekAllAddingCurrent", dbgid)
 			    .detail("Tag", tag.toString())
 			    .detail("Begin", begin)
@@ -798,7 +935,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 				ASSERT_WE_THINK(knownLockedTLogIds.contains(bestSetIdx));
 				bestKnownLockedTLogIds = knownLockedTLogIds[bestSetIdx];
 			}
-			cursors.push_back(makeReference<ILogSystem::SetPeekCursor>(
+			cursors.push_back(makeReference<SetPeekCursor>(
 			    localSets, bestSet, bestServer, tag, lastBegin, end, parallelGetMore, bestKnownLockedTLogIds));
 		}
 		for (int i = 0; begin < lastBegin; i++) {
@@ -812,7 +949,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 				    .detail("End", end)
 				    .detail("LastBegin", lastBegin)
 				    .detail("OldLogDataSize", oldLogData.size());
-				return makeReference<ILogSystem::ServerPeekCursor>(
+				return makeReference<ServerPeekCursor>(
 				    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false);
 			}
 
@@ -824,7 +961,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 				if (log->locality == tagLocalitySpecial) {
 					thisSpecial = true;
 				}
-				if (log->isLocal && log->logServers.size() &&
+				if (log->isLocal && !log->logServers.empty() &&
 				    (log->locality == tagLocalitySpecial || log->locality == tag.locality ||
 				     tag.locality == tagLocalityTxs || tag.locality == tagLocalityLogRouter)) {
 					thisBegin = std::max(thisBegin, log->startVersion);
@@ -835,16 +972,16 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 				}
 			}
 
-			if (!localOldSets.size()) {
+			if (localOldSets.empty()) {
 				TraceEvent("TLogPeekAllNoLocalSets", dbgid)
 				    .detail("Tag", tag.toString())
 				    .detail("Begin", begin)
 				    .detail("End", end)
 				    .detail("LastBegin", lastBegin);
-				if (!cursors.size() && !foundSpecial) {
+				if (cursors.empty() && !foundSpecial) {
 					continue;
 				}
-				return makeReference<ILogSystem::ServerPeekCursor>(
+				return makeReference<ServerPeekCursor>(
 				    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false);
 			}
 			if (thisSpecial) {
@@ -860,29 +997,28 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekAll(UID dbgid,
 					    .detail("BestLogs", localOldSets[bestOldSet]->logServerString())
 					    .detail("LastBegin", lastBegin)
 					    .detail("ThisBegin", thisBegin);
-					cursors.push_back(
-					    makeReference<ILogSystem::SetPeekCursor>(localOldSets,
-					                                             bestOldSet,
-					                                             localOldSets[bestOldSet]->bestLocationFor(tag),
-					                                             tag,
-					                                             thisBegin,
-					                                             std::min(lastBegin, end),
-					                                             parallelGetMore));
+					cursors.push_back(makeReference<SetPeekCursor>(localOldSets,
+					                                               bestOldSet,
+					                                               localOldSets[bestOldSet]->bestLocationFor(tag),
+					                                               tag,
+					                                               thisBegin,
+					                                               std::min(lastBegin, end),
+					                                               parallelGetMore));
 					epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end)));
 				}
 				lastBegin = thisBegin;
 			}
 		}
 
-		return makeReference<ILogSystem::MultiCursor>(cursors, epochEnds);
+		return makeReference<MultiCursor>(cursors, epochEnds);
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid,
-                                                                       Version begin,
-                                                                       Optional<Version> end,
-                                                                       Tag tag,
-                                                                       bool parallelGetMore) {
+Reference<IPeekCursor> LogSystem::peekRemote(UID dbgid,
+                                             Version begin,
+                                             Optional<Version> end,
+                                             Tag tag,
+                                             bool parallelGetMore) {
 	int bestSet = -1;
 	Version lastBegin = recoveredAt.present() ? recoveredAt.get() + 1 : 0;
 	for (int t = 0; t < tLogs.size(); t++) {
@@ -890,7 +1026,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 			lastBegin = std::max(lastBegin, tLogs[t]->startVersion);
 		}
 
-		if (tLogs[t]->logRouters.size()) {
+		if (!tLogs[t]->logRouters.empty()) {
 			ASSERT(bestSet == -1);
 			bestSet = t;
 		}
@@ -900,7 +1036,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 		    .detail("Tag", tag.toString())
 		    .detail("Begin", begin)
 		    .detail("End", end.present() ? end.get() : getPeekEnd());
-		return makeReference<ILogSystem::ServerPeekCursor>(
+		return makeReference<ServerPeekCursor>(
 		    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, parallelGetMore);
 	}
 	if (begin >= lastBegin) {
@@ -911,10 +1047,10 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 		    .detail("BestSet", bestSet)
 		    .detail("BestSetStart", lastBegin)
 		    .detail("LogRouterIds", tLogs[bestSet]->logRouterString());
-		return makeReference<ILogSystem::BufferedCursor>(
+		return makeReference<BufferedCursor>(
 		    tLogs[bestSet]->logRouters, tag, begin, end.present() ? end.get() + 1 : getPeekEnd(), parallelGetMore);
 	} else {
-		std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+		std::vector<Reference<IPeekCursor>> cursors;
 		std::vector<LogMessageVersion> epochEnds;
 		TraceEvent("TLogPeekRemoteAddingBest", dbgid)
 		    .detail("Tag", tag.toString())
@@ -923,7 +1059,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 		    .detail("BestSet", bestSet)
 		    .detail("BestSetStart", lastBegin)
 		    .detail("LogRouterIds", tLogs[bestSet]->logRouterString());
-		cursors.push_back(makeReference<ILogSystem::BufferedCursor>(
+		cursors.push_back(makeReference<BufferedCursor>(
 		    tLogs[bestSet]->logRouters, tag, lastBegin, end.present() ? end.get() + 1 : getPeekEnd(), parallelGetMore));
 		int i = 0;
 		while (begin < lastBegin) {
@@ -934,13 +1070,12 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 				    .detail("End", end.present() ? end.get() : getPeekEnd())
 				    .detail("LastBegin", lastBegin)
 				    .detail("OldLogDataSize", oldLogData.size());
-				return makeReference<ILogSystem::ServerPeekCursor>(
-				    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(),
-				    tag,
-				    begin,
-				    getPeekEnd(),
-				    false,
-				    parallelGetMore);
+				return makeReference<ServerPeekCursor>(Reference<AsyncVar<OptionalInterface<TLogInterface>>>(),
+				                                       tag,
+				                                       begin,
+				                                       getPeekEnd(),
+				                                       false,
+				                                       parallelGetMore);
 			}
 
 			int bestOldSet = -1;
@@ -950,7 +1085,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 					thisBegin = std::max(thisBegin, oldLogData[i].tLogs[t]->startVersion);
 				}
 
-				if (oldLogData[i].tLogs[t]->logRouters.size()) {
+				if (!oldLogData[i].tLogs[t]->logRouters.empty()) {
 					ASSERT(bestOldSet == -1);
 					bestOldSet = t;
 				}
@@ -960,13 +1095,12 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 				    .detail("Tag", tag.toString())
 				    .detail("Begin", begin)
 				    .detail("End", end.present() ? end.get() : getPeekEnd());
-				return makeReference<ILogSystem::ServerPeekCursor>(
-				    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(),
-				    tag,
-				    begin,
-				    getPeekEnd(),
-				    false,
-				    parallelGetMore);
+				return makeReference<ServerPeekCursor>(Reference<AsyncVar<OptionalInterface<TLogInterface>>>(),
+				                                       tag,
+				                                       begin,
+				                                       getPeekEnd(),
+				                                       false,
+				                                       parallelGetMore);
 			}
 
 			if (thisBegin < lastBegin) {
@@ -979,7 +1113,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 				    .detail("LastBegin", lastBegin)
 				    .detail("ThisBegin", thisBegin)
 				    .detail("BestStartVer", oldLogData[i].tLogs[bestOldSet]->startVersion);
-				cursors.push_back(makeReference<ILogSystem::BufferedCursor>(
+				cursors.push_back(makeReference<BufferedCursor>(
 				    oldLogData[i].tLogs[bestOldSet]->logRouters, tag, thisBegin, lastBegin, parallelGetMore));
 				epochEnds.emplace_back(lastBegin);
 				lastBegin = thisBegin;
@@ -987,18 +1121,14 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekRemote(UID dbgid
 			i++;
 		}
 
-		return makeReference<ILogSystem::MultiCursor>(cursors, epochEnds);
+		return makeReference<MultiCursor>(cursors, epochEnds);
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peek(UID dbgid,
-                                                                 Version begin,
-                                                                 Optional<Version> end,
-                                                                 Tag tag,
-                                                                 bool parallelGetMore) {
-	if (!tLogs.size()) {
+Reference<IPeekCursor> LogSystem::peek(UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore) {
+	if (tLogs.empty()) {
 		TraceEvent("TLogPeekNoLogSets", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
-		return makeReference<ILogSystem::ServerPeekCursor>(
+		return makeReference<ServerPeekCursor>(
 		    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false);
 	}
 
@@ -1009,14 +1139,14 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peek(UID dbgid,
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peek(UID dbgid,
-                                                                 Version begin,
-                                                                 Optional<Version> end,
-                                                                 std::vector<Tag> tags,
-                                                                 bool parallelGetMore) {
+Reference<IPeekCursor> LogSystem::peek(UID dbgid,
+                                       Version begin,
+                                       Optional<Version> end,
+                                       std::vector<Tag> tags,
+                                       bool parallelGetMore) {
 	if (tags.empty()) {
 		TraceEvent("TLogPeekNoTags", dbgid).detail("Begin", begin);
-		return makeReference<ILogSystem::ServerPeekCursor>(
+		return makeReference<ServerPeekCursor>(
 		    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), invalidTag, begin, getPeekEnd(), false, false);
 	}
 
@@ -1024,21 +1154,20 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peek(UID dbgid,
 		return peek(dbgid, begin, end, tags[0], parallelGetMore);
 	}
 
-	std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+	std::vector<Reference<IPeekCursor>> cursors;
 	cursors.reserve(tags.size());
 	for (auto tag : tags) {
 		cursors.push_back(peek(dbgid, begin, end, tag, parallelGetMore));
 	}
-	return makeReference<ILogSystem::BufferedCursor>(
-	    cursors, begin, end.present() ? end.get() + 1 : getPeekEnd(), true, false);
+	return makeReference<BufferedCursor>(cursors, begin, end.present() ? end.get() + 1 : getPeekEnd(), true, false);
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
-                                                                      Tag tag,
-                                                                      Version begin,
-                                                                      Version end,
-                                                                      bool useMergePeekCursors,
-                                                                      int8_t peekLocality) {
+Reference<IPeekCursor> LogSystem::peekLocal(UID dbgid,
+                                            Tag tag,
+                                            Version begin,
+                                            Version end,
+                                            bool useMergePeekCursors,
+                                            int8_t peekLocality) {
 	if (tag.locality >= 0 || tag.locality == tagLocalitySpecial) {
 		peekLocality = tag.locality;
 	}
@@ -1048,11 +1177,12 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
 	bool foundSpecial = false;
 	int logCount = 0;
 	for (int t = 0; t < tLogs.size(); t++) {
-		if (tLogs[t]->logServers.size() && tLogs[t]->locality != tagLocalitySatellite) {
+		if (!tLogs[t]->logServers.empty() && tLogs[t]->locality != tagLocalitySatellite) {
 			logCount++;
 		}
-		if (tLogs[t]->logServers.size() && (tLogs[t]->locality == tagLocalitySpecial ||
-		                                    tLogs[t]->locality == peekLocality || peekLocality == tagLocalitySpecial)) {
+		if (!tLogs[t]->logServers.empty() &&
+		    (tLogs[t]->locality == tagLocalitySpecial || tLogs[t]->locality == peekLocality ||
+		     peekLocality == tagLocalitySpecial)) {
 			if (tLogs[t]->locality == tagLocalitySpecial) {
 				foundSpecial = true;
 			}
@@ -1069,7 +1199,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
 		if (useMergePeekCursors || logCount > 1) {
 			throw worker_removed();
 		} else {
-			return makeReference<ILogSystem::ServerPeekCursor>(
+			return makeReference<ServerPeekCursor>(
 			    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false);
 		}
 	}
@@ -1090,24 +1220,24 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
 				ASSERT_WE_THINK(knownLockedTLogIds.contains(bestSet));
 				bestKnownLockedTLogIds = knownLockedTLogIds[bestSet];
 			}
-			return makeReference<ILogSystem::MergedPeekCursor>(tLogs[bestSet]->logServers,
-			                                                   bestServer,
-			                                                   tLogs[bestSet]->logServers.size() + 1 -
-			                                                       tLogs[bestSet]->tLogReplicationFactor,
-			                                                   tag,
-			                                                   begin,
-			                                                   end,
-			                                                   true,
-			                                                   tLogs[bestSet]->tLogLocalities,
-			                                                   tLogs[bestSet]->tLogPolicy,
-			                                                   tLogs[bestSet]->tLogReplicationFactor,
-			                                                   bestKnownLockedTLogIds);
+			return makeReference<MergedPeekCursor>(tLogs[bestSet]->logServers,
+			                                       bestServer,
+			                                       tLogs[bestSet]->logServers.size() + 1 -
+			                                           tLogs[bestSet]->tLogReplicationFactor,
+			                                       tag,
+			                                       begin,
+			                                       end,
+			                                       true,
+			                                       tLogs[bestSet]->tLogLocalities,
+			                                       tLogs[bestSet]->tLogPolicy,
+			                                       tLogs[bestSet]->tLogReplicationFactor,
+			                                       bestKnownLockedTLogIds);
 		} else {
-			return makeReference<ILogSystem::ServerPeekCursor>(
+			return makeReference<ServerPeekCursor>(
 			    tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor(tag)], tag, begin, end, false, false);
 		}
 	} else {
-		std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+		std::vector<Reference<IPeekCursor>> cursors;
 		std::vector<LogMessageVersion> epochEnds;
 
 		if (tLogs[bestSet]->startVersion < end) {
@@ -1126,32 +1256,32 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
 					ASSERT_WE_THINK(knownLockedTLogIds.contains(bestSet));
 					bestKnownLockedTLogIds = knownLockedTLogIds[bestSet];
 				}
-				cursors.push_back(makeReference<ILogSystem::MergedPeekCursor>(tLogs[bestSet]->logServers,
-				                                                              bestServer,
-				                                                              tLogs[bestSet]->logServers.size() + 1 -
-				                                                                  tLogs[bestSet]->tLogReplicationFactor,
-				                                                              tag,
-				                                                              tLogs[bestSet]->startVersion,
-				                                                              end,
-				                                                              true,
-				                                                              tLogs[bestSet]->tLogLocalities,
-				                                                              tLogs[bestSet]->tLogPolicy,
-				                                                              tLogs[bestSet]->tLogReplicationFactor,
-				                                                              bestKnownLockedTLogIds));
+				cursors.push_back(makeReference<MergedPeekCursor>(tLogs[bestSet]->logServers,
+				                                                  bestServer,
+				                                                  tLogs[bestSet]->logServers.size() + 1 -
+				                                                      tLogs[bestSet]->tLogReplicationFactor,
+				                                                  tag,
+				                                                  tLogs[bestSet]->startVersion,
+				                                                  end,
+				                                                  true,
+				                                                  tLogs[bestSet]->tLogLocalities,
+				                                                  tLogs[bestSet]->tLogPolicy,
+				                                                  tLogs[bestSet]->tLogReplicationFactor,
+				                                                  bestKnownLockedTLogIds));
 			} else {
-				cursors.push_back(makeReference<ILogSystem::ServerPeekCursor>(
-				    tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor(tag)],
-				    tag,
-				    tLogs[bestSet]->startVersion,
-				    end,
-				    false,
-				    false));
+				cursors.push_back(
+				    makeReference<ServerPeekCursor>(tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor(tag)],
+				                                    tag,
+				                                    tLogs[bestSet]->startVersion,
+				                                    end,
+				                                    false,
+				                                    false));
 			}
 		}
 		Version lastBegin = tLogs[bestSet]->startVersion;
 		for (int i = 0; begin < lastBegin; i++) {
 			if (i == oldLogData.size()) {
-				if (tag.locality == tagLocalityTxs && cursors.size()) {
+				if (tag.locality == tagLocalityTxs && !cursors.empty()) {
 					break;
 				}
 				TraceEvent("TLogPeekLocalDead", dbgid)
@@ -1167,11 +1297,11 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
 			logCount = 0;
 			bool nextFoundSpecial = false;
 			for (int t = 0; t < oldLogData[i].tLogs.size(); t++) {
-				if (oldLogData[i].tLogs[t]->logServers.size() &&
+				if (!oldLogData[i].tLogs[t]->logServers.empty() &&
 				    oldLogData[i].tLogs[t]->locality != tagLocalitySatellite) {
 					logCount++;
 				}
-				if (oldLogData[i].tLogs[t]->logServers.size() &&
+				if (!oldLogData[i].tLogs[t]->logServers.empty() &&
 				    (oldLogData[i].tLogs[t]->locality == tagLocalitySpecial ||
 				     oldLogData[i].tLogs[t]->locality == peekLocality || peekLocality == tagLocalitySpecial)) {
 					if (oldLogData[i].tLogs[t]->locality == tagLocalitySpecial) {
@@ -1225,37 +1355,37 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLocal(UID dbgid,
 					// detail("LogId",
 					// oldLogData[i].tLogs[bestOldSet]->logServers[tLogs[bestOldSet]->bestLocationFor( tag
 					// )]->get().id());
-					cursors.push_back(makeReference<ILogSystem::MergedPeekCursor>(
-					    oldLogData[i].tLogs[bestOldSet]->logServers,
-					    oldLogData[i].tLogs[bestOldSet]->bestLocationFor(tag),
-					    oldLogData[i].tLogs[bestOldSet]->logServers.size() + 1 -
-					        oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor,
-					    tag,
-					    thisBegin,
-					    std::min(lastBegin, end),
-					    useMergePeekCursors,
-					    oldLogData[i].tLogs[bestOldSet]->tLogLocalities,
-					    oldLogData[i].tLogs[bestOldSet]->tLogPolicy,
-					    oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor));
+					cursors.push_back(
+					    makeReference<MergedPeekCursor>(oldLogData[i].tLogs[bestOldSet]->logServers,
+					                                    oldLogData[i].tLogs[bestOldSet]->bestLocationFor(tag),
+					                                    oldLogData[i].tLogs[bestOldSet]->logServers.size() + 1 -
+					                                        oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor,
+					                                    tag,
+					                                    thisBegin,
+					                                    std::min(lastBegin, end),
+					                                    useMergePeekCursors,
+					                                    oldLogData[i].tLogs[bestOldSet]->tLogLocalities,
+					                                    oldLogData[i].tLogs[bestOldSet]->tLogPolicy,
+					                                    oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor));
 					epochEnds.emplace_back(std::min(lastBegin, end));
 				}
 				lastBegin = thisBegin;
 			}
 		}
 
-		return makeReference<ILogSystem::MultiCursor>(cursors, epochEnds);
+		return makeReference<MultiCursor>(cursors, epochEnds);
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekTxs(UID dbgid,
-                                                                    Version begin,
-                                                                    int8_t peekLocality,
-                                                                    Version localEnd,
-                                                                    bool canDiscardPopped) {
+Reference<IPeekCursor> LogSystem::peekTxs(UID dbgid,
+                                          Version begin,
+                                          int8_t peekLocality,
+                                          Version localEnd,
+                                          bool canDiscardPopped) {
 	Version end = getEnd();
-	if (!tLogs.size()) {
+	if (tLogs.empty()) {
 		TraceEvent("TLogPeekTxsNoLogs", dbgid).log();
-		return makeReference<ILogSystem::ServerPeekCursor>(
+		return makeReference<ServerPeekCursor>(
 		    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), invalidTag, begin, end, false, false);
 	}
 	TraceEvent("TLogPeekTxs", dbgid)
@@ -1271,70 +1401,70 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekTxs(UID dbgid,
 	}
 
 	if (peekLocality < 0 || localEnd == invalidVersion || localEnd <= begin) {
-		std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+		std::vector<Reference<IPeekCursor>> cursors;
 		cursors.reserve(maxTxsTags);
 		for (int i = 0; i < maxTxsTags; i++) {
 			cursors.push_back(peekAll(dbgid, begin, end, Tag(tagLocalityTxs, i), true));
 		}
 
-		return makeReference<ILogSystem::BufferedCursor>(cursors, begin, end, false, canDiscardPopped);
+		return makeReference<BufferedCursor>(cursors, begin, end, false, canDiscardPopped);
 	}
 
 	try {
 		if (localEnd >= end) {
-			std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+			std::vector<Reference<IPeekCursor>> cursors;
 			cursors.reserve(maxTxsTags);
 			for (int i = 0; i < maxTxsTags; i++) {
 				cursors.push_back(peekLocal(dbgid, Tag(tagLocalityTxs, i), begin, end, true, peekLocality));
 			}
 
-			return makeReference<ILogSystem::BufferedCursor>(cursors, begin, end, false, canDiscardPopped);
+			return makeReference<BufferedCursor>(cursors, begin, end, false, canDiscardPopped);
 		}
 
-		std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+		std::vector<Reference<IPeekCursor>> cursors;
 		std::vector<LogMessageVersion> epochEnds;
 
 		cursors.resize(2);
 
-		std::vector<Reference<ILogSystem::IPeekCursor>> localCursors;
-		std::vector<Reference<ILogSystem::IPeekCursor>> allCursors;
+		std::vector<Reference<IPeekCursor>> localCursors;
+		std::vector<Reference<IPeekCursor>> allCursors;
 		for (int i = 0; i < maxTxsTags; i++) {
 			localCursors.push_back(peekLocal(dbgid, Tag(tagLocalityTxs, i), begin, localEnd, true, peekLocality));
 			allCursors.push_back(peekAll(dbgid, localEnd, end, Tag(tagLocalityTxs, i), true));
 		}
 
-		cursors[1] = makeReference<ILogSystem::BufferedCursor>(localCursors, begin, localEnd, false, canDiscardPopped);
-		cursors[0] = makeReference<ILogSystem::BufferedCursor>(allCursors, localEnd, end, false, false);
+		cursors[1] = makeReference<BufferedCursor>(localCursors, begin, localEnd, false, canDiscardPopped);
+		cursors[0] = makeReference<BufferedCursor>(allCursors, localEnd, end, false, false);
 		epochEnds.emplace_back(localEnd);
 
-		return makeReference<ILogSystem::MultiCursor>(cursors, epochEnds);
+		return makeReference<MultiCursor>(cursors, epochEnds);
 	} catch (Error& e) {
 		if (e.code() == error_code_worker_removed) {
-			std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+			std::vector<Reference<IPeekCursor>> cursors;
 			cursors.reserve(maxTxsTags);
 			for (int i = 0; i < maxTxsTags; i++) {
 				cursors.push_back(peekAll(dbgid, begin, end, Tag(tagLocalityTxs, i), true));
 			}
 
-			return makeReference<ILogSystem::BufferedCursor>(cursors, begin, end, false, canDiscardPopped);
+			return makeReference<BufferedCursor>(cursors, begin, end, false, canDiscardPopped);
 		}
 		throw;
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekSingle(UID dbgid,
-                                                                       Version begin,
-                                                                       Tag tag,
-                                                                       std::vector<std::pair<Version, Tag>> history) {
-	while (history.size() && begin >= history.back().first) {
+Reference<IPeekCursor> LogSystem::peekSingle(UID dbgid,
+                                             Version begin,
+                                             Tag tag,
+                                             std::vector<std::pair<Version, Tag>> history) {
+	while (!history.empty() && begin >= history.back().first) {
 		history.pop_back();
 	}
 
-	if (history.size() == 0) {
+	if (history.empty()) {
 		TraceEvent("TLogPeekSingleNoHistory", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
 		return peekLocal(dbgid, tag, begin, getPeekEnd(), false);
 	} else {
-		std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+		std::vector<Reference<IPeekCursor>> cursors;
 		std::vector<LogMessageVersion> epochEnds;
 
 		TraceEvent("TLogPeekSingleAddingLocal", dbgid).detail("Tag", tag.toString()).detail("Begin", history[0].first);
@@ -1354,11 +1484,11 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekSingle(UID dbgid
 			epochEnds.emplace_back(history[i].first);
 		}
 
-		return makeReference<ILogSystem::MultiCursor>(cursors, epochEnds);
+		return makeReference<MultiCursor>(cursors, epochEnds);
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(
+Reference<IPeekCursor> LogSystem::peekLogRouter(
     UID dbgid,
     Version begin,
     Tag tag,
@@ -1389,7 +1519,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(
 			int bestPrimarySetIdx = -1;
 			int bestSatelliteSetIdx = -1;
 			for (auto& log : tLogs) {
-				if (log->isLocal && log->logServers.size()) {
+				if (log->isLocal && !log->logServers.empty()) {
 					TraceEvent("TLogPeekLogRouterLocalSet", dbgid)
 					    .detail("Tag", tag.toString())
 					    .detail("Begin", begin)
@@ -1423,14 +1553,14 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(
 			TraceEvent("TLogPeekLogRouterSets", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
 			// FIXME: do this merge on one of the logs in the other data center to avoid sending multiple copies
 			// across the WAN
-			return makeReference<ILogSystem::SetPeekCursor>(
+			return makeReference<SetPeekCursor>(
 			    localSets, bestSet, bestServer, tag, begin, end.get(), true, bestKnownStoppedTLogIds);
 		} else {
 			int bestPrimarySet = -1;
 			int bestSatelliteSet = -1;
 			for (int i = 0; i < tLogs.size(); i++) {
 				const auto& log = tLogs[i];
-				if (log->logServers.size() && log->isLocal) {
+				if (!log->logServers.empty() && log->isLocal) {
 					if (log->locality == tagLocalitySatellite) {
 						bestSatelliteSet = i;
 						break;
@@ -1449,7 +1579,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(
 			    .detail("Tag", tag.toString())
 			    .detail("Begin", begin)
 			    .detail("LogId", log->logServers[log->bestLocationFor(tag)]->get().id());
-			return makeReference<ILogSystem::ServerPeekCursor>(
+			return makeReference<ServerPeekCursor>(
 			    log->logServers[log->bestLocationFor(tag)], tag, begin, end.get(), false, true);
 		}
 	}
@@ -1463,11 +1593,22 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(
 			}
 		}
 		if (found) {
+			Version oldEnd = firstOld && recoveredAt.present() ? recoveredAt.get() + 1 : old.epochEnd;
+			if (begin > oldEnd) {
+				TraceEvent("TLogPeekLogRouterSkipEmptyOldRange", dbgid)
+				    .detail("Tag", tag.toString())
+				    .detail("Begin", begin)
+				    .detail("OldEnd", oldEnd)
+				    .detail("FirstOld", firstOld);
+				firstOld = false;
+				continue;
+			}
+
 			int bestPrimarySet = 0;
 			int bestSatelliteSet = -1;
 			std::vector<Reference<LogSet>> localSets;
 			for (auto& log : old.tLogs) {
-				if (log->isLocal && log->logServers.size()) {
+				if (log->isLocal && !log->logServers.empty()) {
 					TraceEvent("TLogPeekLogRouterOldLocalSet", dbgid)
 					    .detail("Tag", tag.toString())
 					    .detail("Begin", begin)
@@ -1493,25 +1634,19 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(
 			    .detail("FirstOld", firstOld);
 			// FIXME: do this merge on one of the logs in the other data center to avoid sending multiple copies
 			// across the WAN
-			return makeReference<ILogSystem::SetPeekCursor>(localSets,
-			                                                bestSet,
-			                                                localSets[bestSet]->bestLocationFor(tag),
-			                                                tag,
-			                                                begin,
-			                                                firstOld && recoveredAt.present() ? recoveredAt.get() + 1
-			                                                                                  : old.epochEnd,
-			                                                true);
+			return makeReference<SetPeekCursor>(
+			    localSets, bestSet, localSets[bestSet]->bestLocationFor(tag), tag, begin, oldEnd, true);
 		}
 		firstOld = false;
 	}
-	return makeReference<ILogSystem::ServerPeekCursor>(
+	return makeReference<ServerPeekCursor>(
 	    Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, end.get(), false, false);
 }
 
-Version TagPartitionedLogSystem::getKnownCommittedVersion() {
+Version LogSystem::getKnownCommittedVersion() {
 	Version result = invalidVersion;
 	for (auto& it : lockResults) {
-		auto durableVersionInfo = TagPartitionedLogSystem::getDurableVersion(dbgid, it);
+		auto durableVersionInfo = LogSystem::getDurableVersion(dbgid, it);
 		if (durableVersionInfo.present()) {
 			result = std::max(result, durableVersionInfo.get().knownCommittedVersion);
 		}
@@ -1519,25 +1654,22 @@ Version TagPartitionedLogSystem::getKnownCommittedVersion() {
 	return result;
 }
 
-Future<Void> TagPartitionedLogSystem::onKnownCommittedVersionChange() {
+Future<Void> LogSystem::onKnownCommittedVersionChange() {
 	std::vector<Future<Void>> result;
 	for (auto& it : lockResults) {
-		result.push_back(TagPartitionedLogSystem::getDurableVersionChanged(it));
+		result.push_back(LogSystem::getDurableVersionChanged(it));
 	}
-	if (!result.size()) {
+	if (result.empty()) {
 		return Never();
 	}
 	return waitForAny(result);
 }
 
-void TagPartitionedLogSystem::popLogRouter(Version upTo,
-                                           Tag tag,
-                                           Version durableKnownCommittedVersion,
-                                           int8_t popLocality) {
+void LogSystem::popLogRouter(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) {
 	if (!upTo)
 		return;
 
-	Version lastGenerationStartVersion = TagPartitionedLogSystem::getMaxLocalStartVersion(tLogs);
+	Version lastGenerationStartVersion = LogSystem::getMaxLocalStartVersion(tLogs);
 	if (upTo >= lastGenerationStartVersion) {
 		for (auto& t : tLogs) {
 			if (t->locality == popLocality) {
@@ -1562,7 +1694,7 @@ void TagPartitionedLogSystem::popLogRouter(Version upTo,
 
 	Version nextGenerationStartVersion = lastGenerationStartVersion;
 	for (const auto& old : oldLogData) {
-		Version generationStartVersion = TagPartitionedLogSystem::getMaxLocalStartVersion(old.tLogs);
+		Version generationStartVersion = LogSystem::getMaxLocalStartVersion(old.tLogs);
 		if (generationStartVersion <= upTo) {
 			for (auto& t : old.tLogs) {
 				if (t->locality == popLocality) {
@@ -1593,13 +1725,13 @@ void TagPartitionedLogSystem::popLogRouter(Version upTo,
 	}
 }
 
-void TagPartitionedLogSystem::popTxs(Version upTo, int8_t popLocality) {
+void LogSystem::popTxs(Version upTo, int8_t popLocality) {
 	for (int i = 0; i < txsTags; i++) {
 		pop(upTo, Tag(tagLocalityTxs, i), 0, popLocality);
 	}
 }
 
-void TagPartitionedLogSystem::pop(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) {
+void LogSystem::pop(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality) {
 	if (upTo <= 0)
 		return;
 	if (tag.locality == tagLocalityRemoteLog) {
@@ -1625,28 +1757,28 @@ void TagPartitionedLogSystem::pop(Version upTo, Tag tag, Version durableKnownCom
 	}
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::popFromLog(TagPartitionedLogSystem* self,
-                                                       Reference<AsyncVar<OptionalInterface<TLogInterface>>> log,
-                                                       Tag tag,
-                                                       double delayBeforePop,
-                                                       bool popLogRouter) {
-	state Version last = 0;
-	loop {
-		wait(delay(delayBeforePop, TaskPriority::TLogPop));
+Future<Void> LogSystem::popFromLog(LogSystem* self,
+                                   Reference<AsyncVar<OptionalInterface<TLogInterface>>> log,
+                                   Tag tag,
+                                   double delayBeforePop,
+                                   bool popLogRouter) {
+	Version last = 0;
+	while (true) {
+		co_await delay(delayBeforePop, TaskPriority::TLogPop);
 
 		// to: first is upto version, second is durableKnownComittedVersion
-		state std::pair<Version, Version> to = self->outstandingPops[std::make_pair(log->get().id(), tag)];
+		std::pair<Version, Version> to = self->outstandingPops[std::make_pair(log->get().id(), tag)];
 
 		if (to.first <= last) {
 			self->outstandingPops.erase(std::make_pair(log->get().id(), tag));
-			return Void();
+			co_return;
 		}
 
 		try {
 			if (!log->get().present())
-				return Void();
-			wait(log->get().interf().popMessages.getReply(TLogPopRequest(to.first, to.second, tag),
-			                                              TaskPriority::TLogPop));
+				co_return;
+			co_await log->get().interf().popMessages.getReply(TLogPopRequest(to.first, to.second, tag),
+			                                                  TaskPriority::TLogPop);
 
 			if (popLogRouter) {
 				self->logRouterLastPops[std::make_pair(log->get().id(), tag)] = to.first;
@@ -1659,59 +1791,61 @@ ACTOR Future<Void> TagPartitionedLogSystem::popFromLog(TagPartitionedLogSystem*
 			TraceEvent((e.code() == error_code_broken_promise) ? SevInfo : SevError, "LogPopError", self->dbgid)
 			    .error(e)
 			    .detail("Log", log->get().id());
-			return Void(); // Leaving outstandingPops filled in means no further pop requests to this tlog from this
+			co_return; // Leaving outstandingPops filled in means no further pop requests to this tlog from this
 			// logSystem
 		}
 	}
 }
 
-ACTOR Future<Version> TagPartitionedLogSystem::getPoppedFromTLog(
-    Reference<AsyncVar<OptionalInterface<TLogInterface>>> log,
-    Tag tag) {
+Future<Version> LogSystem::getPoppedFromTLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> log, Tag tag) {
 
-	loop {
-		choose {
-			when(TLogPeekReply rep =
-			         wait(log->get().present() ? brokenPromiseToNever(log->get().interf().peekMessages.getReply(
-			                                         TLogPeekRequest(-1, tag, false, false)))
-			                                   : Never())) {
-				ASSERT(rep.popped.present());
-				return rep.popped.get();
-			}
-			when(wait(log->onChange())) {}
+	while (true) {
+		const auto logInterface = log->get();
+		if (!logInterface.present()) {
+			co_await log->onChange();
+			continue;
+		}
+
+		auto res = co_await race(
+		    brokenPromiseToNever(logInterface.interf().peekMessages.getReply(TLogPeekRequest(-1, tag, false, false))),
+		    log->onChange());
+		if (res.index() == 0) {
+			TLogPeekReply rep = std::get<0>(std::move(res));
+
+			ASSERT(rep.popped.present());
+			co_return rep.popped.get();
 		}
 	}
 }
 
-ACTOR Future<Version> TagPartitionedLogSystem::getPoppedTxs(TagPartitionedLogSystem* self) {
-	state std::vector<std::vector<Future<Version>>> poppedFutures;
-	state std::vector<Future<Void>> poppedReady;
-	if (self->tLogs.size()) {
+Future<Version> LogSystem::getPoppedTxs(LogSystem* self) {
+	std::vector<std::vector<Future<Version>>> poppedFutures;
+	std::vector<Future<Void>> poppedReady;
+	if (!self->tLogs.empty()) {
 		poppedFutures.push_back(std::vector<Future<Version>>());
 		for (auto& it : self->tLogs) {
 			for (auto& log : it->logServers) {
-				poppedFutures.back().push_back(TagPartitionedLogSystem::getPoppedFromTLog(log, Tag(tagLocalityTxs, 0)));
+				poppedFutures.back().push_back(LogSystem::getPoppedFromTLog(log, Tag(tagLocalityTxs, 0)));
 			}
 		}
 		poppedReady.push_back(waitForAny(poppedFutures.back()));
 	}
 
 	for (auto& old : self->oldLogData) {
-		if (old.tLogs.size()) {
+		if (!old.tLogs.empty()) {
 			poppedFutures.push_back(std::vector<Future<Version>>());
 			for (auto& it : old.tLogs) {
 				for (auto& log : it->logServers) {
-					poppedFutures.back().push_back(
-					    TagPartitionedLogSystem::getPoppedFromTLog(log, Tag(tagLocalityTxs, 0)));
+					poppedFutures.back().push_back(LogSystem::getPoppedFromTLog(log, Tag(tagLocalityTxs, 0)));
 				}
 			}
 			poppedReady.push_back(waitForAny(poppedFutures.back()));
 		}
 	}
 
-	state UID dbgid = self->dbgid;
-	state Future<Void> maxGetPoppedDuration = delay(SERVER_KNOBS->TXS_POPPED_MAX_DELAY);
-	wait(waitForAll(poppedReady) || maxGetPoppedDuration);
+	UID dbgid = self->dbgid;
+	Future<Void> maxGetPoppedDuration = delay(SERVER_KNOBS->TXS_POPPED_MAX_DELAY);
+	co_await (waitForAll(poppedReady) || maxGetPoppedDuration);
 
 	if (maxGetPoppedDuration.isReady()) {
 		TraceEvent(SevWarnAlways, "PoppedTxsNotReady", dbgid).log();
@@ -1725,14 +1859,14 @@ ACTOR Future<Version> TagPartitionedLogSystem::getPoppedTxs(TagPartitionedLogSys
 			}
 		}
 	}
-	return maxPopped;
+	co_return maxPopped;
 }
 
-Future<Version> TagPartitionedLogSystem::getTxsPoppedVersion() {
+Future<Version> LogSystem::getTxsPoppedVersion() {
 	return getPoppedTxs(this);
 }
 
-void TagPartitionedLogSystem::updateLogRouter(int logSetIndex, int tagId, TLogInterface const& newLogRouter) {
+void LogSystem::updateLogRouter(int logSetIndex, int tagId, TLogInterface const& newLogRouter) {
 	ASSERT(tagId >= 0 && tagId < tLogs[logSetIndex]->logRouters.size());
 	auto& logSet = tLogs[logSetIndex];
 
@@ -1747,8 +1881,8 @@ void TagPartitionedLogSystem::updateLogRouter(int logSetIndex, int tagId, TLogIn
 	    .detail("Locality", logSet->locality);
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::confirmEpochLive_internal(Reference<LogSet> logSet, Optional<UID> debugID) {
-	state std::vector<Future<Void>> alive;
+Future<Void> LogSystem::confirmEpochLive_internal(Reference<LogSet> logSet, Optional<UID> debugID) {
+	std::vector<Future<Void>> alive;
 	int numPresent = 0;
 	for (auto& t : logSet->logServers) {
 		if (t->get().present()) {
@@ -1760,11 +1894,11 @@ ACTOR Future<Void> TagPartitionedLogSystem::confirmEpochLive_internal(Reference<
 		}
 	}
 
-	wait(quorum(alive, std::min(logSet->tLogReplicationFactor, numPresent - logSet->tLogWriteAntiQuorum)));
+	co_await quorum(alive, std::min(logSet->tLogReplicationFactor, numPresent - logSet->tLogWriteAntiQuorum));
 
-	state std::vector<LocalityEntry> aliveEntries;
-	state std::vector<bool> responded(alive.size(), false);
-	loop {
+	std::vector<LocalityEntry> aliveEntries;
+	std::vector<bool> responded(alive.size(), false);
+	while (true) {
 		for (int i = 0; i < alive.size(); i++) {
 			if (!responded[i] && alive[i].isReady() && !alive[i].isError()) {
 				aliveEntries.push_back(logSet->logEntryArray[i]);
@@ -1773,7 +1907,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::confirmEpochLive_internal(Reference<
 		}
 
 		if (logSet->satisfiesPolicy(aliveEntries)) {
-			return Void();
+			co_return;
 		}
 
 		// The current set of responders that we have weren't enough to form a quorum, so we must
@@ -1785,18 +1919,18 @@ ACTOR Future<Void> TagPartitionedLogSystem::confirmEpochLive_internal(Reference<
 			} else if (alive[i].isReady() && alive[i].isError() &&
 			           alive[i].getError().code() == error_code_tlog_stopped) {
 				// All commits must go to all TLogs.  If any TLog is stopped, then our epoch has ended.
-				return Never();
+				co_await Future<Void>(Never());
 			}
 		}
-		ASSERT(changes.size() != 0);
-		wait(waitForAny(changes));
+		ASSERT(!changes.empty());
+		co_await waitForAny(changes);
 	}
 }
 
-Future<Void> TagPartitionedLogSystem::confirmEpochLive(Optional<UID> debugID) {
+Future<Void> LogSystem::confirmEpochLive(Optional<UID> debugID) {
 	std::vector<Future<Void>> quorumResults;
 	for (auto& it : tLogs) {
-		if (it->isLocal && it->logServers.size()) {
+		if (it->isLocal && !it->logServers.empty()) {
 			quorumResults.push_back(confirmEpochLive_internal(it, debugID));
 		}
 	}
@@ -1804,7 +1938,7 @@ Future<Void> TagPartitionedLogSystem::confirmEpochLive(Optional<UID> debugID) {
 	return waitForAll(quorumResults);
 }
 
-Future<Void> TagPartitionedLogSystem::endEpoch() {
+Future<Void> LogSystem::endEpoch() {
 	std::vector<Future<Void>> lockResults;
 	for (auto& logSet : tLogs) {
 		for (auto& log : logSet->logServers) {
@@ -1814,17 +1948,16 @@ Future<Void> TagPartitionedLogSystem::endEpoch() {
 	return waitForAll(lockResults);
 }
 
-Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
-    RecruitFromConfigurationReply const& recr,
-    Future<RecruitRemoteFromConfigurationReply> const& fRemoteWorkers,
-    DatabaseConfiguration const& config,
-    LogEpoch recoveryCount,
-    Version recoveryTransactionVersion,
-    int8_t primaryLocality,
-    int8_t remoteLocality,
-    std::vector<Tag> const& allTags,
-    Reference<AsyncVar<bool>> const& recruitmentStalled) {
-	return newEpoch(Reference<TagPartitionedLogSystem>::addRef(this),
+Future<Reference<LogSystem>> LogSystem::newEpoch(RecruitFromConfigurationReply const& recr,
+                                                 Future<RecruitRemoteFromConfigurationReply> const& fRemoteWorkers,
+                                                 DatabaseConfiguration const& config,
+                                                 LogEpoch recoveryCount,
+                                                 Version recoveryTransactionVersion,
+                                                 int8_t primaryLocality,
+                                                 int8_t remoteLocality,
+                                                 std::vector<Tag> const& allTags,
+                                                 Reference<AsyncVar<bool>> const& recruitmentStalled) {
+	return newEpoch(Reference<LogSystem>::addRef(this),
 	                recr,
 	                fRemoteWorkers,
 	                config,
@@ -1836,11 +1969,11 @@ Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 	                recruitmentStalled);
 }
 
-LogSystemType TagPartitionedLogSystem::getLogSystemType() const {
+LogSystemType LogSystem::getLogSystemType() const {
 	return logSystemType;
 }
 
-LogSystemConfig TagPartitionedLogSystem::getLogSystemConfig() const {
+LogSystemConfig LogSystem::getLogSystemConfig() const {
 	LogSystemConfig logSystemConfig(epoch);
 	logSystemConfig.logSystemType = logSystemType;
 	logSystemConfig.expectedLogSets = expectedLogSets;
@@ -1854,19 +1987,19 @@ LogSystemConfig TagPartitionedLogSystem::getLogSystemConfig() const {
 	logSystemConfig.knownLockedTLogIds = knownLockedTLogIds;
 	for (const Reference<LogSet>& logSet : tLogs) {
 		if (logSet->isLocal || remoteLogsWrittenToCoreState) {
-			logSystemConfig.tLogs.emplace_back(*logSet);
+			logSystemConfig.tLogs.push_back(toTLogSet(*logSet));
 		}
 	}
 
 	if (!recoveryCompleteWrittenToCoreState.get()) {
 		for (const auto& oldData : oldLogData) {
-			logSystemConfig.oldTLogs.emplace_back(oldData);
+			logSystemConfig.oldTLogs.push_back(toOldTLogConf(oldData));
 		}
 	}
 	return logSystemConfig;
 }
 
-Standalone<StringRef> TagPartitionedLogSystem::getLogsValue() const {
+Standalone<StringRef> LogSystem::getLogsValue() const {
 	std::vector<std::pair<UID, NetworkAddress>> logs;
 	std::vector<std::pair<UID, NetworkAddress>> oldLogs;
 	for (auto& t : tLogs) {
@@ -1892,7 +2025,7 @@ Standalone<StringRef> TagPartitionedLogSystem::getLogsValue() const {
 	return logsValue(logs, oldLogs);
 }
 
-Future<Void> TagPartitionedLogSystem::onLogSystemConfigChange() {
+Future<Void> LogSystem::onLogSystemConfigChange() {
 	std::vector<Future<Void>> changes;
 	changes.push_back(logSystemConfigChanged.onTrigger());
 	for (auto& t : tLogs) {
@@ -1915,12 +2048,12 @@ Future<Void> TagPartitionedLogSystem::onLogSystemConfigChange() {
 	return waitForAny(changes);
 }
 
-Version TagPartitionedLogSystem::getEnd() const {
+Version LogSystem::getEnd() const {
 	ASSERT(recoverAt.present());
 	return recoverAt.get() + 1;
 }
 
-Version TagPartitionedLogSystem::getPeekEnd() const {
+Version LogSystem::getPeekEnd() const {
 	if (recoverAt.present())
 		return getEnd();
 	else
@@ -1936,12 +2069,11 @@ Version TagPartitionedLogSystem::getPeekEnd() const {
  * Returns a vector of Reference<LocalitySet> objects, where each LocalitySet is
  * restricted to the provided locations that fall within its range.
  */
-std::vector<Reference<LocalitySet>> TagPartitionedLogSystem::getPushLocationsForTags(
-    std::vector<int>& fromLocations) const {
+std::vector<Reference<LocalitySet>> LogSystem::getPushLocationsForTags(std::vector<int>& fromLocations) const {
 	std::vector<Reference<LocalitySet>> restrictedLogSets;
 	int locationOffset = 0;
 	for (auto& log : tLogs) {
-		if (!log->isLocal || !log->logServers.size()) {
+		if (!log->isLocal || log->logServers.empty()) {
 			locationOffset += log->logServers.size();
 			continue;
 		}
@@ -1958,14 +2090,14 @@ std::vector<Reference<LocalitySet>> TagPartitionedLogSystem::getPushLocationsFor
 	return restrictedLogSets;
 }
 
-void TagPartitionedLogSystem::getPushLocations(VectorRef<Tag> tags,
-                                               std::vector<int>& locations,
-                                               bool allLocations,
-                                               Optional<std::vector<Reference<LocalitySet>>> fromLocations) const {
+void LogSystem::getPushLocations(VectorRef<Tag> tags,
+                                 std::vector<int>& locations,
+                                 bool allLocations,
+                                 Optional<std::vector<Reference<LocalitySet>>> fromLocations) const {
 	int locationOffset = 0;
 	int setIndex = 0;
 	for (auto& logSet : tLogs) {
-		if (logSet->isLocal && logSet->logServers.size()) {
+		if (logSet->isLocal && !logSet->logServers.empty()) {
 			if (fromLocations.present()) {
 				logSet->getPushLocations(tags, locations, locationOffset, allLocations, fromLocations.get()[setIndex]);
 				setIndex++;
@@ -1977,36 +2109,35 @@ void TagPartitionedLogSystem::getPushLocations(VectorRef<Tag> tags,
 	}
 }
 
-bool TagPartitionedLogSystem::hasRemoteLogs() const {
-	return logRouterTags > 0 || pseudoLocalities.size() > 0;
+bool LogSystem::hasRemoteLogs() const {
+	return logRouterTags > 0 || !pseudoLocalities.empty();
 }
 
-Tag TagPartitionedLogSystem::getRandomRouterTag() const {
+Tag LogSystem::getRandomRouterTag() const {
 	return Tag(tagLocalityLogRouter, deterministicRandom()->randomInt(0, logRouterTags));
 }
 
-Tag TagPartitionedLogSystem::getRandomTxsTag() const {
+Tag LogSystem::getRandomTxsTag() const {
 	return Tag(tagLocalityTxs, deterministicRandom()->randomInt(0, txsTags));
 }
 
-TLogVersion TagPartitionedLogSystem::getTLogVersion() const {
+TLogVersion LogSystem::getTLogVersion() const {
 	return tLogs[0]->tLogVersion;
 }
 
-int TagPartitionedLogSystem::getLogRouterTags() const {
+int LogSystem::getLogRouterTags() const {
 	return logRouterTags;
 }
 
-Version TagPartitionedLogSystem::getBackupStartVersion() const {
-	ASSERT(tLogs.size() > 0);
+Version LogSystem::getBackupStartVersion() const {
+	ASSERT(!tLogs.empty());
 	return backupStartVersion;
 }
 
-std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> TagPartitionedLogSystem::getOldEpochTagsVersionsInfo() const {
+std::map<LogEpoch, EpochTagsVersionsInfo> LogSystem::getOldEpochTagsVersionsInfo() const {
 	std::map<LogEpoch, EpochTagsVersionsInfo> epochInfos;
 	for (const auto& old : oldLogData) {
-		epochInfos.insert(
-		    { old.epoch, ILogSystem::EpochTagsVersionsInfo(old.logRouterTags, old.epochBegin, old.epochEnd) });
+		epochInfos.insert({ old.epoch, EpochTagsVersionsInfo(old.logRouterTags, old.epochBegin, old.epochEnd) });
 		TraceEvent("OldEpochTagsVersions", dbgid)
 		    .detail("Epoch", old.epoch)
 		    .detail("Tags", old.logRouterTags)
@@ -2016,7 +2147,7 @@ std::map<LogEpoch, ILogSystem::EpochTagsVersionsInfo> TagPartitionedLogSystem::g
 	return epochInfos;
 }
 
-inline Reference<LogSet> TagPartitionedLogSystem::getEpochLogSet(LogEpoch epoch) const {
+inline Reference<LogSet> LogSystem::getEpochLogSet(LogEpoch epoch) const {
 	for (const auto& old : oldLogData) {
 		if (epoch == old.epoch)
 			return old.tLogs[0];
@@ -2024,8 +2155,8 @@ inline Reference<LogSet> TagPartitionedLogSystem::getEpochLogSet(LogEpoch epoch)
 	return Reference<LogSet>(nullptr);
 }
 
-void TagPartitionedLogSystem::setBackupWorkers(const std::vector<InitializeBackupReply>& replies) {
-	ASSERT(tLogs.size() > 0);
+void LogSystem::setBackupWorkers(const std::vector<InitializeBackupReply>& replies) {
+	ASSERT(!tLogs.empty());
 
 	Reference<LogSet> logset = tLogs[0]; // Master recruits this epoch's worker first.
 	LogEpoch logsetEpoch = this->epoch;
@@ -2051,7 +2182,7 @@ void TagPartitionedLogSystem::setBackupWorkers(const std::vector<InitializeBacku
 	backupWorkerChanged.trigger();
 }
 
-bool TagPartitionedLogSystem::removeBackupWorker(const BackupWorkerDoneRequest& req) {
+bool LogSystem::removeBackupWorker(const BackupWorkerDoneRequest& req) {
 	bool removed = false;
 	Reference<LogSet> logset = getEpochLogSet(req.backupEpoch);
 	if (logset.isValid()) {
@@ -2067,7 +2198,7 @@ bool TagPartitionedLogSystem::removeBackupWorker(const BackupWorkerDoneRequest&
 	if (removed) {
 		oldestBackupEpoch = epoch;
 		for (const auto& old : oldLogData) {
-			if (old.epoch < oldestBackupEpoch && old.tLogs[0]->backupWorkers.size() > 0) {
+			if (old.epoch < oldestBackupEpoch && !old.tLogs[0]->backupWorkers.empty()) {
 				oldestBackupEpoch = old.epoch;
 			}
 		}
@@ -2084,31 +2215,31 @@ bool TagPartitionedLogSystem::removeBackupWorker(const BackupWorkerDoneRequest&
 	return removed;
 }
 
-LogEpoch TagPartitionedLogSystem::getOldestBackupEpoch() const {
+LogEpoch LogSystem::getOldestBackupEpoch() const {
 	return oldestBackupEpoch;
 }
 
-void TagPartitionedLogSystem::setOldestBackupEpoch(LogEpoch epoch) {
+void LogSystem::setOldestBackupEpoch(LogEpoch epoch) {
 	oldestBackupEpoch = epoch;
 	backupWorkerChanged.trigger();
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::monitorLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> logServer,
-                                                       Reference<AsyncVar<bool>> failed) {
-	state Future<Void> waitFailure;
-	loop {
+Future<Void> LogSystem::monitorLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> logServer,
+                                   Reference<AsyncVar<bool>> failed) {
+	Future<Void> waitFailure;
+	while (true) {
 		if (logServer->get().present())
 			waitFailure = waitFailureTracker(logServer->get().interf().waitFailure, failed);
 		else
 			failed->set(true);
-		wait(logServer->onChange());
+		co_await logServer->onChange();
 	}
 }
 
-Optional<DurableVersionInfo> TagPartitionedLogSystem::getDurableVersion(UID dbgid,
-                                                                        LogLockInfo lockInfo,
-                                                                        std::vector<Reference<AsyncVar<bool>>> failed,
-                                                                        Optional<Version> lastEnd) {
+Optional<DurableVersionInfo> LogSystem::getDurableVersion(UID dbgid,
+                                                          LogLockInfo lockInfo,
+                                                          std::vector<Reference<AsyncVar<bool>>> failed,
+                                                          Optional<Version> lastEnd) {
 
 	Reference<LogSet> logSet = lockInfo.logSet;
 	// To ensure consistent recovery, the number of servers NOT in the write quorum plus the number of servers NOT
@@ -2129,7 +2260,7 @@ Optional<DurableVersionInfo> TagPartitionedLogSystem::getDurableVersion(UID dbgi
 	std::vector<uint16_t> lockedTLogIds;
 	lockedTLogIds.reserve(logSet->logServers.size());
 	for (int t = 0; t < logSet->logServers.size(); t++) {
-		if (lockInfo.replies[t].isReady() && !lockInfo.replies[t].isError() && (!failed.size() || !failed[t]->get())) {
+		if (lockInfo.replies[t].isReady() && !lockInfo.replies[t].isError() && (failed.empty() || !failed[t]->get())) {
 			results.push_back(lockInfo.replies[t].get());
 			availableItems.push_back(logSet->tLogLocalities[t]);
 			sServerState += 'a';
@@ -2160,6 +2291,21 @@ Optional<DurableVersionInfo> TagPartitionedLogSystem::getDurableVersion(UID dbgi
 		bTooManyFailures = true;
 	}
 
+	if (bTooManyFailures) {
+		TraceEvent(SevWarnAlways, "TLogGenerationUnavailable", dbgid)
+		    .detail("CurrentGeneration", lockInfo.isCurrent)
+		    .detail("Locality", logSet->locality)
+		    .detail("StartVersion", logSet->startVersion)
+		    .detail("EpochEnd", lockInfo.epochEnd)
+		    .detail("ReplicationFactor", logSet->tLogReplicationFactor)
+		    .detail("WriteAntiQuorum", logSet->tLogWriteAntiQuorum)
+		    .detail("Required", requiredCount)
+		    .detail("Present", results.size())
+		    .detail("FailedLogsCompletePolicy", failedLogsCompletePolicy)
+		    .detail("ServerState", sServerState)
+		    .detail("LogServers", logSet->logServerString());
+	}
+
 	ASSERT(logSet->logServers.size() == lockInfo.replies.size());
 	if (!bTooManyFailures) {
 		std::sort(results.begin(), results.end(), [](const TLogLockResult& a, const TLogLockResult& b) -> bool {
@@ -2227,8 +2373,7 @@ Optional<DurableVersionInfo> TagPartitionedLogSystem::getDurableVersion(UID dbgi
 	return Optional<DurableVersionInfo>();
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::getDurableVersionChanged(LogLockInfo lockInfo,
-                                                                     std::vector<Reference<AsyncVar<bool>>> failed) {
+Future<Void> LogSystem::getDurableVersionChanged(LogLockInfo lockInfo, std::vector<Reference<AsyncVar<bool>>> failed) {
 	// Wait for anything relevant to change
 	std::vector<Future<Void>> changes;
 	for (int j = 0; j < lockInfo.logSet->logServers.size(); j++) {
@@ -2236,14 +2381,13 @@ ACTOR Future<Void> TagPartitionedLogSystem::getDurableVersionChanged(LogLockInfo
 			changes.push_back(ready(lockInfo.replies[j]));
 		} else {
 			changes.push_back(lockInfo.logSet->logServers[j]->onChange());
-			if (failed.size()) {
+			if (!failed.empty()) {
 				changes.push_back(failed[j]->onChange());
 			}
 		}
 	}
-	ASSERT(changes.size());
-	wait(waitForAny(changes));
-	return Void();
+	ASSERT(!changes.empty());
+	co_await waitForAny(changes);
 }
 
 void getTLogLocIds(const std::vector<Reference<LogSet>>& tLogs,
@@ -2441,25 +2585,25 @@ static bool isSubset(const std::map<uint8_t, std::vector<uint16_t>>& mapA,
 	return true;
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Reference<ILogSystem>>> outLogSystem,
-                                                     UID dbgid,
-                                                     DBCoreState prevState,
-                                                     FutureStream<TLogRejoinRequest> rejoinRequests,
-                                                     LocalityData locality,
-                                                     bool* forceRecovery) {
+Future<Void> LogSystem::epochEnd(Reference<AsyncVar<Reference<LogSystem>>> outLogSystem,
+                                 UID dbgid,
+                                 DBCoreState prevState,
+                                 FutureStream<TLogRejoinRequest> rejoinRequests,
+                                 LocalityData locality,
+                                 bool* forceRecovery) {
 	// Stops a co-quorum of tlogs so that no further versions can be committed until the DBCoreState coordination
 	// state is changed Creates a new logSystem representing the (now frozen) epoch No other important side effects.
 	// The writeQuorum in the master info is from the previous configuration
 
-	if (!prevState.tLogs.size()) {
+	if (prevState.tLogs.empty()) {
 		// This is a brand new database
-		auto logSystem = makeReference<TagPartitionedLogSystem>(dbgid, locality, 0);
+		auto logSystem = makeReference<LogSystem>(dbgid, locality, 0);
 		logSystem->logSystemType = prevState.logSystemType;
 		logSystem->recoverAt = 0;
 		logSystem->knownCommittedVersion = 0;
 		logSystem->stopped = true;
 		outLogSystem->set(logSystem);
-		wait(Future<Void>(Never()));
+		co_await Future<Void>(Never());
 		throw internal_error();
 	}
 
@@ -2493,7 +2637,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 				}
 			}
 
-			while (!foundRemote && modifiedState.oldTLogData.size()) {
+			while (!foundRemote && !modifiedState.oldTLogData.empty()) {
 				for (auto& coreSet : modifiedState.oldTLogData[0].tLogs) {
 					if (coreSet.locality != primaryLocality && coreSet.locality >= tagLocalitySpecial) {
 						foundRemote = true;
@@ -2576,13 +2720,13 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 
 	// trackRejoins listens for rejoin requests from the tLogs that we are recovering from, to learn their
 	// TLogInterfaces
-	state std::vector<LogLockInfo> lockResults;
-	state std::vector<std::pair<Reference<AsyncVar<OptionalInterface<TLogInterface>>>, Reference<IReplicationPolicy>>>
+	std::vector<LogLockInfo> lockResults;
+	std::vector<std::pair<Reference<AsyncVar<OptionalInterface<TLogInterface>>>, Reference<IReplicationPolicy>>>
 	    allLogServers;
-	state std::vector<Reference<LogSet>> logServers;
-	state std::vector<OldLogData> oldLogData;
-	state std::vector<std::vector<Reference<AsyncVar<bool>>>> logFailed;
-	state std::vector<Future<Void>> failureTrackers;
+	std::vector<Reference<LogSet>> logServers;
+	std::vector<OldLogData> oldLogData;
+	std::vector<std::vector<Reference<AsyncVar<bool>>>> logFailed;
+	std::vector<Future<Void>> failureTrackers;
 
 	for (const CoreTLogSet& coreSet : prevState.tLogs) {
 		logServers.push_back(makeReference<LogSet>(coreSet));
@@ -2591,7 +2735,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 		for (const auto& logVar : logServers.back()->logServers) {
 			allLogServers.emplace_back(logVar, coreSet.tLogPolicy);
 			failed.push_back(makeReference<AsyncVar<bool>>());
-			failureTrackers.push_back(TagPartitionedLogSystem::monitorLog(logVar, failed.back()));
+			failureTrackers.push_back(LogSystem::monitorLog(logVar, failed.back()));
 		}
 		logFailed.push_back(failed);
 	}
@@ -2605,7 +2749,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 			}
 		}
 	}
-	state Future<Void> rejoins = TagPartitionedLogSystem::trackRejoins(dbgid, allLogServers, rejoinRequests);
+	Future<Void> rejoins = LogSystem::trackRejoins(dbgid, allLogServers, rejoinRequests);
 
 	lockResults.resize(logServers.size());
 	std::set<int8_t> lockedLocalities;
@@ -2618,7 +2762,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 		lockResults[i].isCurrent = true;
 		lockResults[i].logSet = logServers[i];
 		for (int t = 0; t < logServers[i]->logServers.size(); t++) {
-			lockResults[i].replies.push_back(TagPartitionedLogSystem::lockTLog(dbgid, logServers[i]->logServers[t]));
+			lockResults[i].replies.push_back(LogSystem::lockTLog(dbgid, logServers[i]->logServers[t]));
 		}
 	}
 
@@ -2639,14 +2783,14 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 				lockResult.epochEnd = old.epochEnd;
 				lockResult.logSet = log;
 				for (int t = 0; t < log->logServers.size(); t++) {
-					lockResult.replies.push_back(TagPartitionedLogSystem::lockTLog(dbgid, log->logServers[t]));
+					lockResult.replies.push_back(LogSystem::lockTLog(dbgid, log->logServers[t]));
 				}
 				lockResults.push_back(lockResult);
 			}
 		}
 	}
 	if (*forceRecovery) {
-		state std::vector<LogLockInfo> allLockResults;
+		std::vector<LogLockInfo> allLockResults;
 		ASSERT(lockResults.size() == 1);
 		allLockResults.push_back(lockResults[0]);
 		for (auto& old : oldLogData) {
@@ -2655,16 +2799,16 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 			lockResult.epochEnd = old.epochEnd;
 			lockResult.logSet = old.tLogs[0];
 			for (int t = 0; t < old.tLogs[0]->logServers.size(); t++) {
-				lockResult.replies.push_back(TagPartitionedLogSystem::lockTLog(dbgid, old.tLogs[0]->logServers[t]));
+				lockResult.replies.push_back(LogSystem::lockTLog(dbgid, old.tLogs[0]->logServers[t]));
 			}
 			allLockResults.push_back(lockResult);
 		}
-		state int lockNum = 0;
-		state Version maxRecoveryVersion = 0;
-		state int maxRecoveryIndex = 0;
+		int lockNum = 0;
+		Version maxRecoveryVersion = 0;
+		int maxRecoveryIndex = 0;
 		while (lockNum < allLockResults.size()) {
 
-			auto durableVersionInfo = TagPartitionedLogSystem::getDurableVersion(dbgid, allLockResults[lockNum]);
+			auto durableVersionInfo = LogSystem::getDurableVersion(dbgid, allLockResults[lockNum]);
 			if (durableVersionInfo.present()) {
 				if (durableVersionInfo.get().minimumDurableVersion > maxRecoveryVersion) {
 					TraceEvent("HigherRecoveryVersion", dbgid)
@@ -2675,7 +2819,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 				}
 				lockNum++;
 			} else {
-				wait(TagPartitionedLogSystem::getDurableVersionChanged(allLockResults[lockNum]));
+				co_await LogSystem::getDurableVersionChanged(allLockResults[lockNum]);
 			}
 		}
 		if (maxRecoveryIndex > 0) {
@@ -2687,7 +2831,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 			std::vector<Reference<AsyncVar<bool>>> failed;
 			for (auto& log : logServers[0]->logServers) {
 				failed.push_back(makeReference<AsyncVar<bool>>());
-				failureTrackers.push_back(TagPartitionedLogSystem::monitorLog(log, failed.back()));
+				failureTrackers.push_back(LogSystem::monitorLog(log, failed.back()));
 			}
 			ASSERT(logFailed.size() == 1);
 			logFailed[0] = failed;
@@ -2695,23 +2839,22 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 		}
 	}
 
-	state Optional<Version> lastEnd;
-	state Version knownCommittedVersion = 0;
-	state std::map<uint8_t, std::vector<uint16_t>> lastKnownLockedTLogIds;
-	state bool knownLockedTLogIdsChanged = false;
-	loop {
+	Optional<Version> lastEnd;
+	Version knownCommittedVersion = 0;
+	std::map<uint8_t, std::vector<uint16_t>> lastKnownLockedTLogIds;
+	bool knownLockedTLogIdsChanged = false;
+	while (true) {
 		Version minEnd = std::numeric_limits<Version>::max();
 		Version minDV = std::numeric_limits<Version>::max();
 		Version maxEnd = 0;
-		state std::vector<Future<Void>> changes;
-		state std::vector<std::tuple<int, std::vector<TLogLockResult>, bool>> logGroupResults;
-		state std::map<uint8_t, std::vector<uint16_t>> currentKnownLockedTLogIds;
+		std::vector<Future<Void>> changes;
+		std::vector<std::tuple<int, std::vector<TLogLockResult>, bool>> logGroupResults;
+		std::map<uint8_t, std::vector<uint16_t>> currentKnownLockedTLogIds;
 		for (int log = 0; log < logServers.size(); log++) {
 			if (!logServers[log]->isLocal) {
 				continue;
 			}
-			auto durableVersionInfo =
-			    TagPartitionedLogSystem::getDurableVersion(dbgid, lockResults[log], logFailed[log], lastEnd);
+			auto durableVersionInfo = LogSystem::getDurableVersion(dbgid, lockResults[log], logFailed[log], lastEnd);
 			if (durableVersionInfo.present()) {
 				logGroupResults.emplace_back(logServers[log]->tLogReplicationFactor,
 				                             durableVersionInfo.get().lockResults,
@@ -2730,7 +2873,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 					minEnd = std::min(minEnd, std::get<1>(unicastVersions.get()));
 				}
 			}
-			changes.push_back(TagPartitionedLogSystem::getDurableVersionChanged(lockResults[log], logFailed[log]));
+			changes.push_back(LogSystem::getDurableVersionChanged(lockResults[log], logFailed[log]));
 		}
 		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST && maxEnd > 0 && lastEnd.present() &&
 		    maxEnd >= lastEnd.get()) {
@@ -2747,8 +2890,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 		if (maxEnd > 0 && (!lastEnd.present() || maxEnd < lastEnd.get() || knownLockedTLogIdsChanged)) {
 			CODE_PROBE(lastEnd.present(), "Restarting recovery at an earlier point");
 
-			state Reference<TagPartitionedLogSystem> logSystem =
-			    makeReference<TagPartitionedLogSystem>(dbgid, locality, prevState.recoveryCount);
+			auto logSystem = makeReference<LogSystem>(dbgid, locality, prevState.recoveryCount);
 
 			logSystem->recoverAt = minEnd;
 			lastEnd = minEnd;
@@ -2775,31 +2917,31 @@ ACTOR Future<Void> TagPartitionedLogSystem::epochEnd(Reference<AsyncVar<Referenc
 			outLogSystem->set(logSystem);
 		}
 
-		wait(waitForAny(changes));
+		co_await waitForAny(changes);
 	}
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedLogSystem* self,
-                                                                 std::vector<WorkerInterface> workers,
-                                                                 LogEpoch recoveryCount,
-                                                                 int8_t locality,
-                                                                 Version startVersion,
-                                                                 std::vector<LocalityData> tLogLocalities,
-                                                                 Reference<IReplicationPolicy> tLogPolicy,
-                                                                 bool forRemote) {
-	state std::vector<std::vector<Future<TLogInterface>>> logRouterInitializationReplies;
-	state std::vector<Future<TLogInterface>> allReplies;
+Future<Void> LogSystem::recruitOldLogRouters(LogSystem* self,
+                                             std::vector<WorkerInterface> workers,
+                                             LogEpoch recoveryCount,
+                                             int8_t locality,
+                                             Version startVersion,
+                                             std::vector<LocalityData> tLogLocalities,
+                                             Reference<IReplicationPolicy> tLogPolicy,
+                                             bool forRemote) {
+	std::vector<std::vector<Future<TLogInterface>>> logRouterInitializationReplies;
+	std::vector<Future<TLogInterface>> allReplies;
 	int nextRouter = 0;
-	state Version lastStart = std::numeric_limits<Version>::max();
+	Version lastStart = std::numeric_limits<Version>::max();
 
 	if (!forRemote) {
-		Version maxStart = TagPartitionedLogSystem::getMaxLocalStartVersion(self->tLogs);
+		Version maxStart = LogSystem::getMaxLocalStartVersion(self->tLogs);
 
 		lastStart = std::max(startVersion, maxStart);
 		if (self->logRouterTags == 0) {
 			ASSERT_WE_THINK(false);
 			self->logSystemConfigChanged.trigger();
-			return Void();
+			co_return;
 		}
 
 		bool found = false;
@@ -2855,7 +2997,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL
 	}
 
 	for (auto& old : self->oldLogData) {
-		Version maxStart = TagPartitionedLogSystem::getMaxLocalStartVersion(old.tLogs);
+		Version maxStart = LogSystem::getMaxLocalStartVersion(old.tLogs);
 
 		if (old.logRouterTags == 0 || maxStart >= lastStart) {
 			break;
@@ -2910,14 +3052,14 @@ ACTOR Future<Void> TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL
 		}
 	}
 
-	wait(traceAfter(waitForAll(allReplies), "AllLogRouterRepliesReceived"));
+	co_await traceAfter(waitForAll(allReplies), "AllLogRouterRepliesReceived");
 
 	int nextReplies = 0;
 	lastStart = std::numeric_limits<Version>::max();
 	std::vector<Future<Void>> failed;
 
 	if (!forRemote) {
-		Version maxStart = TagPartitionedLogSystem::getMaxLocalStartVersion(self->tLogs);
+		Version maxStart = LogSystem::getMaxLocalStartVersion(self->tLogs);
 
 		lastStart = std::max(startVersion, maxStart);
 		for (auto& tLogs : self->tLogs) {
@@ -2937,7 +3079,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL
 	}
 
 	for (auto& old : self->oldLogData) {
-		Version maxStart = TagPartitionedLogSystem::getMaxLocalStartVersion(old.tLogs);
+		Version maxStart = LogSystem::getMaxLocalStartVersion(old.tLogs);
 		if (old.logRouterTags == 0 || maxStart >= lastStart) {
 			break;
 		}
@@ -2962,13 +3104,12 @@ ACTOR Future<Void> TagPartitionedLogSystem::recruitOldLogRouters(TagPartitionedL
 
 	if (!forRemote) {
 		self->logSystemConfigChanged.trigger();
-		wait(failed.size() ? tagError<Void>(quorum(failed, 1), tlog_failed()) : Future<Void>(Never()));
+		co_await (!failed.empty() ? tagError<Void>(quorum(failed, 1), tlog_failed()) : Future<Void>(Never()));
 		throw internal_error();
 	}
-	return Void();
 }
 
-Version TagPartitionedLogSystem::getMaxLocalStartVersion(const std::vector<Reference<LogSet>>& tLogs) {
+Version LogSystem::getMaxLocalStartVersion(const std::vector<Reference<LogSet>>& tLogs) {
 	Version maxStart = 0;
 	for (const auto& logSet : tLogs) {
 		if (logSet->isLocal) {
@@ -2978,7 +3119,7 @@ Version TagPartitionedLogSystem::getMaxLocalStartVersion(const std::vector<Refer
 	return maxStart;
 }
 
-std::vector<Tag> TagPartitionedLogSystem::getLocalTags(int8_t locality, const std::vector<Tag>& allTags) {
+std::vector<Tag> LogSystem::getLocalTags(int8_t locality, const std::vector<Tag>& allTags) {
 	std::vector<Tag> localTags;
 	for (const auto& tag : allTags) {
 		if (locality == tagLocalitySpecial || locality == tag.locality || tag.locality < 0) {
@@ -2988,22 +3129,22 @@ std::vector<Tag> TagPartitionedLogSystem::getLocalTags(int8_t locality, const st
 	return localTags;
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSystem* self,
-                                                           Reference<TagPartitionedLogSystem> oldLogSystem,
-                                                           Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
-                                                           DatabaseConfiguration configuration,
-                                                           LogEpoch recoveryCount,
-                                                           Version recoveryTransactionVersion,
-                                                           int8_t remoteLocality,
-                                                           std::vector<Tag> allTags,
-                                                           std::vector<Version> oldGenerationRecoverAtVersions) {
+Future<Void> LogSystem::newRemoteEpoch(LogSystem* self,
+                                       Reference<LogSystem> oldLogSystem,
+                                       Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
+                                       DatabaseConfiguration configuration,
+                                       LogEpoch recoveryCount,
+                                       Version recoveryTransactionVersion,
+                                       int8_t remoteLocality,
+                                       std::vector<Tag> allTags,
+                                       std::vector<Version> oldGenerationRecoverAtVersions) {
 	TraceEvent("RemoteLogRecruitment_WaitingForWorkers").log();
-	state RecruitRemoteFromConfigurationReply remoteWorkers = wait(fRemoteWorkers);
+	RecruitRemoteFromConfigurationReply remoteWorkers = co_await fRemoteWorkers;
 	TraceEvent("RecruitedRemoteLogWorkers")
 	    .detail("TLogs", remoteWorkers.remoteTLogs.size())
 	    .detail("LogRouter", remoteWorkers.logRouters.size());
 
-	state Reference<LogSet> logSet(new LogSet());
+	Reference<LogSet> logSet(new LogSet());
 	logSet->tLogReplicationFactor = configuration.getRemoteTLogReplicationFactor();
 	logSet->tLogVersion = configuration.tLogVersion;
 	logSet->tLogPolicy = configuration.getRemoteTLogPolicy();
@@ -3011,24 +3152,20 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 	logSet->locality = remoteLocality;
 
 	logSet->startVersion = oldLogSystem->knownCommittedVersion + 1;
-	state int lockNum = 0;
-	while (lockNum < oldLogSystem->lockResults.size()) {
+	for (int lockNum = 0; lockNum < oldLogSystem->lockResults.size(); ++lockNum) {
 		if (oldLogSystem->lockResults[lockNum].logSet->locality == remoteLocality) {
-
-			loop {
-				auto durableVersionInfo =
-				    TagPartitionedLogSystem::getDurableVersion(self->dbgid, oldLogSystem->lockResults[lockNum]);
+			while (true) {
+				auto durableVersionInfo = LogSystem::getDurableVersion(self->dbgid, oldLogSystem->lockResults[lockNum]);
 				if (durableVersionInfo.present()) {
 					logSet->startVersion = std::min(std::min(durableVersionInfo.get().knownCommittedVersion + 1,
 					                                         oldLogSystem->lockResults[lockNum].epochEnd),
 					                                logSet->startVersion);
 					break;
 				}
-				wait(TagPartitionedLogSystem::getDurableVersionChanged(oldLogSystem->lockResults[lockNum]));
+				co_await LogSystem::getDurableVersionChanged(oldLogSystem->lockResults[lockNum]);
 			}
 			break;
 		}
-		lockNum++;
 	}
 
 	std::vector<LocalityData> localities;
@@ -3037,20 +3174,20 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 		localities[i] = remoteWorkers.remoteTLogs[i].locality;
 	}
 
-	state Future<Void> oldRouterRecruitment = Void();
+	Future<Void> oldRouterRecruitment = Void();
 	if (logSet->startVersion < oldLogSystem->knownCommittedVersion + 1) {
 		ASSERT(oldLogSystem->logRouterTags > 0);
-		oldRouterRecruitment = TagPartitionedLogSystem::recruitOldLogRouters(self,
-		                                                                     remoteWorkers.logRouters,
-		                                                                     recoveryCount,
-		                                                                     remoteLocality,
-		                                                                     logSet->startVersion,
-		                                                                     localities,
-		                                                                     logSet->tLogPolicy,
-		                                                                     /* forRemote */ true);
+		oldRouterRecruitment = LogSystem::recruitOldLogRouters(self,
+		                                                       remoteWorkers.logRouters,
+		                                                       recoveryCount,
+		                                                       remoteLocality,
+		                                                       logSet->startVersion,
+		                                                       localities,
+		                                                       logSet->tLogPolicy,
+		                                                       /* forRemote */ true);
 	}
 
-	state std::vector<Future<TLogInterface>> logRouterInitializationReplies;
+	std::vector<Future<TLogInterface>> logRouterInitializationReplies;
 	const Version startVersion = oldLogSystem->logRouterTags == 0
 	                                 ? oldLogSystem->recoverAt.get() + 1
 	                                 : std::max(self->tLogs[0]->startVersion, logSet->startVersion);
@@ -3075,7 +3212,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 		    cluster_recovery_failed()));
 	}
 
-	std::vector<Tag> localTags = TagPartitionedLogSystem::getLocalTags(remoteLocality, allTags);
+	std::vector<Tag> localTags = LogSystem::getLocalTags(remoteLocality, allTags);
 	LogSystemConfig oldLogSystemConfig = oldLogSystem->getLogSystemConfig();
 
 	logSet->tLogLocalities.resize(remoteWorkers.remoteTLogs.size());
@@ -3084,9 +3221,9 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 	        .size()); // Dummy interfaces, so that logSystem->getPushLocations() below uses the correct size
 	logSet->updateLocalitySet(localities);
 
-	state std::vector<Future<TLogInterface>> remoteTLogInitializationReplies;
+	std::vector<Future<TLogInterface>> remoteTLogInitializationReplies;
 	std::vector<InitializeTLogRequest> remoteTLogReqs(remoteWorkers.remoteTLogs.size());
-	state std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> allRemoteTLogServers;
+	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> allRemoteTLogServers;
 
 	if (oldLogSystem->logRouterTags == 0) {
 		std::vector<int> locations;
@@ -3098,7 +3235,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 			}
 		}
 
-		if (oldLogSystem->tLogs.size()) {
+		if (!oldLogSystem->tLogs.empty()) {
 			int maxTxsTags = oldLogSystem->txsTags;
 			for (auto& it : oldLogSystem->oldLogData) {
 				maxTxsTags = std::max<int>(maxTxsTags, it.txsTags);
@@ -3115,7 +3252,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 		}
 	}
 
-	if (oldLogSystem->tLogs.size()) {
+	if (!oldLogSystem->tLogs.empty()) {
 		for (int i = 0; i < self->txsTags; i++) {
 			localTags.push_back(Tag(tagLocalityTxs, i));
 		}
@@ -3155,9 +3292,9 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 	    .detail("StartVersion", logSet->startVersion)
 	    .detail("LocalStart", self->tLogs[0]->startVersion)
 	    .detail("LogRouterTags", self->logRouterTags);
-	wait(traceAfter(waitForAll(remoteTLogInitializationReplies), "RemoteTLogInitializationRepliesReceived") &&
-	     traceAfter(waitForAll(logRouterInitializationReplies), "LogRouterInitializationRepliesReceived") &&
-	     traceAfter(oldRouterRecruitment, "OldRouterRecruitmentFinished"));
+	co_await (traceAfter(waitForAll(remoteTLogInitializationReplies), "RemoteTLogInitializationRepliesReceived") &&
+	          traceAfter(waitForAll(logRouterInitializationReplies), "LogRouterInitializationRepliesReceived") &&
+	          traceAfter(oldRouterRecruitment, "OldRouterRecruitmentFinished"));
 
 	for (int i = 0; i < logRouterInitializationReplies.size(); i++) {
 		logSet->logRouters.push_back(makeReference<AsyncVar<OptionalInterface<TLogInterface>>>(
@@ -3185,26 +3322,23 @@ ACTOR Future<Void> TagPartitionedLogSystem::newRemoteEpoch(TagPartitionedLogSyst
 
 	self->remoteRecoveryComplete = waitForAll(recoveryComplete);
 	self->remoteTrackTLogRecovery =
-	    TagPartitionedLogSystem::trackTLogRecoveryActor(allRemoteTLogServers, self->remoteRecoveredVersion);
+	    LogSystem::trackTLogRecoveryActor(allRemoteTLogServers, self->remoteRecoveredVersion);
 	self->tLogs.push_back(logSet);
 	TraceEvent("RemoteLogRecruitment_CompletingRecovery").log();
-	return Void();
-}
-
-ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
-    Reference<TagPartitionedLogSystem> oldLogSystem,
-    RecruitFromConfigurationReply recr,
-    Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
-    DatabaseConfiguration configuration,
-    LogEpoch recoveryCount,
-    Version recoveryTransactionVersion,
-    int8_t primaryLocality,
-    int8_t remoteLocality,
-    std::vector<Tag> allTags,
-    Reference<AsyncVar<bool>> recruitmentStalled) {
-	state double startTime = now();
-	state Reference<TagPartitionedLogSystem> logSystem(
-	    new TagPartitionedLogSystem(oldLogSystem->getDebugID(), oldLogSystem->locality, recoveryCount));
+}
+
+Future<Reference<LogSystem>> LogSystem::newEpoch(Reference<LogSystem> oldLogSystem,
+                                                 RecruitFromConfigurationReply recr,
+                                                 Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
+                                                 DatabaseConfiguration configuration,
+                                                 LogEpoch recoveryCount,
+                                                 Version recoveryTransactionVersion,
+                                                 int8_t primaryLocality,
+                                                 int8_t remoteLocality,
+                                                 std::vector<Tag> allTags,
+                                                 Reference<AsyncVar<bool>> recruitmentStalled) {
+	double startTime = now();
+	Reference<LogSystem> logSystem(new LogSystem(oldLogSystem->getDebugID(), oldLogSystem->locality, recoveryCount));
 	logSystem->logSystemType = LogSystemType::tagPartitioned;
 	logSystem->expectedLogSets = 1;
 	logSystem->recoveredAt = oldLogSystem->recoverAt;
@@ -3242,9 +3376,9 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 	logSystem->tLogs[0]->isLocal = true;
 	logSystem->tLogs[0]->locality = primaryLocality;
 
-	state RegionInfo region = configuration.getRegion(recr.dcId);
+	RegionInfo region = configuration.getRegion(recr.dcId);
 
-	state int maxTxsTags = oldLogSystem->txsTags;
+	int maxTxsTags = oldLogSystem->txsTags;
 	for (auto& it : oldLogSystem->oldLogData) {
 		maxTxsTags = std::max<int>(maxTxsTags, it.txsTags);
 	}
@@ -3280,7 +3414,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		logSystem->expectedLogSets++;
 	}
 
-	if (oldLogSystem->tLogs.size()) {
+	if (!oldLogSystem->tLogs.empty()) {
 		logSystem->oldLogData.emplace_back();
 		logSystem->oldLogData[0].tLogs = oldLogSystem->tLogs;
 		logSystem->oldLogData[0].epochBegin = oldLogSystem->tLogs[0]->startVersion;
@@ -3296,16 +3430,15 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 
 	logSystem->tLogs[0]->startVersion = oldLogSystem->knownCommittedVersion + 1;
 	logSystem->backupStartVersion = oldLogSystem->knownCommittedVersion + 1;
-	state int lockNum = 0;
-	while (lockNum < oldLogSystem->lockResults.size()) {
+	for (int lockNum = 0; lockNum < oldLogSystem->lockResults.size(); ++lockNum) {
 		if (oldLogSystem->lockResults[lockNum].logSet->locality == primaryLocality) {
 			if (oldLogSystem->lockResults[lockNum].isCurrent && oldLogSystem->lockResults[lockNum].logSet->isLocal) {
 				break;
 			}
-			state Future<Void> stalledAfter = setAfter(recruitmentStalled, SERVER_KNOBS->MAX_RECOVERY_TIME, true);
-			loop {
+			Future<Void> stalledAfter = setAfter(recruitmentStalled, SERVER_KNOBS->MAX_RECOVERY_TIME, true);
+			while (true) {
 				auto durableVersionInfo =
-				    TagPartitionedLogSystem::getDurableVersion(logSystem->dbgid, oldLogSystem->lockResults[lockNum]);
+				    LogSystem::getDurableVersion(logSystem->dbgid, oldLogSystem->lockResults[lockNum]);
 				if (durableVersionInfo.present()) {
 					logSystem->tLogs[0]->startVersion =
 					    std::min(std::min(durableVersionInfo.get().knownCommittedVersion + 1,
@@ -3313,12 +3446,11 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 					             logSystem->tLogs[0]->startVersion);
 					break;
 				}
-				wait(TagPartitionedLogSystem::getDurableVersionChanged(oldLogSystem->lockResults[lockNum]));
+				co_await LogSystem::getDurableVersionChanged(oldLogSystem->lockResults[lockNum]);
 			}
 			stalledAfter.cancel();
 			break;
 		}
-		lockNum++;
 	}
 
 	std::vector<LocalityData> localities;
@@ -3327,7 +3459,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		localities[i] = recr.tLogs[i].locality;
 	}
 
-	state Future<Void> oldRouterRecruitment = Never();
+	Future<Void> oldRouterRecruitment = Never();
 	TraceEvent("NewEpochStartVersion", oldLogSystem->getDebugID())
 	    .detail("StartVersion", logSystem->tLogs[0]->startVersion)
 	    .detail("EpochEnd", oldLogSystem->knownCommittedVersion + 1)
@@ -3336,20 +3468,20 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 	if (oldLogSystem->logRouterTags > 0 ||
 	    logSystem->tLogs[0]->startVersion < oldLogSystem->knownCommittedVersion + 1) {
 		// Use log routers to recover [knownCommittedVersion, recoveryVersion] from the old generation.
-		oldRouterRecruitment = TagPartitionedLogSystem::recruitOldLogRouters(oldLogSystem.getPtr(),
-		                                                                     recr.oldLogRouters,
-		                                                                     recoveryCount,
-		                                                                     primaryLocality,
-		                                                                     logSystem->tLogs[0]->startVersion,
-		                                                                     localities,
-		                                                                     logSystem->tLogs[0]->tLogPolicy,
-		                                                                     /* forRemote */ false);
+		oldRouterRecruitment = LogSystem::recruitOldLogRouters(oldLogSystem.getPtr(),
+		                                                       recr.oldLogRouters,
+		                                                       recoveryCount,
+		                                                       primaryLocality,
+		                                                       logSystem->tLogs[0]->startVersion,
+		                                                       localities,
+		                                                       logSystem->tLogs[0]->tLogPolicy,
+		                                                       /* forRemote */ false);
 		if (oldLogSystem->knownCommittedVersion - logSystem->tLogs[0]->startVersion >
 		    SERVER_KNOBS->MAX_RECOVERY_VERSIONS) {
 			// make sure we can recover in the other DC.
 			for (auto& lockResult : oldLogSystem->lockResults) {
 				if (lockResult.logSet->locality == remoteLocality) {
-					if (TagPartitionedLogSystem::getDurableVersion(logSystem->dbgid, lockResult).present()) {
+					if (LogSystem::getDurableVersion(logSystem->dbgid, lockResult).present()) {
 						recruitmentStalled->set(true);
 					}
 				}
@@ -3359,10 +3491,10 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		oldLogSystem->logSystemConfigChanged.trigger();
 	}
 
-	std::vector<Tag> localTags = TagPartitionedLogSystem::getLocalTags(primaryLocality, allTags);
-	state LogSystemConfig oldLogSystemConfig = oldLogSystem->getLogSystemConfig();
+	std::vector<Tag> localTags = LogSystem::getLocalTags(primaryLocality, allTags);
+	LogSystemConfig oldLogSystemConfig = oldLogSystem->getLogSystemConfig();
 
-	state std::vector<Future<TLogInterface>> primaryTLogReplies;
+	std::vector<Future<TLogInterface>> primaryTLogReplies;
 	std::vector<InitializeTLogRequest> reqs(recr.tLogs.size());
 
 	logSystem->tLogs[0]->tLogLocalities.resize(recr.tLogs.size());
@@ -3383,7 +3515,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		reqs[logSystem->tLogs[0]->bestLocationFor(tag)].recoverTags.push_back(tag);
 	}
 
-	if (oldLogSystem->tLogs.size()) {
+	if (!oldLogSystem->tLogs.empty()) {
 		for (int i = 0; i < maxTxsTags; i++) {
 			Tag tag = Tag(tagLocalityTxs, i);
 			Tag pushTag = Tag(tagLocalityTxs, i % logSystem->txsTags);
@@ -3399,7 +3531,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 	}
 
 	// Should be sorted by descending orders of versions.
-	state std::vector<Version> oldGenerationRecoverAtVersions;
+	std::vector<Version> oldGenerationRecoverAtVersions;
 	for (const auto& oldLogGen : logSystem->oldLogData) {
 		if (oldLogGen.recoverAt <= 0) {
 			// When we have an invalid recover at value, it's possible that the previous generations' recover at is not
@@ -3441,11 +3573,11 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		    cluster_recovery_failed()));
 	}
 
-	state std::vector<Future<Void>> recoveryComplete;
-	state std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> allTLogServers;
+	std::vector<Future<Void>> recoveryComplete;
+	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> allTLogServers;
 
 	if (region.satelliteTLogReplicationFactor > 0 && configuration.usableRegions > 1) {
-		state std::vector<Future<TLogInterface>> satelliteInitializationReplies;
+		std::vector<Future<TLogInterface>> satelliteInitializationReplies;
 		std::vector<InitializeTLogRequest> sreqs(recr.satelliteTLogs.size());
 		std::vector<Tag> satelliteTags;
 
@@ -3465,7 +3597,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 				}
 			}
 		}
-		if (oldLogSystem->tLogs.size()) {
+		if (!oldLogSystem->tLogs.empty()) {
 			for (int i = 0; i < maxTxsTags; i++) {
 				Tag tag = Tag(tagLocalityTxs, i);
 				Tag pushTag = Tag(tagLocalityTxs, i % logSystem->txsTags);
@@ -3511,8 +3643,8 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 			    cluster_recovery_failed()));
 		}
 
-		wait(traceAfter(waitForAll(satelliteInitializationReplies), "SatelliteInitializationRepliesReceived") ||
-		     traceAfter(oldRouterRecruitment, "OldRouterRecruitmentFinished"));
+		co_await (traceAfter(waitForAll(satelliteInitializationReplies), "SatelliteInitializationRepliesReceived") ||
+		          traceAfter(oldRouterRecruitment, "OldRouterRecruitmentFinished"));
 		TraceEvent("PrimarySatelliteTLogInitializationComplete", logSystem->getDebugID()).log();
 
 		for (int i = 0; i < satelliteInitializationReplies.size(); i++) {
@@ -3532,8 +3664,8 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		}
 	}
 
-	wait(traceAfter(waitForAll(primaryTLogReplies), "PrimaryTLogRepliesReceived") ||
-	     traceAfter(oldRouterRecruitment, "OldRouterRecruitmentFinished"));
+	co_await (traceAfter(waitForAll(primaryTLogReplies), "PrimaryTLogRepliesReceived") ||
+	          traceAfter(oldRouterRecruitment, "OldRouterRecruitmentFinished"));
 	TraceEvent("PrimaryTLogInitializationComplete", logSystem->getDebugID()).log();
 
 	for (int i = 0; i < primaryTLogReplies.size(); i++) {
@@ -3558,27 +3690,26 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		    cluster_recovery_failed()));
 		allTLogServers.push_back(logSystem->tLogs[0]->logServers[i]);
 	}
-	logSystem->trackTLogRecovery =
-	    TagPartitionedLogSystem::trackTLogRecoveryActor(allTLogServers, logSystem->recoveredVersion);
+	logSystem->trackTLogRecovery = LogSystem::trackTLogRecoveryActor(allTLogServers, logSystem->recoveredVersion);
 	logSystem->recoveryComplete = waitForAll(recoveryComplete);
 
 	if (configuration.usableRegions > 1) {
 		logSystem->hasRemoteServers = true;
-		logSystem->remoteRecovery = TagPartitionedLogSystem::newRemoteEpoch(logSystem.getPtr(),
-		                                                                    oldLogSystem,
-		                                                                    fRemoteWorkers,
-		                                                                    configuration,
-		                                                                    recoveryCount,
-		                                                                    recoveryTransactionVersion,
-		                                                                    remoteLocality,
-		                                                                    allTags,
-		                                                                    oldGenerationRecoverAtVersions);
-		if (oldLogSystem->tLogs.size() > 0 && oldLogSystem->tLogs[0]->locality == tagLocalitySpecial) {
+		logSystem->remoteRecovery = LogSystem::newRemoteEpoch(logSystem.getPtr(),
+		                                                      oldLogSystem,
+		                                                      fRemoteWorkers,
+		                                                      configuration,
+		                                                      recoveryCount,
+		                                                      recoveryTransactionVersion,
+		                                                      remoteLocality,
+		                                                      allTags,
+		                                                      oldGenerationRecoverAtVersions);
+		if (!oldLogSystem->tLogs.empty() && oldLogSystem->tLogs[0]->locality == tagLocalitySpecial) {
 			// The wait is required so that we know both primary logs and remote logs have copied the data between
 			// the known committed version and the recovery version.
 			// FIXME: we can remove this wait once we are able to have log routers which can ship data to the remote
 			// logs without using log router tags.
-			wait(logSystem->remoteRecovery);
+			co_await logSystem->remoteRecovery;
 		}
 	} else {
 		logSystem->hasRemoteServers = false;
@@ -3587,26 +3718,29 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		logSystem->remoteRecoveredVersion->set(MAX_VERSION);
 	}
 
-	return logSystem;
+	co_return logSystem;
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::trackRejoins(
+Future<Void> LogSystem::trackRejoins(
     UID dbgid,
     std::vector<std::pair<Reference<AsyncVar<OptionalInterface<TLogInterface>>>, Reference<IReplicationPolicy>>>
         logServers,
     FutureStream<struct TLogRejoinRequest> rejoinRequests) {
-	state std::map<UID, ReplyPromise<TLogRejoinReply>> lastReply;
-	state std::set<UID> logsWaiting;
-	state double startTime = now();
-	state Future<Void> warnTimeout = delay(SERVER_KNOBS->TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS);
+	std::map<UID, ReplyPromise<TLogRejoinReply>> lastReply;
+	std::set<UID> logsWaiting;
+	double startTime = now();
+	Future<Void> warnTimeout = delay(SERVER_KNOBS->TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS);
 
 	for (const auto& log : logServers) {
 		logsWaiting.insert(log.first->get().id());
 	}
 
 	try {
-		loop choose {
-			when(TLogRejoinRequest req = waitNext(rejoinRequests)) {
+		while (true) {
+			auto res = co_await race(rejoinRequests, warnTimeout);
+			if (res.index() == 0) {
+				TLogRejoinRequest req = std::get<0>(std::move(res));
+
 				int pos = -1;
 				for (int i = 0; i < logServers.size(); i++) {
 					if (logServers[i].first->get().id() == req.myInterface.id()) {
@@ -3634,8 +3768,8 @@ ACTOR Future<Void> TagPartitionedLogSystem::trackRejoins(
 					    .detail("Address", req.myInterface.commit.getEndpoint().getPrimaryAddress().toString());
 					req.reply.send(true);
 				}
-			}
-			when(wait(warnTimeout)) {
+			} else if (res.index() == 1) {
+				// warnTimeout fired
 				for (const auto& logId : logsWaiting) {
 					TraceEvent(SevWarnAlways, "TLogRejoinSlow", dbgid)
 					    .detail("Elapsed", startTime - now())
@@ -3651,32 +3785,35 @@ ACTOR Future<Void> TagPartitionedLogSystem::trackRejoins(
 	}
 }
 
-ACTOR Future<TLogLockResult> TagPartitionedLogSystem::lockTLog(
-    UID myID,
-    Reference<AsyncVar<OptionalInterface<TLogInterface>>> tlog) {
-	TraceEvent("TLogLockStarted", myID).detail("TLog", tlog->get().id()).detail("InfPresent", tlog->get().present());
-	loop {
-		choose {
-			when(TLogLockResult data = wait(
-			         tlog->get().present() ? brokenPromiseToNever(tlog->get().interf().lock.getReply<TLogLockResult>())
-			                               : Never())) {
-				TraceEvent("TLogLocked", myID).detail("TLog", tlog->get().id()).detail("End", data.end);
-				return data;
-			}
-			when(wait(tlog->onChange())) {}
+Future<TLogLockResult> LogSystem::lockTLog(UID myID, Reference<AsyncVar<OptionalInterface<TLogInterface>>> tlog) {
+	const auto initialTLog = tlog->get();
+	TraceEvent("TLogLockStarted", myID).detail("TLog", initialTLog.id()).detail("InfPresent", initialTLog.present());
+	while (true) {
+		const auto tlogInterface = tlog->get();
+		if (!tlogInterface.present()) {
+			co_await tlog->onChange();
+			continue;
+		}
+
+		auto res = co_await race(brokenPromiseToNever(tlogInterface.interf().lock.getReply<TLogLockResult>()),
+		                         tlog->onChange());
+		if (res.index() == 0) {
+			TLogLockResult data = std::get<0>(std::move(res));
+
+			TraceEvent("TLogLocked", myID).detail("TLog", tlogInterface.id()).detail("End", data.end);
+			co_return data;
 		}
 	}
 }
 
-ACTOR Future<Void> TagPartitionedLogSystem::trackTLogRecoveryActor(
-    std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> tlogs,
-    Reference<AsyncVar<Version>> recoveredVersion) {
+Future<Void> LogSystem::trackTLogRecoveryActor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> tlogs,
+                                               Reference<AsyncVar<Version>> recoveredVersion) {
 	if (!SERVER_KNOBS->TRACK_TLOG_RECOVERY) {
-		return Never();
+		co_await Future<Void>(Never());
 	}
 
-	state std::vector<Future<TrackTLogRecoveryReply>> individualTLogRecovery;
-	loop {
+	std::vector<Future<TrackTLogRecoveryReply>> individualTLogRecovery;
+	while (true) {
 		individualTLogRecovery.clear();
 		individualTLogRecovery.reserve(tlogs.size());
 		TrackTLogRecoveryRequest req(recoveredVersion->get());
@@ -3690,7 +3827,7 @@ ACTOR Future<Void> TagPartitionedLogSystem::trackTLogRecoveryActor(
 			    cluster_recovery_failed()));
 		}
 
-		wait(waitForAll(individualTLogRecovery));
+		co_await waitForAll(individualTLogRecovery);
 
 		Version currentRecoveredVersion = MAX_VERSION;
 		for (int i = 0; i < individualTLogRecovery.size(); ++i) {
diff --git a/fdbserver/LogSystemDiskQueueAdapter.actor.cpp b/fdbserver/logsystem/LogSystemDiskQueueAdapter.cpp
similarity index 74%
rename from fdbserver/LogSystemDiskQueueAdapter.actor.cpp
rename to fdbserver/logsystem/LogSystemDiskQueueAdapter.cpp
index 137cd235310..a0fc5afc3f8 100644
--- a/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
+++ b/fdbserver/logsystem/LogSystemDiskQueueAdapter.cpp
@@ -1,5 +1,5 @@
 /*
- * LogSystemDiskQueueAdapter.actor.cpp
+ * LogSystemDiskQueueAdapter.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,15 @@
  * limitations under the License.
  */
 
-#include "fdbserver/IDiskQueue.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/Knobs.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/core/IDiskQueue.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
+#include "fdbserver/core/Knobs.h"
+#include "flow/CoroUtils.h"
 
 class LogSystemDiskQueueAdapterImpl {
 public:
-	ACTOR static Future<Standalone<StringRef>> readNext(LogSystemDiskQueueAdapter* self, int bytes) {
+	static Future<Standalone<StringRef>> readNext(LogSystemDiskQueueAdapter* self, int bytes) {
 		while (self->recoveryQueueDataSize < bytes) {
 			if (self->recoveryLoc == self->logSystem->getEnd()) {
 				// Recovery will be complete once the current recoveryQueue is consumed, so we no longer need
@@ -41,12 +41,43 @@ class LogSystemDiskQueueAdapterImpl {
 			}
 
 			if (!self->cursor->hasMessage()) {
-				loop {
-					choose {
-						when(wait(self->cursor->getMore())) {
-							break;
-						}
-						when(wait(self->localityChanged)) {
+				while (true) {
+					auto choice = co_await race(self->cursor->getMore(),
+					                            self->localityChanged,
+					                            delay(self->peekTypeSwitches == 0
+					                                      ? SERVER_KNOBS->DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME
+					                                      : SERVER_KNOBS->DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME));
+					if (choice.index() == 0) {
+						break;
+					}
+					if (choice.index() == 1) {
+						self->cursor = self->logSystem->peekTxs(
+						    UID(),
+						    self->recoveryLoc,
+						    self->peekLocality ? self->peekLocality->get().primaryLocality : tagLocalityInvalid,
+						    self->peekLocality ? self->peekLocality->get().knownCommittedVersion : invalidVersion,
+						    self->totalRecoveredBytes == 0);
+						self->localityChanged = self->peekLocality->onChange();
+						continue;
+					}
+					if (choice.index() == 2) {
+						self->peekTypeSwitches++;
+						if (self->peekTypeSwitches % 3 == 1) {
+							self->cursor = self->logSystem->peekTxs(UID(),
+							                                        self->recoveryLoc,
+							                                        tagLocalityInvalid,
+							                                        invalidVersion,
+							                                        self->totalRecoveredBytes == 0);
+							self->localityChanged = Never();
+						} else if (self->peekTypeSwitches % 3 == 2) {
+							self->cursor = self->logSystem->peekTxs(
+							    UID(),
+							    self->recoveryLoc,
+							    self->peekLocality ? self->peekLocality->get().secondaryLocality : tagLocalityInvalid,
+							    self->peekLocality ? self->peekLocality->get().knownCommittedVersion : invalidVersion,
+							    self->totalRecoveredBytes == 0);
+							self->localityChanged = self->peekLocality->onChange();
+						} else {
 							self->cursor = self->logSystem->peekTxs(
 							    UID(),
 							    self->recoveryLoc,
@@ -55,39 +86,10 @@ class LogSystemDiskQueueAdapterImpl {
 							    self->totalRecoveredBytes == 0);
 							self->localityChanged = self->peekLocality->onChange();
 						}
-						when(wait(delay(self->peekTypeSwitches == 0
-						                    ? SERVER_KNOBS->DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME
-						                    : SERVER_KNOBS->DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME))) {
-							self->peekTypeSwitches++;
-							if (self->peekTypeSwitches % 3 == 1) {
-								self->cursor = self->logSystem->peekTxs(UID(),
-								                                        self->recoveryLoc,
-								                                        tagLocalityInvalid,
-								                                        invalidVersion,
-								                                        self->totalRecoveredBytes == 0);
-								self->localityChanged = Never();
-							} else if (self->peekTypeSwitches % 3 == 2) {
-								self->cursor = self->logSystem->peekTxs(
-								    UID(),
-								    self->recoveryLoc,
-								    self->peekLocality ? self->peekLocality->get().secondaryLocality
-								                       : tagLocalityInvalid,
-								    self->peekLocality ? self->peekLocality->get().knownCommittedVersion
-								                       : invalidVersion,
-								    self->totalRecoveredBytes == 0);
-								self->localityChanged = self->peekLocality->onChange();
-							} else {
-								self->cursor = self->logSystem->peekTxs(
-								    UID(),
-								    self->recoveryLoc,
-								    self->peekLocality ? self->peekLocality->get().primaryLocality : tagLocalityInvalid,
-								    self->peekLocality ? self->peekLocality->get().knownCommittedVersion
-								                       : invalidVersion,
-								    self->totalRecoveredBytes == 0);
-								self->localityChanged = self->peekLocality->onChange();
-							}
-						}
+						continue;
 					}
+
+					UNREACHABLE();
 				}
 				TraceEvent("PeekNextGetMore")
 				    .detail("Total", self->totalRecoveredBytes)
@@ -143,7 +145,7 @@ class LogSystemDiskQueueAdapterImpl {
 				}
 				if (!self->cursor->hasMessage()) {
 					self->recoveryLoc = self->cursor->version().version;
-					wait(yield());
+					co_await yield();
 					continue;
 				}
 			}
@@ -163,7 +165,7 @@ class LogSystemDiskQueueAdapterImpl {
 		}
 
 		if (self->recoveryQueueDataSize == 0)
-			return Standalone<StringRef>();
+			co_return Standalone<StringRef>();
 
 		ASSERT(self->recoveryQueue[0].size() == self->recoveryQueueDataSize);
 
@@ -172,10 +174,10 @@ class LogSystemDiskQueueAdapterImpl {
 		Standalone<StringRef> result(self->recoveryQueue[0].substr(0, bytes), self->recoveryQueue[0].arena());
 		self->recoveryQueue[0].contents() = self->recoveryQueue[0].substr(bytes);
 		self->recoveryQueueDataSize = self->recoveryQueue[0].size();
-		if (self->recoveryQueue[0].size() == 0) {
+		if (self->recoveryQueue[0].empty()) {
 			self->recoveryQueue.clear();
 		}
-		return result;
+		co_return result;
 	}
 };
 
@@ -191,8 +193,8 @@ IDiskQueue::location LogSystemDiskQueueAdapter::getNextReadLocation() const {
 }
 
 IDiskQueue::location LogSystemDiskQueueAdapter::push(StringRef contents) {
-	while (contents.size()) {
-		int remainder = pushedData.size() == 0 ? 0 : pushedData.back().capacity() - pushedData.back().size();
+	while (!contents.empty()) {
+		int remainder = pushedData.empty() ? 0 : pushedData.back().capacity() - pushedData.back().size();
 
 		if (remainder == 0) {
 			VectorRef<uint8_t> block;
@@ -250,7 +252,7 @@ Future<LogSystemDiskQueueAdapter::CommitMessage> LogSystemDiskQueueAdapter::getC
 	return pcm.getFuture();
 }
 
-LogSystemDiskQueueAdapter* openDiskQueueAdapter(Reference<ILogSystem> logSystem,
+LogSystemDiskQueueAdapter* openDiskQueueAdapter(Reference<LogSystem> logSystem,
                                                 Reference<AsyncVar<PeekTxsInfo>> peekLocality,
                                                 Version txsPoppedVersion) {
 	return new LogSystemDiskQueueAdapter(logSystem, peekLocality, txsPoppedVersion, true);
diff --git a/fdbserver/logsystem/LogSystemFactory.cpp b/fdbserver/logsystem/LogSystemFactory.cpp
new file mode 100644
index 00000000000..391c15a3c6e
--- /dev/null
+++ b/fdbserver/logsystem/LogSystemFactory.cpp
@@ -0,0 +1,44 @@
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/logsystem/LogSystem.h"
+
+Future<Void> recoverAndEndLogSystemEpoch(Reference<AsyncVar<Reference<LogSystem>>> const& outLogSystem,
+                                         UID const& dbgid,
+                                         DBCoreState const& oldState,
+                                         FutureStream<TLogRejoinRequest> const& rejoins,
+                                         LocalityData const& locality,
+                                         bool* forceRecovery) {
+	return LogSystem::recoverAndEndEpoch(outLogSystem, dbgid, oldState, rejoins, locality, forceRecovery);
+}
+
+Reference<LogSystem> makeLogSystemFromLogSystemConfig(UID const& dbgid,
+                                                      LocalityData const& locality,
+                                                      LogSystemConfig const& conf,
+                                                      bool excludeRemote,
+                                                      bool useRecoveredAt,
+                                                      Optional<PromiseStream<Future<Void>>> addActor) {
+	if (conf.logSystemType == LogSystemType::empty) {
+		return Reference<LogSystem>();
+	} else if (conf.logSystemType == LogSystemType::tagPartitioned) {
+		return LogSystem::fromLogSystemConfig(dbgid, locality, conf, excludeRemote, useRecoveredAt, addActor);
+	}
+	throw internal_error();
+}
+
+Reference<LogSystem> makeOldLogSystemFromLogSystemConfig(UID const& dbgid,
+                                                         LocalityData const& locality,
+                                                         LogSystemConfig const& conf) {
+	if (conf.logSystemType == LogSystemType::empty) {
+		return Reference<LogSystem>();
+	} else if (conf.logSystemType == LogSystemType::tagPartitioned) {
+		return LogSystem::fromOldLogSystemConfig(dbgid, locality, conf);
+	}
+	throw internal_error();
+}
+
+Reference<LogSystem> makeLogSystemFromServerDBInfo(UID const& dbgid,
+                                                   ServerDBInfo const& dbInfo,
+                                                   bool useRecoveredAt,
+                                                   Optional<PromiseStream<Future<Void>>> addActor) {
+	return makeLogSystemFromLogSystemConfig(
+	    dbgid, dbInfo.myLocality, dbInfo.logSystemConfig, false, useRecoveredAt, addActor);
+}
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/logsystem/LogSystemPeekCursor.cpp
similarity index 67%
rename from fdbserver/LogSystemPeekCursor.actor.cpp
rename to fdbserver/logsystem/LogSystemPeekCursor.cpp
index 00aa64541c2..7f166fe58fb 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/logsystem/LogSystemPeekCursor.cpp
@@ -1,5 +1,5 @@
 /*
- * LogSystemPeekCursor.actor.cpp
+ * LogSystemPeekCursor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,24 +18,17 @@
  * limitations under the License.
  */
 
-#include "fdbserver/LogSystem.h"
+#include "fdbserver/logsystem/LogSystem.h"
 #include "fdbrpc/FailureMonitor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MutationTracking.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MutationTracking.h"
 #include "fdbrpc/ReplicationUtils.h"
 #include "flow/DebugTrace.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "flow/CoroUtils.h"
 
-// create a peek stream for cursor when it's possible
-ACTOR Future<Void> tryEstablishPeekStream(ILogSystem::ServerPeekCursor* self) {
-	if (self->peekReplyStream.present())
-		return Void();
-	else if (!self->interf || !self->interf->get().present()) {
-		self->peekReplyStream.reset();
-		return Never();
-	}
-	wait(IFailureMonitor::failureMonitor().onStateEqual(self->interf->get().interf().peekStreamMessages.getEndpoint(),
-	                                                    FailureStatus(false)));
+Future<Void> tryEstablishPeekStreamImpl(ServerPeekCursor* self) {
+	co_await IFailureMonitor::failureMonitor().onStateEqual(
+	    self->interf->get().interf().peekStreamMessages.getEndpoint(), FailureStatus(false));
 
 	auto req = TLogPeekStreamRequest(self->messageVersion.version,
 	                                 self->tag,
@@ -49,16 +42,26 @@ ACTOR Future<Void> tryEstablishPeekStream(ILogSystem::ServerPeekCursor* self) {
 	    .detail("PeerAddr", self->interf->get().interf().peekStreamMessages.getEndpoint().getPrimaryAddress())
 	    .detail("PeerAddress", self->interf->get().interf().peekStreamMessages.getEndpoint().getPrimaryAddress())
 	    .detail("PeerToken", self->interf->get().interf().peekStreamMessages.getEndpoint().token);
-	return Void();
 }
 
-ILogSystem::ServerPeekCursor::ServerPeekCursor(Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf,
-                                               Tag tag,
-                                               Version begin,
-                                               Version end,
-                                               bool returnIfBlocked,
-                                               bool parallelGetMore,
-                                               bool returnEmptyIfStopped)
+// create a peek stream for cursor when it's possible
+Future<Void> tryEstablishPeekStream(ServerPeekCursor* self) {
+	if (self->peekReplyStream.present())
+		return Void();
+	else if (!self->interf || !self->interf->get().present()) {
+		self->peekReplyStream.reset();
+		return Never();
+	}
+	return tryEstablishPeekStreamImpl(self);
+}
+
+ServerPeekCursor::ServerPeekCursor(Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf,
+                                   Tag tag,
+                                   Version begin,
+                                   Version end,
+                                   bool returnIfBlocked,
+                                   bool parallelGetMore,
+                                   bool returnEmptyIfStopped)
   : interf(interf), tag(tag), rd(results.arena, results.messages, Unversioned()), messageVersion(begin), end(end),
     poppedVersion(0), hasMsg(false), randomID(deterministicRandom()->randomUniqueID()),
     returnIfBlocked(returnIfBlocked), onlySpilled(false), parallelGetMore(parallelGetMore),
@@ -75,13 +78,13 @@ ILogSystem::ServerPeekCursor::ServerPeekCursor(Reference<AsyncVar<OptionalInterf
 	    .detail("End", end);
 }
 
-ILogSystem::ServerPeekCursor::ServerPeekCursor(TLogPeekReply const& results,
-                                               LogMessageVersion const& messageVersion,
-                                               LogMessageVersion const& end,
-                                               TagsAndMessage const& message,
-                                               bool hasMsg,
-                                               Version poppedVersion,
-                                               Tag tag)
+ServerPeekCursor::ServerPeekCursor(TLogPeekReply const& results,
+                                   LogMessageVersion const& messageVersion,
+                                   LogMessageVersion const& end,
+                                   TagsAndMessage const& message,
+                                   bool hasMsg,
+                                   Version poppedVersion,
+                                   Tag tag)
   : tag(tag), results(results), rd(results.arena, results.messages, Unversioned()), messageVersion(messageVersion),
     end(end), poppedVersion(poppedVersion), messageAndTags(message), hasMsg(hasMsg),
     randomID(deterministicRandom()->randomUniqueID()), returnIfBlocked(false), onlySpilled(false),
@@ -96,29 +99,28 @@ ILogSystem::ServerPeekCursor::ServerPeekCursor(TLogPeekReply const& results,
 	advanceTo(messageVersion);
 }
 
-Reference<ILogSystem::IPeekCursor> ILogSystem::ServerPeekCursor::cloneNoMore() {
-	return makeReference<ILogSystem::ServerPeekCursor>(
-	    results, messageVersion, end, messageAndTags, hasMsg, poppedVersion, tag);
+Reference<IPeekCursor> ServerPeekCursor::cloneNoMore() {
+	return makeReference<ServerPeekCursor>(results, messageVersion, end, messageAndTags, hasMsg, poppedVersion, tag);
 }
 
-void ILogSystem::ServerPeekCursor::setProtocolVersion(ProtocolVersion version) {
+void ServerPeekCursor::setProtocolVersion(ProtocolVersion version) {
 	rd.setProtocolVersion(version);
 }
 
-Arena& ILogSystem::ServerPeekCursor::arena() {
+Arena& ServerPeekCursor::arena() {
 	return results.arena;
 }
 
-ArenaReader* ILogSystem::ServerPeekCursor::reader() {
+ArenaReader* ServerPeekCursor::reader() {
 	return &rd;
 }
 
-bool ILogSystem::ServerPeekCursor::hasMessage() const {
+bool ServerPeekCursor::hasMessage() const {
 	//TraceEvent("SPC_HasMessage", randomID).detail("HasMsg", hasMsg);
 	return hasMsg;
 }
 
-void ILogSystem::ServerPeekCursor::nextMessage() {
+void ServerPeekCursor::nextMessage() {
 	DebugLogTraceEvent("SPC_NextMessage", randomID)
 	    .detail("Tag", tag.toString())
 	    .detail("MessageVersion", messageVersion.toString());
@@ -158,25 +160,25 @@ void ILogSystem::ServerPeekCursor::nextMessage() {
 	    .detail("MessageVersion", messageVersion.toString());
 }
 
-StringRef ILogSystem::ServerPeekCursor::getMessage() {
+StringRef ServerPeekCursor::getMessage() {
 	DebugLogTraceEvent("SPC_GetMessage", randomID).detail("Tag", tag.toString());
 	StringRef message = messageAndTags.getMessageWithoutTags();
 	rd.readBytes(message.size()); // Consumes the message.
 	return message;
 }
 
-StringRef ILogSystem::ServerPeekCursor::getMessageWithTags() {
+StringRef ServerPeekCursor::getMessageWithTags() {
 	StringRef rawMessage = messageAndTags.getRawMessage();
 	rd.readBytes(rawMessage.size() -
 	             TagsAndMessage::getHeaderSize(messageAndTags.tags.size())); // Consumes the message.
 	return rawMessage;
 }
 
-VectorRef<Tag> ILogSystem::ServerPeekCursor::getTags() const {
+VectorRef<Tag> ServerPeekCursor::getTags() const {
 	return messageAndTags.tags;
 }
 
-void ILogSystem::ServerPeekCursor::advanceTo(LogMessageVersion n) {
+void ServerPeekCursor::advanceTo(LogMessageVersion n) {
 	//TraceEvent("SPC_AdvanceTo", randomID).detail("N", n.toString());
 	while (messageVersion < n && hasMessage()) {
 		getMessage();
@@ -195,7 +197,7 @@ void ILogSystem::ServerPeekCursor::advanceTo(LogMessageVersion n) {
 
 // This function is called after the cursor received one TLogPeekReply to update its members, which is the common logic
 // in getMore helper functions.
-void updateCursorWithReply(ILogSystem::ServerPeekCursor* self, const TLogPeekReply& res) {
+void updateCursorWithReply(ServerPeekCursor* self, const TLogPeekReply& res) {
 	self->results = res;
 	self->onlySpilled = res.onlySpilled;
 	if (res.popped.present())
@@ -207,11 +209,11 @@ void updateCursorWithReply(ILogSystem::ServerPeekCursor* self, const TLogPeekRep
 	self->advanceTo(skipSeq);
 }
 
-ACTOR Future<Void> resetChecker(ILogSystem::ServerPeekCursor* self, NetworkAddress addr) {
+Future<Void> resetChecker(ServerPeekCursor* self, NetworkAddress addr) {
 	self->slowReplies = 0;
 	self->unknownReplies = 0;
 	self->fastReplies = 0;
-	wait(delay(SERVER_KNOBS->PEEK_STATS_INTERVAL));
+	co_await delay(SERVER_KNOBS->PEEK_STATS_INTERVAL);
 	TraceEvent("SlowPeekStats", self->randomID)
 	    .detail("PeerAddress", addr)
 	    .detail("SlowReplies", self->slowReplies)
@@ -229,15 +231,13 @@ ACTOR Future<Void> resetChecker(ILogSystem::ServerPeekCursor* self, NetworkAddre
 		FlowTransport::transport().resetConnection(addr);
 		self->lastReset = now();
 	}
-	return Void();
 }
 
-ACTOR Future<TLogPeekReply> recordRequestMetrics(ILogSystem::ServerPeekCursor* self,
-                                                 NetworkAddress addr,
-                                                 Future<TLogPeekReply> in) {
+Future<TLogPeekReply> recordRequestMetrics(ServerPeekCursor* self, NetworkAddress addr, Future<TLogPeekReply> in) {
+	Error err;
 	try {
-		state double startTime = now();
-		TLogPeekReply t = wait(in);
+		double startTime = now();
+		TLogPeekReply t = co_await in;
 		if (now() - self->lastReset > SERVER_KNOBS->PEEK_RESET_INTERVAL) {
 			if (now() - startTime > SERVER_KNOBS->PEEK_MAX_LATENCY) {
 				if (t.messages.size() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES || SERVER_KNOBS->PEEK_COUNT_SMALL_MESSAGES) {
@@ -252,27 +252,18 @@ ACTOR Future<TLogPeekReply> recordRequestMetrics(ILogSystem::ServerPeekCursor* s
 				self->fastReplies++;
 			}
 		}
-		return t;
+		co_return t;
 	} catch (Error& e) {
-		if (e.code() != error_code_broken_promise)
-			throw;
-		wait(Never()); // never return
-		throw internal_error(); // does not happen
+		err = e;
 	}
+	if (err.code() != error_code_broken_promise)
+		throw err;
+	co_await Future<Void>(Never()); // never return
+	throw internal_error(); // does not happen
 }
 
-ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self, TaskPriority taskID) {
-	if (!self->interf || self->isExhausted()) {
-		if (self->hasMessage())
-			return Void();
-		return Never();
-	}
-
-	if (!self->interfaceChanged.isValid()) {
-		self->interfaceChanged = self->interf->onChange();
-	}
-
-	loop {
+Future<Void> serverPeekParallelGetMoreImpl(ServerPeekCursor* self, TaskPriority taskID) {
+	while (true) {
 		DebugLogTraceEvent("SPC_GetMoreP", self->randomID)
 		    .detail("Tag", self->tag.toString())
 		    .detail("Has", self->hasMessage())
@@ -282,7 +273,7 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 		    .detail("Sizes", self->futureResults.size())
 		    .detail("Interf", self->interf->get().present() ? self->interf->get().id() : UID());
 
-		state Version expectedBegin = self->messageVersion.version;
+		Version expectedBegin = self->messageVersion.version;
 		try {
 			if (self->parallelGetMore || self->onlySpilled) {
 				while (self->futureResults.size() < SERVER_KNOBS->PARALLEL_GET_MORE_REQUESTS &&
@@ -304,35 +295,37 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 					throw operation_obsolete();
 				}
 			} else if (self->futureResults.size() == 0) {
-				return Void();
+				co_return;
 			}
 
 			if (self->hasMessage())
-				return Void();
+				co_return;
 
-			choose {
-				when(TLogPeekReply res = wait(self->interf->get().present() ? self->futureResults.front() : Never())) {
-					if (res.begin.get() != expectedBegin) {
-						throw operation_obsolete();
-					}
-					expectedBegin = res.end;
-					self->futureResults.pop_front();
-					updateCursorWithReply(self, res);
-					DebugLogTraceEvent("SPC_GetMoreReply", self->randomID)
-					    .detail("Has", self->hasMessage())
-					    .detail("Tag", self->tag.toString())
-					    .detail("End", res.end)
-					    .detail("Size", self->futureResults.size())
-					    .detail("Popped", res.popped.present() ? res.popped.get() : 0);
-					return Void();
-				}
-				when(wait(self->interfaceChanged)) {
-					self->interfaceChanged = self->interf->onChange();
-					self->randomID = deterministicRandom()->randomUniqueID();
-					self->sequence = 0;
-					self->onlySpilled = false;
-					self->futureResults.clear();
+			Future<TLogPeekReply> peekReply = self->interf->get().present() ? self->futureResults.front() : Never();
+			auto res = co_await race(peekReply, self->interfaceChanged);
+			if (res.index() == 0) {
+				TLogPeekReply reply = std::get<0>(std::move(res));
+
+				if (reply.begin.get() != expectedBegin) {
+					throw operation_obsolete();
 				}
+				self->futureResults.pop_front();
+				updateCursorWithReply(self, reply);
+				DebugLogTraceEvent("SPC_GetMoreReply", self->randomID)
+				    .detail("Has", self->hasMessage())
+				    .detail("Tag", self->tag.toString())
+				    .detail("End", reply.end)
+				    .detail("Size", self->futureResults.size())
+				    .detail("Popped", reply.popped.present() ? reply.popped.get() : 0);
+				co_return;
+			} else if (res.index() == 1) {
+				self->interfaceChanged = self->interf->onChange();
+				self->randomID = deterministicRandom()->randomUniqueID();
+				self->sequence = 0;
+				self->onlySpilled = false;
+				self->futureResults.clear();
+			} else {
+				UNREACHABLE();
 			}
 		} catch (Error& e) {
 			DebugLogTraceEvent("PeekCursorError", self->randomID)
@@ -343,7 +336,7 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 
 			if (e.code() == error_code_end_of_stream) {
 				self->end.reset(self->messageVersion.version);
-				return Void();
+				co_return;
 			} else if (e.code() == error_code_timed_out || e.code() == error_code_operation_obsolete) {
 				TraceEvent ev("PeekCursorTimedOut", self->randomID);
 				// We *should* never get timed_out(), as it means the TLog got stuck while handling a parallel peek,
@@ -375,113 +368,144 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 	}
 }
 
-ACTOR Future<Void> serverPeekStreamGetMore(ILogSystem::ServerPeekCursor* self, TaskPriority taskID) {
+Future<Void> serverPeekParallelGetMore(ServerPeekCursor* self, TaskPriority taskID) {
 	if (!self->interf || self->isExhausted()) {
-		self->peekReplyStream.reset();
 		if (self->hasMessage())
 			return Void();
 		return Never();
 	}
 
-	loop {
+	if (!self->interfaceChanged.isValid()) {
+		self->interfaceChanged = self->interf->onChange();
+	}
+
+	return serverPeekParallelGetMoreImpl(self, taskID);
+}
+
+Future<Void> serverPeekStreamGetMoreImpl(ServerPeekCursor* self, TaskPriority taskID) {
+	while (true) {
+		Optional<Error> err;
 		try {
-			state Version expectedBegin = self->messageVersion.version;
-			state Future<TLogPeekReply> fPeekReply = self->peekReplyStream.present()
-			                                             ? map(waitAndForward(self->peekReplyStream.get().getFuture()),
-			                                                   [](const TLogPeekStreamReply& r) { return r.rep; })
-			                                             : Never();
-			choose {
-				when(wait(self->peekReplyStream.present() ? Never() : tryEstablishPeekStream(self))) {}
-				when(wait(self->interf->onChange())) {
-					self->onlySpilled = false;
-					self->peekReplyStream.reset();
-				}
-				when(TLogPeekReply res = wait(
-				         self->peekReplyStream.present()
-				             ? recordRequestMetrics(
-				                   self,
-				                   self->interf->get().interf().peekStreamMessages.getEndpoint().getPrimaryAddress(),
-				                   fPeekReply)
-				             : Never())) {
-					if (res.begin.get() != expectedBegin) {
-						throw operation_obsolete();
-					}
-					updateCursorWithReply(self, res);
-					expectedBegin = res.end;
-					DebugLogTraceEvent(SevDebug, "SPC_GetMoreB", self->randomID)
-					    .detail("Tag", self->tag)
-					    .detail("Has", self->hasMessage())
-					    .detail("End", res.end)
-					    .detail("Popped", res.popped.present() ? res.popped.get() : 0);
-
-					// NOTE: delay is necessary here since ReplyPromiseStream delivers reply on high priority. Here we
-					// change the priority to the intended one.
-					wait(delay(0, taskID));
-					return Void();
+			Version expectedBegin = self->messageVersion.version;
+			Future<TLogPeekReply> fPeekReply = self->peekReplyStream.present()
+			                                       ? map(waitAndForward(self->peekReplyStream.get().getFuture()),
+			                                             [](const TLogPeekStreamReply& r) { return r.rep; })
+			                                       : Never();
+			Future<Void> establishStream = self->peekReplyStream.present() ? Never() : tryEstablishPeekStream(self);
+			Future<TLogPeekReply> metricsReply =
+			    self->peekReplyStream.present()
+			        ? recordRequestMetrics(
+			              self,
+			              self->interf->get().interf().peekStreamMessages.getEndpoint().getPrimaryAddress(),
+			              fPeekReply)
+			        : Never();
+			auto res = co_await race(establishStream, self->interf->onChange(), metricsReply);
+			if (res.index() == 0) {
+			} else if (res.index() == 1) {
+				self->onlySpilled = false;
+				self->peekReplyStream.reset();
+			} else if (res.index() == 2) {
+				TLogPeekReply reply = std::get<2>(std::move(res));
+
+				if (reply.begin.get() != expectedBegin) {
+					throw operation_obsolete();
 				}
+				updateCursorWithReply(self, reply);
+				DebugLogTraceEvent(SevDebug, "SPC_GetMoreB", self->randomID)
+				    .detail("Tag", self->tag)
+				    .detail("Has", self->hasMessage())
+				    .detail("End", reply.end)
+				    .detail("Popped", reply.popped.present() ? reply.popped.get() : 0);
+
+				// NOTE: delay is necessary here since ReplyPromiseStream delivers reply on high priority. Here we
+				// change the priority to the intended one.
+				co_await delay(0, taskID);
+				co_return;
+			} else {
+				UNREACHABLE();
 			}
 		} catch (Error& e) {
+			err = e;
+		}
+		if (err.present()) {
 			DebugLogTraceEvent(SevDebug, "SPC_GetMoreB_Error", self->randomID)
-			    .errorUnsuppressed(e)
+			    .errorUnsuppressed(err.get())
 			    .detail("Tag", self->tag);
-			if (e.code() == error_code_connection_failed || e.code() == error_code_operation_obsolete ||
-			    e.code() == error_code_request_maybe_delivered) {
+			if (err.get().code() == error_code_connection_failed || err.get().code() == error_code_operation_obsolete ||
+			    err.get().code() == error_code_request_maybe_delivered) {
 				// NOTE: delay in order to avoid the endless retry loop block other tasks
 				self->peekReplyStream.reset();
-				wait(delay(0));
-			} else if (e.code() == error_code_end_of_stream) {
+				co_await delay(0);
+			} else if (err.get().code() == error_code_end_of_stream) {
 				self->peekReplyStream.reset();
 				self->end.reset(self->messageVersion.version);
-				return Void();
+				co_return;
 			} else {
-				throw;
+				throw err.get();
 			}
 		}
 	}
 }
 
-ACTOR Future<Void> serverPeekGetMore(ILogSystem::ServerPeekCursor* self, TaskPriority taskID) {
+Future<Void> serverPeekStreamGetMore(ServerPeekCursor* self, TaskPriority taskID) {
 	if (!self->interf || self->isExhausted()) {
+		self->peekReplyStream.reset();
+		if (self->hasMessage())
+			return Void();
 		return Never();
 	}
+	return serverPeekStreamGetMoreImpl(self, taskID);
+}
+
+Future<Void> serverPeekGetMoreImpl(ServerPeekCursor* self, TaskPriority taskID) {
 	try {
-		loop {
-			choose {
-				when(TLogPeekReply res =
-				         wait(self->interf->get().present()
-				                  ? brokenPromiseToNever(self->interf->get().interf().peekMessages.getReply(
-				                        TLogPeekRequest(self->messageVersion.version,
-				                                        self->tag,
-				                                        self->returnIfBlocked,
-				                                        self->onlySpilled,
-				                                        Optional<std::pair<UID, int>>(),
-				                                        self->end.version,
-				                                        self->returnEmptyIfStopped),
-				                        taskID))
-				                  : Never())) {
-					updateCursorWithReply(self, res);
-					DebugLogTraceEvent("SPC_GetMoreB", self->randomID)
-					    .detail("Tag", self->tag.toString())
-					    .detail("Has", self->hasMessage())
-					    .detail("End", res.end)
-					    .detail("Popped", res.popped.present() ? res.popped.get() : 0);
-					return Void();
-				}
-				when(wait(self->interf->onChange())) {
-					self->onlySpilled = false;
-				}
+		while (true) {
+			Future<TLogPeekReply> peekReply = Never();
+			if (self->interf->get().present()) {
+				peekReply = brokenPromiseToNever(
+				    self->interf->get().interf().peekMessages.getReply(TLogPeekRequest(self->messageVersion.version,
+				                                                                       self->tag,
+				                                                                       self->returnIfBlocked,
+				                                                                       self->onlySpilled,
+				                                                                       Optional<std::pair<UID, int>>(),
+				                                                                       self->end.version,
+				                                                                       self->returnEmptyIfStopped),
+				                                                       taskID));
+			}
+			auto res = co_await race(peekReply, self->interf->onChange());
+			if (res.index() == 0) {
+				TLogPeekReply reply = std::get<0>(std::move(res));
+
+				updateCursorWithReply(self, reply);
+				DebugLogTraceEvent("SPC_GetMoreB", self->randomID)
+				    .detail("Tag", self->tag.toString())
+				    .detail("Has", self->hasMessage())
+				    .detail("End", reply.end)
+				    .detail("Popped", reply.popped.present() ? reply.popped.get() : 0);
+				co_return;
+			} else if (res.index() == 1) {
+				self->onlySpilled = false;
+			} else {
+				UNREACHABLE();
 			}
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_end_of_stream) {
 			self->end.reset(self->messageVersion.version);
-			return Void();
+			co_return;
 		}
 		throw e;
 	}
 }
 
-Future<Void> ILogSystem::ServerPeekCursor::getMore(TaskPriority taskID) {
+Future<Void> serverPeekGetMore(ServerPeekCursor* self, TaskPriority taskID) {
+	if (!self->interf || self->isExhausted()) {
+		return Never();
+	}
+	return serverPeekGetMoreImpl(self, taskID);
+}
+
+Future<Void> ServerPeekCursor::getMore(TaskPriority taskID) {
 	DebugLogTraceEvent("SPC_GetMore", randomID)
 	    .detail("Tag", tag.toString())
 	    .detail("HasMessage", hasMessage())
@@ -505,27 +529,26 @@ Future<Void> ILogSystem::ServerPeekCursor::getMore(TaskPriority taskID) {
 	return more;
 }
 
-ACTOR Future<Void> serverPeekOnFailed(ILogSystem::ServerPeekCursor const* self) {
-	loop {
-		choose {
-			when(wait(self->interf->get().present()
-			              ? IFailureMonitor::failureMonitor().onStateEqual(
-			                    self->interf->get().interf().peekMessages.getEndpoint(), FailureStatus())
-			              : Never())) {
-				return Void();
-			}
-			when(wait(self->interf->get().present()
-			              ? IFailureMonitor::failureMonitor().onStateEqual(
-			                    self->interf->get().interf().peekStreamMessages.getEndpoint(), FailureStatus())
-			              : Never())) {
-				return Void();
-			}
-			when(wait(self->interf->onChange())) {}
+Future<Void> serverPeekOnFailed(ServerPeekCursor const* self) {
+	while (true) {
+		Future<Void> peekMessagesFailed = Never();
+		Future<Void> peekStreamMessagesFailed = Never();
+		if (self->interf->get().present()) {
+			peekMessagesFailed = IFailureMonitor::failureMonitor().onStateEqual(
+			    self->interf->get().interf().peekMessages.getEndpoint(), FailureStatus());
+			peekStreamMessagesFailed = IFailureMonitor::failureMonitor().onStateEqual(
+			    self->interf->get().interf().peekStreamMessages.getEndpoint(), FailureStatus());
+		}
+		auto res = co_await race(peekMessagesFailed, peekStreamMessagesFailed, self->interf->onChange());
+		if (res.index() == 0 || res.index() == 1) {
+			co_return;
+		} else if (res.index() != 2) {
+			UNREACHABLE();
 		}
 	}
 }
 
-Future<Void> ILogSystem::ServerPeekCursor::onFailed() const {
+Future<Void> ServerPeekCursor::onFailed() const {
 	return serverPeekOnFailed(this);
 }
 
@@ -541,38 +564,38 @@ bool isAvailable(Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& in
 	           .isAvailable();
 }
 
-bool ILogSystem::ServerPeekCursor::isActive() const {
+bool ServerPeekCursor::isActive() const {
 	if (isExhausted()) {
 		return false;
 	}
 	return isAvailable(interf);
 }
 
-bool ILogSystem::ServerPeekCursor::isExhausted() const {
+bool ServerPeekCursor::isExhausted() const {
 	return messageVersion >= end;
 }
 
-const LogMessageVersion& ILogSystem::ServerPeekCursor::version() const {
+const LogMessageVersion& ServerPeekCursor::version() const {
 	return messageVersion;
 } // Call only after nextMessage().  The sequence of the current message, or results.end if nextMessage() has returned
   // false.
 
-Version ILogSystem::ServerPeekCursor::getMinKnownCommittedVersion() const {
+Version ServerPeekCursor::getMinKnownCommittedVersion() const {
 	return results.minKnownCommittedVersion;
 }
 
-Optional<UID> ILogSystem::ServerPeekCursor::getPrimaryPeekLocation() const {
+Optional<UID> ServerPeekCursor::getPrimaryPeekLocation() const {
 	if (interf && interf->get().present()) {
 		return interf->get().id();
 	}
 	return Optional<UID>();
 }
 
-Optional<UID> ILogSystem::ServerPeekCursor::getCurrentPeekLocation() const {
-	return ILogSystem::ServerPeekCursor::getPrimaryPeekLocation();
+Optional<UID> ServerPeekCursor::getCurrentPeekLocation() const {
+	return ServerPeekCursor::getPrimaryPeekLocation();
 }
 
-Version ILogSystem::ServerPeekCursor::popped() const {
+Version ServerPeekCursor::popped() const {
 	return poppedVersion;
 }
 
@@ -635,26 +658,24 @@ static bool canReturnEmptyVersionRange(
 	return false;
 }
 
-ILogSystem::MergedPeekCursor::MergedPeekCursor(std::vector<Reference<ILogSystem::IPeekCursor>> const& serverCursors,
-                                               Version begin)
+MergedPeekCursor::MergedPeekCursor(std::vector<Reference<IPeekCursor>> const& serverCursors, Version begin)
   : serverCursors(serverCursors), tag(invalidTag), bestServer(-1), currentCursor(0), readQuorum(serverCursors.size()),
     messageVersion(begin), hasNextMessage(false), randomID(deterministicRandom()->randomUniqueID()),
     tLogReplicationFactor(0) {
 	sortedVersions.resize(serverCursors.size());
 }
 
-ILogSystem::MergedPeekCursor::MergedPeekCursor(
-    std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
-    int bestServerLogId,
-    int readQuorum,
-    Tag tag,
-    Version begin,
-    Version end,
-    bool parallelGetMore,
-    std::vector<LocalityData> const& tLogLocalities,
-    Reference<IReplicationPolicy> const tLogPolicy,
-    int tLogReplicationFactor,
-    const Optional<std::vector<uint16_t>>& knownLockedTLogIds)
+MergedPeekCursor::MergedPeekCursor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
+                                   int bestServerLogId,
+                                   int readQuorum,
+                                   Tag tag,
+                                   Version begin,
+                                   Version end,
+                                   bool parallelGetMore,
+                                   std::vector<LocalityData> const& tLogLocalities,
+                                   Reference<IReplicationPolicy> const tLogPolicy,
+                                   int tLogReplicationFactor,
+                                   const Optional<std::vector<uint16_t>>& knownLockedTLogIds)
   : tag(tag), bestServer(bestServerLogId), currentCursor(0), readQuorum(readQuorum), messageVersion(begin),
     hasNextMessage(false), randomID(deterministicRandom()->randomUniqueID()),
     tLogReplicationFactor(tLogReplicationFactor) {
@@ -675,7 +696,7 @@ ILogSystem::MergedPeekCursor::MergedPeekCursor(
 		    ((SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST && end != std::numeric_limits<Version>::max())
 		         ? canReturnEmptyVersionRange(bestServer, i /*currentServer*/, end, knownLockedTLogIds)
 		         : false);
-		auto cursor = makeReference<ILogSystem::ServerPeekCursor>(
+		auto cursor = makeReference<ServerPeekCursor>(
 		    logServers[i], tag, begin, end, bestServer >= 0, parallelGetMore, returnEmptyIfStopped);
 		//TraceEvent("MPC_Starting", randomID).detail("Cursor", cursor->randomID).detail("End", end);
 		serverCursors.push_back(cursor);
@@ -683,13 +704,13 @@ ILogSystem::MergedPeekCursor::MergedPeekCursor(
 	sortedVersions.resize(serverCursors.size());
 }
 
-ILogSystem::MergedPeekCursor::MergedPeekCursor(std::vector<Reference<ILogSystem::IPeekCursor>> const& serverCursors,
-                                               LogMessageVersion const& messageVersion,
-                                               int bestServer,
-                                               int readQuorum,
-                                               Optional<LogMessageVersion> nextVersion,
-                                               Reference<LogSet> logSet,
-                                               int tLogReplicationFactor)
+MergedPeekCursor::MergedPeekCursor(std::vector<Reference<IPeekCursor>> const& serverCursors,
+                                   LogMessageVersion const& messageVersion,
+                                   int bestServer,
+                                   int readQuorum,
+                                   Optional<LogMessageVersion> nextVersion,
+                                   Reference<LogSet> logSet,
+                                   int tLogReplicationFactor)
   : logSet(logSet), serverCursors(serverCursors), bestServer(bestServer), currentCursor(0), readQuorum(readQuorum),
     nextVersion(nextVersion), messageVersion(messageVersion), hasNextMessage(false),
     randomID(deterministicRandom()->randomUniqueID()), tLogReplicationFactor(tLogReplicationFactor) {
@@ -697,16 +718,16 @@ ILogSystem::MergedPeekCursor::MergedPeekCursor(std::vector<Reference<ILogSystem:
 	calcHasMessage();
 }
 
-Reference<ILogSystem::IPeekCursor> ILogSystem::MergedPeekCursor::cloneNoMore() {
-	std::vector<Reference<ILogSystem::IPeekCursor>> cursors;
+Reference<IPeekCursor> MergedPeekCursor::cloneNoMore() {
+	std::vector<Reference<IPeekCursor>> cursors;
 	for (auto it : serverCursors) {
 		cursors.push_back(it->cloneNoMore());
 	}
-	return makeReference<ILogSystem::MergedPeekCursor>(
+	return makeReference<MergedPeekCursor>(
 	    cursors, messageVersion, bestServer, readQuorum, nextVersion, logSet, tLogReplicationFactor);
 }
 
-void ILogSystem::MergedPeekCursor::setProtocolVersion(ProtocolVersion version) {
+void MergedPeekCursor::setProtocolVersion(ProtocolVersion version) {
 	for (auto it : serverCursors) {
 		if (it->hasMessage()) {
 			it->setProtocolVersion(version);
@@ -714,15 +735,15 @@ void ILogSystem::MergedPeekCursor::setProtocolVersion(ProtocolVersion version) {
 	}
 }
 
-Arena& ILogSystem::MergedPeekCursor::arena() {
+Arena& MergedPeekCursor::arena() {
 	return serverCursors[currentCursor]->arena();
 }
 
-ArenaReader* ILogSystem::MergedPeekCursor::reader() {
+ArenaReader* MergedPeekCursor::reader() {
 	return serverCursors[currentCursor]->reader();
 }
 
-void ILogSystem::MergedPeekCursor::calcHasMessage() {
+void MergedPeekCursor::calcHasMessage() {
 	if (bestServer >= 0) {
 		if (nextVersion.present()) {
 			serverCursors[bestServer]->advanceTo(nextVersion.get());
@@ -753,8 +774,8 @@ void ILogSystem::MergedPeekCursor::calcHasMessage() {
 	}
 }
 
-void ILogSystem::MergedPeekCursor::updateMessage(bool usePolicy) {
-	loop {
+void MergedPeekCursor::updateMessage(bool usePolicy) {
+	while (true) {
 		bool advancedPast = false;
 		sortedVersions.clear();
 		for (int i = 0; i < serverCursors.size(); i++) {
@@ -809,11 +830,11 @@ void ILogSystem::MergedPeekCursor::updateMessage(bool usePolicy) {
 	}
 }
 
-bool ILogSystem::MergedPeekCursor::hasMessage() const {
+bool MergedPeekCursor::hasMessage() const {
 	return hasNextMessage;
 }
 
-void ILogSystem::MergedPeekCursor::nextMessage() {
+void MergedPeekCursor::nextMessage() {
 	nextVersion = version();
 	nextVersion.get().sub++;
 	serverCursors[currentCursor]->nextMessage();
@@ -821,19 +842,19 @@ void ILogSystem::MergedPeekCursor::nextMessage() {
 	ASSERT(hasMessage() || !version().sub);
 }
 
-StringRef ILogSystem::MergedPeekCursor::getMessage() {
+StringRef MergedPeekCursor::getMessage() {
 	return serverCursors[currentCursor]->getMessage();
 }
 
-StringRef ILogSystem::MergedPeekCursor::getMessageWithTags() {
+StringRef MergedPeekCursor::getMessageWithTags() {
 	return serverCursors[currentCursor]->getMessageWithTags();
 }
 
-VectorRef<Tag> ILogSystem::MergedPeekCursor::getTags() const {
+VectorRef<Tag> MergedPeekCursor::getTags() const {
 	return serverCursors[currentCursor]->getTags();
 }
 
-void ILogSystem::MergedPeekCursor::advanceTo(LogMessageVersion n) {
+void MergedPeekCursor::advanceTo(LogMessageVersion n) {
 	bool canChange = false;
 	for (auto& c : serverCursors) {
 		if (c->version() < n) {
@@ -846,15 +867,13 @@ void ILogSystem::MergedPeekCursor::advanceTo(LogMessageVersion n) {
 	}
 }
 
-ACTOR Future<Void> mergedPeekGetMore(ILogSystem::MergedPeekCursor* self,
-                                     LogMessageVersion startVersion,
-                                     TaskPriority taskID) {
-	loop {
+Future<Void> mergedPeekGetMore(MergedPeekCursor* self, LogMessageVersion startVersion, TaskPriority taskID) {
+	while (true) {
 		//TraceEvent("MPC_GetMoreA", self->randomID).detail("Start", startVersion.toString());
 		if (self->bestServer >= 0 && self->serverCursors[self->bestServer]->isActive()) {
 			ASSERT(!self->serverCursors[self->bestServer]->hasMessage());
-			wait(self->serverCursors[self->bestServer]->getMore(taskID) ||
-			     self->serverCursors[self->bestServer]->onFailed());
+			co_await (self->serverCursors[self->bestServer]->getMore(taskID) ||
+			          self->serverCursors[self->bestServer]->onFailed());
 		} else {
 			std::vector<Future<Void>> q;
 			for (auto& c : self->serverCursors) {
@@ -862,17 +881,17 @@ ACTOR Future<Void> mergedPeekGetMore(ILogSystem::MergedPeekCursor* self,
 					q.push_back(c->getMore(taskID));
 				}
 			}
-			wait(quorum(q, 1));
+			co_await quorum(q, 1);
 		}
 		self->calcHasMessage();
 		//TraceEvent("MPC_GetMoreB", self->randomID).detail("HasMessage", self->hasMessage()).detail("Start", startVersion.toString()).detail("Seq", self->version().toString());
 		if (self->hasMessage() || self->version() > startVersion) {
-			return Void();
+			co_return;
 		}
 	}
 }
 
-Future<Void> ILogSystem::MergedPeekCursor::getMore(TaskPriority taskID) {
+Future<Void> MergedPeekCursor::getMore(TaskPriority taskID) {
 	if (more.isValid() && !more.isReady()) {
 		return more;
 	}
@@ -898,43 +917,43 @@ Future<Void> ILogSystem::MergedPeekCursor::getMore(TaskPriority taskID) {
 	return more;
 }
 
-Future<Void> ILogSystem::MergedPeekCursor::onFailed() const {
+Future<Void> MergedPeekCursor::onFailed() const {
 	ASSERT(false);
 	return Never();
 }
 
-bool ILogSystem::MergedPeekCursor::isActive() const {
+bool MergedPeekCursor::isActive() const {
 	ASSERT(false);
 	return false;
 }
 
-bool ILogSystem::MergedPeekCursor::isExhausted() const {
+bool MergedPeekCursor::isExhausted() const {
 	return serverCursors[currentCursor]->isExhausted();
 }
 
-const LogMessageVersion& ILogSystem::MergedPeekCursor::version() const {
+const LogMessageVersion& MergedPeekCursor::version() const {
 	return messageVersion;
 }
 
-Version ILogSystem::MergedPeekCursor::getMinKnownCommittedVersion() const {
+Version MergedPeekCursor::getMinKnownCommittedVersion() const {
 	return serverCursors[currentCursor]->getMinKnownCommittedVersion();
 }
 
-Optional<UID> ILogSystem::MergedPeekCursor::getPrimaryPeekLocation() const {
+Optional<UID> MergedPeekCursor::getPrimaryPeekLocation() const {
 	if (bestServer >= 0) {
 		return serverCursors[bestServer]->getPrimaryPeekLocation();
 	}
 	return Optional<UID>();
 }
 
-Optional<UID> ILogSystem::MergedPeekCursor::getCurrentPeekLocation() const {
+Optional<UID> MergedPeekCursor::getCurrentPeekLocation() const {
 	if (currentCursor >= 0) {
 		return serverCursors[currentCursor]->getPrimaryPeekLocation();
 	}
 	return Optional<UID>();
 }
 
-Version ILogSystem::MergedPeekCursor::popped() const {
+Version MergedPeekCursor::popped() const {
 	Version poppedVersion = 0;
 	for (auto& c : serverCursors) {
 		poppedVersion = std::max(poppedVersion, c->popped());
@@ -942,14 +961,14 @@ Version ILogSystem::MergedPeekCursor::popped() const {
 	return poppedVersion;
 }
 
-ILogSystem::SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
-                                         int bestSet,
-                                         int bestServerLogId,
-                                         Tag tag,
-                                         Version begin,
-                                         Version end,
-                                         bool parallelGetMore,
-                                         const Optional<std::vector<uint16_t>>& knownLockedTLogIds)
+SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
+                             int bestSet,
+                             int bestServerLogId,
+                             Tag tag,
+                             Version begin,
+                             Version end,
+                             bool parallelGetMore,
+                             const Optional<std::vector<uint16_t>>& knownLockedTLogIds)
   : logSets(logSets), tag(tag), bestSet(bestSet), bestServer(bestServerLogId), currentSet(bestSet), currentCursor(0),
     messageVersion(begin), hasNextMessage(false), useBestSet(true), randomID(deterministicRandom()->randomUniqueID()),
     end(end) {
@@ -965,7 +984,7 @@ ILogSystem::SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& l
 			         ? canReturnEmptyVersionRange(
 			               bestServer, j /*currentServer*/, end, knownLockedTLogIds, bestSet, i /* currentSet */)
 			         : false);
-			auto cursor = makeReference<ILogSystem::ServerPeekCursor>(
+			auto cursor = makeReference<ServerPeekCursor>(
 			    logSets[i]->logServers[j], tag, begin, end, true, parallelGetMore, returnEmptyIfStopped);
 			serverCursors[i].push_back(cursor);
 		}
@@ -974,13 +993,13 @@ ILogSystem::SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& l
 	sortedVersions.resize(maxServers);
 }
 
-ILogSystem::SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
-                                         std::vector<std::vector<Reference<IPeekCursor>>> const& serverCursors,
-                                         LogMessageVersion const& messageVersion,
-                                         int bestSet,
-                                         int bestServer,
-                                         Optional<LogMessageVersion> nextVersion,
-                                         bool useBestSet)
+SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
+                             std::vector<std::vector<Reference<IPeekCursor>>> const& serverCursors,
+                             LogMessageVersion const& messageVersion,
+                             int bestSet,
+                             int bestServer,
+                             Optional<LogMessageVersion> nextVersion,
+                             bool useBestSet)
   : logSets(logSets), serverCursors(serverCursors), bestSet(bestSet), bestServer(bestServer), currentSet(bestSet),
     currentCursor(0), nextVersion(nextVersion), messageVersion(messageVersion), hasNextMessage(false),
     useBestSet(useBestSet), randomID(deterministicRandom()->randomUniqueID()) {
@@ -992,19 +1011,18 @@ ILogSystem::SetPeekCursor::SetPeekCursor(std::vector<Reference<LogSet>> const& l
 	calcHasMessage();
 }
 
-Reference<ILogSystem::IPeekCursor> ILogSystem::SetPeekCursor::cloneNoMore() {
-	std::vector<std::vector<Reference<ILogSystem::IPeekCursor>>> cursors;
+Reference<IPeekCursor> SetPeekCursor::cloneNoMore() {
+	std::vector<std::vector<Reference<IPeekCursor>>> cursors;
 	cursors.resize(logSets.size());
 	for (int i = 0; i < logSets.size(); i++) {
 		for (int j = 0; j < logSets[i]->logServers.size(); j++) {
 			cursors[i].push_back(serverCursors[i][j]->cloneNoMore());
 		}
 	}
-	return makeReference<ILogSystem::SetPeekCursor>(
-	    logSets, cursors, messageVersion, bestSet, bestServer, nextVersion, useBestSet);
+	return makeReference<SetPeekCursor>(logSets, cursors, messageVersion, bestSet, bestServer, nextVersion, useBestSet);
 }
 
-void ILogSystem::SetPeekCursor::setProtocolVersion(ProtocolVersion version) {
+void SetPeekCursor::setProtocolVersion(ProtocolVersion version) {
 	for (auto& cursors : serverCursors) {
 		for (auto& it : cursors) {
 			if (it->hasMessage()) {
@@ -1014,15 +1032,15 @@ void ILogSystem::SetPeekCursor::setProtocolVersion(ProtocolVersion version) {
 	}
 }
 
-Arena& ILogSystem::SetPeekCursor::arena() {
+Arena& SetPeekCursor::arena() {
 	return serverCursors[currentSet][currentCursor]->arena();
 }
 
-ArenaReader* ILogSystem::SetPeekCursor::reader() {
+ArenaReader* SetPeekCursor::reader() {
 	return serverCursors[currentSet][currentCursor]->reader();
 }
 
-void ILogSystem::SetPeekCursor::calcHasMessage() {
+void SetPeekCursor::calcHasMessage() {
 	if (bestSet >= 0 && bestServer >= 0) {
 		if (nextVersion.present()) {
 			//TraceEvent("LPC_CalcNext").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage).detail("NextVersion", nextVersion.get().toString());
@@ -1078,8 +1096,8 @@ void ILogSystem::SetPeekCursor::calcHasMessage() {
 	}
 }
 
-void ILogSystem::SetPeekCursor::updateMessage(int logIdx, bool usePolicy) {
-	loop {
+void SetPeekCursor::updateMessage(int logIdx, bool usePolicy) {
+	while (true) {
 		bool advancedPast = false;
 		sortedVersions.clear();
 		for (int i = 0; i < serverCursors[logIdx].size(); i++) {
@@ -1141,11 +1159,11 @@ void ILogSystem::SetPeekCursor::updateMessage(int logIdx, bool usePolicy) {
 	}
 }
 
-bool ILogSystem::SetPeekCursor::hasMessage() const {
+bool SetPeekCursor::hasMessage() const {
 	return hasNextMessage;
 }
 
-void ILogSystem::SetPeekCursor::nextMessage() {
+void SetPeekCursor::nextMessage() {
 	nextVersion = version();
 	nextVersion.get().sub++;
 	serverCursors[currentSet][currentCursor]->nextMessage();
@@ -1153,19 +1171,19 @@ void ILogSystem::SetPeekCursor::nextMessage() {
 	ASSERT(hasMessage() || !version().sub);
 }
 
-StringRef ILogSystem::SetPeekCursor::getMessage() {
+StringRef SetPeekCursor::getMessage() {
 	return serverCursors[currentSet][currentCursor]->getMessage();
 }
 
-StringRef ILogSystem::SetPeekCursor::getMessageWithTags() {
+StringRef SetPeekCursor::getMessageWithTags() {
 	return serverCursors[currentSet][currentCursor]->getMessageWithTags();
 }
 
-VectorRef<Tag> ILogSystem::SetPeekCursor::getTags() const {
+VectorRef<Tag> SetPeekCursor::getTags() const {
 	return serverCursors[currentSet][currentCursor]->getTags();
 }
 
-void ILogSystem::SetPeekCursor::advanceTo(LogMessageVersion n) {
+void SetPeekCursor::advanceTo(LogMessageVersion n) {
 	bool canChange = false;
 	for (auto& cursors : serverCursors) {
 		for (auto& c : cursors) {
@@ -1180,17 +1198,15 @@ void ILogSystem::SetPeekCursor::advanceTo(LogMessageVersion n) {
 	}
 }
 
-ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self,
-                                  LogMessageVersion startVersion,
-                                  TaskPriority taskID) {
-	loop {
+Future<Void> setPeekGetMore(SetPeekCursor* self, LogMessageVersion startVersion, TaskPriority taskID) {
+	while (true) {
 		//TraceEvent("LPC_GetMore1", self->randomID).detail("Start", startVersion.toString()).detail("Tag", self->tag.toString());
 		if (self->bestServer >= 0 && self->bestSet >= 0 &&
 		    self->serverCursors[self->bestSet][self->bestServer]->isActive()) {
 			ASSERT(!self->serverCursors[self->bestSet][self->bestServer]->hasMessage());
 			//TraceEvent("LPC_GetMore2", self->randomID).detail("Start", startVersion.toString()).detail("Tag", self->tag.toString());
-			wait(self->serverCursors[self->bestSet][self->bestServer]->getMore(taskID) ||
-			     self->serverCursors[self->bestSet][self->bestServer]->onFailed());
+			co_await (self->serverCursors[self->bestSet][self->bestServer]->getMore(taskID) ||
+			          self->serverCursors[self->bestSet][self->bestServer]->onFailed());
 			self->useBestSet = true;
 		} else {
 			// FIXME: if best set is exhausted, do not peek remote servers
@@ -1211,7 +1227,7 @@ ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self,
 					self->useBestSet = true;
 					self->calcHasMessage();
 					if (self->hasMessage() || self->version() > startVersion) {
-						return Void();
+						co_return;
 					}
 				}
 
@@ -1225,7 +1241,7 @@ ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self,
 						}
 					}
 				}
-				wait(quorum(q, 1));
+				co_await quorum(q, 1);
 			} else {
 				// FIXME: this will peeking way too many cursors when satellites exist, and does not need to peek
 				// bestSet cursors since we cannot get anymore data from them
@@ -1238,19 +1254,19 @@ ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self,
 						}
 					}
 				}
-				wait(quorum(q, 1));
+				co_await quorum(q, 1);
 				self->useBestSet = false;
 			}
 		}
 		self->calcHasMessage();
 		//TraceEvent("LPC_GetMoreB", self->randomID).detail("HasMessage", self->hasMessage()).detail("Start", startVersion.toString()).detail("Seq", self->version().toString());
 		if (self->hasMessage() || self->version() > startVersion) {
-			return Void();
+			co_return;
 		}
 	}
 }
 
-Future<Void> ILogSystem::SetPeekCursor::getMore(TaskPriority taskID) {
+Future<Void> SetPeekCursor::getMore(TaskPriority taskID) {
 	if (more.isValid() && !more.isReady()) {
 		return more;
 	}
@@ -1272,43 +1288,43 @@ Future<Void> ILogSystem::SetPeekCursor::getMore(TaskPriority taskID) {
 	return more;
 }
 
-Future<Void> ILogSystem::SetPeekCursor::onFailed() const {
+Future<Void> SetPeekCursor::onFailed() const {
 	ASSERT(false);
 	return Never();
 }
 
-bool ILogSystem::SetPeekCursor::isActive() const {
+bool SetPeekCursor::isActive() const {
 	ASSERT(false);
 	return false;
 }
 
-bool ILogSystem::SetPeekCursor::isExhausted() const {
+bool SetPeekCursor::isExhausted() const {
 	return serverCursors[currentSet][currentCursor]->isExhausted();
 }
 
-const LogMessageVersion& ILogSystem::SetPeekCursor::version() const {
+const LogMessageVersion& SetPeekCursor::version() const {
 	return messageVersion;
 }
 
-Version ILogSystem::SetPeekCursor::getMinKnownCommittedVersion() const {
+Version SetPeekCursor::getMinKnownCommittedVersion() const {
 	return serverCursors[currentSet][currentCursor]->getMinKnownCommittedVersion();
 }
 
-Optional<UID> ILogSystem::SetPeekCursor::getPrimaryPeekLocation() const {
+Optional<UID> SetPeekCursor::getPrimaryPeekLocation() const {
 	if (bestServer >= 0 && bestSet >= 0) {
 		return serverCursors[bestSet][bestServer]->getPrimaryPeekLocation();
 	}
 	return Optional<UID>();
 }
 
-Optional<UID> ILogSystem::SetPeekCursor::getCurrentPeekLocation() const {
+Optional<UID> SetPeekCursor::getCurrentPeekLocation() const {
 	if (currentCursor >= 0 && currentSet >= 0) {
 		return serverCursors[currentSet][currentCursor]->getPrimaryPeekLocation();
 	}
 	return Optional<UID>();
 }
 
-Version ILogSystem::SetPeekCursor::popped() const {
+Version SetPeekCursor::popped() const {
 	Version poppedVersion = 0;
 	for (auto& cursors : serverCursors) {
 		for (auto& c : cursors) {
@@ -1318,51 +1334,50 @@ Version ILogSystem::SetPeekCursor::popped() const {
 	return poppedVersion;
 }
 
-ILogSystem::MultiCursor::MultiCursor(std::vector<Reference<IPeekCursor>> cursors,
-                                     std::vector<LogMessageVersion> epochEnds)
+MultiCursor::MultiCursor(std::vector<Reference<IPeekCursor>> cursors, std::vector<LogMessageVersion> epochEnds)
   : cursors(cursors), epochEnds(epochEnds), poppedVersion(0) {
 	for (int i = 0; i < std::min<int>(cursors.size(), SERVER_KNOBS->MULTI_CURSOR_PRE_FETCH_LIMIT); i++) {
 		cursors[cursors.size() - i - 1]->getMore();
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> ILogSystem::MultiCursor::cloneNoMore() {
+Reference<IPeekCursor> MultiCursor::cloneNoMore() {
 	return cursors.back()->cloneNoMore();
 }
 
-void ILogSystem::MultiCursor::setProtocolVersion(ProtocolVersion version) {
+void MultiCursor::setProtocolVersion(ProtocolVersion version) {
 	cursors.back()->setProtocolVersion(version);
 }
 
-Arena& ILogSystem::MultiCursor::arena() {
+Arena& MultiCursor::arena() {
 	return cursors.back()->arena();
 }
 
-ArenaReader* ILogSystem::MultiCursor::reader() {
+ArenaReader* MultiCursor::reader() {
 	return cursors.back()->reader();
 }
 
-bool ILogSystem::MultiCursor::hasMessage() const {
+bool MultiCursor::hasMessage() const {
 	return cursors.back()->hasMessage();
 }
 
-void ILogSystem::MultiCursor::nextMessage() {
+void MultiCursor::nextMessage() {
 	cursors.back()->nextMessage();
 }
 
-StringRef ILogSystem::MultiCursor::getMessage() {
+StringRef MultiCursor::getMessage() {
 	return cursors.back()->getMessage();
 }
 
-StringRef ILogSystem::MultiCursor::getMessageWithTags() {
+StringRef MultiCursor::getMessageWithTags() {
 	return cursors.back()->getMessageWithTags();
 }
 
-VectorRef<Tag> ILogSystem::MultiCursor::getTags() const {
+VectorRef<Tag> MultiCursor::getTags() const {
 	return cursors.back()->getTags();
 }
 
-void ILogSystem::MultiCursor::advanceTo(LogMessageVersion n) {
+void MultiCursor::advanceTo(LogMessageVersion n) {
 	while (cursors.size() > 1 && n >= epochEnds.back()) {
 		poppedVersion = std::max(poppedVersion, cursors.back()->popped());
 		cursors.pop_back();
@@ -1371,7 +1386,7 @@ void ILogSystem::MultiCursor::advanceTo(LogMessageVersion n) {
 	cursors.back()->advanceTo(n);
 }
 
-Future<Void> ILogSystem::MultiCursor::getMore(TaskPriority taskID) {
+Future<Void> MultiCursor::getMore(TaskPriority taskID) {
 	LogMessageVersion startVersion = cursors.back()->version();
 	while (cursors.size() > 1 && cursors.back()->version() >= epochEnds.back()) {
 		poppedVersion = std::max(poppedVersion, cursors.back()->popped());
@@ -1384,43 +1399,43 @@ Future<Void> ILogSystem::MultiCursor::getMore(TaskPriority taskID) {
 	return cursors.back()->getMore(taskID);
 }
 
-Future<Void> ILogSystem::MultiCursor::onFailed() const {
+Future<Void> MultiCursor::onFailed() const {
 	return cursors.back()->onFailed();
 }
 
-bool ILogSystem::MultiCursor::isActive() const {
+bool MultiCursor::isActive() const {
 	return cursors.back()->isActive();
 }
 
-bool ILogSystem::MultiCursor::isExhausted() const {
+bool MultiCursor::isExhausted() const {
 	return cursors.back()->isExhausted();
 }
 
-const LogMessageVersion& ILogSystem::MultiCursor::version() const {
+const LogMessageVersion& MultiCursor::version() const {
 	return cursors.back()->version();
 }
 
-Version ILogSystem::MultiCursor::getMinKnownCommittedVersion() const {
+Version MultiCursor::getMinKnownCommittedVersion() const {
 	return cursors.back()->getMinKnownCommittedVersion();
 }
 
-Optional<UID> ILogSystem::MultiCursor::getPrimaryPeekLocation() const {
+Optional<UID> MultiCursor::getPrimaryPeekLocation() const {
 	return cursors.back()->getPrimaryPeekLocation();
 }
 
-Optional<UID> ILogSystem::MultiCursor::getCurrentPeekLocation() const {
+Optional<UID> MultiCursor::getCurrentPeekLocation() const {
 	return cursors.back()->getCurrentPeekLocation();
 }
 
-Version ILogSystem::MultiCursor::popped() const {
+Version MultiCursor::popped() const {
 	return std::max(poppedVersion, cursors.back()->popped());
 }
 
-ILogSystem::BufferedCursor::BufferedCursor(std::vector<Reference<IPeekCursor>> cursors,
-                                           Version begin,
-                                           Version end,
-                                           bool withTags,
-                                           bool canDiscardPopped)
+BufferedCursor::BufferedCursor(std::vector<Reference<IPeekCursor>> cursors,
+                               Version begin,
+                               Version end,
+                               bool withTags,
+                               bool canDiscardPopped)
   : cursors(cursors), messageIndex(0), messageVersion(begin), end(end), hasNextMessage(false), withTags(withTags),
     knownUnique(false), minKnownCommittedVersion(0), poppedVersion(0), initialPoppedVersion(0),
     canDiscardPopped(canDiscardPopped), randomID(deterministicRandom()->randomUniqueID()) {
@@ -1429,12 +1444,11 @@ ILogSystem::BufferedCursor::BufferedCursor(std::vector<Reference<IPeekCursor>> c
 	cursorMessages.resize(cursors.size());
 }
 
-ILogSystem::BufferedCursor::BufferedCursor(
-    std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
-    Tag tag,
-    Version begin,
-    Version end,
-    bool parallelGetMore)
+BufferedCursor::BufferedCursor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
+                               Tag tag,
+                               Version begin,
+                               Version end,
+                               bool parallelGetMore)
   : messageIndex(0), messageVersion(begin), end(end), hasNextMessage(false), withTags(true), knownUnique(true),
     minKnownCommittedVersion(0), poppedVersion(0), initialPoppedVersion(0), canDiscardPopped(false),
     randomID(deterministicRandom()->randomUniqueID()) {
@@ -1442,72 +1456,68 @@ ILogSystem::BufferedCursor::BufferedCursor(
 	messages.reserve(SERVER_KNOBS->DESIRED_OUTSTANDING_MESSAGES);
 	cursorMessages.resize(logServers.size());
 	for (int i = 0; i < logServers.size(); i++) {
-		auto cursor =
-		    makeReference<ILogSystem::ServerPeekCursor>(logServers[i], tag, begin, end, false, parallelGetMore);
+		auto cursor = makeReference<ServerPeekCursor>(logServers[i], tag, begin, end, false, parallelGetMore);
 		cursors.push_back(cursor);
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> ILogSystem::BufferedCursor::cloneNoMore() {
+Reference<IPeekCursor> BufferedCursor::cloneNoMore() {
 	ASSERT(false);
-	return Reference<ILogSystem::IPeekCursor>();
+	return Reference<IPeekCursor>();
 }
 
-void ILogSystem::BufferedCursor::setProtocolVersion(ProtocolVersion version) {
+void BufferedCursor::setProtocolVersion(ProtocolVersion version) {
 	for (auto& c : cursors) {
 		c->setProtocolVersion(version);
 	}
 }
 
-Arena& ILogSystem::BufferedCursor::arena() {
+Arena& BufferedCursor::arena() {
 	return messages[messageIndex].arena;
 }
 
-ArenaReader* ILogSystem::BufferedCursor::reader() {
+ArenaReader* BufferedCursor::reader() {
 	ASSERT(false);
 	return cursors[0]->reader();
 }
 
-bool ILogSystem::BufferedCursor::hasMessage() const {
+bool BufferedCursor::hasMessage() const {
 	return hasNextMessage;
 }
 
-void ILogSystem::BufferedCursor::nextMessage() {
+void BufferedCursor::nextMessage() {
 	messageIndex++;
 	if (messageIndex == messages.size()) {
 		hasNextMessage = false;
 	}
 }
 
-StringRef ILogSystem::BufferedCursor::getMessage() {
+StringRef BufferedCursor::getMessage() {
 	ASSERT(!withTags);
 	return messages[messageIndex].message;
 }
 
-StringRef ILogSystem::BufferedCursor::getMessageWithTags() {
+StringRef BufferedCursor::getMessageWithTags() {
 	ASSERT(withTags);
 	return messages[messageIndex].message;
 }
 
-VectorRef<Tag> ILogSystem::BufferedCursor::getTags() const {
+VectorRef<Tag> BufferedCursor::getTags() const {
 	ASSERT(withTags);
 	return messages[messageIndex].tags;
 }
 
-void ILogSystem::BufferedCursor::advanceTo(LogMessageVersion n) {
+void BufferedCursor::advanceTo(LogMessageVersion n) {
 	ASSERT(false);
 }
 
-ACTOR Future<Void> bufferedGetMoreLoader(ILogSystem::BufferedCursor* self,
-                                         Reference<ILogSystem::IPeekCursor> cursor,
-                                         int idx,
-                                         TaskPriority taskID) {
-	loop {
-		wait(yield());
+Future<Void> bufferedGetMoreLoader(BufferedCursor* self, Reference<IPeekCursor> cursor, int idx, TaskPriority taskID) {
+	while (true) {
+		co_await yield();
 		if (cursor->version().version >= self->end || self->cursorMessages[idx].size() > self->targetQueueSize) {
-			return Void();
+			co_return;
 		}
-		wait(cursor->getMore(taskID));
+		co_await cursor->getMore(taskID);
 		self->poppedVersion = std::max(self->poppedVersion, cursor->popped());
 		self->minKnownCommittedVersion =
 		    std::max(self->minKnownCommittedVersion, cursor->getMinKnownCommittedVersion());
@@ -1515,22 +1525,22 @@ ACTOR Future<Void> bufferedGetMoreLoader(ILogSystem::BufferedCursor* self,
 			self->initialPoppedVersion = std::max(self->initialPoppedVersion, cursor->popped());
 		}
 		if (cursor->version().version >= self->end) {
-			return Void();
+			co_return;
 		}
 		while (cursor->hasMessage()) {
-			self->cursorMessages[idx].push_back(ILogSystem::BufferedCursor::BufferedMessage(
-			    cursor->arena(),
-			    !self->withTags ? cursor->getMessage() : cursor->getMessageWithTags(),
-			    !self->withTags ? VectorRef<Tag>() : cursor->getTags(),
-			    cursor->version()));
+			self->cursorMessages[idx].push_back(
+			    BufferedCursor::BufferedMessage(cursor->arena(),
+			                                    !self->withTags ? cursor->getMessage() : cursor->getMessageWithTags(),
+			                                    !self->withTags ? VectorRef<Tag>() : cursor->getTags(),
+			                                    cursor->version()));
 			cursor->nextMessage();
 		}
 	}
 }
 
-ACTOR Future<Void> bufferedGetMore(ILogSystem::BufferedCursor* self, TaskPriority taskID) {
+Future<Void> bufferedGetMore(BufferedCursor* self, TaskPriority taskID) {
 	if (self->messageVersion.version >= self->end) {
-		wait(Future<Void>(Never()));
+		co_await Future<Void>(Never());
 		throw internal_error();
 	}
 
@@ -1543,15 +1553,15 @@ ACTOR Future<Void> bufferedGetMore(ILogSystem::BufferedCursor* self, TaskPriorit
 		loaders.push_back(bufferedGetMoreLoader(self, self->cursors[i], i, taskID));
 	}
 
-	state Future<Void> allLoaders = waitForAll(loaders);
-	state Version minVersion;
-	loop {
-		wait(allLoaders || delay(SERVER_KNOBS->DESIRED_GET_MORE_DELAY, taskID));
+	Future<Void> allLoaders = waitForAll(loaders);
+	Version minVersion{ 0 };
+	while (true) {
+		co_await (allLoaders || delay(SERVER_KNOBS->DESIRED_GET_MORE_DELAY, taskID));
 		minVersion = self->end;
 		for (int i = 0; i < self->cursors.size(); i++) {
 			auto cursor = self->cursors[i];
 			while (cursor->hasMessage()) {
-				self->cursorMessages[i].push_back(ILogSystem::BufferedCursor::BufferedMessage(
+				self->cursorMessages[i].push_back(BufferedCursor::BufferedMessage(
 				    cursor->arena(),
 				    !self->withTags ? cursor->getMessage() : cursor->getMessageWithTags(),
 				    !self->withTags ? VectorRef<Tag>() : cursor->getTags(),
@@ -1564,10 +1574,10 @@ ACTOR Future<Void> bufferedGetMore(ILogSystem::BufferedCursor* self, TaskPriorit
 			break;
 		}
 		if (allLoaders.isReady()) {
-			wait(Future<Void>(Never()));
+			co_await Future<Void>(Never());
 		}
 	}
-	wait(yield());
+	co_await yield();
 
 	for (auto& it : self->cursorMessages) {
 		while (!it.empty() && it.front().version.version < minVersion) {
@@ -1585,7 +1595,7 @@ ACTOR Future<Void> bufferedGetMore(ILogSystem::BufferedCursor* self, TaskPriorit
 	self->messageIndex = 0;
 	self->hasNextMessage = self->messages.size() > 0;
 
-	wait(yield());
+	co_await yield();
 	if (self->canDiscardPopped && self->poppedVersion > self->version().version) {
 		TraceEvent(SevWarn, "DiscardingPoppedData", self->randomID)
 		    .detail("Version", self->version().version)
@@ -1599,9 +1609,8 @@ ACTOR Future<Void> bufferedGetMore(ILogSystem::BufferedCursor* self, TaskPriorit
 		    self->messages[self->messages.size() - 1].version.version < self->poppedVersion) {
 			self->hasNextMessage = false;
 		} else {
-			auto iter = std::lower_bound(self->messages.begin(),
-			                             self->messages.end(),
-			                             ILogSystem::BufferedCursor::BufferedMessage(self->poppedVersion));
+			auto iter = std::lower_bound(
+			    self->messages.begin(), self->messages.end(), BufferedCursor::BufferedMessage(self->poppedVersion));
 			self->hasNextMessage = iter != self->messages.end();
 			if (self->hasNextMessage) {
 				self->messageIndex = iter - self->messages.begin();
@@ -1611,10 +1620,9 @@ ACTOR Future<Void> bufferedGetMore(ILogSystem::BufferedCursor* self, TaskPriorit
 	if (self->hasNextMessage) {
 		self->canDiscardPopped = false;
 	}
-	return Void();
 }
 
-Future<Void> ILogSystem::BufferedCursor::getMore(TaskPriority taskID) {
+Future<Void> BufferedCursor::getMore(TaskPriority taskID) {
 	if (hasMessage()) {
 		return Void();
 	}
@@ -1625,41 +1633,41 @@ Future<Void> ILogSystem::BufferedCursor::getMore(TaskPriority taskID) {
 	return more;
 }
 
-Future<Void> ILogSystem::BufferedCursor::onFailed() const {
+Future<Void> BufferedCursor::onFailed() const {
 	ASSERT(false);
 	return Never();
 }
 
-bool ILogSystem::BufferedCursor::isActive() const {
+bool BufferedCursor::isActive() const {
 	ASSERT(false);
 	return false;
 }
 
-bool ILogSystem::BufferedCursor::isExhausted() const {
+bool BufferedCursor::isExhausted() const {
 	ASSERT(false);
 	return false;
 }
 
-const LogMessageVersion& ILogSystem::BufferedCursor::version() const {
+const LogMessageVersion& BufferedCursor::version() const {
 	if (hasNextMessage) {
 		return messages[messageIndex].version;
 	}
 	return messageVersion;
 }
 
-Version ILogSystem::BufferedCursor::getMinKnownCommittedVersion() const {
+Version BufferedCursor::getMinKnownCommittedVersion() const {
 	return minKnownCommittedVersion;
 }
 
-Optional<UID> ILogSystem::BufferedCursor::getPrimaryPeekLocation() const {
+Optional<UID> BufferedCursor::getPrimaryPeekLocation() const {
 	return Optional<UID>();
 }
 
-Optional<UID> ILogSystem::BufferedCursor::getCurrentPeekLocation() const {
+Optional<UID> BufferedCursor::getCurrentPeekLocation() const {
 	return Optional<UID>();
 }
 
-Version ILogSystem::BufferedCursor::popped() const {
+Version BufferedCursor::popped() const {
 	if (initialPoppedVersion == poppedVersion) {
 		return 0;
 	}
diff --git a/fdbserver/TagPartitionedLogSystemRecoveryTests.actor.cpp b/fdbserver/logsystem/LogSystemRecoveryTests.cpp
similarity index 94%
rename from fdbserver/TagPartitionedLogSystemRecoveryTests.actor.cpp
rename to fdbserver/logsystem/LogSystemRecoveryTests.cpp
index ee2bc4f3020..4f9b5fb19ff 100644
--- a/fdbserver/TagPartitionedLogSystemRecoveryTests.actor.cpp
+++ b/fdbserver/logsystem/LogSystemRecoveryTests.cpp
@@ -1,5 +1,5 @@
 /*
- * TagPartitionedLogSystemRecoveryTests.actor.cpp
+ * LogSystemRecoveryTests.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TagPartitionedLogSystem.actor.h"
+#include "fdbserver/logsystem/LogSystem.h"
 #include "flow/UnitTest.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
 
 Reference<LogSet> makeSingleLogSet(const std::vector<TLogInterface>& tlogs, bool isLocal = true) {
-	Reference<LogSet> logSet = makeReference<LogSet>();
+	auto logSet = makeReference<LogSet>();
 	logSet->isLocal = isLocal;
 	for (const auto& tlog : tlogs) {
 		logSet->logServers.push_back(
@@ -57,7 +55,9 @@ std::tuple<int, std::vector<TLogLockResult>, bool> makeLogGroupResults(
 
 } // namespace
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/Simple") {
+void forceLinkLogSystemRecoveryTests() {}
+
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/Simple") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -91,7 +91,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/Simple") {
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/FallbackToMaxKCV") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/FallbackToMaxKCV") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -122,7 +122,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/FallbackToMaxKCV")
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/HaltOnMissingDelivery") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/HaltOnMissingDelivery") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -160,7 +160,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/HaltOnMissingDelive
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/PolicyNotSatisfied") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/PolicyNotSatisfied") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -198,7 +198,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/PolicyNotSatisfied"
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/MinDVRespected") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/MinDVRespected") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -234,7 +234,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/MinDVRespected") {
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/BrokenChain") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/BrokenChain") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -273,7 +273,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/BrokenChain") {
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/MultipleLogSets") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/MultipleLogSets") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -312,7 +312,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/MultipleLogSets") {
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/PartialAvailabilityPolicyFail") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/PartialAvailabilityPolicyFail") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -351,7 +351,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/PartialAvailability
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/VersionsBelowMaxKCV") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/VersionsBelowMaxKCV") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
@@ -388,7 +388,7 @@ TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/VersionsBelowMaxKCV
 	return Void();
 }
 
-TEST_CASE("/TagPartitionedLogSystem/GetRecoverVersionUnicast/RandomVersionsPartialDelivery") {
+TEST_CASE("/LogSystem/GetRecoverVersionUnicast/RandomVersionsPartialDelivery") {
 	if (!SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) {
 		return Void();
 	}
diff --git a/fdbserver/include/fdbserver/ApplyMetadataMutation.h b/fdbserver/logsystem/include/fdbserver/logsystem/ApplyMetadataMutation.h
similarity index 69%
rename from fdbserver/include/fdbserver/ApplyMetadataMutation.h
rename to fdbserver/logsystem/include/fdbserver/logsystem/ApplyMetadataMutation.h
index 0f31b849275..a56b4e76521 100644
--- a/fdbserver/include/fdbserver/ApplyMetadataMutation.h
+++ b/fdbserver/logsystem/include/fdbserver/logsystem/ApplyMetadataMutation.h
@@ -23,18 +23,57 @@
 #pragma once
 
 #include <cstddef>
+#include <map>
+#include <memory>
+#include <set>
+#include <unordered_map>
 
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/MutationList.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/LogProtocolMessage.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/ProxyCommitData.actor.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/LogProtocolMessage.h"
+#include "fdbserver/logsystem/LogSystem.h"
 #include "flow/FastRef.h"
 
+class AccumulativeChecksumBuilder;
+
+// applyMetadataMutations() is shared with resolver/clustercontroller code, so
+// it depends on a narrow commit-proxy view instead of the full ProxyCommitData
+// definition from the commitproxy module.
+struct ApplyMetadataRangeLock {
+	virtual ~ApplyMetadataRangeLock() = default;
+	virtual bool pendingRequest() const = 0;
+	virtual void setPendingRequest(const Key& startKey, const RangeLockStateSet& lockSetState) = 0;
+	virtual void consumePendingRequest(const Key& endKey) = 0;
+};
+
+struct ApplyMutationsData {
+	Future<Void> worker;
+	Version endVersion;
+	Reference<KeyRangeMap<Version>> keyVersion;
+};
+
+struct ApplyMetadataProxyContext {
+	UID dbgid;
+	IKeyValueStore* txnStateStore = nullptr;
+	KeyRangeMap<std::set<Key>>* vecBackupKeys = nullptr;
+	KeyRangeMap<ServerCacheInfo>* keyInfo = nullptr;
+	std::map<Key, ApplyMutationsData>* uid_applyMutationsData = nullptr;
+	PublicRequestStream<CommitTransactionRequest> commit;
+	Database cx;
+	NotifiedVersion* committedVersion = nullptr;
+	std::map<UID, Reference<StorageInfo>>* storageCache = nullptr;
+	std::map<Tag, Version>* tag_popped = nullptr;
+	std::unordered_map<UID, StorageServerInterface>* tssMapping = nullptr;
+	uint16_t commitProxyIndex = 0;
+	std::shared_ptr<AccumulativeChecksumBuilder> acsBuilder = nullptr;
+	Optional<LogEpoch> epoch;
+	ApplyMetadataRangeLock* rangeLock = nullptr;
+};
+
 // Resolver's data for applyMetadataMutations() calls.
 struct ResolverData {
 	const UID dbgid;
@@ -44,7 +83,7 @@ struct ResolverData {
 	// Whether configuration changes. If so, a recovery is forced.
 	bool& confChanges;
 	bool initialCommit = false;
-	Reference<ILogSystem> logSystem = Reference<ILogSystem>();
+	Reference<LogSystem> logSystem = Reference<LogSystem>();
 	LogPushData* toCommit = nullptr;
 	Version popVersion = 0; // exclusive, usually set to commitVersion + 1
 	std::map<UID, Reference<StorageInfo>>* storageCache = nullptr;
@@ -56,7 +95,7 @@ struct ResolverData {
 
 	// For transaction batches that contain metadata mutations
 	ResolverData(UID debugId,
-	             Reference<ILogSystem> logSystem,
+	             Reference<LogSystem> logSystem,
 	             IKeyValueStore* store,
 	             KeyRangeMap<ServerCacheInfo>* info,
 	             LogPushData* toCommit,
@@ -87,9 +126,9 @@ Reference<StorageInfo> getStorageInfo(UID id,
                                       IKeyValueStore* txnStateStore);
 
 void applyMetadataMutations(SpanContext const& spanContext,
-                            ProxyCommitData& proxyCommitData,
+                            const ApplyMetadataProxyContext& proxyMetadata,
                             Arena& arena,
-                            Reference<ILogSystem> logSystem,
+                            Reference<LogSystem> logSystem,
                             const VectorRef<MutationRef>& mutations,
                             LogPushData* pToCommit,
                             bool& confChange,
diff --git a/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h
new file mode 100644
index 00000000000..92da42893a5
--- /dev/null
+++ b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h
@@ -0,0 +1,692 @@
+/*
+ * LogSystem.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_LOGSYSTEM_LOGSYSTEM_H
+#define FDBSERVER_LOGSYSTEM_LOGSYSTEM_H
+
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <set>
+#include <unordered_set>
+#include <vector>
+
+#include "fdbclient/DatabaseConfiguration.h"
+#include "fdbrpc/Replication.h"
+#include "fdbrpc/Locality.h"
+#include "fdbrpc/ReplicationPolicy.h"
+#include "fdbserver/core/BackupProgressTypes.h"
+#include "fdbserver/core/DBCoreState.h"
+#include "fdbserver/core/LogSystemConfig.h"
+#include "fdbserver/core/MutationTracking.h"
+#include "fdbserver/core/OTELSpanContextMessage.h"
+#include "fdbserver/core/SpanContextMessage.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "flow/Arena.h"
+#include "flow/Error.h"
+#include "flow/ActorCollection.h"
+#include "flow/Histogram.h"
+#include "flow/IndexedSet.h"
+#include "flow/Knobs.h"
+
+struct DBCoreState;
+struct LogPushData;
+struct LocalityData;
+struct LogSystem;
+class LogSet;
+
+struct ConnectionResetInfo : public ReferenceCounted<ConnectionResetInfo> {
+	double lastReset;
+	Future<Void> resetCheck;
+	int slowReplies;
+	int fastReplies;
+
+	ConnectionResetInfo() : lastReset(now()), resetCheck(Void()), slowReplies(0), fastReplies(0) {}
+};
+
+struct IPeekCursor {
+	virtual Reference<IPeekCursor> cloneNoMore() = 0;
+
+	virtual void setProtocolVersion(ProtocolVersion version) = 0;
+
+	virtual bool hasMessage() const = 0;
+	virtual VectorRef<Tag> getTags() const = 0;
+	virtual Arena& arena() = 0;
+	virtual ArenaReader* reader() = 0;
+	virtual StringRef getMessage() = 0;
+	virtual StringRef getMessageWithTags() = 0;
+	virtual void nextMessage() = 0;
+	virtual void advanceTo(LogMessageVersion n) = 0;
+	virtual Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) = 0;
+	virtual Future<Void> onFailed() const = 0;
+	virtual bool isActive() const = 0;
+	virtual bool isExhausted() const = 0;
+	virtual const LogMessageVersion& version() const = 0;
+	virtual Version popped() const = 0;
+	virtual Version getMaxKnownVersion() const { return 0; }
+	virtual Version getMinKnownCommittedVersion() const = 0;
+	virtual Optional<UID> getPrimaryPeekLocation() const = 0;
+	virtual Optional<UID> getCurrentPeekLocation() const = 0;
+	virtual void addref() = 0;
+	virtual void delref() = 0;
+};
+
+struct LogPushVersionSet {
+	Version prevVersion;
+	Version version;
+	Version knownCommittedVersion;
+	Version minKnownCommittedVersion;
+};
+
+bool logSystemHasRemoteLogs(LogSystem const& logSystem);
+void logSystemGetPushLocations(
+    LogSystem const& logSystem,
+    VectorRef<Tag> tags,
+    std::vector<int>& locations,
+    bool allLocations = false,
+    Optional<std::vector<Reference<LocalitySet>>> fromLocations = Optional<std::vector<Reference<LocalitySet>>>());
+std::vector<Reference<LocalitySet>> logSystemGetPushLocationsForTags(LogSystem const& logSystem,
+                                                                     std::vector<int>& fromLocations);
+Tag logSystemGetRandomRouterTag(LogSystem const& logSystem);
+int logSystemGetLogRouterTags(LogSystem const& logSystem);
+Tag logSystemGetRandomTxsTag(LogSystem const& logSystem);
+TLogVersion logSystemGetTLogVersion(LogSystem const& logSystem);
+
+struct LengthPrefixedStringRef {
+	uint32_t* length;
+
+	StringRef toStringRef() const {
+		ASSERT(length);
+		return StringRef((uint8_t*)(length + 1), *length);
+	}
+	int expectedSize() const {
+		ASSERT(length);
+		return *length;
+	}
+	uint32_t* getLengthPtr() const { return length; }
+
+	LengthPrefixedStringRef() : length(nullptr) {}
+	explicit LengthPrefixedStringRef(uint32_t* length) : length(length) {}
+};
+
+struct LogPushData : NonCopyable {
+	explicit LogPushData(Reference<LogSystem> logSystem, int tlogCount);
+
+	void addTxsTag();
+
+	void addTag(Tag tag) { next_message_tags.push_back(tag); }
+
+	template <class T>
+	void addTags(T tags) {
+		next_message_tags.insert(next_message_tags.end(), tags.begin(), tags.end());
+	}
+
+	void addTransactionInfo(SpanContext const& context);
+
+	void saveTags(std::set<Tag>& filteredTags) const {
+		for (const auto& tag : written_tags) {
+			filteredTags.insert(tag);
+		}
+	}
+
+	void addWrittenTags(const std::set<Tag>& tags) { written_tags.insert(tags.begin(), tags.end()); }
+
+	void getLocations(const std::set<Tag>& tags, std::set<uint16_t>& writtenTLogs) {
+		std::vector<Tag> vtags(tags.begin(), tags.end());
+		std::vector<int> locations;
+		logSystemGetPushLocations(*logSystem, VectorRef<Tag>((Tag*)vtags.data(), vtags.size()), locations);
+		writtenTLogs.insert(locations.begin(), locations.end());
+	}
+
+	void saveLocations(std::set<uint16_t>& writtenTLogs) {
+		writtenTLogs.insert(msg_locations.begin(), msg_locations.end());
+	}
+
+	void setLogsChanged() { logsChanged = true; }
+	bool haveLogsChanged() const { return logsChanged; }
+
+	void writeMessage(StringRef rawMessageWithoutLength, bool usePreviousLocations);
+
+	void setPushLocationsForTags(std::vector<int> fromLocationsVec) {
+		fromLocations = logSystemGetPushLocationsForTags(*logSystem, fromLocationsVec);
+	}
+
+	template <class T>
+	void writeTypedMessage(T const& item, bool metadataMessage = false, bool allLocations = false);
+
+	Standalone<StringRef> getMessages(int loc) const { return messagesWriter[loc].toValue(); }
+
+	std::vector<Standalone<StringRef>> getAllMessages() const;
+
+	void recordEmptyMessage(int loc, const Standalone<StringRef>& value);
+
+	float getEmptyMessageRatio() const;
+
+	uint32_t getMutationCount() const { return subsequence - 1; }
+
+	void setMutations(uint32_t totalMutations, VectorRef<StringRef> mutations);
+
+	Optional<Tag> savedRandomRouterTag;
+	void storeRandomRouterTag() { savedRandomRouterTag = logSystemGetRandomRouterTag(*logSystem); }
+	int getLogRouterTags() { return logSystemGetLogRouterTags(*logSystem); }
+
+private:
+	Reference<LogSystem> logSystem;
+	std::vector<Tag> next_message_tags;
+	std::vector<Tag> prev_tags;
+	std::set<Tag> written_tags;
+	std::vector<BinaryWriter> messagesWriter;
+	std::vector<bool> messagesWritten;
+	std::vector<int> msg_locations;
+	Optional<std::vector<Reference<LocalitySet>>> fromLocations;
+	std::unordered_set<int> writtenLocations;
+	uint32_t subsequence;
+	SpanContext spanContext;
+	bool logsChanged = false;
+
+	bool writeTransactionInfo(int location, uint32_t subseq);
+
+	Tag chooseRouterTag() {
+		return savedRandomRouterTag.present() ? savedRandomRouterTag.get() : logSystemGetRandomRouterTag(*logSystem);
+	}
+};
+
+// LogSystem info in old epoch
+struct OldLogData {
+	std::vector<Reference<LogSet>> tLogs;
+	int32_t logRouterTags;
+	int32_t txsTags; // The number of txsTags, which may change across generations.
+	Version epochBegin, epochEnd;
+	Version recoverAt;
+	std::set<int8_t> pseudoLocalities;
+	LogEpoch epoch;
+
+	OldLogData() : logRouterTags(0), txsTags(0), epochBegin(0), epochEnd(0), recoverAt(0), epoch(0) {}
+
+	// Constructor for T of OldTLogConf and OldTLogCoreData
+	template <class T>
+	explicit OldLogData(const T& conf);
+};
+
+struct IdToInterf : ReferenceCounted<IdToInterf> {
+	Optional<Version> recoverAt = Optional<Version>();
+	std::map<UID, TLogInterface> lockInterf;
+};
+
+struct LogLockInfo {
+	Version epochEnd;
+	bool isCurrent;
+	Reference<LogSet> logSet;
+	std::vector<Future<TLogLockResult>> replies;
+
+	LogLockInfo() : epochEnd(std::numeric_limits<Version>::max()), isCurrent(false) {}
+};
+
+struct DurableVersionInfo {
+	Version knownCommittedVersion; // maximum of the known committed versions of available tLogs
+	Version minimumDurableVersion; // mimimum of the durable versions of available tLogs
+	std::vector<TLogLockResult> lockResults; // replies from various tLogs
+	bool policyResult; // unavailable tLogs meet the replication policy or not
+	std::vector<uint16_t> knownLockedTLogIds; // tLogs that are known to have been locked
+
+	DurableVersionInfo(Version kcv,
+	                   Version dv,
+	                   std::vector<TLogLockResult>& replies,
+	                   bool meetsPolicy,
+	                   std::vector<uint16_t>& lockedTLogIds)
+	  : knownCommittedVersion(kcv), minimumDurableVersion(dv), lockResults(std::move(replies)),
+	    policyResult(meetsPolicy), knownLockedTLogIds(std::move(lockedTLogIds)) {}
+};
+
+struct LogSystem : ReferenceCounted<LogSystem> {
+	const UID dbgid;
+	LogSystemType logSystemType;
+	std::vector<Reference<LogSet>> tLogs; // LogSets in different locations: primary, satellite, or remote
+	int expectedLogSets;
+	int logRouterTags;
+	int txsTags;
+	UID recruitmentID;
+	int repopulateRegionAntiQuorum;
+	bool stopped;
+	std::set<int8_t> pseudoLocalities; // Represent special localities that will be mapped to tagLocalityLogRouter
+	const LogEpoch epoch;
+	LogEpoch oldestBackupEpoch;
+
+	// new members
+	std::map<Tag, Version> pseudoLocalityPopVersion;
+	Future<Void> rejoins;
+	Future<Void> recoveryComplete;
+	Future<Void> remoteRecovery;
+	Future<Void> remoteRecoveryComplete;
+	Future<Void> trackTLogRecovery;
+	Future<Void> remoteTrackTLogRecovery;
+	Reference<AsyncVar<Version>> recoveredVersion;
+	Reference<AsyncVar<Version>> remoteRecoveredVersion;
+	std::vector<LogLockInfo> lockResults;
+	AsyncVar<bool> recoveryCompleteWrittenToCoreState;
+	bool remoteLogsWrittenToCoreState;
+	bool hasRemoteServers;
+	AsyncTrigger backupWorkerChanged;
+	std::set<UID> removedBackupWorkers; // Workers that are removed before setting them.
+	std::map<uint8_t, std::vector<uint16_t>> knownLockedTLogIds;
+
+	Optional<Version> recoverAt;
+	Optional<Version> recoveredAt;
+	Version knownCommittedVersion;
+	Version backupStartVersion = invalidVersion; // max(tLogs[0].startVersion, previous epochEnd).
+	std::map<UID, Version> rvLogs; // recovery versions per tlog
+	LocalityData locality;
+	// For each currently running popFromLog actor, outstandingPops is
+	// (logID, tag)->(max popped version, durableKnownCommittedVersion).
+	// Why do we need durableKnownCommittedVersion? knownCommittedVersion gives the lower bound of what data
+	// will need to be copied into the next generation to restore the replication factor.
+	// Guess: It probably serves as a minimum version of what data should be on a TLog in the next generation and
+	// sending a pop for anything less than durableKnownCommittedVersion for the TLog will be absurd.
+	std::map<std::pair<UID, Tag>, std::pair<Version, Version>> outstandingPops;
+
+	// Stores each <log router, tag> pair's last popped version. This is used to determine whether we need to pop an old
+	// generation log router.
+	std::map<std::pair<UID, Tag>, Version> logRouterLastPops;
+
+	Optional<PromiseStream<Future<Void>>> addActor;
+	ActorCollection popActors;
+	std::vector<OldLogData> oldLogData; // each element has the log info. in one old epoch.
+	AsyncTrigger logSystemConfigChanged;
+
+	LogSystem(UID dbgid,
+	          LocalityData locality,
+	          LogEpoch e,
+	          Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>())
+	  : dbgid(dbgid), logSystemType(LogSystemType::empty), expectedLogSets(0), logRouterTags(0), txsTags(0),
+	    repopulateRegionAntiQuorum(0), stopped(false), epoch(e), oldestBackupEpoch(0),
+	    recoveredVersion(makeReference<AsyncVar<Version>>(invalidVersion)),
+	    remoteRecoveredVersion(makeReference<AsyncVar<Version>>(invalidVersion)),
+	    recoveryCompleteWrittenToCoreState(false), remoteLogsWrittenToCoreState(false), hasRemoteServers(false),
+	    locality(locality), addActor(addActor), popActors(false) {}
+
+	void stopRejoins();
+
+	void addref();
+
+	void delref();
+
+	std::string describe() const;
+
+	UID getDebugID() const;
+
+	LogSystemType getLogSystemType() const;
+
+	void addPseudoLocality(int8_t locality);
+
+	Tag getPseudoPopTag(Tag tag, ProcessClass::ClassType type) const;
+
+	bool hasPseudoLocality(int8_t locality) const;
+
+	// Return the min version of all pseudoLocalities, i.e., logRouter and backupTag
+	Version popPseudoLocalityTag(Tag tag, Version upTo);
+
+	static Future<Void> recoverAndEndEpoch(Reference<AsyncVar<Reference<LogSystem>>> const& outLogSystem,
+	                                       UID const& dbgid,
+	                                       DBCoreState const& oldState,
+	                                       FutureStream<TLogRejoinRequest> const& rejoins,
+	                                       LocalityData const& locality,
+	                                       bool* forceRecovery);
+
+	static Reference<LogSystem> fromLogSystemConfig(UID const& dbgid,
+	                                                LocalityData const& locality,
+	                                                LogSystemConfig const& lsConf,
+	                                                bool excludeRemote,
+	                                                bool useRecoveredAt,
+	                                                Optional<PromiseStream<Future<Void>>> addActor);
+
+	static Reference<LogSystem> fromOldLogSystemConfig(UID const& dbgid,
+	                                                   LocalityData const& locality,
+	                                                   LogSystemConfig const& lsConf);
+
+	// Convert LogSystem to DBCoreState and override input newState as return value
+	void toCoreState(DBCoreState& newState) const;
+
+	bool remoteStorageRecovered() const;
+
+	// Checks older TLog generations and remove no longer needed generations from the log system.
+	void purgeOldRecoveredGenerationsCoreState(DBCoreState&);
+	void purgeOldRecoveredGenerationsInMemory(const DBCoreState&);
+
+	Future<Void> onCoreStateChanged() const;
+
+	void coreStateWritten(DBCoreState const& newState);
+
+	Future<Void> onError() const;
+
+	static Future<Void> onError_internal(LogSystem const* self);
+
+	static Future<Void> pushResetChecker(Reference<ConnectionResetInfo> self, NetworkAddress addr);
+
+	static Future<TLogCommitReply> recordPushMetrics(Reference<ConnectionResetInfo> self,
+	                                                 Reference<Histogram> dist,
+	                                                 NetworkAddress addr,
+	                                                 Future<TLogCommitReply> in);
+
+	Future<Version> push(const LogPushVersionSet& versionSet,
+	                     LogPushData& data,
+	                     SpanContext const& spanContext,
+	                     Optional<UID> debugID,
+	                     Optional<std::unordered_map<uint16_t, Version>> tpcvMap);
+
+	// Version vector/Unicast specific: reset best server if it is not known to have been locked/stopped.
+	void resetBestServerIfNotLocked(int bestSet,
+	                                int& bestServer,
+	                                Optional<Version> end,
+	                                const Optional<std::map<uint8_t, std::vector<uint16_t>>>& knownLockedTLogIds);
+
+	Reference<IPeekCursor> peekAll(UID dbgid, Version begin, Version end, Tag tag, bool parallelGetMore);
+
+	Reference<IPeekCursor> peekRemote(UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore);
+
+	Reference<IPeekCursor> peek(UID dbgid, Version begin, Optional<Version> end, Tag tag, bool parallelGetMore);
+
+	Reference<IPeekCursor> peek(UID dbgid,
+	                            Version begin,
+	                            Optional<Version> end,
+	                            std::vector<Tag> tags,
+	                            bool parallelGetMore);
+
+	Reference<IPeekCursor> peekLocal(UID dbgid,
+	                                 Tag tag,
+	                                 Version begin,
+	                                 Version end,
+	                                 bool useMergePeekCursors,
+	                                 int8_t peekLocality = tagLocalityInvalid);
+
+	Reference<IPeekCursor> peekTxs(UID dbgid,
+	                               Version begin,
+	                               int8_t peekLocality,
+	                               Version localEnd,
+	                               bool canDiscardPopped);
+
+	Reference<IPeekCursor> peekSingle(
+	    UID dbgid,
+	    Version begin,
+	    Tag tag,
+	    std::vector<std::pair<Version, Tag>> history = std::vector<std::pair<Version, Tag>>());
+
+	// LogRouter or BackupWorker use this function to obtain a cursor for peeking tlogs of a generation (i.e., epoch).
+	// Specifically, the epoch is determined by looking up "dbgid" in tlog sets of generations.
+	// The returned cursor can peek data at the "tag" from the given "begin" version to that epoch's end version or
+	// the recovery version for the latest old epoch. For the current epoch, the cursor has no end version.
+	// For the old epoch, the cursor is provided an end version.
+	Reference<IPeekCursor> peekLogRouter(UID dbgid,
+	                                     Version begin,
+	                                     Tag tag,
+	                                     bool useSatellite,
+	                                     Optional<Version> end = Optional<Version>(),
+	                                     const Optional<std::map<uint8_t, std::vector<uint16_t>>>& knownStoppedTLogIds =
+	                                         Optional<std::map<uint8_t, std::vector<uint16_t>>>());
+
+	Version getKnownCommittedVersion();
+
+	Future<Void> onKnownCommittedVersionChange();
+
+	void popLogRouter(Version upTo, Tag tag, Version durableKnownCommittedVersion, int8_t popLocality);
+
+	void popTxs(Version upTo, int8_t popLocality = tagLocalityInvalid);
+
+	// pop 'tag.locality' type data up to the 'upTo' version
+	void pop(Version upTo, Tag tag, Version durableKnownCommittedVersion = 0, int8_t popLocality = tagLocalityInvalid);
+
+	// pop tag from log up to the version defined in self->outstandingPops[].first
+	static Future<Void> popFromLog(LogSystem* self,
+	                               Reference<AsyncVar<OptionalInterface<TLogInterface>>> log,
+	                               Tag tag,
+	                               double delayBeforePop,
+	                               bool popLogRouter);
+
+	static Future<Version> getPoppedFromTLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> log, Tag tag);
+
+	static Future<Version> getPoppedTxs(LogSystem* self);
+
+	Future<Version> getTxsPoppedVersion();
+
+	static Future<Void> confirmEpochLive_internal(Reference<LogSet> logSet, Optional<UID> debugID);
+
+	// Returns success after confirming that pushes in the current epoch are still possible
+	Future<Void> confirmEpochLive(Optional<UID> debugID = Optional<UID>());
+
+	Future<Void> endEpoch();
+
+	// Call only after end_epoch() has successfully completed.  Returns a new epoch immediately following this one.
+	// The new epoch is only provisional until the caller updates the coordinated DBCoreState.
+	Future<Reference<LogSystem>> newEpoch(RecruitFromConfigurationReply const& recr,
+	                                      Future<RecruitRemoteFromConfigurationReply> const& fRemoteWorkers,
+	                                      DatabaseConfiguration const& config,
+	                                      LogEpoch recoveryCount,
+	                                      Version recoveryTransactionVersion,
+	                                      int8_t primaryLocality,
+	                                      int8_t remoteLocality,
+	                                      std::vector<Tag> const& allTags,
+	                                      Reference<AsyncVar<bool>> const& recruitmentStalled);
+
+	LogSystemConfig getLogSystemConfig() const;
+
+	Standalone<StringRef> getLogsValue() const;
+
+	Future<Void> onLogSystemConfigChange();
+
+	void updateLogRouter(int logSetIndex, int tagId, TLogInterface const& newLogRouter);
+
+	Version getEnd() const;
+
+	Version getPeekEnd() const;
+
+	void getPushLocations(VectorRef<Tag> tags,
+	                      std::vector<int>& locations,
+	                      bool allLocations,
+	                      Optional<std::vector<Reference<LocalitySet>>> fromLocations) const;
+
+	std::vector<Reference<LocalitySet>> getPushLocationsForTags(std::vector<int>& fromLocations) const;
+
+	bool hasRemoteLogs() const;
+
+	Tag getRandomRouterTag() const;
+
+	Tag getRandomTxsTag() const;
+
+	TLogVersion getTLogVersion() const;
+
+	int getLogRouterTags() const;
+
+	Version getBackupStartVersion() const;
+
+	std::map<LogEpoch, EpochTagsVersionsInfo> getOldEpochTagsVersionsInfo() const;
+
+	inline Reference<LogSet> getEpochLogSet(LogEpoch epoch) const;
+
+	void setBackupWorkers(const std::vector<InitializeBackupReply>& replies);
+
+	bool removeBackupWorker(const BackupWorkerDoneRequest& req);
+
+	LogEpoch getOldestBackupEpoch() const;
+
+	void setOldestBackupEpoch(LogEpoch epoch);
+	static Future<Void> monitorLog(Reference<AsyncVar<OptionalInterface<TLogInterface>>> logServer,
+	                               Reference<AsyncVar<bool>> failed);
+
+	// returns the log group's knownComittedVersion, DV, a vector of TLogLockResults for
+	// each tLog in the group, and the result of applying the replication policy check over
+	// unavailable tLogs from the group
+	Optional<DurableVersionInfo> static getDurableVersion(
+	    UID dbgid,
+	    LogLockInfo lockInfo,
+	    std::vector<Reference<AsyncVar<bool>>> failed = std::vector<Reference<AsyncVar<bool>>>(),
+	    Optional<Version> lastEnd = Optional<Version>());
+
+	static Future<Void> getDurableVersionChanged(
+	    LogLockInfo lockInfo,
+	    std::vector<Reference<AsyncVar<bool>>> failed = std::vector<Reference<AsyncVar<bool>>>());
+
+	static Future<Void> epochEnd(Reference<AsyncVar<Reference<LogSystem>>> outLogSystem,
+	                             UID dbgid,
+	                             DBCoreState prevState,
+	                             FutureStream<TLogRejoinRequest> rejoinRequests,
+	                             LocalityData locality,
+	                             bool* forceRecovery);
+
+	static Future<Void> recruitOldLogRouters(LogSystem* self,
+	                                         std::vector<WorkerInterface> workers,
+	                                         LogEpoch recoveryCount,
+	                                         int8_t locality,
+	                                         Version startVersion,
+	                                         std::vector<LocalityData> tLogLocalities,
+	                                         Reference<IReplicationPolicy> tLogPolicy,
+	                                         bool forRemote);
+
+	static Version getMaxLocalStartVersion(const std::vector<Reference<LogSet>>& tLogs);
+
+	static std::vector<Tag> getLocalTags(int8_t locality, const std::vector<Tag>& allTags);
+
+	static Future<Void> newRemoteEpoch(LogSystem* self,
+	                                   Reference<LogSystem> oldLogSystem,
+	                                   Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
+	                                   DatabaseConfiguration configuration,
+	                                   LogEpoch recoveryCount,
+	                                   Version recoveryTransactionVersion,
+	                                   int8_t remoteLocality,
+	                                   std::vector<Tag> allTags,
+	                                   std::vector<Version> oldGenerationRecoverAtVersions);
+
+	static Future<Reference<LogSystem>> newEpoch(Reference<LogSystem> oldLogSystem,
+	                                             RecruitFromConfigurationReply recr,
+	                                             Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers,
+	                                             DatabaseConfiguration configuration,
+	                                             LogEpoch recoveryCount,
+	                                             Version recoveryTransactionVersion,
+	                                             int8_t primaryLocality,
+	                                             int8_t remoteLocality,
+	                                             std::vector<Tag> allTags,
+	                                             Reference<AsyncVar<bool>> recruitmentStalled);
+
+	static Future<Void> trackRejoins(
+	    UID dbgid,
+	    std::vector<std::pair<Reference<AsyncVar<OptionalInterface<TLogInterface>>>, Reference<IReplicationPolicy>>>
+	        logServers,
+	    FutureStream<struct TLogRejoinRequest> rejoinRequests);
+
+	// Keeps track of the recovered generations in all the TLogs in `tlogs` list and updates the latest
+	// `recoveredVersion`.
+	static Future<Void> trackTLogRecoveryActor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> tlogs,
+	                                           Reference<AsyncVar<Version>> recoveredVersion);
+
+	static Future<TLogLockResult> lockTLog(UID myID, Reference<AsyncVar<OptionalInterface<TLogInterface>>> tlog);
+	template <class T>
+	static std::vector<T> getReadyNonError(std::vector<Future<T>> const& futures);
+};
+
+// Recovery version calculation for version vector unicast
+Optional<std::tuple<Version, Version>> getRecoverVersionUnicast(
+    const std::vector<Reference<LogSet>>& logServers,
+    const std::tuple<int, std::vector<TLogLockResult>, bool>& logGroupResults,
+    Version minDV);
+
+template <class T>
+std::vector<T> LogSystem::getReadyNonError(std::vector<Future<T>> const& futures) {
+	// Return the values of those futures which have (non-error) values ready
+	std::vector<T> result;
+	for (auto& f : futures)
+		if (f.isReady() && !f.isError())
+			result.push_back(f.get());
+	return result;
+}
+
+#include "LogSystemTypes.h"
+
+template <class T>
+OldLogData::OldLogData(const T& conf)
+  : logRouterTags(conf.logRouterTags), txsTags(conf.txsTags), epochBegin(conf.epochBegin), epochEnd(conf.epochEnd),
+    recoverAt(conf.recoverAt), pseudoLocalities(conf.pseudoLocalities), epoch(conf.epoch) {
+	tLogs.resize(conf.tLogs.size());
+	for (int j = 0; j < conf.tLogs.size(); j++) {
+		auto logSet = makeReference<LogSet>(conf.tLogs[j]);
+		tLogs[j] = logSet;
+	}
+}
+
+template <class T>
+void LogPushData::writeTypedMessage(T const& item, bool metadataMessage, bool allLocations) {
+	prev_tags.clear();
+	if (logSystemHasRemoteLogs(*logSystem)) {
+		prev_tags.push_back(chooseRouterTag());
+	}
+	for (auto& tag : next_message_tags) {
+		prev_tags.push_back(tag);
+	}
+	msg_locations.clear();
+	logSystemGetPushLocations(*logSystem,
+	                          VectorRef<Tag>((Tag*)prev_tags.data(), prev_tags.size()),
+	                          msg_locations,
+	                          allLocations,
+	                          fromLocations);
+
+	BinaryWriter bw(AssumeVersion(g_network->protocolVersion()));
+
+	if (!metadataMessage) {
+		uint32_t subseq = this->subsequence++;
+		bool updatedLocation = false;
+		for (int loc : msg_locations) {
+			updatedLocation = writeTransactionInfo(loc, subseq) || updatedLocation;
+		}
+		if (!updatedLocation) {
+			this->subsequence--;
+			CODE_PROBE(true, "No new SpanContextMessage written to transaction logs");
+			ASSERT(this->subsequence > 0);
+		}
+	} else {
+		ASSERT(writtenLocations.size() == 0);
+	}
+
+	uint32_t subseq = this->subsequence++;
+	bool first = true;
+	int firstOffset = -1, firstLength = -1;
+	for (int loc : msg_locations) {
+		BinaryWriter& wr = messagesWriter[loc];
+
+		if (first) {
+			firstOffset = wr.getLength();
+			wr << uint32_t(0) << subseq << uint16_t(prev_tags.size());
+			for (auto& tag : prev_tags)
+				wr << tag;
+			wr << item;
+			firstLength = wr.getLength() - firstOffset;
+			*(uint32_t*)((uint8_t*)wr.getData() + firstOffset) = firstLength - sizeof(uint32_t);
+			DEBUG_TAGS_AND_MESSAGE(
+			    "ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength))
+			    .detail("PushLocations", msg_locations);
+			first = false;
+		} else {
+			BinaryWriter& from = messagesWriter[msg_locations[0]];
+			wr.serializeBytes((uint8_t*)from.getData() + firstOffset, firstLength);
+		}
+	}
+	written_tags.insert(next_message_tags.begin(), next_message_tags.end());
+	next_message_tags.clear();
+}
+
+#endif // FDBSERVER_LOGSYSTEM_LOGSYSTEM_H
diff --git a/fdbserver/include/fdbserver/LogSystemDiskQueueAdapter.h b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemDiskQueueAdapter.h
similarity index 89%
rename from fdbserver/include/fdbserver/LogSystemDiskQueueAdapter.h
rename to fdbserver/logsystem/include/fdbserver/logsystem/LogSystemDiskQueueAdapter.h
index bf54ccb8567..b688cf950b3 100644
--- a/fdbserver/include/fdbserver/LogSystemDiskQueueAdapter.h
+++ b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemDiskQueueAdapter.h
@@ -23,7 +23,8 @@
 #pragma once
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/IDiskQueue.h"
+#include "fdbserver/core/IDiskQueue.h"
+#include "fdbserver/logsystem/LogSystem.h"
 
 struct PeekTxsInfo {
 	int8_t primaryLocality;
@@ -43,20 +44,20 @@ struct PeekTxsInfo {
 
 class LogSystemDiskQueueAdapter final : public IDiskQueue {
 public:
-	// This adapter is designed to let KeyValueStoreMemory use ILogSystem
+	// This adapter is designed to let KeyValueStoreMemory use LogSystem
 	// as a backing store, so that the transaction subsystem can in
 	// turn use KeyValueStoreMemory to track configuration information as of
 	// the database version and recover it from the logging subsystem as necessary.
 
 	// Because the transaction subsystem will need to control the actual pushing of
-	// committed information to the ILogSystem, commit() in this interface doesn't directly
-	// call ILogSystem::push().  Instead it makes a commit message available through
+	// committed information to the LogSystem, commit() in this interface doesn't directly
+	// call LogSystem::push().  Instead it makes a commit message available through
 	// getCommitMessage(), and doesn't return until its acknowledge promise is set.
-	// The caller is responsible for calling ILogSystem::push() and ILogSystem::pop() with the results.
+	// The caller is responsible for calling LogSystem::push() and LogSystem::pop() with the results.
 
 	// It does, however, peek the specified tag directly at recovery time.
 
-	LogSystemDiskQueueAdapter(Reference<ILogSystem> logSystem,
+	LogSystemDiskQueueAdapter(Reference<LogSystem> logSystem,
 	                          Reference<AsyncVar<PeekTxsInfo>> peekLocality,
 	                          Version txsPoppedVersion,
 	                          bool recover)
@@ -117,12 +118,12 @@ class LogSystemDiskQueueAdapter final : public IDiskQueue {
 private:
 	Reference<AsyncVar<PeekTxsInfo>> peekLocality;
 	Future<Void> localityChanged;
-	Reference<ILogSystem::IPeekCursor> cursor;
+	Reference<IPeekCursor> cursor;
 	int peekTypeSwitches;
 
 	// Recovery state (used while readNext() is being called repeatedly)
 	bool enableRecovery;
-	Reference<ILogSystem> logSystem;
+	Reference<LogSystem> logSystem;
 	Version startLoc, recoveryLoc, recoveryQueueLoc;
 	std::vector<Standalone<StringRef>> recoveryQueue;
 	int recoveryQueueDataSize;
@@ -138,7 +139,7 @@ class LogSystemDiskQueueAdapter final : public IDiskQueue {
 	friend class LogSystemDiskQueueAdapterImpl;
 };
 
-LogSystemDiskQueueAdapter* openDiskQueueAdapter(Reference<ILogSystem> logSystem,
+LogSystemDiskQueueAdapter* openDiskQueueAdapter(Reference<LogSystem> logSystem,
                                                 Reference<AsyncVar<PeekTxsInfo>> peekLocality,
                                                 Version txsPoppedVersion);
 
diff --git a/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemFactory.h b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemFactory.h
new file mode 100644
index 00000000000..00c0b0b1013
--- /dev/null
+++ b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemFactory.h
@@ -0,0 +1,52 @@
+/*
+ * LogSystemFactory.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_LOGSYSTEM_LOGSYSTEMFACTORY_H
+#define FDBSERVER_LOGSYSTEM_LOGSYSTEMFACTORY_H
+#pragma once
+
+#include "fdbserver/logsystem/LogSystem.h"
+
+Reference<LogSystem> makeLogSystemFromServerDBInfo(
+    UID const& dbgid,
+    ServerDBInfo const& db,
+    bool useRecoveredAt = false,
+    Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>());
+
+Reference<LogSystem> makeLogSystemFromLogSystemConfig(
+    UID const& dbgid,
+    LocalityData const& locality,
+    LogSystemConfig const& conf,
+    bool excludeRemote = false,
+    bool useRecoveredAt = false,
+    Optional<PromiseStream<Future<Void>>> addActor = Optional<PromiseStream<Future<Void>>>());
+
+Reference<LogSystem> makeOldLogSystemFromLogSystemConfig(UID const& dbgid,
+                                                         LocalityData const& locality,
+                                                         LogSystemConfig const& conf);
+
+Future<Void> recoverAndEndLogSystemEpoch(Reference<AsyncVar<Reference<LogSystem>>> const& outLogSystem,
+                                         UID const& dbgid,
+                                         DBCoreState const& oldState,
+                                         FutureStream<TLogRejoinRequest> const& rejoins,
+                                         LocalityData const& locality,
+                                         bool* forceRecovery);
+
+#endif
diff --git a/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemTypes.h b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemTypes.h
new file mode 100644
index 00000000000..974f93f3868
--- /dev/null
+++ b/fdbserver/logsystem/include/fdbserver/logsystem/LogSystemTypes.h
@@ -0,0 +1,358 @@
+/*
+ * LogSystemTypes.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_LOGSYSTEM_LOGSYSTEMTYPES_H
+#define FDBSERVER_LOGSYSTEM_LOGSYSTEMTYPES_H
+#pragma once
+
+#include "fdbserver/core/LogSystemConfig.h"
+#include "fdbserver/core/DBCoreState.h"
+
+class LogSet : NonCopyable, public ReferenceCounted<LogSet> {
+public:
+	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> logServers;
+	std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> logRouters;
+	std::vector<Reference<AsyncVar<OptionalInterface<BackupInterface>>>> backupWorkers;
+	std::vector<Reference<ConnectionResetInfo>> connectionResetTrackers;
+	std::vector<Reference<Histogram>> tlogPushDistTrackers;
+	int32_t tLogWriteAntiQuorum;
+	int32_t tLogReplicationFactor;
+	std::vector<LocalityData> tLogLocalities;
+	TLogVersion tLogVersion;
+	Reference<IReplicationPolicy> tLogPolicy;
+	Reference<LocalitySet> logServerSet;
+	std::vector<int> logIndexArray;
+	std::vector<LocalityEntry> logEntryArray;
+	bool isLocal;
+	int8_t locality;
+	Version startVersion;
+	std::vector<Future<TLogLockResult>> replies;
+	std::vector<std::vector<int>> satelliteTagLocations;
+
+	LogSet()
+	  : tLogWriteAntiQuorum(0), tLogReplicationFactor(0), isLocal(true), locality(tagLocalityInvalid),
+	    startVersion(invalidVersion) {}
+	explicit LogSet(const TLogSet& tlogSet);
+	explicit LogSet(const CoreTLogSet& coreSet);
+
+	std::string logRouterString();
+	bool hasLogRouter(UID id) const;
+	bool hasBackupWorker(UID id) const;
+	std::string logServerString();
+	void populateSatelliteTagLocations(int logRouterTags, int oldLogRouterTags, int txsTags, int oldTxsTags);
+	void checkSatelliteTagLocations();
+	int bestLocationFor(Tag tag);
+	void updateLocalitySet(std::vector<LocalityData> const& localities);
+	bool satisfiesPolicy(const std::vector<LocalityEntry>& locations);
+	void getPushLocations(
+	    VectorRef<Tag> tags,
+	    std::vector<int>& locations,
+	    int locationOffset,
+	    bool allLocations = false,
+	    const Optional<Reference<LocalitySet>>& restrictedLogSet = Optional<Reference<LocalitySet>>());
+
+private:
+	std::vector<LocalityEntry> alsoServers, resultEntries;
+	std::vector<int> newLocations;
+};
+
+class ServerPeekCursor final : public IPeekCursor, public ReferenceCounted<ServerPeekCursor> {
+public:
+	Reference<AsyncVar<OptionalInterface<TLogInterface>>> interf;
+	const Tag tag;
+	TLogPeekReply results;
+	ArenaReader rd;
+	LogMessageVersion messageVersion, end;
+	Version poppedVersion;
+	TagsAndMessage messageAndTags;
+	bool hasMsg;
+	Future<Void> more;
+	UID randomID;
+	bool returnIfBlocked;
+	bool onlySpilled;
+	bool parallelGetMore;
+	bool usePeekStream;
+	int sequence;
+	Deque<Future<TLogPeekReply>> futureResults;
+	Future<Void> interfaceChanged;
+	Optional<ReplyPromiseStream<TLogPeekStreamReply>> peekReplyStream;
+	double lastReset;
+	Future<Void> resetCheck;
+	int slowReplies;
+	int fastReplies;
+	int unknownReplies;
+	bool returnEmptyIfStopped;
+
+	ServerPeekCursor(Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf,
+	                 Tag tag,
+	                 Version begin,
+	                 Version end,
+	                 bool returnIfBlocked,
+	                 bool parallelGetMore,
+	                 bool returnEmtpyIfStopped = false);
+	ServerPeekCursor(TLogPeekReply const& results,
+	                 LogMessageVersion const& messageVersion,
+	                 LogMessageVersion const& end,
+	                 TagsAndMessage const& message,
+	                 bool hasMsg,
+	                 Version poppedVersion,
+	                 Tag tag);
+
+	Reference<IPeekCursor> cloneNoMore() override;
+	void setProtocolVersion(ProtocolVersion version) override;
+	Arena& arena() override;
+	ArenaReader* reader() override;
+	bool hasMessage() const override;
+	void nextMessage() override;
+	StringRef getMessage() override;
+	StringRef getMessageWithTags() override;
+	VectorRef<Tag> getTags() const override;
+	void advanceTo(LogMessageVersion n) override;
+	Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
+	Future<Void> onFailed() const override;
+	bool isActive() const override;
+	bool isExhausted() const override;
+	const LogMessageVersion& version() const override;
+	Version popped() const override;
+	Version getMinKnownCommittedVersion() const override;
+	Optional<UID> getPrimaryPeekLocation() const override;
+	Optional<UID> getCurrentPeekLocation() const override;
+	void addref() override { ReferenceCounted<ServerPeekCursor>::addref(); }
+	void delref() override { ReferenceCounted<ServerPeekCursor>::delref(); }
+	Version getMaxKnownVersion() const override { return results.maxKnownVersion; }
+};
+
+class MergedPeekCursor final : public IPeekCursor, public ReferenceCounted<MergedPeekCursor> {
+public:
+	Reference<LogSet> logSet;
+	std::vector<Reference<IPeekCursor>> serverCursors;
+	std::vector<LocalityEntry> locations;
+	std::vector<std::pair<LogMessageVersion, int>> sortedVersions;
+	Tag tag;
+	int bestServer, currentCursor, readQuorum;
+	Optional<LogMessageVersion> nextVersion;
+	LogMessageVersion messageVersion;
+	bool hasNextMessage;
+	UID randomID;
+	int tLogReplicationFactor;
+	Future<Void> more;
+
+	MergedPeekCursor(std::vector<Reference<IPeekCursor>> const& serverCursors, Version begin);
+	MergedPeekCursor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
+	                 int bestServer,
+	                 int readQuorum,
+	                 Tag tag,
+	                 Version begin,
+	                 Version end,
+	                 bool parallelGetMore,
+	                 std::vector<LocalityData> const& tLogLocalities,
+	                 Reference<IReplicationPolicy> const tLogPolicy,
+	                 int tLogReplicationFactor,
+	                 const Optional<std::vector<uint16_t>>& knownLockedTLogIds = Optional<std::vector<uint16_t>>());
+	MergedPeekCursor(std::vector<Reference<IPeekCursor>> const& serverCursors,
+	                 LogMessageVersion const& messageVersion,
+	                 int bestServer,
+	                 int readQuorum,
+	                 Optional<LogMessageVersion> nextVersion,
+	                 Reference<LogSet> logSet,
+	                 int tLogReplicationFactor);
+
+	Reference<IPeekCursor> cloneNoMore() override;
+	void setProtocolVersion(ProtocolVersion version) override;
+	Arena& arena() override;
+	ArenaReader* reader() override;
+	void calcHasMessage();
+	void updateMessage(bool usePolicy);
+	bool hasMessage() const override;
+	void nextMessage() override;
+	StringRef getMessage() override;
+	StringRef getMessageWithTags() override;
+	VectorRef<Tag> getTags() const override;
+	void advanceTo(LogMessageVersion n) override;
+	Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
+	Future<Void> onFailed() const override;
+	bool isActive() const override;
+	bool isExhausted() const override;
+	const LogMessageVersion& version() const override;
+	Version popped() const override;
+	Version getMinKnownCommittedVersion() const override;
+	Optional<UID> getPrimaryPeekLocation() const override;
+	Optional<UID> getCurrentPeekLocation() const override;
+	void addref() override { ReferenceCounted<MergedPeekCursor>::addref(); }
+	void delref() override { ReferenceCounted<MergedPeekCursor>::delref(); }
+};
+
+class SetPeekCursor final : public IPeekCursor, public ReferenceCounted<SetPeekCursor> {
+public:
+	std::vector<Reference<LogSet>> logSets;
+	std::vector<std::vector<Reference<IPeekCursor>>> serverCursors;
+	Tag tag;
+	int bestSet, bestServer, currentSet, currentCursor;
+	std::vector<LocalityEntry> locations;
+	std::vector<std::pair<LogMessageVersion, int>> sortedVersions;
+	Optional<LogMessageVersion> nextVersion;
+	LogMessageVersion messageVersion;
+	bool hasNextMessage;
+	bool useBestSet;
+	UID randomID;
+	Future<Void> more;
+	Optional<Version> end;
+
+	SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
+	              int bestSet,
+	              int bestServer,
+	              Tag tag,
+	              Version begin,
+	              Version end,
+	              bool parallelGetMore,
+	              const Optional<std::vector<uint16_t>>& knownLockedTLogIds = Optional<std::vector<uint16_t>>());
+	SetPeekCursor(std::vector<Reference<LogSet>> const& logSets,
+	              std::vector<std::vector<Reference<IPeekCursor>>> const& serverCursors,
+	              LogMessageVersion const& messageVersion,
+	              int bestSet,
+	              int bestServer,
+	              Optional<LogMessageVersion> nextVersion,
+	              bool useBestSet);
+
+	Reference<IPeekCursor> cloneNoMore() override;
+	void setProtocolVersion(ProtocolVersion version) override;
+	Arena& arena() override;
+	ArenaReader* reader() override;
+	void calcHasMessage();
+	void updateMessage(int logIdx, bool usePolicy);
+	bool hasMessage() const override;
+	void nextMessage() override;
+	StringRef getMessage() override;
+	StringRef getMessageWithTags() override;
+	VectorRef<Tag> getTags() const override;
+	void advanceTo(LogMessageVersion n) override;
+	Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
+	Future<Void> onFailed() const override;
+	bool isActive() const override;
+	bool isExhausted() const override;
+	const LogMessageVersion& version() const override;
+	Version popped() const override;
+	Version getMinKnownCommittedVersion() const override;
+	Optional<UID> getPrimaryPeekLocation() const override;
+	Optional<UID> getCurrentPeekLocation() const override;
+	void addref() override { ReferenceCounted<SetPeekCursor>::addref(); }
+	void delref() override { ReferenceCounted<SetPeekCursor>::delref(); }
+};
+
+class MultiCursor final : public IPeekCursor, public ReferenceCounted<MultiCursor> {
+public:
+	std::vector<Reference<IPeekCursor>> cursors;
+	std::vector<LogMessageVersion> epochEnds;
+	Version poppedVersion;
+
+	MultiCursor(std::vector<Reference<IPeekCursor>> cursors, std::vector<LogMessageVersion> epochEnds);
+
+	Reference<IPeekCursor> cloneNoMore() override;
+	void setProtocolVersion(ProtocolVersion version) override;
+	Arena& arena() override;
+	ArenaReader* reader() override;
+	bool hasMessage() const override;
+	void nextMessage() override;
+	StringRef getMessage() override;
+	StringRef getMessageWithTags() override;
+	VectorRef<Tag> getTags() const override;
+	void advanceTo(LogMessageVersion n) override;
+	Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
+	Future<Void> onFailed() const override;
+	bool isActive() const override;
+	bool isExhausted() const override;
+	const LogMessageVersion& version() const override;
+	Version popped() const override;
+	Version getMinKnownCommittedVersion() const override;
+	Optional<UID> getPrimaryPeekLocation() const override;
+	Optional<UID> getCurrentPeekLocation() const override;
+	void addref() override { ReferenceCounted<MultiCursor>::addref(); }
+	void delref() override { ReferenceCounted<MultiCursor>::delref(); }
+};
+
+class BufferedCursor final : public IPeekCursor, public ReferenceCounted<BufferedCursor> {
+public:
+	struct BufferedMessage {
+		Arena arena;
+		StringRef message;
+		VectorRef<Tag> tags;
+		LogMessageVersion version;
+
+		BufferedMessage() {}
+		explicit BufferedMessage(Version version) : version(version) {}
+		BufferedMessage(Arena arena, StringRef message, const VectorRef<Tag>& tags, const LogMessageVersion& version)
+		  : arena(arena), message(message), tags(tags), version(version) {}
+
+		bool operator<(BufferedMessage const& r) const { return version < r.version; }
+		bool operator==(BufferedMessage const& r) const { return version == r.version; }
+	};
+
+	std::vector<Reference<IPeekCursor>> cursors;
+	std::vector<Deque<BufferedMessage>> cursorMessages;
+	std::vector<BufferedMessage> messages;
+	int messageIndex;
+	LogMessageVersion messageVersion;
+	Version end;
+	bool hasNextMessage;
+	bool withTags;
+	bool knownUnique;
+	Version minKnownCommittedVersion;
+	Version poppedVersion;
+	Version initialPoppedVersion;
+	bool canDiscardPopped;
+	Future<Void> more;
+	int targetQueueSize;
+	UID randomID;
+
+	BufferedCursor(std::vector<Reference<IPeekCursor>> cursors,
+	               Version begin,
+	               Version end,
+	               bool withTags,
+	               bool canDiscardPopped);
+	BufferedCursor(std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers,
+	               Tag tag,
+	               Version begin,
+	               Version end,
+	               bool parallelGetMore);
+
+	Reference<IPeekCursor> cloneNoMore() override;
+	void setProtocolVersion(ProtocolVersion version) override;
+	Arena& arena() override;
+	ArenaReader* reader() override;
+	bool hasMessage() const override;
+	void nextMessage() override;
+	StringRef getMessage() override;
+	StringRef getMessageWithTags() override;
+	VectorRef<Tag> getTags() const override;
+	void advanceTo(LogMessageVersion n) override;
+	Future<Void> getMore(TaskPriority taskID = TaskPriority::TLogPeekReply) override;
+	Future<Void> onFailed() const override;
+	bool isActive() const override;
+	bool isExhausted() const override;
+	const LogMessageVersion& version() const override;
+	Version popped() const override;
+	Version getMinKnownCommittedVersion() const override;
+	Optional<UID> getPrimaryPeekLocation() const override;
+	Optional<UID> getCurrentPeekLocation() const override;
+	void addref() override { ReferenceCounted<BufferedCursor>::addref(); }
+	void delref() override { ReferenceCounted<BufferedCursor>::delref(); }
+};
+
+#endif
diff --git a/fdbserver/masterserver.swift b/fdbserver/masterserver.swift
deleted file mode 100644
index 2742d42ef0f..00000000000
--- a/fdbserver/masterserver.swift
+++ /dev/null
@@ -1,494 +0,0 @@
-import Flow
-import flow_swift
-@preconcurrency import FDBServer
-import FDBClient
-import fdbclient_swift
-import CxxStdlib
-
-func clamp(_ v: Version, lowerBound: Version, upperBound: Version) -> Version {
-    return max(min(v, upperBound), lowerBound)
-}
-
-@_expose(Cxx)
-public func figureVersion(current: Version,
-                   now: Double,
-                   reference: Version,
-                   toAdd: Int64,
-                   maxVersionRateModifier: Double,
-                   maxVersionRateOffset: Int64) -> Version {
-    // Versions should roughly follow wall-clock time, based on the
-    // system clock of the current machine and an FDB-specific epoch.
-    // Calculate the expected version and determine whether we need to
-    // hand out versions faster or slower to stay in sync with the
-    // clock.
-    let expected = Version(now * Double(getServerKnobs().VERSIONS_PER_SECOND)) - reference
-
-    // Attempt to jump directly to the expected version. But make
-    // sure that versions are still being handed out at a rate
-    // around VERSIONS_PER_SECOND. This rate is scaled depending on
-    // how far off the calculated version is from the expected
-    // version.
-    let maxOffset = min(Int64(Double(toAdd) * maxVersionRateModifier), maxVersionRateOffset)
-    return clamp(expected, lowerBound: current + toAdd - maxOffset,
-                           upperBound: current + toAdd + maxOffset)
-}
-
-@_expose(Cxx)
-public func updateLiveCommittedVersion(myself: MasterData, req: ReportRawCommittedVersionRequest) {
-    myself.minKnownCommittedVersion = max(myself.minKnownCommittedVersion, req.minKnownCommittedVersion)
-
-    if req.version > myself.liveCommittedVersion.get() {
-        if getServerKnobs().ENABLE_VERSION_VECTOR,
-           let writtenTags = Swift.Optional(cxxOptional: req.writtenTags) {
-            let primaryLocality = getServerKnobs().ENABLE_VERSION_VECTOR_HA_OPTIMIZATION ?
-                    myself.locality : Int8(tagLocalityInvalid)
-             myself.ssVersionVector.setVersion(writtenTags, req.version, primaryLocality)
-             myself.versionVectorTagUpdates.addMeasurement(Double(writtenTags.size()))
-        }
-
-        let curTime = now()
-        // add debug here to change liveCommittedVersion to time bound of now()
-        debug_advanceVersionTimestamp(Int64(myself.liveCommittedVersion.get()), curTime + getClientKnobs().MAX_VERSION_CACHE_LAG)
-        // also add req.version but with no time bound
-        debug_advanceVersionTimestamp(Int64(req.version), Double.greatestFiniteMagnitude)
-        myself.databaseLocked = req.locked;
-        myself.proxyMetadataVersion = req.metadataVersion
-        // Note the set call switches context to any waiters on liveCommittedVersion before continuing.
-        myself.liveCommittedVersion.set(Int(req.version))
-    }
-	myself.reportLiveCommittedVersionRequests += 1
-}
-
-extension NotifiedVersionValue {
-    mutating func atLeast(_ limit: VersionMetricHandle.ValueType) async throws {
-        try await self.whenAtLeast(limit).value()
-    }
-}
-
-public class CommitProxyVersionReplies {
-    var replies: [UInt64: GetCommitVersionReply] = [:]
-    var latestRequestNum: NotifiedVersionValue
-
-    public init() {
-        latestRequestNum = NotifiedVersionValue(0)
-    }
-}
-
-@_expose(Cxx)
-public actor MasterDataActor {
-    var lastCommitProxyVersionReplies: [Flow.UID: CommitProxyVersionReplies] = [:]
-
-    public init() {
-    }
-
-    func registerLastCommitProxyVersionReplies(uids: [Flow.UID]) async {
-        lastCommitProxyVersionReplies = [:]
-        for uid in uids {
-            lastCommitProxyVersionReplies[uid] = CommitProxyVersionReplies()
-        }
-    }
-
-    nonisolated public func registerLastCommitProxyVersionReplies(uids: [Flow.UID], result promise: PromiseVoid) {
-        Task {
-            await registerLastCommitProxyVersionReplies(uids: uids)
-            promise.send(Flow.Void())
-        }
-    }
-    nonisolated public func registerLastCommitProxyVersionReplies(uids: [Flow.UID]) -> FutureVoid {
-        let promise = PromiseVoid()
-        registerLastCommitProxyVersionReplies(uids: uids, result: promise)
-        return promise.getFuture()
-    }
-
-    func getVersion(myself: MasterData, req: GetCommitVersionRequest) async -> GetCommitVersionReply? {
-        myself.getCommitVersionRequests += 1
-
-        guard let lastVersionReplies = self.lastCommitProxyVersionReplies[req.requestingProxy] else {
-            // Request from invalid proxy (e.g. from duplicate recruitment request)
-            return nil
-        }
-
-        // CODE_PROBE(lastVersionReplies.latestRequestNum.get() < req.requestNum - 1, "Commit version request queued up")
-        try! await lastVersionReplies.latestRequestNum
-                .atLeast(VersionMetricHandle.ValueType(req.requestNum - UInt64(1)))
-
-        if let lastReply = lastVersionReplies.replies[req.requestNum] {
-            // NOTE: CODE_PROBE is macro, won't be imported
-            // CODE_PROBE(true, "Duplicate request for sequence")
-            return lastReply
-        } else if (req.requestNum <= lastVersionReplies.latestRequestNum.get()) {
-            // NOTE: CODE_PROBE is macro, won't be imported
-            // CODE_PROBE(true, "Old request for previously acknowledged sequence - may be impossible with current FlowTransport");
-            assert(req.requestNum < lastVersionReplies.latestRequestNum.get())
-            // The latest request can never be acknowledged
-            return nil
-        }
-
-        var rep = GetCommitVersionReply()
-
-        if (myself.version == invalidVersion) {
-            myself.lastVersionTime = now()
-            myself.version = myself.recoveryTransactionVersion
-            rep.prevVersion = myself.lastEpochEnd
-        } else {
-            var t1 = now()
-            if BUGGIFY() {
-                t1 = myself.lastVersionTime
-            }
-
-            let toAdd = max(Version(1), min(
-                    getServerKnobs().MAX_READ_TRANSACTION_LIFE_VERSIONS,
-                    Version(Double(getServerKnobs().VERSIONS_PER_SECOND) * (t1 - myself.lastVersionTime))))
-            rep.prevVersion = myself.version
-            if let referenceVersion = Swift.Optional(cxxOptional: myself.referenceVersion) {
-                myself.version = figureVersion(current: myself.version,
-                        now: SwiftGNetwork.timer(),
-                        reference: Version(referenceVersion),
-                        toAdd: toAdd,
-                        maxVersionRateModifier: getServerKnobs().MAX_VERSION_RATE_MODIFIER,
-                        maxVersionRateOffset: getServerKnobs().MAX_VERSION_RATE_OFFSET)
-                assert(myself.version > rep.prevVersion)
-            } else {
-                myself.version += toAdd
-            }
-
-            // NOTE: CODE_PROBE is macro, won't be imported
-            // CODE_PROBE(self.version - rep.prevVersion == 1, "Minimum possible version gap");
-            let _ /*maxVersionGap*/ = myself.version - rep.prevVersion == getServerKnobs().MAX_READ_TRANSACTION_LIFE_VERSIONS
-            // CODE_PROBE(maxVersionGap, "Maximum possible version gap");
-
-            myself.lastVersionTime = t1
-            myself.getResolutionBalancer().setChangesInReply(req.requestingProxy, &rep)
-        }
-
-        rep.version = myself.version
-        rep.requestNum = req.requestNum
-
-        // TODO: highlight some collection use (c++ collection, find on it etc)
-        lastVersionReplies.replies = lastVersionReplies.replies.filter { $0.0 > req.mostRecentProcessedRequestNum }
-        lastVersionReplies.replies[req.requestNum] = rep
-        assert(rep.prevVersion >= 0)
-
-        assert(lastVersionReplies.latestRequestNum.get() == req.requestNum - 1)
-        lastVersionReplies.latestRequestNum.set(Int(req.requestNum))
-        return rep
-    }
-
-    /// Promise type must match result type of the target function.
-    /// If missing, please declare new `using PromiseXXX = Promise<XXX>;` in `swift_<MODULE>_future_support.h` files.
-    nonisolated public func getVersion(myself: MasterData, req: GetCommitVersionRequest,
-                                       result promise: PromiseVoid) {
-        Task {
-            if let rep = await getVersion(myself: myself, req: req) {
-                req.reply.send(rep)
-            } else {
-                req.reply.sendNever()
-            }
-            promise.send(Flow.Void())
-        }
-    }
-    nonisolated public func getVersion(myself: MasterData, req: GetCommitVersionRequest) -> FutureVoid {
-        let promise = PromiseVoid()
-        getVersion(myself: myself, req: req, result: promise)
-        return promise.getFuture()
-    }
-
-    // ACTOR Future<Void> waitForPrev(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
-    @discardableResult
-    func waitForPrev(myself: MasterData, req: ReportRawCommittedVersionRequest) async -> Flow.Void {
-        let startTime = now()
-
-        // TODO: typealias the optional
-        if let prevVersion = Swift.Optional(cxxOptional: req.prevVersion) {
-            try! await myself.liveCommittedVersion.atLeast(Int(prevVersion)) // TODO: try to not need the conversion here
-        }
-
-        let latency = now() - startTime
-
-        myself.waitForPrevLatencies.addMeasurement(latency)
-        myself.waitForPrevCommitRequests += 1
-
-        updateLiveCommittedVersion(myself: myself, req: req)
-
-        return Void()
-    }
-
-    nonisolated public func waitForPrev(myself: MasterData, req: ReportRawCommittedVersionRequest,
-                                        result promise: PromiseVoid) {
-        Task {
-            let rep = await waitForPrev(myself: myself, req: req)
-            req.reply.send(rep)
-            promise.send(rep)
-        }
-    }
-    nonisolated public func waitForPrev(myself: MasterData, req: ReportRawCommittedVersionRequest) -> FutureVoid {
-        let promise = PromiseVoid()
-        waitForPrev(myself: myself, req: req, result: promise)
-        return promise.getFuture()
-    }
-
-    @discardableResult
-    public func provideVersions(myself: MasterData) async throws -> Flow.Void {
-        //  state ActorCollection versionActors(false);
-        //
-        //	loop choose {
-        //		when(GetCommitVersionRequest req = waitNext(myself.myInterface.getCommitVersion.getFuture())) {
-        //			versionActors.add(getVersion(self, req));
-        //		}
-        //		when(wait(versionActors.getResult())) {}
-        //	}
-        try await withThrowingTaskGroup(of: Swift.Void.self) { group in
-            // TODO(swift): the only throw here is from the Sequence, but can it actually ever happen...?
-            for try await req in myself.myInterface.getCommitVersion.getFuture() {
-                _ = group.addTaskUnlessCancelled {
-                    if let rep = await self.getVersion(myself: myself, req: req) {
-                        req.reply.send(rep)
-                    } else {
-                        req.reply.sendNever()
-                    }
-                }
-
-                if Task.isCancelled {
-                    return Flow.Void()
-                }
-            }
-            return Flow.Void()
-        }
-
-    }
-
-    nonisolated public func provideVersions(myself: MasterData, result promise: PromiseVoid) {
-        Task {
-            // TODO(swift): handle the error
-            let void = try await self.provideVersions(myself: myself)
-            promise.send(void)
-        }
-    }
-    nonisolated public func provideVersions(myself: MasterData) -> FutureVoid {
-        let promise = PromiseVoid()
-        provideVersions(myself: myself, result: promise)
-        return promise.getFuture()
-    }
-
-    public func getLiveCommittedVersion(myself: MasterData, _ req: GetRawCommittedVersionRequest) -> GetRawCommittedVersionReply {
-        if req.debugID.present() {
-            // TODO: trace event here
-            //     g_traceBatch.addEvent("TransactionDebug",
-            //		    req.debugID.get().first(),
-            //			"MasterServer.serveLiveCommittedVersion.GetRawCommittedVersion");
-        }
-
-        if myself.liveCommittedVersion.get() == invalidVersion {
-            myself.liveCommittedVersion.set(Int(myself.recoveryTransactionVersion));
-        }
-
-        myself.getLiveCommittedVersionRequests += 1
-
-        var reply = GetRawCommittedVersionReply()
-        let liveCommittedVersion: Int = myself.liveCommittedVersion.get()
-        reply.version = Version(liveCommittedVersion)
-        reply.locked = myself.databaseLocked
-        reply.metadataVersion = myself.proxyMetadataVersion
-        reply.minKnownCommittedVersion = myself.minKnownCommittedVersion
-
-        if (getServerKnobs().ENABLE_VERSION_VECTOR) {
-            myself.versionVectorSizeOnCVReply.addMeasurement(Double(reply.ssVersionVectorDelta.size()))
-        }
-
-        return reply
-    }
-
-    func reportLiveCommittedVersion(myself: MasterData, req: ReportRawCommittedVersionRequest) async -> Void {
-        if let prevVersion = Swift.Optional(cxxOptional: req.prevVersion),
-           getServerKnobs().ENABLE_VERSION_VECTOR &&
-          (myself.liveCommittedVersion.get() != invalidVersion) &&
-          (myself.liveCommittedVersion.get() < prevVersion) {
-            return await waitForPrev(myself: myself, req: req)
-        } else {
-            updateLiveCommittedVersion(myself: myself, req: req)
-            myself.nonWaitForPrevCommitRequests += 1
-            return Void()
-        }
-    }
-
-    @discardableResult
-    public func serveLiveCommittedVersion(myself: MasterData) async -> Flow.Void {
-        // TODO: use TaskPool
-        await withThrowingTaskGroup(of: Swift.Void.self) { group in
-            // getLiveCommittedVersion
-            group.addTask {
-                for try await req in myself.myInterface.getLiveCommittedVersion.getFuture() {
-                    let rep = await self.getLiveCommittedVersion(myself: myself, req)
-                    req.reply.send(rep)
-                }
-            }
-
-            // reportLiveCommittedVersion
-            group.addTask {
-                for try await req in myself.myInterface.reportLiveCommittedVersion.getFuture() {
-                    let rep = await self.reportLiveCommittedVersion(myself: myself, req: req)
-                    req.reply.send(rep)
-                }
-            }
-        }
-
-        return Flow.Void()
-    }
-
-    nonisolated public func serveLiveCommittedVersion(myself: MasterData, result promise: PromiseVoid) {
-        Task {
-            let void = await self.serveLiveCommittedVersion(myself: myself)
-            promise.send(void)
-        }
-    }
-    nonisolated public func serveLiveCommittedVersion(myself: MasterData) -> FutureVoid {
-        let promise = PromiseVoid()
-        serveLiveCommittedVersion(myself: myself, result: promise)
-        return promise.getFuture()
-    }
-
-    func updateRecoveryData(myself: MasterData, req: UpdateRecoveryDataRequest) async -> Flow.Void {
-        STraceEvent("UpdateRecoveryData", myself.dbgid)
-            .detail("ReceivedRecoveryTxnVersion", req.recoveryTransactionVersion)
-            .detail("ReceivedLastEpochEnd", req.lastEpochEnd)
-            .detail("CurrentRecoveryTxnVersion", myself.recoveryTransactionVersion)
-            .detail("CurrentLastEpochEnd", myself.lastEpochEnd)
-            .detail("NumCommitProxies", req.commitProxies.size())
-            .detail("VersionEpoch", req.versionEpoch)
-            .detail("PrimaryLocality", Int(req.primaryLocality)) // TODO: seems we have trouble with Int8->char mapping, so wrap in Int for now
-
-        myself.recoveryTransactionVersion = req.recoveryTransactionVersion
-        myself.lastEpochEnd = req.lastEpochEnd
-
-        if (req.commitProxies.size() > 0) {
-            var registeredUIDs = [UID]()
-            registeredUIDs.reserveCapacity(req.commitProxies.size())
-            for j in 0..<req.commitProxies.size() { // TODO(swift): can we make this be a Sequence?
-                registeredUIDs.append(req.commitProxies[j].id())
-            }
-            await self.registerLastCommitProxyVersionReplies(uids: registeredUIDs)
-        }
-
-        if let versionEpoch = Swift.Optional(cxxOptional: req.versionEpoch) {
-            myself.referenceVersion = OptionalVersion(versionEpoch)
-        } else if BUGGIFY() {
-            // Cannot use a positive version epoch in simulation because of the
-            // clock starting at 0. A positive version epoch would mean the initial
-            // cluster version was negative.
-            // TODO: Increase the size of this interval after fixing the issue
-            // with restoring ranges with large version gaps.
-            let v = swift_get_randomInt64(deterministicRandom(), Int64(-1e6), 0)
-            myself.referenceVersion = OptionalVersion(v)
-        }
-
-        myself.getResolutionBalancer().setCommitProxies(req.commitProxies)
-        myself.getResolutionBalancer().setResolvers(req.resolvers)
-
-        myself.locality = req.primaryLocality
-
-        return Void()
-    }
-
-    @discardableResult
-    public func serveUpdateRecoveryData(myself: MasterData) async throws -> Flow.Void {
-        try await withThrowingTaskGroup(of: Swift.Void.self) { group in
-            // Note: this is an example of one-by-one handling requests, notice the group.next() below.
-            for try await req in myself.myInterface.updateRecoveryData.getFuture() {
-                group.addTask {
-                    let rep = await self.updateRecoveryData(myself: myself, req: req)
-                    req.reply.send(rep)
-                }
-
-                try await group.next()
-            }
-        }
-
-        return Flow.Void()
-    }
-
-    nonisolated public func serveUpdateRecoveryData(myself: MasterData, result promise: PromiseVoid) {
-        Task {
-            // TODO(swift): handle the error
-            let void = try await self.serveUpdateRecoveryData(myself: myself)
-            promise.send(void)
-        }
-    }
-    nonisolated public func serveUpdateRecoveryData(myself: MasterData) -> FutureVoid {
-        let promise = PromiseVoid()
-        serveUpdateRecoveryData(myself: myself, result: promise)
-        return promise.getFuture()
-    }
-
-}
-
-extension MasterData {
-    var swiftActorImpl: MasterDataActor {
-        #if FDBSERVER_FORWARD_DECLARE_SWIFT_APIS
-        // During the generationg of the C++ header for this module, we do not
-        // yet have access to `getSwiftImpl` API.
-        return MasterDataActor()
-        #else
-        return self.getSwiftImpl()
-        #endif
-    }
-}
-
-@_expose(Cxx)
-public func masterServerSwift(
-        mi: MasterInterface,
-        db: AsyncVar_ServerDBInfo,
-        ccInterface: AsyncVar_Optional_ClusterControllerFullInterface,
-        coordinators: ServerCoordinators,
-        lifetime: LifetimeToken,
-        forceRecovery: Bool,
-        masterData: MasterData,
-        promise: PromiseVoid) {
-    Task {
-        let myself = masterData.swiftActorImpl
-
-        do {
-            try await withThrowingTaskGroup(of: Swift.Void.self) { group in
-                group.addTask {
-                    try await traceRole(Role.MASTER, mi.id()).value()
-                }
-                group.addTask {
-                    try await myself.provideVersions(myself: masterData)
-                }
-                group.addTask {
-                    await myself.serveLiveCommittedVersion(myself: masterData)
-                }
-                group.addTask {
-                    try await myself.serveUpdateRecoveryData(myself: masterData)
-                }
-
-//                CODE_PROBE(!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id()),
-//                        "Master born doomed");
-
-                STraceEvent("MasterLifetime", masterData.dbgid)
-                    .detail("LifetimeToken", lifetime.toString())
-
-                while true {
-                    guard (try? await db.onChange().value()) != nil else {
-                        return
-                    }
-
-                    guard lifetime.isStillValid(db.getCopy().masterLifetime, mi.id() == db.getCopy().master.id()) else {
-                        // CODE_PROBE(true, "Master replaced, dying")
-                        if BUGGIFY() {
-                            try? await FlowClock.sleep(for: .seconds(5))
-                        }
-
-                        // throwing out of here, cancels all the other tasks in the group as well.
-                        throw WorkerRemovedError()
-                    }
-                }
-            }
-        } catch {
-            print("Failure: \(error)")
-        }
-
-        promise.send(Void())
-    }
-}
-
-struct WorkerRemovedError: Swift.Error {}
diff --git a/fdbserver/mocks3/CMakeLists.txt b/fdbserver/mocks3/CMakeLists.txt
new file mode 100644
index 00000000000..6b0a011c254
--- /dev/null
+++ b/fdbserver/mocks3/CMakeLists.txt
@@ -0,0 +1,11 @@
+fdb_find_sources(FDBSERVER_MOCKS3_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_mocks3 SRCS ${FDBSERVER_MOCKS3_SRCS})
+
+configure_fdbserver_common_includes(fdbserver_mocks3)
+target_include_directories(fdbserver_mocks3
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_include_directories(fdbserver_mocks3 PRIVATE
+  ${CMAKE_SOURCE_DIR}/contrib/rapidjson)
+target_link_libraries(fdbserver_mocks3 PRIVATE fdbclient)
diff --git a/fdbserver/MockS3Server.actor.cpp b/fdbserver/mocks3/MockS3Server.cpp
similarity index 81%
rename from fdbserver/MockS3Server.actor.cpp
rename to fdbserver/mocks3/MockS3Server.cpp
index f141aceef34..5f5ad918c90 100644
--- a/fdbserver/MockS3Server.actor.cpp
+++ b/fdbserver/mocks3/MockS3Server.cpp
@@ -1,5 +1,5 @@
 /*
- * MockS3Server.actor.cpp
+ * MockS3Server.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/MockS3Server.h"
+#include "fdbserver/mocks3/MockS3Server.h"
 
 #include "fdbrpc/HTTP.h"
 #include "fdbrpc/simulator.h"
@@ -42,35 +42,6 @@
 #include "rapidjson/stringbuffer.h"
 #include "rapidjson/prettywriter.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-/*
- * ACTOR STATE VARIABLE INITIALIZATION REQUIREMENT
- *
- * ACTORs with early returns (before any wait()) crash with canBeSet() assertion if no state
- * variable is declared before the return. The actor compiler generates a member initialization
- * list (": member(value)") in the state class constructor only when it sees state variables.
- * This initialization list ensures the Actor<T> base class and its internal Promise are fully
- * initialized before any code runs. Without it, early returns try to use an uninitialized Promise.
- *
- * FIX: Declare at least one state variable BEFORE any early return. Declaration alone is enough.
- *
- * CORRECT:
- *   ACTOR Future<Void> someActor(...) {
- *       state std::string data;                // Triggers member init list (requires default ctor)
- *       if (earlyExitCondition) return Void(); // Safe - Promise is initialized
- *       data = computeValue();                 // Can initialize later
- *   }
- *
- *   // Or if no default constructor: state MyType x(params); and initialize at declaration
- *
- * WRONG (canBeSet() crash):
- *   ACTOR Future<Void> someActor(...) {
- *       if (earlyExitCondition) return Void(); // CRASH - no member init list generated yet
- *       state std::string data;                // Too late - compiler didn't see it early enough
- *   }
- */
-
 // MockS3 persistence file extensions and constants
 namespace {
 constexpr const char* DEFAULT_MOCKS3_PERSISTENCE_DIR = "simfdb/mocks3";
@@ -91,7 +62,7 @@ struct MockS3GlobalStorage {
 		double lastModified;
 
 		ObjectData() : lastModified(now()) {}
-		ObjectData(const std::string& data) : content(data), lastModified(now()) { etag = generateETag(data); }
+		explicit ObjectData(const std::string& data) : content(data), lastModified(now()) { etag = generateETag(data); }
 
 		static std::string generateETag(const std::string& content) {
 			return "\"" + HTTP::computeMD5Sum(content) + "\"";
@@ -122,9 +93,7 @@ struct MockS3GlobalStorage {
 
 	// Note: In FDB simulation, function-local statics are SHARED across all simulated processes
 	// because they all run on the same OS thread. This is exactly what we want for MockS3 storage.
-	MockS3GlobalStorage() {
-		// Note: Don't use TraceEvent in constructor - may be called before GlobalConfig is initialized
-	}
+	MockS3GlobalStorage() = default;
 
 	// Clear all stored data - called at the start of each simulation test to prevent
 	// data accumulation across multiple tests
@@ -191,7 +160,7 @@ static void createParentDirectories(const std::string& filePath) {
 // (see AsyncFileChaos.h:40). OPEN_NO_AIO controls AsyncFileNonDurable behavior.
 // Uses unique temp filename per caller to avoid race conditions when multiple
 // MockS3 servers write to the same path simultaneously (last writer wins).
-ACTOR static Future<Void> atomicWriteFile(std::string path, std::string content) {
+static Future<Void> atomicWriteFile(std::string path, std::string content) {
 	// Generate unique temp filename to avoid collision with other MockS3 servers
 	// writing to the same path. Each server gets its own temp file, and the
 	// last rename wins (correct S3 semantics).
@@ -201,21 +170,21 @@ ACTOR static Future<Void> atomicWriteFile(std::string path, std::string content)
 	// cause the simulation's random state to diverge between platforms.
 	// We use OPEN_ATOMIC_WRITE_AND_CREATE which creates a .part file internally,
 	// so the full chain is: path.{uuid}.tmp.part -> path.{uuid}.tmp -> path
-	state std::string tempPath = path + "." + nondeterministicRandom()->randomUniqueID().toString() + ".tmp";
+	std::string tempPath = path + "." + nondeterministicRandom()->randomUniqueID().toString() + ".tmp";
 
 	try {
 		// Create all parent directories
 		createParentDirectories(path);
 
 		// Write to unique temp file using atomic write (simulation requires this flag with OPEN_CREATE)
-		state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
+		Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
 		    tempPath,
 		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE |
 		        IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO,
-		    0644));
+		    0644);
 
-		wait(file->write(content.data(), content.size(), 0));
-		wait(file->sync()); // Renames .part -> tempPath
+		co_await file->write(content.data(), content.size(), 0);
+		co_await file->sync(); // Renames .part -> tempPath
 		file = Reference<IAsyncFile>();
 
 		// Rename unique temp file to final path (last writer wins)
@@ -231,26 +200,25 @@ ACTOR static Future<Void> atomicWriteFile(std::string path, std::string content)
 			// Ignore cleanup errors
 		}
 	}
-	return Void();
 }
 
 // ACTOR: Read file content using simulation filesystem without chaos
 // Chaos-free because AsyncFileChaos only affects files with "storage-" in the name
-ACTOR static Future<std::string> readFileContent(std::string path) {
-	state bool exists = fileExists(path); // State variable before any early returns
+static Future<std::string> readFileContent(std::string path) {
+	bool exists = fileExists(path); // State variable before any early returns
 
 	try {
 		if (!exists) {
-			return std::string();
+			co_return std::string();
 		}
 
-		state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
-		    path, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO, 0644));
-		state int64_t fileSize = wait(file->size());
+		Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
+		    path, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO, 0644);
+		int64_t fileSize = co_await file->size();
 
-		state std::string content;
+		std::string content;
 		content.resize(fileSize);
-		int bytesRead = wait(file->read((uint8_t*)content.data(), fileSize, 0));
+		int bytesRead = co_await file->read((uint8_t*)content.data(), fileSize, 0);
 		file = Reference<IAsyncFile>();
 
 		if (bytesRead != fileSize) {
@@ -260,23 +228,22 @@ ACTOR static Future<std::string> readFileContent(std::string path) {
 			    .detail("Actual", bytesRead);
 		}
 
-		return content;
+		co_return content;
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "MockS3PersistenceReadException").error(e).detail("Path", path);
-		return std::string();
+		co_return std::string();
 	}
 }
 
 // ACTOR: Delete file using simulation filesystem
 // Wraps deleteFile with trace events and error handling for MockS3 persistence cleanup
-ACTOR static Future<Void> deletePersistedFile(std::string path) {
+static Future<Void> deletePersistedFile(std::string path) {
 	try {
-		wait(IAsyncFileSystem::filesystem()->deleteFile(path, true)); // Durable delete
+		co_await IAsyncFileSystem::filesystem()->deleteFile(path, true); // Durable delete
 		TraceEvent("MockS3PersistenceDelete").detail("Path", path);
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "MockS3PersistenceDeleteException").error(e).detail("Path", path);
 	}
-	return Void();
 }
 
 // JSON Serialization using rapidjson
@@ -371,9 +338,9 @@ static void deserializeMultipartState(const std::string& jsonStr, MockS3GlobalSt
 }
 
 // Forward declarations for state loading functions
-ACTOR static Future<Void> loadPersistedObjects(std::string persistenceDir);
-ACTOR static Future<Void> loadPersistedMultipartUploads(std::string persistenceDir);
-ACTOR static Future<Void> loadMockS3PersistedStateImpl();
+static Future<Void> loadPersistedObjects(std::string persistenceDir);
+static Future<Void> loadPersistedMultipartUploads(std::string persistenceDir);
+static Future<Void> loadMockS3PersistedStateImpl();
 Future<Void> loadMockS3PersistedStateFuture();
 
 static std::string serializePartMeta(const std::string& etag) {
@@ -390,7 +357,7 @@ static std::string serializePartMeta(const std::string& etag) {
 }
 
 // ACTOR: Persist object data and metadata
-ACTOR static Future<Void> persistObject(std::string bucket, std::string object) {
+static Future<Void> persistObject(std::string bucket, std::string object) {
 	auto& storage = getGlobalStorage();
 
 	// In simulation, automatically enable persistence on first access (process-local storage)
@@ -400,28 +367,28 @@ ACTOR static Future<Void> persistObject(std::string bucket, std::string object)
 	}
 
 	if (!storage.persistenceEnabled) {
-		return Void(); // Persistence not enabled, skip
+		co_return; // Persistence not enabled, skip
 	}
 
 	auto bucketIter = storage.buckets.find(bucket);
 	if (bucketIter == storage.buckets.end()) {
-		return Void();
+		co_return;
 	}
 
 	auto objectIter = bucketIter->second.find(object);
 	if (objectIter == bucketIter->second.end()) {
-		return Void();
+		co_return;
 	}
 
 	// Copy data to state variables (needed across wait() boundaries)
-	state std::string content = objectIter->second.content;
-	state std::string metaJson = serializeObjectMeta(objectIter->second);
+	std::string content = objectIter->second.content;
+	std::string metaJson = serializeObjectMeta(objectIter->second);
 
 	try {
 
 		// Compute paths before wait() calls
-		state std::string dataPath = storage.getObjectDataPath(bucket, object);
-		state std::string metaPath = storage.getObjectMetaPath(bucket, object);
+		std::string dataPath = storage.getObjectDataPath(bucket, object);
+		std::string metaPath = storage.getObjectMetaPath(bucket, object);
 
 		// Skip if already persisted (race condition: multiple processes may try to persist same object)
 		// Check both final file and .part file to close race window
@@ -430,7 +397,7 @@ ACTOR static Future<Void> persistObject(std::string bucket, std::string object)
 			    .detail("Bucket", bucket)
 			    .detail("Object", object)
 			    .detail("DataPath", dataPath);
-			return Void();
+			co_return;
 		}
 
 		TraceEvent("MockS3PersistingObject")
@@ -440,10 +407,10 @@ ACTOR static Future<Void> persistObject(std::string bucket, std::string object)
 		    .detail("Size", content.size());
 
 		// Persist object content
-		wait(atomicWriteFile(dataPath, content));
+		co_await atomicWriteFile(dataPath, content);
 
 		// Persist object metadata
-		wait(atomicWriteFile(metaPath, metaJson));
+		co_await atomicWriteFile(metaPath, metaJson);
 
 		TraceEvent("MockS3ObjectPersisted")
 		    .detail("Bucket", bucket)
@@ -452,14 +419,12 @@ ACTOR static Future<Void> persistObject(std::string bucket, std::string object)
 	} catch (Error& e) {
 		TraceEvent(SevError, "MockS3PersistObjectFailed").error(e).detail("Bucket", bucket).detail("Object", object);
 	}
-
-	return Void();
 }
 
 // ACTOR: Persist multipart upload state
-ACTOR static Future<Void> persistMultipartState(std::string uploadId) {
-	state std::string persistenceDir; // Declare state before any early returns
-	state std::map<int, std::pair<std::string, std::string>> parts;
+static Future<Void> persistMultipartState(std::string uploadId) {
+	std::string persistenceDir; // Declare state before any early returns
+	std::map<int, std::pair<std::string, std::string>> parts;
 
 	auto& storage = getGlobalStorage();
 
@@ -470,12 +435,12 @@ ACTOR static Future<Void> persistMultipartState(std::string uploadId) {
 	}
 
 	if (!storage.persistenceEnabled) {
-		return Void(); // Persistence not enabled, skip
+		co_return; // Persistence not enabled, skip
 	}
 
 	auto uploadIter = storage.multipartUploads.find(uploadId);
 	if (uploadIter == storage.multipartUploads.end()) {
-		return Void();
+		co_return;
 	}
 
 	const auto& upload = uploadIter->second;
@@ -491,25 +456,25 @@ ACTOR static Future<Void> persistMultipartState(std::string uploadId) {
 		// Check both final file and .part file to close race window
 		if (fileExists(statePath) || fileExists(statePath + ".part")) {
 			TraceEvent("MockS3MultipartAlreadyPersisted").detail("UploadId", uploadId).detail("StatePath", statePath);
-			return Void();
+			co_return;
 		}
 
 		std::string stateJson = serializeMultipartState(upload);
-		wait(atomicWriteFile(statePath, stateJson));
+		co_await atomicWriteFile(statePath, stateJson);
 
 		// Persist each part
-		state std::map<int, std::pair<std::string, std::string>>::iterator partIter = parts.begin();
+		auto partIter = parts.begin();
 		while (partIter != parts.end()) {
-			state int partNum = partIter->first;
-			state std::string etag = partIter->second.first;
-			state std::string partData = partIter->second.second;
+			int partNum = partIter->first;
+			std::string etag = partIter->second.first;
+			std::string partData = partIter->second.second;
 
-			state std::string partPath = persistenceDir + "/multipart/" + uploadId + ".part." + std::to_string(partNum);
-			wait(atomicWriteFile(partPath, partData));
+			std::string partPath = persistenceDir + "/multipart/" + uploadId + ".part." + std::to_string(partNum);
+			co_await atomicWriteFile(partPath, partData);
 
-			state std::string partMetaPath = partPath + ".meta.json";
-			state std::string partMetaJson = serializePartMeta(etag);
-			wait(atomicWriteFile(partMetaPath, partMetaJson));
+			std::string partMetaPath = partPath + ".meta.json";
+			std::string partMetaJson = serializePartMeta(etag);
+			co_await atomicWriteFile(partMetaPath, partMetaJson);
 
 			partIter++;
 		}
@@ -518,14 +483,12 @@ ACTOR static Future<Void> persistMultipartState(std::string uploadId) {
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "MockS3PersistMultipartFailed").error(e).detail("UploadId", uploadId);
 	}
-
-	return Void();
 }
 
 // ACTOR: Delete persisted object
-ACTOR static Future<Void> deletePersistedObject(std::string bucket, std::string object) {
-	state std::string dataPath; // Declare state before any early returns
-	state std::string metaPath;
+static Future<Void> deletePersistedObject(std::string bucket, std::string object) {
+	std::string dataPath; // Declare state before any early returns
+	std::string metaPath;
 
 	auto& storage = getGlobalStorage();
 	ASSERT(storage.persistenceEnabled); // Caller should check before calling
@@ -534,8 +497,8 @@ ACTOR static Future<Void> deletePersistedObject(std::string bucket, std::string
 	metaPath = storage.getObjectMetaPath(bucket, object);
 
 	try {
-		wait(deletePersistedFile(dataPath));
-		wait(deletePersistedFile(metaPath));
+		co_await deletePersistedFile(dataPath);
+		co_await deletePersistedFile(metaPath);
 
 		TraceEvent("MockS3ObjectDeleted").detail("Bucket", bucket).detail("Object", object);
 	} catch (Error& e) {
@@ -544,17 +507,15 @@ ACTOR static Future<Void> deletePersistedObject(std::string bucket, std::string
 		    .detail("Bucket", bucket)
 		    .detail("Object", object);
 	}
-
-	return Void();
 }
 
 // ACTOR: Delete persisted multipart upload
-ACTOR static Future<Void> deletePersistedMultipart(std::string uploadId) {
-	state int maxPart; // Declare state before any early returns
-	state std::string persistenceDir;
-	state int partNum;
-	state std::string partPath;
-	state std::string partMetaPath;
+static Future<Void> deletePersistedMultipart(std::string uploadId) {
+	int maxPart{ 0 }; // Declare state before any early returns
+	std::string persistenceDir;
+	int partNum{ 0 };
+	std::string partPath;
+	std::string partMetaPath;
 
 	auto& storage = getGlobalStorage();
 	ASSERT(storage.persistenceEnabled); // Caller should check before calling
@@ -574,7 +535,7 @@ ACTOR static Future<Void> deletePersistedMultipart(std::string uploadId) {
 
 		// Delete state file
 		std::string statePath = persistenceDir + "/multipart/" + uploadId + ".state.json";
-		wait(deletePersistedFile(statePath));
+		co_await deletePersistedFile(statePath);
 
 		// Delete all part files (try all possible part numbers)
 		// Yield very frequently with delays to prevent blocking other MockS3 requests
@@ -582,14 +543,14 @@ ACTOR static Future<Void> deletePersistedMultipart(std::string uploadId) {
 		while (partNum <= maxPart + 10) {
 			partPath = persistenceDir + "/multipart/" + uploadId + ".part." + std::to_string(partNum);
 			partMetaPath = partPath + ".meta.json";
-			wait(deletePersistedFile(partPath));
-			wait(deletePersistedFile(partMetaPath));
+			co_await deletePersistedFile(partPath);
+			co_await deletePersistedFile(partMetaPath);
 			partNum++;
 
 			// Yield every 2 parts with 20ms delay to allow HTTP requests to be processed
 			// For 110 parts: ~55 yields = ~1.1s total delay time
 			if (partNum % 2 == 0) {
-				wait(delay(0.02));
+				co_await delay(0.02);
 			}
 		}
 
@@ -597,8 +558,6 @@ ACTOR static Future<Void> deletePersistedMultipart(std::string uploadId) {
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "MockS3DeletePersistedMultipartFailed").error(e).detail("UploadId", uploadId);
 	}
-
-	return Void();
 }
 
 // Mock S3 Server Implementation for deterministic testing
@@ -608,14 +567,14 @@ class MockS3ServerImpl {
 	using MultipartUpload = MockS3GlobalStorage::MultipartUpload;
 
 	// Note: No TraceEvent here - constructor called before GlobalConfig is initialized
-	MockS3ServerImpl() {}
+	MockS3ServerImpl() = default;
 
-	~MockS3ServerImpl() {}
+	~MockS3ServerImpl() = default;
 
 	// S3 Operation Handlers
-	ACTOR static Future<Void> handleRequest(MockS3ServerImpl* self,
-	                                        Reference<HTTP::IncomingRequest> req,
-	                                        Reference<HTTP::OutgoingResponse> response) {
+	static Future<Void> handleRequest(MockS3ServerImpl* self,
+	                                  Reference<HTTP::IncomingRequest> req,
+	                                  Reference<HTTP::OutgoingResponse> response) {
 
 		TraceEvent("MockS3Request")
 		    .detail("Method", req->verb)
@@ -640,39 +599,39 @@ class MockS3ServerImpl {
 			    .detail("QueryParamCount", queryParams.size());
 
 			// Route to appropriate handler based on operation type
-			if (queryParams.count("uploads")) {
-				wait(self->handleMultipartStart(self, req, response, bucket, object));
-			} else if (queryParams.count("uploadId")) {
-				if (queryParams.count("partNumber")) {
-					wait(self->handleUploadPart(self, req, response, bucket, object, queryParams));
+			if (queryParams.contains("uploads")) {
+				co_await self->handleMultipartStart(self, req, response, bucket, object);
+			} else if (queryParams.contains("uploadId")) {
+				if (queryParams.contains("partNumber")) {
+					co_await self->handleUploadPart(self, req, response, bucket, object, queryParams);
 				} else if (req->verb == "POST") {
-					wait(self->handleMultipartComplete(self, req, response, bucket, object, queryParams));
+					co_await self->handleMultipartComplete(self, req, response, bucket, object, queryParams);
 				} else if (req->verb == "DELETE") {
-					wait(self->handleMultipartAbort(self, req, response, bucket, object, queryParams));
+					co_await self->handleMultipartAbort(self, req, response, bucket, object, queryParams);
 				} else {
 					self->sendError(
 					    response, HTTP::HTTP_STATUS_CODE_BAD_GATEWAY, "InvalidRequest", "Unknown multipart operation");
 				}
-			} else if (queryParams.count("tagging")) {
+			} else if (queryParams.contains("tagging")) {
 				if (req->verb == "PUT") {
-					wait(self->handlePutObjectTags(self, req, response, bucket, object));
+					co_await self->handlePutObjectTags(self, req, response, bucket, object);
 				} else if (req->verb == "GET") {
-					wait(self->handleGetObjectTags(self, req, response, bucket, object));
+					co_await self->handleGetObjectTags(self, req, response, bucket, object);
 				} else {
 					self->sendError(response,
 					                HTTP::HTTP_STATUS_CODE_BAD_GATEWAY,
 					                "MethodNotAllowed",
 					                "Method not allowed for tagging");
 				}
-			} else if (queryParams.count("list-type") || (req->verb == "GET" && object.empty())) {
+			} else if (queryParams.contains("list-type") || (req->verb == "GET" && object.empty())) {
 				// ListObjects operation (when GET request to bucket)
-				wait(self->handleListObjects(self, req, response, bucket, queryParams));
+				co_await self->handleListObjects(self, req, response, bucket, queryParams);
 			} else if (object.empty()) {
 				// Bucket-level operations
 				if (req->verb == "HEAD") {
-					wait(self->handleHeadBucket(self, req, response, bucket));
+					co_await self->handleHeadBucket(self, req, response, bucket);
 				} else if (req->verb == "PUT") {
-					wait(self->handlePutBucket(self, req, response, bucket));
+					co_await self->handlePutBucket(self, req, response, bucket);
 				} else {
 					self->sendError(response,
 					                HTTP::HTTP_STATUS_CODE_BAD_GATEWAY,
@@ -682,13 +641,13 @@ class MockS3ServerImpl {
 			} else {
 				// Basic object operations
 				if (req->verb == "PUT") {
-					wait(self->handlePutObject(self, req, response, bucket, object));
+					co_await self->handlePutObject(self, req, response, bucket, object);
 				} else if (req->verb == "GET") {
-					wait(self->handleGetObject(self, req, response, bucket, object));
+					co_await self->handleGetObject(self, req, response, bucket, object);
 				} else if (req->verb == "DELETE") {
-					wait(self->handleDeleteObject(self, req, response, bucket, object));
+					co_await self->handleDeleteObject(self, req, response, bucket, object);
 				} else if (req->verb == "HEAD") {
-					wait(self->handleHeadObject(self, req, response, bucket, object));
+					co_await self->handleHeadObject(self, req, response, bucket, object);
 				} else {
 					self->sendError(
 					    response, HTTP::HTTP_STATUS_CODE_BAD_GATEWAY, "MethodNotAllowed", "Method not supported");
@@ -699,8 +658,6 @@ class MockS3ServerImpl {
 			TraceEvent(SevError, "MockS3RequestError").error(e).detail("Resource", req->resource);
 			self->sendError(response, 500, "InternalError", "Internal server error");
 		}
-
-		return Void();
 	}
 
 	void parseS3Request(const std::string& resource,
@@ -790,11 +747,11 @@ class MockS3ServerImpl {
 	}
 
 	// Multipart Upload Operations
-	ACTOR static Future<Void> handleMultipartStart(MockS3ServerImpl* self,
-	                                               Reference<HTTP::IncomingRequest> req,
-	                                               Reference<HTTP::OutgoingResponse> response,
-	                                               std::string bucket,
-	                                               std::string object) {
+	static Future<Void> handleMultipartStart(MockS3ServerImpl* self,
+	                                         Reference<HTTP::IncomingRequest> req,
+	                                         Reference<HTTP::OutgoingResponse> response,
+	                                         std::string bucket,
+	                                         std::string object) {
 
 		TraceEvent("MockS3MultipartStart").detail("Bucket", bucket).detail("Object", object);
 
@@ -813,7 +770,7 @@ class MockS3ServerImpl {
 			}
 		}
 
-		state std::string uploadId;
+		std::string uploadId;
 		if (!existingUploadId.empty()) {
 			uploadId = existingUploadId;
 			// No need to persist - already exists and was persisted on first creation
@@ -825,7 +782,7 @@ class MockS3ServerImpl {
 
 			// Persist only the newly created upload
 			if (getGlobalStorage().persistenceEnabled) {
-				wait(persistMultipartState(uploadId));
+				co_await persistMultipartState(uploadId);
 			}
 		}
 
@@ -841,19 +798,17 @@ class MockS3ServerImpl {
 		                         uploadId.c_str());
 
 		self->sendXMLResponse(response, 200, xml);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> handleUploadPart(MockS3ServerImpl* self,
-	                                           Reference<HTTP::IncomingRequest> req,
-	                                           Reference<HTTP::OutgoingResponse> response,
-	                                           std::string bucket,
-	                                           std::string object,
-	                                           std::map<std::string, std::string> queryParams) {
+	static Future<Void> handleUploadPart(MockS3ServerImpl* self,
+	                                     Reference<HTTP::IncomingRequest> req,
+	                                     Reference<HTTP::OutgoingResponse> response,
+	                                     std::string bucket,
+	                                     std::string object,
+	                                     std::map<std::string, std::string> queryParams) {
 
-		state std::string uploadId = queryParams.at("uploadId");
-		state int partNumber = std::stoi(queryParams.at("partNumber"));
+		std::string uploadId = queryParams.at("uploadId");
+		int partNumber = std::stoi(queryParams.at("partNumber"));
 
 		TraceEvent("MockS3UploadPart")
 		    .detail("UploadId", uploadId)
@@ -861,23 +816,23 @@ class MockS3ServerImpl {
 		    .detail("ContentLength", req->data.contentLen)
 		    .detail("ActualContentSize", req->data.content.size())
 		    .detail("ContentPreview",
-		            req->data.content.size() > 0
+		            !req->data.content.empty()
 		                ? req->data.content.substr(0, std::min((size_t)20, req->data.content.size()))
 		                : "EMPTY");
 
 		auto uploadIter = getGlobalStorage().multipartUploads.find(uploadId);
 		if (uploadIter == getGlobalStorage().multipartUploads.end()) {
 			self->sendError(response, HTTP::HTTP_STATUS_CODE_NOT_FOUND, "NoSuchUpload", "Upload not found");
-			return Void();
+			co_return;
 		}
 
 		// Store part data
-		state std::string etag = ObjectData::generateETag(req->data.content);
+		std::string etag = ObjectData::generateETag(req->data.content);
 		uploadIter->second.parts[partNumber] = { etag, req->data.content };
 
 		// Persist multipart state (includes all parts)
 		if (getGlobalStorage().persistenceEnabled) {
-			wait(persistMultipartState(uploadId));
+			co_await persistMultipartState(uploadId);
 		}
 
 		// Return ETag in response
@@ -890,29 +845,27 @@ class MockS3ServerImpl {
 		    .detail("UploadId", uploadId)
 		    .detail("PartNumber", partNumber)
 		    .detail("ETag", etag);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> handleMultipartComplete(MockS3ServerImpl* self,
-	                                                  Reference<HTTP::IncomingRequest> req,
-	                                                  Reference<HTTP::OutgoingResponse> response,
-	                                                  std::string bucket,
-	                                                  std::string object,
-	                                                  std::map<std::string, std::string> queryParams) {
+	static Future<Void> handleMultipartComplete(MockS3ServerImpl* self,
+	                                            Reference<HTTP::IncomingRequest> req,
+	                                            Reference<HTTP::OutgoingResponse> response,
+	                                            std::string bucket,
+	                                            std::string object,
+	                                            std::map<std::string, std::string> queryParams) {
 
-		state std::string uploadId = queryParams.at("uploadId");
+		std::string uploadId = queryParams.at("uploadId");
 
 		TraceEvent("MockS3MultipartComplete").detail("UploadId", uploadId);
 
 		auto uploadIter = getGlobalStorage().multipartUploads.find(uploadId);
 		if (uploadIter == getGlobalStorage().multipartUploads.end()) {
 			self->sendError(response, HTTP::HTTP_STATUS_CODE_NOT_FOUND, "NoSuchUpload", "Upload not found");
-			return Void();
+			co_return;
 		}
 
 		// Combine all parts in order
-		state std::string combinedContent;
+		std::string combinedContent;
 		for (auto& part : uploadIter->second.parts) {
 			combinedContent += part.second.second;
 		}
@@ -922,8 +875,8 @@ class MockS3ServerImpl {
 		    .detail("PartsCount", uploadIter->second.parts.size())
 		    .detail("CombinedSize", combinedContent.size())
 		    .detail("CombinedPreview",
-		            combinedContent.size() > 0 ? combinedContent.substr(0, std::min((size_t)20, combinedContent.size()))
-		                                       : "EMPTY");
+		            !combinedContent.empty() ? combinedContent.substr(0, std::min((size_t)20, combinedContent.size()))
+		                                     : "EMPTY");
 
 		// Create final object
 		ObjectData obj(combinedContent);
@@ -933,20 +886,20 @@ class MockS3ServerImpl {
 		    .detail("UploadId", uploadId)
 		    .detail("StoredSize", getGlobalStorage().buckets[bucket][object].content.size())
 		    .detail("StoredPreview",
-		            getGlobalStorage().buckets[bucket][object].content.size() > 0
+		            !getGlobalStorage().buckets[bucket][object].content.empty()
 		                ? getGlobalStorage().buckets[bucket][object].content.substr(
 		                      0, std::min((size_t)20, getGlobalStorage().buckets[bucket][object].content.size()))
 		                : "EMPTY");
 
 		// Persist final object
 		if (getGlobalStorage().persistenceEnabled) {
-			wait(persistObject(bucket, object));
+			co_await persistObject(bucket, object);
 		}
 
 		// Clean up multipart upload (in-memory and persisted)
 		getGlobalStorage().multipartUploads.erase(uploadId);
 		if (getGlobalStorage().persistenceEnabled) {
-			wait(deletePersistedMultipart(uploadId));
+			co_await deletePersistedMultipart(uploadId);
 		}
 
 		// Generate completion XML response
@@ -963,31 +916,29 @@ class MockS3ServerImpl {
 		self->sendXMLResponse(response, 200, xml);
 
 		TraceEvent("MockS3MultipartCompleted").detail("UploadId", uploadId).detail("FinalSize", combinedContent.size());
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> handleMultipartAbort(MockS3ServerImpl* self,
-	                                               Reference<HTTP::IncomingRequest> req,
-	                                               Reference<HTTP::OutgoingResponse> response,
-	                                               std::string bucket,
-	                                               std::string object,
-	                                               std::map<std::string, std::string> queryParams) {
+	static Future<Void> handleMultipartAbort(MockS3ServerImpl* self,
+	                                         Reference<HTTP::IncomingRequest> req,
+	                                         Reference<HTTP::OutgoingResponse> response,
+	                                         std::string bucket,
+	                                         std::string object,
+	                                         std::map<std::string, std::string> queryParams) {
 
-		state std::string uploadId = queryParams.at("uploadId");
+		std::string uploadId = queryParams.at("uploadId");
 
 		TraceEvent("MockS3MultipartAbort").detail("UploadId", uploadId);
 
 		auto uploadIter = getGlobalStorage().multipartUploads.find(uploadId);
 		if (uploadIter == getGlobalStorage().multipartUploads.end()) {
 			self->sendError(response, HTTP::HTTP_STATUS_CODE_NOT_FOUND, "NoSuchUpload", "Upload not found");
-			return Void();
+			co_return;
 		}
 
 		// Remove multipart upload (in-memory and persisted)
 		getGlobalStorage().multipartUploads.erase(uploadId);
 		if (getGlobalStorage().persistenceEnabled) {
-			wait(deletePersistedMultipart(uploadId));
+			co_await deletePersistedMultipart(uploadId);
 		}
 
 		response->code = 204; // No Content
@@ -995,8 +946,6 @@ class MockS3ServerImpl {
 		response->data.content->discardAll(); // Clear existing content
 
 		TraceEvent("MockS3MultipartAborted").detail("UploadId", uploadId);
-
-		return Void();
 	}
 
 	// Object Tagging Operations
@@ -1072,11 +1021,11 @@ class MockS3ServerImpl {
 	}
 
 	// Basic Object Operations
-	ACTOR static Future<Void> handlePutObject(MockS3ServerImpl* self,
-	                                          Reference<HTTP::IncomingRequest> req,
-	                                          Reference<HTTP::OutgoingResponse> response,
-	                                          std::string bucket,
-	                                          std::string object) {
+	static Future<Void> handlePutObject(MockS3ServerImpl* self,
+	                                    Reference<HTTP::IncomingRequest> req,
+	                                    Reference<HTTP::OutgoingResponse> response,
+	                                    std::string bucket,
+	                                    std::string object) {
 
 		TraceEvent("MockS3PutObject_Debug")
 		    .detail("Bucket", bucket)
@@ -1086,7 +1035,7 @@ class MockS3ServerImpl {
 		    .detail("ContentPreview", req->data.content.substr(0, std::min(100, (int)req->data.content.size())));
 
 		ObjectData obj(req->data.content);
-		state std::string etag = obj.etag;
+		std::string etag = obj.etag;
 		getGlobalStorage().buckets[bucket][object] = std::move(obj);
 
 		TraceEvent("MockS3PutObject_Stored")
@@ -1097,7 +1046,7 @@ class MockS3ServerImpl {
 
 		// Persist object to disk
 		if (getGlobalStorage().persistenceEnabled) {
-			wait(persistObject(bucket, object));
+			co_await persistObject(bucket, object);
 		}
 
 		response->code = 200;
@@ -1111,8 +1060,6 @@ class MockS3ServerImpl {
 		    .detail("ResponseCode", response->code)
 		    .detail("ContentLen", response->data.contentLen)
 		    .detail("HasContent", response->data.content != nullptr);
-
-		return Void();
 	}
 
 	static Future<Void> handleGetObject(MockS3ServerImpl* self,
@@ -1205,11 +1152,11 @@ class MockS3ServerImpl {
 		return Void();
 	}
 
-	ACTOR static Future<Void> handleDeleteObject(MockS3ServerImpl* self,
-	                                             Reference<HTTP::IncomingRequest> req,
-	                                             Reference<HTTP::OutgoingResponse> response,
-	                                             std::string bucket,
-	                                             std::string object) {
+	static Future<Void> handleDeleteObject(MockS3ServerImpl* self,
+	                                       Reference<HTTP::IncomingRequest> req,
+	                                       Reference<HTTP::OutgoingResponse> response,
+	                                       std::string bucket,
+	                                       std::string object) {
 
 		TraceEvent("MockS3DeleteObject").detail("Bucket", bucket).detail("Object", object);
 
@@ -1220,7 +1167,7 @@ class MockS3ServerImpl {
 
 		// Delete persisted object
 		if (getGlobalStorage().persistenceEnabled) {
-			wait(deletePersistedObject(bucket, object));
+			co_await deletePersistedObject(bucket, object);
 		}
 
 		response->code = 204; // No Content
@@ -1228,8 +1175,6 @@ class MockS3ServerImpl {
 		response->data.content->discardAll(); // Clear existing content
 
 		TraceEvent("MockS3ObjectDeleted").detail("Bucket", bucket).detail("Object", object);
-
-		return Void();
 	}
 
 	static Future<Void> handleHeadObject(MockS3ServerImpl* self,
@@ -1289,12 +1234,12 @@ class MockS3ServerImpl {
 		TraceEvent("MockS3ListObjects").detail("Bucket", bucket).detail("QueryParamCount", queryParams.size());
 
 		// Get query parameters for listing
-		std::string prefix = queryParams.count("prefix") ? queryParams.at("prefix") : "";
-		std::string delimiter = queryParams.count("delimiter") ? queryParams.at("delimiter") : "";
-		std::string marker = queryParams.count("marker") ? queryParams.at("marker") : "";
+		std::string prefix = queryParams.contains("prefix") ? queryParams.at("prefix") : "";
+		std::string delimiter = queryParams.contains("delimiter") ? queryParams.at("delimiter") : "";
+		std::string marker = queryParams.contains("marker") ? queryParams.at("marker") : "";
 		std::string continuationToken =
-		    queryParams.count("continuation-token") ? queryParams.at("continuation-token") : "";
-		int maxKeys = queryParams.count("max-keys") ? std::stoi(queryParams.at("max-keys")) : 1000;
+		    queryParams.contains("continuation-token") ? queryParams.at("continuation-token") : "";
+		int maxKeys = queryParams.contains("max-keys") ? std::stoi(queryParams.at("max-keys")) : 1000;
 
 		TraceEvent("MockS3ListObjectsDebug")
 		    .detail("Bucket", bucket)
@@ -1470,7 +1415,7 @@ class MockS3ServerImpl {
 		TraceEvent("MockS3SendXMLResponse_Start")
 		    .detail("Code", code)
 		    .detail("XMLSize", xml.size())
-		    .detail("XMLPreview", xml.size() > 0 ? xml.substr(0, std::min((size_t)50, xml.size())) : "EMPTY");
+		    .detail("XMLPreview", !xml.empty() ? xml.substr(0, std::min((size_t)50, xml.size())) : "EMPTY");
 
 		response->code = code;
 		response->data.headers["Content-Type"] = "application/xml";
@@ -1566,8 +1511,8 @@ Reference<HTTP::IRequestHandler> MockS3RequestHandler::clone() {
 }
 
 // Safe server registration that prevents conflicts (internal implementation)
-ACTOR Future<Void> registerMockS3Server_impl(std::string ip, std::string port) {
-	state std::string serverKey = ip + ":" + port; // State variable before any early returns
+Future<Void> registerMockS3Server_impl(std::string ip, std::string port) {
+	std::string serverKey = ip + ":" + port; // State variable before any early returns
 
 	// DIAGNOSTIC: Enhanced registration logging
 	TraceEvent("MockS3ServerDiagnostic")
@@ -1576,12 +1521,12 @@ ACTOR Future<Void> registerMockS3Server_impl(std::string ip, std::string port) {
 	    .detail("Port", port)
 	    .detail("ServerKey", serverKey)
 	    .detail("IsSimulated", g_network->isSimulated())
-	    .detail("AlreadyRegistered", registeredServers.count(serverKey) > 0);
+	    .detail("AlreadyRegistered", registeredServers.contains(serverKey));
 
 	// Check if server is already registered
-	if (registeredServers.count(serverKey)) {
+	if (registeredServers.contains(serverKey)) {
 		TraceEvent(SevWarn, "MockS3ServerAlreadyRegistered").detail("Address", serverKey);
-		return Void();
+		co_return;
 	}
 
 	try {
@@ -1595,14 +1540,14 @@ ACTOR Future<Void> registerMockS3Server_impl(std::string ip, std::string port) {
 			    .detail("PersistenceDir", persistenceDir);
 
 			// Load any previously persisted state (for crash recovery in simulation)
-			wait(loadMockS3PersistedStateFuture());
+			co_await loadMockS3PersistedStateFuture();
 		}
 
 		TraceEvent("MockS3ServerDiagnostic")
 		    .detail("Phase", "Calling registerSimHTTPServer")
 		    .detail("Address", serverKey);
 
-		wait(g_simulator->registerSimHTTPServer(ip, port, makeReference<MockS3RequestHandler>()));
+		co_await g_simulator->registerSimHTTPServer(ip, port, makeReference<MockS3RequestHandler>());
 		registeredServers[serverKey] = true;
 
 		TraceEvent("MockS3ServerRegistered").detail("Address", serverKey).detail("Success", true);
@@ -1619,12 +1564,10 @@ ACTOR Future<Void> registerMockS3Server_impl(std::string ip, std::string port) {
 		    .detail("ErrorName", e.name());
 		throw;
 	}
-
-	return Void();
 }
 
 // Public Interface Implementation
-ACTOR Future<Void> startMockS3Server(NetworkAddress listenAddress) {
+Future<Void> startMockS3Server(NetworkAddress listenAddress) {
 	TraceEvent("MockS3ServerStarting").detail("ListenAddress", listenAddress.toString());
 
 	try {
@@ -1634,7 +1577,7 @@ ACTOR Future<Void> startMockS3Server(NetworkAddress listenAddress) {
 		    .detail("IsSimulated", g_network->isSimulated());
 
 		// Persistence is automatically enabled in registerMockS3Server_impl()
-		wait(registerMockS3Server_impl(listenAddress.ip.toString(), std::to_string(listenAddress.port)));
+		co_await registerMockS3Server_impl(listenAddress.ip.toString(), std::to_string(listenAddress.port));
 
 		TraceEvent("MockS3ServerStarted")
 		    .detail("ListenAddress", listenAddress.toString())
@@ -1644,8 +1587,6 @@ ACTOR Future<Void> startMockS3Server(NetworkAddress listenAddress) {
 		TraceEvent(SevError, "MockS3ServerStartError").error(e).detail("ListenAddress", listenAddress.toString());
 		throw;
 	}
-
-	return Void();
 }
 
 // Clear all MockS3 global storage - called at the start of each simulation test
@@ -1667,45 +1608,45 @@ bool isMockS3PersistenceEnabled() {
 }
 
 // ACTOR: Load persisted objects from disk
-ACTOR static Future<Void> loadPersistedObjects(std::string persistenceDir) {
-	state std::string objectsDir = persistenceDir + "/objects"; // State variable before any early returns
+static Future<Void> loadPersistedObjects(std::string persistenceDir) {
+	std::string objectsDir = persistenceDir + "/objects"; // State variable before any early returns
 
 	if (!fileExists(objectsDir)) {
 		TraceEvent("MockS3LoadObjects").detail("Status", "NoObjectsDir");
-		return Void();
+		co_return;
 	}
 
 	try {
 		// Get list of bucket directories
-		state std::vector<std::string> buckets = platform::listFiles(objectsDir, "");
+		std::vector<std::string> buckets = platform::listFiles(objectsDir, "");
 		// Sort for deterministic load order (platform::listFiles returns OS-dependent order)
 		std::sort(buckets.begin(), buckets.end());
-		state int bucketIdx = 0;
+		int bucketIdx = 0;
 
 		for (bucketIdx = 0; bucketIdx < buckets.size(); bucketIdx++) {
-			state std::string bucket = buckets[bucketIdx];
+			std::string bucket = buckets[bucketIdx];
 			if (bucket == "." || bucket == "..")
 				continue;
 
-			state std::string bucketDir = objectsDir + "/" + bucket;
+			std::string bucketDir = objectsDir + "/" + bucket;
 			if (!directoryExists(bucketDir))
 				continue;
 
 			// Get all files in the bucket directory (including nested paths)
-			state std::vector<std::string> files = platform::listFiles(bucketDir, "");
+			std::vector<std::string> files = platform::listFiles(bucketDir, "");
 			std::sort(files.begin(), files.end()); // Deterministic order
-			state int fileIdx = 0;
+			int fileIdx = 0;
 
 			for (fileIdx = 0; fileIdx < files.size(); fileIdx++) {
-				state std::string fileName = files[fileIdx];
+				std::string fileName = files[fileIdx];
 
 				// Look for .meta.json files to identify objects
 				if (fileName.size() > OBJECT_META_SUFFIX_LEN &&
 				    fileName.substr(fileName.size() - OBJECT_META_SUFFIX_LEN) == OBJECT_META_SUFFIX) {
 					// Extract object name by removing .meta.json suffix
-					state std::string objectName = fileName.substr(0, fileName.size() - OBJECT_META_SUFFIX_LEN);
-					state std::string dataPath = bucketDir + "/" + objectName + OBJECT_DATA_SUFFIX;
-					state std::string metaPath = bucketDir + "/" + fileName;
+					std::string objectName = fileName.substr(0, fileName.size() - OBJECT_META_SUFFIX_LEN);
+					std::string dataPath = bucketDir + "/" + objectName + OBJECT_DATA_SUFFIX;
+					std::string metaPath = bucketDir + "/" + fileName;
 
 					if (!fileExists(dataPath)) {
 						TraceEvent(SevWarn, "MockS3LoadObjectSkipped")
@@ -1716,8 +1657,8 @@ ACTOR static Future<Void> loadPersistedObjects(std::string persistenceDir) {
 					}
 
 					// Read object content and metadata
-					state std::string content = wait(readFileContent(dataPath));
-					state std::string metaJson = wait(readFileContent(metaPath));
+					std::string content = co_await readFileContent(dataPath);
+					std::string metaJson = co_await readFileContent(metaPath);
 
 					// Parse metadata using rapidjson
 					MockS3GlobalStorage::ObjectData obj(content);
@@ -1736,36 +1677,34 @@ ACTOR static Future<Void> loadPersistedObjects(std::string persistenceDir) {
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "MockS3LoadObjectsFailed").error(e);
 	}
-
-	return Void();
 }
 
 // ACTOR: Load persisted multipart uploads from disk
-ACTOR static Future<Void> loadPersistedMultipartUploads(std::string persistenceDir) {
-	state std::string multipartDir = persistenceDir + "/multipart"; // State variable before any early returns
+static Future<Void> loadPersistedMultipartUploads(std::string persistenceDir) {
+	std::string multipartDir = persistenceDir + "/multipart"; // State variable before any early returns
 
 	if (!fileExists(multipartDir)) {
 		TraceEvent("MockS3LoadMultipart").detail("Status", "NoMultipartDir");
-		return Void();
+		co_return;
 	}
 
 	try {
 		// Get all files in multipart directory
-		state std::vector<std::string> files = platform::listFiles(multipartDir, "");
+		std::vector<std::string> files = platform::listFiles(multipartDir, "");
 		std::sort(files.begin(), files.end()); // Deterministic order
-		state int fileIdx = 0;
+		int fileIdx = 0;
 
 		for (fileIdx = 0; fileIdx < files.size(); fileIdx++) {
-			state std::string fileName = files[fileIdx];
+			std::string fileName = files[fileIdx];
 
 			// Look for .state.json files
 			if (fileName.size() > MULTIPART_STATE_SUFFIX_LEN &&
 			    fileName.substr(fileName.size() - MULTIPART_STATE_SUFFIX_LEN) == MULTIPART_STATE_SUFFIX) {
-				state std::string uploadId = fileName.substr(0, fileName.size() - MULTIPART_STATE_SUFFIX_LEN);
-				state std::string statePath = multipartDir + "/" + fileName;
+				std::string uploadId = fileName.substr(0, fileName.size() - MULTIPART_STATE_SUFFIX_LEN);
+				std::string statePath = multipartDir + "/" + fileName;
 
 				// Read state file
-				state std::string stateJson = wait(readFileContent(statePath));
+				std::string stateJson = co_await readFileContent(statePath);
 				if (stateJson.empty()) {
 					TraceEvent(SevWarn, "MockS3LoadMultipartSkipped")
 					    .detail("UploadId", uploadId)
@@ -1774,23 +1713,23 @@ ACTOR static Future<Void> loadPersistedMultipartUploads(std::string persistenceD
 				}
 
 				// Parse multipart upload state using rapidjson
-				state MockS3GlobalStorage::MultipartUpload upload("", "");
+				MockS3GlobalStorage::MultipartUpload upload("", "");
 				upload.uploadId = uploadId;
 				deserializeMultipartState(stateJson, upload);
 
 				// Load all parts for this upload
-				state int partNum = 1;
-				state int maxAttempts = 10000; // Reasonable limit
+				int partNum = 1;
+				int maxAttempts = 10000; // Reasonable limit
 				for (partNum = 1; partNum <= maxAttempts; partNum++) {
-					state std::string partPath = multipartDir + "/" + uploadId + ".part." + std::to_string(partNum);
-					state std::string partMetaPath = partPath + ".meta.json";
+					std::string partPath = multipartDir + "/" + uploadId + ".part." + std::to_string(partNum);
+					std::string partMetaPath = partPath + ".meta.json";
 
 					if (!fileExists(partPath) || !fileExists(partMetaPath))
 						break; // No more parts
 
 					// Read part data and metadata
-					state std::string partData = wait(readFileContent(partPath));
-					state std::string partMetaJson = wait(readFileContent(partMetaPath));
+					std::string partData = co_await readFileContent(partPath);
+					std::string partMetaJson = co_await readFileContent(partMetaPath);
 
 					// Parse part metadata using rapidjson
 					using namespace rapidjson;
@@ -1820,16 +1759,14 @@ ACTOR static Future<Void> loadPersistedMultipartUploads(std::string persistenceD
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "MockS3LoadMultipartFailed").error(e);
 	}
-
-	return Void();
 }
 
 // ACTOR: Load all persisted state from disk
-ACTOR static Future<Void> loadMockS3PersistedStateImpl() {
-	state std::string persistenceDir; // Declare state before any early returns
+static Future<Void> loadMockS3PersistedStateImpl() {
+	std::string persistenceDir; // Declare state before any early returns
 
 	if (!getGlobalStorage().persistenceEnabled || getGlobalStorage().persistenceLoaded) {
-		return Void();
+		co_return;
 	}
 
 	persistenceDir = getGlobalStorage().persistenceDir;
@@ -1837,10 +1774,10 @@ ACTOR static Future<Void> loadMockS3PersistedStateImpl() {
 
 	try {
 		// Load objects
-		wait(loadPersistedObjects(persistenceDir));
+		co_await loadPersistedObjects(persistenceDir);
 
 		// Load multipart uploads
-		wait(loadPersistedMultipartUploads(persistenceDir));
+		co_await loadPersistedMultipartUploads(persistenceDir);
 
 		getGlobalStorage().persistenceLoaded = true;
 
@@ -1851,8 +1788,6 @@ ACTOR static Future<Void> loadMockS3PersistedStateImpl() {
 		TraceEvent(SevError, "MockS3LoadPersistedStateFailed").error(e);
 		throw;
 	}
-
-	return Void();
 }
 
 // Load persisted state from disk (called at server startup) - returns Future for use in ACTOR context
@@ -1864,7 +1799,7 @@ Future<Void> loadMockS3PersistedStateFuture() {
 }
 
 // Initialize MockS3 persistence for simulation tests (exported for MockS3ServerChaos)
-ACTOR Future<Void> initializeMockS3Persistence(std::string serverKey) {
+Future<Void> initializeMockS3Persistence(std::string serverKey) {
 	if (!getGlobalStorage().persistenceEnabled) {
 		enableMockS3Persistence(DEFAULT_MOCKS3_PERSISTENCE_DIR);
 		TraceEvent("MockS3ServerPersistenceEnabled")
@@ -1872,9 +1807,8 @@ ACTOR Future<Void> initializeMockS3Persistence(std::string serverKey) {
 		    .detail("PersistenceDir", DEFAULT_MOCKS3_PERSISTENCE_DIR);
 
 		// Load any previously persisted state (for crash recovery in simulation)
-		wait(loadMockS3PersistedStateFuture());
+		co_await loadMockS3PersistedStateFuture();
 	}
-	return Void();
 }
 
 // Unit Tests for MockS3Server
@@ -1888,7 +1822,7 @@ TEST_CASE("/MockS3Server/parseS3Request/ValidBucketParameter") {
 	server.parseS3Request(resource, bucket, object, queryParams);
 
 	ASSERT(bucket == "testbucket");
-	ASSERT(object == "");
+	ASSERT(object.empty());
 	ASSERT(queryParams["region"] == "us-east-1");
 
 	return Void();
@@ -1986,7 +1920,7 @@ TEST_CASE("/MockS3Server/parseS3Request/EmptyPath") {
 	server.parseS3Request(resource, bucket, object, queryParams);
 
 	ASSERT(bucket == "testbucket");
-	ASSERT(object == "");
+	ASSERT(object.empty());
 	ASSERT(queryParams["region"] == "us-east-1");
 
 	return Void();
@@ -2002,7 +1936,7 @@ TEST_CASE("/MockS3Server/parseS3Request/OnlyBucketInPath") {
 	server.parseS3Request(resource, bucket, object, queryParams);
 
 	ASSERT(bucket == "testbucket"); // Should use path (like real S3)
-	ASSERT(object == "");
+	ASSERT(object.empty());
 	ASSERT(queryParams["region"] == "us-east-1");
 
 	return Void();
@@ -2036,7 +1970,7 @@ TEST_CASE("/MockS3Server/parseS3Request/ParametersWithoutValues") {
 	server.parseS3Request(resource, bucket, object, queryParams);
 
 	ASSERT(bucket == "testbucket");
-	ASSERT(queryParams["flag"] == ""); // Parameter without value should be empty string
+	ASSERT(queryParams["flag"].empty()); // Parameter without value should be empty string
 	ASSERT(queryParams["region"] == "us-east-1");
 
 	return Void();
@@ -2152,7 +2086,7 @@ TEST_CASE("/MockS3Server/RangeHeader/StartGreaterThanEnd") {
 }
 
 // Real HTTP Server Implementation for ctests
-ACTOR Future<Void> startMockS3ServerReal_impl(NetworkAddress listenAddress, std::string persistenceDir) {
+Future<Void> startMockS3ServerReal_impl(NetworkAddress listenAddress, std::string persistenceDir) {
 	TraceEvent("MockS3ServerRealStarting").detail("ListenAddress", listenAddress.toString());
 
 	// Enable persistence for standalone MockS3Server
@@ -2167,10 +2101,10 @@ ACTOR Future<Void> startMockS3ServerReal_impl(NetworkAddress listenAddress, std:
 		    .detail("PersistenceDir", persistenceDir);
 
 		// Load any previously persisted state (for crash recovery)
-		wait(loadMockS3PersistedStateFuture());
+		co_await loadMockS3PersistedStateFuture();
 	}
 
-	state Reference<HTTP::SimServerContext> server = makeReference<HTTP::SimServerContext>();
+	auto server = makeReference<HTTP::SimServerContext>();
 	server->registerNewServer(listenAddress, makeReference<MockS3RequestHandler>());
 
 	TraceEvent("MockS3ServerRealStarted")
@@ -2178,8 +2112,7 @@ ACTOR Future<Void> startMockS3ServerReal_impl(NetworkAddress listenAddress, std:
 	    .detail("ServerPtr", format("%p", server.getPtr()));
 
 	// Keep the server running indefinitely
-	wait(Never());
-	return Void();
+	co_await Future<Void>(Never());
 }
 
 Future<Void> startMockS3ServerReal(const NetworkAddress& listenAddress, const std::string& persistenceDir) {
@@ -2190,3 +2123,5 @@ Future<Void> startMockS3ServerReal(const NetworkAddress& listenAddress, const st
 Future<Void> registerMockS3Server(std::string ip, std::string port) {
 	return registerMockS3Server_impl(ip, port);
 }
+
+void forceLinkMockS3ServerTests() {}
diff --git a/fdbserver/MockS3ServerChaos.actor.cpp b/fdbserver/mocks3/MockS3ServerChaos.cpp
similarity index 83%
rename from fdbserver/MockS3ServerChaos.actor.cpp
rename to fdbserver/mocks3/MockS3ServerChaos.cpp
index 463d0a52374..1eaac2c86d3 100644
--- a/fdbserver/MockS3ServerChaos.actor.cpp
+++ b/fdbserver/mocks3/MockS3ServerChaos.cpp
@@ -1,5 +1,5 @@
 /*
- * MockS3ServerChaos.actor.cpp
+ * MockS3ServerChaos.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,23 @@
 
 // Design: design/mocks3server_chaos_design.md
 
-#include "fdbserver/MockS3ServerChaos.h"
-#include "fdbserver/MockS3Server.h"
+#include <set>
+
+#include "fdbserver/mocks3/MockS3ServerChaos.h"
+#include "fdbserver/mocks3/MockS3Server.h"
 #include "flow/ChaosMetrics.h"
 #include "fdbrpc/simulator.h"
 #include "flow/Trace.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h"
+
+namespace {
+
+std::set<std::string>& registeredMockS3ChaosServers() {
+	static std::set<std::string> servers;
+	return servers;
+}
+
+} // namespace
 
 // Clear the chaos server registry (for testing/debugging only)
 // NOTE: In production simulation tests, the registry should NOT be cleared between tests,
@@ -34,8 +44,8 @@
 // duplicate registration attempts that would trigger assertions.
 void clearMockS3ChaosRegistry() {
 	if (g_network && g_simulator) {
-		size_t previousSize = g_simulator->registeredMockS3ChaosServers.size();
-		g_simulator->registeredMockS3ChaosServers.clear();
+		size_t previousSize = registeredMockS3ChaosServers().size();
+		registeredMockS3ChaosServers().clear();
 		TraceEvent("MockS3ChaosRegistryCleared").detail("PreviousSize", previousSize);
 	}
 }
@@ -96,7 +106,7 @@ std::string generateS3ErrorXML(const std::string& code, const std::string& messa
 }
 
 // Phase 1: Inject delay if configured
-ACTOR Future<Void> maybeInjectDelay(S3Operation op) {
+Future<Void> maybeInjectDelay(S3Operation op) {
 	auto injector = S3FaultInjector::injector();
 	double delayRate = injector->getDelayRate() * getOperationMultiplier(op);
 
@@ -109,16 +119,14 @@ ACTOR Future<Void> maybeInjectDelay(S3Operation op) {
 		    .detail("Delay", delayTime)
 		    .detail("MaxDelay", maxDelay);
 
-		wait(delay(delayTime));
+		co_await delay(delayTime);
 	}
-
-	return Void();
 }
 
 // Phase 2: Inject errors if configured
-ACTOR Future<bool> maybeInjectError(Reference<HTTP::IncomingRequest> req,
-                                    Reference<HTTP::OutgoingResponse> response,
-                                    S3Operation op) {
+Future<bool> maybeInjectError(Reference<HTTP::IncomingRequest> req,
+                              Reference<HTTP::OutgoingResponse> response,
+                              S3Operation op) {
 	auto injector = S3FaultInjector::injector();
 	double errorRate = injector->getErrorRate() * getOperationMultiplier(op);
 
@@ -218,7 +226,7 @@ ACTOR Future<bool> maybeInjectError(Reference<HTTP::IncomingRequest> req,
 }
 
 // Phase 4: Corrupt response data if configured
-ACTOR Future<Void> maybeCorruptResponse(Reference<HTTP::OutgoingResponse> response, S3Operation op) {
+Future<Void> maybeCorruptResponse(Reference<HTTP::OutgoingResponse> response, S3Operation op) {
 	auto injector = S3FaultInjector::injector();
 	double corruptionRate = injector->getCorruptionRate() * getOperationMultiplier(op);
 
@@ -248,10 +256,10 @@ ACTOR Future<Void> maybeCorruptResponse(Reference<HTTP::OutgoingResponse> respon
 // Core chaos server implementation
 class MockS3ChaosServerImpl {
 public:
-	ACTOR static Future<Void> handleRequest(Reference<HTTP::IncomingRequest> req,
-	                                        Reference<HTTP::OutgoingResponse> response) {
-		// Classify the operation type (must be state since used after wait())
-		state S3Operation op = classifyS3Operation(req->verb, req->resource);
+	static Future<Void> handleRequest(Reference<HTTP::IncomingRequest> req,
+	                                  Reference<HTTP::OutgoingResponse> response) {
+		// Classify the operation type before the first suspension point.
+		S3Operation op = classifyS3Operation(req->verb, req->resource);
 
 		TraceEvent("MockS3ChaosRequest")
 		    .detail("Method", req->verb)
@@ -260,22 +268,22 @@ class MockS3ChaosServerImpl {
 		    .detail("ContentLength", req->data.contentLen);
 
 		// Phase 1: Inject delay if configured
-		wait(maybeInjectDelay(op));
+		co_await maybeInjectDelay(op);
 
 		// Phase 2: Check if we should inject an error
-		state bool errorInjected = wait(maybeInjectError(req, response, op));
+		bool errorInjected = co_await maybeInjectError(req, response, op);
 		if (errorInjected) {
-			return Void();
+			co_return;
 		}
 
 		// Phase 3: Delegate to base MockS3Server for normal processing
 		// Use the exposed processMockS3Request function to access the shared server instance
-		wait(processMockS3Request(req, response));
+		co_await processMockS3Request(req, response);
 
 		// Phase 4: Possibly corrupt the response data
-		wait(maybeCorruptResponse(response, op));
+		co_await maybeCorruptResponse(response, op);
 
-		return Void();
+		co_return;
 	}
 };
 
@@ -290,8 +298,8 @@ Reference<HTTP::IRequestHandler> MockS3ChaosRequestHandler::clone() {
 }
 
 // Safe server registration that prevents conflicts using truly simulator-global registry
-ACTOR Future<Void> registerMockS3ChaosServer(std::string ip, std::string port) {
-	state std::string serverKey = ip + ":" + port;
+Future<Void> registerMockS3ChaosServer(std::string ip, std::string port) {
+	std::string serverKey = ip + ":" + port;
 	ASSERT(g_simulator);
 
 	TraceEvent("MockS3ChaosServerRegistration")
@@ -300,12 +308,12 @@ ACTOR Future<Void> registerMockS3ChaosServer(std::string ip, std::string port) {
 	    .detail("Port", port)
 	    .detail("ServerKey", serverKey)
 	    .detail("IsSimulated", g_network->isSimulated())
-	    .detail("AlreadyRegistered", g_simulator->registeredMockS3ChaosServers.count(serverKey) > 0);
+	    .detail("AlreadyRegistered", registeredMockS3ChaosServers().contains(serverKey));
 
 	// Check if server is already registered using truly simulator-global registry
-	if (g_simulator->registeredMockS3ChaosServers.count(serverKey)) {
+	if (registeredMockS3ChaosServers().contains(serverKey)) {
 		TraceEvent(SevWarn, "MockS3ChaosServerAlreadyRegistered").detail("Address", serverKey);
-		return Void();
+		co_return;
 	}
 
 	try {
@@ -319,19 +327,19 @@ ACTOR Future<Void> registerMockS3ChaosServer(std::string ip, std::string port) {
 			    .detail("PersistenceDir", persistenceDir);
 
 			// Load any previously persisted state (for crash recovery in simulation)
-			wait(loadMockS3PersistedStateFuture());
+			co_await loadMockS3PersistedStateFuture();
 		}
 
-		wait(g_simulator->registerSimHTTPServer(ip, port, makeReference<MockS3ChaosRequestHandler>()));
-		g_simulator->registeredMockS3ChaosServers.insert(serverKey);
+		co_await g_simulator->registerSimHTTPServer(ip, port, makeReference<MockS3ChaosRequestHandler>());
+		registeredMockS3ChaosServers().insert(serverKey);
 
 		// Enable persistence automatically for all MockS3 instances (including chaos)
-		wait(initializeMockS3Persistence(serverKey));
+		co_await initializeMockS3Persistence(serverKey);
 
 		TraceEvent("MockS3ChaosServerRegistered")
 		    .detail("Address", serverKey)
 		    .detail("Success", true)
-		    .detail("TotalRegistered", g_simulator->registeredMockS3ChaosServers.size());
+		    .detail("TotalRegistered", registeredMockS3ChaosServers().size());
 
 	} catch (Error& e) {
 		TraceEvent(SevError, "MockS3ChaosServerRegistrationFailed")
@@ -341,12 +349,10 @@ ACTOR Future<Void> registerMockS3ChaosServer(std::string ip, std::string port) {
 		    .detail("ErrorName", e.name());
 		throw;
 	}
-
-	return Void();
 }
 
 // Public Interface Implementation
-ACTOR Future<Void> startMockS3ServerChaos(NetworkAddress listenAddress) {
+Future<Void> startMockS3ServerChaos(NetworkAddress listenAddress) {
 	ASSERT(g_network->isSimulated());
 
 	TraceEvent("MockS3ChaosServerStart")
@@ -355,11 +361,7 @@ ACTOR Future<Void> startMockS3ServerChaos(NetworkAddress listenAddress) {
 	    .detail("Port", listenAddress.port);
 
 	// Register the chaos-enabled HTTP server
-	wait(registerMockS3ChaosServer(listenAddress.ip.toString(), std::to_string(listenAddress.port)));
+	co_await registerMockS3ChaosServer(listenAddress.ip.toString(), std::to_string(listenAddress.port));
 
 	TraceEvent("MockS3ChaosServerStarted").detail("Address", listenAddress.toString()).detail("Ready", true);
-
-	return Void();
 }
-
-#include "flow/unactorcompiler.h"
\ No newline at end of file
diff --git a/fdbserver/include/fdbserver/MockS3Server.h b/fdbserver/mocks3/include/fdbserver/mocks3/MockS3Server.h
similarity index 98%
rename from fdbserver/include/fdbserver/MockS3Server.h
rename to fdbserver/mocks3/include/fdbserver/mocks3/MockS3Server.h
index fa02851c658..592bad89c24 100644
--- a/fdbserver/include/fdbserver/MockS3Server.h
+++ b/fdbserver/mocks3/include/fdbserver/mocks3/MockS3Server.h
@@ -82,7 +82,7 @@ Future<Void> loadMockS3PersistedStateFuture();
 // Initialize MockS3 persistence for a specific server (simulation)
 // This combines enabling persistence and loading any previously persisted state
 // serverKey: Identifier for the server (e.g., "127.0.0.1:8080")
-Future<Void> initializeMockS3Persistence(std::string const& serverKey);
+Future<Void> initializeMockS3Persistence(std::string serverKey);
 
 // Process a Mock S3 request directly (for wrapping/chaos injection)
 // This is the low-level request processor used by MockS3RequestHandler
diff --git a/fdbserver/include/fdbserver/MockS3ServerChaos.h b/fdbserver/mocks3/include/fdbserver/mocks3/MockS3ServerChaos.h
similarity index 98%
rename from fdbserver/include/fdbserver/MockS3ServerChaos.h
rename to fdbserver/mocks3/include/fdbserver/mocks3/MockS3ServerChaos.h
index 7f25c8e9910..989ab5d2708 100644
--- a/fdbserver/include/fdbserver/MockS3ServerChaos.h
+++ b/fdbserver/mocks3/include/fdbserver/mocks3/MockS3ServerChaos.h
@@ -105,7 +105,7 @@ class MockS3ChaosRequestHandler : public HTTP::IRequestHandler, public Reference
 };
 
 // Public interface
-Future<Void> startMockS3ServerChaos(const NetworkAddress& listenAddress);
+Future<Void> startMockS3ServerChaos(NetworkAddress listenAddress);
 
 // Clear the chaos server registry (for testing/debugging only)
 // NOTE: In production simulation tests, the registry should NOT be cleared between tests,
diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.cpp
similarity index 65%
rename from fdbserver/networktest.actor.cpp
rename to fdbserver/networktest.cpp
index 93d0991268f..5cc52893a50 100644
--- a/fdbserver/networktest.actor.cpp
+++ b/fdbserver/networktest.cpp
@@ -1,5 +1,5 @@
 /*
- * networktest.actor.cpp
+ * networktest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,13 @@
 
 #include "fmt/format.h"
 #include "fdbserver/NetworkTest.h"
-#include "flow/Knobs.h"
 #include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
+#include "flow/Knobs.h"
 #include "flow/UnitTest.h"
 #include <inttypes.h>
 
 #include "flow/IConnection.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 constexpr int WLTOKEN_NETWORKTEST = WLTOKEN_FIRST_AVAILABLE;
 
@@ -64,77 +64,102 @@ NetworkTestInterface::NetworkTestInterface(INetwork* local) {
 	test.makeWellKnownEndpoint(WLTOKEN_NETWORKTEST, TaskPriority::DefaultEndpoint);
 }
 
-ACTOR Future<Void> networkTestServer() {
-	state NetworkTestInterface interf(g_network);
-	state Future<Void> logging = delay(1.0);
-	state double lastTime = now();
-	state int sent = 0;
-	state LatencyStats latency;
+class NetworkTestServer {
+public:
+	NetworkTestServer() : interf(g_network) {}
 
-	loop {
-		choose {
-			when(NetworkTestRequest req = waitNext(interf.test.getFuture())) {
-				LatencyStats::sample sample = latency.tick();
-				req.reply.send(NetworkTestReply(Value(std::string(req.replySize, '.'))));
-				latency.tock(sample);
-				sent++;
-			}
-			when(wait(logging)) {
-				auto spd = sent / (now() - lastTime);
-				if (FLOW_KNOBS->NETWORK_TEST_SCRIPT_MODE) {
-					fprintf(stderr, "%f\t%.3f\t%.3f\n", spd, latency.mean() * 1e6, latency.stddev() * 1e6);
-				} else {
-					fprintf(stderr, "responses per second: %f (%f us)\n", spd, latency.mean() * 1e6);
-				}
-				latency.reset();
-				lastTime = now();
-				sent = 0;
-				logging = delay(1.0);
+	Future<Void> run() { co_await race(requests(), logging()); }
+
+private:
+	Future<Void> requests() {
+		while (true) {
+			NetworkTestRequest req = co_await interf.test.getFuture();
+			LatencyStats::sample sample = latency.tick();
+			req.reply.send(NetworkTestReply(Value(std::string(req.replySize, '.'))));
+			latency.tock(sample);
+			sent++;
+		}
+	}
+
+	Future<Void> logging() {
+		double lastTime = now();
+
+		while (true) {
+			co_await delay(1.0);
+			auto spd = sent / (now() - lastTime);
+			if (FLOW_KNOBS->NETWORK_TEST_SCRIPT_MODE) {
+				fprintf(stderr, "%f\t%.3f\t%.3f\n", spd, latency.mean() * 1e6, latency.stddev() * 1e6);
+			} else {
+				fprintf(stderr, "responses per second: %f (%f us)\n", spd, latency.mean() * 1e6);
 			}
+			latency.reset();
+			lastTime = now();
+			sent = 0;
 		}
 	}
+
+	NetworkTestInterface interf;
+	int sent = 0;
+	LatencyStats latency;
+};
+
+Future<Void> networkTestServer() {
+	NetworkTestServer server;
+	co_await server.run();
 }
 
-ACTOR Future<Void> networkTestStreamingServer() {
-	state NetworkTestInterface interf(g_network);
-	state Future<Void> logging = delay(1.0);
-	state double lastTime = now();
-	state int sent = 0;
-	state LatencyStats latency;
+class NetworkTestStreamingServer {
+public:
+	NetworkTestStreamingServer() : interf(g_network) {}
 
-	loop {
-		try {
-			choose {
-				when(state NetworkTestStreamingRequest req = waitNext(interf.testStream.getFuture())) {
-					state LatencyStats::sample sample = latency.tick();
-					state int i = 0;
-					for (; i < 100; ++i) {
-						wait(req.reply.onReady());
-						req.reply.send(NetworkTestStreamingReply{ i });
-					}
-					req.reply.sendError(end_of_stream());
-					latency.tock(sample);
-					sent++;
+	Future<Void> run() { co_await race(requests(), logging()); }
+
+private:
+	Future<Void> requests() {
+		while (true) {
+			try {
+				NetworkTestStreamingRequest req = co_await interf.testStream.getFuture();
+				LatencyStats::sample sample = latency.tick();
+				for (int i = 0; i < 100; ++i) {
+					co_await req.reply.onReady();
+					req.reply.send(NetworkTestStreamingReply{ i });
 				}
-				when(wait(logging)) {
-					auto spd = sent / (now() - lastTime);
-					if (FLOW_KNOBS->NETWORK_TEST_SCRIPT_MODE) {
-						fprintf(stderr, "%f\t%.3f\t%.3f\n", spd, latency.mean() * 1e6, latency.stddev() * 1e6);
-					} else {
-						fprintf(stderr, "responses per second: %f (%f us)\n", spd, latency.mean() * 1e6);
-					}
-					latency.reset();
-					lastTime = now();
-					sent = 0;
-					logging = delay(1.0);
+				req.reply.sendError(end_of_stream());
+				latency.tock(sample);
+				sent++;
+			} catch (Error& e) {
+				if (e.code() != error_code_operation_obsolete) {
+					throw e;
 				}
 			}
-		} catch (Error& e) {
-			if (e.code() != error_code_operation_obsolete) {
-				throw e;
+		}
+	}
+
+	Future<Void> logging() {
+		double lastTime = now();
+
+		while (true) {
+			co_await delay(1.0);
+			auto spd = sent / (now() - lastTime);
+			if (FLOW_KNOBS->NETWORK_TEST_SCRIPT_MODE) {
+				fprintf(stderr, "%f\t%.3f\t%.3f\n", spd, latency.mean() * 1e6, latency.stddev() * 1e6);
+			} else {
+				fprintf(stderr, "responses per second: %f (%f us)\n", spd, latency.mean() * 1e6);
 			}
+			latency.reset();
+			lastTime = now();
+			sent = 0;
 		}
 	}
+
+	NetworkTestInterface interf;
+	int sent = 0;
+	LatencyStats latency;
+};
+
+Future<Void> networkTestStreamingServer() {
+	NetworkTestStreamingServer server;
+	co_await server.run();
 }
 
 static bool moreRequestsPending(int count) {
@@ -154,42 +179,34 @@ static bool moreLoggingNeeded(int count, int iteration) {
 	}
 }
 
-ACTOR Future<Void> testClient(std::vector<NetworkTestInterface> interfs,
-                              int* sent,
-                              int* completed,
-                              LatencyStats* latency) {
-	state std::string request_payload(FLOW_KNOBS->NETWORK_TEST_REQUEST_SIZE, '.');
-	state LatencyStats::sample sample;
+Future<Void> testClient(std::vector<NetworkTestInterface> interfs, int* sent, int* completed, LatencyStats* latency) {
+	std::string request_payload(FLOW_KNOBS->NETWORK_TEST_REQUEST_SIZE, '.');
 
 	while (moreRequestsPending(*sent)) {
 		(*sent)++;
-		sample = latency->tick();
-		NetworkTestReply rep = wait(
-		    retryBrokenPromise(interfs[deterministicRandom()->randomInt(0, interfs.size())].test,
-		                       NetworkTestRequest(StringRef(request_payload), FLOW_KNOBS->NETWORK_TEST_REPLY_SIZE)));
+		LatencyStats::sample sample = latency->tick();
+		co_await retryBrokenPromise(
+		    interfs[deterministicRandom()->randomInt(0, interfs.size())].test,
+		    NetworkTestRequest(StringRef(request_payload), FLOW_KNOBS->NETWORK_TEST_REPLY_SIZE));
 		latency->tock(sample);
 		(*completed)++;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> testClientStream(std::vector<NetworkTestInterface> interfs,
-                                    int* sent,
-                                    int* completed,
-                                    LatencyStats* latency) {
-	state std::string request_payload(FLOW_KNOBS->NETWORK_TEST_REQUEST_SIZE, '.');
-	state LatencyStats::sample sample;
-
+Future<Void> testClientStream(std::vector<NetworkTestInterface> interfs,
+                              int* sent,
+                              int* completed,
+                              LatencyStats* latency) {
 	while (moreRequestsPending(*sent)) {
 		(*sent)++;
-		sample = latency->tick();
-		state ReplyPromiseStream<NetworkTestStreamingReply> stream =
+		LatencyStats::sample sample = latency->tick();
+		ReplyPromiseStream<NetworkTestStreamingReply> stream =
 		    interfs[deterministicRandom()->randomInt(0, interfs.size())].testStream.getReplyStream(
 		        NetworkTestStreamingRequest{});
-		state int j = 0;
+		int j = 0;
 		try {
-			loop {
-				NetworkTestStreamingReply rep = waitNext(stream.getFuture());
+			while (true) {
+				NetworkTestStreamingReply rep = co_await stream.getFuture();
 				ASSERT(rep.index == j++);
 			}
 		} catch (Error& e) {
@@ -199,15 +216,14 @@ ACTOR Future<Void> testClientStream(std::vector<NetworkTestInterface> interfs,
 		latency->tock(sample);
 		(*completed)++;
 	}
-	return Void();
 }
 
-ACTOR Future<Void> logger(int* sent, int* completed, LatencyStats* latency) {
-	state double lastTime = now();
-	state int logged = 0;
-	state int iteration = 0;
+Future<Void> logger(int* sent, int* completed, LatencyStats* latency) {
+	double lastTime = now();
+	int logged = 0;
+	int iteration = 0;
 	while (moreLoggingNeeded(logged, ++iteration)) {
-		wait(delay(1.0));
+		co_await delay(1.0);
 		auto spd = (*completed - logged) / (now() - lastTime);
 		if (FLOW_KNOBS->NETWORK_TEST_SCRIPT_MODE) {
 			if (iteration == 2) {
@@ -223,7 +239,6 @@ ACTOR Future<Void> logger(int* sent, int* completed, LatencyStats* latency) {
 	}
 	// tell the clients to shut down
 	*sent = -1;
-	return Void();
 }
 
 static void networkTestnanosleep() {
@@ -283,50 +298,49 @@ static void networkTestnanosleep() {
 	return;
 }
 
-ACTOR Future<Void> networkTestClient(std::string testServers) {
+Future<Void> networkTestClient(std::string const& testServers) {
 	if (testServers == "nanosleep") {
 		networkTestnanosleep();
 		// return Void();
 	}
 
-	state std::vector<NetworkTestInterface> interfs;
-	state std::vector<NetworkAddress> servers = NetworkAddress::parseList(testServers);
-	state int sent = 0;
-	state int completed = 0;
-	state LatencyStats latency;
+	std::vector<NetworkTestInterface> interfs;
+	std::vector<NetworkAddress> servers = NetworkAddress::parseList(testServers);
+	int sent = 0;
+	int completed = 0;
+	LatencyStats latency;
 
 	interfs.reserve(servers.size());
 	for (int i = 0; i < servers.size(); i++) {
 		interfs.push_back(NetworkTestInterface(servers[i]));
 	}
 
-	state std::vector<Future<Void>> clients;
+	std::vector<Future<Void>> clients;
 	clients.reserve(FLOW_KNOBS->NETWORK_TEST_CLIENT_COUNT);
 	for (int i = 0; i < FLOW_KNOBS->NETWORK_TEST_CLIENT_COUNT; i++) {
 		clients.push_back(testClient(interfs, &sent, &completed, &latency));
 	}
 	clients.push_back(logger(&sent, &completed, &latency));
 
-	wait(waitForAll(clients));
-	return Void();
+	co_await waitForAll(clients);
 }
 
 struct RandomIntRange {
 	int min;
 	int max;
 
-	RandomIntRange(int low = 0, int high = 0) : min(low), max(high) {}
+	explicit(false) RandomIntRange(int low = 0, int high = 0) : min(low), max(high) {}
 
 	// Accepts strings of the form "min:max" or "N"
 	// where N will be used for both min and max
-	RandomIntRange(std::string str) {
+	explicit(false) RandomIntRange(std::string str) {
 		StringRef high = str;
 		StringRef low = high.eat(":");
-		if (high.size() == 0) {
+		if (high.empty()) {
 			high = low;
 		}
-		min = low.size() == 0 ? 0 : atol(low.toString().c_str());
-		max = high.size() == 0 ? 0 : atol(high.toString().c_str());
+		min = low.empty() ? 0 : atol(low.toString().c_str());
+		max = high.empty() ? 0 : atol(high.toString().c_str());
 		if (min > max) {
 			std::swap(min, max);
 		}
@@ -393,7 +407,7 @@ struct P2PNetworkTest {
 		return s;
 	}
 
-	P2PNetworkTest() {}
+	P2PNetworkTest() = default;
 
 	P2PNetworkTest(std::string listenerAddresses,
 	               std::string remoteAddresses,
@@ -431,17 +445,17 @@ struct P2PNetworkTest {
 
 	NetworkAddress randomRemote() { return remotes[nondeterministicRandom()->randomInt(0, remotes.size())]; }
 
-	ACTOR static Future<Standalone<StringRef>> readMsg(P2PNetworkTest* self, Reference<IConnection> conn) {
-		state Standalone<StringRef> buffer = makeString(sizeof(int));
-		state int writeOffset = 0;
-		state bool gotHeader = false;
+	static Future<Standalone<StringRef>> readMsg(P2PNetworkTest* self, Reference<IConnection> conn) {
+		Standalone<StringRef> buffer = makeString(sizeof(int));
+		int writeOffset = 0;
+		bool gotHeader = false;
 
 		// Fill buffer sequentially until the initial bytesToRead is read (or more), then read
 		// intended message size and add it to bytesToRead, continue if needed until bytesToRead is 0.
-		loop {
+		while (true) {
 			int stutter = self->waitReadMilliseconds.get();
 			if (stutter > 0) {
-				wait(delay(stutter / 1e3));
+				co_await delay(stutter / 1e3);
 			}
 
 			int len = conn->read((uint8_t*)buffer.begin() + writeOffset, (uint8_t*)buffer.end());
@@ -451,12 +465,12 @@ struct P2PNetworkTest {
 			// If buffer is complete, either process it as a header or return it
 			if (writeOffset == buffer.size()) {
 				if (gotHeader) {
-					return buffer;
+					co_return buffer;
 				} else {
 					gotHeader = true;
 					int msgSize = *(int*)buffer.begin();
 					if (msgSize == 0) {
-						return Standalone<StringRef>();
+						co_return Standalone<StringRef>();
 					}
 					buffer = makeString(msgSize);
 					writeOffset = 0;
@@ -464,22 +478,22 @@ struct P2PNetworkTest {
 			}
 
 			if (len == 0) {
-				wait(conn->onReadable());
-				wait(delay(0, TaskPriority::ReadSocket));
+				co_await conn->onReadable();
+				co_await delay(0, TaskPriority::ReadSocket);
 			}
 		}
 	}
 
-	ACTOR static Future<Void> writeMsg(P2PNetworkTest* self, Reference<IConnection> conn, StringRef msg) {
-		state UnsentPacketQueue packets;
+	static Future<Void> writeMsg(P2PNetworkTest* self, Reference<IConnection> conn, StringRef msg) {
+		UnsentPacketQueue packets;
 		PacketWriter writer(packets.getWriteBuffer(msg.size()), nullptr, Unversioned());
 		writer.serializeBinaryItem((int)msg.size());
 		writer.serializeBytes(msg);
 
-		loop {
+		while (true) {
 			int stutter = self->waitWriteMilliseconds.get();
 			if (stutter > 0) {
-				wait(delay(stutter / 1e3));
+				co_await delay(stutter / 1e3);
 			}
 			int sent = conn->write(packets.getUnsent(), FLOW_KNOBS->MAX_PACKET_SEND_BYTES);
 
@@ -492,43 +506,41 @@ struct P2PNetworkTest {
 				break;
 			}
 
-			wait(conn->onWritable());
-			wait(yield(TaskPriority::WriteSocket));
+			co_await conn->onWritable();
+			co_await yield(TaskPriority::WriteSocket);
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> doSession(P2PNetworkTest* self, Reference<IConnection> conn, bool incoming) {
-		state int numRequests;
+	static Future<Void> doSession(P2PNetworkTest* self, Reference<IConnection> conn, bool incoming) {
+		int numRequests{ 0 };
 
 		try {
 			if (incoming) {
-				wait(conn->acceptHandshake());
+				co_await conn->acceptHandshake();
 
 				// Read the number of requests for the session
-				Standalone<StringRef> buf = wait(readMsg(self, conn));
+				Standalone<StringRef> buf = co_await readMsg(self, conn);
 				ASSERT(buf.size() == sizeof(int));
 				numRequests = *(int*)buf.begin();
 			} else {
-				wait(conn->connectHandshake());
+				co_await conn->connectHandshake();
 
 				// Pick the number of requests for the session and send it to remote
 				numRequests = self->requests.get();
-				wait(writeMsg(self, conn, StringRef((const uint8_t*)&numRequests, sizeof(int))));
+				co_await writeMsg(self, conn, StringRef((const uint8_t*)&numRequests, sizeof(int)));
 			}
 
 			while (numRequests > 0) {
 				if (incoming) {
 					// Wait for a request
-					wait(success(readMsg(self, conn)));
+					co_await readMsg(self, conn);
 					// Send a reply
-					wait(writeMsg(self, conn, self->msgBuffer.substr(0, self->replyBytes.get())));
+					co_await writeMsg(self, conn, self->msgBuffer.substr(0, self->replyBytes.get()));
 				} else {
 					// Send a request
-					wait(writeMsg(self, conn, self->msgBuffer.substr(0, self->requestBytes.get())));
+					co_await writeMsg(self, conn, self->msgBuffer.substr(0, self->requestBytes.get()));
 					// Wait for a reply
-					wait(success(readMsg(self, conn)));
+					co_await readMsg(self, conn);
 				}
 
 				if (--numRequests == 0) {
@@ -536,7 +548,7 @@ struct P2PNetworkTest {
 				}
 			}
 
-			wait(delay(self->idleMilliseconds.get() / 1e3));
+			co_await delay(self->idleMilliseconds.get() / 1e3);
 			conn->close();
 
 			if (incoming) {
@@ -550,35 +562,37 @@ struct P2PNetworkTest {
 			    .error(e)
 			    .detail("Remote", conn->getPeerAddress());
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> outgoing(P2PNetworkTest* self) {
-		loop {
-			wait(delay(0, TaskPriority::WriteSocket));
-			state NetworkAddress remote = self->randomRemote();
+	static Future<Void> outgoing(P2PNetworkTest* self) {
+		while (true) {
+			co_await delay(0, TaskPriority::WriteSocket);
+			NetworkAddress remote = self->randomRemote();
 
+			Optional<Error> err;
 			try {
-				state Reference<IConnection> conn = wait(INetworkConnections::net()->connect(remote));
+				Reference<IConnection> conn = co_await INetworkConnections::net()->connect(remote);
 				// printf("Connected to %s\n", remote.toString().c_str());
-				wait(doSession(self, conn, false));
+				co_await doSession(self, conn, false);
 			} catch (Error& e) {
+				err = e;
+			}
+			if (err.present()) {
 				++self->connectErrors;
-				TraceEvent(SevError, "P2POutgoingError").error(e).detail("Remote", remote);
-				wait(delay(1));
+				TraceEvent(SevError, "P2POutgoingError").error(err.get()).detail("Remote", remote);
+				co_await delay(1);
 			}
 		}
 	}
 
-	ACTOR static Future<Void> incoming(P2PNetworkTest* self, Reference<IListener> listener) {
-		state ActorCollection sessions(false);
+	static Future<Void> incoming(P2PNetworkTest* self, Reference<IListener> listener) {
+		ActorCollection sessions(false);
 
-		loop {
-			wait(delay(0, TaskPriority::AcceptSocket));
+		while (true) {
+			co_await delay(0, TaskPriority::AcceptSocket);
 
 			try {
-				state Reference<IConnection> conn = wait(listener->accept());
+				Reference<IConnection> conn = co_await listener->accept();
 				// printf("Connected from %s\n", conn->getPeerAddress().toString().c_str());
 				sessions.add(doSession(self, conn, true));
 			} catch (Error& e) {
@@ -588,9 +602,7 @@ struct P2PNetworkTest {
 		}
 	}
 
-	ACTOR static Future<Void> run_oneshot(P2PNetworkTest* self) {
-		state ActorCollection actors(false);
-
+	static Future<Void> run_oneshot(P2PNetworkTest* self) {
 		self->startTime = now();
 
 		fmt::print("{0} listeners, {1} remotes, {2} outgoing connections\n",
@@ -602,35 +614,33 @@ struct P2PNetworkTest {
 			printf("Remote: %s\n", n.toString().c_str());
 		}
 
-		for (auto el : self->listeners) {
+		for (const auto& el : self->listeners) {
 			printf("Listener: %s\n", el->getListenAddress().toString().c_str());
 		}
 
 		if (!self->listeners.empty()) {
-			state Reference<IConnection> conn1 = wait(self->listeners[0]->accept());
+			Reference<IConnection> conn1 = co_await self->listeners[0]->accept();
 			printf("Server: connected from %s\n", conn1->getPeerAddress().toString().c_str());
 			try {
-				wait(conn1->acceptHandshake());
+				co_await conn1->acceptHandshake();
 				printf("Server: connected from %s, handshake done\n", conn1->getPeerAddress().toString().c_str());
 			} catch (Error& e) {
 				printf("Server: handshake error %s\n", e.what());
 			}
 			threadSleep(11.0);
-			return Void();
+			co_return;
 		}
 
 		if (!self->remotes.empty()) {
-			state Reference<IConnection> conn2 = wait(INetworkConnections::net()->connect(self->remotes[0]));
+			Reference<IConnection> conn2 = co_await INetworkConnections::net()->connect(self->remotes[0]);
 			printf("Client: connected to %s\n", self->remotes[0].toString().c_str());
-			wait(conn2->connectHandshake());
+			co_await conn2->connectHandshake();
 			printf("Client: connected to %s, handshake done\n", self->remotes[0].toString().c_str());
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> run_impl(P2PNetworkTest* self) {
-		state ActorCollection actors(false);
+	static Future<Void> run_impl(P2PNetworkTest* self) {
+		ActorCollection actors(false);
 
 		self->startTime = now();
 		self->globalStartTime = self->startTime;
@@ -644,7 +654,7 @@ struct P2PNetworkTest {
 			printf("Remote: %s\n", n.toString().c_str());
 		}
 
-		for (auto el : self->listeners) {
+		for (const auto& el : self->listeners) {
 			printf("Listener: %s\n", el->getListenAddress().toString().c_str());
 			actors.add(incoming(self, el));
 		}
@@ -668,14 +678,13 @@ struct P2PNetworkTest {
 			}
 		}
 
-		loop {
-			wait(delay(1.0, TaskPriority::Max));
+		while (true) {
+			co_await delay(1.0, TaskPriority::Max);
 			printf("%s\n", self->statsString().c_str());
 			if (self->targetDuration > 0 && now() - self->globalStartTime > self->targetDuration) {
 				break;
 			}
 		}
-		return Void();
 	}
 
 	Future<Void> run() {
@@ -702,35 +711,33 @@ struct P2PNetworkTest {
 // The client will close the connection after a random idleMilliseconds.
 // Reads and writes can optionally preceded by random delays, waitReadMilliseconds and waitWriteMilliseconds.
 TEST_CASE(":/network/p2ptest") {
-	state P2PNetworkTest p2p(params.get("listenerAddresses").orDefault(""),
-	                         params.get("remoteAddresses").orDefault(""),
-	                         params.getInt("connectionsOut").orDefault(1),
-	                         params.get("requestBytes").orDefault("50:100"),
-	                         params.get("replyBytes").orDefault("500:1000"),
-	                         params.get("requests").orDefault("10:10000"),
-	                         params.get("idleMilliseconds").orDefault("0"),
-	                         params.get("waitReadMilliseconds").orDefault("0"),
-	                         params.get("waitWriteMilliseconds").orDefault("0"),
-	                         params.getDouble("targetDuration").orDefault(0.0),
-	                         false);
-
-	wait(p2p.run());
-	return Void();
+	P2PNetworkTest p2p(params.get("listenerAddresses").orDefault(""),
+	                   params.get("remoteAddresses").orDefault(""),
+	                   params.getInt("connectionsOut").orDefault(1),
+	                   params.get("requestBytes").orDefault("50:100"),
+	                   params.get("replyBytes").orDefault("500:1000"),
+	                   params.get("requests").orDefault("10:10000"),
+	                   params.get("idleMilliseconds").orDefault("0"),
+	                   params.get("waitReadMilliseconds").orDefault("0"),
+	                   params.get("waitWriteMilliseconds").orDefault("0"),
+	                   params.getDouble("targetDuration").orDefault(0.0),
+	                   false);
+
+	co_await p2p.run();
 }
 
 TEST_CASE(":/network/p2poneshottest") {
-	state P2PNetworkTest p2p(params.get("listenerAddresses").orDefault(""),
-	                         params.get("remoteAddresses").orDefault(""),
-	                         params.getInt("connectionsOut").orDefault(1),
-	                         params.get("requestBytes").orDefault("50:100"),
-	                         params.get("replyBytes").orDefault("500:1000"),
-	                         params.get("requests").orDefault("10:10000"),
-	                         params.get("idleMilliseconds").orDefault("0"),
-	                         params.get("waitReadMilliseconds").orDefault("0"),
-	                         params.get("waitWriteMilliseconds").orDefault("0"),
-	                         params.getDouble("targetDuration").orDefault(0.0),
-	                         true);
-
-	wait(p2p.run());
-	return Void();
+	P2PNetworkTest p2p(params.get("listenerAddresses").orDefault(""),
+	                   params.get("remoteAddresses").orDefault(""),
+	                   params.getInt("connectionsOut").orDefault(1),
+	                   params.get("requestBytes").orDefault("50:100"),
+	                   params.get("replyBytes").orDefault("500:1000"),
+	                   params.get("requests").orDefault("10:10000"),
+	                   params.get("idleMilliseconds").orDefault("0"),
+	                   params.get("waitReadMilliseconds").orDefault("0"),
+	                   params.get("waitWriteMilliseconds").orDefault("0"),
+	                   params.getDouble("targetDuration").orDefault(0.0),
+	                   true);
+
+	co_await p2p.run();
 }
diff --git a/fdbserver/ratekeeper/CMakeLists.txt b/fdbserver/ratekeeper/CMakeLists.txt
new file mode 100644
index 00000000000..317cc8276c8
--- /dev/null
+++ b/fdbserver/ratekeeper/CMakeLists.txt
@@ -0,0 +1,14 @@
+fdb_find_sources(FDBSERVER_RATEKEEPER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_ratekeeper SRCS ${FDBSERVER_RATEKEEPER_SRCS})
+add_fdbserver_link_test(fdbserver_ratekeeperlinktest
+  fdbserver_ratekeeper
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_ratekeeper)
+target_include_directories(fdbserver_ratekeeper
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_ratekeeper PRIVATE fdbserver_core)
diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/ratekeeper/Ratekeeper.cpp
similarity index 54%
rename from fdbserver/Ratekeeper.actor.cpp
rename to fdbserver/ratekeeper/Ratekeeper.cpp
index fe176cff8d1..f9c876bb22b 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/ratekeeper/Ratekeeper.cpp
@@ -1,5 +1,5 @@
 /*
- * Ratekeeper.actor.cpp
+ * Ratekeeper.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,58 +18,22 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ClientKnobs.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/Ratekeeper.h"
-#include "fdbserver/TagThrottler.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/QuietDatabase.h"
+#include <algorithm>
+
+#include "fdbclient/Knobs.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/ratekeeper/Ratekeeper.h"
+#include "Ratekeeper.h"
+#include "TagThrottler.h"
 #include "flow/OwningResource.h"
 
-#include "flow/actorcompiler.h" // must be last include
-
-const char* limitReasonName[] = {
-	"workload",
-	"storage_server_write_queue_size",
-	"storage_server_write_bandwidth_mvcc",
-	"storage_server_readable_behind",
-	"log_server_mvcc_write_bandwidth",
-	"log_server_write_queue",
-	"storage_server_min_free_space",
-	"storage_server_min_free_space_ratio",
-	"log_server_min_free_space",
-	"log_server_min_free_space_ratio",
-	"storage_server_durability_lag",
-	"storage_server_list_fetch_failed",
-};
-static_assert(sizeof(limitReasonName) / sizeof(limitReasonName[0]) == limitReason_t_end, "limitReasonDesc table size");
-
-int limitReasonEnd = limitReason_t_end;
-
-// NOTE: This has a corresponding table in Script.cs (see RatekeeperReason graph)
-// IF UPDATING THIS ARRAY, UPDATE SCRIPT.CS!
-const char* limitReasonDesc[] = {
-	"Workload or read performance.",
-	"Storage server performance (storage queue).",
-	"Storage server MVCC memory.",
-	"Storage server version falling behind.",
-	"Log server MVCC memory.",
-	"Storage server performance (log queue).",
-	"Storage server running out of space (approaching 100MB limit).",
-	"Storage server running out of space (approaching 5% limit).",
-	"Log server running out of space (approaching 100MB limit).",
-	"Log server running out of space (approaching 5% limit).",
-	"Storage server durable version falling behind.",
-	"Unable to fetch storage server list.",
-};
-
-static_assert(sizeof(limitReasonDesc) / sizeof(limitReasonDesc[0]) == limitReason_t_end, "limitReasonDesc table size");
-
-ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
+#include "flow/CoroUtils.h"
+
+static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
 	try {
-		wait(in);
-		return Void();
+		co_await in;
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled && !errOut.isSet())
 			errOut.sendError(e);
@@ -77,550 +41,535 @@ ACTOR static Future<Void> splitError(Future<Void> in, Promise<Void> errOut) {
 	}
 }
 
-class RatekeeperImpl {
-public:
-	ACTOR static Future<Void> configurationMonitor(Ratekeeper* self) {
-		loop {
-			state ReadYourWritesTransaction tr(self->db);
-
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-					RangeResult results = wait(tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY));
-					ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
-
-					self->configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
-
-					state Future<Void> watchFuture =
-					    tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
-					    tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
-					    tr.watch(failedLocalityVersionKey);
-					wait(tr.commit());
-					wait(watchFuture);
-					break;
-				} catch (Error& e) {
-					wait(tr.onError(e));
-				}
-			}
-		}
-	}
-
-	ACTOR static Future<Void> monitorServerListChange(
-	    Ratekeeper* self,
-	    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-		state std::map<UID, StorageServerInterface> oldServers;
-		state Transaction tr(self->db);
+Future<Void> Ratekeeper::configurationMonitor() {
+	while (true) {
+		ReadYourWritesTransaction tr(db);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				if (now() - self->lastSSListFetchedTimestamp > 2 * SERVER_KNOBS->SERVER_LIST_DELAY) {
-					TraceEvent(SevWarnAlways, "RatekeeperGetSSListLongLatency", self->id)
-					    .detail("Latency", now() - self->lastSSListFetchedTimestamp);
-				}
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-				    wait(NativeAPI::getServerListAndProcessClasses(&tr));
-				self->lastSSListFetchedTimestamp = now();
-
-				std::map<UID, StorageServerInterface> newServers;
-				for (const auto& [ssi, _] : results) {
-					const UID serverId = ssi.id();
-					newServers[serverId] = ssi;
-
-					if (oldServers.contains(serverId)) {
-						if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint() ||
-						    ssi.isAcceptingRequests() != oldServers[serverId].isAcceptingRequests()) {
-							serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
-						}
-						oldServers.erase(serverId);
-					} else {
-						serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
-					}
-				}
+				RangeResult results = co_await tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
+				ASSERT(!results.more && results.size() < CLIENT_KNOBS->TOO_MANY);
 
-				for (const auto& it : oldServers) {
-					serverChanges.send(std::make_pair(it.first, Optional<StorageServerInterface>()));
-				}
+				configuration.fromKeyValues((VectorRef<KeyValueRef>)results);
 
-				oldServers.swap(newServers);
-				tr = Transaction(self->db);
-				wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
+				Future<Void> watchFuture = tr.watch(moveKeysLockOwnerKey) || tr.watch(excludedServersVersionKey) ||
+				                           tr.watch(failedServersVersionKey) || tr.watch(excludedLocalityVersionKey) ||
+				                           tr.watch(failedLocalityVersionKey);
+				co_await tr.commit();
+				co_await watchFuture;
+				break;
 			} catch (Error& e) {
-				if (e.code() != error_code_actor_cancelled) {
-					TraceEvent("RatekeeperGetSSListError", self->id).errorUnsuppressed(e).suppressFor(1.0);
-				}
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
+}
 
-	ACTOR static Future<Void> trackStorageServerQueueInfo(ActorWeakSelfRef<Ratekeeper> self,
-	                                                      StorageServerInterface ssi) {
-		self->storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(self->id, ssi.id(), ssi.locality)));
-		self->healthMetrics.storageStats[ssi.id()] = HealthMetrics::StorageStats();
-		TraceEvent("RkTrackStorageStart", self->id)
-		    .detail("StorageServer", ssi.id())
-		    .detail("Locality", ssi.locality.toString());
+Future<Void> Ratekeeper::monitorServerListChange(
+    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+	std::map<UID, StorageServerInterface> oldServers;
+	Transaction tr(db);
 
-		loop {
-			try {
-				ErrorOr<StorageQueuingMetricsReply> reply = wait(ssi.getQueuingMetrics.getReplyUnlessFailedFor(
-				    StorageQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
-				Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
-				ASSERT(myQueueInfo != self->storageQueueInfo.end());
-				if (reply.present()) {
-					myQueueInfo->value.update(reply.get(), self->smoothTotalDurableBytes);
-					myQueueInfo->value.acceptingRequests = ssi.isAcceptingRequests();
-
-					// Update health stats.
-					auto ssMetrics = self->healthMetrics.storageStats.find(ssi.id());
-					ASSERT(ssMetrics != self->healthMetrics.storageStats.end());
-					ssMetrics->second.storageQueue = myQueueInfo->value.getStorageQueueBytes();
-					ssMetrics->second.storageDurabilityLag = myQueueInfo->value.getDurabilityLag();
-					ssMetrics->second.cpuUsage = reply.get().cpuUsage;
-					ssMetrics->second.diskUsage = reply.get().diskUsage;
-				} else {
-					if (myQueueInfo->value.valid) {
-						TraceEvent("RkStorageServerDidNotRespond", self->id).detail("StorageServer", ssi.id());
+	while (true) {
+		Error err;
+		try {
+			if (now() - lastSSListFetchedTimestamp > 2 * SERVER_KNOBS->SERVER_LIST_DELAY) {
+				TraceEvent(SevWarnAlways, "RatekeeperGetSSListLongLatency", id)
+				    .detail("Latency", now() - lastSSListFetchedTimestamp);
+			}
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
+			    co_await NativeAPI::getServerListAndProcessClasses(&tr);
+			lastSSListFetchedTimestamp = now();
+
+			std::map<UID, StorageServerInterface> newServers;
+			for (const auto& [ssi, _] : results) {
+				const UID serverId = ssi.id();
+				newServers[serverId] = ssi;
+
+				if (oldServers.contains(serverId)) {
+					if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint() ||
+					    ssi.isAcceptingRequests() != oldServers[serverId].isAcceptingRequests()) {
+						serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
 					}
-					myQueueInfo->value.valid = false;
+					oldServers.erase(serverId);
+				} else {
+					serverChanges.send(std::make_pair(serverId, Optional<StorageServerInterface>(ssi)));
 				}
+			}
 
-				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
-				     IFailureMonitor::failureMonitor().onStateEqual(ssi.getQueuingMetrics.getEndpoint(),
-				                                                    FailureStatus(false)));
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					self->storageQueueInfo.erase(ssi.id());
-					self->healthMetrics.storageStats.erase(ssi.id());
-					throw;
-				}
-				// Do no stop tracking storage server. The error might be recoverable.
-				Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
-				ASSERT(myQueueInfo != self->storageQueueInfo.end());
-				myQueueInfo->value.valid = false;
-				TraceEvent("RkTrackStorageError", self->id)
-				    .detail("StorageServer", ssi.id())
-				    .detail("Locality", ssi.locality.toString())
-				    .errorUnsuppressed(e);
+			for (const auto& it : oldServers) {
+				serverChanges.send(std::make_pair(it.first, Optional<StorageServerInterface>()));
 			}
+
+			oldServers.swap(newServers);
+			tr = Transaction(db);
+			co_await delay(SERVER_KNOBS->SERVER_LIST_DELAY);
+			continue;
+		} catch (Error& e) {
+			err = e;
+		}
+		if (err.code() != error_code_actor_cancelled) {
+			TraceEvent("RatekeeperGetSSListError", id).errorUnsuppressed(err).suppressFor(1.0);
 		}
+		co_await tr.onError(err);
 	}
+}
 
-	// works with ExcludeIncludeStorageServersWorkload.actor.cpp to make sure the size of SS list is bounded
-	ACTOR static Future<Void> monitorStorageServerQueueSizeInSimulation(ActorWeakSelfRef<Ratekeeper> self) {
-		if (!g_network->isSimulated()) {
-			return Void();
-		}
-		state int threshold = SERVER_KNOBS->RATEKEEPER_MONITOR_SS_THRESHOLD;
-		state int cnt = 0;
-		state int maxCount = 5;
-		state Transaction tr(self->db);
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-				    wait(NativeAPI::getServerListAndProcessClasses(&tr));
-				state int serverListSize = results.size();
-				state int interfaceSize = self->storageServerInterfaces.size();
-				state int metricSize = self->healthMetrics.storageStats.size();
-				if (interfaceSize - serverListSize > threshold || metricSize - serverListSize > threshold) {
-					cnt++;
-				} else {
-					cnt = 0;
-				}
-				if (cnt > maxCount) {
-					TraceEvent(SevError, "TooManySSInRK")
-					    .detail("Threshold", threshold)
-					    .detail("ServerListSize", serverListSize)
-					    .detail("InterfaceSize", interfaceSize)
-					    .detail("MetricSize", metricSize)
-					    .log();
-				}
+Future<Void> Ratekeeper::trackStorageServerQueueInfo(StorageServerInterface ssi) {
+	storageQueueInfo.insert(mapPair(ssi.id(), StorageQueueInfo(id, ssi.id(), ssi.locality)));
+	healthMetrics.storageStats[ssi.id()] = HealthMetrics::StorageStats();
+	TraceEvent("RkTrackStorageStart", id).detail("StorageServer", ssi.id()).detail("Locality", ssi.locality.toString());
 
-				// Storage related stats should be consistent.
-				for (const auto& [id, _] : self->storageServerInterfaces) {
-					ASSERT(self->storageQueueInfo.find(id) != self->storageQueueInfo.end());
-					ASSERT(self->healthMetrics.storageStats.find(id) != self->healthMetrics.storageStats.end());
+	while (true) {
+		try {
+			ErrorOr<StorageQueuingMetricsReply> reply = co_await ssi.getQueuingMetrics.getReplyUnlessFailedFor(
+			    StorageQueuingMetricsRequest(), 0, 0); // SOMEDAY: or tryGetReply?
+			Map<UID, StorageQueueInfo>::iterator myQueueInfo = storageQueueInfo.find(ssi.id());
+			ASSERT(myQueueInfo != storageQueueInfo.end());
+			if (reply.present()) {
+				myQueueInfo->value.update(reply.get(), smoothTotalDurableBytes);
+				myQueueInfo->value.acceptingRequests = ssi.isAcceptingRequests();
+
+				// Update health stats.
+				auto ssMetrics = healthMetrics.storageStats.find(ssi.id());
+				ASSERT(ssMetrics != healthMetrics.storageStats.end());
+				ssMetrics->second.storageQueue = myQueueInfo->value.getStorageQueueBytes();
+				ssMetrics->second.storageDurabilityLag = myQueueInfo->value.getDurabilityLag();
+				ssMetrics->second.cpuUsage = reply.get().cpuUsage;
+				ssMetrics->second.diskUsage = reply.get().diskUsage;
+			} else {
+				if (myQueueInfo->value.valid) {
+					TraceEvent("RkStorageServerDidNotRespond", id).detail("StorageServer", ssi.id());
 				}
+				myQueueInfo->value.valid = false;
+			}
 
-				// wait for monitorServerListChange to remove the interface
-				wait(delay(SERVER_KNOBS->RATEKEEPER_MONITOR_SS_DELAY));
-				tr = Transaction(self->db);
-			} catch (Error& e) {
-				TraceEvent("RateKeeperFailedToReadSSList").log();
-				wait(tr.onError(e));
+			co_await (delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
+			          IFailureMonitor::failureMonitor().onStateEqual(ssi.getQueuingMetrics.getEndpoint(),
+			                                                         FailureStatus(false)));
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				storageQueueInfo.erase(ssi.id());
+				healthMetrics.storageStats.erase(ssi.id());
+				throw;
 			}
+			// Do no stop tracking storage server. The error might be recoverable.
+			Map<UID, StorageQueueInfo>::iterator myQueueInfo = storageQueueInfo.find(ssi.id());
+			ASSERT(myQueueInfo != storageQueueInfo.end());
+			myQueueInfo->value.valid = false;
+			TraceEvent("RkTrackStorageError", id)
+			    .detail("StorageServer", ssi.id())
+			    .detail("Locality", ssi.locality.toString())
+			    .errorUnsuppressed(e);
 		}
 	}
+}
 
-	ACTOR static Future<Void> trackTLogQueueInfo(Ratekeeper* self, TLogInterface tli) {
-		self->tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
-		state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
-		TraceEvent("RkTrackTLog", self->id).detail("TransactionLog", tli.id());
+// works with ExcludeIncludeStorageServersWorkload.actor.cpp to make sure the size of SS list is bounded
+Future<Void> Ratekeeper::monitorStorageServerQueueSizeInSimulation() {
+	if (!g_network->isSimulated()) {
+		co_return;
+	}
+	int threshold = SERVER_KNOBS->RATEKEEPER_MONITOR_SS_THRESHOLD;
+	int cnt = 0;
+	int maxCount = 5;
+	Transaction tr(db);
+	while (true) {
+		Error err;
 		try {
-			loop {
-				ErrorOr<TLogQueuingMetricsReply> reply = wait(tli.getQueuingMetrics.getReplyUnlessFailedFor(
-				    TLogQueuingMetricsRequest(), 0, 0)); // SOMEDAY: or tryGetReply?
-				if (reply.present()) {
-					myQueueInfo->value.update(reply.get(), self->smoothTotalDurableBytes);
-					myQueueInfo->value.valid = true;
-				} else {
-					if (myQueueInfo->value.valid) {
-						TraceEvent("RkTLogDidNotRespond", self->id).detail("TransactionLog", tli.id());
-					}
-					myQueueInfo->value.valid = false;
-				}
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
+			    co_await NativeAPI::getServerListAndProcessClasses(&tr);
+			int serverListSize = results.size();
+			int interfaceSize = storageServerInterfaces.size();
+			int metricSize = healthMetrics.storageStats.size();
+			if (interfaceSize - serverListSize > threshold || metricSize - serverListSize > threshold) {
+				cnt++;
+			} else {
+				cnt = 0;
+			}
+			if (cnt > maxCount) {
+				TraceEvent(SevError, "TooManySSInRK")
+				    .detail("Threshold", threshold)
+				    .detail("ServerListSize", serverListSize)
+				    .detail("InterfaceSize", interfaceSize)
+				    .detail("MetricSize", metricSize)
+				    .log();
+			}
 
-				wait(delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
-				     IFailureMonitor::failureMonitor().onStateEqual(tli.getQueuingMetrics.getEndpoint(),
-				                                                    FailureStatus(false)));
+			// Storage related stats should be consistent.
+			for (const auto& [id, _] : storageServerInterfaces) {
+				ASSERT(storageQueueInfo.find(id) != storageQueueInfo.end());
+				ASSERT(healthMetrics.storageStats.find(id) != healthMetrics.storageStats.end());
 			}
+
+			// wait for monitorServerListChange to remove the interface
+			co_await delay(SERVER_KNOBS->RATEKEEPER_MONITOR_SS_DELAY);
+			tr = Transaction(db);
+			continue;
 		} catch (Error& e) {
-			// including cancellation
-			self->tlogQueueInfo.erase(myQueueInfo);
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent("RkTrackTLogError", self->id).detail("TransactionLog", tli.id()).errorUnsuppressed(e);
+			err = e;
+		}
+		TraceEvent("RateKeeperFailedToReadSSList").log();
+		co_await tr.onError(err);
+	}
+}
+
+Future<Void> Ratekeeper::trackTLogQueueInfo(TLogInterface tli) {
+	tlogQueueInfo.insert(mapPair(tli.id(), TLogQueueInfo(tli.id())));
+	Map<UID, TLogQueueInfo>::iterator myQueueInfo = tlogQueueInfo.find(tli.id());
+	TraceEvent("RkTrackTLog", id).detail("TransactionLog", tli.id());
+	try {
+		while (true) {
+			ErrorOr<TLogQueuingMetricsReply> reply = co_await tli.getQueuingMetrics.getReplyUnlessFailedFor(
+			    TLogQueuingMetricsRequest(), 0, 0); // SOMEDAY: or tryGetReply?
+			if (reply.present()) {
+				myQueueInfo->value.update(reply.get(), smoothTotalDurableBytes);
+				myQueueInfo->value.valid = true;
+			} else {
+				if (myQueueInfo->value.valid) {
+					TraceEvent("RkTLogDidNotRespond", id).detail("TransactionLog", tli.id());
+				}
+				myQueueInfo->value.valid = false;
 			}
-			throw;
+
+			co_await (delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE) &&
+			          IFailureMonitor::failureMonitor().onStateEqual(tli.getQueuingMetrics.getEndpoint(),
+			                                                         FailureStatus(false)));
+		}
+	} catch (Error& e) {
+		// including cancellation
+		tlogQueueInfo.erase(myQueueInfo);
+		if (e.code() != error_code_actor_cancelled) {
+			TraceEvent("RkTrackTLogError", id).detail("TransactionLog", tli.id()).errorUnsuppressed(e);
 		}
+		throw;
 	}
+}
 
-	ACTOR static Future<Void> trackEachStorageServer(
-	    ActorWeakSelfRef<Ratekeeper> self,
-	    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+Future<Void> Ratekeeper::trackEachStorageServer(
+    FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
+	std::unordered_map<UID, Future<Void>> storageServerTrackers;
+	Promise<Void> err;
 
-		state std::unordered_map<UID, Future<Void>> storageServerTrackers;
-		state Promise<Void> err;
+	while (true) {
+		auto res = co_await race(serverChanges, err.getFuture());
+		ASSERT_EQ(res.index(), 0);
+		std::pair<UID, Optional<StorageServerInterface>> change = std::get<0>(std::move(res));
 
-		loop choose {
-			when(state std::pair<UID, Optional<StorageServerInterface>> change = waitNext(serverChanges)) {
-				wait(delay(0)); // prevent storageServerTracker from getting cancelled while on the call stack
+		co_await delay(0); // prevent storageServerTracker from getting cancelled while on the call stack
 
-				const UID& id = change.first;
-				if (change.second.present()) {
-					if (!change.second.get().isTss()) {
+		const UID& id = change.first;
+		if (change.second.present()) {
+			if (!change.second.get().isTss()) {
 
-						auto& a = storageServerTrackers[change.first];
-						a = Future<Void>();
-						a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
+				auto& a = storageServerTrackers[change.first];
+				a = Future<Void>();
+				a = splitError(trackStorageServerQueueInfo(change.second.get()), err);
 
-						self->storageServerInterfaces[id] = change.second.get();
-					}
-				} else {
-					storageServerTrackers.erase(id);
-					self->storageServerInterfaces.erase(id);
-				}
+				storageServerInterfaces[id] = change.second.get();
 			}
-			when(wait(err.getFuture())) {}
+		} else {
+			storageServerTrackers.erase(id);
+			storageServerInterfaces.erase(id);
 		}
 	}
+}
 
-	ACTOR static Future<Void> monitorHotShards(Ratekeeper* self, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-		loop {
-			wait(delay(SERVER_KNOBS->HOT_SHARD_MONITOR_FREQUENCY));
-			if (!self->ssHighWriteQueue.present()) {
-				continue;
-			}
+Future<Void> Ratekeeper::monitorHotShards(Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	while (true) {
+		co_await delay(SERVER_KNOBS->HOT_SHARD_MONITOR_FREQUENCY);
+		if (!ssHighWriteQueue.present()) {
+			continue;
+		}
 
-			state UID ssi = self->ssHighWriteQueue.get();
-			state SetThrottledShardRequest setReq;
+		UID ssi = ssHighWriteQueue.get();
+		SetThrottledShardRequest setReq;
 
-			// TraceEvent(SevDebug, "SendGetHotShardsRequest");
-			try {
-				GetHotShardsRequest getReq;
-				GetHotShardsReply reply = wait(self->storageServerInterfaces[ssi].getHotShards.getReply(getReq));
-
-				// Backup's restore range can't be throttled, otherwise restore would fail,
-				// i.e., "ApplyMutationsError".
-				KeyRangeRef applyMutationRange("\xfe\xff\xfe"_sr, "\xfe\xff\xff\xff"_sr);
-				for (const auto& shard : reply.hotShards) {
-					if (!shard.intersects(applyMutationRange)) {
-						setReq.throttledShards.push_back(shard);
-					} else {
-						TraceEvent("IgnoreHotShard").detail("Shard", shard);
-					}
+		// TraceEvent(SevDebug, "SendGetHotShardsRequest");
+		try {
+			GetHotShardsRequest getReq;
+			GetHotShardsReply reply = co_await storageServerInterfaces[ssi].getHotShards.getReply(getReq);
+
+			// Backup's restore range can't be throttled, otherwise restore would fail,
+			// i.e., "ApplyMutationsError".
+			KeyRangeRef applyMutationRange("\xfe\xff\xfe"_sr, "\xfe\xff\xff\xff"_sr);
+			for (const auto& shard : reply.hotShards) {
+				if (!shard.intersects(applyMutationRange)) {
+					setReq.throttledShards.push_back(shard);
+				} else {
+					TraceEvent("IgnoreHotShard").detail("Shard", shard);
 				}
-			} catch (Error& e) {
-				TraceEvent(SevWarn, "CannotMonitorHotShardForSS").detail("SS", ssi);
-				continue;
-			}
-			if (!setReq.throttledShards.size()) {
-				continue;
-			}
-			setReq.expirationTime = now() + SERVER_KNOBS->HOT_SHARD_THROTTLING_EXPIRE_AFTER;
-			for (auto& shard : setReq.throttledShards) {
-				TraceEvent(SevInfo, "SendRequestThrottleHotShard")
-				    .detail("Shard", shard)
-				    .detail("DelayUntil", setReq.expirationTime);
-			}
-			for (const auto& cpi : dbInfo->get().client.commitProxies) {
-				cpi.setThrottledShard.send(setReq);
 			}
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "CannotMonitorHotShardForSS").detail("SS", ssi);
+			continue;
+		}
+		if (setReq.throttledShards.empty()) {
+			continue;
+		}
+		setReq.expirationTime = now() + SERVER_KNOBS->HOT_SHARD_THROTTLING_EXPIRE_AFTER;
+		for (auto& shard : setReq.throttledShards) {
+			TraceEvent(SevInfo, "SendRequestThrottleHotShard")
+			    .detail("Shard", shard)
+			    .detail("DelayUntil", setReq.expirationTime);
+		}
+		for (const auto& cpi : dbInfo->get().client.commitProxies) {
+			cpi.setThrottledShard.send(setReq);
 		}
 	}
+}
 
-	ACTOR static Future<Void> run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-		state ActorOwningSelfRef<Ratekeeper> pSelf(
-		    new Ratekeeper(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True)));
-		state Ratekeeper& self = *pSelf;
-		state Future<Void> timeout = Void();
-		state std::vector<Future<Void>> tlogTrackers;
-		state std::vector<TLogInterface> tlogInterfs;
-		state Promise<Void> err;
-		state Future<Void> collection = actorCollection(self.addActor.getFuture());
-
-		TraceEvent("RatekeeperStarting", rkInterf.id());
-		self.addActor.send(waitFailureServer(rkInterf.waitFailure.getFuture()));
-		self.addActor.send(self.configurationMonitor());
-
-		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
-		self.addActor.send(self.monitorServerListChange(serverChanges));
-		self.addActor.send(RatekeeperImpl::trackEachStorageServer(pSelf, serverChanges.getFuture()));
-		self.addActor.send(monitorStorageServerQueueSizeInSimulation(pSelf));
-		self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
-
-		self.addActor.send(self.monitorThrottlingChanges());
-
-		if (SERVER_KNOBS->HOT_SHARD_THROTTLING_ENABLED) {
-			self.addActor.send(self.monitorHotShards(dbInfo));
-		}
+Future<Void> Ratekeeper::handleReportCommitCostEstimationReqs(RatekeeperInterface rkInterf) {
+	while (true) {
+		ReportCommitCostEstimationRequest req = co_await rkInterf.reportCommitCostEstimation.getFuture();
+		updateCommitCostEstimation(req.ssTrTagCommitCost);
+		req.reply.send(Void());
+	}
+}
 
-		self.addActor.send(self.refreshStorageServerCommitCosts());
-
-		TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
-		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG)
-		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
-		    .detail(
-		        "Rate",
-		        (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) /
-		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-		             2.0));
-
-		TraceEvent("RkStorageServerQueueSizeParameters", rkInterf.id())
-		    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER)
-		    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER)
-		    .detail("EBrake", SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES)
-		    .detail(
-		        "Rate",
-		        (SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER - SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER) /
-		            ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
-		             2.0));
-
-		tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
-		tlogTrackers.reserve(tlogInterfs.size());
-		for (int i = 0; i < tlogInterfs.size(); i++) {
-			tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
-		}
+Future<Void> Ratekeeper::handleGetSSVersionLagReqs(RatekeeperInterface rkInterf) {
+	while (true) {
+		GetSSVersionLagRequest req = co_await rkInterf.getSSVersionLag.getFuture();
+		GetSSVersionLagReply reply;
+		getSSVersionLag(reply.maxPrimarySSVersion, reply.maxRemoteSSVersion);
+		req.reply.send(reply);
+	}
+}
 
-		self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
+Future<Void> Ratekeeper::handleGetRateInfoReqs(RatekeeperInterface rkInterf,
+                                               Version* recoveryVersion,
+                                               bool* lastLimited) {
+	while (true) {
+		GetRateInfoRequest req = co_await rkInterf.getRateInfo.getFuture();
+		GetRateInfoReply reply;
 
-		state bool recovering = dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS;
-		state Version recoveryVersion = std::numeric_limits<Version>::max();
-		if (recovering) {
-			self.version_recovery[recoveryVersion] = std::make_pair(now(), Optional<double>());
-		}
+		auto& p = grvProxyInfo[req.requesterID];
+		//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.totalTransactions).detail("Delta", req.totalReleasedTransactions - p.totalTransactions);
+		if (p.totalTransactions > 0) {
+			smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
 
-		try {
-			state bool lastLimited = false;
-			loop choose {
-				when(wait(timeout)) {
-					double actualTps = self.smoothReleasedTransactions.smoothRate();
-					actualTps =
-					    std::max(std::max(1.0, actualTps),
-					             self.smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
-
-					if (self.actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
-						self.actualTpsHistory.pop_front();
-					}
-					self.actualTpsHistory.push_back(actualTps);
+			for (auto const& [tag, count] : req.throttledTagCounts) {
+				tagThrottler->addRequests(tag, count);
+			}
+		}
+		if (p.batchTransactions > 0) {
+			smoothBatchReleasedTransactions.addDelta(req.batchReleasedTransactions - p.batchTransactions);
+		}
 
-					while (self.version_recovery.size() > CLIENT_KNOBS->MAX_GENERATIONS) {
-						self.version_recovery.erase(self.version_recovery.begin());
-					}
+		p.totalTransactions = req.totalReleasedTransactions;
+		p.batchTransactions = req.batchReleasedTransactions;
+		p.version = req.version;
+		maxVersion = std::max(maxVersion, req.version);
 
-					self.updateRate(&self.normalLimits);
-					self.updateRate(&self.batchLimits);
-
-					lastLimited = self.smoothReleasedTransactions.smoothRate() >
-					              SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
-					double tooOld = now() - 1.0;
-					for (auto p = self.grvProxyInfo.begin(); p != self.grvProxyInfo.end();) {
-						if (p->second.lastUpdateTime < tooOld)
-							p = self.grvProxyInfo.erase(p);
-						else
-							++p;
-					}
-					timeout = delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
-				}
-				when(GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
-					GetRateInfoReply reply;
+		if (*recoveryVersion == std::numeric_limits<Version>::max() && version_recovery.contains(*recoveryVersion)) {
+			*recoveryVersion = maxVersion;
+			version_recovery[*recoveryVersion] = version_recovery[std::numeric_limits<Version>::max()];
+			version_recovery.erase(std::numeric_limits<Version>::max());
+		}
 
-					auto& p = self.grvProxyInfo[req.requesterID];
-					//TraceEvent("RKMPU", req.requesterID).detail("TRT", req.totalReleasedTransactions).detail("Last", p.totalTransactions).detail("Delta", req.totalReleasedTransactions - p.totalTransactions);
-					if (p.totalTransactions > 0) {
-						self.smoothReleasedTransactions.addDelta(req.totalReleasedTransactions - p.totalTransactions);
+		p.lastUpdateTime = now();
 
-						for (auto const& [tag, count] : req.throttledTagCounts) {
-							self.tagThrottler->addRequests(tag, count);
-						}
-					}
-					if (p.batchTransactions > 0) {
-						self.smoothBatchReleasedTransactions.addDelta(req.batchReleasedTransactions -
-						                                              p.batchTransactions);
-					}
+		reply.transactionRate = normalLimits.tpsLimit / grvProxyInfo.size();
+		reply.batchTransactionRate = batchLimits.tpsLimit / grvProxyInfo.size();
+		reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
 
-					p.totalTransactions = req.totalReleasedTransactions;
-					p.batchTransactions = req.batchReleasedTransactions;
-					p.version = req.version;
-					self.maxVersion = std::max(self.maxVersion, req.version);
-
-					if (recoveryVersion == std::numeric_limits<Version>::max() &&
-					    self.version_recovery.contains(recoveryVersion)) {
-						recoveryVersion = self.maxVersion;
-						self.version_recovery[recoveryVersion] =
-						    self.version_recovery[std::numeric_limits<Version>::max()];
-						self.version_recovery.erase(std::numeric_limits<Version>::max());
-					}
+		if (p.lastThrottledTagChangeId != tagThrottler->getThrottledTagChangeId() ||
+		    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
+			p.lastThrottledTagChangeId = tagThrottler->getThrottledTagChangeId();
+			p.lastTagPushTime = now();
 
-					p.lastUpdateTime = now();
-
-					reply.transactionRate = self.normalLimits.tpsLimit / self.grvProxyInfo.size();
-					reply.batchTransactionRate = self.batchLimits.tpsLimit / self.grvProxyInfo.size();
-					reply.leaseDuration = SERVER_KNOBS->METRIC_UPDATE_RATE;
-
-					if (p.lastThrottledTagChangeId != self.tagThrottler->getThrottledTagChangeId() ||
-					    now() > p.lastTagPushTime + SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL) {
-						p.lastThrottledTagChangeId = self.tagThrottler->getThrottledTagChangeId();
-						p.lastTagPushTime = now();
-
-						bool returningTagsToProxy{ false };
-						if (SERVER_KNOBS->ENFORCE_TAG_THROTTLING_ON_PROXIES) {
-							auto proxyThrottledTags = self.tagThrottler->getProxyRates(self.grvProxyInfo.size());
-							if (!SERVER_KNOBS->GLOBAL_TAG_THROTTLING_REPORT_ONLY) {
-								returningTagsToProxy = proxyThrottledTags.size() > 0;
-								reply.proxyThrottledTags = std::move(proxyThrottledTags);
-							}
-						} else {
-							auto clientThrottledTags = self.tagThrottler->getClientRates();
-							if (!SERVER_KNOBS->GLOBAL_TAG_THROTTLING_REPORT_ONLY) {
-								returningTagsToProxy = clientThrottledTags.size() > 0;
-								reply.clientThrottledTags = std::move(clientThrottledTags);
-							}
-						}
-						CODE_PROBE(returningTagsToProxy, "Returning tag throttles to a proxy");
-					}
+			auto clientThrottledTags = tagThrottler->getClientRates();
+			bool returningTagsToProxy = !clientThrottledTags.empty();
+			reply.clientThrottledTags = std::move(clientThrottledTags);
+			CODE_PROBE(returningTagsToProxy, "Returning tag throttles to a proxy");
+		}
 
-					reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
-					reply.healthMetrics.tpsLimit = self.normalLimits.tpsLimit;
-					reply.healthMetrics.batchLimited = lastLimited;
+		reply.healthMetrics.update(healthMetrics, true, req.detailed);
+		reply.healthMetrics.tpsLimit = normalLimits.tpsLimit;
+		reply.healthMetrics.batchLimited = *lastLimited;
 
-					req.reply.send(reply);
-				}
-				when(HaltRatekeeperRequest req = waitNext(rkInterf.haltRatekeeper.getFuture())) {
-					req.reply.send(Void());
-					TraceEvent("RatekeeperHalted", rkInterf.id()).detail("ReqID", req.requesterID);
-					break;
-				}
-				when(ReportCommitCostEstimationRequest req =
-				         waitNext(rkInterf.reportCommitCostEstimation.getFuture())) {
-					self.updateCommitCostEstimation(req.ssTrTagCommitCost);
-					req.reply.send(Void());
-				}
-				when(GetSSVersionLagRequest req = waitNext(rkInterf.getSSVersionLag.getFuture())) {
-					GetSSVersionLagReply reply;
-					self.getSSVersionLag(reply.maxPrimarySSVersion, reply.maxRemoteSSVersion);
-					req.reply.send(reply);
-				}
-				when(wait(err.getFuture())) {}
-				when(wait(dbInfo->onChange())) {
-					if (!recovering && dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-						recovering = true;
-						recoveryVersion = self.maxVersion;
-						if (recoveryVersion == 0) {
-							recoveryVersion = std::numeric_limits<Version>::max();
-						}
-						if (self.version_recovery.contains(recoveryVersion)) {
-							auto& it = self.version_recovery[recoveryVersion];
-							double existingEnd = it.second.present() ? it.second.get() : now();
-							double existingDuration = existingEnd - it.first;
-							self.version_recovery[recoveryVersion] =
-							    std::make_pair(now() - existingDuration, Optional<double>());
-						} else {
-							self.version_recovery[recoveryVersion] = std::make_pair(now(), Optional<double>());
-						}
-					}
-					if (recovering && dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
-						recovering = false;
-						self.version_recovery[recoveryVersion].second = now();
-					}
+		req.reply.send(reply);
+	}
+}
 
-					if (tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
-						tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
-						tlogTrackers = std::vector<Future<Void>>();
-						for (int i = 0; i < tlogInterfs.size(); i++)
-							tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
-					}
-					self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
-				}
-				when(wait(collection)) {
-					ASSERT(false);
-					throw internal_error();
-				}
+Future<Void> Ratekeeper::handleDBInfoChanges(Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                             bool* recovering,
+                                             Version* recoveryVersion,
+                                             std::vector<TLogInterface>* tlogInterfs,
+                                             std::vector<Future<Void>>* tlogTrackers,
+                                             Promise<Void>* err) {
+	while (true) {
+		co_await dbInfo->onChange();
+
+		if (!*recovering && dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			*recovering = true;
+			*recoveryVersion = maxVersion;
+			if (*recoveryVersion == 0) {
+				*recoveryVersion = std::numeric_limits<Version>::max();
+			}
+			if (version_recovery.contains(*recoveryVersion)) {
+				auto& it = version_recovery[*recoveryVersion];
+				double existingEnd = it.second.present() ? it.second.get() : now();
+				double existingDuration = existingEnd - it.first;
+				version_recovery[*recoveryVersion] = std::make_pair(now() - existingDuration, Optional<double>());
+			} else {
+				version_recovery[*recoveryVersion] = std::make_pair(now(), Optional<double>());
 			}
-		} catch (Error& err) {
-			TraceEvent("RatekeeperDied", rkInterf.id()).errorUnsuppressed(err);
 		}
-		return Void();
-	}
+		if (*recovering && dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
+			*recovering = false;
+			version_recovery[*recoveryVersion].second = now();
+		}
 
-	ACTOR static Future<Void> refreshStorageServerCommitCosts(Ratekeeper* self) {
-		state double lastBusiestCommitTagPick;
-		state std::vector<Future<Void>> replies;
-		loop {
-			lastBusiestCommitTagPick = now();
-			wait(delay(SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
-
-			replies.clear();
-
-			double elapsed = now() - lastBusiestCommitTagPick;
-			// for each SS, select the busiest commit tag from ssTrTagCommitCost
-			for (auto& [ssId, ssQueueInfo] : self->storageQueueInfo) {
-				// NOTE: In some cases, for unknown reason SS will not respond to the updateCommitCostRequest. Since the
-				// information is not time-sensitive, we do not wait for the replies.
-				replies.push_back(self->storageServerInterfaces[ssId].updateCommitCostRequest.getReply(
-				    ssQueueInfo.refreshCommitCost(elapsed)));
+		if (*tlogInterfs != dbInfo->get().logSystemConfig.allLocalLogs()) {
+			*tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
+			*tlogTrackers = std::vector<Future<Void>>();
+			for (int i = 0; i < tlogInterfs->size(); i++) {
+				tlogTrackers->push_back(splitError(trackTLogQueueInfo((*tlogInterfs)[i]), *err));
 			}
 		}
+		remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
 	}
-}; // class RatekeeperImpl
-
-Future<Void> Ratekeeper::configurationMonitor() {
-	return RatekeeperImpl::configurationMonitor(this);
 }
 
-Future<Void> Ratekeeper::monitorServerListChange(
-    PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges) {
-	return RatekeeperImpl::monitorServerListChange(this, serverChanges);
-}
+Future<Void> Ratekeeper::rateUpdater(bool* lastLimited) {
+	while (true) {
+		double actualTps = smoothReleasedTransactions.smoothRate();
+		actualTps = std::max(std::max(1.0, actualTps),
+		                     smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
 
-Future<Void> Ratekeeper::trackTLogQueueInfo(TLogInterface tli) {
-	return RatekeeperImpl::trackTLogQueueInfo(this, tli);
+		if (actualTpsHistory.size() > SERVER_KNOBS->MAX_TPS_HISTORY_SAMPLES) {
+			actualTpsHistory.pop_front();
+		}
+		actualTpsHistory.push_back(actualTps);
+
+		while (version_recovery.size() > CLIENT_KNOBS->MAX_GENERATIONS) {
+			version_recovery.erase(version_recovery.begin());
+		}
+
+		updateRate(&normalLimits);
+		updateRate(&batchLimits);
+
+		*lastLimited =
+		    smoothReleasedTransactions.smoothRate() > SERVER_KNOBS->LAST_LIMITED_RATIO * batchLimits.tpsLimit;
+		double tooOld = now() - 1.0;
+		for (auto p = grvProxyInfo.begin(); p != grvProxyInfo.end();) {
+			if (p->second.lastUpdateTime < tooOld)
+				p = grvProxyInfo.erase(p);
+			else
+				++p;
+		}
+		co_await delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
+	}
 }
 
-Future<Void> Ratekeeper::monitorThrottlingChanges() {
-	return tagThrottler->monitorThrottlingChanges();
+Future<Void> Ratekeeper::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	ActorOwningSelfRef<Ratekeeper> pSelf(
+	    new Ratekeeper(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True)));
+	Ratekeeper& self = *pSelf;
+	std::vector<Future<Void>> tlogTrackers;
+	std::vector<TLogInterface> tlogInterfs;
+	Promise<Void> err;
+	Future<Void> collection = actorCollection(self.addActor.getFuture());
+
+	TraceEvent("RatekeeperStarting", rkInterf.id());
+	self.addActor.send(waitFailureServer(rkInterf.waitFailure.getFuture()));
+	self.addActor.send(self.configurationMonitor());
+
+	PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges;
+	self.addActor.send(self.monitorServerListChange(serverChanges));
+	self.addActor.send(self.trackEachStorageServer(serverChanges.getFuture()));
+	self.addActor.send(self.monitorStorageServerQueueSizeInSimulation());
+	self.addActor.send(traceRole(Role::RATEKEEPER, rkInterf.id()));
+
+	self.addActor.send(self.monitorThrottlingChanges());
+
+	if (SERVER_KNOBS->HOT_SHARD_THROTTLING_ENABLED) {
+		self.addActor.send(self.monitorHotShards(dbInfo));
+	}
+
+	self.addActor.send(self.refreshStorageServerCommitCosts());
+	self.addActor.send(self.handleReportCommitCostEstimationReqs(rkInterf));
+	self.addActor.send(self.handleGetSSVersionLagReqs(rkInterf));
+
+	TraceEvent("RkTLogQueueSizeParameters", rkInterf.id())
+	    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG)
+	    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
+	    .detail("Rate",
+	            (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) /
+	                ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+	                 2.0));
+
+	TraceEvent("RkStorageServerQueueSizeParameters", rkInterf.id())
+	    .detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER)
+	    .detail("Spring", SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER)
+	    .detail("EBrake", SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES)
+	    .detail("Rate",
+	            (SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER - SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER) /
+	                ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) +
+	                 2.0));
+
+	tlogInterfs = dbInfo->get().logSystemConfig.allLocalLogs();
+	tlogTrackers.reserve(tlogInterfs.size());
+	for (int i = 0; i < tlogInterfs.size(); i++) {
+		tlogTrackers.push_back(splitError(self.trackTLogQueueInfo(tlogInterfs[i]), err));
+	}
+
+	self.remoteDC = dbInfo->get().logSystemConfig.getRemoteDcId();
+
+	bool recovering = dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS;
+	Version recoveryVersion = std::numeric_limits<Version>::max();
+	if (recovering) {
+		self.version_recovery[recoveryVersion] = std::make_pair(now(), Optional<double>());
+	}
+
+	bool lastLimited = false;
+	self.addActor.send(self.handleGetRateInfoReqs(rkInterf, &recoveryVersion, &lastLimited));
+	self.addActor.send(
+	    self.handleDBInfoChanges(dbInfo, &recovering, &recoveryVersion, &tlogInterfs, &tlogTrackers, &err));
+	self.addActor.send(self.rateUpdater(&lastLimited));
+
+	try {
+		auto res = co_await race(rkInterf.haltRatekeeper.getFuture(), err.getFuture(), collection);
+		ASSERT_EQ(res.index(), 0);
+		HaltRatekeeperRequest req = std::get<0>(std::move(res));
+		req.reply.send(Void());
+		TraceEvent("RatekeeperHalted", rkInterf.id()).detail("ReqID", req.requesterID);
+	} catch (Error& err) {
+		TraceEvent("RatekeeperDied", rkInterf.id()).errorUnsuppressed(err);
+	}
 }
 
-Future<Void> Ratekeeper::monitorHotShards(Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	return RatekeeperImpl::monitorHotShards(this, dbInfo);
+Future<Void> Ratekeeper::refreshStorageServerCommitCosts() {
+	double lastBusiestCommitTagPick{ 0 };
+	std::vector<Future<Void>> replies;
+	while (true) {
+		lastBusiestCommitTagPick = now();
+		co_await delay(SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL);
+
+		replies.clear();
+
+		double elapsed = now() - lastBusiestCommitTagPick;
+		// for each SS, select the busiest commit tag from ssTrTagCommitCost
+		for (auto& [ssId, ssQueueInfo] : storageQueueInfo) {
+			// NOTE: In some cases, for unknown reason SS will not respond to the updateCommitCostRequest. Since the
+			// information is not time-sensitive, we do not wait for the replies.
+			replies.push_back(
+			    storageServerInterfaces[ssId].updateCommitCostRequest.getReply(ssQueueInfo.refreshCommitCost(elapsed)));
+		}
+	}
 }
 
-Future<Void> Ratekeeper::run(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	return RatekeeperImpl::run(rkInterf, dbInfo);
+Future<Void> Ratekeeper::monitorThrottlingChanges() {
+	return tagThrottler->monitorThrottlingChanges();
 }
 
 Ratekeeper::Ratekeeper(UID id, Database db)
@@ -634,8 +583,7 @@ Ratekeeper::Ratekeeper(UID id, Database db)
                                                                     SERVER_KNOBS->TARGET_BYTES_PER_TLOG,
                                                                     SERVER_KNOBS->SPRING_BYTES_TLOG,
                                                                     SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE,
-                                                                    SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS,
-                                                                    SERVER_KNOBS->TARGET_BW_LAG),
+                                                                    SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS),
     batchLimits(TransactionPriority::BATCH,
                 "Batch",
                 SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER_BATCH,
@@ -643,15 +591,9 @@ Ratekeeper::Ratekeeper(UID id, Database db)
                 SERVER_KNOBS->TARGET_BYTES_PER_TLOG_BATCH,
                 SERVER_KNOBS->SPRING_BYTES_TLOG_BATCH,
                 SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE_BATCH,
-                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH,
-                SERVER_KNOBS->TARGET_BW_LAG_BATCH),
-    maxVersion(0), blobWorkerTime(now()), unblockedAssignmentTime(now()), anyBlobRanges(false) {
-	if (SERVER_KNOBS->GLOBAL_TAG_THROTTLING) {
-		tagThrottler = std::make_unique<GlobalTagThrottler>(
-		    db, id, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND, SERVER_KNOBS->GLOBAL_TAG_THROTTLING_LIMITING_THRESHOLD);
-	} else {
-		tagThrottler = std::make_unique<TagThrottler>(db, id);
-	}
+                SERVER_KNOBS->TARGET_DURABILITY_LAG_VERSIONS_BATCH),
+    maxVersion(0), unblockedAssignmentTime(now()) {
+	tagThrottler = std::make_unique<TagThrottler>(db, id);
 }
 
 void Ratekeeper::updateCommitCostEstimation(
@@ -678,7 +620,7 @@ static std::string getIgnoredZonesReasons(
 		}
 		ignoredZoneReasons += "] ";
 	}
-	return ignoredZoneReasons.length() ? ignoredZoneReasons : "None";
+	return !ignoredZoneReasons.empty() ? ignoredZoneReasons : "None";
 }
 
 void Ratekeeper::updateRate(RatekeeperLimits* limits) {
@@ -979,16 +921,31 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 
 		limitReason_t tlogLimitReason = limitReason_t::log_server_write_queue;
 
-		int64_t minFreeSpace = std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE,
-		                                (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * tl.getSmoothTotalSpace()));
-
-		worstFreeSpaceTLog =
-		    std::min(worstFreeSpaceTLog, std::max((int64_t)tl.getSmoothFreeSpace() - minFreeSpace, (int64_t)0));
-
-		int64_t springBytes = std::max<int64_t>(
-		    1, std::min<int64_t>(limits->logSpringBytes, (tl.getSmoothFreeSpace() - minFreeSpace) * 0.2));
-		int64_t targetBytes =
-		    std::max<int64_t>(1, std::min(limits->logTargetBytes, (int64_t)tl.getSmoothFreeSpace() - minFreeSpace));
+		const auto smoothTotalSpace = tl.getSmoothTotalSpace();
+		const auto smoothFreeSpace = tl.getSmoothFreeSpace();
+		const auto minFreeSpaceByRatio =
+		    static_cast<int64_t>(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * smoothTotalSpace);
+		const auto minFreeSpace = std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE, minFreeSpaceByRatio);
+		// Start shrinking the tlog queue budget once available space drops below the configured ratio,
+		// then ramp down linearly until reaching minFreeSpace.
+		const auto throttleStartSpaceByRatio =
+		    static_cast<int64_t>(SERVER_KNOBS->TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO * smoothTotalSpace);
+		const auto throttleStartSpace = std::max(minFreeSpace + 1, throttleStartSpaceByRatio);
+		const auto availableAboveMin = smoothFreeSpace - minFreeSpace;
+		const auto availableAboveMinBytes = static_cast<int64_t>(availableAboveMin);
+		const auto throttleWindow = static_cast<double>(throttleStartSpace - minFreeSpace);
+		const auto diskBudgetRatio = std::clamp(availableAboveMin / throttleWindow, 0.0, 1.0);
+
+		worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, std::max(availableAboveMinBytes, int64_t{ 0 }));
+
+		const auto scaledSpringBytes = static_cast<int64_t>(limits->logSpringBytes * diskBudgetRatio);
+		const auto springBytes =
+		    std::max(int64_t{ 1 }, std::min(std::max(int64_t{ 1 }, scaledSpringBytes), availableAboveMinBytes));
+		const auto scaledTargetBytes = static_cast<int64_t>(limits->logTargetBytes * diskBudgetRatio);
+		const auto targetBytes =
+		    std::max(int64_t{ 1 }, std::min(std::max(int64_t{ 1 }, scaledTargetBytes), availableAboveMinBytes));
+
+		CODE_PROBE(diskBudgetRatio < 1.0, "Ratekeeper tlog disk budget ratio below one");
 		if (targetBytes != limits->logTargetBytes) {
 			if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) {
 				tlogLimitReason = limitReason_t::log_server_min_free_space;
@@ -999,9 +956,11 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 				TraceEvent("RatekeeperLimitReasonDetails")
 				    .detail("TLogID", tl.id)
 				    .detail("Reason", tlogLimitReason)
-				    .detail("TLSmoothFreeSpace", tl.getSmoothFreeSpace())
-				    .detail("TLSmoothTotalSpace", tl.getSmoothTotalSpace())
+				    .detail("TLSmoothFreeSpace", smoothFreeSpace)
+				    .detail("TLSmoothTotalSpace", smoothTotalSpace)
 				    .detail("LimitsLogTargetBytes", limits->logTargetBytes)
+				    .detail("ThrottleStartSpace", throttleStartSpace)
+				    .detail("DiskBudgetRatio", diskBudgetRatio)
 				    .detail("TargetBytes", targetBytes)
 				    .detail("MinFreeSpace", minFreeSpace);
 			}
@@ -1173,10 +1132,6 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) {
 	}
 }
 
-Future<Void> Ratekeeper::refreshStorageServerCommitCosts() {
-	return RatekeeperImpl::refreshStorageServerCommitCosts(this);
-}
-
 void Ratekeeper::getSSVersionLag(Version& maxSSPrimaryVersion, Version& maxSSRemoteVersion) {
 	maxSSPrimaryVersion = maxSSRemoteVersion = invalidVersion;
 	for (auto i = storageQueueInfo.begin(); i != storageQueueInfo.end(); ++i) {
@@ -1193,9 +1148,8 @@ void Ratekeeper::getSSVersionLag(Version& maxSSPrimaryVersion, Version& maxSSRem
 	}
 }
 
-ACTOR Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	wait(Ratekeeper::run(rkInterf, dbInfo));
-	return Void();
+Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	co_await Ratekeeper::run(rkInterf, dbInfo);
 }
 
 StorageQueueInfo::StorageQueueInfo(const UID& ratekeeperID_, const UID& id_, const LocalityData& locality_)
@@ -1332,24 +1286,3 @@ void TLogQueueInfo::update(TLogQueuingMetricsReply const& reply, Smoother& smoot
 		smoothTotalSpace.setTotal(reply.storageBytes.total);
 	}
 }
-
-RatekeeperLimits::RatekeeperLimits(TransactionPriority priority,
-                                   std::string context,
-                                   int64_t storageTargetBytes,
-                                   int64_t storageSpringBytes,
-                                   int64_t logTargetBytes,
-                                   int64_t logSpringBytes,
-                                   double maxVersionDifference,
-                                   int64_t durabilityLagTargetVersions,
-                                   double bwLagTarget)
-  : tpsLimit(std::numeric_limits<double>::infinity()), tpsLimitMetric(StringRef("Ratekeeper.TPSLimit" + context)),
-    reasonMetric(StringRef("Ratekeeper.Reason" + context)), storageTargetBytes(storageTargetBytes),
-    storageSpringBytes(storageSpringBytes), logTargetBytes(logTargetBytes), logSpringBytes(logSpringBytes),
-    maxVersionDifference(maxVersionDifference),
-    durabilityLagTargetVersions(durabilityLagTargetVersions +
-                                SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS), // The read transaction life versions
-                                                                                   // are expected to not
-    // be durable on the storage servers
-    lastDurabilityLag(0), durabilityLagLimit(std::numeric_limits<double>::infinity()), bwLagTarget(bwLagTarget),
-    priority(priority), context(context),
-    rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
diff --git a/fdbserver/include/fdbserver/Ratekeeper.h b/fdbserver/ratekeeper/Ratekeeper.h
similarity index 81%
rename from fdbserver/include/fdbserver/Ratekeeper.h
rename to fdbserver/ratekeeper/Ratekeeper.h
index eaf24860717..986800d65f7 100644
--- a/fdbserver/include/fdbserver/Ratekeeper.h
+++ b/fdbserver/ratekeeper/Ratekeeper.h
@@ -26,41 +26,23 @@
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TagThrottle.h"
 #include "fdbrpc/Smoother.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RatekeeperInterface.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/TLogInterface.h"
-
-enum limitReason_t {
-	unlimited, // TODO: rename to workload?
-	storage_server_write_queue_size, // 1
-	storage_server_write_bandwidth_mvcc,
-	storage_server_readable_behind,
-	log_server_mvcc_write_bandwidth,
-	log_server_write_queue, // 5
-	storage_server_min_free_space, // a storage server's normal limits are being reduced by low free space
-	storage_server_min_free_space_ratio, // a storage server's normal limits are being reduced by a low free space ratio
-	log_server_min_free_space,
-	log_server_min_free_space_ratio,
-	storage_server_durability_lag, // 10
-	storage_server_list_fetch_failed,
-	limitReason_t_end
-};
+#include "fdbserver/core/RatekeeperLimitReasons.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "fdbserver/core/TLogInterface.h"
+
+struct ServerDBInfo;
 
 class StorageQueueInfo {
 	uint64_t totalWriteCosts{ 0 };
 	int totalWriteOps{ 0 };
 
-	// refresh periodically
 	TransactionTagMap<TransactionCommitCostEstimation> tagCostEst;
 
 	UID ratekeeperID;
 	Smoother smoothFreeSpace, smoothTotalSpace;
 	Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
-
-	// Currently unused
 	Smoother smoothDurableVersion, smoothLatestVersion;
 
 public:
@@ -74,14 +56,12 @@ class StorageQueueInfo {
 
 	StorageQueueInfo(const UID& id, const LocalityData& locality);
 	StorageQueueInfo(const UID& rateKeeperID, const UID& id, const LocalityData& locality);
-	// Summarizes up the commit cost per storage server. Returns the UpdateCommitCostRequest for corresponding SS.
 	UpdateCommitCostRequest refreshCommitCost(double elapsed);
 	int64_t getStorageQueueBytes() const { return lastReply.bytesInput - smoothDurableBytes.smoothTotal(); }
 	int64_t getDurabilityLag() const { return smoothLatestVersion.smoothTotal() - smoothDurableVersion.smoothTotal(); }
 	void update(StorageQueuingMetricsReply const&, Smoother& smoothTotalDurableBytes);
 	void addCommitCost(TransactionTagRef tagName, TransactionCommitCostEstimation const& cost);
 
-	// Accessor methods for Smoothers
 	double getSmoothFreeSpace() const { return smoothFreeSpace.smoothTotal(); }
 	double getSmoothTotalSpace() const { return smoothTotalSpace.smoothTotal(); }
 	double getSmoothDurableBytes() const { return smoothDurableBytes.smoothTotal(); }
@@ -89,8 +69,6 @@ class StorageQueueInfo {
 	double getVerySmoothDurableBytesRate() const { return verySmoothDurableBytes.smoothRate(); }
 
 	Version getLatestVersion() const { return lastReply.version; }
-
-	// Determine the ratio (limit / current throughput) for throttling based on write queue size
 	Optional<double> getTagThrottlingRatio(int64_t storageTargetBytes, int64_t storageSpringBytes) const;
 };
 
@@ -104,14 +82,13 @@ class TLogQueueInfo {
 	bool valid;
 	UID id;
 
-	// Accessor methods for Smoothers
 	double getSmoothFreeSpace() const { return smoothFreeSpace.smoothTotal(); }
 	double getSmoothTotalSpace() const { return smoothTotalSpace.smoothTotal(); }
 	double getSmoothDurableBytes() const { return smoothDurableBytes.smoothTotal(); }
 	double getSmoothInputBytesRate() const { return smoothInputBytes.smoothRate(); }
 	double getVerySmoothDurableBytesRate() const { return verySmoothDurableBytes.smoothRate(); }
 
-	TLogQueueInfo(UID id);
+	explicit TLogQueueInfo(UID id);
 	Version getLastCommittedVersion() const { return lastReply.v; }
 	void update(TLogQueuingMetricsReply const& reply, Smoother& smoothTotalDurableBytes);
 };
@@ -131,8 +108,6 @@ struct RatekeeperLimits {
 	int64_t lastDurabilityLag;
 	double durabilityLagLimit;
 
-	double bwLagTarget;
-
 	TransactionPriority priority;
 	std::string context;
 
@@ -145,14 +120,10 @@ struct RatekeeperLimits {
 	                 int64_t logTargetBytes,
 	                 int64_t logSpringBytes,
 	                 double maxVersionDifference,
-	                 int64_t durabilityLagTargetVersions,
-	                 double bwLagTarget);
+	                 int64_t durabilityLagTargetVersions);
 };
 
 class Ratekeeper {
-	friend class RatekeeperImpl;
-
-	// Differentiate from GrvProxyInfo in DatabaseContext.h
 	struct GrvProxyInfo {
 		int64_t totalTransactions{ 0 };
 		int64_t batchTransactions{ 0 };
@@ -193,7 +164,6 @@ class Ratekeeper {
 
 	std::unique_ptr<class ITagThrottler> tagThrottler;
 
-	// Maps storage server ID to storage server interface
 	std::unordered_map<UID, StorageServerInterface> storageServerInterfaces;
 
 	RatekeeperLimits normalLimits;
@@ -201,12 +171,9 @@ class Ratekeeper {
 
 	Deque<double> actualTpsHistory;
 	Version maxVersion;
-	double blobWorkerTime;
 	double unblockedAssignmentTime;
 	std::map<Version, Ratekeeper::VersionInfo> version_transactions;
 	std::map<Version, std::pair<double, Optional<double>>> version_recovery;
-	Deque<std::pair<double, Version>> blobWorkerVersionHistory;
-	bool anyBlobRanges;
 	Optional<Key> remoteDC;
 	Optional<UID> ssHighWriteQueue;
 
@@ -231,16 +198,25 @@ class Ratekeeper {
 	void updateRate(RatekeeperLimits* limits);
 	Future<Void> refreshStorageServerCommitCosts();
 	Future<Void> monitorServerListChange(PromiseStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges);
-
-	// SOMEDAY: template trackStorageServerQueueInfo and trackTLogQueueInfo into one function
 	Future<Void> trackStorageServerQueueInfo(StorageServerInterface);
+	Future<Void> monitorStorageServerQueueSizeInSimulation();
 	Future<Void> trackTLogQueueInfo(TLogInterface);
+	Future<Void> trackEachStorageServer(FutureStream<std::pair<UID, Optional<StorageServerInterface>>> serverChanges);
 
 	void tryAutoThrottleTag(TransactionTag, double rate, double busyness, TagThrottledReason);
 	void tryAutoThrottleTag(StorageQueueInfo&, int64_t storageQueue, int64_t storageDurabilityLag);
 	Future<Void> monitorThrottlingChanges();
-	Future<Void> monitorBlobWorkers(Reference<AsyncVar<ServerDBInfo> const> dbInfo);
 	Future<Void> monitorHotShards(Reference<AsyncVar<ServerDBInfo> const> dbInfo);
+	Future<Void> handleReportCommitCostEstimationReqs(RatekeeperInterface rkInterf);
+	Future<Void> handleGetSSVersionLagReqs(RatekeeperInterface rkInterf);
+	Future<Void> handleGetRateInfoReqs(RatekeeperInterface rkInterf, Version* recoveryVersion, bool* lastLimited);
+	Future<Void> handleDBInfoChanges(Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+	                                 bool* recovering,
+	                                 Version* recoveryVersion,
+	                                 std::vector<TLogInterface>* tlogInterfs,
+	                                 std::vector<Future<Void>>* tlogTrackers,
+	                                 Promise<Void>* err);
+	Future<Void> rateUpdater(bool* lastLimited);
 
 	void getSSVersionLag(Version& maxSSPrimaryVersion, Version& maxSSRemoteVersion);
 
diff --git a/fdbserver/ratekeeper/RatekeeperLimits.cpp b/fdbserver/ratekeeper/RatekeeperLimits.cpp
new file mode 100644
index 00000000000..2e1d3fd08b2
--- /dev/null
+++ b/fdbserver/ratekeeper/RatekeeperLimits.cpp
@@ -0,0 +1,41 @@
+/*
+ * RatekeeperLimits.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Ratekeeper.h"
+#include "fdbserver/core/Knobs.h"
+
+RatekeeperLimits::RatekeeperLimits(TransactionPriority priority,
+                                   std::string context,
+                                   int64_t storageTargetBytes,
+                                   int64_t storageSpringBytes,
+                                   int64_t logTargetBytes,
+                                   int64_t logSpringBytes,
+                                   double maxVersionDifference,
+                                   int64_t durabilityLagTargetVersions)
+  : tpsLimit(std::numeric_limits<double>::infinity()), tpsLimitMetric(StringRef("Ratekeeper.TPSLimit" + context)),
+    reasonMetric(StringRef("Ratekeeper.Reason" + context)), storageTargetBytes(storageTargetBytes),
+    storageSpringBytes(storageSpringBytes), logTargetBytes(logTargetBytes), logSpringBytes(logSpringBytes),
+    maxVersionDifference(maxVersionDifference),
+    durabilityLagTargetVersions(durabilityLagTargetVersions +
+                                SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS), // The read transaction life versions
+                                                                                   // are expected to not
+    // be durable on the storage servers
+    lastDurabilityLag(0), durabilityLagLimit(std::numeric_limits<double>::infinity()), priority(priority),
+    context(context), rkUpdateEventCacheHolder(makeReference<EventCacheHolder>("RkUpdate" + context)) {}
diff --git a/fdbserver/RkTagThrottleCollection.cpp b/fdbserver/ratekeeper/RkTagThrottleCollection.cpp
similarity index 99%
rename from fdbserver/RkTagThrottleCollection.cpp
rename to fdbserver/ratekeeper/RkTagThrottleCollection.cpp
index 14817c7187a..cc5b700f16e 100644
--- a/fdbserver/RkTagThrottleCollection.cpp
+++ b/fdbserver/ratekeeper/RkTagThrottleCollection.cpp
@@ -19,8 +19,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RkTagThrottleCollection.h"
+#include "fdbserver/core/Knobs.h"
+#include "RkTagThrottleCollection.h"
 
 double RkTagThrottleCollection::RkTagThrottleData::getTargetRate(Optional<double> requestRate) const {
 	if (limits.tpsRate == 0.0 || !requestRate.present() || requestRate.get() == 0.0 || !rateSet) {
diff --git a/fdbserver/include/fdbserver/RkTagThrottleCollection.h b/fdbserver/ratekeeper/RkTagThrottleCollection.h
similarity index 89%
rename from fdbserver/include/fdbserver/RkTagThrottleCollection.h
rename to fdbserver/ratekeeper/RkTagThrottleCollection.h
index f9f060be2f7..5ccf042acde 100644
--- a/fdbserver/include/fdbserver/RkTagThrottleCollection.h
+++ b/fdbserver/ratekeeper/RkTagThrottleCollection.h
@@ -22,7 +22,7 @@
 #pragma once
 
 #include "fdbclient/Knobs.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TagThrottle.h"
 #include "fdbrpc/Smoother.h"
 
 class RkTagThrottleCollection : NonCopyable {
@@ -34,8 +34,6 @@ class RkTagThrottleCollection : NonCopyable {
 	struct RkTagThrottleData {
 		ClientTagThrottleLimits limits;
 		Smoother clientRate;
-
-		// Only used by auto-throttles
 		double created = now();
 		double lastUpdated = 0;
 		double lastReduced = now();
@@ -57,18 +55,15 @@ class RkTagThrottleCollection : NonCopyable {
 
 public:
 	RkTagThrottleCollection() = default;
-	RkTagThrottleCollection(RkTagThrottleCollection&& other);
+	explicit(false) RkTagThrottleCollection(RkTagThrottleCollection&& other);
 	RkTagThrottleCollection& operator=(RkTagThrottleCollection&& other);
 
-	// Set or update an auto throttling limit for the specified tag and priority combination.
-	// Returns the TPS rate if the throttle is updated, otherwise returns an empty optional
 	Optional<double> autoThrottleTag(UID id,
 	                                 TransactionTag const& tag,
 	                                 double fractionalBusyness,
 	                                 Optional<double> tpsRate = Optional<double>(),
 	                                 Optional<double> expiration = Optional<double>());
 
-	// Set or update a manual tps rate limit for the specified tag and priority combination
 	void manualThrottleTag(UID id,
 	                       TransactionTag const& tag,
 	                       TransactionPriority priority,
diff --git a/fdbserver/TagThrottler.actor.cpp b/fdbserver/ratekeeper/TagThrottler.cpp
similarity index 88%
rename from fdbserver/TagThrottler.actor.cpp
rename to fdbserver/ratekeeper/TagThrottler.cpp
index 9e5896a2fc1..12238e5b808 100644
--- a/fdbserver/TagThrottler.actor.cpp
+++ b/fdbserver/ratekeeper/TagThrottler.cpp
@@ -1,5 +1,5 @@
 /*
- * TagThrottler.h
+ * TagThrottler.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,9 +19,12 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TagThrottler.h"
-#include "fdbserver/RkTagThrottleCollection.h"
-#include "flow/actorcompiler.h" // must be last include
+#include "fdbserver/core/Knobs.h"
+#include "RkTagThrottleCollection.h"
+#include "TagThrottler.h"
+#include "flow/CoroUtils.h"
+
+#include <functional>
 
 class TagThrottlerImpl {
 	Database db;
@@ -31,18 +34,19 @@ class TagThrottlerImpl {
 	bool autoThrottlingEnabled{ false };
 	Future<Void> expiredTagThrottleCleanup;
 
-	ACTOR static Future<Void> monitorThrottlingChanges(TagThrottlerImpl* self) {
-		state bool committed = false;
-		loop {
-			state ReadYourWritesTransaction tr(self->db);
+	static Future<Void> monitorThrottlingChanges(TagThrottlerImpl* self) {
+		bool committed = false;
+		while (true) {
+			ReadYourWritesTransaction tr(self->db);
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 
-					state Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
-					state Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
+					Future<RangeResult> throttledTagKeys = tr.getRange(tagThrottleKeys, CLIENT_KNOBS->TOO_MANY);
+					Future<Optional<Value>> autoThrottlingEnabled = tr.get(tagThrottleAutoEnabledKey);
 
 					if (!committed) {
 						BinaryWriter limitWriter(Unversioned());
@@ -50,7 +54,7 @@ class TagThrottlerImpl {
 						tr.set(tagThrottleLimitKey, limitWriter.toValue());
 					}
 
-					wait(success(throttledTagKeys) && success(autoThrottlingEnabled));
+					co_await (success(throttledTagKeys) && success(autoThrottlingEnabled));
 
 					if (autoThrottlingEnabled.get().present() && autoThrottlingEnabled.get().get() == "0"_sr) {
 						CODE_PROBE(true, "Auto-throttling disabled");
@@ -91,7 +95,7 @@ class TagThrottlerImpl {
 							tagValue.expirationTime = now() + tagValue.initialDuration;
 							BinaryWriter wr(IncludeVersion(ProtocolVersion::withTagThrottleValueReason()));
 							wr << tagValue;
-							state Value value = wr.toValue();
+							Value value = wr.toValue();
 
 							tr.set(entry.key, value);
 						}
@@ -119,18 +123,23 @@ class TagThrottlerImpl {
 					self->throttledTags = std::move(updatedTagThrottles);
 					++self->throttledTagChangeId;
 
-					state Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
-					wait(tr.commit());
+					Future<Void> watchFuture = tr.watch(tagThrottleSignalKey);
+					co_await tr.commit();
 					committed = true;
 
-					wait(watchFuture);
+					co_await watchFuture;
 					TraceEvent("RatekeeperThrottleSignaled", self->id).log();
 					CODE_PROBE(true, "Tag throttle changes detected");
 					break;
 				} catch (Error& e) {
-					TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(e);
-					wait(tr.onError(e));
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+					err = e;
 				}
+
+				TraceEvent("RatekeeperMonitorThrottlingChangesError", self->id).error(err);
+				co_await tr.onError(err);
 			}
 		}
 	}
@@ -160,9 +169,11 @@ class TagThrottlerImpl {
 		return Void();
 	}
 
+	void cleanupExpiredTagThrottles() const { ThrottleApi::expire(db.getReference()); }
+
 public:
 	TagThrottlerImpl(Database db, UID id) : db(db), id(id) {
-		expiredTagThrottleCleanup = recurring([this]() { ThrottleApi::expire(this->db.getReference()); },
+		expiredTagThrottleCleanup = recurring(std::bind_front(&TagThrottlerImpl::cleanupExpiredTagThrottles, this),
 		                                      SERVER_KNOBS->TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL);
 	}
 	Future<Void> monitorThrottlingChanges() { return monitorThrottlingChanges(this); }
diff --git a/fdbserver/include/fdbserver/TagThrottler.h b/fdbserver/ratekeeper/TagThrottler.h
similarity index 50%
rename from fdbserver/include/fdbserver/TagThrottler.h
rename to fdbserver/ratekeeper/TagThrottler.h
index 98da2442a66..a05baf54f01 100644
--- a/fdbserver/include/fdbserver/TagThrottler.h
+++ b/fdbserver/ratekeeper/TagThrottler.h
@@ -21,39 +21,20 @@
 #pragma once
 
 #include "fdbclient/PImpl.h"
-#include "fdbserver/Ratekeeper.h"
+#include "Ratekeeper.h"
 
 class ITagThrottler {
 public:
 	virtual ~ITagThrottler() = default;
-
-	// Poll the system keyspace looking for updates made through the tag throttling API
 	virtual Future<Void> monitorThrottlingChanges() = 0;
-
-	// Increment the number of known requests associated with the specified tag
 	virtual void addRequests(TransactionTag tag, int count) = 0;
-
-	// This throttled tag change ID is used to coordinate updates with the GRV proxies
 	virtual uint64_t getThrottledTagChangeId() const = 0;
-
-	// For each tag and priority combination, return the throughput limit and expiration time
-	// Also, erase expired tags
 	virtual PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() = 0;
-
-	// For each tag and priority combination, return the throughput limit for the cluster
-	// (to be shared across all GRV proxies)
-	virtual TransactionTagMap<double> getProxyRates(int numProxies) = 0;
-
 	virtual int64_t autoThrottleCount() const = 0;
 	virtual uint32_t busyReadTagCount() const = 0;
 	virtual uint32_t busyWriteTagCount() const = 0;
 	virtual int64_t manualThrottleCount() const = 0;
 	virtual bool isAutoThrottlingEnabled() const = 0;
-
-	// Based on the busiest read and write tags in the provided storage queue info, these methods
-	// update tag throttling limits. Unfortunately, the two effective interfaces of the two
-	// implementations of ITagThrottler (GlobalTagThrottler and TagThrottler) have diveraged over
-	// time. As a result, exactly one of the below methods is a noop for each implementation.
 	virtual Future<Void> tryUpdateAutoThrottling(StorageQueueInfo const&) = 0;
 	virtual void updateThrottling(Map<UID, StorageQueueInfo> const&) = 0;
 };
@@ -69,7 +50,6 @@ class TagThrottler : public ITagThrottler {
 	void addRequests(TransactionTag tag, int count) override;
 	uint64_t getThrottledTagChangeId() const override;
 	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() override;
-	TransactionTagMap<double> getProxyRates(int numProxies) override { throw not_implemented(); }
 	int64_t autoThrottleCount() const override;
 	uint32_t busyReadTagCount() const override;
 	uint32_t busyWriteTagCount() const override;
@@ -78,33 +58,3 @@ class TagThrottler : public ITagThrottler {
 	Future<Void> tryUpdateAutoThrottling(StorageQueueInfo const&) override;
 	void updateThrottling(Map<UID, StorageQueueInfo> const&) override {}
 };
-
-class GlobalTagThrottler : public ITagThrottler {
-	PImpl<class GlobalTagThrottlerImpl> impl;
-
-public:
-	GlobalTagThrottler(Database db, UID id, int maxFallingBehind, double limitingThreshold);
-	~GlobalTagThrottler();
-
-	Future<Void> monitorThrottlingChanges() override;
-	void addRequests(TransactionTag tag, int count) override;
-	uint64_t getThrottledTagChangeId() const override;
-
-	int64_t autoThrottleCount() const override;
-	uint32_t busyReadTagCount() const override;
-	uint32_t busyWriteTagCount() const override;
-	int64_t manualThrottleCount() const override;
-	bool isAutoThrottlingEnabled() const override;
-
-	Future<Void> tryUpdateAutoThrottling(StorageQueueInfo const&) override { return Void(); }
-	void updateThrottling(Map<UID, StorageQueueInfo> const&) override;
-	PrioritizedTransactionTagMap<ClientTagThrottleLimits> getClientRates() override;
-	TransactionTagMap<double> getProxyRates(int numProxies) override;
-
-	// Testing only:
-public:
-	void setQuota(TransactionTagRef, ThrottleApi::TagQuotaValue const&);
-	void removeQuota(TransactionTagRef);
-	void removeExpiredTags();
-	uint32_t tagsTracked() const;
-};
diff --git a/fdbserver/include/fdbserver/Knobs.h b/fdbserver/ratekeeper/include/fdbserver/ratekeeper/Ratekeeper.h
similarity index 71%
rename from fdbserver/include/fdbserver/Knobs.h
rename to fdbserver/ratekeeper/include/fdbserver/ratekeeper/Ratekeeper.h
index af68a96fdb8..b4923dc3f2e 100644
--- a/fdbserver/include/fdbserver/Knobs.h
+++ b/fdbserver/ratekeeper/include/fdbserver/ratekeeper/Ratekeeper.h
@@ -1,5 +1,5 @@
 /*
- * Knobs.h
+ * Ratekeeper.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,17 +18,16 @@
  * limitations under the License.
  */
 
-#ifndef FDBSERVER_KNOBS_H
-#define FDBSERVER_KNOBS_H
+#ifndef FDBSERVER_RATEKEEPER_PUBLIC_H
+#define FDBSERVER_RATEKEEPER_PUBLIC_H
 
 #pragma once
 
-#include "fdbclient/IKnobCollection.h"
+#include "fdbserver/core/RatekeeperInterface.h"
+#include "flow/flow.h"
 
-#define SERVER_KNOBS (&IKnobCollection::getGlobalKnobCollection().getServerKnobs())
+struct ServerDBInfo;
 
-inline const ServerKnobs& getServerKnobs() {
-	return *SERVER_KNOBS;
-}
+Future<Void> ratekeeper(RatekeeperInterface rki, Reference<AsyncVar<ServerDBInfo> const> db);
 
-#endif // FDBSERVER_KNOBS_H
+#endif
diff --git a/fdbserver/resolver/CMakeLists.txt b/fdbserver/resolver/CMakeLists.txt
new file mode 100644
index 00000000000..e2aa40c2127
--- /dev/null
+++ b/fdbserver/resolver/CMakeLists.txt
@@ -0,0 +1,15 @@
+fdb_find_sources(FDBSERVER_RESOLVER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_resolver SRCS ${FDBSERVER_RESOLVER_SRCS})
+add_fdbserver_link_test(fdbserver_resolverlinktest
+  fdbserver_resolver
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_resolver)
+target_include_directories(fdbserver_resolver
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_resolver PRIVATE fdbserver_core fdbserver_logsystem)
diff --git a/fdbserver/SkipList.cpp b/fdbserver/resolver/ConflictSet.cpp
similarity index 96%
rename from fdbserver/SkipList.cpp
rename to fdbserver/resolver/ConflictSet.cpp
index 39e1ffe2195..faeab7848cf 100644
--- a/fdbserver/SkipList.cpp
+++ b/fdbserver/resolver/ConflictSet.cpp
@@ -1,5 +1,5 @@
 /*
- * SkipList.cpp
+ * ConflictSet.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -32,7 +32,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/ConflictSet.h"
+#include "ConflictSet.h"
 #include "flow/UnitTest.h"
 
 static std::vector<PerfDoubleCounter*> skc;
@@ -176,7 +176,7 @@ void sortPoints(std::vector<KeyInfo>& points) {
 
 	tasks.emplace_back(0, points.size(), 0);
 
-	while (tasks.size()) {
+	while (!tasks.empty()) {
 		SortTask st = tasks.back();
 		tasks.pop_back();
 
@@ -273,7 +273,7 @@ class SkipList : NonCopyable {
 			n->nPointers = level + 1;
 
 			n->valueLength = value.size();
-			if (value.size() > 0) {
+			if (!value.empty()) {
 				memcpy(n->value(), value.begin(), value.size());
 			}
 			return n;
@@ -421,7 +421,7 @@ class SkipList : NonCopyable {
 		}
 	}
 	~SkipList() { destroy(); }
-	SkipList(SkipList&& other) noexcept : header(other.header) { other.header = nullptr; }
+	explicit(false) SkipList(SkipList&& other) noexcept : header(other.header) { other.header = nullptr; }
 	void operator=(SkipList&& other) noexcept {
 		destroy();
 		header = other.header;
@@ -486,41 +486,6 @@ class SkipList : NonCopyable {
 		}
 	}
 
-	// Splits the version history represented by this skiplist into separate key ranges
-	//   delimited by the given array of keys.  This SkipList is left empty.  this->partition
-	//   is intended to be followed by a call to this->concatenate() recombining the same
-	//   partitions.  In between, operations on each partition must not touch any keys outside
-	//   the partition.  Specifically, the partition to the left of 'key' must not have a range
-	//	 [...,key) inserted, since that would insert an entry at 'key'.
-	// Note this function is not used.
-	void partition(StringRef* begin, int splitCount, SkipList* output) {
-		for (int i = splitCount - 1; i >= 0; i--) {
-			Finger f(header, begin[i]);
-			while (!f.finished())
-				f.nextLevel();
-			split(f, output[i + 1]);
-		}
-		swap(output[0]);
-	}
-
-	// Concatenates multiple SkipList objects into one and stores in input[0].
-	// Note this function is not used.
-	void concatenate(SkipList* input, int count) {
-		std::vector<Finger> ends(count - 1);
-		for (int i = 0; i < ends.size(); i++)
-			input[i].getEnd(ends[i]);
-
-		for (int l = 0; l < MaxLevels; l++) {
-			for (int i = ends.size() - 1; i >= 0; i--) {
-				ends[i].finger[l]->setNext(l, input[i + 1].header->getNext(l));
-				if (l && (!i || ends[i].finger[l] != input[i].header))
-					ends[i].finger[l]->calcVersionForLevel(l);
-				input[i + 1].header->setNext(l, nullptr);
-			}
-		}
-		swap(input[0]);
-	}
-
 	void find(const StringRef* values, Finger* results, int* temp, int count) {
 		// Relying on the ordering of values, descend until the values aren't all in the
 		// same part of the tree
@@ -786,7 +751,7 @@ class SkipList : NonCopyable {
 
 struct ConflictSet {
 	ConflictSet() : removalKey(makeString(0)), oldestVersion(0) {}
-	~ConflictSet() {}
+	~ConflictSet() = default;
 
 	SkipList versionHistory;
 	Key removalKey;
@@ -809,7 +774,7 @@ ConflictBatch::ConflictBatch(ConflictSet* cs,
   : cs(cs), transactionCount(0), conflictingKeyRangeMap(conflictingKeyRangeMap),
     resolveBatchReplyArena(resolveBatchReplyArena) {}
 
-ConflictBatch::~ConflictBatch() {}
+ConflictBatch::~ConflictBatch() = default;
 
 struct TransactionInfo {
 	VectorRef<std::pair<int, int>> readRanges;
@@ -834,9 +799,9 @@ void ConflictBatch::addTransaction(const CommitTransactionRef& tr, Version newOl
 	const int t = transactionCount++;
 
 	Arena& arena = transactionInfo.arena();
-	TransactionInfo* info = new (arena) TransactionInfo;
+	auto* info = new (arena) TransactionInfo;
 	info->reportConflictingKeys = tr.report_conflicting_keys;
-	bool tooOld = tr.read_snapshot < newOldestVersion && tr.read_conflict_ranges.size();
+	bool tooOld = tr.read_snapshot < newOldestVersion && !tr.read_conflict_ranges.empty();
 	if (tooOld && ignoreTooOld()) {
 		bugs->hit();
 		tooOld = false;
diff --git a/fdbserver/include/fdbserver/ConflictSet.h b/fdbserver/resolver/ConflictSet.h
similarity index 92%
rename from fdbserver/include/fdbserver/ConflictSet.h
rename to fdbserver/resolver/ConflictSet.h
index be81985d629..554f380ccea 100644
--- a/fdbserver/include/fdbserver/ConflictSet.h
+++ b/fdbserver/resolver/ConflictSet.h
@@ -26,27 +26,20 @@
 #include <vector>
 
 #include "fdbclient/CommitTransaction.h"
-#include "fdbserver/ResolverBug.h"
+#include "fdbserver/core/ConflictBatch.h"
+#include "fdbserver/resolver/ResolverBug.h"
 
 struct ConflictSet;
 ConflictSet* newConflictSet();
 void clearConflictSet(ConflictSet*, Version);
 void destroyConflictSet(ConflictSet*);
 
-struct ConflictBatch {
+struct ConflictBatch : ConflictBatchStatus {
 	explicit ConflictBatch(ConflictSet*,
 	                       std::map<int, VectorRef<int>>* conflictingKeyRangeMap = nullptr,
 	                       Arena* resolveBatchReplyArena = nullptr);
 	~ConflictBatch();
 
-	enum TransactionCommitResult {
-		TransactionConflict = 0,
-		TransactionTooOld,
-		TransactionUnusedResultValue1,
-		TransactionCommitted,
-		TransactionLockReject,
-	};
-
 	void addTransaction(const CommitTransactionRef& transaction, Version newOldestVersion);
 	void detectConflicts(Version now,
 	                     Version newOldestVersion,
diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/resolver/Resolver.cpp
similarity index 81%
rename from fdbserver/Resolver.actor.cpp
rename to fdbserver/resolver/Resolver.cpp
index 977435975a6..93d5e9e3d91 100644
--- a/fdbserver/Resolver.actor.cpp
+++ b/fdbserver/resolver/Resolver.cpp
@@ -1,5 +1,5 @@
 /*
- * Resolver.actor.cpp
+ * Resolver.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,24 +26,26 @@
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/SystemData.h"
 #include "fdbrpc/Stats.h"
-#include "fdbserver/ApplyMetadataMutation.h"
-#include "fdbserver/ConflictSet.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/LogSystemDiskQueueAdapter.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/ResolverInterface.h"
-#include "fdbserver/RestoreUtil.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/logsystem/ApplyMetadataMutation.h"
+#include "fdbserver/core/ConflictBatch.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/logsystem/LogSystemDiskQueueAdapter.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/ResolverInterface.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/StorageMetrics.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/resolver/Resolver.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "ConflictSet.h"
 #include "flow/ActorCollection.h"
 #include "flow/Error.h"
 #include "flow/Histogram.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 namespace {
 struct ProxyRequestsInfo {
@@ -141,7 +143,7 @@ struct Resolver : ReferenceCounted<Resolver> {
 	// happens at commit proxies and we never "write" to the LogSystem at
 	// Resolvers.
 	LogSystemDiskQueueAdapter* logAdapter = nullptr;
-	Reference<ILogSystem> logSystem;
+	Reference<LogSystem> logSystem;
 	IKeyValueStore* txnStateStore = nullptr;
 	int localTLogCount = -1;
 
@@ -219,8 +221,8 @@ struct Resolver : ReferenceCounted<Resolver> {
 };
 } // namespace
 
-ACTOR Future<Void> versionReady(Resolver* self, ProxyRequestsInfo* proxyInfo, Version prevVersion) {
-	loop {
+Future<Void> versionReady(Resolver* self, ProxyRequestsInfo* proxyInfo, Version prevVersion) {
+	while (true) {
 		if (self->recentStateTransactionsInfo.size() &&
 		    proxyInfo->lastVersion <= self->recentStateTransactionsInfo.firstVersion()) {
 			self->neededVersion.set(std::max(self->neededVersion.get(), prevVersion));
@@ -233,13 +235,13 @@ ACTOR Future<Void> versionReady(Resolver* self, ProxyRequestsInfo* proxyInfo, Ve
 		}
 		self->queueDepthDist->sampleRecordCounter(waiters);
 
-		choose {
-			when(wait(self->version.whenAtLeast(prevVersion))) {
-				// Update queue depth metric after waiting.
-				self->queueDepthDist->sampleRecordCounter(self->version.numWaiting());
-				return Void();
-			}
-			when(wait(self->checkNeededVersion.onTrigger())) {}
+		auto res = co_await race(self->version.whenAtLeast(prevVersion), self->checkNeededVersion.onTrigger());
+		if (res.index() == 0) {
+			// Update queue depth metric after waiting.
+			self->queueDepthDist->sampleRecordCounter(self->version.numWaiting());
+			co_return;
+		} else if (res.index() != 1) {
+			UNREACHABLE();
 		}
 	}
 }
@@ -256,16 +258,13 @@ static bool hasNonLogRouterTags(const std::set<Tag>& allTags) {
 	return false;
 }
 
-ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
-                                ResolveTransactionBatchRequest req,
-                                Reference<AsyncVar<ServerDBInfo> const> db) {
-	state Optional<UID> debugID;
-	state Span span("R:resolveBatch"_loc, req.spanContext);
+Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatchRequest req) {
+	Optional<UID> debugID;
+	Span span("R:resolveBatch"_loc, req.spanContext);
 
 	// The first request (prevVersion < 0) comes from the master
-	state NetworkAddress proxyAddress =
-	    req.prevVersion >= 0 ? req.reply.getEndpoint().getPrimaryAddress() : NetworkAddress();
-	state ProxyRequestsInfo& proxyInfo = self->proxyInfoMap[proxyAddress];
+	NetworkAddress proxyAddress = req.prevVersion >= 0 ? req.reply.getEndpoint().getPrimaryAddress() : NetworkAddress();
+	ProxyRequestsInfo& proxyInfo = self->proxyInfoMap[proxyAddress];
 
 	++self->resolveBatchIn;
 
@@ -290,17 +289,17 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
 	 .detail("LastVersion", proxyInfo.lastVersion).detail("RequestVersion", req.version).detail("NeededVersion",
 		     self->neededVersion.get()) .detail("RecentStateVer", self->recentStateTransactionsInfo.firstVersion());*/
 
-		wait(self->totalStateBytes.onChange() || self->neededVersion.onChange());
+		co_await (self->totalStateBytes.onChange() || self->neededVersion.onChange());
 	}
 
 	if (debugID.present()) {
 		g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "Resolver.resolveBatch.AfterQueueSizeCheck");
 	}
 
-	wait(versionReady(self.getPtr(), &proxyInfo, req.prevVersion));
+	co_await versionReady(self.getPtr(), &proxyInfo, req.prevVersion);
 
 	if (check_yield(TaskPriority::DefaultEndpoint)) {
-		wait(delay(0, TaskPriority::Low) || delay(SERVER_KNOBS->COMMIT_SLEEP_TIME)); // FIXME: Is this still right?
+		co_await (delay(0, TaskPriority::Low) || delay(SERVER_KNOBS->COMMIT_SLEEP_TIME)); // FIXME: Is this still right?
 		g_network->setCurrentTask(TaskPriority::DefaultEndpoint);
 	}
 
@@ -358,11 +357,11 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
 		reply.debugID = req.debugID;
 		reply.committed.resize(reply.arena, req.transactions.size());
 		for (int c = 0; c < commitList.size(); c++)
-			reply.committed[commitList[c]] = ConflictBatch::TransactionCommitted;
+			reply.committed[commitList[c]] = ConflictBatchStatus::TransactionCommitted;
 
 		for (int c = 0; c < tooOldList.size(); c++) {
-			ASSERT(reply.committed[tooOldList[c]] == ConflictBatch::TransactionConflict);
-			reply.committed[tooOldList[c]] = ConflictBatch::TransactionTooOld;
+			ASSERT(reply.committed[tooOldList[c]] == ConflictBatchStatus::TransactionConflict);
+			reply.committed[tooOldList[c]] = ConflictBatchStatus::TransactionTooOld;
 		}
 
 		self->transactionsAccepted += commitList.size();
@@ -403,7 +402,7 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
 			stateBytes += req.transactions[t].mutations.expectedSize();
 			stateTransactions.push_back_deep(
 			    stateTransactions.arena(),
-			    StateTransactionRef(reply.committed[t] == ConflictBatch::TransactionCommitted,
+			    StateTransactionRef(reply.committed[t] == ConflictBatchStatus::TransactionCommitted,
 			                        req.transactions[t].mutations));
 
 			// for (const auto& m : req.transactions[t].mutations)
@@ -411,7 +410,7 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
 
 			// Generate private mutations for metadata mutations
 			// The condition here must match CommitBatch::applyMetadataToCommittedTransactions()
-			if (reply.committed[t] == ConflictBatch::TransactionCommitted && !self->forceRecovery &&
+			if (reply.committed[t] == ConflictBatchStatus::TransactionCommitted && !self->forceRecovery &&
 			    shouldApplyResolverPrivateMutations && (!isLocked || req.transactions[t].lock_aware)) {
 				SpanContext spanContext =
 				    req.transactions[t].spanContext.present() ? req.transactions[t].spanContext.get() : SpanContext();
@@ -558,13 +557,48 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self,
 	self->resolverLatencyDist->sampleSeconds(endTime - req.requestTime());
 
 	++self->resolveBatchOut;
+}
 
-	return Void();
+Future<Void> resolveBatches(Reference<Resolver> self,
+                            FutureStream<ResolveTransactionBatchRequest> resolveRequests,
+                            ActorCollection* actors) {
+	while (true) {
+		ResolveTransactionBatchRequest batch = co_await resolveRequests;
+		actors->add(resolveBatch(self, batch));
+	}
+}
+
+Future<Void> resolveMetricsRequests(Reference<Resolver> self, FutureStream<ResolutionMetricsRequest> metricsRequests) {
+	while (true) {
+		ResolutionMetricsRequest req = co_await metricsRequests;
+		++self->metricsRequests;
+		req.reply.send(
+		    self->iopsSample.getEstimate(SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? normalKeys : allKeys));
+	}
+}
+
+Future<Void> resolveSplitRequests(Reference<Resolver> self, FutureStream<ResolutionSplitRequest> splitRequests) {
+	while (true) {
+		ResolutionSplitRequest req = co_await splitRequests;
+		++self->splitRequests;
+		ResolutionSplitReply rep;
+		rep.key = self->iopsSample.splitEstimate(req.range, req.offset, req.front);
+		rep.used = self->iopsSample.getEstimate(req.front ? KeyRangeRef(req.range.begin, rep.key)
+		                                                  : KeyRangeRef(rep.key, req.range.end));
+		req.reply.send(rep);
+	}
+}
+
+Future<Void> pollMetrics(Reference<Resolver> self) {
+	while (true) {
+		self->iopsSample.poll();
+		co_await delay(SERVER_KNOBS->SAMPLE_POLL_TIME);
+	}
 }
 
 namespace {
 
-// TODO: refactor with the one in CommitProxyServer.actor.cpp
+// TODO: refactor with the one in CommitProxyServer.cpp
 struct TransactionStateResolveContext {
 	// Maximum sequence for txnStateRequest, this is defined when the request last flag is set.
 	Sequence maxSequence = std::numeric_limits<Sequence>::max();
@@ -593,19 +627,17 @@ struct TransactionStateResolveContext {
 	}
 };
 
-ACTOR Future<Void> processCompleteTransactionStateRequest(Reference<Resolver> self,
-                                                          TransactionStateResolveContext* pContext,
-                                                          Reference<AsyncVar<ServerDBInfo> const> db) {
-	state KeyRange txnKeys = allKeys;
-	state std::map<Tag, UID> tag_uid;
+Future<Void> processCompleteTransactionStateRequest(TransactionStateResolveContext* pContext) {
+	KeyRange txnKeys = allKeys;
+	std::map<Tag, UID> tag_uid;
 
 	RangeResult UIDtoTagMap = pContext->pTxnStateStore->readRange(serverTagKeys).get();
 	for (const KeyValueRef& kv : UIDtoTagMap) {
 		tag_uid[decodeServerTagValue(kv.value)] = decodeServerTagKey(kv.key);
 	}
 
-	loop {
-		wait(yield());
+	while (true) {
+		co_await yield();
 
 		RangeResult data =
 		    pContext->pTxnStateStore
@@ -616,11 +648,10 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(Reference<Resolver> se
 
 		((KeyRangeRef&)txnKeys) = KeyRangeRef(keyAfter(data.back().key, txnKeys.arena()), txnKeys.end);
 
-		MutationsVec mutations;
+		Standalone<VectorRef<MutationRef>> mutations;
 		std::vector<std::pair<MapPair<Key, ServerCacheInfo>, int>> keyInfoData;
 		std::vector<UID> src, dest;
 		ServerCacheInfo info;
-		// NOTE: An ACTOR will be compiled into several classes, the this pointer is from one of them.
 		auto updateTagInfo = [pContext = pContext](const std::vector<UID>& uids,
 		                                           std::vector<Tag>& tags,
 		                                           std::vector<Reference<StorageInfo>>& storageInfoItems) {
@@ -663,29 +694,24 @@ ACTOR Future<Void> processCompleteTransactionStateRequest(Reference<Resolver> se
 		ResolverData resolverData(
 		    pContext->pResolverData->dbgid, pContext->pTxnStateStore, &pContext->pResolverData->keyInfo, confChanges);
 		applyMetadataMutations(SpanContext(), resolverData, mutations);
-	} // loop
+	}
 
-	auto lockedKey = pContext->pTxnStateStore->readValue(databaseLockedKey).get();
+	[[maybe_unused]] auto lockedKey = pContext->pTxnStateStore->readValue(databaseLockedKey).get();
 	// pContext->pCommitData->locked = lockedKey.present() && lockedKey.get().size();
 	// pContext->pCommitData->metadataVersion = pContext->pTxnStateStore->readValue(metadataVersionKey).get();
 
 	pContext->pTxnStateStore->enableSnapshot();
-
-	return Void();
 }
 
-ACTOR Future<Void> processTransactionStateRequestPart(Reference<Resolver> self,
-                                                      TransactionStateResolveContext* pContext,
-                                                      TxnStateRequest request,
-                                                      Reference<AsyncVar<ServerDBInfo> const> db) {
+Future<Void> processTransactionStateRequestPart(TransactionStateResolveContext* pContext, TxnStateRequest request) {
 	ASSERT(pContext->pResolverData.getPtr() != nullptr);
 	ASSERT(pContext->pActors != nullptr);
 
 	if (pContext->receivedSequences.contains(request.sequence)) {
 		// This part is already received. Still we will re-broadcast it to other CommitProxies & Resolvers
 		pContext->pActors->send(broadcastTxnRequest(request, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
-		wait(yield());
-		return Void();
+		co_await yield();
+		co_return;
 	}
 
 	if (request.last) {
@@ -704,24 +730,32 @@ ACTOR Future<Void> processTransactionStateRequestPart(Reference<Resolver> self,
 	if (pContext->receivedSequences.size() == pContext->maxSequence) {
 		// Received all components of the txnStateRequest
 		ASSERT(!pContext->processed);
-		wait(processCompleteTransactionStateRequest(self, pContext, db));
+		co_await processCompleteTransactionStateRequest(pContext);
 		pContext->processed = true;
 	}
 
 	pContext->pActors->send(broadcastTxnRequest(request, SERVER_KNOBS->TXN_STATE_SEND_AMOUNT, true));
-	wait(yield());
-	return Void();
+	co_await yield();
+}
+
+Future<Void> resolveTxnStateRequests(FutureStream<TxnStateRequest> txnStateRequests,
+                                     TransactionStateResolveContext* pContext,
+                                     PromiseStream<Future<Void>>* addActor) {
+	while (true) {
+		TxnStateRequest request = co_await txnStateRequests;
+		ASSERT(SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS);
+		addActor->send(processTransactionStateRequestPart(pContext, request));
+	}
 }
 
 } // anonymous namespace
 
-ACTOR Future<Void> resolverCore(ResolverInterface resolver,
-                                InitializeResolverRequest initReq,
-                                Reference<AsyncVar<ServerDBInfo> const> db) {
-	state Reference<Resolver> self(new Resolver(resolver.id(), initReq.commitProxyCount, initReq.resolverCount));
-	state ActorCollection actors(false);
-	state Future<Void> doPollMetrics = self->resolverCount > 1 ? Void() : Future<Void>(Never());
-	state PromiseStream<Future<Void>> addActor;
+Future<Void> resolverCore(ResolverInterface resolver,
+                          InitializeResolverRequest initReq,
+                          Reference<AsyncVar<ServerDBInfo> const> db) {
+	Reference<Resolver> self(new Resolver(resolver.id(), initReq.commitProxyCount, initReq.resolverCount));
+	ActorCollection actors(false);
+	PromiseStream<Future<Void>> addActor;
 	actors.add(waitFailureServer(resolver.waitFailure.getFuture()));
 	actors.add(traceRole(Role::RESOLVER, resolver.id()));
 
@@ -731,22 +765,21 @@ ACTOR Future<Void> resolverCore(ResolverInterface resolver,
 	while (!(initReq.masterLifetime.isEqual(db->get().masterLifetime) &&
 	         db->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION)) {
 		// TraceEvent("ResolverInit2", resolver.id()).detail("LSEpoch", db->get().logSystemConfig.epoch);
-		wait(db->onChange());
+		co_await db->onChange();
 	}
 
 	// Initialize txnStateStore
-	self->logSystem = ILogSystem::fromServerDBInfo(resolver.id(), db->get(), false, addActor);
+	self->logSystem = makeLogSystemFromServerDBInfo(resolver.id(), db->get(), false, addActor);
 	self->localTLogCount = db->get().logSystemConfig.numLogs();
-	state Future<Void> onError =
-	    transformError(actorCollection(addActor.getFuture()), broken_promise(), resolver_failed());
-	state TransactionStateResolveContext transactionStateResolveContext;
+	Future<Void> onError = transformError(actorCollection(addActor.getFuture()), broken_promise(), resolver_failed());
+	TransactionStateResolveContext transactionStateResolveContext;
 	if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS) {
 		self->logAdapter = new LogSystemDiskQueueAdapter(self->logSystem, Reference<AsyncVar<PeekTxsInfo>>(), 1, false);
 		self->txnStateStore = keyValueStoreLogSystem(
-		    self->logAdapter, db, resolver.id(), /*doc=*/2e9, /*u=*/true, /*ment=*/true, /*constants=*/true);
+		    self->logAdapter, db, resolver.id(), 2e9, DisableSnapshot::True, ReplaceContent::True, ExactRecovery::True);
 
 		// wait for txnStateStore recovery
-		wait(success(self->txnStateStore->readValue(StringRef())));
+		co_await self->txnStateStore->readValue(StringRef());
 
 		// This has to be declared after the self->txnStateStore get initialized
 		transactionStateResolveContext = TransactionStateResolveContext(self, &addActor);
@@ -758,67 +791,44 @@ ACTOR Future<Void> resolverCore(ResolverInterface resolver,
 		}
 	}
 
-	loop choose {
-		when(ResolveTransactionBatchRequest batch = waitNext(resolver.resolve.getFuture())) {
-			actors.add(resolveBatch(self, batch, db));
-		}
-		when(ResolutionMetricsRequest req = waitNext(resolver.metrics.getFuture())) {
-			++self->metricsRequests;
-			req.reply.send(self->iopsSample.getEstimate(SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? normalKeys
-			                                                                                               : allKeys));
-		}
-		when(ResolutionSplitRequest req = waitNext(resolver.split.getFuture())) {
-			++self->splitRequests;
-			ResolutionSplitReply rep;
-			rep.key = self->iopsSample.splitEstimate(req.range, req.offset, req.front);
-			rep.used = self->iopsSample.getEstimate(req.front ? KeyRangeRef(req.range.begin, rep.key)
-			                                                  : KeyRangeRef(rep.key, req.range.end));
-			req.reply.send(rep);
-		}
-		when(wait(actors.getResult())) {}
-		when(wait(onError)) {}
-		when(wait(doPollMetrics)) {
-			self->iopsSample.poll();
-			doPollMetrics = delay(SERVER_KNOBS->SAMPLE_POLL_TIME);
-		}
-		when(TxnStateRequest request = waitNext(resolver.txnState.getFuture())) {
-			if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS) {
-				addActor.send(processTransactionStateRequestPart(self, &transactionStateResolveContext, request, db));
-			} else {
-				ASSERT(false);
-			}
-		}
+	actors.add(resolveBatches(self, resolver.resolve.getFuture(), &actors));
+	actors.add(resolveMetricsRequests(self, resolver.metrics.getFuture()));
+	actors.add(resolveSplitRequests(self, resolver.split.getFuture()));
+	actors.add(resolveTxnStateRequests(resolver.txnState.getFuture(), &transactionStateResolveContext, &addActor));
+	if (self->resolverCount > 1) {
+		actors.add(pollMetrics(self));
 	}
+
+	co_await race(actors.getResult(), onError);
+	// The above futures should never be ready
+	UNSTOPPABLE_ASSERT(false);
 }
 
-ACTOR Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db,
-                                uint64_t recoveryCount,
-                                ResolverInterface myInterface) {
-	loop {
+Future<Void> checkRemoved(Reference<AsyncVar<ServerDBInfo> const> db,
+                          uint64_t recoveryCount,
+                          ResolverInterface myInterface) {
+	while (true) {
 		if (db->get().recoveryCount >= recoveryCount &&
 		    std::find(db->get().resolvers.begin(), db->get().resolvers.end(), myInterface) ==
 		        db->get().resolvers.end()) {
 			throw worker_removed();
 		}
-		wait(db->onChange());
+		co_await db->onChange();
 	}
 }
 
-ACTOR Future<Void> resolver(ResolverInterface resolver,
-                            InitializeResolverRequest initReq,
-                            Reference<AsyncVar<ServerDBInfo> const> db) {
+Future<Void> resolver(ResolverInterface resolver,
+                      InitializeResolverRequest initReq,
+                      Reference<AsyncVar<ServerDBInfo> const> db) {
 	try {
-		state Future<Void> core = resolverCore(resolver, initReq, db);
-		loop choose {
-			when(wait(core)) {
-				return Void();
-			}
-			when(wait(checkRemoved(db, initReq.recoveryCount, resolver))) {}
+		Future<Void> core = resolverCore(resolver, initReq, db);
+		while (true) {
+			co_await waitOrError(checkRemoved(db, initReq.recoveryCount, resolver), core);
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled || e.code() == error_code_worker_removed) {
 			TraceEvent("ResolverTerminated", resolver.id()).errorUnsuppressed(e);
-			return Void();
+			co_return;
 		}
 		throw;
 	}
diff --git a/fdbserver/ResolverBug.cpp b/fdbserver/resolver/ResolverBug.cpp
similarity index 94%
rename from fdbserver/ResolverBug.cpp
rename to fdbserver/resolver/ResolverBug.cpp
index 6aafd8d89d0..30e8a80e354 100644
--- a/fdbserver/ResolverBug.cpp
+++ b/fdbserver/resolver/ResolverBug.cpp
@@ -17,7 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "fdbserver/ResolverBug.h"
+#include "fdbserver/resolver/ResolverBug.h"
 
 std::shared_ptr<ISimBug> ResolverBugID::create() const {
 	return std::make_shared<ResolverBug>();
diff --git a/fdbserver/resolver/include/fdbserver/resolver/Resolver.h b/fdbserver/resolver/include/fdbserver/resolver/Resolver.h
new file mode 100644
index 00000000000..16b08fe86da
--- /dev/null
+++ b/fdbserver/resolver/include/fdbserver/resolver/Resolver.h
@@ -0,0 +1,31 @@
+/*
+ * Resolver.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/ResolverInterface.h"
+#include "flow/flow.h"
+
+struct InitializeResolverRequest;
+struct ServerDBInfo;
+
+Future<Void> resolver(ResolverInterface resolver,
+                      InitializeResolverRequest initReq,
+                      Reference<AsyncVar<ServerDBInfo> const> db);
diff --git a/fdbserver/include/fdbserver/ResolverBug.h b/fdbserver/resolver/include/fdbserver/resolver/ResolverBug.h
similarity index 90%
rename from fdbserver/include/fdbserver/ResolverBug.h
rename to fdbserver/resolver/include/fdbserver/resolver/ResolverBug.h
index 62bcea934b3..41d008c7e22 100644
--- a/fdbserver/include/fdbserver/ResolverBug.h
+++ b/fdbserver/resolver/include/fdbserver/resolver/ResolverBug.h
@@ -18,8 +18,8 @@
  * limitations under the License.
  */
 
-#ifndef FDBSERVER_RESOLVER_BUG_H
-#define FDBSERVER_RESOLVER_BUG_H
+#ifndef FDBSERVER_RESOLVER_RESOLVER_BUG_H
+#define FDBSERVER_RESOLVER_RESOLVER_BUG_H
 #pragma once
 
 #include "flow/SimBugInjector.h"
@@ -41,4 +41,4 @@ class ResolverBugID : public IBugIdentifier {
 	std::shared_ptr<ISimBug> create() const override;
 };
 
-#endif // FDBSERVER_RESOLVER_BUG_H
+#endif // FDBSERVER_RESOLVER_RESOLVER_BUG_H
diff --git a/fdbserver/sequencer/CMakeLists.txt b/fdbserver/sequencer/CMakeLists.txt
new file mode 100644
index 00000000000..bce80b1d4ba
--- /dev/null
+++ b/fdbserver/sequencer/CMakeLists.txt
@@ -0,0 +1,14 @@
+fdb_find_sources(FDBSERVER_SEQUENCER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_sequencer SRCS ${FDBSERVER_SEQUENCER_SRCS})
+add_fdbserver_link_test(fdbserver_sequencerlinktest
+  fdbserver_sequencer
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_sequencer)
+target_include_directories(fdbserver_sequencer
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_sequencer PRIVATE fdbserver_core)
diff --git a/fdbserver/include/fdbserver/MasterData.actor.h b/fdbserver/sequencer/MasterData.h
similarity index 50%
rename from fdbserver/include/fdbserver/MasterData.actor.h
rename to fdbserver/sequencer/MasterData.h
index d74d5af0cbb..3b37d535a19 100644
--- a/fdbserver/include/fdbserver/MasterData.actor.h
+++ b/fdbserver/sequencer/MasterData.h
@@ -1,5 +1,5 @@
 /*
- * MasterData.actor.h
+ * MasterData.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,43 +23,17 @@
 #include <map>
 
 #include "fdbrpc/sim_validation.h"
-#include "fdbserver/CoordinatedState.h"
-#include "fdbserver/CoordinationInterface.h" // copy constructors for ServerCoordinators class
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/ResolutionBalancer.actor.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/CoordinatedState.h"
+#include "fdbserver/core/CoordinationInterface.h" // copy constructors for ServerCoordinators class
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "ResolutionBalancer.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "flow/ActorCollection.h"
 #include "flow/Trace.h"
-#include "flow/swift_support.h"
 #include "fdbclient/VersionVector.h"
 
-#ifdef WITH_SWIFT
-#ifndef FDBSERVER_FORWARD_DECLARE_SWIFT_APIS
-// Forward declare C++ MasterData type.
-struct MasterData;
-
-#include "SwiftModules/FDBServer"
-#endif
-#endif // WITH_SWIFT
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_MASTERDATA_ACTOR_G_H)
-#define FDBSERVER_MASTERDATA_ACTOR_G_H
-#include "fdbserver/MasterData.actor.g.h"
-#elif !defined(FDBSERVER_MASTERDATA_ACTOR_H)
-#define FDBSERVER_MASTERDATA_ACTOR_H
-#include "flow/actorcompiler.h" // This must be the last #include
-
-// FIXME(swift): Remove once https://github.com/apple/swift/issues/61620 is fixed.
-#define SWIFT_CXX_REF_MASTERDATA                                                                                       \
-	__attribute__((swift_attr("import_reference"))) __attribute__((swift_attr("retain:addrefMasterData")))             \
-	__attribute__((swift_attr("release:delrefMasterData")))
-
-// A type with Swift value semantics for working with `Counter` types.
 class CounterValue {
-	// FIXME(swift): Delete immortal annotation from `Counter`.
 public:
 	using Value = Counter::Value;
 
@@ -73,20 +47,7 @@ class CounterValue {
 	std::shared_ptr<Counter> value;
 };
 
-// A concrete Optional<Version> type that can be referenced in Swift.
-using OptionalVersion = Optional<Version>;
-
-#ifdef WITH_SWIFT
-#ifdef FDBSERVER_FORWARD_DECLARE_SWIFT_APIS
-// Forward declare the Swift actor.
-namespace fdbserver_swift {
-class MasterDataActor;
-}
-#endif
-#endif
-
-// FIXME (after the one below): Use SWIFT_CXX_REF once https://github.com/apple/swift/issues/61620 is fixed.
-struct SWIFT_CXX_REF_MASTERDATA MasterData : NonCopyable, ReferenceCounted<MasterData> {
+struct MasterData : NonCopyable, ReferenceCounted<MasterData> {
 	UID dbgid;
 
 	Version lastEpochEnd, // The last version in the old epoch not (to be) rolled back in this recovery
@@ -105,7 +66,6 @@ struct SWIFT_CXX_REF_MASTERDATA MasterData : NonCopyable, ReferenceCounted<Maste
 	double lastVersionTime;
 	Optional<Version> referenceVersion;
 
-	// When using Swift master server impl this is declared in Swift.
 	std::map<UID, CommitProxyVersionReplies> lastCommitProxyVersionReplies;
 
 	MasterInterface myInterface;
@@ -138,19 +98,6 @@ struct SWIFT_CXX_REF_MASTERDATA MasterData : NonCopyable, ReferenceCounted<Maste
 	Future<Void> logger;
 	Future<Void> balancer;
 
-#ifdef WITH_SWIFT
-	std::unique_ptr<fdbserver_swift::MasterDataActor> swiftImpl;
-
-#ifndef FDBSERVER_FORWARD_DECLARE_SWIFT_APIS
-	// This function is not available to Swift as the FDBServer header is
-	// being generated, as we do not yet have `MasterDataActor` type definition
-	// in C++.
-	inline fdbserver_swift::MasterDataActor getSwiftImpl() const { return *swiftImpl; }
-#endif
-
-	void setSwiftImpl(fdbserver_swift::MasterDataActor* impl);
-#endif
-
 	MasterData(Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
 	           MasterInterface const& myInterface,
 	           ServerCoordinators const& coordinators,
@@ -160,27 +107,4 @@ struct SWIFT_CXX_REF_MASTERDATA MasterData : NonCopyable, ReferenceCounted<Maste
 	           bool forceRecovery);
 
 	~MasterData();
-
-	// FIXME(swift): return a reference once https://github.com/apple/swift/issues/64315 is fixed.
-	inline ResolutionBalancer* getResolutionBalancer() { return &resolutionBalancer; }
 };
-
-// FIXME(swift): Remove once https://github.com/apple/swift/issues/61620 is fixed.
-inline void addrefMasterData(MasterData* ptr) {
-	addref(ptr);
-}
-
-// FIXME: Remove once https://github.com/apple/swift/issues/61620 is fixed.
-inline void delrefMasterData(MasterData* ptr) {
-	delref(ptr);
-}
-
-using ReferenceMasterData = Reference<MasterData>;
-
-using StdVectorOfUIDs = std::vector<UID>;
-
-// FIXME: Workaround for linker issue (rdar://101092732).
-void swift_workaround_setLatestRequestNumber(NotifiedVersion& latestRequestNum, Version v);
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/ResolutionBalancer.actor.cpp b/fdbserver/sequencer/ResolutionBalancer.cpp
similarity index 81%
rename from fdbserver/ResolutionBalancer.actor.cpp
rename to fdbserver/sequencer/ResolutionBalancer.cpp
index 3ad41ec708b..182f8ed54a4 100644
--- a/fdbserver/ResolutionBalancer.actor.cpp
+++ b/fdbserver/sequencer/ResolutionBalancer.cpp
@@ -1,5 +1,5 @@
 /*
- * ResolutionBalancer.actor.cpp
+ * ResolutionBalancer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbserver/ResolutionBalancer.actor.h"
+#include "ResolutionBalancer.h"
 
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/flow.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 void ResolutionBalancer::setResolvers(const std::vector<ResolverInterface>& v) {
 	resolvers = v;
 	if (resolvers.size() > 1)
@@ -112,22 +110,22 @@ static std::pair<KeyRangeRef, bool> findRange(CoalescedKeyRangeMap<int>& key_res
 }
 
 // Balance key ranges among resolvers so that their load are evenly distributed.
-ACTOR Future<Void> ResolutionBalancer::resolutionBalancing_impl(ResolutionBalancer* self) {
-	wait(self->triggerResolution.onTrigger());
+Future<Void> ResolutionBalancer::resolutionBalancing_impl(ResolutionBalancer* self) {
+	co_await self->triggerResolution.onTrigger();
 
-	state CoalescedKeyRangeMap<int> key_resolver(
+	CoalescedKeyRangeMap<int> key_resolver(
 	    0, SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? normalKeys.end : allKeys.end);
 	key_resolver.insert(SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS ? normalKeys : allKeys, 0);
-	loop {
-		wait(delay(SERVER_KNOBS->MIN_BALANCE_TIME, TaskPriority::ResolutionMetrics));
-		while (self->resolverChanges.get().size())
-			wait(self->resolverChanges.onChange());
-		state std::vector<Future<ResolutionMetricsReply>> futures;
+	while (true) {
+		co_await delay(SERVER_KNOBS->MIN_BALANCE_TIME, TaskPriority::ResolutionMetrics);
+		while (!self->resolverChanges.get().empty())
+			co_await self->resolverChanges.onChange();
+		std::vector<Future<ResolutionMetricsReply>> futures;
 		for (auto& p : self->resolvers)
 			futures.push_back(
 			    brokenPromiseToNever(p.metrics.getReply(ResolutionMetricsRequest(), TaskPriority::ResolutionMetrics)));
-		wait(waitForAll(futures));
-		state IndexedSet<std::pair<int64_t, int>, NoMetric> metrics;
+		co_await waitForAll(futures);
+		IndexedSet<std::pair<int64_t, int>, NoMetric> metrics;
 
 		int64_t total = 0;
 		for (int i = 0; i < futures.size(); i++) {
@@ -137,15 +135,15 @@ ACTOR Future<Void> ResolutionBalancer::resolutionBalancing_impl(ResolutionBalanc
 		}
 		if (metrics.lastItem()->first - metrics.begin()->first > SERVER_KNOBS->MIN_BALANCE_DIFFERENCE) {
 			try {
-				state int src = metrics.lastItem()->second;
-				state int dest = metrics.begin()->second;
-				state int64_t amount = std::min(metrics.lastItem()->first - total / self->resolvers.size(),
-				                                total / self->resolvers.size() - metrics.begin()->first) /
-				                       2;
-				state Standalone<VectorRef<ResolverMoveRef>> movedRanges;
+				int src = metrics.lastItem()->second;
+				int dest = metrics.begin()->second;
+				int64_t amount = std::min(metrics.lastItem()->first - total / self->resolvers.size(),
+				                          total / self->resolvers.size() - metrics.begin()->first) /
+				                 2;
+				Standalone<VectorRef<ResolverMoveRef>> movedRanges;
 
-				loop {
-					state std::pair<KeyRangeRef, bool> range = findRange(key_resolver, movedRanges, src, dest);
+				while (true) {
+					std::pair<KeyRangeRef, bool> range = findRange(key_resolver, movedRanges, src, dest);
 
 					ResolutionSplitRequest req;
 					req.front = range.second;
@@ -153,8 +151,8 @@ ACTOR Future<Void> ResolutionBalancer::resolutionBalancing_impl(ResolutionBalanc
 					req.range = range.first;
 
 					ResolutionSplitReply split =
-					    wait(brokenPromiseToNever(self->resolvers[metrics.lastItem()->second].split.getReply(
-					        req, TaskPriority::ResolutionMetrics)));
+					    co_await brokenPromiseToNever(self->resolvers[metrics.lastItem()->second].split.getReply(
+					        req, TaskPriority::ResolutionMetrics));
 					KeyRangeRef moveRange = range.second ? KeyRangeRef(range.first.begin, split.key)
 					                                     : KeyRangeRef(split.key, range.first.end);
 					movedRanges.push_back_deep(movedRanges.arena(), ResolverMoveRef(moveRange, dest));
diff --git a/fdbserver/include/fdbserver/ResolutionBalancer.actor.h b/fdbserver/sequencer/ResolutionBalancer.h
similarity index 68%
rename from fdbserver/include/fdbserver/ResolutionBalancer.actor.h
rename to fdbserver/sequencer/ResolutionBalancer.h
index 86c4aad6e5e..907773e8745 100644
--- a/fdbserver/include/fdbserver/ResolutionBalancer.actor.h
+++ b/fdbserver/sequencer/ResolutionBalancer.h
@@ -1,5 +1,5 @@
 /*
- * ResolutionBalancer.actor.h
+ * ResolutionBalancer.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,25 +18,19 @@
  * limitations under the License.
  */
 
-#include "fdbclient/CommitProxyInterface.h"
-#include "fdbserver/ResolverInterface.h"
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_RESOLUTION_BALANCER_G_H)
-#define FDBSERVER_RESOLUTION_BALANCER_G_H
-#include "fdbserver/ResolutionBalancer.actor.g.h"
-#elif !defined(FDBSERVER_RESOLUTION_BALANCER_H)
-#define FDBSERVER_RESOLUTION_BALANCER_H
+#pragma once
 
+#include "fdbclient/CommitProxyInterface.h"
 #include <set>
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/core/ResolverInterface.h"
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // must be last include
 
-struct __attribute__((swift_attr("import_reference"))) __attribute__((swift_attr("retain:immortal")))
-__attribute__((swift_attr("release:immortal"))) ResolutionBalancer {
+struct ResolutionBalancer {
 	AsyncVar<Standalone<VectorRef<ResolverMoveRef>>> resolverChanges;
 	Version resolverChangesVersion = invalidVersion;
 	std::set<UID> resolverNeedingChanges;
@@ -47,11 +41,11 @@ __attribute__((swift_attr("release:immortal"))) ResolutionBalancer {
 	std::vector<ResolverInterface> resolvers;
 	AsyncTrigger triggerResolution;
 
-	ResolutionBalancer(Version* version) : pVersion(version) {}
+	explicit(false) ResolutionBalancer(Version* version) : pVersion(version) {}
 
 	Future<Void> resolutionBalancing() { return resolutionBalancing_impl(this); }
 
-	ACTOR static Future<Void> resolutionBalancing_impl(ResolutionBalancer* self);
+	static Future<Void> resolutionBalancing_impl(ResolutionBalancer* self);
 
 	// Sets resolver interfaces. Trigger resolutionBalancing() actor if more
 	// than one resolvers are present.
@@ -61,6 +55,3 @@ __attribute__((swift_attr("release:immortal"))) ResolutionBalancer {
 
 	void setChangesInReply(UID requestingProxy, GetCommitVersionReply& rep);
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/sequencer/include/fdbserver/sequencer/MasterServer.h b/fdbserver/sequencer/include/fdbserver/sequencer/MasterServer.h
new file mode 100644
index 00000000000..97ad8a617c2
--- /dev/null
+++ b/fdbserver/sequencer/include/fdbserver/sequencer/MasterServer.h
@@ -0,0 +1,35 @@
+/*
+ * MasterServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/CoordinationInterface.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "flow/flow.h"
+
+struct ClusterControllerFullInterface;
+struct ServerDBInfo;
+
+Future<Void> masterServer(MasterInterface mi,
+                          Reference<AsyncVar<ServerDBInfo> const> db,
+                          Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
+                          ServerCoordinators serverCoordinators,
+                          LifetimeToken lifetime,
+                          bool forceRecovery);
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/sequencer/masterserver.cpp
similarity index 52%
rename from fdbserver/masterserver.actor.cpp
rename to fdbserver/sequencer/masterserver.cpp
index dc1aa3705c0..63027c324b6 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/sequencer/masterserver.cpp
@@ -1,5 +1,5 @@
 /*
- * masterserver.actor.cpp
+ * masterserver.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,36 +22,30 @@
 #include <iterator>
 
 #include "fdbrpc/sim_validation.h"
-#include "fdbserver/CoordinatedState.h"
-#include "fdbserver/CoordinationInterface.h" // copy constructors for ServerCoordinators class
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MasterInterface.h"
-#include "fdbserver/ResolutionBalancer.actor.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/CoordinatedState.h"
+#include "fdbserver/core/CoordinationInterface.h" // copy constructors for ServerCoordinators class
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MasterInterface.h"
+#include "fdbserver/sequencer/MasterServer.h"
+#include "ResolutionBalancer.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
 #include "flow/Trace.h"
-#include "flow/swift_support.h"
 #include "fdbclient/VersionVector.h"
-#include "fdbserver/MasterData.actor.h"
+#include "MasterData.h"
 
-#ifdef WITH_SWIFT
-#include "SwiftModules/FDBServer"
-#endif
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// Instantiate MasterInterface related templates
 template class ReplyPromise<MasterInterface>;
 template struct NetSAV<MasterInterface>;
 
 void updateLiveCommittedVersion(Reference<MasterData> self, ReportRawCommittedVersionRequest req);
 
-Version figureVersionCxx(Version current,
-                         double now,
-                         Version reference,
-                         int64_t toAdd,
-                         double maxVersionRateModifier,
-                         int64_t maxVersionRateOffset) {
+Version figureVersion(Version current,
+                      double now,
+                      Version reference,
+                      int64_t toAdd,
+                      double maxVersionRateModifier,
+                      int64_t maxVersionRateOffset) {
 	// Versions should roughly follow wall-clock time, based on the
 	// system clock of the current machine and an FDB-specific epoch.
 	// Calculate the expected version and determine whether we need to
@@ -68,68 +62,18 @@ Version figureVersionCxx(Version current,
 	return std::clamp(expected, current + toAdd - maxOffset, current + toAdd + maxOffset);
 }
 
-#ifdef WITH_SWIFT
-Version figureVersion(Version current,
-                      double now,
-                      Version reference,
-                      int64_t toAdd,
-                      double maxVersionRateModifier,
-                      int64_t maxVersionRateOffset) {
-	auto impl = SERVER_KNOBS->FLOW_WITH_SWIFT ? fdbserver_swift::figureVersion : figureVersionCxx;
-	return impl(current, now, reference, toAdd, maxVersionRateModifier, maxVersionRateOffset);
-}
-#else
-Version figureVersion(Version current,
-                      double now,
-                      Version reference,
-                      int64_t toAdd,
-                      double maxVersionRateModifier,
-                      int64_t maxVersionRateOffset) {
-	return figureVersionCxx(current, now, reference, toAdd, maxVersionRateModifier, maxVersionRateOffset);
-}
-#endif
-
-#ifdef WITH_SWIFT
-SWIFT_ACTOR Future<Void> waitForPrev(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		auto future = self->swiftImpl->waitForPrev(self.getPtr(), req);
-		wait(future);
-	} else {
-		state double startTime = now();
-		wait(self->liveCommittedVersion.whenAtLeast(req.prevVersion.get()));
-		double latency = now() - startTime;
-		self->waitForPrevLatencies->addMeasurement(latency);
-		++self->waitForPrevCommitRequests;
-		updateLiveCommittedVersion(self, req);
-		req.reply.send(Void());
-	}
-
-	return Void();
-}
-#else
-ACTOR Future<Void> waitForPrev(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
-	state double startTime = now();
-	wait(self->liveCommittedVersion.whenAtLeast(req.prevVersion.get()));
+Future<Void> waitForPrev(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
+	double startTime = now();
+	co_await self->liveCommittedVersion.whenAtLeast(req.prevVersion.get());
 	self->waitForPrevLatencies->addMeasurement(now() - startTime);
 	++self->waitForPrevCommitRequests;
 	updateLiveCommittedVersion(self, req);
 	req.reply.send(Void());
-
-	return Void();
 }
-#endif
 
-#ifdef WITH_SWIFT
-SWIFT_ACTOR Future<Void> getVersionSwift(Reference<MasterData> self, GetCommitVersionRequest req) {
-	auto future = self->swiftImpl->getVersion(self.getPtr(), req);
-	wait(future);
-	return Void();
-}
-#endif
-
-ACTOR Future<Void> getVersionCxx(Reference<MasterData> self, GetCommitVersionRequest req) {
-	state Span span("M:getVersion"_loc, req.spanContext);
-	state std::map<UID, CommitProxyVersionReplies>::iterator proxyItr =
+Future<Void> getVersion(Reference<MasterData> self, GetCommitVersionRequest req) {
+	Span span("M:getVersion"_loc, req.spanContext);
+	std::map<UID, CommitProxyVersionReplies>::iterator proxyItr =
 	    self->lastCommitProxyVersionReplies.find(req.requestingProxy); // lastCommitProxyVersionReplies never changes
 
 	++self->getCommitVersionRequests;
@@ -137,13 +81,13 @@ ACTOR Future<Void> getVersionCxx(Reference<MasterData> self, GetCommitVersionReq
 	if (proxyItr == self->lastCommitProxyVersionReplies.end()) {
 		// Request from invalid proxy (e.g. from duplicate recruitment request)
 		req.reply.send(Never());
-		return Void();
+		co_return;
 	}
 
 	CODE_PROBE(proxyItr->second.latestRequestNum.get() < req.requestNum - 1,
 	           "Commit version request queued up",
 	           probe::decoration::rare);
-	wait(proxyItr->second.latestRequestNum.whenAtLeast(req.requestNum - 1));
+	co_await proxyItr->second.latestRequestNum.whenAtLeast(req.requestNum - 1);
 
 	auto itr = proxyItr->second.replies.find(req.requestNum);
 	if (itr != proxyItr->second.replies.end()) {
@@ -210,26 +154,7 @@ ACTOR Future<Void> getVersionCxx(Reference<MasterData> self, GetCommitVersionReq
 		ASSERT(proxyItr->second.latestRequestNum.get() == req.requestNum - 1);
 		proxyItr->second.latestRequestNum.set(req.requestNum);
 	}
-
-	return Void();
-}
-
-#ifdef WITH_SWIFT
-ACTOR Future<Void> getVersion(Reference<MasterData> self, GetCommitVersionRequest req) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		wait(getVersionSwift(self, req));
-		return Void();
-	} else {
-		wait(getVersionCxx(self, req));
-		return Void();
-	}
-}
-#else
-ACTOR Future<Void> getVersion(Reference<MasterData> self, GetCommitVersionRequest req) {
-	wait(getVersionCxx(self, req));
-	return Void();
 }
-#endif
 
 CounterValue::CounterValue(std::string const& name, CounterCollection& collection)
   : value(std::make_shared<Counter>(name, collection)) {}
@@ -283,65 +208,21 @@ MasterData::MasterData(Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
 		                                                       SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
 		                                                       SERVER_KNOBS->LATENCY_SKETCH_ACCURACY);
 	}
-
-#ifdef WITH_SWIFT
-	using namespace fdbserver_swift;
-	// FIXME(swift): can we make a cleaner init?
-	swiftImpl.reset(new MasterDataActor((const MasterDataActor&)MasterDataActor::init()));
-#endif
-}
-
-#ifdef WITH_SWIFT
-void MasterData::setSwiftImpl(fdbserver_swift::MasterDataActor* impl) {
-	swiftImpl.reset(impl);
 }
-#endif
 
 MasterData::~MasterData() {}
 
-ACTOR Future<Void> provideVersionsCxx(Reference<MasterData> self) {
-	state ActorCollection versionActors(false);
-
-	loop choose {
-		when(GetCommitVersionRequest req = waitNext(self->myInterface.getCommitVersion.getFuture())) {
-			versionActors.add(getVersion(self, req));
-		}
-		when(wait(versionActors.getResult())) {}
-	}
-}
-
-#ifdef WITH_SWIFT
-SWIFT_ACTOR Future<Void> provideVersionsSwift(Reference<MasterData> self) {
-	auto future = self->swiftImpl->provideVersions(self.getPtr());
-	wait(future);
-	return Void();
-}
-#endif
+Future<Void> provideVersions(Reference<MasterData> self) {
+	ActorCollection versionActors(false);
 
-#ifdef WITH_SWIFT
-ACTOR Future<Void> provideVersions(Reference<MasterData> self) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		wait(provideVersionsSwift(self));
-	} else {
-		wait(provideVersionsCxx(self));
+	while (true) {
+		GetCommitVersionRequest req =
+		    co_await waitOrError(self->myInterface.getCommitVersion.getFuture(), versionActors.getResult());
+		versionActors.add(getVersion(self, req));
 	}
-
-	return Void();
-}
-#else
-ACTOR Future<Void> provideVersions(Reference<MasterData> self) {
-	wait(provideVersionsCxx(self));
-	return Void();
-}
-#endif
-
-#ifdef WITH_SWIFT
-void updateLiveCommittedVersionSwift(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
-	fdbserver_swift::updateLiveCommittedVersion(self.getPtr(), req);
 }
-#endif
 
-void updateLiveCommittedVersionCxx(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
+void updateLiveCommittedVersion(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
 	self->minKnownCommittedVersion = std::max(self->minKnownCommittedVersion, req.minKnownCommittedVersion);
 
 	if (req.version > self->liveCommittedVersion.get()) {
@@ -365,95 +246,63 @@ void updateLiveCommittedVersionCxx(Reference<MasterData> self, ReportRawCommitte
 	++self->reportLiveCommittedVersionRequests;
 }
 
-#ifdef WITH_SWIFT
-void updateLiveCommittedVersion(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		return updateLiveCommittedVersionSwift(self, req);
-	} else {
-		return updateLiveCommittedVersionCxx(self, req);
-	}
-}
-#else
-void updateLiveCommittedVersion(Reference<MasterData> self, ReportRawCommittedVersionRequest req) {
-	return updateLiveCommittedVersionCxx(self, req);
-}
-#endif
+class LiveCommittedVersionServer {
+	Reference<MasterData> self;
 
-#ifdef WITH_SWIFT
-SWIFT_ACTOR Future<Void> serveLiveCommittedVersionSwift(Reference<MasterData> self) {
-	auto future = self->swiftImpl->serveLiveCommittedVersion(self.getPtr());
-	wait(future);
-	return Void();
-}
-#endif
-
-ACTOR Future<Void> serveLiveCommittedVersionCxx(Reference<MasterData> self) {
-	loop {
-		choose {
-			when(GetRawCommittedVersionRequest req = waitNext(self->myInterface.getLiveCommittedVersion.getFuture())) {
-				if (req.debugID.present())
-					g_traceBatch.addEvent("TransactionDebug",
-					                      req.debugID.get().first(),
-					                      "MasterServer.serveLiveCommittedVersion.GetRawCommittedVersion");
-
-				if (self->liveCommittedVersion.get() == invalidVersion) {
-					self->liveCommittedVersion.set(self->recoveryTransactionVersion);
-				}
-				++self->getLiveCommittedVersionRequests;
-				GetRawCommittedVersionReply reply;
-				reply.version = self->liveCommittedVersion.get();
-				reply.locked = self->databaseLocked;
-				reply.metadataVersion = self->proxyMetadataVersion;
-				reply.minKnownCommittedVersion = self->minKnownCommittedVersion;
-				if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
-					self->ssVersionVector.getDelta(req.maxVersion, reply.ssVersionVectorDelta);
-					self->versionVectorSizeOnCVReply->addMeasurement(reply.ssVersionVectorDelta.size());
-				}
-				req.reply.send(reply);
+	Future<Void> serveGetLiveCommittedVersion() {
+		while (true) {
+			GetRawCommittedVersionRequest req = co_await self->myInterface.getLiveCommittedVersion.getFuture();
+			if (req.debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      req.debugID.get().first(),
+				                      "MasterServer.serveLiveCommittedVersion.GetRawCommittedVersion");
+
+			if (self->liveCommittedVersion.get() == invalidVersion) {
+				self->liveCommittedVersion.set(self->recoveryTransactionVersion);
 			}
-			when(ReportRawCommittedVersionRequest req =
-			         waitNext(self->myInterface.reportLiveCommittedVersion.getFuture())) {
-				if (SERVER_KNOBS->ENABLE_VERSION_VECTOR && req.prevVersion.present() &&
-				    (self->liveCommittedVersion.get() != invalidVersion) &&
-				    (self->liveCommittedVersion.get() < req.prevVersion.get())) {
-					self->addActor.send(waitForPrev(self, req));
-				} else {
-					updateLiveCommittedVersion(self, req);
-					++self->nonWaitForPrevCommitRequests;
-					req.reply.send(Void());
-				}
+			++self->getLiveCommittedVersionRequests;
+			GetRawCommittedVersionReply reply;
+			reply.version = self->liveCommittedVersion.get();
+			reply.locked = self->databaseLocked;
+			reply.metadataVersion = self->proxyMetadataVersion;
+			reply.minKnownCommittedVersion = self->minKnownCommittedVersion;
+			if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
+				self->ssVersionVector.getDelta(req.maxVersion, reply.ssVersionVectorDelta);
+				self->versionVectorSizeOnCVReply->addMeasurement(reply.ssVersionVectorDelta.size());
 			}
+			req.reply.send(reply);
 		}
 	}
-}
 
-#ifdef WITH_SWIFT
-ACTOR Future<Void> serveLiveCommittedVersion(Reference<MasterData> self) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		wait(serveLiveCommittedVersionSwift(self));
-	} else {
-		wait(serveLiveCommittedVersionCxx(self));
+	Future<Void> serveReportLiveCommittedVersion() {
+		while (true) {
+			ReportRawCommittedVersionRequest req = co_await self->myInterface.reportLiveCommittedVersion.getFuture();
+			if (SERVER_KNOBS->ENABLE_VERSION_VECTOR && req.prevVersion.present() &&
+			    (self->liveCommittedVersion.get() != invalidVersion) &&
+			    (self->liveCommittedVersion.get() < req.prevVersion.get())) {
+				self->addActor.send(waitForPrev(self, req));
+			} else {
+				updateLiveCommittedVersion(self, req);
+				++self->nonWaitForPrevCommitRequests;
+				req.reply.send(Void());
+			}
+		}
 	}
-	return Void();
-}
-#else
-ACTOR Future<Void> serveLiveCommittedVersion(Reference<MasterData> self) {
-	wait(serveLiveCommittedVersionCxx(self));
-	return Void();
-}
-#endif
 
-#ifdef WITH_SWIFT
-SWIFT_ACTOR Future<Void> updateRecoveryDataSwift(Reference<MasterData> self) {
-	auto future = self->swiftImpl->serveUpdateRecoveryData(self.getPtr());
-	wait(future);
-	return Void();
+public:
+	explicit LiveCommittedVersionServer(Reference<MasterData> self) : self(self) {}
+
+	Future<Void> run() { co_await race(serveGetLiveCommittedVersion(), serveReportLiveCommittedVersion()); }
+};
+
+Future<Void> serveLiveCommittedVersion(Reference<MasterData> self) {
+	LiveCommittedVersionServer server(self);
+	co_await server.run();
 }
-#endif
 
-ACTOR Future<Void> updateRecoveryDataCxx(Reference<MasterData> self) {
-	loop {
-		UpdateRecoveryDataRequest req = waitNext(self->myInterface.updateRecoveryData.getFuture());
+Future<Void> updateRecoveryData(Reference<MasterData> self) {
+	while (true) {
+		UpdateRecoveryDataRequest req = co_await self->myInterface.updateRecoveryData.getFuture();
 		TraceEvent("UpdateRecoveryData", self->dbgid)
 		    .detail("ReceivedRecoveryTxnVersion", req.recoveryTransactionVersion)
 		    .detail("ReceivedLastEpochEnd", req.lastEpochEnd)
@@ -493,22 +342,6 @@ ACTOR Future<Void> updateRecoveryDataCxx(Reference<MasterData> self) {
 	}
 }
 
-#ifdef WITH_SWIFT
-ACTOR Future<Void> updateRecoveryData(Reference<MasterData> self) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		wait(updateRecoveryDataSwift(self));
-	} else {
-		wait(updateRecoveryDataCxx(self));
-	}
-	return Void();
-}
-#else
-ACTOR Future<Void> updateRecoveryData(Reference<MasterData> self) {
-	wait(updateRecoveryDataCxx(self));
-	return Void();
-}
-#endif
-
 static std::set<int> const& normalMasterErrors() {
 	static std::set<int> s;
 	if (s.empty()) {
@@ -530,141 +363,73 @@ static std::set<int> const& normalMasterErrors() {
 	return s;
 }
 
-ACTOR Future<Void> masterServerCxx(MasterInterface mi,
-                                   Reference<AsyncVar<ServerDBInfo> const> db,
-                                   Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
-                                   ServerCoordinators coordinators,
-                                   LifetimeToken lifetime,
-                                   bool forceRecovery) {
-	state Future<Void> ccTimeout = delay(SERVER_KNOBS->CC_INTERFACE_TIMEOUT);
+Future<Void> masterServer(MasterInterface mi,
+                          Reference<AsyncVar<ServerDBInfo> const> db,
+                          Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
+                          ServerCoordinators coordinators,
+                          LifetimeToken lifetime,
+                          bool forceRecovery) {
+	Future<Void> ccTimeout = delay(SERVER_KNOBS->CC_INTERFACE_TIMEOUT);
 	while (!ccInterface->get().present() || db->get().clusterInterface != ccInterface->get().get()) {
-		wait(ccInterface->onChange() || db->onChange() || ccTimeout);
+		co_await race(ccInterface->onChange(), db->onChange(), ccTimeout);
 		if (ccTimeout.isReady()) {
 			TraceEvent("MasterTerminated", mi.id())
 			    .detail("Reason", "Timeout")
 			    .detail("CCInterface", ccInterface->get().present() ? ccInterface->get().get().id() : UID())
 			    .detail("DBInfoInterface", db->get().clusterInterface.id());
-			return Void();
+			co_return;
 		}
 	}
 
-	state Future<Void> onDBChange = Void();
-	wait(onDBChange);
-	state PromiseStream<Future<Void>> addActor;
-	state Reference<MasterData> self(
+	Future<Void> onDBChange = Void();
+	co_await onDBChange;
+	PromiseStream<Future<Void>> addActor;
+	Reference<MasterData> self(
 	    new MasterData(db, mi, coordinators, db->get().clusterInterface, ""_sr, addActor, forceRecovery));
-	state Future<Void> collection = actorCollection(addActor.getFuture());
+	Future<Void> collection = actorCollection(addActor.getFuture());
 
 	addActor.send(traceRole(Role::MASTER, mi.id()));
 	addActor.send(provideVersions(self));
 	addActor.send(serveLiveCommittedVersion(self));
-	addActor.send(updateRecoveryDataCxx(self));
+	addActor.send(updateRecoveryData(self));
 
 	CODE_PROBE(!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id()),
 	           "Master born doomed");
 	TraceEvent("MasterLifetime", self->dbgid).detail("LifetimeToken", lifetime.toString());
 
+	Error err;
 	try {
-		loop choose {
-			when(wait(onDBChange)) {
-				onDBChange = db->onChange();
-				if (!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id())) {
-					TraceEvent("MasterTerminated", mi.id())
-					    .detail("Reason", "LifetimeToken")
-					    .detail("MyToken", lifetime.toString())
-					    .detail("CurrentToken", db->get().masterLifetime.toString());
-					CODE_PROBE(true, "Master replaced, dying");
-					if (BUGGIFY)
-						wait(delay(5));
-					throw worker_removed();
+		while (true) {
+			co_await waitOrError(onDBChange, collection);
+			onDBChange = db->onChange();
+			if (!lifetime.isStillValid(db->get().masterLifetime, mi.id() == db->get().master.id())) {
+				TraceEvent("MasterTerminated", mi.id())
+				    .detail("Reason", "LifetimeToken")
+				    .detail("MyToken", lifetime.toString())
+				    .detail("CurrentToken", db->get().masterLifetime.toString());
+				CODE_PROBE(true, "Master replaced, dying");
+				if (BUGGIFY) {
+					co_await delay(5);
 				}
-			}
-			when(wait(collection)) {
-				ASSERT(false);
-				throw internal_error();
+				throw worker_removed();
 			}
 		}
 	} catch (Error& e) {
-		state Error err = e;
-		if (e.code() != error_code_actor_cancelled) {
-			wait(delay(0.0));
-		}
-		while (!addActor.isEmpty()) {
-			addActor.getFuture().pop();
-		}
-
-		if (normalMasterErrors().contains(err.code())) {
-			TraceEvent("MasterTerminated", mi.id()).error(err);
-			return Void();
-		}
-		throw err;
+		err = e;
 	}
-}
 
-#ifdef WITH_SWIFT
-ACTOR Future<Void> masterServerImpl(MasterInterface mi,
-                                    Reference<AsyncVar<ServerDBInfo> const> db,
-                                    Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
-                                    ServerCoordinators coordinators,
-                                    LifetimeToken lifetime,
-                                    bool forceRecovery) {
-	if (SERVER_KNOBS->FLOW_WITH_SWIFT) {
-		auto promise = Promise<Void>();
-		state PromiseStream<Future<Void>> addActor;
-		state Reference<MasterData> self(
-		    new MasterData(db, mi, coordinators, db->get().clusterInterface, ""_sr, addActor, forceRecovery));
-		fdbserver_swift::masterServerSwift(
-		    mi,
-		    const_cast<AsyncVar<ServerDBInfo>*>(db.getPtr()),
-		    const_cast<AsyncVar<Optional<ClusterControllerFullInterface>>*>(ccInterface.getPtr()),
-		    coordinators,
-		    lifetime,
-		    forceRecovery,
-		    self.getPtr(),
-		    /*result=*/promise);
-		Future<Void> f = promise.getFuture();
-		wait(f);
-		return Void();
-	} else {
-		wait(masterServerCxx(mi, db, ccInterface, coordinators, lifetime, forceRecovery));
-		return Void();
+	if (err.code() != error_code_actor_cancelled) {
+		co_await delay(0.0);
 	}
-}
-#else
-ACTOR Future<Void> masterServerImpl(MasterInterface mi,
-                                    Reference<AsyncVar<ServerDBInfo> const> db,
-                                    Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
-                                    ServerCoordinators coordinators,
-                                    LifetimeToken lifetime,
-                                    bool forceRecovery) {
-	wait(masterServerCxx(mi, db, ccInterface, coordinators, lifetime, forceRecovery));
-	return Void();
-}
-#endif
-
-ACTOR Future<Void> masterServer(MasterInterface mi,
-                                Reference<AsyncVar<ServerDBInfo> const> db,
-                                Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
-                                ServerCoordinators coordinators,
-                                LifetimeToken lifetime,
-                                bool forceRecovery) {
-
-	state Future<Void> ccTimeout = delay(SERVER_KNOBS->CC_INTERFACE_TIMEOUT);
-	while (!ccInterface->get().present() || db->get().clusterInterface != ccInterface->get().get()) {
-		wait(ccInterface->onChange() || db->onChange() || ccTimeout);
-		if (ccTimeout.isReady()) {
-			TraceEvent("MasterTerminated", mi.id())
-			    .detail("Reason", "Timeout")
-			    .detail("CCInterface", ccInterface->get().present() ? ccInterface->get().get().id() : UID())
-			    .detail("DBInfoInterface", db->get().clusterInterface.id());
-			return Void();
-		}
+	while (!addActor.isEmpty()) {
+		addActor.getFuture().pop();
 	}
 
-	state Future<Void> onDBChange = Void();
-	wait(onDBChange);
-	wait(masterServerImpl(mi, db, ccInterface, coordinators, lifetime, forceRecovery));
-	return Void();
+	if (normalMasterErrors().contains(err.code())) {
+		TraceEvent("MasterTerminated", mi.id()).error(err);
+		co_return;
+	}
+	throw err;
 }
 
 TEST_CASE("/fdbserver/MasterServer/FigureVersion/Simple") {
diff --git a/fdbserver/storageserver/CMakeLists.txt b/fdbserver/storageserver/CMakeLists.txt
new file mode 100644
index 00000000000..abf85cb3a31
--- /dev/null
+++ b/fdbserver/storageserver/CMakeLists.txt
@@ -0,0 +1,17 @@
+fdb_find_sources(FDBSERVER_STORAGESERVER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_storageserver SRCS ${FDBSERVER_STORAGESERVER_SRCS})
+add_fdbserver_link_test(fdbserver_storageserverlinktest
+  fdbserver_storageserver
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_storageserver)
+target_include_directories(fdbserver_storageserver
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(fdbserver_storageserver PRIVATE fdbserver_core fdbserver_kvstore fdbserver_logsystem)
diff --git a/fdbserver/ReadLatencySamples.cpp b/fdbserver/storageserver/ReadLatencySamples.cpp
similarity index 97%
rename from fdbserver/ReadLatencySamples.cpp
rename to fdbserver/storageserver/ReadLatencySamples.cpp
index 3929f222777..80d51415b7a 100644
--- a/fdbserver/ReadLatencySamples.cpp
+++ b/fdbserver/storageserver/ReadLatencySamples.cpp
@@ -18,8 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/ReadLatencySamples.h"
-#include "fdbserver/Knobs.h"
+#include "ReadLatencySamples.h"
+#include "fdbserver/core/Knobs.h"
 
 namespace {
 
diff --git a/fdbserver/include/fdbserver/ReadLatencySamples.h b/fdbserver/storageserver/ReadLatencySamples.h
similarity index 100%
rename from fdbserver/include/fdbserver/ReadLatencySamples.h
rename to fdbserver/storageserver/ReadLatencySamples.h
diff --git a/fdbserver/StorageServerUtils.cpp b/fdbserver/storageserver/StorageServerUtils.cpp
similarity index 98%
rename from fdbserver/StorageServerUtils.cpp
rename to fdbserver/storageserver/StorageServerUtils.cpp
index 7fdaf8929a1..ce9070b8fd3 100644
--- a/fdbserver/StorageServerUtils.cpp
+++ b/fdbserver/storageserver/StorageServerUtils.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/StorageServerUtils.h"
+#include "StorageServerUtils.h"
 
 #define PERSIST_PREFIX "\xff\xff"
 
@@ -111,4 +111,4 @@ MoveInShardMetaData decodeMoveInShardValue(const ValueRef& value) {
 	ObjectReader reader(value.begin(), IncludeVersion());
 	reader.deserialize(shard);
 	return shard;
-}
\ No newline at end of file
+}
diff --git a/fdbserver/include/fdbserver/StorageServerUtils.h b/fdbserver/storageserver/StorageServerUtils.h
similarity index 99%
rename from fdbserver/include/fdbserver/StorageServerUtils.h
rename to fdbserver/storageserver/StorageServerUtils.h
index c879dd984c6..3b7ff5b1262 100644
--- a/fdbserver/include/fdbserver/StorageServerUtils.h
+++ b/fdbserver/storageserver/StorageServerUtils.h
@@ -109,7 +109,7 @@ struct MoveInShardMetaData {
 
 class ThroughputLimiter {
 public:
-	ThroughputLimiter(int64_t cap);
+	explicit ThroughputLimiter(int64_t cap);
 
 	Future<Void> ready();
 	void addBytes(int64_t bytes);
diff --git a/fdbserver/TransactionTagCounter.actor.cpp b/fdbserver/storageserver/TransactionTagCounter.cpp
similarity index 76%
rename from fdbserver/TransactionTagCounter.actor.cpp
rename to fdbserver/storageserver/TransactionTagCounter.cpp
index b8f50184975..1dd220606fe 100644
--- a/fdbserver/TransactionTagCounter.actor.cpp
+++ b/fdbserver/storageserver/TransactionTagCounter.cpp
@@ -1,5 +1,5 @@
 /*
- * TransactionTagCounter.actor.cpp
+ * TransactionTagCounter.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/TransactionTagCounter.h"
+#include "fdbserver/core/Knobs.h"
+#include "TransactionTagCounter.h"
+#include "flow/Coroutines.h"
 #include "flow/Trace.h"
-#include "flow/actorcompiler.h"
 
 class TransactionTagCounterImpl {
 	UID thisServerID;
@@ -145,41 +145,37 @@ TagSet getTagSet(TransactionTagRef tag) {
 } // namespace
 
 TEST_CASE("/fdbserver/TransactionTagCounter/IgnoreBeyondMaxTags") {
-	state TransactionTagCounter counter(UID(),
-	                                    /*maxTagsTracked=*/2,
-	                                    /*minRateTracked=*/10.0 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE /
-	                                        CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
+	TransactionTagCounter counter(UID(),
+	                              /*maxTagsTracked=*/2,
+	                              /*minRateTracked=*/10.0 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE /
+	                                  CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
 	counter.startNewInterval();
 	ASSERT_EQ(counter.getBusiestTags().size(), 0);
-	{
-		wait(delay(1.0));
-		counter.addRequest(getTagSet("tagA"_sr), 10 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
-		counter.addRequest(getTagSet("tagA"_sr), 10 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
-		counter.addRequest(getTagSet("tagB"_sr), 15 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
-		counter.addRequest(getTagSet("tagC"_sr), 20 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
-		counter.startNewInterval();
-		auto const busiestTags = counter.getBusiestTags();
-		ASSERT_EQ(busiestTags.size(), 2);
-		ASSERT(containsTag(busiestTags, "tagA"_sr));
-		ASSERT(!containsTag(busiestTags, "tagB"_sr));
-		ASSERT(containsTag(busiestTags, "tagC"_sr));
-	}
-	return Void();
+	co_await delay(1.0);
+	counter.addRequest(getTagSet("tagA"_sr), 10 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
+	counter.addRequest(getTagSet("tagA"_sr), 10 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
+	counter.addRequest(getTagSet("tagB"_sr), 15 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
+	counter.addRequest(getTagSet("tagC"_sr), 20 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
+	counter.startNewInterval();
+	auto const busiestTags = counter.getBusiestTags();
+	ASSERT_EQ(busiestTags.size(), 2);
+	ASSERT(containsTag(busiestTags, "tagA"_sr));
+	ASSERT(!containsTag(busiestTags, "tagB"_sr));
+	ASSERT(containsTag(busiestTags, "tagC"_sr));
+	co_return;
 }
 
 TEST_CASE("/fdbserver/TransactionTagCounter/IgnoreBelowMinRate") {
-	state TransactionTagCounter counter(UID(),
-	                                    /*maxTagsTracked=*/2,
-	                                    /*minRateTracked=*/10.0 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE /
-	                                        CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
+	TransactionTagCounter counter(UID(),
+	                              /*maxTagsTracked=*/2,
+	                              /*minRateTracked=*/10.0 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE /
+	                                  CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
 	counter.startNewInterval();
 	ASSERT_EQ(counter.getBusiestTags().size(), 0);
-	{
-		wait(delay(1.0));
-		counter.addRequest(getTagSet("tagA"_sr), 5 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
-		counter.startNewInterval();
-		auto const busiestTags = counter.getBusiestTags();
-		ASSERT_EQ(busiestTags.size(), 0);
-	}
-	return Void();
+	co_await delay(1.0);
+	counter.addRequest(getTagSet("tagA"_sr), 5 * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
+	counter.startNewInterval();
+	auto const busiestTags = counter.getBusiestTags();
+	ASSERT_EQ(busiestTags.size(), 0);
+	co_return;
 }
diff --git a/fdbserver/include/fdbserver/TransactionTagCounter.h b/fdbserver/storageserver/TransactionTagCounter.h
similarity index 97%
rename from fdbserver/include/fdbserver/TransactionTagCounter.h
rename to fdbserver/storageserver/TransactionTagCounter.h
index a313e51d74b..469c5294c03 100644
--- a/fdbserver/include/fdbserver/TransactionTagCounter.h
+++ b/fdbserver/storageserver/TransactionTagCounter.h
@@ -22,7 +22,7 @@
 
 #include "fdbclient/PImpl.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TagThrottle.h"
 
 class TransactionTagCounter {
 	PImpl<class TransactionTagCounterImpl> impl;
diff --git a/fdbserver/include/fdbserver/StorageCorruptionBug.h b/fdbserver/storageserver/include/fdbserver/storageserver/StorageCorruptionBug.h
similarity index 85%
rename from fdbserver/include/fdbserver/StorageCorruptionBug.h
rename to fdbserver/storageserver/include/fdbserver/storageserver/StorageCorruptionBug.h
index 27a8bbbce4f..f274e1f5b1a 100644
--- a/fdbserver/include/fdbserver/StorageCorruptionBug.h
+++ b/fdbserver/storageserver/include/fdbserver/storageserver/StorageCorruptionBug.h
@@ -18,17 +18,19 @@
  * limitations under the License.
  */
 #pragma once
-#ifndef FDBSERVER_STORAGE_CORRUPTION_BUG_H
-#define FDBSERVER_STORAGE_CORRUPTION_BUG_H
+#ifndef FDBSERVER_STORAGESERVER_STORAGE_CORRUPTION_BUG_H
+#define FDBSERVER_STORAGESERVER_STORAGE_CORRUPTION_BUG_H
+
 #include "flow/SimBugInjector.h"
 
 class StorageCorruptionBug : public ISimBug {
 public:
 	double corruptionProbability = 0.001;
 };
+
 class StorageCorruptionBugID : public IBugIdentifier {
 public:
 	std::shared_ptr<ISimBug> create() const override { return std::make_shared<StorageCorruptionBug>(); }
 };
 
-#endif // FDBSERVER_STORAGE_CORRUPTION_BUG_H
+#endif // FDBSERVER_STORAGESERVER_STORAGE_CORRUPTION_BUG_H
diff --git a/fdbserver/storageserver/include/fdbserver/storageserver/StorageServer.h b/fdbserver/storageserver/include/fdbserver/storageserver/StorageServer.h
new file mode 100644
index 00000000000..40764c8f468
--- /dev/null
+++ b/fdbserver/storageserver/include/fdbserver/storageserver/StorageServer.h
@@ -0,0 +1,49 @@
+/*
+ * StorageServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#ifndef FDBSERVER_STORAGESERVER_STORAGE_SERVER_H
+#define FDBSERVER_STORAGESERVER_STORAGE_SERVER_H
+
+#include "fdbclient/StorageServerInterface.h"
+#include "flow/flow.h"
+
+class IClusterConnectionRecord;
+class IKeyValueStore;
+struct InitializeStorageReply;
+struct ServerDBInfo;
+
+Future<Void> storageServer(IKeyValueStore* persistentData,
+                           StorageServerInterface ssi,
+                           Tag seedTag,
+                           Version startVersion,
+                           Version tssSeedVersion,
+                           ReplyPromise<InitializeStorageReply> recruitReply,
+                           Reference<AsyncVar<ServerDBInfo> const> db,
+                           std::string folder);
+
+Future<Void> storageServer(IKeyValueStore* persistentData,
+                           StorageServerInterface ssi,
+                           Reference<AsyncVar<ServerDBInfo> const> db,
+                           std::string folder,
+                           Promise<Void> recovered,
+                           Reference<IClusterConnectionRecord> connRecord);
+
+#endif
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver/storageserver.actor.cpp
similarity index 78%
rename from fdbserver/storageserver.actor.cpp
rename to fdbserver/storageserver/storageserver.actor.cpp
index b60e9bc81e6..c0c0c51e716 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver/storageserver.actor.cpp
@@ -38,13 +38,35 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/LoadBalance.h"
-#include "fdbserver/OTELSpanContextMessage.h"
+#include "fdbserver/core/AccumulativeChecksumUtil.h"
+#include "fdbserver/core/BulkDumpUtil.h"
+#include "fdbserver/core/BulkLoadUtil.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/FDBRocksDBVersion.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/kvstore/FDBExecHelper.h"
+#include "fdbserver/core/LatencyBandConfig.h"
+#include "fdbserver/core/LogProtocolMessage.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/MutationTracking.h"
+#include "fdbserver/core/OTELSpanContextMessage.h"
+#include "ReadLatencySamples.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
+#include "fdbserver/core/ServerCheckpoint.h"
+#include "fdbserver/core/SpanContextMessage.h"
+#include "fdbserver/storageserver/StorageCorruptionBug.h"
+#include "fdbserver/core/StorageMetrics.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "TransactionTagCounter.h"
+#include "fdbserver/core/WaitFailure.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/Hash3.h"
 #include "flow/Histogram.h"
-#include "flow/PriorityMultiLock.actor.h"
+#include "flow/PriorityMultiLock.h"
 #include "flow/IRandom.h"
 #include "flow/IndexedSet.h"
 #include "flow/SystemMonitor.h"
@@ -52,11 +74,11 @@
 #include "fdbclient/Tracing.h"
 #include "flow/Util.h"
 #include "fdbclient/Atomic.h"
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/Notified.h"
@@ -69,33 +91,13 @@
 #include "fdbrpc/sim_validation.h"
 #include "fdbrpc/Smoother.h"
 #include "fdbrpc/Stats.h"
-#include "fdbserver/AccumulativeChecksumUtil.h"
-#include "fdbserver/BulkDumpUtil.actor.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/LatencyBandConfig.h"
-#include "fdbserver/LogProtocolMessage.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/OTELSpanContextMessage.h"
-#include "fdbserver/Ratekeeper.h"
-#include "fdbserver/ReadLatencySamples.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/ServerCheckpoint.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/SpanContextMessage.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/TransactionTagCounter.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/StorageCorruptionBug.h"
-#include "fdbserver/StorageServerUtils.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/DataMovement.h"
+#include "fdbserver/storageserver/StorageServer.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "StorageServerUtils.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
@@ -103,13 +105,12 @@
 #include "flow/Histogram.h"
 #include "flow/IRandom.h"
 #include "flow/IndexedSet.h"
+#include "flow/CoroUtils.h"
 #include "flow/SystemMonitor.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/Trace.h"
 #include "flow/Util.h"
 #include "flow/genericactors.actor.h"
-#include "fdbserver/FDBRocksDBVersion.h"
-
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 #ifndef __INTEL_COMPILER
@@ -272,16 +273,16 @@ struct MoveInUpdates {
 	Severity logSev;
 
 private:
-	ACTOR static Future<Void> loadUpdates(MoveInUpdates* self, Version begin, Version end);
+	static Future<Void> loadUpdates(MoveInUpdates* self, Version begin, Version end);
 
 	Key getPersistKey(const Version version, const int idx) const;
 };
 
-ACTOR Future<Void> MoveInUpdates::loadUpdates(MoveInUpdates* self, Version begin, Version end) {
+Future<Void> MoveInUpdates::loadUpdates(MoveInUpdates* self, Version begin, Version end) {
 	ASSERT(self->spilled);
 	if (begin >= end) {
 		self->spilled = MoveInUpdatesSpilled::False;
-		return Void();
+		co_return;
 	}
 
 	const Key beginKey = persistUpdatesKey(self->id, begin), endKey = persistUpdatesKey(self->id, end);
@@ -291,14 +292,13 @@ ACTOR Future<Void> MoveInUpdates::loadUpdates(MoveInUpdates* self, Version begin
 	    .detail("BeginKey", beginKey)
 	    .detail("EndKey", endKey);
 	ASSERT(beginKey < endKey);
-	RangeResult res = wait(self->store->readRange(KeyRangeRef(beginKey, endKey),
-	                                              SERVER_KNOBS->FETCH_SHARD_UPDATES_BYTE_LIMIT,
-	                                              SERVER_KNOBS->FETCH_SHARD_UPDATES_BYTE_LIMIT));
+	RangeResult res = co_await self->store->readRange(KeyRangeRef(beginKey, endKey),
+	                                                  SERVER_KNOBS->FETCH_SHARD_UPDATES_BYTE_LIMIT,
+	                                                  SERVER_KNOBS->FETCH_SHARD_UPDATES_BYTE_LIMIT);
 	std::vector<Standalone<VerUpdateRef>> restored;
 	for (int i = 0; i < res.size(); ++i) {
 		const Version version = decodePersistUpdateVersion(res[i].key.removePrefix(self->range.begin));
-		Standalone<VerUpdateRef> vur =
-		    BinaryReader::fromStringRef<Standalone<VerUpdateRef>>(res[i].value, IncludeVersion());
+		auto vur = BinaryReader::fromStringRef<Standalone<VerUpdateRef>>(res[i].value, IncludeVersion());
 		ASSERT(version == vur.version);
 		TraceEvent(self->logSev, "MoveInUpdatesLoadedMutations", self->id)
 		    .detail("Version", version)
@@ -324,8 +324,6 @@ ACTOR Future<Void> MoveInUpdates::loadUpdates(MoveInUpdates* self, Version begin
 	    .detail("MaxVersion", restored.empty() ? invalidVersion : restored.back().version)
 	    .detail("VersionCount", restored.size())
 	    .detail("LastBatch", !res.more);
-
-	return Void();
 }
 
 bool MoveInUpdates::hasNext() const {
@@ -713,12 +711,12 @@ struct StorageServerDisk {
 	void writeMutations(const VectorRef<MutationRef>& mutations, Version debugVersion, const char* debugContext);
 	void writeMutationsBuggy(const VectorRef<MutationRef>& mutations, Version debugVersion, const char* debugContext);
 
-	ACTOR static Future<Key> readFirstKey(IKeyValueStore* storage, KeyRangeRef range, Optional<ReadOptions> options) {
-		RangeResult r = wait(storage->readRange(range, 1, 1 << 30, options));
+	static Future<Key> readFirstKey(IKeyValueStore* storage, KeyRangeRef range, Optional<ReadOptions> options) {
+		RangeResult r = co_await storage->readRange(range, 1, 1 << 30, options);
 		if (r.size())
-			return r[0].key;
+			co_return r[0].key;
 		else
-			return range.end;
+			co_return range.end;
 	}
 };
 
@@ -732,7 +730,8 @@ struct UpdateEagerReadInfo {
 	Arena arena;
 	bool enableClearRangeEagerReads;
 
-	UpdateEagerReadInfo(bool enableClearRangeEagerReads) : enableClearRangeEagerReads(enableClearRangeEagerReads) {}
+	explicit UpdateEagerReadInfo(bool enableClearRangeEagerReads)
+	  : enableClearRangeEagerReads(enableClearRangeEagerReads) {}
 
 	void addMutations(VectorRef<MutationRef> const& mutations) {
 		for (auto& m : mutations)
@@ -832,7 +831,7 @@ struct BusiestWriteTagContext {
 	Reference<EventCacheHolder> busiestWriteTagEventHolder;
 	double lastUpdateTime;
 
-	BusiestWriteTagContext(const UID& thisServerID)
+	explicit BusiestWriteTagContext(const UID& thisServerID)
 	  : busiestWriteTagTrackingKey(thisServerID.toString() + "/BusiestWriteTag"), ratekeeperID(UID()),
 	    busiestWriteTagEventHolder(makeReference<EventCacheHolder>(busiestWriteTagTrackingKey)), lastUpdateTime(-1) {}
 };
@@ -849,7 +848,7 @@ struct SSBulkLoadMetrics {
 };
 
 struct StorageServer : public IStorageMetricsService {
-	typedef VersionedMap<KeyRef, ValueOrClearToRef> VersionedData;
+	using VersionedData = VersionedMap<KeyRef, ValueOrClearToRef>;
 
 private:
 	// versionedData contains sets and clears.
@@ -1109,10 +1108,10 @@ struct StorageServer : public IStorageMetricsService {
 		// With fault injection enabled, the tss will start acting normal for a bit, then after the specified delay
 		// start behaving incorrectly.
 		if (g_network->isSimulated() && !g_simulator->speedUpSimulation &&
-		    g_simulator->tssMode >= ISimulator::TSSMode::EnabledAddDelay) {
+		    simulationPolicyHasCapability(ISimulationPolicy::Capability::StorageReplicaFaultInjection)) {
 			tssFaultInjectTime = now() + deterministicRandom()->randomInt(60, 300);
 			TraceEvent(SevWarnAlways, "TSSInjectFaultEnabled", thisServerID)
-			    .detail("Mode", g_simulator->tssMode)
+			    .detail("Mode", static_cast<int>(fdbSimulationPolicyState().tssMode))
 			    .detail("At", tssFaultInjectTime.get());
 		}
 	}
@@ -1178,8 +1177,8 @@ struct StorageServer : public IStorageMetricsService {
 
 	ProtocolVersion logProtocol;
 
-	Reference<ILogSystem> logSystem;
-	Reference<ILogSystem::IPeekCursor> logCursor;
+	Reference<LogSystem> logSystem;
+	Reference<IPeekCursor> logCursor;
 
 	// The version the cluster starts on. This value is not persisted and may
 	// not be valid after a recovery.
@@ -1302,11 +1301,8 @@ struct StorageServer : public IStorageMetricsService {
 		// Count of all fetchKey clearRange operations to the storage engine.
 		Counter kvClearRangesInFetchKeys;
 
-		// Bytes fetched by fetchChangeFeed for data movements.
-		Counter feedBytesFetched;
-
 		Counter sampledBytesCleared;
-		Counter atomicMutations, changeFeedMutations, changeFeedMutationsDurable;
+		Counter atomicMutations;
 		Counter updateBatches, updateVersions;
 		Counter loops;
 		Counter fetchWaitingMS, fetchWaitingCount, fetchExecutingMS, fetchExecutingCount;
@@ -1331,8 +1327,6 @@ struct StorageServer : public IStorageMetricsService {
 		Counter kvScans;
 		// The count of commit operation to the storage engine.
 		Counter kvCommits;
-		// The count of change feed reads that hit disk
-		Counter changeFeedDiskReads;
 		// The count of ChangeServerKeys actions.
 		Counter changeServerKeysAssigned;
 		Counter changeServerKeysUnassigned;
@@ -1362,10 +1356,8 @@ struct StorageServer : public IStorageMetricsService {
 		    logicalBytesInput("LogicalBytesInput", cc), logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
 		    kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc),
 		    kvClearSingleKey("KVClearSingleKey", cc), kvSystemClearRanges("KVSystemClearRanges", cc),
-		    bytesDurable("BytesDurable", cc), feedBytesFetched("FeedBytesFetched", cc),
-		    sampledBytesCleared("SampledBytesCleared", cc), atomicMutations("AtomicMutations", cc),
-		    changeFeedMutations("ChangeFeedMutations", cc),
-		    changeFeedMutationsDurable("ChangeFeedMutationsDurable", cc), updateBatches("UpdateBatches", cc),
+		    bytesDurable("BytesDurable", cc), sampledBytesCleared("SampledBytesCleared", cc),
+		    atomicMutations("AtomicMutations", cc), updateBatches("UpdateBatches", cc),
 		    updateVersions("UpdateVersions", cc), loops("Loops", cc), fetchWaitingMS("FetchWaitingMS", cc),
 		    fetchWaitingCount("FetchWaitingCount", cc), fetchExecutingMS("FetchExecutingMS", cc),
 		    fetchExecutingCount("FetchExecutingCount", cc), readsRejected("ReadsRejected", cc),
@@ -1374,7 +1366,7 @@ struct StorageServer : public IStorageMetricsService {
 		    quickGetValueMiss("QuickGetValueMiss", cc), quickGetKeyValuesHit("QuickGetKeyValuesHit", cc),
 		    quickGetKeyValuesMiss("QuickGetKeyValuesMiss", cc), kvScanBytes("KVScanBytes", cc),
 		    kvGetBytes("KVGetBytes", cc), eagerReadsKeys("EagerReadsKeys", cc), kvGets("KVGets", cc),
-		    kvScans("KVScans", cc), kvCommits("KVCommits", cc), changeFeedDiskReads("ChangeFeedDiskReads", cc),
+		    kvScans("KVScans", cc), kvCommits("KVCommits", cc),
 		    getMappedRangeBytesQueried("GetMappedRangeBytesQueried", cc),
 		    finishedGetMappedRangeQueries("FinishedGetMappedRangeQueries", cc),
 		    finishedGetMappedRangeSecondaryQueries("FinishedGetMappedRangeSecondaryQueries", cc),
@@ -1640,8 +1632,8 @@ struct StorageServer : public IStorageMetricsService {
 	using isLoadBalancedReply = std::is_base_of<LoadBalancedReply, Reply>;
 
 	template <class Reply>
-	typename std::enable_if<isLoadBalancedReply<Reply>::value, void>::type
-	sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double penalty) {
+	    requires(isLoadBalancedReply<Reply>::value)
+	void sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double penalty) {
 		if (err.code() == error_code_wrong_shard_server) {
 			++counters.wrongShardServer;
 		}
@@ -1652,8 +1644,8 @@ struct StorageServer : public IStorageMetricsService {
 	}
 
 	template <class Reply>
-	typename std::enable_if<!isLoadBalancedReply<Reply>::value, void>::type
-	sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double) {
+	    requires(!isLoadBalancedReply<Reply>::value)
+	void sendErrorWithPenalty(const ReplyPromise<Reply>& promise, const Error& err, double) {
 		if (err.code() == error_code_wrong_shard_server) {
 			++counters.wrongShardServer;
 		}
@@ -1694,20 +1686,21 @@ struct StorageServer : public IStorageMetricsService {
 	void maybeInjectTargetedRestart(Version v) {
 		// inject an SS restart at most once per test
 		if (g_network->isSimulated() && !g_simulator->speedUpSimulation &&
-		    now() > g_simulator->injectTargetedSSRestartTime &&
+		    now() > fdbSimulationPolicyState().injectTargetedSSRestartTime &&
 		    rebootAfterDurableVersion == std::numeric_limits<Version>::max()) {
 			CODE_PROBE(true, "Injecting SS targeted restart");
 			TraceEvent("SimSSInjectTargetedRestart", thisServerID).detail("Version", v);
 			rebootAfterDurableVersion = v;
-			g_simulator->injectTargetedSSRestartTime = std::numeric_limits<double>::max();
+			fdbSimulationPolicyState().injectTargetedSSRestartTime = std::numeric_limits<double>::max();
 		}
 	}
 
 	bool maybeInjectDelay() {
-		if (g_network->isSimulated() && !g_simulator->speedUpSimulation && now() > g_simulator->injectSSDelayTime) {
+		if (g_network->isSimulated() && !g_simulator->speedUpSimulation &&
+		    now() > fdbSimulationPolicyState().injectSSDelayTime) {
 			CODE_PROBE(true, "Injecting SS targeted delay");
 			TraceEvent("SimSSInjectDelay", thisServerID).log();
-			g_simulator->injectSSDelayTime = std::numeric_limits<double>::max();
+			fdbSimulationPolicyState().injectSSDelayTime = std::numeric_limits<double>::max();
 			return true;
 		}
 		return false;
@@ -1944,8 +1937,6 @@ void validate(StorageServer* data, bool force = false) {
 				}
 			}
 
-			// FIXME: do some change feed validation?
-
 			latest.validate();
 			validateRange(latest, allKeys, data->version.get(), data->thisServerID, data->durableVersion.get());
 
@@ -2108,30 +2099,6 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageServer* data, Version versio
 	}
 }
 
-ACTOR Future<Version> waitForMinVersion(StorageServer* data, Version version) {
-	// This could become an Actor transparently, but for now it just does the lookup
-	if (version == latestVersion)
-		version = std::max(Version(1), data->version.get());
-	if (version < data->oldestVersion.get() || version <= 0) {
-		return data->oldestVersion.get();
-	} else if (version <= data->version.get()) {
-		return version;
-	}
-	choose {
-		when(wait(data->version.whenAtLeast(version))) {
-			return version;
-		}
-		when(wait(delay(SERVER_KNOBS->FUTURE_VERSION_DELAY))) {
-			if (deterministicRandom()->random01() < 0.001)
-				TraceEvent(SevWarn, "ShardServerFutureVersion1000x", data->thisServerID)
-				    .detail("Version", version)
-				    .detail("MyVersion", data->version.get())
-				    .detail("ServerID", data->thisServerID);
-			throw future_version();
-		}
-	}
-}
-
 std::vector<StorageServerShard> StorageServer::getStorageServerShards(KeyRangeRef range) {
 	std::vector<StorageServerShard> res;
 	for (auto t : this->shards.intersectingRanges(range)) {
@@ -2172,8 +2139,8 @@ std::shared_ptr<MoveInShard> StorageServer::getMoveInShard(const UID& dataMoveId
 	return shard;
 }
 
-ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
-	state int64_t resultSize = 0;
+Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
+	int64_t resultSize = 0;
 	Span span("SS:getValue"_loc, req.spanContext);
 	// Temporarily disabled -- this path is hit a lot
 	// getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.first();
@@ -2189,11 +2156,11 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 
 		// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 		// so we need to downgrade here
-		wait(data->getQueryDelay());
-		state PriorityMultiLock::Lock readLock = wait(data->getReadLock(req.options));
+		co_await data->getQueryDelay();
+		PriorityMultiLock::Lock readLock = co_await data->getReadLock(req.options);
 
 		// Track time from requestTime through now as read queueing wait time
-		state double queueWaitEnd = g_network->timer();
+		double queueWaitEnd = g_network->timer();
 		data->counters.readLatencySamples.sample(
 		    queueWaitEnd - req.requestTime(), ReadLatencySamples::READ_QUEUE_WAIT, trackedReadType(req));
 
@@ -2202,9 +2169,9 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.DoRead"); //.detail("TaskID", g_network->getCurrentTask());
 
-		state Optional<Value> v;
+		Optional<Value> v;
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
-		state Version version = wait(waitForVersion(data, commitVersion, req.version, req.spanContext));
+		Version version = co_await waitForVersion(data, commitVersion, req.version, req.spanContext);
 		data->counters.readLatencySamples.sample(
 		    g_network->timer() - queueWaitEnd, ReadLatencySamples::READ_VERSION_WAIT, trackedReadType(req));
 
@@ -2213,21 +2180,21 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 			                      req.options.get().debugID.get().first(),
 			                      "getValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
-		state uint64_t changeCounter = data->shardChangeCounter;
+		uint64_t changeCounter = data->shardChangeCounter;
 
 		if (!data->shards[req.key]->isReadable()) {
 			//TraceEvent("WrongShardServer", data->thisServerID).detail("Key", req.key).detail("Version", version).detail("In", "getValueQ");
 			throw wrong_shard_server();
 		}
 
-		state int path = 0;
+		int path = 0;
 		auto i = data->data().at(version).lastLessOrEqual(req.key);
 		if (i && i->isValue() && i.key() == req.key) {
 			v = (Value)i->getValue();
 			path = 1;
 		} else if (!i || !i->isClearTo() || i->getEndKey() <= req.key) {
 			path = 2;
-			Optional<Value> vv = wait(data->storage.readValue(req.key, req.options));
+			Optional<Value> vv = co_await data->storage.readValue(req.key, req.options);
 			data->counters.kvGetBytes += vv.expectedSize();
 			// Validate that while we were reading the data we didn't lose the version or shard
 			if (version < data->storageVersion()) {
@@ -2307,8 +2274,6 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 		    data->latencyBandConfig.get().readConfig.maxReadBytes.orDefault(std::numeric_limits<int>::max());
 		data->counters.readLatencyBands.addMeasurement(duration, 1, Filtered(resultSize > maxReadBytes));
 	}
-
-	return Void();
 }
 
 // Pessimistic estimate the number of overhead bytes used by each
@@ -2492,9 +2457,9 @@ ACTOR Future<Void> watchValueSendReply(StorageServer* data,
 }
 
 // Finds a checkpoint.
-ACTOR Future<Void> getCheckpointQ(StorageServer* self, GetCheckpointRequest req) {
+Future<Void> getCheckpointQ(StorageServer* self, GetCheckpointRequest req) {
 	// Wait until the desired version is durable.
-	wait(self->durableVersion.whenAtLeast(req.version + 1));
+	co_await self->durableVersion.whenAtLeast(req.version + 1);
 
 	TraceEvent(SevDebug, "ServeGetCheckpointVersionSatisfied", self->thisServerID)
 	    .detail("Version", req.version)
@@ -2504,12 +2469,12 @@ ACTOR Future<Void> getCheckpointQ(StorageServer* self, GetCheckpointRequest req)
 	for (const auto& range : req.ranges) {
 		if (!self->isReadable(range)) {
 			req.reply.sendError(wrong_shard_server());
-			return Void();
+			co_return;
 		}
 	}
 
 	try {
-		std::unordered_map<UID, CheckpointMetaData>::iterator it = self->checkpoints.begin();
+		auto it = self->checkpoints.begin();
 		for (; it != self->checkpoints.end(); ++it) {
 			const CheckpointMetaData& md = it->second;
 			if (md.version == req.version && md.format == req.format && req.actionId == md.actionId &&
@@ -2529,46 +2494,44 @@ ACTOR Future<Void> getCheckpointQ(StorageServer* self, GetCheckpointRequest req)
 		}
 		req.reply.sendError(e);
 	}
-	return Void();
 }
 
 // Delete the checkpoint from disk, as well as all related persisted meta data.
-ACTOR Future<Void> deleteCheckpointQ(StorageServer* self, Version version, CheckpointMetaData checkpoint) {
-	wait(delay(0, TaskPriority::Low));
+Future<Void> deleteCheckpointQ(StorageServer* self, Version version, CheckpointMetaData checkpoint) {
+	co_await delay(0, TaskPriority::Low);
 
-	wait(self->durableVersion.whenAtLeast(version));
+	co_await self->durableVersion.whenAtLeast(version);
 
 	TraceEvent(SevInfo, "DeleteCheckpointBegin", self->thisServerID).detail("Checkpoint", checkpoint.toString());
 
 	self->checkpoints.erase(checkpoint.checkpointID);
 
 	try {
-		wait(deleteCheckpoint(checkpoint));
+		co_await deleteCheckpoint(checkpoint);
 	} catch (Error& e) {
 		// TODO: Handle errors more gracefully.
 		throw;
 	}
 
-	state Key persistCheckpointKey(persistCheckpointKeys.begin.toString() + checkpoint.checkpointID.toString());
-	state Key pendingCheckpointKey(persistPendingCheckpointKeys.begin.toString() + checkpoint.checkpointID.toString());
+	Key persistCheckpointKey(persistCheckpointKeys.begin.toString() + checkpoint.checkpointID.toString());
+	Key pendingCheckpointKey(persistPendingCheckpointKeys.begin.toString() + checkpoint.checkpointID.toString());
 	auto& mLV = self->addVersionToMutationLog(self->data().getLatestVersion());
 	self->addMutationToMutationLog(
 	    mLV, MutationRef(MutationRef::ClearRange, pendingCheckpointKey, keyAfter(pendingCheckpointKey)));
 	self->addMutationToMutationLog(
 	    mLV, MutationRef(MutationRef::ClearRange, persistCheckpointKey, keyAfter(persistCheckpointKey)));
 	TraceEvent(SevInfo, "DeleteCheckpointEnd", self->thisServerID).detail("Checkpoint", checkpoint.toString());
-
-	return Void();
 }
 
 // Serves FetchCheckpointRequests.
-ACTOR Future<Void> fetchCheckpointQ(StorageServer* self, FetchCheckpointRequest req) {
+Future<Void> fetchCheckpointQ(StorageServer* self, FetchCheckpointRequest req) {
 	TraceEvent("ServeFetchCheckpointBegin", self->thisServerID)
 	    .detail("CheckpointID", req.checkpointID)
 	    .detail("Token", req.token);
 
-	state ICheckpointReader* reader = nullptr;
-	state int64_t totalSize = 0;
+	ICheckpointReader* reader = nullptr;
+	int64_t totalSize = 0;
+	Optional<Error> errToThrow;
 
 	req.reply.setByteLimit(SERVER_KNOBS->CHECKPOINT_TRANSFER_BLOCK_BYTES);
 
@@ -2577,16 +2540,16 @@ ACTOR Future<Void> fetchCheckpointQ(StorageServer* self, FetchCheckpointRequest
 	if (it == self->checkpoints.end()) {
 		req.reply.sendError(checkpoint_not_found());
 		TraceEvent("ServeFetchCheckpointNotFound", self->thisServerID).detail("CheckpointID", req.checkpointID);
-		return Void();
+		co_return;
 	}
 
 	try {
 		reader = newCheckpointReader(it->second, CheckpointAsKeyValues::False, deterministicRandom()->randomUniqueID());
-		wait(reader->init(req.token));
+		co_await reader->init(req.token);
 
-		loop {
-			state Standalone<StringRef> data = wait(reader->nextChunk(CLIENT_KNOBS->REPLY_BYTE_LIMIT));
-			wait(req.reply.onReady());
+		while (true) {
+			Standalone<StringRef> data = co_await reader->nextChunk(CLIENT_KNOBS->REPLY_BYTE_LIMIT);
+			co_await req.reply.onReady();
 			FetchCheckpointReply reply(req.token);
 			reply.data = data;
 			req.reply.send(reply);
@@ -2608,22 +2571,22 @@ ACTOR Future<Void> fetchCheckpointQ(StorageServer* self, FetchCheckpointRequest
 			if (canReplyWith(e)) {
 				req.reply.sendError(e);
 			}
-			state Error err = e;
-			if (reader != nullptr) {
-				wait(reader->close());
-			}
-			throw err;
+			errToThrow = e;
 		}
 	}
 
-	wait(reader->close());
-	return Void();
+	if (reader != nullptr) {
+		co_await reader->close();
+	}
+	if (errToThrow.present()) {
+		throw errToThrow.get();
+	}
 }
 
 // Serves FetchCheckpointKeyValuesRequest, reads local checkpoint and sends it to the client over wire.
-ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpointKeyValuesRequest req) {
-	wait(self->serveFetchCheckpointParallelismLock.take(TaskPriority::DefaultYield));
-	state FlowLock::Releaser holder(self->serveFetchCheckpointParallelismLock);
+Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpointKeyValuesRequest req) {
+	co_await self->serveFetchCheckpointParallelismLock.take(TaskPriority::DefaultYield);
+	FlowLock::Releaser holder(self->serveFetchCheckpointParallelismLock);
 
 	TraceEvent("ServeFetchCheckpointKeyValuesBegin", self->thisServerID)
 	    .detail("CheckpointID", req.checkpointID)
@@ -2636,10 +2599,10 @@ ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpoin
 	if (it == self->checkpoints.end()) {
 		req.reply.sendError(checkpoint_not_found());
 		TraceEvent("ServeFetchCheckpointNotFound", self->thisServerID).detail("CheckpointID", req.checkpointID);
-		return Void();
+		co_return;
 	}
 
-	state ICheckpointReader* reader = nullptr;
+	ICheckpointReader* reader = nullptr;
 	auto crIt = self->liveCheckpointReaders.find(req.checkpointID);
 	if (crIt != self->liveCheckpointReaders.end()) {
 		reader = crIt->second;
@@ -2648,14 +2611,13 @@ ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpoin
 		self->liveCheckpointReaders[req.checkpointID] = reader;
 	}
 
-	state std::unique_ptr<ICheckpointIterator> iter;
+	std::unique_ptr<ICheckpointIterator> iter;
 	try {
-		wait(reader->init(BinaryWriter::toValue(req.range, IncludeVersion())));
+		co_await reader->init(BinaryWriter::toValue(req.range, IncludeVersion()));
 		iter = reader->getIterator(req.range);
 
-		loop {
-			state RangeResult res =
-			    wait(iter->nextBatch(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
+		while (true) {
+			RangeResult res = co_await iter->nextBatch(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT);
 			if (!res.empty()) {
 				TraceEvent(SevDebug, "FetchCheckpontKeyValuesReadRange", self->thisServerID)
 				    .detail("CheckpointID", req.checkpointID)
@@ -2667,7 +2629,7 @@ ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpoin
 				    .detail("CheckpointID", req.checkpointID);
 			}
 
-			wait(req.reply.onReady());
+			co_await req.reply.onReady();
 			FetchCheckpointKeyValuesStreamReply reply;
 			reply.arena.dependsOn(res.arena());
 			for (int i = 0; i < res.size(); ++i) {
@@ -2697,9 +2659,8 @@ ACTOR Future<Void> fetchCheckpointKeyValuesQ(StorageServer* self, FetchCheckpoin
 	iter.reset();
 	if (!reader->inUse()) {
 		self->liveCheckpointReaders.erase(req.checkpointID);
-		wait(reader->close());
+		co_await reader->close();
 	}
-	return Void();
 }
 
 #ifdef NO_INTELLISENSE
@@ -2712,10 +2673,10 @@ size_t WATCH_OVERHEAD_WATCHQ = 0; // only used in IDE so value is irrelevant
 size_t WATCH_OVERHEAD_WATCHIMPL = 0;
 #endif
 
-ACTOR Future<Void> getShardState_impl(StorageServer* data, GetShardStateRequest req) {
+Future<Void> getShardState_impl(StorageServer* data, GetShardStateRequest req) {
 	ASSERT(req.mode != GetShardStateRequest::NO_WAIT);
 
-	loop {
+	while (true) {
 		std::vector<Future<Void>> onChange;
 
 		for (auto t : data->shards.intersectingRanges(req.keys)) {
@@ -2749,11 +2710,11 @@ ACTOR Future<Void> getShardState_impl(StorageServer* data, GetShardStateRequest
 				rep.shards = data->getStorageServerShards(req.keys);
 			}
 			req.reply.send(rep);
-			return Void();
+			co_return;
 		}
 
-		wait(waitForAll(onChange));
-		wait(delay(0)); // onChange could have been triggered by cancellation, let things settle before rechecking
+		co_await waitForAll(onChange);
+		co_await delay(0); // onChange could have been triggered by cancellation, let things settle before rechecking
 	}
 }
 
@@ -2841,14 +2802,14 @@ static inline void copyOptionalValue(Arena* a,
 		a->dependsOn(optionalValue.get().arena());
 	}
 }
-ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
-                                                    StringRef key,
-                                                    Version version,
-                                                    Arena* a,
-                                                    // To provide span context, tags, debug ID to underlying lookups.
-                                                    GetMappedKeyValuesRequest* pOriginalReq) {
-	state GetValueReqAndResultRef getValue;
-	state double getValueStart = g_network->timer();
+Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
+                                              StringRef key,
+                                              Version version,
+                                              Arena* a,
+                                              // To provide span context, tags, debug ID to underlying lookups.
+                                              GetMappedKeyValuesRequest* pOriginalReq) {
+	GetValueReqAndResultRef getValue;
+	double getValueStart = g_network->timer();
 	getValue.key = key;
 
 	if (data->shards[key]->isReadable()) {
@@ -2860,14 +2821,14 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 			// original request level, rather than individual underlying lookups. The reason is that throttle any
 			// individual underlying lookup will fail the original request, which is not productive.
 			data->actors.add(getValueQ(data, req));
-			GetValueReply reply = wait(req.reply.getFuture());
+			GetValueReply reply = co_await req.reply.getFuture();
 			if (!reply.error.present()) {
 				++data->counters.quickGetValueHit;
 				copyOptionalValue(a, getValue, reply.value);
 				const double duration = g_network->timer() - getValueStart;
 				data->counters.readLatencySamples.sample(
 				    duration, ReadLatencySamples::MAPPED_RANGE_LOCAL, trackedReadType(*pOriginalReq));
-				return getValue;
+				co_return getValue;
 			}
 			// Otherwise fallback.
 		} catch (Error& e) {
@@ -2878,18 +2839,18 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 
 	++data->counters.quickGetValueMiss;
 	if (SERVER_KNOBS->QUICK_GET_VALUE_FALLBACK) {
-		state Transaction tr(data->cx);
+		Transaction tr(data->cx);
 		tr.setVersion(version);
 		// TODO: is DefaultPromiseEndpoint the best priority for this?
 		tr.trState->taskID = TaskPriority::DefaultPromiseEndpoint;
 		Future<Optional<Value>> valueFuture = tr.get(key, Snapshot::True);
 		// TODO: async in case it needs to read from other servers.
-		Optional<Value> valueOption = wait(valueFuture);
+		Optional<Value> valueOption = co_await valueFuture;
 		copyOptionalValue(a, getValue, valueOption);
 		double duration = g_network->timer() - getValueStart;
 		data->counters.readLatencySamples.sample(
 		    duration, ReadLatencySamples::MAPPED_RANGE_REMOTE, trackedReadType(*pOriginalReq));
-		return getValue;
+		co_return getValue;
 	} else {
 		throw quick_get_value_miss();
 	}
@@ -2897,29 +2858,29 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 
 // If limit>=0, it returns the first rows in the range (sorted ascending), otherwise the last rows (sorted descending).
 // readRange has O(|result|) + O(log |data|) cost
-ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
-                                          Version version,
-                                          KeyRange range,
-                                          int limit,
-                                          int* pLimitBytes,
-                                          SpanContext parentSpan,
-                                          Optional<ReadOptions> options) {
-	state GetKeyValuesReply result;
-	state StorageServer::VersionedData::ViewAtVersion view = data->data().at(version);
-	state StorageServer::VersionedData::iterator vCurrent = view.end();
-	state KeyRef readBegin;
-	state KeyRef readEnd;
-	state Key readBeginTemp;
-	state int vCount = 0;
-	state Span span("SS:readRange"_loc, parentSpan);
-	state int resultLogicalSize = 0;
-	state int logicalSize = 0;
+Future<GetKeyValuesReply> readRange(StorageServer* data,
+                                    Version version,
+                                    KeyRange range,
+                                    int limit,
+                                    int* pLimitBytes,
+                                    SpanContext parentSpan,
+                                    Optional<ReadOptions> options) {
+	GetKeyValuesReply result;
+	StorageServer::VersionedData::ViewAtVersion view = data->data().at(version);
+	StorageServer::VersionedData::iterator vCurrent = view.end();
+	KeyRef readBegin;
+	KeyRef readEnd;
+	Key readBeginTemp;
+	int vCount = 0;
+	Span span("SS:readRange"_loc, parentSpan);
+	int resultLogicalSize = 0;
+	int logicalSize = 0;
 
 	// for caching the storage queue results during the first PTree traversal
-	state VectorRef<KeyValueRef> resultCache;
+	VectorRef<KeyValueRef> resultCache;
 
 	// for remembering the position in the resultCache
-	state int pos = 0;
+	int pos = 0;
 
 	// Check if the desired key-range is cached
 	auto containingRange = data->cachedRangeMap.rangeContaining(range.begin);
@@ -2982,7 +2943,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			// Read the data on disk up to vCurrent (or the end of the range)
 			readEnd = vCurrent ? std::min(vCurrent.key(), range.end) : range.end;
 			RangeResult atStorageVersion =
-			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, options));
+			    co_await data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, options);
 			logicalSize = atStorageVersion.logicalSize();
 			data->counters.kvScanBytes += logicalSize;
 			resultLogicalSize += logicalSize;
@@ -3080,7 +3041,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			readBegin = vCurrent ? std::max(vCurrent->isClearTo() ? vCurrent->getEndKey() : vCurrent.key(), range.begin)
 			                     : range.begin;
 			RangeResult atStorageVersion =
-			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, options));
+			    co_await data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, options);
 			logicalSize = atStorageVersion.logicalSize();
 			data->counters.kvScanBytes += logicalSize;
 			resultLogicalSize += logicalSize;
@@ -3138,16 +3099,16 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 	ASSERT(result.data.size() == 0 || *pLimitBytes + result.data.end()[-1].expectedSize() + sizeof(KeyValueRef) > 0);
 	result.more = limit == 0 || *pLimitBytes <= 0; // FIXME: Does this have to be exact?
 	result.version = version;
-	return result;
+	co_return result;
 }
 
-ACTOR Future<Key> findKey(StorageServer* data,
-                          KeySelectorRef sel,
-                          Version version,
-                          KeyRange range,
-                          int* pOffset,
-                          SpanContext parentSpan,
-                          Optional<ReadOptions> options)
+Future<Key> findKey(StorageServer* data,
+                    KeySelectorRef sel,
+                    Version version,
+                    KeyRange range,
+                    int* pOffset,
+                    SpanContext parentSpan,
+                    Optional<ReadOptions> options)
 // Attempts to find the key indicated by sel in the data at version, within range.
 // Precondition: selectorInRange(sel, range)
 // If it is found, offset is set to 0 and a key is returned which falls inside range.
@@ -3163,39 +3124,41 @@ ACTOR Future<Key> findKey(StorageServer* data,
 	ASSERT(selectorInRange(sel, range) && version >= data->oldestVersion.get());
 
 	// Count forward or backward distance items, skipping the first one if it == key and skipEqualKey
-	state bool forward = sel.offset > 0; // If forward, result >= sel.getKey(); else result <= sel.getKey()
-	state int sign = forward ? +1 : -1;
-	state bool skipEqualKey = sel.orEqual == forward;
-	state int distance = forward ? sel.offset : 1 - sel.offset;
-	state Span span("SS.findKey"_loc, parentSpan);
+	bool forward = sel.offset > 0; // If forward, result >= sel.getKey(); else result <= sel.getKey()
+	int sign = forward ? +1 : -1;
+	bool skipEqualKey = sel.orEqual == forward;
+	int distance = forward ? sel.offset : 1 - sel.offset;
+	Span span("SS.findKey"_loc, parentSpan);
 
 	// Don't limit the number of bytes if this is a trivial key selector (there will be at most two items returned from
 	// the read range in this case)
-	state int maxBytes;
+	int maxBytes{ 0 };
 	if (sel.offset <= 1 && sel.offset >= 0)
 		maxBytes = std::numeric_limits<int>::max();
 	else
-		maxBytes = (g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::Disabled && BUGGIFY)
-		               ? SERVER_KNOBS->BUGGIFY_LIMIT_BYTES
-		               : SERVER_KNOBS->STORAGE_LIMIT_BYTES;
-
-	state GetKeyValuesReply rep = wait(
-	    readRange(data,
-	              version,
-	              forward ? KeyRangeRef(sel.getKey(), range.end) : KeyRangeRef(range.begin, keyAfter(sel.getKey())),
-	              (distance + skipEqualKey) * sign,
-	              &maxBytes,
-	              span.context,
-	              options));
-	state bool more = rep.more && rep.data.size() != distance + skipEqualKey;
+		maxBytes =
+		    (g_network->isSimulated() &&
+		     simulationPolicyHasCapability(ISimulationPolicy::Capability::LimitStorageServerReadBytes) && BUGGIFY)
+		        ? SERVER_KNOBS->BUGGIFY_LIMIT_BYTES
+		        : SERVER_KNOBS->STORAGE_LIMIT_BYTES;
+
+	GetKeyValuesReply rep = co_await readRange(data,
+	                                           version,
+	                                           forward ? KeyRangeRef(sel.getKey(), range.end)
+	                                                   : KeyRangeRef(range.begin, keyAfter(sel.getKey())),
+	                                           (distance + skipEqualKey) * sign,
+	                                           &maxBytes,
+	                                           span.context,
+	                                           options);
+	bool more = rep.more && rep.data.size() != distance + skipEqualKey;
 
 	// If we get only one result in the reverse direction as a result of the data being too large, we could get stuck in
 	// a loop
 	if (more && !forward && rep.data.size() == 1) {
 		CODE_PROBE(true, "Reverse key selector returned only one result in range read");
 		maxBytes = std::numeric_limits<int>::max();
-		GetKeyValuesReply rep2 = wait(readRange(
-		    data, version, KeyRangeRef(range.begin, keyAfter(sel.getKey())), -2, &maxBytes, span.context, options));
+		GetKeyValuesReply rep2 = co_await readRange(
+		    data, version, KeyRangeRef(range.begin, keyAfter(sel.getKey())), -2, &maxBytes, span.context, options);
 		rep = rep2;
 		more = rep.more && rep.data.size() != distance + skipEqualKey;
 		ASSERT(rep.data.size() == 2 || !more);
@@ -3214,7 +3177,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
 			data->metrics.notifyBytesReadPerKSecond(sel.getKey(), bytesReadPerKSecond);
 		}
 
-		return rep.data[index].key;
+		co_return rep.data[index].key;
 	} else {
 		if (SERVER_KNOBS->READ_SAMPLING_ENABLED) {
 			int64_t bytesReadPerKSecond = SERVER_KNOBS->EMPTY_READ_PENALTY;
@@ -3235,9 +3198,9 @@ ACTOR Future<Key> findKey(StorageServer* data,
 			// This is possible if key/value pairs are very large and only one result is returned on a last less than
 			// query SOMEDAY: graceful handling of exceptionally sized values
 			ASSERT(returnKey != sel.getKey());
-			return returnKey;
+			co_return returnKey;
 		} else {
-			return forward ? range.end : range.begin;
+			co_return forward ? range.end : range.begin;
 		}
 	}
 }
@@ -3273,31 +3236,31 @@ KeyRange getShardKeyRange(StorageServer* data, const KeySelectorRef& sel)
 }
 
 void maybeInjectConsistencyScanCorruption(UID thisServerID, GetKeyValuesRequest const& req, GetKeyValuesReply& reply) {
-	if (g_simulator->consistencyScanState != ISimulator::SimConsistencyScanState::Enabled_InjectCorruption ||
+	if (fdbSimulationPolicyState().consistencyScanState != FDBSimConsistencyScanState::Enabled_InjectCorruption ||
 	    !req.options.present() || !req.options.get().consistencyCheckStartVersion.present() ||
-	    !g_simulator->consistencyScanCorruptRequestKey.present()) {
+	    !fdbSimulationPolicyState().consistencyScanCorruptRequestKey.present()) {
 		return;
 	}
 
 	UID destination = req.reply.getEndpoint().token;
 
-	ASSERT(g_simulator->consistencyScanInjectedCorruptionType.present() ==
-	       g_simulator->consistencyScanInjectedCorruptionDestination.present());
+	ASSERT(fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.present() ==
+	       fdbSimulationPolicyState().consistencyScanInjectedCorruptionDestination.present());
 	// if we already injected a corruption, reinject it if this request was a retransmit of the same one we corrupted
 	// could also check that this storage sent the corruption but the reply endpoints should be globally unique so this
 	// covers it
-	if (g_simulator->consistencyScanInjectedCorruptionDestination.present() &&
-	    (g_simulator->consistencyScanInjectedCorruptionDestination.get() != destination)) {
+	if (fdbSimulationPolicyState().consistencyScanInjectedCorruptionDestination.present() &&
+	    (fdbSimulationPolicyState().consistencyScanInjectedCorruptionDestination.get() != destination)) {
 		return;
 	}
 
 	CODE_PROBE(true, "consistency check injecting corruption");
-	CODE_PROBE(g_simulator->consistencyScanInjectedCorruptionDestination.present() &&
-	               g_simulator->consistencyScanInjectedCorruptionDestination.get() == destination,
+	CODE_PROBE(fdbSimulationPolicyState().consistencyScanInjectedCorruptionDestination.present() &&
+	               fdbSimulationPolicyState().consistencyScanInjectedCorruptionDestination.get() == destination,
 	           "consistency check re-injecting corruption after retransmit",
 	           probe::decoration::rare);
 
-	g_simulator->consistencyScanInjectedCorruptionDestination = destination;
+	fdbSimulationPolicyState().consistencyScanInjectedCorruptionDestination = destination;
 	// FIXME: reinject same type of corruption once we enable other types
 
 	// FIXME: code probe for each type?
@@ -3305,7 +3268,8 @@ void maybeInjectConsistencyScanCorruption(UID thisServerID, GetKeyValuesRequest
 	if (true /*deterministicRandom()->random01() < 0.3*/) {
 		// flip more flag
 		reply.more = !reply.more;
-		g_simulator->consistencyScanInjectedCorruptionType = ISimulator::SimConsistencyScanCorruptionType::FlipMoreFlag;
+		fdbSimulationPolicyState().consistencyScanInjectedCorruptionType =
+		    FDBSimConsistencyScanCorruptionType::FlipMoreFlag;
 	} else {
 		// FIXME: weird memory issues when messing with actual response data, enable and figure out later
 		ASSERT(false);
@@ -3319,37 +3283,37 @@ void maybeInjectConsistencyScanCorruption(UID thisServerID, GetKeyValuesRequest
 
 		if (reply.data.empty()) {
 			// add row to empty response
-			g_simulator->consistencyScanInjectedCorruptionType =
-			    ISimulator::SimConsistencyScanCorruptionType::AddToEmpty;
-			reply.data.push_back_deep(
-			    reply.arena,
-			    KeyValueRef(g_simulator->consistencyScanCorruptRequestKey.get(), "consistencyCheckCorruptValue"_sr));
+			fdbSimulationPolicyState().consistencyScanInjectedCorruptionType =
+			    FDBSimConsistencyScanCorruptionType::AddToEmpty;
+			reply.data.push_back_deep(reply.arena,
+			                          KeyValueRef(fdbSimulationPolicyState().consistencyScanCorruptRequestKey.get(),
+			                                      "consistencyCheckCorruptValue"_sr));
 		} else if (deterministicRandom()->coinflip() || reply.data.back().value.empty()) {
 			// change value in non-empty response
-			g_simulator->consistencyScanInjectedCorruptionType =
-			    ISimulator::SimConsistencyScanCorruptionType::RemoveLastRow;
+			fdbSimulationPolicyState().consistencyScanInjectedCorruptionType =
+			    FDBSimConsistencyScanCorruptionType::RemoveLastRow;
 			reply.data.pop_back();
 		} else {
 			// chop off last byte of first value
-			g_simulator->consistencyScanInjectedCorruptionType =
-			    ISimulator::SimConsistencyScanCorruptionType::ChangeFirstValue;
+			fdbSimulationPolicyState().consistencyScanInjectedCorruptionType =
+			    FDBSimConsistencyScanCorruptionType::ChangeFirstValue;
 
 			reply.data[0].value = reply.data[0].value.substr(0, reply.data[0].value.size() - 1);
 		}
 	}
 
 	TraceEvent(SevWarnAlways, "InjectedConsistencyScanCorruption", thisServerID)
-	    .detail("CorruptionType", g_simulator->consistencyScanInjectedCorruptionType.get())
+	    .detail("CorruptionType", fdbSimulationPolicyState().consistencyScanInjectedCorruptionType.get())
 	    .detail("Version", req.version)
 	    .detail("Count", reply.data.size());
 }
 
-ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
+Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 // Throws a wrong_shard_server if the keys in the request or result depend on data outside this server OR if a large
 // selector offset prevents all data from being read in one range read
 {
-	state Span span("SS:getKeyValues"_loc, req.spanContext);
-	state int64_t resultSize = 0;
+	Span span("SS:getKeyValues"_loc, req.spanContext);
+	int64_t resultSize = 0;
 
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 
@@ -3364,11 +3328,11 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	wait(data->getQueryDelay());
-	state PriorityMultiLock::Lock readLock = wait(data->getReadLock(req.options));
+	co_await data->getQueryDelay();
+	PriorityMultiLock::Lock readLock = co_await data->getReadLock(req.options);
 
 	// Track time from requestTime through now as read queueing wait time
-	state double queueWaitEnd = g_network->timer();
+	double queueWaitEnd = g_network->timer();
 	data->counters.readLatencySamples.sample(
 	    queueWaitEnd - req.requestTime(), ReadLatencySamples::READ_QUEUE_WAIT, trackedReadType(req));
 
@@ -3378,7 +3342,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValues.Before");
 
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
-		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
+		Version version = co_await waitForVersion(data, commitVersion, req.version, span.context);
 		DisabledTraceEvent("VVV", data->thisServerID)
 		    .detail("Version", version)
 		    .detail("ReqVersion", req.version)
@@ -3390,9 +3354,9 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		data->counters.readLatencySamples.sample(
 		    g_network->timer() - queueWaitEnd, ReadLatencySamples::READ_VERSION_WAIT, trackedReadType(req));
 
-		state uint64_t changeCounter = data->shardChangeCounter;
+		uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
-		state KeyRange shard = getShardKeyRange(data, req.begin);
+		KeyRange shard = getShardKeyRange(data, req.begin);
 
 		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
@@ -3413,16 +3377,16 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			throw wrong_shard_server();
 		}
 
-		state int offset1 = 0;
-		state int offset2;
-		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
-		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, shard, &offset1, span.context, req.options);
-		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
-		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, shard, &offset2, span.context, req.options);
-		state Key begin = wait(fBegin);
-		state Key end = wait(fEnd);
+		int offset1 = 0;
+		int offset2{ 0 };
+		Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
+		                         ? Future<Key>(req.begin.getKey())
+		                         : findKey(data, req.begin, version, shard, &offset1, span.context, req.options);
+		Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
+		                       ? Future<Key>(req.end.getKey())
+		                       : findKey(data, req.end, version, shard, &offset2, span.context, req.options);
+		Key begin = co_await fBegin;
+		Key end = co_await fEnd;
 
 		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent(
@@ -3473,11 +3437,11 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			}
 			req.reply.send(none);
 		} else {
-			state int remainingLimitBytes = req.limitBytes;
+			int remainingLimitBytes = req.limitBytes;
 
-			state double kvReadRange = g_network->timer();
-			GetKeyValuesReply _r = wait(readRange(
-			    data, version, KeyRangeRef(begin, end), req.limit, &remainingLimitBytes, span.context, req.options));
+			double kvReadRange = g_network->timer();
+			GetKeyValuesReply _r = co_await readRange(
+			    data, version, KeyRangeRef(begin, end), req.limit, &remainingLimitBytes, span.context, req.options);
 			const double duration = g_network->timer() - kvReadRange;
 			data->counters.readLatencySamples.sample(duration, ReadLatencySamples::KV_READ_RANGE, trackedReadType(req));
 			GetKeyValuesReply r = _r;
@@ -3551,19 +3515,16 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		                                                        abs(req.begin.offset) > maxSelectorOffset ||
 		                                                        abs(req.end.offset) > maxSelectorOffset));
 	}
-
-	return Void();
 }
 
-ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
-    StorageServer* data,
-    StringRef prefix,
-    Version version,
-    Arena* a,
-    // To provide span context, tags, debug ID to underlying lookups.
-    GetMappedKeyValuesRequest* pOriginalReq) {
-	state GetRangeReqAndResultRef getRange;
-	state double getValuesStart = g_network->timer();
+Future<GetRangeReqAndResultRef> quickGetKeyValues(StorageServer* data,
+                                                  StringRef prefix,
+                                                  Version version,
+                                                  Arena* a,
+                                                  // To provide span context, tags, debug ID to underlying lookups.
+                                                  GetMappedKeyValuesRequest* pOriginalReq) {
+	GetRangeReqAndResultRef getRange;
+	double getValuesStart = g_network->timer();
 	getRange.begin = firstGreaterOrEqual(KeyRef(*a, prefix));
 	getRange.end = firstGreaterOrEqual(strinc(prefix, *a));
 	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
@@ -3592,7 +3553,7 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 		// original request level, rather than individual underlying lookups. The reason is that throttle any individual
 		// underlying lookup will fail the original request, which is not productive.
 		data->actors.add(getKeyValuesQ(data, req));
-		GetKeyValuesReply reply = wait(req.reply.getFuture());
+		GetKeyValuesReply reply = co_await req.reply.getFuture();
 		if (!reply.error.present()) {
 			++data->counters.quickGetKeyValuesHit;
 			// Convert GetKeyValuesReply to RangeResult.
@@ -3605,7 +3566,7 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 				g_traceBatch.addEvent("TransactionDebug",
 				                      pOriginalReq->options.get().debugID.get().first(),
 				                      "storageserver.quickGetKeyValues.AfterLocalFetch");
-			return getRange;
+			co_return getRange;
 		}
 		// Otherwise fallback.
 	} catch (Error& e) {
@@ -3614,7 +3575,7 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 
 	++data->counters.quickGetKeyValuesMiss;
 	if (SERVER_KNOBS->QUICK_GET_KEY_VALUES_FALLBACK) {
-		state Transaction tr(data->cx);
+		Transaction tr(data->cx);
 		tr.setVersion(version);
 		if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present()) {
 			tr.debugTransaction(pOriginalReq->options.get().debugID.get());
@@ -3624,7 +3585,7 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 		Future<RangeResult> rangeResultFuture =
 		    tr.getRange(prefixRange(prefix), GetRangeLimits::ROW_LIMIT_UNLIMITED, Snapshot::True);
 		// TODO: async in case it needs to read from other servers.
-		RangeResult rangeResult = wait(rangeResultFuture);
+		RangeResult rangeResult = co_await rangeResultFuture;
 		a->dependsOn(rangeResult.arena());
 		getRange.result = rangeResult;
 		const double duration = g_network->timer() - getValuesStart;
@@ -3634,7 +3595,7 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 			g_traceBatch.addEvent("TransactionDebug",
 			                      pOriginalReq->options.get().debugID.get().first(),
 			                      "storageserver.quickGetKeyValues.AfterRemoteFetch");
-		return getRange;
+		co_return getRange;
 	} else {
 		throw quick_get_key_values_miss();
 	}
@@ -3813,9 +3774,9 @@ AuditGetShardInfoRes getThisServerShardInfo(StorageServer* data, KeyRange range)
 }
 
 // Check consistency between StorageServer->shardInfo and ServerKeys system key space
-ACTOR Future<Void> auditStorageServerShardQ(StorageServer* data, AuditStorageRequest req) {
+Future<Void> auditStorageServerShardQ(StorageServer* data, AuditStorageRequest req) {
 	ASSERT(req.getType() == AuditType::ValidateStorageServerShard);
-	wait(data->serveAuditStorageParallelismLock.take(TaskPriority::DefaultYield));
+	co_await data->serveAuditStorageParallelismLock.take(TaskPriority::DefaultYield);
 	// The trackShardAssignment is correct when at most 1 auditStorageServerShardQ runs
 	// at a time. Currently, this is guaranteed by setting serveAuditStorageParallelismLock == 1
 	// If serveAuditStorageParallelismLock > 1, we need to check trackShardAssignmentMinVersion
@@ -3827,118 +3788,94 @@ ACTOR Future<Void> auditStorageServerShardQ(StorageServer* data, AuditStorageReq
 		           "ExistStorageServerShardAuditExit") // unexpected
 		    .detail("NewAuditId", req.id)
 		    .detail("NewAuditType", req.getType());
-		return Void();
+		co_return;
 	}
-	state FlowLock::Releaser holder(data->serveAuditStorageParallelismLock);
+	FlowLock::Releaser holder(data->serveAuditStorageParallelismLock);
 	TraceEvent(SevInfo, "SSAuditStorageSsShardBegin", data->thisServerID)
 	    .detail("AuditId", req.id)
 	    .detail("AuditRange", req.range);
-	state AuditStorageState res(req.id, data->thisServerID, req.getType());
-	state std::vector<std::string> errors;
-	state std::vector<Future<Void>> fs;
-	state Transaction tr(data->cx);
-	state AuditGetServerKeysRes serverKeyRes;
-	state Version serverKeyReadAtVersion;
-	state KeyRange serverKeyCompleteRange;
-	state AuditGetKeyServersRes keyServerRes;
-	state Version keyServerReadAtVersion;
-	state KeyRange keyServerCompleteRange;
-	state AuditGetShardInfoRes ownRangesLocalViewRes;
-	state Version localShardInfoReadAtVersion;
+	AuditStorageState res(req.id, data->thisServerID, req.getType());
+	std::vector<std::string> errors;
+	std::vector<Future<Void>> fs;
+	Transaction tr(data->cx);
+	AuditGetServerKeysRes serverKeyRes;
+	Version serverKeyReadAtVersion{ 0 };
+	KeyRange serverKeyCompleteRange;
+	AuditGetKeyServersRes keyServerRes;
+	Version keyServerReadAtVersion{ 0 };
+	KeyRange keyServerCompleteRange;
+	AuditGetShardInfoRes ownRangesLocalViewRes;
+	Version localShardInfoReadAtVersion{ 0 };
 	// We want to find out any mismatch between ownRangesSeenByServerKey and ownRangesLocalView and
 	// ownRangesSeenByKeyServerMap
-	state std::unordered_map<UID, std::vector<KeyRange>> ownRangesSeenByKeyServerMap;
-	state std::vector<KeyRange> ownRangesSeenByServerKey;
-	state std::vector<KeyRange> ownRangesSeenByKeyServer;
-	state std::vector<KeyRange> ownRangesLocalView;
-	state std::string failureReason;
+	std::unordered_map<UID, std::vector<KeyRange>> ownRangesSeenByKeyServerMap;
+	std::vector<KeyRange> ownRangesSeenByServerKey;
+	std::vector<KeyRange> ownRangesSeenByKeyServer;
+	std::vector<KeyRange> ownRangesLocalView;
+	std::string failureReason;
 	// Note that since krmReadRange may not return the value of the entire range at a time
 	// Given req.range, a part of the range is returned, thus, only a part of the range is
 	// able to be compared --- claimRange
 	// Given claimRange, rangeToRead is decided for reading the remaining range
 	// At beginning, rangeToRead is req.range
-	state KeyRange claimRange;
-	state Key rangeToReadBegin = req.range.begin;
-	state KeyRangeRef rangeToRead;
-	state int retryCount = 0;
-	state int64_t cumulatedValidatedLocalShardsNum = 0;
-	state int64_t cumulatedValidatedServerKeysNum = 0;
-	state Reference<IRateControl> rateLimiter =
-	    Reference<IRateControl>(new SpeedLimit(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1));
-	state int64_t remoteReadBytes = 0;
-	state double startTime = now();
-	state double lastRateLimiterWaitTime = 0;
-	state double rateLimiterBeforeWaitTime = 0;
-	state double rateLimiterTotalWaitTime = 0;
+	KeyRange claimRange;
+	Key rangeToReadBegin = req.range.begin;
+	KeyRangeRef rangeToRead;
+	int retryCount = 0;
+	int64_t cumulatedValidatedLocalShardsNum = 0;
+	int64_t cumulatedValidatedServerKeysNum = 0;
+	Reference<IRateControl> rateLimiter = makeReference<SpeedLimit>(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1);
+	int64_t remoteReadBytes = 0;
+	double startTime = now();
+	double lastRateLimiterWaitTime = 0;
+	double rateLimiterBeforeWaitTime = 0;
+	double rateLimiterTotalWaitTime = 0;
 
 	try {
-		loop {
-			try {
-				if (data->version.get() == 0) {
-					failureReason = "SS version is 0";
-					throw audit_storage_failed();
-				}
+		while (true) {
+			{
+				Optional<Error> err;
+				try {
+					if (data->version.get() == 0) {
+						failureReason = "SS version is 0";
+						throw audit_storage_failed();
+					}
 
-				// Read serverKeys and shardInfo
-				errors.clear();
-				// We do not reset retryCount for each partial read range
-				ownRangesLocalView.clear();
-				ownRangesSeenByServerKey.clear();
-				ownRangesSeenByKeyServer.clear();
-				ownRangesSeenByKeyServerMap.clear();
-				rangeToRead = KeyRangeRef(rangeToReadBegin, req.range.end);
-
-				ASSERT(!rangeToRead.empty());
-
-				// At this point, shard assignment history guarantees to contain assignments
-				// from localShardInfoReadAtVersion
-				ownRangesLocalViewRes = getThisServerShardInfo(data, rangeToRead);
-				localShardInfoReadAtVersion = ownRangesLocalViewRes.readAtVersion;
-				if (localShardInfoReadAtVersion != data->version.get()) {
-					TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-					           "SSAuditStorageSsShardGRVMismatchError",
-					           data->thisServerID);
-					throw audit_storage_cancelled();
-				}
+					// Read serverKeys and shardInfo
+					errors.clear();
+					// We do not reset retryCount for each partial read range
+					ownRangesLocalView.clear();
+					ownRangesSeenByServerKey.clear();
+					ownRangesSeenByKeyServer.clear();
+					ownRangesSeenByKeyServerMap.clear();
+					rangeToRead = KeyRangeRef(rangeToReadBegin, req.range.end);
+
+					ASSERT(!rangeToRead.empty());
+
+					// At this point, shard assignment history guarantees to contain assignments
+					// from localShardInfoReadAtVersion
+					ownRangesLocalViewRes = getThisServerShardInfo(data, rangeToRead);
+					localShardInfoReadAtVersion = ownRangesLocalViewRes.readAtVersion;
+					if (localShardInfoReadAtVersion != data->version.get()) {
+						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+						           "SSAuditStorageSsShardGRVMismatchError",
+						           data->thisServerID);
+						throw audit_storage_cancelled();
+					}
 
-				// Request to record shard assignment history at least localShardInfoReadAtVersion
-				data->startTrackShardAssignment(localShardInfoReadAtVersion);
-				TraceEvent(SevVerbose, "SSShardAssignmentHistoryRecordStart", data->thisServerID)
-				    .detail("AuditID", req.id);
+					// Request to record shard assignment history at least localShardInfoReadAtVersion
+					data->startTrackShardAssignment(localShardInfoReadAtVersion);
+					TraceEvent(SevVerbose, "SSShardAssignmentHistoryRecordStart", data->thisServerID)
+					    .detail("AuditID", req.id);
 
-				// Transactional read of serverKeys
-				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				fs.clear();
-				fs.push_back(
-				    store(serverKeyRes, getThisServerKeysFromServerKeys(data->thisServerID, &tr, rangeToRead)));
-				fs.push_back(store(keyServerRes, getShardMapFromKeyServers(data->thisServerID, &tr, rangeToRead)));
-				wait(waitForAll(fs));
-				// Get serverKeys result
-				serverKeyCompleteRange = serverKeyRes.completeRange;
-				serverKeyReadAtVersion = serverKeyRes.readAtVersion;
-				// Get keyServers result
-				keyServerCompleteRange = keyServerRes.completeRange;
-				keyServerReadAtVersion = keyServerRes.readAtVersion;
-				// Get bytes read
-				remoteReadBytes = keyServerRes.readBytes + serverKeyRes.readBytes;
-				// We want to do transactional read at a version newer than data->version
-				while (serverKeyReadAtVersion < localShardInfoReadAtVersion) {
-					if (retryCount >= SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX) {
-						failureReason = "Read serverKeys retry count exceeds the max";
-						throw audit_storage_failed();
-					}
-					wait(rateLimiter->getAllowance(remoteReadBytes)); // RateKeeping
-					retryCount++;
-					wait(delay(0.5));
-					tr.reset();
+					// Transactional read of serverKeys
 					tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 					fs.clear();
 					fs.push_back(
 					    store(serverKeyRes, getThisServerKeysFromServerKeys(data->thisServerID, &tr, rangeToRead)));
 					fs.push_back(store(keyServerRes, getShardMapFromKeyServers(data->thisServerID, &tr, rangeToRead)));
-					wait(waitForAll(fs));
+					co_await waitForAll(fs);
 					// Get serverKeys result
 					serverKeyCompleteRange = serverKeyRes.completeRange;
 					serverKeyReadAtVersion = serverKeyRes.readAtVersion;
@@ -3947,147 +3884,199 @@ ACTOR Future<Void> auditStorageServerShardQ(StorageServer* data, AuditStorageReq
 					keyServerReadAtVersion = keyServerRes.readAtVersion;
 					// Get bytes read
 					remoteReadBytes = keyServerRes.readBytes + serverKeyRes.readBytes;
-				} // retry until serverKeyReadAtVersion is as larger as localShardInfoReadAtVersion
-				// Check versions
-				if (serverKeyReadAtVersion < localShardInfoReadAtVersion) {
-					TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-					           "SSAuditStorageSsShardComparedVersionError",
-					           data->thisServerID);
-					throw audit_storage_cancelled();
-				}
-				if (keyServerReadAtVersion != serverKeyReadAtVersion) {
-					TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-					           "SSAuditStorageSsShardKSVersionMismatchError",
-					           data->thisServerID);
-					throw audit_storage_cancelled();
-				}
-				try {
-					wait(timeoutError(data->version.whenAtLeast(serverKeyReadAtVersion), 30));
-				} catch (Error& e) {
-					TraceEvent(SevWarn, "SSAuditStorageSsShardWaitSSVersionTooLong", data->thisServerID)
-					    .detail("ServerKeyReadAtVersion", serverKeyReadAtVersion)
-					    .detail("SSVersion", data->version.get());
-					failureReason = "SS version takes long time to catch up with serverKeyReadAtVersion";
-					throw audit_storage_failed();
-				}
-				// At this point, shard assignment history guarantees to contain assignments
-				// upto serverKeyReadAtVersion
-
-				// Stop requesting to record shard assignment history
-				data->stopTrackShardAssignment();
-				TraceEvent(SevVerbose, "ShardAssignmentHistoryRecordStop", data->thisServerID)
-				    .detail("AuditID", req.id);
-
-				// check any serverKey update between localShardInfoReadAtVersion and serverKeyReadAtVersion
-				std::vector<std::pair<Version, KeyRangeRef>> shardAssignments =
-				    data->getShardAssignmentHistory(localShardInfoReadAtVersion, serverKeyReadAtVersion);
-				TraceEvent(SevInfo, "SSAuditStorageSsShardGetHistory", data->thisServerID)
-				    .detail("AuditId", req.id)
-				    .detail("AuditRange", req.range)
-				    .detail("ServerKeyAtVersion", serverKeyReadAtVersion)
-				    .detail("LocalShardInfoAtVersion", localShardInfoReadAtVersion)
-				    .detail("ShardAssignmentsCount", shardAssignments.size());
-				// Ideally, we revert ownRangesLocalView changes by shard assignment history
-				// Currently, we give up if any update collected
-				if (!shardAssignments.empty()) {
-					failureReason = "Shard assignment history is not empty";
-					throw audit_storage_failed();
-				}
-				// Get claim range
-				KeyRange claimRange = rangeToRead;
-				claimRange = claimRange & serverKeyCompleteRange;
-				if (claimRange.empty()) {
-					TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-					           "SSAuditStorageSsShardOverlapRangeEmpty",
-					           data->thisServerID);
-					throw audit_storage_cancelled();
-				}
-				claimRange = claimRange & keyServerCompleteRange;
-				if (claimRange.empty()) {
-					TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-					           "SSAuditStorageSsShardOverlapRangeEmpty",
-					           data->thisServerID);
-					throw audit_storage_cancelled();
-				}
-				// We only compare within claimRange
-				// Get ownRangesLocalView within claimRange
-				for (auto& range : ownRangesLocalViewRes.ownRanges) {
-					KeyRange overlappingRange = range & claimRange;
-					if (overlappingRange.empty()) {
-						continue;
+					// We want to do transactional read at a version newer than data->version
+					while (serverKeyReadAtVersion < localShardInfoReadAtVersion) {
+						if (retryCount >= SERVER_KNOBS->AUDIT_RETRY_COUNT_MAX) {
+							failureReason = "Read serverKeys retry count exceeds the max";
+							throw audit_storage_failed();
+						}
+						co_await rateLimiter->getAllowance(remoteReadBytes); // RateKeeping
+						retryCount++;
+						co_await delay(0.5);
+						tr.reset();
+						tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+						fs.clear();
+						fs.push_back(
+						    store(serverKeyRes, getThisServerKeysFromServerKeys(data->thisServerID, &tr, rangeToRead)));
+						fs.push_back(
+						    store(keyServerRes, getShardMapFromKeyServers(data->thisServerID, &tr, rangeToRead)));
+						co_await waitForAll(fs);
+						// Get serverKeys result
+						serverKeyCompleteRange = serverKeyRes.completeRange;
+						serverKeyReadAtVersion = serverKeyRes.readAtVersion;
+						// Get keyServers result
+						keyServerCompleteRange = keyServerRes.completeRange;
+						keyServerReadAtVersion = keyServerRes.readAtVersion;
+						// Get bytes read
+						remoteReadBytes = keyServerRes.readBytes + serverKeyRes.readBytes;
+					} // retry until serverKeyReadAtVersion is as larger as localShardInfoReadAtVersion
+					// Check versions
+					if (serverKeyReadAtVersion < localShardInfoReadAtVersion) {
+						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+						           "SSAuditStorageSsShardComparedVersionError",
+						           data->thisServerID);
+						throw audit_storage_cancelled();
 					}
-					ownRangesLocalView.push_back(overlappingRange);
-				}
-				// Get ownRangesSeenByServerKey within claimRange
-				for (auto& range : serverKeyRes.ownRanges) {
-					KeyRange overlappingRange = range & claimRange;
-					if (overlappingRange.empty()) {
-						continue;
+					if (keyServerReadAtVersion != serverKeyReadAtVersion) {
+						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+						           "SSAuditStorageSsShardKSVersionMismatchError",
+						           data->thisServerID);
+						throw audit_storage_cancelled();
 					}
-					ownRangesSeenByServerKey.push_back(overlappingRange);
-				}
-				// Get ownRangesSeenByKeyServer within claimRange
-				if (keyServerRes.rangeOwnershipMap.contains(data->thisServerID)) {
-					std::vector mergedRanges = coalesceRangeList(keyServerRes.rangeOwnershipMap[data->thisServerID]);
-					for (auto& range : mergedRanges) {
+					try {
+						co_await timeoutError(data->version.whenAtLeast(serverKeyReadAtVersion), 30);
+					} catch (Error& e) {
+						TraceEvent(SevWarn, "SSAuditStorageSsShardWaitSSVersionTooLong", data->thisServerID)
+						    .detail("ServerKeyReadAtVersion", serverKeyReadAtVersion)
+						    .detail("SSVersion", data->version.get());
+						failureReason = "SS version takes long time to catch up with serverKeyReadAtVersion";
+						throw audit_storage_failed();
+					}
+					// At this point, shard assignment history guarantees to contain assignments
+					// upto serverKeyReadAtVersion
+
+					// Stop requesting to record shard assignment history
+					data->stopTrackShardAssignment();
+					TraceEvent(SevVerbose, "ShardAssignmentHistoryRecordStop", data->thisServerID)
+					    .detail("AuditID", req.id);
+
+					// check any serverKey update between localShardInfoReadAtVersion and serverKeyReadAtVersion
+					std::vector<std::pair<Version, KeyRangeRef>> shardAssignments =
+					    data->getShardAssignmentHistory(localShardInfoReadAtVersion, serverKeyReadAtVersion);
+					TraceEvent(SevInfo, "SSAuditStorageSsShardGetHistory", data->thisServerID)
+					    .detail("AuditId", req.id)
+					    .detail("AuditRange", req.range)
+					    .detail("ServerKeyAtVersion", serverKeyReadAtVersion)
+					    .detail("LocalShardInfoAtVersion", localShardInfoReadAtVersion)
+					    .detail("ShardAssignmentsCount", shardAssignments.size());
+					// Ideally, we revert ownRangesLocalView changes by shard assignment history
+					// Currently, we give up if any update collected
+					if (!shardAssignments.empty()) {
+						failureReason = "Shard assignment history is not empty";
+						throw audit_storage_failed();
+					}
+					// Get claim range
+					KeyRange claimRange = rangeToRead;
+					claimRange = claimRange & serverKeyCompleteRange;
+					if (claimRange.empty()) {
+						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+						           "SSAuditStorageSsShardOverlapRangeEmpty",
+						           data->thisServerID);
+						throw audit_storage_cancelled();
+					}
+					claimRange = claimRange & keyServerCompleteRange;
+					if (claimRange.empty()) {
+						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+						           "SSAuditStorageSsShardOverlapRangeEmpty",
+						           data->thisServerID);
+						throw audit_storage_cancelled();
+					}
+					// We only compare within claimRange
+					// Get ownRangesLocalView within claimRange
+					for (auto& range : ownRangesLocalViewRes.ownRanges) {
 						KeyRange overlappingRange = range & claimRange;
 						if (overlappingRange.empty()) {
 							continue;
 						}
-						ownRangesSeenByKeyServer.push_back(overlappingRange);
+						ownRangesLocalView.push_back(overlappingRange);
 					}
-				}
-				TraceEvent(SevInfo, "SSAuditStorageSsShardReadDone", data->thisServerID)
-				    .detail("AuditId", req.id)
-				    .detail("AuditRange", req.range)
-				    .detail("ClaimRange", claimRange)
-				    .detail("ServerKeyAtVersion", serverKeyReadAtVersion)
-				    .detail("ShardInfoAtVersion", data->version.get());
-				// Log statistic
-				cumulatedValidatedLocalShardsNum = cumulatedValidatedLocalShardsNum + ownRangesLocalView.size();
-				cumulatedValidatedServerKeysNum = cumulatedValidatedServerKeysNum + ownRangesSeenByServerKey.size();
-				TraceEvent(SevInfo, "SSAuditStorageStatisticShardInfo", data->thisServerID)
-				    .suppressFor(30.0)
-				    .detail("AuditType", req.getType())
-				    .detail("AuditId", req.id)
-				    .detail("AuditRange", req.range)
-				    .detail("CurrentValidatedLocalShardsNum", ownRangesLocalView.size())
-				    .detail("CurrentValidatedServerKeysNum", ownRangesSeenByServerKey.size())
-				    .detail("CurrentValidatedInclusiveRange", claimRange)
-				    .detail("CumulatedValidatedLocalShardsNum", cumulatedValidatedLocalShardsNum)
-				    .detail("CumulatedValidatedServerKeysNum", cumulatedValidatedServerKeysNum)
-				    .detail("CumulatedValidatedInclusiveRange", KeyRangeRef(req.range.begin, claimRange.end));
-
-				// Compare
-				// Compare keyServers and serverKeys
-				if (ownRangesSeenByKeyServer.empty()) {
-					if (!ownRangesSeenByServerKey.empty()) {
-						std::string error =
-						    format("ServerKeys shows %zu ranges that not appear on keyServers for Server(%s): ",
-						           ownRangesSeenByServerKey.size(),
-						           data->thisServerID.toString().c_str(),
-						           describe(ownRangesSeenByServerKey).c_str());
-						TraceEvent(SevError, "SSAuditStorageSsShardError", data->thisServerID)
-						    .setMaxFieldLength(-1)
-						    .setMaxEventLength(-1)
-						    .detail("AuditId", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("ClaimRange", claimRange)
-						    .detail("ErrorMessage", error)
-						    .detail("MismatchedRangeByLocalView", describe(ownRangesSeenByServerKey))
-						    .detail("AuditServer", data->thisServerID);
+					// Get ownRangesSeenByServerKey within claimRange
+					for (auto& range : serverKeyRes.ownRanges) {
+						KeyRange overlappingRange = range & claimRange;
+						if (overlappingRange.empty()) {
+							continue;
+						}
+						ownRangesSeenByServerKey.push_back(overlappingRange);
 					}
-				} else {
+					// Get ownRangesSeenByKeyServer within claimRange
+					if (keyServerRes.rangeOwnershipMap.contains(data->thisServerID)) {
+						std::vector mergedRanges =
+						    coalesceRangeList(keyServerRes.rangeOwnershipMap[data->thisServerID]);
+						for (auto& range : mergedRanges) {
+							KeyRange overlappingRange = range & claimRange;
+							if (overlappingRange.empty()) {
+								continue;
+							}
+							ownRangesSeenByKeyServer.push_back(overlappingRange);
+						}
+					}
+					TraceEvent(SevInfo, "SSAuditStorageSsShardReadDone", data->thisServerID)
+					    .detail("AuditId", req.id)
+					    .detail("AuditRange", req.range)
+					    .detail("ClaimRange", claimRange)
+					    .detail("ServerKeyAtVersion", serverKeyReadAtVersion)
+					    .detail("ShardInfoAtVersion", data->version.get());
+					// Log statistic
+					cumulatedValidatedLocalShardsNum = cumulatedValidatedLocalShardsNum + ownRangesLocalView.size();
+					cumulatedValidatedServerKeysNum = cumulatedValidatedServerKeysNum + ownRangesSeenByServerKey.size();
+					TraceEvent(SevInfo, "SSAuditStorageStatisticShardInfo", data->thisServerID)
+					    .suppressFor(30.0)
+					    .detail("AuditType", req.getType())
+					    .detail("AuditId", req.id)
+					    .detail("AuditRange", req.range)
+					    .detail("CurrentValidatedLocalShardsNum", ownRangesLocalView.size())
+					    .detail("CurrentValidatedServerKeysNum", ownRangesSeenByServerKey.size())
+					    .detail("CurrentValidatedInclusiveRange", claimRange)
+					    .detail("CumulatedValidatedLocalShardsNum", cumulatedValidatedLocalShardsNum)
+					    .detail("CumulatedValidatedServerKeysNum", cumulatedValidatedServerKeysNum)
+					    .detail("CumulatedValidatedInclusiveRange", KeyRangeRef(req.range.begin, claimRange.end));
+
+					// Compare
+					// Compare keyServers and serverKeys
+					if (ownRangesSeenByKeyServer.empty()) {
+						if (!ownRangesSeenByServerKey.empty()) {
+							std::string error =
+							    format("ServerKeys shows %zu ranges that not appear on keyServers for Server(%s): ",
+							           ownRangesSeenByServerKey.size(),
+							           data->thisServerID.toString().c_str(),
+							           describe(ownRangesSeenByServerKey).c_str());
+							TraceEvent(SevError, "SSAuditStorageSsShardError", data->thisServerID)
+							    .setMaxFieldLength(-1)
+							    .setMaxEventLength(-1)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("ClaimRange", claimRange)
+							    .detail("ErrorMessage", error)
+							    .detail("MismatchedRangeByLocalView", describe(ownRangesSeenByServerKey))
+							    .detail("AuditServer", data->thisServerID);
+						}
+					} else {
+						Optional<std::pair<KeyRange, KeyRange>> anyMismatch =
+						    rangesSame(ownRangesSeenByServerKey, ownRangesSeenByKeyServer);
+						if (anyMismatch.present()) { // mismatch detected
+							KeyRange mismatchedRangeByServerKey = anyMismatch.get().first;
+							KeyRange mismatchedRangeByKeyServer = anyMismatch.get().second;
+							std::string error =
+							    format("KeyServers and serverKeys mismatch on Server(%s): ServerKey: %s; KeyServer: %s",
+							           data->thisServerID.toString().c_str(),
+							           mismatchedRangeByServerKey.toString().c_str(),
+							           mismatchedRangeByKeyServer.toString().c_str());
+							TraceEvent(SevError, "SSAuditStorageSsShardError", data->thisServerID)
+							    .setMaxFieldLength(-1)
+							    .setMaxEventLength(-1)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("ClaimRange", claimRange)
+							    .detail("ErrorMessage", error)
+							    .detail("MismatchedRangeByKeyServer", mismatchedRangeByKeyServer)
+							    .detail("MismatchedRangeByServerKey", mismatchedRangeByServerKey)
+							    .detail("AuditServer", data->thisServerID);
+							errors.push_back(error);
+						}
+					}
+
+					// Compare SS shard info and serverKeys
 					Optional<std::pair<KeyRange, KeyRange>> anyMismatch =
-					    rangesSame(ownRangesSeenByServerKey, ownRangesSeenByKeyServer);
+					    rangesSame(ownRangesSeenByServerKey, ownRangesLocalView);
 					if (anyMismatch.present()) { // mismatch detected
 						KeyRange mismatchedRangeByServerKey = anyMismatch.get().first;
-						KeyRange mismatchedRangeByKeyServer = anyMismatch.get().second;
-						std::string error =
-						    format("KeyServers and serverKeys mismatch on Server(%s): ServerKey: %s; KeyServer: %s",
-						           data->thisServerID.toString().c_str(),
-						           mismatchedRangeByServerKey.toString().c_str(),
-						           mismatchedRangeByKeyServer.toString().c_str());
+						KeyRange mismatchedRangeByLocalView = anyMismatch.get().second;
+						std::string error = format(
+						    "Storage server shard info mismatch on Server(%s): ServerKey: %s; ServerShardInfo: %s",
+						    data->thisServerID.toString().c_str(),
+						    mismatchedRangeByServerKey.toString().c_str(),
+						    mismatchedRangeByLocalView.toString().c_str());
 						TraceEvent(SevError, "SSAuditStorageSsShardError", data->thisServerID)
 						    .setMaxFieldLength(-1)
 						    .setMaxEventLength(-1)
@@ -4095,114 +4084,93 @@ ACTOR Future<Void> auditStorageServerShardQ(StorageServer* data, AuditStorageReq
 						    .detail("AuditRange", req.range)
 						    .detail("ClaimRange", claimRange)
 						    .detail("ErrorMessage", error)
-						    .detail("MismatchedRangeByKeyServer", mismatchedRangeByKeyServer)
+						    .detail("MismatchedRangeByLocalView", mismatchedRangeByLocalView)
 						    .detail("MismatchedRangeByServerKey", mismatchedRangeByServerKey)
 						    .detail("AuditServer", data->thisServerID);
 						errors.push_back(error);
 					}
-				}
-
-				// Compare SS shard info and serverKeys
-				Optional<std::pair<KeyRange, KeyRange>> anyMismatch =
-				    rangesSame(ownRangesSeenByServerKey, ownRangesLocalView);
-				if (anyMismatch.present()) { // mismatch detected
-					KeyRange mismatchedRangeByServerKey = anyMismatch.get().first;
-					KeyRange mismatchedRangeByLocalView = anyMismatch.get().second;
-					std::string error =
-					    format("Storage server shard info mismatch on Server(%s): ServerKey: %s; ServerShardInfo: %s",
-					           data->thisServerID.toString().c_str(),
-					           mismatchedRangeByServerKey.toString().c_str(),
-					           mismatchedRangeByLocalView.toString().c_str());
-					TraceEvent(SevError, "SSAuditStorageSsShardError", data->thisServerID)
-					    .setMaxFieldLength(-1)
-					    .setMaxEventLength(-1)
-					    .detail("AuditId", req.id)
-					    .detail("AuditRange", req.range)
-					    .detail("ClaimRange", claimRange)
-					    .detail("ErrorMessage", error)
-					    .detail("MismatchedRangeByLocalView", mismatchedRangeByLocalView)
-					    .detail("MismatchedRangeByServerKey", mismatchedRangeByServerKey)
-					    .detail("AuditServer", data->thisServerID);
-					errors.push_back(error);
-				}
 
-				// Return result
-				if (!errors.empty()) {
-					TraceEvent(SevVerbose, "SSAuditStorageSsShardErrorEnd", data->thisServerID)
-					    .detail("AuditId", req.id)
-					    .detail("AuditRange", req.range)
-					    .detail("AuditServer", data->thisServerID);
-					res.range = claimRange;
-					res.setPhase(AuditPhase::Error);
-					if (!req.ddId.isValid()) {
-						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-						           "SSAuditStorageSsShardDDIdInvalid",
-						           data->thisServerID);
-						throw audit_storage_cancelled();
-					}
-					res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
-					wait(persistAuditStateByServer(data->cx, res));
-					req.reply.sendError(audit_storage_error());
-					break;
-				} else {
-					// Expand persisted complete range
-					res.range = Standalone(KeyRangeRef(req.range.begin, claimRange.end));
-					res.setPhase(AuditPhase::Complete);
-					if (!req.ddId.isValid()) {
-						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-						           "SSAuditStorageSsShardDDIdInvalid",
-						           data->thisServerID);
-						throw audit_storage_cancelled();
-					}
-					res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
-					wait(persistAuditStateByServer(data->cx, res));
-					if (res.range.end < req.range.end) {
-						TraceEvent(SevInfo, "SSAuditStorageSsShardPartialDone", data->thisServerID)
-						    .suppressFor(10.0)
-						    .detail("AuditId", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("AuditServer", data->thisServerID)
-						    .detail("CompleteRange", res.range)
-						    .detail("ClaimRange", claimRange)
-						    .detail("RangeToReadEnd", req.range.end)
-						    .detail("LastRateLimiterWaitTime", lastRateLimiterWaitTime)
-						    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime);
-						rangeToReadBegin = res.range.end;
-					} else { // complete
-						req.reply.send(res);
-						TraceEvent(SevInfo, "SSAuditStorageSsShardComplete", data->thisServerID)
+					// Return result
+					if (!errors.empty()) {
+						TraceEvent(SevVerbose, "SSAuditStorageSsShardErrorEnd", data->thisServerID)
 						    .detail("AuditId", req.id)
 						    .detail("AuditRange", req.range)
-						    .detail("AuditServer", data->thisServerID)
-						    .detail("ClaimRange", claimRange)
-						    .detail("CompleteRange", res.range)
-						    .detail("NumValidatedLocalShards", cumulatedValidatedLocalShardsNum)
-						    .detail("NumValidatedServerKeys", cumulatedValidatedServerKeysNum)
-						    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime)
-						    .detail("TotalTime", now() - startTime);
+						    .detail("AuditServer", data->thisServerID);
+						res.range = claimRange;
+						res.setPhase(AuditPhase::Error);
+						if (!req.ddId.isValid()) {
+							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+							           "SSAuditStorageSsShardDDIdInvalid",
+							           data->thisServerID);
+							throw audit_storage_cancelled();
+						}
+						res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
+						co_await persistAuditStateByServer(data->cx, res);
+						req.reply.sendError(audit_storage_error());
 						break;
+					} else {
+						// Expand persisted complete range
+						res.range = Standalone(KeyRangeRef(req.range.begin, claimRange.end));
+						res.setPhase(AuditPhase::Complete);
+						if (!req.ddId.isValid()) {
+							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+							           "SSAuditStorageSsShardDDIdInvalid",
+							           data->thisServerID);
+							throw audit_storage_cancelled();
+						}
+						res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
+						co_await persistAuditStateByServer(data->cx, res);
+						if (res.range.end < req.range.end) {
+							TraceEvent(SevInfo, "SSAuditStorageSsShardPartialDone", data->thisServerID)
+							    .suppressFor(10.0)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditServer", data->thisServerID)
+							    .detail("CompleteRange", res.range)
+							    .detail("ClaimRange", claimRange)
+							    .detail("RangeToReadEnd", req.range.end)
+							    .detail("LastRateLimiterWaitTime", lastRateLimiterWaitTime)
+							    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime);
+							rangeToReadBegin = res.range.end;
+						} else { // complete
+							req.reply.send(res);
+							TraceEvent(SevInfo, "SSAuditStorageSsShardComplete", data->thisServerID)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditServer", data->thisServerID)
+							    .detail("ClaimRange", claimRange)
+							    .detail("CompleteRange", res.range)
+							    .detail("NumValidatedLocalShards", cumulatedValidatedLocalShardsNum)
+							    .detail("NumValidatedServerKeys", cumulatedValidatedServerKeysNum)
+							    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime)
+							    .detail("TotalTime", now() - startTime);
+							break;
+						}
 					}
+				} catch (Error& e) {
+					err = e;
 				}
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					// In this case, we need not stop tracking shard assignment
-					// The shard history will not get unboundedly large for this case
-					// When this actor gets cancelled, data will be eventually destroyed
-					// Therefore, the shard history will be destroyed
-					throw e;
+				if (err.present()) {
+					if (err.get().code() == error_code_actor_cancelled) {
+						// In this case, we need not stop tracking shard assignment
+						// The shard history will not get unboundedly large for this case
+						// When this actor gets cancelled, data will be eventually destroyed
+						// Therefore, the shard history will be destroyed
+						throw err.get();
+					}
+					data->stopTrackShardAssignment();
+					co_await tr.onError(err.get());
 				}
-				data->stopTrackShardAssignment();
-				wait(tr.onError(e));
 			}
 
 			rateLimiterBeforeWaitTime = now();
-			wait(rateLimiter->getAllowance(remoteReadBytes)); // RateKeeping
+			co_await rateLimiter->getAllowance(remoteReadBytes); // RateKeeping
 			lastRateLimiterWaitTime = now() - rateLimiterBeforeWaitTime;
 			rateLimiterTotalWaitTime = rateLimiterTotalWaitTime + lastRateLimiterWaitTime;
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
-			return Void(); // silently exit
+			co_return; // silently exit
 		}
 		TraceEvent(SevInfo, "SSAuditStorageSsShardFailed", data->thisServerID)
 		    .errorUnsuppressed(e)
@@ -4229,8 +4197,6 @@ ACTOR Future<Void> auditStorageServerShardQ(StorageServer* data, AuditStorageReq
 	// Make sure the history collection is not open due to this audit
 	data->stopTrackShardAssignment();
 	TraceEvent(SevVerbose, "SSShardAssignmentHistoryRecordStopWhenExit", data->thisServerID).detail("AuditID", req.id);
-
-	return Void();
 }
 
 /*
@@ -4294,16 +4260,16 @@ static Future<ErrorOr<GetKeyValuesReply>> issueGetKeyValuesRequest(StorageServer
 // NOTE: We read the ENTIRE restored keyspace (not just rangeToRead with prefix),
 // because restored keys are stored with their original names under the prefix.
 // E.g., source key "mykey" is restored as "\xff\x02/rlog/mykey"
-ACTOR static Future<std::pair<GetKeyValuesReply, GetKeyValuesReply>> fetchSourceAndRestoredData(StorageServer* data,
-                                                                                                KeyRange rangeToRead,
-                                                                                                Version version,
-                                                                                                int limit,
-                                                                                                int limitBytes) {
+static Future<std::pair<GetKeyValuesReply, GetKeyValuesReply>> fetchSourceAndRestoredData(StorageServer* data,
+                                                                                          KeyRange rangeToRead,
+                                                                                          Version version,
+                                                                                          int limit,
+                                                                                          int limitBytes) {
 	// Construct the restored range by adding the restore prefix to the source range
 	// E.g., if source range is "key1 - key2", restored range is "\xff\x02/rlog/key1 - \xff\x02/rlog/key2"
-	state Key restoredBegin = rangeToRead.begin.withPrefix(validateRestoreLogKeys.begin);
-	state Key restoredEnd = rangeToRead.end.withPrefix(validateRestoreLogKeys.begin);
-	state KeyRange restoredRange = KeyRangeRef(restoredBegin, restoredEnd);
+	Key restoredBegin = rangeToRead.begin.withPrefix(validateRestoreLogKeys.begin);
+	Key restoredEnd = rangeToRead.end.withPrefix(validateRestoreLogKeys.begin);
+	KeyRange restoredRange = KeyRangeRef(restoredBegin, restoredEnd);
 
 	TraceEvent("SSAuditRestoreFetch", data->thisServerID)
 	    .detail("RangeToRead", rangeToRead)
@@ -4312,61 +4278,68 @@ ACTOR static Future<std::pair<GetKeyValuesReply, GetKeyValuesReply>> fetchSource
 	    .detail("Limit", limit)
 	    .detail("LimitBytes", limitBytes);
 
-	// Read source data from user key range (this SS must own it since DD sent request here)
-	state Future<ErrorOr<GetKeyValuesReply>> sourceFuture =
-	    issueGetKeyValuesRequest(data, rangeToRead, version, limit, limitBytes);
-
-	// Read restored data from system key space
-	// NOTE: Use database transaction to read system keys since this SS might not own them
-	// IMPORTANT: Use same byte limits as source query to ensure comparable results, since restored keys
-	// are larger (include prefix), which could cause fewer keys to be returned when byte limit is reached
-	state ErrorOr<GetKeyValuesReply> restoredResult;
+	// Read BOTH source and restored data via database transactions to ensure consistent
+	// key ranges regardless of shard boundaries. Using the local SS for source reads
+	// can miss keys at shard boundaries after a bulkload restore.
+	ErrorOr<GetKeyValuesReply> sourceResult;
+	ErrorOr<GetKeyValuesReply> restoredResult;
 	try {
-		state Transaction tr(data->cx);
+		Transaction tr(data->cx);
 		tr.setVersion(version);
+		tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+		GetRangeLimits sourceLimits(limit, limitBytes);
+		RangeResult sourceData = co_await tr.getRange(rangeToRead, sourceLimits, Snapshot::False, Reverse::False);
+
+		// Convert source to GetKeyValuesReply format
+		GetKeyValuesReply sourceReply;
+		sourceReply.data.append_deep(sourceReply.arena, sourceData.begin(), sourceData.size());
+		sourceReply.more = sourceData.more;
+		sourceReply.version = version;
+		sourceResult = sourceReply;
+
+		// Read restored data with the same limits as source to ensure comparable results
 		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-		GetRangeLimits limits(limit, limitBytes);
-		state RangeResult restoredData = wait(tr.getRange(restoredRange, limits, Snapshot::False, Reverse::False));
+		GetRangeLimits restoredLimits(limit, limitBytes);
+		RangeResult restoredData = co_await tr.getRange(restoredRange, restoredLimits, Snapshot::False, Reverse::False);
 
-		// Convert RangeResult to GetKeyValuesReply format
+		// Convert restored to GetKeyValuesReply format
 		GetKeyValuesReply restoredReply;
 		restoredReply.data.append_deep(restoredReply.arena, restoredData.begin(), restoredData.size());
 		restoredReply.more = restoredData.more;
 		restoredReply.version = version;
 		restoredResult = restoredReply;
 	} catch (Error& e) {
-		restoredResult = e;
+		if (sourceResult.isError() || !sourceResult.present()) {
+			sourceResult = e;
+		} else {
+			restoredResult = e;
+		}
 	}
 
-	state Future<ErrorOr<GetKeyValuesReply>> restoredFuture = Future<ErrorOr<GetKeyValuesReply>>(restoredResult);
-
-	wait(success(sourceFuture) && success(restoredFuture));
-
 	// Check for errors
-	if (sourceFuture.get().isError()) {
-		throw sourceFuture.get().getError();
+	if (sourceResult.isError()) {
+		throw sourceResult.getError();
 	}
-	if (restoredFuture.get().isError()) {
-		throw restoredFuture.get().getError();
+	if (sourceResult.get().error.present()) {
+		throw sourceResult.get().error.get();
 	}
-	if (sourceFuture.get().get().error.present()) {
-		throw sourceFuture.get().get().error.get();
+	if (restoredResult.isError()) {
+		throw restoredResult.getError();
 	}
-	if (restoredFuture.get().get().error.present()) {
-		throw restoredFuture.get().get().error.get();
+	if (restoredResult.get().error.present()) {
+		throw restoredResult.get().error.get();
 	}
 
 	// Log what we fetched
 	TraceEvent("SSAuditRestoreFetchResult", data->thisServerID)
-	    .detail("SourceKeys", sourceFuture.get().get().data.size())
-	    .detail("RestoredKeys", restoredFuture.get().get().data.size())
-	    .detail("SourceBytes", sourceFuture.get().get().data.expectedSize())
-	    .detail("RestoredBytes", restoredFuture.get().get().data.expectedSize())
-	    .detail("SourceMore", sourceFuture.get().get().more)
-	    .detail("RestoredMore", restoredFuture.get().get().more);
+	    .detail("SourceKeys", sourceResult.get().data.size())
+	    .detail("RestoredKeys", restoredResult.get().data.size())
+	    .detail("SourceBytes", sourceResult.get().data.expectedSize())
+	    .detail("RestoredBytes", restoredResult.get().data.expectedSize())
+	    .detail("SourceMore", sourceResult.get().more)
+	    .detail("RestoredMore", restoredResult.get().more);
 
-	return std::make_pair(sourceFuture.get().get(), restoredFuture.get().get());
+	co_return std::make_pair(sourceResult.get(), restoredResult.get());
 }
 
 // Helper: Compare source and restored data, returning validation errors
@@ -4518,10 +4491,10 @@ std::vector<std::string> compareSourceAndRestoredData(UID thisServerID,
 	return errors;
 }
 
-ACTOR Future<Void> auditRestoreQ(StorageServer* data, AuditStorageRequest req) {
+Future<Void> auditRestoreQ(StorageServer* data, AuditStorageRequest req) {
 	ASSERT(req.getType() == AuditType::ValidateRestore);
-	wait(data->serveAuditStorageParallelismLock.take(TaskPriority::DefaultYield));
-	state FlowLock::Releaser holder(data->serveAuditStorageParallelismLock);
+	co_await data->serveAuditStorageParallelismLock.take(TaskPriority::DefaultYield);
+	FlowLock::Releaser holder(data->serveAuditStorageParallelismLock);
 
 	TraceEvent(SevInfo, "SSAuditRestoreBegin", data->thisServerID)
 	    .detail("AuditID", req.id)
@@ -4535,165 +4508,238 @@ ACTOR Future<Void> auditRestoreQ(StorageServer* data, AuditStorageRequest req) {
 		    .detail("AuditRange", req.range)
 		    .detail("Error", "Range must be within normalKeys");
 		req.reply.sendError(audit_storage_failed());
-		return Void();
-	}
-
-	state AuditStorageState res(req.id, req.getType());
-	state std::vector<std::string> errors;
-	state Version version;
-	state KeyRange rangeToRead = req.range;
-	state Key rangeToReadBegin = req.range.begin;
-	state KeyRange claimRange;
-	state int limit = 1e4;
-	state int limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-	state int64_t readBytes = 0;
-	state int64_t numValidatedKeys = 0;
-	state int64_t validatedBytes = 0;
-	state bool complete = false;
-	state double startTime = now();
-	state Reference<IRateControl> rateLimiter =
-	    Reference<IRateControl>(new SpeedLimit(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1));
+		co_return;
+	}
 
-	try {
-		loop {
-			try {
-				readBytes = 0;
-				rangeToRead = KeyRangeRef(rangeToReadBegin, req.range.end);
-				ASSERT(!rangeToRead.empty());
+	AuditStorageState res(req.id, req.getType());
+	std::vector<std::string> errors;
+	Version version{ 0 };
+	KeyRange rangeToRead = req.range;
+	Key rangeToReadBegin = req.range.begin;
+	KeyRange claimRange;
+	int limit = SERVER_KNOBS->AUDIT_RESTORE_BATCH_KEY_LIMIT; // Use knob instead of hardcoded 10K
+	int limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+	int64_t readBytes = 0;
+	Error nonRetryableError;
+	int64_t numValidatedKeys = 0;
+	int64_t validatedBytes = 0;
+	int64_t lastPersistBytes = 0; // Track when we last persisted progress
+	bool complete = false;
+	double startTime = now();
+	Reference<IRateControl> rateLimiter = makeReference<SpeedLimit>(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1);
 
-				TraceEvent(SevDebug, "SSAuditRestoreNewRoundBegin", data->thisServerID)
-				    .suppressFor(10.0)
-				    .detail("AuditID", req.id)
-				    .detail("AuditRange", req.range)
-				    .detail("ReadRangeBegin", rangeToReadBegin)
-				    .detail("ReadRangeEnd", req.range.end);
+	{
+		Optional<Error> err;
+		try {
+			while (true) {
+				try {
+					readBytes = 0;
+					rangeToRead = KeyRangeRef(rangeToReadBegin, req.range.end);
+					ASSERT(!rangeToRead.empty());
 
-				errors.clear();
+					TraceEvent(SevDebug, "SSAuditRestoreNewRoundBegin", data->thisServerID)
+					    .suppressFor(10.0)
+					    .detail("AuditID", req.id)
+					    .detail("AuditRange", req.range)
+					    .detail("ReadRangeBegin", rangeToReadBegin)
+					    .detail("ReadRangeEnd", req.range.end);
 
-				// Use current durable version for reading
-				version = data->version.get();
+					errors.clear();
 
-				// Fetch both source and restored data
-				state std::pair<GetKeyValuesReply, GetKeyValuesReply> replyPair =
-				    wait(fetchSourceAndRestoredData(data, rangeToRead, version, limit, limitBytes));
-				state GetKeyValuesReply sourceReply = replyPair.first;
-				state GetKeyValuesReply restoredReply = replyPair.second;
+					// Use current durable version for reading
+					version = data->version.get();
 
-				readBytes = sourceReply.data.expectedSize() + restoredReply.data.expectedSize();
-				validatedBytes += readBytes;
+					// Fetch both source and restored data
+					std::pair<GetKeyValuesReply, GetKeyValuesReply> replyPair =
+					    co_await fetchSourceAndRestoredData(data, rangeToRead, version, limit, limitBytes);
+					GetKeyValuesReply sourceReply = replyPair.first;
+					GetKeyValuesReply restoredReply = replyPair.second;
 
-				// Check if we've completed reading
-				if (!sourceReply.more) {
-					complete = true;
-				}
+					readBytes = sourceReply.data.expectedSize() + restoredReply.data.expectedSize();
+					validatedBytes += readBytes;
 
-				// Compare source data with restored data
-				claimRange = rangeToRead;
-				state Key lastKey = rangeToRead.begin;
-				errors = compareSourceAndRestoredData(data->thisServerID,
-				                                      req.id,
-				                                      req.range,
-				                                      sourceReply,
-				                                      restoredReply,
-				                                      rangeToRead,
-				                                      version,
-				                                      claimRange,
-				                                      lastKey,
-				                                      numValidatedKeys);
-
-				// Update progress in the database
-				KeyRange completeRange = Standalone(KeyRangeRef(rangeToRead.begin, keyAfter(lastKey)));
-				if (!complete && !completeRange.empty() && claimRange.begin == completeRange.begin) {
-					claimRange = claimRange & completeRange;
-					AuditStorageState progressState(req.id, claimRange, req.getType());
-					progressState.setPhase(AuditPhase::Running);
-					progressState.ddId = req.ddId;
-					progressState.auditServerId = data->thisServerID;
-					wait(persistAuditStateByRange(data->cx, progressState));
-				}
+					// FAST-PATH: Detect completely empty sides early and fail immediately.
+					// This prevents iterating through millions of keys when one side is empty.
 
-				// Apply rate limiting
-				wait(rateLimiter->getAllowance(readBytes));
+					// Case 1: Baseline (restored) is completely empty but source has data
+					// This catches the case where mode=BOTH backup created empty range files.
+					if (numValidatedKeys == 0 && restoredReply.data.empty() && !restoredReply.more &&
+					    !sourceReply.data.empty()) {
+						std::string error =
+						    format("Baseline restore data is completely empty but source has %d keys! "
+						           "This likely means the backup's range file snapshot was empty (mode=BOTH bug). "
+						           "First source key: %s",
+						           sourceReply.data.size(),
+						           sourceReply.data.size() > 0
+						               ? Traceable<StringRef>::toString(sourceReply.data[0].key).c_str()
+						               : "N/A");
+						TraceEvent(SevError, "SSAuditRestoreEmptyBaseline", data->thisServerID)
+						    .detail("AuditID", req.id)
+						    .detail("AuditRange", req.range)
+						    .detail("SourceKeyCount", sourceReply.data.size())
+						    .detail("RestoredKeyCount", restoredReply.data.size())
+						    .detail("ErrorMessage", error);
+						errors.push_back(error);
+						break; // Exit loop immediately
+					}
 
-				// If errors found or complete, break
-				if (!errors.empty() || complete) {
-					break;
-				}
+					// Case 2: Source (bulkload restore) is completely empty but baseline has data
+					// This catches the case where bulkload restore failed but baseline worked.
+					if (numValidatedKeys == 0 && sourceReply.data.empty() && !sourceReply.more &&
+					    !restoredReply.data.empty()) {
+						std::string error =
+						    format("Bulkload restore data is completely empty but baseline has %d keys! "
+						           "This likely means the bulkload restore failed. "
+						           "First baseline key: %s",
+						           restoredReply.data.size(),
+						           restoredReply.data.size() > 0
+						               ? Traceable<StringRef>::toString(restoredReply.data[0].key).c_str()
+						               : "N/A");
+						TraceEvent(SevError, "SSAuditRestoreEmptySource", data->thisServerID)
+						    .detail("AuditID", req.id)
+						    .detail("AuditRange", req.range)
+						    .detail("SourceKeyCount", sourceReply.data.size())
+						    .detail("RestoredKeyCount", restoredReply.data.size())
+						    .detail("ErrorMessage", error);
+						errors.push_back(error);
+						break; // Exit loop immediately
+					}
 
-				// Move to next range
-				rangeToReadBegin = keyAfter(lastKey);
-				if (rangeToReadBegin >= req.range.end) {
-					complete = true;
-					break;
-				}
+					// Check if we've completed reading
+					if (!sourceReply.more) {
+						complete = true;
+					}
 
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				throw;
-			}
-		}
+					// Compare source data with restored data
+					claimRange = rangeToRead;
+					Key lastKey = rangeToRead.begin;
+					errors = compareSourceAndRestoredData(data->thisServerID,
+					                                      req.id,
+					                                      req.range,
+					                                      sourceReply,
+					                                      restoredReply,
+					                                      rangeToRead,
+					                                      version,
+					                                      claimRange,
+					                                      lastKey,
+					                                      numValidatedKeys);
+
+					// Update progress in the database (only persist periodically to reduce transaction overhead)
+					KeyRange completeRange = Standalone(KeyRangeRef(rangeToRead.begin, keyAfter(lastKey)));
+					bool shouldPersist =
+					    (validatedBytes - lastPersistBytes) >= SERVER_KNOBS->AUDIT_PROGRESS_PERSIST_BYTES_INTERVAL;
+					if (!complete && shouldPersist && !completeRange.empty() &&
+					    claimRange.begin == completeRange.begin) {
+						claimRange = claimRange & completeRange;
+						AuditStorageState progressState(req.id, claimRange, req.getType());
+						progressState.setPhase(AuditPhase::Running);
+						progressState.ddId = req.ddId;
+						progressState.auditServerId = data->thisServerID;
+						co_await persistAuditStateByRange(data->cx, progressState);
+						lastPersistBytes = validatedBytes;
+					}
 
-		// Set final state
-		if (!errors.empty()) {
-			res.setPhase(AuditPhase::Error);
-			res.error = errors[0]; // Report first error
-			res.range = req.range;
-			TraceEvent(SevWarn, "SSAuditRestoreComplete", data->thisServerID)
-			    .detail("AuditID", req.id)
-			    .detail("AuditRange", req.range)
-			    .detail("Complete", complete)
-			    .detail("ValidationErrors", errors.size())
-			    .detail("NumValidatedKeys", numValidatedKeys)
-			    .detail("ValidatedBytes", validatedBytes)
-			    .detail("Duration", now() - startTime);
-		} else {
-			res.setPhase(AuditPhase::Complete);
-			res.range = req.range;
-			TraceEvent(SevInfo, "SSAuditRestoreComplete", data->thisServerID)
-			    .detail("AuditID", req.id)
-			    .detail("AuditRange", req.range)
-			    .detail("Complete", complete)
-			    .detail("NumValidatedKeys", numValidatedKeys)
-			    .detail("ValidatedBytes", validatedBytes)
-			    .detail("Duration", now() - startTime);
-		}
+					// Apply rate limiting
+					co_await rateLimiter->getAllowance(readBytes);
 
-		// Persist final audit state
-		res.ddId = req.ddId;
-		res.auditServerId = data->thisServerID;
-		wait(persistAuditStateByRange(data->cx, res));
+					// If errors found or complete, break
+					if (!errors.empty() || complete) {
+						break;
+					}
 
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			throw e;
-		}
-		// Send retryable errors back to DD so it can retry with correct SS
-		if (e.code() == error_code_wrong_shard_server) {
-			req.reply.sendError(e);
-			return Void();
+					// Move to next range
+					rangeToReadBegin = keyAfter(lastKey);
+					if (rangeToReadBegin >= req.range.end) {
+						complete = true;
+						break;
+					}
+
+				} catch (Error& e) {
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+					nonRetryableError = e;
+				}
+				// Handle errors outside catch — co_await not allowed in handlers
+				if (nonRetryableError.isValid() && nonRetryableError.code() != error_code_success) {
+					if (nonRetryableError.code() == error_code_future_version ||
+					    nonRetryableError.code() == error_code_transaction_too_old ||
+					    nonRetryableError.code() == error_code_server_overloaded) {
+						TraceEvent(SevWarn, "SSAuditRestoreRetryableError", data->thisServerID)
+						    .detail("AuditID", req.id)
+						    .detail("Error", nonRetryableError.what())
+						    .detail("Version", version)
+						    .detail("Range", rangeToRead);
+						nonRetryableError = Error();
+						co_await delay(1.0);
+						continue;
+					}
+					throw nonRetryableError;
+				}
+			}
+
+			// Set final state
+			if (!errors.empty()) {
+				res.setPhase(AuditPhase::Error);
+				res.error = errors[0]; // Report first error
+				res.range = req.range;
+				TraceEvent(SevWarn, "SSAuditRestoreComplete", data->thisServerID)
+				    .detail("AuditID", req.id)
+				    .detail("AuditRange", req.range)
+				    .detail("Complete", complete)
+				    .detail("ValidationErrors", errors.size())
+				    .detail("NumValidatedKeys", numValidatedKeys)
+				    .detail("ValidatedBytes", validatedBytes)
+				    .detail("Duration", now() - startTime);
+			} else {
+				res.setPhase(AuditPhase::Complete);
+				res.range = req.range;
+				TraceEvent(SevInfo, "SSAuditRestoreComplete", data->thisServerID)
+				    .detail("AuditID", req.id)
+				    .detail("AuditRange", req.range)
+				    .detail("Complete", complete)
+				    .detail("NumValidatedKeys", numValidatedKeys)
+				    .detail("ValidatedBytes", validatedBytes)
+				    .detail("Duration", now() - startTime);
+			}
+
+			// Persist final audit state
+			res.ddId = req.ddId;
+			res.auditServerId = data->thisServerID;
+			co_await persistAuditStateByRange(data->cx, res);
+
+		} catch (Error& e) {
+			err = e;
+		}
+		if (err.present()) {
+			if (err.get().code() == error_code_actor_cancelled) {
+				throw err.get();
+			}
+			// Send retryable errors back to DD so it can retry with correct SS
+			if (err.get().code() == error_code_wrong_shard_server) {
+				req.reply.sendError(err.get());
+				co_return;
+			}
+			res.setPhase(AuditPhase::Error);
+			res.error = err.get().what();
+			res.range = req.range;
+			TraceEvent(SevWarn, "SSAuditRestoreError", data->thisServerID)
+			    .errorUnsuppressed(err.get())
+			    .detail("AuditID", req.id)
+			    .detail("AuditRange", req.range);
+			res.ddId = req.ddId;
+			res.auditServerId = data->thisServerID;
+			co_await persistAuditStateByRange(data->cx, res);
 		}
-		res.setPhase(AuditPhase::Error);
-		res.error = e.what();
-		res.range = req.range;
-		TraceEvent(SevWarn, "SSAuditRestoreError", data->thisServerID)
-		    .errorUnsuppressed(e)
-		    .detail("AuditID", req.id)
-		    .detail("AuditRange", req.range);
-		res.ddId = req.ddId;
-		res.auditServerId = data->thisServerID;
-		wait(persistAuditStateByRange(data->cx, res));
 	}
 
 	req.reply.send(res);
-	return Void();
 }
 
-ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRequest req) {
+Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRequest req) {
 	ASSERT(req.getType() == AuditType::ValidateHA || req.getType() == AuditType::ValidateReplica);
-	wait(data->serveAuditStorageParallelismLock.take(TaskPriority::DefaultYield));
-	state FlowLock::Releaser holder(data->serveAuditStorageParallelismLock);
+	co_await data->serveAuditStorageParallelismLock.take(TaskPriority::DefaultYield);
+	FlowLock::Releaser holder(data->serveAuditStorageParallelismLock);
 
 	TraceEvent(SevInfo, "SSAuditStorageShardReplicaBegin", data->thisServerID)
 	    .detail("AuditID", req.id)
@@ -4701,187 +4747,250 @@ ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRe
 	    .detail("AuditType", req.type)
 	    .detail("TargetServers", describe(req.targetServers));
 
-	state AuditStorageState res(req.id, req.getType()); // we will set range of audit later
-	state std::vector<Optional<Value>> serverListValues;
-	state std::vector<Future<ErrorOr<GetKeyValuesReply>>> fs;
-	state std::vector<std::string> errors;
-	state Version version;
-	state Transaction tr(data->cx);
-	state KeyRange rangeToRead = req.range;
-	state Key rangeToReadBegin = req.range.begin;
-	state KeyRange claimRange;
-	state int limit = 1e4;
-	state int limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-	state int64_t readBytes = 0;
-	state int64_t numValidatedKeys = 0;
-	state int64_t validatedBytes = 0;
-	state bool complete = false;
-	state int64_t checkTimes = 0;
-	state double startTime = now();
-	state double lastRateLimiterWaitTime = 0;
-	state double rateLimiterBeforeWaitTime = 0;
-	state double rateLimiterTotalWaitTime = 0;
-	state Reference<IRateControl> rateLimiter =
-	    Reference<IRateControl>(new SpeedLimit(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1));
+	AuditStorageState res(req.id, req.getType()); // we will set range of audit later
+	std::vector<Optional<Value>> serverListValues;
+	std::vector<Future<ErrorOr<GetKeyValuesReply>>> fs;
+	std::vector<std::string> errors;
+	Version version{ 0 };
+	Transaction tr(data->cx);
+	KeyRange rangeToRead = req.range;
+	Key rangeToReadBegin = req.range.begin;
+	KeyRange claimRange;
+	int limit = 1e4;
+	int limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+	int64_t readBytes = 0;
+	int64_t numValidatedKeys = 0;
+	int64_t validatedBytes = 0;
+	bool complete = false;
+	int64_t checkTimes = 0;
+	double startTime = now();
+	double lastRateLimiterWaitTime = 0;
+	double rateLimiterBeforeWaitTime = 0;
+	double rateLimiterTotalWaitTime = 0;
+	Reference<IRateControl> rateLimiter = makeReference<SpeedLimit>(SERVER_KNOBS->AUDIT_STORAGE_RATE_PER_SERVER_MAX, 1);
 	try {
-		loop {
-			try {
-				readBytes = 0;
-				rangeToRead = KeyRangeRef(rangeToReadBegin, req.range.end);
-				ASSERT(!rangeToRead.empty());
+		while (true) {
+			{
+				Optional<Error> err;
+				try {
+					readBytes = 0;
+					rangeToRead = KeyRangeRef(rangeToReadBegin, req.range.end);
+					ASSERT(!rangeToRead.empty());
 
-				TraceEvent(SevDebug, "SSAuditStorageShardReplicaNewRoundBegin", data->thisServerID)
-				    .suppressFor(10.0)
-				    .detail("AuditID", req.id)
-				    .detail("AuditRange", req.range)
-				    .detail("AuditType", req.type)
-				    .detail("ReadRangeBegin", rangeToReadBegin)
-				    .detail("ReadRangeEnd", req.range.end);
-				serverListValues.clear();
-				errors.clear();
-				fs.clear();
-				tr.reset();
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+					TraceEvent(SevDebug, "SSAuditStorageShardReplicaNewRoundBegin", data->thisServerID)
+					    .suppressFor(10.0)
+					    .detail("AuditID", req.id)
+					    .detail("AuditRange", req.range)
+					    .detail("AuditType", req.type)
+					    .detail("ReadRangeBegin", rangeToReadBegin)
+					    .detail("ReadRangeEnd", req.range.end);
+					serverListValues.clear();
+					errors.clear();
+					fs.clear();
+					tr.reset();
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				// Get SS interfaces
-				std::vector<Future<Optional<Value>>> serverListEntries;
-				for (const UID& id : req.targetServers) {
-					if (id != data->thisServerID) {
-						serverListEntries.push_back(tr.get(serverListKeyFor(id)));
+					// Get SS interfaces
+					std::vector<Future<Optional<Value>>> serverListEntries;
+					for (const UID& id : req.targetServers) {
+						if (id != data->thisServerID) {
+							serverListEntries.push_back(tr.get(serverListKeyFor(id)));
+						}
 					}
-				}
-				wait(store(serverListValues, getAll(serverListEntries)));
+					serverListValues = co_await getAll(serverListEntries);
 
-				// Decide version to compare
-				wait(store(version, tr.getReadVersion()));
+					// Decide version to compare
+					version = co_await tr.getReadVersion();
 
-				// Read remote servers
-				for (const auto& v : serverListValues) {
-					if (!v.present()) {
-						TraceEvent(SevWarn, "SSAuditStorageShardReplicaRemoteServerNotFound", data->thisServerID)
-						    .detail("AuditID", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("AuditType", req.type);
-						throw audit_storage_failed();
+					// Read remote servers
+					for (const auto& v : serverListValues) {
+						if (!v.present()) {
+							TraceEvent(SevWarn, "SSAuditStorageShardReplicaRemoteServerNotFound", data->thisServerID)
+							    .detail("AuditID", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditType", req.type);
+							throw audit_storage_failed();
+						}
+						StorageServerInterface remoteServer = decodeServerListValue(v.get());
+						GetKeyValuesRequest req;
+						req.begin = firstGreaterOrEqual(rangeToRead.begin);
+						req.end = firstGreaterOrEqual(rangeToRead.end);
+						req.limit = limit;
+						req.limitBytes = limitBytes;
+						req.version = version;
+						req.tags = TagSet();
+						fs.push_back(remoteServer.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 					}
-					StorageServerInterface remoteServer = decodeServerListValue(v.get());
-					GetKeyValuesRequest req;
-					req.begin = firstGreaterOrEqual(rangeToRead.begin);
-					req.end = firstGreaterOrEqual(rangeToRead.end);
-					req.limit = limit;
-					req.limitBytes = limitBytes;
-					req.version = version;
-					req.tags = TagSet();
-					fs.push_back(remoteServer.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
-				}
 
-				// Read local server
-				GetKeyValuesRequest localReq;
-				localReq.begin = firstGreaterOrEqual(rangeToRead.begin);
-				localReq.end = firstGreaterOrEqual(rangeToRead.end);
-				localReq.limit = limit;
-				localReq.limitBytes = limitBytes;
-				localReq.version = version;
-				localReq.tags = TagSet();
-				data->actors.add(getKeyValuesQ(data, localReq));
-				fs.push_back(errorOr(localReq.reply.getFuture()));
-				std::vector<ErrorOr<GetKeyValuesReply>> reps = wait(getAll(fs));
-				// Note: getAll() must keep the order of fs
-
-				// Check read result
-				for (int i = 0; i < reps.size(); ++i) {
-					if (reps[i].isError()) {
-						TraceEvent(SevWarn, "SSAuditStorageShardReplicaGetKeyValuesError", data->thisServerID)
-						    .errorUnsuppressed(reps[i].getError())
-						    .detail("AuditID", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("AuditType", req.type)
-						    .detail("ReplyIndex", i)
-						    .detail("RangeRead", rangeToRead)
-						    .detail("TargetServers", describe(req.targetServers));
-						throw reps[i].getError();
+					// Read local server
+					GetKeyValuesRequest localReq;
+					localReq.begin = firstGreaterOrEqual(rangeToRead.begin);
+					localReq.end = firstGreaterOrEqual(rangeToRead.end);
+					localReq.limit = limit;
+					localReq.limitBytes = limitBytes;
+					localReq.version = version;
+					localReq.tags = TagSet();
+					data->actors.add(getKeyValuesQ(data, localReq));
+					fs.push_back(errorOr(localReq.reply.getFuture()));
+					std::vector<ErrorOr<GetKeyValuesReply>> reps = co_await getAll(fs);
+					// Note: getAll() must keep the order of fs
+
+					// Check read result
+					for (int i = 0; i < reps.size(); ++i) {
+						if (reps[i].isError()) {
+							TraceEvent(SevWarn, "SSAuditStorageShardReplicaGetKeyValuesError", data->thisServerID)
+							    .errorUnsuppressed(reps[i].getError())
+							    .detail("AuditID", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditType", req.type)
+							    .detail("ReplyIndex", i)
+							    .detail("RangeRead", rangeToRead)
+							    .detail("TargetServers", describe(req.targetServers));
+							throw reps[i].getError();
+						}
+						if (reps[i].get().error.present()) {
+							TraceEvent(SevWarn, "SSAuditStorageShardReplicaGetKeyValuesError", data->thisServerID)
+							    .errorUnsuppressed(reps[i].get().error.get())
+							    .detail("AuditID", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditType", req.type)
+							    .detail("ReplyIndex", i)
+							    .detail("RangeRead", rangeToRead)
+							    .detail("TargetServers", describe(req.targetServers));
+							throw reps[i].get().error.get();
+						}
+						readBytes = readBytes + reps[i].get().data.expectedSize();
+						validatedBytes = validatedBytes + reps[i].get().data.expectedSize();
+						// If any of reps finishes read, we think we complete
+						// Even some rep does not finish read, this unfinished rep has more key than
+						// the complete rep, which will lead to missKey inconsistency in
+						// this round of check
+						if (!reps[i].get().more) {
+							complete = true;
+						}
 					}
-					if (reps[i].get().error.present()) {
-						TraceEvent(SevWarn, "SSAuditStorageShardReplicaGetKeyValuesError", data->thisServerID)
-						    .errorUnsuppressed(reps[i].get().error.get())
+
+					// Validation
+					claimRange = rangeToRead;
+					const GetKeyValuesReply& local = reps.back().get();
+					if (serverListValues.size() != reps.size() - 1) {
+						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+						           "SSAuditStorageShardReplicaRepsLengthWrong",
+						           data->thisServerID)
+						    .detail("ServerListValuesSize", serverListValues.size())
+						    .detail("RepsSize", reps.size());
+						throw audit_storage_cancelled();
+					}
+					if (reps.size() == 1) {
+						// if no other server to compare
+						TraceEvent(SevWarn, "SSAuditStorageShardReplicaNothingToCompare", data->thisServerID)
 						    .detail("AuditID", req.id)
 						    .detail("AuditRange", req.range)
 						    .detail("AuditType", req.type)
-						    .detail("ReplyIndex", i)
-						    .detail("RangeRead", rangeToRead)
 						    .detail("TargetServers", describe(req.targetServers));
-						throw reps[i].get().error.get();
-					}
-					readBytes = readBytes + reps[i].get().data.expectedSize();
-					validatedBytes = validatedBytes + reps[i].get().data.expectedSize();
-					// If any of reps finishes read, we think we complete
-					// Even some rep does not finish read, this unfinished rep has more key than
-					// the complete rep, which will lead to missKey inconsistency in
-					// this round of check
-					if (!reps[i].get().more) {
 						complete = true;
 					}
-				}
-
-				// Validation
-				claimRange = rangeToRead;
-				const GetKeyValuesReply& local = reps.back().get();
-				if (serverListValues.size() != reps.size() - 1) {
-					TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-					           "SSAuditStorageShardReplicaRepsLengthWrong",
-					           data->thisServerID)
-					    .detail("ServerListValuesSize", serverListValues.size())
-					    .detail("RepsSize", reps.size());
-					throw audit_storage_cancelled();
-				}
-				if (reps.size() == 1) {
-					// if no other server to compare
-					TraceEvent(SevWarn, "SSAuditStorageShardReplicaNothingToCompare", data->thisServerID)
-					    .detail("AuditID", req.id)
-					    .detail("AuditRange", req.range)
-					    .detail("AuditType", req.type)
-					    .detail("TargetServers", describe(req.targetServers));
-					complete = true;
-				}
-				// Compare local and each remote one by one
-				// The last one of reps is local, so skip it
-				for (int repIdx = 0; repIdx < reps.size() - 1; repIdx++) {
-					const GetKeyValuesReply& remote = reps[repIdx].get();
-					// serverListValues and reps should be same order
-					if (!serverListValues[repIdx].present()) { // if not, already throw audit_storage_failed
-						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-						           "SSAuditStorageShardReplicaRepIdxNotPresent",
-						           data->thisServerID)
-						    .detail("RepIdx", repIdx);
-						throw audit_storage_cancelled();
-					}
-					const StorageServerInterface& remoteServer = decodeServerListValue(serverListValues[repIdx].get());
-					Key lastKey = rangeToRead.begin;
-					const int end = std::min(local.data.size(), remote.data.size());
-					bool missingKey = local.data.size() != remote.data.size();
-					// Compare each key one by one
-					std::string error;
-					int i = 0;
-					for (; i < end; ++i) {
-						KeyValueRef remoteKV = remote.data[i];
-						KeyValueRef localKV = local.data[i];
-						if (!req.range.contains(remoteKV.key) || !req.range.contains(localKV.key)) {
-							TraceEvent(SevWarn, "SSAuditStorageShardReplicaKeyOutOfRange", data->thisServerID)
-							    .detail("AuditRange", req.range)
-							    .detail("RemoteServer", remoteServer.toString())
-							    .detail("LocalKey", localKV.key)
-							    .detail("RemoteKey", remoteKV.key);
-							throw wrong_shard_server();
+					// Compare local and each remote one by one
+					// The last one of reps is local, so skip it
+					for (int repIdx = 0; repIdx < reps.size() - 1; repIdx++) {
+						const GetKeyValuesReply& remote = reps[repIdx].get();
+						// serverListValues and reps should be same order
+						if (!serverListValues[repIdx].present()) { // if not, already throw audit_storage_failed
+							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+							           "SSAuditStorageShardReplicaRepIdxNotPresent",
+							           data->thisServerID)
+							    .detail("RepIdx", repIdx);
+							throw audit_storage_cancelled();
+						}
+						const StorageServerInterface& remoteServer =
+						    decodeServerListValue(serverListValues[repIdx].get());
+						Key lastKey = rangeToRead.begin;
+						const int end = std::min(local.data.size(), remote.data.size());
+						bool missingKey = local.data.size() != remote.data.size();
+						// Compare each key one by one
+						std::string error;
+						int i = 0;
+						for (; i < end; ++i) {
+							KeyValueRef remoteKV = remote.data[i];
+							KeyValueRef localKV = local.data[i];
+							if (!req.range.contains(remoteKV.key) || !req.range.contains(localKV.key)) {
+								TraceEvent(SevWarn, "SSAuditStorageShardReplicaKeyOutOfRange", data->thisServerID)
+								    .detail("AuditRange", req.range)
+								    .detail("RemoteServer", remoteServer.toString())
+								    .detail("LocalKey", localKV.key)
+								    .detail("RemoteKey", remoteKV.key);
+								throw wrong_shard_server();
+							}
+							// Check if mismatch
+							if (remoteKV.key != localKV.key) {
+								error = format("Key Mismatch: local server (%016llx): %s, remote server(%016llx) %s",
+								               data->thisServerID.first(),
+								               Traceable<StringRef>::toString(localKV.key).c_str(),
+								               remoteServer.uniqueID.first(),
+								               Traceable<StringRef>::toString(remoteKV.key).c_str());
+								TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
+								    .setMaxFieldLength(-1)
+								    .setMaxEventLength(-1)
+								    .detail("AuditId", req.id)
+								    .detail("AuditRange", req.range)
+								    .detail("ErrorMessage", error)
+								    .detail("Version", version)
+								    .detail("ClaimRange", claimRange);
+								errors.push_back(error);
+								break;
+							} else if (remoteKV.value != localKV.value) {
+								error = format(
+								    "Value Mismatch for Key %s: local server (%016llx): %s, remote server(%016llx) %s",
+								    Traceable<StringRef>::toString(localKV.key).c_str(),
+								    data->thisServerID.first(),
+								    Traceable<StringRef>::toString(localKV.value).c_str(),
+								    remoteServer.uniqueID.first(),
+								    Traceable<StringRef>::toString(remoteKV.value).c_str());
+								TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
+								    .setMaxFieldLength(-1)
+								    .setMaxEventLength(-1)
+								    .detail("AuditId", req.id)
+								    .detail("AuditRange", req.range)
+								    .detail("ErrorMessage", error)
+								    .detail("Version", version)
+								    .detail("ClaimRange", claimRange);
+								errors.push_back(error);
+								break;
+							} else {
+								TraceEvent(SevVerbose, "SSAuditStorageShardReplicaValidatedKey", data->thisServerID)
+								    .detail("Key", localKV.key);
+							}
+							++numValidatedKeys;
+							lastKey = localKV.key;
 						}
-						// Check if mismatch
-						if (remoteKV.key != localKV.key) {
-							error = format("Key Mismatch: local server (%016llx): %s, remote server(%016llx) %s",
-							               data->thisServerID.first(),
-							               Traceable<StringRef>::toString(localKV.key).c_str(),
-							               remoteServer.uniqueID.first(),
-							               Traceable<StringRef>::toString(remoteKV.key).c_str());
+						KeyRange completeRange = Standalone(KeyRangeRef(rangeToRead.begin, keyAfter(lastKey)));
+						if (completeRange.empty() || claimRange.begin != completeRange.begin) {
+							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+							           "SSAuditStorageShardReplicaCompleteRangeUnexpected",
+							           data->thisServerID)
+							    .detail("ClaimRange", claimRange)
+							    .detail("CompleteRange", completeRange);
+							throw audit_storage_cancelled();
+						}
+						claimRange = claimRange & completeRange;
+						if (!error.empty()) { // if key or value mismatch detected
+							continue; // check next remote server
+						}
+						if (!local.more && !remote.more && local.data.size() == remote.data.size()) {
+							continue; // check next remote server
+						} else if (i >= local.data.size() && !local.more && i < remote.data.size()) {
+							if (!missingKey) {
+								TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+								           "SSAuditStorageShardReplicaMissingKeyUnexpected",
+								           data->thisServerID);
+							}
+							std::string error =
+							    format("Missing key(s) form local server (%lld), next key: %s, remote server(%016llx) ",
+							           data->thisServerID.first(),
+							           Traceable<StringRef>::toString(remote.data[i].key).c_str(),
+							           remoteServer.uniqueID.first());
 							TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
 							    .setMaxFieldLength(-1)
 							    .setMaxEventLength(-1)
@@ -4891,15 +5000,18 @@ ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRe
 							    .detail("Version", version)
 							    .detail("ClaimRange", claimRange);
 							errors.push_back(error);
-							break;
-						} else if (remoteKV.value != localKV.value) {
-							error = format(
-							    "Value Mismatch for Key %s: local server (%016llx): %s, remote server(%016llx) %s",
-							    Traceable<StringRef>::toString(localKV.key).c_str(),
-							    data->thisServerID.first(),
-							    Traceable<StringRef>::toString(localKV.value).c_str(),
-							    remoteServer.uniqueID.first(),
-							    Traceable<StringRef>::toString(remoteKV.value).c_str());
+							continue; // check next remote server
+						} else if (i >= remote.data.size() && !remote.more && i < local.data.size()) {
+							if (!missingKey) {
+								TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+								           "SSAuditStorageShardReplicaMissingKeyUnexpected",
+								           data->thisServerID);
+							}
+							std::string error =
+							    format("Missing key(s) form remote server (%lld), next local server(%016llx) key: %s",
+							           remoteServer.uniqueID.first(),
+							           data->thisServerID.first(),
+							           Traceable<StringRef>::toString(local.data[i].key).c_str());
 							TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
 							    .setMaxFieldLength(-1)
 							    .setMaxEventLength(-1)
@@ -4909,120 +5021,38 @@ ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRe
 							    .detail("Version", version)
 							    .detail("ClaimRange", claimRange);
 							errors.push_back(error);
-							break;
-						} else {
-							TraceEvent(SevVerbose, "SSAuditStorageShardReplicaValidatedKey", data->thisServerID)
-							    .detail("Key", localKV.key);
+							continue; // check next remote server
 						}
-						++numValidatedKeys;
-						lastKey = localKV.key;
 					}
-					KeyRange completeRange = Standalone(KeyRangeRef(rangeToRead.begin, keyAfter(lastKey)));
-					if (completeRange.empty() || claimRange.begin != completeRange.begin) {
-						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-						           "SSAuditStorageShardReplicaCompleteRangeUnexpected",
-						           data->thisServerID)
-						    .detail("ClaimRange", claimRange)
-						    .detail("CompleteRange", completeRange);
-						throw audit_storage_cancelled();
-					}
-					claimRange = claimRange & completeRange;
-					if (!error.empty()) { // if key or value mismatch detected
-						continue; // check next remote server
+
+					if (claimRange.end >= req.range.end) {
+						complete = true;
 					}
-					if (!local.more && !remote.more && local.data.size() == remote.data.size()) {
-						continue; // check next remote server
-					} else if (i >= local.data.size() && !local.more && i < remote.data.size()) {
-						if (!missingKey) {
-							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-							           "SSAuditStorageShardReplicaMissingKeyUnexpected",
-							           data->thisServerID);
-						}
-						std::string error =
-						    format("Missing key(s) form local server (%lld), next key: %s, remote server(%016llx) ",
-						           data->thisServerID.first(),
-						           Traceable<StringRef>::toString(remote.data[i].key).c_str(),
-						           remoteServer.uniqueID.first());
-						TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
-						    .setMaxFieldLength(-1)
-						    .setMaxEventLength(-1)
-						    .detail("AuditId", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("ErrorMessage", error)
-						    .detail("Version", version)
-						    .detail("ClaimRange", claimRange);
-						errors.push_back(error);
-						continue; // check next remote server
-					} else if (i >= remote.data.size() && !remote.more && i < local.data.size()) {
-						if (!missingKey) {
-							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-							           "SSAuditStorageShardReplicaMissingKeyUnexpected",
-							           data->thisServerID);
-						}
-						std::string error =
-						    format("Missing key(s) form remote server (%lld), next local server(%016llx) key: %s",
-						           remoteServer.uniqueID.first(),
-						           data->thisServerID.first(),
-						           Traceable<StringRef>::toString(local.data[i].key).c_str());
+
+					TraceEvent(SevInfo, "SSAuditStorageStatisticValidateReplica", data->thisServerID)
+					    .suppressFor(30.0)
+					    .detail("AuditID", req.id)
+					    .detail("AuditRange", req.range)
+					    .detail("AuditType", req.type)
+					    .detail("AuditServer", data->thisServerID)
+					    .detail("ReplicaServers", req.targetServers)
+					    .detail("CheckTimes", checkTimes)
+					    .detail("NumValidatedKeys", numValidatedKeys)
+					    .detail("CurrentValidatedInclusiveRange", claimRange)
+					    .detail("CumulatedValidatedInclusiveRange", KeyRangeRef(req.range.begin, claimRange.end));
+
+					// Return result
+					if (!errors.empty()) {
 						TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
 						    .setMaxFieldLength(-1)
 						    .setMaxEventLength(-1)
 						    .detail("AuditId", req.id)
 						    .detail("AuditRange", req.range)
-						    .detail("ErrorMessage", error)
+						    .detail("ErrorCount", errors.size())
 						    .detail("Version", version)
 						    .detail("ClaimRange", claimRange);
-						errors.push_back(error);
-						continue; // check next remote server
-					}
-				}
-
-				if (claimRange.end >= req.range.end) {
-					complete = true;
-				}
-
-				TraceEvent(SevInfo, "SSAuditStorageStatisticValidateReplica", data->thisServerID)
-				    .suppressFor(30.0)
-				    .detail("AuditID", req.id)
-				    .detail("AuditRange", req.range)
-				    .detail("AuditType", req.type)
-				    .detail("AuditServer", data->thisServerID)
-				    .detail("ReplicaServers", req.targetServers)
-				    .detail("CheckTimes", checkTimes)
-				    .detail("NumValidatedKeys", numValidatedKeys)
-				    .detail("CurrentValidatedInclusiveRange", claimRange)
-				    .detail("CumulatedValidatedInclusiveRange", KeyRangeRef(req.range.begin, claimRange.end));
-
-				// Return result
-				if (!errors.empty()) {
-					TraceEvent(SevError, "SSAuditStorageShardReplicaError", data->thisServerID)
-					    .setMaxFieldLength(-1)
-					    .setMaxEventLength(-1)
-					    .detail("AuditId", req.id)
-					    .detail("AuditRange", req.range)
-					    .detail("ErrorCount", errors.size())
-					    .detail("Version", version)
-					    .detail("ClaimRange", claimRange);
-					res.range = claimRange;
-					res.setPhase(AuditPhase::Error);
-					if (!req.ddId.isValid()) {
-						TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-						           "SSAuditStorageShardReplicaDDIdInvalid",
-						           data->thisServerID);
-						throw audit_storage_cancelled();
-					}
-					res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
-					wait(persistAuditStateByRange(data->cx, res));
-					req.reply.sendError(audit_storage_error());
-					break;
-				} else {
-					if (complete || checkTimes % 100 == 0) {
-						if (complete) {
-							res.range = req.range;
-						} else {
-							res.range = Standalone(KeyRangeRef(req.range.begin, claimRange.end));
-						}
-						res.setPhase(AuditPhase::Complete);
+						res.range = claimRange;
+						res.setPhase(AuditPhase::Error);
 						if (!req.ddId.isValid()) {
 							TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
 							           "SSAuditStorageShardReplicaDDIdInvalid",
@@ -5030,50 +5060,72 @@ ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRe
 							throw audit_storage_cancelled();
 						}
 						res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
-						TraceEvent(SevInfo, "SSAuditStorageShardReplicaProgressPersist", data->thisServerID)
-						    .suppressFor(10.0)
-						    .detail("AuditId", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("AuditServer", data->thisServerID)
-						    .detail("Progress", res.toString());
-						wait(persistAuditStateByRange(data->cx, res));
-					}
-					// Expand persisted complete range
-					if (complete) {
-						req.reply.send(res);
-						TraceEvent(SevInfo, "SSAuditStorageShardReplicaComplete", data->thisServerID)
-						    .detail("AuditId", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("AuditServer", data->thisServerID)
-						    .detail("ReplicaServers", req.targetServers)
-						    .detail("ClaimRange", claimRange)
-						    .detail("CompleteRange", res.range)
-						    .detail("CheckTimes", checkTimes)
-						    .detail("NumValidatedKeys", numValidatedKeys)
-						    .detail("ValidatedBytes", validatedBytes)
-						    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime)
-						    .detail("TotalTime", now() - startTime);
+						co_await persistAuditStateByRange(data->cx, res);
+						req.reply.sendError(audit_storage_error());
 						break;
 					} else {
-						TraceEvent(SevInfo, "SSAuditStorageShardReplicaPartialDone", data->thisServerID)
-						    .suppressFor(10.0)
-						    .detail("AuditId", req.id)
-						    .detail("AuditRange", req.range)
-						    .detail("AuditServer", data->thisServerID)
-						    .detail("ReplicaServers", req.targetServers)
-						    .detail("ClaimRange", claimRange)
-						    .detail("CompleteRange", res.range)
-						    .detail("LastRateLimiterWaitTime", lastRateLimiterWaitTime)
-						    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime);
-						rangeToReadBegin = claimRange.end;
+						if (complete || checkTimes % 100 == 0) {
+							if (complete) {
+								res.range = req.range;
+							} else {
+								res.range = Standalone(KeyRangeRef(req.range.begin, claimRange.end));
+							}
+							res.setPhase(AuditPhase::Complete);
+							if (!req.ddId.isValid()) {
+								TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+								           "SSAuditStorageShardReplicaDDIdInvalid",
+								           data->thisServerID);
+								throw audit_storage_cancelled();
+							}
+							res.ddId = req.ddId; // used to compare req.ddId with existing persisted ddId
+							TraceEvent(SevInfo, "SSAuditStorageShardReplicaProgressPersist", data->thisServerID)
+							    .suppressFor(10.0)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditServer", data->thisServerID)
+							    .detail("Progress", res.toString());
+							co_await persistAuditStateByRange(data->cx, res);
+						}
+						// Expand persisted complete range
+						if (complete) {
+							req.reply.send(res);
+							TraceEvent(SevInfo, "SSAuditStorageShardReplicaComplete", data->thisServerID)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditServer", data->thisServerID)
+							    .detail("ReplicaServers", req.targetServers)
+							    .detail("ClaimRange", claimRange)
+							    .detail("CompleteRange", res.range)
+							    .detail("CheckTimes", checkTimes)
+							    .detail("NumValidatedKeys", numValidatedKeys)
+							    .detail("ValidatedBytes", validatedBytes)
+							    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime)
+							    .detail("TotalTime", now() - startTime);
+							break;
+						} else {
+							TraceEvent(SevInfo, "SSAuditStorageShardReplicaPartialDone", data->thisServerID)
+							    .suppressFor(10.0)
+							    .detail("AuditId", req.id)
+							    .detail("AuditRange", req.range)
+							    .detail("AuditServer", data->thisServerID)
+							    .detail("ReplicaServers", req.targetServers)
+							    .detail("ClaimRange", claimRange)
+							    .detail("CompleteRange", res.range)
+							    .detail("LastRateLimiterWaitTime", lastRateLimiterWaitTime)
+							    .detail("RateLimiterTotalWaitTime", rateLimiterTotalWaitTime);
+							rangeToReadBegin = claimRange.end;
+						}
 					}
+				} catch (Error& e) {
+					err = e;
+				}
+				if (err.present()) {
+					co_await tr.onError(err.get());
 				}
-			} catch (Error& e) {
-				wait(tr.onError(e));
 			}
 
 			rateLimiterBeforeWaitTime = now();
-			wait(rateLimiter->getAllowance(readBytes)); // RateKeeping
+			co_await rateLimiter->getAllowance(readBytes); // RateKeeping
 			lastRateLimiterWaitTime = now() - rateLimiterBeforeWaitTime;
 			rateLimiterTotalWaitTime = rateLimiterTotalWaitTime + lastRateLimiterWaitTime;
 			++checkTimes;
@@ -5081,7 +5133,7 @@ ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRe
 
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
-			return Void(); // silently exit
+			co_return; // silently exit
 		}
 		TraceEvent(SevInfo, "SSAuditStorageShardReplicaFailed", data->thisServerID)
 		    .errorUnsuppressed(e)
@@ -5098,28 +5150,27 @@ ACTOR Future<Void> auditStorageShardReplicaQ(StorageServer* data, AuditStorageRe
 			req.reply.sendError(audit_storage_failed());
 		}
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> getRangeDataToDump(StorageServer* data,
-                                      KeyRange range,
-                                      Version version,
-                                      std::shared_ptr<RangeDumpRawData> output) {
-	state std::map<Key, Value> kvsToDump;
+Future<Void> getRangeDataToDump(StorageServer* data,
+                                KeyRange range,
+                                Version version,
+                                std::shared_ptr<RangeDumpRawData> output) {
+	std::map<Key, Value> kvsToDump;
+	output->arena = Arena();
 	output->kvs.clear();
 	output->sampled.clear();
-	state std::map<Key, Value> sample;
+	std::map<Key, Value> sample;
 	output->kvsBytes = 0;
-	state Key beginKey = range.begin;
-	output->lastKey = range.begin;
-	state bool immediateError = true;
+	Key beginKey = range.begin;
+	output->lastKey = KeyRef(output->arena, range.begin);
+	bool immediateError = true;
 	// Accumulate data read from local storage to output->kvs and make sampling until any error presents
-	loop {
+	while (true) {
 		// Read data and stop for any error
-		state ErrorOr<GetKeyValuesReply> rep;
+		ErrorOr<GetKeyValuesReply> rep;
 		try {
-			state GetKeyValuesRequest localReq;
+			GetKeyValuesRequest localReq;
 			localReq.begin = firstGreaterOrEqual(beginKey);
 			localReq.end = firstGreaterOrEqual(range.end);
 			localReq.version = version;
@@ -5127,7 +5178,7 @@ ACTOR Future<Void> getRangeDataToDump(StorageServer* data,
 			localReq.limitBytes = SERVER_KNOBS->MOVE_SHARD_KRM_BYTE_LIMIT;
 			localReq.tags = TagSet();
 			data->actors.add(getKeyValuesQ(data, localReq));
-			wait(store(rep, errorOr(localReq.reply.getFuture())));
+			rep = co_await errorOr(localReq.reply.getFuture());
 			if (rep.isError()) {
 				throw rep.getError();
 			}
@@ -5146,13 +5197,15 @@ ACTOR Future<Void> getRangeDataToDump(StorageServer* data,
 
 		// Given the data, create KVS and sample. Stop if the accumulated data size is too large.
 		for (const auto& kv : rep.get().data) { // TODO(BulkDump): directly read from special key space.
-			output->lastKey = kv.key;
-			auto res = output->kvs.insert({ kv.key, kv.value });
+			Key key = KeyRef(output->arena, kv.key);
+			Value value = ValueRef(output->arena, kv.value);
+			output->lastKey = key;
+			auto res = output->kvs.insert({ key, value });
 			ASSERT(res.second);
 			ByteSampleInfo sampleInfo = isKeyValueInSample(KeyValueRef(kv.key, kv.value));
 			if (sampleInfo.inSample) {
-				auto resSample =
-				    output->sampled.insert({ kv.key, BinaryWriter::toValue(sampleInfo.sampledSize, Unversioned()) });
+				Value sampleValue = BinaryWriter::toValue(sampleInfo.sampledSize, Unversioned());
+				auto resSample = output->sampled.insert({ key, ValueRef(output->arena, sampleValue) });
 				ASSERT(resSample.second);
 			}
 			output->kvsBytes = output->kvsBytes + kv.expectedSize();
@@ -5165,7 +5218,7 @@ ACTOR Future<Void> getRangeDataToDump(StorageServer* data,
 		if (output->kvsBytes >= SERVER_KNOBS->SS_BULKDUMP_BATCH_BYTES) {
 			break;
 		} else if (!rep.get().more) {
-			output->lastKey = range.end; // Use the range end as the output->lastKey
+			output->lastKey = KeyRef(output->arena, range.end); // Use the range end as the output->lastKey
 			break;
 		}
 
@@ -5176,7 +5229,6 @@ ACTOR Future<Void> getRangeDataToDump(StorageServer* data,
 	if (immediateError) {
 		throw retry();
 	}
-	return Void();
 }
 
 // The SS actor handling bulk dump task sent from DD.
@@ -5198,28 +5250,28 @@ ACTOR Future<Void> getRangeDataToDump(StorageServer* data,
 // Each task must have one manifest file.
 // If the task's range is empty, data file and sample file do not exist
 // If the task's data size is too small, the sample file may omitted
-ACTOR Future<Void> bulkDumpQ(StorageServer* data, BulkDumpRequest req) {
-	wait(data->serveBulkDumpParallelismLock.take(TaskPriority::DefaultYield));
-	state FlowLock::Releaser holder(data->serveBulkDumpParallelismLock); // A SS can handle one bulkDump task at a time
-	state Key rangeBegin = req.bulkDumpState.getRange().begin;
-	state Key rangeEnd = req.bulkDumpState.getRange().end;
-	state BulkLoadTransportMethod transportMethod = req.bulkDumpState.getTransportMethod();
-	state BulkLoadType dumpType = req.bulkDumpState.getType();
-	state int64_t readBytes = 0;
-	state int retryCount = 0;
-	state uint64_t batchNum = 0;
-	state Version versionToDump;
-	state std::shared_ptr<RangeDumpRawData> rangeDumpRawData;
-	state UID jobId = req.bulkDumpState.getJobId();
-	state std::string rootFolderLocal = data->bulkDumpFolder;
-	state std::string rootFolderRemote = req.bulkDumpState.getJobRoot();
+Future<Void> bulkDumpQ(StorageServer* data, BulkDumpRequest req) {
+	co_await data->serveBulkDumpParallelismLock.take(TaskPriority::DefaultYield);
+	FlowLock::Releaser holder(data->serveBulkDumpParallelismLock); // A SS can handle one bulkDump task at a time
+	Key rangeBegin = req.bulkDumpState.getRange().begin;
+	Key rangeEnd = req.bulkDumpState.getRange().end;
+	BulkLoadTransportMethod transportMethod = req.bulkDumpState.getTransportMethod();
+	BulkLoadType dumpType = req.bulkDumpState.getType();
+	int64_t readBytes = 0;
+	int retryCount = 0;
+	uint64_t batchNum = 0;
+	Version versionToDump{ 0 };
+	std::shared_ptr<RangeDumpRawData> rangeDumpRawData;
+	UID jobId = req.bulkDumpState.getJobId();
+	std::string rootFolderLocal = data->bulkDumpFolder;
+	std::string rootFolderRemote = req.bulkDumpState.getJobRoot();
 	// Use jobId and taskId as the folder to store the data of the task range
 	ASSERT(req.bulkDumpState.getTaskId().present());
-	state std::string taskFolder = getBulkDumpJobTaskFolder(jobId, req.bulkDumpState.getTaskId().get());
-	state BulkLoadFileSet destinationFileSets;
-	state Transaction tr(data->cx);
+	std::string taskFolder = getBulkDumpJobTaskFolder(jobId, req.bulkDumpState.getTaskId().get());
+	BulkLoadFileSet destinationFileSets;
+	Transaction tr(data->cx);
 
-	loop {
+	while (true) {
 		try {
 			// Reset data buffer
 			rangeDumpRawData = std::make_shared<RangeDumpRawData>();
@@ -5228,33 +5280,33 @@ ACTOR Future<Void> bulkDumpQ(StorageServer* data, BulkDumpRequest req) {
 			clearFileFolder(abspath(joinPath(rootFolderLocal, taskFolder)));
 
 			// Dump data of rangeToDump in a relativeFolder
-			state KeyRange rangeToDump = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
+			KeyRange rangeToDump = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
 
 			// relativeFolder = <JobId>/<TaskId>/<batchNum>
 			// relativeFolder remains consistent between local path and remote path
-			state std::string relativeFolder = joinPath(taskFolder, std::to_string(batchNum));
+			std::string relativeFolder = joinPath(taskFolder, std::to_string(batchNum));
 
 			// Get version to dump
 			tr.reset();
-			wait(store(versionToDump, tr.getReadVersion()));
+			versionToDump = co_await tr.getReadVersion();
 
 			// Read data
 			// TODO(BulkDump): Read data from other servers at the versionToDump as much as possible
-			wait(getRangeDataToDump(data, rangeToDump, versionToDump, /*output=*/rangeDumpRawData));
+			co_await getRangeDataToDump(data, rangeToDump, versionToDump, /*output=*/rangeDumpRawData);
 
 			// Generate local file paths and remote file paths
 			// The data in KVStore is dumped to the local folder at first and then
 			// the local files are uploaded to the remote folder
 			// Local files and remotes files have the same relative path but different root
-			state std::pair<BulkLoadFileSet, BulkLoadFileSet> resFileSets =
+			std::pair<BulkLoadFileSet, BulkLoadFileSet> resFileSets =
 			    getLocalRemoteFileSetSetting(versionToDump,
 			                                 relativeFolder,
 			                                 /*rootLocal=*/rootFolderLocal,
 			                                 /*rootRemote=*/rootFolderRemote);
 
 			// The remote file path:
-			state BulkLoadFileSet localFileSetSetting = resFileSets.first;
-			state BulkLoadFileSet remoteFileSetSetting = resFileSets.second;
+			BulkLoadFileSet localFileSetSetting = resFileSets.first;
+			BulkLoadFileSet remoteFileSetSetting = resFileSets.second;
 
 			// Generate byte sampling setting
 			BulkLoadByteSampleSetting byteSampleSetting(0,
@@ -5264,17 +5316,17 @@ ACTOR Future<Void> bulkDumpQ(StorageServer* data, BulkDumpRequest req) {
 			                                            SERVER_KNOBS->MIN_BYTE_SAMPLING_PROBABILITY);
 
 			// Write to SST file
-			state KeyRange dataRange = rangeToDump & KeyRangeRef(rangeBegin, keyAfter(rangeDumpRawData->lastKey));
-			state BulkLoadManifest manifest =
-			    wait(dumpDataFileToLocalDirectory(data->thisServerID,
-			                                      rangeDumpRawData,
-			                                      localFileSetSetting,
-			                                      remoteFileSetSetting,
-			                                      byteSampleSetting,
-			                                      versionToDump,
-			                                      dataRange, // the actual range of the rangeDumpRawData.kvs
-			                                      dumpType,
-			                                      transportMethod));
+			KeyRange dataRange = rangeToDump & KeyRangeRef(rangeBegin, keyAfter(rangeDumpRawData->lastKey));
+			BulkLoadManifest manifest =
+			    co_await dumpDataFileToLocalDirectory(data->thisServerID,
+			                                          rangeDumpRawData,
+			                                          localFileSetSetting,
+			                                          remoteFileSetSetting,
+			                                          byteSampleSetting,
+			                                          versionToDump,
+			                                          dataRange, // the actual range of the rangeDumpRawData.kvs
+			                                          dumpType,
+			                                          transportMethod);
 			readBytes = readBytes + rangeDumpRawData->kvsBytes;
 			TraceEvent(bulkLoadVerboseEventSev(), "SSBulkDumpDataFileGenerated", data->thisServerID)
 			    .detail("TaskID", req.bulkDumpState.getTaskId())
@@ -5291,23 +5343,23 @@ ACTOR Future<Void> bulkDumpQ(StorageServer* data, BulkDumpRequest req) {
 			    .detail("RemoteFileSet", manifest.getFileSet().toString());
 
 			// Upload Files
-			state BulkLoadFileSet localFileSet = localFileSetSetting;
+			BulkLoadFileSet localFileSet = localFileSetSetting;
 			if (!manifest.hasDataFile()) {
 				localFileSet.removeDataFile();
 			}
 			if (!manifest.hasByteSampleFile()) {
 				localFileSet.removeByteSampleFile();
 			}
-			wait(uploadBulkDumpFileSet(
-			    req.bulkDumpState.getTransportMethod(), localFileSet, manifest.getFileSet(), data->thisServerID));
+			co_await uploadBulkDumpFileSet(
+			    req.bulkDumpState.getTransportMethod(), localFileSet, manifest.getFileSet(), data->thisServerID);
 
 			// Progressively set metadata of the data range as complete phase
 			// Persist remoteFilePaths to the corresponding range
 			if (!dataRange.empty()) {
 				// The persisting range (dataRange) must be exactly same as the range presented in the manifest file
 				ASSERT(dataRange == manifest.getRange());
-				wait(persistCompleteBulkDumpRange(data->cx,
-				                                  req.bulkDumpState.generateBulkDumpMetadataToPersist(manifest)));
+				co_await persistCompleteBulkDumpRange(data->cx,
+				                                      req.bulkDumpState.generateBulkDumpMetadataToPersist(manifest));
 			}
 
 			// Move to the next range
@@ -5339,13 +5391,11 @@ ACTOR Future<Void> bulkDumpQ(StorageServer* data, BulkDumpRequest req) {
 			}
 			retryCount++;
 		}
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
 
 	// Do best effort cleanup
 	clearFileFolder(abspath(joinPath(rootFolderLocal, taskFolder)), data->thisServerID, /*ignoreError=*/true);
-
-	return Void();
 }
 
 TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
@@ -5444,25 +5494,24 @@ TEST_CASE("/fdbserver/storageserver/constructMappedKey") {
 }
 
 // Issues a secondary query (either range and point read) and fills results into "kvm".
-ACTOR Future<Void> mapSubquery(StorageServer* data,
-                               Version version,
-                               GetMappedKeyValuesRequest* pOriginalReq,
-                               Arena* pArena,
-                               bool isRangeQuery,
-                               KeyValueRef* it,
-                               MappedKeyValueRef* kvm,
-                               Key mappedKey) {
+Future<Void> mapSubquery(StorageServer* data,
+                         Version version,
+                         GetMappedKeyValuesRequest* pOriginalReq,
+                         Arena* pArena,
+                         bool isRangeQuery,
+                         KeyValueRef* it,
+                         MappedKeyValueRef* kvm,
+                         Key mappedKey) {
 	if (isRangeQuery) {
 		// Use the mappedKey as the prefix of the range query.
-		GetRangeReqAndResultRef getRange = wait(quickGetKeyValues(data, mappedKey, version, pArena, pOriginalReq));
+		GetRangeReqAndResultRef getRange = co_await quickGetKeyValues(data, mappedKey, version, pArena, pOriginalReq);
 		kvm->key = it->key;
 		kvm->value = it->value;
 		kvm->reqAndResult = getRange;
 	} else {
-		GetValueReqAndResultRef getValue = wait(quickGetValue(data, mappedKey, version, pArena, pOriginalReq));
+		GetValueReqAndResultRef getValue = co_await quickGetValue(data, mappedKey, version, pArena, pOriginalReq);
 		kvm->reqAndResult = getValue;
 	}
-	return Void();
 }
 
 int getMappedKeyValueSize(MappedKeyValueRef mappedKeyValue) {
@@ -5480,13 +5529,13 @@ int getMappedKeyValueSize(MappedKeyValueRef mappedKeyValue) {
 	return bytes;
 }
 
-ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
-                                                   GetKeyValuesReply input,
-                                                   StringRef mapper,
-                                                   // To provide span context, tags, debug ID to underlying lookups.
-                                                   GetMappedKeyValuesRequest* pOriginalReq,
-                                                   int* remainingLimitBytes) {
-	state GetMappedKeyValuesReply result;
+Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
+                                             GetKeyValuesReply input,
+                                             StringRef mapper,
+                                             // To provide span context, tags, debug ID to underlying lookups.
+                                             GetMappedKeyValuesRequest* pOriginalReq,
+                                             int* remainingLimitBytes) {
+	GetMappedKeyValuesReply result;
 	result.version = input.version;
 	result.cached = input.cached;
 	result.arena.dependsOn(input.arena);
@@ -5495,7 +5544,7 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
 		g_traceBatch.addEvent(
 		    "TransactionDebug", pOriginalReq->options.get().debugID.get().first(), "storageserver.mapKeyValues.Start");
-	state Tuple mappedKeyFormatTuple;
+	Tuple mappedKeyFormatTuple;
 
 	try {
 		mappedKeyFormatTuple = Tuple::unpack(mapper);
@@ -5503,15 +5552,15 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 		TraceEvent("MapperNotTuple").error(e).detail("Mapper", mapper);
 		throw mapper_not_tuple();
 	}
-	state std::vector<Optional<Tuple>> vt;
-	state bool isRangeQuery = false;
+	std::vector<Optional<Tuple>> vt;
+	bool isRangeQuery = false;
 	preprocessMappedKey(mappedKeyFormatTuple, vt, isRangeQuery);
 
-	state int sz = input.data.size();
+	int sz = input.data.size();
 	const int k = std::min(sz, SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE);
-	state std::vector<MappedKeyValueRef> kvms(k);
-	state std::vector<Future<Void>> subqueries;
-	state int offset = 0;
+	std::vector<MappedKeyValueRef> kvms(k);
+	std::vector<Future<Void>> subqueries;
+	int offset = 0;
 	if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
 		g_traceBatch.addEvent("TransactionDebug",
 		                      pOriginalReq->options.get().debugID.get().first(),
@@ -5535,7 +5584,7 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 			subqueries.push_back(
 			    mapSubquery(data, input.version, pOriginalReq, &result.arena, isRangeQuery, it, kvm, mappedKey));
 		}
-		wait(waitForAll(subqueries));
+		co_await waitForAll(subqueries);
 		if (pOriginalReq->options.present() && pOriginalReq->options.get().debugID.present())
 			g_traceBatch.addEvent("TransactionDebug",
 			                      pOriginalReq->options.get().debugID.get().first(),
@@ -5567,17 +5616,17 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 		g_traceBatch.addEvent("TransactionDebug",
 		                      pOriginalReq->options.get().debugID.get().first(),
 		                      "storageserver.mapKeyValues.AfterAll");
-	return result;
+	co_return result;
 }
 
 // Most of the actor is copied from getKeyValuesQ. I tried to use templates but things become nearly impossible after
 // combining actor shenanigans with template shenanigans.
-ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRequest req)
+Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRequest req)
 // Throws a wrong_shard_server if the keys in the request or result depend on data outside this server OR if a large
 // selector offset prevents all data from being read in one range read
 {
-	state Span span("SS:getMappedKeyValues"_loc, req.spanContext);
-	state int64_t resultSize = 0;
+	Span span("SS:getMappedKeyValues"_loc, req.spanContext);
+	int64_t resultSize = 0;
 
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 
@@ -5591,11 +5640,11 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	wait(data->getQueryDelay());
-	state PriorityMultiLock::Lock readLock = wait(data->getReadLock(req.options));
+	co_await data->getQueryDelay();
+	PriorityMultiLock::Lock readLock = co_await data->getReadLock(req.options);
 
 	// Track time from requestTime through now as read queueing wait time
-	state double queueWaitEnd = g_network->timer();
+	double queueWaitEnd = g_network->timer();
 	data->counters.readLatencySamples.sample(
 	    queueWaitEnd - req.requestTime(), ReadLatencySamples::READ_QUEUE_WAIT, trackedReadType(req));
 
@@ -5605,13 +5654,13 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getMappedKeyValues.Before");
 		// VERSION_VECTOR change
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
-		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
+		Version version = co_await waitForVersion(data, commitVersion, req.version, span.context);
 		data->counters.readLatencySamples.sample(
 		    g_network->timer() - queueWaitEnd, ReadLatencySamples::READ_VERSION_WAIT, trackedReadType(req));
 
-		state uint64_t changeCounter = data->shardChangeCounter;
+		uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
-		state KeyRange shard = getShardKeyRange(data, req.begin);
+		KeyRange shard = getShardKeyRange(data, req.begin);
 
 		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("TransactionDebug",
@@ -5629,16 +5678,16 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 			throw wrong_shard_server();
 		}
 
-		state int offset1 = 0;
-		state int offset2;
-		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
-		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, shard, &offset1, span.context, req.options);
-		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
-		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, shard, &offset2, span.context, req.options);
-		state Key begin = wait(fBegin);
-		state Key end = wait(fEnd);
+		int offset1 = 0;
+		int offset2{ 0 };
+		Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
+		                         ? Future<Key>(req.begin.getKey())
+		                         : findKey(data, req.begin, version, shard, &offset1, span.context, req.options);
+		Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
+		                       ? Future<Key>(req.end.getKey())
+		                       : findKey(data, req.end, version, shard, &offset2, span.context, req.options);
+		Key begin = co_await fBegin;
+		Key end = co_await fEnd;
 
 		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("TransactionDebug",
@@ -5677,24 +5726,24 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 			                                     std::max<KeyRef>(req.begin.getKey(), req.end.getKey())));
 			req.reply.send(none);
 		} else {
-			state int remainingLimitBytes = req.limitBytes;
+			int remainingLimitBytes = req.limitBytes;
 			// create a temporary byte limit for index fetching ONLY, this should be excessive
 			// because readRange is cheap when reading additional bytes
-			state int bytesForIndex =
+			int bytesForIndex =
 			    std::min(req.limitBytes, (int)(req.limitBytes * SERVER_KNOBS->FRACTION_INDEX_BYTELIMIT_PREFETCH));
-			GetKeyValuesReply getKeyValuesReply = wait(readRange(
-			    data, version, KeyRangeRef(begin, end), req.limit, &bytesForIndex, span.context, req.options));
+			GetKeyValuesReply getKeyValuesReply = co_await readRange(
+			    data, version, KeyRangeRef(begin, end), req.limit, &bytesForIndex, span.context, req.options);
 
 			// Unlock read lock before the subqueries because each
 			// subquery will route back to getValueQ or getKeyValuesQ with a new request having the same
 			// read options which will each acquire the ssLock.
 			readLock.release();
 
-			state GetMappedKeyValuesReply r;
+			GetMappedKeyValuesReply r;
 			try {
 				// Map the scanned range to another list of keys and look up.
 				GetMappedKeyValuesReply _r =
-				    wait(mapKeyValues(data, getKeyValuesReply, req.mapper, &req, &remainingLimitBytes));
+				    co_await mapKeyValues(data, getKeyValuesReply, req.mapper, &req, &remainingLimitBytes);
 				r = _r;
 			} catch (Error& e) {
 				// catch txn_too_old here if prefetch runs for too long, and returns it back to client
@@ -5750,16 +5799,14 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 		                                                        abs(req.begin.offset) > maxSelectorOffset ||
 		                                                        abs(req.end.offset) > maxSelectorOffset));
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRequest req)
+Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRequest req)
 // Throws a wrong_shard_server if the keys in the request or result depend on data outside this server OR if a large
 // selector offset prevents all data from being read in one range read
 {
-	state Span span("SS:getKeyValuesStream"_loc, req.spanContext);
-	state int64_t resultSize = 0;
+	Span span("SS:getKeyValuesStream"_loc, req.spanContext);
+	int64_t resultSize = 0;
 
 	req.reply.setByteLimit(SERVER_KNOBS->RANGESTREAM_LIMIT_BYTES);
 	++data->counters.getRangeStreamQueries;
@@ -5772,7 +5819,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	wait(delay(0, TaskPriority::DefaultEndpoint));
+	co_await delay(0, TaskPriority::DefaultEndpoint);
 
 	try {
 		if (req.options.present() && req.options.get().debugID.present())
@@ -5780,11 +5827,11 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			    "TransactionDebug", req.options.get().debugID.get().first(), "storageserver.getKeyValuesStream.Before");
 
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
-		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
+		Version version = co_await waitForVersion(data, commitVersion, req.version, span.context);
 
-		state uint64_t changeCounter = data->shardChangeCounter;
+		uint64_t changeCounter = data->shardChangeCounter;
 		//		try {
-		state KeyRange shard = getShardKeyRange(data, req.begin);
+		KeyRange shard = getShardKeyRange(data, req.begin);
 
 		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("TransactionDebug",
@@ -5802,16 +5849,16 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			throw wrong_shard_server();
 		}
 
-		state int offset1 = 0;
-		state int offset2;
-		state Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
-		                               ? Future<Key>(req.begin.getKey())
-		                               : findKey(data, req.begin, version, shard, &offset1, span.context, req.options);
-		state Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
-		                             ? Future<Key>(req.end.getKey())
-		                             : findKey(data, req.end, version, shard, &offset2, span.context, req.options);
-		state Key begin = wait(fBegin);
-		state Key end = wait(fEnd);
+		int offset1 = 0;
+		int offset2{ 0 };
+		Future<Key> fBegin = req.begin.isFirstGreaterOrEqual()
+		                         ? Future<Key>(req.begin.getKey())
+		                         : findKey(data, req.begin, version, shard, &offset1, span.context, req.options);
+		Future<Key> fEnd = req.end.isFirstGreaterOrEqual()
+		                       ? Future<Key>(req.end.getKey())
+		                       : findKey(data, req.end, version, shard, &offset2, span.context, req.options);
+		Key begin = co_await fBegin;
+		Key end = co_await fEnd;
 		if (req.options.present() && req.options.get().debugID.present())
 			g_traceBatch.addEvent("TransactionDebug",
 			                      req.options.get().debugID.get().first(),
@@ -5849,9 +5896,9 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 			req.reply.send(none);
 			req.reply.sendError(end_of_stream());
 		} else {
-			loop {
-				wait(req.reply.onReady());
-				state PriorityMultiLock::Lock readLock = wait(data->getReadLock(req.options));
+			while (true) {
+				co_await req.reply.onReady();
+				PriorityMultiLock::Lock readLock = co_await data->getReadLock(req.options);
 
 				if (version < data->oldestVersion.get()) {
 					throw transaction_too_old();
@@ -5859,8 +5906,9 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 				// Even if TSS mode is Disabled, this may be the second test in a restarting test where the first run
 				// had it enabled.
-				state int byteLimit =
-				    (BUGGIFY && g_network->isSimulated() && g_simulator->tssMode == ISimulator::TSSMode::Disabled &&
+				int byteLimit =
+				    (BUGGIFY && g_network->isSimulated() &&
+				     simulationPolicyHasCapability(ISimulationPolicy::Capability::LimitStorageServerReadBytes) &&
 				     !data->isTss() && !data->isSSWithTSSPair())
 				        ? 1
 				        : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
@@ -5870,8 +5918,8 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 				    .detail("Begin", begin.printable())
 				    .detail("End", end.printable());
 
-				GetKeyValuesReply _r = wait(readRange(
-				    data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, req.options));
+				GetKeyValuesReply _r = co_await readRange(
+				    data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, req.options);
 				readLock.release();
 				GetKeyValuesStreamReply r(_r);
 
@@ -5941,13 +5989,11 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
 	++data->counters.finishedQueries;
-
-	return Void();
 }
 
-ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
-	state Span span("SS:getKey"_loc, req.spanContext);
-	state int64_t resultSize = 0;
+Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
+	Span span("SS:getKey"_loc, req.spanContext);
+	int64_t resultSize = 0;
 
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 
@@ -5958,26 +6004,26 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 
 	// Active load balancing runs at a very high priority (to obtain accurate queue lengths)
 	// so we need to downgrade here
-	wait(data->getQueryDelay());
-	state PriorityMultiLock::Lock readLock = wait(data->getReadLock(req.options));
+	co_await data->getQueryDelay();
+	PriorityMultiLock::Lock readLock = co_await data->getReadLock(req.options);
 
 	// Track time from requestTime through now as read queueing wait time
-	state double queueWaitEnd = g_network->timer();
+	double queueWaitEnd = g_network->timer();
 	data->counters.readLatencySamples.sample(
 	    queueWaitEnd - req.requestTime(), ReadLatencySamples::READ_QUEUE_WAIT, trackedReadType(req));
 
 	try {
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
-		state Version version = wait(waitForVersion(data, commitVersion, req.version, req.spanContext));
+		Version version = co_await waitForVersion(data, commitVersion, req.version, req.spanContext);
 		data->counters.readLatencySamples.sample(
 		    g_network->timer() - queueWaitEnd, ReadLatencySamples::READ_VERSION_WAIT, trackedReadType(req));
 
-		state uint64_t changeCounter = data->shardChangeCounter;
+		uint64_t changeCounter = data->shardChangeCounter;
 
 		KeyRange shard = getShardKeyRange(data, req.sel);
 
-		state int offset;
-		Key absoluteKey = wait(findKey(data, req.sel, version, shard, &offset, req.spanContext, req.options));
+		int offset{ 0 };
+		Key absoluteKey = co_await findKey(data, req.sel, version, shard, &offset, req.spanContext, req.options);
 
 		data->checkChangeCounter(changeCounter,
 		                         KeyRangeRef(std::min<KeyRef>(req.sel.getKey(), absoluteKey),
@@ -6035,8 +6081,6 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
 		data->counters.readLatencyBands.addMeasurement(
 		    duration, 1, Filtered(resultSize > maxReadBytes || abs(req.sel.offset) > maxSelectorOffset));
 	}
-
-	return Void();
 }
 
 void getQueuingMetrics(StorageServer* self, StorageQueuingMetricsRequest const& req) {
@@ -6068,9 +6112,9 @@ void getQueuingMetrics(StorageServer* self, StorageQueuingMetricsRequest const&
 #pragma region Updates
 #endif
 
-ACTOR Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager) {
+Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager) {
 	eager->finishKeyBegin();
-	state ReadOptions options;
+	ReadOptions options;
 	options.type = ReadType::EAGER;
 	if (eager->enableClearRangeEagerReads) {
 		std::vector<Future<Key>> keyEnd(eager->keyBegin.size());
@@ -6078,8 +6122,8 @@ ACTOR Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager)
 			keyEnd[i] = data->storage.readNextKeyInclusive(eager->keyBegin[i], options);
 		data->counters.eagerReadsKeys += keyEnd.size();
 
-		state Future<std::vector<Key>> futureKeyEnds = getAll(keyEnd);
-		state std::vector<Key> keyEndVal = wait(futureKeyEnds);
+		Future<std::vector<Key>> futureKeyEnds = getAll(keyEnd);
+		std::vector<Key> keyEndVal = co_await futureKeyEnds;
 		for (const auto& key : keyEndVal) {
 			data->counters.kvScanBytes += key.expectedSize();
 		}
@@ -6090,8 +6134,8 @@ ACTOR Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager)
 	for (int i = 0; i < value.size(); i++)
 		value[i] = data->storage.readValuePrefix(eager->keys[i].first, eager->keys[i].second, options);
 
-	state Future<std::vector<Optional<Value>>> futureValues = getAll(value);
-	std::vector<Optional<Value>> optionalValues = wait(futureValues);
+	Future<std::vector<Optional<Value>>> futureValues = getAll(value);
+	std::vector<Optional<Value>> optionalValues = co_await futureValues;
 	for (const auto& value : optionalValues) {
 		if (value.present()) {
 			data->counters.kvGetBytes += value.expectedSize();
@@ -6099,8 +6143,6 @@ ACTOR Future<Void> doEagerReads(StorageServer* data, UpdateEagerReadInfo* eager)
 	}
 	data->counters.eagerReadsKeys += eager->keys.size();
 	eager->value = optionalValues;
-
-	return Void();
 }
 
 bool changeDurableVersion(StorageServer* data, Version desiredDurableVersion) {
@@ -6129,7 +6171,7 @@ bool changeDurableVersion(StorageServer* data, Version desiredDurableVersion) {
 		int64_t bytesDurable = VERSION_OVERHEAD;
 		for (const auto& m : v.mutations) {
 			bytesDurable += mvccStorageBytes(m);
-			auto i = verData.atLatest().find(m.param1);
+			auto i = verData.latestFind(m.param1);
 			if (i) {
 				ASSERT(i.key() == m.param1);
 				ASSERT(i.insertVersion() >= nextDurableVersion);
@@ -6139,7 +6181,7 @@ bool changeDurableVersion(StorageServer* data, Version desiredDurableVersion) {
 			if (m.type == MutationRef::SetValue) {
 				// A set can split a clear, so there might be another entry immediately after this one that should also
 				// be cleaned up
-				i = verData.atLatest().upper_bound(m.param1);
+				i = verData.latestUpperBound(m.param1);
 				if (i) {
 					ASSERT(i.insertVersion() >= nextDurableVersion);
 					if (i.insertVersion() == nextDurableVersion)
@@ -6190,10 +6232,11 @@ bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, U
 	// After this function call, m should be copied into an arena immediately (before modifying data, shards, or eager)
 	if (m.type != MutationRef::ClearRange && m.type != MutationRef::SetValue) {
 		Optional<StringRef> oldVal;
-		auto it = data.atLatest().lastLessOrEqual(m.param1);
-		if (it != data.atLatest().end() && it->isValue() && it.key() == m.param1)
+		auto it = data.latestLastLessOrEqual(m.param1);
+		auto latestEnd = data.latestEnd();
+		if (it != latestEnd && it->isValue() && it.key() == m.param1)
 			oldVal = it->getValue();
-		else if (it != data.atLatest().end() && it->isClearTo() && it->getEndKey() > m.param1) {
+		else if (it != latestEnd && it->isClearTo() && it->getEndKey() > m.param1) {
 			CODE_PROBE(true, "Atomic op right after a clear.");
 		} else {
 			Optional<Value>& oldThing = eager->getValue(m.param1);
@@ -6312,7 +6355,7 @@ void applyMutation(StorageServer* self,
 	if (m.type == MutationRef::SetValue) {
 		// VersionedMap (data) is bookkeeping all empty ranges. If the key to be set is new, it is supposed to be in a
 		// range what was empty. Break the empty range into halves.
-		auto prev = data.atLatest().lastLessOrEqual(m.param1);
+		auto prev = data.latestLastLessOrEqual(m.param1);
 		if (prev && prev->isClearTo() && prev->getEndKey() > m.param1) {
 			ASSERT(prev.key() <= m.param1);
 			KeyRef end = prev->getEndKey();
@@ -6367,7 +6410,7 @@ void removeDataRange(StorageServer* ss,
 	     ++r)
 		range = KeyRangeRef(range.begin, r->end());
 
-	auto endClear = data.atLatest().lastLess(range.end);
+	auto endClear = data.latestLastLess(range.end);
 	if (endClear && endClear->isClearTo() && endClear->getEndKey() > range.end) {
 		// This clear has been bumped up to insertVersion==data.getLatestVersion and needs a corresponding mutation log
 		// entry to forget
@@ -6377,7 +6420,7 @@ void removeDataRange(StorageServer* ss,
 		++ss->counters.kvSystemClearRanges;
 	}
 
-	auto beginClear = data.atLatest().lastLess(range.begin);
+	auto beginClear = data.latestLastLess(range.begin);
 	if (beginClear && beginClear->isClearTo() && beginClear->getEndKey() > range.begin) {
 		// We don't need any special mutationLog entry - because the begin key and insert version are unchanged the
 		// original clear
@@ -6462,11 +6505,11 @@ void splitMutation(StorageServer* data, KeyRangeMap<T>& map, MutationRef const&
 		ASSERT(false); // Unknown mutation type in splitMutations
 }
 
-ACTOR Future<Void> logFetchKeysWarning(AddingShard* shard) {
-	state double startTime = now();
-	loop {
-		state double waitSeconds = BUGGIFY ? 5.0 : 600.0;
-		wait(delay(waitSeconds));
+Future<Void> logFetchKeysWarning(AddingShard* shard) {
+	double startTime = now();
+	while (true) {
+		double waitSeconds = BUGGIFY ? 5.0 : 600.0;
+		co_await delay(waitSeconds);
 
 		const auto traceEventLevel =
 		    waitSeconds > SERVER_KNOBS->FETCH_KEYS_TOO_LONG_TIME_CRITERIA ? SevWarnAlways : SevInfo;
@@ -6524,25 +6567,25 @@ class FetchKeysMetricReporter {
 	}
 };
 
-ACTOR Future<Void> tryGetRange(PromiseStream<RangeResult> results, Transaction* tr, KeyRange keys) {
+Future<Void> tryGetRange(PromiseStream<RangeResult> results, Transaction* tr, KeyRange keys) {
 	if (SERVER_KNOBS->FETCH_USING_STREAMING) {
-		wait(tr->getRangeStream(results, keys, GetRangeLimits(), Snapshot::True));
-		return Void();
+		co_await tr->getRangeStream(results, keys, GetRangeLimits(), Snapshot::True);
+		co_return;
 	}
 
-	state KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
-	state KeySelectorRef end = firstGreaterOrEqual(keys.end);
+	KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
+	KeySelectorRef end = firstGreaterOrEqual(keys.end);
 
 	try {
-		loop {
+		while (true) {
 			GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, SERVER_KNOBS->FETCH_BLOCK_BYTES);
 			limits.minRows = 0;
-			state RangeResult rep = wait(tr->getRange(begin, end, limits, Snapshot::True));
+			RangeResult rep = co_await tr->getRange(begin, end, limits, Snapshot::True);
 			results.send(rep);
 
 			if (!rep.more) {
 				results.sendError(end_of_stream());
-				return Void();
+				co_return;
 			}
 
 			begin = rep.nextBeginKeySelector();
@@ -6575,26 +6618,26 @@ bool fetchKeyCanRetry(const Error& e) {
 	}
 }
 
-ACTOR Future<Void> bulkLoadFetchKeyValueFileToLoad(StorageServer* data,
-                                                   std::string dir,
-                                                   BulkLoadTaskState bulkLoadTaskState,
-                                                   std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) {
+Future<Void> bulkLoadFetchKeyValueFileToLoad(StorageServer* data,
+                                             std::string dir,
+                                             BulkLoadTaskState bulkLoadTaskState,
+                                             std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) {
 	localFileSets->clear();
 	ASSERT(bulkLoadTaskState.getLoadType() == BulkLoadType::SST);
 	TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchSSTFile", data->thisServerID)
 	    .detail("JobID", bulkLoadTaskState.getJobId().toString())
 	    .detail("TaskID", bulkLoadTaskState.getTaskId().toString())
 	    .detail("Dir", abspath(dir));
-	state double fetchStartTime = now();
+	double fetchStartTime = now();
 	// Download data file from fromRemoteFileSet to toLocalFileSet
-	state std::shared_ptr<BulkLoadFileSetKeyMap> fromRemoteFileSets = std::make_shared<BulkLoadFileSetKeyMap>();
+	std::shared_ptr<BulkLoadFileSetKeyMap> fromRemoteFileSets = std::make_shared<BulkLoadFileSetKeyMap>();
 	for (const auto& manifest : bulkLoadTaskState.getManifests()) {
 		fromRemoteFileSets->push_back(std::make_pair(manifest.getRange(), manifest.getFileSet()));
 		// Note that manifest.range may contain more than the task range. We will cut-off data outside the task
 		// range when we read the kvs.
 	}
-	wait(bulkLoadDownloadTaskFileSets(
-	    bulkLoadTaskState.getTransportMethod(), fromRemoteFileSets, localFileSets, dir, data->thisServerID));
+	co_await bulkLoadDownloadTaskFileSets(
+	    bulkLoadTaskState.getTransportMethod(), fromRemoteFileSets, localFileSets, dir, data->thisServerID);
 	// Do not need byte sampling locally in fetchKeys
 	const double duration = now() - fetchStartTime;
 	const int64_t totalBytes = bulkLoadTaskState.getTotalBytes();
@@ -6616,21 +6659,20 @@ ACTOR Future<Void> bulkLoadFetchKeyValueFileToLoad(StorageServer* data,
 	    .detail("Duration", duration)
 	    .detail("TotalBytes", totalBytes)
 	    .detail("Rate", duration == 0 ? -1.0 : (double)totalBytes / duration);
-	return Void();
 }
 
-ACTOR Future<Void> tryGetRangeForBulkLoadFromSST(PromiseStream<RangeResult> results,
-                                                 KeyRange keys,
-                                                 std::string sstFilePath,
-                                                 bool lastOne) {
-	state Key beginKey = keys.begin;
-	state Key endKey = keys.end;
+Future<Void> tryGetRangeForBulkLoadFromSST(PromiseStream<RangeResult> results,
+                                           KeyRange keys,
+                                           std::string sstFilePath,
+                                           bool lastOne) {
+	Key beginKey = keys.begin;
+	Key endKey = keys.end;
 	try {
-		state std::unique_ptr<IRocksDBSstFileReader> reader = newRocksDBSstFileReader(
+		std::unique_ptr<IRocksDBSstFileReader> reader = newRocksDBSstFileReader(
 		    keys, SERVER_KNOBS->SS_BULKLOAD_GETRANGE_BATCH_SIZE, SERVER_KNOBS->FETCH_BLOCK_BYTES);
 		// TODO(BulkLoad): this can be a slow task. We will make this as async call.
 		reader->open(abspath(sstFilePath));
-		loop {
+		while (true) {
 			// TODO(BulkLoad): this is a blocking call. We will make this as async call.
 			RangeResult rep = reader->getRange(KeyRangeRef(beginKey, endKey));
 			if (!rep.more) {
@@ -6638,7 +6680,7 @@ ACTOR Future<Void> tryGetRangeForBulkLoadFromSST(PromiseStream<RangeResult> resu
 					rep.more = false;
 					results.send(rep);
 					results.sendError(end_of_stream());
-					return Void();
+					co_return;
 				} else {
 					if (!rep.empty()) {
 						rep.more = true;
@@ -6646,13 +6688,13 @@ ACTOR Future<Void> tryGetRangeForBulkLoadFromSST(PromiseStream<RangeResult> resu
 						// The reply cannot be empty of the more is true
 						results.send(rep);
 					}
-					return Void();
+					co_return;
 				}
 			} else {
 				results.send(rep);
 			}
 			beginKey = keyAfter(rep.back().key);
-			wait(delay(0.1)); // context switch to avoid busy loop
+			co_await delay(0.1); // context switch to avoid busy loop
 		}
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
@@ -6663,14 +6705,14 @@ ACTOR Future<Void> tryGetRangeForBulkLoadFromSST(PromiseStream<RangeResult> resu
 	}
 }
 
-ACTOR Future<Void> tryGetRangeForBulkLoad(PromiseStream<RangeResult> results,
-                                          KeyRange keys /* only read data within the keys */,
-                                          std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) {
+Future<Void> tryGetRangeForBulkLoad(PromiseStream<RangeResult> results,
+                                    KeyRange keys /* only read data within the keys */,
+                                    std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) {
 	try {
 		// Build bulkLoadFileSetsToLoad
-		state std::vector<std::pair<KeyRange, BulkLoadFileSet>> bulkLoadFileSetsToLoad;
+		std::vector<std::pair<KeyRange, BulkLoadFileSet>> bulkLoadFileSetsToLoad;
 		KeyRangeMap<BulkLoadFileSet> localFileSetMap;
-		for (auto it = localFileSets->begin(); it < localFileSets->end(); it++) {
+		for (auto it = localFileSets->begin(); it < localFileSets->end(); ++it) {
 			localFileSetMap.insert(it->first, it->second);
 		}
 		for (auto range : localFileSetMap.intersectingRanges(keys)) {
@@ -6694,27 +6736,16 @@ ACTOR Future<Void> tryGetRangeForBulkLoad(PromiseStream<RangeResult> results,
 			emptyResult.more = false;
 			results.send(emptyResult);
 			results.sendError(end_of_stream());
-			return Void();
+			co_return;
 		}
 
-		// Streaming results given the input keys using bulkLoadFileSetsToLoad
-		state int i = 0;
-		if (bulkLoadFileSetsToLoad.empty()) {
-			// All ranges are empty - send empty result and signal completion
-			RangeResult emptyResult;
-			emptyResult.more = false;
-			results.send(emptyResult);
-			results.sendError(end_of_stream());
-			return Void();
-		}
-		for (; i < bulkLoadFileSetsToLoad.size(); i++) {
+		for (int i = 0; i < bulkLoadFileSetsToLoad.size(); ++i) {
 			std::string sstFilePath = bulkLoadFileSetsToLoad[i].second.getDataFileFullPath();
 			KeyRange rangeToLoad = bulkLoadFileSetsToLoad[i].first & keys;
 			ASSERT(!rangeToLoad.empty());
-			wait(tryGetRangeForBulkLoadFromSST(
-			    results, rangeToLoad, sstFilePath, i == bulkLoadFileSetsToLoad.size() - 1));
+			co_await tryGetRangeForBulkLoadFromSST(
+			    results, rangeToLoad, sstFilePath, i == bulkLoadFileSetsToLoad.size() - 1);
 		}
-		return Void();
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
 			throw;
@@ -6725,66 +6756,71 @@ ACTOR Future<Void> tryGetRangeForBulkLoad(PromiseStream<RangeResult> results,
 }
 
 // Utility function to process sample files during bulk load
-ACTOR static Future<Void> processSampleFiles(StorageServer* data,
-                                             KeyRange maxRange,
-                                             std::string bulkLoadLocalDir,
-                                             std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) {
-	state BulkLoadFileSetKeyMap::const_iterator iter = localFileSets->begin();
-	state BulkLoadFileSetKeyMap::const_iterator end = localFileSets->end();
-	state std::vector<KeyValue> rawSamples;
-	state std::unique_ptr<IRocksDBSstFileReader> reader;
+static Future<Void> processSampleFiles(StorageServer* data,
+                                       KeyRange maxRange,
+                                       std::string bulkLoadLocalDir,
+                                       std::shared_ptr<BulkLoadFileSetKeyMap> localFileSets) {
+	BulkLoadFileSetKeyMap::const_iterator iter = localFileSets->begin();
+	BulkLoadFileSetKeyMap::const_iterator end = localFileSets->end();
+	std::vector<KeyValue> rawSamples;
+	std::unique_ptr<IRocksDBSstFileReader> reader;
 
 	while (iter != end) {
 		const auto& [range, fileSet] = *iter;
 		if (fileSet.hasByteSampleFile()) {
-			state std::string sampleFilePath = fileSet.getBytesSampleFileFullPath();
-			state int retryCount = 0;
-			state int maxRetries = 10; // Consider making this a KNOB
-			state Error lastError;
+			std::string sampleFilePath = fileSet.getBytesSampleFileFullPath();
+			int retryCount = 0;
+			int maxRetries = 10; // Consider making this a KNOB
+			Error lastError;
 
 			// This outer loop retries reading the entire file if errors occur during opening/reading
 			while (retryCount < maxRetries) {
-				try {
-					// Read all samples from the SST file into memory first
-					// Store as KeyValueRef to keep the original encoded size value
-					rawSamples.clear();
-					reader = newRocksDBSstFileReader();
-					reader->open(abspath(sampleFilePath));
-
-					TraceEvent(SevInfo, "StorageServerProcessingSampleFile", data->thisServerID)
-					    .detail("File", sampleFilePath);
-
-					// Read all samples
-					while (reader->hasNext()) {
-						// Copy the next kv to rawSamples.
-						rawSamples.push_back(reader->next());
-					}
+				{
+					Error err;
+					try {
+						// Read all samples from the SST file into memory first
+						// Store as KeyValueRef to keep the original encoded size value
+						rawSamples.clear();
+						reader = newRocksDBSstFileReader();
+						reader->open(abspath(sampleFilePath));
+
+						TraceEvent(SevInfo, "StorageServerProcessingSampleFile", data->thisServerID)
+						    .detail("File", sampleFilePath);
+
+						// Read all samples
+						while (reader->hasNext()) {
+							// Copy the next kv to rawSamples.
+							rawSamples.push_back(reader->next());
+						}
 
-					// Now apply all read samples to the in-memory set and update metrics
-					for (const auto& kv : rawSamples) {
-						const KeyRef& key = kv.key;
-						if (!maxRange.contains(key)) {
-							// Skip keys outside the maxRange
-							continue;
+						// Now apply all read samples to the in-memory set and update metrics
+						for (const auto& kv : rawSamples) {
+							const KeyRef& key = kv.key;
+							if (!maxRange.contains(key)) {
+								// Skip keys outside the maxRange
+								continue;
+							}
+							int64_t size = BinaryReader::fromStringRef<int64_t>(kv.value, Unversioned());
+							data->metrics.byteSample.sample.insert(key, size);
+							data->metrics.notifyBytes(key, size);
+							data->addMutationToMutationLogOrStorage(
+							    invalidVersion,
+							    MutationRef(
+							        MutationRef::SetValue, key.withPrefix(persistByteSampleKeys.begin), kv.value));
 						}
-						int64_t size = BinaryReader::fromStringRef<int64_t>(kv.value, Unversioned());
-						data->metrics.byteSample.sample.insert(key, size);
-						data->metrics.notifyBytes(key, size);
-						data->addMutationToMutationLogOrStorage(
-						    invalidVersion,
-						    MutationRef(MutationRef::SetValue, key.withPrefix(persistByteSampleKeys.begin), kv.value));
-					}
 
-					// If we get here, processing was successful for this file
-					break; // Exit the retry loop
+						// If we get here, processing was successful for this file
+						break; // Exit the retry loop
 
-				} catch (Error& e) {
-					lastError = e;
+					} catch (Error& e) {
+						err = e;
+					}
+					lastError = err;
 					retryCount++;
 					TraceEvent(retryCount < maxRetries ? SevWarn : SevError,
 					           "StorageServerSampleFileProcessingError",
 					           data->thisServerID)
-					    .error(e) // Log the actual error 'e'
+					    .error(err) // Log the actual error 'e'
 					    .detail("File", sampleFilePath)
 					    // REMOVED: .detail("Key", kv.key).detail("Value", kv.value) as 'kv' is out of scope
 					    .detail("RetryCount", retryCount)
@@ -6794,7 +6830,7 @@ ACTOR static Future<Void> processSampleFiles(StorageServer* data,
 
 					if (retryCount < maxRetries) {
 						// Wait before retrying, with exponential backoff
-						wait(delay(0.1 * pow(2, retryCount))); // Consider adding jitter
+						co_await delay(0.1 * pow(2, retryCount)); // Consider adding jitter
 						continue; // Retry reading the file
 					}
 					// On final retry failure, throw the last error encountered
@@ -6804,37 +6840,36 @@ ACTOR static Future<Void> processSampleFiles(StorageServer* data,
 		}
 		++iter;
 	} // end file iteration loop
-	return Void();
 }
 
-ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
-	state const UID fetchKeysID = deterministicRandom()->randomUniqueID();
-	state TraceInterval interval("FetchKeys");
-	state KeyRange keys = shard->keys;
-	state Future<Void> warningLogger = logFetchKeysWarning(shard);
-	state const double startTime = now();
-	state Version fetchVersion = invalidVersion;
-	state int64_t totalBytes = 0;
-	state int priority = dataMovementPriority(shard->reason);
-	state UID dataMoveId = shard->getSSBulkLoadMetadata().getDataMoveId();
-	state ConductBulkLoad conductBulkLoad = ConductBulkLoad(shard->getSSBulkLoadMetadata().getConductBulkLoad());
-	state std::string bulkLoadLocalDir =
+Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
+	const UID fetchKeysID = deterministicRandom()->randomUniqueID();
+	TraceInterval interval("FetchKeys");
+	KeyRange keys = shard->keys;
+	Future<Void> warningLogger = logFetchKeysWarning(shard);
+	const double startTime = now();
+	Version fetchVersion = invalidVersion;
+	int64_t totalBytes = 0;
+	int priority = dataMovementPriority(shard->reason);
+	UID dataMoveId = shard->getSSBulkLoadMetadata().getDataMoveId();
+	ConductBulkLoad conductBulkLoad = ConductBulkLoad(shard->getSSBulkLoadMetadata().getConductBulkLoad());
+	std::string bulkLoadLocalDir =
 	    joinPath(joinPath(data->bulkLoadFolder, dataMoveId.toString()), fetchKeysID.toString());
-	state std::shared_ptr<BulkLoadFileSetKeyMap> localBulkLoadFileSets;
+	std::shared_ptr<BulkLoadFileSetKeyMap> localBulkLoadFileSets;
 	// Since the fetchKey can split, so multiple fetchzkeys can have the same data move id. We want each fetchkey
 	// downloads its file without conflict, so we add fetchKeysID to the bulkLoadLocalDir.
-	state PromiseStream<Key> destroyedFeeds;
-	state FetchKeysMetricReporter metricReporter(fetchKeysID,
-	                                             startTime,
-	                                             keys,
-	                                             data->fetchKeysHistograms,
-	                                             data->currentRunningFetchKeys,
-	                                             data->counters.bytesFetched,
-	                                             data->counters.kvFetched);
+	PromiseStream<Key> destroyedFeeds;
+	FetchKeysMetricReporter metricReporter(fetchKeysID,
+	                                       startTime,
+	                                       keys,
+	                                       data->fetchKeysHistograms,
+	                                       data->currentRunningFetchKeys,
+	                                       data->counters.bytesFetched,
+	                                       data->counters.kvFetched);
 
 	// Set read options to use non-caching reads and set Fetch type unless low priority data fetching is disabled by
 	// a knob
-	state ReadOptions readOptions = ReadOptions(
+	ReadOptions readOptions = ReadOptions(
 	    {}, SERVER_KNOBS->FETCH_KEYS_LOWER_PRIORITY ? ReadType::FETCH : ReadType::NORMAL, CacheResult::False);
 
 	if (conductBulkLoad) {
@@ -6856,7 +6891,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			    .detail("DataMoveId", dataMoveId)
 			    .detail("Elapsed", now() - startTime);
 		}
-		wait(data->coreStarted.getFuture() && delay(0));
+		co_await (data->coreStarted.getFuture() && delay(0));
 		if (conductBulkLoad) {
 			TraceEvent(SevDebug, "SSBulkLoadFetchKeysAfterCoreStarted", data->thisServerID)
 			    .detail("FKID", fetchKeysID)
@@ -6875,8 +6910,8 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				    .detail("CurrentVersion", data->version.get())
 				    .detail("Elapsed", now() - startTime);
 			}
-			wait(data->version.whenAtLeast(data->version.get() + 1));
-			wait(delay(0));
+			co_await data->version.whenAtLeast(data->version.get() + 1);
+			co_await delay(0);
 			if (conductBulkLoad) {
 				TraceEvent(SevDebug, "SSBulkLoadFetchKeysAfterVersionAdvance", data->thisServerID)
 				    .detail("FKID", fetchKeysID)
@@ -6924,7 +6959,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				    .detail("CurrentDurableVersion", data->durableVersion.get())
 				    .detail("Elapsed", now() - startTime);
 			}
-			wait(data->durableVersion.whenAtLeast(lastAvailable + 1));
+			co_await data->durableVersion.whenAtLeast(lastAvailable + 1);
 			if (conductBulkLoad) {
 				TraceEvent(SevDebug, "SSBulkLoadFetchKeysAfterDurableVersionWait", data->thisServerID)
 				    .detail("FKID", fetchKeysID)
@@ -6945,8 +6980,8 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			    .detail("DataMoveId", dataMoveId)
 			    .detail("Elapsed", now() - startTime);
 		}
-		wait(data->fetchKeysParallelismLock.take(TaskPriority::DefaultYield));
-		state FlowLock::Releaser holdingFKPL(data->fetchKeysParallelismLock);
+		co_await data->fetchKeysParallelismLock.take(TaskPriority::DefaultYield);
+		FlowLock::Releaser holdingFKPL(data->fetchKeysParallelismLock);
 		if (conductBulkLoad) {
 			TraceEvent(SevDebug, "SSBulkLoadFetchKeysAfterParallelismLock", data->thisServerID)
 			    .detail("FKID", fetchKeysID)
@@ -6954,7 +6989,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			    .detail("Elapsed", now() - startTime);
 		}
 
-		state double executeStart = now();
+		double executeStart = now();
 		++data->counters.fetchWaitingCount;
 		data->counters.fetchWaitingMS += 1000 * (executeStart - startTime);
 
@@ -6968,7 +7003,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			    .detail("DataMoveId", dataMoveId)
 			    .detail("Elapsed", now() - startTime);
 		}
-		wait(data->durableVersionLock.take());
+		co_await data->durableVersionLock.take();
 		if (conductBulkLoad) {
 			TraceEvent(SevDebug, "SSBulkLoadFetchKeysAfterDurableVersionLock", data->thisServerID)
 			    .detail("FKID", fetchKeysID)
@@ -6980,19 +7015,19 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 		data->durableVersionLock.release();
 
-		wait(delay(0));
+		co_await delay(0);
 
 		// Get the history
-		state int debug_getRangeRetries = 0;
-		state int debug_nextRetryToLog = 1;
-		state Error lastError;
+		int debug_getRangeRetries = 0;
+		int debug_nextRetryToLog = 1;
+		Error lastError;
 
 		// FIXME: The client cache does not notice when servers are added to a team. To read from a local storage
 		// server we must refresh the cache manually.
 		data->cx->invalidateCache(keys);
 
-		loop {
-			state Transaction tr(data->cx);
+		while (true) {
+			Transaction tr(data->cx);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			if (SERVER_KNOBS->ENABLE_REPLICA_CONSISTENCY_CHECK_ON_DATA_MOVEMENT) {
@@ -7025,7 +7060,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 			if (lastError.code() == error_code_transaction_too_old) {
 				try {
-					Version grvVersion = wait(tr.getRawReadVersion());
+					Version grvVersion = co_await tr.getRawReadVersion();
 					if (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01)) {
 						// Test failed GRV request.
 						throw grv_proxy_memory_limit_exceeded();
@@ -7053,9 +7088,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				shard->updates.pop_front();
 			tr.setVersion(fetchVersion);
 
-			state PromiseStream<RangeResult> results;
-			state Future<Void> hold;
-			state KeyRef rangeEnd;
+			PromiseStream<RangeResult> results;
+			Future<Void> hold;
+			KeyRef rangeEnd;
 			if (conductBulkLoad) {
 				ASSERT(dataMoveIdIsValidForBulkLoad(dataMoveId)); // TODO(BulkLoad): remove dangerous assert
 				// Get the bulkload task metadata from the data move metadata. Note that a SS can receive a data move
@@ -7067,8 +7102,8 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				// Note that it is possible that this SS can never get the bulkload metadata because the bulkload data
 				// move is cancelled or replaced by another data move. In this case, while
 				// getBulkLoadTaskStateFromDataMove get stuck, this fetchKeys is guaranteed to be cancelled.
-				BulkLoadTaskState bulkLoadTaskState = wait(getBulkLoadTaskStateFromDataMove(
-				    data->cx, dataMoveId, /*atLeastVersion=*/data->version.get(), data->thisServerID));
+				BulkLoadTaskState bulkLoadTaskState = co_await getBulkLoadTaskStateFromDataMove(
+				    data->cx, dataMoveId, /*atLeastVersion=*/data->version.get(), data->thisServerID);
 				TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchKey", data->thisServerID)
 				    .detail("DataMoveId", dataMoveId.toString())
 				    .detail("Range", keys)
@@ -7080,9 +7115,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				// We download the data file to local disk and pass the data file path to read in the next step.
 				localBulkLoadFileSets = std::make_shared<BulkLoadFileSetKeyMap>();
 				// A bulkload task can do file ingestion if the task range is aligned with manifests' range.
-				state bool bulkloadCanIngestSSTFile = bulkLoadTaskState.canIngestFile();
-				wait(bulkLoadFetchKeyValueFileToLoad(
-				    data, bulkLoadLocalDir, bulkLoadTaskState, /*output=*/localBulkLoadFileSets));
+				bool bulkloadCanIngestSSTFile = bulkLoadTaskState.canIngestFile();
+				co_await bulkLoadFetchKeyValueFileToLoad(
+				    data, bulkLoadLocalDir, bulkLoadTaskState, /*output=*/localBulkLoadFileSets);
 				TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchKey", data->thisServerID)
 				    .detail("DataMoveId", dataMoveId.toString())
 				    .detail("Range", keys)
@@ -7118,23 +7153,23 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 					data->storage.getKeyValueStore()->clear(keys);
 
 					// Now wait on the durableVersion to be updated so clear has been committed.
-					wait(data->durableVersion.whenAtLeast(data->storageVersion() + 1));
+					co_await data->durableVersion.whenAtLeast(data->storageVersion() + 1);
 
 					// Compact the range before ingestion to optimize storage
-					wait(data->storage.getKeyValueStore()->compactRange(keys));
+					co_await data->storage.getKeyValueStore()->compactRange(keys);
 
 					// Ingest the SST files.
 					// Measure duration at this level so we capture the inter-thread handoff time.
-					state double ingestStartTime = g_network->timer(); // Record start time
-					wait(data->storage.getKeyValueStore()->ingestSSTFiles(localBulkLoadFileSets));
+					double ingestStartTime = g_network->timer(); // Record start time
+					co_await data->storage.getKeyValueStore()->ingestSSTFiles(localBulkLoadFileSets);
 					const double ingestDuration = g_network->timer() - ingestStartTime;
 					data->counters.ingestDurationLatencySample->addMeasurement(ingestDuration);
 
 					// Compact the range after ingestion to avoid accumulating compaction overtime
-					wait(data->storage.getKeyValueStore()->compactRange(keys));
+					co_await data->storage.getKeyValueStore()->compactRange(keys);
 
 					// Process sample files after SST ingestion
-					wait(processSampleFiles(data, keys, bulkLoadLocalDir, localBulkLoadFileSets));
+					co_await processSampleFiles(data, keys, bulkLoadLocalDir, localBulkLoadFileSets);
 
 					// NOTICE: We break the 'fetchKeys' loop here if we successfully ingest the SST files.
 					// EARLY EXIT FROM 'fetchKeys' LOOP!!!
@@ -7161,112 +7196,116 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 				rangeEnd = keys.end;
 			}
 
-			state Key blockBegin = keys.begin;
+			Key blockBegin = keys.begin;
 
-			try {
-				loop {
-					CODE_PROBE(true, "Fetching keys for transferred shard");
-					while (data->fetchKeysBudgetUsed.get()) {
-						std::vector<Future<Void>> delays;
-						if (SERVER_KNOBS->STORAGE_FETCH_KEYS_DELAY > 0) {
-							delays.push_back(delayJittered(SERVER_KNOBS->STORAGE_FETCH_KEYS_DELAY));
-						}
-						delays.push_back(data->fetchKeysBudgetUsed.onChange());
-						wait(waitForAll(delays));
-					}
-					state RangeResult this_block = waitNext(results.getFuture());
-
-					state int expectedBlockSize =
-					    (int)this_block.expectedSize() + (8 - (int)sizeof(KeyValueRef)) * this_block.size();
-
-					TraceEvent(SevDebug, "FetchKeysBlock", data->thisServerID)
-					    .detail("FKID", interval.pairID)
-					    .detail("BlockRows", this_block.size())
-					    .detail("BlockBytes", expectedBlockSize)
-					    .detail("KeyBegin", keys.begin)
-					    .detail("KeyEnd", keys.end)
-					    .detail("Last", this_block.size() ? this_block.end()[-1].key : std::string())
-					    .detail("Version", fetchVersion)
-					    .detail("More", this_block.more)
-					    .detail("DataMoveId", dataMoveId.toString())
-					    .detail("ConductBulkLoad", conductBulkLoad);
-
-					DEBUG_KEY_RANGE("fetchRange", fetchVersion, keys, data->thisServerID);
-					if (MUTATION_TRACKING_ENABLED) {
-						for (auto k = this_block.begin(); k != this_block.end(); ++k) {
-							DEBUG_MUTATION("fetch",
-							               fetchVersion,
-							               MutationRef(MutationRef::SetValue, k->key, k->value),
-							               data->thisServerID);
+			{
+				Error err;
+				try {
+					while (true) {
+						CODE_PROBE(true, "Fetching keys for transferred shard");
+						while (data->fetchKeysBudgetUsed.get()) {
+							std::vector<Future<Void>> delays;
+							if (SERVER_KNOBS->STORAGE_FETCH_KEYS_DELAY > 0) {
+								delays.push_back(delayJittered(SERVER_KNOBS->STORAGE_FETCH_KEYS_DELAY));
+							}
+							delays.push_back(data->fetchKeysBudgetUsed.onChange());
+							co_await waitForAll(delays);
 						}
-					}
-					if (!conductBulkLoad) {
-						metricReporter.addFetchedBytes(expectedBlockSize, this_block.size());
-					}
-					totalBytes += expectedBlockSize;
-
-					if (shard->reason != DataMovementReason::INVALID &&
-					    priority < SERVER_KNOBS->FETCH_KEYS_THROTTLE_PRIORITY_THRESHOLD &&
-					    !data->fetchKeysLimiter.ready().isReady()) {
-						TraceEvent(SevDebug, "FetchKeysThrottling", data->thisServerID).detail("FKID", fetchKeysID);
-						state double ts = now();
-						wait(data->fetchKeysLimiter.ready());
-						TraceEvent(SevDebug, "FetchKeysThrottled", data->thisServerID)
-						    .detail("Priority", priority)
-						    .detail("KeyRange", shard->keys)
-						    .detail("Delay", now() - ts)
-						    .detail("FKID", fetchKeysID);
-					}
+						RangeResult this_block = co_await results.getFuture();
 
-					// Write this_block to storage
-					state Standalone<VectorRef<KeyValueRef>> blockData(this_block, this_block.arena());
-					state Key blockEnd =
-					    this_block.size() > 0 && this_block.more ? keyAfter(this_block.back().key) : keys.end;
-					state KeyRange blockRange(KeyRangeRef(blockBegin, blockEnd));
-					wait(data->storage.replaceRange(blockRange, blockData));
+						int expectedBlockSize =
+						    (int)this_block.expectedSize() + (8 - (int)sizeof(KeyValueRef)) * this_block.size();
 
-					if (conductBulkLoad) {
-						TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchKey", data->thisServerID)
+						TraceEvent(SevDebug, "FetchKeysBlock", data->thisServerID)
 						    .detail("FKID", interval.pairID)
+						    .detail("BlockRows", this_block.size())
+						    .detail("BlockBytes", expectedBlockSize)
+						    .detail("KeyBegin", keys.begin)
+						    .detail("KeyEnd", keys.end)
+						    .detail("Last", this_block.size() ? this_block.end()[-1].key : std::string())
+						    .detail("Version", fetchVersion)
+						    .detail("More", this_block.more)
 						    .detail("DataMoveId", dataMoveId.toString())
-						    .detail("Range", keys)
-						    .detail("BlockRange", blockRange)
-						    .detail("Phase", "Replaced range");
-					}
+						    .detail("ConductBulkLoad", conductBulkLoad);
+
+						DEBUG_KEY_RANGE("fetchRange", fetchVersion, keys, data->thisServerID);
+						if (MUTATION_TRACKING_ENABLED) {
+							for (auto k = this_block.begin(); k != this_block.end(); ++k) {
+								DEBUG_MUTATION("fetch",
+								               fetchVersion,
+								               MutationRef(MutationRef::SetValue, k->key, k->value),
+								               data->thisServerID);
+							}
+						}
+						if (!conductBulkLoad) {
+							metricReporter.addFetchedBytes(expectedBlockSize, this_block.size());
+						}
+						totalBytes += expectedBlockSize;
+
+						if (shard->reason != DataMovementReason::INVALID &&
+						    priority < SERVER_KNOBS->FETCH_KEYS_THROTTLE_PRIORITY_THRESHOLD &&
+						    !data->fetchKeysLimiter.ready().isReady()) {
+							TraceEvent(SevDebug, "FetchKeysThrottling", data->thisServerID).detail("FKID", fetchKeysID);
+							double ts = now();
+							co_await data->fetchKeysLimiter.ready();
+							TraceEvent(SevDebug, "FetchKeysThrottled", data->thisServerID)
+							    .detail("Priority", priority)
+							    .detail("KeyRange", shard->keys)
+							    .detail("Delay", now() - ts)
+							    .detail("FKID", fetchKeysID);
+						}
 
-					data->fetchKeysLimiter.addBytes(expectedBlockSize);
+						// Write this_block to storage
+						Standalone<VectorRef<KeyValueRef>> blockData(this_block, this_block.arena());
+						Key blockEnd =
+						    this_block.size() > 0 && this_block.more ? keyAfter(this_block.back().key) : keys.end;
+						KeyRange blockRange(KeyRangeRef(blockBegin, blockEnd));
+						co_await data->storage.replaceRange(blockRange, blockData);
 
-					state KeyValueRef* kvItr = this_block.begin();
-					for (; kvItr != this_block.end(); ++kvItr) {
-						data->byteSampleApplySet(*kvItr, invalidVersion);
-					}
+						if (conductBulkLoad) {
+							TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchKey", data->thisServerID)
+							    .detail("FKID", interval.pairID)
+							    .detail("DataMoveId", dataMoveId.toString())
+							    .detail("Range", keys)
+							    .detail("BlockRange", blockRange)
+							    .detail("Phase", "Replaced range");
+						}
 
-					if (this_block.more) {
-						blockBegin = this_block.getReadThrough();
-					} else {
-						ASSERT(!this_block.readThrough.present());
-						blockBegin = rangeEnd;
-					}
-					this_block = RangeResult();
+						data->fetchKeysLimiter.addBytes(expectedBlockSize);
+
+						KeyValueRef* kvItr = this_block.begin();
+						for (; kvItr != this_block.end(); ++kvItr) {
+							data->byteSampleApplySet(*kvItr, invalidVersion);
+						}
 
-					++data->counters.kvClearRangesInFetchKeys;
-					data->fetchKeysTotalCommitBytes += expectedBlockSize;
-					data->fetchKeysBytesBudget -= expectedBlockSize;
-					data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
+						if (this_block.more) {
+							blockBegin = this_block.getReadThrough();
+						} else {
+							ASSERT(!this_block.readThrough.present());
+							blockBegin = rangeEnd;
+						}
+						this_block = RangeResult();
+
+						++data->counters.kvClearRangesInFetchKeys;
+						data->fetchKeysTotalCommitBytes += expectedBlockSize;
+						data->fetchKeysBytesBudget -= expectedBlockSize;
+						data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
+					}
+				} catch (Error& e) {
+					err = e;
 				}
-			} catch (Error& e) {
-				if (!fetchKeyCanRetry(e)) {
-					throw e;
+				if (!fetchKeyCanRetry(err)) {
+					throw err;
 				}
 				if (!conductBulkLoad) {
 					data->counters.fetchKeyErrors += 1;
 				}
-				lastError = e;
+				lastError = err;
 				if (lastError.code() == error_code_storage_replica_comparison_error) {
 					// The inconsistency could be because of the inclusion of a rolled back
 					// transaction(s)/version(s) in the returned results. Retry.
 
-					wait(data->knownCommittedVersion.whenAtLeast(fetchVersion));
+					co_await data->knownCommittedVersion.whenAtLeast(fetchVersion);
 				}
 				if (blockBegin == keys.begin) {
 					TraceEvent("FKBlockFail", data->thisServerID)
@@ -7283,7 +7322,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 						    .detail("N", fetchVersion)
 						    .detail("E", data->version.get());
 					}
-					wait(delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					co_await delayJittered(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 					continue;
 				}
 				if (blockBegin < keys.end) {
@@ -7322,8 +7361,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 					// in WaitPrevious phase (hasn't chosen a fetchVersion yet). What we are doing here is expensive
 					// and could get more expensive if we started having many more blocks per shard. May need
 					// optimization in the future.
-					std::deque<Standalone<VerUpdateRef>>::iterator u = updatesToSplit.begin();
-					for (; u != updatesToSplit.end(); ++u) {
+					for (auto u = updatesToSplit.begin(); u != updatesToSplit.end(); ++u) {
 						splitMutations(data, data->shards, *u);
 					}
 
@@ -7346,9 +7384,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			// This block is for the fetchKey without SST ingestion case.
 			// For the SST ingestion case, we have already compacted the range right after ingestion.
 			// Wait until the load data has been committed.
-			wait(data->durableVersion.whenAtLeast(data->storageVersion() + 1));
+			co_await data->durableVersion.whenAtLeast(data->storageVersion() + 1);
 			// Compact the range after ingestion to avoid accumulating compaction overtime.
-			wait(data->storage.getKeyValueStore()->compactRange(keys));
+			co_await data->storage.getKeyValueStore()->compactRange(keys);
 		}
 
 		// We have completed the fetch and write of the data, now we wait for MVCC window to pass.
@@ -7372,11 +7410,11 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		// version being recovered. Instead we wait for the updateStorage loop to commit something (and consequently
 		// also what we have written)
 
-		state Future<Void> fetchDurable = data->durableVersion.whenAtLeast(data->storageVersion() + 1);
-		state Future<Void> dataArrive = data->version.whenAtLeast(fetchVersion);
+		Future<Void> fetchDurable = data->durableVersion.whenAtLeast(data->storageVersion() + 1);
+		Future<Void> dataArrive = data->version.whenAtLeast(fetchVersion);
 
 		holdingFKPL.release();
-		wait(dataArrive && fetchDurable);
+		co_await (dataArrive && fetchDurable);
 
 		TraceEvent(SevDebug, "FKAfterFinalCommit", data->thisServerID)
 		    .detail("FKID", interval.pairID)
@@ -7390,7 +7428,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 		// After we add to the promise readyFetchKeys, update() would provide a pointer to FetchInjectionInfo that
 		// we can put mutation in.
-		FetchInjectionInfo* batch = wait(p.getFuture());
+		FetchInjectionInfo* batch = co_await p.getFuture();
 		TraceEvent(SevDebug, "FKUpdateBatch", data->thisServerID).detail("FKID", interval.pairID);
 
 		// TOOD(gglass): eliminate the need for the FetchingCF phase here.
@@ -7456,7 +7494,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		    .detail("Version", feedTransferredVersion)
 		    .detail("StorageVersion", data->storageVersion());
 
-		state StorageServerShard newShard;
+		StorageServerShard newShard;
 		if (data->shardAware) {
 			newShard = data->shards[keys.begin]->toStorageServerShard();
 			ASSERT(newShard.range == keys);
@@ -7472,7 +7510,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		// this point.
 
 		// Wait for the transferred version (and therefore the shard data) to be committed and durable.
-		wait(data->durableVersion.whenAtLeast(feedTransferredVersion));
+		co_await data->durableVersion.whenAtLeast(feedTransferredVersion);
 
 		ASSERT(data->shards[shard->keys.begin]->assigned() &&
 		       data->shards[shard->keys.begin]->range() ==
@@ -7536,8 +7574,6 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		}
 		throw; // goes nowhere
 	}
-
-	return Void();
 }
 
 AddingShard::AddingShard(StorageServer* server,
@@ -7600,14 +7636,14 @@ void changeServerKeysWithPhysicalShards(StorageServer* data,
                                         EnablePhysicalShardMove enablePSM,
                                         ConductBulkLoad conductBulkLoad);
 
-ACTOR Future<Void> fallBackToAddingShard(StorageServer* data, MoveInShard* moveInShard) {
+Future<Void> fallBackToAddingShard(StorageServer* data, MoveInShard* moveInShard) {
 	if (moveInShard->getPhase() != MoveInPhase::Fetching && moveInShard->getPhase() != MoveInPhase::Ingesting) {
 		TraceEvent(SevError, "FallBackToAddingShardError", data->thisServerID)
 		    .detail("MoveInShard", moveInShard->meta->toString());
 		throw internal_error();
 	}
 	if (moveInShard->failed()) {
-		return Void();
+		co_return;
 	}
 	auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
 	TraceEvent(SevInfo, "FallBackToAddingShardBegin", data->thisServerID)
@@ -7633,15 +7669,13 @@ ACTOR Future<Void> fallBackToAddingShard(StorageServer* data, MoveInShard* moveI
 		}
 	}
 
-	wait(data->durableVersion.whenAtLeast(mLV.version + 1));
-
-	return Void();
+	co_await data->durableVersion.whenAtLeast(mLV.version + 1);
 }
 
-ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
-                                                MoveInShard* moveInShard,
-                                                std::string localRoot,
-                                                BulkLoadTaskState bulkLoadTaskState) {
+Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
+                                          MoveInShard* moveInShard,
+                                          std::string localRoot,
+                                          BulkLoadTaskState bulkLoadTaskState) {
 	ASSERT(bulkLoadTaskState.getLoadType() == BulkLoadType::SST);
 	TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchShardFile", data->thisServerID)
 	    .detail("JobID", bulkLoadTaskState.getJobId().toString())
@@ -7649,13 +7683,13 @@ ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
 	    .detail("MoveInShard", moveInShard->toString())
 	    .detail("LocalRoot", abspath(localRoot));
 
-	state double fetchStartTime = now();
+	double fetchStartTime = now();
 
 	// Step 1: Download files to localRoot
 	// TODO(BulkLoad): support bulkload task mutiple sst for sharded rocksdb.
 	ASSERT(SERVER_KNOBS->MANIFEST_COUNT_MAX_PER_BULKLOAD_TASK == 1);
 	ASSERT(bulkLoadTaskState.getManifests().size() == 1);
-	state BulkLoadFileSet fromRemoteFileSet = bulkLoadTaskState.getManifests()[0].getFileSet();
+	BulkLoadFileSet fromRemoteFileSet = bulkLoadTaskState.getManifests()[0].getFileSet();
 	BulkLoadByteSampleSetting currentClusterByteSampleSetting(
 	    0,
 	    "hashlittle2", // use function name to represent the method
@@ -7669,8 +7703,8 @@ ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
 		fromRemoteFileSet.removeByteSampleFile();
 	}
 	// Download data file and byte sample file from fromRemoteFileSet to toLocalFileSet
-	state BulkLoadFileSet toLocalFileSet = wait(bulkLoadDownloadTaskFileSet(
-	    bulkLoadTaskState.getTransportMethod(), fromRemoteFileSet, localRoot, data->thisServerID));
+	BulkLoadFileSet toLocalFileSet = co_await bulkLoadDownloadTaskFileSet(
+	    bulkLoadTaskState.getTransportMethod(), fromRemoteFileSet, localRoot, data->thisServerID);
 	TraceEvent(bulkLoadVerboseEventSev(), "SSBulkLoadTaskFetchShardSSTFileFetched", data->thisServerID)
 	    .detail("JobID", bulkLoadTaskState.getJobId().toString())
 	    .detail("TaskID", bulkLoadTaskState.getTaskId().toString())
@@ -7688,7 +7722,7 @@ ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
 		    .detail("Reason", "No data to ingest for empty range");
 		// For empty ranges, directly move to Ingesting phase without adding any checkpoint
 		moveInShard->setPhase(MoveInPhase::Ingesting);
-		return Void();
+		co_return;
 	}
 
 	// Step 2: Do byte sampling locally if the remote byte sampling file is not valid nor existing
@@ -7698,11 +7732,11 @@ ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
 		    .detail("JobID", bulkLoadTaskState.getJobId().toString())
 		    .detail("TaskID", bulkLoadTaskState.getTaskId().toString())
 		    .detail("LocalFileSet", toLocalFileSet.toString());
-		state std::string byteSampleFileName =
+		std::string byteSampleFileName =
 		    generateBulkLoadBytesSampleFileNameFromDataFileName(toLocalFileSet.getDataFileName());
 		std::string byteSampleFilePathLocal = abspath(joinPath(toLocalFileSet.getFolder(), byteSampleFileName));
-		bool bytesSampleFileGenerated = wait(doBytesSamplingOnDataFile(
-		    toLocalFileSet.getDataFileFullPath(), byteSampleFilePathLocal, data->thisServerID));
+		bool bytesSampleFileGenerated = co_await doBytesSamplingOnDataFile(
+		    toLocalFileSet.getDataFileFullPath(), byteSampleFilePathLocal, data->thisServerID);
 		if (bytesSampleFileGenerated) {
 			toLocalFileSet.setByteSampleFileName(byteSampleFileName);
 		}
@@ -7715,7 +7749,7 @@ ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
 	    .detail("LocalFileSet", toLocalFileSet.toString());
 
 	// Step 3: Build LocalRecord used by ShardedRocksDB KVStore when injecting data
-	state CheckpointMetaData localRecord;
+	CheckpointMetaData localRecord;
 	localRecord.checkpointID = UID();
 	localRecord.dir = abspath(toLocalFileSet.getFolder());
 	for (const auto& range : moveInShard->ranges()) {
@@ -7758,80 +7792,81 @@ ACTOR Future<Void> bulkLoadFetchShardFileToLoad(StorageServer* data,
 
 	// Step 4: Update the moveInShard phase
 	moveInShard->setPhase(MoveInPhase::Ingesting);
-	return Void();
 }
 
-ACTOR Future<Void> fetchShardCheckpoint(StorageServer* data, MoveInShard* moveInShard, std::string dir) {
+Future<Void> fetchShardCheckpoint(StorageServer* data, MoveInShard* moveInShard, std::string dir) {
 	TraceEvent(SevInfo, "FetchShardCheckpointMetaDataBegin", data->thisServerID)
 	    .detail("MoveInShard", moveInShard->toString());
 	ASSERT(moveInShard->getPhase() == MoveInPhase::Fetching);
 
-	state std::vector<std::pair<KeyRange, CheckpointMetaData>> records;
-	state std::vector<CheckpointMetaData> localRecords;
-	state int attempt = 0; // TODO(heliu): use shard->meta->checkpoints to continue the fetch.
-	state double fetchStartTime = now();
+	std::vector<std::pair<KeyRange, CheckpointMetaData>> records;
+	std::vector<CheckpointMetaData> localRecords;
+	int attempt = 0; // TODO(heliu): use shard->meta->checkpoints to continue the fetch.
+	double fetchStartTime = now();
 
-	loop {
-		wait(delay(0, TaskPriority::FetchKeys));
+	while (true) {
+		co_await delay(0, TaskPriority::FetchKeys);
 		if (moveInShard->failed()) {
-			return Void();
+			co_return;
 		}
 		++attempt;
 		records.clear();
 		for (const auto& range : moveInShard->ranges()) {
 			data->cx->invalidateCache(range);
 		}
-		try {
-			wait(store(records,
-			           getCheckpointMetaData(data->cx,
-			                                 moveInShard->ranges(),
-			                                 moveInShard->meta->createVersion,
-			                                 DataMoveRocksCF,
-			                                 moveInShard->dataMoveId())));
-			if (moveInShard->failed()) {
-				return Void();
-			}
+		{
+			Error err;
+			try {
+				records = co_await getCheckpointMetaData(data->cx,
+				                                         moveInShard->ranges(),
+				                                         moveInShard->meta->createVersion,
+				                                         DataMoveRocksCF,
+				                                         moveInShard->dataMoveId());
+				if (moveInShard->failed()) {
+					co_return;
+				}
 
-			if (g_network->isSimulated()) {
-				for (const auto& [range, record] : records) {
-					TraceEvent(moveInShard->logSev, "FetchShardCheckpointMetaData", data->thisServerID)
-					    .detail("MoveInShardID", moveInShard->id())
-					    .detail("Range", range)
-					    .detail("CheckpointMetaData", record.toString());
+				if (g_network->isSimulated()) {
+					for (const auto& [range, record] : records) {
+						TraceEvent(moveInShard->logSev, "FetchShardCheckpointMetaData", data->thisServerID)
+						    .detail("MoveInShardID", moveInShard->id())
+						    .detail("Range", range)
+						    .detail("CheckpointMetaData", record.toString());
+					}
 				}
-			}
 
-			platform::eraseDirectoryRecursive(dir);
-			ASSERT(platform::createDirectory(dir));
+				platform::eraseDirectoryRecursive(dir);
+				ASSERT(platform::createDirectory(dir));
 
-			TraceEvent(SevInfo, "FetchShardFetchCheckpointsBegin", data->thisServerID)
-			    .detail("MoveInShardID", moveInShard->id());
-			std::vector<Future<CheckpointMetaData>> fFetchCheckpoint;
-			for (const auto& [range, record] : records) {
-				const std::string checkpointDir = joinPath(dir, deterministicRandom()->randomAlphaNumeric(8));
-				if (!platform::createDirectory(checkpointDir)) {
-					throw retry();
+				TraceEvent(SevInfo, "FetchShardFetchCheckpointsBegin", data->thisServerID)
+				    .detail("MoveInShardID", moveInShard->id());
+				std::vector<Future<CheckpointMetaData>> fFetchCheckpoint;
+				for (const auto& [range, record] : records) {
+					const std::string checkpointDir = joinPath(dir, deterministicRandom()->randomAlphaNumeric(8));
+					if (!platform::createDirectory(checkpointDir)) {
+						throw retry();
+					}
+					fFetchCheckpoint.push_back(fetchCheckpointRanges(data->cx, record, checkpointDir, { range }));
 				}
-				fFetchCheckpoint.push_back(fetchCheckpointRanges(data->cx, record, checkpointDir, { range }));
-			}
-			wait(store(localRecords, getAll(fFetchCheckpoint)));
-			if (moveInShard->failed()) {
-				return Void();
+				localRecords = co_await getAll(fFetchCheckpoint);
+				if (moveInShard->failed()) {
+					co_return;
+				}
+				break;
+			} catch (Error& e) {
+				err = e;
 			}
-			break;
-		} catch (Error& err) {
 			TraceEvent(SevWarn, "FetchShardCheckpointsError", data->thisServerID)
 			    .errorUnsuppressed(err)
 			    .detail("Attempt", attempt)
 			    .detail("MoveInShard", moveInShard->toString());
-			state Error e = err;
-			if (e.code() == error_code_actor_cancelled || moveInShard->getPhase() != MoveInPhase::Fetching) {
-				throw e;
-			} else if (attempt > 10 || e.code() == error_code_checkpoint_not_found) {
-				wait(fallBackToAddingShard(data, moveInShard));
-				return Void();
+			if (err.code() == error_code_actor_cancelled || moveInShard->getPhase() != MoveInPhase::Fetching) {
+				throw err;
+			} else if (attempt > 10 || err.code() == error_code_checkpoint_not_found) {
+				co_await fallBackToAddingShard(data, moveInShard);
+				co_return;
 			} else {
-				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::FetchKeys));
+				co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::FetchKeys);
 			}
 		}
 	}
@@ -7851,20 +7886,19 @@ ACTOR Future<Void> fetchShardCheckpoint(StorageServer* data, MoveInShard* moveIn
 	updateMoveInShardMetaData(data, moveInShard);
 
 	TraceEvent(SevInfo, "FetchShardCheckpointsEnd", data->thisServerID).detail("MoveInShard", moveInShard->toString());
-	return Void();
 }
 
-ACTOR Future<Void> fetchShardIngestCheckpoint(StorageServer* data, MoveInShard* moveInShard) {
+Future<Void> fetchShardIngestCheckpoint(StorageServer* data, MoveInShard* moveInShard) {
 	TraceEvent(SevInfo, "FetchShardIngestCheckpointBegin", data->thisServerID)
 	    .detail("Checkpoints", describe(moveInShard->checkpoints()));
 	ASSERT(moveInShard->getPhase() == MoveInPhase::Ingesting);
-	state double startTime = now();
+	double startTime = now();
 
 	try {
-		wait(
-		    data->storage.restore(moveInShard->destShardIdString(), moveInShard->ranges(), moveInShard->checkpoints()));
+		co_await data->storage.restore(
+		    moveInShard->destShardIdString(), moveInShard->ranges(), moveInShard->checkpoints());
 	} catch (Error& e) {
-		state Error err = e;
+		Error err = e;
 		TraceEvent(SevWarn, "FetchShardIngestedCheckpointError", data->thisServerID)
 		    .errorUnsuppressed(e)
 		    .detail("MoveInShard", moveInShard->toString())
@@ -7872,7 +7906,7 @@ ACTOR Future<Void> fetchShardIngestCheckpoint(StorageServer* data, MoveInShard*
 		if (e.code() == error_code_failed_to_restore_checkpoint && !moveInShard->failed()) {
 			moveInShard->setPhase(MoveInPhase::Fetching);
 			updateMoveInShardMetaData(data, moveInShard);
-			return Void();
+			co_return;
 		}
 		throw err;
 	}
@@ -7882,7 +7916,7 @@ ACTOR Future<Void> fetchShardIngestCheckpoint(StorageServer* data, MoveInShard*
 	    .detail("Checkpoints", describe(moveInShard->checkpoints()));
 
 	if (moveInShard->failed()) {
-		return Void();
+		co_return;
 	}
 
 	for (const auto& range : moveInShard->ranges()) {
@@ -7932,32 +7966,30 @@ ACTOR Future<Void> fetchShardIngestCheckpoint(StorageServer* data, MoveInShard*
 	    .detail("Bytes", totalBytes)
 	    .detail("Duration", duration)
 	    .detail("Rate", static_cast<double>(totalBytes) / duration);
-
-	return Void();
 }
 
-ACTOR Future<Void> fetchShardApplyUpdates(StorageServer* data,
-                                          MoveInShard* moveInShard,
-                                          std::shared_ptr<MoveInUpdates> moveInUpdates) {
+Future<Void> fetchShardApplyUpdates(StorageServer* data,
+                                    MoveInShard* moveInShard,
+                                    std::shared_ptr<MoveInUpdates> moveInUpdates) {
 	TraceEvent(SevInfo, "FetchShardApplyUpdatesBegin", data->thisServerID)
 	    .detail("MoveInShard", moveInShard->toString());
 	ASSERT(moveInShard->getPhase() == MoveInPhase::ApplyingUpdates);
-	state double startTime = now();
+	double startTime = now();
 
 	try {
 		if (moveInShard->failed()) {
-			return Void();
+			co_return;
 		}
 
-		loop {
+		while (true) {
 			Promise<FetchInjectionInfo*> p;
 			data->readyFetchKeys.push_back(p);
-			FetchInjectionInfo* batch = wait(p.getFuture());
+			FetchInjectionInfo* batch = co_await p.getFuture();
 			if (moveInShard->failed()) {
-				return Void();
+				co_return;
 			}
 
-			state Version version = data->version.get() + 1;
+			Version version = data->version.get() + 1;
 			data->mutableData().createNewVersion(version);
 			ASSERT(version == data->data().getLatestVersion());
 
@@ -8020,9 +8052,9 @@ ACTOR Future<Void> fetchShardApplyUpdates(StorageServer* data,
 				}
 
 				// Wait for the transferredVersion (and therefore the shard data) to be committed and durable.
-				wait(data->durableVersion.whenAtLeast(mLV.version + 1));
+				co_await data->durableVersion.whenAtLeast(mLV.version + 1);
 				if (moveInShard->failed()) {
-					return Void();
+					co_return;
 				}
 				break;
 			} else {
@@ -8033,7 +8065,7 @@ ACTOR Future<Void> fetchShardApplyUpdates(StorageServer* data,
 				TraceEvent(moveInShard->logSev, "MultipleApplyUpdates").detail("MoveInShard", moveInShard->toString());
 			}
 
-			wait(data->version.whenAtLeast(version + 1));
+			co_await data->version.whenAtLeast(version + 1);
 		}
 
 		moveInShard->setPhase(MoveInPhase::Complete);
@@ -8080,15 +8112,13 @@ ACTOR Future<Void> fetchShardApplyUpdates(StorageServer* data,
 		    .detail("MoveInShard", moveInShard->toString());
 		throw e;
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> cleanUpMoveInShard(StorageServer* data, Version version, MoveInShard* moveInShard) {
+Future<Void> cleanUpMoveInShard(StorageServer* data, Version version, MoveInShard* moveInShard) {
 	TraceEvent(moveInShard->logSev, "CleanUpMoveInShardBegin", data->thisServerID)
 	    .detail("MoveInShard", moveInShard->meta->toString())
 	    .detail("Version", version);
-	wait(data->durableVersion.whenAtLeast(version));
+	co_await data->durableVersion.whenAtLeast(version);
 
 	platform::eraseDirectoryRecursive(fetchedCheckpointDir(data->folder, moveInShard->id()));
 
@@ -8097,7 +8127,7 @@ ACTOR Future<Void> cleanUpMoveInShard(StorageServer* data, Version version, Move
 	data->addMutationToMutationLog(
 	    mLV, MutationRef(MutationRef::ClearRange, persistUpdatesRange.begin, persistUpdatesRange.end));
 
-	state bool clearRecord = true;
+	bool clearRecord = true;
 	if (moveInShard->failed()) {
 		for (const auto& mir : moveInShard->ranges()) {
 			auto existingShards = data->shards.intersectingRanges(mir);
@@ -8113,13 +8143,11 @@ ACTOR Future<Void> cleanUpMoveInShard(StorageServer* data, Version version, Move
 		const Key persistKey = persistMoveInShardKey(moveInShard->id());
 		data->addMutationToMutationLog(mLV, MutationRef(MutationRef::ClearRange, persistKey, keyAfter(persistKey)));
 	}
-	wait(data->durableVersion.whenAtLeast(mLV.version + 1));
+	co_await data->durableVersion.whenAtLeast(mLV.version + 1);
 
 	if (clearRecord) {
 		data->moveInShards.erase(moveInShard->id());
 	}
-
-	return Void();
 }
 
 // It works in the following sequences:
@@ -8129,28 +8157,27 @@ ACTOR Future<Void> cleanUpMoveInShard(StorageServer* data, Version version, Move
 // 4. Apply any new updates accumulated since the checkpoint version.
 // 5. Mark the new shard as read-write
 // 6. Clean up all the checkpoint files etc.
-ACTOR Future<Void> fetchShard(StorageServer* data, MoveInShard* moveInShard) {
+Future<Void> fetchShard(StorageServer* data, MoveInShard* moveInShard) {
 	TraceEvent(SevInfo, "FetchShardBegin", data->thisServerID).detail("MoveInShard", moveInShard->toString());
-	state std::shared_ptr<MoveInUpdates> moveInUpdates = moveInShard->updates;
-	state std::string dir = fetchedCheckpointDir(data->folder, moveInShard->id());
-	state MoveInPhase phase;
+	std::shared_ptr<MoveInUpdates> moveInUpdates = moveInShard->updates;
+	std::string dir = fetchedCheckpointDir(data->folder, moveInShard->id());
+	MoveInPhase phase;
 
 	ASSERT(moveInShard->getPhase() != MoveInPhase::Pending);
 
-	wait(data->coreStarted.getFuture() && data->durableVersion.whenAtLeast(moveInShard->meta->createVersion + 1));
-	wait(data->fetchKeysParallelismLock.take(TaskPriority::DefaultYield));
-	state FlowLock::Releaser holdingFKPL(data->fetchKeysParallelismLock);
+	co_await (data->coreStarted.getFuture() && data->durableVersion.whenAtLeast(moveInShard->meta->createVersion + 1));
+	co_await data->fetchKeysParallelismLock.take(TaskPriority::DefaultYield);
+	FlowLock::Releaser holdingFKPL(data->fetchKeysParallelismLock);
 
-	state BulkLoadTaskState bulkLoadTaskState;
-	state bool conductBulkLoad = moveInShard->meta->conductBulkLoad;
+	BulkLoadTaskState bulkLoadTaskState;
+	bool conductBulkLoad = moveInShard->meta->conductBulkLoad;
 	if (conductBulkLoad) {
 		// Get the bulkload task metadata from the data move metadata. For details, see the comments in the fetchKeys.
-		wait(store(bulkLoadTaskState,
-		           getBulkLoadTaskStateFromDataMove(
-		               data->cx, moveInShard->dataMoveId(), data->version.get(), data->thisServerID)));
+		bulkLoadTaskState = co_await getBulkLoadTaskStateFromDataMove(
+		    data->cx, moveInShard->dataMoveId(), data->version.get(), data->thisServerID);
 	}
 
-	loop {
+	while (true) {
 		phase = moveInShard->getPhase();
 		TraceEvent(moveInShard->logSev, "FetchShardLoop", data->thisServerID)
 		    .detail("MoveInShard", moveInShard->toString());
@@ -8161,16 +8188,16 @@ ACTOR Future<Void> fetchShard(StorageServer* data, MoveInShard* moveInShard) {
 					// Check the correctness: bulkLoadTaskMetadata stored in dataMoveMetadata must have the same
 					// dataMoveId.
 					ASSERT(bulkLoadTaskState.getDataMoveId() != moveInShard->dataMoveId());
-					wait(bulkLoadFetchShardFileToLoad(data, moveInShard, dir, bulkLoadTaskState));
+					co_await bulkLoadFetchShardFileToLoad(data, moveInShard, dir, bulkLoadTaskState);
 				} else {
-					wait(fetchShardCheckpoint(data, moveInShard, dir));
+					co_await fetchShardCheckpoint(data, moveInShard, dir);
 					TraceEvent(SevWarn, "SSBulkLoadTaskFetchShardFailedForNoMetadata", data->thisServerID)
 					    .detail("DataMoveId", moveInShard->dataMoveId());
 				}
 			} else if (phase == MoveInPhase::Ingesting) {
-				wait(fetchShardIngestCheckpoint(data, moveInShard));
+				co_await fetchShardIngestCheckpoint(data, moveInShard);
 			} else if (phase == MoveInPhase::ApplyingUpdates) {
-				wait(fetchShardApplyUpdates(data, moveInShard, moveInUpdates));
+				co_await fetchShardApplyUpdates(data, moveInShard, moveInUpdates);
 			} else if (phase == MoveInPhase::Complete) {
 				data->actors.add(cleanUpMoveInShard(data, data->data().getLatestVersion(), moveInShard));
 				break;
@@ -8185,14 +8212,12 @@ ACTOR Future<Void> fetchShard(StorageServer* data, MoveInShard* moveInShard) {
 			    .detail("MoveInShard", moveInShard->toString());
 			throw e;
 		}
-		wait(delay(1, TaskPriority::FetchKeys));
+		co_await delay(1, TaskPriority::FetchKeys);
 	}
 
 	validate(data);
 
 	TraceEvent(SevInfo, "FetchShardEnd", data->thisServerID).detail("MoveInShard", moveInShard->toString());
-
-	return Void();
 }
 
 MoveInUpdates::MoveInUpdates(UID id,
@@ -8488,33 +8513,32 @@ void ShardInfo::addMutation(Version version, bool fromFetch, MutationRef const&
 	}
 }
 
-ACTOR Future<Void> restoreShards(StorageServer* data,
-                                 Version version,
-                                 RangeResult storageShards,
-                                 RangeResult moveInShards,
-                                 RangeResult assignedShards,
-                                 RangeResult availableShards) {
+Future<Void> restoreShards(StorageServer* data,
+                           Version version,
+                           RangeResult storageShards,
+                           RangeResult moveInShards,
+                           RangeResult assignedShards,
+                           RangeResult availableShards) {
 	TraceEvent(SevInfo, "StorageServerRestoreShardsBegin", data->thisServerID)
 	    .detail("StorageShard", storageShards.size())
 	    .detail("Version", version);
 
-	state int moveInLoc;
+	int moveInLoc{ 0 };
 	for (moveInLoc = 0; moveInLoc < moveInShards.size(); ++moveInLoc) {
 		MoveInShardMetaData meta = decodeMoveInShardValue(moveInShards[moveInLoc].value);
 		TraceEvent(SevInfo, "RestoreMoveInShard", data->thisServerID).detail("MoveInShard", meta.toString());
 		data->moveInShards.emplace(meta.id, std::make_shared<MoveInShard>(data, meta));
-		wait(yield());
+		co_await yield();
 	}
 
-	state int shardLoc;
+	int shardLoc{ 0 };
 	for (shardLoc = 0; shardLoc < storageShards.size(); ++shardLoc) {
 		const KeyRangeRef shardRange(
 		    storageShards[shardLoc].key.removePrefix(persistStorageServerShardKeys.begin),
 		    shardLoc + 1 == storageShards.size()
 		        ? allKeys.end
 		        : storageShards[shardLoc + 1].key.removePrefix(persistStorageServerShardKeys.begin));
-		StorageServerShard shard =
-		    ObjectReader::fromStringRef<StorageServerShard>(storageShards[shardLoc].value, IncludeVersion());
+		auto shard = ObjectReader::fromStringRef<StorageServerShard>(storageShards[shardLoc].value, IncludeVersion());
 		shard.range = shardRange;
 		TraceEvent(SevVerbose, "RestoreShardsStorageShard", data->thisServerID)
 		    .detail("Range", shardRange)
@@ -8538,7 +8562,7 @@ ACTOR Future<Void> restoreShards(StorageServer* data,
 
 		auto ranges = data->shards.getAffectedRangesAfterInsertion(shard.range, Reference<ShardInfo>());
 		for (int i = 0; i < ranges.size(); i++) {
-			KeyRangeRef& range = static_cast<KeyRangeRef&>(ranges[i]);
+			auto& range = static_cast<KeyRangeRef&>(ranges[i]);
 			TraceEvent(SevVerbose, "RestoreShardsAddShard", data->thisServerID)
 			    .detail("Shard", shard.toString())
 			    .detail("Range", range);
@@ -8567,10 +8591,10 @@ ACTOR Future<Void> restoreShards(StorageServer* data,
 			data->newestDirtyVersion.insert(shardRange, version);
 		}
 
-		wait(yield());
+		co_await yield();
 	}
 
-	state int availableLoc;
+	int availableLoc{ 0 };
 	for (availableLoc = 0; availableLoc < availableShards.size(); availableLoc++) {
 		KeyRangeRef shardRange(
 		    availableShards[availableLoc].key.removePrefix(persistShardAvailableKeys.begin),
@@ -8594,10 +8618,10 @@ ACTOR Future<Void> restoreShards(StorageServer* data,
 			}
 		}
 
-		wait(yield());
+		co_await yield();
 	}
 
-	state int assignedLoc;
+	int assignedLoc{ 0 };
 	for (assignedLoc = 0; assignedLoc < assignedShards.size(); ++assignedLoc) {
 		KeyRangeRef shardRange(assignedShards[assignedLoc].key.removePrefix(persistShardAssignedKeys.begin),
 		                       assignedLoc + 1 == assignedShards.size()
@@ -8621,13 +8645,11 @@ ACTOR Future<Void> restoreShards(StorageServer* data,
 			}
 		}
 
-		wait(yield());
+		co_await yield();
 	}
 
 	validate(data, /*force=*/true);
 	TraceEvent(SevInfo, "StorageServerRestoreShardsEnd", data->thisServerID).detail("Version", version);
-
-	return Void();
 }
 
 void changeServerKeys(StorageServer* data,
@@ -9193,15 +9215,12 @@ void StorageServer::addMutation(Version version,
                                 KeyRangeRef const& shard,
                                 UpdateEagerReadInfo* eagerReads) {
 	MutationRef expanded = mutation;
-	MutationRef
-	    nonExpanded; // need to keep non-expanded but atomic converted version of clear mutations for change feeds
 	auto& mLog = addVersionToMutationLog(version);
 
 	if (!convertAtomicOp(expanded, data(), eagerReads, mLog.arena())) {
 		return;
 	}
 	if (expanded.type == MutationRef::ClearRange) {
-		nonExpanded = expanded;
 		expandClear(expanded, data(), eagerReads, shard.end);
 	}
 	expanded = addMutationToMutationLog(mLog, expanded);
@@ -9523,623 +9542,633 @@ class StorageUpdater {
 	}
 };
 
-ACTOR Future<Void> tssDelayForever() {
-	loop {
-		wait(delay(5.0));
+Future<Void> tssDelayForever() {
+	while (true) {
+		co_await delay(5.0);
 		if (g_simulator->speedUpSimulation) {
-			return Void();
+			co_return;
 		}
 	}
 }
 
-ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
-	state double updateStart = g_network->timer();
-	state double decryptionTime = 0;
-	state double start;
-	state bool enableClearRangeEagerReads =
-	    (data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_ROCKSDB_V1 ||
-	     data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_SHARDED_ROCKSDB)
-	        ? SERVER_KNOBS->ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS
-	        : SERVER_KNOBS->ENABLE_CLEAR_RANGE_EAGER_READS;
-	state UpdateEagerReadInfo eager(enableClearRangeEagerReads);
-	try {
+Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
+	double updateStart = g_network->timer();
+	double start{ 0 };
+	bool enableClearRangeEagerReads = (data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_ROCKSDB_V1 ||
+	                                   data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_SHARDED_ROCKSDB)
+	                                      ? SERVER_KNOBS->ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS
+	                                      : SERVER_KNOBS->ENABLE_CLEAR_RANGE_EAGER_READS;
+	UpdateEagerReadInfo eager(enableClearRangeEagerReads);
+	{
+		Error err;
+		try {
+			// If we are disk bound and durableVersion is very old, we need to block updates or we could run out of
+			// memory. This is often referred to as the storage server e-brake (emergency brake)
+
+			// We allow the storage server to make some progress between e-brake periods, referred to as "overage", in
+			// order to ensure that it advances desiredOldestVersion enough for updateStorage to make enough progress on
+			// freeing up queue size. We also increase these limits if speed up simulation was set IF they were
+			// buggified to a very small value.
+			int64_t hardLimit = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES;
+			int64_t hardLimitOverage = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE;
+			if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
+				hardLimit = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM;
+				hardLimitOverage = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM;
+			}
+			double waitStartT = 0;
+			if (data->queueSize() >= hardLimit && data->durableVersion.get() < data->desiredOldestVersion.get() &&
+			    ((data->desiredOldestVersion.get() - SERVER_KNOBS->STORAGE_HARD_LIMIT_VERSION_OVERAGE >
+			      data->lastDurableVersionEBrake) ||
+			     (data->counters.bytesInput.getValue() - hardLimitOverage > data->lastBytesInputEBrake))) {
+
+				while (data->queueSize() >= hardLimit &&
+				       data->durableVersion.get() < data->desiredOldestVersion.get()) {
+					if (now() - waitStartT >= 1) {
+						TraceEvent(SevWarn, "StorageServerUpdateLag", data->thisServerID)
+						    .detail("Version", data->version.get())
+						    .detail("DurableVersion", data->durableVersion.get())
+						    .detail("DesiredOldestVersion", data->desiredOldestVersion.get())
+						    .detail("QueueSize", data->queueSize())
+						    .detail("LastBytesInputEBrake", data->lastBytesInputEBrake)
+						    .detail("LastDurableVersionEBrake", data->lastDurableVersionEBrake);
+						waitStartT = now();
+					}
 
-		// If we are disk bound and durableVersion is very old, we need to block updates or we could run out of
-		// memory. This is often referred to as the storage server e-brake (emergency brake)
-
-		// We allow the storage server to make some progress between e-brake periods, referred to as "overage", in
-		// order to ensure that it advances desiredOldestVersion enough for updateStorage to make enough progress on
-		// freeing up queue size. We also increase these limits if speed up simulation was set IF they were
-		// buggified to a very small value.
-		state int64_t hardLimit = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES;
-		state int64_t hardLimitOverage = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE;
-		if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
-			hardLimit = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM;
-			hardLimitOverage = SERVER_KNOBS->STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM;
-		}
-		state double waitStartT = 0;
-		if (data->queueSize() >= hardLimit && data->durableVersion.get() < data->desiredOldestVersion.get() &&
-		    ((data->desiredOldestVersion.get() - SERVER_KNOBS->STORAGE_HARD_LIMIT_VERSION_OVERAGE >
-		      data->lastDurableVersionEBrake) ||
-		     (data->counters.bytesInput.getValue() - hardLimitOverage > data->lastBytesInputEBrake))) {
-
-			while (data->queueSize() >= hardLimit && data->durableVersion.get() < data->desiredOldestVersion.get()) {
-				if (now() - waitStartT >= 1) {
-					TraceEvent(SevWarn, "StorageServerUpdateLag", data->thisServerID)
-					    .detail("Version", data->version.get())
-					    .detail("DurableVersion", data->durableVersion.get())
-					    .detail("DesiredOldestVersion", data->desiredOldestVersion.get())
-					    .detail("QueueSize", data->queueSize())
-					    .detail("LastBytesInputEBrake", data->lastBytesInputEBrake)
-					    .detail("LastDurableVersionEBrake", data->lastDurableVersionEBrake);
-					waitStartT = now();
+					data->behind = true;
+					co_await delayJittered(.005, TaskPriority::TLogPeekReply);
+				}
+				data->lastBytesInputEBrake = data->counters.bytesInput.getValue();
+				data->lastDurableVersionEBrake = data->durableVersion.get();
+			}
+
+			if (g_network->isSimulated() && data->isTss() &&
+			    simulationPolicyHasCapability(ISimulationPolicy::Capability::StorageReplicaDelay) &&
+			    !g_simulator->speedUpSimulation && data->tssFaultInjectTime.present() &&
+			    data->tssFaultInjectTime.get() < now()) {
+				if (deterministicRandom()->random01() < 0.01) {
+					TraceEvent(SevWarnAlways, "TSSInjectDelayForever", data->thisServerID).log();
+					// small random chance to just completely get stuck here, each tss should eventually hit this in
+					// this mode
+					co_await tssDelayForever();
+				} else {
+					// otherwise pause for part of a second
+					double delayTime = deterministicRandom()->random01();
+					TraceEvent(SevWarnAlways, "TSSInjectDelay", data->thisServerID).detail("Delay", delayTime);
+					co_await delay(delayTime);
 				}
-
-				data->behind = true;
-				wait(delayJittered(.005, TaskPriority::TLogPeekReply));
 			}
-			data->lastBytesInputEBrake = data->counters.bytesInput.getValue();
-			data->lastDurableVersionEBrake = data->durableVersion.get();
-		}
 
-		if (g_network->isSimulated() && data->isTss() && g_simulator->tssMode == ISimulator::TSSMode::EnabledAddDelay &&
-		    !g_simulator->speedUpSimulation && data->tssFaultInjectTime.present() &&
-		    data->tssFaultInjectTime.get() < now()) {
-			if (deterministicRandom()->random01() < 0.01) {
-				TraceEvent(SevWarnAlways, "TSSInjectDelayForever", data->thisServerID).log();
-				// small random chance to just completely get stuck here, each tss should eventually hit this in
-				// this mode
-				wait(tssDelayForever());
-			} else {
-				// otherwise pause for part of a second
-				double delayTime = deterministicRandom()->random01();
-				TraceEvent(SevWarnAlways, "TSSInjectDelay", data->thisServerID).detail("Delay", delayTime);
-				wait(delay(delayTime));
+			if (data->maybeInjectDelay()) {
+				co_await delay(deterministicRandom()->random01() * 10.0);
 			}
-		}
-
-		if (data->maybeInjectDelay()) {
-			wait(delay(deterministicRandom()->random01() * 10.0));
-		}
-
-		while (data->byteSampleClearsTooLarge.get()) {
-			wait(data->byteSampleClearsTooLarge.onChange());
-		}
-
-		state Reference<ILogSystem::IPeekCursor> cursor = data->logCursor;
 
-		state double beforeTLogCursorReads = now();
-		loop {
-			wait(cursor->getMore());
-			if (!cursor->isExhausted()) {
-				break;
+			while (data->byteSampleClearsTooLarge.get()) {
+				co_await data->byteSampleClearsTooLarge.onChange();
 			}
-		}
-		data->tlogCursorReadsLatencyHistogram->sampleSeconds(now() - beforeTLogCursorReads);
-		if (cursor->popped() > 0) {
-			TraceEvent("StorageServerWorkerRemoved", data->thisServerID)
-			    .detail("Reason", "PeekPoppedTLogData")
-			    .detail("Version", cursor->popped());
-			throw worker_removed();
-		}
-
-		++data->counters.updateBatches;
-		data->lastTLogVersion = cursor->getMaxKnownVersion();
-		if (cursor->getMinKnownCommittedVersion() > data->knownCommittedVersion.get()) {
-			data->knownCommittedVersion.set(cursor->getMinKnownCommittedVersion());
-		}
-		data->versionLag = std::max<int64_t>(0, data->lastTLogVersion - data->version.get());
-
-		ASSERT(*pReceivedUpdate == false);
-		*pReceivedUpdate = true;
-
-		start = now();
-		wait(data->durableVersionLock.take(TaskPriority::TLogPeekReply, 1));
-		state FlowLock::Releaser holdingDVL(data->durableVersionLock);
-		if (now() - start > 0.1)
-			TraceEvent("SSSlowTakeLock1", data->thisServerID)
-			    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken)
-			    .detail("Duration", now() - start)
-			    .detail("Version", data->version.get());
-		data->ssVersionLockLatencyHistogram->sampleSeconds(now() - start);
-
-		start = now();
-		state FetchInjectionInfo fii;
-		state Reference<ILogSystem::IPeekCursor> cloneCursor2 = cursor->cloneNoMore();
-
-		// Collect eager read keys.
-		loop {
-			state uint64_t changeCounter = data->shardChangeCounter;
-			bool epochEnd = false;
-			bool hasPrivateData = false;
-			bool firstMutation = true;
-			bool dbgLastMessageWasProtocol = false;
-
-			Reference<ILogSystem::IPeekCursor> cloneCursor1 = cloneCursor2->cloneNoMore();
 
-			cloneCursor1->setProtocolVersion(data->logProtocol);
+			Reference<IPeekCursor> cursor = data->logCursor;
 
-			for (; cloneCursor1->hasMessage(); cloneCursor1->nextMessage()) {
-				ArenaReader& cloneReader = *cloneCursor1->reader();
-
-				if (LogProtocolMessage::isNextIn(cloneReader)) {
-					LogProtocolMessage lpm;
-					cloneReader >> lpm;
-					//TraceEvent(SevDebug, "SSReadingLPM", data->thisServerID).detail("Mutation", lpm);
-					dbgLastMessageWasProtocol = true;
-					cloneCursor1->setProtocolVersion(cloneReader.protocolVersion());
-				} else if (cloneReader.protocolVersion().hasSpanContext() &&
-				           SpanContextMessage::isNextIn(cloneReader)) {
-					SpanContextMessage scm;
-					cloneReader >> scm;
-				} else if (cloneReader.protocolVersion().hasOTELSpanContext() &&
-				           OTELSpanContextMessage::isNextIn(cloneReader)) {
-					OTELSpanContextMessage scm;
-					cloneReader >> scm;
-				} else {
-					MutationRef msg;
-					cloneReader >> msg;
-					if (!msg.validateChecksum()) {
-						TraceEvent(SevError, "ValidateChecksumError", data->thisServerID)
-						    .setMaxFieldLength(-1)
-						    .setMaxEventLength(-1)
-						    .detail("Mutation", msg);
-						ASSERT(false);
-					}
-					// TraceEvent(SevDebug, "SSReadingLog", data->thisServerID).detail("Mutation", msg);
-					if (data->acsValidator != nullptr) {
-						data->acsValidator->incrementTotalMutations();
-						if (isAccumulativeChecksumMutation(msg)) {
-							data->acsValidator->incrementTotalAcsMutations();
-						}
-					}
-					if (firstMutation && msg.param1.startsWith(systemKeys.end)) {
-						hasPrivateData = true;
-					}
-					firstMutation = false;
-					if (msg.param1 == lastEpochEndPrivateKey) {
-						epochEnd = true;
-						ASSERT(dbgLastMessageWasProtocol);
-					}
-					eager.addMutation(msg);
-					dbgLastMessageWasProtocol = false;
+			double beforeTLogCursorReads = now();
+			while (true) {
+				co_await cursor->getMore();
+				if (!cursor->isExhausted()) {
+					break;
 				}
 			}
-
-			// Any fetchKeys which are ready to transition their shards to the adding,transferred state do so
-			// now. If there is an epoch end we skip this step, to increase testability and to prevent inserting
-			// a version in the middle of a rolled back version range.
-			while (!hasPrivateData && !epochEnd && !data->readyFetchKeys.empty()) {
-				auto fk = data->readyFetchKeys.back();
-				data->readyFetchKeys.pop_back();
-				fk.send(&fii);
-				// fetchKeys() would put the data it fetched into the fii. The thread will not return back to
-				// this actor until it was completed.
-			}
-
-			for (auto& c : fii.changes) {
-				eager.addMutations(c.mutations);
+			data->tlogCursorReadsLatencyHistogram->sampleSeconds(now() - beforeTLogCursorReads);
+			if (cursor->popped() > 0) {
+				TraceEvent("StorageServerWorkerRemoved", data->thisServerID)
+				    .detail("Reason", "PeekPoppedTLogData")
+				    .detail("Version", cursor->popped());
+				throw worker_removed();
 			}
 
-			wait(doEagerReads(data, &eager));
-			if (data->shardChangeCounter == changeCounter) {
-				break;
-			}
+			++data->counters.updateBatches;
+			data->lastTLogVersion = cursor->getMaxKnownVersion();
+			if (cursor->getMinKnownCommittedVersion() > data->knownCommittedVersion.get()) {
+				data->knownCommittedVersion.set(cursor->getMinKnownCommittedVersion());
+			}
+			data->versionLag = std::max<int64_t>(0, data->lastTLogVersion - data->version.get());
+
+			ASSERT(*pReceivedUpdate == false);
+			*pReceivedUpdate = true;
+
+			start = now();
+			co_await data->durableVersionLock.take(TaskPriority::TLogPeekReply, 1);
+			FlowLock::Releaser holdingDVL(data->durableVersionLock);
+			if (now() - start > 0.1)
+				TraceEvent("SSSlowTakeLock1", data->thisServerID)
+				    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken)
+				    .detail("Duration", now() - start)
+				    .detail("Version", data->version.get());
+			data->ssVersionLockLatencyHistogram->sampleSeconds(now() - start);
+
+			start = now();
+			FetchInjectionInfo fii;
+			Reference<IPeekCursor> cloneCursor2 = cursor->cloneNoMore();
+
+			// Collect eager read keys.
+			while (true) {
+				uint64_t changeCounter = data->shardChangeCounter;
+				bool epochEnd = false;
+				bool hasPrivateData = false;
+				bool firstMutation = true;
+				bool dbgLastMessageWasProtocol = false;
+
+				Reference<IPeekCursor> cloneCursor1 = cloneCursor2->cloneNoMore();
+
+				cloneCursor1->setProtocolVersion(data->logProtocol);
+
+				for (; cloneCursor1->hasMessage(); cloneCursor1->nextMessage()) {
+					ArenaReader& cloneReader = *cloneCursor1->reader();
+
+					if (LogProtocolMessage::isNextIn(cloneReader)) {
+						LogProtocolMessage lpm;
+						cloneReader >> lpm;
+						//TraceEvent(SevDebug, "SSReadingLPM", data->thisServerID).detail("Mutation", lpm);
+						dbgLastMessageWasProtocol = true;
+						cloneCursor1->setProtocolVersion(cloneReader.protocolVersion());
+					} else if (cloneReader.protocolVersion().hasSpanContext() &&
+					           SpanContextMessage::isNextIn(cloneReader)) {
+						SpanContextMessage scm;
+						cloneReader >> scm;
+					} else if (cloneReader.protocolVersion().hasOTELSpanContext() &&
+					           OTELSpanContextMessage::isNextIn(cloneReader)) {
+						OTELSpanContextMessage scm;
+						cloneReader >> scm;
+					} else {
+						MutationRef msg;
+						cloneReader >> msg;
+						if (!msg.validateChecksum()) {
+							TraceEvent(SevError, "ValidateChecksumError", data->thisServerID)
+							    .setMaxFieldLength(-1)
+							    .setMaxEventLength(-1)
+							    .detail("Mutation", msg);
+							ASSERT(false);
+						}
+						// TraceEvent(SevDebug, "SSReadingLog", data->thisServerID).detail("Mutation", msg);
+						if (data->acsValidator != nullptr) {
+							data->acsValidator->incrementTotalMutations();
+							if (isAccumulativeChecksumMutation(msg)) {
+								data->acsValidator->incrementTotalAcsMutations();
+							}
+						}
+						if (firstMutation && msg.param1.startsWith(systemKeys.end)) {
+							hasPrivateData = true;
+						}
+						firstMutation = false;
+						if (msg.param1 == lastEpochEndPrivateKey) {
+							epochEnd = true;
+							ASSERT(dbgLastMessageWasProtocol);
+						}
+						eager.addMutation(msg);
+						dbgLastMessageWasProtocol = false;
+					}
+				}
 
-			CODE_PROBE(true,
-			           "A fetchKeys completed while we were doing this, so eager might be outdated.  Read it again.");
-			// SOMEDAY: Theoretically we could check the change counters of individual shards and retry the
-			// reads only selectively
-			eager = UpdateEagerReadInfo(enableClearRangeEagerReads);
-			cloneCursor2 = cursor->cloneNoMore();
-		}
-		data->eagerReadsLatencyHistogram->sampleSeconds(now() - start);
+				// Any fetchKeys which are ready to transition their shards to the adding,transferred state do so
+				// now. If there is an epoch end we skip this step, to increase testability and to prevent inserting
+				// a version in the middle of a rolled back version range.
+				while (!hasPrivateData && !epochEnd && !data->readyFetchKeys.empty()) {
+					auto fk = data->readyFetchKeys.back();
+					data->readyFetchKeys.pop_back();
+					fk.send(&fii);
+					// fetchKeys() would put the data it fetched into the fii. The thread will not return back to
+					// this actor until it was completed.
+				}
 
-		if (now() - start > 0.1) {
-			TraceEvent("SSSlowTakeLock2", data->thisServerID)
-			    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken)
-			    .detail("Duration", now() - start)
-			    .detail("Version", data->version.get());
-		}
-		data->updateEagerReads = &eager;
-		data->debug_inApplyUpdate = true;
+				for (auto& c : fii.changes) {
+					eager.addMutations(c.mutations);
+				}
 
-		state StorageUpdater updater(data->lastVersionWithData, data->restoredVersion);
+				co_await doEagerReads(data, &eager);
+				if (data->shardChangeCounter == changeCounter) {
+					break;
+				}
 
-		if (EXPENSIVE_VALIDATION)
-			data->data().atLatest().validate();
-		validate(data);
+				CODE_PROBE(
+				    true,
+				    "A fetchKeys completed while we were doing this, so eager might be outdated.  Read it again.");
+				// SOMEDAY: Theoretically we could check the change counters of individual shards and retry the
+				// reads only selectively
+				eager = UpdateEagerReadInfo(enableClearRangeEagerReads);
+				cloneCursor2 = cursor->cloneNoMore();
+			}
+			data->eagerReadsLatencyHistogram->sampleSeconds(now() - start);
+
+			if (now() - start > 0.1) {
+				TraceEvent("SSSlowTakeLock2", data->thisServerID)
+				    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken)
+				    .detail("Duration", now() - start)
+				    .detail("Version", data->version.get());
+			}
+			data->updateEagerReads = &eager;
+			data->debug_inApplyUpdate = true;
+
+			StorageUpdater updater(data->lastVersionWithData, data->restoredVersion);
+
+			if (EXPENSIVE_VALIDATION)
+				data->data().atLatest().validate();
+			validate(data);
+
+			bool injectedChanges = false;
+			int changeNum = 0;
+			int mutationBytes = 0;
+			double beforeFetchKeysUpdates = now();
+			for (; changeNum < fii.changes.size(); changeNum++) {
+				int mutationNum = 0;
+				VerUpdateRef* pUpdate = &fii.changes[changeNum];
+				for (; mutationNum < pUpdate->mutations.size(); mutationNum++) {
+					updater.applyMutation(data, pUpdate->mutations[mutationNum], pUpdate->version, true);
+					mutationBytes += pUpdate->mutations[mutationNum].totalSize();
+					// data->counters.mutationBytes or data->counters.mutations should not be updated because they
+					// should have counted when the mutations arrive from cursor initially.
+					injectedChanges = true;
+					if (mutationBytes > SERVER_KNOBS->DESIRED_UPDATE_BYTES) {
+						mutationBytes = 0;
+						co_await delay(SERVER_KNOBS->UPDATE_DELAY);
+					}
+				}
+			}
+			data->fetchKeysPTreeUpdatesLatencyHistogram->sampleSeconds(now() - beforeFetchKeysUpdates);
 
-		state bool injectedChanges = false;
-		state int changeNum = 0;
-		state int mutationBytes = 0;
-		state double beforeFetchKeysUpdates = now();
-		for (; changeNum < fii.changes.size(); changeNum++) {
-			state int mutationNum = 0;
-			state VerUpdateRef* pUpdate = &fii.changes[changeNum];
-			for (; mutationNum < pUpdate->mutations.size(); mutationNum++) {
-				updater.applyMutation(data, pUpdate->mutations[mutationNum], pUpdate->version, true);
-				mutationBytes += pUpdate->mutations[mutationNum].totalSize();
-				// data->counters.mutationBytes or data->counters.mutations should not be updated because they
-				// should have counted when the mutations arrive from cursor initially.
-				injectedChanges = true;
+			Version ver = invalidVersion;
+			cloneCursor2->setProtocolVersion(data->logProtocol);
+			SpanContext spanContext = SpanContext();
+			double beforeTLogMsgsUpdates = now();
+			for (; cloneCursor2->hasMessage(); cloneCursor2->nextMessage()) {
 				if (mutationBytes > SERVER_KNOBS->DESIRED_UPDATE_BYTES) {
 					mutationBytes = 0;
-					wait(delay(SERVER_KNOBS->UPDATE_DELAY));
+					// Instead of just yielding, leave time for the storage server to respond to reads
+					co_await delay(SERVER_KNOBS->UPDATE_DELAY);
+				}
+
+				if (cloneCursor2->version().version > ver) {
+					ASSERT(cloneCursor2->version().version > data->version.get());
 				}
-			}
-		}
-		data->fetchKeysPTreeUpdatesLatencyHistogram->sampleSeconds(now() - beforeFetchKeysUpdates);
-
-		state Version ver = invalidVersion;
-		cloneCursor2->setProtocolVersion(data->logProtocol);
-		state SpanContext spanContext = SpanContext();
-		state double beforeTLogMsgsUpdates = now();
-		for (; cloneCursor2->hasMessage(); cloneCursor2->nextMessage()) {
-			if (mutationBytes > SERVER_KNOBS->DESIRED_UPDATE_BYTES) {
-				mutationBytes = 0;
-				// Instead of just yielding, leave time for the storage server to respond to reads
-				wait(delay(SERVER_KNOBS->UPDATE_DELAY));
-			}
-
-			if (cloneCursor2->version().version > ver) {
-				ASSERT(cloneCursor2->version().version > data->version.get());
-			}
-
-			auto& rd = *cloneCursor2->reader();
-
-			if (cloneCursor2->version().version > ver && cloneCursor2->version().version > data->version.get()) {
-				++data->counters.updateVersions;
-				ver = cloneCursor2->version().version;
-			}
-
-			if (LogProtocolMessage::isNextIn(rd)) {
-				LogProtocolMessage lpm;
-				rd >> lpm;
-
-				data->logProtocol = rd.protocolVersion();
-				data->storage.changeLogProtocol(ver, data->logProtocol);
-				cloneCursor2->setProtocolVersion(rd.protocolVersion());
-				spanContext.traceID = UID();
-			} else if (rd.protocolVersion().hasSpanContext() && SpanContextMessage::isNextIn(rd)) {
-				SpanContextMessage scm;
-				rd >> scm;
-				CODE_PROBE(true, "storageserveractor converting SpanContextMessage into OTEL SpanContext");
-				spanContext =
-				    SpanContext(UID(scm.spanContext.first(), scm.spanContext.second()),
-				                0,
-				                scm.spanContext.first() != 0 && scm.spanContext.second() != 0 ? TraceFlags::sampled
-				                                                                              : TraceFlags::unsampled);
-			} else if (rd.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(rd)) {
-				CODE_PROBE(true, "storageserveractor reading OTELSpanContextMessage");
-				OTELSpanContextMessage scm;
-				rd >> scm;
-				spanContext = scm.spanContext;
-			} else {
-				MutationRef msg;
-				rd >> msg;
-				if (data->acsValidator != nullptr && msg.checksum.present() &&
-				    msg.accumulativeChecksumIndex.present() && !isAccumulativeChecksumMutation(msg)) {
-					// We have to check accumulative checksum when iterating through cloneCursor2,
-					// where ss removal by tag assignment takes effect immediately
-					data->acsValidator->addMutation(
-					    msg, data->thisServerID, data->tag, data->version.get(), cloneCursor2->version().version);
+
+				auto& rd = *cloneCursor2->reader();
+
+				if (cloneCursor2->version().version > ver && cloneCursor2->version().version > data->version.get()) {
+					++data->counters.updateVersions;
+					ver = cloneCursor2->version().version;
 				}
 
-				Span span("SS:update"_loc, spanContext);
-
-				// Drop non-private mutations if TSS fault injection is enabled in simulation, or if this is a TSS
-				// in quarantine.
-				if (g_network->isSimulated() && data->isTss() && !g_simulator->speedUpSimulation &&
-				    g_simulator->tssMode == ISimulator::TSSMode::EnabledDropMutations &&
-				    data->tssFaultInjectTime.present() && data->tssFaultInjectTime.get() < now() &&
-				    (msg.type == MutationRef::SetValue || msg.type == MutationRef::ClearRange) &&
-				    (msg.param1.size() < 2 || msg.param1[0] != 0xff || msg.param1[1] != 0xff) &&
-				    deterministicRandom()->random01() < 0.05) {
-					TraceEvent(SevWarnAlways, "TSSInjectDropMutation", data->thisServerID)
-					    .detail("Mutation", msg)
-					    .detail("Version", cloneCursor2->version().toString());
-				} else if (data->isTSSInQuarantine() &&
-				           (msg.param1.size() < 2 || msg.param1[0] != 0xff || msg.param1[1] != 0xff)) {
-					TraceEvent("TSSQuarantineDropMutation", data->thisServerID)
-					    .suppressFor(10.0)
-					    .detail("Version", cloneCursor2->version().toString());
-				} else if (ver != invalidVersion) { // This change belongs to a version < minVersion
-					DEBUG_MUTATION("SSPeek", ver, msg, data->thisServerID);
-					if (ver == data->initialClusterVersion) {
-						//TraceEvent("SSPeekMutation", data->thisServerID).log();
-						// The following trace event may produce a value with special characters
-						TraceEvent("SSPeekMutation", data->thisServerID)
+				if (LogProtocolMessage::isNextIn(rd)) {
+					LogProtocolMessage lpm;
+					rd >> lpm;
+
+					data->logProtocol = rd.protocolVersion();
+					data->storage.changeLogProtocol(ver, data->logProtocol);
+					cloneCursor2->setProtocolVersion(rd.protocolVersion());
+					spanContext.traceID = UID();
+				} else if (rd.protocolVersion().hasSpanContext() && SpanContextMessage::isNextIn(rd)) {
+					SpanContextMessage scm;
+					rd >> scm;
+					CODE_PROBE(true, "storageserveractor converting SpanContextMessage into OTEL SpanContext");
+					spanContext = SpanContext(UID(scm.spanContext.first(), scm.spanContext.second()),
+					                          0,
+					                          scm.spanContext.first() != 0 && scm.spanContext.second() != 0
+					                              ? TraceFlags::sampled
+					                              : TraceFlags::unsampled);
+				} else if (rd.protocolVersion().hasOTELSpanContext() && OTELSpanContextMessage::isNextIn(rd)) {
+					CODE_PROBE(true, "storageserveractor reading OTELSpanContextMessage");
+					OTELSpanContextMessage scm;
+					rd >> scm;
+					spanContext = scm.spanContext;
+				} else {
+					MutationRef msg;
+					rd >> msg;
+					if (data->acsValidator != nullptr && msg.checksum.present() &&
+					    msg.accumulativeChecksumIndex.present() && !isAccumulativeChecksumMutation(msg)) {
+						// We have to check accumulative checksum when iterating through cloneCursor2,
+						// where ss removal by tag assignment takes effect immediately
+						data->acsValidator->addMutation(
+						    msg, data->thisServerID, data->tag, data->version.get(), cloneCursor2->version().version);
+					}
+
+					Span span("SS:update"_loc, spanContext);
+
+					// Drop non-private mutations if TSS fault injection is enabled in simulation, or if this is a TSS
+					// in quarantine.
+					if (g_network->isSimulated() && data->isTss() && !g_simulator->speedUpSimulation &&
+					    simulationPolicyHasCapability(ISimulationPolicy::Capability::StorageReplicaMutationDrop) &&
+					    data->tssFaultInjectTime.present() && data->tssFaultInjectTime.get() < now() &&
+					    (msg.type == MutationRef::SetValue || msg.type == MutationRef::ClearRange) &&
+					    (msg.param1.size() < 2 || msg.param1[0] != 0xff || msg.param1[1] != 0xff) &&
+					    deterministicRandom()->random01() < 0.05) {
+						TraceEvent(SevWarnAlways, "TSSInjectDropMutation", data->thisServerID)
 						    .detail("Mutation", msg)
 						    .detail("Version", cloneCursor2->version().toString());
-					}
+					} else if (data->isTSSInQuarantine() &&
+					           (msg.param1.size() < 2 || msg.param1[0] != 0xff || msg.param1[1] != 0xff)) {
+						TraceEvent("TSSQuarantineDropMutation", data->thisServerID)
+						    .suppressFor(10.0)
+						    .detail("Version", cloneCursor2->version().toString());
+					} else if (ver != invalidVersion) { // This change belongs to a version < minVersion
+						DEBUG_MUTATION("SSPeek", ver, msg, data->thisServerID);
+						if (ver == data->initialClusterVersion) {
+							//TraceEvent("SSPeekMutation", data->thisServerID).log();
+							// The following trace event may produce a value with special characters
+							TraceEvent("SSPeekMutation", data->thisServerID)
+							    .detail("Mutation", msg)
+							    .detail("Version", cloneCursor2->version().toString());
+						}
 
-					updater.applyMutation(data, msg, ver, false);
-					mutationBytes += msg.totalSize();
-					data->counters.mutationBytes += msg.totalSize();
-					data->counters.logicalBytesInput += msg.expectedSize();
-					++data->counters.mutations;
-					switch (msg.type) {
-					case MutationRef::SetValue:
-						++data->counters.setMutations;
-						break;
-					case MutationRef::ClearRange:
-						++data->counters.clearRangeMutations;
-						break;
-					case MutationRef::AddValue:
-					case MutationRef::And:
-					case MutationRef::AndV2:
-					case MutationRef::AppendIfFits:
-					case MutationRef::ByteMax:
-					case MutationRef::ByteMin:
-					case MutationRef::Max:
-					case MutationRef::Min:
-					case MutationRef::MinV2:
-					case MutationRef::Or:
-					case MutationRef::Xor:
-					case MutationRef::CompareAndClear:
-						++data->counters.atomicMutations;
-						break;
-					}
-				} else
-					TraceEvent(SevError, "DiscardingPeekedData", data->thisServerID)
-					    .detail("Mutation", msg)
-					    .detail("Version", cloneCursor2->version().toString());
+						updater.applyMutation(data, msg, ver, false);
+						mutationBytes += msg.totalSize();
+						data->counters.mutationBytes += msg.totalSize();
+						data->counters.logicalBytesInput += msg.expectedSize();
+						++data->counters.mutations;
+						switch (msg.type) {
+						case MutationRef::SetValue:
+							++data->counters.setMutations;
+							break;
+						case MutationRef::ClearRange:
+							++data->counters.clearRangeMutations;
+							break;
+						case MutationRef::AddValue:
+						case MutationRef::And:
+						case MutationRef::AndV2:
+						case MutationRef::AppendIfFits:
+						case MutationRef::ByteMax:
+						case MutationRef::ByteMin:
+						case MutationRef::Max:
+						case MutationRef::Min:
+						case MutationRef::MinV2:
+						case MutationRef::Or:
+						case MutationRef::Xor:
+						case MutationRef::CompareAndClear:
+							++data->counters.atomicMutations;
+							break;
+						}
+					} else
+						TraceEvent(SevError, "DiscardingPeekedData", data->thisServerID)
+						    .detail("Mutation", msg)
+						    .detail("Version", cloneCursor2->version().toString());
+				}
 			}
-		}
 
-		if (data->acsValidator != nullptr) {
-			data->acsValidator->clearCache(data->thisServerID, data->tag, data->version.get());
-		}
+			if (data->acsValidator != nullptr) {
+				data->acsValidator->clearCache(data->thisServerID, data->tag, data->version.get());
+			}
 
-		data->tLogMsgsPTreeUpdatesLatencyHistogram->sampleSeconds(now() - beforeTLogMsgsUpdates);
+			data->tLogMsgsPTreeUpdatesLatencyHistogram->sampleSeconds(now() - beforeTLogMsgsUpdates);
 
-		if (ver != invalidVersion) {
-			data->lastVersionWithData = ver;
-		}
-		ver = cloneCursor2->version().version - 1;
+			if (ver != invalidVersion) {
+				data->lastVersionWithData = ver;
+			}
+			ver = cloneCursor2->version().version - 1;
 
-		if (injectedChanges)
-			data->lastVersionWithData = ver;
+			if (injectedChanges)
+				data->lastVersionWithData = ver;
 
-		data->updateEagerReads = nullptr;
-		data->debug_inApplyUpdate = false;
+			data->updateEagerReads = nullptr;
+			data->debug_inApplyUpdate = false;
 
-		if (ver == invalidVersion && !fii.changes.empty()) {
-			ver = updater.currentVersion;
-		}
+			if (ver == invalidVersion && !fii.changes.empty()) {
+				ver = updater.currentVersion;
+			}
 
-		if (ver != invalidVersion && ver > data->version.get()) {
-			// TODO(alexmiller): Update to version tracking.
-			// DEBUG_KEY_RANGE("SSUpdate", ver, KeyRangeRef());
+			if (ver != invalidVersion && ver > data->version.get()) {
+				// TODO(alexmiller): Update to version tracking.
+				// DEBUG_KEY_RANGE("SSUpdate", ver, KeyRangeRef());
 
-			data->mutableData().createNewVersion(ver);
-			if (data->otherError.getFuture().isReady())
-				data->otherError.getFuture().get();
+				data->mutableData().createNewVersion(ver);
+				if (data->otherError.getFuture().isReady())
+					data->otherError.getFuture().get();
 
-			data->counters.fetchedVersions += (ver - data->version.get());
-			++data->counters.fetchesFromLogs;
-			Optional<UID> curSourceTLogID = cursor->getCurrentPeekLocation();
+				data->counters.fetchedVersions += (ver - data->version.get());
+				++data->counters.fetchesFromLogs;
+				Optional<UID> curSourceTLogID = cursor->getCurrentPeekLocation();
 
-			if (curSourceTLogID != data->sourceTLogID) {
-				data->sourceTLogID = curSourceTLogID;
+				if (curSourceTLogID != data->sourceTLogID) {
+					data->sourceTLogID = curSourceTLogID;
 
-				TraceEvent("StorageServerSourceTLogID", data->thisServerID)
-				    .detail("SourceTLogID",
-				            data->sourceTLogID.present() ? data->sourceTLogID.get().toString() : "unknown")
-				    .trackLatest(data->storageServerSourceTLogIDEventHolder->trackingKey);
-			}
+					TraceEvent("StorageServerSourceTLogID", data->thisServerID)
+					    .detail("SourceTLogID",
+					            data->sourceTLogID.present() ? data->sourceTLogID.get().toString() : "unknown")
+					    .trackLatest(data->storageServerSourceTLogIDEventHolder->trackingKey);
+				}
 
-			data->noRecentUpdates.set(false);
-			data->lastUpdate = now();
+				data->noRecentUpdates.set(false);
+				data->lastUpdate = now();
 
-			data->prevVersion = data->version.get();
-			data->version.set(ver); // Triggers replies to waiting gets for new version(s)
+				data->prevVersion = data->version.get();
+				data->version.set(ver); // Triggers replies to waiting gets for new version(s)
 
-			setDataVersion(data->thisServerID, data->version.get());
-			if (data->otherError.getFuture().isReady())
-				data->otherError.getFuture().get();
+				setDataVersion(data->thisServerID, data->version.get());
+				if (data->otherError.getFuture().isReady())
+					data->otherError.getFuture().get();
 
-			Version maxVersionsInMemory =
-			    (g_network->isSimulated() && g_simulator->speedUpSimulation)
-			        ? std::max(5 * SERVER_KNOBS->VERSIONS_PER_SECOND, SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)
-			        : SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS;
-			for (int i = 0; i < data->recoveryVersionSkips.size(); i++) {
-				maxVersionsInMemory += data->recoveryVersionSkips[i].second;
-			}
+				Version maxVersionsInMemory = (g_network->isSimulated() && g_simulator->speedUpSimulation)
+				                                  ? std::max(5 * SERVER_KNOBS->VERSIONS_PER_SECOND,
+				                                             SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)
+				                                  : SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS;
+				for (int i = 0; i < data->recoveryVersionSkips.size(); i++) {
+					maxVersionsInMemory += data->recoveryVersionSkips[i].second;
+				}
 
-			// Trigger updateStorage if necessary
-			Version proposedOldestVersion =
-			    std::max(data->version.get(), cursor->getMinKnownCommittedVersion()) - maxVersionsInMemory;
-			if (data->primaryLocality == tagLocalitySpecial || data->tag.locality == data->primaryLocality) {
-				proposedOldestVersion = std::max(proposedOldestVersion, data->lastTLogVersion - maxVersionsInMemory);
+				// Trigger updateStorage if necessary
+				Version proposedOldestVersion =
+				    std::max(data->version.get(), cursor->getMinKnownCommittedVersion()) - maxVersionsInMemory;
+				if (data->primaryLocality == tagLocalitySpecial || data->tag.locality == data->primaryLocality) {
+					proposedOldestVersion =
+					    std::max(proposedOldestVersion, data->lastTLogVersion - maxVersionsInMemory);
+				}
+				proposedOldestVersion = std::min(proposedOldestVersion, data->version.get() - 1);
+				proposedOldestVersion = std::max(proposedOldestVersion, data->oldestVersion.get());
+				proposedOldestVersion = std::max(proposedOldestVersion, data->desiredOldestVersion.get());
+				proposedOldestVersion = std::max(proposedOldestVersion, data->initialClusterVersion);
+
+				DisabledTraceEvent("StorageServerUpdated", data->thisServerID)
+				    .detail("Ver", ver)
+				    .detail("DataVersion", data->version.get())
+				    .detail("LastTLogVersion", data->lastTLogVersion)
+				    .detail("NewOldest", data->oldestVersion.get())
+				    .detail("DesiredOldest", data->desiredOldestVersion.get())
+				    .detail("MinKCV", cursor->getMinKnownCommittedVersion())
+				    .detail("MaxVersionInMemory", maxVersionsInMemory)
+				    .detail("Proposed", proposedOldestVersion)
+				    .detail("PrimaryLocality", data->primaryLocality)
+				    .detail("Tag", data->tag.toString());
+
+				while (!data->recoveryVersionSkips.empty() &&
+				       proposedOldestVersion > data->recoveryVersionSkips.front().first) {
+					data->recoveryVersionSkips.pop_front();
+				}
+				data->desiredOldestVersion.set(proposedOldestVersion);
 			}
-			proposedOldestVersion = std::min(proposedOldestVersion, data->version.get() - 1);
-			proposedOldestVersion = std::max(proposedOldestVersion, data->oldestVersion.get());
-			proposedOldestVersion = std::max(proposedOldestVersion, data->desiredOldestVersion.get());
-			proposedOldestVersion = std::max(proposedOldestVersion, data->initialClusterVersion);
 
-			DisabledTraceEvent("StorageServerUpdated", data->thisServerID)
-			    .detail("Ver", ver)
-			    .detail("DataVersion", data->version.get())
-			    .detail("LastTLogVersion", data->lastTLogVersion)
-			    .detail("NewOldest", data->oldestVersion.get())
-			    .detail("DesiredOldest", data->desiredOldestVersion.get())
-			    .detail("MinKCV", cursor->getMinKnownCommittedVersion())
-			    .detail("MaxVersionInMemory", maxVersionsInMemory)
-			    .detail("Proposed", proposedOldestVersion)
-			    .detail("PrimaryLocality", data->primaryLocality)
-			    .detail("Tag", data->tag.toString());
+			validate(data);
 
-			while (!data->recoveryVersionSkips.empty() &&
-			       proposedOldestVersion > data->recoveryVersionSkips.front().first) {
-				data->recoveryVersionSkips.pop_front();
+			if ((data->lastTLogVersion - data->version.get()) < SERVER_KNOBS->STORAGE_RECOVERY_VERSION_LAG_LIMIT) {
+				if (data->registerInterfaceAcceptingRequests.canBeSet()) {
+					data->registerInterfaceAcceptingRequests.send(Void());
+					ErrorOr<Void> e = co_await errorOr(data->interfaceRegistered);
+					if (e.isError()) {
+						TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed", data->thisServerID)
+						    .error(e.getError());
+						throw e.getError();
+					}
+				}
 			}
-			data->desiredOldestVersion.set(proposedOldestVersion);
-		}
 
-		validate(data);
-
-		if ((data->lastTLogVersion - data->version.get()) < SERVER_KNOBS->STORAGE_RECOVERY_VERSION_LAG_LIMIT) {
-			if (data->registerInterfaceAcceptingRequests.canBeSet()) {
-				data->registerInterfaceAcceptingRequests.send(Void());
-				ErrorOr<Void> e = wait(errorOr(data->interfaceRegistered));
-				if (e.isError()) {
-					TraceEvent(SevWarn, "StorageInterfaceRegistrationFailed", data->thisServerID).error(e.getError());
-					throw e.getError();
+			data->logCursor->advanceTo(cloneCursor2->version());
+			if (cursor->version().version >= data->lastTLogVersion) {
+				if (data->behind) {
+					TraceEvent("StorageServerNoLongerBehind", data->thisServerID)
+					    .detail("CursorVersion", cursor->version().version)
+					    .detail("TLogVersion", data->lastTLogVersion);
 				}
+				data->behind = false;
 			}
-		}
+			const double duration = g_network->timer() - updateStart;
+			data->counters.updateLatencySample->addMeasurement(duration);
 
-		data->logCursor->advanceTo(cloneCursor2->version());
-		if (cursor->version().version >= data->lastTLogVersion) {
-			if (data->behind) {
-				TraceEvent("StorageServerNoLongerBehind", data->thisServerID)
-				    .detail("CursorVersion", cursor->version().version)
-				    .detail("TLogVersion", data->lastTLogVersion);
-			}
-			data->behind = false;
+			co_return; // update will get called again ASAP
+		} catch (Error& e) {
+			err = e;
 		}
-		const double duration = g_network->timer() - updateStart;
-		data->counters.updateLatencySample->addMeasurement(duration);
-
-		return Void(); // update will get called again ASAP
-	} catch (Error& err) {
-		state Error e = err;
-		if (e.code() != error_code_worker_removed && e.code() != error_code_please_reboot) {
-			TraceEvent(SevError, "SSUpdateError", data->thisServerID).error(e).backtrace();
-		} else if (e.code() == error_code_please_reboot) {
-			wait(data->durableInProgress);
+		if (err.code() != error_code_worker_removed && err.code() != error_code_please_reboot) {
+			TraceEvent(SevError, "SSUpdateError", data->thisServerID).error(err).backtrace();
+		} else if (err.code() == error_code_please_reboot) {
+			co_await data->durableInProgress;
 		}
-		throw e;
+		throw err;
 	}
 }
 
-ACTOR Future<bool> createSstFileForCheckpointShardBytesSample(StorageServer* data,
-                                                              CheckpointMetaData metaData,
-                                                              std::string bytesSampleFile) {
-	state int failureCount = 0;
-	state std::unique_ptr<IRocksDBSstFileWriter> sstWriter;
-	state int64_t numGetRangeQueries;
-	state int64_t numSampledKeys;
-	state std::vector<KeyRange>::iterator metaDataRangesIter;
-	state Key readBegin;
-	state Key readEnd;
-	state bool anyFileCreated;
+Future<bool> createSstFileForCheckpointShardBytesSample(StorageServer* data,
+                                                        CheckpointMetaData metaData,
+                                                        std::string bytesSampleFile) {
+	int failureCount = 0;
+	std::unique_ptr<IRocksDBSstFileWriter> sstWriter;
+	int64_t numGetRangeQueries{ 0 };
+	int64_t numSampledKeys{ 0 };
+	std::vector<KeyRange>::iterator metaDataRangesIter;
+	Key readBegin;
+	Key readEnd;
+	bool anyFileCreated{ false };
 	TraceEvent(SevDebug, "CheckpointbytesSampleBegin", data->thisServerID).detail("Checkpoint", metaData.toString());
 
-	loop {
-		try {
-			// Any failure leads to retry until retryCount reaches maximum
-			// For each retry, cleanup the bytesSampleFile created in last time
-			ASSERT(directoryExists(parentDirectory(bytesSampleFile)));
-			if (fileExists(abspath(bytesSampleFile))) {
-				deleteFile(abspath(bytesSampleFile));
-			}
-			anyFileCreated = false;
-			sstWriter = newRocksDBSstFileWriter();
-			sstWriter->open(bytesSampleFile);
-			if (sstWriter == nullptr) {
-				break;
-			}
-			ASSERT(metaData.ranges.size() > 0);
-			std::sort(metaData.ranges.begin(), metaData.ranges.end(), [](KeyRange a, KeyRange b) {
-				// Debug usage: make sure no overlapping between compared two ranges
-				/* if (a.begin < b.begin) {
-				    ASSERT(a.end <= b.begin);
-				} else if (a.begin > b.begin) {
-				    ASSERT(a.end >= b.begin);
-				} else {
-				    ASSERT(false);
-				} */
-				// metaData.ranges must be in ascending order
-				// sstWriter requires written keys to be in ascending order
-				return a.begin < b.begin;
-			});
-
-			numGetRangeQueries = 0;
-			numSampledKeys = 0;
-			metaDataRangesIter = metaData.ranges.begin();
-			while (metaDataRangesIter != metaData.ranges.end()) {
-				KeyRange range = *metaDataRangesIter;
-				readBegin = range.begin.withPrefix(persistByteSampleKeys.begin);
-				readEnd = range.end.withPrefix(persistByteSampleKeys.begin);
-				loop {
-					try {
-						RangeResult readResult = wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd),
-						                                                      SERVER_KNOBS->STORAGE_LIMIT_BYTES,
-						                                                      SERVER_KNOBS->STORAGE_LIMIT_BYTES));
-						numGetRangeQueries++;
-						for (int i = 0; i < readResult.size(); i++) {
-							ASSERT(!readResult[i].key.empty() && !readResult[i].value.empty());
-							int64_t size = BinaryReader::fromStringRef<int64_t>(readResult[i].value, Unversioned());
-							KeyRef key = readResult[i].key.removePrefix(persistByteSampleKeys.begin);
-							TraceEvent(SevDebug, "CheckpointbytesSampleKey", data->thisServerID)
-							    // .setMaxFieldLength(10000)
-							    .detail("Checkpoint", metaData.toString())
-							    .detail("SampleKey", key)
-							    .detail("Size", size);
-							sstWriter->write(readResult[i].key, readResult[i].value);
-							numSampledKeys++;
-						}
-						if (readResult.more) {
-							readBegin = readResult.getReadThrough();
-							ASSERT(readBegin <= readEnd);
-						} else {
-							break; // finish for current metaDataRangesIter
-						}
-					} catch (Error& e) {
-						if (failureCount < SERVER_KNOBS->ROCKSDB_CREATE_BYTES_SAMPLE_FILE_RETRY_MAX) {
-							throw retry(); // retry from sketch
-						} else {
-							throw e;
+	while (true) {
+		{
+			Error err;
+			try {
+				// Any failure leads to retry until retryCount reaches maximum
+				// For each retry, cleanup the bytesSampleFile created in last time
+				ASSERT(directoryExists(parentDirectory(bytesSampleFile)));
+				if (fileExists(abspath(bytesSampleFile))) {
+					deleteFile(abspath(bytesSampleFile));
+				}
+				anyFileCreated = false;
+				sstWriter = newRocksDBSstFileWriter();
+				sstWriter->open(bytesSampleFile);
+				if (sstWriter == nullptr) {
+					break;
+				}
+				ASSERT(metaData.ranges.size() > 0);
+				std::sort(metaData.ranges.begin(), metaData.ranges.end(), [](KeyRange a, KeyRange b) {
+					// Debug usage: make sure no overlapping between compared two ranges
+					/* if (a.begin < b.begin) {
+					    ASSERT(a.end <= b.begin);
+					} else if (a.begin > b.begin) {
+					    ASSERT(a.end >= b.begin);
+					} else {
+					    ASSERT(false);
+					} */
+					// metaData.ranges must be in ascending order
+					// sstWriter requires written keys to be in ascending order
+					return a.begin < b.begin;
+				});
+
+				numGetRangeQueries = 0;
+				numSampledKeys = 0;
+				metaDataRangesIter = metaData.ranges.begin();
+				while (metaDataRangesIter != metaData.ranges.end()) {
+					KeyRange range = *metaDataRangesIter;
+					readBegin = range.begin.withPrefix(persistByteSampleKeys.begin);
+					readEnd = range.end.withPrefix(persistByteSampleKeys.begin);
+					while (true) {
+						try {
+							RangeResult readResult =
+							    co_await data->storage.readRange(KeyRangeRef(readBegin, readEnd),
+							                                     SERVER_KNOBS->STORAGE_LIMIT_BYTES,
+							                                     SERVER_KNOBS->STORAGE_LIMIT_BYTES);
+							numGetRangeQueries++;
+							for (int i = 0; i < readResult.size(); i++) {
+								ASSERT(!readResult[i].key.empty() && !readResult[i].value.empty());
+								int64_t size = BinaryReader::fromStringRef<int64_t>(readResult[i].value, Unversioned());
+								KeyRef key = readResult[i].key.removePrefix(persistByteSampleKeys.begin);
+								TraceEvent(SevDebug, "CheckpointbytesSampleKey", data->thisServerID)
+								    // .setMaxFieldLength(10000)
+								    .detail("Checkpoint", metaData.toString())
+								    .detail("SampleKey", key)
+								    .detail("Size", size);
+								sstWriter->write(readResult[i].key, readResult[i].value);
+								numSampledKeys++;
+							}
+							if (readResult.more) {
+								readBegin = readResult.getReadThrough();
+								ASSERT(readBegin <= readEnd);
+							} else {
+								break; // finish for current metaDataRangesIter
+							}
+						} catch (Error& e) {
+							if (failureCount < SERVER_KNOBS->ROCKSDB_CREATE_BYTES_SAMPLE_FILE_RETRY_MAX) {
+								throw retry(); // retry from sketch
+							} else {
+								throw e;
+							}
 						}
 					}
+					metaDataRangesIter++;
 				}
-				metaDataRangesIter++;
-			}
-			anyFileCreated = sstWriter->finish();
-			ASSERT((numSampledKeys > 0 && anyFileCreated) || (numSampledKeys == 0 && !anyFileCreated));
-			TraceEvent(SevDebug, "DumpCheckPointMetaData", data->thisServerID)
-			    .detail("NumSampledKeys", numSampledKeys)
-			    .detail("NumGetRangeQueries", numGetRangeQueries)
-			    .detail("CheckpointID", metaData.checkpointID.toString())
-			    .detail("BytesSampleTempFile", anyFileCreated ? bytesSampleFile : "noFileCreated");
-			break;
+				anyFileCreated = sstWriter->finish();
+				ASSERT((numSampledKeys > 0 && anyFileCreated) || (numSampledKeys == 0 && !anyFileCreated));
+				TraceEvent(SevDebug, "DumpCheckPointMetaData", data->thisServerID)
+				    .detail("NumSampledKeys", numSampledKeys)
+				    .detail("NumGetRangeQueries", numGetRangeQueries)
+				    .detail("CheckpointID", metaData.checkpointID.toString())
+				    .detail("BytesSampleTempFile", anyFileCreated ? bytesSampleFile : "noFileCreated");
+				break;
 
-		} catch (Error& e) {
-			if (e.code() == error_code_retry) {
-				wait(delay(0.5));
+			} catch (Error& e) {
+				err = e;
+			}
+			if (err.code() == error_code_retry) {
+				co_await delay(0.5);
 				failureCount++;
 				continue;
 			} else {
 				TraceEvent(SevDebug, "StorageCreateCheckpointMetaDataSstFileDumpedFailure", data->thisServerID)
 				    .detail("PendingCheckpoint", metaData.toString())
-				    .detail("Error", e.name());
-				throw e;
+				    .detail("Error", err.name());
+				throw err;
 			}
 		}
 	}
 
-	return anyFileCreated;
+	co_return anyFileCreated;
 }
 
-ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData metaData) {
+Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData metaData) {
 	TraceEvent(SevDebug, "SSCreateCheckpoint", data->thisServerID).detail("CheckpointMeta", metaData.toString());
 	ASSERT(std::find(metaData.src.begin(), metaData.src.end(), data->thisServerID) != metaData.src.end() &&
 	       !metaData.ranges.empty());
-	state std::string checkpointDir = serverCheckpointDir(data->checkpointFolder, metaData.checkpointID);
-	state std::string bytesSampleFile = abspath(joinPath(checkpointDir, checkpointBytesSampleFileName));
-	state std::string bytesSampleTempDir = data->folder + checkpointBytesSampleTempFolder;
-	state std::string bytesSampleTempFile =
+	std::string checkpointDir = serverCheckpointDir(data->checkpointFolder, metaData.checkpointID);
+	std::string bytesSampleFile = abspath(joinPath(checkpointDir, checkpointBytesSampleFileName));
+	std::string bytesSampleTempDir = data->folder + checkpointBytesSampleTempFolder;
+	std::string bytesSampleTempFile =
 	    bytesSampleTempDir + "/" + metaData.checkpointID.toString() + "_" + checkpointBytesSampleFileName;
 	const CheckpointRequest req(metaData.version,
 	                            metaData.ranges,
 	                            static_cast<CheckpointFormat>(metaData.format),
 	                            metaData.checkpointID,
 	                            checkpointDir);
-	state CheckpointMetaData checkpointResult;
-	state bool sampleByteSstFileCreated;
+	CheckpointMetaData checkpointResult;
+	bool sampleByteSstFileCreated{ false };
 	std::vector<Future<Void>> createCheckpointActors;
 
 	try {
@@ -10151,7 +10180,7 @@ ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData meta
 		}
 		createCheckpointActors.push_back(store(
 		    sampleByteSstFileCreated, createSstFileForCheckpointShardBytesSample(data, metaData, bytesSampleTempFile)));
-		wait(waitForAll(createCheckpointActors));
+		co_await waitForAll(createCheckpointActors);
 		// Move sst file to the checkpoint folder
 		if (sampleByteSstFileCreated) {
 			ASSERT(directoryExists(abspath(checkpointDir)));
@@ -10187,7 +10216,7 @@ ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData meta
 		Key persistCheckpointKey(persistCheckpointKeys.begin.toString() + checkpointResult.checkpointID.toString());
 		data->storage.clearRange(singleKeyRange(pendingCheckpointKey));
 		data->storage.writeKeyValue(KeyValueRef(persistCheckpointKey, checkpointValue(checkpointResult)));
-		wait(data->storage.commit());
+		co_await data->storage.commit();
 		TraceEvent("StorageCreateCheckpointPersisted", data->thisServerID)
 		    .detail("Checkpoint", checkpointResult.toString());
 	} catch (Error& e) {
@@ -10201,8 +10230,6 @@ ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData meta
 		data->checkpoints[checkpointResult.checkpointID].setState(CheckpointMetaData::Deleting);
 		data->actors.add(deleteCheckpointQ(data, metaData.version, checkpointResult));
 	}
-
-	return Void();
 }
 
 struct UpdateStorageCommitStats {
@@ -10883,21 +10910,19 @@ void StorageServerDisk::changeLogProtocol(Version version, ProtocolVersion proto
 	    MutationRef(MutationRef::SetValue, persistLogProtocol, BinaryWriter::toValue(protocol, Unversioned())));
 }
 
-ACTOR Future<Void> applyByteSampleResult(StorageServer* data,
-                                         IKeyValueStore* storage,
-                                         Key begin,
-                                         Key end,
-                                         std::vector<Standalone<VectorRef<KeyValueRef>>>* results = nullptr) {
-	state int totalFetches = 0;
-	state int totalKeys = 0;
-	state int totalBytes = 0;
-	state ReadOptions readOptions(ReadType::NORMAL, CacheResult::False);
+Future<Void> applyByteSampleResult(StorageServer* data,
+                                   IKeyValueStore* storage,
+                                   Key begin,
+                                   Key end,
+                                   std::vector<Standalone<VectorRef<KeyValueRef>>>* results = nullptr) {
+	int totalFetches = 0;
+	int totalKeys = 0;
+	int totalBytes = 0;
+	ReadOptions readOptions(ReadType::NORMAL, CacheResult::False);
 
-	loop {
-		RangeResult bs = wait(storage->readRange(KeyRangeRef(begin, end),
-		                                         SERVER_KNOBS->STORAGE_LIMIT_BYTES,
-		                                         SERVER_KNOBS->STORAGE_LIMIT_BYTES,
-		                                         readOptions));
+	while (true) {
+		RangeResult bs = co_await storage->readRange(
+		    KeyRangeRef(begin, end), SERVER_KNOBS->STORAGE_LIMIT_BYTES, SERVER_KNOBS->STORAGE_LIMIT_BYTES, readOptions);
 		if (results) {
 			results->push_back(bs.castTo<VectorRef<KeyValueRef>>());
 			data->bytesRestored += bs.logicalSize();
@@ -10936,7 +10961,7 @@ ACTOR Future<Void> applyByteSampleResult(StorageServer* data,
 		}
 
 		if (!results) {
-			wait(delay(SERVER_KNOBS->BYTE_SAMPLE_LOAD_DELAY));
+			co_await delay(SERVER_KNOBS->BYTE_SAMPLE_LOAD_DELAY);
 		}
 	}
 	TraceEvent("RecoveredByteSampleRange", data->thisServerID)
@@ -10945,19 +10970,18 @@ ACTOR Future<Void> applyByteSampleResult(StorageServer* data,
 	    .detail("Fetches", totalFetches)
 	    .detail("Keys", totalKeys)
 	    .detail("ReadBytes", totalBytes);
-	return Void();
 }
 
-ACTOR Future<Void> restoreByteSample(StorageServer* data,
-                                     IKeyValueStore* storage,
-                                     Promise<Void> byteSampleSampleRecovered,
-                                     Future<Void> startRestore) {
-	state std::vector<Standalone<VectorRef<KeyValueRef>>> byteSampleSample;
-	wait(applyByteSampleResult(
-	    data, storage, persistByteSampleSampleKeys.begin, persistByteSampleSampleKeys.end, &byteSampleSample));
+Future<Void> restoreByteSample(StorageServer* data,
+                               IKeyValueStore* storage,
+                               Promise<Void> byteSampleSampleRecovered,
+                               Future<Void> startRestore) {
+	std::vector<Standalone<VectorRef<KeyValueRef>>> byteSampleSample;
+	co_await applyByteSampleResult(
+	    data, storage, persistByteSampleSampleKeys.begin, persistByteSampleSampleKeys.end, &byteSampleSample);
 	byteSampleSampleRecovered.send(Void());
-	wait(startRestore);
-	wait(delay(SERVER_KNOBS->BYTE_SAMPLE_START_DELAY));
+	co_await startRestore;
+	co_await delay(SERVER_KNOBS->BYTE_SAMPLE_START_DELAY);
 
 	size_t bytes_per_fetch = 0;
 	// Since the expected size also includes (as of now) the space overhead of the container, we calculate our own
@@ -10969,7 +10993,7 @@ ACTOR Future<Void> restoreByteSample(StorageServer* data,
 	}
 	bytes_per_fetch = (bytes_per_fetch / SERVER_KNOBS->BYTE_SAMPLE_LOAD_PARALLELISM) + 1;
 
-	state std::vector<Future<Void>> sampleRanges;
+	std::vector<Future<Void>> sampleRanges;
 	int accumulatedSize = 0;
 	Key lastStart =
 	    persistByteSampleKeys.begin; // make sure the first range starts at the absolute beginning of the byte sample
@@ -10987,50 +11011,45 @@ ACTOR Future<Void> restoreByteSample(StorageServer* data,
 	// make sure that the last range goes all the way to the end of the byte sample
 	sampleRanges.push_back(applyByteSampleResult(data, storage, lastStart, persistByteSampleKeys.end));
 
-	wait(waitForAll(sampleRanges));
+	co_await waitForAll(sampleRanges);
 	TraceEvent("RecoveredByteSampleChunkedRead", data->thisServerID).detail("Ranges", sampleRanges.size());
 
 	if (BUGGIFY)
-		wait(delay(deterministicRandom()->random01() * 10.0));
-
-	return Void();
-}
-
-ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* storage) {
-	state Future<Optional<Value>> fFormat = storage->readValue(persistFormat.key);
-	state Future<Optional<Value>> fID = storage->readValue(persistID);
-	state Future<Optional<Value>> ftssPairID = storage->readValue(persistTssPairID);
-	state Future<Optional<Value>> fssPairID = storage->readValue(persistSSPairID);
-	state Future<Optional<Value>> fTssQuarantine = storage->readValue(persistTssQuarantine);
-	state Future<Optional<Value>> fVersion = storage->readValue(persistVersion);
-	state Future<Optional<Value>> fLogProtocol = storage->readValue(persistLogProtocol);
-	state Future<Optional<Value>> fPrimaryLocality = storage->readValue(persistPrimaryLocality);
-	state Future<RangeResult> fShardAssigned = storage->readRange(persistShardAssignedKeys);
-	state Future<RangeResult> fShardAvailable = storage->readRange(persistShardAvailableKeys);
-	state Future<RangeResult> fPendingCheckpoints = storage->readRange(persistPendingCheckpointKeys);
-	state Future<RangeResult> fCheckpoints = storage->readRange(persistCheckpointKeys);
-	state Future<RangeResult> fMoveInShards = storage->readRange(persistMoveInShardsKeyRange());
-	state Future<RangeResult> fStorageShards = storage->readRange(persistStorageServerShardKeys);
-	state Future<RangeResult> fAccumulativeChecksum = storage->readRange(persistAccumulativeChecksumKeys);
-	state Future<RangeResult> fBulkLoadTask = storage->readRange(persistBulkLoadTaskKeys);
-
-	state Promise<Void> byteSampleSampleRecovered;
-	state Promise<Void> startByteSampleRestore;
+		co_await delay(deterministicRandom()->random01() * 10.0);
+}
+
+Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* storage) {
+	Future<Optional<Value>> fFormat = storage->readValue(persistFormat.key);
+	Future<Optional<Value>> fID = storage->readValue(persistID);
+	Future<Optional<Value>> ftssPairID = storage->readValue(persistTssPairID);
+	Future<Optional<Value>> fssPairID = storage->readValue(persistSSPairID);
+	Future<Optional<Value>> fTssQuarantine = storage->readValue(persistTssQuarantine);
+	Future<Optional<Value>> fVersion = storage->readValue(persistVersion);
+	Future<Optional<Value>> fLogProtocol = storage->readValue(persistLogProtocol);
+	Future<Optional<Value>> fPrimaryLocality = storage->readValue(persistPrimaryLocality);
+	Future<RangeResult> fShardAssigned = storage->readRange(persistShardAssignedKeys);
+	Future<RangeResult> fShardAvailable = storage->readRange(persistShardAvailableKeys);
+	Future<RangeResult> fPendingCheckpoints = storage->readRange(persistPendingCheckpointKeys);
+	Future<RangeResult> fCheckpoints = storage->readRange(persistCheckpointKeys);
+	Future<RangeResult> fMoveInShards = storage->readRange(persistMoveInShardsKeyRange());
+	Future<RangeResult> fStorageShards = storage->readRange(persistStorageServerShardKeys);
+	Future<RangeResult> fAccumulativeChecksum = storage->readRange(persistAccumulativeChecksumKeys);
+	Future<RangeResult> fBulkLoadTask = storage->readRange(persistBulkLoadTaskKeys);
+
+	Promise<Void> byteSampleSampleRecovered;
+	Promise<Void> startByteSampleRestore;
 	data->byteSampleRecovery =
 	    restoreByteSample(data, storage, byteSampleSampleRecovered, startByteSampleRestore.getFuture());
 
 	TraceEvent("ReadingDurableState", data->thisServerID).log();
-	wait(waitForAll(
-	    std::vector{ fFormat, fID, ftssPairID, fssPairID, fTssQuarantine, fVersion, fLogProtocol, fPrimaryLocality }));
-	wait(waitForAll(std::vector{ fShardAssigned,
-	                             fShardAvailable,
-	                             fPendingCheckpoints,
-	                             fCheckpoints,
-	                             fMoveInShards,
-	                             fStorageShards,
-	                             fAccumulativeChecksum,
-	                             fBulkLoadTask }));
-	wait(byteSampleSampleRecovered.getFuture());
+	std::vector<Future<Optional<Value>>> pointFutures = { fFormat,        fID,      ftssPairID,   fssPairID,
+		                                                  fTssQuarantine, fVersion, fLogProtocol, fPrimaryLocality };
+	co_await waitForAll(pointFutures);
+	std::vector<Future<RangeResult>> rangeFutures = { fShardAssigned,        fShardAvailable, fPendingCheckpoints,
+		                                              fCheckpoints,          fMoveInShards,   fStorageShards,
+		                                              fAccumulativeChecksum, fBulkLoadTask };
+	co_await waitForAll(rangeFutures);
+	co_await byteSampleSampleRecovered.getFuture();
 	TraceEvent("RestoringDurableState", data->thisServerID).log();
 
 	if (!fFormat.get().present()) {
@@ -11040,7 +11059,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		storage->dispose();
 		data->thisServerID = UID();
 		data->sk = Key();
-		return false;
+		co_return false;
 	} else {
 		TraceEvent(SevVerbose, "RestoringStorageServerWithPhysicalShards", data->thisServerID).log();
 		data->shardAware = fFormat.get().get() == persistShardAwareFormat.value;
@@ -11086,35 +11105,35 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		data->bytesRestored += fPrimaryLocality.get().expectedSize();
 	}
 
-	state Version version = BinaryReader::fromStringRef<Version>(fVersion.get().get(), Unversioned());
+	Version version = BinaryReader::fromStringRef<Version>(fVersion.get().get(), Unversioned());
 	debug_checkRestoredVersion(data->thisServerID, version, "StorageServer");
 	data->setInitialVersion(version);
 	data->bytesRestored += fVersion.get().expectedSize();
 
 	TraceEvent(SevInfo, "StorageServerRestoreVersion", data->thisServerID).detail("Version", version);
 
-	state RangeResult pendingCheckpoints = fPendingCheckpoints.get();
-	state int pCLoc;
+	RangeResult pendingCheckpoints = fPendingCheckpoints.get();
+	int pCLoc{ 0 };
 	for (pCLoc = 0; pCLoc < pendingCheckpoints.size(); ++pCLoc) {
 		CheckpointMetaData metaData = decodeCheckpointValue(pendingCheckpoints[pCLoc].value);
 		data->pendingCheckpoints[metaData.version].push_back(metaData);
-		wait(yield());
+		co_await yield();
 	}
 
-	state RangeResult checkpoints = fCheckpoints.get();
-	state int cLoc;
+	RangeResult checkpoints = fCheckpoints.get();
+	int cLoc{ 0 };
 	for (cLoc = 0; cLoc < checkpoints.size(); ++cLoc) {
 		CheckpointMetaData metaData = decodeCheckpointValue(checkpoints[cLoc].value);
 		data->checkpoints[metaData.checkpointID] = metaData;
 		if (metaData.getState() == CheckpointMetaData::Deleting) {
 			data->actors.add(deleteCheckpointQ(data, version, metaData));
 		}
-		wait(yield());
+		co_await yield();
 	}
 
-	state RangeResult available = fShardAvailable.get();
+	RangeResult available = fShardAvailable.get();
 	data->bytesRestored += available.logicalSize();
-	state int availableLoc;
+	int availableLoc{ 0 };
 	for (availableLoc = 0; availableLoc < available.size(); availableLoc++) {
 		KeyRangeRef keys(available[availableLoc].key.removePrefix(persistShardAvailableKeys.begin),
 		                 availableLoc + 1 == available.size()
@@ -11126,7 +11145,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		/*if(nowAvailable)
 		TraceEvent("AvailableShard", data->thisServerID).detail("RangeBegin", keys.begin).detail("RangeEnd", keys.end);*/
 		data->newestAvailableVersion.insert(keys, nowAvailable ? latestVersion : invalidVersion);
-		wait(yield());
+		co_await yield();
 	}
 
 	// Restore acs validator from persisted disk
@@ -11141,9 +11160,9 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 		}
 	}
 
-	state KeyRangeMap<Optional<UID>> bulkLoadTaskRangeMap; // store dataMoveId on ranges with active bulkload tasks
+	KeyRangeMap<Optional<UID>> bulkLoadTaskRangeMap; // store dataMoveId on ranges with active bulkload tasks
 	bulkLoadTaskRangeMap.insert(allKeys, Optional<UID>());
-	state RangeResult bulkLoadTasks = fBulkLoadTask.get();
+	RangeResult bulkLoadTasks = fBulkLoadTask.get();
 	TraceEvent(SevDebug, "SSRecoveryBulkLoadTasksRead", data->thisServerID)
 	    .detail("TaskCount", bulkLoadTasks.size())
 	    .detail("TasksSize", bulkLoadTasks.expectedSize());
@@ -11174,14 +11193,14 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	// BulkLoadTaskRangeMap range boundary is aligned to the shard assignment boundary, because we persist the bulkload
 	// task metadata and the shard assignment metadata at the same version with the same shard boundary.
 
-	state RangeResult assigned = fShardAssigned.get();
+	RangeResult assigned = fShardAssigned.get();
 	data->bytesRestored += assigned.logicalSize();
 	data->bytesRestored += fStorageShards.get().logicalSize();
 	data->bytesRestored += fMoveInShards.get().logicalSize();
-	state int assignedLoc;
+	int assignedLoc{ 0 };
 	if (data->shardAware) {
 		// TODO(psm): Avoid copying RangeResult around.
-		wait(restoreShards(data, version, fStorageShards.get(), fMoveInShards.get(), assigned, available));
+		co_await restoreShards(data, version, fStorageShards.get(), fMoveInShards.get(), assigned, available);
 	} else {
 		for (assignedLoc = 0; assignedLoc < assigned.size(); assignedLoc++) {
 			KeyRangeRef keys(assigned[assignedLoc].key.removePrefix(persistShardAssignedKeys.begin),
@@ -11279,7 +11298,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 
 			if (!nowAssigned)
 				ASSERT(data->newestAvailableVersion.allEqual(keys, invalidVersion));
-			wait(yield());
+			co_await yield();
 		}
 	}
 
@@ -11291,7 +11310,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	// know" or "isn't really sure there is a good answer" or "author
 	// doesn't know what they are doing".  It looks bad any way you
 	// look at it.
-	wait(delay(0.0001));
+	co_await delay(0.0001);
 
 	if (!data->shardAware) {
 		// Erase data which isn't available (it is from some fetch at a later version)
@@ -11313,7 +11332,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 	validate(data, true);
 	startByteSampleRestore.send(Void());
 
-	return true;
+	co_return true;
 }
 
 Future<bool> StorageServerDisk::restoreDurableState() {
@@ -11346,39 +11365,6 @@ Future<bool> StorageServerDisk::restoreDurableState() {
 //   * probability: probability we select a key-value pair of this size, at random.
 //   * sampledSize: always |key + value| / probability
 //                  represents the amount of key-value space covered by that key.
-ByteSampleInfo isKeyValueInSample(const KeyRef key, int64_t totalKvSize) {
-	ASSERT(totalKvSize >= key.size());
-	ByteSampleInfo info;
-
-	// Pathological case: key size == value size == 0
-	//
-	// It's probability of getting chosen is 0, so let's skip the
-	// computation and avoid dividing by zero.
-	if (totalKvSize == 0) {
-		info.size = 0;
-		info.probability = 0.0;
-		info.inSample = false;
-		info.sampledSize = 0;
-		return info;
-	}
-
-	info.size = totalKvSize;
-
-	uint32_t a = 0;
-	uint32_t b = 0;
-	hashlittle2(key.begin(), key.size(), &a, &b);
-
-	info.probability =
-	    (double)info.size / (key.size() + SERVER_KNOBS->BYTE_SAMPLING_OVERHEAD) / SERVER_KNOBS->BYTE_SAMPLING_FACTOR;
-	// MIN_BYTE_SAMPLING_PROBABILITY is 0.99 only for testing
-	// MIN_BYTE_SAMPLING_PROBABILITY is 0 for other cases
-	info.probability = std::clamp(info.probability, SERVER_KNOBS->MIN_BYTE_SAMPLING_PROBABILITY, 1.0);
-	info.inSample = a / ((1 << 30) * 4.0) < info.probability;
-	info.sampledSize = info.size / info.probability;
-
-	return info;
-}
-
 void StorageServer::addMutationToMutationLogOrStorage(Version ver, MutationRef m) {
 	if (ver != invalidVersion) {
 		addMutationToMutationLog(addVersionToMutationLog(ver), m);
@@ -11473,111 +11459,6 @@ void StorageServer::byteSampleApplyClear(KeyRangeRef range, Version ver) {
 	}
 }
 
-ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest req, Future<Void> timeout) {
-	state PromiseStream<StorageMetrics> change;
-	state StorageMetrics metrics = self->getMetrics(req.keys);
-	state Error error = success();
-	state bool timedout = false;
-
-	// state UID metricReqId = deterministicRandom()->randomUniqueID();
-	DisabledTraceEvent(SevDebug, "WaitMetrics", metricReqId)
-	    .detail("Keys", req.keys)
-	    .detail("Metrics", metrics.toString())
-	    .detail("ReqMin", req.min.toString())
-	    .detail("ReqMax", req.max.toString());
-	if (!req.min.allLessOrEqual(metrics) || !metrics.allLessOrEqual(req.max)) {
-		CODE_PROBE(true, "ShardWaitMetrics return case 1 (quickly)");
-		req.reply.send(metrics);
-		return Void();
-	}
-
-	{
-		auto rs = self->waitMetricsMap.modify(req.keys);
-		for (auto r = rs.begin(); r != rs.end(); ++r)
-			r->value().push_back(change);
-		loop {
-			try {
-				choose {
-					when(StorageMetrics c = waitNext(change.getFuture())) {
-						metrics += c;
-
-						// SOMEDAY: validation! The changes here are possibly partial changes (we receive multiple
-						// messages per
-						//  update to our requested range). This means that the validation would have to occur after
-						//  all the messages for one clear or set have been dispatched.
-
-						/*StorageMetrics m = getMetrics( data, req.keys );
-						bool b = ( m.bytes != metrics.bytes || m.bytesWrittenPerKSecond !=
-						metrics.bytesWrittenPerKSecond
-						|| m.iosPerKSecond != metrics.iosPerKSecond ); if (b) { printf("keys: '%s' - '%s' @%p\n",
-						printable(req.keys.begin).c_str(), printable(req.keys.end).c_str(), this);
-						printf("waitMetrics: desync %d (%lld %lld %lld) != (%lld %lld %lld); +(%lld %lld %lld)\n",
-						b, m.bytes, m.bytesWrittenPerKSecond, m.iosPerKSecond, metrics.bytes,
-						metrics.bytesWrittenPerKSecond, metrics.iosPerKSecond, c.bytes, c.bytesWrittenPerKSecond,
-						c.iosPerKSecond);
-
-						}*/
-					}
-					when(wait(timeout)) {
-						timedout = true;
-					}
-				}
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw; // This is only cancelled when the main loop had exited...no need in this case to clean
-					       // up self
-				error = e;
-				break;
-			}
-
-			if (timedout) {
-				CODE_PROBE(true, "ShardWaitMetrics return on timeout");
-				// FIXME: instead of using random chance, send wrong_shard_server when the call in from
-				// waitMetricsMultiple (requires additional information in the request)
-				if (deterministicRandom()->random01() < SERVER_KNOBS->WAIT_METRICS_WRONG_SHARD_CHANCE) {
-					req.reply.sendError(wrong_shard_server());
-				} else {
-					req.reply.send(metrics);
-				}
-				break;
-			}
-
-			if (!req.min.allLessOrEqual(metrics) || !metrics.allLessOrEqual(req.max)) {
-				CODE_PROBE(true, "ShardWaitMetrics return case 2 (delayed)");
-				req.reply.send(metrics);
-				break;
-			}
-		}
-
-		wait(delay(0)); // prevent iterator invalidation of functions sending changes
-	}
-	// fmt::print("PopWaitMetricsMap {}\n", req.keys.toString());
-	auto rs = self->waitMetricsMap.modify(req.keys);
-	for (auto i = rs.begin(); i != rs.end(); ++i) {
-		auto& x = i->value();
-		for (int j = 0; j < x.size(); j++) {
-			if (x[j] == change) {
-				swapAndPop(&x, j);
-				break;
-			}
-		}
-	}
-	self->waitMetricsMap.coalesce(req.keys);
-
-	if (error.code() != error_code_success) {
-		if (error.code() != error_code_wrong_shard_server)
-			throw error;
-		CODE_PROBE(true, "ShardWaitMetrics delayed wrong_shard_server()");
-		req.reply.sendError(error);
-	}
-
-	return Void();
-}
-
-Future<Void> StorageServerMetrics::waitMetrics(WaitMetricsRequest req, Future<Void> delay) {
-	return ::waitMetrics(this, req, delay);
-}
-
 #ifndef __INTEL_COMPILER
 #pragma endregion
 #endif
@@ -11587,22 +11468,20 @@ Future<Void> StorageServerMetrics::waitMetrics(WaitMetricsRequest req, Future<Vo
 #pragma region Core
 #endif
 
-ACTOR Future<Void> waitMetricsForReal_internal(StorageServer* self, WaitMetricsRequest req) {
+Future<Void> waitMetricsForReal_internal(StorageServer* self, WaitMetricsRequest req) {
 	if (!self->isReadable(req.keys)) {
 		self->sendErrorWithPenalty(req.reply, wrong_shard_server(), self->getPenalty());
 	} else {
-		wait(self->metrics.waitMetrics(req, delayJittered(SERVER_KNOBS->STORAGE_METRIC_TIMEOUT)));
+		co_await self->metrics.waitMetrics(req, delayJittered(SERVER_KNOBS->STORAGE_METRIC_TIMEOUT));
 	}
-	return Void();
 }
 
 Future<Void> StorageServer::waitMetricsForReal(const WaitMetricsRequest& req) {
 	return waitMetricsForReal_internal(this, req);
 }
 
-ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi) {
-
-	wait(self->byteSampleRecovery);
+Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi) {
+	co_await self->byteSampleRecovery;
 	TraceEvent("StorageServerRestoreDurableState", self->thisServerID).detail("RestoredBytes", self->bytesRestored);
 
 	// Logs all counters in `counters.cc` and reset the interval.
@@ -11656,8 +11535,7 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 		    }
 	    }));
 
-	wait(serveStorageMetricsRequests(self, ssi));
-	return Void();
+	co_await serveStorageMetricsRequests(self, ssi);
 }
 
 ACTOR Future<Void> logLongByteSampleRecovery(Future<Void> recovery) {
@@ -11671,16 +11549,17 @@ ACTOR Future<Void> logLongByteSampleRecovery(Future<Void> recovery) {
 	return Void();
 }
 
-ACTOR Future<Void> checkBehind(StorageServer* self) {
-	state int behindCount = 0;
-	loop {
-		wait(delay(SERVER_KNOBS->BEHIND_CHECK_DELAY));
-		state Transaction tr(self->cx);
-		loop {
+Future<Void> checkBehind(StorageServer* self) {
+	int behindCount = 0;
+	while (true) {
+		co_await delay(SERVER_KNOBS->BEHIND_CHECK_DELAY);
+		Transaction tr(self->cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Version readVersion = wait(tr.getRawReadVersion());
+				Version readVersion = co_await tr.getRawReadVersion();
 				if (readVersion > self->version.get() + SERVER_KNOBS->BEHIND_CHECK_VERSIONS) {
 					behindCount++;
 				} else {
@@ -11689,16 +11568,17 @@ ACTOR Future<Void> checkBehind(StorageServer* self) {
 				self->versionBehind = behindCount >= SERVER_KNOBS->BEHIND_CHECK_COUNT;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 }
 
-ACTOR Future<Void> serveGetValueRequests(StorageServer* self, FutureStream<GetValueRequest> getValue) {
+Future<Void> serveGetValueRequests(StorageServer* self, FutureStream<GetValueRequest> getValue) {
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetValue;
-	loop {
-		GetValueRequest req = waitNext(getValue);
+	while (true) {
+		GetValueRequest req = co_await getValue;
 		// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
 		// downgrade before doing real work
 		if (req.options.present() && req.options.get().debugID.present())
@@ -11713,10 +11593,10 @@ ACTOR Future<Void> serveGetValueRequests(StorageServer* self, FutureStream<GetVa
 	}
 }
 
-ACTOR Future<Void> serveGetKeyValuesRequests(StorageServer* self, FutureStream<GetKeyValuesRequest> getKeyValues) {
+Future<Void> serveGetKeyValuesRequests(StorageServer* self, FutureStream<GetKeyValuesRequest> getKeyValues) {
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKeyValues;
-	loop {
-		GetKeyValuesRequest req = waitNext(getKeyValues);
+	while (true) {
+		GetKeyValuesRequest req = co_await getKeyValues;
 
 		// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
 		// downgrade before doing real work
@@ -11724,12 +11604,12 @@ ACTOR Future<Void> serveGetKeyValuesRequests(StorageServer* self, FutureStream<G
 	}
 }
 
-ACTOR Future<Void> serveGetMappedKeyValuesRequests(StorageServer* self,
-                                                   FutureStream<GetMappedKeyValuesRequest> getMappedKeyValues) {
+Future<Void> serveGetMappedKeyValuesRequests(StorageServer* self,
+                                             FutureStream<GetMappedKeyValuesRequest> getMappedKeyValues) {
 	// TODO: Is it fine to keep TransactionLineage::Operation::GetKeyValues here?
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKeyValues;
-	loop {
-		GetMappedKeyValuesRequest req = waitNext(getMappedKeyValues);
+	while (true) {
+		GetMappedKeyValuesRequest req = co_await getMappedKeyValues;
 
 		// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
 		// downgrade before doing real work
@@ -11737,10 +11617,10 @@ ACTOR Future<Void> serveGetMappedKeyValuesRequests(StorageServer* self,
 	}
 }
 
-ACTOR Future<Void> serveGetKeyValuesStreamRequests(StorageServer* self,
-                                                   FutureStream<GetKeyValuesStreamRequest> getKeyValuesStream) {
-	loop {
-		GetKeyValuesStreamRequest req = waitNext(getKeyValuesStream);
+Future<Void> serveGetKeyValuesStreamRequests(StorageServer* self,
+                                             FutureStream<GetKeyValuesStreamRequest> getKeyValuesStream) {
+	while (true) {
+		GetKeyValuesStreamRequest req = co_await getKeyValuesStream;
 		// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
 		// downgrade before doing real work
 		// FIXME: add readGuard again
@@ -11748,41 +11628,40 @@ ACTOR Future<Void> serveGetKeyValuesStreamRequests(StorageServer* self,
 	}
 }
 
-ACTOR Future<Void> serveGetKeyRequests(StorageServer* self, FutureStream<GetKeyRequest> getKey) {
+Future<Void> serveGetKeyRequests(StorageServer* self, FutureStream<GetKeyRequest> getKey) {
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKey;
-	loop {
-		GetKeyRequest req = waitNext(getKey);
+	while (true) {
+		GetKeyRequest req = co_await getKey;
 		// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
 		// downgrade before doing real work
 		self->actors.add(self->readGuard(req, getKeyQ));
 	}
 }
 
-ACTOR Future<Void> watchValueWaitForVersion(StorageServer* self,
-                                            WatchValueRequest req,
-                                            PromiseStream<WatchValueRequest> stream) {
-	state Span span("SS:watchValueWaitForVersion"_loc, req.spanContext);
+Future<Void> watchValueWaitForVersion(StorageServer* self,
+                                      WatchValueRequest req,
+                                      PromiseStream<WatchValueRequest> stream) {
+	Span span("SS:watchValueWaitForVersion"_loc, req.spanContext);
 
 	getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
 	try {
-		wait(success(waitForVersionNoTooOld(self, req.version)));
+		co_await waitForVersionNoTooOld(self, req.version);
 		stream.send(req);
 	} catch (Error& e) {
 		if (!canReplyWith(e))
 			throw e;
 		self->sendErrorWithPenalty(req.reply, e, self->getPenalty());
 	}
-	return Void();
 }
 
-ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream<WatchValueRequest> stream) {
-	loop {
+Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream<WatchValueRequest> stream) {
+	while (true) {
 		getCurrentLineage()->modify(&TransactionLineage::txID) = UID();
-		state WatchValueRequest req = waitNext(stream);
-		state Reference<ServerWatchMetadata> metadata = self->getWatchMetadata(req.key.contents());
-		state Span span("SS:serveWatchValueRequestsImpl"_loc, req.spanContext);
+		WatchValueRequest req = co_await stream;
+		Reference<ServerWatchMetadata> metadata = self->getWatchMetadata(req.key.contents());
+		Span span("SS:serveWatchValueRequestsImpl"_loc, req.spanContext);
 		getCurrentLineage()->modify(&TransactionLineage::txID) = req.spanContext.traceID;
-		state ReadOptions options;
+		ReadOptions options;
 
 		// case 1: no watch set for the current key
 		if (!metadata.isValid()) {
@@ -11833,15 +11712,15 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 		// case 5: watch value differs but their versions are the same (rare case) so check with the SS
 		else {
 			CODE_PROBE(true, "watch version in the map is the same but value is different (case 5)");
-			loop {
+			while (true) {
 				try {
-					state Version latest = self->version.get();
+					Version latest = self->version.get();
 					options.debugID = metadata->debugID;
 
 					GetValueRequest getReq(
 					    span.context, metadata->key, latest, metadata->tags, options, VersionVector());
-					state Future<Void> getValue = getValueQ(self, getReq);
-					GetValueReply reply = wait(getReq.reply.getFuture());
+					Future<Void> getValue = getValueQ(self, getReq);
+					GetValueReply reply = co_await getReq.reply.getFuture();
 					metadata = self->getWatchMetadata(req.key.contents());
 
 					if (metadata.isValid() && reply.value != metadata->value) { // valSS != valMap
@@ -11877,13 +11756,13 @@ ACTOR Future<Void> serveWatchValueRequestsImpl(StorageServer* self, FutureStream
 	}
 }
 
-ACTOR Future<Void> serveWatchValueRequests(StorageServer* self, FutureStream<WatchValueRequest> watchValue) {
-	state PromiseStream<WatchValueRequest> stream;
+Future<Void> serveWatchValueRequests(StorageServer* self, FutureStream<WatchValueRequest> watchValue) {
+	PromiseStream<WatchValueRequest> stream;
 	getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::WatchValue;
 	self->actors.add(serveWatchValueRequestsImpl(self, stream.getFuture()));
 
-	loop {
-		WatchValueRequest req = waitNext(watchValue);
+	while (true) {
+		WatchValueRequest req = co_await watchValue;
 		// TODO: fast load balancing?
 		if (self->shouldRead(req)) {
 			self->actors.add(watchValueWaitForVersion(self, req, stream));
@@ -11894,53 +11773,52 @@ ACTOR Future<Void> serveWatchValueRequests(StorageServer* self, FutureStream<Wat
 // Change feed related message handlers that exist but do nothing.
 // The reason for these existing is that we do not want to change the StorageServerInterface
 // to remove change feed related interfaces.  Figure out how to do that later.
-ACTOR Future<Void> serveChangeFeedStreamRequests(StorageServer* self,
-                                                 FutureStream<ChangeFeedStreamRequest> changeFeedStream) {
-	loop {
-		ChangeFeedStreamRequest req = waitNext(changeFeedStream);
+Future<Void> serveChangeFeedStreamRequests(StorageServer* self,
+                                           FutureStream<ChangeFeedStreamRequest> changeFeedStream) {
+	while (true) {
+		ChangeFeedStreamRequest req = co_await changeFeedStream;
 		TraceEvent(SevWarn, "serveChangeFeedStreamRequests").detail("Unexpected", "Invocation");
 	}
 }
 
-ACTOR Future<Void> serveOverlappingChangeFeedsRequests(
-    StorageServer* self,
-    FutureStream<OverlappingChangeFeedsRequest> overlappingChangeFeeds) {
-	loop {
-		OverlappingChangeFeedsRequest req = waitNext(overlappingChangeFeeds);
+Future<Void> serveOverlappingChangeFeedsRequests(StorageServer* self,
+                                                 FutureStream<OverlappingChangeFeedsRequest> overlappingChangeFeeds) {
+	while (true) {
+		OverlappingChangeFeedsRequest req = co_await overlappingChangeFeeds;
 		TraceEvent(SevWarn, "serveOverlappingChangeFeedsRequests").detail("Unexpected", "Invocation");
 	}
 }
 
-ACTOR Future<Void> serveChangeFeedPopRequests(StorageServer* self, FutureStream<ChangeFeedPopRequest> changeFeedPops) {
-	loop {
-		ChangeFeedPopRequest req = waitNext(changeFeedPops);
+Future<Void> serveChangeFeedPopRequests(StorageServer* self, FutureStream<ChangeFeedPopRequest> changeFeedPops) {
+	while (true) {
+		ChangeFeedPopRequest req = co_await changeFeedPops;
 		TraceEvent(SevWarn, "serveChangeFeedPopRequests").detail("Unexpected", "Invocation");
 	}
 }
 
-ACTOR Future<Void> serveChangeFeedVersionUpdateRequests(
+Future<Void> serveChangeFeedVersionUpdateRequests(
     StorageServer* self,
     FutureStream<ChangeFeedVersionUpdateRequest> changeFeedVersionUpdate) {
-	loop {
-		ChangeFeedVersionUpdateRequest req = waitNext(changeFeedVersionUpdate);
+	while (true) {
+		ChangeFeedVersionUpdateRequest req = co_await changeFeedVersionUpdate;
 		TraceEvent(SevWarn, "serveChangeFeedVersionUpdateRequests").detail("Unexpected", "Invocation");
 	}
 }
 
-ACTOR Future<Void> storageEngineConsistencyCheck(StorageServer* self) {
+Future<Void> storageEngineConsistencyCheck(StorageServer* self) {
 	if (SERVER_KNOBS->STORAGE_SHARD_CONSISTENCY_CHECK_INTERVAL <= 0.0) {
-		return Void();
+		co_return;
 	}
 
 	if (self->storage.getKeyValueStoreType() != KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
-		return Void();
+		co_return;
 	}
 
-	loop {
-		wait(delay(SERVER_KNOBS->STORAGE_SHARD_CONSISTENCY_CHECK_INTERVAL));
+	while (true) {
+		co_await delay(SERVER_KNOBS->STORAGE_SHARD_CONSISTENCY_CHECK_INTERVAL);
 		// Only validate when storage server and storage engine are expected to have the same shard mapping.
 		while (self->pendingAddRanges.size() > 0 || self->pendingRemoveRanges.size() > 0) {
-			wait(delay(5.0));
+			co_await delay(5.0);
 		}
 
 		CoalescedKeyRangeMap<std::string> currentShards;
@@ -11999,13 +11877,13 @@ ACTOR Future<Void> storageEngineConsistencyCheck(StorageServer* self) {
 	}
 }
 
-ACTOR Future<Void> reportStorageServerState(StorageServer* self) {
+Future<Void> reportStorageServerState(StorageServer* self) {
 	if (!SERVER_KNOBS->REPORT_DD_METRICS) {
-		return Void();
+		co_return;
 	}
 
-	loop {
-		wait(delay(SERVER_KNOBS->DD_METRICS_REPORT_INTERVAL));
+	while (true) {
+		co_await delay(SERVER_KNOBS->DD_METRICS_REPORT_INTERVAL);
 
 		const auto numRunningFetchKeys = self->currentRunningFetchKeys.numRunning();
 		if (numRunningFetchKeys == 0) {
@@ -12035,7 +11913,7 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
 	state double lastLoopTopTime = now();
 	state Future<Void> dbInfoChange = Void();
 	state Future<Void> checkLastUpdate = Void();
-	state Future<Void> updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL);
+	state Future<Void> updateProcessStatsTimer = delay(SERVER_KNOBS->STORAGE_UPDATE_PROCESS_STATS_INTERVAL);
 
 	self->actors.add(updateStorage(self));
 	self->actors.add(waitFailureServer(ssi.waitFailure.getFuture()));
@@ -12058,8 +11936,8 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
 	self->actors.add(storageEngineConsistencyCheck(self));
 
 	self->transactionTagCounter.startNewInterval();
-	self->actors.add(
-	    recurring([&]() { self->transactionTagCounter.startNewInterval(); }, SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
+	self->actors.add(recurring(std::bind_front(&TransactionTagCounter::startNewInterval, &self->transactionTagCounter),
+	                           SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL));
 
 	self->coreStarted.send(Void());
 
@@ -12088,7 +11966,7 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
 				CODE_PROBE(self->logSystem, "shardServer dbInfo changed");
 				dbInfoChange = self->db->onChange();
 				if (self->db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
-					self->logSystem = ILogSystem::fromServerDBInfo(self->thisServerID, self->db->get());
+					self->logSystem = makeLogSystemFromServerDBInfo(self->thisServerID, self->db->get());
 					if (self->logSystem) {
 						if (self->db->get().logSystemConfig.recoveredAt.present()) {
 							self->poppedAllAfter = self->db->get().logSystemConfig.recoveredAt.get();
@@ -12215,7 +12093,7 @@ ACTOR Future<Void> storageServerCore(StorageServer* self, StorageServerInterface
 			}
 			when(wait(updateProcessStatsTimer)) {
 				updateProcessStats(self);
-				updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL);
+				updateProcessStatsTimer = delay(SERVER_KNOBS->STORAGE_UPDATE_PROCESS_STATS_INTERVAL);
 			}
 			when(GetHotShardsRequest req = waitNext(ssi.getHotShards.getFuture())) {
 				struct ComparePair {
@@ -12282,47 +12160,52 @@ bool storageServerTerminated(StorageServer& self, IKeyValueStore* persistentData
 	}
 
 	if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed ||
-	    e.code() == error_code_file_not_found || e.code() == error_code_actor_cancelled ||
-	    e.code() == error_code_remote_kvs_cancelled) {
+	    e.code() == error_code_file_not_found || e.code() == error_code_actor_cancelled) {
 		TraceEvent("StorageServerTerminated", self.thisServerID).errorUnsuppressed(e);
 		return true;
 	} else
 		return false;
 }
 
-ACTOR Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<IClusterConnectionRecord> connRecord, UID id) {
+Future<Void> memoryStoreRecover(IKeyValueStore* store, Reference<IClusterConnectionRecord> connRecord, UID id) {
 	if (store->getType() != KeyValueStoreType::MEMORY || connRecord.getPtr() == nullptr) {
-		return Never();
+		co_await Future<Void>(Never());
+		co_return;
 	}
 
 	// create a temp client connect to DB
 	Database cx = Database::createDatabase(connRecord, ApiVersion::LATEST_VERSION);
 
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-	state int noCanRemoveCount = 0;
-	loop {
-		try {
-			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-
-			state bool canRemove = wait(canRemoveStorageServer(tr, id));
-			if (!canRemove) {
-				CODE_PROBE(true,
-				           "it's possible that the caller had a transaction in flight that assigned keys to the "
-				           "server. Wait for it to reverse its mistake.");
-				wait(delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskPriority::UpdateStorage));
-				tr->reset();
-				TraceEvent("RemoveStorageServerRetrying")
-				    .detail("Count", noCanRemoveCount++)
-				    .detail("ServerID", id)
-				    .detail("CanRemove", canRemove);
-			} else {
-				return Void();
+	auto tr = makeReference<ReadYourWritesTransaction>(cx);
+	int noCanRemoveCount = 0;
+	while (true) {
+		{
+			Optional<Error> err;
+			try {
+				tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+
+				bool canRemove = co_await canRemoveStorageServer(tr, id);
+				if (!canRemove) {
+					CODE_PROBE(true,
+					           "it's possible that the caller had a transaction in flight that assigned keys to the "
+					           "server. Wait for it to reverse its mistake.");
+					co_await delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskPriority::UpdateStorage);
+					tr->reset();
+					TraceEvent("RemoveStorageServerRetrying")
+					    .detail("Count", noCanRemoveCount++)
+					    .detail("ServerID", id)
+					    .detail("CanRemove", canRemove);
+				} else {
+					co_return;
+				}
+			} catch (Error& e) {
+				err = e;
+			}
+			if (err.present()) {
+				co_await tr->onError(err.get());
+				TraceEvent("RemoveStorageServerRetrying").error(err.get());
 			}
-		} catch (Error& e) {
-			state Error err = e;
-			wait(tr->onError(e));
-			TraceEvent("RemoveStorageServerRetrying").error(err);
 		}
 	}
 }
@@ -12418,23 +12301,24 @@ ACTOR Future<Void> replaceInterface(StorageServer* self, StorageServerInterface
 	return Void();
 }
 
-ACTOR Future<Void> replaceTSSInterface(StorageServer* self, StorageServerInterface ssi) {
+Future<Void> replaceTSSInterface(StorageServer* self, StorageServerInterface ssi) {
 	// RYW for KeyBackedMap
-	state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
-	state KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
+	auto tr = makeReference<ReadYourWritesTransaction>(self->cx);
+	KeyBackedMap<UID, UID> tssMapDB = KeyBackedMap<UID, UID>(tssMappingKeys.begin);
 
 	ASSERT(ssi.isTss());
 
-	loop {
+	while (true) {
+		Error err;
 		try {
-			state Tag myTag;
+			Tag myTag;
 
 			tr->reset();
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-			Optional<Value> pairTagValue = wait(tr->get(serverTagKeyFor(self->tssPairID.get())));
+			Optional<Value> pairTagValue = co_await tr->get(serverTagKeyFor(self->tssPairID.get()));
 
 			if (!pairTagValue.present()) {
 				CODE_PROBE(true, "Race where tss was down, pair was removed, tss starts back up");
@@ -12452,50 +12336,41 @@ ACTOR Future<Void> replaceTSSInterface(StorageServer* self, StorageServerInterfa
 				tssMapDB.set(tr, self->tssPairID.get(), ssi.id());
 			}
 
-			wait(tr->commit());
+			co_await tr->commit();
 			self->tag = myTag;
-
 			break;
 		} catch (Error& e) {
-			wait(tr->onError(e));
+			err = e;
 		}
+		co_await tr->onError(err);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> storageInterfaceRegistration(StorageServer* self,
-                                                StorageServerInterface ssi,
-                                                Optional<Future<Void>> readyToAcceptRequests) {
-
+Future<Void> storageInterfaceRegistration(StorageServer* self,
+                                          StorageServerInterface ssi,
+                                          Optional<Future<Void>> readyToAcceptRequests) {
 	if (readyToAcceptRequests.present()) {
-		wait(readyToAcceptRequests.get());
+		co_await readyToAcceptRequests.get();
 		ssi.startAcceptingRequests();
 	} else {
 		ssi.stopAcceptingRequests();
 	}
 
-	try {
-		if (self->isTss()) {
-			wait(replaceTSSInterface(self, ssi));
-		} else {
-			wait(replaceInterface(self, ssi));
-		}
-	} catch (Error& e) {
-		throw;
+	if (self->isTss()) {
+		co_await replaceTSSInterface(self, ssi);
+	} else {
+		co_await replaceInterface(self, ssi);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> rocksdbLogCleaner(std::string folder) {
+Future<Void> rocksdbLogCleaner(std::string folder) {
 	std::replace(folder.begin(), folder.end(), '/', '_');
 	if (!folder.empty() && folder[0] == '_') {
 		folder.erase(0, 1);
 	}
 	try {
-		loop {
-			wait(delayJittered(SERVER_KNOBS->STORAGE_ROCKSDB_LOG_CLEAN_UP_DELAY));
+		while (true) {
+			co_await delayJittered(SERVER_KNOBS->STORAGE_ROCKSDB_LOG_CLEAN_UP_DELAY);
 			TraceEvent("CleanUpRocksDBLogs").detail("LogPrefix", folder);
 			auto logFiles = platform::listFiles(SERVER_KNOBS->LOG_DIRECTORY);
 			for (const auto& f : logFiles) {
@@ -12514,21 +12389,20 @@ ACTOR Future<Void> rocksdbLogCleaner(std::string folder) {
 			TraceEvent(SevError, "RocksDBLogCleanerError").errorUnsuppressed(e);
 		}
 	}
-	return Void();
 }
 
 // for creating a new storage server
-ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
-                                 StorageServerInterface ssi,
-                                 Tag seedTag,
-                                 Version startVersion,
-                                 Version tssSeedVersion,
-                                 ReplyPromise<InitializeStorageReply> recruitReply,
-                                 Reference<AsyncVar<ServerDBInfo> const> db,
-                                 std::string folder) {
-	state StorageServer self(persistentData, db, ssi);
+Future<Void> storageServer(IKeyValueStore* persistentData,
+                           StorageServerInterface ssi,
+                           Tag seedTag,
+                           Version startVersion,
+                           Version tssSeedVersion,
+                           ReplyPromise<InitializeStorageReply> recruitReply,
+                           Reference<AsyncVar<ServerDBInfo> const> db,
+                           std::string folder) {
+	StorageServer self(persistentData, db, ssi);
 	self.shardAware = persistentData->shardAware();
-	state Future<Void> ssCore;
+	Future<Void> ssCore;
 	self.initialClusterVersion = startVersion;
 	if (ssi.isTss()) {
 		self.setTssPair(ssi.tssPairID.get());
@@ -12549,78 +12423,82 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 	self.bulkLoadFolder = joinPath(self.folder, serverBulkLoadFolder);
 	self.actors.add(rocksdbLogCleaner(folder));
 
-	try {
-		wait(self.storage.init());
-		TraceEvent("StorageServerInitProgress", ssi.id())
-		    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-		    .detail("Step", "5.StorageInited");
-		wait(self.storage.commit());
-		TraceEvent("StorageServerInitProgress", ssi.id())
-		    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-		    .detail("Step", "6.StorageCommitted");
-		++self.counters.kvCommits;
+	{
+		Error err;
+		try {
+			co_await self.storage.init();
+			TraceEvent("StorageServerInitProgress", ssi.id())
+			    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
+			    .detail("Step", "5.StorageInited");
+			co_await self.storage.commit();
+			TraceEvent("StorageServerInitProgress", ssi.id())
+			    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
+			    .detail("Step", "6.StorageCommitted");
+			++self.counters.kvCommits;
 
-		platform::createDirectory(self.checkpointFolder);
-		platform::createDirectory(self.fetchedCheckpointFolder);
+			platform::createDirectory(self.checkpointFolder);
+			platform::createDirectory(self.fetchedCheckpointFolder);
 
-		clearFileFolder(self.bulkDumpFolder, self.thisServerID, /*ignoreError=*/false);
-		clearFileFolder(self.bulkLoadFolder, self.thisServerID, /*ignoreError=*/false);
+			clearFileFolder(self.bulkDumpFolder, self.thisServerID, /*ignoreError=*/false);
+			clearFileFolder(self.bulkLoadFolder, self.thisServerID, /*ignoreError=*/false);
 
-		if (seedTag == invalidTag) {
-			ssi.startAcceptingRequests();
-			self.registerInterfaceAcceptingRequests.send(Void());
+			if (seedTag == invalidTag) {
+				ssi.startAcceptingRequests();
+				self.registerInterfaceAcceptingRequests.send(Void());
 
-			// Might throw recruitment_failed in case of simultaneous master failure
-			std::pair<Version, Tag> verAndTag = wait(addStorageServer(self.cx, ssi));
-			TraceEvent("StorageServerInitProgress", ssi.id())
-			    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-			    .detail("Step", "8.StorageServerAdded");
+				// Might throw recruitment_failed in case of simultaneous master failure
+				std::pair<Version, Tag> verAndTag = co_await addStorageServer(self.cx, ssi);
+				TraceEvent("StorageServerInitProgress", ssi.id())
+				    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
+				    .detail("Step", "8.StorageServerAdded");
+
+				self.tag = verAndTag.second;
+				if (ssi.isTss()) {
+					self.setInitialVersion(tssSeedVersion);
+				} else {
+					self.setInitialVersion(verAndTag.first - 1);
+				}
 
-			self.tag = verAndTag.second;
-			if (ssi.isTss()) {
-				self.setInitialVersion(tssSeedVersion);
+				TraceEvent("StorageServerInitProgress", ssi.id())
+				    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
+				    // This is an intentionally useless detail to avoid renumbering things.
+				    .detail("Step", "9.SomeLinesOfCodeExecuted");
 			} else {
-				self.setInitialVersion(verAndTag.first - 1);
+				self.tag = seedTag;
 			}
 
+			self.storage.makeNewStorageServerDurable(self.shardAware);
+			co_await self.storage.commit();
 			TraceEvent("StorageServerInitProgress", ssi.id())
 			    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-			    // This is an intentionally useless detail to avoid renumbering things.
-			    .detail("Step", "9.SomeLinesOfCodeExecuted");
-		} else {
-			self.tag = seedTag;
-		}
+			    .detail("Step", "10.NewStorageServerDurable");
+			++self.counters.kvCommits;
 
-		self.storage.makeNewStorageServerDurable(self.shardAware);
-		wait(self.storage.commit());
-		TraceEvent("StorageServerInitProgress", ssi.id())
-		    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-		    .detail("Step", "10.NewStorageServerDurable");
-		++self.counters.kvCommits;
+			self.interfaceRegistered =
+			    storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture());
+			co_await delay(0);
 
-		self.interfaceRegistered =
-		    storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture());
-		wait(delay(0));
-
-		TraceEvent("StorageServerInit", ssi.id())
-		    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-		    .detail("Version", self.version.get())
-		    .detail("SeedTag", seedTag.toString())
-		    .detail("TssPair", ssi.isTss() ? ssi.tssPairID.get().toString() : "");
-		InitializeStorageReply rep;
-		rep.interf = ssi;
-		rep.addedVersion = self.version.get();
-		recruitReply.send(rep);
-		self.byteSampleRecovery = Void();
-		TraceEvent("StorageServerInitProgress", ssi.id())
-		    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
-		    .detail("Step", "11.RecruitReplied");
+			TraceEvent("StorageServerInit", ssi.id())
+			    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
+			    .detail("Version", self.version.get())
+			    .detail("SeedTag", seedTag.toString())
+			    .detail("TssPair", ssi.isTss() ? ssi.tssPairID.get().toString() : "");
+			InitializeStorageReply rep;
+			rep.interf = ssi;
+			rep.addedVersion = self.version.get();
+			recruitReply.send(rep);
+			self.byteSampleRecovery = Void();
+			TraceEvent("StorageServerInitProgress", ssi.id())
+			    .detail("EngineType", self.storage.getKeyValueStoreType().toString())
+			    .detail("Step", "11.RecruitReplied");
 
-		ssCore = storageServerCore(&self, ssi);
-		wait(ssCore);
+			ssCore = storageServerCore(&self, ssi);
+			co_await ssCore;
 
-		throw internal_error();
-	} catch (Error& e) {
+			throw internal_error();
+		} catch (Error& e) {
+			err = e;
+		}
 		// If we die with an error before replying to the recruitment request, send the error to the recruiter
 		// (ClusterController, and from there to the DataDistributionTeamCollection)
 		if (!recruitReply.isSet())
@@ -12633,29 +12511,28 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 
 		self.moveInShards.clear();
 
-		state Error err = e;
 		if (storageServerTerminated(self, persistentData, err)) {
 			ssCore.cancel();
 			self.actors = ActorCollection(false);
-			wait(delay(0));
-			return Void();
+			co_await delay(0);
+			co_return;
 		}
 		ssCore.cancel();
 		self.actors = ActorCollection(false);
-		wait(delay(0));
+		co_await delay(0);
 		throw err;
 	}
 }
 
 // for recovering an existing storage server
-ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
-                                 StorageServerInterface ssi,
-                                 Reference<AsyncVar<ServerDBInfo> const> db,
-                                 std::string folder,
-                                 Promise<Void> recovered,
-                                 Reference<IClusterConnectionRecord> connRecord) {
-	state StorageServer self(persistentData, db, ssi);
-	state Future<Void> ssCore;
+Future<Void> storageServer(IKeyValueStore* persistentData,
+                           StorageServerInterface ssi,
+                           Reference<AsyncVar<ServerDBInfo> const> db,
+                           std::string folder,
+                           Promise<Void> recovered,
+                           Reference<IClusterConnectionRecord> connRecord) {
+	StorageServer self(persistentData, db, ssi);
+	Future<Void> ssCore;
 	self.folder = folder;
 	self.checkpointFolder = joinPath(self.folder, serverCheckpointFolder);
 	self.fetchedCheckpointFolder = joinPath(self.folder, fetchedCheckpointFolder);
@@ -12675,28 +12552,30 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 	clearFileFolder(self.bulkLoadFolder, self.thisServerID, /*ignoreError=*/false);
 
 	self.actors.add(rocksdbLogCleaner(folder));
+	Error err;
 	try {
-		state double start = now();
+		double start = now();
 		TraceEvent("StorageServerRebootStart", self.thisServerID).log();
 
-		wait(self.storage.init());
-		choose {
-			// after a rollback there might be uncommitted changes.
-			// for memory storage engine type, wait until recovery is done before commit
-			when(wait(self.storage.commit())) {}
-
-			when(wait(memoryStoreRecover(persistentData, connRecord, self.thisServerID))) {
-				TraceEvent("DisposeStorageServer", self.thisServerID).log();
-				throw worker_removed();
-			}
+		co_await self.storage.init();
+		// after a rollback there might be uncommitted changes.
+		// for memory storage engine type, wait until recovery is done before commit
+		auto res =
+		    co_await race(self.storage.commit(), memoryStoreRecover(persistentData, connRecord, self.thisServerID));
+		if (res.index() == 1) {
+			TraceEvent("DisposeStorageServer", self.thisServerID).log();
+			throw worker_removed();
+		} else if (res.index() != 0) {
+			UNREACHABLE();
 		}
+
 		++self.counters.kvCommits;
 
-		bool ok = wait(self.storage.restoreDurableState());
+		bool ok = co_await self.storage.restoreDurableState();
 		if (!ok) {
 			if (recovered.canBeSet())
 				recovered.send(Void());
-			return Void();
+			co_return;
 		}
 		TraceEvent("SSTimeRestoreDurableState", self.thisServerID).detail("TimeTaken", now() - start);
 
@@ -12719,48 +12598,48 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
 		if (recovered.canBeSet())
 			recovered.send(Void());
 
-		state Future<Void> f = storageInterfaceRegistration(&self, ssi, {});
-		wait(delay(0));
-		ErrorOr<Void> e = wait(errorOr(f));
+		Future<Void> f = storageInterfaceRegistration(&self, ssi, {});
+		co_await delay(0);
+		ErrorOr<Void> e = co_await errorOr(f);
 		if (e.isError()) {
 			throw f.getError();
 		}
 
 		self.interfaceRegistered =
 		    storageInterfaceRegistration(&self, ssi, self.registerInterfaceAcceptingRequests.getFuture());
-		wait(delay(0));
+		co_await delay(0);
 
 		TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start);
 
 		ssCore = storageServerCore(&self, ssi);
-		wait(ssCore);
+		co_await ssCore;
 
 		throw internal_error();
 	} catch (Error& e) {
+		err = e;
+	}
 
-		self.ssLock->halt();
+	self.ssLock->halt();
 
-		if (self.byteSampleRecovery.isValid()) {
-			self.byteSampleRecovery.cancel();
-		}
+	if (self.byteSampleRecovery.isValid()) {
+		self.byteSampleRecovery.cancel();
+	}
 
-		if (recovered.canBeSet())
-			recovered.send(Void());
+	if (recovered.canBeSet())
+		recovered.send(Void());
 
-		// If the storage server dies while something that uses self is still on the stack,
-		// we want that actor to complete before we terminate and that memory goes out of scope
-		state Error err = e;
-		if (storageServerTerminated(self, persistentData, err)) {
-			ssCore.cancel();
-			self.actors = ActorCollection(false);
-			wait(delay(0));
-			return Void();
-		}
+	// If the storage server dies while something that uses self is still on the stack,
+	// we want that actor to complete before we terminate and that memory goes out of scope
+	if (storageServerTerminated(self, persistentData, err)) {
 		ssCore.cancel();
 		self.actors = ActorCollection(false);
-		wait(delay(0));
-		throw err;
+		co_await delay(0);
+		co_return;
 	}
+	ssCore.cancel();
+	self.actors = ActorCollection(false);
+	co_await delay(0);
+	throw err;
 }
 
 #ifndef __INTEL_COMPILER
@@ -12831,7 +12710,7 @@ void versionedMapTest() {
 	auto after = FastAllocator<ASIZE>::getTotalMemory();
 
 	int count = 0;
-	for (auto i = vm.atLatest().begin(); i != vm.atLatest().end(); ++i)
+	for (auto i = vm.latestBegin(); i != vm.latestEnd(); ++i)
 		++count;
 
 	printf("PTree node is %d bytes, allocated as %d bytes\n", NSIZE, ASIZE);
diff --git a/fdbserver/test.cpp b/fdbserver/test.cpp
deleted file mode 100644
index cabbda24b47..00000000000
--- a/fdbserver/test.cpp
+++ /dev/null
@@ -1,3324 +0,0 @@
-/*
- * tester.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <cinttypes>
-#include <filesystem>
-#include <fstream>
-#include <functional>
-#include <istream>
-#include <iterator>
-#include <map>
-#include <streambuf>
-#include <numeric>
-
-#include <fmt/ranges.h>
-#include <toml.hpp>
-
-#include "flow/ActorCollection.h"
-#include "flow/DeterministicRandom.h"
-#include "flow/Histogram.h"
-#include "flow/ProcessEvents.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbrpc/sim_validation.h"
-#include "fdbrpc/simulator.h"
-#include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
-#include "fdbclient/ClusterInterface.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/SystemData.h"
-#include "fdbclient/DataDistributionConfig.actor.h"
-#include "fdbserver/KnobProtectiveGroups.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbclient/MonitorLeader.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "flow/Platform.h"
-
-#include "flow/Trace.h"
-#include "flow/CoroUtils.h"
-
-WorkloadContext::WorkloadContext() {}
-
-WorkloadContext::WorkloadContext(const WorkloadContext& r)
-  : options(r.options), clientId(r.clientId), clientCount(r.clientCount), sharedRandomNumber(r.sharedRandomNumber),
-    dbInfo(r.dbInfo), ccr(r.ccr), rangesToCheck(r.rangesToCheck) {}
-
-WorkloadContext::~WorkloadContext() {}
-
-const char HEX_CHAR_LOOKUP[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
-
-void emplaceIndex(uint8_t* data, int offset, int64_t index) {
-	for (int i = 0; i < 16; i++) {
-		data[(15 - i) + offset] = HEX_CHAR_LOOKUP[index & 0xf];
-		index = index >> 4;
-	}
-}
-
-Key doubleToTestKey(double p) {
-	return StringRef(format("%016llx", *(uint64_t*)&p));
-}
-
-double testKeyToDouble(const KeyRef& p) {
-	uint64_t x = 0;
-	sscanf(p.toString().c_str(), "%" SCNx64, &x);
-	return *(double*)&x;
-}
-
-Key doubleToTestKey(double p, const KeyRef& prefix) {
-	return doubleToTestKey(p).withPrefix(prefix);
-}
-
-Key KVWorkload::getRandomKey() const {
-	return getRandomKey(absentFrac);
-}
-
-Key KVWorkload::getRandomKey(double absentFrac) const {
-	if (absentFrac > 0.0000001) {
-		return getRandomKey(deterministicRandom()->random01() < absentFrac);
-	} else {
-		return getRandomKey(false);
-	}
-}
-
-Key KVWorkload::getRandomKey(bool absent) const {
-	return keyForIndex(deterministicRandom()->randomInt(0, nodeCount), absent);
-}
-
-Key KVWorkload::keyForIndex(uint64_t index) const {
-	if (absentFrac > 0.0000001) {
-		return keyForIndex(index, deterministicRandom()->random01() < absentFrac);
-	} else {
-		return keyForIndex(index, false);
-	}
-}
-
-int64_t KVWorkload::indexForKey(const KeyRef& key, bool absent) const {
-	int idx = 0;
-	if (nodePrefix > 0) {
-		ASSERT(keyBytes >= 32);
-		idx += 16;
-	}
-	ASSERT(keyBytes >= 16);
-	// extract int64_t index, the reverse process of emplaceIndex()
-	auto end = key.size() - idx - (absent ? 1 : 0);
-	std::string str((char*)key.begin() + idx, end);
-	int64_t res = std::stoll(str, nullptr, 16);
-	return res;
-}
-
-Key KVWorkload::keyForIndex(uint64_t index, bool absent) const {
-	int adjustedKeyBytes = (absent) ? (keyBytes + 1) : keyBytes;
-	Key result = makeString(adjustedKeyBytes);
-	uint8_t* data = mutateString(result);
-	memset(data, '.', adjustedKeyBytes);
-
-	int idx = 0;
-	if (nodePrefix > 0) {
-		ASSERT(keyBytes >= 32);
-		emplaceIndex(data, 0, nodePrefix);
-		idx += 16;
-	}
-	ASSERT(keyBytes >= 16);
-	emplaceIndex(data, idx, (int64_t)index);
-	// ASSERT(indexForKey(result) == (int64_t)index); // debug assert
-
-	return result;
-}
-
-double testKeyToDouble(const KeyRef& p, const KeyRef& prefix) {
-	return testKeyToDouble(p.removePrefix(prefix));
-}
-
-Future<Void> poisson(double* last, double meanInterval) {
-	*last += meanInterval * -log(deterministicRandom()->random01());
-	co_await delayUntil(*last);
-	co_return;
-}
-
-Future<Void> uniform(double* last, double meanInterval) {
-	*last += meanInterval;
-	co_await delayUntil(*last);
-	co_return;
-}
-
-Value getOption(VectorRef<KeyValueRef> options, Key key, Value defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			Value value = options[i].value;
-			options[i].value = ""_sr;
-			return value;
-		}
-
-	return defaultValue;
-}
-
-int getOption(VectorRef<KeyValueRef> options, Key key, int defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			int r;
-			if (sscanf(options[i].value.toString().c_str(), "%d", &r)) {
-				options[i].value = ""_sr;
-				return r;
-			} else {
-				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
-				throw test_specification_invalid();
-			}
-		}
-
-	return defaultValue;
-}
-
-uint64_t getOption(VectorRef<KeyValueRef> options, Key key, uint64_t defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			uint64_t r;
-			if (sscanf(options[i].value.toString().c_str(), "%" SCNd64, &r)) {
-				options[i].value = ""_sr;
-				return r;
-			} else {
-				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
-				throw test_specification_invalid();
-			}
-		}
-
-	return defaultValue;
-}
-
-int64_t getOption(VectorRef<KeyValueRef> options, Key key, int64_t defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			int64_t r;
-			if (sscanf(options[i].value.toString().c_str(), "%" SCNd64, &r)) {
-				options[i].value = ""_sr;
-				return r;
-			} else {
-				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
-				throw test_specification_invalid();
-			}
-		}
-
-	return defaultValue;
-}
-
-double getOption(VectorRef<KeyValueRef> options, Key key, double defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			float r;
-			if (sscanf(options[i].value.toString().c_str(), "%f", &r)) {
-				options[i].value = ""_sr;
-				return r;
-			}
-		}
-
-	return defaultValue;
-}
-
-bool getOption(VectorRef<KeyValueRef> options, Key key, bool defaultValue) {
-	Value p = getOption(options, key, defaultValue ? "true"_sr : "false"_sr);
-	if (p == "true"_sr)
-		return true;
-	if (p == "false"_sr)
-		return false;
-	ASSERT(false);
-	return false; // Assure that compiler is fine with the function
-}
-
-std::vector<std::string> getOption(VectorRef<KeyValueRef> options, Key key, std::vector<std::string> defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			std::vector<std::string> v;
-			int begin = 0;
-			for (int c = 0; c < options[i].value.size(); c++)
-				if (options[i].value[c] == ',') {
-					v.push_back(options[i].value.substr(begin, c - begin).toString());
-					begin = c + 1;
-				}
-			v.push_back(options[i].value.substr(begin).toString());
-			options[i].value = ""_sr;
-			return v;
-		}
-	return defaultValue;
-}
-
-std::vector<int> getOption(VectorRef<KeyValueRef> options, Key key, std::vector<int> defaultValue) {
-	for (int i = 0; i < options.size(); i++)
-		if (options[i].key == key) {
-			std::vector<int> v;
-			int begin = 0;
-			for (int c = 0; c < options[i].value.size(); c++)
-				if (options[i].value[c] == ',') {
-					v.push_back(atoi((char*)options[i].value.begin() + begin));
-					begin = c + 1;
-				}
-			v.push_back(atoi((char*)options[i].value.begin() + begin));
-			options[i].value = ""_sr;
-			return v;
-		}
-	return defaultValue;
-}
-
-bool hasOption(VectorRef<KeyValueRef> options, Key key) {
-	for (const auto& option : options) {
-		if (option.key == key) {
-			return true;
-		}
-	}
-	return false;
-}
-
-// returns unconsumed options
-Standalone<VectorRef<KeyValueRef>> checkAllOptionsConsumed(VectorRef<KeyValueRef> options) {
-	static StringRef nothing = ""_sr;
-	Standalone<VectorRef<KeyValueRef>> unconsumed;
-	for (int i = 0; i < options.size(); i++)
-		if (!(options[i].value == nothing)) {
-			TraceEvent(SevError, "OptionNotConsumed")
-			    .detail("Key", options[i].key.toString().c_str())
-			    .detail("Value", options[i].value.toString().c_str());
-			unconsumed.push_back_deep(unconsumed.arena(), options[i]);
-		}
-	return unconsumed;
-}
-
-CompoundWorkload::CompoundWorkload(WorkloadContext& wcx) : TestWorkload(wcx) {}
-
-CompoundWorkload* CompoundWorkload::add(Reference<TestWorkload>&& w) {
-	workloads.push_back(std::move(w));
-	return this;
-}
-
-std::string CompoundWorkload::description() const {
-	std::vector<std::string> names;
-	names.reserve(workloads.size());
-	for (auto const& w : workloads) {
-		names.push_back(w->description());
-	}
-	return fmt::format("{}", fmt::join(std::move(names), ";"));
-}
-Future<Void> CompoundWorkload::setup(Database const& cx) {
-	std::vector<Future<Void>> all;
-	all.reserve(workloads.size());
-	for (int w = 0; w < workloads.size(); w++)
-		all.push_back(workloads[w]->setup(cx));
-	auto done = waitForAll(all);
-	if (failureInjection.empty()) {
-		return done;
-	}
-	std::vector<Future<Void>> res;
-	res.reserve(failureInjection.size());
-	for (auto& f : failureInjection) {
-		res.push_back(f->setupInjectionWorkload(cx, done));
-	}
-	return waitForAll(res);
-}
-
-Future<Void> CompoundWorkload::start(Database const& cx) {
-	std::vector<Future<Void>> all;
-	all.reserve(workloads.size() + failureInjection.size());
-	auto wCount = std::make_shared<unsigned>(0);
-	auto startWorkload = [&](TestWorkload& workload) -> Future<Void> {
-		auto workloadName = workload.description();
-		++(*wCount);
-		TraceEvent("WorkloadRunStatus").detail("Name", workloadName).detail("Count", *wCount).detail("Phase", "Start");
-		return fmap(
-		    [workloadName, wCount](Void value) {
-			    --(*wCount);
-			    TraceEvent("WorkloadRunStatus")
-			        .detail("Name", workloadName)
-			        .detail("Remaining", *wCount)
-			        .detail("Phase", "End");
-			    return Void();
-		    },
-		    workload.start(cx));
-	};
-	for (auto& workload : workloads) {
-		all.push_back(startWorkload(*workload));
-	}
-	for (auto& workload : failureInjection) {
-		all.push_back(startWorkload(*workload));
-	}
-	return waitForAll(all);
-}
-
-Future<bool> CompoundWorkload::check(Database const& cx) {
-	std::vector<Future<bool>> all;
-	all.reserve(workloads.size() + failureInjection.size());
-	auto wCount = std::make_shared<unsigned>(0);
-	auto starter = [&](TestWorkload& workload) -> Future<bool> {
-		++(*wCount);
-		std::string workloadName = workload.description();
-		TraceEvent("WorkloadCheckStatus")
-		    .detail("Name", workloadName)
-		    .detail("Count", *wCount)
-		    .detail("Phase", "Start");
-		return fmap(
-		    [workloadName, wCount](bool ret) {
-			    --(*wCount);
-			    TraceEvent("WorkloadCheckStatus")
-			        .detail("Name", workloadName)
-			        .detail("Remaining", *wCount)
-			        .detail("Phase", "End");
-			    return ret;
-		    },
-		    workload.check(cx));
-	};
-	for (auto& workload : workloads) {
-		all.push_back(starter(*workload));
-	}
-	for (auto& workload : failureInjection) {
-		all.push_back(starter(*workload));
-	}
-	return allTrue(all);
-}
-
-Future<std::vector<PerfMetric>> getMetricsCompoundWorkload(CompoundWorkload* self) {
-	std::vector<Future<std::vector<PerfMetric>>> results;
-	for (int w = 0; w < self->workloads.size(); w++) {
-		std::vector<PerfMetric> p;
-		results.push_back(self->workloads[w]->getMetrics());
-	}
-	co_await waitForAll(results);
-	std::vector<PerfMetric> res;
-	for (int i = 0; i < results.size(); ++i) {
-		auto const& p = results[i].get();
-		for (auto const& m : p) {
-			res.push_back(m.withPrefix(self->workloads[i]->description() + "."));
-		}
-	}
-	co_return res;
-}
-
-void CompoundWorkload::addFailureInjection(WorkloadRequest& work) {
-	if (!work.runFailureWorkloads) {
-		return;
-	}
-	// Some workloads won't work with some failure injection workloads
-	std::set<std::string> disabledWorkloads;
-	for (auto const& w : workloads) {
-		w->disableFailureInjectionWorkloads(disabledWorkloads);
-	}
-	if (disabledWorkloads.contains("all")) {
-		return;
-	}
-	auto& factories = IFailureInjectorFactory::factories();
-	DeterministicRandom random(sharedRandomNumber);
-	for (auto& factory : factories) {
-		auto workload = factory->create(*this);
-		if (disabledWorkloads.contains(workload->description())) {
-			continue;
-		}
-		if (std::find(work.disabledFailureInjectionWorkloads.begin(),
-		              work.disabledFailureInjectionWorkloads.end(),
-		              workload->description()) != work.disabledFailureInjectionWorkloads.end()) {
-			continue;
-		}
-		while (shouldInjectFailure(random, work, workload)) {
-			workload->initFailureInjectionMode(random);
-			TraceEvent("AddFailureInjectionWorkload")
-			    .detail("Name", workload->description())
-			    .detail("ClientID", work.clientId)
-			    .detail("ClientCount", clientCount)
-			    .detail("Title", work.title);
-			failureInjection.push_back(workload);
-			workload = factory->create(*this);
-		}
-	}
-}
-
-bool CompoundWorkload::shouldInjectFailure(DeterministicRandom& random,
-                                           const WorkloadRequest& work,
-                                           Reference<FailureInjectionWorkload> failure) const {
-	auto desc = failure->description();
-	unsigned alreadyAdded =
-	    std::count_if(workloads.begin(), workloads.end(), [&desc](auto const& w) { return w->description() == desc; });
-	alreadyAdded += std::count_if(
-	    failureInjection.begin(), failureInjection.end(), [&desc](auto const& w) { return w->description() == desc; });
-	return failure->shouldInject(random, work, alreadyAdded);
-}
-
-Future<std::vector<PerfMetric>> CompoundWorkload::getMetrics() {
-	return getMetricsCompoundWorkload(this);
-}
-
-double CompoundWorkload::getCheckTimeout() const {
-	double m = 0;
-	for (int w = 0; w < workloads.size(); w++)
-		m = std::max(workloads[w]->getCheckTimeout(), m);
-	return m;
-}
-
-void CompoundWorkload::getMetrics(std::vector<PerfMetric>&) {
-	ASSERT(false);
-}
-
-void TestWorkload::disableFailureInjectionWorkloads(std::set<std::string>& out) const {}
-
-FailureInjectionWorkload::FailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
-
-void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random) {}
-
-bool FailureInjectionWorkload::shouldInject(DeterministicRandom& random,
-                                            const WorkloadRequest& work,
-                                            const unsigned alreadyAdded) const {
-	return alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
-}
-
-Future<Void> FailureInjectionWorkload::setupInjectionWorkload(const Database& cx, Future<Void> done) {
-	return holdWhile(this->setup(cx), done);
-}
-
-Future<Void> FailureInjectionWorkload::startInjectionWorkload(const Database& cx, Future<Void> done) {
-	return holdWhile(this->start(cx), done);
-}
-
-Future<bool> FailureInjectionWorkload::checkInjectionWorkload(const Database& cx, Future<bool> done) {
-	return holdWhile(this->check(cx), done);
-}
-
-Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
-                                                 Reference<IClusterConnectionRecord> ccr,
-                                                 VectorRef<KeyValueRef> options,
-                                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	Reference<TestWorkload> workload;
-	Value testName = getOption(options, "testName"_sr, "no-test-specified"_sr);
-	WorkloadContext wcx;
-	wcx.clientId = work.clientId;
-	wcx.clientCount = work.clientCount;
-	wcx.ccr = ccr;
-	wcx.dbInfo = dbInfo;
-	wcx.options = options;
-	wcx.sharedRandomNumber = work.sharedRandomNumber;
-	wcx.rangesToCheck = work.rangesToCheck;
-
-	workload = IWorkloadFactory::create(testName.toString(), wcx);
-	if (workload) {
-		co_await workload->initialized();
-	}
-
-	auto unconsumedOptions = checkAllOptionsConsumed(workload ? workload->options : VectorRef<KeyValueRef>());
-	if (!workload || unconsumedOptions.size()) {
-		TraceEvent evt(SevError, "TestCreationError");
-		evt.detail("TestName", testName);
-		if (!workload) {
-			evt.detail("Reason", "Null workload");
-			fprintf(stderr,
-			        "ERROR: Workload could not be created, perhaps testName (%s) is not a valid workload\n",
-			        printable(testName).c_str());
-		} else {
-			evt.detail("Reason", "Not all options consumed");
-			fprintf(stderr, "ERROR: Workload had invalid options. The following were unrecognized:\n");
-			for (int i = 0; i < unconsumedOptions.size(); i++)
-				fprintf(stderr,
-				        " '%s' = '%s'\n",
-				        unconsumedOptions[i].key.toString().c_str(),
-				        unconsumedOptions[i].value.toString().c_str());
-		}
-		throw test_specification_invalid();
-	}
-	co_return workload;
-}
-
-Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
-                                                 Reference<IClusterConnectionRecord> ccr,
-                                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	WorkloadContext wcx;
-	std::vector<Future<Reference<TestWorkload>>> ifaces;
-	if (work.options.size() < 1) {
-		TraceEvent(SevError, "TestCreationError").detail("Reason", "No options provided");
-		fprintf(stderr, "ERROR: No options were provided for workload.\n");
-		throw test_specification_invalid();
-	}
-
-	wcx.clientId = work.clientId;
-	wcx.clientCount = work.clientCount;
-	wcx.sharedRandomNumber = work.sharedRandomNumber;
-	wcx.ccr = ccr;
-	wcx.dbInfo = dbInfo;
-	wcx.rangesToCheck = work.rangesToCheck;
-	// FIXME: Other stuff not filled in; why isn't this constructed here and passed down to the other
-	// getWorkloadIface()?
-	for (int i = 0; i < work.options.size(); i++) {
-		ifaces.push_back(getWorkloadIface(work, ccr, work.options[i], dbInfo));
-	}
-	co_await waitForAll(ifaces);
-	auto compound = makeReference<CompoundWorkload>(wcx);
-	for (int i = 0; i < work.options.size(); i++) {
-		compound->add(ifaces[i].getValue());
-	}
-	compound->addFailureInjection(work);
-	co_return compound;
-}
-
-/**
- * Only works in simulation. This method prints all simulated processes in a human readable form to stdout. It groups
- * processes by data center, data hall, zone, and machine (in this order).
- */
-void printSimulatedTopology() {
-	if (!g_network->isSimulated()) {
-		return;
-	}
-	auto processes = g_simulator->getAllProcesses();
-	std::sort(processes.begin(), processes.end(), [](ISimulator::ProcessInfo* lhs, ISimulator::ProcessInfo* rhs) {
-		auto l = lhs->locality;
-		auto r = rhs->locality;
-		if (l.dcId() != r.dcId()) {
-			return l.dcId() < r.dcId();
-		}
-		if (l.dataHallId() != r.dataHallId()) {
-			return l.dataHallId() < r.dataHallId();
-		}
-		if (l.zoneId() != r.zoneId()) {
-			return l.zoneId() < r.zoneId();
-		}
-		if (l.machineId() != r.zoneId()) {
-			return l.machineId() < r.machineId();
-		}
-		return lhs->address < rhs->address;
-	});
-	printf("Simulated Cluster Topology:\n");
-	printf("===========================\n");
-	Optional<Standalone<StringRef>> dcId, dataHallId, zoneId, machineId;
-	for (auto p : processes) {
-		std::string indent = "";
-		if (dcId != p->locality.dcId()) {
-			dcId = p->locality.dcId();
-			printf("%sdcId: %s\n", indent.c_str(), p->locality.describeDcId().c_str());
-		}
-		indent += "  ";
-		if (dataHallId != p->locality.dataHallId()) {
-			dataHallId = p->locality.dataHallId();
-			printf("%sdataHallId: %s\n", indent.c_str(), p->locality.describeDataHall().c_str());
-		}
-		indent += "  ";
-		if (zoneId != p->locality.zoneId()) {
-			zoneId = p->locality.zoneId();
-			printf("%szoneId: %s\n", indent.c_str(), p->locality.describeZone().c_str());
-		}
-		indent += "  ";
-		if (machineId != p->locality.machineId()) {
-			machineId = p->locality.machineId();
-			printf("%smachineId: %s\n", indent.c_str(), p->locality.describeMachineId().c_str());
-		}
-		indent += "  ";
-		printf("%sAddress: %s\n", indent.c_str(), p->address.toString().c_str());
-		indent += "  ";
-		printf("%sClass: %s\n", indent.c_str(), p->startingClass.toString().c_str());
-		printf("%sName: %s\n", indent.c_str(), p->name.c_str());
-	}
-}
-
-Future<Void> databaseWarmer(Database const& cx) {
-	// C++20 coroutine safety: copy const& params to survive across suspend points
-	Database cxCopy = cx;
-	while (true) {
-		Transaction tr(cxCopy);
-		co_await success(tr.getReadVersion());
-		co_await delay(0.25);
-	}
-	co_return; // Unreachable but required for coroutine
-}
-
-// Tries indefinitely to commit a simple, self conflicting transaction
-Future<Void> pingDatabase(Database cx) {
-	Transaction tr(cx);
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		try {
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Optional<Value> v =
-			    co_await tr.get(StringRef("/Liveness/" + deterministicRandom()->randomUniqueID().toString()));
-			tr.makeSelfConflicting();
-			co_await tr.commit();
-			co_return;
-		} catch (Error& e) {
-			TraceEvent("PingingDatabaseTransactionError").error(e);
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			co_await tr.onError(caughtError);
-		}
-	}
-}
-
-Future<Void> testDatabaseLiveness(Database cx, double databasePingDelay, std::string context, double startDelay = 0.0) {
-	co_await delay(startDelay);
-	while (true) {
-		Error caughtError;
-
-		try {
-			double start = now();
-			auto traceMsg = "PingingDatabaseLiveness_" + context;
-			TraceEvent(traceMsg.c_str()).log();
-			co_await timeoutError(pingDatabase(cx), databasePingDelay);
-			double pingTime = now() - start;
-			ASSERT(pingTime > 0);
-			TraceEvent(("PingingDatabaseLivenessDone_" + context).c_str()).detail("TimeTaken", pingTime);
-			co_await delay(databasePingDelay - pingTime);
-		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled)
-				TraceEvent(SevError, ("PingingDatabaseLivenessError_" + context).c_str())
-				    .error(e)
-				    .detail("PingDelay", databasePingDelay);
-			throw;
-		}
-	}
-}
-
-template <class T>
-void sendResult(ReplyPromise<T>& reply, Optional<ErrorOr<T>> const& result) {
-	auto& res = result.get();
-	if (res.isError())
-		reply.sendError(res.getError());
-	else
-		reply.send(res.get());
-}
-
-Future<Reference<TestWorkload>> getConsistencyCheckUrgentWorkloadIface(WorkloadRequest work,
-                                                                       Reference<IClusterConnectionRecord> ccr,
-                                                                       Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	WorkloadContext wcx;
-	wcx.clientId = work.clientId;
-	wcx.clientCount = work.clientCount;
-	wcx.sharedRandomNumber = work.sharedRandomNumber;
-	wcx.ccr = ccr;
-	wcx.dbInfo = dbInfo;
-	wcx.rangesToCheck = work.rangesToCheck;
-	Reference<TestWorkload> iface = co_await getWorkloadIface(work, ccr, work.options[0], dbInfo);
-	co_return iface;
-}
-
-Future<Void> runConsistencyCheckUrgentWorkloadAsync(Database cx,
-                                                    WorkloadInterface workIface,
-                                                    Reference<TestWorkload> workload) {
-	ReplyPromise<Void> jobReq = co_await workIface.start.getFuture();
-
-	try {
-		TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadReceived", workIface.id())
-		    .detail("WorkloadName", workload->description())
-		    .detail("ClientCount", workload->clientCount)
-		    .detail("ClientId", workload->clientId);
-		co_await workload->start(cx);
-		jobReq.send(Void());
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			throw e;
-		}
-		TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadError", workIface.id())
-		    .errorUnsuppressed(e)
-		    .detail("WorkloadName", workload->description())
-		    .detail("ClientCount", workload->clientCount)
-		    .detail("ClientId", workload->clientId);
-		jobReq.sendError(consistency_check_urgent_task_failed());
-	}
-	co_return;
-}
-
-Future<Void> testerServerConsistencyCheckerUrgentWorkload(WorkloadRequest work,
-                                                          Reference<IClusterConnectionRecord> ccr,
-                                                          Reference<AsyncVar<struct ServerDBInfo> const> dbInfo) {
-	WorkloadInterface workIface;
-	bool replied = false;
-
-	try {
-		Database cx = openDBOnServer(dbInfo);
-		co_await delay(1.0);
-		Reference<TestWorkload> workload = co_await getConsistencyCheckUrgentWorkloadIface(work, ccr, dbInfo);
-		Future<Void> test = runConsistencyCheckUrgentWorkloadAsync(cx, workIface, workload);
-		work.reply.send(workIface);
-		replied = true;
-		co_await test;
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			throw e;
-		}
-		TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterRunWorkloadFailed").errorUnsuppressed(e);
-		if (!replied) {
-			work.reply.sendError(e);
-		}
-	}
-	co_return;
-}
-
-Future<Void> runWorkloadAsync(Database cx,
-                              WorkloadInterface workIface,
-                              Reference<TestWorkload> workload,
-                              double databasePingDelay) {
-	Optional<ErrorOr<Void>> setupResult;
-	Optional<ErrorOr<Void>> startResult;
-	Optional<ErrorOr<CheckReply>> checkResult;
-	ReplyPromise<Void> setupReq;
-	ReplyPromise<Void> startReq;
-	ReplyPromise<CheckReply> checkReq;
-	ReplyPromise<std::vector<PerfMetric>> metricsReq;
-
-	TraceEvent("TestBeginAsync", workIface.id())
-	    .detail("Workload", workload->description())
-	    .detail("DatabasePingDelay", databasePingDelay);
-
-	Future<Void> databaseError =
-	    databasePingDelay == 0.0 ? Never() : testDatabaseLiveness(cx, databasePingDelay, "RunWorkloadAsync");
-
-	ReplyPromise<Void> stopReq;
-	while (true) {
-		int action = 0;
-		co_await Choose()
-		    .When(workIface.setup.getFuture(),
-		          [&](ReplyPromise<Void> const& req) {
-			          setupReq = req;
-			          action = 1;
-		          })
-		    .When(workIface.start.getFuture(),
-		          [&](ReplyPromise<Void> const& req) {
-			          startReq = req;
-			          action = 2;
-		          })
-		    .When(workIface.check.getFuture(),
-		          [&](ReplyPromise<CheckReply> const& req) {
-			          checkReq = req;
-			          action = 3;
-		          })
-		    .When(workIface.metrics.getFuture(),
-		          [&](ReplyPromise<std::vector<PerfMetric>> const& req) {
-			          metricsReq = req;
-			          action = 4;
-		          })
-		    .When(workIface.stop.getFuture(),
-		          [&](ReplyPromise<Void> const& r) {
-			          stopReq = r;
-			          action = 5;
-		          })
-		    .run();
-
-		if (action == 5) {
-			stopReq.send(Void());
-			break;
-		}
-
-		if (action == 1) {
-			// setup
-			printf("Test received trigger for setup...\n");
-			TraceEvent("TestSetupBeginning", workIface.id()).detail("Workload", workload->description());
-			if (!setupResult.present()) {
-				try {
-					co_await (workload->setup(cx) || databaseError);
-					TraceEvent("TestSetupComplete", workIface.id()).detail("Workload", workload->description());
-					setupResult = Void();
-				} catch (Error& e) {
-					setupResult = operation_failed();
-					TraceEvent(SevError, "TestSetupError", workIface.id())
-					    .error(e)
-					    .detail("Workload", workload->description());
-					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
-						throw;
-				}
-			}
-			sendResult(setupReq, setupResult);
-		} else if (action == 2) {
-			// start
-			if (!startResult.present()) {
-				try {
-					TraceEvent("TestStarting", workIface.id()).detail("Workload", workload->description());
-					co_await (workload->start(cx) || databaseError);
-					startResult = Void();
-				} catch (Error& e) {
-					startResult = operation_failed();
-					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
-						throw;
-					TraceEvent(SevError, "TestFailure", workIface.id())
-					    .errorUnsuppressed(e)
-					    .detail("Reason", "Error starting workload")
-					    .detail("Workload", workload->description());
-				}
-				TraceEvent("TestComplete", workIface.id())
-				    .detail("Workload", workload->description())
-				    .detail("OK", !startResult.get().isError());
-				printf("%s complete\n", workload->description().c_str());
-			}
-			sendResult(startReq, startResult);
-		} else if (action == 3) {
-			// check
-			if (!checkResult.present()) {
-				try {
-					TraceEvent("TestChecking", workIface.id()).detail("Workload", workload->description());
-					bool check = co_await timeoutError(workload->check(cx), workload->getCheckTimeout());
-					checkResult = CheckReply{ (!startResult.present() || !startResult.get().isError()) && check };
-				} catch (Error& e) {
-					checkResult = operation_failed();
-					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
-						throw;
-					TraceEvent(SevError, "TestFailure", workIface.id())
-					    .error(e)
-					    .detail("Reason", "Error checking workload")
-					    .detail("Workload", workload->description());
-				}
-				TraceEvent("TestCheckComplete", workIface.id()).detail("Workload", workload->description());
-			}
-			sendResult(checkReq, checkResult);
-		} else if (action == 4) {
-			// metrics
-			try {
-				std::vector<PerfMetric> m = co_await workload->getMetrics();
-				TraceEvent("WorkloadSendMetrics", workIface.id()).detail("Count", m.size());
-				metricsReq.send(m);
-			} catch (Error& e) {
-				if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
-					throw;
-				TraceEvent(SevError, "WorkloadSendMetrics", workIface.id()).error(e);
-				metricsReq.sendError(operation_failed());
-			}
-		}
-	}
-	co_return;
-}
-
-Future<Void> testerServerWorkload(WorkloadRequest work,
-                                  Reference<IClusterConnectionRecord> ccr,
-                                  Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
-                                  LocalityData locality) {
-	WorkloadInterface workIface;
-	bool replied = false;
-	Database cx;
-
-	try {
-		std::map<std::string, std::string> details;
-		details["WorkloadTitle"] = printable(work.title);
-		details["ClientId"] = format("%d", work.clientId);
-		details["ClientCount"] = format("%d", work.clientCount);
-		details["WorkloadTimeout"] = format("%d", work.timeout);
-		startRole(Role::TESTER, workIface.id(), UID(), details);
-
-		if (work.useDatabase) {
-			cx = Database::createDatabase(ccr, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
-			co_await delay(1.0);
-		}
-
-		// add test for "done" ?
-		TraceEvent("WorkloadReceived", workIface.id()).detail("Title", work.title);
-		Reference<TestWorkload> workload = co_await getWorkloadIface(work, ccr, dbInfo);
-		if (!workload) {
-			TraceEvent("TestCreationError").detail("Reason", "Workload could not be created");
-			fprintf(stderr, "ERROR: The workload could not be created.\n");
-			throw test_specification_invalid();
-		}
-		Future<Void> test = runWorkloadAsync(cx, workIface, workload, work.databasePingDelay) ||
-		                    traceRole(Role::TESTER, workIface.id());
-		work.reply.send(workIface);
-		replied = true;
-
-		if (work.timeout > 0) {
-			test = timeoutError(test, work.timeout);
-		}
-
-		co_await test;
-
-		endRole(Role::TESTER, workIface.id(), "Complete");
-	} catch (Error& e) {
-		TraceEvent(SevDebug, "TesterWorkloadFailed").errorUnsuppressed(e);
-		if (!replied) {
-			if (e.code() == error_code_test_specification_invalid)
-				work.reply.sendError(e);
-			else
-				work.reply.sendError(operation_failed());
-		}
-
-		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete ||
-		          e.code() == error_code_actor_cancelled;
-		endRole(Role::TESTER, workIface.id(), "Error", ok, e);
-
-		if (e.code() != error_code_test_specification_invalid && e.code() != error_code_timed_out) {
-			throw; // fatal errors will kill the testerServer as well
-		}
-	}
-	co_return;
-}
-
-Future<Void> testerServerCore(TesterInterface const& interf,
-                              Reference<IClusterConnectionRecord> const& ccr,
-                              Reference<AsyncVar<struct ServerDBInfo> const> const& dbInfo,
-                              LocalityData const& locality,
-                              Optional<std::string> const& expectedWorkLoad) {
-	// C++20 coroutine safety: const& parameters only store the reference in the coroutine frame,
-	// not the object. The referred-to object may be destroyed after the coroutine suspends
-	// (e.g. local variables in a caller's if-block, or temporaries from default arguments).
-	// Copy all const& parameters to ensure they survive across suspend points.
-	TesterInterface interfCopy = interf;
-	Reference<IClusterConnectionRecord> ccrCopy = ccr;
-	Reference<AsyncVar<struct ServerDBInfo> const> dbInfoCopy = dbInfo;
-	LocalityData localityCopy = locality;
-	Optional<std::string> expectedWorkLoadCopy = expectedWorkLoad;
-
-	PromiseStream<Future<Void>> addWorkload;
-	Future<Void> workerFatalError = actorCollection(addWorkload.getFuture());
-
-	// Dedicated to consistencyCheckerUrgent
-	// At any time, we only allow at most 1 consistency checker workload on a server
-	std::pair<int64_t, Future<Void>> consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>());
-
-	TraceEvent("StartingTesterServerCore", interfCopy.id())
-	    .detail("ExpectedWorkload", expectedWorkLoadCopy.present() ? expectedWorkLoadCopy.get() : "[Unset]");
-
-	while (true) {
-		int action = 0;
-		WorkloadRequest workReq;
-		Future<Void> checkerDone =
-		    consistencyCheckerUrgentTester.second.isValid() ? consistencyCheckerUrgentTester.second : Never();
-
-		co_await Choose()
-		    .When(workerFatalError, [&](Void const&) { action = 1; })
-		    .When(checkerDone, [&](Void const&) { action = 2; })
-		    .When(interfCopy.recruitments.getFuture(),
-		          [&](WorkloadRequest const& work) {
-			          workReq = work;
-			          action = 3;
-		          })
-		    .run();
-
-		if (action == 1) {
-			break;
-		}
-
-		if (action == 2) {
-			ASSERT(consistencyCheckerUrgentTester.first != 0);
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadEnd", interfCopy.id())
-			    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first);
-			consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>()); // reset
-			continue;
-		}
-
-		if (action == 3) {
-			if (expectedWorkLoadCopy.present() && expectedWorkLoadCopy.get() != workReq.title) {
-				TraceEvent(SevError, "StartingTesterServerCoreUnexpectedWorkload", interfCopy.id())
-				    .detail("ClientId", workReq.clientId)
-				    .detail("ClientCount", workReq.clientCount)
-				    .detail("ExpectedWorkLoad", expectedWorkLoadCopy.get())
-				    .detail("WorkLoad", workReq.title);
-				// Drop the workload
-			} else if (workReq.title == "ConsistencyCheckUrgent") {
-				// The workload is a consistency checker urgent workload
-				if (workReq.sharedRandomNumber == consistencyCheckerUrgentTester.first) {
-					// A single req can be sent for multiple times. In this case, the sharedRandomNumber is same as
-					// the existing one. For this scenario, we reply an error. This case should be rare.
-					TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterDuplicatedRequest", interfCopy.id())
-					    .detail("ConsistencyCheckerId", workReq.sharedRandomNumber)
-					    .detail("ClientId", workReq.clientId)
-					    .detail("ClientCount", workReq.clientCount);
-					workReq.reply.sendError(consistency_check_urgent_duplicate_request());
-				} else {
-					// When the req.sharedRandomNumber is different from the existing one, the cluster has muiltiple
-					// consistencycheckurgent roles at the same time. Evenutally, the cluster will have only one
-					// consistencycheckurgent role in a stable state. So, in this case, we simply let the new request to
-					// overwrite the old request. After the work is destroyed, the broken_promise will be replied.
-					if (consistencyCheckerUrgentTester.second.isValid() &&
-					    !consistencyCheckerUrgentTester.second.isReady()) {
-						TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_TesterWorkloadConflict", interfCopy.id())
-						    .detail("ExistingConsistencyCheckerId", consistencyCheckerUrgentTester.first)
-						    .detail("ArrivingConsistencyCheckerId", workReq.sharedRandomNumber)
-						    .detail("ClientId", workReq.clientId)
-						    .detail("ClientCount", workReq.clientCount);
-					}
-					consistencyCheckerUrgentTester =
-					    std::make_pair(workReq.sharedRandomNumber,
-					                   testerServerConsistencyCheckerUrgentWorkload(workReq, ccrCopy, dbInfoCopy));
-					TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadInitialized", interfCopy.id())
-					    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first)
-					    .detail("ClientId", workReq.clientId)
-					    .detail("ClientCount", workReq.clientCount);
-				}
-			} else {
-				addWorkload.send(testerServerWorkload(workReq, ccrCopy, dbInfoCopy, localityCopy));
-			}
-		}
-	}
-	co_return;
-}
-
-Future<Void> clearData(Database cx) {
-	TraceEvent("ClearData_Start").detail("Phase", "Enter");
-
-	Transaction tr(cx);
-	TraceEvent("ClearData_TrCreated").detail("Phase", "Loop1_Start");
-
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		TraceEvent("ClearData_Loop1_Iteration").detail("Phase", "TryBlock");
-
-		try {
-			tr.debugTransaction(debugRandom()->randomUniqueID());
-			TraceEvent("ClearData_DebugSet").detail("Phase", "Loop1_Debug");
-
-			ASSERT(tr.trState->readOptions.present() && tr.trState->readOptions.get().debugID.present());
-			TraceEvent("TesterClearingDatabaseStart", tr.trState->readOptions.get().debugID.get()).log();
-
-			// This transaction needs to be self-conflicting, but not conflict consistently with
-			// any other transactions
-			tr.clear(normalKeys);
-			tr.makeSelfConflicting();
-			TraceEvent("ClearData_PreGetReadVersion").detail("Phase", "Loop1_AboutToAwait");
-
-			Version rv = co_await tr.getReadVersion(); // required since we use addReadConflictRange but not get
-			TraceEvent("TesterClearingDatabaseRV", tr.trState->readOptions.get().debugID.get()).detail("RV", rv);
-			TraceEvent("ClearData_PreCommit").detail("Phase", "Loop1_AboutToCommit");
-
-			co_await tr.commit();
-			TraceEvent("TesterClearingDatabase", tr.trState->readOptions.get().debugID.get())
-			    .detail("AtVersion", tr.getCommittedVersion());
-			TraceEvent("ClearData_Loop1_Success").detail("Phase", "Loop1_Complete");
-			break;
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "TesterClearingDatabaseError", tr.trState->readOptions.get().debugID.get()).error(e);
-			TraceEvent("ClearData_Loop1_Catch").detail("Phase", "Loop1_Error").detail("ErrorCode", e.code());
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			TraceEvent("ClearData_Loop1_ErrorHandling").detail("Phase", "Loop1_OnError");
-			co_await tr.onError(caughtError);
-		}
-	}
-
-	TraceEvent("ClearData_Loop1_Done").detail("Phase", "Loop2_Start");
-	tr = Transaction(cx);
-	TraceEvent("ClearData_NewTr").detail("Phase", "Loop2_TrCreated");
-
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		TraceEvent("ClearData_Loop2_Iteration").detail("Phase", "Loop2_TryBlock");
-
-		try {
-			tr.debugTransaction(debugRandom()->randomUniqueID());
-			tr.setOption(FDBTransactionOptions::RAW_ACCESS);
-			TraceEvent("ClearData_Loop2_PreGetRange").detail("Phase", "Loop2_AboutToAwait");
-
-			RangeResult rangeResult = co_await tr.getRange(normalKeys, 1);
-			TraceEvent("ClearData_Loop2_PostGetRange")
-			    .detail("Phase", "Loop2_RangeGot")
-			    .detail("ResultSize", rangeResult.size());
-
-			// If the result is non-empty, it is possible that there is some bug.
-			if (!rangeResult.empty()) {
-				TraceEvent(SevError, "TesterClearFailure").detail("FirstKey", rangeResult[0].key);
-				ASSERT(false);
-			}
-			// TODO(gglass): the assert below is leftover from some tenant logic.  If it fails, take it out.
-			// If not, leave it in and take out this comment.
-			ASSERT(tr.trState->readOptions.present() && tr.trState->readOptions.get().debugID.present());
-			TraceEvent("TesterCheckDatabaseClearedDone", tr.trState->readOptions.get().debugID.get());
-			TraceEvent("ClearData_Loop2_Success").detail("Phase", "Loop2_Complete");
-			break;
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "TesterCheckDatabaseClearedError", tr.trState->readOptions.get().debugID.get())
-			    .error(e);
-			TraceEvent("ClearData_Loop2_Catch").detail("Phase", "Loop2_Error").detail("ErrorCode", e.code());
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			TraceEvent("ClearData_Loop2_ErrorHandling").detail("Phase", "Loop2_OnError");
-			co_await tr.onError(caughtError);
-		}
-	}
-
-	TraceEvent("ClearData_End").detail("Phase", "Success");
-	co_return;
-}
-
-Future<Void> dumpDatabase(Database const& cx, std::string const& outputFilename, KeyRange const& range);
-
-int passCount = 0;
-int failCount = 0;
-
-std::vector<PerfMetric> aggregateMetrics(std::vector<std::vector<PerfMetric>> metrics) {
-	std::map<std::string, std::vector<PerfMetric>> metricMap;
-	for (int i = 0; i < metrics.size(); i++) {
-		std::vector<PerfMetric> workloadMetrics = metrics[i];
-		TraceEvent("MetricsReturned").detail("Count", workloadMetrics.size());
-		for (int m = 0; m < workloadMetrics.size(); m++) {
-			printf("Metric (%d, %d): %s, %f, %s\n",
-			       i,
-			       m,
-			       workloadMetrics[m].name().c_str(),
-			       workloadMetrics[m].value(),
-			       workloadMetrics[m].formatted().c_str());
-			metricMap[workloadMetrics[m].name()].push_back(workloadMetrics[m]);
-		}
-	}
-	TraceEvent("Metric")
-	    .detail("Name", "Reporting Clients")
-	    .detail("Value", (double)metrics.size())
-	    .detail("Formatted", format("%d", metrics.size()).c_str());
-
-	std::vector<PerfMetric> result;
-	std::map<std::string, std::vector<PerfMetric>>::iterator it;
-	for (it = metricMap.begin(); it != metricMap.end(); it++) {
-		auto& vec = it->second;
-		if (!vec.size())
-			continue;
-		double sum = 0;
-		for (int i = 0; i < vec.size(); i++)
-			sum += vec[i].value();
-		if (vec[0].averaged() && vec.size())
-			sum /= vec.size();
-		result.emplace_back(vec[0].name(), sum, Averaged::False, vec[0].format_code());
-	}
-	return result;
-}
-
-void logMetrics(std::vector<PerfMetric> metrics) {
-	for (int idx = 0; idx < metrics.size(); idx++)
-		TraceEvent("Metric")
-		    .detail("Name", metrics[idx].name())
-		    .detail("Value", metrics[idx].value())
-		    .detail("Formatted", format(metrics[idx].format_code().c_str(), metrics[idx].value()));
-}
-
-template <class T>
-void throwIfError(const std::vector<Future<ErrorOr<T>>>& futures, std::string errorMsg) {
-	for (auto& future : futures) {
-		if (future.get().isError()) {
-			TraceEvent(SevError, errorMsg.c_str()).error(future.get().getError());
-			throw future.get().getError();
-		}
-	}
-}
-
-struct TesterConsistencyScanState {
-	bool enabled = false;
-	bool enableAfter = false;
-	bool waitForComplete = false;
-};
-
-Future<Void> checkConsistencyScanAfterTest(Database cx, TesterConsistencyScanState* csState) {
-	if (!csState->enabled) {
-		co_return;
-	}
-
-	// mark it as done so later so this does not repeat
-	csState->enabled = false;
-
-	if (csState->enableAfter || csState->waitForComplete) {
-		printf("Enabling consistency scan after test ...\n");
-		co_await enableConsistencyScanInSim(cx);
-		printf("Enabled consistency scan after test.\n");
-	}
-
-	co_await disableConsistencyScanInSim(cx, csState->waitForComplete);
-
-	co_return;
-}
-
-Future<DistributedTestResults> runWorkload(Database const& cx,
-                                           std::vector<TesterInterface> const& testers,
-                                           TestSpec const& spec) {
-	// C++20 coroutine safety: copy const& params to survive across suspend points
-	Database cxCopy = cx;
-	std::vector<TesterInterface> testersCopy = testers;
-	TestSpec specCopy = spec;
-
-	TraceEvent("TestRunning")
-	    .detail("WorkloadTitle", specCopy.title)
-	    .detail("TesterCount", testersCopy.size())
-	    .detail("Phases", specCopy.phases)
-	    .detail("TestTimeout", specCopy.timeout);
-
-	std::vector<Future<WorkloadInterface>> workRequests;
-	std::vector<std::vector<PerfMetric>> metricsResults;
-
-	int i = 0;
-	int success = 0;
-	int failure = 0;
-	int64_t sharedRandom = deterministicRandom()->randomInt64(0, 10000000);
-	for (; i < testersCopy.size(); i++) {
-		WorkloadRequest req;
-		req.title = specCopy.title;
-		req.useDatabase = specCopy.useDB;
-		req.runFailureWorkloads = specCopy.runFailureWorkloads;
-		req.timeout = specCopy.timeout;
-		req.databasePingDelay = specCopy.useDB ? specCopy.databasePingDelay : 0.0;
-		req.options = specCopy.options;
-		req.clientId = i;
-		req.clientCount = testersCopy.size();
-		req.sharedRandomNumber = sharedRandom;
-		req.disabledFailureInjectionWorkloads = specCopy.disabledFailureInjectionWorkloads;
-		workRequests.push_back(testersCopy[i].recruitments.getReply(req));
-	}
-
-	std::vector<WorkloadInterface> workloads = co_await getAll(workRequests);
-	double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
-	if (g_network->isSimulated() && specCopy.simCheckRelocationDuration)
-		debug_setCheckRelocationDuration(true);
-
-	if (specCopy.phases & TestWorkload::SETUP) {
-		std::vector<Future<ErrorOr<Void>>> setups;
-		printf("setting up test (%s)...\n", printable(specCopy.title).c_str());
-		TraceEvent("TestSetupStart").detail("WorkloadTitle", specCopy.title);
-		setups.reserve(workloads.size());
-		for (int i = 0; i < workloads.size(); i++)
-			setups.push_back(workloads[i].setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
-		co_await waitForAll(setups);
-		throwIfError(setups, "SetupFailedForWorkload" + printable(specCopy.title));
-		TraceEvent("TestSetupComplete").detail("WorkloadTitle", specCopy.title);
-	}
-
-	if (specCopy.phases & TestWorkload::EXECUTION) {
-		TraceEvent("TestStarting").detail("WorkloadTitle", specCopy.title);
-		printf("running test (%s)...\n", printable(specCopy.title).c_str());
-		std::vector<Future<ErrorOr<Void>>> starts;
-		starts.reserve(workloads.size());
-		for (int i = 0; i < workloads.size(); i++)
-			starts.push_back(workloads[i].start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
-		co_await waitForAll(starts);
-		throwIfError(starts, "StartFailedForWorkload" + printable(specCopy.title));
-		printf("%s complete\n", printable(specCopy.title).c_str());
-		TraceEvent("TestComplete").detail("WorkloadTitle", specCopy.title);
-	}
-
-	if (specCopy.phases & TestWorkload::CHECK) {
-		if (specCopy.useDB && (specCopy.phases & TestWorkload::EXECUTION)) {
-			co_await delay(3.0);
-		}
-
-		std::vector<Future<ErrorOr<CheckReply>>> checks;
-		TraceEvent("TestCheckingResults").detail("WorkloadTitle", specCopy.title);
-
-		printf("checking test (%s)...\n", printable(specCopy.title).c_str());
-
-		checks.reserve(workloads.size());
-		for (int i = 0; i < workloads.size(); i++)
-			checks.push_back(workloads[i].check.template getReplyUnlessFailedFor<CheckReply>(waitForFailureTime, 0));
-		co_await waitForAll(checks);
-
-		throwIfError(checks, "CheckFailedForWorkload" + printable(specCopy.title));
-
-		for (int i = 0; i < checks.size(); i++) {
-			if (checks[i].get().get().value)
-				success++;
-			else
-				failure++;
-		}
-		TraceEvent("TestCheckComplete").detail("WorkloadTitle", specCopy.title);
-	}
-
-	if (specCopy.phases & TestWorkload::METRICS) {
-		std::vector<Future<ErrorOr<std::vector<PerfMetric>>>> metricTasks;
-		printf("fetching metrics (%s)...\n", printable(specCopy.title).c_str());
-		TraceEvent("TestFetchingMetrics").detail("WorkloadTitle", specCopy.title);
-		metricTasks.reserve(workloads.size());
-		for (int i = 0; i < workloads.size(); i++)
-			metricTasks.push_back(
-			    workloads[i].metrics.template getReplyUnlessFailedFor<std::vector<PerfMetric>>(waitForFailureTime, 0));
-		co_await waitForAll(metricTasks);
-		throwIfError(metricTasks, "MetricFailedForWorkload" + printable(specCopy.title));
-		for (int i = 0; i < metricTasks.size(); i++) {
-			metricsResults.push_back(metricTasks[i].get().get());
-		}
-	}
-
-	// Stopping the workloads is unreliable, but they have a timeout
-	// FIXME: stop if one of the above phases throws an exception
-	for (int i = 0; i < workloads.size(); i++)
-		workloads[i].stop.send(ReplyPromise<Void>());
-
-	co_return DistributedTestResults(aggregateMetrics(metricsResults), success, failure);
-}
-
-// Sets the database configuration by running the ChangeConfig workload
-Future<Void> changeConfiguration(Database cx, std::vector<TesterInterface> testers, StringRef configMode) {
-	TestSpec spec;
-	Standalone<VectorRef<KeyValueRef>> options;
-	spec.title = "ChangeConfig"_sr;
-	spec.runFailureWorkloads = false;
-	options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ChangeConfig"_sr));
-	options.push_back_deep(options.arena(), KeyValueRef("configMode"_sr, configMode));
-	spec.options.push_back_deep(spec.options.arena(), options);
-
-	DistributedTestResults testResults = co_await runWorkload(cx, testers, spec);
-
-	co_return;
-}
-
-Future<Void> auditStorageCorrectness(Reference<AsyncVar<ServerDBInfo>> dbInfo, AuditType auditType) {
-	if (SERVER_KNOBS->DISABLE_AUDIT_STORAGE_FINAL_REPLICA_CHECK_IN_SIM &&
-	    (auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica)) {
-		co_return;
-	}
-	TraceEvent(SevDebug, "AuditStorageCorrectnessBegin").detail("AuditType", auditType);
-	Database cx;
-	UID auditId;
-	AuditStorageState auditState;
-
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		try {
-			while (dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS ||
-			       !dbInfo->get().distributor.present()) {
-				co_await dbInfo->onChange();
-			}
-			TriggerAuditRequest req(auditType, allKeys, KeyValueStoreType::END); // do not specify engine type to check
-			UID auditId_ = co_await timeoutError(dbInfo->get().distributor.get().triggerAudit.getReply(req), 300);
-			auditId = auditId_;
-			TraceEvent(SevDebug, "AuditStorageCorrectnessTriggered")
-			    .detail("AuditID", auditId)
-			    .detail("AuditType", auditType);
-			break;
-		} catch (Error& e) {
-			TraceEvent(SevWarn, "AuditStorageCorrectnessTriggerError")
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", auditId)
-			    .detail("AuditType", auditType);
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			co_await delay(1);
-		}
-	}
-
-	int retryCount = 0;
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		try {
-			cx = openDBOnServer(dbInfo);
-			AuditStorageState auditState_ = co_await getAuditState(cx, auditType, auditId);
-			auditState = auditState_;
-			if (auditState.getPhase() == AuditPhase::Complete) {
-				break;
-			} else if (auditState.getPhase() == AuditPhase::Running) {
-				TraceEvent("AuditStorageCorrectnessWait")
-				    .detail("AuditID", auditId)
-				    .detail("AuditType", auditType)
-				    .detail("RetryCount", retryCount);
-				co_await delay(25);
-				if (retryCount > 20) {
-					TraceEvent("AuditStorageCorrectnessWaitFailed")
-					    .detail("AuditID", auditId)
-					    .detail("AuditType", auditType);
-					break;
-				}
-				retryCount++;
-				continue;
-			} else if (auditState.getPhase() == AuditPhase::Error) {
-				break;
-			} else if (auditState.getPhase() == AuditPhase::Failed) {
-				break;
-			} else {
-				UNREACHABLE();
-			}
-		} catch (Error& e) {
-			TraceEvent("AuditStorageCorrectnessWaitError")
-			    .errorUnsuppressed(e)
-			    .detail("AuditID", auditId)
-			    .detail("AuditType", auditType)
-			    .detail("AuditState", auditState.toString());
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			co_await delay(1);
-		}
-	}
-	TraceEvent("AuditStorageCorrectnessWaitEnd")
-	    .detail("AuditID", auditId)
-	    .detail("AuditType", auditType)
-	    .detail("AuditState", auditState.toString());
-
-	co_return;
-}
-
-// Runs the consistency check workload, which verifies that the database is in a consistent state
-Future<Void> checkConsistency(Database cx,
-                              std::vector<TesterInterface> testers,
-                              bool doQuiescentCheck,
-                              bool doTSSCheck,
-                              double maxDDRunTime,
-                              double softTimeLimit,
-                              double databasePingDelay,
-                              Reference<AsyncVar<ServerDBInfo>> dbInfo) {
-	TestSpec spec;
-
-	double connectionFailures;
-	if (g_network->isSimulated()) {
-		// NOTE: the value will be reset after consistency check
-		connectionFailures = g_simulator->connectionFailuresDisableDuration;
-		disableConnectionFailures("ConsistencyCheck");
-		g_simulator->isConsistencyChecked = true;
-	}
-
-	Standalone<VectorRef<KeyValueRef>> options;
-	StringRef performQuiescent = "false"_sr;
-	StringRef performTSSCheck = "false"_sr;
-	if (doQuiescentCheck) {
-		performQuiescent = "true"_sr;
-		spec.restorePerpetualWiggleSetting = false;
-	}
-	if (doTSSCheck) {
-		performTSSCheck = "true"_sr;
-	}
-	spec.title = "ConsistencyCheck"_sr;
-	spec.databasePingDelay = databasePingDelay;
-	spec.runFailureWorkloads = false;
-	spec.timeout = 32000;
-	options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ConsistencyCheck"_sr));
-	options.push_back_deep(options.arena(), KeyValueRef("performQuiescentChecks"_sr, performQuiescent));
-	options.push_back_deep(options.arena(), KeyValueRef("performTSSCheck"_sr, performTSSCheck));
-	options.push_back_deep(options.arena(),
-	                       KeyValueRef("maxDDRunTime"_sr, ValueRef(options.arena(), format("%f", maxDDRunTime))));
-	options.push_back_deep(options.arena(), KeyValueRef("distributed"_sr, "false"_sr));
-	spec.options.push_back_deep(spec.options.arena(), options);
-
-	double start = now();
-	bool lastRun = false;
-	while (true) {
-		DistributedTestResults testResults = co_await runWorkload(cx, testers, spec);
-		if (testResults.ok() || lastRun) {
-			if (g_network->isSimulated()) {
-				g_simulator->connectionFailuresDisableDuration = connectionFailures;
-				g_simulator->isConsistencyChecked = false;
-			}
-			co_return;
-		}
-		if (now() - start > softTimeLimit) {
-			spec.options[0].push_back_deep(spec.options.arena(), KeyValueRef("failureIsError"_sr, "true"_sr));
-			lastRun = true;
-		}
-
-		co_await repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck");
-	}
-}
-
-Future<std::unordered_set<int>> runUrgentConsistencyCheckWorkload(
-    Database cx,
-    std::vector<TesterInterface> testers,
-    int64_t consistencyCheckerId,
-    std::unordered_map<int, std::vector<KeyRange>> assignment) {
-	TraceEvent(SevInfo, "ConsistencyCheckUrgent_DispatchWorkloads")
-	    .detail("TesterCount", testers.size())
-	    .detail("ConsistencyCheckerId", consistencyCheckerId);
-
-	// Step 1: Get interfaces for running workloads
-	std::vector<Future<ErrorOr<WorkloadInterface>>> workRequests;
-	Standalone<VectorRef<KeyValueRef>> option;
-	option.push_back_deep(option.arena(), KeyValueRef("testName"_sr, "ConsistencyCheckUrgent"_sr));
-	Standalone<VectorRef<VectorRef<KeyValueRef>>> options;
-	options.push_back_deep(options.arena(), option);
-	for (int i = 0; i < testers.size(); i++) {
-		WorkloadRequest req;
-		req.title = "ConsistencyCheckUrgent"_sr;
-		req.useDatabase = true;
-		req.timeout = 0.0; // disable timeout workload
-		req.databasePingDelay = 0.0; // disable databased ping check
-		req.options = options;
-		req.clientId = i;
-		req.clientCount = testers.size();
-		req.sharedRandomNumber = consistencyCheckerId;
-		req.rangesToCheck = assignment[i];
-		workRequests.push_back(testers[i].recruitments.getReplyUnlessFailedFor(req, 10, 0));
-		// workRequests follows the order of clientId of assignment
-	}
-	co_await waitForAll(workRequests);
-
-	// Step 2: Run workloads via the interfaces
-	TraceEvent(SevInfo, "ConsistencyCheckUrgent_TriggerWorkloads")
-	    .detail("TesterCount", testers.size())
-	    .detail("ConsistencyCheckerId", consistencyCheckerId);
-	std::unordered_set<int> completeClientIds;
-	std::vector<int> clientIds; // record the clientId for jobs
-	std::vector<Future<ErrorOr<Void>>> jobs;
-	for (int i = 0; i < workRequests.size(); i++) {
-		ASSERT(workRequests[i].isReady());
-		if (workRequests[i].get().isError()) {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_FailedToContactTester")
-			    .error(workRequests[i].get().getError())
-			    .detail("TesterCount", testers.size())
-			    .detail("TesterId", i)
-			    .detail("ConsistencyCheckerId", consistencyCheckerId);
-		} else {
-			jobs.push_back(workRequests[i].get().get().start.template getReplyUnlessFailedFor<Void>(10, 0));
-			clientIds.push_back(i);
-		}
-	}
-	co_await waitForAll(jobs);
-	for (int i = 0; i < jobs.size(); i++) {
-		if (jobs[i].isError()) {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RunWorkloadError1")
-			    .errorUnsuppressed(jobs[i].getError())
-			    .detail("ClientId", clientIds[i])
-			    .detail("ClientCount", testers.size())
-			    .detail("ConsistencyCheckerId", consistencyCheckerId);
-		} else if (jobs[i].get().isError()) {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RunWorkloadError2")
-			    .errorUnsuppressed(jobs[i].get().getError())
-			    .detail("ClientId", clientIds[i])
-			    .detail("ClientCount", testers.size())
-			    .detail("ConsistencyCheckerId", consistencyCheckerId);
-		} else {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RunWorkloadComplete")
-			    .detail("ClientId", clientIds[i])
-			    .detail("ClientCount", testers.size())
-			    .detail("ConsistencyCheckerId", consistencyCheckerId);
-			completeClientIds.insert(clientIds[i]); // Add complete clients
-		}
-	}
-
-	TraceEvent(SevInfo, "ConsistencyCheckUrgent_DispatchWorkloadEnd")
-	    .detail("TesterCount", testers.size())
-	    .detail("ConsistencyCheckerId", consistencyCheckerId);
-
-	co_return completeClientIds;
-}
-
-Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::vector<KeyRange> ranges) {
-	// Get the scope of the input list of ranges
-	Key beginKeyToReadKeyServer;
-	Key endKeyToReadKeyServer;
-	for (int i = 0; i < ranges.size(); i++) {
-		if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
-			beginKeyToReadKeyServer = ranges[i].begin;
-		}
-		if (i == 0 || ranges[i].end > endKeyToReadKeyServer) {
-			endKeyToReadKeyServer = ranges[i].end;
-		}
-	}
-	TraceEvent(SevInfo, "ConsistencyCheckUrgent_GetConsistencyCheckShards")
-	    .detail("RangeBegin", beginKeyToReadKeyServer)
-	    .detail("RangeEnd", endKeyToReadKeyServer);
-	// Read KeyServer space within the scope and add shards intersecting with the input ranges
-	std::vector<KeyRange> res;
-	Transaction tr(cx);
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		try {
-			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
-			RangeResult readResult = co_await krmGetRanges(&tr,
-			                                               keyServersPrefix,
-			                                               rangeToRead,
-			                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
-			                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
-			for (int i = 0; i < readResult.size() - 1; ++i) {
-				KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
-				Value valueToCheck = Standalone(readResult[i].value);
-				bool toAdd = false;
-				for (const auto& range : ranges) {
-					if (rangeToCheck.intersects(range) == true) {
-						toAdd = true;
-						break;
-					}
-				}
-				if (toAdd == true) {
-					res.push_back(rangeToCheck);
-				}
-				beginKeyToReadKeyServer = readResult[i + 1].key;
-			}
-			if (beginKeyToReadKeyServer >= endKeyToReadKeyServer) {
-				break;
-			}
-		} catch (Error& e) {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_GetConsistencyCheckShardsRetry").error(e);
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			co_await tr.onError(caughtError);
-		}
-	}
-	co_return res;
-}
-
-Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
-                                                int minTestersExpected) {
-	// Recruit workers
-	int flags = GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
-	Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
-	std::vector<WorkerDetails> workers;
-
-	while (true) {
-		Future<std::vector<WorkerDetails>> getWorkersReq =
-		    cc->get().present() ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
-		                        : Future<std::vector<WorkerDetails>>(Never());
-		Future<Void> ccChange = cc->onChange();
-
-		int action = 0;
-		std::vector<WorkerDetails> gotWorkers;
-
-		co_await Choose()
-		    .When(getWorkersReq,
-		          [&](std::vector<WorkerDetails> const& w) {
-			          gotWorkers = w;
-			          action = 1;
-		          })
-		    .When(ccChange, [&](Void const&) { action = 2; })
-		    .When(testerTimeout, [&](Void const&) { action = 3; })
-		    .run();
-
-		if (action == 1) {
-			if (static_cast<int>(gotWorkers.size()) >= minTestersExpected) {
-				workers = gotWorkers;
-				break;
-			}
-			co_await delay(SERVER_KNOBS->WORKER_POLL_DELAY);
-		} else if (action == 3) {
-			TraceEvent(SevWarnAlways, "TesterRecruitmentTimeout");
-			throw timed_out();
-		}
-		// action == 2: cc changed, loop again
-	}
-	std::vector<TesterInterface> ts;
-	ts.reserve(workers.size());
-	for (int i = 0; i < workers.size(); i++)
-		ts.push_back(workers[i].interf.testerInterface);
-	deterministicRandom()->randomShuffle(ts);
-	co_return ts;
-}
-
-const std::unordered_map<char, uint8_t> parseCharMap{
-	{ '0', 0 },  { '1', 1 },  { '2', 2 },  { '3', 3 },  { '4', 4 },  { '5', 5 },  { '6', 6 },  { '7', 7 },
-	{ '8', 8 },  { '9', 9 },  { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
-	{ 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 },
-};
-
-Optional<Key> getKeyFromString(const std::string& str) {
-	Key emptyKey;
-	if (str.size() == 0) {
-		return emptyKey;
-	}
-	if (str.size() % 4 != 0) {
-		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
-		    .setMaxEventLength(-1)
-		    .setMaxFieldLength(-1)
-		    .detail("Reason", "WrongLength")
-		    .detail("InputStr", str);
-		return Optional<Key>();
-	}
-	std::vector<uint8_t> byteList;
-	for (int i = 0; i < str.size(); i += 4) {
-		if (str.at(i + 0) != '\\' || str.at(i + 1) != 'x') {
-			TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-			           "ConsistencyCheckUrgent_GetKeyFromStringError")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("Reason", "WrongBytePrefix")
-			    .detail("InputStr", str);
-			return Optional<Key>();
-		}
-		const char first = str.at(i + 2);
-		const char second = str.at(i + 3);
-		if (!parseCharMap.contains(first) || !parseCharMap.contains(second)) {
-			TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
-			           "ConsistencyCheckUrgent_GetKeyFromStringError")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("Reason", "WrongByteContent")
-			    .detail("InputStr", str);
-			return Optional<Key>();
-		}
-		uint8_t parsedValue = parseCharMap.at(first) * 16 + parseCharMap.at(second);
-		byteList.push_back(parsedValue);
-	}
-	return Standalone(StringRef(byteList.data(), byteList.size()));
-}
-
-Optional<std::vector<KeyRange>> loadRangesToCheckFromKnob() {
-	// Load string from knob
-	std::vector<std::string> beginKeyStrs = {
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_0,
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_1,
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_2,
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_3,
-	};
-	std::vector<std::string> endKeyStrs = {
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_0,
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_1,
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_2,
-		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_3,
-	};
-
-	// Get keys from strings
-	std::vector<Key> beginKeys;
-	for (const auto& beginKeyStr : beginKeyStrs) {
-		Optional<Key> key = getKeyFromString(beginKeyStr);
-		if (key.present()) {
-			beginKeys.push_back(key.get());
-		} else {
-			return Optional<std::vector<KeyRange>>();
-		}
-	}
-	std::vector<Key> endKeys;
-	for (const auto& endKeyStr : endKeyStrs) {
-		Optional<Key> key = getKeyFromString(endKeyStr);
-		if (key.present()) {
-			endKeys.push_back(key.get());
-		} else {
-			return Optional<std::vector<KeyRange>>();
-		}
-	}
-	if (beginKeys.size() != endKeys.size()) {
-		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
-		    .detail("Reason", "MismatchBeginKeysAndEndKeys");
-		return Optional<std::vector<KeyRange>>();
-	}
-
-	// Get ranges
-	KeyRangeMap<bool> rangeToCheckMap;
-	for (int i = 0; i < beginKeys.size(); i++) {
-		Key rangeBegin = beginKeys[i];
-		Key rangeEnd = endKeys[i];
-		if (rangeBegin.empty() && rangeEnd.empty()) {
-			continue;
-		}
-		if (rangeBegin > allKeys.end) {
-			rangeBegin = allKeys.end;
-		}
-		if (rangeEnd > allKeys.end) {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_ReverseInputRange")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("Index", i)
-			    .detail("RangeBegin", rangeBegin)
-			    .detail("RangeEnd", rangeEnd);
-			rangeEnd = allKeys.end;
-		}
-
-		KeyRange rangeToCheck;
-		if (rangeBegin < rangeEnd) {
-			rangeToCheck = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
-		} else if (rangeBegin > rangeEnd) {
-			rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
-		} else {
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_EmptyInputRange")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("Index", i)
-			    .detail("RangeBegin", rangeBegin)
-			    .detail("RangeEnd", rangeEnd);
-			continue;
-		}
-		rangeToCheckMap.insert(rangeToCheck, true);
-	}
-
-	rangeToCheckMap.coalesce(allKeys);
-
-	std::vector<KeyRange> res;
-	for (auto rangeToCheck : rangeToCheckMap.ranges()) {
-		if (rangeToCheck.value() == true) {
-			res.push_back(rangeToCheck.range());
-		}
-	}
-	TraceEvent e(SevInfo, "ConsistencyCheckUrgent_LoadedInputRange");
-	e.setMaxEventLength(-1);
-	e.setMaxFieldLength(-1);
-	for (int i = 0; i < res.size(); i++) {
-		e.detail("RangeBegin" + std::to_string(i), res[i].begin);
-		e.detail("RangeEnd" + std::to_string(i), res[i].end);
-	}
-	return res;
-}
-
-std::unordered_map<int, std::vector<KeyRange>> makeTaskAssignment(Database cx,
-                                                                  int64_t consistencyCheckerId,
-                                                                  std::vector<KeyRange> shardsToCheck,
-                                                                  int testersCount,
-                                                                  int round) {
-	ASSERT(testersCount >= 1);
-	std::unordered_map<int, std::vector<KeyRange>> assignment;
-
-	std::vector<size_t> shuffledIndices(testersCount);
-	std::iota(shuffledIndices.begin(), shuffledIndices.end(), 0); // creates [0, 1, ..., testersCount - 1]
-	deterministicRandom()->randomShuffle(shuffledIndices);
-
-	int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_BATCH_SHARD_COUNT;
-	int startingPoint = 0;
-	if (shardsToCheck.size() > batchSize * testersCount) {
-		startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * testersCount);
-		// We randomly pick a set of successive shards:
-		// (1) We want to retry for different shards to avoid repeated failure on the same shards
-		// (2) We want to check successive shards to avoid inefficiency incurred by fragments
-	}
-	assignment.clear();
-	for (int i = startingPoint; i < shardsToCheck.size(); i++) {
-		int testerIdx = (i - startingPoint) / batchSize;
-		if (testerIdx > testersCount - 1) {
-			break; // Have filled up all testers
-		}
-		// When assigning a shards/batch to a tester idx, there are certain edge cases which can result in urgent
-		// consistency checker being infinetely stuck in a loop. Examples:
-		//      1. if there is 1 remaining shard, and tester 0 consistently fails, we will still always pick tester 0
-		//      2. if there are 10 remaining shards, and batch size is 10, and tester 0 consistently fails, we will
-		//      still always pick tester 0
-		//      3. if there are 20 remaining shards, and batch size is 10, and testers {0, 1} consistently fail, we will
-		//      keep picking testers {0, 1}
-		// To avoid repeatedly picking the same testers even though they could be failing, shuffledIndices provides an
-		// indirection to a random tester idx. That way, each invocation of makeTaskAssignment won't
-		// result in the same task assignment for the class of edge cases mentioned above.
-		assignment[shuffledIndices[testerIdx]].push_back(shardsToCheck[i]);
-	}
-	std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
-	for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
-		TraceEvent(SevInfo, "ConsistencyCheckUrgent_AssignTaskToTesters")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round)
-		    .detail("ClientId", assignIt->first)
-		    .detail("ShardsCount", assignIt->second.size());
-	}
-	return assignment;
-}
-
-Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
-                                             Database cx,
-                                             Optional<std::vector<TesterInterface>> testers,
-                                             int minTestersExpected) {
-	KeyRangeMap<bool> globalProgressMap; // used to keep track of progress
-	std::unordered_map<int, std::vector<KeyRange>> assignment; // used to keep track of assignment of tasks
-	std::vector<TesterInterface> ts; // used to store testers interface
-	std::vector<KeyRange> rangesToCheck; // get from globalProgressMap
-	std::vector<KeyRange> shardsToCheck; // get from keyServer metadata
-	Optional<double> whenFailedToGetTesterStart;
-
-	// Initialize globalProgressMap
-	Optional<std::vector<KeyRange>> rangesToCheck_ = loadRangesToCheckFromKnob();
-	if (rangesToCheck_.present()) {
-		globalProgressMap.insert(allKeys, true);
-		for (const auto& rangeToCheck : rangesToCheck_.get()) {
-			// Mark rangesToCheck as incomplete
-			// Those ranges will be checked
-			globalProgressMap.insert(rangeToCheck, false);
-		}
-		globalProgressMap.coalesce(allKeys);
-	} else {
-		TraceEvent(SevInfo, "ConsistencyCheckUrgent_FailedToLoadRangeFromKnob");
-		globalProgressMap.insert(allKeys, false);
-	}
-
-	int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
-	int retryTimes = 0;
-	int round = 0;
-
-	// Main loop
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		try {
-			// Step 1: Load ranges to check, if nothing to run, exit
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RoundBegin")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("RetryTimes", retryTimes)
-			    .detail("TesterCount", ts.size())
-			    .detail("Round", round);
-			rangesToCheck.clear();
-			for (auto& range : globalProgressMap.ranges()) {
-				if (!range.value()) { // range that is not finished
-					rangesToCheck.push_back(range.range());
-				}
-			}
-			if (rangesToCheck.size() == 0) {
-				TraceEvent(SevInfo, "ConsistencyCheckUrgent_Complete")
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("RetryTimes", retryTimes)
-				    .detail("Round", round);
-				co_return;
-			}
-
-			// Step 2: Get testers
-			ts.clear();
-			if (!testers.present()) { // In real clusters
-				bool getTesterError = false;
-				Error getTesterErr;
-
-				try {
-					ts = co_await getTesters(cc, minTestersExpected);
-					whenFailedToGetTesterStart.reset();
-				} catch (Error& e) {
-					if (e.code() == error_code_timed_out) {
-						if (!whenFailedToGetTesterStart.present()) {
-							whenFailedToGetTesterStart = now();
-						} else if (now() - whenFailedToGetTesterStart.get() > 3600 * 24) { // 1 day
-							TraceEvent(SevError, "TesterRecruitmentTimeout");
-						}
-					}
-					getTesterErr = e;
-					getTesterError = true;
-				}
-
-				if (getTesterError) {
-					throw getTesterErr;
-				}
-
-				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-					throw operation_failed(); // Introduce random failure
-				}
-			} else { // In simulation
-				ts = testers.get();
-			}
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_GotTesters")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("Round", round)
-			    .detail("RetryTimes", retryTimes)
-			    .detail("TesterCount", ts.size());
-
-			// Step 3: Load shards to check from keyserver space
-			// Shard is the unit for the task assignment
-			shardsToCheck.clear();
-			shardsToCheck = co_await getConsistencyCheckShards(cx, rangesToCheck);
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_GotShardsToCheck")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("Round", round)
-			    .detail("RetryTimes", retryTimes)
-			    .detail("TesterCount", ts.size())
-			    .detail("ShardCount", shardsToCheck.size());
-
-			// Step 4: Assign tasks to clients
-			assignment.clear();
-			assignment = makeTaskAssignment(cx, consistencyCheckerId, shardsToCheck, ts.size(), round);
-
-			// Step 5: Run checking on testers
-			std::unordered_set<int> completeClients =
-			    co_await runUrgentConsistencyCheckWorkload(cx, ts, consistencyCheckerId, assignment);
-			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-				throw operation_failed(); // Introduce random failure
-			}
-			// We use the complete client to decide which ranges are completed
-			for (const auto& clientId : completeClients) {
-				for (const auto& range : assignment[clientId]) {
-					globalProgressMap.insert(range, true); // Mark the ranges as complete
-				}
-			}
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RoundEnd")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("RetryTimes", retryTimes)
-			    .detail("SucceedTesterCount", completeClients.size())
-			    .detail("SucceedTesters", describe(completeClients))
-			    .detail("TesterCount", ts.size())
-			    .detail("Round", round);
-			round++;
-
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw e;
-			} else {
-				TraceEvent(SevInfo, "ConsistencyCheckUrgent_CoreWithRetriableFailure")
-				    .errorUnsuppressed(e)
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("RetryTimes", retryTimes)
-				    .detail("Round", round);
-				caughtError = e;
-				needsErrorHandling = true;
-			}
-		}
-
-		if (needsErrorHandling) {
-			co_await delay(10.0);
-			retryTimes++;
-		} else {
-			co_await delay(10.0); // Backoff 10 seconds for the next round
-		}
-
-		// Decide and enforce the consistencyCheckerId for the next round
-		consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
-	}
-}
-
-Future<Void> runConsistencyCheckerUrgentHolder(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
-                                               Database cx,
-                                               Optional<std::vector<TesterInterface>> testers,
-                                               int minTestersExpected,
-                                               bool repeatRun) {
-	while (true) {
-		co_await runConsistencyCheckerUrgentCore(cc, cx, testers, minTestersExpected);
-		if (!repeatRun) {
-			break;
-		}
-		co_await delay(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME);
-	}
-	co_return;
-}
-
-Future<Void> checkConsistencyUrgentSim(Database cx, std::vector<TesterInterface> testers) {
-	if (SERVER_KNOBS->DISABLE_AUDIT_STORAGE_FINAL_REPLICA_CHECK_IN_SIM) {
-		return Void();
-	}
-	return runConsistencyCheckerUrgentHolder(
-	    Reference<AsyncVar<Optional<ClusterControllerFullInterface>>>(), cx, testers, 1, /*repeatRun=*/false);
-}
-
-Future<bool> runTest(Database cx,
-                     std::vector<TesterInterface> testers,
-                     TestSpec spec,
-                     Reference<AsyncVar<ServerDBInfo>> dbInfo,
-                     TesterConsistencyScanState* consistencyScanState) {
-	DistributedTestResults testResults;
-	double savedDisableDuration = 0;
-	bool hadException = false;
-	Error caughtError;
-
-	try {
-		Future<DistributedTestResults> fTestResults = runWorkload(cx, testers, spec);
-		if (g_network->isSimulated() && spec.simConnectionFailuresDisableDuration > 0) {
-			savedDisableDuration = g_simulator->connectionFailuresDisableDuration;
-			g_simulator->connectionFailuresDisableDuration = spec.simConnectionFailuresDisableDuration;
-		}
-		if (spec.timeout > 0) {
-			fTestResults = timeoutError(fTestResults, spec.timeout);
-		}
-		DistributedTestResults _testResults = co_await fTestResults;
-		printf("Test complete\n");
-		testResults = _testResults;
-		logMetrics(testResults.metrics);
-		if (g_network->isSimulated() && savedDisableDuration > 0) {
-			g_simulator->connectionFailuresDisableDuration = savedDisableDuration;
-		}
-	} catch (Error& e) {
-		if (e.code() == error_code_timed_out) {
-			auto msg = fmt::format("Process timed out after {} seconds", spec.timeout);
-			ProcessEvents::trigger("Timeout"_sr, StringRef(msg), e);
-			TraceEvent(SevError, "TestFailure")
-			    .error(e)
-			    .detail("Reason", "Test timed out")
-			    .detail("Timeout", spec.timeout);
-			fprintf(stderr, "ERROR: Test timed out after %d seconds.\n", spec.timeout);
-			testResults.failures = testers.size();
-			testResults.successes = 0;
-		} else {
-			caughtError = e;
-			hadException = true;
-		}
-	}
-
-	if (hadException) {
-		throw caughtError;
-	}
-
-	bool ok = testResults.ok();
-
-	if (spec.useDB) {
-		printf("%d test clients passed; %d test clients failed\n", testResults.successes, testResults.failures);
-		if (spec.dumpAfterTest) {
-			Error dumpErr;
-			try {
-				co_await timeoutError(dumpDatabase(cx, "dump after " + printable(spec.title) + ".html", allKeys), 30.0);
-			} catch (Error& e) {
-				TraceEvent(SevError, "TestFailure").error(e).detail("Reason", "Unable to dump database");
-				ok = false;
-			}
-
-			co_await delay(1.0);
-		}
-
-		// Disable consistency scan before checkConsistency because otherwise it will prevent quiet database from
-		// quiescing
-		co_await checkConsistencyScanAfterTest(cx, consistencyScanState);
-		printf("Consistency scan done\n");
-
-		// Run the consistency check workload
-		if (spec.runConsistencyCheck) {
-			bool quiescent = g_network->isSimulated() ? !BUGGIFY : spec.waitForQuiescenceEnd;
-			try {
-				printf("Running urgent consistency check...\n");
-				// For testing urgent consistency check
-				co_await timeoutError(checkConsistencyUrgentSim(cx, testers), 20000.0);
-				printf("Urgent consistency check done\nRunning consistency check...\n");
-				co_await timeoutError(checkConsistency(cx,
-				                                       testers,
-				                                       quiescent,
-				                                       spec.runConsistencyCheckOnTSS,
-				                                       spec.maxDDRunTime > 0 ? spec.maxDDRunTime : 10000.0,
-				                                       5000,
-				                                       spec.databasePingDelay,
-				                                       dbInfo),
-				                      20000.0);
-				printf("Consistency check done\n");
-			} catch (Error& e) {
-				TraceEvent(SevError, "TestFailure").error(e).detail("Reason", "Unable to perform consistency check");
-				ok = false;
-			}
-
-			// Run auditStorage at the end of simulation
-			if (quiescent && g_network->isSimulated()) {
-				try {
-					TraceEvent("AuditStorageStart");
-					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateHA), 1500.0);
-					TraceEvent("AuditStorageCorrectnessHADone");
-					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateReplica), 1500.0);
-					TraceEvent("AuditStorageCorrectnessReplicaDone");
-					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateLocationMetadata), 1500.0);
-					TraceEvent("AuditStorageCorrectnessLocationMetadataDone");
-					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateStorageServerShard),
-					                      1500.0);
-					TraceEvent("AuditStorageCorrectnessStorageServerShardDone");
-				} catch (Error& e) {
-					ok = false;
-					TraceEvent(SevError, "TestFailure")
-					    .error(e)
-					    .detail("Reason", "Unable to perform auditStorage check.");
-				}
-			}
-		}
-	}
-
-	TraceEvent(ok ? SevInfo : SevWarnAlways, "TestResults").detail("Workload", spec.title).detail("Passed", (int)ok);
-	//.detail("Metrics", metricSummary);
-
-	if (ok) {
-		passCount++;
-	} else {
-		failCount++;
-	}
-
-	printf("%d test clients passed; %d test clients failed\n", testResults.successes, testResults.failures);
-
-	if (spec.useDB && spec.clearAfterTest) {
-		Error clearErr;
-		try {
-			TraceEvent("TesterClearingDatabase").log();
-			co_await timeoutError(clearData(cx), 1000.0);
-		} catch (Error& e) {
-			TraceEvent(SevError, "ErrorClearingDatabaseAfterTest").error(e);
-			throw; // If we didn't do this, we don't want any later tests to run on this DB
-		}
-
-		co_await delay(1.0);
-	}
-
-	co_return ok;
-}
-
-std::map<std::string, std::function<void(const std::string&)>> testSpecGlobalKeys = {
-	// These are read by SimulatedCluster and used before testers exist.  Thus, they must
-	// be recognized and accepted, but there's no point in placing them into a testSpec.
-	// testClass and testPriority are only used for TestHarness, we'll ignore those here
-	{ "testClass", [](std::string const&) {} },
-	{ "testPriority", [](std::string const&) {} },
-	{ "extraDatabaseMode",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseMode", ""); } },
-	{ "extraDatabaseCount",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseCount", ""); } },
-	{ "configureLocked",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedConfigureLocked", ""); } },
-	{ "minimumReplication",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMinimumReplication", ""); } },
-	{ "minimumRegions",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMinimumRegions", ""); } },
-	{ "logAntiQuorum",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedLogAntiQuorum", ""); } },
-	{ "buggify", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedBuggify", value); } },
-	// The test harness handles NewSeverity events specially.
-	{ "StderrSeverity", [](const std::string& value) { TraceEvent("StderrSeverity").detail("NewSeverity", value); } },
-	{ "ClientInfoLogging",
-	  [](const std::string& value) {
-	      if (value == "false") {
-		      setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
-	      }
-	      // else { } It is enable by default for tester
-	      TraceEvent("TestParserTest").detail("ClientInfoLogging", value);
-	  } },
-	{ "startIncompatibleProcess",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStartIncompatibleProcess", value); } },
-	{ "storageEngineExcludeTypes",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStorageEngineExcludeTypes", ""); } },
-	{ "maxTLogVersion",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMaxTLogVersion", ""); } },
-	{ "disableTss", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableTSS", ""); } },
-	{ "disableHostname",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } },
-	{ "disableRemoteKVS",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedRemoteKVS", ""); } },
-	// TODO(gglass): remove this when it's no longer used.  For now it's a useful signal in a toml file
-	// for tests against functionality that might not actually be tenant related and hence need to keep.
-	{ "allowDefaultTenant",
-	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDefaultTenant", ""); } }
-};
-
-std::map<std::string, std::function<void(const std::string& value, TestSpec* spec)>> testSpecTestKeys = {
-	{ "testTitle",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->title = value;
-	      TraceEvent("TestParserTest").detail("ParsedTest", spec->title);
-	  } },
-	{ "timeout",
-	  [](const std::string& value, TestSpec* spec) {
-	      sscanf(value.c_str(), "%d", &(spec->timeout));
-	      ASSERT(spec->timeout > 0);
-	      TraceEvent("TestParserTest").detail("ParsedTimeout", spec->timeout);
-	  } },
-	{ "databasePingDelay",
-	  [](const std::string& value, TestSpec* spec) {
-	      double databasePingDelay;
-	      sscanf(value.c_str(), "%lf", &databasePingDelay);
-	      ASSERT(databasePingDelay >= 0);
-	      if (!spec->useDB && databasePingDelay > 0) {
-		      TraceEvent(SevError, "TestParserError")
-		          .detail("Reason", "Cannot have non-zero ping delay on test that does not use database")
-		          .detail("PingDelay", databasePingDelay)
-		          .detail("UseDB", spec->useDB);
-		      ASSERT(false);
-	      }
-	      spec->databasePingDelay = databasePingDelay;
-	      TraceEvent("TestParserTest").detail("ParsedPingDelay", spec->databasePingDelay);
-	  } },
-	{ "runSetup",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->phases = TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
-	      if (value == "true")
-		      spec->phases |= TestWorkload::SETUP;
-	      TraceEvent("TestParserTest").detail("ParsedSetupFlag", (spec->phases & TestWorkload::SETUP) != 0);
-	  } },
-	{ "dumpAfterTest",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->dumpAfterTest = (value == "true");
-	      TraceEvent("TestParserTest").detail("ParsedDumpAfter", spec->dumpAfterTest);
-	  } },
-	{ "clearAfterTest",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->clearAfterTest = (value == "true");
-	      TraceEvent("TestParserTest").detail("ParsedClearAfter", spec->clearAfterTest);
-	  } },
-	{ "useDB",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->useDB = (value == "true");
-	      TraceEvent("TestParserTest").detail("ParsedUseDB", spec->useDB);
-	      if (!spec->useDB)
-		      spec->databasePingDelay = 0.0;
-	  } },
-	{ "startDelay",
-	  [](const std::string& value, TestSpec* spec) {
-	      sscanf(value.c_str(), "%lf", &spec->startDelay);
-	      TraceEvent("TestParserTest").detail("ParsedStartDelay", spec->startDelay);
-	  } },
-	{ "runConsistencyCheck",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->runConsistencyCheck = (value == "true");
-	      TraceEvent("TestParserTest").detail("ParsedRunConsistencyCheck", spec->runConsistencyCheck);
-	  } },
-	{ "runConsistencyCheckOnTSS",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->runConsistencyCheckOnTSS = (value == "true");
-	      TraceEvent("TestParserTest").detail("ParsedRunConsistencyCheckOnTSS", spec->runConsistencyCheckOnTSS);
-	  } },
-	{ "maxDDRunTime",
-	  [](const std::string& value, TestSpec* spec) {
-	      sscanf(value.c_str(), "%lf", &(spec->maxDDRunTime));
-	      ASSERT(spec->maxDDRunTime >= 0);
-	      TraceEvent("TestParserTest").detail("ParsedMaxDDRunTime", spec->maxDDRunTime);
-	  } },
-	{ "waitForQuiescence",
-	  [](const std::string& value, TestSpec* spec) {
-	      bool toWait = value == "true";
-	      spec->waitForQuiescenceBegin = toWait;
-	      spec->waitForQuiescenceEnd = toWait;
-	      TraceEvent("TestParserTest").detail("ParsedWaitForQuiescence", toWait);
-	  } },
-	{ "waitForQuiescenceBegin",
-	  [](const std::string& value, TestSpec* spec) {
-	      bool toWait = value == "true";
-	      spec->waitForQuiescenceBegin = toWait;
-	      TraceEvent("TestParserTest").detail("ParsedWaitForQuiescenceBegin", toWait);
-	  } },
-	{ "waitForQuiescenceEnd",
-	  [](const std::string& value, TestSpec* spec) {
-	      bool toWait = value == "true";
-	      spec->waitForQuiescenceEnd = toWait;
-	      TraceEvent("TestParserTest").detail("ParsedWaitForQuiescenceEnd", toWait);
-	  } },
-	{ "simCheckRelocationDuration",
-	  [](const std::string& value, TestSpec* spec) {
-	      spec->simCheckRelocationDuration = (value == "true");
-	      TraceEvent("TestParserTest").detail("ParsedSimCheckRelocationDuration", spec->simCheckRelocationDuration);
-	  } },
-	{ "connectionFailuresDisableDuration",
-	  [](const std::string& value, TestSpec* spec) {
-	      double connectionFailuresDisableDuration;
-	      sscanf(value.c_str(), "%lf", &connectionFailuresDisableDuration);
-	      ASSERT(connectionFailuresDisableDuration >= 0);
-	      spec->simConnectionFailuresDisableDuration = connectionFailuresDisableDuration;
-	      TraceEvent("TestParserTest")
-	          .detail("ParsedSimConnectionFailuresDisableDuration", spec->simConnectionFailuresDisableDuration);
-	  } },
-	{ "simBackupAgents",
-	  [](const std::string& value, TestSpec* spec) {
-	      if (value == "BackupToFile" || value == "BackupToFileAndDB")
-		      spec->simBackupAgents = ISimulator::BackupAgentType::BackupToFile;
-	      else
-		      spec->simBackupAgents = ISimulator::BackupAgentType::NoBackupAgents;
-	      TraceEvent("TestParserTest").detail("ParsedSimBackupAgents", spec->simBackupAgents);
-
-	      if (value == "BackupToDB" || value == "BackupToFileAndDB")
-		      spec->simDrAgents = ISimulator::BackupAgentType::BackupToDB;
-	      else
-		      spec->simDrAgents = ISimulator::BackupAgentType::NoBackupAgents;
-	      TraceEvent("TestParserTest").detail("ParsedSimDrAgents", spec->simDrAgents);
-	  } },
-	{ "checkOnly",
-	  [](const std::string& value, TestSpec* spec) {
-	      if (value == "true")
-		      spec->phases = TestWorkload::CHECK;
-	  } },
-	{ "restorePerpetualWiggleSetting",
-	  [](const std::string& value, TestSpec* spec) {
-	      if (value == "false")
-		      spec->restorePerpetualWiggleSetting = false;
-	  } },
-	{ "runFailureWorkloads",
-	  [](const std::string& value, TestSpec* spec) { spec->runFailureWorkloads = (value == "true"); } },
-	{ "disabledFailureInjectionWorkloads",
-	  [](const std::string& value, TestSpec* spec) {
-	      // Expects a comma separated list of workload names in "value".
-	      // This custom encoding is needed because both text and toml files need to be supported
-	      // and "value" is passed in as a string.
-	      std::stringstream ss(value);
-	      while (ss.good()) {
-		      std::string substr;
-		      getline(ss, substr, ',');
-		      substr = removeWhitespace(substr);
-		      if (!substr.empty()) {
-			      spec->disabledFailureInjectionWorkloads.push_back(substr);
-		      }
-	      }
-	  } },
-};
-
-std::vector<TestSpec> readTests(std::ifstream& ifs) {
-	TestSpec spec;
-	std::vector<TestSpec> result;
-	Standalone<VectorRef<KeyValueRef>> workloadOptions;
-	std::string cline;
-	bool beforeFirstTest = true;
-	bool parsingWorkloads = false;
-
-	while (ifs.good()) {
-		getline(ifs, cline);
-		std::string line = removeWhitespace(cline);
-		if (!line.size() || line.find(';') == 0)
-			continue;
-
-		size_t found = line.find('=');
-		if (found == std::string::npos)
-			// hmmm, not good
-			continue;
-		std::string attrib = removeWhitespace(line.substr(0, found));
-		std::string value = removeWhitespace(line.substr(found + 1));
-
-		if (attrib == "testTitle") {
-			beforeFirstTest = false;
-			parsingWorkloads = false;
-			if (workloadOptions.size()) {
-				spec.options.push_back_deep(spec.options.arena(), workloadOptions);
-				workloadOptions = Standalone<VectorRef<KeyValueRef>>();
-			}
-			if (spec.options.size() && spec.title.size()) {
-				result.push_back(spec);
-				spec = TestSpec();
-			}
-
-			testSpecTestKeys[attrib](value, &spec);
-		} else if (testSpecTestKeys.find(attrib) != testSpecTestKeys.end()) {
-			if (parsingWorkloads)
-				TraceEvent(SevError, "TestSpecTestParamInWorkload").detail("Attrib", attrib).detail("Value", value);
-			testSpecTestKeys[attrib](value, &spec);
-		} else if (testSpecGlobalKeys.find(attrib) != testSpecGlobalKeys.end()) {
-			if (!beforeFirstTest)
-				TraceEvent(SevError, "TestSpecGlobalParamInTest").detail("Attrib", attrib).detail("Value", value);
-			testSpecGlobalKeys[attrib](value);
-		} else {
-			if (attrib == "testName") {
-				parsingWorkloads = true;
-				if (workloadOptions.size()) {
-					TraceEvent("TestParserFlush").detail("Reason", "new (compound) test");
-					spec.options.push_back_deep(spec.options.arena(), workloadOptions);
-					workloadOptions = Standalone<VectorRef<KeyValueRef>>();
-				}
-			}
-
-			workloadOptions.push_back_deep(workloadOptions.arena(), KeyValueRef(StringRef(attrib), StringRef(value)));
-			TraceEvent("TestParserOption").detail("ParsedKey", attrib).detail("ParsedValue", value);
-		}
-	}
-	if (workloadOptions.size())
-		spec.options.push_back_deep(spec.options.arena(), workloadOptions);
-	if (spec.options.size() && spec.title.size()) {
-		result.push_back(spec);
-	}
-
-	return result;
-}
-
-template <typename T>
-std::string toml_to_string(const T& value) {
-	// TOML formatting converts numbers to strings exactly how they're in the file
-	// and thus, is equivalent to testspec.  However, strings are quoted, so we
-	// must remove the quotes.
-	if (value.type() == toml::value_t::string) {
-		const std::string& formatted = toml::format(value);
-		return formatted.substr(1, formatted.size() - 2);
-	} else {
-		return toml::format(value);
-	}
-}
-
-struct TestSet {
-	KnobKeyValuePairs overrideKnobs;
-	std::vector<TestSpec> testSpecs;
-};
-
-namespace {
-
-// In the current TOML scope, look for "knobs" field. If exists, translate all
-// key value pairs into KnobKeyValuePairs
-KnobKeyValuePairs getOverriddenKnobKeyValues(const toml::value& context) {
-	KnobKeyValuePairs result;
-
-	try {
-		const toml::array& overrideKnobs = toml::find(context, "knobs").as_array();
-		for (const toml::value& knob : overrideKnobs) {
-			for (const auto& [key, value_] : knob.as_table()) {
-				const std::string& value = toml_to_string(value_);
-				ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value);
-				if (std::get_if<NoKnobFound>(&parsedValue)) {
-					parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
-				}
-				if (std::get_if<NoKnobFound>(&parsedValue)) {
-					parsedValue = FLOW_KNOBS->parseKnobValue(key, value);
-				}
-				if (std::get_if<NoKnobFound>(&parsedValue)) {
-					TraceEvent(SevError, "TestSpecUnrecognizedKnob")
-					    .detail("KnobName", key)
-					    .detail("OverrideValue", value);
-					continue;
-				}
-				result.set(key, parsedValue);
-			}
-		}
-	} catch (const std::out_of_range&) {
-		// No knobs field in this scope, this is not an error
-	}
-
-	return result;
-}
-
-} // namespace
-
-TestSet readTOMLTests_(std::string fileName) {
-	Standalone<VectorRef<KeyValueRef>> workloadOptions;
-	TestSet result;
-
-	const toml::value& conf = toml::parse(fileName);
-
-	// Parse the global knob changes
-	result.overrideKnobs = getOverriddenKnobKeyValues(conf);
-
-	// Then parse each test
-	const toml::array& tests = toml::find(conf, "test").as_array();
-	for (const toml::value& test : tests) {
-		TestSpec spec;
-
-		// First handle all test-level settings
-		for (const auto& [k, v] : test.as_table()) {
-			if (k == "workload" || k == "knobs") {
-				continue;
-			}
-			if (testSpecTestKeys.find(k) != testSpecTestKeys.end()) {
-				testSpecTestKeys[k](toml_to_string(v), &spec);
-			} else {
-				TraceEvent(SevError, "TestSpecUnrecognizedTestParam")
-				    .detail("Attrib", k)
-				    .detail("Value", toml_to_string(v));
-			}
-		}
-
-		// And then copy the workload attributes to spec.options
-		const toml::array& workloads = toml::find(test, "workload").as_array();
-		for (const toml::value& workload : workloads) {
-			workloadOptions = Standalone<VectorRef<KeyValueRef>>();
-			TraceEvent("TestParserFlush").detail("Reason", "new (compound) test");
-			for (const auto& [attrib, v] : workload.as_table()) {
-				const std::string& value = toml_to_string(v);
-				workloadOptions.push_back_deep(workloadOptions.arena(),
-				                               KeyValueRef(StringRef(attrib), StringRef(value)));
-				TraceEvent("TestParserOption").detail("ParsedKey", attrib).detail("ParsedValue", value);
-			}
-			spec.options.push_back_deep(spec.options.arena(), workloadOptions);
-		}
-
-		// And then copy the knob attributes to spec.overrideKnobs
-		spec.overrideKnobs = getOverriddenKnobKeyValues(test);
-
-		result.testSpecs.push_back(spec);
-	}
-
-	return result;
-}
-
-// A hack to catch and log std::exception, because TOML11 has very useful
-// error messages, but the actor framework can't handle std::exception.
-TestSet readTOMLTests(std::string fileName) {
-	try {
-		return readTOMLTests_(fileName);
-	} catch (std::exception& e) {
-		std::cerr << e.what() << std::endl;
-		TraceEvent("TOMLParseError").detail("Error", printable(e.what()));
-		// TODO: replace with toml_parse_error();
-		throw unknown_error();
-	}
-}
-
-Future<Void> monitorServerDBInfo(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> ccInterface,
-                                 LocalityData locality,
-                                 Reference<AsyncVar<ServerDBInfo>> dbInfo) {
-	// Initially most of the serverDBInfo is not known, but we know our locality right away
-	ServerDBInfo localInfo;
-	localInfo.myLocality = locality;
-	dbInfo->set(localInfo);
-
-	while (true) {
-		GetServerDBInfoRequest req;
-		req.knownServerInfoID = dbInfo->get().id;
-
-		Future<ServerDBInfo> getInfoReq =
-		    ccInterface->get().present() ? brokenPromiseToNever(ccInterface->get().get().getServerDBInfo.getReply(req))
-		                                 : Future<ServerDBInfo>(Never());
-		Future<Void> ccChange = ccInterface->onChange();
-
-		int action = 0;
-		ServerDBInfo gotInfo;
-
-		co_await Choose()
-		    .When(getInfoReq,
-		          [&](ServerDBInfo const& info) {
-			          gotInfo = info;
-			          action = 1;
-		          })
-		    .When(ccChange, [&](Void const&) { action = 2; })
-		    .run();
-
-		if (action == 1) {
-			TraceEvent("GotServerDBInfoChange")
-			    .detail("ChangeID", gotInfo.id)
-			    .detail("MasterID", gotInfo.master.id())
-			    .detail("RatekeeperID", gotInfo.ratekeeper.present() ? gotInfo.ratekeeper.get().id() : UID())
-			    .detail("DataDistributorID", gotInfo.distributor.present() ? gotInfo.distributor.get().id() : UID());
-
-			gotInfo.myLocality = locality;
-			dbInfo->set(gotInfo);
-		} else if (action == 2) {
-			if (ccInterface->get().present())
-				TraceEvent("GotCCInterfaceChange")
-				    .detail("CCID", ccInterface->get().get().id())
-				    .detail("CCMachine", ccInterface->get().get().getWorkers.getEndpoint().getPrimaryAddress());
-		}
-	}
-}
-
-Future<Void> initializeSimConfig(Database db) {
-	Transaction tr(db);
-	ASSERT(g_network->isSimulated());
-	while (true) {
-		bool needsErrorHandling = false;
-		Error caughtError;
-
-		try {
-			DatabaseConfiguration dbConfig = co_await getDatabaseConfiguration(&tr);
-			g_simulator->storagePolicy = dbConfig.storagePolicy;
-			g_simulator->tLogPolicy = dbConfig.tLogPolicy;
-			g_simulator->tLogWriteAntiQuorum = dbConfig.tLogWriteAntiQuorum;
-			g_simulator->usableRegions = dbConfig.usableRegions;
-
-			// If the same region is being shared between the remote and a satellite, then our simulated policy checking
-			// may fail to account for the total number of needed machines when deciding what can be killed. To work
-			// around this, we increase the required transaction logs in the remote policy to include the number of
-			// satellite logs that may get recruited there
-			bool foundSharedDcId = false;
-			std::set<Key> dcIds;
-			int maxSatelliteReplication = 0;
-			for (auto const& r : dbConfig.regions) {
-				if (!dcIds.insert(r.dcId).second) {
-					foundSharedDcId = true;
-				}
-				if (!r.satellites.empty() && r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogUsableDcs > 0) {
-					for (auto const& s : r.satellites) {
-						if (!dcIds.insert(s.dcId).second) {
-							foundSharedDcId = true;
-						}
-					}
-
-					maxSatelliteReplication =
-					    std::max(maxSatelliteReplication, r.satelliteTLogReplicationFactor / r.satelliteTLogUsableDcs);
-				}
-			}
-
-			if (foundSharedDcId) {
-				int totalRequired = std::max(dbConfig.tLogReplicationFactor, dbConfig.remoteTLogReplicationFactor) +
-				                    maxSatelliteReplication;
-				g_simulator->remoteTLogPolicy = Reference<IReplicationPolicy>(
-				    new PolicyAcross(totalRequired, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
-				TraceEvent("ChangingSimTLogPolicyForSharedRemote")
-				    .detail("TotalRequired", totalRequired)
-				    .detail("MaxSatelliteReplication", maxSatelliteReplication)
-				    .detail("ActualPolicy", dbConfig.getRemoteTLogPolicy()->info())
-				    .detail("SimulatorPolicy", g_simulator->remoteTLogPolicy->info());
-			} else {
-				g_simulator->remoteTLogPolicy = dbConfig.getRemoteTLogPolicy();
-			}
-
-			co_return;
-		} catch (Error& e) {
-			caughtError = e;
-			needsErrorHandling = true;
-		}
-
-		if (needsErrorHandling) {
-			co_await tr.onError(caughtError);
-		}
-	}
-}
-
-// Disables connection failures after the given time seconds
-Future<Void> disableConnectionFailuresAfter(double seconds, std::string context) {
-	if (g_network->isSimulated()) {
-		TraceEvent(SevWarnAlways, ("ScheduleDisableConnectionFailures_" + context).c_str())
-		    .detail("At", now() + seconds);
-		co_await delay(seconds);
-		while (true) {
-			double delaySeconds = disableConnectionFailures(context, ForceDisable::False);
-			if (delaySeconds > DISABLE_CONNECTION_FAILURE_MIN_INTERVAL) {
-				co_await delay(delaySeconds);
-			} else {
-				// disableConnectionFailures will always take effect if less than
-				// DISABLE_CONNECTION_FAILURE_MIN_INTERVAL is returned.
-				break;
-			}
-		}
-	}
-	co_return;
-}
-
-/**
- * \brief Test orchestrator: sends test specification to testers in the right order and collects the results.
- *
- * There are multiple actors in this file with similar names (runTest, runTests) and slightly different signatures.
- *
- * This is the actual orchestrator. It reads the test specifications (from tests), prepares the cluster (by running the
- * configure command given in startingConfiguration) and then runs the workload.
- *
- * \param cc The cluster controller interface
- * \param ci Same as cc.clientInterface
- * \param testers The interfaces of the testers that should run the actual workloads
- * \param tests The test specifications to run
- * \param startingConfiguration If non-empty, the orchestrator will attempt to set this configuration before starting
- * the tests.
- * \param locality client locality (it seems this is unused?)
- *
- * \returns A future which will be set after all tests finished.
- */
-Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> cc,
-                      Reference<AsyncVar<Optional<struct ClusterInterface>>> ci,
-                      std::vector<TesterInterface> testers,
-                      std::vector<TestSpec> tests,
-                      StringRef startingConfiguration,
-                      LocalityData locality,
-                      bool restartingTest) {
-	Database cx;
-	Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
-	Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
-
-	bool useDB = false;
-	bool waitForQuiescenceBegin = false;
-	bool waitForQuiescenceEnd = false;
-	bool restorePerpetualWiggleSetting = false;
-	bool perpetualWiggleEnabled = false;
-	bool backupWorkerEnabled = false;
-	double startDelay = 0.0;
-	double databasePingDelay = 1e9;
-	ISimulator::BackupAgentType simBackupAgents = ISimulator::BackupAgentType::NoBackupAgents;
-	ISimulator::BackupAgentType simDrAgents = ISimulator::BackupAgentType::NoBackupAgents;
-	bool enableDD = false;
-	TesterConsistencyScanState consistencyScanState;
-
-	// Gives chance for g_network->run() to run inside event loop and hence let
-	// the tests see correct value for `isOnMainThread()`.
-	co_await yield();
-
-	if (tests.empty())
-		useDB = true;
-	for (auto iter = tests.begin(); iter != tests.end(); ++iter) {
-		if (iter->useDB)
-			useDB = true;
-		if (iter->waitForQuiescenceBegin)
-			waitForQuiescenceBegin = true;
-		if (iter->waitForQuiescenceEnd)
-			waitForQuiescenceEnd = true;
-		if (iter->restorePerpetualWiggleSetting)
-			restorePerpetualWiggleSetting = true;
-		startDelay = std::max(startDelay, iter->startDelay);
-		databasePingDelay = std::min(databasePingDelay, iter->databasePingDelay);
-		if (iter->simBackupAgents != ISimulator::BackupAgentType::NoBackupAgents)
-			simBackupAgents = iter->simBackupAgents;
-
-		if (iter->simDrAgents != ISimulator::BackupAgentType::NoBackupAgents) {
-			simDrAgents = iter->simDrAgents;
-		}
-		enableDD = enableDD || getOption(iter->options[0], "enableDD"_sr, false);
-	}
-
-	if (g_network->isSimulated()) {
-		g_simulator->backupAgents = simBackupAgents;
-		g_simulator->drAgents = simDrAgents;
-	}
-
-	// turn off the database ping functionality if the suite of tests are not going to be using the database
-	if (!useDB)
-		databasePingDelay = 0.0;
-
-	if (useDB) {
-		cx = openDBOnServer(dbInfo);
-	}
-
-	consistencyScanState.enabled = g_network->isSimulated() && deterministicRandom()->coinflip();
-	consistencyScanState.waitForComplete =
-	    consistencyScanState.enabled && waitForQuiescenceEnd && deterministicRandom()->coinflip();
-	consistencyScanState.enableAfter = consistencyScanState.waitForComplete && deterministicRandom()->random01() < 0.1;
-
-	disableConnectionFailures("Tester");
-
-	// Change the configuration (and/or create the database) if necessary
-	printf("startingConfiguration:%s start\n", startingConfiguration.toString().c_str());
-	fmt::print("useDB: {}\n", useDB);
-	printSimulatedTopology();
-	if (useDB && startingConfiguration != StringRef()) {
-		Error configErr;
-		try {
-			co_await timeoutError(changeConfiguration(cx, testers, startingConfiguration), 2000.0);
-			if (g_network->isSimulated() && enableDD) {
-				co_await success(setDDMode(cx, 1));
-			}
-		} catch (Error& e) {
-			TraceEvent(SevError, "TestFailure").error(e).detail("Reason", "Unable to set starting configuration");
-		}
-		std::string_view confView(reinterpret_cast<const char*>(startingConfiguration.begin()),
-		                          startingConfiguration.size());
-		if (restorePerpetualWiggleSetting) {
-			const std::string setting = "perpetual_storage_wiggle:=";
-			auto pos = confView.find(setting);
-			if (pos != confView.npos && confView.at(pos + setting.size()) == '1') {
-				perpetualWiggleEnabled = true;
-			}
-		}
-		const std::string bwSetting = "backup_worker_enabled:=";
-		auto pos = confView.find(bwSetting);
-		if (pos != confView.npos && confView.at(pos + bwSetting.size()) == '1') {
-			backupWorkerEnabled = true;
-		}
-	}
-
-	// Read cluster configuration
-	if (useDB && g_network->isSimulated()) {
-		DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
-
-		g_simulator->storagePolicy = configuration.storagePolicy;
-		g_simulator->tLogPolicy = configuration.tLogPolicy;
-		g_simulator->tLogWriteAntiQuorum = configuration.tLogWriteAntiQuorum;
-		g_simulator->remoteTLogPolicy = configuration.remoteTLogPolicy;
-		g_simulator->usableRegions = configuration.usableRegions;
-		if (configuration.regions.size() > 0) {
-			g_simulator->primaryDcId = configuration.regions[0].dcId;
-			g_simulator->hasSatelliteReplication = configuration.regions[0].satelliteTLogReplicationFactor > 0;
-			if (configuration.regions[0].satelliteTLogUsableDcsFallback > 0) {
-				g_simulator->satelliteTLogPolicyFallback = configuration.regions[0].satelliteTLogPolicyFallback;
-				g_simulator->satelliteTLogWriteAntiQuorumFallback =
-				    configuration.regions[0].satelliteTLogWriteAntiQuorumFallback;
-			} else {
-				g_simulator->satelliteTLogPolicyFallback = configuration.regions[0].satelliteTLogPolicy;
-				g_simulator->satelliteTLogWriteAntiQuorumFallback =
-				    configuration.regions[0].satelliteTLogWriteAntiQuorum;
-			}
-			g_simulator->satelliteTLogPolicy = configuration.regions[0].satelliteTLogPolicy;
-			g_simulator->satelliteTLogWriteAntiQuorum = configuration.regions[0].satelliteTLogWriteAntiQuorum;
-
-			for (auto s : configuration.regions[0].satellites) {
-				g_simulator->primarySatelliteDcIds.push_back(s.dcId);
-			}
-		} else {
-			g_simulator->hasSatelliteReplication = false;
-			g_simulator->satelliteTLogWriteAntiQuorum = 0;
-		}
-
-		if (configuration.regions.size() == 2) {
-			g_simulator->remoteDcId = configuration.regions[1].dcId;
-			ASSERT((!configuration.regions[0].satelliteTLogPolicy && !configuration.regions[1].satelliteTLogPolicy) ||
-			       configuration.regions[0].satelliteTLogPolicy->info() ==
-			           configuration.regions[1].satelliteTLogPolicy->info());
-
-			for (auto s : configuration.regions[1].satellites) {
-				g_simulator->remoteSatelliteDcIds.push_back(s.dcId);
-			}
-		}
-
-		if (restartingTest || g_simulator->usableRegions < 2 || !g_simulator->hasSatelliteReplication) {
-			g_simulator->allowLogSetKills = false;
-		}
-
-		ASSERT(g_simulator->storagePolicy && g_simulator->tLogPolicy);
-		ASSERT(!g_simulator->hasSatelliteReplication || g_simulator->satelliteTLogPolicy);
-
-		// Randomly inject custom shard configuration
-		// TODO:  Move this to a workload representing non-failure behaviors which can be randomly added to any test
-		// run.
-		if (deterministicRandom()->random01() < 0.25) {
-			co_await customShardConfigWorkload(cx);
-		}
-	}
-
-	if (useDB) {
-		if (g_network->isSimulated()) {
-			co_await initializeSimConfig(cx);
-		}
-	}
-
-	if (useDB && waitForQuiescenceBegin) {
-		TraceEvent("TesterStartingPreTestChecks")
-		    .detail("DatabasePingDelay", databasePingDelay)
-		    .detail("StartDelay", startDelay);
-
-		Error quietDbErr;
-		try {
-			co_await (quietDatabase(cx, dbInfo, "Start") ||
-			          (databasePingDelay == 0.0
-			               ? Never()
-			               : testDatabaseLiveness(cx, databasePingDelay, "QuietDatabaseStart", startDelay)));
-		} catch (Error& e) {
-			TraceEvent("QuietDatabaseStartExternalError").error(e);
-			throw;
-		}
-
-		if (perpetualWiggleEnabled) { // restore the enabled perpetual storage wiggle setting
-			printf("Set perpetual_storage_wiggle=1 ...\n");
-			Version cVer = co_await setPerpetualStorageWiggle(cx, true, LockAware::True);
-			(void)cVer;
-			printf("Set perpetual_storage_wiggle=1 Done.\n");
-		}
-
-		if (backupWorkerEnabled) {
-			printf("Enabling backup worker ...\n");
-			co_await enableBackupWorker(cx);
-			printf("Enabled backup worker.\n");
-		}
-
-		// TODO: Move this to a BehaviorInjection workload once that concept exists.
-		if (consistencyScanState.enabled && !consistencyScanState.enableAfter) {
-			printf("Enabling consistency scan ...\n");
-			co_await enableConsistencyScanInSim(cx);
-			printf("Enabled consistency scan.\n");
-		}
-	}
-
-	// Use the first test's connectionFailuresDisableDuration if set
-	double connectionFailuresDisableDuration = 0.0;
-	if (!tests.empty() && tests[0].simConnectionFailuresDisableDuration > 0) {
-		connectionFailuresDisableDuration = tests[0].simConnectionFailuresDisableDuration;
-	}
-
-	// Must be at function scope so the background actor isn't cancelled when the else block exits.
-	// In the original ACTOR code, `state` hoists variables to actor scope regardless of block scope.
-	Future<Void> disabler;
-	if (connectionFailuresDisableDuration > 0) {
-		// Disable connection failures with specified duration
-		disableConnectionFailures("Tester", ForceDisable::True, connectionFailuresDisableDuration);
-	} else {
-		enableConnectionFailures("Tester", FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS);
-		disabler = disableConnectionFailuresAfter(FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, "Tester");
-	}
-	Future<Void> repairDataCenter;
-	if (useDB) {
-		// Keep datacenter repair at the default duration regardless of connection failures setting
-		Future<Void> reconfigure = reconfigureAfter(cx, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, dbInfo, "Tester");
-		repairDataCenter = reconfigure;
-	}
-
-	TraceEvent("TestsExpectedToPass").detail("Count", tests.size());
-	int idx = 0;
-	std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup;
-	for (; idx < tests.size(); idx++) {
-		printf("Run test:%s start\n", tests[idx].title.toString().c_str());
-		knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(tests[idx].overrideKnobs);
-		co_await success(runTest(cx, testers, tests[idx], dbInfo, &consistencyScanState));
-		knobProtectiveGroup.reset(nullptr);
-		printf("Run test:%s Done.\n", tests[idx].title.toString().c_str());
-		// do we handle a failure here?
-	}
-
-	printf("\n%d tests passed; %d tests failed.\n", passCount, failCount);
-
-	// If the database was deleted during the workload we need to recreate the database
-	if (tests.empty() || useDB) {
-		if (waitForQuiescenceEnd) {
-			printf("Waiting for DD to end...\n");
-			TraceEvent("QuietDatabaseEndStart");
-			try {
-				TraceEvent("QuietDatabaseEndWait");
-				Future<Void> waitConsistencyScanEnd = checkConsistencyScanAfterTest(cx, &consistencyScanState);
-				Future<Void> waitQuietDatabaseEnd =
-				    quietDatabase(cx, dbInfo, "End", 0, 2e6, 2e6) ||
-				    (databasePingDelay == 0.0 ? Never()
-				                              : testDatabaseLiveness(cx, databasePingDelay, "QuietDatabaseEnd"));
-
-				co_await (waitConsistencyScanEnd && waitQuietDatabaseEnd);
-			} catch (Error& e) {
-				TraceEvent("QuietDatabaseEndExternalError").error(e);
-				throw;
-			}
-		}
-	}
-	printf("\n");
-
-	co_return;
-}
-
-/**
- * \brief Proxy function that waits until enough testers are available and then calls into the orchestrator.
- *
- * There are multiple actors in this file with similar names (runTest, runTests) and slightly different signatures.
- *
- * This actor wraps the actual orchestrator (also called runTests). But before calling that actor, it waits for enough
- * testers to come up.
- *
- * \param cc The cluster controller interface
- * \param ci Same as cc.clientInterface
- * \param tests The test specifications to run
- * \param minTestersExpected The number of testers to expect. This actor will block until it can find this many testers.
- * \param startingConfiguration If non-empty, the orchestrator will attempt to set this configuration before starting
- * the tests.
- * \param locality client locality (it seems this is unused?)
- *
- * \returns A future which will be set after all tests finished.
- */
-Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> cc,
-                      Reference<AsyncVar<Optional<struct ClusterInterface>>> ci,
-                      std::vector<TestSpec> tests,
-                      test_location_t at,
-                      int minTestersExpected,
-                      StringRef startingConfiguration,
-                      LocalityData locality,
-                      bool restartingTest) {
-	int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) |
-	            GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
-	Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
-	std::vector<WorkerDetails> workers;
-
-	while (true) {
-		Future<std::vector<WorkerDetails>> getWorkersReq =
-		    cc->get().present() ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
-		                        : Future<std::vector<WorkerDetails>>(Never());
-		Future<Void> ccChange = cc->onChange();
-
-		int action = 0;
-		std::vector<WorkerDetails> gotWorkers;
-
-		co_await Choose()
-		    .When(getWorkersReq,
-		          [&](std::vector<WorkerDetails> const& w) {
-			          gotWorkers = w;
-			          action = 1;
-		          })
-		    .When(ccChange, [&](Void const&) { action = 2; })
-		    .When(testerTimeout, [&](Void const&) { action = 3; })
-		    .run();
-
-		if (action == 1) {
-			if (static_cast<int>(gotWorkers.size()) >= minTestersExpected) {
-				workers = gotWorkers;
-				break;
-			}
-			co_await delay(SERVER_KNOBS->WORKER_POLL_DELAY);
-		} else if (action == 3) {
-			TraceEvent(SevError, "TesterRecruitmentTimeout").log();
-			throw timed_out();
-		}
-		// action == 2: cc changed, loop again
-	}
-
-	std::vector<TesterInterface> ts;
-	ts.reserve(workers.size());
-	for (int i = 0; i < workers.size(); i++)
-		ts.push_back(workers[i].interf.testerInterface);
-
-	co_await runTests(cc, ci, ts, tests, startingConfiguration, locality, restartingTest);
-	co_return;
-}
-
-/**
- * \brief Set up testing environment and run the given tests on a cluster.
- *
- * There are multiple actors in this file with similar names (runTest, runTests) and slightly different signatures.
- *
- * This actor is usually the first entry point into the test environment. It itself doesn't implement too much
- * functionality. Its main purpose is to generate the test specification from passed arguments and then call into the
- * correct actor which will orchestrate the actual test.
- *
- * \param connRecord A cluster connection record. Not all tests require a functional cluster but all tests require
- * a cluster record.
- * \param whatToRun TEST_TYPE_FROM_FILE to read the test description from a passed toml file or
- * TEST_TYPE_CONSISTENCY_CHECK to generate a test spec for consistency checking
- * \param at TEST_HERE: this process will act as a test client and execute the given workload. TEST_ON_SERVERS: Run a
- * test client on every worker in the cluster. TEST_ON_TESTERS: Run a test client on all servers with class Test
- * \param minTestersExpected In at is not TEST_HERE, this will instruct the orchestrator until it can find at least
- * minTestersExpected test-clients. This is usually passed through from a command line argument. In simulation, the
- * simulator will pass the number of testers that it started.
- * \param fileName The path to the toml-file containing the test description. Is ignored if whatToRun !=
- * TEST_TYPE_FROM_FILE
- * \param startingConfiguration Can be used to configure a cluster before running the test. If this is an empty string,
- * it will be ignored, otherwise it will be passed to changeConfiguration.
- * \param locality The client locality to be used. This is only used if at == TEST_HERE
- *
- * \returns A future which will be set after all tests finished.
- */
-Future<Void> runTests(Reference<IClusterConnectionRecord> const& connRecord,
-                      test_type_t const& whatToRun,
-                      test_location_t const& at,
-                      int const& minTestersExpected,
-                      std::string const& fileName,
-                      StringRef const& startingConfiguration,
-                      LocalityData const& locality,
-                      UnitTestParameters const& testOptions,
-                      bool const& restartingTest) {
-	// C++20 coroutine safety: copy parameters that might bind to temporaries (default args).
-	// const& parameters only store the reference in the coroutine frame; temporaries are
-	// destroyed after the first suspend point, leaving dangling references.
-	Reference<IClusterConnectionRecord> connRecordCopy = connRecord;
-	test_type_t whatToRunCopy = whatToRun;
-	test_location_t atCopy = at;
-	int minTestersExpectedCopy = minTestersExpected;
-	std::string fileNameCopy = fileName;
-	StringRef startingConfigurationCopy = startingConfiguration;
-	LocalityData localityCopy = locality;
-	UnitTestParameters testOptionsCopy = testOptions;
-	bool restartingTestCopy = restartingTest;
-
-	TestSet testSet;
-	std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup(nullptr);
-	auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
-	auto ci = makeReference<AsyncVar<Optional<ClusterInterface>>>();
-	std::vector<Future<Void>> actors;
-	if (connRecordCopy) {
-		actors.push_back(reportErrors(monitorLeader(connRecordCopy, cc), "MonitorLeader"));
-		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
-	}
-
-	if (whatToRunCopy == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
-		// Need not to set spec here. Will set spec when triggering workload
-	} else if (whatToRunCopy == TEST_TYPE_CONSISTENCY_CHECK) {
-		TestSpec spec;
-		Standalone<VectorRef<KeyValueRef>> options;
-		spec.title = "ConsistencyCheck"_sr;
-		spec.runFailureWorkloads = false;
-		spec.databasePingDelay = 0;
-		spec.timeout = 0;
-		spec.waitForQuiescenceBegin = false;
-		spec.waitForQuiescenceEnd = false;
-		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
-		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ConsistencyCheck"_sr));
-		options.push_back_deep(options.arena(), KeyValueRef("performQuiescentChecks"_sr, "false"_sr));
-		options.push_back_deep(options.arena(), KeyValueRef("distributed"_sr, "false"_sr));
-		options.push_back_deep(options.arena(), KeyValueRef("failureIsError"_sr, "true"_sr));
-		options.push_back_deep(options.arena(), KeyValueRef("indefinite"_sr, "true"_sr));
-		options.push_back_deep(options.arena(), KeyValueRef("rateLimitMax"_sr, StringRef(rateLimitMax)));
-		options.push_back_deep(options.arena(), KeyValueRef("shuffleShards"_sr, "true"_sr));
-		spec.options.push_back_deep(spec.options.arena(), options);
-		testSet.testSpecs.push_back(spec);
-	} else if (whatToRunCopy == TEST_TYPE_UNIT_TESTS) {
-		TestSpec spec;
-		Standalone<VectorRef<KeyValueRef>> options;
-		spec.title = "UnitTests"_sr;
-		spec.startDelay = 0;
-		spec.useDB = false;
-		spec.timeout = 0;
-		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "UnitTests"_sr));
-		options.push_back_deep(options.arena(), KeyValueRef("testsMatching"_sr, fileNameCopy));
-		// Add unit test options as test spec options
-		for (auto& kv : testOptionsCopy.params) {
-			options.push_back_deep(options.arena(), KeyValueRef(kv.first, kv.second));
-		}
-		spec.options.push_back_deep(spec.options.arena(), options);
-		testSet.testSpecs.push_back(spec);
-	} else {
-		std::ifstream ifs;
-		ifs.open(fileNameCopy.c_str(), std::ifstream::in);
-		if (!ifs.good()) {
-			TraceEvent(SevError, "TestHarnessFail")
-			    .detail("Reason", "file open failed")
-			    .detail("File", fileNameCopy.c_str());
-			fprintf(stderr, "ERROR: Could not open file `%s'\n", fileNameCopy.c_str());
-			co_return;
-		}
-		enableClientInfoLogging(); // Enable Client Info logging by default for tester
-		if (fileNameCopy.ends_with(".txt")) {
-			testSet.testSpecs = readTests(ifs);
-		} else if (fileNameCopy.ends_with(".toml")) {
-			// TOML is weird about opening the file as binary on windows, so we
-			// just let TOML re-open the file instead of using ifs.
-			testSet = readTOMLTests(fileNameCopy);
-		} else {
-			TraceEvent(SevError, "TestHarnessFail")
-			    .detail("Reason", "unknown tests specification extension")
-			    .detail("File", fileNameCopy.c_str());
-			co_return;
-		}
-		ifs.close();
-	}
-
-	knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(testSet.overrideKnobs);
-	Future<Void> tests;
-	// These must be at function scope so they aren't destroyed when the if-block exits.
-	// In the original ACTOR code, `state` hoists variables to actor scope regardless of block scope.
-	Database urgentCx;
-	Reference<AsyncVar<ServerDBInfo>> urgentDbInfo;
-	Future<Void> urgentCcMonitor;
-	if (whatToRunCopy == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
-		urgentDbInfo = makeReference<AsyncVar<ServerDBInfo>>();
-		urgentCcMonitor = monitorServerDBInfo(cc, LocalityData(), urgentDbInfo); // FIXME: locality
-		urgentCx = openDBOnServer(urgentDbInfo);
-		tests = reportErrors(
-		    runConsistencyCheckerUrgentHolder(
-		        cc, urgentCx, Optional<std::vector<TesterInterface>>(), minTestersExpectedCopy, /*repeatRun=*/true),
-		    "runConsistencyCheckerUrgentHolder");
-	} else if (atCopy == TEST_HERE) {
-		auto db = makeReference<AsyncVar<ServerDBInfo>>();
-		std::vector<TesterInterface> iTesters(1);
-		actors.push_back(
-		    reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
-		actors.push_back(
-		    reportErrors(testerServerCore(iTesters[0], connRecordCopy, db, localityCopy), "TesterServerCore"));
-		tests =
-		    runTests(cc, ci, iTesters, testSet.testSpecs, startingConfigurationCopy, localityCopy, restartingTestCopy);
-	} else {
-		tests = reportErrors(runTests(cc,
-		                              ci,
-		                              testSet.testSpecs,
-		                              atCopy,
-		                              minTestersExpectedCopy,
-		                              startingConfigurationCopy,
-		                              localityCopy,
-		                              restartingTestCopy),
-		                     "RunTests");
-	}
-
-	Future<Void> actorsFuture = actors.empty() ? Never() : waitForAll(actors);
-	co_await Choose()
-	    .When(tests, [](Void const&) {})
-	    .When(actorsFuture,
-	          [](Void const&) {
-		          ASSERT(false);
-		          throw internal_error();
-	          })
-	    .run();
-}
-
-namespace {
-Future<Void> testExpectedErrorImpl(Future<Void> test,
-                                   const char* testDescr,
-                                   Optional<Error> expectedError,
-                                   Optional<bool*> successFlag,
-                                   std::map<std::string, std::string> details,
-                                   Optional<Error> throwOnError,
-                                   UID id) {
-	Error actualError;
-	bool hadError = false;
-
-	try {
-		co_await test;
-	} catch (Error& e) {
-		if (e.code() == error_code_actor_cancelled) {
-			throw e;
-		}
-		actualError = e;
-		hadError = true;
-		// The test failed as expected
-		if (!expectedError.present() || actualError.code() == expectedError.get().code()) {
-			co_return;
-		}
-	}
-
-	// The test has failed
-	if (successFlag.present()) {
-		*(successFlag.get()) = false;
-	}
-	TraceEvent evt(SevError, "TestErrorFailed", id);
-	evt.detail("TestDescription", testDescr);
-	if (expectedError.present()) {
-		evt.detail("ExpectedError", expectedError.get().name());
-		evt.detail("ExpectedErrorCode", expectedError.get().code());
-	}
-	if (hadError && actualError.isValid()) {
-		evt.detail("ActualError", actualError.name());
-		evt.detail("ActualErrorCode", actualError.code());
-	} else {
-		evt.detail("Reason", "Unexpected success");
-	}
-
-	// Make sure that no duplicate details were provided
-	ASSERT(!details.contains("TestDescription"));
-	ASSERT(!details.contains("ExpectedError"));
-	ASSERT(!details.contains("ExpectedErrorCode"));
-	ASSERT(!details.contains("ActualError"));
-	ASSERT(!details.contains("ActualErrorCode"));
-	ASSERT(!details.contains("Reason"));
-
-	for (auto& p : details) {
-		evt.detail(p.first.c_str(), p.second);
-	}
-	if (throwOnError.present()) {
-		throw throwOnError.get();
-	}
-	co_return;
-}
-} // namespace
-
-Future<Void> testExpectedError(Future<Void> test,
-                               const char* testDescr,
-                               Optional<Error> expectedError,
-                               Optional<bool*> successFlag,
-                               std::map<std::string, std::string> details,
-                               Optional<Error> throwOnError,
-                               UID id) {
-	return testExpectedErrorImpl(test, testDescr, expectedError, successFlag, details, throwOnError, id);
-}
diff --git a/fdbserver/tester/CMakeLists.txt b/fdbserver/tester/CMakeLists.txt
new file mode 100644
index 00000000000..d8650fcf40a
--- /dev/null
+++ b/fdbserver/tester/CMakeLists.txt
@@ -0,0 +1,17 @@
+fdb_find_sources(FDBSERVER_TESTER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_tester SRCS ${FDBSERVER_TESTER_SRCS})
+add_fdbserver_link_test(fdbserver_testerlinktest
+  fdbserver_tester
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_tester)
+target_include_directories(fdbserver_tester
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(fdbserver_tester PRIVATE fdbclient fdbserver_core toml11::toml11)
+
+# Ensure toml11 external project is built before this target if needed
+if(NOT toml11_FOUND AND TARGET toml11Project)
+  add_dependencies(fdbserver_tester toml11Project)
+endif()
diff --git a/fdbserver/tester/ConsistencyChecker.cpp b/fdbserver/tester/ConsistencyChecker.cpp
new file mode 100644
index 00000000000..9d026f71a20
--- /dev/null
+++ b/fdbserver/tester/ConsistencyChecker.cpp
@@ -0,0 +1,760 @@
+/*
+ * ConsistencyChecker.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <numeric>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "flow/DeterministicRandom.h"
+#include "flow/Trace.h"
+#include "flow/genericactors.actor.h"
+#include "fdbrpc/simulator.h"
+#include "fdbclient/Audit.h"
+#include "fdbclient/AuditUtils.h"
+#include "fdbclient/ClusterInterface.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/MonitorLeader.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "ConsistencyChecker.h"
+#include "fdbserver/tester/workloads.h"
+
+Future<Void> auditStorageCorrectness(Reference<AsyncVar<ServerDBInfo>> dbInfo, AuditType auditType) {
+	if (SERVER_KNOBS->DISABLE_AUDIT_STORAGE_FINAL_REPLICA_CHECK_IN_SIM &&
+	    (auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica)) {
+		co_return;
+	}
+	TraceEvent(SevDebug, "AuditStorageCorrectnessBegin").detail("AuditType", auditType);
+	Database cx;
+	UID auditId;
+	AuditStorageState auditState;
+
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		try {
+			while (dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS ||
+			       !dbInfo->get().distributor.present()) {
+				co_await dbInfo->onChange();
+			}
+			TriggerAuditRequest req(auditType, allKeys, KeyValueStoreType::END); // do not specify engine type to check
+			UID auditId_ = co_await timeoutError(dbInfo->get().distributor.get().triggerAudit.getReply(req), 300);
+			auditId = auditId_;
+			TraceEvent(SevDebug, "AuditStorageCorrectnessTriggered")
+			    .detail("AuditID", auditId)
+			    .detail("AuditType", auditType);
+			break;
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "AuditStorageCorrectnessTriggerError")
+			    .errorUnsuppressed(e)
+			    .detail("AuditID", auditId)
+			    .detail("AuditType", auditType);
+			caughtError = e;
+			needsErrorHandling = true;
+		}
+
+		if (needsErrorHandling) {
+			co_await delay(1);
+		}
+	}
+
+	int retryCount = 0;
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		try {
+			cx = openDBOnServer(dbInfo);
+			AuditStorageState auditState_ = co_await getAuditState(cx, auditType, auditId);
+			auditState = auditState_;
+			if (auditState.getPhase() == AuditPhase::Complete) {
+				break;
+			} else if (auditState.getPhase() == AuditPhase::Running) {
+				TraceEvent("AuditStorageCorrectnessWait")
+				    .detail("AuditID", auditId)
+				    .detail("AuditType", auditType)
+				    .detail("RetryCount", retryCount);
+				co_await delay(25);
+				if (retryCount > 20) {
+					TraceEvent("AuditStorageCorrectnessWaitFailed")
+					    .detail("AuditID", auditId)
+					    .detail("AuditType", auditType);
+					break;
+				}
+				retryCount++;
+				continue;
+			} else if (auditState.getPhase() == AuditPhase::Error) {
+				break;
+			} else if (auditState.getPhase() == AuditPhase::Failed) {
+				break;
+			} else {
+				UNREACHABLE();
+			}
+		} catch (Error& e) {
+			TraceEvent("AuditStorageCorrectnessWaitError")
+			    .errorUnsuppressed(e)
+			    .detail("AuditID", auditId)
+			    .detail("AuditType", auditType)
+			    .detail("AuditState", auditState.toString());
+			caughtError = e;
+			needsErrorHandling = true;
+		}
+
+		if (needsErrorHandling) {
+			co_await delay(1);
+		}
+	}
+	TraceEvent("AuditStorageCorrectnessWaitEnd")
+	    .detail("AuditID", auditId)
+	    .detail("AuditType", auditType)
+	    .detail("AuditState", auditState.toString());
+
+	co_return;
+}
+
+// Runs the consistency check workload, which verifies that the database is in a consistent state
+Future<Void> checkConsistency(Database cx,
+                              std::vector<TesterInterface> testers,
+                              bool doQuiescentCheck,
+                              bool doTSSCheck,
+                              double maxDDRunTime,
+                              double softTimeLimit,
+                              double databasePingDelay,
+                              Reference<AsyncVar<ServerDBInfo>> dbInfo) {
+	TestSpec spec;
+
+	double connectionFailures;
+	if (g_network->isSimulated()) {
+		// NOTE: the value will be reset after consistency check
+		connectionFailures = g_simulator->connectionFailuresDisableDuration;
+		disableConnectionFailures("ConsistencyCheck");
+		fdbSimulationPolicyState().isConsistencyChecked = true;
+	}
+
+	Standalone<VectorRef<KeyValueRef>> options;
+	StringRef performQuiescent = "false"_sr;
+	StringRef performTSSCheck = "false"_sr;
+	if (doQuiescentCheck) {
+		performQuiescent = "true"_sr;
+		spec.restorePerpetualWiggleSetting = false;
+	}
+	if (doTSSCheck) {
+		performTSSCheck = "true"_sr;
+	}
+	spec.title = "ConsistencyCheck"_sr;
+	spec.databasePingDelay = databasePingDelay;
+	spec.runFailureWorkloads = false;
+	spec.timeout = 32000;
+	options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ConsistencyCheck"_sr));
+	options.push_back_deep(options.arena(), KeyValueRef("performQuiescentChecks"_sr, performQuiescent));
+	options.push_back_deep(options.arena(), KeyValueRef("performTSSCheck"_sr, performTSSCheck));
+	options.push_back_deep(options.arena(),
+	                       KeyValueRef("maxDDRunTime"_sr, ValueRef(options.arena(), format("%f", maxDDRunTime))));
+	options.push_back_deep(options.arena(), KeyValueRef("distributed"_sr, "false"_sr));
+	spec.options.push_back_deep(spec.options.arena(), options);
+
+	double start = now();
+	bool lastRun = false;
+	while (true) {
+		DistributedTestResults testResults = co_await runWorkload(cx, testers, spec);
+		if (testResults.ok() || lastRun) {
+			if (g_network->isSimulated()) {
+				g_simulator->connectionFailuresDisableDuration = connectionFailures;
+				fdbSimulationPolicyState().isConsistencyChecked = false;
+			}
+			co_return;
+		}
+		if (now() - start > softTimeLimit) {
+			spec.options[0].push_back_deep(spec.options.arena(), KeyValueRef("failureIsError"_sr, "true"_sr));
+			lastRun = true;
+		}
+
+		co_await repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck");
+	}
+}
+
+namespace {
+
+Future<std::unordered_set<int>> runUrgentConsistencyCheckWorkload(
+    Database cx,
+    std::vector<TesterInterface> testers,
+    int64_t consistencyCheckerId,
+    std::unordered_map<int, std::vector<KeyRange>> assignment) {
+	TraceEvent(SevInfo, "ConsistencyCheckUrgent_DispatchWorkloads")
+	    .detail("TesterCount", testers.size())
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	// Step 1: Get interfaces for running workloads
+	std::vector<Future<ErrorOr<WorkloadInterface>>> workRequests;
+	Standalone<VectorRef<KeyValueRef>> option;
+	option.push_back_deep(option.arena(), KeyValueRef("testName"_sr, "ConsistencyCheckUrgent"_sr));
+	Standalone<VectorRef<VectorRef<KeyValueRef>>> options;
+	options.push_back_deep(options.arena(), option);
+	for (int i = 0; i < testers.size(); i++) {
+		WorkloadRequest req;
+		req.title = "ConsistencyCheckUrgent"_sr;
+		req.useDatabase = true;
+		req.timeout = 0.0; // disable timeout workload
+		req.databasePingDelay = 0.0; // disable databased ping check
+		req.options = options;
+		req.clientId = i;
+		req.clientCount = testers.size();
+		req.sharedRandomNumber = consistencyCheckerId;
+		req.rangesToCheck = assignment[i];
+		workRequests.push_back(testers[i].recruitments.getReplyUnlessFailedFor(req, 10, 0));
+		// workRequests follows the order of clientId of assignment
+	}
+	co_await waitForAll(workRequests);
+
+	// Step 2: Run workloads via the interfaces
+	TraceEvent(SevInfo, "ConsistencyCheckUrgent_TriggerWorkloads")
+	    .detail("TesterCount", testers.size())
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+	std::unordered_set<int> completeClientIds;
+	std::vector<int> clientIds; // record the clientId for jobs
+	std::vector<Future<ErrorOr<Void>>> jobs;
+	for (int i = 0; i < workRequests.size(); i++) {
+		ASSERT(workRequests[i].isReady());
+		if (workRequests[i].get().isError()) {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_FailedToContactTester")
+			    .error(workRequests[i].get().getError())
+			    .detail("TesterCount", testers.size())
+			    .detail("TesterId", i)
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		} else {
+			jobs.push_back(workRequests[i].get().get().start.template getReplyUnlessFailedFor<Void>(10, 0));
+			clientIds.push_back(i);
+		}
+	}
+	co_await waitForAll(jobs);
+	for (int i = 0; i < jobs.size(); i++) {
+		if (jobs[i].isError()) {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RunWorkloadError1")
+			    .errorUnsuppressed(jobs[i].getError())
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		} else if (jobs[i].get().isError()) {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RunWorkloadError2")
+			    .errorUnsuppressed(jobs[i].get().getError())
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		} else {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RunWorkloadComplete")
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			completeClientIds.insert(clientIds[i]); // Add complete clients
+		}
+	}
+
+	TraceEvent(SevInfo, "ConsistencyCheckUrgent_DispatchWorkloadEnd")
+	    .detail("TesterCount", testers.size())
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	co_return completeClientIds;
+}
+
+Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::vector<KeyRange> ranges) {
+	// Get the scope of the input list of ranges
+	Key beginKeyToReadKeyServer;
+	Key endKeyToReadKeyServer;
+	for (int i = 0; i < ranges.size(); i++) {
+		if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
+			beginKeyToReadKeyServer = ranges[i].begin;
+		}
+		if (i == 0 || ranges[i].end > endKeyToReadKeyServer) {
+			endKeyToReadKeyServer = ranges[i].end;
+		}
+	}
+	TraceEvent(SevInfo, "ConsistencyCheckUrgent_GetConsistencyCheckShards")
+	    .detail("RangeBegin", beginKeyToReadKeyServer)
+	    .detail("RangeEnd", endKeyToReadKeyServer);
+	// Read KeyServer space within the scope and add shards intersecting with the input ranges
+	std::vector<KeyRange> res;
+	Transaction tr(cx);
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
+			RangeResult readResult = co_await krmGetRanges(&tr,
+			                                               keyServersPrefix,
+			                                               rangeToRead,
+			                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+			                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
+			for (int i = 0; i < readResult.size() - 1; ++i) {
+				KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
+				Value valueToCheck = Standalone(readResult[i].value);
+				bool toAdd = false;
+				for (const auto& range : ranges) {
+					if (rangeToCheck.intersects(range) == true) {
+						toAdd = true;
+						break;
+					}
+				}
+				if (toAdd == true) {
+					res.push_back(rangeToCheck);
+				}
+				beginKeyToReadKeyServer = readResult[i + 1].key;
+			}
+			if (beginKeyToReadKeyServer >= endKeyToReadKeyServer) {
+				break;
+			}
+		} catch (Error& e) {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_GetConsistencyCheckShardsRetry").error(e);
+			caughtError = e;
+			needsErrorHandling = true;
+		}
+
+		if (needsErrorHandling) {
+			co_await tr.onError(caughtError);
+		}
+	}
+	co_return res;
+}
+
+Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                                int minTestersExpected) {
+	// Recruit workers
+	int flags = GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
+	Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
+	std::vector<WorkerDetails> workers;
+
+	while (true) {
+		Future<std::vector<WorkerDetails>> getWorkersReq =
+		    cc->get().present() ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
+		                        : Future<std::vector<WorkerDetails>>(Never());
+		Future<Void> ccChange = cc->onChange();
+
+		int action = 0;
+		std::vector<WorkerDetails> gotWorkers;
+
+		co_await Choose()
+		    .When(getWorkersReq,
+		          [&](std::vector<WorkerDetails> const& w) {
+			          gotWorkers = w;
+			          action = 1;
+		          })
+		    .When(ccChange, [&](Void const&) { action = 2; })
+		    .When(testerTimeout, [&](Void const&) { action = 3; })
+		    .run();
+
+		if (action == 1) {
+			if (static_cast<int>(gotWorkers.size()) >= minTestersExpected) {
+				workers = gotWorkers;
+				break;
+			}
+			co_await delay(SERVER_KNOBS->WORKER_POLL_DELAY);
+		} else if (action == 3) {
+			TraceEvent(SevWarnAlways, "TesterRecruitmentTimeout");
+			throw timed_out();
+		}
+		// action == 2: cc changed, loop again
+	}
+	std::vector<TesterInterface> ts;
+	ts.reserve(workers.size());
+	for (int i = 0; i < workers.size(); i++)
+		ts.push_back(workers[i].interf.testerInterface);
+	deterministicRandom()->randomShuffle(ts);
+	co_return ts;
+}
+
+const std::unordered_map<char, uint8_t> parseCharMap{
+	{ '0', 0 },  { '1', 1 },  { '2', 2 },  { '3', 3 },  { '4', 4 },  { '5', 5 },  { '6', 6 },  { '7', 7 },
+	{ '8', 8 },  { '9', 9 },  { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
+	{ 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 },
+};
+
+Optional<Key> getKeyFromString(const std::string& str) {
+	Key emptyKey;
+	if (str.empty()) {
+		return emptyKey;
+	}
+	if (str.size() % 4 != 0) {
+		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
+		    .setMaxEventLength(-1)
+		    .setMaxFieldLength(-1)
+		    .detail("Reason", "WrongLength")
+		    .detail("InputStr", str);
+		return Optional<Key>();
+	}
+	std::vector<uint8_t> byteList;
+	for (int i = 0; i < str.size(); i += 4) {
+		if (str.at(i + 0) != '\\' || str.at(i + 1) != 'x') {
+			TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+			           "ConsistencyCheckUrgent_GetKeyFromStringError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Reason", "WrongBytePrefix")
+			    .detail("InputStr", str);
+			return Optional<Key>();
+		}
+		const char first = str.at(i + 2);
+		const char second = str.at(i + 3);
+		if (!parseCharMap.contains(first) || !parseCharMap.contains(second)) {
+			TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways,
+			           "ConsistencyCheckUrgent_GetKeyFromStringError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Reason", "WrongByteContent")
+			    .detail("InputStr", str);
+			return Optional<Key>();
+		}
+		uint8_t parsedValue = parseCharMap.at(first) * 16 + parseCharMap.at(second);
+		byteList.push_back(parsedValue);
+	}
+	return Standalone(StringRef(byteList.data(), byteList.size()));
+}
+
+Optional<std::vector<KeyRange>> loadRangesToCheckFromKnob() {
+	// Load string from knob
+	std::vector<std::string> beginKeyStrs = {
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_0,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_1,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_2,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_3,
+	};
+	std::vector<std::string> endKeyStrs = {
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_0,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_1,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_2,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RANGE_END_3,
+	};
+
+	// Get keys from strings
+	std::vector<Key> beginKeys;
+	for (const auto& beginKeyStr : beginKeyStrs) {
+		Optional<Key> key = getKeyFromString(beginKeyStr);
+		if (key.present()) {
+			beginKeys.push_back(key.get());
+		} else {
+			return Optional<std::vector<KeyRange>>();
+		}
+	}
+	std::vector<Key> endKeys;
+	for (const auto& endKeyStr : endKeyStrs) {
+		Optional<Key> key = getKeyFromString(endKeyStr);
+		if (key.present()) {
+			endKeys.push_back(key.get());
+		} else {
+			return Optional<std::vector<KeyRange>>();
+		}
+	}
+	if (beginKeys.size() != endKeys.size()) {
+		TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
+		    .detail("Reason", "MismatchBeginKeysAndEndKeys");
+		return Optional<std::vector<KeyRange>>();
+	}
+
+	// Get ranges
+	KeyRangeMap<bool> rangeToCheckMap;
+	for (int i = 0; i < beginKeys.size(); i++) {
+		Key rangeBegin = beginKeys[i];
+		Key rangeEnd = endKeys[i];
+		if (rangeBegin.empty() && rangeEnd.empty()) {
+			continue;
+		}
+		if (rangeBegin > allKeys.end) {
+			rangeBegin = allKeys.end;
+		}
+		if (rangeEnd > allKeys.end) {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_ReverseInputRange")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Index", i)
+			    .detail("RangeBegin", rangeBegin)
+			    .detail("RangeEnd", rangeEnd);
+			rangeEnd = allKeys.end;
+		}
+
+		KeyRange rangeToCheck;
+		if (rangeBegin < rangeEnd) {
+			rangeToCheck = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
+		} else if (rangeBegin > rangeEnd) {
+			rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
+		} else {
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_EmptyInputRange")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Index", i)
+			    .detail("RangeBegin", rangeBegin)
+			    .detail("RangeEnd", rangeEnd);
+			continue;
+		}
+		rangeToCheckMap.insert(rangeToCheck, true);
+	}
+
+	rangeToCheckMap.coalesce(allKeys);
+
+	std::vector<KeyRange> res;
+	for (auto rangeToCheck : rangeToCheckMap.ranges()) {
+		if (rangeToCheck.value() == true) {
+			res.push_back(rangeToCheck.range());
+		}
+	}
+	TraceEvent e(SevInfo, "ConsistencyCheckUrgent_LoadedInputRange");
+	e.setMaxEventLength(-1);
+	e.setMaxFieldLength(-1);
+	for (int i = 0; i < res.size(); i++) {
+		e.detail("RangeBegin" + std::to_string(i), res[i].begin);
+		e.detail("RangeEnd" + std::to_string(i), res[i].end);
+	}
+	return res;
+}
+
+std::unordered_map<int, std::vector<KeyRange>> makeTaskAssignment(Database cx,
+                                                                  int64_t consistencyCheckerId,
+                                                                  std::vector<KeyRange> shardsToCheck,
+                                                                  int testersCount,
+                                                                  int round) {
+	ASSERT(testersCount >= 1);
+	std::unordered_map<int, std::vector<KeyRange>> assignment;
+
+	std::vector<size_t> shuffledIndices(testersCount);
+	std::iota(shuffledIndices.begin(), shuffledIndices.end(), 0); // creates [0, 1, ..., testersCount - 1]
+	deterministicRandom()->randomShuffle(shuffledIndices);
+
+	int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_BATCH_SHARD_COUNT;
+	int startingPoint = 0;
+	if (shardsToCheck.size() > batchSize * testersCount) {
+		startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * testersCount);
+		// We randomly pick a set of successive shards:
+		// (1) We want to retry for different shards to avoid repeated failure on the same shards
+		// (2) We want to check successive shards to avoid inefficiency incurred by fragments
+	}
+	assignment.clear();
+	for (int i = startingPoint; i < shardsToCheck.size(); i++) {
+		int testerIdx = (i - startingPoint) / batchSize;
+		if (testerIdx > testersCount - 1) {
+			break; // Have filled up all testers
+		}
+		// When assigning a shards/batch to a tester idx, there are certain edge cases which can result in urgent
+		// consistency checker being infinetely stuck in a loop. Examples:
+		//      1. if there is 1 remaining shard, and tester 0 consistently fails, we will still always pick tester 0
+		//      2. if there are 10 remaining shards, and batch size is 10, and tester 0 consistently fails, we will
+		//      still always pick tester 0
+		//      3. if there are 20 remaining shards, and batch size is 10, and testers {0, 1} consistently fail, we will
+		//      keep picking testers {0, 1}
+		// To avoid repeatedly picking the same testers even though they could be failing, shuffledIndices provides an
+		// indirection to a random tester idx. That way, each invocation of makeTaskAssignment won't
+		// result in the same task assignment for the class of edge cases mentioned above.
+		assignment[shuffledIndices[testerIdx]].push_back(shardsToCheck[i]);
+	}
+	std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
+	for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
+		TraceEvent(SevInfo, "ConsistencyCheckUrgent_AssignTaskToTesters")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round)
+		    .detail("ClientId", assignIt->first)
+		    .detail("ShardsCount", assignIt->second.size());
+	}
+	return assignment;
+}
+
+Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                             Database cx,
+                                             Optional<std::vector<TesterInterface>> testers,
+                                             int minTestersExpected) {
+	KeyRangeMap<bool> globalProgressMap; // used to keep track of progress
+	std::unordered_map<int, std::vector<KeyRange>> assignment; // used to keep track of assignment of tasks
+	std::vector<TesterInterface> ts; // used to store testers interface
+	std::vector<KeyRange> rangesToCheck; // get from globalProgressMap
+	std::vector<KeyRange> shardsToCheck; // get from keyServer metadata
+	Optional<double> whenFailedToGetTesterStart;
+
+	// Initialize globalProgressMap
+	Optional<std::vector<KeyRange>> rangesToCheck_ = loadRangesToCheckFromKnob();
+	if (rangesToCheck_.present()) {
+		globalProgressMap.insert(allKeys, true);
+		for (const auto& rangeToCheck : rangesToCheck_.get()) {
+			// Mark rangesToCheck as incomplete
+			// Those ranges will be checked
+			globalProgressMap.insert(rangeToCheck, false);
+		}
+		globalProgressMap.coalesce(allKeys);
+	} else {
+		TraceEvent(SevInfo, "ConsistencyCheckUrgent_FailedToLoadRangeFromKnob");
+		globalProgressMap.insert(allKeys, false);
+	}
+
+	int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+	int retryTimes = 0;
+	int round = 0;
+
+	// Main loop
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		try {
+			// Step 1: Load ranges to check, if nothing to run, exit
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RoundBegin")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("TesterCount", ts.size())
+			    .detail("Round", round);
+			rangesToCheck.clear();
+			for (auto& range : globalProgressMap.ranges()) {
+				if (!range.value()) { // range that is not finished
+					rangesToCheck.push_back(range.range());
+				}
+			}
+			if (rangesToCheck.empty()) {
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_Complete")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				co_return;
+			}
+
+			// Step 2: Get testers
+			ts.clear();
+			if (!testers.present()) { // In real clusters
+				bool getTesterError = false;
+				Error getTesterErr;
+
+				try {
+					ts = co_await getTesters(cc, minTestersExpected);
+					whenFailedToGetTesterStart.reset();
+				} catch (Error& e) {
+					if (e.code() == error_code_timed_out) {
+						if (!whenFailedToGetTesterStart.present()) {
+							whenFailedToGetTesterStart = now();
+						} else if (now() - whenFailedToGetTesterStart.get() > 3600 * 24) { // 1 day
+							TraceEvent(SevError, "TesterRecruitmentTimeout");
+						}
+					}
+					getTesterErr = e;
+					getTesterError = true;
+				}
+
+				if (getTesterError) {
+					throw getTesterErr;
+				}
+
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+			} else { // In simulation
+				ts = testers.get();
+			}
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_GotTesters")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("TesterCount", ts.size());
+
+			// Step 3: Load shards to check from keyserver space
+			// Shard is the unit for the task assignment
+			shardsToCheck.clear();
+			shardsToCheck = co_await getConsistencyCheckShards(cx, rangesToCheck);
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_GotShardsToCheck")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("TesterCount", ts.size())
+			    .detail("ShardCount", shardsToCheck.size());
+
+			// Step 4: Assign tasks to clients
+			assignment.clear();
+			assignment = makeTaskAssignment(cx, consistencyCheckerId, shardsToCheck, ts.size(), round);
+
+			// Step 5: Run checking on testers
+			std::unordered_set<int> completeClients =
+			    co_await runUrgentConsistencyCheckWorkload(cx, ts, consistencyCheckerId, assignment);
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
+			// We use the complete client to decide which ranges are completed
+			for (const auto& clientId : completeClients) {
+				for (const auto& range : assignment[clientId]) {
+					globalProgressMap.insert(range, true); // Mark the ranges as complete
+				}
+			}
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_RoundEnd")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("SucceedTesterCount", completeClients.size())
+			    .detail("SucceedTesters", describe(completeClients))
+			    .detail("TesterCount", ts.size())
+			    .detail("Round", round);
+			round++;
+
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw e;
+			} else {
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_CoreWithRetriableFailure")
+				    .errorUnsuppressed(e)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				caughtError = e;
+				needsErrorHandling = true;
+			}
+		}
+
+		if (needsErrorHandling) {
+			co_await delay(10.0);
+			retryTimes++;
+		} else {
+			co_await delay(10.0); // Backoff 10 seconds for the next round
+		}
+
+		// Decide and enforce the consistencyCheckerId for the next round
+		consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+	}
+}
+
+} // namespace
+
+Future<Void> runConsistencyCheckerUrgentHolder(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                               Database cx,
+                                               Optional<std::vector<TesterInterface>> testers,
+                                               int minTestersExpected,
+                                               bool repeatRun) {
+	while (true) {
+		co_await runConsistencyCheckerUrgentCore(cc, cx, testers, minTestersExpected);
+		if (!repeatRun) {
+			break;
+		}
+		co_await delay(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME);
+	}
+	co_return;
+}
+
+Future<Void> checkConsistencyUrgentSim(Database cx, std::vector<TesterInterface> testers) {
+	if (SERVER_KNOBS->DISABLE_AUDIT_STORAGE_FINAL_REPLICA_CHECK_IN_SIM) {
+		return Void();
+	}
+	return runConsistencyCheckerUrgentHolder(
+	    Reference<AsyncVar<Optional<ClusterControllerFullInterface>>>(), cx, testers, 1, /*repeatRun=*/false);
+}
diff --git a/fdbserver/tester/ConsistencyChecker.h b/fdbserver/tester/ConsistencyChecker.h
new file mode 100644
index 00000000000..9c61f34878c
--- /dev/null
+++ b/fdbserver/tester/ConsistencyChecker.h
@@ -0,0 +1,47 @@
+/*
+ * ConsistencyChecker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/tester/WorkloadUtils.h"
+
+enum class AuditType : uint8_t;
+struct ClusterControllerFullInterface;
+struct ServerDBInfo;
+
+Future<Void> checkConsistency(Database cx,
+                              std::vector<TesterInterface> testers,
+                              bool doQuiescentCheck,
+                              bool doTSSCheck,
+                              double maxDDRunTime,
+                              double softTimeLimit,
+                              double databasePingDelay,
+                              Reference<AsyncVar<ServerDBInfo>> dbInfo);
+Future<Void> auditStorageCorrectness(Reference<AsyncVar<ServerDBInfo>> dbInfo, AuditType auditType);
+Future<Void> checkConsistencyUrgentSim(Database cx, std::vector<TesterInterface> testers);
+Future<Void> runConsistencyCheckerUrgentHolder(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                               Database cx,
+                                               Optional<std::vector<TesterInterface>> testers,
+                                               int minTestersExpected,
+                                               bool repeatRun);
diff --git a/fdbserver/workloads/workloads.actor.cpp b/fdbserver/tester/CustomShardConfigWorkload.cpp
similarity index 67%
rename from fdbserver/workloads/workloads.actor.cpp
rename to fdbserver/tester/CustomShardConfigWorkload.cpp
index 2b2dceb57d9..ff0feb112d6 100644
--- a/fdbserver/workloads/workloads.actor.cpp
+++ b/fdbserver/tester/CustomShardConfigWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * workloads.actor.cpp
+ * CustomShardConfigWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,18 +22,20 @@
 
 #include "flow/flow.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/DataDistributionConfig.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/DataDistributionConfig.h"
+#include "fdbserver/tester/tester.h"
 
-ACTOR Future<Void> customShardConfigWorkload(Database cx) {
-	state ReadYourWritesTransaction tr(cx);
-	state bool verbose = (KEYBACKEDTYPES_DEBUG != 0);
+Future<Void> customShardConfigWorkload(Database const& cxUnsafe) {
+	auto cx = cxUnsafe;
+	ReadYourWritesTransaction tr(cx);
+	bool verbose = (KEYBACKEDTYPES_DEBUG != 0);
 
 	// This has to be optional because state vars need default constructors and VersionOptions types can't be
 	// default constructed and RangeConfigMap uses ObjectWriter which needs VersionOptions.
-	state Optional<DDConfiguration::RangeConfigMap> rangeConfig = DDConfiguration().userRangeConfig();
+	Optional<DDConfiguration::RangeConfigMap> rangeConfig = DDConfiguration().userRangeConfig();
 
-	loop {
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -42,26 +44,26 @@ ACTOR Future<Void> customShardConfigWorkload(Database cx) {
 			if (deterministicRandom()->coinflip()) {
 				TraceEvent("KeyRangeConfigSetDefault").log();
 				CODE_PROBE(true, "Set default shard range");
-				wait(rangeConfig->updateRange(&tr, allKeys.begin, allKeys.end, DDRangeConfig()));
+				co_await rangeConfig->updateRange(&tr, allKeys.begin, allKeys.end, DDRangeConfig());
 			}
 
 			if (deterministicRandom()->coinflip()) {
 				CODE_PROBE(true, "Set range config test cases");
 				TraceEvent("KeyRangeConfigSetTestRanges").log();
 
-				wait(rangeConfig->updateRange(&tr, "\xff\x03"_sr, "\xff\x04"_sr, DDRangeConfig(3)));
-				wait(rangeConfig->updateRange(&tr, "\xff\x06"_sr, "\xff\x07"_sr, DDRangeConfig(6)));
-				wait(rangeConfig->updateRange(&tr, "\xff\x04"_sr, "\xff\x05"_sr, DDRangeConfig(4)));
+				co_await rangeConfig->updateRange(&tr, "\xff\x03"_sr, "\xff\x04"_sr, DDRangeConfig(3));
+				co_await rangeConfig->updateRange(&tr, "\xff\x06"_sr, "\xff\x07"_sr, DDRangeConfig(6));
+				co_await rangeConfig->updateRange(&tr, "\xff\x04"_sr, "\xff\x05"_sr, DDRangeConfig(4));
 
-				wait(rangeConfig->updateRange(&tr, "\xff\x03k"_sr, "\xff\x03z"_sr, DDRangeConfig(3)));
-				wait(rangeConfig->updateRange(&tr, "\xff\x04t"_sr, "\xff\x05h"_sr, DDRangeConfig(3, 1), true));
-				wait(rangeConfig->updateRange(&tr, "\xff\x06u"_sr, "\xff\x07m"_sr, DDRangeConfig({}, 2)));
+				co_await rangeConfig->updateRange(&tr, "\xff\x03k"_sr, "\xff\x03z"_sr, DDRangeConfig(3));
+				co_await rangeConfig->updateRange(&tr, "\xff\x04t"_sr, "\xff\x05h"_sr, DDRangeConfig(3, 1), true);
+				co_await rangeConfig->updateRange(&tr, "\xff\x06u"_sr, "\xff\x07m"_sr, DDRangeConfig({}, 2));
 
-				wait(rangeConfig->updateRange(&tr, "a"_sr, "b"_sr, DDRangeConfig(3, 20), true));
-				wait(rangeConfig->updateRange(&tr, "a"_sr, "a10"_sr, DDRangeConfig(3, 20), true));
+				co_await rangeConfig->updateRange(&tr, "a"_sr, "b"_sr, DDRangeConfig(3, 20), true);
+				co_await rangeConfig->updateRange(&tr, "a"_sr, "a10"_sr, DDRangeConfig(3, 20), true);
 
 				// Key, entryRequiredInDB, expectedRangeConfig
-				state std::vector<std::tuple<Key, bool, DDRangeConfig>> rangeTests = {
+				std::vector<std::tuple<Key, bool, DDRangeConfig>> rangeTests = {
 					{ "\x01"_sr, false, DDRangeConfig() },
 					{ "\xff\x10"_sr, false, DDRangeConfig() },
 					{ "\xff\x03x"_sr, true, DDRangeConfig(3) },
@@ -79,8 +81,8 @@ ACTOR Future<Void> customShardConfigWorkload(Database cx) {
 					{ "b1"_sr, false, DDRangeConfig() }
 				};
 
-				state Reference<DDConfiguration::RangeConfigMapSnapshot> snapshot =
-				    wait(rangeConfig->getSnapshot(&tr, allKeys.begin, allKeys.end));
+				Reference<DDConfiguration::RangeConfigMapSnapshot> snapshot =
+				    co_await rangeConfig->getSnapshot(&tr, allKeys.begin, allKeys.end);
 
 				if (verbose) {
 					fmt::print(
@@ -88,11 +90,10 @@ ACTOR Future<Void> customShardConfigWorkload(Database cx) {
 					    json_spirit::write_string(DDConfiguration::toJSON(*snapshot, true), json_spirit::pretty_print));
 				}
 
-				state int i;
-				for (i = 0; i < rangeTests.size(); ++i) {
-					state Key query = std::get<0>(rangeTests[i]);
+				for (int i = 0; i < rangeTests.size(); ++i) {
+					Key query = std::get<0>(rangeTests[i]);
 					Optional<DDConfiguration::RangeConfigMap::RangeValue> verify =
-					    wait(rangeConfig->getRangeForKey(&tr, query));
+					    co_await rangeConfig->getRangeForKey(&tr, query);
 
 					if (verbose) {
 						if (verify.present()) {
@@ -121,13 +122,13 @@ ACTOR Future<Void> customShardConfigWorkload(Database cx) {
 				}
 			}
 
-			wait(tr.commit());
+			co_await tr.commit();
 			TraceEvent("KeyRangeConfigCommitted").log();
 			break;
 		} catch (Error& e) {
-			TraceEvent("KeyRangeConfigCommitError").error(e);
-			wait(tr.onError(e));
+			err = e;
 		}
+		TraceEvent("KeyRangeConfigCommitError").error(err);
+		co_await tr.onError(err);
 	}
-	return Void();
 }
diff --git a/fdbserver/tester/DatabaseMaintenance.cpp b/fdbserver/tester/DatabaseMaintenance.cpp
new file mode 100644
index 00000000000..8da1cb64804
--- /dev/null
+++ b/fdbserver/tester/DatabaseMaintenance.cpp
@@ -0,0 +1,264 @@
+/*
+ * DatabaseMaintenance.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <cinttypes>
+#include <cstdio>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "flow/DeterministicRandom.h"
+#include "flow/Platform.h"
+#include "flow/Trace.h"
+#include "fdbrpc/sim_validation.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "DatabaseMaintenance.h"
+
+Future<Void> clearData(Database cx) {
+	TraceEvent("ClearData_Start").detail("Phase", "Enter");
+
+	Transaction tr(cx);
+	TraceEvent("ClearData_TrCreated").detail("Phase", "Loop1_Start");
+
+	while (true) {
+		TraceEvent("ClearData_Loop1_Iteration").detail("Phase", "TryBlock");
+
+		Error err;
+		try {
+			tr.debugTransaction(debugRandom()->randomUniqueID());
+			TraceEvent("ClearData_DebugSet").detail("Phase", "Loop1_Debug");
+
+			ASSERT(tr.trState->readOptions.present() && tr.trState->readOptions.get().debugID.present());
+			TraceEvent("TesterClearingDatabaseStart", tr.trState->readOptions.get().debugID.get()).log();
+
+			// This transaction needs to be self-conflicting, but not conflict consistently with
+			// any other transactions
+			tr.clear(normalKeys);
+			tr.makeSelfConflicting();
+			TraceEvent("ClearData_PreGetReadVersion").detail("Phase", "Loop1_AboutToAwait");
+
+			Version rv = co_await tr.getReadVersion(); // required since we use addReadConflictRange but not get
+			TraceEvent("TesterClearingDatabaseRV", tr.trState->readOptions.get().debugID.get()).detail("RV", rv);
+			TraceEvent("ClearData_PreCommit").detail("Phase", "Loop1_AboutToCommit");
+
+			co_await tr.commit();
+			TraceEvent("TesterClearingDatabase", tr.trState->readOptions.get().debugID.get())
+			    .detail("AtVersion", tr.getCommittedVersion());
+			TraceEvent("ClearData_Loop1_Success").detail("Phase", "Loop1_Complete");
+			break;
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "TesterClearingDatabaseError", tr.trState->readOptions.get().debugID.get()).error(e);
+			TraceEvent("ClearData_Loop1_Catch").detail("Phase", "Loop1_Error").detail("ErrorCode", e.code());
+			err = e;
+		}
+
+		TraceEvent("ClearData_Loop1_ErrorHandling").detail("Phase", "Loop1_OnError");
+		co_await tr.onError(err);
+	}
+
+	TraceEvent("ClearData_Loop1_Done").detail("Phase", "Loop2_Start");
+	tr = Transaction(cx);
+	TraceEvent("ClearData_NewTr").detail("Phase", "Loop2_TrCreated");
+
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		TraceEvent("ClearData_Loop2_Iteration").detail("Phase", "Loop2_TryBlock");
+
+		try {
+			tr.debugTransaction(debugRandom()->randomUniqueID());
+			tr.setOption(FDBTransactionOptions::RAW_ACCESS);
+			TraceEvent("ClearData_Loop2_PreGetRange").detail("Phase", "Loop2_AboutToAwait");
+
+			RangeResult rangeResult = co_await tr.getRange(normalKeys, 1);
+			TraceEvent("ClearData_Loop2_PostGetRange")
+			    .detail("Phase", "Loop2_RangeGot")
+			    .detail("ResultSize", rangeResult.size());
+
+			// If the result is non-empty, it is possible that there is some bug.
+			if (!rangeResult.empty()) {
+				TraceEvent(SevError, "TesterClearFailure").detail("FirstKey", rangeResult[0].key);
+				ASSERT(false);
+			}
+			// TODO(gglass): the assert below is leftover from some tenant logic.  If it fails, take it out.
+			// If not, leave it in and take out this comment.
+			ASSERT(tr.trState->readOptions.present() && tr.trState->readOptions.get().debugID.present());
+			TraceEvent("TesterCheckDatabaseClearedDone", tr.trState->readOptions.get().debugID.get());
+			TraceEvent("ClearData_Loop2_Success").detail("Phase", "Loop2_Complete");
+			break;
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "TesterCheckDatabaseClearedError", tr.trState->readOptions.get().debugID.get())
+			    .error(e);
+			TraceEvent("ClearData_Loop2_Catch").detail("Phase", "Loop2_Error").detail("ErrorCode", e.code());
+			caughtError = e;
+			needsErrorHandling = true;
+		}
+
+		if (needsErrorHandling) {
+			TraceEvent("ClearData_Loop2_ErrorHandling").detail("Phase", "Loop2_OnError");
+			co_await tr.onError(caughtError);
+		}
+	}
+
+	TraceEvent("ClearData_End").detail("Phase", "Success");
+	co_return;
+}
+
+namespace {
+
+std::string toHTML(const StringRef& binaryString) {
+	std::string s;
+
+	for (int i = 0; i < binaryString.size(); i++) {
+		uint8_t c = binaryString[i];
+		if (c == '<') {
+			s += "&lt;";
+		} else if (c == '>') {
+			s += "&gt;";
+		} else if (c == '&') {
+			s += "&amp;";
+		} else if (c == '"') {
+			s += "&quot;";
+		} else if (c == ' ') {
+			s += "&nbsp;";
+		} else if (c > 32 && c < 127) {
+			s += c;
+		} else {
+			s += format("<span class=\"binary\">[%02x]</span>", c);
+		}
+	}
+
+	return s;
+}
+
+} // namespace
+
+Future<Void> dumpDatabase(Database const& cx, std::string const& outputFilename, KeyRange const& range) {
+	try {
+		Transaction tr(cx);
+		while (true) {
+			FILE* output = fopen(outputFilename.c_str(), "wt");
+			Error err;
+			bool success = false;
+			try {
+				KeySelectorRef iter = firstGreaterOrEqual(range.begin);
+				Arena arena;
+				fprintf(output, "<html><head><style type=\"text/css\">.binary {color:red}</style></head><body>\n");
+				Version ver = co_await tr.getReadVersion();
+				fprintf(output, "<h3>Database version: %" PRId64 "</h3>", ver);
+
+				while (true) {
+					RangeResult results = co_await tr.getRange(iter, firstGreaterOrEqual(range.end), 1000);
+					for (int r = 0; r < results.size(); r++) {
+						std::string key = toHTML(results[r].key), value = toHTML(results[r].value);
+						fprintf(output, "<p>%s <b>:=</b> %s</p>\n", key.c_str(), value.c_str());
+					}
+					if (results.size() < 1000) {
+						break;
+					}
+					iter = firstGreaterThan(KeyRef(arena, results[results.size() - 1].key));
+				}
+				fprintf(output, "</body></html>");
+				success = true;
+			} catch (Error& e) {
+				err = e;
+			}
+			fclose(output);
+			if (success) {
+				TraceEvent("DatabaseDumped").detail("Filename", outputFilename);
+				co_return;
+			}
+			co_await tr.onError(err);
+		}
+	} catch (Error& e) {
+		TraceEvent(SevError, "DumpDatabaseError").error(e).detail("Filename", outputFilename);
+		throw;
+	}
+}
+
+std::vector<PerfMetric> aggregateMetrics(std::vector<std::vector<PerfMetric>> metrics) {
+	std::map<std::string, std::vector<PerfMetric>> metricMap;
+	for (int i = 0; i < metrics.size(); i++) {
+		std::vector<PerfMetric> workloadMetrics = metrics[i];
+		TraceEvent("MetricsReturned").detail("Count", workloadMetrics.size());
+		for (int m = 0; m < workloadMetrics.size(); m++) {
+			printf("Metric (%d, %d): %s, %f, %s\n",
+			       i,
+			       m,
+			       workloadMetrics[m].name().c_str(),
+			       workloadMetrics[m].value(),
+			       workloadMetrics[m].formatted().c_str());
+			metricMap[workloadMetrics[m].name()].push_back(workloadMetrics[m]);
+		}
+	}
+	TraceEvent("Metric")
+	    .detail("Name", "Reporting Clients")
+	    .detail("Value", (double)metrics.size())
+	    .detail("Formatted", format("%d", metrics.size()).c_str());
+
+	std::vector<PerfMetric> result;
+	std::map<std::string, std::vector<PerfMetric>>::iterator it;
+	for (it = metricMap.begin(); it != metricMap.end(); it++) {
+		auto& vec = it->second;
+		if (vec.empty())
+			continue;
+		double sum = 0;
+		for (int i = 0; i < vec.size(); i++)
+			sum += vec[i].value();
+		if (vec[0].averaged() && !vec.empty())
+			sum /= vec.size();
+		result.emplace_back(vec[0].name(), sum, Averaged::False, vec[0].format_code());
+	}
+	return result;
+}
+
+void logMetrics(std::vector<PerfMetric> metrics) {
+	for (int idx = 0; idx < metrics.size(); idx++)
+		TraceEvent("Metric")
+		    .detail("Name", metrics[idx].name())
+		    .detail("Value", metrics[idx].value())
+		    .detail("Formatted", format(metrics[idx].format_code().c_str(), metrics[idx].value()));
+}
+
+Future<Void> checkConsistencyScanAfterTest(Database cx, TesterConsistencyScanState* csState) {
+	if (!csState->enabled) {
+		co_return;
+	}
+
+	// mark it as done so later so this does not repeat
+	csState->enabled = false;
+
+	if (csState->enableAfter || csState->waitForComplete) {
+		printf("Enabling consistency scan after test ...\n");
+		TraceEvent("TestProgress").log("checkConsistencyScanAfterTest: calling enableConsistencyScanInSim()");
+		co_await enableConsistencyScanInSim(cx);
+		printf("Enabled consistency scan after test.\n");
+		TraceEvent("TestProgress").log("checkConsistencyScanAfterTest: enableConsistencyScanInSim() returned.");
+	}
+
+	TraceEvent("TestProgress").log("checkConsistencyScanAfterTest: calling disableConsistencyScanInSim()");
+	co_await disableConsistencyScanInSim(cx, csState->waitForComplete);
+	TraceEvent("TestProgress").log("checkConsistencyScanAfterTest: disableConsistencyScanInSim() returned.");
+
+	co_return;
+}
diff --git a/fdbserver/tester/DatabaseMaintenance.h b/fdbserver/tester/DatabaseMaintenance.h
new file mode 100644
index 00000000000..04e20eedaee
--- /dev/null
+++ b/fdbserver/tester/DatabaseMaintenance.h
@@ -0,0 +1,37 @@
+/*
+ * DatabaseMaintenance.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "fdbclient/NativeAPI.actor.h"
+
+struct TesterConsistencyScanState {
+	bool enabled = false;
+	bool enableAfter = false;
+	bool waitForComplete = false;
+};
+
+Future<Void> clearData(Database cx);
+Future<Void> dumpDatabase(Database const& cx, std::string const& outputFilename, KeyRange const& range);
+std::vector<PerfMetric> aggregateMetrics(std::vector<std::vector<PerfMetric>> metrics);
+Future<Void> checkConsistencyScanAfterTest(Database cx, TesterConsistencyScanState* csState);
diff --git a/fdbserver/KnobProtectiveGroups.cpp b/fdbserver/tester/KnobProtectiveGroups.cpp
similarity index 88%
rename from fdbserver/KnobProtectiveGroups.cpp
rename to fdbserver/tester/KnobProtectiveGroups.cpp
index 3bd9270eb31..81b7514ca77 100644
--- a/fdbserver/KnobProtectiveGroups.cpp
+++ b/fdbserver/tester/KnobProtectiveGroups.cpp
@@ -18,14 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbserver/KnobProtectiveGroups.h"
+#include "fdbserver/tester/KnobProtectiveGroups.h"
 
 #include <array>
 
 #include "flow/Knobs.h"
 #include "fdbclient/Knobs.h"
-#include "fdbclient/ServerKnobCollection.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 
 void KnobKeyValuePairs::set(const std::string& name, const ParsedKnobValue value) {
 	ASSERT(!knobs.contains(name));
@@ -67,11 +66,9 @@ void KnobProtectiveGroup::snapshotOriginalKnobs() {
 }
 
 void KnobProtectiveGroup::assignKnobs(const KnobKeyValuePairs& overrideKnobs) {
-	auto& mutableServerKnobs = dynamic_cast<ServerKnobCollection&>(IKnobCollection::getMutableGlobalKnobCollection());
-
 	for (const auto& [name, value] : overrideKnobs.getKnobs()) {
 		Standalone<KnobValueRef> valueRef = KnobValueRef::create(value);
-		bool success = mutableServerKnobs.trySetKnob(name, valueRef);
+		bool success = trySetServerKnob(name, valueRef);
 		if (!success) {
 			TraceEvent(SevError, "FailedToAssignKnob")
 			    .detail("KnobName", name)
diff --git a/fdbserver/tester/TestSpecParser.cpp b/fdbserver/tester/TestSpecParser.cpp
new file mode 100644
index 00000000000..fc633df9527
--- /dev/null
+++ b/fdbserver/tester/TestSpecParser.cpp
@@ -0,0 +1,413 @@
+/*
+ * TestSpecParser.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <toml.hpp>
+
+#include "flow/Platform.h"
+#include "flow/Trace.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "TestSpecParser.h"
+
+namespace {
+
+std::map<std::string, std::function<void(const std::string&)>> testSpecGlobalKeys = {
+	// These are read by SimulatedCluster and used before testers exist.  Thus, they must
+	// be recognized and accepted, but there's no point in placing them into a testSpec.
+	// testClass and testPriority are only used for TestHarness, we'll ignore those here
+	{ "testClass", [](std::string const&) {} },
+	{ "testPriority", [](std::string const&) {} },
+	{ "extraDatabaseMode",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseMode", ""); } },
+	{ "extraDatabaseCount",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDatabaseCount", ""); } },
+	{ "configureLocked",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedConfigureLocked", ""); } },
+	{ "minimumReplication",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMinimumReplication", ""); } },
+	{ "minimumRegions",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMinimumRegions", ""); } },
+	{ "logAntiQuorum",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedLogAntiQuorum", ""); } },
+	{ "buggify", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedBuggify", value); } },
+	// The test harness handles NewSeverity events specially.
+	{ "StderrSeverity", [](const std::string& value) { TraceEvent("StderrSeverity").detail("NewSeverity", value); } },
+	{ "ClientInfoLogging",
+	  [](const std::string& value) {
+	      if (value == "false") {
+		      setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
+	      }
+	      // else { } It is enable by default for tester
+	      TraceEvent("TestParserTest").detail("ClientInfoLogging", value);
+	  } },
+	{ "startIncompatibleProcess",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStartIncompatibleProcess", value); } },
+	{ "storageEngineExcludeTypes",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStorageEngineExcludeTypes", ""); } },
+	{ "maxTLogVersion",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMaxTLogVersion", ""); } },
+	{ "disableTss", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableTSS", ""); } },
+	{ "disableHostname",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } },
+	// TODO(gglass): remove this when it's no longer used.  For now it's a useful signal in a toml file
+	// for tests against functionality that might not actually be tenant related and hence need to keep.
+	{ "allowDefaultTenant",
+	  [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDefaultTenant", ""); } }
+};
+
+std::map<std::string, std::function<void(const std::string& value, TestSpec* spec)>> testSpecTestKeys = {
+	{ "testTitle",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->title = value;
+	      TraceEvent("TestParserTest").detail("ParsedTest", spec->title);
+	  } },
+	{ "timeout",
+	  [](const std::string& value, TestSpec* spec) {
+	      sscanf(value.c_str(), "%d", &(spec->timeout));
+	      ASSERT(spec->timeout > 0);
+	      TraceEvent("TestParserTest").detail("ParsedTimeout", spec->timeout);
+	  } },
+	{ "databasePingDelay",
+	  [](const std::string& value, TestSpec* spec) {
+	      double databasePingDelay;
+	      sscanf(value.c_str(), "%lf", &databasePingDelay);
+	      ASSERT(databasePingDelay >= 0);
+	      if (!spec->useDB && databasePingDelay > 0) {
+		      TraceEvent(SevError, "TestParserError")
+		          .detail("Reason", "Cannot have non-zero ping delay on test that does not use database")
+		          .detail("PingDelay", databasePingDelay)
+		          .detail("UseDB", spec->useDB);
+		      ASSERT(false);
+	      }
+	      spec->databasePingDelay = databasePingDelay;
+	      TraceEvent("TestParserTest").detail("ParsedPingDelay", spec->databasePingDelay);
+	  } },
+	{ "runSetup",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->phases = TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
+	      if (value == "true")
+		      spec->phases |= TestWorkload::SETUP;
+	      TraceEvent("TestParserTest").detail("ParsedSetupFlag", (spec->phases & TestWorkload::SETUP) != 0);
+	  } },
+	{ "dumpAfterTest",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->dumpAfterTest = (value == "true");
+	      TraceEvent("TestParserTest").detail("ParsedDumpAfter", spec->dumpAfterTest);
+	  } },
+	{ "clearAfterTest",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->clearAfterTest = (value == "true");
+	      TraceEvent("TestParserTest").detail("ParsedClearAfter", spec->clearAfterTest);
+	  } },
+	{ "useDB",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->useDB = (value == "true");
+	      TraceEvent("TestParserTest").detail("ParsedUseDB", spec->useDB);
+	      if (!spec->useDB)
+		      spec->databasePingDelay = 0.0;
+	  } },
+	{ "startDelay",
+	  [](const std::string& value, TestSpec* spec) {
+	      sscanf(value.c_str(), "%lf", &spec->startDelay);
+	      TraceEvent("TestParserTest").detail("ParsedStartDelay", spec->startDelay);
+	  } },
+	{ "runConsistencyCheck",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->runConsistencyCheck = (value == "true");
+	      TraceEvent("TestParserTest").detail("ParsedRunConsistencyCheck", spec->runConsistencyCheck);
+	  } },
+	{ "runConsistencyCheckOnTSS",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->runConsistencyCheckOnTSS = (value == "true");
+	      TraceEvent("TestParserTest").detail("ParsedRunConsistencyCheckOnTSS", spec->runConsistencyCheckOnTSS);
+	  } },
+	{ "maxDDRunTime",
+	  [](const std::string& value, TestSpec* spec) {
+	      sscanf(value.c_str(), "%lf", &(spec->maxDDRunTime));
+	      ASSERT(spec->maxDDRunTime >= 0);
+	      TraceEvent("TestParserTest").detail("ParsedMaxDDRunTime", spec->maxDDRunTime);
+	  } },
+	{ "waitForQuiescence",
+	  [](const std::string& value, TestSpec* spec) {
+	      bool toWait = value == "true";
+	      spec->waitForQuiescenceBegin = toWait;
+	      spec->waitForQuiescenceEnd = toWait;
+	      TraceEvent("TestParserTest").detail("ParsedWaitForQuiescence", toWait);
+	  } },
+	{ "waitForQuiescenceBegin",
+	  [](const std::string& value, TestSpec* spec) {
+	      bool toWait = value == "true";
+	      spec->waitForQuiescenceBegin = toWait;
+	      TraceEvent("TestParserTest").detail("ParsedWaitForQuiescenceBegin", toWait);
+	  } },
+	{ "waitForQuiescenceEnd",
+	  [](const std::string& value, TestSpec* spec) {
+	      bool toWait = value == "true";
+	      spec->waitForQuiescenceEnd = toWait;
+	      TraceEvent("TestParserTest").detail("ParsedWaitForQuiescenceEnd", toWait);
+	  } },
+	{ "simCheckRelocationDuration",
+	  [](const std::string& value, TestSpec* spec) {
+	      spec->simCheckRelocationDuration = (value == "true");
+	      TraceEvent("TestParserTest").detail("ParsedSimCheckRelocationDuration", spec->simCheckRelocationDuration);
+	  } },
+	{ "connectionFailuresDisableDuration",
+	  [](const std::string& value, TestSpec* spec) {
+	      double connectionFailuresDisableDuration;
+	      sscanf(value.c_str(), "%lf", &connectionFailuresDisableDuration);
+	      ASSERT(connectionFailuresDisableDuration >= 0);
+	      spec->simConnectionFailuresDisableDuration = connectionFailuresDisableDuration;
+	      TraceEvent("TestParserTest")
+	          .detail("ParsedSimConnectionFailuresDisableDuration", spec->simConnectionFailuresDisableDuration);
+	  } },
+	{ "simBackupAgents",
+	  [](const std::string& value, TestSpec* spec) {
+	      if (value == "BackupToFile" || value == "BackupToFileAndDB")
+		      spec->simBackupAgents = FDBBackupAgentType::BackupToFile;
+	      else
+		      spec->simBackupAgents = FDBBackupAgentType::NoBackupAgents;
+	      TraceEvent("TestParserTest").detail("ParsedSimBackupAgents", spec->simBackupAgents);
+
+	      if (value == "BackupToDB" || value == "BackupToFileAndDB")
+		      spec->simDrAgents = FDBBackupAgentType::BackupToDB;
+	      else
+		      spec->simDrAgents = FDBBackupAgentType::NoBackupAgents;
+	      TraceEvent("TestParserTest").detail("ParsedSimDrAgents", spec->simDrAgents);
+	  } },
+	{ "checkOnly",
+	  [](const std::string& value, TestSpec* spec) {
+	      if (value == "true")
+		      spec->phases = TestWorkload::CHECK;
+	  } },
+	{ "restorePerpetualWiggleSetting",
+	  [](const std::string& value, TestSpec* spec) {
+	      if (value == "false")
+		      spec->restorePerpetualWiggleSetting = false;
+	  } },
+	{ "runFailureWorkloads",
+	  [](const std::string& value, TestSpec* spec) { spec->runFailureWorkloads = (value == "true"); } },
+	{ "disabledFailureInjectionWorkloads",
+	  [](const std::string& value, TestSpec* spec) {
+	      // Expects a comma separated list of workload names in "value".
+	      // This custom encoding is needed because both text and toml files need to be supported
+	      // and "value" is passed in as a string.
+	      std::stringstream ss(value);
+	      while (ss.good()) {
+		      std::string substr;
+		      getline(ss, substr, ',');
+		      substr = removeWhitespace(substr);
+		      if (!substr.empty()) {
+			      spec->disabledFailureInjectionWorkloads.push_back(substr);
+		      }
+	      }
+	  } },
+};
+
+template <typename T>
+std::string toml_to_string(const T& value) {
+	// TOML formatting converts numbers to strings exactly how they're in the file
+	// and thus, is equivalent to testspec.  However, strings are quoted, so we
+	// must remove the quotes.
+	if (value.type() == toml::value_t::string) {
+		const std::string& formatted = toml::format(value);
+		return formatted.substr(1, formatted.size() - 2);
+	} else {
+		return toml::format(value);
+	}
+}
+
+} // namespace
+
+std::vector<TestSpec> readTests(std::ifstream& ifs) {
+	TestSpec spec;
+	std::vector<TestSpec> result;
+	Standalone<VectorRef<KeyValueRef>> workloadOptions;
+	std::string cline;
+	bool beforeFirstTest = true;
+	bool parsingWorkloads = false;
+
+	while (ifs.good()) {
+		getline(ifs, cline);
+		std::string line = removeWhitespace(cline);
+		if (line.empty() || line.find(';') == 0)
+			continue;
+
+		size_t found = line.find('=');
+		if (found == std::string::npos)
+			// hmmm, not good
+			continue;
+		std::string attrib = removeWhitespace(line.substr(0, found));
+		std::string value = removeWhitespace(line.substr(found + 1));
+
+		if (attrib == "testTitle") {
+			beforeFirstTest = false;
+			parsingWorkloads = false;
+			if (!workloadOptions.empty()) {
+				spec.options.push_back_deep(spec.options.arena(), workloadOptions);
+				workloadOptions = Standalone<VectorRef<KeyValueRef>>();
+			}
+			if (!spec.options.empty() && !spec.title.empty()) {
+				result.push_back(spec);
+				spec = TestSpec();
+			}
+
+			testSpecTestKeys[attrib](value, &spec);
+		} else if (testSpecTestKeys.find(attrib) != testSpecTestKeys.end()) {
+			if (parsingWorkloads)
+				TraceEvent(SevError, "TestSpecTestParamInWorkload").detail("Attrib", attrib).detail("Value", value);
+			testSpecTestKeys[attrib](value, &spec);
+		} else if (testSpecGlobalKeys.find(attrib) != testSpecGlobalKeys.end()) {
+			if (!beforeFirstTest)
+				TraceEvent(SevError, "TestSpecGlobalParamInTest").detail("Attrib", attrib).detail("Value", value);
+			testSpecGlobalKeys[attrib](value);
+		} else {
+			if (attrib == "testName") {
+				parsingWorkloads = true;
+				if (!workloadOptions.empty()) {
+					TraceEvent("TestParserFlush").detail("Reason", "new (compound) test");
+					spec.options.push_back_deep(spec.options.arena(), workloadOptions);
+					workloadOptions = Standalone<VectorRef<KeyValueRef>>();
+				}
+			}
+
+			workloadOptions.push_back_deep(workloadOptions.arena(), KeyValueRef(StringRef(attrib), StringRef(value)));
+			TraceEvent("TestParserOption").detail("ParsedKey", attrib).detail("ParsedValue", value);
+		}
+	}
+	if (!workloadOptions.empty())
+		spec.options.push_back_deep(spec.options.arena(), workloadOptions);
+	if (!spec.options.empty() && !spec.title.empty()) {
+		result.push_back(spec);
+	}
+
+	return result;
+}
+
+namespace {
+
+// In the current TOML scope, look for "knobs" field. If exists, translate all
+// key value pairs into KnobKeyValuePairs
+KnobKeyValuePairs getOverriddenKnobKeyValues(const toml::value& context) {
+	KnobKeyValuePairs result;
+
+	try {
+		const toml::array& overrideKnobs = toml::find(context, "knobs").as_array();
+		for (const toml::value& knob : overrideKnobs) {
+			for (const auto& [key, value_] : knob.as_table()) {
+				const std::string& value = toml_to_string(value_);
+				ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value);
+				if (std::get_if<NoKnobFound>(&parsedValue)) {
+					parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
+				}
+				if (std::get_if<NoKnobFound>(&parsedValue)) {
+					parsedValue = FLOW_KNOBS->parseKnobValue(key, value);
+				}
+				if (std::get_if<NoKnobFound>(&parsedValue)) {
+					TraceEvent(SevError, "TestSpecUnrecognizedKnob")
+					    .detail("KnobName", key)
+					    .detail("OverrideValue", value);
+					continue;
+				}
+				result.set(key, parsedValue);
+			}
+		}
+	} catch (const std::out_of_range&) {
+		// No knobs field in this scope, this is not an error
+	}
+
+	return result;
+}
+
+} // namespace
+
+namespace {
+
+TestSet readTOMLTests_(std::string fileName) {
+	Standalone<VectorRef<KeyValueRef>> workloadOptions;
+	TestSet result;
+
+	const toml::value& conf = toml::parse(fileName);
+
+	// Parse the global knob changes
+	result.overrideKnobs = getOverriddenKnobKeyValues(conf);
+
+	// Then parse each test
+	const toml::array& tests = toml::find(conf, "test").as_array();
+	for (const toml::value& test : tests) {
+		TestSpec spec;
+
+		// First handle all test-level settings
+		for (const auto& [k, v] : test.as_table()) {
+			if (k == "workload" || k == "knobs") {
+				continue;
+			}
+			if (testSpecTestKeys.find(k) != testSpecTestKeys.end()) {
+				testSpecTestKeys[k](toml_to_string(v), &spec);
+			} else {
+				TraceEvent(SevError, "TestSpecUnrecognizedTestParam")
+				    .detail("Attrib", k)
+				    .detail("Value", toml_to_string(v));
+			}
+		}
+
+		// And then copy the workload attributes to spec.options
+		const toml::array& workloads = toml::find(test, "workload").as_array();
+		for (const toml::value& workload : workloads) {
+			workloadOptions = Standalone<VectorRef<KeyValueRef>>();
+			TraceEvent("TestParserFlush").detail("Reason", "new (compound) test");
+			for (const auto& [attrib, v] : workload.as_table()) {
+				const std::string& value = toml_to_string(v);
+				workloadOptions.push_back_deep(workloadOptions.arena(),
+				                               KeyValueRef(StringRef(attrib), StringRef(value)));
+				TraceEvent("TestParserOption").detail("ParsedKey", attrib).detail("ParsedValue", value);
+			}
+			spec.options.push_back_deep(spec.options.arena(), workloadOptions);
+		}
+
+		// And then copy the knob attributes to spec.overrideKnobs
+		spec.overrideKnobs = getOverriddenKnobKeyValues(test);
+
+		result.testSpecs.push_back(spec);
+	}
+
+	return result;
+}
+
+} // namespace
+
+// A hack to catch and log std::exception, because TOML11 has very useful
+// error messages, but the actor framework can't handle std::exception.
+TestSet readTOMLTests(std::string fileName) {
+	try {
+		return readTOMLTests_(fileName);
+	} catch (std::exception& e) {
+		std::cerr << e.what() << std::endl;
+		TraceEvent("TOMLParseError").detail("Error", printable(e.what()));
+		// TODO: replace with toml_parse_error();
+		throw unknown_error();
+	}
+}
diff --git a/fdbserver/tester/TestSpecParser.h b/fdbserver/tester/TestSpecParser.h
new file mode 100644
index 00000000000..72b4fe999bc
--- /dev/null
+++ b/fdbserver/tester/TestSpecParser.h
@@ -0,0 +1,35 @@
+/*
+ * TestSpecParser.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+
+#include "fdbserver/tester/WorkloadUtils.h"
+
+struct TestSet {
+	KnobKeyValuePairs overrideKnobs;
+	std::vector<TestSpec> testSpecs;
+};
+
+std::vector<TestSpec> readTests(std::ifstream& ifs);
+TestSet readTOMLTests(std::string fileName);
diff --git a/fdbserver/tester/TesterServer.cpp b/fdbserver/tester/TesterServer.cpp
new file mode 100644
index 00000000000..ee861f80696
--- /dev/null
+++ b/fdbserver/tester/TesterServer.cpp
@@ -0,0 +1,623 @@
+/*
+ * TesterServer.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <algorithm>
+#include <cstdio>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "flow/ActorCollection.h"
+#include "flow/CoroUtils.h"
+#include "flow/DeterministicRandom.h"
+#include "flow/Platform.h"
+#include "flow/Trace.h"
+#include "fdbrpc/Locality.h"
+#include "fdbrpc/SimulatorProcessInfo.h"
+#include "fdbrpc/simulator.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "TesterServer.h"
+#include "fdbserver/tester/workloads.h"
+
+namespace {
+
+Standalone<VectorRef<KeyValueRef>> checkAllOptionsConsumed(VectorRef<KeyValueRef> options) {
+	static StringRef nothing = ""_sr;
+	Standalone<VectorRef<KeyValueRef>> unconsumed;
+	for (int i = 0; i < options.size(); i++)
+		if (!(options[i].value == nothing)) {
+			TraceEvent(SevError, "OptionNotConsumed")
+			    .detail("Key", options[i].key.toString().c_str())
+			    .detail("Value", options[i].value.toString().c_str());
+			unconsumed.push_back_deep(unconsumed.arena(), options[i]);
+		}
+	return unconsumed;
+}
+
+Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
+                                                 Reference<IClusterConnectionRecord> ccr,
+                                                 VectorRef<KeyValueRef> options,
+                                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	Reference<TestWorkload> workload;
+	Value testName = getOption(options, "testName"_sr, "no-test-specified"_sr);
+	WorkloadContext wcx;
+	wcx.clientId = work.clientId;
+	wcx.clientCount = work.clientCount;
+	wcx.ccr = ccr;
+	wcx.dbInfo = dbInfo;
+	wcx.options = options;
+	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.rangesToCheck = work.rangesToCheck;
+
+	workload = IWorkloadFactory::create(testName.toString(), wcx);
+	if (workload) {
+		co_await workload->initialized();
+	}
+
+	auto unconsumedOptions = checkAllOptionsConsumed(workload ? workload->options : VectorRef<KeyValueRef>());
+	if (!workload || !unconsumedOptions.empty()) {
+		TraceEvent evt(SevError, "TestCreationError");
+		evt.detail("TestName", testName);
+		if (!workload) {
+			evt.detail("Reason", "Null workload");
+			fprintf(stderr,
+			        "ERROR: Workload could not be created, perhaps testName (%s) is not a valid workload\n",
+			        printable(testName).c_str());
+		} else {
+			evt.detail("Reason", "Not all options consumed");
+			fprintf(stderr, "ERROR: Workload had invalid options. The following were unrecognized:\n");
+			for (int i = 0; i < unconsumedOptions.size(); i++)
+				fprintf(stderr,
+				        " '%s' = '%s'\n",
+				        unconsumedOptions[i].key.toString().c_str(),
+				        unconsumedOptions[i].value.toString().c_str());
+		}
+		throw test_specification_invalid();
+	}
+	co_return workload;
+}
+
+Future<Reference<TestWorkload>> getWorkloadIface(WorkloadRequest work,
+                                                 Reference<IClusterConnectionRecord> ccr,
+                                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	WorkloadContext wcx;
+	std::vector<Future<Reference<TestWorkload>>> ifaces;
+	if (work.options.size() < 1) {
+		TraceEvent(SevError, "TestCreationError").detail("Reason", "No options provided");
+		fprintf(stderr, "ERROR: No options were provided for workload.\n");
+		throw test_specification_invalid();
+	}
+
+	wcx.clientId = work.clientId;
+	wcx.clientCount = work.clientCount;
+	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.ccr = ccr;
+	wcx.dbInfo = dbInfo;
+	wcx.rangesToCheck = work.rangesToCheck;
+	// FIXME: Other stuff not filled in; why isn't this constructed here and passed down to the other
+	// getWorkloadIface()?
+	for (int i = 0; i < work.options.size(); i++) {
+		ifaces.push_back(getWorkloadIface(work, ccr, work.options[i], dbInfo));
+	}
+	co_await waitForAll(ifaces);
+	auto compound = makeReference<CompoundWorkload>(wcx);
+	for (int i = 0; i < work.options.size(); i++) {
+		compound->add(ifaces[i].getValue());
+	}
+	compound->addFailureInjection(work);
+	co_return compound;
+}
+
+} // namespace
+
+/**
+ * Only works in simulation. This method prints all simulated processes in a human readable form to stdout. It groups
+ * processes by data center, data hall, zone, and machine (in this order).
+ */
+void printSimulatedTopology() {
+	if (!g_network->isSimulated()) {
+		return;
+	}
+	auto processes = g_simulator->getAllProcesses();
+	std::sort(processes.begin(), processes.end(), [](ISimulator::ProcessInfo* lhs, ISimulator::ProcessInfo* rhs) {
+		auto l = lhs->locality;
+		auto r = rhs->locality;
+		if (l.dcId() != r.dcId()) {
+			return l.dcId() < r.dcId();
+		}
+		if (l.dataHallId() != r.dataHallId()) {
+			return l.dataHallId() < r.dataHallId();
+		}
+		if (l.zoneId() != r.zoneId()) {
+			return l.zoneId() < r.zoneId();
+		}
+		if (l.machineId() != r.machineId()) {
+			return l.machineId() < r.machineId();
+		}
+		return lhs->address < rhs->address;
+	});
+	printf("Simulated Cluster Topology:\n");
+	printf("===========================\n");
+	Optional<Standalone<StringRef>> dcId, dataHallId, zoneId, machineId;
+	for (auto p : processes) {
+		std::string indent = "";
+		if (dcId != p->locality.dcId()) {
+			dcId = p->locality.dcId();
+			printf("%sdcId: %s\n", indent.c_str(), p->locality.describeDcId().c_str());
+		}
+		indent += "  ";
+		if (dataHallId != p->locality.dataHallId()) {
+			dataHallId = p->locality.dataHallId();
+			printf("%sdataHallId: %s\n", indent.c_str(), p->locality.describeDataHall().c_str());
+		}
+		indent += "  ";
+		if (zoneId != p->locality.zoneId()) {
+			zoneId = p->locality.zoneId();
+			printf("%szoneId: %s\n", indent.c_str(), p->locality.describeZone().c_str());
+		}
+		indent += "  ";
+		if (machineId != p->locality.machineId()) {
+			machineId = p->locality.machineId();
+			printf("%smachineId: %s\n", indent.c_str(), p->locality.describeMachineId().c_str());
+		}
+		indent += "  ";
+		printf("%sAddress: %s\n", indent.c_str(), p->address.toString().c_str());
+		indent += "  ";
+		printf("%sClass: %s\n", indent.c_str(), p->startingClass.toString().c_str());
+		printf("%sName: %s\n", indent.c_str(), p->name.c_str());
+	}
+}
+
+Future<Void> databaseWarmer(Database cx) {
+	while (true) {
+		Transaction tr(cx);
+		co_await tr.getReadVersion();
+		co_await delay(0.25);
+	}
+	co_return; // Unreachable but required for coroutine
+}
+
+namespace {
+
+// Tries indefinitely to commit a simple, self conflicting transaction
+Future<Void> pingDatabase(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> v =
+			    co_await tr.get(StringRef("/Liveness/" + deterministicRandom()->randomUniqueID().toString()));
+			tr.makeSelfConflicting();
+			co_await tr.commit();
+			co_return;
+		} catch (Error& e) {
+			TraceEvent("PingingDatabaseTransactionError").error(e);
+			caughtError = e;
+			needsErrorHandling = true;
+		}
+
+		if (needsErrorHandling) {
+			co_await tr.onError(caughtError);
+		}
+	}
+}
+
+} // namespace
+
+Future<Void> testDatabaseLiveness(Database cx, double databasePingDelay, std::string context, double startDelay) {
+	co_await delay(startDelay);
+	while (true) {
+		Error caughtError;
+
+		try {
+			double start = now();
+			auto traceMsg = "PingingDatabaseLiveness_" + context;
+			TraceEvent(traceMsg.c_str()).log();
+			co_await timeoutError(pingDatabase(cx), databasePingDelay);
+			double pingTime = now() - start;
+			ASSERT(pingTime > 0);
+			TraceEvent(("PingingDatabaseLivenessDone_" + context).c_str()).detail("TimeTaken", pingTime);
+			co_await delay(databasePingDelay - pingTime);
+		} catch (Error& e) {
+			if (e.code() != error_code_actor_cancelled)
+				TraceEvent(SevError, ("PingingDatabaseLivenessError_" + context).c_str())
+				    .error(e)
+				    .detail("PingDelay", databasePingDelay);
+			throw;
+		}
+	}
+}
+
+namespace {
+
+template <class T>
+void sendResult(ReplyPromise<T>& reply, Optional<ErrorOr<T>> const& result) {
+	auto& res = result.get();
+	if (res.isError())
+		reply.sendError(res.getError());
+	else
+		reply.send(res.get());
+}
+
+Future<Reference<TestWorkload>> getConsistencyCheckUrgentWorkloadIface(WorkloadRequest work,
+                                                                       Reference<IClusterConnectionRecord> ccr,
+                                                                       Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	WorkloadContext wcx;
+	wcx.clientId = work.clientId;
+	wcx.clientCount = work.clientCount;
+	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.ccr = ccr;
+	wcx.dbInfo = dbInfo;
+	wcx.rangesToCheck = work.rangesToCheck;
+	Reference<TestWorkload> iface = co_await getWorkloadIface(work, ccr, work.options[0], dbInfo);
+	co_return iface;
+}
+
+Future<Void> runConsistencyCheckUrgentWorkloadAsync(Database cx,
+                                                    WorkloadInterface workIface,
+                                                    Reference<TestWorkload> workload) {
+	ReplyPromise<Void> jobReq = co_await workIface.start.getFuture();
+
+	try {
+		TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadReceived", workIface.id())
+		    .detail("WorkloadName", workload->description())
+		    .detail("ClientCount", workload->clientCount)
+		    .detail("ClientId", workload->clientId);
+		co_await workload->start(cx);
+		jobReq.send(Void());
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			throw e;
+		}
+		TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadError", workIface.id())
+		    .errorUnsuppressed(e)
+		    .detail("WorkloadName", workload->description())
+		    .detail("ClientCount", workload->clientCount)
+		    .detail("ClientId", workload->clientId);
+		jobReq.sendError(consistency_check_urgent_task_failed());
+	}
+	co_return;
+}
+
+Future<Void> testerServerConsistencyCheckerUrgentWorkload(WorkloadRequest work,
+                                                          Reference<IClusterConnectionRecord> ccr,
+                                                          Reference<AsyncVar<struct ServerDBInfo> const> dbInfo) {
+	WorkloadInterface workIface;
+	bool replied = false;
+
+	try {
+		Database cx = openDBOnServer(dbInfo);
+		co_await delay(1.0);
+		Reference<TestWorkload> workload = co_await getConsistencyCheckUrgentWorkloadIface(work, ccr, dbInfo);
+		Future<Void> test = runConsistencyCheckUrgentWorkloadAsync(cx, workIface, workload);
+		work.reply.send(workIface);
+		replied = true;
+		co_await test;
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			throw e;
+		}
+		TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterRunWorkloadFailed").errorUnsuppressed(e);
+		if (!replied) {
+			work.reply.sendError(e);
+		}
+	}
+	co_return;
+}
+
+Future<Void> runWorkloadAsync(Database cx,
+                              WorkloadInterface workIface,
+                              Reference<TestWorkload> workload,
+                              double databasePingDelay) {
+	Optional<ErrorOr<Void>> setupResult;
+	Optional<ErrorOr<Void>> startResult;
+	Optional<ErrorOr<CheckReply>> checkResult;
+	ReplyPromise<Void> setupReq;
+	ReplyPromise<Void> startReq;
+	ReplyPromise<CheckReply> checkReq;
+	ReplyPromise<std::vector<PerfMetric>> metricsReq;
+
+	TraceEvent("TestBeginAsync", workIface.id())
+	    .detail("Workload", workload->description())
+	    .detail("DatabasePingDelay", databasePingDelay);
+
+	Future<Void> databaseError =
+	    databasePingDelay == 0.0 ? Never() : testDatabaseLiveness(cx, databasePingDelay, "RunWorkloadAsync");
+
+	ReplyPromise<Void> stopReq;
+	while (true) {
+		int action = 0;
+		co_await Choose()
+		    .When(workIface.setup.getFuture(),
+		          [&](ReplyPromise<Void> const& req) {
+			          setupReq = req;
+			          action = 1;
+		          })
+		    .When(workIface.start.getFuture(),
+		          [&](ReplyPromise<Void> const& req) {
+			          startReq = req;
+			          action = 2;
+		          })
+		    .When(workIface.check.getFuture(),
+		          [&](ReplyPromise<CheckReply> const& req) {
+			          checkReq = req;
+			          action = 3;
+		          })
+		    .When(workIface.metrics.getFuture(),
+		          [&](ReplyPromise<std::vector<PerfMetric>> const& req) {
+			          metricsReq = req;
+			          action = 4;
+		          })
+		    .When(workIface.stop.getFuture(),
+		          [&](ReplyPromise<Void> const& r) {
+			          stopReq = r;
+			          action = 5;
+		          })
+		    .run();
+
+		if (action == 5) {
+			stopReq.send(Void());
+			break;
+		}
+
+		if (action == 1) {
+			// setup
+			printf("Test received trigger for setup...\n");
+			TraceEvent("TestSetupBeginning", workIface.id()).detail("Workload", workload->description());
+			if (!setupResult.present()) {
+				try {
+					co_await (workload->setup(cx) || databaseError);
+					TraceEvent("TestSetupComplete", workIface.id()).detail("Workload", workload->description());
+					setupResult = Void();
+				} catch (Error& e) {
+					setupResult = operation_failed();
+					TraceEvent(SevError, "TestSetupError", workIface.id())
+					    .error(e)
+					    .detail("Workload", workload->description());
+					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
+						throw;
+				}
+			}
+			sendResult(setupReq, setupResult);
+		} else if (action == 2) {
+			// start
+			if (!startResult.present()) {
+				try {
+					TraceEvent("TestStarting", workIface.id()).detail("Workload", workload->description());
+					co_await (workload->start(cx) || databaseError);
+					startResult = Void();
+				} catch (Error& e) {
+					startResult = operation_failed();
+					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
+						throw;
+					TraceEvent(SevError, "TestFailure", workIface.id())
+					    .errorUnsuppressed(e)
+					    .detail("Reason", "Error starting workload")
+					    .detail("Workload", workload->description());
+				}
+				TraceEvent("TestComplete", workIface.id())
+				    .detail("Workload", workload->description())
+				    .detail("OK", !startResult.get().isError());
+				printf("%s complete\n", workload->description().c_str());
+			}
+			sendResult(startReq, startResult);
+		} else if (action == 3) {
+			// check
+			if (!checkResult.present()) {
+				try {
+					TraceEvent("TestChecking", workIface.id()).detail("Workload", workload->description());
+					bool check = co_await timeoutError(workload->check(cx), workload->getCheckTimeout());
+					checkResult = CheckReply{ (!startResult.present() || !startResult.get().isError()) && check };
+				} catch (Error& e) {
+					checkResult = operation_failed();
+					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
+						throw;
+					TraceEvent(SevError, "TestFailure", workIface.id())
+					    .error(e)
+					    .detail("Reason", "Error checking workload")
+					    .detail("Workload", workload->description());
+				}
+				TraceEvent("TestCheckComplete", workIface.id()).detail("Workload", workload->description());
+			}
+			sendResult(checkReq, checkResult);
+		} else if (action == 4) {
+			// metrics
+			try {
+				std::vector<PerfMetric> m = co_await workload->getMetrics();
+				TraceEvent("WorkloadSendMetrics", workIface.id()).detail("Count", m.size());
+				metricsReq.send(m);
+			} catch (Error& e) {
+				if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
+					throw;
+				TraceEvent(SevError, "WorkloadSendMetrics", workIface.id()).error(e);
+				metricsReq.sendError(operation_failed());
+			}
+		}
+	}
+	co_return;
+}
+
+Future<Void> testerServerWorkload(WorkloadRequest work,
+                                  Reference<IClusterConnectionRecord> ccr,
+                                  Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
+                                  LocalityData locality) {
+	WorkloadInterface workIface;
+	bool replied = false;
+	Database cx;
+
+	try {
+		std::map<std::string, std::string> details;
+		details["WorkloadTitle"] = printable(work.title);
+		details["ClientId"] = format("%d", work.clientId);
+		details["ClientCount"] = format("%d", work.clientCount);
+		details["WorkloadTimeout"] = format("%d", work.timeout);
+		startRole(Role::TESTER, workIface.id(), UID(), details);
+
+		if (work.useDatabase) {
+			cx = Database::createDatabase(ccr, ApiVersion::LATEST_VERSION, IsInternal::True, locality);
+			co_await delay(1.0);
+		}
+
+		// add test for "done" ?
+		TraceEvent("WorkloadReceived", workIface.id()).detail("Title", work.title);
+		Reference<TestWorkload> workload = co_await getWorkloadIface(work, ccr, dbInfo);
+		if (!workload) {
+			TraceEvent("TestCreationError").detail("Reason", "Workload could not be created");
+			fprintf(stderr, "ERROR: The workload could not be created.\n");
+			throw test_specification_invalid();
+		}
+		Future<Void> test = runWorkloadAsync(cx, workIface, workload, work.databasePingDelay) ||
+		                    traceRole(Role::TESTER, workIface.id());
+		work.reply.send(workIface);
+		replied = true;
+
+		if (work.timeout > 0) {
+			test = timeoutError(test, work.timeout);
+		}
+
+		co_await test;
+
+		endRole(Role::TESTER, workIface.id(), "Complete");
+	} catch (Error& e) {
+		TraceEvent(SevDebug, "TesterWorkloadFailed").errorUnsuppressed(e);
+		if (!replied) {
+			if (e.code() == error_code_test_specification_invalid)
+				work.reply.sendError(e);
+			else
+				work.reply.sendError(operation_failed());
+		}
+
+		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete ||
+		          e.code() == error_code_actor_cancelled;
+		endRole(Role::TESTER, workIface.id(), "Error", ok, e);
+
+		if (e.code() != error_code_test_specification_invalid && e.code() != error_code_timed_out) {
+			throw; // fatal errors will kill the testerServer as well
+		}
+	}
+	co_return;
+}
+
+} // namespace
+
+Future<Void> testerServerCore(TesterInterface const& interf,
+                              Reference<IClusterConnectionRecord> const& ccr,
+                              Reference<AsyncVar<struct ServerDBInfo> const> const& dbInfo,
+                              LocalityData const& locality,
+                              Optional<std::string> const& expectedWorkLoad) {
+	// C++20 coroutine safety: const& parameters only store the reference in the coroutine frame,
+	// not the object. The referred-to object may be destroyed after the coroutine suspends
+	// (e.g. local variables in a caller's if-block, or temporaries from default arguments).
+	// Copy all const& parameters to ensure they survive across suspend points.
+	TesterInterface interfCopy = interf;
+	Reference<IClusterConnectionRecord> ccrCopy = ccr;
+	Reference<AsyncVar<struct ServerDBInfo> const> dbInfoCopy = dbInfo;
+	LocalityData localityCopy = locality;
+	Optional<std::string> expectedWorkLoadCopy = expectedWorkLoad;
+
+	PromiseStream<Future<Void>> addWorkload;
+	Future<Void> workerFatalError = actorCollection(addWorkload.getFuture());
+
+	// Dedicated to consistencyCheckerUrgent
+	// At any time, we only allow at most 1 consistency checker workload on a server
+	std::pair<int64_t, Future<Void>> consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>());
+
+	TraceEvent("StartingTesterServerCore", interfCopy.id())
+	    .detail("ExpectedWorkload", expectedWorkLoadCopy.present() ? expectedWorkLoadCopy.get() : "[Unset]");
+
+	while (true) {
+		int action = 0;
+		WorkloadRequest workReq;
+		Future<Void> checkerDone =
+		    consistencyCheckerUrgentTester.second.isValid() ? consistencyCheckerUrgentTester.second : Never();
+
+		co_await Choose()
+		    .When(workerFatalError, [&](Void const&) { action = 1; })
+		    .When(checkerDone, [&](Void const&) { action = 2; })
+		    .When(interfCopy.recruitments.getFuture(),
+		          [&](WorkloadRequest const& work) {
+			          workReq = work;
+			          action = 3;
+		          })
+		    .run();
+
+		if (action == 1) {
+			break;
+		}
+
+		if (action == 2) {
+			ASSERT(consistencyCheckerUrgentTester.first != 0);
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadEnd", interfCopy.id())
+			    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first);
+			consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>()); // reset
+			continue;
+		}
+
+		if (action == 3) {
+			if (expectedWorkLoadCopy.present() && expectedWorkLoadCopy.get() != workReq.title) {
+				TraceEvent(SevError, "StartingTesterServerCoreUnexpectedWorkload", interfCopy.id())
+				    .detail("ClientId", workReq.clientId)
+				    .detail("ClientCount", workReq.clientCount)
+				    .detail("ExpectedWorkLoad", expectedWorkLoadCopy.get())
+				    .detail("WorkLoad", workReq.title);
+				// Drop the workload
+			} else if (workReq.title == "ConsistencyCheckUrgent") {
+				// The workload is a consistency checker urgent workload
+				if (workReq.sharedRandomNumber == consistencyCheckerUrgentTester.first) {
+					// A single req can be sent for multiple times. In this case, the sharedRandomNumber is same as
+					// the existing one. For this scenario, we reply an error. This case should be rare.
+					TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterDuplicatedRequest", interfCopy.id())
+					    .detail("ConsistencyCheckerId", workReq.sharedRandomNumber)
+					    .detail("ClientId", workReq.clientId)
+					    .detail("ClientCount", workReq.clientCount);
+					workReq.reply.sendError(consistency_check_urgent_duplicate_request());
+				} else {
+					// When the req.sharedRandomNumber is different from the existing one, the cluster has muiltiple
+					// consistencycheckurgent roles at the same time. Evenutally, the cluster will have only one
+					// consistencycheckurgent role in a stable state. So, in this case, we simply let the new request to
+					// overwrite the old request. After the work is destroyed, the broken_promise will be replied.
+					if (consistencyCheckerUrgentTester.second.isValid() &&
+					    !consistencyCheckerUrgentTester.second.isReady()) {
+						TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_TesterWorkloadConflict", interfCopy.id())
+						    .detail("ExistingConsistencyCheckerId", consistencyCheckerUrgentTester.first)
+						    .detail("ArrivingConsistencyCheckerId", workReq.sharedRandomNumber)
+						    .detail("ClientId", workReq.clientId)
+						    .detail("ClientCount", workReq.clientCount);
+					}
+					consistencyCheckerUrgentTester =
+					    std::make_pair(workReq.sharedRandomNumber,
+					                   testerServerConsistencyCheckerUrgentWorkload(workReq, ccrCopy, dbInfoCopy));
+					TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadInitialized", interfCopy.id())
+					    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first)
+					    .detail("ClientId", workReq.clientId)
+					    .detail("ClientCount", workReq.clientCount);
+				}
+			} else {
+				addWorkload.send(testerServerWorkload(workReq, ccrCopy, dbInfoCopy, localityCopy));
+			}
+		}
+	}
+	co_return;
+}
diff --git a/fdbserver/tester/TesterServer.h b/fdbserver/tester/TesterServer.h
new file mode 100644
index 00000000000..3c8af854b85
--- /dev/null
+++ b/fdbserver/tester/TesterServer.h
@@ -0,0 +1,28 @@
+/*
+ * TesterServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "fdbclient/NativeAPI.actor.h"
+
+Future<Void> testDatabaseLiveness(Database cx, double databasePingDelay, std::string context, double startDelay = 0.0);
+void printSimulatedTopology();
diff --git a/fdbserver/tester/WorkloadUtils.cpp b/fdbserver/tester/WorkloadUtils.cpp
new file mode 100644
index 00000000000..b0c168dabcc
--- /dev/null
+++ b/fdbserver/tester/WorkloadUtils.cpp
@@ -0,0 +1,439 @@
+/*
+ * WorkloadUtils.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <cinttypes>
+#include <cstdio>
+#include <memory>
+
+#include <fmt/ranges.h>
+
+#include "flow/CoroUtils.h"
+#include "flow/DeterministicRandom.h"
+#include "flow/Trace.h"
+#include "flow/genericactors.actor.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
+
+namespace {
+
+constexpr char HEX_CHAR_LOOKUP[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
+
+Future<std::vector<PerfMetric>> getMetricsCompoundWorkload(CompoundWorkload* self) {
+	std::vector<Future<std::vector<PerfMetric>>> results;
+	for (int w = 0; w < self->workloads.size(); w++) {
+		std::vector<PerfMetric> p;
+		results.push_back(self->workloads[w]->getMetrics());
+	}
+	co_await waitForAll(results);
+	std::vector<PerfMetric> res;
+	for (int i = 0; i < results.size(); ++i) {
+		auto const& p = results[i].get();
+		for (auto const& m : p) {
+			res.push_back(m.withPrefix(self->workloads[i]->description() + "."));
+		}
+	}
+	co_return res;
+}
+
+} // namespace
+
+WorkloadContext::WorkloadContext() = default;
+
+WorkloadContext::WorkloadContext(const WorkloadContext& r) = default;
+
+WorkloadContext::~WorkloadContext() = default;
+
+void emplaceIndex(uint8_t* data, int offset, int64_t index) {
+	for (int i = 0; i < 16; i++) {
+		data[(15 - i) + offset] = HEX_CHAR_LOOKUP[index & 0xf];
+		index = index >> 4;
+	}
+}
+
+Key KVWorkload::getRandomKey() const {
+	return getRandomKey(absentFrac);
+}
+
+Key KVWorkload::getRandomKey(double absentFrac) const {
+	if (absentFrac > 0.0000001) {
+		return getRandomKey(deterministicRandom()->random01() < absentFrac);
+	} else {
+		return getRandomKey(false);
+	}
+}
+
+Key KVWorkload::getRandomKey(bool absent) const {
+	return keyForIndex(deterministicRandom()->randomInt(0, nodeCount), absent);
+}
+
+Key KVWorkload::keyForIndex(uint64_t index) const {
+	if (absentFrac > 0.0000001) {
+		return keyForIndex(index, deterministicRandom()->random01() < absentFrac);
+	} else {
+		return keyForIndex(index, false);
+	}
+}
+
+int64_t KVWorkload::indexForKey(const KeyRef& key, bool absent) const {
+	int idx = 0;
+	if (nodePrefix > 0) {
+		ASSERT(keyBytes >= 32);
+		idx += 16;
+	}
+	ASSERT(keyBytes >= 16);
+	auto end = key.size() - idx - (absent ? 1 : 0);
+	std::string str((char*)key.begin() + idx, end);
+	int64_t res = std::stoll(str, nullptr, 16);
+	return res;
+}
+
+Key KVWorkload::keyForIndex(uint64_t index, bool absent) const {
+	int adjustedKeyBytes = absent ? (keyBytes + 1) : keyBytes;
+	Key result = makeString(adjustedKeyBytes);
+	uint8_t* data = mutateString(result);
+	memset(data, '.', adjustedKeyBytes);
+
+	int idx = 0;
+	if (nodePrefix > 0) {
+		ASSERT(keyBytes >= 32);
+		emplaceIndex(data, 0, nodePrefix);
+		idx += 16;
+	}
+	ASSERT(keyBytes >= 16);
+	emplaceIndex(data, idx, (int64_t)index);
+
+	return result;
+}
+
+Future<Void> poisson(double* last, double meanInterval) {
+	*last += meanInterval * -log(deterministicRandom()->random01());
+	co_await delayUntil(*last);
+	co_return;
+}
+
+Future<Void> uniform(double* last, double meanInterval) {
+	*last += meanInterval;
+	co_await delayUntil(*last);
+	co_return;
+}
+
+Value getOption(VectorRef<KeyValueRef> options, Key key, Value defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			Value value = options[i].value;
+			options[i].value = ""_sr;
+			return value;
+		}
+
+	return defaultValue;
+}
+
+int getOption(VectorRef<KeyValueRef> options, Key key, int defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			int r;
+			if (sscanf(options[i].value.toString().c_str(), "%d", &r)) {
+				options[i].value = ""_sr;
+				return r;
+			} else {
+				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
+				throw test_specification_invalid();
+			}
+		}
+
+	return defaultValue;
+}
+
+uint64_t getOption(VectorRef<KeyValueRef> options, Key key, uint64_t defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			uint64_t r;
+			if (sscanf(options[i].value.toString().c_str(), "%" SCNd64, &r)) {
+				options[i].value = ""_sr;
+				return r;
+			} else {
+				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
+				throw test_specification_invalid();
+			}
+		}
+
+	return defaultValue;
+}
+
+int64_t getOption(VectorRef<KeyValueRef> options, Key key, int64_t defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			int64_t r;
+			if (sscanf(options[i].value.toString().c_str(), "%" SCNd64, &r)) {
+				options[i].value = ""_sr;
+				return r;
+			} else {
+				TraceEvent(SevError, "InvalidTestOption").detail("OptionName", key);
+				throw test_specification_invalid();
+			}
+		}
+
+	return defaultValue;
+}
+
+double getOption(VectorRef<KeyValueRef> options, Key key, double defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			float r;
+			if (sscanf(options[i].value.toString().c_str(), "%f", &r)) {
+				options[i].value = ""_sr;
+				return r;
+			}
+		}
+
+	return defaultValue;
+}
+
+bool getOption(VectorRef<KeyValueRef> options, Key key, bool defaultValue) {
+	Value p = getOption(options, key, defaultValue ? "true"_sr : "false"_sr);
+	if (p == "true"_sr)
+		return true;
+	if (p == "false"_sr)
+		return false;
+	ASSERT(false);
+	return false;
+}
+
+std::vector<std::string> getOption(VectorRef<KeyValueRef> options, Key key, std::vector<std::string> defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			std::vector<std::string> v;
+			int begin = 0;
+			for (int c = 0; c < options[i].value.size(); c++)
+				if (options[i].value[c] == ',') {
+					v.push_back(options[i].value.substr(begin, c - begin).toString());
+					begin = c + 1;
+				}
+			v.push_back(options[i].value.substr(begin).toString());
+			options[i].value = ""_sr;
+			return v;
+		}
+	return defaultValue;
+}
+
+std::vector<int> getOption(VectorRef<KeyValueRef> options, Key key, std::vector<int> defaultValue) {
+	for (int i = 0; i < options.size(); i++)
+		if (options[i].key == key) {
+			std::vector<int> v;
+			int begin = 0;
+			for (int c = 0; c < options[i].value.size(); c++)
+				if (options[i].value[c] == ',') {
+					v.push_back(atoi((char*)options[i].value.begin() + begin));
+					begin = c + 1;
+				}
+			v.push_back(atoi((char*)options[i].value.begin() + begin));
+			options[i].value = ""_sr;
+			return v;
+		}
+	return defaultValue;
+}
+
+bool hasOption(VectorRef<KeyValueRef> options, Key key) {
+	for (const auto& option : options) {
+		if (option.key == key) {
+			return true;
+		}
+	}
+	return false;
+}
+
+CompoundWorkload::CompoundWorkload(WorkloadContext& wcx) : TestWorkload(wcx) {}
+
+CompoundWorkload* CompoundWorkload::add(Reference<TestWorkload>&& w) {
+	workloads.push_back(std::move(w));
+	return this;
+}
+
+std::string CompoundWorkload::description() const {
+	std::vector<std::string> names;
+	names.reserve(workloads.size());
+	for (auto const& w : workloads) {
+		names.push_back(w->description());
+	}
+	return fmt::format("{}", fmt::join(std::move(names), ";"));
+}
+
+Future<Void> CompoundWorkload::setup(Database const& cx) {
+	std::vector<Future<Void>> all;
+	all.reserve(workloads.size());
+	for (int w = 0; w < workloads.size(); w++)
+		all.push_back(workloads[w]->setup(cx));
+	auto done = waitForAll(all);
+	if (failureInjection.empty()) {
+		return done;
+	}
+	std::vector<Future<Void>> res;
+	res.reserve(failureInjection.size());
+	for (auto& f : failureInjection) {
+		res.push_back(f->setupInjectionWorkload(cx, done));
+	}
+	return waitForAll(res);
+}
+
+Future<Void> CompoundWorkload::start(Database const& cx) {
+	std::vector<Future<Void>> all;
+	all.reserve(workloads.size() + failureInjection.size());
+	auto wCount = std::make_shared<unsigned>(0);
+	auto startWorkload = [&](TestWorkload& workload) -> Future<Void> {
+		auto workloadName = workload.description();
+		++(*wCount);
+		TraceEvent("WorkloadRunStatus").detail("Name", workloadName).detail("Count", *wCount).detail("Phase", "Start");
+		return fmap(
+		    [workloadName, wCount](Void value) {
+			    --(*wCount);
+			    TraceEvent("WorkloadRunStatus")
+			        .detail("Name", workloadName)
+			        .detail("Remaining", *wCount)
+			        .detail("Phase", "End");
+			    return Void();
+		    },
+		    workload.start(cx));
+	};
+	for (auto& workload : workloads) {
+		all.push_back(startWorkload(*workload));
+	}
+	for (auto& workload : failureInjection) {
+		all.push_back(startWorkload(*workload));
+	}
+	return waitForAll(all);
+}
+
+Future<bool> CompoundWorkload::check(Database const& cx) {
+	std::vector<Future<bool>> all;
+	all.reserve(workloads.size() + failureInjection.size());
+	auto wCount = std::make_shared<unsigned>(0);
+	auto starter = [&](TestWorkload& workload) -> Future<bool> {
+		++(*wCount);
+		std::string workloadName = workload.description();
+		TraceEvent("WorkloadCheckStatus")
+		    .detail("Name", workloadName)
+		    .detail("Count", *wCount)
+		    .detail("Phase", "Start");
+		return fmap(
+		    [workloadName, wCount](bool ret) {
+			    --(*wCount);
+			    TraceEvent("WorkloadCheckStatus")
+			        .detail("Name", workloadName)
+			        .detail("Remaining", *wCount)
+			        .detail("Phase", "End");
+			    return ret;
+		    },
+		    workload.check(cx));
+	};
+	for (auto& workload : workloads) {
+		all.push_back(starter(*workload));
+	}
+	for (auto& workload : failureInjection) {
+		all.push_back(starter(*workload));
+	}
+	return allTrue(all);
+}
+
+void CompoundWorkload::addFailureInjection(WorkloadRequest& work) {
+	if (!work.runFailureWorkloads) {
+		return;
+	}
+	std::set<std::string> disabledWorkloads;
+	for (auto const& w : workloads) {
+		w->disableFailureInjectionWorkloads(disabledWorkloads);
+	}
+	if (disabledWorkloads.contains("all")) {
+		return;
+	}
+	auto& factories = IFailureInjectorFactory::factories();
+	DeterministicRandom random(sharedRandomNumber);
+	for (auto& factory : factories) {
+		auto workload = factory->create(*this);
+		if (disabledWorkloads.contains(workload->description())) {
+			continue;
+		}
+		if (std::find(work.disabledFailureInjectionWorkloads.begin(),
+		              work.disabledFailureInjectionWorkloads.end(),
+		              workload->description()) != work.disabledFailureInjectionWorkloads.end()) {
+			continue;
+		}
+		while (shouldInjectFailure(random, work, workload)) {
+			workload->initFailureInjectionMode(random);
+			TraceEvent("AddFailureInjectionWorkload")
+			    .detail("Name", workload->description())
+			    .detail("ClientID", work.clientId)
+			    .detail("ClientCount", clientCount)
+			    .detail("Title", work.title);
+			failureInjection.push_back(workload);
+			workload = factory->create(*this);
+		}
+	}
+}
+
+bool CompoundWorkload::shouldInjectFailure(DeterministicRandom& random,
+                                           const WorkloadRequest& work,
+                                           Reference<FailureInjectionWorkload> failure) const {
+	auto desc = failure->description();
+	unsigned alreadyAdded =
+	    std::count_if(workloads.begin(), workloads.end(), [&desc](auto const& w) { return w->description() == desc; });
+	alreadyAdded += std::count_if(
+	    failureInjection.begin(), failureInjection.end(), [&desc](auto const& w) { return w->description() == desc; });
+	return failure->shouldInject(random, work, alreadyAdded);
+}
+
+Future<std::vector<PerfMetric>> CompoundWorkload::getMetrics() {
+	return getMetricsCompoundWorkload(this);
+}
+
+double CompoundWorkload::getCheckTimeout() const {
+	double m = 0;
+	for (int w = 0; w < workloads.size(); w++)
+		m = std::max(workloads[w]->getCheckTimeout(), m);
+	return m;
+}
+
+void CompoundWorkload::getMetrics(std::vector<PerfMetric>&) {
+	ASSERT(false);
+}
+
+void TestWorkload::disableFailureInjectionWorkloads(std::set<std::string>& out) const {}
+
+FailureInjectionWorkload::FailureInjectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+void FailureInjectionWorkload::initFailureInjectionMode(DeterministicRandom& random) {}
+
+bool FailureInjectionWorkload::shouldInject(DeterministicRandom& random,
+                                            const WorkloadRequest& work,
+                                            const unsigned alreadyAdded) const {
+	return alreadyAdded < 3 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
+}
+
+Future<Void> FailureInjectionWorkload::setupInjectionWorkload(const Database& cx, Future<Void> done) {
+	return holdWhile(this->setup(cx), done);
+}
+
+Future<Void> FailureInjectionWorkload::startInjectionWorkload(const Database& cx, Future<Void> done) {
+	return holdWhile(this->start(cx), done);
+}
+
+Future<bool> FailureInjectionWorkload::checkInjectionWorkload(const Database& cx, Future<bool> done) {
+	return holdWhile(this->check(cx), done);
+}
diff --git a/fdbserver/include/fdbserver/KnobProtectiveGroups.h b/fdbserver/tester/include/fdbserver/tester/KnobProtectiveGroups.h
similarity index 86%
rename from fdbserver/include/fdbserver/KnobProtectiveGroups.h
rename to fdbserver/tester/include/fdbserver/tester/KnobProtectiveGroups.h
index abf73a85a86..4ae331a254c 100644
--- a/fdbserver/include/fdbserver/KnobProtectiveGroups.h
+++ b/fdbserver/tester/include/fdbserver/tester/KnobProtectiveGroups.h
@@ -18,13 +18,13 @@
  * limitations under the License.
  */
 
-#ifndef FDBSERVER_KNOBPROTECTIVEGROUPS_H
-#define FDBSERVER_KNOBPROTECTIVEGROUPS_H
+#ifndef FDBSERVER_TESTER_KNOBPROTECTIVEGROUPS_H
+#define FDBSERVER_TESTER_KNOBPROTECTIVEGROUPS_H
 
 #include <array>
 #include <unordered_map>
 
-#include "fdbclient/IKnobCollection.h"
+#include "flow/Knobs.h"
 
 // A list of knob key value pairs
 class KnobKeyValuePairs {
@@ -54,8 +54,8 @@ class KnobProtectiveGroup {
 	void assignKnobs(const KnobKeyValuePairs& overrideKnobs);
 
 public:
-	KnobProtectiveGroup(const KnobKeyValuePairs& overridenKnobs_);
+	explicit KnobProtectiveGroup(const KnobKeyValuePairs& overridenKnobs_);
 	~KnobProtectiveGroup();
 };
 
-#endif // FDBSERVER_KNOBPROTECTIVEGROUPS_H
\ No newline at end of file
+#endif // FDBSERVER_TESTER_KNOBPROTECTIVEGROUPS_H
diff --git a/fdbserver/tester/include/fdbserver/tester/TestEncryptionUtils.h b/fdbserver/tester/include/fdbserver/tester/TestEncryptionUtils.h
new file mode 100644
index 00000000000..3a4c19d7be5
--- /dev/null
+++ b/fdbserver/tester/include/fdbserver/tester/TestEncryptionUtils.h
@@ -0,0 +1,29 @@
+/*
+ * TestEncryptionUtils.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_TESTER_TESTENCRYPTIONUTILS_H
+#define FDBSERVER_TESTER_TESTENCRYPTIONUTILS_H
+#pragma once
+
+#include <string>
+
+std::string getTestEncryptionFileName();
+
+#endif
diff --git a/fdbserver/tester/include/fdbserver/tester/WorkloadUtils.h b/fdbserver/tester/include/fdbserver/tester/WorkloadUtils.h
new file mode 100644
index 00000000000..5d2206674a7
--- /dev/null
+++ b/fdbserver/tester/include/fdbserver/tester/WorkloadUtils.h
@@ -0,0 +1,129 @@
+/*
+ * WorkloadUtils.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_TESTER_WORKLOADUTILS_H
+#define FDBSERVER_TESTER_WORKLOADUTILS_H
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/KnobProtectiveGroups.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbrpc/PerfMetric.h"
+#include "fdbrpc/simulator.h"
+
+struct DistributedTestResults {
+	std::vector<PerfMetric> metrics;
+	int successes, failures;
+
+	DistributedTestResults() {}
+
+	DistributedTestResults(std::vector<PerfMetric> const& metrics, int successes, int failures)
+	  : metrics(metrics), successes(successes), failures(failures) {}
+
+	bool ok() const { return successes && !failures; }
+};
+
+class TestSpec {
+public:
+	TestSpec() {
+		title = StringRef();
+		dumpAfterTest = false;
+		clearAfterTest = g_network->isSimulated();
+		useDB = true;
+		startDelay = 30.0;
+		phases = TestWorkload::SETUP | TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
+		timeout = g_network->isSimulated() ? 15000 : 1500;
+		databasePingDelay = g_network->isSimulated() ? 0.0 : 15.0;
+		runConsistencyCheck = g_network->isSimulated();
+		runConsistencyCheckOnCache = false;
+		runConsistencyCheckOnTSS = true;
+		waitForQuiescenceBegin = true;
+		waitForQuiescenceEnd = true;
+		simCheckRelocationDuration = false;
+		simConnectionFailuresDisableDuration = 0;
+		simBackupAgents = FDBBackupAgentType::NoBackupAgents;
+		simDrAgents = FDBBackupAgentType::NoBackupAgents;
+		restorePerpetualWiggleSetting = true;
+	}
+	TestSpec(StringRef title,
+	         bool dump,
+	         bool clear,
+	         double startDelay = 30.0,
+	         bool useDB = true,
+	         double databasePingDelay = -1.0)
+	  : title(title), dumpAfterTest(dump), clearAfterTest(clear), useDB(useDB), startDelay(startDelay), timeout(600),
+	    databasePingDelay(databasePingDelay), runConsistencyCheck(g_network->isSimulated()),
+	    runConsistencyCheckOnCache(false), runConsistencyCheckOnTSS(false), waitForQuiescenceBegin(true),
+	    waitForQuiescenceEnd(true), restorePerpetualWiggleSetting(true), simCheckRelocationDuration(false),
+	    simConnectionFailuresDisableDuration(0), simBackupAgents(FDBBackupAgentType::NoBackupAgents),
+	    simDrAgents(FDBBackupAgentType::NoBackupAgents) {
+		phases = TestWorkload::SETUP | TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
+		if (databasePingDelay < 0)
+			databasePingDelay = g_network->isSimulated() ? 0.0 : 15.0;
+	}
+
+	Standalone<StringRef> title;
+	bool dumpAfterTest;
+	bool clearAfterTest;
+	bool useDB;
+	bool runFailureWorkloads = true;
+	double startDelay;
+	int phases;
+	Standalone<VectorRef<VectorRef<KeyValueRef>>> options;
+	int timeout;
+	double databasePingDelay;
+	double maxDDRunTime = 0;
+	bool runConsistencyCheck;
+	bool runConsistencyCheckOnCache;
+	bool runConsistencyCheckOnTSS;
+	bool waitForQuiescenceBegin;
+	bool waitForQuiescenceEnd;
+	bool restorePerpetualWiggleSetting;
+	bool simCheckRelocationDuration;
+	double simConnectionFailuresDisableDuration;
+	FDBBackupAgentType simBackupAgents;
+	FDBBackupAgentType simDrAgents;
+
+	KnobKeyValuePairs overrideKnobs;
+	std::vector<std::string> disabledFailureInjectionWorkloads;
+};
+
+Future<DistributedTestResults> runWorkload(Database const& cx,
+                                           std::vector<TesterInterface> const& testers,
+                                           TestSpec const& spec);
+void logMetrics(std::vector<PerfMetric> metrics);
+Future<Void> databaseWarmer(Database cx);
+
+Future<Void> testExpectedError(Future<Void> test,
+                               const char* testDescr,
+                               Optional<Error> expectedError = Optional<Error>(),
+                               Optional<bool*> successFlag = Optional<bool*>(),
+                               std::map<std::string, std::string> details = {},
+                               Optional<Error> throwOnError = Optional<Error>(),
+                               UID id = UID());
+
+#endif
diff --git a/fdbserver/tester/include/fdbserver/tester/tester.h b/fdbserver/tester/include/fdbserver/tester/tester.h
new file mode 100644
index 00000000000..926fc4eaf6f
--- /dev/null
+++ b/fdbserver/tester/include/fdbserver/tester/tester.h
@@ -0,0 +1,51 @@
+/*
+ * tester.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbrpc/Locality.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "flow/UnitTest.h"
+
+Future<Void> testerServerCore(TesterInterface const& interf,
+                              Reference<IClusterConnectionRecord> const& ccr,
+                              Reference<AsyncVar<struct ServerDBInfo> const> const& serverDBInfo,
+                              LocalityData const& locality,
+                              Optional<std::string> const& expectedWorkLoad = Optional<std::string>());
+
+enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS };
+enum test_type_t {
+	TEST_TYPE_FROM_FILE,
+	TEST_TYPE_CONSISTENCY_CHECK,
+	TEST_TYPE_UNIT_TESTS,
+	TEST_TYPE_CONSISTENCY_CHECK_URGENT
+};
+
+Future<Void> runTests(Reference<IClusterConnectionRecord> const& connRecord,
+                      test_type_t const& whatToRun,
+                      test_location_t const& whereToRun,
+                      int const& minTestersExpected,
+                      std::string const& fileName = std::string(),
+                      StringRef const& startingConfiguration = StringRef(),
+                      LocalityData const& locality = LocalityData(),
+                      UnitTestParameters const& testOptions = UnitTestParameters(),
+                      bool const& restartingTest = false);
+
+Future<Void> customShardConfigWorkload(Database const& cx);
diff --git a/fdbserver/include/fdbserver/workloads/workloads.actor.h b/fdbserver/tester/include/fdbserver/tester/workloads.h
similarity index 61%
rename from fdbserver/include/fdbserver/workloads/workloads.actor.h
rename to fdbserver/tester/include/fdbserver/tester/workloads.h
index bc65948ccc9..b49585b2985 100644
--- a/fdbserver/include/fdbserver/workloads/workloads.actor.h
+++ b/fdbserver/tester/include/fdbserver/tester/workloads.h
@@ -1,5 +1,5 @@
 /*
- * workloads.actor.h
+ * workloads.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,39 +18,23 @@
  * limitations under the License.
  */
 
+#ifndef FDBSERVER_TESTER_WORKLOADS_H
+#define FDBSERVER_TESTER_WORKLOADS_H
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_WORKLOADS_ACTOR_G_H)
-#define FDBSERVER_WORKLOADS_ACTOR_G_H
-#include "fdbserver/workloads/workloads.actor.g.h"
-#elif !defined(FDBSERVER_WORKLOADS_ACTOR_H)
-#define FDBSERVER_WORKLOADS_ACTOR_H
 
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/DatabaseContext.h" // for clone()
-#include "fdbserver/KnobProtectiveGroups.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbrpc/simulator.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkloadKeys.h"
 
+#include <algorithm>
 #include <functional>
-
-#include "flow/actorcompiler.h" // has to be last import
-
-/*
- * Gets an Value from a list of key/value pairs, using a default value if the key is not present.
- */
-Value getOption(VectorRef<KeyValueRef> options, Key key, Value defaultValue);
-int getOption(VectorRef<KeyValueRef> options, Key key, int defaultValue);
-uint64_t getOption(VectorRef<KeyValueRef> options, Key key, uint64_t defaultValue);
-int64_t getOption(VectorRef<KeyValueRef> options, Key key, int64_t defaultValue);
-double getOption(VectorRef<KeyValueRef> options, Key key, double defaultValue);
-bool getOption(VectorRef<KeyValueRef> options, Key key, bool defaultValue);
-std::vector<std::string> getOption(VectorRef<KeyValueRef> options,
-                                   Key key,
-                                   std::vector<std::string> defaultValue); // comma-separated strings
-std::vector<int> getOption(VectorRef<KeyValueRef> options,
-                           Key key,
-                           std::vector<int> defaultValue = {}); // comma-separated integers
-bool hasOption(VectorRef<KeyValueRef> options, Key key);
+#include <map>
+#include <set>
+#include <string>
+#include <type_traits>
+#include <vector>
 
 struct WorkloadContext {
 	Standalone<VectorRef<KeyValueRef>> options;
@@ -61,11 +45,24 @@ struct WorkloadContext {
 	std::vector<KeyRange> rangesToCheck; // for urgent consistency checker
 
 	WorkloadContext();
-	WorkloadContext(const WorkloadContext&);
+	explicit(false) WorkloadContext(const WorkloadContext&);
 	~WorkloadContext();
 	WorkloadContext& operator=(const WorkloadContext&) = delete;
 };
 
+Value getOption(VectorRef<KeyValueRef> options, Key key, Value defaultValue);
+int getOption(VectorRef<KeyValueRef> options, Key key, int defaultValue);
+uint64_t getOption(VectorRef<KeyValueRef> options, Key key, uint64_t defaultValue);
+int64_t getOption(VectorRef<KeyValueRef> options, Key key, int64_t defaultValue);
+double getOption(VectorRef<KeyValueRef> options, Key key, double defaultValue);
+bool getOption(VectorRef<KeyValueRef> options, Key key, bool defaultValue);
+std::vector<std::string> getOption(VectorRef<KeyValueRef> options, Key key, std::vector<std::string> defaultValue);
+std::vector<int> getOption(VectorRef<KeyValueRef> options, Key key, std::vector<int> defaultValue = {});
+bool hasOption(VectorRef<KeyValueRef> options, Key key);
+Future<Void> poisson(double* last, double meanInterval);
+Future<Void> uniform(double* last, double meanInterval);
+void emplaceIndex(uint8_t* data, int offset, int64_t index);
+
 struct TestWorkload : NonCopyable, WorkloadContext, ReferenceCounted<TestWorkload> {
 	// Implementations of TestWorkload need to provide their name by defining a static member variable called name:
 	// static constexpr const char* name = "WorkloadName";
@@ -111,9 +108,10 @@ struct TestWorkloadImpl : Workload {
 	static_assert(std::is_same_v<decltype(&TestWorkload::description), decltype(&Workload::description)>,
 	              "Workload must not override TestWorkload::description");
 
-	TestWorkloadImpl(WorkloadContext const& wcx) : Workload(wcx) {}
+	explicit(false) TestWorkloadImpl(WorkloadContext const& wcx) : Workload(wcx) {}
 	template <bool E = isFailureInjectionWorkload>
-	TestWorkloadImpl(WorkloadContext const& wcx, std::enable_if_t<E, NoOptions> o) : Workload(wcx, o) {}
+	    requires(E)
+	TestWorkloadImpl(WorkloadContext const& wcx, NoOptions o) : Workload(wcx, o) {}
 
 	std::string description() const override { return Workload::NAME; }
 };
@@ -122,7 +120,7 @@ struct CompoundWorkload;
 class DeterministicRandom;
 
 struct FailureInjectionWorkload : TestWorkload {
-	FailureInjectionWorkload(WorkloadContext const&);
+	explicit(false) FailureInjectionWorkload(WorkloadContext const&);
 	virtual ~FailureInjectionWorkload() {}
 	virtual void initFailureInjectionMode(DeterministicRandom& random);
 	virtual bool shouldInject(DeterministicRandom& random, const WorkloadRequest& work, const unsigned count) const;
@@ -156,7 +154,7 @@ struct CompoundWorkload : TestWorkload {
 	std::vector<Reference<TestWorkload>> workloads;
 	std::vector<Reference<FailureInjectionWorkload>> failureInjection;
 
-	CompoundWorkload(WorkloadContext& wcx);
+	explicit(false) CompoundWorkload(WorkloadContext& wcx);
 	CompoundWorkload* add(Reference<TestWorkload>&& w);
 	void addFailureInjection(WorkloadRequest& work);
 	bool shouldInjectFailure(DeterministicRandom& random,
@@ -256,7 +254,7 @@ struct WorkloadFactory : IWorkloadFactory {
 	              "Each workload must have a Workload::NAME member");
 	using WorkloadType = TestWorkloadImpl<Workload>;
 	bool runInUntrustedClient;
-	WorkloadFactory(UntrustedMode runInUntrustedClient = UntrustedMode::False)
+	explicit(false) WorkloadFactory(UntrustedMode runInUntrustedClient = UntrustedMode::False)
 	  : runInUntrustedClient(runInUntrustedClient) {
 		auto& f = factories();
 		std::string name = WorkloadType::NAME;
@@ -274,108 +272,6 @@ struct WorkloadFactory : IWorkloadFactory {
 
 #define REGISTER_WORKLOAD(classname) WorkloadFactory<classname> classname##WorkloadFactory
 
-struct DistributedTestResults {
-	std::vector<PerfMetric> metrics;
-	int successes, failures;
-
-	DistributedTestResults() {}
-
-	DistributedTestResults(std::vector<PerfMetric> const& metrics, int successes, int failures)
-	  : metrics(metrics), successes(successes), failures(failures) {}
-
-	bool ok() const { return successes && !failures; }
-};
-
-class TestSpec {
-public:
-	TestSpec() {
-		title = StringRef();
-		dumpAfterTest = false;
-		clearAfterTest = g_network->isSimulated();
-		useDB = true;
-		startDelay = 30.0;
-		phases = TestWorkload::SETUP | TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
-		timeout = g_network->isSimulated() ? 15000 : 1500;
-		databasePingDelay = g_network->isSimulated() ? 0.0 : 15.0;
-		runConsistencyCheck = g_network->isSimulated();
-		runConsistencyCheckOnCache = false;
-		runConsistencyCheckOnTSS = true;
-		waitForQuiescenceBegin = true;
-		waitForQuiescenceEnd = true;
-		simCheckRelocationDuration = false;
-		simConnectionFailuresDisableDuration = 0;
-		simBackupAgents = ISimulator::BackupAgentType::NoBackupAgents;
-		simDrAgents = ISimulator::BackupAgentType::NoBackupAgents;
-		restorePerpetualWiggleSetting = true;
-	}
-	TestSpec(StringRef title,
-	         bool dump,
-	         bool clear,
-	         double startDelay = 30.0,
-	         bool useDB = true,
-	         double databasePingDelay = -1.0)
-	  : title(title), dumpAfterTest(dump), clearAfterTest(clear), useDB(useDB), startDelay(startDelay), timeout(600),
-	    databasePingDelay(databasePingDelay), runConsistencyCheck(g_network->isSimulated()),
-	    runConsistencyCheckOnCache(false), runConsistencyCheckOnTSS(false), waitForQuiescenceBegin(true),
-	    waitForQuiescenceEnd(true), restorePerpetualWiggleSetting(true), simCheckRelocationDuration(false),
-	    simConnectionFailuresDisableDuration(0), simBackupAgents(ISimulator::BackupAgentType::NoBackupAgents),
-	    simDrAgents(ISimulator::BackupAgentType::NoBackupAgents) {
-		phases = TestWorkload::SETUP | TestWorkload::EXECUTION | TestWorkload::CHECK | TestWorkload::METRICS;
-		if (databasePingDelay < 0)
-			databasePingDelay = g_network->isSimulated() ? 0.0 : 15.0;
-	}
-
-	Standalone<StringRef> title;
-	bool dumpAfterTest;
-	bool clearAfterTest;
-	bool useDB;
-	bool runFailureWorkloads = true;
-	double startDelay;
-	int phases;
-	Standalone<VectorRef<VectorRef<KeyValueRef>>> options;
-	int timeout;
-	double databasePingDelay;
-	double maxDDRunTime = 0; // Maximum Data Distributor run time before considered stuck (0 = use default)
-	bool runConsistencyCheck;
-	bool runConsistencyCheckOnCache;
-	bool runConsistencyCheckOnTSS;
-	bool waitForQuiescenceBegin;
-	bool waitForQuiescenceEnd;
-	bool restorePerpetualWiggleSetting; // whether set perpetual_storage_wiggle as the value after run
-	                                    // QuietDatabase. QuietDatabase always disables perpetual storage wiggle on
-	                                    // purpose. If waitForQuiescenceBegin == true and we want to keep perpetual
-	                                    // storage wiggle the same setting as before during testing, this value should
-	                                    // be set true.
-
-	bool simCheckRelocationDuration; // If set to true, then long duration relocations generate SevWarnAlways messages.
-	                                 // Once any workload sets this to true, it will be true for the duration of the
-	                                 // program.  Can only be used in simulation.
-	double simConnectionFailuresDisableDuration;
-	ISimulator::BackupAgentType simBackupAgents; // If set to true, then the simulation runs backup agents on the
-	                                             // workers. Can only be used in simulation.
-	ISimulator::BackupAgentType simDrAgents;
-
-	KnobKeyValuePairs overrideKnobs;
-	std::vector<std::string> disabledFailureInjectionWorkloads;
-};
-
-Future<DistributedTestResults> runWorkload(Database const& cx,
-                                           std::vector<TesterInterface> const& testers,
-                                           TestSpec const& spec);
-
-void logMetrics(std::vector<PerfMetric> metrics);
-
-Future<Void> poisson(double* last, double meanInterval);
-Future<Void> uniform(double* last, double meanInterval);
-
-void emplaceIndex(uint8_t* data, int offset, int64_t index);
-Key doubleToTestKey(double p);
-double testKeyToDouble(const KeyRef& p);
-Key doubleToTestKey(double p, const KeyRef& prefix);
-double testKeyToDouble(const KeyRef& p, const KeyRef& prefix);
-
-ACTOR Future<Void> databaseWarmer(Database cx);
-
 Future<Void> quietDatabase(Database const& cx,
                            Reference<AsyncVar<struct ServerDBInfo> const> const&,
                            std::string phase,
@@ -387,30 +283,4 @@ Future<Void> quietDatabase(Database const& cx,
                            int64_t maxVersionOffset = 1e6,
                            double maxDDRunTime = 0);
 
-/**
- * A utility function for testing error situations. It succeeds if the given test
- * throws an error. If expectedError is provided, it additionally checks if the
- * error code is as expected.
- *
- * In case of a failure, logs an corresponding error in the trace with the given
- * description and details, sets the given success flag to false (optional)
- * and throws the given exception (optional). Note that in case of a successful
- * test execution, the success flag is kept unchanged.
- */
-Future<Void> testExpectedError(Future<Void> test,
-                               const char* testDescr,
-                               Optional<Error> expectedError = Optional<Error>(),
-                               Optional<bool*> successFlag = Optional<bool*>(),
-                               std::map<std::string, std::string> details = {},
-                               Optional<Error> throwOnError = Optional<Error>(),
-                               UID id = UID());
-
-std::string getTestEncryptionFileName();
-
-// This should become a BehaviorInjectionWorkload or perhaps ConfigInjectionWorkload which should be a new class that
-// should represent non-failure behaviors that can be randomly injected into any test run.
-ACTOR Future<Void> customShardConfigWorkload(Database cx);
-
-#include "flow/unactorcompiler.h"
-
 #endif
diff --git a/fdbserver/tester/test.cpp b/fdbserver/tester/test.cpp
new file mode 100644
index 00000000000..a77490c49de
--- /dev/null
+++ b/fdbserver/tester/test.cpp
@@ -0,0 +1,1085 @@
+/*
+ * test.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdio>
+#include <fstream>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <fmt/ranges.h>
+
+#include "flow/CoroUtils.h"
+#include "flow/DeterministicRandom.h"
+#include "flow/Platform.h"
+#include "flow/ProcessEvents.h"
+#include "flow/Trace.h"
+#include "flow/genericactors.actor.h"
+#include "fdbrpc/sim_validation.h"
+#include "fdbclient/ClusterInterface.h"
+#include "fdbclient/DataDistributionConfig.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/MonitorLeader.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/KnobProtectiveGroups.h"
+#include "ConsistencyChecker.h"
+#include "DatabaseMaintenance.h"
+#include "TestSpecParser.h"
+#include "TesterServer.h"
+#include "fdbserver/tester/WorkloadUtils.h"
+#include "fdbserver/tester/tester.h"
+#include "fdbserver/tester/workloads.h"
+
+int passCount = 0;
+int failCount = 0;
+
+template <class T>
+void throwIfError(const std::vector<Future<ErrorOr<T>>>& futures, std::string errorMsg) {
+	for (auto& future : futures) {
+		if (future.get().isError()) {
+			TraceEvent(SevError, errorMsg.c_str()).error(future.get().getError());
+			throw future.get().getError();
+		}
+	}
+}
+
+Future<DistributedTestResults> runWorkload(Database const& cx,
+                                           std::vector<TesterInterface> const& testers,
+                                           TestSpec const& spec) {
+	// C++20 coroutine safety: copy const& params to survive across suspend points
+	Database cxCopy = cx;
+	std::vector<TesterInterface> testersCopy = testers;
+	TestSpec specCopy = spec;
+	std::string name = printable(specCopy.title);
+
+	TraceEvent("TestRunning")
+	    .detail("WorkloadTitle", specCopy.title)
+	    .detail("TesterCount", testersCopy.size())
+	    .detail("Phases", specCopy.phases)
+	    .detail("TestTimeout", specCopy.timeout);
+
+	std::vector<Future<WorkloadInterface>> workRequests;
+	std::vector<std::vector<PerfMetric>> metricsResults;
+
+	int i = 0;
+	int success = 0;
+	int failure = 0;
+	int64_t sharedRandom = deterministicRandom()->randomInt64(0, 10000000);
+	for (; i < testersCopy.size(); i++) {
+		WorkloadRequest req;
+		req.title = specCopy.title;
+		req.useDatabase = specCopy.useDB;
+		req.runFailureWorkloads = specCopy.runFailureWorkloads;
+		req.timeout = specCopy.timeout;
+		req.databasePingDelay = specCopy.useDB ? specCopy.databasePingDelay : 0.0;
+		req.options = specCopy.options;
+		req.clientId = i;
+		req.clientCount = testersCopy.size();
+		req.sharedRandomNumber = sharedRandom;
+		req.disabledFailureInjectionWorkloads = specCopy.disabledFailureInjectionWorkloads;
+		workRequests.push_back(testersCopy[i].recruitments.getReply(req));
+	}
+
+	std::vector<WorkloadInterface> workloads = co_await getAll(workRequests);
+	double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
+	if (g_network->isSimulated() && specCopy.simCheckRelocationDuration)
+		debug_setCheckRelocationDuration(true);
+
+	if (specCopy.phases & TestWorkload::SETUP) {
+		std::vector<Future<ErrorOr<Void>>> setups;
+		printf("setting up test (%s)...\n", printable(specCopy.title).c_str());
+		TraceEvent("TestSetupStart").detail("WorkloadTitle", specCopy.title);
+		setups.reserve(workloads.size());
+		for (const auto& workload : workloads)
+			setups.push_back(workload.setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
+		co_await waitForAll(setups);
+		throwIfError(setups, "SetupFailedForWorkload" + printable(specCopy.title));
+		TraceEvent("TestSetupComplete").detail("WorkloadTitle", specCopy.title);
+		TraceEvent("TestProgress").log("runWorkload: workload [%s] setup finished", name.c_str());
+	}
+
+	if (specCopy.phases & TestWorkload::EXECUTION) {
+		TraceEvent("TestStarting").detail("WorkloadTitle", specCopy.title);
+		printf("running test (%s)...\n", printable(specCopy.title).c_str());
+		std::vector<Future<ErrorOr<Void>>> starts;
+		starts.reserve(workloads.size());
+		for (const auto& workload : workloads)
+			starts.push_back(workload.start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
+		co_await waitForAll(starts);
+		throwIfError(starts, "StartFailedForWorkload" + printable(specCopy.title));
+		printf("%s complete\n", printable(specCopy.title).c_str());
+		TraceEvent("TestComplete").detail("WorkloadTitle", specCopy.title);
+		TraceEvent("TestProgress").log("runWorkload: workload [%s] start finished", name.c_str());
+	}
+
+	if (specCopy.phases & TestWorkload::CHECK) {
+		if (specCopy.useDB && (specCopy.phases & TestWorkload::EXECUTION)) {
+			co_await delay(3.0);
+		}
+
+		std::vector<Future<ErrorOr<CheckReply>>> checks;
+		TraceEvent("TestCheckingResults").detail("WorkloadTitle", specCopy.title);
+
+		printf("checking test (%s)...\n", name.c_str());
+		TraceEvent("TestProgress").log("runWorkload: calling check interface for test [%s]", name.c_str());
+
+		checks.reserve(workloads.size());
+		for (const auto& workload : workloads)
+			checks.push_back(workload.check.template getReplyUnlessFailedFor<CheckReply>(waitForFailureTime, 0));
+		co_await waitForAll(checks);
+
+		throwIfError(checks, "CheckFailedForWorkload" + printable(specCopy.title));
+
+		for (const auto& check : checks) {
+			if (check.get().get().value)
+				success++;
+			else
+				failure++;
+		}
+		TraceEvent("TestCheckComplete").detail("WorkloadTitle", specCopy.title);
+		TraceEvent("TestProgress")
+		    .log("finished check() for test [%s]; success [%d], failure [%d]", name.c_str(), success, failure);
+	}
+
+	if (specCopy.phases & TestWorkload::METRICS) {
+		std::vector<Future<ErrorOr<std::vector<PerfMetric>>>> metricTasks;
+		printf("fetching metrics (%s)...\n", printable(specCopy.title).c_str());
+		TraceEvent("TestFetchingMetrics").detail("WorkloadTitle", specCopy.title);
+		TraceEvent("TestProgress").log("runWorkload: calling metrics interface for test [%s]", name.c_str());
+		metricTasks.reserve(workloads.size());
+		for (const auto& workload : workloads)
+			metricTasks.push_back(
+			    workload.metrics.template getReplyUnlessFailedFor<std::vector<PerfMetric>>(waitForFailureTime, 0));
+		co_await waitForAll(metricTasks);
+		throwIfError(metricTasks, "MetricFailedForWorkload" + printable(specCopy.title));
+		for (const auto& metricTask : metricTasks) {
+			metricsResults.push_back(metricTask.get().get());
+		}
+	}
+
+	// Stopping the workloads is unreliable, but they have a timeout
+	// FIXME: stop if one of the above phases throws an exception
+	for (const auto& workload : workloads)
+		workload.stop.send(ReplyPromise<Void>());
+
+	co_return DistributedTestResults(aggregateMetrics(metricsResults), success, failure);
+}
+
+// Sets the database configuration by running the ChangeConfig workload
+Future<Void> changeConfiguration(Database cx, std::vector<TesterInterface> testers, StringRef configMode) {
+	TestSpec spec;
+	Standalone<VectorRef<KeyValueRef>> options;
+	spec.title = "ChangeConfig"_sr;
+	spec.runFailureWorkloads = false;
+	options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ChangeConfig"_sr));
+	options.push_back_deep(options.arena(), KeyValueRef("configMode"_sr, configMode));
+	spec.options.push_back_deep(spec.options.arena(), options);
+
+	DistributedTestResults testResults = co_await runWorkload(cx, testers, spec);
+
+	co_return;
+}
+
+Future<bool> runTest(Database cx,
+                     std::vector<TesterInterface> testers,
+                     TestSpec spec,
+                     Reference<AsyncVar<ServerDBInfo>> dbInfo,
+                     TesterConsistencyScanState* consistencyScanState) {
+	DistributedTestResults testResults;
+	double savedDisableDuration = 0;
+	bool hadException = false;
+	Error caughtError;
+	std::string name = spec.title.toString();
+
+	try {
+		Future<DistributedTestResults> fTestResults = runWorkload(cx, testers, spec);
+		if (g_network->isSimulated() && spec.simConnectionFailuresDisableDuration > 0) {
+			savedDisableDuration = g_simulator->connectionFailuresDisableDuration;
+			g_simulator->connectionFailuresDisableDuration = spec.simConnectionFailuresDisableDuration;
+		}
+		if (spec.timeout > 0) {
+			fTestResults = timeoutError(fTestResults, spec.timeout);
+		}
+		DistributedTestResults _testResults = co_await fTestResults;
+		printf("Test complete\n");
+		TraceEvent("TestProgress").log("runTest: test [%s] complete", name.c_str());
+		testResults = _testResults;
+		logMetrics(testResults.metrics);
+		if (g_network->isSimulated() && savedDisableDuration > 0) {
+			g_simulator->connectionFailuresDisableDuration = savedDisableDuration;
+		}
+	} catch (Error& e) {
+		if (e.code() == error_code_timed_out) {
+			auto msg = fmt::format("Process timed out after {} seconds", spec.timeout);
+			ProcessEvents::trigger("Timeout"_sr, StringRef(msg), e);
+			TraceEvent(SevError, "TestFailure")
+			    .error(e)
+			    .detail("Reason", "Test timed out")
+			    .detail("Timeout", spec.timeout);
+			fprintf(stderr, "ERROR: Test [%s] timed out after %d seconds.\n", name.c_str(), spec.timeout);
+			testResults.failures = testers.size();
+			testResults.successes = 0;
+		} else {
+			caughtError = e;
+			hadException = true;
+		}
+	}
+
+	if (hadException) {
+		throw caughtError;
+	}
+
+	bool ok = testResults.ok();
+
+	if (spec.useDB) {
+		printf("%d test clients passed; %d test clients failed\n", testResults.successes, testResults.failures);
+		TraceEvent("TestProgress")
+		    .log("runTest: [%d] test clients passed; [%d] test clients failed",
+		         testResults.successes,
+		         testResults.failures);
+		if (spec.dumpAfterTest) {
+			Error dumpErr;
+			try {
+				co_await timeoutError(dumpDatabase(cx, "dump after " + printable(spec.title) + ".html", allKeys), 30.0);
+			} catch (Error& e) {
+				TraceEvent(SevError, "TestFailure").error(e).detail("Reason", "Unable to dump database");
+				ok = false;
+			}
+
+			co_await delay(1.0);
+		}
+
+		TraceEvent("TestProgress").log("runTest: invoking checkConsistencyScanAfterTest()");
+		// Disable consistency scan before checkConsistency because otherwise it will prevent quiet database from
+		// quiescing
+		co_await checkConsistencyScanAfterTest(cx, consistencyScanState);
+		printf("Consistency scan done\n");
+		TraceEvent("TestProgress").log("runTest: checkConsistencyScanAfterTest returned");
+
+		// Run the consistency check workload
+		if (spec.runConsistencyCheck) {
+			bool quiescent = g_network->isSimulated() ? !isGeneralBuggifyEnabled() : spec.waitForQuiescenceEnd;
+			try {
+				if (quiescent) {
+					printf("Running urgent consistency check...\n");
+					TraceEvent("TestProgress").log("Running urgent consistency check");
+					co_await timeoutError(checkConsistencyUrgentSim(cx, testers), 20000.0);
+					printf("Urgent consistency check done\n");
+					TraceEvent("TestProgress").log("Urgent consistency check done");
+				} else {
+					TraceEvent("TestProgress").log("Skipping urgent consistency check for non-quiescent end state");
+				}
+				printf("Running consistency check...\n");
+				TraceEvent("TestProgress").log("Invoking checkConsistency()");
+				co_await timeoutError(checkConsistency(cx,
+				                                       testers,
+				                                       quiescent,
+				                                       spec.runConsistencyCheckOnTSS,
+				                                       spec.maxDDRunTime > 0 ? spec.maxDDRunTime : 10000.0,
+				                                       5000,
+				                                       spec.databasePingDelay,
+				                                       dbInfo),
+				                      20000.0);
+				printf("Consistency check done\n");
+				TraceEvent("TestProgress").log("checkConsistency() returned");
+			} catch (Error& e) {
+				TraceEvent("TestProgress").log("Exception in checkConsistencyUrgentSim or checkConsistency");
+				TraceEvent(SevError, "TestFailure").error(e).detail("Reason", "Unable to perform consistency check");
+				ok = false;
+			}
+
+			// Run auditStorage at the end of simulation
+			if (ok && quiescent && g_network->isSimulated()) {
+				try {
+					TraceEvent("AuditStorageStart");
+					TraceEvent("TestProgress")
+					    .log("runTest: calling auditStorageCorrectness(dbinfo, ValidateHA, 1500.0)");
+					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateHA), 1500.0);
+					TraceEvent("AuditStorageCorrectnessHADone");
+					TraceEvent("TestProgress")
+					    .log("runTest: calling auditStorageCorrectness(dbinfo, ValidateReplica, 1500.0)");
+					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateReplica), 1500.0);
+					TraceEvent("AuditStorageCorrectnessReplicaDone");
+					TraceEvent("TestProgress")
+					    .log("runTest: calling auditStorageCorrectness(dbinfo, ValidateLocationMetadata, 1500.0)");
+					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateLocationMetadata), 1500.0);
+
+					TraceEvent("AuditStorageCorrectnessLocationMetadataDone");
+					TraceEvent("TestProgress")
+					    .log("runTest: calling auditStorageCorrectness(dbinfo, ValidateSTorageServerShard, 1500.0)");
+					co_await timeoutError(auditStorageCorrectness(dbInfo, AuditType::ValidateStorageServerShard),
+					                      1500.0);
+					TraceEvent("AuditStorageCorrectnessStorageServerShardDone");
+					TraceEvent("TestProgress").log("runTest: storage audits completed.");
+				} catch (Error& e) {
+					ok = false;
+					TraceEvent(SevError, "TestFailure")
+					    .error(e)
+					    .detail("Reason", "Unable to perform auditStorage check.");
+				}
+			}
+		}
+	}
+
+	TraceEvent(ok ? SevInfo : SevWarnAlways, "TestResults").detail("Workload", spec.title).detail("Passed", (int)ok);
+	TraceEvent("TestProgress")
+	    .log("runTest: Workload [%s] passed: [%s]", spec.title.toString().c_str(), (ok ? "TRUE" : "FALSE"));
+
+	if (ok) {
+		passCount++;
+	} else {
+		failCount++;
+	}
+
+	printf("%d test clients passed; %d test clients failed\n", testResults.successes, testResults.failures);
+	TraceEvent("TestProgress")
+	    .log(
+	        "runTest: [%d] test clients passed; [%d] test clients failed", testResults.successes, testResults.failures);
+
+	if (spec.useDB && spec.clearAfterTest) {
+		Error clearErr;
+		try {
+			TraceEvent("TesterClearingDatabase").log();
+			TraceEvent("TestProgress").log("runTest: cleaning database via clearData(cx, 1000.0)");
+			co_await timeoutError(clearData(cx), 1000.0);
+		} catch (Error& e) {
+			TraceEvent(SevError, "ErrorClearingDatabaseAfterTest").error(e);
+			throw; // If we didn't do this, we don't want any later tests to run on this DB
+		}
+
+		co_await delay(1.0);
+	}
+
+	co_return ok;
+}
+
+Future<Void> monitorServerDBInfo(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> ccInterface,
+                                 LocalityData locality,
+                                 Reference<AsyncVar<ServerDBInfo>> dbInfo) {
+	// Initially most of the serverDBInfo is not known, but we know our locality right away
+	ServerDBInfo localInfo;
+	localInfo.myLocality = locality;
+	dbInfo->set(localInfo);
+
+	while (true) {
+		GetServerDBInfoRequest req;
+		req.knownServerInfoID = dbInfo->get().id;
+
+		Future<ServerDBInfo> getInfoReq =
+		    ccInterface->get().present() ? brokenPromiseToNever(ccInterface->get().get().getServerDBInfo.getReply(req))
+		                                 : Future<ServerDBInfo>(Never());
+		Future<Void> ccChange = ccInterface->onChange();
+
+		int action = 0;
+		ServerDBInfo gotInfo;
+
+		co_await Choose()
+		    .When(getInfoReq,
+		          [&](ServerDBInfo const& info) {
+			          gotInfo = info;
+			          action = 1;
+		          })
+		    .When(ccChange, [&](Void const&) { action = 2; })
+		    .run();
+
+		if (action == 1) {
+			TraceEvent("GotServerDBInfoChange")
+			    .detail("ChangeID", gotInfo.id)
+			    .detail("MasterID", gotInfo.master.id())
+			    .detail("RatekeeperID", gotInfo.ratekeeper.present() ? gotInfo.ratekeeper.get().id() : UID())
+			    .detail("DataDistributorID", gotInfo.distributor.present() ? gotInfo.distributor.get().id() : UID());
+
+			gotInfo.myLocality = locality;
+			dbInfo->set(gotInfo);
+		} else if (action == 2) {
+			if (ccInterface->get().present())
+				TraceEvent("GotCCInterfaceChange")
+				    .detail("CCID", ccInterface->get().get().id())
+				    .detail("CCMachine", ccInterface->get().get().getWorkers.getEndpoint().getPrimaryAddress());
+		}
+	}
+}
+
+Future<Void> initializeSimConfig(Database db, bool restartingTest) {
+	Transaction tr(db);
+	ASSERT(g_network->isSimulated());
+	while (true) {
+		bool needsErrorHandling = false;
+		Error caughtError;
+
+		try {
+			DatabaseConfiguration dbConfig = co_await getDatabaseConfiguration(&tr);
+			updateFDBSimulationPolicy(dbConfig, restartingTest);
+
+			// If the same region is being shared between the remote and a satellite, then our simulated policy checking
+			// may fail to account for the total number of needed machines when deciding what can be killed. To work
+			// around this, we increase the required transaction logs in the remote policy to include the number of
+			// satellite logs that may get recruited there
+			bool foundSharedDcId = false;
+			std::set<Key> dcIds;
+			int maxSatelliteReplication = 0;
+			for (auto const& r : dbConfig.regions) {
+				if (!dcIds.insert(r.dcId).second) {
+					foundSharedDcId = true;
+				}
+				if (!r.satellites.empty() && r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogUsableDcs > 0) {
+					for (auto const& s : r.satellites) {
+						if (!dcIds.insert(s.dcId).second) {
+							foundSharedDcId = true;
+						}
+					}
+
+					maxSatelliteReplication =
+					    std::max(maxSatelliteReplication, r.satelliteTLogReplicationFactor / r.satelliteTLogUsableDcs);
+				}
+			}
+
+			if (foundSharedDcId) {
+				int totalRequired = std::max(dbConfig.tLogReplicationFactor, dbConfig.remoteTLogReplicationFactor) +
+				                    maxSatelliteReplication;
+				setFDBSimulationPolicyRemoteTLogPolicy(
+				    makeReference<PolicyAcross>(totalRequired, "zoneid", makeReference<PolicyOne>()));
+				TraceEvent("ChangingSimTLogPolicyForSharedRemote")
+				    .detail("TotalRequired", totalRequired)
+				    .detail("MaxSatelliteReplication", maxSatelliteReplication)
+				    .detail("ActualPolicy", dbConfig.getRemoteTLogPolicy()->info())
+				    .detail("SimulatorPolicy", fdbSimulationPolicyState().remoteTLogPolicy->info());
+			} else {
+				setFDBSimulationPolicyRemoteTLogPolicy(dbConfig.getRemoteTLogPolicy());
+			}
+
+			co_return;
+		} catch (Error& e) {
+			caughtError = e;
+			needsErrorHandling = true;
+		}
+
+		if (needsErrorHandling) {
+			co_await tr.onError(caughtError);
+		}
+	}
+}
+
+// Disables connection failures after the given time seconds
+Future<Void> disableConnectionFailuresAfter(double seconds, std::string context) {
+	if (g_network->isSimulated()) {
+		TraceEvent(SevWarnAlways, ("ScheduleDisableConnectionFailures_" + context).c_str())
+		    .detail("At", now() + seconds);
+		co_await delay(seconds);
+		while (true) {
+			double delaySeconds = disableConnectionFailures(context, ForceDisable::False);
+			if (delaySeconds > DISABLE_CONNECTION_FAILURE_MIN_INTERVAL) {
+				co_await delay(delaySeconds);
+			} else {
+				// disableConnectionFailures will always take effect if less than
+				// DISABLE_CONNECTION_FAILURE_MIN_INTERVAL is returned.
+				break;
+			}
+		}
+	}
+	co_return;
+}
+
+/**
+ * Test orchestrator: sends test specification to testers in the right order and collects the results.
+ *
+ * There are multiple actors in this file with similar names (runTest, runTests) and slightly different signatures.
+ *
+ * This is the actual orchestrator. It reads the test specifications (from tests), prepares the cluster (by running the
+ * configure command given in startingConfiguration) and then runs the workload.
+ *
+ * NOTE: this assumes that referenced objects will outlive all suspension points encounted by this function.
+ *
+ * Rturns a future which will be set after all tests finished.
+ */
+Future<Void> runTests7(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> cc,
+                       Reference<AsyncVar<Optional<struct ClusterInterface>>> ci,
+                       std::vector<TesterInterface> testers,
+                       std::vector<TestSpec> tests,
+                       StringRef startingConfiguration,
+                       LocalityData locality,
+                       bool restartingTest) {
+	Database cx;
+	Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
+	Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
+
+	bool useDB = false;
+	bool waitForQuiescenceBegin = false;
+	bool waitForQuiescenceEnd = false;
+	bool restorePerpetualWiggleSetting = false;
+	bool perpetualWiggleEnabled = false;
+	bool backupWorkerEnabled = false;
+	double startDelay = 0.0;
+	double databasePingDelay = 1e9;
+	FDBBackupAgentType simBackupAgents = FDBBackupAgentType::NoBackupAgents;
+	FDBBackupAgentType simDrAgents = FDBBackupAgentType::NoBackupAgents;
+	bool enableDD = false;
+	TesterConsistencyScanState consistencyScanState;
+
+	// Gives chance for g_network->run() to run inside event loop and hence let
+	// the tests see correct value for `isOnMainThread()`.
+	// FIXME: this type of knowledge of random implementation details elsewhere is obviously
+	// undesirable and getting rid of hacks like this to simplify things, reduce cognitive load,
+	// etc would be nice.
+	co_await yield();
+
+	if (tests.empty()) {
+		useDB = true;
+	}
+	for (auto iter = tests.begin(); iter != tests.end(); ++iter) {
+		if (iter->useDB) {
+			useDB = true;
+		}
+		if (iter->waitForQuiescenceBegin) {
+			waitForQuiescenceBegin = true;
+		}
+		if (iter->waitForQuiescenceEnd) {
+			waitForQuiescenceEnd = true;
+		}
+		if (iter->restorePerpetualWiggleSetting) {
+			restorePerpetualWiggleSetting = true;
+		}
+		startDelay = std::max(startDelay, iter->startDelay);
+		databasePingDelay = std::min(databasePingDelay, iter->databasePingDelay);
+		if (iter->simBackupAgents != FDBBackupAgentType::NoBackupAgents) {
+			simBackupAgents = iter->simBackupAgents;
+		}
+
+		if (iter->simDrAgents != FDBBackupAgentType::NoBackupAgents) {
+			simDrAgents = iter->simDrAgents;
+		}
+		enableDD = enableDD || getOption(iter->options[0], "enableDD"_sr, false);
+	}
+
+	if (g_network->isSimulated()) {
+		fdbSimulationPolicyState().backupAgents = simBackupAgents;
+		fdbSimulationPolicyState().drAgents = simDrAgents;
+	}
+
+	// turn off the database ping functionality if the suite of tests are not going to be using the database
+	if (!useDB) {
+		databasePingDelay = 0.0;
+	}
+
+	if (useDB) {
+		cx = openDBOnServer(dbInfo);
+	}
+
+	consistencyScanState.enabled = g_network->isSimulated() && deterministicRandom()->coinflip();
+	consistencyScanState.waitForComplete =
+	    consistencyScanState.enabled && waitForQuiescenceEnd && deterministicRandom()->coinflip();
+	consistencyScanState.enableAfter = consistencyScanState.waitForComplete && deterministicRandom()->random01() < 0.1;
+
+	disableConnectionFailures("Tester");
+
+	// Change the configuration (and/or create the database) if necessary
+	printf("startingConfiguration:%s start\n", startingConfiguration.toString().c_str());
+	fmt::print("useDB: {}\n", useDB);
+
+	TraceEvent("TestProgress")
+	    .log("runTests7: startingConfiguration: [%s]; useDB: [%s]",
+	         startingConfiguration.toString().c_str(),
+	         (useDB ? "TRUE" : "FALSE"));
+
+	// If this is important it should be in the trace file also.  Who wants to read two places
+	// when we could be reading just one?
+	printSimulatedTopology();
+	if (useDB && !startingConfiguration.empty()) {
+		Error configErr;
+		try {
+			co_await timeoutError(changeConfiguration(cx, testers, startingConfiguration), 2000.0);
+			if (g_network->isSimulated() && enableDD) {
+				co_await setDDMode(cx, 1);
+			}
+		} catch (Error& e) {
+			TraceEvent(SevError, "TestFailure").error(e).detail("Reason", "Unable to set starting configuration");
+		}
+		std::string_view confView(reinterpret_cast<const char*>(startingConfiguration.begin()),
+		                          startingConfiguration.size());
+		if (restorePerpetualWiggleSetting) {
+			const std::string setting = "perpetual_storage_wiggle:=";
+			auto pos = confView.find(setting);
+			if (pos != confView.npos && confView.at(pos + setting.size()) == '1') {
+				perpetualWiggleEnabled = true;
+			}
+		}
+		const std::string bwSetting = "backup_worker_enabled:=";
+		auto pos = confView.find(bwSetting);
+		if (pos != confView.npos && confView.at(pos + bwSetting.size()) == '1') {
+			backupWorkerEnabled = true;
+		}
+	}
+
+	// Read cluster configuration
+	if (useDB && g_network->isSimulated()) {
+		DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
+		updateFDBSimulationPolicy(configuration, restartingTest);
+		auto const& simPolicy = fdbSimulationPolicyState();
+		if (configuration.regions.size() == 2) {
+			ASSERT((!configuration.regions[0].satelliteTLogPolicy && !configuration.regions[1].satelliteTLogPolicy) ||
+			       configuration.regions[0].satelliteTLogPolicy->info() ==
+			           configuration.regions[1].satelliteTLogPolicy->info());
+		}
+
+		ASSERT(simPolicy.storagePolicy && simPolicy.tLogPolicy);
+		ASSERT(!simPolicy.hasSatelliteReplication || simPolicy.satelliteTLogPolicy);
+
+		// Randomly inject custom shard configuration
+		// TODO:  Move this to a workload representing non-failure behaviors which can be randomly added to any test
+		// run.
+		if (deterministicRandom()->random01() < 0.25) {
+			co_await customShardConfigWorkload(cx);
+		}
+	}
+
+	if (useDB) {
+		if (g_network->isSimulated()) {
+			co_await initializeSimConfig(cx, restartingTest);
+		}
+	}
+
+	bool runPretestChecks = (useDB && waitForQuiescenceBegin);
+	TraceEvent("TestProgress").log("runTests7: doing PreTestChecks? [%s]", (runPretestChecks ? "TRUE" : "FALSE"));
+
+	if (runPretestChecks) {
+		TraceEvent("TesterStartingPreTestChecks")
+		    .detail("DatabasePingDelay", databasePingDelay)
+		    .detail("StartDelay", startDelay);
+
+		Error quietDbErr;
+		try {
+			co_await (quietDatabase(cx, dbInfo, "Start") ||
+			          (databasePingDelay == 0.0
+			               ? Never()
+			               : testDatabaseLiveness(cx, databasePingDelay, "QuietDatabaseStart", startDelay)));
+		} catch (Error& e) {
+			TraceEvent("QuietDatabaseStartExternalError").error(e);
+			throw;
+		}
+
+		if (perpetualWiggleEnabled) { // restore the enabled perpetual storage wiggle setting
+			printf("Set perpetual_storage_wiggle=1 ...\n");
+			Version cVer = co_await setPerpetualStorageWiggle(cx, true, LockAware::True);
+			(void)cVer;
+			printf("Set perpetual_storage_wiggle=1 Done.\n");
+		}
+
+		if (backupWorkerEnabled) {
+			printf("Enabling backup worker ...\n");
+			co_await enableBackupWorker(cx);
+			printf("Enabled backup worker.\n");
+		}
+
+		// TODO: Move this to a BehaviorInjection workload once that concept exists.
+		if (consistencyScanState.enabled && !consistencyScanState.enableAfter) {
+			printf("Enabling consistency scan ...\n");
+			co_await enableConsistencyScanInSim(cx);
+			printf("Enabled consistency scan.\n");
+		}
+	}
+
+	// Use the first test's connectionFailuresDisableDuration if set
+	double connectionFailuresDisableDuration = 0.0;
+	if (!tests.empty() && tests[0].simConnectionFailuresDisableDuration > 0) {
+		connectionFailuresDisableDuration = tests[0].simConnectionFailuresDisableDuration;
+	}
+
+	// Must be at function scope so the background actor isn't cancelled when the else block exits.
+	// In the original ACTOR code, `state` hoists variables to actor scope regardless of block scope.
+	Future<Void> disabler;
+	if (connectionFailuresDisableDuration > 0) {
+		// Disable connection failures with specified duration
+		disableConnectionFailures("Tester", ForceDisable::True, connectionFailuresDisableDuration);
+	} else {
+		enableConnectionFailures("Tester", FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS);
+		disabler = disableConnectionFailuresAfter(FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, "Tester");
+	}
+	Future<Void> repairDataCenter;
+	if (useDB) {
+		// Keep datacenter repair at the default duration regardless of connection failures setting
+		Future<Void> reconfigure = reconfigureAfter(cx, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, dbInfo, "Tester");
+		repairDataCenter = reconfigure;
+	}
+
+	TraceEvent("TestProgress").log("runTests7: going to run [%d] tests", tests.size());
+	// Also emit legacy log:
+	TraceEvent("TestsExpectedToPass").detail("Count", tests.size());
+
+	int idx = 0;
+	std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup;
+	for (; idx < tests.size(); idx++) {
+		printf("Run test:%s start\n", tests[idx].title.toString().c_str());
+		TraceEvent("TestProgress").log("runTests7: starting test [%s]", tests[idx].title.toString().c_str());
+		knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(tests[idx].overrideKnobs);
+		co_await runTest(cx, testers, tests[idx], dbInfo, &consistencyScanState);
+		knobProtectiveGroup.reset(nullptr);
+		printf("Run test:%s Done.\n", tests[idx].title.toString().c_str());
+		TraceEvent("TestProgress").log("runTests7: done running test [%s]", tests[idx].title.toString().c_str());
+	}
+
+	printf("\n%d tests passed; %d tests failed.\n", passCount, failCount);
+	TraceEvent("TestProgress").log("runTests7: [%d] tests passed; [%d] tests failed.", passCount, failCount);
+
+	bool ranConsistencyScan = false;
+	if (useDB) {
+		if (waitForQuiescenceEnd) {
+			TraceEvent("TestProgress")
+			    .log("runTests7: useDB && waitForQuiescenceEnd ==> invoking checkConsistencyScanAfterTest()");
+			printf("Waiting for DD to end...\n");
+			TraceEvent("QuietDatabaseEndStart");
+			try {
+				TraceEvent("QuietDatabaseEndWait");
+				ranConsistencyScan = true;
+				Future<Void> waitConsistencyScanEnd = checkConsistencyScanAfterTest(cx, &consistencyScanState);
+				Future<Void> waitQuietDatabaseEnd =
+				    quietDatabase(cx, dbInfo, "End", 0, 2e6, 2e6) ||
+				    (databasePingDelay == 0.0 ? Never()
+				                              : testDatabaseLiveness(cx, databasePingDelay, "QuietDatabaseEnd"));
+
+				co_await (waitConsistencyScanEnd && waitQuietDatabaseEnd);
+			} catch (Error& e) {
+				TraceEvent("QuietDatabaseEndExternalError").error(e);
+				throw;
+			}
+		}
+	}
+	if (ranConsistencyScan) {
+		TraceEvent("TestProgress").log("runTests7: finished running consistency scan");
+	} else {
+		TraceEvent("TestProgress").log("runTests7: didn't run consistency scan");
+	}
+
+	printf("\n");
+
+	co_return;
+}
+
+/**
+ * \brief Proxy function that waits until enough testers are available and then calls into the orchestrator.
+ *
+ * There are multiple actors in this file with similar names (runTest, runTests) and slightly different signatures.
+ *
+ * This actor wraps the actual orchestrator (also called runTests). But before calling that actor, it waits for enough
+ * testers to come up.
+ *
+ * \param cc The cluster controller interface
+ * \param ci Same as cc.clientInterface
+ * \param tests The test specifications to run
+ * \param minTestersExpected The number of testers to expect. This actor will block until it can find this many testers.
+ * \param startingConfiguration If non-empty, the orchestrator will attempt to set this configuration before starting
+ * the tests.
+ * \param locality client locality (it seems this is unused?)
+ *
+ * \returns A future which will be set after all tests finished.
+ */
+Future<Void> runTests8(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> cc,
+                       Reference<AsyncVar<Optional<struct ClusterInterface>>> ci,
+                       std::vector<TestSpec> tests,
+                       test_location_t at,
+                       int minTestersExpected,
+                       StringRef startingConfiguration,
+                       LocalityData locality,
+                       bool restartingTest) {
+	int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) |
+	            GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
+	Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
+	std::vector<WorkerDetails> workers;
+
+	while (true) {
+		Future<std::vector<WorkerDetails>> getWorkersReq =
+		    cc->get().present() ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
+		                        : Future<std::vector<WorkerDetails>>(Never());
+		Future<Void> ccChange = cc->onChange();
+
+		int action = 0;
+		std::vector<WorkerDetails> gotWorkers;
+
+		co_await Choose()
+		    .When(getWorkersReq,
+		          [&](std::vector<WorkerDetails> const& w) {
+			          gotWorkers = w;
+			          action = 1;
+		          })
+		    .When(ccChange, [&](Void const&) { action = 2; })
+		    .When(testerTimeout, [&](Void const&) { action = 3; })
+		    .run();
+
+		if (action == 1) {
+			if (static_cast<int>(gotWorkers.size()) >= minTestersExpected) {
+				workers = gotWorkers;
+				break;
+			}
+			co_await delay(SERVER_KNOBS->WORKER_POLL_DELAY);
+		} else if (action == 3) {
+			TraceEvent(SevError, "TesterRecruitmentTimeout").log();
+			throw timed_out();
+		}
+		// action == 2: cc changed, loop again
+	}
+
+	std::vector<TesterInterface> ts;
+	ts.reserve(workers.size());
+	for (const auto& worker : workers)
+		ts.push_back(worker.interf.testerInterface);
+
+	co_await runTests7(cc, ci, ts, tests, startingConfiguration, locality, restartingTest);
+	co_return;
+}
+
+/**
+ * \brief Set up testing environment and run the given tests on a cluster.
+ *
+ * There are multiple actors in this file with similar names (runTest, runTests) and slightly different signatures.
+ *
+ * This actor is usually the first entry point into the test environment. It itself doesn't implement too much
+ * functionality. Its main purpose is to generate the test specification from passed arguments and then call into the
+ * correct actor which will orchestrate the actual test.
+ *
+ * \param connRecord A cluster connection record. Not all tests require a functional cluster but all tests require
+ * a cluster record.
+ * \param whatToRun TEST_TYPE_FROM_FILE to read the test description from a passed toml file or
+ * TEST_TYPE_CONSISTENCY_CHECK to generate a test spec for consistency checking
+ * \param at TEST_HERE: this process will act as a test client and execute the given workload. TEST_ON_SERVERS: Run a
+ * test client on every worker in the cluster. TEST_ON_TESTERS: Run a test client on all servers with class Test
+ * \param minTestersExpected In at is not TEST_HERE, this will instruct the orchestrator until it can find at least
+ * minTestersExpected test-clients. This is usually passed through from a command line argument. In simulation, the
+ * simulator will pass the number of testers that it started.
+ * \param fileName The path to the toml-file containing the test description. Is ignored if whatToRun !=
+ * TEST_TYPE_FROM_FILE
+ * \param startingConfiguration Can be used to configure a cluster before running the test. If this is an empty string,
+ * it will be ignored, otherwise it will be passed to changeConfiguration.
+ * \param locality The client locality to be used. This is only used if at == TEST_HERE
+ *
+ * \returns A future which will be set after all tests finished.
+ */
+Future<Void> runTests(Reference<IClusterConnectionRecord> const& connRecordUnsafe,
+                      test_type_t const& whatToRunUnsafe,
+                      test_location_t const& atUnsafe,
+                      int const& minTestersExpectedUnsafe,
+                      std::string const& fileNameUnsafe,
+                      StringRef const& startingConfigurationUnsafe,
+                      LocalityData const& localityUnsafe,
+                      UnitTestParameters const& testOptionsUnsafe,
+                      bool const& restartingTestUnsafe) {
+	// C++20 coroutine safety: copy parameters that might bind to temporaries (default args).
+	// const& parameters only store the reference in the coroutine frame; temporaries are
+	// destroyed after the first suspend point, leaving dangling references.
+	// Just do this for all parameters, including ones that could be passed by value.
+	Reference<IClusterConnectionRecord> connRecord = connRecordUnsafe;
+	test_type_t whatToRun = whatToRunUnsafe;
+	test_location_t at = atUnsafe;
+	int minTestersExpected = minTestersExpectedUnsafe;
+	std::string fileName = fileNameUnsafe;
+	StringRef startingConfiguration = startingConfigurationUnsafe;
+	LocalityData locality = localityUnsafe;
+	UnitTestParameters testOptions = testOptionsUnsafe;
+	bool restartingTest = restartingTestUnsafe;
+
+	TestSet testSet;
+	std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup(nullptr);
+	auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
+	auto ci = makeReference<AsyncVar<Optional<ClusterInterface>>>();
+	std::vector<Future<Void>> actors;
+	if (connRecord) {
+		actors.push_back(reportErrors(monitorLeader(connRecord, cc), "MonitorLeader"));
+		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
+	}
+
+	TraceEvent("TestProgress").log("runTests: invoked with whatToRun = [%d]", whatToRun);
+
+	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
+		// Need not to set spec here. Will set spec when triggering workload
+	} else if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK) {
+		TestSpec spec;
+		Standalone<VectorRef<KeyValueRef>> options;
+		spec.title = "ConsistencyCheck"_sr;
+		spec.runFailureWorkloads = false;
+		spec.databasePingDelay = 0;
+		spec.timeout = 0;
+		spec.waitForQuiescenceBegin = false;
+		spec.waitForQuiescenceEnd = false;
+		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
+		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "ConsistencyCheck"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("performQuiescentChecks"_sr, "false"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("distributed"_sr, "false"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("failureIsError"_sr, "true"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("indefinite"_sr, "true"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("rateLimitMax"_sr, StringRef(rateLimitMax)));
+		options.push_back_deep(options.arena(), KeyValueRef("shuffleShards"_sr, "true"_sr));
+		spec.options.push_back_deep(spec.options.arena(), options);
+		testSet.testSpecs.push_back(spec);
+	} else if (whatToRun == TEST_TYPE_UNIT_TESTS) {
+		TestSpec spec;
+		Standalone<VectorRef<KeyValueRef>> options;
+		spec.title = "UnitTests"_sr;
+		spec.startDelay = 0;
+		spec.useDB = false;
+		spec.timeout = 0;
+		options.push_back_deep(options.arena(), KeyValueRef("testName"_sr, "UnitTests"_sr));
+		options.push_back_deep(options.arena(), KeyValueRef("testsMatching"_sr, fileName));
+		// Add unit test options as test spec options
+		for (auto& [key, value] : testOptions.params) {
+			options.push_back_deep(options.arena(), KeyValueRef(key, value));
+		}
+		spec.options.push_back_deep(spec.options.arena(), options);
+		testSet.testSpecs.push_back(spec);
+	} else {
+		ASSERT(whatToRun == TEST_TYPE_FROM_FILE);
+		std::ifstream ifs;
+		ifs.open(fileName.c_str(), std::ifstream::in);
+		if (!ifs.good()) {
+			TraceEvent(SevError, "TestHarnessFail")
+			    .detail("Reason", "file open failed")
+			    .detail("File", fileName.c_str());
+			fprintf(stderr, "ERROR: Could not open file `%s'\n", fileName.c_str());
+			co_return;
+		}
+		enableClientInfoLogging(); // Enable Client Info logging by default for tester
+		if (fileName.ends_with(".txt")) {
+			testSet.testSpecs = readTests(ifs);
+		} else if (fileName.ends_with(".toml")) {
+			// TOML is weird about opening the file as binary on windows, so we
+			// just let TOML re-open the file instead of using ifs.
+			testSet = readTOMLTests(fileName);
+		} else {
+			TraceEvent(SevError, "TestHarnessFail")
+			    .detail("Reason", "unknown tests specification extension")
+			    .detail("File", fileName.c_str());
+			co_return;
+		}
+		ifs.close();
+	}
+
+	knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(testSet.overrideKnobs);
+	Future<Void> tests;
+	// These must be at function scope so they aren't destroyed when the if-block exits.
+	// In the original ACTOR code, `state` hoists variables to actor scope regardless of block scope.
+	Database urgentCx;
+	Reference<AsyncVar<ServerDBInfo>> urgentDbInfo;
+	Future<Void> urgentCcMonitor;
+	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
+		urgentDbInfo = makeReference<AsyncVar<ServerDBInfo>>();
+		urgentCcMonitor = monitorServerDBInfo(cc, LocalityData(), urgentDbInfo); // FIXME: locality
+		urgentCx = openDBOnServer(urgentDbInfo);
+		tests = reportErrors(
+		    runConsistencyCheckerUrgentHolder(
+		        cc, urgentCx, Optional<std::vector<TesterInterface>>(), minTestersExpected, /*repeatRun=*/true),
+		    "runConsistencyCheckerUrgentHolder");
+	} else if (at == TEST_HERE) {
+		auto db = makeReference<AsyncVar<ServerDBInfo>>();
+		std::vector<TesterInterface> iTesters(1);
+		actors.push_back(
+		    reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
+		actors.push_back(reportErrors(testerServerCore(iTesters[0], connRecord, db, locality), "TesterServerCore"));
+		tests = runTests7(cc, ci, iTesters, testSet.testSpecs, startingConfiguration, locality, restartingTest);
+	} else {
+		tests = reportErrors(
+		    runTests8(
+		        cc, ci, testSet.testSpecs, at, minTestersExpected, startingConfiguration, locality, restartingTest),
+		    "RunTests");
+	}
+
+	TraceEvent("TestProgress").log("runTests: waiting for actors to finish.");
+
+	Future<Void> actorsFuture = actors.empty() ? Never() : waitForAll(actors);
+	co_await Choose()
+	    .When(tests, [](Void const&) {})
+	    .When(actorsFuture,
+	          [](Void const&) {
+		          ASSERT(false);
+		          throw internal_error();
+	          })
+	    .run();
+}
+
+namespace {
+Future<Void> testExpectedErrorImpl(Future<Void> test,
+                                   const char* testDescr,
+                                   Optional<Error> expectedError,
+                                   Optional<bool*> successFlag,
+                                   std::map<std::string, std::string> details,
+                                   Optional<Error> throwOnError,
+                                   UID id) {
+	Error actualError;
+	bool hadError = false;
+
+	try {
+		co_await test;
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			throw e;
+		}
+		actualError = e;
+		hadError = true;
+		// The test failed as expected
+		if (!expectedError.present() || actualError.code() == expectedError.get().code()) {
+			co_return;
+		}
+	}
+
+	// The test has failed
+	if (successFlag.present()) {
+		*(successFlag.get()) = false;
+	}
+	TraceEvent evt(SevError, "TestErrorFailed", id);
+	evt.detail("TestDescription", testDescr);
+	if (expectedError.present()) {
+		evt.detail("ExpectedError", expectedError.get().name());
+		evt.detail("ExpectedErrorCode", expectedError.get().code());
+	}
+	if (hadError && actualError.isValid()) {
+		evt.detail("ActualError", actualError.name());
+		evt.detail("ActualErrorCode", actualError.code());
+	} else {
+		evt.detail("Reason", "Unexpected success");
+	}
+
+	// Make sure that no duplicate details were provided
+	ASSERT(!details.contains("TestDescription"));
+	ASSERT(!details.contains("ExpectedError"));
+	ASSERT(!details.contains("ExpectedErrorCode"));
+	ASSERT(!details.contains("ActualError"));
+	ASSERT(!details.contains("ActualErrorCode"));
+	ASSERT(!details.contains("Reason"));
+
+	for (auto& [key, value] : details) {
+		evt.detail(key.c_str(), value);
+	}
+	if (throwOnError.present()) {
+		throw throwOnError.get();
+	}
+	co_return;
+}
+} // namespace
+
+Future<Void> testExpectedError(Future<Void> test,
+                               const char* testDescr,
+                               Optional<Error> expectedError,
+                               Optional<bool*> successFlag,
+                               std::map<std::string, std::string> details,
+                               Optional<Error> throwOnError,
+                               UID id) {
+	return testExpectedErrorImpl(test, testDescr, expectedError, successFlag, details, throwOnError, id);
+}
diff --git a/fdbserver/tlog/CMakeLists.txt b/fdbserver/tlog/CMakeLists.txt
new file mode 100644
index 00000000000..99330293f7c
--- /dev/null
+++ b/fdbserver/tlog/CMakeLists.txt
@@ -0,0 +1,15 @@
+fdb_find_sources(FDBSERVER_TLOG_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_tlog SRCS ${FDBSERVER_TLOG_SRCS})
+add_fdbserver_link_test(fdbserver_tloglinktest
+  fdbserver_tlog
+  fdbserver_logsystem
+  fdbserver_core)
+
+configure_fdbserver_common_includes(fdbserver_tlog)
+target_include_directories(fdbserver_tlog
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_tlog PRIVATE fdbserver_core fdbserver_kvstore fdbserver_logsystem)
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/tlog/TLogServer.cpp
similarity index 77%
rename from fdbserver/TLogServer.actor.cpp
rename to fdbserver/tlog/TLogServer.cpp
index 218073f90ca..8a23ef2c6ba 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/tlog/TLogServer.cpp
@@ -1,5 +1,5 @@
 /*
- * TLogServer.actor.cpp
+ * TLogServer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,31 +23,41 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/SpanContextMessage.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/MutationTracking.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/IDiskQueue.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MutationTracking.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/SpanContextMessage.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/tlog/TLogServer.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
 #include "fdbrpc/FailureMonitor.h"
-#include "fdbserver/IDiskQueue.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/Stats.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/FDBExecHelper.actor.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
 #include "flow/Histogram.h"
 #include "flow/DebugTrace.h"
 #include "flow/genericactors.actor.h"
 #include "flow/network.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
+
+namespace {
+
+FDB_BOOLEAN_PARAM(NothingPersistent);
+FDB_BOOLEAN_PARAM(PoppedRecently);
+FDB_BOOLEAN_PARAM(UnpoppedRecovered);
+
+} // namespace
 
 struct TLogQueueEntryRef {
 	UID id;
@@ -96,7 +106,7 @@ struct AlternativeTLogQueueEntryRef {
 	}
 };
 
-typedef Standalone<TLogQueueEntryRef> TLogQueueEntry;
+using TLogQueueEntry = Standalone<TLogQueueEntryRef>;
 struct LogData;
 struct TLogData;
 
@@ -149,13 +159,13 @@ struct TLogQueue final : public IClosable {
 	                        IDiskQueue::location start,
 	                        IDiskQueue::location end);
 
-	ACTOR static Future<TLogQueueEntry> readNext(TLogQueue* self, TLogData* tLog) {
-		state TLogQueueEntry result;
-		state int zeroFillSize = 0;
+	static Future<TLogQueueEntry> readNext(TLogQueue* self, TLogData* tLog) {
+		TLogQueueEntry result;
+		int zeroFillSize = 0;
 
-		loop {
-			state IDiskQueue::location startloc = self->queue->getNextReadLocation();
-			Standalone<StringRef> h = wait(self->queue->readNext(sizeof(uint32_t)));
+		while (true) {
+			IDiskQueue::location startloc = self->queue->getNextReadLocation();
+			Standalone<StringRef> h = co_await self->queue->readNext(sizeof(uint32_t));
 			if (h.size() != sizeof(uint32_t)) {
 				if (h.size()) {
 					CODE_PROBE(true, "Zero fill within size field", probe::decoration::rare);
@@ -167,10 +177,10 @@ struct TLogQueue final : public IClosable {
 				break;
 			}
 
-			state uint32_t payloadSize = *(uint32_t*)h.begin();
+			uint32_t payloadSize = *(uint32_t*)h.begin();
 			ASSERT(payloadSize < (100 << 20));
 
-			Standalone<StringRef> e = wait(self->queue->readNext(payloadSize + 1));
+			Standalone<StringRef> e = co_await self->queue->readNext(payloadSize + 1);
 			if (e.size() != payloadSize + 1) {
 				CODE_PROBE(true, "Zero fill within payload", probe::decoration::rare);
 				zeroFillSize = payloadSize + 1 - e.size();
@@ -184,7 +194,7 @@ struct TLogQueue final : public IClosable {
 				ar >> result;
 				const IDiskQueue::location endloc = self->queue->getNextReadLocation();
 				self->updateVersionSizes(result, tLog, startloc, endloc);
-				return result;
+				co_return result;
 			}
 		}
 		if (zeroFillSize) {
@@ -349,6 +359,7 @@ struct TLogData : NonCopyable {
 	// End of fields used by snapshot based backup and restore
 
 	Reference<AsyncVar<bool>> degraded;
+	Reference<AsyncVar<bool>> lowDiskTLogExclusion;
 	std::vector<TagsAndMessage> tempTagMessages;
 
 	// Distribution of end-to-end server latency of tlog commit requests.
@@ -377,6 +388,7 @@ struct TLogData : NonCopyable {
 	         IDiskQueue* persistentQueue,
 	         Reference<AsyncVar<ServerDBInfo> const> dbInfo,
 	         Reference<AsyncVar<bool>> degraded,
+	         Reference<AsyncVar<bool>> lowDiskTLogExclusion,
 	         std::string folder,
 	         Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck)
 	  : dbgid(dbgid), workerID(workerID), persistentData(persistentData), rawPersistentQueue(persistentQueue),
@@ -386,13 +398,26 @@ struct TLogData : NonCopyable {
 	    targetVolatileBytes(SERVER_KNOBS->TLOG_SPILL_THRESHOLD), overheadBytesInput(0), overheadBytesDurable(0),
 	    peekMemoryLimiter(SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES),
 	    concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopDeadline(0), dataFolder(folder),
-	    degraded(degraded),
+	    degraded(degraded), lowDiskTLogExclusion(lowDiskTLogExclusion),
 	    commitLatencyDist(Histogram::getHistogram("tLog"_sr, "commit"_sr, Histogram::Unit::milliseconds)),
 	    queueWaitLatencyDist(Histogram::getHistogram("tLog"_sr, "QueueWait"_sr, Histogram::Unit::milliseconds)),
 	    timeUntilDurableDist(Histogram::getHistogram("tLog"_sr, "TimeUntilDurable"_sr, Histogram::Unit::milliseconds)),
 	    enablePrimaryTxnSystemHealthCheck(enablePrimaryTxnSystemHealthCheck) {
 		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
 	}
+
+	double availableSpaceRatio(StorageBytes const& kvStoreBytes, StorageBytes const& queueBytes) const {
+		auto ratio = [](StorageBytes const& storageBytes) -> double {
+			return storageBytes.total > 0 ? double(storageBytes.available) / storageBytes.total : 1.0;
+		};
+		return std::min(ratio(kvStoreBytes), ratio(queueBytes));
+	}
+
+	bool shouldAcceptNewData(StorageBytes const& kvStoreBytes,
+	                         StorageBytes const& queueBytes,
+	                         double minAvailableSpaceRatio) const {
+		return minAvailableSpaceRatio <= 0.0 || availableSpaceRatio(kvStoreBytes, queueBytes) >= minAvailableSpaceRatio;
+	}
 };
 
 struct LogData : NonCopyable, public ReferenceCounted<LogData> {
@@ -411,14 +436,14 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		TagData(Tag tag,
 		        Version popped,
 		        IDiskQueue::location poppedLocation,
-		        bool nothingPersistent,
-		        bool poppedRecently,
-		        bool unpoppedRecovered)
+		        NothingPersistent nothingPersistent,
+		        PoppedRecently poppedRecently,
+		        UnpoppedRecovered unpoppedRecovered)
 		  : nothingPersistent(nothingPersistent), poppedRecently(poppedRecently), popped(popped), persistentPopped(0),
 		    versionForPoppedLocation(0), poppedLocation(poppedLocation), unpoppedRecovered(unpoppedRecovered),
 		    tag(tag) {}
 
-		TagData(TagData&& r) noexcept
+		explicit(false) TagData(TagData&& r) noexcept
 		  : versionMessages(std::move(r.versionMessages)), nothingPersistent(r.nothingPersistent),
 		    poppedRecently(r.poppedRecently), popped(r.popped), persistentPopped(r.persistentPopped),
 		    versionForPoppedLocation(r.versionForPoppedLocation), poppedLocation(r.poppedLocation),
@@ -436,11 +461,11 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		}
 
 		// Erase messages not needed to update *from* versions >= before (thus, messages with toversion <= before)
-		ACTOR Future<Void> eraseMessagesBefore(TagData* self,
-		                                       Version before,
-		                                       TLogData* tlogData,
-		                                       Reference<LogData> logData,
-		                                       TaskPriority taskID) {
+		Future<Void> eraseMessagesBefore(TagData* self,
+		                                 Version before,
+		                                 TLogData* tlogData,
+		                                 Reference<LogData> logData,
+		                                 TaskPriority taskID) {
 			while (!self->versionMessages.empty() && self->versionMessages.front().first < before) {
 				Version version = self->versionMessages.front().first;
 				std::pair<int, int>& sizes = logData->version_sizes[version];
@@ -463,10 +488,8 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 				logData->bytesDurable += bytesErased;
 				tlogData->bytesDurable += bytesErased;
 				tlogData->overheadBytesDurable += bytesErased;
-				wait(yield(taskID));
+				co_await yield(taskID);
 			}
-
-			return Void();
 		}
 
 		Future<Void> eraseMessagesBefore(Version before,
@@ -518,11 +541,13 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 	Version minKnownCommittedVersion;
 	Version queuePoppedVersion; // The disk queue has been popped up until the location which represents this version.
 	Version minPoppedTagVersion;
-	Tag minPoppedTag; // The tag that makes tLog hold its data and cause tLog's disk queue increasing.
+	Tag minPoppedTag; // The tag (locality >= 0) that makes tLog hold its data and cause tLog's disk queue increasing.
+	Version minSysPopTagVersion;
+	Tag minSysPopTag; // The system locality tag (locality < 0) with the minimum popped version.
 
 	// For each version above knownCommittedVersion, track:
 	// <Version, PrevVersion (that the sequencer provided), TLogs that the version has been sent to (the tLogs
-	//  are represented by their corresponding positions in "TagPartitionedLogSystem::tLogs")>
+	//  are represented by their corresponding positions in "LogSystem::tLogs")>
 	std::deque<UnknownCommittedVersions> unknownCommittedVersions;
 
 	Deque<std::pair<Version, Standalone<VectorRef<uint8_t>>>> messageBlocks;
@@ -547,9 +572,9 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 	// only callable after getTagData returns a null reference
 	Reference<TagData> createTagData(Tag tag,
 	                                 Version popped,
-	                                 bool nothingPersistent,
-	                                 bool poppedRecently,
-	                                 bool unpoppedRecovered) {
+	                                 NothingPersistent nothingPersistent,
+	                                 PoppedRecently poppedRecently,
+	                                 UnpoppedRecovered unpoppedRecovered) {
 		if (tag.locality != tagLocalityLogRouter && tag.locality != tagLocalityTxs && tag != txsTag && allTags.size() &&
 		    !allTags.contains(tag) && popped <= recoveredAt) {
 			popped = recoveredAt + 1;
@@ -629,7 +654,7 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 
 	std::map<UID, PeekTrackerData> peekTracker;
 
-	Reference<AsyncVar<Reference<ILogSystem>>> logSystem;
+	Reference<AsyncVar<Reference<LogSystem>>> logSystem;
 	Tag remoteTag;
 	bool isPrimary;
 	int logRouterTags;
@@ -660,13 +685,14 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 	                 std::string context)
 	  : initialized(false), queueCommittingVersion(0), knownCommittedVersion(0), durableKnownCommittedVersion(0),
 	    minKnownCommittedVersion(0), queuePoppedVersion(0), minPoppedTagVersion(0), minPoppedTag(invalidTag),
-	    unpoppedRecoveredTagCount(0), cc("TLog", interf.id().toString()), bytesInput("BytesInput", cc),
-	    tagMessageCount("tagMessageCount", cc), bytesDurable("BytesDurable", cc), blockingPeeks("BlockingPeeks", cc),
+	    minSysPopTagVersion(0), minSysPopTag(invalidTag), unpoppedRecoveredTagCount(0),
+	    cc("TLog", interf.id().toString()), bytesInput("BytesInput", cc), tagMessageCount("tagMessageCount", cc),
+	    bytesDurable("BytesDurable", cc), blockingPeeks("BlockingPeeks", cc),
 	    blockingPeekTimeouts("BlockingPeekTimeouts", cc), emptyPeeks("EmptyPeeks", cc),
 	    nonEmptyPeeks("NonEmptyPeeks", cc), persistentDataUpdateBatches("PersistentDataUpdateBatches", cc),
 	    dirtyTagsProcessed("DirtyTagsProcessed", cc), logId(interf.id()), protocolVersion(protocolVersion),
 	    newPersistentDataVersion(invalidVersion), tLogData(tLogData), unrecoveredBefore(1), recoveredAt(1),
-	    recoveryTxnVersion(1), logSystem(new AsyncVar<Reference<ILogSystem>>()), remoteTag(remoteTag),
+	    recoveryTxnVersion(1), logSystem(new AsyncVar<Reference<LogSystem>>()), remoteTag(remoteTag),
 	    isPrimary(isPrimary), logRouterTags(logRouterTags), logRouterPoppedVersion(0), logRouterPopToVersion(0),
 	    locality(tagLocalityInvalid), recruitmentID(recruitmentID), logSpillType(logSpillType),
 	    allTags(tags.begin(), tags.end()), terminated(tLogData->terminated.getFuture()), execOpCommitInProgress(false),
@@ -695,6 +721,9 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 		// for why the TLog thinks it can't throw away data.
 		specialCounter(cc, "MinPoppedTagLocality", [this]() { return this->minPoppedTag.locality; });
 		specialCounter(cc, "MinPoppedTagId", [this]() { return this->minPoppedTag.id; });
+		specialCounter(cc, "MinSysPopTagVersion", [this]() { return this->minSysPopTagVersion; });
+		specialCounter(cc, "MinSysPopTagLocality", [this]() { return this->minSysPopTag.locality; });
+		specialCounter(cc, "MinSysPopTagId", [this]() { return this->minSysPopTag.id; });
 		specialCounter(cc, "SharedBytesInput", [tLogData]() { return tLogData->bytesInput; });
 		specialCounter(cc, "SharedBytesDurable", [tLogData]() { return tLogData->bytesDurable; });
 		specialCounter(cc, "SharedOverheadBytesInput", [tLogData]() { return tLogData->overheadBytesInput; });
@@ -832,8 +861,8 @@ void TLogQueue::updateVersionSizes(const TLogQueueEntry& result,
 	}
 }
 
-ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
-	state Version stopVersion = logData->version.get();
+Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
+	Version stopVersion = logData->version.get();
 
 	CODE_PROBE(true, "TLog stopped by recovering cluster-controller");
 	CODE_PROBE(logData->stopped(), "logData already stopped");
@@ -851,7 +880,7 @@ ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply,
 	}
 
 	// Lock once the current version has been committed
-	wait(logData->queueCommittedVersion.whenAtLeast(stopVersion));
+	co_await logData->queueCommittedVersion.whenAtLeast(stopVersion);
 
 	ASSERT(stopVersion == logData->version.get());
 
@@ -870,7 +899,6 @@ ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply,
 	    .detail("KnownCommitted", result.knownCommittedVersion);
 
 	reply.send(result);
-	return Void();
 }
 
 void updatePersistentPopped(TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data) {
@@ -899,14 +927,14 @@ void updatePersistentPopped(TLogData* self, Reference<LogData> logData, Referenc
 	}
 }
 
-ACTOR Future<Void> updatePoppedLocation(TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data) {
+Future<Void> updatePoppedLocation(TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data) {
 	// For anything spilled by value, we do not need to track its popped location.
 	if (logData->shouldSpillByValue(data->tag)) {
-		return Void();
+		co_return;
 	}
 
 	if (data->versionForPoppedLocation >= data->persistentPopped) {
-		return Void();
+		co_return;
 	}
 	data->versionForPoppedLocation = data->persistentPopped;
 
@@ -914,10 +942,10 @@ ACTOR Future<Void> updatePoppedLocation(TLogData* self, Reference<LogData> logDa
 	// us to remove data that still is pointed to by SpilledData in the btree.
 	if (data->persistentPopped <= logData->persistentDataVersion) {
 		// Recover the next needed location in the Disk Queue from the index.
-		RangeResult kvrefs = wait(self->persistentData->readRange(
+		RangeResult kvrefs = co_await self->persistentData->readRange(
 		    KeyRangeRef(persistTagMessageRefsKey(logData->logId, data->tag, data->persistentPopped),
 		                persistTagMessageRefsKey(logData->logId, data->tag, logData->persistentDataVersion + 1)),
-		    1));
+		    1);
 
 		if (kvrefs.empty()) {
 			// Nothing was persistent after all.
@@ -949,16 +977,14 @@ ACTOR Future<Void> updatePoppedLocation(TLogData* self, Reference<LogData> logDa
 			// to avoid trying to track what the ending location of this TLog instance was.
 		}
 	}
-
-	return Void();
 }
 
 // It runs against the oldest TLog instance, calculates the first location in the disk queue that contains un-popped
 // data, and then issues a pop to the disk queue at that location so that anything earlier can be
 // removed/forgotten/overwritten. In effect, it applies the effect of TLogPop RPCs to disk.
-ACTOR Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
+Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
 	if (!logData->initialized) {
-		return Void();
+		co_return;
 	}
 
 	std::vector<Future<Void>> updates;
@@ -970,7 +996,7 @@ ACTOR Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
 			}
 		}
 	}
-	wait(waitForAll(updates));
+	co_await waitForAll(updates);
 
 	IDiskQueue::location minLocation = 0;
 	Version minVersion = 0;
@@ -980,6 +1006,7 @@ ACTOR Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
 		minVersion = locationIter->key;
 	}
 	logData->minPoppedTagVersion = std::numeric_limits<Version>::max();
+	logData->minSysPopTagVersion = std::numeric_limits<Version>::max();
 
 	for (int tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
 		for (int tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
@@ -989,10 +1016,18 @@ ACTOR Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
 					minLocation = std::min(minLocation, tagData->poppedLocation);
 					minVersion = std::min(minVersion, tagData->popped);
 				}
-				if ((!tagData->nothingPersistent || tagData->versionMessages.size()) &&
-				    tagData->popped < logData->minPoppedTagVersion) {
-					logData->minPoppedTagVersion = tagData->popped;
-					logData->minPoppedTag = tagData->tag;
+				if ((!tagData->nothingPersistent || tagData->versionMessages.size())) {
+					if (tagData->tag.locality >= 0) {
+						if (tagData->popped < logData->minPoppedTagVersion) {
+							logData->minPoppedTagVersion = tagData->popped;
+							logData->minPoppedTag = tagData->tag;
+						}
+					} else {
+						if (tagData->popped < logData->minSysPopTagVersion) {
+							logData->minSysPopTagVersion = tagData->popped;
+							logData->minSysPopTag = tagData->tag;
+						}
+					}
 				}
 			}
 		}
@@ -1008,12 +1043,10 @@ ACTOR Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
 		self->persistentQueue->pop(std::min(minLocation, lastCommittedLocation));
 		logData->queuePoppedVersion = std::max(logData->queuePoppedVersion, minVersion);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logData, Version newPersistentDataVersion) {
-	state BinaryWriter wr(Unversioned());
+Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logData, Version newPersistentDataVersion) {
+	BinaryWriter wr(Unversioned());
 	// PERSIST: Changes self->persistentDataVersion and writes and commits the relevant changes
 	ASSERT(newPersistentDataVersion <= logData->version.get());
 	ASSERT(newPersistentDataVersion <= logData->queueCommittedVersion.get());
@@ -1023,34 +1056,34 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 
 	//TraceEvent("UpdatePersistentData", self->dbgid).detail("Seq", newPersistentDataSeq);
 
-	state bool anyData = false;
+	bool anyData = false;
 
 	// Track total batch runs (including empty batches with no work)
 	logData->persistentDataUpdateBatches += 1;
 
 	// Track successfully processed tags (incremented at end after all work completes)
-	state int64_t tagsProcessedThisBatch = 0;
+	int64_t tagsProcessedThisBatch = 0;
 
 	// For all existing tags
-	state int tagLocality = 0;
-	state int tagId = 0;
+	int tagLocality = 0;
+	int tagId = 0;
 
 	for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
 		for (tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
-			state Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
+			Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
 			if (tagData) {
 				// poppedRecently implies updatePersistentPopped() will touch persistentData
-				state bool tagDidWork = tagData->poppedRecently;
-				wait(tagData->eraseMessagesBefore(tagData->popped, self, logData, TaskPriority::UpdateStorage));
-				state Version currentVersion = 0;
+				bool tagDidWork = tagData->poppedRecently;
+				co_await tagData->eraseMessagesBefore(tagData->popped, self, logData, TaskPriority::UpdateStorage);
+				Version currentVersion = 0;
 				// Clear recently popped versions from persistentData if necessary
 				updatePersistentPopped(self, logData, tagData);
-				state Version lastVersion = std::numeric_limits<Version>::min();
-				state IDiskQueue::location firstLocation = std::numeric_limits<IDiskQueue::location>::max();
+				Version lastVersion = std::numeric_limits<Version>::min();
+				IDiskQueue::location firstLocation = std::numeric_limits<IDiskQueue::location>::max();
 				// Transfer unpopped messages with version numbers less than newPersistentDataVersion to persistentData
-				state std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator msg =
+				std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator msg =
 				    tagData->versionMessages.begin();
-				state int refSpilledTagCount = 0;
+				int refSpilledTagCount = 0;
 				wr = BinaryWriter(AssumeVersion(logData->protocolVersion));
 				// We prefix our spilled locations with a count, so that we can read this back out as a VectorRef.
 				wr << uint32_t(0);
@@ -1101,7 +1134,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 
 						Future<Void> f = yield(TaskPriority::UpdateStorage);
 						if (!f.isReady()) {
-							wait(f);
+							co_await f;
 							msg = std::upper_bound(
 							    tagData->versionMessages.begin(),
 							    tagData->versionMessages.end(),
@@ -1118,7 +1151,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 					tagData->poppedLocation = std::min(tagData->poppedLocation, firstLocation);
 				}
 
-				wait(yield(TaskPriority::UpdateStorage));
+				co_await yield(TaskPriority::UpdateStorage);
 
 				if (tagDidWork) {
 					tagsProcessedThisBatch++;
@@ -1147,15 +1180,15 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	// In some rare simulation tests, particularly with log_spill:=1 configured, the TLOG_MAX_CREATE_DURATION limit is
 	// exceeded, causing SevError trace events and simulation test failure. Increasing the value is a workaround to
 	// avoid these failures.
-	state double tLogMaxCreateDuration = SERVER_KNOBS->TLOG_MAX_CREATE_DURATION;
+	double tLogMaxCreateDuration = SERVER_KNOBS->TLOG_MAX_CREATE_DURATION;
 	if (g_network->isSimulated() && logData->logSpillType == TLogSpillType::VALUE) {
 		tLogMaxCreateDuration = SERVER_KNOBS->TLOG_MAX_CREATE_DURATION * 2;
 	}
 	// SOMEDAY: This seems to be running pretty often, should we slow it down???
 	// This needs a timeout since nothing prevents I/O operations from hanging indefinitely.
-	wait(ioTimeoutError(self->persistentData->commit(), tLogMaxCreateDuration, "TLogCommit"));
+	co_await ioTimeoutError(self->persistentData->commit(), tLogMaxCreateDuration, "TLogCommit");
 
-	wait(delay(0, TaskPriority::UpdateStorage));
+	co_await delay(0, TaskPriority::UpdateStorage);
 
 	// Now that the changes we made to persistentData are durable, erase the data we moved from memory and the queue,
 	// increase bytesDurable accordingly, and update persistentDataDurableVersion.
@@ -1165,9 +1198,9 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
 		for (tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
 			if (logData->tag_data[tagLocality][tagId]) {
-				wait(logData->tag_data[tagLocality][tagId]->eraseMessagesBefore(
-				    newPersistentDataVersion + 1, self, logData, TaskPriority::UpdateStorage));
-				wait(yield(TaskPriority::UpdateStorage));
+				co_await logData->tag_data[tagLocality][tagId]->eraseMessagesBefore(
+				    newPersistentDataVersion + 1, self, logData, TaskPriority::UpdateStorage);
+				co_await yield(TaskPriority::UpdateStorage);
 			}
 		}
 	}
@@ -1175,7 +1208,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 	logData->version_sizes.erase(logData->version_sizes.begin(),
 	                             logData->version_sizes.lower_bound(logData->persistentDataDurableVersion));
 
-	wait(yield(TaskPriority::UpdateStorage));
+	co_await yield(TaskPriority::UpdateStorage);
 
 	while (!logData->messageBlocks.empty() && logData->messageBlocks.front().first <= newPersistentDataVersion) {
 		int64_t bytesErased =
@@ -1183,7 +1216,7 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 		logData->bytesDurable += bytesErased;
 		self->bytesDurable += bytesErased;
 		logData->messageBlocks.pop_front();
-		wait(yield(TaskPriority::UpdateStorage));
+		co_await yield(TaskPriority::UpdateStorage);
 	}
 
 	if (logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) {
@@ -1220,12 +1253,10 @@ ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logDa
 		}
 	}
 	logData->newPersistentDataVersion = invalidVersion;
-
-	return Void();
 }
 
-ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Reference<LogData> logData) {
-	state Version upTo = to;
+Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Reference<LogData> logData) {
+	Version upTo = to;
 	int8_t tagLocality = inputTag.locality;
 	if (isPseudoLocality(tagLocality)) {
 		if (logData->logSystem->get().isValid()) {
@@ -1240,19 +1271,20 @@ ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen
 				TraceEvent(SevWarn, "TLogPopNoPseudoLocality", self->dbgid)
 				    .detail("Locality", tagLocality)
 				    .detail("Version", upTo);
-				return Void();
+				co_return;
 			}
 		} else {
 			TraceEvent(SevWarn, "TLogPopNoLogSystem", self->dbgid)
 			    .detail("Locality", tagLocality)
 			    .detail("Version", upTo);
-			return Void();
+			co_return;
 		}
 	}
-	state Tag tag(tagLocality, inputTag.id);
+	Tag tag(tagLocality, inputTag.id);
 	auto tagData = logData->getTagData(tag);
 	if (!tagData) {
-		tagData = logData->createTagData(tag, upTo, true, true, false);
+		tagData =
+		    logData->createTagData(tag, upTo, NothingPersistent::True, PoppedRecently::True, UnpoppedRecovered::False);
 	} else if (upTo > tagData->popped) {
 		tagData->popped = upTo;
 		tagData->poppedRecently = true;
@@ -1311,20 +1343,19 @@ ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen
 			    .detail("NothingPersistent", tagData->nothingPersistent ? "True" : "False");
 		}
 		if (upTo > logData->persistentDataDurableVersion)
-			wait(tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop));
+			co_await tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop);
 		//TraceEvent("TLogPop", logData->logId).detail("Tag", tag.toString()).detail("To", upTo);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> processPopRequests(TLogData* self, Reference<LogData> logData) {
-	state std::vector<Future<Void>> ignoredPops;
-	state std::map<Tag, Version>::const_iterator it;
-	state int ignoredPopsPlayed = 0;
-	state std::map<Tag, Version> toBePopped;
+Future<Void> processPopRequests(TLogData* self, Reference<LogData> logData) {
+	std::vector<Future<Void>> ignoredPops;
+	std::map<Tag, Version>::const_iterator it;
+	int ignoredPopsPlayed = 0;
+	std::map<Tag, Version> toBePopped;
 
 	while (now() < self->ignorePopDeadline) {
-		wait(delayUntil(self->ignorePopDeadline + 0.0001));
+		co_await delayUntil(self->ignorePopDeadline + 0.0001);
 	}
 
 	toBePopped = std::move(logData->toBePopped);
@@ -1336,15 +1367,14 @@ ACTOR Future<Void> processPopRequests(TLogData* self, Reference<LogData> logData
 		ignoredPops.push_back(tLogPopCore(self, tag, version, logData));
 		if (++ignoredPopsPlayed % SERVER_KNOBS->TLOG_POP_BATCH_SIZE == 0) {
 			CODE_PROBE(true, "Yielding while processing pop requests", probe::decoration::rare);
-			wait(yield());
+			co_await yield();
 		}
 	}
-	wait(waitForAll(ignoredPops));
+	co_await waitForAll(ignoredPops);
 	TraceEvent("ResetIgnorePopRequest", logData->logId).detail("IgnorePopDeadline", self->ignorePopDeadline);
-	return Void();
 }
 
-ACTOR Future<Void> tLogPop(TLogData* self, TLogPopRequest req, Reference<LogData> logData) {
+Future<Void> tLogPop(TLogData* self, TLogPopRequest req, Reference<LogData> logData) {
 	if (now() < self->ignorePopDeadline) {
 		TraceEvent(SevDebug, "IgnoringPopRequest", logData->logId).detail("IgnorePopDeadline", self->ignorePopDeadline);
 
@@ -1360,10 +1390,14 @@ ACTOR Future<Void> tLogPop(TLogData* self, TLogPopRequest req, Reference<LogData
 		    .detail("Tag", req.tag.toString())
 		    .detail("Version", req.to);
 	} else {
-		wait(tLogPopCore(self, req.tag, req.to, logData));
+		co_await tLogPopCore(self, req.tag, req.to, logData);
 	}
 	req.reply.send(Void());
-	return Void();
+}
+
+double getTLogStorageUpdateDelayDuration() {
+	return BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
+	               : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL;
 }
 
 // This function (and updatePersistentData, which is called by this function) run at a low priority and can soak up all
@@ -1371,23 +1405,21 @@ ACTOR Future<Void> tLogPop(TLogData* self, TLogPopRequest req, Reference<LogData
 // latencies for more important work (e.g. commits).
 // This actor is just a loop that calls updatePersistentData and popDiskQueue whenever
 // (a) there's data to be spilled or (b) we should update metadata after some commits have been fully popped.
-ACTOR Future<Void> updateStorage(TLogData* self) {
+Future<Void> updateStorage(TLogData* self) {
 	while (self->spillOrder.size() && !self->id_data.contains(self->spillOrder.front())) {
 		self->spillOrder.pop_front();
 	}
 
 	if (!self->spillOrder.size()) {
-		wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
-		                   : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
-		           TaskPriority::UpdateStorage));
-		return Void();
+		co_await delay(getTLogStorageUpdateDelayDuration(), TaskPriority::UpdateStorage);
+		co_return;
 	}
 
-	state Reference<LogData> logData = self->id_data[self->spillOrder.front()];
-	state Version nextVersion = 0;
-	state int totalSize = 0;
+	Reference<LogData> logData = self->id_data[self->spillOrder.front()];
+	Version nextVersion = 0;
+	int totalSize = 0;
 
-	state FlowLock::Releaser commitLockReleaser;
+	FlowLock::Releaser commitLockReleaser;
 
 	if (logData->stopped()) {
 		if (self->bytesInput - self->bytesDurable >= self->targetVolatileBytes) {
@@ -1402,26 +1434,24 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 					nextVersion = sizeItr == logData->version_sizes.end() ? logData->version.get() : sizeItr->key;
 				}
 
-				wait(logData->queueCommittedVersion.whenAtLeast(nextVersion));
+				co_await logData->queueCommittedVersion.whenAtLeast(nextVersion);
 				if (logData->queueCommittedVersion.get() == std::numeric_limits<Version>::max()) {
-					return Void();
+					co_return;
 				}
-				wait(delay(0, TaskPriority::UpdateStorage));
+				co_await delay(0, TaskPriority::UpdateStorage);
 
 				//TraceEvent("TLogUpdatePersist", self->dbgid).detail("LogId", logData->logId).detail("NextVersion", nextVersion).detail("Version", logData->version.get()).detail("PersistentDataDurableVer", logData->persistentDataDurableVersion).detail("QueueCommitVer", logData->queueCommittedVersion.get()).detail("PersistDataVer", logData->persistentDataVersion);
 				if (nextVersion > logData->persistentDataVersion) {
-					wait(self->persistentDataCommitLock.take());
+					co_await self->persistentDataCommitLock.take();
 					commitLockReleaser = FlowLock::Releaser(self->persistentDataCommitLock);
-					wait(updatePersistentData(self, logData, nextVersion));
+					co_await updatePersistentData(self, logData, nextVersion);
 					// Concurrently with this loop, the last stopped TLog could have been removed.
 					if (self->popOrder.size()) {
-						wait(popDiskQueue(self, self->id_data[self->popOrder.front()]));
+						co_await popDiskQueue(self, self->id_data[self->popOrder.front()]);
 					}
 					commitLockReleaser.release();
 				} else {
-					wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
-					                   : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
-					           TaskPriority::UpdateStorage));
+					co_await delay(getTLogStorageUpdateDelayDuration(), TaskPriority::UpdateStorage);
 				}
 
 				if (logData->removed.isReady()) {
@@ -1432,11 +1462,9 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 			if (logData->persistentDataDurableVersion == logData->version.get()) {
 				self->spillOrder.pop_front();
 			}
-			wait(delay(0.0, TaskPriority::UpdateStorage));
+			co_await delay(0.0, TaskPriority::UpdateStorage);
 		} else {
-			wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
-			                   : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
-			           TaskPriority::UpdateStorage));
+			co_await delay(getTLogStorageUpdateDelayDuration(), TaskPriority::UpdateStorage);
 		}
 	} else if (logData->initialized) {
 		ASSERT(self->spillOrder.size() == 1);
@@ -1459,44 +1487,39 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 
 		//TraceEvent("UpdateStorageVer", logData->logId).detail("NextVersion", nextVersion).detail("PersistentDataVersion", logData->persistentDataVersion).detail("TotalSize", totalSize);
 
-		wait(logData->queueCommittedVersion.whenAtLeast(nextVersion));
+		co_await logData->queueCommittedVersion.whenAtLeast(nextVersion);
 		if (logData->queueCommittedVersion.get() == std::numeric_limits<Version>::max()) {
-			return Void();
+			co_return;
 		}
-		wait(delay(0, TaskPriority::UpdateStorage));
+		co_await delay(0, TaskPriority::UpdateStorage);
 
 		if (nextVersion > logData->persistentDataVersion) {
-			wait(self->persistentDataCommitLock.take());
+			co_await self->persistentDataCommitLock.take();
 			commitLockReleaser = FlowLock::Releaser(self->persistentDataCommitLock);
-			wait(updatePersistentData(self, logData, nextVersion));
+			co_await updatePersistentData(self, logData, nextVersion);
 			if (self->popOrder.size()) {
-				wait(popDiskQueue(self, self->id_data[self->popOrder.front()]));
+				co_await popDiskQueue(self, self->id_data[self->popOrder.front()]);
 			}
 			commitLockReleaser.release();
 		}
 
 		if (totalSize < SERVER_KNOBS->REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT) {
-			wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
-			                   : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
-			           TaskPriority::UpdateStorage));
+			co_await delay(getTLogStorageUpdateDelayDuration(), TaskPriority::UpdateStorage);
 		} else {
 			// recovery wants to commit to persistent data when updatePersistentData is not active, this delay ensures
 			// that immediately after updatePersist returns another one has not been started yet.
-			wait(delay(0.0, TaskPriority::UpdateStorage));
+			co_await delay(0.0, TaskPriority::UpdateStorage);
 		}
 	} else {
-		wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
-		                   : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
-		           TaskPriority::UpdateStorage));
+		co_await delay(getTLogStorageUpdateDelayDuration(), TaskPriority::UpdateStorage);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> updateStorageLoop(TLogData* self) {
-	wait(delay(0, TaskPriority::UpdateStorage));
+Future<Void> updateStorageLoop(TLogData* self) {
+	co_await delay(0, TaskPriority::UpdateStorage);
 
-	loop {
-		wait(updateStorage(self));
+	while (true) {
+		co_await updateStorage(self);
 	}
 }
 
@@ -1570,7 +1593,8 @@ void commitMessages(TLogData* self,
 			}
 			Reference<LogData::TagData> tagData = logData->getTagData(tag);
 			if (!tagData) {
-				tagData = logData->createTagData(tag, 0, true, true, false);
+				tagData = logData->createTagData(
+				    tag, 0, NothingPersistent::True, PoppedRecently::True, UnpoppedRecovered::False);
 			}
 
 			if (version >= tagData->popped) {
@@ -1655,22 +1679,19 @@ std::deque<std::pair<Version, LengthPrefixedStringRef>>& getVersionMessages(Refe
 	return tagData->versionMessages;
 };
 
-ACTOR Future<Void> waitForMessagesForTag(Reference<LogData> self, Tag reqTag, Version reqBegin, double timeout) {
+Future<Void> waitForMessagesForTag(Reference<LogData> self, Tag reqTag, Version reqBegin, double timeoutDelay) {
 	self->blockingPeeks += 1;
 	auto tagData = self->getTagData(reqTag);
 	if (tagData.isValid() && !tagData->versionMessages.empty() && tagData->versionMessages.back().first >= reqBegin) {
-		return Void();
+		co_return;
 	}
-	choose {
-		when(wait(self->waitingTags[reqTag].getFuture())) {
-			// we want the caller to finish first, otherwise the data structure it is building might not be complete
-			wait(delay(0.0));
-		}
-		when(wait(delay(timeout))) {
-			self->blockingPeekTimeouts += 1;
-		}
+	Optional<Void> ready = co_await timeout(self->waitingTags[reqTag].getFuture(), timeoutDelay);
+	if (ready.present()) {
+		// We want the caller to finish first, otherwise the data structure it is building might not be complete.
+		co_await delay(0.0);
+	} else {
+		self->blockingPeekTimeouts += 1;
 	}
-	return Void();
 }
 
 void peekMessagesFromMemory(Reference<LogData> self,
@@ -1728,10 +1749,10 @@ void peekMessagesFromMemory(Reference<LogData> self,
 	}
 }
 
-ACTOR Future<std::vector<StringRef>> parseMessagesForTag(StringRef commitBlob, Tag tag, int logRouters) {
+Future<std::vector<StringRef>> parseMessagesForTag(StringRef commitBlob, Tag tag, int logRouters) {
 	// See the comment in LogSystem.cpp for the binary format of commitBlob.
-	state std::vector<StringRef> relevantMessages;
-	state BinaryReader rd(commitBlob, AssumeVersion(g_network->protocolVersion()));
+	std::vector<StringRef> relevantMessages;
+	BinaryReader rd(commitBlob, AssumeVersion(g_network->protocolVersion()));
 	while (!rd.empty()) {
 		TagsAndMessage tagsAndMessage;
 		tagsAndMessage.loadFromArena(&rd, nullptr);
@@ -1745,13 +1766,13 @@ ACTOR Future<std::vector<StringRef>> parseMessagesForTag(StringRef commitBlob, T
 				break;
 			}
 		}
-		wait(yield());
+		co_await yield();
 	}
-	return relevantMessages;
+	co_return relevantMessages;
 }
 
 // Common logics to peek TLog and create TLogPeekReply that serves both streaming peek or normal peek request
-ACTOR template <typename PromiseType>
+template <typename PromiseType>
 Future<Void> tLogPeekMessages(PromiseType replyPromise,
                               TLogData* self,
                               Reference<LogData> logData,
@@ -1762,11 +1783,11 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
                               Optional<std::pair<UID, int>> reqSequence = Optional<std::pair<UID, int>>(),
                               Optional<Version> reqEnd = Optional<Version>(),
                               Optional<bool> reqReturnEmptyIfStopped = Optional<bool>()) {
-	state BinaryWriter messages(Unversioned());
-	state BinaryWriter messages2(Unversioned());
-	state int sequence = -1;
-	state UID peekId;
-	state double queueStart = now();
+	BinaryWriter messages(Unversioned());
+	BinaryWriter messages2(Unversioned());
+	int sequence = -1;
+	UID peekId;
+	double queueStart = now();
 
 	if (reqTag.locality == tagLocalityTxs && reqTag.id >= logData->txsTags && logData->txsTags > 0) {
 		reqTag.id = reqTag.id % logData->txsTags;
@@ -1817,21 +1838,21 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 				trackerData.idleTime += t;
 			}
 			trackerData.lastUpdate = now();
-			std::pair<Version, bool> prevPeekData = wait(fPrevPeekData);
+			std::pair<Version, bool> prevPeekData = co_await fPrevPeekData;
 			reqBegin = std::max(prevPeekData.first, reqBegin);
 			reqOnlySpilled = prevPeekData.second;
-			wait(yield());
+			co_await yield();
 		} catch (Error& e) {
 			if (e.code() == error_code_timed_out || e.code() == error_code_operation_obsolete) {
 				replyPromise.sendError(e);
-				return Void();
+				co_return;
 			} else {
 				throw;
 			}
 		}
 	}
 
-	state double blockStart = now();
+	double blockStart = now();
 
 	// We need to return data that the caller doesn't already have.
 	// If the requested version is beyond what the tLog currently has, we'll wait for new data.
@@ -1843,7 +1864,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 	// To prevent deadlocks:
 	//   - If a valid 'end' version was provided in the request (the Recovery Version), return with that version.
 	//   - Otherwise, wait for new data as long as the tLog isn't locked.
-	state Optional<Version> replyWithRecoveryVersion = Optional<Version>();
+	Optional<Version> replyWithRecoveryVersion = Optional<Version>();
 	ASSERT(!SERVER_KNOBS->ENABLE_VERSION_VECTOR_REPLY_RECOVERY || SERVER_KNOBS->ENABLE_VERSION_VECTOR);
 	if (logData->version.get() < reqBegin) {
 		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR_REPLY_RECOVERY && SERVER_KNOBS->ENABLE_VERSION_VECTOR &&
@@ -1861,7 +1882,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 					sequenceData.send(std::make_pair(reqBegin, reqOnlySpilled));
 				}
 			}
-			return Void();
+			co_return;
 		}
 	}
 
@@ -1874,8 +1895,8 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 	    .detail("ClusterRecovery", replyWithRecoveryVersion.present() ? replyWithRecoveryVersion.get() : -1);
 	// Wait until we have something to return that the caller doesn't already have
 	if (!replyWithRecoveryVersion.present() && logData->version.get() < reqBegin) {
-		wait(logData->version.whenAtLeast(reqBegin));
-		wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask()));
+		co_await logData->version.whenAtLeast(reqBegin);
+		co_await delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask());
 	}
 	if (!logData->stopped() && reqTag.locality != tagLocalityTxs && reqTag != txsTag &&
 	    logData->version.get() < logData->recoveryTxnVersion) {
@@ -1890,13 +1911,14 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		// Older generation TLog has been stopped and doesn't wait here.
 		// Similarly during recovery, reading transaction state store
 		// doesn't wait here.
-		wait(logData->version.whenAtLeast(logData->recoveryTxnVersion) || logData->stoppedPromise.getFuture());
+		co_await (logData->version.whenAtLeast(logData->recoveryTxnVersion) || logData->stoppedPromise.getFuture());
 	}
 
+	FlowLock::Releaser globalReleaser;
 	if (logData->locality != tagLocalitySatellite && reqTag.locality == tagLocalityLogRouter) {
-		wait(self->concurrentLogRouterReads.take());
-		state FlowLock::Releaser globalReleaser(self->concurrentLogRouterReads);
-		wait(delay(0.0, TaskPriority::Low));
+		co_await self->concurrentLogRouterReads.take();
+		globalReleaser = FlowLock::Releaser(self->concurrentLogRouterReads);
+		co_await delay(0.0, TaskPriority::Low);
 	}
 
 	if (reqBegin <= logData->persistentDataDurableVersion && reqTag.locality != tagLocalityTxs && reqTag != txsTag) {
@@ -1905,17 +1927,17 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		// slightly faster over keeping the rest of the cluster operating normally.
 		// txsTag is only ever peeked on recovery, and we would still wish to prioritize requests
 		// that impact recovery duration.
-		wait(delay(0, TaskPriority::TLogSpilledPeekReply));
+		co_await delay(0, TaskPriority::TLogSpilledPeekReply);
 	}
 
-	state double workStart = now();
-	state Version poppedVer;
-	state Version endVersion;
-	state bool onlySpilled;
+	double workStart = now();
+	Version poppedVer{ 0 };
+	Version endVersion{ 0 };
+	bool onlySpilled{ false };
 
 	// Run the peek logic in a loop to account for the case where there is no data to return to the caller, and we may
 	// want to wait a little bit instead of just sending back an empty message. This feature is controlled by a knob.
-	loop {
+	while (true) {
 		poppedVer = poppedVersion(logData, reqTag);
 
 		auto tagData = logData->getTagData(reqTag);
@@ -1923,8 +1945,8 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 		if (SERVER_KNOBS->ENABLE_VERSION_VECTOR && poppedVer <= reqBegin &&
 		    reqBegin > logData->persistentDataDurableVersion && !reqOnlySpilled && reqTag.locality >= 0 &&
 		    !reqReturnIfBlocked && tagRecovered) {
-			state double startTime = now();
-			wait(waitForMessagesForTag(logData, reqTag, reqBegin, SERVER_KNOBS->BLOCKING_PEEK_TIMEOUT));
+			double startTime = now();
+			co_await waitForMessagesForTag(logData, reqTag, reqBegin, SERVER_KNOBS->BLOCKING_PEEK_TIMEOUT);
 			double latency = now() - startTime;
 			auto [it, inserted] =
 			    logData->blockingPeekLatencies.try_emplace(reqTag,
@@ -1960,14 +1982,14 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 					replyPromise.sendError(operation_obsolete());
 					if (!sequenceData.isSet())
 						sequenceData.sendError(operation_obsolete());
-					return Void();
+					co_return;
 				}
 				if (sequenceData.isSet()) {
 					if (sequenceData.getFuture().get().first != rep.end) {
 						CODE_PROBE(
 						    true, "tlog peek second attempt ended at a different version", probe::decoration::rare);
 						replyPromise.sendError(operation_obsolete());
-						return Void();
+						co_return;
 					}
 				} else {
 					sequenceData.send(std::make_pair(rep.end, rep.onlySpilled));
@@ -1976,7 +1998,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			}
 
 			replyPromise.send(rep);
-			return Void();
+			co_return;
 		}
 
 		ASSERT(reqBegin >= poppedVersion(logData, reqTag));
@@ -2002,12 +2024,12 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			}
 
 			if (logData->shouldSpillByValue(reqTag)) {
-				RangeResult kvs = wait(self->persistentData->readRange(
+				RangeResult kvs = co_await self->persistentData->readRange(
 				    KeyRangeRef(
 				        persistTagMessagesKey(logData->logId, reqTag, reqBegin),
 				        persistTagMessagesKey(logData->logId, reqTag, logData->persistentDataDurableVersion + 1)),
 				    SERVER_KNOBS->DESIRED_TOTAL_BYTES,
-				    SERVER_KNOBS->DESIRED_TOTAL_BYTES));
+				    SERVER_KNOBS->DESIRED_TOTAL_BYTES);
 
 				for (auto& kv : kvs) {
 					auto ver = decodeTagMessagesKey(kv.key);
@@ -2023,19 +2045,19 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 				}
 			} else {
 				// FIXME: Limit to approximately DESIRED_TOTATL_BYTES somehow.
-				RangeResult kvrefs = wait(self->persistentData->readRange(
+				RangeResult kvrefs = co_await self->persistentData->readRange(
 				    KeyRangeRef(
 				        persistTagMessageRefsKey(logData->logId, reqTag, reqBegin),
 				        persistTagMessageRefsKey(logData->logId, reqTag, logData->persistentDataDurableVersion + 1)),
-				    SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK + 1));
+				    SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK + 1);
 
 				//TraceEvent("TLogPeekResults", self->dbgid).detail("ForAddress", replyPromise.getEndpoint().getPrimaryAddress()).detail("Tag1Results", s1).detail("Tag2Results", s2).detail("Tag1ResultsLim", kv1.size()).detail("Tag2ResultsLim", kv2.size()).detail("Tag1ResultsLast", kv1.size() ? kv1[0].key : "").detail("Tag2ResultsLast", kv2.size() ? kv2[0].key : "").detail("Limited", limited).detail("NextEpoch", next_pos.epoch).detail("NextSeq", next_pos.sequence).detail("NowEpoch", self->epoch()).detail("NowSeq", self->sequence.getNextSequence());
 
-				state std::vector<std::pair<IDiskQueue::location, IDiskQueue::location>> commitLocations;
-				state bool earlyEnd = false;
+				std::vector<std::pair<IDiskQueue::location, IDiskQueue::location>> commitLocations;
+				bool earlyEnd = false;
 				uint32_t mutationBytes = 0;
-				state uint64_t commitBytes = 0;
-				state Version firstVersion = std::numeric_limits<Version>::max();
+				uint64_t commitBytes = 0;
+				Version firstVersion = std::numeric_limits<Version>::max();
 				for (int i = 0; i < kvrefs.size() && i < SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK; i++) {
 					auto& kv = kvrefs[i];
 					VectorRef<SpilledData> spilledData;
@@ -2060,19 +2082,19 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 						break;
 				}
 				earlyEnd = earlyEnd || (kvrefs.size() >= SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK + 1);
-				wait(self->peekMemoryLimiter.take(TaskPriority::TLogSpilledPeekReply, commitBytes));
-				state FlowLock::Releaser memoryReservation(self->peekMemoryLimiter, commitBytes);
-				state std::vector<Future<Standalone<StringRef>>> messageReads;
+				co_await self->peekMemoryLimiter.take(TaskPriority::TLogSpilledPeekReply, commitBytes);
+				FlowLock::Releaser memoryReservation(self->peekMemoryLimiter, commitBytes);
+				std::vector<Future<Standalone<StringRef>>> messageReads;
 				messageReads.reserve(commitLocations.size());
 				for (const auto& pair : commitLocations) {
 					messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::True));
 				}
 				commitLocations.clear();
-				wait(waitForAll(messageReads));
+				co_await waitForAll(messageReads);
 
-				state Version lastRefMessageVersion = 0;
-				state int index = 0;
-				loop {
+				Version lastRefMessageVersion = 0;
+				int index = 0;
+				while (true) {
 					if (index >= messageReads.size())
 						break;
 					Standalone<StringRef> queueEntryData = messageReads[index].get();
@@ -2080,7 +2102,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 					const uint32_t length = *(uint32_t*)queueEntryData.begin();
 					queueEntryData = queueEntryData.substr(4, queueEntryData.size() - 4);
 					BinaryReader rd(queueEntryData, IncludeVersion());
-					state TLogQueueEntry entry;
+					TLogQueueEntry entry;
 					rd >> entry >> valid;
 					ASSERT(valid == 0x01);
 					ASSERT(length + sizeof(valid) == queueEntryData.size());
@@ -2088,7 +2110,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 					messages << VERSION_HEADER << entry.version;
 
 					std::vector<StringRef> rawMessages =
-					    wait(parseMessagesForTag(entry.messages, reqTag, logData->logRouterTags));
+					    co_await parseMessagesForTag(entry.messages, reqTag, logData->logRouterTags);
 					for (const StringRef& msg : rawMessages) {
 						messages.serializeBytes(msg);
 						DEBUG_TAGS_AND_MESSAGE("TLogPeekFromDisk", entry.version, msg, logData->logId)
@@ -2129,12 +2151,12 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 			break;
 		}
 
-		state Version waitUntilVersion = logData->version.get() + 1;
+		Version waitUntilVersion = logData->version.get() + 1;
 
 		// Currently, from `reqBegin` to logData->version are all empty peeks. Wait for more versions, or the empty
 		// batching interval has expired.
-		wait(logData->version.whenAtLeast(waitUntilVersion) ||
-		     delay(SERVER_KNOBS->PEEK_BATCHING_EMPTY_MSG_INTERVAL - (now() - blockStart)));
+		co_await (logData->version.whenAtLeast(waitUntilVersion) ||
+		          delay(SERVER_KNOBS->PEEK_BATCHING_EMPTY_MSG_INTERVAL - (now() - blockStart)));
 		if (logData->version.get() < waitUntilVersion) {
 			break; // We know that from `reqBegin` to logData->version are all empty messages. Skip re-executing the
 			       // peek logic.
@@ -2192,14 +2214,14 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 				// response will probably be a waste of CPU.
 				sequenceData.sendError(operation_obsolete());
 			}
-			return Void();
+			co_return;
 		}
 		if (sequenceData.isSet()) {
 			trackerData.duplicatePeeks++;
 			if (sequenceData.getFuture().get().first != reply.end) {
 				CODE_PROBE(true, "tlog peek second attempt ended at a different version (2)");
 				replyPromise.sendError(operation_obsolete());
-				return Void();
+				co_return;
 			}
 		} else {
 			sequenceData.send(std::make_pair(reply.end, reply.onlySpilled));
@@ -2208,41 +2230,40 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
 	}
 
 	replyPromise.send(reply);
-	return Void();
 }
 
 // This actor keep pushing TLogPeekStreamReply until it's removed from the cluster or should recover
-ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Reference<LogData> logData) {
+Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Reference<LogData> logData) {
 	self->activePeekStreams++;
 
-	state Version begin = req.begin;
-	state bool onlySpilled = false;
+	Version begin = req.begin;
+	bool onlySpilled = false;
 	req.reply.setByteLimit(std::min(SERVER_KNOBS->MAXIMUM_PEEK_BYTES, req.limitBytes));
-	loop {
-		state TLogPeekStreamReply reply;
-		state Promise<TLogPeekReply> promise;
-		state Future<TLogPeekReply> future(promise.getFuture());
+	while (true) {
+		TLogPeekStreamReply reply;
+		Promise<TLogPeekReply> promise;
+		Future<TLogPeekReply> future(promise.getFuture());
 		try {
-			wait(req.reply.onReady() && store(reply.rep, future) &&
-			     tLogPeekMessages(promise,
-			                      self,
-			                      logData,
-			                      begin,
-			                      req.tag,
-			                      req.returnIfBlocked,
-			                      onlySpilled,
-			                      Optional<std::pair<UID, int>>(),
-			                      req.end,
-			                      req.returnEmptyIfStopped));
+			co_await (req.reply.onReady() && store(reply.rep, future) &&
+			          tLogPeekMessages(promise,
+			                           self,
+			                           logData,
+			                           begin,
+			                           req.tag,
+			                           req.returnIfBlocked,
+			                           onlySpilled,
+			                           Optional<std::pair<UID, int>>(),
+			                           req.end,
+			                           req.returnEmptyIfStopped));
 
 			reply.rep.begin = begin;
 			req.reply.send(reply);
 			begin = reply.rep.end;
 			onlySpilled = reply.rep.onlySpilled;
 			if (reply.rep.end > logData->version.get()) {
-				wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask()));
+				co_await delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask());
 			} else {
-				wait(delay(0, g_network->getCurrentTask()));
+				co_await delay(0, g_network->getCurrentTask());
 			}
 		} catch (Error& e) {
 			self->activePeekStreams--;
@@ -2254,7 +2275,7 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 
 			if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {
 				req.reply.sendError(e);
-				return Void();
+				co_return;
 			} else {
 				throw;
 			}
@@ -2262,12 +2283,12 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
 	}
 }
 
-ACTOR Future<Void> doQueueCommit(TLogData* self,
-                                 Reference<LogData> logData,
-                                 std::vector<Reference<LogData>> missingFinalCommit) {
-	state Version ver = logData->version.get();
-	state Version commitNumber = self->queueCommitBegin + 1;
-	state Version knownCommittedVersion = logData->knownCommittedVersion;
+Future<Void> doQueueCommit(TLogData* self,
+                           Reference<LogData> logData,
+                           std::vector<Reference<LogData>> missingFinalCommit) {
+	Version ver = logData->version.get();
+	Version commitNumber = self->queueCommitBegin + 1;
+	Version knownCommittedVersion = logData->knownCommittedVersion;
 	self->queueCommitBegin = commitNumber;
 	logData->queueCommittingVersion = ver;
 
@@ -2276,16 +2297,16 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 	self->diskQueueCommitBytes = 0;
 	self->largeDiskQueueCommitBytes.set(false);
 
-	wait(ioDegradedOrTimeoutError(
-	    c, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, self->degraded, SERVER_KNOBS->TLOG_DEGRADED_DURATION, "TLogCommit"));
+	co_await ioDegradedOrTimeoutError(
+	    c, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, self->degraded, SERVER_KNOBS->TLOG_DEGRADED_DURATION, "TLogCommit");
 	if (g_network->isSimulated() && !g_simulator->speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
-		wait(delay(6.0));
+		co_await delay(6.0);
 	}
-	wait(self->queueCommitEnd.whenAtLeast(commitNumber - 1));
+	co_await self->queueCommitEnd.whenAtLeast(commitNumber - 1);
 
 	// Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
 	if (g_network->check_yield(g_network->getCurrentTask())) {
-		wait(delay(0, g_network->getCurrentTask()));
+		co_await delay(0, g_network->getCurrentTask());
 	}
 
 	ASSERT(ver > logData->queueCommittedVersion.get());
@@ -2321,14 +2342,13 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 		CODE_PROBE(true, "A TLog was replaced before having a chance to commit its queue", probe::decoration::rare);
 		it->queueCommittedVersion.set(it->version.get());
 	}
-	return Void();
 }
 
-ACTOR Future<Void> commitQueue(TLogData* self) {
-	state Reference<LogData> logData;
-	state std::vector<Reference<LogData>> missingFinalCommit;
+Future<Void> commitQueue(TLogData* self) {
+	Reference<LogData> logData;
+	std::vector<Reference<LogData>> missingFinalCommit;
 
-	loop {
+	while (true) {
 		int foundCount = 0;
 		for (auto it : self->id_data) {
 			if (!it.second->stopped()) {
@@ -2342,7 +2362,7 @@ ACTOR Future<Void> commitQueue(TLogData* self) {
 
 		ASSERT(foundCount < 2);
 		if (!foundCount) {
-			wait(self->newLogData.onTrigger());
+			co_await self->newLogData.onTrigger();
 			continue;
 		}
 
@@ -2355,16 +2375,16 @@ ACTOR Future<Void> commitQueue(TLogData* self) {
 			logData->committingQueue.send(Void());
 		}
 
-		loop {
+		while (true) {
 			// Insert enough of a delay to allow this tlog to be stopped and a new one registered
 			// before the commit is issued. These are the conditions which trigger a missingFinalCommit.
 			if (BUGGIFY_WITH_PROB(0.0001) && !g_simulator->speedUpSimulation) {
-				wait(delay(1.0));
+				co_await delay(1.0);
 			}
 
 			if (logData->stopped() && logData->version.get() == std::max(logData->queueCommittingVersion,
 			                                                             logData->queueCommittedVersion.get())) {
-				wait(logData->queueCommittedVersion.whenAtLeast(logData->version.get()));
+				co_await logData->queueCommittedVersion.whenAtLeast(logData->version.get());
 				break;
 			}
 
@@ -2372,32 +2392,33 @@ ACTOR Future<Void> commitQueue(TLogData* self) {
 				break;
 			}
 
-			choose {
-				when(wait(logData->version.whenAtLeast(
-				    std::max(logData->queueCommittingVersion, logData->queueCommittedVersion.get()) + 1))) {
-					while (self->queueCommitBegin != self->queueCommitEnd.get() &&
-					       !self->largeDiskQueueCommitBytes.get()) {
-						wait(self->queueCommitEnd.whenAtLeast(self->queueCommitBegin) ||
-						     self->largeDiskQueueCommitBytes.onChange());
-					}
-					if (logData->queueCommittedVersion.get() == std::numeric_limits<Version>::max()) {
-						break;
-					}
-					self->sharedActors.send(doQueueCommit(self, logData, missingFinalCommit));
-					missingFinalCommit.clear();
+			auto res =
+			    co_await race(logData->version.whenAtLeast(
+			                      std::max(logData->queueCommittingVersion, logData->queueCommittedVersion.get()) + 1),
+			                  self->newLogData.onTrigger());
+			if (res.index() == 0) {
+				while (self->queueCommitBegin != self->queueCommitEnd.get() && !self->largeDiskQueueCommitBytes.get()) {
+					co_await (self->queueCommitEnd.whenAtLeast(self->queueCommitBegin) ||
+					          self->largeDiskQueueCommitBytes.onChange());
 				}
-				when(wait(self->newLogData.onTrigger())) {}
+				if (logData->queueCommittedVersion.get() == std::numeric_limits<Version>::max()) {
+					break;
+				}
+				self->sharedActors.send(doQueueCommit(self, logData, missingFinalCommit));
+				missingFinalCommit.clear();
+			} else {
+				ASSERT(res.index() == 1);
 			}
 		}
 	}
 }
 
-ACTOR Future<Void> tLogCommit(TLogData* self,
-                              TLogCommitRequest req,
-                              Reference<LogData> logData,
-                              PromiseStream<Void> warningCollectorInput) {
-	state Span span("TLog:tLogCommit"_loc, req.spanContext);
-	state Optional<UID> tlogDebugID;
+Future<Void> tLogCommit(TLogData* self,
+                        TLogCommitRequest req,
+                        Reference<LogData> logData,
+                        PromiseStream<Void> warningCollectorInput) {
+	Span span("TLog:tLogCommit"_loc, req.spanContext);
+	Optional<UID> tlogDebugID;
 	if (req.debugID.present()) {
 		tlogDebugID = nondeterministicRandom()->randomUniqueID();
 		g_traceBatch.addAttach("CommitAttachID", req.debugID.get().first(), tlogDebugID.get().first());
@@ -2406,17 +2427,17 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 
 	logData->minKnownCommittedVersion = std::max(logData->minKnownCommittedVersion, req.minKnownCommittedVersion);
 
-	wait(logData->version.whenAtLeast(req.prevVersion));
+	co_await logData->version.whenAtLeast(req.prevVersion);
 	// Time until now has been spent waiting in the queue to do actual work.
-	state double queueWaitEndTime = g_network->timer();
+	double queueWaitEndTime = g_network->timer();
 	self->queueWaitLatencyDist->sampleSeconds(queueWaitEndTime - req.requestTime());
 
 	// Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
 	if (g_network->check_yield(g_network->getCurrentTask())) {
-		wait(delay(0, g_network->getCurrentTask()));
+		co_await delay(0, g_network->getCurrentTask());
 	}
 
-	state double waitStartT = 0;
+	double waitStartT = 0;
 	while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped()) {
 		if (now() - waitStartT >= 1) {
 			TraceEvent(SevWarn, "TLogUpdateLag", logData->logId)
@@ -2425,16 +2446,16 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 			    .detail("PersistentDataDurableVersion", logData->persistentDataDurableVersion);
 			waitStartT = now();
 		}
-		wait(delayJittered(.005, TaskPriority::TLogCommit));
+		co_await delayJittered(.005, TaskPriority::TLogCommit);
 	}
 
 	if (logData->stopped()) {
 		req.reply.sendError(tlog_stopped());
-		return Void();
+		co_return;
 	}
 
 	// Not a duplicate (check relies on critical section between here self->version.set() below!)
-	state bool isNotDuplicate = (logData->version.get() == req.prevVersion);
+	bool isNotDuplicate = (logData->version.get() == req.prevVersion);
 	if (isNotDuplicate) {
 		if (req.debugID.present())
 			g_traceBatch.addEvent("CommitDebug", tlogDebugID.get().first(), "TLog.tLogCommit.Before");
@@ -2472,9 +2493,9 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 			g_traceBatch.addEvent("CommitDebug", tlogDebugID.get().first(), "TLog.tLogCommit.AfterTLogCommit");
 	}
 	// Send replies only once all prior messages have been received and committed.
-	state Future<Void> stopped = logData->stopCommit.onTrigger();
-	wait(
-	    timeoutWarning(logData->queueCommittedVersion.whenAtLeast(req.version) || stopped, 0.1, warningCollectorInput));
+	Future<Void> stopped = logData->stopCommit.onTrigger();
+	co_await timeoutWarning(
+	    logData->queueCommittedVersion.whenAtLeast(req.version) || stopped, 0.1, warningCollectorInput);
 
 	// This is the point at which the transaction is durable (unless it timed out, or the tlog stopped).
 	const double durableTime = g_network->timer();
@@ -2482,7 +2503,7 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 	if (stopped.isReady()) {
 		ASSERT(logData->stopped());
 		req.reply.sendError(tlog_stopped());
-		return Void();
+		co_return;
 	}
 
 	if (req.debugID.present())
@@ -2498,15 +2519,13 @@ ACTOR Future<Void> tLogCommit(TLogData* self,
 		self->timeUntilDurableDist->sampleSeconds(durableTime - queueWaitEndTime);
 		self->commitLatencyDist->sampleSeconds(endTime - req.requestTime());
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> initPersistentState(TLogData* self, Reference<LogData> logData) {
-	wait(self->persistentDataCommitLock.take());
-	state FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
+Future<Void> initPersistentState(TLogData* self, Reference<LogData> logData) {
+	co_await self->persistentDataCommitLock.take();
+	FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
 
-	state IKeyValueStore* storage = self->persistentData;
+	IKeyValueStore* storage = self->persistentData;
 	storage->set(
 	    KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistCurrentVersionKeys.begin),
 	                BinaryWriter::toValue(logData->version.get(), Unversioned())));
@@ -2532,24 +2551,23 @@ ACTOR Future<Void> initPersistentState(TLogData* self, Reference<LogData> logDat
 
 	for (auto tag : logData->allTags) {
 		ASSERT(!logData->getTagData(tag));
-		logData->createTagData(tag, 0, true, true, true);
+		logData->createTagData(tag, 0, NothingPersistent::True, PoppedRecently::True, UnpoppedRecovered::True);
 		updatePersistentPopped(self, logData, logData->getTagData(tag));
 	}
 
 	TraceEvent("TLogInitCommit", logData->logId);
-	wait(self->persistentData->commit());
-	return Void();
+	co_await self->persistentData->commit();
 }
 
 // send stopped promise instead of LogData* to avoid reference cycles
-ACTOR Future<Void> rejoinClusterController(TLogData* self,
-                                           TLogInterface tli,
-                                           DBRecoveryCount recoveryCount,
-                                           Promise<Void> stoppedPromise,
-                                           Future<Void> registerWithCC,
-                                           bool isPrimary) {
-	state LifetimeToken lastMasterLifetime;
-	loop {
+Future<Void> rejoinClusterController(TLogData* self,
+                                     TLogInterface tli,
+                                     DBRecoveryCount recoveryCount,
+                                     Promise<Void> stoppedPromise,
+                                     Future<Void> registerWithCC,
+                                     bool isPrimary) {
+	LifetimeToken lastMasterLifetime;
+	while (true) {
 		auto const& inf = self->dbInfo->get();
 		bool isDisplaced =
 		    std::find(inf.priorCommittedLogServers.begin(), inf.priorCommittedLogServers.end(), tli.id()) ==
@@ -2573,7 +2591,7 @@ ACTOR Future<Void> rejoinClusterController(TLogData* self,
 			    .detail("PriorLogs", describe(inf.priorCommittedLogServers))
 			    .detail("OldLogGens", inf.logSystemConfig.oldTLogs.size());
 			if (BUGGIFY)
-				wait(delay(SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * deterministicRandom()->random01()));
+				co_await delay(SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * deterministicRandom()->random01());
 			throw worker_removed();
 		} else if (inf.recoveryCount > recoveryCount && stoppedPromise.canBeSet()) {
 			CODE_PROBE(true, "Stopping tlog because new dbinfo has a higher recovery count");
@@ -2585,7 +2603,7 @@ ACTOR Future<Void> rejoinClusterController(TLogData* self,
 		}
 
 		if (self->terminated.isSet()) {
-			return Void();
+			co_return;
 		}
 
 		if (registerWithCC.isReady()) {
@@ -2597,27 +2615,29 @@ ACTOR Future<Void> rejoinClusterController(TLogData* self,
 				    .detail("ClusterController", self->dbInfo->get().clusterInterface.id())
 				    .detail("DbInfoMasterLifeTime", self->dbInfo->get().masterLifetime.toString())
 				    .detail("LastMasterLifeTime", lastMasterLifetime.toString());
-				choose {
-					when(TLogRejoinReply rep = wait(
-					         brokenPromiseToNever(self->dbInfo->get().clusterInterface.tlogRejoin.getReply(req)))) {
-						if (rep.masterIsRecovered)
-							lastMasterLifetime = self->dbInfo->get().masterLifetime;
-					}
-					when(wait(self->dbInfo->onChange())) {}
+				auto res =
+				    co_await race(brokenPromiseToNever(self->dbInfo->get().clusterInterface.tlogRejoin.getReply(req)),
+				                  self->dbInfo->onChange());
+				if (res.index() == 0) {
+					TLogRejoinReply rep = std::get<0>(std::move(res));
+					if (rep.masterIsRecovered)
+						lastMasterLifetime = self->dbInfo->get().masterLifetime;
+				} else {
+					ASSERT(res.index() == 1);
 				}
 			} else {
-				wait(self->dbInfo->onChange());
+				co_await self->dbInfo->onChange();
 			}
 		} else {
-			wait(registerWithCC || self->dbInfo->onChange());
+			co_await (registerWithCC || self->dbInfo->onChange());
 		}
 	}
 }
 
-ACTOR Future<Void> respondToRecovered(TLogInterface tli, Promise<Void> recoveryComplete) {
-	state bool finishedRecovery = true;
+Future<Void> respondToRecovered(TLogInterface tli, Promise<Void> recoveryComplete) {
+	bool finishedRecovery = true;
 	try {
-		wait(recoveryComplete.getFuture());
+		co_await recoveryComplete.getFuture();
 	} catch (Error& e) {
 		if (e.code() != error_code_end_of_stream) {
 			throw;
@@ -2627,14 +2647,14 @@ ACTOR Future<Void> respondToRecovered(TLogInterface tli, Promise<Void> recoveryC
 
 	// This delay is added for testing purpose in simulation where by setting `disableTLogRecoveryFinish`, we disable
 	// TLogs to send back `TLogRecoveryFinishedRequest`.
-	while (g_network->isSimulated() && g_simulator->disableTLogRecoveryFinish) {
-		TraceEvent("WaitingToBeUnblocked", tli.id());
-		wait(delay(10));
+	while (g_network->isSimulated() && fdbSimulationPolicyState().disableTLogRecoveryFinish) {
+		TraceEvent("WaitingToBeUnblocked", tli.id()).suppressFor(60);
+		co_await delay(10);
 	}
 
 	TraceEvent("TLogRespondToRecovered", tli.id()).detail("Finished", finishedRecovery);
-	loop {
-		TLogRecoveryFinishedRequest req = waitNext(tli.recoveryFinished.getFuture());
+	while (true) {
+		TLogRecoveryFinishedRequest req = co_await tli.recoveryFinished.getFuture();
 		if (finishedRecovery) {
 			req.reply.send(Void());
 		} else {
@@ -2643,8 +2663,16 @@ ACTOR Future<Void> respondToRecovered(TLogInterface tli, Promise<Void> recoveryC
 	}
 }
 
-ACTOR Future<Void> trackRecoveryReq(TLogInterface tli, TrackTLogRecoveryRequest req, Reference<LogData> logData) {
-	loop {
+Future<Void> trackRecoveryReq(TLogInterface tli, TrackTLogRecoveryRequest req, Reference<LogData> logData) {
+	// Block recovery version tracking when disableTLogRecoveryFinish is set in simulation.
+	// This prevents recoveredVersion from advancing in trackTLogRecoveryActor, which in turn
+	// prevents purgeOldRecoveredGenerationsCoreState from GC'ing old TLog generations.
+	// Without this, the GcGenerations test accumulation phase races with generation GC.
+	while (g_network->isSimulated() && fdbSimulationPolicyState().disableTLogRecoveryFinish) {
+		co_await delay(10);
+	}
+
+	while (true) {
 		Version oldestGenerationRecoverAtVersion = invalidVersion;
 		for (const auto& [tag, genVersions] : logData->tagUnpoppedOldGenerations) {
 			ASSERT(!genVersions.empty());
@@ -2663,20 +2691,19 @@ ACTOR Future<Void> trackRecoveryReq(TLogInterface tli, TrackTLogRecoveryRequest
 			break;
 		}
 
-		wait(logData->updateGenerationRecovery.onTrigger());
+		co_await logData->updateGenerationRecovery.onTrigger();
 	}
-	return Void();
 }
 
-ACTOR Future<Void> respondToTrackRecovery(TLogInterface tli, Reference<LogData> logData) {
-	loop {
-		TrackTLogRecoveryRequest req = waitNext(tli.trackRecovery.getFuture());
+Future<Void> respondToTrackRecovery(TLogInterface tli, Reference<LogData> logData) {
+	while (true) {
+		TrackTLogRecoveryRequest req = co_await tli.trackRecovery.getFuture();
 		logData->addActor.send(trackRecoveryReq(tli, req, logData));
 	}
 }
 
-ACTOR Future<Void> cleanupPeekTrackers(LogData* logData) {
-	loop {
+Future<Void> cleanupPeekTrackers(LogData* logData) {
+	while (true) {
 		double minTimeUntilExpiration = SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME;
 		auto it = logData->peekTracker.begin();
 		while (it != logData->peekTracker.end()) {
@@ -2694,12 +2721,12 @@ ACTOR Future<Void> cleanupPeekTrackers(LogData* logData) {
 			}
 		}
 
-		wait(delay(minTimeUntilExpiration));
+		co_await delay(minTimeUntilExpiration);
 	}
 }
 
-ACTOR Future<Void> logPeekTrackers(LogData* logData) {
-	loop {
+Future<Void> logPeekTrackers(LogData* logData) {
+	while (true) {
 		int64_t logThreshold = 1;
 		if (logData->peekTracker.size() > SERVER_KNOBS->PEEK_LOGGING_AMOUNT) {
 			std::vector<int64_t> peekCounts;
@@ -2736,7 +2763,7 @@ ACTOR Future<Void> logPeekTrackers(LogData* logData) {
 			}
 		}
 
-		wait(delay(SERVER_KNOBS->PEEK_LOGGING_DELAY * std::max(1, logCount)));
+		co_await delay(SERVER_KNOBS->PEEK_LOGGING_DELAY * std::max(1, logCount));
 	}
 }
 
@@ -2752,13 +2779,13 @@ void getQueuingMetrics(TLogData* self, Reference<LogData> logData, TLogQueuingMe
 	req.reply.send(reply);
 }
 
-ACTOR Future<Void> tLogEnablePopReq(TLogEnablePopRequest enablePopReq, TLogData* self, Reference<LogData> logData) {
+Future<Void> tLogEnablePopReq(TLogEnablePopRequest enablePopReq, TLogData* self, Reference<LogData> logData) {
 	if (self->ignorePopUid != enablePopReq.snapUID.toString()) {
 		TraceEvent(SevWarn, "TLogPopDisableEnableUidMismatch")
 		    .detail("IgnorePopUid", self->ignorePopUid)
 		    .detail("UidStr", enablePopReq.snapUID.toString());
 		enablePopReq.reply.sendError(operation_failed());
-		return Void();
+		co_return;
 	}
 	TraceEvent("EnableTLogPlayAllIgnoredPops2", logData->logId)
 	    .detail("UidStr", enablePopReq.snapUID.toString())
@@ -2769,19 +2796,20 @@ ACTOR Future<Void> tLogEnablePopReq(TLogEnablePopRequest enablePopReq, TLogData*
 	    .detail("QueueCommittedVersion", logData->queueCommittedVersion.get())
 	    .detail("Version", logData->version.get());
 	self->ignorePopDeadline = 0;
-	wait(processPopRequests(self, logData));
+	co_await processPopRequests(self, logData);
 	enablePopReq.reply.send(Void());
-	return Void();
 }
 
-ACTOR Future<Void> serveTLogInterface(TLogData* self,
-                                      TLogInterface tli,
-                                      Reference<LogData> logData,
-                                      PromiseStream<Void> warningCollectorInput) {
-	state Future<Void> dbInfoChange = Void();
+class ServeTLogInterface {
+	TLogData* self;
+	TLogInterface tli;
+	Reference<LogData> logData;
+	PromiseStream<Void> warningCollectorInput;
+	Future<Void> dbInfoChange;
 
-	loop choose {
-		when(wait(dbInfoChange)) {
+	Future<Void> updateDbInfo() {
+		while (true) {
+			co_await dbInfoChange;
 			dbInfoChange = self->dbInfo->onChange();
 			bool found = false;
 			if (self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
@@ -2793,7 +2821,7 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 				}
 			}
 			if (found && self->dbInfo->get().logSystemConfig.recruitmentID == logData->recruitmentID) {
-				logData->logSystem->set(ILogSystem::fromServerDBInfo(self->dbgid, self->dbInfo->get()));
+				logData->logSystem->set(makeLogSystemFromServerDBInfo(self->dbgid, self->dbInfo->get()));
 				if (!logData->isPrimary) {
 					logData->logSystem->get()->pop(logData->logRouterPoppedVersion,
 					                               logData->remoteTag,
@@ -2806,15 +2834,23 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 					logData->removed = logData->removed && logData->logSystem->get()->endEpoch();
 				}
 			} else {
-				logData->logSystem->set(Reference<ILogSystem>());
+				logData->logSystem->set(Reference<LogSystem>());
 			}
 		}
-		when(TLogPeekStreamRequest req = waitNext(tli.peekStreamMessages.getFuture())) {
+	}
+
+	Future<Void> servePeekStreams() {
+		while (true) {
+			TLogPeekStreamRequest req = co_await tli.peekStreamMessages.getFuture();
 			TraceEvent(SevDebug, "TLogPeekStream", logData->logId)
 			    .detail("Token", tli.peekStreamMessages.getEndpoint().token);
 			logData->addActor.send(tLogPeekStream(self, req, logData));
 		}
-		when(TLogPeekRequest req = waitNext(tli.peekMessages.getFuture())) {
+	}
+
+	Future<Void> servePeeks() {
+		while (true) {
+			TLogPeekRequest req = co_await tli.peekMessages.getFuture();
 			logData->addActor.send(tLogPeekMessages(req.reply,
 			                                        self,
 			                                        logData,
@@ -2826,25 +2862,46 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 			                                        req.end,
 			                                        req.returnEmptyIfStopped));
 		}
-		when(TLogPopRequest req = waitNext(tli.popMessages.getFuture())) {
+	}
+
+	Future<Void> servePops() {
+		while (true) {
+			TLogPopRequest req = co_await tli.popMessages.getFuture();
 			logData->addActor.send(tLogPop(self, req, logData));
 		}
-		when(TLogCommitRequest req = waitNext(tli.commit.getFuture())) {
+	}
+
+	Future<Void> serveCommits() {
+		while (true) {
+			TLogCommitRequest req = co_await tli.commit.getFuture();
 			//TraceEvent("TLogCommitReq", logData->logId).detail("Ver", req.version).detail("PrevVer", req.prevVersion).detail("LogVer", logData->version.get());
 			ASSERT(logData->isPrimary);
 			CODE_PROBE(logData->stopped(), "TLogCommitRequest while stopped");
-			if (!logData->stopped())
-				logData->addActor.send(tLogCommit(self, req, logData, warningCollectorInput));
-			else
+			if (logData->stopped()) {
 				req.reply.sendError(tlog_stopped());
+			} else {
+				logData->addActor.send(tLogCommit(self, req, logData, warningCollectorInput));
+			}
 		}
-		when(ReplyPromise<TLogLockResult> reply = waitNext(tli.lock.getFuture())) {
+	}
+
+	Future<Void> serveLocks() {
+		while (true) {
+			ReplyPromise<TLogLockResult> reply = co_await tli.lock.getFuture();
 			logData->addActor.send(tLogLock(self, reply, logData));
 		}
-		when(TLogQueuingMetricsRequest req = waitNext(tli.getQueuingMetrics.getFuture())) {
+	}
+
+	Future<Void> serveQueuingMetrics() {
+		while (true) {
+			TLogQueuingMetricsRequest req = co_await tli.getQueuingMetrics.getFuture();
 			getQueuingMetrics(self, logData, req);
 		}
-		when(TLogConfirmRunningRequest req = waitNext(tli.confirmRunning.getFuture())) {
+	}
+
+	Future<Void> serveConfirmRunning() {
+		while (true) {
+			TLogConfirmRunningRequest req = co_await tli.confirmRunning.getFuture();
 			if (req.debugID.present()) {
 				UID tlogDebugID = nondeterministicRandom()->randomUniqueID();
 				g_traceBatch.addAttach("TransactionAttachID", req.debugID.get().first(), tlogDebugID.first());
@@ -2855,7 +2912,11 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 			else
 				req.reply.sendError(tlog_stopped());
 		}
-		when(TLogDisablePopRequest req = waitNext(tli.disablePopRequest.getFuture())) {
+	}
+
+	Future<Void> serveDisablePop() {
+		while (true) {
+			TLogDisablePopRequest req = co_await tli.disablePopRequest.getFuture();
 			if (self->ignorePopUid != "") {
 				TraceEvent(SevWarn, "TLogPopDisableonDisable")
 				    .detail("IgnorePopUid", self->ignorePopUid)
@@ -2872,13 +2933,45 @@ ACTOR Future<Void> serveTLogInterface(TLogData* self,
 				req.reply.send(Void());
 			}
 		}
-		when(TLogEnablePopRequest enablePopReq = waitNext(tli.enablePopRequest.getFuture())) {
+	}
+
+	Future<Void> serveEnablePop() {
+		while (true) {
+			TLogEnablePopRequest enablePopReq = co_await tli.enablePopRequest.getFuture();
 			logData->addActor.send(tLogEnablePopReq(enablePopReq, self, logData));
 		}
 	}
+
+public:
+	ServeTLogInterface(TLogData* self,
+	                   TLogInterface tli,
+	                   Reference<LogData> logData,
+	                   PromiseStream<Void> warningCollectorInput)
+	  : self(self), tli(tli), logData(logData), warningCollectorInput(warningCollectorInput), dbInfoChange(Void()) {}
+
+	Future<Void> run() {
+		co_await race(updateDbInfo(),
+		              servePeekStreams(),
+		              servePeeks(),
+		              servePops(),
+		              serveCommits(),
+		              serveLocks(),
+		              serveQueuingMetrics(),
+		              serveConfirmRunning(),
+		              serveDisablePop(),
+		              serveEnablePop());
+	}
+};
+
+Future<Void> serveTLogInterface(TLogData* self,
+                                TLogInterface tli,
+                                Reference<LogData> logData,
+                                PromiseStream<Void> warningCollectorInput) {
+	ServeTLogInterface server(self, tli, logData, warningCollectorInput);
+	co_await server.run();
 }
 
-void removeLog(TLogData* self, Reference<LogData> logData) {
+void removeLog(TLogData* self, Reference<LogData> logData, bool terminateWorkerIfEmpty = true) {
 	TraceEvent("TLogRemoved", self->dbgid)
 	    .detail("LogId", logData->logId)
 	    .detail("Input", logData->bytesInput.getValue())
@@ -2896,29 +2989,84 @@ void removeLog(TLogData* self, Reference<LogData> logData) {
 	while (self->popOrder.size() && !self->id_data.contains(self->popOrder.front())) {
 		self->popOrder.pop_front();
 	}
-
-	if (self->id_data.size() == 0) {
-		throw worker_removed();
-	}
 	if (logData->queueCommittingVersion == 0) {
 		// If the removed tlog never attempted a queue commit, the update storage loop could become stuck waiting for
 		// queueCommittedVersion to advance.
 		logData->queueCommittedVersion.set(std::numeric_limits<Version>::max());
 	}
+	if (self->id_data.size() == 0 && terminateWorkerIfEmpty) {
+		throw worker_removed();
+	}
+}
+
+static void throwLowDiskTLogRecoveryFailed(TLogData* self,
+                                           Reference<LogData> logData,
+                                           Version ver,
+                                           double minAvailableSpaceRatio,
+                                           StorageBytes kvStoreBytes,
+                                           StorageBytes queueBytes) {
+	if (!self->lowDiskTLogExclusion->get()) {
+		self->lowDiskTLogExclusion->set(true);
+	}
+	TraceEvent(SevWarnAlways, "TLogPullAsyncDataLowDiskSpace", logData->logId)
+	    .detail("MinAvailableSpaceRatio", minAvailableSpaceRatio)
+	    .detail("AvailableSpaceRatio", self->availableSpaceRatio(kvStoreBytes, queueBytes))
+	    .detail("KvstoreBytesAvailable", kvStoreBytes.available)
+	    .detail("KvstoreBytesTotal", kvStoreBytes.total)
+	    .detail("QueueDiskBytesAvailable", queueBytes.available)
+	    .detail("QueueDiskBytesTotal", queueBytes.total)
+	    .detail("Version", ver)
+	    .detail("Action", "FailRecoveryAndRecruitNewTLogs");
+	CODE_PROBE(true, "pullAsyncData failed recovery due to TLOG_MIN_AVAILABLE_SPACE_RATIO");
+	throw recruitment_failed();
+}
+
+double effectiveTLogMinAvailableSpaceRatio() {
+	if (g_network->isSimulated() && g_simulator->speedUpSimulation) {
+		return 0.0;
+	}
+	return SERVER_KNOBS->TLOG_MIN_AVAILABLE_SPACE_RATIO;
+}
+
+static void failIfTLogCannotAcceptNewData(TLogData* self, Reference<LogData> logData, Version ver) {
+	StorageBytes kvStoreBytes = self->persistentData->getStorageBytes();
+	StorageBytes queueBytes = self->rawPersistentQueue->getStorageBytes();
+	const double minAvailableSpaceRatio = effectiveTLogMinAvailableSpaceRatio();
+	if (self->shouldAcceptNewData(kvStoreBytes, queueBytes, minAvailableSpaceRatio)) {
+		return;
+	}
+	if (g_network->isSimulated() && !g_simulator->speedUpSimulation) {
+		TraceEvent(SevWarnAlways, "TLogPullAsyncDataLowDiskSpeedUpSimulation", logData->logId)
+		    .detail("MinAvailableSpaceRatio", minAvailableSpaceRatio)
+		    .detail("AvailableSpaceRatio", self->availableSpaceRatio(kvStoreBytes, queueBytes))
+		    .detail("KvstoreBytesAvailable", kvStoreBytes.available)
+		    .detail("KvstoreBytesTotal", kvStoreBytes.total)
+		    .detail("QueueDiskBytesAvailable", queueBytes.available)
+		    .detail("QueueDiskBytesTotal", queueBytes.total)
+		    .detail("Version", ver);
+		g_simulator->speedUpSimulation = true;
+		if (self->shouldAcceptNewData(kvStoreBytes, queueBytes, effectiveTLogMinAvailableSpaceRatio())) {
+			return;
+		}
+	}
+	CODE_PROBE(true, "pullAsyncData blocked by TLOG_MIN_AVAILABLE_SPACE_RATIO");
+	// Outside speedUpSimulation, fail recovery and temporarily exclude this worker from TLog recruitment until disk
+	// space recovers.
+	throwLowDiskTLogRecoveryFailed(self, logData, ver, minAvailableSpaceRatio, kvStoreBytes, queueBytes);
 }
 
 // remote tLog pull data from log routers
-ACTOR Future<Void> pullAsyncData(TLogData* self,
-                                 Reference<LogData> logData,
-                                 std::vector<Tag> tags,
-                                 Version beginVersion,
-                                 Optional<Version> endVersion,
-                                 bool poppedIsKnownCommitted) {
-	state Future<Void> dbInfoChange = Void();
-	state Reference<ILogSystem::IPeekCursor> r;
-	state Version tagAt = beginVersion;
-	state Version lastVer = 0;
-	state double startTime = now();
+Future<Void> pullAsyncData(TLogData* self,
+                           Reference<LogData> logData,
+                           std::vector<Tag> tags,
+                           Version beginVersion,
+                           Optional<Version> endVersion,
+                           bool poppedIsKnownCommitted) {
+	Future<Void> dbInfoChange = Void();
+	Reference<IPeekCursor> r;
+	Version tagAt = beginVersion;
+	Version lastVer = 0;
+	double startTime = now();
 
 	if (endVersion.present()) {
 		TraceEvent("TLogRestoreReplicationFactor", self->dbgid)
@@ -2930,32 +3078,31 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 
 	while (!endVersion.present() || logData->version.get() < endVersion.get()) {
 		// When we just processed some data, we reset the warning start time.
-		state double lastPullAsyncDataWarningTime = now();
-		loop {
+		double lastPullAsyncDataWarningTime = now();
+		while (true) {
 			double waitTime = std::max(
 			    0.0, lastPullAsyncDataWarningTime + SERVER_KNOBS->TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS - now());
-			choose {
-				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
-					break;
-				}
-				when(wait(dbInfoChange)) {
-					if (logData->logSystem->get()) {
-						r = logData->logSystem->get()->peek(logData->logId, tagAt, endVersion, tags, true);
-					} else {
-						r = Reference<ILogSystem::IPeekCursor>();
-					}
-					dbInfoChange = logData->logSystem->onChange();
-				}
-				when(wait(delay(waitTime))) {
-					TraceEvent(SevWarn, "TLogPullAsyncDataSlow", logData->logId)
-					    .detail("Elapsed", now() - startTime)
-					    .detail("Version", logData->version.get());
-					lastPullAsyncDataWarningTime = now();
+			Future<Void> getMoreFuture = r ? r->getMore(TaskPriority::TLogCommit) : Never();
+			auto res = co_await race(getMoreFuture, dbInfoChange, delay(waitTime));
+			if (res.index() == 0) {
+				break;
+			} else if (res.index() == 1) {
+				if (logData->logSystem->get()) {
+					r = logData->logSystem->get()->peek(logData->logId, tagAt, endVersion, tags, true);
+				} else {
+					r = Reference<IPeekCursor>();
 				}
+				dbInfoChange = logData->logSystem->onChange();
+			} else {
+				ASSERT(res.index() == 2);
+				TraceEvent(SevWarn, "TLogPullAsyncDataSlow", logData->logId)
+				    .detail("Elapsed", now() - startTime)
+				    .detail("Version", logData->version.get());
+				lastPullAsyncDataWarningTime = now();
 			}
 		}
 
-		state double waitStartT = 0;
+		double waitStartT = 0;
 		while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped()) {
 			if (now() - waitStartT >= 1) {
 				TraceEvent(SevWarn, "TLogUpdateLag", logData->logId)
@@ -2964,18 +3111,18 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 				    .detail("PersistentDataDurableVersion", logData->persistentDataDurableVersion);
 				waitStartT = now();
 			}
-			wait(delayJittered(.005, TaskPriority::TLogCommit));
+			co_await delayJittered(.005, TaskPriority::TLogCommit);
 		}
 
-		state Version ver = 0;
-		state std::vector<TagsAndMessage> messages;
-		loop {
-			state bool foundMessage = r->hasMessage();
+		Version ver = 0;
+		std::vector<TagsAndMessage> messages;
+		while (true) {
+			bool foundMessage = r->hasMessage();
 			if (!foundMessage || r->version().version != ver) {
 				ASSERT(r->version().version > lastVer);
 				if (ver) {
 					if (logData->stopped() || (endVersion.present() && ver > endVersion.get())) {
-						return Void();
+						co_return;
 					}
 
 					if (poppedIsKnownCommitted) {
@@ -2984,10 +3131,15 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 						    std::max(logData->minKnownCommittedVersion, r->getMinKnownCommittedVersion());
 					}
 
+					failIfTLogCannotAcceptNewData(self, logData, ver);
+					if (logData->stopped()) {
+						co_return;
+					}
+
 					commitMessages(self, logData, ver, messages);
 
 					if (self->terminated.isSet()) {
-						return Void();
+						co_return;
 					}
 
 					// Log the changes to the persistent queue, to be committed by commitQueue()
@@ -3006,7 +3158,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 					// Notifies the commitQueue actor to commit persistentQueue, and also unblocks tLogPeekMessages
 					// actors
 					logData->version.set(ver);
-					wait(yield(TaskPriority::TLogCommit));
+					co_await yield(TaskPriority::TLogCommit);
 				}
 				lastVer = ver;
 				ver = r->version().version;
@@ -3016,7 +3168,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 					ver--;
 					if (ver > logData->version.get()) {
 						if (logData->stopped() || (endVersion.present() && ver > endVersion.get())) {
-							return Void();
+							co_return;
 						}
 
 						if (poppedIsKnownCommitted) {
@@ -3025,8 +3177,13 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 							    std::max(logData->minKnownCommittedVersion, r->getMinKnownCommittedVersion());
 						}
 
+						failIfTLogCannotAcceptNewData(self, logData, ver);
+						if (logData->stopped()) {
+							co_return;
+						}
+
 						if (self->terminated.isSet()) {
-							return Void();
+							co_return;
 						}
 
 						// Log the changes to the persistent queue, to be committed by commitQueue()
@@ -3045,7 +3202,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 						// Notifies the commitQueue actor to commit persistentQueue, and also unblocks tLogPeekMessages
 						// actors
 						logData->version.set(ver);
-						wait(yield(TaskPriority::TLogCommit));
+						co_await yield(TaskPriority::TLogCommit);
 					}
 					break;
 				}
@@ -3057,15 +3214,11 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 
 		tagAt = std::max(r->version().version, logData->version.get() + 1);
 	}
-	return Void();
 }
 
-ACTOR Future<Void> tLogCore(TLogData* self,
-                            Reference<LogData> logData,
-                            TLogInterface tli,
-                            bool pulledRecoveryVersions) {
+Future<Void> tLogCore(TLogData* self, Reference<LogData> logData, TLogInterface tli, bool pulledRecoveryVersions) {
 	if (logData->removed.isReady()) {
-		wait(delay(0)); // to avoid iterator invalidation in restorePersistentState when removed is already ready
+		co_await delay(0); // to avoid iterator invalidation in restorePersistentState when removed is already ready
 		ASSERT(logData->removed.isError());
 
 		if (logData->removed.getError().code() != error_code_worker_removed) {
@@ -3073,13 +3226,13 @@ ACTOR Future<Void> tLogCore(TLogData* self,
 		}
 
 		removeLog(self, logData);
-		return Void();
+		co_return;
 	}
 
-	state PromiseStream<Void> warningCollectorInput;
-	state Future<Void> warningCollector =
+	PromiseStream<Void> warningCollectorInput;
+	[[maybe_unused]] Future<Void> warningCollector =
 	    timeoutWarningCollector(warningCollectorInput.getFuture(), 1.0, "TLogQueueCommitSlow", self->dbgid);
-	state Future<Void> error = actorCollection(logData->addActor.getFuture());
+	Future<Void> error = actorCollection(logData->addActor.getFuture());
 
 	logData->addActor.send(waitFailureServer(tli.waitFailure.getFuture()));
 	logData->addActor.send(logData->removed);
@@ -3122,85 +3275,79 @@ ACTOR Future<Void> tLogCore(TLogData* self,
 	}
 
 	try {
-		wait(error);
+		co_await error;
 		throw internal_error();
 	} catch (Error& e) {
-		if (e.code() != error_code_worker_removed)
+		if (e.code() != error_code_worker_removed && e.code() != error_code_recruitment_failed)
 			throw;
 
-		removeLog(self, logData);
-		return Void();
+		removeLog(self, logData, e.code() != error_code_recruitment_failed);
+		co_return;
 	}
 }
 
-ACTOR Future<Void> checkEmptyQueue(TLogData* self) {
+Future<Void> checkEmptyQueue(TLogData* self) {
 	TraceEvent("TLogCheckEmptyQueueBegin", self->dbgid).log();
 	try {
-		bool recoveryFinished = wait(self->persistentQueue->initializeRecovery(0));
+		bool recoveryFinished = co_await self->persistentQueue->initializeRecovery(0);
 		if (recoveryFinished)
-			return Void();
-		TLogQueueEntry r = wait(self->persistentQueue->readNext(self));
+			co_return;
+		TLogQueueEntry r = co_await self->persistentQueue->readNext(self);
 		throw internal_error();
 	} catch (Error& e) {
 		if (e.code() != error_code_end_of_stream)
 			throw;
 		TraceEvent("TLogCheckEmptyQueueEnd", self->dbgid).log();
-		return Void();
+		co_return;
 	}
 }
 
-ACTOR Future<Void> initPersistentStorage(TLogData* self) {
+Future<Void> initPersistentStorage(TLogData* self) {
 	TraceEvent("TLogInitPersistentStorageStart", self->dbgid);
 
-	wait(self->persistentDataCommitLock.take());
-	state FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
+	co_await self->persistentDataCommitLock.take();
+	FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
 
 	// PERSIST: Initial setup of persistentData for a brand new tLog for a new database
-	state IKeyValueStore* storage = self->persistentData;
+	IKeyValueStore* storage = self->persistentData;
 	storage->set(persistFormat);
 
-	wait(storage->commit());
+	co_await storage->commit();
 
 	TraceEvent("TLogInitPersistentStorageDone", self->dbgid);
-	return Void();
 }
 
 // Recovery persistent state of tLog from disk
-ACTOR Future<Void> restorePersistentState(TLogData* self,
-                                          LocalityData locality,
-                                          Promise<Void> oldLog,
-                                          Promise<Void> recovered,
-                                          PromiseStream<InitializeTLogRequest> tlogRequests) {
-	state double startt = now();
-	state Reference<LogData> logData;
-	state KeyRange tagKeys;
+Future<Void> restorePersistentState(TLogData* self,
+                                    LocalityData locality,
+                                    Promise<Void> oldLog,
+                                    Promise<Void> recovered,
+                                    PromiseStream<InitializeTLogRequest> tlogRequests) {
+	double startt = now();
+	Reference<LogData> logData;
 	// PERSIST: Read basic state from persistentData; replay persistentQueue but don't erase it
 
 	TraceEvent("TLogRestorePersistentState", self->dbgid).log();
 
-	state IKeyValueStore* storage = self->persistentData;
-	state Future<Optional<Value>> fFormat = storage->readValue(persistFormat.key);
-	state Future<Optional<Value>> fRecoveryLocation = storage->readValue(persistRecoveryLocationKey);
-	state Future<RangeResult> fVers = storage->readRange(persistCurrentVersionKeys);
-	state Future<RangeResult> fKnownCommitted = storage->readRange(persistKnownCommittedVersionKeys);
-	state Future<RangeResult> fLocality = storage->readRange(persistLocalityKeys);
-	state Future<RangeResult> fLogRouterTags = storage->readRange(persistLogRouterTagsKeys);
-	state Future<RangeResult> fTxsTags = storage->readRange(persistTxsTagsKeys);
-	state Future<RangeResult> fRecoverCounts = storage->readRange(persistRecoveryCountKeys);
-	state Future<RangeResult> fProtocolVersions = storage->readRange(persistProtocolVersionKeys);
-	state Future<RangeResult> fTLogSpillTypes = storage->readRange(persistTLogSpillTypeKeys);
+	IKeyValueStore* storage = self->persistentData;
+	Future<Optional<Value>> fFormat = storage->readValue(persistFormat.key);
+	Future<Optional<Value>> fRecoveryLocation = storage->readValue(persistRecoveryLocationKey);
+	Future<RangeResult> fVers = storage->readRange(persistCurrentVersionKeys);
+	Future<RangeResult> fKnownCommitted = storage->readRange(persistKnownCommittedVersionKeys);
+	Future<RangeResult> fLocality = storage->readRange(persistLocalityKeys);
+	Future<RangeResult> fLogRouterTags = storage->readRange(persistLogRouterTagsKeys);
+	Future<RangeResult> fTxsTags = storage->readRange(persistTxsTagsKeys);
+	Future<RangeResult> fRecoverCounts = storage->readRange(persistRecoveryCountKeys);
+	Future<RangeResult> fProtocolVersions = storage->readRange(persistProtocolVersionKeys);
+	Future<RangeResult> fTLogSpillTypes = storage->readRange(persistTLogSpillTypeKeys);
 
 	// FIXME: metadata in queue?
 
-	wait(waitForAll(std::vector{ fFormat, fRecoveryLocation }));
-	wait(waitForAll(std::vector{ fVers,
-	                             fKnownCommitted,
-	                             fLocality,
-	                             fLogRouterTags,
-	                             fTxsTags,
-	                             fRecoverCounts,
-	                             fProtocolVersions,
-	                             fTLogSpillTypes }));
+	std::vector<Future<Optional<Value>>> pointFutures = { fFormat, fRecoveryLocation };
+	co_await waitForAll(pointFutures);
+	std::vector<Future<RangeResult>> rangeFutures = { fVers,    fKnownCommitted, fLocality,         fLogRouterTags,
+		                                              fTxsTags, fRecoverCounts,  fProtocolVersions, fTLogSpillTypes };
+	co_await waitForAll(rangeFutures);
 
 	if (fFormat.get().present() && !persistFormatReadableRange.contains(fFormat.get().get())) {
 		// FIXME: remove when we no longer need to test upgrades from 4.X releases
@@ -3216,7 +3363,7 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 
 	if (!fFormat.get().present()) {
-		RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), "\xff"_sr), 1));
+		RangeResult v = co_await self->persistentData->readRange(KeyRangeRef(StringRef(), "\xff"_sr), 1);
 		if (!v.size()) {
 			CODE_PROBE(true, "The DB is completely empty, so it was never initialized.  Delete it.");
 			throw worker_removed();
@@ -3228,50 +3375,49 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 		}
 	}
 
-	state std::vector<Future<ErrorOr<Void>>> removed;
+	std::vector<Future<ErrorOr<Void>>> removed;
 
 	ASSERT(fFormat.get().get() == "FoundationDB/LogServer/3/0"_sr);
 
 	ASSERT(fVers.get().size() == fRecoverCounts.get().size());
 
-	state std::map<UID, int8_t> id_locality;
+	std::map<UID, int8_t> id_locality;
 	for (auto it : fLocality.get()) {
 		id_locality[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistLocalityKeys.begin), Unversioned())] =
 		    BinaryReader::fromStringRef<int8_t>(it.value, Unversioned());
 	}
 
-	state std::map<UID, int> id_logRouterTags;
+	std::map<UID, int> id_logRouterTags;
 	for (auto it : fLogRouterTags.get()) {
 		id_logRouterTags[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistLogRouterTagsKeys.begin),
 		                                                  Unversioned())] =
 		    BinaryReader::fromStringRef<int>(it.value, Unversioned());
 	}
 
-	state std::map<UID, int> id_txsTags;
+	std::map<UID, int> id_txsTags;
 	for (auto it : fTxsTags.get()) {
 		id_txsTags[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistTxsTagsKeys.begin), Unversioned())] =
 		    BinaryReader::fromStringRef<int>(it.value, Unversioned());
 	}
 
-	state std::map<UID, Version> id_knownCommitted;
+	std::map<UID, Version> id_knownCommitted;
 	for (auto it : fKnownCommitted.get()) {
 		id_knownCommitted[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistKnownCommittedVersionKeys.begin),
 		                                                   Unversioned())] =
 		    BinaryReader::fromStringRef<Version>(it.value, Unversioned());
 	}
 
-	state IDiskQueue::location minimumRecoveryLocation = 0;
+	IDiskQueue::location minimumRecoveryLocation = 0;
 	if (fRecoveryLocation.get().present()) {
 		minimumRecoveryLocation =
 		    BinaryReader::fromStringRef<IDiskQueue::location>(fRecoveryLocation.get().get(), Unversioned());
 	}
 
-	state int idx = 0;
-	state Promise<Void> registerWithCC;
-	state std::map<UID, TLogInterface> id_interf;
-	state std::vector<std::pair<Version, UID>> logsByVersion;
-	for (idx = 0; idx < fVers.get().size(); idx++) {
-		state KeyRef rawId = fVers.get()[idx].key.removePrefix(persistCurrentVersionKeys.begin);
+	Promise<Void> registerWithCC;
+	std::map<UID, TLogInterface> id_interf;
+	std::vector<std::pair<Version, UID>> logsByVersion;
+	for (int idx = 0; idx < fVers.get().size(); idx++) {
+		KeyRef rawId = fVers.get()[idx].key.removePrefix(persistCurrentVersionKeys.begin);
 		UID id1 = BinaryReader::fromStringRef<UID>(rawId, Unversioned());
 		UID id2 = BinaryReader::fromStringRef<UID>(
 		    fRecoverCounts.get()[idx].key.removePrefix(persistRecoveryCountKeys.begin), Unversioned());
@@ -3293,10 +3439,10 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 		DUMPTOKEN(recruited.enablePopRequest);
 		DUMPTOKEN(recruited.snapRequest);
 
-		ProtocolVersion protocolVersion =
+		auto protocolVersion =
 		    BinaryReader::fromStringRef<ProtocolVersion>(fProtocolVersions.get()[idx].value, Unversioned());
-		TLogSpillType logSpillType = BinaryReader::fromStringRef<TLogSpillType>(fTLogSpillTypes.get()[idx].value,
-		                                                                        AssumeVersion(protocolVersion));
+		auto logSpillType = BinaryReader::fromStringRef<TLogSpillType>(fTLogSpillTypes.get()[idx].value,
+		                                                               AssumeVersion(protocolVersion));
 
 		// We do not need the remoteTag, because we will not be loading any additional data
 		logData = makeReference<LogData>(self,
@@ -3334,11 +3480,11 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 		    .detail("RecoveryCount", logData->recoveryCount);
 		// Restore popped keys.  Pop operations that took place after the last (committed) updatePersistentDataVersion
 		// might be lost, but that is fine because we will get the corresponding data back, too.
-		tagKeys = prefixRange(rawId.withPrefix(persistTagPoppedKeys.begin));
-		loop {
+		KeyRange tagKeys = prefixRange(rawId.withPrefix(persistTagPoppedKeys.begin));
+		while (true) {
 			if (logData->removed.isReady())
 				break;
-			RangeResult data = wait(self->persistentData->readRange(tagKeys, BUGGIFY ? 3 : 1 << 30, 1 << 20));
+			RangeResult data = co_await self->persistentData->readRange(tagKeys, BUGGIFY ? 3 : 1 << 30, 1 << 20);
 			if (!data.size())
 				break;
 			((KeyRangeRef&)tagKeys) = KeyRangeRef(keyAfter(data.back().key, tagKeys.arena()), tagKeys.end);
@@ -3349,7 +3495,8 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 				TraceEvent("TLogRestorePopped", logData->logId).detail("Tag", tag.toString()).detail("To", popped);
 				auto tagData = logData->getTagData(tag);
 				ASSERT(!tagData);
-				logData->createTagData(tag, popped, false, false, false);
+				logData->createTagData(
+				    tag, popped, NothingPersistent::False, PoppedRecently::False, UnpoppedRecovered::False);
 				logData->getTagData(tag)->persistentPopped = popped;
 			}
 		}
@@ -3362,76 +3509,75 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 	}
 	logsByVersion.clear();
 
-	state Future<Void> allRemoved = waitForAll(removed);
-	state UID lastId = UID(1, 1); // initialized so it will not compare equal to a default UID
-	state double recoverMemoryLimit = SERVER_KNOBS->TLOG_RECOVER_MEMORY_LIMIT;
+	Future<Void> allRemoved = waitForAll(removed);
+	UID lastId = UID(1, 1); // initialized so it will not compare equal to a default UID
+	double recoverMemoryLimit = SERVER_KNOBS->TLOG_RECOVER_MEMORY_LIMIT;
 	if (BUGGIFY)
 		recoverMemoryLimit =
 		    std::max<double>(SERVER_KNOBS->BUGGIFY_RECOVER_MEMORY_LIMIT, (double)SERVER_KNOBS->TLOG_SPILL_THRESHOLD);
 
 	try {
-		bool recoveryFinished = wait(self->persistentQueue->initializeRecovery(minimumRecoveryLocation));
+		bool recoveryFinished = co_await self->persistentQueue->initializeRecovery(minimumRecoveryLocation);
 		if (recoveryFinished)
 			throw end_of_stream();
-		loop {
+		while (true) {
 			if (allRemoved.isReady()) {
 				CODE_PROBE(true, "all tlogs removed during queue recovery");
 				throw worker_removed();
 			}
-			choose {
-				when(TLogQueueEntry qe = wait(self->persistentQueue->readNext(self))) {
-					if (qe.id != lastId) {
-						lastId = qe.id;
-						auto it = self->id_data.find(qe.id);
-						if (it != self->id_data.end()) {
-							logData = it->second;
-						} else {
-							logData = Reference<LogData>();
-						}
+			auto res = co_await race(self->persistentQueue->readNext(self), allRemoved);
+			if (res.index() == 0) {
+				TLogQueueEntry qe = std::get<0>(std::move(res));
+
+				if (qe.id != lastId) {
+					lastId = qe.id;
+					auto it = self->id_data.find(qe.id);
+					if (it != self->id_data.end()) {
+						logData = it->second;
+					} else {
+						logData = Reference<LogData>();
 					}
+				}
 
-					//TraceEvent("TLogRecoveredQE", self->dbgid).detail("LogId", qe.id).detail("Ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size())
-					//	.detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("Version",
-					// logData->version.get());
+				//TraceEvent("TLogRecoveredQE", self->dbgid).detail("LogId", qe.id).detail("Ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size())
+				//	.detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("Version",
+				// logData->version.get());
 
-					if (logData) {
-						if (!self->spillOrder.size() || self->spillOrder.back() != qe.id) {
-							self->spillOrder.push_back(qe.id);
-						}
-						logData->knownCommittedVersion =
-						    std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
-						if (qe.version > logData->version.get()) {
-							commitMessages(self, logData, qe.version, qe.arena(), qe.messages);
-							logData->version.set(qe.version);
-							logData->queueCommittedVersion.set(qe.version);
-
-							while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
-								CODE_PROBE(true, "Flush excess data during TLog queue recovery");
-								TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
-								    .detail("LogId", logData->logId)
-								    .detail("BytesInput", self->bytesInput)
-								    .detail("BytesDurable", self->bytesDurable)
-								    .detail("Version", logData->version.get())
-								    .detail("PVer", logData->persistentDataVersion);
-
-								choose {
-									when(wait(updateStorage(self))) {}
-									when(wait(allRemoved)) {
-										throw worker_removed();
-									}
-								}
+				if (logData) {
+					if (!self->spillOrder.size() || self->spillOrder.back() != qe.id) {
+						self->spillOrder.push_back(qe.id);
+					}
+					logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
+					if (qe.version > logData->version.get()) {
+						commitMessages(self, logData, qe.version, qe.arena(), qe.messages);
+						logData->version.set(qe.version);
+						logData->queueCommittedVersion.set(qe.version);
+
+						while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
+							CODE_PROBE(true, "Flush excess data during TLog queue recovery");
+							TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
+							    .detail("LogId", logData->logId)
+							    .detail("BytesInput", self->bytesInput)
+							    .detail("BytesDurable", self->bytesDurable)
+							    .detail("Version", logData->version.get())
+							    .detail("PVer", logData->persistentDataVersion);
+
+							auto flushRes = co_await race(updateStorage(self), allRemoved);
+							if (flushRes.index() == 1) {
+								throw worker_removed();
 							}
-						} else {
-							// Updating persistRecoveryLocation and persistCurrentVersion at the same time,
-							// transactionally, should mean that we never read any TLogQueueEntry that has already
-							// been spilled.
-							ASSERT_WE_THINK(qe.version == logData->version.get());
+							ASSERT(flushRes.index() == 0);
 						}
+					} else {
+						// Updating persistRecoveryLocation and persistCurrentVersion at the same time,
+						// transactionally, should mean that we never read any TLogQueueEntry that has already
+						// been spilled.
+						ASSERT_WE_THINK(qe.version == logData->version.get());
 					}
 				}
-				when(wait(allRemoved)) {
-					throw worker_removed();
-				}
+			} else {
+				ASSERT(res.index() == 1);
+				throw worker_removed();
 			}
 		}
 	} catch (Error& e) {
@@ -3453,7 +3599,6 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 	if (registerWithCC.canBeSet())
 		registerWithCC.send(Void());
-	return Void();
 }
 
 bool tlogTerminated(TLogData* self, IKeyValueStore* persistentData, TLogQueue* persistentQueue, Error const& e) {
@@ -3475,28 +3620,28 @@ bool tlogTerminated(TLogData* self, IKeyValueStore* persistentData, TLogQueue* p
 		return false;
 }
 
-ACTOR Future<Void> updateLogSystem(TLogData* self,
-                                   Reference<LogData> logData,
-                                   LogSystemConfig recoverFrom,
-                                   Reference<AsyncVar<Reference<ILogSystem>>> logSystem) {
-	loop {
+Future<Void> updateLogSystem(TLogData* self,
+                             Reference<LogData> logData,
+                             LogSystemConfig recoverFrom,
+                             Reference<AsyncVar<Reference<LogSystem>>> logSystem) {
+	while (true) {
 		bool found = self->dbInfo->get().logSystemConfig.recruitmentID == logData->recruitmentID;
 		if (found) {
 			if (self->dbInfo->get().logSystemConfig.isNextGenerationOf(recoverFrom)) {
-				logSystem->set(ILogSystem::fromOldLogSystemConfig(
+				logSystem->set(makeOldLogSystemFromLogSystemConfig(
 				    logData->logId, self->dbInfo->get().myLocality, self->dbInfo->get().logSystemConfig));
 			} else if (self->dbInfo->get().logSystemConfig.isEqualIds(recoverFrom)) {
-				logSystem->set(ILogSystem::fromLogSystemConfig(
+				logSystem->set(makeLogSystemFromLogSystemConfig(
 				    logData->logId, self->dbInfo->get().myLocality, self->dbInfo->get().logSystemConfig, false, true));
 			} else if (self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
-				logSystem->set(ILogSystem::fromLogSystemConfig(
+				logSystem->set(makeLogSystemFromLogSystemConfig(
 				    logData->logId, self->dbInfo->get().myLocality, self->dbInfo->get().logSystemConfig, true));
 			} else {
 				found = false;
 			}
 		}
 		if (!found) {
-			logSystem->set(Reference<ILogSystem>());
+			logSystem->set(Reference<LogSystem>());
 		} else {
 			logData->logSystem->get()->pop(logData->logRouterPoppedVersion,
 			                               logData->remoteTag,
@@ -3515,7 +3660,7 @@ ACTOR Future<Void> updateLogSystem(TLogData* self,
 		for (const auto& it : self->dbInfo->get().logSystemConfig.oldTLogs) {
 			TraceEvent("TLogUpdateOld", self->dbgid).detail("LogId", logData->logId).detail("DbInfo", it.toString());
 		}
-		wait(self->dbInfo->onChange());
+		co_await self->dbInfo->onChange();
 	}
 }
 
@@ -3544,8 +3689,8 @@ void stopAllTLogs(TLogData* self, UID newLogId) {
 }
 
 // Start the tLog role for a worker
-ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, LocalityData locality) {
-	state TLogInterface recruited(self->dbgid, locality);
+Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, LocalityData locality) {
+	TLogInterface recruited(self->dbgid, locality);
 	recruited.initEndpoints();
 
 	DUMPTOKEN(recruited.peekMessages);
@@ -3565,17 +3710,17 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	stopAllTLogs(self, recruited.id());
 
 	bool recovering = (req.recoverFrom.logSystemType == LogSystemType::tagPartitioned);
-	state Reference<LogData> logData = makeReference<LogData>(self,
-	                                                          recruited,
-	                                                          req.remoteTag,
-	                                                          req.isPrimary,
-	                                                          req.logRouterTags,
-	                                                          req.txsTags,
-	                                                          req.recruitmentID,
-	                                                          g_network->protocolVersion(),
-	                                                          req.spillType,
-	                                                          req.allTags,
-	                                                          recovering ? "Recovered" : "Recruited");
+	auto logData = makeReference<LogData>(self,
+	                                      recruited,
+	                                      req.remoteTag,
+	                                      req.isPrimary,
+	                                      req.logRouterTags,
+	                                      req.txsTags,
+	                                      req.recruitmentID,
+	                                      g_network->protocolVersion(),
+	                                      req.spillType,
+	                                      req.allTags,
+	                                      recovering ? "Recovered" : "Recruited");
 	self->id_data[recruited.id()] = logData;
 	logData->locality = req.locality;
 	logData->recoveryCount = req.epoch;
@@ -3590,8 +3735,9 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	    .detail("RecoveryTxnVersion", logData->recoveryTxnVersion)
 	    .detail("IsPrimary", req.isPrimary);
 
-	state Future<Void> updater;
-	state bool pulledRecoveryVersions = false;
+	Future<Void> updater;
+	bool pulledRecoveryVersions = false;
+	Optional<Error> err;
 	try {
 		if (logData->removed.isReady()) {
 			throw logData->removed.getError();
@@ -3599,7 +3745,7 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 
 		if (recovering) {
 			logData->unrecoveredBefore = req.startVersion;
-			state Version recoverAt = req.recoverAt;
+			Version recoverAt = req.recoverAt;
 			logData->recoveredAt = recoverAt;
 			logData->knownCommittedVersion = req.startVersion - 1;
 			logData->persistentDataVersion = logData->unrecoveredBefore - 1;
@@ -3614,9 +3760,9 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 					logData->tagUnpoppedOldGenerations.emplace(tag, req.oldGenerationRecoverAtVersions);
 				}
 			}
-			wait(ioTimeoutError(initPersistentState(self, logData) || logData->removed,
-			                    SERVER_KNOBS->TLOG_MAX_CREATE_DURATION,
-			                    "TLogInit"));
+			co_await ioTimeoutError(initPersistentState(self, logData) || logData->removed,
+			                        SERVER_KNOBS->TLOG_MAX_CREATE_DURATION,
+			                        "TLogInit");
 
 			TraceEvent("TLogRecover", self->dbgid)
 			    .detail("LogId", logData->logId)
@@ -3648,8 +3794,8 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 						// Force gray failure monitoring during recovery.
 						self->enablePrimaryTxnSystemHealthCheck->set(true);
 					}
-					wait(pullAsyncData(self, logData, tags, logData->unrecoveredBefore, recoverAt, true) ||
-					     logData->removed || logData->stopCommit.onTrigger());
+					co_await (pullAsyncData(self, logData, tags, logData->unrecoveredBefore, recoverAt, true) ||
+					          logData->removed || logData->stopCommit.onTrigger());
 					self->enablePrimaryTxnSystemHealthCheck->set(false);
 				} else if (!req.recoverTags.empty()) {
 					ASSERT(logData->unrecoveredBefore > req.knownCommittedVersion);
@@ -3658,16 +3804,16 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 						// Force gray failure monitoring during recovery.
 						self->enablePrimaryTxnSystemHealthCheck->set(true);
 					}
-					wait(pullAsyncData(
-					         self, logData, req.recoverTags, req.knownCommittedVersion + 1, recoverAt, false) ||
-					     logData->removed || logData->stopCommit.onTrigger());
+					co_await (pullAsyncData(
+					              self, logData, req.recoverTags, req.knownCommittedVersion + 1, recoverAt, false) ||
+					          logData->removed || logData->stopCommit.onTrigger());
 					self->enablePrimaryTxnSystemHealthCheck->set(false);
 				}
 				pulledRecoveryVersions = true;
 				logData->knownCommittedVersion = recoverAt;
 			}
 
-			state Version lastVersionPrevEpoch = req.recoverAt;
+			Version lastVersionPrevEpoch = req.recoverAt;
 
 			if ((req.isPrimary || req.recoverFrom.logRouterTags == 0) &&
 			    logData->version.get() < lastVersionPrevEpoch && !logData->stopped()) {
@@ -3694,9 +3840,9 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 			logData->addActor.send(respondToTrackRecovery(recruited, logData));
 		} else {
 			// Brand new tlog, initialization has already been done by caller
-			wait(ioTimeoutError(initPersistentState(self, logData) || logData->removed,
-			                    SERVER_KNOBS->TLOG_MAX_CREATE_DURATION,
-			                    "TLogInit"));
+			co_await ioTimeoutError(initPersistentState(self, logData) || logData->removed,
+			                        SERVER_KNOBS->TLOG_MAX_CREATE_DURATION,
+			                        "TLogInit");
 
 			if (logData->recoveryComplete.isSet()) {
 				throw worker_removed();
@@ -3707,19 +3853,24 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 
 			logData->recoveryComplete.send(Void());
 		}
-		wait(logData->committingQueue.getFuture() || logData->removed);
+		co_await (logData->committingQueue.getFuture() || logData->removed);
 	} catch (Error& e) {
+		err = e;
+	}
+	if (err.present()) {
 		req.reply.sendError(recruitment_failed());
 
-		if (e.code() != error_code_worker_removed) {
-			throw;
+		if (err.get().code() != error_code_worker_removed && err.get().code() != error_code_recruitment_failed) {
+			throw err.get();
 		}
 
-		wait(delay(0.0)); // if multiple recruitment requests were already in the promise stream make sure they are all
-		                  // started before any are removed
+		const bool removePersistentState = err.get().code() != error_code_recruitment_failed;
+		// If multiple recruitment requests were already in the promise stream, make sure they are all started
+		// before any are removed.
+		co_await delay(0.0);
 
-		removeLog(self, logData);
-		return Void();
+		removeLog(self, logData, removePersistentState);
+		co_return;
 	}
 
 	req.reply.send(recruited);
@@ -3731,12 +3882,11 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 	    .detail("AllTags", describe(req.allTags));
 
 	updater = Void();
-	wait(tLogCore(self, logData, recruited, pulledRecoveryVersions));
-	return Void();
+	co_await tLogCore(self, logData, recruited, pulledRecoveryVersions);
 }
 
-ACTOR Future<Void> startSpillingInTenSeconds(TLogData* self, UID tlogId, Reference<AsyncVar<UID>> activeSharedTLog) {
-	wait(delay(10));
+Future<Void> startSpillingInTenSeconds(TLogData* self, UID tlogId, Reference<AsyncVar<UID>> activeSharedTLog) {
+	co_await delay(10);
 	if (activeSharedTLog->get() != tlogId) {
 		// TODO: This should fully spill, but currently doing so will cause us to no longer update poppedVersion
 		// and QuietDatabase will hang thinking our TLog is behind.
@@ -3745,42 +3895,49 @@ ACTOR Future<Void> startSpillingInTenSeconds(TLogData* self, UID tlogId, Referen
 	} else {
 		TraceEvent("SharedTLogSkipSpilling", self->dbgid).detail("NowActive", activeSharedTLog->get());
 	}
-	return Void();
 }
 
 // New tLog (if !recoverFrom.size()) or restore from network
-ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
-                        IDiskQueue* persistentQueue,
-                        Reference<AsyncVar<ServerDBInfo> const> db,
-                        LocalityData locality,
-                        PromiseStream<InitializeTLogRequest> tlogRequests,
-                        UID tlogId,
-                        UID workerID,
-                        bool restoreFromDisk,
-                        Promise<Void> oldLog,
-                        Promise<Void> recovered,
-                        std::string folder,
-                        Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog,
-                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
-	state TLogData self(
-	    tlogId, workerID, persistentData, persistentQueue, db, degraded, folder, enablePrimaryTxnSystemHealthCheck);
-	state Future<Void> error = actorCollection(self.sharedActors.getFuture());
+Future<Void> tLog(IKeyValueStore* persistentData,
+                  IDiskQueue* persistentQueue,
+                  Reference<AsyncVar<ServerDBInfo> const> db,
+                  LocalityData locality,
+                  PromiseStream<InitializeTLogRequest> tlogRequests,
+                  UID tlogId,
+                  UID workerID,
+                  bool restoreFromDisk,
+                  Promise<Void> oldLog,
+                  Promise<Void> recovered,
+                  std::string folder,
+                  Reference<AsyncVar<bool>> degraded,
+                  Reference<AsyncVar<bool>> lowDiskTLogExclusion,
+                  Reference<AsyncVar<UID>> activeSharedTLog,
+                  Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
+	TLogData self(tlogId,
+	              workerID,
+	              persistentData,
+	              persistentQueue,
+	              db,
+	              degraded,
+	              lowDiskTLogExclusion,
+	              folder,
+	              enablePrimaryTxnSystemHealthCheck);
+	Future<Void> error = actorCollection(self.sharedActors.getFuture());
 
 	TraceEvent("SharedTLog", tlogId);
 	try {
-		wait(ioTimeoutError(persistentData->init(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION, "TLogInit"));
+		co_await ioTimeoutError(persistentData->init(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION, "TLogInit");
 
 		if (restoreFromDisk) {
-			wait(restorePersistentState(&self, locality, oldLog, recovered, tlogRequests));
+			co_await restorePersistentState(&self, locality, oldLog, recovered, tlogRequests);
 		} else {
-			wait(ioTimeoutError(checkEmptyQueue(&self) && initPersistentStorage(&self),
-			                    SERVER_KNOBS->TLOG_MAX_CREATE_DURATION,
-			                    "TLogInit"));
+			co_await ioTimeoutError(checkEmptyQueue(&self) && initPersistentStorage(&self),
+			                        SERVER_KNOBS->TLOG_MAX_CREATE_DURATION,
+			                        "TLogInit");
 		}
 
 		// Disk errors need a chance to kill this actor.
-		wait(delay(0.000001));
+		co_await delay(0.000001);
 
 		if (recovered.canBeSet()) {
 			recovered.send(Void());
@@ -3789,33 +3946,34 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 		self.sharedActors.send(commitQueue(&self));
 		self.sharedActors.send(updateStorageLoop(&self));
 		self.sharedActors.send(traceRole(Role::SHARED_TRANSACTION_LOG, tlogId));
-		state Future<Void> activeSharedChange = Void();
-
-		loop {
-			choose {
-				when(state InitializeTLogRequest req = waitNext(tlogRequests.getFuture())) {
-					if (!self.tlogCache.exists(req.recruitmentID)) {
-						self.tlogCache.set(req.recruitmentID, req.reply.getFuture());
-						self.sharedActors.send(
-						    self.tlogCache.removeOnReady(req.recruitmentID, tLogStart(&self, req, locality)));
-					} else {
-						forwardPromise(req.reply, self.tlogCache.get(req.recruitmentID));
-					}
-				}
-				when(wait(error)) {
-					throw internal_error();
+		Future<Void> activeSharedChange = Void();
+
+		while (true) {
+			auto res = co_await race(tlogRequests.getFuture(), error, activeSharedChange);
+			if (res.index() == 0) {
+				InitializeTLogRequest req = std::get<0>(std::move(res));
+
+				if (!self.tlogCache.exists(req.recruitmentID)) {
+					self.tlogCache.set(req.recruitmentID, req.reply.getFuture());
+					self.sharedActors.send(
+					    self.tlogCache.removeOnReady(req.recruitmentID, tLogStart(&self, req, locality)));
+				} else {
+					forwardPromise(Uncancellable{}, req.reply, self.tlogCache.get(req.recruitmentID));
 				}
-				when(wait(activeSharedChange)) {
-					if (activeSharedTLog->get() == tlogId) {
-						TraceEvent("SharedTLogNowActive", self.dbgid).detail("NowActive", activeSharedTLog->get());
-						self.targetVolatileBytes = SERVER_KNOBS->TLOG_SPILL_THRESHOLD;
-					} else {
-						stopAllTLogs(&self, tlogId);
-						TraceEvent("SharedTLogQueueSpilling", self.dbgid).detail("NowActive", activeSharedTLog->get());
-						self.sharedActors.send(startSpillingInTenSeconds(&self, tlogId, activeSharedTLog));
-					}
-					activeSharedChange = activeSharedTLog->onChange();
+			} else if (res.index() == 1) {
+				throw internal_error();
+			} else if (res.index() == 2) {
+				if (activeSharedTLog->get() == tlogId) {
+					TraceEvent("SharedTLogNowActive", self.dbgid).detail("NowActive", activeSharedTLog->get());
+					self.targetVolatileBytes = SERVER_KNOBS->TLOG_SPILL_THRESHOLD;
+				} else {
+					stopAllTLogs(&self, tlogId);
+					TraceEvent("SharedTLogQueueSpilling", self.dbgid).detail("NowActive", activeSharedTLog->get());
+					self.sharedActors.send(startSpillingInTenSeconds(&self, tlogId, activeSharedTLog));
 				}
+				activeSharedChange = activeSharedTLog->onChange();
+			} else {
+				UNREACHABLE();
 			}
 		}
 	} catch (Error& e) {
@@ -3834,9 +3992,7 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 			}
 		}
 
-		if (tlogTerminated(&self, persistentData, self.persistentQueue, e)) {
-			return Void();
-		} else {
+		if (!tlogTerminated(&self, persistentData, self.persistentQueue, e)) {
 			throw;
 		}
 	}
@@ -3853,21 +4009,24 @@ template <class T>
 struct DequeAllocator : std::allocator<T> {
 	template <typename U>
 	struct rebind {
-		typedef DequeAllocator<U> other;
+		using other = DequeAllocator<U>;
 	};
 
-	DequeAllocator() {}
+	DequeAllocator() = default;
 
 	template <typename U>
-	DequeAllocator(DequeAllocator<U> const& u) : std::allocator<T>(u) {}
+	explicit(false) DequeAllocator(DequeAllocator<U> const& u) : std::allocator<T>(u) {}
 
 	T* allocate(std::size_t n) {
+		// Intentionally count allocated bytes for all allocator rebinds, including deque internals that use pointer
+		// types. NOLINTNEXTLINE(bugprone-sizeof-expression)
 		DequeAllocatorStats::allocatedBytes += n * sizeof(T);
 		// fprintf(stderr, "Allocating %lld objects for %lld bytes (total allocated: %lld)\n", n, n * sizeof(T),
 		// DequeAllocatorStats::allocatedBytes);
 		return std::allocator<T>::allocate(n);
 	}
 	void deallocate(T* p, std::size_t n) {
+		// NOLINTNEXTLINE(bugprone-sizeof-expression)
 		DequeAllocatorStats::allocatedBytes -= n * sizeof(T);
 		// fprintf(stderr, "Deallocating %lld objects for %lld bytes (total allocated: %lld)\n", n, n * sizeof(T),
 		// DequeAllocatorStats::allocatedBytes);
@@ -3877,7 +4036,7 @@ struct DequeAllocator : std::allocator<T> {
 
 TEST_CASE("Lfdbserver/tlogserver/VersionMessagesOverheadFactor") {
 
-	typedef std::pair<Version, LengthPrefixedStringRef> TestType; // type used by versionMessages
+	using TestType = std::pair<Version, LengthPrefixedStringRef>; // type used by versionMessages
 
 	for (int i = 1; i < 9; ++i) {
 		for (int j = 0; j < 20; ++j) {
diff --git a/fdbserver/TestTLogServer.actor.cpp b/fdbserver/tlog/TestTLogServer.cpp
similarity index 75%
rename from fdbserver/TestTLogServer.actor.cpp
rename to fdbserver/tlog/TestTLogServer.cpp
index a2bcd642b07..814579da447 100644
--- a/fdbserver/TestTLogServer.actor.cpp
+++ b/fdbserver/tlog/TestTLogServer.cpp
@@ -1,5 +1,5 @@
 /*
- * TestTLogServer.actor.cpp
+ * TestTLogServer.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,23 +18,25 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TestTLogServer.actor.h"
+#include "TestTLogServer.h"
 
 #include <filesystem>
 #include <vector>
 
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/ReplicationPolicy.h"
-#include "fdbserver/TLogInterface.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/IDiskQueue.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/kvstore/IDiskQueue.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "fdbserver/tlog/TLogServer.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/logsystem/LogSystemFactory.h"
 #include "flow/IRandom.h"
 #include "flow/DebugTrace.h"
 
-#include "flow/actorcompiler.h" // must be last include
+#include "flow/CoroUtils.h"
 
 namespace {
 
@@ -168,14 +170,14 @@ Reference<TLogTestContext> initTLogTestContext(TestTLogOptions tLogOptions,
 
 // Create and start a tLog. If optional parmeters are set, the tLog is a new generation of "tLogID"
 // as described by initReq. Otherwise, it is a newborn generation 0 tLog.
-ACTOR Future<Void> getTLogCreateActor(Reference<TLogTestContext> pTLogTestContext,
-                                      TestTLogOptions tLogOptions,
-                                      uint16_t processID,
-                                      InitializeTLogRequest* initReq = nullptr,
-                                      UID tLogID = UID()) {
+Future<Void> getTLogCreateActor(Reference<TLogTestContext> pTLogTestContext,
+                                TestTLogOptions tLogOptions,
+                                uint16_t processID,
+                                InitializeTLogRequest* initReq = nullptr,
+                                UID tLogID = UID()) {
 
 	// build per-tLog state.
-	state Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[processID];
+	Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[processID];
 	pTLogContext->tagProcessID = processID;
 
 	pTLogContext->tLogID = tLogID != UID(0, 0) ? tLogID : deterministicRandom()->randomUniqueID();
@@ -185,7 +187,7 @@ ACTOR Future<Void> getTLogCreateActor(Reference<TLogTestContext> pTLogTestContex
 	std::filesystem::create_directory(tLogOptions.dataFolder);
 
 	// create persistent storage
-	state StorageResources storage = setupPersistentStorage(pTLogContext, *pTLogTestContext, tLogOptions);
+	StorageResources storage = setupPersistentStorage(pTLogContext, *pTLogTestContext, tLogOptions);
 
 	// prepare tLog construction.
 	Standalone<StringRef> machineID = "machine"_sr;
@@ -194,31 +196,33 @@ ACTOR Future<Void> getTLogCreateActor(Reference<TLogTestContext> pTLogTestContex
 	localities.set(StringRef("datacenter"_sr), pTLogTestContext->dcID);
 
 	Reference<AsyncVar<bool>> isDegraded = FlowTransport::transport().getDegraded();
+	Reference<AsyncVar<bool>> lowDiskTLogExclusion(new AsyncVar<bool>(false));
 	Reference<AsyncVar<UID>> activeSharedTLog(new AsyncVar<UID>(pTLogContext->tLogID));
 	Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck(new AsyncVar<bool>(false));
-	state PromiseStream<InitializeTLogRequest> promiseStream = PromiseStream<InitializeTLogRequest>();
+	PromiseStream<InitializeTLogRequest> promiseStream;
 	Promise<Void> oldLog;
 	Promise<Void> recovery;
 
 	// construct tLog.
-	state Future<Void> tl = ::tLog(pTLogContext->persistentData,
-	                               pTLogContext->persistentQueue,
-	                               pTLogTestContext->dbInfoRef,
-	                               localities,
-	                               promiseStream,
-	                               pTLogContext->tLogID,
-	                               pTLogTestContext->workerID,
-	                               false, /* restoreFromDisk */
-	                               oldLog,
-	                               recovery,
-	                               pTLogTestContext->diskQueueBasename,
-	                               isDegraded,
-	                               activeSharedTLog,
-	                               enablePrimaryTxnSystemHealthCheck);
-
-	state InitializeTLogRequest initTLogReq = initReq != nullptr ? *initReq : buildInitializeRequest(*pTLogTestContext);
-
-	TLogInterface interface = wait(promiseStream.getReply(initTLogReq));
+	Future<Void> tl = ::tLog(pTLogContext->persistentData,
+	                         pTLogContext->persistentQueue,
+	                         pTLogTestContext->dbInfoRef,
+	                         localities,
+	                         promiseStream,
+	                         pTLogContext->tLogID,
+	                         pTLogTestContext->workerID,
+	                         false, /* restoreFromDisk */
+	                         oldLog,
+	                         recovery,
+	                         pTLogTestContext->diskQueueBasename,
+	                         isDegraded,
+	                         lowDiskTLogExclusion,
+	                         activeSharedTLog,
+	                         enablePrimaryTxnSystemHealthCheck);
+
+	InitializeTLogRequest initTLogReq = initReq != nullptr ? *initReq : buildInitializeRequest(*pTLogTestContext);
+
+	TLogInterface interface = co_await promiseStream.getReply(initTLogReq);
 	pTLogContext->TestTLogInterface = interface;
 	pTLogContext->init = promiseStream;
 
@@ -228,36 +232,31 @@ ACTOR Future<Void> getTLogCreateActor(Reference<TLogTestContext> pTLogTestContex
 	TraceEvent("TestTLogServerInitializedTLog", pTLogContext->tLogID);
 
 	// wait for either test completion or tLog failure.
-	choose {
-		when(wait(tl)) {}
-		when(bool testCompleted = wait(pTLogContext->TestTLogServerCompleted.getFuture())) {
-			ASSERT_EQ(testCompleted, true);
-		}
+	auto choice = co_await race(tl, pTLogContext->TestTLogServerCompleted.getFuture());
+	if (choice.index() == 1) {
+		bool testCompleted = std::get<1>(std::move(choice));
+		ASSERT_EQ(testCompleted, true);
 	}
 
-	wait(delay(1.0));
+	co_await delay(1.0);
 
 	storage.tempFiles.cleanup();
-
-	return Void();
 }
 
-ACTOR Future<Void> TLogTestContext::sendPushMessages(TLogTestContext* pTLogTestContext) {
+Future<Void> TLogTestContext::sendPushMessages(TLogTestContext* pTLogTestContext) {
 
 	TraceEvent("TestTLogServerEnterPush", pTLogTestContext->workerID);
 
-	state uint16_t logID = 0;
-	for (; logID < pTLogTestContext->numLogServers; logID++) {
-		state Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[logID];
-		bool tLogReady = wait(pTLogContext->TLogStarted.getFuture());
+	for (uint16_t logID = 0; logID < pTLogTestContext->numLogServers; ++logID) {
+		Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[logID];
+		bool tLogReady = co_await pTLogContext->TLogStarted.getFuture();
 		ASSERT_EQ(tLogReady, true);
 	}
 
-	state uint32_t i = 0;
-	state Version prev = pTLogTestContext->initVersion - 1;
-	state Version next = pTLogTestContext->initVersion;
+	Version prev = pTLogTestContext->initVersion - 1;
+	Version next = pTLogTestContext->initVersion;
 
-	for (; i < pTLogTestContext->numCommits; i++) {
+	for (uint32_t i = 0; i < pTLogTestContext->numCommits; ++i) {
 		Standalone<StringRef> key = StringRef(format("key %d", i));
 		Standalone<StringRef> val = StringRef(format("value %d", i));
 		MutationRef m(MutationRef::Type::SetValue, key, val);
@@ -289,10 +288,10 @@ ACTOR Future<Void> TLogTestContext::sendPushMessages(TLogTestContext* pTLogTestC
 			}
 		}
 		if (toCommit.getMutationCount()) {
-			const auto versionSet = ILogSystem::PushVersionSet{ prev, next, prev, prev };
+			const auto versionSet = LogPushVersionSet{ prev, next, prev, prev };
 			Future<Version> loggingComplete =
 			    pTLogTestContext->ls->push(versionSet, toCommit, SpanContext(), UID(), tpcvMap);
-			Version ver = wait(loggingComplete);
+			Version ver = co_await loggingComplete;
 			ASSERT_LE(ver, next);
 		}
 		prev = next;
@@ -302,30 +301,26 @@ ACTOR Future<Void> TLogTestContext::sendPushMessages(TLogTestContext* pTLogTestC
 
 	TraceEvent("TestTLogServerExitPush", pTLogTestContext->workerID)
 	    .detail("NumVersions", pTLogTestContext->tLogOptions.versions.size());
-
-	return Void();
 }
 
 // send peek/pop through a given TLog interface and tag
-ACTOR Future<Void> TLogTestContext::peekCommitMessages(TLogTestContext* pTLogTestContext,
-                                                       uint16_t tagID,
-                                                       uint32_t logID) {
-	state Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[logID];
-	bool tLogReady = wait(pTLogContext->TLogStarted.getFuture());
+Future<Void> TLogTestContext::peekCommitMessages(TLogTestContext* pTLogTestContext, uint16_t tagID, uint32_t logID) {
+	Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[logID];
+	bool tLogReady = co_await pTLogContext->TLogStarted.getFuture();
 	ASSERT_EQ(tLogReady, true);
 
 	// peek from the same tag
-	state Tag tag(pTLogTestContext->tagLocality, tagID);
+	Tag tag(pTLogTestContext->tagLocality, tagID);
 
 	TraceEvent("TestTLogServerEnterPeek", pTLogTestContext->workerID).detail("LogID", logID).detail("Tag", tag);
 
-	state uint32_t i = 0;
-	state std::vector<Version> versions = pTLogTestContext->commitHistory[std::tuple(tagID, logID)];
+	uint32_t i = 0;
+	std::vector<Version> versions = pTLogTestContext->commitHistory[std::tuple(tagID, logID)];
 	for (Version v : versions) {
-		state Version begin = v;
+		Version begin = v;
 		// wait for next message commit
 		::TLogPeekRequest request(begin, tag, false, false);
-		::TLogPeekReply reply = wait(pTLogContext->TestTLogInterface.peekMessages.getReply(request));
+		::TLogPeekReply reply = co_await pTLogContext->TestTLogInterface.peekMessages.getReply(request);
 		TraceEvent("TestTLogServerTryValidateDataOnPeek", pTLogTestContext->workerID)
 		    .detail("R", begin)
 		    .detail("B", reply.begin.present() ? reply.begin.get() : -1)
@@ -382,39 +377,35 @@ ACTOR Future<Void> TLogTestContext::peekCommitMessages(TLogTestContext* pTLogTes
 
 		// go directly to pop as there is no SS.
 		::TLogPopRequest requestPop(begin, begin, tag);
-		wait(pTLogContext->TestTLogInterface.popMessages.getReply(requestPop));
+		co_await pTLogContext->TestTLogInterface.popMessages.getReply(requestPop);
 		i++;
 	}
 
 	TraceEvent("TestTLogServerExitPeek", pTLogTestContext->workerID).detail("LogID", logID).detail("TagID", tag);
-
-	return Void();
 }
 
-ACTOR Future<Void> buildTLogSet(Reference<TLogTestContext> pTLogTestContext) {
-	state TLogSet tLogSet;
-	state uint16_t processID = 0;
+Future<Void> buildTLogSet(Reference<TLogTestContext> pTLogTestContext) {
+	TLogSet tLogSet;
 
 	tLogSet.tLogLocalities.push_back(LocalityData());
-	tLogSet.tLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
+	tLogSet.tLogPolicy = makeReference<PolicyOne>();
 	tLogSet.locality = pTLogTestContext->primaryLocality;
 	tLogSet.isLocal = true;
 	tLogSet.tLogVersion = TLogVersion::V6;
 	tLogSet.tLogReplicationFactor = 1;
-	for (; processID < pTLogTestContext->numLogServers; processID++) {
-		state Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[processID];
-		bool isCreated = wait(pTLogContext->TLogCreated.getFuture());
+	for (uint16_t processID = 0; processID < pTLogTestContext->numLogServers; ++processID) {
+		Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[processID];
+		bool isCreated = co_await pTLogContext->TLogCreated.getFuture();
 		ASSERT_EQ(isCreated, true);
 
 		tLogSet.tLogs.push_back(OptionalInterface<TLogInterface>(pTLogContext->TestTLogInterface));
 	}
 	pTLogTestContext->dbInfo.logSystemConfig.tLogs.push_back(tLogSet);
-	for (processID = 0; processID < pTLogTestContext->numLogServers; processID++) {
+	for (uint16_t processID = 0; processID < pTLogTestContext->numLogServers; ++processID) {
 		Reference<TLogContext> pTLogContext = pTLogTestContext->pTLogContextList[processID];
 		// start transactions
 		pTLogContext->TLogStarted.send(true);
 	}
-	return Void();
 }
 
 // This test creates a tLog and pushes data to it. the If the recovery
@@ -422,14 +413,14 @@ ACTOR Future<Void> buildTLogSet(Reference<TLogTestContext> pTLogTestContext) {
 // and pull data from the old generation. The data is peeked from either the old or new generation
 // depending on the recovery switch, validated, and popped.
 
-ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
-	state std::vector<Future<Void>> tLogActors;
-	state Reference<TLogTestContext> pTLogTestContextEpochOne =
+Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
+	std::vector<Future<Void>> tLogActors;
+	Reference<TLogTestContext> pTLogTestContextEpochOne =
 	    initTLogTestContext(params, Optional<Reference<TLogTestContext>>());
 
 	FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT);
 
-	state uint16_t tLogIdx = 0;
+	uint16_t tLogIdx = 0;
 
 	TraceEvent("TestTLogServerEnterRecoveryTest");
 
@@ -442,26 +433,26 @@ ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
 	}
 
 	// wait for tLogs to be created, and signal pushes can start
-	wait(buildTLogSet(pTLogTestContextEpochOne));
+	co_await buildTLogSet(pTLogTestContextEpochOne);
 
 	PromiseStream<Future<Void>> promises;
 
-	pTLogTestContextEpochOne->ls = ILogSystem::fromServerDBInfo(
+	pTLogTestContextEpochOne->ls = makeLogSystemFromServerDBInfo(
 	    pTLogTestContextEpochOne->logID, pTLogTestContextEpochOne->dbInfo, false, promises);
 
-	wait(pTLogTestContextEpochOne->sendPushMessages());
+	co_await pTLogTestContextEpochOne->sendPushMessages();
 
-	state uint32_t tagID = 0;
-	state std::map<std::tuple<uint32_t, uint32_t>, std::vector<Version>> commitHistory;
+	uint32_t tagID = 0;
+	std::map<std::tuple<uint32_t, uint32_t>, std::vector<Version>> commitHistory;
 
 	if (!pTLogTestContextEpochOne->recover) {
 		commitHistory = pTLogTestContextEpochOne->commitHistory;
 		for (const auto& [loc, ver] : commitHistory) {
-			wait(pTLogTestContextEpochOne->peekCommitMessages(std::get<0>(loc), std::get<1>(loc)));
+			co_await pTLogTestContextEpochOne->peekCommitMessages(std::get<0>(loc), std::get<1>(loc));
 		}
 	} else {
 		// Done with old generation. Lock the old generations of tLogs.
-		state Reference<TLogTestContext> pTLogTestContextEpochTwo =
+		Reference<TLogTestContext> pTLogTestContextEpochTwo =
 		    initTLogTestContext(TestTLogOptions(params), pTLogTestContextEpochOne);
 		// for validation: the expected versions in EpochTwo that were written in EpochOne
 		pTLogTestContextEpochTwo->tLogOptions.versions = pTLogTestContextEpochOne->tLogOptions.versions;
@@ -470,8 +461,8 @@ ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
 		for (const auto& [loc, ver] : commitHistory) {
 			tLogIdx = std::get<1>(loc);
 			tagID = std::get<0>(loc);
-			TLogLockResult data = wait(pTLogTestContextEpochOne->pTLogContextList[tLogIdx]
-			                               ->TestTLogInterface.lock.template getReply<TLogLockResult>());
+			TLogLockResult data = co_await pTLogTestContextEpochOne->pTLogContextList[tLogIdx]
+			                          ->TestTLogInterface.lock.template getReply<TLogLockResult>();
 			TraceEvent("TestTLogServerLockResult")
 			    .detail("KCV", data.knownCommittedVersion)
 			    .detail("T", tagID)
@@ -479,7 +470,7 @@ ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
 
 			Reference<TLogContext> pNewTLogContext(new TLogContext(tLogIdx));
 			pTLogTestContextEpochTwo->pTLogContextList[tLogIdx] = pNewTLogContext;
-			state InitializeTLogRequest req;
+			InitializeTLogRequest req;
 			req.recruitmentID = pTLogTestContextEpochTwo->dbInfo.logSystemConfig.recruitmentID;
 			req.recoverAt = pTLogTestContextEpochOne->tLogOptions.versions.back();
 			// for testing + 10;
@@ -506,14 +497,14 @@ ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
 			                                           &req,
 			                                           pTLogTestContextEpochOne->pTLogContextList[tLogIdx]->tLogID));
 
-			state Reference<TLogContext> pTLogContext = pTLogTestContextEpochTwo->pTLogContextList[tLogIdx];
-			bool isCreated = wait(pTLogContext->TLogCreated.getFuture());
+			Reference<TLogContext> pTLogContext = pTLogTestContextEpochTwo->pTLogContextList[tLogIdx];
+			bool isCreated = co_await pTLogContext->TLogCreated.getFuture();
 			ASSERT_EQ(isCreated, true);
 			pTLogContext->TLogStarted.send(true);
 		}
 
 		for (const auto& [loc, ver] : commitHistory) {
-			wait(pTLogTestContextEpochTwo->peekCommitMessages(std::get<0>(loc), std::get<1>(loc)));
+			co_await pTLogTestContextEpochTwo->peekCommitMessages(std::get<0>(loc), std::get<1>(loc));
 		}
 
 		// signal that tLogs can be destroyed
@@ -527,25 +518,21 @@ ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
 	}
 
 	// wait for tLogs to destruct
-	wait(waitForAll(tLogActors));
+	co_await waitForAll(tLogActors);
 
 	TraceEvent("TestTLogServerExitRecoveryTest");
-
-	return Void();
 }
 
 // TEST_CASE("/fdbserver/test/TestTLogCommits") {
 // 	params.set("recover", static_cast<int64_t>(0));
 // 	TestTLogOptions testTLogOptions(params);
-// 	wait(startTestsTLogRecoveryActors(testTLogOptions));
-// 	return Void();
+// 	co_await startTestsTLogRecoveryActors(testTLogOptions);
 // }
 
 // // Two tlogs, one tag. Write 10 commits up to version V. Start recovery with RV set to V+10.
 // TEST_CASE("/fdbserver/test/TestTLogRecovery1") {
 // 	TestTLogOptions testTLogOptions(params);
-// 	wait(startTestsTLogRecoveryActors(testTLogOptions));
-// 	return Void();
+// 	co_await startTestsTLogRecoveryActors(testTLogOptions);
 // }
 
 // // Four tlogs, two tags.
@@ -554,6 +541,5 @@ ACTOR Future<Void> startTestsTLogRecoveryActors(TestTLogOptions params) {
 // 	params.set("numTags", static_cast<int64_t>(2));
 
 // 	TestTLogOptions testTLogOptions(params);
-// 	wait(startTestsTLogRecoveryActors(testTLogOptions));
-// 	return Void();
+// 	co_await startTestsTLogRecoveryActors(testTLogOptions);
 // }
diff --git a/fdbserver/include/fdbserver/TestTLogServer.actor.h b/fdbserver/tlog/TestTLogServer.h
similarity index 78%
rename from fdbserver/include/fdbserver/TestTLogServer.actor.h
rename to fdbserver/tlog/TestTLogServer.h
index 661bca76609..bc179a8954d 100644
--- a/fdbserver/include/fdbserver/TestTLogServer.actor.h
+++ b/fdbserver/tlog/TestTLogServer.h
@@ -1,5 +1,5 @@
 /*
- * TestTLogServer.actor.h
+ * TestTLogServer.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,24 +18,22 @@
  * limitations under the License.
  */
 
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_TEST_TLOG_ACTOR_G_H)
-#define FDBSERVER_TEST_TLOG_ACTOR_G_H
-#include "fdbserver/TestTLogServer.actor.g.h"
-#elif !defined(FDBSERVER_TEST_TLOG_ACTOR_H)
-#define FDBSERVER_TEST_TLOG_ACTOR_H
+#ifndef FDBSERVER_TLOG_TEST_TLOG_H
+#define FDBSERVER_TLOG_TEST_TLOG_H
 
 #include <memory>
 #include <unordered_map>
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/LogSystem.h"
-#include "fdbserver/ResolverInterface.h"
-#include "fdbserver/TLogInterface.h"
+#include "fdbserver/core/IDiskQueue.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/logsystem/LogSystem.h"
+#include "fdbserver/core/ResolverInterface.h"
+#include "fdbserver/core/TLogInterface.h"
+#include "../logsystem/include/fdbserver/logsystem/LogSystem.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/flow.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 #pragma once
 
 struct TestTLogOptions {
@@ -84,17 +82,17 @@ struct TLogContext : NonCopyable, public ReferenceCounted<TLogContext> {
 	Promise<bool> TLogStarted;
 	Promise<bool> TestTLogServerCompleted;
 
-	TLogContext(uint32_t inProcessID = 0) : tagProcessID(inProcessID) {};
+	explicit TLogContext(uint32_t inProcessID = 0) : tagProcessID(inProcessID) {};
 };
 
 // test state
 struct TLogTestContext : NonCopyable, public ReferenceCounted<TLogTestContext> {
 
-	ACTOR static Future<Void> sendPushMessages(TLogTestContext* pTLogTestContext);
+	static Future<Void> sendPushMessages(TLogTestContext* pTLogTestContext);
 
 	Future<Void> sendPushMessages() { return sendPushMessages(this); }
 
-	ACTOR static Future<Void> sendCommitMessages(TLogTestContext* pTLogTestContext, uint16_t processID);
+	static Future<Void> sendCommitMessages(TLogTestContext* pTLogTestContext, uint16_t processID);
 
 	Future<Void> sendCommitMessages(uint16_t processID = 0) { return sendCommitMessages(this, processID); }
 
@@ -102,9 +100,9 @@ struct TLogTestContext : NonCopyable, public ReferenceCounted<TLogTestContext> {
 		return peekCommitMessages(this, tagID, logID);
 	}
 
-	ACTOR static Future<Void> peekCommitMessages(TLogTestContext* pTLogTestContext, uint16_t logGroupID, uint32_t tag);
+	static Future<Void> peekCommitMessages(TLogTestContext* pTLogTestContext, uint16_t logGroupID, uint32_t tag);
 
-	TLogTestContext(TestTLogOptions& tLogOptions) : tLogOptions(tLogOptions), epoch(1) {}
+	explicit(false) TLogTestContext(TestTLogOptions& tLogOptions) : tLogOptions(tLogOptions), epoch(1) {}
 
 	// paramaters
 	std::string diskQueueBasename;
@@ -122,7 +120,7 @@ struct TLogTestContext : NonCopyable, public ReferenceCounted<TLogTestContext> {
 	// system state
 	UID logID;
 	UID workerID;
-	Reference<ILogSystem> ls;
+	Reference<LogSystem> ls;
 	ServerDBInfo dbInfo;
 	Reference<AsyncVar<ServerDBInfo>> dbInfoRef;
 	Standalone<StringRef> dcID;
@@ -132,5 +130,4 @@ struct TLogTestContext : NonCopyable, public ReferenceCounted<TLogTestContext> {
 	const uint32_t primaryLocality = 0;
 };
 
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_TEST_TLOG_ACTOR_G_H
+#endif // FDBSERVER_TLOG_TEST_TLOG_H
diff --git a/fdbserver/tlog/include/fdbserver/tlog/TLogServer.h b/fdbserver/tlog/include/fdbserver/tlog/TLogServer.h
new file mode 100644
index 00000000000..0a3bc21b838
--- /dev/null
+++ b/fdbserver/tlog/include/fdbserver/tlog/TLogServer.h
@@ -0,0 +1,49 @@
+/*
+ * TLogServer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/core/TLogInterface.h"
+#include "flow/flow.h"
+
+class IDiskQueue;
+class IKeyValueStore;
+struct InitializeTLogRequest;
+struct ServerDBInfo;
+
+double effectiveTLogMinAvailableSpaceRatio();
+
+Future<Void> tLog(IKeyValueStore* persistentData,
+                  IDiskQueue* persistentQueue,
+                  Reference<AsyncVar<ServerDBInfo> const> db,
+                  LocalityData locality,
+                  PromiseStream<InitializeTLogRequest> tlogRequests,
+                  UID tlogId,
+                  UID workerID,
+                  bool restoreFromDisk,
+                  Promise<Void> oldLog,
+                  Promise<Void> recovered,
+                  std::string folder,
+                  Reference<AsyncVar<bool>> degraded,
+                  Reference<AsyncVar<bool>> lowDiskTLogExclusion,
+                  Reference<AsyncVar<UID>> activeSharedTLog,
+                  Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck);
+
+using TLogFn = decltype(&tLog);
diff --git a/fdbserver/worker/CMakeLists.txt b/fdbserver/worker/CMakeLists.txt
new file mode 100644
index 00000000000..3601903eaa2
--- /dev/null
+++ b/fdbserver/worker/CMakeLists.txt
@@ -0,0 +1,54 @@
+fdb_find_sources(FDBSERVER_WORKER_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_worker SRCS ${FDBSERVER_WORKER_SRCS})
+add_fdbserver_link_test(fdbserver_workerlinktest
+  fdbserver_worker
+  fdbserver_backupworker
+  fdbserver_clustercontroller
+  fdbserver_commitproxy
+  fdbserver_consistencyscan
+  fdbserver_coordinator
+  fdbserver_datadistributor
+  fdbserver_grvproxy
+  fdbserver_logrouter
+  fdbserver_logsystem
+  fdbserver_ratekeeper
+  fdbserver_resolver
+  fdbserver_sequencer
+  fdbserver_storageserver
+  fdbserver_tester
+  fdbserver_tlog
+  fdbserver_core
+  fdbctl)
+
+configure_fdbserver_common_includes(fdbserver_worker)
+target_include_directories(fdbserver_worker
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE
+    ${CMAKE_SOURCE_DIR}/fdbserver/include
+    ${CMAKE_BINARY_DIR}/fdbserver/include
+    ${CMAKE_SOURCE_DIR}/fdbctl/include
+    ${CMAKE_BINARY_DIR}/fdbctl/include
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(fdbserver_worker
+  PUBLIC
+    fdbctl
+  PRIVATE
+    fdbserver_backupworker
+    fdbserver_clustercontroller
+    fdbserver_commitproxy
+    fdbserver_consistencyscan
+    fdbserver_coordinator
+    fdbserver_datadistributor
+    fdbserver_grvproxy
+    fdbserver_logrouter
+    fdbserver_logsystem
+    fdbserver_ratekeeper
+    fdbserver_resolver
+    fdbserver_sequencer
+    fdbserver_storageserver
+    fdbserver_tester
+    fdbserver_tlog
+    fdbserver_core)
diff --git a/fdbserver/MetricClient.actor.cpp b/fdbserver/worker/MetricClient.cpp
similarity index 89%
rename from fdbserver/MetricClient.actor.cpp
rename to fdbserver/worker/MetricClient.cpp
index 221e131f630..2cc41cec6fa 100644
--- a/fdbserver/MetricClient.actor.cpp
+++ b/fdbserver/worker/MetricClient.cpp
@@ -1,5 +1,5 @@
 /*
- * MetricClient.actor.cpp
+ * MetricClient.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -17,13 +17,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "fdbserver/MetricClient.h"
+#include "MetricClient.h"
 #include "fdbrpc/Stats.h"
 #include "flow/FastRef.h"
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
 #include "flow/OTELMetrics.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/Msgpack.h"
 #include "flow/Trace.h"
 #include "flow/flow.h"
@@ -34,8 +34,6 @@
 #include "flow/network.h"
 #include "flow/IUDPSocket.h"
 #include "flow/IConnection.h"
-#include "flow/actorcompiler.h"
-
 UDPMetricClient::UDPMetricClient()
   : socket_fd(-1), model(knobToMetricModel(FLOW_KNOBS->METRICS_DATA_MODEL)),
     buf{ MsgpackBuffer{ .buffer = std::make_unique<uint8_t[]>(1024), .data_size = 0, .buffer_size = 1024 } },
@@ -67,29 +65,30 @@ void UDPMetricClient::send(MetricCollection* metrics) {
 
 		// Define custom serialize functions
 		auto f_sums = [](const std::vector<OTEL::OTELSum>& vec, MsgpackBuffer& buf) {
-			typedef void (*func_ptr)(const OTEL::OTELSum&, MsgpackBuffer&);
+			using func_ptr = void (*)(const OTEL::OTELSum&, MsgpackBuffer&);
 			func_ptr f = OTEL::serialize;
 			serialize_vector(vec, buf, f);
 		};
 
 		auto f_hists = [](const std::vector<OTEL::OTELHistogram>& vec, MsgpackBuffer& buf) {
-			typedef void (*func_ptr)(const OTEL::OTELHistogram&, MsgpackBuffer&);
+			using func_ptr = void (*)(const OTEL::OTELHistogram&, MsgpackBuffer&);
 			func_ptr f = OTEL::serialize;
 			serialize_vector(vec, buf, f);
 		};
 
 		auto f_gauge = [](const std::vector<OTEL::OTELGauge>& vec, MsgpackBuffer& buf) {
-			typedef void (*func_ptr)(const OTEL::OTELGauge&, MsgpackBuffer&);
+			using func_ptr = void (*)(const OTEL::OTELGauge&, MsgpackBuffer&);
 			func_ptr f = OTEL::serialize;
 			serialize_vector(vec, buf, f);
 		};
 
 		std::vector<OTEL::OTELSum> currentSums;
 		size_t current_msgpack = 0;
-		for (const auto& [_, s] : metrics->sumMap) {
+		for (auto& [_, s] : metrics->sumMap) {
+			const auto msgpackBytes = s.getMsgpackBytes();
 			if (current_msgpack < MAX_OTELSUM_PACKET_SIZE) {
 				currentSums.push_back(std::move(s));
-				current_msgpack += s.getMsgpackBytes();
+				current_msgpack += msgpackBytes;
 			} else {
 				sums.push_back(std::move(currentSums));
 				currentSums.clear();
@@ -109,7 +108,7 @@ void UDPMetricClient::send(MetricCollection* metrics) {
 
 		// Each histogram should be in a separate because of their large sizes
 		// Expected DDSketch size is ~4200 entries * 9 bytes = 37800
-		for (const auto& [_, h] : metrics->histMap) {
+		for (auto& [_, h] : metrics->histMap) {
 			const std::vector<OTEL::OTELHistogram> singleHist{ std::move(h) };
 			serialize_ext(singleHist, buf, OTEL::OTELMetricType::Hist, f_hists);
 			send_packet(socket_fd, buf.buffer.get(), buf.data_size);
@@ -120,7 +119,7 @@ void UDPMetricClient::send(MetricCollection* metrics) {
 
 		metrics->histMap.clear();
 
-		for (const auto& [_, g] : metrics->gaugeMap) {
+		for (auto& [_, g] : metrics->gaugeMap) {
 			gauges.push_back(std::move(g));
 		}
 		if (!gauges.empty()) {
@@ -146,4 +145,4 @@ void UDPMetricClient::send(MetricCollection* metrics) {
 		}
 		metrics->statsd_message.clear();
 	}
-}
\ No newline at end of file
+}
diff --git a/fdbserver/include/fdbserver/MetricClient.h b/fdbserver/worker/MetricClient.h
similarity index 92%
rename from fdbserver/include/fdbserver/MetricClient.h
rename to fdbserver/worker/MetricClient.h
index d7c4f97bd77..5ecbda55050 100644
--- a/fdbserver/include/fdbserver/MetricClient.h
+++ b/fdbserver/worker/MetricClient.h
@@ -18,13 +18,13 @@
  * limitations under the License.
  */
 
-#ifndef METRIC_CLIENT_H
-#define METRIC_CLIENT_H
+#ifndef FDBSERVER_WORKER_METRIC_CLIENT_H
+#define FDBSERVER_WORKER_METRIC_CLIENT_H
 
-#include "flow/TDMetric.actor.h"
+#include "flow/IUDPSocket.h"
 #include "flow/Msgpack.h"
+#include "flow/TDMetric.h"
 #include "flow/network.h"
-#include "flow/IUDPSocket.h"
 
 class IMetricClient {
 protected:
@@ -52,4 +52,5 @@ class UDPMetricClient : public IMetricClient {
 	UDPMetricClient();
 	void send(MetricCollection*) override;
 };
-#endif
\ No newline at end of file
+
+#endif
diff --git a/fdbserver/MetricLogger.actor.cpp b/fdbserver/worker/MetricLogger.actor.cpp
similarity index 97%
rename from fdbserver/MetricLogger.actor.cpp
rename to fdbserver/worker/MetricLogger.actor.cpp
index 2e69993dbe7..0f2921a5029 100644
--- a/fdbserver/MetricLogger.actor.cpp
+++ b/fdbserver/worker/MetricLogger.actor.cpp
@@ -32,20 +32,21 @@
 #include "flow/OTELMetrics.h"
 #include "flow/SystemMonitor.h"
 #include "flow/UnitTest.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbserver/MetricLogger.actor.h"
-#include "fdbserver/MetricClient.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "MetricLogger.actor.h"
+#include "MetricClient.h"
 #include "flow/flow.h"
 #include "flow/network.h"
 #include "flow/IUDPSocket.h"
 #include "flow/IConnection.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+namespace {
 struct MetricsRule {
-	MetricsRule(bool enabled = false, int minLevel = 0, StringRef const& name = StringRef())
+	explicit(false) MetricsRule(bool enabled = false, int minLevel = 0, StringRef const& name = StringRef())
 	  : namePattern(name), enabled(enabled), minLevel(minLevel) {}
 
 	Standalone<StringRef> typePattern;
@@ -105,7 +106,7 @@ struct MetricsRule {
 };
 
 struct MetricsConfig {
-	MetricsConfig(Key prefix = KeyRef())
+	explicit MetricsConfig(Key prefix = KeyRef())
 	  : space(prefix), ruleMap(space.get("Rules"_sr).key()), addressMap(space.get("Enum"_sr).get("Address"_sr).key()),
 	    nameAndTypeMap(space.get("Enum"_sr).get("NameType"_sr).key()),
 	    ruleChangeKey(space.get("RulesChanged"_sr).key()), enumsChangeKey(space.get("EnumsChanged"_sr).key()),
@@ -113,7 +114,7 @@ struct MetricsConfig {
 
 	Subspace space;
 
-	typedef KeyBackedMap<int64_t, MetricsRule> RuleMapT;
+	using RuleMapT = KeyBackedMap<int64_t, MetricsRule>;
 	RuleMapT ruleMap;
 	RuleMapT::RangeResultType rules;
 
@@ -185,7 +186,7 @@ ACTOR Future<Void> metricRuleUpdater(Database cx, MetricsConfig* config, TDMetri
 // Implementation of IMetricDB
 class MetricDB : public IMetricDB {
 public:
-	MetricDB(ReadYourWritesTransaction* tr = nullptr) : tr(tr) {}
+	explicit MetricDB(ReadYourWritesTransaction* tr = nullptr) : tr(tr) {}
 	~MetricDB() override {}
 
 	// levelKey is the prefix for the entire level, no timestamp at the end
@@ -379,6 +380,7 @@ ACTOR Future<Void> updateMetricRegistration(Database cx, MetricsConfig* config,
 		wait(registrationChange || newMetric);
 	}
 }
+} // namespace
 
 ACTOR Future<Void> runMetrics(Future<Database> fcx, Key prefix) {
 	// Never log to an empty prefix, it's pretty much always a bad idea.
@@ -419,6 +421,7 @@ ACTOR Future<Void> runMetrics(Future<Database> fcx, Key prefix) {
 	return Void();
 }
 
+namespace {
 ACTOR Future<Void> startMetricsSimulationServer(MetricsDataModel model) {
 	if (model == MetricsDataModel::NONE) {
 		return Void{};
@@ -457,6 +460,7 @@ ACTOR Future<Void> startMetricsSimulationServer(MetricsDataModel model) {
 		}
 	}
 }
+} // namespace
 
 ACTOR Future<Void> runMetrics() {
 	state MetricCollection* metrics = nullptr;
diff --git a/fdbserver/include/fdbserver/MetricLogger.actor.h b/fdbserver/worker/MetricLogger.actor.h
similarity index 71%
rename from fdbserver/include/fdbserver/MetricLogger.actor.h
rename to fdbserver/worker/MetricLogger.actor.h
index 2090c3ca470..1b5c05125f5 100644
--- a/fdbserver/include/fdbserver/MetricLogger.actor.h
+++ b/fdbserver/worker/MetricLogger.actor.h
@@ -20,13 +20,17 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_METRICLOGGER_ACTOR_G_H)
-#define FDBSERVER_METRICLOGGER_ACTOR_G_H
-#include "fdbserver/MetricLogger.actor.g.h"
-#elif !defined(FDBSERVER_METRICLOGGER_ACTOR_H)
-#define FDBSERVER_METRICLOGGER_ACTOR_H
+#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_WORKER_METRICLOGGER_ACTOR_G_H)
+#define FDBSERVER_WORKER_METRICLOGGER_ACTOR_G_H
+#include "MetricLogger.actor.g.h"
+#elif !defined(FDBSERVER_WORKER_METRICLOGGER_ACTOR_H)
+#define FDBSERVER_WORKER_METRICLOGGER_ACTOR_H
+
+#include "fdbclient/FDBTypes.h"
+#include "flow/flow.h"
+
+class Database;
+
 #include "flow/actorcompiler.h" // This must be the last #include
 
 ACTOR Future<Void> runMetrics(Future<Database> fcx, Key metricsPrefix);
diff --git a/fdbserver/RoleLineage.actor.cpp b/fdbserver/worker/RoleLineage.cpp
similarity index 95%
rename from fdbserver/RoleLineage.actor.cpp
rename to fdbserver/worker/RoleLineage.cpp
index 78cc7e49f03..d72f29c1706 100644
--- a/fdbserver/RoleLineage.actor.cpp
+++ b/fdbserver/worker/RoleLineage.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/RoleLineage.actor.h"
+#include "RoleLineage.h"
 
 using namespace std::literals;
 
diff --git a/fdbserver/include/fdbserver/RoleLineage.actor.h b/fdbserver/worker/RoleLineage.h
similarity index 62%
rename from fdbserver/include/fdbserver/RoleLineage.actor.h
rename to fdbserver/worker/RoleLineage.h
index c3888ac8e60..51ceba4e5f1 100644
--- a/fdbserver/include/fdbserver/RoleLineage.actor.h
+++ b/fdbserver/worker/RoleLineage.h
@@ -1,5 +1,5 @@
 /*
- * RoleLineage.actor.h
+ * RoleLineage.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,22 +19,12 @@
  */
 
 #pragma once
-#include "flow/flow.h"
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_ROLE_LINEAGE_ACTOR_G_H)
-#define FDBSERVER_ROLE_LINEAGE_ACTOR_G_H
-#include "fdbserver/RoleLineage.actor.g.h"
-#elif !defined(FDBSERVER_ROLE_LINEAGE_ACTOR_H)
-#define FDBSERVER_ROLE_LINEAGE_ACTOR_H
-
-#include "flow/singleton.h"
-#include "fdbrpc/Locality.h"
 #include "fdbclient/ActorLineageProfiler.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 
-#include <string_view>
-#include <msgpack.hpp>
 #include <any>
-#include "flow/actorcompiler.h" // This must be the last include
+#include <optional>
+#include <string_view>
 
 struct RoleLineage : LineageProperties<RoleLineage> {
 	static std::string_view name;
@@ -54,16 +44,3 @@ struct RoleLineageCollector : IALPCollector<RoleLineage> {
 		}
 	}
 };
-
-// creates a new root and sets the role lineage
-ACTOR template <class Fun>
-Future<decltype(std::declval<Fun>()())> runInRole(Fun fun, ProcessClass::ClusterRole role) {
-	getCurrentLineage()->makeRoot();
-	getCurrentLineage()->modify(&RoleLineage::role) = role;
-	decltype(std::declval<Fun>()()) res = wait(fun());
-	return res;
-}
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/fdbserver/worker/include/fdbserver/worker/Worker.h b/fdbserver/worker/include/fdbserver/worker/Worker.h
new file mode 100644
index 00000000000..9a8e9af46b8
--- /dev/null
+++ b/fdbserver/worker/include/fdbserver/worker/Worker.h
@@ -0,0 +1,50 @@
+/*
+ * Worker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef FDBSERVER_WORKER_WORKER_H
+#define FDBSERVER_WORKER_WORKER_H
+
+#include <string>
+
+#include "fdbclient/ClientWorkerInterface.h"
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbrpc/Locality.h"
+#include "flow/flow.h"
+
+struct ClusterControllerFullInterface;
+struct ServerDBInfo;
+
+Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
+                  LocalityData localities,
+                  ProcessClass processClass,
+                  std::string dataFolder,
+                  std::string coordFolder,
+                  int64_t memoryLimit,
+                  std::string metricsConnFile,
+                  std::string metricsPrefix,
+                  int64_t memoryProfilingThreshold,
+                  std::string whitelistBinPaths,
+                  bool consistencyCheckUrgentMode);
+
+void registerThreadForProfiling();
+#endif
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker/worker.actor.cpp
similarity index 85%
rename from fdbserver/worker.actor.cpp
rename to fdbserver/worker/worker.actor.cpp
index 0279bdff522..4dcd26225a1 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker/worker.actor.cpp
@@ -29,11 +29,11 @@
 #include "flow/CodeProbe.h"
 #include "flow/IAsyncFile.h"
 #include "fdbrpc/Locality.h"
-#include "fdbclient/GlobalConfig.actor.h"
+#include "fdbclient/GlobalConfig.h"
 #include "fdbclient/ProcessInterface.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/versions.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/ActorCollection.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
@@ -44,22 +44,35 @@
 #include "flow/Platform.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/SystemMonitor.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/MetricLogger.actor.h"
-#include "fdbserver/BackupInterface.h"
-#include "fdbserver/RoleLineage.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/WaitFailure.h"
-#include "fdbserver/TesterInterface.actor.h" // for poisson()
-#include "fdbserver/IDiskQueue.h"
+#include "MetricLogger.actor.h"
+#include "fdbserver/backupworker/BackupWorker.h"
+#include "fdbserver/clustercontroller/ClusterController.h"
+#include "fdbserver/commitproxy/CommitProxyServer.h"
+#include "fdbserver/consistencyscan/ConsistencyScan.h"
+#include "fdbserver/datadistributor/DataDistributor.h"
+#include "fdbserver/grvproxy/GrvProxyServer.h"
+#include "fdbserver/logrouter/LogRouter.h"
+#include "fdbserver/core/BackupInterface.h"
+#include "RoleLineage.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/worker/Worker.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/ratekeeper/Ratekeeper.h"
+#include "fdbserver/resolver/Resolver.h"
+#include "fdbserver/sequencer/MasterServer.h"
+#include "fdbserver/storageserver/StorageServer.h"
+#include "fdbserver/tlog/TLogServer.h"
+#include "fdbserver/core/WaitFailure.h"
+#include "fdbserver/tester/tester.h"
+#include "fdbserver/kvstore/IDiskQueue.h"
 #include "fdbclient/DatabaseContext.h"
-#include "fdbserver/DataDistributorInterface.h"
-#include "fdbserver/FDBExecHelper.actor.h"
-#include "fdbserver/CoordinationInterface.h"
-#include "fdbserver/RemoteIKeyValueStore.actor.h"
+#include "fdbserver/core/DataDistributorInterface.h"
+#include "fdbserver/coordinator/CoordinationServer.h"
+#include "fdbserver/kvstore/FDBExecHelper.h"
+#include "fdbserver/core/CoordinationInterface.h"
 #include "fdbclient/MonitorLeader.h"
 #include "fdbclient/ClientWorkerInterface.h"
 #include "flow/Profiler.h"
@@ -90,6 +103,7 @@
 #include <unistd.h>
 #include <execinfo.h>
 #endif
+#include "fdbserver/core/TesterInterface.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 #if CENABLED(0, NOT_IN_CLEAN)
@@ -101,97 +115,6 @@ extern IKeyValueStore* keyValueStoreCompressTestData(IKeyValueStore* store);
 #define KV_STORE(filename, uid) keyValueStoreMemory(filename, uid)
 #endif
 
-template class RequestStream<RecruitMasterRequest, false>;
-template struct NetNotifiedQueue<RecruitMasterRequest, false>;
-
-template class RequestStream<InitializeCommitProxyRequest, false>;
-template struct NetNotifiedQueue<InitializeCommitProxyRequest, false>;
-
-template class RequestStream<InitializeGrvProxyRequest, false>;
-template struct NetNotifiedQueue<InitializeGrvProxyRequest, false>;
-
-template class RequestStream<GetServerDBInfoRequest, false>;
-template struct NetNotifiedQueue<GetServerDBInfoRequest, false>;
-
-namespace {
-RoleLineageCollector roleLineageCollector;
-}
-
-ACTOR Future<std::vector<Endpoint>> tryDBInfoBroadcast(RequestStream<UpdateServerDBInfoRequest> stream,
-                                                       UpdateServerDBInfoRequest req) {
-	ErrorOr<std::vector<Endpoint>> rep =
-	    wait(stream.getReplyUnlessFailedFor(req, SERVER_KNOBS->DBINFO_FAILED_DELAY, 0));
-	if (rep.present()) {
-		return rep.get();
-	}
-	req.broadcastInfo.push_back(stream.getEndpoint());
-	return req.broadcastInfo;
-}
-
-ACTOR Future<std::vector<Endpoint>> broadcastDBInfoRequest(UpdateServerDBInfoRequest req,
-                                                           int sendAmount,
-                                                           Optional<Endpoint> sender,
-                                                           bool sendReply) {
-	state std::vector<Future<std::vector<Endpoint>>> replies;
-	state ReplyPromise<std::vector<Endpoint>> reply = req.reply;
-	resetReply(req);
-	int currentStream = 0;
-	std::vector<Endpoint> broadcastEndpoints = req.broadcastInfo;
-	for (int i = 0; i < sendAmount && currentStream < broadcastEndpoints.size(); i++) {
-		std::vector<Endpoint> endpoints;
-		RequestStream<UpdateServerDBInfoRequest> cur(broadcastEndpoints[currentStream++]);
-		while (currentStream < broadcastEndpoints.size() * (i + 1) / sendAmount) {
-			endpoints.push_back(broadcastEndpoints[currentStream++]);
-		}
-		req.broadcastInfo = endpoints;
-		replies.push_back(tryDBInfoBroadcast(cur, req));
-		resetReply(req);
-	}
-	wait(waitForAll(replies));
-	std::vector<Endpoint> notUpdated;
-	if (sender.present()) {
-		notUpdated.push_back(sender.get());
-	}
-	for (auto& it : replies) {
-		notUpdated.insert(notUpdated.end(), it.get().begin(), it.get().end());
-	}
-	if (sendReply) {
-		reply.send(notUpdated);
-	}
-	return notUpdated;
-}
-
-ACTOR static Future<Void> extractClientInfo(Reference<AsyncVar<ServerDBInfo> const> db,
-                                            Reference<AsyncVar<ClientDBInfo>> info) {
-	state std::vector<UID> lastCommitProxyUIDs;
-	state std::vector<CommitProxyInterface> lastCommitProxies;
-	state std::vector<UID> lastGrvProxyUIDs;
-	state std::vector<GrvProxyInterface> lastGrvProxies;
-	loop {
-		ClientDBInfo ni = db->get().client;
-		shrinkProxyList(ni, lastCommitProxyUIDs, lastCommitProxies, lastGrvProxyUIDs, lastGrvProxies);
-		info->setUnconditional(ni);
-		wait(db->onChange());
-	}
-}
-
-Database openDBOnServer(Reference<AsyncVar<ServerDBInfo> const> const& db,
-                        TaskPriority taskID,
-                        LockAware lockAware,
-                        EnableLocalityLoadBalance enableLocalityLoadBalance) {
-	auto info = makeReference<AsyncVar<ClientDBInfo>>();
-	auto cx = DatabaseContext::create(info,
-	                                  extractClientInfo(db, info),
-	                                  enableLocalityLoadBalance ? db->get().myLocality : LocalityData(),
-	                                  enableLocalityLoadBalance,
-	                                  taskID,
-	                                  lockAware);
-	cx->globalConfig->init(db, std::addressof(db->get().client));
-	cx->globalConfig->trigger(samplingFrequency, samplingProfilerUpdateFrequency);
-	cx->globalConfig->trigger(samplingWindow, samplingProfilerUpdateWindow);
-	return cx;
-}
-
 struct ErrorInfo {
 	Error error;
 	const Role& role;
@@ -203,6 +126,14 @@ struct ErrorInfo {
 	}
 };
 
+struct PrimaryAndRemoteAddresses {
+	std::vector<NetworkAddress> primary;
+	std::vector<NetworkAddress> remote;
+};
+
+namespace {
+RoleLineageCollector roleLineageCollector;
+
 Error checkIOTimeout(Error const& e) {
 	// Convert all_errors to io_timeout if global timeout bool was set
 	bool timeoutOccurred = (bool)g_network->global(INetwork::enASIOTimedOut);
@@ -222,14 +153,14 @@ Error checkIOTimeout(Error const& e) {
 	return e;
 }
 
-ACTOR Future<Void> forwardError(PromiseStream<ErrorInfo> errors, Role role, UID id, Future<Void> process) {
+Future<Void> forwardError(PromiseStream<ErrorInfo> errors, Role role, UID id, Future<Void> process) {
 	try {
-		wait(process);
+		co_await process;
 		errors.send(ErrorInfo(success(), role, id));
-		return Void();
+		co_return;
 	} catch (Error& e) {
 		errors.send(ErrorInfo(e, role, id));
-		return Void();
+		co_return;
 	}
 }
 
@@ -278,7 +209,7 @@ Future<Void> handleIOErrors(Future<Void> actor, IClosable* store, UID id, Future
 	return handleIOErrors(actor, storeError, id, onClosed);
 }
 
-Future<Void> deregisterGrpcService(const UID& id) {
+Future<Void> deregisterGrpcService(UID id) {
 #ifdef FLOW_GRPC_ENABLED
 	if (g_network->isSimulated()) {
 		return Void();
@@ -297,7 +228,6 @@ ACTOR Future<Void> workerHandleErrors(FutureStream<ErrorInfo> errors) {
 			ErrorInfo err = _err;
 			bool ok = err.error.code() == error_code_success || err.error.code() == error_code_please_reboot ||
 			          err.error.code() == error_code_actor_cancelled ||
-			          err.error.code() == error_code_remote_kvs_cancelled ||
 			          err.error.code() == error_code_coordinators_changed || // The worker server was cancelled
 			          err.error.code() == error_code_shutdown_in_progress ||
 			          err.error.code() == error_code_audit_storage_task_outdated; // Expected during DD failover
@@ -327,23 +257,23 @@ ACTOR Future<Void> workerHandleErrors(FutureStream<ErrorInfo> errors) {
 // Improve simulation code coverage by sometimes deferring the destruction of workerInterface (and therefore "endpoint
 // not found" responses to clients
 //		for an extra second, so that clients are more likely to see broken_promise errors
-ACTOR template <class T>
+template <class T>
 Future<Void> zombie(T workerInterface, Future<Void> worker) {
 	try {
-		wait(worker);
+		co_await worker;
 		if (BUGGIFY)
-			wait(delay(1.0));
-		return Void();
+			co_await delay(1.0);
+		co_return;
 	} catch (Error& e) {
 		throw;
 	}
 }
 
-ACTOR Future<Void> loadedPonger(FutureStream<LoadedPingRequest> pings) {
-	state Standalone<StringRef> payloadBack(std::string(20480, '.'));
+Future<Void> loadedPonger(FutureStream<LoadedPingRequest> pings) {
+	Standalone<StringRef> payloadBack(std::string(20480, '.'));
 
-	loop {
-		LoadedPingRequest pong = waitNext(pings);
+	while (true) {
+		LoadedPingRequest pong = co_await pings;
 		LoadedReply rep;
 		rep.payload = (pong.loadReply ? payloadBack : ""_sr);
 		rep.id = pong.id;
@@ -507,7 +437,7 @@ TLogFn tLogFnForOptions(TLogOptions options) {
 }
 
 struct DiskStore {
-	enum COMPONENT { TLogData, Storage, BlobWorker, UNSET };
+	enum COMPONENT { TLogData, Storage, UNSET };
 
 	UID storeID = UID();
 	std::string filename = ""; // For KVStoreMemory just the base filename to be passed to IDiskQueue
@@ -631,7 +561,6 @@ ACTOR Future<Void> registrationClient(Reference<AsyncVar<Optional<ClusterControl
 			incorrectTime = Optional<double>();
 		}
 
-		Optional<EncryptKeyProxyInterface> fakeEpkInterf;
 		RegisterWorkerRequest request(interf,
 		                              initialClass,
 		                              processClass,
@@ -639,7 +568,6 @@ ACTOR Future<Void> registrationClient(Reference<AsyncVar<Optional<ClusterControl
 		                              requestGeneration++,
 		                              ddInterf->get(),
 		                              rkInterf->get(),
-		                              fakeEpkInterf,
 		                              csInterf->get(),
 		                              degraded->get(),
 		                              recoveredDiskFiles.isSet(),
@@ -867,22 +795,6 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimaryDc") {
 
 } // namespace
 
-// Returns true if `address` is used in the db (indicated by `dbInfo`) transaction system and in the db's primary
-// satellite DC.
-bool addressInDbAndPrimarySatelliteDc(const NetworkAddress& address, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	for (const auto& logSet : dbInfo->get().logSystemConfig.tLogs) {
-		if (logSet.isLocal && logSet.locality == tagLocalitySatellite) {
-			for (const auto& tlog : logSet.tLogs) {
-				if (tlog.present() && tlog.interf().addresses().contains(address)) {
-					return true;
-				}
-			}
-		}
-	}
-
-	return false;
-}
-
 bool addressesInDbAndPrimarySatelliteDc(const NetworkAddressList& addresses,
                                         Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
 	return addressInDbAndPrimarySatelliteDc(addresses.address, dbInfo) ||
@@ -943,36 +855,6 @@ TEST_CASE("/fdbserver/worker/addressInDbAndPrimarySatelliteDc") {
 
 } // namespace
 
-bool addressInDbAndRemoteDc(const NetworkAddress& address,
-                            Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                            Optional<std::vector<NetworkAddress>> storageServers) {
-	const auto& dbi = dbInfo->get();
-
-	for (const auto& logSet : dbi.logSystemConfig.tLogs) {
-		if (logSet.isLocal || logSet.locality == tagLocalitySatellite) {
-			continue;
-		}
-		for (const auto& tlog : logSet.tLogs) {
-			if (tlog.present() && tlog.interf().addresses().contains(address)) {
-				return true;
-			}
-		}
-
-		for (const auto& logRouter : logSet.logRouters) {
-			if (logRouter.present() && logRouter.interf().addresses().contains(address)) {
-				return true;
-			}
-		}
-	}
-
-	if (storageServers.present() &&
-	    (std::find(storageServers.get().begin(), storageServers.get().end(), address) != storageServers.get().end())) {
-		return true;
-	}
-
-	return false;
-}
-
 bool addressesInDbAndRemoteDc(
     const NetworkAddressList& addresses,
     Reference<AsyncVar<ServerDBInfo> const> dbInfo,
@@ -1135,11 +1017,6 @@ bool isDegradedPeer(const UpdateWorkerHealthRequest& lastReq, const NetworkAddre
 	return false;
 }
 
-struct PrimaryAndRemoteAddresses {
-	std::vector<NetworkAddress> primary;
-	std::vector<NetworkAddress> remote;
-};
-
 // Check if the current worker is a transaction worker, and is experiencing degraded or disconnected peers.
 UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
                                             const LocalityData& locality,
@@ -1391,16 +1268,16 @@ static Optional<Standalone<StringRef>> getPrimaryDCId(const ServerDBInfo& dbInfo
 // This actor makes a network call, and if that call fails, an empty optional is returned in addition to a
 // TraceEvent being logged. Intentionally, this actor does not implement a retry policy, but the client can
 // choose to retry by waiting on this actor again.
-ACTOR Future<Optional<PrimaryAndRemoteAddresses>> getStorageServers(Database db,
-                                                                    Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
-	state Optional<PrimaryAndRemoteAddresses> ret;
-	state Transaction tr(db);
+Future<Optional<PrimaryAndRemoteAddresses>> getStorageServers(Database db,
+                                                              Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	Optional<PrimaryAndRemoteAddresses> ret;
+	Transaction tr(db);
 	try {
 		tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-		    wait(NativeAPI::getServerListAndProcessClasses(&tr));
+		    co_await NativeAPI::getServerListAndProcessClasses(&tr);
 		PrimaryAndRemoteAddresses storageServers;
 		const auto primaryDCId = getPrimaryDCId(dbInfo->get());
 		for (auto& [ssi, _] : results) {
@@ -1426,7 +1303,7 @@ ACTOR Future<Optional<PrimaryAndRemoteAddresses>> getStorageServers(Database db,
 			TraceEvent("GetStorageServersError").error(e);
 		}
 	}
-	return ret;
+	co_return ret;
 }
 
 // The actor that actively monitors the health of local and peer servers, and reports anomaly to the cluster controller.
@@ -1489,7 +1366,10 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 	}
 }
 
+} // namespace
+
 #if (defined(__linux__) || defined(__FreeBSD__)) && defined(USE_GPERFTOOLS)
+namespace {
 // A set of threads that should be profiled
 std::set<std::thread::id> profiledThreads;
 
@@ -1497,6 +1377,7 @@ std::set<std::thread::id> profiledThreads;
 int filter_in_thread(void* arg) {
 	return profiledThreads.contains(std::this_thread::get_id()) ? 1 : 0;
 }
+} // namespace
 #endif
 
 // Enables the calling thread to be profiled
@@ -1511,6 +1392,8 @@ void registerThreadForProfiling() {
 #endif
 }
 
+namespace {
+
 // Starts or stops the CPU profiler
 void updateCpuProfiler(ProfilerRequest req) {
 	switch (req.type) {
@@ -1553,17 +1436,17 @@ void updateCpuProfiler(ProfilerRequest req) {
 	}
 }
 
-ACTOR Future<Void> runCpuProfiler(ProfilerRequest req) {
+Future<Void> runCpuProfiler(ProfilerRequest req) {
 	if (req.action == ProfilerRequest::Action::RUN) {
 		req.action = ProfilerRequest::Action::ENABLE;
 		updateCpuProfiler(req);
-		wait(delay(req.duration));
+		co_await delay(req.duration);
 		req.action = ProfilerRequest::Action::DISABLE;
 		updateCpuProfiler(req);
-		return Void();
+		co_return;
 	} else {
 		updateCpuProfiler(req);
-		return Void();
+		co_return;
 	}
 }
 
@@ -1579,14 +1462,14 @@ void runHeapProfiler(const char* msg) {
 #endif
 }
 
-ACTOR Future<Void> runProfiler(ProfilerRequest req) {
+Future<Void> runProfiler(ProfilerRequest req) {
 	if (req.type == ProfilerRequest::Type::GPROF_HEAP) {
 		runHeapProfiler("User triggered heap dump");
 	} else {
-		wait(runCpuProfiler(req));
+		co_await runCpuProfiler(req);
 	}
 
-	return Void();
+	co_return;
 }
 
 bool checkHighMemory(int64_t threshold, bool* error) {
@@ -1623,11 +1506,11 @@ bool checkHighMemory(int64_t threshold, bool* error) {
 }
 
 // Runs heap profiler when RSS memory usage is high.
-ACTOR Future<Void> monitorHighMemory(int64_t threshold) {
+Future<Void> monitorHighMemory(int64_t threshold) {
 	if (threshold <= 0)
-		return Void();
+		co_return;
 
-	loop {
+	while (true) {
 		bool err = false;
 		bool highmem = checkHighMemory(threshold, &err);
 		if (err)
@@ -1635,9 +1518,9 @@ ACTOR Future<Void> monitorHighMemory(int64_t threshold) {
 
 		if (highmem)
 			runHeapProfiler("Highmem heap dump");
-		wait(delay(SERVER_KNOBS->HEAP_PROFILER_INTERVAL));
+		co_await delay(SERVER_KNOBS->HEAP_PROFILER_INTERVAL);
 	}
-	return Void();
+	co_return;
 }
 
 struct StorageDiskCleaner {
@@ -1689,26 +1572,26 @@ struct TrackRunningStorage {
 	};
 };
 
-ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValueStoreType>>* runningStorages,
-                                                 std::unordered_map<UID, StorageDiskCleaner>* storageCleaners,
-                                                 Future<Void> prevStorageServer,
-                                                 KeyValueStoreType storeType,
-                                                 std::string filename,
-                                                 UID id,
-                                                 LocalityData locality,
-                                                 bool isTss,
-                                                 Reference<AsyncVar<ServerDBInfo> const> db,
-                                                 std::string folder,
-                                                 ActorCollection* filesClosed,
-                                                 int64_t memoryLimit,
-                                                 IKeyValueStore* store,
-                                                 bool validateDataFiles,
-                                                 Promise<Void>* rebootKVStore) {
-	state TrackRunningStorage _(id, storeType, locality, filename, runningStorages, storageCleaners);
-	loop {
-		ErrorOr<Void> e = wait(errorOr(prevStorageServer));
+Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValueStoreType>>* runningStorages,
+                                           std::unordered_map<UID, StorageDiskCleaner>* storageCleaners,
+                                           Future<Void> prevStorageServer,
+                                           KeyValueStoreType storeType,
+                                           std::string filename,
+                                           UID id,
+                                           LocalityData locality,
+                                           bool isTss,
+                                           Reference<AsyncVar<ServerDBInfo> const> db,
+                                           std::string folder,
+                                           ActorCollection* filesClosed,
+                                           int64_t memoryLimit,
+                                           IKeyValueStore* store,
+                                           bool validateDataFiles,
+                                           Promise<Void>* rebootKVStore) {
+	TrackRunningStorage _(id, storeType, locality, filename, runningStorages, storageCleaners);
+	while (true) {
+		ErrorOr<Void> e = co_await errorOr(prevStorageServer);
 		if (!e.isError())
-			return Void();
+			co_return;
 		else if (e.getError().code() != error_code_please_reboot &&
 		         e.getError().code() != error_code_please_reboot_kv_store)
 			throw e.getError();
@@ -1720,21 +1603,9 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
 		if (e.getError().code() == error_code_please_reboot_kv_store) {
 			// Add the to actorcollection to make sure filesClosed not return
 			filesClosed->add(rebootKVStore->getFuture());
-			wait(delay(SERVER_KNOBS->REBOOT_KV_STORE_DELAY));
+			co_await delay(SERVER_KNOBS->REBOOT_KV_STORE_DELAY);
 			// reopen KV store
-			store = openKVStore(
-			    storeType,
-			    filename,
-			    id,
-			    memoryLimit,
-			    false,
-			    validateDataFiles,
-			    SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled */
-			        (g_network->isSimulated()
-			             ? (/* Disable for RocksDB */ storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
-			                deterministicRandom()->coinflip())
-			             : true),
-			    db);
+			store = openKVStore(storeType, filename, id, memoryLimit, false, validateDataFiles, db);
 			Promise<Void> nextRebootKVStorePromise;
 			filesClosed->add(store->onClosed() ||
 			                 nextRebootKVStorePromise
@@ -1778,101 +1649,16 @@ ACTOR Future<Void> storageServerRollbackRebooter(std::set<std::pair<UID, KeyValu
 	}
 }
 
-// FIXME:  This will not work correctly in simulation as all workers would share the same roles map
-std::set<std::pair<std::string, std::string>> g_roles;
-
-Standalone<StringRef> roleString(std::set<std::pair<std::string, std::string>> roles, bool with_ids) {
-	std::string result;
-	for (auto& r : roles) {
-		if (!result.empty())
-			result.append(",");
-		result.append(r.first);
-		if (with_ids) {
-			result.append(":");
-			result.append(r.second);
-		}
-	}
-	return StringRef(result);
-}
-
-void startRole(const Role& role,
-               UID roleId,
-               UID workerId,
-               const std::map<std::string, std::string>& details,
-               const std::string& origination) {
-	if (role.includeInTraceRoles) {
-		addTraceRole(role.abbreviation);
-	}
-
-	TraceEvent ev("Role", roleId);
-	ev.detail("As", role.roleName)
-	    .detail("Transition", "Begin")
-	    .detail("Origination", origination)
-	    .detail("OnWorker", workerId);
-	for (auto it = details.begin(); it != details.end(); it++)
-		ev.detail(it->first.c_str(), it->second);
-
-	ev.trackLatest(roleId.shortString() + ".Role");
-
-	// Update roles map, log Roles metrics
-	g_roles.insert({ role.roleName, roleId.shortString() });
-	StringMetricHandle("Roles"_sr) = roleString(g_roles, false);
-	StringMetricHandle("RolesWithIDs"_sr) = roleString(g_roles, true);
-	if (g_network->isSimulated())
-		g_simulator->addRole(g_network->getLocalAddress(), role.roleName);
-}
-
-void endRole(const Role& role, UID id, std::string reason, bool ok, Error e) {
-	{
-		TraceEvent ev("Role", id);
-		if (e.code() != invalid_error_code)
-			ev.errorUnsuppressed(e);
-		ev.detail("Transition", "End").detail("As", role.roleName).detail("Reason", reason);
-
-		ev.trackLatest(id.shortString() + ".Role");
-	}
-
-	if (!ok) {
-		std::string type = role.roleName + "Failed";
-
-		TraceEvent err(SevError, type.c_str(), id);
-		if (e.code() != invalid_error_code) {
-			err.errorUnsuppressed(e);
-		}
-		err.detail("Reason", reason);
-	}
-
-	latestEventCache.clear(id.shortString());
-
-	// Update roles map, log Roles metrics
-	g_roles.erase({ role.roleName, id.shortString() });
-	StringMetricHandle("Roles"_sr) = roleString(g_roles, false);
-	StringMetricHandle("RolesWithIDs"_sr) = roleString(g_roles, true);
-	if (g_network->isSimulated())
-		g_simulator->removeRole(g_network->getLocalAddress(), role.roleName);
-
-	if (role.includeInTraceRoles) {
-		removeTraceRole(role.abbreviation);
-	}
-}
-
-ACTOR Future<Void> traceRole(Role role, UID roleId) {
-	loop {
-		wait(delay(SERVER_KNOBS->WORKER_LOGGING_INTERVAL));
-		TraceEvent("Role", roleId).detail("Transition", "Refresh").detail("As", role.roleName);
-	}
-}
-
-ACTOR Future<Void> workerSnapCreate(
+Future<Void> workerSnapCreate(
     WorkerSnapRequest snapReq,
     std::string snapFolder,
     std::map<std::string, WorkerSnapRequest>* snapReqMap /* ongoing snapshot requests */,
     std::map<std::string, ErrorOr<Void>>*
         snapReqResultMap /* finished snapshot requests, expired in SNAP_MINIMUM_TIME_GAP seconds */) {
-	state ExecCmdValueString snapArg(snapReq.snapPayload);
-	state std::string snapReqKey = snapReq.snapUID.toString() + snapReq.role.toString();
+	ExecCmdValueString snapArg(snapReq.snapPayload);
+	std::string snapReqKey = snapReq.snapUID.toString() + snapReq.role.toString();
 	try {
-		int err = wait(execHelper(&snapArg, snapReq.snapUID, snapFolder, snapReq.role.toString()));
+		int err = co_await execHelper(&snapArg, snapReq.snapUID, snapFolder, snapReq.role.toString());
 		std::string uidStr = snapReq.snapUID.toString();
 		TraceEvent("ExecTraceWorker")
 		    .detail("Uid", uidStr)
@@ -1899,19 +1685,16 @@ ACTOR Future<Void> workerSnapCreate(
 			throw e;
 		}
 	}
-	return Void();
+	co_return;
 }
 
-// TODO: `issues` is right now only updated by `monitorTraceLogIssues` and thus is being `set` on every update.
-// It could be changed to `insert` and `trigger` later if we want to use it as a generic way for the caller of this
-// function to report issues to cluster controller.
-ACTOR Future<Void> monitorTraceLogIssues(Reference<AsyncVar<std::set<std::string>>> issues) {
-	state bool pingTimeout = false;
-	loop {
-		wait(delay(SERVER_KNOBS->TRACE_LOG_FLUSH_FAILURE_CHECK_INTERVAL_SECONDS));
+Future<Void> monitorTraceLogIssues(Reference<AsyncVar<std::set<std::string>>> traceLogIssues) {
+	bool pingTimeout = false;
+	while (true) {
+		co_await delay(SERVER_KNOBS->TRACE_LOG_FLUSH_FAILURE_CHECK_INTERVAL_SECONDS);
 		Future<Void> pingAck = pingTraceLogWriterThread();
 		try {
-			wait(timeoutError(pingAck, SERVER_KNOBS->TRACE_LOG_PING_TIMEOUT_SECONDS));
+			co_await timeoutError(pingAck, SERVER_KNOBS->TRACE_LOG_PING_TIMEOUT_SECONDS);
 		} catch (Error& e) {
 			if (e.code() == error_code_timed_out) {
 				pingTimeout = true;
@@ -1926,7 +1709,55 @@ ACTOR Future<Void> monitorTraceLogIssues(Reference<AsyncVar<std::set<std::string
 			_issues.insert("trace_log_writer_thread_unresponsive");
 			pingTimeout = false;
 		}
-		issues->set(_issues);
+		traceLogIssues->set(_issues);
+	}
+}
+
+static const std::string excludeFromTLogRecruitmentLowDiskIssue = "exclude_from_tlog_recruitment_low_disk";
+
+Future<Void> monitorTLogIssues(std::string folder,
+                               Reference<AsyncVar<bool>> lowDiskTLogExclusion,
+                               Reference<AsyncVar<std::set<std::string>>> tlogIssues) {
+	while (true) {
+		std::set<std::string> currentIssues;
+		if (lowDiskTLogExclusion->get()) {
+			int64_t free = 0;
+			int64_t total = 0;
+			g_network->getDiskBytes(folder, free, total);
+			const double minAvailableSpaceRatio = effectiveTLogMinAvailableSpaceRatio();
+			const bool recovered =
+			    minAvailableSpaceRatio <= 0.0 || total <= 0 || double(free) / total >= minAvailableSpaceRatio;
+			if (recovered) {
+				lowDiskTLogExclusion->set(false);
+			} else {
+				currentIssues.insert(excludeFromTLogRecruitmentLowDiskIssue);
+			}
+		}
+
+		if (tlogIssues->get() != currentIssues) {
+			tlogIssues->set(currentIssues);
+		}
+
+		if (lowDiskTLogExclusion->get()) {
+			co_await timeout(lowDiskTLogExclusion->onChange(), 1.0);
+		} else {
+			co_await lowDiskTLogExclusion->onChange();
+		}
+	}
+}
+
+Future<Void> combineWorkerIssues(Reference<AsyncVar<std::set<std::string>> const> traceLogIssues,
+                                 Reference<AsyncVar<std::set<std::string>> const> tlogIssues,
+                                 Reference<AsyncVar<std::set<std::string>>> issues) {
+	while (true) {
+		std::set<std::string> mergedIssues = traceLogIssues->get();
+		const auto& currentTLogIssues = tlogIssues->get();
+		mergedIssues.insert(currentTLogIssues.begin(), currentTLogIssues.end());
+		if (issues->get() != mergedIssues) {
+			issues->set(mergedIssues);
+		}
+
+		co_await race(traceLogIssues->onChange(), tlogIssues->onChange());
 	}
 }
 
@@ -1958,17 +1789,17 @@ struct SharedLogsValue {
 	  : actor(actor), uid(uid), requests(requests) {}
 };
 
-ACTOR Future<Void> chaosMetricsLogger() {
+Future<Void> chaosMetricsLogger() {
 
 	auto res = g_network->global(INetwork::enChaosMetrics);
 	if (!res)
-		return Void();
+		co_return;
 
-	state ChaosMetrics* chaosMetrics = static_cast<ChaosMetrics*>(res);
+	ChaosMetrics* chaosMetrics = static_cast<ChaosMetrics*>(res);
 	chaosMetrics->clear();
 
-	loop {
-		wait(delay(FLOW_KNOBS->CHAOS_LOGGING_INTERVAL));
+	while (true) {
+		co_await delay(FLOW_KNOBS->CHAOS_LOGGING_INTERVAL);
 
 		TraceEvent e("ChaosMetrics");
 		double elapsed = now() - chaosMetrics->startTime;
@@ -1981,30 +1812,30 @@ ACTOR Future<Void> chaosMetricsLogger() {
 
 static const std::string clusterIdFilename = "clusterId";
 
-ACTOR Future<Void> createClusterIdFile(std::string folder, UID clusterId) {
-	state std::string clusterIdPath = joinPath(folder, clusterIdFilename);
+Future<Void> createClusterIdFile(std::string folder, UID clusterId) {
+	std::string clusterIdPath = joinPath(folder, clusterIdFilename);
 	if (fileExists(clusterIdPath)) {
-		return Void();
+		co_return;
 	}
-	loop {
+	while (true) {
 		try {
-			state ErrorOr<Reference<IAsyncFile>> clusterIdFile =
-			    wait(errorOr(IAsyncFileSystem::filesystem(g_network)->open(
-			        clusterIdPath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0600)));
+			ErrorOr<Reference<IAsyncFile>> clusterIdFile =
+			    co_await errorOr(IAsyncFileSystem::filesystem(g_network)->open(
+			        clusterIdPath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0600));
 
 			if (clusterIdFile.isError() && clusterIdFile.getError().code() == error_code_file_not_found &&
 			    !fileExists(clusterIdPath)) {
-				Reference<IAsyncFile> _clusterIdFile = wait(IAsyncFileSystem::filesystem()->open(
+				Reference<IAsyncFile> _clusterIdFile = co_await IAsyncFileSystem::filesystem()->open(
 				    clusterIdPath,
 				    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_LOCK |
 				        IAsyncFile::OPEN_READWRITE,
-				    0600));
+				    0600);
 				clusterIdFile = _clusterIdFile;
 				BinaryWriter wr(IncludeVersion());
 				wr << clusterId;
-				wait(clusterIdFile.get()->write(wr.getData(), wr.getLength(), 0));
-				wait(clusterIdFile.get()->sync());
-				return Void();
+				co_await clusterIdFile.get()->write(wr.getData(), wr.getLength(), 0);
+				co_await clusterIdFile.get()->sync();
+				co_return;
 			} else {
 				throw clusterIdFile.getError();
 			}
@@ -2025,66 +1856,68 @@ ACTOR Future<Void> createClusterIdFile(std::string folder, UID clusterId) {
 
 // Updates this processes cluster ID based off the cluster ID received in the
 // ServerDBInfo. Persists the cluster ID to disk if it does not already exist.
-ACTOR Future<Void> updateClusterId(UID ccClusterId, Reference<AsyncVar<Optional<UID>>> clusterId, std::string folder) {
+Future<Void> updateClusterId(UID ccClusterId, Reference<AsyncVar<Optional<UID>>> clusterId, std::string folder) {
 	if (!clusterId->get().present() && ccClusterId.isValid()) {
-		wait(createClusterIdFile(folder, ccClusterId));
+		co_await createClusterIdFile(folder, ccClusterId);
 		clusterId->set(ccClusterId);
 	}
-	return Void();
+	co_return;
 }
 
-ACTOR Future<Void> deleteStorageFile(KeyValueStoreType storeType,
-                                     std::string filename,
-                                     UID storeID,
-                                     int64_t memoryLimit,
-                                     Reference<AsyncVar<ServerDBInfo>> dbInfo) {
-	state IKeyValueStore* kvs = openKVStore(storeType, filename, storeID, memoryLimit, false, false, false, dbInfo, {});
-	wait(ready(kvs->init()));
+Future<Void> deleteStorageFile(KeyValueStoreType storeType,
+                               std::string filename,
+                               UID storeID,
+                               int64_t memoryLimit,
+                               Reference<AsyncVar<ServerDBInfo>> dbInfo) {
+	IKeyValueStore* kvs = openKVStore(storeType, filename, storeID, memoryLimit, false, false, dbInfo, {});
+	co_await ready(kvs->init());
 	TraceEvent("KVSRemoved").detail("Reason", "WorkerRemoved");
 	kvs->dispose();
 	CODE_PROBE(true, "Removed stale disk file");
 	TraceEvent("RemoveStorageDisk").detail("Filename", filename).detail("StoreID", storeID);
-	return Void();
+	co_return;
 }
 
-ACTOR Future<Void> cleanupStaleStorageDisk(Reference<AsyncVar<ServerDBInfo>> dbInfo,
-                                           std::unordered_map<UID, StorageDiskCleaner>* cleaners,
-                                           UID storeID,
-                                           StorageDiskCleaner cleaner,
-                                           int64_t memoryLimit) {
-	state int retries = 0;
-	loop {
+Future<Void> cleanupStaleStorageDisk(Reference<AsyncVar<ServerDBInfo>> dbInfo,
+                                     std::unordered_map<UID, StorageDiskCleaner>* cleaners,
+                                     UID storeID,
+                                     StorageDiskCleaner cleaner,
+                                     int64_t memoryLimit) {
+	int retries = 0;
+	while (true) {
+		Optional<Error> err;
 		try {
 			if (retries > SERVER_KNOBS->STORAGE_DISK_CLEANUP_MAX_RETRIES) {
 				TraceEvent("SkipDiskCleanup").detail("Filename", cleaner.filename).detail("StoreID", storeID);
-				return Void();
+				co_return;
 			}
 
 			TraceEvent("StorageServerLivenessCheck").detail("StoreID", storeID).detail("Retry", retries);
 			Reference<CommitProxyInfo> commitProxies(new CommitProxyInfo(dbInfo->get().client.commitProxies));
 			if (commitProxies->size() == 0) {
 				TraceEvent("SkipDiskCleanup").log();
-				return Void();
+				co_return;
 			}
 			GetStorageServerRejoinInfoRequest request(storeID, cleaner.locality.dcId());
 			GetStorageServerRejoinInfoReply _rep =
-			    wait(basicLoadBalance(commitProxies, &CommitProxyInterface::getStorageServerRejoinInfo, request));
+			    co_await basicLoadBalance(commitProxies, &CommitProxyInterface::getStorageServerRejoinInfo, request);
 			// a successful response means the storage server is still alive
 			retries++;
 		} catch (Error& e) {
-			// error worker_removed indicates the storage server has been removed, so it's safe to delete its data
-			if (e.code() == error_code_worker_removed) {
-				// delete the files on disk
-				if (fileExists(cleaner.filename)) {
-					wait(deleteStorageFile(cleaner.storeType, cleaner.filename, storeID, memoryLimit, dbInfo));
-				}
-
-				// remove the cleaner
-				cleaners->erase(storeID);
-				return Void();
+			err = e;
+		}
+		// error worker_removed indicates the storage server has been removed, so it's safe to delete its data
+		if (err.present() && err.get().code() == error_code_worker_removed) {
+			// delete the files on disk
+			if (fileExists(cleaner.filename)) {
+				co_await deleteStorageFile(cleaner.storeType, cleaner.filename, storeID, memoryLimit, dbInfo);
 			}
+
+			// remove the cleaner
+			cleaners->erase(storeID);
+			co_return;
 		}
-		wait(delay(SERVER_KNOBS->STORAGE_DISK_CLEANUP_RETRY_INTERVAL));
+		co_await delay(SERVER_KNOBS->STORAGE_DISK_CLEANUP_RETRY_INTERVAL);
 	}
 }
 
@@ -2110,25 +1943,26 @@ bool skipInitRspInSim(const UID workerInterfID, const bool allowDropInSim) {
 }
 
 #ifdef FLOW_GRPC_ENABLED
-ACTOR Future<Void> registerWorkerGrpcServices(UID id, Reference<IClusterConnectionRecord> ccr) {
+Future<Void> registerWorkerGrpcServices(UID id, Reference<IClusterConnectionRecord> ccr) {
 	if (GrpcServer::instance() == nullptr) {
-		return Never();
+		co_await Future<Void>(Never());
 	} else if (g_network->isSimulated()) {
-		return Never();
+		co_await Future<Void>(Never());
 	}
 
 	auto db = Database::createDatabase(ccr, ApiVersion::LATEST_VERSION);
-	Reference<IDatabase> idb = wait(safeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(db)));
+	Reference<IDatabase> idb = co_await safeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(db));
 	auto services = GrpcServer::ServiceList{ std::make_shared<fdbctl::ControlServiceImpl>(idb) };
 	GrpcServer::instance()->registerRoleServices(UID(), services);
 	TraceEvent("WorkerGrpcServerStart").detail("Address", GrpcServer::instance()->getAddress());
-	return Never();
+	co_await Future<Void>(Never());
 }
 #else
-ACTOR Future<Void> registerWorkerGrpcServices(UID id, Reference<IClusterConnectionRecord> ccr) {
+Future<Void> registerWorkerGrpcServices(UID id, Reference<IClusterConnectionRecord> ccr) {
 	return Never();
 }
 #endif
+} // namespace
 
 ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
                                 Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
@@ -2163,6 +1997,10 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	state Future<Void> metricsLogger;
 	state Future<Void> chaosMetricsActor;
 	state Reference<AsyncVar<bool>> degraded = FlowTransport::transport().getDegraded();
+	state Reference<AsyncVar<std::set<std::string>>> issues(new AsyncVar<std::set<std::string>>());
+	state Reference<AsyncVar<std::set<std::string>>> traceLogIssues(new AsyncVar<std::set<std::string>>());
+	state Reference<AsyncVar<std::set<std::string>>> tlogIssues(new AsyncVar<std::set<std::string>>());
+	state Reference<AsyncVar<bool>> lowDiskTLogExclusion(new AsyncVar<bool>(false));
 	// tLogFnForOptions() can return a function that doesn't correspond with the FDB version that the
 	// TLogVersion represents.  This can be done if the newer TLog doesn't support a requested option.
 	// As (store type, spill type) can map to the same TLogFn across multiple TLogVersions, we need to
@@ -2172,7 +2010,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	state Reference<AsyncVar<UID>> activeSharedTLog(new AsyncVar<UID>());
 	state WorkerCache<InitializeBackupReply> backupWorkerCache;
 	state WorkerCache<TLogInterface> logRouterCache;
-	state Future<Void> blobWorkerFuture = Void();
 
 	state WorkerSnapRequest lastSnapReq;
 	// Here the key is UID+role, as we still send duplicate requests to a process which is both storage and tlog
@@ -2192,8 +2029,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 	interf.initEndpoints();
 
-	state Reference<AsyncVar<std::set<std::string>>> issues(new AsyncVar<std::set<std::string>>());
-
 	state Future<Void> updateClusterIdFuture;
 
 	// When set to true, the health monitor running in this worker starts monitor other transaction process in this
@@ -2238,7 +2073,9 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	                               "DegradedReset"));
 	errorForwarders.add(loadedPonger(interf.debugPing.getFuture()));
 	errorForwarders.add(waitFailureServer(interf.waitFailure.getFuture()));
-	errorForwarders.add(monitorTraceLogIssues(issues));
+	errorForwarders.add(monitorTraceLogIssues(traceLogIssues));
+	errorForwarders.add(monitorTLogIssues(folder, lowDiskTLogExclusion, tlogIssues));
+	errorForwarders.add(combineWorkerIssues(traceLogIssues, tlogIssues, issues));
 	errorForwarders.add(
 	    testerServerCore(interf.testerInterface,
 	                     connRecord,
@@ -2296,21 +2133,14 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				LocalLineage _;
 				getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::ClusterRole::Storage;
 
-				IKeyValueStore* kv = openKVStore(
-				    s.storeType,
-				    s.filename,
-				    s.storeID,
-				    memoryLimit,
-				    false,
-				    validateDataFiles,
-				    SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled */
-				        (g_network->isSimulated()
-				             ? (/* Disable for RocksDB */ s.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
-				                s.storeType != KeyValueStoreType::SSD_SHARDED_ROCKSDB &&
-				                deterministicRandom()->coinflip())
-				             : true),
-				    dbInfo,
-				    /* document constants =*/0);
+				IKeyValueStore* kv = openKVStore(s.storeType,
+				                                 s.filename,
+				                                 s.storeID,
+				                                 memoryLimit,
+				                                 false,
+				                                 validateDataFiles,
+				                                 dbInfo,
+				                                 /* document constants =*/0);
 				Future<Void> kvClosed =
 				    kv->onClosed() ||
 				    rebootKVSPromise.getFuture() /* clear the onClosed() Future in actorCollection when rebooting */;
@@ -2391,8 +2221,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					logQueueBasename = fileLogQueuePrefix.toString() + optionsString.toString() + "-";
 				}
 				ASSERT_WE_THINK(abspath(parentDirectory(s.filename)) == folder);
-				IKeyValueStore* kv = openKVStore(
-				    s.storeType, s.filename, s.storeID, memoryLimit, validateDataFiles, false, false, dbInfo);
+				IKeyValueStore* kv =
+				    openKVStore(s.storeType, s.filename, s.storeID, memoryLimit, validateDataFiles, false, dbInfo);
 				const DiskQueueVersion dqv = s.tLogOptions.getDiskQueueVersion();
 				const int64_t diskQueueWarnSize =
 				    s.tLogOptions.spillType == TLogSpillType::VALUE ? 10 * SERVER_KNOBS->TARGET_BYTES_PER_TLOG : -1;
@@ -2428,6 +2258,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				                         recovery,
 				                         folder,
 				                         degraded,
+				                         lowDiskTLogExclusion,
 				                         activeSharedTLog,
 				                         enablePrimaryTxnSystemHealthCheck);
 				recoveries.push_back(recovery.getFuture());
@@ -2489,8 +2320,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 		loop choose {
 			when(UpdateServerDBInfoRequest req = waitNext(interf.updateServerDBInfo.getFuture())) {
-				ServerDBInfo localInfo = BinaryReader::fromStringRef<ServerDBInfo>(
-				    req.serializedDbInfo, AssumeVersion(g_network->protocolVersion()));
+				auto localInfo = BinaryReader::fromStringRef<ServerDBInfo>(req.serializedDbInfo,
+				                                                           AssumeVersion(g_network->protocolVersion()));
 				localInfo.myLocality = locality;
 
 				if (localInfo.infoGeneration < dbInfo->get().infoGeneration &&
@@ -2710,7 +2541,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					InitializeBackupReply reply(recruited, req.backupEpoch);
 					backupReady.send(reply);
 				} else {
-					forwardPromise(req.reply, backupWorkerCache.get(req.reqId));
+					forwardPromise(Uncancellable{}, req.reply, backupWorkerCache.get(req.reqId));
 				}
 			}
 			when(InitializeTLogRequest req = waitNext(interf.tLog.getFuture())) {
@@ -2747,7 +2578,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					std::string filename =
 					    filenameFromId(req.storeType, folder, prefix.toString() + tLogOptions.toPrefix(), logId);
 					IKeyValueStore* data =
-					    openKVStore(req.storeType, filename, logId, memoryLimit, false, false, false, dbInfo);
+					    openKVStore(req.storeType, filename, logId, memoryLimit, false, false, dbInfo);
 					const DiskQueueVersion dqv = tLogOptions.getDiskQueueVersion();
 					IDiskQueue* queue = openDiskQueue(
 					    joinPath(folder,
@@ -2771,6 +2602,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					                               Promise<Void>(),
 					                               folder,
 					                               degraded,
+					                               lowDiskTLogExclusion,
 					                               activeSharedTLog,
 					                               enablePrimaryTxnSystemHealthCheck);
 					tLogCore = handleIOErrors(tLogCore, data, logId);
@@ -2860,21 +2692,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					                   folder,
 					                   isTss ? testingStoragePrefix.toString() : fileStoragePrefix.toString(),
 					                   recruited.id());
-					IKeyValueStore* data = openKVStore(
-					    req.storeType,
-					    filename,
-					    recruited.id(),
-					    memoryLimit,
-					    false,
-					    false,
-					    SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled */
-					        (g_network->isSimulated()
-					             ? (/* Disable for RocksDB */ req.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
-					                req.storeType != KeyValueStoreType::SSD_SHARDED_ROCKSDB &&
-					                deterministicRandom()->coinflip())
-					             : true),
-					    dbInfo,
-					    0);
+					IKeyValueStore* data =
+					    openKVStore(req.storeType, filename, recruited.id(), memoryLimit, false, false, dbInfo, 0);
 					TraceEvent("StorageServerInitProgress", recruited.id())
 					    .detail("ReqID", req.reqId)
 					    .detail("StorageType", req.storeType.toString())
@@ -2934,7 +2753,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				startRole(Role::COMMIT_PROXY, recruited.id(), interf.id(), details);
 
 				DUMPTOKEN(recruited.commit);
-				DUMPTOKEN(recruited.getConsistentReadVersion);
 				DUMPTOKEN(recruited.getKeyServersLocations);
 				DUMPTOKEN(recruited.getStorageServerRejoinInfo);
 				DUMPTOKEN(recruited.waitFailure);
@@ -3023,7 +2841,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 						logRouterReady.send(recruited);
 					}
 				} else {
-					forwardPromise(req.reply, logRouterCache.get(req.reqId));
+					forwardPromise(Uncancellable{}, req.reply, logRouterCache.get(req.reqId));
 				}
 			}
 			when(CoordinationPingMessage m = waitNext(interf.coordinationPing.getFuture())) {
@@ -3163,9 +2981,6 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 		endRole(Role::WORKER, interf.id(), "WorkerError", ok, e);
 		errorForwarders.clear(false);
 		sharedLogs.clear();
-		// blobWorkerFuture is also in errorForwarders so it's double refcounted. If we don't cancel it, it'll never
-		// close it's IKVS and this will hang, leaving a zombie worker
-		blobWorkerFuture.cancel();
 
 		if (e.code() != error_code_actor_cancelled) {
 			// actor_cancelled:
@@ -3182,18 +2997,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	}
 }
 
-ACTOR Future<Void> extractClusterInterface(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> in,
-                                           Reference<AsyncVar<Optional<ClusterInterface>>> out) {
-	loop {
-		if (in->get().present()) {
-			out->set(in->get().get().clientInterface);
-		} else {
-			out->set(Optional<ClusterInterface>());
-		}
-		wait(in->onChange());
-	}
-}
-
+namespace {
 static std::set<int> const& normalWorkerErrors() {
 	static std::set<int> s;
 	if (s.empty()) {
@@ -3205,13 +3009,13 @@ static std::set<int> const& normalWorkerErrors() {
 	return s;
 }
 
-ACTOR Future<Void> printTimeout() {
-	wait(delay(5));
+Future<Void> printTimeout() {
+	co_await delay(5);
 	if (!g_network->isSimulated()) {
 		fprintf(stderr, "Warning: FDBD has not joined the cluster after 5 seconds.\n");
 		fprintf(stderr, "  Check configuration and availability using the 'status' command with the fdbcli\n");
 	}
-	return Void();
+	co_return;
 }
 
 ACTOR Future<Void> printOnFirstConnected(Reference<AsyncVar<Optional<ClusterInterface>> const> ci) {
@@ -3255,10 +3059,10 @@ ClusterControllerPriorityInfo getCCPriorityInfo(std::string filePath, ProcessCla
 	return priorityInfo;
 }
 
-ACTOR Future<Void> monitorAndWriteCCPriorityInfo(std::string filePath,
-                                                 Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo) {
-	loop {
-		wait(asyncPriorityInfo->onChange());
+Future<Void> monitorAndWriteCCPriorityInfo(std::string filePath,
+                                           Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo) {
+	while (true) {
+		co_await asyncPriorityInfo->onChange();
 		std::string contents(BinaryWriter::toValue(asyncPriorityInfo->get(),
 		                                           IncludeVersion(ProtocolVersion::withClusterControllerPriorityInfo()))
 		                         .toString());
@@ -3268,11 +3072,11 @@ ACTOR Future<Void> monitorAndWriteCCPriorityInfo(std::string filePath,
 
 static const std::string versionFileName = "sw-version";
 
-ACTOR Future<SWVersion> testSoftwareVersionCompatibility(std::string folder, ProtocolVersion currentVersion) {
+Future<SWVersion> testSoftwareVersionCompatibility(std::string folder, ProtocolVersion currentVersion) {
 	try {
-		state std::string versionFilePath = joinPath(folder, versionFileName);
-		state ErrorOr<Reference<IAsyncFile>> versionFile = wait(
-		    errorOr(IAsyncFileSystem::filesystem(g_network)->open(versionFilePath, IAsyncFile::OPEN_READONLY, 0600)));
+		std::string versionFilePath = joinPath(folder, versionFileName);
+		ErrorOr<Reference<IAsyncFile>> versionFile = co_await errorOr(
+		    IAsyncFileSystem::filesystem(g_network)->open(versionFilePath, IAsyncFile::OPEN_READONLY, 0600));
 
 		if (versionFile.isError()) {
 			if (versionFile.getError().code() == error_code_file_not_found && !fileExists(versionFilePath)) {
@@ -3280,7 +3084,7 @@ ACTOR Future<SWVersion> testSoftwareVersionCompatibility(std::string folder, Pro
 				// installation or an upgrade from a version that does not support version files.
 				// Either way, we can safely continue running this version of software.
 				TraceEvent(SevInfo, "NoPreviousSWVersion").log();
-				return SWVersion();
+				co_return SWVersion();
 			} else {
 				// Dangerous to continue if we cannot do a software compatibility test
 				throw versionFile.getError();
@@ -3288,9 +3092,9 @@ ACTOR Future<SWVersion> testSoftwareVersionCompatibility(std::string folder, Pro
 		} else {
 			// Test whether the most newest software version that has been run on this cluster is
 			// compatible with the current software version
-			state int64_t filesize = wait(versionFile.get()->size());
-			state Standalone<StringRef> buf = makeString(filesize);
-			int readLen = wait(versionFile.get()->read(mutateString(buf), filesize, 0));
+			int64_t filesize = co_await versionFile.get()->size();
+			Standalone<StringRef> buf = makeString(filesize);
+			int readLen = co_await versionFile.get()->read(mutateString(buf), filesize, 0);
 			if (filesize == 0 || readLen != filesize) {
 				throw file_corrupt();
 			}
@@ -3299,7 +3103,7 @@ ACTOR Future<SWVersion> testSoftwareVersionCompatibility(std::string folder, Pro
 				SWVersion swversion = ObjectReader::fromStringRef<SWVersion>(buf, IncludeVersion());
 				ProtocolVersion lowestCompatibleVersion(swversion.lowestCompatibleProtocolVersion());
 				if (currentVersion >= lowestCompatibleVersion) {
-					return swversion;
+					co_return swversion;
 				} else {
 					throw incompatible_software_version();
 				}
@@ -3317,35 +3121,35 @@ ACTOR Future<SWVersion> testSoftwareVersionCompatibility(std::string folder, Pro
 	}
 }
 
-ACTOR Future<Void> updateNewestSoftwareVersion(std::string folder,
-                                               ProtocolVersion currentVersion,
-                                               ProtocolVersion latestVersion,
-                                               ProtocolVersion minCompatibleVersion) {
+Future<Void> updateNewestSoftwareVersion(std::string folder,
+                                         ProtocolVersion currentVersion,
+                                         ProtocolVersion latestVersion,
+                                         ProtocolVersion minCompatibleVersion) {
 
 	ASSERT(currentVersion >= minCompatibleVersion);
 
 	try {
-		state std::string versionFilePath = joinPath(folder, versionFileName);
-		ErrorOr<Reference<IAsyncFile>> versionFile = wait(
-		    errorOr(IAsyncFileSystem::filesystem(g_network)->open(versionFilePath, IAsyncFile::OPEN_READONLY, 0600)));
+		std::string versionFilePath = joinPath(folder, versionFileName);
+		ErrorOr<Reference<IAsyncFile>> versionFile = co_await errorOr(
+		    IAsyncFileSystem::filesystem(g_network)->open(versionFilePath, IAsyncFile::OPEN_READONLY, 0600));
 
 		if (versionFile.isError() &&
 		    (versionFile.getError().code() != error_code_file_not_found || fileExists(versionFilePath))) {
 			throw versionFile.getError();
 		}
 
-		state Reference<IAsyncFile> newVersionFile = wait(IAsyncFileSystem::filesystem()->open(
+		Reference<IAsyncFile> newVersionFile = co_await IAsyncFileSystem::filesystem()->open(
 		    versionFilePath,
 		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE,
-		    0600));
+		    0600);
 
 		SWVersion swVersion(latestVersion, currentVersion, minCompatibleVersion);
 		Value s = swVersionValue(swVersion);
-		ErrorOr<Void> e = wait(holdWhile(s, errorOr(newVersionFile->write(s.begin(), s.size(), 0))));
+		ErrorOr<Void> e = co_await holdWhile(s, errorOr(newVersionFile->write(s.begin(), s.size(), 0)));
 		if (e.isError()) {
 			throw e.getError();
 		}
-		wait(newVersionFile->sync());
+		co_await newVersionFile->sync();
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
 			throw;
@@ -3354,12 +3158,12 @@ ACTOR Future<Void> updateNewestSoftwareVersion(std::string folder,
 		throw;
 	}
 
-	return Void();
+	co_return;
 }
 
-ACTOR Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFolder, UID processIDUid) {
+Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFolder, UID processIDUid) {
 	ErrorOr<SWVersion> swVersion =
-	    wait(errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion())));
+	    co_await errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion()));
 	if (swVersion.isError()) {
 		TraceEvent(SevWarnAlways, "SWVersionCompatibilityCheckError", processIDUid).error(swVersion.getError());
 		throw swVersion.getError();
@@ -3369,24 +3173,24 @@ ACTOR Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFol
 
 	if (!swVersion.get().isValid() ||
 	    currentProtocolVersion() > ProtocolVersion(swVersion.get().newestProtocolVersion())) {
-		ErrorOr<Void> updatedSWVersion = wait(errorOr(updateNewestSoftwareVersion(
-		    dataFolder, currentProtocolVersion(), currentProtocolVersion(), minCompatibleProtocolVersion)));
+		ErrorOr<Void> updatedSWVersion = co_await errorOr(updateNewestSoftwareVersion(
+		    dataFolder, currentProtocolVersion(), currentProtocolVersion(), minCompatibleProtocolVersion));
 		if (updatedSWVersion.isError()) {
 			throw updatedSWVersion.getError();
 		}
 	} else if (currentProtocolVersion() < ProtocolVersion(swVersion.get().newestProtocolVersion())) {
-		ErrorOr<Void> updatedSWVersion = wait(
-		    errorOr(updateNewestSoftwareVersion(dataFolder,
-		                                        currentProtocolVersion(),
-		                                        ProtocolVersion(swVersion.get().newestProtocolVersion()),
-		                                        ProtocolVersion(swVersion.get().lowestCompatibleProtocolVersion()))));
+		ErrorOr<Void> updatedSWVersion = co_await errorOr(
+		    updateNewestSoftwareVersion(dataFolder,
+		                                currentProtocolVersion(),
+		                                ProtocolVersion(swVersion.get().newestProtocolVersion()),
+		                                ProtocolVersion(swVersion.get().lowestCompatibleProtocolVersion())));
 		if (updatedSWVersion.isError()) {
 			throw updatedSWVersion.getError();
 		}
 	}
 
 	ErrorOr<SWVersion> newSWVersion =
-	    wait(errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion())));
+	    co_await errorOr(testSoftwareVersionCompatibility(dataFolder, currentProtocolVersion()));
 	if (newSWVersion.isError()) {
 		TraceEvent(SevWarnAlways, "SWVersionCompatibilityCheckError", processIDUid).error(newSWVersion.getError());
 		throw newSWVersion.getError();
@@ -3394,7 +3198,7 @@ ACTOR Future<Void> testAndUpdateSoftwareVersionCompatibility(std::string dataFol
 
 	TraceEvent(SevInfo, "VerifiedNewSoftwareVersion", processIDUid).detail("SWVersion", newSWVersion.get());
 
-	return Void();
+	co_return;
 }
 
 static const std::string swversionTestDirName = "sw-version-test";
@@ -3660,47 +3464,51 @@ TEST_CASE("/fdbserver/storageengine/clearInflightCommits") {
 	platform::eraseDirectoryRecursive(testDir);
 	return Void();
 }
-} // anonymous namespace
+} // namespace
 
-ACTOR Future<UID> createAndLockProcessIdFile(std::string folder) {
-	state UID processIDUid;
+Future<UID> createAndLockProcessIdFile(std::string folder) {
+	UID processIDUid;
 	platform::createDirectory(folder);
 
-	loop {
+	while (true) {
 		try {
-			state std::string lockFilePath = joinPath(folder, "processId");
-			state ErrorOr<Reference<IAsyncFile>> lockFile = wait(errorOr(IAsyncFileSystem::filesystem(g_network)->open(
-			    lockFilePath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0600)));
+			std::string lockFilePath = joinPath(folder, "processId");
+			ErrorOr<Reference<IAsyncFile>> lockFile = co_await errorOr(IAsyncFileSystem::filesystem(g_network)->open(
+			    lockFilePath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0600));
 
 			if (lockFile.isError() && lockFile.getError().code() == error_code_file_not_found &&
 			    !fileExists(lockFilePath)) {
-				Reference<IAsyncFile> _lockFile = wait(IAsyncFileSystem::filesystem()->open(
+				Reference<IAsyncFile> _lockFile = co_await IAsyncFileSystem::filesystem()->open(
 				    lockFilePath,
 				    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_LOCK |
 				        IAsyncFile::OPEN_READWRITE,
-				    0600));
+				    0600);
 				lockFile = _lockFile;
 				processIDUid = deterministicRandom()->randomUniqueID();
 				BinaryWriter wr(IncludeVersion(ProtocolVersion::withProcessIDFile()));
 				wr << processIDUid;
-				wait(lockFile.get()->write(wr.getData(), wr.getLength(), 0));
-				wait(lockFile.get()->sync());
+				co_await lockFile.get()->write(wr.getData(), wr.getLength(), 0);
+				co_await lockFile.get()->sync();
 			} else {
 				if (lockFile.isError())
 					throw lockFile.getError(); // If we've failed to open the file, throw an exception
 
-				int64_t fileSize = wait(lockFile.get()->size());
-				state Key fileData = makeString(fileSize);
-				wait(success(lockFile.get()->read(mutateString(fileData), fileSize, 0)));
+				int64_t fileSize = co_await lockFile.get()->size();
+				Key fileData = makeString(fileSize);
+				co_await lockFile.get()->read(mutateString(fileData), fileSize, 0);
+				bool deleteCorruptProcessIdFile = false;
 				try {
 					processIDUid = BinaryReader::fromStringRef<UID>(fileData, IncludeVersion());
-					return processIDUid;
+					co_return processIDUid;
 				} catch (Error& e) {
 					if (!g_network->isSimulated()) {
-						throw;
+						throw e;
 					}
 					lockFile = ErrorOr<Reference<IAsyncFile>>();
-					wait(IAsyncFileSystem::filesystem()->deleteFile(lockFilePath, true));
+					deleteCorruptProcessIdFile = true;
+				}
+				if (deleteCorruptProcessIdFile) {
+					co_await IAsyncFileSystem::filesystem()->deleteFile(lockFilePath, true);
 				}
 			}
 		} catch (Error& e) {
@@ -3718,16 +3526,16 @@ ACTOR Future<UID> createAndLockProcessIdFile(std::string folder) {
 	}
 }
 
-ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGeneration(
+Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGeneration(
     Reference<IClusterConnectionRecord> connRecord,
     Reference<AsyncVar<Value>> result,
     MonitorLeaderInfo info) {
 	ClusterConnectionString cs = info.intermediateConnRecord->getConnectionString();
-	state int coordinatorsSize = cs.hostnames.size() + cs.coords.size();
-	state ElectionResultRequest request;
-	state int index = 0;
-	state int successIndex = 0;
-	state std::vector<LeaderElectionRegInterface> leaderElectionServers;
+	int coordinatorsSize = cs.hostnames.size() + cs.coords.size();
+	ElectionResultRequest request;
+	int index = 0;
+	int successIndex = 0;
+	std::vector<LeaderElectionRegInterface> leaderElectionServers;
 
 	leaderElectionServers.reserve(coordinatorsSize);
 	for (const auto& h : cs.hostnames) {
@@ -3742,18 +3550,17 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 	request.hostnames = cs.hostnames;
 	request.coordinators = cs.coords;
 
-	loop {
+	while (true) {
 		LeaderElectionRegInterface interf = leaderElectionServers[index];
 		bool usingHostname = interf.hostname.present();
 		request.reply = ReplyPromise<Optional<LeaderInfo>>();
 
-		state ErrorOr<Optional<LeaderInfo>> leader;
+		ErrorOr<Optional<LeaderInfo>> leader;
 		if (usingHostname) {
-			wait(store(
-			    leader,
-			    tryGetReplyFromHostname(request, interf.hostname.get(), WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT)));
+			leader = co_await tryGetReplyFromHostname(
+			    request, interf.hostname.get(), WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT);
 		} else {
-			wait(store(leader, interf.electionResult.tryGetReply(request)));
+			leader = co_await interf.electionResult.tryGetReply(request);
 		}
 
 		if (leader.present()) {
@@ -3761,7 +3568,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 				if (leader.get().get().forward) {
 					info.intermediateConnRecord = connRecord->makeIntermediateRecord(
 					    ClusterConnectionString(leader.get().get().serializedInfo.toString()));
-					return info;
+					co_return info;
 				}
 				if (connRecord != info.intermediateConnRecord) {
 					if (!info.hasConnected) {
@@ -3771,7 +3578,8 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 						    .detail("CurrentConnectionString",
 						            info.intermediateConnRecord->getConnectionString().toString());
 					}
-					wait(connRecord->setAndPersistConnectionString(info.intermediateConnRecord->getConnectionString()));
+					co_await connRecord->setAndPersistConnectionString(
+					    info.intermediateConnRecord->getConnectionString());
 					info.intermediateConnRecord = connRecord;
 				}
 
@@ -3793,18 +3601,18 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderWithDelayedCandidacyImplOneGenerati
 		} else {
 			index = (index + 1) % coordinatorsSize;
 			if (index == successIndex) {
-				wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY));
+				co_await delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY);
 			}
 		}
 	}
 }
 
-ACTOR Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClusterConnectionRecord> connRecord,
-                                                                 Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
-	state MonitorLeaderInfo info(connRecord);
-	loop {
+Future<Void> monitorLeaderWithDelayedCandidacyImplInternal(Reference<IClusterConnectionRecord> connRecord,
+                                                           Reference<AsyncVar<Value>> outSerializedLeaderInfo) {
+	MonitorLeaderInfo info(connRecord);
+	while (true) {
 		MonitorLeaderInfo _info =
-		    wait(monitorLeaderWithDelayedCandidacyImplOneGeneration(connRecord, outSerializedLeaderInfo, info));
+		    co_await monitorLeaderWithDelayedCandidacyImplOneGeneration(connRecord, outSerializedLeaderInfo, info);
 		info = _info;
 	}
 }
@@ -3858,12 +3666,12 @@ ACTOR Future<Void> monitorLeaderWithDelayedCandidacy(
 
 extern void setupStackSignal();
 
-ACTOR Future<Void> serveProtocolInfo() {
-	state PublicRequestStream<ProtocolInfoRequest> protocolInfo(
+Future<Void> serveProtocolInfo() {
+	PublicRequestStream<ProtocolInfoRequest> protocolInfo(
 	    PeerCompatibilityPolicy{ RequirePeer::AtLeast, ProtocolVersion::withStableInterfaces() });
 	protocolInfo.makeWellKnownEndpoint(WLTOKEN_PROTOCOL_INFO, TaskPriority::DefaultEndpoint);
-	loop {
-		state ProtocolInfoRequest req = waitNext(protocolInfo.getFuture());
+	while (true) {
+		ProtocolInfoRequest req = co_await protocolInfo.getFuture();
 		req.reply.send(ProtocolInfoReply{ g_network->protocolVersion() });
 	}
 }
@@ -3909,25 +3717,27 @@ Optional<UID> readClusterId(std::string filePath) {
 	br >> clusterId;
 	return clusterId;
 }
+} // anonymous namespace
 
-ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
-                        LocalityData localities,
-                        ProcessClass processClass,
-                        std::string dataFolder,
-                        std::string coordFolder,
-                        int64_t memoryLimit,
-                        std::string metricsConnFile,
-                        std::string metricsPrefix,
-                        int64_t memoryProfileThreshold,
-                        std::string whitelistBinPaths,
-                        bool consistencyCheckUrgentMode) {
-	state std::vector<Future<Void>> actors;
+Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
+                  LocalityData localities,
+                  ProcessClass processClass,
+                  std::string dataFolder,
+                  std::string coordFolder,
+                  int64_t memoryLimit,
+                  std::string metricsConnFile,
+                  std::string metricsPrefix,
+                  int64_t memoryProfileThreshold,
+                  std::string whitelistBinPaths,
+                  bool consistencyCheckUrgentMode) {
+	std::vector<Future<Void>> actors;
 	// setupStackSignal();
 	getCurrentLineage()->modify(&RoleLineage::role) = ProcessClass::Worker;
 
 	actors.push_back(serveProtocolInfo());
 	actors.push_back(serveProcess());
 
+	Error caughtErr;
 	try {
 		ServerCoordinators coordinators(connRecord);
 		if (g_network->isSimulated()) {
@@ -3946,11 +3756,11 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 			actors.push_back(coordinationServer(coordFolder, connRecord));
 		}
 
-		state UID processIDUid = wait(createAndLockProcessIdFile(dataFolder));
+		UID processIDUid = co_await createAndLockProcessIdFile(dataFolder);
 		localities.set(LocalityData::keyProcessId, processIDUid.toString());
 		// Only one process can execute on a dataFolder from this point onwards
 
-		wait(testAndUpdateSoftwareVersionCompatibility(dataFolder, processIDUid));
+		co_await testAndUpdateSoftwareVersionCompatibility(dataFolder, processIDUid);
 
 		std::string fitnessFilePath = joinPath(dataFolder, "fitness");
 		auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
@@ -3996,42 +3806,25 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 		                                    "WorkerServer",
 		                                    UID(),
 		                                    &normalWorkerErrors()));
-		state Future<Void> firstConnect = reportErrors(printOnFirstConnected(ci), "ClusterFirstConnectedError");
+		Future<Void> firstConnect = reportErrors(printOnFirstConnected(ci), "ClusterFirstConnectedError");
 
-		wait(quorum(actors, 1));
+		co_await quorum(actors, 1);
 		ASSERT(false); // None of these actors should terminate normally
 		throw internal_error();
 	} catch (Error& e) {
-		// Make sure actors are cancelled before recoveredDiskFiles is destructed.
-		// Otherwise, these actors may get a broken promise error.
-		for (auto f : actors)
-			f.cancel();
-		state Error err = checkIOTimeout(e);
-		if (e.code() == error_code_invalid_cluster_id) {
-			// If this process tried to join an invalid cluster, become a
-			// zombie and wait for manual action by the operator.
-			TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "ZombieProcess").error(e);
-			wait(Never());
-		}
-		throw err;
-	}
-}
-
-const Role Role::WORKER("Worker", "WK", false);
-const Role Role::STORAGE_SERVER("StorageServer", "SS");
-const Role Role::TESTING_STORAGE_SERVER("TestingStorageServer", "ST");
-const Role Role::TRANSACTION_LOG("TLog", "TL");
-const Role Role::SHARED_TRANSACTION_LOG("SharedTLog", "SL", false);
-const Role Role::COMMIT_PROXY("CommitProxyServer", "CP");
-const Role Role::GRV_PROXY("GrvProxyServer", "GP");
-const Role Role::MASTER("MasterServer", "MS");
-const Role Role::RESOLVER("Resolver", "RV");
-const Role Role::CLUSTER_CONTROLLER("ClusterController", "CC");
-const Role Role::TESTER("Tester", "TS");
-const Role Role::LOG_ROUTER("LogRouter", "LR");
-const Role Role::DATA_DISTRIBUTOR("DataDistributor", "DD");
-const Role Role::RATEKEEPER("Ratekeeper", "RK");
-const Role Role::COORDINATOR("Coordinator", "CD");
-const Role Role::BACKUP("Backup", "BK");
-const Role Role::ENCRYPT_KEY_PROXY("EncryptKeyProxy", "EP");
-const Role Role::CONSISTENCYSCAN("ConsistencyScan", "CS");
+		caughtErr = e;
+	}
+
+	// Make sure actors are cancelled before recoveredDiskFiles is destructed.
+	// Otherwise, these actors may get a broken promise error.
+	for (auto f : actors)
+		f.cancel();
+	Error err = checkIOTimeout(caughtErr);
+	if (caughtErr.code() == error_code_invalid_cluster_id) {
+		// If this process tried to join an invalid cluster, become a
+		// zombie and wait for manual action by the operator.
+		TraceEvent(g_network->isSimulated() ? SevWarnAlways : SevError, "ZombieProcess").error(caughtErr);
+		co_await Future<Void>(Never());
+	}
+	throw err;
+}
diff --git a/fdbserver/workloads/ApiCorrectness.actor.cpp b/fdbserver/workloads/ApiCorrectness.cpp
similarity index 72%
rename from fdbserver/workloads/ApiCorrectness.actor.cpp
rename to fdbserver/workloads/ApiCorrectness.cpp
index d7bf55f59c3..02a829d8119 100644
--- a/fdbserver/workloads/ApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/ApiCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * ApiCorrectness.actor.cpp
+ * ApiCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,16 +19,15 @@
  */
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/QuietDatabase.h"
+#include "fdbserver/core/QuietDatabase.h"
 
-#include "fdbserver/MutationTracking.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/ApiWorkload.h"
-#include "fdbserver/workloads/MemoryKeyValueStore.h"
+#include "fdbserver/core/MutationTracking.h"
+#include "fdbserver/tester/workloads.h"
+#include "ApiWorkload.h"
+#include "MemoryKeyValueStore.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 #include "flow/genericactors.actor.h"
 
 // An enum of API operation types used in the random test
@@ -103,7 +102,7 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 	// Maximum time to reset DB to the original state
 	double resetDBTimeout;
 
-	ApiCorrectnessWorkload(WorkloadContext const& wcx)
+	explicit ApiCorrectnessWorkload(WorkloadContext const& wcx)
 	  : ApiWorkload(wcx), numRandomOperations("Num Random Operations") {
 		numGets = getOption(options, "numGets"_sr, 1000);
 		numGetRanges = getOption(options, "numGetRanges"_sr, 100);
@@ -141,14 +140,14 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		}
 	}
 
-	~ApiCorrectnessWorkload() override {}
+	~ApiCorrectnessWorkload() override = default;
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
 		m.emplace_back("Number of Random Operations Performed", numRandomOperations.getValue(), Averaged::False);
 	}
 
-	ACTOR Future<Void> performSetup(Database cx, ApiCorrectnessWorkload* self) {
-		DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(cx));
+	Future<Void> performSetup(Database cx, ApiCorrectnessWorkload* self) {
+		DatabaseConfiguration dbConfig = co_await getDatabaseConfiguration(cx);
 		// Choose a random transaction type (NativeAPI, ReadYourWrites, ThreadSafe, MultiVersion)
 		std::vector<TransactionType> types;
 		types.push_back(NATIVE);
@@ -156,37 +155,33 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		types.push_back(THREAD_SAFE);
 		types.push_back(MULTI_VERSION);
 
-		wait(self->chooseTransactionFactory(cx, types));
-
-		return Void();
+		co_await self->chooseTransactionFactory(cx, types);
 	}
 
 	Future<Void> performSetup(Database const& cx) override { return performSetup(cx, this); }
 
-	ACTOR Future<Void> performTest(Database cx, Standalone<VectorRef<KeyValueRef>> data, ApiCorrectnessWorkload* self) {
+	Future<Void> performTest(Database cx, Standalone<VectorRef<KeyValueRef>> data, ApiCorrectnessWorkload* self) {
 		// Run the scripted test for a maximum of 10 minutes
-		wait(timeout(self->runScriptedTest(self, data), 600, Void()));
+		co_await timeout(self->runScriptedTest(self, data), 600, Void());
 
 		if (!self->hasFailed()) {
 			// Return database to original state (for a maximum of resetDBTimeout seconds)
 			try {
-				wait(timeoutError(::success(self->runSet(data, self)), self->resetDBTimeout));
+				co_await timeoutError(::success(self->runSet(data, self)), self->resetDBTimeout);
 			} catch (Error& e) {
 				if (e.code() == error_code_timed_out) {
 					if (!self->hasFailed())
 						self->testFailure("Timeout during database reset");
 
-					return Void();
+					co_return;
 				}
 
 				throw;
 			}
 
 			// Run the random test for the user-specified duration
-			wait(timeout(self->runRandomTest(self, data), self->randomTestDuration, Void()));
+			co_await timeout(self->runRandomTest(self, data), self->randomTestDuration, Void());
 		}
-
-		return Void();
 	}
 
 	Future<Void> performTest(Database const& cx, Standalone<VectorRef<KeyValueRef>> const& data) override {
@@ -194,52 +189,49 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 	}
 
 	// Run a scripted set of API operations
-	ACTOR Future<Void> runScriptedTest(ApiCorrectnessWorkload* self, VectorRef<KeyValueRef> data) {
+	Future<Void> runScriptedTest(ApiCorrectnessWorkload* self, VectorRef<KeyValueRef> data) {
 		// Test the set function
-		bool setResult = wait(self->runSet(data, self));
+		bool setResult = co_await self->runSet(data, self);
 		if (!setResult)
-			return Void();
+			co_return;
 
 		// Test the get function
-		wait(::success(self->runGet(data, self->numGets, self)));
+		co_await ::success(self->runGet(data, self->numGets, self));
 
 		// Test the getRange function
-		state int i;
-		for (i = 0; i < self->numGetRanges; i++)
-			wait(::success(self->runGetRange(data, self)));
+		for (int i = 0; i < self->numGetRanges; i++)
+			co_await ::success(self->runGetRange(data, self));
 
 		// Test the getRange function using key selectors
-		for (i = 0; i < self->numGetRangeSelectors; i++)
-			wait(::success(self->runGetRangeSelector(data, self)));
+		for (int i = 0; i < self->numGetRangeSelectors; i++)
+			co_await ::success(self->runGetRangeSelector(data, self));
 
 		// Test the getKey function
-		wait(::success(self->runGetKey(data, self->numGetKeys, self)));
+		co_await ::success(self->runGetKey(data, self->numGetKeys, self));
 
 		// Test the clear function
-		bool clearResult = wait(self->runClear(data, self->numClears, self));
+		bool clearResult = co_await self->runClear(data, self->numClears, self);
 		if (!clearResult)
-			return Void();
+			co_return;
 
 		// Test the clear function using keyRanges
-		for (i = 0; i < self->numClearRanges; i++) {
+		for (int i = 0; i < self->numClearRanges; i++) {
 			// Alternate restoring the database to its original state and clearing a single range
 			if (self->store.size() < self->minSizeAfterClear) {
-				bool resetResult = wait(self->runSet(data, self));
+				bool resetResult = co_await self->runSet(data, self);
 				if (!resetResult)
-					return Void();
+					co_return;
 			}
 
-			bool clearRangeResults = wait(self->runClearRange(data, self));
+			bool clearRangeResults = co_await self->runClearRange(data, self);
 			if (!clearRangeResults)
-				return Void();
+				co_return;
 		}
-
-		return Void();
 	}
 
 	// Generate and execute a sequence of random operations
-	ACTOR Future<Void> runRandomTest(ApiCorrectnessWorkload* self, Standalone<VectorRef<KeyValueRef>> data) {
-		loop {
+	Future<Void> runRandomTest(ApiCorrectnessWorkload* self, Standalone<VectorRef<KeyValueRef>> data) {
+		while (true) {
 			double setProbability = 1 - ((double)self->store.size()) / self->maxRandomTestKeys;
 			int pdfArray[] = { 0,
 				               (int)(100 * setProbability),
@@ -278,7 +270,7 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 				int minKeyLength = useShortKeys ? self->minShortKeyLength : self->minLongKeyLength;
 				int maxKeyLength = useShortKeys ? self->maxShortKeyLength : self->maxLongKeyLength;
 
-				state Standalone<VectorRef<KeyValueRef>> newData =
+				Standalone<VectorRef<KeyValueRef>> newData =
 				    self->generateData(std::min((uint64_t)100, self->maxRandomTestKeys - self->store.size()),
 				                       minKeyLength,
 				                       maxKeyLength,
@@ -289,66 +281,67 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 
 				data.append_deep(data.arena(), newData.begin(), newData.size());
 
-				bool result = wait(self->runSet(newData, self));
+				bool result = co_await self->runSet(newData, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 
 			// Test the get operation
 			else if (operation == GET) {
-				bool result = wait(self->runGet(data, 10, self));
+				bool result = co_await self->runGet(data, 10, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 
 			// Test the getRange operation
 			else if (operation == GET_RANGE) {
-				bool result = wait(self->runGetRange(data, self));
+				bool result = co_await self->runGetRange(data, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 
 			// Test the getRange operation with key selectors
 			else if (operation == GET_RANGE_SELECTOR) {
-				bool result = wait(self->runGetRangeSelector(data, self));
+				bool result = co_await self->runGetRangeSelector(data, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 
 			// Test the getKey operation
 			else if (operation == GET_KEY) {
-				bool result = wait(self->runGetKey(data, 10, self));
+				bool result = co_await self->runGetKey(data, 10, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 
 			// Test the clear operation
 			else if (operation == CLEAR) {
-				bool result = wait(self->runClear(data, 10, self));
+				bool result = co_await self->runClear(data, 10, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 
 			// Test the clear operation (using key range)
 			else if (operation == CLEAR_RANGE) {
-				bool result = wait(self->runClearRange(data, self));
+				bool result = co_await self->runClearRange(data, self);
 				if (!result)
-					return Void();
+					co_return;
 			}
 		}
 	}
 
 	// Adds the key-value pairs in data to the database and memory store
-	ACTOR Future<bool> runSet(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
-		state int currentIndex = 0;
+	Future<bool> runSet(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
+		int currentIndex = 0;
 		while (currentIndex < data.size()) {
-			state Reference<TransactionWrapper> transaction = self->createTransaction();
+			Reference<TransactionWrapper> transaction = self->createTransaction();
 
 			// Set keys in the database
-			loop {
+			while (true) {
+				Error err;
 				try {
 					// For now, make this transaction self-conflicting to avoid commit errors
-					Optional<Value> value = wait(transaction->get(data[currentIndex].key));
+					Optional<Value> value = co_await transaction->get(data[currentIndex].key);
 
 					for (int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, data.size());
 					     i++) {
@@ -356,7 +349,7 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 						transaction->set(data[i].key, data[i].value);
 					}
 
-					wait(transaction->commit());
+					co_await transaction->commit();
 					for (int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, data.size());
 					     i++)
 						DEBUG_MUTATION("ApiCorrectnessSet",
@@ -366,8 +359,9 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 					currentIndex += self->maxKeysPerTransaction;
 					break;
 				} catch (Error& e) {
-					wait(transaction->onError(e));
+					err = e;
 				}
+				co_await transaction->onError(err);
 			}
 		}
 
@@ -378,36 +372,37 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		}
 
 		// Check that the database and memory store are the same
-		bool result = wait(self->compareDatabaseToMemory());
+		bool result = co_await self->compareDatabaseToMemory();
 		if (!result)
 			self->testFailure("Set resulted in incorrect database");
 
-		return result;
+		co_return result;
 	}
 
 	// Gets a specified number of values from the database and memory store and compares them, returning true if all
 	// results were the same
-	ACTOR Future<bool> runGet(VectorRef<KeyValueRef> data, int numReads, ApiCorrectnessWorkload* self) {
+	Future<bool> runGet(VectorRef<KeyValueRef> data, int numReads, ApiCorrectnessWorkload* self) {
 		// Generate a set of random keys to get
-		state Standalone<VectorRef<KeyRef>> keys;
+		Standalone<VectorRef<KeyRef>> keys;
 		for (int i = 0; i < numReads; i++)
 			keys.push_back_deep(keys.arena(), self->selectRandomKey(data, 0.9));
 
-		state std::vector<Optional<Value>> values;
+		std::vector<Optional<Value>> values;
 
-		state int currentIndex = 0;
+		int currentIndex = 0;
 		while (currentIndex < keys.size()) {
-			state Reference<TransactionWrapper> transaction = self->createTransaction();
+			Reference<TransactionWrapper> transaction = self->createTransaction();
 
 			// Get the values from the database
-			loop {
+			while (true) {
+				Error err;
 				try {
-					state std::vector<Future<Optional<Value>>> dbValueFutures;
+					std::vector<Future<Optional<Value>>> dbValueFutures;
 					for (int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size());
 					     i++)
 						dbValueFutures.push_back(transaction->get(keys[i]));
 
-					wait(waitForAll(dbValueFutures));
+					co_await waitForAll(dbValueFutures);
 
 					for (int i = 0; i < dbValueFutures.size(); i++)
 						values.push_back(dbValueFutures[i].get());
@@ -416,8 +411,9 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 
 					break;
 				} catch (Error& e) {
-					wait(transaction->onError(e));
+					err = e;
 				}
+				co_await transaction->onError(err);
 			}
 		}
 
@@ -434,45 +430,47 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		if (!result)
 			self->testFailure("Get returned incorrect results");
 
-		return result;
+		co_return result;
 	}
 
 	// Gets a single range of values from the database and memory stores and compares them, returning true if the
 	// results were the same
-	ACTOR Future<bool> runGetRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
-		state Reverse reverse(deterministicRandom()->coinflip());
+	Future<bool> runGetRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
+		Reverse reverse(deterministicRandom()->coinflip());
 
 		// Generate a random range
 		Key key = self->selectRandomKey(data, 0.5);
 		Key key2 = self->selectRandomKey(data, 0.5);
 
-		state Key start = std::min(key, key2);
-		state Key end = std::max(key, key2);
+		Key start = std::min(key, key2);
+		Key end = std::max(key, key2);
 
 		// Generate a random maximum number of results
-		state int limit = deterministicRandom()->randomInt(0, 101);
+		int limit = deterministicRandom()->randomInt(0, 101);
 
 		// Get the range from memory
-		state RangeResult storeResults = self->store.getRange(KeyRangeRef(start, end), limit, reverse);
+		RangeResult storeResults = self->store.getRange(KeyRangeRef(start, end), limit, reverse);
 
 		// Get the range from the database
-		state RangeResult dbResults;
-		state Version readVersion;
+		RangeResult dbResults;
+		Version readVersion{ 0 };
 
-		state Reference<TransactionWrapper> transaction = self->createTransaction();
+		Reference<TransactionWrapper> transaction = self->createTransaction();
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				Version version = wait(transaction->getReadVersion());
+				Version version = co_await transaction->getReadVersion();
 				readVersion = version;
 
 				KeyRangeRef range(start, end);
-				RangeResult rangeResults = wait(transaction->getRange(range, limit, reverse));
+				RangeResult rangeResults = co_await transaction->getRange(range, limit, reverse);
 				dbResults = rangeResults;
 				break;
 			} catch (Error& e) {
-				wait(transaction->onError(e));
+				err = e;
 			}
+			co_await transaction->onError(err);
 		}
 
 		// Compare the ranges
@@ -480,13 +478,13 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		if (!result)
 			self->testFailure("GetRange returned incorrect results");
 
-		return result;
+		co_return result;
 	}
 
 	// Gets a single range of values using key selectors from the database and memory store and compares them, returning
 	// true if the results were the same
-	ACTOR Future<bool> runGetRangeSelector(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
-		state Reverse reverse(deterministicRandom()->coinflip());
+	Future<bool> runGetRangeSelector(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
+		Reverse reverse(deterministicRandom()->coinflip());
 
 		KeySelector selectors[2];
 		Key keys[2];
@@ -498,7 +496,7 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		// Don't use key selectors which would return results outside the key-space of this client unless this client
 		// is the first or last client
 		for (int i = 0; i < 2; i++) {
-			loop {
+			while (true) {
 				// Gradually decrease the maximum offset to increase the likelihood of finding a valid key selector in a
 				// small store
 				selectors[i] =
@@ -506,22 +504,22 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 				keys[i] = self->store.getKey(selectors[i]);
 
 				if (keys[i].startsWith(StringRef(self->clientPrefix)) ||
-				    (keys[i].size() == 0 && self->clientPrefixInt == 0) ||
+				    (keys[i].empty() && self->clientPrefixInt == 0) ||
 				    (keys[i].startsWith("\xff"_sr) && self->clientPrefixInt == self->clientCount - 1)) {
 					break;
 				}
 
 				// Don't loop forever trying to generate valid key selectors if there are no keys in the store
 				if (++currentSelectorAttempts == maxSelectorAttempts)
-					return true;
+					co_return true;
 			}
 		}
 
-		state KeySelector startSelector;
-		state KeySelector endSelector;
+		KeySelector startSelector;
+		KeySelector endSelector;
 
-		state Key startKey;
-		state Key endKey;
+		Key startKey;
+		Key endKey;
 
 		// Make sure startKey is less than endKey
 		if (keys[0] < keys[1]) {
@@ -537,23 +535,24 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		}
 
 		// Choose a random maximum number of results
-		state int limit = deterministicRandom()->randomInt(0, 101);
+		int limit = deterministicRandom()->randomInt(0, 101);
 
 		// Get the range from the memory store
-		state RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), limit, reverse);
+		RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), limit, reverse);
 
 		// Get the range from the database
-		state RangeResult dbResults;
+		RangeResult dbResults;
 
-		state Reference<TransactionWrapper> transaction = self->createTransaction();
-		state Version readVersion;
+		Reference<TransactionWrapper> transaction = self->createTransaction();
+		Version readVersion{ 0 };
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				Version version = wait(transaction->getReadVersion());
+				Version version = co_await transaction->getReadVersion();
 				readVersion = version;
 
-				RangeResult range = wait(transaction->getRange(startSelector, endSelector, limit, reverse));
+				RangeResult range = co_await transaction->getRange(startSelector, endSelector, limit, reverse);
 
 				if (endKey == self->store.endKey()) {
 					for (int i = 0; i < range.size(); i++) {
@@ -569,8 +568,9 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 
 				break;
 			} catch (Error& e) {
-				wait(transaction->onError(e));
+				err = e;
 			}
+			co_await transaction->onError(err);
 		}
 
 		// Compare the results
@@ -579,33 +579,34 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		if (!result)
 			self->testFailure("GetRange (KeySelector) returned incorrect results");
 
-		return result;
+		co_return result;
 	}
 
 	// Gets a specified number of keys from the database and memory store and compares them, returning true if all
 	// results were the same
-	ACTOR Future<bool> runGetKey(VectorRef<KeyValueRef> data, int numGetKeys, ApiCorrectnessWorkload* self) {
+	Future<bool> runGetKey(VectorRef<KeyValueRef> data, int numGetKeys, ApiCorrectnessWorkload* self) {
 		// Generate a set of random key selectors
-		state Standalone<VectorRef<KeySelectorRef>> selectors;
+		Standalone<VectorRef<KeySelectorRef>> selectors;
 		for (int i = 0; i < numGetKeys; i++)
 			selectors.push_back_deep(selectors.arena(), self->generateKeySelector(data, 100));
 
-		state Standalone<VectorRef<KeyRef>> keys;
+		Standalone<VectorRef<KeyRef>> keys;
 
-		state int currentIndex = 0;
+		int currentIndex = 0;
 		while (currentIndex < selectors.size()) {
 			// Get the keys from the database
-			state Reference<TransactionWrapper> transaction = self->createTransaction();
+			Reference<TransactionWrapper> transaction = self->createTransaction();
 
-			loop {
+			while (true) {
+				Error err;
 				try {
-					state std::vector<Future<Standalone<KeyRef>>> dbKeyFutures;
+					std::vector<Future<Standalone<KeyRef>>> dbKeyFutures;
 					for (int i = currentIndex;
 					     i < std::min(currentIndex + self->maxKeysPerTransaction, selectors.size());
 					     i++)
 						dbKeyFutures.push_back(transaction->getKey(selectors[i]));
 
-					wait(waitForAll(dbKeyFutures));
+					co_await waitForAll(dbKeyFutures);
 
 					for (int i = 0; i < dbKeyFutures.size(); i++)
 						keys.push_back_deep(keys.arena(), dbKeyFutures[i].get());
@@ -614,16 +615,16 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 
 					break;
 				} catch (Error& e) {
-					wait(transaction->onError(e));
+					err = e;
 				}
+				co_await transaction->onError(err);
 			}
 		}
 
-		state bool result = true;
+		bool result = true;
 
 		// Get the keys from the memory store and compare them
-		state int i;
-		for (i = 0; i < selectors.size(); i++) {
+		for (int i = 0; i < selectors.size(); i++) {
 			Key key = self->store.getKey(selectors[i]);
 			if (keys[i].startsWith(StringRef(self->clientPrefix)) && keys[i] != key)
 				result = false;
@@ -638,23 +639,24 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 				       selectors[i].toString().c_str(),
 				       printable(keys[i]).c_str(),
 				       printable(key).c_str());
-				state int dir = selectors[i].offset > 0 ? 1 : -1;
-				state int j;
-				for (j = 0; j <= abs(selectors[i].offset); j++) {
-					state KeySelector sel = KeySelectorRef(selectors[i].getKey(), selectors[i].orEqual, j * dir);
-					state Key storeKey = self->store.getKey(sel);
+				int dir = selectors[i].offset > 0 ? 1 : -1;
+				for (int j = 0; j <= abs(selectors[i].offset); j++) {
+					KeySelector sel = KeySelectorRef(selectors[i].getKey(), selectors[i].orEqual, j * dir);
+					Key storeKey = self->store.getKey(sel);
 
-					state Reference<TransactionWrapper> tr = self->createTransaction();
+					Reference<TransactionWrapper> tr = self->createTransaction();
 
-					state Key dbKey;
-					loop {
+					Key dbKey;
+					while (true) {
+						Error err;
 						try {
-							Key key = wait(tr->getKey(sel));
+							Key key = co_await tr->getKey(sel);
 							dbKey = key;
 							break;
 						} catch (Error& e) {
-							wait(tr->onError(e));
+							err = e;
 						}
+						co_await tr->onError(err);
 					}
 
 					if (!(storeKey == self->store.startKey() && dbKey < StringRef(self->clientPrefix)) &&
@@ -672,24 +674,25 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		if (!result)
 			self->testFailure("GetKey returned incorrect results");
 
-		return result;
+		co_return result;
 	}
 
 	// Clears a specified number of keys from the database and memory store
-	ACTOR Future<bool> runClear(VectorRef<KeyValueRef> data, int numClears, ApiCorrectnessWorkload* self) {
+	Future<bool> runClear(VectorRef<KeyValueRef> data, int numClears, ApiCorrectnessWorkload* self) {
 		// Generate a random set of keys to clear
-		state Standalone<VectorRef<KeyRef>> keys;
+		Standalone<VectorRef<KeyRef>> keys;
 		for (int i = 0; i < numClears; i++)
 			keys.push_back_deep(keys.arena(), self->selectRandomKey(data, 0.9));
 
-		state int currentIndex = 0;
+		int currentIndex = 0;
 		while (currentIndex < keys.size()) {
 			// Clear the keys from the database
-			state Reference<TransactionWrapper> transaction = self->createTransaction();
-			loop {
+			Reference<TransactionWrapper> transaction = self->createTransaction();
+			while (true) {
+				Error err;
 				try {
 					// For now, make this transaction self-conflicting to avoid commit errors
-					Optional<Value> value = wait(transaction->get(keys[0]));
+					Optional<Value> value = co_await transaction->get(keys[0]);
 
 					for (int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size());
 					     i++) {
@@ -697,7 +700,7 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 						transaction->clear(keys[i]);
 					}
 
-					wait(transaction->commit());
+					co_await transaction->commit();
 					for (int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size());
 					     i++)
 						DEBUG_MUTATION("ApiCorrectnessClear",
@@ -707,8 +710,9 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 					currentIndex += self->maxKeysPerTransaction;
 					break;
 				} catch (Error& e) {
-					wait(transaction->onError(e));
+					err = e;
 				}
+				co_await transaction->onError(err);
 			}
 		}
 
@@ -719,54 +723,56 @@ struct ApiCorrectnessWorkload : ApiWorkload {
 		}
 
 		// Check that the database and memory store are the same
-		bool result = wait(self->compareDatabaseToMemory());
+		bool result = co_await self->compareDatabaseToMemory();
 		if (!result)
 			self->testFailure("Clear resulted in incorrect database");
 
-		return result;
+		co_return result;
 	}
 
 	// Clears a single range of keys from the database and memory store
-	ACTOR Future<bool> runClearRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
+	Future<bool> runClearRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload* self) {
 		// Generate a random range to clear
 		Key key = self->selectRandomKey(data, 0.5);
 		Key key2 = self->selectRandomKey(data, 0.5);
 
-		state Key start = std::min(key, key2);
-		state Key end = std::max(key, key2);
+		Key start = std::min(key, key2);
+		Key end = std::max(key, key2);
 
 		// Clear the range in memory
 		self->store.clear(KeyRangeRef(start, end));
 
 		// Clear the range in the database
-		state Reference<TransactionWrapper> transaction = self->createTransaction();
+		Reference<TransactionWrapper> transaction = self->createTransaction();
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// For now, make this transaction self-conflicting to avoid commit errors
-				Optional<Value> value = wait(transaction->get(start));
+				Optional<Value> value = co_await transaction->get(start);
 
-				state KeyRangeRef range(start, end);
+				KeyRangeRef range(start, end);
 				if (!range.empty()) {
 					transaction->addReadConflictRange(range);
 				}
 				transaction->clear(range);
-				wait(transaction->commit());
+				co_await transaction->commit();
 				DEBUG_KEY_RANGE("ApiCorrectnessClear", transaction->getCommittedVersion(), range);
 				break;
 			} catch (Error& e) {
-				wait(transaction->onError(e));
+				err = e;
 			}
+			co_await transaction->onError(err);
 		}
 
 		self->debugKey(KeyRangeRef(start, end), "ClearRange");
 
 		// Check that the database and memory store are the same
-		bool result = wait(self->compareDatabaseToMemory());
+		bool result = co_await self->compareDatabaseToMemory();
 		if (!result)
 			self->testFailure("Clear (range) resulted in incorrect database");
 
-		return result;
+		co_return result;
 	}
 };
 
diff --git a/fdbserver/workloads/ApiWorkload.actor.cpp b/fdbserver/workloads/ApiWorkload.cpp
similarity index 73%
rename from fdbserver/workloads/ApiWorkload.actor.cpp
rename to fdbserver/workloads/ApiWorkload.cpp
index 05fb4161f20..6c7e6660b6b 100644
--- a/fdbserver/workloads/ApiWorkload.actor.cpp
+++ b/fdbserver/workloads/ApiWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * ApiWorkload.actor.cpp
+ * ApiWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,12 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/ApiWorkload.h"
+#include "ApiWorkload.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/MultiVersionTransaction.h"
 
 #include "fdbrpc/simulator.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
@@ -32,22 +33,22 @@
 #include <cinttypes>
 #include "fmt/format.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 // Clears the keyspace used by this test
-ACTOR Future<Void> clearKeyspace(ApiWorkload* self) {
-	loop {
-		state Reference<TransactionWrapper> transaction = self->createTransaction();
+Future<Void> clearKeyspace(ApiWorkload* self) {
+	while (true) {
+		Reference<TransactionWrapper> transaction = self->createTransaction();
+		Error err;
 		try {
 			KeyRange range(KeyRangeRef(StringRef(format("%010d", self->clientPrefixInt)),
 			                           StringRef(format("%010d", self->clientPrefixInt + 1))));
 
 			transaction->clear(range);
-			wait(transaction->commit());
-			return Void();
+			co_await transaction->commit();
+			co_return;
 		} catch (Error& e) {
-			wait(transaction->onError(e));
+			err = e;
 		}
+		co_await transaction->onError(err);
 	}
 }
 
@@ -55,15 +56,13 @@ Future<Void> ApiWorkload::clearKeyspace() {
 	return ::clearKeyspace(this);
 }
 
-ACTOR Future<Void> setup(Database cx, ApiWorkload* self) {
-	self->transactionFactory = Reference<TransactionFactoryInterface>(
-	    new TransactionFactory<FlowTransactionWrapper<Transaction>, const Database>(cx, cx, false));
+Future<Void> setup(Database cx, ApiWorkload* self) {
+	self->transactionFactory =
+	    makeReference<TransactionFactory<FlowTransactionWrapper<Transaction>, const Database>>(cx, cx, false);
 
 	// Clear keyspace before running
-	wait(timeoutError(self->clearKeyspace(), 600));
-	wait(self->performSetup(cx));
-
-	return Void();
+	co_await timeoutError(self->clearKeyspace(), 600);
+	co_await self->performSetup(cx);
 }
 
 Future<Void> ApiWorkload::setup(Database const& cx) {
@@ -73,14 +72,14 @@ Future<Void> ApiWorkload::setup(Database const& cx) {
 	return Void();
 }
 
-ACTOR Future<Void> start(Database cx, ApiWorkload* self) {
+Future<Void> start(Database cx, ApiWorkload* self) {
 	// Generate the data to store in this client's key-space
-	state Standalone<VectorRef<KeyValueRef>> data = self->generateData(self->numKeys * self->shortKeysRatio,
-	                                                                   self->minShortKeyLength,
-	                                                                   self->maxShortKeyLength,
-	                                                                   self->minValueLength,
-	                                                                   self->maxValueLength,
-	                                                                   self->clientPrefix);
+	Standalone<VectorRef<KeyValueRef>> data = self->generateData(self->numKeys * self->shortKeysRatio,
+	                                                             self->minShortKeyLength,
+	                                                             self->maxShortKeyLength,
+	                                                             self->minValueLength,
+	                                                             self->maxValueLength,
+	                                                             self->clientPrefix);
 	Standalone<VectorRef<KeyValueRef>> bigKeyData = self->generateData(self->numKeys * (1 - self->shortKeysRatio),
 	                                                                   self->minLongKeyLength,
 	                                                                   self->maxLongKeyLength,
@@ -91,13 +90,11 @@ ACTOR Future<Void> start(Database cx, ApiWorkload* self) {
 	data.append_deep(data.arena(), bigKeyData.begin(), bigKeyData.size());
 
 	try {
-		wait(self->performTest(cx, data));
+		co_await self->performTest(cx, data);
 	} catch (Error& e) {
 		if (e.code() != error_code_actor_cancelled)
 			self->testFailure(format("Unhandled error %d: %s", e.code(), e.name()));
 	}
-
-	return Void();
 }
 
 Future<Void> ApiWorkload::start(Database const& cx) {
@@ -170,41 +167,42 @@ bool ApiWorkload::compareResults(VectorRef<KeyValueRef> dbResults,
 }
 
 // Compares the contents of this client's key-space in the database with the in-memory key-value store
-ACTOR Future<bool> compareDatabaseToMemory(ApiWorkload* self) {
-	state Key startKey(self->clientPrefix);
-	state Key endKey(self->clientPrefix + "\xff");
-	state int resultsPerRange = 100;
-	state double startTime = now();
+Future<bool> compareDatabaseToMemory(ApiWorkload* self) {
+	Key startKey(self->clientPrefix);
+	Key endKey(self->clientPrefix + "\xff");
+	int resultsPerRange = 100;
+	double startTime = now();
 
-	loop {
+	while (true) {
 		// Fetch a subset of the results from each of the database and the memory store and compare them
-		state RangeResult storeResults =
-		    self->store.getRange(KeyRangeRef(startKey, endKey), resultsPerRange, Reverse::False);
+		RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), resultsPerRange, Reverse::False);
 
-		state Reference<TransactionWrapper> transaction = self->createTransaction();
-		state KeyRangeRef range(startKey, endKey);
+		Reference<TransactionWrapper> transaction = self->createTransaction();
+		KeyRangeRef range(startKey, endKey);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state RangeResult dbResults = wait(transaction->getRange(range, resultsPerRange, Reverse::False));
+				RangeResult dbResults = co_await transaction->getRange(range, resultsPerRange, Reverse::False);
 
 				// Compare results of database and memory store
-				Version v = wait(transaction->getReadVersion());
+				Version v = co_await transaction->getReadVersion();
 				if (!self->compareResults(dbResults, storeResults, v)) {
 					TraceEvent(SevError, "FailedComparisonToMemory").detail("StartTime", startTime);
-					return false;
+					co_return false;
 				}
 
 				// If there are no more results, then return success
 				if (storeResults.size() < resultsPerRange)
-					return true;
+					co_return true;
 
 				startKey = dbResults[dbResults.size() - 1].key;
 
 				break;
 			} catch (Error& e) {
-				wait(transaction->onError(e));
+				err = e;
 			}
+			co_await transaction->onError(err);
 		}
 	}
 }
@@ -260,14 +258,14 @@ Key ApiWorkload::generateKey(VectorRef<KeyValueRef> const& data,
 	}
 
 	// If encryption validation is enabled; slip "marker pattern" at random location in generated key
-	if (g_network->isSimulated() && g_simulator->dataAtRestPlaintextMarker.present() &&
-	    keyLength + 1 > g_simulator->dataAtRestPlaintextMarker.get().size()) {
-		int len = keyLength - g_simulator->dataAtRestPlaintextMarker.get().size();
+	if (g_network->isSimulated() && fdbSimulationPolicyState().dataAtRestPlaintextMarker.present() &&
+	    keyLength + 1 > fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().size()) {
+		int len = keyLength - fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().size();
 		// Avoid updating the first byte of the key
 		int idx = len > 1 ? deterministicRandom()->randomInt(1, len) : 1;
 		memcpy(&keyBuffer[idx],
-		       g_simulator->dataAtRestPlaintextMarker.get().c_str(),
-		       g_simulator->dataAtRestPlaintextMarker.get().size());
+		       fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().c_str(),
+		       fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().size());
 		//TraceEvent(SevDebug, "ModifiedKey").suppressFor(5).detail("Key", keyBuffer);
 	}
 
@@ -301,13 +299,13 @@ Value ApiWorkload::generateValue(int minValueLength, int maxValueLength) {
 	std::string ret(std::string(valueLength, 'x'));
 
 	// If encryption validation is enabled; slip "marker pattern" at random location in generated key
-	if (g_network->isSimulated() && g_simulator->dataAtRestPlaintextMarker.present() &&
-	    valueLength > g_simulator->dataAtRestPlaintextMarker.get().size()) {
-		int len = valueLength - g_simulator->dataAtRestPlaintextMarker.get().size();
+	if (g_network->isSimulated() && fdbSimulationPolicyState().dataAtRestPlaintextMarker.present() &&
+	    valueLength > fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().size()) {
+		int len = valueLength - fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().size();
 		int idx = deterministicRandom()->randomInt(0, len);
 		memcpy(&ret[idx],
-		       g_simulator->dataAtRestPlaintextMarker.get().c_str(),
-		       g_simulator->dataAtRestPlaintextMarker.get().size());
+		       fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().c_str(),
+		       fdbSimulationPolicyState().dataAtRestPlaintextMarker.get().size());
 		//TraceEvent("ModifiedValue").suppressFor(5).detail("Value", ret);
 	}
 	return Value(ret);
@@ -319,37 +317,35 @@ Value ApiWorkload::generateValue() {
 }
 
 // Creates a random transaction factory to produce transaction of one of the TransactionType choices
-ACTOR Future<Void> chooseTransactionFactory(Database cx, std::vector<TransactionType> choices, ApiWorkload* self) {
+Future<Void> chooseTransactionFactory(Database cx, std::vector<TransactionType> choices, ApiWorkload* self) {
 	TransactionType transactionType = deterministicRandom()->randomChoice(choices);
 	self->transactionType = transactionType;
 
 	if (transactionType == NATIVE) {
 		printf("client %d: Running NativeAPI Transactions\n", self->clientPrefixInt);
-		self->transactionFactory = Reference<TransactionFactoryInterface>(
-		    new TransactionFactory<FlowTransactionWrapper<Transaction>, const Database>(
-		        cx, self->extraDB, self->useExtraDB));
+		self->transactionFactory =
+		    makeReference<TransactionFactory<FlowTransactionWrapper<Transaction>, const Database>>(
+		        cx, self->extraDB, self->useExtraDB);
 	} else if (transactionType == READ_YOUR_WRITES) {
 		printf("client %d: Running ReadYourWrites Transactions\n", self->clientPrefixInt);
-		self->transactionFactory = Reference<TransactionFactoryInterface>(
-		    new TransactionFactory<FlowTransactionWrapper<ReadYourWritesTransaction>, const Database>(
-		        cx, self->extraDB, self->useExtraDB));
+		self->transactionFactory =
+		    makeReference<TransactionFactory<FlowTransactionWrapper<ReadYourWritesTransaction>, const Database>>(
+		        cx, self->extraDB, self->useExtraDB);
 	} else if (transactionType == THREAD_SAFE) {
 		printf("client %d: Running ThreadSafe Transactions\n", self->clientPrefixInt);
 		Reference<IDatabase> dbHandle =
-		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
-		self->transactionFactory = Reference<TransactionFactoryInterface>(
-		    new TransactionFactory<ThreadTransactionWrapper, Reference<IDatabase>>(dbHandle, dbHandle, false));
+		    co_await unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx));
+		self->transactionFactory = makeReference<TransactionFactory<ThreadTransactionWrapper, Reference<IDatabase>>>(
+		    dbHandle, dbHandle, false);
 	} else if (transactionType == MULTI_VERSION) {
 		printf("client %d: Running Multi-Version Transactions\n", self->clientPrefixInt);
 		MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
 		Reference<IDatabase> threadSafeHandle =
-		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
+		    co_await unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx));
 		Reference<IDatabase> dbHandle = MultiVersionDatabase::debugCreateFromExistingDatabase(threadSafeHandle);
-		self->transactionFactory = Reference<TransactionFactoryInterface>(
-		    new TransactionFactory<ThreadTransactionWrapper, Reference<IDatabase>>(dbHandle, dbHandle, false));
+		self->transactionFactory = makeReference<TransactionFactory<ThreadTransactionWrapper, Reference<IDatabase>>>(
+		    dbHandle, dbHandle, false);
 	}
-
-	return Void();
 }
 
 Future<Void> ApiWorkload::chooseTransactionFactory(Database const& cx, std::vector<TransactionType> const& choices) {
diff --git a/fdbserver/include/fdbserver/workloads/ApiWorkload.h b/fdbserver/workloads/ApiWorkload.h
similarity index 97%
rename from fdbserver/include/fdbserver/workloads/ApiWorkload.h
rename to fdbserver/workloads/ApiWorkload.h
index 8c9cd1ed948..aa6c22dfb11 100644
--- a/fdbserver/include/fdbserver/workloads/ApiWorkload.h
+++ b/fdbserver/workloads/ApiWorkload.h
@@ -22,11 +22,11 @@
 #define FDBSERVER_APIWORKLOAD_H
 #pragma once
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/ThreadSafeTransaction.h"
-#include "fdbserver/workloads/MemoryKeyValueStore.h"
+#include "MemoryKeyValueStore.h"
 
 // an enumeration of apis being tested
 enum TransactionType { NATIVE, READ_YOUR_WRITES, THREAD_SAFE, MULTI_VERSION };
@@ -259,7 +259,7 @@ struct TransactionFactory : public TransactionFactoryInterface {
 
 	// Creates a new transaction
 	Reference<TransactionWrapper> createTransaction() override {
-		return Reference<TransactionWrapper>(new T(dbHandle, extraDbHandle, useExtraDB));
+		return makeReference<T>(dbHandle, extraDbHandle, useExtraDB);
 	}
 };
 
@@ -267,7 +267,7 @@ struct ApiWorkload : TestWorkload {
 	bool useExtraDB;
 	Database extraDB;
 
-	ApiWorkload(WorkloadContext const& wcx, int maxClients = -1)
+	explicit ApiWorkload(WorkloadContext const& wcx, int maxClients = -1)
 	  : TestWorkload(wcx), maxClients(maxClients), success(true), transactionFactory(nullptr) {
 		clientPrefixInt = getOption(options, "clientId"_sr, clientId);
 		clientPrefix = format("%010d", clientPrefixInt);
@@ -282,10 +282,10 @@ struct ApiWorkload : TestWorkload {
 		minValueLength = getOption(options, "minValueLength"_sr, 1);
 		maxValueLength = getOption(options, "maxValueLength"_sr, 10000);
 
-		useExtraDB = g_network->isSimulated() && !g_simulator->extraDatabases.empty();
+		useExtraDB = g_network->isSimulated() && !fdbSimulationPolicyState().extraDatabases.empty();
 		if (useExtraDB) {
-			ASSERT(g_simulator->extraDatabases.size() == 1);
-			extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+			ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+			extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 		}
 	}
 
diff --git a/fdbserver/workloads/AsyncFile.cpp b/fdbserver/workloads/AsyncFile.cpp
index 1bbd2942abc..7a3c8a0d709 100644
--- a/fdbserver/workloads/AsyncFile.cpp
+++ b/fdbserver/workloads/AsyncFile.cpp
@@ -18,9 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
-#include "fdbserver/workloads/AsyncFile.actor.h"
+#include "AsyncFile.h"
 
 // class RandomByteGenerator
 
diff --git a/fdbserver/include/fdbserver/workloads/AsyncFile.actor.h b/fdbserver/workloads/AsyncFile.h
similarity index 69%
rename from fdbserver/include/fdbserver/workloads/AsyncFile.actor.h
rename to fdbserver/workloads/AsyncFile.h
index 71184bae0a0..c7ce6ff0975 100644
--- a/fdbserver/include/fdbserver/workloads/AsyncFile.actor.h
+++ b/fdbserver/workloads/AsyncFile.h
@@ -1,5 +1,5 @@
 /*
- * AsyncFile.actor.h
+ * AsyncFile.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,17 +20,8 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(WORKLOADS_ASYNCFILE_ACTOR_G_H)
-#define WORKLOADS_ASYNCFILE_ACTOR_G_H
-#include "fdbserver/workloads/AsyncFile.actor.g.h"
-#elif !defined(WORKLOADS_ASYNCFILE_ACTOR_H)
-#define WORKLOADS_ASYNCFILE_ACTOR_H
-
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/IAsyncFile.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 class RandomByteGenerator {
 private:
@@ -76,7 +67,7 @@ struct AsyncFileWorkload : TestWorkload {
 
 	std::string path;
 
-	AsyncFileWorkload(WorkloadContext const&);
+	explicit(false) AsyncFileWorkload(WorkloadContext const&);
 	~AsyncFileWorkload() override {}
 
 	// Allocates a buffer of a given size.  If necessary, the buffer will be aligned to 4K
@@ -85,19 +76,15 @@ struct AsyncFileWorkload : TestWorkload {
 	Future<bool> check(Database const& cx) override;
 
 	// Opens a file for AsyncFile operations.  If the path is empty, then creates a file and fills it with random data
-	ACTOR Future<Void> openFile(AsyncFileWorkload* self,
-	                            int64_t flags,
-	                            int64_t mode,
-	                            uint64_t size,
-	                            bool fillFile = false) {
-		state RandomByteGenerator rbg;
+	Future<Void> openFile(AsyncFileWorkload* self, int64_t flags, int64_t mode, uint64_t size, bool fillFile = false) {
+		RandomByteGenerator rbg;
 
 		if (self->fileHandle.getPtr() != nullptr) {
 			self->fileHandle->file = Reference<IAsyncFile>(nullptr);
-			wait(delay(0.1));
+			co_await delay(0.1);
 		}
 
-		state bool fileCreated = self->path.length() == 0;
+		bool fileCreated = self->path.length() == 0;
 		if (fileCreated) {
 			self->path = "asyncfile." + deterministicRandom()->randomUniqueID().toString();
 			flags &= ~IAsyncFile::OPEN_READONLY;
@@ -113,23 +100,23 @@ struct AsyncFileWorkload : TestWorkload {
 			flags |= IAsyncFile::OPEN_UNCACHED;
 
 		try {
-			state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(self->path, flags, 0666));
+			Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(self->path, flags, 0666);
 			if (self->fileHandle.getPtr() == nullptr)
 				self->fileHandle = makeReference<AsyncFileHandle>(file, self->path, fileCreated);
 			else
 				self->fileHandle->file = file;
 
 			if (fileCreated || fillFile) {
-				state int64_t oldSize = wait(file->size());
-				state int64_t newSize = (size + _PAGE_SIZE - 1) & ~(int64_t(_PAGE_SIZE - 1)); // align size up
+				int64_t oldSize = co_await file->size();
+				int64_t newSize = (size + _PAGE_SIZE - 1) & ~(int64_t(_PAGE_SIZE - 1)); // align size up
 
 				if (!fileCreated)
-					wait(file->truncate(newSize));
+					co_await file->truncate(newSize);
 
-				state int chunkSize = 4 << 16;
-				state Reference<AsyncFileBuffer> data = self->allocateBuffer(chunkSize);
-				state int64_t i;
-				state Future<Void> lastWrite = Void();
+				int chunkSize = 4 << 16;
+				Reference<AsyncFileBuffer> data = self->allocateBuffer(chunkSize);
+				int64_t i{ 0 };
+				Future<Void> lastWrite = Void();
 				for (i = oldSize & ~(chunkSize - 1); i < newSize; i += chunkSize) {
 					if (i >> 30 != (i + (chunkSize)) >> 30) // each GB
 						printf("Building test file: %d GB\n", int((i + (chunkSize)) >> 30));
@@ -137,18 +124,13 @@ struct AsyncFileWorkload : TestWorkload {
 						rbg.writeRandomBytesToBuffer(data->buffer, chunkSize);
 					auto w = lastWrite;
 					lastWrite = file->write(data->buffer, chunkSize, i);
-					wait(w);
+					co_await w;
 				}
-				wait(lastWrite);
+				co_await lastWrite;
 			}
 		} catch (Error& error) {
 			TraceEvent(SevError, "TestFailure").detail("Reason", "Could not open file");
 			throw;
 		}
-
-		return Void();
 	}
 };
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/fdbserver/workloads/AsyncFileCorrectness.actor.cpp b/fdbserver/workloads/AsyncFileCorrectness.cpp
similarity index 87%
rename from fdbserver/workloads/AsyncFileCorrectness.actor.cpp
rename to fdbserver/workloads/AsyncFileCorrectness.cpp
index 130307bd19f..6a868fe1418 100644
--- a/fdbserver/workloads/AsyncFileCorrectness.actor.cpp
+++ b/fdbserver/workloads/AsyncFileCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * AsyncFileCorrectness.actor.cpp
+ * AsyncFileCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,12 +21,11 @@
 #include <cinttypes>
 
 #include "fmt/format.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
 #include "flow/IRandom.h"
 #include "flow/SystemMonitor.h"
-#include "fdbserver/workloads/AsyncFile.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "AsyncFile.h"
 
 // An enumeration representing the type of operation to be performed in a correctness test operation
 enum OperationType { READ, WRITE, SYNC, REOPEN, TRUNCATE };
@@ -74,7 +73,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 	double averageCpuUtilization;
 	PerfIntCounter numOperations;
 
-	AsyncFileCorrectnessWorkload(WorkloadContext const& wcx)
+	explicit AsyncFileCorrectnessWorkload(WorkloadContext const& wcx)
 	  : AsyncFileWorkload(wcx), memoryFile(nullptr), success(true), numOperations("Num Operations") {
 		maxOperationSize = getOption(options, "maxOperationSize"_sr, 4096);
 		numSimultaneousOperations = getOption(options, "numSimultaneousOperations"_sr, 10);
@@ -94,7 +93,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 		}
 	}
 
-	~AsyncFileCorrectnessWorkload() override {}
+	~AsyncFileCorrectnessWorkload() override = default;
 
 	Future<Void> setup(Database const& cx) override {
 		if (enabled)
@@ -103,7 +102,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _setup(AsyncFileCorrectnessWorkload* self) {
+	Future<Void> _setup(AsyncFileCorrectnessWorkload* self) {
 		// Create the memory version of the file, the file locks, and the valid mask
 		self->memoryFile = self->allocateBuffer(self->targetFileSize);
 		self->fileLock.resize(self->targetFileSize, 0);
@@ -111,9 +110,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 		self->fileSize = 0;
 
 		// Create or open the file being used for testing
-		wait(self->openFile(self, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, 0666, self->fileSize, true));
-
-		return Void();
+		co_await self->openFile(self, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, 0666, self->fileSize, true);
 	}
 
 	// Updates the memory buffer, locks, and validity mask to a new file size
@@ -144,37 +141,35 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _start(AsyncFileCorrectnessWorkload* self) {
-		state StatisticsState statState;
+	Future<Void> _start(AsyncFileCorrectnessWorkload* self) {
+		StatisticsState statState;
 		customSystemMonitor("AsyncFile Metrics", &statState);
 
-		wait(timeout(self->runCorrectnessTest(self), self->testDuration, Void()));
+		co_await timeout(self->runCorrectnessTest(self), self->testDuration, Void());
 
 		SystemStatistics stats = customSystemMonitor("AsyncFile Metrics", &statState);
 		self->averageCpuUtilization = stats.processCPUSeconds / stats.elapsed;
 
 		// Try to let the IO operations finish so we can clean up after them
-		wait(timeout(waitForAll(self->operations), 10, Void()));
-
-		return Void();
+		co_await timeout(waitForAll(self->operations), 10, Void());
 	}
 
-	ACTOR Future<Void> runCorrectnessTest(AsyncFileCorrectnessWorkload* self) {
-		state std::vector<OperationInfo> postponedOperations;
-		state int validOperations = 0;
+	Future<Void> runCorrectnessTest(AsyncFileCorrectnessWorkload* self) {
+		std::vector<OperationInfo> postponedOperations;
+		int validOperations = 0;
 
-		loop {
-			wait(delay(0));
+		while (true) {
+			co_await delay(0);
 
 			// Fill the operations buffer with random operations
-			while (self->operations.size() < self->numSimultaneousOperations && postponedOperations.size() == 0) {
+			while (self->operations.size() < self->numSimultaneousOperations && postponedOperations.empty()) {
 				self->operations.push_back(
 				    self->processOperation(self, self->generateOperation(self->operations.size(), false)));
 				validOperations++;
 			}
 
 			// Get the first operation that finishes
-			OperationInfo info = wait(waitForFirst(self->operations));
+			OperationInfo info = co_await waitForFirst(self->operations);
 
 			// If it is a read, check that it matches what our memory representation has
 			if (info.operation == READ) {
@@ -192,13 +187,13 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 						// know
 						if (isValid && memcmp(&self->fileValidityMask[info.offset + start],
 						                      &info.data->buffer[start],
-						                      i - start)) {
+						                      i - start) != 0) {
 							printf("Read returned incorrect results at %" PRIu64 " of length %" PRIu64 "\n",
 							       info.offset,
 							       info.length);
 
 							self->success = false;
-							return Void();
+							co_return;
 						}
 						// Otherwise, skip the comparison and just update what we know
 						else if (!isValid) {
@@ -227,7 +222,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 				memset(&self->fileLock[info.offset], 0, info.length * sizeof(uint32_t));
 
 			// Only generate new operations if we don't have a postponed operation in queue
-			if (postponedOperations.size() == 0) {
+			if (postponedOperations.empty()) {
 				// Insert a new operation into the operations buffer
 				OperationInfo newOperation = self->generateOperation(info.index);
 
@@ -240,16 +235,16 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 			}
 
 			// If there is a postponed operation, clear the queue so that we can run it
-			if (postponedOperations.size() > 0) {
+			if (!postponedOperations.empty()) {
 				self->operations[info.index] = Never();
 				validOperations--;
 			}
 
 			// If there are no operations being processed and postponed operations are waiting, run them now
-			while (validOperations == 0 && postponedOperations.size() > 0) {
+			while (validOperations == 0 && !postponedOperations.empty()) {
 				self->operations.clear();
 				self->operations.push_back(self->processOperation(self, postponedOperations.front()));
-				OperationInfo info = wait(self->operations.front());
+				OperationInfo info = co_await self->operations.front();
 				postponedOperations.erase(postponedOperations.begin());
 				self->operations.clear();
 			}
@@ -358,15 +353,15 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 	}
 
 	// Performs an operation on a file and the memory representation of that file
-	ACTOR Future<OperationInfo> processOperation(AsyncFileCorrectnessWorkload* self, OperationInfo info) {
+	Future<OperationInfo> processOperation(AsyncFileCorrectnessWorkload* self, OperationInfo info) {
 		if (info.operation == READ) {
 			info.data = self->allocateBuffer(info.length);
 
 			// Perform the read.  Don't allow it to be cancelled (because the underlying IO may not be cancellable) and
 			// don't allow objects that the read uses to be deleted
-			int numRead = wait(uncancellable(
+			int numRead = co_await uncancellable(
 			    holdWhile(self->fileHandle,
-			              holdWhile(info, self->fileHandle->file->read(info.data->buffer, info.length, info.offset)))));
+			              holdWhile(info, self->fileHandle->file->read(info.data->buffer, info.length, info.offset))));
 
 			if (numRead != std::min(info.length, self->fileSize - info.offset)) {
 				printf("Read reported incorrect number of bytes at %" PRIu64 " of length %" PRIu64 "\n",
@@ -382,19 +377,19 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 
 			// Perform the write.  Don't allow it to be cancelled (because the underlying IO may not be cancellable) and
 			// don't allow objects that the write uses to be deleted
-			wait(uncancellable(holdWhile(
-			    self->fileHandle,
-			    holdWhile(info, self->fileHandle->file->write(info.data->buffer, info.length, info.offset)))));
+			co_await uncancellable(
+			    holdWhile(self->fileHandle,
+			              holdWhile(info, self->fileHandle->file->write(info.data->buffer, info.length, info.offset))));
 
 			// If we wrote past the end of the file, update the size of the file
 			self->fileSize = std::max((int64_t)(info.offset + info.length), self->fileSize);
 		} else if (info.operation == SYNC) {
 			info.data = Reference<AsyncFileBuffer>(nullptr);
-			wait(self->fileHandle->file->sync());
+			co_await self->fileHandle->file->sync();
 		} else if (info.operation == REOPEN) {
 			// Will fail if the file does not exist
-			wait(self->openFile(self, IAsyncFile::OPEN_READWRITE, 0666, 0, false));
-			int64_t fileSize = wait(self->fileHandle->file->size());
+			co_await self->openFile(self, IAsyncFile::OPEN_READWRITE, 0666, 0, false);
+			int64_t fileSize = co_await self->fileHandle->file->size();
 			int64_t fileSizeChange = fileSize - self->fileSize;
 			if (fileSizeChange >= _PAGE_SIZE) {
 				fmt::print("Reopened file increased in size by {0} bytes (at most {1} allowed)\n",
@@ -410,9 +405,9 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 		} else if (info.operation == TRUNCATE) {
 			// Perform the truncate.  Don't allow it to be cancelled (because the underlying IO may not be cancellable)
 			// and don't allow file handle to be deleted
-			wait(uncancellable(holdWhile(self->fileHandle, self->fileHandle->file->truncate(info.offset))));
+			co_await uncancellable(holdWhile(self->fileHandle, self->fileHandle->file->truncate(info.offset)));
 
-			int64_t fileSize = wait(self->fileHandle->file->size());
+			int64_t fileSize = co_await self->fileHandle->file->size();
 			if (fileSize != info.offset) {
 				printf("Incorrect file size reported after truncate\n");
 				self->success = false;
@@ -422,7 +417,7 @@ struct AsyncFileCorrectnessWorkload : public AsyncFileWorkload {
 		}
 
 		++self->numOperations;
-		return info;
+		co_return info;
 	}
 
 	Future<bool> check(Database const& cx) override { return success; }
diff --git a/fdbserver/workloads/AsyncFileRead.actor.cpp b/fdbserver/workloads/AsyncFileRead.cpp
similarity index 76%
rename from fdbserver/workloads/AsyncFileRead.actor.cpp
rename to fdbserver/workloads/AsyncFileRead.cpp
index c38182a410b..bde3dc0fb8b 100644
--- a/fdbserver/workloads/AsyncFileRead.actor.cpp
+++ b/fdbserver/workloads/AsyncFileRead.cpp
@@ -1,5 +1,5 @@
 /*
- * AsyncFileRead.actor.cpp
+ * AsyncFileRead.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,12 @@
 #include <utility>
 #include <vector>
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
 #include "flow/SystemMonitor.h"
 #include "flow/IAsyncFile.h"
-#include "fdbserver/workloads/AsyncFile.actor.h"
+#include "AsyncFile.h"
 #include "flow/DeterministicRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 static const double ROLL_TIME = 5.0;
 
@@ -172,7 +171,8 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 	IOLog* ioLog;
 	RandomByteGenerator rbg;
 
-	AsyncFileReadWorkload(WorkloadContext const& wcx) : AsyncFileWorkload(wcx), bytesRead("Bytes Read"), ioLog(0) {
+	explicit AsyncFileReadWorkload(WorkloadContext const& wcx)
+	  : AsyncFileWorkload(wcx), bytesRead("Bytes Read"), ioLog(0) {
 		// Only run on one client
 		numParallelReads = getOption(options, "numParallelReads"_sr, 0);
 		readSize = getOption(options, "readSize"_sr, _PAGE_SIZE);
@@ -184,7 +184,7 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 		fixedRate = getOption(options, "fixedRate"_sr, 0.0);
 	}
 
-	~AsyncFileReadWorkload() override {}
+	~AsyncFileReadWorkload() override = default;
 
 	Future<Void> setup(Database const& cx) override {
 		if (enabled)
@@ -193,7 +193,7 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _setup(AsyncFileReadWorkload* self) {
+	Future<Void> _setup(AsyncFileReadWorkload* self) {
 		// Allow only 4K aligned reads if doing unbuffered IO
 		if (self->unbufferedIO && self->readSize % AsyncFileWorkload::_PAGE_SIZE != 0)
 			self->readSize = std::max(AsyncFileWorkload::_PAGE_SIZE,
@@ -203,75 +203,68 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 		for (int i = 0; i < self->numParallelReads; i++)
 			self->readBuffers.push_back(self->allocateBuffer(self->readSize));
 
-		wait(self->openFile(
-		    self, IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0666, self->fileSize, self->fileSize != 0));
+		co_await self->openFile(
+		    self, IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0666, self->fileSize, self->fileSize != 0);
 
-		int64_t fileSize = wait(self->fileHandle->file->size());
+		int64_t fileSize = co_await self->fileHandle->file->size();
 		self->fileSize = fileSize;
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (enabled)
-			return _start(this);
+			return _start();
 
 		return Void();
 	}
 
-	ACTOR Future<Void> _start(AsyncFileReadWorkload* self) {
-		state StatisticsState statState;
+	Future<Void> _start() {
+		StatisticsState statState;
 		customSystemMonitor("AsyncFile Metrics", &statState);
 
-		wait(timeout(self->runReadTest(self), self->testDuration, Void()));
+		co_await timeout(runReadTest(this), testDuration, Void());
 
 		SystemStatistics stats = customSystemMonitor("AsyncFile Metrics", &statState);
-		self->averageCpuUtilization = stats.processCPUSeconds / stats.elapsed;
+		averageCpuUtilization = stats.processCPUSeconds / stats.elapsed;
 
 		// Try to let the IO operations finish so we can clean up after them
-		wait(timeout(waitForAll(self->readFutures), 10, Void()));
-
-		return Void();
+		co_await timeout(waitForAll(readFutures), 10, Void());
 	}
 
-	ACTOR static Future<Void> readLoop(AsyncFileReadWorkload* self, int bufferIndex, double fixedRate) {
-		state bool writeFlag = false;
-		state double begin = 0.0;
-		state double lastTime = now();
-		loop {
+	Future<Void> readLoop(int bufferIndex, double fixedRate) {
+		bool writeFlag = false;
+		double begin = 0.0;
+		double lastTime = now();
+		while (true) {
 			if (fixedRate)
-				wait(poisson(&lastTime, 1.0 / fixedRate));
+				co_await poisson(&lastTime, 1.0 / fixedRate);
 
 			// state Future<Void> d = delay( 1/25. * (.75 + 0.5*deterministicRandom()->random01()) );
 			int64_t offset;
-			if (self->unbufferedIO)
-				offset = (int64_t)(deterministicRandom()->random01() * (self->fileSize - 1) /
-				                   AsyncFileWorkload::_PAGE_SIZE) *
+			if (unbufferedIO)
+				offset = (int64_t)(deterministicRandom()->random01() * (fileSize - 1) / AsyncFileWorkload::_PAGE_SIZE) *
 				         AsyncFileWorkload::_PAGE_SIZE;
 			else
-				offset = (int64_t)(deterministicRandom()->random01() * (self->fileSize - 1));
+				offset = (int64_t)(deterministicRandom()->random01() * (fileSize - 1));
 
-			writeFlag = deterministicRandom()->random01() < self->writeFraction;
+			writeFlag = deterministicRandom()->random01() < writeFraction;
 			if (writeFlag)
-				self->rbg.writeRandomBytesToBuffer((char*)self->readBuffers[bufferIndex]->buffer, self->readSize);
+				rbg.writeRandomBytesToBuffer((char*)readBuffers[bufferIndex]->buffer, readSize);
 
-			auto r =
-			    writeFlag
-			        ? tag(self->fileHandle->file->write(self->readBuffers[bufferIndex]->buffer, self->readSize, offset),
-			              self->readSize)
-			        : self->fileHandle->file->read(self->readBuffers[bufferIndex]->buffer, self->readSize, offset);
+			auto r = writeFlag
+			             ? tag(fileHandle->file->write(readBuffers[bufferIndex]->buffer, readSize, offset), readSize)
+			             : fileHandle->file->read(readBuffers[bufferIndex]->buffer, readSize, offset);
 			begin = now();
-			if (self->ioLog)
-				self->ioLog->logIOIssue(writeFlag, begin);
-			wait(success(uncancellable(holdWhile(self->fileHandle, holdWhile(self->readBuffers[bufferIndex], r)))));
-			if (self->ioLog)
-				self->ioLog->logIOCompletion(writeFlag, begin, now());
-			self->bytesRead += self->readSize;
+			if (ioLog)
+				ioLog->logIOIssue(writeFlag, begin);
+			co_await uncancellable(holdWhile(fileHandle, holdWhile(readBuffers[bufferIndex], r)));
+			if (ioLog)
+				ioLog->logIOCompletion(writeFlag, begin, now());
+			bytesRead += readSize;
 			// wait(d);
 		}
 	}
 
-	ACTOR Future<Void> runReadTest(AsyncFileReadWorkload* self) {
+	Future<Void> runReadTest(AsyncFileReadWorkload* self) {
 		if (self->unbatched) {
 			self->ioLog = new IOLog();
 
@@ -279,15 +272,15 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 
 			readers.reserve(self->numParallelReads);
 			for (int i = 0; i < self->numParallelReads; i++)
-				readers.push_back(readLoop(self, i, self->fixedRate / self->numParallelReads));
-			wait(waitForAll(readers));
+				readers.push_back(readLoop(i, self->fixedRate / self->numParallelReads));
+			co_await waitForAll(readers);
 
 			delete self->ioLog;
-			return Void();
+			co_return;
 		}
 
-		state int64_t offset = self->fileSize;
-		loop {
+		int64_t offset = self->fileSize;
+		while (true) {
 			// Read consecutive chunks of the file using different actors
 			for (int i = 0; i < self->numParallelReads; i++) {
 				if (self->sequential) {
@@ -311,12 +304,12 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload {
 				              self->fileHandle->file->read(self->readBuffers[i]->buffer, self->readSize, offset)))));
 			}
 
-			wait(waitForAll(self->readFutures));
+			co_await waitForAll(self->readFutures);
 			self->bytesRead += self->readSize * self->numParallelReads;
 
 			self->readFutures.clear();
 
-			wait(delay(0));
+			co_await delay(0);
 		}
 	}
 
diff --git a/fdbserver/workloads/AsyncFileWrite.actor.cpp b/fdbserver/workloads/AsyncFileWrite.cpp
similarity index 81%
rename from fdbserver/workloads/AsyncFileWrite.actor.cpp
rename to fdbserver/workloads/AsyncFileWrite.cpp
index 823305afa09..d4efa13bd4e 100644
--- a/fdbserver/workloads/AsyncFileWrite.actor.cpp
+++ b/fdbserver/workloads/AsyncFileWrite.cpp
@@ -1,5 +1,5 @@
 /*
- * AsyncFileWrite.actor.cpp
+ * AsyncFileWrite.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,12 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
 #include "flow/SystemMonitor.h"
 #include "flow/IAsyncFile.h"
-#include "fdbserver/workloads/AsyncFile.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "AsyncFile.h"
 
 struct AsyncFileWriteWorkload : public AsyncFileWorkload {
 	static constexpr auto NAME = "AsyncFileWrite";
@@ -46,7 +45,7 @@ struct AsyncFileWriteWorkload : public AsyncFileWorkload {
 	double averageCpuUtilization;
 	PerfIntCounter bytesWritten;
 
-	AsyncFileWriteWorkload(WorkloadContext const& wcx)
+	explicit AsyncFileWriteWorkload(WorkloadContext const& wcx)
 	  : AsyncFileWorkload(wcx), writeBuffer(nullptr), bytesWritten("Bytes Written") {
 		numParallelWrites = getOption(options, "numParallelWrites"_sr, 0);
 		writeSize = getOption(options, "writeSize"_sr, _PAGE_SIZE);
@@ -61,7 +60,7 @@ struct AsyncFileWriteWorkload : public AsyncFileWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _setup(AsyncFileWriteWorkload* self) {
+	Future<Void> _setup(AsyncFileWriteWorkload* self) {
 		// Allow only 4K aligned writes if using unbuffered IO
 		if (self->unbufferedIO && self->writeSize % AsyncFileWorkload::_PAGE_SIZE != 0)
 			self->writeSize = std::max(AsyncFileWorkload::_PAGE_SIZE,
@@ -74,41 +73,37 @@ struct AsyncFileWriteWorkload : public AsyncFileWorkload {
 		if (self->sequential)
 			initialSize = 0;
 
-		wait(self->openFile(self, IAsyncFile::OPEN_READWRITE, 0666, initialSize));
+		co_await self->openFile(self, IAsyncFile::OPEN_READWRITE, 0666, initialSize);
 
-		int64_t fileSize = wait(self->fileHandle->file->size());
+		int64_t fileSize = co_await self->fileHandle->file->size();
 		if (fileSize != 0)
 			self->fileSize = fileSize;
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (enabled)
-			return _start(this);
+			return _start();
 
 		return Void();
 	}
 
-	ACTOR Future<Void> _start(AsyncFileWriteWorkload* self) {
-		state StatisticsState statState;
+	Future<Void> _start() {
+		StatisticsState statState;
 		customSystemMonitor("AsyncFile Metrics", &statState);
 
-		wait(timeout(self->runWriteTest(self), self->testDuration, Void()));
+		co_await timeout(runWriteTest(this), testDuration, Void());
 
 		SystemStatistics stats = customSystemMonitor("AsyncFile Metrics", &statState);
-		self->averageCpuUtilization = stats.processCPUSeconds / stats.elapsed;
+		averageCpuUtilization = stats.processCPUSeconds / stats.elapsed;
 
 		// Try to let the IO complete so we can clean up after them
-		wait(timeout(waitForAll(self->writeFutures), 10, Void()));
-
-		return Void();
+		co_await timeout(waitForAll(writeFutures), 10, Void());
 	}
 
-	ACTOR Future<Void> runWriteTest(AsyncFileWriteWorkload* self) {
-		state int64_t offset = self->fileSize;
-		state Future<Void> prevSync = Void();
-		loop {
+	Future<Void> runWriteTest(AsyncFileWriteWorkload* self) {
+		int64_t offset = self->fileSize;
+		Future<Void> prevSync = Void();
+		while (true) {
 			// Write chunks of the file using different actors
 			for (int i = 0; i < self->numParallelWrites; i++) {
 				// Perform the write.  Don't allow it to be cancelled (because the underlying IO may not be cancellable)
@@ -134,8 +129,8 @@ struct AsyncFileWriteWorkload : public AsyncFileWorkload {
 					offset = (int64_t)(deterministicRandom()->random01() * (self->fileSize - 1));
 			}
 
-			wait(waitForAll(self->writeFutures));
-			wait(prevSync);
+			co_await waitForAll(self->writeFutures);
+			co_await prevSync;
 			prevSync = self->fileHandle->file->sync();
 
 			self->writeFutures.clear();
diff --git a/fdbserver/workloads/AtomicOps.actor.cpp b/fdbserver/workloads/AtomicOps.cpp
similarity index 74%
rename from fdbserver/workloads/AtomicOps.actor.cpp
rename to fdbserver/workloads/AtomicOps.cpp
index a9efcb3a7c4..94cfbf68a6f 100644
--- a/fdbserver/workloads/AtomicOps.actor.cpp
+++ b/fdbserver/workloads/AtomicOps.cpp
@@ -1,5 +1,5 @@
 /*
- * AtomicOps.actor.cpp
+ * AtomicOps.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // #define SevAtomicOpDebug SevInfo
@@ -38,7 +38,7 @@ struct AtomicOpsWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	uint64_t lbsum, ubsum; // The lower bound and upper bound sum of operations when opType = AddValue
 
-	AtomicOpsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), opNum(0) {
+	explicit AtomicOpsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), opNum(0) {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
 		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
@@ -140,13 +140,14 @@ struct AtomicOpsWorkload : TestWorkload {
 		return std::make_pair(logKey, debugKey);
 	}
 
-	ACTOR Future<Void> _setup(Database cx, AtomicOpsWorkload* self) {
+	Future<Void> _setup(Database cx, AtomicOpsWorkload* self) {
 		// Sanity check if log keyspace has elements
-		state ReadYourWritesTransaction tr1(cx);
-		loop {
+		ReadYourWritesTransaction tr1(cx);
+		while (true) {
+			Error err;
 			try {
 				Key begin(std::string("log"));
-				RangeResult log = wait(tr1.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
+				RangeResult log = co_await tr1.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY);
 				if (!log.empty()) {
 					TraceEvent(SevError, "AtomicOpSetup")
 					    .detail("LogKeySpace", "Not empty")
@@ -159,47 +160,49 @@ struct AtomicOpsWorkload : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr1.onError(e));
+				err = e;
 			}
+			co_await tr1.onError(err);
 		}
 
-		state int g = 0;
-		for (; g < 100; g++) {
-			state ReadYourWritesTransaction tr(cx);
-			loop {
+		for (int g = 0; g < 100; g++) {
+			ReadYourWritesTransaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < self->nodeCount / 100; i++) {
 						uint64_t intValue = 0;
 						tr.set(StringRef(format("ops%08x%08x", g, i)),
 						       StringRef((const uint8_t*)&intValue, sizeof(intValue)));
 					}
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> atomicOpWorker(Database cx, AtomicOpsWorkload* self, double delay) {
-		state double lastTime = now();
-		loop {
-			wait(poisson(&lastTime, delay));
-			state ReadYourWritesTransaction tr(cx);
-			loop {
+	Future<Void> atomicOpWorker(Database cx, AtomicOpsWorkload* self, double delay) {
+		double lastTime = now();
+		while (true) {
+			co_await poisson(&lastTime, delay);
+			ReadYourWritesTransaction tr(cx);
+			while (true) {
 				int group = deterministicRandom()->randomInt(0, 100);
-				state uint64_t intValue = deterministicRandom()->randomInt(0, 10000000);
-				state Key val = StringRef((const uint8_t*)&intValue, sizeof(intValue));
-				state std::pair<Key, Key> logDebugKey = self->logDebugKey(group);
+				uint64_t intValue = deterministicRandom()->randomInt(0, 10000000);
+				Key val = StringRef((const uint8_t*)&intValue, sizeof(intValue));
+				std::pair<Key, Key> logDebugKey = self->logDebugKey(group);
 				int nodeIndex = deterministicRandom()->randomInt(0, self->nodeCount / 100);
-				state Key opsKey(format("ops%08x%08x", group, nodeIndex));
+				Key opsKey(format("ops%08x%08x", group, nodeIndex));
+				Error err;
 				try {
 					tr.set(logDebugKey.first, val); // set log key
 					tr.set(logDebugKey.second, opsKey); // set debug key; one opsKey can have multiple logs key
 					tr.atomicOp(opsKey, val, self->opType);
-					wait(tr.commit());
+					co_await tr.commit();
 					TraceEvent(SevAtomicOpDebug, "AtomicOpWorker")
 					    .detail("OpsKey", opsKey)
 					    .detail("LogKey", logDebugKey.first)
@@ -210,24 +213,26 @@ struct AtomicOpsWorkload : TestWorkload {
 					}
 					break;
 				} catch (Error& e) {
-					if (e.code() == 1021) {
-						self->ubsum += intValue;
-						TraceEvent(SevInfo, "TxnCommitUnknownResult")
-						    .detail("Value", intValue)
-						    .detail("LogKey", logDebugKey.first)
-						    .detail("OpsKey", opsKey);
-					}
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.code() == 1021) {
+					self->ubsum += intValue;
+					TraceEvent(SevInfo, "TxnCommitUnknownResult")
+					    .detail("Value", intValue)
+					    .detail("LogKey", logDebugKey.first)
+					    .detail("OpsKey", opsKey);
 				}
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR Future<Void> dumpLogKV(Database cx, int g) {
-		state ReadYourWritesTransaction tr(cx);
+	Future<Void> dumpLogKV(Database cx, int g) {
+		ReadYourWritesTransaction tr(cx);
+		Error err;
 		try {
 			Key begin(format("log%08x", g));
-			RangeResult log = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
+			RangeResult log = co_await tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY);
 			if (log.more) {
 				TraceEvent(SevError, "LogHitTxnLimits").detail("Result", log.toString());
 			}
@@ -243,17 +248,18 @@ struct AtomicOpsWorkload : TestWorkload {
 				    .detail("CurSum", sum);
 			}
 		} catch (Error& e) {
-			TraceEvent("DumpLogKVError").detail("Error", e.what());
-			wait(tr.onError(e));
+			err = e;
 		}
-		return Void();
+		TraceEvent("DumpLogKVError").detail("Error", err.what());
+		co_await tr.onError(err);
 	}
 
-	ACTOR Future<Void> dumpDebugKV(Database cx, int g) {
-		state ReadYourWritesTransaction tr(cx);
+	Future<Void> dumpDebugKV(Database cx, int g) {
+		ReadYourWritesTransaction tr(cx);
+		Error err;
 		try {
 			Key begin(format("debug%08x", g));
-			RangeResult debuglog = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
+			RangeResult debuglog = co_await tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY);
 			if (debuglog.more) {
 				TraceEvent(SevError, "DebugLogHitTxnLimits").detail("Result", debuglog.toString());
 			}
@@ -261,17 +267,18 @@ struct AtomicOpsWorkload : TestWorkload {
 				TraceEvent("AtomicOpDebug").detail("Key", kv.key).detail("Val", kv.value);
 			}
 		} catch (Error& e) {
-			TraceEvent("DumpDebugKVError").detail("Error", e.what());
-			wait(tr.onError(e));
+			err = e;
 		}
-		return Void();
+		TraceEvent("DumpDebugKVError").detail("Error", err.what());
+		co_await tr.onError(err);
 	}
 
-	ACTOR Future<Void> dumpOpsKV(Database cx, int g) {
-		state ReadYourWritesTransaction tr(cx);
+	Future<Void> dumpOpsKV(Database cx, int g) {
+		ReadYourWritesTransaction tr(cx);
+		Error err;
 		try {
 			Key begin(format("ops%08x", g));
-			RangeResult ops = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
+			RangeResult ops = co_await tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY);
 			if (ops.more) {
 				TraceEvent(SevError, "OpsHitTxnLimits").detail("Result", ops.toString());
 			}
@@ -287,32 +294,32 @@ struct AtomicOpsWorkload : TestWorkload {
 				    .detail("CurSum", sum);
 			}
 		} catch (Error& e) {
-			TraceEvent("DumpOpsKVError").detail("Error", e.what());
-			wait(tr.onError(e));
+			err = e;
 		}
-		return Void();
+		TraceEvent("DumpOpsKVError").detail("Error", err.what());
+		co_await tr.onError(err);
 	}
 
-	ACTOR Future<Void> validateOpsKey(Database cx, AtomicOpsWorkload* self, int g) {
+	Future<Void> validateOpsKey(Database cx, AtomicOpsWorkload* self, int g) {
 		// Get mapping between opsKeys and debugKeys
-		state ReadYourWritesTransaction tr1(cx);
-		state std::map<Key, Key> records; // <ops, debugKey>
-		RangeResult debuglog = wait(tr1.getRange(prefixRange(format("debug%08x", g)), CLIENT_KNOBS->TOO_MANY));
+		ReadYourWritesTransaction tr1(cx);
+		std::map<Key, Key> records; // <ops, debugKey>
+		RangeResult debuglog = co_await tr1.getRange(prefixRange(format("debug%08x", g)), CLIENT_KNOBS->TOO_MANY);
 		if (debuglog.more) {
 			TraceEvent(SevError, "DebugLogHitTxnLimits").detail("Result", debuglog.toString());
-			return Void();
+			co_return;
 		}
 		for (auto& kv : debuglog) {
 			records[kv.value] = kv.key;
 		}
 
 		// Get log key's value and assign it to the associated debugKey
-		state ReadYourWritesTransaction tr2(cx);
-		state std::map<Key, int64_t> logVal; // debugKey, log's value
-		RangeResult log = wait(tr2.getRange(prefixRange(format("log%08x", g)), CLIENT_KNOBS->TOO_MANY));
+		ReadYourWritesTransaction tr2(cx);
+		std::map<Key, int64_t> logVal; // debugKey, log's value
+		RangeResult log = co_await tr2.getRange(prefixRange(format("log%08x", g)), CLIENT_KNOBS->TOO_MANY);
 		if (log.more) {
 			TraceEvent(SevError, "LogHitTxnLimits").detail("Result", log.toString());
-			return Void();
+			co_return;
 		}
 		for (auto& kv : log) {
 			uint64_t intValue = 0;
@@ -321,12 +328,12 @@ struct AtomicOpsWorkload : TestWorkload {
 		}
 
 		// Get opsKeys and validate if it has correct value
-		state ReadYourWritesTransaction tr3(cx);
-		state std::map<Key, int64_t> opsVal; // ops key, ops value
-		RangeResult ops = wait(tr3.getRange(prefixRange(format("ops%08x", g)), CLIENT_KNOBS->TOO_MANY));
+		ReadYourWritesTransaction tr3(cx);
+		std::map<Key, int64_t> opsVal; // ops key, ops value
+		RangeResult ops = co_await tr3.getRange(prefixRange(format("ops%08x", g)), CLIENT_KNOBS->TOO_MANY);
 		if (ops.more) {
 			TraceEvent(SevError, "OpsHitTxnLimits").detail("Result", ops.toString());
-			return Void();
+			co_return;
 		}
 		// Validate if ops' key value is consistent with logs' key value
 		for (auto& kv : ops) {
@@ -353,21 +360,21 @@ struct AtomicOpsWorkload : TestWorkload {
 				TraceEvent(SevError, "MissingOpsKey2").detail("OpsKey", kv.first).detail("DebugKey", kv.second);
 			}
 		}
-		return Void();
 	}
 
-	ACTOR Future<bool> _check(Database cx, AtomicOpsWorkload* self) {
-		state int g = 0;
-		state bool ret = true;
-		for (; g < 100; g++) {
-			state ReadYourWritesTransaction tr(cx);
-			state RangeResult log;
-			loop {
+	Future<bool> _check(Database cx, AtomicOpsWorkload* self) {
+		bool ret = true;
+		for (int g = 0; g < 100; ++g) {
+			ReadYourWritesTransaction tr(cx);
+			RangeResult log;
+			while (true) {
+				Error err;
 				try {
 					{
 						// Calculate the accumulated value in the log keyspace for the group g
 						Key begin(format("log%08x", g));
-						RangeResult log_ = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
+						RangeResult log_ =
+						    co_await tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY);
 						log = log_;
 						uint64_t zeroValue = 0;
 						tr.set("xlogResult"_sr, StringRef((const uint8_t*)&zeroValue, sizeof(zeroValue)));
@@ -381,7 +388,8 @@ struct AtomicOpsWorkload : TestWorkload {
 					{
 						// Calculate the accumulated value in the ops keyspace for the group g
 						Key begin(format("ops%08x", g));
-						RangeResult ops = wait(tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY));
+						RangeResult ops =
+						    co_await tr.getRange(KeyRangeRef(begin, strinc(begin)), CLIENT_KNOBS->TOO_MANY);
 						uint64_t zeroValue = 0;
 						tr.set("xopsResult"_sr, StringRef((const uint8_t*)&zeroValue, sizeof(zeroValue)));
 						for (auto& kv : ops) {
@@ -419,20 +427,21 @@ struct AtomicOpsWorkload : TestWorkload {
 								    .detail("Size", opsResultStr.size())
 								    .detail("LowerBoundSum", self->lbsum)
 								    .detail("UpperBoundSum", self->ubsum);
-								wait(self->dumpLogKV(cx, g));
-								wait(self->dumpDebugKV(cx, g));
-								wait(self->dumpOpsKV(cx, g));
-								wait(self->validateOpsKey(cx, self, g));
+								co_await self->dumpLogKV(cx, g);
+								co_await self->dumpDebugKV(cx, g);
+								co_await self->dumpOpsKV(cx, g);
+								co_await self->validateOpsKey(cx, self, g);
 							}
 						}
 						break;
 					}
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
-		return ret;
+		co_return ret;
 	}
 };
 
diff --git a/fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp b/fdbserver/workloads/AtomicOpsApiCorrectness.cpp
similarity index 58%
rename from fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
rename to fdbserver/workloads/AtomicOpsApiCorrectness.cpp
index ae3d7ec8011..fc3f227b72d 100644
--- a/fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/AtomicOpsApiCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * AtomicOpsApiCorrectness.actor.cpp
+ * AtomicOpsApiCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/RunRYWTransaction.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
@@ -40,7 +40,7 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 	}
 
 public:
-	AtomicOpsApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit AtomicOpsApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		opType = getOption(options, "opType"_sr, -1);
 	}
 
@@ -90,33 +90,36 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
 	// Test Atomic ops on non existing keys that results in a set
-	ACTOR Future<Void> testAtomicOpSetOnNonExistingKey(Database cx,
-	                                                   AtomicOpsApiCorrectnessWorkload* self,
-	                                                   uint32_t opType,
-	                                                   Key key) {
-		state uint64_t intValue = deterministicRandom()->randomInt(0, 10000000);
-		state Value val = StringRef((const uint8_t*)&intValue, sizeof(intValue));
+	Future<Void> testAtomicOpSetOnNonExistingKey(Database cx,
+	                                             AtomicOpsApiCorrectnessWorkload* self,
+	                                             uint32_t opType,
+	                                             Key key) {
+		uint64_t intValue = deterministicRandom()->randomInt(0, 10000000);
+		Value val = StringRef((const uint8_t*)&intValue, sizeof(intValue));
 
 		// Do operation on Storage Server
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->clear(key);
 					return Void();
-				}));
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				});
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->atomicOp(key, val, opType);
 					return Void();
-				}));
+				});
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
-				wait(delay(1));
+				err = e;
 			}
+			TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", err.code());
+			co_await delay(1);
 		}
+
 		{
-			Optional<Value> outputVal = wait(runRYWTransaction(
-			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); }));
+			Optional<Value> outputVal = co_await runRYWTransaction(
+			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); });
 			uint64_t output = 0;
 			ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
 			memcpy(&output, outputVal.get().begin(), outputVal.get().size());
@@ -133,11 +136,11 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 		{
 			// Do operation on RYW Layer
 			Optional<Value> outputVal =
-			    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+			    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 				    tr->clear(key);
 				    tr->atomicOp(key, val, opType);
 				    return tr->get(key);
-			    }));
+			    });
 			uint64_t output = 0;
 			ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
 			memcpy(&output, outputVal.get().begin(), outputVal.get().size());
@@ -150,37 +153,39 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 				self->testFailed = true;
 			}
 		}
-		return Void();
 	}
 
 	// Test Atomic ops on non existing keys that results in a unset
-	ACTOR Future<Void> testAtomicOpUnsetOnNonExistingKey(Database cx,
-	                                                     AtomicOpsApiCorrectnessWorkload* self,
-	                                                     uint32_t opType,
-	                                                     Key key) {
-		state uint64_t intValue = deterministicRandom()->randomInt(0, 10000000);
-		state Value val = StringRef((const uint8_t*)&intValue, sizeof(intValue));
+	Future<Void> testAtomicOpUnsetOnNonExistingKey(Database cx,
+	                                               AtomicOpsApiCorrectnessWorkload* self,
+	                                               uint32_t opType,
+	                                               Key key) {
+		uint64_t intValue = deterministicRandom()->randomInt(0, 10000000);
+		Value val = StringRef((const uint8_t*)&intValue, sizeof(intValue));
 
 		// Do operation on Storage Server
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->clear(key);
 					return Void();
-				}));
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				});
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->atomicOp(key, val, opType);
 					return Void();
-				}));
+				});
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
-				wait(delay(1));
+				err = e;
 			}
+			TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", err.code());
+			co_await delay(1);
 		}
+
 		{
-			Optional<Value> outputVal = wait(runRYWTransaction(
-			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); }));
+			Optional<Value> outputVal = co_await runRYWTransaction(
+			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); });
 			uint64_t output = 0;
 			ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
 			memcpy(&output, outputVal.get().begin(), outputVal.get().size());
@@ -197,11 +202,11 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 		{
 			// Do operation on RYW Layer
 			Optional<Value> outputVal =
-			    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+			    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 				    tr->clear(key);
 				    tr->atomicOp(key, val, opType);
 				    return tr->get(key);
-			    }));
+			    });
 			uint64_t output = 0;
 			ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
 			memcpy(&output, outputVal.get().begin(), outputVal.get().size());
@@ -214,20 +219,19 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 				self->testFailed = true;
 			}
 		}
-		return Void();
 	}
 
-	typedef std::function<Value(Value, Value)> DoAtomicOpOnEmptyValueFunction;
+	using DoAtomicOpOnEmptyValueFunction = std::function<Value(Value, Value)>;
 
 	// Test Atomic Ops when one of the value is empty
-	ACTOR Future<Void> testAtomicOpOnEmptyValue(Database cx,
-	                                            AtomicOpsApiCorrectnessWorkload* self,
-	                                            uint32_t opType,
-	                                            Key key,
-	                                            DoAtomicOpOnEmptyValueFunction opFunc) {
-		state Value existingVal;
-		state Value otherVal;
-		state uint64_t val = deterministicRandom()->randomInt(0, 10000000);
+	Future<Void> testAtomicOpOnEmptyValue(Database cx,
+	                                      AtomicOpsApiCorrectnessWorkload* self,
+	                                      uint32_t opType,
+	                                      Key key,
+	                                      DoAtomicOpOnEmptyValueFunction opFunc) {
+		Value existingVal;
+		Value otherVal;
+		uint64_t val = deterministicRandom()->randomInt(0, 10000000);
 		if (deterministicRandom()->random01() < 0.5) {
 			existingVal = StringRef((const uint8_t*)&val, sizeof(val));
 			otherVal = StringRef();
@@ -236,25 +240,28 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 			existingVal = StringRef();
 		}
 		// Do operation on Storage Server
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->set(key, existingVal);
 					return Void();
-				}));
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				});
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->atomicOp(key, otherVal, opType);
 					return Void();
-				}));
+				});
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
-				wait(delay(1));
+				err = e;
 			}
+			TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", err.code());
+			co_await delay(1);
 		}
+
 		{
-			Optional<Value> outputVal = wait(runRYWTransaction(
-			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); }));
+			Optional<Value> outputVal = co_await runRYWTransaction(
+			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); });
 			ASSERT(outputVal.present());
 			Value output = outputVal.get();
 			if (output != opFunc(existingVal, otherVal)) {
@@ -270,11 +277,11 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 		{
 			// Do operation on RYW Layer
 			Optional<Value> outputVal =
-			    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+			    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 				    tr->set(key, existingVal);
 				    tr->atomicOp(key, otherVal, opType);
 				    return tr->get(key);
-			    }));
+			    });
 			ASSERT(outputVal.present());
 			Value output = outputVal.get();
 			if (output != opFunc(existingVal, otherVal)) {
@@ -286,45 +293,47 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 				self->testFailed = true;
 			}
 		}
-		return Void();
 	}
 
-	typedef std::function<uint64_t(uint64_t, uint64_t)> DoAtomicOpFunction;
+	using DoAtomicOpFunction = std::function<uint64_t(uint64_t, uint64_t)>;
 
 	// Test atomic ops in the normal case when the existing value is present
-	ACTOR Future<Void> testAtomicOpApi(Database cx,
-	                                   AtomicOpsApiCorrectnessWorkload* self,
-	                                   uint32_t opType,
-	                                   Key key,
-	                                   DoAtomicOpFunction opFunc) {
-		state uint64_t intValue1 = deterministicRandom()->randomInt(0, 10000000);
-		state uint64_t intValue2 = deterministicRandom()->randomInt(0, 10000000);
-		state Value val1 = StringRef((const uint8_t*)&intValue1, sizeof(intValue1));
-		state Value val2 = StringRef((const uint8_t*)&intValue2, sizeof(intValue2));
+	Future<Void> testAtomicOpApi(Database cx,
+	                             AtomicOpsApiCorrectnessWorkload* self,
+	                             uint32_t opType,
+	                             Key key,
+	                             DoAtomicOpFunction opFunc) {
+		uint64_t intValue1 = deterministicRandom()->randomInt(0, 10000000);
+		uint64_t intValue2 = deterministicRandom()->randomInt(0, 10000000);
+		Value val1 = StringRef((const uint8_t*)&intValue1, sizeof(intValue1));
+		Value val2 = StringRef((const uint8_t*)&intValue2, sizeof(intValue2));
 
 		// Do operation on Storage Server
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// Set the key to a random value
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->set(key, val1);
 					return Void();
-				}));
+				});
 				// Do atomic op
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->atomicOp(key, val2, opType);
 					return Void();
-				}));
+				});
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
-				wait(delay(1));
+				err = e;
 			}
+			TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", err.code());
+			co_await delay(1);
 		}
+
 		{
 			// Compare result
-			Optional<Value> outputVal = wait(runRYWTransaction(
-			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); }));
+			Optional<Value> outputVal = co_await runRYWTransaction(
+			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); });
 			uint64_t output = 0;
 			ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
 			memcpy(&output, outputVal.get().begin(), outputVal.get().size());
@@ -343,11 +352,11 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 		{
 			// Do operation at RYW layer
 			Optional<Value> outputVal =
-			    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+			    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 				    tr->set(key, val1);
 				    tr->atomicOp(key, val2, opType);
 				    return tr->get(key);
-			    }));
+			    });
 			// Compare result
 			uint64_t output = 0;
 			ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
@@ -363,22 +372,20 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 				self->testFailed = true;
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> testCompareAndClearAtomicOpApi(Database cx,
-	                                                  AtomicOpsApiCorrectnessWorkload* self,
-	                                                  Key key,
-	                                                  bool keySet) {
-		state uint64_t opType = MutationRef::CompareAndClear;
-		state uint64_t intValue1 = deterministicRandom()->randomInt(0, 10000000);
-		state uint64_t intValue2 =
+	Future<Void> testCompareAndClearAtomicOpApi(Database cx,
+	                                            AtomicOpsApiCorrectnessWorkload* self,
+	                                            Key key,
+	                                            bool keySet) {
+		uint64_t opType = MutationRef::CompareAndClear;
+		uint64_t intValue1 = deterministicRandom()->randomInt(0, 10000000);
+		uint64_t intValue2 =
 		    deterministicRandom()->coinflip() ? intValue1 : deterministicRandom()->randomInt(0, 10000000);
 
-		state Value val1 = StringRef((const uint8_t*)&intValue1, sizeof(intValue1));
-		state Value val2 = StringRef((const uint8_t*)&intValue2, sizeof(intValue2));
-		state std::function<Optional<uint64_t>(uint64_t, uint64_t)> opFunc = [keySet](uint64_t val1, uint64_t val2) {
+		Value val1 = StringRef((const uint8_t*)&intValue1, sizeof(intValue1));
+		Value val2 = StringRef((const uint8_t*)&intValue2, sizeof(intValue2));
+		std::function<Optional<uint64_t>(uint64_t, uint64_t)> opFunc = [keySet](uint64_t val1, uint64_t val2) {
 			if (!keySet || val1 == val2) {
 				return Optional<uint64_t>();
 			} else {
@@ -387,35 +394,37 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 		};
 
 		// Do operation on Storage Server
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// Set the key to a random value
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					if (keySet) {
 						tr->set(key, val1);
 					} else {
 						tr->clear(key);
 					}
 					return Void();
-				}));
+				});
 
 				// Do atomic op
-				wait(runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransactionNoRetry(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->atomicOp(key, val2, opType);
 					return Void();
-				}));
+				});
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
-				wait(delay(1));
+				err = e;
 			}
+			TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", err.code());
+			co_await delay(1);
 		}
 
-		state Optional<uint64_t> expectedOutput;
+		Optional<uint64_t> expectedOutput;
 		{
 			// Compare result
-			Optional<Value> outputVal = wait(runRYWTransaction(
-			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); }));
+			Optional<Value> outputVal = co_await runRYWTransaction(
+			    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> { return tr->get(key); });
 			Optional<uint64_t> expectedOutput_ = opFunc(intValue1, intValue2);
 			expectedOutput = expectedOutput_;
 
@@ -440,7 +449,7 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 		{
 			// Do operation at RYW layer
 			Optional<Value> outputVal =
-			    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+			    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 				    if (keySet) {
 					    tr->set(key, val1);
 				    } else {
@@ -448,7 +457,7 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 				    }
 				    tr->atomicOp(key, val2, opType);
 				    return tr->get(key);
-			    }));
+			    });
 
 			// Compare result
 			ASSERT(outputVal.present() == expectedOutput.present());
@@ -468,174 +477,157 @@ struct AtomicOpsApiCorrectnessWorkload : TestWorkload {
 				}
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> testMin(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state int currentApiVersion = getApiVersion(cx);
-		state Key key = self->getTestKey("test_key_min_");
+	Future<Void> testMin(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		int currentApiVersion = getApiVersion(cx);
+		Key key = self->getTestKey("test_key_min_");
 
 		TraceEvent("AtomicOpCorrectnessApiWorkload").detail("OpType", "MIN");
 		// API Version 500
 		setApiVersion(&cx, 500);
 		TraceEvent(SevInfo, "Running Atomic Op Min Correctness Test Api Version 500").log();
-		wait(self->testAtomicOpUnsetOnNonExistingKey(cx, self, MutationRef::Min, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::Min, key, [](uint64_t val1, uint64_t val2) { return val1 < val2 ? val1 : val2; }));
-		wait(self->testAtomicOpOnEmptyValue(cx, self, MutationRef::Min, key, [](Value v1, Value v2) -> Value {
+		co_await self->testAtomicOpUnsetOnNonExistingKey(cx, self, MutationRef::Min, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::Min, key, [](uint64_t val1, uint64_t val2) { return val1 < val2 ? val1 : val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::Min, key, [](Value v1, Value v2) -> Value {
 			uint64_t zeroVal = 0;
-			if (v2.size() == 0)
+			if (v2.empty())
 				return StringRef();
 			else
 				return StringRef((const uint8_t*)&zeroVal, sizeof(zeroVal));
-		}));
+		});
 
 		// Current API Version
 		setApiVersion(&cx, currentApiVersion);
 		TraceEvent(SevInfo, "Running Atomic Op Min Correctness Current Api Version")
 		    .detail("Version", currentApiVersion);
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Min, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::Min, key, [](uint64_t val1, uint64_t val2) { return val1 < val2 ? val1 : val2; }));
-		wait(self->testAtomicOpOnEmptyValue(cx, self, MutationRef::Min, key, [](Value v1, Value v2) -> Value {
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Min, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::Min, key, [](uint64_t val1, uint64_t val2) { return val1 < val2 ? val1 : val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::Min, key, [](Value v1, Value v2) -> Value {
 			uint64_t zeroVal = 0;
-			if (v2.size() == 0)
+			if (v2.empty())
 				return StringRef();
 			else
 				return StringRef((const uint8_t*)&zeroVal, sizeof(zeroVal));
-		}));
-
-		return Void();
+		});
 	}
 
-	ACTOR Future<Void> testMax(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_max_");
+	Future<Void> testMax(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_max_");
 
 		TraceEvent(SevInfo, "Running Atomic Op MAX Correctness Current Api Version").log();
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Max, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::Max, key, [](uint64_t val1, uint64_t val2) { return val1 > val2 ? val1 : val2; }));
-		wait(self->testAtomicOpOnEmptyValue(
-		    cx, self, MutationRef::Max, key, [](Value v1, Value v2) -> Value { return v2.size() ? v2 : StringRef(); }));
-
-		return Void();
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Max, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::Max, key, [](uint64_t val1, uint64_t val2) { return val1 > val2 ? val1 : val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::Max, key, [](Value v1, Value v2) -> Value {
+			return !v2.empty() ? v2 : StringRef();
+		});
 	}
 
-	ACTOR Future<Void> testAnd(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state int currentApiVersion = getApiVersion(cx);
-		state Key key = self->getTestKey("test_key_and_");
+	Future<Void> testAnd(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		int currentApiVersion = getApiVersion(cx);
+		Key key = self->getTestKey("test_key_and_");
 
 		TraceEvent("AtomicOpCorrectnessApiWorkload").detail("OpType", "AND");
 		// API Version 500
 		setApiVersion(&cx, 500);
 		TraceEvent(SevInfo, "Running Atomic Op AND Correctness Test Api Version 500").log();
-		wait(self->testAtomicOpUnsetOnNonExistingKey(cx, self, MutationRef::And, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::And, key, [](uint64_t val1, uint64_t val2) { return val1 & val2; }));
-		wait(self->testAtomicOpOnEmptyValue(cx, self, MutationRef::And, key, [](Value v1, Value v2) -> Value {
+		co_await self->testAtomicOpUnsetOnNonExistingKey(cx, self, MutationRef::And, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::And, key, [](uint64_t val1, uint64_t val2) { return val1 & val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::And, key, [](Value v1, Value v2) -> Value {
 			uint64_t zeroVal = 0;
-			if (v2.size() == 0)
+			if (v2.empty())
 				return StringRef();
 			else
 				return StringRef((const uint8_t*)&zeroVal, sizeof(zeroVal));
-		}));
+		});
 
 		// Current API Version
 		setApiVersion(&cx, currentApiVersion);
 		TraceEvent(SevInfo, "Running Atomic Op AND Correctness Current Api Version")
 		    .detail("Version", currentApiVersion);
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::And, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::And, key, [](uint64_t val1, uint64_t val2) { return val1 & val2; }));
-		wait(self->testAtomicOpOnEmptyValue(cx, self, MutationRef::And, key, [](Value v1, Value v2) -> Value {
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::And, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::And, key, [](uint64_t val1, uint64_t val2) { return val1 & val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::And, key, [](Value v1, Value v2) -> Value {
 			uint64_t zeroVal = 0;
-			if (v2.size() == 0)
+			if (v2.empty())
 				return StringRef();
 			else
 				return StringRef((const uint8_t*)&zeroVal, sizeof(zeroVal));
-		}));
-
-		return Void();
+		});
 	}
 
-	ACTOR Future<Void> testOr(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_or_");
+	Future<Void> testOr(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_or_");
 
 		TraceEvent(SevInfo, "Running Atomic Op OR Correctness Current Api Version").log();
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Or, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::Or, key, [](uint64_t val1, uint64_t val2) { return val1 | val2; }));
-		wait(self->testAtomicOpOnEmptyValue(
-		    cx, self, MutationRef::Or, key, [](Value v1, Value v2) -> Value { return v2.size() ? v2 : StringRef(); }));
-
-		return Void();
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Or, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::Or, key, [](uint64_t val1, uint64_t val2) { return val1 | val2; });
+		co_await self->testAtomicOpOnEmptyValue(
+		    cx, self, MutationRef::Or, key, [](Value v1, Value v2) -> Value { return !v2.empty() ? v2 : StringRef(); });
 	}
 
-	ACTOR Future<Void> testXor(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_xor_");
+	Future<Void> testXor(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_xor_");
 
 		TraceEvent(SevInfo, "Running Atomic Op XOR Correctness Current Api Version").log();
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Xor, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::Xor, key, [](uint64_t val1, uint64_t val2) { return val1 ^ val2; }));
-		wait(self->testAtomicOpOnEmptyValue(
-		    cx, self, MutationRef::Xor, key, [](Value v1, Value v2) -> Value { return v2.size() ? v2 : StringRef(); }));
-
-		return Void();
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::Xor, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::Xor, key, [](uint64_t val1, uint64_t val2) { return val1 ^ val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::Xor, key, [](Value v1, Value v2) -> Value {
+			return !v2.empty() ? v2 : StringRef();
+		});
 	}
 
-	ACTOR Future<Void> testAdd(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_add_");
+	Future<Void> testAdd(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_add_");
 		TraceEvent(SevInfo, "Running Atomic Op ADD Correctness Current Api Version").log();
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::AddValue, key));
-		wait(self->testAtomicOpApi(
-		    cx, self, MutationRef::AddValue, key, [](uint64_t val1, uint64_t val2) { return val1 + val2; }));
-		wait(self->testAtomicOpOnEmptyValue(cx, self, MutationRef::AddValue, key, [](Value v1, Value v2) -> Value {
-			return v2.size() ? v2 : StringRef();
-		}));
-
-		return Void();
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::AddValue, key);
+		co_await self->testAtomicOpApi(
+		    cx, self, MutationRef::AddValue, key, [](uint64_t val1, uint64_t val2) { return val1 + val2; });
+		co_await self->testAtomicOpOnEmptyValue(cx, self, MutationRef::AddValue, key, [](Value v1, Value v2) -> Value {
+			return !v2.empty() ? v2 : StringRef();
+		});
 	}
 
-	ACTOR Future<Void> testCompareAndClear(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_compare_and_clear_");
+	Future<Void> testCompareAndClear(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_compare_and_clear_");
 		TraceEvent(SevInfo, "Running Atomic Op COMPARE_AND_CLEAR Correctness Current Api Version").log();
-		wait(self->testCompareAndClearAtomicOpApi(cx, self, key, true));
-		wait(self->testCompareAndClearAtomicOpApi(cx, self, key, false));
-		return Void();
+		co_await self->testCompareAndClearAtomicOpApi(cx, self, key, true);
+		co_await self->testCompareAndClearAtomicOpApi(cx, self, key, false);
 	}
 
-	ACTOR Future<Void> testByteMin(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_byte_min_");
+	Future<Void> testByteMin(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_byte_min_");
 
 		TraceEvent(SevInfo, "Running Atomic Op BYTE_MIN Correctness Current Api Version").log();
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::ByteMin, key));
-		wait(self->testAtomicOpApi(cx, self, MutationRef::ByteMin, key, [](uint64_t val1, uint64_t val2) {
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::ByteMin, key);
+		co_await self->testAtomicOpApi(cx, self, MutationRef::ByteMin, key, [](uint64_t val1, uint64_t val2) {
 			return StringRef((const uint8_t*)&val1, sizeof(val1)) < StringRef((const uint8_t*)&val2, sizeof(val2))
 			           ? val1
 			           : val2;
-		}));
-		wait(self->testAtomicOpOnEmptyValue(
-		    cx, self, MutationRef::ByteMin, key, [](Value v1, Value v2) -> Value { return StringRef(); }));
-
-		return Void();
+		});
+		co_await self->testAtomicOpOnEmptyValue(
+		    cx, self, MutationRef::ByteMin, key, [](Value v1, Value v2) -> Value { return StringRef(); });
 	}
 
-	ACTOR Future<Void> testByteMax(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
-		state Key key = self->getTestKey("test_key_byte_max_");
+	Future<Void> testByteMax(Database cx, AtomicOpsApiCorrectnessWorkload* self) {
+		Key key = self->getTestKey("test_key_byte_max_");
 
 		TraceEvent(SevInfo, "Running Atomic Op BYTE_MAX Correctness Current Api Version").log();
-		wait(self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::ByteMax, key));
-		wait(self->testAtomicOpApi(cx, self, MutationRef::ByteMax, key, [](uint64_t val1, uint64_t val2) {
+		co_await self->testAtomicOpSetOnNonExistingKey(cx, self, MutationRef::ByteMax, key);
+		co_await self->testAtomicOpApi(cx, self, MutationRef::ByteMax, key, [](uint64_t val1, uint64_t val2) {
 			return StringRef((const uint8_t*)&val1, sizeof(val1)) > StringRef((const uint8_t*)&val2, sizeof(val2))
 			           ? val1
 			           : val2;
-		}));
-		wait(self->testAtomicOpOnEmptyValue(
-		    cx, self, MutationRef::ByteMax, key, [](Value v1, Value v2) -> Value { return v1.size() ? v1 : v2; }));
-
-		return Void();
+		});
+		co_await self->testAtomicOpOnEmptyValue(
+		    cx, self, MutationRef::ByteMax, key, [](Value v1, Value v2) -> Value { return !v1.empty() ? v1 : v2; });
 	}
 };
 
diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.cpp
similarity index 50%
rename from fdbserver/workloads/AtomicRestore.actor.cpp
rename to fdbserver/workloads/AtomicRestore.cpp
index 671fbfca5d6..cb831fa6c71 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.cpp
@@ -1,5 +1,5 @@
 /*
- * AtomicRestore.actor.cpp
+ * AtomicRestore.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,41 +18,31 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RestoreCommon.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 // Workload to test atomicRestore().
 
 struct AtomicRestoreWorkload : TestWorkload {
 	static constexpr auto NAME = "AtomicRestore";
 	double startAfter, restoreAfter;
-	bool fastRestore; // true: use fast restore, false: use old style restore
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
-	UsePartitionedLog usePartitionedLogs{ false };
+	MutationLogType mutationLogType{ MutationLogType::DEFAULT };
 	Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix
 	// CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix
 
-	AtomicRestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit AtomicRestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 
 		startAfter = getOption(options, "startAfter"_sr, 10.0);
 		restoreAfter = getOption(options, "restoreAfter"_sr, 20.0);
-		fastRestore = getOption(options, "fastRestore"_sr, false);
-		if (!fastRestore) {
-			addDefaultBackupRanges(backupRanges);
-		} else {
-			// Fast restore doesn't support multiple ranges yet
-			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-		}
-		usePartitionedLogs.set(
-		    getOption(options, "usePartitionedLogs"_sr, deterministicRandom()->random01() < 0.5 ? true : false));
+		addDefaultBackupRanges(backupRanges);
+		mutationLogType = static_cast<MutationLogType>(
+		    getOption(options, "mutationLogType"_sr, deterministicRandom()->randomInt(0, 2)));
 
 		addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
 		removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
@@ -71,10 +61,10 @@ struct AtomicRestoreWorkload : TestWorkload {
 		// 	}
 		// }
 		TraceEvent("AtomicRestoreWorkload").detail("AddPrefix", addPrefix).detail("RemovePrefix", removePrefix);
-		ASSERT(addPrefix.size() == 0 && removePrefix.size() == 0);
+		ASSERT(addPrefix.empty() && removePrefix.empty());
 		// Do not support removePrefix right now because we must ensure all backup keys have the removePrefix
 		// otherwise, test will fail because fast restore will simply add the removePrefix to every key in the end.
-		ASSERT(removePrefix.size() == 0);
+		ASSERT(removePrefix.empty());
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
@@ -82,70 +72,63 @@ struct AtomicRestoreWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	bool hasPrefix() const { return addPrefix != ""_sr || removePrefix != ""_sr; }
+	bool hasPrefix() const { return !addPrefix.empty() || !removePrefix.empty(); }
 
-	ACTOR static Future<Void> _start(Database cx, AtomicRestoreWorkload* self) {
-		state FileBackupAgent backupAgent;
+	Future<Void> _start(Database cx) {
+		FileBackupAgent backupAgent;
 
-		wait(delay(self->startAfter * deterministicRandom()->random01()));
-		TraceEvent("AtomicRestore_Start").detail("UsePartitionedLog", self->usePartitionedLogs);
+		co_await delay(startAfter * deterministicRandom()->random01());
+		TraceEvent("AtomicRestore_Start").detail("MutationLogType", mutationLogType);
 
-		state std::string backupContainer = "file://simfdb/backups/";
+		std::string backupContainer = "file://simfdb/backups/";
 		try {
-			wait(backupAgent.submitBackup(cx,
-			                              StringRef(backupContainer),
-			                              {},
-			                              deterministicRandom()->randomInt(0, 60),
-			                              deterministicRandom()->randomInt(0, 100),
-			                              BackupAgentBase::getDefaultTagName(),
-			                              self->backupRanges,
-			                              StopWhenDone::False,
-			                              self->usePartitionedLogs,
-			                              IncrementalBackupOnly::False,
-			                              {}));
+			co_await backupAgent.submitBackup(cx,
+			                                  StringRef(backupContainer),
+			                                  {},
+			                                  deterministicRandom()->randomInt(0, 60),
+			                                  deterministicRandom()->randomInt(0, 100),
+			                                  BackupAgentBase::getDefaultTagName(),
+			                                  backupRanges,
+			                                  StopWhenDone::False,
+			                                  mutationLogType,
+			                                  IncrementalBackupOnly::False,
+			                                  {});
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 				throw;
 		}
 
 		TraceEvent("AtomicRestore_Wait").log();
-		wait(success(backupAgent.waitBackup(cx, BackupAgentBase::getDefaultTagName(), StopWhenDone::False)));
+		co_await backupAgent.waitBackup(cx, BackupAgentBase::getDefaultTagName(), StopWhenDone::False);
 		TraceEvent("AtomicRestore_BackupStart").log();
-		wait(delay(self->restoreAfter * deterministicRandom()->random01()));
+		co_await delay(restoreAfter * deterministicRandom()->random01());
 		TraceEvent("AtomicRestore_RestoreStart").log();
 
-		if (self->fastRestore) { // New fast parallel restore
-			TraceEvent(SevInfo, "AtomicParallelRestore").log();
-			wait(backupAgent.atomicParallelRestore(
-			    cx, BackupAgentBase::getDefaultTag(), self->backupRanges, self->addPrefix, self->removePrefix));
-		} else { // Old style restore
-			loop {
-				try {
-					wait(success(backupAgent.atomicRestore(
-					    cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef())));
-					break;
-				} catch (Error& e) {
-					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-						throw;
-				}
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+		while (true) {
+			try {
+				co_await backupAgent.atomicRestore(
+				    cx, BackupAgentBase::getDefaultTag(), backupRanges, StringRef(), StringRef());
+				break;
+			} catch (Error& e) {
+				if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
+					throw;
 			}
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 		}
 
 		// SOMEDAY: Remove after backup agents can exist quiescently
-		if (g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) {
-			g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+		if (fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::BackupToFile) {
+			fdbSimulationPolicyState().backupAgents = FDBBackupAgentType::NoBackupAgents;
 		}
 
 		TraceEvent("AtomicRestore_Done").log();
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/AtomicSwitchover.actor.cpp b/fdbserver/workloads/AtomicSwitchover.cpp
similarity index 59%
rename from fdbserver/workloads/AtomicSwitchover.actor.cpp
rename to fdbserver/workloads/AtomicSwitchover.cpp
index 4ca095b0943..f1a30623b57 100644
--- a/fdbserver/workloads/AtomicSwitchover.actor.cpp
+++ b/fdbserver/workloads/AtomicSwitchover.cpp
@@ -1,5 +1,5 @@
 /*
- * AtomicSwitchover.actor.cpp
+ * AtomicSwitchover.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,10 @@
  */
 
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 // A workload which test the correctness of backup and restore process
 struct AtomicSwitchoverWorkload : TestWorkload {
@@ -32,7 +31,7 @@ struct AtomicSwitchoverWorkload : TestWorkload {
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
 	Database extraDB;
 
-	AtomicSwitchoverWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit AtomicSwitchoverWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 
 		switch1delay = getOption(options, "switch1delay"_sr, 50.0);
 		switch2delay = getOption(options, "switch2delay"_sr, 50.0);
@@ -40,62 +39,61 @@ struct AtomicSwitchoverWorkload : TestWorkload {
 
 		addDefaultBackupRanges(backupRanges);
 
-		ASSERT(g_simulator->extraDatabases.size() == 1);
-		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+		ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 	}
 
 	Future<Void> setup(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _setup(cx, this);
+		return _setup(cx);
 	}
 
-	ACTOR static Future<Void> _setup(Database cx, AtomicSwitchoverWorkload* self) {
-		state DatabaseBackupAgent backupAgent(cx);
+	Future<Void> _setup(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
 		try {
 			TraceEvent("AS_Submit1").log();
-			wait(backupAgent.submitBackup(self->extraDB,
-			                              BackupAgentBase::getDefaultTag(),
-			                              self->backupRanges,
-			                              StopWhenDone::False,
-			                              StringRef(),
-			                              StringRef(),
-			                              LockDB::True));
+			co_await backupAgent.submitBackup(extraDB,
+			                                  BackupAgentBase::getDefaultTag(),
+			                                  backupRanges,
+			                                  StopWhenDone::False,
+			                                  StringRef(),
+			                                  StringRef(),
+			                                  LockDB::True);
 			TraceEvent("AS_Submit2").log();
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_duplicate)
 				throw;
 		}
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> diffRanges(Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                     StringRef backupPrefix,
-	                                     Database src,
-	                                     Database dest) {
-		state int rangeIndex;
-		for (rangeIndex = 0; rangeIndex < ranges.size(); ++rangeIndex) {
-			state KeyRangeRef range = ranges[rangeIndex];
-			state Key begin = range.begin;
-			loop {
-				state Transaction tr(src);
-				state Transaction tr2(dest);
+	static Future<Void> diffRanges(Standalone<VectorRef<KeyRangeRef>> ranges,
+	                               StringRef backupPrefix,
+	                               Database src,
+	                               Database dest) {
+		for (int rangeIndex = 0; rangeIndex < ranges.size(); ++rangeIndex) {
+			KeyRangeRef range = ranges[rangeIndex];
+			Key begin = range.begin;
+			while (true) {
+				Transaction tr(src);
+				Transaction tr2(dest);
+				Error err;
 				try {
-					loop {
-						state Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
-						state Future<RangeResult> bkpFuture =
+					while (true) {
+						Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
+						Future<RangeResult> bkpFuture =
 						    tr2.getRange(KeyRangeRef(begin, range.end).withPrefix(backupPrefix), 1000);
-						wait(success(srcFuture) && success(bkpFuture));
+						co_await (success(srcFuture) && success(bkpFuture));
 
 						auto src = srcFuture.get().begin();
 						auto bkp = bkpFuture.get().begin();
@@ -155,46 +153,43 @@ struct AtomicSwitchoverWorkload : TestWorkload {
 
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, AtomicSwitchoverWorkload* self) {
-		state DatabaseBackupAgent backupAgent(cx);
-		state DatabaseBackupAgent restoreTool(self->extraDB);
+	Future<Void> _start(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
+		DatabaseBackupAgent restoreTool(extraDB);
 
 		TraceEvent("AS_Wait1").log();
-		wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::False)));
+		co_await backupAgent.waitBackup(extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::False);
 		TraceEvent("AS_Ready1").log();
-		wait(delay(deterministicRandom()->random01() * self->switch1delay));
+		co_await delay(deterministicRandom()->random01() * switch1delay);
 		TraceEvent("AS_Switch1").log();
-		wait(backupAgent.atomicSwitchover(
-		    self->extraDB, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
+		co_await backupAgent.atomicSwitchover(
+		    extraDB, BackupAgentBase::getDefaultTag(), backupRanges, StringRef(), StringRef());
 		TraceEvent("AS_Wait2").log();
-		wait(success(restoreTool.waitBackup(cx, BackupAgentBase::getDefaultTag(), StopWhenDone::False)));
+		co_await restoreTool.waitBackup(cx, BackupAgentBase::getDefaultTag(), StopWhenDone::False);
 		TraceEvent("AS_Ready2").log();
-		wait(delay(deterministicRandom()->random01() * self->switch2delay));
+		co_await delay(deterministicRandom()->random01() * switch2delay);
 		TraceEvent("AS_Switch2").log();
-		wait(restoreTool.atomicSwitchover(
-		    cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef()));
+		co_await restoreTool.atomicSwitchover(
+		    cx, BackupAgentBase::getDefaultTag(), backupRanges, StringRef(), StringRef());
 		TraceEvent("AS_Wait3").log();
-		wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::False)));
+		co_await backupAgent.waitBackup(extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::False);
 		TraceEvent("AS_Ready3").log();
-		wait(delay(deterministicRandom()->random01() * self->stopDelay));
+		co_await delay(deterministicRandom()->random01() * stopDelay);
 		TraceEvent("AS_Abort").log();
-		wait(backupAgent.abortBackup(self->extraDB, BackupAgentBase::getDefaultTag()));
+		co_await backupAgent.abortBackup(extraDB, BackupAgentBase::getDefaultTag());
 		TraceEvent("AS_Done").log();
 
 		// SOMEDAY: Remove after backup agents can exist quiescently
-		if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
-			g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+		if (fdbSimulationPolicyState().drAgents == FDBBackupAgentType::BackupToDB) {
+			fdbSimulationPolicyState().drAgents = FDBBackupAgentType::NoBackupAgents;
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/AutomaticIdempotencyWorkload.actor.cpp b/fdbserver/workloads/AutomaticIdempotencyWorkload.cpp
similarity index 64%
rename from fdbserver/workloads/AutomaticIdempotencyWorkload.actor.cpp
rename to fdbserver/workloads/AutomaticIdempotencyWorkload.cpp
index ca33e83fab0..e01de12672d 100644
--- a/fdbserver/workloads/AutomaticIdempotencyWorkload.actor.cpp
+++ b/fdbserver/workloads/AutomaticIdempotencyWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * AutomaticIdempotencyWorkload.actor.cpp
+ * AutomaticIdempotencyWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,12 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/RunRYWTransaction.h"
+#include "fdbserver/tester/workloads.h"
+#include "flow/CoroUtils.h"
 
 namespace {
 struct ValueType {
@@ -70,7 +70,7 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 	KeyBackedProperty<SharedConfiguration, TupleCodec<SharedConfiguration>, false> sharedConfigProperty;
 	SharedConfiguration sharedConfig;
 
-	AutomaticIdempotencyWorkload(WorkloadContext const& wcx)
+	explicit AutomaticIdempotencyWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), sharedConfigProperty("/testConfig"_sr) {
 		numTransactions = getOption(options, "numTransactions"_sr, 500);
 		keyPrefix = KeyRef(getOption(options, "keyPrefix"_sr, "/autoIdempotency/"_sr));
@@ -84,80 +84,86 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> _setup(AutomaticIdempotencyWorkload* self, Database cx) {
-		if (self->clientId == 0) {
-			wait(self->sharedConfigProperty.set(cx.getReference(), self->sharedConfig));
+	Future<Void> _setup(Database cx) {
+		if (clientId == 0) {
+			co_await sharedConfigProperty.set(cx.getReference(), sharedConfig);
 		} else {
-			loop {
-				Optional<SharedConfiguration> sharedConfig = wait(self->sharedConfigProperty.get(cx.getReference()));
+			while (true) {
+				Optional<SharedConfiguration> sharedConfig = co_await sharedConfigProperty.get(cx.getReference());
 				if (sharedConfig.present()) {
-					self->sharedConfig = sharedConfig.get();
+					this->sharedConfig = sharedConfig.get();
 					break;
 				}
-				wait(delay(1.0));
+				co_await delay(1.0);
 			}
 		}
 
-		if (self->sharedConfig.disableAutomaticIdempotency) {
-			self->automaticPercentage = 0;
+		if (sharedConfig.disableAutomaticIdempotency) {
+			automaticPercentage = 0;
 		}
-
-		return Void();
 	}
 
-	Future<Void> setup(Database const& cx) override { return _setup(this, cx); }
+	Future<Void> setup(Database const& cx) override { return _setup(cx); }
 
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
+	Future<Void> start(Database const& cx) override { return _start(cx); }
 
-	ACTOR static Future<Void> _start(AutomaticIdempotencyWorkload* self, Database cx) {
-		state int i = 0;
-		for (; i < self->numTransactions; ++i) {
+	Future<Void> _start(Database cx) {
+		for (int i = 0; i < numTransactions; ++i) {
 			// Half direct representation, half indirect representation
 			int length = deterministicRandom()->coinflip() ? 16 : deterministicRandom()->randomInt(17, 256);
-			state Value idempotencyId = makeString(length);
+			Value idempotencyId = makeString(length);
 			deterministicRandom()->randomBytes(mutateString(idempotencyId), length);
 			TraceEvent("IdempotencyIdWorkloadTransaction").detail("Id", idempotencyId);
-			wait(runRYWTransaction(
-			    cx, [self = self, idempotencyId = idempotencyId](Reference<ReadYourWritesTransaction> tr) {
+			bool automatic = deterministicRandom()->random01() < automaticPercentage;
+			Reference<ReadYourWritesTransaction> committedTr;
+			co_await runRYWTransaction(
+			    cx,
+			    [this, idempotencyId = idempotencyId, automatic, &committedTr](
+			        Reference<ReadYourWritesTransaction> tr) {
+				    committedTr = tr;
 				    // If we don't set AUTOMATIC_IDEMPOTENCY the idempotency id won't automatically get cleaned up, so
 				    // it should create work for the cleaner.
 				    tr->setOption(FDBTransactionOptions::IDEMPOTENCY_ID, idempotencyId);
-				    if (deterministicRandom()->random01() < self->automaticPercentage) {
+				    if (automatic) {
 					    // We also want to exercise the automatic idempotency code path.
 					    tr->setOption(FDBTransactionOptions::AUTOMATIC_IDEMPOTENCY);
 				    }
-				    uint32_t index = self->keyPrefix.size();
+				    uint32_t index = keyPrefix.size();
 				    Value suffix = makeString(14);
 				    memset(mutateString(suffix), 0, 10);
 				    memcpy(mutateString(suffix) + 10, &index, 4);
-				    tr->atomicOp(self->keyPrefix.withSuffix(suffix),
+				    tr->atomicOp(keyPrefix.withSuffix(suffix),
 				                 ObjectWriter::toValue(ValueType{ idempotencyId, int64_t(now()) }, Unversioned()),
 				                 MutationRef::SetVersionstampedKey);
 				    return Future<Void>(Void());
-			    }));
+			    });
+			if (committedTr->getCommittedVersion() == invalidVersion) {
+				TraceEvent(SevError, "IdempotencyCommitMissingVersion")
+				    .detail("Id", idempotencyId)
+				    .detail("AutomaticIdempotency", automatic);
+				ok = false;
+			}
 		}
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override {
 		if (clientId != 0) {
 			return true;
 		}
-		return testAll(this, cx);
+		return testAll(cx);
 	}
 
-	ACTOR static Future<bool> testAll(AutomaticIdempotencyWorkload* self, Database db) {
-		wait(runRYWTransaction(db,
-		                       [=](Reference<ReadYourWritesTransaction> tr) { return logIdempotencyIds(self, tr); }));
-		wait(runRYWTransaction(db, [=](Reference<ReadYourWritesTransaction> tr) { return testIdempotency(self, tr); }));
-		wait(testCleaner(self, db));
-		return self->ok;
+	Future<bool> testAll(Database db) {
+		co_await runRYWTransaction(db,
+		                           [this](Reference<ReadYourWritesTransaction> tr) { return logIdempotencyIds(tr); });
+		co_await runRYWTransaction(db, [this](Reference<ReadYourWritesTransaction> tr) { return testIdempotency(tr); });
+		co_await testCleaner(db);
+		co_return ok;
 	}
 
-	ACTOR static Future<Void> logIdempotencyIds(AutomaticIdempotencyWorkload* self,
-	                                            Reference<ReadYourWritesTransaction> tr) {
+	Future<Void> logIdempotencyIds(Reference<ReadYourWritesTransaction> tr) {
 		tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-		RangeResult result = wait(tr->getRange(idempotencyIdKeys, CLIENT_KNOBS->TOO_MANY));
+		RangeResult result = co_await tr->getRange(idempotencyIdKeys, CLIENT_KNOBS->TOO_MANY);
 		ASSERT(!result.more);
 		for (const auto& [k, v] : result) {
 			Version commitVersion;
@@ -178,13 +184,11 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 				    .detail("Id", id);
 			}
 		}
-		return Void();
 	}
 
 	// Check that each transaction committed exactly once.
-	ACTOR static Future<Void> testIdempotency(AutomaticIdempotencyWorkload* self,
-	                                          Reference<ReadYourWritesTransaction> tr) {
-		RangeResult result = wait(tr->getRange(prefixRange(self->keyPrefix), CLIENT_KNOBS->TOO_MANY));
+	Future<Void> testIdempotency(Reference<ReadYourWritesTransaction> tr) {
+		RangeResult result = co_await tr->getRange(prefixRange(keyPrefix), CLIENT_KNOBS->TOO_MANY);
 		ASSERT(!result.more);
 		std::unordered_set<Value> ids;
 		// Make sure they're all unique - ie no transaction committed twice
@@ -194,7 +198,7 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 		for (const auto& [k, rawValue] : result) {
 			auto v = ObjectReader::fromStringRef<ValueType>(rawValue, Unversioned());
 			BinaryReader reader(k, Unversioned());
-			reader.readBytes(self->keyPrefix.size());
+			reader.readBytes(keyPrefix.size());
 			Version commitVersion;
 			reader >> commitVersion;
 			commitVersion = bigEndian64(commitVersion);
@@ -207,11 +211,10 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 			    .detail("Id", v.idempotencyId)
 			    .detail("CreatedTime", v.createdTime);
 		}
-		if (ids.size() != self->clientCount * self->numTransactions) {
-			self->ok = false;
+		if (ids.size() != clientCount * numTransactions) {
+			ok = false;
 		}
-		ASSERT_EQ(ids.size(), self->clientCount * self->numTransactions);
-		return Void();
+		ASSERT_EQ(ids.size(), clientCount * numTransactions);
 	}
 
 	std::vector<Key> idempotencyKeyValueToTestKeys(KeyValueRef kv, Version* commitVersion, int64_t* timestamp) {
@@ -244,39 +247,38 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 	}
 
 	// Returns the largest gap between createdTime and the idempotency timestamp
-	ACTOR static Future<int64_t> getMaxTimestampDelta(AutomaticIdempotencyWorkload* self,
-	                                                  Database db,
-	                                                  int64_t numCreatedTimes) {
-		state std::vector<int64_t> timestamps;
-		state std::vector<Key> keys;
+	Future<int64_t> getMaxTimestampDelta(Database db, int64_t numCreatedTimes) {
+		std::vector<int64_t> timestamps;
+		std::vector<Key> keys;
 
-		RangeResult result = wait(runRYWTransaction(db, [](Reference<ReadYourWritesTransaction> tr) {
+		RangeResult result = co_await runRYWTransaction(db, [](Reference<ReadYourWritesTransaction> tr) {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			return tr->getRange(idempotencyIdKeys, CLIENT_KNOBS->TOO_MANY);
-		}));
+		});
 
 		ASSERT(!result.more);
 
 		for (const auto& kv : result) {
 			Version commitVersion;
 			int64_t timestamp;
-			std::vector<Key> decodedKeys = self->idempotencyKeyValueToTestKeys(kv, &commitVersion, &timestamp);
+			std::vector<Key> decodedKeys = idempotencyKeyValueToTestKeys(kv, &commitVersion, &timestamp);
 			for (auto const& key : decodedKeys) {
 				timestamps.push_back(timestamp);
 				keys.push_back(key);
 			}
 		}
 
-		state ReadYourWritesTransaction tr(db);
-		loop {
+		ReadYourWritesTransaction tr(db);
+		while (true) {
+			Error err;
 			try {
-				state std::vector<Future<Optional<Value>>> futures;
+				std::vector<Future<Optional<Value>>> futures;
 
 				for (auto const& key : keys) {
 					futures.push_back(tr.get(key));
 				}
 
-				wait(waitForAll(futures));
+				co_await waitForAll(futures);
 
 				int64_t maxCreatedTimeDelta = 0;
 				for (int i = 0; i < futures.size(); ++i) {
@@ -286,39 +288,40 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 					maxCreatedTimeDelta = std::max(timestamps[i] - e.createdTime, maxCreatedTimeDelta);
 				}
 
-				if (self->automaticPercentage == 0) {
+				if (automaticPercentage == 0) {
 					ASSERT_EQ(futures.size(), numCreatedTimes);
 				}
 
-				return maxCreatedTimeDelta;
+				co_return maxCreatedTimeDelta;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<int64_t> getOldestCreatedTime(AutomaticIdempotencyWorkload* self, Database db) {
-		state ReadYourWritesTransaction tr(db);
-		state Key key;
-		state Version commitVersion;
+	Future<int64_t> getOldestCreatedTime(Database db) {
+		ReadYourWritesTransaction tr(db);
+		Key key;
+		Version commitVersion{ 0 };
 
-		state RangeResult result = wait(runRYWTransaction(db, [](Reference<ReadYourWritesTransaction> tr) {
+		RangeResult result = co_await runRYWTransaction(db, [](Reference<ReadYourWritesTransaction> tr) {
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			return tr->getRange(idempotencyIdKeys, 1);
-		}));
+		});
 
 		if (result.empty()) {
 			TraceEvent("AutomaticIdempotencyNoIdsLeft").log();
-			return -1;
+			co_return -1;
 		}
 
 		int64_t timestamp;
-		key = self->idempotencyKeyValueToTestKeys(result[0], &commitVersion, &timestamp)[0];
+		key = idempotencyKeyValueToTestKeys(result[0], &commitVersion, &timestamp)[0];
 
 		// We need to use a different transaction because we set READ_SYSTEM_KEYS on this one, and we might
 		// be using a tenant.
-		Optional<Value> entry =
-		    wait(runRYWTransaction(db, [key = key](Reference<ReadYourWritesTransaction> tr) { return tr->get(key); }));
+		Optional<Value> entry = co_await runRYWTransaction(
+		    db, [key = key](Reference<ReadYourWritesTransaction> tr) { return tr->get(key); });
 
 		if (!entry.present()) {
 			TraceEvent(SevError, "AutomaticIdempotencyKeyMissing")
@@ -329,29 +332,28 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 		ASSERT(entry.present());
 
 		auto e = ObjectReader::fromStringRef<ValueType>(entry.get(), Unversioned());
-		return e.createdTime;
+		co_return e.createdTime;
 	}
 
-	ACTOR static Future<bool> testCleanerOneIteration(AutomaticIdempotencyWorkload* self,
-	                                                  Database db,
-	                                                  ActorCollection* actors,
-	                                                  int64_t minAgeSeconds,
-	                                                  int64_t maxTimestampDelta,
-	                                                  const std::vector<int64_t>* createdTimes) {
-		state Future<Void> cleaner = recurringAsync(
+	Future<bool> testCleanerOneIteration(Database db,
+	                                     ActorCollection* actors,
+	                                     int64_t minAgeSeconds,
+	                                     int64_t maxTimestampDelta,
+	                                     const std::vector<int64_t>* createdTimes) {
+		Future<Void> cleaner = recurringAsync(
 		    [db = db, minAgeSeconds = minAgeSeconds]() { return cleanIdempotencyIds(db, minAgeSeconds); },
-		    self->pollingInterval,
+		    pollingInterval,
 		    true,
-		    self->pollingInterval);
+		    pollingInterval);
 
-		state int64_t oldestCreatedTime;
-		state int64_t successes = 0;
+		int64_t oldestCreatedTime{ 0 };
+		int64_t successes = 0;
 		actors->add(cleaner);
-		loop {
+		while (true) {
 			// Oldest created time of a transaction from the workload which still has an idempotency id
-			wait(store(oldestCreatedTime, getOldestCreatedTime(self, db)));
+			oldestCreatedTime = co_await getOldestCreatedTime(db);
 			if (oldestCreatedTime == -1) {
-				return true; // Test can't make meaningful progress anymore
+				co_return true; // Test can't make meaningful progress anymore
 			}
 
 			// oldestCreatedTime could seem too high if there's a large gap in the age
@@ -369,15 +371,15 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 				oldestCreatedTime = *iter + 1;
 			}
 			auto maxActualAge = int64_t(now()) - oldestCreatedTime;
-			if (maxActualAge > minAgeSeconds * self->slop) {
+			if (maxActualAge > minAgeSeconds * slop) {
 				CODE_PROBE(true, "Idempotency cleaner more to clean");
 				TraceEvent("AutomaticIdempotencyCleanerMoreToClean")
 				    .detail("MaxActualAge", maxActualAge)
 				    .detail("MinAgePolicy", minAgeSeconds);
 				successes = 0;
 				// Cleaning should happen eventually
-			} else if (maxActualAge < minAgeSeconds / self->slop) {
-				bool ok = self->automaticPercentage == 0;
+			} else if (maxActualAge < minAgeSeconds / slop) {
+				bool ok = automaticPercentage == 0;
 				TraceEvent(ok ? SevInfo : SevError, "AutomaticIdempotencyCleanedTooMuch")
 				    .detail("MaxActualAge", maxActualAge)
 				    .detail("MinAgePolicy", minAgeSeconds)
@@ -385,7 +387,7 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 				    .detail("OldestCreatedTime", oldestCreatedTime)
 				    .detail("MaxTimestampDelta", maxTimestampDelta);
 				if (!ok) {
-					self->ok = false;
+					this->ok = false;
 				}
 				ASSERT(ok);
 				successes = 0;
@@ -399,15 +401,14 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 					break;
 				}
 			}
-			wait(delay(self->pollingInterval));
+			co_await delay(pollingInterval);
 		}
 		cleaner.cancel();
-		return false;
+		co_return false;
 	}
 
-	ACTOR static Future<std::vector<int64_t>> getCreatedTimes(AutomaticIdempotencyWorkload* self,
-	                                                          Reference<ReadYourWritesTransaction> tr) {
-		RangeResult result = wait(tr->getRange(prefixRange(self->keyPrefix), CLIENT_KNOBS->TOO_MANY));
+	Future<std::vector<int64_t>> getCreatedTimes(Reference<ReadYourWritesTransaction> tr) {
+		RangeResult result = co_await tr->getRange(prefixRange(keyPrefix), CLIENT_KNOBS->TOO_MANY);
 		ASSERT(!result.more);
 		std::vector<int64_t> createdTimes;
 		for (const auto& [k, v] : result) {
@@ -415,43 +416,45 @@ struct AutomaticIdempotencyWorkload : TestWorkload {
 			createdTimes.emplace_back(e.createdTime);
 		}
 		std::sort(createdTimes.begin(), createdTimes.end());
-		return createdTimes;
+		co_return createdTimes;
 	}
 
 	// Check that min age is respected. Also test that we can tolerate concurrent cleaners.
-	ACTOR static Future<Void> testCleaner(AutomaticIdempotencyWorkload* self, Database db) {
-		state ActorCollection actors;
-		state int64_t minAgeSeconds;
-		state std::vector<int64_t> createdTimes;
+	Future<Void> testCleaner(Database db) {
+		ActorCollection actors;
+		int64_t minAgeSeconds{ 0 };
+		std::vector<int64_t> createdTimes;
 
 		// Initialize minAgeSeconds to match the current status
-		wait(store(minAgeSeconds, fmap([](int64_t t) { return int64_t(now()) - t; }, getOldestCreatedTime(self, db))) &&
-		     store(createdTimes, runRYWTransaction(db, [self = self](Reference<ReadYourWritesTransaction> tr) {
-			           return getCreatedTimes(self, tr);
-		           })));
+		co_await (store(minAgeSeconds, fmap([](int64_t t) { return int64_t(now()) - t; }, getOldestCreatedTime(db))) &&
+		          store(createdTimes, runRYWTransaction(db, [this](Reference<ReadYourWritesTransaction> tr) {
+			                return getCreatedTimes(tr);
+		                })));
 
-		state int64_t maxTimestampDelta = wait(getMaxTimestampDelta(self, db, createdTimes.size()));
+		int64_t maxTimestampDelta = co_await getMaxTimestampDelta(db, createdTimes.size());
 
 		// Slowly and somewhat randomly allow the cleaner to do more cleaning. Observe that it cleans some, but not too
 		// much.
-		loop {
-			minAgeSeconds *= 1 / (self->slop * 2);
-			if (minAgeSeconds < self->minMinAgeSeconds) {
+		while (true) {
+			minAgeSeconds *= 1 / (slop * 2);
+			if (minAgeSeconds < minMinAgeSeconds) {
 				break;
 			}
-			choose {
-				when(bool done = wait(
-				         testCleanerOneIteration(self, db, &actors, minAgeSeconds, maxTimestampDelta, &createdTimes))) {
-					if (done) {
-						break;
-					}
-				}
-				when(wait(actors.getResult())) {
-					ASSERT(false);
+			auto choice =
+			    co_await race(testCleanerOneIteration(db, &actors, minAgeSeconds, maxTimestampDelta, &createdTimes),
+			                  actors.getResult());
+			if (choice.index() == 0) {
+				bool done = std::get<0>(std::move(choice));
+
+				if (done) {
+					break;
 				}
+			} else if (choice.index() == 1) {
+				ASSERT(false);
+			} else {
+				UNREACHABLE();
 			}
 		}
-		return Void();
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
diff --git a/fdbserver/workloads/BackgroundSelectors.actor.cpp b/fdbserver/workloads/BackgroundSelectors.cpp
similarity index 68%
rename from fdbserver/workloads/BackgroundSelectors.actor.cpp
rename to fdbserver/workloads/BackgroundSelectors.cpp
index 1e4a0204e97..22ccf88b952 100644
--- a/fdbserver/workloads/BackgroundSelectors.actor.cpp
+++ b/fdbserver/workloads/BackgroundSelectors.cpp
@@ -1,5 +1,5 @@
 /*
- * BackgroundSelectors.actor.cpp
+ * BackgroundSelectors.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,9 +19,8 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 KeySelector randomizedSelector(const KeyRef& key, bool orEqual, int offset) {
 	if (orEqual && deterministicRandom()->random01() > 0.5)
@@ -37,7 +36,7 @@ struct BackgroundSelectorWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter operations, checks, retries;
 
-	BackgroundSelectorWorkload(WorkloadContext const& wcx)
+	explicit BackgroundSelectorWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), operations("Operations"), checks("Checks"), retries("Retries") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		actorsPerClient = std::max(getOption(options, "actorsPerClient"_sr, 1), 1);
@@ -50,13 +49,10 @@ struct BackgroundSelectorWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
-	ACTOR Future<Void> _start(Database cx, BackgroundSelectorWorkload* self) {
-		for (int c = 0; c < self->actorsPerClient; c++)
-			self->clients.push_back(timeout(self->backgroundSelectorWorker(cx, self), self->testDuration, Void()));
-		wait(waitForAll(self->clients));
-		return Void();
+	Future<Void> start(Database const& cx) override {
+		for (int c = 0; c < actorsPerClient; c++)
+			clients.push_back(timeout(backgroundSelectorWorker(cx, this), testDuration, Void()));
+		co_await waitForAll(clients);
 	}
 
 	Future<bool> check(Database const& cx) override {
@@ -76,58 +72,60 @@ struct BackgroundSelectorWorkload : TestWorkload {
 		m.push_back(retries.getMetric());
 	}
 
-	ACTOR Future<Void> backgroundSelectorWorker(Database cx, BackgroundSelectorWorkload* self) {
-		state double lastTime = now();
-		state bool forward;
-		state int direction;
-		state Key startKey;
-		state Key endKey;
-		state int diff;
-		state Transaction tr(cx);
-		state RangeResult rangeResult;
-		state Standalone<StringRef> startResult;
-		state Standalone<StringRef> endResult;
-		state int startDrift;
-		state int endDrift;
-		state bool restartProcess;
-
-		loop {
+	Future<Void> backgroundSelectorWorker(Database cx, BackgroundSelectorWorkload* self) {
+		double lastTime = now();
+		bool forward{ false };
+		int direction{ 0 };
+		Key startKey;
+		Key endKey;
+		int diff{ 0 };
+		Transaction tr(cx);
+		RangeResult rangeResult;
+		Standalone<StringRef> startResult;
+		Standalone<StringRef> endResult;
+		int startDrift{ 0 };
+		int endDrift{ 0 };
+		bool restartProcess{ false };
+
+		while (true) {
 			forward = deterministicRandom()->randomInt(0, 2) != 0;
 			direction = forward ? 1 : -1;
 			diff = deterministicRandom()->randomInt(0, self->maxDiff);
 
 			// Setup start and end key
-			loop {
+			while (true) {
+				Error err;
 				try {
 					if (forward) {
 						{
-							Standalone<StringRef> res = wait(tr.getKey(KeySelectorRef(allKeys.begin, false, 1)));
+							Standalone<StringRef> res = co_await tr.getKey(KeySelectorRef(allKeys.begin, false, 1));
 							startKey = res;
 						}
 
 						{
-							Standalone<StringRef> res = wait(tr.getKey(randomizedSelector(startKey, true, diff)));
+							Standalone<StringRef> res = co_await tr.getKey(randomizedSelector(startKey, true, diff));
 							endKey = res;
 						}
 					} else {
 						{
-							Standalone<StringRef> res = wait(tr.getKey(KeySelectorRef(allKeys.end, false, 0)));
+							Standalone<StringRef> res = co_await tr.getKey(KeySelectorRef(allKeys.end, false, 0));
 							endKey = res;
 						}
 
 						{
-							Standalone<StringRef> res = wait(tr.getKey(randomizedSelector(endKey, true, -1 * diff)));
+							Standalone<StringRef> res = co_await tr.getKey(randomizedSelector(endKey, true, -1 * diff));
 							startKey = res;
 						}
 					}
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 
-			loop {
-				wait(poisson(&lastTime, 1.0 / self->transactionsPerSecond));
+			while (true) {
+				co_await poisson(&lastTime, 1.0 / self->transactionsPerSecond);
 				tr.reset();
 				startDrift = direction * deterministicRandom()->randomInt(self->minDrift, self->maxDrift);
 				endDrift = direction * deterministicRandom()->randomInt(self->minDrift, self->maxDrift);
@@ -142,35 +140,37 @@ struct BackgroundSelectorWorkload : TestWorkload {
 					diff++;
 				}
 
-				loop {
+				while (true) {
+					Error err;
 					try {
 						if (diff < 0) {
 							RangeResult rangeResult_ =
-							    wait(tr.getRange(randomizedSelector(endKey, true, endDrift),
-							                     randomizedSelector(startKey, true, startDrift + 1),
-							                     self->resultLimit));
+							    co_await tr.getRange(randomizedSelector(endKey, true, endDrift),
+							                         randomizedSelector(startKey, true, startDrift + 1),
+							                         self->resultLimit);
 							rangeResult = rangeResult_;
 							Standalone<StringRef> endResult_ =
-							    wait(tr.getKey(randomizedSelector(startKey, true, startDrift)));
+							    co_await tr.getKey(randomizedSelector(startKey, true, startDrift));
 							endResult = endResult_;
 							Standalone<StringRef> startResult_ =
-							    wait(tr.getKey(randomizedSelector(endKey, true, endDrift)));
+							    co_await tr.getKey(randomizedSelector(endKey, true, endDrift));
 							startResult = startResult_;
 						} else {
-							RangeResult rangeResult_ = wait(tr.getRange(randomizedSelector(startKey, true, startDrift),
-							                                            randomizedSelector(endKey, true, endDrift + 1),
-							                                            self->resultLimit));
+							RangeResult rangeResult_ =
+							    co_await tr.getRange(randomizedSelector(startKey, true, startDrift),
+							                         randomizedSelector(endKey, true, endDrift + 1),
+							                         self->resultLimit);
 							rangeResult = rangeResult_;
 							Standalone<StringRef> startResult_ =
-							    wait(tr.getKey(randomizedSelector(startKey, true, startDrift)));
+							    co_await tr.getKey(randomizedSelector(startKey, true, startDrift));
 							startResult = startResult_;
 							Standalone<StringRef> endResult_ =
-							    wait(tr.getKey(randomizedSelector(endKey, true, endDrift)));
+							    co_await tr.getKey(randomizedSelector(endKey, true, endDrift));
 							endResult = endResult_;
 						}
 
 						restartProcess = false;
-						if (rangeResult.size() == 0) {
+						if (rangeResult.empty()) {
 							restartProcess = true;
 							break;
 						}
@@ -203,9 +203,10 @@ struct BackgroundSelectorWorkload : TestWorkload {
 
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
-						++self->retries;
+						err = e;
 					}
+					co_await tr.onError(err);
+					++self->retries;
 				}
 				++self->operations;
 				if (restartProcess)
diff --git a/fdbserver/workloads/Backup.actor.cpp b/fdbserver/workloads/Backup.cpp
similarity index 66%
rename from fdbserver/workloads/Backup.actor.cpp
rename to fdbserver/workloads/Backup.cpp
index 0b84accbd65..cf1738a5ea9 100644
--- a/fdbserver/workloads/Backup.actor.cpp
+++ b/fdbserver/workloads/Backup.cpp
@@ -1,5 +1,5 @@
 /*
- * Backup.actor.cpp
+ * Backup.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,13 @@
 
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // A workload which only performs backup operations. A separate workload is used to perform restore operations.
 struct BackupWorkload : TestWorkload {
@@ -38,14 +38,14 @@ struct BackupWorkload : TestWorkload {
 	bool differentialBackup;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
 	LockDB locked{ false };
-	UsePartitionedLog usePartitionedLog{ true };
+	MutationLogType mutationLogType{ MutationLogType::PARTITIONED_LOG };
 	bool allowPauses;
 	Optional<std::string> encryptionKeyFileName;
 
-	BackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit BackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
-		bool partitioned = getOption(options, "usePartitionedLog"_sr, true);
-		usePartitionedLog.set(partitioned);
+		mutationLogType = static_cast<MutationLogType>(
+		    getOption(options, "mutationLogType"_sr, static_cast<int>(MutationLogType::PARTITIONED_LOG)));
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
 		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
 		if (backupAfter > minBackupAfter) {
@@ -97,57 +97,55 @@ struct BackupWorkload : TestWorkload {
 		    .detail("StopDifferentialAfter", stopDifferentialAfter)
 		    .detail("Encrypted", encryptionKeyFileName.present());
 
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
-		loop {
-			wait(backupAgent->changePause(cx, true));
+	static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
+		while (true) {
+			co_await backupAgent->changePause(cx, true);
 			TraceEvent("BW_AgentPaused").log();
-			wait(delay(30 * deterministicRandom()->random01()));
-			wait(backupAgent->changePause(cx, false));
+			co_await delay(30 * deterministicRandom()->random01());
+			co_await backupAgent->changePause(cx, false);
 			TraceEvent("BW_AgentResumed").log();
-			wait(delay(120 * deterministicRandom()->random01()));
+			co_await delay(120 * deterministicRandom()->random01());
 		}
 	}
 
 	// Resume the backup agent if it is paused
-	ACTOR static Future<Void> resumeAgent(Database cx, FileBackupAgent* backupAgent) {
+	static Future<Void> resumeAgent(Database cx, FileBackupAgent* backupAgent) {
 		TraceEvent("BW_AgentResuming").log();
-		wait(backupAgent->changePause(cx, false));
+		co_await backupAgent->changePause(cx, false);
 		TraceEvent("BW_AgentResumed").log();
-		return Void();
 	}
 
-	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
-		state FileBackupAgent agent;
-		loop {
-			bool active = wait(agent.checkActive(cx));
+	static Future<Void> statusLoop(Database cx, std::string tag) {
+		FileBackupAgent agent;
+		while (true) {
+			bool active = co_await agent.checkActive(cx);
 			TraceEvent("BW_AgentActivityCheck").detail("IsActive", active);
-			std::string status = wait(agent.getStatus(cx, ShowErrors::True, tag));
+			std::string status = co_await agent.getStatus(cx, ShowErrors::True, tag);
 			puts(status.c_str());
-			std::string statusJSON = wait(agent.getStatusJSON(cx, tag));
+			std::string statusJSON = co_await agent.getStatusJSON(cx, tag);
 			puts(statusJSON.c_str());
-			wait(delay(2.0));
+			co_await delay(2.0);
 		}
 	}
 
-	ACTOR static Future<Void> doBackup(BackupWorkload* self,
-	                                   double startDelay,
-	                                   FileBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   double stopDifferentialDelay) {
+	Future<Void> doBackup(double startDelay,
+	                      FileBackupAgent* backupAgent,
+	                      Database cx,
+	                      Key tag,
+	                      Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                      double stopDifferentialDelay) {
 
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 
-		state Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
-		wait(delay(startDelay));
+		Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
+		co_await delay(startDelay);
 
 		if (startDelay || BUGGIFY) {
 			TraceEvent("BW_DoBackupAbortBackup1", randomID)
@@ -155,7 +153,7 @@ struct BackupWorkload : TestWorkload {
 			    .detail("StartDelay", startDelay);
 
 			try {
-				wait(backupAgent->abortBackup(cx, tag.toString()));
+				co_await backupAgent->abortBackup(cx, tag.toString());
 			} catch (Error& e) {
 				TraceEvent("BW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
 				if (e.code() != error_code_backup_unneeded)
@@ -167,20 +165,22 @@ struct BackupWorkload : TestWorkload {
 		    .detail("Tag", printable(tag))
 		    .detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
 
-		state std::string backupContainer = "file://simfdb/backups/";
-		state Future<Void> status = statusLoop(cx, tag.toString());
+		std::string backupContainer = "file://simfdb/backups/";
+		Future<Void> status = statusLoop(cx, tag.toString());
 		try {
-			wait(backupAgent->submitBackup(cx,
-			                               StringRef(backupContainer),
-			                               {},
-			                               deterministicRandom()->randomInt(0, 60),
-			                               deterministicRandom()->randomInt(0, 2000),
-			                               tag.toString(),
-			                               backupRanges,
-			                               StopWhenDone{ !stopDifferentialDelay },
-			                               self->usePartitionedLog,
-			                               IncrementalBackupOnly::False,
-			                               self->encryptionKeyFileName));
+			co_await backupAgent->submitBackup(cx,
+			                                   StringRef(backupContainer),
+			                                   {},
+			                                   deterministicRandom()->randomInt(0, 60),
+			                                   deterministicRandom()->randomInt(0, 2000),
+			                                   tag.toString(),
+			                                   backupRanges,
+			                                   StopWhenDone{ !stopDifferentialDelay },
+			                                   mutationLogType,
+			                                   IncrementalBackupOnly::False,
+			                                   encryptionKeyFileName,
+			                                   encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+			                                   /*snapshotMode=*/0);
 		} catch (Error& e) {
 			TraceEvent("BW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@@ -191,34 +191,34 @@ struct BackupWorkload : TestWorkload {
 		if (stopDifferentialDelay) {
 			CODE_PROBE(!stopDifferentialFuture.isReady(),
 			           "Restore starts at specified time - stopDifferential not ready");
-			wait(stopDifferentialFuture);
+			co_await stopDifferentialFuture;
 			TraceEvent("BW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
 			    .detail("DifferentialAfter", stopDifferentialDelay);
 
 			try {
 				if (BUGGIFY) {
-					state KeyBackedTag backupTag = makeBackupTag(tag.toString());
+					KeyBackedTag backupTag = makeBackupTag(tag.toString());
 					TraceEvent("BW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName);
 
 					// Wait until the backup is in a restorable state and get the status, URL, and UID atomically
-					state Reference<IBackupContainer> lastBackupContainer;
-					state UID lastBackupUID;
-					state EBackupState resultWait = wait(backupAgent->waitBackup(
-					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID));
+					Reference<IBackupContainer> lastBackupContainer;
+					UID lastBackupUID;
+					EBackupState resultWait = co_await backupAgent->waitBackup(
+					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID);
 
 					TraceEvent("BW_DoBackupWaitForRestorable", randomID)
 					    .detail("Tag", backupTag.tagName)
 					    .detail("Result", BackupAgentBase::getStateText(resultWait));
 
-					state bool restorable = false;
+					bool restorable = false;
 					if (lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
-						wait(ready(fdesc));
+						Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
+						co_await ready(fdesc);
 
 						if (!fdesc.isError()) {
-							state BackupDescription desc = fdesc.get();
-							wait(desc.resolveVersionTimes(cx));
+							BackupDescription desc = fdesc.get();
+							co_await desc.resolveVersionTimes(cx);
 							printf("BackupDescription:\n%s\n", desc.toString().c_str());
 							restorable = desc.maxRestorableVersion.present();
 						}
@@ -262,12 +262,12 @@ struct BackupWorkload : TestWorkload {
 						    .detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
 						    .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
 						    .detail("Restorable", restorable);
-						wait(backupAgent->abortBackup(cx, tag.toString()));
+						co_await backupAgent->abortBackup(cx, tag.toString());
 					} else {
 						TraceEvent("BW_DoBackupDiscontinueBackup", randomID)
 						    .detail("Tag", printable(tag))
 						    .detail("DifferentialAfter", stopDifferentialDelay);
-						wait(backupAgent->discontinueBackup(cx, tag));
+						co_await backupAgent->discontinueBackup(cx, tag);
 					}
 				}
 
@@ -275,7 +275,7 @@ struct BackupWorkload : TestWorkload {
 					TraceEvent("BW_DoBackupDiscontinueBackup", randomID)
 					    .detail("Tag", printable(tag))
 					    .detail("DifferentialAfter", stopDifferentialDelay);
-					wait(backupAgent->discontinueBackup(cx, tag));
+					co_await backupAgent->discontinueBackup(cx, tag);
 				}
 			} catch (Error& e) {
 				TraceEvent("BW_DoBackupDiscontinueBackupException", randomID).error(e).detail("Tag", printable(tag));
@@ -286,73 +286,69 @@ struct BackupWorkload : TestWorkload {
 
 		// Wait for the backup to complete
 		TraceEvent("BW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
-		state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True));
+		EBackupState statusValue = co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True);
 
-		std::string statusText = wait(backupAgent->getStatus(cx, ShowErrors::True, tag.toString()));
+		std::string statusText = co_await backupAgent->getStatus(cx, ShowErrors::True, tag.toString());
 		// Can we validate anything about status?
 
 		TraceEvent("BW_DoBackupComplete", randomID)
 		    .detail("Tag", printable(tag))
 		    .detail("Status", statusText)
 		    .detail("StatusValue", BackupAgentBase::getStateText(statusValue));
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, BackupWorkload* self) {
-		state FileBackupAgent backupAgent;
-		state Future<Void> cp;
+	Future<Void> _start(Database cx) {
+		FileBackupAgent backupAgent;
+		Future<Void> cp;
 		TraceEvent("BW_Arguments")
-		    .detail("BackupTag", printable(self->backupTag))
-		    .detail("BackupAfter", self->backupAfter)
-		    .detail("RestoreAfter", self->restoreAfter)
-		    .detail("AbortAndRestartAfter", self->abortAndRestartAfter)
-		    .detail("DifferentialAfter", self->stopDifferentialAfter);
-
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		if (self->allowPauses && BUGGIFY) {
+		    .detail("BackupTag", printable(backupTag))
+		    .detail("BackupAfter", backupAfter)
+		    .detail("RestoreAfter", restoreAfter)
+		    .detail("AbortAndRestartAfter", abortAndRestartAfter)
+		    .detail("DifferentialAfter", stopDifferentialAfter);
+
+		UID randomID = nondeterministicRandom()->randomUniqueID();
+		if (allowPauses && BUGGIFY) {
 			cp = changePaused(cx, &backupAgent);
 		} else {
 			cp = resumeAgent(cx, &backupAgent);
 		}
 
-		if (self->encryptionKeyFileName.present()) {
-			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		if (encryptionKeyFileName.present()) {
+			co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 		}
 
 		try {
-			state Future<Void> startRestore = delay(self->restoreAfter);
+			Future<Void> startRestore = delay(restoreAfter);
 
 			// backup
-			wait(delay(self->backupAfter));
+			co_await delay(backupAfter);
 
-			TraceEvent("BW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
-			state Future<Void> b =
-			    doBackup(self, 0, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter);
+			TraceEvent("BW_DoBackup1", randomID).detail("Tag", printable(backupTag));
+			Future<Void> b = doBackup(0, &backupAgent, cx, backupTag, backupRanges, stopDifferentialAfter);
 
 			TraceEvent("BW_DoBackupWait", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 			try {
-				wait(b);
+				co_await b;
 			} catch (Error& e) {
 				if (e.code() != error_code_database_locked)
 					throw;
-				return Void();
+				co_return;
 			}
 			TraceEvent("BW_DoBackupDone", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 
-			wait(startRestore);
+			co_await startRestore;
 
 			// We can't remove after backup agents since the restore also needs them.
-			// I.e., g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents
+			// I.e., fdbSimulationPolicyState().backupAgents = FDBBackupAgentType::NoBackupAgents
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupCorrectness").error(e).GetLastError();
 			throw;
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
deleted file mode 100644
index f6d269e3a11..00000000000
--- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp
+++ /dev/null
@@ -1,807 +0,0 @@
-/*
- * BackupAndParallelRestoreCorrectness.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbserver/RestoreCommon.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-#define TEST_ABORT_FASTRESTORE 0
-
-struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
-	static constexpr auto NAME = "BackupAndParallelRestoreCorrectness";
-	double backupAfter, restoreAfter, abortAndRestartAfter;
-	double backupStartAt, restoreStartAfterBackupFinished, stopDifferentialAfter;
-	Key backupTag;
-	int backupRangesCount, backupRangeLengthMax;
-	bool differentialBackup, performRestore, agentRequest;
-	Standalone<VectorRef<KeyRangeRef>> backupRanges;
-	static int backupAgentRequests;
-	LockDB locked{ false };
-	bool allowPauses;
-	bool shareLogRange;
-	UsePartitionedLog usePartitionedLogs{ false };
-	Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix
-	// CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix
-	Optional<std::string> encryptionKeyFileName;
-
-	std::map<Standalone<KeyRef>, Standalone<ValueRef>> dbKVs;
-
-	// This workload is not compatible with RandomRangeLock workload because they will race in locked range
-	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
-		out.insert({ "RandomRangeLock" });
-	}
-
-	BackupAndParallelRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		locked.set(sharedRandomNumber % 2);
-		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
-		restoreAfter = getOption(options, "restoreAfter"_sr, 35.0);
-		performRestore = getOption(options, "performRestore"_sr, true);
-		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
-		backupRangesCount = getOption(options, "backupRangesCount"_sr, 5);
-		backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
-		abortAndRestartAfter =
-		    getOption(options,
-		              "abortAndRestartAfter"_sr,
-		              deterministicRandom()->random01() < 0.5
-		                  ? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter
-		                  : 0.0);
-		differentialBackup =
-		    getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false);
-		stopDifferentialAfter =
-		    getOption(options,
-		              "stopDifferentialAfter"_sr,
-		              differentialBackup ? deterministicRandom()->random01() *
-		                                           (restoreAfter - std::max(abortAndRestartAfter, backupAfter)) +
-		                                       std::max(abortAndRestartAfter, backupAfter)
-		                                 : 0.0);
-		agentRequest = getOption(options, "simBackupAgents"_sr, true);
-		allowPauses = getOption(options, "allowPauses"_sr, true);
-		shareLogRange = getOption(options, "shareLogRange"_sr, false);
-		usePartitionedLogs.set(getOption(options, "usePartitionedLogs"_sr, deterministicRandom()->coinflip()));
-		addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
-		removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
-
-		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.5)) {
-			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
-		}
-
-		KeyRef beginRange;
-		KeyRef endRange;
-		UID randomID = nondeterministicRandom()->randomUniqueID();
-
-		// Correctness is not clean for addPrefix feature yet. Uncomment below to enable the test
-		// Generate addPrefix
-		// if (addPrefix.size() == 0 && removePrefix.size() == 0) {
-		// 	if (deterministicRandom()->random01() < 0.5) { // Generate random addPrefix
-		// 		int len = deterministicRandom()->randomInt(1, 100);
-		// 		std::string randomStr = deterministicRandom()->randomAlphaNumeric(len);
-		// 		TraceEvent("BackupAndParallelRestoreCorrectness")
-		// 		    .detail("GenerateAddPrefix", randomStr)
-		// 		    .detail("Length", len)
-		// 		    .detail("StrLen", randomStr.size());
-		// 		addPrefix = Key(randomStr);
-		// 	}
-		// }
-		TraceEvent("BackupAndParallelRestoreCorrectness")
-		    .detail("AddPrefix", addPrefix)
-		    .detail("RemovePrefix", removePrefix);
-		ASSERT(addPrefix.size() == 0 && removePrefix.size() == 0);
-		// Do not support removePrefix right now because we must ensure all backup keys have the removePrefix
-		// otherwise, test will fail because fast restore will simply add the removePrefix to every key in the end.
-		ASSERT(removePrefix.size() == 0);
-
-		if (shareLogRange) {
-			bool beforePrefix = sharedRandomNumber & 1;
-			if (beforePrefix)
-				backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr));
-			else
-				backupRanges.push_back_deep(backupRanges.arena(),
-				                            KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end));
-		} else if (backupRangesCount <= 0) {
-			backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-		} else {
-			// Add backup ranges
-			std::set<std::string> rangeEndpoints;
-			while (rangeEndpoints.size() < backupRangesCount * 2) {
-				rangeEndpoints.insert(deterministicRandom()->randomAlphaNumeric(
-				    deterministicRandom()->randomInt(1, backupRangeLengthMax + 1)));
-			}
-
-			// Create ranges from the keys, in order, to prevent overlaps
-			std::vector<std::string> sortedEndpoints(rangeEndpoints.begin(), rangeEndpoints.end());
-			sort(sortedEndpoints.begin(), sortedEndpoints.end());
-			for (auto i = sortedEndpoints.begin(); i != sortedEndpoints.end(); ++i) {
-				const std::string& start = *i++;
-				backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(start, *i));
-
-				// Track the added range
-				TraceEvent("BARW_BackupCorrectnessRange", randomID)
-				    .detail("RangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange))
-				    .detail("RangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange));
-			}
-		}
-	}
-
-	Future<Void> setup(Database const& cx) override { return Void(); }
-
-	Future<Void> start(Database const& cx) override {
-		if (clientId != 0)
-			return Void();
-
-		TraceEvent(SevInfo, "BARW_Param").detail("Locked", locked);
-		TraceEvent(SevInfo, "BARW_Param").detail("BackupAfter", backupAfter);
-		TraceEvent(SevInfo, "BARW_Param").detail("RestoreAfter", restoreAfter);
-		TraceEvent(SevInfo, "BARW_Param").detail("PerformRestore", performRestore);
-		TraceEvent(SevInfo, "BARW_Param").detail("BackupTag", printable(backupTag).c_str());
-		TraceEvent(SevInfo, "BARW_Param").detail("BackupRangesCount", backupRangesCount);
-		TraceEvent(SevInfo, "BARW_Param").detail("BackupRangeLengthMax", backupRangeLengthMax);
-		TraceEvent(SevInfo, "BARW_Param").detail("AbortAndRestartAfter", abortAndRestartAfter);
-		TraceEvent(SevInfo, "BARW_Param").detail("DifferentialBackup", differentialBackup);
-		TraceEvent(SevInfo, "BARW_Param").detail("StopDifferentialAfter", stopDifferentialAfter);
-		TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest);
-		TraceEvent(SevInfo, "BARW_Param").detail("Encrypted", encryptionKeyFileName.present());
-
-		return _start(cx, this);
-	}
-
-	bool hasPrefix() const { return addPrefix != ""_sr || removePrefix != ""_sr; }
-
-	Future<bool> check(Database const& cx) override { return true; }
-
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-
-	ACTOR static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
-		loop {
-			wait(backupAgent->changePause(cx, true));
-			wait(delay(30 * deterministicRandom()->random01()));
-			wait(backupAgent->changePause(cx, false));
-			wait(delay(120 * deterministicRandom()->random01()));
-		}
-	}
-
-	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
-		state FileBackupAgent agent;
-		loop {
-			std::string status = wait(agent.getStatus(cx, ShowErrors::True, tag));
-			puts(status.c_str());
-			wait(delay(2.0));
-		}
-	}
-
-	ACTOR static Future<Void> doBackup(BackupAndParallelRestoreCorrectnessWorkload* self,
-	                                   double startDelay,
-	                                   FileBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   double stopDifferentialDelay,
-	                                   Promise<Void> submitted) {
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		state Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
-
-		wait(delay(startDelay));
-
-		if (startDelay || BUGGIFY) {
-			TraceEvent("BARW_DoBackupAbortBackup1", randomID)
-			    .detail("Tag", printable(tag))
-			    .detail("StartDelay", startDelay);
-
-			try {
-				wait(backupAgent->abortBackup(cx, tag.toString()));
-			} catch (Error& e) {
-				TraceEvent("BARW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
-				if (e.code() != error_code_backup_unneeded)
-					throw;
-			}
-		}
-
-		TraceEvent("BARW_DoBackupSubmitBackup", randomID)
-		    .detail("Tag", printable(tag))
-		    .detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
-
-		state std::string backupContainer = "file://simfdb/backups/";
-		state Future<Void> status = statusLoop(cx, tag.toString());
-		try {
-			wait(backupAgent->submitBackup(cx,
-			                               StringRef(backupContainer),
-			                               {},
-			                               deterministicRandom()->randomInt(0, 60),
-			                               deterministicRandom()->randomInt(0, 100),
-			                               tag.toString(),
-			                               backupRanges,
-			                               StopWhenDone{ !stopDifferentialDelay },
-			                               self->usePartitionedLogs,
-			                               IncrementalBackupOnly::False,
-			                               self->encryptionKeyFileName));
-		} catch (Error& e) {
-			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
-			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-				throw;
-		}
-
-		submitted.send(Void());
-
-		// Stop the differential backup, if enabled
-		if (stopDifferentialDelay) {
-			CODE_PROBE(!stopDifferentialFuture.isReady(),
-			           "Restore starts at specified time - stopDifferential not ready");
-			wait(stopDifferentialFuture);
-			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
-			    .detail("Tag", printable(tag))
-			    .detail("DifferentialAfter", stopDifferentialDelay);
-
-			try {
-				if (BUGGIFY) {
-					state KeyBackedTag backupTag = makeBackupTag(tag.toString());
-					TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName);
-					// Wait until the backup is in a restorable state and get the status, URL, and UID atomically
-					state Reference<IBackupContainer> lastBackupContainer;
-					state UID lastBackupUID;
-					state EBackupState resultWait = wait(backupAgent->waitBackup(
-					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID));
-
-					TraceEvent("BARW_DoBackupWaitForRestorable", randomID)
-					    .detail("Tag", backupTag.tagName)
-					    .detail("Result", BackupAgentBase::getStateText(resultWait));
-
-					state bool restorable = false;
-					if (lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
-						wait(ready(fdesc));
-
-						if (!fdesc.isError()) {
-							state BackupDescription desc = fdesc.get();
-							wait(desc.resolveVersionTimes(cx));
-							printf("BackupDescription:\n%s\n", desc.toString().c_str());
-							restorable = desc.maxRestorableVersion.present();
-						}
-					}
-
-					TraceEvent("BARW_LastBackupContainer", randomID)
-					    .detail("BackupTag", printable(tag))
-					    .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
-					    .detail("LastBackupUID", lastBackupUID)
-					    .detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
-					    .detail("Restorable", restorable);
-
-					// Do not check the backup, if aborted
-					if (resultWait == EBackupState::STATE_ABORTED) {
-					}
-					// Ensure that a backup container was found
-					else if (!lastBackupContainer) {
-						TraceEvent(SevError, "BARW_MissingBackupContainer", randomID)
-						    .detail("LastBackupUID", lastBackupUID)
-						    .detail("BackupTag", printable(tag))
-						    .detail("WaitStatus", resultWait);
-						printf("BackupCorrectnessMissingBackupContainer   tag: %s  status: %s\n",
-						       printable(tag).c_str(),
-						       BackupAgentBase::getStateText(resultWait));
-					}
-					// Check that backup is restorable
-					else if (!restorable) {
-						TraceEvent(SevError, "BARW_NotRestorable", randomID)
-						    .detail("LastBackupUID", lastBackupUID)
-						    .detail("BackupTag", printable(tag))
-						    .detail("BackupFolder", lastBackupContainer->getURL())
-						    .detail("WaitStatus", BackupAgentBase::getStateText(resultWait));
-						printf("BackupCorrectnessNotRestorable:  tag: %s\n", printable(tag).c_str());
-					}
-
-					// Abort the backup, if not the first backup because the second backup may have aborted the backup
-					// by now
-					if (startDelay) {
-						TraceEvent("BARW_DoBackupAbortBackup2", randomID)
-						    .detail("Tag", printable(tag))
-						    .detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
-						    .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
-						    .detail("Restorable", restorable);
-						wait(backupAgent->abortBackup(cx, tag.toString()));
-					} else {
-						TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
-						    .detail("Tag", printable(tag))
-						    .detail("DifferentialAfter", stopDifferentialDelay);
-						wait(backupAgent->discontinueBackup(cx, tag));
-					}
-				}
-
-				else {
-					TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
-					    .detail("Tag", printable(tag))
-					    .detail("DifferentialAfter", stopDifferentialDelay);
-					wait(backupAgent->discontinueBackup(cx, tag));
-				}
-			} catch (Error& e) {
-				TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).error(e).detail("Tag", printable(tag));
-				if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-					throw;
-			}
-		}
-
-		// Wait for the backup to complete
-		TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
-		state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True));
-
-		state std::string statusText;
-
-		std::string _statusText = wait(backupAgent->getStatus(cx, ShowErrors::True, tag.toString()));
-		statusText = _statusText;
-		// Can we validate anything about status?
-
-		TraceEvent("BARW_DoBackupComplete", randomID)
-		    .detail("Tag", printable(tag))
-		    .detail("Status", statusText)
-		    .detail("StatusValue", BackupAgentBase::getStateText(statusValue));
-
-		return Void();
-	}
-
-	// This actor attempts to restore the database without clearing the keyspace.
-	// TODO: Enable this function in correctness test
-	ACTOR static Future<Void> attemptDirtyRestore(BackupAndParallelRestoreCorrectnessWorkload* self,
-	                                              Database cx,
-	                                              FileBackupAgent* backupAgent,
-	                                              Standalone<StringRef> lastBackupContainer,
-	                                              UID randomID) {
-		state Transaction tr(cx);
-		state int rowCount = 0;
-		loop {
-			try {
-				RangeResult existingRows = wait(tr.getRange(normalKeys, 1));
-				rowCount = existingRows.size();
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-
-		// Try doing a restore without clearing the keys
-		if (rowCount > 0) {
-			try {
-				// TODO: Change to my restore agent code
-				TraceEvent(SevError, "MXFastRestore").detail("RestoreFunction", "ShouldChangeToMyOwnRestoreLogic");
-				wait(success(backupAgent->restore(cx,
-				                                  cx,
-				                                  self->backupTag,
-				                                  KeyRef(lastBackupContainer),
-				                                  {},
-				                                  WaitForComplete::True,
-				                                  ::invalidVersion,
-				                                  Verbose::True,
-				                                  normalKeys,
-				                                  Key(),
-				                                  Key(),
-				                                  self->locked,
-				                                  OnlyApplyMutationLogs::False,
-				                                  InconsistentSnapshotOnly::False,
-				                                  ::invalidVersion,
-				                                  self->encryptionKeyFileName)));
-				TraceEvent(SevError, "BARW_RestoreAllowedOverwrittingDatabase", randomID).log();
-				ASSERT(false);
-			} catch (Error& e) {
-				if (e.code() != error_code_restore_destination_not_empty) {
-					throw;
-				}
-			}
-		}
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> _start(Database cx, BackupAndParallelRestoreCorrectnessWorkload* self) {
-		state FileBackupAgent backupAgent;
-		state Future<Void> extraBackup;
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		state int restoreIndex = 0;
-		state ReadYourWritesTransaction tr2(cx);
-
-		TraceEvent("BARW_Arguments")
-		    .detail("BackupTag", printable(self->backupTag))
-		    .detail("PerformRestore", self->performRestore)
-		    .detail("BackupAfter", self->backupAfter)
-		    .detail("RestoreAfter", self->restoreAfter)
-		    .detail("AbortAndRestartAfter", self->abortAndRestartAfter)
-		    .detail("DifferentialAfter", self->stopDifferentialAfter);
-
-		if (self->allowPauses && BUGGIFY) {
-			state Future<Void> cp = changePaused(cx, &backupAgent);
-		}
-
-		// Increment the backup agent requests
-		if (self->agentRequest) {
-			BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests++;
-		}
-
-		if (self->encryptionKeyFileName.present()) {
-			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
-		}
-
-		try {
-			state Future<Void> startRestore = delay(self->restoreAfter);
-
-			// backup
-			wait(delay(self->backupAfter));
-
-			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
-			state Promise<Void> submitted;
-			state Future<Void> b = doBackup(
-			    self, 0, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter, submitted);
-
-			if (self->abortAndRestartAfter) {
-				TraceEvent("BARW_DoBackup2", randomID)
-				    .detail("Tag", printable(self->backupTag))
-				    .detail("AbortWait", self->abortAndRestartAfter);
-				wait(submitted.getFuture());
-				b = b && doBackup(self,
-				                  self->abortAndRestartAfter,
-				                  &backupAgent,
-				                  cx,
-				                  self->backupTag,
-				                  self->backupRanges,
-				                  self->stopDifferentialAfter,
-				                  Promise<Void>());
-			}
-
-			TraceEvent("BARW_DoBackupWait", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
-			try {
-				wait(b);
-			} catch (Error& e) {
-				if (e.code() != error_code_database_locked)
-					throw;
-				if (self->performRestore)
-					throw;
-				return Void();
-			}
-			TraceEvent("BARW_DoBackupDone", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
-
-			state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
-			UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx.getReference()));
-			state UID logUid = uidFlag.first;
-			state Key destUidValue = wait(BackupConfig(logUid).destUidValue().getD(cx.getReference()));
-			state Reference<IBackupContainer> lastBackupContainer =
-			    wait(BackupConfig(logUid).backupContainer().getD(cx.getReference()));
-
-			// Occasionally start yet another backup that might still be running when we restore
-			if (!self->locked && BUGGIFY) {
-				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
-				try {
-					// Note the "partitionedLog" must be false, because we change
-					// the configuration to disable backup workers before restore.
-					extraBackup = backupAgent.submitBackup(cx,
-					                                       "file://simfdb/backups/"_sr,
-					                                       {},
-					                                       deterministicRandom()->randomInt(0, 60),
-					                                       deterministicRandom()->randomInt(0, 100),
-					                                       self->backupTag.toString(),
-					                                       self->backupRanges,
-					                                       StopWhenDone::True,
-					                                       UsePartitionedLog::False,
-					                                       IncrementalBackupOnly::False,
-					                                       self->encryptionKeyFileName);
-				} catch (Error& e) {
-					TraceEvent("BARW_SubmitBackup2Exception", randomID)
-					    .error(e)
-					    .detail("BackupTag", printable(self->backupTag));
-					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-						throw;
-				}
-			}
-
-			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
-			wait(startRestore);
-
-			if (lastBackupContainer && self->performRestore) {
-				if (deterministicRandom()->random01() < 0.5) {
-					printf("TODO: Check if restore can succeed if dirty restore is performed first\n");
-					// TODO: To support restore even after we attempt dirty restore. Not implemented in the 1st version
-					// fast restore
-					// wait(attemptDirtyRestore(self, cx, &backupAgent, StringRef(lastBackupContainer->getURL()),
-					// randomID));
-				}
-
-				// We must ensure no backup workers are running, otherwise the clear DB
-				// below can be picked up by backup workers and applied during restore.
-				wait(success(ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=0", true)));
-
-				// Clear DB before restore
-				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-					for (auto& kvrange : self->backupRanges)
-						tr->clear(kvrange);
-					return Void();
-				}));
-
-				// restore database
-				TraceEvent("BAFRW_Restore", randomID)
-				    .detail("LastBackupContainer", lastBackupContainer->getURL())
-				    .detail("RestoreAfter", self->restoreAfter)
-				    .detail("BackupTag", printable(self->backupTag));
-				// start restoring
-
-				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(),
-				                                                 lastBackupContainer->getProxy(),
-				                                                 lastBackupContainer->getEncryptionKeyFileName());
-				BackupDescription desc = wait(container->describeBackup());
-				ASSERT(self->usePartitionedLogs == desc.partitioned);
-				ASSERT(desc.minRestorableVersion.present()); // We must have a valid backup now.
-
-				state Version targetVersion = -1;
-				if (desc.maxRestorableVersion.present()) {
-					if (deterministicRandom()->random01() < 0.1) {
-						targetVersion = desc.minRestorableVersion.get();
-					} else if (deterministicRandom()->random01() < 0.1) {
-						targetVersion = desc.maxRestorableVersion.get();
-					} else if (deterministicRandom()->random01() < 0.5) {
-						targetVersion = (desc.minRestorableVersion.get() != desc.maxRestorableVersion.get())
-						                    ? deterministicRandom()->randomInt64(desc.minRestorableVersion.get(),
-						                                                         desc.maxRestorableVersion.get())
-						                    : desc.maxRestorableVersion.get();
-					}
-				}
-
-				TraceEvent("BAFRW_Restore", randomID)
-				    .detail("LastBackupContainer", lastBackupContainer->getURL())
-				    .detail("MinRestorableVersion", desc.minRestorableVersion.get())
-				    .detail("MaxRestorableVersion", desc.maxRestorableVersion.get())
-				    .detail("ContiguousLogEnd", desc.contiguousLogEnd.get())
-				    .detail("TargetVersion", targetVersion);
-
-				state std::vector<Future<Version>> restores;
-				state std::vector<Standalone<StringRef>> restoreTags;
-
-				// Submit parallel restore requests
-				TraceEvent("BackupAndParallelRestoreWorkload")
-				    .detail("PrepareRestores", self->backupRanges.size())
-				    .detail("AddPrefix", self->addPrefix)
-				    .detail("RemovePrefix", self->removePrefix);
-				wait(backupAgent.submitParallelRestore(cx,
-				                                       self->backupTag,
-				                                       self->backupRanges,
-				                                       KeyRef(lastBackupContainer->getURL()),
-				                                       lastBackupContainer->getProxy(),
-				                                       targetVersion,
-				                                       self->locked,
-				                                       randomID,
-				                                       self->addPrefix,
-				                                       self->removePrefix));
-				TraceEvent("BackupAndParallelRestoreWorkload")
-				    .detail("TriggerRestore", "Setting up restoreRequestTriggerKey");
-
-				// Sometimes kill and restart the restore
-				// In real cluster, aborting a restore needs:
-				// (1) kill restore cluster; (2) clear dest. DB restore system keyspace.
-				// TODO: Consider gracefully abort a restore and restart.
-				if (BUGGIFY && TEST_ABORT_FASTRESTORE) {
-					TraceEvent(SevError, "FastRestore").detail("Buggify", "NotImplementedYet");
-					wait(delay(deterministicRandom()->randomInt(0, 10)));
-					for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) {
-						FileBackupAgent::ERestoreState rs =
-						    wait(backupAgent.abortRestore(cx, restoreTags[restoreIndex]));
-						// The restore may have already completed, or the abort may have been done before the restore
-						// was even able to start.  Only run a new restore if the previous one was actually aborted.
-						if (rs == FileBackupAgent::ERestoreState::ABORTED) {
-							wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-								tr->clear(self->backupRanges[restoreIndex]);
-								return Void();
-							}));
-							// TODO: Not Implemented yet
-							// restores[restoreIndex] = backupAgent.restore(cx, restoreTags[restoreIndex],
-							// KeyRef(lastBackupContainer->getURL()), true, -1, true, self->backupRanges[restoreIndex],
-							// Key(), Key(), self->locked);
-						}
-					}
-				}
-
-				// Wait for parallel restore to finish before we can proceed
-				TraceEvent("FastRestoreWorkload").detail("WaitForRestoreToFinish", randomID);
-				// Do not unlock DB when restore finish because we need to transformDatabaseContents
-				wait(backupAgent.parallelRestoreFinish(cx, randomID, UnlockDB{ !self->hasPrefix() }));
-				TraceEvent("FastRestoreWorkload").detail("RestoreFinished", randomID);
-
-				for (auto& restore : restores) {
-					ASSERT(!restore.isError());
-				}
-
-				// If addPrefix or removePrefix set, we want to transform the effect by copying data
-				if (self->hasPrefix()) {
-					wait(transformRestoredDatabase(cx, self->backupRanges, self->addPrefix, self->removePrefix));
-					wait(unlockDatabase(cx, randomID));
-				}
-			}
-
-			// Q: What is the extra backup and why do we need to care about it?
-			if (extraBackup.isValid()) { // SOMEDAY: Handle this case
-				TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(self->backupTag));
-				try {
-					wait(extraBackup);
-				} catch (Error& e) {
-					TraceEvent("BARW_ExtraBackupException", randomID)
-					    .error(e)
-					    .detail("BackupTag", printable(self->backupTag));
-					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-						throw;
-				}
-
-				TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(self->backupTag));
-				try {
-					wait(backupAgent.abortBackup(cx, self->backupTag.toString()));
-				} catch (Error& e) {
-					TraceEvent("BARW_AbortBackupExtraException", randomID).error(e);
-					if (e.code() != error_code_backup_unneeded)
-						throw;
-				}
-			}
-
-			state Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
-			state Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
-			state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
-			state Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
-			state int displaySystemKeys = 0;
-
-			// Ensure that there is no left over key within the backup subspace
-			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-
-				TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(self->backupTag));
-
-				try {
-					tr->reset();
-					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-
-					// Check the left over tasks
-					// We have to wait for the list to empty since an abort and get status
-					// can leave extra tasks in the queue
-					TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(self->backupTag));
-					state int64_t taskCount = wait(backupAgent.getTaskCount(tr));
-					state int waitCycles = 0;
-
-					loop {
-						waitCycles++;
-
-						TraceEvent("BARW_NonzeroTaskWait", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
-						    .detail("TaskCount", taskCount)
-						    .detail("WaitCycles", waitCycles);
-						printf("%.6f %-10s Wait #%4d for %lld tasks to end\n",
-						       now(),
-						       randomID.toString().c_str(),
-						       waitCycles,
-						       (long long)taskCount);
-
-						wait(delay(5.0));
-						wait(tr->commit());
-						tr = makeReference<ReadYourWritesTransaction>(cx);
-						int64_t _taskCount = wait(backupAgent.getTaskCount(tr));
-						taskCount = _taskCount;
-
-						if (!taskCount) {
-							break;
-						}
-					}
-
-					if (taskCount) {
-						displaySystemKeys++;
-						TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
-						    .detail("TaskCount", taskCount)
-						    .detail("WaitCycles", waitCycles);
-						printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long)taskCount);
-					}
-
-					RangeResult agentValues =
-					    wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
-
-					// Error if the system keyspace for the backup tag is not empty
-					if (agentValues.size() > 0) {
-						displaySystemKeys++;
-						printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n",
-						       agentValues.size(),
-						       printable(backupAgentKey).c_str());
-						TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
-						    .detail("LeftOverKeys", agentValues.size())
-						    .detail("KeySpace", printable(backupAgentKey));
-						for (auto& s : agentValues) {
-							TraceEvent("BARW_LeftOverKey", randomID)
-							    .detail("Key", printable(StringRef(s.key.toString())))
-							    .detail("Value", printable(StringRef(s.value.toString())));
-							printf("   Key: %-50s  Value: %s\n",
-							       printable(StringRef(s.key.toString())).c_str(),
-							       printable(StringRef(s.value.toString())).c_str());
-						}
-					} else {
-						printf("No left over backup agent configuration keys\n");
-					}
-
-					Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
-					if (latestVersion.present()) {
-						TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
-						    .detail("BackupLatestVersionsKey", backupLatestVersionsKey.printable())
-						    .detail("DestUidValue", destUidValue.printable());
-					} else {
-						printf("No left over backup version key\n");
-					}
-
-					RangeResult versions = wait(tr->getRange(
-					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1));
-					if (!self->shareLogRange || !versions.size()) {
-						RangeResult logValues = wait(
-						    tr->getRange(KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100));
-
-						// Error if the log/mutation keyspace for the backup tag  is not empty
-						if (logValues.size() > 0) {
-							displaySystemKeys++;
-							printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n",
-							       logValues.size(),
-							       printable(backupLogValuesKey).c_str());
-							TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID)
-							    .detail("BackupTag", printable(self->backupTag))
-							    .detail("LeftOverKeys", logValues.size())
-							    .detail("KeySpace", printable(backupLogValuesKey));
-						} else {
-							printf("No left over backup log keys\n");
-						}
-					}
-
-					break;
-				} catch (Error& e) {
-					TraceEvent("BARW_CheckException", randomID).error(e);
-					wait(tr->onError(e));
-				}
-			}
-
-			if (displaySystemKeys) {
-				wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
-			}
-
-			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
-
-			// Decrement the backup agent requests
-			if (self->agentRequest) {
-				BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests--;
-			}
-
-			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
-			    (!BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests)) {
-				g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
-			}
-		} catch (Error& e) {
-			TraceEvent(SevError, "BackupAndParallelRestoreCorrectness").error(e).GetLastError();
-			throw;
-		}
-		return Void();
-	}
-};
-
-int BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests = 0;
-
-WorkloadFactory<BackupAndParallelRestoreCorrectnessWorkload> BackupAndParallelRestoreCorrectnessWorkloadFactory;
diff --git a/fdbserver/workloads/BackupAndRestoreValidation.actor.cpp b/fdbserver/workloads/BackupAndRestoreValidation.actor.cpp
deleted file mode 100644
index 7a288470b82..00000000000
--- a/fdbserver/workloads/BackupAndRestoreValidation.actor.cpp
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * BackupAndRestoreValidation.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbclient/SystemData.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// Simplified backup and restore workload specifically for restore validation testing
-// This avoids the complexity of BackupAndRestoreCorrectness which is used by many tests
-
-// Completion marker key to signal that restore is fully done
-const KeyRef restoreValidationCompletionKey = "\xff\x02/restoreValidationComplete"_sr;
-struct BackupAndRestoreValidationWorkload : TestWorkload {
-	static constexpr auto NAME = "BackupAndRestoreValidation";
-	double backupAfter, restoreAfter;
-	Key backupTag;
-	Key addPrefix; // Prefix to add during restore (e.g., \xff\x02/rlog/)
-
-	BackupAndRestoreValidationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
-		restoreAfter = getOption(options, "restoreAfter"_sr, 30.0);
-		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
-		addPrefix = unprintable(getOption(options, "addPrefix"_sr, ""_sr).toString());
-
-		TraceEvent("BARV_Init")
-		    .detail("BackupAfter", backupAfter)
-		    .detail("RestoreAfter", restoreAfter)
-		    .detail("AddPrefix", printable(addPrefix));
-	}
-
-	Future<Void> setup(Database const& cx) override { return Void(); }
-
-	Future<Void> start(Database const& cx) override {
-		if (clientId != 0)
-			return Void();
-		return _start(cx, this);
-	}
-
-	Future<bool> check(Database const& cx) override { return true; }
-
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-
-	ACTOR static Future<Void> doBackup(BackupAndRestoreValidationWorkload* self,
-	                                   FileBackupAgent* backupAgent,
-	                                   Database cx) {
-		state std::string backupContainer = "file://simfdb/backups/";
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
-
-		// Only backup normal user keys (not system keys)
-		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
-
-		TraceEvent("BARV_SubmitBackup").detail("Tag", printable(self->backupTag)).detail("Container", backupContainer);
-
-		try {
-			wait(backupAgent->submitBackup(cx,
-			                               StringRef(backupContainer),
-			                               {},
-			                               deterministicRandom()->randomInt(0, 60),
-			                               deterministicRandom()->randomInt(0, 100),
-			                               self->backupTag.toString(),
-			                               backupRanges,
-			                               StopWhenDone{ true }));
-		} catch (Error& e) {
-			TraceEvent("BARV_SubmitBackupException").error(e);
-			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
-				throw;
-		}
-
-		// Wait for backup to complete
-		TraceEvent("BARV_WaitBackup").detail("Tag", printable(self->backupTag));
-		state EBackupState statusValue =
-		    wait(backupAgent->waitBackup(cx, self->backupTag.toString(), StopWhenDone::True));
-
-		TraceEvent("BARV_BackupComplete")
-		    .detail("Tag", printable(self->backupTag))
-		    .detail("Status", BackupAgentBase::getStateText(statusValue));
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> doRestore(BackupAndRestoreValidationWorkload* self,
-	                                    FileBackupAgent* backupAgent,
-	                                    Database cx,
-	                                    Reference<IBackupContainer> backupContainer) {
-		state Standalone<VectorRef<KeyRangeRef>> restoreRanges;
-
-		// Restore normal user keys only
-		restoreRanges.push_back_deep(restoreRanges.arena(), normalKeys);
-
-		state Standalone<StringRef> restoreTag(self->backupTag.toString() + "_restore");
-
-		TraceEvent("BARV_StartRestore")
-		    .detail("Tag", printable(restoreTag))
-		    .detail("Container", backupContainer->getURL())
-		    .detail("AddPrefix", printable(self->addPrefix));
-
-		// Don't clear keys - we want to keep original data for validation comparison
-		// The restore will put data at the addPrefix location
-
-		wait(success(backupAgent->restore(cx,
-		                                  cx,
-		                                  restoreTag,
-		                                  KeyRef(backupContainer->getURL()),
-		                                  backupContainer->getProxy(),
-		                                  restoreRanges,
-		                                  WaitForComplete::True,
-		                                  ::invalidVersion,
-		                                  Verbose::True,
-		                                  self->addPrefix,
-		                                  Key(), // removePrefix
-		                                  LockDB{ false },
-		                                  UnlockDB::True,
-		                                  OnlyApplyMutationLogs::False,
-		                                  InconsistentSnapshotOnly::False,
-		                                  ::invalidVersion,
-		                                  backupContainer->getEncryptionKeyFileName())));
-
-		TraceEvent("BARV_RestoreComplete")
-		    .detail("Tag", printable(restoreTag))
-		    .detail("AddPrefix", printable(self->addPrefix));
-
-		// Wait a bit to ensure all restored data is committed and visible
-		// The restore API returns success before all data is fully flushed to storage servers
-		wait(delay(5.0));
-		TraceEvent("BARV_RestoreDataStabilizationWait").detail("WaitTime", 5.0);
-
-		// Write a completion marker so RestoreValidation knows restore is fully done
-		state Key completionMarker = restoreValidationCompletionKey;
-		state Transaction markTr(cx);
-		loop {
-			try {
-				markTr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				markTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				markTr.set(completionMarker, "1"_sr);
-				wait(markTr.commit());
-				TraceEvent("BARV_RestoreCompletionMarkerSet").detail("MarkerKey", printable(completionMarker));
-				break;
-			} catch (Error& e) {
-				wait(markTr.onError(e));
-			}
-		}
-
-		// Unlock the database after restore completes
-		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr->clear(databaseLockedKey);
-			return Void();
-		}));
-
-		TraceEvent("BARV_DatabaseUnlocked").detail("Tag", printable(restoreTag));
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> _start(Database cx, BackupAndRestoreValidationWorkload* self) {
-		// Only run on client 0 to avoid conflicts
-		if (self->clientId != 0) {
-			return Void();
-		}
-
-		state FileBackupAgent backupAgent;
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		state int retryCount = 0;
-
-		loop {
-			try {
-				// Wait before starting backup
-				wait(delay(self->backupAfter));
-
-				// Perform backup
-				TraceEvent("BARV_StartBackup", randomID)
-				    .detail("Tag", printable(self->backupTag))
-				    .detail("RetryCount", retryCount);
-				wait(doBackup(self, &backupAgent, cx));
-
-				// Get backup container info
-				state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
-				UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx.getReference()));
-				state UID logUid = uidFlag.first;
-				state Reference<IBackupContainer> backupContainer =
-				    wait(BackupConfig(logUid).backupContainer().getD(cx.getReference()));
-
-				// Wait before starting restore
-				wait(delay(self->restoreAfter - self->backupAfter));
-
-				// Perform restore with prefix
-				TraceEvent("BARV_StartRestore", randomID)
-				    .detail("Tag", printable(self->backupTag))
-				    .detail("Container", backupContainer->getURL());
-				wait(doRestore(self, &backupAgent, cx, backupContainer));
-
-				TraceEvent("BARV_Complete", randomID).detail("Tag", printable(self->backupTag));
-				break; // Success!
-
-			} catch (Error& e) {
-				// Retry on transient errors from buggify chaos injection
-				if (e.code() == error_code_grv_proxy_memory_limit_exceeded ||
-				    e.code() == error_code_commit_proxy_memory_limit_exceeded ||
-				    e.code() == error_code_database_locked || e.code() == error_code_transaction_too_old ||
-				    e.code() == error_code_future_version) {
-					retryCount++;
-					double backoff = std::min(1.0, 0.1 * retryCount);
-					TraceEvent(SevWarn, "BARV_RetryableError", randomID)
-					    .error(e)
-					    .detail("RetryCount", retryCount)
-					    .detail("BackoffSeconds", backoff);
-					wait(delay(backoff));
-					// Reset state and retry
-					self->backupAfter = 0.0; // Don't wait again
-					self->restoreAfter = self->restoreAfter - self->backupAfter;
-					// Loop will retry
-				} else {
-					TraceEvent(SevError, "BARV_Error", randomID).error(e).detail("RetryCount", retryCount);
-					throw;
-				}
-			}
-		}
-
-		return Void();
-	}
-};
-
-WorkloadFactory<BackupAndRestoreValidationWorkload> BackupAndRestoreValidationWorkloadFactory;
diff --git a/fdbserver/workloads/BackupAndRestoreValidation.cpp b/fdbserver/workloads/BackupAndRestoreValidation.cpp
new file mode 100644
index 00000000000..b6470f117de
--- /dev/null
+++ b/fdbserver/workloads/BackupAndRestoreValidation.cpp
@@ -0,0 +1,238 @@
+/*
+ * BackupAndRestoreValidation.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/BackupAgent.h"
+#include "fdbclient/BackupContainer.h"
+#include "fdbclient/SystemData.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+// Simplified backup and restore workload specifically for restore validation testing
+// This avoids the complexity of BackupAndRestoreCorrectness which is used by many tests
+
+// Completion marker key to signal that restore is fully done
+const KeyRef restoreValidationCompletionKey = "\xff\x02/restoreValidationComplete"_sr;
+struct BackupAndRestoreValidationWorkload : TestWorkload {
+	static constexpr auto NAME = "BackupAndRestoreValidation";
+	double backupAfter, restoreAfter;
+	Key backupTag;
+	Key addPrefix; // Prefix to add during restore (e.g., \xff\x02/rlog/)
+
+	explicit BackupAndRestoreValidationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
+		restoreAfter = getOption(options, "restoreAfter"_sr, 30.0);
+		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		addPrefix = unprintable(getOption(options, "addPrefix"_sr, ""_sr).toString());
+
+		TraceEvent("BARV_Init")
+		    .detail("BackupAfter", backupAfter)
+		    .detail("RestoreAfter", restoreAfter)
+		    .detail("AddPrefix", printable(addPrefix));
+	}
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0)
+			return Void();
+		return _start(cx);
+	}
+
+	Future<bool> check(Database const& cx) override { return true; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	Future<Void> doBackup(FileBackupAgent* backupAgent, Database cx) {
+		std::string backupContainer = "file://simfdb/backups/";
+		Standalone<VectorRef<KeyRangeRef>> backupRanges;
+
+		// Only backup normal user keys (not system keys)
+		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
+
+		TraceEvent("BARV_SubmitBackup").detail("Tag", printable(backupTag)).detail("Container", backupContainer);
+
+		try {
+			co_await backupAgent->submitBackup(cx,
+			                                   StringRef(backupContainer),
+			                                   {},
+			                                   deterministicRandom()->randomInt(0, 60),
+			                                   deterministicRandom()->randomInt(0, 100),
+			                                   backupTag.toString(),
+			                                   backupRanges,
+			                                   StopWhenDone{ true });
+		} catch (Error& e) {
+			TraceEvent("BARV_SubmitBackupException").error(e);
+			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
+				throw;
+		}
+
+		// Wait for backup to complete
+		TraceEvent("BARV_WaitBackup").detail("Tag", printable(backupTag));
+		EBackupState statusValue = co_await backupAgent->waitBackup(cx, backupTag.toString(), StopWhenDone::True);
+
+		TraceEvent("BARV_BackupComplete")
+		    .detail("Tag", printable(backupTag))
+		    .detail("Status", BackupAgentBase::getStateText(statusValue));
+	}
+
+	Future<Void> doRestore(FileBackupAgent* backupAgent, Database cx, Reference<IBackupContainer> backupContainer) {
+		Standalone<VectorRef<KeyRangeRef>> restoreRanges;
+
+		// Restore normal user keys only
+		restoreRanges.push_back_deep(restoreRanges.arena(), normalKeys);
+
+		Standalone<StringRef> restoreTag(backupTag.toString() + "_restore");
+
+		TraceEvent("BARV_StartRestore")
+		    .detail("Tag", printable(restoreTag))
+		    .detail("Container", backupContainer->getURL())
+		    .detail("AddPrefix", printable(addPrefix));
+
+		// Don't clear keys - we want to keep original data for validation comparison
+		// The restore will put data at the addPrefix location
+
+		co_await backupAgent->restore(cx,
+		                              cx,
+		                              restoreTag,
+		                              KeyRef(backupContainer->getURL()),
+		                              backupContainer->getProxy(),
+		                              restoreRanges,
+		                              WaitForComplete::True,
+		                              ::invalidVersion,
+		                              Verbose::True,
+		                              addPrefix,
+		                              Key(), // removePrefix
+		                              LockDB{ false },
+		                              UnlockDB::True,
+		                              OnlyApplyMutationLogs::False,
+		                              InconsistentSnapshotOnly::False,
+		                              ::invalidVersion,
+		                              backupContainer->getEncryptionKeyFileName());
+
+		TraceEvent("BARV_RestoreComplete")
+		    .detail("Tag", printable(restoreTag))
+		    .detail("AddPrefix", printable(addPrefix));
+
+		// Wait a bit to ensure all restored data is committed and visible
+		// The restore API returns success before all data is fully flushed to storage servers
+		co_await delay(5.0);
+		TraceEvent("BARV_RestoreDataStabilizationWait").detail("WaitTime", 5.0);
+
+		// Write a completion marker so RestoreValidation knows restore is fully done
+		Key completionMarker = restoreValidationCompletionKey;
+		Transaction markTr(cx);
+		while (true) {
+			Error err;
+			try {
+				markTr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				markTr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				markTr.set(completionMarker, "1"_sr);
+				co_await markTr.commit();
+				TraceEvent("BARV_RestoreCompletionMarkerSet").detail("MarkerKey", printable(completionMarker));
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await markTr.onError(err);
+		}
+
+		// Unlock the database after restore completes
+		co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr->clear(databaseLockedKey);
+			return Void();
+		});
+
+		TraceEvent("BARV_DatabaseUnlocked").detail("Tag", printable(restoreTag));
+	}
+
+	Future<Void> _start(Database cx) {
+		// Only run on client 0 to avoid conflicts
+		if (clientId != 0) {
+			co_return;
+		}
+
+		FileBackupAgent backupAgent;
+		UID randomID = nondeterministicRandom()->randomUniqueID();
+		int retryCount = 0;
+
+		while (true) {
+			Error err;
+			try {
+				// Wait before starting backup
+				co_await delay(backupAfter);
+
+				// Perform backup
+				TraceEvent("BARV_StartBackup", randomID)
+				    .detail("Tag", printable(backupTag))
+				    .detail("RetryCount", retryCount);
+				co_await doBackup(&backupAgent, cx);
+
+				// Get backup container info
+				KeyBackedTag keyBackedTag = makeBackupTag(backupTag.toString());
+				UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
+				UID logUid = uidFlag.first;
+				Reference<IBackupContainer> backupContainer =
+				    co_await BackupConfig(logUid).backupContainer().getD(cx.getReference());
+
+				// Wait before starting restore
+				co_await delay(restoreAfter - backupAfter);
+
+				// Perform restore with prefix
+				TraceEvent("BARV_StartRestore", randomID)
+				    .detail("Tag", printable(backupTag))
+				    .detail("Container", backupContainer->getURL());
+				co_await doRestore(&backupAgent, cx, backupContainer);
+
+				TraceEvent("BARV_Complete", randomID).detail("Tag", printable(backupTag));
+				break; // Success!
+
+			} catch (Error& e) {
+				err = e;
+			}
+			// Retry on transient errors from buggify chaos injection
+			if (err.code() == error_code_grv_proxy_memory_limit_exceeded ||
+			    err.code() == error_code_commit_proxy_memory_limit_exceeded ||
+			    err.code() == error_code_database_locked || err.code() == error_code_transaction_too_old ||
+			    err.code() == error_code_future_version) {
+				retryCount++;
+				double backoff = std::min(1.0, 0.1 * retryCount);
+				TraceEvent(SevWarn, "BARV_RetryableError", randomID)
+				    .error(err)
+				    .detail("RetryCount", retryCount)
+				    .detail("BackoffSeconds", backoff);
+				co_await delay(backoff);
+				// Reset state and retry
+				backupAfter = 0.0; // Don't wait again
+				restoreAfter = restoreAfter - backupAfter;
+				// Loop will retry
+			} else {
+				TraceEvent(SevError, "BARV_Error", randomID).error(err).detail("RetryCount", retryCount);
+				throw err;
+			}
+		}
+	}
+};
+
+WorkloadFactory<BackupAndRestoreValidationWorkload> BackupAndRestoreValidationWorkloadFactory;
diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.cpp
similarity index 65%
rename from fdbserver/workloads/BackupCorrectness.actor.cpp
rename to fdbserver/workloads/BackupCorrectness.cpp
index 16421682370..f1458983bba 100644
--- a/fdbserver/workloads/BackupCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupCorrectness.actor.cpp
+ * BackupCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,17 +19,17 @@
  */
 
 #include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
+#include "BulkSetup.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // TODO: explain the purpose of this workload and how it different from the
 // 20+ (literally) other backup/restore workloads.
@@ -59,7 +59,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		out.insert({ "RandomRangeLock" });
 	}
 
-	BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
 		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
@@ -186,7 +186,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		return _setup(cx, this);
 	}
 
-	ACTOR Future<Void> _setup(Database cx, BackupAndRestoreCorrectnessWorkload* self) {
+	Future<Void> _setup(Database cx, BackupAndRestoreCorrectnessWorkload* self) {
 		if (BUGGIFY) {
 			for (auto r : getSystemBackupRanges()) {
 				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
@@ -216,75 +216,75 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest);
 		TraceEvent(SevInfo, "BARW_Param").detail("Encrypted", encryptionKeyFileName.present());
 
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override {
 		if (clientId != 0)
 			return true;
 		else
-			return _check(cx, this);
+			return _check(cx);
 	}
 
-	ACTOR static Future<bool> _check(Database cx, BackupAndRestoreCorrectnessWorkload* self) {
-		state Transaction tr(cx);
-		loop {
+	Future<bool> _check(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				state int restoreIndex;
-				for (restoreIndex = 0; restoreIndex < self->skippedRestoreRanges.size(); restoreIndex++) {
-					state KeyRangeRef range = self->skippedRestoreRanges[restoreIndex];
-					Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
-					RangeResult res = wait(tr.getRange(range, GetRangeLimits::ROW_LIMIT_UNLIMITED));
+				for (int restoreIndex = 0; restoreIndex < skippedRestoreRanges.size(); restoreIndex++) {
+					KeyRangeRef range = skippedRestoreRanges[restoreIndex];
+					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
+					RangeResult res = co_await tr.getRange(range, GetRangeLimits::ROW_LIMIT_UNLIMITED);
 					if (!res.empty()) {
 						TraceEvent(SevError, "BARW_UnexpectedRangePresent").detail("Range", printable(range));
-						return false;
+						co_return false;
 					}
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return true;
+		co_return true;
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
-		loop {
-			wait(backupAgent->changePause(cx, true));
-			wait(delay(30 * deterministicRandom()->random01()));
-			wait(backupAgent->changePause(cx, false));
-			wait(delay(120 * deterministicRandom()->random01()));
+	static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
+		while (true) {
+			co_await backupAgent->changePause(cx, true);
+			co_await delay(30 * deterministicRandom()->random01());
+			co_await backupAgent->changePause(cx, false);
+			co_await delay(120 * deterministicRandom()->random01());
 		}
 	}
 
-	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
-		state FileBackupAgent agent;
-		loop {
-			bool active = wait(agent.checkActive(cx));
+	static Future<Void> statusLoop(Database cx, std::string tag) {
+		FileBackupAgent agent;
+		while (true) {
+			bool active = co_await agent.checkActive(cx);
 			TraceEvent("BARW_AgentActivityCheck").detail("IsActive", active);
-			std::string status = wait(agent.getStatus(cx, ShowErrors::True, tag));
+			std::string status = co_await agent.getStatus(cx, ShowErrors::True, tag);
 			puts(status.c_str());
-			std::string statusJSON = wait(agent.getStatusJSON(cx, tag));
+			std::string statusJSON = co_await agent.getStatusJSON(cx, tag);
 			puts(statusJSON.c_str());
-			wait(delay(2.0));
+			co_await delay(2.0);
 		}
 	}
 
-	ACTOR static Future<Void> doBackup(BackupAndRestoreCorrectnessWorkload* self,
-	                                   double startDelay,
-	                                   FileBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   double stopDifferentialDelay,
-	                                   Promise<Void> submitted) {
+	Future<Void> doBackup(double startDelay,
+	                      FileBackupAgent* backupAgent,
+	                      Database cx,
+	                      Key tag,
+	                      Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                      double stopDifferentialDelay,
+	                      Promise<Void> submitted) {
 
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 
-		state Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
-		wait(delay(startDelay));
+		Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
+		co_await delay(startDelay);
 
 		if (startDelay || BUGGIFY) {
 			TraceEvent("BARW_DoBackupAbortBackup1", randomID)
@@ -292,7 +292,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 			    .detail("StartDelay", startDelay);
 
 			try {
-				wait(backupAgent->abortBackup(cx, tag.toString()));
+				co_await backupAgent->abortBackup(cx, tag.toString());
 			} catch (Error& e) {
 				TraceEvent("BARW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
 				if (e.code() != error_code_backup_unneeded)
@@ -304,20 +304,22 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		    .detail("Tag", printable(tag))
 		    .detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
 
-		state std::string backupContainer = "file://simfdb/backups/";
-		state Future<Void> status = statusLoop(cx, tag.toString());
+		std::string backupContainer = "file://simfdb/backups/";
+		Future<Void> status = statusLoop(cx, tag.toString());
 		try {
-			wait(backupAgent->submitBackup(cx,
-			                               StringRef(backupContainer),
-			                               {},
-			                               deterministicRandom()->randomInt(0, 60),
-			                               deterministicRandom()->randomInt(0, 2000),
-			                               tag.toString(),
-			                               backupRanges,
-			                               StopWhenDone{ !stopDifferentialDelay },
-			                               UsePartitionedLog::False,
-			                               IncrementalBackupOnly::False,
-			                               self->encryptionKeyFileName));
+			co_await backupAgent->submitBackup(cx,
+			                                   StringRef(backupContainer),
+			                                   {},
+			                                   deterministicRandom()->randomInt(0, 60),
+			                                   deterministicRandom()->randomInt(0, 2000),
+			                                   tag.toString(),
+			                                   backupRanges,
+			                                   StopWhenDone{ !stopDifferentialDelay },
+			                                   MutationLogType::DEFAULT,
+			                                   IncrementalBackupOnly::False,
+			                                   encryptionKeyFileName,
+			                                   encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+			                                   /*snapshotMode=*/0);
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@@ -330,34 +332,34 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		if (stopDifferentialDelay) {
 			CODE_PROBE(!stopDifferentialFuture.isReady(),
 			           "Restore starts at specified time - stopDifferential not ready");
-			wait(stopDifferentialFuture);
+			co_await stopDifferentialFuture;
 			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
 			    .detail("DifferentialAfter", stopDifferentialDelay);
 
 			try {
 				if (BUGGIFY) {
-					state KeyBackedTag backupTag = makeBackupTag(tag.toString());
+					KeyBackedTag backupTag = makeBackupTag(tag.toString());
 					TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName);
 
 					// Wait until the backup is in a restorable state and get the status, URL, and UID atomically
-					state Reference<IBackupContainer> lastBackupContainer;
-					state UID lastBackupUID;
-					state EBackupState resultWait = wait(backupAgent->waitBackup(
-					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID));
+					Reference<IBackupContainer> lastBackupContainer;
+					UID lastBackupUID;
+					EBackupState resultWait = co_await backupAgent->waitBackup(
+					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID);
 
 					TraceEvent("BARW_DoBackupWaitForRestorable", randomID)
 					    .detail("Tag", backupTag.tagName)
 					    .detail("Result", BackupAgentBase::getStateText(resultWait));
 
-					state bool restorable = false;
+					bool restorable = false;
 					if (lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
-						wait(ready(fdesc));
+						Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
+						co_await ready(fdesc);
 
 						if (!fdesc.isError()) {
-							state BackupDescription desc = fdesc.get();
-							wait(desc.resolveVersionTimes(cx));
+							BackupDescription desc = fdesc.get();
+							co_await desc.resolveVersionTimes(cx);
 							printf("BackupDescription:\n%s\n", desc.toString().c_str());
 							restorable = desc.maxRestorableVersion.present();
 						}
@@ -401,12 +403,12 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						    .detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
 						    .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
 						    .detail("Restorable", restorable);
-						wait(backupAgent->abortBackup(cx, tag.toString()));
+						co_await backupAgent->abortBackup(cx, tag.toString());
 					} else {
 						TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
 						    .detail("Tag", printable(tag))
 						    .detail("DifferentialAfter", stopDifferentialDelay);
-						wait(backupAgent->discontinueBackup(cx, tag));
+						co_await backupAgent->discontinueBackup(cx, tag);
 					}
 				}
 
@@ -414,7 +416,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 					TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
 					    .detail("Tag", printable(tag))
 					    .detail("DifferentialAfter", stopDifferentialDelay);
-					wait(backupAgent->discontinueBackup(cx, tag));
+					co_await backupAgent->discontinueBackup(cx, tag);
 				}
 			} catch (Error& e) {
 				TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).error(e).detail("Tag", printable(tag));
@@ -425,11 +427,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 		// Wait for the backup to complete
 		TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
-		state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True));
+		EBackupState statusValue = co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True);
 
-		state std::string statusText;
+		std::string statusText;
 
-		std::string _statusText = wait(backupAgent->getStatus(cx, ShowErrors::True, tag.toString()));
+		std::string _statusText = co_await backupAgent->getStatus(cx, ShowErrors::True, tag.toString());
 		statusText = _statusText;
 		// Can we validate anything about status?
 
@@ -437,50 +439,49 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		    .detail("Tag", printable(tag))
 		    .detail("Status", statusText)
 		    .detail("StatusValue", BackupAgentBase::getStateText(statusValue));
-
-		return Void();
 	}
 
 	/**
 	    This actor attempts to restore the database without clearing the keyspace.
 	 */
-	ACTOR static Future<Void> attemptDirtyRestore(BackupAndRestoreCorrectnessWorkload* self,
-	                                              Database cx,
-	                                              FileBackupAgent* backupAgent,
-	                                              Standalone<StringRef> lastBackupContainer,
-	                                              UID randomID,
-	                                              Optional<std::string> encryptionKeyFileName) {
-		state Transaction tr(cx);
-		state int rowCount = 0;
-		loop {
+	Future<Void> attemptDirtyRestore(Database cx,
+	                                 FileBackupAgent* backupAgent,
+	                                 Standalone<StringRef> lastBackupContainer,
+	                                 UID randomID,
+	                                 Optional<std::string> encryptionKeyFileName) {
+		Transaction tr(cx);
+		int rowCount = 0;
+		while (true) {
+			Error err;
 			try {
-				RangeResult existingRows = wait(tr.getRange(normalKeys, 1));
+				RangeResult existingRows = co_await tr.getRange(normalKeys, 1);
 				rowCount = existingRows.size();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		// Try doing a restore without clearing the keys
 		if (rowCount > 0) {
 			try {
-				wait(success(backupAgent->restore(cx,
-				                                  cx,
-				                                  self->backupTag,
-				                                  KeyRef(lastBackupContainer),
-				                                  {},
-				                                  WaitForComplete::True,
-				                                  ::invalidVersion,
-				                                  Verbose::True,
-				                                  normalKeys,
-				                                  Key(),
-				                                  Key(),
-				                                  self->locked,
-				                                  OnlyApplyMutationLogs::False,
-				                                  InconsistentSnapshotOnly::False,
-				                                  ::invalidVersion,
-				                                  encryptionKeyFileName)));
+				co_await backupAgent->restore(cx,
+				                              cx,
+				                              backupTag,
+				                              KeyRef(lastBackupContainer),
+				                              {},
+				                              WaitForComplete::True,
+				                              ::invalidVersion,
+				                              Verbose::True,
+				                              normalKeys,
+				                              Key(),
+				                              Key(),
+				                              locked,
+				                              OnlyApplyMutationLogs::False,
+				                              InconsistentSnapshotOnly::False,
+				                              ::invalidVersion,
+				                              encryptionKeyFileName);
 				TraceEvent(SevError, "BARW_RestoreAllowedOverwrittingDatabase", randomID).log();
 				ASSERT(false);
 			} catch (Error& e) {
@@ -489,174 +490,169 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 				}
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> clearAndRestoreSystemKeys(Database cx,
-	                                                    BackupAndRestoreCorrectnessWorkload* self,
-	                                                    FileBackupAgent* backupAgent,
-	                                                    Version targetVersion,
-	                                                    Reference<IBackupContainer> lastBackupContainer,
-	                                                    Standalone<VectorRef<KeyRangeRef>> systemRestoreRanges) {
+	static Future<Void> clearAndRestoreSystemKeys(Database cx,
+	                                              BackupAndRestoreCorrectnessWorkload* self,
+	                                              FileBackupAgent* backupAgent,
+	                                              Version targetVersion,
+	                                              Reference<IBackupContainer> lastBackupContainer,
+	                                              Standalone<VectorRef<KeyRangeRef>> systemRestoreRanges) {
 		// restore system keys before restoring any other ranges
-		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+		co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			for (auto& range : systemRestoreRanges)
 				tr->clear(range);
 			return Void();
-		}));
-		state Standalone<StringRef> restoreTag(self->backupTag.toString() + "_system");
+		});
+		Standalone<StringRef> restoreTag(self->backupTag.toString() + "_system");
 		printf("BackupCorrectness, backupAgent.restore is called for tag:%s\n", restoreTag.toString().c_str());
-		wait(success(backupAgent->restore(cx,
-		                                  cx,
-		                                  restoreTag,
-		                                  KeyRef(lastBackupContainer->getURL()),
-		                                  lastBackupContainer->getProxy(),
-		                                  systemRestoreRanges,
-		                                  WaitForComplete::True,
-		                                  targetVersion,
-		                                  Verbose::True,
-		                                  Key(),
-		                                  Key(),
-		                                  self->locked,
-		                                  UnlockDB::True,
-		                                  OnlyApplyMutationLogs::False,
-		                                  InconsistentSnapshotOnly::False,
-		                                  ::invalidVersion,
-		                                  lastBackupContainer->getEncryptionKeyFileName())));
+		co_await backupAgent->restore(cx,
+		                              cx,
+		                              restoreTag,
+		                              KeyRef(lastBackupContainer->getURL()),
+		                              lastBackupContainer->getProxy(),
+		                              systemRestoreRanges,
+		                              WaitForComplete::True,
+		                              targetVersion,
+		                              Verbose::True,
+		                              Key(),
+		                              Key(),
+		                              self->locked,
+		                              UnlockDB::True,
+		                              OnlyApplyMutationLogs::False,
+		                              InconsistentSnapshotOnly::False,
+		                              ::invalidVersion,
+		                              lastBackupContainer->getEncryptionKeyFileName());
 		printf("BackupCorrectness, backupAgent.restore finished for tag:%s\n", restoreTag.toString().c_str());
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, BackupAndRestoreCorrectnessWorkload* self) {
-		state FileBackupAgent backupAgent;
-		state Future<Void> extraBackup;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> _start(Database cx) {
+		FileBackupAgent backupAgent;
+		Future<Void> extraBackup;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		TraceEvent("BARW_Arguments")
-		    .detail("BackupTag", printable(self->backupTag))
-		    .detail("PerformRestore", self->performRestore)
-		    .detail("BackupAfter", self->backupAfter)
-		    .detail("RestoreAfter", self->restoreAfter)
-		    .detail("AbortAndRestartAfter", self->abortAndRestartAfter)
-		    .detail("DifferentialAfter", self->stopDifferentialAfter);
-
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		if (self->allowPauses && BUGGIFY) {
-			state Future<Void> cp = changePaused(cx, &backupAgent);
+		    .detail("BackupTag", printable(backupTag))
+		    .detail("PerformRestore", performRestore)
+		    .detail("BackupAfter", backupAfter)
+		    .detail("RestoreAfter", restoreAfter)
+		    .detail("AbortAndRestartAfter", abortAndRestartAfter)
+		    .detail("DifferentialAfter", stopDifferentialAfter);
+
+		UID randomID = nondeterministicRandom()->randomUniqueID();
+		if (allowPauses && BUGGIFY) {
+			Future<Void> cp = changePaused(cx, &backupAgent);
 		}
 
 		// Increment the backup agent requests
-		if (self->agentRequest) {
+		if (agentRequest) {
 			BackupAndRestoreCorrectnessWorkload::backupAgentRequests++;
 		}
 
-		if (self->encryptionKeyFileName.present()) {
-			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		if (encryptionKeyFileName.present()) {
+			co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 		}
 
 		try {
-			state Future<Void> startRestore = delay(self->restoreAfter);
+			Future<Void> startRestore = delay(restoreAfter);
 
 			// backup
-			wait(delay(self->backupAfter));
+			co_await delay(backupAfter);
 
-			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
-			state Promise<Void> submitted;
-			state Future<Void> b = doBackup(
-			    self, 0, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter, submitted);
+			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(backupTag));
+			Promise<Void> submitted;
+			Future<Void> b = doBackup(0, &backupAgent, cx, backupTag, backupRanges, stopDifferentialAfter, submitted);
 
-			if (self->abortAndRestartAfter) {
+			if (abortAndRestartAfter) {
 				TraceEvent("BARW_DoBackup2", randomID)
-				    .detail("Tag", printable(self->backupTag))
-				    .detail("AbortWait", self->abortAndRestartAfter);
-				wait(submitted.getFuture());
-				b = b && doBackup(self,
-				                  self->abortAndRestartAfter,
+				    .detail("Tag", printable(backupTag))
+				    .detail("AbortWait", abortAndRestartAfter);
+				co_await submitted.getFuture();
+				b = b && doBackup(abortAndRestartAfter,
 				                  &backupAgent,
 				                  cx,
-				                  self->backupTag,
-				                  self->backupRanges,
-				                  self->stopDifferentialAfter,
+				                  backupTag,
+				                  backupRanges,
+				                  stopDifferentialAfter,
 				                  Promise<Void>());
 			}
 
 			TraceEvent("BARW_DoBackupWait", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 			try {
-				wait(b);
+				co_await b;
 			} catch (Error& e) {
 				if (e.code() != error_code_database_locked)
 					throw;
-				if (self->performRestore)
+				if (performRestore)
 					throw;
-				return Void();
+				co_return;
 			}
 			TraceEvent("BARW_DoBackupDone", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 
-			state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
-			UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx.getReference()));
-			state UID logUid = uidFlag.first;
-			state Key destUidValue = wait(BackupConfig(logUid).destUidValue().getD(cx.getReference()));
-			state Reference<IBackupContainer> lastBackupContainer =
-			    wait(BackupConfig(logUid).backupContainer().getD(cx.getReference()));
+			KeyBackedTag keyBackedTag = makeBackupTag(backupTag.toString());
+			UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
+			UID logUid = uidFlag.first;
+			Key destUidValue = co_await BackupConfig(logUid).destUidValue().getD(cx.getReference());
+			Reference<IBackupContainer> lastBackupContainer =
+			    co_await BackupConfig(logUid).backupContainer().getD(cx.getReference());
 
 			// Occasionally start yet another backup that might still be running when we restore
-			if (!self->locked && BUGGIFY) {
-				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
+			if (!locked && BUGGIFY) {
+				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(backupTag));
 				try {
 					extraBackup = backupAgent.submitBackup(cx,
 					                                       "file://simfdb/backups/"_sr,
 					                                       {},
 					                                       deterministicRandom()->randomInt(0, 60),
 					                                       deterministicRandom()->randomInt(0, 100),
-					                                       self->backupTag.toString(),
-					                                       self->backupRanges,
+					                                       backupTag.toString(),
+					                                       backupRanges,
 					                                       StopWhenDone::True);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)
-					    .detail("BackupTag", printable(self->backupTag));
+					    .detail("BackupTag", printable(backupTag));
 					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 						throw;
 				}
 			}
 
 			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
-			wait(startRestore);
+			co_await startRestore;
 
-			if (lastBackupContainer && self->performRestore) {
+			if (lastBackupContainer && performRestore) {
 				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(),
 				                                                 lastBackupContainer->getProxy(),
-				                                                 lastBackupContainer->getEncryptionKeyFileName());
-				state BackupDescription desc = wait(container->describeBackup());
+				                                                 lastBackupContainer->getEncryptionKeyFileName(),
+				                                                 lastBackupContainer->getEncryptionBlockSize());
+				BackupDescription desc = co_await container->describeBackup();
 
 				if (deterministicRandom()->random01() < 0.5) {
-					wait(attemptDirtyRestore(self,
-					                         cx,
-					                         &backupAgent,
-					                         StringRef(lastBackupContainer->getURL()),
-					                         randomID,
-					                         lastBackupContainer->getEncryptionKeyFileName()));
+					co_await attemptDirtyRestore(cx,
+					                             &backupAgent,
+					                             StringRef(lastBackupContainer->getURL()),
+					                             randomID,
+					                             lastBackupContainer->getEncryptionKeyFileName());
 				}
 
-				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					for (auto& kvrange : self->backupRanges)
+					for (auto& kvrange : backupRanges)
 						tr->clear(kvrange);
 					return Void();
-				}));
+				});
 
 				// restore database
 				TraceEvent("BARW_Restore", randomID)
 				    .detail("LastBackupContainer", lastBackupContainer->getURL())
-				    .detail("RestoreAfter", self->restoreAfter)
-				    .detail("BackupTag", printable(self->backupTag));
+				    .detail("RestoreAfter", restoreAfter)
+				    .detail("BackupTag", printable(backupTag));
 
-				state Version targetVersion = -1;
+				Version targetVersion = -1;
 				if (desc.maxRestorableVersion.present()) {
 					if (deterministicRandom()->random01() < 0.1) {
 						targetVersion = desc.minRestorableVersion.get();
@@ -672,20 +668,20 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 				TraceEvent("BARW_RestoreDebug").detail("TargetVersion", targetVersion);
 
-				state std::vector<Future<Version>> restores;
-				state std::vector<Standalone<StringRef>> restoreTags;
-				state bool multipleRangesInOneTag = false;
-				state int restoreIndex = 0;
+				std::vector<Future<Version>> restores;
+				std::vector<Standalone<StringRef>> restoreTags;
+				bool multipleRangesInOneTag = false;
+				int restoreIndex = 0;
 				// make sure system keys are not present in the restoreRanges as they will get restored first separately
 				// from the rest
 				Standalone<VectorRef<KeyRangeRef>> modifiedRestoreRanges;
 				Standalone<VectorRef<KeyRangeRef>> systemRestoreRanges;
-				for (int i = 0; i < self->restoreRanges.size(); ++i) {
-					if (!self->restoreRanges[i].intersects(getSystemBackupRanges())) {
-						modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), self->restoreRanges[i]);
+				for (int i = 0; i < restoreRanges.size(); ++i) {
+					if (!restoreRanges[i].intersects(getSystemBackupRanges())) {
+						modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), restoreRanges[i]);
 					} else {
-						KeyRangeRef normalKeyRange = self->restoreRanges[i] & normalKeys;
-						KeyRangeRef systemKeyRange = self->restoreRanges[i] & systemKeys;
+						KeyRangeRef normalKeyRange = restoreRanges[i] & normalKeys;
+						KeyRangeRef systemKeyRange = restoreRanges[i] & systemKeys;
 						if (!normalKeyRange.empty()) {
 							modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), normalKeyRange);
 						}
@@ -694,18 +690,17 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						}
 					}
 				}
-				self->restoreRanges = modifiedRestoreRanges;
+				restoreRanges = modifiedRestoreRanges;
 				if (!systemRestoreRanges.empty()) {
 					// We are able to restore system keys first since we restore an entire cluster at once rather than
 					// partial key ranges.
-					wait(clearAndRestoreSystemKeys(
-					    cx, self, &backupAgent, targetVersion, lastBackupContainer, systemRestoreRanges));
+					co_await clearAndRestoreSystemKeys(
+					    cx, this, &backupAgent, targetVersion, lastBackupContainer, systemRestoreRanges);
 				}
 				if (deterministicRandom()->random01() < 0.5) {
-					for (restoreIndex = 0; restoreIndex < self->restoreRanges.size(); restoreIndex++) {
-						auto range = self->restoreRanges[restoreIndex];
-						Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" +
-						                                 std::to_string(restoreIndex));
+					for (restoreIndex = 0; restoreIndex < restoreRanges.size(); restoreIndex++) {
+						auto range = restoreRanges[restoreIndex];
+						Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
 						restoreTags.push_back(restoreTag);
 						printf("BackupCorrectness, restore for each range: backupAgent.restore is called for "
 						       "restoreIndex:%d tag:%s ranges:%s\n",
@@ -723,7 +718,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						                                       range,
 						                                       Key(),
 						                                       Key(),
-						                                       self->locked,
+						                                       locked,
 						                                       OnlyApplyMutationLogs::False,
 						                                       InconsistentSnapshotOnly::False,
 						                                       ::invalidVersion,
@@ -731,7 +726,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 					}
 				} else {
 					multipleRangesInOneTag = true;
-					Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
+					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
 					restoreTags.push_back(restoreTag);
 					printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d tag:%s\n",
 					       restoreIndex,
@@ -741,13 +736,13 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 					                                       restoreTag,
 					                                       KeyRef(lastBackupContainer->getURL()),
 					                                       lastBackupContainer->getProxy(),
-					                                       self->restoreRanges,
+					                                       restoreRanges,
 					                                       WaitForComplete::True,
 					                                       targetVersion,
 					                                       Verbose::True,
 					                                       Key(),
 					                                       Key(),
-					                                       self->locked,
+					                                       locked,
 					                                       UnlockDB::True,
 					                                       OnlyApplyMutationLogs::False,
 					                                       InconsistentSnapshotOnly::False,
@@ -757,31 +752,32 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 				// Sometimes kill and restart the restore
 				if (BUGGIFY) {
-					wait(delay(deterministicRandom()->randomInt(0, 10)));
+					co_await delay(deterministicRandom()->randomInt(0, 10));
 					if (multipleRangesInOneTag) {
-						FileBackupAgent::ERestoreState rs = wait(backupAgent.abortRestore(cx, restoreTags[0]));
+						FileBackupAgent::ERestoreState rs = co_await backupAgent.abortRestore(cx, restoreTags[0]);
 						// The restore may have already completed, or the abort may have been done before the restore
 						// was even able to start.  Only run a new restore if the previous one was actually aborted.
 						if (rs == FileBackupAgent::ERestoreState::ABORTED) {
-							wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-								tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-								for (auto& range : self->restoreRanges)
-									tr->clear(range);
-								return Void();
-							}));
+							co_await runRYWTransaction(cx,
+							                           [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+								                           tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+								                           for (auto& range : restoreRanges)
+									                           tr->clear(range);
+								                           return Void();
+							                           });
 							restores[restoreIndex] =
 							    backupAgent.restore(cx,
 							                        cx,
 							                        restoreTags[restoreIndex],
 							                        KeyRef(lastBackupContainer->getURL()),
 							                        lastBackupContainer->getProxy(),
-							                        self->restoreRanges,
+							                        restoreRanges,
 							                        WaitForComplete::True,
 							                        ::invalidVersion,
 							                        Verbose::True,
 							                        Key(),
 							                        Key(),
-							                        self->locked,
+							                        locked,
 							                        UnlockDB::True,
 							                        OnlyApplyMutationLogs::False,
 							                        InconsistentSnapshotOnly::False,
@@ -791,17 +787,17 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 					} else {
 						for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) {
 							FileBackupAgent::ERestoreState rs =
-							    wait(backupAgent.abortRestore(cx, restoreTags[restoreIndex]));
+							    co_await backupAgent.abortRestore(cx, restoreTags[restoreIndex]);
 							// The restore may have already completed, or the abort may have been done before the
 							// restore was even able to start.  Only run a new restore if the previous one was actually
 							// aborted.
 							if (rs == FileBackupAgent::ERestoreState::ABORTED) {
-								wait(
-								    runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+								co_await runRYWTransaction(
+								    cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 									    tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-									    tr->clear(self->restoreRanges[restoreIndex]);
+									    tr->clear(restoreRanges[restoreIndex]);
 									    return Void();
-								    }));
+								    });
 								restores[restoreIndex] =
 								    backupAgent.restore(cx,
 								                        cx,
@@ -811,10 +807,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 								                        WaitForComplete::True,
 								                        ::invalidVersion,
 								                        Verbose::True,
-								                        self->restoreRanges[restoreIndex],
+								                        restoreRanges[restoreIndex],
 								                        Key(),
 								                        Key(),
-								                        self->locked,
+								                        locked,
 								                        OnlyApplyMutationLogs::False,
 								                        InconsistentSnapshotOnly::False,
 								                        ::invalidVersion,
@@ -824,7 +820,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 					}
 				}
 
-				wait(waitForAll(restores));
+				co_await waitForAll(restores);
 
 				for (auto& restore : restores) {
 					ASSERT(!restore.isError());
@@ -832,20 +828,20 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 			}
 
 			if (extraBackup.isValid()) {
-				TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(backupTag));
 				try {
-					wait(extraBackup);
+					co_await extraBackup;
 				} catch (Error& e) {
 					TraceEvent("BARW_ExtraBackupException", randomID)
 					    .error(e)
-					    .detail("BackupTag", printable(self->backupTag));
+					    .detail("BackupTag", printable(backupTag));
 					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 						throw;
 				}
 
-				TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(backupTag));
 				try {
-					wait(backupAgent.abortBackup(cx, self->backupTag.toString()));
+					co_await backupAgent.abortBackup(cx, backupTag.toString());
 				} catch (Error& e) {
 					TraceEvent("BARW_AbortBackupExtraException", randomID).error(e);
 					if (e.code() != error_code_backup_unneeded)
@@ -853,40 +849,41 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 				}
 			}
 
-			state Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
-			state Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
-			state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
-			state Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
-			state int displaySystemKeys = 0;
+			Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
+			Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
+			Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
+			Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
+			int displaySystemKeys = 0;
 
 			// Ensure that there is no left over key within the backup subspace
-			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+			while (true) {
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-				TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(backupTag));
 
+				Error err;
 				try {
 					// Check the left over tasks
 					// We have to wait for the list to empty since an abort and get status
 					// can leave extra tasks in the queue
-					TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(self->backupTag));
-					state int64_t taskCount = wait(backupAgent.getTaskCount(tr));
-					state int waitCycles = 0;
+					TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(backupTag));
+					int64_t taskCount = co_await backupAgent.getTaskCount(tr);
+					int waitCycles = 0;
 
 					if ((taskCount) && false) {
 						TraceEvent("BARW_EndingNonzeroTaskCount", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("TaskCount", taskCount)
 						    .detail("WaitCycles", waitCycles);
 						printf("EndingNonZeroTasks: %ld\n", (long)taskCount);
-						wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
+						co_await TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef());
 					}
 
 					while (taskCount > 0) {
 						waitCycles++;
 
 						TraceEvent("BARW_NonzeroTaskWait", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("TaskCount", taskCount)
 						    .detail("WaitCycles", waitCycles);
 						printf("%.6f %-10s Wait #%4d for %lld tasks to end\n",
@@ -895,23 +892,23 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						       waitCycles,
 						       (long long)taskCount);
 
-						wait(delay(5.0));
+						co_await delay(5.0);
 
 						tr = makeReference<ReadYourWritesTransaction>(cx);
-						wait(store(taskCount, backupAgent.getTaskCount(tr)));
+						taskCount = co_await backupAgent.getTaskCount(tr);
 					}
 
 					RangeResult agentValues =
-					    wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
+					    co_await tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100);
 
 					// Error if the system keyspace for the backup tag is not empty
-					if (agentValues.size() > 0) {
+					if (!agentValues.empty()) {
 						displaySystemKeys++;
 						printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n",
 						       agentValues.size(),
 						       printable(backupAgentKey).c_str());
 						TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("LeftOverKeys", agentValues.size())
 						    .detail("KeySpace", printable(backupAgentKey));
 						for (auto& s : agentValues) {
@@ -926,30 +923,30 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 						printf("No left over backup agent configuration keys\n");
 					}
 
-					Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
+					Optional<Value> latestVersion = co_await tr->get(backupLatestVersionsKey);
 					if (latestVersion.present()) {
 						TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("BackupLatestVersionsKey", backupLatestVersionsKey.printable())
 						    .detail("DestUidValue", destUidValue.printable());
 					} else {
 						printf("No left over backup version key\n");
 					}
 
-					RangeResult versions = wait(tr->getRange(
-					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1));
-					if (!self->shareLogRange || !versions.size()) {
-						RangeResult logValues = wait(
-						    tr->getRange(KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100));
+					RangeResult versions = co_await tr->getRange(
+					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1);
+					if (!shareLogRange || versions.empty()) {
+						RangeResult logValues = co_await tr->getRange(
+						    KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100);
 
 						// Error if the log/mutation keyspace for the backup tag  is not empty
-						if (logValues.size() > 0) {
+						if (!logValues.empty()) {
 							displaySystemKeys++;
 							printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n",
 							       logValues.size(),
 							       printable(backupLogValuesKey).c_str());
 							TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID)
-							    .detail("BackupTag", printable(self->backupTag))
+							    .detail("BackupTag", printable(backupTag))
 							    .detail("LeftOverKeys", logValues.size())
 							    .detail("KeySpace", printable(backupLogValuesKey));
 						} else {
@@ -959,32 +956,32 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 
 					break;
 				} catch (Error& e) {
-					TraceEvent("BARW_CheckException", randomID).error(e);
-					wait(tr->onError(e));
+					err = e;
 				}
+				TraceEvent("BARW_CheckException", randomID).error(err);
+				co_await tr->onError(err);
 			}
 
 			if (displaySystemKeys) {
-				wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
+				co_await TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef());
 			}
 
-			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
+			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(backupTag));
 
 			// Decrement the backup agent requests
-			if (self->agentRequest) {
+			if (agentRequest) {
 				BackupAndRestoreCorrectnessWorkload::backupAgentRequests--;
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
+			if ((fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::BackupToFile) &&
 			    (!BackupAndRestoreCorrectnessWorkload::backupAgentRequests)) {
-				g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				fdbSimulationPolicyState().backupAgents = FDBBackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestoreCorrectness").error(e).GetLastError();
 			throw;
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BackupCorrectnessPartitioned.actor.cpp b/fdbserver/workloads/BackupCorrectnessPartitioned.cpp
similarity index 66%
rename from fdbserver/workloads/BackupCorrectnessPartitioned.actor.cpp
rename to fdbserver/workloads/BackupCorrectnessPartitioned.cpp
index d2e40be2da7..5e67fa0038e 100644
--- a/fdbserver/workloads/BackupCorrectnessPartitioned.actor.cpp
+++ b/fdbserver/workloads/BackupCorrectnessPartitioned.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupCorrectness.actor.cpp
+ * BackupCorrectnessPartitioned.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,17 +19,17 @@
  */
 
 #include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
+#include "BulkSetup.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // A workload for
 // https://github.com/apple/foundationdb/blob/main/design/backup_v2_partitioned_logs.md
@@ -57,7 +57,7 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 	bool defaultBackup;
 	Optional<std::string> encryptionKeyFileName;
 
-	BackupAndRestorePartitionedCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit BackupAndRestorePartitionedCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
 		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
@@ -184,7 +184,7 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 		return _setup(cx, this);
 	}
 
-	ACTOR Future<Void> _setup(Database cx, BackupAndRestorePartitionedCorrectnessWorkload* self) {
+	Future<Void> _setup(Database cx, BackupAndRestorePartitionedCorrectnessWorkload* self) {
 		if (BUGGIFY) {
 			for (auto r : getSystemBackupRanges()) {
 				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
@@ -215,75 +215,75 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 		    .detail("AgentRequest", agentRequest)
 		    .detail("Encrypted", encryptionKeyFileName.present());
 
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override {
 		if (clientId != 0)
 			return true;
 		else
-			return _check(cx, this);
+			return _check(cx);
 	}
 
-	ACTOR static Future<bool> _check(Database cx, BackupAndRestorePartitionedCorrectnessWorkload* self) {
-		state Transaction tr(cx);
-		loop {
+	Future<bool> _check(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				state int restoreIndex;
-				for (restoreIndex = 0; restoreIndex < self->skippedRestoreRanges.size(); restoreIndex++) {
-					state KeyRangeRef range = self->skippedRestoreRanges[restoreIndex];
-					Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
-					RangeResult res = wait(tr.getRange(range, GetRangeLimits::ROW_LIMIT_UNLIMITED));
+				for (int restoreIndex = 0; restoreIndex < skippedRestoreRanges.size(); restoreIndex++) {
+					KeyRangeRef range = skippedRestoreRanges[restoreIndex];
+					Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
+					RangeResult res = co_await tr.getRange(range, GetRangeLimits::ROW_LIMIT_UNLIMITED);
 					if (!res.empty()) {
 						TraceEvent(SevError, "BARW_UnexpectedRangePresent").detail("Range", printable(range));
-						return false;
+						co_return false;
 					}
 				}
 				TraceEvent("BackupCorrectnessFinish").log();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return true;
+		co_return true;
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
-		loop {
-			wait(backupAgent->changePause(cx, true));
-			wait(delay(30 * deterministicRandom()->random01()));
-			wait(backupAgent->changePause(cx, false));
-			wait(delay(120 * deterministicRandom()->random01()));
+	static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
+		while (true) {
+			co_await backupAgent->changePause(cx, true);
+			co_await delay(30 * deterministicRandom()->random01());
+			co_await backupAgent->changePause(cx, false);
+			co_await delay(120 * deterministicRandom()->random01());
 		}
 	}
 
-	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
-		state FileBackupAgent agent;
-		loop {
-			bool active = wait(agent.checkActive(cx));
+	static Future<Void> statusLoop(Database cx, std::string tag) {
+		FileBackupAgent agent;
+		while (true) {
+			bool active = co_await agent.checkActive(cx);
 			TraceEvent("BARW_AgentActivityCheck").detail("IsActive", active);
-			std::string status = wait(agent.getStatus(cx, ShowErrors::True, tag));
+			std::string status = co_await agent.getStatus(cx, ShowErrors::True, tag);
 			puts(status.c_str());
-			std::string statusJSON = wait(agent.getStatusJSON(cx, tag));
+			std::string statusJSON = co_await agent.getStatusJSON(cx, tag);
 			puts(statusJSON.c_str());
-			wait(delay(2.0));
+			co_await delay(2.0);
 		}
 	}
 
-	ACTOR static Future<Void> doBackup(BackupAndRestorePartitionedCorrectnessWorkload* self,
-	                                   double startDelay,
-	                                   FileBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   double stopDifferentialDelay) {
+	Future<Void> doBackup(double startDelay,
+	                      FileBackupAgent* backupAgent,
+	                      Database cx,
+	                      Key tag,
+	                      Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                      double stopDifferentialDelay) {
 
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 
-		state Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
-		wait(delay(startDelay));
+		Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
+		co_await delay(startDelay);
 
 		if (startDelay || BUGGIFY) {
 			TraceEvent("BARW_DoBackupAbortBackup1", randomID)
@@ -291,7 +291,7 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 			    .detail("StartDelay", startDelay);
 
 			try {
-				wait(backupAgent->abortBackup(cx, tag.toString()));
+				co_await backupAgent->abortBackup(cx, tag.toString());
 			} catch (Error& e) {
 				TraceEvent("BARW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
 				if (e.code() != error_code_backup_unneeded)
@@ -303,20 +303,22 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 		    .detail("Tag", printable(tag))
 		    .detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
 
-		state std::string backupContainer = "file://simfdb/backups/";
-		state Future<Void> status = statusLoop(cx, tag.toString());
+		std::string backupContainer = "file://simfdb/backups/";
+		Future<Void> status = statusLoop(cx, tag.toString());
 		try {
-			wait(backupAgent->submitBackup(cx,
-			                               StringRef(backupContainer),
-			                               {},
-			                               deterministicRandom()->randomInt(0, 60),
-			                               deterministicRandom()->randomInt(0, 2000),
-			                               tag.toString(),
-			                               backupRanges,
-			                               StopWhenDone{ !stopDifferentialDelay },
-			                               UsePartitionedLog::True, // enable partitioned log here
-			                               IncrementalBackupOnly::False,
-			                               self->encryptionKeyFileName));
+			co_await backupAgent->submitBackup(cx,
+			                                   StringRef(backupContainer),
+			                                   {},
+			                                   deterministicRandom()->randomInt(0, 60),
+			                                   deterministicRandom()->randomInt(0, 2000),
+			                                   tag.toString(),
+			                                   backupRanges,
+			                                   StopWhenDone{ !stopDifferentialDelay },
+			                                   MutationLogType::PARTITIONED_LOG, // enable partitioned log here
+			                                   IncrementalBackupOnly::False,
+			                                   encryptionKeyFileName,
+			                                   encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+			                                   /*snapshotMode=*/0);
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@@ -327,34 +329,34 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 		if (stopDifferentialDelay) {
 			CODE_PROBE(!stopDifferentialFuture.isReady(),
 			           "Restore starts at specified time - stopDifferential not ready");
-			wait(stopDifferentialFuture);
+			co_await stopDifferentialFuture;
 			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
 			    .detail("DifferentialAfter", stopDifferentialDelay);
 
 			try {
 				if (BUGGIFY) {
-					state KeyBackedTag backupTag = makeBackupTag(tag.toString());
+					KeyBackedTag backupTag = makeBackupTag(tag.toString());
 					TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName);
 
 					// Wait until the backup is in a restorable state and get the status, URL, and UID atomically
-					state Reference<IBackupContainer> lastBackupContainer;
-					state UID lastBackupUID;
-					state EBackupState resultWait = wait(backupAgent->waitBackup(
-					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID));
+					Reference<IBackupContainer> lastBackupContainer;
+					UID lastBackupUID;
+					EBackupState resultWait = co_await backupAgent->waitBackup(
+					    cx, backupTag.tagName, StopWhenDone::False, &lastBackupContainer, &lastBackupUID);
 
 					TraceEvent("BARW_DoBackupWaitForRestorable", randomID)
 					    .detail("Tag", backupTag.tagName)
 					    .detail("Result", BackupAgentBase::getStateText(resultWait));
 
-					state bool restorable = false;
+					bool restorable = false;
 					if (lastBackupContainer) {
-						state Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
-						wait(ready(fdesc));
+						Future<BackupDescription> fdesc = lastBackupContainer->describeBackup();
+						co_await ready(fdesc);
 
 						if (!fdesc.isError()) {
-							state BackupDescription desc = fdesc.get();
-							wait(desc.resolveVersionTimes(cx));
+							BackupDescription desc = fdesc.get();
+							co_await desc.resolveVersionTimes(cx);
 							printf("BackupDescription:\n%s\n", desc.toString().c_str());
 							restorable = desc.maxRestorableVersion.present();
 						}
@@ -398,12 +400,12 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 						    .detail("WaitStatus", BackupAgentBase::getStateText(resultWait))
 						    .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
 						    .detail("Restorable", restorable);
-						wait(backupAgent->abortBackup(cx, tag.toString()));
+						co_await backupAgent->abortBackup(cx, tag.toString());
 					} else {
 						TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
 						    .detail("Tag", printable(tag))
 						    .detail("DifferentialAfter", stopDifferentialDelay);
-						wait(backupAgent->discontinueBackup(cx, tag));
+						co_await backupAgent->discontinueBackup(cx, tag);
 					}
 				}
 
@@ -411,7 +413,7 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 					TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
 					    .detail("Tag", printable(tag))
 					    .detail("DifferentialAfter", stopDifferentialDelay);
-					wait(backupAgent->discontinueBackup(cx, tag));
+					co_await backupAgent->discontinueBackup(cx, tag);
 				}
 			} catch (Error& e) {
 				TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).error(e).detail("Tag", printable(tag));
@@ -422,11 +424,11 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 
 		// Wait for the backup to complete
 		TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
-		state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True));
+		EBackupState statusValue = co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True);
 
-		state std::string statusText;
+		std::string statusText;
 
-		std::string _statusText = wait(backupAgent->getStatus(cx, ShowErrors::True, tag.toString()));
+		std::string _statusText = co_await backupAgent->getStatus(cx, ShowErrors::True, tag.toString());
 		statusText = _statusText;
 		// Can we validate anything about status?
 
@@ -434,114 +436,111 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 		    .detail("Tag", printable(tag))
 		    .detail("Status", statusText)
 		    .detail("StatusValue", BackupAgentBase::getStateText(statusValue));
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> clearAndRestoreSystemKeys(Database cx,
-	                                                    BackupAndRestorePartitionedCorrectnessWorkload* self,
-	                                                    FileBackupAgent* backupAgent,
-	                                                    Version targetVersion,
-	                                                    Reference<IBackupContainer> lastBackupContainer,
-	                                                    Standalone<VectorRef<KeyRangeRef>> systemRestoreRanges) {
+	static Future<Void> clearAndRestoreSystemKeys(Database cx,
+	                                              BackupAndRestorePartitionedCorrectnessWorkload* self,
+	                                              FileBackupAgent* backupAgent,
+	                                              Version targetVersion,
+	                                              Reference<IBackupContainer> lastBackupContainer,
+	                                              Standalone<VectorRef<KeyRangeRef>> systemRestoreRanges) {
 		// restore system keys before restoring any other ranges
-		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+		co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			for (auto& range : systemRestoreRanges) {
 				tr->clear(range);
 			}
 			return Void();
-		}));
-		state Standalone<StringRef> restoreTag(self->backupTag.toString() + "_system");
+		});
+		Standalone<StringRef> restoreTag(self->backupTag.toString() + "_system");
 		printf("BackupCorrectness, backupAgent.restore is called for tag:%s\n", restoreTag.toString().c_str());
-		wait(success(backupAgent->restore(cx,
-		                                  cx,
-		                                  restoreTag,
-		                                  KeyRef(lastBackupContainer->getURL()),
-		                                  lastBackupContainer->getProxy(),
-		                                  systemRestoreRanges,
-		                                  WaitForComplete::True,
-		                                  targetVersion,
-		                                  Verbose::True,
-		                                  Key(),
-		                                  Key(),
-		                                  self->locked,
-		                                  UnlockDB::True,
-		                                  OnlyApplyMutationLogs::False,
-		                                  InconsistentSnapshotOnly::False,
-		                                  ::invalidVersion,
-		                                  self->encryptionKeyFileName)));
+		co_await backupAgent->restore(cx,
+		                              cx,
+		                              restoreTag,
+		                              KeyRef(lastBackupContainer->getURL()),
+		                              lastBackupContainer->getProxy(),
+		                              systemRestoreRanges,
+		                              WaitForComplete::True,
+		                              targetVersion,
+		                              Verbose::True,
+		                              Key(),
+		                              Key(),
+		                              self->locked,
+		                              UnlockDB::True,
+		                              OnlyApplyMutationLogs::False,
+		                              InconsistentSnapshotOnly::False,
+		                              ::invalidVersion,
+		                              self->encryptionKeyFileName);
 		printf("BackupCorrectness, backupAgent.restore finished for tag:%s\n", restoreTag.toString().c_str());
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, BackupAndRestorePartitionedCorrectnessWorkload* self) {
-		state FileBackupAgent backupAgent;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> _start(Database cx) {
+		FileBackupAgent backupAgent;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		TraceEvent("BARW_Arguments")
-		    .detail("BackupTag", printable(self->backupTag))
-		    .detail("PerformRestore", self->performRestore)
-		    .detail("BackupAfter", self->backupAfter)
-		    .detail("RestoreAfter", self->restoreAfter)
-		    .detail("AbortAndRestartAfter", self->abortAndRestartAfter)
-		    .detail("DifferentialAfter", self->stopDifferentialAfter);
-
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
-		if (self->allowPauses && BUGGIFY) {
-			state Future<Void> cp = changePaused(cx, &backupAgent);
+		    .detail("BackupTag", printable(backupTag))
+		    .detail("PerformRestore", performRestore)
+		    .detail("BackupAfter", backupAfter)
+		    .detail("RestoreAfter", restoreAfter)
+		    .detail("AbortAndRestartAfter", abortAndRestartAfter)
+		    .detail("DifferentialAfter", stopDifferentialAfter);
+
+		UID randomID = nondeterministicRandom()->randomUniqueID();
+		if (allowPauses && BUGGIFY) {
+			Future<Void> cp = changePaused(cx, &backupAgent);
 		}
 
 		// Increment the backup agent requests
-		if (self->agentRequest) {
+		if (agentRequest) {
 			BackupAndRestorePartitionedCorrectnessWorkload::backupAgentRequests++;
 		}
 
-		if (self->encryptionKeyFileName.present()) {
-			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		if (encryptionKeyFileName.present()) {
+			co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 		}
 
 		try {
-			state Future<Void> startRestore = delay(self->restoreAfter);
+			Future<Void> startRestore = delay(restoreAfter);
 
 			// backup
-			wait(delay(self->backupAfter));
+			co_await delay(backupAfter);
 
-			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
-			state Future<Void> b =
-			    doBackup(self, 0, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter);
+			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(backupTag));
+			Future<Void> b = doBackup(0, &backupAgent, cx, backupTag, backupRanges, stopDifferentialAfter);
 
 			TraceEvent("BARW_DoBackupWait", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 			try {
-				wait(b);
+				co_await b;
 			} catch (Error& e) {
 				if (e.code() != error_code_database_locked)
 					throw;
-				if (self->performRestore)
+				if (performRestore)
 					throw;
-				return Void();
+				co_return;
 			}
 			TraceEvent("BARW_DoBackupDone", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 
-			state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
-			UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx.getReference()));
-			state UID logUid = uidFlag.first;
-			state Key destUidValue = wait(BackupConfig(logUid).destUidValue().getD(cx.getReference()));
-			state Reference<IBackupContainer> lastBackupContainer =
-			    wait(BackupConfig(logUid).backupContainer().getD(cx.getReference()));
+			KeyBackedTag keyBackedTag = makeBackupTag(backupTag.toString());
+			UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
+			UID logUid = uidFlag.first;
+			Key destUidValue = co_await BackupConfig(logUid).destUidValue().getD(cx.getReference());
+			Reference<IBackupContainer> lastBackupContainer =
+			    co_await BackupConfig(logUid).backupContainer().getD(cx.getReference());
 
 			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
-			wait(startRestore);
+			co_await startRestore;
 			// between here
-			if (lastBackupContainer && self->performRestore) {
+			if (lastBackupContainer && performRestore) {
 				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(),
 				                                                 lastBackupContainer->getProxy(),
-				                                                 lastBackupContainer->getEncryptionKeyFileName());
-				BackupDescription desc = wait(container->describeBackup());
-				state Version targetVersion = -1;
+				                                                 lastBackupContainer->getEncryptionKeyFileName(),
+				                                                 lastBackupContainer->getEncryptionBlockSize());
+				BackupDescription desc = co_await container->describeBackup();
+				Version targetVersion = -1;
 				if (desc.maxRestorableVersion.present()) {
 					if (deterministicRandom()->random01() < 0.1) {
 						targetVersion = desc.minRestorableVersion.get();
@@ -554,35 +553,35 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 						                    : desc.maxRestorableVersion.get();
 					}
 				}
-				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					for (auto& kvrange : self->backupRanges) {
+					for (auto& kvrange : backupRanges) {
 						// version needs to be decided before this transaction otherwise
 						// this clear mutation might be backup as well
 						tr->clear(kvrange);
 					}
 					return Void();
-				}));
+				});
 
 				// restore database
 				TraceEvent("BARW_Restore", randomID)
 				    .detail("LastBackupContainer", lastBackupContainer->getURL())
-				    .detail("RestoreAfter", self->restoreAfter)
-				    .detail("BackupTag", printable(self->backupTag))
+				    .detail("RestoreAfter", restoreAfter)
+				    .detail("BackupTag", printable(backupTag))
 				    .detail("TargetVersion", targetVersion);
-				state std::vector<Future<Version>> restores;
-				state std::vector<Standalone<StringRef>> restoreTags;
-				state int restoreIndex = 0;
+				std::vector<Future<Version>> restores;
+				std::vector<Standalone<StringRef>> restoreTags;
+				int restoreIndex = 0;
 				// make sure system keys are not present in the restoreRanges as they will get restored first
 				// separately from the rest
 				Standalone<VectorRef<KeyRangeRef>> modifiedRestoreRanges;
 				Standalone<VectorRef<KeyRangeRef>> systemRestoreRanges;
-				for (int i = 0; i < self->restoreRanges.size(); ++i) {
-					if (!self->restoreRanges[i].intersects(getSystemBackupRanges())) {
-						modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), self->restoreRanges[i]);
+				for (int i = 0; i < restoreRanges.size(); ++i) {
+					if (!restoreRanges[i].intersects(getSystemBackupRanges())) {
+						modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), restoreRanges[i]);
 					} else {
-						KeyRangeRef normalKeyRange = self->restoreRanges[i] & normalKeys;
-						KeyRangeRef systemKeyRange = self->restoreRanges[i] & systemKeys;
+						KeyRangeRef normalKeyRange = restoreRanges[i] & normalKeys;
+						KeyRangeRef systemKeyRange = restoreRanges[i] & systemKeys;
 						if (!normalKeyRange.empty()) {
 							modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), normalKeyRange);
 						}
@@ -591,13 +590,13 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 						}
 					}
 				}
-				self->restoreRanges = modifiedRestoreRanges;
+				restoreRanges = modifiedRestoreRanges;
 				if (!systemRestoreRanges.empty()) {
-					wait(clearAndRestoreSystemKeys(
-					    cx, self, &backupAgent, targetVersion, lastBackupContainer, systemRestoreRanges));
+					co_await clearAndRestoreSystemKeys(
+					    cx, this, &backupAgent, targetVersion, lastBackupContainer, systemRestoreRanges);
 				}
 
-				Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
+				Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
 				restoreTags.push_back(restoreTag);
 				printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d tag:%s "
 				       "TargetVersion:%" PRId64 "\n",
@@ -609,49 +608,50 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 				                                       restoreTag,
 				                                       KeyRef(lastBackupContainer->getURL()),
 				                                       lastBackupContainer->getProxy(),
-				                                       self->restoreRanges,
+				                                       restoreRanges,
 				                                       WaitForComplete::True,
 				                                       targetVersion,
 				                                       Verbose::True,
 				                                       Key(),
 				                                       Key(),
-				                                       self->locked,
+				                                       locked,
 				                                       UnlockDB::True,
 				                                       OnlyApplyMutationLogs::False,
 				                                       InconsistentSnapshotOnly::False,
 				                                       ::invalidVersion,
-				                                       self->encryptionKeyFileName));
-				wait(waitForAll(restores));
+				                                       encryptionKeyFileName));
+				co_await waitForAll(restores);
 
 				for (auto& restore : restores) {
 					ASSERT(!restore.isError());
 				}
 			}
-			state Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
-			state Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
-			state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
-			state Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
-			state int displaySystemKeys = 0;
+			Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
+			Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
+			Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
+			Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
+			int displaySystemKeys = 0;
 
 			// Ensure that there is no left over key within the backup subspace
-			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+			while (true) {
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-				TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(backupTag));
 
+				Error err;
 				try {
 					// Check the left over tasks
 					// We have to wait for the list to empty since an abort and get status
 					// can leave extra tasks in the queue
-					TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(self->backupTag));
-					state int64_t taskCount = wait(backupAgent.getTaskCount(tr));
-					state int waitCycles = 0;
+					TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(backupTag));
+					int64_t taskCount = co_await backupAgent.getTaskCount(tr);
+					int waitCycles = 0;
 
 					while (taskCount > 0) {
 						waitCycles++;
 
 						TraceEvent("BARW_NonzeroTaskWait", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("TaskCount", taskCount)
 						    .detail("WaitCycles", waitCycles);
 						printf("%.6f %-10s Wait #%4d for %lld tasks to end\n",
@@ -660,23 +660,23 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 						       waitCycles,
 						       (long long)taskCount);
 
-						wait(delay(5.0));
+						co_await delay(5.0);
 
 						tr = makeReference<ReadYourWritesTransaction>(cx);
-						wait(store(taskCount, backupAgent.getTaskCount(tr)));
+						taskCount = co_await backupAgent.getTaskCount(tr);
 					}
 
 					RangeResult agentValues =
-					    wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
+					    co_await tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100);
 
 					// Error if the system keyspace for the backup tag is not empty
-					if (agentValues.size() > 0) {
+					if (!agentValues.empty()) {
 						displaySystemKeys++;
 						printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n",
 						       agentValues.size(),
 						       printable(backupAgentKey).c_str());
 						TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("LeftOverKeys", agentValues.size())
 						    .detail("KeySpace", printable(backupAgentKey));
 						for (auto& s : agentValues) {
@@ -691,30 +691,30 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 						printf("No left over backup agent configuration keys\n");
 					}
 
-					Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
+					Optional<Value> latestVersion = co_await tr->get(backupLatestVersionsKey);
 					if (latestVersion.present()) {
 						TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("BackupLatestVersionsKey", backupLatestVersionsKey.printable())
 						    .detail("DestUidValue", destUidValue.printable());
 					} else {
 						printf("No left over backup version key\n");
 					}
 
-					RangeResult versions = wait(tr->getRange(
-					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1));
-					if (!self->shareLogRange || !versions.size()) {
-						RangeResult logValues = wait(
-						    tr->getRange(KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100));
+					RangeResult versions = co_await tr->getRange(
+					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1);
+					if (!shareLogRange || versions.empty()) {
+						RangeResult logValues = co_await tr->getRange(
+						    KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100);
 
 						// Error if the log/mutation keyspace for the backup tag  is not empty
-						if (logValues.size() > 0) {
+						if (!logValues.empty()) {
 							displaySystemKeys++;
 							printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n",
 							       logValues.size(),
 							       printable(backupLogValuesKey).c_str());
 							TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID)
-							    .detail("BackupTag", printable(self->backupTag))
+							    .detail("BackupTag", printable(backupTag))
 							    .detail("LeftOverKeys", logValues.size())
 							    .detail("KeySpace", printable(backupLogValuesKey));
 						} else {
@@ -724,32 +724,32 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 
 					break;
 				} catch (Error& e) {
-					TraceEvent("BARW_CheckException", randomID).error(e);
-					wait(tr->onError(e));
+					err = e;
 				}
+				TraceEvent("BARW_CheckException", randomID).error(err);
+				co_await tr->onError(err);
 			}
 
 			if (displaySystemKeys) {
-				wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
+				co_await TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef());
 			}
 
-			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
+			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(backupTag));
 
 			// Decrement the backup agent requets
-			if (self->agentRequest) {
+			if (agentRequest) {
 				BackupAndRestorePartitionedCorrectnessWorkload::backupAgentRequests--;
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
+			if ((fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::BackupToFile) &&
 			    (!BackupAndRestorePartitionedCorrectnessWorkload::backupAgentRequests)) {
-				g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				fdbSimulationPolicyState().backupAgents = FDBBackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestorePartitionedCorrectness").error(e).GetLastError();
 			throw;
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BackupS3BlobCorrectness.actor.cpp b/fdbserver/workloads/BackupS3BlobCorrectness.cpp
similarity index 67%
rename from fdbserver/workloads/BackupS3BlobCorrectness.actor.cpp
rename to fdbserver/workloads/BackupS3BlobCorrectness.cpp
index 5c039a187f9..b7c3f4cc4e4 100644
--- a/fdbserver/workloads/BackupS3BlobCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupS3BlobCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupS3BlobCorrectness.actor.cpp
+ * BackupS3BlobCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -80,21 +80,21 @@
 
 #include <atomic>
 #include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbserver/MockS3Server.h"
-#include "fdbserver/MockS3ServerChaos.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
+#include "BulkSetup.h"
+#include "fdbserver/mocks3/MockS3Server.h"
+#include "fdbserver/mocks3/MockS3ServerChaos.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // Counters to verify BulkDump/BulkLoad were actually used
 // These are incremented by trace event handlers in the backup/restore code paths
@@ -146,7 +146,7 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		out.insert({ "RandomRangeLock" });
 	}
 
-	BackupS3BlobCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit BackupS3BlobCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
 		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
@@ -268,7 +268,7 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		return _setup(cx, this);
 	}
 
-	ACTOR Future<Void> _setup(Database cx, BackupS3BlobCorrectnessWorkload* self) {
+	Future<Void> _setup(Database cx, BackupS3BlobCorrectnessWorkload* self) {
 		// S3-specific: Register MockS3Server or MockS3ServerChaos for blobstore URLs in simulation
 		// Only client 0 registers the server to avoid duplicates
 		// Persistence is automatically enabled in registration
@@ -283,7 +283,7 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 
 			if (self->enableChaos) {
 				NetworkAddress listenAddress(IPAddress(0x7f000001), 8080);
-				wait(startMockS3ServerChaos(listenAddress));
+				co_await startMockS3ServerChaos(listenAddress);
 
 				// Configure chaos rates
 				auto injector = S3FaultInjector::injector();
@@ -302,7 +302,7 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 				    .detail("CorruptionRate", self->corruptionRate)
 				    .detail("MaxDelay", self->maxDelay);
 			} else {
-				wait(registerMockS3Server("127.0.0.1", "8080"));
+				co_await registerMockS3Server("127.0.0.1", "8080");
 				TraceEvent("BS3BCW_RegisteredMockS3")
 				    .detail("Address", "127.0.0.1:8080")
 				    .detail("ClientId", self->clientId);
@@ -312,8 +312,6 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		// Backup everything
 		self->backupRanges.push_back_deep(self->backupRanges.arena(), normalKeys);
 		self->restoreRanges.push_back_deep(self->restoreRanges.arena(), normalKeys);
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -322,50 +320,86 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		if (clientId != 0) {
 			return Void();
 		}
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
-		loop {
-			wait(backupAgent->taskBucket->changePause(cx, deterministicRandom()->coinflip()));
-			wait(delay(30 * deterministicRandom()->random01()));
+	static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
+		while (true) {
+			co_await backupAgent->taskBucket->changePause(cx, deterministicRandom()->coinflip());
+			co_await delay(30 * deterministicRandom()->random01());
 		}
 	}
 
-	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
-		state FileBackupAgent agent;
-		loop {
-			bool active = wait(agent.checkActive(cx));
+	static Future<Void> statusLoop(Database cx, std::string tag) {
+		FileBackupAgent agent;
+		while (true) {
+			bool active = co_await agent.checkActive(cx);
 			TraceEvent("BS3BCW_AgentActivityCheck").detail("IsActive", active);
-			std::string statusText = wait(agent.getStatus(cx, ShowErrors::True, tag));
+			std::string statusText = co_await agent.getStatus(cx, ShowErrors::True, tag);
 			// S3-specific: Suppress backup status output during testing to reduce noise
 			// puts(statusText.c_str());
-			std::string statusJSON = wait(agent.getStatusJSON(cx, tag));
+			std::string statusJSON = co_await agent.getStatusJSON(cx, tag);
 			// puts(statusJSON.c_str());
-			wait(delay(2.0));
+			co_await delay(2.0);
+		}
+	}
+
+	static Future<Void> verifyBulkDumpObservability(Database cx, UID expectedBackupUID, std::string backupTag) {
+		double startTime = now();
+		double maxWaitTime = 30.0;
+
+		TraceEvent("BS3BCW_ObservabilityCheckStart")
+		    .detail("ExpectedBackupUID", expectedBackupUID)
+		    .detail("BackupTag", backupTag);
+
+		loop {
+			Optional<BulkDumpProgress> progressOpt = co_await getBulkDumpProgress(cx);
+
+			if (progressOpt.present()) {
+				BulkDumpProgress progress = progressOpt.get();
+
+				TraceEvent("BS3BCW_ObservabilityProgress")
+				    .detail("JobId", progress.jobId)
+				    .detail("TotalTasks", progress.totalTasks)
+				    .detail("CompleteTasks", progress.completeTasks)
+				    .detail("ProgressPercent", progress.progressPercent())
+				    .detail("TotalBytes", progress.totalBytes)
+				    .detail("CompletedBytes", progress.completedBytes);
+
+				Optional<BulkDumpOwnerInfo> ownerInfo = co_await getBulkDumpOwner(cx, progress.jobId);
+				if (ownerInfo.present() && ownerInfo.get().ownerUID == expectedBackupUID) {
+					TraceEvent("BS3BCW_ObservabilityVerified").detail("BackupUID", expectedBackupUID);
+					co_return;
+				}
+			}
+
+			if (now() - startTime > maxWaitTime) {
+				TraceEvent(SevWarn, "BS3BCW_ObservabilityTimeout");
+				co_return;
+			}
+
+			co_await delay(5.0);
 		}
 	}
 
-	// Wait for a backup to become restorable, with retries
-	// This handles cases where cluster recoveries delay snapshot completion
-	ACTOR static Future<Void> waitForRestorable(Reference<IBackupContainer> backupContainer, int maxAttempts) {
-		state int restorabilityCheckAttempts = 0;
-		state bool isRestorable = false;
-		state int64_t lastSnapshotBytes = 0;
+	static Future<Void> waitForRestorable(Reference<IBackupContainer> backupContainer, int maxAttempts) {
+		int restorabilityCheckAttempts = 0;
+		bool isRestorable = false;
+		int64_t lastSnapshotBytes = 0;
 
 		while (!isRestorable && restorabilityCheckAttempts < maxAttempts) {
-			BackupDescription desc = wait(backupContainer->describeBackup());
+			BackupDescription desc = co_await backupContainer->describeBackup();
 			isRestorable = desc.maxRestorableVersion.present();
 			lastSnapshotBytes = desc.snapshotBytes;
 			if (!isRestorable) {
 				TraceEvent("BS3BCW_WaitingForRestorable")
 				    .detail("Attempt", restorabilityCheckAttempts)
 				    .detail("SnapshotBytes", lastSnapshotBytes);
-				wait(delay(2.0));
+				co_await delay(2.0);
 				restorabilityCheckAttempts++;
 			}
 		}
@@ -373,7 +407,7 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		// Do one final check after the loop to catch snapshots that completed
 		// between the last check and now
 		if (!isRestorable) {
-			BackupDescription finalDesc = wait(backupContainer->describeBackup());
+			BackupDescription finalDesc = co_await backupContainer->describeBackup();
 			isRestorable = finalDesc.maxRestorableVersion.present();
 			lastSnapshotBytes = finalDesc.snapshotBytes;
 			if (isRestorable) {
@@ -391,28 +425,25 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		TraceEvent("BS3BCW_BackupRestorable")
 		    .detail("AttemptsNeeded", restorabilityCheckAttempts)
 		    .detail("SnapshotBytes", lastSnapshotBytes);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> doBackup(BackupS3BlobCorrectnessWorkload* self,
-	                                   double startDelay,
-	                                   FileBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   double stopDifferentialDelay,
-	                                   Promise<Void> submitted) {
+	Future<Void> doBackup(double startDelay,
+	                      FileBackupAgent* backupAgent,
+	                      Database cx,
+	                      Key tag,
+	                      Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                      double stopDifferentialDelay,
+	                      Promise<Void> submitted) {
 
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 
 		// Increment the backup agent requests
-		if (self->agentRequest) {
+		if (agentRequest) {
 			BackupS3BlobCorrectnessWorkload::backupAgentRequests++;
 		}
 
-		state Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
-		wait(delay(startDelay));
+		Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
+		co_await delay(startDelay);
 
 		// S3-specific: Conditional cleanup matching original BackupCorrectness behavior
 		// Only abort existing backups on first call (startDelay > 0) or randomly (BUGGIFY)
@@ -423,7 +454,7 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 			    .detail("StartDelay", startDelay);
 
 			try {
-				wait(backupAgent->abortBackup(cx, tag.toString()));
+				co_await backupAgent->abortBackup(cx, tag.toString());
 			} catch (Error& e) {
 				TraceEvent("BS3BCW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
 				if (e.code() != error_code_backup_unneeded)
@@ -433,9 +464,9 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 
 		TraceEvent("BS3BCW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
 
-		state EBackupState statusValue;
+		EBackupState statusValue;
 		try {
-			EBackupState _statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::False));
+			EBackupState _statusValue = co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::False);
 			statusValue = _statusValue;
 		} catch (Error& e) {
 			// If backup_unneeded, it means there's no active backup (possibly completed from previous test)
@@ -454,14 +485,14 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 
 		if (statusValue == EBackupState::STATE_COMPLETED) {
 			TraceEvent("BS3BCW_DoBackupDiscontinued", randomID).detail("Tag", printable(tag));
-			return Void();
+			co_return;
 		}
 
 		if (statusValue != EBackupState::STATE_NEVERRAN) {
 			TraceEvent("BS3BCW_DoBackupAbortBackup2", randomID).detail("Tag", printable(tag));
 
 			try {
-				wait(backupAgent->abortBackup(cx, tag.toString()));
+				co_await backupAgent->abortBackup(cx, tag.toString());
 			} catch (Error& e) {
 				TraceEvent("BS3BCW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
 				if (e.code() != error_code_backup_unneeded)
@@ -474,24 +505,25 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		    .detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
 
 		// S3-specific: Use configurable backup URL and snapshot intervals
-		state std::string backupContainer = self->backupURL;
-		state Future<Void> status = statusLoop(cx, tag.toString());
+		std::string backupContainer = backupURL;
+		Future<Void> status = statusLoop(cx, tag.toString());
 
 		// Testing v1 (non-partitioned) backup approach
 		// This does not require backup workers
 		try {
-			wait(backupAgent->submitBackup(cx,
-			                               StringRef(backupContainer),
-			                               {},
-			                               self->initSnapshotInterval,
-			                               self->snapshotInterval,
-			                               tag.toString(),
-			                               backupRanges,
-			                               StopWhenDone{ !stopDifferentialDelay },
-			                               UsePartitionedLog::False,
-			                               IncrementalBackupOnly::False,
-			                               self->encryptionKeyFileName,
-			                               self->snapshotMode));
+			co_await backupAgent->submitBackup(cx,
+			                                   StringRef(backupContainer),
+			                                   {},
+			                                   initSnapshotInterval,
+			                                   snapshotInterval,
+			                                   tag.toString(),
+			                                   backupRanges,
+			                                   StopWhenDone{ !stopDifferentialDelay },
+			                                   MutationLogType::DEFAULT,
+			                                   IncrementalBackupOnly::False,
+			                                   encryptionKeyFileName,
+			                                   encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+			                                   snapshotMode);
 		} catch (Error& e) {
 			TraceEvent("BS3BCW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@@ -500,16 +532,28 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 
 		submitted.send(Void());
 
+		Future<Void> observabilityCheck = Void();
+		if (snapshotMode == 1 || snapshotMode == 2) {
+			KeyBackedTag keyBackedTag = makeBackupTag(tag.toString());
+			try {
+				UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
+				UID backupUID = uidFlag.first;
+				observabilityCheck = verifyBulkDumpObservability(cx, backupUID, tag.toString());
+			} catch (Error& e) {
+				TraceEvent(SevWarn, "BS3BCW_CouldNotGetBackupUID").error(e);
+			}
+		}
+
 		TraceEvent("BS3BCW_DoBackupWaitToDiscontinue", randomID)
 		    .detail("Tag", printable(tag))
 		    .detail("DifferentialAfter", stopDifferentialDelay);
 
 		try {
-			wait(success(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True)));
+			co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True);
 		} catch (Error& e) {
 			if (e.code() == error_code_backup_unneeded) {
 				TraceEvent("BS3BCW_DoBackupWaitToDiscontinueUnneeded", randomID).detail("Tag", printable(tag));
-				return Void();
+				co_return;
 			}
 			throw;
 		}
@@ -517,11 +561,11 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		TraceEvent("BS3BCW_DoBackupDiscontinueBackup", randomID).detail("Tag", printable(tag));
 
 		try {
-			wait(backupAgent->discontinueBackup(cx, tag));
+			co_await backupAgent->discontinueBackup(cx, tag);
 		} catch (Error& e) {
 			if (e.code() == error_code_backup_unneeded) {
 				TraceEvent("BS3BCW_DoBackupDiscontinueBackupUnneeded", randomID).detail("Tag", printable(tag));
-				return Void();
+				co_return;
 			}
 			throw;
 		}
@@ -529,30 +573,28 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		TraceEvent("BS3BCW_DoBackupWaitForDiscontinued", randomID).detail("Tag", printable(tag));
 
 		try {
-			wait(success(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True)));
+			co_await backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::True);
 		} catch (Error& e) {
 			if (e.code() == error_code_backup_unneeded) {
 				TraceEvent("BS3BCW_DoBackupWaitForDiscontinuedUnneeded", randomID).detail("Tag", printable(tag));
-				return Void();
+				co_return;
 			}
 			throw;
 		}
 
 		TraceEvent("BS3BCW_DoBackupComplete", randomID).detail("Tag", printable(tag));
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, BackupS3BlobCorrectnessWorkload* self) {
-		state FileBackupAgent backupAgent;
-		state Future<Void> stopDifferentialBackup = delay(self->stopDifferentialAfter);
+	Future<Void> _start(Database cx) {
+		FileBackupAgent backupAgent;
+		Future<Void> stopDifferentialBackup = delay(stopDifferentialAfter);
 
 		TraceEvent("BS3BCW_Arguments")
-		    .detail("BackupAfter", self->backupAfter)
-		    .detail("RestoreAfter", self->restoreAfter)
-		    .detail("RestoreStartAfterBackupFinished", self->restoreStartAfterBackupFinished)
-		    .detail("AbortAndRestartAfter", self->abortAndRestartAfter)
-		    .detail("DifferentialAfter", self->stopDifferentialAfter);
+		    .detail("BackupAfter", backupAfter)
+		    .detail("RestoreAfter", restoreAfter)
+		    .detail("RestoreStartAfterBackupFinished", restoreStartAfterBackupFinished)
+		    .detail("AbortAndRestartAfter", abortAndRestartAfter)
+		    .detail("DifferentialAfter", stopDifferentialAfter);
 
 		// S3-specific: Clean up state on every test restart to prevent restore_invalid_version errors
 		// The test harness restarts tests multiple times. Without cleanup, FDB metadata points to
@@ -561,10 +603,10 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		// 1. Abort any existing backup (clears FDB metadata)
 		// 2. Clear MockS3 storage (removes old backup data)
 		// This ensures FDB and S3 are always in sync at the start of each test run.
-		if (self->agentRequest) {
-			TraceEvent("BS3BCW_CleanupOnRestart").detail("Tag", printable(self->backupTag));
+		if (agentRequest) {
+			TraceEvent("BS3BCW_CleanupOnRestart").detail("Tag", printable(backupTag));
 			try {
-				wait(backupAgent.abortBackup(cx, self->backupTag.toString()));
+				co_await backupAgent.abortBackup(cx, backupTag.toString());
 			} catch (Error& e) {
 				if (e.code() != error_code_backup_unneeded)
 					throw;
@@ -575,80 +617,73 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 		// metadata that was just initialized when the server was registered. The persistence system
 		// handles cleanup properly on its own.
 
-		if (self->agentRequest) {
-			state Promise<Void> submitted;
-			state Future<Void> b = doBackup(self,
-			                                self->backupAfter,
-			                                &backupAgent,
-			                                cx,
-			                                self->backupTag,
-			                                self->backupRanges,
-			                                self->stopDifferentialAfter,
-			                                submitted);
-
-			if (self->abortAndRestartAfter) {
-				TraceEvent("BS3BCW_AbortAndRestartAfter").detail("AbortAndRestartAfter", self->abortAndRestartAfter);
-				wait(submitted.getFuture());
-				wait(delay(self->abortAndRestartAfter - self->backupAfter));
-				TraceEvent("BS3BCW_AbortBackup").detail("Tag", printable(self->backupTag));
+		if (agentRequest) {
+			Promise<Void> submitted;
+			Future<Void> b =
+			    doBackup(backupAfter, &backupAgent, cx, backupTag, backupRanges, stopDifferentialAfter, submitted);
+
+			if (abortAndRestartAfter) {
+				TraceEvent("BS3BCW_AbortAndRestartAfter").detail("AbortAndRestartAfter", abortAndRestartAfter);
+				co_await submitted.getFuture();
+				co_await delay(abortAndRestartAfter - backupAfter);
+				TraceEvent("BS3BCW_AbortBackup").detail("Tag", printable(backupTag));
 				try {
-					wait(backupAgent.abortBackup(cx, self->backupTag.toString()));
+					co_await backupAgent.abortBackup(cx, backupTag.toString());
 				} catch (Error& e) {
 					if (e.code() != error_code_backup_unneeded)
 						throw;
 				}
-				TraceEvent("BS3BCW_AbortComplete").detail("Tag", printable(self->backupTag));
-				wait(b);
-				TraceEvent("BS3BCW_RestartBackup").detail("Tag", printable(self->backupTag));
-				b = doBackup(self,
-				             0,
+				TraceEvent("BS3BCW_AbortComplete").detail("Tag", printable(backupTag));
+				co_await b;
+				TraceEvent("BS3BCW_RestartBackup").detail("Tag", printable(backupTag));
+				b = doBackup(0,
 				             &backupAgent,
 				             cx,
-				             self->backupTag,
-				             self->backupRanges,
-				             self->stopDifferentialAfter - self->abortAndRestartAfter,
+				             backupTag,
+				             backupRanges,
+				             stopDifferentialAfter - abortAndRestartAfter,
 				             Promise<Void>());
 			}
 
-			if (self->performRestore) {
+			if (performRestore) {
 				// Adaptive timing: Wait for backup to complete, then wait additional time
 				// This ensures the backup metadata is written before restore starts
 				TraceEvent("BS3BCW_WaitingForBackupCompletion").detail("WaitingForBackup", true);
-				wait(b);
+				co_await b;
 				TraceEvent("BS3BCW_BackupCompleted").detail("BackupFinished", true);
 
 				// Wait additional time after backup completes for metadata to be written
-				if (self->restoreStartAfterBackupFinished > 0) {
+				if (restoreStartAfterBackupFinished > 0) {
 					TraceEvent("BS3BCW_WaitingAfterBackupComplete")
-					    .detail("DelaySeconds", self->restoreStartAfterBackupFinished);
-					wait(delay(self->restoreStartAfterBackupFinished));
+					    .detail("DelaySeconds", restoreStartAfterBackupFinished);
+					co_await delay(restoreStartAfterBackupFinished);
 				}
 
 				TraceEvent("BS3BCW_StartingRestore").detail("RestoreStarting", true);
 
 				// Get the backup container to restore from
-				state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
-				UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx.getReference()));
-				state UID logUid = uidFlag.first;
-				state Reference<IBackupContainer> lastBackupContainer =
-				    wait(BackupConfig(logUid).backupContainer().getD(cx.getReference()));
+				KeyBackedTag keyBackedTag = makeBackupTag(backupTag.toString());
+				UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
+				UID logUid = uidFlag.first;
+				Reference<IBackupContainer> lastBackupContainer =
+				    co_await BackupConfig(logUid).backupContainer().getD(cx.getReference());
 
 				// Wait for backup to become restorable if it's still in progress
 				// Increased timeout for complex multi-region configs
 				if (lastBackupContainer) {
-					wait(waitForRestorable(lastBackupContainer, 150));
+					co_await waitForRestorable(lastBackupContainer, 150);
 
 					// Generate a lock UID for the entire clear+restore operation
-					state UID lockUID = deterministicRandom()->randomUniqueID();
+					UID lockUID = deterministicRandom()->randomUniqueID();
 
 					// Lock the database to prevent other workloads from seeing inconsistent state
 					// during clear+restore. This makes workloads like Cycle get database_locked
 					// errors (retriable) instead of encountering missing keys.
-					wait(lockDatabase(cx, lockUID));
+					co_await lockDatabase(cx, lockUID);
 
 					TraceEvent("BS3BCW_DatabaseLocked")
 					    .detail("LockUID", lockUID)
-					    .detail("BackupTag", printable(self->backupTag));
+					    .detail("BackupTag", printable(backupTag));
 
 					// BulkLoad validation: compare BulkLoad restore vs traditional restore
 					// 1. Traditional restore with addPrefix to system keyspace (\xff\x02/rlog/)
@@ -657,37 +692,37 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 					// 4. audit_storage validate_restore compares normalKeys vs prefixed data
 
 					// Prefix for validation - restored data goes to system keyspace
-					state Key validationPrefix = "\xff\x02/rlog/"_sr;
-					state Key validationPrefixEnd = "\xff\x02/rlog0"_sr;
+					Key validationPrefix = "\xff\x02/rlog/"_sr;
+					Key validationPrefixEnd = "\xff\x02/rlog0"_sr;
 
-					if (self->performValidation) {
+					if (performValidation) {
 						// Step 1: Restore with prefix using TRADITIONAL (rangefile) mode
 						TraceEvent("BS3BCW_ValidationStep1_TraditionalRestore")
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("ValidationPrefix", printable(validationPrefix));
 
-						Standalone<StringRef> validationRestoreTag(self->backupTag.toString() + "_validate");
+						Standalone<StringRef> validationRestoreTag(backupTag.toString() + "_validate");
 						try {
 							Version validationVersion =
-							    wait(backupAgent.restore(cx,
-							                             cx,
-							                             validationRestoreTag,
-							                             KeyRef(lastBackupContainer->getURL()),
-							                             lastBackupContainer->getProxy(),
-							                             self->restoreRanges,
-							                             WaitForComplete::True,
-							                             ::invalidVersion,
-							                             Verbose::True,
-							                             validationPrefix, // addPrefix
-							                             Key(), // removePrefix
-							                             LockDB::False, // already locked
-							                             UnlockDB::False, // don't unlock yet
-							                             OnlyApplyMutationLogs::False,
-							                             InconsistentSnapshotOnly::False,
-							                             ::invalidVersion,
-							                             lastBackupContainer->getEncryptionKeyFileName(),
-							                             lockUID,
-							                             true)); // useRangeFileRestore = true (traditional mode)
+							    co_await backupAgent.restore(cx,
+							                                 cx,
+							                                 validationRestoreTag,
+							                                 KeyRef(lastBackupContainer->getURL()),
+							                                 lastBackupContainer->getProxy(),
+							                                 restoreRanges,
+							                                 WaitForComplete::True,
+							                                 ::invalidVersion,
+							                                 Verbose::True,
+							                                 validationPrefix, // addPrefix
+							                                 Key(), // removePrefix
+							                                 LockDB::False, // already locked
+							                                 UnlockDB::False, // don't unlock yet
+							                                 OnlyApplyMutationLogs::False,
+							                                 InconsistentSnapshotOnly::False,
+							                                 ::invalidVersion,
+							                                 lastBackupContainer->getEncryptionKeyFileName(),
+							                                 lockUID,
+							                                 true); // useRangeFileRestore = true (traditional mode)
 
 							TraceEvent("BS3BCW_ValidationStep1_Complete")
 							    .detail("ValidationVersion", validationVersion)
@@ -699,111 +734,113 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 					}
 
 					// Step 2: Clear the backup ranges before restoring (unless skipDirtyRestore is true)
-					if (!self->skipDirtyRestore) {
+					if (!skipDirtyRestore) {
 						TraceEvent("BS3BCW_ClearingNormalKeys");
-						wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+						co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 							tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 							tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-							for (auto& kvrange : self->backupRanges)
+							for (auto& kvrange : backupRanges)
 								tr->clear(kvrange);
 							return Void();
-						}));
+						});
 					}
 
 					// Step 3: Perform the restore (with BulkLoad if configured)
 					TraceEvent("BS3BCW_Restore")
 					    .detail("LastBackupContainer", lastBackupContainer->getURL())
-					    .detail("BackupTag", printable(self->backupTag))
+					    .detail("BackupTag", printable(backupTag))
 					    .detail("LockUID", lockUID)
-					    .detail("SkipDirtyRestore", self->skipDirtyRestore);
+					    .detail("SkipDirtyRestore", skipDirtyRestore);
 
-					Standalone<StringRef> restoreTag(self->backupTag.toString() + "_restore");
+					Standalone<StringRef> restoreTag(backupTag.toString() + "_restore");
 					// Pass lockDB=False since we already locked, unlockDB=True to release when done,
 					// and our lockUID so restore uses the same lock for checkDatabaseLock calls
-					Version v = wait(backupAgent.restore(cx,
-					                                     cx,
-					                                     restoreTag,
-					                                     KeyRef(lastBackupContainer->getURL()),
-					                                     lastBackupContainer->getProxy(),
-					                                     self->restoreRanges,
-					                                     WaitForComplete::True,
-					                                     ::invalidVersion,
-					                                     Verbose::True,
-					                                     Key(),
-					                                     Key(),
-					                                     LockDB::False,
-					                                     UnlockDB::True,
-					                                     OnlyApplyMutationLogs::False,
-					                                     InconsistentSnapshotOnly::False,
-					                                     ::invalidVersion,
-					                                     lastBackupContainer->getEncryptionKeyFileName(),
-					                                     lockUID,
-					                                     self->useRangeFileRestore));
+					Version v = co_await backupAgent.restore(cx,
+					                                         cx,
+					                                         restoreTag,
+					                                         KeyRef(lastBackupContainer->getURL()),
+					                                         lastBackupContainer->getProxy(),
+					                                         restoreRanges,
+					                                         WaitForComplete::True,
+					                                         ::invalidVersion,
+					                                         Verbose::True,
+					                                         Key(),
+					                                         Key(),
+					                                         LockDB::False,
+					                                         UnlockDB::True,
+					                                         OnlyApplyMutationLogs::False,
+					                                         InconsistentSnapshotOnly::False,
+					                                         ::invalidVersion,
+					                                         lastBackupContainer->getEncryptionKeyFileName(),
+					                                         lockUID,
+					                                         useRangeFileRestore);
 
 					TraceEvent("BS3BCW_RestoreComplete")
-					    .detail("BackupTag", printable(self->backupTag))
+					    .detail("BackupTag", printable(backupTag))
 					    .detail("RestoreVersion", v);
 
 					// ASSERT: When useRangeFileRestore=false, BulkLoad MUST have been used
-					if (!self->useRangeFileRestore) {
+					if (!useRangeFileRestore) {
 						int bulkLoadCount = g_bulkLoadRestoreTaskCompleteCount.load();
 						TraceEvent("BS3BCW_AssertBulkLoadUsed")
-						    .detail("UseRangeFileRestore", self->useRangeFileRestore)
+						    .detail("UseRangeFileRestore", useRangeFileRestore)
 						    .detail("BulkLoadRestoreTaskCompleteCount", bulkLoadCount);
 						// FAIL if BulkLoad didn't run
 						ASSERT(bulkLoadCount > 0);
 					}
 
 					// Step 4: Run audit to compare BulkLoad-restored vs traditional-restored
-					if (self->performValidation) {
+					if (performValidation) {
 						TraceEvent("BS3BCW_ValidationStep4_AuditStarting")
 						    .detail("Comparing", "BulkLoad-restored (normalKeys) vs traditional-restored (prefix)");
 
-						state Reference<IClusterConnectionRecord> clusterFile = cx->getConnectionRecord();
-						state UID auditId;
-						state int auditRetryCount = 0;
-						state int maxAuditRetries = 5;
+						Reference<IClusterConnectionRecord> clusterFile = cx->getConnectionRecord();
+						UID auditId;
+						int auditRetryCount = 0;
+						int maxAuditRetries = 5;
 
-						state Error auditError;
-						loop {
+						Error auditError;
+						while (true) {
+							Error err;
 							try {
-								UID scheduleResult = wait(timeoutError(auditStorage(clusterFile,
-								                                                    normalKeys,
-								                                                    AuditType::ValidateRestore,
-								                                                    KeyValueStoreType::END,
-								                                                    300.0),
-								                                       60.0));
+								UID scheduleResult = co_await timeoutError(auditStorage(clusterFile,
+								                                                        normalKeys,
+								                                                        AuditType::ValidateRestore,
+								                                                        KeyValueStoreType::END,
+								                                                        300.0),
+								                                           60.0);
 								auditId = scheduleResult;
 								break;
 							} catch (Error& e) {
-								auditError = e;
-								if (auditError.code() == error_code_timed_out ||
-								    auditError.code() == error_code_audit_storage_failed) {
-									auditRetryCount++;
-									if (auditRetryCount < maxAuditRetries) {
-										TraceEvent(SevWarn, "BS3BCW_ValidationAuditRetry")
-										    .error(auditError)
-										    .detail("RetryCount", auditRetryCount);
-										wait(delay(2.0 * auditRetryCount));
-										continue;
-									}
+								err = e;
+							}
+							auditError = err;
+							if (auditError.code() == error_code_timed_out ||
+							    auditError.code() == error_code_audit_storage_failed) {
+								auditRetryCount++;
+								if (auditRetryCount < maxAuditRetries) {
+									TraceEvent(SevWarn, "BS3BCW_ValidationAuditRetry")
+									    .error(auditError)
+									    .detail("RetryCount", auditRetryCount);
+									co_await delay(2.0 * auditRetryCount);
+									continue;
 								}
-								throw auditError;
 							}
+							throw auditError;
 						}
 
 						TraceEvent("BS3BCW_ValidationAuditScheduled").detail("AuditID", auditId);
 
 						// Monitor audit progress
-						state double auditStartTime = now();
-						state double maxAuditWaitTime = 300.0;
-						state AuditPhase finalPhase = AuditPhase::Invalid;
+						double auditStartTime = now();
+						double maxAuditWaitTime = 300.0;
+						AuditPhase finalPhase = AuditPhase::Invalid;
 
-						loop {
-							wait(delay(5.0));
+						while (true) {
+							co_await delay(5.0);
 
 							std::vector<AuditStorageState> auditStates =
-							    wait(getAuditStates(cx, AuditType::ValidateRestore, true));
+							    co_await getAuditStates(cx, AuditType::ValidateRestore, true);
 
 							for (const auto& auditState : auditStates) {
 								if (auditState.id == auditId) {
@@ -841,34 +878,32 @@ struct BackupS3BlobCorrectnessWorkload : TestWorkload {
 						    .detail("Result", "BulkLoad produces identical results to traditional restore");
 
 						// Step 5: Clean up validation data from system keyspace
-						wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+						co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 							tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 							tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 							tr->clear(KeyRangeRef(validationPrefix, validationPrefixEnd));
 							return Void();
-						}));
+						});
 
 						TraceEvent("BS3BCW_ValidationComplete")
-						    .detail("BackupTag", printable(self->backupTag))
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("AuditID", auditId);
 					}
 				}
 			}
 
 			// ASSERT: Verify BulkDump was used when snapshotMode=1 (BULKDUMP) or 2 (BOTH)
-			if (self->snapshotMode == 1 || self->snapshotMode == 2) {
+			if (snapshotMode == 1 || snapshotMode == 2) {
 				int bulkDumpCount = g_bulkDumpTaskCompleteCount.load();
 				TraceEvent("BS3BCW_AssertBulkDumpUsed")
-				    .detail("SnapshotMode", self->snapshotMode)
+				    .detail("SnapshotMode", snapshotMode)
 				    .detail("BulkDumpTaskCompleteCount", bulkDumpCount);
 				// FAIL if BulkDump didn't run
 				ASSERT(bulkDumpCount > 0);
 			}
 
-			wait(b);
+			co_await b;
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BackupToDBAbort.actor.cpp b/fdbserver/workloads/BackupToDBAbort.cpp
similarity index 51%
rename from fdbserver/workloads/BackupToDBAbort.actor.cpp
rename to fdbserver/workloads/BackupToDBAbort.cpp
index 92611309b6e..6985c6909f7 100644
--- a/fdbserver/workloads/BackupToDBAbort.actor.cpp
+++ b/fdbserver/workloads/BackupToDBAbort.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupToDBAbort.actor.cpp
+ * BackupToDBAbort.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbclient/BackupAgent.actor.h"
-#include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/BackupAgent.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/ApiVersion.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct BackupToDBAbort : TestWorkload {
 	static constexpr auto NAME = "BackupToDBAbort";
@@ -38,8 +35,8 @@ struct BackupToDBAbort : TestWorkload {
 
 		addDefaultBackupRanges(backupRanges);
 
-		ASSERT(g_simulator->extraDatabases.size() == 1);
-		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+		ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 
 		lockid = UID(0xbeeffeed, 0xdecaf00d);
 	}
@@ -47,67 +44,62 @@ struct BackupToDBAbort : TestWorkload {
 	Future<Void> setup(const Database& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _setup(this, cx);
+		return _setup(cx);
 	}
 
-	ACTOR static Future<Void> _setup(BackupToDBAbort* self, Database cx) {
-		state DatabaseBackupAgent backupAgent(cx);
+	Future<Void> _setup(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
 		try {
 			TraceEvent("BDBA_Submit1").log();
-			wait(backupAgent.submitBackup(self->extraDB,
-			                              BackupAgentBase::getDefaultTag(),
-			                              self->backupRanges,
-			                              StopWhenDone::False,
-			                              StringRef(),
-			                              StringRef(),
-			                              LockDB::True));
+			co_await backupAgent.submitBackup(extraDB,
+			                                  BackupAgentBase::getDefaultTag(),
+			                                  backupRanges,
+			                                  StopWhenDone::False,
+			                                  StringRef(),
+			                                  StringRef(),
+			                                  LockDB::True);
 			TraceEvent("BDBA_Submit2").log();
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_duplicate)
 				throw;
 		}
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(this, cx);
+		return _start(cx);
 	}
 
-	ACTOR static Future<Void> _start(BackupToDBAbort* self, Database cx) {
-		state DatabaseBackupAgent backupAgent(cx);
+	Future<Void> _start(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
 
-		TraceEvent("BDBA_Start").detail("Delay", self->abortDelay);
-		wait(delay(self->abortDelay));
+		TraceEvent("BDBA_Start").detail("Delay", abortDelay);
+		co_await delay(abortDelay);
 		TraceEvent("BDBA_Wait").log();
-		wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::False)));
+		co_await backupAgent.waitBackup(extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::False);
 		TraceEvent("BDBA_Lock").log();
-		wait(lockDatabase(cx, self->lockid));
+		co_await lockDatabase(cx, lockid);
 		TraceEvent("BDBA_Abort").log();
-		wait(backupAgent.abortBackup(self->extraDB, BackupAgentBase::getDefaultTag()));
+		co_await backupAgent.abortBackup(extraDB, BackupAgentBase::getDefaultTag());
 		TraceEvent("BDBA_Unlock").log();
-		wait(backupAgent.unlockBackup(self->extraDB, BackupAgentBase::getDefaultTag()));
+		co_await backupAgent.unlockBackup(extraDB, BackupAgentBase::getDefaultTag());
 		TraceEvent("BDBA_End").log();
 
 		// SOMEDAY: Remove after backup agents can exist quiescently
-		if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
-			g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+		if (fdbSimulationPolicyState().drAgents == FDBBackupAgentType::BackupToDB) {
+			fdbSimulationPolicyState().drAgents = FDBBackupAgentType::NoBackupAgents;
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<bool> _check(BackupToDBAbort* self, Database cx) {
+	Future<bool> check(const Database& cx) override {
 		TraceEvent("BDBA_UnlockPrimary").log();
 		// Too much of the tester framework expects the primary database to be unlocked, so we unlock it
 		// once all of the workloads have finished.
-		wait(unlockDatabase(cx, self->lockid));
-		return true;
+		co_await unlockDatabase(cx, lockid);
+		co_return true;
 	}
 
-	Future<bool> check(const Database& cx) override { return _check(this, cx); }
-
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
diff --git a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp b/fdbserver/workloads/BackupToDBCorrectness.cpp
similarity index 65%
rename from fdbserver/workloads/BackupToDBCorrectness.actor.cpp
rename to fdbserver/workloads/BackupToDBCorrectness.cpp
index 219d08488e7..e5fd3db67f0 100644
--- a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp
+++ b/fdbserver/workloads/BackupToDBCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupToDBCorrectness.actor.cpp
+ * BackupToDBCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,15 +19,14 @@
  */
 
 #include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "flow/ApiVersion.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // This workload tests backing up one cluster to another.
 
@@ -55,7 +54,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 		out.insert({ "RandomRangeLock" });
 	}
 
-	BackupToDBCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit BackupToDBCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
 		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
@@ -103,7 +102,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 				backupPrefix = backupPrefix.withPrefix("\x00\x00\00"_sr);
 			}
 
-			ASSERT(backupPrefix != StringRef());
+			ASSERT(!backupPrefix.empty());
 		}
 
 		KeyRef beginRange;
@@ -148,8 +147,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			}
 		}
 
-		ASSERT(g_simulator->extraDatabases.size() == 1);
-		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+		ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 
 		TraceEvent("BARW_Start").detail("Locked", locked);
 	}
@@ -161,7 +160,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 		return _setup(cx, this);
 	}
 
-	ACTOR Future<Void> _setup(Database cx, BackupToDBCorrectnessWorkload* self) {
+	Future<Void> _setup(Database cx, BackupToDBCorrectnessWorkload* self) {
 		if (BUGGIFY) {
 			for (auto r : getSystemBackupRanges()) {
 				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
@@ -174,7 +173,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
@@ -182,46 +181,48 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
 	// Reads a series of key ranges and returns each range.
-	ACTOR static Future<std::vector<RangeResult>> readRanges(Database cx,
-	                                                         Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                                         StringRef removePrefix) {
-		loop {
-			state Transaction tr(cx);
+	static Future<std::vector<RangeResult>> readRanges(Database cx,
+	                                                   Standalone<VectorRef<KeyRangeRef>> ranges,
+	                                                   StringRef removePrefix) {
+		while (true) {
+			Transaction tr(cx);
+			Error err;
 			try {
-				state std::vector<Future<RangeResult>> results;
+				std::vector<Future<RangeResult>> results;
 				for (auto& range : ranges) {
 					results.push_back(tr.getRange(range.removePrefix(removePrefix), 1000));
 				}
-				wait(waitForAll(results));
+				co_await waitForAll(results);
 
 				std::vector<RangeResult> ret;
-				for (auto result : results) {
+				for (const auto& result : results) {
 					ret.push_back(result.get());
 				}
-				return ret;
+				co_return ret;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> diffRanges(Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                     StringRef backupPrefix,
-	                                     Database src,
-	                                     Database dest) {
-		state int rangeIndex;
-		for (rangeIndex = 0; rangeIndex < ranges.size(); ++rangeIndex) {
-			state KeyRangeRef range = ranges[rangeIndex];
-			state Key begin = range.begin;
-			loop {
-				state Transaction tr(src);
-				state Transaction tr2(dest);
+	static Future<Void> diffRanges(Standalone<VectorRef<KeyRangeRef>> ranges,
+	                               StringRef backupPrefix,
+	                               Database src,
+	                               Database dest) {
+		for (int rangeIndex = 0; rangeIndex < ranges.size(); ++rangeIndex) {
+			KeyRangeRef range = ranges[rangeIndex];
+			Key begin = range.begin;
+			while (true) {
+				Transaction tr(src);
+				Transaction tr2(dest);
+				Error err;
 				try {
-					loop {
-						state Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
-						state Future<RangeResult> bkpFuture =
+					while (true) {
+						Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
+						Future<RangeResult> bkpFuture =
 						    tr2.getRange(KeyRangeRef(begin, range.end).withPrefix(backupPrefix), 1000);
-						wait(success(srcFuture) && success(bkpFuture));
+						co_await (success(srcFuture) && success(bkpFuture));
 
 						auto src = srcFuture.get().begin();
 						auto bkp = bkpFuture.get().begin();
@@ -281,27 +282,25 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> doBackup(BackupToDBCorrectnessWorkload* self,
-	                                   double startDelay,
-	                                   DatabaseBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges,
-	                                   double stopDifferentialDelay,
-	                                   Promise<Void> submitted) {
+	Future<Void> doBackup(double startDelay,
+	                      DatabaseBackupAgent* backupAgent,
+	                      Database cx,
+	                      Key tag,
+	                      Standalone<VectorRef<KeyRangeRef>> backupRanges,
+	                      double stopDifferentialDelay,
+	                      Promise<Void> submitted) {
 
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 
-		state Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
-		wait(delay(startDelay));
+		Future<Void> stopDifferentialFuture = delay(stopDifferentialDelay);
+		co_await delay(startDelay);
 
 		if (startDelay || BUGGIFY) {
 			TraceEvent("BARW_DoBackupAbortBackup1", randomID)
@@ -309,13 +308,13 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			    .detail("StartDelay", startDelay);
 
 			try {
-				wait(backupAgent->abortBackup(cx, tag));
+				co_await backupAgent->abortBackup(cx, tag);
 			} catch (Error& e) {
 				TraceEvent("BARW_DoBackupAbortBackupException", randomID).error(e).detail("Tag", printable(tag));
 				if (e.code() != error_code_backup_unneeded)
 					throw;
 			}
-			wait(backupAgent->unlockBackup(cx, tag));
+			co_await backupAgent->unlockBackup(cx, tag);
 		}
 
 		// In prior versions of submitBackup, we have seen a rare bug where
@@ -333,14 +332,14 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 		try {
 			try {
-				wait(backupAgent->submitBackup(cx,
-				                               tag,
-				                               backupRanges,
-				                               StopWhenDone{ !stopDifferentialDelay },
-				                               self->backupPrefix,
-				                               StringRef(),
-				                               LockDB{ self->locked },
-				                               DatabaseBackupAgent::PreBackupAction::CLEAR));
+				co_await backupAgent->submitBackup(cx,
+				                                   tag,
+				                                   backupRanges,
+				                                   StopWhenDone{ !stopDifferentialDelay },
+				                                   backupPrefix,
+				                                   StringRef(),
+				                                   LockDB{ locked },
+				                                   DatabaseBackupAgent::PreBackupAction::CLEAR);
 			} catch (Error& e) {
 				TraceEvent("BARW_SubmitBackup1Exception", randomID).error(e);
 				if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) {
@@ -356,23 +355,23 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 		submitted.send(Void());
 
-		state UID logUid = wait(backupAgent->getLogUid(cx, tag));
+		UID logUid = co_await backupAgent->getLogUid(cx, tag);
 
 		// Stop the differential backup, if enabled
 		if (stopDifferentialDelay) {
 			CODE_PROBE(!stopDifferentialFuture.isReady(),
 			           "Restore starts at specified time - stopDifferential not ready");
-			wait(stopDifferentialFuture);
+			co_await stopDifferentialFuture;
 			TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID)
 			    .detail("Tag", printable(tag))
 			    .detail("DifferentialAfter", stopDifferentialDelay);
 
-			state bool aborted = false;
+			bool aborted = false;
 			try {
 				if (BUGGIFY) {
 					TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", printable(tag));
 					// Wait until the backup is in a restorable state
-					state EBackupState resultWait = wait(backupAgent->waitBackup(cx, tag, StopWhenDone::False));
+					EBackupState resultWait = co_await backupAgent->waitBackup(cx, tag, StopWhenDone::False);
 
 					TraceEvent("BARW_LastBackupFolder", randomID)
 					    .detail("BackupTag", printable(tag))
@@ -386,12 +385,12 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 						    .detail("Tag", printable(tag))
 						    .detail("WaitStatus", resultWait);
 						aborted = true;
-						wait(backupAgent->abortBackup(cx, tag));
+						co_await backupAgent->abortBackup(cx, tag);
 					} else {
 						TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
 						    .detail("Tag", printable(tag))
 						    .detail("DifferentialAfter", stopDifferentialDelay);
-						wait(backupAgent->discontinueBackup(cx, tag));
+						co_await backupAgent->discontinueBackup(cx, tag);
 					}
 				}
 
@@ -399,7 +398,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 					TraceEvent("BARW_DoBackupDiscontinueBackup", randomID)
 					    .detail("Tag", printable(tag))
 					    .detail("DifferentialAfter", stopDifferentialDelay);
-					wait(backupAgent->discontinueBackup(cx, tag));
+					co_await backupAgent->discontinueBackup(cx, tag);
 				}
 			} catch (Error& e) {
 				TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).error(e).detail("Tag", printable(tag));
@@ -408,22 +407,22 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			}
 
 			if (aborted) {
-				wait(backupAgent->unlockBackup(cx, tag));
+				co_await backupAgent->unlockBackup(cx, tag);
 			}
 		}
 
 		// Wait for the backup to complete
 		TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
 
-		UID _destUid = wait(backupAgent->getDestUid(cx, logUid));
-		self->destUid = _destUid;
+		UID _destUid = co_await backupAgent->getDestUid(cx, logUid);
+		destUid = _destUid;
 
-		state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag, StopWhenDone::True));
-		wait(backupAgent->unlockBackup(cx, tag));
+		EBackupState statusValue = co_await backupAgent->waitBackup(cx, tag, StopWhenDone::True);
+		co_await backupAgent->unlockBackup(cx, tag);
 
-		state std::string statusText;
+		std::string statusText;
 
-		std::string _statusText = wait(backupAgent->getStatus(cx, 5, tag));
+		std::string _statusText = co_await backupAgent->getStatus(cx, 5, tag);
 		statusText = _statusText;
 		// Can we validate anything about status?
 
@@ -431,36 +430,35 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 		    .detail("Tag", printable(tag))
 		    .detail("Status", statusText)
 		    .detail("StatusValue", statusValue);
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> checkData(Database cx,
-	                                    UID logUid,
-	                                    UID destUid,
-	                                    UID randomID,
-	                                    Key tag,
-	                                    DatabaseBackupAgent* backupAgent,
-	                                    bool shareLogRange) {
-		state Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
-		state Key backupLogValuesKey = uidPrefixKey(backupLogKeys.begin, destUid);
-		state Key backupLatestVersionsPath = uidPrefixKey(backupLatestVersionsPrefix, destUid);
-		state Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
-		state int displaySystemKeys = 0;
+	static Future<Void> checkData(Database cx,
+	                              UID logUid,
+	                              UID destUid,
+	                              UID randomID,
+	                              Key tag,
+	                              DatabaseBackupAgent* backupAgent,
+	                              bool shareLogRange) {
+		Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
+		Key backupLogValuesKey = uidPrefixKey(backupLogKeys.begin, destUid);
+		Key backupLatestVersionsPath = uidPrefixKey(backupLatestVersionsPrefix, destUid);
+		Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
+		int displaySystemKeys = 0;
 
 		// Ensure that there is no left over key within the backup subspace
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
 			TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(tag));
 
+			Error err;
 			try {
 				// Check the left over tasks
 				// We have to wait for the list to empty since an abort and get status
 				// can leave extra tasks in the queue
 				TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(tag));
-				state int64_t taskCount = wait(backupAgent->getTaskCount(tr));
-				state int waitCycles = 0;
+				int64_t taskCount = co_await backupAgent->getTaskCount(tr);
+				int waitCycles = 0;
 
 				if ((taskCount) && false) {
 					TraceEvent("BARW_EndingNonzeroTaskCount", randomID)
@@ -468,7 +466,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 					    .detail("TaskCount", taskCount)
 					    .detail("WaitCycles", waitCycles);
 					printf("EndingNonZeroTasks: %ld\n", (long)taskCount);
-					wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
+					co_await TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef());
 				}
 
 				while (taskCount > 0) {
@@ -484,16 +482,16 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 					       waitCycles,
 					       (long long)taskCount);
 
-					wait(delay(5.0));
+					co_await delay(5.0);
 					tr = makeReference<ReadYourWritesTransaction>(cx);
-					wait(store(taskCount, backupAgent->getTaskCount(tr)));
+					taskCount = co_await backupAgent->getTaskCount(tr);
 				}
 
 				RangeResult agentValues =
-				    wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
+				    co_await tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100);
 
 				// Error if the system keyspace for the backup tag is not empty
-				if (agentValues.size() > 0) {
+				if (!agentValues.empty()) {
 					displaySystemKeys++;
 					printf("BackupCorrectnessLeftoverMutationKeys: (%d) %s\n",
 					       agentValues.size(),
@@ -514,7 +512,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 					printf("No left over backup agent configuration keys\n");
 				}
 
-				Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
+				Optional<Value> latestVersion = co_await tr->get(backupLatestVersionsKey);
 				if (latestVersion.present()) {
 					TraceEvent(SevError, "BackupCorrectnessLeftoverVersionKey", randomID)
 					    .detail("BackupTag", printable(tag))
@@ -524,14 +522,14 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 					printf("No left over backup version key\n");
 				}
 
-				RangeResult versions = wait(
-				    tr->getRange(KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1));
-				if (!shareLogRange || !versions.size()) {
-					RangeResult logValues =
-					    wait(tr->getRange(KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100));
+				RangeResult versions = co_await tr->getRange(
+				    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1);
+				if (!shareLogRange || versions.empty()) {
+					RangeResult logValues = co_await tr->getRange(
+					    KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100);
 
 					// Error if the log/mutation keyspace for the backup tag is not empty
-					if (logValues.size() > 0) {
+					if (!logValues.empty()) {
 						displaySystemKeys++;
 						printf("BackupCorrectnessLeftoverLogKeys: (%d) %s\n",
 						       logValues.size(),
@@ -556,123 +554,116 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 				break;
 			} catch (Error& e) {
-				TraceEvent("BARW_CheckException", randomID).error(e);
-				wait(tr->onError(e));
+				err = e;
 			}
+			TraceEvent("BARW_CheckException", randomID).error(err);
+			co_await tr->onError(err);
 		}
 
 		if (displaySystemKeys) {
-			wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
+			co_await TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef());
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, BackupToDBCorrectnessWorkload* self) {
-		state DatabaseBackupAgent backupAgent(cx);
-		state DatabaseBackupAgent restoreTool(self->extraDB);
-		state Future<Void> extraBackup;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> _start(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
+		DatabaseBackupAgent restoreTool(extraDB);
+		Future<Void> extraBackup;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		TraceEvent("BARW_Arguments")
-		    .detail("BackupTag", printable(self->backupTag))
-		    .detail("BackupAfter", self->backupAfter)
-		    .detail("AbortAndRestartAfter", self->abortAndRestartAfter)
-		    .detail("DifferentialAfter", self->stopDifferentialAfter);
+		    .detail("BackupTag", printable(backupTag))
+		    .detail("BackupAfter", backupAfter)
+		    .detail("AbortAndRestartAfter", abortAndRestartAfter)
+		    .detail("DifferentialAfter", stopDifferentialAfter);
 
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 
 		// Increment the backup agent requests
-		if (self->agentRequest) {
+		if (agentRequest) {
 			BackupToDBCorrectnessWorkload::drAgentRequests++;
 		}
 
 		try {
-			state Future<Void> startRestore = delay(self->restoreAfter);
+			Future<Void> startRestore = delay(restoreAfter);
 
 			// backup
-			wait(delay(self->backupAfter));
-
-			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
-			state Promise<Void> submitted;
-			state Future<Void> b = doBackup(self,
-			                                0,
-			                                &backupAgent,
-			                                self->extraDB,
-			                                self->backupTag,
-			                                self->backupRanges,
-			                                self->stopDifferentialAfter,
-			                                submitted);
-
-			if (self->abortAndRestartAfter) {
+			co_await delay(backupAfter);
+
+			TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(backupTag));
+			Promise<Void> submitted;
+			Future<Void> b =
+			    doBackup(0, &backupAgent, extraDB, backupTag, backupRanges, stopDifferentialAfter, submitted);
+
+			if (abortAndRestartAfter) {
 				TraceEvent("BARW_DoBackup2", randomID)
-				    .detail("Tag", printable(self->backupTag))
-				    .detail("AbortWait", self->abortAndRestartAfter);
-				wait(submitted.getFuture());
+				    .detail("Tag", printable(backupTag))
+				    .detail("AbortWait", abortAndRestartAfter);
+				co_await submitted.getFuture();
 
-				b = b && doBackup(self,
-				                  self->abortAndRestartAfter,
+				b = b && doBackup(abortAndRestartAfter,
 				                  &backupAgent,
-				                  self->extraDB,
-				                  self->backupTag,
-				                  self->backupRanges,
-				                  self->stopDifferentialAfter,
+				                  extraDB,
+				                  backupTag,
+				                  backupRanges,
+				                  stopDifferentialAfter,
 				                  Promise<Void>());
 			}
 
 			TraceEvent("BARW_DoBackupWait", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
-			wait(b);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
+			co_await b;
 			TraceEvent("BARW_DoBackupDone", randomID)
-			    .detail("BackupTag", printable(self->backupTag))
-			    .detail("AbortAndRestartAfter", self->abortAndRestartAfter);
+			    .detail("BackupTag", printable(backupTag))
+			    .detail("AbortAndRestartAfter", abortAndRestartAfter);
 
-			state UID logUid = wait(backupAgent.getLogUid(self->extraDB, self->backupTag));
+			UID logUid = co_await backupAgent.getLogUid(extraDB, backupTag);
 
 			// Occasionally start yet another backup that might still be running when we restore
-			if (!self->locked && self->extraPrefix != self->backupPrefix && BUGGIFY) {
-				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
+			if (!locked && extraPrefix != backupPrefix && BUGGIFY) {
+				TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(backupTag));
 				try {
-					extraBackup = backupAgent.submitBackup(self->extraDB,
-					                                       self->backupTag,
-					                                       self->backupRanges,
+					extraBackup = backupAgent.submitBackup(extraDB,
+					                                       backupTag,
+					                                       backupRanges,
 					                                       StopWhenDone::True,
-					                                       self->extraPrefix,
+					                                       extraPrefix,
 					                                       StringRef(),
-					                                       self->locked,
+					                                       locked,
 					                                       DatabaseBackupAgent::PreBackupAction::CLEAR);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)
-					    .detail("BackupTag", printable(self->backupTag));
+					    .detail("BackupTag", printable(backupTag));
 					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 						throw;
 				}
 			}
 
 			CODE_PROBE(!startRestore.isReady(), "Restore starts at specified time");
-			wait(startRestore);
+			co_await startRestore;
 
-			if (self->performRestore) {
+			if (performRestore) {
 				// restore database
 				TraceEvent("BARW_Restore", randomID)
-				    .detail("RestoreAfter", self->restoreAfter)
-				    .detail("BackupTag", printable(self->restoreTag));
-				// wait(diffRanges(self->backupRanges, self->backupPrefix, cx, self->extraDB));
+				    .detail("RestoreAfter", restoreAfter)
+				    .detail("BackupTag", printable(restoreTag));
+				// wait(diffRanges(backupRanges, backupPrefix, cx, extraDB));
 
-				state Standalone<VectorRef<KeyRangeRef>> restoreRange;
-				state Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
-				for (auto r : self->backupRanges) {
+				Standalone<VectorRef<KeyRangeRef>> restoreRange;
+				Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
+				for (auto r : backupRanges) {
 					if (!r.intersects(getSystemBackupRanges())) {
 						restoreRange.push_back_deep(
 						    restoreRange.arena(),
-						    KeyRangeRef(r.begin.withPrefix(self->backupPrefix), r.end.withPrefix(self->backupPrefix)));
+						    KeyRangeRef(r.begin.withPrefix(backupPrefix), r.end.withPrefix(backupPrefix)));
 					} else {
 						KeyRangeRef normalKeyRange = r & normalKeys;
 						KeyRangeRef systemKeyRange = r & systemKeys;
 						if (!normalKeyRange.empty()) {
 							restoreRange.push_back_deep(restoreRange.arena(),
-							                            KeyRangeRef(normalKeyRange.begin.withPrefix(self->backupPrefix),
-							                                        normalKeyRange.end.withPrefix(self->backupPrefix)));
+							                            KeyRangeRef(normalKeyRange.begin.withPrefix(backupPrefix),
+							                                        normalKeyRange.end.withPrefix(backupPrefix)));
 						}
 						if (!systemKeyRange.empty()) {
 							systemRestoreRange.push_back_deep(systemRestoreRange.arena(), systemKeyRange);
@@ -682,16 +673,16 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 
 				// restore system keys first before restoring user data
 				if (!systemRestoreRange.empty()) {
-					state Key systemRestoreTag = "restore_system"_sr;
+					Key systemRestoreTag = "restore_system"_sr;
 					try {
-						wait(restoreTool.submitBackup(cx,
-						                              systemRestoreTag,
-						                              systemRestoreRange,
-						                              StopWhenDone::True,
-						                              StringRef(),
-						                              self->backupPrefix,
-						                              self->locked,
-						                              DatabaseBackupAgent::PreBackupAction::CLEAR));
+						co_await restoreTool.submitBackup(cx,
+						                                  systemRestoreTag,
+						                                  systemRestoreRange,
+						                                  StopWhenDone::True,
+						                                  StringRef(),
+						                                  backupPrefix,
+						                                  locked,
+						                                  DatabaseBackupAgent::PreBackupAction::CLEAR);
 					} catch (Error& e) {
 						TraceEvent("BARW_DoBackupSubmitBackupException", randomID)
 						    .error(e)
@@ -699,35 +690,35 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 						if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 							throw;
 					}
-					wait(success(restoreTool.waitBackup(cx, systemRestoreTag)));
-					wait(restoreTool.unlockBackup(cx, systemRestoreTag));
+					co_await restoreTool.waitBackup(cx, systemRestoreTag);
+					co_await restoreTool.unlockBackup(cx, systemRestoreTag);
 				}
 
 				try {
-					wait(restoreTool.submitBackup(cx,
-					                              self->restoreTag,
-					                              restoreRange,
-					                              StopWhenDone::True,
-					                              StringRef(),
-					                              self->backupPrefix,
-					                              self->locked,
-					                              DatabaseBackupAgent::PreBackupAction::CLEAR));
+					co_await restoreTool.submitBackup(cx,
+					                                  restoreTag,
+					                                  restoreRange,
+					                                  StopWhenDone::True,
+					                                  StringRef(),
+					                                  backupPrefix,
+					                                  locked,
+					                                  DatabaseBackupAgent::PreBackupAction::CLEAR);
 				} catch (Error& e) {
 					TraceEvent("BARW_DoBackupSubmitBackupException", randomID)
 					    .error(e)
-					    .detail("Tag", printable(self->restoreTag));
+					    .detail("Tag", printable(restoreTag));
 					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 						throw;
 				}
 
-				wait(success(restoreTool.waitBackup(cx, self->restoreTag)));
-				wait(restoreTool.unlockBackup(cx, self->restoreTag));
+				co_await restoreTool.waitBackup(cx, restoreTag);
+				co_await restoreTool.unlockBackup(cx, restoreTag);
 
 				// Make sure no more data is written to the restored range
 				// after the restore completes.
-				state std::vector<RangeResult> res1 = wait(readRanges(cx, restoreRange, self->backupPrefix));
-				wait(delay(5));
-				state std::vector<RangeResult> res2 = wait(readRanges(cx, restoreRange, self->backupPrefix));
+				std::vector<RangeResult> res1 = co_await readRanges(cx, restoreRange, backupPrefix);
+				co_await delay(5);
+				std::vector<RangeResult> res2 = co_await readRanges(cx, restoreRange, backupPrefix);
 				ASSERT(res1.size() == res2.size());
 				for (int i = 0; i < res1.size(); ++i) {
 					auto range1 = res1.at(i);
@@ -741,27 +732,27 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 			}
 
 			if (extraBackup.isValid()) {
-				TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(backupTag));
 				try {
-					wait(extraBackup);
+					co_await extraBackup;
 				} catch (Error& e) {
 					TraceEvent("BARW_ExtraBackupException", randomID)
 					    .error(e)
-					    .detail("BackupTag", printable(self->backupTag));
+					    .detail("BackupTag", printable(backupTag));
 					if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 						throw;
 				}
 
-				TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(backupTag));
 				try {
 					// This abort can race with submitBackup such that destUID may
 					// not be set yet. Adding "waitForDestUID" flag to avoid the race.
-					wait(backupAgent.abortBackup(self->extraDB,
-					                             self->backupTag,
-					                             PartialBackup::False,
-					                             AbortOldBackup::False,
-					                             DstOnly::False,
-					                             WaitForDestUID::True));
+					co_await backupAgent.abortBackup(extraDB,
+					                                 backupTag,
+					                                 PartialBackup::False,
+					                                 AbortOldBackup::False,
+					                                 DstOnly::False,
+					                                 WaitForDestUID::True);
 				} catch (Error& e) {
 					TraceEvent("BARW_AbortBackupExtraException", randomID).error(e);
 					if (e.code() != error_code_backup_unneeded)
@@ -769,33 +760,29 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
 				}
 			}
 
-			wait(checkData(
-			    self->extraDB, logUid, self->destUid, randomID, self->backupTag, &backupAgent, self->shareLogRange));
+			co_await checkData(extraDB, logUid, destUid, randomID, backupTag, &backupAgent, shareLogRange);
 
-			if (self->performRestore) {
-				state UID restoreUid = wait(backupAgent.getLogUid(self->extraDB, self->restoreTag));
-				wait(checkData(
-				    cx, restoreUid, restoreUid, randomID, self->restoreTag, &restoreTool, self->shareLogRange));
+			if (performRestore) {
+				UID restoreUid = co_await backupAgent.getLogUid(extraDB, restoreTag);
+				co_await checkData(cx, restoreUid, restoreUid, randomID, restoreTag, &restoreTool, shareLogRange);
 			}
 
-			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
+			TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(backupTag));
 
 			// Decrement the backup agent requests
-			if (self->agentRequest) {
+			if (agentRequest) {
 				BackupToDBCorrectnessWorkload::drAgentRequests--;
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) &&
+			if ((fdbSimulationPolicyState().drAgents == FDBBackupAgentType::BackupToDB) &&
 			    (!BackupToDBCorrectnessWorkload::drAgentRequests)) {
-				g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				fdbSimulationPolicyState().drAgents = FDBBackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestoreCorrectness").error(e);
 			throw;
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp b/fdbserver/workloads/BackupToDBUpgrade.cpp
similarity index 64%
rename from fdbserver/workloads/BackupToDBUpgrade.actor.cpp
rename to fdbserver/workloads/BackupToDBUpgrade.cpp
index f6092dc3ec7..8432e063231 100644
--- a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp
+++ b/fdbserver/workloads/BackupToDBUpgrade.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupToDBUpgrade.actor.cpp
+ * BackupToDBUpgrade.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,12 @@
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
+#include "fdbclient/ManagementAPI.h"
 #include "flow/ApiVersion.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // TODO: explain the purpose of this workload and how it different from the
 // 20+ (literally) other backup/restore workloads.
@@ -44,7 +43,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 		out.insert({ "RandomRangeLock" });
 	}
 
-	BackupToDBUpgradeWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit BackupToDBUpgradeWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		backupAfter = getOption(options, "backupAfter"_sr, deterministicRandom()->random01() * 10.0);
 		backupPrefix = getOption(options, "backupPrefix"_sr, StringRef());
 		backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
@@ -55,7 +54,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 		extraPrefix = backupPrefix.withPrefix("\xfe\xff\xfe"_sr);
 		backupPrefix = backupPrefix.withPrefix("\xfe\xff\xff"_sr);
 
-		ASSERT(backupPrefix != StringRef());
+		ASSERT(!backupPrefix.empty());
 
 		KeyRef beginRange;
 		KeyRef endRange;
@@ -86,8 +85,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 			}
 		}
 
-		ASSERT(g_simulator->extraDatabases.size() == 1);
-		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+		ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 
 		TraceEvent("DRU_Start").log();
 	}
@@ -95,43 +94,44 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 	Future<Void> setup(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _setup(cx, this);
+		return _setup(cx);
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> doBackup(BackupToDBUpgradeWorkload* self,
-	                                   DatabaseBackupAgent* backupAgent,
-	                                   Database cx,
-	                                   Key tag,
-	                                   Standalone<VectorRef<KeyRangeRef>> backupRanges) {
+	Future<Void> doBackup(DatabaseBackupAgent* backupAgent,
+	                      Database cx,
+	                      Key tag,
+	                      Standalone<VectorRef<KeyRangeRef>> backupRanges) {
 		try {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->extraDB));
-			loop {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(extraDB));
+			while (true) {
+				Error err;
 				try {
-					for (auto r : self->backupRanges) {
+					for (auto r : backupRanges) {
 						if (!r.empty()) {
-							auto targetRange = r.withPrefix(self->backupPrefix);
+							auto targetRange = r.withPrefix(backupPrefix);
 							printf("Clearing %s in destination\n", printable(targetRange).c_str());
 							tr->addReadConflictRange(targetRange);
 							tr->clear(targetRange);
 						}
 					}
-					wait(backupAgent->submitBackup(
-					    tr, tag, backupRanges, StopWhenDone::False, self->backupPrefix, StringRef()));
-					wait(tr->commit());
+					co_await backupAgent->submitBackup(
+					    tr, tag, backupRanges, StopWhenDone::False, backupPrefix, StringRef());
+					co_await tr->commit();
 					break;
 				} catch (Error& e) {
-					wait(tr->onError(e));
+					err = e;
 				}
+				co_await tr->onError(err);
 			}
 
 			TraceEvent("DRU_DoBackupInDifferentialMode").detail("Tag", printable(tag));
@@ -142,37 +142,32 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 			}
 		}
 
-		wait(success(backupAgent->waitBackup(self->extraDB, tag, StopWhenDone::False)));
-
-		return Void();
+		co_await backupAgent->waitBackup(extraDB, tag, StopWhenDone::False);
 	}
 
-	ACTOR static Future<Void> checkData(Database cx,
-	                                    UID logUid,
-	                                    UID destUid,
-	                                    Key tag,
-	                                    DatabaseBackupAgent* backupAgent) {
-		state Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
-		state Key backupLogValuesKey = uidPrefixKey(backupLogKeys.begin, destUid);
-		state Key backupLatestVersionsPath = uidPrefixKey(backupLatestVersionsPrefix, destUid);
-		state Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
-		state int displaySystemKeys = 0;
+	static Future<Void> checkData(Database cx, UID logUid, UID destUid, Key tag, DatabaseBackupAgent* backupAgent) {
+		Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
+		Key backupLogValuesKey = uidPrefixKey(backupLogKeys.begin, destUid);
+		Key backupLatestVersionsPath = uidPrefixKey(backupLatestVersionsPrefix, destUid);
+		Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
+		int displaySystemKeys = 0;
 
 		ASSERT(destUid.isValid());
 
 		// Ensure that there is no left over key within the backup subspace
-		loop {
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
 			TraceEvent("DRU_CheckLeftoverkeys").detail("BackupTag", printable(tag));
 
+			Error err;
 			try {
 				// Check the left over tasks
 				// We have to wait for the list to empty since an abort and get status
 				// can leave extra tasks in the queue
 				TraceEvent("DRU_CheckLeftovertasks").detail("BackupTag", printable(tag));
-				state int64_t taskCount = wait(backupAgent->getTaskCount(tr));
-				state int waitCycles = 0;
+				int64_t taskCount = co_await backupAgent->getTaskCount(tr);
+				int waitCycles = 0;
 
 				if ((taskCount) && false) {
 					TraceEvent("DRU_EndingNonzeroTaskCount")
@@ -180,7 +175,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 					    .detail("TaskCount", taskCount)
 					    .detail("WaitCycles", waitCycles);
 					printf("EndingNonZeroTasks: %ld\n", (long)taskCount);
-					wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
+					co_await TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef());
 				}
 
 				while (taskCount > 0) {
@@ -192,16 +187,16 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 					    .detail("WaitCycles", waitCycles);
 					printf("%.6f Wait #%4d for %lld tasks to end\n", now(), waitCycles, (long long)taskCount);
 
-					wait(delay(20.0));
-					tr = Reference<ReadYourWritesTransaction>(new ReadYourWritesTransaction(cx));
-					wait(store(taskCount, backupAgent->getTaskCount(tr)));
+					co_await delay(20.0);
+					tr = makeReference<ReadYourWritesTransaction>(cx);
+					taskCount = co_await backupAgent->getTaskCount(tr);
 				}
 
 				RangeResult agentValues =
-				    wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
+				    co_await tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100);
 
 				// Error if the system keyspace for the backup tag is not empty
-				if (agentValues.size() > 0) {
+				if (!agentValues.empty()) {
 					displaySystemKeys++;
 					printf("BackupCorrectnessLeftoverMutationKeys: (%d) %s\n",
 					       agentValues.size(),
@@ -222,7 +217,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 					printf("No left over backup agent configuration keys\n");
 				}
 
-				Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
+				Optional<Value> latestVersion = co_await tr->get(backupLatestVersionsKey);
 				if (latestVersion.present()) {
 					TraceEvent(SevError, "BackupCorrectnessLeftoverVersionKey")
 					    .detail("BackupTag", printable(tag))
@@ -232,14 +227,14 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 					printf("No left over backup version key\n");
 				}
 
-				RangeResult versions = wait(
-				    tr->getRange(KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1));
-				if (!versions.size()) {
-					RangeResult logValues =
-					    wait(tr->getRange(KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100));
+				RangeResult versions = co_await tr->getRange(
+				    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1);
+				if (versions.empty()) {
+					RangeResult logValues = co_await tr->getRange(
+					    KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100);
 
 					// Error if the log/mutation keyspace for the backup tag is not empty
-					if (logValues.size() > 0) {
+					if (!logValues.empty()) {
 						displaySystemKeys++;
 						printf("BackupCorrectnessLeftoverLogKeys: (%d) %s\n",
 						       logValues.size(),
@@ -264,68 +259,65 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 
 				break;
 			} catch (Error& e) {
-				TraceEvent("DRU_CheckError").error(e);
-				wait(tr->onError(e));
+				err = e;
 			}
+			TraceEvent("DRU_CheckError").error(err);
+			co_await tr->onError(err);
 		}
 
 		if (displaySystemKeys) {
-			wait(TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef()));
+			co_await TaskBucket::debugPrintRange(cx, "\xff"_sr, StringRef());
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _setup(Database cx, BackupToDBUpgradeWorkload* self) {
-		state DatabaseBackupAgent backupAgent(cx);
+	Future<Void> _setup(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
 
 		if (BUGGIFY) {
 			for (auto r : getSystemBackupRanges()) {
-				self->backupRanges.push_back_deep(self->backupRanges.arena(), r);
+				backupRanges.push_back_deep(backupRanges.arena(), r);
 			}
 		}
 
 		try {
-			wait(delay(self->backupAfter));
+			co_await delay(backupAfter);
 
-			TraceEvent("DRU_DoBackup").detail("Tag", printable(self->backupTag));
-			state Future<Void> b = doBackup(self, &backupAgent, self->extraDB, self->backupTag, self->backupRanges);
+			TraceEvent("DRU_DoBackup").detail("Tag", printable(backupTag));
+			Future<Void> b = doBackup(&backupAgent, extraDB, backupTag, backupRanges);
 
-			TraceEvent("DRU_DoBackupWait").detail("BackupTag", printable(self->backupTag));
-			wait(b);
-			TraceEvent("DRU_DoBackupWaitEnd").detail("BackupTag", printable(self->backupTag));
+			TraceEvent("DRU_DoBackupWait").detail("BackupTag", printable(backupTag));
+			co_await b;
+			TraceEvent("DRU_DoBackupWaitEnd").detail("BackupTag", printable(backupTag));
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupToDBUpgradeSetupError").error(e);
 			throw;
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> diffRanges(Standalone<VectorRef<KeyRangeRef>> ranges,
-	                                     StringRef backupPrefix,
-	                                     Database src,
-	                                     Database dest) {
-		state int rangeIndex;
-		for (rangeIndex = 0; rangeIndex < ranges.size(); ++rangeIndex) {
-			state KeyRangeRef range = ranges[rangeIndex];
-			state Key begin = range.begin;
+	static Future<Void> diffRanges(Standalone<VectorRef<KeyRangeRef>> ranges,
+	                               StringRef backupPrefix,
+	                               Database src,
+	                               Database dest) {
+		for (int rangeIndex = 0; rangeIndex < ranges.size(); ++rangeIndex) {
+			KeyRangeRef range = ranges[rangeIndex];
+			Key begin = range.begin;
 			if (range.empty()) {
 				continue;
 			}
-			loop {
-				state Transaction tr(src);
-				state Transaction tr2(dest);
+			while (true) {
+				Transaction tr(src);
+				Transaction tr2(dest);
+				Error err;
 				try {
-					loop {
+					while (true) {
 						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 						tr.setOption(FDBTransactionOptions::RAW_ACCESS);
 						tr2.setOption(FDBTransactionOptions::LOCK_AWARE);
 						tr2.setOption(FDBTransactionOptions::RAW_ACCESS);
-						state Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
-						state Future<RangeResult> bkpFuture =
+						Future<RangeResult> srcFuture = tr.getRange(KeyRangeRef(begin, range.end), 1000);
+						Future<RangeResult> bkpFuture =
 						    tr2.getRange(KeyRangeRef(begin, range.end).withPrefix(backupPrefix), 1000);
-						wait(success(srcFuture) && success(bkpFuture));
+						co_await (success(srcFuture) && success(bkpFuture));
 
 						auto src = srcFuture.get().begin();
 						auto bkp = bkpFuture.get().begin();
@@ -385,94 +377,101 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, BackupToDBUpgradeWorkload* self) {
-		state DatabaseBackupAgent backupAgent(cx);
-		state DatabaseBackupAgent restoreTool(self->extraDB);
-		state Standalone<VectorRef<KeyRangeRef>> prevBackupRanges;
-		state UID logUid;
-		state Version commitVersion;
+	Future<Void> _start(Database cx) {
+		DatabaseBackupAgent backupAgent(cx);
+		DatabaseBackupAgent restoreTool(extraDB);
+		Standalone<VectorRef<KeyRangeRef>> prevBackupRanges;
+		UID logUid;
+		Version commitVersion{ 0 };
 
-		state Future<Void> stopDifferential = delay(self->stopDifferentialAfter);
-		state Future<Void> waitUpgrade = backupAgent.waitUpgradeToLatestDrVersion(self->extraDB, self->backupTag);
-		wait(success(stopDifferential) && success(waitUpgrade));
-		TraceEvent("DRU_WaitDifferentialEnd").detail("Tag", printable(self->backupTag));
+		Future<Void> stopDifferential = delay(stopDifferentialAfter);
+		Future<Void> waitUpgrade = backupAgent.waitUpgradeToLatestDrVersion(extraDB, backupTag);
+		co_await (success(stopDifferential) && success(waitUpgrade));
+		TraceEvent("DRU_WaitDifferentialEnd").detail("Tag", printable(backupTag));
 
 		try {
 			// Get restore ranges before aborting
-			state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(self->extraDB));
-			loop {
+			Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(extraDB));
+			while (true) {
+				Error err;
 				try {
 					// Get backup ranges
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					UID _logUid = wait(backupAgent.getLogUid(tr, self->backupTag));
+					UID _logUid = co_await backupAgent.getLogUid(tr, backupTag);
 					logUid = _logUid;
 
 					Optional<Key> backupKeysPacked =
-					    wait(tr->get(backupAgent.config.get(BinaryWriter::toValue(logUid, Unversioned()))
-					                     .pack(BackupAgentBase::keyConfigBackupRanges)));
+					    co_await tr->get(backupAgent.config.get(BinaryWriter::toValue(logUid, Unversioned()))
+					                         .pack(BackupAgentBase::keyConfigBackupRanges));
 					ASSERT(backupKeysPacked.present());
 
 					BinaryReader br(backupKeysPacked.get(), IncludeVersion());
 					prevBackupRanges = Standalone<VectorRef<KeyRangeRef>>();
 					br >> prevBackupRanges;
-					wait(lockDatabase(tr, logUid));
+					co_await lockDatabase(tr, logUid);
 					tr->addWriteConflictRange(singleKeyRange(StringRef()));
-					wait(tr->commit());
+					co_await tr->commit();
 					commitVersion = tr->getCommittedVersion();
 					break;
 				} catch (Error& e) {
-					TraceEvent("DRU_GetRestoreRangeError").error(e);
-					wait(tr->onError(e));
+					err = e;
 				}
+				TraceEvent("DRU_GetRestoreRangeError").error(err);
+				co_await tr->onError(err);
 			}
 
 			TraceEvent("DRU_Locked").detail("LockedVersion", commitVersion);
 
 			// Wait for the destination to apply mutations up to the lock commit before switching over.
-			state ReadYourWritesTransaction versionCheckTr(self->extraDB);
-			loop {
+			ReadYourWritesTransaction versionCheckTr(extraDB);
+			while (true) {
+				Error err;
 				try {
 					versionCheckTr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					versionCheckTr.setOption(FDBTransactionOptions::LOCK_AWARE);
-					Optional<Value> v = wait(versionCheckTr.get(
-					    BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin)));
+					Optional<Value> v = co_await versionCheckTr.get(
+					    BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin));
 					TraceEvent("DRU_Applied")
 					    .detail("AppliedVersion",
 					            v.present() ? BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) : -1);
 					if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >= commitVersion)
 						break;
 
-					state Future<Void> versionWatch = versionCheckTr.watch(
+					Future<Void> versionWatch = versionCheckTr.watch(
 					    BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin));
-					wait(versionCheckTr.commit());
-					wait(versionWatch);
+					co_await versionCheckTr.commit();
+					co_await versionWatch;
 					versionCheckTr.reset();
 				} catch (Error& e) {
-					TraceEvent("DRU_GetAppliedVersionError").error(e);
-					wait(versionCheckTr.onError(e));
+					err = e;
+				}
+				if (!err.isValid()) {
+					continue;
 				}
+				TraceEvent("DRU_GetAppliedVersionError").error(err);
+				co_await versionCheckTr.onError(err);
 			}
 
 			TraceEvent("DRU_DiffRanges").log();
-			wait(diffRanges(prevBackupRanges, self->backupPrefix, cx, self->extraDB));
+			co_await diffRanges(prevBackupRanges, backupPrefix, cx, extraDB);
 
 			// abort backup
-			TraceEvent("DRU_AbortBackup").detail("Tag", printable(self->backupTag));
-			wait(backupAgent.abortBackup(self->extraDB, self->backupTag));
-			wait(unlockDatabase(self->extraDB, logUid));
+			TraceEvent("DRU_AbortBackup").detail("Tag", printable(backupTag));
+			co_await backupAgent.abortBackup(extraDB, backupTag);
+			co_await unlockDatabase(extraDB, logUid);
 
 			// restore database
-			TraceEvent("DRU_PrepareRestore").detail("RestoreTag", printable(self->restoreTag));
-			state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
-			loop {
+			TraceEvent("DRU_PrepareRestore").detail("RestoreTag", printable(restoreTag));
+			Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
+			while (true) {
+				Error err;
 				try {
 					tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr2->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -483,50 +482,49 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
 							tr2->clear(r);
 						}
 					}
-					wait(tr2->commit());
+					co_await tr2->commit();
 					break;
 				} catch (Error& e) {
-					TraceEvent("DRU_RestoreSetupError").errorUnsuppressed(e);
-					wait(tr2->onError(e));
+					err = e;
 				}
+				TraceEvent("DRU_RestoreSetupError").errorUnsuppressed(err);
+				co_await tr2->onError(err);
 			}
 
-			state Standalone<VectorRef<KeyRangeRef>> restoreRanges;
+			Standalone<VectorRef<KeyRangeRef>> restoreRanges;
 			for (auto r : prevBackupRanges) {
 				restoreRanges.push_back_deep(
 				    restoreRanges.arena(),
-				    KeyRangeRef(r.begin.withPrefix(self->backupPrefix), r.end.withPrefix(self->backupPrefix)));
+				    KeyRangeRef(r.begin.withPrefix(backupPrefix), r.end.withPrefix(backupPrefix)));
 			}
 
 			// start restoring db
 			try {
-				TraceEvent("DRU_RestoreDb").detail("RestoreTag", printable(self->restoreTag));
-				wait(restoreTool.submitBackup(
-				    cx, self->restoreTag, restoreRanges, StopWhenDone::True, StringRef(), self->backupPrefix));
+				TraceEvent("DRU_RestoreDb").detail("RestoreTag", printable(restoreTag));
+				co_await restoreTool.submitBackup(
+				    cx, restoreTag, restoreRanges, StopWhenDone::True, StringRef(), backupPrefix);
 			} catch (Error& e) {
-				TraceEvent("DRU_RestoreSubmitBackupError").error(e).detail("Tag", printable(self->restoreTag));
+				TraceEvent("DRU_RestoreSubmitBackupError").error(e).detail("Tag", printable(restoreTag));
 				if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 					throw;
 			}
 
-			wait(success(restoreTool.waitBackup(cx, self->restoreTag)));
-			wait(restoreTool.unlockBackup(cx, self->restoreTag));
-			wait(checkData(self->extraDB, logUid, logUid, self->backupTag, &backupAgent));
+			co_await restoreTool.waitBackup(cx, restoreTag);
+			co_await restoreTool.unlockBackup(cx, restoreTag);
+			co_await checkData(extraDB, logUid, logUid, backupTag, &backupAgent);
 
-			state UID restoreUid = wait(restoreTool.getLogUid(cx, self->restoreTag));
-			wait(checkData(cx, restoreUid, restoreUid, self->restoreTag, &restoreTool));
+			UID restoreUid = co_await restoreTool.getLogUid(cx, restoreTag);
+			co_await checkData(cx, restoreUid, restoreUid, restoreTag, &restoreTool);
 
-			TraceEvent("DRU_Complete").detail("BackupTag", printable(self->backupTag));
+			TraceEvent("DRU_Complete").detail("BackupTag", printable(backupTag));
 
-			if (g_simulator->drAgents == ISimulator::BackupAgentType::BackupToDB) {
-				g_simulator->drAgents = ISimulator::BackupAgentType::NoBackupAgents;
+			if (fdbSimulationPolicyState().drAgents == FDBBackupAgentType::BackupToDB) {
+				fdbSimulationPolicyState().drAgents = FDBBackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestoreCorrectnessError").error(e);
 			throw;
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BulkDumping.actor.cpp b/fdbserver/workloads/BulkDumping.cpp
similarity index 76%
rename from fdbserver/workloads/BulkDumping.actor.cpp
rename to fdbserver/workloads/BulkDumping.cpp
index 67a5079c1aa..633fe3ff3de 100644
--- a/fdbserver/workloads/BulkDumping.actor.cpp
+++ b/fdbserver/workloads/BulkDumping.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkDumping.actor.cpp
+ * BulkDumping.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,17 +21,16 @@
 #include "fdbclient/BulkDumping.h"
 #include "fdbclient/BulkLoading.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/MockS3Server.h"
-#include "fdbserver/MockS3ServerChaos.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/mocks3/MockS3Server.h"
+#include "fdbserver/mocks3/MockS3ServerChaos.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Platform.h"
 #include "fdbrpc/simulator.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 const std::string simulationBulkDumpFolder = joinPath("simfdb", "bulkdump");
 
@@ -72,7 +71,7 @@ struct BulkDumping : TestWorkload {
 		             "BulkLoading" });
 	}
 
-	BulkDumping(WorkloadContext const& wcx)
+	explicit BulkDumping(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), enabled(true), pass(true), cancelTimes(0),
 	    maxCancelTimes(getOption(options, "maxCancelTimes"_sr, deterministicRandom()->randomInt(0, 2))),
 	    bulkLoadTransportMethod(
@@ -96,8 +95,6 @@ struct BulkDumping : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return _setup(this, cx); }
 
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
@@ -110,7 +107,7 @@ struct BulkDumping : TestWorkload {
 	}
 
 	KeyRange getRandomRange(BulkDumping* self, KeyRange maxRange) const {
-		loop {
+		while (true) {
 			Standalone<StringRef> keyA = self->getRandomStringRef();
 			Standalone<StringRef> keyB = self->getRandomStringRef();
 			if (!maxRange.contains(keyA) || !maxRange.contains(keyB)) {
@@ -140,27 +137,29 @@ struct BulkDumping : TestWorkload {
 		return kvs; // ordered
 	}
 
-	ACTOR Future<Void> setKeys(Database cx, std::map<Key, Value> kvs) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> setKeys(Database cx, std::map<Key, Value> kvs) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				for (const auto& [key, value] : kvs) {
 					tr.set(key, value);
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent("BulkDumpingWorkLoadSetKey")
 				    .detail("KeyCount", kvs.size())
 				    .detail("Version", tr.getCommittedVersion());
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> waitUntilDumpJobComplete(BulkDumping* self, Database cx) {
-		state double startTime = now();
-		loop {
+	Future<Void> waitUntilDumpJobComplete(BulkDumping* self, Database cx) {
+		double startTime = now();
+		while (true) {
 			// Check for timeout to prevent infinite waiting
 			if (now() - startTime > self->jobCompletionTimeout) {
 				TraceEvent(SevWarnAlways, "BulkDumpingWorkLoadDumpJobTimeout").detail("WaitTime", now() - startTime);
@@ -168,28 +167,32 @@ struct BulkDumping : TestWorkload {
 				break;
 			}
 			// Create a fresh transaction each iteration to avoid stale reads
-			state Transaction tr(cx);
+			Transaction tr(cx);
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<BulkDumpState> aliveJob = wait(getSubmittedBulkDumpJob(&tr));
+				Optional<BulkDumpState> aliveJob = co_await getSubmittedBulkDumpJob(&tr);
 				if (!aliveJob.present()) {
 					break;
 				}
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid() && err.code() == error_code_actor_cancelled) {
+				throw err;
 			}
-			wait(delay(30.0));
+			if (err.isValid()) {
+				co_await tr.onError(err);
+			}
+			co_await delay(30.0);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> clearDatabase(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> clearDatabase(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -198,54 +201,59 @@ struct BulkDumping : TestWorkload {
 				tr.clear(bulkLoadJobKeys);
 				tr.clear(bulkLoadTaskKeys);
 				tr.clear(bulkLoadJobHistoryKeys);
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> clearRangeData(Database cx, KeyRange range) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> clearRangeData(Database cx, KeyRange range) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.clear(range);
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
 	// Return error tasks
 	// If allowIntermediateStates is true, tasks in Running/Complete phases are allowed (for timeout scenarios)
-	ACTOR Future<std::vector<BulkLoadTaskState>> validateAllBulkLoadTaskAcknowledgedOrError(
+	Future<std::vector<BulkLoadTaskState>> validateAllBulkLoadTaskAcknowledgedOrError(
 	    Database cx,
 	    UID jobId,
 	    KeyRange jobRange,
 	    bool allowIntermediateStates = false) {
-		state RangeResult rangeResult;
-		state Key beginKey = jobRange.begin;
-		state Key endKey = jobRange.end;
-		state Transaction tr(cx);
-		state std::vector<BulkLoadTaskState> errorTasks;
+		RangeResult rangeResult;
+		Key beginKey = jobRange.begin;
+		Key endKey = jobRange.end;
+		Transaction tr(cx);
+		std::vector<BulkLoadTaskState> errorTasks;
 		TraceEvent("BulkDumpingWorkLoad")
 		    .detail("Phase", "ValidateAllBulkLoadTaskAcknowledgedOrError")
 		    .detail("Job", jobId.toString())
 		    .detail("Range", jobRange)
 		    .detail("AllowIntermediateStates", allowIntermediateStates);
 		while (beginKey < endKey) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				rangeResult.clear();
-				wait(store(rangeResult, krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey))));
-				for (int i = 0; i < rangeResult.size() - 1; ++i) {
+				rangeResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey));
+				if (rangeResult.empty()) {
+					break;
+				}
+				for (int i = 0; i < static_cast<int>(rangeResult.size()) - 1; ++i) {
 					if (rangeResult[i].value.empty()) {
 						continue;
 					}
@@ -275,18 +283,21 @@ struct BulkDumping : TestWorkload {
 				}
 				beginKey = rangeResult.back().key;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
-		return errorTasks;
+		co_return errorTasks;
 	}
 
-	ACTOR Future<std::vector<BulkLoadTaskState>> waitUntilLoadJobCompleteOrError(BulkDumping* self,
-	                                                                             Database cx,
-	                                                                             UID jobId,
-	                                                                             KeyRange jobRange) {
-		state double startTime = now();
-		loop {
+	Future<std::vector<BulkLoadTaskState>> waitUntilLoadJobCompleteOrError(BulkDumping* self,
+	                                                                       Database cx,
+	                                                                       UID jobId,
+	                                                                       KeyRange jobRange) {
+		double startTime = now();
+		while (true) {
 			// Check for timeout to prevent infinite waiting
 			if (now() - startTime > self->jobCompletionTimeout) {
 				TraceEvent(SevWarnAlways, "BulkDumpingWorkLoadJobTimeout")
@@ -295,11 +306,11 @@ struct BulkDumping : TestWorkload {
 				    .detail("WaitTime", now() - startTime);
 				// Timeout: validate what we have and return (allow intermediate states since job may still be running)
 				std::vector<BulkLoadTaskState> errorTasks =
-				    wait(self->validateAllBulkLoadTaskAcknowledgedOrError(cx, jobId, jobRange, true));
-				return errorTasks;
+				    co_await self->validateAllBulkLoadTaskAcknowledgedOrError(cx, jobId, jobRange, true);
+				co_return errorTasks;
 			}
 
-			Optional<BulkLoadJobState> runningJob = wait(getRunningBulkLoadJob(cx));
+			Optional<BulkLoadJobState> runningJob = co_await getRunningBulkLoadJob(cx);
 			if (runningJob.present()) {
 				ASSERT(runningJob.get().getJobId() == jobId);
 				// During the wait for the job completion, we may inject the job cancellation.
@@ -308,30 +319,30 @@ struct BulkDumping : TestWorkload {
 				// ensure the job can complete fast.
 				if (SERVER_KNOBS->BULKLOAD_SIM_FAILURE_INJECTION && self->cancelTimes < self->maxCancelTimes &&
 				    deterministicRandom()->random01() < 0.1) {
-					wait(cancelBulkLoadJob(cx, jobId));
+					co_await cancelBulkLoadJob(cx, jobId);
 					self->cancelTimes++; // Inject cancellation. Then the bulkload job should run again.
 					TraceEvent("BulkDumpingWorkLoad").detail("Phase", "Job Cancelled").detail("Job", jobId.toString());
-					wait(self->clearRangeData(cx, jobRange));
+					co_await self->clearRangeData(cx, jobRange);
 					TraceEvent("BulkDumpingWorkLoad")
 					    .detail("Phase", "Data Cleared")
 					    .detail("Job", jobId.toString())
 					    .detail("JobRange", jobRange);
-					return std::vector<BulkLoadTaskState>();
+					co_return std::vector<BulkLoadTaskState>();
 				}
-				wait(delay(10.0));
+				co_await delay(10.0);
 				continue;
 			}
 			std::vector<BulkLoadTaskState> errorTasks =
-			    wait(self->validateAllBulkLoadTaskAcknowledgedOrError(cx, jobId, jobRange));
-			return errorTasks;
+			    co_await self->validateAllBulkLoadTaskAcknowledgedOrError(cx, jobId, jobRange);
+			co_return errorTasks;
 		}
 	}
 
-	ACTOR Future<Void> validateBulkLoadJobHistory(Database cx,
-	                                              UID jobId,
-	                                              bool hasError,
-	                                              bool bulkDumpRangeContainBulkLoadRange) {
-		std::vector<BulkLoadJobState> jobHistory = wait(getBulkLoadJobFromHistory(cx));
+	Future<Void> validateBulkLoadJobHistory(Database cx,
+	                                        UID jobId,
+	                                        bool hasError,
+	                                        bool bulkDumpRangeContainBulkLoadRange) {
+		std::vector<BulkLoadJobState> jobHistory = co_await getBulkLoadJobFromHistory(cx);
 		Optional<BulkLoadJobState> jobInHistory;
 		for (const auto& job : jobHistory) {
 			ASSERT(job.isValid());
@@ -351,15 +362,15 @@ struct BulkDumping : TestWorkload {
 		} else {
 			ASSERT(jobInHistory.get().getPhase() == BulkLoadJobPhase::Complete);
 		}
-		return Void();
 	}
 
-	ACTOR Future<std::map<Key, Value>> getAllKVSFromDB(Database cx) {
-		state Transaction tr(cx);
-		state std::map<Key, Value> kvs;
-		loop {
+	Future<std::map<Key, Value>> getAllKVSFromDB(Database cx) {
+		Transaction tr(cx);
+		std::map<Key, Value> kvs;
+		while (true) {
+			Error err;
 			try {
-				RangeResult kvsRes = wait(tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult kvsRes = co_await tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!kvsRes.more);
 				kvs.clear();
 				for (auto& kv : kvsRes) {
@@ -368,10 +379,11 @@ struct BulkDumping : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return kvs;
+		co_return kvs;
 	}
 
 	bool keyContainedInRanges(const Key& key, const std::vector<KeyRange>& ranges) {
@@ -443,14 +455,14 @@ struct BulkDumping : TestWorkload {
 	// (9) Validate the loaded data in DB is same as the data in DB before dumping within the bulkdump job range and
 	// bulkload job range. Note that the bulkload job can be unretriable error. In this case, we ignore the error range;
 	// (10) Validate the bulk load job history.
-	ACTOR Future<Void> _start(BulkDumping* self, Database cx) {
-		if (self->clientId != 0) {
-			return Void();
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0) {
+			co_return;
 		}
 
 		// Cleanup any leftover state from previous test iterations BEFORE starting work
 		// This ensures we start clean even if a previous iteration timed out or crashed
-		if (self->bulkLoadTransportMethod == BulkLoadTransportMethod::BLOBSTORE && g_network->isSimulated()) {
+		if (bulkLoadTransportMethod == BulkLoadTransportMethod::BLOBSTORE && g_network->isSimulated()) {
 			// Clear MockS3Server storage to prevent memory accumulation over test iterations
 			clearMockS3Storage();
 		}
@@ -462,88 +474,87 @@ struct BulkDumping : TestWorkload {
 			disableConnectionFailures("BulkDumping");
 		}
 
-		state KeyRange bulkDumpJobRange =
-		    deterministicRandom()->coinflip() ? normalKeys : self->getRandomRange(self, normalKeys);
+		KeyRange bulkDumpJobRange = deterministicRandom()->coinflip() ? normalKeys : getRandomRange(this, normalKeys);
 
-		state bool bulkDumpRangeContainBulkLoadRange = true; // Will set to false if the bulk load job range is not
+		bool bulkDumpRangeContainBulkLoadRange = true; // Will set to false if the bulk load job range is not
 		// contained in the bulk dump job range. In this case, the bulk load job will be failed fast with error.
 		// So, when set to false, skip the processCheck() in the end of the workload.
 		// Also, check bulkload job history to ensure the job is failed with the expected error.
 
-		state std::map<Key, Value> kvs = self->generateOrderedKVS(self, normalKeys, 1000);
-		wait(self->setKeys(cx, kvs));
+		std::map<Key, Value> kvs = generateOrderedKVS(this, normalKeys, 1000);
+		co_await setKeys(cx, kvs);
 
 		// BulkLoad uses range lock
-		wait(registerRangeLockOwner(cx, rangeLockNameForBulkLoad, rangeLockNameForBulkLoad));
+		co_await registerRangeLockOwner(cx, rangeLockNameForBulkLoad, rangeLockNameForBulkLoad);
 
-		std::vector<RangeLockOwner> lockOwners = wait(getAllRangeLockOwners(cx));
+		std::vector<RangeLockOwner> lockOwners = co_await getAllRangeLockOwners(cx);
 		ASSERT(lockOwners.size() == 1 && lockOwners[0].getOwnerUniqueId() == rangeLockNameForBulkLoad);
 
 		// Submit a bulk dump job
-		state int oldBulkDumpMode = 0;
+		int oldBulkDumpMode = 0;
 		TraceEvent("BulkDumpingWorkLoad").detail("Phase", "Setting BulkDump Mode");
 		try {
-			wait(store(oldBulkDumpMode, timeoutError(setBulkDumpMode(cx, 1), self->modeSetTimeout))); // Enable bulkDump
+			oldBulkDumpMode = co_await timeoutError(setBulkDumpMode(cx, 1), modeSetTimeout); // Enable bulkDump
 			TraceEvent("BulkDumpingWorkLoad").detail("Phase", "BulkDump Mode Set").detail("OldMode", oldBulkDumpMode);
 		} catch (Error& e) {
 			if (e.code() == error_code_timed_out) {
 				TraceEvent(SevWarnAlways, "BulkDumpingWorkLoadSetDumpModeTimeout")
-				    .detail("TimeoutSeconds", self->modeSetTimeout);
+				    .detail("TimeoutSeconds", modeSetTimeout);
 				throw;
 			}
 			throw;
 		}
-		state std::string dumpFolder = self->jobRoot.empty() ? simulationBulkDumpFolder : self->jobRoot;
-		state BulkDumpState bulkDumpJob =
-		    createBulkDumpJob(bulkDumpJobRange, dumpFolder, BulkLoadType::SST, self->bulkLoadTransportMethod);
+		std::string dumpFolder = jobRoot.empty() ? simulationBulkDumpFolder : jobRoot;
+		BulkDumpState bulkDumpJob =
+		    createBulkDumpJob(bulkDumpJobRange, dumpFolder, BulkLoadType::SST, bulkLoadTransportMethod);
 		TraceEvent("BulkDumpingWorkLoad").detail("Phase", "Submitting Dump Job").detail("Job", bulkDumpJob.getJobId());
-		wait(timeoutError(submitBulkDumpJob(cx, bulkDumpJob), self->jobSubmitTimeout));
+		co_await timeoutError(submitBulkDumpJob(cx, bulkDumpJob), jobSubmitTimeout);
 		TraceEvent("BulkDumpingWorkLoad")
 		    .detail("Phase", "Dump Job Submitted")
-		    .detail("TransportMethod", convertBulkLoadTransportMethodToString(self->bulkLoadTransportMethod))
+		    .detail("TransportMethod", convertBulkLoadTransportMethodToString(bulkLoadTransportMethod))
 		    .detail("JobRoot", dumpFolder)
 		    .detail("Job", bulkDumpJob.toString());
 
 		// Wait until the dump job completes
-		wait(self->waitUntilDumpJobComplete(self, cx));
+		co_await waitUntilDumpJobComplete(this, cx);
 		TraceEvent("BulkDumpingWorkLoad").detail("Phase", "Dump Job Complete").detail("Job", bulkDumpJob.toString());
 
 		// Clear database
-		wait(self->clearDatabase(cx));
+		co_await clearDatabase(cx);
 		TraceEvent("BulkDumpingWorkLoad").detail("Phase", "Clear DB").detail("Job", bulkDumpJob.toString());
 
 		// Submit a bulk load job
-		state int oldBulkLoadMode = 0;
+		int oldBulkLoadMode = 0;
 		TraceEvent("BulkDumpingWorkLoad")
 		    .detail("Phase", "Setting BulkLoad Mode")
 		    .detail("Job", bulkDumpJob.toString());
 		try {
-			wait(store(oldBulkLoadMode, timeoutError(setBulkLoadMode(cx, 1), self->modeSetTimeout))); // Enable bulkLoad
+			oldBulkLoadMode = co_await timeoutError(setBulkLoadMode(cx, 1), modeSetTimeout); // Enable bulkLoad
 			TraceEvent("BulkDumpingWorkLoad").detail("Phase", "BulkLoad Mode Set").detail("OldMode", oldBulkLoadMode);
 		} catch (Error& e) {
 			if (e.code() == error_code_timed_out) {
 				TraceEvent(SevWarnAlways, "BulkDumpingWorkLoadSetModeTimeout")
 				    .detail("Job", bulkDumpJob.toString())
-				    .detail("TimeoutSeconds", self->modeSetTimeout);
+				    .detail("TimeoutSeconds", modeSetTimeout);
 				throw;
 			}
 			throw;
 		}
-		loop {
+		while (true) {
 			// We randomly injects the job cancellation when waiting for the job completion to test the job
 			// cancellation. If the job is cancelled, we should re-submit the job.
-			state bool hasError = false;
-			state int oldCancelTimes = self->cancelTimes;
-			state KeyRange bulkLoadJobRange =
+			bool hasError = false;
+			int oldCancelTimes = cancelTimes;
+			KeyRange bulkLoadJobRange =
 			    deterministicRandom()->coinflip()
 			        ? bulkDumpJob.getJobRange()
-			        : self->getRandomRange(self, deterministicRandom()->coinflip() ? normalKeys : bulkDumpJobRange);
-			state UID dataSourceId = bulkDumpJob.getJobId();
-			state std::string dataSourceRoot = bulkDumpJob.getJobRoot();
-			state BulkLoadJobState bulkLoadJob =
-			    createBulkLoadJob(dataSourceId, bulkLoadJobRange, dataSourceRoot, self->bulkLoadTransportMethod);
+			        : getRandomRange(this, deterministicRandom()->coinflip() ? normalKeys : bulkDumpJobRange);
+			UID dataSourceId = bulkDumpJob.getJobId();
+			std::string dataSourceRoot = bulkDumpJob.getJobRoot();
+			BulkLoadJobState bulkLoadJob =
+			    createBulkLoadJob(dataSourceId, bulkLoadJobRange, dataSourceRoot, bulkLoadTransportMethod);
 			TraceEvent("BulkDumpingWorkLoad").detail("Phase", "Submitting Load Job").detail("JobId", dataSourceId);
-			wait(timeoutError(submitBulkLoadJob(cx, bulkLoadJob), self->jobSubmitTimeout));
+			co_await timeoutError(submitBulkLoadJob(cx, bulkLoadJob), jobSubmitTimeout);
 			TraceEvent("BulkDumpingWorkLoad")
 			    .detail("Phase", "Load Job Submitted")
 			    .detail("JobId", dataSourceId)
@@ -552,15 +563,15 @@ struct BulkDumping : TestWorkload {
 			    .detail("TransportMethod", bulkLoadJob.getTransportMethod());
 
 			// Wait until the load job complete
-			state std::vector<KeyRange> errorRanges;
-			state std::vector<BulkLoadTaskState> errorTasks = wait(
-			    self->waitUntilLoadJobCompleteOrError(self, cx, bulkLoadJob.getJobId(), bulkLoadJob.getJobRange()));
-			// waitUntilLoadJobCompleteOrError can cancel the job and set self->cancelled to true.
+			std::vector<KeyRange> errorRanges;
+			std::vector<BulkLoadTaskState> errorTasks =
+			    co_await waitUntilLoadJobCompleteOrError(this, cx, bulkLoadJob.getJobId(), bulkLoadJob.getJobRange());
+			// waitUntilLoadJobCompleteOrError can cancel the job and set cancelled to true.
 			// If this happens, the current job is intentionally cancelled and we should retry the job.
-			ASSERT(self->cancelTimes >= oldCancelTimes);
-			if (self->cancelTimes > oldCancelTimes) {
-				// self->cancelTimes increments when waitUntilLoadJobCompleteOrError injects job cancellation
-				wait(delay(deterministicRandom()->random01() * 10.0));
+			ASSERT(cancelTimes >= oldCancelTimes);
+			if (cancelTimes > oldCancelTimes) {
+				// cancelTimes increments when waitUntilLoadJobCompleteOrError injects job cancellation
+				co_await delay(deterministicRandom()->random01() * 10.0);
 				continue;
 			}
 			for (const auto& errorTask : errorTasks) {
@@ -575,9 +586,9 @@ struct BulkDumping : TestWorkload {
 			    .detail("BulkLoadTransportMethod", bulkLoadJob.getTransportMethod());
 
 			// Check the loaded data in DB is same as the data in DB before dumping
-			std::map<Key, Value> newKvs = wait(self->getAllKVSFromDB(cx));
+			std::map<Key, Value> newKvs = co_await getAllKVSFromDB(cx);
 			if (bulkDumpJobRange.contains(bulkLoadJobRange)) {
-				self->processCheck(self, kvs, newKvs, bulkLoadJobRange, bulkDumpJobRange, errorRanges);
+				processCheck(this, kvs, newKvs, bulkLoadJobRange, bulkDumpJobRange, errorRanges);
 				bulkDumpRangeContainBulkLoadRange = true;
 			} else {
 				TraceEvent(SevWarnAlways, "BulkDumpingWorkLoad")
@@ -588,26 +599,24 @@ struct BulkDumping : TestWorkload {
 			}
 
 			// Acknowledge any error task of the job
-			wait(acknowledgeAllErrorBulkLoadTasks(cx, bulkLoadJob.getJobId(), bulkLoadJob.getJobRange()));
-			wait(self->validateBulkLoadJobHistory(
-			    cx, bulkLoadJob.getJobId(), hasError, bulkDumpRangeContainBulkLoadRange));
+			co_await acknowledgeAllErrorBulkLoadTasks(cx, bulkLoadJob.getJobId(), bulkLoadJob.getJobRange());
+			co_await validateBulkLoadJobHistory(
+			    cx, bulkLoadJob.getJobId(), hasError, bulkDumpRangeContainBulkLoadRange);
 			break;
 		}
 
 		// Make sure all ranges locked by the workload are unlocked
 		std::vector<std::pair<KeyRange, RangeLockState>> res =
-		    wait(findExclusiveReadLockOnRange(cx, normalKeys, rangeLockNameForBulkLoad));
+		    co_await findExclusiveReadLockOnRange(cx, normalKeys, rangeLockNameForBulkLoad);
 		ASSERT(res.empty());
 
-		wait(removeRangeLockOwner(cx, rangeLockNameForBulkLoad));
+		co_await removeRangeLockOwner(cx, rangeLockNameForBulkLoad);
 
-		std::vector<RangeLockOwner> lockOwnersAfterRemove = wait(getAllRangeLockOwners(cx));
+		std::vector<RangeLockOwner> lockOwnersAfterRemove = co_await getAllRangeLockOwners(cx);
 		ASSERT(lockOwnersAfterRemove.empty());
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _setup(BulkDumping* self, Database cx) {
+	Future<Void> _setup(BulkDumping* self, Database cx) {
 		// Only client 0 registers the MockS3Server to avoid duplicates
 		if (self->clientId == 0 && self->bulkLoadTransportMethod == BulkLoadTransportMethod::BLOBSTORE) {
 			// Check if we're using a local mock server URL pattern
@@ -618,7 +627,7 @@ struct BulkDumping : TestWorkload {
 			if (useMockS3 && g_network->isSimulated()) {
 				// Check if 127.0.0.1:8080 is already registered in simulator's httpHandlers
 				std::string serverKey = "127.0.0.1:8080";
-				bool alreadyRegistered = g_simulator->httpHandlers.count(serverKey) > 0;
+				bool alreadyRegistered = g_simulator->httpHandlers.contains(serverKey);
 
 				if (alreadyRegistered) {
 					TraceEvent("BulkDumpingWorkload")
@@ -633,7 +642,7 @@ struct BulkDumping : TestWorkload {
 
 					// Start MockS3ServerChaos - has internal duplicate detection
 					NetworkAddress listenAddress(IPAddress(0x7f000001), 8080);
-					wait(startMockS3ServerChaos(listenAddress));
+					co_await startMockS3ServerChaos(listenAddress);
 
 					TraceEvent("BulkDumpingWorkload")
 					    .detail("Phase", "MockS3ServerChaos Started")
@@ -644,7 +653,7 @@ struct BulkDumping : TestWorkload {
 					    .detail("JobRoot", self->jobRoot);
 
 					// Register MockS3Server with persistence enabled
-					wait(registerMockS3Server("127.0.0.1", "8080"));
+					co_await registerMockS3Server("127.0.0.1", "8080");
 
 					TraceEvent("BulkDumpingWorkload")
 					    .detail("Phase", "MockS3Server Registered")
@@ -671,8 +680,6 @@ struct BulkDumping : TestWorkload {
 			    .detail("DelayRate", self->delayRate)
 			    .detail("CorruptionRate", self->corruptionRate);
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/BulkLoad.actor.cpp b/fdbserver/workloads/BulkLoad.cpp
similarity index 86%
rename from fdbserver/workloads/BulkLoad.actor.cpp
rename to fdbserver/workloads/BulkLoad.cpp
index c83f4751684..b444887c901 100644
--- a/fdbserver/workloads/BulkLoad.actor.cpp
+++ b/fdbserver/workloads/BulkLoad.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkLoad.actor.cpp
+ * BulkLoad.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,9 +20,7 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct BulkLoadWorkload : TestWorkload {
 	static constexpr auto NAME = "BulkLoad";
@@ -36,7 +34,7 @@ struct BulkLoadWorkload : TestWorkload {
 	PerfIntCounter transactions, retries;
 	DDSketch<double> latencies;
 
-	BulkLoadWorkload(WorkloadContext const& wcx)
+	explicit BulkLoadWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), clientCount(wcx.clientCount), transactions("Transactions"), retries("Retries"), latencies() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		actorCount = getOption(options, "actorCount"_sr, 20);
@@ -76,14 +74,15 @@ struct BulkLoadWorkload : TestWorkload {
 		m.emplace_back("98% Latency (ms, averaged)", 1000 * latencies.percentile(0.98), Averaged::True);
 	}
 
-	ACTOR Future<Void> bulkLoadClient(Database cx, BulkLoadWorkload* self, int clientId, int actorId) {
-		state uint64_t totalBytes = 0;
-		state int idx = 0;
-		loop {
-			state double tstart = now();
-			state Transaction tr(cx);
-			loop {
-				state uint64_t txnBytes = 0;
+	Future<Void> bulkLoadClient(Database cx, BulkLoadWorkload* self, int clientId, int actorId) {
+		uint64_t totalBytes = 0;
+		int idx = 0;
+		while (true) {
+			double tstart = now();
+			Transaction tr(cx);
+			while (true) {
+				uint64_t txnBytes = 0;
+				Error err;
 				try {
 					for (int i = 0; i < self->writesPerTransaction; i++) {
 						std::string key = format("%s/bulkload/%04x/%04x/%08x",
@@ -95,20 +94,21 @@ struct BulkLoadWorkload : TestWorkload {
 						txnBytes += key.size() + self->value.size();
 					}
 					tr.makeSelfConflicting();
-					wait(success(tr.getReadVersion()));
-					wait(tr.commit());
+					co_await tr.getReadVersion();
+					co_await tr.commit();
 					totalBytes += txnBytes;
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
-					++self->retries;
+					err = e;
 				}
+				co_await tr.onError(err);
+				++self->retries;
 			}
 			self->latencies.addSample(now() - tstart);
 			++self->transactions;
 			idx += self->writesPerTransaction;
 			if (totalBytes > self->targetBytes / self->clientCount / self->actorCount)
-				return Void();
+				co_return;
 		}
 	}
 };
diff --git a/fdbserver/workloads/BulkLoading.actor.cpp b/fdbserver/workloads/BulkLoading.cpp
similarity index 72%
rename from fdbserver/workloads/BulkLoading.actor.cpp
rename to fdbserver/workloads/BulkLoading.cpp
index 888c6ae8cd5..2465e2085b0 100644
--- a/fdbserver/workloads/BulkLoading.actor.cpp
+++ b/fdbserver/workloads/BulkLoading.cpp
@@ -1,5 +1,5 @@
 /*
- * BulkLoading.actor.cpp
+ * BulkLoading.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,13 +19,13 @@
  */
 
 #include "fdbclient/BulkLoading.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/BulkLoadUtil.h"
+#include "fdbserver/core/RocksDBCheckpointUtils.h"
+#include "fdbserver/core/StorageMetrics.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 const std::string simulationBulkLoadFolder = joinPath("simfdb", "bulkload");
@@ -78,120 +78,129 @@ struct BulkLoading : TestWorkload {
 		             "BulkDumping" });
 	}
 
-	BulkLoading(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {}
+	explicit BulkLoading(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> clearAllBulkLoadTask(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> clearAllBulkLoadTask(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				wait(krmSetRange(&tr, bulkLoadTaskPrefix, normalKeys, bulkLoadTaskStateValue(BulkLoadTaskState())));
-				wait(tr.commit());
+				co_await krmSetRange(&tr, bulkLoadTaskPrefix, normalKeys, bulkLoadTaskStateValue(BulkLoadTaskState()));
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
 	// Submit task can be failed due to range lock reject
-	ACTOR Future<bool> submitBulkLoadTask(Database cx, BulkLoadTaskState bulkLoadTask) {
-		loop {
+	Future<bool> submitBulkLoadTask(Database cx, BulkLoadTaskState bulkLoadTask) {
+		while (true) {
+			Error err;
 			try {
-				state Transaction tr(cx);
-				wait(setBulkLoadSubmissionTransaction(&tr, bulkLoadTask));
-				wait(takeExclusiveReadLockOnRange(&tr, bulkLoadTask.getRange(), rangeLockNameForBulkLoad));
-				wait(tr.commit());
+				Transaction tr(cx);
+				co_await setBulkLoadSubmissionTransaction(&tr, bulkLoadTask);
+				co_await takeExclusiveReadLockOnRange(&tr, bulkLoadTask.getRange(), rangeLockNameForBulkLoad);
+				co_await tr.commit();
 				TraceEvent(SevDebug, "BulkLoadingSubmitBulkLoadTask")
 				    .detail("BulkLoadTaskState", bulkLoadTask.toString());
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevWarn, "BulkLoadingSubmitBulkLoadTaskError")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .errorUnsuppressed(e)
-				    .detail("BulkLoadTaskState", bulkLoadTask.toString());
-				if (e.code() == error_code_range_lock_reject) {
-					return false;
-				}
-				wait(delay(0.1));
+				err = e;
 			}
+			TraceEvent(SevWarn, "BulkLoadingSubmitBulkLoadTaskError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .errorUnsuppressed(err)
+			    .detail("BulkLoadTaskState", bulkLoadTask.toString());
+			if (err.code() == error_code_range_lock_reject) {
+				co_return false;
+			}
+			co_await delay(0.1);
 		}
-		return true;
+		co_return true;
 	}
 
 	// Finish task must always succeed
-	ACTOR Future<Void> finalizeBulkLoadTask(Database cx, KeyRange range, UID taskId) {
-		loop {
+	Future<Void> finalizeBulkLoadTask(Database cx, KeyRange range, UID taskId) {
+		while (true) {
+			Error err;
 			try {
-				state Transaction tr(cx);
-				wait(setBulkLoadFinalizeTransaction(&tr, range, taskId));
-				wait(releaseExclusiveReadLockOnRange(&tr, range, rangeLockNameForBulkLoad));
-				wait(tr.commit());
+				Transaction tr(cx);
+				co_await setBulkLoadFinalizeTransaction(&tr, range, taskId);
+				co_await releaseExclusiveReadLockOnRange(&tr, range, rangeLockNameForBulkLoad);
+				co_await tr.commit();
 				TraceEvent(SevDebug, "BulkLoadingAcknowledgeBulkLoadTask")
 				    .detail("TaskID", taskId.toString())
 				    .detail("TaskRange", range);
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevWarn, "BulkLoadingAcknowledgeBulkLoadTaskError")
-				    .errorUnsuppressed(e)
-				    .detail("TaskID", taskId.toString())
-				    .detail("TaskRange", range);
-				ASSERT(e.code() != error_code_bulkload_task_outdated && e.code() != error_code_range_lock_reject);
-				wait(delay(0.1));
+				err = e;
 			}
+			TraceEvent(SevWarn, "BulkLoadingAcknowledgeBulkLoadTaskError")
+			    .errorUnsuppressed(err)
+			    .detail("TaskID", taskId.toString())
+			    .detail("TaskRange", range);
+			ASSERT(err.code() != error_code_bulkload_task_outdated && err.code() != error_code_range_lock_reject);
+			co_await delay(0.1);
 		}
-		return Void();
 	}
 
-	ACTOR Future<bool> checkDDEnabled(Database cx) {
-		loop {
-			state Transaction tr(cx);
+	Future<bool> checkDDEnabled(Database cx) {
+		while (true) {
+			Transaction tr(cx);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			Error err;
 			try {
-				state int ddMode = 1;
-				Optional<Value> mode = wait(tr.get(dataDistributionModeKey));
+				int ddMode = 1;
+				Optional<Value> mode = co_await tr.get(dataDistributionModeKey);
 				if (mode.present()) {
 					BinaryReader rd(mode.get(), Unversioned());
 					rd >> ddMode;
 				}
-				return ddMode == 1;
+				co_return ddMode == 1;
 
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
 	// First return value is whether all tasks are complete or error.
 	// Second return value is the error tasks.
-	ACTOR Future<std::pair<bool, std::vector<BulkLoadTaskState>>> checkAllTaskCompleteOrError(Database cx) {
-		state Transaction tr(cx);
-		state Key beginKey = allKeys.begin;
-		state Key endKey = allKeys.end;
-		state std::vector<BulkLoadTaskState> errorTasks;
+	Future<std::pair<bool, std::vector<BulkLoadTaskState>>> checkAllTaskCompleteOrError(Database cx) {
+		Transaction tr(cx);
+		Key beginKey = allKeys.begin;
+		Key endKey = allKeys.end;
+		std::vector<BulkLoadTaskState> errorTasks;
 		while (beginKey < endKey) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				RangeResult res =
-				    wait(krmGetRanges(&tr, bulkLoadTaskPrefix, Standalone(KeyRangeRef(beginKey, endKey))));
-				for (int i = 0; i < res.size() - 1; i++) {
+				    co_await krmGetRanges(&tr, bulkLoadTaskPrefix, Standalone(KeyRangeRef(beginKey, endKey)));
+				if (res.empty()) {
+					break;
+				}
+				for (int i = 0; i < static_cast<int>(res.size()) - 1; i++) {
 					if (!res[i].value.empty()) {
 						BulkLoadTaskState bulkLoadTaskState = decodeBulkLoadTaskState(res[i].value);
 						if (!bulkLoadTaskState.isValid()) {
 							continue;
 						}
-						// We do not check manifest because we are not fully setting manifest in this simulation test
+						// We do not check manifest because we are not fully setting manifest in this simulation
+						// test
 						if (bulkLoadTaskState.getRange() != KeyRangeRef(res[i].key, res[i + 1].key)) {
 							continue; // Ignore outdated task
 						}
@@ -201,7 +210,7 @@ struct BulkLoading : TestWorkload {
 							    .setMaxEventLength(-1)
 							    .setMaxFieldLength(-1)
 							    .detail("Task", bulkLoadTaskState.toString());
-							return std::make_pair(false, errorTasks);
+							co_return std::make_pair(false, errorTasks);
 						}
 						if (bulkLoadTaskState.phase == BulkLoadPhase::Error) {
 							TraceEvent("BulkLoadingWorkLoadFailedTasks")
@@ -214,33 +223,40 @@ struct BulkLoading : TestWorkload {
 				}
 				beginKey = res[res.size() - 1].key;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
-		return std::make_pair(true, errorTasks);
+		co_return std::make_pair(true, errorTasks);
 	}
 
-	ACTOR Future<std::vector<BulkLoadTaskState>> waitUntilAllTaskCompleteOrError(BulkLoading* self, Database cx) {
-		loop {
-			std::pair<bool, std::vector<BulkLoadTaskState>> res = wait(self->checkAllTaskCompleteOrError(cx));
+	Future<std::vector<BulkLoadTaskState>> waitUntilAllTaskCompleteOrError(BulkLoading* self, Database cx) {
+		while (true) {
+			std::pair<bool, std::vector<BulkLoadTaskState>> res = co_await self->checkAllTaskCompleteOrError(cx);
 			if (res.first) { // If all tasks are complete or error
-				return res.second; // Return errorTasks
+				co_return res.second; // Return errorTasks
 			}
-			wait(delay(10.0));
+			co_await delay(10.0);
 		}
 	}
 
-	ACTOR Future<bool> checkBulkLoadMetadataCleared(BulkLoading* self, Database cx) {
-		state Key beginKey = normalKeys.begin;
-		state Key endKey = normalKeys.end;
-		state Transaction tr(cx);
+	Future<bool> checkBulkLoadMetadataCleared(BulkLoading* self, Database cx) {
+		Key beginKey = normalKeys.begin;
+		Key endKey = normalKeys.end;
+		Transaction tr(cx);
 		while (beginKey < endKey) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				RangeResult res = wait(krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey)));
+				RangeResult res = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(beginKey, endKey));
+				if (res.empty()) {
+					break;
+				}
 				int clearedCount = 0;
 				int nonEmptyCount = 0;
-				for (int i = 0; i < res.size() - 1; i++) {
+				for (int i = 0; i < static_cast<int>(res.size()) - 1; i++) {
 					ASSERT(!self->initializeBulkLoadMetadata || !res[i].value.empty());
 					if (res[i].value.empty()) {
 						continue;
@@ -256,7 +272,7 @@ struct BulkLoading : TestWorkload {
 						    .setMaxEventLength(-1)
 						    .setMaxFieldLength(-1)
 						    .detail("BulkLoadTask", bulkLoadTaskState.toString());
-						return false;
+						co_return false;
 					}
 					ASSERT(bulkLoadTaskState.getRange().contains(currentRange));
 					nonEmptyCount++;
@@ -268,10 +284,13 @@ struct BulkLoading : TestWorkload {
 				}
 				beginKey = res.back().key;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
-		return true;
+		co_return true;
 	}
 
 	bool keyContainedInRanges(const Key& key, const std::vector<KeyRange>& ranges) {
@@ -283,16 +302,17 @@ struct BulkLoading : TestWorkload {
 		return false;
 	}
 
-	ACTOR Future<std::vector<KeyValue>> getKvsFromDB(BulkLoading* self,
-	                                                 Database cx,
-	                                                 std::vector<KeyRange> ignoreRanges,
-	                                                 std::vector<KeyRange> loadedRanges) {
-		state std::vector<KeyValue> res;
-		state Transaction tr(cx);
+	Future<std::vector<KeyValue>> getKvsFromDB(BulkLoading* self,
+	                                           Database cx,
+	                                           std::vector<KeyRange> ignoreRanges,
+	                                           std::vector<KeyRange> loadedRanges) {
+		std::vector<KeyValue> res;
+		Transaction tr(cx);
 		TraceEvent("BulkLoadingWorkLoadGetKVSFromDBStart");
-		loop {
+		while (true) {
+			Error err;
 			try {
-				RangeResult result = wait(tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult result = co_await tr.getRange(normalKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!result.more);
 				for (int i = 0; i < result.size(); i++) {
 					if (self->keyContainedInRanges(result[i].key, ignoreRanges)) {
@@ -305,11 +325,12 @@ struct BulkLoading : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 		TraceEvent("BulkLoadingWorkLoadGetKVSFromDBDone");
-		return res;
+		co_return res;
 	}
 
 	Standalone<StringRef> getRandomStringRef() const {
@@ -320,7 +341,7 @@ struct BulkLoading : TestWorkload {
 	}
 
 	KeyRange getRandomRange(BulkLoading* self, KeyRange scope) const {
-		loop {
+		while (true) {
 			Standalone<StringRef> keyA = self->getRandomStringRef();
 			Standalone<StringRef> keyB = self->getRandomStringRef();
 			if (!scope.contains(keyA) || !scope.contains(keyB)) {
@@ -400,7 +421,7 @@ struct BulkLoading : TestWorkload {
 
 		res.setByteSampleFileName(sampleFileName);
 		std::string bytesSampleFile = res.getBytesSampleFileFullPath();
-		if (bytesSample.size() > 0) {
+		if (!bytesSample.empty()) {
 			sstWriter->open(abspath(bytesSampleFile));
 			for (const auto& kv : bytesSample) {
 				sstWriter->write(kv.key, kv.value);
@@ -508,14 +529,14 @@ struct BulkLoading : TestWorkload {
 	}
 
 	// Issue three non-overlapping tasks and check data consistency and correctness
-	ACTOR Future<Void> simpleTest(BulkLoading* self, Database cx) {
+	Future<Void> simpleTest(BulkLoading* self, Database cx) {
 		TraceEvent("BulkLoadingWorkLoadSimpleTestBegin");
-		state int oldBulkLoadMode = 0;
-		state std::vector<BulkLoadTaskState> bulkLoadTaskStates;
-		state std::vector<KeyRange> taskRanges;
-		state std::vector<KeyRange> errorRanges;
-		state std::vector<BulkLoadTaskTestUnit> taskUnits;
-		state int i = 0;
+		int oldBulkLoadMode = 0;
+		std::vector<BulkLoadTaskState> bulkLoadTaskStates;
+		std::vector<KeyRange> taskRanges;
+		std::vector<KeyRange> errorRanges;
+		std::vector<BulkLoadTaskTestUnit> taskUnits;
+		int i = 0;
 		for (i = 0; i < 2; i++) {
 			std::string indexStr = std::to_string(i);
 			std::string indexStrNext = std::to_string(i + 1);
@@ -528,24 +549,24 @@ struct BulkLoading : TestWorkload {
 			bulkLoadTaskStates.push_back(taskUnit.bulkLoadTask);
 			taskRanges.push_back(taskUnit.bulkLoadTask.getRange());
 			taskUnits.push_back(taskUnit);
-			bool succeed = wait(self->submitBulkLoadTask(cx, taskUnit.bulkLoadTask));
+			bool succeed = co_await self->submitBulkLoadTask(cx, taskUnit.bulkLoadTask);
 			ASSERT(succeed);
 		}
 
 		TraceEvent("BulkLoadingWorkLoadSimpleTestIssuedTasks");
-		wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 1)));
+		oldBulkLoadMode = co_await setBulkLoadMode(cx, 1);
 		TraceEvent("BulkLoadingWorkLoadSimpleTestSetMode").detail("OldMode", oldBulkLoadMode).detail("NewMode", 1);
-		std::vector<BulkLoadTaskState> errorTasks = wait(self->waitUntilAllTaskCompleteOrError(self, cx));
+		std::vector<BulkLoadTaskState> errorTasks = co_await self->waitUntilAllTaskCompleteOrError(self, cx);
 		for (const auto& errorTask : errorTasks) {
 			errorRanges.push_back(errorTask.getRange());
 		}
 		TraceEvent("BulkLoadingWorkLoadSimpleTestAllComplete");
 
 		// Check data
-		wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 0)));
+		oldBulkLoadMode = co_await setBulkLoadMode(cx, 0);
 		TraceEvent("BulkLoadingWorkLoadSimpleTestSetMode").detail("OldMode", oldBulkLoadMode).detail("NewMode", 0);
-		state std::vector<KeyValue> dbkvs = wait(self->getKvsFromDB(self, cx, errorRanges, taskRanges));
-		state std::vector<KeyValue> kvs;
+		std::vector<KeyValue> dbkvs = co_await self->getKvsFromDB(self, cx, errorRanges, taskRanges);
+		std::vector<KeyValue> kvs;
 		for (int j = 0; j < taskUnits.size(); j++) {
 			bool rangeTaskError = false;
 			for (const auto& errorRange : errorRanges) {
@@ -562,99 +583,101 @@ struct BulkLoading : TestWorkload {
 		ASSERT(self->checkSame(self, kvs, dbkvs));
 
 		// Check bulk load metadata
-		wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 1)));
+		oldBulkLoadMode = co_await setBulkLoadMode(cx, 1);
 		TraceEvent("BulkLoadingWorkLoadSimpleTestSetMode").detail("OldMode", oldBulkLoadMode).detail("NewMode", 1);
 		for (i = 0; i < bulkLoadTaskStates.size(); i++) {
-			wait(self->finalizeBulkLoadTask(cx, bulkLoadTaskStates[i].getRange(), bulkLoadTaskStates[i].getTaskId()));
+			co_await self->finalizeBulkLoadTask(
+			    cx, bulkLoadTaskStates[i].getRange(), bulkLoadTaskStates[i].getTaskId());
 		}
-		wait(acknowledgeAllErrorBulkLoadTasks(cx, self->jobId, normalKeys));
-		loop {
-			bool cleared = wait(self->checkBulkLoadMetadataCleared(self, cx));
+		co_await acknowledgeAllErrorBulkLoadTasks(cx, self->jobId, normalKeys);
+		while (true) {
+			bool cleared = co_await self->checkBulkLoadMetadataCleared(self, cx);
 			if (cleared) {
 				break;
 			}
-			wait(delay(1.0));
+			co_await delay(1.0);
 		}
 		TraceEvent("BulkLoadingWorkLoadSimpleTestComplete");
-		return Void();
 	}
 
-	ACTOR Future<Void> setKeys(Database cx, std::vector<KeyValue> kvs) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> setKeys(Database cx, std::vector<KeyValue> kvs) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				for (const auto& kv : kvs) {
 					tr.set(kv.key, kv.value);
 				}
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> backgroundWriteTraffic(BulkLoading* self, Database cx) {
-		loop {
+	Future<Void> backgroundWriteTraffic(BulkLoading* self, Database cx) {
+		while (true) {
 			int keyCount = deterministicRandom()->randomInt(1, 20);
 			std::vector<KeyValue> kvs = self->generateOrderedKVS(self, normalKeys, keyCount);
-			wait(self->setKeys(cx, kvs));
+			co_await self->setKeys(cx, kvs);
 			double delayTime = deterministicRandom()->random01() * 5.0;
-			wait(delay(delayTime));
+			co_await delay(delayTime);
 		}
 	}
 
-	ACTOR Future<Void> complexTest(BulkLoading* self, Database cx) {
-		state KeyRangeMap<Optional<BulkLoadTaskTestUnit>> taskMap;
+	Future<Void> complexTest(BulkLoading* self, Database cx) {
+		KeyRangeMap<Optional<BulkLoadTaskTestUnit>> taskMap;
 		taskMap.insert(allKeys, Optional<BulkLoadTaskTestUnit>());
-		state int i = 0;
-		state int oldBulkLoadMode = 0;
-		state BulkLoadTaskTestUnit taskUnit;
-		state std::vector<KeyRange> outdatedRanges;
-		state std::vector<KeyRange> errorRanges;
+		int i = 0;
+		int oldBulkLoadMode = 0;
+		BulkLoadTaskTestUnit taskUnit;
+		std::vector<KeyRange> outdatedRanges;
+		std::vector<KeyRange> errorRanges;
 
 		// Run tasks
-		wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 1)));
+		oldBulkLoadMode = co_await setBulkLoadMode(cx, 1);
 		TraceEvent("BulkLoadingWorkLoadComplexTestSetMode").detail("OldMode", oldBulkLoadMode).detail("NewMode", 1);
 		for (; i < 3; i++) {
 			std::string folderPath = joinPath(simulationBulkLoadFolder, std::to_string(i));
 			int dataSize = std::pow(10, deterministicRandom()->randomInt(0, 4));
 			taskUnit = self->generateBulkLoadTaskUnit(self, folderPath, dataSize);
 			ASSERT(normalKeys.contains(taskUnit.bulkLoadTask.getRange()));
-			bool succeed = wait(self->submitBulkLoadTask(cx, taskUnit.bulkLoadTask));
+			bool succeed = co_await self->submitBulkLoadTask(cx, taskUnit.bulkLoadTask);
 			if (succeed) {
 				taskMap.insert(taskUnit.bulkLoadTask.getRange(), taskUnit);
 			}
 			if (deterministicRandom()->coinflip()) {
-				std::vector<BulkLoadTaskState> errorTasks = wait(self->waitUntilAllTaskCompleteOrError(self, cx));
+				std::vector<BulkLoadTaskState> errorTasks = co_await self->waitUntilAllTaskCompleteOrError(self, cx);
 			}
 			if (deterministicRandom()->coinflip()) {
-				wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 0)));
+				oldBulkLoadMode = co_await setBulkLoadMode(cx, 0);
 				TraceEvent("BulkLoadingWorkLoadComplexTestSetMode")
 				    .detail("OldMode", oldBulkLoadMode)
 				    .detail("NewMode", 0);
-				wait(delay(deterministicRandom()->random01() * 5));
-				wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 1)));
+				co_await delay(deterministicRandom()->random01() * 5);
+				oldBulkLoadMode = co_await setBulkLoadMode(cx, 1);
 				TraceEvent("BulkLoadingWorkLoadComplexTestSetMode")
 				    .detail("OldMode", oldBulkLoadMode)
 				    .detail("NewMode", 1);
 			}
 			if (deterministicRandom()->coinflip()) {
-				wait(delay(deterministicRandom()->random01() * 5));
+				co_await delay(deterministicRandom()->random01() * 5);
 			}
 		}
 		// Wait until all tasks have completed
-		std::vector<BulkLoadTaskState> errorTasks = wait(self->waitUntilAllTaskCompleteOrError(self, cx));
+		std::vector<BulkLoadTaskState> errorTasks = co_await self->waitUntilAllTaskCompleteOrError(self, cx);
 		for (const auto& errorTask : errorTasks) {
 			errorRanges.push_back(errorTask.getRange()); // for any error range, do not check data
 		}
-		wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 0))); // trigger DD restart
+		oldBulkLoadMode = co_await setBulkLoadMode(cx, 0); // trigger DD restart
 		TraceEvent("BulkLoadingWorkLoadComplexTestSetMode").detail("OldMode", oldBulkLoadMode).detail("NewMode", 0);
 
 		// Check correctness
-		state std::vector<KeyValue> kvs;
-		state std::vector<BulkLoadTaskState> bulkLoadTaskStates;
-		state std::vector<KeyRange> completeTaskRanges;
+		std::vector<KeyValue> kvs;
+		std::vector<BulkLoadTaskState> bulkLoadTaskStates;
+		std::vector<KeyRange> completeTaskRanges;
 		for (auto& range : taskMap.ranges()) {
 			if (!range.value().present()) {
 				continue;
@@ -691,35 +714,35 @@ struct BulkLoading : TestWorkload {
 		ignoreRanges.reserve(outdatedRanges.size() + errorRanges.size());
 		ignoreRanges.insert(ignoreRanges.end(), outdatedRanges.begin(), outdatedRanges.end());
 		ignoreRanges.insert(ignoreRanges.end(), errorRanges.begin(), errorRanges.end());
-		std::vector<KeyValue> dbkvs = wait(self->getKvsFromDB(self, cx, ignoreRanges, completeTaskRanges));
+		std::vector<KeyValue> dbkvs = co_await self->getKvsFromDB(self, cx, ignoreRanges, completeTaskRanges);
 		ASSERT(self->checkSame(self, kvs, dbkvs));
 
 		// Clear metadata
-		wait(store(oldBulkLoadMode, setBulkLoadMode(cx, 1)));
+		oldBulkLoadMode = co_await setBulkLoadMode(cx, 1);
 		TraceEvent("BulkLoadingWorkLoadComplexTestSetMode").detail("OldMode", oldBulkLoadMode).detail("NewMode", 1);
 		for (i = 0; i < bulkLoadTaskStates.size(); i++) {
-			wait(self->finalizeBulkLoadTask(cx, bulkLoadTaskStates[i].getRange(), bulkLoadTaskStates[i].getTaskId()));
+			co_await self->finalizeBulkLoadTask(
+			    cx, bulkLoadTaskStates[i].getRange(), bulkLoadTaskStates[i].getTaskId());
 		}
-		wait(acknowledgeAllErrorBulkLoadTasks(cx, self->jobId, normalKeys));
-		loop {
-			bool cleared = wait(self->checkBulkLoadMetadataCleared(self, cx));
+		co_await acknowledgeAllErrorBulkLoadTasks(cx, self->jobId, normalKeys);
+		while (true) {
+			bool cleared = co_await self->checkBulkLoadMetadataCleared(self, cx);
 			if (cleared) {
 				break;
 			}
-			wait(delay(1.0));
+			co_await delay(1.0);
 		}
 
 		// Make sure all ranges locked by the workload are unlocked
 		std::vector<std::pair<KeyRange, RangeLockState>> res =
-		    wait(findExclusiveReadLockOnRange(cx, normalKeys, rangeLockNameForBulkLoad));
+		    co_await findExclusiveReadLockOnRange(cx, normalKeys, rangeLockNameForBulkLoad);
 		ASSERT(res.empty());
 		TraceEvent("BulkLoadingWorkLoadComplexTestComplete");
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(BulkLoading* self, Database cx) {
-		if (self->clientId != 0) {
-			return Void();
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0) {
+			co_return;
 		}
 
 		if (g_network->isSimulated()) {
@@ -729,39 +752,37 @@ struct BulkLoading : TestWorkload {
 			disableConnectionFailures("BulkLoading");
 		}
 
-		if (self->initializeBulkLoadMetadata) {
-			wait(self->clearAllBulkLoadTask(cx));
+		if (initializeBulkLoadMetadata) {
+			co_await clearAllBulkLoadTask(cx);
 		}
 
 		// Run background traffic
-		if (self->backgroundTrafficEnabled) {
-			state std::vector<Future<Void>> trafficActors;
+		if (backgroundTrafficEnabled) {
+			std::vector<Future<Void>> trafficActors;
 			int actorCount = deterministicRandom()->randomInt(1, 10);
 			for (int i = 0; i < actorCount; i++) {
-				trafficActors.push_back(self->backgroundWriteTraffic(self, cx));
+				trafficActors.push_back(backgroundWriteTraffic(this, cx));
 			}
 		}
 
-		wait(registerRangeLockOwner(cx, rangeLockNameForBulkLoad, rangeLockNameForBulkLoad));
+		co_await registerRangeLockOwner(cx, rangeLockNameForBulkLoad, rangeLockNameForBulkLoad);
 
-		std::vector<RangeLockOwner> lockOwners = wait(getAllRangeLockOwners(cx));
+		std::vector<RangeLockOwner> lockOwners = co_await getAllRangeLockOwners(cx);
 		ASSERT(lockOwners.size() == 1 && lockOwners[0].getOwnerUniqueId() == rangeLockNameForBulkLoad);
 
 		// Run test
 		if (deterministicRandom()->coinflip()) {
 			// Inject data to three non-overlapping ranges
-			wait(self->simpleTest(self, cx));
+			co_await simpleTest(this, cx);
 		} else {
 			// Inject data to many ranges and those ranges can be overlapping
-			wait(self->complexTest(self, cx));
+			co_await complexTest(this, cx);
 		}
 
-		wait(removeRangeLockOwner(cx, rangeLockNameForBulkLoad));
+		co_await removeRangeLockOwner(cx, rangeLockNameForBulkLoad);
 
-		std::vector<RangeLockOwner> lockOwnersAfterRemove = wait(getAllRangeLockOwners(cx));
+		std::vector<RangeLockOwner> lockOwnersAfterRemove = co_await getAllRangeLockOwners(cx);
 		ASSERT(lockOwnersAfterRemove.empty());
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/include/fdbserver/workloads/BulkSetup.actor.h b/fdbserver/workloads/BulkSetup.h
similarity index 71%
rename from fdbserver/include/fdbserver/workloads/BulkSetup.actor.h
rename to fdbserver/workloads/BulkSetup.h
index 20134877299..1c0442def5e 100644
--- a/fdbserver/include/fdbserver/workloads/BulkSetup.actor.h
+++ b/fdbserver/workloads/BulkSetup.h
@@ -1,5 +1,5 @@
 /*
- * BulkSetup.actor.h
+ * BulkSetup.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,8 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated
-// version of this file.  In intellisense use the source version.
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_BULK_SETUP_ACTOR_G_H)
-#define FDBSERVER_BULK_SETUP_ACTOR_G_H
-#include "fdbserver/workloads/BulkSetup.actor.g.h"
-#elif !defined(FDBSERVER_BULK_SETUP_ACTOR_H)
-#define FDBSERVER_BULK_SETUP_ACTOR_H
+#ifndef FDBSERVER_BULK_SETUP_H
+#define FDBSERVER_BULK_SETUP_H
 
 #include <string>
 #include <utility>
@@ -34,11 +29,11 @@
 
 #include "fdbclient/ClientBooleanParams.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/QuietDatabase.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/tester/WorkloadUtils.h"
 #include "fdbrpc/simulator.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 template <class T>
 struct sfinae_true : std::true_type {};
@@ -58,41 +53,47 @@ void setAuthToken(T const& self, Transaction& tr) {
 	}
 }
 
-ACTOR template <class T>
+template <class T>
 Future<bool> checkRangeSimpleValueSize(Database cx, T* workload, uint64_t begin, uint64_t end) {
-	loop {
-		state Transaction tr(cx);
+	while (true) {
+		Transaction tr(cx);
+		Error err;
 		setAuthToken(*workload, tr);
 		try {
-			state Standalone<KeyValueRef> firstKV = (*workload)(begin);
-			state Standalone<KeyValueRef> lastKV = (*workload)(end - 1);
-			state Future<Optional<Value>> first = tr.get(firstKV.key);
-			state Future<Optional<Value>> last = tr.get(lastKV.key);
-			wait(success(first) && success(last));
-			return first.get().present() && last.get().present();
+			Standalone<KeyValueRef> firstKV = (*workload)(begin);
+			Standalone<KeyValueRef> lastKV = (*workload)(end - 1);
+			Future<Optional<Value>> first = tr.get(firstKV.key);
+			Future<Optional<Value>> last = tr.get(lastKV.key);
+			co_await (success(first) && success(last));
+			co_return first.get().present() && last.get().present();
 		} catch (Error& e) {
 			TraceEvent("CheckRangeError").error(e).detail("Begin", begin).detail("End", end);
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.isValid()) {
+			co_await tr.onError(err);
 		}
 	}
 }
 
 // Returns true if the range was added
-ACTOR template <class T>
+template <class T>
 Future<uint64_t> setupRange(Database cx, T* workload, uint64_t begin, uint64_t end) {
-	state uint64_t bytesInserted = 0;
-	loop {
-		state Transaction tr(cx);
+	uint64_t bytesInserted = 0;
+	while (true) {
+		Transaction tr(cx);
+		Error err;
 		setAuthToken(*workload, tr);
 		try {
-			if (deterministicRandom()->random01() < 0.001)
+			if (deterministicRandom()->random01() < 0.001) {
 				tr.debugTransaction(deterministicRandom()->randomUniqueID());
+			}
 
-			state Standalone<KeyValueRef> sampleKV = (*workload)(begin);
+			Standalone<KeyValueRef> sampleKV = (*workload)(begin);
 			if (!g_network->isSimulated() || !g_simulator->speedUpSimulation) {
 				// Don't issue reads if speedUpSimulation is true. So this transaction
 				// becomes blind writes, to avoid transaction_too_old errors.
-				Optional<Value> f = wait(tr.get(sampleKV.key));
+				Optional<Value> f = co_await tr.get(sampleKV.key);
 				if (f.present()) {
 					//				if( sampleKV.value.size() == f.get().size() ) {
 					//TraceEvent("BulkSetupRangeAlreadyPresent")
@@ -101,7 +102,7 @@ Future<uint64_t> setupRange(Database cx, T* workload, uint64_t begin, uint64_t e
 					//	.detail("Version", tr.getReadVersion().get())
 					//	.detail("Key", printable((*workload)(begin).key))
 					//	.detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken);
-					return bytesInserted; // The transaction already completed!
+					co_return bytesInserted; // The transaction already completed!
 					//}
 					//TraceEvent(SevError, "BulkSetupConflict")
 					//	.detail("Begin", begin)
@@ -119,40 +120,45 @@ Future<uint64_t> setupRange(Database cx, T* workload, uint64_t begin, uint64_t e
 				tr.set(kv.key, kv.value, AddConflictRange::False);
 				bytesInserted += kv.key.size() + kv.value.size();
 			}
-			wait(tr.commit());
-			return bytesInserted;
+			co_await tr.commit();
+			co_return bytesInserted;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.isValid()) {
+			co_await tr.onError(err);
 		}
 	}
 }
 
-ACTOR template <class T>
+template <class T>
 Future<uint64_t> setupRangeWorker(Database cx,
                                   T* workload,
                                   std::vector<std::pair<uint64_t, uint64_t>>* jobs,
                                   double maxKeyInsertRate,
                                   int keySaveIncrement,
                                   int actorId) {
-	state double nextStart;
-	state uint64_t loadedRanges = 0;
-	state int lastStoredKeysLoaded = 0;
-	state uint64_t keysLoaded = 0;
-	state uint64_t bytesStored = 0;
+	double nextStart;
+	uint64_t loadedRanges = 0;
+	int lastStoredKeysLoaded = 0;
+	uint64_t keysLoaded = 0;
+	uint64_t bytesStored = 0;
 	while (jobs->size()) {
-		state std::pair<uint64_t, uint64_t> job = jobs->back();
+		std::pair<uint64_t, uint64_t> job = jobs->back();
 		jobs->pop_back();
 		nextStart = now() + (job.second - job.first) / maxKeyInsertRate;
-		uint64_t numBytes = wait(setupRange(cx, workload, job.first, job.second));
-		if (numBytes > 0)
+		uint64_t numBytes = co_await setupRange(cx, workload, job.first, job.second);
+		if (numBytes > 0) {
 			loadedRanges++;
+		}
 
 		if (keySaveIncrement > 0) {
 			keysLoaded += job.second - job.first;
 			bytesStored += numBytes;
 
 			if (keysLoaded - lastStoredKeysLoaded >= keySaveIncrement || jobs->size() == 0) {
-				state Transaction tr(cx);
+				Transaction tr(cx);
+				Error err;
 				setAuthToken(*workload, tr);
 				try {
 					std::string countKey = format("keycount|%d|%d", workload->clientId, actorId);
@@ -161,18 +167,22 @@ Future<uint64_t> setupRangeWorker(Database cx,
 					tr.set(StringRef(countKey), StringRef((uint8_t*)&keysLoaded, sizeof(uint64_t)));
 					tr.set(StringRef(bytesKey), StringRef((uint8_t*)&bytesStored, sizeof(uint64_t)));
 
-					wait(tr.commit());
+					co_await tr.commit();
 					lastStoredKeysLoaded = keysLoaded;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.isValid()) {
+					co_await tr.onError(err);
 				}
 			}
 		}
 
-		if (now() < nextStart)
-			wait(delayUntil(nextStart));
+		if (now() < nextStart) {
+			co_await delayUntil(nextStart);
+		}
 	}
-	return loadedRanges;
+	co_return loadedRanges;
 }
 
 // Periodically determines how many keys have been inserted.  If the count has just exceeded a count of interest,
@@ -180,25 +190,26 @@ Future<uint64_t> setupRangeWorker(Database cx,
 // the countsOfInterest vector.
 
 // Expects countsOfInterest to be sorted in ascending order
-ACTOR Future<std::vector<std::pair<uint64_t, double>>> trackInsertionCount(Database cx,
-                                                                           std::vector<uint64_t> countsOfInterest,
-                                                                           double checkInterval);
+Future<std::vector<std::pair<uint64_t, double>>> trackInsertionCount(Database cx,
+                                                                     std::vector<uint64_t> countsOfInterest,
+                                                                     double checkInterval);
 
-ACTOR template <class T>
+template <class T>
 Future<Void> waitForLowInFlight(Database cx, T* workload) {
-	state Future<Void> timeout = delay(600.0);
-	loop {
+	Future<Void> timeout = delay(600.0);
+	while (true) {
+		bool shouldDelay = false;
 		try {
 			if (timeout.isReady()) {
 				throw timed_out();
 			}
 
-			int64_t inFlight = wait(getDataInFlight(cx, workload->dbInfo));
+			int64_t inFlight = co_await getDataInFlight(cx, workload->dbInfo);
 			TraceEvent("DynamicWarming").detail("InFlight", inFlight);
 			if (inFlight > 1e6) { // Wait for just 1 MB to be in flight
-				wait(delay(1.0));
+				co_await delay(1.0);
 			} else {
-				wait(delay(1.0));
+				co_await delay(1.0);
 				TraceEvent("DynamicWarmingDone").log();
 				break;
 			}
@@ -206,17 +217,19 @@ Future<Void> waitForLowInFlight(Database cx, T* workload) {
 			if (e.code() == error_code_attribute_not_found ||
 			    (e.code() == error_code_timed_out && !timeout.isReady())) {
 				// DD may not be initialized yet and attribute "DataInFlight" can be missing
-				wait(delay(1.0));
+				shouldDelay = true;
 			} else {
 				TraceEvent(SevWarn, "WaitForLowInFlightError").error(e);
 				throw;
 			}
 		}
+		if (shouldDelay) {
+			co_await delay(1.0);
+		}
 	}
-	return Void();
 }
 
-ACTOR template <class T>
+template <class T>
 Future<Void> bulkSetup(Database cx,
                        T* workload,
                        uint64_t nodeCount,
@@ -231,12 +244,11 @@ Future<Void> bulkSetup(Database cx,
                        double keyCheckInterval = 0.1,
                        uint64_t startNodeIdx = 0,
                        uint64_t endNodeIdx = 0) {
+	std::vector<std::pair<uint64_t, uint64_t>> jobs;
+	uint64_t startNode = startNodeIdx ? startNodeIdx : (nodeCount * workload->clientId) / workload->clientCount;
+	uint64_t endNode = endNodeIdx ? endNodeIdx : (nodeCount * (workload->clientId + 1)) / workload->clientCount;
 
-	state std::vector<std::pair<uint64_t, uint64_t>> jobs;
-	state uint64_t startNode = startNodeIdx ? startNodeIdx : (nodeCount * workload->clientId) / workload->clientCount;
-	state uint64_t endNode = endNodeIdx ? endNodeIdx : (nodeCount * (workload->clientId + 1)) / workload->clientCount;
-
-	state double start = now();
+	double start = now();
 
 	TraceEvent("BulkSetupStart")
 	    .detail("NodeCount", nodeCount)
@@ -247,7 +259,7 @@ Future<Void> bulkSetup(Database cx,
 	// For bulk data schemes where the value of the key is not critical to operation, check to
 	//  see if the database has already been set up.
 	if (valuesInconsequential) {
-		bool present = wait(checkRangeSimpleValueSize(cx, workload, startNode, endNode));
+		bool present = co_await checkRangeSimpleValueSize(cx, workload, startNode, endNode);
 		if (present) {
 			TraceEvent("BulkSetupRangeAlreadyPresent")
 			    .detail("Begin", startNode)
@@ -255,7 +267,7 @@ Future<Void> bulkSetup(Database cx,
 			    .detail("CheckMethod", "SimpleValueSize");
 			setupTime.send(0.0);
 			ratesAtKeyCounts.send(std::vector<std::pair<uint64_t, double>>());
-			return Void();
+			co_return;
 		} else {
 			TraceEvent("BulkRangeNotFound")
 			    .detail("Begin", startNode)
@@ -264,9 +276,9 @@ Future<Void> bulkSetup(Database cx,
 		}
 	}
 
-	wait(delay(deterministicRandom()->random01() / 4)); // smear over .25 seconds
+	co_await delay(deterministicRandom()->random01() / 4); // smear over .25 seconds
 
-	state int BULK_SETUP_WORKERS = 40;
+	int BULK_SETUP_WORKERS = 40;
 	// See that each chunk inserted is about 10KB
 	int size_total = 0;
 	for (int i = 0; i < 100; i++) {
@@ -274,7 +286,7 @@ Future<Void> bulkSetup(Database cx,
 		    (*workload)(startNode + (uint64_t)(deterministicRandom()->random01() * (endNode - startNode)));
 		size_total += sampleKV.key.size() + sampleKV.value.size();
 	}
-	state int BULK_SETUP_RANGE_SIZE = size_total == 0 ? 50 : std::max(1, 10000 / (size_total / 100));
+	int BULK_SETUP_RANGE_SIZE = size_total == 0 ? 50 : std::max(1, 10000 / (size_total / 100));
 
 	TraceEvent((workload->description() + "SetupStart").c_str())
 	    .detail("ClientIdx", workload->clientId)
@@ -285,34 +297,41 @@ Future<Void> bulkSetup(Database cx,
 	    .detail("SetupActors", BULK_SETUP_WORKERS);
 
 	// create a random vector of range-create jobs
-	for (uint64_t n = startNode; n < endNode; n += BULK_SETUP_RANGE_SIZE)
+	for (uint64_t n = startNode; n < endNode; n += BULK_SETUP_RANGE_SIZE) {
 		jobs.emplace_back(n, std::min(endNode, n + BULK_SETUP_RANGE_SIZE));
+	}
 	deterministicRandom()->randomShuffle(jobs);
 
 	// fire up the workers and wait for them to eat all the jobs
 	double maxWorkerInsertRate = maxKeyInsertRate / BULK_SETUP_WORKERS / workload->clientCount;
-	state std::vector<Future<uint64_t>> fs;
+	std::vector<Future<uint64_t>> fs;
 
-	state Future<std::vector<std::pair<uint64_t, double>>> insertionTimes = std::vector<std::pair<uint64_t, double>>();
+	Future<std::vector<std::pair<uint64_t, double>>> insertionTimes = std::vector<std::pair<uint64_t, double>>();
 
 	if (insertionCountsToMeasure.size() > 0) {
 		std::sort(insertionCountsToMeasure.begin(), insertionCountsToMeasure.end());
-		for (int i = 0; i < insertionCountsToMeasure.size(); i++)
-			if (insertionCountsToMeasure[i] > nodeCount)
+		for (int i = 0; i < insertionCountsToMeasure.size(); i++) {
+			if (insertionCountsToMeasure[i] > nodeCount) {
 				insertionCountsToMeasure.erase(insertionCountsToMeasure.begin() + i, insertionCountsToMeasure.end());
+			}
+		}
 
-		if (workload->clientId == 0)
+		if (workload->clientId == 0) {
 			insertionTimes = trackInsertionCount(cx, insertionCountsToMeasure, keyCheckInterval);
+		}
 
-		if (keySaveIncrement <= 0)
+		if (keySaveIncrement <= 0) {
 			keySaveIncrement = BULK_SETUP_RANGE_SIZE;
-	} else
+		}
+	} else {
 		keySaveIncrement = 0;
+	}
 
-	for (int j = 0; j < BULK_SETUP_WORKERS; j++)
+	for (int j = 0; j < BULK_SETUP_WORKERS; j++) {
 		fs.push_back(setupRangeWorker(cx, workload, &jobs, maxWorkerInsertRate, keySaveIncrement, j));
+	}
 	try {
-		wait(success(insertionTimes) && waitForAll(fs));
+		co_await (success(insertionTimes) && waitForAll(fs));
 	} catch (Error& e) {
 		if (e.code() == error_code_operation_failed) {
 			TraceEvent(SevError, "BulkSetupFailed").error(e);
@@ -323,10 +342,11 @@ Future<Void> bulkSetup(Database cx,
 	ratesAtKeyCounts.send(insertionTimes.get());
 
 	uint64_t rangesInserted = 0;
-	for (int i = 0; i < fs.size(); i++)
+	for (int i = 0; i < fs.size(); i++) {
 		rangesInserted += fs[i].get();
+	}
 
-	state double elapsed = now() - start;
+	double elapsed = now() - start;
 	setupTime.send(elapsed);
 	TraceEvent("SetupLoadComplete")
 	    .detail("LoadedRanges", rangesInserted)
@@ -335,15 +355,19 @@ Future<Void> bulkSetup(Database cx,
 
 	// Here we wait for data in flight to go to 0 (this will not work on a database with other users)
 	if (postSetupWarming != 0) {
+		bool shouldWarmDatabase = false;
 		try {
-			wait(delay(5.0));
-			wait(waitForLowInFlight(cx, workload)); // Wait for the data distribution in a small test to start
+			co_await delay(5.0);
+			co_await waitForLowInFlight(cx, workload); // Wait for the data distribution in a small test to start
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled)
+			if (e.code() == error_code_actor_cancelled) {
 				throw;
+			}
 			TraceEvent("DynamicWarmingError").error(e);
-			if (postSetupWarming > 0)
-				wait(timeout(databaseWarmer(cx), postSetupWarming, Void()));
+			shouldWarmDatabase = postSetupWarming > 0;
+		}
+		if (shouldWarmDatabase) {
+			co_await timeout(databaseWarmer(cx), postSetupWarming, Void());
 		}
 	}
 
@@ -351,9 +375,6 @@ Future<Void> bulkSetup(Database cx,
 	    .detail("ClientIdx", workload->clientId)
 	    .detail("WarmingDelay", postSetupWarming)
 	    .detail("KeyLoadElapsedTime", elapsed);
-	return Void();
 }
 
-#include "flow/unactorcompiler.h"
-
 #endif
diff --git a/fdbserver/workloads/CMakeLists.txt b/fdbserver/workloads/CMakeLists.txt
new file mode 100644
index 00000000000..9a90ab84605
--- /dev/null
+++ b/fdbserver/workloads/CMakeLists.txt
@@ -0,0 +1,17 @@
+fdb_find_sources(FDBSERVER_WORKLOADS_SRCS)
+
+add_flow_target(STATIC_LIBRARY NAME fdbserver_workloads SRCS ${FDBSERVER_WORKLOADS_SRCS})
+
+configure_fdbserver_common_includes(fdbserver_workloads)
+target_include_directories(fdbserver_workloads PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(fdbserver_workloads PRIVATE
+  fdbserver_consistencyscan
+  fdbserver_core
+  fdbserver_kvstore
+  fdbserver_worker
+  fdbserver_tester
+  fdbserver_datadistributor
+  fdbserver_resolver
+  fdbserver_storageserver
+  fdbserver_mocks3)
diff --git a/fdbserver/workloads/ChangeConfig.actor.cpp b/fdbserver/workloads/ChangeConfig.actor.cpp
deleted file mode 100644
index ef922bb4d85..00000000000
--- a/fdbserver/workloads/ChangeConfig.actor.cpp
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * ChangeConfig.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/Schemas.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbrpc/simulator.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-struct ChangeConfigWorkload : TestWorkload {
-	static constexpr auto NAME = "ChangeConfig";
-	double minDelayBeforeChange, maxDelayBeforeChange;
-	std::string configMode;
-	std::string networkAddresses;
-	int coordinatorChanges;
-
-	ChangeConfigWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		minDelayBeforeChange = getOption(options, "minDelayBeforeChange"_sr, 0.0);
-		maxDelayBeforeChange = getOption(options, "maxDelayBeforeChange"_sr, 0.0);
-		ASSERT(maxDelayBeforeChange >= minDelayBeforeChange);
-
-		configMode = getOption(options, "configMode"_sr, StringRef()).toString();
-		networkAddresses = getOption(options, "coordinators"_sr, StringRef()).toString();
-		coordinatorChanges = getOption(options, "coordinatorChanges"_sr, 1);
-		if (networkAddresses != "auto") {
-			coordinatorChanges = 1;
-		}
-	}
-
-	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
-
-	Future<Void> start(Database const& cx) override {
-		if (this->clientId != 0) {
-			return Void();
-		}
-		return changeConfigClient(cx->clone(), this);
-	}
-
-	Future<bool> check(Database const& cx) override { return true; }
-
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-
-	std::string getConfigMode(const std::string& mode, bool existingDB) {
-		std::string res = mode;
-		if (existingDB) {
-			size_t pos = res.find("new ");
-			if (pos != std::string::npos) {
-				res.replace(pos, 4, "");
-			}
-		}
-		return res;
-	}
-
-	ACTOR static Future<Void> configureExtraDatabase(ChangeConfigWorkload* self, Database db) {
-		wait(delay(5 * deterministicRandom()->random01()));
-		if (self->configMode.size()) {
-			state bool existingDB = false;
-			if (g_simulator->startingDisabledConfiguration != "") {
-				wait(success(
-				    ManagementAPI::changeConfig(db.getReference(), g_simulator->startingDisabledConfiguration, true)));
-				TraceEvent("WaitForReplicasExtra").log();
-				wait(waitForFullReplication(db));
-				TraceEvent("WaitForReplicasExtraEnd").log();
-				existingDB = true;
-			}
-			std::string mode = self->getConfigMode(self->configMode, existingDB);
-			wait(success(ManagementAPI::changeConfig(db.getReference(), mode, true)));
-		}
-		if (self->networkAddresses.size()) {
-			if (self->networkAddresses == "auto") {
-				wait(coordinatorsChangeActor(db, self, true));
-			} else {
-				wait(coordinatorsChangeActor(db, self));
-			}
-		}
-
-		wait(delay(5 * deterministicRandom()->random01()));
-		return Void();
-	}
-
-	Future<Void> configureExtraDatabases(ChangeConfigWorkload* self) {
-		std::vector<Future<Void>> futures;
-		if (g_network->isSimulated()) {
-			for (auto extraDatabase : g_simulator->extraDatabases) {
-				Database db = Database::createSimulatedExtraDatabase(extraDatabase);
-				futures.push_back(configureExtraDatabase(self, db));
-			}
-		}
-		return waitForAll(futures);
-	}
-
-	ACTOR static Future<Void> changeConfigClient(Database cx, ChangeConfigWorkload* self) {
-		wait(delay(self->minDelayBeforeChange +
-		           deterministicRandom()->random01() * (self->maxDelayBeforeChange - self->minDelayBeforeChange)));
-
-		state bool extraConfigureBefore = deterministicRandom()->random01() < 0.5;
-		if (extraConfigureBefore) {
-			wait(self->configureExtraDatabases(self));
-		}
-
-		if (self->configMode.size()) {
-			state bool existingDB = false;
-			if (g_network->isSimulated() && g_simulator->startingDisabledConfiguration != "") {
-				wait(success(
-				    ManagementAPI::changeConfig(cx.getReference(), g_simulator->startingDisabledConfiguration, true)));
-				TraceEvent("WaitForReplicas").log();
-				wait(waitForFullReplication(cx));
-				TraceEvent("WaitForReplicasEnd").log();
-				existingDB = true;
-			}
-			std::string mode = self->getConfigMode(self->configMode, existingDB);
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), mode, true)));
-		}
-
-		if (self->networkAddresses.size()) {
-			state int i;
-			for (i = 0; i < self->coordinatorChanges; ++i) {
-				if (i > 0) {
-					wait(delay(20));
-				}
-				wait(coordinatorsChangeActor(cx, self, self->networkAddresses == "auto"));
-			}
-		}
-
-		if (!extraConfigureBefore) {
-			wait(self->configureExtraDatabases(self));
-		}
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> coordinatorsChangeActor(Database cx,
-	                                                  ChangeConfigWorkload* self,
-	                                                  bool autoChange = false) {
-		state ReadYourWritesTransaction tr(cx);
-		state int notEnoughMachineResults = 0;
-		state std::string desiredCoordinatorsKey;
-
-		if (autoChange) {
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
-					Optional<Value> newCoordinatorsKey = wait(tr.get("auto_coordinators"_sr.withPrefix(
-					    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
-					ASSERT(newCoordinatorsKey.present());
-					desiredCoordinatorsKey = newCoordinatorsKey.get().toString();
-					tr.reset();
-					break;
-				} catch (Error& e) {
-					if (e.code() == error_code_special_keys_api_failure) {
-						Optional<Value> errorMsg =
-						    wait(tr.get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-						ASSERT(errorMsg.present());
-						std::string errorStr;
-						auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-						auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-						TraceEvent(SevDebug, "GetAutoCoordinatorsChange")
-						    .detail("ErrorMessage", valueObj["message"].get_str());
-						ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-						ASSERT(valueObj["command"].get_str() == "auto_coordinators");
-						if (valueObj["retriable"].get_bool() && notEnoughMachineResults < 1) {
-							notEnoughMachineResults++;
-							wait(delay(1.0));
-							tr.reset();
-						} else {
-							break;
-						}
-					} else {
-						wait(tr.onError(e));
-					}
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				}
-			}
-		} else {
-			desiredCoordinatorsKey = self->networkAddresses;
-		}
-
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-				tr.set("processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
-				       Value(desiredCoordinatorsKey));
-				TraceEvent(SevDebug, "CoordinatorsChangeBeforeCommit")
-				    .detail("Auto", autoChange)
-				    .detail("NewCoordinatorsKey", describe(desiredCoordinatorsKey));
-				wait(tr.commit());
-				ASSERT(false);
-			} catch (Error& e) {
-				state Error err(e);
-				if (e.code() == error_code_special_keys_api_failure) {
-					Optional<Value> errorMsg =
-					    wait(tr.get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-					ASSERT(errorMsg.present());
-					std::string errorStr;
-					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-					auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-					TraceEvent(SevDebug, "CoordinatorsChangeError")
-					    .detail("Auto", autoChange)
-					    .detail("ErrorMessage", valueObj["message"].get_str());
-					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-					ASSERT(valueObj["command"].get_str() == "coordinators");
-					break;
-				} else {
-					wait(tr.onError(err));
-				}
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-			}
-		}
-		return Void();
-	}
-};
-
-WorkloadFactory<ChangeConfigWorkload> ChangeConfigWorkloadFactory;
diff --git a/fdbserver/workloads/ChangeConfig.cpp b/fdbserver/workloads/ChangeConfig.cpp
new file mode 100644
index 00000000000..f9ff279d3d0
--- /dev/null
+++ b/fdbserver/workloads/ChangeConfig.cpp
@@ -0,0 +1,229 @@
+/*
+ * ChangeConfig.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/Schemas.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbrpc/simulator.h"
+
+struct ChangeConfigWorkload : TestWorkload {
+	static constexpr auto NAME = "ChangeConfig";
+	double minDelayBeforeChange, maxDelayBeforeChange;
+	std::string configMode;
+	std::string networkAddresses;
+	int coordinatorChanges;
+
+	explicit ChangeConfigWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		minDelayBeforeChange = getOption(options, "minDelayBeforeChange"_sr, 0.0);
+		maxDelayBeforeChange = getOption(options, "maxDelayBeforeChange"_sr, 0.0);
+		ASSERT(maxDelayBeforeChange >= minDelayBeforeChange);
+
+		configMode = getOption(options, "configMode"_sr, StringRef()).toString();
+		networkAddresses = getOption(options, "coordinators"_sr, StringRef()).toString();
+		coordinatorChanges = getOption(options, "coordinatorChanges"_sr, 1);
+		if (networkAddresses != "auto") {
+			coordinatorChanges = 1;
+		}
+	}
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
+
+	Future<Void> start(Database const& cx) override {
+		if (this->clientId != 0) {
+			return Void();
+		}
+		return changeConfigClient(cx->clone());
+	}
+
+	Future<bool> check(Database const& cx) override { return true; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	std::string getConfigMode(const std::string& mode, bool existingDB) {
+		std::string res = mode;
+		if (existingDB) {
+			size_t pos = res.find("new ");
+			if (pos != std::string::npos) {
+				res.replace(pos, 4, "");
+			}
+		}
+		return res;
+	}
+
+	Future<Void> configureExtraDatabase(Database db) {
+		co_await delay(5 * deterministicRandom()->random01());
+		if (!configMode.empty()) {
+			bool existingDB = false;
+			if (!fdbSimulationPolicyState().startingDisabledConfiguration.empty()) {
+				co_await ManagementAPI::changeConfig(
+				    db.getReference(), fdbSimulationPolicyState().startingDisabledConfiguration, true);
+				TraceEvent("WaitForReplicasExtra").log();
+				co_await waitForFullReplication(db);
+				TraceEvent("WaitForReplicasExtraEnd").log();
+				existingDB = true;
+			}
+			std::string mode = getConfigMode(configMode, existingDB);
+			co_await ManagementAPI::changeConfig(db.getReference(), mode, true);
+		}
+		if (!networkAddresses.empty()) {
+			if (networkAddresses == "auto") {
+				co_await coordinatorsChangeActor(db, true);
+			} else {
+				co_await coordinatorsChangeActor(db);
+			}
+		}
+
+		co_await delay(5 * deterministicRandom()->random01());
+	}
+
+	Future<Void> configureExtraDatabases() {
+		std::vector<Future<Void>> futures;
+		if (g_network->isSimulated()) {
+			for (const auto& extraDatabase : fdbSimulationPolicyState().extraDatabases) {
+				Database db = Database::createSimulatedExtraDatabase(extraDatabase);
+				futures.push_back(configureExtraDatabase(db));
+			}
+		}
+		return waitForAll(futures);
+	}
+
+	Future<Void> changeConfigClient(Database cx) {
+		co_await delay(minDelayBeforeChange +
+		               deterministicRandom()->random01() * (maxDelayBeforeChange - minDelayBeforeChange));
+
+		bool extraConfigureBefore = deterministicRandom()->random01() < 0.5;
+		if (extraConfigureBefore) {
+			co_await configureExtraDatabases();
+		}
+
+		if (!configMode.empty()) {
+			bool existingDB = false;
+			if (g_network->isSimulated() && !fdbSimulationPolicyState().startingDisabledConfiguration.empty()) {
+				co_await ManagementAPI::changeConfig(
+				    cx.getReference(), fdbSimulationPolicyState().startingDisabledConfiguration, true);
+				TraceEvent("WaitForReplicas").log();
+				co_await waitForFullReplication(cx);
+				TraceEvent("WaitForReplicasEnd").log();
+				existingDB = true;
+			}
+			std::string mode = getConfigMode(configMode, existingDB);
+			co_await ManagementAPI::changeConfig(cx.getReference(), mode, true);
+		}
+
+		if (!networkAddresses.empty()) {
+			for (int i = 0; i < coordinatorChanges; ++i) {
+				if (i > 0) {
+					co_await delay(20);
+				}
+				co_await coordinatorsChangeActor(cx, networkAddresses == "auto");
+			}
+		}
+
+		if (!extraConfigureBefore) {
+			co_await configureExtraDatabases();
+		}
+	}
+
+	Future<Void> coordinatorsChangeActor(Database cx, bool autoChange = false) {
+		ReadYourWritesTransaction tr(cx);
+		int notEnoughMachineResults = 0;
+		std::string desiredCoordinatorsKey;
+
+		if (autoChange) {
+			while (true) {
+				Error err;
+				try {
+					tr.setOption(FDBTransactionOptions::RAW_ACCESS);
+					Optional<Value> newCoordinatorsKey = co_await tr.get("auto_coordinators"_sr.withPrefix(
+					    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin));
+					ASSERT(newCoordinatorsKey.present());
+					desiredCoordinatorsKey = newCoordinatorsKey.get().toString();
+					tr.reset();
+					break;
+				} catch (Error& e) {
+					err = e;
+				}
+				if (err.code() == error_code_special_keys_api_failure) {
+					Optional<Value> errorMsg =
+					    co_await tr.get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+					ASSERT(errorMsg.present());
+					std::string errorStr;
+					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+					auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+					TraceEvent(SevDebug, "GetAutoCoordinatorsChange")
+					    .detail("ErrorMessage", valueObj["message"].get_str());
+					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+					ASSERT(valueObj["command"].get_str() == "auto_coordinators");
+					if (valueObj["retriable"].get_bool() && notEnoughMachineResults < 1) {
+						notEnoughMachineResults++;
+						co_await delay(1.0);
+						tr.reset();
+					} else {
+						break;
+					}
+				} else {
+					co_await tr.onError(err);
+				}
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			}
+		} else {
+			desiredCoordinatorsKey = networkAddresses;
+		}
+
+		while (true) {
+			Error caughtErr;
+			try {
+				tr.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+				tr.set("processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+				       Value(desiredCoordinatorsKey));
+				TraceEvent(SevDebug, "CoordinatorsChangeBeforeCommit")
+				    .detail("Auto", autoChange)
+				    .detail("NewCoordinatorsKey", describe(desiredCoordinatorsKey));
+				co_await tr.commit();
+				ASSERT(false);
+			} catch (Error& e) {
+				caughtErr = e;
+			}
+			Error err(caughtErr);
+			if (caughtErr.code() == error_code_special_keys_api_failure) {
+				Optional<Value> errorMsg =
+				    co_await tr.get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+				ASSERT(errorMsg.present());
+				std::string errorStr;
+				auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+				auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+				TraceEvent(SevDebug, "CoordinatorsChangeError")
+				    .detail("Auto", autoChange)
+				    .detail("ErrorMessage", valueObj["message"].get_str());
+				ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+				ASSERT(valueObj["command"].get_str() == "coordinators");
+				break;
+			} else {
+				if (err.isValid()) {
+					co_await tr.onError(err);
+				}
+			}
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+		}
+	}
+};
+
+WorkloadFactory<ChangeConfigWorkload> ChangeConfigWorkloadFactory;
diff --git a/fdbserver/workloads/ClearSingleRange.actor.cpp b/fdbserver/workloads/ClearSingleRange.cpp
similarity index 65%
rename from fdbserver/workloads/ClearSingleRange.actor.cpp
rename to fdbserver/workloads/ClearSingleRange.cpp
index ee9c99fdd6f..4fe793704f1 100644
--- a/fdbserver/workloads/ClearSingleRange.actor.cpp
+++ b/fdbserver/workloads/ClearSingleRange.cpp
@@ -1,5 +1,5 @@
 /*
- * ClearSingleRange.actor.cpp
+ * ClearSingleRange.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,9 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 struct ClearSingleRange : TestWorkload {
 	static constexpr auto NAME = "ClearSingleRange";
@@ -30,7 +29,7 @@ struct ClearSingleRange : TestWorkload {
 	Key end;
 	double startDelay;
 
-	ClearSingleRange(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ClearSingleRange(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		begin = getOption(options, "begin"_sr, normalKeys.begin);
 		end = getOption(options, "end"_sr, normalKeys.end);
 		startDelay = getOption(options, "beginClearRange"_sr, 10.0);
@@ -38,28 +37,29 @@ struct ClearSingleRange : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return clientId != 0 ? Void() : fdbClientClearRange(cx, this); }
+	Future<Void> start(Database const& cx) override { return clientId != 0 ? Void() : fdbClientClearRange(cx); }
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> fdbClientClearRange(Database db, ClearSingleRange* self) {
-		state Transaction tr(db);
+	Future<Void> fdbClientClearRange(Database db) {
+		Transaction tr(db);
+		Error err;
 		try {
 			TraceEvent("ClearSingleRange")
-			    .detail("Begin", printable(self->begin))
-			    .detail("End", printable(self->end))
-			    .detail("StartDelay", self->startDelay);
+			    .detail("Begin", printable(begin))
+			    .detail("End", printable(end))
+			    .detail("StartDelay", startDelay);
 			tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-			wait(delay(self->startDelay));
-			tr.clear(KeyRangeRef(self->begin, self->end));
-			wait(tr.commit());
+			co_await delay(startDelay);
+			tr.clear(KeyRangeRef(begin, end));
+			co_await tr.commit();
 		} catch (Error& e) {
-			TraceEvent("ClearRangeError").error(e);
-			wait(tr.onError(e));
+			err = e;
 		}
-		return Void();
+		TraceEvent("ClearRangeError").error(err);
+		co_await tr.onError(err);
 	}
 };
 
diff --git a/fdbserver/workloads/ClientMetric.actor.cpp b/fdbserver/workloads/ClientMetric.cpp
similarity index 67%
rename from fdbserver/workloads/ClientMetric.actor.cpp
rename to fdbserver/workloads/ClientMetric.cpp
index 9d1ff667cea..109a2632fab 100644
--- a/fdbserver/workloads/ClientMetric.actor.cpp
+++ b/fdbserver/workloads/ClientMetric.cpp
@@ -1,5 +1,5 @@
 /*
- * ClientMetric.actor.cpp
+ * ClientMetric.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunTransaction.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunTransaction.h"
 #include "fdbclient/Tuple.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 static const StringRef sampleTrInfoKey =
     "\xff\x02/fdbClientInfo/client_latency/SSSSSSSSSS/RRRRRRRRRRRRRRRR/NNNNTTTT/XXXX/"_sr;
@@ -41,7 +40,7 @@ struct ClientMetricWorkload : TestWorkload {
 	int64_t trInfoSizeLimit;
 	std::vector<Future<Void>> clients;
 
-	ClientMetricWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ClientMetricWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		samplingProbability = getOption(options,
 		                                "samplingProbability"_sr,
 		                                deterministicRandom()->random01()); // rand range 0 - 1
@@ -68,21 +67,20 @@ struct ClientMetricWorkload : TestWorkload {
 		if (this->clientId != 0) {
 			return Void();
 		}
-		return _start(this, cx);
+		return _start(cx);
 	}
 
-	ACTOR Future<Void> _start(ClientMetricWorkload* self, Database cx) {
+	Future<Void> _start(Database cx) {
 		try {
-			self->clients.push_back(timeout(self->runner(cx, self), self->testDuration, Void()));
-			wait(waitForAll(self->clients));
+			clients.push_back(timeout(runner(cx, this), testDuration, Void()));
+			co_await waitForAll(clients);
 		} catch (Error& e) {
 			TraceEvent("ClientMetricError::_start").error(e);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) {
-		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+	Future<Void> changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) {
+		co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 			Tuple rate = Tuple::makeTuple(sampleProbability);
 			Tuple size = Tuple::makeTuple(sizeLimit);
 			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -92,19 +90,18 @@ struct ClientMetricWorkload : TestWorkload {
 			          << std::endl;
 
 			return Void();
-		}));
-		return Void();
+		});
 	}
 
-	ACTOR Future<RangeResult> latencyRangeQuery(Database cx, int keysLimit, bool reverse) {
-		state KeySelector begin =
-		    firstGreaterOrEqual(CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin));
-		state KeySelector end = firstGreaterOrEqual(strinc(begin.getKey()));
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state RangeResult txInfoEntries;
+	Future<RangeResult> latencyRangeQuery(Database cx, int keysLimit, bool reverse) {
+		KeySelector begin = firstGreaterOrEqual(CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin));
+		KeySelector end = firstGreaterOrEqual(strinc(begin.getKey()));
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		RangeResult txInfoEntries;
 		// wait to make sure client metrics are updated
-		wait(delay(CLIENT_KNOBS->CSI_STATUS_DELAY));
-		loop {
+		co_await delay(CLIENT_KNOBS->CSI_STATUS_DELAY);
+		while (true) {
+			Error err;
 			try {
 				tr->reset();
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -121,10 +118,10 @@ struct ClientMetricWorkload : TestWorkload {
 					sizeLimitStr = std::to_string(sizeLimit);
 				}
 				std::cout << "Read from globalconfig: rate=" << sampleRateStr << " size=" << sizeLimitStr << std::endl;
-				state RangeResult kvRange = wait(
-				    tr->getRange(begin, end, keysLimit, Snapshot::False, reverse ? Reverse::True : Reverse::False));
+				RangeResult kvRange = co_await tr->getRange(
+				    begin, end, keysLimit, Snapshot::False, reverse ? Reverse::True : Reverse::False);
 				if (kvRange.empty()) {
-					wait(delay(1.0));
+					co_await delay(1.0);
 					std::cout << "WaitingForLatencyMetricToBePresent" << std::endl;
 					TraceEvent("WaitingForLatencyMetricToBePresent").log();
 					continue;
@@ -133,33 +130,38 @@ struct ClientMetricWorkload : TestWorkload {
 				txInfoEntries.append(txInfoEntries.arena(), kvRange.begin(), kvRange.size());
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 		for (auto& kv : txInfoEntries) {
 			uint64_t vs = getVersionStamp(kv.key);
 			std::cout << "VersionStamp is " << vs << std::endl;
 		}
-		return txInfoEntries;
+		co_return txInfoEntries;
 	}
 
-	ACTOR Future<Void> writeRandomKeys(Database cx, int total) {
-		state int cnt = 0;
-		state Transaction tr(cx);
+	Future<Void> writeRandomKeys(Database cx, int total) {
+		int cnt = 0;
+		Transaction tr(cx);
 		try {
-			loop {
+			while (true) {
+				Error err;
 				try {
-					wait(delay(0.001));
+					co_await delay(0.001);
 					tr.reset();
 					tr.set(Key(deterministicRandom()->randomAlphaNumeric(10)),
 					       Value(Key(deterministicRandom()->randomAlphaNumeric(10))));
-					wait(tr.commit());
+					co_await tr.commit();
 					if (cnt >= total) {
 						break;
 					}
 					++cnt;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.isValid()) {
+					co_await tr.onError(err);
 				}
 			}
 		} catch (Error& e) {
@@ -167,31 +169,33 @@ struct ClientMetricWorkload : TestWorkload {
 			throw;
 		}
 		std::cout << "writeRandomKeys finish, written=" << cnt << std::endl;
-		return Void();
 	}
 
-	ACTOR Future<uint64_t> writeKeysAndGetLatencyVersion(Database cx,
-	                                                     ClientMetricWorkload* self,
-	                                                     int numKeys,
-	                                                     uint64_t previousVS) {
-		state int retry = 0;
-		state int max_retry = 10;
-		state int keysLimit = 1;
-		loop {
+	Future<uint64_t> writeKeysAndGetLatencyVersion(Database cx,
+	                                               ClientMetricWorkload* self,
+	                                               int numKeys,
+	                                               uint64_t previousVS) {
+		int retry = 0;
+		int max_retry = 50;
+		int keysLimit = 1;
+		while (true) {
 			if (retry > max_retry) {
-				// this should not happen, it should succeed after a few retry
+				TraceEvent(SevError, "WriteKeysAndGetLatencyVersionFailed")
+				    .detail("Retry", retry)
+				    .detail("MaxRetry", max_retry)
+				    .detail("PreviousVS", previousVS);
 				ASSERT(false);
 			}
 			// write random keys to generate some latency metrics
-			wait(self->writeRandomKeys(cx, numKeys));
+			co_await self->writeRandomKeys(cx, numKeys);
 			// get the latest latency metric and parse its version stamp
-			RangeResult r = wait(self->latencyRangeQuery(cx, keysLimit, true));
-			if (r.size() == 0) {
+			RangeResult r = co_await self->latencyRangeQuery(cx, keysLimit, true);
+			if (r.empty()) {
 				// latency metrics might not be present due to transaction batching, retry a few times
 				++retry;
 				continue;
 			}
-			ASSERT(r.size() > 0);
+			ASSERT(!r.empty());
 			// [0] is the latest version, as we have reverse = true
 			KeyRef latest = r[0].key;
 			uint64_t vs = getVersionStamp(latest);
@@ -201,7 +205,7 @@ struct ClientMetricWorkload : TestWorkload {
 				++retry;
 				continue;
 			}
-			return vs;
+			co_return vs;
 		}
 	}
 
@@ -209,24 +213,23 @@ struct ClientMetricWorkload : TestWorkload {
 	//      write some random keys, check the latency metric and the latest version stamp vs1
 	//      write some other random keys, check the latency metric and latest version stamp again vs2
 	//      vs2 should be strictly larger than vs1, to verify new latency metrics are added
-	ACTOR Future<Void> runner(Database cx, ClientMetricWorkload* self) {
+	Future<Void> runner(Database cx, ClientMetricWorkload* self) {
 		try {
-			state int initialWrites = deterministicRandom()->randomInt(100, 200);
-			state int secondWrites = deterministicRandom()->randomInt(100, 200);
+			int initialWrites = deterministicRandom()->randomInt(100, 200);
+			int secondWrites = deterministicRandom()->randomInt(100, 200);
 
-			state uint64_t zeroVS = 0;
-			state uint64_t vs1 = wait(self->writeKeysAndGetLatencyVersion(cx, self, initialWrites, zeroVS));
+			uint64_t zeroVS = 0;
+			uint64_t vs1 = co_await self->writeKeysAndGetLatencyVersion(cx, self, initialWrites, zeroVS);
 			std::cout << "vs1=" << vs1 << std::endl;
 			ASSERT(vs1 > zeroVS);
 
-			state uint64_t vs2 = wait(self->writeKeysAndGetLatencyVersion(cx, self, secondWrites, vs1));
+			uint64_t vs2 = co_await self->writeKeysAndGetLatencyVersion(cx, self, secondWrites, vs1);
 			std::cout << "vs2=" << vs2 << std::endl;
 			ASSERT(vs2 > vs1);
 
 		} catch (Error& e) {
 			TraceEvent("ClientMetricError").error(e);
 		}
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp b/fdbserver/workloads/ClientTransactionProfileCorrectness.cpp
similarity index 85%
rename from fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
rename to fdbserver/workloads/ClientTransactionProfileCorrectness.cpp
index a2e830981a4..c27974b958b 100644
--- a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
+++ b/fdbserver/workloads/ClientTransactionProfileCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * ClientTransactionProfileCorrectness.actor.cpp
+ * ClientTransactionProfileCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbclient/Tuple.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 static const Key CLIENT_LATENCY_INFO_PREFIX = "client_latency/"_sr;
 static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = "client_latency_counter/"_sr;
@@ -122,21 +122,21 @@ struct ParserBase {
 	std::function<void(BinaryReader&)> parseErrorCommit = parseEventErrorCommit;
 	virtual ~ParserBase() = 0;
 };
-ParserBase::~ParserBase() {}
+ParserBase::~ParserBase() = default;
 
 struct Parser_V1 : ParserBase {
-	~Parser_V1() override {}
+	~Parser_V1() override = default;
 };
 struct Parser_V2 : ParserBase {
 	Parser_V2() { parseGetVersion = parseEventGetVersion_V2; }
-	~Parser_V2() override {}
+	~Parser_V2() override = default;
 };
 struct Parser_V3 : ParserBase {
 	Parser_V3() {
 		parseGetVersion = parseEventGetVersion_V3;
 		parseCommit = parseEventCommit_V2;
 	}
-	~Parser_V3() override {}
+	~Parser_V3() override = default;
 };
 
 struct ParserFactory {
@@ -201,7 +201,7 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 	double samplingProbability;
 	int64_t trInfoSizeLimit;
 
-	ClientTransactionProfileCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ClientTransactionProfileCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		samplingProbability = getOption(options,
 		                                "samplingProbability"_sr,
 		                                deterministicRandom()->random01() / 10); // rand range 0 - 0.1
@@ -216,8 +216,7 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override {
 		if (clientId == 0) {
-			IKnobCollection::getMutableGlobalKnobCollection().setKnob("csi_status_delay",
-			                                                          KnobValueRef::create(double{ 2.0 })); // 2 seconds
+			setServerKnob("csi_status_delay", KnobValueRef::create(double{ 2.0 })); // 2 seconds
 			return changeProfilingParameters(cx, trInfoSizeLimit, samplingProbability);
 		}
 		return Void();
@@ -287,9 +286,9 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 		return true;
 	}
 
-	ACTOR Future<Void> changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) {
+	Future<Void> changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) {
 
-		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+		co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
 			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -298,37 +297,36 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
 			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
 			return Void();
-		}));
-		return Void();
+		});
 	}
 
-	ACTOR Future<bool> _check(Database cx, ClientTransactionProfileCorrectnessWorkload* self) {
-		wait(self->changeProfilingParameters(cx, self->trInfoSizeLimit, 0)); // Disable sampling
+	Future<bool> _check(Database cx, ClientTransactionProfileCorrectnessWorkload* self) {
+		co_await self->changeProfilingParameters(cx, self->trInfoSizeLimit, 0); // Disable sampling
 		// FIXME: Better way to ensure that all client profile data has been flushed to the database
-		wait(delay(CLIENT_KNOBS->CSI_STATUS_DELAY));
+		co_await delay(CLIENT_KNOBS->CSI_STATUS_DELAY);
 
-		state Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
-		state int64_t counter;
-		state RangeResult txInfoEntries;
+		Key clientLatencyAtomicCtr = CLIENT_LATENCY_INFO_CTR_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
+		int64_t counter{ 0 };
+		RangeResult txInfoEntries;
 		Optional<Value> ctrValue =
-		    wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+		    co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
 			    tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			    tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			    return tr->get(clientLatencyAtomicCtr);
-		    }));
+		    });
 		counter = ctrValue.present() ? BinaryReader::fromStringRef<int64_t>(ctrValue.get(), Unversioned()) : 0;
-		state Key clientLatencyName = CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
-
-		state KeySelector begin =
-		    firstGreaterOrEqual(CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin));
-		state KeySelector end = firstGreaterOrEqual(strinc(begin.getKey()));
-		state int keysLimit = 10;
-		state Transaction tr(cx);
-		loop {
+		Key clientLatencyName = CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin);
+
+		KeySelector begin = firstGreaterOrEqual(CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin));
+		KeySelector end = firstGreaterOrEqual(strinc(begin.getKey()));
+		int keysLimit = 10;
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				state RangeResult kvRange = wait(tr.getRange(begin, end, keysLimit));
+				RangeResult kvRange = co_await tr.getRange(begin, end, keysLimit);
 				if (kvRange.empty())
 					break;
 				txInfoEntries.arena().dependsOn(kvRange.arena());
@@ -336,9 +334,12 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 				begin = firstGreaterThan(kvRange.back().key);
 				tr.reset();
 			} catch (Error& e) {
-				if (e.code() == error_code_transaction_too_old)
-					keysLimit = std::max(1, keysLimit / 2);
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid() && err.code() == error_code_transaction_too_old)
+				keysLimit = std::max(1, keysLimit / 2);
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
 
@@ -356,7 +357,7 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 		TraceEvent(SevInfo, "ClientTransactionProfilingContentsSize").detail("ContentsSize", contentsSize);
 
 		// Check if the data format is as expected
-		return self->checkTxInfoEntriesFormat(txInfoEntries);
+		co_return self->checkTxInfoEntriesFormat(txInfoEntries);
 	}
 
 	Future<bool> check(Database const& cx) override {
diff --git a/fdbserver/workloads/ClientWorkload.actor.cpp b/fdbserver/workloads/ClientWorkload.cpp
similarity index 57%
rename from fdbserver/workloads/ClientWorkload.actor.cpp
rename to fdbserver/workloads/ClientWorkload.cpp
index 3eec908fd34..e171d1cea3d 100644
--- a/fdbserver/workloads/ClientWorkload.actor.cpp
+++ b/fdbserver/workloads/ClientWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * ClientWorkload.actor.cpp
+ * ClientWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,15 @@
  * limitations under the License.
  */
 
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 
 #include <fmt/format.h>
+#include <type_traits>
 
 #include "flow/ApiVersion.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 class WorkloadProcessState {
 	IPAddress childAddress;
@@ -34,68 +34,64 @@ class WorkloadProcessState {
 	Future<Void> processActor;
 	Promise<Void> init;
 
-	WorkloadProcessState(int clientId) : clientId(clientId) { processActor = processStart(this); }
+	explicit WorkloadProcessState(int clientId) : clientId(clientId) { processActor = processStart(); }
 
 	~WorkloadProcessState() {
 		TraceEvent("ShutdownClientForWorkload", id).log();
 		g_simulator->destroyProcess(childProcess);
 	}
 
-	ACTOR static Future<Void> initializationDone(WorkloadProcessState* self, ISimulator::ProcessInfo* parent) {
-		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
-		self->init.send(Void());
-		wait(Never());
+	Future<Void> initializationDone(ISimulator::ProcessInfo* parent) {
+		co_await g_simulator->onProcess(parent, TaskPriority::DefaultYield);
+		init.send(Void());
+		co_await Future<Void>(Never());
 		ASSERT(false); // does not happen
-		return Void();
 	}
 
-	ACTOR static Future<Void> processStart(WorkloadProcessState* self) {
-		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
-		state std::vector<Future<Void>> futures;
+	Future<Void> processStart() {
+		ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
+		std::vector<Future<Void>> futures;
 		if (parent->address.isV6()) {
-			self->childAddress =
-			    IPAddress::parse(fmt::format("2001:fdb1:fdb2:fdb3:fdb4:fdb5:fdb6:{:04x}", self->clientId + 2)).get();
+			childAddress =
+			    IPAddress::parse(fmt::format("2001:fdb1:fdb2:fdb3:fdb4:fdb5:fdb6:{:04x}", clientId + 2)).get();
 		} else {
-			self->childAddress = IPAddress::parse(fmt::format("192.168.0.{}", self->clientId + 2)).get();
+			childAddress = IPAddress::parse(fmt::format("192.168.0.{}", clientId + 2)).get();
 		}
-		self->processName = fmt::format("TestClient{}", self->clientId);
+		processName = fmt::format("TestClient{}", clientId);
 		Standalone<StringRef> newZoneId(deterministicRandom()->randomUniqueID().toString());
 		auto locality = LocalityData(Optional<Standalone<StringRef>>(), newZoneId, newZoneId, parent->locality.dcId());
 		auto dataFolder = joinPath(popPath(parent->dataFolder), deterministicRandom()->randomUniqueID().toString());
 		platform::createDirectory(dataFolder);
-		TraceEvent("StartingClientWorkloadProcess", self->id)
-		    .detail("Name", self->processName)
-		    .detail("Address", self->childAddress);
-		self->childProcess = g_simulator->newProcess(self->processName.c_str(),
-		                                             self->childAddress,
-		                                             1,
-		                                             parent->address.isTLS(),
-		                                             1,
-		                                             locality,
-		                                             ProcessClass(ProcessClass::TesterClass, ProcessClass::AutoSource),
-		                                             dataFolder.c_str(),
-		                                             parent->coordinationFolder.c_str(),
-		                                             parent->protocolVersion,
-		                                             false);
-		self->childProcess->excludeFromRestarts = true;
-		wait(g_simulator->onProcess(self->childProcess, TaskPriority::DefaultYield));
+		TraceEvent("StartingClientWorkloadProcess", id).detail("Name", processName).detail("Address", childAddress);
+		childProcess = g_simulator->newProcess(processName.c_str(),
+		                                       childAddress,
+		                                       1,
+		                                       parent->address.isTLS(),
+		                                       1,
+		                                       locality,
+		                                       ProcessClass(ProcessClass::TesterClass, ProcessClass::AutoSource),
+		                                       dataFolder.c_str(),
+		                                       parent->coordinationFolder.c_str(),
+		                                       parent->protocolVersion,
+		                                       false);
+		childProcess->excludeFromRestarts = true;
+		co_await g_simulator->onProcess(childProcess, TaskPriority::DefaultYield);
 		try {
 			FlowTransport::createInstance(true, 1, WLTOKEN_RESERVED_COUNT);
 			Sim2FileSystem::newFileSystem();
 			auto addr = g_simulator->getCurrentProcess()->address;
 			futures.push_back(FlowTransport::transport().bind(addr, addr));
-			futures.push_back(success((self->childProcess->onShutdown())));
-			TraceEvent("ClientWorkloadProcessInitialized", self->id).log();
-			futures.push_back(initializationDone(self, parent));
-			wait(waitForAny(futures));
+			futures.push_back(success(childProcess->onShutdown()));
+			TraceEvent("ClientWorkloadProcessInitialized", id).log();
+			futures.push_back(initializationDone(parent));
+			co_await waitForAny(futures);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
-				return Void();
+				co_return;
 			}
 			ASSERT(false);
 		}
 		ASSERT(false);
-		return Void();
 	}
 
 	static std::vector<WorkloadProcessState*>& states() {
@@ -142,25 +138,22 @@ struct WorkloadProcess {
 		}
 	}
 
-	ACTOR static Future<Void> openDatabase(WorkloadProcess* self,
-	                                       ClientWorkload::CreateWorkload childCreator,
-	                                       WorkloadContext wcx) {
-		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
-		state Optional<Error> err;
+	Future<Void> openDatabase(ClientWorkload::CreateWorkload childCreator, WorkloadContext wcx) {
+		ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
+		Optional<Error> err;
 		wcx.dbInfo = Reference<AsyncVar<struct ServerDBInfo> const>();
-		wait(self->processState->initialized());
-		wait(g_simulator->onProcess(self->childProcess(), TaskPriority::DefaultYield));
+		co_await processState->initialized();
+		co_await g_simulator->onProcess(childProcess(), TaskPriority::DefaultYield);
 		try {
-			self->createDatabase(childCreator, wcx);
+			createDatabase(childCreator, wcx);
 		} catch (Error& e) {
 			ASSERT(e.code() != error_code_actor_cancelled);
 			err = e;
 		}
-		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
+		co_await g_simulator->onProcess(parent, TaskPriority::DefaultYield);
 		if (err.present()) {
 			throw err.get();
 		}
-		return Void();
 	}
 
 	ISimulator::ProcessInfo* childProcess() { return processState->childProcess; }
@@ -175,55 +168,62 @@ struct WorkloadProcess {
 		childWorkloadContext.ccr = wcx.ccr;
 		childWorkloadContext.options = wcx.options;
 		childWorkloadContext.sharedRandomNumber = wcx.sharedRandomNumber;
-		databaseOpened = openDatabase(this, childCreator, childWorkloadContext);
+		databaseOpened = openDatabase(childCreator, childWorkloadContext);
 	}
 
-	ACTOR static void destroy(WorkloadProcess* self) {
-		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
-		wait(g_simulator->onProcess(self->childProcess(), TaskPriority::DefaultYield));
+	Future<Void> destroy() {
+		ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
+		co_await g_simulator->onProcess(childProcess(), TaskPriority::DefaultYield);
 		TraceEvent("DeleteWorkloadProcess").backtrace();
-		delete self;
-		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
+		delete this;
+		co_await g_simulator->onProcess(parent, TaskPriority::DefaultYield);
 	}
 
 	std::string description() { return desc; }
 
 	// This actor will keep a reference to a future alive, switch to another process and then return. If the future
 	// count of `f` is 1, this will cause the future to be destroyed in the process `process`
-	ACTOR template <class T>
-	static void cancelChild(ISimulator::ProcessInfo* process, Future<T> f) {
-		wait(g_simulator->onProcess(process, TaskPriority::DefaultYield));
+	template <class T>
+	static Future<Void> cancelChild(ISimulator::ProcessInfo* process, Future<T> f) {
+		co_await g_simulator->onProcess(process, TaskPriority::DefaultYield);
 	}
 
-	ACTOR template <class Ret, class Fun>
-	Future<Ret> runActor(WorkloadProcess* self, Fun f) {
-		state Optional<Error> err;
-		state Ret res;
-		state Future<Ret> fut;
-		state ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
-		wait(self->databaseOpened);
-		wait(g_simulator->onProcess(self->childProcess(), TaskPriority::DefaultYield));
+	template <class Ret, class Fun>
+	Future<Ret> runActor(Fun f) {
+		Optional<Error> err;
+		using ResultHolder = std::conditional_t<std::is_same_v<Ret, Void>, bool, Ret>;
+		[[maybe_unused]] ResultHolder res{};
+		Future<Ret> fut;
+		ISimulator::ProcessInfo* parent = g_simulator->getCurrentProcess();
+		co_await databaseOpened;
+		co_await g_simulator->onProcess(childProcess(), TaskPriority::DefaultYield);
 		try {
-			fut = f(self->cx);
-			Ret r = wait(fut);
-			res = r;
+			fut = f(cx);
+			if constexpr (std::is_same_v<Ret, Void>) {
+				co_await fut;
+			} else {
+				res = co_await fut;
+			}
 		} catch (Error& e) {
 			// if we're getting cancelled, we could run in the scope of the parent process, but we're not allowed to
 			// cancel `fut` in any other process than the child process. So we're going to pass the future to an
 			// uncancellable actor (it has to be uncancellable because if we got cancelled here we can't wait on
 			// anything) which will then destroy the future on the child process.
-			cancelChild(self->childProcess(), fut);
+			cancelChild(childProcess(), fut);
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
 			}
 			err = e;
 		}
 		fut = Future<Ret>();
-		wait(g_simulator->onProcess(parent, TaskPriority::DefaultYield));
+		co_await g_simulator->onProcess(parent, TaskPriority::DefaultYield);
 		if (err.present()) {
 			throw err.get();
 		}
-		return res;
+		if constexpr (std::is_same_v<Ret, Void>) {
+		} else {
+			co_return res;
+		}
 	}
 };
 
@@ -232,7 +232,7 @@ ClientWorkload::ClientWorkload(CreateWorkload const& childCreator, WorkloadConte
 
 ClientWorkload::~ClientWorkload() {
 	TraceEvent(SevDebug, "DestroyClientWorkload").backtrace();
-	WorkloadProcess::destroy(impl);
+	impl->destroy();
 }
 
 std::string ClientWorkload::description() const {
@@ -244,17 +244,16 @@ Future<Void> ClientWorkload::initialized() {
 }
 
 Future<Void> ClientWorkload::setup(Database const& cx) {
-	return impl->runActor<Void>(impl, [this](Database const& db) { return impl->child->setup(db); });
+	return impl->runActor<Void>([this](Database const& db) { return impl->child->setup(db); });
 }
 Future<Void> ClientWorkload::start(Database const& cx) {
-	return impl->runActor<Void>(impl, [this](Database const& db) { return impl->child->start(db); });
+	return impl->runActor<Void>([this](Database const& db) { return impl->child->start(db); });
 }
 Future<bool> ClientWorkload::check(Database const& cx) {
-	return impl->runActor<bool>(impl, [this](Database const& db) { return impl->child->check(db); });
+	return impl->runActor<bool>([this](Database const& db) { return impl->child->check(db); });
 }
 Future<std::vector<PerfMetric>> ClientWorkload::getMetrics() {
-	return impl->runActor<std::vector<PerfMetric>>(impl,
-	                                               [this](Database const& db) { return impl->child->getMetrics(); });
+	return impl->runActor<std::vector<PerfMetric>>([this](Database const& db) { return impl->child->getMetrics(); });
 }
 void ClientWorkload::getMetrics(std::vector<PerfMetric>& m) {
 	ASSERT(false);
diff --git a/fdbserver/workloads/ClogRemoteTLog.actor.cpp b/fdbserver/workloads/ClogRemoteTLog.cpp
similarity index 79%
rename from fdbserver/workloads/ClogRemoteTLog.actor.cpp
rename to fdbserver/workloads/ClogRemoteTLog.cpp
index 86a507297cb..32f6d6f3052 100644
--- a/fdbserver/workloads/ClogRemoteTLog.actor.cpp
+++ b/fdbserver/workloads/ClogRemoteTLog.cpp
@@ -7,9 +7,10 @@
 #include "fdbrpc/PerfMetric.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Buggify.h"
 #include "flow/Error.h"
 #include "flow/IPAddress.h"
@@ -20,8 +21,6 @@
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 struct ClogRemoteTLog : TestWorkload {
 	static constexpr auto NAME = "ClogRemoteTLog";
 
@@ -53,7 +52,7 @@ struct ClogRemoteTLog : TestWorkload {
 	    cloggedRemoteTLog; // set after clogging is done, we use this state to ensure that it's
 	                       // eventually not present in dbInfo (which implies it was excluded by gray failure)
 
-	ClogRemoteTLog(const WorkloadContext& wctx) : TestWorkload(wctx) {
+	explicit ClogRemoteTLog(const WorkloadContext& wctx) : TestWorkload(wctx) {
 		enabled =
 		    (clientId == 0); // only run this workload for a single client, and that too the first client (by its id)
 		testDuration = getOption(options, "testDuration"_sr, 120);
@@ -139,18 +138,18 @@ struct ClogRemoteTLog : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Optional<double>> measureMaxSSLag(ClogRemoteTLog* self, Database db) {
-		StatusObject status = wait(StatusClient::statusFetcher(db));
+	Future<Optional<double>> measureMaxSSLag(Database db) {
+		StatusObject status = co_await StatusClient::statusFetcher(db);
 		StatusObjectReader reader(status);
 		StatusObjectReader cluster;
 		StatusObjectReader processMap;
 		if (!reader.get("cluster", cluster)) {
 			TraceEvent("NoCluster");
-			return Optional<double>();
+			co_return Optional<double>();
 		}
 		if (!cluster.get("processes", processMap)) {
 			TraceEvent("NoProcesses");
-			return Optional<double>();
+			co_return Optional<double>();
 		}
 		double maxSSLag{ -1 };
 		for (auto p : processMap.obj()) {
@@ -176,11 +175,11 @@ struct ClogRemoteTLog : TestWorkload {
 		}
 		TraceEvent("MaxSSDataLag")
 		    .detail("SecondLag", maxSSLag == -1 ? "none" : std::to_string(maxSSLag))
-		    .detail("SecondThreshold", self->lagThreshold);
+		    .detail("SecondThreshold", lagThreshold);
 		if (maxSSLag == -1) {
-			return Optional<double>();
+			co_return Optional<double>();
 		} else {
-			return maxSSLag;
+			co_return maxSSLag;
 		}
 	}
 
@@ -219,14 +218,14 @@ struct ClogRemoteTLog : TestWorkload {
 		return false;
 	}
 
-	ACTOR static Future<bool> grayFailureStatusCheck(Database db, NetworkAddress cloggedRemoteTLog) {
-		StatusObject status = wait(StatusClient::statusFetcher(db));
+	static Future<bool> grayFailureStatusCheck(Database db, NetworkAddress cloggedRemoteTLog) {
+		StatusObject status = co_await StatusClient::statusFetcher(db);
 		StatusObjectReader reader(status);
 
 		if (statusError(reader)) {
 			// If there is some error to get the status (e.g. network issue), we let gray failure status check pass
 			// since that's not what we are testing for here.
-			return true;
+			co_return true;
 		}
 
 		StatusObjectReader cluster;
@@ -234,7 +233,7 @@ struct ClogRemoteTLog : TestWorkload {
 		StatusObjectReader grayFailure;
 		if (!cluster.get("gray_failure", grayFailure)) {
 			TraceEvent("NoGrayFailure");
-			return false;
+			co_return false;
 		}
 		ASSERT(grayFailure.has("excluded_servers"));
 		StatusArray excludedProcesses = grayFailure["excluded_servers"].get_array();
@@ -245,38 +244,40 @@ struct ClogRemoteTLog : TestWorkload {
 			    .detail("Address", process["address"].get_str())
 			    .detail("Ts", process["time"].get_real());
 		}
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<std::vector<IPAddress>> getRemoteSSIPs(Database db) {
-		state std::vector<IPAddress> ret;
-		state Transaction tr(db);
-		loop {
+	static Future<std::vector<IPAddress>> getRemoteSSIPs(Database db) {
+		std::vector<IPAddress> ret;
+		Transaction tr(db);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-				    wait(NativeAPI::getServerListAndProcessClasses(&tr));
+				    co_await NativeAPI::getServerListAndProcessClasses(&tr);
 				for (auto& [ssi, p] : results) {
-					if (ssi.locality.dcId().present() && g_simulator->remoteDcId.present() &&
-					    ssi.locality.dcId().get() == g_simulator->remoteDcId.get()) {
+					if (ssi.locality.dcId().present() && fdbSimulationPolicyState().remoteDcId.present() &&
+					    ssi.locality.dcId().get() == fdbSimulationPolicyState().remoteDcId.get()) {
 						ret.push_back(ssi.address().ip);
 					}
 				}
-				return ret;
+				co_return ret;
 			} catch (Error& e) {
-				if (e.code() != error_code_actor_cancelled) {
-					TraceEvent("GetRemoteSSIPsError").error(e);
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.code() != error_code_actor_cancelled) {
+				TraceEvent("GetRemoteSSIPsError").error(err);
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	static std::vector<NetworkAddress> getRemoteTLogs(ClogRemoteTLog* self) {
+	std::vector<NetworkAddress> getRemoteTLogs() {
 		std::vector<NetworkAddress> remoteTLogIPs;
-		for (const auto& tLogSet : self->dbInfo->get().logSystemConfig.tLogs) {
+		for (const auto& tLogSet : dbInfo->get().logSystemConfig.tLogs) {
 			if (tLogSet.isLocal) {
 				continue;
 			}
@@ -287,20 +288,20 @@ struct ClogRemoteTLog : TestWorkload {
 		return remoteTLogIPs;
 	}
 
-	ACTOR static Future<Void> clogRemoteTLog(ClogRemoteTLog* self, Database db) {
-		wait(delay(self->clogInitDelay));
+	Future<Void> clogRemoteTLog(Database db) {
+		co_await delay(clogInitDelay);
 
 		// Ensure db is ready
-		while (self->dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
-			wait(self->dbInfo->onChange());
+		while (dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
+			co_await dbInfo->onChange();
 		}
 
 		// Then, get all remote TLog IPs
-		state std::vector<NetworkAddress> remoteTLogs = getRemoteTLogs(self);
+		std::vector<NetworkAddress> remoteTLogs = getRemoteTLogs();
 		ASSERT(!remoteTLogs.empty());
 
 		// Then, get all remote SS IPs
-		std::vector<IPAddress> remoteSSIPs = wait(getRemoteSSIPs(db));
+		std::vector<IPAddress> remoteSSIPs = co_await getRemoteSSIPs(db);
 		ASSERT(!remoteSSIPs.empty());
 
 		// Then, attempt to find a remote tlog that is not on the same machine as a remote SS
@@ -314,15 +315,15 @@ struct ClogRemoteTLog : TestWorkload {
 		// If we can find such a machine that is just running a remote tlog, then we will do extra checking at the end
 		// (in check() method). If we can't find such a machine, we pick a random machhine and still run the test to
 		// ensure no crashes or correctness issues are observed.
-		self->cloggedRemoteTLog = isolatedRemoteTLog.present()
-		                              ? isolatedRemoteTLog.get()
-		                              : self->cloggedRemoteTLog =
-		                                    remoteTLogs[deterministicRandom()->randomInt(0, remoteTLogs.size())];
-		ASSERT(self->cloggedRemoteTLog.present());
+		cloggedRemoteTLog = isolatedRemoteTLog.present()
+		                        ? isolatedRemoteTLog.get()
+		                        : cloggedRemoteTLog =
+		                              remoteTLogs[deterministicRandom()->randomInt(0, remoteTLogs.size())];
+		ASSERT(cloggedRemoteTLog.present());
 
 		// Then, find all processes that the remote tlog will have degraded connection with
-		IPAddress cc = self->dbInfo->get().clusterInterface.address().ip;
-		state std::vector<IPAddress> processes;
+		IPAddress cc = dbInfo->get().clusterInterface.address().ip;
+		std::vector<IPAddress> processes;
 		for (const auto& process : g_simulator->getAllProcesses()) {
 			const auto& ip = process->address.ip;
 			if (process->startingClass != ProcessClass::TesterClass && ip != cc) {
@@ -334,25 +335,24 @@ struct ClogRemoteTLog : TestWorkload {
 		// Finally, start the clogging between the remote tlog and the processes calculated above
 		int numClogged{ 0 };
 		for (const auto& ip : processes) {
-			if (self->cloggedRemoteTLog.get().ip == ip) {
+			if (cloggedRemoteTLog.get().ip == ip) {
 				continue;
 			}
-			double clogDuration = self->testDuration * (0.5 + 0.4 * deterministicRandom()->random01());
+			double clogDuration = testDuration * (0.5 + 0.4 * deterministicRandom()->random01());
 			// clogDuration must be less than testDuration to ensure that the clogging ends before the test ends
-			g_simulator->clogPair(ip, self->cloggedRemoteTLog.get().ip, clogDuration);
+			g_simulator->clogPair(ip, cloggedRemoteTLog.get().ip, clogDuration);
 			TraceEvent("ClogRemoteTLog")
-			    .detail("SrcIP", self->cloggedRemoteTLog->ip)
+			    .detail("SrcIP", cloggedRemoteTLog->ip)
 			    .detail("DstIP", ip)
 			    .detail("Duration", clogDuration);
 			numClogged++;
 		}
 
 		if (isolatedRemoteTLog.present() && numClogged > 1) {
-			self->doCheck = true;
+			doCheck = true;
 		}
 
-		wait(Never());
-		return Void();
+		co_await Future<Void>(Never());
 	}
 
 	// Returns true if and only if the provided remote tlog `addr` is not in dbInfo
@@ -370,21 +370,20 @@ struct ClogRemoteTLog : TestWorkload {
 		return true;
 	}
 
-	ACTOR Future<Void> workload(ClogRemoteTLog* self, Database db) {
-		state Future<Void> clog = self->clogRemoteTLog(self, db);
-		state TestState testState = TestState::TEST_INIT;
+	Future<Void> workload(ClogRemoteTLog* self, Database db) {
+		Future<Void> clog = clogRemoteTLog(db);
+		TestState testState = TestState::TEST_INIT;
 		self->actualStatePath.push_back(testState);
-		state bool statusCheckPassed = false;
-		loop {
-			wait(delay(self->lagMeasurementFrequency));
-			Optional<double> ssLag = wait(measureMaxSSLag(self, db));
+		bool statusCheckPassed = false;
+		while (true) {
+			co_await delay(self->lagMeasurementFrequency);
+			Optional<double> ssLag = co_await measureMaxSSLag(db);
 			if (!ssLag.present()) {
 				continue;
 			}
 			// See if ss lag state changed
-			state TestState localState =
-			    ssLag.get() < self->lagThreshold ? TestState::SS_LAG_NORMAL : TestState::SS_LAG_HIGH;
-			state bool stateTransition = localState != testState;
+			TestState localState = ssLag.get() < self->lagThreshold ? TestState::SS_LAG_NORMAL : TestState::SS_LAG_HIGH;
+			bool stateTransition = localState != testState;
 			// If ss lag state did not change, see if clogged remote tlog got excluded
 			if (!stateTransition) {
 				const bool acceptingCommits = self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS;
@@ -402,7 +401,7 @@ struct ClogRemoteTLog : TestWorkload {
 				    remoteTLogNotInDbInfo(self->cloggedRemoteTLog.get(), self->dbInfo->get())) {
 					localState = TestState::CLOGGED_REMOTE_TLOG_EXCLUDED;
 					if (!statusCheckPassed) {
-						wait(store(statusCheckPassed, grayFailureStatusCheck(db, self->cloggedRemoteTLog.get())));
+						statusCheckPassed = co_await grayFailureStatusCheck(db, self->cloggedRemoteTLog.get());
 						ASSERT(statusCheckPassed);
 					}
 					stateTransition = localState != testState;
diff --git a/fdbserver/workloads/ClogSingleConnection.actor.cpp b/fdbserver/workloads/ClogSingleConnection.cpp
similarity index 87%
rename from fdbserver/workloads/ClogSingleConnection.actor.cpp
rename to fdbserver/workloads/ClogSingleConnection.cpp
index 956fd8ed723..cd36287e83d 100644
--- a/fdbserver/workloads/ClogSingleConnection.actor.cpp
+++ b/fdbserver/workloads/ClogSingleConnection.cpp
@@ -1,5 +1,5 @@
 /*
- * ClogSingleConnection.actor.cpp
+ * ClogSingleConnection.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 class ClogSingleConnectionWorkload : public TestWorkload {
 	double delaySeconds;
@@ -30,7 +27,7 @@ class ClogSingleConnectionWorkload : public TestWorkload {
 
 public:
 	static constexpr auto NAME = "ClogSingleConnection";
-	ClogSingleConnectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ClogSingleConnectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		auto minDelay = getOption(options, "minDelay"_sr, 0.0);
 		auto maxDelay = getOption(options, "maxDelay"_sr, 10.0);
 		ASSERT_LE(minDelay, maxDelay);
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.cpp
similarity index 77%
rename from fdbserver/workloads/ClogTlog.actor.cpp
rename to fdbserver/workloads/ClogTlog.cpp
index abae815babc..ffacb3a9f7f 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.cpp
@@ -1,5 +1,5 @@
 /*
- * ClogTlog.actor.cpp
+ * ClogTlog.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,15 @@
  * limitations under the License.
  */
 
-#include "fdbclient/GenericManagementAPI.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/GenericManagementAPI.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "flow/CodeProbe.h"
 #include "flow/NetworkAddress.h"
@@ -34,7 +34,7 @@
 #include "flow/Trace.h"
 #include "flow/flow.h"
 #include "flow/network.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 // This workload tests a gray failure scenario: a single TLog is network-partitioned from
 // the rest of the cluster, except the cluster controller. This will cause proxies to
@@ -51,7 +51,7 @@ struct ClogTlogWorkload : TestWorkload {
 	Optional<NetworkAddress> tlog; // the tlog to be clogged with all other processes except the CC
 	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
 
-	ClogTlogWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ClogTlogWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		testDuration = getOption(options, "testDuration"_sr, 1000.0);
 	}
@@ -79,7 +79,7 @@ struct ClogTlogWorkload : TestWorkload {
 				ips.push_back(ip);
 			}
 		}
-		ASSERT(ips.size() > 0);
+		ASSERT(!ips.empty());
 
 		// Find all primary tlogs
 		std::vector<NetworkAddress> logs; // all primary logs except CC
@@ -97,7 +97,7 @@ struct ClogTlogWorkload : TestWorkload {
 				}
 			}
 		}
-		ASSERT(logs.size() > 0 && tlog.present());
+		ASSERT(!logs.empty() && tlog.present());
 
 		// clog pairs
 		for (const auto& ip : ips) {
@@ -128,50 +128,54 @@ struct ClogTlogWorkload : TestWorkload {
 		cloggedPairs.clear();
 	}
 
-	ACTOR static Future<Void> excludeFailedLog(ClogTlogWorkload* self, Database cx) {
-		loop choose {
-			when(wait(self->dbInfo->onChange())) {
-				if (self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
-					return Void();
+	Future<Void> excludeFailedLog(Database cx) {
+		while (true) {
+			auto choice = co_await race(dbInfo->onChange(), delay(30));
+			if (choice.index() == 0) {
+
+				if (dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
+					co_return;
 				}
-			}
-			when(wait(delay(30))) {
+			} else if (choice.index() == 1) {
+
 				// recovery state hasn't changed in 30s, exclude the failed tlog
 				CODE_PROBE(true, "Exclude failed tlog");
 				TraceEvent("ExcludeFailedLog")
-				    .detail("TLog", self->tlog.get())
-				    .detail("RecoveryState", self->dbInfo->get().recoveryState);
-				std::string modes = "exclude=" + formatIpPort(self->tlog.get().ip, self->tlog.get().port);
-				ConfigurationResult r = wait(ManagementAPI::changeConfig(cx.getReference(), modes, /*force=*/true));
+				    .detail("TLog", tlog.get())
+				    .detail("RecoveryState", dbInfo->get().recoveryState);
+				std::string modes = "exclude=" + formatIpPort(tlog.get().ip, tlog.get().port);
+				ConfigurationResult r = co_await ManagementAPI::changeConfig(cx.getReference(), modes, /*force=*/true);
 				TraceEvent("ExcludeFailedLog").detail("Result", r);
-				return Void();
+				co_return;
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
 
-	ACTOR Future<Void> clogClient(ClogTlogWorkload* self, Database cx) {
+	Future<Void> clogClient(ClogTlogWorkload* self, Database cx) {
 		if (deterministicRandom()->coinflip()) {
 			self->useDisconnection = true;
 		}
 
 		// Let cycle workload issue some transactions.
-		wait(delay(20.0));
+		co_await delay(20.0);
 
 		while (self->dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
-			wait(self->dbInfo->onChange());
+			co_await self->dbInfo->onChange();
 		}
 
 		double startTime = now();
-		state double workloadEnd = now() + self->testDuration - 10;
+		double workloadEnd = now() + self->testDuration - 10;
 		TraceEvent("ClogTlog").detail("StartTime", startTime).detail("EndTime", workloadEnd);
 
 		// Clog and wait for recovery to happen
 		self->clogTlog(workloadEnd - now());
 		while (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
-			wait(self->dbInfo->onChange());
+			co_await self->dbInfo->onChange();
 		}
 
-		state bool useGrayFailureToRecover = false;
+		bool useGrayFailureToRecover = false;
 		if (deterministicRandom()->coinflip() && self->useDisconnection) {
 			// Use gray failure instead of exclusion to recover the cluster.
 			TraceEvent("ClogTLogUseGrayFailreToRecover").log();
@@ -180,21 +184,25 @@ struct ClogTlogWorkload : TestWorkload {
 
 		// start exclusion and wait for fully recovery. When using gray failure, the cluster should recover by itself
 		// eventually.
-		state Future<Void> excludeLog = useGrayFailureToRecover ? Never() : excludeFailedLog(self, cx);
-		state Future<Void> onChange = self->dbInfo->onChange();
-		loop choose {
-			when(wait(onChange)) {
+		Future<Void> excludeLog = useGrayFailureToRecover ? Never() : excludeFailedLog(cx);
+		Future<Void> onChange = self->dbInfo->onChange();
+		while (true) {
+			auto choice = co_await race(onChange, delayUntil(workloadEnd));
+			if (choice.index() == 0) {
+
 				if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
 					TraceEvent("ClogDoneFullyRecovered").log();
 					self->unclogAll();
-					return Void();
+					co_return;
 				}
 				onChange = self->dbInfo->onChange();
-			}
-			when(wait(delayUntil(workloadEnd))) {
+			} else if (choice.index() == 1) {
+
 				// Expect to reach fully recovered state before workload ends
 				TraceEvent(SevError, "ClogTLogFailure").detail("RecoveryState", self->dbInfo->get().recoveryState);
-				return Void();
+				co_return;
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
diff --git a/fdbserver/workloads/CommitBugCheck.actor.cpp b/fdbserver/workloads/CommitBugCheck.cpp
similarity index 56%
rename from fdbserver/workloads/CommitBugCheck.actor.cpp
rename to fdbserver/workloads/CommitBugCheck.cpp
index 5ac258632bc..fc4c7549b09 100644
--- a/fdbserver/workloads/CommitBugCheck.actor.cpp
+++ b/fdbserver/workloads/CommitBugCheck.cpp
@@ -1,5 +1,5 @@
 /*
- * CommitBugCheck.actor.cpp
+ * CommitBugCheck.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,137 +18,147 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 // Regression tests for 2 commit related bugs
 struct CommitBugWorkload : TestWorkload {
 	static constexpr auto NAME = "CommitBug";
 	bool success;
 
-	CommitBugWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { success = true; }
+	explicit CommitBugWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { success = true; }
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
 	Future<Void> start(Database const& cx) override { return timeout(bug1(cx, this) && bug2(cx, this), 60, Void()); }
 
-	ACTOR Future<Void> bug1(Database cx, CommitBugWorkload* self) {
-		state Key key = StringRef(format("B1Key%d", self->clientId));
-		state Value val1 = "Value1"_sr;
-		state Value val2 = "Value2"_sr;
+	Future<Void> bug1(Database cx, CommitBugWorkload* self) {
+		Key key = StringRef(format("B1Key%d", self->clientId));
+		Value val1 = "Value1"_sr;
+		Value val2 = "Value2"_sr;
 
-		loop {
-			state Transaction tr(cx);
-			loop {
+		while (true) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					tr.set(key, val1);
-					wait(tr.commit());
+					co_await tr.commit();
 					tr.reset();
 					break;
 				} catch (Error& e) {
-					TraceEvent("CommitBugSetVal1Error").error(e);
-					CODE_PROBE(e.code() == error_code_commit_unknown_result, "Commit unknown result");
-					wait(tr.onError(e));
+					err = e;
 				}
+				TraceEvent("CommitBugSetVal1Error").error(err);
+				CODE_PROBE(err.code() == error_code_commit_unknown_result, "Commit unknown result");
+				co_await tr.onError(err);
 			}
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr.set(key, val2);
-					wait(tr.commit());
+					co_await tr.commit();
 					tr.reset();
 					break;
 				} catch (Error& e) {
-					TraceEvent("CommitBugSetVal2Error").error(e);
-					wait(tr.onError(e));
+					err = e;
 				}
+				TraceEvent("CommitBugSetVal2Error").error(err);
+				co_await tr.onError(err);
 			}
 
-			loop {
+			while (true) {
+				Error err;
 				try {
-					Optional<Value> v = wait(tr.get(key));
+					Optional<Value> v = co_await tr.get(key);
 					if (!v.present() || v.get() != val2) {
 						TraceEvent(SevError, "CommitBugFailed")
 						    .detail("Value", v.present() ? printable(v.get()) : "Not present");
 						self->success = false;
-						return Void();
+						co_return;
 					}
 
 					break;
 				} catch (Error& e) {
-					TraceEvent("CommitBugGetValError").error(e);
-					wait(tr.onError(e));
+					err = e;
 				}
+				TraceEvent("CommitBugGetValError").error(err);
+				co_await tr.onError(err);
 			}
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr.clear(key);
-					wait(tr.commit());
+					co_await tr.commit();
 					tr.reset();
 					break;
 				} catch (Error& e) {
-					TraceEvent("CommitBugClearValError").error(e);
-					wait(tr.onError(e));
+					err = e;
 				}
+				TraceEvent("CommitBugClearValError").error(err);
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR Future<Void> bug2(Database cx, CommitBugWorkload* self) {
-		state Key key = StringRef(format("B2Key%d", self->clientId));
+	Future<Void> bug2(Database cx, CommitBugWorkload* self) {
+		Key key = StringRef(format("B2Key%d", self->clientId));
 
-		state int i;
-		for (i = 0; i < 1000; ++i) {
-			state Transaction tr(cx);
+		for (int i = 0; i < 1000; ++i) {
+			Transaction tr(cx);
 
-			loop {
+			while (true) {
+				Error caughtErr;
 				try {
-					Optional<Value> val = wait(tr.get(key));
-					state int num = 0;
+					Optional<Value> val = co_await tr.get(key);
+					int num = 0;
 					if (val.present()) {
 						num = atoi(val.get().toString().c_str());
 						if (num != i) {
 							TraceEvent(SevError, "CommitBug2Failed").detail("Value", num).detail("Expected", i);
 							self->success = false;
-							return Void();
+							co_return;
 						}
 					}
 
 					TraceEvent("CommitBug2SetKey").detail("Num", i + 1);
 					tr.set(key, StringRef(format("%d", i + 1)));
-					wait(tr.commit());
+					co_await tr.commit();
 					TraceEvent("CommitBug2SetCompleted").detail("Num", i + 1);
 					break;
 				} catch (Error& error) {
-					state Error e = error;
-					if (e.code() != error_code_not_committed && e.code() != error_code_transaction_too_old) {
-						tr.reset();
-						loop {
+					caughtErr = error;
+				}
+				Error e = caughtErr;
+				if (e.code() != error_code_not_committed && e.code() != error_code_transaction_too_old) {
+					tr.reset();
+					while (true) {
+						{
+							Error caughtErr;
 							try {
 								TraceEvent("CommitBug2SetKey").detail("Num", i + 1);
 								tr.set(key, StringRef(format("%d", i + 1)));
 								TraceEvent("CommitBug2SetCompleted").detail("Num", i + 1);
-								wait(tr.commit());
+								co_await tr.commit();
 								break;
 							} catch (Error& err) {
-								wait(tr.onError(err));
+								caughtErr = err;
 							}
+							co_await tr.onError(caughtErr);
 						}
+					}
 
-						break;
-					} else {
-						CODE_PROBE(true, "Commit conflict");
+					break;
+				} else {
+					CODE_PROBE(true, "Commit conflict");
 
-						TraceEvent("CommitBug2Error").error(e).detail("AttemptedNum", i + 1);
-						wait(tr.onError(e));
-					}
+					TraceEvent("CommitBug2Error").error(e).detail("AttemptedNum", i + 1);
+					co_await tr.onError(e);
 				}
 			}
 		}
-
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return success; }
diff --git a/fdbserver/workloads/ConfigureDatabase.actor.cpp b/fdbserver/workloads/ConfigureDatabase.cpp
similarity index 79%
rename from fdbserver/workloads/ConfigureDatabase.actor.cpp
rename to fdbserver/workloads/ConfigureDatabase.cpp
index 3fdf9054ad5..5be1bd81e6e 100644
--- a/fdbserver/workloads/ConfigureDatabase.actor.cpp
+++ b/fdbserver/workloads/ConfigureDatabase.cpp
@@ -1,5 +1,5 @@
 /*
- * ConfigureDatabase.actor.cpp
+ * ConfigureDatabase.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,17 +22,17 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/SimulatedCluster.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/datadistributor/SimulatedCluster.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 static const char* storageMigrationTypes[] = { "perpetual_storage_wiggle=0 storage_migration_type=aggressive",
 	                                           "perpetual_storage_wiggle=1",
@@ -54,9 +54,9 @@ static const char* backupTypes[] = { "backup_worker_enabled:=0", "backup_worker_
 
 std::string generateRegions() {
 	std::string result;
-	if (g_simulator->physicalDatacenters == 1 ||
-	    (g_simulator->physicalDatacenters == 2 && deterministicRandom()->random01() < 0.25) ||
-	    g_simulator->physicalDatacenters == 3) {
+	if (fdbSimulationPolicyState().physicalDatacenters == 1 ||
+	    (fdbSimulationPolicyState().physicalDatacenters == 2 && deterministicRandom()->random01() < 0.25) ||
+	    fdbSimulationPolicyState().physicalDatacenters == 3) {
 		return " usable_regions=1 regions=\"\"";
 	}
 
@@ -91,7 +91,7 @@ std::string generateRegions() {
 
 	int maxSatelliteLogs = getMaxSatelliteLogs();
 
-	if (g_simulator->physicalDatacenters > 3 && deterministicRandom()->random01() < 0.5) {
+	if (fdbSimulationPolicyState().physicalDatacenters > 3 && deterministicRandom()->random01() < 0.5) {
 		StatusObject primarySatelliteObj;
 		primarySatelliteObj["id"] = "2";
 		primarySatelliteObj["priority"] = 1;
@@ -108,7 +108,7 @@ std::string generateRegions() {
 			remoteSatelliteObj["satellite_logs"] = deterministicRandom()->randomInt(1, maxSatelliteLogs + 1);
 		remoteDcArr.push_back(remoteSatelliteObj);
 
-		if (g_simulator->physicalDatacenters > 5 && deterministicRandom()->random01() < 0.5) {
+		if (fdbSimulationPolicyState().physicalDatacenters > 5 && deterministicRandom()->random01() < 0.5) {
 			StatusObject primarySatelliteObjB;
 			primarySatelliteObjB["id"] = "4";
 			primarySatelliteObjB["priority"] = 1;
@@ -240,26 +240,21 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter retries;
 
-	ConfigureDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), retries("Retries") {
+	explicit ConfigureDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), retries("Retries") {
 		testDuration = getOption(options, "testDuration"_sr, 200.0);
 		allowDescriptorChange =
 		    getOption(options, "allowDescriptorChange"_sr, SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT);
-		allowTestStorageMigration =
-		    getOption(options, "allowTestStorageMigration"_sr, false) && g_simulator->allowStorageMigrationTypeChange;
+		allowTestStorageMigration = getOption(options, "allowTestStorageMigration"_sr, false) &&
+		                            fdbSimulationPolicyState().allowStorageMigrationTypeChange;
 		storageMigrationCompatibleConf = getOption(options, "storageMigrationCompatibleConf"_sr, false);
 		waitStoreTypeCheck = getOption(options, "waitStoreTypeCheck"_sr, false);
 		downgradeTest1 = getOption(options, "downgradeTest1"_sr, false);
 		storageEngineExcludeTypes = getOption(options, "storageEngineExcludeTypes"_sr);
-		g_simulator->usableRegions = 1;
+		fdbSimulationPolicyState().usableRegions = 1;
 	}
 
 	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("Attrition"); }
 
-	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
-
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-	Future<bool> check(Database const& cx) override { return _check(this, cx); }
-
 	void getMetrics(std::vector<PerfMetric>& m) override { m.push_back(retries.getMetric()); }
 
 	static inline uint64_t valueToUInt64(const StringRef& v) {
@@ -268,31 +263,27 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 		return x;
 	}
 
-	static inline Standalone<StringRef> getDatabaseName(ConfigureDatabaseWorkload* self, int dbIndex) {
-		return StringRef(format("DestroyDB%d", dbIndex));
-	}
+	inline Standalone<StringRef> getDatabaseName(int dbIndex) { return StringRef(format("DestroyDB%d", dbIndex)); }
 
 	static Future<ConfigurationResult> IssueConfigurationChange(Database cx, const std::string& config, bool force) {
 		printf("Issuing configuration change: %s\n", config.c_str());
 		return ManagementAPI::changeConfig(cx.getReference(), config, force);
 	}
 
-	ACTOR Future<Void> _setup(Database cx, ConfigureDatabaseWorkload* self) {
-		wait(success(ManagementAPI::changeConfig(cx.getReference(), "single storage_migration_type=aggressive", true)));
-		return Void();
+	Future<Void> setup(Database const& cx) override {
+		co_await ManagementAPI::changeConfig(cx.getReference(), "single storage_migration_type=aggressive", true);
 	}
 
-	ACTOR Future<Void> _start(ConfigureDatabaseWorkload* self, Database cx) {
-		DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> start(Database const& cx) override {
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		TraceEvent("ConfigureDatabase_Config").detail("Config", config.toString());
 		if (!SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-			self->storageEngineExcludeTypes.push_back((int)SimulationStorageEngine::SHARDED_ROCKSDB);
+			storageEngineExcludeTypes.push_back((int)SimulationStorageEngine::SHARDED_ROCKSDB);
 		}
-		if (self->clientId == 0) {
-			self->clients.push_back(timeout(self->singleDB(self, cx), self->testDuration, Void()));
-			wait(waitForAll(self->clients));
+		if (clientId == 0) {
+			clients.push_back(timeout(singleDB(this, cx), testDuration, Void()));
+			co_await waitForAll(clients);
 		}
-		return Void();
 	}
 
 	// Returns true iff aggressive migration was triggered.
@@ -300,10 +291,10 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 	// storage servers to the new storage engine. As an example, if the DC has just 1 SS, DD will decide not to do
 	// perpetual wiggle because there's no room. In such cases, we issue aggressive migration which can do in-place
 	// migration.
-	ACTOR Future<bool> issueAggressiveMigrationIfNeeded(Database cx,
-	                                                    DatabaseConfiguration conf,
-	                                                    std::vector<StorageServerInterface> storageServers) {
-		state std::unordered_map<std::string /* dc id */, int /* number of ss in that dc id */> dcIdToSSCount;
+	Future<bool> issueAggressiveMigrationIfNeeded(Database cx,
+	                                              DatabaseConfiguration conf,
+	                                              std::vector<StorageServerInterface> storageServers) {
+		std::unordered_map<std::string /* dc id */, int /* number of ss in that dc id */> dcIdToSSCount;
 		for (const auto& ss : storageServers) {
 			if (ss.locality.dcId().present()) {
 				const auto& dcId = ss.locality.dcId().get().toString();
@@ -316,39 +307,38 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 
 		for (const auto& [_, ssCount] : dcIdToSSCount) {
 			if (ssCount <= conf.storageTeamSize) {
-				wait(success(IssueConfigurationChange(cx, "storage_migration_type=aggressive", false)));
-				return true;
+				co_await IssueConfigurationChange(cx, "storage_migration_type=aggressive", false);
+				co_return true;
 			}
 		}
 
-		return false;
+		co_return false;
 	}
 
-	ACTOR Future<bool> _check(ConfigureDatabaseWorkload* self, Database cx) {
-		wait(delay(30.0));
+	Future<bool> check(Database const& cx) override {
+		co_await delay(30.0);
 		// only storage_migration_type=gradual && perpetual_storage_wiggle=1 need this check because in QuietDatabase
 		// perpetual wiggle will be forced to close For other cases, later ConsistencyCheck will check KV store type
 		// there
-		if (self->allowTestStorageMigration || self->waitStoreTypeCheck) {
-			state bool aggressiveMigrationTriggered = false;
-			loop {
+		if (allowTestStorageMigration || waitStoreTypeCheck) {
+			bool aggressiveMigrationTriggered = false;
+			while (true) {
 				// There exists a race where the check can start before the last transaction that singleDB issued
 				// finishes, if singleDB gets actor cancelled from a timeout at the end of a test. This means the
 				// configuration needs to be re-read in case it changed since the last loop, since it could
 				// read a stale storage engine type from the configuration initially.
-				state DatabaseConfiguration conf = wait(getDatabaseConfiguration(cx));
+				DatabaseConfiguration conf = co_await getDatabaseConfiguration(cx);
 
-				state std::string wiggleLocalityKeyValue = conf.perpetualStorageWiggleLocality;
-				state std::vector<std::pair<Optional<Value>, Optional<Value>>> wiggleLocalityKeyValues =
+				std::string wiggleLocalityKeyValue = conf.perpetualStorageWiggleLocality;
+				std::vector<std::pair<Optional<Value>, Optional<Value>>> wiggleLocalityKeyValues =
 				    ParsePerpetualStorageWiggleLocality(wiggleLocalityKeyValue);
-				state int i;
+				int i{ 0 };
 
-				state bool pass = true;
-				state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+				bool pass = true;
+				std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
 
 				if (!aggressiveMigrationTriggered) {
-					wait(store(aggressiveMigrationTriggered,
-					           self->issueAggressiveMigrationIfNeeded(cx, conf, storageServers)));
+					aggressiveMigrationTriggered = co_await issueAggressiveMigrationIfNeeded(cx, conf, storageServers);
 				}
 
 				for (i = 0; i < storageServers.size(); i++) {
@@ -358,7 +348,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 					     localityMatchInList(wiggleLocalityKeyValues, storageServers[i].locality))) {
 						ReplyPromise<KeyValueStoreType> typeReply;
 						ErrorOr<KeyValueStoreType> keyValueStoreType =
-						    wait(storageServers[i].getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0));
+						    co_await storageServers[i].getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0);
 						if (keyValueStoreType.present() && keyValueStoreType.get() != conf.storageServerStoreType) {
 							TraceEvent(SevWarn, "ConfigureDatabase_WrongStoreType")
 							    .suppressFor(5.0)
@@ -374,10 +364,10 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				}
 				if (pass)
 					break;
-				wait(delay(g_network->isSimulated() ? 2.0 : 30.0));
+				co_await delay(g_network->isSimulated() ? 2.0 : 30.0);
 			}
 		}
-		return true;
+		co_return true;
 	}
 
 	static int randomRoleNumber() {
@@ -385,13 +375,13 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 		return i ? i : -1;
 	}
 
-	ACTOR Future<Void> singleDB(ConfigureDatabaseWorkload* self, Database cx) {
-		state Transaction tr;
-		loop {
+	Future<Void> singleDB(ConfigureDatabaseWorkload* self, Database cx) {
+		Transaction tr;
+		while (true) {
 			if (g_simulator->speedUpSimulation) {
-				return Void();
+				co_return;
 			}
-			state int randomChoice;
+			int randomChoice{ 0 };
 			if (self->allowTestStorageMigration) {
 				randomChoice = (deterministicRandom()->random01() < 0.375) ? deterministicRandom()->randomInt(0, 3)
 				                                                           : deterministicRandom()->randomInt(4, 9);
@@ -402,26 +392,26 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				randomChoice = deterministicRandom()->randomInt(0, 8);
 			}
 			if (randomChoice == 0) {
-				wait(success(
-				    runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
-					    return tr->get("This read is only to ensure that the database recovered"_sr);
-				    })));
-				wait(delay(20 + 10 * deterministicRandom()->random01()));
+				co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Optional<Value>> {
+					return tr->get("This read is only to ensure that the database recovered"_sr);
+				});
+				co_await delay(20 + 10 * deterministicRandom()->random01());
 			} else if (randomChoice < 3) {
 				double waitDuration = 3.0 * deterministicRandom()->random01();
 				//TraceEvent("ConfigureTestWaitAfter").detail("WaitDuration",waitDuration);
-				wait(delay(waitDuration));
+				co_await delay(waitDuration);
 			} else if (randomChoice == 3) {
 				//TraceEvent("ConfigureTestConfigureBegin").detail("NewConfig", newConfig);
 				int maxRedundancies = sizeof(redundancies) / sizeof(redundancies[0]);
-				if (g_simulator->physicalDatacenters == 2 || g_simulator->physicalDatacenters > 3) {
+				if (fdbSimulationPolicyState().physicalDatacenters == 2 ||
+				    fdbSimulationPolicyState().physicalDatacenters > 3) {
 					maxRedundancies--; // There are not enough machines for triple replication in fearless
 					                   // configurations
 				}
 				int redundancy = deterministicRandom()->randomInt(0, maxRedundancies);
 				std::string config = redundancies[redundancy];
 
-				if (config == "triple" && g_simulator->physicalDatacenters == 3) {
+				if (config == "triple" && fdbSimulationPolicyState().physicalDatacenters == 3) {
 					config = "three_data_hall ";
 				}
 
@@ -441,7 +431,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				if (deterministicRandom()->random01() < 0.5)
 					config += " resolvers=" + format("%d", randomRoleNumber());
 
-				wait(success(IssueConfigurationChange(cx, config, false)));
+				co_await IssueConfigurationChange(cx, config, false);
 
 				//TraceEvent("ConfigureTestConfigureEnd").detail("NewConfig", newConfig);
 			} else if (randomChoice == 4) {
@@ -455,7 +445,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				if (deterministicRandom()->randomInt(0, 2))
 					ch = nameQuorumChange(format(desiredClusterName.c_str(), deterministicRandom()->randomInt(0, 100)),
 					                      ch);
-				wait(success(changeQuorum(cx, ch)));
+				co_await changeQuorum(cx, ch);
 				//TraceEvent("ConfigureTestConfigureEnd").detail("NewQuorum", s);
 			} else if (randomChoice == 5) {
 				int storeType = 0;
@@ -492,7 +482,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 				default:
 					ASSERT(false);
 				}
-				wait(success(IssueConfigurationChange(cx, storeTypeStr, true)));
+				co_await IssueConfigurationChange(cx, storeTypeStr, true);
 			} else if (randomChoice == 6) {
 				// Some configurations will be invalid, and that's fine.
 				int length = sizeof(logTypes) / sizeof(logTypes[0]);
@@ -501,21 +491,20 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 					length -= 1;
 				}
 
-				wait(success(
-				    IssueConfigurationChange(cx, logTypes[deterministicRandom()->randomInt(0, length)], false)));
+				co_await IssueConfigurationChange(cx, logTypes[deterministicRandom()->randomInt(0, length)], false);
 			} else if (randomChoice == 7) {
-				wait(success(IssueConfigurationChange(
+				co_await IssueConfigurationChange(
 				    cx,
 				    backupTypes[deterministicRandom()->randomInt(0, sizeof(backupTypes) / sizeof(backupTypes[0]))],
-				    false)));
+				    false);
 			} else if (randomChoice == 8) {
 				if (self->allowTestStorageMigration) {
 					CODE_PROBE(true, "storage migration type change");
 
 					// randomly configuring perpetual_storage_wiggle_locality
-					state std::string randomPerpetualWiggleLocality;
+					std::string randomPerpetualWiggleLocality;
 					if (deterministicRandom()->random01() < 0.25) {
-						state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+						std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
 						std::string localityFilter;
 						int selectSSCount =
 						    deterministicRandom()->randomInt(1, std::min(4, (int)(storageServers.size() + 1)));
@@ -530,7 +519,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 							StringRef randomLocalityKey =
 							    localityKeys[deterministicRandom()->randomInt(0, localityKeys.size())];
 							if (randomSS.locality.isPresent(randomLocalityKey)) {
-								if (localityFilter.size() > 0) {
+								if (!localityFilter.empty()) {
 									localityFilter += ";";
 								}
 								localityFilter += randomLocalityKey.toString() + ":" +
@@ -538,18 +527,18 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 							}
 						}
 
-						if (localityFilter.size() > 0) {
+						if (!localityFilter.empty()) {
 							TraceEvent("ConfigureTestSettingWiggleLocality").detail("LocalityFilter", localityFilter);
 							randomPerpetualWiggleLocality = " perpetual_storage_wiggle_locality=" + localityFilter;
 						}
 					}
 
-					wait(success(IssueConfigurationChange(
+					co_await IssueConfigurationChange(
 					    cx,
 					    storageMigrationTypes[deterministicRandom()->randomInt(
 					        0, sizeof(storageMigrationTypes) / sizeof(storageMigrationTypes[0]))] +
 					        randomPerpetualWiggleLocality,
-					    false)));
+					    false);
 				}
 			} else {
 				ASSERT(false);
diff --git a/fdbserver/workloads/ConflictRange.actor.cpp b/fdbserver/workloads/ConflictRange.cpp
similarity index 74%
rename from fdbserver/workloads/ConflictRange.actor.cpp
rename to fdbserver/workloads/ConflictRange.cpp
index 3222d3e873b..4cc9ac488b2 100644
--- a/fdbserver/workloads/ConflictRange.actor.cpp
+++ b/fdbserver/workloads/ConflictRange.cpp
@@ -1,5 +1,5 @@
 /*
- * ConflictRange.actor.cpp
+ * ConflictRange.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,14 +19,13 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
+#include "fdbclient/ManagementAPI.h"
 
 // For this test to report properly buggify must be disabled (flow.h) , and failConnection must be disabled in
-// (sim2.actor.cpp)
+// (sim2.cpp)
 
 struct ConflictRangeWorkload : TestWorkload {
 	static constexpr auto NAME = "ConflictRange";
@@ -43,7 +42,7 @@ struct ConflictRangeWorkload : TestWorkload {
 		out.insert({ "RandomRangeLock" });
 	}
 
-	ConflictRangeWorkload(WorkloadContext const& wcx)
+	explicit ConflictRangeWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), withConflicts("WithConflicts"), withoutConflicts("withoutConflicts"), retries("Retries") {
 		minOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 2);
 		maxOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 4);
@@ -55,8 +54,6 @@ struct ConflictRangeWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	Future<bool> check(Database const& cx) override {
 		clients.clear();
 		return true;
@@ -68,46 +65,46 @@ struct ConflictRangeWorkload : TestWorkload {
 		m.push_back(retries.getMetric());
 	}
 
-	ACTOR Future<Void> _start(Database cx, ConflictRangeWorkload* self) {
-		if (self->clientId == 0)
-			wait(timeout(self->conflictRangeClient(cx, self), self->testDuration, Void()));
-		return Void();
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0)
+			co_await timeout(conflictRangeClient(cx, this), testDuration, Void());
 	}
 
-	ACTOR Future<Void> conflictRangeClient(Database cx, ConflictRangeWorkload* self) {
-		state std::string clientID;
-		state std::string myKeyA;
-		state std::string myKeyB;
-		state std::string myValue;
-		state bool onEqualA;
-		state bool onEqualB;
-		state int offsetA;
-		state int offsetB;
-		state int randomLimit;
-		state Reverse reverse = Reverse::False;
-		state bool randomSets = false;
-		state std::set<int> insertedSet;
-		state RangeResult originalResults;
-		state Standalone<StringRef> firstElement;
-
-		state std::set<int> clearedSet;
-		state int clearedBegin;
-		state int clearedEnd;
+	Future<Void> conflictRangeClient(Database cx, ConflictRangeWorkload* self) {
+		std::string clientID;
+		std::string myKeyA;
+		std::string myKeyB;
+		std::string myValue;
+		bool onEqualA{ false };
+		bool onEqualB{ false };
+		int offsetA{ 0 };
+		int offsetB{ 0 };
+		int randomLimit{ 0 };
+		Reverse reverse = Reverse::False;
+		bool randomSets = false;
+		std::set<int> insertedSet;
+		RangeResult originalResults;
+		Standalone<StringRef> firstElement;
+
+		std::set<int> clearedSet;
+		int clearedBegin{ 0 };
+		int clearedEnd{ 0 };
 
 		if (g_network->isSimulated()) {
-			wait(timeKeeperSetDisable(cx));
+			co_await timeKeeperSetDisable(cx);
 		}
 
 		// Set one key after the end of the tested range. If this key is included in the result, then
 		// we may have drifted into the system key-space and cannot evaluate the result.
-		state Key sentinelKey = StringRef(format("%010d", self->maxKeySpace));
+		Key sentinelKey = StringRef(format("%010d", self->maxKeySpace));
 
-		loop {
+		while (true) {
 			randomSets = !randomSets;
 
 			// Initialize the database with random values.
-			loop {
-				state Transaction tr0(cx);
+			while (true) {
+				Transaction tr0(cx);
+				Error err;
 				try {
 					TraceEvent("ConflictRangeReset").log();
 					insertedSet.clear();
@@ -138,25 +135,27 @@ struct ConflictRangeWorkload : TestWorkload {
 
 					tr0.set(sentinelKey, deterministicRandom()->randomUniqueID().toString());
 
-					wait(tr0.commit());
+					co_await tr0.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr0.onError(e));
+					err = e;
 				}
+				co_await tr0.onError(err);
 			}
 
 			firstElement = Key(StringRef(format("%010d", *(insertedSet.begin()))));
 
-			state Transaction tr1(cx);
-			state Transaction tr2(cx);
-			state Transaction tr3(cx);
-			state Transaction tr4(cx);
-			state ReadYourWritesTransaction trRYOW(cx);
+			Transaction tr1(cx);
+			Transaction tr2(cx);
+			Transaction tr3(cx);
+			Transaction tr4(cx);
+			ReadYourWritesTransaction trRYOW(cx);
 
+			Error err;
 			try {
 				// Generate a random getRange operation and execute it, if it produces results, save them, otherwise
 				// retry.
-				loop {
+				while (true) {
 					myKeyA = format("%010d", deterministicRandom()->randomInt(0, self->maxKeySpace));
 					myKeyB = format("%010d", deterministicRandom()->randomInt(0, self->maxKeySpace));
 					onEqualA = deterministicRandom()->randomInt(0, 2) != 0;
@@ -166,12 +165,12 @@ struct ConflictRangeWorkload : TestWorkload {
 					randomLimit = deterministicRandom()->randomInt(1, self->maxKeySpace);
 					reverse.set(deterministicRandom()->coinflip());
 
-					RangeResult res = wait(tr1.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-					                                    KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-					                                    randomLimit,
-					                                    Snapshot::False,
-					                                    reverse));
-					if (res.size()) {
+					RangeResult res = co_await tr1.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+					                                        KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+					                                        randomLimit,
+					                                        Snapshot::False,
+					                                        reverse);
+					if (!res.empty()) {
 						originalResults = res;
 						break;
 					}
@@ -182,12 +181,12 @@ struct ConflictRangeWorkload : TestWorkload {
 					for (auto iter = clearedSet.begin(); iter != clearedSet.end(); ++iter)
 						tr1.set(StringRef(format("%010d", (*iter))),
 						        deterministicRandom()->randomUniqueID().toString());
-					wait(tr1.commit());
+					co_await tr1.commit();
 					tr1 = Transaction(cx);
 				}
 
 				// Create two transactions with the same read version
-				Version readVersion = wait(tr2.getReadVersion());
+				Version readVersion = co_await tr2.getReadVersion();
 
 				if (self->testReadYourWrites) {
 					trRYOW.setVersion(readVersion);
@@ -223,28 +222,28 @@ struct ConflictRangeWorkload : TestWorkload {
 					}
 				}
 
-				wait(tr2.commit());
+				co_await tr2.commit();
 
-				state bool foundConflict = false;
+				bool foundConflict = false;
 				try {
 					// Do the generated getRange in the other transaction and commit.
 					if (self->testReadYourWrites) {
 						trRYOW.clear(KeyRangeRef(StringRef(format("%010d", clearedBegin)),
 						                         StringRef(format("%010d", clearedEnd))));
-						RangeResult res = wait(trRYOW.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-						                                       KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-						                                       randomLimit,
-						                                       Snapshot::False,
-						                                       reverse));
-						wait(trRYOW.commit());
+						RangeResult res = co_await trRYOW.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+						                                           KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+						                                           randomLimit,
+						                                           Snapshot::False,
+						                                           reverse);
+						co_await trRYOW.commit();
 					} else {
 						tr3.clear(StringRef(format("%010d", self->maxKeySpace + 1)));
-						RangeResult res = wait(tr3.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-						                                    KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-						                                    randomLimit,
-						                                    Snapshot::False,
-						                                    reverse));
-						wait(tr3.commit());
+						RangeResult res = co_await tr3.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+						                                        KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+						                                        randomLimit,
+						                                        Snapshot::False,
+						                                        reverse);
+						co_await tr3.commit();
 					}
 				} catch (Error& e) {
 					if (e.code() != error_code_not_committed)
@@ -258,15 +257,15 @@ struct ConflictRangeWorkload : TestWorkload {
 					if (self->testReadYourWrites) {
 						tr1.clear(KeyRangeRef(StringRef(format("%010d", clearedBegin)),
 						                      StringRef(format("%010d", clearedEnd))));
-						wait(tr1.commit());
+						co_await tr1.commit();
 						tr1 = Transaction(cx);
 					}
 
-					RangeResult res = wait(tr4.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-					                                    KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-					                                    randomLimit,
-					                                    Snapshot::False,
-					                                    reverse));
+					RangeResult res = co_await tr4.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+					                                        KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+					                                        randomLimit,
+					                                        Snapshot::False,
+					                                        reverse);
 					++self->withConflicts;
 
 					if (res.size() == originalResults.size()) {
@@ -277,8 +276,8 @@ struct ConflictRangeWorkload : TestWorkload {
 						// Discard known cases where conflicts do not change the results
 						if (originalResults.size() == randomLimit &&
 						    ((offsetB <= 0 && !reverse) || (offsetA > 1 && reverse))) {
-							// Hit limit but end offset goes into the range, so changes could effect results even though
-							// in this instance they did not
+							// Hit limit but end offset goes into the range, so changes could effect results even
+							// though in this instance they did not
 							throw not_committed();
 						}
 
@@ -289,23 +288,23 @@ struct ConflictRangeWorkload : TestWorkload {
 						}
 
 						if (largestResult >= sentinelKey) {
-							// Results go into server keyspace, so if a key selector does not fully resolve offset, a
-							// change won't effect results
+							// Results go into server keyspace, so if a key selector does not fully resolve offset,
+							// a change won't effect results
 							throw not_committed();
 						}
 
 						if ((smallestResult == firstElement ||
 						     smallestResult == StringRef(format("%010d", *(insertedSet.begin())))) &&
 						    offsetA < 0) {
-							// Results return the first element, and the begin offset is negative, so if a key selector
-							// does not fully resolve the offset, a change won't effect results
+							// Results return the first element, and the begin offset is negative, so if a key
+							// selector does not fully resolve the offset, a change won't effect results
 							throw not_committed();
 						}
 
 						if ((myKeyA > myKeyB || (myKeyA == myKeyB && onEqualA && !onEqualB)) &&
 						    originalResults.size() == randomLimit) {
-							// The begin key is less than the end key, so changes in this range only effect the end key
-							// selector, but because we hit the limit this does not change the results
+							// The begin key is less than the end key, so changes in this range only effect the end
+							// key selector, but because we hit the limit this does not change the results
 							throw not_committed();
 						}
 
@@ -335,9 +334,9 @@ struct ConflictRangeWorkload : TestWorkload {
 						    .detail("Original", keyStr2);
 
 						tr4 = Transaction(cx);
-						RangeResult res = wait(tr4.getRange(
+						RangeResult res = co_await tr4.getRange(
 						    KeyRangeRef(StringRef(format("%010d", 0)), StringRef(format("%010d", self->maxKeySpace))),
-						    200));
+						    200);
 						std::string allKeyEntries = "";
 						for (int i = 0; i < res.size(); i++) {
 							allKeyEntries += printable(res[i].key) + " ";
@@ -348,11 +347,11 @@ struct ConflictRangeWorkload : TestWorkload {
 					throw not_committed();
 				} else {
 					// If the commit is successful, check that the result matches the first execution.
-					RangeResult res = wait(tr4.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-					                                    KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-					                                    randomLimit,
-					                                    Snapshot::False,
-					                                    reverse));
+					RangeResult res = co_await tr4.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+					                                        KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+					                                        randomLimit,
+					                                        Snapshot::False,
+					                                        reverse);
 					++self->withoutConflicts;
 
 					if (res.size() == originalResults.size()) {
@@ -397,16 +396,19 @@ struct ConflictRangeWorkload : TestWorkload {
 					}
 				}
 			} catch (Error& e) {
-				state Error e2 = e;
-				if (e2.code() != error_code_not_committed)
-					++self->retries;
-
-				wait(tr1.onError(e2));
-				wait(tr2.onError(e2));
-				wait(tr3.onError(e2));
-				wait(tr4.onError(e2));
-				wait(trRYOW.onError(e2));
+				err = e;
+			}
+			if (!err.isValid()) {
+				continue;
 			}
+			if (err.code() != error_code_not_committed)
+				++self->retries;
+
+			co_await tr1.onError(err);
+			co_await tr2.onError(err);
+			co_await tr3.onError(err);
+			co_await tr4.onError(err);
+			co_await trRYOW.onError(err);
 		}
 	}
 };
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.cpp
similarity index 76%
rename from fdbserver/workloads/ConsistencyCheck.actor.cpp
rename to fdbserver/workloads/ConsistencyCheck.cpp
index 354da67a25f..aa9748cde50 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.cpp
@@ -1,5 +1,5 @@
 /*
- * ConsistencyCheck.actor.cpp
+ * ConsistencyCheck.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,22 +25,24 @@
 #include "flow/ProcessEvents.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/IRateControl.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/StorageMetrics.actor.h"
-#include "fdbserver/DataDistribution.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/TSSMappingUtil.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/consistencyscan/ConsistencyScan.h"
+#include "fdbserver/core/StorageMetrics.h"
+#include "fdbserver/datadistributor/DataDistribution.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/TSSMappingUtil.h"
 #include "flow/DeterministicRandom.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/network.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 // #define SevCCheckInfo SevVerbose
 #define SevCCheckInfo SevInfo
@@ -111,7 +113,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	OnTimeout onTimeout;
 	ProcessEvents::Event onTimeoutEvent;
 
-	ConsistencyCheckWorkload(WorkloadContext const& wcx)
+	explicit ConsistencyCheckWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), onTimeout(*this), onTimeoutEvent({ "Timeout"_sr, "TracedTooManyLines"_sr }, onTimeout) {
 		performQuiescentChecks = getOption(options, "performQuiescentChecks"_sr, false);
 		performTSSCheck = getOption(options, "performTSSCheck"_sr, true);
@@ -134,21 +136,21 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
 
-	ACTOR Future<Void> _setup(Database cx, ConsistencyCheckWorkload* self) {
+	Future<Void> _setup(Database cx, ConsistencyCheckWorkload* self) {
 		// If performing quiescent checks, wait for the database to go quiet
 		if (self->firstClient && self->performQuiescentChecks) {
 			if (g_network->isSimulated()) {
-				wait(timeKeeperSetDisable(cx));
+				co_await timeKeeperSetDisable(cx);
 			}
 
 			try {
-				wait(timeoutError(
+				co_await timeoutError(
 				    quietDatabase(
 				        cx, self->dbInfo, "ConsistencyCheckStart", 0, 1e5, 0, 0, 30e6, 1e6, self->maxDDRunTime),
-				    self->maxDDRunTime)); // FIXME: should be zero?
+				    self->maxDDRunTime); // FIXME: should be zero?
 				if (g_network->isSimulated()) {
-					g_simulator->quiesced = true;
-					TraceEvent("ConsistencyCheckQuiesced").detail("Quiesced", g_simulator->quiesced);
+					fdbSimulationPolicyState().quiesced = true;
+					TraceEvent("ConsistencyCheckQuiesced").detail("Quiesced", fdbSimulationPolicyState().quiesced);
 				}
 			} catch (Error& e) {
 				TraceEvent("ConsistencyCheck_QuietDatabaseError").error(e);
@@ -158,7 +160,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		self->monitorConsistencyCheckSettingsActor = self->monitorConsistencyCheckSettings(cx, self);
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -182,70 +183,75 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		failEvent.detail("Reason", "Consistency check: " + message);
 	}
 
-	ACTOR Future<Void> monitorConsistencyCheckSettings(Database cx, ConsistencyCheckWorkload* self) {
-		loop {
-			state ReadYourWritesTransaction tr(cx);
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				state Optional<Value> ccSuspendVal = wait(tr.get(fdbShouldConsistencyCheckBeSuspended));
-				bool ccSuspend = ccSuspendVal.present()
-				                     ? BinaryReader::fromStringRef<bool>(ccSuspendVal.get(), Unversioned())
-				                     : false;
-				self->suspendConsistencyCheck.set(ccSuspend);
-				state Future<Void> watchCCSuspendFuture = tr.watch(fdbShouldConsistencyCheckBeSuspended);
-				wait(tr.commit());
-				wait(watchCCSuspendFuture);
-			} catch (Error& e) {
-				wait(tr.onError(e));
+	Future<Void> monitorConsistencyCheckSettings(Database cx, ConsistencyCheckWorkload* self) {
+		while (true) {
+			ReadYourWritesTransaction tr(cx);
+			{
+				Error err;
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+					Optional<Value> ccSuspendVal = co_await tr.get(fdbShouldConsistencyCheckBeSuspended);
+					bool ccSuspend = ccSuspendVal.present()
+					                     ? BinaryReader::fromStringRef<bool>(ccSuspendVal.get(), Unversioned())
+					                     : false;
+					self->suspendConsistencyCheck.set(ccSuspend);
+					Future<Void> watchCCSuspendFuture = tr.watch(fdbShouldConsistencyCheckBeSuspended);
+					co_await tr.commit();
+					co_await watchCCSuspendFuture;
+				} catch (Error& e) {
+					err = e;
+				}
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR Future<Void> _start(Database cx, ConsistencyCheckWorkload* self) {
-		loop {
+	Future<Void> _start(Database cx, ConsistencyCheckWorkload* self) {
+		while (true) {
 			while (self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Suspended").log();
-				wait(self->suspendConsistencyCheck.onChange());
+				co_await self->suspendConsistencyCheck.onChange();
 			}
 			TraceEvent("ConsistencyCheck_StartingOrResuming").log();
-			choose {
-				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite)
-						break;
-					self->repetitions++;
-					wait(delay(5.0));
-				}
-				when(wait(self->suspendConsistencyCheck.onChange())) {}
+			auto choice = co_await race(self->runCheck(cx, self), self->suspendConsistencyCheck.onChange());
+			if (choice.index() == 0) {
+				if (!self->indefinite)
+					break;
+				self->repetitions++;
+				co_await delay(5.0);
+			} else if (choice.index() == 1) {
+			} else {
+				UNREACHABLE();
 			}
 		}
 		if (self->firstClient && g_network->isSimulated() && self->performQuiescentChecks) {
-			g_simulator->quiesced = false;
-			TraceEvent("ConsistencyCheckQuiescedEnd").detail("Quiesced", g_simulator->quiesced);
+			fdbSimulationPolicyState().quiesced = false;
+			TraceEvent("ConsistencyCheckQuiescedEnd").detail("Quiesced", fdbSimulationPolicyState().quiesced);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> runCheck(Database cx, ConsistencyCheckWorkload* self) {
+	Future<Void> runCheck(Database cx, ConsistencyCheckWorkload* self) {
 		CODE_PROBE(self->performQuiescentChecks, "Quiescent consistency check");
 		CODE_PROBE(!self->performQuiescentChecks, "Non-quiescent consistency check");
-		state double consistenyCheckerBeginTime = now();
+		double consistenyCheckerBeginTime = now();
 
 		if (self->firstClient || self->distributed) {
 			try {
-				state DatabaseConfiguration configuration;
-				state std::map<UID, StorageServerInterface> tssMapping;
+				DatabaseConfiguration configuration;
+				std::map<UID, StorageServerInterface> tssMapping;
 
-				state Transaction tr(cx);
+				Transaction tr(cx);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				loop {
+				while (true) {
+					Error err;
 					try {
 						if (self->performTSSCheck) {
 							tssMapping.clear();
-							wait(readTSSMapping(&tr, &tssMapping));
+							co_await readTSSMapping(&tr, &tssMapping);
 						}
-						RangeResult res = wait(tr.getRange(configKeys, 1000));
+						RangeResult res = co_await tr.getRange(configKeys, 1000);
 						if (res.size() == 1000) {
 							TraceEvent("ConsistencyCheck_TooManyConfigOptions").log();
 							self->testFailure("Read too many configuration options");
@@ -254,18 +260,19 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							configuration.set(res[i].key, res[i].value);
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 				}
 
 				// Perform quiescence-only checks
 				if (self->firstClient && self->performQuiescentChecks) {
 					// Check for undesirable servers (storage servers with exact same network address or using the wrong
 					// key value store type)
-					state bool hasUndesirableServers = wait(self->checkForUndesirableServers(cx, configuration, self));
+					bool hasUndesirableServers = co_await self->checkForUndesirableServers(cx, configuration, self);
 
 					// Check that nothing is in-flight or in queue in data distribution
-					int64_t inDataDistributionQueue = wait(getDataDistributionQueueSize(cx, self->dbInfo, true));
+					int64_t inDataDistributionQueue = co_await getDataDistributionQueueSize(cx, self->dbInfo, true);
 					if (inDataDistributionQueue > 0) {
 						TraceEvent("ConsistencyCheck_NonZeroDataDistributionQueue")
 						    .detail("QueueSize", inDataDistributionQueue);
@@ -274,7 +281,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 					// Check that the number of process (and machine) teams is no larger than
 					// the allowed maximum number of teams
-					bool teamCollectionValid = wait(getTeamCollectionValid(cx, self->dbInfo));
+					bool teamCollectionValid = co_await getTeamCollectionValid(cx, self->dbInfo);
 					if (!teamCollectionValid) {
 						TraceEvent(SevError, "ConsistencyCheck_TooManyTeams").log();
 						self->testFailure("The number of process or machine teams is larger than the allowed maximum "
@@ -282,7 +289,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					}
 
 					// Check that nothing is in the TLog queues
-					std::pair<int64_t, int64_t> maxTLogQueueInfo = wait(getTLogQueueInfo(cx, self->dbInfo));
+					std::pair<int64_t, int64_t> maxTLogQueueInfo = co_await getTLogQueueInfo(cx, self->dbInfo);
 					if (maxTLogQueueInfo.first > 1e5) // FIXME: Should be zero?
 					{
 						TraceEvent("ConsistencyCheck_NonZeroTLogQueue").detail("MaxQueueSize", maxTLogQueueInfo.first);
@@ -298,7 +305,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					// Check that nothing is in the storage server queues
 					try {
 						int64_t maxStorageServerQueueSize =
-						    wait(getMaxStorageServerQueueSize(cx, self->dbInfo, invalidVersion));
+						    co_await getMaxStorageServerQueueSize(cx, self->dbInfo, invalidVersion);
 						if (maxStorageServerQueueSize > 0) {
 							TraceEvent("ConsistencyCheck_ExceedStorageServerQueueLimit")
 							    .detail("MaxQueueSize", maxStorageServerQueueSize);
@@ -318,24 +325,24 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							throw;
 					}
 
-					wait(::success(self->checkForStorage(cx, configuration, tssMapping, self)));
-					wait(::success(self->checkForExtraDataStores(cx, self)));
-					wait(::success(self->checkStorageMetadata(cx, self)));
+					co_await self->checkForStorage(cx, configuration, tssMapping, self);
+					co_await self->checkForExtraDataStores(cx, self);
+					co_await self->checkStorageMetadata(cx, self);
 
 					// Check that each machine is operating as its desired class
-					bool usingDesiredClasses = wait(self->checkUsingDesiredClasses(cx, self));
+					bool usingDesiredClasses = co_await self->checkUsingDesiredClasses(cx, self);
 					if (!usingDesiredClasses)
 						self->testFailure("Cluster has machine(s) not using requested classes");
 
-					bool workerListCorrect = wait(self->checkWorkerList(cx, self));
+					bool workerListCorrect = co_await self->checkWorkerList(cx, self);
 					if (!workerListCorrect)
 						self->testFailure("Worker list incorrect");
 
-					bool coordinatorsCorrect = wait(self->checkCoordinators(cx));
+					bool coordinatorsCorrect = co_await self->checkCoordinators(cx);
 					if (!coordinatorsCorrect)
 						self->testFailure("Coordinators incorrect");
 
-					bool consistencyScanStopped = wait(self->checkConsistencyScan(cx));
+					bool consistencyScanStopped = co_await self->checkConsistencyScan(cx);
 					if (!consistencyScanStopped)
 						self->testFailure("Consistency scan active");
 
@@ -346,45 +353,45 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 
 				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
-				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-				bool keyServerResult = wait(getKeyServers(cx,
-				                                          keyServerPromise,
-				                                          keyServersKeys,
-				                                          self->performQuiescentChecks,
-				                                          self->failureIsError,
-				                                          &self->success));
+				Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
+				bool keyServerResult = co_await getKeyServers(cx,
+				                                              keyServerPromise,
+				                                              keyServersKeys,
+				                                              self->performQuiescentChecks,
+				                                              self->failureIsError,
+				                                              &self->success);
 				if (keyServerResult) {
-					state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+					std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 					    keyServerPromise.getFuture().get();
 
 					// Get the locations of all the shards in the database
-					state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
-					bool keyLocationResult = wait(getKeyLocations(
-					    cx, keyServers, keyLocationPromise, self->performQuiescentChecks, &self->success));
+					Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
+					bool keyLocationResult = co_await getKeyLocations(
+					    cx, keyServers, keyLocationPromise, self->performQuiescentChecks, &self->success);
 					if (keyLocationResult) {
-						state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
+						Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
 
 						// Check that each shard has the same data on all storage servers that it resides on
-						wait(checkDataConsistency(cx,
-						                          keyLocations,
-						                          configuration,
-						                          tssMapping,
-						                          self->performQuiescentChecks,
-						                          self->performTSSCheck,
-						                          self->firstClient,
-						                          self->failureIsError,
-						                          self->clientId,
-						                          self->clientCount,
-						                          self->distributed,
-						                          self->shuffleShards,
-						                          self->shardSampleFactor,
-						                          self->sharedRandomNumber,
-						                          self->repetitions,
-						                          &(self->bytesReadInPreviousRound),
-						                          true,
-						                          self->rateLimitMax,
-						                          CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,
-						                          &self->success));
+						co_await checkDataConsistency(cx,
+						                              keyLocations,
+						                              configuration,
+						                              tssMapping,
+						                              self->performQuiescentChecks,
+						                              self->performTSSCheck,
+						                              self->firstClient,
+						                              self->failureIsError,
+						                              self->clientId,
+						                              self->clientCount,
+						                              self->distributed,
+						                              self->shuffleShards,
+						                              self->shardSampleFactor,
+						                              self->sharedRandomNumber,
+						                              self->repetitions,
+						                              &(self->bytesReadInPreviousRound),
+						                              true,
+						                              self->rateLimitMax,
+						                              CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,
+						                              &self->success);
 					}
 				}
 			} catch (Error& e) {
@@ -402,8 +409,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		TraceEvent("ConsistencyCheck_FinishedCheck")
 		    .detail("Repetitions", self->repetitions)
 		    .detail("TimeSpan", now() - consistenyCheckerBeginTime);
-
-		return Void();
 	}
 
 	// Comparison function used to compare map elements by value
@@ -414,14 +419,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	// Returns true if any storage servers have the exact same network address or are not using the correct key value
 	// store type
-	ACTOR Future<bool> checkForUndesirableServers(Database cx,
-	                                              DatabaseConfiguration configuration,
-	                                              ConsistencyCheckWorkload* self) {
-		state int i;
-		state int j;
-		state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
-		state std::string wiggleLocalityKeyValue = configuration.perpetualStorageWiggleLocality;
-		state std::vector<std::pair<Optional<Value>, Optional<Value>>> wiggleLocalityKeyValues =
+	Future<bool> checkForUndesirableServers(Database cx,
+	                                        DatabaseConfiguration configuration,
+	                                        ConsistencyCheckWorkload* self) {
+		int i{ 0 };
+		int j{ 0 };
+		std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
+		std::string wiggleLocalityKeyValue = configuration.perpetualStorageWiggleLocality;
+		std::vector<std::pair<Optional<Value>, Optional<Value>>> wiggleLocalityKeyValues =
 		    ParsePerpetualStorageWiggleLocality(configuration.perpetualStorageWiggleLocality);
 
 		// Check each pair of storage servers for an address match
@@ -429,7 +434,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			// Check that each storage server has the correct key value store type
 			ReplyPromise<KeyValueStoreType> typeReply;
 			ErrorOr<KeyValueStoreType> keyValueStoreType =
-			    wait(storageServers[i].getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0));
+			    co_await storageServers[i].getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0);
 
 			if (!keyValueStoreType.present()) {
 				TraceEvent("ConsistencyCheck_ServerUnavailable").detail("ServerID", storageServers[i].id());
@@ -446,7 +451,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						    .detail("DesiredType", configuration.perpetualStoreType.toString())
 						    .detail("IsPerpetualStoreType", true);
 						self->testFailure("Storage server has wrong key-value store type");
-						return true;
+						co_return true;
 					}
 				} else if ((!storageServers[i].isTss() &&
 				            keyValueStoreType.get() != configuration.storageServerStoreType) ||
@@ -458,7 +463,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("DesiredType", configuration.perpetualStoreType.toString())
 					    .detail("IsPerpetualStoreType", false);
 					self->testFailure("Storage server has wrong key-value store type");
-					return true;
+					co_return true;
 				}
 			} else if (((!storageServers[i].isTss() &&
 			             keyValueStoreType.get() != configuration.storageServerStoreType) ||
@@ -472,7 +477,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("DesiredType", configuration.storageServerStoreType.toString())
 				    .detail("IsPerpetualStoreType", false);
 				self->testFailure("Storage server has wrong key-value store type");
-				return true;
+				co_return true;
 			}
 
 			// Check each pair of storage servers for an address match
@@ -483,57 +488,62 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("StorageServer2", storageServers[j].id())
 					    .detail("Address", storageServers[i].address());
 					self->testFailure("Multiple storage servers have the same address");
-					return true;
+					co_return true;
 				}
 			}
 		}
 
-		return false;
+		co_return false;
 	}
 
 	// Every storage server should have it metadata populated and no metadata leak when the database reach the quiescent
 	// state
-	ACTOR Future<bool> checkStorageMetadata(Database cx, ConsistencyCheckWorkload* self) {
-		state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(
-		    serverMetadataKeys.begin, IncludeVersion());
-		state std::vector<StorageServerInterface> servers;
-		state std::unordered_map<UID, StorageMetadataType> id_ssi;
-		state Transaction tr(cx);
-		loop {
+	Future<bool> checkStorageMetadata(Database cx, ConsistencyCheckWorkload* self) {
+		KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
+		                                                                                     IncludeVersion());
+		std::vector<StorageServerInterface> servers;
+		std::unordered_map<UID, StorageMetadataType> id_ssi;
+		Transaction tr(cx);
+		while (true) {
 			servers.clear();
 			id_ssi.clear();
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			try {
-				state KeyBackedRangeResult<std::pair<UID, StorageMetadataType>> metadata =
-				    wait(metadataMap.getRange(&tr, {}, {}, CLIENT_KNOBS->TOO_MANY));
-				ASSERT(!metadata.more && metadata.results.size() < CLIENT_KNOBS->TOO_MANY);
-				RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
-				ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
-				ASSERT_EQ(metadata.results.size(), serverList.size());
-				id_ssi = std::unordered_map<UID, StorageMetadataType>(metadata.results.begin(), metadata.results.end());
-				servers.reserve(serverList.size());
-				for (int i = 0; i < serverList.size(); i++)
-					servers.push_back(decodeServerListValue(serverList[i].value));
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
+			{
+				Error err;
+				try {
+					KeyBackedRangeResult<std::pair<UID, StorageMetadataType>> metadata =
+					    co_await metadataMap.getRange(&tr, {}, {}, CLIENT_KNOBS->TOO_MANY);
+					ASSERT(!metadata.more && metadata.results.size() < CLIENT_KNOBS->TOO_MANY);
+					RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+					ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
+					ASSERT_EQ(metadata.results.size(), serverList.size());
+					id_ssi =
+					    std::unordered_map<UID, StorageMetadataType>(metadata.results.begin(), metadata.results.end());
+					servers.reserve(serverList.size());
+					for (int i = 0; i < serverList.size(); i++)
+						servers.push_back(decodeServerListValue(serverList[i].value));
+					break;
+				} catch (Error& e) {
+					err = e;
+				}
+				co_await tr.onError(err);
 			}
 		}
 
 		for (auto& ssi : servers) {
 			ASSERT(id_ssi.contains(ssi.id()));
 		}
-		return true;
+		co_return true;
 	}
 
 	// Returns false if any worker that should have a storage server does not have one
-	ACTOR Future<bool> checkForStorage(Database cx,
-	                                   DatabaseConfiguration configuration,
-	                                   std::map<UID, StorageServerInterface> tssMapping,
-	                                   ConsistencyCheckWorkload* self) {
-		state std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
-		state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+	Future<bool> checkForStorage(Database cx,
+	                             DatabaseConfiguration configuration,
+	                             std::map<UID, StorageServerInterface> tssMapping,
+	                             ConsistencyCheckWorkload* self) {
+		std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
+		std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
 		std::vector<Optional<Key>> missingStorage; // vector instead of a set to get the count
 
 		for (int i = 0; i < workers.size(); i++) {
@@ -559,14 +569,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			}
 		}
 
-		int missingDc0 = configuration.regions.size() == 0
+		int missingDc0 = configuration.regions.empty()
 		                     ? 0
 		                     : std::count(missingStorage.begin(), missingStorage.end(), configuration.regions[0].dcId);
 		int missingDc1 = configuration.regions.size() < 2
 		                     ? 0
 		                     : std::count(missingStorage.begin(), missingStorage.end(), configuration.regions[1].dcId);
 
-		if ((configuration.regions.size() == 0 && missingStorage.size()) ||
+		if ((configuration.regions.empty() && !missingStorage.empty()) ||
 		    (configuration.regions.size() == 1 && missingDc0) ||
 		    (configuration.regions.size() == 2 && configuration.usableRegions == 1 && missingDc0 && missingDc1) ||
 		    (configuration.regions.size() == 2 && configuration.usableRegions > 1 && (missingDc0 || missingDc1))) {
@@ -592,27 +602,27 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			if (!couldExpectMissingTss || countMissing > acceptableTssMissing) {
 				self->testFailure("No storage server on worker");
-				return false;
+				co_return false;
 			} else {
 				TraceEvent(SevWarn, "ConsistencyCheck_TSSMissing").log();
 			}
 		}
 
-		return true;
+		co_return true;
 	}
 
-	ACTOR Future<bool> checkForExtraDataStores(Database cx, ConsistencyCheckWorkload* self) {
-		state std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
-		state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
-		state std::vector<WorkerInterface> coordWorkers = wait(getCoordWorkers(cx, self->dbInfo));
+	Future<bool> checkForExtraDataStores(Database cx, ConsistencyCheckWorkload* self) {
+		std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
+		std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
+		std::vector<WorkerInterface> coordWorkers = co_await getCoordWorkers(cx, self->dbInfo);
 		auto& db = self->dbInfo->get();
-		state std::vector<TLogInterface> logs = db.logSystemConfig.allPresentLogs();
+		std::vector<TLogInterface> logs = db.logSystemConfig.allPresentLogs();
 
-		state std::vector<WorkerDetails>::iterator itr;
-		state bool foundExtraDataStore = false;
-		state std::vector<struct ProcessInfo*> protectedProcessesToKill;
+		std::vector<WorkerDetails>::iterator itr;
+		bool foundExtraDataStore = false;
+		std::vector<struct ProcessInfo*> protectedProcessesToKill;
 
-		state std::map<NetworkAddress, std::set<UID>> statefulProcesses;
+		std::map<NetworkAddress, std::set<UID>> statefulProcesses;
 		for (const auto& ss : storageServers) {
 			statefulProcesses[ss.address()].insert(ss.id());
 			// A process may have two addresses (same ip, different ports)
@@ -651,13 +661,13 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 		for (itr = workers.begin(); itr != workers.end(); ++itr) {
 			ErrorOr<Standalone<VectorRef<UID>>> stores =
-			    wait(itr->interf.diskStoreRequest.getReplyUnlessFailedFor(DiskStoreRequest(false), 2, 0));
+			    co_await itr->interf.diskStoreRequest.getReplyUnlessFailedFor(DiskStoreRequest(false), 2, 0);
 			if (stores.isError()) {
 				TraceEvent("ConsistencyCheck_GetDataStoreFailure")
 				    .error(stores.getError())
 				    .detail("Address", itr->interf.address());
 				self->testFailure("Failed to get data stores");
-				return false;
+				co_return false;
 			}
 
 			TraceEvent(SevCCheckInfo, "ConsistencyCheck_ExtraDataStore")
@@ -687,7 +697,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ProcessPrimaryAddress", p->address)
 					    .detail("ProcessAddresses", p->addresses.toString())
 					    .detail("DataStoreID", id)
-					    .detail("Protected", g_simulator->protectedAddresses.contains(itr->interf.address()))
+					    .detail("Protected", g_simulator->isProtectedAddress(itr->interf.address()))
 					    .detail("Reliable", p->isReliable())
 					    .detail("ReliableInfo", p->getReliableInfo())
 					    .detail("KillOrRebootProcess", p->address);
@@ -704,18 +714,18 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 		if (foundExtraDataStore) {
 			self->testFailure("Extra data stores present on workers");
-			return false;
+			co_return false;
 		}
 
-		return true;
+		co_return true;
 	}
 
-	ACTOR Future<bool> checkWorkerList(Database cx, ConsistencyCheckWorkload* self) {
-		if (!g_simulator->extraDatabases.empty()) {
-			return true;
+	Future<bool> checkWorkerList(Database cx, ConsistencyCheckWorkload* self) {
+		if (!fdbSimulationPolicyState().extraDatabases.empty()) {
+			co_return true;
 		}
 
-		std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
+		std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
 		std::set<NetworkAddress> workerAddresses;
 
 		for (const auto& it : workers) {
@@ -723,7 +733,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			ISimulator::ProcessInfo* info = g_simulator->getProcessByAddress(addr);
 			if (!info || info->failed) {
 				TraceEvent("ConsistencyCheck_FailedWorkerInList").detail("Addr", it.interf.address());
-				return false;
+				co_return false;
 			}
 			workerAddresses.insert(NetworkAddress(addr.ip, addr.port, true, addr.isTLS()));
 		}
@@ -736,12 +746,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    all[i]->protocolVersion == g_network->protocolVersion()) {
 				if (!workerAddresses.contains(all[i]->address)) {
 					TraceEvent("ConsistencyCheck_WorkerMissingFromList").detail("Addr", all[i]->address);
-					return false;
+					co_return false;
 				}
 			}
 		}
 
-		return true;
+		co_return true;
 	}
 
 	static ProcessClass::Fitness getBestAvailableFitness(
@@ -763,25 +773,26 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return "NotSet";
 	}
 
-	ACTOR Future<bool> checkCoordinators(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	Future<bool> checkCoordinators(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				Optional<Value> currentKey = wait(tr.get(coordinatorsKey));
+				Optional<Value> currentKey = co_await tr.get(coordinatorsKey);
 
 				if (!currentKey.present()) {
 					TraceEvent("ConsistencyCheck_NoCoordinatorKey").log();
-					return false;
+					co_return false;
 				}
 
 				ClusterConnectionString old(currentKey.get().toString());
-				state std::vector<NetworkAddress> oldCoordinators = wait(old.tryResolveHostnames());
+				std::vector<NetworkAddress> oldCoordinators = co_await old.tryResolveHostnames();
 
-				std::vector<ProcessData> workers = wait(::getWorkers(&tr));
+				std::vector<ProcessData> workers = co_await ::getWorkers(&tr);
 
 				std::map<NetworkAddress, LocalityData> addr_locality;
-				for (auto w : workers) {
+				for (const auto& w : workers) {
 					addr_locality[w.address] = w.locality;
 				}
 
@@ -793,27 +804,28 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							TraceEvent("ConsistencyCheck_BadCoordinator")
 							    .detail("Addr", addr)
 							    .detail("NotFound", findResult == addr_locality.end());
-							return false;
+							co_return false;
 						}
 						checkDuplicates.insert(findResult->second.zoneId());
 					}
 				}
 
-				return true;
+				co_return true;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
 	// Returns true if all machines in the cluster that specified a desired class are operating in that class
-	ACTOR Future<bool> checkUsingDesiredClasses(Database cx, ConsistencyCheckWorkload* self) {
-		state Optional<Key> expectedPrimaryDcId;
-		state Optional<Key> expectedRemoteDcId;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
-		state std::vector<WorkerDetails> allWorkers = wait(getWorkers(self->dbInfo));
-		state std::vector<WorkerDetails> nonExcludedWorkers =
-		    wait(getWorkers(self->dbInfo, GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY));
+	Future<bool> checkUsingDesiredClasses(Database cx, ConsistencyCheckWorkload* self) {
+		Optional<Key> expectedPrimaryDcId;
+		Optional<Key> expectedRemoteDcId;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
+		std::vector<WorkerDetails> allWorkers = co_await getWorkers(self->dbInfo);
+		std::vector<WorkerDetails> nonExcludedWorkers =
+		    co_await getWorkers(self->dbInfo, GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY);
 		auto& db = self->dbInfo->get();
 
 		std::map<NetworkAddress, WorkerDetails> allWorkerProcessMap;
@@ -839,12 +851,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		if (!allWorkerProcessMap.contains(db.clusterInterface.clientInterface.address())) {
 			TraceEvent("ConsistencyCheck_CCNotInWorkerList")
 			    .detail("CCAddress", db.clusterInterface.clientInterface.address().toString());
-			return false;
+			co_return false;
 		}
 		if (!allWorkerProcessMap.contains(db.master.address())) {
 			TraceEvent("ConsistencyCheck_MasterNotInWorkerList")
 			    .detail("MasterAddress", db.master.address().toString());
-			return false;
+			co_return false;
 		}
 
 		Optional<Key> ccDcId =
@@ -855,14 +867,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			TraceEvent("ConsistencyCheck_CCAndMasterNotInSameDC")
 			    .detail("ClusterControllerDcId", getOptionalString(ccDcId))
 			    .detail("MasterDcId", getOptionalString(masterDcId));
-			return false;
+			co_return false;
 		}
 		// Check if master and cluster controller are in the desired DC for fearless cluster when running under
 		// simulation
 		// FIXME: g_simulator->datacenterDead could return false positives. Relaxing checks until it is fixed.
-		if (g_network->isSimulated() && config.usableRegions > 1 && g_simulator->primaryDcId.present() &&
-		    !g_simulator->datacenterDead(g_simulator->primaryDcId) &&
-		    !g_simulator->datacenterDead(g_simulator->remoteDcId)) {
+		if (g_network->isSimulated() && config.usableRegions > 1 && fdbSimulationPolicyState().primaryDcId.present() &&
+		    !g_simulator->datacenterDead(fdbSimulationPolicyState().primaryDcId) &&
+		    !g_simulator->datacenterDead(fdbSimulationPolicyState().remoteDcId)) {
 			expectedPrimaryDcId = config.regions[0].dcId;
 			expectedRemoteDcId = config.regions[1].dcId;
 			// If the priorities are equal, either could be the primary
@@ -876,13 +888,13 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				TraceEvent("ConsistencyCheck_ClusterControllerDcNotBest")
 				    .detail("PreferredDcId", getOptionalString(expectedPrimaryDcId))
 				    .detail("ExistingDcId", getOptionalString(ccDcId));
-				return false;
+				co_return false;
 			}
 			if (masterDcId != expectedPrimaryDcId) {
 				TraceEvent("ConsistencyCheck_MasterDcNotBest")
 				    .detail("PreferredDcId", getOptionalString(expectedPrimaryDcId))
 				    .detail("ExistingDcId", getOptionalString(masterDcId));
-				return false;
+				co_return false;
 			}
 		}
 
@@ -899,7 +911,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			                ? nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()]
 			                      .processClass.machineClassFitness(ProcessClass::ClusterController)
 			                : -1);
-			return false;
+			co_return false;
 		}
 
 		// Check Master
@@ -923,7 +935,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			                ? nonExcludedWorkerProcessMap[db.master.address()].processClass.machineClassFitness(
 			                      ProcessClass::Master)
 			                : -1);
-			return false;
+			co_return false;
 		}
 
 		// Check commit proxy
@@ -940,7 +952,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				                ? nonExcludedWorkerProcessMap[commitProxy.address()].processClass.machineClassFitness(
 				                      ProcessClass::CommitProxy)
 				                : -1);
-				return false;
+				co_return false;
 			}
 		}
 
@@ -958,7 +970,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				                ? nonExcludedWorkerProcessMap[grvProxy.address()].processClass.machineClassFitness(
 				                      ProcessClass::GrvProxy)
 				                : -1);
-				return false;
+				co_return false;
 			}
 		}
 
@@ -976,27 +988,27 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				                ? nonExcludedWorkerProcessMap[resolver.address()].processClass.machineClassFitness(
 				                      ProcessClass::Resolver)
 				                : -1);
-				return false;
+				co_return false;
 			}
 		}
 
 		// Check LogRouter
-		if (g_network->isSimulated() && config.usableRegions > 1 && g_simulator->primaryDcId.present() &&
-		    !g_simulator->datacenterDead(g_simulator->primaryDcId) &&
-		    !g_simulator->datacenterDead(g_simulator->remoteDcId)) {
+		if (g_network->isSimulated() && config.usableRegions > 1 && fdbSimulationPolicyState().primaryDcId.present() &&
+		    !g_simulator->datacenterDead(fdbSimulationPolicyState().primaryDcId) &&
+		    !g_simulator->datacenterDead(fdbSimulationPolicyState().remoteDcId)) {
 			for (auto& tlogSet : db.logSystemConfig.tLogs) {
-				if (!tlogSet.isLocal && tlogSet.logRouters.size()) {
+				if (!tlogSet.isLocal && !tlogSet.logRouters.empty()) {
 					for (auto& logRouter : tlogSet.logRouters) {
 						if (!nonExcludedWorkerProcessMap.contains(logRouter.interf().address())) {
 							TraceEvent("ConsistencyCheck_LogRouterNotInNonExcludedWorkers")
 							    .detail("Id", logRouter.id());
-							return false;
+							co_return false;
 						}
 						if (logRouter.interf().filteredLocality.dcId() != expectedRemoteDcId) {
 							TraceEvent("ConsistencyCheck_LogRouterNotBestDC")
 							    .detail("expectedDC", getOptionalString(expectedRemoteDcId))
 							    .detail("ActualDC", getOptionalString(logRouter.interf().filteredLocality.dcId()));
-							return false;
+							co_return false;
 						}
 					}
 				}
@@ -1018,7 +1030,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			            ? nonExcludedWorkerProcessMap[db.distributor.get().address()].processClass.machineClassFitness(
 			                  ProcessClass::DataDistributor)
 			            : -1);
-			return false;
+			co_return false;
 		}
 
 		// Check Ratekeeper
@@ -1034,7 +1046,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			            ? nonExcludedWorkerProcessMap[db.ratekeeper.get().address()].processClass.machineClassFitness(
 			                  ProcessClass::Ratekeeper)
 			            : -1);
-			return false;
+			co_return false;
 		}
 
 		// Check ConsistencyScan
@@ -1049,29 +1061,31 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			                ? nonExcludedWorkerProcessMap[db.consistencyScan.get().address()]
 			                      .processClass.machineClassFitness(ProcessClass::ConsistencyScan)
 			                : -1);
-			return false;
+			co_return false;
 		}
 
 		// TODO: Check Tlog
 
-		return true;
+		co_return true;
 	}
 
 	// returns true if stopped, false otherwise
-	ACTOR Future<bool> checkConsistencyScan(Database cx) {
+	Future<bool> checkConsistencyScan(Database cx) {
 		if (!g_network->isSimulated()) {
-			return true;
+			co_return true;
 		}
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state ConsistencyScanState cs;
-		loop {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		ConsistencyScanState cs;
+		while (true) {
+			Error err;
 			try {
 				SystemDBWriteLockedNow(cx.getReference())->setOptions(tr);
-				ConsistencyScanState::Config config = wait(cs.config().getD(tr));
-				return !config.enabled;
+				ConsistencyScanState::Config config = co_await cs.config().getD(tr);
+				co_return !config.enabled;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
diff --git a/fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp b/fdbserver/workloads/ConsistencyCheckUrgent.cpp
similarity index 77%
rename from fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp
rename to fdbserver/workloads/ConsistencyCheckUrgent.cpp
index e47072fdbe4..d91ce9ee6b5 100644
--- a/fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheckUrgent.cpp
@@ -1,5 +1,5 @@
 /*
- * ConsistencyCheckUrgent.actor.cpp
+ * ConsistencyCheckUrgent.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,17 +25,16 @@
 #include "flow/ProcessEvents.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/IRateControl.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/DeterministicRandom.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/network.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // The ConsistencyCheckUrgent workload is designed to support the consistency check
 // urgent feature, a distributed version of the consistency check which emphasizes
@@ -55,15 +54,16 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 
 	int64_t consistencyCheckerId;
 
-	ConsistencyCheckUrgentWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ConsistencyCheckUrgentWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		consistencyCheckerId = sharedRandomNumber;
 	}
 
-	ACTOR Future<std::vector<std::pair<KeyRange, Value>>>
-	getKeyLocationsForRangeList(Database cx, std::vector<KeyRange> ranges, ConsistencyCheckUrgentWorkload* self) {
+	Future<std::vector<std::pair<KeyRange, Value>>> getKeyLocationsForRangeList(Database cx,
+	                                                                            std::vector<KeyRange> ranges,
+	                                                                            ConsistencyCheckUrgentWorkload* self) {
 		// Get the scope of the input list of ranges
-		state Key beginKeyToReadKeyServer;
-		state Key endKeyToReadKeyServer;
+		Key beginKeyToReadKeyServer;
+		Key endKeyToReadKeyServer;
 		for (int i = 0; i < ranges.size(); i++) {
 			if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
 				beginKeyToReadKeyServer = ranges[i].begin;
@@ -78,19 +78,20 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 		    .detail("ClientCount", self->clientCount)
 		    .detail("ClientId", self->clientId);
 		// Read KeyServer space within the scope and add shards intersecting with the input ranges
-		state std::vector<std::pair<KeyRange, Value>> res;
-		state Transaction tr(cx);
-		loop {
+		std::vector<std::pair<KeyRange, Value>> res;
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
-				RangeResult readResult = wait(krmGetRanges(&tr,
-				                                           keyServersPrefix,
-				                                           rangeToRead,
-				                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
-				                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
+				RangeResult readResult = co_await krmGetRanges(&tr,
+				                                               keyServersPrefix,
+				                                               rangeToRead,
+				                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+				                                               SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES);
 				for (int i = 0; i < readResult.size() - 1; ++i) {
 					KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
 					Value valueToCheck = Standalone(readResult[i].value);
@@ -110,57 +111,64 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					break;
 				}
 			} catch (Error& e) {
+				err = e;
+			}
+			if (err.isValid()) {
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterRetryGetKeyLocationsForRangeList")
-				    .error(e)
+				    .error(err)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId);
-				wait(tr.onError(e));
+				co_await tr.onError(err);
 			}
 		}
-		return res;
+		co_return res;
 	}
 
-	ACTOR Future<Version> getVersion(Database cx) {
-		loop {
-			state Transaction tr(cx);
+	Future<Version> getVersion(Database cx) {
+		while (true) {
+			Transaction tr(cx);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Error err;
 			try {
-				Version version = wait(tr.getReadVersion());
-				return version;
+				Version version = co_await tr.getReadVersion();
+				co_return version;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> checkDataConsistencyUrgent(Database cx,
-	                                              std::vector<KeyRange> rangesToCheck,
-	                                              ConsistencyCheckUrgentWorkload* self,
-	                                              int consistencyCheckEpoch) {
+	Future<Void> checkDataConsistencyUrgent(Database cx,
+	                                        std::vector<KeyRange> rangesToCheck,
+	                                        ConsistencyCheckUrgentWorkload* self,
+	                                        int consistencyCheckEpoch) {
 		// Get shard locations of the input rangesToCheck
-		state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
-		state int retryCount = 0;
-		loop {
+		std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
+		int retryCount = 0;
+		while (true) {
+			Error err;
 			try {
 				shardLocationPairList.clear();
-				wait(store(shardLocationPairList, self->getKeyLocationsForRangeList(cx, rangesToCheck, self)));
+				shardLocationPairList = co_await self->getKeyLocationsForRangeList(cx, rangesToCheck, self);
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterGetKeyLocationListFailed")
-				    .error(e)
-				    .detail("RetryCount", retryCount)
-				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-				    .detail("ClientId", self->clientId)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-				wait(delay(5.0));
-				retryCount++;
-				if (retryCount > 50) {
-					throw timed_out();
-				}
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			}
+			TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterGetKeyLocationListFailed")
+			    .error(err)
+			    .detail("RetryCount", retryCount)
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+			    .detail("ClientId", self->clientId)
+			    .detail("ClientCount", self->clientCount)
+			    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+			co_await delay(5.0);
+			retryCount++;
+			if (retryCount > 50) {
+				throw timed_out();
 			}
 		}
 
@@ -173,31 +181,31 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 
 		// Do consistency check shard by shard
-		state Reference<IRateControl> rateLimiter =
-		    Reference<IRateControl>(new SpeedLimit(CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX, 1));
-		state KeyRangeMap<bool> failedRanges; // Used to collect failed ranges in the current checkDataConsistency
+		Reference<IRateControl> rateLimiter =
+		    makeReference<SpeedLimit>(CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX, 1);
+		KeyRangeMap<bool> failedRanges; // Used to collect failed ranges in the current checkDataConsistency
 		failedRanges.insert(allKeys, false); // Initialized with false and will set any failed range as true later
 		// Which will be used to start the next consistencyCheckEpoch of the checkDataConsistency
-		state int64_t numShardThisClient = shardLocationPairList.size();
-		state int64_t numShardToCheck = -1;
-		state int64_t numCompleteShards = 0;
-		state int64_t numFailedShards = 0;
-		state int shardIdx = 0;
-		for (; shardIdx < shardLocationPairList.size(); ++shardIdx) {
+		int64_t numShardThisClient = shardLocationPairList.size();
+		int64_t numShardToCheck = -1;
+		int64_t numCompleteShards = 0;
+		int64_t numFailedShards = 0;
+		for (int shardIdx = 0; shardIdx < shardLocationPairList.size(); ++shardIdx) {
 			numShardToCheck = numShardThisClient - shardIdx;
-			state KeyRangeRef range = shardLocationPairList[shardIdx].first;
+			KeyRangeRef range = shardLocationPairList[shardIdx].first;
 			// Step 1: Get source server id of the shard
-			state std::vector<UID> sourceStorageServers;
-			state std::vector<UID> destStorageServers;
-			state std::unordered_map<UID, Tag> storageServerToTagMap; // populated only when version vector is enabled
+			std::vector<UID> sourceStorageServers;
+			std::vector<UID> destStorageServers;
+			std::unordered_map<UID, Tag> storageServerToTagMap; // populated only when version vector is enabled
 			retryCount = 0;
-			loop {
+			while (true) {
+				Error err;
 				try {
 					Transaction tr(cx);
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 					tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-					RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 					decodeKeyServersValue(UIDtoTagMap,
 					                      shardLocationPairList[shardIdx].second,
@@ -212,27 +220,28 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					}
 					break;
 				} catch (Error& e) {
-					if (e.code() == error_code_actor_cancelled) {
-						throw e;
-					}
-					TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterGetUIDtoTagMapFailed")
-					    .error(e)
-					    .detail("RetryCount", retryCount)
-					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-					    .detail("ClientId", self->clientId)
-					    .detail("ClientCount", self->clientCount)
-					    .detail("ShardBegin", range.begin)
-					    .detail("ShardEnd", range.end)
-					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-					wait(delay(5.0));
-					retryCount++;
-					if (retryCount > 50) {
-						throw timed_out();
-					}
+					err = e;
+				}
+				if (err.code() == error_code_actor_cancelled) {
+					throw err;
+				}
+				TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterGetUIDtoTagMapFailed")
+				    .error(err)
+				    .detail("RetryCount", retryCount)
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ShardBegin", range.begin)
+				    .detail("ShardEnd", range.end)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				co_await delay(5.0);
+				retryCount++;
+				if (retryCount > 50) {
+					throw timed_out();
 				}
 			}
 
-			if (sourceStorageServers.size() == 0) {
+			if (sourceStorageServers.empty()) {
 				TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_TesterEmptySourceServers")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("ClientId", self->clientId)
@@ -255,10 +264,11 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 			}
 
 			// Step 2: Get server interfaces
-			state std::vector<UID> storageServers = sourceStorageServers; // We check source server
-			state std::vector<StorageServerInterface> storageServerInterfaces;
+			std::vector<UID> storageServers = sourceStorageServers; // We check source server
+			std::vector<StorageServerInterface> storageServerInterfaces;
 			retryCount = 0;
-			loop {
+			while (true) {
+				Error err;
 				try {
 					Transaction tr(cx);
 					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -267,7 +277,7 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					std::vector<Future<Optional<Value>>> serverListEntries;
 					for (int s = 0; s < storageServers.size(); s++)
 						serverListEntries.push_back(tr.get(serverListKeyFor(storageServers[s])));
-					state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+					std::vector<Optional<Value>> serverListValues = co_await getAll(serverListEntries);
 					for (int s = 0; s < serverListValues.size(); s++) {
 						if (!serverListValues[s].present()) {
 							TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterGetServerInterfaceFailed")
@@ -296,30 +306,32 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					}
 					break;
 				} catch (Error& e) {
-					if (e.code() == error_code_actor_cancelled) {
-						throw e;
-					}
-					wait(delay(5.0));
-					retryCount++;
-					if (retryCount > 10) {
-						// SS could be removed from the cluster
-						throw timed_out();
-					}
+					err = e;
+				}
+				if (err.code() == error_code_actor_cancelled) {
+					throw err;
+				}
+				co_await delay(5.0);
+				retryCount++;
+				if (retryCount > 10) {
+					// SS could be removed from the cluster
+					throw timed_out();
 				}
 			}
 
 			// Step 3: Read a limited number of entries at a time, repeating until all keys in the shard have been read
-			state int64_t totalReadAmount = 0;
-			state int64_t shardReadAmount = 0;
-			state int64_t shardKeyCompared = 0;
-			state bool valueAvailableToCheck = true;
-			state KeySelector begin = firstGreaterOrEqual(range.begin);
-			loop {
+			int64_t totalReadAmount = 0;
+			int64_t shardReadAmount = 0;
+			int64_t shardKeyCompared = 0;
+			bool valueAvailableToCheck = true;
+			KeySelector begin = firstGreaterOrEqual(range.begin);
+			while (true) {
+				Error err;
 				try {
 					// Get the min version of the storage servers
-					Version version = wait(self->getVersion(cx));
+					Version version = co_await self->getVersion(cx);
 
-					state GetKeyValuesRequest req;
+					GetKeyValuesRequest req;
 					req.begin = begin;
 					req.end = firstGreaterOrEqual(range.end);
 					req.limit = 1e4;
@@ -331,8 +343,8 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					req.tags = TagSet();
 
 					// Try getting the entries in the specified range
-					state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
-					state int j = 0;
+					std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+					int j = 0;
 					for (j = 0; j < storageServerInterfaces.size(); j++) {
 						resetReply(req);
 						if (SERVER_KNOBS->ENABLE_VERSION_VECTOR) {
@@ -342,13 +354,14 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 						keyValueFutures.push_back(
 						    storageServerInterfaces[j].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 					}
-					wait(waitForAll(keyValueFutures));
+					co_await waitForAll(keyValueFutures);
 
 					for (j = 0; j < keyValueFutures.size(); j++) {
 						ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
 						if (!rangeResult.present() || rangeResult.get().error.present()) {
 							valueAvailableToCheck = false;
 							TraceEvent e(SevInfo, "ConsistencyCheckUrgent_TesterGetRangeError");
+							e.suppressFor(60.0);
 							e.detail("ResultPresent", rangeResult.present());
 							e.detail("StorageServer", storageServerInterfaces[j].uniqueID);
 							if (rangeResult.present()) {
@@ -369,6 +382,7 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					if (!valueAvailableToCheck) {
 						failedRanges.insert(range, true);
 						TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterShardAddedToRetry")
+						    .suppressFor(60.0)
 						    .setMaxEventLength(-1)
 						    .setMaxFieldLength(-1)
 						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -380,13 +394,13 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 						break;
 					}
 
-					state int firstValidServer = -1;
+					int firstValidServer = -1;
 					totalReadAmount = 0;
 					for (j = 0; j < keyValueFutures.size(); j++) {
 						ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
 						ASSERT(rangeResult.present() && !rangeResult.get().error.present());
 
-						state GetKeyValuesReply current = rangeResult.get();
+						GetKeyValuesReply current = rangeResult.get();
 						totalReadAmount += current.data.expectedSize();
 						// If we haven't encountered a valid storage server yet, then mark this as the baseline
 						// to compare against
@@ -478,7 +492,7 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					}
 
 					// RateKeeping
-					wait(rateLimiter->getAllowance(totalReadAmount));
+					co_await rateLimiter->getAllowance(totalReadAmount);
 
 					shardReadAmount += totalReadAmount;
 
@@ -486,7 +500,7 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					ASSERT(firstValidServer != -1);
 					if (keyValueFutures[firstValidServer].get().get().more) {
 						VectorRef<KeyValueRef> result = keyValueFutures[firstValidServer].get().get().data;
-						ASSERT(result.size() > 0);
+						ASSERT(!result.empty());
 						begin = firstGreaterThan(result[result.size() - 1].key);
 						ASSERT(begin.getKey() != allKeys.end);
 					} else {
@@ -494,12 +508,16 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 					}
 
 				} catch (Error& e) {
-					if (e.code() == error_code_actor_cancelled) {
-						throw e;
-					}
-					TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterRetryDataConsistency").error(e);
-					wait(delay(5.0));
+					err = e;
+				}
+				if (!err.isValid()) {
+					continue;
 				}
+				if (err.code() == error_code_actor_cancelled) {
+					throw err;
+				}
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterRetryDataConsistency").error(err);
+				co_await delay(5.0);
 			}
 
 			// Step 4: if the shard failed to check, add it to retry queue
@@ -509,6 +527,7 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 				// by the next consistencyCheckEpoch
 				numFailedShards++;
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterShardFailed")
+				    .suppressFor(60.0)
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -547,24 +566,24 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 
 		// For the failed ranges, trigger next epoch of checkDataConsistencyUrgent
 		failedRanges.coalesce(allKeys);
-		state std::vector<KeyRange> failedRangesToCheck;
-		state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
-		state KeyRangeMap<bool>::iterator failedRangesIter = failedRangesList.begin();
-		for (; failedRangesIter != failedRangesList.end(); ++failedRangesIter) {
+		std::vector<KeyRange> failedRangesToCheck;
+		KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
+		for (auto failedRangesIter = failedRangesList.begin(); failedRangesIter != failedRangesList.end();
+		     ++failedRangesIter) {
 			if (failedRangesIter->value()) {
 				failedRangesToCheck.push_back(failedRangesIter->range());
 			}
 		}
-		if (failedRangesToCheck.size() > 0) { // Retry for any failed shard
+		if (!failedRangesToCheck.empty()) { // Retry for any failed shard
 			if (consistencyCheckEpoch < CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_RETRY_DEPTH_MAX) {
-				wait(delay(60.0)); // Backoff 1 min
+				co_await delay(60.0); // Backoff 1 min
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterRetryFailedRanges")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
 				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
 				    .detail("ClientId", self->clientId)
 				    .detail("ClientCount", self->clientCount);
-				wait(self->checkDataConsistencyUrgent(cx, failedRangesToCheck, self, consistencyCheckEpoch + 1));
+				co_await self->checkDataConsistencyUrgent(cx, failedRangesToCheck, self, consistencyCheckEpoch + 1);
 			} else {
 				TraceEvent(SevWarn, "ConsistencyCheckUrgent_TesterRetryDepthMax")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -584,38 +603,37 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 			    .detail("ClientId", self->clientId)
 			    .detail("ClientCount", self->clientCount);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, ConsistencyCheckUrgentWorkload* self) {
+	Future<Void> _start(Database cx) {
 		try {
 			TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterStart")
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("ClientId", self->clientId);
-			if (self->rangesToCheck.size() == 0) {
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("ClientCount", clientCount)
+			    .detail("ClientId", clientId);
+			if (rangesToCheck.empty()) {
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterExit")
 				    .detail("Reason", "AssignedEmptyRangeToCheck")
-				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("ClientId", self->clientId);
-				return Void();
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("ClientCount", clientCount)
+				    .detail("ClientId", clientId);
+				co_return;
 			}
 			if (g_network->isSimulated() && deterministicRandom()->coinflip()) {
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterMimicFailure")
-				    .detail("ClientCount", self->clientCount)
-				    .detail("ClientId", self->clientId);
+				    .detail("ClientCount", clientCount)
+				    .detail("ClientId", clientId);
 				throw operation_failed(); // mimic tester failure
 			}
-			wait(self->checkDataConsistencyUrgent(cx,
-			                                      self->rangesToCheck,
-			                                      self,
-			                                      /*consistencyCheckEpoch=*/0));
+			co_await checkDataConsistencyUrgent(cx,
+			                                    rangesToCheck,
+			                                    this,
+			                                    /*consistencyCheckEpoch=*/0);
 			TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterExit")
 			    .detail("Reason", "CompleteCheck")
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("ClientId", self->clientId);
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("ClientCount", clientCount)
+			    .detail("ClientId", clientId);
 		} catch (Error& e) {
 			std::string reason;
 			Severity sev = SevInfo;
@@ -632,19 +650,18 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload {
 			TraceEvent(sev, "ConsistencyCheckUrgent_TesterExit")
 			    .error(e)
 			    .detail("Reason", reason)
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("ClientId", self->clientId);
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("ClientCount", clientCount)
+			    .detail("ClientId", clientId);
 			throw consistency_check_urgent_task_failed();
 		}
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
 	Future<Void> start(Database const& cx) override {
 		TraceEvent("ConsistencyCheckUrgent_EnterWorkload").detail("ConsistencyCheckerId", consistencyCheckerId);
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/CpuProfiler.actor.cpp b/fdbserver/workloads/CpuProfiler.cpp
similarity index 72%
rename from fdbserver/workloads/CpuProfiler.actor.cpp
rename to fdbserver/workloads/CpuProfiler.cpp
index ac4ec8642bc..9fb1a14cc5a 100644
--- a/fdbserver/workloads/CpuProfiler.actor.cpp
+++ b/fdbserver/workloads/CpuProfiler.cpp
@@ -1,5 +1,5 @@
 /*
- * CpuProfiler.actor.cpp
+ * CpuProfiler.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
 
 // A workload which starts the CPU profiler at a given time and duration on all workers in a cluster
 struct CpuProfilerWorkload : TestWorkload {
@@ -42,7 +41,7 @@ struct CpuProfilerWorkload : TestWorkload {
 	// A list of worker interfaces which have had profiling turned on
 	std::vector<WorkerInterface> profilingWorkers;
 
-	CpuProfilerWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit CpuProfilerWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		initialDelay = getOption(options, "initialDelay"_sr, 0.0);
 		duration = getOption(options, "duration"_sr, -1.0);
 		roles = getOption(options, "roles"_sr, std::vector<std::string>());
@@ -52,11 +51,11 @@ struct CpuProfilerWorkload : TestWorkload {
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
 	// Turns the profiler on or off
-	ACTOR Future<Void> updateProfiler(bool enabled, Database cx, CpuProfilerWorkload* self) {
+	Future<Void> updateProfiler(bool enabled, Database cx, CpuProfilerWorkload* self) {
 		if (self->clientId == 0) {
 			// If we are turning the profiler on, get a list of workers in the system
 			if (enabled) {
-				std::vector<WorkerDetails> _workers = wait(getWorkers(self->dbInfo));
+				std::vector<WorkerDetails> _workers = co_await getWorkers(self->dbInfo);
 				std::vector<WorkerInterface> workers;
 				for (int i = 0; i < _workers.size(); i++) {
 					if (self->roles.empty() ||
@@ -68,8 +67,8 @@ struct CpuProfilerWorkload : TestWorkload {
 				self->profilingWorkers = workers;
 			}
 
-			state std::vector<Future<ErrorOr<Void>>> replies;
-			state int i;
+			std::vector<Future<ErrorOr<Void>>> replies;
+			int i{ 0 };
 			// Send a ProfilerRequest to each worker
 			for (i = 0; i < self->profilingWorkers.size(); i++) {
 				ProfilerRequest req;
@@ -84,7 +83,7 @@ struct CpuProfilerWorkload : TestWorkload {
 				replies.push_back(self->profilingWorkers[i].clientInterface.profiler.tryGetReply(req));
 			}
 
-			wait(waitForAll(replies));
+			co_await waitForAll(replies);
 
 			// Check that all workers succeeded if turning the profiler on
 			if (enabled)
@@ -94,40 +93,32 @@ struct CpuProfilerWorkload : TestWorkload {
 
 			TraceEvent("DoneSignalingProfiler").log();
 		}
-
-		return Void();
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
-	ACTOR Future<Void> _start(Database cx, CpuProfilerWorkload* self) {
-		wait(delay(self->initialDelay));
-		if (self->clientId == 0)
+	Future<Void> start(Database const& cx) override {
+		co_await delay(initialDelay);
+		if (clientId == 0)
 			TraceEvent("SignalProfilerOn").log();
-		wait(timeoutError(self->updateProfiler(true, cx, self), 60.0));
+		co_await timeoutError(updateProfiler(true, cx, this), 60.0);
 
 		// If a duration was given, let the duration elapse and then shut the profiler off
-		if (self->duration > 0) {
-			wait(delay(self->duration));
-			if (self->clientId == 0)
+		if (duration > 0) {
+			co_await delay(duration);
+			if (clientId == 0)
 				TraceEvent("SignalProfilerOff").log();
-			wait(timeoutError(self->updateProfiler(false, cx, self), 60.0));
+			co_await timeoutError(updateProfiler(false, cx, this), 60.0);
 		}
-
-		return Void();
 	}
 
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
-
-	ACTOR Future<bool> _check(Database cx, CpuProfilerWorkload* self) {
+	Future<bool> check(Database const& cx) override {
 		// If no duration was given, then shut the profiler off now
-		if (self->duration <= 0) {
-			if (self->clientId == 0)
+		if (duration <= 0) {
+			if (clientId == 0)
 				TraceEvent("SignalProfilerOff").log();
-			wait(timeoutError(self->updateProfiler(false, cx, self), 60.0));
+			co_await timeoutError(updateProfiler(false, cx, this), 60.0);
 		}
 
-		return self->success;
+		co_return success;
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.cpp
similarity index 79%
rename from fdbserver/workloads/Cycle.actor.cpp
rename to fdbserver/workloads/Cycle.cpp
index 6460b65cdb5..ff2b60ef7cb 100644
--- a/fdbserver/workloads/Cycle.actor.cpp
+++ b/fdbserver/workloads/Cycle.cpp
@@ -1,5 +1,5 @@
 /*
- * Cycle.actor.cpp
+ * Cycle.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -27,11 +27,9 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 struct CycleWorkload : TestWorkload, Arena {
 	static constexpr auto NAME = "Cycle";
@@ -45,7 +43,7 @@ struct CycleWorkload : TestWorkload, Arena {
 	PerfIntCounter transactions, retries, tooOldRetries, commitFailedRetries;
 	PerfDoubleCounter totalLatency;
 
-	CycleWorkload(WorkloadContext const& wcx)
+	explicit CycleWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), tooOldRetries("Retries.too_old"),
 	    commitFailedRetries("Retries.commit_failed"), totalLatency("Latency") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -66,9 +64,7 @@ struct CycleWorkload : TestWorkload, Arena {
 		if (skipSetup) {
 			return Void();
 		}
-		// TODO(gglass): possible cleanup here.  Leftover tenant stuff.
-		Future<Void> prepare = Void();
-		return runAfter(prepare, [this, cx](Void) { return bulkSetup(cx, this, nodeCount, Promise<double>()); });
+		return bulkSetup(cx, this, nodeCount, Promise<double>());
 	}
 	Future<Void> start(Database const& cx) override {
 		for (int c = 0; c < actorCount; c++)
@@ -115,44 +111,45 @@ struct CycleWorkload : TestWorkload, Arena {
 		    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken);
 	}
 
-	ACTOR Future<Void> cycleClient(Database cx, CycleWorkload* self, double delay) {
-		state double lastTime = now();
+	Future<Void> cycleClient(Database cx, CycleWorkload* self, double delay) {
+		double lastTime = now();
 		TraceEvent("CycleClientStart").log();
 		try {
-			loop {
-				wait(poisson(&lastTime, delay));
+			while (true) {
+				co_await poisson(&lastTime, delay);
 
-				state double tstart = now();
-				state int r = deterministicRandom()->randomInt(0, self->nodeCount);
-				state Transaction tr(cx);
+				double tstart = now();
+				int r = deterministicRandom()->randomInt(0, self->nodeCount);
+				Transaction tr(cx);
 				if (deterministicRandom()->random01() <= self->traceParentProbability) {
-					state Span span("CycleClient"_loc);
+					Span span("CycleClient"_loc);
 					TraceEvent("CycleTracingTransaction", span.context.traceID).log();
 					tr.setOption(FDBTransactionOptions::SPAN_PARENT,
 					             BinaryWriter::toValue(span.context, IncludeVersion()));
 				}
 				while (true) {
+					Error err;
 					try {
 						// Reverse next and next^2 node
-						Optional<Value> v = wait(tr.get(self->key(r)));
+						Optional<Value> v = co_await tr.get(self->key(r));
 						if (!v.present()) {
 							self->badRead("KeyR", r, tr);
 						}
-						state int r2 = self->fromValue(v.get());
-						Optional<Value> v2 = wait(tr.get(self->key(r2)));
+						int r2 = self->fromValue(v.get());
+						Optional<Value> v2 = co_await tr.get(self->key(r2));
 						if (!v2.present())
 							self->badRead("KeyR2", r2, tr);
-						state int r3 = self->fromValue(v2.get());
-						Optional<Value> v3 = wait(tr.get(self->key(r3)));
+						int r3 = self->fromValue(v2.get());
+						Optional<Value> v3 = co_await tr.get(self->key(r3));
 						if (!v3.present())
 							self->badRead("KeyR3", r3, tr);
-						state int r4 = self->fromValue(v3.get());
+						int r4 = self->fromValue(v3.get());
 
-						// Single key clear range op will be converted to point delete inside storage engine. Generating
-						// a larger range here to increase test coverage.
-						tr.clear(
-						    self->keyRange(r),
-						    AddConflictRange::True); //< Shouldn't have an effect, but will break with wrong ordering
+						// Single key clear range op will be converted to point delete inside storage engine.
+						// Generating a larger range here to increase test coverage.
+						tr.clear(self->keyRange(r),
+						         AddConflictRange::True); //< Shouldn't have an effect, but will break with wrong
+						                                  // ordering
 						tr.set(self->key(r), self->value(r3));
 						tr.set(self->key(r2), self->value(r4));
 						tr.set(self->key(r3), self->value(r2));
@@ -160,7 +157,7 @@ struct CycleWorkload : TestWorkload, Arena {
 						// TraceEvent("CyclicTest2").detail("RawKey", r2).detail("RawValue", r4).detail("Key", self->key(r2).toString()).detail("Value", self->value(r4).toString()).log();
 						// TraceEvent("CyclicTest3").detail("RawKey", r3).detail("RawValue", r2).detail("Key", self->key(r3).toString()).detail("Value", self->value(r2).toString()).log();
 
-						wait(tr.commit());
+						co_await tr.commit();
 						// TraceEvent("CyclicTestCommit")
 						// 	.detail("R1", r)
 						// 	.detail("R2", r2)
@@ -169,12 +166,13 @@ struct CycleWorkload : TestWorkload, Arena {
 						// 	.log();
 						break;
 					} catch (Error& e) {
-						if (e.code() == error_code_transaction_too_old)
-							++self->tooOldRetries;
-						else if (e.code() == error_code_not_committed)
-							++self->commitFailedRetries;
-						wait(tr.onError(e));
+						err = e;
 					}
+					if (err.code() == error_code_transaction_too_old)
+						++self->tooOldRetries;
+					else if (err.code() == error_code_not_committed)
+						++self->commitFailedRetries;
+					co_await tr.onError(err);
 					++self->retries;
 				}
 				++self->transactions;
@@ -263,7 +261,7 @@ struct CycleWorkload : TestWorkload, Arena {
 		return true;
 	}
 
-	ACTOR Future<bool> cycleCheck(Database cx, CycleWorkload* self, bool ok) {
+	Future<bool> cycleCheck(Database cx, CycleWorkload* self, bool ok) {
 		if (self->transactions.getMetric().value() < self->testDuration * self->minExpectedTransactionsPerSecond) {
 			TraceEvent(SevWarnAlways, "TestFailure")
 			    .detail("Reason", "Rate below desired rate")
@@ -279,32 +277,34 @@ struct CycleWorkload : TestWorkload, Arena {
 		}
 		if (!self->clientId) {
 			// One client checks the validity of the cycle
-			state Transaction tr(cx);
-			state int retryCount = 0;
-			loop {
+			Transaction tr(cx);
+			int retryCount = 0;
+			while (true) {
+				Error err;
 				try {
-					state Version v = wait(tr.getReadVersion());
-					RangeResult data = wait(tr.getRange(firstGreaterOrEqual(doubleToTestKey(0.0, self->keyPrefix)),
-					                                    firstGreaterOrEqual(doubleToTestKey(1.0, self->keyPrefix)),
-					                                    self->nodeCount + 1));
+					Version v = co_await tr.getReadVersion();
+					RangeResult data = co_await tr.getRange(firstGreaterOrEqual(doubleToTestKey(0.0, self->keyPrefix)),
+					                                        firstGreaterOrEqual(doubleToTestKey(1.0, self->keyPrefix)),
+					                                        self->nodeCount + 1);
 					ok = self->cycleCheckData(data, v) && ok;
 					break;
 				} catch (Error& e) {
-					retryCount++;
-					TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "CycleCheckError").error(e);
-					if (g_network->isSimulated() && retryCount > 50) {
-						CODE_PROBE(true, "Cycle check enable speedUpSimulation because too many transaction_too_old()");
-						// try to make the read window back to normal size (5 * version_per_sec)
-						g_simulator->speedUpSimulation = true;
-					}
-					wait(tr.onError(e));
+					err = e;
+				}
+				retryCount++;
+				TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "CycleCheckError").error(err);
+				if (g_network->isSimulated() && retryCount > 50) {
+					CODE_PROBE(true, "Cycle check enable speedUpSimulation because too many transaction_too_old()");
+					// try to make the read window back to normal size (5 * version_per_sec)
+					g_simulator->speedUpSimulation = true;
 				}
+				co_await tr.onError(err);
 			}
 		}
 		if (!self->unseedCheck) {
 			ASSERT(noUnseed);
 		}
-		return ok;
+		co_return ok;
 	}
 };
 
diff --git a/fdbserver/workloads/DDBalance.actor.cpp b/fdbserver/workloads/DDBalance.cpp
similarity index 72%
rename from fdbserver/workloads/DDBalance.actor.cpp
rename to fdbserver/workloads/DDBalance.cpp
index fee8b2696c8..88b5038c7e4 100644
--- a/fdbserver/workloads/DDBalance.actor.cpp
+++ b/fdbserver/workloads/DDBalance.cpp
@@ -1,5 +1,5 @@
 /*
- * DDBalance.actor.cpp
+ * DDBalance.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,9 +20,9 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/WorkloadUtils.h"
 
 struct DDBalanceWorkload : TestWorkload {
 	static constexpr auto NAME = "DDBalance";
@@ -35,7 +35,7 @@ struct DDBalanceWorkload : TestWorkload {
 	PerfIntCounter bin_shifts, operations, retries;
 	DDSketch<double> latencies;
 
-	DDBalanceWorkload(WorkloadContext const& wcx)
+	explicit DDBalanceWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), bin_shifts("Bin_Shifts"), operations("Operations"), retries("Retries"), latencies() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		binCount = getOption(options, "binCount"_sr, 1000);
@@ -56,13 +56,10 @@ struct DDBalanceWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return ddbalanceSetup(cx, this); }
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
-	ACTOR Future<Void> _start(Database cx, DDBalanceWorkload* self) {
-		for (int c = 0; c < self->moversPerClient; c++)
-			self->clients.push_back(timeout(self->ddBalanceMover(cx, self, c), self->testDuration, Void()));
-		wait(waitForAll(self->clients));
-		return Void();
+	Future<Void> start(Database const& cx) override {
+		for (int c = 0; c < moversPerClient; c++)
+			clients.push_back(timeout(ddBalanceMover(cx, this, c), testDuration, Void()));
+		co_await waitForAll(clients);
 	}
 
 	Future<bool> check(Database const& cx) override {
@@ -92,16 +89,16 @@ struct DDBalanceWorkload : TestWorkload {
 
 	Value value(int n) { return doubleToTestKey(n); }
 
-	ACTOR Future<Void> setKeyIfNotPresent(Transaction* tr, Key key, Value val) {
-		Optional<Value> f = wait(tr->get(key));
+	Future<Void> setKeyIfNotPresent(Transaction* tr, Key key, Value val) {
+		Optional<Value> f = co_await tr->get(key);
 		if (!f.present())
 			tr->set(key, val);
-		return Void();
 	}
 
-	ACTOR Future<Void> ddbalanceSetupRange(Database cx, DDBalanceWorkload* self, int begin, int end) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> ddbalanceSetupRange(Database cx, DDBalanceWorkload* self, int begin, int end) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				std::vector<Future<Void>> setActors;
 				for (int n = begin; n < end; n++) {
@@ -110,19 +107,19 @@ struct DDBalanceWorkload : TestWorkload {
 					setActors.push_back(self->setKeyIfNotPresent(
 					    &tr, self->key(self->currentbin, objectnum, moverid, self->clientId), self->value(objectnum)));
 				}
-				wait(waitForAll(setActors));
-				wait(tr.commit());
+				co_await waitForAll(setActors);
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> ddbalanceSetup(Database cx, DDBalanceWorkload* self) {
-		state int i;
-		state std::vector<int> order;
+	Future<Void> ddbalanceSetup(Database cx, DDBalanceWorkload* self) {
+		int i{ 0 };
+		std::vector<int> order;
 
 		for (int o = 0; o <= self->nodesPerActor * self->actorsPerClient / 10; o++)
 			order.push_back(o * 10);
@@ -134,14 +131,12 @@ struct DDBalanceWorkload : TestWorkload {
 				fs.push_back(self->ddbalanceSetupRange(cx, self, order[i], order[i] + 10));
 				i++;
 			}
-			wait(waitForAll(fs));
+			co_await waitForAll(fs);
 		}
 
 		if (self->warmingDelay > 0) {
-			wait(timeout(databaseWarmer(cx), self->warmingDelay, Void()));
+			co_await timeout(databaseWarmer(cx), self->warmingDelay, Void());
 		}
-
-		return Void();
 	}
 
 	bool shouldRecord(double clientBegin) {
@@ -150,35 +145,36 @@ struct DDBalanceWorkload : TestWorkload {
 		       (n > (clientBegin + testDuration * 0.125) && n < (clientBegin + testDuration * 0.875));
 	}
 
-	ACTOR Future<Void> ddBalanceWorker(Database cx,
-	                                   DDBalanceWorkload* self,
-	                                   int moverId,
-	                                   int sourceBin,
-	                                   int destinationBin,
-	                                   int begin,
-	                                   int end,
-	                                   double clientBegin,
-	                                   double* lastTime,
-	                                   double delay) {
-		state int i;
-		state int j;
-		state int moves;
-		state int maxMovedAmount = 0;
+	Future<Void> ddBalanceWorker(Database cx,
+	                             DDBalanceWorkload* self,
+	                             int moverId,
+	                             int sourceBin,
+	                             int destinationBin,
+	                             int begin,
+	                             int end,
+	                             double clientBegin,
+	                             double* lastTime,
+	                             double delay) {
+		int i{ 0 };
+		int j{ 0 };
+		int moves{ 0 };
+		int maxMovedAmount = 0;
 		for (i = begin; i < end;) {
-			wait(poisson(lastTime, delay));
-			state double tstart = now();
-			state Transaction tr(cx);
-			loop {
-				state int startvalue = i;
+			co_await poisson(lastTime, delay);
+			double tstart = now();
+			Transaction tr(cx);
+			while (true) {
+				int startvalue = i;
 				moves = 0;
+				Error err;
 				try {
 					for (j = 0; i < end && j < self->writesPerTransaction; j++) {
-						state Key myKey = self->key(sourceBin, i, moverId, self->clientId);
-						state Key nextKey = self->key(destinationBin, i, moverId, self->clientId);
+						Key myKey = self->key(sourceBin, i, moverId, self->clientId);
+						Key nextKey = self->key(destinationBin, i, moverId, self->clientId);
 						moves++;
 						i++;
 
-						Optional<Value> f = wait(tr.get(myKey));
+						Optional<Value> f = co_await tr.get(myKey);
 						if (f.present()) {
 							maxMovedAmount++;
 							tr.set(nextKey, f.get());
@@ -191,14 +187,15 @@ struct DDBalanceWorkload : TestWorkload {
 							    .detail("NextBin", destinationBin);
 						}
 					}
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
-					if (self->shouldRecord(clientBegin))
-						++self->retries;
-					i = startvalue;
+					err = e;
 				}
+				co_await tr.onError(err);
+				if (self->shouldRecord(clientBegin))
+					++self->retries;
+				i = startvalue;
 			}
 
 			tr = Transaction();
@@ -219,19 +216,17 @@ struct DDBalanceWorkload : TestWorkload {
 			    .detail("NextBin", destinationBin);
 			ASSERT(false);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> ddBalanceMover(Database cx, DDBalanceWorkload* self, int moverId) {
-		state int currentBin = self->currentbin;
-		state int nextBin = 0;
-		state int key_space_drift = 0;
+	Future<Void> ddBalanceMover(Database cx, DDBalanceWorkload* self, int moverId) {
+		int currentBin = self->currentbin;
+		int nextBin = 0;
+		int key_space_drift = 0;
 
-		state double clientBegin = now();
-		state double lastTime = now();
+		double clientBegin = now();
+		double lastTime = now();
 
-		loop {
+		while (true) {
 			nextBin = deterministicRandom()->randomInt(key_space_drift, self->binCount + key_space_drift);
 			while (nextBin == currentBin)
 				nextBin = deterministicRandom()->randomInt(key_space_drift, self->binCount + key_space_drift);
@@ -249,7 +244,7 @@ struct DDBalanceWorkload : TestWorkload {
 				                                   clientBegin,
 				                                   &lastTime,
 				                                   1.0 / self->transactionsPerSecond));
-			wait(waitForAll(fs));
+			co_await waitForAll(fs);
 
 			currentBin = nextBin;
 			key_space_drift += self->keySpaceDriftFactor;
diff --git a/fdbserver/workloads/DDMetrics.actor.cpp b/fdbserver/workloads/DDMetrics.cpp
similarity index 65%
rename from fdbserver/workloads/DDMetrics.actor.cpp
rename to fdbserver/workloads/DDMetrics.cpp
index c817b92ffc8..710db797423 100644
--- a/fdbserver/workloads/DDMetrics.actor.cpp
+++ b/fdbserver/workloads/DDMetrics.cpp
@@ -1,5 +1,5 @@
 /*
- * DDMetrics.actor.cpp
+ * DDMetrics.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,51 +19,48 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/Status.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
 
 struct DDMetricsWorkload : TestWorkload {
 	static constexpr auto NAME = "DDMetrics";
 	double startDelay, ddDone;
 
-	DDMetricsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddDone(0.0) {
+	explicit DDMetricsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddDone(0.0) {
 		startDelay = getOption(options, "beginPoll"_sr, 10.0);
 	}
 
-	ACTOR Future<int> getHighPriorityRelocationsInFlight(Database cx, DDMetricsWorkload* self) {
-		WorkerInterface masterWorker = wait(getMasterWorker(cx, self->dbInfo));
+	Future<int> getHighPriorityRelocationsInFlight(Database cx, DDMetricsWorkload* self) {
+		WorkerInterface masterWorker = co_await getMasterWorker(cx, self->dbInfo);
 
 		TraceEvent("GetHighPriorityReliocationsInFlight").detail("Stage", "ContactingMaster");
 		TraceEventFields md =
-		    wait(timeoutError(masterWorker.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0));
+		    co_await timeoutError(masterWorker.eventLogRequest.getReply(EventLogRequest("MovingData"_sr)), 1.0);
 		int relocations;
 		sscanf(md.getValue("UnhealthyRelocations").c_str(), "%d", &relocations);
-		return relocations;
+		co_return relocations;
 	}
 
-	ACTOR Future<Void> work(Database cx, DDMetricsWorkload* self) {
+	Future<Void> work(Database cx, DDMetricsWorkload* self) {
 		try {
 			TraceEvent("DDMetricsWaiting").detail("StartDelay", self->startDelay);
-			wait(delay(self->startDelay));
+			co_await delay(self->startDelay);
 			TraceEvent("DDMetricsStarting").log();
-			state double startTime = now();
-			loop {
-				wait(delay(2.5));
-				int dif = wait(self->getHighPriorityRelocationsInFlight(cx, self));
+			double startTime = now();
+			while (true) {
+				co_await delay(2.5);
+				int dif = co_await self->getHighPriorityRelocationsInFlight(cx, self);
 				TraceEvent("DDMetricsCheck").detail("DIF", dif);
 				if (dif == 0) {
 					self->ddDone = now() - startTime;
-					return Void();
+					co_return;
 				}
 			}
 		} catch (Error& e) {
 			TraceEvent("DDMetricsError").error(e);
 		}
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override { return clientId == 0 ? work(cx, this) : Void(); }
diff --git a/fdbserver/workloads/DDMetricsExclude.actor.cpp b/fdbserver/workloads/DDMetricsExclude.cpp
similarity index 69%
rename from fdbserver/workloads/DDMetricsExclude.actor.cpp
rename to fdbserver/workloads/DDMetricsExclude.cpp
index 02a0ce1441a..f439c312019 100644
--- a/fdbserver/workloads/DDMetricsExclude.actor.cpp
+++ b/fdbserver/workloads/DDMetricsExclude.cpp
@@ -1,5 +1,5 @@
 /*
- * DDMetricsExclude.actor.cpp
+ * DDMetricsExclude.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "fdbclient/StatusClient.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct DDMetricsExcludeWorkload : TestWorkload {
 	static constexpr auto NAME = "DDMetricsExclude";
@@ -36,7 +35,7 @@ struct DDMetricsExcludeWorkload : TestWorkload {
 	double peakInFlight;
 	double movingDataPerSec;
 
-	DDMetricsExcludeWorkload(WorkloadContext const& wcx)
+	explicit DDMetricsExcludeWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), ddDone(0.0), peakMovingData(0.0), peakInQueue(0.0), peakInFlight(0.0),
 	    movingDataPerSec(0.0) {
 		excludeIp = getOption(options, "excludeIp"_sr, Value("127.0.0.1"_sr));
@@ -47,9 +46,9 @@ struct DDMetricsExcludeWorkload : TestWorkload {
 		return Standalone<StringRef>(format("Value/%080d", deterministicRandom()->randomInt(0, 10e6)));
 	}
 
-	ACTOR static Future<double> getMovingDataAmount(Database cx, DDMetricsExcludeWorkload* self) {
+	Future<double> getMovingDataAmount(Database cx) {
 		try {
-			StatusObject statusObj = wait(StatusClient::statusFetcher(cx));
+			StatusObject statusObj = co_await StatusClient::statusFetcher(cx);
 			StatusObjectReader statusObjCluster;
 			((StatusObjectReader)statusObj).get("cluster", statusObjCluster);
 			StatusObjectReader statusObjData;
@@ -58,42 +57,40 @@ struct DDMetricsExcludeWorkload : TestWorkload {
 				StatusObjectReader movingData = statusObjData.last();
 				double dataInQueue, dataInFlight;
 				if (movingData.get("in_queue_bytes", dataInQueue) && movingData.get("in_flight_bytes", dataInFlight)) {
-					self->peakInQueue = std::max(self->peakInQueue, dataInQueue);
-					self->peakInFlight = std::max(self->peakInFlight, dataInFlight);
-					return dataInQueue + dataInFlight;
+					peakInQueue = std::max(peakInQueue, dataInQueue);
+					peakInFlight = std::max(peakInFlight, dataInFlight);
+					co_return dataInQueue + dataInFlight;
 				}
 			}
 		} catch (Error& e) {
 			TraceEvent("DDMetricsExcludeGetMovingDataError").error(e);
 			throw;
 		}
-		return -1.0;
+		co_return -1.0;
 	}
 
-	ACTOR static Future<Void> _start(Database cx, DDMetricsExcludeWorkload* self) {
+	Future<Void> start(Database const& cx) override {
 		try {
-			state std::vector<AddressExclusion> excluded;
-			excluded.push_back(AddressExclusion(IPAddress::parse(self->excludeIp.toString()).get(), self->excludePort));
-			wait(excludeServers(cx, excluded));
-			state double startTime = now();
-			loop {
-				wait(delay(2.5));
-				double movingData = wait(self->getMovingDataAmount(cx, self));
-				self->peakMovingData = std::max(self->peakMovingData, movingData);
+			std::vector<AddressExclusion> excluded;
+			excluded.push_back(AddressExclusion(IPAddress::parse(excludeIp.toString()).get(), excludePort));
+			co_await excludeServers(cx, excluded);
+			double startTime = now();
+			while (true) {
+				co_await delay(2.5);
+				double movingData = co_await getMovingDataAmount(cx);
+				peakMovingData = std::max(peakMovingData, movingData);
 				TraceEvent("DDMetricsExcludeCheck").detail("MovingData", movingData);
 				if (movingData == 0.0) {
-					self->ddDone = now() - startTime;
-					return Void();
+					ddDone = now() - startTime;
+					co_return;
 				}
 			}
 		} catch (Error& e) {
 			TraceEvent("DDMetricsExcludeError").error(e);
 		}
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	Future<bool> check(Database const& cx) override {
 		movingDataPerSec = peakMovingData / ddDone;
 		return true;
diff --git a/fdbserver/workloads/DataDistributionMetrics.actor.cpp b/fdbserver/workloads/DataDistributionMetrics.cpp
similarity index 61%
rename from fdbserver/workloads/DataDistributionMetrics.actor.cpp
rename to fdbserver/workloads/DataDistributionMetrics.cpp
index a8c7b0b5649..4affe9b216d 100644
--- a/fdbserver/workloads/DataDistributionMetrics.actor.cpp
+++ b/fdbserver/workloads/DataDistributionMetrics.cpp
@@ -1,5 +1,5 @@
 /*
- * DataDistributionMetrics.actor.cpp
+ * DataDistributionMetrics.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,10 @@
 
 #include <boost/lexical_cast.hpp>
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/Schemas.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last include
 
 struct DataDistributionMetricsWorkload : KVWorkload {
@@ -36,7 +36,7 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 	PerfIntCounter commits, errors;
 	double delayPerLoop;
 
-	DataDistributionMetricsWorkload(WorkloadContext const& wcx)
+	explicit DataDistributionMetricsWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), numShards(0), avgBytes(0), commits("Commits"), errors("Errors") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		// transaction time out duration(ms)
@@ -54,47 +54,49 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 
 	Key keyForIndex(int n) { return doubleToTestKey((double)n / nodeCount, keyPrefix); }
 
-	ACTOR static Future<Void> ddRWClient(Database cx, DataDistributionMetricsWorkload* self) {
-		loop {
-			state ReadYourWritesTransaction tr(cx);
-			state int i;
+	Future<Void> ddRWClient(Database cx) {
+		while (true) {
+			ReadYourWritesTransaction tr(cx);
+			int i{ 0 };
+			Error err;
 			try {
-				for (i = 0; i < self->readPerTx; ++i)
-					wait(success(
-					    tr.get(self->keyForIndex(deterministicRandom()->randomInt(0, self->nodeCount))))); // read
-				for (i = 0; i < self->writePerTx; ++i)
-					tr.set(self->keyForIndex(deterministicRandom()->randomInt(0, self->nodeCount)),
+				for (i = 0; i < readPerTx; ++i)
+					co_await tr.get(keyForIndex(deterministicRandom()->randomInt(0, nodeCount))); // read
+				for (i = 0; i < writePerTx; ++i)
+					tr.set(keyForIndex(deterministicRandom()->randomInt(0, nodeCount)),
 					       getRandomValue()); // write
-				wait(tr.commit());
-				++self->commits;
+				co_await tr.commit();
+				++commits;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR Future<Void> resultConsistencyCheckClient(Database cx, DataDistributionMetricsWorkload* self) {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		loop {
+	Future<Void> resultConsistencyCheckClient(Database cx, DataDistributionMetricsWorkload* self) {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		while (true) {
 			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
-			tr->setOption(FDBTransactionOptions::TIMEOUT,
-			              StringRef((uint8_t*)&self->transactionTimeLimit, sizeof(int64_t)));
+			tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t*)&transactionTimeLimit, sizeof(int64_t)));
+			Error err;
 			try {
-				wait(delay(self->delayPerLoop));
+				co_await delay(self->delayPerLoop);
 				int startIndex = deterministicRandom()->randomInt(0, self->nodeCount - 1);
 				int endIndex = deterministicRandom()->randomInt(startIndex + 1, self->nodeCount);
-				state Key startKey = self->keyForIndex(startIndex);
-				state Key endKey = self->keyForIndex(endIndex);
-				// Find the last key <= startKey and use as the begin of the range. Since "Key()" is always the starting
-				// point, this key selector will never do cross_module_range_read. In addition, the first key in the
-				// result will be the last one <= startKey (Condition #1)
-				state KeySelector begin =
-				    KeySelectorRef(startKey.withPrefix(ddStatsRange.begin, startKey.arena()), true, 0);
-				// Find the last key less than endKey, move forward 2 keys, and use this key as the (exclusive) end of
-				// the range. If we didn't read through the end of the range, then the second last key
-				// in the result will be the last key less than endKey. (Condition #2)
-				state KeySelector end = KeySelectorRef(endKey.withPrefix(ddStatsRange.begin, endKey.arena()), false, 2);
-				RangeResult result = wait(tr->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
+				Key startKey = self->keyForIndex(startIndex);
+				Key endKey = self->keyForIndex(endIndex);
+				// Find the last key <= startKey and use as the begin of the range. Since "Key()" is always the
+				// starting point, this key selector will never do cross_module_range_read. In addition, the first
+				// key in the result will be the last one <= startKey (Condition #1)
+				KeySelector begin = KeySelectorRef(startKey.withPrefix(ddStatsRange.begin, startKey.arena()), true, 0);
+				// Find the last key less than endKey, move forward 2 keys, and use this key as the (exclusive) end
+				// of the range. If we didn't read through the end of the range, then the second last key in the
+				// result will be the last key less than endKey. (Condition #2)
+				KeySelector end = KeySelectorRef(endKey.withPrefix(ddStatsRange.begin, endKey.arena()), false, 2);
+				RangeResult result = co_await tr->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY));
 				// Condition #1 and #2 can be broken if multiple rpc calls happened in one getRange
 				if (result.size() > 1) {
 					if (result[0].key > begin.getKey() || result[1].key <= begin.getKey()) {
@@ -129,37 +131,41 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 					// 	    .detail("EndKeySelector", end);
 				}
 			} catch (Error& e) {
-				// Ignore timed_out error and cross_module_read, the end key selector may read through the end
-				if (e.code() == error_code_timed_out || e.code() == error_code_transaction_timed_out) {
-					tr->reset();
-					continue;
-				}
-				wait(tr->onError(e));
+				err = e;
+			}
+			// Ignore timed_out error and cross_module_read, the end key selector may read through the end
+			if (err.isValid() &&
+			    (err.code() == error_code_timed_out || err.code() == error_code_transaction_timed_out)) {
+				tr->reset();
+				continue;
+			}
+			if (err.isValid()) {
+				co_await tr->onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<bool> _check(Database cx, DataDistributionMetricsWorkload* self) {
-		if (self->errors.getValue() > 0) {
+	Future<bool> _check(Database cx) {
+		if (errors.getValue() > 0) {
 			TraceEvent(SevError, "TestFailure").detail("Reason", "GetRange Results Inconsistent");
-			return false;
+			co_return false;
 		}
 		// TODO : find why this not work
 		// wait(quietDatabase(cx, self->dbInfo, "PopulateTPCC"));
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state int i;
-		state int retries = 0;
-		loop {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		int i{ 0 };
+		int retries = 0;
+		while (true) {
 			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
-			tr->setOption(FDBTransactionOptions::TIMEOUT,
-			              StringRef((uint8_t*)&self->transactionTimeLimit, sizeof(int64_t)));
+			tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t*)&transactionTimeLimit, sizeof(int64_t)));
+			Error err;
 			try {
-				state RangeResult result = wait(tr->getRange(ddStatsRange, CLIENT_KNOBS->TOO_MANY));
+				RangeResult result = co_await tr->getRange(ddStatsRange, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!result.more);
-				self->numShards = result.size();
+				numShards = result.size();
 				// There's no guarantee that #shards <= CLIENT_KNOBS->SHARD_COUNT_LIMIT all the time
-				ASSERT(self->numShards >= 1);
-				state int64_t totalBytes = 0;
+				ASSERT(numShards >= 1);
+				int64_t totalBytes = 0;
 				auto schema = readJSONStrictly(JSONSchemas::dataDistributionStatsSchema.toString()).get_obj();
 				for (i = 0; i < result.size(); ++i) {
 					ASSERT(result[i].key.startsWith(ddStatsRange.begin));
@@ -170,42 +176,41 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 						TraceEvent(SevError, "DataDistributionStatsSchemaValidationFailed")
 						    .detail("ErrorStr", errorStr.c_str())
 						    .detail("JSON", json_spirit::write_string(json_spirit::mValue(result[i].value.toString())));
-						return false;
+						co_return false;
 					}
 					totalBytes += valueObj["shard_bytes"].get_int64();
 				}
-				self->avgBytes = totalBytes / self->numShards;
+				avgBytes = totalBytes / numShards;
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_timed_out || e.code() == error_code_transaction_timed_out) {
-					tr->reset();
-					// The RPC call may in some corner cases get no response
-					if (++retries > 10)
-						break;
-					continue;
-				}
-				wait(tr->onError(e));
+				err = e;
+			}
+			if (err.code() == error_code_timed_out || err.code() == error_code_transaction_timed_out) {
+				tr->reset();
+				// The RPC call may in some corner cases get no response
+				if (++retries > 10)
+					break;
+				continue;
 			}
+			co_await tr->onError(err);
 		}
-		return true;
+		co_return true;
 	}
 
-	ACTOR Future<Void> _start(Database cx, DataDistributionMetricsWorkload* self) {
+	Future<Void> start(Database const& cx) override {
 		std::vector<Future<Void>> clients;
-		clients.push_back(self->resultConsistencyCheckClient(cx, self));
-		for (int i = 0; i < self->actorCount; ++i)
-			clients.push_back(self->ddRWClient(cx, self));
-		wait(timeout(waitForAll(clients), self->testDuration, Void()));
-		wait(delay(5.0));
-		return Void();
+		clients.push_back(resultConsistencyCheckClient(cx, this));
+		for (int i = 0; i < actorCount; ++i)
+			clients.push_back(ddRWClient(cx));
+		co_await timeout(waitForAll(clients), testDuration, Void());
+		co_await delay(5.0);
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 
 	Future<bool> check(Database const& cx) override {
 		if (clientId == 0)
-			return _check(cx, this);
+			return _check(cx);
 		return true;
 	}
 
diff --git a/fdbserver/workloads/DataLossRecovery.actor.cpp b/fdbserver/workloads/DataLossRecovery.cpp
similarity index 71%
rename from fdbserver/workloads/DataLossRecovery.actor.cpp
rename to fdbserver/workloads/DataLossRecovery.cpp
index 2f24b48e532..d3de9a72f90 100644
--- a/fdbserver/workloads/DataLossRecovery.actor.cpp
+++ b/fdbserver/workloads/DataLossRecovery.cpp
@@ -1,5 +1,5 @@
 /*
- * DataLossRecovery.actor.cpp
+ * DataLossRecovery.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,17 +22,16 @@
 #include <limits>
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/Knobs.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 std::string printValue(const ErrorOr<Optional<Value>>& value) {
@@ -51,7 +50,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
 	bool pass;
 	NetworkAddress addr;
 
-	DataLossRecoveryWorkload(WorkloadContext const& wcx)
+	explicit DataLossRecoveryWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), startMoveKeysParallelismLock(5), finishMoveKeysParallelismLock(5), enabled(!clientId),
 	    pass(true) {}
 
@@ -72,141 +71,140 @@ struct DataLossRecoveryWorkload : TestWorkload {
 		if (!enabled) {
 			return Void();
 		}
-		return _start(this, cx);
+		return _start(cx);
 	}
 
-	ACTOR Future<Void> _start(DataLossRecoveryWorkload* self, Database cx) {
-		state Key key = "TestKey"_sr;
-		state Key endKey = "TestKey0"_sr;
-		state Value oldValue = "TestValue"_sr;
-		state Value newValue = "TestNewValue"_sr;
+	Future<Void> _start(Database cx) {
+		Key key = "TestKey"_sr;
+		Key endKey = "TestKey0"_sr;
+		Value oldValue = "TestValue"_sr;
+		Value newValue = "TestNewValue"_sr;
 
 		TraceEvent("DataLossRecovery").detail("Phase", "Starting");
-		wait(self->writeAndVerify(self, cx, key, oldValue));
+		co_await writeAndVerify(this, cx, key, oldValue);
 
 		TraceEvent("DataLossRecovery").detail("Phase", "InitialWrites");
 		// Move [key, endKey) to team: {address}.
-		state NetworkAddress address = wait(self->disableDDAndMoveShard(self, cx, KeyRangeRef(key, endKey)));
+		NetworkAddress address = co_await disableDDAndMoveShard(this, cx, KeyRangeRef(key, endKey));
 		TraceEvent("DataLossRecovery").detail("Phase", "Moved");
-		wait(self->readAndVerify(self, cx, key, oldValue));
+		co_await readAndVerify(this, cx, key, oldValue);
 		TraceEvent("DataLossRecovery").detail("Phase", "ReadAfterMove");
 
 		// Kill team {address}, and expect read to timeout.
-		self->killProcess(self, address);
+		killProcess(this, address);
 		TraceEvent("DataLossRecovery").detail("Phase", "KilledProcess");
-		wait(self->readAndVerify(self, cx, key, timed_out()));
+		co_await readAndVerify(this, cx, key, timed_out());
 		TraceEvent("DataLossRecovery").detail("Phase", "VerifiedReadTimeout");
 
 		// Reenable DD and exclude address as fail, so that [key, endKey) will be dropped and moved to a new team.
 		// Expect read to return 'value not found'.
-		wait(success(setDDMode(cx, 1)));
-		wait(self->exclude(cx, address));
+		co_await setDDMode(cx, 1);
+		co_await exclude(cx, address);
 		TraceEvent("DataLossRecovery").detail("Phase", "Excluded");
-		wait(self->readAndVerify(self, cx, key, Optional<Value>()));
+		co_await readAndVerify(this, cx, key, Optional<Value>());
 		TraceEvent("DataLossRecovery").detail("Phase", "VerifiedDataDropped");
 
 		// Write will scceed.
-		wait(self->writeAndVerify(self, cx, key, newValue));
-
-		return Void();
+		co_await writeAndVerify(this, cx, key, newValue);
 	}
 
-	ACTOR Future<Void> readAndVerify(DataLossRecoveryWorkload* self,
-	                                 Database cx,
-	                                 Key key,
-	                                 ErrorOr<Optional<Value>> expectedValue) {
-		state Transaction tr(cx);
+	Future<Void> readAndVerify(DataLossRecoveryWorkload* self,
+	                           Database cx,
+	                           Key key,
+	                           ErrorOr<Optional<Value>> expectedValue) {
+		Transaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// add timeout to read so test fails faster if something goes wrong
-				state Optional<Value> res = wait(timeoutError(tr.get(key), 90.0));
+				Optional<Value> res = co_await timeoutError(tr.get(key), 90.0);
 				const bool equal = !expectedValue.isError() && res == expectedValue.get();
 				if (!equal) {
 					self->validationFailed(expectedValue, ErrorOr<Optional<Value>>(res));
 				}
 				break;
 			} catch (Error& e) {
-				if (expectedValue.isError() && expectedValue.getError().code() == e.code()) {
-					break;
-				}
-				wait(tr.onError(e));
+				err = e;
 			}
+			if (expectedValue.isError() && expectedValue.getError().code() == err.code()) {
+				break;
+			}
+			co_await tr.onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> writeAndVerify(DataLossRecoveryWorkload* self, Database cx, Key key, Optional<Value> value) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> writeAndVerify(DataLossRecoveryWorkload* self, Database cx, Key key, Optional<Value> value) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				if (value.present()) {
 					tr.set(key, value.get());
 				} else {
 					tr.clear(key);
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
-		wait(self->readAndVerify(self, cx, key, value));
-
-		return Void();
+		co_await self->readAndVerify(self, cx, key, value);
 	}
 
-	ACTOR Future<Void> exclude(Database cx, NetworkAddress addr) {
-		state Transaction tr(cx);
-		state std::vector<AddressExclusion> servers;
+	Future<Void> exclude(Database cx, NetworkAddress addr) {
+		Transaction tr(cx);
+		std::vector<AddressExclusion> servers;
 		servers.push_back(AddressExclusion(addr.ip, addr.port));
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(excludeServers(&tr, servers, true));
-				wait(tr.commit());
+				co_await excludeServers(&tr, servers, true);
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		// Wait until all data are moved out of servers.
-		std::set<NetworkAddress> inProgress = wait(checkForExcludingServers(cx, servers, true));
+		std::set<NetworkAddress> inProgress = co_await checkForExcludingServers(cx, servers, true);
 		ASSERT(inProgress.empty());
 
 		TraceEvent("ExcludedFailedServer").detail("Address", addr.toString());
-		return Void();
 	}
 
 	// Move keys to a random selected team consisting of a single SS, after disabling DD, so that keys won't be
 	// kept in the new team until DD is enabled.
 	// Returns the address of the single SS of the new team.
-	ACTOR Future<NetworkAddress> disableDDAndMoveShard(DataLossRecoveryWorkload* self, Database cx, KeyRange keys) {
+	Future<NetworkAddress> disableDDAndMoveShard(DataLossRecoveryWorkload* self, Database cx, KeyRange keys) {
 		// Disable DD to avoid DD undoing of our move.
-		wait(success(setDDMode(cx, 0)));
+		co_await setDDMode(cx, 0);
 		TraceEvent("DataLossRecovery").detail("Phase", "DisabledDD");
-		state NetworkAddress addr;
+		NetworkAddress addr;
 
 		// Pick a random SS as the dest, keys will reside on a single server after the move.
-		state std::vector<UID> dest;
+		std::vector<UID> dest;
 		while (dest.empty()) {
-			state std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
+			std::vector<StorageServerInterface> interfs = co_await getStorageServers(cx);
 			if (!interfs.empty()) {
-				state StorageServerInterface interf = interfs[deterministicRandom()->randomInt(0, interfs.size())];
-				if (!g_simulator->protectedAddresses.contains(interf.address())) {
+				StorageServerInterface interf = interfs[deterministicRandom()->randomInt(0, interfs.size())];
+				if (!g_simulator->isProtectedAddress(interf.address())) {
 					// We need to avoid selecting a storage server that is already dead at this point, otherwise
 					// the test will hang. This is achieved by sending a GetStorageMetrics RPC. This is a necessary
 					// check for this test because DD has been disabled and the proper mechanism that removes bad
 					// storage servers are not taking place in the scope of this function.
-					state Future<ErrorOr<GetStorageMetricsReply>> metricsRequest = interf.getStorageMetrics.tryGetReply(
+					Future<ErrorOr<GetStorageMetricsReply>> metricsRequest = interf.getStorageMetrics.tryGetReply(
 					    GetStorageMetricsRequest(), TaskPriority::DataDistributionLaunch);
 
-					state ErrorOr<GetStorageMetricsReply> rep = wait(metricsRequest);
+					ErrorOr<GetStorageMetricsReply> rep = co_await metricsRequest;
 					if (rep.isError()) {
 						// Delay 1s to avoid tight spin loop in case all options fail
-						wait(delay(1.0));
+						co_await delay(1.0);
 						continue;
 					}
 					dest.push_back(interf.uniqueID);
@@ -215,18 +213,19 @@ struct DataLossRecoveryWorkload : TestWorkload {
 			}
 		}
 
-		state UID owner = deterministicRandom()->randomUniqueID();
-		state DDEnabledState ddEnabledState;
+		UID owner = deterministicRandom()->randomUniqueID();
+		DDEnabledState ddEnabledState;
 
-		state Transaction tr(cx);
+		Transaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				BinaryWriter wrMyOwner(Unversioned());
 				wrMyOwner << owner;
 				tr.set(moveKeysLockOwnerKey, wrMyOwner.toValue());
-				wait(tr.commit());
+				co_await tr.commit();
 
 				MoveKeysLock moveKeysLock;
 				moveKeysLock.myOwner = owner;
@@ -275,35 +274,40 @@ struct DataLossRecoveryWorkload : TestWorkload {
 					                                          CancelConflictingDataMoves::True,
 					                                          Optional<BulkLoadTaskState>());
 				}
-				wait(moveKeys(cx, *params));
+				co_await moveKeys(cx, *params);
 				break;
 			} catch (Error& e) {
-				TraceEvent("DataLossRecovery").error(e).detail("Phase", "MoveRangeError");
-				if (e.code() == error_code_movekeys_conflict) {
-					// Conflict on moveKeysLocks with the current running DD is expected, just retry.
-					tr.reset();
-				} else {
-					wait(tr.onError(e));
-				}
+				err = e;
+			}
+			TraceEvent("DataLossRecovery").error(err).detail("Phase", "MoveRangeError");
+			if (err.code() == error_code_movekeys_conflict ||
+			    err.code() == error_code_finish_move_keys_too_many_retries) {
+				// Conflicts on moveKeysLocks with the current running DD and finishMoveKeys retry exhaustion are both
+				// expected transient move outcomes, so retry the move from a fresh transaction.
+				tr.reset();
+			} else {
+				co_await tr.onError(err);
 			}
 		}
 
 		TraceEvent("TestKeyMoved").detail("NewTeam", describe(dest)).detail("Address", addr.toString());
 
-		state Transaction validateTr(cx);
-		loop {
+		Transaction validateTr(cx);
+		while (true) {
+			Error err;
 			try {
-				Standalone<VectorRef<const char*>> addresses = wait(validateTr.getAddressesForKey(keys.begin));
+				Standalone<VectorRef<const char*>> addresses = co_await validateTr.getAddressesForKey(keys.begin);
 				// The move function is not what we are testing here, crash the test if the move fails.
 				ASSERT(addresses.size() == 1);
 				ASSERT(std::string(addresses[0]) == addr.toString());
 				break;
 			} catch (Error& e) {
-				wait(validateTr.onError(e));
+				err = e;
 			}
+			co_await validateTr.onError(err);
 		}
 
-		return addr;
+		co_return addr;
 	}
 
 	void killProcess(DataLossRecoveryWorkload* self, const NetworkAddress& addr) {
diff --git a/fdbserver/workloads/DcLag.actor.cpp b/fdbserver/workloads/DcLag.cpp
similarity index 79%
rename from fdbserver/workloads/DcLag.actor.cpp
rename to fdbserver/workloads/DcLag.cpp
index a300c1e651a..6368d5b75ef 100644
--- a/fdbserver/workloads/DcLag.actor.cpp
+++ b/fdbserver/workloads/DcLag.cpp
@@ -1,5 +1,5 @@
 /*
- * DcLag.actor.cpp
+ * DcLag.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,11 +22,12 @@
 #include "fdbclient/StatusClient.h"
 #include "fdbrpc/Locality.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "flow/CodeProbe.h"
 #include "flow/NetworkAddress.h"
@@ -34,7 +35,7 @@
 #include "flow/Trace.h"
 #include "flow/flow.h"
 #include "flow/network.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 // This workload tests a gray failure scenario: a satellite TLog is have network issue
 // for sending packets to the remote data center's log routers. This will cause these
@@ -49,7 +50,7 @@ struct DcLagWorkload : TestWorkload {
 	double startDelay;
 	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
 
-	DcLagWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit DcLagWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		testDuration = getOption(options, "testDuration"_sr, 1000.0);
 		startDelay = getOption(options, "startDelay"_sr, 10.0);
@@ -75,11 +76,12 @@ struct DcLagWorkload : TestWorkload {
 		std::vector<IPAddress> ips; // all remote process IPs
 		for (const auto& process : g_simulator->getAllProcesses()) {
 			const auto& ip = process->address.ip;
-			if (process->locality.dcId().present() && process->locality.dcId() == g_simulator->remoteDcId) {
+			if (process->locality.dcId().present() &&
+			    process->locality.dcId() == fdbSimulationPolicyState().remoteDcId) {
 				ips.push_back(ip);
 			}
 		}
-		ASSERT(ips.size() > 0);
+		ASSERT(!ips.empty());
 
 		// Find all primary satellite tlogs
 		std::vector<NetworkAddress> logs; // all primary satellite logs
@@ -117,40 +119,40 @@ struct DcLagWorkload : TestWorkload {
 		cloggedPairs.clear();
 	}
 
-	ACTOR static Future<Optional<double>> fetchDatacenterLag(Database cx) {
-		StatusObject result = wait(StatusClient::statusFetcher(cx));
+	static Future<Optional<double>> fetchDatacenterLag(Database cx) {
+		StatusObject result = co_await StatusClient::statusFetcher(cx);
 		StatusObjectReader statusObj(result);
 		StatusObjectReader statusObjCluster;
 		if (!statusObj.get("cluster", statusObjCluster)) {
 			TraceEvent("DcLagNoCluster");
-			return Optional<double>();
+			co_return Optional<double>();
 		}
 
 		StatusObjectReader dcLag;
 		if (!statusObjCluster.get("datacenter_lag", dcLag)) {
 			TraceEvent("DcLagNoLagData");
-			return Optional<double>();
+			co_return Optional<double>();
 		}
 
 		Version versions = 0;
 		double seconds = 0;
 		if (!dcLag.get("versions", versions)) {
 			TraceEvent("DcLagNoVersions");
-			return Optional<double>();
+			co_return Optional<double>();
 		}
 		if (!dcLag.get("seconds", seconds)) {
 			TraceEvent("DcLagNoSeconds");
-			return Optional<double>();
+			co_return Optional<double>();
 		}
 		TraceEvent("DcLag").detail("Versions", versions).detail("Seconds", seconds);
-		return seconds;
+		co_return seconds;
 	}
 
-	ACTOR Future<Void> clogClient(DcLagWorkload* self, Database cx) {
-		wait(delay(self->startDelay));
+	Future<Void> clogClient(DcLagWorkload* self, Database cx) {
+		co_await delay(self->startDelay);
 
 		while (self->dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
-			wait(self->dbInfo->onChange());
+			co_await self->dbInfo->onChange();
 		}
 
 		double startTime = now();
@@ -159,17 +161,20 @@ struct DcLagWorkload : TestWorkload {
 
 		// Clog and wait for recovery to happen
 		if (!self->clogTlog(self->testDuration)) {
-			return Void(); // skip the test if no satellite found
+			co_return; // skip the test if no satellite found
 		}
 
-		state Future<Optional<double>> status = Never();
-		state bool lagged = false;
-		loop choose {
-			when(wait(delay(5.0))) {
+		Future<Optional<double>> status = Never();
+		bool lagged = false;
+		while (true) {
+			auto choice = co_await race(delay(5.0), status);
+			if (choice.index() == 0) {
+
 				// Fetch DC lag every 5s
 				status = fetchDatacenterLag(cx);
-			}
-			when(Optional<double> lag = wait(status)) {
+			} else if (choice.index() == 1) {
+				Optional<double> lag = std::get<1>(std::move(choice));
+
 				if (lag.present() && lag.get() >= SERVER_KNOBS->LOG_ROUTER_PEEK_SWITCH_DC_TIME - 10.0) {
 					// Detect DC Lag happened before Log router switch DC reactions
 					lagged = true;
@@ -178,12 +183,14 @@ struct DcLagWorkload : TestWorkload {
 				if (lagged && lag.present() && lag.get() < 5.0) {
 					TraceEvent("DcLagRecovered");
 					self->unclogAll();
-					return Void();
+					co_return;
 				}
 				status = Never();
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
 };
 
-WorkloadFactory<DcLagWorkload> DcLagWorkloadFactory;
\ No newline at end of file
+WorkloadFactory<DcLagWorkload> DcLagWorkloadFactory;
diff --git a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp b/fdbserver/workloads/DifferentClustersSameRV.cpp
similarity index 51%
rename from fdbserver/workloads/DifferentClustersSameRV.actor.cpp
rename to fdbserver/workloads/DifferentClustersSameRV.cpp
index 53f1b28c698..925d42dd144 100644
--- a/fdbserver/workloads/DifferentClustersSameRV.actor.cpp
+++ b/fdbserver/workloads/DifferentClustersSameRV.cpp
@@ -1,5 +1,5 @@
 /*
- * DifferentClustersSameRV.actor.cpp
+ * DifferentClustersSameRV.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,13 +19,12 @@
  */
 
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ApiVersion.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 void traceError(const char* filename, int line, Error const& e) {
 	TraceEvent("DifferentClustersSameRVWorkload_Error").error(e).detail("File", filename).detail("Line", line);
@@ -44,9 +43,9 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 	Value keyToWatch;
 	bool switchComplete = false;
 
-	DifferentClustersSameRVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		ASSERT(g_simulator->extraDatabases.size() == 1);
-		extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+	explicit DifferentClustersSameRVWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+		extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 		testDuration = getOption(options, "testDuration"_sr, 100.0);
 		switchAfter = getOption(options, "switchAfter"_sr, 50.0);
 		keyToRead = getOption(options, "keyToRead"_sr, "someKey"_sr);
@@ -57,31 +56,32 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 		if (clientId != 0) {
 			return Void();
 		}
-		return _setup(cx, this);
+		return _setup(cx, extraDB);
 	}
 
-	ACTOR static Future<Void> _setup(Database cx, DifferentClustersSameRVWorkload* self) {
-		state Version rv1;
-		state Version rv2;
-		state Version newClusterVersion;
-		state Transaction tr1(cx);
-		state Transaction tr2(self->extraDB);
+	static Future<Void> _setup(Database cx, Database extraDB) {
+		Version rv1{ 0 };
+		Version rv2{ 0 };
+		Version newClusterVersion{ 0 };
+		Transaction tr1(cx);
+		Transaction tr2(extraDB);
 		TraceEvent("DifferentClustersSameRVWorkload");
 
 		// we want to advance the read version of both clusters so that they are roughly the same. This makes the test
 		// more effective (since it's more likely that we can read from both clusters with the same version).
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(store(rv1, tr1.getReadVersion()) && store(rv2, tr2.getReadVersion()));
+				co_await (store(rv1, tr1.getReadVersion()) && store(rv2, tr2.getReadVersion()));
 				break;
 			} catch (Error& e) {
-				wait(tr1.onError(e) && tr2.onError(e));
+				err = e;
 			}
+			co_await (tr1.onError(err) && tr2.onError(err));
 		}
 		newClusterVersion = std::max(rv1, rv2) + 10e6;
-		wait(::advanceVersion(cx, newClusterVersion) && ::advanceVersion(self->extraDB, newClusterVersion));
+		co_await (::advanceVersion(cx, newClusterVersion) && ::advanceVersion(extraDB, newClusterVersion));
 		TraceEvent("DifferentClustersSameRVWorkload_AdvancedVersion").detail("Version", newClusterVersion);
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -90,10 +90,10 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 		}
 		auto switchConnFileDb = Database::createDatabase(cx->getConnectionRecord(), -1);
 		originalDB = cx;
-		std::vector<Future<Void>> clients = { readerClientSeparateDBs(cx, this),
-			                                  doSwitch(switchConnFileDb, this),
-			                                  writerClient(cx, this),
-			                                  writerClient(extraDB, this) };
+		std::vector<Future<Void>> clients = { readerClientSeparateDBs(cx, extraDB, keyToRead),
+			                                  doSwitch(switchConnFileDb),
+			                                  writerClient(cx, keyToRead),
+			                                  writerClient(extraDB, keyToRead) };
 		return success(timeout(waitForAll(clients), testDuration));
 	}
 
@@ -107,119 +107,138 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<std::pair<Version, Optional<Value>>> doRead(Database cx,
-	                                                                DifferentClustersSameRVWorkload* self) {
-		state Transaction tr(cx);
-		loop {
+	static Future<std::pair<Version, Optional<Value>>> doRead(Database cx, Value const& keyToRead) {
+		Transaction tr(cx);
+		while (true) {
 			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			Error err;
 			try {
-				state Version rv = wait(tr.getReadVersion());
-				Optional<Value> val1 = wait(tr.get(self->keyToRead));
-				return std::make_pair(rv, val1);
+				Version rv = co_await tr.getReadVersion();
+				Optional<Value> val1 = co_await tr.get(keyToRead);
+				co_return std::make_pair(rv, val1);
 			} catch (Error& e) {
-				TRACE_ERROR(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TRACE_ERROR(err);
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> doWrite(Database cx, Value key, Optional<Value> val) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> doWrite(Database cx, Value key, Optional<Value> val) {
+		Transaction tr(cx);
+		while (true) {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Error err;
 			try {
 				if (val.present()) {
 					tr.set(key, val.get());
 				} else {
 					tr.clear(key);
 				}
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				TRACE_ERROR(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TRACE_ERROR(err);
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> advanceVersion(Database cx, Version v) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> advanceVersion(Database cx, Version v) {
+		Transaction tr(cx);
+		while (true) {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Error err;
 			try {
-				Version extraDBVersion = wait(tr.getReadVersion());
+				Version extraDBVersion = co_await tr.getReadVersion();
 				if (extraDBVersion <= v) {
 					tr.set(minRequiredCommitVersionKey, BinaryWriter::toValue(v + 1, Unversioned()));
-					wait(tr.commit());
+					co_await tr.commit();
+					continue;
 				} else {
-					return Void();
+					co_return;
 				}
 			} catch (Error& e) {
-				TRACE_ERROR(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TRACE_ERROR(err);
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> doSwitch(Database cx, DifferentClustersSameRVWorkload* self) {
-		state UID lockUid = deterministicRandom()->randomUniqueID();
-		wait(delay(self->switchAfter));
-		state Future<Void> watchFuture;
-		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) mutable -> Future<Void> {
-			watchFuture = tr->watch(self->keyToWatch);
-			return Void();
-		}));
-		wait(lockDatabase(self->originalDB, lockUid) && lockDatabase(self->extraDB, lockUid));
+	Future<Void> doSwitch(Database cx) {
+		UID lockUid = deterministicRandom()->randomUniqueID();
+		co_await delay(switchAfter);
+		Future<Void> watchFuture;
+		co_await runRYWTransaction(
+		    cx, [this, &watchFuture](Reference<ReadYourWritesTransaction> tr) mutable -> Future<Void> {
+			    watchFuture = tr->watch(keyToWatch);
+			    return Void();
+		    });
+		co_await (lockDatabase(originalDB, lockUid) && lockDatabase(extraDB, lockUid));
 		TraceEvent("DifferentClusters_LockedDatabases").log();
-		std::pair<Version, Optional<Value>> read1 = wait(doRead(self->originalDB, self));
-		state Version rv = read1.first;
-		state Optional<Value> val1 = read1.second;
-		wait(doWrite(self->extraDB, self->keyToRead, val1));
+		std::pair<Version, Optional<Value>> read1 = co_await doRead(originalDB, keyToRead);
+		Version rv = read1.first;
+		Optional<Value> val1 = read1.second;
+		co_await doWrite(extraDB, keyToRead, val1);
 		TraceEvent("DifferentClusters_CopiedDatabase").log();
-		wait(advanceVersion(self->extraDB, rv));
+		co_await advanceVersion(extraDB, rv);
 		TraceEvent("DifferentClusters_AdvancedVersion").log();
-		wait(cx->switchConnectionRecord(
-		    makeReference<ClusterConnectionMemoryRecord>(self->extraDB->getConnectionRecord()->getConnectionString())));
+		co_await cx->switchConnectionRecord(
+		    makeReference<ClusterConnectionMemoryRecord>(extraDB->getConnectionRecord()->getConnectionString()));
 		TraceEvent("DifferentClusters_SwitchedConnectionFile").log();
-		state Transaction tr(cx);
+		Transaction tr(cx);
 		tr.setVersion(rv);
 		tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+		Error readErr;
+		bool readFailed = false;
 		try {
-			Optional<Value> val2 = wait(tr.get(self->keyToRead));
+			Optional<Value> val2 = co_await tr.get(keyToRead);
 			// We read the same key at the same read version with the same db, we must get the same value (or fail to
 			// read)
 			ASSERT(val1 == val2);
-		} catch (Error& e) {
-			TraceEvent("DifferentClusters_ReadError").error(e);
-			TRACE_ERROR(e);
-			wait(tr.onError(e));
+		} catch (Error& err) {
+			readErr = err;
+			readFailed = true;
+		}
+		if (readFailed) {
+			TraceEvent("DifferentClusters_ReadError").error(readErr);
+			TRACE_ERROR(readErr);
+			co_await tr.onError(readErr);
 		}
 		// In an actual switch we would call switchConnectionRecord after unlocking the database. But it's possible
 		// that a storage server serves a read at |rv| even after the recovery caused by unlocking the database, and we
 		// want to make that more likely for this test. So read at |rv| then unlock.
-		wait(unlockDatabase(self->extraDB, lockUid));
+		co_await unlockDatabase(extraDB, lockUid);
 		TraceEvent("DifferentClusters_UnlockedExtraDB").log();
 		ASSERT(!watchFuture.isReady() || watchFuture.isError());
-		wait(doWrite(self->extraDB, self->keyToWatch, Optional<Value>{ ""_sr }));
+		co_await doWrite(extraDB, keyToWatch, Optional<Value>{ ""_sr });
 		TraceEvent("DifferentClusters_WaitingForWatch").log();
+		Error watchErr;
+		bool watchFailed = false;
 		try {
-			wait(timeoutError(watchFuture, (self->testDuration - self->switchAfter) / 2));
-		} catch (Error& e) {
-			TraceEvent("DifferentClusters_WatchError").error(e);
-			wait(tr.onError(e));
+			co_await timeoutError(watchFuture, (testDuration - switchAfter) / 2);
+		} catch (Error& err) {
+			watchErr = err;
+			watchFailed = true;
+		}
+		if (watchFailed) {
+			TraceEvent("DifferentClusters_WatchError").error(watchErr);
+			co_await tr.onError(watchErr);
 		}
 		TraceEvent("DifferentClusters_Done").log();
-		self->switchComplete = true;
-		wait(unlockDatabase(self->originalDB, lockUid)); // So quietDatabase can finish
-		return Void();
+		switchComplete = true;
+		co_await unlockDatabase(originalDB, lockUid); // So quietDatabase can finish
 	}
 
-	ACTOR static Future<Void> writerClient(Database cx, DifferentClustersSameRVWorkload* self) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> writerClient(Database cx, Value const& keyToRead) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				Optional<Value> value = wait(tr.get(self->keyToRead));
+				Optional<Value> value = co_await tr.get(keyToRead);
 				int x = 0;
 				if (value.present()) {
 					BinaryReader r(value.get(), Unversioned());
@@ -228,57 +247,65 @@ struct DifferentClustersSameRVWorkload : TestWorkload {
 				x += 1;
 				BinaryWriter w(Unversioned());
 				serializer(w, x);
-				tr.set(self->keyToRead, w.toValue());
-				wait(tr.commit());
+				tr.set(keyToRead, w.toValue());
+				co_await tr.commit();
 				tr.reset();
+				continue;
 			} catch (Error& e) {
-				TRACE_ERROR(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TRACE_ERROR(err);
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Optional<Value>> readAtVersion(DifferentClustersSameRVWorkload* self,
-	                                                   const char* name,
-	                                                   Transaction* tr,
-	                                                   Version version) {
-		state Optional<Value> res;
+	static Future<Optional<Value>> readAtVersion(Value const& keyToRead,
+	                                             const char* name,
+	                                             Transaction* tr,
+	                                             Version version) {
+		Optional<Value> res;
 		try {
 			tr->reset();
 			tr->setVersion(version);
-			wait(store(res, tr->get(self->keyToRead)));
-			return res;
+			res = co_await tr->get(keyToRead);
+			co_return res;
 		} catch (Error& e) {
 			TraceEvent(name).error(e);
 			throw e;
 		}
 	}
 
-	ACTOR static Future<Void> readerClientSeparateDBs(Database cx, DifferentClustersSameRVWorkload* self) {
-		state Transaction tr1(cx);
-		state Transaction tr2(self->extraDB);
-		state Version rv1;
-		state Version rv2;
-		state Optional<Value> val1;
-		state Optional<Value> val2;
-		loop {
+	static Future<Void> readerClientSeparateDBs(Database cx, Database extraDB, Value const& keyToRead) {
+		Transaction tr1(cx);
+		Transaction tr2(extraDB);
+		Version rv1{ 0 };
+		Version rv2{ 0 };
+		Optional<Value> val1;
+		Optional<Value> val2;
+		while (true) {
 			tr1.reset();
 			tr2.reset();
 			tr1.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			tr2.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+			Error err;
 			try {
-				wait(store(rv1, tr1.getReadVersion()) && store(rv2, tr2.getReadVersion()));
+				co_await (store(rv1, tr1.getReadVersion()) && store(rv2, tr2.getReadVersion()));
 				TraceEvent("DifferentClustersSameRVWorkload_GotReadVersion").detail("RV1", rv1).detail("RV2", rv2);
-				state Version rv = std::min(rv1, rv2);
-				wait(store(val1, readAtVersion(self, "DifferentClustersSameRVWorkload_Transaction1_Error", &tr1, rv)) &&
-				     store(val2, readAtVersion(self, "DifferentClustersSameRVWorkload_Transaction2_Error", &tr2, rv)));
+				Version rv = std::min(rv1, rv2);
+				co_await (
+				    store(val1,
+				          readAtVersion(keyToRead, "DifferentClustersSameRVWorkload_Transaction1_Error", &tr1, rv)) &&
+				    store(val2,
+				          readAtVersion(keyToRead, "DifferentClustersSameRVWorkload_Transaction2_Error", &tr2, rv)));
 				// We're reading from different db's with the same read version. We can get a different value.
 				CODE_PROBE(val1.present() != val2.present() || !val1.present() || val1.get() != val2.get(),
 				           "reading from different dbs with the same version");
+				continue;
 			} catch (Error& e) {
-				TRACE_ERROR(e);
-				wait(tr1.onError(e) && tr2.onError(e));
+				err = e;
 			}
+			TRACE_ERROR(err);
+			co_await (tr1.onError(err) && tr2.onError(err));
 		}
 	}
 };
diff --git a/fdbserver/workloads/DiskDurability.actor.cpp b/fdbserver/workloads/DiskDurability.cpp
similarity index 54%
rename from fdbserver/workloads/DiskDurability.actor.cpp
rename to fdbserver/workloads/DiskDurability.cpp
index ed9a5e5a202..6495e17f6ae 100644
--- a/fdbserver/workloads/DiskDurability.actor.cpp
+++ b/fdbserver/workloads/DiskDurability.cpp
@@ -1,5 +1,5 @@
 /*
- * DiskDurability.actor.cpp
+ * DiskDurability.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,38 +18,34 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
 #include "flow/SystemMonitor.h"
 #include "flow/IAsyncFile.h"
-#include "fdbserver/workloads/AsyncFile.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "AsyncFile.h"
 
 struct DiskDurabilityWorkload : public AsyncFileWorkload {
 	static constexpr auto NAME = "DiskDurability";
 	struct FileBlock {
-		FileBlock(int blockNum) : blockNum(blockNum), lastData(0), lock(new FlowLock(1)) {}
-		~FileBlock() {}
+		explicit FileBlock(int blockNum) : blockNum(blockNum), lastData(0), lock(new FlowLock(1)) {}
+		~FileBlock() = default;
 		int blockNum;
 		int64_t lastData;
 		Reference<FlowLock> lock;
 
-		ACTOR static Future<Void> test_impl(FileBlock* self,
-		                                    Reference<AsyncFileHandle> file,
-		                                    int pages,
-		                                    Reference<AsyncFileBuffer> buffer) {
-			wait(self->lock->take());
+		Future<Void> test_impl(Reference<AsyncFileHandle> file, int pages, Reference<AsyncFileBuffer> buffer) {
+			co_await lock->take();
 
-			state int64_t offset = (int64_t)self->blockNum * pages * _PAGE_SIZE;
-			state int size = pages * _PAGE_SIZE;
+			int64_t offset = (int64_t)blockNum * pages * _PAGE_SIZE;
+			int size = pages * _PAGE_SIZE;
 
-			state int64_t newData = 0;
-			if (self->lastData == 0)
+			int64_t newData = 0;
+			if (lastData == 0)
 				newData = deterministicRandom()->randomInt64(std::numeric_limits<int64_t>::min(),
 				                                             std::numeric_limits<int64_t>::max());
 			else {
 				++newData;
-				int readBytes = wait(file->file->read(buffer->buffer, size, offset));
+				int readBytes = co_await file->file->read(buffer->buffer, size, offset);
 				ASSERT(readBytes == size);
 			}
 
@@ -57,12 +53,12 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload {
 				newData = 1;
 			int64_t* arr = (int64_t*)buffer->buffer;
 			for (int i = 0, imax = size / sizeof(int64_t); i < imax; ++i) {
-				if (self->lastData != 0 && arr[i] != self->lastData) {
+				if (lastData != 0 && arr[i] != lastData) {
 					TraceEvent(SevError, "WriteWasNotDurable")
 					    .detail("Filename", file->path)
 					    .detail("Offset", offset)
 					    .detail("OpSize", size)
-					    .detail("Expected", self->lastData)
+					    .detail("Expected", lastData)
 					    .detail("Found", arr[i])
 					    .detail("Index", i);
 					throw io_error();
@@ -70,14 +66,13 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload {
 				arr[i] = newData;
 			}
 
-			wait(file->file->write(buffer->buffer, size, offset));
-			self->lock->release(1);
-			self->lastData = newData;
-			return Void();
+			co_await file->file->write(buffer->buffer, size, offset);
+			lock->release(1);
+			lastData = newData;
 		}
 
 		Future<Void> test(Reference<AsyncFileHandle> file, int pages, Reference<AsyncFileBuffer> buffer) {
-			return test_impl(this, file, pages, buffer);
+			return test_impl(file, pages, buffer);
 		}
 	};
 
@@ -87,7 +82,7 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload {
 	int writers;
 	double syncInterval;
 
-	DiskDurabilityWorkload(WorkloadContext const& wcx) : AsyncFileWorkload(wcx) {
+	explicit DiskDurabilityWorkload(WorkloadContext const& wcx) : AsyncFileWorkload(wcx) {
 		writers = getOption(options, "writers"_sr, 1);
 		filePages = getOption(options, "filePages"_sr, 1000000);
 		fileSize = filePages * _PAGE_SIZE;
@@ -98,42 +93,40 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload {
 		syncInterval = (double)(getOption(options, "syncIntervalMs"_sr, 2000)) / 1000;
 	}
 
-	~DiskDurabilityWorkload() override {}
+	~DiskDurabilityWorkload() override = default;
 
 	Future<Void> setup(Database const& cx) override {
 		if (enabled)
-			return _setup(this);
+			return _setup();
 
 		return Void();
 	}
 
-	ACTOR Future<Void> _setup(DiskDurabilityWorkload* self) {
-		ASSERT(!self->path.empty());
+	Future<Void> _setup() {
+		ASSERT(!path.empty());
 
 		int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE;
 
-		if (self->unbufferedIO)
+		if (unbufferedIO)
 			flags |= IAsyncFile::OPEN_UNBUFFERED;
-		if (self->uncachedIO)
+		if (uncachedIO)
 			flags |= IAsyncFile::OPEN_UNCACHED;
 
 		try {
-			state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(self->path, flags, 0666));
-			if (self->fileHandle.getPtr() == nullptr)
-				self->fileHandle = makeReference<AsyncFileHandle>(file, self->path, false);
+			Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(path, flags, 0666);
+			if (fileHandle.getPtr() == nullptr)
+				fileHandle = makeReference<AsyncFileHandle>(file, path, false);
 			else
-				self->fileHandle->file = file;
+				fileHandle->file = file;
 		} catch (Error& error) {
 			TraceEvent(SevError, "TestFailure").detail("Reason", "Could not open file");
 			throw;
 		}
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (enabled)
-			return _start(this);
+			return _start();
 
 		return Void();
 	}
@@ -145,39 +138,37 @@ struct DiskDurabilityWorkload : public AsyncFileWorkload {
 		return x;
 	}
 
-	ACTOR static Future<Void> worker(DiskDurabilityWorkload* self) {
-		state Reference<AsyncFileBuffer> buffer = makeReference<AsyncFileBuffer>(_PAGE_SIZE, true);
-		state int logfp = (int)ceil(log2(self->filePages));
-		loop {
+	Future<Void> worker() {
+		auto buffer = makeReference<AsyncFileBuffer>(_PAGE_SIZE, true);
+		int logfp = (int)ceil(log2(filePages));
+		while (true) {
 			int block = intHash(std::min<int>(
 			                deterministicRandom()->randomInt(0, 1 << deterministicRandom()->randomInt(0, logfp)),
-			                self->filePages - 1)) %
-			            self->filePages;
-			wait(self->blocks[block].test(self->fileHandle, self->pagesPerWrite, buffer));
+			                filePages - 1)) %
+			            filePages;
+			co_await blocks[block].test(fileHandle, pagesPerWrite, buffer);
 		}
 	}
 
-	ACTOR static Future<Void> syncLoop(DiskDurabilityWorkload* self) {
-		loop {
-			wait(delay(deterministicRandom()->random01() * self->syncInterval));
-			wait(self->fileHandle->file->sync());
+	Future<Void> syncLoop() {
+		while (true) {
+			co_await delay(deterministicRandom()->random01() * syncInterval);
+			co_await fileHandle->file->sync();
 		}
 	}
 
-	ACTOR Future<Void> _start(DiskDurabilityWorkload* self) {
-		self->blocks.reserve(self->filePages);
-		for (int i = 0; i < self->filePages; ++i)
-			self->blocks.push_back(FileBlock(i));
+	Future<Void> _start() {
+		blocks.reserve(filePages);
+		for (int i = 0; i < filePages; ++i)
+			blocks.push_back(FileBlock(i));
 
-		state std::vector<Future<Void>> tasks;
-		tasks.push_back(syncLoop(self));
+		std::vector<Future<Void>> tasks;
+		tasks.push_back(syncLoop());
 
-		for (int i = 0; i < self->writers; ++i)
-			tasks.push_back(worker(self));
+		for (int i = 0; i < writers; ++i)
+			tasks.push_back(worker());
 
-		wait(timeout(waitForAll(tasks), self->testDuration, Void()));
-
-		return Void();
+		co_await timeout(waitForAll(tasks), testDuration, Void());
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
diff --git a/fdbserver/workloads/DiskDurabilityTest.actor.cpp b/fdbserver/workloads/DiskDurabilityTest.cpp
similarity index 66%
rename from fdbserver/workloads/DiskDurabilityTest.actor.cpp
rename to fdbserver/workloads/DiskDurabilityTest.cpp
index 2d16e18a20c..3d93c438ede 100644
--- a/fdbserver/workloads/DiskDurabilityTest.actor.cpp
+++ b/fdbserver/workloads/DiskDurabilityTest.cpp
@@ -1,5 +1,5 @@
 /*
- * DiskDurabilityTest.actor.cpp
+ * DiskDurabilityTest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,9 @@
 
 #include <cinttypes>
 #include "fmt/format.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/IAsyncFile.h"
 #include "fdbclient/FDBTypes.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct DiskDurabilityTest : TestWorkload {
 	static constexpr auto NAME = "DiskDurabilityTest";
@@ -31,7 +30,7 @@ struct DiskDurabilityTest : TestWorkload {
 	std::string filename;
 	KeyRange range, metrics;
 
-	DiskDurabilityTest(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit DiskDurabilityTest(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		filename = getOption(options, "filename"_sr, "durability_test.bin"_sr).toString();
 		auto prefix = getOption(options, "prefix"_sr, "/DiskDurabilityTest/"_sr);
@@ -42,7 +41,7 @@ struct DiskDurabilityTest : TestWorkload {
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override {
 		if (enabled)
-			return durabilityTest(this, cx);
+			return durabilityTest(cx);
 		return Void();
 	}
 	Future<bool> check(Database const& cx) override { return true; }
@@ -73,42 +72,43 @@ struct DiskDurabilityTest : TestWorkload {
 		return ipage[0];
 	}
 
-	ACTOR static Future<Void> durabilityTest(DiskDurabilityTest* self, Database db) {
-		state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(
-		    self->filename,
+	Future<Void> durabilityTest(Database db) {
+		Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
+		    filename,
 		    IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNBUFFERED |
 		        IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_LOCK,
-		    0600));
-		state std::vector<uint8_t> pagedata(4096 * 128);
-		state uint8_t* page = (uint8_t*)((intptr_t(&pagedata[0]) | intptr_t(4095)) + 1);
+		    0600);
+		std::vector<uint8_t> pagedata(4096 * 128);
+		uint8_t* page = (uint8_t*)((intptr_t(&pagedata[0]) | intptr_t(4095)) + 1);
 
-		state int64_t size = wait(file->size());
-		state bool failed = false;
-		state int verifyPages;
+		int64_t size = co_await file->size();
+		bool failed = false;
+		int verifyPages{ 0 };
 
 		// Verify
-		state Transaction tr(db);
-		loop {
+		Transaction tr(db);
+		while (true) {
+			Error err;
 			try {
-				state RangeResult r = wait(tr.getRange(self->range, GetRangeLimits(1000000)));
+				RangeResult r = co_await tr.getRange(range, GetRangeLimits(1000000));
 				verifyPages = r.size();
-				state int i;
-				for (i = 0; i < r.size(); i++) {
-					int bytesRead = wait(file->read(page, 4096, self->decodeKey(r[i].key) * 4096));
-					if (bytesRead != 4096 || self->decodePage(page) != self->decodeValue(r[i].value)) {
+				for (int i = 0; i < r.size(); i++) {
+					int bytesRead = co_await file->read(page, 4096, decodeKey(r[i].key) * 4096);
+					if (bytesRead != 4096 || decodePage(page) != decodeValue(r[i].value)) {
 						printf("ValidationError\n");
 						TraceEvent(SevError, "ValidationError")
-						    .detail("At", self->decodeKey(r[i].key))
-						    .detail("Expected", self->decodeValue(r[i].value))
-						    .detail("Found", self->decodePage(page))
+						    .detail("At", decodeKey(r[i].key))
+						    .detail("Expected", decodeValue(r[i].value))
+						    .detail("Found", decodePage(page))
 						    .detail("Read", bytesRead);
 						failed = true;
 					}
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		if (failed)
@@ -118,9 +118,9 @@ struct DiskDurabilityTest : TestWorkload {
 		TraceEvent(SevInfo, "Verified").detail("Pages", verifyPages).detail("Of", size / 4096);
 
 		// Run
-		state bool first = true;
-		loop {
-			state std::vector<int64_t> targetPages;
+		bool first = true;
+		while (true) {
+			std::vector<int64_t> targetPages;
 			for (int i = deterministicRandom()->randomInt(1, 100); i > 0 && targetPages.size() < size / 4096; i--) {
 				auto p = deterministicRandom()->randomInt(0, size / 4096);
 				if (std::find(targetPages.begin(), targetPages.end(), p) == targetPages.end())
@@ -131,53 +131,57 @@ struct DiskDurabilityTest : TestWorkload {
 				size += 4096;
 			}
 
-			state std::vector<int64_t> targetValues(targetPages.size());
+			std::vector<int64_t> targetValues(targetPages.size());
 			for (auto& v : targetValues)
 				v = deterministicRandom()->randomUniqueID().first();
 
 			tr.reset();
-			loop {
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < targetPages.size(); i++)
-						tr.clear(self->encodeKey(targetPages[i]));
+						tr.clear(encodeKey(targetPages[i]));
 
 					if (!first) {
-						Optional<Value> v = wait(tr.get("syncs"_sr.withPrefix(self->metrics.begin)));
-						int64_t count = v.present() ? self->decodeValue(v.get()) : 0;
+						Optional<Value> v = co_await tr.get("syncs"_sr.withPrefix(metrics.begin));
+						int64_t count = v.present() ? decodeValue(v.get()) : 0;
 						count++;
-						tr.set("syncs"_sr.withPrefix(self->metrics.begin), self->encodeValue(count));
+						tr.set("syncs"_sr.withPrefix(metrics.begin), encodeValue(count));
 					}
 
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 			tr.reset();
-			state Future<Version> rv = tr.getReadVersion(); // hide this latency
+			Future<Version> rv = tr.getReadVersion(); // hide this latency
 
 			std::vector<Future<Void>> fresults;
 
 			for (int i = 0; i < targetPages.size(); i++) {
 				uint8_t* p = page + 4096 * i;
-				self->encodePage(p, targetValues[i]);
+				encodePage(p, targetValues[i]);
 				fresults.push_back(file->write(p, 4096, targetPages[i] * 4096));
 			}
 
-			wait(waitForAll(fresults));
+			co_await waitForAll(fresults);
 
-			wait(file->sync());
+			co_await file->sync();
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < targetPages.size(); i++)
-						tr.set(self->encodeKey(targetPages[i]), self->encodeValue(targetValues[i]));
-					wait(tr.commit());
+						tr.set(encodeKey(targetPages[i]), encodeValue(targetValues[i]));
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 
 			first = false;
diff --git a/fdbserver/workloads/DiskFailureInjection.actor.cpp b/fdbserver/workloads/DiskFailureInjection.cpp
similarity index 64%
rename from fdbserver/workloads/DiskFailureInjection.actor.cpp
rename to fdbserver/workloads/DiskFailureInjection.cpp
index 2efa9cea755..746ed0cb1ef 100644
--- a/fdbserver/workloads/DiskFailureInjection.actor.cpp
+++ b/fdbserver/workloads/DiskFailureInjection.cpp
@@ -1,5 +1,5 @@
 /*
- * DiskFailureInjection.actor.cpp
+ * DiskFailureInjection.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,12 +19,12 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/Status.actor.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/WorkerEvents.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
@@ -49,7 +49,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 
 	DiskFailureInjectionWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {}
 
-	DiskFailureInjectionWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+	explicit DiskFailureInjectionWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		startDelay = getOption(options, "startDelay"_sr, startDelay);
 		testDuration = getOption(options, "testDuration"_sr, testDuration);
@@ -77,14 +77,14 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 	// 2. Starting the actor that injects failures on chosen storage servers
 	Future<Void> start(Database const& cx) override {
 		if (enabled) {
-			auto result = diskFailureInjectionClient<WorkerInterface>(cx, this);
+			auto result = diskFailureInjectionClient<WorkerInterface>(cx);
 			//  In verification mode, we want to wait until periodicEventBroadcast actor returns which indicates that
 			//  a non-zero chaosMetric was found.
 			if (verificationMode) {
-				return (periodicEventBroadcast(this) && delay(testDuration)) || result;
+				return (periodicEventBroadcast() && delay(testDuration)) || result;
 			} else {
 				// Else we honor the testDuration
-				return timeout(periodicEventBroadcast(this) && result, testDuration, Void());
+				return timeout(periodicEventBroadcast() && result, testDuration, Void());
 			}
 		} else
 			return Void();
@@ -109,11 +109,11 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 	}
 
 	// Sets the disk delay request
-	ACTOR void injectDiskDelays(WorkerInterface worker,
-	                            double stallInterval,
-	                            double stallPeriod,
-	                            double throttlePeriod) {
-		state Future<Void> res;
+	Future<Void> injectDiskDelays(WorkerInterface worker,
+	                              double stallInterval,
+	                              double stallPeriod,
+	                              double throttlePeriod) {
+		Future<Void> res;
 		SetFailureInjection::DiskFailureCommand diskFailure;
 		diskFailure.stallInterval = stallInterval;
 		diskFailure.stallPeriod = stallPeriod;
@@ -121,36 +121,36 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 		SetFailureInjection req;
 		req.diskFailure = diskFailure;
 		res = worker.clientInterface.setFailureInjection.getReply(req);
-		wait(ready(res));
+		co_await ready(res);
 		checkDiskFailureInjectionResult(res, worker);
 	}
 
 	// Sets the disk corruption request
-	ACTOR void injectBitFlips(WorkerInterface worker, double percentage) {
-		state Future<Void> res;
+	Future<Void> injectBitFlips(WorkerInterface worker, double percentage) {
+		Future<Void> res;
 		SetFailureInjection::FlipBitsCommand flipBits;
 		flipBits.percentBitFlips = percentage;
 		SetFailureInjection req;
 		req.flipBits = flipBits;
 		res = worker.clientInterface.setFailureInjection.getReply(req);
-		wait(ready(res));
+		co_await ready(res);
 		checkDiskFailureInjectionResult(res, worker);
 	}
 
 	// Choose random storage servers to inject disk failures.
 	// We currently only inject disk failure on storage servers. Can be expanded to include
 	// other worker types in future
-	ACTOR template <class W>
-	Future<Void> diskFailureInjectionClient(Database cx, DiskFailureInjectionWorkload* self) {
-		wait(::delay(self->startDelay));
-		state double lastTime = now();
-		state std::vector<W> machines;
-		state int throttledWorkers = 0;
-		state int corruptedWorkers = 0;
-		loop {
-			wait(poisson(&lastTime, 1));
+	template <class W>
+	Future<Void> diskFailureInjectionClient(Database cx) {
+		co_await ::delay(startDelay);
+		double lastTime = now();
+		std::vector<W> machines;
+		int throttledWorkers = 0;
+		int corruptedWorkers = 0;
+		while (true) {
+			co_await poisson(&lastTime, 1);
 			try {
-				std::pair<std::vector<W>, int> m = wait(getStorageWorkers(cx, self->dbInfo, false));
+				std::pair<std::vector<W>, int> m = co_await getStorageWorkers(cx, dbInfo, false);
 				if (m.second > 0) {
 					throw operation_failed();
 				}
@@ -164,74 +164,72 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 			auto machine = deterministicRandom()->randomChoice(machines);
 
 			// If we have already chosen this worker, then just continue
-			if (find(self->chosenWorkers.begin(), self->chosenWorkers.end(), machine.address()) !=
-			    self->chosenWorkers.end()) {
+			if (find(chosenWorkers.begin(), chosenWorkers.end(), machine.address()) != chosenWorkers.end()) {
 				continue;
 			}
 
 			// Keep track of chosen workers for verification purpose
-			self->chosenWorkers.emplace_back(machine.address());
-			if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle))
-				self->injectDiskDelays(machine, self->stallInterval, self->stallPeriod, self->throttlePeriod);
-			if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) {
+			chosenWorkers.emplace_back(machine.address());
+			if (throttleDisk && (throttledWorkers++ < workersToThrottle))
+				injectDiskDelays(machine, stallInterval, stallPeriod, throttlePeriod);
+			if (corruptFile && (corruptedWorkers++ < workersToCorrupt)) {
 				if (g_simulator == g_network)
-					g_simulator->corruptWorkerMap[machine.address()] = true;
-				self->injectBitFlips(machine, self->percentBitFlips);
+					fdbSimulationPolicyState().corruptWorkerMap[machine.address()] = true;
+				injectBitFlips(machine, percentBitFlips);
 			}
 		}
 	}
 
 	// Resend the chaos event to previously chosen workers, in case some workers got restarted and lost their chaos
 	// config
-	ACTOR static Future<Void> reSendChaos(DiskFailureInjectionWorkload* self) {
-		state int throttledWorkers = 0;
-		state int corruptedWorkers = 0;
-		state std::map<NetworkAddress, WorkerInterface> workersMap;
-		state std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
-		for (auto worker : workers) {
+	Future<Void> reSendChaos() {
+		int throttledWorkers = 0;
+		int corruptedWorkers = 0;
+		std::map<NetworkAddress, WorkerInterface> workersMap;
+		std::vector<WorkerDetails> workers = co_await getWorkers(dbInfo);
+		for (const auto& worker : workers) {
 			workersMap[worker.interf.address()] = worker.interf;
 		}
 		TraceEvent("ResendChaos")
-		    .detail("ChosenWorkersSize", self->chosenWorkers.size())
+		    .detail("ChosenWorkersSize", chosenWorkers.size())
 		    .detail("FoundWorkers", workersMap.size())
 		    .detail("ResendToNumber",
-		            std::count_if(self->chosenWorkers.begin(),
-		                          self->chosenWorkers.end(),
+		            std::count_if(chosenWorkers.begin(),
+		                          chosenWorkers.end(),
 		                          [&map = std::as_const(workersMap)](auto const& addr) { return map.contains(addr); }));
-		for (auto& workerAddress : self->chosenWorkers) {
+		for (auto& workerAddress : chosenWorkers) {
 			auto itr = workersMap.find(workerAddress);
 			if (itr != workersMap.end()) {
-				if (self->throttleDisk && (throttledWorkers++ < self->workersToThrottle)) {
-					self->injectDiskDelays(itr->second, self->stallInterval, self->stallPeriod, self->throttlePeriod);
+				if (throttleDisk && (throttledWorkers++ < workersToThrottle)) {
+					injectDiskDelays(itr->second, stallInterval, stallPeriod, throttlePeriod);
 				}
-				if (self->corruptFile && (corruptedWorkers++ < self->workersToCorrupt)) {
+				if (corruptFile && (corruptedWorkers++ < workersToCorrupt)) {
 					if (g_simulator == g_network)
-						g_simulator->corruptWorkerMap[workerAddress] = true;
-					self->injectBitFlips(itr->second, self->percentBitFlips);
+						fdbSimulationPolicyState().corruptWorkerMap[workerAddress] = true;
+					injectBitFlips(itr->second, percentBitFlips);
 				}
 			}
 		}
-		return Void();
 	}
 
 	// Fetches chaosMetrics and verifies that chaos events are happening for enabled workers
-	ACTOR static Future<int> chaosGetStatus(DiskFailureInjectionWorkload* self) {
-		state int foundChaosMetrics = 0;
-		state std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
+	Future<int> chaosGetStatus() {
+		int foundChaosMetrics = 0;
+		std::vector<WorkerDetails> workers = co_await getWorkers(dbInfo);
 
-		Future<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventsFuture;
-		latestEventsFuture = latestEventOnWorkers(workers, "ChaosMetrics");
-		state Optional<std::pair<WorkerEvents, std::set<std::string>>> workerEvents = wait(latestEventsFuture);
+		AsyncResult<Optional<std::pair<WorkerEvents, std::set<std::string>>>> latestEventsFuture =
+		    latestEventOnWorkers(workers, "ChaosMetrics");
+		Optional<std::pair<WorkerEvents, std::set<std::string>>> workerEvents = co_await std::move(latestEventsFuture);
 
-		state WorkerEvents cMetrics = workerEvents.present() ? workerEvents.get().first : WorkerEvents();
+		WorkerEvents cMetrics = workerEvents.present() ? workerEvents.get().first : WorkerEvents();
 
 		// Check if any of the chosen workers for chaos events have non-zero chaosMetrics
 		try {
-			for (auto& workerAddress : self->chosenWorkers) {
+			for (auto& workerAddress : chosenWorkers) {
 				auto chaosMetrics = cMetrics.find(workerAddress);
 				if (chaosMetrics != cMetrics.end()) {
 					// we expect diskDelays to be non-zero for chosenWorkers for throttleDisk event
-					if (self->throttleDisk) {
+					if (throttleDisk) {
 						int diskDelays = chaosMetrics->second.getInt("DiskDelays");
 						if (diskDelays > 0) {
 							foundChaosMetrics += diskDelays;
@@ -239,7 +237,7 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 					}
 
 					// we expect bitFlips to be non-zero for chosenWorkers for corruptFile event
-					if (self->corruptFile) {
+					if (corruptFile) {
 						int bitFlips = chaosMetrics->second.getInt("BitFlips");
 						if (bitFlips > 0) {
 							foundChaosMetrics += bitFlips;
@@ -255,26 +253,24 @@ struct DiskFailureInjectionWorkload : FailureInjectionWorkload {
 			}
 		}
 
-		return foundChaosMetrics;
+		co_return foundChaosMetrics;
 	}
 
 	// Periodically re-send the chaos event in case of a process restart
-	ACTOR static Future<Void> periodicEventBroadcast(DiskFailureInjectionWorkload* self) {
-		wait(::delay(self->startDelay));
-		state double start = now();
-		state double elapsed = 0.0;
+	Future<Void> periodicEventBroadcast() {
+		co_await ::delay(startDelay);
+		double start = now();
+		double elapsed = 0.0;
 
-		loop {
-			wait(delayUntil(start + elapsed));
-			wait(reSendChaos(self));
-			elapsed += self->periodicBroadcastInterval;
-			wait(delayUntil(start + elapsed));
-			int foundChaosMetrics = wait(chaosGetStatus(self));
+		while (true) {
+			co_await delayUntil(start + elapsed);
+			co_await reSendChaos();
+			elapsed += periodicBroadcastInterval;
+			co_await delayUntil(start + elapsed);
+			int foundChaosMetrics = co_await chaosGetStatus();
 			if (foundChaosMetrics > 0) {
-				TraceEvent("FoundChaos")
-				    .detail("ChaosMetricCount", foundChaosMetrics)
-				    .detail("ClientID", self->clientId);
-				return Void();
+				TraceEvent("FoundChaos").detail("ChaosMetricCount", foundChaosMetrics).detail("ClientID", clientId);
+				break;
 			}
 		}
 	}
diff --git a/fdbserver/workloads/ExcludeIncludeStorageServersWorkload.actor.cpp b/fdbserver/workloads/ExcludeIncludeStorageServersWorkload.cpp
similarity index 75%
rename from fdbserver/workloads/ExcludeIncludeStorageServersWorkload.actor.cpp
rename to fdbserver/workloads/ExcludeIncludeStorageServersWorkload.cpp
index 4c19b9d5f09..4bc9df90b91 100644
--- a/fdbserver/workloads/ExcludeIncludeStorageServersWorkload.actor.cpp
+++ b/fdbserver/workloads/ExcludeIncludeStorageServersWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * ExcludeIncludeStorageServersWorkload.actor.cpp
+ * ExcludeIncludeStorageServersWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,13 +19,13 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/ManagementAPI.h"
 
 // This test creates a scenario that large number of SS join the cluster, by excluding and including a random
 // SS consistently for many times.
@@ -46,11 +46,11 @@ struct ExcludeIncludeStorageServersWorkload : TestWorkload {
 
 	std::map<AddressExclusion, std::set<AddressExclusion>> machineProcesses; // ip -> ip:port
 
-	ExcludeIncludeStorageServersWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ExcludeIncludeStorageServersWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled =
 		    !clientId && g_network->isSimulated(); // only do this on the "first" client, and only when in simulation
 		if (g_network->isSimulated()) {
-			g_simulator->allowLogSetKills = false;
+			fdbSimulationPolicyState().allowLogSetKills = false;
 		}
 	}
 
@@ -67,31 +67,32 @@ struct ExcludeIncludeStorageServersWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override {
 		if (!enabled)
 			return Void();
-		return workloadMain(this, cx);
+		return workloadMain(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>&) override {}
 
-	ACTOR static Future<Void> workloadMain(ExcludeIncludeStorageServersWorkload* self, Database cx) {
-		state int round = deterministicRandom()->randomInt(10, 80);
-		state std::set<AddressExclusion> servers;
-		state std::set<AddressExclusion>::iterator it;
-		state Transaction tr(cx);
-		state double timeout = 100.0;
-		state int timeoutCnt = 0;
-		state int maxTimeout = std::min(30, round);
+	Future<Void> workloadMain(Database cx) {
+		int round = deterministicRandom()->randomInt(10, 80);
+		std::set<AddressExclusion> servers;
+		std::set<AddressExclusion>::iterator it;
+		Transaction tr(cx);
+		double timeout = 100.0;
+		int timeoutCnt = 0;
+		int maxTimeout = std::min(30, round);
 		TraceEvent("WorkloadStart").detail("Round", round).log();
-		loop {
+		while (true) {
 			if (timeoutCnt > maxTimeout) {
 				TraceEvent("QuitEarlyNotCompleteServerExclude").log();
 				break;
 			}
+			Error err;
 			try {
 				servers.clear();
 				// including an invalid address means include everything(clear all exclude prefix)
-				wait(includeServers(cx, std::vector<AddressExclusion>(1)));
+				co_await includeServers(cx, std::vector<AddressExclusion>(1));
 
 				tr = Transaction(cx);
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@@ -100,15 +101,15 @@ struct ExcludeIncludeStorageServersWorkload : TestWorkload {
 
 				// get all storage servers
 				std::vector<std::pair<StorageServerInterface, ProcessClass>> results =
-				    wait(NativeAPI::getServerListAndProcessClasses(&tr));
+				    co_await NativeAPI::getServerListAndProcessClasses(&tr);
 				for (auto& [ssi, p] : results) {
-					if (!g_simulator->protectedAddresses.contains(ssi.address())) {
+					if (!g_simulator->isProtectedAddress(ssi.address())) {
 						servers.insert(AddressExclusion(ssi.address().ip, ssi.address().port));
 					}
 				}
 
 				// get all TLogs and remove from SS candidate set
-				Optional<Standalone<StringRef>> value = wait(tr.get(logsKey));
+				Optional<Standalone<StringRef>> value = co_await tr.get(logsKey);
 				ASSERT(value.present());
 				auto logs = decodeLogsValue(value.get());
 				for (auto const& log : logs.first) {
@@ -116,7 +117,7 @@ struct ExcludeIncludeStorageServersWorkload : TestWorkload {
 				}
 
 				// get all log routers and remove from SS candidate set
-				for (const auto& tLogSet : self->dbInfo->get().logSystemConfig.tLogs) {
+				for (const auto& tLogSet : dbInfo->get().logSystemConfig.tLogs) {
 					for (const auto& logRouter : tLogSet.logRouters) {
 						if (logRouter.present()) {
 							const auto& logRouterInterf = logRouter.interf();
@@ -134,28 +135,30 @@ struct ExcludeIncludeStorageServersWorkload : TestWorkload {
 
 				// find a SS and exclude it
 				it = std::next(servers.begin(), deterministicRandom()->randomInt(0, servers.size()));
-				wait(excludeServers(cx, std::vector<AddressExclusion>{ *it }));
+				co_await excludeServers(cx, std::vector<AddressExclusion>{ *it });
 				// timeoutError() is needed because sometimes excluding process can take forever
-				std::set<NetworkAddress> inProgress = wait(
-				    timeoutError(checkForExcludingServers(cx, std::vector<AddressExclusion>{ *it }, true), timeout));
+				std::set<NetworkAddress> inProgress = co_await timeoutError(
+				    checkForExcludingServers(cx, std::vector<AddressExclusion>{ *it }, true), timeout);
 				ASSERT(inProgress.empty());
 				if (--round <= 0) {
 					break;
 				}
 			} catch (Error& e) {
-				if (e.code() == error_code_timed_out) {
-					// it might never be excluded from serverList
-					timeoutCnt++;
-					continue;
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid() && err.code() == error_code_timed_out) {
+				// it might never be excluded from serverList
+				timeoutCnt++;
+				continue;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
 		// if it is still in the middle of a exclude, then DD cannot finish and test would timeout
-		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
+		co_await includeServers(cx, std::vector<AddressExclusion>(1));
 
 		TraceEvent("WorkloadFinish").detail("QuitEarly", round > 0).detail("TimeoutCount", timeoutCnt).log();
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/ExpectStableThroughput.actor.cpp b/fdbserver/workloads/ExpectStableThroughput.actor.cpp
deleted file mode 100644
index 2a4203e513d..00000000000
--- a/fdbserver/workloads/ExpectStableThroughput.actor.cpp
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * ExpectStableThroughput.actor.cpp
- */
-
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last include
-
-// This workload is meant to be run with the ThroughputQuotaWorklaod.
-// The ThroughputQuotaWorkload sets a total quota, and then this workload runs
-// with tagged transactions for a long duration, attempting to achieve a higher
-// throughput than the specified quota. The check phase of this workload then
-// verifies that the achieved throughput is near the total quota.
-//
-// TODO:
-//   - Test write workloads
-//   - Randomize the number of operations per transaction
-//   - Test multi-page operations
-class ExpectStableThroughputWorkload : public TestWorkload {
-	// Metrics:
-	uint64_t totalCost{ 0 };
-	double throttledDuration{ 0.0 };
-	int tagThrottledErrors{ 0 };
-
-	// Parameters:
-	double testDuration;
-	uint64_t expectedThroughputPagesRate;
-	TransactionTag throttlingTag;
-	double errorTolerance;
-	Key keyPrefix;
-	int numActors;
-	double attemptedTransactionRatePerActor;
-	double warmupTime;
-	int opsPerTransaction;
-
-	double startTime;
-
-	void finishTransaction(Transaction const& tr) {
-		if (now() > startTime + warmupTime) {
-			totalCost += tr.getTotalCost();
-			throttledDuration += tr.getTagThrottledDuration();
-		}
-	}
-
-	Key getKey(int index) { return Key(format("%06d", index)).withPrefix(keyPrefix); }
-
-	ACTOR static Future<Void> runTransaction(ExpectStableThroughputWorkload* self, Database cx) {
-		state Transaction tr(cx);
-		state std::vector<Future<Void>> futures;
-		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::AUTO_THROTTLE_TAG, self->throttlingTag);
-				futures.clear();
-				futures.reserve(self->opsPerTransaction);
-				for (int i = 0; i < self->opsPerTransaction; ++i) {
-					futures.push_back(success(tr.get(self->getKey(i))));
-				}
-				wait(waitForAll(futures));
-				self->finishTransaction(tr);
-				return Void();
-			} catch (Error& e) {
-				self->finishTransaction(tr);
-				if (e.code() == error_code_proxy_tag_throttled && now() > self->startTime + self->warmupTime) {
-					++self->tagThrottledErrors;
-				}
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> runClient(ExpectStableThroughputWorkload* self, Database cx) {
-		loop {
-			wait(delay(1 / self->attemptedTransactionRatePerActor) && runTransaction(self, cx));
-		}
-	}
-
-public:
-	static constexpr auto NAME = "ExpectStableThroughput";
-	explicit ExpectStableThroughputWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		testDuration = getOption(options, "testDuration"_sr, 1200.0);
-		throttlingTag = getOption(options, "throttlingTag"_sr, "testTag"_sr);
-		expectedThroughputPagesRate = getOption(options, "expectedThroughputPagesRate"_sr, 1);
-		errorTolerance = getOption(options, "errorTolerance"_sr, 0.2);
-		keyPrefix = getOption(options, "keyPrefix"_sr, "testKey"_sr);
-		numActors = getOption(options, "numActors"_sr, 100);
-		attemptedTransactionRatePerActor = getOption(options, "attemptedTransactionRatePerActor"_sr, 0.4);
-		warmupTime = getOption(options, "warmupTime"_sr, 10.0);
-		opsPerTransaction = getOption(options, "opsPerTransaction"_sr, 5);
-	}
-
-	Future<Void> setup(Database const& cx) override { return Void(); }
-
-	Future<Void> start(Database const& cx) override {
-		if (clientId != 0 || !SERVER_KNOBS->GLOBAL_TAG_THROTTLING) {
-			return Void();
-		}
-		startTime = now();
-		std::vector<Future<Void>> clients;
-		for (int i = 0; i < numActors; ++i) {
-			clients.push_back(runClient(this, cx));
-		}
-		return success(timeout(waitForAll(clients), warmupTime + testDuration));
-	}
-
-	Future<bool> check(Database const& cx) override {
-		if (clientId != 0 || !SERVER_KNOBS->GLOBAL_TAG_THROTTLING) {
-			return true;
-		}
-		auto const expectedTotalCost =
-		    testDuration * expectedThroughputPagesRate * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
-		bool const passed = (static_cast<double>(expectedTotalCost) * (1.0 - errorTolerance) <= totalCost) &&
-		                    (totalCost <= static_cast<double>(expectedTotalCost) * (1.0 + errorTolerance));
-		auto const severity = passed ? SevInfo : SevError;
-		TraceEvent(severity, "CheckingStableThroughput")
-		    .detail("ExpectedTotalCost", expectedTotalCost)
-		    .detail("ErrorTolerance", errorTolerance)
-		    .detail("TotalCost", totalCost);
-		return passed;
-	}
-	void getMetrics(std::vector<PerfMetric>& m) override {
-		m.emplace_back("Total Cost", totalCost, Averaged::False);
-		m.emplace_back("TagThrottled Errors", tagThrottledErrors, Averaged::False);
-		m.emplace_back("Total Throttling Duration", throttledDuration, Averaged::False);
-	}
-
-	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
-		out.insert("Attrition");
-		out.insert("RandomClogging");
-	}
-};
-
-WorkloadFactory<ExpectStableThroughputWorkload> ExpectStableThroughputWorkloadFactory;
diff --git a/fdbserver/workloads/ExternalWorkload.actor.cpp b/fdbserver/workloads/ExternalWorkload.cpp
similarity index 93%
rename from fdbserver/workloads/ExternalWorkload.actor.cpp
rename to fdbserver/workloads/ExternalWorkload.cpp
index 0ffda02640d..5e7d8468d53 100644
--- a/fdbserver/workloads/ExternalWorkload.actor.cpp
+++ b/fdbserver/workloads/ExternalWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * ExternalWorkload.actor.cpp
+ * ExternalWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,9 +22,7 @@
 #include "flow/Platform.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "foundationdb/CppWorkload.h"
-#include "fdbserver/workloads/workloads.actor.h"
-
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/tester/workloads.h"
 
 extern void flushTraceFileVoid();
 
@@ -33,7 +31,7 @@ namespace {
 template <class T>
 struct FDBPromiseImpl : FDBPromise {
 	Promise<T> impl;
-	FDBPromiseImpl(Promise<T> impl) : impl(impl) {}
+	explicit FDBPromiseImpl(Promise<T> impl) : impl(impl) {}
 	void send(void* value) override {
 		if (g_network->isOnMainThread()) {
 			impl.send(*reinterpret_cast<T*>(value));
@@ -46,10 +44,10 @@ struct FDBPromiseImpl : FDBPromise {
 	}
 };
 
-ACTOR template <class F, class T>
-void keepAlive(F until, T db) {
+template <class F, class T>
+Future<Void> keepAlive(F until, T db) {
 	try {
-		wait(success(until));
+		co_await until;
 	} catch (...) {
 	}
 }
@@ -264,23 +262,23 @@ class Workload : public FDBWorkload {
 	capi::FDBWorkload::FDBWorkload_VT* vt;
 
 public:
-	Workload(capi::FDBWorkload c_workload) : inner(c_workload.inner), vt(c_workload.vt) {}
+	explicit Workload(capi::FDBWorkload c_workload) : inner(c_workload.inner), vt(c_workload.vt) {}
 	~Workload() { this->vt->free(this->inner); }
 
-	virtual bool init(FDBWorkloadContext* context) override { return true; }
-	virtual void setup(FDBDatabase* db, GenericPromise<bool> done) override {
+	bool init(FDBWorkloadContext* context) override { return true; }
+	void setup(FDBDatabase* db, GenericPromise<bool> done) override {
 		return this->vt->setup(this->inner, (capi::FDBDatabase*)db, promise::wrap(done));
 	}
-	virtual void start(FDBDatabase* db, GenericPromise<bool> done) override {
+	void start(FDBDatabase* db, GenericPromise<bool> done) override {
 		return this->vt->start(this->inner, (capi::FDBDatabase*)db, promise::wrap(done));
 	}
-	virtual void check(FDBDatabase* db, GenericPromise<bool> done) override {
+	void check(FDBDatabase* db, GenericPromise<bool> done) override {
 		return this->vt->check(this->inner, (capi::FDBDatabase*)db, promise::wrap(done));
 	}
-	virtual void getMetrics(std::vector<FDBPerfMetric>& out) const override {
+	void getMetrics(std::vector<FDBPerfMetric>& out) const override {
 		this->vt->getMetrics(this->inner, metrics::wrap(&out));
 	}
-	virtual double getCheckTimeout() override { return this->vt->getCheckTimeout(this->inner); }
+	double getCheckTimeout() override { return this->vt->getCheckTimeout(this->inner); }
 };
 } // namespace translator
 
@@ -363,12 +361,11 @@ struct ExternalWorkload : TestWorkload, FDBWorkloadContext {
 		}
 	}
 
-	ACTOR Future<Void> assertTrue(StringRef stage, Future<bool> f) {
-		bool res = wait(f);
+	Future<Void> assertTrue(StringRef stage, Future<bool> f) {
+		bool res = co_await f;
 		if (!res) {
 			TraceEvent(SevError, "ExternalWorkloadFailure").detail("Stage", stage);
 		}
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override {
diff --git a/fdbserver/workloads/FailoverWithSSLag.actor.cpp b/fdbserver/workloads/FailoverWithSSLag.cpp
similarity index 69%
rename from fdbserver/workloads/FailoverWithSSLag.actor.cpp
rename to fdbserver/workloads/FailoverWithSSLag.cpp
index 70fb4f25d82..df99861592d 100644
--- a/fdbserver/workloads/FailoverWithSSLag.actor.cpp
+++ b/fdbserver/workloads/FailoverWithSSLag.cpp
@@ -1,5 +1,5 @@
 /*
- * FailoverWithSSLag.actor.cpp
+ * FailoverWithSSLag.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,17 +19,18 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/StatusClient.h"
-#include "flow/actorcompiler.h" // This must be the last include.
+#include "flow/CoroUtils.h"
 
 // This actor tests failover with remote tlogs being in sync with primary but with remote storage servers lagging
 // behind the primary. Failover shouldn't complete until the remote storage servers also get in sync with the primary.
@@ -41,12 +42,12 @@ struct FailoverWithSSLagWorkload : TestWorkload {
 	std::vector<IPAddress> tlogs; // remote tlogs
 	std::vector<IPAddress> storages; // remote storages
 
-	FailoverWithSSLagWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit FailoverWithSSLagWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled =
 		    !clientId && g_network->isSimulated(); // only do this on the "first" client, and only when in simulation
 		testDuration = getOption(options, "testDuration"_sr, 400.0);
 		testSuccess = true;
-		g_simulator->usableRegions = 2;
+		fdbSimulationPolicyState().usableRegions = 2;
 	}
 
 	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
@@ -100,7 +101,8 @@ struct FailoverWithSSLagWorkload : TestWorkload {
 
 		// Find all remote storage servers.
 		for (const auto& process : g_simulator->getAllProcesses()) {
-			if (process->locality.dcId().present() && process->locality.dcId() == g_simulator->remoteDcId &&
+			if (process->locality.dcId().present() &&
+			    process->locality.dcId() == fdbSimulationPolicyState().remoteDcId &&
 			    g_simulator->hasRole(process->address, "StorageServer")) {
 				storages.push_back(process->address.ip);
 			}
@@ -135,40 +137,40 @@ struct FailoverWithSSLagWorkload : TestWorkload {
 		return true;
 	}
 
-	ACTOR static Future<Optional<Version>> fetchStorageServerLag(FailoverWithSSLagWorkload* self, Database cx) {
-		state double startTime = now();
-		StatusObject result = wait(StatusClient::statusFetcher(cx));
+	Future<Optional<Version>> fetchStorageServerLag(Database cx) {
+		double startTime = now();
+		StatusObject result = co_await StatusClient::statusFetcher(cx);
 		double duration = now() - startTime;
 
 		StatusObjectReader statusObj(result);
 		StatusObjectReader statusObjCluster;
 		if (!statusObj.get("cluster", statusObjCluster)) {
 			TraceEvent("SSLagNoCluster");
-			return Optional<Version>();
+			co_return Optional<Version>();
 		}
 
 		// Fetch the lag between primary and remote tlogs.
 		Version tlogLagInVersions = 0;
 		double tlogLagInSeconds = 0;
-		if (!self->fetchLagFromStatusObject("logserver_lag", statusObjCluster, tlogLagInVersions, tlogLagInSeconds)) {
+		if (!fetchLagFromStatusObject("logserver_lag", statusObjCluster, tlogLagInVersions, tlogLagInSeconds)) {
 			TraceEvent("NoLogServerLagData");
-			return Optional<Version>();
+			co_return Optional<Version>();
 		}
 
 		// Fetch the lag between primary and remote storage servers.
 		Version ssLagInVersions = 0;
 		double ssLagInSeconds = 0;
-		if (!self->fetchLagFromStatusObject("storageserver_lag", statusObjCluster, ssLagInVersions, ssLagInSeconds)) {
+		if (!fetchLagFromStatusObject("storageserver_lag", statusObjCluster, ssLagInVersions, ssLagInSeconds)) {
 			TraceEvent("NoStorageServerLagData");
-			return Optional<Version>();
+			co_return Optional<Version>();
 		}
 
 		// Fetch the lag between primary and remote data centers.
 		Version dcLagInVersions = 0;
 		double dcLagInSeconds = 0;
-		if (!self->fetchLagFromStatusObject("datacenter_lag", statusObjCluster, dcLagInVersions, dcLagInSeconds)) {
+		if (!fetchLagFromStatusObject("datacenter_lag", statusObjCluster, dcLagInVersions, dcLagInSeconds)) {
 			TraceEvent("NoDataCenterLagData");
-			return Optional<Version>();
+			co_return Optional<Version>();
 		}
 
 		TraceEvent("LagInfo")
@@ -180,82 +182,86 @@ struct FailoverWithSSLagWorkload : TestWorkload {
 		    .detail("DataCenterLagInSeconds", dcLagInSeconds)
 		    .detail("TimeToFetchStatus", duration);
 
-		return ssLagInVersions;
+		co_return ssLagInVersions;
 	}
 
-	ACTOR static Future<Void> waitForRemoteDataCenterToLag(FailoverWithSSLagWorkload* self, Database cx) {
-		state Future<Optional<Version>> ssLag = Never();
-		loop choose {
-			when(wait(delay(5.0))) {
+	Future<Void> waitForRemoteDataCenterToLag(Database cx) {
+		Future<Optional<Version>> ssLag = Never();
+		while (true) {
+			auto choice = co_await race(delay(5.0), ssLag);
+			if (choice.index() == 0) {
+
 				// Fetch SS lag every 5s.
-				ssLag = self->fetchStorageServerLag(self, cx);
-			}
-			when(Optional<Version> versionLag = wait(ssLag)) {
+				ssLag = fetchStorageServerLag(cx);
+			} else if (choice.index() == 1) {
+				Optional<Version> versionLag = std::get<1>(std::move(choice));
+
 				if (versionLag.present() && versionLag.get() >= SERVER_KNOBS->MAX_VERSION_DIFFERENCE) {
 					TraceEvent("SSLag").detail("Versions", versionLag.get());
-					return Void();
+					co_return;
 				}
 				ssLag = Never();
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
 
-	ACTOR static Future<Void> failover(FailoverWithSSLagWorkload* self, Database cx) {
+	Future<Void> failover(Database cx) {
 		TraceEvent("FailoverBegin").log();
 
-		wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->disablePrimary, true)));
+		co_await ManagementAPI::changeConfig(cx.getReference(), fdbSimulationPolicyState().disablePrimary, true);
 		TraceEvent("Failover_WaitFor_PrimaryDatacenterKey").log();
 
 		// when failover, primaryDC should change to 1
-		wait(waitForPrimaryDC(cx, "1"_sr));
+		co_await waitForPrimaryDC(cx, "1"_sr);
 		TraceEvent("FailoverComplete").log();
-		return Void();
 	}
 
-	ACTOR static Future<Void> doFailover(FailoverWithSSLagWorkload* self, Database cx) {
-		state bool connectionsClogged = true;
-		state bool failoverCompleted = false;
-		loop choose {
-			// NOTE: We don't have a way of verifying that failover is blocked because of the
-			// data center/storage server lag. So verify that failover is blocked for 100 seconds
-			// (which is way longer than the time needed to complete failover) and then unclog
-			// connections and let failover make progress.
-			when(wait(delay(100.0))) {
+	Future<Void> doFailover(Database cx) {
+		bool connectionsClogged = true;
+		bool failoverCompleted = false;
+		while (true) {
+			auto choice = co_await race(delay(100.0), failover(cx));
+			if (choice.index() == 0) {
+
 				if (connectionsClogged) {
 					if (failoverCompleted) {
 						// Failover completed even while the remote storages are clogged, which
 						// shouldn't happen. Mark the test as failed.
-						self->testSuccess = false;
-						return Void();
+						testSuccess = false;
+						co_return;
 					}
-					self->clogUnclogRemoteStorages(false /* clog */);
+					clogUnclogRemoteStorages(false /* clog */);
 					connectionsClogged = false;
 				}
-			}
-			when(wait(self->failover(self, cx))) {
+			} else if (choice.index() == 1) {
+
 				if (connectionsClogged) {
 					// Failover completed even while the remote storages are clogged, which
 					// shouldn't happen. Mark the test as failed.
-					self->testSuccess = false;
-					return Void();
+					testSuccess = false;
+					co_return;
 				}
 				failoverCompleted = true;
 
 				// Verify that the storage server lag has gone below the threshold.
-				state Future<Optional<Version>> ssLag = self->fetchStorageServerLag(self, cx);
-				Optional<Version> versionLag = wait(ssLag);
+				Future<Optional<Version>> ssLag = fetchStorageServerLag(cx);
+				Optional<Version> versionLag = co_await ssLag;
 				if (versionLag.present() && versionLag.get() >= SERVER_KNOBS->MAX_VERSION_DIFFERENCE) {
 					TraceEvent("SSLag").detail("Versions", versionLag.get());
-					self->testSuccess = false;
+					testSuccess = false;
 				}
-				return Void();
+				co_return;
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
 
-	ACTOR Future<Void> clogClient(FailoverWithSSLagWorkload* self, Database cx) {
+	Future<Void> clogClient(FailoverWithSSLagWorkload* self, Database cx) {
 		while (self->dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
-			wait(self->dbInfo->onChange());
+			co_await self->dbInfo->onChange();
 		}
 
 		// Clog connections between remote tlogs and storage servers.
@@ -263,17 +269,15 @@ struct FailoverWithSSLagWorkload : TestWorkload {
 			// Couldn't find remote tlogs/storage servers. Probably configuration will
 			// need to be adjusted.
 			self->testSuccess = false;
-			return Void();
+			co_return;
 		}
 
 		// Wait until the data center/storage server lag goes above the threshold.
-		wait(self->waitForRemoteDataCenterToLag(self, cx));
+		co_await self->waitForRemoteDataCenterToLag(cx);
 
 		// Initiate failover and verify that it doesn't complete until the data center/
 		// storage server lag gets below the threshold.
-		wait(self->doFailover(self, cx));
-
-		return Void();
+		co_await self->doFailover(cx);
 	}
 };
 
diff --git a/fdbserver/workloads/FastTriggeredWatches.actor.cpp b/fdbserver/workloads/FastTriggeredWatches.cpp
similarity index 71%
rename from fdbserver/workloads/FastTriggeredWatches.actor.cpp
rename to fdbserver/workloads/FastTriggeredWatches.cpp
index ac5ee962b42..ac44d7d416d 100644
--- a/fdbserver/workloads/FastTriggeredWatches.actor.cpp
+++ b/fdbserver/workloads/FastTriggeredWatches.cpp
@@ -1,5 +1,5 @@
 /*
- * FastTriggeredWatches.actor.cpp
+ * FastTriggeredWatches.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
 
 struct FastTriggeredWatchesWorkload : TestWorkload {
 	static constexpr auto NAME = "FastTriggeredWatches";
@@ -34,7 +33,7 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
 	PerfIntCounter operations, retries;
 	Value defaultValue;
 
-	FastTriggeredWatchesWorkload(WorkloadContext const& wcx)
+	explicit FastTriggeredWatchesWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), operations("Operations"), retries("Retries") {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		nodes = getOption(options, "nodes"_sr, 100);
@@ -54,108 +53,114 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _setup(Database cx, FastTriggeredWatchesWorkload* self) {
-		state Transaction tr(cx);
+	Future<Void> _setup(Database cx, FastTriggeredWatchesWorkload* self) {
+		Transaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				for (int i = 0; i < self->nodes; i += 2)
 					tr.set(self->keyForIndex(i), self->defaultValue);
 
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0)
-			return _start(cx, this);
+			return _start(cx);
 		return Void();
 	}
 
-	ACTOR Future<Version> setter(Database cx, Key key, Optional<Value> value) {
-		state ReadYourWritesTransaction tr(cx);
+	Future<Version> setter(Database cx, Key key, Optional<Value> value) {
+		ReadYourWritesTransaction tr(cx);
 		// set the value of key and return the commit version
-		wait(delay(deterministicRandom()->random01()));
-		loop {
+		co_await delay(deterministicRandom()->random01());
+		while (true) {
+			Error err;
 			try {
 				if (value.present())
 					tr.set(key, value.get());
 				else
 					tr.clear(key);
 				//TraceEvent("FTWSetBegin").detail("Key", printable(key)).detail("Value", printable(value));
-				wait(tr.commit());
+				co_await tr.commit();
 				//TraceEvent("FTWSetEnd").detail("Key", printable(key)).detail("Value", printable(value)).detail("Ver", tr.getCommittedVersion());
-				return tr.getCommittedVersion();
+				co_return tr.getCommittedVersion();
 			} catch (Error& e) {
-				//TraceEvent("FTWSetError").error(e).detail("Key", printable(key)).detail("Value", printable(value));
-				wait(tr.onError(e));
+				err = e;
 			}
+			//TraceEvent("FTWSetError").error(e).detail("Key", printable(key)).detail("Value", printable(value));
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> _start(Database cx, FastTriggeredWatchesWorkload* self) {
-		state double testStart = now();
-		state Version lastReadVersion = 0;
+	Future<Void> _start(Database cx) {
+		double testStart = now();
+		Version lastReadVersion = 0;
 		try {
-			loop {
-				state double getDuration = 0;
-				state double watchEnd = 0;
-				state bool watchCommitted = false;
-				state Key setKey = self->keyForIndex(deterministicRandom()->randomInt(0, self->nodes));
-				state Optional<Value> setValue;
+			while (true) {
+				double getDuration = 0;
+				double watchEnd = 0;
+				bool watchCommitted = false;
+				Key setKey = keyForIndex(deterministicRandom()->randomInt(0, nodes));
+				Optional<Value> setValue;
 				if (deterministicRandom()->random01() > 0.5)
 					setValue = StringRef(format("%010d", deterministicRandom()->randomInt(0, 1000)));
 				// Set the value at setKey to something random
-				state Future<Version> setFuture = self->setter(cx, setKey, setValue);
-				wait(delay(deterministicRandom()->random01()));
-				state Version watchCommitVersion = 0;
-				loop {
-					state ReadYourWritesTransaction tr(cx);
+				Future<Version> setFuture = setter(cx, setKey, setValue);
+				co_await delay(deterministicRandom()->random01());
+				Version watchCommitVersion = 0;
+				while (true) {
+					ReadYourWritesTransaction tr(cx);
 
+					Error err;
 					try {
-						Optional<Value> val = wait(tr.get(setKey));
+						Optional<Value> val = co_await tr.get(setKey);
 						if (watchCommitted) {
 							getDuration = now() - watchEnd;
 						}
 						lastReadVersion = tr.getReadVersion().get();
 						//TraceEvent("FTWGet").detail("Key", printable(setKey)).detail("Value", printable(val)).detail("Ver", tr.getReadVersion().get());
-						// if the value is already setValue then there is no point setting a watch so break out of the
-						// loop
+						// if the value is already setValue then there is no point setting a watch so break out of
+						// the loop
 						if (val == setValue)
 							break;
 						ASSERT(!watchCommitted);
 						tr.addWriteConflictRange(singleKeyRange(""_sr));
-						// set a watch and wait for it to be triggered (i.e for self->setter to set the value)
-						state Future<Void> watchFuture = tr.watch(setKey);
-						wait(tr.commit());
+						// set a watch and wait for it to be triggered (i.e for setter to set the value)
+						Future<Void> watchFuture = tr.watch(setKey);
+						co_await tr.commit();
 						watchCommitVersion = tr.getCommittedVersion();
 
 						//TraceEvent("FTWStartWatch").detail("Key", printable(setKey));
-						wait(watchFuture);
+						co_await watchFuture;
 						watchEnd = now();
 						watchCommitted = true;
 					} catch (Error& e) {
-						//TraceEvent("FTWWatchError").error(e).detail("Key", printable(setKey));
-						wait(tr.onError(e));
+						err = e;
+					}
+					//TraceEvent("FTWWatchError").error(e).detail("Key", printable(setKey));
+					if (err.isValid()) {
+						co_await tr.onError(err);
 					}
 				}
-				Version keySetVersion = wait(setFuture);
+				Version keySetVersion = co_await setFuture;
 				int64_t versionDelta = lastReadVersion - std::max(keySetVersion, watchCommitVersion);
 				//TraceEvent("FTWWatchDone").detail("Key", printable(setKey));
 				// Assert that the time from setting the key to triggering the watch is no greater than 25s
 				ASSERT(!watchCommitted || versionDelta >= SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT ||
 				       versionDelta < SERVER_KNOBS->VERSIONS_PER_SECOND * (25 + getDuration));
 
-				if (now() - testStart > self->testDuration)
+				if (now() - testStart > testDuration)
 					break;
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			TraceEvent(SevError, "FastWatchError").errorUnsuppressed(e);
 			throw;
diff --git a/fdbserver/workloads/FileSystem.actor.cpp b/fdbserver/workloads/FileSystem.cpp
similarity index 70%
rename from fdbserver/workloads/FileSystem.actor.cpp
rename to fdbserver/workloads/FileSystem.cpp
index 65aae6880df..094b16d3517 100644
--- a/fdbserver/workloads/FileSystem.actor.cpp
+++ b/fdbserver/workloads/FileSystem.cpp
@@ -1,5 +1,5 @@
 /*
- * FileSystem.actor.cpp
+ * FileSystem.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,9 +20,8 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 struct FileSystemWorkload : TestWorkload {
 	static constexpr auto NAME = "FileSystem";
@@ -40,10 +39,10 @@ struct FileSystemWorkload : TestWorkload {
 	public:
 		virtual Future<Optional<Version>> run(FileSystemWorkload* self, Transaction* tr) = 0;
 		virtual const char* name() = 0;
-		virtual ~FileSystemOp() {}
+		virtual ~FileSystemOp() = default;
 	};
 
-	FileSystemWorkload(WorkloadContext const& wcx)
+	explicit FileSystemWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), queries("Queries"), writes("Latency"), latencies(), writeLatencies() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0) / clientCount;
@@ -64,8 +63,6 @@ struct FileSystemWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return nodeSetup(cx, this); }
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	Future<bool> check(Database const& cx) override {
 		clients.clear();
 		return true;
@@ -116,31 +113,32 @@ struct FileSystemWorkload : TestWorkload {
 		tr->set("/files/path/" + path, format("%016llx", id));
 	}
 
-	ACTOR Future<Void> setupRange(Database cx, FileSystemWorkload* self, int begin, int end) {
-		state Transaction tr(cx);
+	Future<Void> setupRange(Database cx, FileSystemWorkload* self, int begin, int end) {
+		Transaction tr(cx);
 		while (true) {
+			Error err;
 			try {
-				Optional<Value> f = wait(tr.get(self->keyForFileID(begin)));
+				Optional<Value> f = co_await tr.get(self->keyForFileID(begin));
 				if (f.present())
 					break; // The transaction already completed!
 
 				for (int n = begin; n < end; n++)
 					self->initializeFile(&tr, self, n);
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> nodeSetup(Database cx, FileSystemWorkload* self) {
-		state int i;
-		state std::vector<int> order;
-		state int nodesToSetUp = self->fileCount / self->clientCount + 1;
-		state int startingNode = nodesToSetUp * self->clientId;
-		state int batchCount = 5;
+	Future<Void> nodeSetup(Database cx, FileSystemWorkload* self) {
+		int i{ 0 };
+		std::vector<int> order;
+		int nodesToSetUp = self->fileCount / self->clientCount + 1;
+		int startingNode = nodesToSetUp * self->clientId;
+		int batchCount = 5;
 		for (int o = 0; o <= nodesToSetUp / batchCount; o++)
 			order.push_back(o * batchCount);
 		deterministicRandom()->randomShuffle(order);
@@ -154,42 +152,38 @@ struct FileSystemWorkload : TestWorkload {
 				    std::min(startingNode + order[i] + batchCount, nodesToSetUp * (self->clientId + 1))));
 				i++;
 			}
-			wait(waitForAll(fs));
+			co_await waitForAll(fs);
 		}
 		TraceEvent("FileSetupOK")
 		    .detail("ClientIdx", self->clientId)
 		    .detail("ClientCount", self->clientCount)
 		    .detail("StartingFile", startingNode)
 		    .detail("FilesToSetUp", nodesToSetUp);
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, FileSystemWorkload* self) {
-		state FileSystemOp* operation;
-		if (self->operationName == "deletionQuery")
+	Future<Void> start(Database const& cx) override {
+		FileSystemOp* operation;
+		if (operationName == "deletionQuery")
 			operation = new ServerDeletionCountQuery();
 		else
 			operation = new RecentModificationQuery();
-		wait(timeout(self->operationClient(cx, self, operation, 0.01), 1.0, Void()));
-		self->queries.clear();
-		self->writes.clear();
+		co_await timeout(operationClient(cx, this, operation, 0.01), 1.0, Void());
+		queries.clear();
+		writes.clear();
 
-		if (self->performingWrites) {
-			for (int c = 0; c < self->writeActorCount; c++) {
-				self->clients.push_back(timeout(self->writeClient(cx, self), self->testDuration, Void()));
+		if (performingWrites) {
+			for (int c = 0; c < writeActorCount; c++) {
+				clients.push_back(timeout(writeClient(cx, this), testDuration, Void()));
 			}
 		}
-		for (int c = 0; c < self->actorCount; c++) {
-			self->clients.push_back(
-			    timeout(self->operationClient(cx, self, operation, self->actorCount / self->transactionsPerSecond),
-			            self->testDuration,
-			            Void()));
+		for (int c = 0; c < actorCount; c++) {
+			clients.push_back(timeout(
+			    operationClient(cx, this, operation, actorCount / transactionsPerSecond), testDuration, Void()));
 		}
-		wait(waitForAll(self->clients));
+		co_await waitForAll(clients);
 
-		wait(delay(0.01)); // Make sure the deletion happens after actor cancellation
+		co_await delay(0.01); // Make sure the deletion happens after actor cancellation
 		delete operation;
-		return Void();
 	}
 
 	bool shouldRecord(double clientBegin) {
@@ -198,21 +192,23 @@ struct FileSystemWorkload : TestWorkload {
 		       (n > (clientBegin + testDuration * 0.125) && n < (clientBegin + testDuration * 0.875));
 	}
 
-	ACTOR Future<Void> operationClient(Database cx, FileSystemWorkload* self, FileSystemOp* operation, double delay) {
-		state double clientBegin = now();
-		state double lastTime = now();
-		loop {
-			wait(poisson(&lastTime, delay));
-			state double tstart = now();
-			state Transaction tr(cx);
-			loop {
+	Future<Void> operationClient(Database cx, FileSystemWorkload* self, FileSystemOp* operation, double delay) {
+		double clientBegin = now();
+		double lastTime = now();
+		while (true) {
+			co_await poisson(&lastTime, delay);
+			double tstart = now();
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					Optional<Version> ver = wait(operation->run(self, &tr));
+					Optional<Version> ver = co_await operation->run(self, &tr);
 					if (ver.present())
 						break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 			if (self->shouldRecord(clientBegin)) {
 				++self->queries;
@@ -228,22 +224,23 @@ struct FileSystemWorkload : TestWorkload {
 		return x;
 	}
 
-	ACTOR Future<Void> writeClient(Database cx, FileSystemWorkload* self) {
-		state double clientBegin = now();
-		loop {
-			state int fileID = deterministicRandom()->randomInt(0, self->fileCount);
-			state bool isDeleting = deterministicRandom()->random01() < 0.25;
-			state int size = isDeleting ? 0 : deterministicRandom()->randomInt(0, std::numeric_limits<int>::max());
-			state std::string keyStr = self->keyForFileID(fileID).toString();
-			state double tstart = now();
-			state Transaction tr(cx);
-			loop {
+	Future<Void> writeClient(Database cx, FileSystemWorkload* self) {
+		double clientBegin = now();
+		while (true) {
+			int fileID = deterministicRandom()->randomInt(0, self->fileCount);
+			bool isDeleting = deterministicRandom()->random01() < 0.25;
+			int size = isDeleting ? 0 : deterministicRandom()->randomInt(0, std::numeric_limits<int>::max());
+			std::string keyStr = self->keyForFileID(fileID).toString();
+			double tstart = now();
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					state double time = now();
+					double time = now();
 					if (isDeleting) {
-						state Optional<Value> deleted = wait(tr.get(StringRef(keyStr + "/deleted")));
+						Optional<Value> deleted = co_await tr.get(StringRef(keyStr + "/deleted"));
 						ASSERT(deleted.present());
-						Optional<Value> serverStr = wait(tr.get(StringRef(keyStr + "/server")));
+						Optional<Value> serverStr = co_await tr.get(StringRef(keyStr + "/server"));
 						ASSERT(serverStr.present());
 						int serverID = testKeyToInt(serverStr.get());
 						if (deleted.get().toString() == "1") {
@@ -258,11 +255,12 @@ struct FileSystemWorkload : TestWorkload {
 						tr.set(keyStr + "/size", format("%d", size));
 					}
 					tr.set(keyStr + "/lastupdated", doubleToTestKey(time));
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 			if (self->shouldRecord(clientBegin)) {
 				++self->writes;
@@ -271,13 +269,13 @@ struct FileSystemWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Optional<Version>> modificationQuery(FileSystemWorkload* self, Transaction* tr) {
-		state uint64_t userID = deterministicRandom()->randomInt(0, self->userIDCount);
-		state std::string base = format("/files/user/%016llx", userID);
+	Future<Optional<Version>> modificationQuery(FileSystemWorkload* self, Transaction* tr) {
+		uint64_t userID = deterministicRandom()->randomInt(0, self->userIDCount);
+		std::string base = format("/files/user/%016llx", userID);
 		if (self->loggingQueries)
 			TraceEvent("UserQuery").detail("UserID", userID).detail("PathBase", base);
 		Key keyEnd(base + "/updated0");
-		RangeResult val = wait(tr->getRange(firstGreaterOrEqual(keyEnd) - 10, firstGreaterOrEqual(keyEnd), 10));
+		RangeResult val = co_await tr->getRange(firstGreaterOrEqual(keyEnd) - 10, firstGreaterOrEqual(keyEnd), 10);
 		Key keyBegin(base + "/updated/");
 		for (int i = val.size() - 1; i >= 0; i--) {
 			if (val[i].key.startsWith(keyBegin) && self->loggingQueries) {
@@ -288,23 +286,23 @@ struct FileSystemWorkload : TestWorkload {
 				break;
 			}
 		}
-		return Optional<Version>(Version(0));
+		co_return Optional<Version>(Version(0));
 	}
 
-	ACTOR Future<Optional<Version>> deletionQuery(FileSystemWorkload* self, Transaction* tr) {
-		state uint64_t serverID = deterministicRandom()->randomInt(0, self->serverCount);
-		state std::string base = format("/files/server/%08x/deleted", serverID);
+	Future<Optional<Version>> deletionQuery(FileSystemWorkload* self, Transaction* tr) {
+		uint64_t serverID = deterministicRandom()->randomInt(0, self->serverCount);
+		std::string base = format("/files/server/%08x/deleted", serverID);
 		if (self->loggingQueries)
 			TraceEvent("DeletionQuery").detail("ServerID", serverID).detail("PathBase", base);
-		state Key keyBegin(base + "/");
-		state Key keyEnd(base + "0");
-		state KeySelectorRef begin = firstGreaterThan(keyBegin);
-		state KeySelectorRef end = firstGreaterOrEqual(keyEnd);
-		state int transferred = 1000;
-		state int transferSize = 1000;
-		state uint64_t deletedFiles = 0;
+		Key keyBegin(base + "/");
+		Key keyEnd(base + "0");
+		KeySelectorRef begin = firstGreaterThan(keyBegin);
+		KeySelectorRef end = firstGreaterOrEqual(keyEnd);
+		int transferred = 1000;
+		int transferSize = 1000;
+		uint64_t deletedFiles = 0;
 		while (transferred == transferSize) {
-			RangeResult val = wait(tr->getRange(begin, end, transferSize));
+			RangeResult val = co_await tr->getRange(begin, end, transferSize);
 			transferred = val.size();
 			deletedFiles += transferred;
 			begin = begin + transferred;
@@ -315,7 +313,7 @@ struct FileSystemWorkload : TestWorkload {
 			    .detail("PathBase", base)
 			    .detail("DeletedFiles", deletedFiles);
 		}
-		return Optional<Version>(Version(0));
+		co_return Optional<Version>(Version(0));
 	}
 
 	class RecentModificationQuery : public FileSystemOp {
diff --git a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp b/fdbserver/workloads/FuzzApiCorrectness.cpp
similarity index 89%
rename from fdbserver/workloads/FuzzApiCorrectness.actor.cpp
rename to fdbserver/workloads/FuzzApiCorrectness.cpp
index c2fade5b1ef..3e4dbbf5c47 100644
--- a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/FuzzApiCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * FuzzApiCorrectness.actor.cpp
+ * FuzzApiCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,22 +22,22 @@
 #include <mutex>
 #include <functional>
 #include <sstream>
+#include <type_traits>
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbclient/GenericManagementAPI.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "flow/ActorCollection.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Arena.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace ph = std::placeholders;
 
-// FIXME(gglass): figure out what this test is actually tring to test, and
+// FIXME(gglass): figure out what this test is actually trying to test, and
 // restructure it to be useful in a post-tenant world.  For now leave the
 // tenant crap in so that the distribution of keys/values generated is
 // actually sane.  An earlier attempt at removing the tenant stuff caused
@@ -47,14 +47,12 @@ namespace ph = std::placeholders;
 // Maybe we should put it out of its misery i.e. delete it.  TBD.
 
 // Putting back some tenant crap so that this test works
-typedef StringRef TenantNameRef;
-typedef Standalone<TenantNameRef> TenantName;
-typedef StringRef TenantGroupNameRef;
-typedef Standalone<TenantGroupNameRef> TenantGroupName;
+using TenantNameRef = StringRef;
+using TenantName = Standalone<TenantNameRef>;
 
 // This allows us to dictate which exceptions we SHOULD get.
 // We can use this to suppress expected exceptions, and take action
-// if we don't get an exception wqe should have gotten.
+// if we don't get an exception we should have gotten.
 struct ExceptionContract {
 	enum occurance_t { Never = 0, Possible = 1, Always = 2 };
 
@@ -147,13 +145,12 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 	std::set<TenantName> createdTenants;
 	int numTenants;
-	int numTenantGroups;
 	int minTenantNum = -1;
 
 	// Map from tenant number to key prefix
 	std::map<int, std::string> keyPrefixes;
 
-	FuzzApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), operationId(0), success(true) {
+	explicit FuzzApiCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), operationId(0), success(true) {
 		std::call_once(onceFlag, [&]() { addTestCases(); });
 
 		testDuration = getOption(options, "testDuration"_sr, 60.0);
@@ -170,7 +167,6 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 		specialKeysWritesEnabled = useSystemKeys && deterministicRandom()->coinflip();
 
 		numTenants = 0;
-		numTenantGroups = 0;
 
 		// See https://github.com/apple/foundationdb/issues/2424
 		if (BUGGIFY) {
@@ -223,14 +219,6 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	}
 
 	static TenantName getTenant(int num) { return TenantNameRef(format("tenant_%d", num)); }
-	Optional<TenantGroupName> getTenantGroup(int num) {
-		int groupNum = num % (numTenantGroups + 1);
-		if (groupNum == numTenantGroups - 1) {
-			return Optional<TenantGroupName>();
-		} else {
-			return TenantGroupNameRef(format("tenantgroup_%d", groupNum));
-		}
-	}
 	bool canUseTenant(Optional<TenantName> tenant) {
 		return !tenant.present() || createdTenants.contains(tenant.get());
 	}
@@ -243,13 +231,12 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _setup(Database cx, FuzzApiCorrectnessWorkload* self) {
-		Reference<IDatabase> db = wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
+	Future<Void> _setup(Database cx, FuzzApiCorrectnessWorkload* self) {
+		Reference<IDatabase> db =
+		    co_await unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx));
 		self->db = db;
 
 		ASSERT(self->numTenants == 0);
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -291,42 +278,49 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 	// Prevent a write only transaction whose commit was previously cancelled from being reordered after this
 	// transaction
-	ACTOR Future<Void> writeBarrier(Reference<IDatabase> db) {
-		state Reference<ITransaction> tr = db->createTransaction();
-		loop {
+	Future<Void> writeBarrier(Reference<IDatabase> db) {
+		Reference<ITransaction> tr = db->createTransaction();
+		while (true) {
+			Error err;
+			bool hasErr = false;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 				// Write-only transactions have a self-conflict in the system keys
 				tr->addWriteConflictRange(allKeys);
 				tr->clear(normalKeys);
-				wait(unsafeThreadFutureToFuture(tr->commit()));
-				return Void();
+				co_await unsafeThreadFutureToFuture(tr->commit());
+				break;
 			} catch (Error& e) {
-				wait(unsafeThreadFutureToFuture(tr->onError(e)));
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
+				co_await unsafeThreadFutureToFuture(tr->onError(err));
 			}
 		}
 	}
 
-	ACTOR Future<Void> loadAndRun(FuzzApiCorrectnessWorkload* self, Database cx) {
-		state double startTime = now();
-		state int nodesPerTenant = std::max<int>(1, self->nodes / (self->numTenants + 1));
-		state int keysPerBatch =
+	Future<Void> loadAndRun(FuzzApiCorrectnessWorkload* self, Database cx) {
+		double startTime = now();
+		int nodesPerTenant = std::max<int>(1, self->nodes / (self->numTenants + 1));
+		int keysPerBatch =
 		    std::min<int64_t>(1000,
 		                      1 + CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT / 2 /
 		                              (self->getKeyForIndex(-1, nodesPerTenant).size() + self->valueSizeRange.second));
 		try {
-			loop {
-				state int tenantNum = self->minTenantNum;
-				for (; tenantNum < self->numTenants; ++tenantNum) {
-					state int i = 0;
-					wait(self->writeBarrier(self->db));
-					for (; i < nodesPerTenant; i += keysPerBatch) {
+			while (true) {
+				// TODO: Remove tenant-related logic
+				for (int tenantNum = self->minTenantNum; tenantNum < self->numTenants; ++tenantNum) {
+					co_await self->writeBarrier(self->db);
+					for (int i = 0; i < nodesPerTenant; i += keysPerBatch) {
 						ASSERT(tenantNum < 0);
-						state Reference<ITransaction> tr = self->db->createTransaction();
-						loop {
+						Reference<ITransaction> tr = self->db->createTransaction();
+						while (true) {
 							if (now() - startTime > self->testDuration)
-								return Void();
+								co_return;
+							Error err;
+							bool hasErr = false;
 							try {
 								if (self->useSystemKeys && tenantNum == -1) {
 									tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@@ -355,26 +349,30 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 										}
 									}
 								}
-								wait(unsafeThreadFutureToFuture(tr->commit()));
+								co_await unsafeThreadFutureToFuture(tr->commit());
 								//TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr->getCommittedVersion());
 								break;
 							} catch (Error& e) {
-								wait(unsafeThreadFutureToFuture(tr->onError(e)));
+								err = e;
+								hasErr = true;
+							}
+							if (hasErr) {
+								co_await unsafeThreadFutureToFuture(tr->onError(err));
 							}
 						}
 					}
 				}
 
-				loop {
+				while (true) {
 					try {
-						wait(self->randomTransaction(self, cx) && delay(self->numOps * .001));
+						co_await (self->randomTransaction(self, cx) && delay(self->numOps * .001));
 					} catch (Error& e) {
 						if (e.code() != error_code_not_committed)
 							throw e;
 						break;
 					}
 					if (now() - startTime > self->testDuration)
-						return Void();
+						co_return;
 				}
 			}
 		} catch (Error& e) {
@@ -383,20 +381,20 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> randomTransaction(FuzzApiCorrectnessWorkload* self, Database cx) {
-		state Reference<ITransaction> tr;
-		state bool readYourWritesDisabled = deterministicRandom()->coinflip();
-		state bool readAheadDisabled = deterministicRandom()->coinflip();
-		state std::vector<Future<Void>> operations;
-		state int waitLocation = 0;
+	Future<Void> randomTransaction(FuzzApiCorrectnessWorkload* self, Database cx) {
+		Reference<ITransaction> tr;
+		bool readYourWritesDisabled = deterministicRandom()->coinflip();
+		bool readAheadDisabled = deterministicRandom()->coinflip();
+		std::vector<Future<Void>> operations;
+		int waitLocation = 0;
 
-		state int tenantNum = -1;
+		int tenantNum = -1;
 		tr = self->db->createTransaction();
 
-		state bool rawAccess = tenantNum == -1 && deterministicRandom()->coinflip();
+		bool rawAccess = tenantNum == -1 && deterministicRandom()->coinflip();
 
-		loop {
-			state bool cancelled = false;
+		while (true) {
+			bool cancelled = false;
 			if (readYourWritesDisabled)
 				tr->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
 			if (readAheadDisabled)
@@ -415,17 +413,17 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 			}
 			tr->addWriteConflictRange(self->conflictRange);
 
+			Error err;
+			bool hasErr = false;
 			try {
-				state int numWaits = deterministicRandom()->randomInt(1, 5);
-				state int i = 0;
+				int numWaits = deterministicRandom()->randomInt(1, 5);
 
 				// Code path here will create a bunch of ops, wait for ONE of them to complete,
 				// then add more ops, wait for another one, and so on.
-				for (; i < numWaits; i++) {
-					state int numOps = deterministicRandom()->randomInt(1, self->numOps);
-					state int j = 0;
-					for (; j < numOps; j++) {
-						state int operationType = deterministicRandom()->randomInt(0, testCases.size());
+				for (int i = 0; i < numWaits; ++i) {
+					int numOps = deterministicRandom()->randomInt(1, self->numOps);
+					for (int j = 0; j < numOps; ++j) {
+						int operationType = deterministicRandom()->randomInt(0, testCases.size());
 						printf("%d: Selected Operation %d\n", self->operationId + 1, operationType);
 						try {
 							operations.push_back(testCases[operationType](++self->operationId, self, tr));
@@ -440,14 +438,14 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 					// Wait for a random op to complete.
 					if (waitLocation < operations.size()) {
 						int waitOp = deterministicRandom()->randomInt(waitLocation, operations.size());
-						wait(operations[waitOp]);
-						wait(delay(0.000001)); // to ensure errors have propagated from reads to commits
+						co_await operations[waitOp];
+						co_await delay(0.000001); // to ensure errors have propagated from reads to commits
 						waitLocation = operations.size();
 					}
 				}
-				wait(waitForAll(operations));
+				co_await waitForAll(operations);
 				try {
-					wait(timeoutError(unsafeThreadFutureToFuture(tr->commit()), 30));
+					co_await timeoutError(unsafeThreadFutureToFuture(tr->commit()), 30);
 				} catch (Error& e) {
 					if (e.code() == error_code_client_invalid_operation ||
 					    e.code() == error_code_transaction_too_large || e.code() == error_code_invalid_option) {
@@ -456,61 +454,83 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
+				err = e;
+				hasErr = true;
+			}
+			if (hasErr) {
 				operations.clear();
 				waitLocation = 0;
 
-				if (e.code() == error_code_not_committed || e.code() == error_code_commit_unknown_result || cancelled) {
+				if (err.code() == error_code_not_committed || err.code() == error_code_commit_unknown_result ||
+				    err.code() == error_code_transaction_cancelled || cancelled) {
 					throw not_committed();
 				}
 
 				try {
-					wait(unsafeThreadFutureToFuture(tr->onError(e)));
-				} catch (Error& e) {
-					if (e.code() == error_code_transaction_timed_out) {
+					co_await unsafeThreadFutureToFuture(tr->onError(err));
+				} catch (Error& err) {
+					if (err.code() == error_code_transaction_timed_out) {
 						throw not_committed();
 					}
-					throw e;
+					throw err;
 				}
 			}
 		}
-		return Void();
 	}
 
 	template <typename Subclass, typename T>
 	struct BaseTest {
-		typedef T value_type;
-
-		ACTOR static Future<Void> runTest(unsigned int id, FuzzApiCorrectnessWorkload* wl, Reference<ITransaction> tr) {
-			state Subclass self(id, wl, tr);
+		using value_type = T;
 
+		static Future<Void> runTest(unsigned int id, FuzzApiCorrectnessWorkload* wl, Reference<ITransaction> tr) {
+			auto self = std::make_unique<Subclass>(id, wl, tr);
 			try {
-				value_type result = wait(timeoutError(BaseTest::runTest2(tr, &self), 1000));
-				self.contract.handleNotThrown(tr);
-				return self.errorCheck(tr, result);
+				Future<value_type> future = unsafeThreadFutureToFuture(self->createFuture(tr));
+				return runTestAsync(tr, std::move(self), future);
 			} catch (Error& e) {
-				self.contract.handleException(e, tr);
+				return handleCaughtException(tr, std::move(self), e);
 			}
-			return Void();
 		}
 
-		ACTOR static Future<value_type> runTest2(Reference<ITransaction> tr, Subclass* self) {
-			state Future<value_type> future = unsafeThreadFutureToFuture(self->createFuture(tr));
+		static Future<Void> runTestAsync(Reference<ITransaction> tr,
+		                                 std::unique_ptr<Subclass> self,
+		                                 Future<value_type> future) {
+			Optional<Error> err;
+			try {
+				// Create may have added some other things to do before we can get the result.
+				for (auto i = self->pre_steps.begin(); i != self->pre_steps.end(); ++i) {
+					co_await unsafeThreadFutureToFuture(*i);
+				}
 
-			// Create may have added some other things to do before
-			// we can get the result.
-			state std::vector<ThreadFuture<Void>>::iterator i;
-			for (i = self->pre_steps.begin(); i != self->pre_steps.end(); ++i) {
-				wait(unsafeThreadFutureToFuture(*i));
+				co_await timeoutError(ready(future), 1000);
+
+				if (future.isError()) {
+					err = future.getError();
+				} else {
+					ASSERT(future.isValid());
+					self->contract.handleNotThrown(tr);
+					if constexpr (std::is_same_v<value_type, Void>) {
+						self->errorCheck(tr, Void());
+					} else {
+						self->errorCheck(tr, future.get());
+					}
+				}
+			} catch (Error& e) {
+				err = e;
 			}
-
-			value_type result = wait(future);
-			if (future.isError()) {
-				self->contract.handleException(future.getError(), tr);
-			} else {
-				ASSERT(future.isValid());
+			if (err.present()) {
+				co_await handleCaughtException(tr, std::move(self), err.get());
 			}
+		}
 
-			return result;
+		static Future<Void> handleCaughtException(Reference<ITransaction> tr,
+		                                          std::unique_ptr<Subclass> self,
+		                                          Error err) {
+			self->contract.handleException(err, tr);
+			try {
+				co_await unsafeThreadFutureToFuture(tr.castTo<ThreadSafeTransaction>()->checkDeferredError());
+			} catch (Error&) {
+			}
 		}
 
 		virtual ThreadFuture<value_type> createFuture(Reference<ITransaction> tr) = 0;
@@ -666,7 +686,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 
 	template <typename Subclass>
 	struct BaseTestCallback : BaseTest<Subclass, Void> {
-		typedef typename BaseTest<Subclass, Void>::value_type value_type;
+		using value_type = typename BaseTest<Subclass, Void>::value_type;
 
 		BaseTestCallback(unsigned int id, FuzzApiCorrectnessWorkload* wl, const char* func)
 		  : BaseTest<Subclass, Void>(id, wl, func) {}
@@ -686,7 +706,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestSetVersion : public BaseTest<TestSetVersion, Version> {
-		typedef BaseTest<TestSetVersion, Version> base_type;
+		using base_type = BaseTest<TestSetVersion, Version>;
 		Version v;
 
 		TestSetVersion(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -718,7 +738,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGet : public BaseTest<TestGet, Optional<Value>> {
-		typedef BaseTest<TestGet, Optional<Value>> base_type;
+		using base_type = BaseTest<TestGet, Optional<Value>>;
 		Key key;
 
 		TestGet(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -756,7 +776,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGetKey : public BaseTest<TestGetKey, Key> {
-		typedef BaseTest<TestGetKey, Key> base_type;
+		using base_type = BaseTest<TestGetKey, Key>;
 		KeySelector keysel;
 
 		TestGetKey(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -781,7 +801,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGetRange0 : public BaseTest<TestGetRange0, RangeResult> {
-		typedef BaseTest<TestGetRange0, RangeResult> base_type;
+		using base_type = BaseTest<TestGetRange0, RangeResult>;
 		KeySelector keysel1, keysel2;
 		int limit;
 
@@ -832,7 +852,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGetRange1 : public BaseTest<TestGetRange1, RangeResult> {
-		typedef BaseTest<TestGetRange1, RangeResult> base_type;
+		using base_type = BaseTest<TestGetRange1, RangeResult>;
 		KeySelector keysel1, keysel2;
 		GetRangeLimits limits;
 
@@ -878,7 +898,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGetRange2 : public BaseTest<TestGetRange2, RangeResult> {
-		typedef BaseTest<TestGetRange2, RangeResult> base_type;
+		using base_type = BaseTest<TestGetRange2, RangeResult>;
 		Key key1, key2;
 		int limit;
 
@@ -939,7 +959,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGetRange3 : public BaseTest<TestGetRange3, RangeResult> {
-		typedef BaseTest<TestGetRange3, RangeResult> base_type;
+		using base_type = BaseTest<TestGetRange3, RangeResult>;
 		Key key1, key2;
 		GetRangeLimits limits;
 
@@ -994,7 +1014,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestGetAddressesForKey : public BaseTest<TestGetAddressesForKey, Standalone<VectorRef<const char*>>> {
-		typedef BaseTest<TestGetAddressesForKey, Standalone<VectorRef<const char*>>> base_type;
+		using base_type = BaseTest<TestGetAddressesForKey, Standalone<VectorRef<const char*>>>;
 		Key key;
 
 		TestGetAddressesForKey(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1016,7 +1036,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestAddReadConflictRange : public BaseTestCallback<TestAddReadConflictRange> {
-		typedef BaseTest<TestAddReadConflictRange, Void> base_type;
+		using base_type = BaseTest<TestAddReadConflictRange, Void>;
 		Key key1, key2;
 
 		TestAddReadConflictRange(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1038,7 +1058,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestAtomicOp : public BaseTestCallback<TestAtomicOp> {
-		typedef BaseTest<TestAtomicOp, Void> base_type;
+		using base_type = BaseTest<TestAtomicOp, Void>;
 		Key key;
 		Value value;
 		uint8_t op;
@@ -1110,7 +1130,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestSet : public BaseTestCallback<TestSet> {
-		typedef BaseTest<TestSet, Void> base_type;
+		using base_type = BaseTest<TestSet, Void>;
 		Key key;
 		Value value;
 
@@ -1147,7 +1167,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestClear0 : public BaseTestCallback<TestClear0> {
-		typedef BaseTest<TestClear0, Void> base_type;
+		using base_type = BaseTest<TestClear0, Void>;
 		Key key1, key2;
 
 		TestClear0(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1185,7 +1205,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestClear1 : public BaseTestCallback<TestClear1> {
-		typedef BaseTest<TestClear1, Void> base_type;
+		using base_type = BaseTest<TestClear1, Void>;
 		Key key1, key2;
 
 		TestClear1(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1223,7 +1243,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestClear2 : public BaseTestCallback<TestClear2> {
-		typedef BaseTest<TestClear2, Void> base_type;
+		using base_type = BaseTest<TestClear2, Void>;
 		Key key;
 
 		TestClear2(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1251,7 +1271,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestWatch : public BaseTest<TestWatch, Void> {
-		typedef BaseTest<TestWatch, Void> base_type;
+		using base_type = BaseTest<TestWatch, Void>;
 		Key key;
 
 		TestWatch(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1281,7 +1301,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestAddWriteConflictRange : public BaseTestCallback<TestAddWriteConflictRange> {
-		typedef BaseTest<TestAddWriteConflictRange, Void> base_type;
+		using base_type = BaseTest<TestAddWriteConflictRange, Void>;
 		Key key1, key2;
 
 		TestAddWriteConflictRange(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
@@ -1303,7 +1323,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestSetOption : public BaseTestCallback<TestSetOption> {
-		typedef BaseTest<TestSetOption, Void> base_type;
+		using base_type = BaseTest<TestSetOption, Void>;
 		int op;
 		Optional<Standalone<StringRef>> val;
 
@@ -1389,7 +1409,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 	};
 
 	struct TestOnError : public BaseTestCallback<TestOnError> {
-		typedef BaseTest<TestOnError, Void> base_type;
+		using base_type = BaseTest<TestOnError, Void>;
 		int errorcode;
 
 		TestOnError(unsigned int id, FuzzApiCorrectnessWorkload* workload, Reference<ITransaction> tr)
diff --git a/fdbserver/workloads/GcGenerations.actor.cpp b/fdbserver/workloads/GcGenerations.actor.cpp
deleted file mode 100644
index 48aa7814713..00000000000
--- a/fdbserver/workloads/GcGenerations.actor.cpp
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * GcGenerations.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/StatusClient.h"
-#include "fdbrpc/Locality.h"
-#include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbrpc/simulator.h"
-#include "flow/CodeProbe.h"
-#include "flow/NetworkAddress.h"
-#include "flow/Error.h"
-#include "flow/Trace.h"
-#include "flow/flow.h"
-#include "flow/network.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// This workload tests that when TRACK_TLOG_RECOVERY is turned on, older TLog generations can be garbage collected
-// during the recovery before reaching fully_recovered.
-struct GcGenerationsWorkload : TestWorkload {
-	static constexpr auto NAME = "GcGenerations";
-	bool enabled;
-	double testDuration;
-	double startDelay;
-	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
-
-	GcGenerationsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		enabled = !clientId; // only do this on the "first" client
-		testDuration = getOption(options, "testDuration"_sr, 1000.0);
-		startDelay = getOption(options, "startDelay"_sr, 30.0);
-	}
-
-	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
-		out.insert("Attrition");
-		out.insert("RandomClogging");
-	}
-
-	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override {
-		if (g_network->isSimulated() && enabled)
-			return timeout(reportErrors(gcGenerationsTestClient(this, cx), "GcGenerationsError"), testDuration, Void());
-		else
-			return Void();
-	}
-	Future<bool> check(Database const& cx) override { return true; }
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-
-	void unclogAll() {
-		TraceEvent("GcGenerationsUnclogRemote").detail("UnclogConnectionCount", cloggedPairs.size());
-		// unclog previously clogged connections
-		for (const auto& pair : cloggedPairs) {
-			g_simulator->unclogPair(pair.first, pair.second);
-		}
-		cloggedPairs.clear();
-	}
-
-	ACTOR Future<Void> clogRemoteDc(GcGenerationsWorkload* self, Database cx) {
-		Optional<ClusterConnectionString> csOptional = wait(getConnectionString(cx));
-		state std::vector<NetworkAddress> coordinators;
-		if (csOptional.present()) {
-			ClusterConnectionString cs = csOptional.get();
-			wait(store(coordinators, cs.tryResolveHostnames()));
-		}
-
-		auto isCoordinator = [](const std::vector<NetworkAddress>& coordinators, const IPAddress& ip) {
-			for (const auto& c : coordinators) {
-				if (c.ip == ip) {
-					return true;
-				}
-			}
-			return false;
-		};
-
-		std::vector<IPAddress> ips; // all non-remote process IPs
-		std::vector<IPAddress> remoteIps; // all remote process IPs
-		for (const auto& process : g_simulator->getAllProcesses()) {
-			const auto& ip = process->address.ip;
-			if (process->locality.dcId().present() && process->locality.dcId() == g_simulator->remoteDcId &&
-			    !isCoordinator(coordinators, ip)) {
-				remoteIps.push_back(ip);
-			} else {
-				ips.push_back(ip);
-			}
-		}
-		ASSERT(ips.size() > 0);
-		ASSERT(remoteIps.size() > 0);
-
-		for (const auto& ip : ips) {
-			for (const auto& remoteIp : remoteIps) {
-				g_simulator->clogPair(ip, remoteIp, 10000);
-				g_simulator->clogPair(remoteIp, ip, 10000);
-				self->cloggedPairs.emplace_back(ip, remoteIp);
-				self->cloggedPairs.emplace_back(remoteIp, ip);
-			}
-		}
-
-		TraceEvent("PartitionRemoteDc")
-		    .detail("RemoteDc", g_simulator->remoteDcId)
-		    .detail("CloggedRemoteProcess", describe(remoteIps));
-		return Void();
-	}
-
-	ACTOR Future<Void> dbAvailable(GcGenerationsWorkload* self) {
-		while (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-			wait(self->dbInfo->onChange());
-		}
-		return Void();
-	}
-
-	ACTOR Future<Void> generateMultipleTxnGenerations(GcGenerationsWorkload* self, Database cx) {
-		wait(self->clogRemoteDc(self, cx));
-		state int i = 0;
-		state int generationCount = 0;
-		for (; i < 6; ++i) {
-			// Sometimes Cycle Setup can take a long time, so we need to extend
-			// connection failures injection for clogRemoteDc() to work properly.
-			// We also want to make sure connection failures are still active when we
-			// are triggering recoveries within this actor, so the assertion below
-			// is true.
-			extendConnectionFailures("GcGenerations", FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS);
-
-			wait(delay(30));
-			TraceEvent("WaitingForDbAvailable")
-			    .detail("Index", i)
-			    .detail("RecoveryState", self->dbInfo->get().recoveryState);
-			wait(self->dbAvailable(self));
-			generationCount = self->dbInfo->get().logSystemConfig.oldTLogs.size();
-			TraceEvent("WaitingForDbAvailableDone")
-			    .detail("Index", i)
-			    .detail("Master", self->dbInfo->get().master.address());
-			g_simulator->rebootProcess(g_simulator->getProcessByAddress(self->dbInfo->get().master.address()),
-			                           ISimulator::KillType::Reboot);
-
-			// Wait for recovery
-			while (self->dbInfo->get().logSystemConfig.oldTLogs.size() == generationCount ||
-			       self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
-				wait(self->dbInfo->onChange());
-			}
-			TraceEvent("CurrentGenerations")
-			    .detail("PrevCount", generationCount)
-			    .detail("New", self->dbInfo->get().logSystemConfig.oldTLogs.size());
-			ASSERT(self->dbInfo->get().logSystemConfig.oldTLogs.size() > generationCount);
-			generationCount = self->dbInfo->get().logSystemConfig.oldTLogs.size();
-		}
-		TraceEvent("AfterMultipleRecovery")
-		    .detail("OldGenerationCount", self->dbInfo->get().logSystemConfig.oldTLogs.size());
-		return Void();
-	}
-
-	ACTOR Future<Void> generationReduced(GcGenerationsWorkload* self) {
-		TraceEvent("WaitForGenerationReduction")
-		    .detail("GenerationCount", self->dbInfo->get().logSystemConfig.oldTLogs.size())
-		    .detail("RecoveryState", self->dbInfo->get().recoveryState);
-		while (self->dbInfo->get().logSystemConfig.oldTLogs.size() > 1 ||
-		       self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-			TraceEvent("WaitForGenerationReduction")
-			    .detail("GenerationCount", self->dbInfo->get().logSystemConfig.oldTLogs.size())
-			    .detail("RecoveryState", self->dbInfo->get().recoveryState);
-			wait(self->dbInfo->onChange());
-		}
-		TraceEvent("WaitForGenerationReduction")
-		    .detail("GenerationCount", self->dbInfo->get().logSystemConfig.oldTLogs.size())
-		    .detail("RecoveryState", self->dbInfo->get().recoveryState);
-		return Void();
-	}
-
-	ACTOR Future<Void> gcGenerationsTestClient(GcGenerationsWorkload* self, Database cx) {
-		g_simulator->disableTLogRecoveryFinish = true;
-
-		wait(delay(self->startDelay));
-
-		TraceEvent("WaitingForDbAvailable").detail("RecoveryState", self->dbInfo->get().recoveryState);
-		while (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-			wait(self->dbInfo->onChange());
-		}
-
-		double startTime = now();
-		double workloadEnd = now() + self->testDuration;
-		TraceEvent("GcGenerations").detail("StartTime", startTime).detail("EndTime", workloadEnd);
-
-		wait(self->generateMultipleTxnGenerations(self, cx));
-		self->unclogAll();
-		disableConnectionFailures("GcGenerations");
-
-		wait(self->generationReduced(self));
-
-		g_simulator->disableTLogRecoveryFinish = false;
-
-		TraceEvent("WaitingForDbFullyRecovered").detail("RecoveryState", self->dbInfo->get().recoveryState);
-		while (self->dbInfo->get().recoveryState != RecoveryState::FULLY_RECOVERED) {
-			wait(self->dbInfo->onChange());
-		}
-
-		TraceEvent("GcGenerationsWorkloadFinish").log();
-		return Void();
-	}
-};
-
-WorkloadFactory<GcGenerationsWorkload> GcGenerationsWorkloadFactory;
\ No newline at end of file
diff --git a/fdbserver/workloads/GcGenerations.cpp b/fdbserver/workloads/GcGenerations.cpp
new file mode 100644
index 00000000000..3c6ee63624f
--- /dev/null
+++ b/fdbserver/workloads/GcGenerations.cpp
@@ -0,0 +1,267 @@
+/*
+ * GcGenerations.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/StatusClient.h"
+#include "fdbrpc/Locality.h"
+#include "fdbrpc/SimulatorProcessInfo.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbrpc/simulator.h"
+#include "flow/CodeProbe.h"
+#include "flow/NetworkAddress.h"
+#include "flow/Error.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+
+// This workload tests that when TRACK_TLOG_RECOVERY is turned on, older TLog generations can be garbage collected
+// during the recovery before reaching fully_recovered.
+struct GcGenerationsWorkload : TestWorkload {
+	static constexpr auto NAME = "GcGenerations";
+	bool enabled;
+	double testDuration;
+	double startDelay;
+	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
+
+	explicit GcGenerationsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		enabled = !clientId; // only do this on the "first" client
+		testDuration = getOption(options, "testDuration"_sr, 1000.0);
+		startDelay = getOption(options, "startDelay"_sr, 30.0);
+	}
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
+		out.insert("Attrition");
+		out.insert("RandomClogging");
+	}
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override {
+		if (g_network->isSimulated() && enabled)
+			return timeout(reportErrors(gcGenerationsTestClient(this, cx), "GcGenerationsError"), testDuration, Void());
+		else
+			return Void();
+	}
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	// Ensure simulator state is cleaned up even if the workload is cancelled by timeout.
+	// Without this, a timeout leaves the cluster permanently degraded: remote DC clogged,
+	// connection failures active, disableTLogRecoveryFinish=true — causing Cycle check to fail.
+	~GcGenerationsWorkload() {
+		if (g_network && g_network->isSimulated()) {
+			unclogAll();
+			disableConnectionFailures("GcGenerations");
+			fdbSimulationPolicyState().disableTLogRecoveryFinish = false;
+		}
+	}
+
+	void unclogAll() {
+		TraceEvent("GcGenerationsUnclogRemote").detail("UnclogConnectionCount", cloggedPairs.size());
+		// unclog previously clogged connections
+		for (const auto& pair : cloggedPairs) {
+			g_simulator->unclogPair(pair.first, pair.second);
+		}
+		cloggedPairs.clear();
+	}
+
+	Future<Void> clogRemoteDc(GcGenerationsWorkload* self, Database cx) {
+		Optional<ClusterConnectionString> csOptional = co_await getConnectionString(cx);
+		std::vector<NetworkAddress> coordinators;
+		if (csOptional.present()) {
+			ClusterConnectionString cs = csOptional.get();
+			coordinators = co_await cs.tryResolveHostnames();
+		}
+
+		auto isCoordinator = [](const std::vector<NetworkAddress>& coordinators, const IPAddress& ip) {
+			for (const auto& c : coordinators) {
+				if (c.ip == ip) {
+					return true;
+				}
+			}
+			return false;
+		};
+
+		std::vector<IPAddress> ips; // all non-remote process IPs
+		std::vector<IPAddress> remoteIps; // all remote process IPs
+		for (const auto& process : g_simulator->getAllProcesses()) {
+			const auto& ip = process->address.ip;
+			if (process->locality.dcId().present() &&
+			    process->locality.dcId() == fdbSimulationPolicyState().remoteDcId && !isCoordinator(coordinators, ip)) {
+				remoteIps.push_back(ip);
+			} else {
+				ips.push_back(ip);
+			}
+		}
+		ASSERT(!ips.empty());
+		ASSERT(!remoteIps.empty());
+
+		for (const auto& ip : ips) {
+			for (const auto& remoteIp : remoteIps) {
+				g_simulator->clogPair(ip, remoteIp, 10000);
+				g_simulator->clogPair(remoteIp, ip, 10000);
+				self->cloggedPairs.emplace_back(ip, remoteIp);
+				self->cloggedPairs.emplace_back(remoteIp, ip);
+			}
+		}
+
+		TraceEvent("PartitionRemoteDc")
+		    .detail("RemoteDc", fdbSimulationPolicyState().remoteDcId)
+		    .detail("CloggedRemoteProcess", describe(remoteIps));
+	}
+
+	bool isMasterInRemoteDc(GcGenerationsWorkload* self) {
+		auto masterAddr = self->dbInfo->get().master.address();
+		auto* masterProc = g_simulator->getProcessByAddress(masterAddr);
+		return !masterProc || !masterProc->locality.dcId().present() ||
+		       masterProc->locality.dcId() == fdbSimulationPolicyState().remoteDcId;
+	}
+
+	// Wait for the DB to reach ACCEPTING_COMMITS. If rebootRemoteDcMaster is true and
+	// the master is in the remote DC, reboot it to force the CC to elect a primary DC
+	// master. This is required when the remote DC is clogged (otherwise recovery can
+	// never complete), but must be disabled once the remote DC is unclogged — otherwise
+	// every CC re-election that lands in the remote DC triggers another reboot, producing
+	// a tight loop that prevents recovery from ever reaching ACCEPTING_COMMITS.
+	Future<Void> dbAvailable(GcGenerationsWorkload* self, bool rebootRemoteDcMaster) {
+		while (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			co_await self->dbInfo->onChange();
+			if (rebootRemoteDcMaster && self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS &&
+			    self->isMasterInRemoteDc(self)) {
+				auto masterAddr = self->dbInfo->get().master.address();
+				auto* masterProc = g_simulator->getProcessByAddress(masterAddr);
+				TraceEvent("DbAvailableRebootRemoteMaster").detail("MasterAddr", masterAddr);
+				if (masterProc) {
+					g_simulator->rebootProcess(masterProc, ISimulator::KillType::Reboot);
+				}
+			}
+		}
+	}
+
+	Future<Void> generateMultipleTxnGenerations(GcGenerationsWorkload* self, Database cx) {
+		co_await self->clogRemoteDc(self, cx);
+		int generationCount = 0;
+		int successfulReboots = 0;
+		while (successfulReboots < 6) {
+			// Re-enable connection failures each iteration to keep the partition active.
+			// Using enableConnectionFailures (not extendConnectionFailures) resets
+			// connectionFailureEnableTime, which prevents the peek cursor assertion
+			// in LogSystemPeekCursor from firing while clogged pairs are still active.
+			enableConnectionFailures("GcGenerations", FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS);
+
+			co_await delay(30);
+			TraceEvent("WaitingForDbAvailable")
+			    .detail("Iteration", successfulReboots)
+			    .detail("RecoveryState", self->dbInfo->get().recoveryState);
+			co_await self->dbAvailable(self, /*rebootRemoteDcMaster=*/true);
+			generationCount = self->dbInfo->get().logSystemConfig.oldTLogs.size();
+
+			// Only reboot the master if it's in the primary DC. If it's in the clogged
+			// remote DC, recovery will stall because the master can't communicate with
+			// primary DC processes. Loop back and try again.
+			if (self->isMasterInRemoteDc(self)) {
+				TraceEvent("RetryingRemoteDcMaster")
+				    .detail("Iteration", successfulReboots)
+				    .detail("MasterAddr", self->dbInfo->get().master.address());
+				continue;
+			}
+
+			auto masterAddr = self->dbInfo->get().master.address();
+			TraceEvent("RebootingPrimaryDcMaster").detail("Iteration", successfulReboots).detail("Master", masterAddr);
+			g_simulator->rebootProcess(g_simulator->getProcessByAddress(masterAddr), ISimulator::KillType::Reboot);
+
+			// Wait for recovery to create a new generation.
+			while (self->dbInfo->get().logSystemConfig.oldTLogs.size() == generationCount ||
+			       self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
+				co_await self->dbInfo->onChange();
+			}
+			TraceEvent("CurrentGenerations")
+			    .detail("Iteration", successfulReboots)
+			    .detail("PrevCount", generationCount)
+			    .detail("New", self->dbInfo->get().logSystemConfig.oldTLogs.size());
+			ASSERT(self->dbInfo->get().logSystemConfig.oldTLogs.size() > generationCount);
+			generationCount = self->dbInfo->get().logSystemConfig.oldTLogs.size();
+			++successfulReboots;
+		}
+		TraceEvent("AfterMultipleRecovery")
+		    .detail("OldGenerationCount", self->dbInfo->get().logSystemConfig.oldTLogs.size());
+	}
+
+	Future<Void> gcGenerationsTestClient(GcGenerationsWorkload* self, Database cx) {
+		co_await delay(self->startDelay);
+
+		TraceEvent("WaitingForDbAvailable").detail("RecoveryState", self->dbInfo->get().recoveryState);
+		while (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			co_await self->dbInfo->onChange();
+		}
+
+		double startTime = now();
+		double workloadEnd = now() + self->testDuration;
+		TraceEvent("GcGenerations").detail("StartTime", startTime).detail("EndTime", workloadEnd);
+
+		// Block TLog recovery while creating generations to test generation accumulation during recovery
+		fdbSimulationPolicyState().disableTLogRecoveryFinish = true;
+
+		co_await self->generateMultipleTxnGenerations(self, cx);
+		self->unclogAll();
+		disableConnectionFailures("GcGenerations");
+
+		// Unblock TLogs before waiting for generation reduction.
+		// The trackRecoveryReq blocking prevented TLogs from reporting recovered state
+		// during accumulation. The current recovery's tracking is now stale (FinalUpdate
+		// will never fire), so we must trigger a fresh recovery by rebooting the master.
+		// The new recovery starts with clean tracking state, allowing GC to proceed.
+		fdbSimulationPolicyState().disableTLogRecoveryFinish = false;
+
+		// Reboot the master to trigger fresh recoveries with clean tracking state.
+		// GC may need multiple recovery cycles: remote TLogs must catch up from old
+		// generations before remoteRecoveredVersion advances past their recoverAt,
+		// and purgeOldRecoveredGenerationsCoreState only purges generations below that.
+		// Retry periodically until oldTLogs is reduced.
+		// Note: the remote DC is unclogged now, so any master (including remote DC)
+		// can coordinate recovery. No need for the primary-DC-only guard here.
+		while (self->dbInfo->get().logSystemConfig.oldTLogs.size() > 1) {
+			co_await self->dbAvailable(self, /*rebootRemoteDcMaster=*/false);
+			auto masterAddr = self->dbInfo->get().master.address();
+			TraceEvent("RebootMasterForGC").detail("Master", masterAddr);
+			g_simulator->rebootProcess(g_simulator->getProcessByAddress(masterAddr), ISimulator::KillType::Reboot);
+			// Give this recovery cycle time to GC before retrying.
+			co_await delay(60);
+			co_await self->dbAvailable(self, /*rebootRemoteDcMaster=*/false);
+			TraceEvent("GcGenerationsWaitingForReduction")
+			    .detail("OldTLogs", self->dbInfo->get().logSystemConfig.oldTLogs.size())
+			    .detail("RecoveryState", self->dbInfo->get().recoveryState);
+		}
+
+		TraceEvent("WaitingForDbFullyRecovered").detail("RecoveryState", self->dbInfo->get().recoveryState);
+		while (self->dbInfo->get().recoveryState != RecoveryState::FULLY_RECOVERED) {
+			co_await self->dbInfo->onChange();
+		}
+
+		TraceEvent("GcGenerationsWorkloadFinish").log();
+	}
+};
+
+WorkloadFactory<GcGenerationsWorkload> GcGenerationsWorkloadFactory;
diff --git a/fdbserver/workloads/GetEstimatedRangeSize.actor.cpp b/fdbserver/workloads/GetEstimatedRangeSize.cpp
similarity index 70%
rename from fdbserver/workloads/GetEstimatedRangeSize.actor.cpp
rename to fdbserver/workloads/GetEstimatedRangeSize.cpp
index 24d727dab6f..41d4e1d4632 100644
--- a/fdbserver/workloads/GetEstimatedRangeSize.actor.cpp
+++ b/fdbserver/workloads/GetEstimatedRangeSize.cpp
@@ -1,5 +1,5 @@
 /*
- * GetEstimatedRangeSize.actor.cpp
+ * GetEstimatedRangeSize.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,17 +18,14 @@
  * limitations under the License.
  */
 
-#include <cstring>
-
 #include "fdbrpc/simulator.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 #include "flow/Error.h"
 #include "flow/Trace.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct GetEstimatedRangeSizeWorkload : TestWorkload {
 	static constexpr auto NAME = "GetEstimatedRangeSize";
@@ -37,7 +34,7 @@ struct GetEstimatedRangeSizeWorkload : TestWorkload {
 	Key keyPrefix;
 	bool checkOnly;
 
-	GetEstimatedRangeSizeWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit GetEstimatedRangeSizeWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		nodeCount = getOption(options, "nodeCount"_sr, 10000);
 		keyPrefix = unprintable(getOption(options, "keyPrefix"_sr, ""_sr).toString());
@@ -67,7 +64,7 @@ struct GetEstimatedRangeSizeWorkload : TestWorkload {
 		if (clientId > 0) {
 			return Void();
 		}
-		return checkSize(this, cx);
+		return checkSize(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
@@ -80,37 +77,39 @@ struct GetEstimatedRangeSizeWorkload : TestWorkload {
 	int fromValue(const ValueRef& v) { return testKeyToDouble(v, keyPrefix); }
 	Standalone<KeyValueRef> operator()(int n) { return KeyValueRef(key(n), value((n + 1) % nodeCount)); }
 
-	ACTOR static Future<Void> checkSize(GetEstimatedRangeSizeWorkload* self, Database cx) {
-		state int64_t size = wait(getSize(self, cx));
-		ASSERT(sizeIsAsExpected(self, size));
-		return Void();
+	Future<Void> checkSize(Database cx) {
+		int64_t size = co_await getSize(cx);
+		ASSERT(sizeIsAsExpected(size));
 	}
 
-	static bool sizeIsAsExpected(GetEstimatedRangeSizeWorkload* self, int64_t size) {
-		int nodeSize = self->key(0).size() + self->value(0).size();
+	bool sizeIsAsExpected(int64_t size) {
+		int nodeSize = key(0).size() + value(0).size();
 		// We use a wide range to avoid flakiness because the underlying function
 		// is making an estimation.
-		return size > self->nodeCount * nodeSize / 2 && size < self->nodeCount * nodeSize * 5;
+		return size > nodeCount * nodeSize / 2 && size < nodeCount * nodeSize * 5;
 	}
 
-	ACTOR static Future<int64_t> getSize(GetEstimatedRangeSizeWorkload* self, Database cx) {
-		state ReadYourWritesTransaction tr(cx);
-		state double totalDelay = 0.0;
+	Future<int64_t> getSize(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		double totalDelay = 0.0;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state int64_t size = wait(tr.getEstimatedRangeSizeBytes(normalKeys));
+				int64_t size = co_await tr.getEstimatedRangeSizeBytes(normalKeys);
 				TraceEvent(SevDebug, "GetSizeResult").detail("Size", size);
-				if (!sizeIsAsExpected(self, size) && totalDelay < 300.0) {
+				if (!sizeIsAsExpected(size) && totalDelay < 300.0) {
 					totalDelay += 5.0;
-					wait(delay(5.0));
+					co_await delay(5.0);
+					continue;
 				} else {
-					return size;
+					co_return size;
 				}
 			} catch (Error& e) {
-				TraceEvent(SevDebug, "GetSizeError").errorUnsuppressed(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent(SevDebug, "GetSizeError").errorUnsuppressed(err);
+			co_await tr.onError(err);
 		}
 	}
 };
diff --git a/fdbserver/workloads/GetMappedRange.actor.cpp b/fdbserver/workloads/GetMappedRange.cpp
similarity index 60%
rename from fdbserver/workloads/GetMappedRange.actor.cpp
rename to fdbserver/workloads/GetMappedRange.cpp
index 3b883f642d3..e868003fe9f 100644
--- a/fdbserver/workloads/GetMappedRange.actor.cpp
+++ b/fdbserver/workloads/GetMappedRange.cpp
@@ -1,5 +1,5 @@
 /*
- * GetMappedRange.actor.cpp
+ * GetMappedRange.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,16 +19,16 @@
  */
 
 #include <algorithm>
-#include "fdbclient/MutationLogReader.actor.h"
+#include "fdbclient/MutationLogReader.h"
 #include "fdbclient/StatusClient.h"
 #include "fdbclient/Tuple.h"
-#include "fdbserver/workloads/ApiWorkload.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/Knobs.h"
+#include "ApiWorkload.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/Knobs.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 const Value EMPTY = Tuple().pack();
 ValueRef SOMETHING = "SOMETHING"_sr;
@@ -51,7 +51,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	double checkStorageQueueSeconds;
 	uint64_t queueMaxLength;
 
-	GetMappedRangeWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx) {
+	explicit GetMappedRangeWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		checkStorageQueueSeconds = getOption(options, "checkStorageQueueSeconds"_sr, 60.0);
 		queueMaxLength = getOption(options, "queueMaxLength"_sr, UINT64_C(100));
@@ -68,13 +68,12 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> performSetup(Database cx, GetMappedRangeWorkload* self) {
+	Future<Void> performSetup(Database cx, GetMappedRangeWorkload* self) {
 		std::vector<TransactionType> types;
 		types.push_back(NATIVE);
 		types.push_back(READ_YOUR_WRITES);
 
-		wait(self->chooseTransactionFactory(cx, types));
-		return Void();
+		co_await self->chooseTransactionFactory(cx, types);
 	}
 
 	Future<Void> performSetup(Database const& cx) override { return performSetup(cx, this); }
@@ -95,44 +94,47 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	static Value recordValue(int i) { return Tuple::makeTuple(dataOfRecord(i)).pack(); }
 	static Value recordValue(int i, int split) { return Tuple::makeTuple(dataOfRecord(i, split)).pack(); }
 
-	ACTOR Future<Void> fillInRecords(Database cx, int n, GetMappedRangeWorkload* self) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> fillInRecords(Database cx, int n, GetMappedRangeWorkload* self) {
+		Transaction tr(cx);
+		while (true) {
 			std::cout << "start fillInRecords n=" << n << std::endl;
 			// TODO: When n is large, split into multiple transactions.
 			recordSize = 0;
 			indexSize = 0;
-			try {
-				for (int i = 0; i < n; i++) {
-					if (self->SPLIT_RECORDS) {
-						for (int split = 0; split < SPLIT_SIZE; split++) {
-							tr.set(recordKey(i, split), recordValue(i, split));
+			{
+				Error err;
+				try {
+					for (int i = 0; i < n; i++) {
+						if (self->SPLIT_RECORDS) {
+							for (int split = 0; split < SPLIT_SIZE; split++) {
+								tr.set(recordKey(i, split), recordValue(i, split));
+								if (i == 0) {
+									recordSize +=
+									    recordKey(i, split).size() + recordValue(i, split).size() + sizeof(KeyValueRef);
+								}
+							}
+						} else {
+							tr.set(recordKey(i), recordValue(i));
 							if (i == 0) {
-								recordSize +=
-								    recordKey(i, split).size() + recordValue(i, split).size() + sizeof(KeyValueRef);
+								recordSize += recordKey(i).size() + recordValue(i).size() + sizeof(KeyValueRef);
 							}
 						}
-					} else {
-						tr.set(recordKey(i), recordValue(i));
+						tr.set(indexEntryKey(i), EMPTY);
 						if (i == 0) {
-							recordSize += recordKey(i).size() + recordValue(i).size() + sizeof(KeyValueRef);
+							indexSize += indexEntryKey(i).size() + sizeof(KeyValueRef);
 						}
 					}
-					tr.set(indexEntryKey(i), EMPTY);
-					if (i == 0) {
-						indexSize += indexEntryKey(i).size() + sizeof(KeyValueRef);
-					}
+					co_await tr.commit();
+					std::cout << "finished fillInRecords with version " << tr.getCommittedVersion() << " recordSize "
+					          << recordSize << " indexSize " << indexSize << std::endl;
+					break;
+				} catch (Error& e) {
+					err = e;
 				}
-				wait(tr.commit());
-				std::cout << "finished fillInRecords with version " << tr.getCommittedVersion() << " recordSize "
-				          << recordSize << " indexSize " << indexSize << std::endl;
-				break;
-			} catch (Error& e) {
 				std::cout << "failed fillInRecords, retry" << std::endl;
-				wait(tr.onError(e));
+				co_await tr.onError(err);
 			}
 		}
-		return Void();
 	}
 
 	static void showResult(const RangeResult& result) {
@@ -142,21 +144,22 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		}
 	}
 
-	ACTOR Future<Void> scanRange(Database cx, KeyRangeRef range) {
+	Future<Void> scanRange(Database cx, KeyRangeRef range) {
 		std::cout << "start scanRange " << range.toString() << std::endl;
 		// TODO: When n is large, split into multiple transactions.
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				RangeResult result = wait(tr.getRange(range, CLIENT_KNOBS->TOO_MANY));
+				RangeResult result = co_await tr.getRange(range, CLIENT_KNOBS->TOO_MANY);
 				//			showResult(result);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 		std::cout << "finished scanRange" << std::endl;
-		return Void();
 	}
 
 	// Return true if need to retry.
@@ -205,68 +208,72 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		return false;
 	}
 
-	ACTOR Future<MappedRangeResult> scanMappedRangeWithLimits(Database cx,
-	                                                          KeySelector beginSelector,
-	                                                          KeySelector endSelector,
-	                                                          Key mapper,
-	                                                          int limit,
-	                                                          int byteLimit,
-	                                                          int expectedBeginId,
-	                                                          GetMappedRangeWorkload* self,
-	                                                          bool allMissing) {
+	Future<MappedRangeResult> scanMappedRangeWithLimits(Database cx,
+	                                                    KeySelector beginSelector,
+	                                                    KeySelector endSelector,
+	                                                    Key mapper,
+	                                                    int limit,
+	                                                    int byteLimit,
+	                                                    int expectedBeginId,
+	                                                    GetMappedRangeWorkload* self,
+	                                                    bool allMissing) {
 
 		std::cout << "start scanMappedRangeWithLimits beginSelector:" << beginSelector.toString()
 		          << " endSelector:" << endSelector.toString() << " expectedBeginId:" << expectedBeginId
 		          << " limit:" << limit << " byteLimit: " << byteLimit << "  recordSize: " << recordSize
 		          << " STRICTLY_ENFORCE_BYTE_LIMIT: " << SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT << " allMissing "
 		          << allMissing << std::endl;
-		loop {
-			state Reference<TransactionWrapper> tr = self->createTransaction();
-			try {
-				MappedRangeResult result = wait(tr->getMappedRange(beginSelector,
-				                                                   endSelector,
-				                                                   mapper,
-				                                                   GetRangeLimits(limit, byteLimit),
-				                                                   self->snapshot,
-				                                                   Reverse::False));
-				//			showResult(result);
-				if (self->BAD_MAPPER) {
-					TraceEvent("GetMappedRangeWorkloadShouldNotReachable").detail("ResultSize", result.size());
-				}
-				std::cout << "result.size()=" << result.size() << std::endl;
-				std::cout << "result.more=" << result.more << std::endl;
-				ASSERT(result.size() <= limit);
-				int expectedId = expectedBeginId;
-				bool needRetry = false;
-				int cnt = 0;
-				const MappedKeyValueRef* it = result.begin();
-				for (; cnt < result.size(); cnt++, it++) {
-					if (validateRecord(expectedId, it, self, allMissing)) {
-						needRetry = true;
-						break;
+		while (true) {
+			Reference<TransactionWrapper> tr = self->createTransaction();
+			{
+				Error err;
+				try {
+					MappedRangeResult result = co_await tr->getMappedRange(beginSelector,
+					                                                       endSelector,
+					                                                       mapper,
+					                                                       GetRangeLimits(limit, byteLimit),
+					                                                       self->snapshot,
+					                                                       Reverse::False);
+					//			showResult(result);
+					if (self->BAD_MAPPER) {
+						TraceEvent("GetMappedRangeWorkloadShouldNotReachable").detail("ResultSize", result.size());
 					}
-					expectedId++;
-				}
-				if (needRetry) {
-					continue;
+					std::cout << "result.size()=" << result.size() << std::endl;
+					std::cout << "result.more=" << result.more << std::endl;
+					ASSERT(result.size() <= limit);
+					int expectedId = expectedBeginId;
+					bool needRetry = false;
+					int cnt = 0;
+					const MappedKeyValueRef* it = result.begin();
+					for (; cnt < result.size(); cnt++, it++) {
+						if (validateRecord(expectedId, it, self, allMissing)) {
+							needRetry = true;
+							break;
+						}
+						expectedId++;
+					}
+					if (needRetry) {
+						continue;
+					}
+					std::cout << "finished scanMappedRangeWithLimits" << std::endl;
+					co_return result;
+				} catch (Error& e) {
+					err = e;
 				}
-				std::cout << "finished scanMappedRangeWithLimits" << std::endl;
-				return result;
-			} catch (Error& e) {
-				if ((self->BAD_MAPPER && e.code() == error_code_mapper_bad_index) ||
-				    (!SERVER_KNOBS->QUICK_GET_VALUE_FALLBACK && e.code() == error_code_quick_get_value_miss) ||
+				if ((self->BAD_MAPPER && err.code() == error_code_mapper_bad_index) ||
+				    (!SERVER_KNOBS->QUICK_GET_VALUE_FALLBACK && err.code() == error_code_quick_get_value_miss) ||
 				    (!SERVER_KNOBS->QUICK_GET_KEY_VALUES_FALLBACK &&
-				     e.code() == error_code_quick_get_key_values_miss)) {
-					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(e);
-					return MappedRangeResult();
-				} else if (e.code() == error_code_commit_proxy_memory_limit_exceeded ||
-				           e.code() == error_code_operation_cancelled) {
+				     err.code() == error_code_quick_get_key_values_miss)) {
+					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(err);
+					co_return MappedRangeResult();
+				} else if (err.code() == error_code_commit_proxy_memory_limit_exceeded ||
+				           err.code() == error_code_operation_cancelled) {
 					// requests have overwhelmed commit proxy, rest a bit
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 					continue;
 				} else {
-					std::cout << "scan error " << e.what() << "  code is " << e.code() << std::endl;
-					wait(tr->onError(e));
+					std::cout << "scan error " << err.what() << "  code is " << err.code() << std::endl;
+					co_await tr->onError(err);
 				}
 				std::cout << "failed scanMappedRangeWithLimits" << std::endl;
 			}
@@ -275,47 +282,47 @@ struct GetMappedRangeWorkload : ApiWorkload {
 
 	// if sendFirstRequestIndefinitely is true, then this method would send the first request indefinitely
 	// it is in order to test the metric
-	ACTOR Future<Void> submitSmallRequestIndefinitely(Database cx,
-	                                                  int beginId,
-	                                                  int endId,
-	                                                  Key mapper,
-	                                                  GetMappedRangeWorkload* self) {
+	Future<Void> submitSmallRequestIndefinitely(Database cx,
+	                                            int beginId,
+	                                            int endId,
+	                                            Key mapper,
+	                                            GetMappedRangeWorkload* self) {
 		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
-		state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
+		KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
 		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
-		state KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
-		state int limit = 1;
-		state int byteLimit = 10000;
+		KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
+		int limit = 1;
+		int byteLimit = 10000;
 		while (true) {
-			MappedRangeResult result = wait(self->scanMappedRangeWithLimits(
-			    cx, beginSelector, endSelector, mapper, limit, byteLimit, beginId, self, false));
+			MappedRangeResult result = co_await self->scanMappedRangeWithLimits(
+			    cx, beginSelector, endSelector, mapper, limit, byteLimit, beginId, self, false);
 			if (result.empty()) {
 				TraceEvent("EmptyResult");
 			}
 			// to avoid requests make proxy memory overwhelmed
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 		}
 	}
 
-	ACTOR Future<Void> scanMappedRange(Database cx,
-	                                   int beginId,
-	                                   int endId,
-	                                   Key mapper,
-	                                   GetMappedRangeWorkload* self,
-	                                   bool allMissing = false) {
+	Future<Void> scanMappedRange(Database cx,
+	                             int beginId,
+	                             int endId,
+	                             Key mapper,
+	                             GetMappedRangeWorkload* self,
+	                             bool allMissing = false) {
 		Key beginTuple = Tuple::makeTuple(prefix, INDEX, indexKey(beginId)).getDataAsStandalone();
-		state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
+		KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
 		Key endTuple = Tuple::makeTuple(prefix, INDEX, indexKey(endId)).getDataAsStandalone();
-		state KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
-		state int limit = 100;
-		state int byteLimit = deterministicRandom()->randomInt(1, 9) * 10000;
-		state int expectedBeginId = beginId;
+		KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
+		int limit = 100;
+		int byteLimit = deterministicRandom()->randomInt(1, 9) * 10000;
+		int expectedBeginId = beginId;
 		std::cout << "ByteLimit: " << byteLimit << " limit: " << limit
 		          << " FRACTION_INDEX_BYTELIMIT_PREFETCH: " << SERVER_KNOBS->FRACTION_INDEX_BYTELIMIT_PREFETCH
 		          << " MAX_PARALLEL_QUICK_GET_VALUE: " << SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE << std::endl;
 		while (true) {
-			MappedRangeResult result = wait(self->scanMappedRangeWithLimits(
-			    cx, beginSelector, endSelector, mapper, limit, byteLimit, expectedBeginId, self, allMissing));
+			MappedRangeResult result = co_await self->scanMappedRangeWithLimits(
+			    cx, beginSelector, endSelector, mapper, limit, byteLimit, expectedBeginId, self, allMissing);
 			expectedBeginId += result.size();
 			if (result.more) {
 				if (result.empty()) {
@@ -351,8 +358,6 @@ struct GetMappedRangeWorkload : ApiWorkload {
 			}
 		}
 		ASSERT(expectedBeginId == endId);
-
-		return Void();
 	}
 
 	static void conflictWriteOnRecord(int conflictRecordId,
@@ -375,11 +380,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		std::cout << "conflict write to " << printable(writeKey) << std::endl;
 	}
 
-	static Future<MappedRangeResult> runGetMappedRange(int beginId,
-	                                                   int endId,
-	                                                   Reference<TransactionWrapper>& tr,
-	                                                   GetMappedRangeWorkload* self) {
-		Key mapper = getMapper(self, false);
+	Future<MappedRangeResult> runGetMappedRange(int beginId, int endId, Reference<TransactionWrapper>& tr) {
+		Key mapper = getMapper(false);
 		Key beginTuple = Tuple::makeTuple(prefix, INDEX, indexKey(beginId)).getDataAsStandalone();
 		KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
 		Key endTuple = Tuple::makeTuple(prefix, INDEX, indexKey(endId)).getDataAsStandalone();
@@ -388,68 +390,76 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		                          endSelector,
 		                          mapper,
 		                          GetRangeLimits(GetRangeLimits::ROW_LIMIT_UNLIMITED),
-		                          self->snapshot,
+		                          snapshot,
 		                          Reverse::False);
 	}
 
 	// If another transaction writes to our read set (the scanned ranges) before we commit, the transaction should
 	// fail.
-	ACTOR Future<Void> testSerializableConflicts(GetMappedRangeWorkload* self) {
+	Future<Void> testSerializableConflicts(GetMappedRangeWorkload* self) {
 		std::cout << "testSerializableConflicts" << std::endl;
 
-		loop {
-			state Reference<TransactionWrapper> tr1 = self->createTransaction();
-			try {
-				MappedRangeResult result = wait(runGetMappedRange(5, 10, tr1, self));
-
-				// Commit another transaction that has conflict writes.
-				loop {
-					state Reference<TransactionWrapper> tr2 = self->createTransaction();
-					try {
-						conflictWriteOnRecord(7, tr2, self);
-						wait(tr2->commit());
-						break;
-					} catch (Error& e) {
-						std::cout << "tr2 error " << e.what() << std::endl;
-						wait(tr2->onError(e));
+		while (true) {
+			Reference<TransactionWrapper> tr1 = self->createTransaction();
+			{
+				Error err;
+				try {
+					MappedRangeResult result = co_await runGetMappedRange(5, 10, tr1);
+
+					// Commit another transaction that has conflict writes.
+					while (true) {
+						Reference<TransactionWrapper> tr2 = self->createTransaction();
+						{
+							Error err;
+							try {
+								conflictWriteOnRecord(7, tr2, self);
+								co_await tr2->commit();
+								break;
+							} catch (Error& e) {
+								err = e;
+							}
+							std::cout << "tr2 error " << err.what() << std::endl;
+							co_await tr2->onError(err);
+						}
 					}
-				}
 
-				// Do some writes so that tr1 is not read-only.
-				tr1->set(SOMETHING, SOMETHING);
-				wait(tr1->commit());
-				UNREACHABLE();
-			} catch (Error& e) {
-				if (e.code() == error_code_not_committed) {
+					// Do some writes so that tr1 is not read-only.
+					tr1->set(SOMETHING, SOMETHING);
+					co_await tr1->commit();
+					UNREACHABLE();
+				} catch (Error& e) {
+					err = e;
+				}
+				if (err.code() == error_code_not_committed) {
 					std::cout << "tr1 failed because of conflicts (as expected)" << std::endl;
-					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(e);
-					return Void();
+					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(err);
+					co_return;
 				} else {
-					std::cout << "tr1 error " << e.what() << std::endl;
-					wait(tr1->onError(e));
+					std::cout << "tr1 error " << err.what() << std::endl;
+					co_await tr1->onError(err);
 				}
 			}
 		}
 	}
 
 	// checking the max storage queue length is bounded
-	ACTOR static Future<Void> reportMetric(GetMappedRangeWorkload* self, Database cx) {
-		loop {
-			StatusObject result = wait(StatusClient::statusFetcher(cx));
+	Future<Void> reportMetric(Database cx) {
+		while (true) {
+			StatusObject result = co_await StatusClient::statusFetcher(cx);
 			StatusObjectReader statusObj(result);
-			state StatusObjectReader statusObjCluster;
-			state StatusObjectReader processesMap;
-			state int64_t queryQueueMax = 0;
-			state int waitInterval = 2;
+			StatusObjectReader statusObjCluster;
+			StatusObjectReader processesMap;
+			int64_t queryQueueMax = 0;
+			int waitInterval = 2;
 			if (!statusObj.get("cluster", statusObjCluster)) {
 				TraceEvent("NoCluster");
-				wait(delay(waitInterval));
+				co_await delay(waitInterval);
 				continue;
 			}
 
 			if (!statusObjCluster.get("processes", processesMap)) {
 				TraceEvent("NoProcesses");
-				wait(delay(waitInterval));
+				co_await delay(waitInterval);
 				continue;
 			}
 			for (auto proc : processesMap.obj()) {
@@ -462,68 +472,72 @@ struct GetMappedRangeWorkload : ApiWorkload {
 							CODE_PROBE(queryQueueMax > 0, " SS query queue is non-empty");
 							TraceEvent(SevDebug, "QueryQueueMax")
 							    .detail("Value", queryQueueMax)
-							    .detail("MaxLength", self->queueMaxLength);
-							ASSERT(queryQueueMax < self->queueMaxLength);
+							    .detail("MaxLength", queueMaxLength);
+							ASSERT(queryQueueMax < queueMaxLength);
 						}
 					}
 				} else {
 					TraceEvent("NoRoles");
 				}
 			}
-			wait(delay(waitInterval));
+			co_await delay(waitInterval);
 		}
 	}
 
 	// If the same transaction writes to the read set (the scanned ranges) before reading, it should throw read your
 	// write exception.
-	ACTOR Future<Void> testRYW(GetMappedRangeWorkload* self) {
+	Future<Void> testRYW(GetMappedRangeWorkload* self) {
 		std::cout << "testRYW" << std::endl;
-		loop {
-			state Reference<TransactionWrapper> tr1 = self->createTransaction();
-			try {
-				// Write something that will be read in getMappedRange.
-				conflictWriteOnRecord(7, tr1, self);
-				MappedRangeResult result = wait(runGetMappedRange(5, 10, tr1, self));
-				UNREACHABLE();
-			} catch (Error& e) {
-				if (e.code() == error_code_get_mapped_range_reads_your_writes) {
+		while (true) {
+			Reference<TransactionWrapper> tr1 = self->createTransaction();
+			{
+				Error err;
+				try {
+					// Write something that will be read in getMappedRange.
+					conflictWriteOnRecord(7, tr1, self);
+					MappedRangeResult result = co_await runGetMappedRange(5, 10, tr1);
+					UNREACHABLE();
+				} catch (Error& e) {
+					err = e;
+				}
+				if (err.code() == error_code_get_mapped_range_reads_your_writes) {
 					std::cout << "tr1 failed because of read your writes (as expected)" << std::endl;
-					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(e);
-					return Void();
+					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(err);
+					co_return;
 				} else {
-					std::cout << "tr1 error " << e.what() << std::endl;
-					wait(tr1->onError(e));
+					std::cout << "tr1 error " << err.what() << std::endl;
+					co_await tr1->onError(err);
 				}
 			}
 		}
 	}
 
-	ACTOR static Future<Void> testMetric(Database cx,
-	                                     GetMappedRangeWorkload* self,
-	                                     int beginId,
-	                                     int endId,
-	                                     Key mapper,
-	                                     int seconds) {
-		loop choose {
-			when(wait(reportMetric(self, cx))) {
+	Future<Void> testMetric(Database cx, int beginId, int endId, Key mapper, int seconds) {
+		while (true) {
+			auto choice = co_await race(
+			    reportMetric(cx), submitSmallRequestIndefinitely(cx, 10, 490, mapper, this), delay(seconds));
+			if (choice.index() == 0) {
+
 				TraceEvent(SevError, "Error: ReportMetric has ended");
-				return Void();
-			}
-			when(wait(self->submitSmallRequestIndefinitely(cx, 10, 490, mapper, self))) {
+				co_return;
+			} else if (choice.index() == 1) {
+
 				TraceEvent(SevError, "Error: submitSmallRequestIndefinitely has ended");
-				return Void();
-			}
-			when(wait(delay(seconds))) {
-				return Void();
+				co_return;
+			} else if (choice.index() == 2) {
+
+				co_return;
+			} else {
+				UNREACHABLE();
 			}
 		}
 	}
 
-	ACTOR Future<Void> _start(Database cx, GetMappedRangeWorkload* self) {
+	Future<Void> _start(Database cx, GetMappedRangeWorkload* self) {
 		TraceEvent("GetMappedRangeWorkloadConfig").detail("BadMapper", self->BAD_MAPPER);
 
 		// TODO: Use toml to config
-		wait(self->fillInRecords(cx, 500, self));
+		co_await self->fillInRecords(cx, 500, self);
 
 		if (self->transactionType == NATIVE) {
 			self->snapshot = Snapshot::True;
@@ -531,11 +545,11 @@ struct GetMappedRangeWorkload : ApiWorkload {
 			self->snapshot = Snapshot::False;
 			const double rand = deterministicRandom()->random01();
 			if (rand < 0.1) {
-				wait(self->testSerializableConflicts(self));
-				return Void();
+				co_await self->testSerializableConflicts(self);
+				co_return;
 			} else if (rand < 0.2) {
-				wait(self->testRYW(self));
-				return Void();
+				co_await self->testRYW(self);
+				co_return;
 			} else {
 				// Test the happy path where there is no conflicts or RYW
 			}
@@ -546,26 +560,25 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		std::cout << "Test configuration: transactionType:" << self->transactionType << " snapshot:" << self->snapshot
 		          << "bad_mapper:" << self->BAD_MAPPER << std::endl;
 
-		state Key mapper = getMapper(self, false);
+		Key mapper = getMapper(false);
 		// The scanned range cannot be too large to hit get_mapped_key_values_has_more. We have a unit validating the
 		// error is thrown when the range is large.
-		state bool originalStrictlyEnforeByteLimit = SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT;
-		(const_cast<ServerKnobs*> SERVER_KNOBS)->STRICTLY_ENFORCE_BYTE_LIMIT = deterministicRandom()->coinflip();
-		wait(self->scanMappedRange(cx, 10, 490, mapper, self));
-		wait(testMetric(cx, self, 10, 490, mapper, self->checkStorageQueueSeconds));
+		bool originalStrictlyEnforeByteLimit = SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT;
+		(const_cast<ServerKnobs*>(SERVER_KNOBS))->STRICTLY_ENFORCE_BYTE_LIMIT = deterministicRandom()->coinflip();
+		co_await self->scanMappedRange(cx, 10, 490, mapper, self);
+		co_await testMetric(cx, 10, 490, mapper, self->checkStorageQueueSeconds);
 
 		// reset it to default
-		(const_cast<ServerKnobs*> SERVER_KNOBS)->STRICTLY_ENFORCE_BYTE_LIMIT = originalStrictlyEnforeByteLimit;
-		return Void();
+		(const_cast<ServerKnobs*>(SERVER_KNOBS))->STRICTLY_ENFORCE_BYTE_LIMIT = originalStrictlyEnforeByteLimit;
 	}
 
-	static Key getMapper(GetMappedRangeWorkload* self, bool mapperForAllMissing) {
+	Key getMapper(bool mapperForAllMissing) {
 		Tuple mapperTuple;
-		if (self->BAD_MAPPER) {
+		if (BAD_MAPPER) {
 			mapperTuple << prefix << RECORD << "{K[xxx]}"_sr;
 		} else {
 			mapperTuple << prefix << RECORD << (mapperForAllMissing ? "{K[2]}"_sr : "{K[3]}"_sr);
-			if (self->SPLIT_RECORDS) {
+			if (SPLIT_RECORDS) {
 				mapperTuple << "{...}"_sr;
 			}
 		}
diff --git a/fdbserver/workloads/GetRangeStream.actor.cpp b/fdbserver/workloads/GetRangeStream.cpp
similarity index 50%
rename from fdbserver/workloads/GetRangeStream.actor.cpp
rename to fdbserver/workloads/GetRangeStream.cpp
index 5d2d73ffa4d..69044259eb6 100644
--- a/fdbserver/workloads/GetRangeStream.actor.cpp
+++ b/fdbserver/workloads/GetRangeStream.cpp
@@ -1,5 +1,5 @@
 /*
- * GetRangeStream.actor.cpp
+ * GetRangeStream.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,9 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 struct GetRangeStream : TestWorkload {
 	static constexpr auto NAME = "GetRangeStream";
@@ -32,7 +31,7 @@ struct GetRangeStream : TestWorkload {
 	Key end;
 	bool printKVPairs;
 
-	GetRangeStream(WorkloadContext const& wcx) : TestWorkload(wcx), bytesRead("BytesRead") {
+	explicit GetRangeStream(WorkloadContext const& wcx) : TestWorkload(wcx), bytesRead("BytesRead") {
 		useGetRange = getOption(options, "useGetRange"_sr, false);
 		begin = getOption(options, "begin"_sr, normalKeys.begin);
 		end = getOption(options, "end"_sr, normalKeys.end);
@@ -42,85 +41,90 @@ struct GetRangeStream : TestWorkload {
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
 	Future<Void> start(Database const& cx) override {
-		return clientId != 0 ? Void() : useGetRange ? fdbClientGetRange(cx, this) : fdbClientStream(cx, this);
+		return clientId != 0 ? Void() : useGetRange ? fdbClientGetRange(cx) : fdbClientStream(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override { m.push_back(bytesRead.getMetric()); }
 
-	ACTOR static Future<Void> logThroughput(GetRangeStream* self, Key* next) {
-		loop {
-			state int64_t last = self->bytesRead.getValue();
-			state double before = g_network->now();
-			wait(delay(1));
-			state double after = g_network->now();
+	Future<Void> logThroughput(Key* next) {
+		while (true) {
+			int64_t last = bytesRead.getValue();
+			double before = g_network->now();
+			co_await delay(1);
+			double after = g_network->now();
 			if (after > before) {
 				printf("throughput: %g bytes/s, next: %s\n",
-				       (self->bytesRead.getValue() - last) / (after - before),
+				       (bytesRead.getValue() - last) / (after - before),
 				       printable(*next).c_str());
 			}
 		}
 	}
 
-	ACTOR static Future<Void> fdbClientGetRange(Database db, GetRangeStream* self) {
-		state Transaction tx(db);
-		state Key next = self->begin;
-		state Future<Void> logFuture = logThroughput(self, &next);
-		loop {
+	Future<Void> fdbClientGetRange(Database db) {
+		Transaction tx(db);
+		Key next = begin;
+		Future<Void> logFuture = logThroughput(&next);
+		while (true) {
+			Error err;
 			try {
-				Standalone<RangeResultRef> range = wait(
-				    tx.getRange(KeySelector(firstGreaterOrEqual(next), next.arena()),
-				                KeySelector(firstGreaterOrEqual(self->end)),
-				                GetRangeLimits(GetRangeLimits::ROW_LIMIT_UNLIMITED, CLIENT_KNOBS->REPLY_BYTE_LIMIT)));
+				Standalone<RangeResultRef> range = co_await tx.getRange(
+				    KeySelector(firstGreaterOrEqual(next), next.arena()),
+				    KeySelector(firstGreaterOrEqual(end)),
+				    GetRangeLimits(GetRangeLimits::ROW_LIMIT_UNLIMITED, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
 				for (const auto& [k, v] : range) {
-					if (self->printKVPairs) {
+					if (printKVPairs) {
 						printf("%s -> %s\n", printable(k).c_str(), printable(v).c_str());
 					}
-					self->bytesRead += k.size() + v.size();
+					bytesRead += k.size() + v.size();
 				}
 				if (!range.more) {
 					break;
 				}
 				next = keyAfter(range.back().key);
 			} catch (Error& e) {
-				wait(tx.onError(e));
+				err = e;
 			}
+			co_await tx.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> fdbClientStream(Database db, GetRangeStream* self) {
-		state Transaction tx(db);
-		state Key next = self->begin;
-		state Future<Void> logFuture = logThroughput(self, &next);
-		loop {
-			state PromiseStream<Standalone<RangeResultRef>> results;
+	Future<Void> fdbClientStream(Database db) {
+		Transaction tx(db);
+		Key next = begin;
+		Future<Void> logFuture = logThroughput(&next);
+		while (true) {
+			PromiseStream<Standalone<RangeResultRef>> results;
+			Error err;
 			try {
-				state Future<Void> stream = tx.getRangeStream(results,
-				                                              KeySelector(firstGreaterOrEqual(next), next.arena()),
-				                                              KeySelector(firstGreaterOrEqual(self->end)),
-				                                              GetRangeLimits());
-				loop {
-					Standalone<RangeResultRef> range = waitNext(results.getFuture());
+				Future<Void> stream = tx.getRangeStream(results,
+				                                        KeySelector(firstGreaterOrEqual(next), next.arena()),
+				                                        KeySelector(firstGreaterOrEqual(end)),
+				                                        GetRangeLimits());
+				while (true) {
+					Standalone<RangeResultRef> range = co_await results.getFuture();
 					for (const auto& [k, v] : range) {
-						if (self->printKVPairs) {
+						if (printKVPairs) {
 							printf("%s -> %s\n", printable(k).c_str(), printable(v).c_str());
 						}
-						self->bytesRead += k.size() + v.size();
+						bytesRead += k.size() + v.size();
 					}
-					if (range.size()) {
+					if (!range.empty()) {
 						next = keyAfter(range.back().key);
 					}
 				}
 			} catch (Error& e) {
-				if (e.code() == error_code_end_of_stream) {
-					break;
-				}
-				wait(tx.onError(e));
+				err = e;
+			}
+			if (!err.isValid()) {
+				continue;
+			}
+			if (err.code() == error_code_end_of_stream) {
+				break;
 			}
+			co_await tx.onError(err);
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/HTTPKeyValueStore.actor.cpp b/fdbserver/workloads/HTTPKeyValueStore.cpp
similarity index 74%
rename from fdbserver/workloads/HTTPKeyValueStore.actor.cpp
rename to fdbserver/workloads/HTTPKeyValueStore.cpp
index be867c125c2..5c80ad24a4b 100644
--- a/fdbserver/workloads/HTTPKeyValueStore.actor.cpp
+++ b/fdbserver/workloads/HTTPKeyValueStore.cpp
@@ -1,5 +1,5 @@
 /*
- * HTTPKeyValueStore.actor.cpp
+ * HTTPKeyValueStore.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,14 +21,9 @@
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
-#include "flow/Util.h"
 #include "flow/serialize.h"
 #include "fdbrpc/HTTP.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include <cstring>
-#include <limits>
-
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 /*
  * Implements a basic put/get key-value store over HTTP to test the http client and simulated server code.
@@ -101,10 +96,10 @@ void httpKVProcessGet(Reference<SimHTTPKVStore> kvStore,
 }
 
 // key header always exists for both put and get
-ACTOR Future<Void> httpKVRequestCallback(Reference<SimHTTPKVStore> kvStore,
-                                         Reference<HTTP::IncomingRequest> req,
-                                         Reference<HTTP::OutgoingResponse> response) {
-	wait(delay(0));
+Future<Void> httpKVRequestCallback(Reference<SimHTTPKVStore> kvStore,
+                                   Reference<HTTP::IncomingRequest> req,
+                                   Reference<HTTP::OutgoingResponse> response) {
+	co_await delay(0);
 
 	ASSERT(req->verb == HTTP::HTTP_VERB_PUT || req->verb == HTTP::HTTP_VERB_GET);
 	ASSERT_EQ(req->resource, "/kv");
@@ -142,8 +137,6 @@ ACTOR Future<Void> httpKVRequestCallback(Reference<SimHTTPKVStore> kvStore,
 	} else {
 		httpKVProcessGet(kvStore, key, req, response);
 	}
-
-	return Void();
 }
 
 static Reference<SimHTTPKVStore> globalKVStore = Reference<SimHTTPKVStore>();
@@ -153,7 +146,7 @@ struct KeyValueRequestHandler : HTTP::IRequestHandler, ReferenceCounted<KeyValue
 	// global kv store for all request handler instances during simulation
 	Reference<SimHTTPKVStore> myKVStore;
 
-	KeyValueRequestHandler() {}
+	KeyValueRequestHandler() = default;
 
 	Future<Void> handleRequest(Reference<HTTP::IncomingRequest> req,
 	                           Reference<HTTP::OutgoingResponse> response) override {
@@ -197,7 +190,7 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 
 	PerfIntCounter getCount, putCount, connectCount, failedConnectCount;
 
-	HTTPKeyValueStoreWorkload(WorkloadContext const& wcx)
+	explicit HTTPKeyValueStoreWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), putCount("PutCount"), getCount("GetCount"), connectCount("ConnectCount"),
 	    failedConnectCount("FailedConnectCount") {
 		testDuration = getOption(options, "testDuration"_sr, 30.0);
@@ -220,45 +213,45 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 	}
 
 	// handles retrying on timeout and reinitializing connection like other users of HTTP (S3BlobStore, RestClient)
-	ACTOR Future<Reference<HTTP::IncomingResponse>> doKVRequest(HTTPKeyValueStoreWorkload* self,
-	                                                            std::string key,
-	                                                            Optional<std::string> value) {
-		state UnsentPacketQueue content;
-		state int seqNo = self->nextSeqNo;
+	Future<Reference<HTTP::IncomingResponse>> doKVRequest(HTTPKeyValueStoreWorkload* self,
+	                                                      std::string key,
+	                                                      Optional<std::string> value) {
+		UnsentPacketQueue content;
+		int seqNo = self->nextSeqNo;
 		++self->nextSeqNo;
-		loop {
+		while (true) {
+			Error err;
 			try {
 				while (!self->conn) {
-					// sometimes do resolve and connect directly, other times simulate what rest kms connector does and
-					// resolve endpoints themself and then connect to one directly
+					// sometimes do resolve and connect directly, other times simulate what rest kms connector does
+					// and resolve endpoints themself and then connect to one directly
 					if (self->manualResolve) {
-						state std::vector<NetworkAddress> addrs =
-						    wait(INetworkConnections::net()->resolveTCPEndpoint(self->hostname, self->service));
+						std::vector<NetworkAddress> addrs =
+						    co_await INetworkConnections::net()->resolveTCPEndpoint(self->hostname, self->service);
 						ASSERT(!addrs.empty());
 						int idx = deterministicRandom()->randomInt(0, addrs.size());
-						wait(store(self->conn,
-						           timeoutError(INetworkConnections::net()->connect(
-						                            addrs[idx].ip.toString(), std::to_string(addrs[idx].port), false),
-						                        FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT)));
+						self->conn =
+						    co_await timeoutError(INetworkConnections::net()->connect(
+						                              addrs[idx].ip.toString(), std::to_string(addrs[idx].port), false),
+						                          FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
 
 					} else {
-						wait(store(
-						    self->conn,
-						    timeoutError(INetworkConnections::net()->connect(self->hostname, self->service, false),
-						                 FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT)));
+						self->conn = co_await timeoutError(
+						    INetworkConnections::net()->connect(self->hostname, self->service, false),
+						    FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
 					}
 					if (self->conn.isValid()) {
-						wait(self->conn->connectHandshake());
+						co_await self->conn->connectHandshake();
 						++self->connectCount;
 					} else {
-						wait(delay(0.1));
+						co_await delay(0.1);
 						++self->failedConnectCount;
 					}
 				}
 
 				content.discardAll();
-				state Reference<HTTP::OutgoingRequest> req = makeReference<HTTP::OutgoingRequest>();
-				state UID requestID = deterministicRandom()->randomUniqueID();
+				auto req = makeReference<HTTP::OutgoingRequest>();
+				UID requestID = deterministicRandom()->randomUniqueID();
 				req->data.content = &content;
 				req->data.contentLen = 0;
 				req->resource = "/kv";
@@ -278,10 +271,10 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 					req->verb = HTTP::HTTP_VERB_GET;
 				}
 
-				state Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
-				state int64_t bytes_sent = 0;
-				Reference<HTTP::IncomingResponse> response = wait(
-				    timeoutError(HTTP::doRequest(self->conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 5.0));
+				Reference<IRateControl> sendReceiveRate = makeReference<Unlimited>();
+				int64_t bytes_sent = 0;
+				Reference<HTTP::IncomingResponse> response = co_await timeoutError(
+				    HTTP::doRequest(self->conn, req, sendReceiveRate, &bytes_sent, sendReceiveRate), 5.0);
 
 				// sometimes randomly close connection anyway
 				if (BUGGIFY_WITH_PROB(0.1)) {
@@ -298,31 +291,32 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 				ASSERT(response->data.headers.contains("UID"));
 				ASSERT_EQ(response->data.headers["UID"], requestID.toString());
 
-				return response;
+				co_return response;
 			} catch (Error& e) {
-				if (e.code() == error_code_operation_cancelled) {
-					throw e;
-				}
-				if (DEBUG_HTTPKV) {
-					fmt::print("REQ: ERROR: {0}\n", e.name());
-				}
-				if (self->conn) {
-					self->conn->close();
-					self->conn.clear();
-				}
-				if (e.code() != error_code_timed_out && e.code() != error_code_connection_failed &&
-				    e.code() != error_code_lookup_failed) {
-					throw e;
-				}
+				err = e;
+			}
+			if (err.code() == error_code_operation_cancelled) {
+				throw err;
+			}
+			if (DEBUG_HTTPKV) {
+				fmt::print("REQ: ERROR: {0}\n", err.name());
+			}
+			if (self->conn) {
+				self->conn->close();
+				self->conn.clear();
+			}
+			if (err.code() != error_code_timed_out && err.code() != error_code_connection_failed &&
+			    err.code() != error_code_lookup_failed) {
+				throw err;
+			}
 
-				// request got timed out or connection could not be established, close conn and try again
+			// request got timed out or connection could not be established, close conn and try again
 
-				wait(delay(0.1));
-			}
+			co_await delay(0.1);
 		}
 	}
 
-	ACTOR Future<Void> put(HTTPKeyValueStoreWorkload* self, std::string key, std::string value) {
+	Future<Void> put(HTTPKeyValueStoreWorkload* self, std::string key, std::string value) {
 		// TODO do http request
 		std::pair<std::string, std::string> active = { key, value };
 		self->activePut = active;
@@ -331,7 +325,7 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 			fmt::print("CL:put {0}:{1} = {2}\n", self->clientId, key, value);
 		}
 
-		Reference<HTTP::IncomingResponse> response = wait(self->doKVRequest(self, key, value));
+		Reference<HTTP::IncomingResponse> response = co_await self->doKVRequest(self, key, value);
 
 		if (DEBUG_HTTPKV) {
 			fmt::print("CL:put {0}:{1} = {2} DONE\n", self->clientId, key, value);
@@ -341,17 +335,15 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 		self->activePut.reset();
 		self->myData[key] = value;
 		++self->putCount;
-
-		return Void();
 	}
 
-	ACTOR Future<Void> get(HTTPKeyValueStoreWorkload* self, std::string key, bool checkActive) {
+	Future<Void> get(HTTPKeyValueStoreWorkload* self, std::string key, bool checkActive) {
 
 		if (DEBUG_HTTPKV) {
 			fmt::print("CL:get {0}:{1}\n", self->clientId, key);
 		}
 
-		Reference<HTTP::IncomingResponse> response = wait(self->doKVRequest(self, key, {}));
+		Reference<HTTP::IncomingResponse> response = co_await self->doKVRequest(self, key, {});
 
 		if (DEBUG_HTTPKV) {
 			fmt::print("CL:get {0}:{1} = {2} DONE\n", self->clientId, key, response->data.content);
@@ -366,21 +358,19 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 			ASSERT(contentCorrect || inFlightCorrect);
 		}
 		++self->getCount;
-
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override { return _setup(this); }
 
-	ACTOR Future<Void> _setup(HTTPKeyValueStoreWorkload* self) {
+	Future<Void> _setup(HTTPKeyValueStoreWorkload* self) {
 		ASSERT(g_network->isSimulated());
 		if (self->clientId == 0) {
 			TraceEvent("SimHTTPKeyValueStoreRegistering");
 			if (DEBUG_HTTPKV) {
 				fmt::print("Registering sim http kv server\n");
 			}
-			wait(g_simulator->registerSimHTTPServer(
-			    self->hostname, self->service, makeReference<KeyValueRequestHandler>()));
+			co_await g_simulator->registerSimHTTPServer(
+			    self->hostname, self->service, makeReference<KeyValueRequestHandler>());
 			if (DEBUG_HTTPKV) {
 				fmt::print("Registered sim http kv server\n");
 			}
@@ -389,14 +379,11 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 
 		TraceEvent("SimHTTPKeyValueStoreLoading");
 
-		state int i;
-		for (i = 0; i < self->nodeCount; i++) {
-			wait(self->put(self, self->getKey(i), self->randomValue()));
+		for (int i = 0; i < self->nodeCount; i++) {
+			co_await self->put(self, self->getKey(i), self->randomValue());
 		}
 
 		TraceEvent("SimHTTPKeyValueStoreLoaded");
-
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -410,7 +397,7 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 		return _check(this);
 	}
 
-	ACTOR Future<bool> _check(HTTPKeyValueStoreWorkload* self) {
+	Future<bool> _check(HTTPKeyValueStoreWorkload* self) {
 		// reset conn since it could have been cancelled during part of initialization
 		if (self->conn) {
 			self->conn->close();
@@ -418,9 +405,8 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 		}
 
 		TraceEvent("SimHTTPKeyValueStoreWorkloadChecking");
-		state int i;
-		for (i = 0; i < self->nodeCount; i++) {
-			wait(self->get(self, self->getKey(i), true));
+		for (int i = 0; i < self->nodeCount; i++) {
+			co_await self->get(self, self->getKey(i), true);
 		}
 
 		TraceEvent("SimHTTPKeyValueStoreWorkloadChecked");
@@ -430,7 +416,7 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 			self->conn.clear();
 		}
 
-		return true;
+		co_return true;
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
@@ -440,23 +426,23 @@ struct HTTPKeyValueStoreWorkload : TestWorkload {
 		m.push_back(failedConnectCount.getMetric());
 	}
 
-	ACTOR Future<Void> httpKeyValueClient(HTTPKeyValueStoreWorkload* self) {
+	Future<Void> httpKeyValueClient(HTTPKeyValueStoreWorkload* self) {
 		TraceEvent("SimHTTPKeyValueStoreWorkloadStarting");
-		state double last = now();
-		loop {
-			state Future<Void> waitNextOp = poisson(&last, 1.0 / self->opsPerSecond);
+		double last = now();
+		while (true) {
+			Future<Void> waitNextOp = poisson(&last, 1.0 / self->opsPerSecond);
 
 			int key = deterministicRandom()->randomInt(0, self->nodeCount);
 
 			if (deterministicRandom()->coinflip()) {
-				wait(self->put(self, self->getKey(key), self->randomValue()));
+				co_await self->put(self, self->getKey(key), self->randomValue());
 			} else {
-				wait(self->get(self, self->getKey(key), false));
+				co_await self->get(self, self->getKey(key), false);
 			}
 
-			wait(waitNextOp);
+			co_await waitNextOp;
 		}
 	}
 };
 
-WorkloadFactory<HTTPKeyValueStoreWorkload> HTTPKeyValueStoreWorkloadFactory;
\ No newline at end of file
+WorkloadFactory<HTTPKeyValueStoreWorkload> HTTPKeyValueStoreWorkloadFactory;
diff --git a/fdbserver/workloads/HealthMetricsApi.actor.cpp b/fdbserver/workloads/HealthMetricsApi.cpp
similarity index 74%
rename from fdbserver/workloads/HealthMetricsApi.actor.cpp
rename to fdbserver/workloads/HealthMetricsApi.cpp
index 122d3576591..095c50510f9 100644
--- a/fdbserver/workloads/HealthMetricsApi.actor.cpp
+++ b/fdbserver/workloads/HealthMetricsApi.cpp
@@ -1,5 +1,5 @@
 /*
- * HealthMetricsApi.actor.cpp
+ * HealthMetricsApi.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 
 // NOTE: it might be simpler to test health metrics via something
 // other than simulation. Testing equivalent to what this workload does can
@@ -57,29 +56,25 @@ struct HealthMetricsApiWorkload : TestWorkload {
 	bool gotMetrics = false;
 	static constexpr auto NAME = "HealthMetricsApi";
 
-	HealthMetricsApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit HealthMetricsApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 120.0);
 		healthMetricsCheckInterval = getOption(options, "healthMetricsCheckInterval"_sr, 1.0);
 		sendDetailedHealthMetrics = getOption(options, "sendDetailedHealthMetrics"_sr, true);
 		maxAllowedStaleness = getOption(options, "maxAllowedStaleness"_sr, 60.0);
 	}
 
-	ACTOR static Future<Void> _setup(Database cx, HealthMetricsApiWorkload* self) {
-		if (!self->sendDetailedHealthMetrics) {
+	Future<Void> setup(Database const& cx) override {
+		if (!sendDetailedHealthMetrics) {
 			// Internally cached health metrics time out after this knob.  Wait
 			// an extra second to avoid any off-by-1 ">" vs ">=" type issues.
-			wait(delay(1 + CLIENT_KNOBS->DETAILED_HEALTH_METRICS_MAX_STALENESS));
+			co_await delay(1 + CLIENT_KNOBS->DETAILED_HEALTH_METRICS_MAX_STALENESS);
 			cx->healthMetrics.storageStats.clear();
 			cx->healthMetrics.tLogQueue.clear();
 		}
-		return Void();
 	}
-	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
-	ACTOR static Future<Void> _start(Database cx, HealthMetricsApiWorkload* self) {
-		wait(timeout(healthMetricsChecker(cx, self), self->testDuration, Void()));
-		return Void();
+	Future<Void> start(Database const& cx) override {
+		co_await timeout(healthMetricsChecker(cx), testDuration, Void());
 	}
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 
 	Future<bool> check(Database const& cx) override {
 		if (!gotMetrics) {
@@ -139,27 +134,25 @@ struct HealthMetricsApiWorkload : TestWorkload {
 		m.emplace_back("DetailedWorstDiskUsage", detailedWorstDiskUsage, Averaged::True);
 	}
 
-	ACTOR static Future<Void> healthMetricsChecker(Database cx, HealthMetricsApiWorkload* self) {
-		state int repeated = 0;
-		state HealthMetrics healthMetrics;
-		loop {
-			wait(delay(self->healthMetricsCheckInterval));
-			HealthMetrics newHealthMetrics = wait(cx->getHealthMetrics(self->sendDetailedHealthMetrics));
+	Future<Void> healthMetricsChecker(Database cx) {
+		int repeated = 0;
+		HealthMetrics healthMetrics;
+		while (true) {
+			co_await delay(healthMetricsCheckInterval);
+			HealthMetrics newHealthMetrics = co_await cx->getHealthMetrics(sendDetailedHealthMetrics);
 			if (healthMetrics == newHealthMetrics) {
-				if (++repeated > self->maxAllowedStaleness / self->healthMetricsCheckInterval)
-					self->healthMetricsStoppedUpdating = true;
+				if (++repeated > maxAllowedStaleness / healthMetricsCheckInterval)
+					healthMetricsStoppedUpdating = true;
 			} else
 				repeated = 0;
 			healthMetrics = newHealthMetrics;
 
-			self->worstStorageQueue = std::max(self->worstStorageQueue, healthMetrics.worstStorageQueue);
-			self->worstLimitingStorageQueue =
-			    std::max(self->worstLimitingStorageQueue, healthMetrics.limitingStorageQueue);
-			self->worstStorageDurabilityLag =
-			    std::max(self->worstStorageDurabilityLag, healthMetrics.worstStorageDurabilityLag);
-			self->worstLimitingStorageDurabilityLag =
-			    std::max(self->worstLimitingStorageDurabilityLag, healthMetrics.limitingStorageDurabilityLag);
-			self->worstTLogQueue = std::max(self->worstTLogQueue, healthMetrics.worstTLogQueue);
+			worstStorageQueue = std::max(worstStorageQueue, healthMetrics.worstStorageQueue);
+			worstLimitingStorageQueue = std::max(worstLimitingStorageQueue, healthMetrics.limitingStorageQueue);
+			worstStorageDurabilityLag = std::max(worstStorageDurabilityLag, healthMetrics.worstStorageDurabilityLag);
+			worstLimitingStorageDurabilityLag =
+			    std::max(worstLimitingStorageDurabilityLag, healthMetrics.limitingStorageDurabilityLag);
+			worstTLogQueue = std::max(worstTLogQueue, healthMetrics.worstTLogQueue);
 
 			TraceEvent("HealthMetrics")
 			    .detail("WorstStorageQueue", healthMetrics.worstStorageQueue)
@@ -178,15 +171,15 @@ struct HealthMetricsApiWorkload : TestWorkload {
 			for (const auto& ss : healthMetrics.storageStats) {
 				gotStorageStats = true;
 				auto storageStats = ss.second;
-				self->detailedWorstStorageQueue = std::max(self->detailedWorstStorageQueue, storageStats.storageQueue);
+				detailedWorstStorageQueue = std::max(detailedWorstStorageQueue, storageStats.storageQueue);
 				traceStorageQueue.detail(format("Storage-%s", ss.first.toString().c_str()), storageStats.storageQueue);
-				self->detailedWorstStorageDurabilityLag =
-				    std::max(self->detailedWorstStorageDurabilityLag, storageStats.storageDurabilityLag);
+				detailedWorstStorageDurabilityLag =
+				    std::max(detailedWorstStorageDurabilityLag, storageStats.storageDurabilityLag);
 				traceStorageDurabilityLag.detail(format("Storage-%s", ss.first.toString().c_str()),
 				                                 storageStats.storageDurabilityLag);
-				self->detailedWorstCpuUsage = std::max(self->detailedWorstCpuUsage, storageStats.cpuUsage);
+				detailedWorstCpuUsage = std::max(detailedWorstCpuUsage, storageStats.cpuUsage);
 				traceCpuUsage.detail(format("Storage-%s", ss.first.toString().c_str()), storageStats.cpuUsage);
-				self->detailedWorstDiskUsage = std::max(self->detailedWorstDiskUsage, storageStats.diskUsage);
+				detailedWorstDiskUsage = std::max(detailedWorstDiskUsage, storageStats.diskUsage);
 				traceDiskUsage.detail(format("Storage-%s", ss.first.toString().c_str()), storageStats.diskUsage);
 			}
 			TraceEvent traceTLogQueue("TLogQueue");
@@ -194,12 +187,12 @@ struct HealthMetricsApiWorkload : TestWorkload {
 			bool gotTLogQueue = false;
 			for (const auto& ss : healthMetrics.tLogQueue) {
 				gotTLogQueue = true;
-				self->detailedWorstTLogQueue = std::max(self->detailedWorstTLogQueue, ss.second);
+				detailedWorstTLogQueue = std::max(detailedWorstTLogQueue, ss.second);
 				traceTLogQueue.detail(format("TLog-%s", ss.first.toString().c_str()), ss.second);
 			}
-			if (!self->gotMetrics && gotStorageStats && gotTLogQueue) {
+			if (!gotMetrics && gotStorageStats && gotTLogQueue) {
 				TraceEvent("HealthMetricsGotFullResult");
-				self->gotMetrics = true;
+				gotMetrics = true;
 			}
 		};
 	}
diff --git a/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp b/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.cpp
similarity index 58%
rename from fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp
rename to fdbserver/workloads/HighContentionPrefixAllocatorWorkload.cpp
index b3d43b0d0ca..aee3388648c 100644
--- a/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.actor.cpp
+++ b/fdbserver/workloads/HighContentionPrefixAllocatorWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * HighContentionPrefixAllocatorWorkload.actor.cpp
+ * HighContentionPrefixAllocatorWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbclient/HighContentionPrefixAllocator.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/HighContentionPrefixAllocator.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 // This workload tests the basic contract of the high contention allocator
 struct HighContentionPrefixAllocatorWorkload : TestWorkload {
@@ -36,7 +35,7 @@ struct HighContentionPrefixAllocatorWorkload : TestWorkload {
 	int expectedPrefixes = 0;
 	std::set<Key> allocatedPrefixes;
 
-	HighContentionPrefixAllocatorWorkload(WorkloadContext const& wcx)
+	explicit HighContentionPrefixAllocatorWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), allocatorSubspace("test_subspace"_sr), allocator(allocatorSubspace) {
 		numRounds = getOption(options, "numRounds"_sr, 100);
 		maxTransactionsPerRound = getOption(options, "maxTransactionsPerRound"_sr, 20);
@@ -45,28 +44,29 @@ struct HighContentionPrefixAllocatorWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	ACTOR static Future<Void> runAllocationTransaction(Database cx, HighContentionPrefixAllocatorWorkload* self) {
-		state Reference<ReadYourWritesTransaction> tr = cx->createTransaction();
+	Future<Void> runAllocationTransaction(Database cx) {
+		Reference<ReadYourWritesTransaction> tr = cx->createTransaction();
 
-		state int numAllocations = deterministicRandom()->randomInt(1, self->maxAllocationsPerTransaction + 1);
-		self->expectedPrefixes += numAllocations;
+		int numAllocations = deterministicRandom()->randomInt(1, maxAllocationsPerTransaction + 1);
+		expectedPrefixes += numAllocations;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state std::vector<Future<Key>> futures;
+				std::vector<Future<Key>> futures;
 				for (int i = 0; i < numAllocations; ++i) {
-					futures.push_back(self->allocator.allocate(tr));
+					futures.push_back(allocator.allocate(tr));
 				}
 
-				wait(waitForAll(futures));
-				wait(tr->commit());
+				co_await waitForAll(futures);
+				co_await tr->commit();
 
-				for (auto f : futures) {
+				for (const auto& f : futures) {
 					Key prefix = f.get();
 
 					// There should be no previously allocated prefix that is prefixed by our newly allocated one
-					auto itr = self->allocatedPrefixes.lower_bound(prefix);
-					if (itr != self->allocatedPrefixes.end() && itr->startsWith(prefix)) {
+					auto itr = allocatedPrefixes.lower_bound(prefix);
+					if (itr != allocatedPrefixes.end() && itr->startsWith(prefix)) {
 						TraceEvent(SevError, "HighContentionAllocationWorkloadFailure")
 						    .detail("Reason", "Prefix collision")
 						    .detail("AllocatedPrefix", prefix)
@@ -76,7 +76,7 @@ struct HighContentionPrefixAllocatorWorkload : TestWorkload {
 					}
 
 					// There should be no previously allocated prefix that is a prefix of our newly allocated one
-					if (itr != self->allocatedPrefixes.begin()) {
+					if (itr != allocatedPrefixes.begin()) {
 						--itr;
 
 						if (prefix.startsWith(*itr)) {
@@ -90,67 +90,64 @@ struct HighContentionPrefixAllocatorWorkload : TestWorkload {
 					}
 
 					// This is technically redundant, but the prefix should not have been allocated previously
-					ASSERT(self->allocatedPrefixes.insert(f.get()).second);
+					ASSERT(allocatedPrefixes.insert(f.get()).second);
 				}
 
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> runTest(Database cx, HighContentionPrefixAllocatorWorkload* self) {
-		state int roundNum = 0;
-		for (; roundNum < self->numRounds; ++roundNum) {
+	Future<Void> runTest(Database cx) {
+		for (int roundNum = 0; roundNum < numRounds; ++roundNum) {
 			std::vector<Future<Void>> futures;
-			int numTransactions = deterministicRandom()->randomInt(1, self->maxTransactionsPerRound + 1);
+			int numTransactions = deterministicRandom()->randomInt(1, maxTransactionsPerRound + 1);
 			for (int i = 0; i < numTransactions; ++i) {
-				futures.push_back(runAllocationTransaction(cx, self));
+				futures.push_back(runAllocationTransaction(cx));
 			}
 
-			wait(waitForAll(futures));
+			co_await waitForAll(futures);
 		}
-
-		return Void();
 	}
 
-	Future<Void> start(Database const& cx) override { return runTest(cx, this); }
+	Future<Void> start(Database const& cx) override { return runTest(cx); }
 
-	ACTOR static Future<bool> _check(Database cx, HighContentionPrefixAllocatorWorkload* self) {
-		if (self->expectedPrefixes != self->allocatedPrefixes.size()) {
+	Future<bool> check(Database const& cx) override {
+		if (expectedPrefixes != allocatedPrefixes.size()) {
 			TraceEvent(SevError, "HighContentionAllocationWorkloadFailure")
 			    .detail("Reason", "Incorrect Number of Prefixes Allocated")
-			    .detail("NumAllocated", self->allocatedPrefixes.size())
-			    .detail("Expected", self->expectedPrefixes);
+			    .detail("NumAllocated", allocatedPrefixes.size())
+			    .detail("Expected", expectedPrefixes);
 
-			return false;
+			co_return false;
 		}
 
-		state Reference<ReadYourWritesTransaction> tr = cx->createTransaction();
-		loop {
+		Reference<ReadYourWritesTransaction> tr = cx->createTransaction();
+		while (true) {
+			Error err;
 			try {
-				state Key k1 = wait(tr->getKey(firstGreaterOrEqual(""_sr)));
-				Key k2 = wait(tr->getKey(lastLessThan("\xff"_sr)));
-				if (!k1.startsWith(self->allocatorSubspace.key()) || !k2.startsWith(self->allocatorSubspace.key())) {
+				Key k1 = co_await tr->getKey(firstGreaterOrEqual(""_sr));
+				Key k2 = co_await tr->getKey(lastLessThan("\xff"_sr));
+				if (!k1.startsWith(allocatorSubspace.key()) || !k2.startsWith(allocatorSubspace.key())) {
 					TraceEvent(SevError, "HighContentionAllocationWorkloadFailure")
 					    .detail("Reason", "Keys written outside allocator subspace")
 					    .detail("MinKey", k1)
 					    .detail("MaxKey", k2);
 
-					return false;
+					co_return false;
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 
-		return true;
+		co_return true;
 	}
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
diff --git a/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp b/fdbserver/workloads/IDDTxnProcessorApiCorrectness.cpp
similarity index 61%
rename from fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp
rename to fdbserver/workloads/IDDTxnProcessorApiCorrectness.cpp
index 72701471413..0d27a282772 100644
--- a/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp
+++ b/fdbserver/workloads/IDDTxnProcessorApiCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * IDDTxnProcessorApiCorrectness.actor.cpp
+ * IDDTxnProcessorApiCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,15 +19,14 @@
  */
 
 #include "fdbclient/FDBOptions.g.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/DDTxnProcessor.h"
-#include "fdbserver/MoveKeys.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
+#include "fdbserver/core/MoveKeys.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/Knobs.h"
 #include "fdbclient/VersionedMap.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 std::string describe(const DDShardInfo& a) {
 	std::string res = "key: " + a.key.toString() + "\n";
@@ -126,14 +125,14 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 	// counters
 	int testRawStart = 0, testAll = 0, testRawFinish = 0;
 
-	IDDTxnProcessorApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddContext(UID()) {
+	explicit IDDTxnProcessorApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ddContext(UID()) {
 		enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		testStartOnly = getOption(options, "testStartOnly"_sr, false);
 	}
 
 	Future<Void> setup(Database const& cx) override { return enabled ? _setup(cx, this) : Void(); }
-	Future<Void> start(Database const& cx) override { return enabled ? _start(cx, this) : Void(); }
+	Future<Void> start(Database const& cx) override { return enabled ? _start(cx) : Void(); }
 
 	// This workload is not compatible with RandomMoveKeys workload because they will race in changing the DD mode.
 	// Other workload injections may make no sense because this workload only use the DB at beginning to reading the
@@ -142,28 +141,22 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 		out.insert({ "RandomMoveKeys", "Attrition" });
 	}
 
-	ACTOR static Future<Void> readRealInitialDataDistribution(IDDTxnProcessorApiWorkload* self) {
-		loop {
-			wait(store(self->ddContext.lock, ::readMoveKeysLock(self->real->context())));
+	Future<Void> readRealInitialDataDistribution() {
+		while (true) {
+			ddContext.lock = co_await ::readMoveKeysLock(real->context());
 			// read real InitialDataDistribution
 			try {
-				wait(store(self->realInitDD,
-				           self->real->getInitialDataDistribution(self->ddContext.id(),
-				                                                  self->ddContext.lock,
-				                                                  {},
-				                                                  self->ddContext.ddEnabledState.get(),
-				                                                  SkipDDModeCheck::True)));
-				std::cout << "Finish read real InitialDataDistribution: server size "
-				          << self->realInitDD->allServers.size() << ", shard size: " << self->realInitDD->shards.size()
-				          << std::endl;
+				realInitDD = co_await real->getInitialDataDistribution(
+				    ddContext.id(), ddContext.lock, {}, ddContext.ddEnabledState.get(), SkipDDModeCheck::True);
+				std::cout << "Finish read real InitialDataDistribution: server size " << realInitDD->allServers.size()
+				          << ", shard size: " << realInitDD->shards.size() << std::endl;
 				break;
 			} catch (Error& e) {
 				if (e.code() != error_code_movekeys_conflict)
 					throw;
 			}
 		}
-		self->updateBoundaries();
-		return Void();
+		updateBoundaries();
 	}
 
 	// according to boundaries, generate valid ranges for moveKeys operation
@@ -217,47 +210,41 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 		boundaries = std::vector<Key>(tempBoundaries.begin(), tempBoundaries.end());
 	}
 
-	ACTOR Future<Void> _setup(Database cx, IDDTxnProcessorApiWorkload* self) {
-		int oldMode = wait(setDDMode(cx, 0));
+	Future<Void> _setup(Database cx, IDDTxnProcessorApiWorkload* self) {
+		int oldMode = co_await setDDMode(cx, 0);
 		TraceEvent("IDDTxnApiTestStartModeSetting").detail("OldValue", oldMode).log();
 
 		self->real = std::make_shared<DDTxnProcessorTester>(cx);
 		// Get the database configuration so as to use proper team size
-		wait(store(self->ddContext.configuration, self->real->getDatabaseConfiguration()));
+		self->ddContext.configuration = co_await self->real->getDatabaseConfiguration();
 		ASSERT(self->ddContext.configuration.storageTeamSize > 0);
 		// FIXME: add support for generating random teams across DCs
 		ASSERT_EQ(self->ddContext.usableRegions(), 1);
-		wait(readRealInitialDataDistribution(self));
-		return Void();
+		co_await readRealInitialDataDistribution();
 	}
 
-	ACTOR Future<Void> _start(Database cx, IDDTxnProcessorApiWorkload* self) {
+	Future<Void> _start(Database cx) {
 
-		self->mgs = std::make_shared<MockGlobalState>();
-		self->mgs->configuration = self->ddContext.configuration;
-		self->mgs->restrictSize = false; // no need to check the validity of shard size
+		mgs = std::make_shared<MockGlobalState>();
+		mgs->configuration = ddContext.configuration;
+		mgs->restrictSize = false; // no need to check the validity of shard size
 
-		self->mock = std::make_shared<DDMockTxnProcessorTester>(self->mgs);
-		self->mock->setupMockGlobalState(self->realInitDD);
+		mock = std::make_shared<DDMockTxnProcessorTester>(mgs);
+		mock->setupMockGlobalState(realInitDD);
 
 		Reference<InitialDataDistribution> mockInitData =
-		    self->mock
-		        ->getInitialDataDistribution(self->ddContext.id(),
-		                                     self->ddContext.lock,
-		                                     {},
-		                                     self->ddContext.ddEnabledState.get(),
-		                                     SkipDDModeCheck::True)
+		    mock->getInitialDataDistribution(
+		            ddContext.id(), ddContext.lock, {}, ddContext.ddEnabledState.get(), SkipDDModeCheck::True)
 		        .get();
 
-		verifyInitDataEqual(self->realInitDD, mockInitData);
+		verifyInitDataEqual(realInitDD, mockInitData);
 
-		wait(timeout(reportErrors(self->worker(cx, self), "IDDTxnProcessorApiWorkload"), self->testDuration, Void()));
+		co_await timeout(reportErrors(worker(cx, this), "IDDTxnProcessorApiWorkload"), testDuration, Void());
 
 		// Always set the DD mode back, even if we die with an error
 		TraceEvent("IDDTxnApiTestDoneMoving").log();
-		int oldValue = wait(setDDMode(cx, 1));
+		int oldValue = co_await setDDMode(cx, 1);
 		TraceEvent("IDDTxnApiTestDoneModeSetting").detail("OldValue", oldValue);
-		return Void();
 	}
 
 	void verifyServerKeyDest(MoveKeysParams& params) const {
@@ -277,13 +264,13 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 			ASSERT(mgs->serverIsDestForShard(id, keys));
 		}
 	}
-	ACTOR static Future<Void> testRawMovementApi(IDDTxnProcessorApiWorkload* self) {
-		state TraceInterval relocateShardInterval("RelocateShard_TestRawMovementApi");
-		state FlowLock fl1(1);
-		state FlowLock fl2(1);
-		state std::map<UID, StorageServerInterface> emptyTssMapping;
-		state Reference<InitialDataDistribution> mockInitData;
-		state MoveKeysParams realParams = wait(generateMoveKeysParams(self));
+	Future<Void> testRawMovementApi() {
+		TraceInterval relocateShardInterval("RelocateShard_TestRawMovementApi");
+		FlowLock fl1(1);
+		FlowLock fl2(1);
+		std::map<UID, StorageServerInterface> emptyTssMapping;
+		Reference<InitialDataDistribution> mockInitData;
+		MoveKeysParams realParams = co_await generateMoveKeysParams();
 		realParams.startMoveKeysParallelismLock = &fl1;
 		realParams.finishMoveKeysParallelismLock = &fl2;
 		realParams.relocationIntervalId = relocateShardInterval.pairID;
@@ -291,64 +278,62 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 		    .detail("Key", realParams.keys)
 		    .detail("Dest", realParams.destinationTeam);
 
-		state MoveKeysParams mockParams = realParams;
+		MoveKeysParams mockParams = realParams;
 		mockParams.dataMovementComplete = Promise<Void>();
 
-		loop {
+		while (true) {
 			realParams.dataMovementComplete.reset();
 			mockParams.dataMovementComplete.reset();
 
-			wait(store(realParams.lock, self->real->takeMoveKeysLock(UID())));
+			realParams.lock = co_await real->takeMoveKeysLock(UID());
+			Error err;
 			try {
 				// test start
-				wait(self->mock->testRawStartMovement(mockParams, emptyTssMapping));
-				wait(self->real->testRawStartMovement(realParams, emptyTssMapping));
+				co_await mock->testRawStartMovement(mockParams, emptyTssMapping);
+				co_await real->testRawStartMovement(realParams, emptyTssMapping);
 
-				self->verifyServerKeyDest(realParams);
+				verifyServerKeyDest(realParams);
 				// test finish or started but cancelled movement
-				if (self->testStartOnly || deterministicRandom()->coinflip()) {
+				if (testStartOnly || deterministicRandom()->coinflip()) {
 					CODE_PROBE(true, "RawMovementApi partial started");
-					self->testRawStart++;
+					testRawStart++;
 					break;
 				}
 
 				// The real transaction should finish first because mock transaction will always success.
-				wait(self->real->testRawFinishMovement(realParams, emptyTssMapping));
-				wait(self->mock->testRawFinishMovement(mockParams, emptyTssMapping));
-				self->testRawFinish++;
+				co_await real->testRawFinishMovement(realParams, emptyTssMapping);
+				co_await mock->testRawFinishMovement(mockParams, emptyTssMapping);
+				testRawFinish++;
 				break;
 			} catch (Error& e) {
-				if (e.code() != error_code_movekeys_conflict)
-					throw;
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				// Keep trying to get the moveKeysLock
+				err = e;
 			}
+			if (err.code() != error_code_movekeys_conflict)
+				throw err;
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			// Keep trying to get the moveKeysLock
 		}
 
 		// read initial data again
-		wait(readRealInitialDataDistribution(self));
-		mockInitData = self->mock
-		                   ->getInitialDataDistribution(self->ddContext.id(),
-		                                                self->ddContext.lock,
-		                                                {},
-		                                                self->ddContext.ddEnabledState.get(),
-		                                                SkipDDModeCheck::True)
-		                   .get();
-
-		verifyInitDataEqual(self->realInitDD, mockInitData);
+		co_await readRealInitialDataDistribution();
+		mockInitData =
+		    mock->getInitialDataDistribution(
+		            ddContext.id(), ddContext.lock, {}, ddContext.ddEnabledState.get(), SkipDDModeCheck::True)
+		        .get();
+
+		verifyInitDataEqual(realInitDD, mockInitData);
 		TraceEvent(SevDebug, relocateShardInterval.end(), relocateShardInterval.pairID);
 		// The simulator have chances generating a scenario when after the first setupMockGlobalState call, there is a
 		// new storage server join the cluster, there's no way for mock DD to know the new storage server without
 		// calling setupMockGlobalState again.
-		self->mock->setupMockGlobalState(self->realInitDD);
-		return Void();
+		mock->setupMockGlobalState(realInitDD);
 	}
 
-	ACTOR static Future<MoveKeysParams> generateMoveKeysParams(IDDTxnProcessorApiWorkload* self) { // always empty
-		state MoveKeysLock lock = wait(takeMoveKeysLock(self->real->context(), UID()));
+	Future<MoveKeysParams> generateMoveKeysParams() { // always empty
+		MoveKeysLock lock = co_await takeMoveKeysLock(real->context(), UID());
 
-		KeyRange keys = self->getRandomKeys();
-		std::vector<UID> destTeam = self->getRandomTeam();
+		KeyRange keys = getRandomKeys();
+		std::vector<UID> destTeam = getRandomTeam();
 		std::sort(destTeam.begin(), destTeam.end());
 		const UID dataMoveId = newDataMoveId(deterministicRandom()->randomUInt64(),
 		                                     AssignEmptyRange(false),
@@ -356,43 +341,43 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 		                                     DataMovementReason::INVALID,
 		                                     UnassignShard(false));
 		if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
-			return MoveKeysParams(dataMoveId,
-			                      std::vector<KeyRange>{ keys },
-			                      destTeam,
-			                      destTeam,
-			                      lock,
-			                      Promise<Void>(),
-			                      nullptr,
-			                      nullptr,
-			                      false,
-			                      UID(),
-			                      self->ddContext.ddEnabledState.get(),
-			                      CancelConflictingDataMoves::True,
-			                      Optional<BulkLoadTaskState>());
+			co_return MoveKeysParams(dataMoveId,
+			                         std::vector<KeyRange>{ keys },
+			                         destTeam,
+			                         destTeam,
+			                         lock,
+			                         Promise<Void>(),
+			                         nullptr,
+			                         nullptr,
+			                         false,
+			                         UID(),
+			                         ddContext.ddEnabledState.get(),
+			                         CancelConflictingDataMoves::True,
+			                         Optional<BulkLoadTaskState>());
 		} else {
-			return MoveKeysParams(dataMoveId,
-			                      keys,
-			                      destTeam,
-			                      destTeam,
-			                      lock,
-			                      Promise<Void>(),
-			                      nullptr,
-			                      nullptr,
-			                      false,
-			                      UID(),
-			                      self->ddContext.ddEnabledState.get(),
-			                      CancelConflictingDataMoves::True,
-			                      Optional<BulkLoadTaskState>());
+			co_return MoveKeysParams(dataMoveId,
+			                         keys,
+			                         destTeam,
+			                         destTeam,
+			                         lock,
+			                         Promise<Void>(),
+			                         nullptr,
+			                         nullptr,
+			                         false,
+			                         UID(),
+			                         ddContext.ddEnabledState.get(),
+			                         CancelConflictingDataMoves::True,
+			                         Optional<BulkLoadTaskState>());
 		}
 	}
 
-	ACTOR static Future<Void> testMoveKeys(IDDTxnProcessorApiWorkload* self) {
-		state TraceInterval relocateShardInterval("RelocateShard_TestMoveKeys");
-		state FlowLock fl1(1);
-		state FlowLock fl2(1);
-		state std::map<UID, StorageServerInterface> emptyTssMapping;
-		state Reference<InitialDataDistribution> mockInitData;
-		state MoveKeysParams realParams = wait(generateMoveKeysParams(self));
+	Future<Void> testMoveKeys() {
+		TraceInterval relocateShardInterval("RelocateShard_TestMoveKeys");
+		FlowLock fl1(1);
+		FlowLock fl2(1);
+		std::map<UID, StorageServerInterface> emptyTssMapping;
+		Reference<InitialDataDistribution> mockInitData;
+		MoveKeysParams realParams = co_await generateMoveKeysParams();
 		realParams.startMoveKeysParallelismLock = &fl1;
 		realParams.finishMoveKeysParallelismLock = &fl2;
 		realParams.relocationIntervalId = relocateShardInterval.pairID;
@@ -400,55 +385,53 @@ struct IDDTxnProcessorApiWorkload : TestWorkload {
 		    .detail("Key", realParams.keys)
 		    .detail("Dest", realParams.destinationTeam);
 
-		state MoveKeysParams mockParams = realParams;
+		MoveKeysParams mockParams = realParams;
 		mockParams.dataMovementComplete = Promise<Void>();
 
-		loop {
+		while (true) {
 			realParams.dataMovementComplete.reset();
 			mockParams.dataMovementComplete.reset();
-			wait(store(realParams.lock, self->real->takeMoveKeysLock(UID())));
+			realParams.lock = co_await real->takeMoveKeysLock(UID());
+			Error err;
 			try {
-				wait(self->mock->moveKeys(mockParams));
-				wait(self->real->moveKeys(realParams));
-				self->testAll++;
+				co_await mock->moveKeys(mockParams);
+				co_await real->moveKeys(realParams);
+				testAll++;
 				break;
 			} catch (Error& e) {
-				if (e.code() != error_code_movekeys_conflict)
-					throw;
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				// Keep trying to get the moveKeysLock
+				err = e;
 			}
+			if (err.code() != error_code_movekeys_conflict)
+				throw err;
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			// Keep trying to get the moveKeysLock
 		}
 
 		// read initial data again
-		wait(readRealInitialDataDistribution(self));
-		mockInitData = self->mock
-		                   ->getInitialDataDistribution(self->ddContext.id(),
-		                                                self->ddContext.lock,
-		                                                {},
-		                                                self->ddContext.ddEnabledState.get(),
-		                                                SkipDDModeCheck::True)
-		                   .get();
-
-		verifyInitDataEqual(self->realInitDD, mockInitData);
+		co_await readRealInitialDataDistribution();
+		mockInitData =
+		    mock->getInitialDataDistribution(
+		            ddContext.id(), ddContext.lock, {}, ddContext.ddEnabledState.get(), SkipDDModeCheck::True)
+		        .get();
+
+		verifyInitDataEqual(realInitDD, mockInitData);
 		TraceEvent(SevDebug, relocateShardInterval.end(), relocateShardInterval.pairID);
-		self->mock->setupMockGlobalState(self->realInitDD); // in case SS remove or recruit
-		return Void();
+		mock->setupMockGlobalState(realInitDD); // in case SS remove or recruit
 	}
 
-	ACTOR Future<Void> worker(Database cx, IDDTxnProcessorApiWorkload* self) {
-		state int choice = 0;
-		state int maxChoice = self->testStartOnly ? 1 : 2;
-		loop {
+	Future<Void> worker(Database cx, IDDTxnProcessorApiWorkload* self) {
+		int choice = 0;
+		int maxChoice = self->testStartOnly ? 1 : 2;
+		while (true) {
 			choice = deterministicRandom()->randomInt(0, maxChoice);
 			if (choice == 0) { // test rawStartMovement and rawFinishMovement separately
-				wait(testRawMovementApi(self));
+				co_await testRawMovementApi();
 			} else if (choice == 1) { // test moveKeys
-				wait(testMoveKeys(self));
+				co_await testMoveKeys();
 			} else {
 				ASSERT(false);
 			}
-			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 		}
 	}
 
diff --git a/fdbserver/workloads/Increment.actor.cpp b/fdbserver/workloads/Increment.cpp
similarity index 80%
rename from fdbserver/workloads/Increment.actor.cpp
rename to fdbserver/workloads/Increment.cpp
index 759370a7326..ba95521ab28 100644
--- a/fdbserver/workloads/Increment.actor.cpp
+++ b/fdbserver/workloads/Increment.cpp
@@ -1,5 +1,5 @@
 /*
- * Increment.actor.cpp
+ * Increment.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,9 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 struct Increment : TestWorkload {
 	static constexpr auto NAME = "Increment";
@@ -33,7 +32,7 @@ struct Increment : TestWorkload {
 	PerfIntCounter transactions, retries, tooOldRetries, commitFailedRetries;
 	PerfDoubleCounter totalLatency;
 
-	Increment(WorkloadContext const& wcx)
+	explicit Increment(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), tooOldRetries("Retries.too_old"),
 	    commitFailedRetries("Retries.commit_failed"), totalLatency("Latency") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -71,15 +70,16 @@ struct Increment : TestWorkload {
 
 	static Key intToTestKey(int i) { return StringRef(format("%016d", i)); }
 
-	ACTOR Future<Void> incrementClient(Database cx, Increment* self, double delay) {
-		state double lastTime = now();
+	Future<Void> incrementClient(Database cx, Increment* self, double delay) {
+		double lastTime = now();
 		try {
-			loop {
-				wait(poisson(&lastTime, delay));
+			while (true) {
+				co_await poisson(&lastTime, delay);
 
-				state double tstart = now();
-				state Transaction tr(cx);
+				double tstart = now();
+				Transaction tr(cx);
 				while (true) {
+					Error err;
 					try {
 						tr.atomicOp(intToTestKey(deterministicRandom()->randomInt(0, self->nodeCount / 2)),
 						            "\x01"_sr,
@@ -88,15 +88,16 @@ struct Increment : TestWorkload {
 						    intToTestKey(deterministicRandom()->randomInt(self->nodeCount / 2, self->nodeCount)),
 						    "\x01"_sr,
 						    MutationRef::AddValue);
-						wait(tr.commit());
+						co_await tr.commit();
 						break;
 					} catch (Error& e) {
-						if (e.code() == error_code_transaction_too_old)
-							++self->tooOldRetries;
-						else if (e.code() == error_code_not_committed)
-							++self->commitFailedRetries;
-						wait(tr.onError(e));
+						err = e;
 					}
+					if (err.code() == error_code_transaction_too_old)
+						++self->tooOldRetries;
+					else if (err.code() == error_code_not_committed)
+						++self->commitFailedRetries;
+					co_await tr.onError(err);
 					++self->retries;
 				}
 				++self->transactions;
@@ -109,7 +110,7 @@ struct Increment : TestWorkload {
 	}
 	bool incrementCheckData(const VectorRef<KeyValueRef>& data, Version v, Increment* self) {
 		CODE_PROBE(self->transactions.getValue(), "incrementCheckData transaction has value");
-		if (self->transactions.getValue() && data.size() == 0) {
+		if (self->transactions.getValue() && data.empty()) {
 			TraceEvent(SevError, "TestFailure")
 			    .detail("Reason", "No successful increments")
 			    .detail("Before", nodeCount)
@@ -138,7 +139,7 @@ struct Increment : TestWorkload {
 		}
 		return true;
 	}
-	ACTOR Future<bool> incrementCheck(Database cx, Increment* self, bool ok) {
+	Future<bool> incrementCheck(Database cx, Increment* self, bool ok) {
 		if (self->transactions.getMetric().value() < self->testDuration * self->minExpectedTransactionsPerSecond) {
 			TraceEvent(SevWarnAlways, "TestFailure")
 			    .detail("Reason", "Rate below desired rate")
@@ -154,24 +155,26 @@ struct Increment : TestWorkload {
 		}
 		if (!self->clientId) {
 			// One client checks the validity of the cycle
-			state Transaction tr(cx);
-			state int retryCount = 0;
-			loop {
+			Transaction tr(cx);
+			int retryCount = 0;
+			while (true) {
+				Error err;
 				try {
-					state Version v = wait(tr.getReadVersion());
-					RangeResult data = wait(tr.getRange(firstGreaterOrEqual(intToTestKey(0)),
-					                                    firstGreaterOrEqual(intToTestKey(self->nodeCount)),
-					                                    self->nodeCount + 1));
+					Version v = co_await tr.getReadVersion();
+					RangeResult data = co_await tr.getRange(firstGreaterOrEqual(intToTestKey(0)),
+					                                        firstGreaterOrEqual(intToTestKey(self->nodeCount)),
+					                                        self->nodeCount + 1);
 					ok = self->incrementCheckData(data, v, self) && ok;
 					break;
 				} catch (Error& e) {
-					retryCount++;
-					TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "IncrementCheckError").error(e);
-					wait(tr.onError(e));
+					err = e;
 				}
+				retryCount++;
+				TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "IncrementCheckError").error(err);
+				co_await tr.onError(err);
 			}
 		}
-		return ok;
+		co_return ok;
 	}
 };
 
diff --git a/fdbserver/workloads/IncrementalBackup.actor.cpp b/fdbserver/workloads/IncrementalBackup.cpp
similarity index 52%
rename from fdbserver/workloads/IncrementalBackup.actor.cpp
rename to fdbserver/workloads/IncrementalBackup.cpp
index b59bc506c08..91659e4907c 100644
--- a/fdbserver/workloads/IncrementalBackup.actor.cpp
+++ b/fdbserver/workloads/IncrementalBackup.cpp
@@ -1,5 +1,5 @@
 /*
- * IncrementalBackup.actor.cpp
+ * IncrementalBackup.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,20 +21,20 @@
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/Knobs.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
 #include "flow/Arena.h"
 #include "flow/Platform.h"
 #include "flow/Trace.h"
 #include "flow/serialize.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // TODO: explain the purpose of this workload and how it different from the
 // 20+ (literally) other backup/restore workloads.
@@ -55,7 +55,7 @@ struct IncrementalBackupWorkload : TestWorkload {
 	Standalone<StringRef> blobManifestUrl;
 	Optional<std::string> encryptionKeyFileName;
 
-	IncrementalBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit IncrementalBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
 		tag = getOption(options, "tag"_sr, "default"_sr);
 		submitOnly = getOption(options, "submitOnly"_sr, false);
@@ -84,65 +84,65 @@ struct IncrementalBackupWorkload : TestWorkload {
 		if (clientId) {
 			return Void();
 		}
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override {
 		if (clientId) {
 			return true;
 		}
-		return _check(cx, this);
+		return _check(cx);
 	}
 
-	ACTOR static Future<bool> _check(Database cx, IncrementalBackupWorkload* self) {
-		if (self->waitForBackup) {
+	Future<bool> _check(Database cx) {
+		if (waitForBackup) {
 			// Undergoing recovery with the snapshot system keys set will pause the backup agent
 			// Pre-emptively unpause any backup agents before attempting to wait to avoid getting stuck
-			wait(self->backupAgent.changePause(cx, false));
-			state Reference<IBackupContainer> backupContainer;
-			state UID backupUID;
-			state Version v;
-			state Transaction tr(cx);
-			loop {
+			co_await backupAgent.changePause(cx, false);
+			Reference<IBackupContainer> backupContainer;
+			UID backupUID;
+			Version v{ 0 };
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					wait(store(v, tr.getReadVersion()));
+					v = co_await tr.getReadVersion();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
-			loop {
+			while (true) {
 				// Wait for backup container to be created and avoid race condition
 				TraceEvent("IBackupWaitContainer").log();
-				wait(success(self->backupAgent.waitBackup(
-				    cx, self->tag.toString(), StopWhenDone::False, &backupContainer, &backupUID)));
+				co_await backupAgent.waitBackup(cx, tag.toString(), StopWhenDone::False, &backupContainer, &backupUID);
 
-				state Optional<std::string> restoreEncryptionKeyFileName;
-				if (self->encryptionKeyFileName.present() && fileExists(self->encryptionKeyFileName.get())) {
-					restoreEncryptionKeyFileName = self->encryptionKeyFileName.get();
+				Optional<std::string> restoreEncryptionKeyFileName;
+				if (encryptionKeyFileName.present() && fileExists(encryptionKeyFileName.get())) {
+					restoreEncryptionKeyFileName = encryptionKeyFileName.get();
 				}
 
 				if (!backupContainer.isValid()) {
 					TraceEvent("IBackupCheckListContainersAttempt").log();
-					state std::vector<std::string> containers =
-					    wait(IBackupContainer::listContainers(self->backupDir.toString(), {}));
+					std::vector<std::string> containers =
+					    co_await IBackupContainer::listContainers(backupDir.toString(), {});
 					TraceEvent("IBackupCheckListContainersSuccess")
 					    .detail("Size", containers.size())
 					    .detail("First", containers.front());
-					if (containers.size()) {
-						backupContainer =
-						    IBackupContainer::openContainer(containers.front(), {}, restoreEncryptionKeyFileName);
+					if (!containers.empty()) {
+						backupContainer = IBackupContainer::openContainer(containers.front(), {}, {}, 0);
 					}
 				}
-				state bool e = wait(backupContainer->exists());
+				bool e = co_await backupContainer->exists();
 				if (e)
 					break;
-				wait(delay(5.0));
+				co_await delay(5.0);
 			}
-			state int tries = 0;
-			loop {
+			int tries = 0;
+			while (true) {
 				tries++;
-				state BackupDescription desc = wait(backupContainer->describeBackup(true));
+				BackupDescription desc = co_await backupContainer->describeBackup(true);
 				TraceEvent("IBackupVersionGate")
 				    .detail("MaxLogEndVersion", desc.maxLogEnd.present() ? desc.maxLogEnd.get() : invalidVersion)
 				    .detail("ContiguousLogEndVersion",
@@ -150,21 +150,21 @@ struct IncrementalBackupWorkload : TestWorkload {
 				    .detail("TargetVersion", v);
 
 				if (!desc.contiguousLogEnd.present()) {
-					wait(delay(5.0));
+					co_await delay(5.0);
 					continue;
 				}
 				if (desc.contiguousLogEnd.get() >= v)
 					break;
-				if (self->waitRetries != -1 && tries > self->waitRetries)
+				if (waitRetries != -1 && tries > waitRetries)
 					break;
 				// Avoid spamming requests with a delay
-				wait(delay(5.0));
+				co_await delay(5.0);
 			}
 		}
-		if (self->stopBackup) {
+		if (stopBackup) {
 			try {
 				TraceEvent("IBackupDiscontinueBackup").log();
-				wait(self->backupAgent.discontinueBackup(cx, self->tag));
+				co_await backupAgent.discontinueBackup(cx, tag);
 			} catch (Error& e) {
 				TraceEvent("IBackupDiscontinueBackupException").error(e);
 				if (e.code() != error_code_backup_unneeded) {
@@ -172,36 +172,38 @@ struct IncrementalBackupWorkload : TestWorkload {
 				}
 			}
 		}
-		return true;
+		co_return true;
 	}
 
-	ACTOR static Future<Void> _start(Database cx, IncrementalBackupWorkload* self) {
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> _start(Database cx) {
+		Standalone<VectorRef<KeyRangeRef>> backupRanges;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		addDefaultBackupRanges(backupRanges);
 
-		if (self->submitOnly) {
-			if (self->encryptionKeyFileName.present()) {
-				wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		if (submitOnly) {
+			if (encryptionKeyFileName.present()) {
+				co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 			}
 
 			TraceEvent("IBackupSubmitAttempt").log();
 			try {
 				Optional<std::string> blobManifestUrl;
-				if (!self->blobManifestUrl.empty()) {
-					blobManifestUrl = self->blobManifestUrl.toString();
+				if (!this->blobManifestUrl.empty()) {
+					blobManifestUrl = this->blobManifestUrl.toString();
 				}
-				wait(self->backupAgent.submitBackup(cx,
-				                                    self->backupDir,
-				                                    {},
-				                                    0,
-				                                    1e8,
-				                                    self->tag.toString(),
-				                                    backupRanges,
-				                                    StopWhenDone::False,
-				                                    UsePartitionedLog::False,
-				                                    IncrementalBackupOnly::True,
-				                                    self->encryptionKeyFileName));
+				co_await backupAgent.submitBackup(cx,
+				                                  backupDir,
+				                                  {},
+				                                  0,
+				                                  1e8,
+				                                  tag.toString(),
+				                                  backupRanges,
+				                                  StopWhenDone::False,
+				                                  MutationLogType::DEFAULT,
+				                                  IncrementalBackupOnly::True,
+				                                  encryptionKeyFileName,
+				                                  encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+				                                  0);
 			} catch (Error& e) {
 				TraceEvent("IBackupSubmitError").error(e);
 				if (e.code() != error_code_backup_duplicate) {
@@ -210,42 +212,44 @@ struct IncrementalBackupWorkload : TestWorkload {
 			}
 			TraceEvent("IBackupSubmitSuccess").log();
 		}
-		if (self->restoreOnly) {
-			if (self->clearBackupAgentKeys) {
-				state Transaction clearTr(cx);
+		if (restoreOnly) {
+			if (clearBackupAgentKeys) {
+				Transaction clearTr(cx);
 				// Clear Relevant System Keys
-				loop {
+				while (true) {
+					Error err;
 					try {
 						clearTr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 						clearTr.setOption(FDBTransactionOptions::LOCK_AWARE);
 						clearTr.clear(fileBackupPrefixRange);
-						wait(clearTr.commit());
+						co_await clearTr.commit();
 						break;
 					} catch (Error& e) {
-						wait(clearTr.onError(e));
+						err = e;
 					}
+					co_await clearTr.onError(err);
 				}
 			}
-			state Reference<IBackupContainer> backupContainer;
-			state UID backupUID;
-			state Version beginVersion = invalidVersion;
-			wait(success(self->backupAgent.waitBackup(
-			    cx, self->tag.toString(), StopWhenDone::False, &backupContainer, &backupUID)));
+			Reference<IBackupContainer> backupContainer;
+			UID backupUID;
+			Version beginVersion = invalidVersion;
+			co_await backupAgent.waitBackup(cx, tag.toString(), StopWhenDone::False, &backupContainer, &backupUID);
 
-			state Optional<std::string> restoreEncryptionKeyFileName;
-			if (self->encryptionKeyFileName.present() && fileExists(self->encryptionKeyFileName.get())) {
-				restoreEncryptionKeyFileName = self->encryptionKeyFileName.get();
+			Optional<std::string> restoreEncryptionKeyFileName;
+			if (encryptionKeyFileName.present() && fileExists(encryptionKeyFileName.get())) {
+				restoreEncryptionKeyFileName = encryptionKeyFileName.get();
 			}
 
-			if (self->checkBeginVersion) {
+			if (checkBeginVersion) {
 				TraceEvent("IBackupReadSystemKeys").log();
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-				loop {
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+				while (true) {
+					Error err;
 					try {
 						tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 						tr->setOption(FDBTransactionOptions::LOCK_AWARE);
-						state Optional<Value> writeFlag = wait(tr->get(writeRecoveryKey));
-						state Optional<Value> versionValue = wait(tr->get(snapshotEndVersionKey));
+						Optional<Value> writeFlag = co_await tr->get(writeRecoveryKey);
+						Optional<Value> versionValue = co_await tr->get(snapshotEndVersionKey);
 						TraceEvent("IBackupCheckSpecialKeys")
 						    .detail("WriteRecoveryValue", writeFlag.present() ? writeFlag.get().toString() : "N/A")
 						    .detail("EndVersionValue", versionValue.present() ? versionValue.get().toString() : "N/A");
@@ -258,24 +262,24 @@ struct IncrementalBackupWorkload : TestWorkload {
 						TraceEvent("IBackupCheckBeginVersion").detail("Version", beginVersion);
 						break;
 					} catch (Error& e) {
-						TraceEvent("IBackupReadSystemKeysError").error(e);
-						if (e.code() == error_code_key_not_found) {
-							throw;
-						}
-						wait(tr->onError(e));
+						err = e;
+					}
+					TraceEvent("IBackupReadSystemKeysError").error(err);
+					if (err.code() == error_code_key_not_found) {
+						throw err;
 					}
+					co_await tr->onError(err);
 				}
 			}
 			TraceEvent("IBackupStartListContainersAttempt").log();
-			state std::vector<std::string> containers =
-			    wait(IBackupContainer::listContainers(self->backupDir.toString(), {}));
+			std::vector<std::string> containers = co_await IBackupContainer::listContainers(backupDir.toString(), {});
 			TraceEvent("IBackupStartListContainersSuccess")
 			    .detail("Size", containers.size())
 			    .detail("First", containers.front());
-			state Key backupURL = Key(containers.front());
+			Key backupURL = Key(containers.front());
 
-			state Standalone<VectorRef<KeyRangeRef>> restoreRange;
-			state Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
+			Standalone<VectorRef<KeyRangeRef>> restoreRange;
+			Standalone<VectorRef<KeyRangeRef>> systemRestoreRange;
 			for (auto r : backupRanges) {
 				if (!r.intersects(getSystemBackupRanges())) {
 					restoreRange.push_back_deep(restoreRange.arena(), r);
@@ -292,45 +296,44 @@ struct IncrementalBackupWorkload : TestWorkload {
 			}
 			if (!systemRestoreRange.empty()) {
 				TraceEvent("IBackupSystemRestoreAttempt").detail("BeginVersion", beginVersion);
-				wait(success(self->backupAgent.restore(cx,
-				                                       cx,
-				                                       "system_restore"_sr,
-				                                       backupURL,
-				                                       {},
-				                                       systemRestoreRange,
-				                                       WaitForComplete::True,
-				                                       invalidVersion,
-				                                       Verbose::True,
-				                                       Key(),
-				                                       Key(),
-				                                       LockDB::True,
-				                                       UnlockDB::True,
-				                                       OnlyApplyMutationLogs::True,
-				                                       InconsistentSnapshotOnly::False,
-				                                       beginVersion,
-				                                       restoreEncryptionKeyFileName)));
+				co_await backupAgent.restore(cx,
+				                             cx,
+				                             "system_restore"_sr,
+				                             backupURL,
+				                             {},
+				                             systemRestoreRange,
+				                             WaitForComplete::True,
+				                             invalidVersion,
+				                             Verbose::True,
+				                             Key(),
+				                             Key(),
+				                             LockDB::True,
+				                             UnlockDB::True,
+				                             OnlyApplyMutationLogs::True,
+				                             InconsistentSnapshotOnly::False,
+				                             beginVersion,
+				                             restoreEncryptionKeyFileName);
 			}
 			TraceEvent("IBackupRestoreAttempt").detail("BeginVersion", beginVersion);
-			wait(success(self->backupAgent.restore(cx,
-			                                       cx,
-			                                       Key(self->tag.toString()),
-			                                       backupURL,
-			                                       {},
-			                                       restoreRange,
-			                                       WaitForComplete::True,
-			                                       invalidVersion,
-			                                       Verbose::True,
-			                                       Key(),
-			                                       Key(),
-			                                       LockDB::True,
-			                                       UnlockDB::True,
-			                                       OnlyApplyMutationLogs::True,
-			                                       InconsistentSnapshotOnly::False,
-			                                       beginVersion,
-			                                       restoreEncryptionKeyFileName)));
+			co_await backupAgent.restore(cx,
+			                             cx,
+			                             Key(tag.toString()),
+			                             backupURL,
+			                             {},
+			                             restoreRange,
+			                             WaitForComplete::True,
+			                             invalidVersion,
+			                             Verbose::True,
+			                             Key(),
+			                             Key(),
+			                             LockDB::True,
+			                             UnlockDB::True,
+			                             OnlyApplyMutationLogs::True,
+			                             InconsistentSnapshotOnly::False,
+			                             beginVersion,
+			                             restoreEncryptionKeyFileName);
 			TraceEvent("IBackupRestoreSuccess").log();
 		}
-		return Void();
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
diff --git a/fdbserver/workloads/IndexScan.actor.cpp b/fdbserver/workloads/IndexScan.cpp
similarity index 62%
rename from fdbserver/workloads/IndexScan.actor.cpp
rename to fdbserver/workloads/IndexScan.cpp
index 855bfd0aee0..412cc1539a3 100644
--- a/fdbserver/workloads/IndexScan.actor.cpp
+++ b/fdbserver/workloads/IndexScan.cpp
@@ -1,5 +1,5 @@
 /*
- * IndexScan.actor.cpp
+ * IndexScan.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct IndexScanWorkload : KVWorkload {
 	constexpr static auto NAME = "IndexScan";
@@ -32,7 +31,7 @@ struct IndexScanWorkload : KVWorkload {
 	double totalTimeFetching, testDuration, transactionDuration;
 	bool singleProcess, readYourWrites;
 
-	IndexScanWorkload(WorkloadContext const& wcx)
+	explicit IndexScanWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), rowsRead(0), chunks(0), failedTransactions(0), scans(0) {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		bytesPerRead = getOption(options, "bytesPerRead"_sr, 80000);
@@ -50,7 +49,7 @@ struct IndexScanWorkload : KVWorkload {
 		if (singleProcess && clientId != 0) {
 			return Void();
 		}
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(const Database&) override { return true; }
@@ -68,62 +67,63 @@ struct IndexScanWorkload : KVWorkload {
 		m.emplace_back("Rows/chunk", chunks == 0 ? 0 : rowsRead / (double)chunks, Averaged::True);
 	}
 
-	ACTOR Future<Void> _start(Database cx, IndexScanWorkload* self) {
+	Future<Void> _start(Database cx) {
 		// Boilerplate: "warm" the location cache so that the location of all keys is known before test starts
-		state double startTime = now();
-		loop {
-			state Transaction tr(cx);
+		double startTime = now();
+		while (true) {
+			Transaction tr(cx);
+			Error err;
 			try {
-				wait(tr.warmRange(allKeys));
+				co_await tr.warmRange(allKeys);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		// Wait some small amount of time for things to "settle". Maybe this is historical?
-		wait(delay(std::max(0.1, 1.0 - (now() - startTime))));
+		co_await delay(std::max(0.1, 1.0 - (now() - startTime)));
 
-		wait(timeout(serialScans(cx, self), self->testDuration, Void()));
-		return Void();
+		co_await timeout(serialScans(cx), testDuration, Void());
 	}
 
-	ACTOR static Future<Void> serialScans(Database cx, IndexScanWorkload* self) {
-		state double start = now();
+	Future<Void> serialScans(Database cx) {
+		double start = now();
 		try {
-			loop {
-				wait(scanDatabase(cx, self));
+			while (true) {
+				co_await scanDatabase(cx);
 			}
 		} catch (...) {
-			self->totalTimeFetching = now() - start;
+			totalTimeFetching = now() - start;
 			throw;
 		}
 	}
 
-	ACTOR static Future<Void> scanDatabase(Database cx, IndexScanWorkload* self) {
-		state int startNode =
-		    deterministicRandom()->randomInt(0, self->nodeCount / 2); // start in the first half of the database
-		state KeySelector begin = firstGreaterOrEqual(self->keyForIndex(startNode));
-		state KeySelector end = firstGreaterThan(self->keyForIndex(self->nodeCount));
-		state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, self->bytesPerRead);
-
-		state int rowsRead;
-		state int chunks;
-		state double startTime;
-		loop {
-			state ReadYourWritesTransaction tr(cx);
-			if (!self->readYourWrites)
+	Future<Void> scanDatabase(Database cx) {
+		int startNode = deterministicRandom()->randomInt(0, nodeCount / 2); // start in the first half of the database
+		KeySelector begin = firstGreaterOrEqual(keyForIndex(startNode));
+		KeySelector end = firstGreaterThan(keyForIndex(nodeCount));
+		GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, bytesPerRead);
+
+		int rowsRead{ 0 };
+		int chunks{ 0 };
+		double startTime{ 0 };
+		while (true) {
+			ReadYourWritesTransaction tr(cx);
+			if (!readYourWrites)
 				tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
 			startTime = now();
 			rowsRead = 0;
 			chunks = 0;
 
+			Error err;
 			try {
-				loop {
-					RangeResult r = wait(tr.getRange(begin, end, limits));
+				while (true) {
+					RangeResult r = co_await tr.getRange(begin, end, limits);
 					chunks++;
 					rowsRead += r.size();
-					if (!r.size() || !r.more || (now() - startTime) > self->transactionDuration) {
+					if (r.empty() || !r.more || (now() - startTime) > transactionDuration) {
 						break;
 					}
 					begin = firstGreaterThan(r[r.size() - 1].key);
@@ -131,16 +131,16 @@ struct IndexScanWorkload : KVWorkload {
 
 				break;
 			} catch (Error& e) {
-				if (e.code() != error_code_actor_cancelled)
-					++self->failedTransactions;
-				wait(tr.onError(e));
+				err = e;
 			}
+			if (err.code() != error_code_actor_cancelled)
+				++failedTransactions;
+			co_await tr.onError(err);
 		}
 
-		self->rowsRead += rowsRead;
-		self->chunks += chunks;
-		self->scans++;
-		return Void();
+		this->rowsRead += rowsRead;
+		this->chunks += chunks;
+		this->scans++;
 	}
 };
 
diff --git a/fdbserver/workloads/Inventory.actor.cpp b/fdbserver/workloads/Inventory.cpp
similarity index 82%
rename from fdbserver/workloads/Inventory.actor.cpp
rename to fdbserver/workloads/Inventory.cpp
index 6beed58a44f..3de924fd4c1 100644
--- a/fdbserver/workloads/Inventory.actor.cpp
+++ b/fdbserver/workloads/Inventory.cpp
@@ -1,5 +1,5 @@
 /*
- * Inventory.actor.cpp
+ * Inventory.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,9 +19,8 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 // SOMEDAY: Make this actually run on multiple clients
 
@@ -37,7 +36,7 @@ struct InventoryTestWorkload : TestWorkload {
 	PerfIntCounter transactions, retries;
 	PerfDoubleCounter totalLatency;
 
-	InventoryTestWorkload(WorkloadContext const& wcx)
+	explicit InventoryTestWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), totalLatency("Latency") {
 		actorCount = getOption(options, "actorCount"_sr, 500);
 		nProducts = getOption(options, "nProducts"_sr, 100000);
@@ -100,10 +99,10 @@ struct InventoryTestWorkload : TestWorkload {
 		// return s;
 	}
 
-	ACTOR Future<bool> inventoryTestCheck(Database cx, InventoryTestWorkload* self) {
+	Future<bool> inventoryTestCheck(Database cx, InventoryTestWorkload* self) {
 		if (self->failures()) {
 			TraceEvent(SevError, "TestFailure").detail("Reason", "There were client failures.");
-			return false;
+			co_return false;
 		}
 		/*if (self->transactions.getValue() < .9 * self->transactionsPerSecond * self->testDuration) {
 		    TraceEvent(SevError, "TestFailure").detail("Reason", "Less than 90% desired transaction rate.");
@@ -111,11 +110,12 @@ struct InventoryTestWorkload : TestWorkload {
 		}*/
 		self->clients.clear();
 
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				RangeResult data = wait(tr.getRange(
-				    firstGreaterOrEqual(doubleToTestKey(0)), firstGreaterOrEqual(doubleToTestKey(1)), self->nProducts));
+				RangeResult data = co_await tr.getRange(
+				    firstGreaterOrEqual(doubleToTestKey(0)), firstGreaterOrEqual(doubleToTestKey(1)), self->nProducts);
 
 				std::map<Key, int> actualResults;
 				for (int i = 0; i < data.size(); i++)
@@ -141,34 +141,34 @@ struct InventoryTestWorkload : TestWorkload {
 						    .detail("MaxExpected", self->maxExpectedResults[i->first]);
 					}
 				if (error)
-					return false;
-				return true;
+					co_return false;
+				co_return true;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> inventoryTestWrite(Transaction* tr, Key key) {
-		Optional<Value> val = wait(tr->get(key));
+	Future<Void> inventoryTestWrite(Transaction* tr, Key key) {
+		Optional<Value> val = co_await tr->get(key);
 		int count = !val.present() ? 0 : atoi(val.get().toString().c_str());
 		ASSERT(count >= 0 && count < 1000000);
 		tr->set(key, format("%d", count + 1));
-		return Void();
 	}
 
-	ACTOR Future<Void> inventoryTestClient(Database cx,
-	                                       InventoryTestWorkload* self,
-	                                       double transactionDelay,
-	                                       double fractionWriteTransactions,
-	                                       int productsPerWrite) {
-		state double lastTime = now();
-		loop {
-			wait(poisson(&lastTime, transactionDelay));
-			state double st = now();
-			state Transaction tr(cx);
+	Future<Void> inventoryTestClient(Database cx,
+	                                 InventoryTestWorkload* self,
+	                                 double transactionDelay,
+	                                 double fractionWriteTransactions,
+	                                 int productsPerWrite) {
+		double lastTime = now();
+		while (true) {
+			co_await poisson(&lastTime, transactionDelay);
+			double st = now();
+			Transaction tr(cx);
 			if (deterministicRandom()->random01() < fractionWriteTransactions) {
-				state std::set<Key> products;
+				std::set<Key> products;
 				for (int i = 0; i < productsPerWrite; i++)
 					products.insert(self->chooseProduct());
 				for (auto p = products.begin(); p != products.end(); ++p)
@@ -177,20 +177,22 @@ struct InventoryTestWorkload : TestWorkload {
 					std::vector<Future<Void>> todo;
 					for (auto p = products.begin(); p != products.end(); ++p)
 						todo.push_back(self->inventoryTestWrite(&tr, *p));
+					Error err;
 					try {
 						try {
-							wait(waitForAll(todo));
+							co_await waitForAll(todo);
 						} catch (Error& e) {
 							if (e.code() == error_code_actor_cancelled)
 								for (auto p = products.begin(); p != products.end(); ++p)
 									self->maxExpectedResults[*p]--;
 							throw e;
 						}
-						wait(tr.commit());
+						co_await tr.commit();
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 					++self->retries;
 					for (auto p = products.begin(); p != products.end(); ++p)
 						self->maxExpectedResults[*p]++;
@@ -198,13 +200,15 @@ struct InventoryTestWorkload : TestWorkload {
 				for (auto p = products.begin(); p != products.end(); ++p)
 					self->minExpectedResults[*p]++;
 			} else {
-				loop {
+				while (true) {
+					Error err;
 					try {
-						Optional<Value> val = wait(tr.get(self->chooseProduct()));
+						Optional<Value> val = co_await tr.get(self->chooseProduct());
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 				}
 			}
 			self->totalLatency += now() - st;
diff --git a/fdbserver/workloads/KVStoreTest.actor.cpp b/fdbserver/workloads/KVStoreTest.cpp
similarity index 81%
rename from fdbserver/workloads/KVStoreTest.actor.cpp
rename to fdbserver/workloads/KVStoreTest.cpp
index 7bc73c71631..515bbf6a1e4 100644
--- a/fdbserver/workloads/KVStoreTest.actor.cpp
+++ b/fdbserver/workloads/KVStoreTest.cpp
@@ -1,5 +1,5 @@
 /*
- * KVStoreTest.actor.cpp
+ * KVStoreTest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,18 +22,18 @@
 #include <cinttypes>
 #include "fdbclient/FDBTypes.h"
 #include "fmt/format.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
 #include "flow/ActorCollection.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 extern IKeyValueStore* makeDummyKeyValueStore();
 
 template <class T>
 class TestHistogram {
 public:
-	TestHistogram(int minSamples = 100) : minSamples(minSamples) { reset(); }
+	explicit TestHistogram(int minSamples = 100) : minSamples(minSamples) { reset(); }
 
 	void reset() {
 		N = 0;
@@ -155,12 +155,12 @@ struct KVTest {
 	}
 };
 
-ACTOR Future<Void> testKVRead(KVTest* test, Key key, TestHistogram<float>* latency, PerfIntCounter* count) {
+Future<Void> testKVRead(KVTest* test, Key key, TestHistogram<float>* latency, PerfIntCounter* count) {
 	// state Version s1 = test->lastCommit;
-	state Version s2 = test->lastDurable;
+	Version s2 = test->lastDurable;
 
-	state double begin = timer();
-	Optional<Value> val = wait(test->store->readValue(key));
+	double begin = timer();
+	Optional<Value> val = co_await test->store->readValue(key);
 	latency->addSample(timer() - begin);
 	++*count;
 	Version v = val.present() ? BinaryReader::fromStringRef<Version>(val.get(), Unversioned()) : test->startVersion;
@@ -171,31 +171,29 @@ ACTOR Future<Void> testKVRead(KVTest* test, Key key, TestHistogram<float>* laten
 	ASSERT(s2 <= v || test->get(key, s2) == v); // Causal consistency
 	ASSERT(v <= test->lastCommit); // read committed
 	// ASSERT( v <= test->lastSet );  // read uncommitted
-	return Void();
 }
 
-ACTOR Future<Void> testKVReadSaturation(KVTest* test, TestHistogram<float>* latency, PerfIntCounter* count) {
+Future<Void> testKVReadSaturation(KVTest* test, TestHistogram<float>* latency, PerfIntCounter* count) {
 	while (true) {
-		state double begin = timer();
-		Optional<Value> val = wait(test->store->readValue(test->randomKey()));
+		double begin = timer();
+		Optional<Value> val = co_await test->store->readValue(test->randomKey());
 		latency->addSample(timer() - begin);
 		++*count;
-		wait(delay(0));
+		co_await delay(0);
 	}
 }
 
-ACTOR Future<Void> testKVCommit(KVTest* test, TestHistogram<float>* latency, PerfIntCounter* count) {
-	state Version v = test->lastSet;
+Future<Void> testKVCommit(KVTest* test, TestHistogram<float>* latency, PerfIntCounter* count) {
+	Version v = test->lastSet;
 	test->lastCommit = v;
-	state double begin = timer();
-	wait(test->store->commit());
+	double begin = timer();
+	co_await test->store->commit();
 	++*count;
 	latency->addSample(timer() - begin);
 	test->lastDurable = std::max(test->lastDurable, v);
-	return Void();
 }
 
-Future<Void> testKVStore(struct KVStoreTestWorkload* const&);
+Future<Void> testKVStore(struct KVStoreTestWorkload* workload);
 
 struct KVStoreTestWorkload : TestWorkload {
 	static constexpr auto NAME = "KVStoreTest";
@@ -210,7 +208,7 @@ struct KVStoreTestWorkload : TestWorkload {
 	double setupTook;
 	KeyValueStoreType storeType;
 
-	KVStoreTestWorkload(WorkloadContext const& wcx)
+	explicit KVStoreTestWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), reads("Reads"), sets("Sets"), commits("Commits"), setupTook(0) {
 		enabled = !clientId; // only do this on the "first" client
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -255,25 +253,25 @@ struct KVStoreTestWorkload : TestWorkload {
 
 WorkloadFactory<KVStoreTestWorkload> KVStoreTestWorkloadFactory;
 
-ACTOR Future<Void> testKVStoreMain(KVStoreTestWorkload* workload, KVTest* ptest) {
-	state KVTest& test = *ptest;
-	state ActorCollectionNoErrors ac;
-	state std::deque<Future<Void>> reads;
-	state BinaryWriter wr(Unversioned());
-	state int64_t commitsStarted = 0;
+Future<Void> testKVStoreMain(KVStoreTestWorkload* workload, KVTest* ptest) {
+	KVTest& test = *ptest;
+	ActorCollectionNoErrors ac;
+	std::deque<Future<Void>> reads;
+	BinaryWriter wr(Unversioned());
+	int64_t commitsStarted = 0;
 	// test.store = makeDummyKeyValueStore();
-	state int extraBytes = workload->valueBytes - sizeof(test.lastSet);
-	state int i;
+	int extraBytes = workload->valueBytes - sizeof(test.lastSet);
+	int i{ 0 };
 	ASSERT(extraBytes >= 0);
-	state char* extraValue = new char[extraBytes];
+	char* extraValue = new char[extraBytes];
 	memset(extraValue, '.', extraBytes);
 
 	if (workload->doCount) {
-		state int64_t count = 0;
-		state Key k;
-		state double cst = timer();
+		int64_t count = 0;
+		Key k;
+		double cst = timer();
 		while (true) {
-			RangeResult kv = wait(test.store->readRange(KeyRangeRef(k, "\xff\xff\xff\xff"_sr), 1000));
+			RangeResult kv = co_await test.store->readRange(KeyRangeRef(k, "\xff\xff\xff\xff"_sr), 1000);
 			count += kv.size();
 			if (kv.size() < 1000)
 				break;
@@ -289,23 +287,23 @@ ACTOR Future<Void> testKVStoreMain(KVStoreTestWorkload* workload, KVTest* ptest)
 		wr.serializeBytes(extraValue, extraBytes);
 
 		printf("Building %d nodes: ", workload->nodeCount);
-		state double setupBegin = timer();
-		state Future<Void> lastCommit = Void();
+		double setupBegin = timer();
+		Future<Void> lastCommit = Void();
 		for (i = 0; i < workload->nodeCount; i++) {
 			test.store->set(KeyValueRef(test.makeKey(i), wr.toValue()));
 			if (!((i + 1) % 10000) || i + 1 == workload->nodeCount) {
-				wait(lastCommit);
+				co_await lastCommit;
 				lastCommit = test.store->commit();
 				printf("ETA: %f seconds\n", (timer() - setupBegin) / i * (workload->nodeCount - i));
 			}
 		}
-		wait(lastCommit);
+		co_await lastCommit;
 		workload->setupTook = timer() - setupBegin;
 		TraceEvent("KVStoreSetup").detail("Count", workload->nodeCount).detail("Took", workload->setupTook);
 	}
 
-	state double t = now();
-	state double stopAt = t + workload->testDuration;
+	double t = now();
+	double stopAt = t + workload->testDuration;
 	if (workload->saturation) {
 		if (workload->commitFraction) {
 			while (now() < stopAt) {
@@ -318,19 +316,19 @@ ACTOR Future<Void> testKVStoreMain(KVStoreTestWorkload* workload, KVTest* ptest)
 					++workload->sets;
 				}
 				++commitsStarted;
-				wait(testKVCommit(&test, &workload->commitLatency, &workload->commits));
+				co_await testKVCommit(&test, &workload->commitLatency, &workload->commits);
 			}
 		} else {
 			std::vector<Future<Void>> actors;
 			actors.reserve(100);
 			for (int a = 0; a < 100; a++)
 				actors.push_back(testKVReadSaturation(&test, &workload->readLatency, &workload->reads));
-			wait(timeout(waitForAll(actors), workload->testDuration, Void()));
+			co_await timeout(waitForAll(actors), workload->testDuration, Void());
 		}
 	} else {
 		while (t < stopAt) {
 			double end = now();
-			loop {
+			while (true) {
 				t += 1.0 / workload->operationsPerSecond;
 				double op = deterministicRandom()->random01();
 				if (op < workload->commitFraction) {
@@ -354,32 +352,30 @@ ACTOR Future<Void> testKVStoreMain(KVStoreTestWorkload* workload, KVTest* ptest)
 				if (t >= end)
 					break;
 			}
-			wait(delayUntil(t));
+			co_await delayUntil(t);
 		}
 	}
 
 	if (workload->doClear) {
-		state int chunk = 1000000;
+		int chunk = 1000000;
 		t = timer();
 		for (i = 0; i < workload->nodeCount; i += chunk) {
 			test.store->clear(KeyRangeRef(test.makeKey(i), test.makeKey(i + chunk)));
-			wait(test.store->commit());
+			co_await test.store->commit();
 		}
 		TraceEvent("KVStoreClear").detail("Took", timer() - t);
 	}
-
-	return Void();
 }
 
-ACTOR Future<Void> testKVStore(KVStoreTestWorkload* workload) {
-	state KVTest test(workload->nodeCount, !workload->filename.size(), workload->keyBytes);
-	state Error err;
+Future<Void> testKVStore(KVStoreTestWorkload* workload) {
+	KVTest test(workload->nodeCount, workload->filename.empty(), workload->keyBytes);
+	Error err;
 
 	// wait( delay(1) );
 	TraceEvent("GO").log();
 
 	UID id = deterministicRandom()->randomUniqueID();
-	std::string fn = workload->filename.size() ? workload->filename : id.toString();
+	std::string fn = !workload->filename.empty() ? workload->filename : id.toString();
 	if (workload->storeType == KeyValueStoreType::SSD_BTREE_V2) {
 		test.store = keyValueStoreSQLite(fn, id, KeyValueStoreType::SSD_BTREE_V2);
 	} else if (workload->storeType == KeyValueStoreType::SSD_BTREE_V1) {
@@ -399,15 +395,16 @@ ACTOR Future<Void> testKVStore(KVStoreTestWorkload* workload) {
 		ASSERT(false);
 	}
 
-	wait(test.store->init());
+	co_await test.store->init();
 
-	state Future<Void> main = testKVStoreMain(workload, &test);
+	Future<Void> main = testKVStoreMain(workload, &test);
 	try {
-		choose {
-			when(wait(main)) {}
-			when(wait(test.store->getError())) {
-				ASSERT(false);
-			}
+		auto choice = co_await race(main, test.store->getError());
+		if (choice.index() == 0) {
+		} else if (choice.index() == 1) {
+			ASSERT(false);
+		} else {
+			UNREACHABLE();
 		}
 	} catch (Error& e) {
 		err = e;
@@ -416,8 +413,7 @@ ACTOR Future<Void> testKVStore(KVStoreTestWorkload* workload) {
 
 	Future<Void> c = test.store->onClosed();
 	test.close();
-	wait(c);
+	co_await c;
 	if (err.code() != invalid_error_code)
 		throw err;
-	return Void();
 }
diff --git a/fdbserver/workloads/KillRegion.actor.cpp b/fdbserver/workloads/KillRegion.cpp
similarity index 65%
rename from fdbserver/workloads/KillRegion.actor.cpp
rename to fdbserver/workloads/KillRegion.cpp
index 77e48cefb14..0d2a4b84ead 100644
--- a/fdbserver/workloads/KillRegion.actor.cpp
+++ b/fdbserver/workloads/KillRegion.cpp
@@ -19,72 +19,71 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/RecoveryState.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/RecoveryState.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last include.
+#include "fdbclient/ManagementAPI.h"
+#include "flow/CoroUtils.h"
 
 struct KillRegionWorkload : TestWorkload {
 	static constexpr auto NAME = "KillRegion";
 	bool enabled;
 	double testDuration;
 
-	KillRegionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit KillRegionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled =
 		    !clientId && g_network->isSimulated(); // only do this on the "first" client, and only when in simulation
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
-		g_simulator->usableRegions = 1;
+		fdbSimulationPolicyState().usableRegions = 1;
 	}
 
 	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
 
 	Future<Void> setup(Database const& cx) override {
 		if (enabled) {
-			return _setup(this, cx);
+			return _setup(cx);
 		}
 		return Void();
 	}
 	Future<Void> start(Database const& cx) override {
 		if (enabled) {
-			return killRegion(this, cx);
+			return killRegion(cx);
 		}
 		return Void();
 	}
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> _setup(KillRegionWorkload* self, Database cx) {
+	Future<Void> _setup(Database cx) {
 		TraceEvent("ForceRecovery_DisablePrimaryBegin").log();
-		wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->disablePrimary, true)));
+		co_await ManagementAPI::changeConfig(cx.getReference(), fdbSimulationPolicyState().disablePrimary, true);
 		TraceEvent("ForceRecovery_WaitForRemote").log();
-		wait(waitForPrimaryDC(cx, "1"_sr));
+		co_await waitForPrimaryDC(cx, "1"_sr);
 		TraceEvent("ForceRecovery_DisablePrimaryComplete").log();
-		return Void();
 	}
 
-	ACTOR static Future<Void> waitForStorageRecovered(KillRegionWorkload* self) {
-		while (self->dbInfo->get().recoveryState < RecoveryState::STORAGE_RECOVERED) {
-			wait(self->dbInfo->onChange());
+	Future<Void> waitForStorageRecovered() {
+		while (dbInfo->get().recoveryState < RecoveryState::STORAGE_RECOVERED) {
+			co_await dbInfo->onChange();
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> killRegion(KillRegionWorkload* self, Database cx) {
+	Future<Void> killRegion(Database cx) {
 		ASSERT(g_network->isSimulated());
 		if (deterministicRandom()->random01() < 0.5) {
 			TraceEvent("ForceRecovery_DisableRemoteBegin").log();
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->disableRemote, true)));
+			co_await ManagementAPI::changeConfig(cx.getReference(), fdbSimulationPolicyState().disableRemote, true);
 			TraceEvent("ForceRecovery_WaitForPrimary").log();
-			wait(waitForPrimaryDC(cx, "0"_sr));
+			co_await waitForPrimaryDC(cx, "0"_sr);
 			TraceEvent("ForceRecovery_DisableRemoteComplete").log();
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), g_simulator->originalRegions, true)));
+			co_await ManagementAPI::changeConfig(cx.getReference(), fdbSimulationPolicyState().originalRegions, true);
 		}
 		TraceEvent("ForceRecovery_Wait").log();
-		wait(delay(deterministicRandom()->random01() * self->testDuration));
+		co_await delay(deterministicRandom()->random01() * testDuration);
 
 		// FIXME: killDataCenter breaks simulation if forceKill=false, since some processes can survive and
 		// partially complete a recovery
@@ -103,11 +102,11 @@ struct KillRegionWorkload : TestWorkload {
 
 		TraceEvent("ForceRecovery_Begin").log();
 
-		wait(forceRecovery(cx->getConnectionRecord(), "1"_sr));
+		co_await forceRecovery(cx->getConnectionRecord(), "1"_sr);
 
 		TraceEvent("ForceRecovery_UsableRegions").log();
 
-		DatabaseConfiguration conf = wait(getDatabaseConfiguration(cx));
+		DatabaseConfiguration conf = co_await getDatabaseConfiguration(cx);
 
 		TraceEvent("ForceRecovery_GotConfig")
 		    .setMaxEventLength(11000)
@@ -115,23 +114,19 @@ struct KillRegionWorkload : TestWorkload {
 		    .detail("Conf", conf.toString());
 
 		if (conf.usableRegions > 1) {
-			loop {
+			while (true) {
 				// only needed if force recovery was unnecessary and we killed the secondary
-				wait(success(ManagementAPI::changeConfig(
-				    cx.getReference(), g_simulator->disablePrimary + " repopulate_anti_quorum=1", true)));
-				choose {
-					when(wait(waitForStorageRecovered(self))) {
-						break;
-					}
-					when(wait(delay(300.0))) {}
+				co_await ManagementAPI::changeConfig(
+				    cx.getReference(), fdbSimulationPolicyState().disablePrimary + " repopulate_anti_quorum=1", true);
+				auto result = co_await race(waitForStorageRecovered(), delay(300.0));
+				if (result.index() == 0) {
+					break;
 				}
 			}
-			wait(success(ManagementAPI::changeConfig(cx.getReference(), "usable_regions=1", true)));
+			co_await ManagementAPI::changeConfig(cx.getReference(), "usable_regions=1", true);
 		}
 
 		TraceEvent("ForceRecovery_Complete").log();
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/LocalRatekeeper.actor.cpp b/fdbserver/workloads/LocalRatekeeper.cpp
similarity index 69%
rename from fdbserver/workloads/LocalRatekeeper.actor.cpp
rename to fdbserver/workloads/LocalRatekeeper.cpp
index a63c3443b50..d581ad23d45 100644
--- a/fdbserver/workloads/LocalRatekeeper.actor.cpp
+++ b/fdbserver/workloads/LocalRatekeeper.cpp
@@ -1,5 +1,5 @@
 /*
- * LocalRatekeeper.actor.cpp
+ * LocalRatekeeper.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,27 +19,30 @@
  */
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include <fdbserver/Knobs.h>
-#include <flow/actorcompiler.h>
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/Knobs.h"
 
 namespace {
 
-ACTOR Future<StorageServerInterface> getRandomStorage(Database cx) {
-	state Transaction tr(cx);
-	loop {
+Future<StorageServerInterface> getRandomStorage(Database cx) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.reset();
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-			RangeResult range = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
-			if (range.size() > 0) {
+			RangeResult range = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+			if (!range.empty()) {
 				auto idx = deterministicRandom()->randomInt(0, range.size());
-				return decodeServerListValue(range[idx].value);
+				co_return decodeServerListValue(range[idx].value);
 			} else {
-				wait(delay(1.0));
+				co_await delay(1.0);
 			}
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
+		}
+		if (err.isValid()) {
+			co_await tr.onError(err);
 		}
 	}
 }
@@ -51,19 +54,19 @@ struct LocalRatekeeperWorkload : TestWorkload {
 	double blockWritesFor;
 	bool testFailed = false;
 
-	LocalRatekeeperWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit LocalRatekeeperWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		startAfter = getOption(options, "startAfter"_sr, startAfter);
 		blockWritesFor = getOption(
 		    options, "blockWritesFor"_sr, double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX) / double(1e6));
 	}
 
-	ACTOR static Future<Void> testStorage(LocalRatekeeperWorkload* self, Database cx, StorageServerInterface ssi) {
-		state Transaction tr(cx);
-		state std::vector<Future<GetValueReply>> requests;
+	Future<Void> testStorage(Database cx, StorageServerInterface ssi) {
+		Transaction tr(cx);
+		std::vector<Future<GetValueReply>> requests;
 		requests.reserve(100);
-		loop {
-			state StorageQueuingMetricsReply metrics =
-			    wait(brokenPromiseToNever(ssi.getQueuingMetrics.getReply(StorageQueuingMetricsRequest{})));
+		while (true) {
+			StorageQueuingMetricsReply metrics =
+			    co_await brokenPromiseToNever(ssi.getQueuingMetrics.getReply(StorageQueuingMetricsRequest{}));
 			auto durabilityLag = metrics.version - metrics.durableVersion;
 			double expectedRateLimit = 1.0;
 			if (durabilityLag >= SERVER_KNOBS->STORAGE_DURABILITY_LAG_HARD_MAX) {
@@ -75,22 +78,24 @@ struct LocalRatekeeperWorkload : TestWorkload {
 			}
 			if (expectedRateLimit < metrics.localRateLimit - 0.01 ||
 			    expectedRateLimit > metrics.localRateLimit + 0.01) {
-				self->testFailed = true;
+				testFailed = true;
 				TraceEvent(SevError, "StorageRateLimitTooFarOff")
 				    .detail("Storage", ssi.id())
 				    .detail("Expected", expectedRateLimit)
 				    .detail("Actual", metrics.localRateLimit);
 			}
 			tr.reset();
-			state Version readVersion = invalidVersion;
-			loop {
+			Version readVersion = invalidVersion;
+			while (true) {
+				Error err;
 				try {
-					Version v = wait(tr.getReadVersion());
+					Version v = co_await tr.getReadVersion();
 					readVersion = v;
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 			requests.clear();
 			// we send 100 requests to this storage node and count how many of those get rejected
@@ -101,12 +106,12 @@ struct LocalRatekeeperWorkload : TestWorkload {
 				req.key = "/lkfs"_sr;
 				requests.emplace_back(brokenPromiseToNever(ssi.getValue.getReply(req)));
 			}
-			wait(waitForAllReady(requests));
+			co_await waitForAllReady(requests);
 			int failedRequests = 0;
 			int errors = 0;
 			for (const auto& resp : requests) {
 				if (resp.isError()) {
-					self->testFailed = true;
+					testFailed = true;
 					++errors;
 					TraceEvent(SevError, "LoadBalancedResponseReturnedError").error(resp.getError());
 				} else if (resp.get().error.present() && resp.get().error.get().code() == error_code_future_version) {
@@ -117,22 +122,21 @@ struct LocalRatekeeperWorkload : TestWorkload {
 				}
 			}
 			TraceEvent("RejectedVersions").detail("NumRejected", failedRequests);
-			if (self->testFailed) {
-				return Void();
+			if (testFailed) {
+				co_return;
 			}
-			wait(delay(5.0));
+			co_await delay(5.0);
 		}
 	}
 
-	ACTOR static Future<Void> _start(LocalRatekeeperWorkload* self, Database cx) {
-		wait(delay(self->startAfter));
-		state StorageServerInterface ssi = wait(getRandomStorage(cx));
-		g_simulator->disableFor(format("%s/updateStorage", ssi.id().toString().c_str()), now() + self->blockWritesFor);
-		state Future<Void> done = delay(self->blockWritesFor);
+	Future<Void> _start(Database cx) {
+		co_await delay(startAfter);
+		StorageServerInterface ssi = co_await getRandomStorage(cx);
+		g_simulator->disableFor(format("%s/updateStorage", ssi.id().toString().c_str()), now() + blockWritesFor);
+		Future<Void> done = delay(blockWritesFor);
 		// not much will happen until the storage goes over the soft limit
-		wait(delay(double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX / 1e6)));
-		wait(testStorage(self, cx, ssi) || done);
-		return Void();
+		co_await delay(double(SERVER_KNOBS->STORAGE_DURABILITY_LAG_SOFT_MAX / 1e6));
+		co_await (testStorage(cx, ssi) || done);
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -140,7 +144,7 @@ struct LocalRatekeeperWorkload : TestWorkload {
 		if (clientId != 0 || !g_network->isSimulated()) {
 			return Void();
 		}
-		return _start(this, cx);
+		return _start(cx);
 	}
 	Future<bool> check(Database const& cx) override { return !testFailed; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
diff --git a/fdbserver/workloads/LockDatabase.actor.cpp b/fdbserver/workloads/LockDatabase.cpp
similarity index 57%
rename from fdbserver/workloads/LockDatabase.actor.cpp
rename to fdbserver/workloads/LockDatabase.cpp
index f0c9fbdf94c..3e9e99f60b7 100644
--- a/fdbserver/workloads/LockDatabase.actor.cpp
+++ b/fdbserver/workloads/LockDatabase.cpp
@@ -1,5 +1,5 @@
 /*
- * LockDatabase.actor.cpp
+ * LockDatabase.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,9 @@
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbclient/ManagementAPI.h"
 
 struct LockDatabaseWorkload : TestWorkload {
 	static constexpr auto NAME = "LockDatabase";
@@ -32,7 +31,7 @@ struct LockDatabaseWorkload : TestWorkload {
 	bool ok;
 	bool onlyCheckLocked;
 
-	LockDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ok(true) {
+	explicit LockDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), ok(true) {
 		lockAfter = getOption(options, "lockAfter"_sr, 0.0);
 		unlockAfter = getOption(options, "unlockAfter"_sr, 10.0);
 		onlyCheckLocked = getOption(options, "onlyCheckLocked"_sr, false);
@@ -43,7 +42,7 @@ struct LockDatabaseWorkload : TestWorkload {
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0)
-			return onlyCheckLocked ? timeout(checkLocked(cx, this), 60, Void()) : lockWorker(cx, this);
+			return onlyCheckLocked ? timeout(checkLocked(cx), 60, Void()) : lockWorker(cx);
 		return Void();
 	}
 
@@ -51,39 +50,42 @@ struct LockDatabaseWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<RangeResult> lockAndSave(Database cx, LockDatabaseWorkload* self, UID lockID) {
-		state Transaction tr(cx);
-		loop {
+	Future<RangeResult> lockAndSave(Database cx, UID lockID) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				wait(lockDatabase(&tr, lockID));
-				state RangeResult data = wait(tr.getRange(normalKeys, 50000));
+				co_await lockDatabase(&tr, lockID);
+				RangeResult data = co_await tr.getRange(normalKeys, 50000);
 				ASSERT(!data.more);
-				wait(tr.commit());
-				return data;
+				co_await tr.commit();
+				co_return data;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> unlockAndCheck(Database cx, LockDatabaseWorkload* self, UID lockID, RangeResult data) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> unlockAndCheck(Database cx, UID lockID, RangeResult data) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> val = wait(tr.get(databaseLockedKey));
+				Optional<Value> val = co_await tr.get(databaseLockedKey);
 				if (!val.present())
-					return Void();
+					co_return;
 
-				wait(unlockDatabase(&tr, lockID));
-				state RangeResult data2 = wait(tr.getRange(normalKeys, 50000));
+				co_await unlockDatabase(&tr, lockID);
+				RangeResult data2 = co_await tr.getRange(normalKeys, 50000);
 				if (data.size() != data2.size()) {
 					TraceEvent(SevError, "DataChangedWhileLocked")
 					    .detail("BeforeSize", data.size())
 					    .detail("AfterSize", data2.size());
-					self->ok = false;
+					ok = false;
 				} else if (data != data2) {
 					TraceEvent(SevError, "DataChangedWhileLocked").detail("Size", data.size());
 					for (int i = 0; i < data.size(); i++) {
@@ -94,41 +96,42 @@ struct LockDatabaseWorkload : TestWorkload {
 							    .detail("After", printable(data2[i]));
 						}
 					}
-					self->ok = false;
+					ok = false;
 				}
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> checkLocked(Database cx, LockDatabaseWorkload* self) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> checkLocked(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				Version v = wait(tr.getReadVersion());
+				Version v = co_await tr.getReadVersion();
 				TraceEvent(SevError, "GotVersionWhileLocked").detail("Version", v);
-				self->ok = false;
-				return Void();
+				ok = false;
+				co_return;
 			} catch (Error& e) {
-				CODE_PROBE(e.code() == error_code_database_locked, "Database confirmed locked");
-				wait(tr.onError(e));
+				err = e;
 			}
+			CODE_PROBE(err.code() == error_code_database_locked, "Database confirmed locked");
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> lockWorker(Database cx, LockDatabaseWorkload* self) {
-		state UID lockID = deterministicRandom()->randomUniqueID();
-		wait(delay(self->lockAfter));
-		state RangeResult data = wait(lockAndSave(cx, self, lockID));
-		state Future<Void> checker = checkLocked(cx, self);
-		wait(delay(self->unlockAfter - self->lockAfter));
+	Future<Void> lockWorker(Database cx) {
+		UID lockID = deterministicRandom()->randomUniqueID();
+		co_await delay(lockAfter);
+		RangeResult data = co_await lockAndSave(cx, lockID);
+		Future<Void> checker = checkLocked(cx);
+		co_await delay(unlockAfter - lockAfter);
 		checker.cancel();
-		wait(unlockAndCheck(cx, self, lockID, data));
-
-		return Void();
+		co_await unlockAndCheck(cx, lockID, data);
 	}
 };
 
diff --git a/fdbserver/workloads/LockDatabaseFrequently.actor.cpp b/fdbserver/workloads/LockDatabaseFrequently.cpp
similarity index 55%
rename from fdbserver/workloads/LockDatabaseFrequently.actor.cpp
rename to fdbserver/workloads/LockDatabaseFrequently.cpp
index e9ba9bc2f7b..5622d902ce9 100644
--- a/fdbserver/workloads/LockDatabaseFrequently.actor.cpp
+++ b/fdbserver/workloads/LockDatabaseFrequently.cpp
@@ -1,5 +1,5 @@
 /*
- * LockDatabaseFrequently.actor.cpp
+ * LockDatabaseFrequently.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,8 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
+#include "fdbclient/ManagementAPI.h"
 
 struct LockDatabaseFrequentlyWorkload : TestWorkload {
 	static constexpr auto NAME = "LockDatabaseFrequently";
@@ -31,14 +29,14 @@ struct LockDatabaseFrequentlyWorkload : TestWorkload {
 	double testDuration;
 	PerfIntCounter lockCount{ "LockCount" };
 
-	LockDatabaseFrequentlyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit LockDatabaseFrequentlyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		delayBetweenLocks = getOption(options, "delayBetweenLocks"_sr, 0.1);
 		testDuration = getOption(options, "testDuration"_sr, 60);
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return clientId == 0 ? worker(this, cx) : Void(); }
+	Future<Void> start(Database const& cx) override { return clientId == 0 ? worker(cx) : Void(); }
 
 	Future<bool> check(Database const& cx) override { return true; }
 
@@ -48,27 +46,23 @@ struct LockDatabaseFrequentlyWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> worker(LockDatabaseFrequentlyWorkload* self, Database cx) {
-		state Future<Void> end = delay(self->testDuration);
-		state double lastLock = g_network->now();
-		state double lastUnlock = g_network->now() + self->delayBetweenLocks / 2;
-		loop {
-			wait(lockAndUnlock(self, cx, &lastLock, &lastUnlock));
-			++self->lockCount;
+	Future<Void> worker(Database cx) {
+		Future<Void> end = delay(testDuration);
+		double lastLock = g_network->now();
+		double lastUnlock = g_network->now() + delayBetweenLocks / 2;
+		while (true) {
+			co_await lockAndUnlock(cx, &lastLock, &lastUnlock);
+			++lockCount;
 			if (end.isReady()) {
-				return Void();
+				co_return;
 			}
 		}
 	}
 
-	ACTOR static Future<Void> lockAndUnlock(LockDatabaseFrequentlyWorkload* self,
-	                                        Database cx,
-	                                        double* lastLock,
-	                                        double* lastUnlock) {
-		state UID uid = deterministicRandom()->randomUniqueID();
-		wait(lockDatabase(cx, uid) && poisson(lastLock, self->delayBetweenLocks));
-		wait(unlockDatabase(cx, uid) && poisson(lastUnlock, self->delayBetweenLocks));
-		return Void();
+	Future<Void> lockAndUnlock(Database cx, double* lastLock, double* lastUnlock) {
+		UID uid = deterministicRandom()->randomUniqueID();
+		co_await (lockDatabase(cx, uid) && poisson(lastLock, delayBetweenLocks));
+		co_await (unlockDatabase(cx, uid) && poisson(lastUnlock, delayBetweenLocks));
 	}
 };
 
diff --git a/fdbserver/workloads/LogMetrics.actor.cpp b/fdbserver/workloads/LogMetrics.cpp
similarity index 67%
rename from fdbserver/workloads/LogMetrics.actor.cpp
rename to fdbserver/workloads/LogMetrics.cpp
index 2d4e1e2ed11..e9fd4e88745 100644
--- a/fdbserver/workloads/LogMetrics.actor.cpp
+++ b/fdbserver/workloads/LogMetrics.cpp
@@ -1,5 +1,5 @@
 /*
- * LogMetrics.actor.cpp
+ * LogMetrics.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,14 +20,14 @@
 
 #include "flow/SystemMonitor.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/MasterInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct LogMetricsWorkload : TestWorkload {
@@ -36,7 +36,7 @@ struct LogMetricsWorkload : TestWorkload {
 	std::string dataFolder;
 	double logAt, logDuration, logsPerSecond;
 
-	LogMetricsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit LogMetricsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		logAt = getOption(options, "logAt"_sr, 0.0);
 		logDuration = getOption(options, "logDuration"_sr, 30.0);
 		logsPerSecond = getOption(options, "logsPerSecond"_sr, 20);
@@ -47,13 +47,13 @@ struct LogMetricsWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override {
 		if (clientId)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
-	ACTOR Future<Void> setSystemRate(LogMetricsWorkload* self, Database cx, uint32_t rate) {
+	Future<Void> setSystemRate(LogMetricsWorkload* self, Database cx, uint32_t rate) {
 		// set worker interval and ss interval
-		state BinaryWriter br(Unversioned());
-		std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
+		BinaryWriter br(Unversioned());
+		std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
 		// std::vector<Future<Void>> replies;
 		TraceEvent("RateChangeTrigger").log();
 		SetMetricsLogRateRequest req(rate);
@@ -63,32 +63,30 @@ struct LogMetricsWorkload : TestWorkload {
 		// wait( waitForAll( replies ) );
 
 		br << rate;
-		loop {
-			state Transaction tr(cx);
+		while (true) {
+			Transaction tr(cx);
+			Error err;
 			try {
-				wait(success(tr.getReadVersion()));
+				co_await tr.getReadVersion();
 				tr.set(fastLoggingEnabled, br.toValue());
 				tr.makeSelfConflicting();
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, LogMetricsWorkload* self) {
-		wait(delay(self->logAt));
+	Future<Void> _start(Database cx) {
+		co_await delay(logAt);
 
-		wait(self->setSystemRate(self, cx, self->logsPerSecond));
-		wait(timeout(recurring(&systemMonitor, 1.0 / self->logsPerSecond), self->logDuration, Void()));
+		co_await setSystemRate(this, cx, logsPerSecond);
+		co_await timeout(recurring(&systemMonitor, 1.0 / logsPerSecond), logDuration, Void());
 
 		// We're done, set everything back
-		wait(self->setSystemRate(self, cx, 1.0));
-
-		return Void();
+		co_await setSystemRate(this, cx, 1.0);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/LowLatency.actor.cpp b/fdbserver/workloads/LowLatency.cpp
similarity index 68%
rename from fdbserver/workloads/LowLatency.actor.cpp
rename to fdbserver/workloads/LowLatency.cpp
index f72514fe886..45f8095201c 100644
--- a/fdbserver/workloads/LowLatency.actor.cpp
+++ b/fdbserver/workloads/LowLatency.cpp
@@ -1,5 +1,5 @@
 /*
- * LowLatency.actor.cpp
+ * LowLatency.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
 
 struct LowLatencyWorkload : TestWorkload {
 	static constexpr auto NAME = "LowLatency";
@@ -38,7 +36,7 @@ struct LowLatencyWorkload : TestWorkload {
 	Key testKey;
 	bool ok;
 
-	LowLatencyWorkload(WorkloadContext const& wcx)
+	explicit LowLatencyWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), operations("Operations"), retries("Retries"), ok(true) {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		maxGRVLatency = getOption(options, "maxGRVLatency"_sr, 20.0);
@@ -52,63 +50,63 @@ struct LowLatencyWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override {
 		if (g_network->isSimulated()) {
-			IKnobCollection::getMutableGlobalKnobCollection().setKnob("min_delay_cc_worst_fit_candidacy_seconds",
-			                                                          KnobValueRef::create(double{ 5.0 }));
-			IKnobCollection::getMutableGlobalKnobCollection().setKnob("max_delay_cc_worst_fit_candidacy_seconds",
-			                                                          KnobValueRef::create(double{ 10.0 }));
+			setServerKnob("min_delay_cc_worst_fit_candidacy_seconds", KnobValueRef::create(double{ 5.0 }));
+			setServerKnob("max_delay_cc_worst_fit_candidacy_seconds", KnobValueRef::create(double{ 10.0 }));
 		}
 		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0)
-			return _start(cx, this);
+			return _start(cx);
 		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, LowLatencyWorkload* self) {
-		state double testStart = now();
+	Future<Void> _start(Database cx) {
+		double testStart = now();
 		try {
-			loop {
-				wait(delay(self->checkDelay));
-				state Transaction tr(cx);
-				state double operationStart = now();
-				state bool doCommit = self->testWrites && deterministicRandom()->coinflip();
-				state double maxLatency = doCommit ? self->maxCommitLatency : self->maxGRVLatency;
-				++self->operations;
-				loop {
+			while (true) {
+				co_await delay(checkDelay);
+				Transaction tr(cx);
+				double operationStart = now();
+				bool doCommit = testWrites && deterministicRandom()->coinflip();
+				double maxLatency = doCommit ? maxCommitLatency : maxGRVLatency;
+				++operations;
+				while (true) {
+					Error err;
 					try {
-						TraceEvent("LowLatencyTransactionStart").detail("Retries", self->retries.getValue());
+						TraceEvent("LowLatencyTransactionStart").detail("Retries", retries.getValue());
 						tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 						tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 						if (doCommit) {
 							TraceEvent("LowLatencyTransactionCommitStart");
-							tr.set(self->testKey, ""_sr);
-							wait(tr.commit());
+							tr.set(testKey, ""_sr);
+							co_await tr.commit();
 							TraceEvent("LowLatencyTransactionCommitFinish");
 						} else {
 							TraceEvent("LowLatencyTransactionGRVStart");
-							wait(success(tr.getReadVersion()));
+							co_await tr.getReadVersion();
 							TraceEvent("LowLatencyTransactionGRVFinish");
 						}
 						break;
 					} catch (Error& e) {
-						TraceEvent("LowLatencyTransactionFailed").errorUnsuppressed(e);
-						wait(tr.onError(e));
-						++self->retries;
+						err = e;
 					}
+					TraceEvent("LowLatencyTransactionFailed").errorUnsuppressed(err);
+					co_await tr.onError(err);
+					++retries;
 				}
 				if (now() - operationStart > maxLatency) {
 					TraceEvent(SevError, "LatencyTooLarge")
 					    .detail("MaxLatency", maxLatency)
 					    .detail("ObservedLatency", now() - operationStart)
 					    .detail("IsCommit", doCommit);
-					self->ok = false;
+					ok = false;
 				}
-				if (now() - testStart > self->testDuration)
+				if (now() - testStart > testDuration)
 					break;
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			TraceEvent(SevError, "LowLatencyError").errorUnsuppressed(e);
 			throw;
diff --git a/fdbserver/workloads/MachineAttrition.actor.cpp b/fdbserver/workloads/MachineAttrition.cpp
similarity index 74%
rename from fdbserver/workloads/MachineAttrition.actor.cpp
rename to fdbserver/workloads/MachineAttrition.cpp
index 130546a6efc..c38a3da98f4 100644
--- a/fdbserver/workloads/MachineAttrition.actor.cpp
+++ b/fdbserver/workloads/MachineAttrition.cpp
@@ -1,5 +1,5 @@
 /*
- * MachineAttrition.actor.cpp
+ * MachineAttrition.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,15 +21,15 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/CoordinationInterface.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "flow/FaultInjection.h"
 #include "flow/DeterministicRandom.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 static std::set<int> const& normalAttritionErrors() {
 	static std::set<int> s;
@@ -40,25 +40,27 @@ static std::set<int> const& normalAttritionErrors() {
 	return s;
 }
 
-ACTOR Future<bool> ignoreSSFailuresForDuration(Database cx, double duration) {
+Future<bool> ignoreSSFailuresForDuration(Database cx, double duration) {
 	// duration doesn't matter since this won't timeout
 	TraceEvent("IgnoreSSFailureStart").log();
-	wait(success(setHealthyZone(cx, ignoreSSFailuresZoneString, 0)));
+	co_await setHealthyZone(cx, ignoreSSFailuresZoneString, 0);
 	TraceEvent("IgnoreSSFailureWait").log();
-	wait(delay(duration));
+	co_await delay(duration);
 	TraceEvent("IgnoreSSFailureClear").log();
-	state Transaction tr(cx);
-	loop {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.clear(healthyZoneKey);
-			wait(tr.commit());
+			co_await tr.commit();
 			TraceEvent("IgnoreSSFailureComplete").log();
-			return true;
+			co_return true;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -92,7 +94,7 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 		iterate = true;
 	}
 
-	MachineAttritionWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+	explicit MachineAttritionWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		// only do this on the "first" client, and only when in simulation and only when fault injection is enabled
 		enabled = !clientId && g_network->isSimulated() && faultInjectionActivated;
 		machinesToKill = getOption(options, "machinesToKill"_sr, machinesToKill);
@@ -109,10 +111,10 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 		killProcess = getOption(options, "killProcess"_sr, killProcess);
 		killZone = getOption(options, "killZone"_sr, killZone);
 		killSelf = getOption(options, "killSelf"_sr, killSelf);
-		killAll =
-		    getOption(options,
-		              "killAll"_sr,
-		              g_network->isSimulated() && !g_simulator->extraDatabases.empty() && BUGGIFY_WITH_PROB(0.01));
+		killAll = getOption(options,
+		                    "killAll"_sr,
+		                    g_network->isSimulated() && !fdbSimulationPolicyState().extraDatabases.empty() &&
+		                        BUGGIFY_WITH_PROB(0.01));
 		targetIds = getOption(options, "targetIds"_sr, std::vector<std::string>());
 		replacement = getOption(options, "replacement"_sr, reboot && deterministicRandom()->random01() < 0.5);
 		waitForVersion = getOption(options, "waitForVersion"_sr, waitForVersion);
@@ -122,7 +124,7 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 	bool shouldInject(DeterministicRandom& random,
 	                  const WorkloadRequest& work,
 	                  const unsigned alreadyAdded) const override {
-		if (g_network->isSimulated() && !g_simulator->extraDatabases.empty()) {
+		if (g_network->isSimulated() && !fdbSimulationPolicyState().extraDatabases.empty()) {
 			// Remove this as soon as we track extra databases properly
 			return false;
 		}
@@ -143,9 +145,9 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 				dataHalls.emplace(process->locality.dataHallId().castTo<StringRef>());
 				zones.emplace(process->locality.zoneId().castTo<StringRef>());
 			}
-			killDc = dataCenters.size() > 0 && random.random01() > (dataHalls.size() < 0 ? 0.1 : 0.25);
-			killDatahall = dataHalls.size() > 0 && killDc && random.random01() < 0.5;
-			killZone = zones.size() > 0 && random.random01() < 0.2;
+			killDc = !dataCenters.empty() && random.random01() > (dataHalls.size() < 0 ? 0.1 : 0.25);
+			killDatahall = !dataHalls.empty() && killDc && random.random01() < 0.5;
+			killZone = !zones.empty() && random.random01() < 0.2;
 		}
 		TraceEvent("AddingFailureInjection")
 		    .detail("Reboot", reboot)
@@ -189,14 +191,14 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 
 			return timeout(
 			    reportErrorsExcept(
-			        machineKillWorker(this, meanDelay, cx), "machineKillWorkerError", UID(), &normalAttritionErrors()),
+			        machineKillWorker(meanDelay, cx), "machineKillWorkerError", UID(), &normalAttritionErrors()),
 			    testDuration,
 			    Void());
 		}
 		if (!clientId && !g_network->isSimulated()) {
 			return timeout(
 			    reportErrorsExcept(
-			        noSimMachineKillWorker(this, cx), "noSimMachineKillWorkerError", UID(), &normalAttritionErrors()),
+			        noSimMachineKillWorker(cx), "noSimMachineKillWorkerError", UID(), &normalAttritionErrors()),
 			    testDuration,
 			    Void());
 		}
@@ -226,19 +228,19 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> noSimMachineKillWorker(MachineAttritionWorkload* self, Database cx) {
+	Future<Void> noSimMachineKillWorker(Database cx) {
 		ASSERT(!g_network->isSimulated());
-		state int killedWorkers = 0;
-		state std::vector<WorkerDetails> allWorkers =
-		    wait(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest()));
+		int killedWorkers = 0;
+		std::vector<WorkerDetails> allWorkers =
+		    co_await dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest());
 		// Can reuse reboot request to send to each interface since no reply promise needed
-		state RebootRequest rbReq;
-		if (self->reboot) {
-			rbReq.waitForDuration = self->suspendDuration;
+		RebootRequest rbReq;
+		if (reboot) {
+			rbReq.waitForDuration = suspendDuration;
 		} else {
 			rbReq.waitForDuration = std::numeric_limits<uint32_t>::max();
 		}
-		state std::vector<WorkerDetails> workers;
+		std::vector<WorkerDetails> workers;
 		// Pre-processing step: remove all testers from list of workers
 		for (const auto& worker : allWorkers) {
 			if (noSimIsViableKill(worker)) {
@@ -246,99 +248,100 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 			}
 		}
 		deterministicRandom()->randomShuffle(workers);
-		wait(delay(self->liveDuration));
+		co_await delay(liveDuration);
 		// if a specific kill is requested, it must be accompanied by a set of target IDs otherwise no kills will
 		// occur
-		if (self->killDc) {
-			TraceEvent("Assassination").detail("TargetDataCenterIds", describe(self->targetIds));
+		if (killDc) {
+			TraceEvent("Assassination").detail("TargetDataCenterIds", describe(targetIds));
 			sendRebootRequests(workers,
-			                   self->targetIds,
+			                   targetIds,
 			                   rbReq,
 			                   // idAccess lambda
 			                   [](WorkerDetails worker) { return worker.interf.locality.dcId(); });
-		} else if (self->killMachine) {
-			TraceEvent("Assassination").detail("TargetMachineIds", describe(self->targetIds));
+		} else if (killMachine) {
+			TraceEvent("Assassination").detail("TargetMachineIds", describe(targetIds));
 			sendRebootRequests(workers,
-			                   self->targetIds,
+			                   targetIds,
 			                   rbReq,
 			                   // idAccess lambda
 			                   [](WorkerDetails worker) { return worker.interf.locality.machineId(); });
-		} else if (self->killDatahall) {
-			TraceEvent("Assassination").detail("TargetDatahallIds", describe(self->targetIds));
+		} else if (killDatahall) {
+			TraceEvent("Assassination").detail("TargetDatahallIds", describe(targetIds));
 			sendRebootRequests(workers,
-			                   self->targetIds,
+			                   targetIds,
 			                   rbReq,
 			                   // idAccess lambda
 			                   [](WorkerDetails worker) { return worker.interf.locality.dataHallId(); });
-		} else if (self->killProcess) {
-			TraceEvent("Assassination").detail("TargetProcessIds", describe(self->targetIds));
+		} else if (killProcess) {
+			TraceEvent("Assassination").detail("TargetProcessIds", describe(targetIds));
 			sendRebootRequests(workers,
-			                   self->targetIds,
+			                   targetIds,
 			                   rbReq,
 			                   // idAccess lambda
 			                   [](WorkerDetails worker) { return worker.interf.locality.processId(); });
-		} else if (self->killZone) {
-			TraceEvent("Assassination").detail("TargetZoneIds", describe(self->targetIds));
+		} else if (killZone) {
+			TraceEvent("Assassination").detail("TargetZoneIds", describe(targetIds));
 			sendRebootRequests(workers,
-			                   self->targetIds,
+			                   targetIds,
 			                   rbReq,
 			                   // idAccess lambda
 			                   [](WorkerDetails worker) { return worker.interf.locality.zoneId(); });
 		} else {
-			while (killedWorkers < self->workersToKill && workers.size() > self->workersToLeave) {
+			while (killedWorkers < workersToKill && workers.size() > workersToLeave) {
 				TraceEvent("WorkerKillBegin")
 				    .detail("KilledWorkers", killedWorkers)
-				    .detail("WorkersToKill", self->workersToKill)
-				    .detail("WorkersToLeave", self->workersToLeave)
+				    .detail("WorkersToKill", workersToKill)
+				    .detail("WorkersToLeave", workersToLeave)
 				    .detail("Workers", workers.size());
-				if (self->waitForVersion) {
-					state Transaction tr(cx);
-					loop {
+				if (waitForVersion) {
+					Transaction tr(cx);
+					while (true) {
+						Error err;
 						try {
 							tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 							tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-							wait(success(tr.getReadVersion()));
+							co_await tr.getReadVersion();
 							break;
 						} catch (Error& e) {
-							wait(tr.onError(e));
+							err = e;
 						}
+						co_await tr.onError(err);
 					}
 				}
 				// Pick a worker to kill
-				state WorkerDetails targetWorker;
+				WorkerDetails targetWorker;
 				targetWorker = workers.back();
 				TraceEvent("Assassination")
 				    .detail("TargetWorker", targetWorker.interf.locality.toString())
 				    .detail("ZoneId", targetWorker.interf.locality.zoneId())
 				    .detail("KilledWorkers", killedWorkers)
-				    .detail("WorkersToKill", self->workersToKill)
-				    .detail("WorkersToLeave", self->workersToLeave)
+				    .detail("WorkersToKill", workersToKill)
+				    .detail("WorkersToLeave", workersToLeave)
 				    .detail("Workers", workers.size());
 				targetWorker.interf.clientInterface.reboot.send(rbReq);
 				killedWorkers++;
 				workers.pop_back();
 			}
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> machineKillWorker(MachineAttritionWorkload* self, double meanDelay, Database cx) {
+	Future<Void> machineKillWorker(double meanDelay, Database cx) {
 		ASSERT(g_network->isSimulated());
-		state double delayBeforeKill;
-		state double suspendDuration = self->suspendDuration;
-		state double startTime = now();
+		double delayBeforeKill{ 0 };
+		double suspendDuration = this->suspendDuration;
+		double startTime = now();
 
-		loop {
-			if (self->killDc) {
+		while (true) {
+			if (killDc) {
 				delayBeforeKill = deterministicRandom()->random01() * meanDelay;
-				wait(delay(delayBeforeKill));
+				co_await delay(delayBeforeKill);
 
 				// decide on a machine to kill
-				ASSERT(self->machines.size());
-				Optional<Standalone<StringRef>> target = self->machines.back().dcId();
+				ASSERT(!machines.empty());
+				Optional<Standalone<StringRef>> target = machines.back().dcId();
 
 				ISimulator::KillType kt = ISimulator::KillType::Reboot;
-				if (!self->reboot) {
+				if (!reboot) {
 					int killType = deterministicRandom()->randomInt(0, 3); // FIXME: enable disk stalls
 					if (killType == 0)
 						kt = ISimulator::KillType::KillInstantly;
@@ -351,83 +354,85 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 				}
 				TraceEvent("Assassination")
 				    .detail("TargetDatacenter", target)
-				    .detail("Reboot", self->reboot)
+				    .detail("Reboot", reboot)
 				    .detail("KillType", kt);
 
 				g_simulator->killDataCenter(target, kt);
-			} else if (self->killDatahall) {
+			} else if (killDatahall) {
 				delayBeforeKill = deterministicRandom()->random01() * meanDelay;
-				wait(delay(delayBeforeKill));
+				co_await delay(delayBeforeKill);
 
 				// It only makes sense to kill a single data hall.
-				ASSERT(self->targetIds.size() == 1);
-				auto target = self->targetIds.front();
+				ASSERT(targetIds.size() == 1);
+				auto target = targetIds.front();
 
 				auto kt = ISimulator::KillType::KillInstantly;
 				TraceEvent("Assassination").detail("TargetDataHall", target).detail("KillType", kt);
 
 				g_simulator->killDataHall(target, kt);
-			} else if (self->killAll) {
-				state ISimulator::KillType kt = ISimulator::KillType::RebootProcessAndSwitch;
+			} else if (killAll) {
+				ISimulator::KillType kt = ISimulator::KillType::RebootProcessAndSwitch;
 				TraceEvent("Assassination").detail("KillType", kt);
 				g_simulator->killAll(kt, true);
 				g_simulator->toggleGlobalSwitchCluster();
-				wait(delay(self->testDuration / 2));
+				co_await delay(testDuration / 2);
 				g_simulator->killAll(kt, true);
 				g_simulator->toggleGlobalSwitchCluster();
 			} else {
-				state int killedMachines = 0;
-				while (killedMachines < self->machinesToKill && self->machines.size() > self->machinesToLeave) {
+				int killedMachines = 0;
+				while (killedMachines < machinesToKill && machines.size() > machinesToLeave) {
 					TraceEvent("WorkerKillBegin")
 					    .detail("KilledMachines", killedMachines)
-					    .detail("MachinesToKill", self->machinesToKill)
-					    .detail("MachinesToLeave", self->machinesToLeave)
-					    .detail("Machines", self->machines.size());
+					    .detail("MachinesToKill", machinesToKill)
+					    .detail("MachinesToLeave", machinesToLeave)
+					    .detail("Machines", machines.size());
 					CODE_PROBE(true, "Killing a machine");
 
 					delayBeforeKill = deterministicRandom()->random01() * meanDelay;
-					wait(delay(delayBeforeKill));
+					co_await delay(delayBeforeKill);
 					TraceEvent("WorkerKillAfterDelay").log();
 
-					if (self->waitForVersion) {
-						state Transaction tr(cx);
-						loop {
+					if (waitForVersion) {
+						Transaction tr(cx);
+						while (true) {
+							Error err;
 							try {
 								tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 								tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-								wait(success(tr.getReadVersion()));
+								co_await tr.getReadVersion();
 								break;
 							} catch (Error& e) {
-								wait(tr.onError(e));
+								err = e;
 							}
+							co_await tr.onError(err);
 						}
 					}
 
 					// decide on a machine to kill
-					state LocalityData targetMachine = self->machines.back();
+					LocalityData targetMachine = machines.back();
 					if (BUGGIFY_WITH_PROB(0.01)) {
 						CODE_PROBE(true, "Marked a zone for maintenance before killing it");
-						wait(success(
-						    setHealthyZone(cx, targetMachine.zoneId().get(), deterministicRandom()->random01() * 20)));
-					} else if (!g_simulator->willRestart && BUGGIFY_WITH_PROB(0.005)) {
+						co_await setHealthyZone(
+						    cx, targetMachine.zoneId().get(), deterministicRandom()->random01() * 20);
+					} else if (!fdbSimulationPolicyState().willRestart && BUGGIFY_WITH_PROB(0.005)) {
 						// don't do this in restarting test, since test could exit before it is unset, and restarted
 						// test would never unset it
 						CODE_PROBE(true, "Disable DD for all storage server failures");
-						self->ignoreSSFailures =
+						ignoreSSFailures =
 						    uncancellable(ignoreSSFailuresForDuration(cx, deterministicRandom()->random01() * 5));
 					}
 
 					TraceEvent("Assassination")
 					    .detail("TargetMachine", targetMachine.toString())
 					    .detail("ZoneId", targetMachine.zoneId())
-					    .detail("Reboot", self->reboot)
+					    .detail("Reboot", reboot)
 					    .detail("KilledMachines", killedMachines)
-					    .detail("MachinesToKill", self->machinesToKill)
-					    .detail("MachinesToLeave", self->machinesToLeave)
-					    .detail("Machines", self->machines.size())
-					    .detail("Replace", self->replacement);
+					    .detail("MachinesToKill", machinesToKill)
+					    .detail("MachinesToLeave", machinesToLeave)
+					    .detail("Machines", machines.size())
+					    .detail("Replace", replacement);
 
-					if (self->reboot) {
+					if (reboot) {
 						if (deterministicRandom()->random01() > 0.5) {
 							g_simulator->rebootProcess(targetMachine.zoneId(), deterministicRandom()->random01() > 0.5);
 						} else {
@@ -436,14 +441,14 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 					} else {
 						auto randomDouble = deterministicRandom()->random01();
 						TraceEvent("WorkerKill")
-						    .detail("MachineCount", self->machines.size())
+						    .detail("MachineCount", machines.size())
 						    .detail("RandomValue", randomDouble);
 						if (randomDouble < 0.33) {
 							TraceEvent("RebootAndDelete").detail("TargetMachine", targetMachine.toString());
 							g_simulator->killZone(targetMachine.zoneId(), ISimulator::KillType::RebootAndDelete);
 						} else {
 							auto kt = ISimulator::KillType::KillInstantly;
-							if (self->allowFaultInjection) {
+							if (allowFaultInjection) {
 								if (randomDouble < 0.50) {
 									kt = ISimulator::KillType::InjectFaults;
 								}
@@ -461,30 +466,29 @@ struct MachineAttritionWorkload : FailureInjectionWorkload {
 					}
 
 					killedMachines++;
-					if (self->replacement) {
+					if (replacement) {
 						// Replace by reshuffling, since we always pick from the back.
-						deterministicRandom()->randomShuffle(self->machines);
+						deterministicRandom()->randomShuffle(machines);
 					} else {
-						self->machines.pop_back();
+						machines.pop_back();
 					}
 
-					wait(delay(meanDelay - delayBeforeKill) && success(self->ignoreSSFailures));
+					co_await (delay(meanDelay - delayBeforeKill) && success(ignoreSSFailures));
 
 					delayBeforeKill = deterministicRandom()->random01() * meanDelay;
 					TraceEvent("WorkerKillAfterMeanDelay").detail("DelayBeforeKill", delayBeforeKill);
 				}
 			}
-			if (!self->iterate || now() - startTime > self->maxRunDuration) {
+			if (!iterate || now() - startTime > maxRunDuration) {
 				break;
 			} else {
-				wait(delay(suspendDuration));
-				suspendDuration *= self->backoff;
+				co_await delay(suspendDuration);
+				suspendDuration *= backoff;
 			}
 		}
 
-		if (self->killSelf)
+		if (killSelf)
 			throw please_reboot();
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.cpp
similarity index 63%
rename from fdbserver/workloads/Mako.actor.cpp
rename to fdbserver/workloads/Mako.cpp
index ff7df7b82a2..29413483433 100644
--- a/fdbserver/workloads/Mako.actor.cpp
+++ b/fdbserver/workloads/Mako.cpp
@@ -1,5 +1,5 @@
 /*
- * Mako.actor.cpp
+ * Mako.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,13 +19,12 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/zipf.h"
 #include "crc32/crc32c.h"
-#include "flow/actorcompiler.h"
 
 enum {
 	OP_GETREADVERSION,
@@ -73,7 +72,7 @@ struct MakoWorkload : TestWorkload {
 		                                              "SGETRANGE", "UPDATE",   "INSERT",     "INSERTRANGE",
 		                                              "CLEAR",     "SETCLEAR", "CLEARRANGE", "SETCLEARRANGE",
 		                                              "COMMIT" };
-	MakoWorkload(WorkloadContext const& wcx)
+	explicit MakoWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), loadTime(0.0), xacts("Transactions"), retries("Retries"), conflicts("Conflicts"),
 	    commits("Commits"), totalOps("Operations") {
 		// init parameters from test file
@@ -170,13 +169,13 @@ struct MakoWorkload : TestWorkload {
 	Future<Void> setup(Database const& cx) override {
 		if (doChecksumVerificationOnly)
 			return Void();
-		return _setup(cx, this);
+		return _setup(cx);
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (doChecksumVerificationOnly)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override {
@@ -184,7 +183,7 @@ struct MakoWorkload : TestWorkload {
 			return true;
 		}
 		// verify checksum consistency
-		return dochecksumVerification(cx, this);
+		return dochecksumVerification(cx);
 	}
 
 	// disable the default timeout setting
@@ -195,8 +194,7 @@ struct MakoWorkload : TestWorkload {
 		if (populateData) {
 			m.emplace_back("Mean load time (seconds)", loadTime, Averaged::True);
 			// The importing rate of keys, controlled by parameter "insertionCountsToMeasure"
-			auto ratesItr = ratesAtKeyCounts.begin();
-			for (; ratesItr != ratesAtKeyCounts.end(); ratesItr++) {
+			for (auto ratesItr = ratesAtKeyCounts.begin(); ratesItr != ratesAtKeyCounts.end(); ++ratesItr) {
 				m.emplace_back(
 				    format("%lld keys imported bytes/sec", ratesItr->first), ratesItr->second, Averaged::False);
 			}
@@ -285,12 +283,12 @@ struct MakoWorkload : TestWorkload {
 		return digits;
 	}
 
-	static void updateCSFlags(MakoWorkload* self, std::vector<bool>& flags, uint64_t startIdx, uint64_t endIdx) {
+	void updateCSFlags(std::vector<bool>& flags, uint64_t startIdx, uint64_t endIdx) {
 		// We deal with cases where rowCount % csCount != 0 and csPartitionSize % csSize != 0;
 		// In particular, all keys with index in range [csSize * csPartitionSize, rowCount) will not be used for
 		// checksum By the same way, for any i in range [0, csSize): keys with index in range [ i*csPartitionSize,
 		// i*csPartitionSize + csCount*csStepSizeInPartition) will not be used for checksum
-		uint64_t boundary = self->csSize * self->csPartitionSize;
+		uint64_t boundary = csSize * csPartitionSize;
 		if (startIdx >= boundary)
 			return;
 		else if (endIdx > boundary)
@@ -302,304 +300,301 @@ struct MakoWorkload : TestWorkload {
 
 		if (startIdx + 1 == endIdx) {
 			// single key case
-			startIdx = startIdx % self->csPartitionSize;
-			if ((startIdx < self->csCount * self->csStepSizeInPartition) &&
-			    (startIdx % self->csStepSizeInPartition == 0)) {
-				flags.at(startIdx / self->csStepSizeInPartition) = true;
+			startIdx = startIdx % csPartitionSize;
+			if ((startIdx < csCount * csStepSizeInPartition) && (startIdx % csStepSizeInPartition == 0)) {
+				flags.at(startIdx / csStepSizeInPartition) = true;
 			}
 		} else {
 			// key range case
-			uint64_t count = self->csCount;
-			uint64_t base = (startIdx / self->csPartitionSize) * self->csPartitionSize;
+			uint64_t count = csCount;
+			uint64_t base = (startIdx / csPartitionSize) * csPartitionSize;
 			startIdx -= base;
 			endIdx -= base;
-			uint64_t startStepIdx = std::min(startIdx / self->csStepSizeInPartition, self->csCount - 1);
+			uint64_t startStepIdx = std::min(startIdx / csStepSizeInPartition, csCount - 1);
 
 			// if changed range size is more than one csPartitionSize, which means every checksum needs to be updated
-			if ((endIdx - startIdx) < self->csPartitionSize) {
+			if ((endIdx - startIdx) < csPartitionSize) {
 				uint64_t endStepIdx;
-				if (endIdx > self->csPartitionSize) {
-					endStepIdx =
-					    self->csCount +
-					    std::min((endIdx - 1 - self->csPartitionSize) / self->csStepSizeInPartition, self->csCount);
+				if (endIdx > csPartitionSize) {
+					endStepIdx = csCount + std::min((endIdx - 1 - csPartitionSize) / csStepSizeInPartition, csCount);
 				} else {
-					endStepIdx = std::min((endIdx - 1) / self->csStepSizeInPartition, self->csCount - 1);
+					endStepIdx = std::min((endIdx - 1) / csStepSizeInPartition, csCount - 1);
 				}
 				// All the left boundary of csStep should be updated
 				// Also, check the startIdx whether it is the left boundary of a csStep
-				if (startIdx == self->csStepSizeInPartition * startStepIdx)
+				if (startIdx == csStepSizeInPartition * startStepIdx)
 					flags[startStepIdx] = true;
 				count = endStepIdx - startStepIdx;
 			}
 			for (int i = 1; i <= count; ++i) {
-				flags[(startStepIdx + i) % self->csCount] = true;
+				flags[(startStepIdx + i) % csCount] = true;
 			}
 		}
 	}
 	Standalone<KeyValueRef> operator()(uint64_t n) { return KeyValueRef(keyForIndex(n), randomValue()); }
 
-	ACTOR static Future<Void> tracePeriodically(MakoWorkload* self) {
-		state double start = timer();
-		state double elapsed = 0.0;
-		state int64_t last_ops = 0;
-		state int64_t last_xacts = 0;
-
-		loop {
-			elapsed += self->periodicLoggingInterval;
-			wait(delayUntil(start + elapsed));
-			TraceEvent((self->description() + "_CommitLatency").c_str())
-			    .detail("Mean", self->opLatencies[OP_COMMIT].mean())
-			    .detail("Median", self->opLatencies[OP_COMMIT].median())
-			    .detail("Percentile5", self->opLatencies[OP_COMMIT].percentile(.05))
-			    .detail("Percentile95", self->opLatencies[OP_COMMIT].percentile(.95))
-			    .detail("Count", self->opCounters[OP_COMMIT].getValue())
+	Future<Void> tracePeriodically() {
+		double start = timer();
+		double elapsed = 0.0;
+		int64_t last_ops = 0;
+		int64_t last_xacts = 0;
+
+		while (true) {
+			elapsed += periodicLoggingInterval;
+			co_await delayUntil(start + elapsed);
+			TraceEvent((description() + "_CommitLatency").c_str())
+			    .detail("Mean", opLatencies[OP_COMMIT].mean())
+			    .detail("Median", opLatencies[OP_COMMIT].median())
+			    .detail("Percentile5", opLatencies[OP_COMMIT].percentile(.05))
+			    .detail("Percentile95", opLatencies[OP_COMMIT].percentile(.95))
+			    .detail("Count", opCounters[OP_COMMIT].getValue())
 			    .detail("Elapsed", elapsed);
-			TraceEvent((self->description() + "_GRVLatency").c_str())
-			    .detail("Mean", self->opLatencies[OP_GETREADVERSION].mean())
-			    .detail("Median", self->opLatencies[OP_GETREADVERSION].median())
-			    .detail("Percentile5", self->opLatencies[OP_GETREADVERSION].percentile(.05))
-			    .detail("Percentile95", self->opLatencies[OP_GETREADVERSION].percentile(.95))
-			    .detail("Count", self->opCounters[OP_GETREADVERSION].getValue());
+			TraceEvent((description() + "_GRVLatency").c_str())
+			    .detail("Mean", opLatencies[OP_GETREADVERSION].mean())
+			    .detail("Median", opLatencies[OP_GETREADVERSION].median())
+			    .detail("Percentile5", opLatencies[OP_GETREADVERSION].percentile(.05))
+			    .detail("Percentile95", opLatencies[OP_GETREADVERSION].percentile(.95))
+			    .detail("Count", opCounters[OP_GETREADVERSION].getValue());
 
 			std::string ts = format("T=%04.0fs: ", elapsed);
-			self->periodicMetrics.emplace_back(ts + "Transactions/sec",
-			                                   (self->xacts.getValue() - last_xacts) / self->periodicLoggingInterval,
-			                                   Averaged::False);
-			self->periodicMetrics.emplace_back(ts + "Operations/sec",
-			                                   (self->totalOps.getValue() - last_ops) / self->periodicLoggingInterval,
-			                                   Averaged::False);
-
-			last_xacts = self->xacts.getValue();
-			last_ops = self->totalOps.getValue();
+			periodicMetrics.emplace_back(
+			    ts + "Transactions/sec", (xacts.getValue() - last_xacts) / periodicLoggingInterval, Averaged::False);
+			periodicMetrics.emplace_back(
+			    ts + "Operations/sec", (totalOps.getValue() - last_ops) / periodicLoggingInterval, Averaged::False);
+
+			last_xacts = xacts.getValue();
+			last_ops = totalOps.getValue();
 		}
 	}
-	ACTOR Future<Void> _setup(Database cx, MakoWorkload* self) {
+	Future<Void> _setup(Database cx) {
 		// use all the clients to populate data
-		if (self->populateData) {
-			state Promise<double> loadTime;
-			state Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts;
-
-			wait(bulkSetup(cx,
-			               self,
-			               self->rowCount,
-			               loadTime,
-			               self->insertionCountsToMeasure.empty(),
-			               self->warmingDelay,
-			               self->maxInsertRate,
-			               self->insertionCountsToMeasure,
-			               ratesAtKeyCounts));
+		if (populateData) {
+			Promise<double> loadTime;
+			Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts;
+
+			co_await bulkSetup(cx,
+			                   this,
+			                   rowCount,
+			                   loadTime,
+			                   insertionCountsToMeasure.empty(),
+			                   warmingDelay,
+			                   maxInsertRate,
+			                   insertionCountsToMeasure,
+			                   ratesAtKeyCounts);
 
 			// This is the setup time
-			self->loadTime = loadTime.getFuture().get();
+			this->loadTime = loadTime.getFuture().get();
 			// This is the rates of importing keys
-			self->ratesAtKeyCounts = ratesAtKeyCounts.getFuture().get();
+			this->ratesAtKeyCounts = ratesAtKeyCounts.getFuture().get();
 		}
 		// Use one client to initialize checksums
-		if (self->checksumVerification && self->clientId == 0) {
-			wait(generateChecksum(cx, self));
+		if (checksumVerification && clientId == 0) {
+			co_await generateChecksum(cx);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, MakoWorkload* self) {
-		// TODO: Do I need to read data to warm the cache of the keySystem like ReadWrite.actor.cpp (line 465)?
-		if (self->runBenchmark) {
-			wait(self->_runBenchmark(cx, self));
+	Future<Void> _start(Database cx) {
+		// TODO: Do I need to read data to warm the cache of the keySystem like ReadWrite.cpp (line 465)?
+		if (runBenchmark) {
+			co_await _runBenchmark(cx);
 		}
-		if (!self->preserveData && self->clientId == 0) {
-			wait(self->cleanup(cx, self));
+		if (!preserveData && clientId == 0) {
+			co_await cleanup(cx);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> _runBenchmark(Database cx, MakoWorkload* self) {
+	Future<Void> _runBenchmark(Database cx) {
 		std::vector<Future<Void>> clients;
-		clients.reserve(self->actorCountPerClient);
-		for (int c = 0; c < self->actorCountPerClient; ++c) {
-			clients.push_back(self->makoClient(cx, self, self->actorCountPerClient / self->transactionsPerSecond, c));
+		clients.reserve(actorCountPerClient);
+		for (int c = 0; c < actorCountPerClient; ++c) {
+			clients.push_back(makoClient(cx, actorCountPerClient / transactionsPerSecond, c));
 		}
 
-		if (self->enableLogging)
-			clients.push_back(tracePeriodically(self));
+		if (enableLogging)
+			clients.push_back(tracePeriodically());
 
-		wait(timeout(waitForAll(clients), self->testDuration, Void()));
-		return Void();
+		co_await timeout(waitForAll(clients), testDuration, Void());
 	}
 
-	ACTOR Future<Void> makoClient(Database cx, MakoWorkload* self, double delay, int actorIndex) {
-
-		state Key rkey, rkey2;
-		state Value rval;
-		state ReadYourWritesTransaction tr(cx);
-		state bool doCommit;
-		state int i, count;
-		state uint64_t range, indBegin, indEnd, rangeLen;
-		state KeyRangeRef rkeyRangeRef;
-		state std::vector<int> perOpCount(MAX_OP, 0);
+	Future<Void> makoClient(Database cx, double delay, int actorIndex) {
+
+		Key rkey;
+		Key rkey2;
+		Value rval;
+		ReadYourWritesTransaction tr(cx);
+		bool doCommit{ false };
+		int i{ 0 };
+		int count{ 0 };
+		uint64_t range{ 0 };
+		uint64_t indBegin{ 0 };
+		uint64_t indEnd{ 0 };
+		uint64_t rangeLen{ 0 };
+		KeyRangeRef rkeyRangeRef;
+		std::vector<int> perOpCount(MAX_OP, 0);
 		// flag at index-i indicates whether checksum-i need to be updated
-		state std::vector<bool> csChangedFlags(self->csCount, false);
-		state double lastTime = timer();
-		state double commitStart;
+		std::vector<bool> csChangedFlags(csCount, false);
+		double lastTime = timer();
+		double commitStart{ 0 };
 
 		TraceEvent("ClientStarting")
 		    .detail("ActorIndex", actorIndex)
-		    .detail("ClientIndex", self->clientId)
-		    .detail("NumActors", self->actorCountPerClient);
+		    .detail("ClientIndex", clientId)
+		    .detail("NumActors", actorCountPerClient);
 
-		loop {
+		while (true) {
 			// used for throttling
-			wait(poisson(&lastTime, delay));
+			co_await poisson(&lastTime, delay);
+			Error err;
 			try {
 				// user-defined value: whether commit read-only ops or not; default is false
-				doCommit = self->commitGet;
+				doCommit = commitGet;
 				for (i = 0; i < MAX_OP; ++i) {
 					if (i == OP_COMMIT)
 						continue;
-					for (count = 0; count < self->operations[i][0]; ++count) {
-						range = self->operations[i][1];
+					for (count = 0; count < operations[i][0]; ++count) {
+						range = operations[i][1];
 						rangeLen = digits(range);
 						// generate random key-val pair for operation
-						indBegin = self->getRandomKeyIndex(self->rowCount);
-						rkey = self->keyForIndex(indBegin);
-						rval = self->randomValue();
-						indEnd = std::min(indBegin + range, self->rowCount);
-						rkey2 = self->keyForIndex(indEnd);
+						indBegin = getRandomKeyIndex(rowCount);
+						rkey = keyForIndex(indBegin);
+						rval = randomValue();
+						indEnd = std::min(indBegin + range, rowCount);
+						rkey2 = keyForIndex(indEnd);
 						// KeyRangeRef(min, maxPlusOne)
 						rkeyRangeRef = KeyRangeRef(rkey, rkey2);
 
 						// used for mako-level consistency check
-						if (self->checksumVerification) {
+						if (checksumVerification) {
 							if (i == OP_INSERT | i == OP_UPDATE | i == OP_CLEAR) {
-								updateCSFlags(self, csChangedFlags, indBegin, indBegin + 1);
+								updateCSFlags(csChangedFlags, indBegin, indBegin + 1);
 							} else if (i == OP_CLEARRANGE) {
-								updateCSFlags(self, csChangedFlags, indBegin, indEnd);
+								updateCSFlags(csChangedFlags, indBegin, indEnd);
 							}
 						}
 
 						if (i == OP_GETREADVERSION) {
-							wait(logLatency(tr.getReadVersion(), &self->opLatencies[i]));
+							co_await logLatency(tr.getReadVersion(), &opLatencies[i]);
 						} else if (i == OP_GET) {
-							wait(logLatency(tr.get(rkey, Snapshot::False), &self->opLatencies[i]));
+							co_await logLatency(tr.get(rkey, Snapshot::False), &opLatencies[i]);
 						} else if (i == OP_GETRANGE) {
-							wait(logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, Snapshot::False),
-							                &self->opLatencies[i]));
+							co_await logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, Snapshot::False),
+							                    &opLatencies[i]);
 						} else if (i == OP_SGET) {
-							wait(logLatency(tr.get(rkey, Snapshot::True), &self->opLatencies[i]));
+							co_await logLatency(tr.get(rkey, Snapshot::True), &opLatencies[i]);
 						} else if (i == OP_SGETRANGE) {
 							// do snapshot get range here
-							wait(logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, Snapshot::True),
-							                &self->opLatencies[i]));
+							co_await logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, Snapshot::True),
+							                    &opLatencies[i]);
 						} else if (i == OP_UPDATE) {
-							wait(logLatency(tr.get(rkey, Snapshot::False), &self->opLatencies[OP_GET]));
-							if (self->latencyForLocalOperation) {
+							co_await logLatency(tr.get(rkey, Snapshot::False), &opLatencies[OP_GET]);
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.set(rkey, rval);
-								self->opLatencies[OP_INSERT].addSample(timer() - opBegin);
+								opLatencies[OP_INSERT].addSample(timer() - opBegin);
 							} else {
 								tr.set(rkey, rval);
 							}
 							doCommit = true;
 						} else if (i == OP_INSERT) {
-							// generate an (almost) unique key here, it starts with 'mako' and then comes with randomly
-							// generated characters
-							randStr(reinterpret_cast<char*>(mutateString(rkey)) + self->KEYPREFIXLEN,
-							        self->keyBytes - self->KEYPREFIXLEN);
-							if (self->latencyForLocalOperation) {
+							// generate an (almost) unique key here, it starts with 'mako' and then comes with
+							// randomly generated characters
+							randStr(reinterpret_cast<char*>(mutateString(rkey)) + KEYPREFIXLEN,
+							        keyBytes - KEYPREFIXLEN);
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.set(rkey, rval);
-								self->opLatencies[OP_INSERT].addSample(timer() - opBegin);
+								opLatencies[OP_INSERT].addSample(timer() - opBegin);
 							} else {
 								tr.set(rkey, rval);
 							}
 							doCommit = true;
 						} else if (i == OP_INSERTRANGE) {
 							char* rkeyPtr = reinterpret_cast<char*>(mutateString(rkey));
-							randStr(rkeyPtr + self->KEYPREFIXLEN, self->keyBytes - self->KEYPREFIXLEN);
+							randStr(rkeyPtr + KEYPREFIXLEN, keyBytes - KEYPREFIXLEN);
 							for (int range_i = 0; range_i < range; ++range_i) {
-								format("%0.*d", rangeLen, range_i).copy(rkeyPtr + self->keyBytes - rangeLen, rangeLen);
-								if (self->latencyForLocalOperation) {
+								format("%0.*d", rangeLen, range_i).copy(rkeyPtr + keyBytes - rangeLen, rangeLen);
+								if (latencyForLocalOperation) {
 									double opBegin = timer();
-									tr.set(rkey, self->randomValue());
-									self->opLatencies[OP_INSERT].addSample(timer() - opBegin);
+									tr.set(rkey, randomValue());
+									opLatencies[OP_INSERT].addSample(timer() - opBegin);
 								} else {
-									tr.set(rkey, self->randomValue());
+									tr.set(rkey, randomValue());
 								}
 							}
 							doCommit = true;
 						} else if (i == OP_CLEAR) {
-							if (self->latencyForLocalOperation) {
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.clear(rkey);
-								self->opLatencies[OP_CLEAR].addSample(timer() - opBegin);
+								opLatencies[OP_CLEAR].addSample(timer() - opBegin);
 							} else {
 								tr.clear(rkey);
 							}
 							doCommit = true;
 						} else if (i == OP_SETCLEAR) {
-							randStr(reinterpret_cast<char*>(mutateString(rkey)) + self->KEYPREFIXLEN,
-							        self->keyBytes - self->KEYPREFIXLEN);
-							if (self->latencyForLocalOperation) {
+							randStr(reinterpret_cast<char*>(mutateString(rkey)) + KEYPREFIXLEN,
+							        keyBytes - KEYPREFIXLEN);
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.set(rkey, rval);
-								self->opLatencies[OP_INSERT].addSample(timer() - opBegin);
+								opLatencies[OP_INSERT].addSample(timer() - opBegin);
 							} else {
 								tr.set(rkey, rval);
 							}
-							wait(self->updateCSBeforeCommit(&tr, self, &csChangedFlags));
+							co_await updateCSBeforeCommit(&tr, &csChangedFlags);
 							// commit the change and update metrics
 							commitStart = timer();
-							wait(tr.commit());
-							self->opLatencies[OP_COMMIT].addSample(timer() - commitStart);
+							co_await tr.commit();
+							opLatencies[OP_COMMIT].addSample(timer() - commitStart);
 							++perOpCount[OP_COMMIT];
 							tr.reset();
-							if (self->latencyForLocalOperation) {
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.clear(rkey);
-								self->opLatencies[OP_CLEAR].addSample(timer() - opBegin);
+								opLatencies[OP_CLEAR].addSample(timer() - opBegin);
 							} else {
 								tr.clear(rkey);
 							}
 							doCommit = true;
 						} else if (i == OP_CLEARRANGE) {
-							if (self->latencyForLocalOperation) {
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.clear(rkeyRangeRef);
-								self->opLatencies[OP_CLEARRANGE].addSample(timer() - opBegin);
+								opLatencies[OP_CLEARRANGE].addSample(timer() - opBegin);
 							} else {
 								tr.clear(rkeyRangeRef);
 							}
 							doCommit = true;
 						} else if (i == OP_SETCLEARRANGE) {
 							char* rkeyPtr = reinterpret_cast<char*>(mutateString(rkey));
-							randStr(rkeyPtr + self->KEYPREFIXLEN, self->keyBytes - self->KEYPREFIXLEN);
-							state std::string scr_start_key;
-							state std::string scr_end_key;
-							state KeyRangeRef scr_key_range_ref;
+							randStr(rkeyPtr + KEYPREFIXLEN, keyBytes - KEYPREFIXLEN);
+							std::string scr_start_key;
+							std::string scr_end_key;
+							KeyRangeRef scr_key_range_ref;
 							for (int range_i = 0; range_i < range; ++range_i) {
-								format("%0.*d", rangeLen, range_i).copy(rkeyPtr + self->keyBytes - rangeLen, rangeLen);
-								if (self->latencyForLocalOperation) {
+								format("%0.*d", rangeLen, range_i).copy(rkeyPtr + keyBytes - rangeLen, rangeLen);
+								if (latencyForLocalOperation) {
 									double opBegin = timer();
-									tr.set(rkey, self->randomValue());
-									self->opLatencies[OP_INSERT].addSample(timer() - opBegin);
+									tr.set(rkey, randomValue());
+									opLatencies[OP_INSERT].addSample(timer() - opBegin);
 								} else {
-									tr.set(rkey, self->randomValue());
+									tr.set(rkey, randomValue());
 								}
 								if (range_i == 0)
 									scr_start_key = rkey.toString();
 							}
 							scr_end_key = rkey.toString();
 							scr_key_range_ref = KeyRangeRef(KeyRef(scr_start_key), KeyRef(scr_end_key));
-							wait(self->updateCSBeforeCommit(&tr, self, &csChangedFlags));
+							co_await updateCSBeforeCommit(&tr, &csChangedFlags);
 							commitStart = timer();
-							wait(tr.commit());
-							self->opLatencies[OP_COMMIT].addSample(timer() - commitStart);
+							co_await tr.commit();
+							opLatencies[OP_COMMIT].addSample(timer() - commitStart);
 							++perOpCount[OP_COMMIT];
 							tr.reset();
-							if (self->latencyForLocalOperation) {
+							if (latencyForLocalOperation) {
 								double opBegin = timer();
 								tr.clear(scr_key_range_ref);
-								self->opLatencies[OP_CLEARRANGE].addSample(timer() - opBegin);
+								opLatencies[OP_CLEARRANGE].addSample(timer() - opBegin);
 							} else {
 								tr.clear(scr_key_range_ref);
 							}
@@ -610,59 +605,58 @@ struct MakoWorkload : TestWorkload {
 				}
 
 				if (doCommit) {
-					wait(self->updateCSBeforeCommit(&tr, self, &csChangedFlags));
+					co_await updateCSBeforeCommit(&tr, &csChangedFlags);
 					commitStart = timer();
-					wait(tr.commit());
-					self->opLatencies[OP_COMMIT].addSample(timer() - commitStart);
+					co_await tr.commit();
+					opLatencies[OP_COMMIT].addSample(timer() - commitStart);
 					++perOpCount[OP_COMMIT];
 				}
 				// successfully finish the transaction, update metrics
-				++self->xacts;
+				++xacts;
 				for (int op = 0; op < MAX_OP; ++op) {
-					self->opCounters[op] += perOpCount[op];
-					self->totalOps += perOpCount[op];
+					opCounters[op] += perOpCount[op];
+					totalOps += perOpCount[op];
 				}
 			} catch (Error& e) {
-				TraceEvent("FailedToExecOperations").error(e);
-				if (e.code() == error_code_operation_cancelled)
-					throw;
-				else if (e.code() == error_code_not_committed)
-					++self->conflicts;
-
-				wait(tr.onError(e));
-				++self->retries;
+				err = e;
 			}
+			TraceEvent("FailedToExecOperations").error(err);
+			if (err.code() == error_code_operation_cancelled)
+				throw err;
+			else if (err.code() == error_code_not_committed)
+				++conflicts;
+			co_await tr.onError(err);
+			++retries;
 			// reset all the operations' counters to 0
 			std::fill(perOpCount.begin(), perOpCount.end(), 0);
 			tr.reset();
 		}
 	}
 
-	ACTOR Future<Void> cleanup(Database cx, MakoWorkload* self) {
+	Future<Void> cleanup(Database cx) {
 		// clear all data starts with 'mako' in the database
-		state std::string keyPrefix(self->keyPrefix);
-		state ReadYourWritesTransaction tr(cx);
+		std::string keyPrefix(this->keyPrefix);
+		ReadYourWritesTransaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.clear(prefixRange(keyPrefix));
-				wait(tr.commit());
-				TraceEvent("CleanUpMakoRelatedData").detail("KeyPrefix", self->keyPrefix);
+				co_await tr.commit();
+				TraceEvent("CleanUpMakoRelatedData").detail("KeyPrefix", this->keyPrefix);
 				break;
 			} catch (Error& e) {
-				TraceEvent("FailedToCleanData").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("FailedToCleanData").error(err);
+			co_await tr.onError(err);
 		}
-
-		return Void();
 	}
-	ACTOR template <class T>
+	template <class T>
 	static Future<Void> logLatency(Future<T> f, DDSketch<double>* opLatencies) {
-		state double opBegin = timer();
-		wait(success(f));
+		double opBegin = timer();
+		co_await f;
 		opLatencies->addSample(timer() - opBegin);
-		return Void();
 	}
 
 	int64_t getRandomKeyIndex(uint64_t rowCount) {
@@ -776,14 +770,14 @@ struct MakoWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR static Future<uint32_t> calcCheckSum(ReadYourWritesTransaction* tr, MakoWorkload* self, int csIndex) {
-		state uint32_t result = 0;
-		state int i;
-		state Key csKey;
-		for (i = 0; i < self->csSize; ++i) {
-			int idx = csIndex * self->csStepSizeInPartition + i * self->csPartitionSize;
-			csKey = self->keyForIndex(idx);
-			Optional<Value> temp = wait(tr->get(csKey));
+	Future<uint32_t> calcCheckSum(ReadYourWritesTransaction* tr, int csIndex) {
+		uint32_t result = 0;
+		int i{ 0 };
+		Key csKey;
+		for (i = 0; i < csSize; ++i) {
+			int idx = csIndex * csStepSizeInPartition + i * csPartitionSize;
+			csKey = keyForIndex(idx);
+			Optional<Value> temp = co_await tr->get(csKey);
 			if (temp.present()) {
 				Value val = temp.get();
 				result = crc32c_append(result, val.begin(), val.size());
@@ -792,28 +786,29 @@ struct MakoWorkload : TestWorkload {
 				result = crc32c_append(result, csKey.begin(), csKey.size());
 			}
 		}
-		return result;
+		co_return result;
 	}
 
-	ACTOR static Future<bool> dochecksumVerification(Database cx, MakoWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
-		state int csIdx;
-		state Value csValue;
+	Future<bool> dochecksumVerification(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		int csIdx{ 0 };
+		Value csValue;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
-				for (csIdx = 0; csIdx < self->csCount; ++csIdx) {
-					Optional<Value> temp = wait(tr.get(self->csKeys[csIdx]));
+				for (csIdx = 0; csIdx < csCount; ++csIdx) {
+					Optional<Value> temp = co_await tr.get(csKeys[csIdx]);
 					if (!temp.present()) {
 						TraceEvent(SevError, "TestFailure")
 						    .detail("Reason", "NoExistingChecksum")
 						    .detail("missedChecksumIndex", csIdx);
-						return false;
+						co_return false;
 					} else {
 						csValue = temp.get();
 						ASSERT(csValue.size() == sizeof(uint32_t));
-						uint32_t calculatedCS = wait(calcCheckSum(&tr, self, csIdx));
+						uint32_t calculatedCS = co_await calcCheckSum(&tr, csIdx);
 						uint32_t existingCS = *(reinterpret_cast<const uint32_t*>(csValue.begin()));
 						if (existingCS != calculatedCS) {
 							TraceEvent(SevError, "TestFailure")
@@ -821,63 +816,60 @@ struct MakoWorkload : TestWorkload {
 							    .detail("ChecksumIndex", csIdx)
 							    .detail("ExistingChecksum", existingCS)
 							    .detail("CurrentChecksum", calculatedCS);
-							return false;
+							co_return false;
 						}
 						TraceEvent("ChecksumVerificationPass")
 						    .detail("ChecksumIndex", csIdx)
 						    .detail("ChecksumValue", existingCS);
 					}
 				}
-				return true;
+				co_return true;
 			} catch (Error& e) {
-				TraceEvent("FailedToCalculateChecksum").error(e).detail("ChecksumIndex", csIdx);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("FailedToCalculateChecksum").error(err).detail("ChecksumIndex", csIdx);
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> generateChecksum(Database cx, MakoWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
-		state int csIdx;
-		loop {
+	Future<Void> generateChecksum(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		int csIdx{ 0 };
+		while (true) {
+			Error err;
 			try {
-				for (csIdx = 0; csIdx < self->csCount; ++csIdx) {
-					Optional<Value> temp = wait(tr.get(self->csKeys[csIdx]));
+				for (csIdx = 0; csIdx < csCount; ++csIdx) {
+					Optional<Value> temp = co_await tr.get(csKeys[csIdx]);
 					if (temp.present())
-						TraceEvent("DuplicatePopulationOnSamePrefix").detail("KeyPrefix", self->keyPrefix);
-					wait(self->updateCheckSum(&tr, self, csIdx));
+						TraceEvent("DuplicatePopulationOnSamePrefix").detail("KeyPrefix", keyPrefix);
+					co_await updateCheckSum(&tr, csIdx);
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				TraceEvent("FailedToGenerateChecksumForPopulatedData").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("FailedToGenerateChecksumForPopulatedData").error(err);
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> updateCheckSum(ReadYourWritesTransaction* tr, MakoWorkload* self, int csIdx) {
-		state uint32_t csVal = wait(calcCheckSum(tr, self, csIdx));
+	Future<Void> updateCheckSum(ReadYourWritesTransaction* tr, int csIdx) {
+		uint32_t csVal = co_await calcCheckSum(tr, csIdx);
 		TraceEvent("UpdateCheckSum").detail("ChecksumIndex", csIdx).detail("Checksum", csVal);
-		tr->set(self->csKeys[csIdx], ValueRef(reinterpret_cast<const uint8_t*>(&csVal), sizeof(uint32_t)));
-		return Void();
+		tr->set(csKeys[csIdx], ValueRef(reinterpret_cast<const uint8_t*>(&csVal), sizeof(uint32_t)));
 	}
 
-	ACTOR static Future<Void> updateCSBeforeCommit(ReadYourWritesTransaction* tr,
-	                                               MakoWorkload* self,
-	                                               std::vector<bool>* flags) {
-		if (!self->checksumVerification)
-			return Void();
+	Future<Void> updateCSBeforeCommit(ReadYourWritesTransaction* tr, std::vector<bool>* flags) {
+		if (!checksumVerification)
+			co_return;
 
-		state int csIdx;
-		for (csIdx = 0; csIdx < self->csCount; ++csIdx) {
+		for (int csIdx = 0; csIdx < csCount; ++csIdx) {
 			if ((*flags)[csIdx]) {
-				wait(updateCheckSum(tr, self, csIdx));
+				co_await updateCheckSum(tr, csIdx);
 				(*flags)[csIdx] = false;
 			}
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/MaxGrvQueueDelay.cpp b/fdbserver/workloads/MaxGrvQueueDelay.cpp
new file mode 100644
index 00000000000..d388c5b3c81
--- /dev/null
+++ b/fdbserver/workloads/MaxGrvQueueDelay.cpp
@@ -0,0 +1,208 @@
+/*
+ * MaxGrvQueueDelay.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "flow/Buggify.h"
+#include "flow/Error.h"
+#include "flow/Trace.h"
+#include "flow/genericactors.actor.h"
+
+// This workload validates the MAX_GRV_QUEUE_DELAY transaction option end-to-end.
+// The test file lowers GRV ratekeeper limits so a burst of uncached GRV requests
+// should produce proxy-side transaction_grv_queue_rejected errors, while a request
+// without the option and a request with a permissive threshold still complete.
+struct MaxGrvQueueDelayWorkload : TestWorkload {
+	static constexpr auto NAME = "MaxGrvQueueDelay";
+
+	int requestCount;
+	int minRejected;
+	int64_t maxQueueDelayMS;
+	int64_t permissiveMaxQueueDelayMS;
+	int warmupRequestCount;
+	double warmupDelay;
+	double startAfter;
+	double testTimeout;
+
+	bool done{ false };
+	bool failed{ false };
+	PerfIntCounter requests;
+	PerfIntCounter successes;
+	PerfIntCounter rejected;
+	PerfIntCounter unexpectedErrors;
+
+	MaxGrvQueueDelayWorkload(WorkloadContext const& wcx)
+	  : TestWorkload(wcx), requests("Requests"), successes("Successes"), rejected("Rejected"),
+	    unexpectedErrors("UnexpectedErrors") {
+		requestCount = getOption(options, "requestCount"_sr, 50);
+		minRejected = getOption(options, "minRejected"_sr, 1);
+		maxQueueDelayMS = getOption(options, "maxQueueDelayMS"_sr, int64_t{ 0 });
+		permissiveMaxQueueDelayMS = getOption(options, "permissiveMaxQueueDelayMS"_sr, int64_t{ 60000 });
+		warmupRequestCount = getOption(options, "warmupRequestCount"_sr, 0);
+		warmupDelay = getOption(options, "warmupDelay"_sr, 0.0);
+		startAfter = getOption(options, "startAfter"_sr, 5.0);
+		testTimeout = getOption(options, "testTimeout"_sr, 30.0);
+	}
+
+	static bool runTest() { return g_network->isSimulated() && !isGeneralBuggifyEnabled(); }
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
+
+	Future<Void> setup(Database const& cx) override {
+		TraceEvent("MaxGrvQueueDelaySetup")
+		    .detail("RequestCount", requestCount)
+		    .detail("MinRejected", minRejected)
+		    .detail("MaxQueueDelayMS", maxQueueDelayMS)
+		    .detail("PermissiveMaxQueueDelayMS", permissiveMaxQueueDelayMS)
+		    .detail("WarmupRequestCount", warmupRequestCount)
+		    .detail("WarmupDelay", warmupDelay)
+		    .detail("StartAfter", startAfter)
+		    .detail("TestTimeout", testTimeout);
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0 || !runTest()) {
+			return Void();
+		}
+		return timeout(reportErrors(run(cx), "MaxGrvQueueDelayError"), testTimeout, Void());
+	}
+
+	Future<bool> check(Database const& cx) override {
+		if (clientId != 0 || !runTest()) {
+			return true;
+		}
+		if (!done) {
+			TraceEvent(SevError, "MaxGrvQueueDelayCheckFailed")
+			    .detail("Reason", "WorkloadDidNotComplete")
+			    .detail("Rejected", rejected.getValue())
+			    .detail("Successes", successes.getValue())
+			    .detail("UnexpectedErrors", unexpectedErrors.getValue());
+			return false;
+		}
+		return !failed;
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override {
+		m.emplace_back("Requests", requests.getValue(), Averaged::False);
+		m.emplace_back("Successes", successes.getValue(), Averaged::False);
+		m.emplace_back("Rejected", rejected.getValue(), Averaged::False);
+		m.emplace_back("Unexpected Errors", unexpectedErrors.getValue(), Averaged::False);
+	}
+
+	Future<Version> getReadVersion(Database cx, Optional<int64_t> maxDelayMS) {
+		Transaction tr(cx);
+		tr.setOption(FDBTransactionOptions::SKIP_GRV_CACHE);
+		if (maxDelayMS.present()) {
+			int64_t optionValue = maxDelayMS.get();
+			tr.setOption(FDBTransactionOptions::MAX_GRV_QUEUE_DELAY,
+			             StringRef(reinterpret_cast<uint8_t*>(&optionValue), sizeof(optionValue)));
+		}
+		Version version = co_await tr.getReadVersion();
+		co_return version;
+	}
+
+	Future<Void> verifyBaseline(Database cx) {
+		ErrorOr<Version> withoutOption = co_await errorOr(getReadVersion(cx, Optional<int64_t>()));
+		if (withoutOption.isError()) {
+			failed = true;
+			++unexpectedErrors;
+			TraceEvent(SevError, "MaxGrvQueueDelayBaselineFailed")
+			    .error(withoutOption.getError())
+			    .detail("HasOption", false);
+			co_return;
+		}
+		++successes;
+
+		ErrorOr<Version> permissive =
+		    co_await errorOr(getReadVersion(cx, Optional<int64_t>(permissiveMaxQueueDelayMS)));
+		if (permissive.isError()) {
+			failed = true;
+			++unexpectedErrors;
+			TraceEvent(SevError, "MaxGrvQueueDelayBaselineFailed")
+			    .error(permissive.getError())
+			    .detail("HasOption", true)
+			    .detail("MaxQueueDelayMS", permissiveMaxQueueDelayMS);
+			co_return;
+		}
+		++successes;
+	}
+
+	Future<Void> run(Database cx) {
+		co_await delay(startAfter);
+		co_await verifyBaseline(cx);
+		if (failed) {
+			done = true;
+			co_return;
+		}
+
+		std::vector<Future<ErrorOr<Version>>> warmupFutures;
+		warmupFutures.reserve(warmupRequestCount);
+		for (int i = 0; i < warmupRequestCount; ++i) {
+			warmupFutures.push_back(errorOr(getReadVersion(cx, Optional<int64_t>())));
+		}
+		if (warmupDelay > 0.0) {
+			co_await delay(warmupDelay);
+		}
+
+		std::vector<Future<ErrorOr<Version>>> futures;
+		futures.reserve(requestCount);
+		for (int i = 0; i < requestCount; ++i) {
+			++requests;
+			futures.push_back(errorOr(getReadVersion(cx, Optional<int64_t>(maxQueueDelayMS))));
+		}
+
+		co_await waitForAll(futures);
+
+		for (auto const& f : futures) {
+			ASSERT(f.isReady());
+			ErrorOr<Version> result = f.get();
+			if (!result.isError()) {
+				++successes;
+			} else if (result.getError().code() == error_code_transaction_grv_queue_rejected) {
+				++rejected;
+			} else {
+				++unexpectedErrors;
+				failed = true;
+				TraceEvent(SevError, "MaxGrvQueueDelayUnexpectedError")
+				    .error(result.getError())
+				    .detail("ExpectedError", error_code_transaction_grv_queue_rejected);
+			}
+		}
+
+		if (rejected.getValue() < minRejected) {
+			failed = true;
+			TraceEvent(SevError, "MaxGrvQueueDelayTooFewRejections")
+			    .detail("Rejected", rejected.getValue())
+			    .detail("MinRejected", minRejected)
+			    .detail("Successes", successes.getValue())
+			    .detail("Requests", requests.getValue());
+		} else {
+			TraceEvent("MaxGrvQueueDelayComplete")
+			    .detail("Rejected", rejected.getValue())
+			    .detail("Successes", successes.getValue())
+			    .detail("Requests", requests.getValue());
+		}
+
+		done = true;
+	}
+};
+
+WorkloadFactory<MaxGrvQueueDelayWorkload> MaxGrvQueueDelayWorkloadFactory;
diff --git a/fdbserver/workloads/MemoryKeyValueStore.cpp b/fdbserver/workloads/MemoryKeyValueStore.cpp
index 24b15be2911..115b5b0ae3c 100644
--- a/fdbserver/workloads/MemoryKeyValueStore.cpp
+++ b/fdbserver/workloads/MemoryKeyValueStore.cpp
@@ -18,11 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/MemoryKeyValueStore.h"
+#include "MemoryKeyValueStore.h"
 
 // Get the value associated with a key
 Optional<Value> MemoryKeyValueStore::get(KeyRef key) const {
-	std::map<Key, Value>::const_iterator value = store.find(key);
+	auto value = store.find(key);
 	if (value != store.end())
 		return value->second;
 	else
@@ -32,7 +32,7 @@ Optional<Value> MemoryKeyValueStore::get(KeyRef key) const {
 // Returns the key designated by a key selector
 Key MemoryKeyValueStore::getKey(KeySelectorRef selector) const {
 	// Begin by getting the start key referenced by the key selector
-	std::map<Key, Value>::const_iterator mapItr = store.lower_bound(selector.getKey());
+	auto mapItr = store.lower_bound(selector.getKey());
 
 	// Update the iterator position if necessary based on the value of orEqual
 	int count = 0;
@@ -81,16 +81,16 @@ Key MemoryKeyValueStore::getKey(KeySelectorRef selector) const {
 RangeResult MemoryKeyValueStore::getRange(KeyRangeRef range, int limit, Reverse reverse) const {
 	RangeResult results;
 	if (!reverse) {
-		std::map<Key, Value>::const_iterator mapItr = store.lower_bound(range.begin);
-
-		for (; mapItr != store.end() && mapItr->first < range.end && results.size() < limit; mapItr++)
+		for (auto mapItr = store.lower_bound(range.begin);
+		     mapItr != store.end() && mapItr->first < range.end && results.size() < limit;
+		     mapItr++)
 			results.push_back_deep(results.arena(), KeyValueRef(mapItr->first, mapItr->second));
 	}
 
 	// Support for reverse getRange queries is supported, but not tested at this time.  This is because reverse range
 	// queries have been disallowed by the database at the API level
 	else {
-		std::map<Key, Value>::const_iterator mapItr = store.lower_bound(range.end);
+		auto mapItr = store.lower_bound(range.end);
 		if (mapItr == store.begin())
 			return results;
 
diff --git a/fdbserver/include/fdbserver/workloads/MemoryKeyValueStore.h b/fdbserver/workloads/MemoryKeyValueStore.h
similarity index 97%
rename from fdbserver/include/fdbserver/workloads/MemoryKeyValueStore.h
rename to fdbserver/workloads/MemoryKeyValueStore.h
index 2da763ec95b..ebb0f6710ca 100644
--- a/fdbserver/include/fdbserver/workloads/MemoryKeyValueStore.h
+++ b/fdbserver/workloads/MemoryKeyValueStore.h
@@ -23,7 +23,7 @@
 #pragma once
 
 #include "fdbrpc/fdbrpc.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 
 #include <map>
 
diff --git a/fdbserver/workloads/MemoryLifetime.actor.cpp b/fdbserver/workloads/MemoryLifetime.cpp
similarity index 62%
rename from fdbserver/workloads/MemoryLifetime.actor.cpp
rename to fdbserver/workloads/MemoryLifetime.cpp
index b872bf776c2..602075ab302 100644
--- a/fdbserver/workloads/MemoryLifetime.actor.cpp
+++ b/fdbserver/workloads/MemoryLifetime.cpp
@@ -1,5 +1,5 @@
 /*
- * MemoryLifetime.actor.cpp
+ * MemoryLifetime.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,18 +19,17 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct MemoryLifetime : KVWorkload {
 	static constexpr auto NAME = "MemoryLifetime";
 	double testDuration;
 	std::vector<Future<Void>> clients;
 
-	MemoryLifetime(WorkloadContext const& wcx) : KVWorkload(wcx) {
+	explicit MemoryLifetime(WorkloadContext const& wcx) : KVWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 60.0);
 	}
 
@@ -42,50 +41,45 @@ struct MemoryLifetime : KVWorkload {
 
 	Standalone<KeyValueRef> operator()(uint64_t n) { return KeyValueRef(keyForIndex(n, false), randomValue()); }
 
-	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
-
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> _setup(Database cx, MemoryLifetime* self) {
-		state Promise<double> loadTime;
-		wait(bulkSetup(cx, self, self->nodeCount, loadTime));
-		return Void();
+	Future<Void> setup(Database const& cx) override {
+		Promise<double> loadTime;
+		co_await bulkSetup(cx, this, nodeCount, loadTime);
 	}
 
-	ACTOR Future<Void> _start(Database cx, MemoryLifetime* self) {
-		state double startTime = now();
-		state ReadYourWritesTransaction tr(cx);
-		state Reverse reverse = Reverse::False;
-		state Snapshot snapshot = Snapshot::False;
-		loop {
+	Future<Void> start(Database const& cx) override {
+		double startTime = now();
+		ReadYourWritesTransaction tr(cx);
+		Reverse reverse = Reverse::False;
+		Snapshot snapshot = Snapshot::False;
+		while (true) {
+			Error err;
 			try {
 				tr = ReadYourWritesTransaction(cx);
 				int op = deterministicRandom()->randomInt(0, 4);
 				if (op == 0) {
 					reverse.set(deterministicRandom()->coinflip());
-					state Key getRange_startKey = self->getRandomKey();
-					state KeyRange getRange_queryRange =
-					    reverse ? KeyRangeRef(normalKeys.begin, keyAfter(getRange_startKey))
-					            : KeyRangeRef(getRange_startKey, normalKeys.end);
-					state bool getRange_randomStart = deterministicRandom()->random01();
-					state Value getRange_newValue = self->randomValue();
+					Key getRange_startKey = getRandomKey();
+					KeyRange getRange_queryRange = reverse ? KeyRangeRef(normalKeys.begin, keyAfter(getRange_startKey))
+					                                       : KeyRangeRef(getRange_startKey, normalKeys.end);
+					bool getRange_randomStart = deterministicRandom()->random01();
+					Value getRange_newValue = randomValue();
 					snapshot.set(deterministicRandom()->coinflip());
 
 					//TraceEvent("MemoryLifetimeCheck").detail("IsReverse", reverse).detail("StartKey", printable(getRange_startKey)).detail("RandomStart", getRange_randomStart).detail("NewValue", getRange_newValue.size()).detail("IsSnapshot", snapshot);
 					if (getRange_randomStart)
 						tr.set(getRange_startKey, getRange_newValue);
-					state RangeResult getRange_res1 =
-					    wait(tr.getRange(getRange_queryRange, GetRangeLimits(4000), snapshot, reverse));
+					RangeResult getRange_res1 =
+					    co_await tr.getRange(getRange_queryRange, GetRangeLimits(4000), snapshot, reverse);
 					tr = ReadYourWritesTransaction(cx);
-					wait(delay(0.01));
+					co_await delay(0.01);
 					if (getRange_randomStart)
 						tr.set(getRange_startKey, getRange_newValue);
 					RangeResult getRange_res2 =
-					    wait(tr.getRange(getRange_queryRange, GetRangeLimits(4000), snapshot, reverse));
+					    co_await tr.getRange(getRange_queryRange, GetRangeLimits(4000), snapshot, reverse);
 					ASSERT(getRange_res1.size() == getRange_res2.size());
 					for (int i = 0; i < getRange_res1.size(); i++) {
 						if (getRange_res1[i].key != getRange_res2[i].key) {
@@ -110,51 +104,54 @@ struct MemoryLifetime : KVWorkload {
 						}
 					}
 				} else if (op == 1) {
-					state Key get_startKey = self->getRandomKey();
-					state bool get_randomStart = deterministicRandom()->random01();
-					state Value get_newValue = self->randomValue();
+					Key get_startKey = getRandomKey();
+					bool get_randomStart = deterministicRandom()->random01();
+					Value get_newValue = randomValue();
 					snapshot.set(deterministicRandom()->coinflip());
 
 					if (get_randomStart)
 						tr.set(get_startKey, get_newValue);
-					state Optional<Value> get_res1 = wait(tr.get(get_startKey, snapshot));
+					Optional<Value> get_res1 = co_await tr.get(get_startKey, snapshot);
 					tr = ReadYourWritesTransaction(cx);
-					wait(delay(0.01));
+					co_await delay(0.01);
 					if (get_randomStart)
 						tr.set(get_startKey, get_newValue);
-					Optional<Value> get_res2 = wait(tr.get(get_startKey, snapshot));
+					Optional<Value> get_res2 = co_await tr.get(get_startKey, snapshot);
 					ASSERT(get_res1 == get_res2);
 				} else if (op == 2) {
-					state KeySelector getKey_selector = self->getRandomKeySelector();
-					state bool getKey_randomStart = deterministicRandom()->random01();
-					state Value getKey_newValue = self->randomValue();
+					KeySelector getKey_selector = getRandomKeySelector();
+					bool getKey_randomStart = deterministicRandom()->random01();
+					Value getKey_newValue = randomValue();
 					snapshot.set(deterministicRandom()->coinflip());
 
 					if (getKey_randomStart)
 						tr.set(getKey_selector.getKey(), getKey_newValue);
-					state Key getKey_res1 = wait(tr.getKey(getKey_selector, snapshot));
+					Key getKey_res1 = co_await tr.getKey(getKey_selector, snapshot);
 					tr = ReadYourWritesTransaction(cx);
-					wait(delay(0.01));
+					co_await delay(0.01);
 					if (getKey_randomStart)
 						tr.set(getKey_selector.getKey(), getKey_newValue);
-					Key getKey_res2 = wait(tr.getKey(getKey_selector, snapshot));
+					Key getKey_res2 = co_await tr.getKey(getKey_selector, snapshot);
 					ASSERT(getKey_res1 == getKey_res2);
 				} else if (op == 3) {
-					state Key getAddress_startKey = self->getRandomKey();
-					state Standalone<VectorRef<const char*>> getAddress_res1 =
-					    wait(tr.getAddressesForKey(getAddress_startKey));
+					Key getAddress_startKey = getRandomKey();
+					Standalone<VectorRef<const char*>> getAddress_res1 =
+					    co_await tr.getAddressesForKey(getAddress_startKey);
 					tr = ReadYourWritesTransaction(cx);
-					wait(delay(0.01));
-					// we cannot check the contents like other operations so just touch all the values to make sure we
-					// dont crash
+					co_await delay(0.01);
+					// we cannot check the contents like other operations so just touch all the values to make sure
+					// we dont crash
 					for (int i = 0; i < getAddress_res1.size(); i++) {
 						ASSERT(NetworkAddress::parseOptional(getAddress_res1[i]).present());
 					}
 				}
-				if (now() - startTime > self->testDuration)
-					return Void();
+				if (now() - startTime > testDuration)
+					co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
 	}
diff --git a/fdbserver/workloads/MetricLogging.actor.cpp b/fdbserver/workloads/MetricLogging.cpp
similarity index 74%
rename from fdbserver/workloads/MetricLogging.actor.cpp
rename to fdbserver/workloads/MetricLogging.cpp
index 1055826b280..e871335f847 100644
--- a/fdbserver/workloads/MetricLogging.actor.cpp
+++ b/fdbserver/workloads/MetricLogging.cpp
@@ -1,5 +1,5 @@
 /*
- * MetricLogging.actor.cpp
+ * MetricLogging.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,8 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "flow/TDMetric.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TDMetric.h"
+#include "fdbserver/tester/workloads.h"
 
 struct MetricLoggingWorkload : TestWorkload {
 	static constexpr auto NAME = "MetricLogging";
@@ -35,7 +33,7 @@ struct MetricLoggingWorkload : TestWorkload {
 	std::vector<BoolMetricHandle> boolMetrics;
 	std::vector<Int64MetricHandle> int64Metrics;
 
-	MetricLoggingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), changes("Changes") {
+	explicit MetricLoggingWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), changes("Changes") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		actorCount = getOption(options, "actorCount"_sr, 1);
 		metricCount = getOption(options, "metricCount"_sr, 1);
@@ -51,18 +49,15 @@ struct MetricLoggingWorkload : TestWorkload {
 		}
 	}
 
-	Future<Void> setup(Database const& cx) override { return _setup(this, cx); }
-
-	ACTOR Future<Void> _setup(MetricLoggingWorkload* self, Database cx) {
-		wait(delay(2.0));
-		for (int i = 0; i < self->metricCount; i++) {
-			if (self->testBool) {
-				self->boolMetrics[i]->setConfig(true);
+	Future<Void> setup(Database const& cx) override {
+		co_await delay(2.0);
+		for (int i = 0; i < metricCount; i++) {
+			if (testBool) {
+				boolMetrics[i]->setConfig(true);
 			} else {
-				self->int64Metrics[i]->setConfig(true);
+				int64Metrics[i]->setConfig(true);
 			}
 		}
-		return Void();
 	}
 
 	Future<Void> start(Database const& cx) override {
@@ -81,9 +76,9 @@ struct MetricLoggingWorkload : TestWorkload {
 		m.emplace_back("Changes/sec", changes.getValue() / testDuration, Averaged::False);
 	}
 
-	ACTOR Future<Void> MetricLoggingClient(Database cx, MetricLoggingWorkload* self, int clientId, int actorId) {
-		state BinaryWriter writer(Unversioned());
-		loop {
+	Future<Void> MetricLoggingClient(Database cx, MetricLoggingWorkload* self, int clientId, int actorId) {
+		BinaryWriter writer(Unversioned());
+		while (true) {
 			for (int i = 0; i < 100; i++) {
 				if (self->testBool) {
 					self->boolMetrics[self->changes.getValue() % self->metricCount]->toggle();
@@ -92,7 +87,7 @@ struct MetricLoggingWorkload : TestWorkload {
 				}
 				++self->changes;
 			}
-			wait(yield());
+			co_await yield();
 		}
 	}
 };
diff --git a/fdbserver/workloads/MiniCycle.actor.cpp b/fdbserver/workloads/MiniCycle.cpp
similarity index 81%
rename from fdbserver/workloads/MiniCycle.actor.cpp
rename to fdbserver/workloads/MiniCycle.cpp
index 324119812b3..8d457e8e4aa 100644
--- a/fdbserver/workloads/MiniCycle.actor.cpp
+++ b/fdbserver/workloads/MiniCycle.cpp
@@ -1,5 +1,5 @@
 /*
- * MiniCycle.actor.cpp
+ * MiniCycle.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,16 +20,15 @@
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "flow/serialize.h"
-#include <cstring>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 struct MiniCycleWorkload : TestWorkload {
 	static constexpr auto NAME = "MiniCycle";
@@ -42,7 +41,7 @@ struct MiniCycleWorkload : TestWorkload {
 	PerfIntCounter transactions, retries, tooOldRetries, commitFailedRetries;
 	PerfDoubleCounter totalLatency;
 
-	MiniCycleWorkload(WorkloadContext const& wcx)
+	explicit MiniCycleWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), tooOldRetries("Retries.too_old"),
 	    commitFailedRetries("Retries.commit_failed"), totalLatency("Latency") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -73,26 +72,28 @@ struct MiniCycleWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override { return Void(); }
 	Future<bool> check(Database const& cx) override { return _check(cx->clone(), this); }
 
-	ACTOR Future<bool> _check(Database cx, MiniCycleWorkload* self) {
-		state std::vector<Future<Void>> cycleClients;
+	Future<bool> _check(Database cx, MiniCycleWorkload* self) {
+		std::vector<Future<Void>> cycleClients;
 		for (int c = 0; c < self->clientCount; c++)
 			cycleClients.push_back(
 			    timeout(self->cycleClient(cx->clone(), self, self->actorCount / self->transactionsPerSecond),
 			            self->testDuration,
 			            Void()));
 
-		state Future<Void> end = delay(self->testDuration);
-		state bool ok = true;
-		loop {
-			choose {
-				when(bool ret = wait(self->_checkCycle(cx->clone(), self, ok))) {
-					ok = ret && ok;
-					if (!ok)
-						return false;
-				}
-				when(wait(end)) {
-					break;
-				}
+		Future<Void> end = delay(self->testDuration);
+		bool ok = true;
+		while (true) {
+			auto choice = co_await race(self->_checkCycle(cx->clone(), self, ok), end);
+			if (choice.index() == 0) {
+				bool ret = std::get<0>(std::move(choice));
+
+				ok = ret && ok;
+				if (!ok)
+					co_return false;
+			} else if (choice.index() == 1) {
+				break;
+			} else {
+				UNREACHABLE();
 			}
 		}
 
@@ -105,15 +106,15 @@ struct MiniCycleWorkload : TestWorkload {
 		cycleClients.clear();
 
 		printf("Beginning full cycle check...");
-		bool ret = wait(self->_checkCycle(cx->clone(), self, ok));
-		return ret;
+		bool ret = co_await self->_checkCycle(cx->clone(), self, ok);
+		co_return ret;
 	}
 
-	ACTOR Future<bool> _checkCycle(Database cx, MiniCycleWorkload* self, bool ok) {
-		state std::vector<Future<bool>> checkClients;
+	Future<bool> _checkCycle(Database cx, MiniCycleWorkload* self, bool ok) {
+		std::vector<Future<bool>> checkClients;
 		for (int c = 0; c < self->clientCount; c++)
 			checkClients.push_back(self->cycleCheckClient(cx->clone(), self, ok));
-		bool ret = wait(allTrue(checkClients));
+		bool ret = co_await allTrue(checkClients);
 
 		// Check for errors in the cycle clients
 		int errors = 0;
@@ -121,7 +122,7 @@ struct MiniCycleWorkload : TestWorkload {
 			errors += checkClients[c].isError();
 		if (errors)
 			TraceEvent(SevError, "TestFailure").detail("Reason", "There were checker errors.");
-		return ret;
+		co_return ret;
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
@@ -164,34 +165,35 @@ struct MiniCycleWorkload : TestWorkload {
 		    .detailf("From", "%016llx", debug_lastLoadBalanceResultEndpointToken);
 	}
 
-	ACTOR Future<Void> cycleClient(Database cx, MiniCycleWorkload* self, double delay) {
-		state double lastTime = now();
+	Future<Void> cycleClient(Database cx, MiniCycleWorkload* self, double delay) {
+		double lastTime = now();
 		try {
-			loop {
-				wait(poisson(&lastTime, delay));
+			while (true) {
+				co_await poisson(&lastTime, delay);
 
-				state double tstart = now();
-				state int r =
+				double tstart = now();
+				int r =
 				    deterministicRandom()->randomInt(self->beginKey(self->clientId), self->endKey(self->clientId) - 1);
-				state Transaction tr(cx);
+				Transaction tr(cx);
 				if (deterministicRandom()->random01() >= self->traceParentProbability) {
-					state Span span("MiniCycleClient"_loc);
+					Span span("MiniCycleClient"_loc);
 					TraceEvent("MiniCycleTracingTransaction", span.context.traceID).log();
 					tr.setOption(FDBTransactionOptions::SPAN_PARENT,
 					             BinaryWriter::toValue(span.context, Unversioned()));
 				}
 				while (true) {
+					Error err;
 					try {
 						// Reverse next and next^2 node
-						Optional<Value> v = wait(tr.get(self->key(r)));
+						Optional<Value> v = co_await tr.get(self->key(r));
 						if (!v.present())
 							self->badRead("KeyR", r, tr);
-						state int r2 = self->fromValue(v.get());
-						Optional<Value> v2 = wait(tr.get(self->key(r2)));
+						int r2 = self->fromValue(v.get());
+						Optional<Value> v2 = co_await tr.get(self->key(r2));
 						if (!v2.present())
 							self->badRead("KeyR2", r2, tr);
-						state int r3 = self->fromValue(v2.get());
-						Optional<Value> v3 = wait(tr.get(self->key(r3)));
+						int r3 = self->fromValue(v2.get());
+						Optional<Value> v3 = co_await tr.get(self->key(r3));
 						if (!v3.present())
 							self->badRead("KeyR3", r3, tr);
 						int r4 = self->fromValue(v3.get());
@@ -204,16 +206,17 @@ struct MiniCycleWorkload : TestWorkload {
 						// TraceEvent("CyclicTest").detail("Key", self->key(r2).toString()).detail("Value", self->value(r4).toString());
 						// TraceEvent("CyclicTest").detail("Key", self->key(r3).toString()).detail("Value", self->value(r2).toString());
 
-						wait(tr.commit());
+						co_await tr.commit();
 						// TraceEvent("MiniCycleCommit");
 						break;
 					} catch (Error& e) {
-						if (e.code() == error_code_transaction_too_old)
-							++self->tooOldRetries;
-						else if (e.code() == error_code_not_committed)
-							++self->commitFailedRetries;
-						wait(tr.onError(e));
+						err = e;
 					}
+					if (err.code() == error_code_transaction_too_old)
+						++self->tooOldRetries;
+					else if (err.code() == error_code_not_committed)
+						++self->commitFailedRetries;
+					co_await tr.onError(err);
 					++self->retries;
 				}
 				++self->transactions;
@@ -302,7 +305,7 @@ struct MiniCycleWorkload : TestWorkload {
 		}
 		return true;
 	}
-	ACTOR Future<bool> cycleCheckClient(Database cx, MiniCycleWorkload* self, bool ok) {
+	Future<bool> cycleCheckClient(Database cx, MiniCycleWorkload* self, bool ok) {
 		if (self->transactions.getMetric().value() < self->testDuration * self->minExpectedTransactionsPerSecond) {
 			TraceEvent(SevWarnAlways, "TestFailure")
 			    .detail("Reason", "Rate below desired rate")
@@ -318,27 +321,29 @@ struct MiniCycleWorkload : TestWorkload {
 		}
 
 		// One client checks the validity of the cycle at a time
-		wait(self->checkLock.take());
-		state FlowLock::Releaser releaser(self->checkLock);
+		co_await self->checkLock.take();
+		FlowLock::Releaser releaser(self->checkLock);
 
-		state Transaction tr(cx);
-		state int retryCount = 0;
-		loop {
+		Transaction tr(cx);
+		int retryCount = 0;
+		while (true) {
+			Error err;
 			try {
-				state Version v = wait(tr.getReadVersion());
-				RangeResult data = wait(
-				    tr.getRange(firstGreaterOrEqual(doubleToTestKey(self->beginKey(self->clientId), self->keyPrefix)),
-				                firstGreaterOrEqual(doubleToTestKey(self->endKey(self->clientId), self->keyPrefix)),
-				                self->cycleSize(self->clientId) + 1));
+				Version v = co_await tr.getReadVersion();
+				RangeResult data = co_await tr.getRange(
+				    firstGreaterOrEqual(doubleToTestKey(self->beginKey(self->clientId), self->keyPrefix)),
+				    firstGreaterOrEqual(doubleToTestKey(self->endKey(self->clientId), self->keyPrefix)),
+				    self->cycleSize(self->clientId) + 1);
 				ok = self->cycleCheckData(data, v, self->clientId) && ok;
 				break;
 			} catch (Error& e) {
-				retryCount++;
-				TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "MiniCycleCheckError").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			retryCount++;
+			TraceEvent(retryCount > 20 ? SevWarnAlways : SevWarn, "MiniCycleCheckError").error(err);
+			co_await tr.onError(err);
 		}
-		return ok;
+		co_return ok;
 	}
 };
 
diff --git a/fdbserver/workloads/MockDDReadWrite.actor.cpp b/fdbserver/workloads/MockDDReadWrite.cpp
similarity index 90%
rename from fdbserver/workloads/MockDDReadWrite.actor.cpp
rename to fdbserver/workloads/MockDDReadWrite.cpp
index 89a90348b75..af553791f5c 100644
--- a/fdbserver/workloads/MockDDReadWrite.actor.cpp
+++ b/fdbserver/workloads/MockDDReadWrite.cpp
@@ -1,5 +1,5 @@
 /*
- * MockDDReadWrite.actor.cpp
+ * MockDDReadWrite.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,10 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/MockDDTest.h"
-#include "fdbserver/MockDataDistributor.h"
-#include "fdbserver/DDTxnProcessor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "MockDDTest.h"
+#include "fdbserver/datadistributor/MockDataDistributor.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
 
 class MockDDReadWriteWorkload : public MockDDTestWorkload {
 public:
@@ -70,4 +69,4 @@ class MockDDReadWriteWorkload : public MockDDTestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<MockDDReadWriteWorkload> MockDDReadWriteWorkload;
\ No newline at end of file
+WorkloadFactory<MockDDReadWriteWorkload> MockDDReadWriteWorkload;
diff --git a/fdbserver/workloads/MockDDTest.actor.cpp b/fdbserver/workloads/MockDDTest.cpp
similarity index 97%
rename from fdbserver/workloads/MockDDTest.actor.cpp
rename to fdbserver/workloads/MockDDTest.cpp
index ca6c02d015b..e3f6066902e 100644
--- a/fdbserver/workloads/MockDDTest.actor.cpp
+++ b/fdbserver/workloads/MockDDTest.cpp
@@ -1,5 +1,5 @@
 /*
- * MockDDTest.actor.cpp
+ * MockDDTest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,8 +18,7 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/MockDDTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "MockDDTest.h"
 
 KeyRange MockDDTestWorkload::getRandomRange(double offset) const {
 	double len = deterministicRandom()->random01() * this->maxKeyspace;
@@ -122,4 +121,4 @@ Future<Void> MockDDTestWorkload::setup(Database const& cx) {
 	sharedMgs->initializeClusterLayout(dbConfig);
 	sharedMgs->initializeAsEmptyDatabaseMGS(dbConfig.db);
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbserver/include/fdbserver/workloads/MockDDTest.h b/fdbserver/workloads/MockDDTest.h
similarity index 93%
rename from fdbserver/include/fdbserver/workloads/MockDDTest.h
rename to fdbserver/workloads/MockDDTest.h
index 40fb987e5fe..253d76ad900 100644
--- a/fdbserver/include/fdbserver/workloads/MockDDTest.h
+++ b/fdbserver/workloads/MockDDTest.h
@@ -20,10 +20,10 @@
 #ifndef FOUNDATIONDB_MOCKDDTEST_H
 #define FOUNDATIONDB_MOCKDDTEST_H
 
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/DDTxnProcessor.h"
-#include "fdbserver/MoveKeys.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
+#include "fdbserver/core/MoveKeys.h"
 #include "fdbclient/StorageServerInterface.h"
 
 // other Mock DD workload can derive from this class to use the common settings
diff --git a/fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp b/fdbserver/workloads/MockDDTrackerShardEvaluator.cpp
similarity index 83%
rename from fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp
rename to fdbserver/workloads/MockDDTrackerShardEvaluator.cpp
index 3e7a8260305..f7080c853cb 100644
--- a/fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp
+++ b/fdbserver/workloads/MockDDTrackerShardEvaluator.cpp
@@ -1,5 +1,5 @@
 /*
- * MockDDTrackerShardEvaluator.actor.cpp
+ * MockDDTrackerShardEvaluator.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,8 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/MockDDTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "MockDDTest.h"
+#include "flow/CoroUtils.h"
 
 class MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
 public:
@@ -50,7 +50,7 @@ class MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
 	int checkMinSizeSplit = 0;
 	int checkMinWriteSplit = 0;
 
-	MockDDTrackerShardEvaluatorWorkload(WorkloadContext const& wcx)
+	explicit MockDDTrackerShardEvaluatorWorkload(WorkloadContext const& wcx)
 	  : MockDDTestWorkload(wcx), ddcx(deterministicRandom()->randomUniqueID()) {
 		checkMinShardCount = getOption(options, "checkMinShardCount"_sr, checkMinShardCount);
 		checkMinSizeSplit = getOption(options, "checkMinSizeSplit"_sr, checkMinSizeSplit);
@@ -67,20 +67,24 @@ class MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
 		return Void();
 	}
 
-	ACTOR static Future<Void> relocateShardReporter(MockDDTrackerShardEvaluatorWorkload* self,
-	                                                FutureStream<RelocateShard> input) {
-		loop {
+	Future<Void> relocateShardReporter(FutureStream<RelocateShard> input) {
+		while (true) {
+			Error err;
 			try {
-				choose {
-					when(RelocateShard rs = waitNext(input)) {
-						++self->rsReasonCounts[rs.reason];
-					}
+				auto choice = co_await race(input);
+				if (choice.index() == 0) {
+					RelocateShard rs = std::get<0>(std::move(choice));
+
+					++rsReasonCounts[rs.reason];
+				} else {
+					UNREACHABLE();
 				}
 			} catch (Error& e) {
-				if (e.code() != error_code_wrong_shard_server)
-					throw e;
-				wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY));
+				err = e;
 			}
+			if (err.code() != error_code_wrong_shard_server)
+				throw err;
+			co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY);
 		}
 	}
 
@@ -90,14 +94,15 @@ class MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
 
 		// start mock servers
 		actors.add(waitForAll(sharedMgs->runAllMockServers()));
+		actors.add(relocateShardReporter(output.getFuture()));
 
 		// start tracker
 		Reference<InitialDataDistribution> initData =
 		    mock->getInitialDataDistribution(ddcx.id(), ddcx.lock, {}, ddcx.ddEnabledState.get(), SkipDDModeCheck::True)
 		        .get();
-		Reference<PhysicalShardCollection> physicalShardCollection = makeReference<PhysicalShardCollection>();
-		Reference<BulkLoadTaskCollection> bulkLoadTaskCollection = makeReference<BulkLoadTaskCollection>(ddcx.id());
-		Reference<AsyncVar<bool>> zeroHealthyTeams = makeReference<AsyncVar<bool>>(false);
+		auto physicalShardCollection = makeReference<PhysicalShardCollection>();
+		auto bulkLoadTaskCollection = makeReference<BulkLoadTaskCollection>(ddcx.id());
+		auto zeroHealthyTeams = makeReference<AsyncVar<bool>>(false);
 
 		shardTracker = makeReference<DataDistributionTracker>(
 		    DataDistributionTrackerInitParams{ .db = mock,
@@ -120,8 +125,6 @@ class MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload {
 		                                        triggerStorageQueueRebalance.getFuture(),
 		                                        triggerShardBulkLoading.getFuture()));
 
-		actors.add(relocateShardReporter(this, output.getFuture()));
-
 		return timeout(reportErrors(actors.getResult(), "MockDDTrackerShardEvaluatorWorkload"), testDuration, Void());
 	}
 
diff --git a/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp b/fdbserver/workloads/MutationLogReaderCorrectness.cpp
similarity index 80%
rename from fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp
rename to fdbserver/workloads/MutationLogReaderCorrectness.cpp
index 6195ae223c0..77c39801325 100644
--- a/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp
+++ b/fdbserver/workloads/MutationLogReaderCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * MutationLogReaderCorrectness.actor.cpp
+ * MutationLogReaderCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,14 +22,13 @@
 #include <limits>
 #include "fmt/format.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
-#include "fdbclient/MutationLogReader.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/MutationLogReader.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct MutationLogReaderCorrectnessWorkload : TestWorkload {
 	static constexpr auto NAME = "MutationLogReaderCorrectness";
@@ -50,10 +49,10 @@ struct MutationLogReaderCorrectnessWorkload : TestWorkload {
 
 	Value recordValue(int index) {
 		Version v = recordVersion(index);
-		return StringRef(format("%" PRId64 " (%" PRIx64 ")", v, v));
+		return StringRef(format("%lld (%llx)", static_cast<long long>(v), static_cast<long long>(v)));
 	}
 
-	MutationLogReaderCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit MutationLogReaderCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		uid = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
 		baLogRangePrefix = uid.withPrefix(backupLogKeys.begin);
@@ -75,46 +74,47 @@ struct MutationLogReaderCorrectnessWorkload : TestWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, MutationLogReaderCorrectnessWorkload* self) {
-		state Transaction tr(cx);
-		state int iStart = 0;
-		state int batchSize = 1000;
+	Future<Void> _start(Database cx, MutationLogReaderCorrectnessWorkload* self) {
+		Transaction tr(cx);
+		int iStart = 0;
+		int batchSize = 1000;
 		fmt::print("Records: {}\n", self->records);
 		fmt::print("BeginVersion: {}\n", self->beginVersion);
 		fmt::print("EndVersion: {}\n", self->endVersion);
 
 		while (iStart < self->records) {
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr.reset();
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-					int i = iStart;
-					state int iEnd = std::min(iStart + batchSize, self->records);
+					int iEnd = std::min(iStart + batchSize, self->records);
 
-					for (; i < iEnd; ++i) {
+					for (int i = iStart; i < iEnd; ++i) {
 						Key key = self->recordKey(i);
 						Value value = self->recordValue(i);
 						tr.set(key, value);
 					}
 
-					wait(tr.commit());
+					co_await tr.commit();
 					iStart = iEnd;
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 
-		state Reference<MutationLogReader> reader = wait(MutationLogReader::Create(
-		    cx, self->beginVersion, self->endVersion, self->uid, backupLogKeys.begin, /*pipelineDepth=*/1));
+		Reference<MutationLogReader> reader = co_await MutationLogReader::Create(
+		    cx, self->beginVersion, self->endVersion, self->uid, backupLogKeys.begin, /*pipelineDepth=*/1);
 
-		state int nextExpectedRecord = 0;
+		int nextExpectedRecord = 0;
 
 		try {
-			loop {
-				state Standalone<RangeResultRef> results = wait(reader->getNext());
+			while (true) {
+				Standalone<RangeResultRef> results = co_await reader->getNext();
 
 				for (const auto& rec : results) {
 					Key expectedKey = self->recordKey(nextExpectedRecord);
@@ -149,8 +149,6 @@ struct MutationLogReaderCorrectnessWorkload : TestWorkload {
 		printf("records found:    %d\n", nextExpectedRecord);
 
 		ASSERT_EQ(nextExpectedRecord, self->records);
-
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/ParallelRestore.actor.cpp b/fdbserver/workloads/ParallelRestore.actor.cpp
deleted file mode 100644
index 3f0a470baaf..00000000000
--- a/fdbserver/workloads/ParallelRestore.actor.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * ParallelRestore.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbrpc/simulator.h"
-#include "fdbclient/BackupContainer.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbserver/RestoreWorkerInterface.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// A workload which test the correctness of backup and restore process
-struct RunRestoreWorkerWorkload : TestWorkload {
-	static constexpr auto NAME = "RunRestoreWorkerWorkload";
-
-	Future<Void> worker;
-	RunRestoreWorkerWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		TraceEvent("RunRestoreWorkerWorkloadMX").log();
-	}
-
-	Future<Void> setup(Database const& cx) override { return Void(); }
-
-	Future<Void> start(Database const& cx) override {
-		int num_myWorkers = SERVER_KNOBS->FASTRESTORE_NUM_APPLIERS + SERVER_KNOBS->FASTRESTORE_NUM_LOADERS + 1;
-		TraceEvent("RunParallelRestoreWorkerWorkload")
-		    .detail("Start", "RestoreToolDB")
-		    .detail("Workers", num_myWorkers);
-		printf("RunParallelRestoreWorkerWorkload, we will start %d restore workers\n", num_myWorkers);
-		std::vector<Future<Void>> myWorkers;
-		myWorkers.reserve(num_myWorkers);
-		for (int i = 0; i < num_myWorkers; ++i) {
-			myWorkers.push_back(_restoreWorker(cx, LocalityData()));
-		}
-		printf("RunParallelRestoreWorkerWorkload, wait on reply from %ld restore workers\n", myWorkers.size());
-		worker = waitForAll(myWorkers);
-		printf("RunParallelRestoreWorkerWorkload, got all replies from restore workers\n");
-		return Void();
-	}
-
-	Future<bool> check(Database const& cx) override { return true; }
-
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-};
-
-WorkloadFactory<RunRestoreWorkerWorkload> RunRestoreWorkerWorkloadFactory;
diff --git a/fdbserver/workloads/Performance.actor.cpp b/fdbserver/workloads/Performance.cpp
similarity index 79%
rename from fdbserver/workloads/Performance.actor.cpp
rename to fdbserver/workloads/Performance.cpp
index 0bd6e8ed778..44434b300af 100644
--- a/fdbserver/workloads/Performance.actor.cpp
+++ b/fdbserver/workloads/Performance.cpp
@@ -1,5 +1,5 @@
 /*
- * Performance.actor.cpp
+ * Performance.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,11 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/tester/WorkloadUtils.h"
+#include "flow/CoroUtils.h"
 
 // TODO: explain purpose of this workload. Obviously simulation is aimed at correctness,
 // not performance, so a workload literally named Performance has some explaining to do.
@@ -37,12 +38,12 @@ struct PerformanceWorkload : TestWorkload {
 	PerfMetric latencyBaseline, latencySaturation;
 	PerfMetric maxAchievedTPS;
 
-	PerformanceWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit PerformanceWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		probeWorkload = getOption(options, "probeWorkload"_sr, "ReadWrite"_sr);
 
 		// "Consume" all options and save for later tests
 		for (int i = 0; i < options.size(); i++) {
-			if (options[i].value.size()) {
+			if (!options[i].value.empty()) {
 				savedOptions.push_back_deep(savedOptions.arena(), KeyValueRef(options[i].key, options[i].value));
 				printf("saved option (%d): '%s'='%s'\n",
 				       i,
@@ -106,19 +107,19 @@ struct PerformanceWorkload : TestWorkload {
 	}
 
 	// FIXME: does not use testers which are recruited on workers
-	ACTOR Future<std::vector<TesterInterface>> getTesters(PerformanceWorkload* self) {
-		state std::vector<WorkerDetails> workers;
-
-		loop {
-			choose {
-				when(
-				    std::vector<WorkerDetails> w = wait(
-				        brokenPromiseToNever(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest(
-				            GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY))))) {
-					workers = w;
-					break;
-				}
-				when(wait(self->dbInfo->onChange())) {}
+	Future<std::vector<TesterInterface>> getTesters(PerformanceWorkload* self) {
+		std::vector<WorkerDetails> workers;
+
+		while (true) {
+			auto choice = co_await race(
+			    brokenPromiseToNever(self->dbInfo->get().clusterInterface.getWorkers.getReply(GetWorkersRequest(
+			        GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY))),
+			    self->dbInfo->onChange());
+			if (choice.index() == 0) {
+				std::vector<WorkerDetails> w = std::get<0>(std::move(choice));
+
+				workers = w;
+				break;
 			}
 		}
 
@@ -126,22 +127,20 @@ struct PerformanceWorkload : TestWorkload {
 		ts.reserve(workers.size());
 		for (int i = 0; i < workers.size(); i++)
 			ts.push_back(workers[i].interf.testerInterface);
-		return ts;
+		co_return ts;
 	}
 
-	ACTOR Future<Void> _setup(Database cx, PerformanceWorkload* self) {
-		state Standalone<VectorRef<VectorRef<KeyValueRef>>> options = self->getOpts(1000.0);
+	Future<Void> _setup(Database cx, PerformanceWorkload* self) {
+		Standalone<VectorRef<VectorRef<KeyValueRef>>> options = self->getOpts(1000.0);
 		self->logOptions(options);
 
-		std::vector<TesterInterface> testers = wait(self->getTesters(self));
+		std::vector<TesterInterface> testers = co_await self->getTesters(self);
 		self->testers = testers;
 
 		TestSpec spec("PerformanceSetup"_sr, false, false);
 		spec.options = options;
 		spec.phases = TestWorkload::SETUP;
-		DistributedTestResults results = wait(runWorkload(cx, testers, spec));
-
-		return Void();
+		DistributedTestResults results = co_await runWorkload(cx, testers, spec);
 	}
 
 	PerfMetric getNamedMetric(std::string name, std::vector<PerfMetric> metrics) {
@@ -153,13 +152,13 @@ struct PerformanceWorkload : TestWorkload {
 		return PerfMetric();
 	}
 
-	ACTOR Future<Void> getSaturation(Database cx, PerformanceWorkload* self) {
-		state double tps = 400;
-		state bool reported = false;
-		state bool retry = false;
-		state double multiplier = 2.0;
+	Future<Void> getSaturation(Database cx, PerformanceWorkload* self) {
+		double tps = 400;
+		bool reported = false;
+		bool retry = false;
+		double multiplier = 2.0;
 
-		loop {
+		while (true) {
 			Standalone<VectorRef<VectorRef<KeyValueRef>>> options = self->getOpts(tps);
 			TraceEvent start("PerformaceProbeStarting");
 			start.detail("RateTarget", tps);
@@ -169,12 +168,12 @@ struct PerformanceWorkload : TestWorkload {
 					             printable(options[i][j].key) + "=" + printable(options[i][j].value));
 				}
 			}
-			state DistributedTestResults results;
+			DistributedTestResults results;
 			try {
 				TestSpec spec("PerformanceRun"_sr, false, false);
 				spec.phases = TestWorkload::EXECUTION | TestWorkload::METRICS;
 				spec.options = options;
-				DistributedTestResults r = wait(runWorkload(cx, self->testers, spec));
+				DistributedTestResults r = co_await runWorkload(cx, self->testers, spec);
 				results = r;
 			} catch (Error& e) {
 				TraceEvent("PerformanceRunError")
@@ -207,7 +206,7 @@ struct PerformanceWorkload : TestWorkload {
 					retry = true;
 				} else if (multiplier < 2.0) {
 					evt.detail("Saturation", "final");
-					return Void();
+					co_return;
 				} else {
 					tps /= 2;
 					multiplier = 1.189;
@@ -218,16 +217,13 @@ struct PerformanceWorkload : TestWorkload {
 			}
 			tps *= retry ? 1.0 : multiplier;
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, PerformanceWorkload* self) {
-		wait(self->getSaturation(cx, self));
+	Future<Void> _start(Database cx, PerformanceWorkload* self) {
+		co_await self->getSaturation(cx, self);
 		TraceEvent("PerformanceSaturation")
 		    .detail("SaturationRate", self->maxAchievedTPS.value())
 		    .detail("SaturationLatency", self->latencySaturation.value());
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/PerpetualWiggleStatsWorkload.actor.cpp b/fdbserver/workloads/PerpetualWiggleStatsWorkload.actor.cpp
deleted file mode 100644
index 56c6a3a95af..00000000000
--- a/fdbserver/workloads/PerpetualWiggleStatsWorkload.actor.cpp
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * PerpetualWiggleStatsWorkload.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbserver/DDTeamCollection.h"
-#include "fdbclient/FDBOptions.g.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/DDTxnProcessor.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbclient/StorageServerInterface.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbclient/VersionedMap.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// just compare the int part and smoothed duration is enough for the test.
-bool storageWiggleStatsEqual(StorageWiggleMetrics const& a, StorageWiggleMetrics const& b) {
-
-	bool res = a.finished_wiggle == b.finished_wiggle && a.finished_round == b.finished_round &&
-	           std::abs(a.smoothed_wiggle_duration.getTotal() - b.smoothed_wiggle_duration.getTotal()) < 0.0001 &&
-	           std::abs(a.smoothed_round_duration.getTotal() - b.smoothed_round_duration.getTotal()) < 0.0001;
-	if (!res) {
-		std::cout << a.finished_wiggle << " | " << b.finished_wiggle << "\n";
-		std::cout << a.finished_round << " | " << b.finished_round << "\n";
-		std::cout << a.smoothed_wiggle_duration.getTotal() << " | " << b.smoothed_wiggle_duration.getTotal() << "\n";
-		std::cout << a.smoothed_round_duration.getTotal() << " | " << b.smoothed_round_duration.getTotal() << "\n";
-	}
-	return res;
-}
-
-namespace {
-ACTOR Future<bool> IssueConfigurationChange(Database cx, std::string config, bool force) {
-	printf("Issuing configuration change: %s\n", config.c_str());
-	state ConfigurationResult res = wait(ManagementAPI::changeConfig(cx.getReference(), config, force));
-	if (res != ConfigurationResult::SUCCESS) {
-		return false;
-	}
-	wait(delay(5.0)); // wait for read window
-	return true;
-}
-} // namespace
-
-// a wrapper for test protected method
-struct DDTeamCollectionTester : public DDTeamCollection {
-	ActorCollection actors;
-	void resetStorageWiggleState() {
-		wigglingId.reset();
-		storageWiggler = makeReference<StorageWiggler>(this);
-	}
-	ACTOR static Future<Void> testRestoreAndResetStats(DDTeamCollectionTester* self, StorageWiggleMetrics metrics) {
-		std::cout << "testRestoreAndResetStats ...\n";
-		wait(self->storageWiggler->restoreStats());
-		ASSERT(storageWiggleStatsEqual(self->storageWiggler->metrics, metrics));
-		// disable PW
-		{
-			bool success = wait(IssueConfigurationChange(
-			    self->dbContext(), "perpetual_storage_wiggle=0", deterministicRandom()->coinflip()));
-			if (!success)
-				return Void();
-		}
-
-		// restore from FDB
-		metrics.reset();
-		wait(self->storageWiggler->restoreStats());
-		ASSERT(storageWiggleStatsEqual(self->storageWiggler->metrics, metrics));
-		return Void();
-	}
-
-	ACTOR static Future<Void> switchPerpetualWiggleWhenDDIsDead(DDTeamCollectionTester* self,
-	                                                            StorageWiggleMetrics metrics) {
-		std::cout << "switchPerpetualWiggleWhenDDIsDead ...\n";
-		wait(self->storageWiggler->restoreStats());
-		ASSERT(storageWiggleStatsEqual(self->storageWiggler->metrics, metrics));
-		// disable PW
-		{
-			bool success = wait(IssueConfigurationChange(
-			    self->dbContext(), "perpetual_storage_wiggle=0", deterministicRandom()->coinflip()));
-			if (!success)
-				return Void();
-		}
-		// enable PW
-		{
-			bool success = wait(IssueConfigurationChange(
-			    self->dbContext(), "perpetual_storage_wiggle=1", deterministicRandom()->coinflip()));
-			if (!success)
-				return Void();
-		}
-		// restart
-		wait(self->storageWiggler->restoreStats());
-		metrics.reset();
-		ASSERT(storageWiggleStatsEqual(self->storageWiggler->metrics, metrics));
-		return Void();
-	}
-
-	// reset stats shouldn't be overwritten when PW is disabled
-	ACTOR static Future<Void> finishWiggleAfterPWDisabled(DDTeamCollectionTester* self, StorageWiggleMetrics metrics) {
-		std::cout << "finishWiggleAfterPWDisabled ...\n";
-		wait(self->storageWiggler->restoreStats());
-		ASSERT(storageWiggleStatsEqual(self->storageWiggler->metrics, metrics));
-		// disable PW
-		{
-			bool success = wait(IssueConfigurationChange(
-			    self->dbContext(), "perpetual_storage_wiggle=0", deterministicRandom()->coinflip()));
-			if (!success)
-				return Void();
-		}
-		wait(self->storageWiggler->finishWiggle());
-
-		// restart perpetual wiggle
-		{
-			bool success = wait(IssueConfigurationChange(
-			    self->dbContext(), "perpetual_storage_wiggle=1", deterministicRandom()->coinflip()));
-			if (!success)
-				return Void();
-		}
-		wait(self->storageWiggler->restoreStats());
-		metrics.reset();
-		ASSERT(storageWiggleStatsEqual(self->storageWiggler->metrics, metrics));
-		return Void();
-	}
-
-	DDTeamCollectionTester(DDTeamCollectionInitParams const& params) : DDTeamCollection(params) {}
-};
-
-StorageWiggleMetrics getRandomWiggleMetrics() {
-	StorageWiggleMetrics res;
-	res.smoothed_round_duration.reset(deterministicRandom()->randomUInt32());
-	res.smoothed_wiggle_duration.reset(deterministicRandom()->randomUInt32());
-	res.finished_round = deterministicRandom()->randomUInt32();
-	res.finished_wiggle = deterministicRandom()->randomUInt32();
-	return res;
-}
-
-struct PerpetualWiggleStatsWorkload : public TestWorkload {
-
-	static constexpr auto NAME = "PerpetualWiggleStatsWorkload";
-	StorageWiggleMetrics lastMetrics;
-
-	PerpetualWiggleStatsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
-
-	ACTOR static Future<Void> _setup(PerpetualWiggleStatsWorkload* self, Database cx) {
-		wait(success(setDDMode(cx, 0)));
-		wait(success(takeMoveKeysLock(cx, UID()))); // force current DD to quit
-		bool success = wait(IssueConfigurationChange(cx, "storage_migration_type=disabled", true));
-		ASSERT(success);
-		wait(delay(30.0)); // make sure the DD has already quit before the test start
-		return Void();
-	}
-
-	ACTOR static Future<Void> prepareTestEnv(PerpetualWiggleStatsWorkload* self, Database cx) {
-		// enable perpetual wiggle
-		bool change = wait(IssueConfigurationChange(cx, "perpetual_storage_wiggle=1", true));
-		ASSERT(change);
-		// update wiggle metrics
-		self->lastMetrics = getRandomWiggleMetrics();
-		auto& lastMetrics = self->lastMetrics;
-		wait(success(runRYWTransaction(cx, [&lastMetrics](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
-			StorageWiggleData wiggleData;
-			return wiggleData.updateStorageWiggleMetrics(tr, lastMetrics, PrimaryRegion(true));
-		})));
-		return Void();
-	}
-
-	Future<Void> setup(Database const& cx) override {
-		if (clientId == 0) {
-			return _setup(this, cx); // force to disable DD
-		}
-		return Void();
-	}
-
-	Future<Void> start(Database const& cx) override {
-		if (clientId == 0) {
-			return _start(this, cx);
-		}
-		return Void();
-	};
-
-	Future<bool> check(Database const& cx) override { return true; };
-
-	ACTOR static Future<Void> _start(PerpetualWiggleStatsWorkload* self, Database cx) {
-		state Reference<IDDTxnProcessor> db = makeReference<DDTxnProcessor>(cx);
-		state DatabaseConfiguration conf;
-		state DDTeamCollectionTester tester({ db,
-		                                      UID(8, 6),
-		                                      MoveKeysLock(),
-		                                      PromiseStream<RelocateShard>(),
-		                                      makeReference<ShardsAffectedByTeamFailure>(),
-		                                      conf,
-		                                      {},
-		                                      {},
-		                                      Future<Void>(Void()),
-		                                      makeReference<AsyncVar<bool>>(true),
-		                                      IsPrimary::True,
-		                                      makeReference<AsyncVar<bool>>(false),
-		                                      makeReference<AsyncVar<bool>>(false),
-		                                      PromiseStream<GetMetricsRequest>(),
-		                                      Promise<UID>(),
-		                                      PromiseStream<Promise<int>>(),
-		                                      PromiseStream<Promise<int64_t>>(),
-		                                      PromiseStream<RebalanceStorageQueueRequest>(),
-		                                      makeReference<BulkLoadTaskCollection>(UID(8, 6)) });
-		tester.configuration.storageTeamSize = 3;
-		tester.configuration.perpetualStorageWiggleSpeed = 1;
-
-		wait(prepareTestEnv(self, cx));
-		tester.resetStorageWiggleState();
-		wait(DDTeamCollectionTester::testRestoreAndResetStats(&tester, self->lastMetrics));
-
-		wait(prepareTestEnv(self, cx));
-		tester.resetStorageWiggleState();
-		wait(DDTeamCollectionTester::switchPerpetualWiggleWhenDDIsDead(&tester, self->lastMetrics));
-
-		wait(prepareTestEnv(self, cx));
-		tester.resetStorageWiggleState();
-		wait(DDTeamCollectionTester::finishWiggleAfterPWDisabled(&tester, self->lastMetrics));
-
-		wait(success(setDDMode(cx, 1)));
-		return Void();
-	}
-
-	void getMetrics(std::vector<PerfMetric>& m) override { return; }
-};
-
-WorkloadFactory<PerpetualWiggleStatsWorkload> PerpetualWiggleStatsWorkload;
\ No newline at end of file
diff --git a/fdbserver/workloads/PerpetualWiggleStatsWorkload.cpp b/fdbserver/workloads/PerpetualWiggleStatsWorkload.cpp
new file mode 100644
index 00000000000..fc694c01a3b
--- /dev/null
+++ b/fdbserver/workloads/PerpetualWiggleStatsWorkload.cpp
@@ -0,0 +1,233 @@
+/*
+ * PerpetualWiggleStatsWorkload.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/datadistributor/DDTeamCollection.h"
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbclient/VersionedMap.h"
+#include "fdbclient/ReadYourWrites.h"
+
+// just compare the int part and smoothed duration is enough for the test.
+bool storageWiggleStatsEqual(StorageWiggleMetrics const& a, StorageWiggleMetrics const& b) {
+
+	bool res = a.finished_wiggle == b.finished_wiggle && a.finished_round == b.finished_round &&
+	           std::abs(a.smoothed_wiggle_duration.getTotal() - b.smoothed_wiggle_duration.getTotal()) < 0.0001 &&
+	           std::abs(a.smoothed_round_duration.getTotal() - b.smoothed_round_duration.getTotal()) < 0.0001;
+	if (!res) {
+		std::cout << a.finished_wiggle << " | " << b.finished_wiggle << "\n";
+		std::cout << a.finished_round << " | " << b.finished_round << "\n";
+		std::cout << a.smoothed_wiggle_duration.getTotal() << " | " << b.smoothed_wiggle_duration.getTotal() << "\n";
+		std::cout << a.smoothed_round_duration.getTotal() << " | " << b.smoothed_round_duration.getTotal() << "\n";
+	}
+	return res;
+}
+
+namespace {
+Future<bool> IssueConfigurationChange(Database cx, std::string config, bool force) {
+	printf("Issuing configuration change: %s\n", config.c_str());
+	ConfigurationResult res = co_await ManagementAPI::changeConfig(cx.getReference(), config, force);
+	if (res != ConfigurationResult::SUCCESS) {
+		co_return false;
+	}
+	co_await delay(5.0); // wait for read window
+	co_return true;
+}
+} // namespace
+
+// a wrapper for test protected method
+struct DDTeamCollectionTester : public DDTeamCollection {
+	ActorCollection actors;
+	void resetStorageWiggleState() {
+		wigglingId.reset();
+		storageWiggler = makeReference<StorageWiggler>(this);
+	}
+	Future<Void> testRestoreAndResetStats(StorageWiggleMetrics metrics) {
+		std::cout << "testRestoreAndResetStats ...\n";
+		co_await storageWiggler->restoreStats();
+		ASSERT(storageWiggleStatsEqual(storageWiggler->metrics, metrics));
+		// disable PW
+		{
+			bool success = co_await IssueConfigurationChange(
+			    dbContext(), "perpetual_storage_wiggle=0", deterministicRandom()->coinflip());
+			if (!success)
+				co_return;
+		}
+
+		// restore from FDB
+		metrics.reset();
+		co_await storageWiggler->restoreStats();
+		ASSERT(storageWiggleStatsEqual(storageWiggler->metrics, metrics));
+	}
+
+	Future<Void> switchPerpetualWiggleWhenDDIsDead(StorageWiggleMetrics metrics) {
+		std::cout << "switchPerpetualWiggleWhenDDIsDead ...\n";
+		co_await storageWiggler->restoreStats();
+		ASSERT(storageWiggleStatsEqual(storageWiggler->metrics, metrics));
+		// disable PW
+		{
+			bool success = co_await IssueConfigurationChange(
+			    dbContext(), "perpetual_storage_wiggle=0", deterministicRandom()->coinflip());
+			if (!success)
+				co_return;
+		}
+		// enable PW
+		{
+			bool success = co_await IssueConfigurationChange(
+			    dbContext(), "perpetual_storage_wiggle=1", deterministicRandom()->coinflip());
+			if (!success)
+				co_return;
+		}
+		// restart
+		co_await storageWiggler->restoreStats();
+		metrics.reset();
+		ASSERT(storageWiggleStatsEqual(storageWiggler->metrics, metrics));
+	}
+
+	// reset stats shouldn't be overwritten when PW is disabled
+	Future<Void> finishWiggleAfterPWDisabled(StorageWiggleMetrics metrics) {
+		std::cout << "finishWiggleAfterPWDisabled ...\n";
+		co_await storageWiggler->restoreStats();
+		ASSERT(storageWiggleStatsEqual(storageWiggler->metrics, metrics));
+		// disable PW
+		{
+			bool success = co_await IssueConfigurationChange(
+			    dbContext(), "perpetual_storage_wiggle=0", deterministicRandom()->coinflip());
+			if (!success)
+				co_return;
+		}
+		co_await storageWiggler->finishWiggle();
+
+		// restart perpetual wiggle
+		{
+			bool success = co_await IssueConfigurationChange(
+			    dbContext(), "perpetual_storage_wiggle=1", deterministicRandom()->coinflip());
+			if (!success)
+				co_return;
+		}
+		co_await storageWiggler->restoreStats();
+		metrics.reset();
+		ASSERT(storageWiggleStatsEqual(storageWiggler->metrics, metrics));
+	}
+
+	explicit DDTeamCollectionTester(DDTeamCollectionInitParams const& params) : DDTeamCollection(params) {}
+};
+
+StorageWiggleMetrics getRandomWiggleMetrics() {
+	StorageWiggleMetrics res;
+	res.smoothed_round_duration.reset(deterministicRandom()->randomUInt32());
+	res.smoothed_wiggle_duration.reset(deterministicRandom()->randomUInt32());
+	res.finished_round = deterministicRandom()->randomUInt32();
+	res.finished_wiggle = deterministicRandom()->randomUInt32();
+	return res;
+}
+
+struct PerpetualWiggleStatsWorkload : public TestWorkload {
+
+	static constexpr auto NAME = "PerpetualWiggleStatsWorkload";
+	StorageWiggleMetrics lastMetrics;
+
+	explicit PerpetualWiggleStatsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+	Future<Void> _setup(Database cx) {
+		co_await setDDMode(cx, 0);
+		co_await takeMoveKeysLock(cx, UID()); // force current DD to quit
+		bool success = co_await IssueConfigurationChange(cx, "storage_migration_type=disabled", true);
+		ASSERT(success);
+		co_await delay(30.0); // make sure the DD has already quit before the test start
+	}
+
+	Future<Void> prepareTestEnv(Database cx) {
+		// enable perpetual wiggle
+		bool change = co_await IssueConfigurationChange(cx, "perpetual_storage_wiggle=1", true);
+		ASSERT(change);
+		// update wiggle metrics
+		lastMetrics = getRandomWiggleMetrics();
+		auto& lastMetrics = this->lastMetrics;
+		co_await runRYWTransaction(cx, [&lastMetrics](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+			StorageWiggleData wiggleData;
+			return wiggleData.updateStorageWiggleMetrics(tr, lastMetrics, PrimaryRegion(true));
+		});
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0) {
+			return _setup(cx); // force to disable DD
+		}
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0) {
+			return _start(cx);
+		}
+		return Void();
+	};
+
+	Future<bool> check(Database const& cx) override { return true; };
+
+	Future<Void> _start(Database cx) {
+		Reference<IDDTxnProcessor> db = makeReference<DDTxnProcessor>(cx);
+		DatabaseConfiguration conf;
+		DDTeamCollectionTester tester({ db,
+		                                UID(8, 6),
+		                                MoveKeysLock(),
+		                                PromiseStream<RelocateShard>(),
+		                                makeReference<ShardsAffectedByTeamFailure>(),
+		                                conf,
+		                                {},
+		                                {},
+		                                Future<Void>(Void()),
+		                                makeReference<AsyncVar<bool>>(true),
+		                                IsPrimary::True,
+		                                makeReference<AsyncVar<bool>>(false),
+		                                makeReference<AsyncVar<bool>>(false),
+		                                PromiseStream<GetMetricsRequest>(),
+		                                Promise<UID>(),
+		                                PromiseStream<Promise<int>>(),
+		                                PromiseStream<Promise<int64_t>>(),
+		                                PromiseStream<RebalanceStorageQueueRequest>(),
+		                                makeReference<BulkLoadTaskCollection>(UID(8, 6)) });
+		tester.configuration.storageTeamSize = 3;
+		tester.configuration.perpetualStorageWiggleSpeed = 1;
+
+		co_await prepareTestEnv(cx);
+		tester.resetStorageWiggleState();
+		co_await tester.testRestoreAndResetStats(lastMetrics);
+
+		co_await prepareTestEnv(cx);
+		tester.resetStorageWiggleState();
+		co_await tester.switchPerpetualWiggleWhenDDIsDead(lastMetrics);
+
+		co_await prepareTestEnv(cx);
+		tester.resetStorageWiggleState();
+		co_await tester.finishWiggleAfterPWDisabled(lastMetrics);
+
+		co_await setDDMode(cx, 1);
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override { return; }
+};
+
+WorkloadFactory<PerpetualWiggleStatsWorkload> PerpetualWiggleStatsWorkload;
diff --git a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.cpp
similarity index 67%
rename from fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
rename to fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.cpp
index df964170256..1d6f4cd70b3 100644
--- a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
+++ b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * PerpetualWiggleStorageMigrationWorkload.actor.cpp
+ * PerpetualWiggleStorageMigrationWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,29 +18,27 @@
  * limitations under the License.
  */
 
-#include "fdbserver/DDTeamCollection.h"
+#include "fdbserver/datadistributor/DDTeamCollection.h"
 #include "fdbclient/FDBOptions.g.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/DDSharedContext.h"
-#include "fdbserver/DDTxnProcessor.h"
-#include "fdbserver/MoveKeys.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/datadistributor/DDSharedContext.h"
+#include "fdbserver/datadistributor/DDTxnProcessor.h"
+#include "fdbserver/core/MoveKeys.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbclient/VersionedMap.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
-ACTOR Future<bool> IssueConfigurationChange(Database cx, std::string config, bool force) {
+Future<bool> IssueConfigurationChange(Database cx, std::string config, bool force) {
 	printf("Issuing configuration change: %s\n", config.c_str());
-	state ConfigurationResult res = wait(ManagementAPI::changeConfig(cx.getReference(), config, force));
+	ConfigurationResult res = co_await ManagementAPI::changeConfig(cx.getReference(), config, force);
 	if (res != ConfigurationResult::SUCCESS) {
-		return false;
+		co_return false;
 	}
-	wait(delay(5.0)); // wait for read window
-	return true;
+	co_await delay(5.0); // wait for read window
+	co_return true;
 }
 
 constexpr bool hasRocksDB =
@@ -57,7 +55,7 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 
 	static constexpr auto NAME = "PerpetualWiggleStorageMigrationWorkload";
 
-	PerpetualWiggleStorageMigrationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+	explicit PerpetualWiggleStorageMigrationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
 	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
 		// This test requires exclude/include runs smoothly, so we disable all the failure injection workloads.
@@ -66,27 +64,27 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0) {
-			return _start(this, cx);
+			return startImpl(cx);
 		}
 		return Void();
 	};
 
 	Future<bool> check(Database const& cx) override { return true; };
 
-	ACTOR static Future<Void> _start(PerpetualWiggleStorageMigrationWorkload* self, Database cx) {
+	Future<Void> startImpl(Database cx) {
 		if (!hasRocksDB) {
 			// RocksDB, which is required by this test, is not supported
-			return Void();
+			co_return;
 		}
-		state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+		std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
 		// The test should have enough storage servers to exclude.
 		ASSERT(storageServers.size() > 3);
 
 		// Pick a storage process to exclude and later include. This process should always use storage engine from
 		// `storage_engine` configuration.
-		state StorageServerInterface ssToExcludeInclude =
+		StorageServerInterface ssToExcludeInclude =
 		    storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
-		state ISimulator::ProcessInfo* p = g_simulator->getProcessByAddress(ssToExcludeInclude.address());
+		ISimulator::ProcessInfo* p = g_simulator->getProcessByAddress(ssToExcludeInclude.address());
 		while (!p->isReliable()) {
 			ssToExcludeInclude = storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
 			p = g_simulator->getProcessByAddress(ssToExcludeInclude.address());
@@ -97,8 +95,7 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 		    .detail("Address", ssToExcludeInclude.address());
 
 		// Pick a storage process to migrate to storage engine specified in `perpetual_storage_wiggle_engine`.
-		state StorageServerInterface ssToWiggle =
-		    storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
+		StorageServerInterface ssToWiggle = storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
 		while (ssToExcludeInclude.locality.processId() == ssToWiggle.locality.processId()) {
 			ssToWiggle = storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
 		}
@@ -109,47 +106,46 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 		// Issue a configuration change to ONLY migrate `ssToWiggle` using perpetual wiggle.
 		std::string migrationLocality =
 		    LocalityData::keyProcessId.toString() + ":" + ssToWiggle.locality.processId()->toString();
-		bool change =
-		    wait(IssueConfigurationChange(cx,
-		                                  "perpetual_storage_wiggle_engine=ssd-rocksdb-v1 perpetual_storage_wiggle=1 "
-		                                  "storage_migration_type=gradual perpetual_storage_wiggle_locality=" +
-		                                      migrationLocality,
-		                                  true));
+		bool change = co_await IssueConfigurationChange(
+		    cx,
+		    "perpetual_storage_wiggle_engine=ssd-rocksdb-v1 perpetual_storage_wiggle=1 "
+		    "storage_migration_type=gradual perpetual_storage_wiggle_locality=" +
+		        migrationLocality,
+		    true);
 		TraceEvent("Test_ConfigChangeDone").detail("Success", change);
 		ASSERT(change);
 
-		wait(excludeIncludeServer(cx, ssToExcludeInclude));
+		co_await excludeIncludeServer(cx, ssToExcludeInclude);
 
-		wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, /*wiggleStorageType=*/"ssd-rocksdb-v1"));
+		co_await validateDatabase(cx, ssToExcludeInclude, ssToWiggle, /*wiggleStorageType=*/"ssd-rocksdb-v1");
 
 		// We probablistically validate that resetting perpetual_storage_wiggle_engine to none works as expected.
 		if (deterministicRandom()->coinflip()) {
 			TraceEvent("Test_ClearPerpetualStorageWiggleEngine").log();
-			bool change = wait(IssueConfigurationChange(cx, "perpetual_storage_wiggle_engine=none", true));
+			bool change = co_await IssueConfigurationChange(cx, "perpetual_storage_wiggle_engine=none", true);
 			TraceEvent("Test_ClearPerpetualStorageWiggleEngineDone").detail("Success", change);
 			ASSERT(change);
 
 			// Next, we run exclude and then include `ssToWiggle`. Because perpetual_storage_wiggle_engine is set to
 			// none, the engine for `ssToWiggle` should be `storage_engine`.
-			wait(excludeIncludeServer(cx, ssToWiggle));
-			wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, /*wiggleStorageType=*/"ssd-2"));
+			co_await excludeIncludeServer(cx, ssToWiggle);
+			co_await validateDatabase(cx, ssToExcludeInclude, ssToWiggle, /*wiggleStorageType=*/"ssd-2");
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> excludeIncludeServer(Database cx, StorageServerInterface ssToExcludeInclude) {
+	static Future<Void> excludeIncludeServer(Database cx, StorageServerInterface ssToExcludeInclude) {
 		// Now, let's exclude `ssToExcludeInclude` process and include it again. The new SS created on this process
 		// should always uses `storage_engine` config, which is `ssd-2`.
-		state std::vector<AddressExclusion> servers;
+		std::vector<AddressExclusion> servers;
 		servers.push_back(AddressExclusion(ssToExcludeInclude.address().ip, ssToExcludeInclude.address().port));
 
 		// Since we have enough storage servers and there won't be any failure, let's use exclude failed to make sure
 		// the exclude process can succeed.
-		wait(excludeServers(cx, servers, true));
+		co_await excludeServers(cx, servers, true);
 		TraceEvent("Test_DoneExcludeServer").log();
 
 		try {
-			std::set<NetworkAddress> inProgress = wait(checkForExcludingServers(cx, servers, true));
+			std::set<NetworkAddress> inProgress = co_await checkForExcludingServers(cx, servers, true);
 			ASSERT(inProgress.empty());
 		} catch (Error& e) {
 			if (e.code() == error_code_timed_out) {
@@ -162,20 +158,18 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 		TraceEvent("Test_CheckingExcludeServerDone").log();
 
 		// Include all the processes the cluster knows.
-		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
+		co_await includeServers(cx, std::vector<AddressExclusion>(1));
 		TraceEvent("Test_IncludeServerDone").log();
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> validateDatabase(Database cx,
-	                                           StorageServerInterface ssToExcludeInclude,
-	                                           StorageServerInterface ssToWiggle,
-	                                           std::string wiggleStorageType) {
+	static Future<Void> validateDatabase(Database cx,
+	                                     StorageServerInterface ssToExcludeInclude,
+	                                     StorageServerInterface ssToWiggle,
+	                                     std::string wiggleStorageType) {
 		// Wait until `ssToExcludeInclude` to be recruited as storage server again.
-		state int missingTargetCount = 0;
-		loop {
-			std::vector<StorageServerInterface> allStorageServers = wait(getStorageServers(cx));
+		int missingTargetCount = 0;
+		while (true) {
+			std::vector<StorageServerInterface> allStorageServers = co_await getStorageServers(cx);
 			bool foundTarget = false;
 			for (auto& ss : allStorageServers) {
 				if (ss.address() == ssToExcludeInclude.address()) {
@@ -192,26 +186,26 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 				// the process class). So we don't wait indefinitely here.
 				break;
 			}
-			wait(delay(20));
+			co_await delay(20);
 		}
 
 		// Wait until wiggle process to migrate to new storage engine.
-		state int missingWiggleStorageCount = 0;
-		state std::vector<StorageServerInterface> allSSes;
-		state bool doneCheckingWiggleStorage = false;
-		state bool containWiggleStorage = false;
-		loop {
-			std::vector<StorageServerInterface> SSes = wait(getStorageServers(cx));
+		int missingWiggleStorageCount = 0;
+		std::vector<StorageServerInterface> allSSes;
+		bool doneCheckingWiggleStorage = false;
+		bool containWiggleStorage = false;
+		while (true) {
+			std::vector<StorageServerInterface> SSes = co_await getStorageServers(cx);
 			allSSes = SSes;
 
-			state int i = 0;
+			int i = 0;
 			containWiggleStorage = false;
 			doneCheckingWiggleStorage = false;
 			for (i = 0; i < allSSes.size(); ++i) {
-				state StorageServerInterface ssInterface = allSSes[i];
-				state ReplyPromise<KeyValueStoreType> typeReply;
+				StorageServerInterface ssInterface = allSSes[i];
+				ReplyPromise<KeyValueStoreType> typeReply;
 				ErrorOr<KeyValueStoreType> keyValueStoreType =
-				    wait(ssInterface.getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0));
+				    co_await ssInterface.getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0);
 				if (keyValueStoreType.present()) {
 					TraceEvent(SevDebug, "Test_KvStorageType")
 					    .detail("StorageServer", ssInterface.address())
@@ -245,7 +239,7 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 					break;
 				}
 			}
-			wait(delay(20));
+			co_await delay(20);
 		}
 
 		if (!doneCheckingWiggleStorage) {
@@ -254,7 +248,6 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 			// storage engine in the last check.
 			ASSERT(!containWiggleStorage);
 		}
-		return Void();
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override { return; }
diff --git a/fdbserver/workloads/PhysicalShardMove.actor.cpp b/fdbserver/workloads/PhysicalShardMove.cpp
similarity index 58%
rename from fdbserver/workloads/PhysicalShardMove.actor.cpp
rename to fdbserver/workloads/PhysicalShardMove.cpp
index 9e0eaa7a2ad..7f549d4e347 100644
--- a/fdbserver/workloads/PhysicalShardMove.actor.cpp
+++ b/fdbserver/workloads/PhysicalShardMove.cpp
@@ -18,23 +18,21 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/ServerCheckpoint.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/ServerCheckpoint.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
 #include <cstdint>
 #include <limits>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
 std::string printValue(const ErrorOr<Optional<Value>>& value) {
 	if (value.isError()) {
@@ -53,7 +51,8 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 	const bool enabled;
 	bool pass;
 
-	PhysicalShardMoveWorkLoad(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
+	explicit PhysicalShardMoveWorkLoad(WorkloadContext const& wcx)
+	  : TestWorkload(wcx), enabled(!clientId), pass(true) {}
 
 	void validationFailed(ErrorOr<Optional<Value>> expectedValue, ErrorOr<Optional<Value>> actualValue) {
 		TraceEvent(SevError, "TestFailed")
@@ -68,7 +67,7 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		if (!enabled) {
 			return Void();
 		}
-		return _start(this, cx);
+		return _start(cx);
 	}
 
 	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override {
@@ -76,79 +75,77 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		out.insert("Attrition");
 	}
 
-	ACTOR Future<Void> _start(PhysicalShardMoveWorkLoad* self, Database cx) {
-		wait(success(setDDMode(cx, 0)));
-		state std::vector<UID> teamA;
-		state std::map<Key, Value> kvs({ { "TestKeyA"_sr, "TestValueA"_sr },
-		                                 { "TestKeyAB"_sr, "TestValueAB"_sr },
-		                                 { "TestKeyAD"_sr, "TestValueAD"_sr },
-		                                 { "TestKeyB"_sr, "TestValueB"_sr },
-		                                 { "TestKeyBA"_sr, "TestValueBA"_sr },
-		                                 { "TestKeyC"_sr, "TestValueC"_sr },
-		                                 { "TestKeyD"_sr, "TestValueD"_sr },
-		                                 { "TestKeyE"_sr, "TestValueE"_sr },
-		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
-
-		wait(success(self->populateData(self, cx, &kvs)));
+	Future<Void> _start(Database cx) {
+		co_await setDDMode(cx, 0);
+		std::vector<UID> teamA;
+		std::map<Key, Value> kvs({ { "TestKeyA"_sr, "TestValueA"_sr },
+		                           { "TestKeyAB"_sr, "TestValueAB"_sr },
+		                           { "TestKeyAD"_sr, "TestValueAD"_sr },
+		                           { "TestKeyB"_sr, "TestValueB"_sr },
+		                           { "TestKeyBA"_sr, "TestValueBA"_sr },
+		                           { "TestKeyC"_sr, "TestValueC"_sr },
+		                           { "TestKeyD"_sr, "TestValueD"_sr },
+		                           { "TestKeyE"_sr, "TestValueE"_sr },
+		                           { "TestKeyF"_sr, "TestValueF"_sr } });
+
+		co_await populateData(this, cx, &kvs);
 
 		TraceEvent("TestValueWritten").log();
 
-		state std::unordered_set<UID> excludes;
-		state std::unordered_set<UID> includes;
-		state int teamSize = 1;
-		state DataMovementReason dataMoveReason = static_cast<DataMovementReason>(
+		std::unordered_set<UID> excludes;
+		std::unordered_set<UID> includes;
+		int teamSize = 1;
+		DataMovementReason dataMoveReason = static_cast<DataMovementReason>(
 		    deterministicRandom()->randomInt(1, static_cast<int>(DataMovementReason::NUMBER_OF_REASONS)));
-		state KeyRangeRef currentRange = KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr);
-		wait(store(teamA,
-		           self->moveShard(self,
-		                           cx,
-		                           newDataMoveId(deterministicRandom()->randomUInt64(),
-		                                         AssignEmptyRange::False,
-		                                         deterministicRandom()->random01() <
-		                                                 SERVER_KNOBS->DD_PHYSICAL_SHARD_MOVE_PROBABILITY
-		                                             ? DataMoveType::PHYSICAL
-		                                             : DataMoveType::LOGICAL,
-		                                         dataMoveReason),
-		                           currentRange,
-		                           teamSize,
-		                           includes,
-		                           excludes)));
+		KeyRangeRef currentRange = KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr);
+		teamA = co_await moveShard(
+		    this,
+		    cx,
+		    newDataMoveId(deterministicRandom()->randomUInt64(),
+		                  AssignEmptyRange::False,
+		                  deterministicRandom()->random01() < SERVER_KNOBS->DD_PHYSICAL_SHARD_MOVE_PROBABILITY
+		                      ? DataMoveType::PHYSICAL
+		                      : DataMoveType::LOGICAL,
+		                  dataMoveReason),
+		    currentRange,
+		    teamSize,
+		    includes,
+		    excludes);
 		TraceEvent(SevDebug, "TestMovedRange1").detail("Range", currentRange).detail("Team", describe(teamA));
 
 		excludes.insert(teamA.begin(), teamA.end());
 
-		state uint64_t sh0 = deterministicRandom()->randomUInt64();
-		state uint64_t sh1 = deterministicRandom()->randomUInt64();
-		state uint64_t sh2 = deterministicRandom()->randomUInt64();
+		uint64_t sh0 = deterministicRandom()->randomUInt64();
+		uint64_t sh1 = deterministicRandom()->randomUInt64();
+		uint64_t sh2 = deterministicRandom()->randomUInt64();
 
 		// Move range [TestKeyA, TestKeyB) to sh0.
 		currentRange = KeyRangeRef("TestKeyA"_sr, "TestKeyB"_sr);
-		wait(store(teamA,
-		           self->moveShard(self,
-		                           cx,
-		                           newDataMoveId(sh0,
-		                                         AssignEmptyRange::False,
-		                                         deterministicRandom()->random01() <
-		                                                 SERVER_KNOBS->DD_PHYSICAL_SHARD_MOVE_PROBABILITY
-		                                             ? DataMoveType::PHYSICAL
-		                                             : DataMoveType::LOGICAL,
-		                                         dataMoveReason),
-		                           currentRange,
-		                           teamSize,
-		                           includes,
-		                           excludes)));
+		teamA = co_await moveShard(
+		    this,
+		    cx,
+		    newDataMoveId(sh0,
+		                  AssignEmptyRange::False,
+		                  deterministicRandom()->random01() < SERVER_KNOBS->DD_PHYSICAL_SHARD_MOVE_PROBABILITY
+		                      ? DataMoveType::PHYSICAL
+		                      : DataMoveType::LOGICAL,
+		                  dataMoveReason),
+		    currentRange,
+		    teamSize,
+		    includes,
+		    excludes);
 		TraceEvent(SevDebug, "TestMovedRange2").detail("Range", currentRange).detail("Team", describe(teamA));
 
-		state std::vector<KeyRange> checkpointRanges;
+		std::vector<KeyRange> checkpointRanges;
 		checkpointRanges.push_back(KeyRangeRef("TestKeyA"_sr, "TestKeyAC"_sr));
-		wait(self->checkpointRestore(self, cx, checkpointRanges, checkpointRanges, CheckpointAsKeyValues::True, &kvs));
+		co_await checkpointRestore(this, cx, checkpointRanges, checkpointRanges, CheckpointAsKeyValues::True, &kvs);
 		TraceEvent(SevDebug, "TestCheckpointRestored1");
 
 		// Move range [TestKeyD, TestKeyF) to sh0;
 		includes.insert(teamA.begin(), teamA.end());
 		currentRange = KeyRangeRef("TestKeyD"_sr, "TestKeyF"_sr);
-		state std::vector<UID> teamE = wait(self->moveShard(
-		    self,
+		std::vector<UID> teamE = co_await moveShard(
+		    this,
 		    cx,
 		    newDataMoveId(sh0,
 		                  AssignEmptyRange::False,
@@ -159,16 +156,16 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		    currentRange,
 		    teamSize,
 		    includes,
-		    excludes));
+		    excludes);
 		TraceEvent(SevDebug, "TestMovedRange3").detail("Range", currentRange).detail("Team", describe(teamE));
 		ASSERT(std::equal(teamA.begin(), teamA.end(), teamE.begin()));
 
-		state int teamIdx = 0;
+		int teamIdx = 0;
 		for (teamIdx = 0; teamIdx < teamA.size(); ++teamIdx) {
 			TraceEvent("TestGettingServerShards", teamA[teamIdx])
 			    .detail("Range", KeyRangeRef("TestKeyD"_sr, "TestKeyF"_sr));
 			std::vector<StorageServerShard> shards =
-			    wait(self->getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyD"_sr, "TestKeyF"_sr)));
+			    co_await getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyD"_sr, "TestKeyF"_sr));
 			ASSERT(shards.size() == 1);
 			ASSERT(shards[0].desiredId == sh0);
 			ASSERT(shards[0].id == sh0);
@@ -178,14 +175,14 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		checkpointRanges.clear();
 		checkpointRanges.push_back(KeyRangeRef("TestKeyA"_sr, "TestKeyB"_sr));
 		checkpointRanges.push_back(KeyRangeRef("TestKeyD"_sr, "TestKeyE"_sr));
-		wait(self->checkpointRestore(self, cx, checkpointRanges, checkpointRanges, CheckpointAsKeyValues::True, &kvs));
+		co_await checkpointRestore(this, cx, checkpointRanges, checkpointRanges, CheckpointAsKeyValues::True, &kvs);
 		TraceEvent(SevDebug, "TestCheckpointRestored2");
 
 		// Move range [TestKeyB, TestKeyC) to sh1, on the same server.
 		currentRange = KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr);
 		includes.insert(teamA.begin(), teamA.end());
-		state std::vector<UID> teamB = wait(self->moveShard(
-		    self,
+		std::vector<UID> teamB = co_await moveShard(
+		    this,
 		    cx,
 		    newDataMoveId(sh1,
 		                  AssignEmptyRange::False,
@@ -196,14 +193,14 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		    currentRange,
 		    teamSize,
 		    includes,
-		    excludes));
+		    excludes);
 		TraceEvent(SevDebug, "TestMovedRange4").detail("Range", currentRange).detail("Team", describe(teamB));
 		ASSERT(std::equal(teamA.begin(), teamA.end(), teamB.begin()));
 
 		teamIdx = 0;
 		for (teamIdx = 0; teamIdx < teamA.size(); ++teamIdx) {
 			std::vector<StorageServerShard> shards =
-			    wait(self->getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr)));
+			    co_await getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr));
 			TraceEvent("TestStorageServerShards", teamA[teamIdx]).detail("Shards", describe(shards));
 			ASSERT(shards.size() == 2);
 			ASSERT(shards[0].desiredId == sh0);
@@ -216,12 +213,12 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		std::vector<KeyRange> restoreRanges;
 		restoreRanges.push_back(KeyRangeRef("TestKeyA"_sr, "TestKeyB"_sr));
 		restoreRanges.push_back(KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr));
-		wait(self->checkpointRestore(self, cx, checkpointRanges, restoreRanges, CheckpointAsKeyValues::True, &kvs));
+		co_await checkpointRestore(this, cx, checkpointRanges, restoreRanges, CheckpointAsKeyValues::True, &kvs);
 		TraceEvent(SevDebug, "TestCheckpointRestored3");
 
 		currentRange = KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr);
-		state std::vector<UID> teamC = wait(self->moveShard(
-		    self,
+		std::vector<UID> teamC = co_await moveShard(
+		    this,
 		    cx,
 		    newDataMoveId(sh2,
 		                  AssignEmptyRange::False,
@@ -232,13 +229,13 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		    currentRange,
 		    teamSize,
 		    includes,
-		    excludes));
+		    excludes);
 		TraceEvent(SevDebug, "TestMovedRange5").detail("Range", currentRange).detail("Team", describe(teamC));
 		ASSERT(std::equal(teamA.begin(), teamA.end(), teamC.begin()));
 
 		for (teamIdx = 0; teamIdx < teamA.size(); ++teamIdx) {
 			std::vector<StorageServerShard> shards =
-			    wait(self->getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr)));
+			    co_await getStorageServerShards(cx, teamA[teamIdx], KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr));
 			ASSERT(shards.size() == 2);
 			ASSERT(shards[0].desiredId == sh0);
 			ASSERT(shards[1].id == sh1);
@@ -247,27 +244,26 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		}
 
 		{
-			int _ = wait(setDDMode(cx, 1));
+			int _ = co_await setDDMode(cx, 1);
 			(void)_;
 		}
 
-		wait(self->validateData(self, cx, KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr), &kvs));
+		co_await validateData(this, cx, KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr), &kvs);
 		TraceEvent("TestValueVerified").log();
-
-		return Void();
 	}
 
-	ACTOR Future<Void> deleteCheckpoints(Database cx, std::vector<UID> checkpointIds) {
+	Future<Void> deleteCheckpoints(Database cx, std::vector<UID> checkpointIds) {
 		TraceEvent(SevDebug, "DataMoveDeleteCheckpoints").detail("Checkpoints", describe(checkpointIds));
 
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				std::vector<Future<Optional<Value>>> checkpointEntries;
 				for (const UID& id : checkpointIds) {
 					checkpointEntries.push_back(tr.get(checkpointKeyFor(id)));
 				}
-				std::vector<Optional<Value>> checkpointValues = wait(getAll(checkpointEntries));
+				std::vector<Optional<Value>> checkpointValues = co_await getAll(checkpointEntries);
 
 				for (int i = 0; i < checkpointIds.size(); ++i) {
 					const auto& value = checkpointValues[i];
@@ -284,51 +280,51 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 					tr.clear(singleKeyRange(key));
 					TraceEvent(SevDebug, "DataMoveDeleteCheckpoint").detail("Checkpoint", checkpoint.toString());
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> checkpointRestore(PhysicalShardMoveWorkLoad* self,
-	                                     Database cx,
-	                                     std::vector<KeyRange> checkpointRanges,
-	                                     std::vector<KeyRange> restoreRanges,
-	                                     CheckpointAsKeyValues asKeyValues,
-	                                     std::map<Key, Value>* kvs) {
+	Future<Void> checkpointRestore(PhysicalShardMoveWorkLoad* self,
+	                               Database cx,
+	                               std::vector<KeyRange> checkpointRanges,
+	                               std::vector<KeyRange> restoreRanges,
+	                               CheckpointAsKeyValues asKeyValues,
+	                               std::map<Key, Value>* kvs) {
 
 		// Create checkpoint.
 		TraceEvent(SevDebug, "TestCreatingCheckpoint").detail("Ranges", describe(checkpointRanges));
-		state Transaction tr(cx);
-		state CheckpointFormat format = DataMoveRocksCF;
-		state UID dataMoveId = deterministicRandom()->randomUniqueID();
+		Transaction tr(cx);
+		CheckpointFormat format = DataMoveRocksCF;
+		UID dataMoveId = deterministicRandom()->randomUniqueID();
 		TraceEvent("CheckpointRestore").detail("DMID1", dataMoveId.first()).detail("DMID2", dataMoveId.second());
-		state Version version;
+		Version version{ 0 };
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				wait(createCheckpoint(&tr, checkpointRanges, format, dataMoveId));
-				wait(tr.commit());
+				co_await createCheckpoint(&tr, checkpointRanges, format, dataMoveId);
+				co_await tr.commit();
 				version = tr.getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		// Fetch checkpoint meta data.
-		state std::vector<std::pair<KeyRange, CheckpointMetaData>> records;
-		loop {
+		std::vector<std::pair<KeyRange, CheckpointMetaData>> records;
+		while (true) {
 			records.clear();
 			try {
-				wait(store(records,
-				           getCheckpointMetaData(cx, restoreRanges, version, format, Optional<UID>(dataMoveId))));
+				records = co_await getCheckpointMetaData(cx, restoreRanges, version, format, Optional<UID>(dataMoveId));
 				TraceEvent(SevDebug, "TestCheckpointMetaDataFetched")
 				    .detail("Range", describe(checkpointRanges))
 				    .detail("Version", version);
@@ -346,12 +342,12 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		}
 
 		// Fetch checkpoint.
-		state std::string checkpointDir = abspath("fetchedCheckpoints" + deterministicRandom()->randomAlphaNumeric(6));
+		std::string checkpointDir = abspath("fetchedCheckpoints" + deterministicRandom()->randomAlphaNumeric(6));
 		platform::eraseDirectoryRecursive(checkpointDir);
 		ASSERT(platform::createDirectory(checkpointDir));
-		state std::vector<Future<CheckpointMetaData>> checkpointFutures;
-		state std::vector<CheckpointMetaData> fetchedCheckpoints;
-		loop {
+		std::vector<Future<CheckpointMetaData>> checkpointFutures;
+		std::vector<CheckpointMetaData> fetchedCheckpoints;
+		while (true) {
 			checkpointFutures.clear();
 			try {
 				if (asKeyValues) {
@@ -375,7 +371,7 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 					ASSERT(platform::createDirectory(currentDir));
 					checkpointFutures.push_back(fetchCheckpoint(cx, records.front().second, currentDir));
 				}
-				wait(store(fetchedCheckpoints, getAll(checkpointFutures)));
+				fetchedCheckpoints = co_await getAll(checkpointFutures);
 				TraceEvent(SevDebug, "TestCheckpointFetched").detail("Checkpoints", describe(fetchedCheckpoints));
 				break;
 			} catch (Error& e) {
@@ -387,17 +383,17 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		for (const auto& it : records) {
 			checkpointIds.push_back(it.second.checkpointID);
 		}
-		wait(self->deleteCheckpoints(cx, checkpointIds));
+		co_await self->deleteCheckpoints(cx, checkpointIds);
 
 		// Restore KVS.
-		state std::string rocksDBTestDir = "rocksdb-kvstore-test-restored-db";
+		std::string rocksDBTestDir = "rocksdb-kvstore-test-restored-db";
 		platform::eraseDirectoryRecursive(rocksDBTestDir);
-		state std::string shardId = "restored-shard";
-		state IKeyValueStore* kvStore = keyValueStoreShardedRocksDB(
+		std::string shardId = "restored-shard";
+		IKeyValueStore* kvStore = keyValueStoreShardedRocksDB(
 		    rocksDBTestDir, deterministicRandom()->randomUniqueID(), KeyValueStoreType::SSD_SHARDED_ROCKSDB);
-		wait(kvStore->init());
+		co_await kvStore->init();
 		try {
-			wait(kvStore->restore(shardId, restoreRanges, fetchedCheckpoints));
+			co_await kvStore->restore(shardId, restoreRanges, fetchedCheckpoints);
 		} catch (Error& e) {
 			TraceEvent(SevError, "TestRestoreCheckpointError")
 			    .errorUnsuppressed(e)
@@ -407,7 +403,7 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 		TraceEvent(SevDebug, "TestCheckpointRestored").detail("Checkpoint", describe(fetchedCheckpoints));
 
 		// Validate the restored kv-store.
-		RangeResult kvRange = wait(kvStore->readRange(normalKeys));
+		RangeResult kvRange = co_await kvStore->readRange(normalKeys);
 		ASSERT(!kvRange.more);
 		std::unordered_map<Key, Value> kvsKvs;
 		for (int i = 0; i < kvRange.size(); ++i) {
@@ -439,54 +435,52 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 
 		Future<Void> close = kvStore->onClosed();
 		kvStore->dispose();
-		wait(close);
+		co_await close;
 		platform::eraseDirectoryRecursive(rocksDBTestDir);
 		platform::eraseDirectoryRecursive(checkpointDir);
 
 		TraceEvent(SevDebug, "TestRocksDBClosed").detail("Checkpoint", describe(fetchedCheckpoints));
-
-		return Void();
 	}
 
-	ACTOR Future<Version> populateData(PhysicalShardMoveWorkLoad* self, Database cx, std::map<Key, Value>* kvs) {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state Version version;
-		state UID debugID;
-		loop {
+	Future<Version> populateData(PhysicalShardMoveWorkLoad* self, Database cx, std::map<Key, Value>* kvs) {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		Version version{ 0 };
+		UID debugID;
+		while (true) {
 			debugID = deterministicRandom()->randomUniqueID();
+			Error err;
 			try {
 				tr->debugTransaction(debugID);
 				for (const auto& [key, value] : *kvs) {
 					tr->set(key, value);
 				}
-				wait(tr->commit());
+				co_await tr->commit();
 				version = tr->getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				TraceEvent("TestCommitError").errorUnsuppressed(e);
-				wait(tr->onError(e));
+				err = e;
 			}
+			TraceEvent("TestCommitError").errorUnsuppressed(err);
+			co_await tr->onError(err);
 		}
 
 		TraceEvent("PopulateTestDataDone")
 		    .detail("CommitVersion", tr->getCommittedVersion())
 		    .detail("DebugID", debugID);
 
-		return version;
+		co_return version;
 	}
 
-	ACTOR Future<Void> validateData(PhysicalShardMoveWorkLoad* self,
-	                                Database cx,
-	                                KeyRange range,
-	                                std::map<Key, Value>* kvs) {
-		state Transaction tr(cx);
-		state UID debugID;
-		loop {
+	Future<Void> validateData(PhysicalShardMoveWorkLoad* self, Database cx, KeyRange range, std::map<Key, Value>* kvs) {
+		Transaction tr(cx);
+		UID debugID;
+		while (true) {
 			debugID = deterministicRandom()->randomUniqueID();
+			Error err;
 			try {
 				TraceEvent("TestValidateDataBegin").detail("DebugID", debugID);
 				tr.debugTransaction(debugID);
-				RangeResult res = wait(tr.getRange(range, CLIENT_KNOBS->TOO_MANY));
+				RangeResult res = co_await tr.getRange(range, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!res.more && res.size() < CLIENT_KNOBS->TOO_MANY);
 
 				for (const auto& kv : res) {
@@ -494,55 +488,55 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
-				TraceEvent("TestCommitError").errorUnsuppressed(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("TestCommitError").errorUnsuppressed(err);
+			co_await tr.onError(err);
 		}
 
 		TraceEvent("ValidateTestDataDone").detail("DebugID", debugID);
-
-		return Void();
 	}
 
-	ACTOR Future<Void> readAndVerify(PhysicalShardMoveWorkLoad* self,
-	                                 Database cx,
-	                                 Key key,
-	                                 ErrorOr<Optional<Value>> expectedValue) {
-		state Transaction tr(cx);
-		state Version readVersion;
+	Future<Void> readAndVerify(PhysicalShardMoveWorkLoad* self,
+	                           Database cx,
+	                           Key key,
+	                           ErrorOr<Optional<Value>> expectedValue) {
+		Transaction tr(cx);
+		Version readVersion{ 0 };
 		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				Version _readVersion = wait(tr.getReadVersion());
+				Version _readVersion = co_await tr.getReadVersion();
 				readVersion = _readVersion;
-				state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
+				Optional<Value> res = co_await timeoutError(tr.get(key), 30.0);
 				const bool equal = !expectedValue.isError() && res == expectedValue.get();
 				if (!equal) {
 					self->validationFailed(expectedValue, ErrorOr<Optional<Value>>(res));
 				}
 				break;
 			} catch (Error& e) {
-				TraceEvent("TestReadError").errorUnsuppressed(e);
-				if (expectedValue.isError() && expectedValue.getError().code() == e.code()) {
-					break;
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			TraceEvent("TestReadError").errorUnsuppressed(err);
+			if (expectedValue.isError() && expectedValue.getError().code() == err.code()) {
+				break;
 			}
+			co_await tr.onError(err);
 		}
 
 		TraceEvent("TestReadSuccess").detail("Version", readVersion);
-
-		return Void();
 	}
 
-	ACTOR Future<Version> writeAndVerify(PhysicalShardMoveWorkLoad* self, Database cx, Key key, Optional<Value> value) {
+	Future<Version> writeAndVerify(PhysicalShardMoveWorkLoad* self, Database cx, Key key, Optional<Value> value) {
 		// state Transaction tr(cx);
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state Version version;
-		state UID debugID;
-		loop {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		Version version{ 0 };
+		UID debugID;
+		while (true) {
 			debugID = deterministicRandom()->randomUniqueID();
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->debugTransaction(debugID);
@@ -553,33 +547,34 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 				} else {
 					tr->clear(key);
 				}
-				wait(timeoutError(tr->commit(), 30.0));
+				co_await timeoutError(tr->commit(), 30.0);
 				version = tr->getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				TraceEvent("TestCommitError").errorUnsuppressed(e);
-				wait(tr->onError(e));
+				err = e;
 			}
+			TraceEvent("TestCommitError").errorUnsuppressed(err);
+			co_await tr->onError(err);
 		}
 
 		TraceEvent("TestCommitSuccess").detail("CommitVersion", tr->getCommittedVersion()).detail("DebugID", debugID);
 
-		wait(self->readAndVerify(self, cx, key, value));
+		co_await self->readAndVerify(self, cx, key, value);
 
-		return version;
+		co_return version;
 	}
 
 	// Move keys to a random selected team consisting of a single SS, this requires DD is disabled to prevent shards
 	// being moved by DD automatically. Returns the address of the single SS of the new team.
-	ACTOR Future<std::vector<UID>> moveShard(PhysicalShardMoveWorkLoad* self,
-	                                         Database cx,
-	                                         UID dataMoveId,
-	                                         KeyRange keys,
-	                                         int teamSize,
-	                                         std::unordered_set<UID> includes,
-	                                         std::unordered_set<UID> excludes) {
+	Future<std::vector<UID>> moveShard(PhysicalShardMoveWorkLoad* self,
+	                                   Database cx,
+	                                   UID dataMoveId,
+	                                   KeyRange keys,
+	                                   int teamSize,
+	                                   std::unordered_set<UID> includes,
+	                                   std::unordered_set<UID> excludes) {
 		// Pick a random SS as the dest, keys will reside on a single server after the move.
-		std::vector<StorageServerInterface> interfs = wait(getStorageServers(cx));
+		std::vector<StorageServerInterface> interfs = co_await getStorageServers(cx);
 		ASSERT(interfs.size() > teamSize - includes.size());
 		while (includes.size() < teamSize) {
 			const auto& interf = interfs[deterministicRandom()->randomInt(0, interfs.size())];
@@ -588,27 +583,27 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 			}
 		}
 
-		state std::vector<UID> dests(includes.begin(), includes.end());
-		state UID owner = deterministicRandom()->randomUniqueID();
-		state DDEnabledState ddEnabledState;
+		std::vector<UID> dests(includes.begin(), includes.end());
+		UID owner = deterministicRandom()->randomUniqueID();
+		DDEnabledState ddEnabledState;
 
-		state Transaction tr(cx);
+		Transaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				TraceEvent("TestMoveShard").detail("Range", keys.toString());
-				state MoveKeysLock moveKeysLock = wait(takeMoveKeysLock(cx, owner));
+				MoveKeysLock moveKeysLock = co_await takeMoveKeysLock(cx, owner);
 
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				state RangeResult dataMoves = wait(tr.getRange(dataMoveKeys, CLIENT_KNOBS->TOO_MANY));
-				Version readVersion = wait(tr.getReadVersion());
+				RangeResult dataMoves = co_await tr.getRange(dataMoveKeys, CLIENT_KNOBS->TOO_MANY);
+				Version readVersion = co_await tr.getReadVersion();
 				TraceEvent("TestMoveShardReadDataMoves")
 				    .detail("DataMoves", dataMoves.size())
 				    .detail("ReadVersion", readVersion);
-				state int i = 0;
-				for (; i < dataMoves.size(); ++i) {
+				for (int i = 0; i < dataMoves.size(); ++i) {
 					UID dataMoveId = decodeDataMoveKey(dataMoves[i].key);
-					state DataMoveMetaData dataMove = decodeDataMoveValue(dataMoves[i].value);
+					DataMoveMetaData dataMove = decodeDataMoveValue(dataMoves[i].value);
 					ASSERT(dataMoveId == dataMove.id);
 					TraceEvent("TestCancelDataMoveBegin").detail("DataMove", dataMove.toString());
 					if (dataMove.ranges.empty()) {
@@ -619,59 +614,62 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 						TraceEvent("TestCancelEmptyDataMoveEnd").detail("DataMove", dataMove.toString());
 						continue;
 					}
-					wait(cleanUpDataMove(cx,
-					                     dataMoveId,
-					                     moveKeysLock,
-					                     &self->cleanUpDataMoveParallelismLock,
-					                     dataMove.ranges.front(),
-					                     &ddEnabledState));
+					co_await cleanUpDataMove(cx,
+					                         dataMoveId,
+					                         moveKeysLock,
+					                         &self->cleanUpDataMoveParallelismLock,
+					                         dataMove.ranges.front(),
+					                         &ddEnabledState);
 					TraceEvent("TestCancelDataMoveEnd").detail("DataMove", dataMove.toString());
 				}
 
 				TraceEvent("TestMoveShardStartMoveKeys").detail("DataMove", dataMoveId);
-				wait(moveKeys(cx,
-				              MoveKeysParams(dataMoveId,
-				                             std::vector<KeyRange>{ keys },
-				                             dests,
-				                             dests,
-				                             moveKeysLock,
-				                             Promise<Void>(),
-				                             &self->startMoveKeysParallelismLock,
-				                             &self->finishMoveKeysParallelismLock,
-				                             false,
-				                             deterministicRandom()->randomUniqueID(), // for logging only
-				                             &ddEnabledState,
-				                             CancelConflictingDataMoves::False,
-				                             Optional<BulkLoadTaskState>())));
+				co_await moveKeys(cx,
+				                  MoveKeysParams(dataMoveId,
+				                                 std::vector<KeyRange>(1, keys),
+				                                 dests,
+				                                 dests,
+				                                 moveKeysLock,
+				                                 Promise<Void>(),
+				                                 &self->startMoveKeysParallelismLock,
+				                                 &self->finishMoveKeysParallelismLock,
+				                                 false,
+				                                 deterministicRandom()->randomUniqueID(), // for logging only
+				                                 &ddEnabledState,
+				                                 CancelConflictingDataMoves::False,
+				                                 Optional<BulkLoadTaskState>()));
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_movekeys_conflict) {
-					// Conflict on moveKeysLocks with the current running DD is expected, just retry.
-					tr.reset();
-				} else {
-					wait(tr.onError(e));
-				}
+				err = e;
+			}
+			if (err.code() == error_code_movekeys_conflict) {
+				// Conflict on moveKeysLocks with the current running DD is expected, just retry.
+				tr.reset();
+			} else {
+				co_await tr.onError(err);
 			}
 		}
 
 		TraceEvent("TestMoveShardComplete").detail("Range", keys.toString()).detail("NewTeam", describe(dests));
 
-		return dests;
+		co_return dests;
 	}
 
-	ACTOR Future<std::vector<StorageServerShard>> getStorageServerShards(Database cx, UID ssId, KeyRange range) {
-		state Transaction tr(cx);
-		loop {
+	Future<std::vector<StorageServerShard>> getStorageServerShards(Database cx, UID ssId, KeyRange range) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				Optional<Value> serverListValue = wait(tr.get(serverListKeyFor(ssId)));
+				Optional<Value> serverListValue = co_await tr.get(serverListKeyFor(ssId));
 				ASSERT(serverListValue.present());
-				state StorageServerInterface ssi = decodeServerListValue(serverListValue.get());
+				StorageServerInterface ssi = decodeServerListValue(serverListValue.get());
 				GetShardStateRequest req(range, GetShardStateRequest::READABLE, true);
-				GetShardStateReply rep = wait(ssi.getShardState.getReply(req, TaskPriority::DefaultEndpoint));
-				return rep.shards;
+				GetShardStateReply rep = co_await ssi.getShardState.getReply(req, TaskPriority::DefaultEndpoint);
+				co_return rep.shards;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
@@ -680,4 +678,4 @@ struct PhysicalShardMoveWorkLoad : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<PhysicalShardMoveWorkLoad> PhysicalShardMoveWorkLoadFactory;
\ No newline at end of file
+WorkloadFactory<PhysicalShardMoveWorkLoad> PhysicalShardMoveWorkLoadFactory;
diff --git a/fdbserver/workloads/Ping.actor.cpp b/fdbserver/workloads/Ping.cpp
similarity index 76%
rename from fdbserver/workloads/Ping.actor.cpp
rename to fdbserver/workloads/Ping.cpp
index 81c55f6d6f1..f100e1b1e15 100644
--- a/fdbserver/workloads/Ping.actor.cpp
+++ b/fdbserver/workloads/Ping.cpp
@@ -1,5 +1,5 @@
 /*
- * Ping.actor.cpp
+ * Ping.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,10 @@
 
 #include "flow/ActorCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/QuietDatabase.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/QuietDatabase.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct PingWorkloadInterface {
@@ -51,7 +51,7 @@ struct PingWorkload : TestWorkload {
 	PerfDoubleCounter totalMessageLatency;
 	PerfDoubleCounter maxMessageLatency;
 
-	PingWorkload(WorkloadContext const& wcx)
+	explicit PingWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), messages("Messages"), totalMessageLatency("TotalLatency"),
 	    maxMessageLatency("Max Latency (ms)") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -98,37 +98,38 @@ struct PingWorkload : TestWorkload {
 		m.push_back(maxMessageLatency.getMetric());
 	}
 
-	ACTOR Future<Void> persistInterface(PingWorkload* self, Database cx) {
-		state Transaction tr(cx);
+	Future<Void> persistInterface(PingWorkload* self, Database cx) {
+		Transaction tr(cx);
 		BinaryWriter wr(IncludeVersion());
 		wr << self->interf;
-		state Standalone<StringRef> serializedInterface = wr.toValue();
-		loop {
+		Standalone<StringRef> serializedInterface = wr.toValue();
+		while (true) {
+			Error err;
 			try {
-				Optional<Value> val = wait(tr.get(StringRef(format("Ping/Client/%d", self->clientId))));
+				Optional<Value> val = co_await tr.get(StringRef(format("Ping/Client/%d", self->clientId)));
 				if (val.present()) {
 					if (val.get() != serializedInterface)
 						throw operation_failed();
 					break;
 				}
 				tr.set(format("Ping/Client/%d", self->clientId), serializedInterface);
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<std::vector<PingWorkloadInterface>> fetchInterfaces(PingWorkload* self, Database cx) {
-		state Transaction tr(cx);
-		loop {
+	Future<std::vector<PingWorkloadInterface>> fetchInterfaces(PingWorkload* self, Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				state std::vector<PingWorkloadInterface> result;
-				state int i;
-				for (i = 0; i < self->clientCount; i++) {
-					Optional<Value> val = wait(tr.get(StringRef(format("Ping/Client/%d", i))));
+				std::vector<PingWorkloadInterface> result;
+				for (int i = 0; i < self->clientCount; i++) {
+					Optional<Value> val = co_await tr.get(StringRef(format("Ping/Client/%d", i)));
 					if (!val.present()) {
 						throw operation_failed();
 					}
@@ -137,27 +138,28 @@ struct PingWorkload : TestWorkload {
 					br >> interf;
 					result.push_back(interf);
 				}
-				return result;
+				co_return result;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> pinger(PingWorkload* self, std::vector<RequestStream<LoadedPingRequest>> peers) {
-		state double lastTime = now();
+	Future<Void> pinger(PingWorkload* self, std::vector<RequestStream<LoadedPingRequest>> peers) {
+		double lastTime = now();
 
-		loop {
-			wait(poisson(&lastTime, self->actorCount / self->operationsPerSecond));
+		while (true) {
+			co_await poisson(&lastTime, self->actorCount / self->operationsPerSecond);
 			auto& peer = deterministicRandom()->randomChoice(peers);
-			state NetworkAddress addr = peer.getEndpoint().getPrimaryAddress();
-			state double before = now();
+			NetworkAddress addr = peer.getEndpoint().getPrimaryAddress();
+			double before = now();
 
 			LoadedPingRequest req;
 			req.id = deterministicRandom()->randomUniqueID();
 			req.payload = self->usePayload ? self->payloadOut : ""_sr;
 			req.loadReply = self->usePayload;
-			LoadedReply rep = wait(peer.getReply(req));
+			LoadedReply rep = co_await peer.getReply(req);
 
 			double elapsed = now() - before;
 			self->totalMessageLatency += elapsed;
@@ -168,8 +170,8 @@ struct PingWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> pinger(PingWorkload* self, Database cx) {
-		std::vector<PingWorkloadInterface> testers = wait(self->fetchInterfaces(self, cx));
+	Future<Void> pinger(PingWorkload* self, Database cx) {
+		std::vector<PingWorkloadInterface> testers = co_await self->fetchInterfaces(self, cx);
 		std::vector<RequestStream<LoadedPingRequest>> peers;
 		peers.reserve(testers.size());
 		for (int i = 0; i < testers.size(); i++)
@@ -178,12 +180,11 @@ struct PingWorkload : TestWorkload {
 		pingers.reserve(self->actorCount);
 		for (int i = 0; i < self->actorCount; i++)
 			pingers.push_back(self->pinger(self, peers));
-		wait(waitForAll(pingers));
-		return Void();
+		co_await waitForAll(pingers);
 	}
 
-	ACTOR Future<Void> workerPinger(PingWorkload* self) {
-		std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
+	Future<Void> workerPinger(PingWorkload* self) {
+		std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
 		std::vector<RequestStream<LoadedPingRequest>> peers;
 		peers.reserve(workers.size());
 		for (int i = 0; i < workers.size(); i++)
@@ -192,8 +193,7 @@ struct PingWorkload : TestWorkload {
 		pingers.reserve(self->actorCount);
 		for (int i = 0; i < self->actorCount; i++)
 			pingers.push_back(self->pinger(self, peers));
-		wait(waitForAll(pingers));
-		return Void();
+		co_await waitForAll(pingers);
 	}
 
 	// ACTOR Future<Void> poisson_spin( double *last, double meanInterval ) {
@@ -206,25 +206,25 @@ struct PingWorkload : TestWorkload {
 	// 	return Void();
 	// }
 
-	ACTOR Future<Void> payloadSender(PingWorkload* self, Database cx) {
-		state std::vector<RequestStream<LoadedPingRequest>> endpoints;
-		state double lastTime = timer();
-		state PromiseStream<Future<Void>> addActor;
-		state Future<Void> collection = actorCollection(addActor.getFuture());
+	Future<Void> payloadSender(PingWorkload* self, Database cx) {
+		std::vector<RequestStream<LoadedPingRequest>> endpoints;
+		double lastTime = timer();
+		PromiseStream<Future<Void>> addActor;
+		Future<Void> collection = actorCollection(addActor.getFuture());
 
 		if (self->workerBroadcast) {
-			std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
+			std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
 			for (int i = 0; i < workers.size(); i++)
 				endpoints.push_back(workers[i].interf.debugPing);
 		} else {
-			std::vector<PingWorkloadInterface> peers = wait(self->fetchInterfaces(self, cx));
+			std::vector<PingWorkloadInterface> peers = co_await self->fetchInterfaces(self, cx);
 			for (int i = 0; i < peers.size(); i++)
 				endpoints.push_back(peers[i].payloadPing);
 		}
 
 		// std::random_shuffle( peers.begin(), peers.end() );
-		loop {
-			wait(poisson(&lastTime, 1.0 / 6.0));
+		while (true) {
+			co_await poisson(&lastTime, 1.0 / 6.0);
 			addActor.send(self->payloadPinger(self, cx, endpoints));
 		}
 	}
@@ -242,14 +242,12 @@ struct PingWorkload : TestWorkload {
 	// 	return Void();
 	// }
 
-	ACTOR Future<Void> payloadPinger(PingWorkload* self,
-	                                 Database cx,
-	                                 std::vector<RequestStream<LoadedPingRequest>> peers) {
+	Future<Void> payloadPinger(PingWorkload* self, Database cx, std::vector<RequestStream<LoadedPingRequest>> peers) {
 		// state std::vector<PingWorkloadInterface> peers = wait( self->fetchInterfaces( self, cx ) );
 
 		// loop {
-		state double start = now();
-		state UID pingId(deterministicRandom()->randomUniqueID());
+		double start = now();
+		UID pingId(deterministicRandom()->randomUniqueID());
 		std::vector<Future<Void>> replies;
 		for (int i = 0; i < peers.size(); i++) {
 			LoadedPingRequest req;
@@ -262,12 +260,11 @@ struct PingWorkload : TestWorkload {
 			// replies.push_back( self->payloadDelayer( req, peers[i].payloadPing ) );
 		}
 		TraceEvent("PayloadPingSent", pingId).log();
-		wait(waitForAll(replies));
+		co_await waitForAll(replies);
 		double elapsed = now() - start;
 		TraceEvent("PayloadPingDone", pingId).detail("Elapsed", elapsed);
 		// 	wait( delay( deterministicRandom()->random01() / 100 ) );
 		// }
-		return Void();
 	}
 
 	// ACTOR Future<Void> packetPonger( PingWorkload* self, LoadedPingRequest req ) {
@@ -281,12 +278,12 @@ struct PingWorkload : TestWorkload {
 	// 	return Void();
 	// }
 
-	ACTOR Future<Void> ponger(PingWorkload* self) {
+	Future<Void> ponger(PingWorkload* self) {
 		// state PromiseStream<Future<Void>> addActor;
 		// state Future<Void> pongCollection = actorCollection( addActor.getFuture() );
 
-		loop {
-			LoadedPingRequest req = waitNext(self->interf.payloadPing.getFuture());
+		while (true) {
+			LoadedPingRequest req = co_await self->interf.payloadPing.getFuture();
 			// double end = timer() + (deterministicRandom()->random01() / 200);
 			// while( timer() < end )
 			// 	_mm_pause();
diff --git a/fdbserver/workloads/PrivateEndpoints.actor.cpp b/fdbserver/workloads/PrivateEndpoints.cpp
similarity index 83%
rename from fdbserver/workloads/PrivateEndpoints.actor.cpp
rename to fdbserver/workloads/PrivateEndpoints.cpp
index 365d5306a19..b7238e4866b 100644
--- a/fdbserver/workloads/PrivateEndpoints.actor.cpp
+++ b/fdbserver/workloads/PrivateEndpoints.cpp
@@ -1,5 +1,5 @@
 /*
- * PrivateEndpoints.actor.cpp
+ * PrivateEndpoints.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,9 +18,9 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 
-#include "flow/actorcompiler.h" // has to be last include
+#include "flow/CoroUtils.h"
 
 namespace {
 
@@ -54,11 +54,10 @@ struct PrivateEndpoints : TestWorkload {
 		}
 	}
 
-	ACTOR template <class T>
+	template <class T>
 	static Future<Void> assumeFailure(Future<T> f) {
 		try {
-			T t = wait(f);
-			(void)t;
+			co_await f;
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
@@ -73,7 +72,6 @@ struct PrivateEndpoints : TestWorkload {
 				TraceEvent(SevError, "WrongErrorCode").error(e);
 			}
 		}
-		return Void();
 	}
 
 	template <class I, class RT>
@@ -105,39 +103,39 @@ struct PrivateEndpoints : TestWorkload {
 		addTestFor(&CommitProxyInterface::exclusionSafetyCheckReq);
 		// addTestFor(&CommitProxyInterface::getDDMetrics);
 	}
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
+	Future<Void> start(Database const& cx) override { return _start(cx); }
 	Future<bool> check(Database const& cx) override { return success; }
 	void getMetrics(std::vector<PerfMetric>& m) override {
 		m.emplace_back("Successes", double(numSuccesses), Averaged::True);
 	}
 
-	ACTOR static Future<Void> _start(PrivateEndpoints* self, Database cx) {
-		state Reference<AsyncVar<ClientDBInfo>> clientInfo = cx->clientInfo;
-		state Future<Void> end;
-		TraceEvent("PrivateEndpointTestStartWait").detail("WaitTime", self->startAfter).log();
-		wait(delay(self->startAfter));
-		TraceEvent("PrivateEndpointTestStart").detail("RunFor", self->runFor).log();
-		end = delay(self->runFor);
+	Future<Void> _start(Database cx) {
+		Reference<AsyncVar<ClientDBInfo>> clientInfo = cx->clientInfo;
+		TraceEvent("PrivateEndpointTestStartWait").detail("WaitTime", startAfter).log();
+		co_await delay(startAfter);
+		TraceEvent("PrivateEndpointTestStart").detail("RunFor", runFor).log();
+		Future<Void> end = delay(runFor);
 		try {
-			loop {
-				auto testFuture = deterministicRandom()->randomChoice(self->testFunctions)(cx->clientInfo);
-				choose {
-					when(wait(end)) {
+			while (true) {
+				auto testFuture = deterministicRandom()->randomChoice(testFunctions)(clientInfo);
+				{
+					auto choice = co_await race(end, testFuture);
+					if (choice.index() == 0) {
 						TraceEvent("PrivateEndpointTestDone").log();
-						return Void();
-					}
-					when(wait(testFuture)) {
-						++self->numSuccesses;
+						co_return;
+					} else if (choice.index() == 1) {
+						++numSuccesses;
+					} else {
+						UNREACHABLE();
 					}
 				}
-				wait(delay(0.2));
+				co_await delay(0.2);
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "PrivateEndpointTestError").errorUnsuppressed(e);
 			ASSERT(false);
 		}
 		UNREACHABLE();
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/ProtocolVersion.actor.cpp b/fdbserver/workloads/ProtocolVersion.cpp
similarity index 70%
rename from fdbserver/workloads/ProtocolVersion.actor.cpp
rename to fdbserver/workloads/ProtocolVersion.cpp
index aba48b50fa6..ec1b39537d4 100644
--- a/fdbserver/workloads/ProtocolVersion.actor.cpp
+++ b/fdbserver/workloads/ProtocolVersion.cpp
@@ -1,5 +1,5 @@
 /*
- * ProtocolVersion.actor.cpp
+ * ProtocolVersion.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,20 +18,17 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "flow/actorcompiler.h" // This must be the last include
 
 struct ProtocolVersionWorkload : TestWorkload {
 	static constexpr auto NAME = "ProtocolVersion";
 
-	ProtocolVersionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+	explicit ProtocolVersionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-
-	ACTOR Future<Void> _start(ProtocolVersionWorkload* self, Database cx) {
-		state std::vector<ISimulator::ProcessInfo*> allProcesses = g_simulator->getAllProcesses();
-		state std::vector<ISimulator::ProcessInfo*>::iterator diffVersionProcess =
+	Future<Void> start(Database const& cx) override {
+		std::vector<ISimulator::ProcessInfo*> allProcesses = g_simulator->getAllProcesses();
+		auto diffVersionProcess =
 		    find_if(allProcesses.begin(), allProcesses.end(), [](const ISimulator::ProcessInfo* p) {
 			    return p->protocolVersion != currentProtocolVersion();
 		    });
@@ -40,10 +37,9 @@ struct ProtocolVersionWorkload : TestWorkload {
 
 		RequestStream<ProtocolInfoRequest> requestStream{ Endpoint::wellKnown({ (*diffVersionProcess)->addresses },
 			                                                                  WLTOKEN_PROTOCOL_INFO) };
-		ProtocolInfoReply reply = wait(retryBrokenPromise(requestStream, ProtocolInfoRequest{}));
+		ProtocolInfoReply reply = co_await retryBrokenPromise(requestStream, ProtocolInfoRequest{});
 
 		ASSERT(reply.version != g_network->protocolVersion());
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/PubSubMultiples.actor.cpp b/fdbserver/workloads/PubSubMultiples.cpp
similarity index 69%
rename from fdbserver/workloads/PubSubMultiples.actor.cpp
rename to fdbserver/workloads/PubSubMultiples.cpp
index a7289421733..70426d6b2a9 100644
--- a/fdbserver/workloads/PubSubMultiples.actor.cpp
+++ b/fdbserver/workloads/PubSubMultiples.cpp
@@ -1,5 +1,5 @@
 /*
- * PubSubMultiples.actor.cpp
+ * PubSubMultiples.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,8 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/pubsub.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "pubsub.h"
+#include "fdbserver/tester/workloads.h"
 
 struct PubSubMultiplesWorkload : TestWorkload {
 	static constexpr auto NAME = "PubSubMultiples";
@@ -33,7 +31,7 @@ struct PubSubMultiplesWorkload : TestWorkload {
 	std::vector<Future<Void>> inboxWatchers;
 	PerfIntCounter messages;
 
-	PubSubMultiplesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), messages("Messages") {
+	explicit PubSubMultiplesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), messages("Messages") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		messagesPerSecond = getOption(options, "messagesPerSecond"_sr, 500.0) / clientCount;
 		actorCount = getOption(options, "actorsPerClient"_sr, 20);
@@ -53,19 +51,19 @@ struct PubSubMultiplesWorkload : TestWorkload {
 	Key keyForInbox(int i) { return StringRef(format("/PSM/inbox/%d", i)); }
 	Value valueForUInt(uint64_t i) { return StringRef(format("%llx", i)); }
 
-	ACTOR Future<Void> createNodeSwath(PubSubMultiplesWorkload* self, int actor, Database cx) {
-		state PubSub ps(cx);
-		state std::vector<uint64_t> feeds;
-		state std::vector<uint64_t> inboxes;
-		state int idx;
-		for (idx = 0; idx < self->inboxesPerActor; idx++) {
-			uint64_t feedIdx = wait(ps.createFeed(StringRef()));
+	Future<Void> createNodeSwath(PubSubMultiplesWorkload* self, int actor, Database cx) {
+		PubSub ps(cx);
+		std::vector<uint64_t> feeds;
+		std::vector<uint64_t> inboxes;
+		for (int idx = 0; idx < self->inboxesPerActor; idx++) {
+			uint64_t feedIdx = co_await ps.createFeed(StringRef());
 			feeds.push_back(feedIdx);
-			uint64_t inboxIdx = wait(ps.createInbox(StringRef()));
+			uint64_t inboxIdx = co_await ps.createInbox(StringRef());
 			inboxes.push_back(inboxIdx);
 		}
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				for (int idx = 0; idx < self->inboxesPerActor; idx++) {
 					int offset = (self->clientId * self->clientCount * self->actorCount * self->inboxesPerActor) +
@@ -73,45 +71,43 @@ struct PubSubMultiplesWorkload : TestWorkload {
 					tr.set(self->keyForFeed(offset), self->valueForUInt(feeds[idx]));
 					tr.set(self->keyForInbox(offset), self->valueForUInt(inboxes[idx]));
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> createNodes(PubSubMultiplesWorkload* self, Database cx) {
-		state PubSub ps(cx);
+	Future<Void> createNodes(PubSubMultiplesWorkload* self, Database cx) {
+		PubSub ps(cx);
 		std::vector<Future<Void>> actors;
 		actors.reserve(self->actorCount);
 		for (int i = 0; i < self->actorCount; i++)
 			actors.push_back(self->createNodeSwath(self, i, cx->clone()));
-		wait(waitForAll(actors));
+		co_await waitForAll(actors);
 		TraceEvent("PSMNodesCreated").detail("ClientIdx", self->clientId);
-		return Void();
 	}
 
-	/*ACTOR*/ Future<Void> createSubscriptions(PubSubMultiplesWorkload* self, int actor, Database cx) {
+	Future<Void> createSubscriptions(PubSubMultiplesWorkload* self, int actor, Database cx) {
 		// create the "multiples" subscriptions for each owned inbox
 		return Void();
 	}
 
-	/*ACTOR*/ Future<Void> messageSender(PubSubMultiplesWorkload* self, Database cx) {
+	Future<Void> messageSender(PubSubMultiplesWorkload* self, Database cx) {
 		// use a possion loop and post messages to feeds
 		return Void();
 	}
 
-	ACTOR Future<Void> startTests(PubSubMultiplesWorkload* self, Database cx) {
+	Future<Void> startTests(PubSubMultiplesWorkload* self, Database cx) {
 		std::vector<Future<Void>> subscribers;
 		subscribers.reserve(self->actorCount);
 		for (int i = 0; i < self->actorCount; i++)
 			subscribers.push_back(self->createSubscriptions(self, i, cx));
-		wait(waitForAll(subscribers));
+		co_await waitForAll(subscribers);
 
-		state Future<Void> sender = self->messageSender(self, cx);
-		return Void();
+		Future<Void> sender = self->messageSender(self, cx);
 	}
 };
 
diff --git a/fdbserver/workloads/QueuePush.actor.cpp b/fdbserver/workloads/QueuePush.cpp
similarity index 80%
rename from fdbserver/workloads/QueuePush.actor.cpp
rename to fdbserver/workloads/QueuePush.cpp
index 64c39605759..15f0cc643ee 100644
--- a/fdbserver/workloads/QueuePush.actor.cpp
+++ b/fdbserver/workloads/QueuePush.cpp
@@ -1,5 +1,5 @@
 /*
- * QueuePush.actor.cpp
+ * QueuePush.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,9 +22,8 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 const int keyBytes = 16;
 
@@ -41,7 +40,7 @@ struct QueuePushWorkload : TestWorkload {
 	PerfIntCounter transactions, retries;
 	DDSketch<double> commitLatencies, GRVLatencies;
 
-	QueuePushWorkload(WorkloadContext const& wcx)
+	explicit QueuePushWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), commitLatencies(), GRVLatencies() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		actorCount = getOption(options, "actorCount"_sr, 50);
@@ -55,8 +54,6 @@ struct QueuePushWorkload : TestWorkload {
 		startingKey = "0000000000000001"_sr;
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
@@ -94,35 +91,35 @@ struct QueuePushWorkload : TestWorkload {
 			throw client_invalid_operation();
 	}
 
-	ACTOR Future<Void> _start(Database cx, QueuePushWorkload* self) {
-		for (int i = 0; i < self->actorCount; i++) {
-			self->clients.push_back(self->writeClient(cx, self));
+	Future<Void> start(Database const& cx) override {
+		for (int i = 0; i < actorCount; i++) {
+			clients.push_back(writeClient(cx, this));
 		}
 
-		wait(timeout(waitForAll(self->clients), self->testDuration, Void()));
-		self->clients.clear();
-		return Void();
+		co_await timeout(waitForAll(clients), testDuration, Void());
+		clients.clear();
 	}
 
-	ACTOR Future<Void> writeClient(Database cx, QueuePushWorkload* self) {
-		loop {
-			state Transaction tr(cx);
-			loop {
+	Future<Void> writeClient(Database cx, QueuePushWorkload* self) {
+		while (true) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					state double start = now();
-					wait(success(tr.getReadVersion()));
+					double start = now();
+					co_await tr.getReadVersion();
 					self->GRVLatencies.addSample(now() - start);
 
 					// Get the last key in the database with a snapshot read
-					state Key lastKey;
+					Key lastKey;
 
 					if (self->forward) {
-						Key _lastKey = wait(tr.getKey(lastLessThan(self->endingKey), Snapshot::True));
+						Key _lastKey = co_await tr.getKey(lastLessThan(self->endingKey), Snapshot::True);
 						lastKey = _lastKey;
-						if (lastKey == StringRef())
+						if (lastKey.empty())
 							lastKey = self->startingKey;
 					} else {
-						Key _lastKey = wait(tr.getKey(firstGreaterThan(self->startingKey), Snapshot::True));
+						Key _lastKey = co_await tr.getKey(firstGreaterThan(self->startingKey), Snapshot::True);
 						lastKey = _lastKey;
 						if (!normalKeys.contains(lastKey))
 							lastKey = self->endingKey;
@@ -138,13 +135,14 @@ struct QueuePushWorkload : TestWorkload {
 						       StringRef(self->valueString));
 
 					start = now();
-					wait(tr.commit());
+					co_await tr.commit();
 					self->commitLatencies.addSample(now() - start);
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
-					++self->retries;
+					err = e;
 				}
+				co_await tr.onError(err);
+				++self->retries;
 			}
 			++self->transactions;
 		}
diff --git a/fdbserver/workloads/RYWDisable.actor.cpp b/fdbserver/workloads/RYWDisable.cpp
similarity index 72%
rename from fdbserver/workloads/RYWDisable.actor.cpp
rename to fdbserver/workloads/RYWDisable.cpp
index 168b198f72c..3a2d33481c9 100644
--- a/fdbserver/workloads/RYWDisable.actor.cpp
+++ b/fdbserver/workloads/RYWDisable.cpp
@@ -1,5 +1,5 @@
 /*
- * RYWDisable.actor.cpp
+ * RYWDisable.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,9 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct RYWDisableWorkload : TestWorkload {
 	static constexpr auto NAME = "RYWDisable";
@@ -31,7 +30,7 @@ struct RYWDisableWorkload : TestWorkload {
 	double testDuration;
 	std::vector<Future<Void>> clients;
 
-	RYWDisableWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit RYWDisableWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		nodes = getOption(options, "nodes"_sr, 100);
 		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
@@ -41,31 +40,31 @@ struct RYWDisableWorkload : TestWorkload {
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0)
-			return _start(cx, this);
+			return _start(cx);
 		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, RYWDisableWorkload* self) {
-		state double testStart = now();
+	Future<Void> _start(Database cx) {
+		double testStart = now();
 
-		loop {
-			state ReadYourWritesTransaction tr(cx);
-			loop {
+		while (true) {
+			ReadYourWritesTransaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					// do some operations
-					state int opType = deterministicRandom()->randomInt(0, 4);
-					state bool shouldError = true;
+					int opType = deterministicRandom()->randomInt(0, 4);
+					bool shouldError = true;
 
 					if (opType == 0) {
 						//TraceEvent("RYWSetting");
-						tr.set(self->keyForIndex(deterministicRandom()->randomInt(0, self->nodes)), StringRef());
+						tr.set(keyForIndex(deterministicRandom()->randomInt(0, nodes)), StringRef());
 					} else if (opType == 1) {
 						//TraceEvent("RYWGetNoWait");
-						Future<Optional<Value>> _ =
-						    tr.get(self->keyForIndex(deterministicRandom()->randomInt(0, self->nodes)));
+						Future<Optional<Value>> _ = tr.get(keyForIndex(deterministicRandom()->randomInt(0, nodes)));
 					} else if (opType == 2) {
 						//TraceEvent("RYWGetAndWait");
-						wait(success(tr.get(self->keyForIndex(deterministicRandom()->randomInt(0, self->nodes)))));
+						co_await tr.get(keyForIndex(deterministicRandom()->randomInt(0, nodes)));
 					} else {
 						//TraceEvent("RYWNoOp");
 						shouldError = false;
@@ -82,17 +81,20 @@ struct RYWDisableWorkload : TestWorkload {
 						ASSERT(e.code() == error_code_client_invalid_operation);
 					}
 
-					wait(delay(0.1));
+					co_await delay(0.1);
 
-					if (now() - testStart > self->testDuration)
-						return Void();
+					if (now() - testStart > testDuration)
+						co_return;
 
 					if (deterministicRandom()->random01() < 0.5)
 						break;
 
 					tr.reset();
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.isValid()) {
+					co_await tr.onError(err);
 				}
 			}
 		}
diff --git a/fdbserver/workloads/RYWPerformance.actor.cpp b/fdbserver/workloads/RYWPerformance.actor.cpp
deleted file mode 100644
index 626620b03e6..00000000000
--- a/fdbserver/workloads/RYWPerformance.actor.cpp
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * RYWPerformance.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-struct RYWPerformanceWorkload : TestWorkload {
-	static constexpr auto NAME = "RYWPerformance";
-
-	int keyBytes, nodes, ranges;
-	RYWPerformanceWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		nodes = getOption(options, "nodes"_sr, 10000);
-		ranges = getOption(options, "ranges"_sr, 10);
-		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
-	}
-
-	Future<Void> setup(Database const& cx) override {
-		if (clientId == 0)
-			return _setup(cx, this);
-		return Void();
-	}
-
-	ACTOR Future<Void> _setup(Database cx, RYWPerformanceWorkload* self) {
-		state Transaction tr(cx);
-
-		loop {
-			try {
-				for (int i = 0; i < self->nodes; i++)
-					tr.set(self->keyForIndex(i), "bar"_sr);
-
-				wait(tr.commit());
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-
-		return Void();
-	}
-
-	Future<Void> start(Database const& cx) override {
-		if (clientId == 0)
-			return _start(cx, this);
-		return Void();
-	}
-
-	ACTOR static Future<Void> fillCache(ReadYourWritesTransaction* tr, RYWPerformanceWorkload* self, int type) {
-		state int i;
-		if (type == 0) {
-			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 1) {
-			std::vector<Future<Optional<Value>>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->get(self->keyForIndex(i)));
-			}
-			wait(waitForAll(gets));
-		} else if (type == 2) {
-			std::vector<Future<Optional<Value>>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->get(self->keyForIndex(i)));
-			}
-			wait(waitForAll(gets));
-			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 3) {
-			std::vector<Future<Optional<Value>>> gets;
-			for (i = 0; i < self->nodes; i += 2) {
-				gets.push_back(tr->get(self->keyForIndex(i)));
-			}
-			wait(waitForAll(gets));
-			for (i = 1; i < self->nodes; i += 2) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 4) {
-			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
-		} else if (type == 5) {
-			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
-			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 6) {
-			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
-			for (i = 0; i < self->nodes; i += 2) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 7) {
-			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
-			for (i = 0; i < self->nodes; i++) {
-				tr->clear(self->keyForIndex(i));
-			}
-		} else if (type == 8) {
-			wait(success(tr->getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
-			for (i = 0; i < self->nodes; i += 2) {
-				tr->clear(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 1)));
-			}
-		} else if (type == 9) {
-			std::vector<Future<RangeResult>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->getRange(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 2)), self->nodes));
-			}
-			wait(waitForAll(gets));
-		} else if (type == 10) {
-			std::vector<Future<RangeResult>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->getRange(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 2)), self->nodes));
-			}
-			wait(waitForAll(gets));
-			for (i = 0; i < self->nodes; i++) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 11) {
-			std::vector<Future<RangeResult>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->getRange(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 2)), self->nodes));
-			}
-			wait(waitForAll(gets));
-			for (i = 0; i < self->nodes; i += 2) {
-				tr->set(self->keyForIndex(i), "foo"_sr);
-			}
-		} else if (type == 12) {
-			std::vector<Future<RangeResult>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->getRange(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 2)), self->nodes));
-			}
-			wait(waitForAll(gets));
-			for (i = 0; i < self->nodes; i++) {
-				tr->clear(self->keyForIndex(i));
-			}
-		} else if (type == 13) {
-			std::vector<Future<RangeResult>> gets;
-			for (i = 0; i < self->nodes; i++) {
-				gets.push_back(tr->getRange(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 2)), self->nodes));
-			}
-			wait(waitForAll(gets));
-			for (i = 0; i < self->nodes; i += 2) {
-				tr->clear(KeyRangeRef(self->keyForIndex(i), self->keyForIndex(i + 1)));
-			}
-		}
-		return Void();
-	}
-
-	ACTOR static Future<Void> test_get_single(Database cx, RYWPerformanceWorkload* self, int cacheType) {
-		state int i;
-		state ReadYourWritesTransaction tr(cx);
-
-		loop {
-			try {
-				wait(self->fillCache(&tr, self, cacheType));
-
-				state double startTime = timer();
-
-				for (i = 0; i < self->nodes; i++) {
-					wait(success(tr.get(self->keyForIndex(self->nodes / 2))));
-				}
-
-				fprintf(stderr, "%f", self->nodes / (timer() - startTime));
-
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> test_get_many_sequential(Database cx, RYWPerformanceWorkload* self, int cacheType) {
-		state int i;
-		state ReadYourWritesTransaction tr(cx);
-
-		loop {
-			try {
-				wait(self->fillCache(&tr, self, cacheType));
-
-				state double startTime = timer();
-
-				for (i = 0; i < self->nodes; i++) {
-					wait(success(tr.get(self->keyForIndex(i))));
-				}
-
-				fprintf(stderr, "%f", self->nodes / (timer() - startTime));
-
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> test_get_range_basic(Database cx, RYWPerformanceWorkload* self, int cacheType) {
-		state int i;
-		state ReadYourWritesTransaction tr(cx);
-
-		loop {
-			try {
-				wait(self->fillCache(&tr, self, cacheType));
-
-				state double startTime = timer();
-
-				for (i = 0; i < self->ranges; i++) {
-					wait(success(
-					    tr.getRange(KeyRangeRef(self->keyForIndex(0), self->keyForIndex(self->nodes)), self->nodes)));
-				}
-
-				fprintf(stderr, "%f", self->ranges / (timer() - startTime));
-
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> test_interleaved_sets_gets(Database cx, RYWPerformanceWorkload* self, int cacheType) {
-		state int i;
-		state ReadYourWritesTransaction tr(cx);
-
-		loop {
-			try {
-				wait(self->fillCache(&tr, self, cacheType));
-
-				tr.set(self->keyForIndex(self->nodes / 2), self->keyForIndex(self->nodes));
-
-				state double startTime = timer();
-
-				for (i = 0; i < self->nodes; i++) {
-					wait(success(tr.get(self->keyForIndex(self->nodes / 2))));
-					tr.set(self->keyForIndex(self->nodes / 2), self->keyForIndex(i));
-				}
-
-				fprintf(stderr, "%f", self->nodes / (timer() - startTime));
-
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-
-	ACTOR static Future<Void> _start(Database cx, RYWPerformanceWorkload* self) {
-		state int i;
-		fprintf(stderr, "test_get_single, ");
-		for (i = 0; i < 14; i++) {
-			wait(self->test_get_single(cx, self, i));
-			if (i == 13)
-				fprintf(stderr, "\n");
-			else
-				fprintf(stderr, ", ");
-		}
-		fprintf(stderr, "test_get_many_sequential, ");
-		for (i = 0; i < 14; i++) {
-			wait(self->test_get_many_sequential(cx, self, i));
-			if (i == 13)
-				fprintf(stderr, "\n");
-			else
-				fprintf(stderr, ", ");
-		}
-		fprintf(stderr, "test_get_range_basic, ");
-		for (i = 4; i < 14; i++) {
-			wait(self->test_get_range_basic(cx, self, i));
-			if (i == 13)
-				fprintf(stderr, "\n");
-			else
-				fprintf(stderr, ", ");
-		}
-		fprintf(stderr, "test_interleaved_sets_gets, ");
-		for (i = 0; i < 14; i++) {
-			wait(self->test_interleaved_sets_gets(cx, self, i));
-			if (i == 13)
-				fprintf(stderr, "\n");
-			else
-				fprintf(stderr, ", ");
-		}
-		return Void();
-	}
-
-	Future<bool> check(Database const& cx) override { return true; }
-
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-
-	Key keyForIndex(uint64_t index) {
-		Key result = makeString(keyBytes);
-		uint8_t* data = mutateString(result);
-		memset(data, '.', keyBytes);
-
-		double d = double(index) / nodes;
-		emplaceIndex(data, 0, *(int64_t*)&d);
-
-		return result;
-	}
-};
-
-WorkloadFactory<RYWPerformanceWorkload> RYWPerformanceWorkloadFactory;
diff --git a/fdbserver/workloads/RYWPerformance.cpp b/fdbserver/workloads/RYWPerformance.cpp
new file mode 100644
index 00000000000..d1018d634c2
--- /dev/null
+++ b/fdbserver/workloads/RYWPerformance.cpp
@@ -0,0 +1,318 @@
+/*
+ * RYWPerformance.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbserver/tester/workloads.h"
+
+struct RYWPerformanceWorkload : TestWorkload {
+	static constexpr auto NAME = "RYWPerformance";
+
+	int keyBytes, nodes, ranges;
+	explicit RYWPerformanceWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		nodes = getOption(options, "nodes"_sr, 10000);
+		ranges = getOption(options, "ranges"_sr, 10);
+		keyBytes = std::max(getOption(options, "keyBytes"_sr, 16), 16);
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if (clientId == 0)
+			return _setup(cx, this);
+		return Void();
+	}
+
+	Future<Void> _setup(Database cx, RYWPerformanceWorkload* self) {
+		Transaction tr(cx);
+
+		while (true) {
+			Error err;
+			try {
+				for (int i = 0; i < self->nodes; i++)
+					tr.set(self->keyForIndex(i), "bar"_sr);
+
+				co_await tr.commit();
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0)
+			return _start(cx);
+		return Void();
+	}
+
+	Future<Void> fillCache(ReadYourWritesTransaction* tr, int type) {
+		int i{ 0 };
+		if (type == 0) {
+			for (i = 0; i < nodes; i++) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 1) {
+			std::vector<Future<Optional<Value>>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->get(keyForIndex(i)));
+			}
+			co_await waitForAll(gets);
+		} else if (type == 2) {
+			std::vector<Future<Optional<Value>>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->get(keyForIndex(i)));
+			}
+			co_await waitForAll(gets);
+			for (i = 0; i < nodes; i++) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 3) {
+			std::vector<Future<Optional<Value>>> gets;
+			for (i = 0; i < nodes; i += 2) {
+				gets.push_back(tr->get(keyForIndex(i)));
+			}
+			co_await waitForAll(gets);
+			for (i = 1; i < nodes; i += 2) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 4) {
+			co_await tr->getRange(KeyRangeRef(keyForIndex(0), keyForIndex(nodes)), nodes);
+		} else if (type == 5) {
+			co_await tr->getRange(KeyRangeRef(keyForIndex(0), keyForIndex(nodes)), nodes);
+			for (i = 0; i < nodes; i++) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 6) {
+			co_await tr->getRange(KeyRangeRef(keyForIndex(0), keyForIndex(nodes)), nodes);
+			for (i = 0; i < nodes; i += 2) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 7) {
+			co_await tr->getRange(KeyRangeRef(keyForIndex(0), keyForIndex(nodes)), nodes);
+			for (i = 0; i < nodes; i++) {
+				tr->clear(keyForIndex(i));
+			}
+		} else if (type == 8) {
+			co_await tr->getRange(KeyRangeRef(keyForIndex(0), keyForIndex(nodes)), nodes);
+			for (i = 0; i < nodes; i += 2) {
+				tr->clear(KeyRangeRef(keyForIndex(i), keyForIndex(i + 1)));
+			}
+		} else if (type == 9) {
+			std::vector<Future<RangeResult>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->getRange(KeyRangeRef(keyForIndex(i), keyForIndex(i + 2)), nodes));
+			}
+			co_await waitForAll(gets);
+		} else if (type == 10) {
+			std::vector<Future<RangeResult>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->getRange(KeyRangeRef(keyForIndex(i), keyForIndex(i + 2)), nodes));
+			}
+			co_await waitForAll(gets);
+			for (i = 0; i < nodes; i++) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 11) {
+			std::vector<Future<RangeResult>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->getRange(KeyRangeRef(keyForIndex(i), keyForIndex(i + 2)), nodes));
+			}
+			co_await waitForAll(gets);
+			for (i = 0; i < nodes; i += 2) {
+				tr->set(keyForIndex(i), "foo"_sr);
+			}
+		} else if (type == 12) {
+			std::vector<Future<RangeResult>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->getRange(KeyRangeRef(keyForIndex(i), keyForIndex(i + 2)), nodes));
+			}
+			co_await waitForAll(gets);
+			for (i = 0; i < nodes; i++) {
+				tr->clear(keyForIndex(i));
+			}
+		} else if (type == 13) {
+			std::vector<Future<RangeResult>> gets;
+			for (i = 0; i < nodes; i++) {
+				gets.push_back(tr->getRange(KeyRangeRef(keyForIndex(i), keyForIndex(i + 2)), nodes));
+			}
+			co_await waitForAll(gets);
+			for (i = 0; i < nodes; i += 2) {
+				tr->clear(KeyRangeRef(keyForIndex(i), keyForIndex(i + 1)));
+			}
+		}
+	}
+
+	Future<Void> test_get_single(Database cx, int cacheType) {
+		int i{ 0 };
+		ReadYourWritesTransaction tr(cx);
+
+		while (true) {
+			Error err;
+			try {
+				co_await fillCache(&tr, cacheType);
+
+				double startTime = timer();
+
+				for (i = 0; i < nodes; i++) {
+					co_await tr.get(keyForIndex(nodes / 2));
+				}
+
+				fprintf(stderr, "%f", nodes / (timer() - startTime));
+
+				co_return;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+
+	Future<Void> test_get_many_sequential(Database cx, int cacheType) {
+		int i{ 0 };
+		ReadYourWritesTransaction tr(cx);
+
+		while (true) {
+			Error err;
+			try {
+				co_await fillCache(&tr, cacheType);
+
+				double startTime = timer();
+
+				for (i = 0; i < nodes; i++) {
+					co_await tr.get(keyForIndex(i));
+				}
+
+				fprintf(stderr, "%f", nodes / (timer() - startTime));
+
+				co_return;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+
+	Future<Void> test_get_range_basic(Database cx, int cacheType) {
+		int i{ 0 };
+		ReadYourWritesTransaction tr(cx);
+
+		while (true) {
+			Error err;
+			try {
+				co_await fillCache(&tr, cacheType);
+
+				double startTime = timer();
+
+				for (i = 0; i < ranges; i++) {
+					co_await tr.getRange(KeyRangeRef(keyForIndex(0), keyForIndex(nodes)), nodes);
+				}
+
+				fprintf(stderr, "%f", ranges / (timer() - startTime));
+
+				co_return;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+
+	Future<Void> test_interleaved_sets_gets(Database cx, int cacheType) {
+		int i{ 0 };
+		ReadYourWritesTransaction tr(cx);
+
+		while (true) {
+			Error err;
+			try {
+				co_await fillCache(&tr, cacheType);
+
+				tr.set(keyForIndex(nodes / 2), keyForIndex(nodes));
+
+				double startTime = timer();
+
+				for (i = 0; i < nodes; i++) {
+					co_await tr.get(keyForIndex(nodes / 2));
+					tr.set(keyForIndex(nodes / 2), keyForIndex(i));
+				}
+
+				fprintf(stderr, "%f", nodes / (timer() - startTime));
+
+				co_return;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+
+	Future<Void> _start(Database cx) {
+		int i{ 0 };
+		fprintf(stderr, "test_get_single, ");
+		for (i = 0; i < 14; i++) {
+			co_await test_get_single(cx, i);
+			if (i == 13)
+				fprintf(stderr, "\n");
+			else
+				fprintf(stderr, ", ");
+		}
+		fprintf(stderr, "test_get_many_sequential, ");
+		for (i = 0; i < 14; i++) {
+			co_await test_get_many_sequential(cx, i);
+			if (i == 13)
+				fprintf(stderr, "\n");
+			else
+				fprintf(stderr, ", ");
+		}
+		fprintf(stderr, "test_get_range_basic, ");
+		for (i = 4; i < 14; i++) {
+			co_await test_get_range_basic(cx, i);
+			if (i == 13)
+				fprintf(stderr, "\n");
+			else
+				fprintf(stderr, ", ");
+		}
+		fprintf(stderr, "test_interleaved_sets_gets, ");
+		for (i = 0; i < 14; i++) {
+			co_await test_interleaved_sets_gets(cx, i);
+			if (i == 13)
+				fprintf(stderr, "\n");
+			else
+				fprintf(stderr, ", ");
+		}
+	}
+
+	Future<bool> check(Database const& cx) override { return true; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	Key keyForIndex(uint64_t index) {
+		Key result = makeString(keyBytes);
+		uint8_t* data = mutateString(result);
+		memset(data, '.', keyBytes);
+
+		double d = double(index) / nodes;
+		emplaceIndex(data, 0, *(int64_t*)&d);
+
+		return result;
+	}
+};
+
+WorkloadFactory<RYWPerformanceWorkload> RYWPerformanceWorkloadFactory;
diff --git a/fdbserver/workloads/RandomClogging.actor.cpp b/fdbserver/workloads/RandomClogging.cpp
similarity index 67%
rename from fdbserver/workloads/RandomClogging.actor.cpp
rename to fdbserver/workloads/RandomClogging.cpp
index 6756238534b..65913e9ccc7 100644
--- a/fdbserver/workloads/RandomClogging.actor.cpp
+++ b/fdbserver/workloads/RandomClogging.cpp
@@ -1,5 +1,5 @@
 /*
- * RandomClogging.actor.cpp
+ * RandomClogging.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,10 +21,8 @@
 #include "flow/DeterministicRandom.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 struct RandomCloggingWorkload : FailureInjectionWorkload {
 	static constexpr auto NAME = "RandomClogging";
@@ -38,7 +36,7 @@ struct RandomCloggingWorkload : FailureInjectionWorkload {
 
 	RandomCloggingWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {}
 
-	RandomCloggingWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+	explicit RandomCloggingWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		testDuration = getOption(options, "testDuration"_sr, testDuration);
 		scale = getOption(options, "scale"_sr, scale);
@@ -63,32 +61,30 @@ struct RandomCloggingWorkload : FailureInjectionWorkload {
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override {
 		if (g_network->isSimulated() && enabled) {
-			return _start(this);
+			return startImpl();
 		}
 		return Void();
 	}
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> _start(RandomCloggingWorkload* self) {
-		state Future<Void> done = delay(self->maxRunDuration);
-		loop {
-			wait(done ||
-			     timeout(reportErrors(self->swizzleClog ? self->swizzleClogClient(self) : self->clogClient(self),
-			                          "RandomCloggingError"),
-			             self->testDuration,
-			             Void()));
-			if (!done.isReady() && self->iterate) {
-				wait(delay(self->suspend));
-				self->suspend *= self->backoff;
-			} else {
-				return Void();
+	Future<Void> startImpl() {
+		Future<Void> done = delay(maxRunDuration);
+		while (true) {
+			co_await (done ||
+			          timeout(reportErrors(swizzleClog ? swizzleClogClient() : clogClient(), "RandomCloggingError"),
+			                  testDuration,
+			                  Void()));
+			if (done.isReady() || !iterate) {
+				break;
 			}
+			co_await delay(suspend);
+			suspend *= backoff;
 		}
 	}
 
-	ACTOR void doClog(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) {
-		wait(::delay(delay));
+	Future<Void> doClog(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) {
+		co_await ::delay(delay);
 		g_simulator->clogInterface(machine->address.ip, t);
 	}
 
@@ -99,28 +95,28 @@ struct RandomCloggingWorkload : FailureInjectionWorkload {
 			g_simulator->clogPair(m1->address.ip, m2->address.ip, t);
 	}
 
-	ACTOR Future<Void> clogClient(RandomCloggingWorkload* self) {
-		state double lastTime = now();
-		state double workloadEnd = now() + self->testDuration;
-		loop {
-			wait(poisson(&lastTime, self->scale / self->clogginess));
+	Future<Void> clogClient() {
+		double lastTime = now();
+		double workloadEnd = now() + testDuration;
+		while (true) {
+			co_await poisson(&lastTime, scale / clogginess);
 			auto machine = deterministicRandom()->randomChoice(g_simulator->getAllProcesses());
-			double t = self->scale * 10.0 * exp(-10.0 * deterministicRandom()->random01());
+			double t = scale * 10.0 * exp(-10.0 * deterministicRandom()->random01());
 			t = std::max(0.0, std::min(t, workloadEnd - now()));
-			self->doClog(machine, t);
+			doClog(machine, t);
 
-			t = self->scale * 20.0 * exp(-10.0 * deterministicRandom()->random01());
+			t = scale * 20.0 * exp(-10.0 * deterministicRandom()->random01());
 			t = std::max(0.0, std::min(t, workloadEnd - now()));
-			self->clogRandomPair(t);
+			clogRandomPair(t);
 		}
 	}
 
-	ACTOR Future<Void> swizzleClogClient(RandomCloggingWorkload* self) {
-		state double lastTime = now();
-		state double workloadEnd = now() + self->testDuration;
-		loop {
-			wait(poisson(&lastTime, self->scale / self->clogginess));
-			double t = self->scale * 10.0 * exp(-10.0 * deterministicRandom()->random01());
+	Future<Void> swizzleClogClient() {
+		double lastTime = now();
+		double workloadEnd = now() + testDuration;
+		while (true) {
+			co_await poisson(&lastTime, scale / clogginess);
+			double t = scale * 10.0 * exp(-10.0 * deterministicRandom()->random01());
 			t = std::max(0.0, std::min(t, workloadEnd - now()));
 
 			// randomly choose half of the machines in the cluster to all clog up,
@@ -134,10 +130,10 @@ struct RandomCloggingWorkload : FailureInjectionWorkload {
 					ends.push_back(deterministicRandom()->random01() * t / 2 + t / 2);
 				}
 			for (int i = 0; i < 10; i++)
-				self->clogRandomPair(t);
+				clogRandomPair(t);
 
 			for (int i = 0; i < swizzled.size(); i++)
-				self->doClog(swizzled[i], ends[i] - starts[i], starts[i]);
+				doClog(swizzled[i], ends[i] - starts[i], starts[i]);
 		}
 	}
 };
diff --git a/fdbserver/workloads/RandomMoveKeys.actor.cpp b/fdbserver/workloads/RandomMoveKeys.cpp
similarity index 80%
rename from fdbserver/workloads/RandomMoveKeys.actor.cpp
rename to fdbserver/workloads/RandomMoveKeys.cpp
index 5cdaf3d3210..83296f31ea0 100644
--- a/fdbserver/workloads/RandomMoveKeys.actor.cpp
+++ b/fdbserver/workloads/RandomMoveKeys.cpp
@@ -1,5 +1,5 @@
 /*
- * RandomMoveKeys.actor.cpp
+ * RandomMoveKeys.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,15 +21,14 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/StorageServerInterface.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbserver/MoveKeys.actor.h"
+#include "fdbclient/ManagementAPI.h"
+#include "fdbserver/core/MoveKeys.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/QuietDatabase.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/QuietDatabase.h"
 #include "flow/DeterministicRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct MoveKeysWorkload : FailureInjectionWorkload {
 	static constexpr auto NAME = "RandomMoveKeys";
@@ -43,7 +42,7 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 		enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
 	}
 
-	MoveKeysWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+	explicit MoveKeysWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId && g_network->isSimulated(); // only do this on the "first" client
 		meanDelay = getOption(options, "meanDelay"_sr, meanDelay);
 		testDuration = getOption(options, "testDuration"_sr, testDuration);
@@ -51,7 +50,6 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 
 	bool shouldInject(DeterministicRandom& random,
 	                  const WorkloadRequest& work,
@@ -59,34 +57,34 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 		return alreadyAdded < 1 && work.useDatabase && 0.1 / (1 + alreadyAdded) > random.random01();
 	}
 
-	ACTOR Future<Void> _start(Database cx, MoveKeysWorkload* self) {
-		if (self->enabled) {
+	Future<Void> start(Database const& cx) override {
+		if (enabled) {
 			// Get the database configuration so as to use proper team size
-			state Transaction tr(cx);
-			loop {
+			Transaction tr(cx);
+			while (true) {
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
+				Error err;
 				try {
-					RangeResult res = wait(tr.getRange(configKeys, 1000));
+					RangeResult res = co_await tr.getRange(configKeys, 1000);
 					ASSERT(res.size() < 1000);
 					for (int i = 0; i < res.size(); i++)
-						self->configuration.set(res[i].key, res[i].value);
+						configuration.set(res[i].key, res[i].value);
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 
-			state int oldMode = wait(setDDMode(cx, 0));
+			int oldMode = co_await setDDMode(cx, 0);
 			TraceEvent("RMKStartModeSetting").log();
-			wait(timeout(
-			    reportErrors(self->worker(cx, self), "MoveKeysWorkloadWorkerError"), self->testDuration, Void()));
+			co_await timeout(reportErrors(worker(cx, this), "MoveKeysWorkloadWorkerError"), testDuration, Void());
 			// Always set the DD mode back, even if we die with an error
 			TraceEvent("RMKDoneMoving").log();
-			wait(success(setDDMode(cx, oldMode)));
+			co_await setDDMode(cx, oldMode);
 			TraceEvent("RMKDoneModeSetting").log();
 		}
-		return Void();
 	}
 
 	double getCheckTimeout() const override { return testDuration / 2 + 1; }
@@ -113,7 +111,7 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 		std::set<StorageServerInterface> t;
 		std::set<Optional<Standalone<StringRef>>> machines;
 		std::set<Optional<Standalone<StringRef>>> dataHalls;
-		while (t.size() < teamSize && storageServers.size()) {
+		while (t.size() < teamSize && !storageServers.empty()) {
 			auto s = storageServers.back();
 			storageServers.pop_back();
 			if (!machines.contains(s.locality.zoneId()) &&
@@ -134,14 +132,14 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 		return std::vector<StorageServerInterface>(t.begin(), t.end());
 	}
 
-	ACTOR Future<Void> doMoveKeys(Database cx,
-	                              MoveKeysWorkload* self,
-	                              KeyRange keys,
-	                              std::vector<StorageServerInterface> destinationTeam,
-	                              MoveKeysLock lock) {
-		state TraceInterval relocateShardInterval("RelocateShard");
-		state FlowLock fl1(1);
-		state FlowLock fl2(1);
+	Future<Void> doMoveKeys(Database cx,
+	                        MoveKeysWorkload* self,
+	                        KeyRange keys,
+	                        std::vector<StorageServerInterface> destinationTeam,
+	                        MoveKeysLock lock) {
+		TraceInterval relocateShardInterval("RelocateShard");
+		FlowLock fl1(1);
+		FlowLock fl2(1);
 		std::string desc;
 		for (int s = 0; s < destinationTeam.size(); s++)
 			desc +=
@@ -159,8 +157,8 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 		    .detail("DestinationTeam", desc);
 
 		try {
-			state Promise<Void> signal;
-			state DDEnabledState ddEnabledState;
+			Promise<Void> signal;
+			DDEnabledState ddEnabledState;
 			std::unique_ptr<MoveKeysParams> params;
 			if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) {
 				UID dataMoveId =
@@ -204,9 +202,9 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 				                                          CancelConflictingDataMoves::True,
 				                                          Optional<BulkLoadTaskState>());
 			}
-			wait(moveKeys(cx, *params));
+			co_await moveKeys(cx, *params);
 			TraceEvent(relocateShardInterval.end()).detail("Result", "Success");
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			TraceEvent(relocateShardInterval.end(), self->dbInfo->get().master.id()).errorUnsuppressed(e);
 			throw;
@@ -228,34 +226,35 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 		servers.resize(o);
 	}
 
-	ACTOR Future<Void> forceMasterFailure(Database cx, MoveKeysWorkload* self) {
+	Future<Void> forceMasterFailure(Database cx, MoveKeysWorkload* self) {
 		ASSERT(g_network->isSimulated());
-		loop {
+		while (true) {
 			if (g_simulator->killZone(self->dbInfo->get().master.locality.zoneId(), ISimulator::KillType::Reboot, true))
-				return Void();
-			wait(delay(1.0));
+				co_return;
+			co_await delay(1.0);
 		}
 	}
 
-	ACTOR Future<Void> worker(Database cx, MoveKeysWorkload* self) {
-		state KeyRangeMap<std::vector<StorageServerInterface>> inFlight;
-		state KeyRangeActorMap inFlightActors;
-		state double lastTime = now();
+	Future<Void> worker(Database cx, MoveKeysWorkload* self) {
+		KeyRangeMap<std::vector<StorageServerInterface>> inFlight;
+		KeyRangeActorMap inFlightActors;
+		double lastTime = now();
 
 		ASSERT(self->configuration.storageTeamSize > 0);
 
 		if (self->configuration.usableRegions > 1) { // FIXME: add support for generating random teams across DCs
-			return Void();
+			co_return;
 		}
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state MoveKeysLock lock = wait(takeMoveKeysLock(cx, UID()));
-				state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+				MoveKeysLock lock = co_await takeMoveKeysLock(cx, UID());
+				std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
 				eliminateDuplicates(storageServers);
 
-				loop {
-					wait(poisson(&lastTime, self->meanDelay));
+				while (true) {
+					co_await poisson(&lastTime, self->meanDelay);
 
 					KeyRange keys = self->getRandomKeys();
 					std::vector<StorageServerInterface> team =
@@ -272,11 +271,15 @@ struct MoveKeysWorkload : FailureInjectionWorkload {
 					}
 				}
 			} catch (Error& e) {
-				if (e.code() != error_code_movekeys_conflict && e.code() != error_code_operation_failed)
-					throw;
-				wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				// Keep trying to get the moveKeysLock
+				err = e;
 			}
+			if (!err.isValid()) {
+				continue;
+			}
+			if (err.code() != error_code_movekeys_conflict && err.code() != error_code_operation_failed)
+				throw err;
+			co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+			// Keep trying to get the moveKeysLock
 		}
 	}
 };
diff --git a/fdbserver/workloads/RandomRangeLock.actor.cpp b/fdbserver/workloads/RandomRangeLock.cpp
similarity index 79%
rename from fdbserver/workloads/RandomRangeLock.actor.cpp
rename to fdbserver/workloads/RandomRangeLock.cpp
index 0aaaf9aea5d..3f34a1efe9b 100644
--- a/fdbserver/workloads/RandomRangeLock.actor.cpp
+++ b/fdbserver/workloads/RandomRangeLock.cpp
@@ -1,5 +1,5 @@
 /*
- * RandomRangeLock.actor.cpp
+ * RandomRangeLock.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,17 +19,16 @@
  */
 
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/RangeLock.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct RandomRangeLockWorkload : FailureInjectionWorkload {
 	static constexpr auto NAME = "RandomRangeLock";
@@ -46,14 +45,13 @@ struct RandomRangeLockWorkload : FailureInjectionWorkload {
 		enabled &= (clientId == 0) && g_network->isSimulated();
 	}
 
-	RandomRangeLockWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+	explicit RandomRangeLockWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = (clientId == 0) && g_network->isSimulated();
 		maxLockDuration = getOption(options, "maxLockDuration"_sr, maxLockDuration);
 		maxStartDelay = getOption(options, "maxStartDelay"_sr, maxStartDelay);
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
@@ -83,26 +81,26 @@ struct RandomRangeLockWorkload : FailureInjectionWorkload {
 		}
 	}
 
-	ACTOR Future<Void> lockActor(Database cx, RandomRangeLockWorkload* self, std::string rangeLockOwnerNamePrefix) {
-		state double testDuration = deterministicRandom()->random01() * self->maxLockDuration;
-		state double testStartDelay = deterministicRandom()->random01() * self->maxStartDelay;
-		state std::string rangeLockOwnerName =
+	Future<Void> lockActor(Database cx, RandomRangeLockWorkload* self, std::string rangeLockOwnerNamePrefix) {
+		double testDuration = deterministicRandom()->random01() * self->maxLockDuration;
+		double testStartDelay = deterministicRandom()->random01() * self->maxStartDelay;
+		std::string rangeLockOwnerName =
 		    rangeLockOwnerNamePrefix + "-" + std::to_string(deterministicRandom()->randomInt(0, self->lockActorCount));
 		// Here we intentionally introduced duplicated owner name between different lockActor
 		std::string lockOwnerDescription = rangeLockOwnerName + ":" + self->getRandomStringRef().toString();
-		wait(registerRangeLockOwner(cx, rangeLockOwnerName, lockOwnerDescription));
-		wait(delay(testStartDelay));
-		state KeyRange range = self->getRandomRange(self);
+		co_await registerRangeLockOwner(cx, rangeLockOwnerName, lockOwnerDescription);
+		co_await delay(testStartDelay);
+		KeyRange range = self->getRandomRange(self);
 		TraceEvent(SevWarnAlways, "InjectRangeLockSubmit")
 		    .detail("RangeLockOwnerName", rangeLockOwnerName)
 		    .detail("Range", range)
 		    .detail("LockStartDelayTime", testStartDelay)
 		    .detail("LockTime", testDuration);
 		try {
-			Optional<RangeLockOwner> owner = wait(getRangeLockOwner(cx, rangeLockOwnerName));
+			Optional<RangeLockOwner> owner = co_await getRangeLockOwner(cx, rangeLockOwnerName);
 			ASSERT(owner.present());
 			ASSERT(owner.get().getOwnerUniqueId() == rangeLockOwnerName);
-			wait(takeExclusiveReadLockOnRange(cx, range, rangeLockOwnerName));
+			co_await takeExclusiveReadLockOnRange(cx, range, rangeLockOwnerName);
 			TraceEvent(SevWarnAlways, "InjectRangeLocked")
 			    .detail("RangeLockOwnerName", rangeLockOwnerName)
 			    .detail("Range", range)
@@ -130,7 +128,7 @@ struct RandomRangeLockWorkload : FailureInjectionWorkload {
 				throw e;
 			}
 		}
-		wait(delay(testDuration));
+		co_await delay(testDuration);
 
 		TraceEvent(SevWarnAlways, "InjectRangeUnlockSubmit")
 		    .detail("RangeLockOwnerName", rangeLockOwnerName)
@@ -138,7 +136,7 @@ struct RandomRangeLockWorkload : FailureInjectionWorkload {
 		    .detail("LockStartDelayTime", testStartDelay)
 		    .detail("LockTime", testDuration);
 		try {
-			wait(releaseExclusiveReadLockOnRange(cx, range, rangeLockOwnerName));
+			co_await releaseExclusiveReadLockOnRange(cx, range, rangeLockOwnerName);
 			TraceEvent(SevWarnAlways, "InjectRangeUnlocked")
 			    .detail("RangeLockOwnerName", rangeLockOwnerName)
 			    .detail("Range", range);
@@ -165,12 +163,10 @@ struct RandomRangeLockWorkload : FailureInjectionWorkload {
 				throw e;
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, RandomRangeLockWorkload* self) {
-		if (self->enabled) {
+	Future<Void> start(Database const& cx) override {
+		if (enabled) {
 			// Run lockActorCount number of actor concurrently.
 			// Each actor conducts (1) locking a range for a while and (2) unlocking the range.
 			// Each actor randomly generate a uniqueId as the lock owner.
@@ -180,24 +176,23 @@ struct RandomRangeLockWorkload : FailureInjectionWorkload {
 			// The rangeLock mechanism should approperiately handled those conflict.
 			// When all actors complete, it is expected that all locks are removed,
 			// and this injected workload should not block other workloads.
-			state std::string rangeLockOwnerNamePrefix = "Owner" + std::to_string(self->clientId);
+			std::string rangeLockOwnerNamePrefix = "Owner" + std::to_string(clientId);
 			std::vector<Future<Void>> actors;
-			for (int i = 0; i < self->lockActorCount; i++) {
-				actors.push_back(self->lockActor(cx, self, rangeLockOwnerNamePrefix));
+			for (int i = 0; i < lockActorCount; i++) {
+				actors.push_back(lockActor(cx, this, rangeLockOwnerNamePrefix));
 			}
-			wait(waitForAll(actors));
+			co_await waitForAll(actors);
 
 			// Make sure all ranges locked by the workload client are unlocked
-			state int j = 0;
-			for (; j < self->lockActorCount; j++) {
-				std::vector<std::pair<KeyRange, RangeLockState>> res = wait(
-				    findExclusiveReadLockOnRange(cx, normalKeys, rangeLockOwnerNamePrefix + "-" + std::to_string(j)));
+			int j = 0;
+			for (; j < lockActorCount; j++) {
+				std::vector<std::pair<KeyRange, RangeLockState>> res = co_await findExclusiveReadLockOnRange(
+				    cx, normalKeys, rangeLockOwnerNamePrefix + "-" + std::to_string(j));
 				ASSERT(res.empty());
 			}
 
 			TraceEvent("RandomRangeLockWorkloadEnd").detail("OwnerPrefix", rangeLockOwnerNamePrefix);
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/RandomSelector.actor.cpp b/fdbserver/workloads/RandomSelector.cpp
similarity index 65%
rename from fdbserver/workloads/RandomSelector.actor.cpp
rename to fdbserver/workloads/RandomSelector.cpp
index bd910409c09..b04d4122be5 100644
--- a/fdbserver/workloads/RandomSelector.actor.cpp
+++ b/fdbserver/workloads/RandomSelector.cpp
@@ -1,5 +1,5 @@
 /*
- * RandomSelector.actor.cpp
+ * RandomSelector.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,9 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct RandomSelectorWorkload : TestWorkload {
 	static constexpr auto NAME = "RandomSelector";
@@ -35,7 +34,7 @@ struct RandomSelectorWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter transactions, retries;
 
-	RandomSelectorWorkload(WorkloadContext const& wcx)
+	explicit RandomSelectorWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries") {
 
 		minOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 10);
@@ -65,53 +64,54 @@ struct RandomSelectorWorkload : TestWorkload {
 		m.push_back(retries.getMetric());
 	}
 
-	ACTOR Future<Void> randomSelectorSetup(Database cx, RandomSelectorWorkload* self) {
-		state Value myValue = StringRef(format("%d", deterministicRandom()->randomInt(0, 10000000)));
-		state Transaction tr(cx);
-		state std::string clientID;
+	Future<Void> randomSelectorSetup(Database cx, RandomSelectorWorkload* self) {
+		Value myValue = StringRef(format("%d", deterministicRandom()->randomInt(0, 10000000)));
+		Transaction tr(cx);
+		std::string clientID;
 
 		clientID = format("%08d", self->clientId);
-		loop {
+		while (true) {
+			Error err;
 			try {
 				for (int i = 0; i < self->maxOffset; i++) {
 					tr.set(StringRef(clientID + "a/" + format("%010d", i)), myValue);
 					tr.set(StringRef(clientID + "c/" + format("%010d", i)), myValue);
 					tr.set(StringRef(clientID + "e/" + format("%010d", i)), myValue);
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
-				tr.reset();
+				err = e;
 			}
+			co_await tr.onError(err);
+			tr.reset();
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> randomSelectorClient(Database cx, RandomSelectorWorkload* self) {
-		state int i;
-		state int j;
-		state std::string clientID;
-		state std::string myKeyA;
-		state std::string myKeyB;
-		state std::string myValue;
-		state std::string myRandomIDKey;
-		state bool onEqualA;
-		state bool onEqualB;
-		state int offsetA;
-		state int offsetB;
-		state int randomLimit;
-		state int randomByteLimit;
-		state Reverse reverse = Reverse::False;
-		state Error error;
+	Future<Void> randomSelectorClient(Database cx, RandomSelectorWorkload* self) {
+		int i{ 0 };
+		int j{ 0 };
+		std::string clientID;
+		std::string myKeyA;
+		std::string myKeyB;
+		std::string myValue;
+		std::string myRandomIDKey;
+		bool onEqualA{ false };
+		bool onEqualB{ false };
+		int offsetA{ 0 };
+		int offsetB{ 0 };
+		int randomLimit{ 0 };
+		int randomByteLimit{ 0 };
+		Reverse reverse = Reverse::False;
+		Error error;
 
 		clientID = format("%08d", self->clientId);
 
-		loop {
-			state Transaction tr(cx);
+		while (true) {
+			Transaction tr(cx);
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr.clear(KeyRangeRef(StringRef(clientID + "b/"), StringRef(clientID + "c/")));
 					tr.clear(KeyRangeRef(StringRef(clientID + "d/"), StringRef(clientID + "e/")));
@@ -124,16 +124,18 @@ struct RandomSelectorWorkload : TestWorkload {
 						tr.set(StringRef(clientID + "d/" + myKeyA), myValue);
 						//TraceEvent("RYOWInit").detail("Key",myKeyA).detail("Value",myValue);
 					}
-					wait(tr.commit());
+					co_await tr.commit();
 					tr.reset();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 
-			state ReadYourWritesTransaction trRYOW(cx);
+			ReadYourWritesTransaction trRYOW(cx);
 
+			Error err;
 			try {
 				for (i = 0; i < deterministicRandom()->randomInt(self->minOperationsPerTransaction,
 				                                                 self->maxOperationsPerTransaction + 1);
@@ -146,30 +148,34 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWset").detail("Key",myKeyA).detail("Value",myValue);
 						trRYOW.set(StringRef(clientID + "b/" + myKeyA), myValue);
 
-						loop {
+						while (true) {
+							Error err;
 							try {
 								tr.set(StringRef(clientID + "d/" + myKeyA), myValue);
-								wait(tr.commit());
+								co_await tr.commit();
 								tr.reset();
 								break;
 							} catch (Error& e) {
-								wait(tr.onError(e));
+								err = e;
 							}
+							co_await tr.onError(err);
 						}
 					} else if (j < 4) {
 						myKeyA = format("%010d", deterministicRandom()->randomInt(0, self->maxKeySpace + 1));
 						//TraceEvent("RYOWclear").detail("Key",myKeyA);
 						trRYOW.clear(StringRef(clientID + "b/" + myKeyA));
 
-						loop {
+						while (true) {
+							Error err;
 							try {
 								tr.clear(StringRef(clientID + "d/" + myKeyA));
-								wait(tr.commit());
+								co_await tr.commit();
 								tr.reset();
 								break;
 							} catch (Error& e) {
-								wait(tr.onError(e));
+								err = e;
 							}
+							co_await tr.onError(err);
 						}
 
 					} else if (j < 5) {
@@ -182,44 +188,52 @@ struct RandomSelectorWorkload : TestWorkload {
 						trRYOW.clear(
 						    KeyRangeRef(StringRef(clientID + "b/" + myKeyA), StringRef(clientID + "b/" + myKeyB)));
 
-						loop {
+						while (true) {
+							Error err;
 							try {
 								tr.clear(KeyRangeRef(StringRef(clientID + "d/" + myKeyA),
 								                     StringRef(clientID + "d/" + myKeyB)));
-								wait(tr.commit());
+								co_await tr.commit();
 								tr.reset();
 								break;
 							} catch (Error& e) {
-								wait(tr.onError(e));
+								err = e;
 							}
+							co_await tr.onError(err);
 						}
 					} else if (j < 6) {
 						myKeyA = format("%010d", deterministicRandom()->randomInt(0, self->maxKeySpace + 1));
 
-						state Optional<Value> getTest1;
+						Optional<Value> getTest1;
 
-						Optional<Value> getTest = wait(trRYOW.get(StringRef(clientID + "b/" + myKeyA)));
+						Optional<Value> getTest = co_await trRYOW.get(StringRef(clientID + "b/" + myKeyA));
 						getTest1 = getTest;
 
-						loop {
-							try {
-								Optional<Value> getTest2 = wait(tr.get(StringRef(clientID + "d/" + myKeyA)));
+						while (true) {
+							{
+								Error err;
+								try {
+									Optional<Value> getTest2 = co_await tr.get(StringRef(clientID + "d/" + myKeyA));
 
-								if ((getTest1.present() && (!getTest2.present() || getTest1.get() != getTest2.get())) ||
-								    (!getTest1.present() && getTest2.present())) {
-									TraceEvent(SevError, "RanSelTestFailure")
-									    .detail("Reason", "The get results did not match")
-									    .detail("KeyA", myKeyA)
-									    .detail("RYOW", getTest1.present() ? printable(getTest1.get()) : "Not Present")
-									    .detail("Regular",
-									            getTest2.present() ? printable(getTest2.get()) : "Not Present");
-									self->fail = true;
-								}
+									if ((getTest1.present() &&
+									     (!getTest2.present() || getTest1.get() != getTest2.get())) ||
+									    (!getTest1.present() && getTest2.present())) {
+										TraceEvent(SevError, "RanSelTestFailure")
+										    .detail("Reason", "The get results did not match")
+										    .detail("KeyA", myKeyA)
+										    .detail("RYOW",
+										            getTest1.present() ? printable(getTest1.get()) : "Not Present")
+										    .detail("Regular",
+										            getTest2.present() ? printable(getTest2.get()) : "Not Present");
+										self->fail = true;
+									}
 
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								wait(tr.onError(e));
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								co_await tr.onError(err);
 								tr.reset();
 							}
 						}
@@ -230,18 +244,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWadd").detail("Key",myKeyA).detail("Value", "\\x01");
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::AddValue);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::AddValue);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::AddValue);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -254,18 +272,23 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWappendIfFits").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::AppendIfFits);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::AppendIfFits);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(
+									    StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::AppendIfFits);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -278,18 +301,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWand").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::And);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::And);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::And);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -302,18 +329,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWor").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::Or);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Or);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Or);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -326,18 +357,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWxor").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::Xor);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Xor);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Xor);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -350,18 +385,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWmax").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::Max);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Max);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Max);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -374,18 +413,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWmin").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::Min);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Min);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::Min);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -398,18 +441,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWbytemin").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::ByteMin);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::ByteMin);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::ByteMin);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -422,18 +469,22 @@ struct RandomSelectorWorkload : TestWorkload {
 						//TraceEvent("RYOWbytemax").detail("Key",myKeyA).detail("Value", myValue);
 						trRYOW.atomicOp(StringRef(clientID + "b/" + myKeyA), myValue, MutationRef::ByteMax);
 
-						loop {
-							try {
-								tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
-								tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::ByteMax);
-								wait(tr.commit());
-								tr.reset();
-								break;
-							} catch (Error& e) {
-								error = e;
-								wait(tr.onError(e));
+						while (true) {
+							{
+								Error err;
+								try {
+									tr.set(StringRef(clientID + "z/" + myRandomIDKey), StringRef());
+									tr.atomicOp(StringRef(clientID + "d/" + myKeyA), myValue, MutationRef::ByteMax);
+									co_await tr.commit();
+									tr.reset();
+									break;
+								} catch (Error& e) {
+									err = e;
+								}
+								error = err;
+								co_await tr.onError(err);
 								if (error.code() == error_code_commit_unknown_result) {
-									Optional<Value> thing = wait(tr.get(StringRef(clientID + "z/" + myRandomIDKey)));
+									Optional<Value> thing = co_await tr.get(StringRef(clientID + "z/" + myRandomIDKey));
 									if (thing.present())
 										break;
 								}
@@ -455,23 +506,24 @@ struct RandomSelectorWorkload : TestWorkload {
 
 						//TraceEvent("RYOWgetRange").detail("KeyA", myKeyA).detail("KeyB", myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB).detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("RandomLimit",randomLimit).detail("RandomByteLimit", randomByteLimit).detail("Reverse", reverse);
 
-						state RangeResult getRangeTest1;
-						RangeResult getRangeTest =
-						    wait(trRYOW.getRange(KeySelectorRef(StringRef(clientID + "b/" + myKeyA), onEqualA, offsetA),
-						                         KeySelectorRef(StringRef(clientID + "b/" + myKeyB), onEqualB, offsetB),
-						                         randomLimit,
-						                         Snapshot::False,
-						                         reverse));
+						RangeResult getRangeTest1;
+						RangeResult getRangeTest = co_await trRYOW.getRange(
+						    KeySelectorRef(StringRef(clientID + "b/" + myKeyA), onEqualA, offsetA),
+						    KeySelectorRef(StringRef(clientID + "b/" + myKeyB), onEqualB, offsetB),
+						    randomLimit,
+						    Snapshot::False,
+						    reverse);
 						getRangeTest1 = getRangeTest;
 
-						loop {
+						while (true) {
+							Error err;
 							try {
-								RangeResult getRangeTest2 = wait(
-								    tr.getRange(KeySelectorRef(StringRef(clientID + "d/" + myKeyA), onEqualA, offsetA),
-								                KeySelectorRef(StringRef(clientID + "d/" + myKeyB), onEqualB, offsetB),
-								                randomLimit,
-								                Snapshot::False,
-								                reverse));
+								RangeResult getRangeTest2 = co_await tr.getRange(
+								    KeySelectorRef(StringRef(clientID + "d/" + myKeyA), onEqualA, offsetA),
+								    KeySelectorRef(StringRef(clientID + "d/" + myKeyB), onEqualB, offsetB),
+								    randomLimit,
+								    Snapshot::False,
+								    reverse);
 
 								bool fail = false;
 								if (getRangeTest1.size() != getRangeTest2.size()) {
@@ -523,24 +575,26 @@ struct RandomSelectorWorkload : TestWorkload {
 								tr.reset();
 								break;
 							} catch (Error& e) {
-								wait(tr.onError(e));
+								err = e;
 							}
+							co_await tr.onError(err);
 						}
 					}
 				}
 
-				wait(trRYOW.commit());
+				co_await trRYOW.commit();
 
 				++self->transactions;
 
-				state Transaction finalTransaction(cx);
+				Transaction finalTransaction(cx);
 
-				loop {
+				while (true) {
+					Error err;
 					try {
-						state RangeResult finalTest1 = wait(finalTransaction.getRange(
-						    KeyRangeRef(StringRef(clientID + "b/"), StringRef(clientID + "c/")), self->maxKeySpace));
-						RangeResult finalTest2 = wait(finalTransaction.getRange(
-						    KeyRangeRef(StringRef(clientID + "d/"), StringRef(clientID + "e/")), self->maxKeySpace));
+						RangeResult finalTest1 = co_await finalTransaction.getRange(
+						    KeyRangeRef(StringRef(clientID + "b/"), StringRef(clientID + "c/")), self->maxKeySpace);
+						RangeResult finalTest2 = co_await finalTransaction.getRange(
+						    KeyRangeRef(StringRef(clientID + "d/"), StringRef(clientID + "e/")), self->maxKeySpace);
 
 						if (finalTest1.size() != finalTest2.size()) {
 							TraceEvent(SevError, "RanSelTestFailure")
@@ -560,13 +614,17 @@ struct RandomSelectorWorkload : TestWorkload {
 							}
 						break;
 					} catch (Error& e) {
-						wait(finalTransaction.onError(e));
+						err = e;
 					}
+					co_await finalTransaction.onError(err);
 				}
 			} catch (Error& e) {
-				wait(trRYOW.onError(e));
-				++self->retries;
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await trRYOW.onError(err);
 			}
+			++self->retries;
 		}
 	}
 };
diff --git a/fdbserver/workloads/RangeLock.actor.cpp b/fdbserver/workloads/RangeLock.cpp
similarity index 72%
rename from fdbserver/workloads/RangeLock.actor.cpp
rename to fdbserver/workloads/RangeLock.cpp
index 2e9daa387b1..85335fbc9d5 100644
--- a/fdbserver/workloads/RangeLock.actor.cpp
+++ b/fdbserver/workloads/RangeLock.cpp
@@ -1,5 +1,5 @@
 /*
- * RangeLock.actor.cpp
+ * RangeLock.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,14 +18,13 @@
  * limitations under the License.
  */
 
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/RangeLock.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 struct RangeLocking : TestWorkload {
 	static constexpr auto NAME = "RangeLocking";
@@ -43,8 +42,8 @@ struct RangeLocking : TestWorkload {
 	struct KVOperation {
 		std::variant<KeyRange, KeyValue> params;
 
-		KVOperation(KeyRange range) : params(range) {}
-		KVOperation(KeyValue keyValue) : params(keyValue) {}
+		explicit KVOperation(KeyRange range) : params(range) {}
+		explicit KVOperation(KeyValue keyValue) : params(keyValue) {}
 
 		std::string toString() const {
 			std::string res = "KVOperation: ";
@@ -79,7 +78,7 @@ struct RangeLocking : TestWorkload {
 	std::vector<KVOperation> kvOperations;
 	std::map<Key, Value> kvs;
 
-	RangeLocking(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {
+	explicit RangeLocking(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {
 		lockedRangeMap.insert(allKeys, false);
 	}
 
@@ -87,68 +86,74 @@ struct RangeLocking : TestWorkload {
 		return registerRangeLockOwner(cx, rangeLockOwnerName, rangeLockOwnerName);
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> setKey(Database cx, Key key, Value value) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> setKey(Database cx, Key key, Value value) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.set(key, value);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> clearKey(Database cx, Key key) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> clearKey(Database cx, Key key) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.clear(key);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> clearRange(Database cx, KeyRange range) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> clearRange(Database cx, KeyRange range) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.clear(range);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Optional<Value>> getKey(Database cx, Key key) {
-		state Transaction tr(cx);
-		loop {
+	Future<Optional<Value>> getKey(Database cx, Key key) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				Optional<Value> value = wait(tr.get(key));
-				return value;
+				Optional<Value> value = co_await tr.get(key);
+				co_return value;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
 	std::string getLockRangesString(const std::vector<std::pair<KeyRange, RangeLockState>>& locks) {
 		std::string res = "";
 		int count = 0;
-		for (const auto& lock : locks) {
-			res = res + lock.first.toString();
+		for (const auto& [lockedRange, _lockState] : locks) {
+			res = res + lockedRange.toString();
 			if (count < locks.size()) {
 				res = res + ", ";
 			}
@@ -170,16 +175,15 @@ struct RangeLocking : TestWorkload {
 		return Standalone(KeyRangeRef(beginKey, endKey));
 	}
 
-	ACTOR Future<Void> updateDBWithRandomOperations(RangeLocking* self, Database cx) {
+	Future<Void> updateDBWithRandomOperations(RangeLocking* self, Database cx) {
 		self->kvOperations.clear();
-		state int iterationCount = deterministicRandom()->randomInt(1, 10);
-		state int i = 0;
-		for (; i < iterationCount; i++) {
-			state bool acceptedByDB = true;
+		int iterationCount = deterministicRandom()->randomInt(1, 10);
+		for (int i = 0; i < iterationCount; ++i) {
+			bool acceptedByDB = true;
 			if (deterministicRandom()->coinflip()) {
-				state KeyValue kv = self->getRandomKeyValue();
+				KeyValue kv = self->getRandomKeyValue();
 				try {
-					wait(self->setKey(cx, kv.key, kv.value));
+					co_await self->setKey(cx, kv.key, kv.value);
 				} catch (Error& e) {
 					if (e.code() != error_code_transaction_rejected_range_locked) {
 						throw e;
@@ -195,9 +199,9 @@ struct RangeLocking : TestWorkload {
 					    .detail("Accepted", acceptedByDB);
 				}
 			} else {
-				state KeyRange range = self->getRandomRange();
+				KeyRange range = self->getRandomRange();
 				try {
-					wait(self->clearRange(cx, range));
+					co_await self->clearRange(cx, range);
 				} catch (Error& e) {
 					if (e.code() != error_code_transaction_rejected_range_locked) {
 						throw e;
@@ -213,19 +217,17 @@ struct RangeLocking : TestWorkload {
 				}
 			}
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> updateLockMapWithRandomOperation(RangeLocking* self, Database cx) {
+	Future<Void> updateLockMapWithRandomOperation(RangeLocking* self, Database cx) {
 		self->lockRangeOperations.clear();
-		state int i = 0;
-		state int iterationCount = deterministicRandom()->randomInt(1, 10);
-		for (; i < iterationCount; i++) {
-			state KeyRange range = self->getRandomRange();
-			state bool lock = deterministicRandom()->coinflip();
+		int iterationCount = deterministicRandom()->randomInt(1, 10);
+		for (int i = 0; i < iterationCount; ++i) {
+			KeyRange range = self->getRandomRange();
+			bool lock = deterministicRandom()->coinflip();
 			if (lock) {
 				try {
-					wait(takeExclusiveReadLockOnRange(cx, range, self->rangeLockOwnerName));
+					co_await takeExclusiveReadLockOnRange(cx, range, self->rangeLockOwnerName);
 					if (self->verboseLogging) {
 						TraceEvent("RangeLockWorkLoadHistory").detail("Ops", "Lock").detail("Range", range);
 					}
@@ -242,7 +244,7 @@ struct RangeLocking : TestWorkload {
 				}
 			} else {
 				try {
-					wait(releaseExclusiveReadLockOnRange(cx, range, self->rangeLockOwnerName));
+					co_await releaseExclusiveReadLockOnRange(cx, range, self->rangeLockOwnerName);
 					if (self->verboseLogging) {
 						TraceEvent("RangeLockWorkLoadHistory").detail("Ops", "Unlock").detail("Range", range);
 					}
@@ -260,7 +262,6 @@ struct RangeLocking : TestWorkload {
 			}
 			self->lockRangeOperations.push_back(LockRangeOperation(range, lock));
 		}
-		return Void();
 	}
 
 	bool operationRejectByLocking(RangeLocking* self, const KVOperation& kvOperation) {
@@ -310,39 +311,43 @@ struct RangeLocking : TestWorkload {
 		return;
 	}
 
-	ACTOR Future<std::map<Key, Value>> getKVSFromDB(RangeLocking* self, Database cx) {
-		state std::map<Key, Value> kvsFromDB;
-		state Key beginKey = normalKeys.begin;
-		state Key endKey = normalKeys.end;
-		loop {
-			state Transaction tr(cx);
+	Future<std::map<Key, Value>> getKVSFromDB(RangeLocking* self, Database cx) {
+		std::map<Key, Value> kvsFromDB;
+		Key beginKey = normalKeys.begin;
+		Key endKey = normalKeys.end;
+		while (true) {
+			Transaction tr(cx);
 			KeyRange rangeToRead = KeyRangeRef(beginKey, endKey);
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				RangeResult res = wait(tr.getRange(rangeToRead, GetRangeLimits()));
+				RangeResult res = co_await tr.getRange(rangeToRead, GetRangeLimits());
 				for (int i = 0; i < res.size(); i++) {
 					kvsFromDB[res[i].key] = res[i].value;
 				}
-				if (res.size() > 0) {
+				if (!res.empty()) {
 					beginKey = keyAfter(res.end()[-1].key);
 				} else {
 					break;
 				}
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
-		return kvsFromDB;
+		co_return kvsFromDB;
 	}
 
-	ACTOR Future<std::vector<KeyRange>> getLockedRangesFromDB(Database cx) {
-		state std::vector<std::pair<KeyRange, RangeLockState>> res;
-		wait(store(res, findExclusiveReadLockOnRange(cx, normalKeys)));
+	Future<std::vector<KeyRange>> getLockedRangesFromDB(Database cx) {
+		std::vector<std::pair<KeyRange, RangeLockState>> res;
+		res = co_await findExclusiveReadLockOnRange(cx, normalKeys);
 		std::vector<KeyRange> ranges;
-		for (const auto& lock : res) {
-			ranges.push_back(lock.first);
+		for (const auto& [lockedRange, _lockState] : res) {
+			ranges.push_back(lockedRange);
 		}
-		return coalesceRangeList(ranges);
+		co_return coalesceRangeList(ranges);
 	}
 
 	std::vector<KeyRange> getLockedRangesFromMemory(RangeLocking* self) {
@@ -355,9 +360,9 @@ struct RangeLocking : TestWorkload {
 		return coalesceRangeList(res);
 	}
 
-	ACTOR Future<Void> checkKVCorrectness(RangeLocking* self, Database cx) {
-		state std::map<Key, Value> currentKvsInDB;
-		wait(store(currentKvsInDB, self->getKVSFromDB(self, cx)));
+	Future<Void> checkKVCorrectness(RangeLocking* self, Database cx) {
+		std::map<Key, Value> currentKvsInDB;
+		currentKvsInDB = co_await self->getKVSFromDB(self, cx);
 		for (const auto& [key, value] : currentKvsInDB) {
 			if (self->kvs.find(key) == self->kvs.end()) {
 				TraceEvent(SevError, "RangeLockWorkLoadHistory")
@@ -365,7 +370,7 @@ struct RangeLocking : TestWorkload {
 				    .detail("Key", key)
 				    .detail("Value", value);
 				self->shouldExit = true;
-				return Void();
+				co_return;
 			} else if (self->kvs[key] != value) {
 				TraceEvent(SevError, "RangeLockWorkLoadHistory")
 				    .detail("Ops", "CheckMismatchValue")
@@ -373,7 +378,7 @@ struct RangeLocking : TestWorkload {
 				    .detail("MemValue", self->kvs[key])
 				    .detail("DBValue", value);
 				self->shouldExit = true;
-				return Void();
+				co_return;
 			}
 		}
 		for (const auto& [key, value] : self->kvs) {
@@ -383,7 +388,7 @@ struct RangeLocking : TestWorkload {
 				    .detail("Key", key)
 				    .detail("Value", value);
 				self->shouldExit = true;
-				return Void();
+				co_return;
 			}
 		}
 		if (self->verboseLogging) {
@@ -398,13 +403,11 @@ struct RangeLocking : TestWorkload {
 				i++;
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> checkLockCorrectness(RangeLocking* self, Database cx) {
-		state std::vector<KeyRange> currentLockRangesInDB;
-		wait(store(currentLockRangesInDB, self->getLockedRangesFromDB(cx)));
+	Future<Void> checkLockCorrectness(RangeLocking* self, Database cx) {
+		std::vector<KeyRange> currentLockRangesInDB;
+		currentLockRangesInDB = co_await self->getLockedRangesFromDB(cx);
 		std::vector<KeyRange> currentLockRangesInMemory = self->getLockedRangesFromMemory(self);
 		for (int i = 0; i < currentLockRangesInDB.size(); i++) {
 			if (i >= currentLockRangesInMemory.size()) {
@@ -412,7 +415,7 @@ struct RangeLocking : TestWorkload {
 				    .detail("Ops", "CheckDBUniqueLockedRange")
 				    .detail("Range", currentLockRangesInDB[i]);
 				self->shouldExit = true;
-				return Void();
+				co_return;
 			}
 			if (currentLockRangesInDB[i] != currentLockRangesInMemory[i]) {
 				TraceEvent(SevError, "RangeLockWorkLoadHistory")
@@ -420,7 +423,7 @@ struct RangeLocking : TestWorkload {
 				    .detail("RangeMemory", currentLockRangesInMemory[i])
 				    .detail("RangeDB", currentLockRangesInDB[i]);
 				self->shouldExit = true;
-				return Void();
+				co_return;
 			}
 		}
 		for (int i = currentLockRangesInDB.size(); i < currentLockRangesInMemory.size(); i++) {
@@ -428,7 +431,7 @@ struct RangeLocking : TestWorkload {
 			    .detail("Ops", "CheckMemoryUniqueLockedRange")
 			    .detail("Key", currentLockRangesInMemory[i]);
 			self->shouldExit = true;
-			return Void();
+			co_return;
 		}
 		if (self->verboseLogging) {
 			TraceEvent e("RangeLockWorkLoadHistory");
@@ -441,33 +444,31 @@ struct RangeLocking : TestWorkload {
 				i++;
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> complexTest(RangeLocking* self, Database cx) {
-		state int iterationCount = 100;
-		state int iteration = 0;
-		loop {
+	Future<Void> complexTest(RangeLocking* self, Database cx) {
+		int iterationCount = 100;
+		int iteration = 0;
+		while (true) {
 			if (iteration > iterationCount || self->shouldExit) {
 				break;
 			}
 			if (deterministicRandom()->coinflip()) {
-				wait(self->updateLockMapWithRandomOperation(self, cx));
+				co_await self->updateLockMapWithRandomOperation(self, cx);
 				self->updateInMemoryLockStatus(self);
 			}
 			TraceEvent("RangeLockWorkloadProgress")
 			    .detail("Iteration", iteration)
 			    .detail("IterationCount", iterationCount)
 			    .detail("Phase", "UpdateLock");
-			wait(self->checkLockCorrectness(self, cx));
+			co_await self->checkLockCorrectness(self, cx);
 			TraceEvent("RangeLockWorkloadProgress")
 			    .detail("Iteration", iteration)
 			    .detail("IterationCount", iterationCount)
 			    .detail("Phase", "CheckLockCorrectness");
 			if (deterministicRandom()->coinflip()) {
 				try {
-					wait(self->updateDBWithRandomOperations(self, cx));
+					co_await self->updateDBWithRandomOperations(self, cx);
 					self->updateInMemoryKVSStatus(self);
 				} catch (Error& e) {
 					ASSERT(e.code() == error_code_transaction_rejected_range_locked);
@@ -478,32 +479,31 @@ struct RangeLocking : TestWorkload {
 			    .detail("Iteration", iteration)
 			    .detail("IterationCount", iterationCount)
 			    .detail("Phase", "UpdateDB");
-			wait(self->checkKVCorrectness(self, cx));
+			co_await self->checkKVCorrectness(self, cx);
 			TraceEvent("RangeLockWorkloadProgress")
 			    .detail("Iteration", iteration)
 			    .detail("IterationCount", iterationCount)
 			    .detail("Phase", "CheckDBCorrectness");
 			iteration++;
 		}
-		std::vector<std::pair<KeyRange, RangeLockState>> locks2 = wait(findExclusiveReadLockOnRange(cx, normalKeys));
-		for (const auto& lock : locks2) {
+		std::vector<std::pair<KeyRange, RangeLockState>> locks2 = co_await findExclusiveReadLockOnRange(cx, normalKeys);
+		for (const auto& [lockedRange, lockState] : locks2) {
 			TraceEvent("RangeLockWorkloadProgress")
 			    .detail("Phase", "BeforeLockRelease")
-			    .detail("Range", lock.first)
-			    .detail("State", lock.second.toString());
+			    .detail("Range", lockedRange)
+			    .detail("State", lockState.toString());
 		}
-		wait(releaseExclusiveReadLockByUser(cx, self->rangeLockOwnerName));
-		std::vector<std::pair<KeyRange, RangeLockState>> locks = wait(findExclusiveReadLockOnRange(cx, normalKeys));
-		for (const auto& lock : locks) {
+		co_await releaseExclusiveReadLockByUser(cx, self->rangeLockOwnerName);
+		std::vector<std::pair<KeyRange, RangeLockState>> locks = co_await findExclusiveReadLockOnRange(cx, normalKeys);
+		for (const auto& [lockedRange, lockState] : locks) {
 			TraceEvent("RangeLockWorkloadProgress")
 			    .detail("Phase", "AfterLockRelease")
-			    .detail("Range", lock.first)
-			    .detail("State", lock.second.toString());
+			    .detail("Range", lockedRange)
+			    .detail("State", lockState.toString());
 		}
 		ASSERT(locks.empty());
 
 		TraceEvent("RangeLockWorkloadProgress").detail("Phase", "End");
-		return Void();
 	}
 
 	bool sameRangeList(const std::vector<KeyRange>& rangesA,
@@ -537,24 +537,24 @@ struct RangeLocking : TestWorkload {
 		return true;
 	}
 
-	ACTOR Future<Void> testUnlockByUser(RangeLocking* self, Database cx) {
-		state int i = 0;
-		state int j = 0;
-		state std::unordered_map<RangeLockOwnerName, std::vector<KeyRange>> rangeLocks;
-		state RangeLockOwnerName rangeLockOwnerName;
-		state std::vector<RangeLockOwnerName> candidates;
-		state KeyRange rangeToLock;
-		state std::vector<KeyRange> lockedRanges;
-		state std::vector<RangeLockOwnerName> usersToUnlock; // can contain duplicated users
-		state std::vector<std::pair<KeyRange, RangeLockState>> locksPerUser;
+	Future<Void> testUnlockByUser(RangeLocking* self, Database cx) {
+		int i = 0;
+		int j = 0;
+		std::unordered_map<RangeLockOwnerName, std::vector<KeyRange>> rangeLocks;
+		RangeLockOwnerName rangeLockOwnerName;
+		std::vector<RangeLockOwnerName> candidates;
+		KeyRange rangeToLock;
+		std::vector<KeyRange> lockedRanges;
+		std::vector<RangeLockOwnerName> usersToUnlock; // can contain duplicated users
+		std::vector<std::pair<KeyRange, RangeLockState>> locksPerUser;
 		for (; i < 100; i++) {
 			rangeLockOwnerName = "TestUnlockByUser" + std::to_string(i);
-			wait(registerRangeLockOwner(cx, rangeLockOwnerName, rangeLockOwnerName));
+			co_await registerRangeLockOwner(cx, rangeLockOwnerName, rangeLockOwnerName);
 			lockedRanges.clear();
 			for (; j < 2; j++) {
 				try {
 					rangeToLock = self->getRandomRange();
-					wait(takeExclusiveReadLockOnRange(cx, rangeToLock, rangeLockOwnerName));
+					co_await takeExclusiveReadLockOnRange(cx, rangeToLock, rangeLockOwnerName);
 					lockedRanges.push_back(rangeToLock);
 					TraceEvent("RangeLockWorkloadTestUnlockRangeByUser")
 					    .detail("Ops", "LockRange")
@@ -575,14 +575,14 @@ struct RangeLocking : TestWorkload {
 			usersToUnlock.push_back(deterministicRandom()->randomChoice(candidates));
 		}
 		for (i = 0; i < usersToUnlock.size(); i++) {
-			wait(releaseExclusiveReadLockByUser(cx, usersToUnlock[i]));
+			co_await releaseExclusiveReadLockByUser(cx, usersToUnlock[i]);
 			TraceEvent("RangeLockWorkloadTestUnlockRangeByUser")
 			    .detail("Ops", "Unlock by user")
 			    .detail("User", usersToUnlock[i]);
 		}
 		for (i = 0; i < candidates.size(); i++) {
 			locksPerUser.clear();
-			wait(store(locksPerUser, findExclusiveReadLockOnRange(cx, normalKeys, candidates[i])));
+			locksPerUser = co_await findExclusiveReadLockOnRange(cx, normalKeys, candidates[i]);
 			if (std::find(usersToUnlock.begin(), usersToUnlock.end(), candidates[i]) != usersToUnlock.end()) {
 				TraceEvent("RangeLockWorkloadTestUnlockRangeByUser")
 				    .detail("Ops", "Find unlocked user")
@@ -591,9 +591,9 @@ struct RangeLocking : TestWorkload {
 				ASSERT(locksPerUser.empty());
 			} else {
 				std::vector<KeyRange> lockedRangeFromMetadata;
-				for (const auto& lock : locksPerUser) {
-					ASSERT(lock.first == lock.second.getRange());
-					lockedRangeFromMetadata.push_back(lock.first);
+				for (const auto& [lockedRange, lockState] : locksPerUser) {
+					ASSERT(lockedRange == lockState.getRange());
+					lockedRangeFromMetadata.push_back(lockedRange);
 				}
 				TraceEvent("RangeLockWorkloadTestUnlockRangeByUser")
 				    .detail("Ops", "Find locked user")
@@ -604,16 +604,14 @@ struct RangeLocking : TestWorkload {
 				                           candidates[i]));
 			}
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(RangeLocking* self, Database cx) {
-		if (self->clientId != 0) {
-			return Void();
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0) {
+			co_return;
 		}
-		wait(self->complexTest(self, cx));
-		wait(self->testUnlockByUser(self, cx));
-		return Void();
+		co_await complexTest(this, cx);
+		co_await testUnlockByUser(this, cx);
 	}
 };
 
diff --git a/fdbserver/workloads/ReadAfterWrite.actor.cpp b/fdbserver/workloads/ReadAfterWrite.cpp
similarity index 60%
rename from fdbserver/workloads/ReadAfterWrite.actor.cpp
rename to fdbserver/workloads/ReadAfterWrite.cpp
index 210f77180a0..e1f2bf531b3 100644
--- a/fdbserver/workloads/ReadAfterWrite.actor.cpp
+++ b/fdbserver/workloads/ReadAfterWrite.cpp
@@ -1,5 +1,5 @@
 /*
- * ReadAfterWrite.actor.cpp
+ * ReadAfterWrite.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,18 +21,17 @@
 #include <vector>
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // If the log->storage propagation delay is longer than 1 second, then it's likely that our read
 // will see a `future_version` error from the storage server.  We need to retry the read until
 // a value is returned, or a different error is thrown.
-ACTOR Future<double> latencyOfRead(Transaction* tr, Key k) {
-	state double start = timer();
-	loop {
+Future<double> latencyOfRead(Transaction* tr, Key k) {
+	double start = timer();
+	while (true) {
 		try {
-			wait(success(tr->get(k)));
+			co_await tr->get(k);
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_future_version) {
@@ -41,7 +40,7 @@ ACTOR Future<double> latencyOfRead(Transaction* tr, Key k) {
 			throw;
 		}
 	}
-	return timer() - start;
+	co_return timer() - start;
 }
 
 // Measure the latency of a storage server making a committed value available for reading.
@@ -51,61 +50,64 @@ struct ReadAfterWriteWorkload : KVWorkload {
 	double testDuration;
 	DDSketch<double> propagationLatency;
 
-	ReadAfterWriteWorkload(WorkloadContext const& wcx) : KVWorkload(wcx), propagationLatency() {
+	explicit ReadAfterWriteWorkload(WorkloadContext const& wcx) : KVWorkload(wcx), propagationLatency() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	ACTOR static Future<Void> benchmark(Database cx, ReadAfterWriteWorkload* self) {
-		loop {
-			state Key key = self->getRandomKey();
-			state Transaction writeTr(cx);
-			state Transaction baselineReadTr(cx);
-			state Transaction afterWriteTr(cx);
+	Future<Void> benchmark(Database cx) {
+		while (true) {
+			Key key = getRandomKey();
+			Transaction writeTr(cx);
+			Transaction baselineReadTr(cx);
+			Transaction afterWriteTr(cx);
 
+			Error err;
 			try {
-				state Version readVersion = wait(writeTr.getReadVersion());
+				Version readVersion = co_await writeTr.getReadVersion();
 
 				// We do a read in this writeTransaction only to enforce that `readVersion` is already on a storage
-				// server after we commit.  Its existence or non-existence is irrelevant.  We write back the exact same
-				// value (or clear the key, if empty) so that the database state is not mutated.  This means this
-				// workload can be paired with any other workload, and it won't affect any results.
-				Optional<Value> value = wait(writeTr.get(key));
+				// server after we commit.  Its existence or non-existence is irrelevant.  We write back the exact
+				// same value (or clear the key, if empty) so that the database state is not mutated.  This means
+				// this workload can be paired with any other workload, and it won't affect any results.
+				Optional<Value> value = co_await writeTr.get(key);
 				if (value.present()) {
 					writeTr.set(key, value.get());
 				} else {
 					writeTr.clear(key);
 				}
 
-				wait(writeTr.commit());
+				co_await writeTr.commit();
 
 				Version commitVersion = writeTr.getCommittedVersion();
 
 				baselineReadTr.setVersion(readVersion);
 				afterWriteTr.setVersion(commitVersion);
 
-				state double baselineLatency = 0;
-				state double afterWriteLatency = 0;
+				double baselineLatency = 0;
+				double afterWriteLatency = 0;
 
-				wait(store(baselineLatency, latencyOfRead(&baselineReadTr, key)) &&
-				     store(afterWriteLatency, latencyOfRead(&afterWriteTr, key)));
+				co_await (store(baselineLatency, latencyOfRead(&baselineReadTr, key)) &&
+				          store(afterWriteLatency, latencyOfRead(&afterWriteTr, key)));
 
-				// By reading the same key at two different versions, we should be able to measure the latency of the
-				// network, the storage server overhead, and the propagation delay, and then with our baseline read,
-				// subtract out the network and the storage server overhead, leaving only the propagation delay.
-				self->propagationLatency.addSample(std::max<double>(afterWriteLatency - baselineLatency, 0));
+				// By reading the same key at two different versions, we should be able to measure the latency of
+				// the network, the storage server overhead, and the propagation delay, and then with our baseline
+				// read, subtract out the network and the storage server overhead, leaving only the propagation
+				// delay.
+				propagationLatency.addSample(std::max<double>(afterWriteLatency - baselineLatency, 0));
 			} catch (Error& e) {
-				wait(writeTr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await writeTr.onError(err);
 			}
 		}
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-	ACTOR Future<Void> _start(Database cx, ReadAfterWriteWorkload* self) {
-		state Future<Void> lifetime = benchmark(cx, self);
-		wait(delay(self->testDuration));
-		return Void();
+	Future<Void> start(Database const& cx) override {
+		Future<Void> lifetime = benchmark(cx);
+		co_await delay(testDuration);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/ReadHotDetection.actor.cpp b/fdbserver/workloads/ReadHotDetection.cpp
similarity index 70%
rename from fdbserver/workloads/ReadHotDetection.actor.cpp
rename to fdbserver/workloads/ReadHotDetection.cpp
index 8dd782c71a6..f5b5826326b 100644
--- a/fdbserver/workloads/ReadHotDetection.actor.cpp
+++ b/fdbserver/workloads/ReadHotDetection.cpp
@@ -1,5 +1,5 @@
 /*
- * ReadHotDetection.actor.cpp
+ * ReadHotDetection.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,11 +20,10 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct ReadHotDetectionWorkload : TestWorkload {
 	static constexpr auto NAME = "ReadHotDetection";
@@ -38,7 +37,7 @@ struct ReadHotDetectionWorkload : TestWorkload {
 	KeyRange wholeRange;
 	bool passed;
 
-	ReadHotDetectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ReadHotDetectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 120.0);
 		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 1000.0) / clientCount;
 		actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
@@ -46,7 +45,33 @@ struct ReadHotDetectionWorkload : TestWorkload {
 		readKey = StringRef(format("testkey%08x", deterministicRandom()->randomInt(0, keyCount)));
 	}
 
-	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
+	Future<Void> setup(Database const& cx) override {
+		Standalone<StringRef> largeValue;
+		Standalone<StringRef> smallValue;
+		largeValue = randomString(largeValue.arena(), 100000);
+		smallValue = randomString(smallValue.arena(), 100);
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
+			try {
+				for (int i = 0; i < keyCount; i++) {
+					Standalone<StringRef> key = StringRef(format("testkey%08x", i));
+					if (key == readKey) {
+						tr.set(key, largeValue);
+					} else {
+						tr.set(key, deterministicRandom()->random01() > 0.8 ? largeValue : smallValue);
+					}
+				}
+				co_await tr.commit();
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+		wholeRange = KeyRangeRef(""_sr, "\xff"_sr);
+		// TraceEvent("RHDLog").detail("Phase", "DoneSetup");
+	}
 
 	Future<Void> start(Database const& cx) override {
 		for (int c = 0; c < actorCount; c++) {
@@ -66,43 +91,18 @@ struct ReadHotDetectionWorkload : TestWorkload {
 		return passed;
 	}
 
-	ACTOR Future<Void> _setup(Database cx, ReadHotDetectionWorkload* self) {
-		state Standalone<StringRef> largeValue;
-		state Standalone<StringRef> smallValue;
-		largeValue = self->randomString(largeValue.arena(), 100000);
-		smallValue = self->randomString(smallValue.arena(), 100);
-		state ReadYourWritesTransaction tr(cx);
-		loop {
-			try {
-				for (int i = 0; i < self->keyCount; i++) {
-					Standalone<StringRef> key = StringRef(format("testkey%08x", i));
-					if (key == self->readKey) {
-						tr.set(key, largeValue);
-					} else {
-						tr.set(key, deterministicRandom()->random01() > 0.8 ? largeValue : smallValue);
-					}
-				}
-				wait(tr.commit());
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-		self->wholeRange = KeyRangeRef(""_sr, "\xff"_sr);
-		// TraceEvent("RHDLog").detail("Phase", "DoneSetup");
-		return Void();
-	}
-
-	ACTOR Future<Void> _check(Database cx, ReadHotDetectionWorkload* self) {
-		loop {
-			state Transaction tr(cx);
+	Future<Void> _check(Database cx, ReadHotDetectionWorkload* self) {
+		while (true) {
+			Transaction tr(cx);
+			Error err;
 			try {
-				StorageMetrics sm = wait(cx->getStorageMetrics(self->wholeRange, 100));
+				StorageMetrics sm = co_await cx->getStorageMetrics(self->wholeRange, 100);
 				(void)sm; // suppress unused variable warning
 				// TraceEvent("RHDCheckPhaseLog")
 				//     .detail("KeyRangeSize", sm.bytes)
 				//     .detail("KeyRangeReadBandwidth", sm.bytesReadPerKSecond);
-				Standalone<VectorRef<ReadHotRangeWithMetrics>> keyRanges = wait(cx->getReadHotRanges(self->wholeRange));
+				Standalone<VectorRef<ReadHotRangeWithMetrics>> keyRanges =
+				    co_await cx->getReadHotRanges(self->wholeRange);
 				// TraceEvent("RHDCheckPhaseLog")
 				//     .detail("KeyRangesSize", keyRanges.size())
 				//     .detail("ReadKey", self->readKey.printable().c_str())
@@ -112,7 +112,7 @@ struct ReadHotDetectionWorkload : TestWorkload {
 				for (const auto& kr : keyRanges) {
 					if (kr.keys.contains(self->readKey)) {
 						self->passed = true;
-						return Void();
+						co_return;
 					}
 				}
 				// The key ranges deemed read hot does not contain the readKey, which is impossible here.
@@ -122,33 +122,37 @@ struct ReadHotDetectionWorkload : TestWorkload {
 				// 	.detail("KeyRangesBackBeginKey", keyRanges.back().begin)
 				// 	.detail("KeyRangesBackEndKey", keyRanges.back().end);
 				// for(auto kr : keyRanges) {
-				// 	TraceEvent("RHDCheckPhaseFailed").detail("KeyRagneBegin", kr.begin).detail("KeyRagneEnd", kr.end);
+				// 	TraceEvent("RHDCheckPhaseFailed").detail("KeyRagneBegin", kr.begin).detail("KeyRagneEnd",
+				// kr.end);
 				// }
 				self->passed = false;
 			} catch (Error& e) {
-				// TraceEvent("RHDCheckPhaseReadGotError").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			// TraceEvent("RHDCheckPhaseReadGotError").error(e);
+			co_await tr.onError(err);
 		}
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> keyReader(Database cx, ReadHotDetectionWorkload* self, double delay, bool useReadKey) {
-		state double lastTime = now();
-		loop {
-			wait(poisson(&lastTime, delay));
-			state ReadYourWritesTransaction tr(cx);
-			loop {
+	Future<Void> keyReader(Database cx, ReadHotDetectionWorkload* self, double delay, bool useReadKey) {
+		double lastTime = now();
+		while (true) {
+			co_await poisson(&lastTime, delay);
+			ReadYourWritesTransaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					Optional<Value> v = wait(tr.get(
+					Optional<Value> v = co_await tr.get(
 					    useReadKey
 					        ? self->readKey
-					        : StringRef(format("testkey%08x", deterministicRandom()->randomInt(0, self->keyCount)))));
+					        : StringRef(format("testkey%08x", deterministicRandom()->randomInt(0, self->keyCount))));
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 	}
diff --git a/fdbserver/workloads/ReadWrite.actor.cpp b/fdbserver/workloads/ReadWrite.cpp
similarity index 53%
rename from fdbserver/workloads/ReadWrite.actor.cpp
rename to fdbserver/workloads/ReadWrite.cpp
index c5be76c31a2..62230fcc262 100644
--- a/fdbserver/workloads/ReadWrite.actor.cpp
+++ b/fdbserver/workloads/ReadWrite.cpp
@@ -1,5 +1,5 @@
 /*
- * ReadWrite.actor.cpp
+ * ReadWrite.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,36 +25,38 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbserver/workloads/ReadWriteWorkload.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
+#include "ReadWriteWorkload.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TDMetric.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TDMetric.h"
+#include "flow/CoroUtils.h"
 
 struct ReadWriteCommonImpl {
 	// trace methods
-	ACTOR static Future<bool> traceDumpWorkers(Reference<AsyncVar<ServerDBInfo> const> db) {
+	static Future<bool> traceDumpWorkers(Reference<AsyncVar<ServerDBInfo> const> db) {
 		try {
-			loop {
-				choose {
-					when(wait(db->onChange())) {}
-
-					when(ErrorOr<std::vector<WorkerDetails>> workerList =
-					         wait(db->get().clusterInterface.getWorkers.tryGetReply(GetWorkersRequest()))) {
-						if (workerList.present()) {
-							std::vector<Future<ErrorOr<Void>>> dumpRequests;
-							dumpRequests.reserve(workerList.get().size());
-							for (int i = 0; i < workerList.get().size(); i++)
-								dumpRequests.push_back(workerList.get()[i].interf.traceBatchDumpRequest.tryGetReply(
-								    TraceBatchDumpRequest()));
-							wait(waitForAll(dumpRequests));
-							return true;
-						}
-						wait(delay(1.0));
+			while (true) {
+				auto choice = co_await race(db->onChange(),
+				                            db->get().clusterInterface.getWorkers.tryGetReply(GetWorkersRequest()));
+				if (choice.index() == 0) {
+				} else if (choice.index() == 1) {
+					ErrorOr<std::vector<WorkerDetails>> workerList = std::get<1>(std::move(choice));
+
+					if (workerList.present()) {
+						std::vector<Future<ErrorOr<Void>>> dumpRequests;
+						dumpRequests.reserve(workerList.get().size());
+						for (int i = 0; i < workerList.get().size(); i++)
+							dumpRequests.push_back(
+							    workerList.get()[i].interf.traceBatchDumpRequest.tryGetReply(TraceBatchDumpRequest()));
+						co_await waitForAll(dumpRequests);
+						co_return true;
 					}
+					co_await delay(1.0);
+				} else {
+					UNREACHABLE();
 				}
 			}
 		} catch (Error& e) {
@@ -63,150 +65,14 @@ struct ReadWriteCommonImpl {
 		}
 	}
 
-	ACTOR static Future<Void> tracePeriodically(ReadWriteCommon* self) {
-		state double start = now();
-		state double elapsed = 0.0;
-		state int64_t last_ops = 0;
-
-		loop {
-			elapsed += self->periodicLoggingInterval;
-			wait(delayUntil(start + elapsed));
-
-			TraceEvent((self->description() + "_RowReadLatency").c_str())
-			    .detail("Mean", self->readLatencies.mean())
-			    .detail("Median", self->readLatencies.median())
-			    .detail("Percentile5", self->readLatencies.percentile(.05))
-			    .detail("Percentile95", self->readLatencies.percentile(.95))
-			    .detail("Percentile99", self->readLatencies.percentile(.99))
-			    .detail("Percentile99_9", self->readLatencies.percentile(.999))
-			    .detail("Max", self->readLatencies.max())
-			    .detail("Count", self->readLatencyCount)
-			    .detail("Elapsed", elapsed);
-
-			TraceEvent((self->description() + "_GRVLatency").c_str())
-			    .detail("Mean", self->GRVLatencies.mean())
-			    .detail("Median", self->GRVLatencies.median())
-			    .detail("Percentile5", self->GRVLatencies.percentile(.05))
-			    .detail("Percentile95", self->GRVLatencies.percentile(.95))
-			    .detail("Percentile99", self->GRVLatencies.percentile(.99))
-			    .detail("Percentile99_9", self->GRVLatencies.percentile(.999))
-			    .detail("Max", self->GRVLatencies.max());
-
-			TraceEvent((self->description() + "_CommitLatency").c_str())
-			    .detail("Mean", self->commitLatencies.mean())
-			    .detail("Median", self->commitLatencies.median())
-			    .detail("Percentile5", self->commitLatencies.percentile(.05))
-			    .detail("Percentile95", self->commitLatencies.percentile(.95))
-			    .detail("Percentile99", self->commitLatencies.percentile(.99))
-			    .detail("Percentile99_9", self->commitLatencies.percentile(.999))
-			    .detail("Max", self->commitLatencies.max());
-
-			TraceEvent((self->description() + "_TotalLatency").c_str())
-			    .detail("Mean", self->latencies.mean())
-			    .detail("Median", self->latencies.median())
-			    .detail("Percentile5", self->latencies.percentile(.05))
-			    .detail("Percentile95", self->latencies.percentile(.95))
-			    .detail("Percentile99", self->latencies.percentile(.99))
-			    .detail("Percentile99_9", self->latencies.percentile(.999))
-			    .detail("Max", self->latencies.max());
-
-			int64_t ops =
-			    (self->aTransactions.getValue() * (self->readsPerTransactionA + self->writesPerTransactionA)) +
-			    (self->bTransactions.getValue() * (self->readsPerTransactionB + self->writesPerTransactionB));
-			bool recordBegin = self->shouldRecord(std::max(now() - self->periodicLoggingInterval, self->clientBegin));
-			bool recordEnd = self->shouldRecord(now());
-			if (recordBegin && recordEnd) {
-				std::string ts = format("T=%04.0fs:", elapsed);
-				self->periodicMetrics.emplace_back(
-				    ts + "Operations/sec", (ops - last_ops) / self->periodicLoggingInterval, Averaged::False);
-
-				// if(self->rampUpLoad) {
-				self->periodicMetrics.emplace_back(
-				    ts + "Mean Latency (ms)", 1000 * self->latencies.mean(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Median Latency (ms, averaged)", 1000 * self->latencies.median(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "5% Latency (ms, averaged)", 1000 * self->latencies.percentile(.05), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "95% Latency (ms, averaged)", 1000 * self->latencies.percentile(.95), Averaged::True);
-
-				self->periodicMetrics.emplace_back(
-				    ts + "Mean Row Read Latency (ms)", 1000 * self->readLatencies.mean(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Median Row Read Latency (ms, averaged)", 1000 * self->readLatencies.median(), Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "5% Row Read Latency (ms, averaged)",
-				                                   1000 * self->readLatencies.percentile(.05),
-				                                   Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "95% Row Read Latency (ms, averaged)",
-				                                   1000 * self->readLatencies.percentile(.95),
-				                                   Averaged::True);
-
-				self->periodicMetrics.emplace_back(
-				    ts + "Mean Total Read Latency (ms)", 1000 * self->fullReadLatencies.mean(), Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "Median Total Read Latency (ms, averaged)",
-				                                   1000 * self->fullReadLatencies.median(),
-				                                   Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "5% Total Read Latency (ms, averaged)",
-				                                   1000 * self->fullReadLatencies.percentile(.05),
-				                                   Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "95% Total Read Latency (ms, averaged)",
-				                                   1000 * self->fullReadLatencies.percentile(.95),
-				                                   Averaged::True);
-
-				self->periodicMetrics.emplace_back(
-				    ts + "Mean GRV Latency (ms)", 1000 * self->GRVLatencies.mean(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Median GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.median(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "5% GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.percentile(.05), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "95% GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.percentile(.95), Averaged::True);
-
-				self->periodicMetrics.emplace_back(
-				    ts + "Mean Commit Latency (ms)", 1000 * self->commitLatencies.mean(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Median Commit Latency (ms, averaged)", 1000 * self->commitLatencies.median(), Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "5% Commit Latency (ms, averaged)",
-				                                   1000 * self->commitLatencies.percentile(.05),
-				                                   Averaged::True);
-				self->periodicMetrics.emplace_back(ts + "95% Commit Latency (ms, averaged)",
-				                                   1000 * self->commitLatencies.percentile(.95),
-				                                   Averaged::True);
-				//}
-
-				self->periodicMetrics.emplace_back(
-				    ts + "Max Latency (ms, averaged)", 1000 * self->latencies.max(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Max Row Read Latency (ms, averaged)", 1000 * self->readLatencies.max(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Max Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.max(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Max GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.max(), Averaged::True);
-				self->periodicMetrics.emplace_back(
-				    ts + "Max Commit Latency (ms, averaged)", 1000 * self->commitLatencies.max(), Averaged::True);
-			}
-			last_ops = ops;
-
-			// if(self->rampUpLoad) {
-			self->latencies.clear();
-			self->readLatencies.clear();
-			self->fullReadLatencies.clear();
-			self->GRVLatencies.clear();
-			self->commitLatencies.clear();
-			//}
-
-			self->readLatencyTotal = 0.0;
-			self->readLatencyCount = 0;
-		}
-	}
-	ACTOR static Future<Void> logLatency(Future<Optional<Value>> f,
-	                                     DDSketch<double>* latencies,
-	                                     double* totalLatency,
-	                                     int* latencyCount,
-	                                     EventMetricHandle<ReadMetric> readMetric,
-	                                     bool shouldRecord) {
-		state double readBegin = now();
-		Optional<Value> value = wait(f);
+	static Future<Void> logLatency(Future<Optional<Value>> f,
+	                               DDSketch<double>* latencies,
+	                               double* totalLatency,
+	                               int* latencyCount,
+	                               EventMetricHandle<ReadMetricDescriptor> readMetric,
+	                               bool shouldRecord) {
+		double readBegin = now();
+		Optional<Value> value = co_await f;
 
 		double latency = now() - readBegin;
 		readMetric->readLatency = latency * 1e9;
@@ -217,16 +83,15 @@ struct ReadWriteCommonImpl {
 			++*latencyCount;
 			latencies->addSample(latency);
 		}
-		return Void();
 	}
-	ACTOR static Future<Void> logLatency(Future<RangeResult> f,
-	                                     DDSketch<double>* latencies,
-	                                     double* totalLatency,
-	                                     int* latencyCount,
-	                                     EventMetricHandle<ReadMetric> readMetric,
-	                                     bool shouldRecord) {
-		state double readBegin = now();
-		RangeResult value = wait(f);
+	static Future<Void> logLatency(Future<RangeResult> f,
+	                               DDSketch<double>* latencies,
+	                               double* totalLatency,
+	                               int* latencyCount,
+	                               EventMetricHandle<ReadMetricDescriptor> readMetric,
+	                               bool shouldRecord) {
+		double readBegin = now();
+		RangeResult value = co_await f;
 
 		double latency = now() - readBegin;
 		readMetric->readLatency = latency * 1e9;
@@ -237,35 +102,150 @@ struct ReadWriteCommonImpl {
 			++*latencyCount;
 			latencies->addSample(latency);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> setup(Database cx, ReadWriteCommon* self) {
-		if (!self->doSetup)
-			return Void();
-
-		state Promise<double> loadTime;
-		state Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts;
-
-		wait(bulkSetup(cx,
-		               self,
-		               self->nodeCount,
-		               loadTime,
-		               self->insertionCountsToMeasure.empty(),
-		               self->warmingDelay,
-		               self->maxInsertRate,
-		               self->insertionCountsToMeasure,
-		               ratesAtKeyCounts));
-
-		self->loadTime = loadTime.getFuture().get();
-		self->ratesAtKeyCounts = ratesAtKeyCounts.getFuture().get();
-
-		return Void();
+	static Future<Void> setup(Database cx, ReadWriteCommon& self) {
+		if (!self.doSetup)
+			co_return;
+
+		Promise<double> loadTime;
+		Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts;
+
+		co_await bulkSetup(cx,
+		                   &self,
+		                   self.nodeCount,
+		                   loadTime,
+		                   self.insertionCountsToMeasure.empty(),
+		                   self.warmingDelay,
+		                   self.maxInsertRate,
+		                   self.insertionCountsToMeasure,
+		                   ratesAtKeyCounts);
+
+		self.loadTime = loadTime.getFuture().get();
+		self.ratesAtKeyCounts = ratesAtKeyCounts.getFuture().get();
 	}
 };
 
 Future<Void> ReadWriteCommon::tracePeriodically() {
-	return ReadWriteCommonImpl::tracePeriodically(this);
+	double start = now();
+	double elapsed = 0.0;
+	int64_t last_ops = 0;
+
+	while (true) {
+		elapsed += periodicLoggingInterval;
+		co_await delayUntil(start + elapsed);
+
+		TraceEvent((description() + "_RowReadLatency").c_str())
+		    .detail("Mean", readLatencies.mean())
+		    .detail("Median", readLatencies.median())
+		    .detail("Percentile5", readLatencies.percentile(.05))
+		    .detail("Percentile95", readLatencies.percentile(.95))
+		    .detail("Percentile99", readLatencies.percentile(.99))
+		    .detail("Percentile99_9", readLatencies.percentile(.999))
+		    .detail("Max", readLatencies.max())
+		    .detail("Count", readLatencyCount)
+		    .detail("Elapsed", elapsed);
+
+		TraceEvent((description() + "_GRVLatency").c_str())
+		    .detail("Mean", GRVLatencies.mean())
+		    .detail("Median", GRVLatencies.median())
+		    .detail("Percentile5", GRVLatencies.percentile(.05))
+		    .detail("Percentile95", GRVLatencies.percentile(.95))
+		    .detail("Percentile99", GRVLatencies.percentile(.99))
+		    .detail("Percentile99_9", GRVLatencies.percentile(.999))
+		    .detail("Max", GRVLatencies.max());
+
+		TraceEvent((description() + "_CommitLatency").c_str())
+		    .detail("Mean", commitLatencies.mean())
+		    .detail("Median", commitLatencies.median())
+		    .detail("Percentile5", commitLatencies.percentile(.05))
+		    .detail("Percentile95", commitLatencies.percentile(.95))
+		    .detail("Percentile99", commitLatencies.percentile(.99))
+		    .detail("Percentile99_9", commitLatencies.percentile(.999))
+		    .detail("Max", commitLatencies.max());
+
+		TraceEvent((description() + "_TotalLatency").c_str())
+		    .detail("Mean", latencies.mean())
+		    .detail("Median", latencies.median())
+		    .detail("Percentile5", latencies.percentile(.05))
+		    .detail("Percentile95", latencies.percentile(.95))
+		    .detail("Percentile99", latencies.percentile(.99))
+		    .detail("Percentile99_9", latencies.percentile(.999))
+		    .detail("Max", latencies.max());
+
+		int64_t ops = (aTransactions.getValue() * (readsPerTransactionA + writesPerTransactionA)) +
+		              (bTransactions.getValue() * (readsPerTransactionB + writesPerTransactionB));
+		bool recordBegin = shouldRecord(std::max(now() - periodicLoggingInterval, clientBegin));
+		bool recordEnd = shouldRecord(now());
+		if (recordBegin && recordEnd) {
+			std::string ts = format("T=%04.0fs:", elapsed);
+			periodicMetrics.emplace_back(
+			    ts + "Operations/sec", (ops - last_ops) / periodicLoggingInterval, Averaged::False);
+
+			periodicMetrics.emplace_back(ts + "Mean Latency (ms)", 1000 * latencies.mean(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Median Latency (ms, averaged)", 1000 * latencies.median(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "5% Latency (ms, averaged)", 1000 * latencies.percentile(.05), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "95% Latency (ms, averaged)", 1000 * latencies.percentile(.95), Averaged::True);
+
+			periodicMetrics.emplace_back(
+			    ts + "Mean Row Read Latency (ms)", 1000 * readLatencies.mean(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Median Row Read Latency (ms, averaged)", 1000 * readLatencies.median(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "5% Row Read Latency (ms, averaged)", 1000 * readLatencies.percentile(.05), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "95% Row Read Latency (ms, averaged)", 1000 * readLatencies.percentile(.95), Averaged::True);
+
+			periodicMetrics.emplace_back(
+			    ts + "Mean Total Read Latency (ms)", 1000 * fullReadLatencies.mean(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Median Total Read Latency (ms, averaged)", 1000 * fullReadLatencies.median(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "5% Total Read Latency (ms, averaged)", 1000 * fullReadLatencies.percentile(.05), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "95% Total Read Latency (ms, averaged)", 1000 * fullReadLatencies.percentile(.95), Averaged::True);
+
+			periodicMetrics.emplace_back(ts + "Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "5% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile(.05), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "95% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile(.95), Averaged::True);
+
+			periodicMetrics.emplace_back(
+			    ts + "Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "5% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile(.05), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "95% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile(.95), Averaged::True);
+
+			periodicMetrics.emplace_back(ts + "Max Latency (ms, averaged)", 1000 * latencies.max(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Max Row Read Latency (ms, averaged)", 1000 * readLatencies.max(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Max Total Read Latency (ms, averaged)", 1000 * fullReadLatencies.max(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Max GRV Latency (ms, averaged)", 1000 * GRVLatencies.max(), Averaged::True);
+			periodicMetrics.emplace_back(
+			    ts + "Max Commit Latency (ms, averaged)", 1000 * commitLatencies.max(), Averaged::True);
+		}
+		last_ops = ops;
+
+		latencies.clear();
+		readLatencies.clear();
+		fullReadLatencies.clear();
+		GRVLatencies.clear();
+		commitLatencies.clear();
+
+		readLatencyTotal = 0.0;
+		readLatencyCount = 0;
+	}
 }
 
 Future<Void> ReadWriteCommon::logLatency(Future<Optional<Value>> f, bool shouldRecord) {
@@ -279,7 +259,7 @@ Future<Void> ReadWriteCommon::logLatency(Future<RangeResult> f, bool shouldRecor
 }
 
 Future<Void> ReadWriteCommon::setup(Database const& cx) {
-	return ReadWriteCommonImpl::setup(cx, this);
+	return ReadWriteCommonImpl::setup(cx, *this);
 }
 
 Future<bool> ReadWriteCommon::check(Database const& cx) {
@@ -318,8 +298,7 @@ void ReadWriteCommon::getMetrics(std::vector<PerfMetric>& m) {
 	    "Bytes written/sec", (writes * (keyBytes + (minValueBytes + maxValueBytes) * 0.5)) / duration, Averaged::False);
 	m.insert(m.end(), periodicMetrics.begin(), periodicMetrics.end());
 
-	std::vector<std::pair<uint64_t, double>>::iterator ratesItr = ratesAtKeyCounts.begin();
-	for (; ratesItr != ratesAtKeyCounts.end(); ratesItr++)
+	for (auto ratesItr = ratesAtKeyCounts.begin(); ratesItr != ratesAtKeyCounts.end(); ++ratesItr)
 		m.emplace_back(format("%lld keys imported bytes/sec", ratesItr->first), ratesItr->second, Averaged::False);
 }
 
@@ -335,15 +314,17 @@ bool ReadWriteCommon::shouldRecord(double checkTime) {
 static Future<Version> nextRV;
 static Version lastRV = invalidVersion;
 
-ACTOR static Future<Version> getNextRV(Database db) {
-	state Transaction tr(db);
-	loop {
+static Future<Version> getNextRV(Database db) {
+	Transaction tr(db);
+	while (true) {
+		Error err;
 		try {
-			Version v = wait(tr.getReadVersion());
-			return v;
+			Version v = co_await tr.getReadVersion();
+			co_return v;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -384,7 +365,7 @@ struct ReadWriteWorkload : ReadWriteCommon {
 	// hot traffic pattern
 	double hotKeyFraction, forceHotProbability = 0; // key based hot traffic setting
 
-	ReadWriteWorkload(WorkloadContext const& wcx)
+	explicit ReadWriteWorkload(WorkloadContext const& wcx)
 	  : ReadWriteCommon(wcx), dependentReads(false), adjacentReads(false), adjacentWrites(false) {
 		extraReadConflictRangesPerTransaction = getOption(options, "extraReadConflictRangesPerTransaction"_sr, 0);
 		extraWriteConflictRangesPerTransaction = getOption(options, "extraWriteConflictRangesPerTransaction"_sr, 0);
@@ -484,83 +465,83 @@ struct ReadWriteWorkload : ReadWriteCommon {
 		}
 	}
 
-	Future<Void> start(Database const& cx) override { return timeout(_start(cx, this), testDuration, Void()); }
+	Future<Void> start(Database const& cx) override { return timeout(_start(cx), testDuration, Void()); }
 
-	ACTOR template <class Trans>
-	static Future<Void> readOp(Trans* tr, std::vector<int64_t> keys, ReadWriteWorkload* self, bool shouldRecord) {
-		if (!keys.size())
-			return Void();
-		if (!self->dependentReads) {
+	template <class Trans>
+	Future<Void> readOp(Trans* tr, std::vector<int64_t> keys, bool shouldRecord) {
+		if (keys.empty())
+			co_return;
+		if (!dependentReads) {
 			std::vector<Future<Void>> readers;
-			if (self->rangeReads) {
+			if (rangeReads) {
 				for (int op = 0; op < keys.size(); op++) {
-					++self->totalReadsMetric;
-					readers.push_back(self->logLatency(
-					    tr->getRange(KeyRangeRef(self->keyForIndex(keys[op]), Key(strinc(self->keyForIndex(keys[op])))),
-					                 GetRangeLimits(-1, 80000)),
-					    shouldRecord));
+					++totalReadsMetric;
+					readers.push_back(
+					    logLatency(tr->getRange(KeyRangeRef(keyForIndex(keys[op]), Key(strinc(keyForIndex(keys[op])))),
+					                            GetRangeLimits(-1, 80000)),
+					               shouldRecord));
 				}
 			} else {
 				for (int op = 0; op < keys.size(); op++) {
-					++self->totalReadsMetric;
-					readers.push_back(self->logLatency(tr->get(self->keyForIndex(keys[op])), shouldRecord));
+					++totalReadsMetric;
+					readers.push_back(logLatency(tr->get(keyForIndex(keys[op])), shouldRecord));
 				}
 			}
-			wait(waitForAll(readers));
+			co_await waitForAll(readers);
 		} else {
-			state int op;
+			int op{ 0 };
 			for (op = 0; op < keys.size(); op++) {
-				++self->totalReadsMetric;
-				wait(self->logLatency(tr->get(self->keyForIndex(keys[op])), shouldRecord));
+				++totalReadsMetric;
+				co_await logLatency(tr->get(keyForIndex(keys[op])), shouldRecord);
 			}
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> _start(Database cx, ReadWriteWorkload* self) {
+	Future<Void> _start(Database cx) {
 		// Read one record from the database to warm the cache of keyServers
-		state std::vector<int64_t> keys;
-		keys.push_back(deterministicRandom()->randomInt64(0, self->nodeCount));
-		state double startTime = now();
-		loop {
-			state Transaction tr(cx);
-			try {
-				self->setupTransaction(tr);
-				wait(self->readOp(&tr, keys, self, false));
-				wait(tr.warmRange(allKeys));
-				break;
-			} catch (Error& e) {
-				if (e.code() == error_code_tag_throttled) {
-					++self->transactionsTagThrottled;
+		std::vector<int64_t> keys;
+		keys.push_back(deterministicRandom()->randomInt64(0, nodeCount));
+		double startTime = now();
+		while (true) {
+			Transaction tr(cx);
+			{
+				Error err;
+				try {
+					setupTransaction(tr);
+					co_await readOp(&tr, keys, false);
+					co_await tr.warmRange(allKeys);
+					break;
+				} catch (Error& e) {
+					err = e;
 				}
-				wait(tr.onError(e));
+				if (err.code() == error_code_tag_throttled) {
+					++transactionsTagThrottled;
+				}
+				co_await tr.onError(err);
 			}
 		}
 
-		wait(delay(std::max(0.1, 1.0 - (now() - startTime))));
+		co_await delay(std::max(0.1, 1.0 - (now() - startTime)));
 
 		std::vector<Future<Void>> clients;
-		if (self->enableReadLatencyLogging)
-			clients.push_back(self->tracePeriodically());
+		if (enableReadLatencyLogging)
+			clients.push_back(tracePeriodically());
 
-		self->clientBegin = now();
-		for (int c = 0; c < self->actorCount; c++) {
+		clientBegin = now();
+		for (int c = 0; c < actorCount; c++) {
 			Future<Void> worker;
-			if (self->useRYW)
-				worker = self->randomReadWriteClient<ReadYourWritesTransaction>(
-				    cx, self, self->actorCount / self->transactionsPerSecond, c);
+			if (useRYW)
+				worker =
+				    randomReadWriteClient<ReadYourWritesTransaction>(cx, this, actorCount / transactionsPerSecond, c);
 			else
-				worker = self->randomReadWriteClient<Transaction>(
-				    cx, self, self->actorCount / self->transactionsPerSecond, c);
+				worker = randomReadWriteClient<Transaction>(cx, this, actorCount / transactionsPerSecond, c);
 			clients.push_back(worker);
 		}
 
-		if (!self->cancelWorkersAtDuration)
-			self->clients = clients; // Don't cancel them until check()
+		if (!cancelWorkersAtDuration)
+			this->clients = clients; // Don't cancel them until check()
 
-		wait(self->cancelWorkersAtDuration ? timeout(waitForAll(clients), self->testDuration, Void())
-		                                   : delay(self->testDuration));
-		return Void();
+		co_await (cancelWorkersAtDuration ? timeout(waitForAll(clients), testDuration, Void()) : delay(testDuration));
 	}
 
 	int64_t getRandomKey(uint64_t nodeCount) {
@@ -581,25 +562,25 @@ struct ReadWriteWorkload : ReadWriteCommon {
 		return alpha;
 	}
 
-	ACTOR template <class Trans>
+	template <class Trans>
 	Future<Void> randomReadWriteClient(Database cx, ReadWriteWorkload* self, double delay, int clientIndex) {
-		state double startTime = now();
-		state double lastTime = now();
-		state double GRVStartTime;
-		state UID debugID;
+		double startTime = now();
+		double lastTime = now();
+		double GRVStartTime{ 0 };
+		UID debugID;
 
 		if (self->rampUpConcurrency) {
-			wait(::delay(self->testDuration / 2 *
-			             (double(clientIndex) / self->actorCount +
-			              double(self->clientId) / self->clientCount / self->actorCount)));
+			co_await ::delay(self->testDuration / 2 *
+			                 (double(clientIndex) / self->actorCount +
+			                  double(self->clientId) / self->clientCount / self->actorCount));
 			TraceEvent("ClientStarting")
 			    .detail("ActorIndex", clientIndex)
 			    .detail("ClientIndex", self->clientId)
 			    .detail("NumActors", clientIndex * self->clientCount + self->clientId + 1);
 		}
 
-		loop {
-			wait(poisson(&lastTime, delay));
+		while (true) {
+			co_await poisson(&lastTime, delay);
 
 			if (self->rampUpConcurrency) {
 				if (now() - startTime >= self->testDuration / 2 *
@@ -609,22 +590,22 @@ struct ReadWriteWorkload : ReadWriteCommon {
 					    .detail("ActorIndex", clientIndex)
 					    .detail("ClientIndex", self->clientId)
 					    .detail("NumActors", clientIndex * self->clientCount + self->clientId);
-					wait(Never());
+					co_await Future<Void>(Never());
 				}
 			}
 
 			if (!self->rampUpLoad || deterministicRandom()->random01() < self->sweepAlpha(startTime)) {
-				state double tstart = now();
-				state bool aTransaction = deterministicRandom()->random01() >
-				                          (self->rampTransactionType ? self->sweepAlpha(startTime) : self->alpha);
+				double tstart = now();
+				bool aTransaction = deterministicRandom()->random01() >
+				                    (self->rampTransactionType ? self->sweepAlpha(startTime) : self->alpha);
 
-				state std::vector<int64_t> keys;
-				state std::vector<Value> values;
-				state std::vector<KeyRange> extra_ranges;
+				std::vector<int64_t> keys;
+				std::vector<Value> values;
+				std::vector<KeyRange> extra_ranges;
 				int reads = aTransaction ? self->readsPerTransactionA : self->readsPerTransactionB;
-				state int writes = aTransaction ? self->writesPerTransactionA : self->writesPerTransactionB;
-				state int extra_read_conflict_ranges = writes ? self->extraReadConflictRangesPerTransaction : 0;
-				state int extra_write_conflict_ranges = writes ? self->extraWriteConflictRangesPerTransaction : 0;
+				int writes = aTransaction ? self->writesPerTransactionA : self->writesPerTransactionB;
+				int extra_read_conflict_ranges = writes ? self->extraReadConflictRangesPerTransaction : 0;
+				int extra_write_conflict_ranges = writes ? self->extraWriteConflictRangesPerTransaction : 0;
 				if (!self->adjacentReads) {
 					for (int op = 0; op < reads; op++)
 						keys.push_back(self->getRandomKey(self->nodeCount));
@@ -642,7 +623,7 @@ struct ReadWriteWorkload : ReadWriteCommon {
 				for (int op = 0; op < extra_read_conflict_ranges + extra_write_conflict_ranges; op++)
 					extra_ranges.push_back(singleKeyRange(deterministicRandom()->randomUniqueID().toString()));
 
-				state Trans tr(cx);
+				Trans tr(cx);
 
 				if (tstart - self->clientBegin > self->debugTime &&
 				    tstart - self->clientBegin <= self->debugTime + self->debugInterval) {
@@ -657,7 +638,8 @@ struct ReadWriteWorkload : ReadWriteCommon {
 				self->transactionSuccessMetric->retries = 0;
 				self->transactionSuccessMetric->commitLatency = -1;
 
-				loop {
+				while (true) {
+					Error err;
 					try {
 						self->setupTransaction(tr);
 
@@ -665,7 +647,7 @@ struct ReadWriteWorkload : ReadWriteCommon {
 						self->transactionFailureMetric->startLatency = -1;
 
 						Version v =
-						    wait(self->inconsistentReads ? getInconsistentReadVersion(cx) : tr.getReadVersion());
+						    co_await (self->inconsistentReads ? getInconsistentReadVersion(cx) : tr.getReadVersion());
 						if (self->inconsistentReads)
 							tr.setVersion(v);
 
@@ -675,8 +657,8 @@ struct ReadWriteWorkload : ReadWriteCommon {
 						if (self->shouldRecord())
 							self->GRVLatencies.addSample(grvLatency);
 
-						state double readStart = now();
-						wait(self->readOp(&tr, keys, self, self->shouldRecord()));
+						double readStart = now();
+						co_await self->readOp(&tr, keys, self->shouldRecord());
 
 						double readLatency = now() - readStart;
 						if (self->shouldRecord())
@@ -698,8 +680,8 @@ struct ReadWriteWorkload : ReadWriteCommon {
 						for (int op = 0; op < extra_write_conflict_ranges; op++)
 							tr.addWriteConflictRange(extra_ranges[op + extra_read_conflict_ranges]);
 
-						state double commitStart = now();
-						wait(tr.commit());
+						double commitStart = now();
+						co_await tr.commit();
 
 						double commitLatency = now() - commitStart;
 						self->transactionSuccessMetric->commitLatency = commitLatency * 1e9;
@@ -708,21 +690,22 @@ struct ReadWriteWorkload : ReadWriteCommon {
 
 						break;
 					} catch (Error& e) {
-						if (e.code() == error_code_tag_throttled) {
-							++self->transactionsTagThrottled;
-						}
+						err = e;
+					}
+					if (err.code() == error_code_tag_throttled) {
+						++self->transactionsTagThrottled;
+					}
 
-						self->transactionFailureMetric->errorCode = e.code();
-						self->transactionFailureMetric->log();
+					self->transactionFailureMetric->errorCode = err.code();
+					self->transactionFailureMetric->log();
 
-						wait(tr.onError(e));
+					co_await tr.onError(err);
 
-						++self->transactionSuccessMetric->retries;
-						++self->totalRetriesMetric;
+					++self->transactionSuccessMetric->retries;
+					++self->totalRetriesMetric;
 
-						if (self->shouldRecord())
-							++self->retries;
-					}
+					if (self->shouldRecord())
+						++self->retries;
 				}
 
 				if (debugID != UID())
@@ -747,24 +730,25 @@ struct ReadWriteWorkload : ReadWriteCommon {
 	}
 };
 
-ACTOR Future<std::vector<std::pair<uint64_t, double>>> trackInsertionCount(Database cx,
-                                                                           std::vector<uint64_t> countsOfInterest,
-                                                                           double checkInterval) {
-	state KeyRange keyPrefix = KeyRangeRef(std::string("keycount"), std::string("keycount") + char(255));
-	state KeyRange bytesPrefix = KeyRangeRef(std::string("bytesstored"), std::string("bytesstored") + char(255));
-	state Transaction tr(cx);
-	state uint64_t lastInsertionCount = 0;
-	state int currentCountIndex = 0;
+Future<std::vector<std::pair<uint64_t, double>>> trackInsertionCount(Database cx,
+                                                                     std::vector<uint64_t> countsOfInterest,
+                                                                     double checkInterval) {
+	KeyRange keyPrefix = KeyRangeRef(std::string("keycount"), std::string("keycount") + char(255));
+	KeyRange bytesPrefix = KeyRangeRef(std::string("bytesstored"), std::string("bytesstored") + char(255));
+	Transaction tr(cx);
+	uint64_t lastInsertionCount = 0;
+	int currentCountIndex = 0;
 
-	state std::vector<std::pair<uint64_t, double>> countInsertionRates;
+	std::vector<std::pair<uint64_t, double>> countInsertionRates;
 
-	state double startTime = now();
+	double startTime = now();
 
 	while (currentCountIndex < countsOfInterest.size()) {
+		Error err;
 		try {
-			state Future<RangeResult> countFuture = tr.getRange(keyPrefix, 1000000000);
-			state Future<RangeResult> bytesFuture = tr.getRange(bytesPrefix, 1000000000);
-			wait(success(countFuture) && success(bytesFuture));
+			Future<RangeResult> countFuture = tr.getRange(keyPrefix, 1000000000);
+			Future<RangeResult> bytesFuture = tr.getRange(bytesPrefix, 1000000000);
+			co_await (success(countFuture) && success(bytesFuture));
 
 			RangeResult counts = countFuture.get();
 			RangeResult bytes = bytesFuture.get();
@@ -784,13 +768,14 @@ ACTOR Future<std::vector<std::pair<uint64_t, double>>> trackInsertionCount(Datab
 				                                 bytesInserted / (now() - startTime));
 
 			lastInsertionCount = numInserted;
-			wait(delay(checkInterval));
+			co_await delay(checkInterval);
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 
-	return countInsertionRates;
+	co_return countInsertionRates;
 }
 
 WorkloadFactory<ReadWriteWorkload> ReadWriteWorkloadFactory;
diff --git a/fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h b/fdbserver/workloads/ReadWriteWorkload.h
similarity index 76%
rename from fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h
rename to fdbserver/workloads/ReadWriteWorkload.h
index d0005d11890..95343046df6 100644
--- a/fdbserver/include/fdbserver/workloads/ReadWriteWorkload.actor.h
+++ b/fdbserver/workloads/ReadWriteWorkload.h
@@ -19,33 +19,50 @@
  */
 
 #pragma once
-#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_READWRITEWORKLOAD_ACTOR_G_H)
-#define FDBSERVER_READWRITEWORKLOAD_ACTOR_G_H
-#include "fdbserver/workloads/ReadWriteWorkload.actor.g.h"
-#elif !defined(FDBSERVER_READWRITEWORKLOAD_ACTOR_H)
-#define FDBSERVER_READWRITEWORKLOAD_ACTOR_H
 
 #include "fdbrpc/DDSketch.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/TDMetric.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "flow/TDMetric.h"
 #include <boost/lexical_cast.hpp>
-#include "flow/actorcompiler.h" // This must be the last #include.
-DESCR struct TransactionSuccessMetric {
-	int64_t totalLatency; // ns
-	int64_t startLatency; // ns
-	int64_t commitLatency; // ns
-	int64_t retries; // count
+
+struct TransactionSuccessMetricDescriptor {
+	int64_t totalLatency;
+	int64_t startLatency;
+	int64_t commitLatency;
+	int64_t retries;
 };
 
-DESCR struct TransactionFailureMetric {
-	int64_t startLatency; // ns
-	int64_t errorCode; // flow error code
+template <>
+struct Descriptor<TransactionSuccessMetricDescriptor>
+  : DescribeType<TransactionSuccessMetricDescriptor,
+                 "TransactionSuccessMetric",
+                 DescribeField<&TransactionSuccessMetricDescriptor::totalLatency, "totalLatency", "ns">,
+                 DescribeField<&TransactionSuccessMetricDescriptor::startLatency, "startLatency", "ns">,
+                 DescribeField<&TransactionSuccessMetricDescriptor::commitLatency, "commitLatency", "ns">,
+                 DescribeField<&TransactionSuccessMetricDescriptor::retries, "retries", "count">> {};
+
+struct TransactionFailureMetricDescriptor {
+	int64_t startLatency;
+	int64_t errorCode;
 };
 
-DESCR struct ReadMetric {
-	int64_t readLatency; // ns
+template <>
+struct Descriptor<TransactionFailureMetricDescriptor>
+  : DescribeType<TransactionFailureMetricDescriptor,
+                 "TransactionFailureMetric",
+                 DescribeField<&TransactionFailureMetricDescriptor::startLatency, "startLatency", "ns">,
+                 DescribeField<&TransactionFailureMetricDescriptor::errorCode, "errorCode", "flow error code">> {};
+
+struct ReadMetricDescriptor {
+	int64_t readLatency;
 };
 
+template <>
+struct Descriptor<ReadMetricDescriptor>
+  : DescribeType<ReadMetricDescriptor,
+                 "ReadMetric",
+                 DescribeField<&ReadMetricDescriptor::readLatency, "readLatency", "ns">> {};
+
 // Common ReadWrite test settings
 struct ReadWriteCommon : KVWorkload {
 	static constexpr double sampleError = 0.01;
@@ -72,9 +89,9 @@ struct ReadWriteCommon : KVWorkload {
 	// states of metric
 	Int64MetricHandle totalReadsMetric;
 	Int64MetricHandle totalRetriesMetric;
-	EventMetricHandle<TransactionSuccessMetric> transactionSuccessMetric;
-	EventMetricHandle<TransactionFailureMetric> transactionFailureMetric;
-	EventMetricHandle<ReadMetric> readMetric;
+	EventMetricHandle<TransactionSuccessMetricDescriptor> transactionSuccessMetric;
+	EventMetricHandle<TransactionFailureMetricDescriptor> transactionFailureMetric;
+	EventMetricHandle<ReadMetricDescriptor> readMetric;
 	PerfIntCounter aTransactions, bTransactions, retries;
 	DDSketch<double> latencies, readLatencies, commitLatencies, GRVLatencies, fullReadLatencies;
 	double readLatencyTotal;
@@ -164,6 +181,3 @@ struct ReadWriteCommon : KVWorkload {
 	Standalone<KeyValueRef> operator()(uint64_t n);
 	bool shouldRecord(double checkTime = now());
 };
-
-#include "flow/unactorcompiler.h"
-#endif // FDBSERVER_READWRITEWORKLOAD_ACTOR_H
diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.cpp
similarity index 81%
rename from fdbserver/workloads/RemoveServersSafely.actor.cpp
rename to fdbserver/workloads/RemoveServersSafely.cpp
index 298d3621c33..b639a2ebc02 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.cpp
@@ -1,5 +1,5 @@
 /*
- * RemoveServersSafely.actor.cpp
+ * RemoveServersSafely.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,13 +19,14 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/ManagementAPI.h"
+#include "flow/CoroUtils.h"
 
 template <>
 std::string describe(uint32_t const& item) {
@@ -44,7 +45,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 	std::map<AddressExclusion, Optional<Standalone<StringRef>>> machine_ids; // ip -> Locality Zone id
 	std::map<AddressExclusion, std::set<AddressExclusion>> machineProcesses; // ip -> ip:port
 
-	RemoveServersSafelyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit RemoveServersSafelyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled =
 		    !clientId && g_network->isSimulated(); // only do this on the "first" client, and only when in simulation
 		minMachinesToKill = getOption(options, "minMachinesToKill"_sr, 1);
@@ -57,7 +58,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		kill2Timeout = getOption(options, "kill2Timeout"_sr, 6000.0);
 		killProcesses = deterministicRandom()->random01() < 0.5;
 		if (g_network->isSimulated()) {
-			g_simulator->allowLogSetKills = false;
+			fdbSimulationPolicyState().allowLogSetKills = false;
 		}
 	}
 
@@ -82,7 +83,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Zoneid", it->locality.zoneId().get().toString())
 			    .detail("MachineId", it->locality.machineId().get().toString());
 
-			if (!g_simulator->protectedAddresses.contains(it->address))
+			if (!g_simulator->isProtectedAddress(it->address))
 				processAddrs.push_back(pAddr);
 			machineProcesses[machineIp].insert(pAddr);
 
@@ -145,7 +146,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		if (!enabled)
 			return Void();
 		double delay = deterministicRandom()->random01() * (maxDelay - minDelay) + minDelay;
-		return workloadMain(this, cx, delay, toKill1, toKill2);
+		return workloadMain(cx, delay, toKill1, toKill2);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
@@ -172,7 +173,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		for (auto& netAddr : netAddrs) {
 			auto machineIpPorts = machineProcesses.find(netAddr);
 			if (machineIpPorts != machineProcesses.end()) {
-				ASSERT(machineIpPorts->second.size());
+				ASSERT(!machineIpPorts->second.empty());
 				for (auto& processAdd : machineIpPorts->second)
 					processAddrs.insert(processAdd);
 			} else {
@@ -191,7 +192,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("Failed", processInfo->failed)
 				    .detail("Excluded", processInfo->excluded)
 				    .detail("Rebooting", processInfo->rebooting)
-				    .detail("Protected", g_simulator->protectedAddresses.contains(processInfo->address));
+				    .detail("Protected", g_simulator->isProtectedAddress(processInfo->address));
 			} else {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "ProcessNotToKill")
@@ -200,7 +201,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("Failed", processInfo->failed)
 				    .detail("Excluded", processInfo->excluded)
 				    .detail("Rebooting", processInfo->rebooting)
-				    .detail("Protected", g_simulator->protectedAddresses.contains(processInfo->address));
+				    .detail("Protected", g_simulator->isProtectedAddress(processInfo->address));
 			}
 		}
 		TraceEvent("RemoveAndKill", functionId)
@@ -335,18 +336,17 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 	// toKill1 and toKill2 are two random subsets of all processes. If simply kill all processes in toKill1 or toKill2,
 	// we may kill too many processes to make the cluster unavailable and stuck.
-	ACTOR static Future<Void> workloadMain(RemoveServersSafelyWorkload* self,
-	                                       Database cx,
-	                                       double waitSeconds,
-	                                       std::set<AddressExclusion> toKill1,
-	                                       std::set<AddressExclusion> toKill2) {
-		wait(updateProcessIds(cx));
-		wait(delay(waitSeconds));
+	Future<Void> workloadMain(Database cx,
+	                          double waitSeconds,
+	                          std::set<AddressExclusion> toKill1,
+	                          std::set<AddressExclusion> toKill2) {
+		co_await updateProcessIds(cx);
+		co_await delay(waitSeconds);
 
 		// Removing the first set of machines might legitimately bring the database down, so a timeout is not an error
-		state std::vector<NetworkAddress> firstCoordinators;
-		state std::vector<ISimulator::ProcessInfo*> killProcArray;
-		state bool bClearedFirst;
+		std::vector<NetworkAddress> firstCoordinators;
+		std::vector<ISimulator::ProcessInfo*> killProcArray;
+		bool bClearedFirst{ false };
 
 		TraceEvent("RemoveAndKill")
 		    .detail("Step", "exclude list first")
@@ -356,18 +356,18 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 		// toKill1 may kill too many servers to make cluster unavailable.
 		// Get the processes in toKill1 that are safe to kill
-		killProcArray = self->protectServers(toKill1);
+		killProcArray = protectServers(toKill1);
 		// Update the kill networks to the killable processes
-		toKill1 = self->getNetworks(killProcArray);
+		toKill1 = getNetworks(killProcArray);
 		TraceEvent("RemoveAndKill")
 		    .detail("Step", "exclude list first")
 		    .detail("ToKillModified", describe(toKill1))
 		    .detail("KillTotalModified", toKill1.size())
 		    .detail("ClusterAvailable", g_simulator->isAvailable());
 
-		self->excludeAddresses(toKill1);
+		excludeAddresses(toKill1);
 
-		Optional<Void> result = wait(timeout(removeAndKill(self, cx, toKill1, nullptr, false), self->kill1Timeout));
+		Optional<Void> result = co_await timeout(removeAndKill(cx, toKill1, nullptr, false), kill1Timeout);
 
 		bClearedFirst = result.present();
 		TraceEvent("RemoveAndKill")
@@ -380,7 +380,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 		// Include the servers, if unable to exclude
 		// Reinclude when buggify is on to increase the surface area of the next set of excludes
-		state bool failed = true;
+		bool failed = true;
 		if (!bClearedFirst || BUGGIFY) {
 			// Get the updated list of processes which may have changed due to reboots, deletes, etc
 			TraceEvent("RemoveAndKill")
@@ -388,25 +388,25 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("KillTotal", toKill1.size())
 			    .detail("ToKill", describe(toKill1))
 			    .detail("ClusterAvailable", g_simulator->isAvailable());
-			wait(includeServers(cx, std::vector<AddressExclusion>(1)));
-			wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
-			wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
-			self->includeAddresses(toKill1);
+			co_await includeServers(cx, std::vector<AddressExclusion>(1));
+			co_await includeLocalities(cx, std::vector<std::string>(), failed, true);
+			co_await includeLocalities(cx, std::vector<std::string>(), !failed, true);
+			includeAddresses(toKill1);
 		}
 
 		// toKill2 may kill too many servers to make cluster unavailable.
 		// Get the processes in toKill2 that are safe to kill
-		killProcArray = self->protectServers(toKill2);
+		killProcArray = protectServers(toKill2);
 
 		// Update the kill networks to the killable processes
-		toKill2 = self->getNetworks(killProcArray);
+		toKill2 = getNetworks(killProcArray);
 
 		TraceEvent("RemoveAndKill")
 		    .detail("Step", "exclude list second")
 		    .detail("KillTotal", toKill2.size())
 		    .detail("ToKill", describe(toKill2))
 		    .detail("ClusterAvailable", g_simulator->isAvailable());
-		self->excludeAddresses(toKill2);
+		excludeAddresses(toKill2);
 
 		// The second set of machines is selected so that we can always make progress without it, even after the
 		// permitted number of other permanent failures so we expect to succeed after a finite amount of time
@@ -416,10 +416,10 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("KillTotal", toKill2.size())
 		    .detail("Processes", killProcArray.size())
 		    .detail("ClusterAvailable", g_simulator->isAvailable());
-		wait(reportErrors(timeoutError(removeAndKill(self, cx, toKill2, bClearedFirst ? &toKill1 : nullptr, true),
-		                               self->kill2Timeout),
-		                  "RemoveServersSafelyError",
-		                  UID()));
+		co_await reportErrors(
+		    timeoutError(removeAndKill(cx, toKill2, bClearedFirst ? &toKill1 : nullptr, true), kill2Timeout),
+		    "RemoveServersSafelyError",
+		    UID());
 
 		TraceEvent("RemoveAndKill")
 		    .detail("Step", "excluded second list")
@@ -433,12 +433,10 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("KillTotal", toKill2.size())
 		    .detail("ToKill", describe(toKill2))
 		    .detail("ClusterAvailable", g_simulator->isAvailable());
-		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
-		wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
-		wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
-		self->includeAddresses(toKill2);
-
-		return Void();
+		co_await includeServers(cx, std::vector<AddressExclusion>(1));
+		co_await includeLocalities(cx, std::vector<std::string>(), failed, true);
+		co_await includeLocalities(cx, std::vector<std::string>(), !failed, true);
+		includeAddresses(toKill2);
 	}
 
 	std::vector<ISimulator::ProcessInfo*> killAddresses(std::set<AddressExclusion> const& killAddrs) {
@@ -459,14 +457,14 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("ClusterAvailable", g_simulator->isAvailable())
 			    .detail("RemoveViaClear", removeViaClear);
 			for (auto& killProcess : killProcArray) {
-				if (g_simulator->protectedAddresses.contains(killProcess->address))
+				if (g_simulator->isProtectedAddress(killProcess->address))
 					TraceEvent("RemoveAndKill", functionId)
 					    .detail("Step", "NoKill Process")
 					    .detail("Process", describe(*killProcess))
 					    .detail("Failed", killProcess->failed)
 					    .detail("Rebooting", killProcess->rebooting)
 					    .detail("ClusterAvailable", g_simulator->isAvailable())
-					    .detail("Protected", g_simulator->protectedAddresses.contains(killProcess->address));
+					    .detail("Protected", g_simulator->isProtectedAddress(killProcess->address));
 				else if (removeViaClear) {
 					g_simulator->rebootProcess(killProcess, ISimulator::KillType::RebootProcessAndDelete);
 					TraceEvent("RemoveAndKill", functionId)
@@ -475,12 +473,12 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 					    .detail("Failed", killProcess->failed)
 					    .detail("Rebooting", killProcess->rebooting)
 					    .detail("ClusterAvailable", g_simulator->isAvailable())
-					    .detail("Protected", g_simulator->protectedAddresses.contains(killProcess->address));
+					    .detail("Protected", g_simulator->isProtectedAddress(killProcess->address));
 				}
 				/*
 				                else {
 				                    g_simulator->killProcess( killProcess, ISimulator::KillType::KillInstantly );
-				                    TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator->isAvailable()).detail("Protected", g_simulator->protectedAddresses.contains(killProcess->address));
+				                    TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator->isAvailable()).detail("Protected", g_simulator->isProtectedAddress(killProcess->address));
 				                }
 				*/
 			}
@@ -513,17 +511,16 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 	// If a process is rebooted, it's processid will change. So we need to monitor
 	// such changes and re-issue the locality-based exclusion again.
-	ACTOR static Future<Void> checkLocalityChange(RemoveServersSafelyWorkload* self,
-	                                              Database cx,
-	                                              std::vector<AddressExclusion> toKillArray,
-	                                              std::unordered_set<std::string> origKillLocalities,
-	                                              bool markExcludeAsFailed) {
-		state std::unordered_set<std::string> killLocalities = origKillLocalities;
-
-		loop {
-			wait(delay(10.0));
-			wait(self->updateProcessIds(cx));
-			std::unordered_set<std::string> toKillLocalities = self->getLocalitiesFromAddresses(toKillArray);
+	Future<Void> checkLocalityChange(Database cx,
+	                                 std::vector<AddressExclusion> toKillArray,
+	                                 std::unordered_set<std::string> origKillLocalities,
+	                                 bool markExcludeAsFailed) {
+		std::unordered_set<std::string> killLocalities = origKillLocalities;
+
+		while (true) {
+			co_await delay(10.0);
+			co_await updateProcessIds(cx);
+			std::unordered_set<std::string> toKillLocalities = getLocalitiesFromAddresses(toKillArray);
 			if (toKillLocalities == killLocalities) {
 				continue;
 			}
@@ -538,12 +535,12 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			killLocalities = toKillLocalities;
 
 			// Include back the localities that are no longer in the kill list
-			state bool failed = true;
-			wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
-			wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
+			bool failed = true;
+			co_await includeLocalities(cx, std::vector<std::string>(), failed, true);
+			co_await includeLocalities(cx, std::vector<std::string>(), !failed, true);
 
 			// Exclude the localities that are now in the kill list
-			wait(excludeLocalities(cx, killLocalities, markExcludeAsFailed));
+			co_await excludeLocalities(cx, killLocalities, markExcludeAsFailed);
 			TraceEvent("RemoveAndKill")
 			    .detail("Step", "new localities excluded")
 			    .detail("Localities", describe(killLocalities));
@@ -552,42 +549,41 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 	// Attempts to exclude a set of processes, and once the exclusion is successful it kills them.
 	// If markExcludeAsFailed is true, then it is an error if we cannot complete the exclusion.
-	ACTOR static Future<Void> removeAndKill(RemoveServersSafelyWorkload* self,
-	                                        Database cx,
-	                                        std::set<AddressExclusion> toKill,
-	                                        std::set<AddressExclusion>* pIncAddrs,
-	                                        bool markExcludeAsFailed) {
-		state UID functionId = nondeterministicRandom()->randomUniqueID();
+	Future<Void> removeAndKill(Database cx,
+	                           std::set<AddressExclusion> toKill,
+	                           std::set<AddressExclusion>* pIncAddrs,
+	                           bool markExcludeAsFailed) {
+		UID functionId = nondeterministicRandom()->randomUniqueID();
 
 		// First clear the exclusion list and exclude the given list
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "Including all")
 		    .detail("ClusterAvailable", g_simulator->isAvailable())
 		    .detail("MarkExcludeAsFailed", markExcludeAsFailed);
-		state bool failed = true;
-		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
-		wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
-		wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
+		bool failed = true;
+		co_await includeServers(cx, std::vector<AddressExclusion>(1));
+		co_await includeLocalities(cx, std::vector<std::string>(), failed, true);
+		co_await includeLocalities(cx, std::vector<std::string>(), !failed, true);
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "Included all")
 		    .detail("ClusterAvailable", g_simulator->isAvailable())
 		    .detail("MarkExcludeAsFailed", markExcludeAsFailed);
 		// Reinclude the addresses that were excluded, if present
 		if (pIncAddrs) {
-			self->includeAddresses(*pIncAddrs);
+			includeAddresses(*pIncAddrs);
 		}
 
-		state std::vector<ISimulator::ProcessInfo*> killProcArray;
-		state std::vector<AddressExclusion> toKillArray;
-		state std::vector<AddressExclusion> toKillMarkFailedArray;
-		state AddressExclusion coordExcl;
+		std::vector<ISimulator::ProcessInfo*> killProcArray;
+		std::vector<AddressExclusion> toKillArray;
+		std::vector<AddressExclusion> toKillMarkFailedArray;
+		AddressExclusion coordExcl;
 		// Exclude a coordinator under buggify, but only if fault tolerance is > 0 and kill set is non-empty already
-		if (BUGGIFY && toKill.size()) {
-			Optional<ClusterConnectionString> csOptional = wait(getConnectionString(cx));
-			state std::vector<NetworkAddress> coordinators;
+		if (BUGGIFY && !toKill.empty()) {
+			Optional<ClusterConnectionString> csOptional = co_await getConnectionString(cx);
+			std::vector<NetworkAddress> coordinators;
 			if (csOptional.present()) {
 				ClusterConnectionString cs = csOptional.get();
-				wait(store(coordinators, cs.tryResolveHostnames()));
+				coordinators = co_await cs.tryResolveHostnames();
 			}
 			if (coordinators.size() > 2) {
 				auto randomCoordinator = deterministicRandom()->randomChoice(coordinators);
@@ -599,10 +595,10 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		}
 		std::copy(toKill.begin(), toKill.end(), std::back_inserter(toKillArray));
 		if (markExcludeAsFailed) {
-			state int retries = 0;
-			loop {
-				state bool safe = false;
-				state std::set<AddressExclusion> failSet =
+			int retries = 0;
+			while (true) {
+				bool safe = false;
+				std::set<AddressExclusion> failSet =
 				    random_subset(toKillArray, deterministicRandom()->randomInt(0, toKillArray.size() + 1));
 				toKillMarkFailedArray.resize(failSet.size());
 				std::copy(failSet.begin(), failSet.end(), toKillMarkFailedArray.begin());
@@ -613,19 +609,24 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "SafetyCheck")
 				    .detail("Exclusions", describe(toKillMarkFailedArray));
-				choose {
-					when(bool _safe = wait(checkSafeExclusions(cx, toKillMarkFailedArray))) {
-						safe = _safe && self->protectServers(std::set<AddressExclusion>(toKillMarkFailedArray.begin(),
-						                                                                toKillMarkFailedArray.end()))
+				{
+					auto choice = co_await race(checkSafeExclusions(cx, toKillMarkFailedArray), delay(5.0));
+					if (choice.index() == 0) {
+						bool _safe = std::get<0>(std::move(choice));
+
+						safe = _safe && protectServers(std::set<AddressExclusion>(toKillMarkFailedArray.begin(),
+						                                                          toKillMarkFailedArray.end()))
 						                        .size() == toKillMarkFailedArray.size();
-					}
-					when(wait(delay(5.0))) {
+					} else if (choice.index() == 1) {
+
 						TraceEvent("RemoveAndKill", functionId)
 						    .detail("Step", "SafetyCheckTimedOut")
 						    .detail("Exclusions", describe(toKillMarkFailedArray));
+					} else {
+						UNREACHABLE();
 					}
 				}
-				if (retries == self->maxSafetyCheckRetries) {
+				if (retries == maxSafetyCheckRetries) {
 					// Do not mark as failed if limit is reached
 					TraceEvent("RemoveAndKill", functionId)
 					    .detail("Step", "SafetyCheckLimitReached")
@@ -662,7 +663,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			toKill.erase(removeServer);
 			toKill.insert(coordExcl);
 		}
-		killProcArray = self->getProcesses(toKill);
+		killProcArray = getProcesses(toKill);
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "Activate Server Exclusion")
 		    .detail("KillAddrs", toKill.size())
@@ -674,11 +675,11 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		    .detail("ClusterAvailable", g_simulator->isAvailable())
 		    .detail("MarkExcludeAsFailed", markExcludeAsFailed);
 
-		state bool excludeLocalitiesInsteadOfServers = deterministicRandom()->coinflip();
-		state std::unordered_set<std::string> toKillLocalitiesFailed;
+		bool excludeLocalitiesInsteadOfServers = deterministicRandom()->coinflip();
+		std::unordered_set<std::string> toKillLocalitiesFailed;
 		if (markExcludeAsFailed) {
-			toKillLocalitiesFailed = self->getLocalitiesFromAddresses(toKillMarkFailedArray);
-			if (excludeLocalitiesInsteadOfServers && toKillLocalitiesFailed.size() > 0) {
+			toKillLocalitiesFailed = getLocalitiesFromAddresses(toKillMarkFailedArray);
+			if (excludeLocalitiesInsteadOfServers && !toKillLocalitiesFailed.empty()) {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "Excluding localities with failed option")
 				    .detail("FailedAddressesSize", toKillMarkFailedArray.size())
@@ -686,19 +687,19 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("FailedLocaitiesSize", toKillLocalitiesFailed.size())
 				    .detail("FailedLocaities", describe(toKillLocalitiesFailed));
 
-				wait(excludeLocalities(cx, toKillLocalitiesFailed, true));
+				co_await excludeLocalities(cx, toKillLocalitiesFailed, true);
 			} else {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "Excluding servers with failed option")
 				    .detail("FailedAddressesSize", toKillMarkFailedArray.size())
 				    .detail("FailedAddresses", describe(toKillMarkFailedArray));
 
-				wait(excludeServers(cx, toKillMarkFailedArray, true));
+				co_await excludeServers(cx, toKillMarkFailedArray, true);
 			}
 		}
 
-		state std::unordered_set<std::string> toKillLocalities = self->getLocalitiesFromAddresses(toKillArray);
-		if (excludeLocalitiesInsteadOfServers && toKillLocalities.size() > 0) {
+		std::unordered_set<std::string> toKillLocalities = getLocalitiesFromAddresses(toKillArray);
+		if (excludeLocalitiesInsteadOfServers && !toKillLocalities.empty()) {
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "Excluding localities without failed option")
 			    .detail("AddressesSize", toKillArray.size())
@@ -706,48 +707,46 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("LocaitiesSize", toKillLocalities.size())
 			    .detail("Locaities", describe(toKillLocalities));
 
-			wait(excludeLocalities(cx, toKillLocalities, false));
+			co_await excludeLocalities(cx, toKillLocalities, false);
 		} else {
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "Excluding servers without failed option")
 			    .detail("AddressesSize", toKillArray.size())
 			    .detail("Addresses", describe(toKillArray));
 
-			wait(excludeServers(cx, toKillArray));
+			co_await excludeServers(cx, toKillArray);
 		}
 
 		// We need to skip at least the quorum change if there's nothing to kill, because there might not be enough
 		// servers left alive to do a coordinators auto (?)
-		if (toKill.size()) {
+		if (!toKill.empty()) {
 			// Wait for removal to be safe
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "Wait For Server Exclusion")
 			    .detail("Addresses", describe(toKill))
 			    .detail("ClusterAvailable", g_simulator->isAvailable());
 			if (excludeLocalitiesInsteadOfServers) {
-				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)) ||
-				     checkLocalityChange(self,
-				                         cx,
-				                         toKillArray,
-				                         toKillLocalities,
-				                         markExcludeAsFailed && toKillLocalitiesFailed.size() > 0));
+				co_await (
+				    success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)) ||
+				    checkLocalityChange(
+				        cx, toKillArray, toKillLocalities, markExcludeAsFailed && !toKillLocalitiesFailed.empty()));
 			} else {
-				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
+				co_await checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */);
 			}
 
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "coordinators auto")
-			    .detail("DesiredCoordinators", g_simulator->desiredCoordinators)
+			    .detail("DesiredCoordinators", fdbSimulationPolicyState().desiredCoordinators)
 			    .detail("ClusterAvailable", g_simulator->isAvailable());
 
 			// Setup the coordinators BEFORE the exclusion
 			// Otherwise, we may end up with NotEnoughMachinesForCoordinators
-			state int cycle = 0;
-			state int nQuorum;
+			int cycle = 0;
+			int nQuorum{ 0 };
 			while (true) {
 				cycle++;
-				nQuorum = ((g_simulator->desiredCoordinators + 1) / 2) * 2 - 1;
-				CoordinatorsResult result = wait(changeQuorum(cx, autoQuorumChange(nQuorum)));
+				nQuorum = ((fdbSimulationPolicyState().desiredCoordinators + 1) / 2) * 2 - 1;
+				CoordinatorsResult result = co_await changeQuorum(cx, autoQuorumChange(nQuorum));
 				TraceEvent(result == CoordinatorsResult::SUCCESS || result == CoordinatorsResult::SAME_NETWORK_ADDRESSES
 				               ? SevInfo
 				               : SevWarn,
@@ -756,12 +755,12 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				    .detail("Result", (int)result)
 				    .detail("Attempt", cycle)
 				    .detail("Quorum", nQuorum)
-				    .detail("DesiredCoordinators", g_simulator->desiredCoordinators);
+				    .detail("DesiredCoordinators", fdbSimulationPolicyState().desiredCoordinators);
 				if (result == CoordinatorsResult::SUCCESS || result == CoordinatorsResult::SAME_NETWORK_ADDRESSES)
 					break;
 			}
 
-			self->killAddresses(toKill);
+			killAddresses(toKill);
 		} else {
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "nothing to clear")
@@ -771,8 +770,6 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		TraceEvent("RemoveAndKill", functionId)
 		    .detail("Step", "done")
 		    .detail("ClusterAvailable", g_simulator->isAvailable());
-
-		return Void();
 	}
 
 	static std::vector<ISimulator::ProcessInfo*> getServers() {
@@ -853,8 +850,8 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 	// Update the g_simulator processes list with the process ids
 	// of the workers, that are generated as part of worker creation.
-	ACTOR static Future<Void> updateProcessIds(Database cx) {
-		std::vector<ProcessData> workers = wait(getWorkers(cx));
+	static Future<Void> updateProcessIds(Database cx) {
+		std::vector<ProcessData> workers = co_await getWorkers(cx);
 		std::unordered_map<NetworkAddress, int> addressToIndexMap;
 		for (int i = 0; i < workers.size(); i++) {
 			addressToIndexMap[workers[i].address] = i;
@@ -869,8 +866,6 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 				}
 			}
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/ReportConflictingKeys.actor.cpp b/fdbserver/workloads/ReportConflictingKeys.cpp
similarity index 85%
rename from fdbserver/workloads/ReportConflictingKeys.actor.cpp
rename to fdbserver/workloads/ReportConflictingKeys.cpp
index 1cc0cb9a4f1..2421a79e37f 100644
--- a/fdbserver/workloads/ReportConflictingKeys.actor.cpp
+++ b/fdbserver/workloads/ReportConflictingKeys.cpp
@@ -1,5 +1,5 @@
 /*
- * ReportConflictingKeys.actor.cpp
+ * ReportConflictingKeys.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,13 +21,12 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 // For this test to report properly buggify must be disabled (flow.h) , and failConnection must be disabled in
-// (sim2.actor.cpp)
+// (sim2.cpp)
 struct ReportConflictingKeysWorkload : TestWorkload {
 	static constexpr auto NAME = "ReportConflictingKeys";
 
@@ -43,7 +42,7 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 		out.insert({ "RandomRangeLock" });
 	}
 
-	ReportConflictingKeysWorkload(WorkloadContext const& wcx)
+	explicit ReportConflictingKeysWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), invalidReports("InvalidReports"), commits("commits"), conflicts("Conflicts"),
 	    xacts("Transactions") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -65,12 +64,9 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(const Database& cx) override { return _start(cx->clone(), this); }
+	Future<Void> start(const Database& cx) override { return _start(cx->clone()); }
 
-	ACTOR Future<Void> _start(Database cx, ReportConflictingKeysWorkload* self) {
-		wait(timeout(self->conflictingClient(cx, self), self->testDuration, Void()));
-		return Void();
-	}
+	Future<Void> _start(Database cx) { co_await timeout(conflictingClient(cx, this), testDuration, Void()); }
 
 	Future<bool> check(Database const& cx) override { return invalidReports.getValue() == 0; }
 
@@ -133,17 +129,18 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 		// \xff\xff/transaction/conflicting_keys0)
 		auto resultFuture = ryw->getRange(conflictingKeysRange, CLIENT_KNOBS->TOO_MANY);
 		auto result = resultFuture.get();
-		ASSERT(!result.more && result.size() == 0);
+		ASSERT(!result.more && result.empty());
 	}
 
-	ACTOR Future<Void> conflictingClient(Database cx, ReportConflictingKeysWorkload* self) {
+	Future<Void> conflictingClient(Database cx, ReportConflictingKeysWorkload* self) {
 
-		state Reference<ReadYourWritesTransaction> tr1(new ReadYourWritesTransaction(cx));
-		state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
-		state std::vector<KeyRange> readConflictRanges;
-		state std::vector<KeyRange> writeConflictRanges;
+		Reference<ReadYourWritesTransaction> tr1(new ReadYourWritesTransaction(cx));
+		Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
+		std::vector<KeyRange> readConflictRanges;
+		std::vector<KeyRange> writeConflictRanges;
 
-		loop {
+		while (true) {
+			Optional<Error> err;
 			try {
 				// set the flag for empty key range testing
 				tr1->setOption(FDBTransactionOptions::REPORT_CONFLICTING_KEYS);
@@ -160,22 +157,22 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 				// We have the two tx with same grv, then commit the first
 				// If the second one is not able to commit due to conflicts, verify the returned conflicting keys
 				// Otherwise, there is no conflicts between tr1's writeConflictRange and tr2's readConflictRange
-				Version readVersion = wait(tr1->getReadVersion());
+				Version readVersion = co_await tr1->getReadVersion();
 				tr2->setVersion(readVersion);
 				self->addRandomReadConflictRange(tr1.getPtr(), nullptr);
 				self->addRandomWriteConflictRange(tr1.getPtr(), &writeConflictRanges);
 				++self->commits;
-				wait(tr1->commit());
+				co_await tr1->commit();
 				++self->xacts;
 				// tr1 should never have conflicting keys, test again after the commit
 				self->emptyConflictingKeysTest(tr1);
 
-				state bool foundConflict = false;
+				bool foundConflict = false;
 				try {
 					self->addRandomReadConflictRange(tr2.getPtr(), &readConflictRanges);
 					self->addRandomWriteConflictRange(tr2.getPtr(), nullptr);
 					++self->commits;
-					wait(tr2->commit());
+					co_await tr2->commit();
 					++self->xacts;
 				} catch (Error& e) {
 					if (e.code() != error_code_not_committed)
@@ -184,29 +181,29 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 					++self->conflicts;
 				}
 				// These two conflict sets should not be empty
-				ASSERT(readConflictRanges.size());
-				ASSERT(writeConflictRanges.size());
+				ASSERT(!readConflictRanges.empty());
+				ASSERT(!writeConflictRanges.empty());
 				// check API correctness
 				if (foundConflict) {
 					// \xff\xff/transaction/conflicting_keys is always initialized to false, skip it here
-					state KeyRange ckr = KeyRangeRef(keyAfter(""_sr.withPrefix(conflictingKeysRange.begin)),
-					                                 "\xff\xff"_sr.withPrefix(conflictingKeysRange.begin));
-					// The getRange here using the special key prefix "\xff\xff/transaction/conflicting_keys/" happens
-					// locally Thus, the error handling is not needed here
-					state Future<RangeResult> conflictingKeyRangesFuture = tr2->getRange(ckr, CLIENT_KNOBS->TOO_MANY);
+					KeyRange ckr = KeyRangeRef(keyAfter(""_sr.withPrefix(conflictingKeysRange.begin)),
+					                           "\xff\xff"_sr.withPrefix(conflictingKeysRange.begin));
+					// The getRange here using the special key prefix "\xff\xff/transaction/conflicting_keys/"
+					// happens locally Thus, the error handling is not needed here
+					Future<RangeResult> conflictingKeyRangesFuture = tr2->getRange(ckr, CLIENT_KNOBS->TOO_MANY);
 					ASSERT(conflictingKeyRangesFuture.isReady());
 
-					wait(validateSpecialSubrangeRead(tr2.getPtr(),
-					                                 firstGreaterOrEqual(ckr.begin),
-					                                 firstGreaterOrEqual(ckr.end),
-					                                 GetRangeLimits(),
-					                                 Reverse::False,
-					                                 conflictingKeyRangesFuture.get()));
+					co_await validateSpecialSubrangeRead(tr2.getPtr(),
+					                                     firstGreaterOrEqual(ckr.begin),
+					                                     firstGreaterOrEqual(ckr.end),
+					                                     GetRangeLimits(),
+					                                     Reverse::False,
+					                                     conflictingKeyRangesFuture.get());
 
 					tr2 = makeReference<ReadYourWritesTransaction>(cx);
 
 					const RangeResult conflictingKeyRanges = conflictingKeyRangesFuture.get();
-					ASSERT(conflictingKeyRanges.size() > 0);
+					ASSERT(!conflictingKeyRanges.empty());
 					ASSERT(conflictingKeyRanges.size() <= readConflictRanges.size() * 2);
 					ASSERT(conflictingKeyRanges.size() % 2 == 0);
 					ASSERT(!conflictingKeyRanges.more);
@@ -289,9 +286,12 @@ struct ReportConflictingKeysWorkload : TestWorkload {
 					}
 				}
 			} catch (Error& e) {
-				state Error e2 = e;
-				wait(tr1->onError(e2));
-				wait(tr2->onError(e2));
+				err = e;
+			}
+			if (err.present()) {
+				Error e2 = err.get();
+				co_await tr1->onError(e2);
+				co_await tr2->onError(e2);
 			}
 			readConflictRanges.clear();
 			writeConflictRanges.clear();
diff --git a/fdbserver/workloads/ResolverBug.actor.cpp b/fdbserver/workloads/ResolverBug.cpp
similarity index 66%
rename from fdbserver/workloads/ResolverBug.actor.cpp
rename to fdbserver/workloads/ResolverBug.cpp
index 506bd172da2..df15f4990f5 100644
--- a/fdbserver/workloads/ResolverBug.actor.cpp
+++ b/fdbserver/workloads/ResolverBug.cpp
@@ -18,11 +18,9 @@
  * limitations under the License.
  */
 #include "flow/ProcessEvents.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/ResolverBug.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/resolver/ResolverBug.h"
+#include "fdbserver/core/ServerDBInfo.h"
 
 namespace {
 
@@ -34,7 +32,7 @@ struct ResolverBugWorkload : TestWorkload {
 	KeyRef controlKey = "workload_control"_sr;
 	Promise<Void> bugFound;
 
-	ResolverBugWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit ResolverBugWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		disableFailureInjections = getOption(options, "disableFailureInjections"_sr, true);
 		resolverBug.ignoreTooOldProbability = getOption(options, "ignoreTooOldProbability"_sr, 0.0);
 		resolverBug.ignoreWriteSetProbability = getOption(options, "ignoreWriteSetProbability"_sr, 0.0);
@@ -76,18 +74,16 @@ struct ResolverBugWorkload : TestWorkload {
 		return IWorkloadFactory::create("Cycle", wcx);
 	}
 
-	ACTOR static Future<Void> waitForPhase(std::shared_ptr<ResolverBug> bug, int phase) {
+	static Future<Void> waitForPhase(std::shared_ptr<ResolverBug> bug, int phase) {
 		while (bug->currentPhase != phase) {
-			wait(delay(0.5));
+			co_await delay(0.5);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> waitForPhaseDone(std::shared_ptr<ResolverBug> bug, int phase, int clientCount) {
+	static Future<Void> waitForPhaseDone(std::shared_ptr<ResolverBug> bug, int phase, int clientCount) {
 		while (std::count(bug->cycleState.begin(), bug->cycleState.end(), phase) != clientCount) {
-			wait(delay(0.5));
+			co_await delay(0.5);
 		}
-		return Void();
 	}
 
 	struct ReportTraces {
@@ -97,56 +93,56 @@ struct ResolverBugWorkload : TestWorkload {
 
 	struct OnTestFailure {
 		std::shared_ptr<ResolverBug> bug;
-		OnTestFailure(std::shared_ptr<ResolverBug> bug) : bug(bug) {}
+		explicit OnTestFailure(std::shared_ptr<ResolverBug> bug) : bug(bug) {}
 
 		void operator()(StringRef, auto const& data, Error const&) {
-			BaseTraceEvent* trace = std::any_cast<BaseTraceEvent*>(data);
+			auto* trace = std::any_cast<BaseTraceEvent*>(data);
 			if (trace->getSeverity() == SevError) {
 				bug->bugFound = true;
 			}
 		}
 	};
 
-	ACTOR static Future<Void> driveWorkload(std::shared_ptr<ResolverBug> bug, int clientCount) {
-		state ReportTraces _;
-		state OnTestFailure onTestFailure(bug);
-		state ProcessEvents::Event ev("TraceEvent::TestFailure"_sr, onTestFailure);
-		loop {
+	static Future<Void> driveWorkload(std::shared_ptr<ResolverBug> bug, int clientCount) {
+		ReportTraces _;
+		OnTestFailure onTestFailure(bug);
+		ProcessEvents::Event ev("TraceEvent::TestFailure"_sr, onTestFailure);
+		while (true) {
 			bug->currentPhase = 1;
-			wait(waitForPhaseDone(bug, 1, clientCount));
+			co_await waitForPhaseDone(bug, 1, clientCount);
 			SimBugInjector().enable();
 			bug->currentPhase = 2;
-			wait(waitForPhaseDone(bug, 2, clientCount));
+			co_await waitForPhaseDone(bug, 2, clientCount);
 			SimBugInjector().disable();
 			bug->currentPhase = 3;
-			wait(waitForPhaseDone(bug, 3, clientCount));
+			co_await waitForPhaseDone(bug, 3, clientCount);
 		}
 	}
 
-	ACTOR static Future<Void> _start(ResolverBugWorkload* self, Database cx) {
-		state Reference<TestWorkload> cycle;
-		state std::shared_ptr<ResolverBug> bug = SimBugInjector().get<ResolverBug>(ResolverBugID(), true);
-		loop {
-			wait(waitForPhase(bug, 1));
-			cycle = self->createCycle();
-			wait(cycle->setup(cx));
-			bug->cycleState[self->clientId] = 1;
-			wait(waitForPhase(bug, 2));
-			wait(cycle->start(cx));
-			bug->cycleState[self->clientId] = 2;
-			wait(waitForPhase(bug, 3));
-			wait(success(cycle->check(cx)));
-			bug->cycleState[self->clientId] = 3;
+	Future<Void> _start(Database cx) {
+		Reference<TestWorkload> cycle;
+		std::shared_ptr<ResolverBug> bug = SimBugInjector().get<ResolverBug>(ResolverBugID(), true);
+		while (true) {
+			co_await waitForPhase(bug, 1);
+			cycle = createCycle();
+			co_await cycle->setup(cx);
+			bug->cycleState[clientId] = 1;
+			co_await waitForPhase(bug, 2);
+			co_await cycle->start(cx);
+			bug->cycleState[clientId] = 2;
+			co_await waitForPhase(bug, 3);
+			co_await cycle->check(cx);
+			bug->cycleState[clientId] = 3;
 		}
 	}
 
-	ACTOR static Future<Void> onBug(std::shared_ptr<ResolverBug> bug) {
-		loop {
+	static Future<Void> onBug(std::shared_ptr<ResolverBug> bug) {
+		while (true) {
 			if (bug->bugFound) {
 				TraceEvent("NegativeTestSuccess").log();
-				return Void();
+				co_return;
 			}
-			wait(delay(0.5));
+			co_await delay(0.5);
 		}
 	}
 
@@ -156,7 +152,7 @@ struct ResolverBugWorkload : TestWorkload {
 		if (clientId == 0) {
 			futures.push_back(driveWorkload(bug, clientCount));
 		}
-		futures.push_back(_start(this, cx->clone()));
+		futures.push_back(_start(cx->clone()));
 		return onBug(bug) || waitForAll(futures);
 	}
 	Future<bool> check(Database const& cx) override { return true; };
diff --git a/fdbserver/workloads/Restore.actor.cpp b/fdbserver/workloads/Restore.cpp
similarity index 63%
rename from fdbserver/workloads/Restore.actor.cpp
rename to fdbserver/workloads/Restore.cpp
index cc3389db1e6..a06b599111d 100644
--- a/fdbserver/workloads/Restore.actor.cpp
+++ b/fdbserver/workloads/Restore.cpp
@@ -1,5 +1,5 @@
 /*
- * BackupCorrectness.actor.cpp
+ * Restore.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,17 +19,16 @@
  */
 
 #include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // TODO: explain the purpose of this workload and how it different from the
 // 20+ (literally) other backup/restore workloads.
@@ -46,7 +45,7 @@ struct RestoreWorkload : TestWorkload {
 	bool shouldSkipRestoreRanges;
 	UID randomID;
 
-	RestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit RestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		performRestore = getOption(options, "performRestore"_sr, true);
 		backupTag1 = getOption(options, "backupTag1"_sr, BackupAgentBase::getDefaultTag());
@@ -86,86 +85,86 @@ struct RestoreWorkload : TestWorkload {
 		    .detail("BackupTag", printable(backupTag).c_str())
 		    .detail("AgentRequest", agentRequest);
 
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
-		loop {
-			wait(backupAgent->changePause(cx, true));
+	static Future<Void> changePaused(Database cx, FileBackupAgent* backupAgent) {
+		while (true) {
+			co_await backupAgent->changePause(cx, true);
 			TraceEvent("RW_AgentPaused").log();
-			wait(delay(30 * deterministicRandom()->random01()));
-			wait(backupAgent->changePause(cx, false));
+			co_await delay(30 * deterministicRandom()->random01());
+			co_await backupAgent->changePause(cx, false);
 			TraceEvent("RW_AgentResumed").log();
-			wait(delay(120 * deterministicRandom()->random01()));
+			co_await delay(120 * deterministicRandom()->random01());
 		}
 	}
 
 	// Resume the backup agent if it is paused
-	ACTOR static Future<Void> resumeAgent(Database cx, FileBackupAgent* backupAgent) {
+	static Future<Void> resumeAgent(Database cx, FileBackupAgent* backupAgent) {
 		TraceEvent("RW_AgentResuming").log();
-		wait(backupAgent->changePause(cx, false));
+		co_await backupAgent->changePause(cx, false);
 		TraceEvent("RW_AgentResumed").log();
-		return Void();
 	}
 
-	ACTOR static Future<Void> statusLoop(Database cx, std::string tag) {
-		state FileBackupAgent agent;
-		loop {
-			bool active = wait(agent.checkActive(cx));
+	static Future<Void> statusLoop(Database cx, std::string tag) {
+		FileBackupAgent agent;
+		while (true) {
+			bool active = co_await agent.checkActive(cx);
 			TraceEvent("RW_AgentActivityCheck").detail("IsActive", active);
-			std::string status = wait(agent.getStatus(cx, ShowErrors::True, tag));
+			std::string status = co_await agent.getStatus(cx, ShowErrors::True, tag);
 			puts(status.c_str());
-			std::string statusJSON = wait(agent.getStatusJSON(cx, tag));
+			std::string statusJSON = co_await agent.getStatusJSON(cx, tag);
 			puts(statusJSON.c_str());
-			wait(delay(10.0));
+			co_await delay(10.0);
 		}
 	}
 
-	ACTOR static Future<Void> _start(Database cx, RestoreWorkload* self) {
-		state FileBackupAgent backupAgent;
-		state Future<Void> cp;
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> _start(Database cx) {
+		FileBackupAgent backupAgent;
+		Future<Void> cp;
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		TraceEvent("RW_Arguments")
-		    .detail("BackupTag", printable(self->backupTag))
-		    .detail("PerformRestore", self->performRestore)
-		    .detail("AllowPauses", self->allowPauses);
+		    .detail("BackupTag", printable(backupTag))
+		    .detail("PerformRestore", performRestore)
+		    .detail("AllowPauses", allowPauses);
 
-		if (self->allowPauses && BUGGIFY) {
+		if (allowPauses && BUGGIFY) {
 			cp = changePaused(cx, &backupAgent);
 		} else {
 			cp = resumeAgent(cx, &backupAgent);
 		}
-		state Future<Void> status = statusLoop(cx, self->backupTag.toString());
+		Future<Void> status = statusLoop(cx, backupTag.toString());
 
 		// Increment the backup agent requests
-		if (self->agentRequest) {
+		if (agentRequest) {
 			RestoreWorkload::backupAgentRequests++;
 		}
 
 		try {
-			state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
-			UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx.getReference()));
-			state UID logUid = uidFlag.first;
-			state Key destUidValue = wait(BackupConfig(logUid).destUidValue().getD(cx.getReference()));
-			state Reference<IBackupContainer> lastBackupContainer =
-			    wait(BackupConfig(logUid).backupContainer().getD(cx.getReference()));
-
-			if (lastBackupContainer && self->performRestore) {
+			KeyBackedTag keyBackedTag = makeBackupTag(backupTag.toString());
+			UidAndAbortedFlagT uidFlag = co_await keyBackedTag.getOrThrow(cx.getReference());
+			UID logUid = uidFlag.first;
+			Key destUidValue = co_await BackupConfig(logUid).destUidValue().getD(cx.getReference());
+			Reference<IBackupContainer> lastBackupContainer =
+			    co_await BackupConfig(logUid).backupContainer().getD(cx.getReference());
+
+			if (lastBackupContainer && performRestore) {
 				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(),
 				                                                 lastBackupContainer->getProxy(),
-				                                                 lastBackupContainer->getEncryptionKeyFileName());
-				BackupDescription desc = wait(container->describeBackup());
-				TraceEvent("RW_Restore", self->randomID)
+				                                                 lastBackupContainer->getEncryptionKeyFileName(),
+				                                                 lastBackupContainer->getEncryptionBlockSize());
+				BackupDescription desc = co_await container->describeBackup();
+				TraceEvent("RW_Restore", randomID)
 				    .setMaxEventLength(12000)
 				    .detail("LastBackupContainer", lastBackupContainer->getURL())
-				    .detail("BackupTag", printable(self->backupTag))
+				    .detail("BackupTag", printable(backupTag))
 				    .setMaxFieldLength(10000)
 				    .detail("Description", desc.toString());
-				state Version targetVersion = -1;
+				Version targetVersion = -1;
 				if (desc.maxRestorableVersion.present()) {
 					if (deterministicRandom()->random01() < 0.1) {
 						targetVersion = desc.minRestorableVersion.get();
@@ -178,120 +177,121 @@ struct RestoreWorkload : TestWorkload {
 						                    : desc.maxRestorableVersion.get();
 					}
 				}
-				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+				co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
 					tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					for (auto& kvrange : self->backupRanges) {
+					for (auto& kvrange : backupRanges) {
 						// version needs to be decided before this transaction otherwise
 						// this clear mutation might be backup as well
 						tr->clear(kvrange);
 					}
 					return Void();
-				}));
+				});
 
-				TraceEvent("RW_Restore", self->randomID)
+				TraceEvent("RW_Restore", randomID)
 				    .detail("LastBackupContainer", lastBackupContainer->getURL())
-				    .detail("BackupTag", printable(self->backupTag))
+				    .detail("BackupTag", printable(backupTag))
 				    .detail("TargetVersion", targetVersion);
-				state int restoreIndex = 0;
+				int restoreIndex = 0;
 				// make sure system keys are not present in the restoreRanges as they will get restored first separately
 				// from the rest
 				Standalone<VectorRef<KeyRangeRef>> modifiedRestoreRanges;
-				for (int i = 0; i < self->restoreRanges.size(); ++i) {
-					if (!self->restoreRanges[i].intersects(getSystemBackupRanges())) {
-						modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), self->restoreRanges[i]);
+				for (int i = 0; i < restoreRanges.size(); ++i) {
+					if (!restoreRanges[i].intersects(getSystemBackupRanges())) {
+						modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), restoreRanges[i]);
 					} else {
-						KeyRangeRef normalKeyRange = self->restoreRanges[i] & normalKeys;
+						KeyRangeRef normalKeyRange = restoreRanges[i] & normalKeys;
 						if (!normalKeyRange.empty()) {
 							modifiedRestoreRanges.push_back_deep(modifiedRestoreRanges.arena(), normalKeyRange);
 						}
 					}
 				}
-				self->restoreRanges = modifiedRestoreRanges;
+				restoreRanges = modifiedRestoreRanges;
 
-				Standalone<StringRef> restoreTag(self->backupTag.toString() + "_" + std::to_string(restoreIndex));
+				Standalone<StringRef> restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex));
 				printf("BackupCorrectness, backupAgent.restore is called for restoreIndex:%d tag:%s\n",
 				       restoreIndex,
 				       restoreTag.toString().c_str());
-				TraceEvent("RW_RestoreRanges", self->randomID)
+				TraceEvent("RW_RestoreRanges", randomID)
 				    .detail("RestoreIndex", restoreIndex)
 				    .detail("RestoreTag", printable(restoreTag))
-				    .detail("RestoreRanges", self->restoreRanges.size());
-				state Future<Version> restore;
+				    .detail("RestoreRanges", restoreRanges.size());
+				Future<Version> restore;
 				restore = backupAgent.restore(cx,
 				                              cx,
 				                              restoreTag,
 				                              KeyRef(lastBackupContainer->getURL()),
 				                              lastBackupContainer->getProxy(),
-				                              self->restoreRanges,
+				                              restoreRanges,
 				                              WaitForComplete::True,
 				                              targetVersion,
 				                              Verbose::True,
 				                              Key(),
 				                              Key(),
-				                              self->locked,
+				                              locked,
 				                              UnlockDB::True,
 				                              OnlyApplyMutationLogs::False,
 				                              InconsistentSnapshotOnly::False,
 				                              ::invalidVersion,
 				                              lastBackupContainer->getEncryptionKeyFileName());
 
-				wait(success(restore));
+				co_await restore;
 				ASSERT(!restore.isError());
 			}
-			state Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
-			state Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
-			state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
-			state Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
-			state int displaySystemKeys = 0;
+			Key backupAgentKey = uidPrefixKey(logRangesRange.begin, logUid);
+			Key backupLogValuesKey = destUidValue.withPrefix(backupLogKeys.begin);
+			Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
+			Key backupLatestVersionsKey = uidPrefixKey(backupLatestVersionsPath, logUid);
+			int displaySystemKeys = 0;
 
 			// Ensure that there is no left over key within the backup subspace
-			loop {
-				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+			while (true) {
+				Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
 
-				TraceEvent("RW_CheckLeftoverKeys", self->randomID).detail("BackupTag", printable(self->backupTag));
+				TraceEvent("RW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(backupTag));
 
+				Error err;
 				try {
 					// Check the left over tasks
 					// We have to wait for the list to empty since an abort and get status
 					// can leave extra tasks in the queue
-					TraceEvent("RW_CheckLeftoverTasks", self->randomID).detail("BackupTag", printable(self->backupTag));
-					state int64_t taskCount = wait(backupAgent.getTaskCount(tr));
-					state int waitCycles = 0;
+					TraceEvent("RW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(backupTag));
+					int64_t taskCount = co_await backupAgent.getTaskCount(tr);
+					int waitCycles = 0;
 
 					while (taskCount > 0) {
 						waitCycles++;
 
-						TraceEvent("RW_NonzeroTaskWait", self->randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						TraceEvent("RW_NonzeroTaskWait", randomID)
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("TaskCount", taskCount)
 						    .detail("WaitCycles", waitCycles);
 						printf("%.6f %-10s Wait #%4d for %lld tasks to end\n",
 						       now(),
-						       self->randomID.toString().c_str(),
+						       randomID.toString().c_str(),
 						       waitCycles,
 						       (long long)taskCount);
 
-						wait(delay(5.0));
+						co_await delay(5.0);
 
 						tr = makeReference<ReadYourWritesTransaction>(cx);
-						wait(store(taskCount, backupAgent.getTaskCount(tr)));
+						taskCount = co_await backupAgent.getTaskCount(tr);
 					}
 
 					RangeResult agentValues =
-					    wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
+					    co_await tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100);
 
 					// Error if the system keyspace for the backup tag is not empty
-					if (agentValues.size() > 0) {
+					if (!agentValues.empty()) {
 						displaySystemKeys++;
 						printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n",
 						       agentValues.size(),
 						       printable(backupAgentKey).c_str());
-						TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", self->randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID)
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("LeftOverKeys", agentValues.size())
 						    .detail("KeySpace", printable(backupAgentKey));
 						for (auto& s : agentValues) {
-							TraceEvent("RW_LeftOverKey", self->randomID)
+							TraceEvent("RW_LeftOverKey", randomID)
 							    .detail("Key", printable(StringRef(s.key.toString())))
 							    .detail("Value", printable(StringRef(s.value.toString())));
 							printf("   Key: %-50s  Value: %s\n",
@@ -302,30 +302,30 @@ struct RestoreWorkload : TestWorkload {
 						printf("No left over backup agent configuration keys\n");
 					}
 
-					Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
+					Optional<Value> latestVersion = co_await tr->get(backupLatestVersionsKey);
 					if (latestVersion.present()) {
-						TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", self->randomID)
-						    .detail("BackupTag", printable(self->backupTag))
+						TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID)
+						    .detail("BackupTag", printable(backupTag))
 						    .detail("BackupLatestVersionsKey", backupLatestVersionsKey.printable())
 						    .detail("DestUidValue", destUidValue.printable());
 					} else {
 						printf("No left over backup version key\n");
 					}
 
-					RangeResult versions = wait(tr->getRange(
-					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1));
-					if (!self->shareLogRange || !versions.size()) {
-						RangeResult logValues = wait(
-						    tr->getRange(KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100));
+					RangeResult versions = co_await tr->getRange(
+					    KeyRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath))), 1);
+					if (!shareLogRange || versions.empty()) {
+						RangeResult logValues = co_await tr->getRange(
+						    KeyRange(KeyRangeRef(backupLogValuesKey, strinc(backupLogValuesKey))), 100);
 
 						// Error if the log/mutation keyspace for the backup tag  is not empty
-						if (logValues.size() > 0) {
+						if (!logValues.empty()) {
 							displaySystemKeys++;
 							printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n",
 							       logValues.size(),
 							       printable(backupLogValuesKey).c_str());
-							TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", self->randomID)
-							    .detail("BackupTag", printable(self->backupTag))
+							TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID)
+							    .detail("BackupTag", printable(backupTag))
 							    .detail("LeftOverKeys", logValues.size())
 							    .detail("KeySpace", printable(backupLogValuesKey));
 						} else {
@@ -335,32 +335,32 @@ struct RestoreWorkload : TestWorkload {
 
 					break;
 				} catch (Error& e) {
-					TraceEvent("RW_CheckException", self->randomID).error(e);
-					wait(tr->onError(e));
+					err = e;
 				}
+				TraceEvent("RW_CheckException", randomID).error(err);
+				co_await tr->onError(err);
 			}
 
 			if (displaySystemKeys) {
-				wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
+				co_await TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef());
 			}
 
-			TraceEvent("RW_Complete", self->randomID).detail("BackupTag", printable(self->backupTag));
+			TraceEvent("RW_Complete", randomID).detail("BackupTag", printable(backupTag));
 
 			// Decrement the backup agent requets
-			if (self->agentRequest) {
+			if (agentRequest) {
 				RestoreWorkload::backupAgentRequests--;
 			}
 
 			// SOMEDAY: Remove after backup agents can exist quiescently
-			if ((g_simulator->backupAgents == ISimulator::BackupAgentType::BackupToFile) &&
+			if ((fdbSimulationPolicyState().backupAgents == FDBBackupAgentType::BackupToFile) &&
 			    (!RestoreWorkload::backupAgentRequests)) {
-				g_simulator->backupAgents = ISimulator::BackupAgentType::NoBackupAgents;
+				fdbSimulationPolicyState().backupAgents = FDBBackupAgentType::NoBackupAgents;
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "BackupAndRestorePartitionedCorrectness").error(e).GetLastError();
 			throw;
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/RestoreBackup.actor.cpp b/fdbserver/workloads/RestoreBackup.cpp
similarity index 59%
rename from fdbserver/workloads/RestoreBackup.actor.cpp
rename to fdbserver/workloads/RestoreBackup.cpp
index b488e22ca81..f9f36cb41c8 100644
--- a/fdbserver/workloads/RestoreBackup.actor.cpp
+++ b/fdbserver/workloads/RestoreBackup.cpp
@@ -1,5 +1,5 @@
 /*
- * RestoreBackup.actor.cpp
+ * RestoreBackup.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,16 +20,16 @@
 
 #include "fdbclient/DatabaseConfiguration.h"
 #include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
 
 // TODO: explain the purpose of this workload and how it different from the
 // 20+ (literally) other backup/restore workloads.
@@ -45,7 +45,7 @@ struct RestoreBackupWorkload : TestWorkload {
 	StopWhenDone stopWhenDone{ false };
 	Optional<std::string> encryptionKeyFileName;
 
-	RestoreBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit RestoreBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
 		tag = getOption(options, "tag"_sr, "default"_sr);
 		delayFor = getOption(options, "delayFor"_sr, 10.0);
@@ -60,27 +60,29 @@ struct RestoreBackupWorkload : TestWorkload {
 
 	static constexpr auto NAME = "RestoreBackup";
 
-	ACTOR static Future<Void> waitOnBackup(RestoreBackupWorkload* self, Database cx) {
-		state Version waitForVersion;
-		state UID backupUID;
-		state Transaction tr(cx);
-		loop {
+	Future<Void> waitOnBackup(Database cx) {
+		Version waitForVersion{ 0 };
+		UID backupUID;
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				Version v = wait(tr.getReadVersion());
+				Version v = co_await tr.getReadVersion();
 				waitForVersion = v;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		EBackupState backupState = wait(self->backupAgent.waitBackup(
-		    cx, self->tag.toString(), self->stopWhenDone, &self->backupContainer, &backupUID));
+		EBackupState backupState =
+		    co_await backupAgent.waitBackup(cx, tag.toString(), stopWhenDone, &backupContainer, &backupUID);
 		if (backupState == EBackupState::STATE_COMPLETED) {
-			return Void();
+			co_return;
 		} else if (backupState == EBackupState::STATE_RUNNING_DIFFERENTIAL) {
-			ASSERT(!self->stopWhenDone);
-			loop {
-				BackupDescription desc = wait(self->backupContainer->describeBackup(true));
+			ASSERT(!stopWhenDone);
+			while (true) {
+				BackupDescription desc = co_await backupContainer->describeBackup(true);
 				TraceEvent("BackupVersionGate")
 				    .detail("MaxLogEndVersion", desc.maxLogEnd.present() ? desc.maxLogEnd.get() : invalidVersion)
 				    .detail("ContiguousLogEndVersion",
@@ -89,68 +91,69 @@ struct RestoreBackupWorkload : TestWorkload {
 				if (desc.contiguousLogEnd.present() && desc.contiguousLogEnd.get() >= waitForVersion) {
 					try {
 						TraceEvent("DiscontinuingBackup").log();
-						wait(self->backupAgent.discontinueBackup(cx, self->tag));
+						co_await backupAgent.discontinueBackup(cx, tag);
 					} catch (Error& e) {
 						TraceEvent("ErrorDiscontinuingBackup").error(e);
 						if (e.code() != error_code_backup_unneeded) {
 							throw;
 						}
 					}
-					return Void();
+					co_return;
 				}
-				wait(delay(5.0));
+				co_await delay(5.0);
 			}
 		} else {
 			TraceEvent(SevError, "BadBackupState").detail("BackupState", BackupAgentBase::getStateText(backupState));
 			ASSERT(false);
-			return Void();
+			co_return;
 		}
 	}
 
-	ACTOR static Future<Void> clearDatabase(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> clearDatabase(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.clear(normalKeys);
 				for (auto& r : getSystemBackupRanges()) {
 					tr.clear(r);
 				}
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> _start(RestoreBackupWorkload* self, Database cx) {
-		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
-		wait(delay(self->delayFor));
-		wait(waitOnBackup(self, cx));
-		wait(clearDatabase(cx));
+	Future<Void> _start(Database cx) {
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
+		co_await delay(delayFor);
+		co_await waitOnBackup(cx);
+		co_await clearDatabase(cx);
 
-		wait(success(self->backupAgent.restore(cx,
-		                                       cx,
-		                                       self->tag,
-		                                       Key(self->backupContainer->getURL()),
-		                                       self->backupContainer->getProxy(),
-		                                       WaitForComplete::True,
-		                                       ::invalidVersion,
-		                                       Verbose::True,
-		                                       KeyRange(),
-		                                       Key(),
-		                                       Key(),
-		                                       LockDB::True,
-		                                       OnlyApplyMutationLogs::False,
-		                                       InconsistentSnapshotOnly::False,
-		                                       ::invalidVersion,
-		                                       self->encryptionKeyFileName)));
-		return Void();
+		co_await backupAgent.restore(cx,
+		                             cx,
+		                             tag,
+		                             Key(backupContainer->getURL()),
+		                             backupContainer->getProxy(),
+		                             WaitForComplete::True,
+		                             ::invalidVersion,
+		                             Verbose::True,
+		                             KeyRange(),
+		                             Key(),
+		                             Key(),
+		                             LockDB::True,
+		                             OnlyApplyMutationLogs::False,
+		                             InconsistentSnapshotOnly::False,
+		                             ::invalidVersion,
+		                             encryptionKeyFileName);
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(this, cx); }
+	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(cx); }
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
diff --git a/fdbserver/workloads/RestoreMultiRanges.actor.cpp b/fdbserver/workloads/RestoreMultiRanges.cpp
similarity index 52%
rename from fdbserver/workloads/RestoreMultiRanges.actor.cpp
rename to fdbserver/workloads/RestoreMultiRanges.cpp
index 2ad9d8a58da..f69fb927272 100644
--- a/fdbserver/workloads/RestoreMultiRanges.actor.cpp
+++ b/fdbserver/workloads/RestoreMultiRanges.cpp
@@ -1,5 +1,5 @@
 /*
- * RestoreMultiRanges.actor.cpp
+ * RestoreMultiRanges.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,11 +21,11 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
 
 struct RestoreMultiRangesWorkload : TestWorkload {
 
@@ -33,7 +33,7 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 	Reference<IBackupContainer> backupContainer;
 	Optional<std::string> encryptionKeyFileName;
 
-	RestoreMultiRangesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit RestoreMultiRangesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.5)) {
 			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
 		}
@@ -41,22 +41,25 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 
 	static constexpr const char* NAME = "RestoreMultiRanges";
 
-	ACTOR static Future<Void> clearDatabase(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> clearDatabase(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.clear(normalKeys);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> prepareDatabase(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	static Future<Void> prepareDatabase(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.reset();
 				tr.set("a"_sr, "a"_sr);
@@ -64,11 +67,12 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 				tr.set("b"_sr, "b"_sr);
 				tr.set("bb"_sr, "bb"_sr);
 				tr.set("bbb"_sr, "bbb"_sr);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
@@ -84,23 +88,24 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR static Future<bool> verifyDatabase(Database cx) {
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+	static Future<bool> verifyDatabase(Database cx) {
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 		TraceEvent("RestoreMultiRanges_Verify").detail("UID", randomID);
-		state Transaction tr(cx);
-		state KeyRangeRef range("a"_sr, "z"_sr);
-		loop {
+		Transaction tr(cx);
+		KeyRangeRef range("a"_sr, "z"_sr);
+		while (true) {
+			Error err;
 			try {
 				tr.reset();
 				tr.debugTransaction(randomID);
-				RangeResult kvs = wait(tr.getRange(range, 10));
+				RangeResult kvs = co_await tr.getRange(range, 10);
 				if (kvs.size() != 4) {
 					logTestData(kvs);
 					TraceEvent(SevError, "TestFailureInfo")
 					    .detail("DataSize", kvs.size())
 					    .detail("Expect", 4)
 					    .detail("Workload", NAME);
-					return false;
+					co_return false;
 				}
 				KeyRef keys[4] = { "a"_sr, "aaaa"_sr, "bb"_sr, "bbb"_sr };
 				for (size_t i = 0; i < 4; ++i) {
@@ -109,82 +114,84 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 						    .detail("ExpectKey", keys[i])
 						    .detail("Got", kvs[i].key)
 						    .detail("Index", i);
-						return false;
+						co_return false;
 					}
 				}
 				TraceEvent("RestoreMultiRanges_VerifyPassed");
-				return true;
+				co_return true;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> _start(RestoreMultiRangesWorkload* self, Database cx) {
+	Future<Void> _start(Database cx) {
 		TraceEvent("RestoreMultiRanges_StartBackup");
-		wait(clearDatabase(cx));
-		wait(prepareDatabase(cx));
+		co_await clearDatabase(cx);
+		co_await prepareDatabase(cx);
 
-		if (self->encryptionKeyFileName.present()) {
-			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		if (encryptionKeyFileName.present()) {
+			co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 		}
 
-		state std::string backupContainer = "file://simfdb/backups/";
-		state std::string tagName = "default";
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
+		std::string backupContainer = "file://simfdb/backups/";
+		std::string tagName = "default";
+		Standalone<VectorRef<KeyRangeRef>> backupRanges;
 		backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef("a"_sr, "z"_sr));
 		TraceEvent("RestoreMultiRanges_SubmitBackup");
 		try {
-			wait(self->backupAgent.submitBackup(cx,
-			                                    StringRef(backupContainer),
-			                                    {},
-			                                    deterministicRandom()->randomInt(0, 60),
-			                                    deterministicRandom()->randomInt(0, 100),
-			                                    tagName,
-			                                    backupRanges,
-			                                    StopWhenDone::True,
-			                                    UsePartitionedLog::False,
-			                                    IncrementalBackupOnly::False,
-			                                    self->encryptionKeyFileName));
+			co_await backupAgent.submitBackup(cx,
+			                                  StringRef(backupContainer),
+			                                  {},
+			                                  deterministicRandom()->randomInt(0, 60),
+			                                  deterministicRandom()->randomInt(0, 100),
+			                                  tagName,
+			                                  backupRanges,
+			                                  StopWhenDone::True,
+			                                  MutationLogType::DEFAULT,
+			                                  IncrementalBackupOnly::False,
+			                                  encryptionKeyFileName,
+			                                  encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+			                                  0);
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 				throw;
 		}
 
 		TraceEvent("RestoreMultiRanges_WaitBackup");
-		state Reference<IBackupContainer> container;
-		wait(success(self->backupAgent.waitBackup(cx, tagName, StopWhenDone::True, &container)));
+		Reference<IBackupContainer> container;
+		co_await backupAgent.waitBackup(cx, tagName, StopWhenDone::True, &container);
 
 		TraceEvent("RestoreMultiRanges_ClearDatabase");
-		wait(clearDatabase(cx));
+		co_await clearDatabase(cx);
 
 		TraceEvent("RestoreMultiRanges_Restore");
-		state Standalone<VectorRef<KeyRangeRef>> ranges;
+		Standalone<VectorRef<KeyRangeRef>> ranges;
 		ranges.push_back_deep(ranges.arena(), KeyRangeRef("a"_sr, "aaaaa"_sr));
 		ranges.push_back_deep(ranges.arena(), KeyRangeRef("bb"_sr, "bbbbb"_sr)); // Skip "b"
-		wait(success(self->backupAgent.restore(cx,
-		                                       cx,
-		                                       Key(tagName),
-		                                       Key(container->getURL()),
-		                                       {},
-		                                       ranges,
-		                                       WaitForComplete::True,
-		                                       ::invalidVersion,
-		                                       Verbose::True,
-		                                       Key(),
-		                                       Key(),
-		                                       LockDB::True,
-		                                       UnlockDB::True,
-		                                       OnlyApplyMutationLogs::False,
-		                                       InconsistentSnapshotOnly::False,
-		                                       ::invalidVersion,
-		                                       self->encryptionKeyFileName)));
+		co_await backupAgent.restore(cx,
+		                             cx,
+		                             Key(tagName),
+		                             Key(container->getURL()),
+		                             {},
+		                             ranges,
+		                             WaitForComplete::True,
+		                             ::invalidVersion,
+		                             Verbose::True,
+		                             Key(),
+		                             Key(),
+		                             LockDB::True,
+		                             UnlockDB::True,
+		                             OnlyApplyMutationLogs::False,
+		                             InconsistentSnapshotOnly::False,
+		                             ::invalidVersion,
+		                             encryptionKeyFileName);
 		TraceEvent("RestoreMultiRanges_Success");
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(this, cx); }
+	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(cx); }
 	Future<bool> check(Database const& cx) override { return verifyDatabase(cx); }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
diff --git a/fdbserver/workloads/RestoreValidation.actor.cpp b/fdbserver/workloads/RestoreValidation.cpp
similarity index 71%
rename from fdbserver/workloads/RestoreValidation.actor.cpp
rename to fdbserver/workloads/RestoreValidation.cpp
index ec6e3268d00..6e0782dbea8 100644
--- a/fdbserver/workloads/RestoreValidation.actor.cpp
+++ b/fdbserver/workloads/RestoreValidation.cpp
@@ -1,5 +1,5 @@
 /*
- * RestoreValidation.actor.cpp
+ * RestoreValidation.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -37,11 +37,11 @@
 // - Error detection: Fails the test if audit finds missing or mismatched keys
 
 #include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
+#include "fdbclient/AuditUtils.h"
 #include "fdbclient/ClusterConnectionFile.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct RestoreValidationWorkload : TestWorkload {
@@ -54,7 +54,7 @@ struct RestoreValidationWorkload : TestWorkload {
 	double checkInterval;
 	double maxWaitTime;
 
-	RestoreValidationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit RestoreValidationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		validateAfter = getOption(options, "validateAfter"_sr, 50.0);
 		validationRange = normalKeys;
 		expectedPhase = getOption(options, "expectedPhase"_sr, (int)AuditPhase::Complete);
@@ -73,7 +73,7 @@ struct RestoreValidationWorkload : TestWorkload {
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0) {
-			return _start(this, cx);
+			return _start(cx);
 		}
 		return Void();
 	}
@@ -82,31 +82,32 @@ struct RestoreValidationWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> _start(RestoreValidationWorkload* self, Database cx) {
+	Future<Void> _start(Database cx) {
 		// Only run on client 0 to avoid conflicts (backup/restore runs on client 0)
-		if (self->clientId != 0) {
-			return Void();
+		if (clientId != 0) {
+			co_return;
 		}
 
 		// Wait for the specified time before starting validation
-		TraceEvent("RestoreValidationWorkloadWaiting").detail("WaitTime", self->validateAfter);
-		wait(delay(self->validateAfter));
+		TraceEvent("RestoreValidationWorkloadWaiting").detail("WaitTime", validateAfter);
+		co_await delay(validateAfter);
 
 		// Wait for restore completion marker
 		// BackupAndRestoreValidation sets this key when restore is fully complete
-		state Key completionMarker = "\xff\x02/restoreValidationComplete"_sr;
-		state int checkAttempts = 0;
+		Key completionMarker = "\xff\x02/restoreValidationComplete"_sr;
+		int checkAttempts = 0;
 
 		TraceEvent("RestoreValidationWaitingForRestoreCompletion")
 		    .detail("CompletionMarker", printable(completionMarker));
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state Transaction tr(cx);
+				Transaction tr(cx);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				Optional<Value> markerValue = wait(tr.get(completionMarker));
+				Optional<Value> markerValue = co_await tr.get(completionMarker);
 				if (markerValue.present()) {
 					TraceEvent("RestoreValidationRestoreComplete").detail("CheckAttempts", checkAttempts);
 					break;
@@ -120,52 +121,56 @@ struct RestoreValidationWorkload : TestWorkload {
 					    .detail("CheckAttempts", checkAttempts)
 					    .detail("WaitTimeSeconds", checkAttempts * 5);
 				}
-				wait(delay(5.0));
+				co_await delay(5.0);
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw;
-				}
-				// Retry on transient errors from buggify chaos injection
-				if (e.code() == error_code_grv_proxy_memory_limit_exceeded ||
-				    e.code() == error_code_commit_proxy_memory_limit_exceeded ||
-				    e.code() == error_code_database_locked || e.code() == error_code_transaction_too_old ||
-				    e.code() == error_code_future_version || e.code() == error_code_audit_storage_failed ||
-				    e.code() == error_code_tag_throttled) {
-					TraceEvent(SevWarn, "RestoreValidationRetryableError")
-					    .error(e)
-					    .detail("CheckAttempts", checkAttempts);
-					wait(delay(1.0)); // Backoff before retry
-					// Loop will retry
-				} else {
-					throw;
-				}
+				err = e;
+			}
+			if (!err.isValid()) {
+				continue;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			}
+			// Retry on transient errors from buggify chaos injection
+			if (err.code() == error_code_grv_proxy_memory_limit_exceeded ||
+			    err.code() == error_code_commit_proxy_memory_limit_exceeded ||
+			    err.code() == error_code_database_locked || err.code() == error_code_transaction_too_old ||
+			    err.code() == error_code_future_version || err.code() == error_code_audit_storage_failed ||
+			    err.code() == error_code_tag_throttled) {
+				TraceEvent(SevWarn, "RestoreValidationRetryableError")
+				    .error(err)
+				    .detail("CheckAttempts", checkAttempts);
+				co_await delay(1.0); // Backoff before retry
+				// Loop will retry
+			} else {
+				throw err;
 			}
 		}
 
-		TraceEvent("RestoreValidationWorkloadStarting").detail("Range", self->validationRange);
+		TraceEvent("RestoreValidationWorkloadStarting").detail("Range", validationRange);
 
-		state int auditRetryCount = 0;
-		state int maxAuditRetries = 5;
+		int auditRetryCount = 0;
+		int maxAuditRetries = 5;
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// Trigger the audit_storage validate_restore command
-				state AuditType auditType = AuditType::ValidateRestore;
-				state Reference<IClusterConnectionRecord> clusterFile = cx->getConnectionRecord();
+				AuditType auditType = AuditType::ValidateRestore;
+				Reference<IClusterConnectionRecord> clusterFile = cx->getConnectionRecord();
 
 				TraceEvent("RestoreValidationTriggeringAudit")
 				    .detail("AuditType", (int)auditType)
-				    .detail("Range", self->validationRange)
+				    .detail("Range", validationRange)
 				    .detail("RetryCount", auditRetryCount);
 
 				// Trigger the audit using ManagementAPI with timeout
 				// Use shorter timeout for scheduling (60s) to detect cluster issues early
-				state UID auditId;
+				UID auditId;
 				try {
-					UID scheduleResult = wait(timeoutError(
-					    auditStorage(
-					        clusterFile, self->validationRange, auditType, KeyValueStoreType::END, self->maxWaitTime),
-					    60.0));
+					UID scheduleResult = co_await timeoutError(
+					    auditStorage(clusterFile, validationRange, auditType, KeyValueStoreType::END, maxWaitTime),
+					    60.0);
 					auditId = scheduleResult;
 				} catch (Error& e) {
 					if (e.code() == error_code_timed_out) {
@@ -187,29 +192,29 @@ struct RestoreValidationWorkload : TestWorkload {
 				    .detail("RetryCount", auditRetryCount);
 
 				// Monitor audit progress
-				state double startTime = now();
-				state double lastReportTime = startTime;
-				state AuditPhase finalPhase = AuditPhase::Invalid;
-				state std::string errorMessage;
+				double startTime = now();
+				double lastReportTime = startTime;
+				AuditPhase finalPhase = AuditPhase::Invalid;
+				std::string errorMessage;
 
-				loop {
-					wait(delay(self->checkInterval));
+				while (true) {
+					co_await delay(checkInterval);
 
 					// Get audit status (newFirst=true to get latest states first)
 					// Add timeout to handle cluster recovery/instability
-					state std::vector<AuditStorageState> auditStates;
+					std::vector<AuditStorageState> auditStates;
 					try {
 						std::vector<AuditStorageState> states =
-						    wait(timeoutError(getAuditStates(cx, auditType, true), 60.0));
+						    co_await timeoutError(getAuditStates(cx, auditType, true), 60.0);
 						auditStates = states;
 					} catch (Error& e) {
 						if (e.code() == error_code_timed_out) {
 							// Cluster is likely recovering - check overall timeout and continue
-							if (now() - startTime > self->maxWaitTime) {
+							if (now() - startTime > maxWaitTime) {
 								TraceEvent(SevError, "RestoreValidationTimeout")
 								    .detail("AuditID", auditId)
 								    .detail("ElapsedTime", now() - startTime)
-								    .detail("MaxWaitTime", self->maxWaitTime)
+								    .detail("MaxWaitTime", maxWaitTime)
 								    .detail("Reason", "getAuditStates timed out");
 								throw timed_out();
 							}
@@ -219,9 +224,9 @@ struct RestoreValidationWorkload : TestWorkload {
 					}
 
 					// Filter for our audit ID
-					state bool foundOurAudit = false;
-					state bool allComplete = true;
-					state bool anyError = false;
+					bool foundOurAudit = false;
+					bool allComplete = true;
+					bool anyError = false;
 
 					for (const auto& auditState : auditStates) {
 						if (auditState.id == auditId) {
@@ -266,11 +271,11 @@ struct RestoreValidationWorkload : TestWorkload {
 					}
 
 					// Check timeout
-					if (now() - startTime > self->maxWaitTime) {
+					if (now() - startTime > maxWaitTime) {
 						TraceEvent(SevError, "RestoreValidationTimeout")
 						    .detail("AuditID", auditId)
 						    .detail("ElapsedTime", now() - startTime)
-						    .detail("MaxWaitTime", self->maxWaitTime);
+						    .detail("MaxWaitTime", maxWaitTime);
 						throw timed_out();
 					}
 				}
@@ -279,17 +284,17 @@ struct RestoreValidationWorkload : TestWorkload {
 				TraceEvent("RestoreValidationComplete")
 				    .detail("AuditID", auditId)
 				    .detail("FinalPhase", (int)finalPhase)
-				    .detail("ExpectedPhase", self->expectedPhase)
+				    .detail("ExpectedPhase", expectedPhase)
 				    .detail("ErrorMessage", errorMessage)
 				    .detail("ElapsedTime", now() - startTime);
 
-				if (self->expectSuccess) {
+				if (expectSuccess) {
 					if (finalPhase != AuditPhase::Complete) {
 						// Log as warning since we may retry - only becomes error if all retries fail
 						TraceEvent(SevWarn, "RestoreValidationUnexpectedPhase")
 						    .detail("AuditID", auditId)
 						    .detail("FinalPhase", (int)finalPhase)
-						    .detail("ExpectedPhase", self->expectedPhase)
+						    .detail("ExpectedPhase", expectedPhase)
 						    .detail("ErrorMessage", errorMessage);
 						throw audit_storage_failed();
 					}
@@ -303,7 +308,7 @@ struct RestoreValidationWorkload : TestWorkload {
 					if (finalPhase == AuditPhase::Complete) {
 						TraceEvent(SevError, "RestoreValidationUnexpectedSuccess")
 						    .detail("AuditID", auditId)
-						    .detail("ExpectedPhase", self->expectedPhase);
+						    .detail("ExpectedPhase", expectedPhase);
 						throw audit_storage_task_outdated();
 					}
 				}
@@ -312,30 +317,29 @@ struct RestoreValidationWorkload : TestWorkload {
 				break; // Success!
 
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw;
-				}
-				// Retry audit on failures caused by cluster instability during buggify
-				if (e.code() == error_code_audit_storage_failed && auditRetryCount < maxAuditRetries) {
-					auditRetryCount++;
-					double backoff = std::min(10.0, 2.0 * auditRetryCount);
-					TraceEvent(SevWarn, "RestoreValidationAuditRetry")
-					    .error(e)
-					    .detail("RetryCount", auditRetryCount)
-					    .detail("MaxRetries", maxAuditRetries)
-					    .detail("BackoffSeconds", backoff);
-					wait(delay(backoff));
-					// Loop will retry the entire audit
-				} else {
-					TraceEvent(SevError, "RestoreValidationError")
-					    .errorUnsuppressed(e)
-					    .detail("RetryCount", auditRetryCount);
-					throw;
-				}
+				err = e;
+			}
+			if (err.code() == error_code_actor_cancelled) {
+				throw err;
+			}
+			// Retry audit on failures caused by cluster instability during buggify
+			if (err.code() == error_code_audit_storage_failed && auditRetryCount < maxAuditRetries) {
+				auditRetryCount++;
+				double backoff = std::min(10.0, 2.0 * auditRetryCount);
+				TraceEvent(SevWarn, "RestoreValidationAuditRetry")
+				    .error(err)
+				    .detail("RetryCount", auditRetryCount)
+				    .detail("MaxRetries", maxAuditRetries)
+				    .detail("BackoffSeconds", backoff);
+				co_await delay(backoff);
+				// Loop will retry the entire audit
+			} else {
+				TraceEvent(SevError, "RestoreValidationError")
+				    .errorUnsuppressed(err)
+				    .detail("RetryCount", auditRetryCount);
+				throw err;
 			}
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/Rollback.actor.cpp b/fdbserver/workloads/Rollback.cpp
similarity index 78%
rename from fdbserver/workloads/Rollback.actor.cpp
rename to fdbserver/workloads/Rollback.cpp
index 94d3a8dd32c..52bb36e9dec 100644
--- a/fdbserver/workloads/Rollback.actor.cpp
+++ b/fdbserver/workloads/Rollback.cpp
@@ -1,5 +1,5 @@
 /*
- * Rollback.actor.cpp
+ * Rollback.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,12 @@
 
 #include "flow/DeterministicRandom.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/MasterInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/ServerDBInfo.h"
 
 // Choose a random proxy and a random tLog, represented as unclogTlog.
 // The workload first clogs network link between the chosen proxy and all tLogs but the unclogTlog;
@@ -40,7 +39,7 @@ struct RollbackWorkload : FailureInjectionWorkload {
 
 	RollbackWorkload(WorkloadContext const& wcx, NoOptions) : FailureInjectionWorkload(wcx) {}
 
-	RollbackWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
+	explicit RollbackWorkload(WorkloadContext const& wcx) : FailureInjectionWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		meanDelay = getOption(options, "meanDelay"_sr, meanDelay); // Only matters if multiple==true
 		clogDuration = getOption(options, "clogDuration"_sr, clogDuration);
@@ -66,25 +65,25 @@ struct RollbackWorkload : FailureInjectionWorkload {
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> simulateFailure(Database cx, RollbackWorkload* self) {
-		state ServerDBInfo system = self->dbInfo->get();
+	Future<Void> simulateFailure(Database cx, RollbackWorkload* self) {
+		ServerDBInfo system = self->dbInfo->get();
 		auto tlogs = system.logSystemConfig.allPresentLogs();
 
 		if (tlogs.empty() || system.client.commitProxies.empty()) {
 			TraceEvent(SevInfo, "UnableToTriggerRollback").detail("Reason", "No tlogs in System Map");
-			return Void();
+			co_return;
 		}
 
-		state CommitProxyInterface proxy = deterministicRandom()->randomChoice(system.client.commitProxies);
+		CommitProxyInterface proxy = deterministicRandom()->randomChoice(system.client.commitProxies);
 
 		int utIndex = deterministicRandom()->randomInt(0, tlogs.size());
-		state NetworkAddress uncloggedTLog = tlogs[utIndex].address();
+		NetworkAddress uncloggedTLog = tlogs[utIndex].address();
 
 		for (int t = 0; t < tlogs.size(); t++)
 			if (t != utIndex)
 				if (tlogs[t].address().ip == proxy.address().ip) {
 					TraceEvent(SevInfo, "UnableToTriggerRollback").detail("Reason", "proxy-clogged tLog shared IPs");
-					return Void();
+					co_return;
 				}
 
 		TraceEvent("AttemptingToTriggerRollback")
@@ -100,7 +99,7 @@ struct RollbackWorkload : FailureInjectionWorkload {
 		}
 
 		// While the clogged machines are still clogged...
-		wait(delay(self->clogDuration / 3));
+		co_await delay(self->clogDuration / 3);
 		system = self->dbInfo->get();
 
 		if (self->enableFailures) {
@@ -112,23 +111,21 @@ struct RollbackWorkload : FailureInjectionWorkload {
 			g_simulator->clogInterface(proxy.address().ip, self->clogDuration, ClogAll);
 			g_simulator->clogInterface(uncloggedTLog.ip, self->clogDuration, ClogAll);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> rollbackFailureWorker(Database cx, RollbackWorkload* self, double delay) {
-		state PromiseStream<Void> events;
+	Future<Void> rollbackFailureWorker(Database cx, RollbackWorkload* self, double delay) {
+		PromiseStream<Void> events;
 		if (self->multiple) {
-			state double lastTime = now();
-			loop {
-				wait(poisson(&lastTime, delay));
-				wait(self->simulateFailure(cx, self));
+			double lastTime = now();
+			while (true) {
+				co_await poisson(&lastTime, delay);
+				co_await self->simulateFailure(cx, self);
 			}
 		} else {
-			wait(::delay(deterministicRandom()->random01() *
-			             std::max(0.0, self->testDuration - self->clogDuration * 13.0)));
-			wait(self->simulateFailure(cx, self));
+			co_await ::delay(deterministicRandom()->random01() *
+			                 std::max(0.0, self->testDuration - self->clogDuration * 13.0));
+			co_await self->simulateFailure(cx, self);
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/RyowCorrectness.actor.cpp b/fdbserver/workloads/RyowCorrectness.cpp
similarity index 79%
rename from fdbserver/workloads/RyowCorrectness.actor.cpp
rename to fdbserver/workloads/RyowCorrectness.cpp
index 4d80b4010fe..543d0446a09 100644
--- a/fdbserver/workloads/RyowCorrectness.actor.cpp
+++ b/fdbserver/workloads/RyowCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * RyowCorrectness.actor.cpp
+ * RyowCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,11 +20,10 @@
 
 #include <vector>
 
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/MemoryKeyValueStore.h"
-#include "fdbserver/workloads/ApiWorkload.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "MemoryKeyValueStore.h"
+#include "ApiWorkload.h"
 
 #define TRACE_TRANSACTION 0
 
@@ -62,17 +61,16 @@ struct RyowCorrectnessWorkload : ApiWorkload {
 	// The number of operations to perform on a transaction in between commits
 	int opsPerTransaction;
 
-	RyowCorrectnessWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx, 1) {
+	explicit RyowCorrectnessWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx, 1) {
 		duration = getOption(options, "duration"_sr, 60);
 		opsPerTransaction = getOption(options, "opsPerTransaction"_sr, 50);
 	}
 
-	ACTOR Future<Void> performSetup(Database cx, RyowCorrectnessWorkload* self) {
+	Future<Void> performSetup(Database cx, RyowCorrectnessWorkload* self) {
 		std::vector<TransactionType> types;
 		types.push_back(READ_YOUR_WRITES);
 
-		wait(self->chooseTransactionFactory(cx, types));
-		return Void();
+		co_await self->chooseTransactionFactory(cx, types);
 	}
 
 	Future<Void> performSetup(Database const& cx) override { return performSetup(cx, this); }
@@ -221,35 +219,35 @@ struct RyowCorrectnessWorkload : ApiWorkload {
 	}
 
 	// Applies a sequence of operations to the database and returns the results
-	ACTOR Future<std::vector<RangeResult>> applySequenceToDatabase(Reference<TransactionWrapper> transaction,
-	                                                               std::vector<Operation> sequence,
-	                                                               RyowCorrectnessWorkload* self) {
-		state bool dontUpdateResults = false;
-		state std::vector<RangeResult> results;
-		loop {
+	Future<std::vector<RangeResult>> applySequenceToDatabase(Reference<TransactionWrapper> transaction,
+	                                                         std::vector<Operation> sequence,
+	                                                         RyowCorrectnessWorkload* self) {
+		bool dontUpdateResults = false;
+		std::vector<RangeResult> results;
+		while (true) {
+			Error err;
 			try {
-				state int i;
-				for (i = 0; i < sequence.size(); ++i) {
-					state Operation op = sequence[i];
+				for (int i = 0; i < sequence.size(); ++i) {
+					Operation op = sequence[i];
 
 					if (op.type == Operation::SET) {
 						transaction->set(op.beginKey, op.value);
 					} else if (op.type == Operation::GET) {
-						Optional<Value> val = wait(transaction->get(op.beginKey));
+						Optional<Value> val = co_await transaction->get(op.beginKey);
 						if (!dontUpdateResults)
 							self->pushKVPair(results, op.beginKey, val);
 					} else if (op.type == Operation::GET_RANGE) {
 						KeyRangeRef range(op.beginKey, op.endKey);
-						RangeResult result = wait(transaction->getRange(range, op.limit, op.reverse));
+						RangeResult result = co_await transaction->getRange(range, op.limit, op.reverse);
 						if (!dontUpdateResults)
 							results.push_back((RangeResultRef)result);
 					} else if (op.type == Operation::GET_RANGE_SELECTOR) {
 						RangeResult result =
-						    wait(transaction->getRange(op.beginSelector, op.endSelector, op.limit, op.reverse));
+						    co_await transaction->getRange(op.beginSelector, op.endSelector, op.limit, op.reverse);
 						if (!dontUpdateResults)
 							results.push_back((RangeResultRef)result);
 					} else if (op.type == Operation::GET_KEY) {
-						Key key = wait(transaction->getKey(op.beginSelector));
+						Key key = co_await transaction->getKey(op.beginSelector);
 						if (!dontUpdateResults)
 							self->pushKVPair(results, key, Value());
 					} else if (op.type == Operation::CLEAR) {
@@ -260,18 +258,19 @@ struct RyowCorrectnessWorkload : ApiWorkload {
 					}
 				}
 
-				wait(transaction->commit());
-				return results;
+				co_await transaction->commit();
+				co_return results;
 			} catch (Error& e) {
-				// If the transaction was possibly committed, then keep the results that we got (since they might change
-				// the next time around the loop), but try to commit the transaction again
-				if (e.code() == error_code_commit_unknown_result)
-					dontUpdateResults = true;
-				else if (!dontUpdateResults)
-					results.clear();
-
-				wait(transaction->onError(e));
+				err = e;
 			}
+			// If the transaction was possibly committed, then keep the results that we got (since they might
+			// change the next time around the loop), but try to commit the transaction again
+			if (err.code() == error_code_commit_unknown_result)
+				dontUpdateResults = true;
+			else if (!dontUpdateResults)
+				results.clear();
+
+			co_await transaction->onError(err);
 		}
 	}
 
@@ -326,27 +325,25 @@ struct RyowCorrectnessWorkload : ApiWorkload {
 	}
 
 	// Execute transactions with multiple random operations each
-	ACTOR Future<Void> performTest(Database cx,
-	                               Standalone<VectorRef<KeyValueRef>> data,
-	                               RyowCorrectnessWorkload* self) {
-		loop {
-			state Reference<TransactionWrapper> transaction = self->createTransaction();
-			state std::vector<Operation> sequence = self->generateOperationSequence(data);
-			state std::vector<RangeResult> storeResults = self->applySequenceToStore(sequence);
-			state std::vector<RangeResult> dbResults = wait(self->applySequenceToDatabase(transaction, sequence, self));
-
-			Version readVersion = wait(transaction->getReadVersion());
-
-			state bool result = self->compareResults(dbResults, storeResults, sequence, readVersion);
+	Future<Void> performTest(Database cx, Standalone<VectorRef<KeyValueRef>> data, RyowCorrectnessWorkload* self) {
+		while (true) {
+			Reference<TransactionWrapper> transaction = self->createTransaction();
+			std::vector<Operation> sequence = self->generateOperationSequence(data);
+			std::vector<RangeResult> storeResults = self->applySequenceToStore(sequence);
+			std::vector<RangeResult> dbResults = co_await self->applySequenceToDatabase(transaction, sequence, self);
+
+			Version readVersion = co_await transaction->getReadVersion();
+
+			bool result = self->compareResults(dbResults, storeResults, sequence, readVersion);
 			if (!result)
 				self->testFailure("Transaction results did not match");
 
-			bool result2 = wait(self->compareDatabaseToMemory());
+			bool result2 = co_await self->compareDatabaseToMemory();
 			if (result && !result2)
 				self->testFailure("Database contents did not match");
 
 			if (!result || !result2)
-				return Void();
+				co_return;
 		}
 	}
 
diff --git a/fdbserver/workloads/S3ClientWorkload.actor.cpp b/fdbserver/workloads/S3ClientWorkload.cpp
similarity index 83%
rename from fdbserver/workloads/S3ClientWorkload.actor.cpp
rename to fdbserver/workloads/S3ClientWorkload.cpp
index ba4582f45dd..b8e2dc3bdc5 100644
--- a/fdbserver/workloads/S3ClientWorkload.actor.cpp
+++ b/fdbserver/workloads/S3ClientWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * S3ClientWorkload.actor.cpp
+ * S3ClientWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,8 +18,8 @@
  * limitations under the License.
  */
 
-#include "fdbserver/MockS3Server.h"
-#include "fdbserver/MockS3ServerChaos.h"
+#include "fdbserver/mocks3/MockS3Server.h"
+#include "fdbserver/mocks3/MockS3ServerChaos.h"
 
 #include "fdbrpc/HTTP.h"
 #include "fdbrpc/simulator.h"
@@ -28,17 +28,15 @@
 #include "flow/IRandom.h"
 #include "flow/serialize.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/S3Client.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/BulkLoadUtil.actor.h"
+#include "fdbclient/S3Client.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/BulkLoadUtil.h"
 #include "flow/Platform.h"
 
 #include <string>
 #include <vector>
 #include <libgen.h>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 // Test s3client operations against s3.
 // Run this workload with ../build_output/bin/fdbserver -r simulation -f
 // ../src/foundationdb/tests/slow/S3ClientWorkload.toml
@@ -58,7 +56,7 @@ struct S3ClientWorkload : TestWorkload {
 	double corruptionRate;
 	double maxDelay;
 
-	S3ClientWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {
+	explicit S3ClientWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {
 		s3Url = getOption(options, "s3Url"_sr, ""_sr).toString();
 		if (s3Url.empty()) {
 			// Default location for s3 instance.
@@ -76,7 +74,7 @@ struct S3ClientWorkload : TestWorkload {
 		corruptionRate = getOption(options, "corruptionRate"_sr, 0.01);
 		maxDelay = getOption(options, "maxDelay"_sr, 2.0);
 	}
-	~S3ClientWorkload() {
+	~S3ClientWorkload() override {
 		if (pass) {
 			TraceEvent("S3ClientWorkloadPass");
 		} else {
@@ -90,8 +88,6 @@ struct S3ClientWorkload : TestWorkload {
 		return _setup(this, cx);
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
@@ -157,11 +153,11 @@ struct S3ClientWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> _start(S3ClientWorkload* self, Database cx) {
-		if (self->clientId != 0) {
+	Future<Void> start(Database const& cx) override {
+		if (clientId != 0) {
 			// Our simulation test can trigger multiple same workloads at the same time
 			// Only run one time workload in the simulation
-			return Void();
+			co_return;
 		}
 		if (g_network->isSimulated()) {
 			// Network partition between CC and DD can cause DD no longer existing,
@@ -171,12 +167,12 @@ struct S3ClientWorkload : TestWorkload {
 		}
 
 		// --- BEGIN PRE-TEST CLEANUP ---
-		state std::string download_path_to_clean = "downloaded_credentials";
+		std::string download_path_to_clean = "downloaded_credentials";
 		try {
 			// Attempt to delete the credentials file if it exists
-			if (fileExists(self->credentials)) {
-				deleteFile(self->credentials);
-				TraceEvent(SevDebug, "S3ClientWorkloadCleanedPreExistingFile").detail("File", self->credentials);
+			if (fileExists(credentials)) {
+				deleteFile(credentials);
+				TraceEvent(SevDebug, "S3ClientWorkloadCleanedPreExistingFile").detail("File", credentials);
 			}
 			// Attempt to delete the download target as a directory or file unconditionally
 			// Use eraseDirectoryRecursive as it might exist as a directory
@@ -194,9 +190,8 @@ struct S3ClientWorkload : TestWorkload {
 		// Create a unique directory for this workload instance inside simfdb
 		// This keeps test artifacts in the simulation directory like other workloads
 		// Use deterministic directory name instead of random UID to ensure deterministic behavior
-		state std::string uniqueRunDir =
-		    joinPath(self->simfdbDir,
-		             format("s3_workload_run_%08x_%08x", self->clientId, deterministicRandom()->randomInt(0, 1000000)));
+		std::string uniqueRunDir = joinPath(
+		    simfdbDir, format("s3_workload_run_%08x_%08x", clientId, deterministicRandom()->randomInt(0, 1000000)));
 		try {
 			platform::createDirectory(uniqueRunDir);
 			TraceEvent(SevDebug, "S3ClientWorkloadCreatedRunDir").detail("Dir", uniqueRunDir);
@@ -207,43 +202,47 @@ struct S3ClientWorkload : TestWorkload {
 		// --- END PER-RUN ISOLATION & CLEANUP ---
 
 		// Modify paths to be within the unique directory
-		self->credentials = joinPath(uniqueRunDir, "S3ClientWorkload.blob-credentials.json");
-		state std::string download = joinPath(uniqueRunDir, "downloaded_credentials");
+		credentials = joinPath(uniqueRunDir, "S3ClientWorkload.blob-credentials.json");
+		std::string download = joinPath(uniqueRunDir, "downloaded_credentials");
 
 		// Setup the credentials file inside the unique directory.
-		self->setupCredentialsFile();
+		setupCredentialsFile();
 
 		// Create a unique object key for S3 (using only the base filename)
 		std::string baseFilename =
-		    ::basename(const_cast<char*>(self->credentials.c_str())); // Gets filename from the *new* path
+		    ::basename(const_cast<char*>(credentials.c_str())); // Gets filename from the *new* path
 		// Use deterministic ID based on client ID and test context instead of random UID
 		// This ensures identical behavior across determinism check runs
-		std::string deterministicId = format("%08x_%08x", self->clientId, deterministicRandom()->randomInt(0, 1000000));
+		std::string deterministicId = format("%08x_%08x", clientId, deterministicRandom()->randomInt(0, 1000000));
 		std::string uniqueObjectKey = baseFilename + "_" + deterministicId;
-		state std::string file_url = self->addFileToUrl(uniqueObjectKey, self->s3Url);
-		state bool uploaded = false; // Track if upload started/succeeded
-		state Optional<Error> errorToThrow; // State variable to hold error
+		std::string file_url = addFileToUrl(uniqueObjectKey, s3Url);
+		bool uploaded = false; // Track if upload started/succeeded
+		Optional<Error> errorToThrow; // State variable to hold error
 
+		Error err;
 		try {
 			// Use original local path (now inside unique dir) for source, unique URL for destination
-			wait(copyUpFile(self->credentials, file_url));
+			co_await copyUpFile(credentials, file_url);
 			uploaded = true; // Mark as uploaded only after wait() succeeds
-			wait(copyDownFile(file_url, download));
-			wait(deleteResource(file_url)); // Attempt deletion on success path
+			co_await copyDownFile(file_url, download);
+			co_await deleteResource(file_url); // Attempt deletion on success path
 		} catch (Error& e) {
+			err = e;
+		}
+		if (err.isValid()) {
 			TraceEvent(SevError, "S3ClientWorkloadError") // Log original error
-			    .error(e)
+			    .error(err)
 			    .detail("S3URL", file_url)
-			    .detail("Path", self->credentials)
+			    .detail("Path", credentials)
 			    .detail("Download", download);
 
 			// Store the original error BEFORE attempting cleanup
-			errorToThrow = e;
+			errorToThrow = err;
 
 			// --- Attempt S3 cleanup even on failure ---
 			if (uploaded) { // Only try to delete if we think it was uploaded
 				try {
-					wait(deleteResource(file_url));
+					co_await deleteResource(file_url);
 					TraceEvent(SevWarn, "S3ClientWorkloadCleanedS3AfterError").detail("S3URL", file_url);
 				} catch (Error& cleanup_e) {
 					// Log cleanup error but don't overwrite original error
@@ -252,7 +251,6 @@ struct S3ClientWorkload : TestWorkload {
 					    .detail("S3URL", file_url);
 				}
 			}
-			// --- End S3 cleanup attempt ---
 		}
 
 		// Check if an error occurred and throw it now
@@ -262,7 +260,7 @@ struct S3ClientWorkload : TestWorkload {
 
 		// Compare the contents of the original and downloaded files
 		// Paths are now inside uniqueRunDir
-		std::string originalContent = readFileBytes(self->credentials, 1024 * 1024); // 1MB max size
+		std::string originalContent = readFileBytes(credentials, 1024 * 1024); // 1MB max size
 		std::string downloadedContent = readFileBytes(download, 1024 * 1024); // 1MB max size
 		if (originalContent != downloadedContent) {
 			TraceEvent(SevError, "S3ClientWorkloadContentMismatch")
@@ -274,12 +272,12 @@ struct S3ClientWorkload : TestWorkload {
 		// Cleanup local files - each operation handles its own errors non-fatally
 		// Delete credentials file
 		try {
-			if (fileExists(self->credentials)) {
-				deleteFile(self->credentials);
-				TraceEvent(SevDebug, "S3ClientWorkloadDeletedCredentials").detail("File", self->credentials);
+			if (fileExists(credentials)) {
+				deleteFile(credentials);
+				TraceEvent(SevDebug, "S3ClientWorkloadDeletedCredentials").detail("File", credentials);
 			}
 		} catch (Error& e) {
-			TraceEvent(SevWarn, "S3ClientWorkloadCredentialsCleanupFailed").error(e).detail("File", self->credentials);
+			TraceEvent(SevWarn, "S3ClientWorkloadCredentialsCleanupFailed").error(e).detail("File", credentials);
 		}
 
 		// Delete download file
@@ -302,11 +300,9 @@ struct S3ClientWorkload : TestWorkload {
 			    .detail("Dir", uniqueRunDir)
 			    .detail("Reason", "Non-fatal in simulation");
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> _setup(S3ClientWorkload* self, Database cx) {
+	Future<Void> _setup(S3ClientWorkload* self, Database cx) {
 		// Only client 0 registers the MockS3Server to avoid unnecessary duplicate trace events
 		// Note: Both startMockS3ServerChaos() and registerSimHTTPServer() have internal duplicate detection
 		if (self->clientId == 0) {
@@ -318,7 +314,7 @@ struct S3ClientWorkload : TestWorkload {
 			if (useMockS3 && g_network->isSimulated()) {
 				// Check if 127.0.0.1:8080 is already registered in simulator's httpHandlers
 				std::string serverKey = "127.0.0.1:8080";
-				bool alreadyRegistered = g_simulator->httpHandlers.count(serverKey) > 0;
+				bool alreadyRegistered = g_simulator->httpHandlers.contains(serverKey);
 
 				if (alreadyRegistered) {
 					TraceEvent("S3ClientWorkload")
@@ -333,7 +329,7 @@ struct S3ClientWorkload : TestWorkload {
 
 					// Start MockS3ServerChaos - has internal duplicate detection
 					NetworkAddress listenAddress(IPAddress(0x7f000001), 8080);
-					wait(startMockS3ServerChaos(listenAddress));
+					co_await startMockS3ServerChaos(listenAddress);
 
 					TraceEvent("S3ClientWorkload")
 					    .detail("Phase", "MockS3ServerChaos Started")
@@ -345,7 +341,7 @@ struct S3ClientWorkload : TestWorkload {
 
 					// Register regular MockS3Server using the proper registration function
 					// which automatically enables persistence
-					wait(registerMockS3Server("127.0.0.1", "8080"));
+					co_await registerMockS3Server("127.0.0.1", "8080");
 
 					TraceEvent("S3ClientWorkload")
 					    .detail("Phase", "MockS3Server Registered")
@@ -372,8 +368,6 @@ struct S3ClientWorkload : TestWorkload {
 			    .detail("DelayRate", self->delayRate)
 			    .detail("CorruptionRate", self->corruptionRate);
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/SaveAndKill.actor.cpp b/fdbserver/workloads/SaveAndKill.cpp
similarity index 83%
rename from fdbserver/workloads/SaveAndKill.actor.cpp
rename to fdbserver/workloads/SaveAndKill.cpp
index 9da47212a76..5f093994dfc 100644
--- a/fdbserver/workloads/SaveAndKill.actor.cpp
+++ b/fdbserver/workloads/SaveAndKill.cpp
@@ -1,5 +1,5 @@
 /*
- * SaveAndKill.actor.cpp
+ * SaveAndKill.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,12 @@
  */
 
 #include "fdbclient/DatabaseConfiguration.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
 #include "flow/Knobs.h"
 
@@ -36,7 +37,6 @@
 #define state
 #undef max
 #undef min
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct SaveAndKillWorkload : TestWorkload {
 	static constexpr auto NAME = "SaveAndKill";
@@ -45,7 +45,7 @@ struct SaveAndKillWorkload : TestWorkload {
 	double testDuration;
 	int isRestoring;
 
-	SaveAndKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit SaveAndKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		restartInfo = getOption(options, "restartInfoLocation"_sr, "simfdb/restartInfo.ini"_sr).toString();
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		isRestoring = getOption(options, "isRestoring"_sr, 0);
@@ -56,24 +56,23 @@ struct SaveAndKillWorkload : TestWorkload {
 		g_simulator->disableSwapsToAll();
 		return Void();
 	}
-	Future<Void> start(Database const& cx) override { return _start(this, cx); }
-
-	ACTOR Future<Void> _start(SaveAndKillWorkload* self, Database cx) {
-		state int i;
-		wait(delay(deterministicRandom()->random01() * self->testDuration));
-		DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> start(Database const& cx) override {
+		int i{ 0 };
+		co_await delay(deterministicRandom()->random01() * testDuration);
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 
 		CSimpleIni ini;
 		ini.SetUnicode();
-		ini.LoadFile(self->restartInfo.c_str());
+		ini.LoadFile(restartInfo.c_str());
 
-		ini.SetValue("RESTORE", "isRestoring", format("%d", self->isRestoring).c_str());
+		ini.SetValue("RESTORE", "isRestoring", format("%d", isRestoring).c_str());
 		ini.SetValue("META", "processesPerMachine", format("%d", g_simulator->processesPerMachine).c_str());
 		ini.SetValue("META", "listenersPerProcess", format("%d", g_simulator->listenersPerProcess).c_str());
-		ini.SetValue("META", "desiredCoordinators", format("%d", g_simulator->desiredCoordinators).c_str());
-		ini.SetValue("META", "connectionString", g_simulator->connectionString.c_str());
-		ini.SetValue("META", "testerCount", format("%d", g_simulator->testerCount).c_str());
-		ini.SetValue("META", "tssMode", format("%d", g_simulator->tssMode).c_str());
+		ini.SetValue(
+		    "META", "desiredCoordinators", format("%d", fdbSimulationPolicyState().desiredCoordinators).c_str());
+		ini.SetValue("META", "connectionString", fdbSimulationPolicyState().connectionString.c_str());
+		ini.SetValue("META", "testerCount", format("%d", fdbSimulationPolicyState().testerCount).c_str());
+		ini.SetValue("META", "tssMode", format("%d", static_cast<int>(fdbSimulationPolicyState().tssMode)).c_str());
 		ini.SetValue("META", "mockDNS", INetworkConnections::net()->convertMockDNSToString().c_str());
 		ini.SetBoolValue("META", "enableShardEncodeLocationMetadata", SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
 		ini.SetBoolValue("META", "encryptHeaderAuthTokenEnabled", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
@@ -143,19 +142,17 @@ struct SaveAndKillWorkload : TestWorkload {
 		}
 
 		ini.SetValue("META", "machineCount", format("%d", machines.size()).c_str());
-		ini.SaveFile(self->restartInfo.c_str());
+		ini.SaveFile(restartInfo.c_str());
 
 		for (auto process = allProcessesMap.begin(); process != allProcessesMap.end(); process++) {
 			g_simulator->killProcess(process->second, ISimulator::KillType::Reboot);
 		}
 
 		for (i = 0; i < 100; i++) {
-			wait(delay(0.0));
+			co_await delay(0.0);
 		}
 
 		g_simulator->stop();
-
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/SelectorCorrectness.actor.cpp b/fdbserver/workloads/SelectorCorrectness.cpp
similarity index 79%
rename from fdbserver/workloads/SelectorCorrectness.actor.cpp
rename to fdbserver/workloads/SelectorCorrectness.cpp
index 145f547ed8e..2fa697a30d4 100644
--- a/fdbserver/workloads/SelectorCorrectness.actor.cpp
+++ b/fdbserver/workloads/SelectorCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * SelectorCorrectness.actor.cpp
+ * SelectorCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,10 +19,9 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct SelectorCorrectnessWorkload : TestWorkload {
 	static constexpr auto NAME = "SelectorCorrectness";
@@ -34,7 +33,7 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter transactions, retries;
 
-	SelectorCorrectnessWorkload(WorkloadContext const& wcx)
+	explicit SelectorCorrectnessWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries") {
 
 		minOperationsPerTransaction = getOption(options, "minOperationsPerTransaction"_sr, 10);
@@ -62,24 +61,27 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 		m.push_back(retries.getMetric());
 	}
 
-	ACTOR Future<Void> SelectorCorrectnessSetup(Database cx, SelectorCorrectnessWorkload* self) {
-		state Value myValue = StringRef(format("%010d", deterministicRandom()->randomInt(0, 10000000)));
-		state Transaction tr(cx);
+	Future<Void> SelectorCorrectnessSetup(Database cx, SelectorCorrectnessWorkload* self) {
+		Value myValue = StringRef(format("%010d", deterministicRandom()->randomInt(0, 10000000)));
+		Transaction tr(cx);
 
 		if (!self->testReadYourWrites) {
-			loop {
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < self->maxKeySpace; i += 2)
 						tr.set(StringRef(format("%010d", i)), myValue);
 
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		} else {
-			loop {
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < self->maxKeySpace; i += 4)
 						tr.set(StringRef(format("%010d", i)), myValue);
@@ -87,36 +89,35 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 						if (deterministicRandom()->random01() > 0.5)
 							tr.set(StringRef(format("%010d", i)), myValue);
 
-					wait(tr.commit());
+					co_await tr.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> SelectorCorrectnessClient(Database cx, SelectorCorrectnessWorkload* self) {
-		state int i;
-		state int j;
-		state std::string myKeyA;
-		state std::string myKeyB;
-		state Value myValue;
-		state bool onEqualA;
-		state bool onEqualB;
-		state int offsetA;
-		state int offsetB;
-		state Standalone<StringRef> maxKey;
-		state Reverse reverse = Reverse::False;
+	Future<Void> SelectorCorrectnessClient(Database cx, SelectorCorrectnessWorkload* self) {
+		int i{ 0 };
+		int j{ 0 };
+		std::string myKeyA;
+		std::string myKeyB;
+		Value myValue;
+		bool onEqualA{ false };
+		bool onEqualB{ false };
+		int offsetA{ 0 };
+		int offsetB{ 0 };
+		Standalone<StringRef> maxKey;
+		Reverse reverse = Reverse::False;
 
 		maxKey = Standalone<StringRef>(format("%010d", self->maxKeySpace + 1));
 
-		loop {
+		while (true) {
 
-			state Transaction tr(cx);
-			state ReadYourWritesTransaction trRYOW(cx);
+			Transaction tr(cx);
+			ReadYourWritesTransaction trRYOW(cx);
 
 			if (self->testReadYourWrites) {
 				myValue = StringRef(format("%010d", deterministicRandom()->randomInt(0, 10000000)));
@@ -127,17 +128,18 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 						trRYOW.set(StringRef(format("%010d", i)), myValue);
 			}
 
+			Error err;
 			try {
 				for (i = 0; i < deterministicRandom()->randomInt(self->minOperationsPerTransaction,
 				                                                 self->maxOperationsPerTransaction + 1);
 				     i++) {
 					j = deterministicRandom()->randomInt(0, 2);
 					if (j < 1) {
-						state int searchInt = deterministicRandom()->randomInt(0, self->maxKeySpace);
+						int searchInt = deterministicRandom()->randomInt(0, self->maxKeySpace);
 						myKeyA = format("%010d", searchInt);
 
 						if (self->testReadYourWrites) {
-							Optional<Value> getTest = wait(trRYOW.get(StringRef(myKeyA)));
+							Optional<Value> getTest = co_await trRYOW.get(StringRef(myKeyA));
 							if ((searchInt % 2 == 0 && !getTest.present()) ||
 							    (searchInt % 2 == 1 && getTest.present())) {
 								TraceEvent(SevError, "RanSelTestFailure")
@@ -145,7 +147,7 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 								    .detail("KeyA", myKeyA);
 							}
 						} else {
-							Optional<Value> getTest = wait(tr.get(StringRef(myKeyA)));
+							Optional<Value> getTest = co_await tr.get(StringRef(myKeyA));
 							if ((searchInt % 2 == 0 && !getTest.present()) ||
 							    (searchInt % 2 == 1 && getTest.present())) {
 								TraceEvent(SevError, "RanSelTestFailure")
@@ -167,7 +169,7 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 						reverse.set(deterministicRandom()->coinflip());
 
 						//TraceEvent("RYOWgetRange").detail("KeyA", myKeyA).detail("KeyB", myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB).detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("Direction",direction);
-						state int expectedSize =
+						int expectedSize =
 						    (std::min(abmax + 2 * offsetB - (abmax % 2 == 1 ? 1 : (onEqualB ? 0 : 2)),
 						              self->maxKeySpace) -
 						     (std::max(abmin + 2 * offsetA - (abmin % 2 == 1 ? 1 : (onEqualA ? 0 : 2)), 0))) /
@@ -175,11 +177,11 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 
 						if (self->testReadYourWrites) {
 							RangeResult getRangeTest =
-							    wait(trRYOW.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-							                         KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-							                         2 * (self->maxKeySpace + self->maxOffset),
-							                         Snapshot::False,
-							                         reverse));
+							    co_await trRYOW.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+							                             KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+							                             2 * (self->maxKeySpace + self->maxOffset),
+							                             Snapshot::False,
+							                             reverse);
 
 							int trueSize = 0;
 							while (trueSize < getRangeTest.size() &&
@@ -203,11 +205,11 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 							}
 						} else {
 							RangeResult getRangeTest =
-							    wait(tr.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
-							                     KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
-							                     2 * (self->maxKeySpace + self->maxOffset),
-							                     Snapshot::False,
-							                     reverse));
+							    co_await tr.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA),
+							                         KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB),
+							                         2 * (self->maxKeySpace + self->maxOffset),
+							                         Snapshot::False,
+							                         reverse);
 
 							int trueSize = 0;
 							while (trueSize < getRangeTest.size() &&
@@ -237,10 +239,13 @@ struct SelectorCorrectnessWorkload : TestWorkload {
 				trRYOW.reset();
 				++self->transactions;
 			} catch (Error& e) {
-				wait(trRYOW.onError(e));
-				trRYOW.reset();
-				++self->retries;
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await trRYOW.onError(err);
 			}
+			trRYOW.reset();
+			++self->retries;
 		}
 	}
 };
diff --git a/fdbserver/workloads/Serializability.actor.cpp b/fdbserver/workloads/Serializability.cpp
similarity index 57%
rename from fdbserver/workloads/Serializability.actor.cpp
rename to fdbserver/workloads/Serializability.cpp
index a6fa9d9a5c7..566261b54fd 100644
--- a/fdbserver/workloads/Serializability.actor.cpp
+++ b/fdbserver/workloads/Serializability.cpp
@@ -1,5 +1,5 @@
 /*
- * Serializability.actor.cpp
+ * Serializability.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "flow/ActorCollection.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct SerializabilityWorkload : TestWorkload {
 	static constexpr auto NAME = "Serializability";
@@ -66,7 +65,7 @@ struct SerializabilityWorkload : TestWorkload {
 		Optional<KeyRange> readConflictOp;
 	};
 
-	SerializabilityWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), success(true) {
+	explicit SerializabilityWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), success(true) {
 		testDuration = getOption(options, "testDuration"_sr, 30.0);
 		numOps = getOption(options, "numOps"_sr, 21);
 		nodes = getOption(options, "nodes"_sr, 1000);
@@ -228,15 +227,14 @@ struct SerializabilityWorkload : TestWorkload {
 		futures.back() = tag(::success(futures.back()), T());
 	}
 
-	ACTOR static Future<Void> runTransaction(ReadYourWritesTransaction* tr,
-	                                         std::vector<TransactionOperation> ops,
-	                                         std::vector<Future<Optional<Value>>>* getFutures,
-	                                         std::vector<Future<Key>>* getKeyFutures,
-	                                         std::vector<Future<RangeResult>>* getRangeFutures,
-	                                         std::vector<Future<Void>>* watchFutures,
-	                                         bool checkSnapshotReads) {
-		state int opNum = 0;
-		for (; opNum < ops.size(); opNum++) {
+	static Future<Void> runTransaction(ReadYourWritesTransaction* tr,
+	                                   std::vector<TransactionOperation> ops,
+	                                   std::vector<Future<Optional<Value>>>* getFutures,
+	                                   std::vector<Future<Key>>* getKeyFutures,
+	                                   std::vector<Future<RangeResult>>* getRangeFutures,
+	                                   std::vector<Future<Void>>* watchFutures,
+	                                   bool checkSnapshotReads) {
+		for (int opNum = 0; opNum < ops.size(); ++opNum) {
 			if (ops[opNum].getKeyOp.present()) {
 				auto& op = ops[opNum].getKeyOp.get();
 				//TraceEvent("SRL_GetKey").detail("Key", op.key.toString()).detail("Snapshot", op.snapshot);
@@ -283,57 +281,54 @@ struct SerializabilityWorkload : TestWorkload {
 
 			// sometimes wait for a random operation
 			if (deterministicRandom()->random01() < 0.2) {
-				state int waitType = deterministicRandom()->randomInt(0, 4);
-				loop {
-					if (waitType == 0 && getFutures->size()) {
-						wait(::success(deterministicRandom()->randomChoice(*getFutures)));
+				int waitType = deterministicRandom()->randomInt(0, 4);
+				while (true) {
+					if (waitType == 0 && !getFutures->empty()) {
+						co_await deterministicRandom()->randomChoice(*getFutures);
 						break;
-					} else if (waitType == 1 && getKeyFutures->size()) {
-						wait(::success(deterministicRandom()->randomChoice(*getKeyFutures)));
+					} else if (waitType == 1 && !getKeyFutures->empty()) {
+						co_await deterministicRandom()->randomChoice(*getKeyFutures);
 						break;
-					} else if (waitType == 2 && getRangeFutures->size()) {
-						wait(::success(deterministicRandom()->randomChoice(*getRangeFutures)));
+					} else if (waitType == 2 && !getRangeFutures->empty()) {
+						co_await deterministicRandom()->randomChoice(*getRangeFutures);
 						break;
 					} else if (waitType == 3) {
-						wait(delay(0.001 * deterministicRandom()->random01()));
+						co_await delay(0.001 * deterministicRandom()->random01());
 						break;
 					}
 					waitType = (waitType + 1) % 4;
 				}
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<RangeResult> getDatabaseContents(Database cx, int nodes) {
-		state ReadYourWritesTransaction tr(cx);
+	static Future<RangeResult> getDatabaseContents(Database cx, int nodes) {
+		ReadYourWritesTransaction tr(cx);
 
-		RangeResult result = wait(tr.getRange(normalKeys, nodes + 1));
+		RangeResult result = co_await tr.getRange(normalKeys, nodes + 1);
 		ASSERT(result.size() <= nodes);
-		return result;
+		co_return result;
 	}
 
-	ACTOR static Future<Void> resetDatabase(Database cx, Standalone<VectorRef<KeyValueRef>> data) {
-		state ReadYourWritesTransaction tr(cx);
+	static Future<Void> resetDatabase(Database cx, Standalone<VectorRef<KeyValueRef>> data) {
+		ReadYourWritesTransaction tr(cx);
 
 		tr.clear(normalKeys);
 		for (auto kv : data)
 			tr.set(kv.key, kv.value);
-		wait(tr.commit());
+		co_await tr.commit();
 		//TraceEvent("SRL_Reset");
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, SerializabilityWorkload* self) {
-		state double startTime = now();
+	Future<Void> _start(Database cx, SerializabilityWorkload* self) {
+		double startTime = now();
 
-		loop {
-			state std::vector<ReadYourWritesTransaction> tr;
-			state std::vector<std::vector<Future<Optional<Value>>>> getFutures;
-			state std::vector<std::vector<Future<Key>>> getKeyFutures;
-			state std::vector<std::vector<Future<RangeResult>>> getRangeFutures;
-			state std::vector<std::vector<Future<Void>>> watchFutures;
+		while (true) {
+			std::vector<ReadYourWritesTransaction> tr;
+			std::vector<std::vector<Future<Optional<Value>>>> getFutures;
+			std::vector<std::vector<Future<Key>>> getKeyFutures;
+			std::vector<std::vector<Future<RangeResult>>> getRangeFutures;
+			std::vector<std::vector<Future<Void>>> watchFutures;
 
 			for (int i = 0; i < 5; i++) {
 				tr.push_back(ReadYourWritesTransaction(cx));
@@ -343,83 +338,68 @@ struct SerializabilityWorkload : TestWorkload {
 				watchFutures.push_back(std::vector<Future<Void>>());
 			}
 
-			try {
-				if (now() - startTime > self->testDuration)
-					return Void();
-
-				// Generate initial data
-				state Standalone<VectorRef<KeyValueRef>> initialData;
-				int initialAmount = deterministicRandom()->randomInt(0, 100);
-				for (int i = 0; i < initialAmount; i++) {
-					Key key = self->getRandomKey();
-					Value value = self->getRandomValue();
-					initialData.push_back_deep(initialData.arena(), KeyValueRef(key, value));
-					//TraceEvent("SRL_Init").detail("Key", printable(key)).detail("Value", printable(value));
-				}
-
-				// Generate three random transactions
-				state std::vector<TransactionOperation> a = self->randomTransaction();
-				state std::vector<TransactionOperation> b = self->randomTransaction();
-				state std::vector<TransactionOperation> c = self->randomTransaction();
-
-				// reset database to known state
-				wait(resetDatabase(cx, initialData));
-
-				wait(runTransaction(
-				    &tr[0], a, &getFutures[0], &getKeyFutures[0], &getRangeFutures[0], &watchFutures[0], true));
-				wait(tr[0].commit());
+			{
+				Error err;
+				try {
+					if (now() - startTime > self->testDuration)
+						co_return;
+
+					// Generate initial data
+					Standalone<VectorRef<KeyValueRef>> initialData;
+					int initialAmount = deterministicRandom()->randomInt(0, 100);
+					for (int i = 0; i < initialAmount; i++) {
+						Key key = self->getRandomKey();
+						Value value = self->getRandomValue();
+						initialData.push_back_deep(initialData.arena(), KeyValueRef(key, value));
+						//TraceEvent("SRL_Init").detail("Key", printable(key)).detail("Value", printable(value));
+					}
 
-				//TraceEvent("SRL_FinishedA");
+					// Generate three random transactions
+					std::vector<TransactionOperation> a = self->randomTransaction();
+					std::vector<TransactionOperation> b = self->randomTransaction();
+					std::vector<TransactionOperation> c = self->randomTransaction();
 
-				wait(runTransaction(
-				    &tr[1], b, &getFutures[0], &getKeyFutures[0], &getRangeFutures[0], &watchFutures[0], true));
-				wait(tr[1].commit());
+					// reset database to known state
+					co_await resetDatabase(cx, initialData);
 
-				//TraceEvent("SRL_FinishedB");
+					co_await runTransaction(
+					    &tr[0], a, &getFutures[0], &getKeyFutures[0], &getRangeFutures[0], &watchFutures[0], true);
+					co_await tr[0].commit();
 
-				wait(runTransaction(
-				    &tr[2], c, &getFutures[2], &getKeyFutures[2], &getRangeFutures[2], &watchFutures[2], false));
-				wait(tr[2].commit());
+					//TraceEvent("SRL_FinishedA");
 
-				// get contents of database
-				state RangeResult result1 = wait(getDatabaseContents(cx, self->nodes));
+					co_await runTransaction(
+					    &tr[1], b, &getFutures[0], &getKeyFutures[0], &getRangeFutures[0], &watchFutures[0], true);
+					co_await tr[1].commit();
 
-				// reset database to known state
-				wait(resetDatabase(cx, initialData));
+					//TraceEvent("SRL_FinishedB");
 
-				wait(runTransaction(
-				    &tr[3], a, &getFutures[3], &getKeyFutures[3], &getRangeFutures[3], &watchFutures[3], true));
-				wait(runTransaction(
-				    &tr[3], b, &getFutures[3], &getKeyFutures[3], &getRangeFutures[3], &watchFutures[3], true));
-				wait(runTransaction(
-				    &tr[4], c, &getFutures[4], &getKeyFutures[4], &getRangeFutures[4], &watchFutures[4], false));
-				wait(tr[3].commit());
-				wait(tr[4].commit());
+					co_await runTransaction(
+					    &tr[2], c, &getFutures[2], &getKeyFutures[2], &getRangeFutures[2], &watchFutures[2], false);
+					co_await tr[2].commit();
 
-				// get contents of database
-				RangeResult result2 = wait(getDatabaseContents(cx, self->nodes));
+					// get contents of database
+					RangeResult result1 = co_await getDatabaseContents(cx, self->nodes);
 
-				if (result1.size() != result2.size()) {
-					TraceEvent(SevError, "SRL_ResultMismatch")
-					    .detail("Size1", result1.size())
-					    .detail("Size2", result2.size());
+					// reset database to known state
+					co_await resetDatabase(cx, initialData);
 
-					for (auto kv : result1)
-						TraceEvent("SRL_Result1").detail("Kv", printable(kv));
-					for (auto kv : result2)
-						TraceEvent("SRL_Result2").detail("Kv", printable(kv));
+					co_await runTransaction(
+					    &tr[3], a, &getFutures[3], &getKeyFutures[3], &getRangeFutures[3], &watchFutures[3], true);
+					co_await runTransaction(
+					    &tr[3], b, &getFutures[3], &getKeyFutures[3], &getRangeFutures[3], &watchFutures[3], true);
+					co_await runTransaction(
+					    &tr[4], c, &getFutures[4], &getKeyFutures[4], &getRangeFutures[4], &watchFutures[4], false);
+					co_await tr[3].commit();
+					co_await tr[4].commit();
 
-					ASSERT(false);
-				}
+					// get contents of database
+					RangeResult result2 = co_await getDatabaseContents(cx, self->nodes);
 
-				for (int i = 0; i < result1.size(); i++) {
-					if (result1[i] != result2[i]) {
+					if (result1.size() != result2.size()) {
 						TraceEvent(SevError, "SRL_ResultMismatch")
-						    .detail("I", i)
-						    .detail("Result1", printable(result1[i]))
-						    .detail("Result2", printable(result2[i]))
-						    .detail("Result1Value", printable(result1[i].value))
-						    .detail("Result2Value", printable(result2[i].value));
+						    .detail("Size1", result1.size())
+						    .detail("Size2", result2.size());
 
 						for (auto kv : result1)
 							TraceEvent("SRL_Result1").detail("Kv", printable(kv));
@@ -428,50 +408,50 @@ struct SerializabilityWorkload : TestWorkload {
 
 						ASSERT(false);
 					}
-				}
 
-				for (int i = 0; i < getFutures[0].size(); i++) {
-					ASSERT(getFutures[0][i].get() == getFutures[3][i].get());
-				}
-				for (int i = 0; i < getFutures[1].size(); i++) {
-					ASSERT(getFutures[1][i].get() == getFutures[3][getFutures[0].size() + i].get());
-				}
-				for (int i = 0; i < getFutures[2].size(); i++) {
-					ASSERT(getFutures[2][i].get() == getFutures[4][i].get());
-				}
+					for (int i = 0; i < result1.size(); i++) {
+						if (result1[i] != result2[i]) {
+							TraceEvent(SevError, "SRL_ResultMismatch")
+							    .detail("I", i)
+							    .detail("Result1", printable(result1[i]))
+							    .detail("Result2", printable(result2[i]))
+							    .detail("Result1Value", printable(result1[i].value))
+							    .detail("Result2Value", printable(result2[i].value));
 
-				for (int i = 0; i < getKeyFutures[0].size(); i++) {
-					ASSERT(getKeyFutures[0][i].get() == getKeyFutures[3][i].get());
-				}
-				for (int i = 0; i < getKeyFutures[1].size(); i++) {
-					ASSERT(getKeyFutures[1][i].get() == getKeyFutures[3][getKeyFutures[0].size() + i].get());
-				}
-				for (int i = 0; i < getKeyFutures[2].size(); i++) {
-					ASSERT(getKeyFutures[2][i].get() == getKeyFutures[4][i].get());
-				}
+							for (auto kv : result1)
+								TraceEvent("SRL_Result1").detail("Kv", printable(kv));
+							for (auto kv : result2)
+								TraceEvent("SRL_Result2").detail("Kv", printable(kv));
 
-				for (int i = 0; i < getRangeFutures[0].size(); i++) {
-					if (getRangeFutures[0][i].get().size() != getRangeFutures[3][i].get().size()) {
-						TraceEvent(SevError, "SRL_ResultMismatch")
-						    .detail("Size1", getRangeFutures[0][i].get().size())
-						    .detail("Size2", getRangeFutures[3][i].get().size());
+							ASSERT(false);
+						}
+					}
 
-						for (auto kv : getRangeFutures[0][i].get())
-							TraceEvent("SRL_Result1").detail("Kv", printable(kv));
-						for (auto kv : getRangeFutures[3][i].get())
-							TraceEvent("SRL_Result2").detail("Kv", printable(kv));
+					for (int i = 0; i < getFutures[0].size(); i++) {
+						ASSERT(getFutures[0][i].get() == getFutures[3][i].get());
+					}
+					for (int i = 0; i < getFutures[1].size(); i++) {
+						ASSERT(getFutures[1][i].get() == getFutures[3][getFutures[0].size() + i].get());
+					}
+					for (int i = 0; i < getFutures[2].size(); i++) {
+						ASSERT(getFutures[2][i].get() == getFutures[4][i].get());
+					}
 
-						ASSERT(false);
+					for (int i = 0; i < getKeyFutures[0].size(); i++) {
+						ASSERT(getKeyFutures[0][i].get() == getKeyFutures[3][i].get());
+					}
+					for (int i = 0; i < getKeyFutures[1].size(); i++) {
+						ASSERT(getKeyFutures[1][i].get() == getKeyFutures[3][getKeyFutures[0].size() + i].get());
+					}
+					for (int i = 0; i < getKeyFutures[2].size(); i++) {
+						ASSERT(getKeyFutures[2][i].get() == getKeyFutures[4][i].get());
 					}
 
-					for (int j = 0; j < getRangeFutures[0][i].get().size(); j++) {
-						if (getRangeFutures[0][i].get()[j] != getRangeFutures[3][i].get()[j]) {
+					for (int i = 0; i < getRangeFutures[0].size(); i++) {
+						if (getRangeFutures[0][i].get().size() != getRangeFutures[3][i].get().size()) {
 							TraceEvent(SevError, "SRL_ResultMismatch")
-							    .detail("J", j)
-							    .detail("Result1", printable(getRangeFutures[0][i].get()[j]))
-							    .detail("Result2", printable(getRangeFutures[3][i].get()[j]))
-							    .detail("Result1Value", printable(getRangeFutures[0][i].get()[j].value))
-							    .detail("Result2Value", printable(getRangeFutures[3][i].get()[j].value));
+							    .detail("Size1", getRangeFutures[0][i].get().size())
+							    .detail("Size2", getRangeFutures[3][i].get().size());
 
 							for (auto kv : getRangeFutures[0][i].get())
 								TraceEvent("SRL_Result1").detail("Kv", printable(kv));
@@ -480,37 +460,37 @@ struct SerializabilityWorkload : TestWorkload {
 
 							ASSERT(false);
 						}
-					}
-
-					ASSERT(getRangeFutures[0][i].get() == getRangeFutures[3][i].get());
-				}
-
-				for (int i = 0; i < getRangeFutures[1].size(); i++) {
-					ASSERT(getRangeFutures[1][i].get() == getRangeFutures[3][getRangeFutures[0].size() + i].get());
-				}
 
-				for (int i = 0; i < getRangeFutures[2].size(); i++) {
-					if (getRangeFutures[2][i].get().size() != getRangeFutures[4][i].get().size()) {
-						TraceEvent(SevError, "SRL_ResultMismatch")
-						    .detail("Size1", getRangeFutures[2][i].get().size())
-						    .detail("Size2", getRangeFutures[4][i].get().size());
+						for (int j = 0; j < getRangeFutures[0][i].get().size(); j++) {
+							if (getRangeFutures[0][i].get()[j] != getRangeFutures[3][i].get()[j]) {
+								TraceEvent(SevError, "SRL_ResultMismatch")
+								    .detail("J", j)
+								    .detail("Result1", printable(getRangeFutures[0][i].get()[j]))
+								    .detail("Result2", printable(getRangeFutures[3][i].get()[j]))
+								    .detail("Result1Value", printable(getRangeFutures[0][i].get()[j].value))
+								    .detail("Result2Value", printable(getRangeFutures[3][i].get()[j].value));
+
+								for (auto kv : getRangeFutures[0][i].get())
+									TraceEvent("SRL_Result1").detail("Kv", printable(kv));
+								for (auto kv : getRangeFutures[3][i].get())
+									TraceEvent("SRL_Result2").detail("Kv", printable(kv));
+
+								ASSERT(false);
+							}
+						}
 
-						for (auto kv : getRangeFutures[2][i].get())
-							TraceEvent("SRL_Result1").detail("Kv", printable(kv));
-						for (auto kv : getRangeFutures[4][i].get())
-							TraceEvent("SRL_Result2").detail("Kv", printable(kv));
+						ASSERT(getRangeFutures[0][i].get() == getRangeFutures[3][i].get());
+					}
 
-						ASSERT(false);
+					for (int i = 0; i < getRangeFutures[1].size(); i++) {
+						ASSERT(getRangeFutures[1][i].get() == getRangeFutures[3][getRangeFutures[0].size() + i].get());
 					}
 
-					for (int j = 0; j < getRangeFutures[2][i].get().size(); j++) {
-						if (getRangeFutures[2][i].get()[j] != getRangeFutures[4][i].get()[j]) {
+					for (int i = 0; i < getRangeFutures[2].size(); i++) {
+						if (getRangeFutures[2][i].get().size() != getRangeFutures[4][i].get().size()) {
 							TraceEvent(SevError, "SRL_ResultMismatch")
-							    .detail("J", j)
-							    .detail("Result1", printable(getRangeFutures[2][i].get()[j]))
-							    .detail("Result2", printable(getRangeFutures[4][i].get()[j]))
-							    .detail("Result1Value", printable(getRangeFutures[2][i].get()[j].value))
-							    .detail("Result2Value", printable(getRangeFutures[4][i].get()[j].value));
+							    .detail("Size1", getRangeFutures[2][i].get().size())
+							    .detail("Size2", getRangeFutures[4][i].get().size());
 
 							for (auto kv : getRangeFutures[2][i].get())
 								TraceEvent("SRL_Result1").detail("Kv", printable(kv));
@@ -519,13 +499,34 @@ struct SerializabilityWorkload : TestWorkload {
 
 							ASSERT(false);
 						}
-					}
 
-					ASSERT(getRangeFutures[2][i].get() == getRangeFutures[4][i].get());
+						for (int j = 0; j < getRangeFutures[2][i].get().size(); j++) {
+							if (getRangeFutures[2][i].get()[j] != getRangeFutures[4][i].get()[j]) {
+								TraceEvent(SevError, "SRL_ResultMismatch")
+								    .detail("J", j)
+								    .detail("Result1", printable(getRangeFutures[2][i].get()[j]))
+								    .detail("Result2", printable(getRangeFutures[4][i].get()[j]))
+								    .detail("Result1Value", printable(getRangeFutures[2][i].get()[j].value))
+								    .detail("Result2Value", printable(getRangeFutures[4][i].get()[j].value));
+
+								for (auto kv : getRangeFutures[2][i].get())
+									TraceEvent("SRL_Result1").detail("Kv", printable(kv));
+								for (auto kv : getRangeFutures[4][i].get())
+									TraceEvent("SRL_Result2").detail("Kv", printable(kv));
+
+								ASSERT(false);
+							}
+						}
+
+						ASSERT(getRangeFutures[2][i].get() == getRangeFutures[4][i].get());
+					}
+				} catch (Error& e) {
+					err = e;
+				}
+				if (err.isValid()) {
+					ReadYourWritesTransaction trErr(cx);
+					co_await trErr.onError(err);
 				}
-			} catch (Error& e) {
-				state ReadYourWritesTransaction trErr(cx);
-				wait(trErr.onError(e));
 			}
 		}
 	}
diff --git a/fdbserver/workloads/Sideband.actor.cpp b/fdbserver/workloads/Sideband.cpp
similarity index 70%
rename from fdbserver/workloads/Sideband.actor.cpp
rename to fdbserver/workloads/Sideband.cpp
index 01064e3ab56..73bb47f826d 100644
--- a/fdbserver/workloads/Sideband.actor.cpp
+++ b/fdbserver/workloads/Sideband.cpp
@@ -1,5 +1,5 @@
 /*
- * Sideband.actor.cpp
+ * Sideband.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,16 +19,15 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 
 struct SidebandMessage {
 	constexpr static FileIdentifier file_identifier = 11862046;
 	uint64_t key;
 	Version commitVersion;
 
-	SidebandMessage() {}
+	SidebandMessage() = default;
 	SidebandMessage(uint64_t key, Version commitVersion) : key(key), commitVersion(commitVersion) {}
 
 	template <class Ar>
@@ -58,7 +57,7 @@ struct SidebandWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter messages, consistencyErrors, keysUnexpectedlyPresent;
 
-	SidebandWorkload(WorkloadContext const& wcx)
+	explicit SidebandWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), messages("Messages"), consistencyErrors("Causal Consistency Errors"),
 	    keysUnexpectedlyPresent("KeysUnexpectedlyPresent") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -89,36 +88,38 @@ struct SidebandWorkload : TestWorkload {
 		m.push_back(keysUnexpectedlyPresent.getMetric());
 	}
 
-	ACTOR Future<Void> persistInterface(SidebandWorkload* self, Database cx) {
-		state Transaction tr(cx);
+	Future<Void> persistInterface(SidebandWorkload* self, Database cx) {
+		Transaction tr(cx);
 		BinaryWriter wr(IncludeVersion());
 		wr << self->interf;
-		state Standalone<StringRef> serializedInterface = wr.toValue();
-		loop {
+		Standalone<StringRef> serializedInterface = wr.toValue();
+		while (true) {
+			Error err;
 			try {
-				Optional<Value> val = wait(tr.get(StringRef(format("Sideband/Client/%d", self->clientId))));
+				Optional<Value> val = co_await tr.get(StringRef(format("Sideband/Client/%d", self->clientId)));
 				if (val.present()) {
 					if (val.get() != serializedInterface)
 						throw operation_failed();
 					break;
 				}
 				tr.set(format("Sideband/Client/%d", self->clientId), serializedInterface);
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 		TraceEvent("SidebandPersisted", self->interf.id()).detail("ClientIdx", self->clientId);
-		return Void();
 	}
 
-	ACTOR Future<SidebandInterface> fetchSideband(SidebandWorkload* self, Database cx) {
-		state Transaction tr(cx);
-		loop {
+	Future<SidebandInterface> fetchSideband(SidebandWorkload* self, Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				Optional<Value> val =
-				    wait(tr.get(StringRef(format("Sideband/Client/%d", (self->clientId + 1) % self->clientCount))));
+				    co_await tr.get(StringRef(format("Sideband/Client/%d", (self->clientId + 1) % self->clientCount)));
 				if (!val.present()) {
 					throw operation_failed();
 				}
@@ -126,53 +127,57 @@ struct SidebandWorkload : TestWorkload {
 				BinaryReader br(val.get(), IncludeVersion());
 				br >> sideband;
 				TraceEvent("SidebandFetched", sideband.id()).detail("ClientIdx", self->clientId);
-				return sideband;
+				co_return sideband;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> mutator(SidebandWorkload* self, Database cx) {
-		state SidebandInterface checker = wait(self->fetchSideband(self, cx->clone()));
-		state double lastTime = now();
-		state Version commitVersion;
+	Future<Void> mutator(SidebandWorkload* self, Database cx) {
+		SidebandInterface checker = co_await self->fetchSideband(self, cx->clone());
+		double lastTime = now();
+		Version commitVersion{ 0 };
 
-		loop {
-			wait(poisson(&lastTime, 1.0 / self->operationsPerSecond));
-			state Transaction tr(cx);
-			state uint64_t key = deterministicRandom()->randomUniqueID().hash();
+		while (true) {
+			co_await poisson(&lastTime, 1.0 / self->operationsPerSecond);
+			Transaction tr(cx);
+			uint64_t key = deterministicRandom()->randomUniqueID().hash();
 
-			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", key));
-			loop {
+			Standalone<StringRef> messageKey(format("Sideband/Message/%llx", key));
+			while (true) {
+				Error err;
 				try {
-					Optional<Value> val = wait(tr.get(messageKey));
+					Optional<Value> val = co_await tr.get(messageKey);
 					if (val.present()) {
 						commitVersion = tr.getReadVersion().get();
 						++self->keysUnexpectedlyPresent;
 						break;
 					}
 					tr.set(messageKey, "deadbeef"_sr);
-					wait(tr.commit());
+					co_await tr.commit();
 					commitVersion = tr.getCommittedVersion();
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 			++self->messages;
 			checker.updates.send(SidebandMessage(key, commitVersion));
 		}
 	}
 
-	ACTOR Future<Void> checker(SidebandWorkload* self, Database cx) {
-		loop {
-			state SidebandMessage message = waitNext(self->interf.updates.getFuture());
-			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.key));
-			state Transaction tr(cx);
-			loop {
+	Future<Void> checker(SidebandWorkload* self, Database cx) {
+		while (true) {
+			SidebandMessage message = co_await self->interf.updates.getFuture();
+			Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.key));
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					Optional<Value> val = wait(tr.get(messageKey));
+					Optional<Value> val = co_await tr.get(messageKey);
 					if (!val.present()) {
 						TraceEvent(SevError, "CausalConsistencyError", self->interf.id())
 						    .detail("MessageKey", messageKey.toString().c_str())
@@ -183,8 +188,9 @@ struct SidebandWorkload : TestWorkload {
 					}
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 	}
diff --git a/fdbserver/workloads/SidebandSingle.actor.cpp b/fdbserver/workloads/SidebandSingle.cpp
similarity index 75%
rename from fdbserver/workloads/SidebandSingle.actor.cpp
rename to fdbserver/workloads/SidebandSingle.cpp
index 894a93ab8bc..407821aea96 100644
--- a/fdbserver/workloads/SidebandSingle.actor.cpp
+++ b/fdbserver/workloads/SidebandSingle.cpp
@@ -1,5 +1,5 @@
 /*
- * SidebandSingle.actor.cpp
+ * SidebandSingle.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,8 +20,8 @@
 
 #include "fdbclient/Knobs.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 /*
@@ -43,7 +43,7 @@ struct SidebandSingleWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter messages, consistencyErrors, keysUnexpectedlyPresent;
 
-	SidebandSingleWorkload(WorkloadContext const& wcx)
+	explicit SidebandSingleWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), messages("Messages"), consistencyErrors("Causal Consistency Errors"),
 	    keysUnexpectedlyPresent("KeysUnexpectedlyPresent") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -80,44 +80,48 @@ struct SidebandSingleWorkload : TestWorkload {
 		m.push_back(keysUnexpectedlyPresent.getMetric());
 	}
 
-	ACTOR Future<Void> mutator(SidebandSingleWorkload* self, Database cx) {
-		state double lastTime = now();
-		state Version commitVersion;
-		state bool unknown = false;
+	Future<Void> mutator(SidebandSingleWorkload* self, Database cx) {
+		double lastTime = now();
+		Version commitVersion{ 0 };
+		bool unknown = false;
 
-		loop {
-			wait(poisson(&lastTime, 1.0 / self->operationsPerSecond));
-			state Transaction tr0(cx);
-			state Transaction tr(cx);
-			state uint64_t key = deterministicRandom()->randomUniqueID().hash();
+		while (true) {
+			co_await poisson(&lastTime, 1.0 / self->operationsPerSecond);
+			Transaction tr0(cx);
+			Transaction tr(cx);
+			uint64_t key = deterministicRandom()->randomUniqueID().hash();
 
-			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", key));
+			Standalone<StringRef> messageKey(format("Sideband/Message/%llx", key));
 			// first set, this is the "old" value, always retry
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr0.set(messageKey, "oldbeef"_sr);
-					wait(tr0.commit());
+					co_await tr0.commit();
 					break;
 				} catch (Error& e) {
-					wait(tr0.onError(e));
+					err = e;
 				}
+				co_await tr0.onError(err);
 			}
 			// second set, the checker should see this, no retries on unknown result
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tr.set(messageKey, "deadbeef"_sr);
-					wait(tr.commit());
+					co_await tr.commit();
 					commitVersion = tr.getCommittedVersion();
 					break;
 				} catch (Error& e) {
-					if (e.code() == error_code_commit_unknown_result) {
-						unknown = true;
-						++self->messages;
-						self->interf.send(std::pair(key, invalidVersion));
-						break;
-					}
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.code() == error_code_commit_unknown_result) {
+					unknown = true;
+					++self->messages;
+					self->interf.send(std::pair(key, invalidVersion));
+					break;
 				}
+				co_await tr.onError(err);
 			}
 			if (unknown) {
 				unknown = false;
@@ -128,21 +132,22 @@ struct SidebandSingleWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<Void> checker(SidebandSingleWorkload* self, Database cx) {
+	Future<Void> checker(SidebandSingleWorkload* self, Database cx) {
 		// Required for GRV Cache to work in simulation.
 		// Normally, MVC would set the shared state and it is verified upon setting the
 		// transaction option for GRV Cache.
 		// In simulation, explicitly initialize it when used for tests.
 		cx->initSharedState();
-		loop {
+		while (true) {
 			// Pair represents <Key, commitVersion>
-			state std::pair<uint64_t, Version> message = waitNext(self->interf.getFuture());
-			state Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.first));
-			state Transaction tr(cx);
-			loop {
+			std::pair<uint64_t, Version> message = co_await self->interf.getFuture();
+			Standalone<StringRef> messageKey(format("Sideband/Message/%llx", message.first));
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::USE_GRV_CACHE);
-					state Optional<Value> val = wait(tr.get(messageKey));
+					Optional<Value> val = co_await tr.get(messageKey);
 					if (!val.present()) {
 						TraceEvent(SevError, "CausalConsistencyError1")
 						    .detail("MessageKey", messageKey.toString().c_str())
@@ -160,16 +165,18 @@ struct SidebandSingleWorkload : TestWorkload {
 							break;
 						}
 						// check again without cache, and if it's the same, that's expected
-						state Transaction tr2(cx);
-						state Optional<Value> val2;
-						loop {
+						Transaction tr2(cx);
+						Optional<Value> val2;
+						while (true) {
+							Error err;
 							try {
-								wait(store(val2, tr2.get(messageKey)));
+								val2 = co_await tr2.get(messageKey);
 								break;
 							} catch (Error& e) {
-								TraceEvent("DebugSidebandNoCacheError").errorUnsuppressed(e);
-								wait(tr2.onError(e));
+								err = e;
 							}
+							TraceEvent("DebugSidebandNoCacheError").errorUnsuppressed(err);
+							co_await tr2.onError(err);
 						}
 						if (val != val2) {
 							TraceEvent(SevError, "CausalConsistencyError3")
@@ -184,9 +191,10 @@ struct SidebandSingleWorkload : TestWorkload {
 					}
 					break;
 				} catch (Error& e) {
-					TraceEvent("DebugSidebandCheckError").errorUnsuppressed(e);
-					wait(tr.onError(e));
+					err = e;
 				}
+				TraceEvent("DebugSidebandCheckError").errorUnsuppressed(err);
+				co_await tr.onError(err);
 			}
 		}
 	}
diff --git a/fdbserver/workloads/SimpleAtomicAdd.actor.cpp b/fdbserver/workloads/SimpleAtomicAdd.cpp
similarity index 53%
rename from fdbserver/workloads/SimpleAtomicAdd.actor.cpp
rename to fdbserver/workloads/SimpleAtomicAdd.cpp
index b3c64f73049..b590bceed5e 100644
--- a/fdbserver/workloads/SimpleAtomicAdd.actor.cpp
+++ b/fdbserver/workloads/SimpleAtomicAdd.cpp
@@ -1,5 +1,5 @@
 /*
- * SimpleAtomicAdd.actor.cpp
+ * SimpleAtomicAdd.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,9 @@
 
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct SimpleAtomicAddWorkload : TestWorkload {
 	static constexpr auto NAME = "SimpleAtomicAdd";
@@ -36,7 +35,7 @@ struct SimpleAtomicAddWorkload : TestWorkload {
 	double testDuration;
 	std::vector<Future<Void>> clients;
 
-	SimpleAtomicAddWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit SimpleAtomicAddWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		addValue = getOption(options, "addValue"_sr, 1);
 		iterations = getOption(options, "iterations"_sr, 100);
@@ -51,71 +50,73 @@ struct SimpleAtomicAddWorkload : TestWorkload {
 		if (clientId) {
 			return Void();
 		}
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override {
 		if (clientId) {
 			return true;
 		}
-		return _check(cx, this);
+		return _check(cx);
 	}
 
-	ACTOR static Future<Void> _start(Database cx, SimpleAtomicAddWorkload* self) {
-		if (self->initialize) {
-			wait(setInitialValue(cx, self));
+	Future<Void> _start(Database cx) {
+		if (initialize) {
+			co_await setInitialValue(cx);
 		}
-		for (int i = 0; i < self->iterations; ++i) {
-			self->clients.push_back(timeout(applyAtomicAdd(cx, self), self->testDuration, Void()));
+		for (int i = 0; i < iterations; ++i) {
+			clients.push_back(timeout(applyAtomicAdd(cx), testDuration, Void()));
 		}
-		waitForAll(self->clients);
-		return Void();
+		waitForAll(clients);
 	}
 
-	ACTOR static Future<Void> setInitialValue(Database cx, SimpleAtomicAddWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
-		state Value val = StringRef((const uint8_t*)&self->initialValue, sizeof(self->initialValue));
-		loop {
+	Future<Void> setInitialValue(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		Value val = StringRef((const uint8_t*)&initialValue, sizeof(initialValue));
+		while (true) {
+			Error err;
 			try {
-				TraceEvent("SAASetInitialValue").detail("Key", self->sumKey).detail("Value", val);
-				tr.set(self->sumKey, val);
-				wait(tr.commit());
+				TraceEvent("SAASetInitialValue").detail("Key", sumKey).detail("Value", val);
+				tr.set(sumKey, val);
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				TraceEvent("SAASetInitialValueError").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("SAASetInitialValueError").error(err);
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<Void> applyAtomicAdd(Database cx, SimpleAtomicAddWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
-		state Value val = StringRef((const uint8_t*)&self->addValue, sizeof(self->addValue));
-		loop {
+	Future<Void> applyAtomicAdd(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		Value val = StringRef((const uint8_t*)&addValue, sizeof(addValue));
+		while (true) {
+			Error err;
 			try {
-				TraceEvent("SAABegin").detail("Key", self->sumKey).detail("Value", val);
-				tr.atomicOp(self->sumKey, val, MutationRef::AddValue);
-				wait(tr.commit());
+				TraceEvent("SAABegin").detail("Key", sumKey).detail("Value", val);
+				tr.atomicOp(sumKey, val, MutationRef::AddValue);
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				TraceEvent("SAABeginError").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("SAABeginError").error(err);
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR static Future<bool> _check(Database cx, SimpleAtomicAddWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
-		state uint64_t expectedValue = self->addValue * self->iterations;
-		if (self->initialize) {
-			expectedValue += self->initialValue;
+	Future<bool> _check(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		uint64_t expectedValue = addValue * iterations;
+		if (initialize) {
+			expectedValue += initialValue;
 		}
-		loop {
+		while (true) {
+			Error err;
 			try {
 				TraceEvent("SAACheckKey").log();
-				Optional<Value> actualValue = wait(tr.get(self->sumKey));
+				Optional<Value> actualValue = co_await tr.get(sumKey);
 				uint64_t actualValueInt = 0;
 				if (actualValue.present()) {
 					memcpy(&actualValueInt, actualValue.get().begin(), actualValue.get().size());
@@ -123,11 +124,12 @@ struct SimpleAtomicAddWorkload : TestWorkload {
 				TraceEvent("SAACheckEqual")
 				    .detail("ExpectedValue", expectedValue)
 				    .detail("ActualValue", actualValueInt);
-				return (expectedValue == actualValueInt);
+				co_return (expectedValue == actualValueInt);
 			} catch (Error& e) {
-				TraceEvent("SAACheckError").error(e);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("SAACheckError").error(err);
+			co_await tr.onError(err);
 		}
 	}
 
diff --git a/fdbserver/workloads/SkewedReadWrite.actor.cpp b/fdbserver/workloads/SkewedReadWrite.cpp
similarity index 72%
rename from fdbserver/workloads/SkewedReadWrite.actor.cpp
rename to fdbserver/workloads/SkewedReadWrite.cpp
index 1aa1cf0922e..b2498ce7e23 100644
--- a/fdbserver/workloads/SkewedReadWrite.actor.cpp
+++ b/fdbserver/workloads/SkewedReadWrite.cpp
@@ -1,5 +1,5 @@
 /*
- * ReadWrite.actor.cpp
+ * SkewedReadWrite.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,15 +24,14 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbserver/workloads/ReadWriteWorkload.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
+#include "ReadWriteWorkload.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "flow/TDMetric.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TDMetric.h"
+#include "fdbclient/RunRYWTransaction.h"
 
 struct SkewedReadWriteWorkload : ReadWriteCommon {
 	static constexpr auto NAME = "SkewedReadWrite";
@@ -43,13 +42,13 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 	double hotReadWriteServerOverlap; // the portion of intersection of write and hot server
 
 	// hot server state
-	typedef std::vector<std::pair<int64_t, int64_t>> IndexRangeVec;
+	using IndexRangeVec = std::vector<std::pair<int64_t, int64_t>>;
 	// keyForIndex generate key from index. So for a shard range, recording the start and end is enough
 	std::vector<std::pair<UID, IndexRangeVec>> serverShards; // storage server and the shards it owns
 	std::map<UID, StorageServerInterface> serverInterfaces;
 	int hotServerCount = 0, currentHotRound = -1;
 
-	SkewedReadWriteWorkload(WorkloadContext const& wcx) : ReadWriteCommon(wcx) {
+	explicit SkewedReadWriteWorkload(WorkloadContext const& wcx) : ReadWriteCommon(wcx) {
 		descriptionString = getOption(options, "description"_sr, "SkewedReadWrite"_sr);
 		hotServerFraction = getOption(options, "hotServerFraction"_sr, 0.2);
 		hotServerShardFraction = getOption(options, "hotServerShardFraction"_sr, 1.0);
@@ -60,11 +59,9 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 		ASSERT((hotServerReadFrac >= hotServerFraction || hotServerWriteFrac >= hotServerFraction) && skewRound > 0);
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	void debugPrintServerShards() const {
 		std::cout << std::hex;
-		for (auto it : this->serverShards) {
+		for (const auto& it : this->serverShards) {
 			std::cout << serverInterfaces.at(it.first).address().toString() << ": [";
 			for (auto p : it.second) {
 				std::cout << "[" << p.first << "," << p.second << "], ";
@@ -75,61 +72,60 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 
 	// for each boundary except the last one in boundaries, found the first existed key generated from keyForIndex as
 	// beginIdx, found the last existed key generated from keyForIndex the endIdx.
-	ACTOR static Future<IndexRangeVec> convertKeyBoundaryToIndexShard(Database cx,
-	                                                                  SkewedReadWriteWorkload* self,
-	                                                                  Standalone<VectorRef<KeyRef>> boundaries) {
-		state IndexRangeVec res;
-		state int i = 0;
-		for (; i < boundaries.size() - 1; ++i) {
+	Future<IndexRangeVec> convertKeyBoundaryToIndexShard(Database cx, Standalone<VectorRef<KeyRef>> boundaries) {
+		IndexRangeVec res;
+		for (int i = 0; i < boundaries.size() - 1; ++i) {
 			KeyRangeRef currentShard = KeyRangeRef(boundaries[i], boundaries[i + 1]);
 			// std::cout << currentShard.toString() << "\n";
-			std::vector<RangeResult> ranges = wait(runRYWTransaction(
+			std::vector<RangeResult> ranges = co_await runRYWTransaction(
 			    cx, [currentShard](Reference<ReadYourWritesTransaction> tr) -> Future<std::vector<RangeResult>> {
 				    std::vector<Future<RangeResult>> f;
 				    f.push_back(tr->getRange(currentShard, 1, Snapshot::False, Reverse::False));
 				    f.push_back(tr->getRange(currentShard, 1, Snapshot::False, Reverse::True));
 				    return getAll(f);
-			    }));
+			    });
 			ASSERT(ranges[0].size() == 1 && ranges[1].size() == 1);
-			res.emplace_back(self->indexForKey(ranges[0][0].key), self->indexForKey(ranges[1][0].key));
+			res.emplace_back(indexForKey(ranges[0][0].key), indexForKey(ranges[1][0].key));
 		}
 
 		ASSERT(res.size() == boundaries.size() - 1);
-		return res;
+		co_return res;
 	}
 
-	ACTOR static Future<Void> updateServerShards(Database cx, SkewedReadWriteWorkload* self) {
-		state RangeResult serverList;
-		state RangeResult range;
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		loop {
+	Future<Void> updateServerShards(Database cx) {
+		RangeResult serverList;
+		RangeResult range;
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		while (true) {
 			// read in transaction to ensure two key ranges are transactionally consistent
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				state Future<RangeResult> serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> rangeF = tr->getRange(serverKeysRange, CLIENT_KNOBS->TOO_MANY);
-				wait(store(serverList, serverListF));
-				wait(store(range, rangeF));
+				Future<RangeResult> serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+				Future<RangeResult> rangeF = tr->getRange(serverKeysRange, CLIENT_KNOBS->TOO_MANY);
+				serverList = co_await serverListF;
+				range = co_await rangeF;
 				break;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 		// decode server interfaces
-		self->serverInterfaces.clear();
+		serverInterfaces.clear();
 		for (int i = 0; i < serverList.size(); i++) {
 			auto ssi = decodeServerListValue(serverList[i].value);
-			self->serverInterfaces.emplace(ssi.id(), ssi);
+			serverInterfaces.emplace(ssi.id(), ssi);
 		}
 		// clear self->serverShards
-		self->serverShards.clear();
+		serverShards.clear();
 
 		// leftEdge < workloadBegin < workloadEnd
-		Key workloadBegin = self->keyForIndex(0), workloadEnd = self->keyForIndex(self->nodeCount);
+		Key workloadBegin = keyForIndex(0), workloadEnd = keyForIndex(nodeCount);
 		Key leftEdge(allKeys.begin);
 		std::vector<UID> leftServer; // left server owns the range [leftEdge, workloadBegin)
 		KeyRangeRef workloadRange(workloadBegin, workloadEnd);
-		state std::map<Key, std::vector<UID>> beginServers; // begin index to server ID
+		std::map<Key, std::vector<UID>> beginServers; // begin index to server ID
 
 		for (auto kv = range.begin(); kv != range.end(); kv++) {
 			if (serverHasKey(kv->value)) {
@@ -147,9 +143,9 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 				}
 			}
 		}
-		ASSERT(beginServers.size() == 0 || beginServers.begin()->first >= workloadBegin);
+		ASSERT(beginServers.empty() || beginServers.begin()->first >= workloadBegin);
 		// handle the left boundary
-		if (beginServers.size() == 0 || beginServers.begin()->first > workloadBegin) {
+		if (beginServers.empty() || beginServers.begin()->first > workloadBegin) {
 			beginServers[workloadBegin] = leftServer;
 		}
 		Standalone<VectorRef<KeyRef>> keyBegins;
@@ -159,7 +155,7 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 		// deep count because wait below will destruct workloadEnd
 		keyBegins.push_back_deep(keyBegins.arena(), workloadEnd);
 
-		IndexRangeVec indexShards = wait(convertKeyBoundaryToIndexShard(cx, self, keyBegins));
+		IndexRangeVec indexShards = co_await convertKeyBoundaryToIndexShard(cx, keyBegins);
 		ASSERT(beginServers.size() == indexShards.size());
 		// sort shard begin idx
 		// build self->serverShards, starting from the left shard
@@ -172,19 +168,18 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 			++i;
 		}
 		// self->serverShards is ordered by UID
-		for (auto it : serverShards) {
-			self->serverShards.emplace_back(it);
+		for (const auto& it : serverShards) {
+			this->serverShards.emplace_back(it);
 		}
 		//		if (self->clientId == 0) {
 		//			self->debugPrintServerShards();
 		//		}
-		return Void();
 	}
 
-	ACTOR template <class Trans>
+	template <class Trans>
 	Future<Void> readOp(Trans* tr, std::vector<int64_t> keys, SkewedReadWriteWorkload* self, bool shouldRecord) {
-		if (!keys.size())
-			return Void();
+		if (keys.empty())
+			co_return;
 
 		std::vector<Future<Void>> readers;
 		for (int op = 0; op < keys.size(); op++) {
@@ -192,8 +187,7 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 			readers.push_back(self->logLatency(tr->get(self->keyForIndex(keys[op])), shouldRecord));
 		}
 
-		wait(waitForAll(readers));
-		return Void();
+		co_await waitForAll(readers);
 	}
 
 	void startReadWriteClients(Database cx, std::vector<Future<Void>>& clients) {
@@ -209,22 +203,20 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 		}
 	}
 
-	ACTOR static Future<Void> _start(Database cx, SkewedReadWriteWorkload* self) {
-		state std::vector<Future<Void>> clients;
-		if (self->enableReadLatencyLogging)
-			clients.push_back(self->tracePeriodically());
+	Future<Void> start(Database const& cx) override {
+		std::vector<Future<Void>> clients;
+		if (enableReadLatencyLogging)
+			clients.push_back(tracePeriodically());
 
-		wait(updateServerShards(cx, self));
-		for (self->currentHotRound = 0; self->currentHotRound < self->skewRound; ++self->currentHotRound) {
-			self->setHotServers();
-			self->startReadWriteClients(cx, clients);
-			wait(timeout(waitForAll(clients), self->testDuration / self->skewRound, Void()));
+		co_await updateServerShards(cx);
+		for (currentHotRound = 0; currentHotRound < skewRound; ++currentHotRound) {
+			setHotServers();
+			startReadWriteClients(cx, clients);
+			co_await timeout(waitForAll(clients), testDuration / skewRound, Void());
 			clients.clear();
-			wait(delay(5.0));
-			wait(updateServerShards(cx, self));
+			co_await delay(5.0);
+			co_await updateServerShards(cx);
 		}
-
-		return Void();
 	}
 
 	// calculate hot server count
@@ -264,23 +256,23 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 		return deterministicRandom()->randomInt64(0, nodeCount);
 	}
 
-	ACTOR template <class Trans>
+	template <class Trans>
 	Future<Void> randomReadWriteClient(Database cx, SkewedReadWriteWorkload* self, double delay, int clientIndex) {
-		state double lastTime = now();
-		state double GRVStartTime;
-		state UID debugID;
+		double lastTime = now();
+		double GRVStartTime{ 0 };
+		UID debugID;
 
-		loop {
-			wait(poisson(&lastTime, delay));
+		while (true) {
+			co_await poisson(&lastTime, delay);
 
-			state double tstart = now();
-			state bool aTransaction = deterministicRandom()->random01() > self->alpha;
+			double tstart = now();
+			bool aTransaction = deterministicRandom()->random01() > self->alpha;
 
-			state std::vector<int64_t> keys;
-			state std::vector<Value> values;
-			state std::vector<KeyRange> extra_ranges;
+			std::vector<int64_t> keys;
+			std::vector<Value> values;
+			std::vector<KeyRange> extra_ranges;
 			int reads = aTransaction ? self->readsPerTransactionA : self->readsPerTransactionB;
-			state int writes = aTransaction ? self->writesPerTransactionA : self->writesPerTransactionB;
+			int writes = aTransaction ? self->writesPerTransactionA : self->writesPerTransactionB;
 			for (int op = 0; op < reads; op++)
 				keys.push_back(self->getRandomKey(self->nodeCount));
 
@@ -288,7 +280,7 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 			for (int op = 0; op < writes; op++)
 				values.push_back(self->randomValue());
 
-			state Trans tr(cx);
+			Trans tr(cx);
 
 			if (tstart - self->clientBegin > self->debugTime &&
 			    tstart - self->clientBegin <= self->debugTime + self->debugInterval) {
@@ -302,7 +294,8 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 			self->transactionSuccessMetric->retries = 0;
 			self->transactionSuccessMetric->commitLatency = -1;
 
-			loop {
+			while (true) {
+				Error err;
 				try {
 					GRVStartTime = now();
 					self->transactionFailureMetric->startLatency = -1;
@@ -313,8 +306,8 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 					if (self->shouldRecord())
 						self->GRVLatencies.addSample(grvLatency);
 
-					state double readStart = now();
-					wait(self->readOp(&tr, keys, self, self->shouldRecord()));
+					double readStart = now();
+					co_await self->readOp(&tr, keys, self, self->shouldRecord());
 
 					double readLatency = now() - readStart;
 					if (self->shouldRecord())
@@ -326,8 +319,8 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 					for (int op = 0; op < writes; op++)
 						tr.set(self->keyForIndex(self->getRandomKey(self->nodeCount, false), false), values[op]);
 
-					state double commitStart = now();
-					wait(tr.commit());
+					double commitStart = now();
+					co_await tr.commit();
 
 					double commitLatency = now() - commitStart;
 					self->transactionSuccessMetric->commitLatency = commitLatency * 1e9;
@@ -336,17 +329,18 @@ struct SkewedReadWriteWorkload : ReadWriteCommon {
 
 					break;
 				} catch (Error& e) {
-					self->transactionFailureMetric->errorCode = e.code();
-					self->transactionFailureMetric->log();
+					err = e;
+				}
+				self->transactionFailureMetric->errorCode = err.code();
+				self->transactionFailureMetric->log();
 
-					wait(tr.onError(e));
+				co_await tr.onError(err);
 
-					++self->transactionSuccessMetric->retries;
-					++self->totalRetriesMetric;
+				++self->transactionSuccessMetric->retries;
+				++self->totalRetriesMetric;
 
-					if (self->shouldRecord())
-						++self->retries;
-				}
+				if (self->shouldRecord())
+					++self->retries;
 			}
 
 			if (debugID != UID())
diff --git a/fdbserver/workloads/SlowTaskWorkload.actor.cpp b/fdbserver/workloads/SlowTaskWorkload.cpp
similarity index 89%
rename from fdbserver/workloads/SlowTaskWorkload.actor.cpp
rename to fdbserver/workloads/SlowTaskWorkload.cpp
index 00722107ce9..b46260dbd5b 100644
--- a/fdbserver/workloads/SlowTaskWorkload.actor.cpp
+++ b/fdbserver/workloads/SlowTaskWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * SlowTaskWorkload.actor.cpp
+ * SlowTaskWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,15 +21,14 @@
 #include <cinttypes>
 
 #include "fmt/format.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/SignalSafeUnwind.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 // Stress test the slow task profiler or flow profiler
 struct SlowTaskWorkload : TestWorkload {
 	static constexpr auto NAME = "SlowTaskWorkload";
 
-	SlowTaskWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+	explicit SlowTaskWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
 	Future<Void> start(Database const& cx) override {
 		setupRunLoopProfiler();
@@ -40,8 +39,8 @@ struct SlowTaskWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> go() {
-		wait(delay(1));
+	static Future<Void> go() {
+		co_await delay(1);
 		int64_t phc = dl_iterate_phdr_calls;
 		int64_t startProfilesDisabled = getNumProfilesDisabled();
 		int64_t startProfilesOverflowed = getNumProfilesOverflowed();
@@ -63,8 +62,6 @@ struct SlowTaskWorkload : TestWorkload {
 		           getNumProfilesDisabled() - startProfilesDisabled,
 		           getNumProfilesOverflowed() - startProfilesOverflowed,
 		           getNumProfilesCaptured() - startProfilesCaptured);
-
-		return Void();
 	}
 
 	static void do_slow_exception_thing(int64_t* exc_count) {
diff --git a/fdbserver/workloads/SnapTest.actor.cpp b/fdbserver/workloads/SnapTest.cpp
similarity index 69%
rename from fdbserver/workloads/SnapTest.actor.cpp
rename to fdbserver/workloads/SnapTest.cpp
index 9311d5633b3..a21e0bbc8a8 100644
--- a/fdbserver/workloads/SnapTest.actor.cpp
+++ b/fdbserver/workloads/SnapTest.cpp
@@ -1,5 +1,5 @@
 /*
- * SnapTest.actor.cpp
+ * SnapTest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,18 +19,17 @@
  */
 
 #include <boost/lexical_cast.hpp>
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/SimpleIni.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/Status.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/FDBSimulationPolicy.h"
+#include "BulkSetup.h"
+#include "fdbserver/tester/workloads.h"
 
 struct SnapTestWorkload : TestWorkload {
 	static constexpr auto NAME = "SnapTest";
@@ -49,7 +48,7 @@ struct SnapTestWorkload : TestWorkload {
 	bool attemptDuplicateSnapshot = false;
 
 public: // ctor & dtor
-	SnapTestWorkload(WorkloadContext const& wcx)
+	explicit SnapTestWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), numSnaps(0), maxSnapDelay(0.0), testID(0), snapUID() {
 		TraceEvent("SnapTestWorkloadConstructor").log();
 		std::string workloadName = "SnapTest";
@@ -61,7 +60,7 @@ struct SnapTestWorkload : TestWorkload {
 		restartInfoLocation = getOption(options, "restartInfoLocation"_sr, "simfdb/restartInfo.ini"_sr).toString();
 		// default behavior is to retry until success
 		retryLimit = getOption(options, "retryLimit"_sr, -1);
-		g_simulator->allowLogSetKills = false;
+		fdbSimulationPolicyState().allowLogSetKills = false;
 		{
 			double duplicateSnapshotProbability = getOption(options, "duplicateSnapshotProbability"_sr, 0.1);
 			if (deterministicRandom()->random01() < duplicateSnapshotProbability) {
@@ -78,7 +77,7 @@ struct SnapTestWorkload : TestWorkload {
 	Future<Void> start(Database const& cx) override {
 		TraceEvent("SnapTestWorkloadStart").log();
 		if (clientId == 0) {
-			return _start(cx, this);
+			return _start(cx);
 		}
 		return Void();
 	}
@@ -109,9 +108,9 @@ struct SnapTestWorkload : TestWorkload {
 		out.insert("Attrition");
 	}
 
-	ACTOR Future<Void> _create_keys(Database cx, std::string prefix, bool even = true) {
-		state Transaction tr(cx);
-		state std::vector<int64_t> keys;
+	Future<Void> _create_keys(Database cx, std::string prefix, bool even = true) {
+		Transaction tr(cx);
+		std::vector<int64_t> keys;
 
 		keys.reserve(1000);
 		for (int i = 0; i < 1000; i++) {
@@ -119,7 +118,8 @@ struct SnapTestWorkload : TestWorkload {
 		}
 
 		tr.reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				for (auto id : keys) {
 					if (even) {
@@ -137,103 +137,106 @@ struct SnapTestWorkload : TestWorkload {
 					Value val1Ref(Val1);
 					tr.set(key1Ref, val1Ref, AddConflictRange::False);
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(Database cx, SnapTestWorkload* self) {
-		state Transaction tr(cx);
-		state bool snapFailed = false;
-		state Future<Void> duplicateSnapStatus;
+	Future<Void> _start(Database cx) {
+		Transaction tr(cx);
+		bool snapFailed = false;
+		Future<Void> duplicateSnapStatus;
 
-		if (self->testID == 0) {
+		if (testID == 0) {
 			// create even keys before the snapshot
-			wait(self->_create_keys(cx, "snapKey"));
-		} else if (self->testID == 1) {
+			co_await _create_keys(cx, "snapKey");
+		} else if (testID == 1) {
 			// create a snapshot
-			state double toDelay = fmod(deterministicRandom()->randomUInt32(), self->maxSnapDelay);
+			double toDelay = fmod(deterministicRandom()->randomUInt32(), maxSnapDelay);
 			TraceEvent("ToDelay").detail("Value", toDelay);
-			ASSERT(toDelay < self->maxSnapDelay);
-			wait(delay(toDelay));
+			ASSERT(toDelay < maxSnapDelay);
+			co_await delay(toDelay);
 
-			state int retry = 0;
-			loop {
-				self->snapUID = deterministicRandom()->randomUniqueID();
+			int retry = 0;
+			while (true) {
+				snapUID = deterministicRandom()->randomUniqueID();
+				Error err;
 				try {
-					state StringRef snapCmdRef = "/bin/snap_create.sh"_sr;
+					StringRef snapCmdRef = "/bin/snap_create.sh"_sr;
 
-					state Future<Void> status = snapCreate(cx, snapCmdRef, self->snapUID);
-					if (self->attemptDuplicateSnapshot) {
-						wait(delay(deterministicRandom()->random01()));
-						duplicateSnapStatus = snapCreate(cx, snapCmdRef, self->snapUID);
+					Future<Void> status = snapCreate(cx, snapCmdRef, snapUID);
+					if (attemptDuplicateSnapshot) {
+						co_await delay(deterministicRandom()->random01());
+						duplicateSnapStatus = snapCreate(cx, snapCmdRef, snapUID);
 					}
-					ErrorOr<Void> statusErr = wait(errorOr(status));
+					ErrorOr<Void> statusErr = co_await errorOr(status);
 					if (statusErr.isError() && statusErr.getError().code() != error_code_duplicate_snapshot_request) {
 						// First request is expected to fail with duplicate_snapshot_request error
 						// Any other errors should be thrown
 						throw statusErr.getError();
 					}
-					if (self->attemptDuplicateSnapshot) {
+					if (attemptDuplicateSnapshot) {
 						// If duplicate, the first request is discarded, wait for the latest one
-						wait(duplicateSnapStatus);
+						co_await duplicateSnapStatus;
 					}
 					break;
 				} catch (Error& e) {
-					TraceEvent("SnapTestCreateError")
-					    .error(e)
-					    .detail("SnapUID", self->snapUID)
-					    .detail("Duplicate", self->attemptDuplicateSnapshot);
-					++retry;
-					// snap v2 can fail for many reasons, so retry until specified times and then fail it
-					if (self->retryLimit != -1 && retry > self->retryLimit) {
-						snapFailed = true;
-						break;
-					}
-					// increase the retry wait time to avoid endless retry where DD is always disabled by snapshot
-					wait(delay(retry * SERVER_KNOBS->SNAP_MINIMUM_TIME_GAP));
+					err = e;
 				}
+				TraceEvent("SnapTestCreateError")
+				    .error(err)
+				    .detail("SnapUID", snapUID)
+				    .detail("Duplicate", attemptDuplicateSnapshot);
+				++retry;
+				// snap v2 can fail for many reasons, so retry until specified times and then fail it
+				if (retryLimit != -1 && retry > retryLimit) {
+					snapFailed = true;
+					break;
+				}
+				// increase the retry wait time to avoid endless retry where DD is always disabled by snapshot
+				co_await delay(retry * SERVER_KNOBS->SNAP_MINIMUM_TIME_GAP);
 			}
 			CSimpleIni ini;
 			ini.SetUnicode();
-			ini.LoadFile(self->restartInfoLocation.c_str());
-			std::string uidStr = self->snapUID.toString();
+			ini.LoadFile(restartInfoLocation.c_str());
+			std::string uidStr = snapUID.toString();
 			ini.SetValue("RESTORE", "RestoreSnapUID", uidStr.c_str());
 			ini.SetValue("RESTORE", "BackupFailed", format("%d", snapFailed).c_str());
-			ini.SaveFile(self->restartInfoLocation.c_str());
+			ini.SaveFile(restartInfoLocation.c_str());
 			// write the snapUID to a file
 			auto const severity = snapFailed ? SevError : SevInfo;
 			TraceEvent(severity, "SnapshotCreateStatus").detail("Status", !snapFailed ? "Success" : "Failure");
-			self->snapSucceeded = !snapFailed;
-		} else if (self->testID == 2) {
+			snapSucceeded = !snapFailed;
+		} else if (testID == 2) {
 			// create odd keys after the snapshot
-			wait(self->_create_keys(cx, "snapKey", false /*even*/));
-		} else if (self->testID == 3) {
+			co_await _create_keys(cx, "snapKey", false /*even*/);
+		} else if (testID == 3) {
 			CSimpleIni ini;
 			ini.SetUnicode();
-			ini.LoadFile(self->restartInfoLocation.c_str());
+			ini.LoadFile(restartInfoLocation.c_str());
 			bool backupFailed = atoi(ini.GetValue("RESTORE", "BackupFailed"));
 			if (backupFailed) {
 				// since backup failed, skip the restore checking
 				TraceEvent(SevWarnAlways, "BackupFailedSkippingRestoreCheck").log();
-				return Void();
+				co_return;
 			}
-			state KeySelector begin = firstGreaterOrEqual(normalKeys.begin);
-			state KeySelector end = firstGreaterOrEqual(normalKeys.end);
-			state int cnt = 0;
+			KeySelector begin = firstGreaterOrEqual(normalKeys.begin);
+			KeySelector end = firstGreaterOrEqual(normalKeys.end);
+			int cnt = 0;
 			// read the entire normalKeys range and look at keys prefixed
 			// with snapKeys 1) validate that all key ids are even ie -
 			// created before snap 2) values are same as the key id 3) # of
 			// keys adds up to the total keys created before snap
 			tr.reset();
-			loop {
+			while (true) {
+				Error err;
 				try {
-					RangeResult kvRange = wait(tr.getRange(begin, end, 1000));
-					if (!kvRange.more && kvRange.size() == 0) {
+					RangeResult kvRange = co_await tr.getRange(begin, end, 1000);
+					if (!kvRange.more && kvRange.empty()) {
 						TraceEvent("SnapTestNoMoreEntries").log();
 						break;
 					}
@@ -255,24 +258,27 @@ struct SnapTestWorkload : TestWorkload {
 					}
 					begin = firstGreaterThan(kvRange.end()[-1].key);
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.isValid()) {
+					co_await tr.onError(err);
 				}
 			}
 			if (cnt != 1000) {
 				TraceEvent(SevError, "SnapTestVerifyCntValue").detail("Value", cnt);
 				throw operation_failed();
 			}
-		} else if (self->testID == 4) {
+		} else if (testID == 4) {
 			// create a snapshot with a non whitelisted binary path and operation
 			// should fail
-			state bool testedFailure = false;
+			bool testedFailure = false;
 			snapFailed = false;
-			loop {
-				self->snapUID = deterministicRandom()->randomUniqueID();
+			while (true) {
+				snapUID = deterministicRandom()->randomUniqueID();
 				try {
 					StringRef snapCmdRef = "/bin/snap_create1.sh"_sr;
-					Future<Void> status = snapCreate(cx, snapCmdRef, self->snapUID);
-					wait(status);
+					Future<Void> status = snapCreate(cx, snapCmdRef, snapUID);
+					co_await status;
 					break;
 				} catch (Error& e) {
 					if (e.code() == error_code_snap_not_fully_recovered_unsupported) {
@@ -287,8 +293,7 @@ struct SnapTestWorkload : TestWorkload {
 			}
 			ASSERT(testedFailure || snapFailed);
 		}
-		wait(delay(0.0));
-		return Void();
+		co_await delay(0.0);
 	}
 };
 
diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.cpp
similarity index 73%
rename from fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
rename to fdbserver/workloads/SpecialKeySpaceCorrectness.cpp
index e049fcc3650..1e6563fab9d 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * SpecialKeySpaceCorrectness.actor.cpp
+ * SpecialKeySpaceCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,17 +21,16 @@
 #include "boost/lexical_cast.hpp"
 #include "boost/algorithm/string.hpp"
 
-#include "fdbclient/GlobalConfig.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/GlobalConfig.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/Schemas.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/SpecialKeySpace.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/IRandom.h"
-#include "flow/actorcompiler.h"
 
 struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 	static constexpr auto NAME = "SpecialKeySpaceCorrectness";
@@ -45,7 +44,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 	Standalone<VectorRef<KeyRangeRef>> keys;
 	Standalone<VectorRef<KeyRangeRef>> rwKeys;
 
-	SpecialKeySpaceCorrectnessWorkload(WorkloadContext const& wcx)
+	explicit SpecialKeySpaceCorrectnessWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), wrongResults("Wrong Results"), keysCount("Number of generated keys") {
 		minKeysPerRange = getOption(options, "minKeysPerRange"_sr, 1);
 		maxKeysPerRange = getOption(options, "maxKeysPerRange"_sr, 100);
@@ -62,7 +61,6 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 	}
 
 	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	Future<bool> check(Database const& cx) override { return wrongResults.getValue() == 0; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 	// disable the default timeout setting
@@ -113,21 +111,20 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				               Value(deterministicRandom()->randomAlphaNumeric(self->valBytes)));
 			}
 		}
-		ASSERT(rwImpls.size() > 0);
+		ASSERT(!rwImpls.empty());
 
 		return Void();
 	}
-	ACTOR Future<Void> _start(Database cx, SpecialKeySpaceCorrectnessWorkload* self) {
+	Future<Void> start(Database const& cx) override {
 		testRywLifetime(cx);
-		wait(timeout(self->testSpecialKeySpaceErrors(cx, self) && self->getRangeCallActor(cx, self) &&
-		                 testConflictRanges(cx, /*read*/ true, self) && testConflictRanges(cx, /*read*/ false, self) &&
-		                 self->metricsApiCorrectnessActor(cx, self),
-		             self->testDuration,
-		             Void()));
+		co_await timeout(testSpecialKeySpaceErrors(cx, this) && getRangeCallActor(cx, this) &&
+		                     testConflictRanges(cx, /*read*/ true) && testConflictRanges(cx, /*read*/ false) &&
+		                     metricsApiCorrectnessActor(cx, this),
+		                 testDuration,
+		                 Void());
 		// Only use one client to avoid potential conflicts on changing cluster configuration
-		if (self->clientId == 0)
-			wait(self->managementApiCorrectnessActor(cx, self));
-		return Void();
+		if (clientId == 0)
+			co_await managementApiCorrectnessActor(cx, this);
 	}
 
 	// This would be a unit test except we need a Database to create an ryw transaction
@@ -146,15 +143,15 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		ASSERT(f.getError().code() == error_code_transaction_cancelled);
 	}
 
-	ACTOR Future<Void> getRangeCallActor(Database cx, SpecialKeySpaceCorrectnessWorkload* self) {
-		state double lastTime = now();
-		state Reverse reverse = Reverse::False;
-		loop {
-			wait(poisson(&lastTime, 1.0 / self->transactionsPerSecond));
+	Future<Void> getRangeCallActor(Database cx, SpecialKeySpaceCorrectnessWorkload* self) {
+		double lastTime = now();
+		Reverse reverse = Reverse::False;
+		while (true) {
+			co_await poisson(&lastTime, 1.0 / self->transactionsPerSecond);
 			reverse.set(deterministicRandom()->coinflip());
-			state GetRangeLimits limit = self->randomLimits();
-			state KeySelector begin = self->randomKeySelector();
-			state KeySelector end = self->randomKeySelector();
+			GetRangeLimits limit = self->randomLimits();
+			KeySelector begin = self->randomKeySelector();
+			KeySelector end = self->randomKeySelector();
 			auto correctResultFuture = self->ryw->getRange(begin, end, limit, Snapshot::False, reverse);
 			ASSERT(correctResultFuture.isReady());
 			auto correctResult = correctResultFuture.getValue();
@@ -260,7 +257,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 	}
 
 	KeyRange randomRWKeyRange() {
-		ASSERT(rwImpls.size() > 0);
+		ASSERT(!rwImpls.empty());
 		Key prefix = rwImpls[deterministicRandom()->randomInt(0, rwImpls.size())]->getKeyRange().begin;
 		Key rkey1 = Key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, keyBytes)))
 		                .withPrefix(prefix);
@@ -313,45 +310,46 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		return limit;
 	}
 
-	ACTOR Future<Void> testSpecialKeySpaceErrors(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
-		state Database cx = cx_->clone();
+	Future<Void> testSpecialKeySpaceErrors(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
+		Database cx = cx_->clone();
 		// TODO: explaining the purpose of and difference between `tx`, `defaultTx1`, and `defaultTx2`.
-		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
-		state Reference<ReadYourWritesTransaction> defaultTx1 = makeReference<ReadYourWritesTransaction>(cx);
-		state Reference<ReadYourWritesTransaction> defaultTx2 = makeReference<ReadYourWritesTransaction>(cx);
-		state bool disableRyw = deterministicRandom()->coinflip();
+		auto tx = makeReference<ReadYourWritesTransaction>(cx);
+		auto defaultTx1 = makeReference<ReadYourWritesTransaction>(cx);
+		auto defaultTx2 = makeReference<ReadYourWritesTransaction>(cx);
+		bool disableRyw = deterministicRandom()->coinflip();
 
 		// Formerly tenant check that conflict ranges stay the same after commit
 		// and depending on if RYW is disabled
 		// TODO(gglass): is this still needed?
 		{
-			state RangeResult readresult1;
-			state RangeResult readresult2;
-			state RangeResult writeResult1;
-			state RangeResult writeResult2;
+			RangeResult readresult1;
+			RangeResult readresult2;
+			RangeResult writeResult1;
+			RangeResult writeResult2;
 			try {
 				if (disableRyw) {
 					defaultTx1->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
 				}
 				defaultTx1->addReadConflictRange(singleKeyRange("testKeylll"_sr));
 				defaultTx1->addWriteConflictRange(singleKeyRange("testKeylll"_sr));
-				wait(store(readresult1, defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
-				wait(store(writeResult1, defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
-				wait(defaultTx1->commit());
+				readresult1 = co_await defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY);
+				writeResult1 = co_await defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY);
+				co_await defaultTx1->commit();
 				CODE_PROBE(true, "conflict range commit succeeded");
 			} catch (Error& e) {
 				if (e.code() == error_code_actor_cancelled)
 					throw;
 				CODE_PROBE(true, "conflict range commit error thrown");
 			}
-			wait(store(readresult2, defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
-			wait(store(writeResult2, defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY)));
+			readresult2 = co_await defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY);
+			writeResult2 = co_await defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(readresult1 == readresult2);
 			ASSERT(writeResult1 == writeResult2);
 			defaultTx1->reset();
 		}
 
-		loop {
+		while (true) {
+			Error caughtErr;
 			try {
 				if (disableRyw) {
 					defaultTx1->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
@@ -359,28 +357,36 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				}
 				defaultTx1->setOption(FDBTransactionOptions::REPORT_CONFLICTING_KEYS);
 				defaultTx2->setOption(FDBTransactionOptions::REPORT_CONFLICTING_KEYS);
-				wait(success(defaultTx1->getReadVersion()));
-				wait(success(defaultTx2->getReadVersion()));
+				co_await defaultTx1->getReadVersion();
+				co_await defaultTx2->getReadVersion();
 				defaultTx1->addReadConflictRange(singleKeyRange("foo"_sr));
 				defaultTx1->addWriteConflictRange(singleKeyRange("foo"_sr));
 				defaultTx2->addWriteConflictRange(singleKeyRange("foo"_sr));
-				wait(defaultTx2->commit());
-				try {
-					wait(defaultTx1->commit());
-					ASSERT(false);
-				} catch (Error& e) {
-					state Error err = e;
+				co_await defaultTx2->commit();
+				{
+					Error caughtErr;
+					try {
+						co_await defaultTx1->commit();
+						ASSERT(false);
+					} catch (Error& e) {
+						caughtErr = e;
+					}
+					Error err = caughtErr;
 					if (err.code() != error_code_not_committed) {
-						wait(defaultTx1->onError(err));
-						wait(defaultTx2->onError(err));
+						if (err.isValid()) {
+							co_await defaultTx1->onError(err);
+						}
+						if (err.isValid()) {
+							co_await defaultTx2->onError(err);
+						}
 						continue;
 					}
-					state RangeResult readConflictRange =
-					    wait(defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY));
-					state RangeResult writeConflictRange =
-					    wait(defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY));
-					state RangeResult conflictKeys =
-					    wait(defaultTx1->getRange(conflictingKeysRange, CLIENT_KNOBS->TOO_MANY));
+					RangeResult readConflictRange =
+					    co_await defaultTx1->getRange(readConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY);
+					RangeResult writeConflictRange =
+					    co_await defaultTx1->getRange(writeConflictRangeKeysRange, CLIENT_KNOBS->TOO_MANY);
+					RangeResult conflictKeys =
+					    co_await defaultTx1->getRange(conflictingKeysRange, CLIENT_KNOBS->TOO_MANY);
 
 					// size is 2 because singleKeyRange includes the key after
 					ASSERT(readConflictRange.size() == 2 &&
@@ -394,16 +400,17 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					break;
 				}
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw;
-				wait(defaultTx2->onError(e));
+				caughtErr = e;
 			}
+			if (caughtErr.code() == error_code_actor_cancelled)
+				throw caughtErr;
+			co_await defaultTx2->onError(caughtErr);
 		}
 		// begin key outside module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transactio"_sr, "\xff\xff/transaction0"_sr),
-			                          CLIENT_KNOBS->TOO_MANY)));
+			co_await tx->getRange(KeyRangeRef("\xff\xff/transactio"_sr, "\xff\xff/transaction0"_sr),
+			                      CLIENT_KNOBS->TOO_MANY);
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -414,8 +421,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// end key outside module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction1"_sr),
-			                          CLIENT_KNOBS->TOO_MANY)));
+			co_await tx->getRange(KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction1"_sr),
+			                      CLIENT_KNOBS->TOO_MANY);
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -426,8 +433,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// both begin and end outside module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transaction"_sr, "\xff\xff/transaction1"_sr),
-			                          CLIENT_KNOBS->TOO_MANY)));
+			co_await tx->getRange(KeyRangeRef("\xff\xff/transaction"_sr, "\xff\xff/transaction1"_sr),
+			                      CLIENT_KNOBS->TOO_MANY);
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -438,8 +445,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// legal range read using the module range
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction0"_sr),
-			                          CLIENT_KNOBS->TOO_MANY)));
+			co_await tx->getRange(KeyRangeRef("\xff\xff/transaction/"_sr, "\xff\xff/transaction0"_sr),
+			                      CLIENT_KNOBS->TOO_MANY);
 			CODE_PROBE(true, "read transaction special keyrange");
 			tx->reset();
 		} catch (Error& e) {
@@ -452,9 +459,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			const KeyRef startKey = "\xff\xff/transactio"_sr;
 			const KeyRef endKey = "\xff\xff/transaction1"_sr;
 			RangeResult result =
-			    wait(tx->getRange(KeyRangeRef(startKey, endKey), GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
+			    co_await tx->getRange(KeyRangeRef(startKey, endKey), GetRangeLimits(CLIENT_KNOBS->TOO_MANY));
 			// The whole transaction module should be empty
-			ASSERT(!result.size());
+			ASSERT(result.empty());
 			tx->reset();
 		} catch (Error& e) {
 			throw;
@@ -465,7 +472,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			tx->addReadConflictRange(singleKeyRange("testKey"_sr));
 			KeySelector begin = KeySelectorRef(readConflictRangeKeysRange.begin, false, 1);
 			KeySelector end = KeySelectorRef("\xff\xff/transaction0"_sr, false, 0);
-			wait(success(tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY))));
+			co_await tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY));
 			CODE_PROBE(true, "end key selector inside module range");
 			tx->reset();
 		} catch (Error& e) {
@@ -474,9 +481,9 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		// No module found error case with keys
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			wait(success(tx->getRange(
+			co_await tx->getRange(
 			    KeyRangeRef("\xff\xff/A_no_module_related_prefix"_sr, "\xff\xff/I_am_also_not_in_any_module"_sr),
-			    CLIENT_KNOBS->TOO_MANY)));
+			    CLIENT_KNOBS->TOO_MANY);
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -489,7 +496,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 			KeySelector begin = KeySelectorRef("\xff\xff/zzz_i_am_not_a_module"_sr, false, 1);
 			KeySelector end = KeySelectorRef("\xff\xff/zzz_to_be_the_final_one"_sr, false, 2);
-			wait(success(tx->getRange(begin, end, CLIENT_KNOBS->TOO_MANY)));
+			co_await tx->getRange(begin, end, CLIENT_KNOBS->TOO_MANY);
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -503,7 +510,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			const KeyRef key = "\xff\xff/cluster_file_path"_sr;
 			KeySelector begin = KeySelectorRef(key, false, 0);
 			KeySelector end = KeySelectorRef(keyAfter(key), false, 2);
-			RangeResult result = wait(tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
+			RangeResult result = co_await tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY));
 			ASSERT(result.readToBegin && result.readThroughEnd);
 			tx->reset();
 		} catch (Error& e) {
@@ -515,7 +522,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			const KeyRef key = "\xff\xff/transaction/a_to_be_the_first"_sr;
 			KeySelector begin = KeySelectorRef(key, false, 0);
 			KeySelector end = KeySelectorRef(key, false, 2);
-			RangeResult result = wait(tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
+			RangeResult result = co_await tx->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY));
 			ASSERT(result.readToBegin && !result.readThroughEnd);
 			tx->reset();
 		} catch (Error& e) {
@@ -563,7 +570,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			const KeySelector startKeySelector = KeySelectorRef("\xff\xff/test"_sr, true, -200);
 			const KeySelector endKeySelector = KeySelectorRef("test"_sr, true, -10);
 			RangeResult result =
-			    wait(tx->getRange(startKeySelector, endKeySelector, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
+			    co_await tx->getRange(startKeySelector, endKeySelector, GetRangeLimits(CLIENT_KNOBS->TOO_MANY));
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
@@ -572,35 +579,37 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 			tx->reset();
 		}
 		// test case when registered range is the same as the underlying module
+		Error err;
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			state RangeResult result =
-			    wait(tx->getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
-			                      CLIENT_KNOBS->TOO_MANY));
+			RangeResult result =
+			    co_await tx->getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
+			                          CLIENT_KNOBS->TOO_MANY);
 			// Note: there's possibility we get zero workers
-			if (result.size()) {
-				state KeyValueRef entry = deterministicRandom()->randomChoice(result);
-				Optional<Value> singleRes = wait(tx->get(entry.key));
+			if (!result.empty()) {
+				KeyValueRef entry = deterministicRandom()->randomChoice(result);
+				Optional<Value> singleRes = co_await tx->get(entry.key);
 				if (singleRes.present())
 					ASSERT(singleRes.get() == entry.value);
 			}
 			tx->reset();
 		} catch (Error& e) {
-			wait(tx->onError(e));
+			err = e;
+		}
+		if (err.isValid()) {
+			co_await tx->onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR static Future<Void> testConflictRanges(Database cx_, bool read, SpecialKeySpaceCorrectnessWorkload* self) {
-		state StringRef prefix = read ? readConflictRangeKeysRange.begin : writeConflictRangeKeysRange.begin;
+	Future<Void> testConflictRanges(Database cx_, bool read) {
+		StringRef prefix = read ? readConflictRangeKeysRange.begin : writeConflictRangeKeysRange.begin;
 		CODE_PROBE(read, "test read conflict range special key implementation");
 		CODE_PROBE(!read, "test write conflict range special key implementation");
 		// Get a default special key range instance
 		Database cx = cx_->clone();
-		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
-		state Reference<ReadYourWritesTransaction> referenceTx = makeReference<ReadYourWritesTransaction>(cx);
-		state bool ryw = deterministicRandom()->coinflip();
+		auto tx = makeReference<ReadYourWritesTransaction>(cx);
+		auto referenceTx = makeReference<ReadYourWritesTransaction>(cx);
+		bool ryw = deterministicRandom()->coinflip();
 		tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 		if (!ryw) {
 			tx->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
@@ -608,8 +617,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		referenceTx->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		referenceTx->setVersion(100); // Prevent this from doing a GRV or committing
 		referenceTx->clear(normalKeys);
-		int numKeys = deterministicRandom()->randomInt(1, self->conflictRangeSizeFactor) * 4;
-		state std::vector<std::string> keys; // Must all be distinct
+		int numKeys = deterministicRandom()->randomInt(1, conflictRangeSizeFactor) * 4;
+		std::vector<std::string> keys; // Must all be distinct
 		keys.resize(numKeys);
 		int lastKey = 0;
 		for (auto& key : keys) {
@@ -644,27 +653,27 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		}
 		if (!read && deterministicRandom()->coinflip()) {
 			try {
-				wait(tx->commit());
+				co_await tx->commit();
 			} catch (Error& e) {
 				if (e.code() == error_code_actor_cancelled)
 					throw;
-				return Void();
+				co_return;
 			}
 			CODE_PROBE(true, "Read write conflict range of committed transaction");
 		}
 		try {
-			wait(success(tx->get("\xff\xff/1314109/i_hope_this_isn't_registered"_sr)));
+			co_await tx->get("\xff\xff/1314109/i_hope_this_isn't_registered"_sr);
 			ASSERT(false);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled)
 				throw;
 			ASSERT(e.code() == error_code_special_keys_no_module_found);
 		}
-		for (int i = 0; i < self->conflictRangeSizeFactor; ++i) {
+		for (int i = 0; i < conflictRangeSizeFactor; ++i) {
 			GetRangeLimits limit;
 			KeySelector begin;
 			KeySelector end;
-			loop {
+			while (true) {
 				begin = firstGreaterOrEqual(deterministicRandom()->randomChoice(keys));
 				end = firstGreaterOrEqual(deterministicRandom()->randomChoice(keys));
 				if (begin.getKey() < end.getKey())
@@ -695,7 +704,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					    .detail("End", end)
 					    .detail("Ryw", ryw);
 					had_error = true;
-					++self->wrongResults;
+					++wrongResults;
 				}
 				++correct_iter;
 				++test_iter;
@@ -711,7 +720,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    .detail("Ryw", ryw);
 				++correct_iter;
 				had_error = true;
-				++self->wrongResults;
+				++wrongResults;
 			}
 			while (test_iter != testResultFuture.get().end()) {
 				TraceEvent(SevError, "TestFailure")
@@ -724,12 +733,11 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				    .detail("Ryw", ryw);
 				++test_iter;
 				had_error = true;
-				++self->wrongResults;
+				++wrongResults;
 			}
 			if (had_error)
 				break;
 		}
-		return Void();
 	}
 
 	bool getRangeResultInOrder(const RangeResult& result) {
@@ -746,46 +754,48 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		return true;
 	}
 
-	ACTOR Future<Void> managementApiCorrectnessActor(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
+	Future<Void> managementApiCorrectnessActor(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
 		// All management api related tests that cannot run with failure injections
-		state Database cx = cx_->clone();
-		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
+		Database cx = cx_->clone();
+		auto tx = makeReference<ReadYourWritesTransaction>(cx);
 		// test change coordinators and cluster description
 		// we randomly pick one process(not coordinator) and add it, in this case, it should always succeed
 		{
-			state std::string new_cluster_description;
-			state std::string new_coordinator_process;
-			state std::vector<std::string> old_coordinators_processes;
-			state bool possible_to_add_coordinator;
-			state KeyRange coordinators_key_range =
+			std::string new_cluster_description;
+			std::string new_coordinator_process;
+			std::vector<std::string> old_coordinators_processes;
+			bool possible_to_add_coordinator{ false };
+			KeyRange coordinators_key_range =
 			    KeyRangeRef("process/"_sr, "process0"_sr)
 			        .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"));
-			state unsigned retries = 0;
-			state bool changeCoordinatorsSucceeded = true;
-			loop {
+			unsigned retries = 0;
+			bool changeCoordinatorsSucceeded = true;
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-					Optional<Value> ccStrValue = wait(tx->get(coordinatorsKey));
+					Optional<Value> ccStrValue = co_await tx->get(coordinatorsKey);
 					ASSERT(ccStrValue.present()); // Otherwise, database is in a bad state
 					ClusterConnectionString ccStr(ccStrValue.get().toString());
-					// choose a new description if configuration allows transactions across differently named clusters
+					// choose a new description if configuration allows transactions across differently named
+					// clusters
 					new_cluster_description = SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT
 					                              ? deterministicRandom()->randomAlphaNumeric(8)
 					                              : ccStr.clusterKeyName().toString();
 					// get current coordinators
-					Optional<Value> processes_key = wait(tx->get(
-					    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
+					Optional<Value> processes_key = co_await tx->get(
+					    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")));
 					ASSERT(processes_key.present());
 					boost::split(
 					    old_coordinators_processes, processes_key.get().toString(), [](char c) { return c == ','; });
 					// pick up one non-coordinator process if possible
-					std::vector<ProcessData> workers = wait(getWorkers(&tx->getTransaction()));
+					std::vector<ProcessData> workers = co_await getWorkers(&tx->getTransaction());
 					std::string old_coordinators_processes_string = describe(old_coordinators_processes);
 					TraceEvent(SevDebug, "CoordinatorsManualChange")
 					    .detail("OldCoordinators", old_coordinators_processes_string)
 					    .detail("WorkerSize", workers.size());
 					if (workers.size() > old_coordinators_processes.size()) {
-						loop {
+						while (true) {
 							auto worker = deterministicRandom()->randomChoice(workers);
 							new_coordinator_process = worker.address.toString();
 							if (old_coordinators_processes_string.find(new_coordinator_process) == std::string::npos) {
@@ -799,38 +809,43 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					tx->reset();
 					break;
 				} catch (Error& e) {
-					wait(tx->onError(e));
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					err = e;
 				}
+				co_await tx->onError(err);
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 			}
 			TraceEvent(SevDebug, "CoordinatorsManualChange")
 			    .detail("NewCoordinator", possible_to_add_coordinator ? new_coordinator_process : "")
 			    .detail("NewClusterDescription", new_cluster_description);
 			if (possible_to_add_coordinator) {
-				loop {
-					try {
-						std::string new_processes_key(new_coordinator_process);
-						tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-						tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-						for (const auto& address : old_coordinators_processes) {
-							new_processes_key += "," + address;
+				while (true) {
+					{
+						Error err;
+						try {
+							std::string new_processes_key(new_coordinator_process);
+							tx->setOption(FDBTransactionOptions::RAW_ACCESS);
+							tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+							for (const auto& address : old_coordinators_processes) {
+								new_processes_key += "," + address;
+							}
+							tx->set("processes"_sr.withPrefix(
+							            SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+							        Value(new_processes_key));
+							// update cluster description
+							tx->set("cluster_description"_sr.withPrefix(
+							            SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
+							        Value(new_cluster_description));
+							co_await tx->commit();
+							ASSERT(false);
+						} catch (Error& e) {
+							err = e;
 						}
-						tx->set(
-						    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
-						    Value(new_processes_key));
-						// update cluster description
-						tx->set("cluster_description"_sr.withPrefix(
-						            SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
-						        Value(new_cluster_description));
-						wait(tx->commit());
-						ASSERT(false);
-					} catch (Error& e) {
-						TraceEvent(SevDebug, "CoordinatorsManualChange").error(e);
+						TraceEvent(SevDebug, "CoordinatorsManualChange").error(err);
 						// if we repeat doing the change, we will get the error:
 						// CoordinatorsResult::SAME_NETWORK_ADDRESSES
-						if (e.code() == error_code_special_keys_api_failure) {
-							Optional<Value> errorMsg =
-							    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
+						if (err.code() == error_code_special_keys_api_failure) {
+							Optional<Value> errorMsg = co_await tx->get(
+							    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
 							ASSERT(errorMsg.present());
 							std::string errorStr;
 							auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
@@ -857,15 +872,18 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 								break;
 							}
 						} else {
-							wait(tx->onError(e));
+							if (err.isValid()) {
+								co_await tx->onError(err);
+							}
 						}
-						wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+						co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 					}
 				}
 				// change successful, now check it is already changed
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-					Optional<Value> res = wait(tx->get(coordinatorsKey));
+					Optional<Value> res = co_await tx->get(coordinatorsKey);
 					ASSERT(res.present()); // Otherwise, database is in a bad state
 					ClusterConnectionString csNew(res.get().toString());
 					// verify the cluster description
@@ -873,7 +891,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					       new_cluster_description == csNew.clusterKeyName().toString());
 					ASSERT(!changeCoordinatorsSucceeded ||
 					       csNew.hostnames.size() + csNew.coords.size() == old_coordinators_processes.size() + 1);
-					std::vector<NetworkAddress> newCoordinators = wait(csNew.tryResolveHostnames());
+					std::vector<NetworkAddress> newCoordinators = co_await csNew.tryResolveHostnames();
 					// verify the coordinators' addresses
 					for (const auto& network_address : newCoordinators) {
 						std::string address_str = network_address.toString();
@@ -884,68 +902,76 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					}
 					tx->reset();
 				} catch (Error& e) {
-					wait(tx->onError(e));
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					err = e;
+				}
+				if (err.isValid()) {
+					co_await tx->onError(err);
 				}
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 				// change back to original settings
 				while (changeCoordinatorsSucceeded) {
+					Error err;
 					try {
 						std::string new_processes_key;
 						tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 						tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 						for (const auto& address : old_coordinators_processes) {
-							new_processes_key += new_processes_key.size() ? "," : "";
+							new_processes_key += !new_processes_key.empty() ? "," : "";
 							new_processes_key += address;
 						}
 						tx->set(
 						    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")),
 						    Value(new_processes_key));
-						wait(tx->commit());
+						co_await tx->commit();
 						ASSERT(false);
 					} catch (Error& e) {
-						TraceEvent(SevDebug, "CoordinatorsManualChangeRevert").error(e);
-						if (e.code() == error_code_special_keys_api_failure) {
-							Optional<Value> errorMsg =
-							    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-							ASSERT(errorMsg.present());
-							std::string errorStr;
-							auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-							auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-							// special_key_space_management_api_error_msg schema validation
-							ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-							TraceEvent(SevDebug, "CoordinatorsManualChangeRevert")
-							    .detail("ErrorMessage", valueObj["message"].get_str());
-							ASSERT(valueObj["command"].get_str() == "coordinators");
-							if (valueObj["retriable"].get_bool()) {
-								tx->reset();
-							} else if (valueObj["message"].get_str() ==
-							           "No change (existing configuration satisfies request)") {
-								tx->reset();
-								break;
-							} else {
-								TraceEvent(SevError, "CoordinatorsManualChangeRevert")
-								    .detail("UnexpectedError", valueObj["message"].get_str());
-								throw special_keys_api_failure();
-							}
+						err = e;
+					}
+					TraceEvent(SevDebug, "CoordinatorsManualChangeRevert").error(err);
+					if (err.code() == error_code_special_keys_api_failure) {
+						Optional<Value> errorMsg =
+						    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+						ASSERT(errorMsg.present());
+						std::string errorStr;
+						auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+						auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+						// special_key_space_management_api_error_msg schema validation
+						ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+						TraceEvent(SevDebug, "CoordinatorsManualChangeRevert")
+						    .detail("ErrorMessage", valueObj["message"].get_str());
+						ASSERT(valueObj["command"].get_str() == "coordinators");
+						if (valueObj["retriable"].get_bool()) {
+							tx->reset();
+						} else if (valueObj["message"].get_str() ==
+						           "No change (existing configuration satisfies request)") {
+							tx->reset();
+							break;
 						} else {
-							wait(tx->onError(e));
+							TraceEvent(SevError, "CoordinatorsManualChangeRevert")
+							    .detail("UnexpectedError", valueObj["message"].get_str());
+							throw special_keys_api_failure();
+						}
+					} else {
+						if (err.isValid()) {
+							co_await tx->onError(err);
 						}
-						wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
 					}
+					co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 				}
 			}
 		}
 		// data_distribution & maintenance get
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// maintenance
-				RangeResult maintenanceKVs = wait(
-				    tx->getRange(SpecialKeySpace::getManagementApiCommandRange("maintenance"), CLIENT_KNOBS->TOO_MANY));
+				RangeResult maintenanceKVs = co_await tx->getRange(
+				    SpecialKeySpace::getManagementApiCommandRange("maintenance"), CLIENT_KNOBS->TOO_MANY);
 				// By default, no maintenance is going on
-				ASSERT(!maintenanceKVs.more && !maintenanceKVs.size());
+				ASSERT(!maintenanceKVs.more && maintenanceKVs.empty());
 				// datadistribution
-				RangeResult ddKVs = wait(tx->getRange(SpecialKeySpace::getManagementApiCommandRange("datadistribution"),
-				                                      CLIENT_KNOBS->TOO_MANY));
+				RangeResult ddKVs = co_await tx->getRange(
+				    SpecialKeySpace::getManagementApiCommandRange("datadistribution"), CLIENT_KNOBS->TOO_MANY);
 				// By default, data_distribution/mode := "-1"
 				ASSERT(!ddKVs.more && ddKVs.size() == 1);
 				ASSERT(ddKVs[0].key ==
@@ -955,14 +981,16 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				tx->reset();
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevDebug, "MaintenanceGet").error(e);
-				wait(tx->onError(e));
+				err = e;
 			}
+			TraceEvent(SevDebug, "MaintenanceGet").error(err);
+			co_await tx->onError(err);
 		}
 		// maintenance set
 		{
 			// Make sure setting more than one zone as maintenance will fail
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -973,79 +1001,90 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					tx->set(Key(deterministicRandom()->randomAlphaNumeric(9))
 					            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("maintenance")),
 					        Value(boost::lexical_cast<std::string>(deterministicRandom()->randomInt(1, 100))));
-					wait(tx->commit());
+					co_await tx->commit();
 					ASSERT(false);
 				} catch (Error& e) {
-					TraceEvent(SevDebug, "MaintenanceSetMoreThanOneZone").error(e);
-					if (e.code() == error_code_special_keys_api_failure) {
-						Optional<Value> errorMsg =
-						    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-						ASSERT(errorMsg.present());
-						std::string errorStr;
-						auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-						auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-						// special_key_space_management_api_error_msg schema validation
-						ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-						ASSERT(valueObj["command"].get_str() == "maintenance" && !valueObj["retriable"].get_bool());
-						TraceEvent(SevDebug, "MaintenanceSetMoreThanOneZone")
-						    .detail("ErrorMessage", valueObj["message"].get_str());
-						tx->reset();
-						break;
-					} else {
-						wait(tx->onError(e));
+					err = e;
+				}
+				TraceEvent(SevDebug, "MaintenanceSetMoreThanOneZone").error(err);
+				if (err.code() == error_code_special_keys_api_failure) {
+					Optional<Value> errorMsg =
+					    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+					ASSERT(errorMsg.present());
+					std::string errorStr;
+					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+					auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+					// special_key_space_management_api_error_msg schema validation
+					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+					ASSERT(valueObj["command"].get_str() == "maintenance" && !valueObj["retriable"].get_bool());
+					TraceEvent(SevDebug, "MaintenanceSetMoreThanOneZone")
+					    .detail("ErrorMessage", valueObj["message"].get_str());
+					tx->reset();
+					break;
+				} else {
+					if (err.isValid()) {
+						co_await tx->onError(err);
 					}
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
 				}
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 			}
 			// Disable DD for SS failures
-			state int ignoreSSFailuresRetry = 0;
-			loop {
+			int ignoreSSFailuresRetry = 0;
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 					tx->set(ignoreSSFailuresZoneString.withPrefix(
 					            SpecialKeySpace::getManagementApiCommandPrefix("maintenance")),
 					        Value(boost::lexical_cast<std::string>(0)));
-					wait(tx->commit());
+					co_await tx->commit();
 					tx->reset();
 					ignoreSSFailuresRetry++;
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 				} catch (Error& e) {
-					TraceEvent(SevDebug, "MaintenanceDDIgnoreSSFailures").error(e);
-					// the second commit will fail since maintenance not allowed to use while DD disabled for SS
-					// failures
-					if (e.code() == error_code_special_keys_api_failure) {
-						Optional<Value> errorMsg =
-						    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-						ASSERT(errorMsg.present());
-						std::string errorStr;
-						auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-						auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-						// special_key_space_management_api_error_msg schema validation
-						ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-						ASSERT(valueObj["command"].get_str() == "maintenance" && !valueObj["retriable"].get_bool());
-						ASSERT(ignoreSSFailuresRetry > 0);
-						TraceEvent(SevDebug, "MaintenanceDDIgnoreSSFailures")
-						    .detail("Retry", ignoreSSFailuresRetry)
-						    .detail("ErrorMessage", valueObj["message"].get_str());
-						tx->reset();
-						break;
-					} else {
-						wait(tx->onError(e));
-					}
+					err = e;
+				}
+				if (!err.isValid()) {
 					ignoreSSFailuresRetry++;
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
+					continue;
+				}
+				TraceEvent(SevDebug, "MaintenanceDDIgnoreSSFailures").error(err);
+				// the second commit will fail since maintenance not allowed to use while DD disabled
+				// for SS failures
+				if (err.code() == error_code_special_keys_api_failure) {
+					Optional<Value> errorMsg =
+					    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+					ASSERT(errorMsg.present());
+					std::string errorStr;
+					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+					auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+					// special_key_space_management_api_error_msg schema validation
+					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+					ASSERT(valueObj["command"].get_str() == "maintenance" && !valueObj["retriable"].get_bool());
+					ASSERT(ignoreSSFailuresRetry > 0);
+					TraceEvent(SevDebug, "MaintenanceDDIgnoreSSFailures")
+					    .detail("Retry", ignoreSSFailuresRetry)
+					    .detail("ErrorMessage", valueObj["message"].get_str());
+					tx->reset();
+					break;
+				} else {
+					co_await tx->onError(err);
 				}
+				ignoreSSFailuresRetry++;
+				co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY);
 			}
 			// set dd mode to 0 and disable DD for rebalance
-			state uint8_t ddIgnoreValue = DDIgnore::NONE;
+			uint8_t ddIgnoreValue = DDIgnore::NONE;
 			if (deterministicRandom()->coinflip()) {
 				ddIgnoreValue |= DDIgnore::REBALANCE_READ;
 			}
 			if (deterministicRandom()->coinflip()) {
 				ddIgnoreValue |= DDIgnore::REBALANCE_DISK;
 			}
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -1053,40 +1092,44 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					tx->set("mode"_sr.withPrefix(ddPrefix), "0"_sr);
 					tx->set("rebalance_ignored"_sr.withPrefix(ddPrefix),
 					        BinaryWriter::toValue(ddIgnoreValue, Unversioned()));
-					wait(tx->commit());
+					co_await tx->commit();
 					tx->reset();
 					break;
 				} catch (Error& e) {
-					TraceEvent(SevDebug, "DataDistributionDisableModeAndRebalance").error(e);
-					wait(tx->onError(e));
+					err = e;
 				}
+				TraceEvent(SevDebug, "DataDistributionDisableModeAndRebalance").error(err);
+				co_await tx->onError(err);
 			}
 			// verify underlying system keys are consistent with the change
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 					// check DD disabled for SS failures
-					Optional<Value> val1 = wait(tx->get(healthyZoneKey));
+					Optional<Value> val1 = co_await tx->get(healthyZoneKey);
 					ASSERT(val1.present());
 					auto healthyZone = decodeHealthyZoneValue(val1.get());
 					ASSERT(healthyZone.first == ignoreSSFailuresZoneString);
 					// check DD mode
-					Optional<Value> val2 = wait(tx->get(dataDistributionModeKey));
+					Optional<Value> val2 = co_await tx->get(dataDistributionModeKey);
 					ASSERT(val2.present());
 					// mode should be set to 0
 					ASSERT(BinaryReader::fromStringRef<int>(val2.get(), Unversioned()) == 0);
 					// check DD disabled for rebalance
-					Optional<Value> val3 = wait(tx->get(rebalanceDDIgnoreKey));
+					Optional<Value> val3 = co_await tx->get(rebalanceDDIgnoreKey);
 					ASSERT(val3.present() &&
 					       BinaryReader::fromStringRef<uint8_t>(val3.get(), Unversioned()) == ddIgnoreValue);
 					tx->reset();
 					break;
 				} catch (Error& e) {
-					wait(tx->onError(e));
+					err = e;
 				}
+				co_await tx->onError(err);
 			}
 			// then, clear all changes
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -1095,50 +1138,51 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
 					tx->clear("mode"_sr.withPrefix(ddPrefix));
 					tx->clear("rebalance_ignored"_sr.withPrefix(ddPrefix));
-					wait(tx->commit());
+					co_await tx->commit();
 					tx->reset();
 					break;
 				} catch (Error& e) {
-					wait(tx->onError(e));
+					err = e;
 				}
+				co_await tx->onError(err);
 			}
 			// verify all changes are cleared
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 					// check DD SSFailures key
-					Optional<Value> val1 = wait(tx->get(healthyZoneKey));
+					Optional<Value> val1 = co_await tx->get(healthyZoneKey);
 					ASSERT(!val1.present());
 					// check DD mode
-					Optional<Value> val2 = wait(tx->get(dataDistributionModeKey));
+					Optional<Value> val2 = co_await tx->get(dataDistributionModeKey);
 					ASSERT(!val2.present());
 					// check DD rebalance key
-					Optional<Value> val3 = wait(tx->get(rebalanceDDIgnoreKey));
+					Optional<Value> val3 = co_await tx->get(rebalanceDDIgnoreKey);
 					ASSERT(!val3.present());
 					tx->reset();
 					break;
 				} catch (Error& e) {
-					wait(tx->onError(e));
+					err = e;
 				}
+				co_await tx->onError(err);
 			}
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> metricsApiCorrectnessActor(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
-		state Database cx = cx_->clone();
-		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
+	Future<Void> metricsApiCorrectnessActor(Database cx_, SpecialKeySpaceCorrectnessWorkload* self) {
+		Database cx = cx_->clone();
+		auto tx = makeReference<ReadYourWritesTransaction>(cx);
 		tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 		{
-			Optional<Value> metrics = wait(tx->get("fault_tolerance_metrics_json"_sr.withPrefix(
-			    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::METRICS).begin)));
+			Optional<Value> metrics = co_await tx->get("fault_tolerance_metrics_json"_sr.withPrefix(
+			    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::METRICS).begin));
 			ASSERT(metrics.present());
 			auto metricsObj = readJSONStrictly(metrics.get().toString()).get_obj();
 			auto schema = readJSONStrictly(JSONSchemas::faultToleranceStatusSchema.toString()).get_obj();
 			std::string errorStr;
 			ASSERT(schemaMatch(schema, metricsObj, errorStr, SevError, true));
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/SpecialKeySpaceRobustness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceRobustness.cpp
similarity index 61%
rename from fdbserver/workloads/SpecialKeySpaceRobustness.actor.cpp
rename to fdbserver/workloads/SpecialKeySpaceRobustness.cpp
index 1161bc8fa7b..07b3c344f91 100644
--- a/fdbserver/workloads/SpecialKeySpaceRobustness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceRobustness.cpp
@@ -1,5 +1,5 @@
 /*
- * SpecialKeySpaceRobustness.actor.cpp
+ * SpecialKeySpaceRobustness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -21,30 +21,28 @@
 #include "boost/lexical_cast.hpp"
 #include "boost/algorithm/string.hpp"
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/Schemas.h"
-#include "fdbclient/SpecialKeySpace.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/SpecialKeySpace.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h"
 
 struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 	static constexpr auto NAME = "SpecialKeySpaceRobustness";
 
-	SpecialKeySpaceRobustnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+	explicit SpecialKeySpaceRobustnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
 	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
 	Future<Void> _setup(Database cx, SpecialKeySpaceRobustnessWorkload* self) { return Void(); }
 
-	ACTOR Future<Void> _start(Database cx, SpecialKeySpaceRobustnessWorkload* self) {
+	Future<Void> start(Database const& cx) override {
 		// Only use one client to avoid potential conflicts on changing cluster configuration
-		if (self->clientId == 0)
-			wait(self->managementApiCorrectnessActor(cx, self));
-		return Void();
+		if (clientId == 0)
+			co_await managementApiCorrectnessActor(cx, this);
 	}
 
 	bool getRangeResultInOrder(const RangeResult& result) {
@@ -62,26 +60,26 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 	}
 
 	// A test utility that exclude the worker with `workerAddress` using `command` and return the value of `versionKey`.
-	ACTOR static Future<Optional<Value>> runExcludeAndGetVersionKey(Reference<ReadYourWritesTransaction> tx,
-	                                                                std::string workerAddress,
-	                                                                std::string command,
-	                                                                KeyRef versionKey) {
+	static Future<Optional<Value>> runExcludeAndGetVersionKey(Reference<ReadYourWritesTransaction> tx,
+	                                                          std::string workerAddress,
+	                                                          std::string command,
+	                                                          KeyRef versionKey) {
 		tx->reset();
 		tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 		tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 		tx->set(Key(workerAddress).withPrefix(SpecialKeySpace::getManagementApiCommandPrefix(command)), ValueRef());
-		wait(tx->commit());
+		co_await tx->commit();
 		tx->reset();
 
 		tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-		Optional<Value> versionKeyValue = wait(tx->get(versionKey));
+		Optional<Value> versionKeyValue = co_await tx->get(versionKey);
 		tx->reset();
-		return versionKeyValue;
+		co_return versionKeyValue;
 	}
 
-	ACTOR Future<Void> managementApiCorrectnessActor(Database cx, SpecialKeySpaceRobustnessWorkload* self) {
+	Future<Void> managementApiCorrectnessActor(Database cx, SpecialKeySpaceRobustnessWorkload* self) {
 		// Management api related tests that can run during failure injections
-		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
+		auto tx = makeReference<ReadYourWritesTransaction>(cx);
 		// test ordered option keys
 		{
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
@@ -92,50 +90,55 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 				        .withSuffix(option),
 				    ValueRef());
 			}
-			RangeResult result = wait(tx->getRange(
+			RangeResult result = co_await tx->getRange(
 			    KeyRangeRef("options/"_sr, "options0"_sr)
 			        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin),
-			    CLIENT_KNOBS->TOO_MANY));
+			    CLIENT_KNOBS->TOO_MANY);
 			ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY);
 			ASSERT(result.size() == SpecialKeySpace::getManagementApiOptionsSet().size());
 			ASSERT(self->getRangeResultInOrder(result));
 			tx->reset();
 		}
 		// "exclude" error message schema check
+		Error err;
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 			tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 			tx->set("Invalid_Network_Address"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")),
 			        ValueRef());
-			wait(tx->commit());
+			co_await tx->commit();
 			ASSERT(false);
 		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled)
-				throw;
-			if (e.code() == error_code_special_keys_api_failure) {
-				Optional<Value> errorMsg =
-				    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-				ASSERT(errorMsg.present());
-				std::string errorStr;
-				auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-				auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-				// special_key_space_management_api_error_msg schema validation
-				ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-				ASSERT(valueObj["command"].get_str() == "exclude" && !valueObj["retriable"].get_bool());
-			} else {
-				TraceEvent(SevDebug, "UnexpectedError").error(e).detail("Command", "Exclude");
-				wait(tx->onError(e));
+			err = e;
+		}
+		if (err.code() == error_code_actor_cancelled)
+			throw err;
+		if (err.code() == error_code_special_keys_api_failure) {
+			Optional<Value> errorMsg =
+			    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+			ASSERT(errorMsg.present());
+			std::string errorStr;
+			auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+			auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+			// special_key_space_management_api_error_msg schema validation
+			ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+			ASSERT(valueObj["command"].get_str() == "exclude" && !valueObj["retriable"].get_bool());
+		} else {
+			TraceEvent(SevDebug, "UnexpectedError").error(err).detail("Command", "Exclude");
+			if (err.isValid()) {
+				co_await tx->onError(err);
 			}
-			tx->reset();
 		}
+		tx->reset();
 		// "Exclude" same address multiple times, and only the first excluson should trigger a system metadata update.
 		{
+			Error err;
 			try {
-				state std::string excludeWorker;
-				state std::string excludeCommand;
-				state std::string failedCommand;
-				state KeyRef excludeVersionKey;
-				state KeyRef failedVersionKey;
+				std::string excludeWorker;
+				std::string excludeCommand;
+				std::string failedCommand;
+				KeyRef excludeVersionKey;
+				KeyRef failedVersionKey;
 
 				// Test exclude servers and exclude localities randomly.
 				if (deterministicRandom()->coinflip()) {
@@ -158,38 +161,41 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 				    .detail("ExcludeFailedCommand", failedCommand);
 
 				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				state Optional<Value> versionKey0 = wait(tx->get(excludeVersionKey));
+				Optional<Value> versionKey0 = co_await tx->get(excludeVersionKey);
 
-				state Optional<Value> versionKey1 =
-				    wait(runExcludeAndGetVersionKey(tx, excludeWorker, excludeCommand, excludeVersionKey));
+				Optional<Value> versionKey1 =
+				    co_await runExcludeAndGetVersionKey(tx, excludeWorker, excludeCommand, excludeVersionKey);
 				ASSERT(versionKey1.present());
 				ASSERT(versionKey0 != versionKey1);
 				Optional<Value> versionKey2 =
-				    wait(runExcludeAndGetVersionKey(tx, excludeWorker, excludeCommand, excludeVersionKey));
+				    co_await runExcludeAndGetVersionKey(tx, excludeWorker, excludeCommand, excludeVersionKey);
 				ASSERT(versionKey2.present());
 				// Exclude the same worker twice. The second exclusion shouldn't trigger a system metadata update.
 				ASSERT(versionKey1 == versionKey2);
 
 				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				state Optional<Value> versionKey3 = wait(tx->get(failedVersionKey));
+				Optional<Value> versionKey3 = co_await tx->get(failedVersionKey);
 
-				state Optional<Value> versionKey4 =
-				    wait(runExcludeAndGetVersionKey(tx, excludeWorker, failedCommand, failedVersionKey));
+				Optional<Value> versionKey4 =
+				    co_await runExcludeAndGetVersionKey(tx, excludeWorker, failedCommand, failedVersionKey);
 				ASSERT(versionKey4.present());
 				ASSERT(versionKey3 != versionKey4);
 				Optional<Value> versionKey5 =
-				    wait(runExcludeAndGetVersionKey(tx, excludeWorker, failedCommand, failedVersionKey));
+				    co_await runExcludeAndGetVersionKey(tx, excludeWorker, failedCommand, failedVersionKey);
 				ASSERT(versionKey5.present());
 				// Exclude the same worker twice. The second exclusion shouldn't trigger a system metadata update.
 				ASSERT(versionKey4 == versionKey5);
 
 				tx->reset();
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw;
-				if (e.code() == error_code_special_keys_api_failure) {
+				err = e;
+			}
+			if (err.isValid()) {
+				if (err.code() == error_code_actor_cancelled)
+					throw err;
+				if (err.code() == error_code_special_keys_api_failure) {
 					Optional<Value> errorMsg =
-					    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
+					    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
 					ASSERT(errorMsg.present());
 					std::string errorStr;
 					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
@@ -201,29 +207,32 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 					       !valueObj["retriable"].get_bool());
 				} else {
 					TraceEvent(SevDebug, "UnexpectedError")
-					    .error(e)
+					    .error(err)
 					    .detail("Command", "Exclude")
 					    .detail("Test", "Repeated exclusions");
-					wait(tx->onError(e));
+					if (err.isValid()) {
+						co_await tx->onError(err);
+					}
 				}
-				tx->reset();
 			}
+			tx->reset();
 		}
 		// "setclass"
 		{
+			Error err;
 			try {
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				// test getRange
-				state RangeResult result = wait(tx->getRange(
+				RangeResult result = co_await tx->getRange(
 				    KeyRangeRef("process/class_type/"_sr, "process/class_type0"_sr)
 				        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
-				    CLIENT_KNOBS->TOO_MANY));
+				    CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY);
 				ASSERT(self->getRangeResultInOrder(result));
 				// check correctness of classType of each process
-				std::vector<ProcessData> workers = wait(getWorkers(&tx->getTransaction()));
-				if (workers.size()) {
+				std::vector<ProcessData> workers = co_await getWorkers(&tx->getTransaction());
+				if (!workers.empty()) {
 					for (const auto& worker : workers) {
 						Key addr =
 						    Key("process/class_type/" + formatIpPort(worker.address.ip, worker.address.port))
@@ -240,32 +249,35 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 						// Each process should find its corresponding element
 						ASSERT(found);
 					}
-					state ProcessData worker = deterministicRandom()->randomChoice(workers);
-					state Key addr =
+					ProcessData worker = deterministicRandom()->randomChoice(workers);
+					Key addr =
 					    Key("process/class_type/" + formatIpPort(worker.address.ip, worker.address.port))
 					        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin);
 					tx->set(addr, "InvalidProcessType"_sr);
 					// test ryw
-					Optional<Value> processType = wait(tx->get(addr));
+					Optional<Value> processType = co_await tx->get(addr);
 					ASSERT(processType.present() && processType.get() == "InvalidProcessType"_sr);
 					// test ryw disabled
 					tx->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
-					Optional<Value> originalProcessType = wait(tx->get(addr));
+					Optional<Value> originalProcessType = co_await tx->get(addr);
 					ASSERT(originalProcessType.present() &&
 					       originalProcessType.get() == worker.processClass.toString());
 					// test error handling (invalid value type)
-					wait(tx->commit());
+					co_await tx->commit();
 					ASSERT(false);
 				} else {
 					// If no worker process returned, skip the test
 					TraceEvent(SevDebug, "EmptyWorkerListInSetClassTest").log();
 				}
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw;
-				if (e.code() == error_code_special_keys_api_failure) {
+				err = e;
+			}
+			if (err.isValid()) {
+				if (err.code() == error_code_actor_cancelled)
+					throw err;
+				if (err.code() == error_code_special_keys_api_failure) {
 					Optional<Value> errorMsg =
-					    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
+					    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
 					ASSERT(errorMsg.present());
 					std::string errorStr;
 					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
@@ -274,26 +286,29 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
 					ASSERT(valueObj["command"].get_str() == "setclass" && !valueObj["retriable"].get_bool());
 				} else {
-					TraceEvent(SevDebug, "UnexpectedError").error(e).detail("Command", "Setclass");
-					wait(tx->onError(e));
+					TraceEvent(SevDebug, "UnexpectedError").error(err).detail("Command", "Setclass");
+					if (err.isValid()) {
+						co_await tx->onError(err);
+					}
 				}
-				tx->reset();
 			}
+			tx->reset();
 		}
 		// read class_source
 		{
+			Error err;
 			try {
 				// test getRange
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-				state RangeResult class_source_result = wait(tx->getRange(
+				RangeResult class_source_result = co_await tx->getRange(
 				    KeyRangeRef("process/class_source/"_sr, "process/class_source0"_sr)
 				        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
-				    CLIENT_KNOBS->TOO_MANY));
+				    CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!class_source_result.more && class_source_result.size() < CLIENT_KNOBS->TOO_MANY);
 				ASSERT(self->getRangeResultInOrder(class_source_result));
 				// check correctness of classType of each process
-				std::vector<ProcessData> workers = wait(getWorkers(&tx->getTransaction()));
-				if (workers.size()) {
+				std::vector<ProcessData> workers = co_await getWorkers(&tx->getTransaction());
+				if (!workers.empty()) {
 					for (const auto& worker : workers) {
 						Key addr =
 						    Key("process/class_source/" + formatIpPort(worker.address.ip, worker.address.port))
@@ -313,20 +328,19 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 						ASSERT(found);
 					}
 					ProcessData worker = deterministicRandom()->randomChoice(workers);
-					state std::string address = formatIpPort(worker.address.ip, worker.address.port);
+					std::string address = formatIpPort(worker.address.ip, worker.address.port);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 					tx->set(
 					    Key("process/class_type/" + address)
 					        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
-					    Value(worker.processClass.toString())); // Set it as the same class type as before, thus only
-					                                            // class source will be changed
-					wait(tx->commit());
+					    Value(worker.processClass.toString())); // Set it as the same class type as before, thus
+					                                            // only class source will be changed
+					co_await tx->commit();
 					tx->reset();
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-					Optional<Value> class_source = wait(tx->get(
+					Optional<Value> class_source = co_await tx->get(
 					    Key("process/class_source/" + address)
-					        .withPrefix(
-					            SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin)));
+					        .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin));
 					TraceEvent(SevDebug, "SetClassSourceDebug")
 					    .detail("Address", address)
 					    .detail("Present", class_source.present())
@@ -336,8 +350,8 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 						if (class_source.get() == "command_line"_sr) {
 							// if the process is rebooted after the commit,
 							// the class source is changed back to command_line
-							state LocalityData _locality(worker.locality);
-							std::vector<ProcessData> _workers = wait(getWorkers(&tx->getTransaction()));
+							LocalityData _locality(worker.locality);
+							std::vector<ProcessData> _workers = co_await getWorkers(&tx->getTransaction());
 							bool found = false;
 							for (const auto& w : _workers) {
 								auto w_addr = formatIpPort(w.address.ip, w.address.port);
@@ -359,12 +373,16 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 					TraceEvent(SevDebug, "EmptyWorkerListInSetClassTest").log();
 				}
 			} catch (Error& e) {
-				wait(tx->onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tx->onError(err);
 			}
 		}
 		// test lock and unlock
 		// maske sure we lock the database
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -372,80 +390,92 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 				UID uid = deterministicRandom()->randomUniqueID();
 				tx->set(SpecialKeySpace::getManagementApiCommandPrefix("lock"), uid.toString());
 				// commit
-				wait(tx->commit());
+				co_await tx->commit();
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevDebug, "DatabaseLockFailure").error(e);
-				// In case commit_unknown_result is thrown by buggify, we may try to lock more than once
-				// The second lock commit will throw special_keys_api_failure error
-				if (e.code() == error_code_special_keys_api_failure) {
-					Optional<Value> errorMsg =
-					    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-					ASSERT(errorMsg.present());
-					std::string errorStr;
-					auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-					auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-					// special_key_space_management_api_error_msg schema validation
-					ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-					ASSERT(valueObj["command"].get_str() == "lock" && !valueObj["retriable"].get_bool());
-					break;
-				} else if (e.code() == error_code_database_locked) {
-					// Database is already locked. This can happen if a previous attempt
-					// failed with unknown_result.
-					break;
-				} else {
-					wait(tx->onError(e));
-				}
+				err = e;
+			}
+			TraceEvent(SevDebug, "DatabaseLockFailure").error(err);
+			// In case commit_unknown_result is thrown by buggify, we may try to lock more than once
+			// The second lock commit will throw special_keys_api_failure error
+			if (err.code() == error_code_special_keys_api_failure) {
+				Optional<Value> errorMsg =
+				    co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin);
+				ASSERT(errorMsg.present());
+				std::string errorStr;
+				auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
+				auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
+				// special_key_space_management_api_error_msg schema validation
+				ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
+				ASSERT(valueObj["command"].get_str() == "lock" && !valueObj["retriable"].get_bool());
+				break;
+			} else if (err.code() == error_code_database_locked) {
+				// Database is already locked. This can happen if a previous attempt
+				// failed with unknown_result.
+				break;
+			} else {
+				co_await tx->onError(err);
 			}
 		}
 		TraceEvent(SevDebug, "DatabaseLocked").log();
 		// if database locked, fdb read should get database_locked error
 		tx->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-				RangeResult res = wait(tx->getRange(normalKeys, 1));
+				RangeResult res = co_await tx->getRange(normalKeys, 1);
 			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw;
-				if (e.code() == error_code_grv_proxy_memory_limit_exceeded ||
-				    e.code() == error_code_batch_transaction_throttled) {
-					wait(tx->onError(e));
-				} else {
-					ASSERT(e.code() == error_code_database_locked);
-					break;
+				err = e;
+			}
+			if (!err.isValid()) {
+				ASSERT(false);
+			}
+			if (err.code() == error_code_actor_cancelled)
+				throw err;
+			if (err.code() == error_code_grv_proxy_memory_limit_exceeded ||
+			    err.code() == error_code_batch_transaction_throttled) {
+				if (err.isValid()) {
+					co_await tx->onError(err);
 				}
+			} else {
+				ASSERT(err.code() == error_code_database_locked);
+				break;
 			}
 		}
 		// make sure we unlock the database
 		// unlock is idempotent, thus we can commit many times until successful
 		tx->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 				// unlock the database
 				tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("lock"));
-				wait(tx->commit());
+				co_await tx->commit();
 				TraceEvent(SevDebug, "DatabaseUnlocked").log();
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevDebug, "DatabaseUnlockFailure").error(e);
-				ASSERT(e.code() != error_code_database_locked);
-				wait(tx->onError(e));
+				err = e;
 			}
+			TraceEvent(SevDebug, "DatabaseUnlockFailure").error(err);
+			ASSERT(err.code() != error_code_database_locked);
+			co_await tx->onError(err);
 		}
 
 		tx->reset();
-		loop {
+		while (true) {
+			Error err;
 			try {
 				// read should be successful
 				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-				RangeResult res = wait(tx->getRange(normalKeys, 1));
+				RangeResult res = co_await tx->getRange(normalKeys, 1);
 				break;
 			} catch (Error& e) {
-				wait(tx->onError(e));
+				err = e;
 			}
+			co_await tx->onError(err);
 		}
 
 		// test consistencycheck which only used by ConsistencyCheck Workload
@@ -453,13 +483,14 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 		// no existing workloads can modify the key
 		tx->reset();
 		{
+			Error err;
 			try {
 				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> val1 = wait(tx->get(fdbShouldConsistencyCheckBeSuspended));
-				state bool ccSuspendSetting =
+				Optional<Value> val1 = co_await tx->get(fdbShouldConsistencyCheckBeSuspended);
+				bool ccSuspendSetting =
 				    val1.present() ? BinaryReader::fromStringRef<bool>(val1.get(), Unversioned()) : false;
 				Optional<Value> val2 =
-				    wait(tx->get(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck")));
+				    co_await tx->get(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck"));
 				// Make sure the read result from special key consistency with the system key
 				ASSERT(ccSuspendSetting ? val2.present() : !val2.present());
 				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
@@ -467,127 +498,148 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload {
 				ASSERT(!ccSuspendSetting);
 				// Disable consistencycheck
 				tx->set(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck"), ValueRef());
-				wait(tx->commit());
+				co_await tx->commit();
 				tx->reset();
 				// Read system key to make sure it is disabled
 				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> val3 = wait(tx->get(fdbShouldConsistencyCheckBeSuspended));
+				Optional<Value> val3 = co_await tx->get(fdbShouldConsistencyCheckBeSuspended);
 				bool ccSuspendSetting2 =
 				    val3.present() ? BinaryReader::fromStringRef<bool>(val3.get(), Unversioned()) : false;
 				ASSERT(ccSuspendSetting2);
 				tx->reset();
 			} catch (Error& e) {
-				wait(tx->onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tx->onError(err);
 			}
 		}
 		// make sure we enable consistencycheck by the end
 		{
-			loop {
+			while (true) {
+				Error err;
 				try {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 					tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck"));
-					wait(tx->commit());
+					co_await tx->commit();
 					tx->reset();
 					break;
 				} catch (Error& e) {
-					wait(tx->onError(e));
+					err = e;
 				}
+				co_await tx->onError(err);
 			}
 		}
 		// coordinators
 		// test read, makes sure it's the same as reading from coordinatorsKey
-		loop {
-			try {
-				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				Optional<Value> res = wait(tx->get(coordinatorsKey));
-				ASSERT(res.present()); // Otherwise, database is in a bad state
-				state ClusterConnectionString cs(res.get().toString());
-				Optional<Value> coordinator_processes_key = wait(
-				    tx->get("processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators"))));
-				ASSERT(coordinator_processes_key.present());
-				state std::vector<std::string> process_addresses;
-				boost::split(
-				    process_addresses, coordinator_processes_key.get().toString(), [](char c) { return c == ','; });
-				ASSERT(process_addresses.size() == cs.coords.size() + cs.hostnames.size());
-				// compare the coordinator process network addresses one by one
-				std::vector<NetworkAddress> coordinators = wait(cs.tryResolveHostnames());
-				for (const auto& network_address : coordinators) {
-					ASSERT(std::find(process_addresses.begin(), process_addresses.end(), network_address.toString()) !=
-					       process_addresses.end());
+		while (true) {
+			{
+				Error err;
+				try {
+					tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+					Optional<Value> res = co_await tx->get(coordinatorsKey);
+					ASSERT(res.present()); // Otherwise, database is in a bad state
+					ClusterConnectionString cs(res.get().toString());
+					Optional<Value> coordinator_processes_key = co_await tx->get(
+					    "processes"_sr.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("coordinators")));
+					ASSERT(coordinator_processes_key.present());
+					std::vector<std::string> process_addresses;
+					boost::split(
+					    process_addresses, coordinator_processes_key.get().toString(), [](char c) { return c == ','; });
+					ASSERT(process_addresses.size() == cs.coords.size() + cs.hostnames.size());
+					// compare the coordinator process network addresses one by one
+					std::vector<NetworkAddress> coordinators = co_await cs.tryResolveHostnames();
+					for (const auto& network_address : coordinators) {
+						ASSERT(std::find(process_addresses.begin(),
+						                 process_addresses.end(),
+						                 network_address.toString()) != process_addresses.end());
+					}
+					tx->reset();
+					break;
+				} catch (Error& e) {
+					err = e;
 				}
-				tx->reset();
-				break;
-			} catch (Error& e) {
-				wait(tx->onError(e));
+				co_await tx->onError(err);
 			}
-		}
-		// advanceversion
+		} // advanceversion
+		Error advanceVersionErr;
 		try {
 			tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-			Version v1 = wait(tx->getReadVersion());
+			Version v1 = co_await tx->getReadVersion();
 			TraceEvent(SevDebug, "InitialReadVersion").detail("Version", v1);
-			state Version v2 = 2 * v1;
-			loop {
-				try {
-					// loop until the grv is larger than the set version
-					Version v3 = wait(tx->getReadVersion());
-					if (v3 > v2) {
-						TraceEvent(SevDebug, "AdvanceVersionSuccess").detail("Version", v3);
-						break;
+			Version v2 = 2 * v1;
+			while (true) {
+				{
+					Error err;
+					try {
+						// loop until the grv is larger than the set version
+						Version v3 = co_await tx->getReadVersion();
+						if (v3 > v2) {
+							TraceEvent(SevDebug, "AdvanceVersionSuccess").detail("Version", v3);
+							break;
+						}
+						tx->setOption(FDBTransactionOptions::RAW_ACCESS);
+						tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+						// force the cluster to recover at v2
+						tx->set(SpecialKeySpace::getManagementApiCommandPrefix("advanceversion"), std::to_string(v2));
+						co_await tx->commit();
+						ASSERT(false); // Should fail with commit_unknown_result
+					} catch (Error& e) {
+						err = e;
+					}
+					TraceEvent(SevDebug, "AdvanceVersionCommitFailure").error(err);
+					if (err.isValid()) {
+						co_await tx->onError(err);
 					}
-					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					// force the cluster to recover at v2
-					tx->set(SpecialKeySpace::getManagementApiCommandPrefix("advanceversion"), std::to_string(v2));
-					wait(tx->commit());
-					ASSERT(false); // Should fail with commit_unknown_result
-				} catch (Error& e) {
-					TraceEvent(SevDebug, "AdvanceVersionCommitFailure").error(e);
-					wait(tx->onError(e));
 				}
 			}
 			tx->reset();
 		} catch (Error& e) {
-			wait(tx->onError(e));
+			advanceVersionErr = e;
+		}
+		if (advanceVersionErr.isValid()) {
+			co_await tx->onError(advanceVersionErr);
 		}
 		// make sure when we change dd related special keys, we grab the two system keys,
 		// i.e. moveKeysLockOwnerKey and moveKeysLockWriteKey
 		{
-			state Reference<ReadYourWritesTransaction> tr1(new ReadYourWritesTransaction(cx));
-			state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
-			loop {
+			Reference<ReadYourWritesTransaction> tr1(new ReadYourWritesTransaction(cx));
+			Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
+			while (true) {
+				Error err;
 				try {
 					tr1->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tr1->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 					tr2->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 
-					Version readVersion = wait(tr1->getReadVersion());
+					Version readVersion = co_await tr1->getReadVersion();
 					tr2->setVersion(readVersion);
 					KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
 					tr1->set("mode"_sr.withPrefix(ddPrefix), "1"_sr);
-					wait(tr1->commit());
+					co_await tr1->commit();
 					// randomly read the moveKeysLockOwnerKey/moveKeysLockWriteKey
 					// both of them should be grabbed when changing dd mode
-					wait(success(
-					    tr2->get(deterministicRandom()->coinflip() ? moveKeysLockOwnerKey : moveKeysLockWriteKey)));
+					co_await tr2->get(deterministicRandom()->coinflip() ? moveKeysLockOwnerKey : moveKeysLockWriteKey);
 					// tr2 should never succeed, just write to a key to make it not a read-only transaction
 					tr2->addWriteConflictRange(singleKeyRange(""_sr));
-					wait(tr2->commit());
+					co_await tr2->commit();
 					ASSERT(false); // commit should always fail due to conflict
 				} catch (Error& e) {
-					if (e.code() != error_code_not_committed) {
-						// when buggify is enabled, it's possible we get other retriable errors
-						wait(tr2->onError(e));
-						tr1->reset();
-					} else {
-						// loop until we get conflict error
-						break;
+					err = e;
+				}
+				if (err.code() != error_code_not_committed) {
+					// when buggify is enabled, it's possible we get other retriable errors
+					if (err.isValid()) {
+						co_await tr2->onError(err);
 					}
+					tr1->reset();
+				} else {
+					// loop until we get conflict error
+					break;
 				}
 			}
 		}
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/StatusWorkload.actor.cpp b/fdbserver/workloads/StatusWorkload.cpp
similarity index 79%
rename from fdbserver/workloads/StatusWorkload.actor.cpp
rename to fdbserver/workloads/StatusWorkload.cpp
index 2f3f035afe6..09d195fcae7 100644
--- a/fdbserver/workloads/StatusWorkload.actor.cpp
+++ b/fdbserver/workloads/StatusWorkload.cpp
@@ -1,5 +1,5 @@
 /*
- * StatusWorkload.actor.cpp
+ * StatusWorkload.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,33 +19,34 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbclient/StatusClient.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/Schemas.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/ManagementAPI.h"
 
 struct StatusWorkload : TestWorkload {
 	static constexpr auto NAME = "Status";
 
-	double testDuration, requestsPerSecond;
+	double testDuration, requestsPerSecond, maxAcceptableStatusLatency;
 	bool enableLatencyBands;
 
 	Future<Void> latencyBandActor;
 
 	PerfIntCounter requests, replies, errors, totalSize;
+	double worstLatency = 0;
 	Optional<StatusObject> parsedSchema;
 
-	StatusWorkload(WorkloadContext const& wcx)
+	explicit StatusWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), requests("Status requests issued"), replies("Status replies received"),
 	    errors("Status Errors"), totalSize("Status reply size sum") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		requestsPerSecond = getOption(options, "requestsPerSecond"_sr, 0.5);
+		maxAcceptableStatusLatency = getOption(options, "maxAcceptableStatusLatency"_sr, 0.0);
 		enableLatencyBands = getOption(options, "enableLatencyBands"_sr, deterministicRandom()->random01() < 0.5);
 		auto statusSchemaStr = getOption(options, "schema"_sr, JSONSchemas::statusSchema);
-		if (statusSchemaStr.size()) {
+		if (!statusSchemaStr.empty()) {
 			json_spirit::mValue schema = readJSONStrictly(statusSchemaStr.toString());
 			parsedSchema = schema.get_obj();
 
@@ -67,7 +68,17 @@ struct StatusWorkload : TestWorkload {
 
 		return success(timeout(fetcher(cx, this), testDuration));
 	}
-	Future<bool> check(Database const& cx) override { return errors.getValue() == 0; }
+	Future<bool> check(Database const& cx) override {
+		if (errors.getValue() != 0)
+			return false;
+		if (maxAcceptableStatusLatency > 0 && worstLatency > maxAcceptableStatusLatency) {
+			TraceEvent(SevError, "StatusLatencyExceeded")
+			    .detail("WorstLatency", worstLatency)
+			    .detail("MaxAcceptable", maxAcceptableStatusLatency);
+			return false;
+		}
+		return true;
+	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
 		if (clientId != 0)
@@ -78,6 +89,7 @@ struct StatusWorkload : TestWorkload {
 		m.emplace_back(
 		    "Average Reply Size", replies.getValue() ? totalSize.getValue() / replies.getValue() : 0, Averaged::False);
 		m.push_back(errors.getMetric());
+		m.emplace_back("Worst Latency", worstLatency, Averaged::True);
 	}
 
 	static void schemaCoverageRequirements(StatusObject const& schema, std::string schema_path = std::string()) {
@@ -87,11 +99,11 @@ struct StatusWorkload : TestWorkload {
 
 				schemaCoverage(spath, false);
 
-				if (skv.second.type() == json_spirit::array_type && skv.second.get_array().size()) {
+				if (skv.second.type() == json_spirit::array_type && !skv.second.get_array().empty()) {
 					if (skv.second.get_array()[0].type() != json_spirit::str_type)
 						schemaCoverageRequirements(skv.second.get_array()[0].get_obj(), spath + "[0]");
 				} else if (skv.second.type() == json_spirit::obj_type) {
-					if (skv.second.get_obj().count("$enum")) {
+					if (skv.second.get_obj().contains("$enum")) {
 						for (auto& enum_item : skv.second.get_obj().at("$enum").get_array())
 							schemaCoverage(spath + ".$enum." + enum_item.get_str(), false);
 					} else
@@ -127,10 +139,11 @@ struct StatusWorkload : TestWorkload {
 		return result + "]";
 	}
 
-	ACTOR Future<Void> configureLatencyBands(StatusWorkload* self, Database cx) {
-		loop {
-			state Transaction tr(cx);
-			loop {
+	Future<Void> configureLatencyBands(StatusWorkload* self, Database cx) {
+		while (true) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
@@ -153,39 +166,41 @@ struct StatusWorkload : TestWorkload {
 					    "}";
 
 					tr.set(latencyBandConfigKey, ValueRef(config));
-					wait(tr.commit());
+					co_await tr.commit();
 					tr.reset();
 
 					if (deterministicRandom()->random01() < 0.3) {
-						return Void();
+						co_return;
 					}
 
-					wait(delay(deterministicRandom()->random01() * 120));
+					co_await delay(deterministicRandom()->random01() * 120);
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
+				}
+				if (err.isValid()) {
+					co_await tr.onError(err);
 				}
 			}
 		}
 	}
 
-	ACTOR Future<Void> fetcher(Database cx, StatusWorkload* self) {
-		state double lastTime = now();
+	Future<Void> fetcher(Database cx, StatusWorkload* self) {
+		double lastTime = now();
 
-		loop {
-			wait(poisson(&lastTime, 1.0 / self->requestsPerSecond));
+		while (true) {
+			co_await poisson(&lastTime, 1.0 / self->requestsPerSecond);
 			try {
 				// Since we count the requests that start, we could potentially never really hear back?
 				++self->requests;
-				state double issued = now();
-				StatusObject result = wait(StatusClient::statusFetcher(cx));
+				double issued = now();
+				StatusObject result = co_await StatusClient::statusFetcher(cx);
 				++self->replies;
 				BinaryWriter br(AssumeVersion(g_network->protocolVersion()));
 				save(br, result);
 				self->totalSize += br.getLength();
-				TraceEvent("StatusWorkloadReply")
-				    .detail("ReplySize", br.getLength())
-				    .detail("Latency",
-				            now() - issued); //.detail("Reply", json_spirit::write_string(json_spirit::mValue(result)));
+				double latency = now() - issued;
+				self->worstLatency = std::max(self->worstLatency, latency);
+				TraceEvent("StatusWorkloadReply").detail("ReplySize", br.getLength()).detail("Latency", latency);
 				std::string errorStr;
 				if (self->parsedSchema.present() &&
 				    !schemaMatch(self->parsedSchema.get(), result, errorStr, SevError, true)) {
diff --git a/fdbserver/workloads/StorageCorruption.actor.cpp b/fdbserver/workloads/StorageCorruption.cpp
similarity index 74%
rename from fdbserver/workloads/StorageCorruption.actor.cpp
rename to fdbserver/workloads/StorageCorruption.cpp
index 6587c267701..e5c35de2144 100644
--- a/fdbserver/workloads/StorageCorruption.actor.cpp
+++ b/fdbserver/workloads/StorageCorruption.cpp
@@ -1,5 +1,5 @@
 /*
- * StorageCorruptionBug.h
+ * StorageCorruption.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/StorageCorruptionBug.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/storageserver/StorageCorruptionBug.h"
+#include "fdbclient/ManagementAPI.h"
 #include "flow/ProcessEvents.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 namespace {
 
 struct StorageCorruptionWorkload : TestWorkload {
@@ -34,7 +32,7 @@ struct StorageCorruptionWorkload : TestWorkload {
 	SimBugInjector bugInjector;
 	double testDuration;
 
-	StorageCorruptionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit StorageCorruptionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		bugInjector.enable();
 		bug = bugInjector.enable<StorageCorruptionBug>(StorageCorruptionBugID());
 		bugInjector.disable();
@@ -44,28 +42,27 @@ struct StorageCorruptionWorkload : TestWorkload {
 
 	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
 
-	ACTOR static Future<Void> _start(Self* self, Database cx) {
-		wait(success(setDDMode(cx, 0)));
-		self->bugInjector.enable();
-		wait(delay(self->testDuration));
-		self->bug->corruptionProbability = 0.0;
-		TraceEvent("CorruptionInjections").detail("NumCorruptions", self->bug->numHits()).log();
-		self->bugInjector.disable();
+	Future<Void> _start(Database cx) {
+		co_await setDDMode(cx, 0);
+		bugInjector.enable();
+		co_await delay(testDuration);
+		bug->corruptionProbability = 0.0;
+		TraceEvent("CorruptionInjections").detail("NumCorruptions", bug->numHits()).log();
+		bugInjector.disable();
 		ProcessEvents::uncancellableEvent("ConsistencyCheckFailure"_sr,
 		                                  [](StringRef, std::any const& data, Error const&) {
 			                                  if (std::any_cast<BaseTraceEvent*>(data)->getSeverity() == SevError) {
 				                                  TraceEvent("NegativeTestSuccess");
 			                                  }
 		                                  });
-		wait(success(setDDMode(cx, 1)));
-		return Void();
+		co_await setDDMode(cx, 1);
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0) {
 			return Void();
 		}
-		return _start(this, cx->clone());
+		return _start(cx->clone());
 	}
 	Future<bool> check(Database const& cx) override { return true; }
 
@@ -75,4 +72,4 @@ struct StorageCorruptionWorkload : TestWorkload {
 
 WorkloadFactory<StorageCorruptionWorkload> factory;
 
-} // namespace
\ No newline at end of file
+} // namespace
diff --git a/fdbserver/workloads/StorageServerCheckpointRestoreTest.actor.cpp b/fdbserver/workloads/StorageServerCheckpointRestoreTest.cpp
similarity index 60%
rename from fdbserver/workloads/StorageServerCheckpointRestoreTest.actor.cpp
rename to fdbserver/workloads/StorageServerCheckpointRestoreTest.cpp
index 78e7cbc0401..e8983364ece 100644
--- a/fdbserver/workloads/StorageServerCheckpointRestoreTest.actor.cpp
+++ b/fdbserver/workloads/StorageServerCheckpointRestoreTest.cpp
@@ -1,5 +1,5 @@
 /*
- *PhysicalShardMove.actor.cpp
+ * StorageServerCheckpointRestoreTest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,23 +18,21 @@
  * limitations under the License.
  */
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/ServerCheckpoint.actor.h"
-#include "fdbserver/MoveKeys.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/kvstore/IKeyValueStore.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/ServerCheckpoint.h"
+#include "fdbserver/core/MoveKeys.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
 #include <cstdint>
 #include <limits>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 namespace {
 std::string printValue(const ErrorOr<Optional<Value>>& value) {
 	if (value.isError()) {
@@ -49,7 +47,8 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 	const bool enabled;
 	bool pass;
 
-	SSCheckpointRestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
+	explicit SSCheckpointRestoreWorkload(WorkloadContext const& wcx)
+	  : TestWorkload(wcx), enabled(!clientId), pass(true) {}
 
 	void validationFailed(ErrorOr<Optional<Value>> expectedValue, ErrorOr<Optional<Value>> actualValue) {
 		TraceEvent(SevError, "TestFailed")
@@ -72,22 +71,22 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 		out.insert("Attrition");
 	}
 
-	ACTOR Future<Void> _start(SSCheckpointRestoreWorkload* self, Database cx) {
-		state Key key = "TestKey"_sr;
-		state Key endKey = "TestKey0"_sr;
-		state Value oldValue = "TestValue"_sr;
-		state KeyRange testRange = KeyRangeRef(key, endKey);
-		state std::vector<std::pair<KeyRange, CheckpointMetaData>> records;
+	Future<Void> _start(SSCheckpointRestoreWorkload* self, Database cx) {
+		Key key = "TestKey"_sr;
+		Key endKey = "TestKey0"_sr;
+		Value oldValue = "TestValue"_sr;
+		KeyRange testRange = KeyRangeRef(key, endKey);
+		std::vector<std::pair<KeyRange, CheckpointMetaData>> records;
 
 		TraceEvent("TestCheckpointRestoreBegin");
-		wait(success(setDDMode(cx, 0)));
-		state Version version = wait(self->writeAndVerify(self, cx, key, oldValue));
+		co_await setDDMode(cx, 0);
+		Version version = co_await self->writeAndVerify(self, cx, key, oldValue);
 
 		TraceEvent("TestCreatingCheckpoint").detail("Range", testRange);
 		// Create checkpoint.
-		state Transaction tr(cx);
-		state CheckpointFormat format = DataMoveRocksCF;
-		state UID dataMoveId =
+		Transaction tr(cx);
+		CheckpointFormat format = DataMoveRocksCF;
+		UID dataMoveId =
 		    newDataMoveId(deterministicRandom()->randomUInt64(),
 		                  AssignEmptyRange(false),
 		                  deterministicRandom()->random01() < SERVER_KNOBS->DD_PHYSICAL_SHARD_MOVE_PROBABILITY
@@ -95,17 +94,19 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 		                      : DataMoveType::LOGICAL,
 		                  DataMovementReason::TEAM_HEALTHY,
 		                  UnassignShard(false));
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				wait(createCheckpoint(&tr, { testRange }, format, dataMoveId));
-				wait(tr.commit());
+				co_await createCheckpoint(&tr, std::vector<KeyRange>(1, testRange), format, dataMoveId);
+				co_await tr.commit();
 				version = tr.getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		TraceEvent("TestCheckpointCreated")
@@ -114,11 +115,11 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 		    .detail("DataMoveID", dataMoveId);
 
 		// Fetch checkpoint meta data.
-		loop {
+		while (true) {
 			records.clear();
 			try {
-				wait(store(records,
-				           getCheckpointMetaData(cx, { testRange }, version, format, Optional<UID>(dataMoveId))));
+				records = co_await getCheckpointMetaData(
+				    cx, std::vector<KeyRange>(1, testRange), version, format, Optional<UID>(dataMoveId));
 				break;
 			} catch (Error& e) {
 				TraceEvent("TestFetchCheckpointMetadataError")
@@ -133,40 +134,41 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 
 		TraceEvent("TestCheckpointFetched").detail("Range", testRange).detail("Version", version);
 
-		state std::string pwd = platform::getWorkingDirectory();
-		state std::string folder = pwd + "/checkpoints";
+		std::string pwd = platform::getWorkingDirectory();
+		std::string folder = pwd + "/checkpoints";
 		platform::eraseDirectoryRecursive(folder);
 		ASSERT(platform::createDirectory(folder));
 
 		// Fetch checkpoint.
-		state std::vector<CheckpointMetaData> fetchedCheckpoints;
-		state std::vector<std::pair<KeyRange, CheckpointMetaData>>::iterator it = records.begin();
-		for (; it != records.end(); ++it) {
-			loop {
+		std::vector<CheckpointMetaData> fetchedCheckpoints;
+		for (auto it = records.begin(); it != records.end(); ++it) {
+			while (true) {
 				TraceEvent("TestFetchingCheckpoint").detail("Checkpoint", it->second.toString());
+				Error err;
 				try {
-					state CheckpointMetaData record = wait(fetchCheckpoint(cx, it->second, folder));
+					CheckpointMetaData record = co_await fetchCheckpoint(cx, it->second, folder);
 					fetchedCheckpoints.push_back(record);
 					TraceEvent("TestCheckpointFetched").detail("Checkpoint", record.toString());
 					break;
 				} catch (Error& e) {
-					TraceEvent("TestFetchCheckpointError")
-					    .errorUnsuppressed(e)
-					    .detail("Checkpoint", it->second.toString());
-					wait(delay(1));
+					err = e;
 				}
+				TraceEvent("TestFetchCheckpointError")
+				    .errorUnsuppressed(err)
+				    .detail("Checkpoint", it->second.toString());
+				co_await delay(1);
 			}
 		}
 
-		state std::string rocksDBTestDir = "rocksdb-kvstore-test-db";
+		std::string rocksDBTestDir = "rocksdb-kvstore-test-db";
 		platform::eraseDirectoryRecursive(rocksDBTestDir);
 
 		// Restore KVS.
-		state IKeyValueStore* kvStore = keyValueStoreRocksDB(
+		IKeyValueStore* kvStore = keyValueStoreRocksDB(
 		    rocksDBTestDir, deterministicRandom()->randomUniqueID(), KeyValueStoreType::SSD_ROCKSDB_V1);
-		wait(kvStore->init());
+		co_await kvStore->init();
 		try {
-			wait(kvStore->restore(fetchedCheckpoints));
+			co_await kvStore->restore(fetchedCheckpoints);
 		} catch (Error& e) {
 			TraceEvent(SevError, "TestRestoreCheckpointError").errorUnsuppressed(e);
 		}
@@ -174,18 +176,20 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 		// Compare the keyrange between the original database and the one restored from checkpoint.
 		// For now, it should have been a single key.
 		tr.reset();
-		state RangeResult res;
-		loop {
+		RangeResult res;
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(store(res, tr.getRange(KeyRangeRef(key, endKey), CLIENT_KNOBS->TOO_MANY)));
+				res = co_await tr.getRange(KeyRangeRef(key, endKey), CLIENT_KNOBS->TOO_MANY);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
-		RangeResult kvRange = wait(kvStore->readRange(testRange));
+		RangeResult kvRange = co_await kvStore->readRange(testRange);
 		ASSERT(res.size() == kvRange.size());
 		for (int i = 0; i < res.size(); ++i) {
 			ASSERT(res[i] == kvRange[i]);
@@ -193,64 +197,58 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 
 		Future<Void> close = kvStore->onClosed();
 		kvStore->dispose();
-		wait(close);
-
-		{
-			int ignore = wait(setDDMode(cx, 1));
-			(void)ignore;
-		}
-		return Void();
+		co_await close;
+		co_await setDDMode(cx, 1);
 	}
 
-	ACTOR Future<Void> readAndVerify(SSCheckpointRestoreWorkload* self,
-	                                 Database cx,
-	                                 Key key,
-	                                 ErrorOr<Optional<Value>> expectedValue) {
-		state Transaction tr(cx);
+	Future<Void> readAndVerify(SSCheckpointRestoreWorkload* self,
+	                           Database cx,
+	                           Key key,
+	                           ErrorOr<Optional<Value>> expectedValue) {
+		Transaction tr(cx);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
+				Optional<Value> res = co_await timeoutError(tr.get(key), 30.0);
 				const bool equal = !expectedValue.isError() && res == expectedValue.get();
 				if (!equal) {
 					self->validationFailed(expectedValue, ErrorOr<Optional<Value>>(res));
 				}
 				break;
 			} catch (Error& e) {
-				if (expectedValue.isError() && expectedValue.getError().code() == e.code()) {
-					break;
-				}
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (expectedValue.isError() && expectedValue.getError().code() == err.code()) {
+				break;
 			}
+			co_await tr.onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Version> writeAndVerify(SSCheckpointRestoreWorkload* self,
-	                                     Database cx,
-	                                     Key key,
-	                                     Optional<Value> value) {
-		state Transaction tr(cx);
-		state Version version;
-		loop {
+	Future<Version> writeAndVerify(SSCheckpointRestoreWorkload* self, Database cx, Key key, Optional<Value> value) {
+		Transaction tr(cx);
+		Version version{ 0 };
+		while (true) {
+			Error err;
 			try {
 				if (value.present()) {
 					tr.set(key, value.get());
 				} else {
 					tr.clear(key);
 				}
-				wait(timeoutError(tr.commit(), 30.0));
+				co_await timeoutError(tr.commit(), 30.0);
 				version = tr.getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
-		wait(self->readAndVerify(self, cx, key, value));
+		co_await self->readAndVerify(self, cx, key, value);
 
-		return version;
+		co_return version;
 	}
 
 	Future<bool> check(Database const& cx) override { return pass; }
@@ -258,4 +256,4 @@ struct SSCheckpointRestoreWorkload : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<SSCheckpointRestoreWorkload> SSCheckpointRestoreWorkloadFactory;
\ No newline at end of file
+WorkloadFactory<SSCheckpointRestoreWorkload> SSCheckpointRestoreWorkloadFactory;
diff --git a/fdbserver/workloads/Storefront.actor.cpp b/fdbserver/workloads/Storefront.cpp
similarity index 78%
rename from fdbserver/workloads/Storefront.actor.cpp
rename to fdbserver/workloads/Storefront.cpp
index bdc24e35bd9..a748905de75 100644
--- a/fdbserver/workloads/Storefront.actor.cpp
+++ b/fdbserver/workloads/Storefront.cpp
@@ -1,5 +1,5 @@
 /*
- * Storefront.actor.cpp
+ * Storefront.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,16 +19,15 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 // Storefront workload will maintain 2 tables: one for orders and one for items
 // Items table will have an entry for each item and the current total of "unfilled" orders
 // Orders table will have an entry for each order with a 16-character representation list of each item ordered
 
-typedef uint64_t orderID;
+using orderID = uint64_t;
 
 struct StorefrontWorkload : TestWorkload {
 	static constexpr auto NAME = "Storefront";
@@ -42,7 +41,7 @@ struct StorefrontWorkload : TestWorkload {
 	PerfIntCounter transactions, retries, spuriousCommitFailures;
 	PerfDoubleCounter totalLatency;
 
-	StorefrontWorkload(WorkloadContext const& wcx)
+	explicit StorefrontWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"),
 	    spuriousCommitFailures("Spurious Commit Failures"), totalLatency("Total Latency") {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -100,9 +99,9 @@ struct StorefrontWorkload : TestWorkload {
 
 	Standalone<KeyValueRef> operator()(int n) { return KeyValueRef(itemKey(n), itemValue(0)); }
 
-	ACTOR Future<Void> itemUpdater(Transaction* tr, StorefrontWorkload* self, int item, int quantity) {
-		state Key iKey = self->itemKey(item);
-		Optional<Value> val = wait(tr->get(iKey));
+	Future<Void> itemUpdater(Transaction* tr, StorefrontWorkload* self, int item, int quantity) {
+		Key iKey = self->itemKey(item);
+		Optional<Value> val = co_await tr->get(iKey);
 		if (!val.present()) {
 			TraceEvent(SevError, "StorefrontItemMissing")
 			    .detail("Key", printable(iKey))
@@ -113,43 +112,43 @@ struct StorefrontWorkload : TestWorkload {
 		}
 		int currentCount = valueToInt(val.get());
 		tr->set(iKey, self->itemValue(currentCount + quantity));
-		return Void();
 	}
 
-	ACTOR Future<Void> orderingClient(Database cx, StorefrontWorkload* self, double delay) {
-		state double lastTime = now();
+	Future<Void> orderingClient(Database cx, StorefrontWorkload* self, double delay) {
+		double lastTime = now();
 		try {
-			loop {
-				wait(poisson(&lastTime, delay));
-
-				state double tstart = now();
-				state int itemsToOrder = deterministicRandom()->randomInt(1, self->maxOrderSize);
-				state orderID id = deterministicRandom()->randomUniqueID().hash();
-				state Key orderKey = self->orderKey(id);
-				state Transaction tr(cx);
-				loop {
+			while (true) {
+				co_await poisson(&lastTime, delay);
+
+				double tstart = now();
+				int itemsToOrder = deterministicRandom()->randomInt(1, self->maxOrderSize);
+				orderID id = deterministicRandom()->randomUniqueID().hash();
+				Key orderKey = self->orderKey(id);
+				Transaction tr(cx);
+				while (true) {
+					Error err;
 					try {
-						Optional<Value> order = wait(tr.get(orderKey));
+						Optional<Value> order = co_await tr.get(orderKey);
 						if (order.present()) {
 							++self->spuriousCommitFailures;
 							break; // the order was already committed
 						}
 
 						// pick items
-						state std::map<int, int> items;
+						std::map<int, int> items;
 						for (int i = 0; i < itemsToOrder; i++)
 							items[deterministicRandom()->randomInt(0, self->itemCount)]++;
 
 						// create "value"
-						state std::vector<int> itemList;
+						std::vector<int> itemList;
 						std::map<int, int>::iterator it;
-						state std::vector<Future<Void>> updaters;
+						std::vector<Future<Void>> updaters;
 						for (it = items.begin(); it != items.end(); it++) {
 							for (int i = 0; i < it->second; i++)
 								itemList.push_back(it->first);
 							updaters.push_back(self->itemUpdater(&tr, self, it->first, it->second));
 						}
-						wait(waitForAll(updaters));
+						co_await waitForAll(updaters);
 						updaters.clear();
 
 						// set value for the order
@@ -157,12 +156,13 @@ struct StorefrontWorkload : TestWorkload {
 						wr << itemList;
 						tr.set(orderKey, wr.toValue());
 
-						wait(tr.commit());
+						co_await tr.commit();
 						self->orders[id] = items; // save this in a local list to test durability
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 					++self->retries;
 				}
 				++self->transactions;
@@ -174,17 +174,18 @@ struct StorefrontWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR Future<bool> orderChecker(Database cx, StorefrontWorkload* self, std::vector<orderID> ids) {
-		state Transaction tr(cx);
-		state int idx = 0;
-		loop {
+	Future<bool> orderChecker(Database cx, StorefrontWorkload* self, std::vector<orderID> ids) {
+		Transaction tr(cx);
+		int idx = 0;
+		while (true) {
+			Error err;
 			try {
 				for (; idx < ids.size(); idx++) {
-					state orderID id = ids[idx];
-					Optional<Value> val = wait(tr.get(self->orderKey(id)));
+					orderID id = ids[idx];
+					Optional<Value> val = co_await tr.get(self->orderKey(id));
 					if (!val.present()) {
 						TraceEvent(SevError, "TestFailure").detail("Reason", "OrderNotPresent").detail("OrderID", id);
-						return false;
+						co_return false;
 					}
 					std::vector<int> itemList;
 					std::map<int, int>::iterator it;
@@ -198,19 +199,20 @@ struct StorefrontWorkload : TestWorkload {
 						TraceEvent(SevError, "TestFailure")
 						    .detail("Reason", "OrderContentsMismatch")
 						    .detail("OrderID", id);
-						return false;
+						co_return false;
 					}
 				}
-				return true;
+				co_return true;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<bool> inventoryCheck(Database cx, StorefrontWorkload* self, bool ok) {
-		state std::vector<Future<bool>> checkers;
-		state std::map<orderID, std::map<int, int>>::iterator it(self->orders.begin());
+	Future<bool> inventoryCheck(Database cx, StorefrontWorkload* self, bool ok) {
+		std::vector<Future<bool>> checkers;
+		auto it = self->orders.begin();
 		while (it != self->orders.end()) {
 			for (int a = 0; a < self->actorCount && it != self->orders.end(); a++) {
 				std::vector<orderID> orderIDs;
@@ -220,7 +222,7 @@ struct StorefrontWorkload : TestWorkload {
 				}
 				checkers.push_back(self->orderChecker(cx->clone(), self, orderIDs));
 			}
-			wait(waitForAll(checkers));
+			co_await waitForAll(checkers);
 			for (int c = 0; c < checkers.size(); c++)
 				ok = ok && !checkers[c].isError() && checkers[c].isReady() && checkers[c].get();
 			checkers.clear();
@@ -228,7 +230,7 @@ struct StorefrontWorkload : TestWorkload {
 
 		// FIXME: match order table with inventory table
 
-		return ok;
+		co_return ok;
 	}
 };
 
diff --git a/fdbserver/workloads/StreamingRangeRead.actor.cpp b/fdbserver/workloads/StreamingRangeRead.cpp
similarity index 57%
rename from fdbserver/workloads/StreamingRangeRead.actor.cpp
rename to fdbserver/workloads/StreamingRangeRead.cpp
index ca614cd7313..eb6c9dbf557 100644
--- a/fdbserver/workloads/StreamingRangeRead.actor.cpp
+++ b/fdbserver/workloads/StreamingRangeRead.cpp
@@ -1,5 +1,5 @@
 /*
- * StreamingRangeRead.actor.cpp
+ * StreamingRangeRead.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,33 +20,30 @@
 
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "flow/serialize.h"
-#include <cstring>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR Future<Void> streamUsingGetRange(PromiseStream<RangeResult> results, Transaction* tr, KeyRange keys) {
-	state KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
-	state KeySelectorRef end = firstGreaterOrEqual(keys.end);
+Future<Void> streamUsingGetRange(PromiseStream<RangeResult> results, Transaction* tr, KeyRange keys) {
+	KeySelectorRef begin = firstGreaterOrEqual(keys.begin);
+	KeySelectorRef end = firstGreaterOrEqual(keys.end);
 
 	try {
-		loop {
+		while (true) {
 			GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, 1e6);
 			limits.minRows = 0;
-			state RangeResult rep = wait(tr->getRange(begin, end, limits, Snapshot::True));
+			RangeResult rep = co_await tr->getRange(begin, end, limits, Snapshot::True);
 
 			results.send(rep);
 
 			if (!rep.more) {
 				results.sendError(end_of_stream());
-				return Void();
+				co_return;
 			}
 
 			begin = rep.nextBeginKeySelector();
@@ -60,10 +57,10 @@ ACTOR Future<Void> streamUsingGetRange(PromiseStream<RangeResult> results, Trans
 	}
 }
 
-ACTOR Future<Void> convertStream(PromiseStream<RangeResult> input, PromiseStream<KeyValue> output) {
+Future<Void> convertStream(PromiseStream<RangeResult> input, PromiseStream<KeyValue> output) {
 	try {
-		loop {
-			RangeResult res = waitNext(input.getFuture());
+		while (true) {
+			RangeResult res = co_await input.getFuture();
 			for (auto& kv : res) {
 				output.send(kv);
 			}
@@ -74,7 +71,6 @@ ACTOR Future<Void> convertStream(PromiseStream<RangeResult> input, PromiseStream
 		}
 		output.sendError(e);
 	}
-	return Void();
 }
 
 struct StreamingRangeReadWorkload : KVWorkload {
@@ -82,7 +78,7 @@ struct StreamingRangeReadWorkload : KVWorkload {
 	double testDuration;
 	Future<Void> client;
 
-	StreamingRangeReadWorkload(WorkloadContext const& wcx) : KVWorkload(wcx) {
+	explicit StreamingRangeReadWorkload(WorkloadContext const& wcx) : KVWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 60.0);
 	}
 
@@ -90,7 +86,7 @@ struct StreamingRangeReadWorkload : KVWorkload {
 
 	Future<Void> setup(Database const& cx) override { return bulkSetup(cx, this, nodeCount, Promise<double>()); }
 	Future<Void> start(Database const& cx) override {
-		client = timeout(streamingClient(cx->clone(), this), testDuration, Void());
+		client = timeout(streamingClient(cx->clone()), testDuration, Void());
 		return delay(testDuration);
 	}
 
@@ -102,29 +98,35 @@ struct StreamingRangeReadWorkload : KVWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
 	// Reads the database using both the normal get range API and the streaming API and compares the results
-	ACTOR Future<Void> streamingClient(Database cx, StreamingRangeReadWorkload* self) {
-		state Transaction tr(cx);
-		state Key next;
-		state Future<Void> rateLimit = delay(0.01);
-		loop {
-			state PromiseStream<RangeResult> streamRaw;
-			state PromiseStream<RangeResult> compareRaw;
-			state PromiseStream<KeyValue> streamResults;
-			state PromiseStream<KeyValue> compareResults;
-
+	Future<Void> streamingClient(Database cx) {
+		Transaction tr(cx);
+		Key next;
+		Future<Void> rateLimit = delay(0.01);
+		while (true) {
+			PromiseStream<RangeResult> streamRaw;
+			PromiseStream<RangeResult> compareRaw;
+			PromiseStream<KeyValue> streamResults;
+			PromiseStream<KeyValue> compareResults;
+			Future<Void> compareConvert;
+			Future<Void> streamConvert;
+			Future<Void> compare;
+			Future<Void> stream;
+
+			Error err;
+			bool failed = false;
 			try {
-				state Future<Void> compareConvert = convertStream(compareRaw, compareResults);
-				state Future<Void> streamConvert = convertStream(streamRaw, streamResults);
-				state Future<Void> compare = streamUsingGetRange(compareRaw, &tr, KeyRangeRef(next, normalKeys.end));
-				state Future<Void> stream = tr.getRangeStream(streamRaw,
-				                                              KeySelector(firstGreaterOrEqual(next), next.arena()),
-				                                              KeySelector(firstGreaterOrEqual(normalKeys.end)),
-				                                              GetRangeLimits());
-				loop {
-					state Optional<KeyValue> cmp;
-					state Optional<KeyValue> res;
+				compareConvert = convertStream(compareRaw, compareResults);
+				streamConvert = convertStream(streamRaw, streamResults);
+				compare = streamUsingGetRange(compareRaw, &tr, KeyRangeRef(next, normalKeys.end));
+				stream = tr.getRangeStream(streamRaw,
+				                           KeySelector(firstGreaterOrEqual(next), next.arena()),
+				                           KeySelector(firstGreaterOrEqual(normalKeys.end)),
+				                           GetRangeLimits());
+				while (true) {
+					Optional<KeyValue> cmp;
+					Optional<KeyValue> res;
 					try {
-						KeyValue _cmp = waitNext(streamResults.getFuture());
+						KeyValue _cmp = co_await streamResults.getFuture();
 						cmp = _cmp;
 					} catch (Error& e) {
 						if (e.code() != error_code_end_of_stream) {
@@ -133,7 +135,7 @@ struct StreamingRangeReadWorkload : KVWorkload {
 						cmp = Optional<KeyValue>();
 					}
 					try {
-						KeyValue _res = waitNext(compareResults.getFuture());
+						KeyValue _res = co_await compareResults.getFuture();
 						res = _res;
 					} catch (Error& e) {
 						if (e.code() != error_code_end_of_stream) {
@@ -153,9 +155,13 @@ struct StreamingRangeReadWorkload : KVWorkload {
 					}
 				}
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+				failed = true;
+			}
+			if (failed) {
+				co_await tr.onError(err);
 			}
-			wait(rateLimit);
+			co_await rateLimit;
 			rateLimit = delay(0.01);
 		}
 	}
diff --git a/fdbserver/workloads/StreamingRead.actor.cpp b/fdbserver/workloads/StreamingRead.cpp
similarity index 78%
rename from fdbserver/workloads/StreamingRead.actor.cpp
rename to fdbserver/workloads/StreamingRead.cpp
index b303256f450..0d20dce6f7e 100644
--- a/fdbserver/workloads/StreamingRead.actor.cpp
+++ b/fdbserver/workloads/StreamingRead.cpp
@@ -1,5 +1,5 @@
 /*
- * StreamingRead.actor.cpp
+ * StreamingRead.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,10 +20,9 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 struct StreamingReadWorkload : TestWorkload {
 	static constexpr auto NAME = "StreamingRead";
@@ -39,7 +38,7 @@ struct StreamingReadWorkload : TestWorkload {
 	PerfIntCounter readValueBytes;
 	DDSketch<double> latencies;
 
-	StreamingReadWorkload(WorkloadContext const& wcx)
+	explicit StreamingReadWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), readKeys("Keys Read"), readValueBytes("Value Bytes Read"),
 	    latencies() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -96,24 +95,24 @@ struct StreamingReadWorkload : TestWorkload {
 
 	Standalone<KeyValueRef> operator()(int n) { return KeyValueRef(keyForIndex(n), constantValue); }
 
-	ACTOR Future<Void> streamingReadClient(Database cx, StreamingReadWorkload* self, int clientId, int actorId) {
-		state int minIndex = actorId * self->nodeCount / self->actorCount;
-		state int maxIndex = std::min((actorId + 1) * self->nodeCount / self->actorCount, self->nodeCount);
-		state int currentIndex = minIndex;
-
-		loop {
-			state double tstart = now();
-			state Transaction tr(cx);
-			state int rangeSize = (double)self->readsPerTransaction / self->rangesPerTransaction + 0.5;
-			state int range = 0;
-			loop {
-				state int thisRangeSize =
-				    (range < self->rangesPerTransaction - 1)
-				        ? rangeSize
-				        : self->readsPerTransaction - (self->rangesPerTransaction - 1) * rangeSize;
+	Future<Void> streamingReadClient(Database cx, StreamingReadWorkload* self, int clientId, int actorId) {
+		int minIndex = actorId * self->nodeCount / self->actorCount;
+		int maxIndex = std::min((actorId + 1) * self->nodeCount / self->actorCount, self->nodeCount);
+		int currentIndex = minIndex;
+
+		while (true) {
+			double tstart = now();
+			Transaction tr(cx);
+			int rangeSize = (double)self->readsPerTransaction / self->rangesPerTransaction + 0.5;
+			int range = 0;
+			while (true) {
+				int thisRangeSize = (range < self->rangesPerTransaction - 1)
+				                        ? rangeSize
+				                        : self->readsPerTransaction - (self->rangesPerTransaction - 1) * rangeSize;
 				if (self->readSequentially && thisRangeSize > maxIndex - minIndex)
 					thisRangeSize = maxIndex - minIndex;
-				loop {
+				while (true) {
+					Error err;
 					try {
 						if (!self->readSequentially)
 							currentIndex = deterministicRandom()->randomInt(0, self->nodeCount - thisRangeSize);
@@ -121,9 +120,9 @@ struct StreamingReadWorkload : TestWorkload {
 							currentIndex = minIndex;
 
 						RangeResult values =
-						    wait(tr.getRange(firstGreaterOrEqual(self->keyForIndex(currentIndex)),
-						                     firstGreaterOrEqual(self->keyForIndex(currentIndex + thisRangeSize)),
-						                     thisRangeSize));
+						    co_await tr.getRange(firstGreaterOrEqual(self->keyForIndex(currentIndex)),
+						                         firstGreaterOrEqual(self->keyForIndex(currentIndex + thisRangeSize)),
+						                         thisRangeSize);
 
 						for (int i = 0; i < values.size(); i++)
 							self->readValueBytes += values[i].value.size();
@@ -134,8 +133,9 @@ struct StreamingReadWorkload : TestWorkload {
 						self->readKeys += values.size();
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 				}
 
 				if (now() - tstart > 3)
diff --git a/fdbserver/workloads/SubmitBackup.actor.cpp b/fdbserver/workloads/SubmitBackup.cpp
similarity index 61%
rename from fdbserver/workloads/SubmitBackup.actor.cpp
rename to fdbserver/workloads/SubmitBackup.cpp
index 86f2d9b1895..65205325944 100644
--- a/fdbserver/workloads/SubmitBackup.actor.cpp
+++ b/fdbserver/workloads/SubmitBackup.cpp
@@ -1,5 +1,5 @@
 /*
- * SubmitBackup.actor.cpp
+ * SubmitBackup.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,16 +18,11 @@
  * limitations under the License.
  */
 
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbrpc/simulator.h"
-#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupAgent.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/BackupContainerFileSystem.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/tester/TestEncryptionUtils.h"
 
 struct SubmitBackupWorkload : TestWorkload {
 	static constexpr auto NAME = "SubmitBackup";
@@ -43,7 +38,7 @@ struct SubmitBackupWorkload : TestWorkload {
 	IncrementalBackupOnly incremental{ false };
 	Optional<std::string> encryptionKeyFileName;
 
-	SubmitBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit SubmitBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
 		tag = getOption(options, "tag"_sr, "default"_sr);
 		delayFor = getOption(options, "delayFor"_sr, 10.0);
@@ -57,38 +52,39 @@ struct SubmitBackupWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> _start(SubmitBackupWorkload* self, Database cx) {
-		wait(delay(self->delayFor));
-		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
+	Future<Void> _start(Database cx) {
+		co_await delay(delayFor);
+		Standalone<VectorRef<KeyRangeRef>> backupRanges;
 		addDefaultBackupRanges(backupRanges);
 
-		if (self->encryptionKeyFileName.present()) {
-			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		if (encryptionKeyFileName.present()) {
+			co_await BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get());
 		}
 
 		try {
-			wait(self->backupAgent.submitBackup(cx,
-			                                    self->backupDir,
-			                                    {},
-			                                    self->initSnapshotInterval,
-			                                    self->snapshotInterval,
-			                                    self->tag.toString(),
-			                                    backupRanges,
-			                                    self->stopWhenDone,
-			                                    UsePartitionedLog::False,
-			                                    self->incremental,
-			                                    self->encryptionKeyFileName));
+			co_await backupAgent.submitBackup(cx,
+			                                  backupDir,
+			                                  {},
+			                                  initSnapshotInterval,
+			                                  snapshotInterval,
+			                                  tag.toString(),
+			                                  backupRanges,
+			                                  stopWhenDone,
+			                                  MutationLogType::DEFAULT,
+			                                  incremental,
+			                                  encryptionKeyFileName,
+			                                  encryptionKeyFileName.present() ? DEFAULT_ENCRYPTION_BLOCK_SIZE : 0,
+			                                  0);
 		} catch (Error& e) {
 			TraceEvent("BackupSubmitError").error(e);
 			if (e.code() != error_code_backup_duplicate) {
 				throw;
 			}
 		}
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(this, cx); }
+	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(cx); }
 	Future<bool> check(Database const& cx) override { return true; }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
diff --git a/fdbserver/workloads/SuspendProcesses.actor.cpp b/fdbserver/workloads/SuspendProcesses.cpp
similarity index 73%
rename from fdbserver/workloads/SuspendProcesses.actor.cpp
rename to fdbserver/workloads/SuspendProcesses.cpp
index e9ccb57fcfa..56e863f6759 100644
--- a/fdbserver/workloads/SuspendProcesses.actor.cpp
+++ b/fdbserver/workloads/SuspendProcesses.cpp
@@ -1,5 +1,5 @@
 /*
- * SuspendProcesses.actor.cpp
+ * SuspendProcesses.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,13 +20,12 @@
 
 #include <boost/algorithm/string/predicate.hpp>
 
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/StatusClient.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
 
 struct SuspendProcessesWorkload : TestWorkload {
 	static constexpr auto NAME = "SuspendProcesses";
@@ -34,7 +33,7 @@ struct SuspendProcessesWorkload : TestWorkload {
 	double suspendTimeDuration;
 	double waitTimeDuration;
 
-	SuspendProcessesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit SuspendProcessesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		prefixSuspendProcesses = getOption(options, "prefixesSuspendProcesses"_sr, std::vector<std::string>());
 		waitTimeDuration = getOption(options, "waitTimeDuration"_sr, 0);
 		suspendTimeDuration = getOption(options, "suspendTimeDuration"_sr, 0);
@@ -42,22 +41,23 @@ struct SuspendProcessesWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	ACTOR Future<Void> _start(Database cx, SuspendProcessesWorkload* self) {
-		wait(delay(self->waitTimeDuration));
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+	Future<Void> _start(Database cx) {
+		co_await delay(waitTimeDuration);
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				RangeResult kvs =
-				    wait(tr.getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
-				                     CLIENT_KNOBS->TOO_MANY));
+				RangeResult kvs = co_await tr.getRange(
+				    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
+				    CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!kvs.more);
 				std::vector<Standalone<StringRef>> suspendProcessInterfaces;
 				for (auto it : kvs) {
 					auto ip_port = (it.key.endsWith(":tls"_sr) ? it.key.removeSuffix(":tls"_sr) : it.key)
 					                   .removePrefix("\xff\xff/worker_interfaces/"_sr);
-					for (auto& killProcess : self->prefixSuspendProcesses) {
+					for (auto& killProcess : prefixSuspendProcesses) {
 						if (boost::starts_with(ip_port.toString().c_str(), killProcess.c_str())) {
 							suspendProcessInterfaces.push_back(it.value);
 							TraceEvent("SuspendProcessSelectedProcess").detail("IpPort", printable(ip_port));
@@ -66,19 +66,20 @@ struct SuspendProcessesWorkload : TestWorkload {
 				}
 				for (auto& interf : suspendProcessInterfaces) {
 					BinaryReader::fromStringRef<ClientWorkerInterface>(interf, IncludeVersion())
-					    .reboot.send(RebootRequest(false, false, self->suspendTimeDuration));
+					    .reboot.send(RebootRequest(false, false, suspendTimeDuration));
 				}
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/TagThrottleApi.actor.cpp b/fdbserver/workloads/TagThrottleApi.cpp
similarity index 65%
rename from fdbserver/workloads/TagThrottleApi.actor.cpp
rename to fdbserver/workloads/TagThrottleApi.cpp
index 7019b6aba16..ccb5c22bba4 100644
--- a/fdbserver/workloads/TagThrottleApi.actor.cpp
+++ b/fdbserver/workloads/TagThrottleApi.cpp
@@ -1,5 +1,5 @@
 /*
- * TagThrottle.actor.cpp
+ * TagThrottleApi.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,12 +19,11 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/TagThrottle.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct TagThrottleApiWorkload : TestWorkload {
 	bool autoThrottleEnabled;
@@ -32,7 +31,7 @@ struct TagThrottleApiWorkload : TestWorkload {
 
 	constexpr static auto NAME = "TagThrottleApi";
 
-	TagThrottleApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit TagThrottleApiWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
 		autoThrottleEnabled = SERVER_KNOBS->AUTO_TAG_THROTTLING_ENABLED;
 	}
@@ -43,7 +42,7 @@ struct TagThrottleApiWorkload : TestWorkload {
 	}
 
 	Future<Void> start(Database const& cx) override {
-		if (SERVER_KNOBS->GLOBAL_TAG_THROTTLING || this->clientId != 0)
+		if (this->clientId != 0)
 			return Void();
 		return timeout(runThrottleApi(this, cx), testDuration, Void());
 	}
@@ -68,32 +67,32 @@ struct TagThrottleApiWorkload : TestWorkload {
 		return throttleType;
 	}
 
-	ACTOR Future<Void> throttleTag(
+	Future<Void> throttleTag(
 	    Database cx,
 	    std::map<std::pair<TransactionTag, TransactionPriority>, TagThrottleInfo>* manuallyThrottledTags) {
-		state TransactionTag tag =
+		TransactionTag tag =
 		    TransactionTagRef(deterministicRandom()->randomChoice(DatabaseContext::debugTransactionTagChoices));
-		state TransactionPriority priority = deterministicRandom()->randomChoice(allTransactionPriorities);
-		state double rate = deterministicRandom()->random01() * 20;
-		state double duration = 1 + deterministicRandom()->random01() * 19;
+		TransactionPriority priority = deterministicRandom()->randomChoice(allTransactionPriorities);
+		double rate = deterministicRandom()->random01() * 20;
+		double duration = 1 + deterministicRandom()->random01() * 19;
 
 		TagSet tagSet;
 		tagSet.addTag(tag);
 
 		try {
-			wait(ThrottleApi::throttleTags(cx.getReference(),
-			                               tagSet,
-			                               rate,
-			                               duration,
-			                               TagThrottleType::MANUAL,
-			                               priority,
-			                               Optional<double>(),
-			                               TagThrottledReason::MANUAL));
+			co_await ThrottleApi::throttleTags(cx.getReference(),
+			                                   tagSet,
+			                                   rate,
+			                                   duration,
+			                                   TagThrottleType::MANUAL,
+			                                   priority,
+			                                   Optional<double>(),
+			                                   TagThrottledReason::MANUAL);
 		} catch (Error& e) {
-			state Error err = e;
+			Error err = e;
 			if (e.code() == error_code_too_many_tag_throttles) {
 				ASSERT(manuallyThrottledTags->size() >= SERVER_KNOBS->MAX_MANUAL_THROTTLED_TRANSACTION_TAGS);
-				return Void();
+				co_return;
 			}
 
 			throw err;
@@ -103,25 +102,23 @@ struct TagThrottleApiWorkload : TestWorkload {
 		    std::make_pair(tag, priority),
 		    TagThrottleInfo(
 		        tag, TagThrottleType::MANUAL, priority, rate, now() + duration, duration, TagThrottledReason::MANUAL));
-
-		return Void();
 	}
 
-	ACTOR Future<Void> unthrottleTag(
+	Future<Void> unthrottleTag(
 	    Database cx,
 	    std::map<std::pair<TransactionTag, TransactionPriority>, TagThrottleInfo>* manuallyThrottledTags) {
-		state TransactionTag tag =
+		TransactionTag tag =
 		    TransactionTagRef(deterministicRandom()->randomChoice(DatabaseContext::debugTransactionTagChoices));
 		TagSet tagSet;
 		tagSet.addTag(tag);
 
-		state Optional<TagThrottleType> throttleType = TagThrottleApiWorkload::randomTagThrottleType();
+		Optional<TagThrottleType> throttleType = TagThrottleApiWorkload::randomTagThrottleType();
 		Optional<TransactionPriority> priority = deterministicRandom()->coinflip()
 		                                             ? Optional<TransactionPriority>()
 		                                             : deterministicRandom()->randomChoice(allTransactionPriorities);
 
-		state bool erased = false;
-		state double maxExpiration = 0;
+		bool erased = false;
+		double maxExpiration = 0;
 		if (!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) {
 			for (auto p : allTransactionPriorities) {
 				if (!priority.present() || priority.get() == p) {
@@ -135,23 +132,21 @@ struct TagThrottleApiWorkload : TestWorkload {
 			}
 		}
 
-		bool removed = wait(ThrottleApi::unthrottleTags(cx.getReference(), tagSet, throttleType, priority));
+		bool removed = co_await ThrottleApi::unthrottleTags(cx.getReference(), tagSet, throttleType, priority);
 		if (removed) {
 			ASSERT(erased || !throttleType.present() || throttleType.get() == TagThrottleType::AUTO);
 		} else {
 			ASSERT(maxExpiration < now());
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> getTags(
+	Future<Void> getTags(
 	    TagThrottleApiWorkload* self,
 	    Database cx,
 	    std::map<std::pair<TransactionTag, TransactionPriority>, TagThrottleInfo> const* manuallyThrottledTags) {
 
 		std::vector<TagThrottleInfo> tags =
-		    wait(ThrottleApi::getThrottledTags(cx.getReference(), CLIENT_KNOBS->TOO_MANY));
+		    co_await ThrottleApi::getThrottledTags(cx.getReference(), CLIENT_KNOBS->TOO_MANY);
 
 		int manualThrottledTags = 0;
 		int activeAutoThrottledTags = 0;
@@ -180,28 +175,26 @@ struct TagThrottleApiWorkload : TestWorkload {
 		}
 
 		ASSERT(manualThrottledTags >= minManualThrottledTags && manualThrottledTags <= maxManualThrottledTags);
-		return Void();
 	}
 
-	ACTOR Future<Void> getRecommendedTags(TagThrottleApiWorkload* self, Database cx) {
+	Future<Void> getRecommendedTags(TagThrottleApiWorkload* self, Database cx) {
 		std::vector<TagThrottleInfo> tags =
-		    wait(ThrottleApi::getRecommendedTags(cx.getReference(), CLIENT_KNOBS->TOO_MANY));
+		    co_await ThrottleApi::getRecommendedTags(cx.getReference(), CLIENT_KNOBS->TOO_MANY);
 
 		for (auto& tag : tags) {
 			ASSERT(tag.throttleType == TagThrottleType::AUTO);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> unthrottleTagGroup(
+	Future<Void> unthrottleTagGroup(
 	    Database cx,
 	    std::map<std::pair<TransactionTag, TransactionPriority>, TagThrottleInfo>* manuallyThrottledTags) {
-		state Optional<TagThrottleType> throttleType = TagThrottleApiWorkload::randomTagThrottleType();
-		state Optional<TransactionPriority> priority =
-		    deterministicRandom()->coinflip() ? Optional<TransactionPriority>()
-		                                      : deterministicRandom()->randomChoice(allTransactionPriorities);
+		Optional<TagThrottleType> throttleType = TagThrottleApiWorkload::randomTagThrottleType();
+		Optional<TransactionPriority> priority = deterministicRandom()->coinflip()
+		                                             ? Optional<TransactionPriority>()
+		                                             : deterministicRandom()->randomChoice(allTransactionPriorities);
 
-		bool unthrottled = wait(ThrottleApi::unthrottleAll(cx.getReference(), throttleType, priority));
+		bool unthrottled = co_await ThrottleApi::unthrottleAll(cx.getReference(), throttleType, priority);
 		if (!throttleType.present() || throttleType.get() == TagThrottleType::MANUAL) {
 			bool unthrottleExpected = false;
 			bool empty = manuallyThrottledTags->empty();
@@ -223,46 +216,42 @@ struct TagThrottleApiWorkload : TestWorkload {
 				ASSERT(unthrottled || !unthrottleExpected);
 			}
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> enableAutoThrottling(TagThrottleApiWorkload* self, Database cx) {
-		state Reference<DatabaseContext> db = cx.getReference();
+	Future<Void> enableAutoThrottling(TagThrottleApiWorkload* self, Database cx) {
+		Reference<DatabaseContext> db = cx.getReference();
 		if (deterministicRandom()->coinflip()) {
-			wait(ThrottleApi::enableAuto(db, true));
+			co_await ThrottleApi::enableAuto(db, true);
 			self->autoThrottleEnabled = true;
 			if (deterministicRandom()->coinflip()) {
-				wait(success(ThrottleApi::unthrottleAll(db, TagThrottleType::AUTO, Optional<TransactionPriority>())));
+				co_await ThrottleApi::unthrottleAll(db, TagThrottleType::AUTO, Optional<TransactionPriority>());
 			}
 		} else {
-			wait(ThrottleApi::enableAuto(db, false));
+			co_await ThrottleApi::enableAuto(db, false);
 			self->autoThrottleEnabled = false;
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<Void> runThrottleApi(TagThrottleApiWorkload* self, Database cx) {
-		state std::map<std::pair<TransactionTag, TransactionPriority>, TagThrottleInfo> manuallyThrottledTags;
-		loop {
+	Future<Void> runThrottleApi(TagThrottleApiWorkload* self, Database cx) {
+		std::map<std::pair<TransactionTag, TransactionPriority>, TagThrottleInfo> manuallyThrottledTags;
+		while (true) {
 			double delayTime = deterministicRandom()->random01() * 5;
-			wait(delay(delayTime));
+			co_await delay(delayTime);
 
-			state int action = deterministicRandom()->randomInt(0, 6);
+			int action = deterministicRandom()->randomInt(0, 6);
 
 			if (action == 0) {
-				wait(self->throttleTag(cx, &manuallyThrottledTags));
+				co_await self->throttleTag(cx, &manuallyThrottledTags);
 			} else if (action == 1) {
-				wait(self->unthrottleTag(cx, &manuallyThrottledTags));
+				co_await self->unthrottleTag(cx, &manuallyThrottledTags);
 			} else if (action == 2) {
-				wait(self->getTags(self, cx, &manuallyThrottledTags));
+				co_await self->getTags(self, cx, &manuallyThrottledTags);
 			} else if (action == 3) {
-				wait(self->unthrottleTagGroup(cx, &manuallyThrottledTags));
+				co_await self->unthrottleTagGroup(cx, &manuallyThrottledTags);
 			} else if (action == 4) {
-				wait(self->enableAutoThrottling(self, cx));
+				co_await self->enableAutoThrottling(self, cx);
 			} else if (action == 5) {
-				wait(self->getRecommendedTags(self, cx));
+				co_await self->getRecommendedTags(self, cx);
 			}
 		}
 	}
diff --git a/fdbserver/workloads/TargetedKill.actor.cpp b/fdbserver/workloads/TargetedKill.cpp
similarity index 85%
rename from fdbserver/workloads/TargetedKill.actor.cpp
rename to fdbserver/workloads/TargetedKill.cpp
index 525ca81d081..78ae9427c45 100644
--- a/fdbserver/workloads/TargetedKill.actor.cpp
+++ b/fdbserver/workloads/TargetedKill.cpp
@@ -1,5 +1,5 @@
 /*
- * TargetedKill.actor.cpp
+ * TargetedKill.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,15 +19,14 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/MasterInterface.h"
+#include "fdbserver/core/MasterInterface.h"
 #include "fdbclient/SystemData.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "fdbserver/QuietDatabase.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/core/QuietDatabase.h"
 
 struct TargetedKillWorkload : TestWorkload {
 	static constexpr auto NAME = "TargetedKill";
@@ -39,7 +38,7 @@ struct TargetedKillWorkload : TestWorkload {
 	bool reboot;
 	double suspendDuration;
 
-	TargetedKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit TargetedKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 		killAt = getOption(options, "killAt"_sr, 5.0);
 		reboot = getOption(options, "reboot"_sr, false);
@@ -93,16 +92,16 @@ struct TargetedKillWorkload : TestWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> assassin(Database cx, TargetedKillWorkload* self) {
-		wait(delay(self->killAt));
-		state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
-		state std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
+	Future<Void> assassin(Database cx, TargetedKillWorkload* self) {
+		co_await delay(self->killAt);
+		std::vector<StorageServerInterface> storageServers = co_await getStorageServers(cx);
+		std::vector<WorkerDetails> workers = co_await getWorkers(self->dbInfo);
 
-		state NetworkAddress machine;
-		state NetworkAddress ccAddr;
-		state int killed = 0;
-		state int s = 0;
-		state int j = 0;
+		NetworkAddress machine;
+		NetworkAddress ccAddr;
+		int killed = 0;
+		int s = 0;
+		int j = 0;
 		if (self->machineToKill == "master") {
 			machine = self->dbInfo->get().master.address();
 		} else if (self->machineToKill == "commitproxy") {
@@ -144,10 +143,10 @@ struct TargetedKillWorkload : TestWorkload {
 				machine = ssi.address();
 				if (machine != ccAddr) {
 					TraceEvent("IsolatedMark").detail("TargetedMachine", machine).detail("Role", self->machineToKill);
-					wait(self->killEndpoint(workers, machine, cx, self));
+					co_await self->killEndpoint(workers, machine, cx, self);
 					killed++;
 					if (killed == self->numKillStorages)
-						return Void();
+						co_return;
 				}
 				s = ++s % storageServers.size();
 			}
@@ -157,9 +156,7 @@ struct TargetedKillWorkload : TestWorkload {
 
 		TraceEvent("IsolatedMark").detail("TargetedMachine", machine).detail("Role", self->machineToKill);
 
-		wait(self->killEndpoint(workers, machine, cx, self));
-
-		return Void();
+		co_await self->killEndpoint(workers, machine, cx, self);
 	}
 };
 
diff --git a/fdbserver/workloads/TaskBucketCorrectness.actor.cpp b/fdbserver/workloads/TaskBucketCorrectness.cpp
similarity index 72%
rename from fdbserver/workloads/TaskBucketCorrectness.actor.cpp
rename to fdbserver/workloads/TaskBucketCorrectness.cpp
index 961915fc801..99bd4cc86bd 100644
--- a/fdbserver/workloads/TaskBucketCorrectness.actor.cpp
+++ b/fdbserver/workloads/TaskBucketCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * TaskBucketCorrectness.actor.cpp
+ * TaskBucketCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,8 +23,7 @@
 #include "fdbclient/Tuple.h"
 #include "fdbclient/TaskBucket.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct SayHelloTaskFunc : TaskFuncBase {
 	static StringRef name;
@@ -44,10 +43,10 @@ struct SayHelloTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 		// check task version
 		uint32_t taskVersion = task->getVersion();
 		if (taskVersion > SayHelloTaskFunc::version) {
@@ -58,14 +57,14 @@ struct SayHelloTaskFunc : TaskFuncBase {
 			    .detail("FuncVersion", v);
 		}
 
-		state Reference<TaskFuture> done = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		wait(taskBucket->finish(tr, task));
+		Reference<TaskFuture> done = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
+		co_await taskBucket->finish(tr, task);
 
 		if (BUGGIFY)
-			wait(delay(10));
+			co_await delay(10);
 
-		state Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
-		state Key value;
+		Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
+		Key value;
 		auto itor = task->params.find("name"_sr);
 		if (itor != task->params.end()) {
 			value = itor->value;
@@ -75,7 +74,7 @@ struct SayHelloTaskFunc : TaskFuncBase {
 		}
 
 		if (!task->params["chained"_sr].compare("false"_sr)) {
-			wait(done->set(tr, taskBucket));
+			co_await done->set(tr, taskBucket);
 		} else {
 			int subtaskCount = atoi(task->params["subtaskCount"_sr].toString().c_str());
 			int currTaskNumber = atoi(value.removePrefix("task_"_sr).toString().c_str());
@@ -84,7 +83,7 @@ struct SayHelloTaskFunc : TaskFuncBase {
 			    .detail("CurrTaskNumber", currTaskNumber);
 
 			if (currTaskNumber < subtaskCount - 1) {
-				state std::vector<Reference<TaskFuture>> vectorFuture;
+				std::vector<Reference<TaskFuture>> vectorFuture;
 				auto new_task = makeReference<Task>(SayHelloTaskFunc::name,
 				                                    SayHelloTaskFunc::version,
 				                                    StringRef(),
@@ -96,15 +95,13 @@ struct SayHelloTaskFunc : TaskFuncBase {
 				new_task->params[Task::reservedTaskParamKeyDone] = taskDone->key;
 				taskBucket->addTask(tr, new_task);
 				vectorFuture.push_back(taskDone);
-				wait(done->join(tr, taskBucket, vectorFuture));
+				co_await done->join(tr, taskBucket, vectorFuture);
 			} else {
-				wait(done->set(tr, taskBucket));
+				co_await done->set(tr, taskBucket);
 			}
 		}
 
 		tr->set(key, value);
-
-		return Void();
 	}
 };
 StringRef SayHelloTaskFunc::name = "SayHello"_sr;
@@ -128,12 +125,12 @@ struct SayHelloToEveryoneTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
 		Reference<TaskFuture> done = futureBucket->unpack(task->params[Task::reservedTaskParamKeyDone]);
-		state std::vector<Reference<TaskFuture>> vectorFuture;
+		std::vector<Reference<TaskFuture>> vectorFuture;
 
 		int subtaskCount = 1;
 		if (!task->params["chained"_sr].compare("false"_sr)) {
@@ -151,15 +148,13 @@ struct SayHelloToEveryoneTaskFunc : TaskFuncBase {
 			vectorFuture.push_back(taskDone);
 		}
 
-		wait(done->join(tr, taskBucket, vectorFuture));
-		wait(taskBucket->finish(tr, task));
+		co_await done->join(tr, taskBucket, vectorFuture);
+		co_await taskBucket->finish(tr, task);
 
 		Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
 		Value value = "Hello, Everyone!"_sr;
 		TraceEvent("TaskBucketCorrectnessSayHello").detail("SayHelloToEveryoneTaskFunc", printable(value));
 		tr->set(key, value);
-
-		return Void();
 	}
 };
 StringRef SayHelloToEveryoneTaskFunc::name = "SayHelloToEveryone"_sr;
@@ -183,18 +178,16 @@ struct SaidHelloTaskFunc : TaskFuncBase {
 		return _finish(tr, tb, fb, task);
 	};
 
-	ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
-	                                  Reference<TaskBucket> taskBucket,
-	                                  Reference<FutureBucket> futureBucket,
-	                                  Reference<Task> task) {
-		wait(taskBucket->finish(tr, task));
+	static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
+	                            Reference<TaskBucket> taskBucket,
+	                            Reference<FutureBucket> futureBucket,
+	                            Reference<Task> task) {
+		co_await taskBucket->finish(tr, task);
 
 		Key key = StringRef("Hello_" + deterministicRandom()->randomUniqueID().toString());
 		Value value = "Said hello to everyone!"_sr;
 		TraceEvent("TaskBucketCorrectnessSayHello").detail("SaidHelloTaskFunc", printable(value));
 		tr->set(key, value);
-
-		return Void();
 	}
 };
 StringRef SaidHelloTaskFunc::name = "SaidHello"_sr;
@@ -207,26 +200,22 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 	bool chained;
 	int subtaskCount;
 
-	TaskBucketCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit TaskBucketCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		chained = getOption(options, "chained"_sr, false);
 		subtaskCount = getOption(options, "subtaskCount"_sr, 20);
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
-
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> addInitTasks(Reference<ReadYourWritesTransaction> tr,
-	                                Reference<TaskBucket> taskBucket,
-	                                Reference<FutureBucket> futureBucket,
-	                                bool chained,
-	                                int subtaskCount) {
-		state Key addedInitKey("addedInitTasks"_sr);
-		Optional<Standalone<StringRef>> res = wait(tr->get(addedInitKey));
+	Future<Void> addInitTasks(Reference<ReadYourWritesTransaction> tr,
+	                          Reference<TaskBucket> taskBucket,
+	                          Reference<FutureBucket> futureBucket,
+	                          bool chained,
+	                          int subtaskCount) {
+		Key addedInitKey("addedInitTasks"_sr);
+		Optional<Standalone<StringRef>> res = co_await tr->get(addedInitKey);
 		if (res.present())
-			return Void();
+			co_return;
 		tr->set(addedInitKey, "true"_sr);
 
 		Reference<TaskFuture> allDone = futureBucket->future(tr);
@@ -240,81 +229,89 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 		taskBucket->addTask(tr, task);
 		auto taskDone = makeReference<Task>(
 		    SaidHelloTaskFunc::name, SaidHelloTaskFunc::version, StringRef(), deterministicRandom()->randomInt(0, 2));
-		wait(allDone->onSetAddTask(tr, taskBucket, taskDone));
-		return Void();
+		co_await allDone->onSetAddTask(tr, taskBucket, taskDone);
 	}
 
-	ACTOR Future<Void> _start(Database cx, TaskBucketCorrectnessWorkload* self) {
-		state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
-		state Subspace taskSubspace("backup-agent"_sr);
-		state Reference<TaskBucket> taskBucket(new TaskBucket(taskSubspace.get("tasks"_sr)));
-		state Reference<FutureBucket> futureBucket(new FutureBucket(taskSubspace.get("futures"_sr)));
+	Future<Void> start(Database const& cx) override {
+		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
+		Subspace taskSubspace("backup-agent"_sr);
+		Reference<TaskBucket> taskBucket(new TaskBucket(taskSubspace.get("tasks"_sr)));
+		Reference<FutureBucket> futureBucket(new FutureBucket(taskSubspace.get("futures"_sr)));
 
+		Error err;
 		try {
-			if (self->clientId == 0) {
+			if (clientId == 0) {
 				TraceEvent("TaskBucketCorrectness").detail("ClearingDb", "...");
-				wait(taskBucket->clear(cx));
+				co_await taskBucket->clear(cx);
 
 				TraceEvent("TaskBucketCorrectness").detail("AddingTasks", "...");
-				wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
-					return self->addInitTasks(tr, taskBucket, futureBucket, self->chained, self->subtaskCount);
-				}));
+				co_await runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
+					return addInitTasks(tr, taskBucket, futureBucket, chained, subtaskCount);
+				});
 
 				TraceEvent("TaskBucketCorrectness").detail("RunningTasks", "...");
 			}
 
-			loop {
-				try {
-					bool oneTaskDone = wait(taskBucket->doOne(cx, futureBucket));
-					if (!oneTaskDone) {
-						bool isEmpty = wait(taskBucket->isEmpty(cx));
-						if (isEmpty) {
-							wait(delay(5.0));
-							state bool isFutureEmpty = wait(futureBucket->isEmpty(cx));
-							if (isFutureEmpty)
-								break;
-							else {
-								wait(TaskBucket::debugPrintRange(
-								    cx, taskSubspace.key(), StringRef(format("client_%d", self->clientId))));
-								TraceEvent("TaskBucketCorrectness").detail("FutureIsNotEmpty", "...");
+			while (true) {
+				{
+					Error err;
+					try {
+						bool oneTaskDone = co_await taskBucket->doOne(cx, futureBucket);
+						if (!oneTaskDone) {
+							bool isEmpty = co_await taskBucket->isEmpty(cx);
+							if (isEmpty) {
+								co_await delay(5.0);
+								bool isFutureEmpty = co_await futureBucket->isEmpty(cx);
+								if (isFutureEmpty)
+									break;
+								else {
+									co_await TaskBucket::debugPrintRange(
+									    cx, taskSubspace.key(), StringRef(format("client_%d", clientId)));
+									TraceEvent("TaskBucketCorrectness").detail("FutureIsNotEmpty", "...");
+								}
+							} else {
+								co_await delay(1.0);
 							}
-						} else {
-							wait(delay(1.0));
 						}
+					} catch (Error& e) {
+						err = e;
+					}
+					if (err.isValid()) {
+						if (err.code() == error_code_timed_out)
+							TraceEvent(SevWarn, "TaskBucketCorrectness").error(err);
+						else
+							co_await tr->onError(err);
 					}
-				} catch (Error& e) {
-					if (e.code() == error_code_timed_out)
-						TraceEvent(SevWarn, "TaskBucketCorrectness").error(e);
-					else
-						wait(tr->onError(e));
 				}
 			}
 
-			if (self->clientId == 0) {
+			if (clientId == 0) {
 				TraceEvent("TaskBucketCorrectness").detail("NotTasksRemain", "...");
-				wait(TaskBucket::debugPrintRange(cx, StringRef(), StringRef()));
+				co_await TaskBucket::debugPrintRange(cx, StringRef(), StringRef());
 			}
 		} catch (Error& e) {
-			TraceEvent(SevError, "TaskBucketCorrectness").error(e);
-			wait(tr->onError(e));
+			err = e;
+		}
+		if (err.isValid()) {
+			TraceEvent(SevError, "TaskBucketCorrectness").error(err);
+			co_await tr->onError(err);
 		}
-
-		return Void();
 	}
 
-	ACTOR Future<bool> _check(Database cx, TaskBucketCorrectnessWorkload* self) {
-		bool ret = wait(runRYWTransaction(
-		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return self->checkSayHello(tr, self->subtaskCount); }));
-		return ret;
+	Future<bool> check(Database const& cx) override {
+		bool ret = co_await runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return checkSayHello(tr, subtaskCount); });
+		co_return ret;
 	}
 
-	ACTOR Future<bool> checkSayHello(Reference<ReadYourWritesTransaction> tr, int subTaskCount) {
-		state std::set<std::string> data = { "Hello, Everyone!", "Said hello to everyone!" };
+	Future<bool> checkSayHello(Reference<ReadYourWritesTransaction> tr, int subTaskCount) {
+		std::set<std::string> data = { "Hello, Everyone!", "Said hello to everyone!" };
 		for (int i = 0; i < subTaskCount; i++) {
 			data.insert(format("task_%d", i));
 		}
 
-		RangeResult values = wait(tr->getRange(KeyRangeRef("Hello_\x00"_sr, "Hello_\xff"_sr), CLIENT_KNOBS->TOO_MANY));
+		RangeResult values =
+		    co_await tr->getRange(KeyRangeRef("Hello_\x00"_sr, "Hello_\xff"_sr), CLIENT_KNOBS->TOO_MANY);
 		if (values.size() != data.size()) {
 			TraceEvent(SevError, "CheckSayHello")
 			    .detail("CountNotMatchIs", values.size())
@@ -322,19 +319,19 @@ struct TaskBucketCorrectnessWorkload : TestWorkload {
 			for (auto& s : values) {
 				TraceEvent("CheckSayHello").detail("Item", printable(s)).detail("Value", printable(s.value));
 			}
-			return false;
+			co_return false;
 		}
 
 		for (auto& s : values) {
 			// TraceEvent("CheckSayHello").detail("Item", printable(s)).detail("Value", printable(s.value));
 			data.erase(s.value.toString());
 		}
-		if (data.size() != 0) {
+		if (!data.empty()) {
 			TraceEvent(SevError, "CheckSayHello").detail("DataNotMatch", data.size());
-			return false;
+			co_return false;
 		}
 
-		return true;
+		co_return true;
 	}
 };
 
@@ -347,7 +344,7 @@ void print_subspace_key(const Subspace& subspace, int id) {
 TEST_CASE("/fdbclient/TaskBucket/Subspace") {
 	Subspace subspace_test;
 	print_subspace_key(subspace_test, 0);
-	ASSERT(subspace_test.key().toString() == "");
+	ASSERT(subspace_test.key().toString().empty());
 
 	Subspace subspace_test1("abc"_sr);
 	print_subspace_key(subspace_test1, 1);
diff --git a/fdbserver/workloads/ThreadSafety.actor.cpp b/fdbserver/workloads/ThreadSafety.cpp
similarity index 83%
rename from fdbserver/workloads/ThreadSafety.actor.cpp
rename to fdbserver/workloads/ThreadSafety.cpp
index 8018c6db36f..b78fe2ae914 100644
--- a/fdbserver/workloads/ThreadSafety.actor.cpp
+++ b/fdbserver/workloads/ThreadSafety.cpp
@@ -1,5 +1,5 @@
 /*
- * ThreadSafety.actor.cpp
+ * ThreadSafety.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,12 +20,11 @@
 
 #include "fdbrpc/simulator.h"
 #include "flow/DeterministicRandom.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "fdbclient/ThreadSafeTransaction.h"
 #include "fdbclient/MultiVersionTransaction.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct ThreadSafetyWorkload;
 
@@ -92,8 +91,8 @@ struct Barrier {
 private:
 	void fire() {
 		numReached = 0;
-		for (int i = 0; i < events.size(); ++i)
-			events[i]->set();
+		for (auto& event : events)
+			event->set();
 
 		events.clear();
 	}
@@ -122,7 +121,7 @@ struct ThreadSafetyWorkload : TestWorkload {
 
 	Reference<ITransaction> tr;
 
-	ThreadSafetyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), stopped(false) {
+	explicit ThreadSafetyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), stopped(false) {
 
 		threadsPerClient = getOption(options, "threadsPerClient"_sr, 3);
 		threadDuration = getOption(options, "threadDuration"_sr, 60.0);
@@ -138,52 +137,47 @@ struct ThreadSafetyWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
-	ACTOR Future<Void> _start(Database cx, ThreadSafetyWorkload* self) {
-		state std::vector<ThreadInfo*> threadInfo;
+	Future<Void> start(Database const& cx) override {
+		std::vector<ThreadInfo*> threadInfo;
 
 		Reference<IDatabase> dbRef =
-		    wait(unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx)));
-		self->db = dbRef;
+		    co_await unsafeThreadFutureToFuture(ThreadSafeDatabase::createFromExistingDatabase(cx));
+		db = dbRef;
 
 		if (deterministicRandom()->coinflip()) {
 			MultiVersionApi::api->selectApiVersion(cx->apiVersion.version());
-			self->db = MultiVersionDatabase::debugCreateFromExistingDatabase(dbRef);
+			db = MultiVersionDatabase::debugCreateFromExistingDatabase(dbRef);
 		}
 
-		state int i;
-		for (i = 0; i < self->threadsPerClient; ++i) {
-			threadInfo.push_back(new ThreadInfo(i, self));
-			g_network->startThread(self->threadStart, threadInfo[i]);
+		for (int i = 0; i < threadsPerClient; ++i) {
+			threadInfo.push_back(new ThreadInfo(i, this));
+			g_network->startThread(threadStart, threadInfo[i]);
 		}
 
-		wait(delay(self->threadDuration));
+		co_await delay(threadDuration);
 
 		// Signals the threads to stop
-		self->mutex.enter();
-		self->stopped = true;
-		self->mutex.leave();
+		mutex.enter();
+		stopped = true;
+		mutex.leave();
 
-		for (i = 0; i < threadInfo.size(); ++i) {
+		for (int i = 0; i < threadInfo.size(); ++i) {
 			try {
-				wait(threadInfo[i]->done.getFuture());
+				co_await threadInfo[i]->done.getFuture();
 			} catch (Error& e) {
-				self->success = false;
-				printf("Thread %d.%d failed: %s\n", self->clientId, i, e.name());
+				success = false;
+				printf("Thread %d.%d failed: %s\n", clientId, i, e.name());
 				TraceEvent(SevError, "ThreadSafety_ThreadFailed").error(e);
 			}
 
 			delete threadInfo[i];
 		}
-
-		return Void();
 	}
 
 	THREAD_FUNC threadStart(void* arg) {
 		ThreadInfo* info = (ThreadInfo*)arg;
 
-		Error error(error_code_success);
+		Error error;
 		try {
 			info->self->runTest(info);
 		} catch (Error& e) {
@@ -220,7 +214,7 @@ struct ThreadSafetyWorkload : TestWorkload {
 		}
 		mutex.leave();
 
-		loop {
+		while (true) {
 			// Perform a sequence of random operations
 			for (int i = 0; i < info->random.randomInt(1, 10); ++i) {
 				int operation = info->random.randomInt(0, 6);
diff --git a/fdbserver/workloads/Throttling.actor.cpp b/fdbserver/workloads/Throttling.cpp
similarity index 63%
rename from fdbserver/workloads/Throttling.actor.cpp
rename to fdbserver/workloads/Throttling.cpp
index 3c91c134805..6aa14a91d3b 100644
--- a/fdbserver/workloads/Throttling.actor.cpp
+++ b/fdbserver/workloads/Throttling.cpp
@@ -1,5 +1,5 @@
 /*
- * Throttling.actor.cpp
+ * Throttling.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,11 +20,10 @@
 
 #include <boost/lexical_cast.hpp>
 
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/Schemas.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last include
+#include "fdbserver/tester/workloads.h"
 
 struct TokenBucket {
 	static constexpr const double addTokensInterval = 0.1;
@@ -35,31 +34,31 @@ struct TokenBucket {
 	double bucketSize;
 	Future<Void> tokenAdderActor;
 
-	ACTOR static Future<Void> tokenAdder(TokenBucket* self) {
-		loop {
-			self->bucketSize = std::min(self->bucketSize + self->transactionRate * addTokensInterval, self->maxBurst);
+	Future<Void> tokenAdder() {
+		while (true) {
+			bucketSize = std::min(bucketSize + transactionRate * addTokensInterval, maxBurst);
 			if (deterministicRandom()->randomInt(0, 100) == 0)
 				TraceEvent("AddingTokensx100")
-				    .detail("BucketSize", self->bucketSize)
-				    .detail("TransactionRate", self->transactionRate);
-			wait(delay(addTokensInterval));
+				    .detail("BucketSize", bucketSize)
+				    .detail("TransactionRate", transactionRate);
+			co_await delay(addTokensInterval);
 		}
 	}
 
-	TokenBucket(double maxBurst = 1000) : transactionRate(0), maxBurst(maxBurst), bucketSize(maxBurst) {
-		tokenAdderActor = tokenAdder(this);
+	explicit TokenBucket(double maxBurst = 1000) : transactionRate(0), maxBurst(maxBurst), bucketSize(maxBurst) {
+		tokenAdderActor = tokenAdder();
 	}
 
-	ACTOR static Future<Void> startTransaction(TokenBucket* self) {
-		state double sleepTime = addTokensInterval;
-		loop {
-			if (self->bucketSize >= 1.0) {
-				--self->bucketSize;
-				return Void();
+	Future<Void> startTransaction() {
+		double sleepTime = addTokensInterval;
+		while (true) {
+			if (bucketSize >= 1.0) {
+				--bucketSize;
+				co_return;
 			}
 			if (deterministicRandom()->randomInt(0, 100) == 0)
 				TraceEvent("ThrottlingTransactionx100").detail("SleepTime", sleepTime);
-			wait(delay(sleepTime));
+			co_await delay(sleepTime);
 			sleepTime = std::min(sleepTime * 2, maxSleepTime);
 		}
 	}
@@ -82,7 +81,7 @@ struct ThrottlingWorkload : KVWorkload {
 
 	static constexpr auto NAME = "Throttling";
 
-	ThrottlingWorkload(WorkloadContext const& wcx) : KVWorkload(wcx), transactionsCommitted(0) {
+	explicit ThrottlingWorkload(WorkloadContext const& wcx) : KVWorkload(wcx), transactionsCommitted(0) {
 		testDuration = getOption(options, "testDuration"_sr, 60.0);
 		actorsPerClient = getOption(options, "actorsPerClient"_sr, 10);
 		writesPerTransaction = getOption(options, "writesPerTransaction"_sr, 10);
@@ -96,24 +95,23 @@ struct ThrottlingWorkload : KVWorkload {
 		return Standalone<StringRef>(format("Value/%d", deterministicRandom()->randomInt(0, 10e6)));
 	}
 
-	ACTOR static Future<Void> clientActor(Database cx, ThrottlingWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
+	Future<Void> clientActor(Database cx) {
+		ReadYourWritesTransaction tr(cx);
 
-		loop {
-			wait(TokenBucket::startTransaction(&self->tokenBucket));
+		while (true) {
+			co_await tokenBucket.startTransaction();
 			tr.reset();
 			try {
-				state int i;
-				for (i = 0; i < self->readsPerTransaction; ++i) {
-					state Optional<Value> value = wait(tr.get(self->getRandomKey()));
+				for (int i = 0; i < readsPerTransaction; ++i) {
+					Optional<Value> value = co_await tr.get(getRandomKey());
 				}
-				for (i = 0; i < self->writesPerTransaction; ++i) {
-					tr.set(self->getRandomKey(), getRandomValue());
+				for (int i = 0; i < writesPerTransaction; ++i) {
+					tr.set(getRandomKey(), getRandomValue());
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 				if (deterministicRandom()->randomInt(0, 1000) == 0)
 					TraceEvent("TransactionCommittedx1000").log();
-				++self->transactionsCommitted;
+				++transactionsCommitted;
 			} catch (Error& e) {
 				if (e.code() == error_code_actor_cancelled)
 					throw;
@@ -122,17 +120,16 @@ struct ThrottlingWorkload : KVWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> specialKeysActor(Database cx, ThrottlingWorkload* self) {
-		state ReadYourWritesTransaction tr(cx);
-		state json_spirit::mValue aggregateSchema =
-		    readJSONStrictly(JSONSchemas::aggregateHealthSchema.toString()).get_obj();
-		state json_spirit::mValue storageSchema =
-		    readJSONStrictly(JSONSchemas::storageHealthSchema.toString()).get_obj();
-		state json_spirit::mValue logSchema = readJSONStrictly(JSONSchemas::logHealthSchema.toString()).get_obj();
-		loop {
+	Future<Void> specialKeysActor(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		json_spirit::mValue aggregateSchema = readJSONStrictly(JSONSchemas::aggregateHealthSchema.toString()).get_obj();
+		json_spirit::mValue storageSchema = readJSONStrictly(JSONSchemas::storageHealthSchema.toString()).get_obj();
+		json_spirit::mValue logSchema = readJSONStrictly(JSONSchemas::logHealthSchema.toString()).get_obj();
+		while (true) {
+			Error err;
 			try {
 				RangeResult result =
-				    wait(tr.getRange(prefixRange("\xff\xff/metrics/health/"_sr), CLIENT_KNOBS->TOO_MANY));
+				    co_await tr.getRange(prefixRange("\xff\xff/metrics/health/"_sr), CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!result.more);
 				for (const auto& [k, v] : result) {
 					ASSERT(k.startsWith("\xff\xff/metrics/health/"_sr));
@@ -144,10 +141,10 @@ struct ThrottlingWorkload : KVWorkload {
 							TraceEvent(SevError, "AggregateHealthSchemaValidationFailed")
 							    .detail("ErrorStr", errorStr.c_str())
 							    .detail("JSON", json_spirit::write_string(json_spirit::mValue(v.toString())));
-							self->correctSpecialKeys = false;
+							correctSpecialKeys = false;
 						}
 						auto tpsLimit = valueObj.at("tps_limit").get_real();
-						self->tokenBucket.transactionRate = tpsLimit * self->throttlingMultiplier / self->clientCount;
+						tokenBucket.transactionRate = tpsLimit * throttlingMultiplier / clientCount;
 					} else if (k.removePrefix("\xff\xff/metrics/health/"_sr).startsWith("storage/"_sr)) {
 						CODE_PROBE(true, "Test storage health metrics schema");
 						UID::fromString(k.removePrefix("\xff\xff/metrics/health/storage/"_sr)
@@ -157,7 +154,7 @@ struct ThrottlingWorkload : KVWorkload {
 							TraceEvent(SevError, "StorageHealthSchemaValidationFailed")
 							    .detail("ErrorStr", errorStr.c_str())
 							    .detail("JSON", json_spirit::write_string(json_spirit::mValue(v.toString())));
-							self->correctSpecialKeys = false;
+							correctSpecialKeys = false;
 						}
 					} else if (k.removePrefix("\xff\xff/metrics/health/"_sr).startsWith("log/"_sr)) {
 						CODE_PROBE(true, "Test log health metrics schema");
@@ -168,32 +165,32 @@ struct ThrottlingWorkload : KVWorkload {
 							TraceEvent(SevError, "LogHealthSchemaValidationFailed")
 							    .detail("ErrorStr", errorStr.c_str())
 							    .detail("JSON", json_spirit::write_string(json_spirit::mValue(v.toString())));
-							self->correctSpecialKeys = false;
+							correctSpecialKeys = false;
 						}
 					} else {
 						ASSERT(false); // Unrecognized key
 					}
 				}
-				wait(delayJittered(5));
+				co_await delayJittered(5);
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
+			}
+			if (err.isValid()) {
+				co_await tr.onError(err);
 			}
 		}
 	}
 
-	ACTOR static Future<Void> _start(Database cx, ThrottlingWorkload* self) {
-		state std::vector<Future<Void>> clientActors;
-		state int actorId;
-		for (actorId = 0; actorId < self->actorsPerClient; ++actorId) {
-			clientActors.push_back(timeout(clientActor(cx, self), self->testDuration, Void()));
+	Future<Void> start(Database const& cx) override {
+		std::vector<Future<Void>> clientActors;
+		for (int actorId = 0; actorId < actorsPerClient; ++actorId) {
+			clientActors.push_back(timeout(clientActor(cx), testDuration, Void()));
 		}
-		clientActors.push_back(timeout(specialKeysActor(cx, self), self->testDuration, Void()));
-		clientActors.push_back(timeout(self->tokenBucket.tokenAdderActor, self->testDuration, Void()));
-		wait(delay(self->testDuration));
-		return Void();
+		clientActors.push_back(timeout(specialKeysActor(cx), testDuration, Void()));
+		clientActors.push_back(timeout(tokenBucket.tokenAdderActor, testDuration, Void()));
+		co_await delay(testDuration);
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	Future<bool> check(Database const& cx) override { return correctSpecialKeys; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
diff --git a/fdbserver/workloads/Throughput.actor.cpp b/fdbserver/workloads/Throughput.cpp
similarity index 76%
rename from fdbserver/workloads/Throughput.actor.cpp
rename to fdbserver/workloads/Throughput.cpp
index 2e2fd630b13..a58418fbfc1 100644
--- a/fdbserver/workloads/Throughput.actor.cpp
+++ b/fdbserver/workloads/Throughput.cpp
@@ -1,5 +1,5 @@
 /*
- * Throughput.actor.cpp
+ * Throughput.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,12 +20,11 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ActorCollection.h"
 #include "fdbrpc/Smoother.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct ITransactor : ReferenceCounted<ITransactor> {
 	struct Stats {
@@ -47,7 +46,7 @@ struct ITransactor : ReferenceCounted<ITransactor> {
 	};
 
 	virtual Future<Void> doTransaction(Database const&, Stats* stats) = 0;
-	virtual ~ITransactor() {}
+	virtual ~ITransactor() = default;
 };
 
 struct RWTransactor : ITransactor {
@@ -97,16 +96,16 @@ struct RWTransactor : ITransactor {
 		return rwTransaction(db, Reference<RWTransactor>::addRef(this), stats);
 	}
 
-	ACTOR static Future<Optional<Value>> getLatency(Future<Optional<Value>> f, double* t) {
-		Optional<Value> v = wait(f);
+	static Future<Optional<Value>> getLatency(Future<Optional<Value>> f, double* t) {
+		Optional<Value> v = co_await f;
 		*t += now();
-		return v;
+		co_return v;
 	}
 
-	ACTOR static Future<Void> rwTransaction(Database db, Reference<RWTransactor> self, Stats* stats) {
-		state std::vector<Key> keys;
-		state std::vector<Value> values;
-		state Transaction tr(db);
+	static Future<Void> rwTransaction(Database db, Reference<RWTransactor> self, Stats* stats) {
+		std::vector<Key> keys;
+		std::vector<Value> values;
+		Transaction tr(db);
 
 		for (int op = 0; op < self->reads || op < self->writes; op++)
 			keys.push_back(self->randomKey());
@@ -114,22 +113,23 @@ struct RWTransactor : ITransactor {
 		for (int op = 0; op < self->writes; op++)
 			values.push_back(self->randomValue());
 
-		loop {
+		while (true) {
+			Error err;
 			try {
-				state double t_start = now();
-				wait(success(tr.getReadVersion()));
-				state double t_rv = now();
-				state double rrLatency = -t_rv * self->reads;
+				double t_start = now();
+				co_await tr.getReadVersion();
+				double t_rv = now();
+				double rrLatency = -t_rv * self->reads;
 
-				state std::vector<Future<Optional<Value>>> reads;
+				std::vector<Future<Optional<Value>>> reads;
 				reads.reserve(self->reads);
 				for (int i = 0; i < self->reads; i++)
 					reads.push_back(getLatency(tr.get(keys[i]), &rrLatency));
-				wait(waitForAll(reads));
+				co_await waitForAll(reads);
 				for (int i = 0; i < self->writes; i++)
 					tr.set(keys[i], values[i]);
-				state double t_beforeCommit = now();
-				wait(tr.commit());
+				double t_beforeCommit = now();
+				co_await tr.commit();
 
 				stats->transactions++;
 				stats->reads += self->reads;
@@ -139,12 +139,11 @@ struct RWTransactor : ITransactor {
 				stats->rowReadLatency += rrLatency / self->reads;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
-				stats->retries++;
+				err = e;
 			}
+			co_await tr.onError(err);
+			stats->retries++;
 		}
-
-		return Void();
 	}
 };
 
@@ -196,7 +195,7 @@ struct IMeasurer : ReferenceCounted<IMeasurer> {
 		return *this;
 	} // allow copy operator for non-reference counted instances of subclasses
 
-	virtual ~IMeasurer() {}
+	virtual ~IMeasurer() = default;
 };
 
 struct MeasureSinglePeriod : IMeasurer {
@@ -260,7 +259,7 @@ struct MeasurePeriodically : IMeasurer {
 
 	Future<Void> start() override {
 		msp.start();
-		return periodicActor(this);
+		return periodicActor();
 	}
 	void addTransaction(ITransactor::Stats* st, double now) override { msp.addTransaction(st, now); }
 	void getMetrics(std::vector<PerfMetric>& m) override {
@@ -281,13 +280,13 @@ struct MeasurePeriodically : IMeasurer {
 		msp.start();
 	}
 
-	ACTOR static Future<Void> periodicActor(MeasurePeriodically* self) {
-		state double startT = now();
-		state double elapsed = 0;
-		loop {
-			elapsed += self->period;
-			wait(delayUntil(startT + elapsed));
-			self->nextPeriod(elapsed);
+	Future<Void> periodicActor() {
+		double startT = now();
+		double elapsed = 0;
+		while (true) {
+			elapsed += period;
+			co_await delayUntil(startT + elapsed);
+			nextPeriod(elapsed);
 		}
 	}
 };
@@ -320,7 +319,7 @@ struct ThroughputWorkload : TestWorkload {
 	int activeActors;
 	double totalLatencyIntegral, totalTransactionsIntegral, startT;
 
-	ThroughputWorkload(WorkloadContext const& wcx)
+	explicit ThroughputWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), activeActors(0), totalLatencyIntegral(0), totalTransactionsIntegral(0) {
 		auto multi = makeReference<MeasureMulti>();
 		measurer = multi;
@@ -335,38 +334,38 @@ struct ThroughputWorkload : TestWorkload {
 		double sweepDelay = getOption(options, "sweepDelay"_sr, 0);
 		double zeroPaddingRatio = getOption(options, "zeroPaddingRatio"_sr, 0.15);
 
-		auto AType = Reference<ITransactor>(new RWTransactor(getOption(options, "readsPerTransactionA"_sr, 10),
-		                                                     getOption(options, "writesPerTransactionA"_sr, 0),
-		                                                     keyCount,
-		                                                     keyBytes,
-		                                                     minValueBytes,
-		                                                     maxValueBytes,
-		                                                     zeroPaddingRatio));
-		auto BType = Reference<ITransactor>(new RWTransactor(getOption(options, "readsPerTransactionB"_sr, 5),
-		                                                     getOption(options, "writesPerTransactionB"_sr, 5),
-		                                                     keyCount,
-		                                                     keyBytes,
-		                                                     minValueBytes,
-		                                                     maxValueBytes,
-		                                                     zeroPaddingRatio));
+		auto AType = makeReference<RWTransactor>(getOption(options, "readsPerTransactionA"_sr, 10),
+		                                         getOption(options, "writesPerTransactionA"_sr, 0),
+		                                         keyCount,
+		                                         keyBytes,
+		                                         minValueBytes,
+		                                         maxValueBytes,
+		                                         zeroPaddingRatio);
+		auto BType = makeReference<RWTransactor>(getOption(options, "readsPerTransactionB"_sr, 5),
+		                                         getOption(options, "writesPerTransactionB"_sr, 5),
+		                                         keyCount,
+		                                         keyBytes,
+		                                         minValueBytes,
+		                                         maxValueBytes,
+		                                         zeroPaddingRatio);
 
 		if (sweepDuration > 0) {
-			op = Reference<ITransactor>(new SweepTransactor(sweepDuration, sweepDelay, AType, BType));
+			op = makeReference<SweepTransactor>(sweepDuration, sweepDelay, AType, BType);
 		} else {
-			op = Reference<ITransactor>(new ABTransactor(getOption(options, "alpha"_sr, 0.1), AType, BType));
+			op = makeReference<ABTransactor>(getOption(options, "alpha"_sr, 0.1), AType, BType);
 		}
 
 		double measureDelay = getOption(options, "measureDelay"_sr, 50.0);
 		double measureDuration = getOption(options, "measureDuration"_sr, 10.0);
-		multi->ms.push_back(Reference<IMeasurer>(new MeasureSinglePeriod(measureDelay, measureDuration)));
+		multi->ms.push_back(makeReference<MeasureSinglePeriod>(measureDelay, measureDuration));
 
 		double measurePeriod = getOption(options, "measurePeriod"_sr, 0.0);
 		std::vector<std::string> periodicMetrics =
 		    getOption(options, "measurePeriodicMetrics"_sr, std::vector<std::string>());
 		if (measurePeriod) {
-			ASSERT(periodicMetrics.size() != 0);
-			multi->ms.push_back(Reference<IMeasurer>(new MeasurePeriodically(
-			    measurePeriod, std::set<std::string>(periodicMetrics.begin(), periodicMetrics.end()))));
+			ASSERT(!periodicMetrics.empty());
+			multi->ms.push_back(makeReference<MeasurePeriodically>(
+			    measurePeriod, std::set<std::string>(periodicMetrics.begin(), periodicMetrics.end())));
 		}
 
 		Pgain = getOption(options, "ProportionalGain"_sr, 0.1);
@@ -386,32 +385,32 @@ struct ThroughputWorkload : TestWorkload {
 		Future<Void> ac = actorCollection(add.getFuture(), &activeActors);
 		Future<Void> r = timeout(measurer->start() && ac, testDuration, Void());
 		ASSERT(!ac.isReady()); // ... because else the following line would create an unbreakable reference cycle
-		add.send(throughputActor(cx, this, add));
+		add.send(throughputActor(cx, add));
 		return r;
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
 
-	ACTOR static Future<Void> throughputActor(Database db, ThroughputWorkload* self, PromiseStream<Future<Void>> add) {
-		state double before = now();
-		state ITransactor::Stats stats;
-		wait(self->op->doTransaction(db, &stats));
-		state double after = now();
+	Future<Void> throughputActor(Database db, PromiseStream<Future<Void>> add) {
+		double before = now();
+		ITransactor::Stats stats;
+		co_await op->doTransaction(db, &stats);
+		double after = now();
 
-		wait(delay(0.0));
+		co_await delay(0.0);
 		stats.totalLatency = after - before;
-		self->measurer->addTransaction(&stats, after);
+		measurer->addTransaction(&stats, after);
 
-		self->totalLatencyIntegral += after - before;
-		self->totalTransactionsIntegral += 1;
+		totalLatencyIntegral += after - before;
+		totalTransactionsIntegral += 1;
 
-		double error = after - before - self->targetLatency;
+		double error = after - before - targetLatency;
 		// Ideally ierror would be integral [avg. transaction latency - targetLatency] dt.
 		// Actually we calculate integral[ transaction latency - targetLatency ] dtransaction and change units.
-		double ierror = (self->totalLatencyIntegral - self->totalTransactionsIntegral * self->targetLatency) /
-		                self->totalTransactionsIntegral * (after - self->startT);
+		double ierror = (totalLatencyIntegral - totalTransactionsIntegral * targetLatency) / totalTransactionsIntegral *
+		                (after - startT);
 
-		double desiredSuccessors = 1 - (error * self->Pgain + ierror * self->Igain) / self->targetLatency;
+		double desiredSuccessors = 1 - (error * Pgain + ierror * Igain) / targetLatency;
 
 		// if (deterministicRandom()->random01() < .001) TraceEvent("ThroughputControl").detail("Error",
 		// error).detail("IError", ierror).detail("DesiredSuccessors", desiredSuccessors).detail("ActiveActors",
@@ -422,13 +421,12 @@ struct ThroughputWorkload : TestWorkload {
 		// SOMEDAY: How can we prevent the number of actors on different clients from diverging?
 
 		int successors = deterministicRandom()->random01() + desiredSuccessors;
-		if (successors < 1 && self->activeActors <= 1)
+		if (successors < 1 && activeActors <= 1)
 			successors = 1;
-		if (successors > 1 && self->activeActors >= 200000)
+		if (successors > 1 && activeActors >= 200000)
 			successors = 1;
 		for (int s = 0; s < successors; s++)
-			add.send(throughputActor(db, self, add));
-		return Void();
+			add.send(throughputActor(db, add));
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override { measurer->getMetrics(m); }
diff --git a/fdbserver/workloads/ThroughputQuota.actor.cpp b/fdbserver/workloads/ThroughputQuota.actor.cpp
deleted file mode 100644
index 17795108307..00000000000
--- a/fdbserver/workloads/ThroughputQuota.actor.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * ThroughputQuota.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/TagThrottle.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-// This workload sets the throughput quota of a tag during the setup phase
-class ThroughputQuotaWorkload : public TestWorkload {
-	TransactionTag transactionTag;
-	int64_t reservedQuotaInPages{ 0 };
-	int64_t totalQuotaInPages{ 0 };
-
-	int64_t getReservedQuota() const { return reservedQuotaInPages * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE; }
-
-	int64_t getTotalQuota() const { return totalQuotaInPages * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE; }
-
-	ACTOR static Future<Void> setup(ThroughputQuotaWorkload* self, Database cx) {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		loop {
-			try {
-				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				TraceEvent("ThroughputQuotaWorkload_SettingTagQuota")
-				    .detail("Tag", printable(self->transactionTag))
-				    .detail("ReservedQuota", self->getReservedQuota())
-				    .detail("TotalQuota", self->getTotalQuota());
-				ThrottleApi::setTagQuota(tr, self->transactionTag, self->getReservedQuota(), self->getTotalQuota());
-				wait(tr->commit());
-				return Void();
-			} catch (Error& e) {
-				TraceEvent("ThroughputQuotaWorkload_SetupError").error(e);
-				wait(tr->onError(e));
-			}
-		};
-	}
-
-public:
-	static constexpr auto NAME = "ThroughputQuota";
-	explicit ThroughputQuotaWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		transactionTag = getOption(options, "transactionTag"_sr, "sampleTag"_sr);
-		reservedQuotaInPages = getOption(options, "reservedQuotaInPages"_sr, 0);
-		totalQuotaInPages = getOption(options, "totalQuotaInPages"_sr, 0);
-	}
-
-	Future<Void> setup(Database const& cx) override { return clientId ? Void() : setup(this, cx); }
-	Future<Void> start(Database const& cx) override { return Void(); }
-	Future<bool> check(Database const& cx) override { return true; }
-	void getMetrics(std::vector<PerfMetric>& m) override {}
-};
-
-WorkloadFactory<ThroughputQuotaWorkload> ThroughputQuotaWorkloadFactory;
diff --git a/fdbserver/workloads/TimeKeeperCorrectness.actor.cpp b/fdbserver/workloads/TimeKeeperCorrectness.cpp
similarity index 61%
rename from fdbserver/workloads/TimeKeeperCorrectness.actor.cpp
rename to fdbserver/workloads/TimeKeeperCorrectness.cpp
index a00393d737d..4c907f7892a 100644
--- a/fdbserver/workloads/TimeKeeperCorrectness.actor.cpp
+++ b/fdbserver/workloads/TimeKeeperCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * TimeKeeperCorrectness.actor.cpp
+ * TimeKeeperCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,11 +18,10 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbclient/SystemData.h"
-#include "fdbclient/KeyBackedTypes.actor.h"
-#include "fdbserver/Knobs.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbclient/KeyBackedTypes.h"
+#include "fdbserver/core/Knobs.h"
 
 struct TimeKeeperCorrectnessWorkload : TestWorkload {
 	static constexpr auto NAME = "TimeKeeperCorrectness";
@@ -30,7 +29,7 @@ struct TimeKeeperCorrectnessWorkload : TestWorkload {
 	double testDuration;
 	std::map<int64_t, Version> inMemTimeKeeper;
 
-	TimeKeeperCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit TimeKeeperCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 20.0);
 	}
 
@@ -38,66 +37,66 @@ struct TimeKeeperCorrectnessWorkload : TestWorkload {
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR static Future<Void> _start(Database cx, TimeKeeperCorrectnessWorkload* self) {
+	Future<Void> start(Database const& cx) override {
 		TraceEvent(SevInfo, "TKCorrectness_Start").log();
 
-		state double start = now();
-		while (now() - start > self->testDuration) {
-			state Transaction tr(cx);
+		double start = now();
+		while (now() - start > testDuration) {
+			Transaction tr(cx);
 
-			loop {
+			while (true) {
+				Error err;
 				try {
-					state int64_t curTime = now();
-					Version v = wait(tr.getReadVersion());
-					self->inMemTimeKeeper[curTime] = v;
+					int64_t curTime = now();
+					Version v = co_await tr.getReadVersion();
+					inMemTimeKeeper[curTime] = v;
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 
 			// For every sample from Timekeeper collect two samples here.
-			wait(delay(std::min(SERVER_KNOBS->TIME_KEEPER_DELAY / 10, (int64_t)1L)));
+			co_await delay(std::min(SERVER_KNOBS->TIME_KEEPER_DELAY / 10, (int64_t)1L));
 		}
 
 		TraceEvent(SevInfo, "TKCorrectness_Completed").log();
-		return Void();
 	}
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
-	ACTOR static Future<bool> _check(Database cx, TimeKeeperCorrectnessWorkload* self) {
-		state KeyBackedMap<int64_t, Version> dbTimeKeeper = KeyBackedMap<int64_t, Version>(timeKeeperPrefixRange.begin);
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+	Future<bool> check(Database const& cx) override {
+		KeyBackedMap<int64_t, Version> dbTimeKeeper = KeyBackedMap<int64_t, Version>(timeKeeperPrefixRange.begin);
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
 
 		TraceEvent(SevInfo, "TKCorrectness_CheckStart")
 		    .detail("TimeKeeperMaxEntries", SERVER_KNOBS->TIME_KEEPER_MAX_ENTRIES)
 		    .detail("TimeKeeperDelay", SERVER_KNOBS->TIME_KEEPER_DELAY);
 
-		loop {
+		while (true) {
+			Error err;
 			try {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				KeyBackedRangeResult<std::pair<int64_t, Version>> allItems =
-				    wait(dbTimeKeeper.getRange(tr, 0, Optional<int64_t>(), self->inMemTimeKeeper.size() + 2));
+				    co_await dbTimeKeeper.getRange(tr, 0, Optional<int64_t>(), inMemTimeKeeper.size() + 2);
 
 				if (allItems.results.size() > SERVER_KNOBS->TIME_KEEPER_MAX_ENTRIES + 1) {
 					TraceEvent(SevError, "TKCorrectness_TooManyEntries")
 					    .detail("Expected", SERVER_KNOBS->TIME_KEEPER_MAX_ENTRIES + 1)
 					    .detail("Found", allItems.results.size());
-					return false;
+					co_return false;
 				}
 
-				if (allItems.results.size() < self->testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY) {
+				if (allItems.results.size() < testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY) {
 					TraceEvent(SevWarnAlways, "TKCorrectness_TooFewEntries")
-					    .detail("Expected", self->testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY)
+					    .detail("Expected", testDuration / SERVER_KNOBS->TIME_KEEPER_DELAY)
 					    .detail("Found", allItems.results.size());
 				}
 
 				for (auto item : allItems.results) {
-					auto it = self->inMemTimeKeeper.lower_bound(item.first);
-					if (it == self->inMemTimeKeeper.end()) {
+					auto it = inMemTimeKeeper.lower_bound(item.first);
+					if (it == inMemTimeKeeper.end()) {
 						continue;
 					}
 
@@ -107,19 +106,18 @@ struct TimeKeeperCorrectnessWorkload : TestWorkload {
 						    .detail("ClusterVersion", item.second)
 						    .detail("LocalTime", it->first)
 						    .detail("LocalVersion", it->second);
-						return false;
+						co_return false;
 					}
 				}
 
 				TraceEvent(SevInfo, "TKCorrectness_Passed").log();
-				return true;
+				co_return true;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
-
-	Future<bool> check(Database const& cx) override { return _check(cx, this); }
 };
 
 WorkloadFactory<TimeKeeperCorrectnessWorkload> TimeKeeperCorrectnessWorkloadFactory;
diff --git a/fdbserver/workloads/TransactionCost.actor.cpp b/fdbserver/workloads/TransactionCost.cpp
similarity index 74%
rename from fdbserver/workloads/TransactionCost.actor.cpp
rename to fdbserver/workloads/TransactionCost.cpp
index b59b8cb2779..ae80a018e17 100644
--- a/fdbserver/workloads/TransactionCost.actor.cpp
+++ b/fdbserver/workloads/TransactionCost.cpp
@@ -1,5 +1,5 @@
 /*
- * TransactionCost.actor.cpp
+ * TransactionCost.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,8 +19,7 @@
  */
 
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h"
+#include "fdbserver/tester/workloads.h"
 
 class TransactionCostWorkload : public TestWorkload {
 	int iterations{ 1000 };
@@ -63,18 +62,18 @@ class TransactionCostWorkload : public TestWorkload {
 	};
 
 	class ReadLargeValueTest : public ITest {
-		ACTOR static Future<Void> setup(TransactionCostWorkload const* workload,
-		                                ReadLargeValueTest* self,
-		                                Database cx) {
-			state Transaction tr(cx);
-			loop {
+		Future<Void> setupImpl(TransactionCostWorkload const& workload, Database cx) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
-					tr.set(workload->getKey(self->testNumber), getValue(CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE));
-					wait(tr.commit());
-					return Void();
+					tr.set(workload.getKey(testNumber), getValue(CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE));
+					co_await tr.commit();
+					co_return;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 
@@ -82,7 +81,7 @@ class TransactionCostWorkload : public TestWorkload {
 		explicit ReadLargeValueTest(int64_t testNumber) : ITest(testNumber) {}
 
 		Future<Void> setup(TransactionCostWorkload const& workload, Database const& cx) override {
-			return setup(&workload, this, cx);
+			return setupImpl(workload, cx);
 		}
 
 		Future<Void> exec(TransactionCostWorkload const& workload, Reference<ReadYourWritesTransaction> tr) override {
@@ -102,7 +101,7 @@ class TransactionCostWorkload : public TestWorkload {
 		}
 
 		int64_t expectedFinalCost() const override {
-			return CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
+			return CLIENT_KNOBS->TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
 		}
 	};
 
@@ -116,8 +115,7 @@ class TransactionCostWorkload : public TestWorkload {
 		}
 
 		int64_t expectedFinalCost() const override {
-			return 2 * CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO *
-			       CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
+			return 2 * CLIENT_KNOBS->TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
 		}
 	};
 
@@ -133,8 +131,7 @@ class TransactionCostWorkload : public TestWorkload {
 		}
 
 		int64_t expectedFinalCost() const override {
-			return 10 * CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO *
-			       CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
+			return 10 * CLIENT_KNOBS->TAG_THROTTLING_RW_FUNGIBILITY_RATIO * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
 		}
 	};
 
@@ -151,18 +148,20 @@ class TransactionCostWorkload : public TestWorkload {
 	};
 
 	class ReadRangeTest : public ITest {
-		ACTOR static Future<Void> setup(ReadRangeTest* self, TransactionCostWorkload const* workload, Database cx) {
-			state Transaction tr(cx);
-			loop {
+		Future<Void> setupImpl(TransactionCostWorkload const& workload, Database cx) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < 10; ++i) {
-						tr.set(workload->getKey(self->testNumber, i), workload->getValue(20));
+						tr.set(workload.getKey(testNumber, i), workload.getValue(20));
 					}
-					wait(tr.commit());
-					return Void();
+					co_await tr.commit();
+					co_return;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 
@@ -170,7 +169,7 @@ class TransactionCostWorkload : public TestWorkload {
 		explicit ReadRangeTest(int64_t testNumber) : ITest(testNumber) {}
 
 		Future<Void> setup(TransactionCostWorkload const& workload, Database const& cx) override {
-			return setup(this, &workload, cx);
+			return setupImpl(workload, cx);
 		}
 
 		Future<Void> exec(TransactionCostWorkload const& workload, Reference<ReadYourWritesTransaction> tr) override {
@@ -182,20 +181,20 @@ class TransactionCostWorkload : public TestWorkload {
 	};
 
 	class ReadMultipleValuesTest : public ITest {
-		ACTOR static Future<Void> setup(ReadMultipleValuesTest* self,
-		                                TransactionCostWorkload const* workload,
-		                                Database cx) {
-			state Transaction tr(cx);
-			loop {
+		Future<Void> setupImpl(TransactionCostWorkload const& workload, Database cx) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < 10; ++i) {
-						tr.set(workload->getKey(self->testNumber, i), workload->getValue(20));
+						tr.set(workload.getKey(testNumber, i), workload.getValue(20));
 					}
-					wait(tr.commit());
-					return Void();
+					co_await tr.commit();
+					co_return;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 
@@ -203,7 +202,7 @@ class TransactionCostWorkload : public TestWorkload {
 		explicit ReadMultipleValuesTest(int64_t testNumber) : ITest(testNumber) {}
 
 		Future<Void> setup(TransactionCostWorkload const& workload, Database const& cx) override {
-			return setup(this, &workload, cx);
+			return setupImpl(workload, cx);
 		}
 
 		Future<Void> exec(TransactionCostWorkload const& workload, Reference<ReadYourWritesTransaction> tr) override {
@@ -218,21 +217,21 @@ class TransactionCostWorkload : public TestWorkload {
 	};
 
 	class LargeReadRangeTest : public ITest {
-		ACTOR static Future<Void> setup(LargeReadRangeTest* self,
-		                                TransactionCostWorkload const* workload,
-		                                Database cx) {
-			state Transaction tr(cx);
-			loop {
+		Future<Void> setupImpl(TransactionCostWorkload const& workload, Database cx) {
+			Transaction tr(cx);
+			while (true) {
+				Error err;
 				try {
 					for (int i = 0; i < 10; ++i) {
-						tr.set(workload->getKey(self->testNumber, i),
-						       workload->getValue(CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE));
+						tr.set(workload.getKey(testNumber, i),
+						       workload.getValue(CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE));
 					}
-					wait(tr.commit());
-					return Void();
+					co_await tr.commit();
+					co_return;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 
@@ -240,7 +239,7 @@ class TransactionCostWorkload : public TestWorkload {
 		explicit LargeReadRangeTest(int64_t testNumber) : ITest(testNumber) {}
 
 		Future<Void> setup(TransactionCostWorkload const& workload, Database const& cx) override {
-			return setup(this, &workload, cx);
+			return setupImpl(workload, cx);
 		}
 
 		Future<Void> exec(TransactionCostWorkload const& workload, Reference<ReadYourWritesTransaction> tr) override {
@@ -274,38 +273,38 @@ class TransactionCostWorkload : public TestWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> runTest(TransactionCostWorkload* self, Database cx, ITest* test) {
-		wait(test->setup(*self, cx));
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		if (self->debugTransactions) {
+	Future<Void> runTest(Database cx, ITest* test) {
+		co_await test->setup(*this, cx);
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		if (debugTransactions) {
 			test->debugTransaction(*tr);
 		}
-		loop {
+		while (true) {
+			Error err;
 			try {
-				wait(test->exec(*self, tr));
-				wait(tr->commit());
+				co_await test->exec(*this, tr);
+				co_await tr->commit();
 				ASSERT_EQ(tr->getTotalCost(), test->expectedFinalCost());
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				err = e;
 			}
+			co_await tr->onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> start(TransactionCostWorkload* self, Database cx) {
-		state uint64_t testNumber = 0;
-		state Future<Void> f;
+	Future<Void> runTests(Database cx) {
+		Future<Void> f;
 		// Must use shared_ptr because Flow doesn't support perfect forwarding into actors
-		state std::shared_ptr<ITest> test;
-		for (; testNumber < self->iterations; ++testNumber) {
+		std::shared_ptr<ITest> test;
+		for (uint64_t testNumber = 0; testNumber < iterations; ++testNumber) {
 			test = createRandomTest(testNumber);
-			wait(runTest(self, cx, test.get()));
+			co_await runTest(cx, test.get());
 		}
-		return Void();
 	}
 
 public:
-	TransactionCostWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit TransactionCostWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		iterations = getOption(options, "iterations"_sr, 1000);
 		prefix = getOption(options, "prefix"_sr, "transactionCost/"_sr);
 		debugTransactions = getOption(options, "debug"_sr, false);
@@ -315,7 +314,7 @@ class TransactionCostWorkload : public TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return clientId ? Void() : start(this, cx); }
+	Future<Void> start(Database const& cx) override { return clientId ? Void() : runTests(cx); }
 
 	Future<bool> check(Database const& cx) override { return true; }
 
diff --git a/fdbserver/workloads/TriggerRecovery.actor.cpp b/fdbserver/workloads/TriggerRecovery.cpp
similarity index 67%
rename from fdbserver/workloads/TriggerRecovery.actor.cpp
rename to fdbserver/workloads/TriggerRecovery.cpp
index 88beaedfff5..8b5f5a9d587 100644
--- a/fdbserver/workloads/TriggerRecovery.actor.cpp
+++ b/fdbserver/workloads/TriggerRecovery.cpp
@@ -1,5 +1,5 @@
 /*
- * TriggerRecovery.actor.cpp
+ * TriggerRecovery.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/ServerDBInfo.h"
 #include "fdbclient/Status.h"
 #include "fdbclient/StatusClient.h"
-#include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/RunRYWTransaction.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbclient/ManagementAPI.h"
+#include "fdbclient/RunRYWTransaction.h"
 
 struct TriggerRecoveryLoopWorkload : TestWorkload {
 	static constexpr auto NAME = "TriggerRecoveryLoop";
@@ -36,7 +35,7 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 	Optional<int32_t> originalNumOfResolvers;
 	Optional<int32_t> currentNumOfResolvers;
 
-	TriggerRecoveryLoopWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit TriggerRecoveryLoopWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		startTime = getOption(options, "startTime"_sr, 0.0);
 		numRecoveries = getOption(options, "numRecoveries"_sr, deterministicRandom()->randomInt(1, 10));
 		delayBetweenRecoveries = getOption(options, "delayBetweenRecoveries"_sr, 0.0);
@@ -48,10 +47,9 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 		    .detail("DelayBetweenRecoveries", delayBetweenRecoveries);
 	}
 
-	ACTOR Future<Void> setOriginalNumOfResolvers(Database cx, TriggerRecoveryLoopWorkload* self) {
-		DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
+	Future<Void> setOriginalNumOfResolvers(Database cx, TriggerRecoveryLoopWorkload* self) {
+		DatabaseConfiguration config = co_await getDatabaseConfiguration(cx);
 		self->originalNumOfResolvers = self->currentNumOfResolvers = config.getDesiredResolvers();
-		return Void();
 	}
 
 	Future<Void> setup(Database const& cx) override {
@@ -61,28 +59,27 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 		return Void();
 	}
 
-	ACTOR Future<Void> returnIfClusterRecovered(Database cx) {
-		loop {
-			state ReadYourWritesTransaction tr(cx);
+	Future<Void> returnIfClusterRecovered(Database cx) {
+		while (true) {
+			ReadYourWritesTransaction tr(cx);
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-				state Version v = wait(tr.getReadVersion());
+				Version v = co_await tr.getReadVersion();
 				tr.makeSelfConflicting();
-				wait(tr.commit());
+				co_await tr.commit();
 				TraceEvent(SevInfo, "TriggerRecoveryLoop_ClusterVersion").detail("Version", v);
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> changeResolverConfig(Database cx,
-	                                        TriggerRecoveryLoopWorkload* self,
-	                                        bool setToOriginal = false) {
-		state int32_t numResolversToSet;
+	Future<Void> changeResolverConfig(Database cx, TriggerRecoveryLoopWorkload* self, bool setToOriginal = false) {
+		int32_t numResolversToSet{ 0 };
 		if (setToOriginal) {
 			numResolversToSet = self->originalNumOfResolvers.get();
 		} else {
@@ -90,10 +87,10 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 			                        ? self->originalNumOfResolvers.get() + 1
 			                        : self->originalNumOfResolvers.get();
 		}
-		state StringRef configStr(format("resolvers=%d", numResolversToSet));
-		loop {
+		StringRef configStr(format("resolvers=%d", numResolversToSet));
+		while (true) {
 			Optional<ConfigureAutoResult> conf;
-			ConfigurationResult r = wait(ManagementAPI::changeConfig(cx.getReference(), { configStr }, conf, true));
+			ConfigurationResult r = co_await ManagementAPI::changeConfig(cx.getReference(), { configStr }, conf, true);
 			if (r == ConfigurationResult::SUCCESS) {
 				self->currentNumOfResolvers = numResolversToSet;
 				TraceEvent(SevInfo, "TriggerRecoveryLoop_ChangeResolverConfigSuccess")
@@ -101,20 +98,20 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 				break;
 			}
 			TraceEvent(SevWarn, "TriggerRecoveryLoop_ChangeResolverConfigFailed").detail("Result", r);
-			wait(delay(1.0));
+			co_await delay(1.0);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> killAll(Database cx) {
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+	Future<Void> killAll(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				RangeResult kvs =
-				    wait(tr.getRange(KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
-				                     CLIENT_KNOBS->TOO_MANY));
+				RangeResult kvs = co_await tr.getRange(
+				    KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr),
+				    CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!kvs.more);
 				std::map<Key, Value> address_interface;
 				for (auto it : kvs) {
@@ -122,7 +119,7 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 					                   .removePrefix("\xff\xff/worker_interfaces/"_sr);
 					address_interface[ip_port] = it.value;
 				}
-				for (auto it : address_interface) {
+				for (const auto& it : address_interface) {
 					if (cx->apiVersionAtLeast(700))
 						BinaryReader::fromStringRef<ClientWorkerInterface>(it.second, IncludeVersion())
 						    .reboot.send(RebootRequest());
@@ -130,42 +127,42 @@ struct TriggerRecoveryLoopWorkload : TestWorkload {
 						tr.set("\xff\xff/reboot_worker"_sr, it.second);
 				}
 				TraceEvent(SevInfo, "TriggerRecoveryLoop_AttempedKillAll").log();
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> _start(Database cx, TriggerRecoveryLoopWorkload* self) {
-		wait(delay(self->startTime));
-		state int numRecoveriesDone = 0;
+	Future<Void> _start(Database cx) {
+		co_await delay(startTime);
+		int numRecoveriesDone = 0;
 		try {
-			loop {
-				if (deterministicRandom()->random01() < self->killAllProportion) {
-					wait(self->killAll(cx));
+			while (true) {
+				if (deterministicRandom()->random01() < killAllProportion) {
+					co_await killAll(cx);
 				} else {
-					wait(self->changeResolverConfig(cx, self));
+					co_await changeResolverConfig(cx, this);
 				}
 				numRecoveriesDone++;
 				TraceEvent(SevInfo, "TriggerRecoveryLoop_AttempedRecovery").detail("RecoveryNum", numRecoveriesDone);
-				if (numRecoveriesDone == self->numRecoveries) {
+				if (numRecoveriesDone == numRecoveries) {
 					break;
 				}
-				wait(delay(self->delayBetweenRecoveries));
-				wait(self->returnIfClusterRecovered(cx));
+				co_await delay(delayBetweenRecoveries);
+				co_await returnIfClusterRecovered(cx);
 			}
 		} catch (Error& e) {
 			// Dummy catch here to give a chance to reset number of resolvers to its original value
 		}
-		wait(self->changeResolverConfig(cx, self, true));
-		return Void();
+		co_await changeResolverConfig(cx, this, true);
 	}
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId != 0)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/TxnTimeout.actor.cpp b/fdbserver/workloads/TxnTimeout.cpp
similarity index 62%
rename from fdbserver/workloads/TxnTimeout.actor.cpp
rename to fdbserver/workloads/TxnTimeout.cpp
index 46bfa20a79a..3309edd347a 100644
--- a/fdbserver/workloads/TxnTimeout.actor.cpp
+++ b/fdbserver/workloads/TxnTimeout.cpp
@@ -1,5 +1,5 @@
 /*
- * TxnTimeout.actor.cpp
+ * TxnTimeout.cpp
  *
  * This workload validates that FoundationDB's transaction timeout mechanisms work correctly
  * by creating long-running transactions that intentionally approach (but don't exceed) the
@@ -18,9 +18,9 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/PerfMetric.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/ServerDBInfo.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/ServerDBInfo.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Buggify.h"
 #include "flow/Error.h"
 #include "flow/Optional.h"
@@ -28,8 +28,6 @@
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 struct TxnTimeout : TestWorkload {
 	static constexpr auto NAME = "TxnTimeout";
 
@@ -44,7 +42,7 @@ struct TxnTimeout : TestWorkload {
 	int txnsSucceeded{ 0 }; // Number of transactions that completed successfully
 	int txnsFailed{ 0 }; // Number of transactions that failed with unexpected errors
 
-	TxnTimeout(const WorkloadContext& wctx) : TestWorkload(wctx) {
+	explicit TxnTimeout(const WorkloadContext& wctx) : TestWorkload(wctx) {
 		// Parse workload configuration from TOML test definition
 		testDuration = getOption(options, "testDuration"_sr, 120.0);
 		actorsPerClient = getOption(options, "actorsPerClient"_sr, 1);
@@ -108,46 +106,46 @@ struct TxnTimeout : TestWorkload {
 	// Initializes the database with test data for each actor to operate on
 	// Each actor creates nodeCountPerClientPerActor keys initialized to value "0"
 	// Keys are batched into transactions for efficiency
-	ACTOR static Future<Void> populateDatabase(TxnTimeout* self, Database db, int actorIdx) {
-		state int nodeIdx = 0;
+	Future<Void> populateDatabase(Database db, int actorIdx) {
+		int nodeIdx = 0;
 		// Batch size is 1/4 of total keys, resulting in 4 batches per actor
-		state int batchSize = std::max(1, self->nodeCountPerClientPerActor / 4);
+		int batchSize = std::max(1, nodeCountPerClientPerActor / 4);
 
-		while (nodeIdx < self->nodeCountPerClientPerActor) {
-			state Transaction tr(db);
-			loop {
+		while (nodeIdx < nodeCountPerClientPerActor) {
+			Transaction tr(db);
+			while (true) {
+				Error err;
 				try {
 					// Batch up to batchSize keys in a single transaction
-					state int batchEnd = std::min(nodeIdx + batchSize, self->nodeCountPerClientPerActor);
+					int batchEnd = std::min(nodeIdx + batchSize, nodeCountPerClientPerActor);
 					for (int i = nodeIdx; i < batchEnd; i++) {
-						Key key = makeKey(self->clientId, actorIdx, i);
+						Key key = makeKey(clientId, actorIdx, i);
 						tr.set(key, "0"_sr);
 					}
-					wait(tr.commit());
+					co_await tr.commit();
 					nodeIdx = batchEnd;
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
+					err = e;
 				}
+				co_await tr.onError(err);
 			}
 		}
 
 		TraceEvent("TxnTimeoutPopulateComplete")
-		    .detail("ClientId", self->clientId)
+		    .detail("ClientId", clientId)
 		    .detail("ActorIdx", actorIdx)
 		    .detail("KeysCreated", nodeIdx);
-		return Void();
 	}
 
 	// Runs database population concurrently across actors and clients
-	ACTOR static Future<Void> populateDatabaseAllActors(TxnTimeout* self, Database db) {
-		state std::vector<Future<Void>> populationActors;
-		for (int actorIdx = 0; actorIdx < self->actorsPerClient; ++actorIdx) {
-			populationActors.push_back(populateDatabase(self, db, actorIdx));
+	Future<Void> populateDatabaseAllActors(Database db) {
+		std::vector<Future<Void>> populationActors;
+		for (int actorIdx = 0; actorIdx < actorsPerClient; ++actorIdx) {
+			populationActors.push_back(populateDatabase(db, actorIdx));
 		}
-		wait(waitForAll(populationActors));
-		TraceEvent("TxnTimeoutPopulateAllComplete").detail("ClientId", self->clientId);
-		return Void();
+		co_await waitForAll(populationActors);
+		TraceEvent("TxnTimeoutPopulateAllComplete").detail("ClientId", clientId);
 	}
 
 	/*
@@ -168,64 +166,65 @@ struct TxnTimeout : TestWorkload {
 	 * - Version jumps due to recovery (>MAX_WRITE_TRANSACTION_LIFE_VERSIONS) are tolerated
 	 * - Any other transaction_too_old or similar timeout errors are counted as failures
 	 */
-	ACTOR static Future<Void> txnClient(TxnTimeout* self, Database db, int actorIdx) {
-		state int nodeIdx = 0;
-		state double workloadStartTime = now();
+	Future<Void> txnClient(Database db, int actorIdx) {
+		int nodeIdx = 0;
+		double workloadStartTime = now();
 
 		// Run transactions for 80% of test duration to allow time for cleanup
-		state double runDuration = self->testDuration * 0.8;
+		double runDuration = testDuration * 0.8;
 
-		loop {
+		while (true) {
 			// Cycle through all keys for this actor
-			if (nodeIdx == self->nodeCountPerClientPerActor) {
+			if (nodeIdx == nodeCountPerClientPerActor) {
 				nodeIdx = 0;
 			}
 
 			// Stop when we've reached the target run duration
 			if (now() - workloadStartTime > runDuration) {
 				TraceEvent("TxnTimeoutClientComplete")
-				    .detail("ClientId", self->clientId)
+				    .detail("ClientId", clientId)
 				    .detail("ActorIdx", actorIdx)
 				    .detail("Duration", now() - workloadStartTime);
 				break;
 			}
 
-			state Transaction tr(db);
-			state Version readVersion = 0;
-			state double txnStartTime = now();
-			self->txnsTotal++;
+			Transaction tr(db);
+			Version readVersion = 0;
+			double txnStartTime = now();
+			txnsTotal++;
 
-			loop {
+			while (true) {
+				Error caughtErr;
 				try {
 					// Generate the same key pattern as in populate phase
-					state Key key = makeKey(self->clientId, actorIdx, nodeIdx);
+					Key key = makeKey(clientId, actorIdx, nodeIdx);
 
 					// Get read version and read the current value
-					state double readStartTime = now();
-					Version rv = wait(tr.getReadVersion());
+					double readStartTime = now();
+					Version rv = co_await tr.getReadVersion();
 					readVersion = rv;
 
-					state Optional<Value> val = wait(tr.get(key));
-					state double readDuration = now() - readStartTime;
+					Optional<Value> val = co_await tr.get(key);
+					double readDuration = now() - readStartTime;
 
 					// Artificial delay to extend transaction lifetime to target duration
 					// This is the core of the test: keeping transactions open longer than the usual 5 seconds
-					if (self->txnMinDuration > readDuration) {
-						wait(delay(self->txnMinDuration - readDuration));
+					if (txnMinDuration > readDuration) {
+						co_await delay(txnMinDuration - readDuration);
 					}
 
 					// Perform write operation (increment counter)
-					state int currentVal = std::stoi(val.get().toString());
-					state std::string newVal = std::to_string(currentVal + 1);
+					int currentVal = std::stoi(val.get().toString());
+					std::string newVal = std::to_string(currentVal + 1);
 					tr.set(key, StringRef(newVal));
 
 					// Commit and measure total transaction latency
-					wait(tr.commit());
-					state double txnLatency = now() - txnStartTime;
+					co_await tr.commit();
+					double txnLatency = now() - txnStartTime;
 
-					self->txnsSucceeded++;
+					txnsSucceeded++;
 					TraceEvent("TxnTimeoutTxnSuccess")
-					    .detail("ClientId", self->clientId)
+					    .detail("ClientId", clientId)
 					    .detail("ActorIdx", actorIdx)
 					    .detail("Key", key)
 					    .detail("OldValue", currentVal)
@@ -236,58 +235,58 @@ struct TxnTimeout : TestWorkload {
 					break;
 
 				} catch (Error& e) {
-					state Error err = e;
-					state bool isExpectedError = err.code() == error_code_future_version ||
-					                             err.code() == error_code_commit_unknown_result ||
-					                             err.code() == error_code_process_behind;
+					caughtErr = e;
+				}
+				Error err = caughtErr;
+				bool isExpectedError = err.code() == error_code_future_version ||
+				                       err.code() == error_code_commit_unknown_result ||
+				                       err.code() == error_code_process_behind;
 
-					TraceEvent(isExpectedError ? SevInfo : SevWarn, "TxnTimeoutTxnError")
-					    .detail("ClientId", self->clientId)
+				TraceEvent(isExpectedError ? SevInfo : SevWarn, "TxnTimeoutTxnError")
+				    .detail("ClientId", clientId)
+				    .detail("ActorIdx", actorIdx)
+				    .detail("RecoveryState", dbInfo->get().recoveryState)
+				    .detail("ReadVersion", readVersion)
+				    .errorUnsuppressed(err);
+
+				co_await tr.onError(err);
+
+				// Check if version jumped significantly (e.g stale read version, recovery)
+				Transaction rvTr(db);
+				Version newReadVersion = co_await rvTr.getReadVersion();
+				// The version delta is "best guess" because the newReadVersion could itself be stale, therefore
+				// the delta could be smaller than (sequencer commit version - readVersion)
+				Version versionDelta = newReadVersion - readVersion;
+				const bool isHighVersionJump = versionDelta > SERVER_KNOBS->MAX_WRITE_TRANSACTION_LIFE_VERSIONS;
+				const double txnDuration = now() - txnStartTime;
+				const bool tooMuchTimeHasPassed =
+				    txnDuration >
+				    ((double)SERVER_KNOBS->MAX_WRITE_TRANSACTION_LIFE_VERSIONS / SERVER_KNOBS->VERSIONS_PER_SECOND);
+
+				if (!isExpectedError && !isHighVersionJump && !tooMuchTimeHasPassed) {
+					txnsFailed++;
+					TraceEvent(SevError, "TxnTimeoutUnexpectedFailure")
+					    .detail("ClientId", clientId)
 					    .detail("ActorIdx", actorIdx)
-					    .detail("RecoveryState", self->dbInfo->get().recoveryState)
-					    .detail("ReadVersion", readVersion)
+					    .detail("OldReadVersion", readVersion)
+					    .detail("NewReadVersion", newReadVersion)
+					    .detail("VersionDelta", versionDelta)
+					    .detail("TxnDuration", txnDuration)
 					    .errorUnsuppressed(err);
-
-					wait(tr.onError(err));
-
-					// Check if version jumped significantly (e.g stale read version, recovery)
-					state Transaction rvTr(db);
-					Version newReadVersion = wait(rvTr.getReadVersion());
-					// The version delta is "best guess" because the newReadVersion could itself be stale, therefore
-					// the delta could be smaller than (sequencer commit version - readVersion)
-					Version versionDelta = newReadVersion - readVersion;
-					const bool isHighVersionJump = versionDelta > SERVER_KNOBS->MAX_WRITE_TRANSACTION_LIFE_VERSIONS;
-					const double txnDuration = now() - txnStartTime;
-					const bool tooMuchTimeHasPassed =
-					    txnDuration >
-					    ((double)SERVER_KNOBS->MAX_WRITE_TRANSACTION_LIFE_VERSIONS / SERVER_KNOBS->VERSIONS_PER_SECOND);
-
-					if (!isExpectedError && !isHighVersionJump && !tooMuchTimeHasPassed) {
-						self->txnsFailed++;
-						TraceEvent(SevError, "TxnTimeoutUnexpectedFailure")
-						    .detail("ClientId", self->clientId)
-						    .detail("ActorIdx", actorIdx)
-						    .detail("OldReadVersion", readVersion)
-						    .detail("NewReadVersion", newReadVersion)
-						    .detail("VersionDelta", versionDelta)
-						    .detail("TxnDuration", txnDuration)
-						    .errorUnsuppressed(err);
-					} else {
-						TraceEvent("TxnTimeoutExpectedFailure")
-						    .detail("ClientId", self->clientId)
-						    .detail("ActorIdx", actorIdx)
-						    .detail("OldReadVersion", readVersion)
-						    .detail("NewReadVersion", newReadVersion)
-						    .detail("VersionDelta", versionDelta)
-						    .detail("IsExpectedError", isExpectedError)
-						    .detail("IsHighVersionJump", isHighVersionJump);
-					}
-
-					txnStartTime = now();
+				} else {
+					TraceEvent("TxnTimeoutExpectedFailure")
+					    .detail("ClientId", clientId)
+					    .detail("ActorIdx", actorIdx)
+					    .detail("OldReadVersion", readVersion)
+					    .detail("NewReadVersion", newReadVersion)
+					    .detail("VersionDelta", versionDelta)
+					    .detail("IsExpectedError", isExpectedError)
+					    .detail("IsHighVersionJump", isHighVersionJump);
 				}
+
+				txnStartTime = now();
 			}
 		}
-		return Void();
 	}
 
 	/*
@@ -297,21 +296,21 @@ struct TxnTimeout : TestWorkload {
 	 * Phase 2: Run concurrent transaction clients that test timeout behavior
 	 * Phase 3: Report final metrics
 	 */
-	ACTOR Future<Void> workload(TxnTimeout* self, Database db) {
+	Future<Void> workload(TxnTimeout* self, Database db) {
 		TraceEvent("TxnTimeoutWorkloadStart")
 		    .detail("ClientId", self->clientId)
 		    .detail("TestDuration", self->testDuration)
 		    .detail("ActorsPerClient", self->actorsPerClient);
 
 		// Phase 1: Initialize database with test data
-		wait(populateDatabaseAllActors(self, db));
+		co_await populateDatabaseAllActors(db);
 
 		// Phase 2: Run transaction clients that test timeout behavior
-		state std::vector<Future<Void>> txnClients;
+		std::vector<Future<Void>> txnClients;
 		for (int actorIdx = 0; actorIdx < self->actorsPerClient; ++actorIdx) {
-			txnClients.emplace_back(txnClient(self, db, actorIdx));
+			txnClients.emplace_back(txnClient(db, actorIdx));
 		}
-		wait(waitForAll(txnClients));
+		co_await waitForAll(txnClients);
 
 		// Phase 3: Report final metrics
 		TraceEvent("TxnTimeoutWorkloadComplete")
@@ -320,8 +319,6 @@ struct TxnTimeout : TestWorkload {
 		    .detail("TxnsFailed", self->txnsFailed)
 		    .detail("TxnsTotal", self->txnsTotal)
 		    .detail("SuccessRate", self->txnsTotal > 0 ? (double)self->txnsSucceeded / self->txnsTotal : 0.0);
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/UDPWorkload.actor.cpp b/fdbserver/workloads/UDPWorkload.actor.cpp
deleted file mode 100644
index 72809f80d3b..00000000000
--- a/fdbserver/workloads/UDPWorkload.actor.cpp
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * UDPWorkload.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "fdbclient/FDBTypes.h"
-#include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/ActorCollection.h"
-#include "flow/Arena.h"
-#include "flow/Error.h"
-#include "flow/IRandom.h"
-#include "flow/flow.h"
-#include "flow/network.h"
-#include "flow/serialize.h"
-#include <functional>
-#include <limits>
-#include <unordered_map>
-#include <vector>
-#include <memory>
-#include <functional>
-#include "flow/IUDPSocket.h"
-#include "flow/IConnection.h"
-#include "flow/actorcompiler.h" // has to be last include
-
-namespace {
-
-struct UDPWorkload : TestWorkload {
-	constexpr static auto NAME = "UDPWorkload";
-	// config
-	Key keyPrefix;
-	double runFor;
-	int minPort, maxPort;
-	//  members
-	NetworkAddress serverAddress;
-	Reference<IUDPSocket> serverSocket;
-	std::unordered_map<NetworkAddress, unsigned> sent, received, acked, successes;
-	PromiseStream<NetworkAddress> toAck;
-
-	UDPWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
-		keyPrefix = getOption(options, "keyPrefix"_sr, "/udp/"_sr);
-		runFor = getOption(options, "runFor"_sr, 60.0);
-		minPort = getOption(options, "minPort"_sr, 5000);
-		maxPort = getOption(options, "minPort"_sr, 6000);
-		for (auto p : { minPort, maxPort }) {
-			ASSERT(p > 0 && p < std::numeric_limits<unsigned short>::max());
-		}
-	}
-
-	ACTOR static Future<Void> _setup(UDPWorkload* self, Database cx) {
-		state NetworkAddress localAddress(g_network->getLocalAddress().ip,
-		                                  deterministicRandom()->randomInt(self->minPort, self->maxPort + 1),
-		                                  true,
-		                                  false);
-		state Key key = self->keyPrefix.withSuffix(BinaryWriter::toValue(self->clientId, Unversioned()));
-		state Value serializedLocalAddress = BinaryWriter::toValue(localAddress, IncludeVersion());
-		state ReadYourWritesTransaction tr(cx);
-		Reference<IUDPSocket> s = wait(INetworkConnections::net()->createUDPSocket(localAddress.isV6()));
-		self->serverSocket = std::move(s);
-		self->serverSocket->bind(localAddress);
-		self->serverAddress = localAddress;
-		loop {
-			try {
-				Optional<Value> v = wait(tr.get(key));
-				if (v.present()) {
-					return Void();
-				}
-				tr.set(key, serializedLocalAddress);
-				wait(tr.commit());
-				return Void();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-	}
-	Future<Void> setup(Database const& cx) override { return _setup(this, cx); }
-
-	class Message {
-		int _type = 0;
-
-	public:
-		enum class Type : uint8_t { PING, PONG };
-		Message() {}
-		explicit Message(Type t) {
-			switch (t) {
-			case Type::PING:
-				_type = 0;
-				break;
-			case Type::PONG:
-				_type = 1;
-				break;
-			default:
-				UNSTOPPABLE_ASSERT(false);
-			}
-		}
-
-		Type type() const {
-			switch (_type) {
-			case 0:
-				return Type::PING;
-			case 1:
-				return Type::PONG;
-			default:
-				UNSTOPPABLE_ASSERT(false);
-			}
-		}
-
-		template <class Ar>
-		void serialize(Ar& ar) {
-			serializer(ar, _type);
-		}
-	};
-
-	static Message ping() { return Message{ Message::Type::PING }; }
-	static Message pong() { return Message{ Message::Type::PONG }; }
-
-	ACTOR static Future<Void> _receiver(UDPWorkload* self) {
-		state Standalone<StringRef> packetString = makeString(IUDPSocket::MAX_PACKET_SIZE);
-		state uint8_t* packet = mutateString(packetString);
-		state NetworkAddress peerAddress;
-		loop {
-			int sz = wait(self->serverSocket->receiveFrom(packet, packet + IUDPSocket::MAX_PACKET_SIZE, &peerAddress));
-			auto msg = BinaryReader::fromStringRef<Message>(packetString.substr(0, sz), IncludeVersion());
-			if (msg.type() == Message::Type::PONG) {
-				self->successes[peerAddress] += 1;
-			} else if (msg.type() == Message::Type::PING) {
-				self->received[peerAddress] += 1;
-				self->toAck.send(peerAddress);
-			} else {
-				UNSTOPPABLE_ASSERT(false);
-			}
-		}
-	}
-
-	ACTOR static Future<Void> serverSender(UDPWorkload* self, std::vector<NetworkAddress>* remotes) {
-		state Standalone<StringRef> packetString;
-		state NetworkAddress peer;
-		loop {
-			choose {
-				when(wait(delay(0.1))) {
-					peer = deterministicRandom()->randomChoice(*remotes);
-					packetString = BinaryWriter::toValue(Message{ Message::Type::PING }, IncludeVersion());
-					self->sent[peer] += 1;
-				}
-				when(NetworkAddress p = waitNext(self->toAck.getFuture())) {
-					peer = p;
-					packetString = BinaryWriter::toValue(pong(), IncludeVersion());
-					self->acked[peer] += 1;
-				}
-			}
-			int res = wait(self->serverSocket->sendTo(packetString.begin(), packetString.end(), peer));
-			ASSERT(res == packetString.size());
-		}
-	}
-
-	ACTOR static Future<Void> clientReceiver(UDPWorkload* self, Reference<IUDPSocket> socket, Future<Void> done) {
-		state Standalone<StringRef> packetString = makeString(IUDPSocket::MAX_PACKET_SIZE);
-		state uint8_t* packet = mutateString(packetString);
-		state NetworkAddress peer;
-		state Future<Void> finished = Never();
-		loop {
-			choose {
-				when(int sz = wait(socket->receiveFrom(packet, packet + IUDPSocket::MAX_PACKET_SIZE, &peer))) {
-					auto res = BinaryReader::fromStringRef<Message>(packetString.substr(0, sz), IncludeVersion());
-					ASSERT(res.type() == Message::Type::PONG);
-					self->successes[peer] += 1;
-				}
-				when(wait(done)) {
-					finished = delay(1.0);
-					done = Never();
-				}
-				when(wait(finished)) {
-					return Void();
-				}
-			}
-		}
-	}
-
-	ACTOR static Future<Void> clientSender(UDPWorkload* self, std::vector<NetworkAddress>* remotes) {
-		state AsyncVar<Reference<IUDPSocket>> socket;
-		state Standalone<StringRef> sendString;
-		state ActorCollection actors(false);
-		state NetworkAddress peer;
-
-		loop {
-			choose {
-				when(wait(delay(0.1))) {}
-				when(wait(actors.getResult())) {
-					UNSTOPPABLE_ASSERT(false);
-				}
-			}
-			if (!socket.get().isValid() || deterministicRandom()->random01() < 0.05) {
-				peer = deterministicRandom()->randomChoice(*remotes);
-				Reference<IUDPSocket> s = wait(INetworkConnections::net()->createUDPSocket(peer));
-				socket.set(s);
-				socket = s;
-				actors.add(clientReceiver(self, socket.get(), socket.onChange()));
-			}
-			sendString = BinaryWriter::toValue(ping(), IncludeVersion());
-			int res = wait(socket.get()->send(sendString.begin(), sendString.end()));
-			ASSERT(res == sendString.size());
-			self->sent[peer] += 1;
-		}
-	}
-
-	ACTOR static Future<Void> _start(UDPWorkload* self, Database cx) {
-		state ReadYourWritesTransaction tr(cx);
-		state std::vector<NetworkAddress> remotes;
-		loop {
-			try {
-				RangeResult range = wait(tr.getRange(prefixRange(self->keyPrefix), CLIENT_KNOBS->TOO_MANY));
-				ASSERT(!range.more);
-				for (auto const& p : range) {
-					auto cID = BinaryReader::fromStringRef<decltype(self->clientId)>(
-					    p.key.removePrefix(self->keyPrefix), Unversioned());
-					if (cID != self->clientId) {
-						remotes.emplace_back(BinaryReader::fromStringRef<NetworkAddress>(p.value, IncludeVersion()));
-					}
-				}
-				break;
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
-		}
-		wait(clientSender(self, &remotes) && serverSender(self, &remotes) && _receiver(self));
-		UNSTOPPABLE_ASSERT(false);
-		return Void();
-	}
-	Future<Void> start(Database const& cx) override { return delay(runFor) || _start(this, cx); }
-	Future<bool> check(Database const& cx) override { return true; }
-	void getMetrics(std::vector<PerfMetric>& m) override {
-		unsigned totalReceived = 0, totalSent = 0, totalAcked = 0, totalSuccess = 0;
-		for (const auto& p : sent) {
-			totalSent += p.second;
-		}
-		for (const auto& p : received) {
-			totalReceived += p.second;
-		}
-		for (const auto& p : acked) {
-			totalAcked += p.second;
-		}
-		for (const auto& p : successes) {
-			totalSuccess += p.second;
-		}
-		m.emplace_back("Sent", totalSent, Averaged::False);
-		m.emplace_back("Received", totalReceived, Averaged::False);
-		m.emplace_back("Acknknowledged", totalAcked, Averaged::False);
-		m.emplace_back("Successes", totalSuccess, Averaged::False);
-	}
-};
-
-} // namespace
-
-WorkloadFactory<UDPWorkload> UDPWorkloadFactory;
diff --git a/fdbserver/workloads/UDPWorkload.cpp b/fdbserver/workloads/UDPWorkload.cpp
new file mode 100644
index 00000000000..28b99593a0b
--- /dev/null
+++ b/fdbserver/workloads/UDPWorkload.cpp
@@ -0,0 +1,266 @@
+/*
+ * UDPWorkload.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbserver/tester/workloads.h"
+#include "flow/ActorCollection.h"
+#include "flow/Arena.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+#include "flow/serialize.h"
+#include <functional>
+#include <limits>
+#include <unordered_map>
+#include <vector>
+#include <memory>
+#include "flow/IUDPSocket.h"
+#include "flow/IConnection.h"
+
+namespace {
+
+struct UDPWorkload : TestWorkload {
+	constexpr static auto NAME = "UDPWorkload";
+	// config
+	Key keyPrefix;
+	double runFor;
+	int minPort, maxPort;
+	//  members
+	NetworkAddress serverAddress;
+	Reference<IUDPSocket> serverSocket;
+	std::unordered_map<NetworkAddress, unsigned> sent, received, acked, successes;
+	PromiseStream<NetworkAddress> toAck;
+
+	explicit(false) UDPWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		keyPrefix = getOption(options, "keyPrefix"_sr, "/udp/"_sr);
+		runFor = getOption(options, "runFor"_sr, 60.0);
+		minPort = getOption(options, "minPort"_sr, 5000);
+		maxPort = getOption(options, "minPort"_sr, 6000);
+		for (auto p : { minPort, maxPort }) {
+			ASSERT(p > 0 && p < std::numeric_limits<unsigned short>::max());
+		}
+	}
+
+	Future<Void> _setup(Database cx) {
+		NetworkAddress localAddress(
+		    g_network->getLocalAddress().ip, deterministicRandom()->randomInt(minPort, maxPort + 1), true, false);
+		Key key = keyPrefix.withSuffix(BinaryWriter::toValue(clientId, Unversioned()));
+		Value serializedLocalAddress = BinaryWriter::toValue(localAddress, IncludeVersion());
+		ReadYourWritesTransaction tr(cx);
+		auto socketFuture = INetworkConnections::net()->createUDPSocket(localAddress.isV6());
+		serverSocket = co_await socketFuture;
+		serverSocket->bind(localAddress);
+		serverAddress = localAddress;
+		Error err;
+		while (true) {
+			try {
+				Optional<Value> v = co_await tr.get(key);
+				if (v.present()) {
+					co_return;
+				}
+				tr.set(key, serializedLocalAddress);
+				co_await tr.commit();
+				co_return;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+	}
+	Future<Void> setup(Database const& cx) override { return _setup(cx); }
+
+	class Message {
+		int _type = 0;
+
+	public:
+		enum class Type : uint8_t { PING, PONG };
+		Message() = default;
+		explicit Message(Type t) {
+			switch (t) {
+			case Type::PING:
+				_type = 0;
+				break;
+			case Type::PONG:
+				_type = 1;
+				break;
+			default:
+				UNSTOPPABLE_ASSERT(false);
+			}
+		}
+
+		Type type() const {
+			switch (_type) {
+			case 0:
+				return Type::PING;
+			case 1:
+				return Type::PONG;
+			default:
+				UNSTOPPABLE_ASSERT(false);
+			}
+		}
+
+		template <class Ar>
+		void serialize(Ar& ar) {
+			serializer(ar, _type);
+		}
+	};
+
+	static Message ping() { return Message{ Message::Type::PING }; }
+	static Message pong() { return Message{ Message::Type::PONG }; }
+
+	// TODO: Overload timeout to support FutureStream to replace this helper function
+	static Future<NetworkAddress> nextToAck(FutureStream<NetworkAddress> toAck) { co_return co_await toAck; }
+
+	Future<Void> _receiver() {
+		Standalone<StringRef> packetString = makeString(IUDPSocket::MAX_PACKET_SIZE);
+		uint8_t* packet = mutateString(packetString);
+		NetworkAddress peerAddress;
+		while (true) {
+			int sz = co_await serverSocket->receiveFrom(packet, packet + IUDPSocket::MAX_PACKET_SIZE, &peerAddress);
+			auto msg = BinaryReader::fromStringRef<Message>(packetString.substr(0, sz), IncludeVersion());
+			if (msg.type() == Message::Type::PONG) {
+				successes[peerAddress] += 1;
+			} else if (msg.type() == Message::Type::PING) {
+				received[peerAddress] += 1;
+				toAck.send(peerAddress);
+			} else {
+				UNSTOPPABLE_ASSERT(false);
+			}
+		}
+	}
+
+	Future<Void> serverSender(std::vector<NetworkAddress>* remotes) {
+		Standalone<StringRef> packetString;
+		NetworkAddress peer;
+		while (true) {
+			auto const p = co_await timeout(nextToAck(toAck.getFuture()), 0.1);
+			if (p.present()) {
+				peer = p.get();
+				packetString = BinaryWriter::toValue(pong(), IncludeVersion());
+				acked[peer] += 1;
+			} else {
+				peer = deterministicRandom()->randomChoice(*remotes);
+				packetString = BinaryWriter::toValue(Message{ Message::Type::PING }, IncludeVersion());
+				sent[peer] += 1;
+			}
+			int res = co_await serverSocket->sendTo(packetString.begin(), packetString.end(), peer);
+			ASSERT_EQ(res, packetString.size());
+		}
+	}
+
+	Future<Void> clientReceiver(Reference<IUDPSocket> socket, Future<Void> done) {
+		Standalone<StringRef> packetString = makeString(IUDPSocket::MAX_PACKET_SIZE);
+		uint8_t* packet = mutateString(packetString);
+		NetworkAddress peer;
+		Future<Void> finished = Never();
+		while (true) {
+			auto const res =
+			    co_await race(socket->receiveFrom(packet, packet + IUDPSocket::MAX_PACKET_SIZE, &peer), done, finished);
+			if (res.index() == 0) {
+				int const sz = std::get<0>(res);
+				auto resp = BinaryReader::fromStringRef<Message>(packetString.substr(0, sz), IncludeVersion());
+				ASSERT(resp.type() == Message::Type::PONG);
+				successes[peer] += 1;
+			} else if (res.index() == 1) {
+				// done
+				finished = delay(1.0);
+				done = Never();
+			} else {
+				// finished
+				co_return;
+			}
+		}
+	}
+
+	Future<Void> clientSender(std::vector<NetworkAddress>* remotes) {
+		AsyncVar<Reference<IUDPSocket>> socket;
+		Standalone<StringRef> sendString;
+		ActorCollection actors(false);
+		NetworkAddress peer;
+
+		while (true) {
+			auto delayResult = co_await race(delay(0.1), actors.getResult());
+			UNSTOPPABLE_ASSERT(delayResult.index() == 0);
+			if (!socket.get().isValid() || deterministicRandom()->random01() < 0.05) {
+				peer = deterministicRandom()->randomChoice(*remotes);
+				Reference<IUDPSocket> s = co_await INetworkConnections::net()->createUDPSocket(peer);
+				socket.set(s);
+				actors.add(clientReceiver(socket.get(), socket.onChange()));
+			}
+			sendString = BinaryWriter::toValue(ping(), IncludeVersion());
+			int res = co_await socket.get()->send(sendString.begin(), sendString.end());
+			ASSERT_EQ(res, sendString.size());
+			sent[peer] += 1;
+		}
+	}
+
+	Future<Void> _start(Database cx) {
+		ReadYourWritesTransaction tr(cx);
+		std::vector<NetworkAddress> remotes;
+		while (true) {
+			Error err;
+			try {
+				RangeResult range = co_await tr.getRange(prefixRange(keyPrefix), CLIENT_KNOBS->TOO_MANY);
+				ASSERT(!range.more);
+				for (auto const& p : range) {
+					auto cID =
+					    BinaryReader::fromStringRef<decltype(clientId)>(p.key.removePrefix(keyPrefix), Unversioned());
+					if (cID != clientId) {
+						remotes.emplace_back(BinaryReader::fromStringRef<NetworkAddress>(p.value, IncludeVersion()));
+					}
+				}
+				break;
+			} catch (Error& e) {
+				err = e;
+			}
+			co_await tr.onError(err);
+		}
+		co_await (clientSender(&remotes) && serverSender(&remotes) && _receiver());
+		UNSTOPPABLE_ASSERT(false);
+	}
+
+	Future<Void> start(Database const& cx) override { return delay(runFor) || _start(cx); }
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {
+		unsigned totalReceived = 0, totalSent = 0, totalAcked = 0, totalSuccess = 0;
+		for (const auto& [_destination, sentCount] : sent) {
+			totalSent += sentCount;
+		}
+		for (const auto& [_source, receivedCount] : received) {
+			totalReceived += receivedCount;
+		}
+		for (const auto& [_destination, ackedCount] : acked) {
+			totalAcked += ackedCount;
+		}
+		for (const auto& [_destination, successCount] : successes) {
+			totalSuccess += successCount;
+		}
+		m.emplace_back("Sent", totalSent, Averaged::False);
+		m.emplace_back("Received", totalReceived, Averaged::False);
+		m.emplace_back("Acknknowledged", totalAcked, Averaged::False);
+		m.emplace_back("Successes", totalSuccess, Averaged::False);
+	}
+};
+
+} // namespace
+
+WorkloadFactory<UDPWorkload> UDPWorkloadFactory;
diff --git a/fdbserver/workloads/UnitPerf.actor.cpp b/fdbserver/workloads/UnitPerf.cpp
similarity index 74%
rename from fdbserver/workloads/UnitPerf.actor.cpp
rename to fdbserver/workloads/UnitPerf.cpp
index a38a223b548..dcf73b5e904 100644
--- a/fdbserver/workloads/UnitPerf.actor.cpp
+++ b/fdbserver/workloads/UnitPerf.cpp
@@ -1,5 +1,5 @@
 /*
- * UnitPerf.actor.cpp
+ * UnitPerf.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,41 +18,37 @@
  * limitations under the License.
  */
 
-#include "fdbrpc/ActorFuzz.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
+#include "fdbserver/tester/workloads.h"
 
-ACTOR Future<Void> sleepyActor(double interval, int* counter) {
-	loop {
-		wait(delay(interval));
+Future<Void> sleepyActor(double interval, int* counter) {
+	while (true) {
+		co_await delay(interval);
 		++*counter;
 	}
 }
 
-ACTOR Future<Void> unitPerfTest() {
+Future<Void> unitPerfTest() {
 	printf("\n");
 
-	state int counter = 0;
-	state std::vector<Future<Void>> sleepy;
+	int counter = 0;
+	std::vector<Future<Void>> sleepy;
 	sleepy.reserve(100000);
 	for (int i = 0; i < 100000; i++)
 		sleepy.push_back(sleepyActor(.1, &counter));
 
-	wait(delay(10));
+	co_await delay(10);
 	sleepy.clear();
 	TraceEvent("Completed").detail("Count", counter);
 	printf("Completed: %d\n", counter);
 
 	printf("\n");
-	return Void();
 }
 
 struct UnitPerfWorkload : TestWorkload {
 	static constexpr auto NAME = "UnitPerf";
 	bool enabled;
 
-	UnitPerfWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit UnitPerfWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
 	}
 
diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.cpp
similarity index 79%
rename from fdbserver/workloads/UnitTests.actor.cpp
rename to fdbserver/workloads/UnitTests.cpp
index 1a6c7f783a1..3208dabe242 100644
--- a/fdbserver/workloads/UnitTests.actor.cpp
+++ b/fdbserver/workloads/UnitTests.cpp
@@ -1,5 +1,5 @@
 /*
- * UnitTests.actor.cpp
+ * UnitTests.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,26 +18,24 @@
  * limitations under the License.
  */
 
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 void forceLinkIndexedSetTests();
 void forceLinkDequeTests();
 void forceLinkFlowTests();
 void forceLinkCoroTests();
-void forceLinkVersionedMapTests();
 void forceLinkMemcpyTests();
-void forceLinkMemcpyPerfTests();
 void forceLinkStreamCipherTests();
-void forceLinkParallelStreamTests();
 void forceLinkSimExternalConnectionTests();
 void forceLinkMutationLogReaderTests();
 void forceLinkIThreadPoolTests();
+void forceLinkNet2FileSystemTests();
 void forceLinkJsonWebKeySetTests();
 void forceLinkVersionVectorTests();
 void forceLinkRESTClientTests();
 void forceLinkRESTUtilsTests();
+void forceLinkCompressedIntTests();
 void forceLinkCompressionUtilsTest();
 void forceLinkAtomicTests();
 void forceLinkIdempotencyIdTests();
@@ -50,6 +48,12 @@ void forceLinkActorFuzzUnitTests();
 void forceLinkGrpcTests();
 void forceLinkGrpcTests2();
 void forceLinkSimpleCounterTests();
+void forceLinkLogSystemRecoveryTests();
+void forceLinkIPagerTests();
+void forceLinkMockS3ServerTests();
+void forceLinkAuditUtilsTests();
+void forceLinkClusterHealthMonitorTests();
+void forceLinkGrvQueueDelayTests();
 
 struct UnitTestWorkload : TestWorkload {
 	static constexpr auto NAME = "UnitTests";
@@ -64,7 +68,7 @@ struct UnitTestWorkload : TestWorkload {
 	PerfIntCounter testsAvailable, testsExecuted, testsFailed;
 	PerfDoubleCounter totalWallTime, totalSimTime;
 
-	UnitTestWorkload(WorkloadContext const& wcx)
+	explicit UnitTestWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), testsAvailable("Test Cases Available"), testsExecuted("Test Cases Executed"),
 	    testsFailed("Test Cases Failed"), totalWallTime("Total wall clock time (s)"),
 	    totalSimTime("Total flow time (s)") {
@@ -88,7 +92,7 @@ struct UnitTestWorkload : TestWorkload {
 
 		// Consume all remaining options as testParams which the unit test can access
 		for (auto& kv : options) {
-			if (kv.value.size() != 0) {
+			if (!kv.value.empty()) {
 				testParams.set(kv.key.toString(), getOption(options, kv.key, StringRef()).toString());
 			}
 		}
@@ -97,18 +101,17 @@ struct UnitTestWorkload : TestWorkload {
 		forceLinkDequeTests();
 		forceLinkFlowTests();
 		forceLinkCoroTests();
-		forceLinkVersionedMapTests();
 		forceLinkMemcpyTests();
-		forceLinkMemcpyPerfTests();
 		forceLinkStreamCipherTests();
-		forceLinkParallelStreamTests();
 		forceLinkSimExternalConnectionTests();
 		forceLinkMutationLogReaderTests();
 		forceLinkIThreadPoolTests();
+		forceLinkNet2FileSystemTests();
 		forceLinkJsonWebKeySetTests();
 		forceLinkVersionVectorTests();
 		forceLinkRESTClientTests();
 		forceLinkRESTUtilsTests();
+		forceLinkCompressedIntTests();
 		forceLinkCompressionUtilsTest();
 		forceLinkAtomicTests();
 		forceLinkIdempotencyIdTests();
@@ -118,6 +121,12 @@ struct UnitTestWorkload : TestWorkload {
 		forceLinkRandomKeyValueUtilsTests();
 		forceLinkActorFuzzUnitTests();
 		forceLinkSimpleCounterTests();
+		forceLinkLogSystemRecoveryTests();
+		forceLinkIPagerTests();
+		forceLinkMockS3ServerTests();
+		forceLinkAuditUtilsTests();
+		forceLinkClusterHealthMonitorTests();
+		forceLinkGrvQueueDelayTests();
 
 #ifdef FLOW_GRPC_ENABLED
 		forceLinkGrpcTests();
@@ -131,7 +140,7 @@ struct UnitTestWorkload : TestWorkload {
 	}
 	Future<Void> start(Database const& cx) override {
 		if (enabled)
-			return runUnitTests(this);
+			return runUnitTests();
 		return Void();
 	}
 	Future<bool> check(Database const& cx) override { return testsFailed.getValue() == 0; }
@@ -148,7 +157,7 @@ struct UnitTestWorkload : TestWorkload {
 			return false;
 		}
 
-		for (auto ignorePatt : testsIgnored) {
+		for (const auto& ignorePatt : testsIgnored) {
 			if (StringRef(testName).startsWith(ignorePatt)) {
 				return false;
 			}
@@ -157,12 +166,12 @@ struct UnitTestWorkload : TestWorkload {
 		return true;
 	}
 
-	ACTOR static Future<Void> runUnitTests(UnitTestWorkload* self) {
-		state std::vector<UnitTest*> tests;
+	Future<Void> runUnitTests() {
+		std::vector<UnitTest*> tests;
 
 		for (auto test = g_unittests.tests; test != nullptr; test = test->next) {
-			if (self->testMatched(test->name)) {
-				++self->testsAvailable;
+			if (testMatched(test->name)) {
+				++testsAvailable;
 				tests.push_back(test);
 			}
 		}
@@ -173,21 +182,21 @@ struct UnitTestWorkload : TestWorkload {
 
 		fprintf(stdout, "Found %zu tests\n", tests.size());
 
-		if (tests.size() == 0) {
+		if (tests.empty()) {
 			TraceEvent(SevError, "NoMatchingUnitTests")
-			    .detail("TestPattern", self->testPattern)
-			    .detail("TestsIgnored", self->testsIgnored);
-			++self->testsFailed;
-			return Void();
+			    .detail("TestPattern", testPattern)
+			    .detail("TestsIgnored", testsIgnored);
+			++testsFailed;
+			co_return;
 		}
 
 		deterministicRandom()->randomShuffle(tests);
-		if (self->testRunLimit > 0 && tests.size() > self->testRunLimit)
-			tests.resize(self->testRunLimit);
+		if (testRunLimit > 0 && tests.size() > testRunLimit)
+			tests.resize(testRunLimit);
 
-		state std::vector<UnitTest*>::iterator t;
+		std::vector<UnitTest*>::iterator t;
 		for (t = tests.begin(); t != tests.end(); ++t) {
-			state UnitTest* test = *t;
+			UnitTest* test = *t;
 			printf("Testing %s\n", test->name);
 
 			TraceEvent(SevInfo, "RunningUnitTest")
@@ -196,26 +205,26 @@ struct UnitTestWorkload : TestWorkload {
 			    .detail("Line", test->line)
 			    .detail("Rand", deterministicRandom()->randomInt(0, 100001));
 
-			state Error result = success();
-			state double start_now = now();
-			state double start_timer = timer();
+			Error result = success();
+			double start_now = now();
+			double start_timer = timer();
 
-			platform::createDirectory(self->testParams.getDataDir());
+			platform::createDirectory(testParams.getDataDir());
 			try {
-				wait(test->func(self->testParams));
+				co_await test->func(testParams);
 			} catch (Error& e) {
-				++self->testsFailed;
+				++testsFailed;
 				result = e;
 			}
-			if (self->cleanupAfterTests) {
-				platform::eraseDirectoryRecursive(self->testParams.getDataDir());
+			if (cleanupAfterTests) {
+				platform::eraseDirectoryRecursive(testParams.getDataDir());
 			}
-			++self->testsExecuted;
+			++testsExecuted;
 			double wallTime = timer() - start_timer;
 			double simTime = now() - start_now;
 
-			self->totalWallTime += wallTime;
-			self->totalSimTime += simTime;
+			totalWallTime += wallTime;
+			totalSimTime += simTime;
 			TraceEvent(result.code() != error_code_success ? SevError : SevInfo, "UnitTest")
 			    .errorUnsuppressed(result)
 			    .detail("Name", test->name)
@@ -224,14 +233,12 @@ struct UnitTestWorkload : TestWorkload {
 			    .detail("WallTime", wallTime)
 			    .detail("FlowTime", simTime);
 		}
-
-		return Void();
 	}
 };
 
 WorkloadFactory<UnitTestWorkload> UnitTestWorkloadFactory;
 
 TEST_CASE("/fdbserver/UnitTestWorkload/long delay") {
-	wait(delay(60));
-	return Void();
+	co_await delay(60);
+	co_return;
 }
diff --git a/fdbserver/workloads/Unreadable.actor.cpp b/fdbserver/workloads/Unreadable.cpp
similarity index 92%
rename from fdbserver/workloads/Unreadable.actor.cpp
rename to fdbserver/workloads/Unreadable.cpp
index de94add1410..7156752fc9c 100644
--- a/fdbserver/workloads/Unreadable.actor.cpp
+++ b/fdbserver/workloads/Unreadable.cpp
@@ -1,5 +1,5 @@
 /*
- * Unreadable.actor.cpp
+ * Unreadable.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct UnreadableWorkload : TestWorkload {
 	static constexpr auto NAME = "Unreadable";
@@ -31,7 +30,7 @@ struct UnreadableWorkload : TestWorkload {
 	double testDuration;
 	std::vector<Future<Void>> clients;
 
-	UnreadableWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit UnreadableWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)100000);
 	}
@@ -40,7 +39,7 @@ struct UnreadableWorkload : TestWorkload {
 
 	Future<Void> start(Database const& cx) override {
 		if (clientId == 0)
-			return _start(cx, this);
+			return _start(cx);
 		return Void();
 	}
 
@@ -292,26 +291,24 @@ struct UnreadableWorkload : TestWorkload {
 		           .present());
 	}
 
-	ACTOR Future<Void> _start(Database cx, UnreadableWorkload* self) {
-		state int testCount = 0;
-		state Reverse reverse = Reverse::False;
-		state Snapshot snapshot = Snapshot::False;
-		for (; testCount < 100; testCount++) {
+	Future<Void> _start(Database cx) {
+		Reverse reverse = Reverse::False;
+		Snapshot snapshot = Snapshot::False;
+		for (int testCount = 0; testCount < 100; ++testCount) {
 			//TraceEvent("RYWT_Start").detail("TestCount", testCount);
-			state ReadYourWritesTransaction tr(cx);
-			state Arena arena;
-
-			state std::map<KeyRef, ValueRef> setMap;
-			state KeyRangeMap<bool> unreadableMap;
-
-			state int opCount = 0;
-			state KeyRangeRef range;
-			state KeyRef key;
-			state ValueRef value;
-			state int limit;
-			state KeySelectorRef begin;
-			state KeySelectorRef end;
-			state bool bypassUnreadable = deterministicRandom()->coinflip();
+			ReadYourWritesTransaction tr(cx);
+			Arena arena;
+
+			std::map<KeyRef, ValueRef> setMap;
+			KeyRangeMap<bool> unreadableMap;
+
+			KeyRangeRef range;
+			KeyRef key;
+			ValueRef value;
+			int limit{ 0 };
+			KeySelectorRef begin;
+			KeySelectorRef end;
+			bool bypassUnreadable = deterministicRandom()->coinflip();
 			if (bypassUnreadable) {
 				tr.setOption(FDBTransactionOptions::BYPASS_UNREADABLE);
 			}
@@ -319,7 +316,7 @@ struct UnreadableWorkload : TestWorkload {
 			setMap[normalKeys.begin] = ValueRef();
 			setMap[normalKeys.end] = ValueRef();
 
-			for (; opCount < 500; opCount++) {
+			for (int opCount = 0; opCount < 500; ++opCount) {
 				int r = deterministicRandom()->randomInt(0, 19);
 				if (r <= 10) {
 					key = RandomTestImpl::getRandomKey(arena);
@@ -363,7 +360,7 @@ struct UnreadableWorkload : TestWorkload {
 						tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
 
 					ErrorOr<RangeResult> value =
-					    wait(errorOr(tr.getRange(range, CLIENT_KNOBS->TOO_MANY, snapshot, reverse)));
+					    co_await errorOr(tr.getRange(range, CLIENT_KNOBS->TOO_MANY, snapshot, reverse));
 
 					if (snapshot)
 						tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_ENABLE);
@@ -398,7 +395,7 @@ struct UnreadableWorkload : TestWorkload {
 						tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
 
 					//TraceEvent("RYWT_GetRangeBefore").detail("Reverse", reverse).detail("Begin", begin.toString()).detail("End", end.toString()).detail("Limit", limit);
-					ErrorOr<RangeResult> value = wait(errorOr(tr.getRange(begin, end, limit, snapshot, reverse)));
+					ErrorOr<RangeResult> value = co_await errorOr(tr.getRange(begin, end, limit, snapshot, reverse));
 
 					if (snapshot)
 						tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_ENABLE);
@@ -441,7 +438,7 @@ struct UnreadableWorkload : TestWorkload {
 					if (snapshot)
 						tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
 
-					ErrorOr<Optional<Value>> value = wait(errorOr(tr.get(key, snapshot)));
+					ErrorOr<Optional<Value>> value = co_await errorOr(tr.get(key, snapshot));
 
 					if (snapshot)
 						tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_ENABLE);
@@ -469,8 +466,6 @@ struct UnreadableWorkload : TestWorkload {
 				}
 			}
 		}
-
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/ValidateStorage.actor.cpp b/fdbserver/workloads/ValidateStorage.cpp
similarity index 62%
rename from fdbserver/workloads/ValidateStorage.actor.cpp
rename to fdbserver/workloads/ValidateStorage.cpp
index c9a4ac367a8..e1494e9a23d 100644
--- a/fdbserver/workloads/ValidateStorage.actor.cpp
+++ b/fdbserver/workloads/ValidateStorage.cpp
@@ -1,5 +1,5 @@
 /*
- * ValidateStorage.actor.cpp
+ * ValidateStorage.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,20 +19,17 @@
  */
 
 #include "fdbclient/Audit.h"
-#include "fdbclient/AuditUtils.actor.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/AuditUtils.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/QuietDatabase.h"
+#include "fdbserver/core/Knobs.h"
+#include "fdbserver/core/QuietDatabase.h"
 #include "fdbrpc/simulator.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/flow.h"
 #include <cstdint>
-#include <limits>
-
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace {
 std::string printValue(const ErrorOr<Optional<Value>>& value) {
@@ -91,7 +88,7 @@ struct ValidateStorage : TestWorkload {
 		pass = false;
 	}
 
-	ValidateStorage(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
+	explicit ValidateStorage(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId), pass(true) {}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
@@ -99,27 +96,28 @@ struct ValidateStorage : TestWorkload {
 		if (!enabled) {
 			return Void();
 		}
-		return _start(this, cx);
+		return _start(cx);
 	}
 
-	ACTOR Future<UID> triggerAuditStorageForType(Database cx,
-	                                             AuditType type,
-	                                             std::string context,
-	                                             KeyRange auditRange = allKeys) {
+	Future<UID> triggerAuditStorageForType(Database cx,
+	                                       AuditType type,
+	                                       std::string context,
+	                                       KeyRange auditRange = allKeys) {
 		// Send audit request until the cluster accepts the request
-		state UID auditId;
+		UID auditId;
 		std::vector<KeyValueStoreType> storageEngineCollection = { KeyValueStoreType::SSD_ROCKSDB_V1,
 			                                                       KeyValueStoreType::SSD_SHARDED_ROCKSDB,
 			                                                       KeyValueStoreType::SSD_BTREE_V2,
 			                                                       KeyValueStoreType::END };
-		state KeyValueStoreType storageEngine = deterministicRandom()->randomChoice(storageEngineCollection);
-		loop {
+		KeyValueStoreType storageEngine = deterministicRandom()->randomChoice(storageEngineCollection);
+		while (true) {
+			Error err;
 			try {
-				UID auditId_ = wait(auditStorage(cx->getConnectionRecord(),
-				                                 auditRange,
-				                                 type,
-				                                 storageEngine,
-				                                 /*timeoutSecond=*/300));
+				UID auditId_ = co_await auditStorage(cx->getConnectionRecord(),
+				                                     auditRange,
+				                                     type,
+				                                     storageEngine,
+				                                     /*timeoutSecond=*/300);
 				auditId = auditId_;
 				TraceEvent("TestAuditStorageTriggered")
 				    .detail("Context", context)
@@ -129,30 +127,32 @@ struct ValidateStorage : TestWorkload {
 				    .detail("AuditRange", auditRange);
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevWarn, "TestAuditStorageError")
-				    .errorUnsuppressed(e)
-				    .detail("Context", context)
-				    .detail("AuditType", type)
-				    .detail("AuditStorageEngine", storageEngine)
-				    .detail("AuditRange", auditRange);
-				if (auditRange.empty() && e.code() == error_code_audit_storage_failed) {
-					break;
-				}
-				wait(delay(1));
+				err = e;
+			}
+			TraceEvent(SevWarn, "TestAuditStorageError")
+			    .errorUnsuppressed(err)
+			    .detail("Context", context)
+			    .detail("AuditType", type)
+			    .detail("AuditStorageEngine", storageEngine)
+			    .detail("AuditRange", auditRange);
+			if (auditRange.empty() && err.code() == error_code_audit_storage_failed) {
+				break;
 			}
+			co_await delay(1);
 		}
-		return auditId;
+		co_return auditId;
 	}
 
-	ACTOR Future<Void> waitAuditStorageUntilComplete(Database cx,
-	                                                 AuditType type,
-	                                                 UID auditId,
-	                                                 std::string context,
-	                                                 bool stopWaitWhenCleared) {
-		state AuditStorageState auditState;
-		loop {
+	Future<Void> waitAuditStorageUntilComplete(Database cx,
+	                                           AuditType type,
+	                                           UID auditId,
+	                                           std::string context,
+	                                           bool stopWaitWhenCleared) {
+		AuditStorageState auditState;
+		while (true) {
+			Error err;
 			try {
-				AuditStorageState auditState_ = wait(getAuditState(cx, type, auditId));
+				AuditStorageState auditState_ = co_await getAuditState(cx, type, auditId);
 				auditState = auditState_;
 				if (auditState.getPhase() == AuditPhase::Complete) {
 					break;
@@ -161,7 +161,7 @@ struct ValidateStorage : TestWorkload {
 					    .detail("Context", context)
 					    .detail("AuditID", auditId)
 					    .detail("AuditType", type);
-					wait(delay(30));
+					co_await delay(30);
 					continue;
 				} else if (auditState.getPhase() == AuditPhase::Error) {
 					break;
@@ -171,37 +171,37 @@ struct ValidateStorage : TestWorkload {
 					UNREACHABLE();
 				}
 			} catch (Error& e) {
-				if (stopWaitWhenCleared && e.code() == error_code_key_not_found) {
-					break; // this audit has been cleared
-				}
-				TraceEvent("TestAuditStorageWaitError")
-				    .errorUnsuppressed(e)
-				    .detail("Context", context)
-				    .detail("AuditID", auditId)
-				    .detail("AuditType", type)
-				    .detail("AuditState", auditState.toString());
-				wait(delay(1));
+				err = e;
 			}
+			if (stopWaitWhenCleared && err.code() == error_code_key_not_found) {
+				break; // this audit has been cleared
+			}
+			TraceEvent("TestAuditStorageWaitError")
+			    .errorUnsuppressed(err)
+			    .detail("Context", context)
+			    .detail("AuditID", auditId)
+			    .detail("AuditType", type)
+			    .detail("AuditState", auditState.toString());
+			co_await delay(1);
 		}
 		TraceEvent("TestAuditStorageEnd")
 		    .detail("Context", context)
 		    .detail("AuditID", auditId)
 		    .detail("AuditType", type)
 		    .detail("AuditState", auditState.toString());
-		return Void();
 	}
 
-	ACTOR Future<Void> checkAuditStorageInternalState(Database cx, AuditType type, UID auditId, std::string context) {
+	Future<Void> checkAuditStorageInternalState(Database cx, AuditType type, UID auditId, std::string context) {
 		// Check the number of existing persisted audits is no more than PERSIST_FINISH_AUDIT_COUNT
-		state Transaction tr(cx);
-		loop {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				state RangeResult res = wait(tr.getRange(auditKeyRange(type), GetRangeLimits()));
+				RangeResult res = co_await tr.getRange(auditKeyRange(type), GetRangeLimits());
 				ASSERT(!res.more);
-				state int i = 0;
-				for (; i < res.size(); ++i) {
+				for (int i = 0; i < res.size(); ++i) {
 					AuditStorageState existingAuditState = decodeAuditStorageState(res[i].value);
 					TraceEvent("TestAuditStorageCheckPersistStateExists")
 					    .detail("Context", context)
@@ -214,12 +214,12 @@ struct ValidateStorage : TestWorkload {
 					       existingAuditState.getPhase() == AuditPhase::Running);
 					if (existingAuditState.getPhase() == AuditPhase::Complete) {
 						if (type == AuditType::ValidateStorageServerShard) {
-							RangeResult serverBasedRes = wait(tr.getRange(
-							    auditServerBasedProgressRangeFor(type, existingAuditState.id), GetRangeLimits()));
+							RangeResult serverBasedRes = co_await tr.getRange(
+							    auditServerBasedProgressRangeFor(type, existingAuditState.id), GetRangeLimits());
 							ASSERT(serverBasedRes.empty() && !serverBasedRes.more);
 						} else {
-							RangeResult rangeBasedRes = wait(tr.getRange(
-							    auditRangeBasedProgressRangeFor(type, existingAuditState.id), GetRangeLimits()));
+							RangeResult rangeBasedRes = co_await tr.getRange(
+							    auditRangeBasedProgressRangeFor(type, existingAuditState.id), GetRangeLimits());
 							ASSERT(rangeBasedRes.empty() && !rangeBasedRes.more);
 						}
 					}
@@ -234,56 +234,57 @@ struct ValidateStorage : TestWorkload {
 					    .detail("Context", context)
 					    .detail("AuditID", auditId)
 					    .detail("AuditType", type);
-					wait(delay(30));
+					co_await delay(30);
 					tr.reset();
 					continue;
 				}
 				break;
 			} catch (Error& e) {
-				TraceEvent("TestAuditStorageCheckPersistStateError")
-				    .errorUnsuppressed(e)
-				    .detail("Context", context)
-				    .detail("AuditID", auditId)
-				    .detail("AuditType", type);
-				wait(tr.onError(e));
+				err = e;
 			}
+			TraceEvent("TestAuditStorageCheckPersistStateError")
+			    .errorUnsuppressed(err)
+			    .detail("Context", context)
+			    .detail("AuditID", auditId)
+			    .detail("AuditType", type);
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> waitCancelAuditStorageUntilComplete(Database cx, AuditType type, UID auditId) {
-		loop {
+	Future<Void> waitCancelAuditStorageUntilComplete(Database cx, AuditType type, UID auditId) {
+		while (true) {
+			Error err;
 			try {
-				state UID auditId_ = wait(cancelAuditStorage(cx->getConnectionRecord(),
-				                                             type,
-				                                             auditId,
-				                                             /*timeoutSecond=*/300));
+				UID auditId_ = co_await cancelAuditStorage(cx->getConnectionRecord(),
+				                                           type,
+				                                           auditId,
+				                                           /*timeoutSecond=*/300);
 				ASSERT(auditId == auditId_);
 				TraceEvent("TestAuditStorageWaitCancelAuditStorageUntilComplete")
 				    .detail("AuditID", auditId)
 				    .detail("AuditType", type);
 				break;
 			} catch (Error& e) {
-				TraceEvent(SevWarn, "TestAuditStorageWaitCancelAuditStorageUntilCompleteError")
-				    .errorUnsuppressed(e)
-				    .detail("AuditID", auditId)
-				    .detail("AuditType", type);
-				wait(delay(1));
+				err = e;
 			}
+			TraceEvent(SevWarn, "TestAuditStorageWaitCancelAuditStorageUntilCompleteError")
+			    .errorUnsuppressed(err)
+			    .detail("AuditID", auditId)
+			    .detail("AuditType", type);
+			co_await delay(1);
 		}
-		return Void();
 	}
 
-	ACTOR Future<Void> _start(ValidateStorage* self, Database cx) {
+	Future<Void> _start(Database cx) {
 		TraceEvent("ValidateStorageTestBegin");
-		state std::map<Key, Value> kvs({ { "TestKeyA"_sr, "TestValueA"_sr },
-		                                 { "TestKeyB"_sr, "TestValueB"_sr },
-		                                 { "TestKeyC"_sr, "TestValueC"_sr },
-		                                 { "TestKeyD"_sr, "TestValueD"_sr },
-		                                 { "TestKeyE"_sr, "TestValueE"_sr },
-		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
+		std::map<Key, Value> kvs({ { "TestKeyA"_sr, "TestValueA"_sr },
+		                           { "TestKeyB"_sr, "TestValueB"_sr },
+		                           { "TestKeyC"_sr, "TestValueC"_sr },
+		                           { "TestKeyD"_sr, "TestValueD"_sr },
+		                           { "TestKeyE"_sr, "TestValueE"_sr },
+		                           { "TestKeyF"_sr, "TestValueF"_sr } });
 
-		Version ver = wait(self->populateData(self, cx, &kvs));
+		Version ver = co_await populateData(this, cx, &kvs);
 
 		TraceEvent("TestValueWritten").detail("AtVersion", ver);
 
@@ -292,65 +293,65 @@ struct ValidateStorage : TestWorkload {
 			disableConnectionFailures("AuditStorage");
 		}
 
-		self->testStringToAuditPhaseFunctionality();
+		testStringToAuditPhaseFunctionality();
 		TraceEvent("TestAuditStorageStringToAuditPhaseFuncionalityDone");
 
-		wait(self->testSSUserDataValidation(self, cx, KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr)));
+		co_await testSSUserDataValidation(this, cx, KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr));
 		TraceEvent("TestAuditStorageValidateValueDone");
 
-		wait(self->testAuditStorageFunctionality(self, cx));
+		co_await testAuditStorageFunctionality(this, cx);
 		TraceEvent("TestAuditStorageFunctionalityDone");
 
-		wait(self->testAuditStorageIDGenerator(self, cx));
+		co_await testAuditStorageIDGenerator(this, cx);
 		TraceEvent("TestAuditStorageIDGeneratorDone");
 
-		wait(self->testAuditStorageConcurrentRunForDifferentType(self, cx));
+		co_await testAuditStorageConcurrentRunForDifferentType(this, cx);
 		TraceEvent("TestAuditStorageConcurrentRunForDifferentTypeDone");
 
-		wait(self->testAuditStorageConcurrentRunForSameType(self, cx));
+		co_await testAuditStorageConcurrentRunForSameType(this, cx);
 		TraceEvent("TestAuditStorageConcurrentRunForSameTypeDone");
 
-		wait(self->testAuditStorageCancellation(self, cx));
+		co_await testAuditStorageCancellation(this, cx);
 		TraceEvent("TestAuditStorageCancellationDone");
 
-		wait(self->testAuditStorageProgress(self, cx));
+		co_await testAuditStorageProgress(this, cx);
 		TraceEvent("TestAuditStorageProgressDone");
 
-		wait(self->testAuditStorageWhenDDSecurityMode(self, cx));
+		co_await testAuditStorageWhenDDSecurityMode(this, cx);
 		TraceEvent("TestAuditStorageWhenDDSecurityModeDone");
 
-		wait(self->testAuditStorageWhenDDBackToNormalMode(self, cx));
+		co_await testAuditStorageWhenDDBackToNormalMode(this, cx);
 		TraceEvent("TestAuditStorageWhenDDBackToNormalModeDone");
-
-		return Void();
 	}
 
-	ACTOR Future<Version> populateData(ValidateStorage* self, Database cx, std::map<Key, Value>* kvs) {
-		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		state Version version;
-		state UID debugID;
+	Future<Version> populateData(ValidateStorage* self, Database cx, std::map<Key, Value>* kvs) {
+		auto tr = makeReference<ReadYourWritesTransaction>(cx);
+		Version version{ 0 };
+		UID debugID;
 
-		loop {
+		while (true) {
 			debugID = deterministicRandom()->randomUniqueID();
+			Error err;
 			try {
 				tr->debugTransaction(debugID);
 				for (const auto& [key, value] : *kvs) {
 					tr->set(key, value);
 				}
-				wait(tr->commit());
+				co_await tr->commit();
 				version = tr->getCommittedVersion();
 				break;
 			} catch (Error& e) {
-				TraceEvent("TestCommitError").errorUnsuppressed(e);
-				wait(tr->onError(e));
+				err = e;
 			}
+			TraceEvent("TestCommitError").errorUnsuppressed(err);
+			co_await tr->onError(err);
 		}
 
 		TraceEvent("PopulateTestDataDone")
 		    .detail("CommitVersion", tr->getCommittedVersion())
 		    .detail("DebugID", debugID);
 
-		return version;
+		co_return version;
 	}
 
 	void testStringToAuditPhaseFunctionality() {
@@ -388,22 +389,23 @@ struct ValidateStorage : TestWorkload {
 		return;
 	}
 
-	ACTOR Future<Void> testSSUserDataValidation(ValidateStorage* self, Database cx, KeyRange range) {
+	Future<Void> testSSUserDataValidation(ValidateStorage* self, Database cx, KeyRange range) {
 		TraceEvent("TestSSUserDataValidationBegin").detail("Range", range);
-		state Transaction tr(cx);
+		Transaction tr(cx);
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-		state int retryCount = 0;
-		loop {
+		int retryCount = 0;
+		while (true) {
+			Error err;
 			try {
-				state RangeResult shards =
-				    wait(krmGetRanges(&tr, keyServersPrefix, range, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY));
+				RangeResult shards =
+				    co_await krmGetRanges(&tr, keyServersPrefix, range, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!shards.empty() && !shards.more);
 
-				state RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+				RangeResult UIDtoTagMap = co_await tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY);
 				ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 
-				state int i = 0;
+				int i = 0;
 				for (i = 0; i < shards.size() - 1; ++i) {
 					std::vector<UID> src;
 					std::vector<UID> dest;
@@ -411,7 +413,7 @@ struct ValidateStorage : TestWorkload {
 					decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId);
 
 					const int idx = deterministicRandom()->randomInt(0, src.size());
-					Optional<Value> serverListValue = wait(tr.get(serverListKeyFor(src[idx])));
+					Optional<Value> serverListValue = co_await tr.get(serverListKeyFor(src[idx]));
 					ASSERT(serverListValue.present());
 					const StorageServerInterface ssi = decodeServerListValue(serverListValue.get());
 					TraceEvent("TestSSUserDataValidationSendingRequest")
@@ -422,79 +424,77 @@ struct ValidateStorage : TestWorkload {
 					                        AuditType::ValidateHA);
 					req.ddId = deterministicRandom()->randomUniqueID();
 					Optional<AuditStorageState> vResult =
-					    wait(timeout<AuditStorageState>(ssi.auditStorage.getReply(req), 5));
+					    co_await timeout<AuditStorageState>(ssi.auditStorage.getReply(req), 5);
 					if (!vResult.present()) {
-						return Void();
+						co_return;
 					}
 				}
 				break;
 			} catch (Error& e) {
-				if (retryCount > 5) {
-					TraceEvent(SevWarnAlways, "TestSSUserDataValidationFailed")
-					    .errorUnsuppressed(e)
-					    .detail("Range", range);
-					break;
-				} else {
-					TraceEvent(SevWarn, "TestSSUserDataValidationFailedRetry")
-					    .errorUnsuppressed(e)
-					    .detail("Range", range)
-					    .detail("RetryCount", retryCount);
-					wait(delay(1));
-					retryCount++;
-					continue;
-				}
+				err = e;
+			}
+			if (retryCount > 5) {
+				TraceEvent(SevWarnAlways, "TestSSUserDataValidationFailed")
+				    .errorUnsuppressed(err)
+				    .detail("Range", range);
+				break;
+			} else {
+				TraceEvent(SevWarn, "TestSSUserDataValidationFailedRetry")
+				    .errorUnsuppressed(err)
+				    .detail("Range", range)
+				    .detail("RetryCount", retryCount);
+				co_await delay(1);
+				retryCount++;
+				continue;
 			}
 		}
 
 		TraceEvent("TestSSUserDataValidationDone").detail("Range", range);
-
-		return Void();
 	}
 
-	ACTOR Future<UID> auditStorageForType(ValidateStorage* self,
-	                                      Database cx,
-	                                      AuditType type,
-	                                      std::string context,
-	                                      bool stopWaitWhenCleared = false) {
+	Future<UID> auditStorageForType(ValidateStorage* self,
+	                                Database cx,
+	                                AuditType type,
+	                                std::string context,
+	                                bool stopWaitWhenCleared = false) {
 		std::vector<KeyRangeRef> auditKeysCollection = { partialKeys1, partialKeys2, partialKeys3, partialKeys4,
 			                                             partialKeys5, partialKeys6, partialKeys7, allKeys };
-		state KeyRangeRef auditRange = deterministicRandom()->randomChoice(auditKeysCollection);
+		KeyRangeRef auditRange = deterministicRandom()->randomChoice(auditKeysCollection);
 		// Send audit request until the server accepts the request
-		state UID auditId = wait(self->triggerAuditStorageForType(cx, type, context, auditRange));
+		UID auditId = co_await self->triggerAuditStorageForType(cx, type, context, auditRange);
 		if (auditRange.empty()) {
 			ASSERT(!auditId.isValid());
-			return UID();
+			co_return UID();
 		}
 		// Wait until the request completes
-		wait(self->waitAuditStorageUntilComplete(cx, type, auditId, context, stopWaitWhenCleared));
+		co_await self->waitAuditStorageUntilComplete(cx, type, auditId, context, stopWaitWhenCleared);
 		// Check internal persist state
-		wait(self->checkAuditStorageInternalState(cx, type, auditId, context));
-		return auditId;
+		co_await self->checkAuditStorageInternalState(cx, type, auditId, context);
+		co_return auditId;
 	}
 
-	ACTOR Future<Void> testAuditStorageFunctionality(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageFunctionality(ValidateStorage* self, Database cx) {
 		UID auditIdA =
-		    wait(self->auditStorageForType(self, cx, AuditType::ValidateHA, "TestAuditStorageFunctionality"));
+		    co_await self->auditStorageForType(self, cx, AuditType::ValidateHA, "TestAuditStorageFunctionality");
 		TraceEvent("TestFunctionalityHADone", auditIdA);
 		UID auditIdB =
-		    wait(self->auditStorageForType(self, cx, AuditType::ValidateReplica, "TestAuditStorageFunctionality"));
+		    co_await self->auditStorageForType(self, cx, AuditType::ValidateReplica, "TestAuditStorageFunctionality");
 		TraceEvent("TestFunctionalityReplicaDone", auditIdB);
-		UID auditIdC = wait(
-		    self->auditStorageForType(self, cx, AuditType::ValidateLocationMetadata, "TestAuditStorageFunctionality"));
+		UID auditIdC = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateLocationMetadata, "TestAuditStorageFunctionality");
 		TraceEvent("TestFunctionalityShardLocationMetadataDone", auditIdC);
-		UID auditIdD = wait(self->auditStorageForType(
-		    self, cx, AuditType::ValidateStorageServerShard, "TestAuditStorageFunctionality"));
+		UID auditIdD = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateStorageServerShard, "TestAuditStorageFunctionality");
 		TraceEvent("TestFunctionalitySSShardInfoDone", auditIdD);
-		wait(self->testGetAuditStateWhenNoOngingAudit(self, cx));
+		co_await self->testGetAuditStateWhenNoOngingAudit(self, cx);
 		TraceEvent("TestGetAuditStateDone");
-		return Void();
 	}
 
-	ACTOR Future<Void> testAuditStorageIDGenerator(ValidateStorage* self, Database cx) {
-		state AuditType type = AuditType::ValidateReplica;
+	Future<Void> testAuditStorageIDGenerator(ValidateStorage* self, Database cx) {
+		AuditType type = AuditType::ValidateReplica;
 		TraceEvent("TestAuditStorageIDGeneratorBegin").detail("AuditType", type);
-		state UID auditIdA = wait(self->auditStorageForType(self, cx, type, "FirstRunInTestIDGenerator"));
-		state UID auditIdB = wait(self->auditStorageForType(self, cx, type, "SecondRunInTestIDGenerator"));
+		UID auditIdA = co_await self->auditStorageForType(self, cx, type, "FirstRunInTestIDGenerator");
+		UID auditIdB = co_await self->auditStorageForType(self, cx, type, "SecondRunInTestIDGenerator");
 		if (auditIdA == auditIdB && auditIdA.isValid()) {
 			TraceEvent(SevError, "TestAuditStorageIDGeneratorError")
 			    .detail("AuditType", type)
@@ -503,31 +503,30 @@ struct ValidateStorage : TestWorkload {
 		}
 		TraceEvent("TestAuditStorageIDGeneratorEnd").detail("AuditType", type);
 		;
-		return Void();
 	}
 
-	ACTOR Future<Void> testGetAuditStateWhenNoOngingAuditForType(ValidateStorage* self, Database cx, AuditType type) {
+	Future<Void> testGetAuditStateWhenNoOngingAuditForType(ValidateStorage* self, Database cx, AuditType type) {
 		TraceEvent("TestGetAuditStateBegin").detail("AuditType", type);
-		std::vector<AuditStorageState> res1 = wait(getAuditStates(cx, type, /*newFirst=*/true, 1));
+		std::vector<AuditStorageState> res1 = co_await getAuditStates(cx, type, /*newFirst=*/true, 1);
 		if (res1.size() > 1) { // == 0 if empty range when testAuditStorageFunctionality
 			TraceEvent(SevError, "TestGetAuditStatesError").detail("ActualResSize", res1.size());
 		}
 		std::vector<AuditStorageState> res2 =
-		    wait(getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Invalid));
-		if (res2.size() != 0) {
+		    co_await getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Invalid);
+		if (!res2.empty()) {
 			TraceEvent(SevError, "TestExistingInvalidAudit")
 			    .detail("ActualResSize", res2.size())
 			    .detail("InputPhase", AuditPhase::Invalid);
 		}
 		std::vector<AuditStorageState> res3 =
-		    wait(getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Running));
-		if (res3.size() != 0) {
+		    co_await getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Running);
+		if (!res3.empty()) {
 			TraceEvent(SevError, "TestExistingRunningAudit")
 			    .detail("ActualResSize", res3.size())
 			    .detail("InputPhase", AuditPhase::Running);
 		}
 		std::vector<AuditStorageState> res4 =
-		    wait(getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Complete));
+		    co_await getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Complete);
 		for (const auto& auditState : res4) {
 			if (auditState.getPhase() != AuditPhase::Complete) {
 				TraceEvent(SevError, "TestGetAuditStatesByPhaseError")
@@ -536,7 +535,7 @@ struct ValidateStorage : TestWorkload {
 			}
 		}
 		std::vector<AuditStorageState> res5 =
-		    wait(getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Failed));
+		    co_await getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Failed);
 		for (const auto& auditState : res5) {
 			if (auditState.getPhase() != AuditPhase::Failed) {
 				TraceEvent(SevError, "TestGetAuditStatesByPhaseError")
@@ -545,7 +544,7 @@ struct ValidateStorage : TestWorkload {
 			}
 		}
 		std::vector<AuditStorageState> res6 =
-		    wait(getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Error));
+		    co_await getAuditStates(cx, type, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY, AuditPhase::Error);
 		for (const auto& auditState : res6) {
 			if (auditState.getPhase() != AuditPhase::Error) {
 				TraceEvent(SevError, "TestGetAuditStatesByPhaseError")
@@ -554,28 +553,26 @@ struct ValidateStorage : TestWorkload {
 			}
 		}
 		TraceEvent("TestGetAuditStateEnd").detail("AuditType", type);
-		return Void();
 	}
 
-	ACTOR Future<Void> testGetAuditStateWhenNoOngingAudit(ValidateStorage* self, Database cx) {
-		wait(self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateHA));
+	Future<Void> testGetAuditStateWhenNoOngingAudit(ValidateStorage* self, Database cx) {
+		co_await self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateHA);
 		TraceEvent("TestGetAuditStateHADone");
 
-		wait(self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateReplica));
+		co_await self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateReplica);
 		TraceEvent("TestGetAuditStateReplicaDone");
 
-		wait(self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateLocationMetadata));
+		co_await self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateLocationMetadata);
 		TraceEvent("TestGetAuditStateShardLocationMetadataDone");
 
-		wait(self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateStorageServerShard));
+		co_await self->testGetAuditStateWhenNoOngingAuditForType(self, cx, AuditType::ValidateStorageServerShard);
 		TraceEvent("TestGetAuditStateSSShardInfoDone");
-		return Void();
 	}
 
-	ACTOR Future<Void> testAuditStorageConcurrentRunForDifferentType(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageConcurrentRunForDifferentType(ValidateStorage* self, Database cx) {
 		TraceEvent("TestAuditStorageConcurrentRunForDifferentTypeBegin");
-		state std::vector<Future<Void>> fs;
-		state std::vector<UID> auditIds = { UID(), UID(), UID(), UID() };
+		std::vector<Future<Void>> fs;
+		std::vector<UID> auditIds = { UID(), UID(), UID(), UID() };
 		fs.push_back(
 		    store(auditIds[0],
 		          self->auditStorageForType(self, cx, AuditType::ValidateHA, "TestConcurrentRunForDifferentType")));
@@ -588,16 +585,15 @@ struct ValidateStorage : TestWorkload {
 		fs.push_back(store(auditIds[3],
 		                   self->auditStorageForType(
 		                       self, cx, AuditType::ValidateStorageServerShard, "TestConcurrentRunForDifferentType")));
-		wait(waitForAll(fs));
+		co_await waitForAll(fs);
 		fs.clear();
 		TraceEvent("TestAuditStorageConcurrentRunForDifferentTypeEnd");
-		return Void();
 	}
 
-	ACTOR Future<Void> testAuditStorageConcurrentRunForSameType(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageConcurrentRunForSameType(ValidateStorage* self, Database cx) {
 		TraceEvent("TestAuditStorageConcurrentRunForSameTypeBegin");
-		state std::vector<Future<Void>> fs;
-		state std::vector<UID> auditIds = { UID(), UID(), UID(), UID() };
+		std::vector<Future<Void>> fs;
+		std::vector<UID> auditIds = { UID(), UID(), UID(), UID() };
 		fs.push_back(store(
 		    auditIds[0],
 		    self->auditStorageForType(
@@ -614,29 +610,28 @@ struct ValidateStorage : TestWorkload {
 		    auditIds[3],
 		    self->auditStorageForType(
 		        self, cx, AuditType::ValidateReplica, "TestConcurrentRunForSameType", /*stopWaitWhenCleared=*/true)));
-		wait(waitForAll(fs));
+		co_await waitForAll(fs);
 		TraceEvent("TestAuditStorageConcurrentRunForSameTypeEnd");
-		return Void();
 	}
 
-	ACTOR Future<Void> testAuditStorageCancellation(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageCancellation(ValidateStorage* self, Database cx) {
 		TraceEvent("TestAuditStorageCancellationBegin");
-		state UID auditId = wait(self->triggerAuditStorageForType(cx, AuditType::ValidateHA, "TestAuditCancellation"));
-		state std::vector<Future<Void>> fs;
+		UID auditId = co_await self->triggerAuditStorageForType(cx, AuditType::ValidateHA, "TestAuditCancellation");
+		std::vector<Future<Void>> fs;
 		fs.push_back(self->waitCancelAuditStorageUntilComplete(cx, AuditType::ValidateHA, auditId));
 		fs.push_back(self->waitCancelAuditStorageUntilComplete(cx, AuditType::ValidateHA, auditId));
 		fs.push_back(self->waitCancelAuditStorageUntilComplete(cx, AuditType::ValidateReplica, auditId));
 		fs.push_back(self->checkAuditStorageInternalState(cx, AuditType::ValidateHA, auditId, "TestAuditCancellation"));
 		fs.push_back(
 		    self->checkAuditStorageInternalState(cx, AuditType::ValidateReplica, auditId, "TestAuditCancellation"));
-		wait(waitForAll(fs));
+		co_await waitForAll(fs);
 		fs.clear();
 		fs.push_back(self->checkAuditStorageInternalState(cx, AuditType::ValidateHA, auditId, "TestAuditCancellation"));
 		fs.push_back(
 		    self->checkAuditStorageInternalState(cx, AuditType::ValidateReplica, auditId, "TestAuditCancellation"));
-		wait(waitForAll(fs));
-		state std::vector<AuditStorageState> currentAuditStates1 =
-		    wait(getAuditStates(cx, AuditType::ValidateHA, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY));
+		co_await waitForAll(fs);
+		std::vector<AuditStorageState> currentAuditStates1 =
+		    co_await getAuditStates(cx, AuditType::ValidateHA, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY);
 		for (const auto& auditState : currentAuditStates1) {
 			if (auditState.id == auditId && auditState.getPhase() != AuditPhase::Failed) {
 				TraceEvent(SevError, "TestAuditStorageCancellation1Error")
@@ -645,8 +640,8 @@ struct ValidateStorage : TestWorkload {
 				    .detail("AuditPhase", auditState.getPhase());
 			}
 		}
-		state std::vector<AuditStorageState> currentAuditStates2 =
-		    wait(getAuditStates(cx, AuditType::ValidateReplica, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY));
+		std::vector<AuditStorageState> currentAuditStates2 =
+		    co_await getAuditStates(cx, AuditType::ValidateReplica, /*newFirst=*/true, CLIENT_KNOBS->TOO_MANY);
 		for (const auto& auditState : currentAuditStates2) {
 			if (auditState.id == auditId && auditState.getPhase() != AuditPhase::Failed) {
 				TraceEvent(SevError, "TestAuditStorageCancellation2Error")
@@ -656,53 +651,52 @@ struct ValidateStorage : TestWorkload {
 			}
 		}
 		TraceEvent("TestAuditStorageCancellationEnd");
-		return Void();
 	}
 
-	ACTOR Future<Void> persistAuditStateByRange(ValidateStorage* self, Database cx, AuditStorageState auditState) {
-		state Transaction tr(cx);
-		state RangeResult auditStates;
-		loop {
+	Future<Void> persistAuditStateByRange(ValidateStorage* self, Database cx, AuditStorageState auditState) {
+		Transaction tr(cx);
+		RangeResult auditStates;
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-				wait(krmSetRange(&tr,
-				                 auditRangeBasedProgressPrefixFor(auditState.getType(), auditState.id),
-				                 auditState.range,
-				                 auditStorageStateValue(auditState)));
-				wait(tr.commit());
+				co_await krmSetRange(&tr,
+				                     auditRangeBasedProgressPrefixFor(auditState.getType(), auditState.id),
+				                     auditState.range,
+				                     auditStorageStateValue(auditState));
+				co_await tr.commit();
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
-		return Void();
 	}
 
 	// Test audit progress persist invariant
-	ACTOR Future<Void> testAuditStorageProgress(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageProgress(ValidateStorage* self, Database cx) {
 		TraceEvent("TestAuditStorageProgressBegin");
-		state UID auditId = deterministicRandom()->randomUniqueID();
-		state AuditType auditType = AuditType::ValidateHA;
-		state UID ddId = deterministicRandom()->randomUniqueID();
+		UID auditId = deterministicRandom()->randomUniqueID();
+		AuditType auditType = AuditType::ValidateHA;
+		UID ddId = deterministicRandom()->randomUniqueID();
 		std::vector<KeyRange> progressRangesCollection = {
 			KeyRangeRef("TestKeyA"_sr, "TestKeyB"_sr),   KeyRangeRef("TestKeyB"_sr, "TestKeyBB"_sr),
 			KeyRangeRef("TestKeyA"_sr, "TestKeyBBB"_sr), KeyRangeRef("TestKeyE"_sr, "TestKeyF"_sr),
 			KeyRangeRef("TestKeyC"_sr, "TestKeyD"_sr),   KeyRangeRef("TestKeyBBB"_sr, "TestKeyC"_sr),
 			KeyRangeRef("TestKeyBB"_sr, "TestKeyBC"_sr),
 		};
-		state std::vector<KeyRange> progressRanges = shuffleRanges(progressRangesCollection);
-		state int i = 0;
-		state std::vector<KeyRange> alreadyPersisteRanges;
-		for (; i < progressRanges.size(); i++) {
-			state AuditStorageState auditState(auditId, auditType);
+		std::vector<KeyRange> progressRanges = shuffleRanges(progressRangesCollection);
+		std::vector<KeyRange> alreadyPersisteRanges;
+		for (int i = 0; i < progressRanges.size(); ++i) {
+			AuditStorageState auditState(auditId, auditType);
 			auditState.range = progressRanges[i];
 			auditState.ddId = ddId;
 			auditState.setPhase(AuditPhase::Complete);
-			wait(self->persistAuditStateByRange(self, cx, auditState));
+			co_await self->persistAuditStateByRange(self, cx, auditState);
 			alreadyPersisteRanges.push_back(progressRanges[i]);
-			std::vector<AuditStorageState> auditStates = wait(getAuditStateByRange(cx, auditType, auditId, allKeys));
+			std::vector<AuditStorageState> auditStates = co_await getAuditStateByRange(cx, auditType, auditId, allKeys);
 			for (int i = 0; i < auditStates.size(); i++) {
 				KeyRange toCompare = auditStates[i].range;
 				bool overlapped = false;
@@ -735,45 +729,42 @@ struct ValidateStorage : TestWorkload {
 			}
 		}
 		TraceEvent("TestAuditStorageProgressEnd");
-		return Void();
 	}
 
-	ACTOR Future<Void> testAuditStorageWhenDDSecurityMode(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageWhenDDSecurityMode(ValidateStorage* self, Database cx) {
 		TraceEvent("TestAuditStorageWhenDDSecurityModeBegin");
-		wait(success(setDDMode(cx, 2)));
+		co_await setDDMode(cx, 2);
 		UID auditIdA =
-		    wait(self->auditStorageForType(self, cx, AuditType::ValidateHA, "TestAuditStorageWhenDDSecurityMode"));
+		    co_await self->auditStorageForType(self, cx, AuditType::ValidateHA, "TestAuditStorageWhenDDSecurityMode");
 		TraceEvent("TestFunctionalityHADoneWhenDDSecurityMode", auditIdA);
-		UID auditIdB =
-		    wait(self->auditStorageForType(self, cx, AuditType::ValidateReplica, "TestAuditStorageWhenDDSecurityMode"));
+		UID auditIdB = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateReplica, "TestAuditStorageWhenDDSecurityMode");
 		TraceEvent("TestFunctionalityReplicaDoneWhenDDSecurityMode", auditIdB);
-		UID auditIdC = wait(self->auditStorageForType(
-		    self, cx, AuditType::ValidateLocationMetadata, "TestAuditStorageWhenDDSecurityMode"));
+		UID auditIdC = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateLocationMetadata, "TestAuditStorageWhenDDSecurityMode");
 		TraceEvent("TestFunctionalityShardLocationMetadataDoneWhenDDSecurityMode", auditIdC);
-		UID auditIdD = wait(self->auditStorageForType(
-		    self, cx, AuditType::ValidateStorageServerShard, "TestAuditStorageWhenDDSecurityMode"));
+		UID auditIdD = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateStorageServerShard, "TestAuditStorageWhenDDSecurityMode");
 		TraceEvent("TestFunctionalitySSShardInfoDoneWhenDDSecurityMode", auditIdD);
 		TraceEvent("TestAuditStorageWhenDDSecurityModeEnd");
-		return Void();
 	}
 
-	ACTOR Future<Void> testAuditStorageWhenDDBackToNormalMode(ValidateStorage* self, Database cx) {
+	Future<Void> testAuditStorageWhenDDBackToNormalMode(ValidateStorage* self, Database cx) {
 		TraceEvent("TestAuditStorageWhenDDBackToNormalModeBegin");
-		wait(success(setDDMode(cx, 1)));
-		UID auditIdA =
-		    wait(self->auditStorageForType(self, cx, AuditType::ValidateHA, "TestAuditStorageWhenDDBackToNormalMode"));
+		co_await setDDMode(cx, 1);
+		UID auditIdA = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateHA, "TestAuditStorageWhenDDBackToNormalMode");
 		TraceEvent("TestFunctionalityHADoneWhenDDBackToNormalMode", auditIdA);
-		UID auditIdB = wait(
-		    self->auditStorageForType(self, cx, AuditType::ValidateReplica, "TestAuditStorageWhenDDBackToNormalMode"));
+		UID auditIdB = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateReplica, "TestAuditStorageWhenDDBackToNormalMode");
 		TraceEvent("TestFunctionalityReplicaDoneWhenDDBackToNormalMode", auditIdB);
-		UID auditIdC = wait(self->auditStorageForType(
-		    self, cx, AuditType::ValidateLocationMetadata, "TestAuditStorageWhenDDBackToNormalMode"));
+		UID auditIdC = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateLocationMetadata, "TestAuditStorageWhenDDBackToNormalMode");
 		TraceEvent("TestFunctionalityShardLocationMetadataDoneWhenDDBackToNormalMode", auditIdC);
-		UID auditIdD = wait(self->auditStorageForType(
-		    self, cx, AuditType::ValidateStorageServerShard, "TestAuditStorageWhenDDBackToNormalMode"));
+		UID auditIdD = co_await self->auditStorageForType(
+		    self, cx, AuditType::ValidateStorageServerShard, "TestAuditStorageWhenDDBackToNormalMode");
 		TraceEvent("TestFunctionalitySSShardInfoDoneWhenDDBackToNormalMode", auditIdD);
 		TraceEvent("TestAuditStorageWhenDDBackToNormalModeEnd");
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
@@ -781,4 +772,4 @@ struct ValidateStorage : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<ValidateStorage> ValidateStorageFactory;
\ No newline at end of file
+WorkloadFactory<ValidateStorage> ValidateStorageFactory;
diff --git a/fdbserver/workloads/VersionStamp.actor.cpp b/fdbserver/workloads/VersionStamp.cpp
similarity index 66%
rename from fdbserver/workloads/VersionStamp.actor.cpp
rename to fdbserver/workloads/VersionStamp.cpp
index 7926a93ad32..5d35218ba11 100644
--- a/fdbserver/workloads/VersionStamp.actor.cpp
+++ b/fdbserver/workloads/VersionStamp.cpp
@@ -1,5 +1,5 @@
 /*
- * VersionStamp.actor.cpp
+ * VersionStamp.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,12 +20,11 @@
 
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/ApiVersion.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 struct VersionStampWorkload : TestWorkload {
 	static constexpr auto NAME = "VersionStamp";
@@ -46,7 +45,7 @@ struct VersionStampWorkload : TestWorkload {
 	bool soleOwnerOfMetadataVersionKey;
 	bool allowMetadataVersionKey;
 
-	VersionStampWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit VersionStampWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		testDuration = getOption(options, "testDuration"_sr, 60.0);
 		transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 5000.0);
 		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)10000);
@@ -80,7 +79,7 @@ struct VersionStampWorkload : TestWorkload {
 
 		cx->apiVersion = ApiVersion(apiVersion);
 		if (clientId == 0)
-			return _start(cx, this, 1 / transactionsPerSecond);
+			return _start(cx, 1 / transactionsPerSecond);
 		return Void();
 	}
 
@@ -153,50 +152,55 @@ struct VersionStampWorkload : TestWorkload {
 		return { bigEndian64(parsedVersion), parsedVersionstamp };
 	}
 
-	ACTOR Future<bool> _check(Database cx, VersionStampWorkload* self) {
+	Future<bool> _check(Database cx, VersionStampWorkload* self) {
 		if (self->validateExtraDB) {
-			ASSERT(g_simulator->extraDatabases.size() == 1);
-			cx = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+			ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+			cx = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 		}
-		state ReadYourWritesTransaction tr(cx);
+		ReadYourWritesTransaction tr(cx);
 		// We specifically wish to grab the smallest read version that we can get and maintain it, to
 		// have the strictest check we can on versionstamps monotonically increasing.
-		state Version readVersion;
-		loop {
+		Version readVersion{ 0 };
+		while (true) {
+			Error err;
 			try {
-				Version _readVersion = wait(tr.getReadVersion());
+				Version _readVersion = co_await tr.getReadVersion();
 				readVersion = _readVersion;
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 
 		if (BUGGIFY) {
 			if (deterministicRandom()->random01() < 0.5) {
-				loop {
+				while (true) {
+					Error err;
 					try {
 						tr.makeSelfConflicting();
-						wait(tr.commit());
+						co_await tr.commit();
 						readVersion = tr.getCommittedVersion() - 1;
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 				}
 			}
 			tr.reset();
 			tr.setVersion(readVersion);
 		}
 
-		state RangeResult result;
-		loop {
+		RangeResult result;
+		while (true) {
+			Error err;
 			try {
-				RangeResult result_ = wait(tr.getRange(
-				    KeyRangeRef(self->vsValuePrefix, endOfRange(self->vsValuePrefix)), self->nodeCount + 1));
+				RangeResult result_ = co_await tr.getRange(
+				    KeyRangeRef(self->vsValuePrefix, endOfRange(self->vsValuePrefix)), self->nodeCount + 1);
 				result = result_;
 				if (self->allowMetadataVersionKey && self->key_commit.contains(metadataVersionKey)) {
-					Optional<Value> mVal = wait(tr.get(metadataVersionKey));
+					Optional<Value> mVal = co_await tr.get(metadataVersionKey);
 					if (mVal.present()) {
 						result.push_back_deep(result.arena(), KeyValueRef(metadataVersionKey, mVal.get()));
 					}
@@ -205,7 +209,7 @@ struct VersionStampWorkload : TestWorkload {
 				if (self->failIfDataLost) {
 					ASSERT(result.size() == self->key_commit.size());
 				} else {
-					CODE_PROBE(result.size() > 0, "Not all data should always be lost.");
+					CODE_PROBE(!result.empty(), "Not all data should always be lost.");
 				}
 
 				//TraceEvent("VST_Check0").detail("Size", result.size()).detail("NodeCount", self->nodeCount).detail("KeyCommit", self->key_commit.size()).detail("ReadVersion", readVersion);
@@ -253,14 +257,14 @@ struct VersionStampWorkload : TestWorkload {
 					}
 				}
 
-				RangeResult result__ = wait(
-				    tr.getRange(KeyRangeRef(self->vsKeyPrefix, endOfRange(self->vsKeyPrefix)), self->nodeCount + 1));
+				RangeResult result__ = co_await tr.getRange(
+				    KeyRangeRef(self->vsKeyPrefix, endOfRange(self->vsKeyPrefix)), self->nodeCount + 1);
 				result = result__;
 				ASSERT(result.size() <= self->nodeCount);
 				if (self->failIfDataLost) {
 					ASSERT(result.size() == self->versionStampKey_commit.size());
 				} else {
-					CODE_PROBE(result.size() > 0, "Not all data should always be lost (2)");
+					CODE_PROBE(!result.empty(), "Not all data should always be lost (2)");
 				}
 
 				//TraceEvent("VST_Check1").detail("Size", result.size()).detail("VsKeyCommitSize", self->versionStampKey_commit.size());
@@ -298,43 +302,43 @@ struct VersionStampWorkload : TestWorkload {
 				}
 				break;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 		TraceEvent("VST_CheckEnd").log();
-		return true;
+		co_return true;
 	}
 
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<Void> _start(Database cx, VersionStampWorkload* self, double delay) {
-		state double startTime = now();
-		state double lastTime = now();
-		state Database extraDB;
+	Future<Void> _start(Database cx, double delay) {
+		double startTime = now();
+		double lastTime = now();
+		Database extraDB;
 
-		if (!g_simulator->extraDatabases.empty()) {
-			ASSERT(g_simulator->extraDatabases.size() == 1);
-			extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+		if (!fdbSimulationPolicyState().extraDatabases.empty()) {
+			ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+			extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 		}
 
-		state Future<Void> metadataWatch = Void();
-		loop {
-			wait(poisson(&lastTime, delay));
+		Future<Void> metadataWatch = Void();
+		while (true) {
+			co_await poisson(&lastTime, delay);
 			bool oldVSFormat = !cx->apiVersionAtLeast(520);
 
-			state bool cx_is_primary = true;
-			state ReadYourWritesTransaction tr(cx);
-			state Key key = self->keyForIndex(deterministicRandom()->randomInt(0, self->nodeCount));
-			state Value value(std::string(deterministicRandom()->randomInt(10, 100), 'x'));
-			state Key versionStampKey =
-			    self->versionStampKeyForIndex(deterministicRandom()->randomInt(0, self->nodeCount), oldVSFormat);
-			state StringRef prefix = versionStampKey.substr(0, 20 + self->vsKeyPrefix.size());
-			state Key endOfRange = self->endOfRange(prefix);
-			state KeyRangeRef range(prefix, endOfRange);
-			state Standalone<StringRef> committedVersionStamp;
-			state Version committedVersion;
+			bool cx_is_primary = true;
+			ReadYourWritesTransaction tr(cx);
+			Key key = keyForIndex(deterministicRandom()->randomInt(0, nodeCount));
+			Value value(std::string(deterministicRandom()->randomInt(10, 100), 'x'));
+			Key versionStampKey = versionStampKeyForIndex(deterministicRandom()->randomInt(0, nodeCount), oldVSFormat);
+			StringRef prefix = versionStampKey.substr(0, 20 + vsKeyPrefix.size());
+			Key end = endOfRange(prefix);
+			KeyRangeRef range(prefix, end);
+			Standalone<StringRef> committedVersionStamp;
+			Version committedVersion{ 0 };
 
-			state Value versionStampValue;
+			Value versionStampValue;
 
 			if (key == metadataVersionKey) {
 				value = metadataVersionRequiredValue;
@@ -345,109 +349,123 @@ struct VersionStampWorkload : TestWorkload {
 				versionStampValue = value.withSuffix("\x00\x00\x00\x00"_sr);
 			}
 
-			state bool ryw = deterministicRandom()->coinflip();
-			loop {
+			bool ryw = deterministicRandom()->coinflip();
+			while (true) {
 				if (!ryw) {
 					tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
 				}
-				state bool error = false;
-				state Error err;
+				bool error = false;
+				Optional<Error> err;
 				//TraceEvent("VST_CommitBegin").detail("Key", printable(key)).detail("VsKey", printable(versionStampKey)).detail("Clear", printable(range));
-				state Key testKey;
-				state Future<Void> nextMetadataWatch;
-				try {
-					tr.atomicOp(key, versionStampValue, MutationRef::SetVersionstampedValue);
-					if (key == metadataVersionKey) {
-						testKey = "testKey" + deterministicRandom()->randomUniqueID().toString();
-						tr.atomicOp(testKey, versionStampValue, MutationRef::SetVersionstampedValue);
-					}
-					tr.clear(range);
-					tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
-					if (key == metadataVersionKey) {
-						nextMetadataWatch = tr.watch(versionStampKey);
-					}
-					state Future<Standalone<StringRef>> fTrVs = tr.getVersionstamp();
-					wait(tr.commit());
+				Key testKey;
+				Future<Void> nextMetadataWatch;
+				{
+					Optional<Error> caughtErr;
+					try {
+						tr.atomicOp(key, versionStampValue, MutationRef::SetVersionstampedValue);
+						if (key == metadataVersionKey) {
+							testKey = "testKey" + deterministicRandom()->randomUniqueID().toString();
+							tr.atomicOp(testKey, versionStampValue, MutationRef::SetVersionstampedValue);
+						}
+						tr.clear(range);
+						tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
+						if (key == metadataVersionKey) {
+							nextMetadataWatch = tr.watch(versionStampKey);
+						}
+						Future<Standalone<StringRef>> fTrVs = tr.getVersionstamp();
+						co_await tr.commit();
 
-					committedVersion = tr.getCommittedVersion();
-					Standalone<StringRef> committedVersionStamp_ = wait(fTrVs);
-					committedVersionStamp = committedVersionStamp_;
+						committedVersion = tr.getCommittedVersion();
+						Standalone<StringRef> committedVersionStamp_ = co_await fTrVs;
+						committedVersionStamp = committedVersionStamp_;
 
-					if (key == metadataVersionKey) {
-						wait(timeoutError(metadataWatch, 30));
-						nextMetadataWatch = metadataWatch;
-					}
+						if (key == metadataVersionKey) {
+							co_await timeoutError(metadataWatch, 30);
+							nextMetadataWatch = metadataWatch;
+						}
 
-				} catch (Error& e) {
-					err = e;
-					if (err.code() == error_code_database_locked && !g_simulator->extraDatabases.empty()) {
-						//TraceEvent("VST_CommitDatabaseLocked");
-						cx_is_primary = !cx_is_primary;
-						tr = ReadYourWritesTransaction(cx_is_primary ? cx : extraDB);
-						break;
-					} else if (err.code() == error_code_commit_unknown_result) {
-						//TraceEvent("VST_CommitUnknownResult").error(e).detail("Key", printable(key)).detail("VsKey", printable(versionStampKey));
-						loop {
-							state ReadYourWritesTransaction cur_tr(cx_is_primary ? cx : extraDB);
-							cur_tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-							try {
-								Optional<Value> vs_value = wait(cur_tr.get(key == metadataVersionKey ? testKey : key));
-								if (!vs_value.present()) {
-									error = true;
-									break;
-								}
-								const Version value_version = versionFromValue(vs_value.get()).first;
-								//TraceEvent("VST_CommitUnknownRead").detail("VsValue", vs_value.present() ? printable(vs_value.get()) : "did not exist");
-								const auto& value_ts =
-								    self->key_commit[key == metadataVersionKey ? metadataVersionKey
-								                                               : key.removePrefix(self->vsValuePrefix)];
-								const auto& iter = std::find_if(
-								    value_ts.cbegin(),
-								    value_ts.cend(),
-								    [value_version](const std::pair<Version, Standalone<StringRef>>& pair) {
-									    return value_version == pair.first;
-								    });
-								if (iter == value_ts.cend()) {
-									// The commit was successful, and thus we need to record the new data.
-									committedVersion = value_version;
-									committedVersionStamp = vs_value.get().substr(0, 10);
-								} else {
-									error = true;
+					} catch (Error& e) {
+						caughtErr = e;
+					}
+					if (caughtErr.present()) {
+						err = caughtErr.get();
+						if (err.get().code() == error_code_database_locked &&
+						    !fdbSimulationPolicyState().extraDatabases.empty()) {
+							//TraceEvent("VST_CommitDatabaseLocked");
+							cx_is_primary = !cx_is_primary;
+							tr = ReadYourWritesTransaction(cx_is_primary ? cx : extraDB);
+							break;
+						} else if (err.get().code() == error_code_commit_unknown_result) {
+							//TraceEvent("VST_CommitUnknownResult").error(e).detail("Key", printable(key)).detail("VsKey", printable(versionStampKey));
+							while (true) {
+								ReadYourWritesTransaction cur_tr(cx_is_primary ? cx : extraDB);
+								cur_tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+								Error err;
+								try {
+									Optional<Value> vs_value =
+									    co_await cur_tr.get(key == metadataVersionKey ? testKey : key);
+									if (!vs_value.present()) {
+										error = true;
+										break;
+									}
+									const Version value_version = versionFromValue(vs_value.get()).first;
+									//TraceEvent("VST_CommitUnknownRead").detail("VsValue", vs_value.present() ? printable(vs_value.get()) : "did not exist");
+									const auto& value_ts =
+									    key_commit[key == metadataVersionKey ? metadataVersionKey
+									                                         : key.removePrefix(vsValuePrefix)];
+									const auto& iter = std::find_if(
+									    value_ts.cbegin(),
+									    value_ts.cend(),
+									    [value_version](const std::pair<Version, Standalone<StringRef>>& pair) {
+										    return value_version == pair.first;
+									    });
+									if (iter == value_ts.cend()) {
+										// The commit was successful, and thus we need to record the new data.
+										committedVersion = value_version;
+										committedVersionStamp = vs_value.get().substr(0, 10);
+									} else {
+										error = true;
+										break;
+									}
 									break;
+								} catch (Error& caughtErr) {
+									err = caughtErr;
 								}
-								break;
-							} catch (Error& e) {
-								wait(cur_tr.onError(e));
+								co_await cur_tr.onError(err);
 							}
+						} else {
+							error = true;
 						}
-					} else {
-						error = true;
 					}
 				}
 
 				if (error) {
-					TraceEvent("VST_CommitFailed")
-					    .error(err)
-					    .detail("Key", printable(key))
-					    .detail("VsKey", printable(versionStampKey));
-					wait(tr.onError(err));
+					TraceEvent event("VST_CommitFailed");
+					if (err.present()) {
+						event.error(err.get());
+					}
+					event.detail("Key", printable(key)).detail("VsKey", printable(versionStampKey));
+					if (err.present()) {
+						co_await tr.onError(err.get());
+					} else {
+						tr = ReadYourWritesTransaction(cx_is_primary ? cx : extraDB);
+					}
 					continue;
 				}
 
-				const Standalone<StringRef> vsKeyKey = versionStampKey.removePrefix(self->vsKeyPrefix).substr(4, 16);
+				const Standalone<StringRef> vsKeyKey = versionStampKey.removePrefix(vsKeyPrefix).substr(4, 16);
 				const auto& committedVersionPair = std::make_pair(committedVersion, committedVersionStamp);
 				//TraceEvent("VST_CommitSuccess").detail("Key", printable(key)).detail("VsKey", printable(versionStampKey)).detail("VsKeyKey", printable(vsKeyKey)).detail("Clear", printable(range)).detail("Version", tr.getCommittedVersion()).detail("VsValue", printable(committedVersionPair.second));
-				self->key_commit[key == metadataVersionKey ? metadataVersionKey : key.removePrefix(self->vsValuePrefix)]
-				    .push_back(committedVersionPair);
-				self->versionStampKey_commit[vsKeyKey].push_back(committedVersionPair);
+				key_commit[key == metadataVersionKey ? metadataVersionKey : key.removePrefix(vsValuePrefix)].push_back(
+				    committedVersionPair);
+				versionStampKey_commit[vsKeyKey].push_back(committedVersionPair);
 				break;
 			}
 
-			if (now() - startTime > self->testDuration)
+			if (now() - startTime > testDuration)
 				break;
 		}
-		//TraceEvent("VST_Start").detail("Count", count).detail("NodeCount", self->nodeCount);
-		return Void();
+		//TraceEvent("VST_Start").detail("Count", count).detail("NodeCount", nodeCount);
 	}
 };
 
diff --git a/fdbserver/workloads/WatchAndWait.actor.cpp b/fdbserver/workloads/WatchAndWait.cpp
similarity index 69%
rename from fdbserver/workloads/WatchAndWait.actor.cpp
rename to fdbserver/workloads/WatchAndWait.cpp
index 0daeb497528..be27b89361e 100644
--- a/fdbserver/workloads/WatchAndWait.actor.cpp
+++ b/fdbserver/workloads/WatchAndWait.cpp
@@ -1,5 +1,5 @@
 /*
- * WatchAndWait.actor.cpp
+ * WatchAndWait.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,11 +19,10 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "BulkSetup.h"
 #include "fdbclient/ReadYourWrites.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/tester/workloads.h"
 
 struct WatchAndWaitWorkload : TestWorkload {
 	static constexpr auto NAME = "WatchAndWait";
@@ -36,7 +35,8 @@ struct WatchAndWaitWorkload : TestWorkload {
 	std::vector<Future<Void>> clients;
 	PerfIntCounter triggers, retries;
 
-	WatchAndWaitWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), triggers("Triggers"), retries("Retries") {
+	explicit WatchAndWaitWorkload(WorkloadContext const& wcx)
+	  : TestWorkload(wcx), triggers("Triggers"), retries("Retries") {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		watchCount = getOption(options, "watchCount"_sr, (uint64_t)10000);
 		nodeCount = getOption(options, "nodeCount"_sr, (uint64_t)100000);
@@ -58,8 +58,6 @@ struct WatchAndWaitWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	Key keyForIndex(uint64_t index) const {
 		Key result = makeString(keyBytes);
 		uint8_t* data = mutateString(result);
@@ -86,39 +84,40 @@ struct WatchAndWaitWorkload : TestWorkload {
 		m.push_back(retries.getMetric());
 	}
 
-	ACTOR Future<Void> _start(Database cx, WatchAndWaitWorkload* self) {
-		state std::vector<Future<Void>> watches;
-		uint64_t endNode = (self->nodeCount * (self->clientId + 1)) / self->clientCount;
-		uint64_t startNode = (self->nodeCount * self->clientId) / self->clientCount;
-		uint64_t NodesPerWatch = self->nodeCount / self->watchCount;
+	Future<Void> start(Database const& cx) override {
+		std::vector<Future<Void>> watches;
+		uint64_t endNode = (nodeCount * (clientId + 1)) / clientCount;
+		uint64_t startNode = (nodeCount * clientId) / clientCount;
+		uint64_t NodesPerWatch = nodeCount / watchCount;
 		TraceEvent("WatchAndWaitExpect")
-		    .detail("Duration", self->testDuration)
+		    .detail("Duration", testDuration)
 		    .detail("ExpectedCount", (endNode - startNode) / NodesPerWatch)
 		    .detail("End", endNode)
 		    .detail("Start", startNode)
 		    .detail("Npw", NodesPerWatch);
 		for (uint64_t i = startNode; i < endNode; i += NodesPerWatch) {
-			watches.push_back(self->watchAndWait(cx, self, i));
+			watches.push_back(watchAndWait(cx, this, i));
 		}
-		wait(delay(self->testDuration)); // || waitForAll( watches )
-		TraceEvent("WatchAndWaitEnd").detail("Duration", self->testDuration);
-		return Void();
+		co_await delay(testDuration); // || waitForAll( watches )
+		TraceEvent("WatchAndWaitEnd").detail("Duration", testDuration);
 	}
 
-	ACTOR Future<Void> watchAndWait(Database cx, WatchAndWaitWorkload* self, int index) {
+	Future<Void> watchAndWait(Database cx, WatchAndWaitWorkload* self, int index) {
 		try {
-			state ReadYourWritesTransaction tr(cx);
-			loop {
+			ReadYourWritesTransaction tr(cx);
+			while (true) {
 				cx->maxOutstandingWatches = 1e6;
+				Error err;
 				try {
-					state Future<Void> watch = tr.watch(self->keyForIndex(index));
-					wait(tr.commit());
-					wait(watch);
+					Future<Void> watch = tr.watch(self->keyForIndex(index));
+					co_await tr.commit();
+					co_await watch;
 					++self->triggers;
 				} catch (Error& e) {
-					++self->retries;
-					wait(tr.onError(e));
+					err = e;
 				}
+				++self->retries;
+				co_await tr.onError(err);
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "WatchAndWaitError").error(e);
diff --git a/fdbserver/workloads/Watches.actor.cpp b/fdbserver/workloads/Watches.cpp
similarity index 89%
rename from fdbserver/workloads/Watches.actor.cpp
rename to fdbserver/workloads/Watches.cpp
index cc60d55753d..c94d1c58ee3 100644
--- a/fdbserver/workloads/Watches.actor.cpp
+++ b/fdbserver/workloads/Watches.cpp
@@ -1,5 +1,5 @@
 /*
- * Watches.actor.cpp
+ * Watches.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,11 +20,11 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "flow/CodeProbe.h"
 #include "flow/Coroutines.h"
 #include "flow/DeterministicRandom.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct WatchesWorkload : TestWorkload {
@@ -37,7 +37,7 @@ struct WatchesWorkload : TestWorkload {
 	DDSketch<double> cycleLatencies;
 	std::vector<int> nodeOrder;
 
-	WatchesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), cycles("Cycles"), cycleLatencies() {
+	explicit WatchesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), cycles("Cycles"), cycleLatencies() {
 		testDuration = getOption(options, "testDuration"_sr, 600.0);
 		nodes = getOption(options, "nodeCount"_sr, 100);
 		extraPerNode = getOption(options, "extraPerNode"_sr, 1000);
@@ -124,16 +124,17 @@ struct WatchesWorkload : TestWorkload {
 		}
 	}
 
-	ACTOR static Future<Void> watcher(Database cx, Key watchKey, Key setKey, int extraNodes) {
-		state Optional<Optional<Value>> lastValue;
+	static Future<Void> watcher(Database cx, Key watchKey, Key setKey, int extraNodes) {
+		Optional<Optional<Value>> lastValue;
 
-		loop {
-			loop {
-				state std::unique_ptr<Transaction> tr = std::make_unique<Transaction>(cx);
+		while (true) {
+			while (true) {
+				std::unique_ptr<Transaction> tr = std::make_unique<Transaction>(cx);
+				Error err;
 				try {
-					state Future<Optional<Value>> setValueFuture = tr->get(setKey);
-					state Optional<Value> watchValue = wait(tr->get(watchKey));
-					Optional<Value> setValue = wait(setValueFuture);
+					Future<Optional<Value>> setValueFuture = tr->get(setKey);
+					Optional<Value> watchValue = co_await tr->get(watchKey);
+					Optional<Value> setValue = co_await setValueFuture;
 
 					if (lastValue.present() && lastValue.get() == watchValue) {
 						TraceEvent(SevError, "WatcherTriggeredWithoutChanging")
@@ -152,25 +153,26 @@ struct WatchesWorkload : TestWorkload {
 						else
 							tr->clear(setKey);
 						//TraceEvent("WatcherSetStart").detail("Watch", printable(watchKey)).detail("Set", printable(setKey)).detail("Value", printable( watchValue ) );
-						wait(tr->commit());
+						co_await tr->commit();
 						//TraceEvent("WatcherSetFinish").detail("Watch", printable(watchKey)).detail("Set", printable(setKey)).detail("Value", printable( watchValue ) ).detail("Ver", tr->getCommittedVersion());
 					} else {
 						//TraceEvent("WatcherWatch").detail("Watch", printable(watchKey));
-						state Future<Void> watchFuture = tr->watch(makeReference<Watch>(watchKey, watchValue));
-						wait(tr->commit());
+						Future<Void> watchFuture = tr->watch(makeReference<Watch>(watchKey, watchValue));
+						co_await tr->commit();
 						if (BUGGIFY) {
 							// Make watch future outlive transaction
 							tr.reset();
 						}
-						wait(watchFuture);
+						co_await watchFuture;
 						if (watchValue.present())
 							lastValue = watchValue;
 					}
 					break;
 				} catch (Error& e) {
-					if (tr != nullptr) {
-						wait(tr->onError(e));
-					}
+					err = e;
+				}
+				if (tr != nullptr) {
+					co_await tr->onError(err);
 				}
 			}
 		}
@@ -183,7 +185,7 @@ struct WatchesWorkload : TestWorkload {
 		Optional<Value> startValue;
 		double startTime = now();
 		double chainStartTime;
-		loop {
+		while (true) {
 			bool isValue = deterministicRandom()->random01() > 0.5;
 			Value assignedValue = Value(deterministicRandom()->randomUniqueID().toString());
 			bool firstAttempt = true;
diff --git a/fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp b/fdbserver/workloads/WatchesSameKeyCorrectness.cpp
similarity index 51%
rename from fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp
rename to fdbserver/workloads/WatchesSameKeyCorrectness.cpp
index 3b10357113e..6370d66f804 100644
--- a/fdbserver/workloads/WatchesSameKeyCorrectness.actor.cpp
+++ b/fdbserver/workloads/WatchesSameKeyCorrectness.cpp
@@ -1,5 +1,5 @@
 /*
- * WatchesSameKeyCorrectness.actor.cpp
+ * WatchesSameKeyCorrectness.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,9 +20,9 @@
 
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "flow/DeterministicRandom.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct WatchesSameKeyWorkload : TestWorkload {
@@ -30,16 +30,16 @@ struct WatchesSameKeyWorkload : TestWorkload {
 	int numWatches;
 	std::vector<Future<Void>> cases;
 
-	WatchesSameKeyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+	explicit WatchesSameKeyWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		numWatches = getOption(options, "numWatches"_sr, 3);
 	}
 
 	Future<Void> setup(Database const& cx) override {
-		cases.push_back(case1(cx, "foo1"_sr, this));
-		cases.push_back(case2(cx, "foo2"_sr, this));
-		cases.push_back(case3(cx, "foo3"_sr, this));
-		cases.push_back(case4(cx, "foo4"_sr, this));
-		cases.push_back(case5(cx, "foo5"_sr, this));
+		cases.push_back(case1(cx, "foo1"_sr));
+		cases.push_back(case2(cx, "foo2"_sr));
+		cases.push_back(case3(cx, "foo3"_sr));
+		cases.push_back(case4(cx, "foo4"_sr));
+		cases.push_back(case5(cx, "foo5"_sr));
 		return Void();
 	}
 
@@ -55,10 +55,11 @@ struct WatchesSameKeyWorkload : TestWorkload {
 		return ok;
 	}
 
-	ACTOR static Future<Void> setKeyRandomValue(Database cx, Key key, Optional<Value> val) {
+	static Future<Void> setKeyRandomValue(Database cx, Key key, Optional<Value> val) {
 		// set value at key to val if provided (random otherwise)
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				Value valS;
 				if (!val.present())
@@ -66,173 +67,186 @@ struct WatchesSameKeyWorkload : TestWorkload {
 				else
 					valS = val.get();
 				tr.set(key, valS);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Future<Void>> watchKey(Database cx, Key key) {
+	static Future<Future<Void>> watchKey(Database cx, Key key) {
 		// sets a watch on a key and returns future
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				state Future<Void> watchFuture = tr.watch(key);
-				wait(tr.commit());
-				return watchFuture;
+				Future<Void> watchFuture = tr.watch(key);
+				co_await tr.commit();
+				co_return watchFuture;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> case1(Database cx, Key key, WatchesSameKeyWorkload* self) {
+	Future<Void> case1(Database cx, Key key) {
 		/**
 		 * Tests case 2 in the design doc:
 		 *  - we get a watch that has the same value as a key in the watch map
 		 * */
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				state std::vector<Future<Void>> watchFutures;
-				state int i;
+				std::vector<Future<Void>> watchFutures;
+				int i{ 0 };
 
 				tr.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
-				for (i = 0; i < self->numWatches; i++) { // set watches for a given k/v pair set above
+				for (i = 0; i < numWatches; i++) { // set watches for a given k/v pair set above
 					watchFutures.push_back(tr.watch(key));
 				}
-				wait(tr.commit());
+				co_await tr.commit();
 
-				wait(setKeyRandomValue(cx, key, Optional<Value>())); // trigger all watches created above
+				co_await setKeyRandomValue(cx, key, Optional<Value>()); // trigger all watches created above
 				for (i = 0; i < watchFutures.size(); i++) {
-					wait(watchFutures[i]);
+					co_await watchFutures[i];
 				}
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> case2(Database cx, Key key, WatchesSameKeyWorkload* self) {
+	Future<Void> case2(Database cx, Key key) {
 		/**
 		 * Tests case 3 in the design doc:
 		 * 	- we get a watch that has a different value than the key in the map but the version is larger
 		 * */
-		state ReadYourWritesTransaction tr(cx);
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		while (true) {
+			Error err;
 			try {
-				state std::vector<Future<Void>> watchFutures;
-				state Future<Void> watch1 = wait(watchKey(cx, key));
-				state int i;
+				std::vector<Future<Void>> watchFutures;
+				Future<Void> watch1 = co_await watchKey(cx, key);
+				int i{ 0 };
 
 				tr.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
-				for (i = 0; i < self->numWatches; i++) { // set watches for a given k/v pair set above
+				for (i = 0; i < numWatches; i++) { // set watches for a given k/v pair set above
 					watchFutures.push_back(tr.watch(key));
 				}
-				wait(tr.commit());
-				wait(watch1);
-				wait(setKeyRandomValue(cx, key, Optional<Value>())); // trigger remaining watches
+				co_await tr.commit();
+				co_await watch1;
+				co_await setKeyRandomValue(cx, key, Optional<Value>()); // trigger remaining watches
 				for (i = 0; i < watchFutures.size(); i++) {
-					wait(watchFutures[i]);
+					co_await watchFutures[i];
 				}
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR static Future<Void> case3(Database cx, Key key, WatchesSameKeyWorkload* self) {
+	Future<Void> case3(Database cx, Key key) {
 		/**
 		 * Tests case 2 for the storage server response:
 		 * 	- i.e ABA but when the storage server responds the future count == 1 so we do nothing (no refire)
 		 * */
-		state ReadYourWritesTransaction tr(cx);
-		state ReadYourWritesTransaction tr2(cx);
-		state Value val;
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		ReadYourWritesTransaction tr2(cx);
+		Value val;
+		while (true) {
+			Error err;
 			try {
 				val = deterministicRandom()->randomUniqueID().toString();
 				tr2.set(key, val);
-				state Future<Void> watch1 = tr2.watch(key);
-				wait(tr2.commit());
-				wait(setKeyRandomValue(cx, key, Optional<Value>()));
+				Future<Void> watch1 = tr2.watch(key);
+				co_await tr2.commit();
+				co_await setKeyRandomValue(cx, key, Optional<Value>());
 
 				tr.set(key, val);
-				state Future<Void> watch2 = tr.watch(key);
-				wait(tr.commit());
+				Future<Void> watch2 = tr.watch(key);
+				co_await tr.commit();
 
 				watch1.cancel();
 				watch2.cancel();
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e) && tr2.onError(e));
+				err = e;
 			}
+			co_await (tr.onError(err) && tr2.onError(err));
 		}
 	}
 
-	ACTOR static Future<Void> case4(Database cx, Key key, WatchesSameKeyWorkload* self) {
+	Future<Void> case4(Database cx, Key key) {
 		/**
 		 * Tests case 3 for the storage server response:
 		 * 	- i.e ABA but when the storage server responds the future count > 1 so we refire request to SS
 		 * */
-		state ReadYourWritesTransaction tr(cx);
-		state ReadYourWritesTransaction tr2(cx);
-		loop {
+		ReadYourWritesTransaction tr(cx);
+		ReadYourWritesTransaction tr2(cx);
+		while (true) {
+			Error err;
 			try {
 				// watch1 and watch2 are set on the same k/v pair
-				state Value val(deterministicRandom()->randomUniqueID().toString());
+				Value val(deterministicRandom()->randomUniqueID().toString());
 				tr2.set(key, val);
-				state Future<Void> watch1 = tr2.watch(key);
-				wait(tr2.commit());
-				wait(setKeyRandomValue(cx, key, Optional<Value>()));
+				Future<Void> watch1 = tr2.watch(key);
+				co_await tr2.commit();
+				co_await setKeyRandomValue(cx, key, Optional<Value>());
 				tr.set(key, val); // trigger ABA (line above changes value and this line changes it back)
-				state Future<Void> watch2 = tr.watch(key);
-				wait(tr.commit());
+				Future<Void> watch2 = tr.watch(key);
+				co_await tr.commit();
 
-				wait(setKeyRandomValue(
+				co_await setKeyRandomValue(
 				    cx,
 				    key,
-				    Optional<Value>())); // since ABA has occurred we need to trigger the watches with a new value
-				wait(watch1);
-				wait(watch2);
-				return Void();
+				    Optional<Value>()); // since ABA has occurred we need to trigger the watches with a new value
+				co_await watch1;
+				co_await watch2;
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e) && tr2.onError(e));
+				err = e;
 			}
+			co_await (tr.onError(err) && tr2.onError(err));
 		}
 	}
 
-	ACTOR static Future<Void> case5(Database cx, Key key, WatchesSameKeyWorkload* self) {
+	Future<Void> case5(Database cx, Key key) {
 		/**
 		 * Tests case 5 in the design doc:
 		 * 	- i.e values of watches are different but versions are the same
 		 * */
-		state ReadYourWritesTransaction tr1(cx);
-		state ReadYourWritesTransaction tr2(cx);
-		loop {
+		ReadYourWritesTransaction tr1(cx);
+		ReadYourWritesTransaction tr2(cx);
+		while (true) {
+			Error err;
 			try {
 				tr1.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 				tr2.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 				tr1.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
 				tr2.set(key, Value(deterministicRandom()->randomUniqueID().toString()));
-				state Future<Void> watch1 = tr1.watch(key);
-				state Future<Void> watch2 = tr2.watch(key);
-				// each watch commits with a different value but (hopefully) the same version since there is no write
-				// conflict range
-				wait(tr1.commit() && tr2.commit());
+				Future<Void> watch1 = tr1.watch(key);
+				Future<Void> watch2 = tr2.watch(key);
+				// each watch commits with a different value but (hopefully) the same version since there is no
+				// write conflict range
+				co_await (tr1.commit() && tr2.commit());
 
-				wait(watch1 || watch2); // since we enter case 5 at least one of the watches should be fired
-				wait(setKeyRandomValue(cx, key, Optional<Value>())); // fire the watch that possibly wasn't triggered
-				wait(watch1 && watch2);
+				co_await (watch1 || watch2); // since we enter case 5 at least one of the watches should be fired
+				co_await setKeyRandomValue(cx, key, Optional<Value>()); // fire the watch that possibly wasn't triggered
+				co_await (watch1 && watch2);
 
-				return Void();
+				co_return;
 			} catch (Error& e) {
-				wait(tr1.onError(e) && tr2.onError(e));
+				err = e;
 			}
+			co_await (tr1.onError(err) && tr2.onError(err));
 		}
 	}
 
diff --git a/fdbserver/workloads/WorkerErrors.actor.cpp b/fdbserver/workloads/WorkerErrors.cpp
similarity index 61%
rename from fdbserver/workloads/WorkerErrors.actor.cpp
rename to fdbserver/workloads/WorkerErrors.cpp
index a566add1cee..288574c3b0a 100644
--- a/fdbserver/workloads/WorkerErrors.actor.cpp
+++ b/fdbserver/workloads/WorkerErrors.cpp
@@ -1,5 +1,5 @@
 /*
- * WorkerErrors.actor.cpp
+ * WorkerErrors.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,30 +20,28 @@
 
 #include "flow/ActorCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbserver/ServerDBInfo.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/core/QuietDatabase.h"
+#include "fdbserver/core/ServerDBInfo.h"
 
 struct WorkerErrorsWorkload : TestWorkload {
 	static constexpr auto NAME = "WorkerErrors";
 
-	WorkerErrorsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+	explicit WorkerErrorsWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 
-	ACTOR Future<std::vector<TraceEventFields>> latestEventOnWorkers(std::vector<WorkerDetails> workers) {
-		state std::vector<Future<TraceEventFields>> eventTraces;
+	Future<std::vector<TraceEventFields>> latestEventOnWorkers(std::vector<WorkerDetails> workers) {
+		std::vector<Future<TraceEventFields>> eventTraces;
 		eventTraces.reserve(workers.size());
 		for (int c = 0; c < workers.size(); c++) {
 			eventTraces.push_back(workers[c].interf.eventLogRequest.getReply(EventLogRequest()));
 		}
 
-		wait(timeoutError(waitForAll(eventTraces), 2.0));
+		co_await timeoutError(waitForAll(eventTraces), 2.0);
 
 		std::vector<TraceEventFields> results;
 		results.reserve(eventTraces.size());
@@ -51,16 +49,15 @@ struct WorkerErrorsWorkload : TestWorkload {
 			results.push_back(eventTraces[i].get());
 		}
 
-		return results;
+		co_return results;
 	}
 
-	ACTOR Future<Void> _start(Database cx, WorkerErrorsWorkload* self) {
-		state std::vector<WorkerDetails> workers = wait(getWorkers(self->dbInfo));
-		std::vector<TraceEventFields> errors = wait(self->latestEventOnWorkers(workers));
-		for (auto e : errors) {
+	Future<Void> start(Database const& cx) override {
+		std::vector<WorkerDetails> workers = co_await getWorkers(dbInfo);
+		std::vector<TraceEventFields> errors = co_await latestEventOnWorkers(workers);
+		for (const auto& e : errors) {
 			printf("%s\n", e.toString().c_str());
 		}
-		return Void();
 	}
 
 	Future<bool> check(Database const& cx) override { return true; }
diff --git a/fdbserver/workloads/WriteBandwidth.actor.cpp b/fdbserver/workloads/WriteBandwidth.cpp
similarity index 72%
rename from fdbserver/workloads/WriteBandwidth.actor.cpp
rename to fdbserver/workloads/WriteBandwidth.cpp
index 1f9c9aa37e4..9045cebda56 100644
--- a/fdbserver/workloads/WriteBandwidth.actor.cpp
+++ b/fdbserver/workloads/WriteBandwidth.cpp
@@ -1,5 +1,5 @@
 /*
- * WriteBandwidth.actor.cpp
+ * WriteBandwidth.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,11 +22,10 @@
 
 #include "fdbrpc/DDSketch.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
 
 struct WriteBandwidthWorkload : KVWorkload {
 	static constexpr auto NAME = "WriteBandwidth";
@@ -37,7 +36,7 @@ struct WriteBandwidthWorkload : KVWorkload {
 	PerfIntCounter transactions, retries;
 	DDSketch<double> commitLatencies, GRVLatencies;
 
-	WriteBandwidthWorkload(WorkloadContext const& wcx)
+	explicit WriteBandwidthWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), loadTime(0.0), transactions("Transactions"), retries("Retries"), commitLatencies(),
 	    GRVLatencies() {
 		testDuration = getOption(options, "testDuration"_sr, 10.0);
@@ -46,9 +45,6 @@ struct WriteBandwidthWorkload : KVWorkload {
 		maxInsertRate = getOption(options, "maxInsertRate"_sr, 1e12);
 	}
 
-	Future<Void> setup(Database const& cx) override { return _setup(cx, this); }
-	Future<Void> start(Database const& cx) override { return _start(cx, this); }
-
 	Future<bool> check(Database const& cx) override { return true; }
 
 	void getMetrics(std::vector<PerfMetric>& m) override {
@@ -80,33 +76,32 @@ struct WriteBandwidthWorkload : KVWorkload {
 
 	Standalone<KeyValueRef> operator()(uint64_t n) { return KeyValueRef(keyForIndex(n, false), randomValue()); }
 
-	ACTOR Future<Void> _setup(Database cx, WriteBandwidthWorkload* self) {
-		state Promise<double> loadTime;
-		state Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts;
+	Future<Void> setup(Database const& cx) override {
+		Promise<double> loadTime;
+		Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts;
 
-		wait(bulkSetup(cx, self, self->nodeCount, loadTime, true, self->warmingDelay, self->maxInsertRate));
-		self->loadTime = loadTime.getFuture().get();
-		return Void();
+		co_await bulkSetup(cx, this, nodeCount, loadTime, true, warmingDelay, maxInsertRate);
+		this->loadTime = loadTime.getFuture().get();
 	}
 
-	ACTOR Future<Void> _start(Database cx, WriteBandwidthWorkload* self) {
-		for (int i = 0; i < self->actorCount; i++) {
-			self->clients.push_back(self->writeClient(cx, self));
+	Future<Void> start(Database const& cx) override {
+		for (int i = 0; i < actorCount; i++) {
+			clients.push_back(writeClient(cx, this));
 		}
 
-		wait(timeout(waitForAll(self->clients), self->testDuration, Void()));
-		self->clients.clear();
-		return Void();
+		co_await timeout(waitForAll(clients), testDuration, Void());
+		clients.clear();
 	}
 
-	ACTOR Future<Void> writeClient(Database cx, WriteBandwidthWorkload* self) {
-		loop {
-			state Transaction tr(cx);
-			state uint64_t startIdx = deterministicRandom()->random01() * (self->nodeCount - self->keysPerTransaction);
-			loop {
+	Future<Void> writeClient(Database cx, WriteBandwidthWorkload* self) {
+		while (true) {
+			Transaction tr(cx);
+			uint64_t startIdx = deterministicRandom()->random01() * (self->nodeCount - self->keysPerTransaction);
+			while (true) {
+				Error err;
 				try {
-					state double start = now();
-					wait(success(tr.getReadVersion()));
+					double start = now();
+					co_await tr.getReadVersion();
 					self->GRVLatencies.addSample(now() - start);
 
 					// Predefine a single large write conflict range over the whole key space
@@ -118,13 +113,14 @@ struct WriteBandwidthWorkload : KVWorkload {
 						tr.set(self->keyForIndex(startIdx + i, false), self->randomValue(), AddConflictRange::False);
 
 					start = now();
-					wait(tr.commit());
+					co_await tr.commit();
 					self->commitLatencies.addSample(now() - start);
 					break;
 				} catch (Error& e) {
-					wait(tr.onError(e));
-					++self->retries;
+					err = e;
 				}
+				co_await tr.onError(err);
+				++self->retries;
 			}
 			++self->transactions;
 		}
diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.cpp
similarity index 57%
rename from fdbserver/workloads/WriteDuringRead.actor.cpp
rename to fdbserver/workloads/WriteDuringRead.cpp
index 8b32dbdce8a..e5ff6e74f19 100644
--- a/fdbserver/workloads/WriteDuringRead.actor.cpp
+++ b/fdbserver/workloads/WriteDuringRead.cpp
@@ -1,5 +1,5 @@
 /*
- * WriteDuringRead.actor.cpp
+ * WriteDuringRead.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,15 +19,15 @@
  */
 
 #include "fdbclient/ClusterConnectionMemoryRecord.h"
-#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "flow/ActorCollection.h"
-#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/tester/workloads.h"
 #include "fdbclient/Atomic.h"
 #include "flow/ApiVersion.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 struct WriteDuringReadWorkload : TestWorkload {
 	static constexpr auto NAME = "WriteDuringRead";
@@ -58,7 +58,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 	bool useExtraDB;
 	double zeroPaddingRatio;
 
-	WriteDuringReadWorkload(WorkloadContext const& wcx)
+	explicit WriteDuringReadWorkload(WorkloadContext const& wcx)
 	  : TestWorkload(wcx), transactions("Transactions"), retries("Retries"), success(true) {
 		testDuration = getOption(options, "testDuration"_sr, 60.0);
 		slowModeStart = getOption(options, "slowModeStart"_sr, 1000.0);
@@ -91,10 +91,10 @@ struct WriteDuringReadWorkload : TestWorkload {
 		CODE_PROBE(adjacentKeys && (nodes + minNode) > CLIENT_KNOBS->KEY_SIZE_LIMIT,
 		           "WriteDuringReadWorkload testing large keys");
 
-		useExtraDB = g_network->isSimulated() && !g_simulator->extraDatabases.empty();
+		useExtraDB = g_network->isSimulated() && !fdbSimulationPolicyState().extraDatabases.empty();
 		if (useExtraDB) {
-			ASSERT(g_simulator->extraDatabases.size() == 1);
-			extraDB = Database::createSimulatedExtraDatabase(g_simulator->extraDatabases[0]);
+			ASSERT(fdbSimulationPolicyState().extraDatabases.size() == 1);
+			extraDB = Database::createSimulatedExtraDatabase(fdbSimulationPolicyState().extraDatabases[0]);
 			useSystemKeys = false;
 		}
 
@@ -156,23 +156,23 @@ struct WriteDuringReadWorkload : TestWorkload {
 		return iter->first;
 	}
 
-	ACTOR Future<Void> getKeyAndCompare(ReadYourWritesTransaction* tr,
-	                                    KeySelector key,
-	                                    Snapshot snapshot,
-	                                    bool readYourWritesDisabled,
-	                                    bool snapshotRYWDisabled,
-	                                    WriteDuringReadWorkload* self,
-	                                    bool* doingCommit,
-	                                    int64_t* memLimit) {
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+	Future<Void> getKeyAndCompare(ReadYourWritesTransaction* tr,
+	                              KeySelector key,
+	                              Snapshot snapshot,
+	                              bool readYourWritesDisabled,
+	                              bool snapshotRYWDisabled,
+	                              WriteDuringReadWorkload* self,
+	                              bool* doingCommit,
+	                              int64_t* memLimit) {
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 		//TraceEvent("WDRGetKey", randomID);
 		try {
-			state Key memRes = self->memoryGetKey(readYourWritesDisabled || (snapshot && snapshotRYWDisabled)
-			                                          ? &self->lastCommittedDatabase
-			                                          : &self->memoryDatabase,
-			                                      key);
+			Key memRes = self->memoryGetKey(readYourWritesDisabled || (snapshot && snapshotRYWDisabled)
+			                                    ? &self->lastCommittedDatabase
+			                                    : &self->memoryDatabase,
+			                                key);
 			*memLimit -= memRes.expectedSize();
-			Key _res = wait(tr->getKey(key, snapshot));
+			Key _res = co_await tr->getKey(key, snapshot);
 			Key res = _res;
 			*memLimit += memRes.expectedSize();
 			if (self->useSystemKeys && res > self->getKeyForIndex(self->nodes))
@@ -187,14 +187,14 @@ struct WriteDuringReadWorkload : TestWorkload {
 				    .detail("DbResult", res);
 				self->success = false;
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			//TraceEvent("WDRGetKeyError", randomID).error(e,true);
 			if (e.code() == error_code_used_during_commit) {
 				ASSERT(*doingCommit);
-				return Void();
+				co_return;
 			} else if (e.code() == error_code_transaction_cancelled)
-				return Void();
+				co_return;
 			throw;
 		}
 	}
@@ -215,7 +215,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 
 		Standalone<VectorRef<KeyValueRef>> results;
 		if (reverse) {
-			loop {
+			while (true) {
 				if (beginIter == endIter || limit.reachedBy(results))
 					break;
 
@@ -229,25 +229,25 @@ struct WriteDuringReadWorkload : TestWorkload {
 		return results;
 	}
 
-	ACTOR Future<Void> getRangeAndCompare(ReadYourWritesTransaction* tr,
-	                                      KeySelector begin,
-	                                      KeySelector end,
-	                                      GetRangeLimits limit,
-	                                      Snapshot snapshot,
-	                                      Reverse reverse,
-	                                      bool readYourWritesDisabled,
-	                                      bool snapshotRYWDisabled,
-	                                      WriteDuringReadWorkload* self,
-	                                      bool* doingCommit,
-	                                      int64_t* memLimit) {
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+	Future<Void> getRangeAndCompare(ReadYourWritesTransaction* tr,
+	                                KeySelector begin,
+	                                KeySelector end,
+	                                GetRangeLimits limit,
+	                                Snapshot snapshot,
+	                                Reverse reverse,
+	                                bool readYourWritesDisabled,
+	                                bool snapshotRYWDisabled,
+	                                WriteDuringReadWorkload* self,
+	                                bool* doingCommit,
+	                                int64_t* memLimit) {
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 		/*TraceEvent("WDRGetRange", randomID).detail("BeginKey", begin.getKey()).detail("BeginOffset", begin.offset).detail("BeginOrEqual", begin.orEqual)
 		    .detail("EndKey", end.getKey()).detail("EndOffset", end.offset).detail("EndOrEqual", end.orEqual)
 		    .detail("Limit", limit.rows).detail("Snapshot", snapshot).detail("Reverse",
 		   reverse).detail("ReadYourWritesDisabled", readYourWritesDisabled);*/
 
 		try {
-			state Standalone<VectorRef<KeyValueRef>> memRes = self->memoryGetRange(
+			Standalone<VectorRef<KeyValueRef>> memRes = self->memoryGetRange(
 			    readYourWritesDisabled || (snapshot && snapshotRYWDisabled) ? &self->lastCommittedDatabase
 			                                                                : &self->memoryDatabase,
 			    begin,
@@ -255,7 +255,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 			    limit,
 			    reverse);
 			*memLimit -= memRes.expectedSize();
-			RangeResult _res = wait(tr->getRange(begin, end, limit, snapshot, reverse));
+			RangeResult _res = co_await tr->getRange(begin, end, limit, snapshot, reverse);
 			RangeResult res = _res;
 			*memLimit += memRes.expectedSize();
 
@@ -301,7 +301,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 					    .detail("ReadYourWritesDisabled", readYourWritesDisabled);
 
 					self->success = false;
-					return Void();
+					co_return;
 				}
 
 				for (int i = 0; i < res.size(); i++) {
@@ -325,12 +325,12 @@ struct WriteDuringReadWorkload : TestWorkload {
 						    .detail("DbResultValueSize", res[i].value.size())
 						    .detail("ReadYourWritesDisabled", readYourWritesDisabled);
 						self->success = false;
-						return Void();
+						co_return;
 					}
 				}
 			} else {
 				if (res.size() > memRes.size() || (res.size() < memRes.size() && !res.more) ||
-				    (res.size() == 0 && res.more && !resized)) {
+				    (res.empty() && res.more && !resized)) {
 					TraceEvent(SevError, "WDRGetRangeWrongResult", randomID)
 					    .detail("BeginKey", begin.getKey())
 					    .detail("BeginOffset", begin.offset)
@@ -349,7 +349,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 					    .detail("SystemKeyCount", systemKeyCount);
 
 					self->success = false;
-					return Void();
+					co_return;
 				}
 
 				for (int i = 0; i < res.size(); i++) {
@@ -374,18 +374,18 @@ struct WriteDuringReadWorkload : TestWorkload {
 						    .detail("ReadYourWritesDisabled", readYourWritesDisabled)
 						    .detail("More", res.more);
 						self->success = false;
-						return Void();
+						co_return;
 					}
 				}
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			//TraceEvent("WDRGetRangeError", randomID).error(e,true);
 			if (e.code() == error_code_used_during_commit) {
 				ASSERT(*doingCommit);
-				return Void();
+				co_return;
 			} else if (e.code() == error_code_transaction_cancelled)
-				return Void();
+				co_return;
 			throw;
 		}
 	}
@@ -398,23 +398,23 @@ struct WriteDuringReadWorkload : TestWorkload {
 			return iter->second;
 	}
 
-	ACTOR Future<Void> getAndCompare(ReadYourWritesTransaction* tr,
-	                                 Key key,
-	                                 Snapshot snapshot,
-	                                 bool readYourWritesDisabled,
-	                                 bool snapshotRYWDisabled,
-	                                 WriteDuringReadWorkload* self,
-	                                 bool* doingCommit,
-	                                 int64_t* memLimit) {
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+	Future<Void> getAndCompare(ReadYourWritesTransaction* tr,
+	                           Key key,
+	                           Snapshot snapshot,
+	                           bool readYourWritesDisabled,
+	                           bool snapshotRYWDisabled,
+	                           WriteDuringReadWorkload* self,
+	                           bool* doingCommit,
+	                           int64_t* memLimit) {
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 		//TraceEvent("WDRGet", randomID);
 		try {
-			state Optional<Value> memRes = self->memoryGet(readYourWritesDisabled || (snapshot && snapshotRYWDisabled)
-			                                                   ? &self->lastCommittedDatabase
-			                                                   : &self->memoryDatabase,
-			                                               key);
+			Optional<Value> memRes = self->memoryGet(readYourWritesDisabled || (snapshot && snapshotRYWDisabled)
+			                                             ? &self->lastCommittedDatabase
+			                                             : &self->memoryDatabase,
+			                                         key);
 			*memLimit -= memRes.expectedSize();
-			Optional<Value> res = wait(tr->get(key, snapshot));
+			Optional<Value> res = co_await tr->get(key, snapshot);
 			*memLimit += memRes.expectedSize();
 			if (res != memRes) {
 				TraceEvent(SevError, "WDRGetWrongResult", randomID)
@@ -425,84 +425,85 @@ struct WriteDuringReadWorkload : TestWorkload {
 				    .detail("RywDisable", readYourWritesDisabled);
 				self->success = false;
 			}
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			//TraceEvent("WDRGetError", randomID).error(e,true);
 			if (e.code() == error_code_used_during_commit) {
 				ASSERT(*doingCommit);
-				return Void();
+				co_return;
 			} else if (e.code() == error_code_transaction_cancelled)
-				return Void();
+				co_return;
 			throw;
 		}
 	}
 
-	ACTOR Future<Void> watchAndCompare(ReadYourWritesTransaction* tr,
-	                                   Key key,
-	                                   bool readYourWritesDisabled,
-	                                   WriteDuringReadWorkload* self,
-	                                   bool* doingCommit,
-	                                   int64_t* memLimit) {
-		state UID randomID = nondeterministicRandom()->randomUniqueID();
+	Future<Void> watchAndCompare(ReadYourWritesTransaction* tr,
+	                             Key key,
+	                             bool readYourWritesDisabled,
+	                             WriteDuringReadWorkload* self,
+	                             bool* doingCommit,
+	                             int64_t* memLimit) {
+		UID randomID = nondeterministicRandom()->randomUniqueID();
 		// SOMEDAY: test setting a low outstanding watch limit
 		if (readYourWritesDisabled) // Only tests RYW activated watches
-			return Void();
+			co_return;
 
 		//TraceEvent("WDRWatch", randomID).detail("Key", key);
 		try {
-			state int changeNum = self->changeCount[key];
-			state Optional<Value> memRes = self->memoryGet(&self->memoryDatabase, key);
+			int changeNum = self->changeCount[key];
+			Optional<Value> memRes = self->memoryGet(&self->memoryDatabase, key);
 			*memLimit -= memRes.expectedSize();
 
 			// This if block prevents formatting issues with clang-format
 			if (1) {
-				choose {
-					when(wait(tr->watch(key))) {
-						if (changeNum == self->changeCount[key]) {
-							TraceEvent(SevError, "WDRWatchWrongResult", randomID)
-							    .detail("Reason", "Triggered without changing")
-							    .detail("Key", key)
-							    .detail("Value", changeNum)
-							    .detail("DuringCommit", *doingCommit);
-						}
+				auto choice = co_await race(tr->watch(key), self->finished.onTrigger());
+				if (choice.index() == 0) {
+					if (changeNum == self->changeCount[key]) {
+						TraceEvent(SevError, "WDRWatchWrongResult", randomID)
+						    .detail("Reason", "Triggered without changing")
+						    .detail("Key", key)
+						    .detail("Value", changeNum)
+						    .detail("DuringCommit", *doingCommit);
 					}
-					when(wait(self->finished.onTrigger())) {
-						Optional<Value> memRes2 = self->memoryGet(&self->memoryDatabase, key);
-						if (memRes != memRes2) {
-							TraceEvent(SevError, "WDRWatchWrongResult", randomID)
-							    .detail("Reason", "Changed without triggering")
-							    .detail("Key", key)
-							    .detail("Value1", memRes)
-							    .detail("Value2", memRes2);
-						}
+				} else if (choice.index() == 1) {
+
+					Optional<Value> memRes2 = self->memoryGet(&self->memoryDatabase, key);
+					if (memRes != memRes2) {
+						TraceEvent(SevError, "WDRWatchWrongResult", randomID)
+						    .detail("Reason", "Changed without triggering")
+						    .detail("Key", key)
+						    .detail("Value1", memRes)
+						    .detail("Value2", memRes2);
 					}
+				} else {
+					UNREACHABLE();
 				}
 			}
 			*memLimit += memRes.expectedSize();
 
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			// check for transaction cancelled if the watch was not committed
 			//TraceEvent("WDRWatchError", randomID).error(e,true);
 			if (e.code() == error_code_used_during_commit) {
 				ASSERT(*doingCommit);
-				return Void();
+				co_return;
 			} else if (e.code() == error_code_transaction_cancelled)
-				return Void();
+				co_return;
 			throw;
 		}
 	}
 
-	ACTOR Future<Void> commitAndUpdateMemory(ReadYourWritesTransaction* tr,
-	                                         WriteDuringReadWorkload* self,
-	                                         bool* cancelled,
-	                                         bool readYourWritesDisabled,
-	                                         bool snapshotRYWDisabled,
-	                                         bool readAheadDisabled,
-	                                         bool useBatchPriority,
-	                                         bool* doingCommit,
-	                                         double* startTime,
-	                                         Key timebombStr) {
+	Future<Void> commitAndUpdateMemory(ReadYourWritesTransaction* tr,
+	                                   WriteDuringReadWorkload* self,
+	                                   bool* cancelled,
+	                                   bool readYourWritesDisabled,
+	                                   bool snapshotRYWDisabled,
+	                                   bool readAheadDisabled,
+	                                   bool useBatchPriority,
+	                                   bool* doingCommit,
+	                                   double* startTime,
+	                                   Key timebombStr) {
 		// state UID randomID = nondeterministicRandom()->randomUniqueID();
 		//TraceEvent("WDRCommit", randomID);
 		try {
@@ -548,10 +549,10 @@ struct WriteDuringReadWorkload : TestWorkload {
 				}
 			}
 
-			state int64_t txnSize = tr->getApproximateSize();
-			state std::map<Key, Value> committedDB = self->memoryDatabase;
+			int64_t txnSize = tr->getApproximateSize();
+			std::map<Key, Value> committedDB = self->memoryDatabase;
 			*doingCommit = true;
-			wait(tr->commit());
+			co_await tr->commit();
 			*doingCommit = false;
 			self->finished.trigger();
 			self->dataWritten += txnSize;
@@ -579,7 +580,7 @@ struct WriteDuringReadWorkload : TestWorkload {
 			//TraceEvent("WDRCommitSuccess", randomID).detail("CommittedVersion", tr->getCommittedVersion());
 			self->lastCommittedDatabase = committedDB;
 
-			return Void();
+			co_return;
 		} catch (Error& e) {
 			//TraceEvent("WDRCommitCancelled", randomID).error(e,true);
 			if (e.code() == error_code_actor_cancelled || e.code() == error_code_transaction_cancelled ||
@@ -608,43 +609,45 @@ struct WriteDuringReadWorkload : TestWorkload {
 
 	// Prevent a write only transaction whose commit was previously cancelled from being reordered after this
 	// transaction
-	ACTOR Future<Void> writeBarrier(Database cx) {
-		state Transaction tr(cx);
-		loop {
+	Future<Void> writeBarrier(Database cx) {
+		Transaction tr(cx);
+		while (true) {
+			Error err;
 			try {
 				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
 				// Write-only transactions have a self-conflict in the system keys
 				tr.addWriteConflictRange(allKeys);
-				wait(tr.commit());
-				return Void();
+				co_await tr.commit();
+				co_return;
 			} catch (Error& e) {
-				wait(tr.onError(e));
+				err = e;
 			}
+			co_await tr.onError(err);
 		}
 	}
 
-	ACTOR Future<Void> loadAndRun(Database cx, WriteDuringReadWorkload* self) {
-		state double startTime = now();
-		loop {
-			wait(self->writeBarrier(cx));
+	Future<Void> loadAndRun(Database cx, WriteDuringReadWorkload* self) {
+		double startTime = now();
+		while (true) {
+			co_await self->writeBarrier(cx);
 
-			state int i = 0;
-			state int keysPerBatch =
+			int keysPerBatch =
 			    std::min<int64_t>(1000,
 			                      1 + CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT / 6 /
 			                              (self->getKeyForIndex(self->nodes).size() + self->valueSizeRange.second));
 			self->memoryDatabase = std::map<Key, Value>();
-			for (; i < self->nodes; i += keysPerBatch) {
-				state Transaction tr(cx);
-				loop {
+			for (int i = 0; i < self->nodes; i += keysPerBatch) {
+				Transaction tr(cx);
+				while (true) {
+					Error err;
 					try {
 						if (now() - startTime > self->testDuration || self->dataWritten >= self->maximumDataWritten)
-							return Void();
+							co_return;
 						if (self->useSystemKeys)
 							tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 
-						state int64_t txnSize = 0;
+						int64_t txnSize = 0;
 						if (i == 0) {
 							tr.clear(normalKeys);
 						}
@@ -668,36 +671,37 @@ struct WriteDuringReadWorkload : TestWorkload {
 								}
 							}
 						}
-						wait(tr.commit());
+						co_await tr.commit();
 						self->dataWritten += txnSize;
 						//TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr.getCommittedVersion());
 						break;
 					} catch (Error& e) {
-						wait(tr.onError(e));
+						err = e;
 					}
+					co_await tr.onError(err);
 				}
 			}
 			self->lastCommittedDatabase = self->memoryDatabase;
 			self->addedConflicts.insert(allKeys, false);
 			//TraceEvent("WDRInit");
 
-			loop {
-				wait(delay(now() - startTime > self->slowModeStart ||
-				                   (g_network->isSimulated() && g_simulator->speedUpSimulation)
-				               ? 1.0
-				               : 0.1));
+			while (true) {
+				co_await delay(now() - startTime > self->slowModeStart ||
+				                       (g_network->isSimulated() && g_simulator->speedUpSimulation)
+				                   ? 1.0
+				                   : 0.1);
 				try {
-					wait(self->randomTransaction(
+					co_await self->randomTransaction(
 					    (self->useExtraDB && deterministicRandom()->random01() < 0.5) ? self->extraDB : cx,
 					    self,
-					    startTime));
+					    startTime);
 				} catch (Error& e) {
 					if (e.code() != error_code_not_committed)
 						throw;
 					break;
 				}
 				if (now() - startTime > self->testDuration || self->dataWritten >= self->maximumDataWritten)
-					return Void();
+					co_return;
 			}
 		}
 	}
@@ -775,49 +779,49 @@ struct WriteDuringReadWorkload : TestWorkload {
 		return Value();
 	}
 
-	ACTOR Future<Void> randomTransaction(Database cx, WriteDuringReadWorkload* self, double testStartTime) {
-		state ReadYourWritesTransaction tr(cx);
-		state bool readYourWritesDisabled = deterministicRandom()->random01() < 0.5;
-		state bool readAheadDisabled = deterministicRandom()->random01() < 0.5;
-		state bool snapshotRYWDisabled = deterministicRandom()->random01() < 0.5;
-		state bool useBatchPriority = deterministicRandom()->random01() < 0.5;
-		state int64_t timebomb =
+	Future<Void> randomTransaction(Database cx, WriteDuringReadWorkload* self, double testStartTime) {
+		ReadYourWritesTransaction tr(cx);
+		bool readYourWritesDisabled = deterministicRandom()->random01() < 0.5;
+		bool readAheadDisabled = deterministicRandom()->random01() < 0.5;
+		bool snapshotRYWDisabled = deterministicRandom()->random01() < 0.5;
+		bool useBatchPriority = deterministicRandom()->random01() < 0.5;
+		int64_t timebomb =
 		    (FLOW_KNOBS->MAX_BUGGIFIED_DELAY == 0.0 && deterministicRandom()->random01() < 0.01)
 		        ? deterministicRandom()->randomInt64(1, 6000)
 		        : 0; // timebomb check can fail incorrectly if simulation injects delay longer than the timebomb
-		state std::vector<Future<Void>> operations;
-		state ActorCollection commits(false);
-		state std::vector<Future<Void>> watches;
-		state int changeNum = 1;
-		state bool doingCommit = false;
-		state int waitLocation = 0;
-		state double startTime = now();
-
-		state bool disableGetKey = BUGGIFY;
-		state bool disableGetRange = BUGGIFY;
-		state bool disableGet = BUGGIFY;
-		state bool disableCommit = BUGGIFY;
-		state bool disableClearRange = BUGGIFY;
-		state bool disableClear = BUGGIFY;
-		state bool disableWatch = BUGGIFY;
-		state bool disableWriteConflictRange = BUGGIFY;
-		state bool disableDelay = BUGGIFY;
-		state bool disableReset = BUGGIFY;
-		state bool disableReadConflictRange = BUGGIFY;
-		state bool disableSet = BUGGIFY;
-		state bool disableAtomicOp = BUGGIFY;
-
-		state Key timebombStr = makeString(8);
+		std::vector<Future<Void>> operations;
+		ActorCollection commits(false);
+		std::vector<Future<Void>> watches;
+		int changeNum = 1;
+		bool doingCommit = false;
+		int waitLocation = 0;
+		double startTime = now();
+
+		bool disableGetKey = BUGGIFY;
+		bool disableGetRange = BUGGIFY;
+		bool disableGet = BUGGIFY;
+		bool disableCommit = BUGGIFY;
+		bool disableClearRange = BUGGIFY;
+		bool disableClear = BUGGIFY;
+		bool disableWatch = BUGGIFY;
+		bool disableWriteConflictRange = BUGGIFY;
+		bool disableDelay = BUGGIFY;
+		bool disableReset = BUGGIFY;
+		bool disableReadConflictRange = BUGGIFY;
+		bool disableSet = BUGGIFY;
+		bool disableAtomicOp = BUGGIFY;
+
+		Key timebombStr = makeString(8);
 		uint8_t* data = mutateString(timebombStr);
 		memcpy(data, &timebomb, 8);
 
-		loop {
+		while (true) {
 			if (now() - testStartTime > self->testDuration) {
-				return Void();
+				co_return;
 			}
 
-			state int64_t memLimit = 1e8;
-			state bool cancelled = false;
+			int64_t memLimit = 1e8;
+			bool cancelled = false;
 			if (readYourWritesDisabled)
 				tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
 			if (snapshotRYWDisabled)
@@ -831,244 +835,254 @@ struct WriteDuringReadWorkload : TestWorkload {
 			tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
 			tr.addWriteConflictRange(self->conflictRange);
 			self->addedConflicts.insert(self->conflictRange, true);
-			try {
-				state int numWaits = deterministicRandom()->randomInt(1, 5);
-				state int i = 0;
-				for (; i < numWaits && memLimit > 0; i++) {
-					//TraceEvent("WDROps").detail("Count", i).detail("Max", numWaits).detail("ReadYourWritesDisabled",readYourWritesDisabled);
-					state int numOps = deterministicRandom()->randomInt(1, self->numOps);
-					state int j = 0;
-					for (; j < numOps && memLimit > 0; j++) {
-						if (commits.getResult().isError())
-							throw commits.getResult().getError();
-						try {
-							state int operationType = deterministicRandom()->randomInt(0, 21);
-							if (operationType == 0 && !disableGetKey) {
-								operations.push_back(
-								    self->getKeyAndCompare(&tr,
-								                           self->getRandomKeySelector(),
-								                           Snapshot{ deterministicRandom()->coinflip() },
-								                           readYourWritesDisabled,
-								                           snapshotRYWDisabled,
-								                           self,
-								                           &doingCommit,
-								                           &memLimit));
-							} else if (operationType == 1 && !disableGetRange) {
-								operations.push_back(
-								    self->getRangeAndCompare(&tr,
-								                             self->getRandomKeySelector(),
-								                             self->getRandomKeySelector(),
-								                             self->getRandomLimits(),
-								                             Snapshot{ deterministicRandom()->coinflip() },
-								                             Reverse{ deterministicRandom()->coinflip() },
-								                             readYourWritesDisabled,
-								                             snapshotRYWDisabled,
-								                             self,
-								                             &doingCommit,
-								                             &memLimit));
-							} else if (operationType == 2 && !disableGet) {
-								operations.push_back(self->getAndCompare(&tr,
-								                                         self->getRandomKey(),
-								                                         Snapshot{ deterministicRandom()->coinflip() },
-								                                         readYourWritesDisabled,
-								                                         snapshotRYWDisabled,
-								                                         self,
-								                                         &doingCommit,
-								                                         &memLimit));
-							} else if (operationType == 3 && !disableCommit) {
-								if (!self->rarelyCommit || deterministicRandom()->random01() < 1.0 / self->numOps) {
-									Future<Void> commit = self->commitAndUpdateMemory(&tr,
-									                                                  self,
-									                                                  &cancelled,
-									                                                  readYourWritesDisabled,
-									                                                  snapshotRYWDisabled,
-									                                                  readAheadDisabled,
-									                                                  useBatchPriority,
-									                                                  &doingCommit,
-									                                                  &startTime,
-									                                                  timebombStr);
-									operations.push_back(commit);
-									commits.add(commit);
-								}
-							} else if (operationType == 4 && !disableClearRange) {
-								KeyRange range = self->getRandomRange(self->maxClearSize);
-								self->changeCount.insert(range, changeNum++);
-								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRClearRange").detail("Begin", range).detail("NoConflict", noConflict);
-								if (noConflict)
-									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-								tr.clear(range);
-								if (!noConflict) {
-									KeyRangeRef conflict(
-									    range.begin.substr(
-									        0, std::min<int>(range.begin.size(), getMaxClearKeySize(range.begin) + 1)),
-									    range.end.substr(
-									        0, std::min<int>(range.end.size(), getMaxClearKeySize(range.end) + 1)));
-									self->addedConflicts.insert(conflict, true);
-								}
-								self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
-								                           self->memoryDatabase.lower_bound(range.end));
-							} else if (operationType == 5 && !disableClear) {
-								Key key = self->getRandomKey();
-								self->changeCount.insert(key, changeNum++);
-								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRClear").detail("Key", key).detail("NoConflict", noConflict);
-								if (noConflict)
-									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-								tr.clear(key);
-								if (!noConflict && key.size() <= getMaxClearKeySize(key)) {
-									self->addedConflicts.insert(key, true);
-								}
-								self->memoryDatabase.erase(key);
-							} else if (operationType == 6 && !disableWatch) {
-								watches.push_back(self->watchAndCompare(
-								    &tr, self->getRandomKey(), readYourWritesDisabled, self, &doingCommit, &memLimit));
-							} else if (operationType == 7 && !disableWriteConflictRange) {
-								KeyRange range = self->getRandomRange(self->nodes);
-								//TraceEvent("WDRAddWriteConflict").detail("Range", range);
-								tr.addWriteConflictRange(range);
-								KeyRangeRef conflict(
-								    range.begin.substr(
-								        0, std::min<int>(range.begin.size(), getMaxKeySize(range.begin) + 1)),
-								    range.end.substr(0, std::min<int>(range.end.size(), getMaxKeySize(range.end) + 1)));
-								self->addedConflicts.insert(conflict, true);
-							} else if (operationType == 8 && !disableDelay) {
-								double maxTime = 6.0;
-								if (timebomb > 0)
-									maxTime = startTime + timebomb / 1000.0 - now();
-								operations.push_back(
-								    delay(deterministicRandom()->random01() * deterministicRandom()->random01() *
-								          deterministicRandom()->random01() * maxTime));
-							} else if (operationType == 9 && !disableReset) {
-								if (deterministicRandom()->random01() < 0.001) {
-									//TraceEvent("WDRReset");
-									tr.reset();
-									self->memoryDatabase = self->lastCommittedDatabase;
-									self->addedConflicts.insert(allKeys, false);
-									if (readYourWritesDisabled)
-										tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
-									if (snapshotRYWDisabled)
-										tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
-									if (readAheadDisabled)
-										tr.setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
-									if (self->useSystemKeys)
-										tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-									tr.addWriteConflictRange(self->conflictRange);
-									self->addedConflicts.insert(self->conflictRange, true);
-									startTime = now();
-									tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
-								}
-							} else if (operationType == 10 && !disableReadConflictRange) {
-								KeyRange range = self->getRandomRange(self->maxClearSize);
-								tr.addReadConflictRange(range);
-							} else if (operationType == 11 && !disableAtomicOp) {
-								if (!self->useSystemKeys && deterministicRandom()->random01() < 0.01) {
-									Key versionStampKey = self->getRandomVersionStampKey();
-									Value value = self->getRandomValue();
-									KeyRangeRef range = getVersionstampKeyRange(versionStampKey.arena(),
-									                                            versionStampKey,
-									                                            tr.getCachedReadVersion(),
-									                                            normalKeys.end);
+			{
+				Error err;
+				try {
+					int numWaits = deterministicRandom()->randomInt(1, 5);
+					for (int i = 0; i < numWaits && memLimit > 0; ++i) {
+						//TraceEvent("WDROps").detail("Count", i).detail("Max", numWaits).detail("ReadYourWritesDisabled",readYourWritesDisabled);
+						int numOps = deterministicRandom()->randomInt(1, self->numOps);
+						for (int j = 0; j < numOps && memLimit > 0; ++j) {
+							if (commits.getResult().isError())
+								throw commits.getResult().getError();
+							try {
+								int operationType = deterministicRandom()->randomInt(0, 21);
+								if (operationType == 0 && !disableGetKey) {
+									operations.push_back(
+									    self->getKeyAndCompare(&tr,
+									                           self->getRandomKeySelector(),
+									                           Snapshot{ deterministicRandom()->coinflip() },
+									                           readYourWritesDisabled,
+									                           snapshotRYWDisabled,
+									                           self,
+									                           &doingCommit,
+									                           &memLimit));
+								} else if (operationType == 1 && !disableGetRange) {
+									operations.push_back(
+									    self->getRangeAndCompare(&tr,
+									                             self->getRandomKeySelector(),
+									                             self->getRandomKeySelector(),
+									                             self->getRandomLimits(),
+									                             Snapshot{ deterministicRandom()->coinflip() },
+									                             Reverse{ deterministicRandom()->coinflip() },
+									                             readYourWritesDisabled,
+									                             snapshotRYWDisabled,
+									                             self,
+									                             &doingCommit,
+									                             &memLimit));
+								} else if (operationType == 2 && !disableGet) {
+									operations.push_back(
+									    self->getAndCompare(&tr,
+									                        self->getRandomKey(),
+									                        Snapshot{ deterministicRandom()->coinflip() },
+									                        readYourWritesDisabled,
+									                        snapshotRYWDisabled,
+									                        self,
+									                        &doingCommit,
+									                        &memLimit));
+								} else if (operationType == 3 && !disableCommit) {
+									if (!self->rarelyCommit || deterministicRandom()->random01() < 1.0 / self->numOps) {
+										Future<Void> commit = self->commitAndUpdateMemory(&tr,
+										                                                  self,
+										                                                  &cancelled,
+										                                                  readYourWritesDisabled,
+										                                                  snapshotRYWDisabled,
+										                                                  readAheadDisabled,
+										                                                  useBatchPriority,
+										                                                  &doingCommit,
+										                                                  &startTime,
+										                                                  timebombStr);
+										operations.push_back(commit);
+										commits.add(commit);
+									}
+								} else if (operationType == 4 && !disableClearRange) {
+									KeyRange range = self->getRandomRange(self->maxClearSize);
 									self->changeCount.insert(range, changeNum++);
-									//TraceEvent("WDRVersionStamp").detail("VersionStampKey", versionStampKey).detail("Range", range);
-									tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
+									bool noConflict = deterministicRandom()->random01() < 0.5;
+									//TraceEvent("WDRClearRange").detail("Begin", range).detail("NoConflict", noConflict);
+									if (noConflict)
+										tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
 									tr.clear(range);
+									if (!noConflict) {
+										KeyRangeRef conflict(
+										    range.begin.substr(
+										        0,
+										        std::min<int>(range.begin.size(), getMaxClearKeySize(range.begin) + 1)),
+										    range.end.substr(
+										        0, std::min<int>(range.end.size(), getMaxClearKeySize(range.end) + 1)));
+										self->addedConflicts.insert(conflict, true);
+									}
+									self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
+									                           self->memoryDatabase.lower_bound(range.end));
+								} else if (operationType == 5 && !disableClear) {
+									Key key = self->getRandomKey();
+									self->changeCount.insert(key, changeNum++);
+									bool noConflict = deterministicRandom()->random01() < 0.5;
+									//TraceEvent("WDRClear").detail("Key", key).detail("NoConflict", noConflict);
+									if (noConflict)
+										tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
+									tr.clear(key);
+									if (!noConflict && key.size() <= getMaxClearKeySize(key)) {
+										self->addedConflicts.insert(key, true);
+									}
+									self->memoryDatabase.erase(key);
+								} else if (operationType == 6 && !disableWatch) {
+									watches.push_back(self->watchAndCompare(&tr,
+									                                        self->getRandomKey(),
+									                                        readYourWritesDisabled,
+									                                        self,
+									                                        &doingCommit,
+									                                        &memLimit));
+								} else if (operationType == 7 && !disableWriteConflictRange) {
+									KeyRange range = self->getRandomRange(self->nodes);
+									//TraceEvent("WDRAddWriteConflict").detail("Range", range);
+									tr.addWriteConflictRange(range);
 									KeyRangeRef conflict(
 									    range.begin.substr(
-									        0, std::min<int>(range.begin.size(), getMaxClearKeySize(range.begin) + 1)),
+									        0, std::min<int>(range.begin.size(), getMaxKeySize(range.begin) + 1)),
 									    range.end.substr(
-									        0, std::min<int>(range.end.size(), getMaxClearKeySize(range.end) + 1)));
+									        0, std::min<int>(range.end.size(), getMaxKeySize(range.end) + 1)));
 									self->addedConflicts.insert(conflict, true);
-									self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
-									                           self->memoryDatabase.lower_bound(range.end));
-								} else {
+								} else if (operationType == 8 && !disableDelay) {
+									double maxTime = 6.0;
+									if (timebomb > 0)
+										maxTime = startTime + timebomb / 1000.0 - now();
+									operations.push_back(
+									    delay(deterministicRandom()->random01() * deterministicRandom()->random01() *
+									          deterministicRandom()->random01() * maxTime));
+								} else if (operationType == 9 && !disableReset) {
+									if (deterministicRandom()->random01() < 0.001) {
+										//TraceEvent("WDRReset");
+										tr.reset();
+										self->memoryDatabase = self->lastCommittedDatabase;
+										self->addedConflicts.insert(allKeys, false);
+										if (readYourWritesDisabled)
+											tr.setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
+										if (snapshotRYWDisabled)
+											tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE);
+										if (readAheadDisabled)
+											tr.setOption(FDBTransactionOptions::READ_AHEAD_DISABLE);
+										if (self->useSystemKeys)
+											tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+										tr.addWriteConflictRange(self->conflictRange);
+										self->addedConflicts.insert(self->conflictRange, true);
+										startTime = now();
+										tr.setOption(FDBTransactionOptions::TIMEOUT, timebombStr);
+									}
+								} else if (operationType == 10 && !disableReadConflictRange) {
+									KeyRange range = self->getRandomRange(self->maxClearSize);
+									tr.addReadConflictRange(range);
+								} else if (operationType == 11 && !disableAtomicOp) {
+									if (!self->useSystemKeys && deterministicRandom()->random01() < 0.01) {
+										Key versionStampKey = self->getRandomVersionStampKey();
+										Value value = self->getRandomValue();
+										KeyRangeRef range = getVersionstampKeyRange(versionStampKey.arena(),
+										                                            versionStampKey,
+										                                            tr.getCachedReadVersion(),
+										                                            normalKeys.end);
+										self->changeCount.insert(range, changeNum++);
+										//TraceEvent("WDRVersionStamp").detail("VersionStampKey", versionStampKey).detail("Range", range);
+										tr.atomicOp(versionStampKey, value, MutationRef::SetVersionstampedKey);
+										tr.clear(range);
+										KeyRangeRef conflict(
+										    range.begin.substr(
+										        0,
+										        std::min<int>(range.begin.size(), getMaxClearKeySize(range.begin) + 1)),
+										    range.end.substr(
+										        0, std::min<int>(range.end.size(), getMaxClearKeySize(range.end) + 1)));
+										self->addedConflicts.insert(conflict, true);
+										self->memoryDatabase.erase(self->memoryDatabase.lower_bound(range.begin),
+										                           self->memoryDatabase.lower_bound(range.end));
+									} else {
+										Key key = self->getRandomKey();
+										Value value = self->getRandomValue();
+										MutationRef::Type opType;
+										switch (deterministicRandom()->randomInt(0, 8)) {
+										case 0:
+											opType = MutationRef::AddValue;
+											break;
+										case 1:
+											opType = MutationRef::And;
+											break;
+										case 2:
+											opType = MutationRef::Or;
+											break;
+										case 3:
+											opType = MutationRef::Xor;
+											break;
+										case 4:
+											opType = MutationRef::Max;
+											break;
+										case 5:
+											opType = MutationRef::Min;
+											break;
+										case 6:
+											opType = MutationRef::ByteMin;
+											break;
+										case 7:
+											opType = MutationRef::ByteMax;
+											break;
+										}
+										self->changeCount.insert(key, changeNum++);
+										bool noConflict = deterministicRandom()->random01() < 0.5;
+										//TraceEvent("WDRAtomicOp").detail("Key", key).detail("Value", value.size()).detail("NoConflict", noConflict);
+										if (noConflict)
+											tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
+										tr.atomicOp(key, value, opType);
+										//TraceEvent("WDRAtomicOpSuccess").detail("Key", key).detail("Value", value.size());
+										if (!noConflict && key.size() <= getMaxWriteKeySize(key, self->useSystemKeys)) {
+											self->addedConflicts.insert(key, true);
+										}
+										Optional<Value> existing = self->memoryGet(&self->memoryDatabase, key);
+										self->memoryDatabase[key] =
+										    self->applyAtomicOp(existing.present() ? Optional<StringRef>(existing.get())
+										                                           : Optional<StringRef>(),
+										                        value,
+										                        opType);
+									}
+								} else if (operationType > 11 && !disableSet) {
 									Key key = self->getRandomKey();
 									Value value = self->getRandomValue();
-									MutationRef::Type opType;
-									switch (deterministicRandom()->randomInt(0, 8)) {
-									case 0:
-										opType = MutationRef::AddValue;
-										break;
-									case 1:
-										opType = MutationRef::And;
-										break;
-									case 2:
-										opType = MutationRef::Or;
-										break;
-									case 3:
-										opType = MutationRef::Xor;
-										break;
-									case 4:
-										opType = MutationRef::Max;
-										break;
-									case 5:
-										opType = MutationRef::Min;
-										break;
-									case 6:
-										opType = MutationRef::ByteMin;
-										break;
-									case 7:
-										opType = MutationRef::ByteMax;
-										break;
-									}
 									self->changeCount.insert(key, changeNum++);
 									bool noConflict = deterministicRandom()->random01() < 0.5;
-									//TraceEvent("WDRAtomicOp").detail("Key", key).detail("Value", value.size()).detail("NoConflict", noConflict);
+									//TraceEvent("WDRSet").detail("Key", key).detail("Value", value.size()).detail("NoConflict", noConflict);
 									if (noConflict)
 										tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-									tr.atomicOp(key, value, opType);
-									//TraceEvent("WDRAtomicOpSuccess").detail("Key", key).detail("Value", value.size());
+									tr.set(key, value);
 									if (!noConflict && key.size() <= getMaxWriteKeySize(key, self->useSystemKeys)) {
 										self->addedConflicts.insert(key, true);
 									}
-									Optional<Value> existing = self->memoryGet(&self->memoryDatabase, key);
-									self->memoryDatabase[key] =
-									    self->applyAtomicOp(existing.present() ? Optional<StringRef>(existing.get())
-									                                           : Optional<StringRef>(),
-									                        value,
-									                        opType);
+									//TraceEvent("WDRSetSuccess").detail("Key", key).detail("Value", value.size());
+									self->memoryDatabase[key] = value;
 								}
-							} else if (operationType > 11 && !disableSet) {
-								Key key = self->getRandomKey();
-								Value value = self->getRandomValue();
-								self->changeCount.insert(key, changeNum++);
-								bool noConflict = deterministicRandom()->random01() < 0.5;
-								//TraceEvent("WDRSet").detail("Key", key).detail("Value", value.size()).detail("NoConflict", noConflict);
-								if (noConflict)
-									tr.setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
-								tr.set(key, value);
-								if (!noConflict && key.size() <= getMaxWriteKeySize(key, self->useSystemKeys)) {
-									self->addedConflicts.insert(key, true);
+							} catch (Error& e) {
+								if (e.code() == error_code_used_during_commit) {
+									ASSERT(doingCommit);
+								} else if (e.code() != error_code_transaction_cancelled) {
+									throw;
 								}
-								//TraceEvent("WDRSetSuccess").detail("Key", key).detail("Value", value.size());
-								self->memoryDatabase[key] = value;
-							}
-						} catch (Error& e) {
-							if (e.code() == error_code_used_during_commit) {
-								ASSERT(doingCommit);
-							} else if (e.code() != error_code_transaction_cancelled) {
-								throw;
 							}
 						}
-					}
 
-					if (waitLocation < operations.size()) {
-						int waitOp = deterministicRandom()->randomInt(waitLocation, operations.size());
-						//TraceEvent("WDRWait").detail("Op", waitOp).detail("Operations", operations.size()).detail("WaitLocation", waitLocation);
-						wait(operations[waitOp]);
-						wait(delay(0.000001)); // to ensure errors have propagated from reads to commits
-						waitLocation = operations.size();
+						if (waitLocation < operations.size()) {
+							int waitOp = deterministicRandom()->randomInt(waitLocation, operations.size());
+							//TraceEvent("WDRWait").detail("Op", waitOp).detail("Operations", operations.size()).detail("WaitLocation", waitLocation);
+							co_await operations[waitOp];
+							co_await delay(0.000001); // to ensure errors have propagated from reads to commits
+							waitLocation = operations.size();
+						}
 					}
+					co_await waitForAll(operations);
+					ASSERT(timebomb == 0 || 1000 * (now() - startTime) <= timebomb + 1);
+					co_await tr.debug_onIdle();
+					co_await delay(0.000001); // to ensure triggered watches have a change to register
+					self->finished.trigger();
+					co_await waitForAll(watches); // only for errors, should have all returned
+					self->changeCount.insert(allKeys, 0);
+					break;
+				} catch (Error& e) {
+					err = e;
 				}
-				wait(waitForAll(operations));
-				ASSERT(timebomb == 0 || 1000 * (now() - startTime) <= timebomb + 1);
-				wait(tr.debug_onIdle());
-				wait(delay(0.000001)); // to ensure triggered watches have a change to register
-				self->finished.trigger();
-				wait(waitForAll(watches)); // only for errors, should have all returned
-				self->changeCount.insert(allKeys, 0);
-				break;
-			} catch (Error& e) {
 				operations.clear();
 				commits.clear(false);
 				waitLocation = 0;
@@ -1076,23 +1090,23 @@ struct WriteDuringReadWorkload : TestWorkload {
 				self->changeCount.insert(allKeys, 0);
 				doingCommit = false;
 				//TraceEvent("WDRError").errorUnsuppressed(e);
-				if (e.code() == error_code_database_locked) {
+				if (err.code() == error_code_database_locked) {
 					self->memoryDatabase = self->lastCommittedDatabase;
 					self->addedConflicts.insert(allKeys, false);
-					return Void();
+					co_return;
 				}
-				if (e.code() == error_code_not_committed || e.code() == error_code_commit_unknown_result ||
-				    e.code() == error_code_transaction_too_large || e.code() == error_code_key_too_large ||
-				    e.code() == error_code_value_too_large || e.code() == error_code_too_many_watches || cancelled)
+				if (err.code() == error_code_not_committed || err.code() == error_code_commit_unknown_result ||
+				    err.code() == error_code_transaction_too_large || err.code() == error_code_key_too_large ||
+				    err.code() == error_code_value_too_large || err.code() == error_code_too_many_watches || cancelled)
 					throw not_committed();
 				try {
-					wait(tr.onError(e));
-				} catch (Error& e) {
-					if (e.code() == error_code_transaction_timed_out) {
+					co_await tr.onError(err);
+				} catch (Error& err) {
+					if (err.code() == error_code_transaction_timed_out) {
 						ASSERT(timebomb != 0 && 1000 * (now() - startTime) >= timebomb - 1);
 						throw not_committed();
 					}
-					throw e;
+					throw err;
 				}
 				self->memoryDatabase = self->lastCommittedDatabase;
 				self->addedConflicts.insert(allKeys, false);
@@ -1100,7 +1114,6 @@ struct WriteDuringReadWorkload : TestWorkload {
 		}
 		self->memoryDatabase = self->lastCommittedDatabase;
 		self->addedConflicts.insert(allKeys, false);
-		return Void();
 	}
 };
 
diff --git a/fdbserver/workloads/WriteTagThrottling.actor.cpp b/fdbserver/workloads/WriteTagThrottling.cpp
similarity index 80%
rename from fdbserver/workloads/WriteTagThrottling.actor.cpp
rename to fdbserver/workloads/WriteTagThrottling.cpp
index bf87f43453f..66fa6d54933 100644
--- a/fdbserver/workloads/WriteTagThrottling.actor.cpp
+++ b/fdbserver/workloads/WriteTagThrottling.cpp
@@ -1,5 +1,5 @@
 /*
- * WriteThrottling.actor.cpp
+ * WriteTagThrottling.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,12 @@
  * limitations under the License.
  */
 
-#include "fdbserver/TesterInterface.actor.h"
-#include "fdbserver/workloads/workloads.actor.h"
-#include "fdbserver/workloads/BulkSetup.actor.h"
-#include "fdbserver/WorkerInterface.actor.h"
+#include "fdbserver/core/TesterInterface.h"
+#include "fdbserver/tester/workloads.h"
+#include "BulkSetup.h"
+#include "fdbserver/core/WorkerInterface.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/TagThrottle.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "fdbclient/TagThrottle.h"
 
 // workload description:
 // This workload aims to test whether we can throttling some bad clients that doing penetrating write on write hot-spot
@@ -62,7 +61,7 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 	static constexpr int MIN_TAGS_PER_TRANSACTION = 1;
 	static constexpr int MIN_TRANSACTION_TAG_LENGTH = 2;
 
-	WriteTagThrottlingWorkload(WorkloadContext const& wcx)
+	explicit WriteTagThrottlingWorkload(WorkloadContext const& wcx)
 	  : KVWorkload(wcx), badActorReadLatency(), goodActorReadLatency(), badActorCommitLatency(),
 	    goodActorCommitLatency() {
 		testDuration = getOption(options, "testDuration"_sr, 120.0);
@@ -90,16 +89,15 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 		goodTag = TransactionTag(std::string("gT"));
 	}
 
-	ACTOR static Future<Void> _setup(Database cx, WriteTagThrottlingWorkload* self) {
+	Future<Void> _setup(Database cx) {
 		ASSERT(CLIENT_KNOBS->MAX_TAGS_PER_TRANSACTION >= MIN_TAGS_PER_TRANSACTION &&
 		       CLIENT_KNOBS->MAX_TRANSACTION_TAG_LENGTH >= MIN_TRANSACTION_TAG_LENGTH);
-		if (self->populateData) {
-			wait(bulkSetup(cx, self, self->keyCount, Promise<double>()));
+		if (populateData) {
+			co_await bulkSetup(cx, this, keyCount, Promise<double>());
 		}
-		if (self->clientId == 0) {
-			wait(ThrottleApi::enableAuto(cx.getReference(), true));
+		if (clientId == 0) {
+			co_await ThrottleApi::enableAuto(cx.getReference(), true);
 		}
-		return Void();
 	}
 	Future<Void> setup(const Database& cx) override {
 		if (CLIENT_KNOBS->MAX_TAGS_PER_TRANSACTION < MIN_TAGS_PER_TRANSACTION ||
@@ -107,25 +105,24 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 			fastSuccess = true;
 			return Void();
 		}
-		return _setup(cx, this);
+		return _setup(cx);
 	}
-	ACTOR static Future<Void> _start(Database cx, WriteTagThrottlingWorkload* self) {
+	Future<Void> _start(Database cx) {
 		std::vector<Future<Void>> clientActors;
 		int actorId;
-		for (actorId = 0; actorId < self->goodActorPerClient; ++actorId) {
-			clientActors.push_back(clientActor(false, actorId, 0, cx, self));
+		for (actorId = 0; actorId < goodActorPerClient; ++actorId) {
+			clientActors.push_back(clientActor(false, actorId, 0, cx, this));
 		}
-		for (actorId = 0; actorId < self->badActorPerClient; ++actorId) {
-			clientActors.push_back(clientActor(true, actorId, self->badOpRate, cx, self));
+		for (actorId = 0; actorId < badActorPerClient; ++actorId) {
+			clientActors.push_back(clientActor(true, actorId, badOpRate, cx, this));
 		}
-		clientActors.push_back(throttledTagUpdater(cx, self));
-		wait(timeout(waitForAll(clientActors), self->testDuration, Void()));
-		return Void();
+		clientActors.push_back(throttledTagUpdater(cx));
+		co_await timeout(waitForAll(clientActors), testDuration, Void());
 	}
 	Future<Void> start(Database const& cx) override {
 		if (fastSuccess)
 			return Void();
-		return _start(cx, this);
+		return _start(cx);
 	}
 	Future<bool> check(Database const& cx) override {
 		if (fastSuccess)
@@ -214,22 +211,22 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 	Value generateVal() { return Value(deterministicRandom()->randomAlphaNumeric(maxValueBytes)); }
 
 	// read and write value on particular/random Key
-	ACTOR static Future<Void> clientActor(bool isBadActor,
-	                                      int actorId,
-	                                      double badOpRate,
-	                                      Database cx,
-	                                      WriteTagThrottlingWorkload* self) {
-		state int startIdx = (self->clientId * self->badActorPerClient + actorId) * self->rangeEachBadActor;
-		state int availableRange = std::max(int(self->rangeEachBadActor * self->hotRangeRate), 1);
-		state double lastTime = now();
-		state double opStart;
-		state Key key;
+	static Future<Void> clientActor(bool isBadActor,
+	                                int actorId,
+	                                double badOpRate,
+	                                Database cx,
+	                                WriteTagThrottlingWorkload* self) {
+		int startIdx = (self->clientId * self->badActorPerClient + actorId) * self->rangeEachBadActor;
+		int availableRange = std::max(int(self->rangeEachBadActor * self->hotRangeRate), 1);
+		double lastTime = now();
+		double opStart{ 0 };
+		Key key;
 		try {
-			loop {
-				wait(poisson(&lastTime, self->trInterval));
-				state double trStart;
-				state Transaction tr(cx);
-				state int i;
+			while (true) {
+				co_await poisson(&lastTime, self->trInterval);
+				double trStart{ 0 };
+				Transaction tr(cx);
+				int i{ 0 };
 				// give tag to client
 				if (self->writeThrottle) {
 					ASSERT(CLIENT_KNOBS->MAX_TAGS_PER_TRANSACTION >= MIN_TAGS_PER_TRANSACTION);
@@ -244,6 +241,7 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 
 				trStart = now();
 				while (true) {
+					Error err;
 					try {
 						for (i = 0; i < self->numClearPerTr; ++i) {
 							bool useClearKey = deterministicRandom()->random01() < badOpRate;
@@ -259,27 +257,28 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 							ASSERT(self->keyCount >= actorId);
 							key = self->generateKey(useReadKey, startIdx, availableRange);
 							opStart = now();
-							Optional<Value> v = wait(tr.get(key));
+							Optional<Value> v = co_await tr.get(key);
 							double duration = now() - opStart;
 							isBadActor ? self->badActorReadLatency.addSample(duration)
 							           : self->goodActorReadLatency.addSample(duration);
 						}
 						opStart = now();
-						wait(tr.commit());
+						co_await tr.commit();
 						double duration = now() - opStart;
 						isBadActor ? self->badActorCommitLatency.addSample(duration)
 						           : self->goodActorCommitLatency.addSample(duration);
 						break;
 					} catch (Error& e) {
-						if (e.code() == error_code_transaction_too_old) {
-							isBadActor ? ++self->badActorTooOldRetries : ++self->goodActorTooOldRetries;
-						} else if (e.code() == error_code_not_committed) {
-							isBadActor ? ++self->badActorCommitFailedRetries : ++self->goodActorCommitFailedRetries;
-						} else if (e.code() == error_code_tag_throttled) {
-							isBadActor ? ++self->badActorThrottleRetries : ++self->goodActorThrottleRetries;
-						}
-						wait(tr.onError(e));
+						err = e;
+					}
+					if (err.code() == error_code_transaction_too_old) {
+						isBadActor ? ++self->badActorTooOldRetries : ++self->goodActorTooOldRetries;
+					} else if (err.code() == error_code_not_committed) {
+						isBadActor ? ++self->badActorCommitFailedRetries : ++self->goodActorCommitFailedRetries;
+					} else if (err.code() == error_code_tag_throttled) {
+						isBadActor ? ++self->badActorThrottleRetries : ++self->goodActorThrottleRetries;
 					}
+					co_await tr.onError(err);
 					isBadActor ? ++self->badActorRetries : ++self->goodActorRetries;
 				}
 				double duration = now() - trStart;
@@ -302,13 +301,13 @@ struct WriteTagThrottlingWorkload : KVWorkload {
 			throttledTags.insert(tag.tag.toString());
 		}
 	}
-	ACTOR static Future<Void> throttledTagUpdater(Database cx, WriteTagThrottlingWorkload* self) {
-		state std::vector<TagThrottleInfo> tags;
-		state Reference<DatabaseContext> db = cx.getReference();
-		loop {
-			wait(delay(1.0));
-			wait(store(tags, ThrottleApi::getThrottledTags(db, CLIENT_KNOBS->TOO_MANY, ContainsRecommended::True)));
-			self->recordThrottledTags(tags);
+	Future<Void> throttledTagUpdater(Database cx) {
+		std::vector<TagThrottleInfo> tags;
+		Reference<DatabaseContext> db = cx.getReference();
+		while (true) {
+			co_await delay(1.0);
+			tags = co_await ThrottleApi::getThrottledTags(db, CLIENT_KNOBS->TOO_MANY, ContainsRecommended::True);
+			recordThrottledTags(tags);
 		}
 	}
 
diff --git a/fdbserver/pubsub.actor.cpp b/fdbserver/workloads/pubsub.cpp
similarity index 62%
rename from fdbserver/pubsub.actor.cpp
rename to fdbserver/workloads/pubsub.cpp
index 1a17c9014a3..c813a5bed45 100644
--- a/fdbserver/pubsub.actor.cpp
+++ b/fdbserver/workloads/pubsub.cpp
@@ -1,5 +1,5 @@
 /*
- * pubsub.actor.cpp
+ * pubsub.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,8 +20,7 @@
 
 #include <cinttypes>
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/pubsub.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "pubsub.h"
 
 Value uInt64ToValue(uint64_t v) {
 	return StringRef(format("%016llx", v));
@@ -101,88 +100,93 @@ Key keyForDisptchEntry(uint64_t message) {
 
 PubSub::PubSub(Database _cx) : cx(_cx) {}
 
-ACTOR Future<uint64_t> _createFeed(Database cx, Standalone<StringRef> metadata) {
-	state uint64_t id(deterministicRandom()->randomUniqueID().first()); // SOMEDAY: this should be an atomic increment
+Future<uint64_t> _createFeed(Database cx, Standalone<StringRef> metadata) {
+	uint64_t id(deterministicRandom()->randomUniqueID().first()); // SOMEDAY: this should be an atomic increment
 	TraceEvent("PubSubCreateFeed").detail("Feed", id);
-	state Transaction tr(cx);
-	loop {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			state Optional<Value> val = wait(tr.get(keyForFeed(id)));
+			Optional<Value> val = co_await tr.get(keyForFeed(id));
 			while (val.present()) {
 				id = id + deterministicRandom()->randomInt(1, 100);
-				Optional<Value> v = wait(tr.get(keyForFeed(id)));
+				Optional<Value> v = co_await tr.get(keyForFeed(id));
 				val = v;
 			}
 			tr.set(keyForFeed(id), metadata);
 			tr.set(keyForFeedSubscriberCount(id), uInt64ToValue(0));
 			tr.set(keyForFeedMessageCount(id), uInt64ToValue(0));
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return id;
+	co_return id;
 }
 
 Future<uint64_t> PubSub::createFeed(Standalone<StringRef> metadata) {
 	return _createFeed(cx, metadata);
 }
 
-ACTOR Future<uint64_t> _createInbox(Database cx, Standalone<StringRef> metadata) {
-	state uint64_t id = deterministicRandom()->randomUniqueID().first();
+Future<uint64_t> _createInbox(Database cx, Standalone<StringRef> metadata) {
+	uint64_t id = deterministicRandom()->randomUniqueID().first();
 	TraceEvent("PubSubCreateInbox").detail("Inbox", id);
-	state Transaction tr(cx);
-	loop {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
-			state Optional<Value> val = wait(tr.get(keyForInbox(id)));
+			Optional<Value> val = co_await tr.get(keyForInbox(id));
 			while (val.present()) {
 				id += deterministicRandom()->randomInt(1, 100);
-				Optional<Value> v = wait(tr.get(keyForFeed(id)));
+				Optional<Value> v = co_await tr.get(keyForFeed(id));
 				val = v;
 			}
 			tr.set(keyForInbox(id), metadata);
 			tr.set(keyForInboxSubscriptionCount(id), uInt64ToValue(0));
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return id;
+	co_return id;
 }
 
 Future<uint64_t> PubSub::createInbox(Standalone<StringRef> metadata) {
 	return _createInbox(cx, metadata);
 }
 
-ACTOR Future<bool> _createSubscription(Database cx, uint64_t feed, uint64_t inbox) {
-	state Transaction tr(cx);
+Future<bool> _createSubscription(Database cx, uint64_t feed, uint64_t inbox) {
+	Transaction tr(cx);
 	TraceEvent("PubSubCreateSubscription").detail("Feed", feed).detail("Inbox", inbox);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			Optional<Value> subscription = wait(tr.get(keyForInboxSubscription(inbox, feed)));
+			Optional<Value> subscription = co_await tr.get(keyForInboxSubscription(inbox, feed));
 			if (subscription.present()) {
 				// For idempotency, this could exist from a previous transaction from us that succeeded
-				return true;
+				co_return true;
 			}
-			Optional<Value> inboxVal = wait(tr.get(keyForInbox(inbox)));
+			Optional<Value> inboxVal = co_await tr.get(keyForInbox(inbox));
 			if (!inboxVal.present()) {
-				return false;
+				co_return false;
 			}
-			Optional<Value> feedVal = wait(tr.get(keyForFeed(feed)));
+			Optional<Value> feedVal = co_await tr.get(keyForFeed(feed));
 			if (!feedVal.present()) {
-				return false;
+				co_return false;
 			}
 
 			// Update the subscriptions of the inbox
-			Optional<Value> subscriptionCountVal = wait(tr.get(keyForInboxSubscriptionCount(inbox)));
+			Optional<Value> subscriptionCountVal = co_await tr.get(keyForInboxSubscriptionCount(inbox));
 			uint64_t subscriptionCount = valueToUInt64(subscriptionCountVal.get()); // throws if count not present
 			tr.set(keyForInboxSubscription(inbox, feed), StringRef());
 			tr.set(keyForInboxSubscriptionCount(inbox), uInt64ToValue(subscriptionCount + 1));
 
 			// Update the subscribers of the feed
-			Optional<Value> subscriberCountVal = wait(tr.get(keyForFeedSubscriberCount(feed)));
+			Optional<Value> subscriberCountVal = co_await tr.get(keyForFeedSubscriberCount(feed));
 			uint64_t subscriberCount = valueToUInt64(subscriberCountVal.get()); // throws if count not present
 			tr.set(keyForFeedSubscriber(feed, inbox), StringRef());
 			tr.set(keyForFeedSubscriberCount(inbox), uInt64ToValue(subscriberCount + 1));
@@ -190,13 +194,14 @@ ACTOR Future<bool> _createSubscription(Database cx, uint64_t feed, uint64_t inbo
 			// Add inbox as watcher of feed.
 			tr.set(keyForFeedWatcher(feed, inbox), StringRef());
 
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return true;
+	co_return true;
 }
 
 Future<bool> PubSub::createSubscription(uint64_t feed, uint64_t inbox) {
@@ -206,22 +211,21 @@ Future<bool> PubSub::createSubscription(uint64_t feed, uint64_t inbox) {
 // Since we are not relying on "read-your-own-writes", we need to keep track of
 //  the highest-numbered inbox that we've cleared from the watchers list and
 //  make sure that further requests start after this inbox.
-ACTOR Future<Void> updateFeedWatchers(Transaction* tr, uint64_t feed) {
-	state StringRef watcherPrefix = keyForFeedWatcherPrefix(feed);
-	state uint64_t highestInbox;
-	state bool first = true;
-	loop {
+Future<Void> updateFeedWatchers(Transaction* tr, uint64_t feed) {
+	StringRef watcherPrefix = keyForFeedWatcherPrefix(feed);
+	uint64_t highestInbox{ 0 };
+	bool first = true;
+	while (true) {
 		// Grab watching inboxes in swaths of 100
-		state RangeResult watchingInboxes =
-		    wait((*tr).getRange(firstGreaterOrEqual(keyForFeedWatcher(feed, first ? 0 : highestInbox + 1)),
-		                        firstGreaterOrEqual(keyForFeedWatcher(feed, UINT64_MAX)),
-		                        100)); // REVIEW: does 100 make sense?
-		if (!watchingInboxes.size())
+		RangeResult watchingInboxes =
+		    co_await (*tr).getRange(firstGreaterOrEqual(keyForFeedWatcher(feed, first ? 0 : highestInbox + 1)),
+		                            firstGreaterOrEqual(keyForFeedWatcher(feed, UINT64_MAX)),
+		                            100); // REVIEW: does 100 make sense?
+		if (watchingInboxes.empty())
 			// If there are no watchers, return.
-			return Void();
+			co_return;
 		first = false;
-		state int idx = 0;
-		for (; idx < watchingInboxes.size(); idx++) {
+		for (int idx = 0; idx < watchingInboxes.size(); idx++) {
 			KeyRef key = watchingInboxes[idx].key;
 			StringRef inboxStr = key.removePrefix(watcherPrefix);
 			uint64_t inbox = valueToUInt64(inboxStr);
@@ -233,7 +237,7 @@ ACTOR Future<Void> updateFeedWatchers(Transaction* tr, uint64_t feed) {
 		}
 		if (watchingInboxes.size() < 100)
 			// If there were fewer watchers returned that we asked for, we're done.
-			return Void();
+			co_return;
 	}
 }
 
@@ -247,22 +251,23 @@ ACTOR Future<Void> updateFeedWatchers(Transaction* tr, uint64_t feed) {
  *        Move feed in and remove watchers in one transaction, possibly
  * SOMEDAY: create a global list of the most-subscribed-to feeds that all inbox reads check
  */
-ACTOR Future<uint64_t> _postMessage(Database cx, uint64_t feed, Standalone<StringRef> data) {
-	state Transaction tr(cx);
-	state uint64_t messageId = UINT64_MAX - (uint64_t)now();
+Future<uint64_t> _postMessage(Database cx, uint64_t feed, Standalone<StringRef> data) {
+	Transaction tr(cx);
+	uint64_t messageId = UINT64_MAX - (uint64_t)now();
 	TraceEvent("PubSubPost").detail("Feed", feed).detail("Message", messageId);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			Optional<Value> feedValue = wait(tr.get(keyForFeed(feed)));
+			Optional<Value> feedValue = co_await tr.get(keyForFeed(feed));
 			if (!feedValue.present()) {
 				// No such feed!!
-				return uint64_t(0);
+				co_return uint64_t(0);
 			}
 
 			// Get globally latest message, set our ID to that less one
-			state RangeResult latestMessage = wait(
-			    tr.getRange(firstGreaterOrEqual(keyForMessage(0)), firstGreaterOrEqual(keyForMessage(UINT64_MAX)), 1));
-			if (!latestMessage.size()) {
+			RangeResult latestMessage = co_await tr.getRange(
+			    firstGreaterOrEqual(keyForMessage(0)), firstGreaterOrEqual(keyForMessage(UINT64_MAX)), 1);
+			if (latestMessage.empty()) {
 				messageId = UINT64_MAX - 1;
 			} else {
 				StringRef messageStr = latestMessage[0].key.removePrefix(messagePrefix);
@@ -272,14 +277,16 @@ ACTOR Future<uint64_t> _postMessage(Database cx, uint64_t feed, Standalone<Strin
 			tr.set(keyForMessage(messageId), StringRef());
 			tr.set(keyForDisptchEntry(messageId), StringRef());
 
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 	tr = Transaction(cx);
-	loop {
+	while (true) {
+		Error err;
 		try {
 			// Record this ID as the "latest message" for a feed
 			tr.set(keyForFeedLatestMessage(feed), uInt64ToValue(messageId));
@@ -288,57 +295,57 @@ ACTOR Future<uint64_t> _postMessage(Database cx, uint64_t feed, Standalone<Strin
 			tr.set(keyForFeedMessage(feed, messageId), StringRef());
 
 			// Update the count of message that this feed has published
-			Optional<Value> cntValue = wait(tr.get(keyForFeedMessageCount(feed)));
+			Optional<Value> cntValue = co_await tr.get(keyForFeedMessageCount(feed));
 			uint64_t messageCount(valueToUInt64(cntValue.get()) + 1);
 			tr.set(keyForFeedMessageCount(feed), uInt64ToValue(messageCount));
 
 			// Go through the list of watching inboxes
-			wait(updateFeedWatchers(&tr, feed));
+			co_await updateFeedWatchers(&tr, feed);
 
 			// Post the real message data; clear the "dispatching" entry
 			tr.set(keyForMessage(messageId), data);
 			tr.clear(keyForDisptchEntry(messageId));
 
-			wait(tr.commit());
+			co_await tr.commit();
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
-	return messageId;
+	co_return messageId;
 }
 
 Future<uint64_t> PubSub::postMessage(uint64_t feed, Standalone<StringRef> data) {
 	return _postMessage(cx, feed, data);
 }
 
-ACTOR Future<int> singlePassInboxCacheUpdate(Database cx, uint64_t inbox, int swath) {
-	state Transaction tr(cx);
-	loop {
+Future<int> singlePassInboxCacheUpdate(Database cx, uint64_t inbox, int swath) {
+	Transaction tr(cx);
+	while (true) {
+		Error err;
 		try {
 			// For each stale feed, update cache with latest message id
-			state RangeResult staleFeeds =
-			    wait(tr.getRange(firstGreaterOrEqual(keyForInboxStaleFeed(inbox, 0)),
-			                     firstGreaterOrEqual(keyForInboxStaleFeed(inbox, UINT64_MAX)),
-			                     swath)); // REVIEW: does 100 make sense?
+			RangeResult staleFeeds = co_await tr.getRange(firstGreaterOrEqual(keyForInboxStaleFeed(inbox, 0)),
+			                                              firstGreaterOrEqual(keyForInboxStaleFeed(inbox, UINT64_MAX)),
+			                                              swath); // REVIEW: does 100 make sense?
 			// printf("  --> stale feeds list size: %d\n", staleFeeds.size());
-			if (!staleFeeds.size())
+			if (staleFeeds.empty())
 				// If there are no stale feeds, return.
-				return 0;
-			state StringRef stalePrefix = keyForInboxStalePrefix(inbox);
-			state int idx = 0;
-			for (; idx < staleFeeds.size(); idx++) {
+				co_return 0;
+			StringRef stalePrefix = keyForInboxStalePrefix(inbox);
+			for (int idx = 0; idx < staleFeeds.size(); idx++) {
 				StringRef feedStr = staleFeeds[idx].key.removePrefix(stalePrefix);
 				// printf("  --> clearing stale entry: %s\n", feedStr.toString().c_str());
-				state uint64_t feed = valueToUInt64(feedStr);
+				uint64_t feed = valueToUInt64(feedStr);
 
 				// SOMEDAY: change this to be a range query for the highest #'ed message
-				Optional<Value> v = wait(tr.get(keyForFeedLatestMessage(feed)));
-				state Value latestMessageValue = v.get();
+				Optional<Value> v = co_await tr.get(keyForFeedLatestMessage(feed));
+				Value latestMessageValue = v.get();
 				// printf("  --> latest message from feed: %s\n", latestMessageValue.toString().c_str());
 
 				// find the messageID which is currently cached for this feed
-				Optional<Value> lastCachedValue = wait(tr.get(keyForInboxCacheByFeed(inbox, feed)));
+				Optional<Value> lastCachedValue = co_await tr.get(keyForInboxCacheByFeed(inbox, feed));
 				if (lastCachedValue.present()) {
 					uint64_t lastCachedId = valueToUInt64(lastCachedValue.get());
 					// clear out the cache entry in the "by-ID" list for this feed
@@ -356,121 +363,121 @@ ACTOR Future<int> singlePassInboxCacheUpdate(Database cx, uint64_t inbox, int sw
 				tr.set(keyForFeedWatcher(feed, inbox), StringRef());
 				// printf("  --> adding watch to feed: %s\n", keyForFeedWatcher(feed, inbox).toString().c_str());
 			}
-			wait(tr.commit());
-			return staleFeeds.size();
+			co_await tr.commit();
+			co_return staleFeeds.size();
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
 // SOMEDAY: evaluate if this could lead to painful loop if there's one frequent feed.
-ACTOR Future<Void> updateInboxCache(Database cx, uint64_t inbox) {
-	state int swath = 100;
-	state int updatedEntries = swath;
+Future<Void> updateInboxCache(Database cx, uint64_t inbox) {
+	int swath = 100;
+	int updatedEntries = swath;
 	while (updatedEntries >= swath) {
-		int retVal = wait(singlePassInboxCacheUpdate(cx, inbox, swath));
-		updatedEntries = retVal;
+		updatedEntries = co_await singlePassInboxCacheUpdate(cx, inbox, swath);
 	}
-	return Void();
 }
 
-ACTOR Future<MessageId> getFeedLatestAtOrAfter(Transaction* tr, Feed feed, MessageId position) {
-	state RangeResult lastMessageRange = wait((*tr).getRange(firstGreaterOrEqual(keyForFeedMessage(feed, position)),
-	                                                         firstGreaterOrEqual(keyForFeedMessage(feed, UINT64_MAX)),
-	                                                         1));
-	if (!lastMessageRange.size())
-		return uint64_t(0);
+Future<MessageId> getFeedLatestAtOrAfter(Transaction* tr, Feed feed, MessageId position) {
+	RangeResult lastMessageRange = co_await (*tr).getRange(firstGreaterOrEqual(keyForFeedMessage(feed, position)),
+	                                                       firstGreaterOrEqual(keyForFeedMessage(feed, UINT64_MAX)),
+	                                                       1);
+	if (lastMessageRange.empty())
+		co_return uint64_t(0);
 	KeyValueRef m = lastMessageRange[0];
 	StringRef prefix = keyForFeedMessagePrefix(feed);
 	StringRef mIdStr = m.key.removePrefix(prefix);
-	return valueToUInt64(mIdStr);
+	co_return valueToUInt64(mIdStr);
 }
 
-ACTOR Future<Message> getMessage(Transaction* tr, Feed feed, MessageId id) {
-	state Message m;
+Future<Message> getMessage(Transaction* tr, Feed feed, MessageId id) {
+	Message m;
 	m.originatorFeed = feed;
 	m.messageId = id;
-	Optional<Value> data = wait(tr->get(keyForMessage(id)));
+	Optional<Value> data = co_await tr->get(keyForMessage(id));
 	m.data = data.get();
-	return m;
+	co_return m;
 }
 
-ACTOR Future<std::vector<Message>> _listInboxMessages(Database cx, uint64_t inbox, int count, uint64_t cursor);
+Future<std::vector<Message>> _listInboxMessages(Database cx, uint64_t inbox, int count, uint64_t cursor);
 
 // inboxes with MANY fast feeds may be punished by the following checks
 // SOMEDAY: add a check on global lists (or on dispatching list)
-ACTOR Future<std::vector<Message>> _listInboxMessages(Database cx, uint64_t inbox, int count, uint64_t cursor) {
+Future<std::vector<Message>> _listInboxMessages(Database cx, uint64_t inbox, int count, uint64_t cursor) {
 	TraceEvent("PubSubListInbox").detail("Inbox", inbox).detail("Count", count).detail("Cursor", cursor);
-	wait(updateInboxCache(cx, inbox));
-	state StringRef perIdPrefix = keyForInboxCacheByIDPrefix(inbox);
-	loop {
-		state Transaction tr(cx);
-		state std::vector<Message> messages;
-		state std::map<MessageId, Feed> feedLatest;
+	co_await updateInboxCache(cx, inbox);
+	StringRef perIdPrefix = keyForInboxCacheByIDPrefix(inbox);
+	while (true) {
+		Transaction tr(cx);
+		std::vector<Message> messages;
+		std::map<MessageId, Feed> feedLatest;
+		Error err;
 		try {
 			// Fetch all cached entries for all the feeds to which we are subscribed
-			Optional<Value> cntValue = wait(tr.get(keyForInboxSubscriptionCount(inbox)));
+			Optional<Value> cntValue = co_await tr.get(keyForInboxSubscriptionCount(inbox));
 			uint64_t subscriptions = valueToUInt64(cntValue.get());
-			state RangeResult feeds = wait(tr.getRange(firstGreaterOrEqual(keyForInboxCacheByID(inbox, 0)),
-			                                           firstGreaterOrEqual(keyForInboxCacheByID(inbox, UINT64_MAX)),
-			                                           subscriptions));
-			if (!feeds.size())
-				return messages;
+			RangeResult feeds = co_await tr.getRange(firstGreaterOrEqual(keyForInboxCacheByID(inbox, 0)),
+			                                         firstGreaterOrEqual(keyForInboxCacheByID(inbox, UINT64_MAX)),
+			                                         subscriptions);
+			if (feeds.empty())
+				co_return messages;
 
 			// read cache into map, replace entries newer than cursor with the newest older than cursor
-			state int idx = 0;
-			for (; idx < feeds.size(); idx++) {
+			for (int idx = 0; idx < feeds.size(); idx++) {
 				StringRef mIdStr = feeds[idx].key.removePrefix(perIdPrefix);
 				MessageId messageId = valueToUInt64(mIdStr);
-				state Feed feed = valueToUInt64(feeds[idx].value);
+				Feed feed = valueToUInt64(feeds[idx].value);
 				// printf(" -> cached message %016llx from feed %016llx\n", messageId, feed);
 				if (messageId >= cursor) {
 					// printf(" -> entering message %016llx from feed %016llx\n", messageId, feed);
 					feedLatest.emplace(messageId, feed);
 				} else {
 					// replace this with the first message older than the cursor
-					MessageId mId = wait(getFeedLatestAtOrAfter(&tr, feed, cursor));
+					MessageId mId = co_await getFeedLatestAtOrAfter(&tr, feed, cursor);
 					if (mId) {
 						feedLatest.emplace(mId, feed);
 					}
 				}
 			}
 			// There were some cached feeds, but none with messages older than "cursor"
-			if (!feedLatest.size())
-				return messages;
+			if (feedLatest.empty())
+				co_return messages;
 
 			// Check the list of dispatching messages to make sure there are no older ones than ours
-			state MessageId earliestMessage = feedLatest.begin()->first;
-			RangeResult dispatching = wait(tr.getRange(firstGreaterOrEqual(keyForDisptchEntry(earliestMessage)),
-			                                           firstGreaterOrEqual(keyForDisptchEntry(UINT64_MAX)),
-			                                           1));
+			MessageId earliestMessage = feedLatest.begin()->first;
+			RangeResult dispatching = co_await tr.getRange(firstGreaterOrEqual(keyForDisptchEntry(earliestMessage)),
+			                                               firstGreaterOrEqual(keyForDisptchEntry(UINT64_MAX)),
+			                                               1);
 			// If there are messages "older" than ours, try this again
 			//  (with a new transaction and a flush of the "stale" feeds
-			if (dispatching.size()) {
-				std::vector<Message> r = wait(_listInboxMessages(cx, inbox, count, earliestMessage));
-				return r;
+			if (!dispatching.empty()) {
+				std::vector<Message> r = co_await _listInboxMessages(cx, inbox, count, earliestMessage);
+				co_return r;
 			}
 
-			while (messages.size() < count && feedLatest.size() > 0) {
-				std::map<MessageId, Feed>::iterator latest = feedLatest.begin();
-				state MessageId id = latest->first;
-				state Feed f = latest->second;
+			while (messages.size() < count && !feedLatest.empty()) {
+				auto latest = feedLatest.begin();
+				MessageId id = latest->first;
+				Feed f = latest->second;
 				feedLatest.erase(latest);
 
-				Message m = wait(getMessage(&tr, f, id));
+				Message m = co_await getMessage(&tr, f, id);
 				messages.push_back(m);
 
-				MessageId nextMessage = wait(getFeedLatestAtOrAfter(&tr, f, id + 1));
+				MessageId nextMessage = co_await getFeedLatestAtOrAfter(&tr, f, id + 1);
 				if (nextMessage) {
 					feedLatest.emplace(nextMessage, f);
 				}
 			}
 
-			return messages;
+			co_return messages;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
@@ -478,29 +485,30 @@ Future<std::vector<Message>> PubSub::listInboxMessages(uint64_t inbox, int count
 	return _listInboxMessages(cx, inbox, count, cursor);
 }
 
-ACTOR Future<std::vector<Message>> _listFeedMessages(Database cx, Feed feed, int count, uint64_t cursor) {
-	state std::vector<Message> messages;
-	state Transaction tr(cx);
+Future<std::vector<Message>> _listFeedMessages(Database cx, Feed feed, int count, uint64_t cursor) {
+	std::vector<Message> messages;
+	Transaction tr(cx);
 	TraceEvent("PubSubListFeed").detail("Feed", feed).detail("Count", count).detail("Cursor", cursor);
-	loop {
+	while (true) {
+		Error err;
 		try {
-			state RangeResult messageIds = wait(tr.getRange(firstGreaterOrEqual(keyForFeedMessage(feed, cursor)),
-			                                                firstGreaterOrEqual(keyForFeedMessage(feed, UINT64_MAX)),
-			                                                count));
-			if (!messageIds.size())
-				return messages;
-
-			state int idx = 0;
-			for (; idx < messageIds.size(); idx++) {
+			RangeResult messageIds = co_await tr.getRange(firstGreaterOrEqual(keyForFeedMessage(feed, cursor)),
+			                                              firstGreaterOrEqual(keyForFeedMessage(feed, UINT64_MAX)),
+			                                              count);
+			if (messageIds.empty())
+				co_return messages;
+
+			for (int idx = 0; idx < messageIds.size(); idx++) {
 				StringRef mIdStr = messageIds[idx].key.removePrefix(keyForFeedMessagePrefix(feed));
 				MessageId messageId = valueToUInt64(mIdStr);
-				Message m = wait(getMessage(&tr, feed, messageId));
+				Message m = co_await getMessage(&tr, feed, messageId);
 				messages.push_back(m);
 			}
-			return messages;
+			co_return messages;
 		} catch (Error& e) {
-			wait(tr.onError(e));
+			err = e;
 		}
+		co_await tr.onError(err);
 	}
 }
 
diff --git a/fdbserver/include/fdbserver/pubsub.h b/fdbserver/workloads/pubsub.h
similarity index 98%
rename from fdbserver/include/fdbserver/pubsub.h
rename to fdbserver/workloads/pubsub.h
index cd0f974c6eb..ec9f7f91344 100644
--- a/fdbserver/include/fdbserver/pubsub.h
+++ b/fdbserver/workloads/pubsub.h
@@ -75,7 +75,7 @@ class Message {
 
 class PubSub {
 public:
-	PubSub(Database cx);
+	explicit PubSub(Database cx);
 
 	Future<Feed> createFeed(Standalone<StringRef> metadata);
 
diff --git a/flow/ActorCollection.actor.cpp b/flow/ActorCollection.actor.cpp
index 4ad4bd3b460..5500cfe1e81 100644
--- a/flow/ActorCollection.actor.cpp
+++ b/flow/ActorCollection.actor.cpp
@@ -29,12 +29,12 @@ struct Runner : public boost::intrusive::list_base_hook<>, FastAllocated<Runner>
 };
 
 // An intrusive list of Runners, which are FastAllocated.  Each runner holds a handler future
-typedef boost::intrusive::list<Runner, boost::intrusive::constant_time_size<false>> RunnerList;
+using RunnerList = boost::intrusive::list<Runner, boost::intrusive::constant_time_size<false>>;
 
 // The runners list in the ActorCollection must be destroyed when the actor is destructed rather
 // than before returning or throwing
 struct RunnerListDestroyer : NonCopyable {
-	RunnerListDestroyer(RunnerList* list) : list(list) {}
+	explicit RunnerListDestroyer(RunnerList* list) : list(list) {}
 
 	~RunnerListDestroyer() {
 		list->clear_and_dispose([](Runner* r) { delete r; });
@@ -161,8 +161,8 @@ TEST_CASE("/flow/actorCollection/testCancel") {
 	return Void();
 }
 
-ACTOR Future<Void> failedActor() {
-	throw operation_failed();
+Future<Void> failedActor() {
+	return operation_failed();
 }
 
 // test contract that even if the actor collection has stopped and new actors are added to the promise stream, they are
diff --git a/flow/Arena.cpp b/flow/Arena.cpp
index 288f8b693b3..3afabf8deff 100644
--- a/flow/Arena.cpp
+++ b/flow/Arena.cpp
@@ -107,11 +107,18 @@ void makeUndefined(void*, size_t) {}
 #endif
 } // namespace
 
-Arena::Arena() : impl(nullptr) {}
+static SimpleCounter<int64_t>* arenasCreated(void) {
+	static SimpleCounter<int64_t>* p = SimpleCounter<int64_t>::makeCounter("/flow/arena/arenasCreated");
+	return p;
+}
+
+Arena::Arena() {
+	arenasCreated()->increment(1);
+}
+
 Arena::Arena(size_t reservedSize) : impl(0) {
 	UNSTOPPABLE_ASSERT(reservedSize < std::numeric_limits<int>::max());
-	static SimpleCounter<int64_t>* created = SimpleCounter<int64_t>::makeCounter("/flow/arena/arenasCreated");
-	created->increment(1);
+	arenasCreated()->increment(1);
 	if (reservedSize) {
 		allowAccess(impl.getPtr());
 		ArenaBlock::create((int)reservedSize, impl);
@@ -124,6 +131,49 @@ Arena::Arena(const Arena& r) = default;
 Arena::Arena(Arena&& r) noexcept = default;
 Arena& Arena::operator=(const Arena& r) = default;
 Arena& Arena::operator=(Arena&& r) noexcept = default;
+
+std::string StringRef::toHexString(int limit) const {
+	if (limit < 0)
+		limit = length;
+	std::string rv;
+	if (length > limit) {
+		// If limit is high enough split it so that 2/3 of limit is used to show prefix bytes and the rest is used
+		// for suffix bytes
+		if (limit >= 9) {
+			int suffix = limit / 3;
+			return substr(0, limit - suffix).toHexString() + "..." + substr(length - suffix, suffix).toHexString() +
+			       format(" [%d bytes]", length);
+		}
+		rv = substr(0, limit).toHexString() + format("...[%d]", length);
+	} else {
+		rv.reserve(length * 7);
+		for (int i = 0; i < length; i++) {
+			uint8_t b = (*this)[i];
+			if (isalnum(b))
+				rv.append(format("%02x (%c) ", b, b));
+			else
+				rv.append(format("%02x ", b));
+		}
+		if (!rv.empty())
+			rv.resize(rv.size() - 1);
+	}
+	bytesCopied()->increment(rv.length());
+	return rv;
+}
+
+std::string StringRef::toFullHexStringPlain() const {
+	std::string s;
+	s.reserve(length * 7);
+	for (int i = 0; i < length; i++) {
+		uint8_t b = (*this)[i];
+		s.append(format("%02x ", b));
+	}
+	if (!s.empty())
+		s.resize(s.size() - 1);
+	bytesCopied()->increment(s.length());
+	return s;
+}
+
 void Arena::dependsOn(const Arena& p) {
 	// x.dependsOn(y) is a no-op if they refer to the same ArenaBlocks.
 	// They will already have the same lifetime.
@@ -228,7 +278,7 @@ size_t ArenaBlock::totalSize(std::unordered_set<ArenaBlock*>& visited) const {
 		static SimpleCounter<int64_t>* count =
 		    SimpleCounter<int64_t>::makeCounter("/flow/arena/totalSizeBlocksExamined");
 		count->increment(1);
-		ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + o);
+		auto* r = (ArenaBlockRef*)((char*)getData() + o);
 		makeDefined(r, sizeof(ArenaBlockRef));
 		if (r->aligned4kBufferSize != 0) {
 			totalSizeEstimate += r->aligned4kBufferSize;
@@ -271,7 +321,7 @@ void ArenaBlock::getUniqueBlocks(std::set<ArenaBlock*>& a) {
 
 	int o = nextBlockOffset;
 	while (o) {
-		ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + o);
+		auto* r = (ArenaBlockRef*)((char*)getData() + o);
 		makeDefined(r, sizeof(ArenaBlockRef));
 
 		// If next is valid recursively count its blocks
@@ -298,7 +348,7 @@ int ArenaBlock::addUsed(int bytes) {
 }
 
 void ArenaBlock::makeReference(ArenaBlock* next) {
-	ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + bigUsed);
+	auto* r = (ArenaBlockRef*)((char*)getData() + bigUsed);
 	makeDefined(r, sizeof(ArenaBlockRef));
 	r->aligned4kBufferSize = 0;
 	r->next = next;
@@ -310,7 +360,7 @@ void ArenaBlock::makeReference(ArenaBlock* next) {
 }
 
 void* ArenaBlock::make4kAlignedBuffer(uint32_t size) {
-	ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + bigUsed);
+	auto* r = (ArenaBlockRef*)((char*)getData() + bigUsed);
 	makeDefined(r, sizeof(ArenaBlockRef));
 	r->aligned4kBufferSize = size;
 	r->aligned4kBuffer = allocateFast4kAligned(size);
@@ -482,7 +532,7 @@ void ArenaBlock::destroy() {
 	Arena stackArena;
 	VectorRef<ArenaBlock*> stack(&tinyStack, 1);
 
-	while (stack.size()) {
+	while (!stack.empty()) {
 		ArenaBlock* b = stack.end()[-1];
 		stack.pop_back();
 		allowAccess(b);
@@ -490,7 +540,7 @@ void ArenaBlock::destroy() {
 		if (!b->isTiny()) {
 			int o = b->nextBlockOffset;
 			while (o) {
-				ArenaBlockRef* br = (ArenaBlockRef*)((char*)b->getData() + o);
+				auto* br = (ArenaBlockRef*)((char*)b->getData() + o);
 				makeDefined(br, sizeof(ArenaBlockRef));
 
 				// If aligned4kBuffer is valid, free it
@@ -852,12 +902,12 @@ TEST_CASE("flow/StringRef/eat") {
 	str = "testcase"_sr;
 	first = str.eat("/"_sr);
 	ASSERT(first == "testcase"_sr);
-	ASSERT(str == ""_sr);
+	ASSERT(str.empty());
 
 	str = "testcase/"_sr;
 	first = str.eat("/"_sr);
 	ASSERT(first == "testcase"_sr);
-	ASSERT(str == ""_sr);
+	ASSERT(str.empty());
 
 	str = "test/case/extra"_sr;
 	first = str.eat("/"_sr);
@@ -875,7 +925,26 @@ TEST_CASE("flow/StringRef/eat") {
 	first = str.eat("/", &hasSep);
 	ASSERT(!hasSep);
 	ASSERT(first == "testcase"_sr);
-	ASSERT(str == ""_sr);
+	ASSERT(str.empty());
+
+	return Void();
+}
+
+// Verifies StringRef(const char*) borrows the pointer directly (like std::string_view)
+// rather than routing through a temporary std::string, which would leave a dangling pointer.
+TEST_CASE("/flow/StringRef/ConstCharConstructor") {
+	StringRef s("Hello world!");
+	ASSERT(s == "Hello world!"_sr);
+	ASSERT(s.size() == 12);
+
+	const char* cstr = "another string";
+	StringRef s2(cstr);
+	ASSERT(s2 == "another string"_sr);
+	ASSERT(s2.begin() == reinterpret_cast<const uint8_t*>(cstr));
+
+	StringRef empty("");
+	ASSERT(empty.empty());
+	ASSERT(empty.size() == 0);
 
 	return Void();
 }
diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt
index 0163fc27bb7..e677ed52331 100644
--- a/flow/CMakeLists.txt
+++ b/flow/CMakeLists.txt
@@ -16,6 +16,7 @@ list(REMOVE_ITEM FLOW_SRCS LinkTest.cpp)
 list(REMOVE_ITEM FLOW_SRCS TLSTest.cpp)
 list(REMOVE_ITEM FLOW_SRCS MkCertCli.cpp)
 list(REMOVE_ITEM FLOW_SRCS acac.cpp)
+list(REMOVE_ITEM FLOW_SRCS FlowTest.cpp)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
   list(APPEND FLOW_SRCS aarch64/memcmp.S aarch64/memcpy.S)
@@ -116,25 +117,33 @@ endif()
 # executable so the linker will throw errors if it can't find the declaration of
 # a symbol.
 add_flow_target(LINK_TEST NAME flowlinktest SRCS LinkTest.cpp)
-target_link_libraries(flowlinktest PRIVATE flow stacktrace)
+target_link_libraries(flowlinktest PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,flow>" stacktrace)
+
+add_flow_target(EXECUTABLE NAME flow_test SRCS FlowTest.cpp)
+set_target_properties(flow_test PROPERTIES EXCLUDE_FROM_ALL TRUE)
+target_link_libraries(flow_test PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,flow>")
 
 set(IS_ARM_MAC NO)
 if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
   set(IS_ARM_MAC YES)
 endif()
 
-foreach(ft flow flow_sampling flowlinktest)
+foreach(ft flow flow_sampling flowlinktest flow_test)
   target_include_directories(
     ${ft}
     PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include"
            "${CMAKE_CURRENT_BINARY_DIR}/include"
-           "${CMAKE_SOURCE_DIR}/contrib/libb64/include")
+           "${CMAKE_SOURCE_DIR}/contrib/libb64/include"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
 
   if(FLOW_USE_ZSTD)
     target_include_directories(${ft} PRIVATE SYSTEM ${ZSTD_LIB_INCLUDE_DIR})
   endif()
   if(USE_JEMALLOC)
     target_include_directories(${ft} PRIVATE ${jemalloc_INCLUDE_DIRS})
+    if(ft STREQUAL "flowlinktest" OR ft STREQUAL "flow_test")
+      target_link_libraries(${ft} PRIVATE jemalloc::jemalloc)
+    endif()
   endif()
 
   target_link_libraries(${ft} PRIVATE stacktrace)
@@ -200,6 +209,11 @@ else()
 endif()
 target_link_libraries(mkcert PUBLIC flow)
 
+if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
+                                     # x-compiling.
+  add_subdirectory(bench EXCLUDE_FROM_ALL)
+endif()
+
 set(FLOW_BINARY_DIR "${CMAKE_BINARY_DIR}/flow")
 if(WITH_SWIFT)
   include(GenerateModulemap)
diff --git a/flow/CodeProbe.cpp b/flow/CodeProbe.cpp
index 1ab1ee02c0d..fccdb3557aa 100644
--- a/flow/CodeProbe.cpp
+++ b/flow/CodeProbe.cpp
@@ -115,18 +115,18 @@ struct CodeProbes {
 
 	void verify() const {
 		std::map<std::pair<std::string_view, std::string_view>, ICodeProbe const*> comments;
-		for (auto probe : codeProbes) {
-			auto file = probe.first.file;
-			auto comment = probe.second->comment();
+		for (const auto& [probeLocation, probe] : codeProbes) {
+			auto file = probeLocation.file;
+			auto comment = probe->comment();
 			auto commentEntry = std::make_pair(file, std::string_view(comment));
-			ASSERT(file == probe.second->filename());
+			ASSERT(file == probe->filename());
 			auto iter = comments.find(commentEntry);
-			if (iter != comments.end() && probe.second->line() != iter->second->line()) {
+			if (iter != comments.end() && probe->line() != iter->second->line()) {
 				fmt::print("ERROR ({}:{}): {} isn't unique in file {}. Previously seen here: {}:{}\n",
-				           probe.first.file,
-				           probe.first.line,
+				           probeLocation.file,
+				           probeLocation.line,
 				           iter->first.second,
-				           probe.second->filename(),
+				           probe->filename(),
 				           iter->second->filename(),
 				           iter->second->line());
 				// ICodeProbe const& fst = *iter->second;
@@ -141,7 +141,7 @@ struct CodeProbes {
 				// fmt::print("\t2nd CompUnit: {}\n", snd.compilationUnit());
 				// fmt::print("\n");
 			} else {
-				comments.emplace(commentEntry, probe.second);
+				comments.emplace(commentEntry, probe);
 			}
 		}
 	}
@@ -156,13 +156,13 @@ struct CodeProbes {
 		} else {
 			std::vector<std::string_view> files;
 			fmt::print("\t<CoverageCases>\n");
-			for (auto probe : codeProbes) {
-				files.push_back(probe.first.file);
+			for (const auto& [probeLocation, probe] : codeProbes) {
+				files.push_back(probeLocation.file);
 				fmt::print("\t\t<Case File=\"{}\" Line=\"{}\" Comment=\"{}\" Condition=\"{}\"/>\n",
-				           probe.first.file,
-				           probe.first.line,
-				           probe.second->comment(),
-				           probe.second->condition());
+				           probeLocation.file,
+				           probeLocation.line,
+				           probe->comment(),
+				           probe->condition());
 			}
 			fmt::print("\t</CoverageCases>\n");
 			fmt::print("\t<Inputs>\n");
@@ -188,28 +188,27 @@ struct CodeProbes {
 		} while (false);
 		auto contexts = fromStrings(context);
 		const ICodeProbe* prev = nullptr;
-		for (auto probe : codeProbes) {
-			auto p = probe.second;
+		for (const auto& [_probeLocation, probe] : codeProbes) {
 			if (!contexts.empty()) {
 				bool print = false;
 				for (auto c : contexts) {
-					print = print || p->expectInContext(c);
+					print = print || probe->expectInContext(c);
 				}
 				if (!print) {
 					continue;
 				}
 			}
-			if (prev == nullptr || *prev != *probe.second) {
+			if (prev == nullptr || *prev != *probe) {
 				fmt::print(
 				    "{{ \"File\": \"{}\", \"Line\": {}, \"Comment\": \"{}\", \"Condition\": \"{}\", \"Function\": "
 				    "\"{}\" }}\n",
-				    probe.first.file,
-				    p->line(),
-				    p->comment(),
-				    p->condition(),
-				    p->function());
+				    probe->filename(),
+				    probe->line(),
+				    probe->comment(),
+				    probe->condition(),
+				    probe->function());
 			}
-			prev = probe.second;
+			prev = probe;
 		}
 	}
 };
@@ -223,10 +222,10 @@ size_t hash_value(CodeProbes::Location const& location) {
 
 void CodeProbes::traceMissedProbes(Optional<ExecutionContext> context) const {
 	boost::unordered_map<Location, bool> locations;
-	for (auto probe : codeProbes) {
+	for (const auto& [probeLocation, probe] : codeProbes) {
 		decltype(locations.begin()) iter;
-		std::tie(iter, std::ignore) = locations.emplace(probe.first, false);
-		iter->second = iter->second || probe.second->wasHit();
+		std::tie(iter, std::ignore) = locations.emplace(probeLocation, false);
+		iter->second = iter->second || probe->wasHit();
 	}
 	for (const auto& [loc, probe] : codeProbes) {
 		auto iter = locations.find(loc);
@@ -255,7 +254,7 @@ void traceMissedProbes(Optional<ExecutionContext> context) {
 	CodeProbes::instance().traceMissedProbes(context);
 }
 
-ICodeProbe::~ICodeProbe() {}
+ICodeProbe::~ICodeProbe() = default;
 
 bool ICodeProbe::operator==(const ICodeProbe& other) const {
 	return filename() == other.filename() && line() == other.line();
diff --git a/flow/CompressedInt.actor.cpp b/flow/CompressedInt.cpp
similarity index 93%
rename from flow/CompressedInt.actor.cpp
rename to flow/CompressedInt.cpp
index 1a27289ade2..cc9bce2d591 100644
--- a/flow/CompressedInt.actor.cpp
+++ b/flow/CompressedInt.cpp
@@ -1,5 +1,5 @@
 /*
- * CompressedInt.actor.cpp
+ * CompressedInt.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,7 +22,7 @@
 #include "flow/CompressedInt.h"
 
 void printBitsLittle(size_t const size, void const* const ptr) {
-	unsigned char* b = (unsigned char*)ptr;
+	auto* b = static_cast<unsigned char const*>(ptr);
 	unsigned char byte;
 	int i, j;
 
@@ -37,7 +37,7 @@ void printBitsLittle(size_t const size, void const* const ptr) {
 }
 
 void printBitsBig(size_t const size, void const* const ptr) {
-	unsigned char* b = (unsigned char*)ptr;
+	auto* b = static_cast<unsigned char const*>(ptr);
 	unsigned char byte;
 	int i, j;
 
@@ -57,7 +57,7 @@ void testCompressedInt(IntType n, StringRef rep = StringRef()) {
 	CompressedInt<IntType> cn(n);
 
 	w << cn;
-	if (rep.size() != 0 && w.toValue() != rep) {
+	if (!rep.empty() && w.toValue() != rep) {
 		printf("WRONG ENCODING:\n");
 		printf("  test value (BigE):  ");
 		printBitsLittle(sizeof(IntType), &n);
@@ -85,6 +85,8 @@ void testCompressedInt(IntType n, StringRef rep = StringRef()) {
 	}
 }
 
+void forceLinkCompressedIntTests() {}
+
 TEST_CASE("/flow/compressed_ints") {
 	testCompressedInt<int>(-2, "\x7e"_sr);
 	testCompressedInt<int>(-1, "\x7f"_sr);
diff --git a/flow/CompressionUtils.cpp b/flow/CompressionUtils.cpp
index b82964b4a8f..53917714b7c 100644
--- a/flow/CompressionUtils.cpp
+++ b/flow/CompressionUtils.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "flow/CompressionUtils.h"
+#include "CompressionUtils.h"
 
 #include "flow/Arena.h"
 #include "flow/Error.h"
diff --git a/flow/include/flow/CompressionUtils.h b/flow/CompressionUtils.h
similarity index 100%
rename from flow/include/flow/CompressionUtils.h
rename to flow/CompressionUtils.h
diff --git a/fdbrpc/CoroTests.cpp b/flow/CoroTests.cpp
similarity index 67%
rename from fdbrpc/CoroTests.cpp
rename to flow/CoroTests.cpp
index 8d4c712f388..5af20f95996 100644
--- a/fdbrpc/CoroTests.cpp
+++ b/flow/CoroTests.cpp
@@ -20,13 +20,22 @@
 
 #include "flow/UnitTest.h"
 #include "flow/IAsyncFile.h"
-#include "fdbrpc/fdbrpc.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/FlowThread.h"
+#include "flow/Trace.h"
+#include "flow/TLSConfig.h"
 
-#include <sstream>
+#include <algorithm>
 #include <cstdint>
-#include <ranges>
+#include <functional>
+#include <iostream>
 #include <iterator>
+#include <limits>
+#include <ranges>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <variant>
+#include <vector>
 
 #include <fmt/base.h>
 #include <fmt/format.h>
@@ -173,6 +182,16 @@ Future<T> templateActor(T t) {
 	return t;
 }
 
+Future<int> coroValueAfterYield(int value) {
+	co_await delay(0);
+	co_return value;
+}
+
+Future<int> coroErrorAfterYield() {
+	co_await delay(0);
+	throw operation_failed();
+}
+
 // bool expectActorCount(int x) { return actorCount == x; }
 bool expectActorCount(int) {
 	return true;
@@ -330,6 +349,55 @@ TEST_CASE("/flow/coro/cancel2") {
 	return Void();
 }
 
+TEST_CASE("/flow/coro/errorOr/ReadyValue") {
+	ErrorOr<int> result = co_await coro::errorOr(Future<int>(42));
+	ASSERT(result.present());
+	ASSERT_EQ(result.get(), 42);
+}
+
+TEST_CASE("/flow/coro/errorOr/ReadyError") {
+	ErrorOr<int> result = co_await coro::errorOr(Future<int>(operation_failed()));
+	ASSERT(result.isError());
+	ASSERT_EQ(result.getError().code(), error_code_operation_failed);
+}
+
+TEST_CASE("/flow/coro/errorOr/AsyncValue") {
+	ErrorOr<int> result = co_await coro::errorOr(coroValueAfterYield(7));
+	ASSERT(result.present());
+	ASSERT_EQ(result.get(), 7);
+}
+
+TEST_CASE("/flow/coro/errorOr/AsyncError") {
+	ErrorOr<int> result = co_await coro::errorOr(coroErrorAfterYield());
+	ASSERT(result.isError());
+	ASSERT_EQ(result.getError().code(), error_code_operation_failed);
+}
+
+TEST_CASE("/flow/coro/ignore/ReadyValue") {
+	Void ignored = co_await coro::ignore(Future<int>(42));
+	(void)ignored;
+}
+
+TEST_CASE("/flow/coro/ignore/ReadyError") {
+	try {
+		co_await coro::ignore(Future<int>(operation_failed()));
+		ASSERT(false);
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_operation_failed);
+	}
+}
+
+TEST_CASE("/flow/coro/errorOrIgnore/AsyncValue") {
+	ErrorOr<Void> result = co_await coro::errorOr(coro::ignore(coroValueAfterYield(9)));
+	ASSERT(result.present());
+}
+
+TEST_CASE("/flow/coro/errorOrIgnore/AsyncError") {
+	ErrorOr<Void> result = co_await coro::errorOr(coro::ignore(coroErrorAfterYield()));
+	ASSERT(result.isError());
+	ASSERT_EQ(result.getError().code(), error_code_operation_failed);
+}
+
 TEST_CASE("/flow/coro/trivial_actors") {
 	ASSERT(expectActorCount(0));
 
@@ -863,7 +931,7 @@ struct YAMRandom {
 		} else if (op == 1) {
 			onchanges.push_back(trigger([this]() { this->randomOp(); }, yam.onChange(k)));
 		} else if (op == 2) {
-			if (onchanges.size()) {
+			if (!onchanges.empty()) {
 				int i = deterministicRandom()->randomInt(0, onchanges.size());
 				onchanges[i] = onchanges.back();
 				onchanges.pop_back();
@@ -1105,8 +1173,8 @@ Future<int> Outer::Foo5::foo() {
 struct Tracker {
 	int copied;
 	bool moved;
-	Tracker(int copied = 0) : copied(copied), moved(false) {}
-	Tracker(Tracker&& other) : Tracker(other.copied) {
+	explicit Tracker(int copied = 0) : copied(copied), moved(false) {}
+	explicit(false) Tracker(Tracker&& other) : Tracker(other.copied) {
 		ASSERT(!other.moved);
 		other.moved = true;
 	}
@@ -1117,7 +1185,7 @@ struct Tracker {
 		this->copied = other.copied;
 		return *this;
 	}
-	Tracker(const Tracker& other) : Tracker(other.copied + 1) { ASSERT(!other.moved); }
+	explicit(false) Tracker(const Tracker& other) : Tracker(other.copied + 1) { ASSERT(!other.moved); }
 	Tracker& operator=(const Tracker& other) {
 		ASSERT(!other.moved);
 		this->moved = false;
@@ -1133,6 +1201,79 @@ struct Tracker {
 	}
 };
 
+struct LifetimeTracked {
+	inline static int liveCount = 0;
+
+	LifetimeTracked() { ++liveCount; }
+	explicit(false) LifetimeTracked(const LifetimeTracked&) { ++liveCount; }
+	explicit(false) LifetimeTracked(LifetimeTracked&&) noexcept { ++liveCount; }
+	LifetimeTracked& operator=(const LifetimeTracked&) = default;
+	LifetimeTracked& operator=(LifetimeTracked&&) noexcept = default;
+	~LifetimeTracked() { --liveCount; }
+};
+
+AsyncResult<Tracker> immediateAsyncResultTracker() {
+	co_return Tracker{};
+}
+
+AsyncResult<Tracker> delayedAsyncResultTracker(Future<Void> signal) {
+	co_await signal;
+	co_return Tracker{};
+}
+
+AsyncResult<int> immediateAsyncResultInt(int value) {
+	co_return value;
+}
+
+AsyncResult<int> delayedAsyncResultInt(Future<Void> signal, int value) {
+	co_await signal;
+	co_return value;
+}
+
+AsyncResult<int> failingAsyncResultInt(Future<Void> signal) {
+	co_await signal;
+	throw io_error();
+}
+
+AsyncResult<Void> noThrowOnCancelAsyncResult(Future<Void> signal, int* cleanupCount, NoThrowOnCancel = {}) {
+	struct CleanupCounter {
+		int* count;
+		~CleanupCounter() { ++*count; }
+	} cleanupCounter{ cleanupCount };
+
+	try {
+		co_await signal;
+	} catch (Error&) {
+		// NoThrowOnCancel should destroy the frame and run RAII cleanup without
+		// resuming here to throw actor_cancelled().
+		*cleanupCount += 100;
+	}
+	co_return;
+}
+
+AsyncResult<int> trackedAsyncResultInt(Future<Void> signal, int value, int* cancelledCount, int* completedCount) {
+	LifetimeTracked lifetime;
+	try {
+		co_await signal;
+		++*completedCount;
+		co_return value;
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			++*cancelledCount;
+		}
+		throw;
+	}
+}
+
+AsyncResult<Void> delayedAsyncResultVoid(Future<Void> signal) {
+	co_await signal;
+	co_return;
+}
+
+AsyncResult<LifetimeTracked> immediateAsyncResultLifetimeTracked() {
+	co_return LifetimeTracked{};
+}
+
 } // namespace
 
 TEST_CASE("/flow/coro/PromiseStream/move") {
@@ -1202,6 +1343,252 @@ TEST_CASE("/flow/coro/PromiseStream/move2") {
 	ASSERT(movedTracker.copied == 0);
 }
 
+TEST_CASE("/flow/coro/AsyncResult/move") {
+	{
+		Tracker tracker = co_await immediateAsyncResultTracker();
+		ASSERT(!tracker.moved);
+		ASSERT(tracker.copied == 0);
+	}
+
+	{
+		Promise<Void> signal;
+		AsyncResult<Tracker> result = delayedAsyncResultTracker(signal.getFuture());
+		signal.send(Void());
+		Tracker tracker = co_await result;
+		ASSERT(!tracker.moved);
+		ASSERT(tracker.copied == 0);
+	}
+}
+
+TEST_CASE("/flow/coro/AsyncResult/releaseDestroysState") {
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+
+	{
+		AsyncResult<LifetimeTracked> result = immediateAsyncResultLifetimeTracked();
+		ASSERT_GT(LifetimeTracked::liveCount, 0);
+	}
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+
+	{
+		LifetimeTracked value = co_await immediateAsyncResultLifetimeTracked();
+		ASSERT_GT(LifetimeTracked::liveCount, 0);
+	}
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+}
+
+TEST_CASE("/flow/coro/AsyncResult/noThrowOnCancel") {
+	Promise<Void> signal;
+	int cleanupCount = 0;
+	AsyncResult<Void> result = noThrowOnCancelAsyncResult(signal.getFuture(), &cleanupCount);
+
+	ASSERT(signal.getFutureReferenceCount() > 0);
+	result.cancel();
+	ASSERT(result.isReady() && result.isError() && result.getError().code() == error_code_actor_cancelled);
+	ASSERT_EQ(cleanupCount, 1);
+	ASSERT(signal.getFutureReferenceCount() == 0);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/quorumAsyncResultReady") {
+	std::vector<AsyncResult<int>> results;
+	results.push_back(immediateAsyncResultInt(1));
+	results.push_back(immediateAsyncResultInt(2));
+
+	Future<Void> q = quorum(std::move(results), 2);
+	ASSERT(q.isReady());
+	co_await q;
+}
+
+TEST_CASE("/flow/coro/quorumAsyncResultSuccess") {
+	std::vector<Promise<Void>> signals(3);
+	std::vector<AsyncResult<int>> results;
+	for (int i = 0; i < signals.size(); ++i) {
+		results.push_back(delayedAsyncResultInt(signals[i].getFuture(), i));
+	}
+
+	Future<Void> q = quorum(std::move(results), 2);
+	ASSERT(!q.isReady());
+	signals[0].send(Void());
+	ASSERT(!q.isReady());
+	signals[1].send(Void());
+	ASSERT(q.isReady());
+	signals[2].send(Void());
+	co_await q;
+}
+
+TEST_CASE("/flow/coro/quorumAsyncResultVoid") {
+	Promise<Void> signal1;
+	Promise<Void> signal2;
+	std::vector<AsyncResult<Void>> results;
+	results.push_back(delayedAsyncResultVoid(signal1.getFuture()));
+	results.push_back(delayedAsyncResultVoid(signal2.getFuture()));
+
+	Future<Void> q = quorum(std::move(results), 1);
+	ASSERT(!q.isReady());
+	signal1.send(Void());
+	ASSERT(q.isReady());
+	signal2.send(Void());
+	co_await q;
+}
+
+TEST_CASE("/flow/coro/quorumAsyncResultError") {
+	Promise<Void> successSignal;
+	Promise<Void> errorSignal;
+	std::vector<AsyncResult<int>> results;
+	results.push_back(delayedAsyncResultInt(successSignal.getFuture(), 1));
+	results.push_back(failingAsyncResultInt(errorSignal.getFuture()));
+
+	Future<Void> q = quorum(std::move(results), 2);
+	ASSERT(!q.isReady());
+	successSignal.send(Void());
+	ASSERT(!q.isReady());
+	errorSignal.send(Void());
+	try {
+		co_await q;
+		ASSERT(false);
+	} catch (Error const& e) {
+		ASSERT_EQ(e.code(), error_code_io_error);
+	}
+}
+
+TEST_CASE("/flow/coro/quorumAsyncResultCancelsRemainingProducers") {
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+
+	int cancelledCount = 0;
+	int completedCount = 0;
+	std::vector<Promise<Void>> signals(3);
+	std::vector<AsyncResult<int>> results;
+	for (int i = 0; i < signals.size(); ++i) {
+		results.push_back(trackedAsyncResultInt(signals[i].getFuture(), i, &cancelledCount, &completedCount));
+	}
+
+	Future<Void> q = quorum(std::move(results), 1);
+	signals[0].send(Void());
+	co_await q;
+
+	ASSERT_EQ(completedCount, 1);
+	ASSERT_EQ(cancelledCount, 2);
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+}
+
+TEST_CASE("/flow/coro/quorumAsyncResultDropCancelsProducers") {
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+
+	int cancelledCount = 0;
+	int completedCount = 0;
+	std::vector<Promise<Void>> signals(2);
+	{
+		std::vector<AsyncResult<int>> results;
+		for (int i = 0; i < signals.size(); ++i) {
+			results.push_back(trackedAsyncResultInt(signals[i].getFuture(), i, &cancelledCount, &completedCount));
+		}
+
+		Future<Void> q = quorum(std::move(results), 2);
+		ASSERT(!q.isReady());
+	}
+
+	ASSERT_EQ(completedCount, 0);
+	ASSERT_EQ(cancelledCount, 2);
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+	co_return;
+}
+
+TEST_CASE("/flow/coro/getAllAsyncResultReady") {
+	std::vector<AsyncResult<int>> results;
+	results.push_back(immediateAsyncResultInt(3));
+	results.push_back(immediateAsyncResultInt(5));
+
+	std::vector<int> output = co_await getAll(std::move(results));
+	ASSERT_EQ(output.size(), 2);
+	ASSERT_EQ(output[0], 3);
+	ASSERT_EQ(output[1], 5);
+}
+
+TEST_CASE("/flow/coro/getAllAsyncResultSuccess") {
+	Promise<Void> signal1;
+	Promise<Void> signal2;
+	std::vector<AsyncResult<int>> results;
+	results.push_back(delayedAsyncResultInt(signal1.getFuture(), 7));
+	results.push_back(delayedAsyncResultInt(signal2.getFuture(), 11));
+
+	Future<std::vector<int>> all = getAll(std::move(results));
+	ASSERT(!all.isReady());
+	signal2.send(Void());
+	ASSERT(!all.isReady());
+	signal1.send(Void());
+
+	std::vector<int> output = co_await all;
+	ASSERT_EQ(output.size(), 2);
+	ASSERT_EQ(output[0], 7);
+	ASSERT_EQ(output[1], 11);
+}
+
+TEST_CASE("/flow/coro/getAllAsyncResultError") {
+	Promise<Void> successSignal;
+	Promise<Void> errorSignal;
+	std::vector<AsyncResult<int>> results;
+	results.push_back(delayedAsyncResultInt(successSignal.getFuture(), 13));
+	results.push_back(failingAsyncResultInt(errorSignal.getFuture()));
+
+	Future<std::vector<int>> all = getAll(std::move(results));
+	ASSERT(!all.isReady());
+	errorSignal.send(Void());
+	try {
+		co_await all;
+		ASSERT(false);
+	} catch (Error const& e) {
+		ASSERT_EQ(e.code(), error_code_io_error);
+	}
+	successSignal.send(Void());
+}
+
+TEST_CASE("/flow/coro/getAllAsyncResultErrorCancelsRemainingProducers") {
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+
+	int cancelledCount = 0;
+	int completedCount = 0;
+	Promise<Void> successSignal;
+	Promise<Void> errorSignal;
+	std::vector<AsyncResult<int>> results;
+	results.push_back(trackedAsyncResultInt(successSignal.getFuture(), 17, &cancelledCount, &completedCount));
+	results.push_back(failingAsyncResultInt(errorSignal.getFuture()));
+
+	Future<std::vector<int>> all = getAll(std::move(results));
+	errorSignal.send(Void());
+	try {
+		co_await all;
+		ASSERT(false);
+	} catch (Error const& e) {
+		ASSERT_EQ(e.code(), error_code_io_error);
+	}
+
+	ASSERT_EQ(completedCount, 0);
+	ASSERT_EQ(cancelledCount, 1);
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+}
+
+TEST_CASE("/flow/coro/getAllAsyncResultDropCancelsProducers") {
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+
+	int cancelledCount = 0;
+	int completedCount = 0;
+	std::vector<Promise<Void>> signals(2);
+	{
+		std::vector<AsyncResult<int>> results;
+		for (int i = 0; i < signals.size(); ++i) {
+			results.push_back(trackedAsyncResultInt(signals[i].getFuture(), i, &cancelledCount, &completedCount));
+		}
+
+		Future<std::vector<int>> all = getAll(std::move(results));
+		ASSERT(!all.isReady());
+	}
+
+	ASSERT_EQ(completedCount, 0);
+	ASSERT_EQ(cancelledCount, 2);
+	ASSERT_EQ(LifetimeTracked::liveCount, 0);
+	co_return;
+}
+
 namespace {
 
 constexpr double mutexTestDelay = 0.00001;
@@ -1341,6 +1728,67 @@ struct LifetimeLogger {
 	int id;
 };
 
+enum class NoThrowOnCancelEvent { Start, LifetimeConstructed, WaitReturned, CatchBlock, AfterWait, LifetimeDestroyed };
+
+struct NoThrowOnCancelRecorder {
+	std::vector<NoThrowOnCancelEvent> events;
+
+	int count(NoThrowOnCancelEvent event) const { return std::count(events.begin(), events.end(), event); }
+
+	void record(NoThrowOnCancelEvent event) { events.push_back(event); }
+};
+
+struct NoThrowOnCancelLifetimeTracker {
+	explicit NoThrowOnCancelLifetimeTracker(NoThrowOnCancelRecorder& recorder) : recorder(recorder) {
+		recorder.record(NoThrowOnCancelEvent::LifetimeConstructed);
+	}
+	~NoThrowOnCancelLifetimeTracker() { recorder.record(NoThrowOnCancelEvent::LifetimeDestroyed); }
+
+	NoThrowOnCancelRecorder& recorder;
+};
+
+void assertNoThrowOnCancelDestroyedAtWait(NoThrowOnCancelRecorder const& recorder) {
+	const std::vector<NoThrowOnCancelEvent> expected{
+		NoThrowOnCancelEvent::Start,
+		NoThrowOnCancelEvent::LifetimeConstructed,
+		NoThrowOnCancelEvent::LifetimeDestroyed,
+	};
+	ASSERT(recorder.events == expected);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::WaitReturned), 0);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::CatchBlock), 0);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::AfterWait), 0);
+}
+
+void assertNoThrowOnCancelCompleted(NoThrowOnCancelRecorder const& recorder) {
+	const std::vector<NoThrowOnCancelEvent> expected{
+		NoThrowOnCancelEvent::Start,     NoThrowOnCancelEvent::LifetimeConstructed, NoThrowOnCancelEvent::WaitReturned,
+		NoThrowOnCancelEvent::AfterWait, NoThrowOnCancelEvent::LifetimeDestroyed,
+	};
+	ASSERT(recorder.events == expected);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::CatchBlock), 0);
+}
+
+void assertNoThrowOnCancelCaughtError(NoThrowOnCancelRecorder const& recorder) {
+	const std::vector<NoThrowOnCancelEvent> expected{
+		NoThrowOnCancelEvent::Start,     NoThrowOnCancelEvent::LifetimeConstructed, NoThrowOnCancelEvent::CatchBlock,
+		NoThrowOnCancelEvent::AfterWait, NoThrowOnCancelEvent::LifetimeDestroyed,
+	};
+	ASSERT(recorder.events == expected);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::WaitReturned), 0);
+}
+
+void assertNoThrowOnCancelDestroyedAfterFirstWait(NoThrowOnCancelRecorder const& recorder) {
+	const std::vector<NoThrowOnCancelEvent> expected{
+		NoThrowOnCancelEvent::Start,
+		NoThrowOnCancelEvent::LifetimeConstructed,
+		NoThrowOnCancelEvent::WaitReturned,
+		NoThrowOnCancelEvent::LifetimeDestroyed,
+	};
+	ASSERT(recorder.events == expected);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::CatchBlock), 0);
+	ASSERT_EQ(recorder.count(NoThrowOnCancelEvent::AfterWait), 0);
+}
+
 template <typename T>
 Future<Void> simple_await_test(std::stringstream& ss, Future<T> f) {
 	ss << "start. ";
@@ -1377,6 +1825,73 @@ Future<Void> actor_cancel_test(std::stringstream& ss) {
 	ss << "after co_return. ";
 }
 
+Future<Void> noThrowOnCancelTest(NoThrowOnCancelRecorder& recorder, Future<Void> signal, NoThrowOnCancel = {}) {
+	recorder.record(NoThrowOnCancelEvent::Start);
+
+	NoThrowOnCancelLifetimeTracker tracker(recorder);
+
+	try {
+		co_await signal;
+		recorder.record(NoThrowOnCancelEvent::WaitReturned);
+	} catch (Error&) {
+		// NoThrowOnCancel should bypass coroutine catches on cancellation.
+		recorder.record(NoThrowOnCancelEvent::CatchBlock);
+	}
+
+	recorder.record(NoThrowOnCancelEvent::AfterWait);
+}
+
+Future<int> noThrowOnCancelValueTest(NoThrowOnCancelRecorder& recorder, Future<Void> signal, NoThrowOnCancel = {}) {
+	recorder.record(NoThrowOnCancelEvent::Start);
+
+	NoThrowOnCancelLifetimeTracker tracker(recorder);
+
+	co_await signal;
+	recorder.record(NoThrowOnCancelEvent::WaitReturned);
+	recorder.record(NoThrowOnCancelEvent::AfterWait);
+	co_return 42;
+}
+
+Future<Void> noThrowOnCancelSequentialAwaitsTest(NoThrowOnCancelRecorder& recorder,
+                                                 Future<Void> firstSignal,
+                                                 Future<Void> secondSignal,
+                                                 NoThrowOnCancel = {}) {
+	recorder.record(NoThrowOnCancelEvent::Start);
+
+	NoThrowOnCancelLifetimeTracker tracker(recorder);
+
+	co_await firstSignal;
+	recorder.record(NoThrowOnCancelEvent::WaitReturned);
+	co_await secondSignal;
+	recorder.record(NoThrowOnCancelEvent::AfterWait);
+}
+
+Future<int> noThrowOnCancelFutureStreamTest(NoThrowOnCancelRecorder& recorder,
+                                            FutureStream<int> stream,
+                                            NoThrowOnCancel = {}) {
+	recorder.record(NoThrowOnCancelEvent::Start);
+
+	NoThrowOnCancelLifetimeTracker tracker(recorder);
+
+	int value = co_await stream;
+	recorder.record(NoThrowOnCancelEvent::WaitReturned);
+	recorder.record(NoThrowOnCancelEvent::AfterWait);
+	co_return value;
+}
+
+Future<int> noThrowOnCancelThreadFutureStreamTest(NoThrowOnCancelRecorder& recorder,
+                                                  ThreadFutureStream<int> stream,
+                                                  NoThrowOnCancel = {}) {
+	recorder.record(NoThrowOnCancelEvent::Start);
+
+	NoThrowOnCancelLifetimeTracker tracker(recorder);
+
+	int value = co_await stream;
+	recorder.record(NoThrowOnCancelEvent::WaitReturned);
+	recorder.record(NoThrowOnCancelEvent::AfterWait);
+	co_return value;
+}
+
 Future<Void> actor_throw_test(std::stringstream& ss) {
 	ss << "start. ";
 
@@ -1867,7 +2382,9 @@ TEST_CASE("/flow/coro/generators") {
 	testFibDivisible();
 	co_await testEmptyGenerator();
 	co_await testSimpleGenerator();
-	co_await testReadLines();
+	if (IAsyncFileSystem::filesystem() != nullptr) {
+		co_await testReadLines();
+	}
 	testElementWalker();
 }
 
@@ -1919,6 +2436,40 @@ TEST_CASE("/flow/coro/actor") {
 		                    "~LifetimeLogger(0). ");
 	}
 
+	TraceEvent("CoroNoThrowOnCancelTestStart");
+	{
+		NoThrowOnCancelRecorder recorder;
+		Promise<Void> signal;
+		Future<Void> f = noThrowOnCancelTest(recorder, signal.getFuture());
+		ASSERT(signal.getFutureReferenceCount() > 0);
+		f.cancel();
+		ASSERT(f.isReady() && f.isError() && f.getError().code() == error_code_actor_cancelled);
+		ASSERT(signal.getFutureReferenceCount() == 0);
+		TraceEvent("CoroNoThrowOnCancelTestResult")
+		    .detail("Scenario", "ExplicitCancel")
+		    .detail("EventCount", recorder.events.size())
+		    .detail("WaitReturnedCount", recorder.count(NoThrowOnCancelEvent::WaitReturned))
+		    .detail("CatchBlockCount", recorder.count(NoThrowOnCancelEvent::CatchBlock))
+		    .detail("AfterWaitCount", recorder.count(NoThrowOnCancelEvent::AfterWait));
+		assertNoThrowOnCancelDestroyedAtWait(recorder);
+	}
+	{
+		NoThrowOnCancelRecorder recorder;
+		Promise<Void> signal;
+		{
+			Future<Void> f = noThrowOnCancelTest(recorder, signal.getFuture());
+			ASSERT(signal.getFutureReferenceCount() > 0);
+		}
+		ASSERT(signal.getFutureReferenceCount() == 0);
+		TraceEvent("CoroNoThrowOnCancelTestResult")
+		    .detail("Scenario", "DropFuture")
+		    .detail("EventCount", recorder.events.size())
+		    .detail("WaitReturnedCount", recorder.count(NoThrowOnCancelEvent::WaitReturned))
+		    .detail("CatchBlockCount", recorder.count(NoThrowOnCancelEvent::CatchBlock))
+		    .detail("AfterWaitCount", recorder.count(NoThrowOnCancelEvent::AfterWait));
+		assertNoThrowOnCancelDestroyedAtWait(recorder);
+	}
+
 	std::cout << std::endl;
 	std::cout << "actor_throw_test\n";
 	std::cout << "================\n";
@@ -1942,6 +2493,90 @@ TEST_CASE("/flow/coro/actor") {
 	co_await stackMemoryTest();
 }
 
+TEST_CASE("/flow/coro/noThrowOnCancel/normalCompletionVoid") {
+	NoThrowOnCancelRecorder recorder;
+	Promise<Void> signal;
+	Future<Void> f = noThrowOnCancelTest(recorder, signal.getFuture());
+	ASSERT(signal.getFutureReferenceCount() > 0);
+
+	signal.send(Void());
+	ASSERT(f.isReady() && !f.isError());
+	ASSERT(signal.getFutureReferenceCount() == 0);
+	assertNoThrowOnCancelCompleted(recorder);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/noThrowOnCancel/normalCompletionValue") {
+	NoThrowOnCancelRecorder recorder;
+	Promise<Void> signal;
+	Future<int> f = noThrowOnCancelValueTest(recorder, signal.getFuture());
+	ASSERT(signal.getFutureReferenceCount() > 0);
+
+	signal.send(Void());
+	ASSERT(f.isReady() && !f.isError());
+	ASSERT_EQ(f.get(), 42);
+	ASSERT(signal.getFutureReferenceCount() == 0);
+	assertNoThrowOnCancelCompleted(recorder);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/noThrowOnCancel/awaitedFutureErrorRunsCatch") {
+	NoThrowOnCancelRecorder recorder;
+	Promise<Void> signal;
+	Future<Void> f = noThrowOnCancelTest(recorder, signal.getFuture());
+	ASSERT(signal.getFutureReferenceCount() > 0);
+
+	signal.sendError(io_error());
+	ASSERT(f.isReady() && !f.isError());
+	ASSERT(signal.getFutureReferenceCount() == 0);
+	assertNoThrowOnCancelCaughtError(recorder);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/noThrowOnCancel/sequentialAwaitsCancelSecond") {
+	NoThrowOnCancelRecorder recorder;
+	Promise<Void> firstSignal;
+	Promise<Void> secondSignal;
+	Future<Void> f = noThrowOnCancelSequentialAwaitsTest(recorder, firstSignal.getFuture(), secondSignal.getFuture());
+	ASSERT(firstSignal.getFutureReferenceCount() > 0);
+
+	firstSignal.send(Void());
+	ASSERT(!f.isReady());
+	ASSERT(secondSignal.getFutureReferenceCount() > 0);
+
+	f.cancel();
+	ASSERT(f.isReady() && f.isError() && f.getError().code() == error_code_actor_cancelled);
+	ASSERT(secondSignal.getFutureReferenceCount() == 0);
+	assertNoThrowOnCancelDestroyedAfterFirstWait(recorder);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/noThrowOnCancel/futureStreamCancel") {
+	NoThrowOnCancelRecorder recorder;
+	PromiseStream<int> stream;
+	Future<int> f = noThrowOnCancelFutureStreamTest(recorder, stream.getFuture());
+	ASSERT(stream.getFutureReferenceCount() > 0);
+
+	f.cancel();
+	ASSERT(f.isReady() && f.isError() && f.getError().code() == error_code_actor_cancelled);
+	ASSERT(stream.getFutureReferenceCount() == 0);
+	assertNoThrowOnCancelDestroyedAtWait(recorder);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/noThrowOnCancel/threadFutureStreamCancel") {
+	NoThrowOnCancelRecorder recorder;
+	ThreadReturnPromiseStream<int> stream;
+	Future<int> f = noThrowOnCancelThreadFutureStreamTest(recorder, stream.getFuture());
+	ASSERT(stream.getFutureReferenceCount() > 0);
+
+	f.cancel();
+	ASSERT(f.isReady() && f.isError() && f.getError().code() == error_code_actor_cancelled);
+	ASSERT(stream.getFutureReferenceCount() == 0);
+	assertNoThrowOnCancelDestroyedAtWait(recorder);
+	return Void();
+}
+
 TEST_CASE("/flow/coro/chooseCancelWaiting") {
 	Promise<Void> voidPromise;
 	Promise<int> intPromise;
@@ -1985,3 +2620,135 @@ TEST_CASE("/flow/coro/chooseRepeatedCancel") {
 	ASSERT(chooseFuture.getError().code() == error_code_actor_cancelled);
 	return Void();
 }
+
+TEST_CASE("/flow/coro/raceReady") {
+	Future<std::variant<int, std::string>> raced = race(Future<int>(7), Future<std::string>("later"));
+	ASSERT(raced.isReady());
+	auto result = raced.get();
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 7);
+
+	Future<std::variant<int, std::string>> errorFirst = race(Future<int>(io_error()), Future<std::string>("later"));
+	ASSERT(errorFirst.isReady());
+	ASSERT(errorFirst.isError());
+	ASSERT_EQ(errorFirst.getError().code(), error_code_io_error);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/raceReadyFirstArgumentWinsTie") {
+	// When both inputs are already ready, race() breaks ties by argument order.
+	Future<std::variant<int, std::string>> raced = race(Future<int>(7), Future<std::string>("winner"));
+	ASSERT(raced.isReady());
+	auto result = raced.get();
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 7);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/raceSuccess") {
+	Promise<int> intPromise;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced = race(intPromise.getFuture(), stringPromise.getFuture());
+	stringPromise.send("winner");
+	auto result = co_await raced;
+	ASSERT_EQ(result.index(), 1);
+	ASSERT_EQ(std::get<1>(result), "winner");
+	co_return;
+}
+
+TEST_CASE("/flow/coro/raceSuccessFirstArgument") {
+	Promise<int> intPromise;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced = race(intPromise.getFuture(), stringPromise.getFuture());
+	intPromise.send(42);
+	auto result = co_await raced;
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 42);
+	co_return;
+}
+
+TEST_CASE("/flow/coro/raceError") {
+	Promise<int> intPromise;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced = race(intPromise.getFuture(), stringPromise.getFuture());
+	intPromise.sendError(io_error());
+	try {
+		co_await raced;
+		ASSERT(false);
+	} catch (Error const& e) {
+		ASSERT_EQ(e.code(), error_code_io_error);
+	}
+	co_return;
+}
+
+TEST_CASE("/flow/coro/raceCancel") {
+	Promise<int> intPromise;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced = race(intPromise.getFuture(), stringPromise.getFuture());
+	raced.cancel();
+	ASSERT(raced.isReady());
+	ASSERT(raced.isError());
+	ASSERT_EQ(raced.getError().code(), error_code_actor_cancelled);
+	intPromise.send(1);
+	stringPromise.send("late");
+	ASSERT_EQ(raced.getError().code(), error_code_actor_cancelled);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/raceStreamReady") {
+	PromiseStream<int> intStream;
+	intStream.send(11);
+	Future<std::variant<int, std::string>> raced = race(intStream.getFuture(), Future<std::string>("later"));
+	ASSERT(raced.isReady());
+	auto result = raced.get();
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 11);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/raceStreamSuccess") {
+	PromiseStream<int> intStream;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced = race(intStream.getFuture(), stringPromise.getFuture());
+	intStream.send(13);
+	auto result = co_await raced;
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 13);
+	co_return;
+}
+
+TEST_CASE("/flow/coro/raceAsyncResultReady") {
+	Future<std::variant<int, std::string>> raced = race(immediateAsyncResultInt(17), Future<std::string>("later"));
+	ASSERT(raced.isReady());
+	auto result = raced.get();
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 17);
+	return Void();
+}
+
+TEST_CASE("/flow/coro/raceAsyncResultSuccess") {
+	Promise<Void> signal;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced =
+	    race(delayedAsyncResultInt(signal.getFuture(), 19), stringPromise.getFuture());
+	signal.send(Void());
+	auto result = co_await raced;
+	ASSERT_EQ(result.index(), 0);
+	ASSERT_EQ(std::get<0>(result), 19);
+	co_return;
+}
+
+TEST_CASE("/flow/coro/raceAsyncResultError") {
+	Promise<Void> signal;
+	Promise<std::string> stringPromise;
+	Future<std::variant<int, std::string>> raced =
+	    race(failingAsyncResultInt(signal.getFuture()), stringPromise.getFuture());
+	signal.send(Void());
+	try {
+		co_await raced;
+		ASSERT(false);
+	} catch (Error const& e) {
+		ASSERT_EQ(e.code(), error_code_io_error);
+	}
+	co_return;
+}
diff --git a/flow/Deque.cpp b/flow/Deque.cpp
index 7626ef4be17..f3ee8425b4e 100644
--- a/flow/Deque.cpp
+++ b/flow/Deque.cpp
@@ -84,7 +84,7 @@ struct RandomlyThrows {
 	RandomlyThrows() = default;
 	explicit RandomlyThrows(int data) : data(data) {}
 	~RandomlyThrows() = default;
-	RandomlyThrows(const RandomlyThrows& other) : data(other.data) { randomlyThrow(); }
+	explicit(false) RandomlyThrows(const RandomlyThrows& other) : data(other.data) { randomlyThrow(); }
 	RandomlyThrows& operator=(const RandomlyThrows& other) {
 		data = other.data;
 		randomlyThrow();
diff --git a/flow/DeterministicRandom.cpp b/flow/DeterministicRandom.cpp
index b2f8a8fc33b..8a3079dfce7 100644
--- a/flow/DeterministicRandom.cpp
+++ b/flow/DeterministicRandom.cpp
@@ -27,25 +27,14 @@
 
 uint64_t DeterministicRandom::gen64() {
 	uint64_t curr = next;
-
-	// RE: the previous implementation of this function: order of
-	// evaluation of arguments to the ^ operator is not specified, so
-	// two rng() calls in the same ^ expression may not produce a
-	// consistent 64-bit value across compilations, because we don't
-	// know if the first call was used for the higher order bits and
-	// the second call for the low order bits, or vice-versa.
-	// See https://en.cppreference.com/w/cpp/language/eval_order.html
-	next = (uint64_t(rng()) << 32);
-	next ^= rng();
+	next = rng();
 	if (TRACE_SAMPLE())
 		TraceEvent(SevSample, "Random").log();
 	return curr;
 }
 
-DeterministicRandom::DeterministicRandom(uint32_t seed, bool useRandLog)
-  : rng((unsigned long)seed), next(0), useRandLog(useRandLog) {
-	next = (uint64_t(rng()) << 32);
-	next ^= rng();
+DeterministicRandom::DeterministicRandom(uint64_t seed, bool useRandLog) : rng(seed), next(0), useRandLog(useRandLog) {
+	next = rng();
 }
 
 double DeterministicRandom::random01() {
@@ -164,10 +153,9 @@ uint64_t DeterministicRandom::peek() const {
 	return next;
 }
 
-void DeterministicRandom::resetSeed(uint32_t seed) {
-	rng.seed((unsigned long)seed);
-	next = (uint64_t(rng()) << 32);
-	next ^= rng();
+void DeterministicRandom::resetSeed(uint64_t seed) {
+	rng.seed(seed);
+	next = rng();
 }
 
 void DeterministicRandom::addref() {
diff --git a/flow/Error.cpp b/flow/Error.cpp
index bd78836af1c..b6c1fe2a6d6 100644
--- a/flow/Error.cpp
+++ b/flow/Error.cpp
@@ -18,8 +18,6 @@
  * limitations under the License.
  */
 
-#include <iostream>
-
 #include "flow/Error.h"
 #include "flow/Knobs.h"
 #include "flow/Trace.h"
@@ -215,7 +213,6 @@ const std::set<int> transactionRetryableErrors = { error_code_not_committed,
 	                                               error_code_process_behind,
 	                                               error_code_batch_transaction_throttled,
 	                                               error_code_tag_throttled,
-	                                               error_code_proxy_tag_throttled,
 	                                               // maybe committed error
 	                                               error_code_cluster_version_changed,
 	                                               error_code_commit_unknown_result };
diff --git a/flow/include/flow/EventTypes.actor.h b/flow/EventTypes.h
similarity index 59%
rename from flow/include/flow/EventTypes.actor.h
rename to flow/EventTypes.h
index 07d0110b312..0d8f588f545 100644
--- a/flow/include/flow/EventTypes.actor.h
+++ b/flow/EventTypes.h
@@ -1,5 +1,5 @@
 /*
- * EventTypes.actor.h
+ * EventTypes.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,23 +20,16 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_EVENTTYPES_ACTOR_G_H)
-#define FLOW_EVENTTYPES_ACTOR_G_H
-#include "flow/EventTypes.actor.g.h"
-#elif !defined(FLOW_EVENTTYPES_ACTOR_H)
-#define FLOW_EVENTTYPES_ACTOR_H
-
 #include "flow/flow.h"
-#include "flow/TDMetric.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/TDMetric.h"
 
-DESCR struct TraceEventNameID {
+struct TraceEventNameIDDescriptor {
 	Standalone<StringRef> name;
 	Standalone<StringRef> id;
 };
 
-#include "flow/unactorcompiler.h"
-
-#endif
+template <>
+struct Descriptor<TraceEventNameIDDescriptor> : DescribeType<TraceEventNameIDDescriptor,
+                                                             "TraceEventNameID",
+                                                             DescribeField<&TraceEventNameIDDescriptor::name, "name">,
+                                                             DescribeField<&TraceEventNameIDDescriptor::id, "id">> {};
diff --git a/flow/FileTraceLogWriter.cpp b/flow/FileTraceLogWriter.cpp
index 7eb89a80c1a..c1c56d3bb86 100644
--- a/flow/FileTraceLogWriter.cpp
+++ b/flow/FileTraceLogWriter.cpp
@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-#include "flow/FileTraceLogWriter.h"
+#include "FileTraceLogWriter.h"
 #include "flow/Platform.h"
 #include "flow/flow.h"
 #include "flow/ThreadHelper.actor.h"
@@ -49,7 +49,7 @@
 #include <cmath>
 
 struct IssuesListImpl {
-	IssuesListImpl() {}
+	IssuesListImpl() = default;
 	void addIssue(std::string const& issue) {
 		MutexHolder h(mutex);
 		issues.insert(issue);
@@ -258,7 +258,7 @@ void FileTraceLogWriter::cleanupTraceFiles() {
 			std::sort(existingTraceFiles.begin(), existingTraceFiles.end(), std::greater<std::string>());
 
 			uint64_t runningTotal = 0;
-			std::vector<std::string>::iterator fileListIterator = existingTraceFiles.begin();
+			auto fileListIterator = existingTraceFiles.begin();
 
 			while (runningTotal < maxLogsSize && fileListIterator != existingTraceFiles.end()) {
 				runningTotal += (fileSize(joinPath(directory, *fileListIterator)) + FLOW_KNOBS->ZERO_LENGTH_FILE_PAD);
diff --git a/flow/include/flow/FileTraceLogWriter.h b/flow/FileTraceLogWriter.h
similarity index 100%
rename from flow/include/flow/FileTraceLogWriter.h
rename to flow/FileTraceLogWriter.h
diff --git a/flow/FlowTest.cpp b/flow/FlowTest.cpp
new file mode 100644
index 00000000000..87b178c99e9
--- /dev/null
+++ b/flow/FlowTest.cpp
@@ -0,0 +1,362 @@
+/*
+ * FlowTest.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/Platform.h"
+#include "flow/TLSConfig.h"
+#include "flow/Trace.h"
+#include "flow/UnitTest.h"
+#include "flow/network.h"
+#include "SimpleOpt/SimpleOpt.h"
+
+#include <fmt/format.h>
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#ifndef _WIN32
+#include <unistd.h>
+#endif
+
+namespace {
+
+struct FlowTestOptions {
+	std::string testPattern;
+	std::vector<std::string> testsIgnored;
+	std::string dataDir = "flow_test_data";
+	uint64_t randomSeed = 0;
+	int maxTestCases = -1;
+	bool cleanupAfterTests = true;
+	bool listTests = false;
+	bool showHelp = false;
+};
+
+struct FlowTestResult {
+	int testsAvailable = 0;
+	int testsExecuted = 0;
+	int testsFailed = 0;
+};
+
+enum FlowTestOption {
+	OPT_HELP,
+	OPT_FILTER,
+	OPT_IGNORE,
+	OPT_DATA_DIR,
+	OPT_SEED,
+	OPT_MAX_TEST_CASES,
+	OPT_NO_CLEANUP,
+	OPT_LIST,
+};
+
+CSimpleOpt::SOption flowTestOptions[] = { { OPT_HELP, "-h", SO_NONE },
+	                                      { OPT_HELP, "--help", SO_NONE },
+	                                      { OPT_FILTER, "-f", SO_REQ_SEP },
+	                                      { OPT_FILTER, "--filter", SO_REQ_SEP },
+	                                      { OPT_IGNORE, "--ignore", SO_REQ_SEP },
+	                                      { OPT_DATA_DIR, "--data-dir", SO_REQ_SEP },
+	                                      { OPT_SEED, "--seed", SO_REQ_SEP },
+	                                      { OPT_MAX_TEST_CASES, "--max-test-cases", SO_REQ_SEP },
+	                                      { OPT_NO_CLEANUP, "--no-cleanup", SO_NONE },
+	                                      { OPT_LIST, "--list", SO_NONE },
+	                                      SO_END_OF_OPTIONS };
+
+void printUsage(const char* program) {
+	fmt::print(stderr,
+	           "Usage: {} [OPTIONS]\n"
+	           "\n"
+	           "Run TEST_CASEs linked into the flow library.\n"
+	           "\n"
+	           "Options:\n"
+	           "  -f, --filter PREFIX       Run tests whose names start with PREFIX\n"
+	           "      --ignore PREFIX       Skip tests whose names start with PREFIX\n"
+	           "      --data-dir DIR        Per-test data directory (default: flow_test_data)\n"
+	           "      --seed N              Deterministic random seed (default: random)\n"
+	           "      --max-test-cases N    Stop after N matching tests\n"
+	           "      --no-cleanup          Keep the data directory after each test\n"
+	           "      --list                Print matching test names without running them\n"
+	           "  -h, --help                Show this help\n",
+	           program);
+}
+
+bool parseInt(const char* text, int* value) {
+	char* end = nullptr;
+	long parsed = strtol(text, &end, 10);
+	if (*text == '\0' || *end != '\0') {
+		return false;
+	}
+	*value = static_cast<int>(parsed);
+	return true;
+}
+
+bool parseUInt64(const char* text, uint64_t* value) {
+	char* end = nullptr;
+	uint64_t parsed = strtoull(text, &end, 10);
+	if (*text == '\0' || *end != '\0') {
+		return false;
+	}
+	*value = parsed;
+	return true;
+}
+
+bool parseArgs(int argc, char** argv, FlowTestOptions* options) {
+	CSimpleOpt args(argc, argv, flowTestOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
+	while (args.Next()) {
+		if (auto err = args.LastError()) {
+			switch (err) {
+			case SO_ARG_INVALID_DATA:
+				fmt::print(stderr, "ERROR: Invalid argument to option `{}`\n", args.OptionText());
+				break;
+			case SO_ARG_INVALID:
+				fmt::print(stderr, "ERROR: Argument given to no-argument option `{}`\n", args.OptionText());
+				break;
+			case SO_ARG_MISSING:
+				fmt::print(stderr, "ERROR: Argument missing for option `{}`\n", args.OptionText());
+				break;
+			case SO_OPT_INVALID:
+				fmt::print(stderr, "ERROR: Unknown option `{}`\n", args.OptionText());
+				break;
+			default:
+				fmt::print(
+				    stderr, "ERROR: Unknown error {} with option `{}`\n", static_cast<int>(err), args.OptionText());
+				break;
+			}
+			return false;
+		}
+
+		switch (args.OptionId()) {
+		case OPT_HELP:
+			options->showHelp = true;
+			return true;
+		case OPT_FILTER:
+			options->testPattern = args.OptionArg();
+			break;
+		case OPT_IGNORE:
+			options->testsIgnored.emplace_back(args.OptionArg());
+			break;
+		case OPT_DATA_DIR:
+			options->dataDir = args.OptionArg();
+			break;
+		case OPT_SEED:
+			if (!parseUInt64(args.OptionArg(), &options->randomSeed)) {
+				fmt::print(stderr, "ERROR: --seed requires a uint32 value\n");
+				return false;
+			}
+			break;
+		case OPT_MAX_TEST_CASES:
+			if (!parseInt(args.OptionArg(), &options->maxTestCases)) {
+				fmt::print(stderr, "ERROR: --max-test-cases requires an integer value\n");
+				return false;
+			}
+			break;
+		case OPT_NO_CLEANUP:
+			options->cleanupAfterTests = false;
+			break;
+		case OPT_LIST:
+			options->listTests = true;
+			break;
+		default:
+			fmt::print(stderr, "ERROR: Unknown option id {}\n", args.OptionId());
+			return false;
+		}
+	}
+
+	if (args.FileCount() > 0) {
+		fmt::print(stderr, "ERROR: Unexpected argument `{}`\n", args.File(0));
+		return false;
+	}
+
+	return true;
+}
+
+bool startsWith(std::string_view value, std::string_view prefix) {
+	return value.size() >= prefix.size() && value.substr(0, prefix.size()) == prefix;
+}
+
+bool testMatched(const FlowTestOptions& options, std::string_view testName) {
+	if (!startsWith(testName, options.testPattern)) {
+		return false;
+	}
+
+	for (const auto& ignorePattern : options.testsIgnored) {
+		if (startsWith(testName, ignorePattern)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+std::vector<UnitTest*> collectTests(const FlowTestOptions& options) {
+	std::vector<UnitTest*> tests;
+	for (auto test = g_unittests.tests; test != nullptr; test = test->next) {
+		if (testMatched(options, test->name)) {
+			tests.push_back(test);
+		}
+	}
+
+	std::sort(tests.begin(), tests.end(), [](auto lhs, auto rhs) {
+		return std::string_view(lhs->name) < std::string_view(rhs->name);
+	});
+	return tests;
+}
+
+Future<Void> runFlowTests(const FlowTestOptions& options, FlowTestResult* result) {
+	std::vector<UnitTest*> tests = collectTests(options);
+	result->testsAvailable = tests.size();
+
+	fmt::print(stdout, "Found {} flow tests\n", tests.size());
+
+	if (options.listTests) {
+		for (auto test : tests) {
+			fmt::print(stdout, "{}\n", test->name);
+		}
+		co_return;
+	}
+
+	if (tests.empty()) {
+		TraceEvent(SevError, "NoMatchingFlowTests").detail("TestPattern", options.testPattern);
+		++result->testsFailed;
+		co_return;
+	}
+
+	if (options.maxTestCases > 0 && tests.size() > static_cast<size_t>(options.maxTestCases)) {
+		tests.resize(options.maxTestCases);
+	}
+
+	UnitTestParameters testParams;
+	testParams.setDataDir(options.dataDir);
+
+	for (auto test : tests) {
+		fmt::print(stdout, "Testing {}\n", test->name);
+
+		TraceEvent(SevInfo, "RunningFlowTest")
+		    .detail("Name", test->name)
+		    .detail("File", test->file)
+		    .detail("Line", test->line)
+		    .detail("Rand", deterministicRandom()->randomInt(0, 100001));
+
+		Error resultCode = success();
+		double startNow = now();
+		double startTimer = timer();
+
+		platform::createDirectory(testParams.getDataDir());
+		try {
+			co_await test->func(testParams);
+		} catch (Error& e) {
+			resultCode = e;
+			++result->testsFailed;
+		}
+		if (options.cleanupAfterTests) {
+			platform::eraseDirectoryRecursive(testParams.getDataDir());
+		}
+		++result->testsExecuted;
+
+		double wallTime = timer() - startTimer;
+		double flowTime = now() - startNow;
+		TraceEvent(resultCode.code() != error_code_success ? SevError : SevInfo, "FlowTest")
+		    .errorUnsuppressed(resultCode)
+		    .detail("Name", test->name)
+		    .detail("File", test->file)
+		    .detail("Line", test->line)
+		    .detail("WallTime", wallTime)
+		    .detail("FlowTime", flowTime);
+
+		if (resultCode.code() != error_code_success) {
+			fmt::print(stderr, "Test failed: {}: {}\n", test->name, resultCode.what());
+		}
+	}
+}
+
+Future<Void> stopNetworkAfter(Future<Void> what, int* exitCode) {
+	try {
+		co_await what;
+	} catch (Error& e) {
+		fmt::print(stderr, "Unexpected flow_test error: {}\n", e.what());
+		*exitCode = 1;
+	} catch (std::exception& e) {
+		fmt::print(stderr, "Unexpected flow_test exception: {}\n", e.what());
+		*exitCode = 1;
+	} catch (...) {
+		fmt::print(stderr, "Unexpected flow_test exception\n");
+		*exitCode = 1;
+	}
+	g_network->stop();
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+	platformInit();
+	Error::init();
+	setvbuf(stdout, nullptr, _IOLBF, BUFSIZ);
+	setvbuf(stderr, nullptr, _IOLBF, BUFSIZ);
+
+	FlowTestOptions options;
+	if (!parseArgs(argc, argv, &options)) {
+		printUsage(argv[0]);
+		return 1;
+	}
+	if (options.showHelp) {
+		printUsage(argv[0]);
+		return 0;
+	}
+
+	if (options.randomSeed == 0) {
+		options.randomSeed = platform::getRandomSeed();
+	}
+	setThreadLocalDeterministicRandomSeed(options.randomSeed);
+	fmt::print(stdout, "Random seed is {}\n", options.randomSeed);
+
+	std::string originalWorkingDirectory = platform::getWorkingDirectory();
+	std::string runDirectory = joinPath("/tmp", format("flow_test.%d.%llu", ::getpid(), options.randomSeed));
+	platform::createDirectory(runDirectory);
+	if (::chdir(runDirectory.c_str()) != 0) {
+		fmt::print(stderr, "ERROR: Could not chdir to {}\n", runDirectory);
+		return 1;
+	}
+
+	g_network = newNet2(TLSConfig());
+	openTraceFile({}, 10 << 20, 10 << 20, ".", "flow_test");
+
+	int exitCode = 0;
+	FlowTestResult result;
+	Future<Void> done = stopNetworkAfter(runFlowTests(options, &result), &exitCode);
+	g_network->run();
+	flushTraceFileVoid();
+
+	fmt::print(
+	    stdout, "\n{} tests passed; {} tests failed.\n", result.testsExecuted - result.testsFailed, result.testsFailed);
+
+	if (result.testsFailed != 0) {
+		exitCode = 1;
+	}
+
+	if (::chdir(originalWorkingDirectory.c_str()) == 0) {
+		platform::eraseDirectoryRecursive(runDirectory);
+	}
+	return exitCode;
+}
diff --git a/flow/Histogram.cpp b/flow/Histogram.cpp
index 35e5f7befbe..f9304796019 100644
--- a/flow/Histogram.cpp
+++ b/flow/Histogram.cpp
@@ -27,7 +27,6 @@
 // scoped to the right "machine".
 // either we pull g_simulator into flow, or flow (and the I/O path) will be unable to log performance
 // metrics.
-#include <limits>
 
 #pragma region HistogramRegistry
 
diff --git a/flow/Hostname.actor.cpp b/flow/Hostname.cpp
similarity index 75%
rename from flow/Hostname.actor.cpp
rename to flow/Hostname.cpp
index d5dc96b7ab3..e03555d0903 100644
--- a/flow/Hostname.actor.cpp
+++ b/flow/Hostname.cpp
@@ -1,5 +1,5 @@
 /*
- * Hostname.actor.cpp
+ * Hostname.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -25,8 +25,6 @@
 #include "flow/IConnection.h"
 #include "flow/UnitTest.h"
 
-#include "flow/actorcompiler.h" // has to be last include
-
 namespace {
 
 const static std::regex validation("^([\\w\\-]+\\.?)+:([\\d]+){1,}(:tls)?$");
@@ -60,31 +58,31 @@ Hostname Hostname::parse(const std::string& s) {
 	return Hostname(f.substr(0, colonPos), f.substr(colonPos + 1), isTLS);
 }
 
-ACTOR Future<Optional<NetworkAddress>> resolveImpl(const Hostname* self) {
+Future<Optional<NetworkAddress>> resolveImpl(const Hostname* self) {
 	try {
 		std::vector<NetworkAddress> addresses =
-		    wait(INetworkConnections::net()->resolveTCPEndpointWithDNSCache(self->host, self->service));
+		    co_await INetworkConnections::net()->resolveTCPEndpointWithDNSCache(self->host, self->service);
 		NetworkAddress address = INetworkConnections::pickOneAddress(addresses);
 		address.flags = 0; // Reset the parsed address to public
 		address.fromHostname = NetworkAddressFromHostname::True;
 		if (self->isTLS) {
 			address.flags |= NetworkAddress::FLAG_TLS;
 		}
-		return address;
+		co_return address;
 	} catch (...) {
-		return Optional<NetworkAddress>();
+		co_return Optional<NetworkAddress>();
 	}
 }
 
-ACTOR Future<NetworkAddress> resolveWithRetryImpl(const Hostname* self) {
-	state double resolveInterval = FLOW_KNOBS->HOSTNAME_RESOLVE_INIT_INTERVAL;
-	loop {
+Future<NetworkAddress> resolveWithRetryImpl(const Hostname* self) {
+	double resolveInterval = FLOW_KNOBS->HOSTNAME_RESOLVE_INIT_INTERVAL;
+	while (true) {
 		try {
-			Optional<NetworkAddress> address = wait(resolveImpl(self));
+			Optional<NetworkAddress> address = co_await resolveImpl(self);
 			if (address.present()) {
-				return address.get();
+				co_return address.get();
 			}
-			wait(delay(resolveInterval));
+			co_await delay(resolveInterval);
 			resolveInterval = std::min(2 * resolveInterval, FLOW_KNOBS->HOSTNAME_RESOLVE_MAX_INTERVAL);
 		} catch (Error& e) {
 			ASSERT(e.code() == error_code_actor_cancelled);
@@ -117,7 +115,7 @@ Optional<NetworkAddress> Hostname::resolveBlocking() const {
 	}
 }
 
-TEST_CASE("/flow/Hostname/hostname") {
+TEST_CASE("/flow/Hostname/parse") {
 	std::string hn1s = "localhost:1234";
 	std::string hn2s = "host-name:1234";
 	std::string hn3s = "host.name:1234";
@@ -132,6 +130,9 @@ TEST_CASE("/flow/Hostname/hostname") {
 	std::string hn11s = "[::1]:4800";
 	std::string hn12s = "[::1]:4800:tls";
 	std::string hn13s = "1234";
+	// This host name is guaranteed not to be resolvable:
+	// https://www.rfc-editor.org/rfc/rfc2606.html
+	std::string unresolvableHostnameString = "host-name.invalid:1234";
 
 	auto hn1 = Hostname::parse(hn1s);
 	ASSERT(hn1.toString() == hn1s);
@@ -139,7 +140,7 @@ TEST_CASE("/flow/Hostname/hostname") {
 	ASSERT(hn1.service == "1234");
 	ASSERT(!hn1.isTLS);
 
-	state Hostname hn2 = Hostname::parse(hn2s);
+	Hostname hn2 = Hostname::parse(hn2s);
 	ASSERT(hn2.toString() == hn2s);
 	ASSERT(hn2.host == "host-name");
 	ASSERT(hn2.service == "1234");
@@ -167,36 +168,19 @@ TEST_CASE("/flow/Hostname/hostname") {
 	ASSERT(!Hostname::isHostname(hn12s));
 	ASSERT(!Hostname::isHostname(hn13s));
 
-	state Optional<NetworkAddress> optionalAddress = wait(hn2.resolve());
+	Hostname unresolvableHostname = Hostname::parse(unresolvableHostnameString);
+
+	Optional<NetworkAddress> optionalAddress = co_await unresolvableHostname.resolve();
 	ASSERT(!optionalAddress.present());
 
-	optionalAddress = hn2.resolveBlocking();
+	optionalAddress = unresolvableHostname.resolveBlocking();
 	ASSERT(!optionalAddress.present());
 
-	state NetworkAddress address;
+	NetworkAddress address;
 	try {
-		wait(timeoutError(store(address, hn2.resolveWithRetry()), 1));
+		address = co_await timeoutError(unresolvableHostname.resolveWithRetry(), 1);
 	} catch (Error& e) {
 		ASSERT(e.code() == error_code_timed_out);
 	}
 	ASSERT(address == NetworkAddress());
-
-	state NetworkAddress addressSource = NetworkAddress::parse("127.0.0.0:1234");
-	INetworkConnections::net()->addMockTCPEndpoint("host-name", "1234", { addressSource });
-
-	// Test resolve.
-	wait(store(optionalAddress, hn2.resolve()));
-	ASSERT(optionalAddress.present() && optionalAddress.get() == addressSource);
-	optionalAddress = Optional<NetworkAddress>();
-
-	// Test resolveBlocking.
-	optionalAddress = hn2.resolveBlocking();
-	ASSERT(optionalAddress.present() && optionalAddress.get() == addressSource);
-	optionalAddress = Optional<NetworkAddress>();
-
-	// Test resolveWithRetry.
-	wait(store(address, hn2.resolveWithRetry()));
-	ASSERT(address == addressSource);
-
-	return Void();
 }
diff --git a/flow/IAsyncFile.actor.cpp b/flow/IAsyncFile.actor.cpp
deleted file mode 100644
index 976ee30195d..00000000000
--- a/flow/IAsyncFile.actor.cpp
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * IAsyncFile.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "flow/IAsyncFile.h"
-#include "flow/Error.h"
-#include "flow/Knobs.h"
-#include "flow/Platform.h"
-#include "flow/UnitTest.h"
-#include <iostream>
-#include "flow/actorcompiler.h" // has to be last include
-
-IAsyncFile::~IAsyncFile() = default;
-
-const static unsigned int ONE_MEGABYTE = 1 << 20;
-const static unsigned int FOUR_KILOBYTES = 4 << 10;
-
-ACTOR static Future<Void> zeroRangeHelper(Reference<IAsyncFile> f, int64_t offset, int64_t length, int fixedbyte) {
-	state int64_t pos = offset;
-	state void* zeros = aligned_alloc(ONE_MEGABYTE, ONE_MEGABYTE);
-	memset(zeros, fixedbyte, ONE_MEGABYTE);
-
-	while (pos < offset + length) {
-		state int len = std::min<int64_t>(ONE_MEGABYTE, offset + length - pos);
-		wait(f->write(zeros, len, pos));
-		pos += len;
-		wait(yield());
-	}
-
-	aligned_free(zeros);
-	return Void();
-}
-
-Future<Void> IAsyncFile::zeroRange(int64_t offset, int64_t length) {
-	return uncancellable(zeroRangeHelper(Reference<IAsyncFile>::addRef(this), offset, length, 0));
-}
-
-TEST_CASE("/fileio/zero") {
-	state std::string filename = "/tmp/__ZEROJUNK__";
-	state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-	    filename, IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0));
-
-	// Verify that we can grow a file with zero().
-	wait(f->sync());
-	wait(f->zeroRange(0, ONE_MEGABYTE));
-	int64_t size = wait(f->size());
-	ASSERT(ONE_MEGABYTE == size);
-
-	// Verify that zero() does, in fact, zero.
-	wait(zeroRangeHelper(f, 0, ONE_MEGABYTE, 0xff));
-	wait(f->zeroRange(0, ONE_MEGABYTE));
-	state uint8_t* page = (uint8_t*)malloc(FOUR_KILOBYTES);
-	int n = wait(f->read(page, FOUR_KILOBYTES, 0));
-	ASSERT(n == FOUR_KILOBYTES);
-	for (int i = 0; i < FOUR_KILOBYTES; i++) {
-		ASSERT(page[i] == 0);
-	}
-	free(page);
-
-	// Destruct our file and remove it.
-	f.clear();
-	wait(IAsyncFileSystem::filesystem()->deleteFile(filename, true));
-	return Void();
-}
-
-ACTOR static Future<Void> incrementalDeleteHelper(std::string filename,
-                                                  bool mustBeDurable,
-                                                  int64_t truncateAmt,
-                                                  double interval) {
-	state Reference<IAsyncFile> file;
-	state int64_t remainingFileSize;
-	state bool exists = fileExists(filename);
-
-	if (exists) {
-		Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-		    filename, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, 0));
-		file = f;
-
-		int64_t fileSize = wait(file->size());
-		remainingFileSize = fileSize;
-	}
-
-	wait(IAsyncFileSystem::filesystem()->deleteFile(filename, mustBeDurable));
-
-	if (exists) {
-		for (; remainingFileSize > 0; remainingFileSize -= truncateAmt) {
-			wait(file->truncate(remainingFileSize));
-			wait(file->sync());
-			wait(delay(interval));
-		}
-	}
-
-	return Void();
-}
-
-Future<Void> IAsyncFileSystem::incrementalDeleteFile(const std::string& filename, bool mustBeDurable) {
-	return uncancellable(incrementalDeleteHelper(filename,
-	                                             mustBeDurable,
-	                                             FLOW_KNOBS->INCREMENTAL_DELETE_TRUNCATE_AMOUNT,
-	                                             FLOW_KNOBS->INCREMENTAL_DELETE_INTERVAL));
-}
-
-TEST_CASE("/fileio/incrementalDelete") {
-	// about 5GB
-	state int64_t fileSize = 5e9;
-	state std::string filename = "/tmp/__JUNK__";
-	state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-	    filename, IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0));
-	wait(f->sync());
-	wait(f->truncate(fileSize));
-	// close the file by deleting the reference
-	f.clear();
-	wait(IAsyncFileSystem::filesystem()->incrementalDeleteFile(filename, true));
-	return Void();
-}
-
-TEST_CASE("/fileio/rename") {
-	// create a file
-	state int64_t fileSize = 100e6;
-	state std::string filename = "/tmp/__JUNK__." + deterministicRandom()->randomUniqueID().toString();
-	state std::string renamedFile = "/tmp/__RENAMED_JUNK__." + deterministicRandom()->randomUniqueID().toString();
-	state std::unique_ptr<char[]> data(new char[4096]);
-	state std::unique_ptr<char[]> readData(new char[4096]);
-	state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-	    filename,
-	    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE,
-	    0644));
-	;
-	wait(f->sync());
-	wait(f->truncate(fileSize));
-	memset(data.get(), 0, 4096);
-	// write a random string at the beginning of the file which we can verify after rename
-	for (int i = 0; i < 16; ++i) {
-		data[i] = deterministicRandom()->randomAlphaNumeric();
-	}
-	// write first and block
-	wait(f->write(data.get(), 4096, 0));
-	wait(f->write(data.get(), 4096, fileSize - 4096));
-	wait(f->sync());
-	// close file
-	f.clear();
-	wait(IAsyncFileSystem::filesystem()->renameFile(filename, renamedFile));
-	Reference<IAsyncFile> _f = wait(IAsyncFileSystem::filesystem()->open(renamedFile, IAsyncFile::OPEN_READONLY, 0));
-	f = _f;
-
-	// verify rename happened
-	bool renamedExists = false;
-	auto bName = basename(renamedFile);
-	auto files = platform::listFiles("/tmp/");
-	for (const auto& file : files) {
-		if (file == bName) {
-			renamedExists = true;
-		}
-		ASSERT(file != filename);
-	}
-	ASSERT(renamedExists);
-
-	// verify magic string at beginning of file
-	int length = wait(f->read(readData.get(), 4096, 0));
-	ASSERT(length == 4096);
-	ASSERT(memcmp(readData.get(), data.get(), 4096) == 0);
-	// close the file
-	f.clear();
-
-	// clean up
-	wait(IAsyncFileSystem::filesystem()->deleteFile(renamedFile, true));
-	return Void();
-}
-
-// Truncating to extend size should zero the new data
-TEST_CASE("/fileio/truncateAndRead") {
-	state std::string filename = "/tmp/__JUNK__";
-	state Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(
-	    filename, IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0));
-	state std::array<char, 4096> data;
-	wait(f->sync());
-	wait(f->truncate(4096));
-	int length = wait(f->read(&data[0], 4096, 0));
-	ASSERT(length == 4096);
-	for (auto c : data) {
-		ASSERT(c == '\0');
-	}
-	// close the file by deleting the reference
-	f.clear();
-	wait(IAsyncFileSystem::filesystem()->incrementalDeleteFile(filename, true));
-	return Void();
-}
diff --git a/flow/IAsyncFile.cpp b/flow/IAsyncFile.cpp
new file mode 100644
index 00000000000..3829647e00e
--- /dev/null
+++ b/flow/IAsyncFile.cpp
@@ -0,0 +1,81 @@
+/*
+ * IAsyncFile.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/IAsyncFile.h"
+#include "flow/Knobs.h"
+#include "flow/Platform.h"
+
+IAsyncFile::~IAsyncFile() = default;
+
+const static unsigned int ONE_MEGABYTE = 1 << 20;
+
+static Future<Void> zeroRangeHelper(Reference<IAsyncFile> f, int64_t offset, int64_t length, int fixedbyte) {
+	int64_t pos = offset;
+	void* zeros = aligned_alloc(ONE_MEGABYTE, ONE_MEGABYTE);
+	memset(zeros, fixedbyte, ONE_MEGABYTE);
+
+	while (pos < offset + length) {
+		int len = std::min<int64_t>(ONE_MEGABYTE, offset + length - pos);
+		co_await f->write(zeros, len, pos);
+		pos += len;
+		co_await yield();
+	}
+
+	aligned_free(zeros);
+}
+
+Future<Void> IAsyncFile::zeroRange(int64_t offset, int64_t length) {
+	return uncancellable(zeroRangeHelper(Reference<IAsyncFile>::addRef(this), offset, length, 0));
+}
+
+static Future<Void> incrementalDeleteHelper(std::string filename,
+                                            bool mustBeDurable,
+                                            int64_t truncateAmt,
+                                            double interval) {
+	Reference<IAsyncFile> file;
+	int64_t remainingFileSize{ 0 };
+	bool exists = fileExists(filename);
+
+	if (exists) {
+		Reference<IAsyncFile> f = co_await IAsyncFileSystem::filesystem()->open(
+		    filename, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, 0);
+		file = f;
+
+		int64_t fileSize = co_await file->size();
+		remainingFileSize = fileSize;
+	}
+
+	co_await IAsyncFileSystem::filesystem()->deleteFile(filename, mustBeDurable);
+
+	if (exists) {
+		for (; remainingFileSize > 0; remainingFileSize -= truncateAmt) {
+			co_await file->truncate(remainingFileSize);
+			co_await file->sync();
+			co_await delay(interval);
+		}
+	}
+}
+
+Future<Void> IAsyncFileSystem::incrementalDeleteFile(const std::string& filename, bool mustBeDurable) {
+	return uncancellable(incrementalDeleteHelper(filename,
+	                                             mustBeDurable,
+	                                             FLOW_KNOBS->INCREMENTAL_DELETE_TRUNCATE_AMOUNT,
+	                                             FLOW_KNOBS->INCREMENTAL_DELETE_INTERVAL));
+}
diff --git a/flow/IThreadPool.cpp b/flow/IThreadPool.cpp
index 733cc0db68e..446fbf0b778 100644
--- a/flow/IThreadPool.cpp
+++ b/flow/IThreadPool.cpp
@@ -20,7 +20,6 @@
 
 #include "flow/IThreadPool.h"
 
-#include <algorithm>
 // The ifndef's allow us to compile with pre-built boost.  Otherwise, we get
 // errors about double-defines.  As of this writing, the automatically downloaded
 // build of boost doesn't define these, but the pre-built version does.  (The old
@@ -76,10 +75,12 @@ class ThreadPool final : public IThreadPool, public ReferenceCounted<ThreadPool>
 
 	struct ActionWrapper {
 		PThreadAction action;
-		ActionWrapper(PThreadAction action) : action(action) {}
+		explicit ActionWrapper(PThreadAction action) : action(action) {}
 		// HACK: Boost won't use move constructors, so we just assume the last copy made is the one that will be called
 		// or cancelled
-		ActionWrapper(ActionWrapper const& r) : action(r.action) { const_cast<ActionWrapper&>(r).action = nullptr; }
+		explicit(false) ActionWrapper(ActionWrapper const& r) : action(r.action) {
+			const_cast<ActionWrapper&>(r).action = nullptr;
+		}
 		void operator()() {
 			Thread::dispatch(action);
 			action = nullptr;
@@ -94,7 +95,7 @@ class ThreadPool final : public IThreadPool, public ReferenceCounted<ThreadPool>
 
 public:
 	ThreadPool(int stackSize, int pri) : dontstop(ios), mode(Run), stackSize(stackSize), pri(pri) {}
-	~ThreadPool() override {}
+	~ThreadPool() override = default;
 	Future<Void> stop(Error const& e = success()) override {
 		if (mode == Shutdown)
 			return Void();
@@ -137,7 +138,7 @@ class ThreadPool final : public IThreadPool, public ReferenceCounted<ThreadPool>
 };
 
 Reference<IThreadPool> createGenericThreadPool(int stackSize, int pri) {
-	return Reference<IThreadPool>(new ThreadPool(stackSize, pri));
+	return makeReference<ThreadPool>(stackSize, pri);
 }
 
 thread_local IThreadPoolReceiver* ThreadPool::Thread::threadUserObject;
diff --git a/flow/IThreadPoolTest.actor.cpp b/flow/IThreadPoolTest.cpp
similarity index 68%
rename from flow/IThreadPoolTest.actor.cpp
rename to flow/IThreadPoolTest.cpp
index 47f67707a68..0f7a1a3ea34 100644
--- a/flow/IThreadPoolTest.actor.cpp
+++ b/flow/IThreadPoolTest.cpp
@@ -1,5 +1,5 @@
 /*
- * IThreadPoolTest.actor.cpp
+ * IThreadPoolTest.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,8 +26,8 @@
 #include <pthread.h>
 #include <iostream>
 
+#include "flow/Coroutines.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // has to be last include
 
 void forceLinkIThreadPoolTests() {}
 
@@ -57,33 +57,43 @@ struct ThreadNameReceiver final : IThreadPoolReceiver {
 	}
 };
 
-TEST_CASE("/flow/IThreadPool/NamedThread") {
-	noUnseed = true;
-
-	state Reference<IThreadPool> pool = createGenericThreadPool();
-	pool->addThread(new ThreadNameReceiver(), "thread-foo");
-
-	// Warning: this action is a little racy with the call to `pthread_setname_np`. In practice,
-	// ~nothing should depend on the thread name being set instantaneously. If this test ever
-	// flakes, we can make `startThread` in platform a little bit more complex to clearly order
-	// the actions.
+Future<std::string> getThreadName(Reference<IThreadPool> pool) {
 	auto* a = new ThreadNameReceiver::GetNameAction();
 	auto fut = a->name.getFuture();
 	pool->post(a);
+	return fut;
+}
 
-	std::string name = wait(fut);
-	if (name != "thread-foo") {
-		std::cout << "Incorrect thread name: " << name << std::endl;
-		ASSERT(false);
+Future<bool> waitForThreadName(Reference<IThreadPool> pool, std::string const& expectedName) {
+	// startThread() sets the pthread name from the creating thread after pthread_create(), so the worker may
+	// briefly report its default name before the requested name is visible. Some environments also report
+	// ENOENT from pthread_setname_np(), which startThread() treats as non-fatal.
+	double deadline = now() + 5.0;
+	std::string lastName;
+	while (now() < deadline) {
+		lastName = co_await getThreadName(pool);
+		if (lastName == expectedName) {
+			co_return true;
+		}
+		co_await delay(0.01);
 	}
+	std::cout << "Thread name was not set; last observed thread name: " << lastName << std::endl;
+	co_return false;
+}
+
+TEST_CASE("/flow/IThreadPool/NamedThread") {
+	noUnseed = true;
+
+	Reference<IThreadPool> pool = createGenericThreadPool();
+	pool->addThread(new ThreadNameReceiver(), "thread-foo");
 
-	wait(pool->stop());
+	ASSERT(co_await waitForThreadName(pool, "thread-foo"));
 
-	return Void();
+	co_await pool->stop();
 }
 
 struct ThreadSafePromiseStreamSender final : IThreadPoolReceiver {
-	ThreadSafePromiseStreamSender(ThreadReturnPromiseStream<std::string>* notifications)
+	explicit ThreadSafePromiseStreamSender(ThreadReturnPromiseStream<std::string>* notifications)
 	  : notifications(notifications) {}
 	void init() override {}
 
@@ -117,28 +127,39 @@ struct ThreadSafePromiseStreamSender final : IThreadPoolReceiver {
 TEST_CASE("/flow/IThreadPool/ThreadReturnPromiseStream") {
 	noUnseed = true;
 
-	state std::unique_ptr<ThreadReturnPromiseStream<std::string>> notifications(
-	    new ThreadReturnPromiseStream<std::string>());
+	std::unique_ptr<ThreadReturnPromiseStream<std::string>> notifications(new ThreadReturnPromiseStream<std::string>());
 
-	state Reference<IThreadPool> pool = createGenericThreadPool();
+	Reference<IThreadPool> pool = createGenericThreadPool();
 	pool->addThread(new ThreadSafePromiseStreamSender(notifications.get()), "thread-foo");
 
-	// Warning: this action is a little racy with the call to `pthread_setname_np`. In practice,
-	// ~nothing should depend on the thread name being set instantaneously. If this test ever
-	// flakes, we can make `startThread` in platform a little bit more complex to clearly order
-	// the actions.
-	state int num = 3;
-	for (int i = 0; i < num; ++i) {
+	ThreadFutureStream<std::string> futs = notifications->getFuture();
+	bool threadNameAvailable = false;
+	double deadline = now() + 5.0;
+	std::string lastName;
+	while (now() < deadline) {
 		auto* a = new ThreadSafePromiseStreamSender::GetNameAction();
 		pool->post(a);
+		lastName = co_await futs;
+		if (lastName == "thread-foo") {
+			threadNameAvailable = true;
+			break;
+		}
+		co_await delay(0.01);
+	}
+	if (!threadNameAvailable) {
+		std::cout << "Thread name was not set; last observed thread name: " << lastName << std::endl;
 	}
 
-	state ThreadFutureStream<std::string> futs = notifications->getFuture();
+	int num = 3;
+	for (int i = 0; i < num; ++i) {
+		auto* a = new ThreadSafePromiseStreamSender::GetNameAction();
+		pool->post(a);
+	}
 
-	state int n = 0;
+	int n = 0;
 	while (n < num) {
-		std::string name = waitNext(futs);
-		if (name != "thread-foo") {
+		std::string name = co_await futs;
+		if (threadNameAvailable && name != "thread-foo") {
 			std::cout << "Incorrect thread name: " << name << std::endl;
 			ASSERT(false);
 		}
@@ -151,15 +172,13 @@ TEST_CASE("/flow/IThreadPool/ThreadReturnPromiseStream") {
 	pool->post(faultyAction);
 
 	try {
-		std::string name = waitNext(futs);
+		std::string name = co_await futs;
 		ASSERT(false);
 	} catch (Error& e) {
 		ASSERT(e.isInjectedFault());
 	}
 
-	wait(pool->stop());
-
-	return Void();
+	co_await pool->stop();
 }
 
 struct MockReceiver : public IThreadPoolReceiver {
@@ -194,24 +213,22 @@ Reference<IThreadPool> initTestPool() {
 TEST_CASE("/flow/IThreadPool/ExplicitStop") {
 	noUnseed = true;
 
-	state Reference<IThreadPool> pool = initTestPool();
+	Reference<IThreadPool> pool = initTestPool();
 	auto task = new MockTask();
 	auto future = task->promise.getFuture();
 	pool->post(task);
-	wait(future);
-	wait(pool->stop());
-	return Void();
+	co_await future;
+	co_await pool->stop();
 }
 
 TEST_CASE("/flow/IThreadPool/ImplicitStop") {
 	noUnseed = true;
 
-	state Reference<IThreadPool> pool = initTestPool();
+	Reference<IThreadPool> pool = initTestPool();
 	auto task = new MockTask();
 	auto future = task->promise.getFuture();
 	pool->post(task);
-	wait(future);
-	return Void();
+	co_await future;
 }
 
 #else
diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp
index 6d38c0f8fa2..628ee8beb3c 100644
--- a/flow/IndexedSet.cpp
+++ b/flow/IndexedSet.cpp
@@ -396,7 +396,7 @@ TEST_CASE("/flow/IndexedSets/ints") {
 	ASSERT(is.find(20) != is.end());
 
 	for (int i = 20; i < 200; i += 10) {
-		is.insert(std::move(i), 3);
+		is.insert(i, 3);
 		is.testonly_assertBalanced();
 		ASSERT(is.find(i) != is.end());
 	}
@@ -429,9 +429,9 @@ TEST_CASE("/flow/IndexedSet/data constructor and destructor calls match") {
 	count = 0;
 	struct Counter {
 		int value;
-		Counter(int value) : value(value) { count++; }
+		explicit Counter(int value) : value(value) { count++; }
 		~Counter() { count--; }
-		Counter(const Counter& r) : value(r.value) { count++; }
+		explicit(false) Counter(const Counter& r) : value(r.value) { count++; }
 		void operator=(const Counter& r) { value = r.value; }
 		int compare(const Counter& r) const { return ::compare(value, r.value); }
 		bool operator<(const Counter& r) const { return value < r.value; }
@@ -455,7 +455,7 @@ TEST_CASE("/flow/IndexedSet/comparison to std::set") {
 	std::set<int> ss;
 	for (int i = 0; i < 1000000; i++) {
 		int p = deterministicRandom()->randomInt(0, 2000000);
-		is.insert(std::move(p), 1);
+		is.insert(p, 1);
 		ss.insert(p);
 	}
 
diff --git a/flow/JsonTraceLogFormatter.cpp b/flow/JsonTraceLogFormatter.cpp
index 3e5239b5676..2647fd89a47 100644
--- a/flow/JsonTraceLogFormatter.cpp
+++ b/flow/JsonTraceLogFormatter.cpp
@@ -18,8 +18,7 @@
  * limitations under the License.
  */
 
-#include "flow/flow.h"
-#include "flow/JsonTraceLogFormatter.h"
+#include "JsonTraceLogFormatter.h"
 
 #include <sstream>
 
diff --git a/flow/include/flow/JsonTraceLogFormatter.h b/flow/JsonTraceLogFormatter.h
similarity index 100%
rename from flow/include/flow/JsonTraceLogFormatter.h
rename to flow/JsonTraceLogFormatter.h
diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 35a8e1ed41a..9cc88258523 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -49,6 +49,8 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( HOSTNAME_RECONNECT_INIT_INTERVAL,                    .05 );
 	init( HOSTNAME_RECONNECT_MAX_INTERVAL,                     1.0 );
 	init( ENABLE_COORDINATOR_DNS_CACHE,                      false ); if( randomize && BUGGIFY ) ENABLE_COORDINATOR_DNS_CACHE = true;
+	init( COORDINATOR_DNS_CACHE_REFRESH_INTERVAL,             3.0 );
+	init( COORDINATOR_DNS_CACHE_TTL,                         30.0 );
 	init( CACHE_REFRESH_INTERVAL_WHEN_ALL_ALTERNATIVES_FAILED, 1.0 );
 
 	init( DELAY_JITTER_OFFSET,                                 0.9 );
@@ -159,7 +161,6 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( SIM_PAGE_CACHE_64K,                                  1e7 );
 	init( BUGGIFY_SIM_PAGE_CACHE_4K,                           1e6 );
 	init( BUGGIFY_SIM_PAGE_CACHE_64K,                          1e6 );
-	init( BLOB_WORKER_PAGE_CACHE,                            500e6 );
 	init( MAX_EVICT_ATTEMPTS,                                  100 ); if( randomize && BUGGIFY ) MAX_EVICT_ATTEMPTS = 2;
 	init( CACHE_EVICTION_POLICY,                          "random" );
 	init( PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION,                 0.1 ); if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 0.0; else if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 1.0;
@@ -173,10 +174,6 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( EIO_MAX_PARALLELISM,                                  4  );
 	init( EIO_USE_ODIRECT,                                      0  );
 
-	//AsyncFileEncrypted
-	init( ENCRYPTION_BLOCK_SIZE,                              4096 );
-	init( MAX_DECRYPTED_BLOCKS,                                 10 );
-
 	//AsyncFileKAIO
 	init( MAX_OUTSTANDING,                                      64 );
 	init( MIN_SUBMIT,                                           10 );
@@ -409,15 +406,15 @@ static bool safe_stob(std::string const& str) {
 // invalid_option_value exception.
 ParsedKnobValue Knobs::parseKnobValue(std::string const& knob, std::string const& value) const {
 	try {
-		if (double_knobs.count(knob)) {
+		if (double_knobs.contains(knob)) {
 			return safe_stod(value);
-		} else if (bool_knobs.count(knob)) {
+		} else if (bool_knobs.contains(knob)) {
 			return safe_stob(value);
-		} else if (int64_knobs.count(knob)) {
+		} else if (int64_knobs.contains(knob)) {
 			return safe_stoi64(value);
-		} else if (int_knobs.count(knob)) {
+		} else if (int_knobs.contains(knob)) {
 			return safe_stoi(value);
-		} else if (string_knobs.count(knob)) {
+		} else if (string_knobs.contains(knob)) {
 			return value;
 		}
 		return NoKnobFound{};
@@ -427,7 +424,7 @@ ParsedKnobValue Knobs::parseKnobValue(std::string const& knob, std::string const
 }
 
 bool Knobs::setKnob(std::string const& knob, int value) {
-	if (!int_knobs.count(knob)) {
+	if (!int_knobs.contains(knob)) {
 		return false;
 	}
 	*int_knobs[knob].value = value;
@@ -436,7 +433,7 @@ bool Knobs::setKnob(std::string const& knob, int value) {
 }
 
 bool Knobs::setKnob(std::string const& knob, int64_t value) {
-	if (!int64_knobs.count(knob)) {
+	if (!int64_knobs.contains(knob)) {
 		return false;
 	}
 	*int64_knobs[knob].value = value;
@@ -445,7 +442,7 @@ bool Knobs::setKnob(std::string const& knob, int64_t value) {
 }
 
 bool Knobs::setKnob(std::string const& knob, bool value) {
-	if (!bool_knobs.count(knob)) {
+	if (!bool_knobs.contains(knob)) {
 		return false;
 	}
 	*bool_knobs[knob].value = value;
@@ -454,7 +451,7 @@ bool Knobs::setKnob(std::string const& knob, bool value) {
 }
 
 bool Knobs::setKnob(std::string const& knob, double value) {
-	if (!double_knobs.count(knob)) {
+	if (!double_knobs.contains(knob)) {
 		return false;
 	}
 	*double_knobs[knob].value = value;
@@ -463,7 +460,7 @@ bool Knobs::setKnob(std::string const& knob, double value) {
 }
 
 bool Knobs::setKnob(std::string const& knob, std::string const& value) {
-	if (!string_knobs.count(knob)) {
+	if (!string_knobs.contains(knob)) {
 		return false;
 	}
 	*string_knobs[knob].value = value;
@@ -472,19 +469,19 @@ bool Knobs::setKnob(std::string const& knob, std::string const& value) {
 }
 
 ParsedKnobValue Knobs::getKnob(const std::string& name) const {
-	if (double_knobs.count(name) > 0) {
+	if (double_knobs.contains(name)) {
 		return ParsedKnobValue{ *double_knobs.at(name).value };
 	}
-	if (int64_knobs.count(name) > 0) {
+	if (int64_knobs.contains(name)) {
 		return ParsedKnobValue{ *int64_knobs.at(name).value };
 	}
-	if (int_knobs.count(name) > 0) {
+	if (int_knobs.contains(name)) {
 		return ParsedKnobValue{ *int_knobs.at(name).value };
 	}
-	if (string_knobs.count(name) > 0) {
+	if (string_knobs.contains(name)) {
 		return ParsedKnobValue{ *string_knobs.at(name).value };
 	}
-	if (bool_knobs.count(name) > 0) {
+	if (bool_knobs.contains(name)) {
 		return ParsedKnobValue{ *bool_knobs.at(name).value };
 	}
 
@@ -492,35 +489,35 @@ ParsedKnobValue Knobs::getKnob(const std::string& name) const {
 }
 
 void Knobs::initKnob(double& knob, double value, std::string const& name) {
-	if (!explicitlySetKnobs.count(toLower(name))) {
+	if (!explicitlySetKnobs.contains(toLower(name))) {
 		knob = value;
 		double_knobs[toLower(name)] = KnobValue<double>{ &knob };
 	}
 }
 
 void Knobs::initKnob(int64_t& knob, int64_t value, std::string const& name) {
-	if (!explicitlySetKnobs.count(toLower(name))) {
+	if (!explicitlySetKnobs.contains(toLower(name))) {
 		knob = value;
 		int64_knobs[toLower(name)] = KnobValue<int64_t>{ &knob };
 	}
 }
 
 void Knobs::initKnob(int& knob, int value, std::string const& name) {
-	if (!explicitlySetKnobs.count(toLower(name))) {
+	if (!explicitlySetKnobs.contains(toLower(name))) {
 		knob = value;
 		int_knobs[toLower(name)] = KnobValue<int>{ &knob };
 	}
 }
 
 void Knobs::initKnob(std::string& knob, const std::string& value, const std::string& name) {
-	if (!explicitlySetKnobs.count(toLower(name))) {
+	if (!explicitlySetKnobs.contains(toLower(name))) {
 		knob = value;
 		string_knobs[toLower(name)] = KnobValue<std::string>{ &knob };
 	}
 }
 
 void Knobs::initKnob(bool& knob, bool value, std::string const& name) {
-	if (!explicitlySetKnobs.count(toLower(name))) {
+	if (!explicitlySetKnobs.contains(toLower(name))) {
 		knob = value;
 		bool_knobs[toLower(name)] = KnobValue<bool>{ &knob };
 	}
diff --git a/flow/include/flow/MetricSample.h b/flow/MetricSample.h
similarity index 100%
rename from flow/include/flow/MetricSample.h
rename to flow/MetricSample.h
diff --git a/flow/MkCert.cpp b/flow/MkCert.cpp
index b4f971aefa8..8a2bbad7cf4 100644
--- a/flow/MkCert.cpp
+++ b/flow/MkCert.cpp
@@ -26,7 +26,6 @@
 #include "flow/ScopeExit.h"
 #include "flow/Trace.h"
 
-#include <limits>
 #include <memory>
 #include <string>
 #include <cstring>
diff --git a/flow/MkCertCli.cpp b/flow/MkCertCli.cpp
index 0b2f7c31858..02b3f615910 100644
--- a/flow/MkCertCli.cpp
+++ b/flow/MkCertCli.cpp
@@ -31,7 +31,7 @@
 #include "flow/Platform.h"
 #include "flow/ScopeExit.h"
 #include "SimpleOpt/SimpleOpt.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/Trace.h"
 
 enum EMkCertOpt : int {
diff --git a/flow/Net2.actor.cpp b/flow/Net2.cpp
similarity index 87%
rename from flow/Net2.actor.cpp
rename to flow/Net2.cpp
index b92e24e5314..7252abaa3fd 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.cpp
@@ -1,5 +1,5 @@
 /*
- * Net2.actor.cpp
+ * Net2.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -24,6 +24,7 @@
 #include "flow/Arena.h"
 #include "flow/Knobs.h"
 #include "flow/Platform.h"
+#include "flow/SimpleCounter.h"
 #include "flow/Trace.h"
 #include "flow/swift.h"
 #include "flow/swift_concurrency_hooks.h"
@@ -52,13 +53,13 @@
 #include "flow/TaskQueue.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/ChaosMetrics.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/AsioReactor.h"
 #include "flow/Profiler.h"
 #include "flow/ProtocolVersion.h"
 #include "flow/SendBufferIterator.h"
-#include "flow/TLSConfig.actor.h"
-#include "flow/WatchFile.actor.h"
+#include "flow/TLSConfig.h"
+#include "flow/WatchFile.h"
 #include "flow/genericactors.actor.h"
 #include "flow/Util.h"
 #include "flow/UnitTest.h"
@@ -73,7 +74,7 @@
 #ifdef WIN32
 #include <mmsystem.h>
 #endif
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include "flow/CoroUtils.h"
 
 // Defined to track the stack limit
 extern "C" intptr_t g_stackYieldLimit;
@@ -109,13 +110,22 @@ void initProfiling() {
 }
 #endif
 
-DESCR struct SlowTask {
-	int64_t clocks; // clocks
-	int64_t duration; // ns
-	int64_t priority; // priority level
-	int64_t numYields; // count
+struct SlowTaskDescriptor {
+	int64_t clocks;
+	int64_t duration;
+	int64_t priority;
+	int64_t numYields;
 };
 
+template <>
+struct Descriptor<SlowTaskDescriptor>
+  : DescribeType<SlowTaskDescriptor,
+                 "SlowTask",
+                 DescribeField<&SlowTaskDescriptor::clocks, "clocks", "clocks">,
+                 DescribeField<&SlowTaskDescriptor::duration, "duration", "ns">,
+                 DescribeField<&SlowTaskDescriptor::priority, "priority", "priority level">,
+                 DescribeField<&SlowTaskDescriptor::numYields, "numYields", "count">> {};
+
 namespace N2 { // No indent, it's the whole file
 
 class Peer;
@@ -236,6 +246,7 @@ class Net2 final : public INetwork, public INetworkConnections {
 	TLSConfig tlsConfig;
 	Reference<TLSPolicy> activeTlsPolicy;
 	Future<Void> backgroundCertRefresh;
+	Future<Void> dnsCacheRefreshActor;
 	ETLSInitState tlsInitializedState;
 
 	INetworkConnections* network; // initially this, but can be changed
@@ -266,7 +277,7 @@ class Net2 final : public INetwork, public INetworkConnections {
 	struct PromiseTask final : public FastAllocated<PromiseTask> {
 		Promise<Void> promise;
 		swift::Job* _Nullable swiftJob = nullptr;
-		PromiseTask() {}
+		PromiseTask() = default;
 		explicit PromiseTask(Promise<Void>&& promise) noexcept : promise(std::move(promise)) {}
 		explicit PromiseTask(swift::Job* swiftJob) : swiftJob(swiftJob) {}
 
@@ -324,7 +335,7 @@ class Net2 final : public INetwork, public INetworkConnections {
 	DoubleMetricHandle countReactTime;
 	BoolMetricHandle awakeMetric;
 
-	EventMetricHandle<SlowTask> slowTaskMetric;
+	EventMetricHandle<SlowTaskDescriptor> slowTaskMetric;
 
 	std::vector<std::string> blobCredentialFiles;
 	Optional<std::string> proxy;
@@ -362,18 +373,18 @@ class BindPromise {
 	NetworkAddress peerAddr;
 
 public:
-	BindPromise(const char* errContext, UID errID) : errContext(errContext), errID(errID) {}
-	BindPromise(AuditedEvent auditedEvent, UID errID) : errContext(auditedEvent), errID(errID) {}
-	BindPromise(BindPromise const& r) : p(r.p), errContext(r.errContext), errID(r.errID), peerAddr(r.peerAddr) {}
-	BindPromise(BindPromise&& r) noexcept
+	BindPromise(const char* errContext, UID errID, NetworkAddress peerAddr)
+	  : errContext(errContext), errID(errID), peerAddr(peerAddr) {}
+	BindPromise(AuditedEvent auditedEvent, UID errID, NetworkAddress peerAddr)
+	  : errContext(auditedEvent), errID(errID), peerAddr(peerAddr) {}
+	explicit(false) BindPromise(BindPromise const& r) = default;
+	explicit(false) BindPromise(BindPromise&& r) noexcept
 	  : p(std::move(r.p)), errContext(r.errContext), errID(r.errID), peerAddr(r.peerAddr) {}
 
 	Future<Void> getFuture() const { return p.getFuture(); }
 
 	NetworkAddress getPeerAddr() const { return peerAddr; }
 
-	void setPeerAddr(const NetworkAddress& addr) { peerAddr = addr; }
-
 	void operator()(const boost::system::error_code& error, size_t bytesWritten = 0) {
 		try {
 			if (error) {
@@ -424,19 +435,19 @@ class Connection final : public IConnection, ReferenceCounted<Connection> {
 	  : id(nondeterministicRandom()->randomUniqueID()), socket(io_service) {}
 
 	// This is not part of the IConnection interface, because it is wrapped by INetwork::connect()
-	ACTOR static Future<Reference<IConnection>> connect(boost::asio::io_service* ios, NetworkAddress addr) {
-		state Reference<Connection> self(new Connection(*ios));
+	static Future<Reference<IConnection>> connect(boost::asio::io_service* ios, NetworkAddress addr) {
+		Reference<Connection> self(new Connection(*ios));
 
 		self->peer_address = addr;
 		try {
 			auto to = tcpEndpoint(addr);
-			BindPromise p("N2_ConnectError", self->id);
+			BindPromise p("N2_ConnectError", self->id, self->peer_address);
 			Future<Void> onConnected = p.getFuture();
 			self->socket.async_connect(to, std::move(p));
 
-			wait(onConnected);
+			co_await onConnected;
 			self->init();
-			return self;
+			co_return self;
 		} catch (Error&) {
 			// Either the connection failed, or was cancelled by the caller
 			self->closeSocket();
@@ -457,7 +468,7 @@ class Connection final : public IConnection, ReferenceCounted<Connection> {
 	// returns when write() can write at least one byte
 	Future<Void> onWritable() override {
 		++g_net2->countWriteProbes;
-		BindPromise p("N2_WriteProbeError", id);
+		BindPromise p("N2_WriteProbeError", id, peer_address);
 		auto f = p.getFuture();
 		socket.async_write_some(boost::asio::null_buffers(), std::move(p));
 		return f;
@@ -466,7 +477,7 @@ class Connection final : public IConnection, ReferenceCounted<Connection> {
 	// returns when read() can read at least one byte
 	Future<Void> onReadable() override {
 		++g_net2->countReadProbes;
-		BindPromise p("N2_ReadProbeError", id);
+		BindPromise p("N2_ReadProbeError", id, peer_address);
 		auto f = p.getFuture();
 		socket.async_read_some(boost::asio::null_buffers(), std::move(p));
 		return f;
@@ -597,8 +608,9 @@ class ReadPromise {
 
 public:
 	ReadPromise(const char* errContext, UID errID) : errContext(errContext), errID(errID) {}
-	ReadPromise(ReadPromise const& other) = default;
-	ReadPromise(ReadPromise&& other) : p(std::move(other.p)), errContext(other.errContext), errID(other.errID) {}
+	explicit(false) ReadPromise(ReadPromise const& other) = default;
+	explicit(false) ReadPromise(ReadPromise&& other)
+	  : p(std::move(other.p)), errContext(other.errContext), errID(other.errID) {}
 
 	std::shared_ptr<udp::endpoint>& getEndpoint() { return endpoint; }
 
@@ -627,25 +639,25 @@ class UDPSocket : public IUDPSocket, ReferenceCounted<UDPSocket> {
 	bool isPublic = false;
 
 public:
-	ACTOR static Future<Reference<IUDPSocket>> connect(boost::asio::io_service* io_service,
-	                                                   Optional<NetworkAddress> toAddress,
-	                                                   bool isV6) {
-		state Reference<UDPSocket> self(new UDPSocket(*io_service, toAddress, isV6));
+	static Future<Reference<IUDPSocket>> connect(boost::asio::io_service* io_service,
+	                                             Optional<NetworkAddress> toAddress,
+	                                             bool isV6) {
+		Reference<UDPSocket> self(new UDPSocket(*io_service, toAddress, isV6));
 		ASSERT(!toAddress.present() || toAddress.get().ip.isV6() == isV6);
 		if (!toAddress.present()) {
-			return self;
+			co_return self;
 		}
 		try {
 			if (toAddress.present()) {
 				auto to = udpEndpoint(toAddress.get());
-				BindPromise p("N2_UDPConnectError", self->id);
+				BindPromise p("N2_UDPConnectError", self->id, toAddress.get());
 				Future<Void> onConnected = p.getFuture();
 				self->socket.async_connect(to, std::move(p));
 
-				wait(onConnected);
+				co_await onConnected;
 			}
 			self->init();
-			return self;
+			co_return self;
 		} catch (...) {
 			self->closeSocket();
 			throw;
@@ -794,19 +806,20 @@ class Listener final : public IListener, ReferenceCounted<Listener> {
 	NetworkAddress getListenAddress() const override { return listenAddress; }
 
 private:
-	ACTOR static Future<Reference<IConnection>> doAccept(Listener* self) {
-		state Reference<Connection> conn(new Connection(self->io_service));
-		state tcp::acceptor::endpoint_type peer_endpoint;
+	static Future<Reference<IConnection>> doAccept(Listener* self) {
+		Reference<Connection> conn(new Connection(self->io_service));
+		tcp::acceptor::endpoint_type peer_endpoint;
 		try {
-			BindPromise p("N2_AcceptError", UID());
+			// Peer address not known until accept succeeds
+			BindPromise p("N2_AcceptError", UID(), NetworkAddress());
 			auto f = p.getFuture();
 			self->acceptor.async_accept(conn->getSocket(), peer_endpoint, std::move(p));
-			wait(f);
+			co_await f;
 			auto peer_address = peer_endpoint.address().is_v6() ? IPAddress(peer_endpoint.address().to_v6().to_bytes())
 			                                                    : IPAddress(peer_endpoint.address().to_v4().to_ulong());
 			conn->accept(NetworkAddress(peer_address, peer_endpoint.port()));
 
-			return conn;
+			co_return conn;
 		} catch (...) {
 			conn->close();
 			throw;
@@ -814,10 +827,10 @@ class Listener final : public IListener, ReferenceCounted<Listener> {
 	}
 };
 
-typedef boost::asio::ssl::stream<boost::asio::ip::tcp::socket&> ssl_socket;
+using ssl_socket = boost::asio::ssl::stream<boost::asio::ip::tcp::socket&>;
 
 struct SSLHandshakerThread final : IThreadPoolReceiver {
-	SSLHandshakerThread() {}
+	SSLHandshakerThread() = default;
 	void init() override {}
 
 	struct Handshake final : TypedAction<SSLHandshakerThread, Handshake> {
@@ -896,17 +909,17 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 	    ssl_sock(socket, context->mutate()), sslContext(context) {}
 
 	// This is not part of the IConnection interface, because it is wrapped by INetwork::connect()
-	ACTOR static Future<Reference<IConnection>> connect(boost::asio::io_service* ios,
-	                                                    Reference<ReferencedObject<boost::asio::ssl::context>> context,
-	                                                    NetworkAddress addr,
-	                                                    tcp::socket* existingSocket = nullptr) {
+	static Future<Reference<IConnection>> connect(boost::asio::io_service* ios,
+	                                              Reference<ReferencedObject<boost::asio::ssl::context>> context,
+	                                              NetworkAddress addr,
+	                                              tcp::socket* existingSocket = nullptr) {
 		std::pair<IPAddress, uint16_t> peerIP = std::make_pair(addr.ip, addr.port);
 		auto iter(g_network->networkInfo.serverTLSConnectionThrottler.find(peerIP));
 		if (iter != g_network->networkInfo.serverTLSConnectionThrottler.end()) {
 			if (now() < iter->second.second) {
 				if (iter->second.first >= FLOW_KNOBS->TLS_CLIENT_CONNECTION_THROTTLE_ATTEMPTS) {
 					TraceEvent("TLSOutgoingConnectionThrottlingWarning").suppressFor(1.0).detail("PeerIP", addr);
-					wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
+					co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
 					static SimpleCounter<int64_t>* countClientTLSHandshakeThrottled =
 					    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakeThrottled");
 					countClientTLSHandshakeThrottled->increment(1);
@@ -921,20 +934,20 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 			Reference<SSLConnection> self(new SSLConnection(context, existingSocket));
 			self->peer_address = addr;
 			self->init();
-			return self;
+			co_return self;
 		}
 
-		state Reference<SSLConnection> self(new SSLConnection(*ios, context));
+		Reference<SSLConnection> self(new SSLConnection(*ios, context));
 		self->peer_address = addr;
 		try {
 			auto to = tcpEndpoint(self->peer_address);
-			BindPromise p("N2_ConnectError", self->id);
+			BindPromise p("N2_ConnectError", self->id, self->peer_address);
 			Future<Void> onConnected = p.getFuture();
 			self->socket.async_connect(to, std::move(p));
 
-			wait(onConnected);
+			co_await onConnected;
 			self->init();
-			return self;
+			co_return self;
 		} catch (Error&) {
 			// Either the connection failed, or was cancelled by the caller
 			self->closeSocket();
@@ -943,7 +956,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 	}
 
 	// Connect with hostname for SNI (Server Name Indication) support
-	ACTOR static Future<Reference<IConnection>> connectWithHostname(
+	static Future<Reference<IConnection>> connectWithHostname(
 	    boost::asio::io_service* ios,
 	    Reference<ReferencedObject<boost::asio::ssl::context>> context,
 	    NetworkAddress addr,
@@ -954,7 +967,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 			if (now() < iter->second.second) {
 				if (iter->second.first >= FLOW_KNOBS->TLS_CLIENT_CONNECTION_THROTTLE_ATTEMPTS) {
 					TraceEvent("TLSOutgoingConnectionThrottlingWarning").suppressFor(1.0).detail("PeerIP", addr);
-					wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
+					co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
 					throw connection_failed();
 				}
 			} else {
@@ -962,7 +975,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 			}
 		}
 
-		state Reference<SSLConnection> self(new SSLConnection(*ios, context));
+		Reference<SSLConnection> self(new SSLConnection(*ios, context));
 		self->peer_address = addr;
 		self->sni_hostname = hostname; // Store hostname for SNI during handshake
 
@@ -970,16 +983,16 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 
 		try {
 			auto to = tcpEndpoint(self->peer_address);
-			BindPromise p("N2_ConnectError", self->id);
+			BindPromise p("N2_ConnectError", self->id, self->peer_address);
 			Future<Void> onConnected = p.getFuture();
 			self->socket.async_connect(to, std::move(p));
 
-			wait(onConnected);
+			co_await onConnected;
 
 			// SNI will be set later in doConnectHandshake before SSL handshake
 
 			self->init();
-			return self;
+			co_return self;
 		} catch (Error&) {
 			// Either the connection failed, or was cancelled by the caller
 			self->closeSocket();
@@ -993,8 +1006,8 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 		init();
 	}
 
-	ACTOR static void doAcceptHandshake(Reference<SSLConnection> self, Promise<Void> connected) {
-		state Hold<int> holder;
+	static Future<Void> doAcceptHandshake(Uncancellable, Reference<SSLConnection> self, Promise<Void> connected) {
+		Hold<int> holder;
 
 		try {
 			Future<Void> onHandshook;
@@ -1021,13 +1034,12 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 				static SimpleCounter<int64_t>* countServerTLSHandshakesOnMainThread =
 				    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakesOnMainThread");
 				countServerTLSHandshakesOnMainThread->increment(1);
-				BindPromise p("N2_AcceptHandshakeError"_audit, self->id);
-				p.setPeerAddr(self->getPeerAddress());
+				BindPromise p("N2_AcceptHandshakeError"_audit, self->id, self->getPeerAddress());
 				onHandshook = p.getFuture();
 				self->ssl_sock.async_handshake(boost::asio::ssl::stream_base::server, std::move(p));
 			}
-			wait(onHandshook);
-			wait(delay(0, TaskPriority::Handshake));
+			co_await onHandshook;
+			co_await delay(0, TaskPriority::Handshake);
 			connected.send(Void());
 		} catch (...) {
 			self->closeSocket();
@@ -1035,8 +1047,8 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 		}
 	}
 
-	ACTOR static Future<Void> acceptHandshakeWrapper(Reference<SSLConnection> self) {
-		state std::pair<IPAddress, uint16_t> peerIP;
+	static Future<Void> acceptHandshakeWrapper(Reference<SSLConnection> self) {
+		std::pair<IPAddress, uint16_t> peerIP;
 		peerIP = std::make_pair(self->getPeerAddress().ip, static_cast<uint16_t>(0));
 		auto iter(g_network->networkInfo.serverTLSConnectionThrottler.find(peerIP));
 		if (iter != g_network->networkInfo.serverTLSConnectionThrottler.end()) {
@@ -1045,7 +1057,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 					TraceEvent("TLSIncomingConnectionThrottlingWarning")
 					    .suppressFor(1.0)
 					    .detail("PeerIP", peerIP.first.toString());
-					wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT));
+					co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
 					self->closeSocket();
 					static SimpleCounter<int64_t>* countServerTLSHandshakeThrottled =
 					    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakeThrottled");
@@ -1057,29 +1069,26 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 			}
 		}
 
-		wait(g_network->networkInfo.handshakeLock->take(
-		    getTaskPriorityFromInt(FLOW_KNOBS->TLS_HANDSHAKE_FLOWLOCK_PRIORITY)));
-		state FlowLock::Releaser releaser(*g_network->networkInfo.handshakeLock);
+		co_await g_network->networkInfo.handshakeLock->take(
+		    getTaskPriorityFromInt(FLOW_KNOBS->TLS_HANDSHAKE_FLOWLOCK_PRIORITY));
+		FlowLock::Releaser releaser(*g_network->networkInfo.handshakeLock);
 		static SimpleCounter<int64_t>* countServerTLSHandshakeLocked =
 		    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakeLocked");
 		countServerTLSHandshakeLocked->increment(1);
 
 		Promise<Void> connected;
-		doAcceptHandshake(self, connected);
+		doAcceptHandshake(Uncancellable(), self, connected);
 		try {
-			choose {
-				when(wait(connected.getFuture())) {
-					static SimpleCounter<int64_t>* countServerTLSHandshakesSucceed =
-					    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakesSucceed");
-					countServerTLSHandshakesSucceed->increment(1);
-					return Void();
-				}
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
-					static SimpleCounter<int64_t>* countServerTLSHandshakesTimedout =
-					    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakesTimedout");
-					countServerTLSHandshakesTimedout->increment(1);
-					throw connection_failed();
-				}
+			auto res = co_await timeout(connected.getFuture(), FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
+			if (res.present()) {
+				static SimpleCounter<int64_t>* countServerTLSHandshakesSucceed =
+				    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakesSucceed");
+				countServerTLSHandshakesSucceed->increment(1);
+			} else {
+				static SimpleCounter<int64_t>* countServerTLSHandshakesTimedout =
+				    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ServerTLSHandshakesTimedout");
+				countServerTLSHandshakesTimedout->increment(1);
+				throw connection_failed();
 			}
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled) {
@@ -1099,8 +1108,8 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 
 	Future<Void> acceptHandshake() override { return acceptHandshakeWrapper(Reference<SSLConnection>::addRef(this)); }
 
-	ACTOR static void doConnectHandshake(Reference<SSLConnection> self, Promise<Void> connected) {
-		state Hold<int> holder;
+	static Future<Void> doConnectHandshake(Uncancellable, Reference<SSLConnection> self, Promise<Void> connected) {
+		Hold<int> holder;
 
 		try {
 			Future<Void> onHandshook;
@@ -1146,13 +1155,12 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 				static SimpleCounter<int64_t>* countClientTLSHandshakesOnMainThread =
 				    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakesOnMainThread");
 				countClientTLSHandshakesOnMainThread->increment(1);
-				BindPromise p("N2_ConnectHandshakeError"_audit, self->id);
-				p.setPeerAddr(self->getPeerAddress());
+				BindPromise p("N2_ConnectHandshakeError"_audit, self->id, self->getPeerAddress());
 				onHandshook = p.getFuture();
 				self->ssl_sock.async_handshake(boost::asio::ssl::stream_base::client, std::move(p));
 			}
-			wait(onHandshook);
-			wait(delay(0, TaskPriority::Handshake));
+			co_await onHandshook;
+			co_await delay(0, TaskPriority::Handshake);
 			connected.send(Void());
 		} catch (...) {
 			self->closeSocket();
@@ -1160,30 +1168,27 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 		}
 	}
 
-	ACTOR static Future<Void> connectHandshakeWrapper(Reference<SSLConnection> self) {
-		wait(g_network->networkInfo.handshakeLock->take(
-		    getTaskPriorityFromInt(FLOW_KNOBS->TLS_HANDSHAKE_FLOWLOCK_PRIORITY)));
-		state FlowLock::Releaser releaser(*g_network->networkInfo.handshakeLock);
+	static Future<Void> connectHandshakeWrapper(Reference<SSLConnection> self) {
+		co_await g_network->networkInfo.handshakeLock->take(
+		    getTaskPriorityFromInt(FLOW_KNOBS->TLS_HANDSHAKE_FLOWLOCK_PRIORITY));
+		FlowLock::Releaser releaser(*g_network->networkInfo.handshakeLock);
 		static SimpleCounter<int64_t>* countClientTLSHandshakeLocked =
 		    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakeLocked");
 		countClientTLSHandshakeLocked->increment(1);
 
 		Promise<Void> connected;
-		doConnectHandshake(self, connected);
+		doConnectHandshake(Uncancellable(), self, connected);
 		try {
-			choose {
-				when(wait(connected.getFuture())) {
-					static SimpleCounter<int64_t>* countClientTLSHandshakesSucceed =
-					    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakesSucceed");
-					countClientTLSHandshakesSucceed->increment(1);
-					return Void();
-				}
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
-					static SimpleCounter<int64_t>* countClientTLSHandshakesTimedout =
-					    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakesTimedout");
-					countClientTLSHandshakesTimedout->increment(1);
-					throw connection_failed();
-				}
+			auto res = co_await timeout(connected.getFuture(), FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
+			if (res.present()) {
+				static SimpleCounter<int64_t>* countClientTLSHandshakesSucceed =
+				    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakesSucceed");
+				countClientTLSHandshakesSucceed->increment(1);
+			} else {
+				static SimpleCounter<int64_t>* countClientTLSHandshakesTimedout =
+				    SimpleCounter<int64_t>::makeCounter("/Net2/TLS/ClientTLSHandshakesTimedout");
+				countClientTLSHandshakesTimedout->increment(1);
+				throw connection_failed();
 			}
 		} catch (Error& e) {
 			// Either the connection failed, or was cancelled by the caller
@@ -1207,7 +1212,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 	// returns when write() can write at least one byte
 	Future<Void> onWritable() override {
 		++g_net2->countWriteProbes;
-		BindPromise p("N2_WriteProbeError", id);
+		BindPromise p("N2_WriteProbeError", id, peer_address);
 		auto f = p.getFuture();
 		socket.async_write_some(boost::asio::null_buffers(), std::move(p));
 		return f;
@@ -1216,7 +1221,7 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 	// returns when read() can read at least one byte
 	Future<Void> onReadable() override {
 		++g_net2->countReadProbes;
-		BindPromise p("N2_ReadProbeError", id);
+		BindPromise p("N2_ReadProbeError", id, peer_address);
 		auto f = p.getFuture();
 		socket.async_read_some(boost::asio::null_buffers(), std::move(p));
 		return f;
@@ -1372,20 +1377,21 @@ class SSLListener final : public IListener, ReferenceCounted<SSLListener> {
 	NetworkAddress getListenAddress() const override { return listenAddress; }
 
 private:
-	ACTOR static Future<Reference<IConnection>> doAccept(SSLListener* self) {
-		state Reference<SSLConnection> conn(new SSLConnection(self->io_service, self->contextVar->get()));
-		state tcp::acceptor::endpoint_type peer_endpoint;
+	static Future<Reference<IConnection>> doAccept(SSLListener* self) {
+		Reference<SSLConnection> conn(new SSLConnection(self->io_service, self->contextVar->get()));
+		tcp::acceptor::endpoint_type peer_endpoint;
 		try {
-			BindPromise p("N2_AcceptError", UID());
+			// Peer address not known until accept succeeds
+			BindPromise p("N2_AcceptError", UID(), NetworkAddress());
 			auto f = p.getFuture();
 			self->acceptor.async_accept(conn->getSocket(), peer_endpoint, std::move(p));
-			wait(f);
+			co_await f;
 			auto peer_address = peer_endpoint.address().is_v6() ? IPAddress(peer_endpoint.address().to_v6().to_bytes())
 			                                                    : IPAddress(peer_endpoint.address().to_v4().to_ulong());
 
 			conn->accept(NetworkAddress(peer_address, peer_endpoint.port(), false, true));
 
-			return conn;
+			co_return conn;
 		} catch (...) {
 			conn->close();
 			throw;
@@ -1395,6 +1401,8 @@ class SSLListener final : public IListener, ReferenceCounted<SSLListener> {
 
 // 5MB for loading files into memory
 
+static Future<Void> coordinatorDNSCacheRefresh(Net2* self);
+
 Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics)
   : globals(enumGlobal::COUNT), useThreadPool(useThreadPool), reactor(this),
     sslContextVar({ ReferencedObject<boost::asio::ssl::context>::from(
@@ -1426,24 +1434,24 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics)
 	updateNow();
 }
 
-ACTOR static Future<Void> reloadCertificatesOnChange(
+static Future<Void> reloadCertificatesOnChange(
     TLSConfig config,
     std::function<void()> onPolicyFailure,
     AsyncVar<Reference<ReferencedObject<boost::asio::ssl::context>>>* contextVar,
     Reference<TLSPolicy>* policy) {
 	if (FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS <= 0) {
-		return Void();
+		co_return;
 	}
-	loop {
+	while (true) {
 		// Early in bootup, the filesystem might not be initialized yet.  Wait until it is.
 		if (IAsyncFileSystem::filesystem() != nullptr) {
 			break;
 		}
-		wait(delay(1.0));
+		co_await delay(1.0);
 	}
-	state int mismatches = 0;
-	state AsyncTrigger fileChanged;
-	state std::vector<Future<Void>> lifetimes;
+	int mismatches = 0;
+	AsyncTrigger fileChanged;
+	std::vector<Future<Void>> lifetimes;
 	const int& intervalSeconds = FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS;
 	lifetimes.push_back(watchFileForChanges(
 	    config.getCertificatePathSync(), &fileChanged, &intervalSeconds, "TLSCertificateRefreshStatError"));
@@ -1451,12 +1459,12 @@ ACTOR static Future<Void> reloadCertificatesOnChange(
 	    watchFileForChanges(config.getKeyPathSync(), &fileChanged, &intervalSeconds, "TLSKeyRefreshStatError"));
 	lifetimes.push_back(
 	    watchFileForChanges(config.getCAPathSync(), &fileChanged, &intervalSeconds, "TLSCARefreshStatError"));
-	loop {
-		wait(fileChanged.onTrigger());
+	while (true) {
+		co_await fileChanged.onTrigger();
 		TraceEvent("TLSCertificateRefreshBegin").log();
 
 		try {
-			LoadedTLSConfig loaded = wait(config.loadAsync());
+			LoadedTLSConfig loaded = co_await config.loadAsync();
 			boost::asio::ssl::context context(boost::asio::ssl::context::tls);
 			ConfigureSSLContext(loaded, context);
 			*policy = makeReference<TLSPolicy>(loaded, onPolicyFailure);
@@ -1541,15 +1549,15 @@ void Net2::initTLS(ETLSInitState targetState) {
 	tlsInitializedState = targetState;
 }
 
-ACTOR Future<Void> Net2::logTimeOffset() {
-	loop {
+Future<Void> Net2::logTimeOffset() {
+	while (true) {
 		double processTime = timer_monotonic();
 		double systemTime = timer();
 		TraceEvent("ProcessTimeOffset")
 		    .detailf("ProcessTime", "%lf", processTime)
 		    .detailf("SystemTime", "%lf", systemTime)
 		    .detailf("OffsetFromSystemTime", "%lf", processTime - systemTime);
-		wait(::delay(FLOW_KNOBS->TIME_OFFSET_LOGGING_INTERVAL));
+		co_await ::delay(FLOW_KNOBS->TIME_OFFSET_LOGGING_INTERVAL);
 	}
 }
 
@@ -1587,10 +1595,15 @@ ActorLineageSet& Net2::getActorLineageSet() {
 }
 #endif
 
+// TIP: ways to test this code:
+//
+// fdbserver -r test -f tests/noSim/RandomUnitTests.toml
+// fdbserver -r unittests -f noSim
 void Net2::run() {
 	TraceEvent::setNetworkThread();
 	TraceEvent("Net2Running").log();
 	thread_network = this;
+	dnsCacheRefreshActor = coordinatorDNSCacheRefresh(this);
 
 	unsigned int tasksSinceReact = 0;
 
@@ -1608,8 +1621,8 @@ void Net2::run() {
 	}
 
 	// Get the address to the launch function
-	typedef void (*runCycleFuncPtr)();
-	runCycleFuncPtr runFunc = reinterpret_cast<runCycleFuncPtr>(
+	using runCycleFuncPtr = void (*)();
+	auto runFunc = reinterpret_cast<runCycleFuncPtr>(
 	    reinterpret_cast<flowGlobalType>(g_network->global(INetwork::enRunCycleFunc)));
 
 	started.store(true);
@@ -1678,6 +1691,7 @@ void Net2::run() {
 		[[maybe_unused]] int queueSize = taskQueue.getNumReadyTasks();
 
 		FDB_TRACE_PROBE(run_loop_tasks_start, queueSize);
+		int tasksExecuted = 0;
 		while (taskQueue.hasReadyTask()) {
 			++countTasks;
 			currentTaskID = taskQueue.getReadyTaskID();
@@ -1686,6 +1700,7 @@ void Net2::run() {
 			taskQueue.popReadyTask();
 
 			try {
+				++tasksExecuted;
 				++tasksSinceReact;
 				(*task)();
 			} catch (Error& e) {
@@ -1721,6 +1736,9 @@ void Net2::run() {
 			taskBegin = newTaskBegin;
 			tscBegin = tscNow;
 		}
+		static SimpleCounter<int64_t>* callbacksExecuted =
+		    SimpleCounter<int64_t>::makeCounter("/Net2/callbacksExecuted");
+		callbacksExecuted->increment(tasksExecuted);
 
 		trackAtPriority(TaskPriority::RunLoop, taskBegin);
 
@@ -1759,7 +1777,7 @@ void Net2::run() {
 			if (other_offset) {
 				size_t iter_offset = 0;
 				while (iter_offset < other_offset) {
-					ProfilingSample* ps = (ProfilingSample*)(other_backtraces + iter_offset);
+					auto* ps = (ProfilingSample*)(other_backtraces + iter_offset);
 					TraceEvent(SevWarn, "Net2RunLoopTrace")
 					    .detailf("TraceTime", "%.6f", ps->timestamp)
 					    .detail("Trace", platform::format_backtrace(ps->frames, ps->length));
@@ -1772,17 +1790,24 @@ void Net2::run() {
 		}
 #endif
 		nnow = timer_monotonic();
+		auto time_delta = nnow - now;
+
+		static SimpleCounter<double>* exec_time = SimpleCounter<double>::makeCounter("/Net2/mainThreadExecutionTime");
+		exec_time->increment(time_delta);
 
-		if ((nnow - now) > FLOW_KNOBS->SLOW_LOOP_CUTOFF &&
-		    nondeterministicRandom()->random01() < (nnow - now) * FLOW_KNOBS->SLOW_LOOP_SAMPLING_RATE)
-			TraceEvent("SomewhatSlowRunLoopBottom")
-			    .detail("Elapsed", nnow - now); // This includes the time spent running tasks
+		if (time_delta > FLOW_KNOBS->SLOW_LOOP_CUTOFF &&
+		    nondeterministicRandom()->random01() < time_delta * FLOW_KNOBS->SLOW_LOOP_SAMPLING_RATE) {
+			TraceEvent("SomewhatSlowRunLoopBottom").detail("Elapsed", time_delta);
+		}
 	}
 
 	for (auto& fn : stopCallbacks) {
 		fn();
 	}
 
+	// Emit at least one batch of counters, for manual inspection.
+	simpleCounterReport();
+
 #ifdef WIN32
 	timeEndPeriod(1);
 #endif
@@ -1925,7 +1950,7 @@ Future<Void> Net2::delay(double seconds, TaskPriority taskId) {
 	                     // as infinite
 		return Never();
 
-	PromiseTask* t = new PromiseTask;
+	auto* t = new PromiseTask;
 	if (seconds <= 0.) {
 		taskQueue.addReady(taskId, t);
 	} else {
@@ -1942,9 +1967,9 @@ Future<Void> Net2::orderedDelay(double seconds, TaskPriority taskId) {
 
 void Net2::_swiftEnqueue(void* _job) {
 #ifdef WITH_SWIFT
-	swift::Job* job = (swift::Job*)_job;
+	auto* job = (swift::Job*)_job;
 	TaskPriority priority = swift_priority_to_net2(job->getPriority());
-	PromiseTask* t = new PromiseTask(job);
+	auto* t = new PromiseTask(job);
 	taskQueue.addReady(priority, t);
 #endif
 }
@@ -1952,7 +1977,7 @@ void Net2::_swiftEnqueue(void* _job) {
 void Net2::onMainThread(Promise<Void>&& signal, TaskPriority taskID) {
 	if (stopped)
 		return;
-	PromiseTask* p = new PromiseTask(std::move(signal));
+	auto* p = new PromiseTask(std::move(signal));
 	if (taskQueue.addReadyThreadSafe(isOnMainThread(), taskID, p)) {
 		reactor.wake();
 	}
@@ -1996,12 +2021,10 @@ Future<Reference<IUDPSocket>> Net2::createUDPSocket(bool isV6) {
 	return UDPSocket::connect(&reactor.ios, Optional<NetworkAddress>(), isV6);
 }
 
-ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* self,
-                                                                         std::string host,
-                                                                         std::string service) {
-	state tcp::resolver tcpResolver(self->reactor.ios);
+static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* self, std::string host, std::string service) {
+	tcp::resolver tcpResolver(self->reactor.ios);
 	Promise<std::vector<NetworkAddress>> promise;
-	state Future<std::vector<NetworkAddress>> result = promise.getFuture();
+	Future<std::vector<NetworkAddress>> result = promise.getFuture();
 
 	tcpResolver.async_resolve(
 	    host, service, [promise](const boost::system::error_code& ec, tcp::resolver::iterator iter) {
@@ -2034,34 +2057,66 @@ ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* s
 		    }
 	    });
 
-	try {
-		wait(ready(result));
-	} catch (Error& e) {
-		if (e.code() == error_code_lookup_failed) {
-			self->dnsCache.remove(host, service);
-		}
-		throw e;
-	}
+	co_await ready(result);
 	tcpResolver.cancel();
-	std::vector<NetworkAddress> ret = result.get();
-	self->dnsCache.add(host, service, ret);
-
-	return ret;
+	co_return result.get();
 }
 
 Future<std::vector<NetworkAddress>> Net2::resolveTCPEndpoint(const std::string& host, const std::string& service) {
 	return resolveTCPEndpoint_impl(this, host, service);
 }
 
+static Future<Void> coordinatorDNSCacheRefresh(Net2* self) {
+	if (!FLOW_KNOBS->ENABLE_COORDINATOR_DNS_CACHE) {
+		co_return;
+	}
+	loop {
+		co_await delay(FLOW_KNOBS->COORDINATOR_DNS_CACHE_REFRESH_INTERVAL);
+		std::vector<std::string> keys = self->dnsCache.getKeys();
+		for (int i = 0; i < keys.size(); i++) {
+			auto colonPos = keys[i].find(':');
+			if (colonPos == std::string::npos) {
+				continue;
+			}
+			std::string host = keys[i].substr(0, colonPos);
+			std::string service = keys[i].substr(colonPos + 1);
+
+			// Evict if stale
+			double secondsSinceLastAccess = now() - self->dnsCache.getLastAccess(host, service).orDefault(0.0);
+			if (secondsSinceLastAccess > FLOW_KNOBS->COORDINATOR_DNS_CACHE_TTL) {
+				self->dnsCache.remove(host, service);
+				TraceEvent("DNSCacheEntryEvicted")
+				    .detail("Host", host)
+				    .detail("Service", service)
+				    .detail("SecondsSinceLastAccess", secondsSinceLastAccess);
+				continue;
+			}
+
+			try {
+				std::vector<NetworkAddress> newAddrs = co_await resolveTCPEndpoint_impl(self, host, service);
+				self->dnsCache.update(host, service, newAddrs);
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled) {
+					throw;
+				}
+				TraceEvent(SevWarn, "DNSCacheRefreshFailed").error(e).detail("Host", host).detail("Service", service);
+			}
+		}
+	}
+}
+
 Future<std::vector<NetworkAddress>> Net2::resolveTCPEndpointWithDNSCache(const std::string& host,
                                                                          const std::string& service) {
 	if (FLOW_KNOBS->ENABLE_COORDINATOR_DNS_CACHE) {
 		Optional<std::vector<NetworkAddress>> cache = dnsCache.find(host, service);
 		if (cache.present()) {
-			return cache.get();
+			co_return cache.get();
 		}
+		std::vector<NetworkAddress> addresses = co_await resolveTCPEndpoint_impl(this, host, service);
+		dnsCache.add(host, service, addresses);
+		co_return addresses;
 	}
-	return resolveTCPEndpoint_impl(this, host, service);
+	co_return co_await resolveTCPEndpoint_impl(this, host, service);
 }
 
 std::vector<NetworkAddress> Net2::resolveTCPEndpointBlocking(const std::string& host, const std::string& service) {
@@ -2085,7 +2140,6 @@ std::vector<NetworkAddress> Net2::resolveTCPEndpointBlocking(const std::string&
 		}
 		return addrs;
 	} catch (...) {
-		dnsCache.remove(host, service);
 		throw lookup_failed();
 	}
 }
@@ -2098,7 +2152,11 @@ std::vector<NetworkAddress> Net2::resolveTCPEndpointBlockingWithDNSCache(const s
 			return cache.get();
 		}
 	}
-	return resolveTCPEndpointBlocking(host, service);
+	auto result = resolveTCPEndpointBlocking(host, service);
+	if (FLOW_KNOBS->ENABLE_COORDINATOR_DNS_CACHE) {
+		dnsCache.add(host, service, result);
+	}
+	return result;
 }
 
 bool Net2::isAddressOnThisHost(NetworkAddress const& addr) const {
@@ -2133,9 +2191,9 @@ Reference<IListener> Net2::listen(NetworkAddress localAddr) {
 	try {
 		if (localAddr.isTLS()) {
 			initTLS(ETLSInitState::LISTEN);
-			return Reference<IListener>(new SSLListener(reactor.ios, &this->sslContextVar, localAddr));
+			return makeReference<SSLListener>(reactor.ios, &this->sslContextVar, localAddr);
 		}
-		return Reference<IListener>(new Listener(reactor.ios, localAddr));
+		return makeReference<Listener>(reactor.ios, localAddr);
 	} catch (boost::system::system_error const& e) {
 		Error x;
 		if (e.code().value() == EADDRINUSE)
@@ -2266,7 +2324,7 @@ struct TestGVR {
 	Optional<std::pair<UID, UID>> debugID;
 	Promise<Optional<Standalone<StringRef>>> reply;
 
-	TestGVR() {}
+	TestGVR() = default;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -2278,9 +2336,9 @@ template <class F>
 THREAD_HANDLE startThreadF(F&& func) {
 	struct Thing {
 		F f;
-		Thing(F&& f) : f(std::move(f)) {}
+		explicit Thing(F&& f) : f(std::move(f)) {}
 		THREAD_FUNC start(void* p) {
-			Thing* self = (Thing*)p;
+			auto* self = (Thing*)p;
 			self->f();
 			delete self;
 			THREAD_RETURN;
@@ -2337,10 +2395,10 @@ TEST_CASE("flow/Net2/ThreadSafeQueue/Threaded") {
 	noUnseed = true; // multi-threading inherently non-deterministic
 
 	ThreadSafeQueue<int> queue;
-	state std::vector<QueueTestThreadState> perThread = { QueueTestThreadState(0, 1000000),
-		                                                  QueueTestThreadState(1, 100000),
-		                                                  QueueTestThreadState(2, 1000000) };
-	state std::vector<Future<Void>> doneProducing;
+	std::vector<QueueTestThreadState> perThread = { QueueTestThreadState(0, 1000000),
+		                                            QueueTestThreadState(1, 100000),
+		                                            QueueTestThreadState(2, 1000000) };
+	std::vector<Future<Void>> doneProducing;
 
 	int total = 0;
 	for (int t = 0; t < perThread.size(); ++t) {
@@ -2376,29 +2434,28 @@ TEST_CASE("flow/Net2/ThreadSafeQueue/Threaded") {
 			queue.canSleep();
 	}
 
-	wait(waitForAll(doneProducing));
+	co_await waitForAll(doneProducing);
 
 	// Make sure we continue on the main thread.
 	Promise<Void> signal;
-	state Future<Void> doneConsuming = signal.getFuture();
+	Future<Void> doneConsuming = signal.getFuture();
 	g_network->onMainThread(std::move(signal), TaskPriority::DefaultOnMainThread);
-	wait(doneConsuming);
+	co_await doneConsuming;
 
 	for (int t = 0; t < perThread.size(); ++t) {
 		waitThread(perThread[t].handle);
 		perThread[t].checkDone();
 	}
-	return Void();
 }
 
 TEST_CASE("noSim/flow/Net2/onMainThreadFIFO") {
 	// Verifies that signals processed by onMainThread() are executed in order.
 	noUnseed = true; // multi-threading inherently non-deterministic
 
-	state std::vector<QueueTestThreadState> perThread = { QueueTestThreadState(0, 1000000),
-		                                                  QueueTestThreadState(1, 100000),
-		                                                  QueueTestThreadState(2, 1000000) };
-	state std::vector<Future<Void>> doneProducing;
+	std::vector<QueueTestThreadState> perThread = { QueueTestThreadState(0, 1000000),
+		                                            QueueTestThreadState(1, 100000),
+		                                            QueueTestThreadState(2, 1000000) };
+	std::vector<Future<Void>> doneProducing;
 	for (int t = 0; t < perThread.size(); ++t) {
 		auto& s = perThread[t];
 		doneProducing.push_back(s.doneProducing.getFuture());
@@ -2415,77 +2472,16 @@ TEST_CASE("noSim/flow/Net2/onMainThreadFIFO") {
 			s.doneProducing.send(Void());
 		});
 	}
-	wait(waitForAll(doneProducing));
+	co_await waitForAll(doneProducing);
 
 	// Wait for one more onMainThread to wait for all scheduled signals to be executed.
 	Promise<Void> signal;
-	state Future<Void> doneConsuming = signal.getFuture();
+	Future<Void> doneConsuming = signal.getFuture();
 	g_network->onMainThread(std::move(signal), TaskPriority::DefaultOnMainThread);
-	wait(doneConsuming);
+	co_await doneConsuming;
 
 	for (int t = 0; t < perThread.size(); ++t) {
 		waitThread(perThread[t].handle);
 		perThread[t].checkDone();
 	}
-	return Void();
 }
-
-void net2_test() {
-	/*
-	g_network = newNet2();  // for promise serialization below
-
-	Endpoint destination;
-
-	printf("  Used: %lld\n", FastAllocator<4096>::getTotalMemory());
-
-	char junk[100];
-
-	double before = timer();
-
-	std::vector<TestGVR> reqs;
-	reqs.reserve( 10000 );
-
-	int totalBytes = 0;
-	for(int j=0; j<1000; j++) {
-	    UnsentPacketQueue unsent;
-	    ReliablePacketList reliable;
-
-	    reqs.resize(10000);
-	    for(int i=0; i<10000; i++) {
-	        TestGVR &req = reqs[i];
-	        req.key = "Foobar"_sr;
-
-	        SerializeSource<TestGVR> what(req);
-
-	        SendBuffer* pb = unsent.getWriteBuffer();
-	        ReliablePacket* rp = new ReliablePacket;  // 0
-
-	        PacketWriter wr(pb,rp,AssumeVersion(g_network->protocolVersion()));
-	        //BinaryWriter wr;
-	        SplitBuffer packetLen;
-	        uint32_t len = 0;
-	        wr.writeAhead(sizeof(len), &packetLen);
-	        wr << destination.token;
-	        //req.reply.getEndpoint();
-	        what.serializePacketWriter(wr);
-	        //wr.serializeBytes(junk, 43);
-
-	        unsent.setWriteBuffer(wr.finish());
-	        len = wr.size() - sizeof(len);
-	        packetLen.write(&len, sizeof(len));
-
-	        //totalBytes += wr.getLength();
-	        totalBytes += wr.size();
-
-	        if (rp) reliable.insert(rp);
-	    }
-	    reqs.clear();
-	    unsent.discardAll();
-	    reliable.discardAll();
-	}
-
-	printf("SimSend x 1Kx10K: %0.2f sec\n", timer()-before);
-	printf("  Bytes: %d\n", totalBytes);
-	printf("  Used: %lld\n", FastAllocator<4096>::getTotalMemory());
-	*/
-};
diff --git a/flow/Net2Packet.cpp b/flow/Net2Packet.cpp
index 3377a565ffb..efa77e115d5 100644
--- a/flow/Net2Packet.cpp
+++ b/flow/Net2Packet.cpp
@@ -190,7 +190,7 @@ PacketBuffer* ReliablePacketList::compact(PacketBuffer* into, PacketBuffer* end)
 			if (len > into->bytes_unwritten()) {
 				// We have to split this ReliablePacket
 				len = into->bytes_unwritten();
-				ReliablePacket* e = new ReliablePacket;
+				auto* e = new ReliablePacket;
 				e->cont = c->cont;
 				e->buffer = c->buffer;
 				e->buffer->addref();
diff --git a/flow/Platform.actor.cpp b/flow/Platform.cpp
similarity index 97%
rename from flow/Platform.actor.cpp
rename to flow/Platform.cpp
index 731c2a40325..b4552e8ac70 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.cpp
@@ -1,5 +1,5 @@
 /*
- * Platform.actor.cpp
+ * Platform.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -52,12 +52,10 @@
 #include "flow/Error.h"
 #include "flow/FaultInjection.h"
 #include "flow/Knobs.h"
-#include "flow/Platform.actor.h"
 #include "flow/ScopeExit.h"
 #include "flow/SimpleCounter.h"
 #include "flow/StreamCipher.h"
 #include "flow/Trace.h"
-#include "flow/Trace.h"
 #include "flow/UnitTest.h"
 #include "flow/Util.h"
 
@@ -118,7 +116,6 @@ static_assert(std::is_same<boost::asio::ip::address_v6::bytes_type, std::array<u
 /* Needed for memory allocation */
 #include <linux/mman.h>
 /* Needed for processor affinity */
-#include <sched.h>
 /* Needed for getProcessorTime* and setpriority */
 #include <sys/syscall.h>
 /* Needed for setpriority */
@@ -164,6 +161,7 @@ static_assert(std::is_same<boost::asio::ip::address_v6::bytes_type, std::array<u
 
 #ifdef __APPLE__
 /* Needed for cross-platform 'environ' */
+#include <sys/random.h>
 #include <crt_externs.h>
 #include <mach-o/dyld.h>
 #include <mach/mach.h>
@@ -173,6 +171,8 @@ static_assert(std::is_same<boost::asio::ip::address_v6::bytes_type, std::array<u
 #include <netinet/in.h>
 #include <sys/mount.h>
 #include <sys/param.h>
+/* Needed for getentropy() */
+#include <sys/random.h>
 #include <sys/sysctl.h>
 #include <sys/syslimits.h>
 #include <sys/uio.h>
@@ -186,8 +186,6 @@ static_assert(std::is_same<boost::asio::ip::address_v6::bytes_type, std::array<u
 
 #endif // __unixish__
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 std::string removeWhitespace(const std::string& t) {
 	static const std::string ws(" \t\r");
 	std::string str = t;
@@ -702,7 +700,7 @@ const char* getInterfaceName(const IPAddress& _ip) {
 				break;
 			}
 		} else if (iter->ifa_addr->sa_family == AF_INET6 && _ip.isV6()) {
-			struct sockaddr_in6* ifa_addr = reinterpret_cast<struct sockaddr_in6*>(iter->ifa_addr);
+			auto* ifa_addr = reinterpret_cast<struct sockaddr_in6*>(iter->ifa_addr);
 			if (memcmp(_ip.toV6().data(), &ifa_addr->sin6_addr, 16) == 0) {
 				ifa_name = iter->ifa_name;
 				break;
@@ -1575,7 +1573,7 @@ SystemStatistics getSystemStatistics(std::string const& dataFolder,
 	returnStats.machineCommittedRAM = memInfo.committed;
 	returnStats.machineAvailableRAM = memInfo.available;
 
-	if (dataFolder != "") {
+	if (!dataFolder.empty()) {
 		int64_t diskTotal, diskFree;
 		getDiskBytes(dataFolder, diskFree, diskTotal);
 		returnStats.processDiskTotalBytes = diskTotal;
@@ -1587,7 +1585,7 @@ SystemStatistics getSystemStatistics(std::string const& dataFolder,
 		initPdhStrings(*statState, dataFolder);
 
 		TraceEvent("SetupQuery").log();
-		handlePdhStatus(PdhOpenQuery(nullptr, NULL, &(*statState)->Query), "PdhOpenQuery");
+		handlePdhStatus(PdhOpenQuery(nullptr, nullptr, &(*statState)->Query), "PdhOpenQuery");
 
 		if (!(*statState)->pdhStrings.diskDevice.empty()) {
 			handlePdhStatus(
@@ -1736,7 +1734,7 @@ SystemStatistics getSystemStatistics(std::string const& dataFolder,
 
 	(*statState)->machineLastRetransSegs = machineRetransSegs;
 
-	if (dataFolder != "") {
+	if (!dataFolder.empty()) {
 		DiskStatistics currentDiskStats = getDiskStatistics(dataFolder);
 
 		returnStats.processDiskQueueDepth = currentDiskStats.currentIOs;
@@ -2206,39 +2204,62 @@ void setAffinity(int proc) {
 
 namespace platform {
 
-int getRandomSeed() {
+uint64_t getRandomSeed() {
 	INJECT_FAULT(platform_error, "getRandomSeed"); // getting a random seed failed
-	int randomSeed;
-
+	uint64_t randomSeed;
 #ifdef _WIN32
-	if (rand_s((unsigned int*)&randomSeed) != 0) {
+	uint32_t high, low;
+	if (rand_s(&low) != 0 || rand_s(&high) != 0 {
 		TraceEvent(SevError, "WindowsRandomSeedError").log();
 		throw platform_error();
 	}
+	randomSeed = (uint64_t(high) << 32) | uint64_t(low);
 #else
-	int devRandom = open("/dev/urandom", O_RDONLY | O_CLOEXEC);
-	if (devRandom == -1) {
-		TraceEvent(SevError, "OpenURandom").GetLastError();
-		throw platform_error();
-	}
-	int nbytes = read(devRandom, &randomSeed, sizeof(randomSeed));
-	close(devRandom);
-	if (nbytes != sizeof(randomSeed)) {
-		TraceEvent(SevError, "ReadURandom").GetLastError();
+	if (getentropy(&randomSeed, sizeof(randomSeed)) != 0) {
+		TraceEvent(SevError, "GetEntropyError").GetLastError();
 		throw platform_error();
 	}
 #endif
-
 	return randomSeed;
 }
+
+void getRandomBytes(void* buf, size_t len) {
+	INJECT_FAULT(platform_error, "getRandomBytes"); // getting random bytes failed
+	ASSERT(len == 0 || buf != nullptr);
+#ifdef _WIN32
+	size_t pos = 0;
+	while (pos < len) {
+		unsigned int val;
+		if (rand_s(&val) != 0) {
+			TraceEvent(SevError, "WindowsRandomBytesError").log();
+			throw platform_error();
+		}
+		size_t chunk = std::min(sizeof(val), len - pos);
+		memcpy(static_cast<uint8_t*>(buf) + pos, &val, chunk);
+		pos += chunk;
+	}
+#else
+	uint8_t* p = static_cast<uint8_t*>(buf);
+	while (len > 0) {
+		size_t chunk = std::min(len, size_t(256));
+		if (getentropy(p, chunk) != 0) {
+			TraceEvent(SevError, "GetEntropyError").detail("Errno", errno);
+			throw platform_error();
+		}
+		p += chunk;
+		len -= chunk;
+	}
+#endif
+}
+
 } // namespace platform
 
 std::string joinPath(std::string const& directory, std::string const& filename) {
 	auto d = directory;
 	auto f = filename;
-	while (f.size() && (f[0] == '/' || f[0] == CANONICAL_PATH_SEPARATOR))
+	while (!f.empty() && (f[0] == '/' || f[0] == CANONICAL_PATH_SEPARATOR))
 		f = f.substr(1);
-	while (d.size() && (d.back() == '/' || d.back() == CANONICAL_PATH_SEPARATOR))
+	while (!d.empty() && (d.back() == '/' || d.back() == CANONICAL_PATH_SEPARATOR))
 		d.resize(d.size() - 1);
 	return d + CANONICAL_PATH_SEPARATOR + f;
 }
@@ -2480,9 +2501,9 @@ std::string cleanPath(std::string const& path) {
 
 	StringRef p(path);
 
-	while (p.size() != 0) {
+	while (!p.empty()) {
 		StringRef part = p.eat(separator);
-		if (part.size() == 0 || (part.size() == 1 && part[0] == '.'))
+		if (part.empty() || (part.size() == 1 && part[0] == '.'))
 			continue;
 		if (part == dotdot) {
 			if (!finalParts.empty() && finalParts.back() != dotdot) {
@@ -2654,16 +2675,16 @@ bool acceptDirectory(FILE_ATTRIBUTE_DATA fileAttributes, std::string const& name
 	return (fileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
 }
 
-ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
-                                                 std::string extension,
-                                                 bool directoryOnly,
-                                                 bool async) {
+AsyncResult<std::vector<std::string>> findFiles(std::string directory,
+                                                std::string extension,
+                                                bool directoryOnly,
+                                                bool async) {
 	INJECT_FAULT(platform_error, "findFiles"); // findFiles failed (Win32)
-	state std::vector<std::string> result;
-	state int64_t tsc_begin = timestampCounter();
+	std::vector<std::string> result;
+	int64_t tsc_begin = timestampCounter();
 
-	state WIN32_FIND_DATA fd;
-	state HANDLE h = FindFirstFile((directory + "/*" + extension).c_str(), &fd);
+	WIN32_FIND_DATA fd;
+	HANDLE h = FindFirstFile((directory + "/*" + extension).c_str(), &fd);
 	if (h == INVALID_HANDLE_VALUE) {
 		if (GetLastError() != ERROR_FILE_NOT_FOUND && GetLastError() != ERROR_PATH_NOT_FOUND) {
 			TraceEvent(SevError, "FindFirstFile")
@@ -2673,7 +2694,7 @@ ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
 			throw platform_error();
 		}
 	} else {
-		loop {
+		while (true) {
 			std::string name = fd.cFileName;
 			if ((directoryOnly && acceptDirectory(fd.dwFileAttributes, name, extension)) ||
 			    (!directoryOnly && acceptFile(fd.dwFileAttributes, name, extension))) {
@@ -2682,7 +2703,7 @@ ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
 			if (!FindNextFile(h, &fd))
 				break;
 			if (async && timestampCounter() - tsc_begin > FLOW_KNOBS->TSC_YIELD_TIME && !g_network->isSimulated()) {
-				wait(yield());
+				co_await yield();
 				tsc_begin = timestampCounter();
 			}
 		}
@@ -2697,7 +2718,7 @@ ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
 		FindClose(h);
 	}
 	std::sort(result.begin(), result.end());
-	return result;
+	co_return result;
 }
 
 #elif (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
@@ -2711,18 +2732,16 @@ bool acceptDirectory(FILE_ATTRIBUTE_DATA fileAttributes, std::string const& name
 	return S_ISDIR(fileAttributes);
 }
 
-ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
-                                                 std::string extension,
-                                                 bool directoryOnly,
-                                                 bool async) {
+AsyncResult<std::vector<std::string>> findFiles(std::string directory,
+                                                std::string extension,
+                                                bool directoryOnly,
+                                                bool async) {
 	INJECT_FAULT(platform_error, "findFiles"); // findFiles failed
-	state std::vector<std::string> result;
-	state int64_t tsc_begin = timestampCounter();
-
-	state DIR* dip = nullptr;
+	std::vector<std::string> result;
+	int64_t tsc_begin = timestampCounter();
 
-	if ((dip = opendir(directory.c_str())) != nullptr) {
-		loop {
+	if (DIR* dip = opendir(directory.c_str()); dip != nullptr) {
+		while (true) {
 			struct dirent* dit;
 			dit = readdir(dip);
 			if (dit == nullptr) {
@@ -2748,7 +2767,7 @@ ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
 				result.push_back(name);
 			}
 			if (async && timestampCounter() - tsc_begin > FLOW_KNOBS->TSC_YIELD_TIME && !g_network->isSimulated()) {
-				wait(yield());
+				co_await yield();
 				tsc_begin = timestampCounter();
 			}
 		}
@@ -2756,7 +2775,7 @@ ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
 		closedir(dip);
 	}
 	std::sort(result.begin(), result.end());
-	return result;
+	co_return result;
 }
 
 #else
@@ -2766,19 +2785,19 @@ ACTOR Future<std::vector<std::string>> findFiles(std::string directory,
 namespace platform {
 
 std::vector<std::string> listFiles(std::string const& directory, std::string const& extension) {
-	return findFiles(directory, extension, false /* directoryOnly */, false).get();
-}
-
-Future<std::vector<std::string>> listFilesAsync(std::string const& directory, std::string const& extension) {
-	return findFiles(directory, extension, false /* directoryOnly */, true);
+	auto result = findFiles(directory, extension, false /* directoryOnly */, false);
+	if (result.isError()) {
+		throw result.getError();
+	}
+	return std::move(result).get();
 }
 
 std::vector<std::string> listDirectories(std::string const& directory) {
-	return findFiles(directory, "", true /* directoryOnly */, false).get();
-}
-
-Future<std::vector<std::string>> listDirectoriesAsync(std::string const& directory) {
-	return findFiles(directory, "", true /* directoryOnly */, true);
+	auto result = findFiles(directory, "", true /* directoryOnly */, false);
+	if (result.isError()) {
+		throw result.getError();
+	}
+	return std::move(result).get();
 }
 
 void findFilesRecursively(std::string const& path, std::vector<std::string>& out) {
@@ -2795,19 +2814,18 @@ void findFilesRecursively(std::string const& path, std::vector<std::string>& out
 	}
 }
 
-ACTOR Future<Void> findFilesRecursivelyAsync(std::string path, std::vector<std::string>* out) {
+AsyncResult<Void> findFilesRecursivelyAsync(std::string path, std::vector<std::string>* out) {
 	// Add files to output, prefixing path
-	state std::vector<std::string> files = wait(listFilesAsync(path, ""));
+	std::vector<std::string> files = co_await findFiles(path, "", false /* directoryOnly */, true);
 	for (auto const& f : files)
 		out->push_back(joinPath(path, f));
 
 	// Recurse for directories
-	state std::vector<std::string> directories = wait(listDirectoriesAsync(path));
+	std::vector<std::string> directories = co_await findFiles(path, "", true /* directoryOnly */, true);
 	for (auto const& dir : directories) {
 		if (dir != "." && dir != "..")
-			wait(findFilesRecursivelyAsync(joinPath(path, dir), out));
+			co_await findFilesRecursivelyAsync(joinPath(path, dir), out);
 	}
-	return Void();
 }
 
 } // namespace platform
@@ -2879,7 +2897,7 @@ struct ThreadCreateArgs {
 };
 
 void* runFunc(void* a) {
-	ThreadCreateArgs* args = (ThreadCreateArgs*)a;
+	auto* args = (ThreadCreateArgs*)a;
 	try {
 		(void)args->func(args->arg);
 	} catch (std::exception& e) {
@@ -2909,7 +2927,7 @@ THREAD_HANDLE startThread(void* (*func)(void*), void* arg, int stackSize, const
 		};
 	}
 
-	ThreadCreateArgs* args = new ThreadCreateArgs(func, arg);
+	auto* args = new ThreadCreateArgs(func, arg);
 	pthread_create(&t, &attr, &runFunc, args);
 	pthread_attr_destroy(&attr);
 
@@ -3875,15 +3893,14 @@ void profileHandler(int sig) {
 
 	// We are casting away the volatile-ness of the backtrace array, but we believe that should be reasonably safe in
 	// the signal handler
-	ProfilingSample* ps =
-	    const_cast<ProfilingSample*>((volatile ProfilingSample*)(net2backtraces + net2backtraces_offset));
+	auto* ps = const_cast<ProfilingSample*>((volatile ProfilingSample*)(net2backtraces + net2backtraces_offset));
 
 	// We can only read the check thread time in a signal handler if the atomic is lock free.
 	// We can't get the time from a timer() call because it's not signal safe.
 #ifndef WITH_SWIFT
 	ps->timestamp = checkThreadTime.is_lock_free() ? checkThreadTime.load() : 0;
 #else
-	// FIXME: problem with Swift build: lib/libflow.a(Platform.actor.g.cpp.o):Platform.actor.g.cpp:function
+	// FIXME: problem with Swift build: lib/libflow.a(Platform.cpp.o):Platform.cpp:function
 	// profileHandler(int): error: undefined reference to '__atomic_is_lock_free'
 	ps->timestamp = 0;
 #endif /* WITH_SWIFT */
@@ -4104,7 +4121,7 @@ void stopRunLoopProfiler() {
 	std::unique_lock<std::mutex> lock(loopProfilerThreadMutex);
 	loopProfilerStopRequested.store(true);
 	if (loopProfilerThread) {
-		pthread_join(loopProfilerThread.value(), NULL);
+		pthread_join(loopProfilerThread.value(), nullptr);
 		loopProfilerThread = {};
 	}
 #endif
diff --git a/flow/ProcessEvents.cpp b/flow/ProcessEvents.cpp
index b1efd5354d1..163b38fa515 100644
--- a/flow/ProcessEvents.cpp
+++ b/flow/ProcessEvents.cpp
@@ -86,7 +86,7 @@ struct ProcessEventsImpl {
 				// it's possible that the callback has been removed in
 				// which case attempting to call it will be undefined
 				// behavior.
-				if (toRemove.count(c.first) == 0) {
+				if (!toRemove.contains(c.first)) {
 					c.second->callback(name, data, e);
 				}
 			} catch (...) {
diff --git a/flow/Profiler.actor.cpp b/flow/Profiler.cpp
similarity index 90%
rename from flow/Profiler.actor.cpp
rename to flow/Profiler.cpp
index 1a9738383c6..2fba014e50d 100644
--- a/flow/Profiler.actor.cpp
+++ b/flow/Profiler.cpp
@@ -1,5 +1,5 @@
 /*
- * Profiler.actor.cpp
+ * Profiler.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,7 +23,6 @@
 
 #ifdef __linux__
 
-#include <execinfo.h>
 #include <signal.h>
 #include <sys/time.h>
 #include <stdlib.h>
@@ -31,7 +30,6 @@
 #include <link.h>
 
 #include "flow/Platform.h"
-#include "flow/actorcompiler.h" // This must be the last include.
 
 extern volatile thread_local int flowProfilingEnabled;
 
@@ -49,14 +47,14 @@ struct SignalClosure {
 		// async signal safe!
 		// This is intended to work as a SIGPROF handler for past and future versions of the flow profiler (when
 		// multiple are running in a process!) So don't change what it does without really good reason
-		SignalClosure* closure = (SignalClosure*)(si->si_value.sival_ptr);
+		auto* closure = static_cast<SignalClosure*>(si->si_value.sival_ptr);
 		closure->func(s, si, ucontext, closure->userdata);
 	}
 };
 
 struct SyncFileForSim : ReferenceCounted<SyncFileForSim> {
 	FILE* f;
-	SyncFileForSim(std::string const& filename) { f = fopen(filename.c_str(), "wb"); }
+	explicit SyncFileForSim(std::string const& filename) { f = fopen(filename.c_str(), "wb"); }
 
 	bool isOpen() const { return f != nullptr; }
 
@@ -180,12 +178,12 @@ struct Profiler {
 		return 0;
 	}
 
-	ACTOR static Future<Void> profile(Profiler* self, int period, std::string outfn) {
+	static Future<Void> profile(Profiler* self, int period, std::string outfn) {
 		// Open and truncate output file
-		state Reference<SyncFileForSim> outFile = makeReference<SyncFileForSim>(outfn);
+		auto outFile = makeReference<SyncFileForSim>(outfn);
 		if (!outFile->isOpen()) {
 			TraceEvent(SevWarn, "FailedToOpenProfilingOutputFile").detail("Filename", outfn).GetLastError();
-			return Void();
+			co_return;
 		}
 
 		// According to folk wisdom, calling this once before setting up the signal handler makes
@@ -202,7 +200,7 @@ struct Profiler {
 			self->environmentInfoWriter << uint8_t(0);
 
 		self->output_buffer = new OutputBuffer;
-		state OutputBuffer* otherBuffer = new OutputBuffer;
+		auto* otherBuffer = new OutputBuffer;
 
 		// The profilingSignals signal set will be used by enableSignal
 		sigemptyset(&self->profilingSignals);
@@ -230,29 +228,30 @@ struct Profiler {
 		sev._sigev_un._tid = sys_gettid();
 		if (timer_create(CLOCK_THREAD_CPUTIME_ID, &sev, &self->periodicTimer) != 0) {
 			TraceEvent(SevWarn, "FailedToCreateProfilingTimer").GetLastError();
-			return Void();
+			co_return;
 		}
 		self->timerInitialized = true;
 		if (timer_settime(self->periodicTimer, 0, &tv, nullptr) != 0) {
 			TraceEvent(SevWarn, "FailedToSetProfilingTimer").GetLastError();
-			return Void();
+			co_return;
 		}
 
-		state int64_t outOffset = 0;
-		wait(outFile->truncate(outOffset));
+		int64_t outOffset = 0;
+		co_await outFile->truncate(outOffset);
 
-		wait(outFile->write(self->environmentInfoWriter.getData(), self->environmentInfoWriter.getLength(), outOffset));
+		co_await outFile->write(
+		    self->environmentInfoWriter.getData(), self->environmentInfoWriter.getLength(), outOffset);
 		outOffset += self->environmentInfoWriter.getLength();
 
-		loop {
-			wait(self->network->delay(1.0, TaskPriority::Min) || self->network->delay(2.0, TaskPriority::Max));
+		while (true) {
+			co_await (self->network->delay(1.0, TaskPriority::Min) || self->network->delay(2.0, TaskPriority::Max));
 
 			self->enableSignal(false);
 			std::swap(self->output_buffer, otherBuffer);
 			self->enableSignal(true);
 
-			wait(otherBuffer->writeTo(outFile, outOffset));
-			wait(outFile->flush());
+			co_await otherBuffer->writeTo(outFile, outOffset);
+			co_await outFile->flush();
 			otherBuffer->clear();
 		}
 	}
diff --git a/flow/SimBugInjector.cpp b/flow/SimBugInjector.cpp
index 8e6e595f248..84304644f9a 100644
--- a/flow/SimBugInjector.cpp
+++ b/flow/SimBugInjector.cpp
@@ -63,7 +63,7 @@ unsigned ISimBug::numHits() const {
 	return ISimBugImpl::get(impl)->numHits;
 }
 
-IBugIdentifier::~IBugIdentifier() {}
+IBugIdentifier::~IBugIdentifier() = default;
 
 bool SimBugInjector::isEnabled() const {
 	return simBugInjector != nullptr && simBugInjector->isEnabled;
@@ -86,9 +86,8 @@ void SimBugInjector::disable() {
 }
 
 void SimBugInjector::reset() {
-	if (simBugInjector) {
-		delete simBugInjector;
-	}
+
+	delete simBugInjector;
 }
 
 std::shared_ptr<ISimBug> SimBugInjector::getImpl(const IBugIdentifier& id, bool getDisabled /* = false */) const {
@@ -113,4 +112,4 @@ std::shared_ptr<ISimBug> SimBugInjector::enableImpl(const IBugIdentifier& id) {
 		res = id.create();
 	}
 	return res;
-}
\ No newline at end of file
+}
diff --git a/flow/SimpleCounter.cpp b/flow/SimpleCounter.cpp
index 6f6b54b551d..962036163d6 100644
--- a/flow/SimpleCounter.cpp
+++ b/flow/SimpleCounter.cpp
@@ -30,7 +30,7 @@ static std::string hierarchicalToPrometheus(const std::string input) {
 	std::string output;
 	for (char ch : input) {
 		if (ch == '/') {
-			if (output.size() > 0) {
+			if (!output.empty()) {
 				output += '_';
 			}
 		} else {
@@ -158,7 +158,7 @@ TEST_CASE("/flow/simplecounter/int64") {
 	// NOTE: the following is written as >= 103 and not == 103 because
 	// "unit tests" actually run in fdbserver, so any background
 	// logic, like for example simpleCounterReport() being called
-	// above from fdbserver.actor.cpp, will affect the execution
+	// above from fdbserver.cpp, will affect the execution
 	// environment.
 	ASSERT(intCounters.size() >= 103);
 
@@ -194,7 +194,7 @@ TEST_CASE("/flow/simplecounter/double") {
 	ASSERT(baz->get() == expectedSum);
 
 	std::vector<SimpleCounter<double>*> doubleCounters = SimpleCounter<double>::getCounters();
-	ASSERT(doubleCounters.size() >= 1);
+	ASSERT(!doubleCounters.empty());
 
 	// Give asserts here a chance to run.
 	simpleCounterReport();
diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp
index 9f69c235f25..7cdd282c574 100644
--- a/flow/StreamCipher.cpp
+++ b/flow/StreamCipher.cpp
@@ -59,8 +59,8 @@ StreamCipherKey const* StreamCipherKey::getGlobalCipherKey() {
 }
 
 void StreamCipherKey::cleanup() noexcept {
-	for (const auto& itr : cipherKeys) {
-		itr.second->reset();
+	for (const auto& [_cipherId, cipherKey] : cipherKeys) {
+		cipherKey->reset();
 	}
 }
 
@@ -108,8 +108,8 @@ HMAC_CTX* StreamCipher::getHmacCtx() {
 }
 
 void StreamCipher::cleanup() noexcept {
-	for (auto itr : ctxs) {
-		EVP_CIPHER_CTX_free(itr.second);
+	for (const auto& [_ctxId, cipherContext] : ctxs) {
+		EVP_CIPHER_CTX_free(cipherContext);
 	}
 }
 
@@ -160,7 +160,7 @@ StringRef DecryptionStreamCipher::finish(Arena& arena) {
 }
 
 HmacSha256StreamCipher::HmacSha256StreamCipher() : cipher(EVP_MAX_KEY_LENGTH) {
-	HMAC_Init_ex(cipher.getHmacCtx(), NULL, 0, EVP_sha256(), nullptr);
+	HMAC_Init_ex(cipher.getHmacCtx(), nullptr, 0, EVP_sha256(), nullptr);
 }
 
 StringRef HmacSha256StreamCipher::finish(Arena& arena) {
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 5bf684c3b42..88dfd70482a 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -18,12 +18,9 @@
  * limitations under the License.
  */
 
-#include <fstream>
-
 #include "flow/flow.h"
-#include "flow/Histogram.h"
 #include "flow/Platform.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/SystemMonitor.h"
 
 #if defined(ALLOC_INSTRUMENTATION) && defined(__linux__)
@@ -47,6 +44,49 @@ double machineStartTime() {
 	return ::machineState.monitorStartTime;
 }
 
+void NetworkData::init() {
+	bytesSent = Int64Metric::getValueOrDefault("Net2.BytesSent"_sr);
+	countPacketsReceived = Int64Metric::getValueOrDefault("Net2.CountPacketsReceived"_sr);
+	countPacketsGenerated = Int64Metric::getValueOrDefault("Net2.CountPacketsGenerated"_sr);
+	bytesReceived = Int64Metric::getValueOrDefault("Net2.BytesReceived"_sr);
+	countWriteProbes = Int64Metric::getValueOrDefault("Net2.CountWriteProbes"_sr);
+	countReadProbes = Int64Metric::getValueOrDefault("Net2.CountReadProbes"_sr);
+	countReads = Int64Metric::getValueOrDefault("Net2.CountReads"_sr);
+	countWouldBlock = Int64Metric::getValueOrDefault("Net2.CountWouldBlock"_sr);
+	countWrites = Int64Metric::getValueOrDefault("Net2.CountWrites"_sr);
+	countRunLoop = Int64Metric::getValueOrDefault("Net2.CountRunLoop"_sr);
+	countCantSleep = Int64Metric::getValueOrDefault("Net2.CountCantSleep"_sr);
+	countWontSleep = Int64Metric::getValueOrDefault("Net2.CountWontSleep"_sr);
+	countTimers = Int64Metric::getValueOrDefault("Net2.CountTimers"_sr);
+	countTasks = Int64Metric::getValueOrDefault("Net2.CountTasks"_sr);
+	countYields = Int64Metric::getValueOrDefault("Net2.CountYields"_sr);
+	countYieldBigStack = Int64Metric::getValueOrDefault("Net2.CountYieldBigStack"_sr);
+	countYieldCalls = Int64Metric::getValueOrDefault("Net2.CountYieldCalls"_sr);
+	countASIOEvents = Int64Metric::getValueOrDefault("Net2.CountASIOEvents"_sr);
+	countYieldCallsTrue = Int64Metric::getValueOrDefault("Net2.CountYieldCallsTrue"_sr);
+	countRunLoopProfilingSignals = Int64Metric::getValueOrDefault("Net2.CountRunLoopProfilingSignals"_sr);
+	countConnEstablished = Int64Metric::getValueOrDefault("Net2.CountConnEstablished"_sr);
+	countConnClosedWithError = Int64Metric::getValueOrDefault("Net2.CountConnClosedWithError"_sr);
+	countConnClosedWithoutError = Int64Metric::getValueOrDefault("Net2.CountConnClosedWithoutError"_sr);
+	countTLSPolicyFailures = Int64Metric::getValueOrDefault("Net2.CountTLSPolicyFailures"_sr);
+	countLaunchTime = DoubleMetric::getValueOrDefault("Net2.CountLaunchTime"_sr);
+	countReactTime = DoubleMetric::getValueOrDefault("Net2.CountReactTime"_sr);
+	countFileLogicalWrites = Int64Metric::getValueOrDefault("AsyncFile.CountLogicalWrites"_sr);
+	countFileLogicalReads = Int64Metric::getValueOrDefault("AsyncFile.CountLogicalReads"_sr);
+	countAIOSubmit = Int64Metric::getValueOrDefault("AsyncFile.CountAIOSubmit"_sr);
+	countAIOCollect = Int64Metric::getValueOrDefault("AsyncFile.CountAIOCollect"_sr);
+	countFileCacheWrites = Int64Metric::getValueOrDefault("AsyncFile.CountCacheWrites"_sr);
+	countFileCacheReads = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReads"_sr);
+	countFileCacheWritesBlocked = Int64Metric::getValueOrDefault("AsyncFile.CountCacheWritesBlocked"_sr);
+	countFileCacheReadsBlocked = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReadsBlocked"_sr);
+	countFileCachePageReadsMerged = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMerged"_sr);
+	countFileCacheFinds = Int64Metric::getValueOrDefault("AsyncFile.CountCacheFinds"_sr);
+	countFileCacheReadBytes = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReadBytes"_sr);
+	countFilePageCacheHits = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsHit"_sr);
+	countFilePageCacheMisses = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMissed"_sr);
+	countFilePageCacheEvictions = Int64Metric::getValueOrDefault("EvictablePageCache.CacheEvictions"_sr);
+}
+
 void systemMonitor() {
 	static StatisticsState statState = StatisticsState();
 #if !DEBUG_DETERMINISM
diff --git a/flow/TDMetric.cpp b/flow/TDMetric.cpp
index 09645aab9bd..7403d0b4be5 100644
--- a/flow/TDMetric.cpp
+++ b/flow/TDMetric.cpp
@@ -20,7 +20,7 @@
 
 #include "flow/Error.h"
 #include "flow/OTELMetrics.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/flow.h"
 #include <cctype>
 #include <cstddef>
@@ -84,7 +84,7 @@ void MetricKeyRef::writeMetricName(BinaryWriter& wr) const {
 	wr.serializeBytes("\x00"_sr);
 }
 
-const Standalone<StringRef> MetricKeyRef::packLatestKey() const {
+Standalone<StringRef> MetricKeyRef::packLatestKey() const {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(prefix);
 	wr.serializeBytes("\x01TDMetricsLastValue\x00"_sr);
@@ -92,7 +92,7 @@ const Standalone<StringRef> MetricKeyRef::packLatestKey() const {
 	return wr.toValue();
 }
 
-const Standalone<StringRef> MetricKeyRef::packDataKey(int64_t time) const {
+Standalone<StringRef> MetricKeyRef::packDataKey(int64_t time) const {
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(prefix);
 	if (isField())
@@ -108,7 +108,7 @@ const Standalone<StringRef> MetricKeyRef::packDataKey(int64_t time) const {
 	return wr.toValue();
 }
 
-const Standalone<StringRef> MetricKeyRef::packFieldRegKey() const {
+Standalone<StringRef> MetricKeyRef::packFieldRegKey() const {
 	ASSERT(isField());
 	BinaryWriter wr(Unversioned());
 	wr.serializeBytes(prefix);
diff --git a/flow/TLSConfig.actor.cpp b/flow/TLSConfig.cpp
similarity index 95%
rename from flow/TLSConfig.actor.cpp
rename to flow/TLSConfig.cpp
index 5222c36f0ac..a6eefe27cc0 100644
--- a/flow/TLSConfig.actor.cpp
+++ b/flow/TLSConfig.cpp
@@ -1,5 +1,5 @@
 /*
- * TLSConfig.actor.cpp
+ * TLSConfig.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -19,7 +19,7 @@
  */
 
 #define PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #undef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP
 
 // To force typeinfo to only be emitted once.
@@ -58,10 +58,8 @@ TLSPolicy::~TLSPolicy() {}
 #include "flow/Trace.h"
 #include "flow/genericactors.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
 std::vector<std::string> LoadedTLSConfig::getVerifyPeers() const {
-	if (tlsVerifyPeers.size()) {
+	if (!tlsVerifyPeers.empty()) {
 		return tlsVerifyPeers;
 	}
 
@@ -78,7 +76,7 @@ bool LoadedTLSConfig::getDisablePlainTextConnection() const {
 }
 
 std::string LoadedTLSConfig::getPassword() const {
-	if (tlsPassword.size()) {
+	if (!tlsPassword.empty()) {
 		return tlsPassword;
 	}
 
@@ -132,18 +130,18 @@ void ConfigureSSLContext(const LoadedTLSConfig& loaded, boost::asio::ssl::contex
 		                                  size_t, boost::asio::ssl::context::password_purpose) { return password; });
 
 		const std::string& CABytes = loaded.getCABytes();
-		if (CABytes.size()) {
+		if (!CABytes.empty()) {
 			context.add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size()));
 		}
 
 		const std::string& keyBytes = loaded.getKeyBytes();
-		if (keyBytes.size()) {
+		if (!keyBytes.empty()) {
 			context.use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()),
 			                        boost::asio::ssl::context::pem);
 		}
 
 		const std::string& certBytes = loaded.getCertificateBytes();
-		if (certBytes.size()) {
+		if (!certBytes.empty()) {
 			context.use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size()));
 		}
 	} catch (boost::system::system_error& e) {
@@ -182,7 +180,7 @@ void ConfigureSSLStream(Reference<TLSPolicy> policy,
 }
 
 std::string TLSConfig::getCertificatePathSync() const {
-	if (tlsCertPath.size()) {
+	if (!tlsCertPath.empty()) {
 		return tlsCertPath;
 	}
 
@@ -200,7 +198,7 @@ std::string TLSConfig::getCertificatePathSync() const {
 }
 
 std::string TLSConfig::getKeyPathSync() const {
-	if (tlsKeyPath.size()) {
+	if (!tlsKeyPath.empty()) {
 		return tlsKeyPath;
 	}
 
@@ -218,7 +216,7 @@ std::string TLSConfig::getKeyPathSync() const {
 }
 
 std::string TLSConfig::getCAPathSync() const {
-	if (tlsCAPath.size()) {
+	if (!tlsCAPath.empty()) {
 		return tlsCAPath;
 	}
 
@@ -247,7 +245,7 @@ LoadedTLSConfig TLSConfig::loadSync() const {
 	LoadedTLSConfig loaded;
 
 	const std::string certPath = getCertificatePathSync();
-	if (certPath.size()) {
+	if (!certPath.empty()) {
 		try {
 			loaded.tlsCertBytes = readFileBytes(certPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
 		} catch (Error& e) {
@@ -259,7 +257,7 @@ LoadedTLSConfig TLSConfig::loadSync() const {
 	}
 
 	const std::string keyPath = getKeyPathSync();
-	if (keyPath.size()) {
+	if (!keyPath.empty()) {
 		try {
 			loaded.tlsKeyBytes = readFileBytes(keyPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
 		} catch (Error& e) {
@@ -271,7 +269,7 @@ LoadedTLSConfig TLSConfig::loadSync() const {
 	}
 
 	const std::string CAPath = getCAPathSync();
-	if (CAPath.size()) {
+	if (!CAPath.empty()) {
 		try {
 			loaded.tlsCABytes = readFileBytes(CAPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
 		} catch (Error& e) {
@@ -297,44 +295,43 @@ TLSPolicy::TLSPolicy(const LoadedTLSConfig& loaded, std::function<void()> on_fai
 
 // And now do the same thing, but async...
 
-ACTOR static Future<Void> readEntireFile(std::string filename, std::string* destination) {
-	state Reference<IAsyncFile> file =
-	    wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
-	state int64_t filesize = wait(file->size());
+static Future<Void> readEntireFile(std::string filename, std::string* destination) {
+	Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
+	    filename, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0);
+	int64_t filesize = co_await file->size();
 	if (filesize > FLOW_KNOBS->CERT_FILE_MAX_SIZE) {
 		throw file_too_large();
 	}
 	destination->resize(filesize);
-	wait(success(file->read(&((*destination)[0]), filesize, 0)));
-	return Void();
+	co_await file->read(&((*destination)[0]), filesize, 0);
 }
 
-ACTOR Future<LoadedTLSConfig> TLSConfig::loadAsync(const TLSConfig* self) {
-	state LoadedTLSConfig loaded;
-	state std::vector<Future<Void>> reads;
+Future<LoadedTLSConfig> TLSConfig::loadAsync(const TLSConfig* self) {
+	LoadedTLSConfig loaded;
+	std::vector<Future<Void>> reads;
 
-	state int32_t certIdx = -1;
-	state int32_t keyIdx = -1;
-	state int32_t caIdx = -1;
+	int32_t certIdx = -1;
+	int32_t keyIdx = -1;
+	int32_t caIdx = -1;
 
-	state std::string certPath = self->getCertificatePathSync();
-	if (certPath.size()) {
+	std::string certPath = self->getCertificatePathSync();
+	if (!certPath.empty()) {
 		reads.push_back(readEntireFile(certPath, &loaded.tlsCertBytes));
 		certIdx = reads.size() - 1;
 	} else {
 		loaded.tlsCertBytes = self->tlsCertBytes;
 	}
 
-	state std::string keyPath = self->getKeyPathSync();
-	if (keyPath.size()) {
+	std::string keyPath = self->getKeyPathSync();
+	if (!keyPath.empty()) {
 		reads.push_back(readEntireFile(keyPath, &loaded.tlsKeyBytes));
 		keyIdx = reads.size() - 1;
 	} else {
 		loaded.tlsKeyBytes = self->tlsKeyBytes;
 	}
 
-	state std::string CAPath = self->getCAPathSync();
-	if (CAPath.size()) {
+	std::string CAPath = self->getCAPathSync();
+	if (!CAPath.empty()) {
 		reads.push_back(readEntireFile(CAPath, &loaded.tlsCABytes));
 		caIdx = reads.size() - 1;
 	} else {
@@ -342,7 +339,7 @@ ACTOR Future<LoadedTLSConfig> TLSConfig::loadAsync(const TLSConfig* self) {
 	}
 
 	try {
-		wait(waitForAll(reads));
+		co_await waitForAll(reads);
 	} catch (Error& e) {
 		if (certIdx != -1 && reads[certIdx].isError()) {
 			fprintf(stderr, "Warning: Error reading TLS Certificate [%s]: %s\n", certPath.c_str(), e.what());
@@ -362,7 +359,7 @@ ACTOR Future<LoadedTLSConfig> TLSConfig::loadAsync(const TLSConfig* self) {
 	loaded.endpointType = self->endpointType;
 	loaded.tlsDisablePlainTextConnection = self->tlsDisablePlainTextConnection;
 
-	return loaded;
+	co_return loaded;
 }
 
 std::string TLSPolicy::ErrorString(boost::system::error_code e) {
@@ -739,7 +736,7 @@ class PeerVerifier {
 	bool isOk() const noexcept { return m_verified; }
 	bool isErr() const noexcept { return !isOk(); }
 	const std::vector<std::string>& getFailureReasons() const noexcept { return m_failureReasons; }
-	const std::string_view getSuccessReason() const noexcept { return m_successReason; }
+	std::string_view getSuccessReason() const noexcept { return m_successReason; }
 };
 
 std::string getX509Name(const X509_NAME* name) {
@@ -947,7 +944,7 @@ bool PeerVerifier::verifyRule(const TLSPolicy::Rule& rule) {
 }
 
 bool PeerVerifier::verify() {
-	if (m_rules.size() == 0) {
+	if (m_rules.empty()) {
 		m_successReason = "No rule defined"sv;
 		return true;
 	}
@@ -1037,6 +1034,7 @@ bool TLSPolicy::verify_peer(bool preverified, X509_STORE_CTX* store_ctx, const N
 
 	return verifier.isOk();
 }
+
 struct CryptoLibHandle {
 	void* lib = nullptr;
 	void* func = nullptr;
@@ -1111,4 +1109,4 @@ void TLSConfig::setPassword(const std::string& password) {
 	} else {
 		tlsPassword = password;
 	}
-}
+}
\ No newline at end of file
diff --git a/flow/ThreadHelper.actor.cpp b/flow/ThreadHelper.cpp
similarity index 81%
rename from flow/ThreadHelper.actor.cpp
rename to flow/ThreadHelper.cpp
index 686dfc128db..7a95d1276bc 100644
--- a/flow/ThreadHelper.actor.cpp
+++ b/flow/ThreadHelper.cpp
@@ -1,5 +1,5 @@
 /*
- * ThreadHelper.actor.cpp
+ * ThreadHelper.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,15 +18,14 @@
  * limitations under the License.
  */
 
-#include <string>
 #include <thread>
 
 #include "flow/flow.h"
 #include "flow/network.h"
 #include "flow/ThreadHelper.actor.h"
+#include "flow/Coroutines.h"
 #include "flow/Error.h"
 #include "flow/UnitTest.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 ThreadCallback* ThreadCallback::addCallback(ThreadCallback* cb) {
 	return (new ThreadMultiCallback())->addCallback(this)->addCallback(cb);
@@ -40,7 +39,7 @@ struct ThreadFutureSendObj {
 
 // A simple thread object that cancels the threadFuture
 struct ThreadFutureCancelObj {
-	ThreadFutureCancelObj(ThreadFuture<Void> f) : f(f) {}
+	explicit ThreadFutureCancelObj(ThreadFuture<Void> f) : f(f) {}
 	void operator()() { f.cancel(); }
 	ThreadFuture<Void> f;
 };
@@ -49,29 +48,27 @@ struct ThreadFutureCancelObj {
 TEST_CASE("/flow/safeThreadFutureToFuture/Send") {
 	// std::thread is not working in simulation at present, disable this in simulation
 	if (g_network->isSimulated())
-		return Void();
+		co_return;
 	auto* tsav = new ThreadSingleAssignmentVar<Void>;
-	state std::thread thread = std::thread{ ThreadFutureSendObj{ tsav } };
+	std::thread thread = std::thread{ ThreadFutureSendObj{ tsav } };
 	ThreadFuture<Void> f(tsav);
 	// change this to unsafeThreadFutureToFuture will get a data-race failure
-	wait(safeThreadFutureToFuture(f));
+	co_await safeThreadFutureToFuture(f);
 	thread.join();
-	return Void();
 }
 
 // Test the case where the underlying threadFuture is cancelled
 TEST_CASE("/flow/safeThreadFutureToFuture/Cancel") {
 	// std::thread is not working in simulation at present, disable this in simulation
 	if (g_network->isSimulated())
-		return Void();
+		co_return;
 	ThreadFuture<Void> f = onMainThread([]() -> Future<Void> { return Never(); });
-	state std::thread thread = std::thread{ ThreadFutureCancelObj(f) };
+	std::thread thread = std::thread{ ThreadFutureCancelObj(f) };
 	try {
-		wait(safeThreadFutureToFuture(f)); // this actor should get actor_cancelled
+		co_await safeThreadFutureToFuture(f); // this actor should get actor_cancelled
 		ASSERT(false);
 	} catch (Error& e) {
 		ASSERT(e.code() == error_code_actor_cancelled);
 	}
 	thread.join();
-	return Void();
 }
diff --git a/flow/ThreadPrimitives.cpp b/flow/ThreadPrimitives.cpp
index a239599dc52..19873368ae4 100644
--- a/flow/ThreadPrimitives.cpp
+++ b/flow/ThreadPrimitives.cpp
@@ -19,12 +19,6 @@
  */
 
 #include "flow/ThreadPrimitives.h"
-#include "flow/Trace.h"
-#include <stdint.h>
-#include <iostream>
-#include <errno.h>
-#include <string.h>
-#include <stdio.h>
 
 #ifdef _WIN32
 #include <windows.h>
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index cbb9d83f3ad..68d8fe45462 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -20,30 +20,27 @@
 
 #include "flow/ActorContext.h"
 #include "flow/Trace.h"
-#include "flow/FileTraceLogWriter.h"
+#include "FileTraceLogWriter.h"
 #include "flow/Knobs.h"
-#include "flow/XmlTraceLogFormatter.h"
-#include "flow/JsonTraceLogFormatter.h"
+#include "XmlTraceLogFormatter.h"
+#include "JsonTraceLogFormatter.h"
 #include "flow/flow.h"
 #include "flow/DeterministicRandom.h"
 #include "flow/ProcessEvents.h"
 #include <exception>
-#include <stdlib.h>
 #include <stdarg.h>
 #include <cctype>
 #include <time.h>
 #include <set>
-#include <unordered_set>
 #include <string_view>
 #include <iomanip>
 #include "flow/IThreadPool.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/FastRef.h"
-#include "flow/EventTypes.actor.h"
-#include "flow/TDMetric.actor.h"
-#include "flow/MetricSample.h"
+#include "EventTypes.h"
+#include "flow/TDMetric.h"
+#include "MetricSample.h"
 #include "flow/network.h"
-#include "flow/SimBugInjector.h"
 
 #ifdef _WIN32
 #include <windows.h>
@@ -64,7 +61,7 @@ unsigned tracedLines = 0;
 thread_local int failedLineOverflow = 0;
 bool g_traceProcessEvents = false;
 
-ITraceLogIssuesReporter::~ITraceLogIssuesReporter() {}
+ITraceLogIssuesReporter::~ITraceLogIssuesReporter() = default;
 
 struct SuppressionMap {
 	struct SuppressionInfo {
@@ -79,7 +76,7 @@ struct SuppressionMap {
 	int64_t checkAndInsertSuppression(std::string type, double duration, bool& suppress) {
 		ASSERT(g_network);
 		if (suppressionMap.size() >= FLOW_KNOBS->MAX_TRACE_SUPPRESSIONS) {
-			TraceEvent(SevWarnAlways, "ClearingTraceSuppressionMap").log();
+			TraceEvent(SevWarnAlways, "ClearingTraceSuppressionMap").writeEvent();
 			suppressionMap.clear();
 		}
 
@@ -134,11 +131,11 @@ struct TraceLog {
 	uint64_t rollsize;
 	Mutex mutex;
 
-	EventMetricHandle<TraceEventNameID> SevErrorNames;
-	EventMetricHandle<TraceEventNameID> SevWarnAlwaysNames;
-	EventMetricHandle<TraceEventNameID> SevWarnNames;
-	EventMetricHandle<TraceEventNameID> SevInfoNames;
-	EventMetricHandle<TraceEventNameID> SevDebugNames;
+	EventMetricHandle<TraceEventNameIDDescriptor> SevErrorNames;
+	EventMetricHandle<TraceEventNameIDDescriptor> SevWarnAlwaysNames;
+	EventMetricHandle<TraceEventNameIDDescriptor> SevWarnNames;
+	EventMetricHandle<TraceEventNameIDDescriptor> SevInfoNames;
+	EventMetricHandle<TraceEventNameIDDescriptor> SevDebugNames;
 
 	struct RoleInfo {
 		std::map<std::string, int> roles;
@@ -146,7 +143,7 @@ struct TraceLog {
 
 		void refreshRolesString() {
 			rolesString = "";
-			for (auto itr : roles) {
+			for (const auto& itr : roles) {
 				if (!rolesString.empty()) {
 					rolesString += ",";
 				}
@@ -261,7 +258,7 @@ struct TraceLog {
 		struct WriteBuffer final : TypedAction<WriterThread, WriteBuffer> {
 			std::vector<TraceEventFields> events;
 
-			WriteBuffer(std::vector<TraceEventFields> events) : events(events) {}
+			explicit WriteBuffer(std::vector<TraceEventFields> events) : events(events) {}
 			double getTimeEstimate() const override { return .001; }
 		};
 		void action(WriteBuffer& a) {
@@ -277,7 +274,7 @@ struct TraceLog {
 		struct Ping final : TypedAction<WriterThread, Ping> {
 			ThreadReturnPromise<Void> ack;
 
-			explicit Ping() {};
+			explicit Ping() = default;
 			double getTimeEstimate() const override { return 0; }
 		};
 		void action(Ping& ping) {
@@ -328,7 +325,7 @@ struct TraceLog {
 		    issues));
 
 		if (g_network->isSimulated())
-			writer = Reference<IThreadPool>(new DummyThreadPool());
+			writer = makeReference<DummyThreadPool>();
 		else
 			writer = createGenericThreadPool();
 		writer->addThread(new WriterThread(barriers, logWriter, formatter), "fdb-trace-log");
@@ -375,7 +372,7 @@ struct TraceLog {
 		fields.addField("LogGroup", logGroup);
 
 		RoleInfo const& r = mutateRoleInfo();
-		if (r.rolesString.size() > 0) {
+		if (!r.rolesString.empty()) {
 			fields.addField("Roles", r.rolesString);
 		}
 
@@ -435,7 +432,7 @@ struct TraceLog {
 	void logMetrics(int severity, const char* name, UID id, uint64_t event_ts) {
 		ASSERT(TraceEvent::isNetworkThread() && logTraceEventMetrics);
 
-		EventMetricHandle<TraceEventNameID>* m = nullptr;
+		EventMetricHandle<TraceEventNameIDDescriptor>* m = nullptr;
 		switch (severity) {
 		case SevError:
 			m = &SevErrorNames;
@@ -469,7 +466,7 @@ struct TraceLog {
 
 		MutexHolder hold(mutex);
 		bool roll = false;
-		if (!eventBuffer.size())
+		if (eventBuffer.empty())
 			return Void(); // SOMEDAY: maybe we still roll the tracefile here?
 
 		if (rollsize && bufferLength + loggedLength > rollsize) // SOMEDAY: more conditions to roll
@@ -577,7 +574,7 @@ struct TraceLog {
 
 	void addUniversalTraceField(const std::string& name, const std::string& value) {
 		MutexHolder holder(mutex);
-		ASSERT(universalFields.count(name) == 0);
+		ASSERT(!universalFields.contains(name));
 		universalFields[name] = value;
 	}
 
@@ -685,17 +682,17 @@ bool traceFormatImpl(std::string& format) {
 	std::transform(format.begin(), format.end(), format.begin(), ::tolower);
 	if (format == "xml") {
 		if (!validate) {
-			g_traceLog.formatter = Reference<ITraceLogFormatter>(new XmlTraceLogFormatter());
+			g_traceLog.formatter = makeReference<XmlTraceLogFormatter>();
 		}
 		return true;
 	} else if (format == "json") {
 		if (!validate) {
-			g_traceLog.formatter = Reference<ITraceLogFormatter>(new JsonTraceLogFormatter());
+			g_traceLog.formatter = makeReference<JsonTraceLogFormatter>();
 		}
 		return true;
 	} else {
 		if (!validate) {
-			g_traceLog.formatter = Reference<ITraceLogFormatter>(new XmlTraceLogFormatter());
+			g_traceLog.formatter = makeReference<XmlTraceLogFormatter>();
 		}
 		return false;
 	}
@@ -1169,6 +1166,22 @@ BaseTraceEvent& BaseTraceEvent::detailf(std::string key, const char* valueFormat
 	}
 	return *this;
 }
+
+BaseTraceEvent& BaseTraceEvent::log(const char* valueFormat, ...) {
+	if (enabled) {
+		va_list args;
+		va_start(args, valueFormat);
+		std::string value;
+		int result = vsformat(value, valueFormat, args);
+		va_end(args);
+
+		ASSERT(result >= 0);
+		std::string key = "LogMessage";
+		detailImpl(std::move(key), std::move(value));
+	}
+	return *this;
+}
+
 BaseTraceEvent& BaseTraceEvent::detailfNoMetric(std::string&& key, const char* valueFormat, ...) {
 	if (enabled) {
 		va_list args;
@@ -1189,7 +1202,7 @@ BaseTraceEvent& BaseTraceEvent::detailfNoMetric(std::string&& key, const char* v
 BaseTraceEvent& BaseTraceEvent::trackLatest(const std::string& trackingKey) {
 	ASSERT(!logged);
 	this->trackingKey = trackingKey;
-	ASSERT(this->trackingKey.size() != 0 && this->trackingKey[0] != '/' && this->trackingKey[0] != '\\');
+	ASSERT(!this->trackingKey.empty() && this->trackingKey[0] != '/' && this->trackingKey[0] != '\\');
 	return *this;
 }
 
@@ -1315,7 +1328,7 @@ BaseTraceEvent& BaseTraceEvent::backtrace(const std::string& prefix) {
 	return detail(prefix + "Backtrace", platform::get_backtrace());
 }
 
-void BaseTraceEvent::log() {
+void BaseTraceEvent::writeEvent() {
 	if (!logged) {
 		init();
 		++g_allocation_tracing_disabled;
@@ -1379,12 +1392,12 @@ void BaseTraceEvent::log() {
 BaseTraceEvent::~BaseTraceEvent() {
 	//  fprintf(stderr, "[%s:%d](%s) ~BaseTraceEvent [%p] \n", __FILE_NAME__, __LINE__, __FUNCTION__,
 	//          this);
-	log();
+	writeEvent();
 	if (failedLineOverflow == 1) {
 		failedLineOverflow = 2;
 		auto msg = fmt::format("Traced {} lines", tracedLines);
 		ProcessEvents::trigger("TracedTooManyLines"_sr, StringRef(msg), test_failed());
-		TraceEvent(SevError, "TracedTooManyLines").log();
+		TraceEvent(SevError, "TracedTooManyLines").writeEvent();
 		crashAndDie();
 	}
 }
diff --git a/flow/WipedString.cpp b/flow/WipedString.cpp
index e9200eca30b..78e441b836a 100644
--- a/flow/WipedString.cpp
+++ b/flow/WipedString.cpp
@@ -1,5 +1,4 @@
 #include "flow/FastAlloc.h"
-#include "flow/ScopeExit.h"
 #include "flow/UnitTest.h"
 #include "flow/WipedString.h"
 #include "flow/ObjectSerializer.h"
diff --git a/flow/WriteOnlySet.actor.cpp b/flow/WriteOnlySet.cpp
similarity index 96%
rename from flow/WriteOnlySet.actor.cpp
rename to flow/WriteOnlySet.cpp
index fed388942a8..fbb20c96638 100644
--- a/flow/WriteOnlySet.actor.cpp
+++ b/flow/WriteOnlySet.cpp
@@ -1,5 +1,5 @@
 /*
- * WriteOnlySet.actor.cpp
+ * WriteOnlySet.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,7 +26,6 @@
 #include <chrono>
 #include <random>
 #include <thread>
-#include "flow/actorcompiler.h" // has to be last include
 
 #ifdef ENABLE_SAMPLING
 template <class T, class IndexType, IndexType CAPACITY>
@@ -176,9 +175,9 @@ using TestSet = WriteOnlySet<TestObject, unsigned, 128>;
 using Clock = std::chrono::steady_clock;
 
 // An actor that can join a set of threads in an async way.
-ACTOR Future<Void> threadjoiner(std::shared_ptr<std::vector<std::thread>> threads, std::shared_ptr<TestSet> set) {
-	loop {
-		wait(delay(0.1));
+Future<Void> threadjoiner(std::shared_ptr<std::vector<std::thread>> threads, std::shared_ptr<TestSet> set) {
+	while (true) {
+		co_await delay(0.1);
 		for (unsigned i = 0;;) {
 			if (threads->size() == i) {
 				break;
@@ -197,7 +196,7 @@ ACTOR Future<Void> threadjoiner(std::shared_ptr<std::vector<std::thread>> thread
 		if (threads->empty()) {
 			set->copy();
 			ASSERT(instanceCounter.load() == 0);
-			return Void();
+			co_return;
 		}
 	}
 }
@@ -255,7 +254,7 @@ void writer(std::shared_ptr<TestSet> set, std::chrono::seconds runFor) {
 TEST_CASE("/flow/WriteOnlySet") {
 	if (g_network->isSimulated()) {
 		// This test is not deterministic, so we shouldn't run it in simulation
-		return Void();
+		co_return;
 	}
 	auto set = std::make_shared<TestSet>();
 	auto threads = std::make_shared<std::vector<std::thread>>();
@@ -264,13 +263,13 @@ TEST_CASE("/flow/WriteOnlySet") {
 		threads->emplace_back([set, runFor]() { writer(set, runFor); });
 	}
 	threads->emplace_back([set, runFor]() { testCopier(set, runFor); });
-	wait(threadjoiner(threads, set));
+	co_await threadjoiner(threads, set);
 	TraceEvent("WriteOnlySetTestResult")
 	    .detail("Inserts", numInserts.load())
 	    .detail("Erases", numErase.load())
 	    .detail("Copies", numCopied.load())
 	    .detail("LockedErase", numLockedErase.load());
-	return Void();
+	co_return;
 }
 } // namespace
 #endif
diff --git a/flow/XmlTraceLogFormatter.cpp b/flow/XmlTraceLogFormatter.cpp
index af5173cd356..dbd57acdf00 100644
--- a/flow/XmlTraceLogFormatter.cpp
+++ b/flow/XmlTraceLogFormatter.cpp
@@ -19,7 +19,9 @@
  */
 
 #include "flow/flow.h"
-#include "flow/XmlTraceLogFormatter.h"
+#include "flow/ScopeExit.h"
+#include "flow/UnitTest.h"
+#include "XmlTraceLogFormatter.h"
 
 void XmlTraceLogFormatter::addref() {
 	ReferenceCounted<XmlTraceLogFormatter>::addref();
@@ -41,6 +43,8 @@ const char* XmlTraceLogFormatter::getFooter() const {
 	return "</Trace>\r\n";
 }
 
+static Severity xmlIllegalCharSeverity = SevWarnAlways;
+
 void XmlTraceLogFormatter::escape(std::ostringstream& oss, std::string source) const {
 	for (;;) {
 		int index = source.find_first_of(std::string({ '&', '"', '<', '>', '\r', '\n', '\0' }));
@@ -61,7 +65,7 @@ void XmlTraceLogFormatter::escape(std::ostringstream& oss, std::string source) c
 			oss << " ";
 		} else if (source[index] == '\0') {
 			oss << " ";
-			TraceEvent(SevWarnAlways, "StrippedIllegalCharacterFromTraceEvent")
+			TraceEvent(xmlIllegalCharSeverity, "StrippedIllegalCharacterFromTraceEvent")
 			    .detail("Source", StringRef(source).printable())
 			    .detail("Character", StringRef(source.substr(index, 1)).printable());
 		} else {
@@ -78,7 +82,7 @@ std::string XmlTraceLogFormatter::formatEvent(const TraceEventFields& fields) co
 	std::ostringstream oss;
 	oss << "<Event ";
 
-	for (auto itr : fields) {
+	for (const auto& itr : fields) {
 		escape(oss, itr.first);
 		oss << "=\"";
 		escape(oss, itr.second);
@@ -88,3 +92,52 @@ std::string XmlTraceLogFormatter::formatEvent(const TraceEventFields& fields) co
 	oss << "/>\n";
 	return std::move(oss).str();
 }
+
+// Test case to ensure that TraceEvent details with XML junk get escaped.
+//
+// This doesn't attempt to ensure that whatever we log is valid XML.
+// We might be logging a response from a remote server which may or
+// may not be 100% well formed.  In this situation we want to know
+// exactly what it is telling us, so we escape meta characters but log
+// the string regardless of its parseability.
+TEST_CASE("/flow/XmlTraceEscape") {
+	std::string garbage_input = "<root attr=\"value&weird\">\n"
+	                            "  <child>Some <b>bold</b> & \"quoted\" text</child>\n"
+	                            "  <child2><![CDATA[<not>really</not> xml]]></child2>\n"
+	                            "  <empty/>\n"
+	                            "  <<<mismatched>>>>>>><<\n"
+	                            "  <weird attr='><&\"'><inner>><<&&\"\"</inner></weird>\n"
+	                            "  <!-- comment with <tags> & stuff -->\n"
+	                            "  <deep>\n"
+	                            "    <level1>\n"
+	                            "      <level2>\n"
+	                            "        <level3 attr=\"a&b<c>d\">text & more text</level3>\n"
+	                            "      </level2>\n"
+	                            "    </level1>\n"
+	                            "  </deep>\n"
+	                            "</root>"
+	                            "more junk\n"
+	                            "<<even more>\n";
+
+	auto defaultSeverity = xmlIllegalCharSeverity;
+	ScopeExit cleanup([&]() { xmlIllegalCharSeverity = defaultSeverity; });
+
+	xmlIllegalCharSeverity = SevInfo;
+
+	TraceEvent event("TestEvent");
+	event.detail("Garbage", garbage_input);
+
+	XmlTraceLogFormatter formatter;
+	std::ostringstream output;
+	for (const auto& f : event.getFields()) {
+		formatter.escape(output, f.first);
+		formatter.escape(output, f.second);
+	}
+
+	std::string str = output.str();
+	for (const char ch : str) {
+		ASSERT(ch != '<' && ch != '>');
+	}
+
+	return Void();
+}
diff --git a/flow/include/flow/XmlTraceLogFormatter.h b/flow/XmlTraceLogFormatter.h
similarity index 100%
rename from flow/include/flow/XmlTraceLogFormatter.h
rename to flow/XmlTraceLogFormatter.h
diff --git a/flow/actorcompiler/ActorCompiler.cs b/flow/actorcompiler/ActorCompiler.cs
index 7a40cb2776c..adf138eb27b 100644
--- a/flow/actorcompiler/ActorCompiler.cs
+++ b/flow/actorcompiler/ActorCompiler.cs
@@ -237,73 +237,6 @@ class CallbackVar : StateVar
         public int CallbackGroup;
     }
 
-    class DescrCompiler
-    {
-        Descr descr;
-        string memberIndentStr;
-
-        public DescrCompiler(Descr descr, int braceDepth)
-        {
-            this.descr = descr;
-            this.memberIndentStr = new string('\t', braceDepth);
-        }
-        public void Write(TextWriter writer, out int lines)
-        {
-            lines = 0;
-
-            writer.WriteLine(memberIndentStr + "template<> struct Descriptor<struct {0}> {{", descr.name);
-            writer.WriteLine(memberIndentStr + "\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", descr.name);
-            writer.WriteLine(memberIndentStr + "\ttypedef {0} type;", descr.name);
-            lines += 3;
-
-            foreach (var dec in descr.body)
-            {
-                writer.WriteLine(memberIndentStr + "\tstruct {0}Descriptor {{", dec.name);
-                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef name() {{ return \"{0}\"_sr; }}", dec.name);
-                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", dec.type);
-                writer.WriteLine(memberIndentStr + "\t\tstatic StringRef comment() {{ return \"{0}\"_sr; }}", dec.comment);
-                writer.WriteLine(memberIndentStr + "\t\ttypedef {0} type;", dec.type);
-                writer.WriteLine(memberIndentStr + "\t\tstatic inline type get({0}& from);", descr.name);
-                writer.WriteLine(memberIndentStr + "\t};");
-                lines += 7;
-            }
-
-            writer.Write(memberIndentStr + "\ttypedef std::tuple<");
-            bool FirstDesc = true;
-            foreach (var dec in descr.body)
-            {
-                if (!FirstDesc)
-                    writer.Write(",");
-                writer.Write("{0}Descriptor", dec.name);
-                FirstDesc = false;
-            }
-            writer.Write("> fields;\n");
-            writer.WriteLine(memberIndentStr + "\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size<fields>::value>::type field_indexes;");
-            writer.WriteLine(memberIndentStr + "};");
-            if(descr.superClassList != null)
-                writer.WriteLine(memberIndentStr + "struct {0} : {1} {{", descr.name, descr.superClassList);
-            else
-                writer.WriteLine(memberIndentStr + "struct {0} {{", descr.name);
-            lines += 4;
-
-            foreach (var dec in descr.body)
-            {
-                writer.WriteLine(memberIndentStr + "\t{0} {1}; //{2}", dec.type, dec.name, dec.comment);
-                lines++;
-            }
-
-            writer.WriteLine(memberIndentStr + "};");
-            lines++;
-
-            foreach (var dec in descr.body)
-            {
-                writer.WriteLine(memberIndentStr + "{0} Descriptor<{1}>::{2}Descriptor::get({1}& from) {{ return from.{2}; }}", dec.type, descr.name, dec.name);
-                lines++;
-            }
-
-        }
-    }
-
     class ActorCompiler
     {
         Actor actor;
@@ -869,7 +802,7 @@ void CompileStatement(ChooseStatement stmt, Context cx)
 
             var exitFunc = getFunction("exitChoose", "");
             exitFunc.returnType = "void";
-            exitFunc.WriteLine("if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", This);
+            exitFunc.WriteLine("if (actorWaitStateIsWaiting({0}->actor_wait_state)) {0}->actor_wait_state = ACTOR_WAIT_STATE_NOT_WAITING;", This);
             foreach(var ch in choices)
                 exitFunc.WriteLine("{0}->{1}::remove();", This, ch.CallbackTypeInStateClass);
             exitFunc.endIsUnreachable = true;
@@ -991,7 +924,7 @@ void CompileStatement(ChooseStatement stmt, Context cx)
                     firstChoice = false;
                     LineNumber(cx.target, stmt.FirstSourceLine);
                     if (actor.IsCancellable())
-                        cx.target.WriteLine("if ({1}->actor_wait_state < 0) return {0};", cx.catchFErr.call("actor_cancelled()", AdjustLoopDepth(cx.tryLoopDepth)), This);
+                        cx.target.WriteLine("if (actorWaitStateIsCancelled({1}->actor_wait_state)) return {0};", cx.catchFErr.call("actor_cancelled()", AdjustLoopDepth(cx.tryLoopDepth)), This);
                 }
 
                 cx.target.WriteLine("if ({0}.isReady()) {{ if ({0}.isError()) return {2}; else return {1}; }};", ch.Future, ch.Body.call(ch.Future + "." + getFunc + "()", "loopDepth"), cx.catchFErr.call(ch.Future + ".getError()", AdjustLoopDepth(cx.tryLoopDepth)));
@@ -1330,7 +1263,7 @@ void WriteCancelFunc(TextWriter writer)
                 };
                 cancelFunc.Indent(codeIndent);
                 cancelFunc.WriteLine("auto wait_state = this->actor_wait_state;");
-                cancelFunc.WriteLine("this->actor_wait_state = -1;");
+                cancelFunc.WriteLine("this->actor_wait_state = ACTOR_WAIT_STATE_CANCELLED;");
                 cancelFunc.WriteLine("switch (wait_state) {");
                 int lastGroup = -1;
                 foreach (var cb in callbacks.OrderBy(cb => cb.CallbackGroup))
diff --git a/flow/actorcompiler/ActorParser.cs b/flow/actorcompiler/ActorParser.cs
index a130eb9b992..36392627d0c 100644
--- a/flow/actorcompiler/ActorParser.cs
+++ b/flow/actorcompiler/ActorParser.cs
@@ -381,20 +381,6 @@ public void Write(System.IO.TextWriter writer, string destFileName)
                         outLine++;
                     }
                 }
-                else if (tokens[i].Value == "DESCR")
-                {
-                    int end;
-                    var descr = ParseDescr(i, out end);
-                    int lines;
-                    new DescrCompiler(descr, tokens[i].BraceDepth).Write(writer, out lines);
-                    i = end;
-                    outLine += lines;
-                    if (i != tokens.Length && LineNumbersEnabled)
-                    {
-                        writer.WriteLine("#line {0} \"{1}\"", tokens[i].SourceLine, sourceFile);
-                        outLine++;
-                    }
-                }
                 else if (tokens[i].Value == "class" || tokens[i].Value == "struct" || tokens[i].Value == "union")
                 {
                     writer.Write(tokens[i].Value);
@@ -518,20 +504,6 @@ VarDeclaration ParseVarDeclaration(TokenRange tokens)
         readonly Func<Token, bool> Whitespace = (Token t) => t.IsWhitespace;
         readonly Func<Token, bool> NonWhitespace = (Token t) => !t.IsWhitespace;
 
-        void ParseDescrHeading(Descr descr, TokenRange toks)
-        {
-            toks.First(NonWhitespace).Assert("non-struct DESCR!", t => t.Value == "struct");
-            toks = toks.SkipWhile(Whitespace).Skip(1).SkipWhile(Whitespace);
-
-            var colon = toks.FirstOrDefault(t => t.Value == ":");
-            if (colon != null)
-            {
-                descr.superClassList = str(range(colon.Position + 1, toks.End)).Trim();
-                toks = range(toks.Begin, colon.Position);
-            }
-            descr.name = str(toks).Trim();
-        }
-
         void ParseTestCaseHeading(Actor actor, TokenRange toks)
         {
             actor.isStatic = true;
@@ -894,22 +866,6 @@ void ParseCatchStatement(TokenRange toks, Statement prevStatement)
 
         static readonly HashSet<string> IllegalKeywords = new HashSet<string> { "goto", "do", "finally", "__if_exists", "__if_not_exists" };
 
-        void ParseDeclaration(TokenRange toks, List<Declaration> declarations)
-        {
-            Declaration dec = new Declaration();
-
-            Token delim = toks.First(t => t.Value == ";");
-            var nameRange = range(toks.Begin, delim.Position).RevSkipWhile(Whitespace).RevTakeWhile(NonWhitespace);
-            var typeRange = range(toks.Begin, nameRange.Begin);
-            var commentRange = range(delim.Position + 1, toks.End);
-
-            dec.name = str(nameRange).Trim();
-            dec.type = str(typeRange).Trim();
-            dec.comment = str(commentRange).Trim().TrimStart('/');
-
-            declarations.Add(dec);
-        }
-
         void ParseStatement(TokenRange toks, List<Statement> statements)
         {
             toks = toks.SkipWhile(Whitespace);
@@ -971,31 +927,6 @@ Statement ParseCompoundStatement(TokenRange toks)
             }
         }
 
-        List<Declaration> ParseDescrCodeBlock(TokenRange toks)
-        {
-            List<Declaration> declarations = new List<Declaration>();
-            while (true)
-            {
-                Token delim = toks.FirstOrDefault(t => t.Value == ";");
-                if (delim == null)
-                    break;
-
-                int pos = delim.Position + 1;
-                var potentialComment = range(pos, toks.End).SkipWhile(t => t.Value == "\t" || t.Value == " ");
-                if (!potentialComment.IsEmpty && potentialComment.First().Value.StartsWith("//"))
-                {
-                    pos = potentialComment.First().Position + 1;
-                }
-
-                ParseDeclaration(range(toks.Begin, pos), declarations);
-
-                toks = range(pos, toks.End);
-            }
-            if (!toks.All(Whitespace))
-                throw new Error(toks.First(NonWhitespace).SourceLine, "Trailing unterminated statement in code block");
-            return declarations;
-        }
-
         CodeBlock ParseCodeBlock(TokenRange toks)
         {
             List<Statement> statements = new List<Statement>();
@@ -1022,21 +953,6 @@ TokenRange range(int beginPos, int endPos)
             return new TokenRange(tokens, beginPos, endPos);
         }
 
-        Descr ParseDescr(int pos, out int end)
-        {
-            var descr = new Descr();
-            var toks = range(pos + 1, tokens.Length);
-            var heading = toks.TakeWhile(t => t.Value != "{");
-            var body = range(heading.End + 1, tokens.Length)
-                .TakeWhile(t => t.BraceDepth > toks.First().BraceDepth || t.Value == ";" ); //assumes no whitespace between the last "}" and the ";"
-
-            ParseDescrHeading(descr, heading);
-            descr.body = ParseDescrCodeBlock(body);
-
-            end = body.End + 1;
-            return descr;
-        }
-
         Actor ParseActor( int pos, out int end ) {
             var actor = new Actor();
             var head_token = tokens[pos];
diff --git a/flow/actorcompiler/ParseTree.cs b/flow/actorcompiler/ParseTree.cs
index 4fc4c680ab7..bcc4f137e23 100644
--- a/flow/actorcompiler/ParseTree.cs
+++ b/flow/actorcompiler/ParseTree.cs
@@ -215,13 +215,6 @@ public override bool containsWait()
     };
 
 
-    class Declaration
-    {
-        public string type;
-        public string name;
-        public string comment;
-    };
-
     class Actor
     {
         public List<string> attributes = new List<string>();
@@ -242,11 +235,4 @@ class Actor
         public bool IsCancellable() { return returnType != null && !isUncancellable; }
         public void SetUncancellable() { isUncancellable = true; }
     };
-
-    class Descr
-    {
-        public string name;
-        public string superClassList;
-        public List<Declaration> body;
-    };
 };
diff --git a/flow/actorcompiler/usertype.dat b/flow/actorcompiler/usertype.dat
index 77a51990089..613d82fe794 100644
--- a/flow/actorcompiler/usertype.dat
+++ b/flow/actorcompiler/usertype.dat
@@ -5,4 +5,3 @@ choose
 when
 wait
 waitNext
-DESCR
\ No newline at end of file
diff --git a/flow/actorcompiler_py/actor_compiler.py b/flow/actorcompiler_py/actor_compiler.py
index 5a6a6c6e9a4..649821d9a9d 100644
--- a/flow/actorcompiler_py/actor_compiler.py
+++ b/flow/actorcompiler_py/actor_compiler.py
@@ -12,8 +12,6 @@
     ChooseStatement,
     CodeBlock,
     ContinueStatement,
-    Declaration,
-    Descr,
     ForStatement,
     IfStatement,
     LoopStatement,
@@ -213,71 +211,6 @@ def copy_from(self, other: "Context") -> None:
         self.try_loop_depth = other.try_loop_depth
 
 
-class DescrCompiler:
-    def __init__(self, descr: "Descr", brace_depth: int) -> None:
-        self.descr = descr
-        self.member_indent_str = "\t" * brace_depth
-
-    def write(self, writer) -> int:
-        lines = 0
-        indent = self.member_indent_str
-        writer.write(
-            f"{indent}template<> struct Descriptor<struct {self.descr.name}> {{\n"
-        )
-        writer.write(
-            f'{indent}\tstatic StringRef typeName() {{ return "{self.descr.name}"_sr; }}\n'
-        )
-        writer.write(f"{indent}\ttypedef {self.descr.name} type;\n")
-        lines += 3
-        for dec in self.descr.body:
-            writer.write(f"{indent}\tstruct {dec.name}Descriptor {{\n")
-            writer.write(
-                f'{indent}\t\tstatic StringRef name() {{ return "{dec.name}"_sr; }}\n'
-            )
-            writer.write(
-                f'{indent}\t\tstatic StringRef typeName() {{ return "{dec.type}"_sr; }}\n'
-            )
-            writer.write(
-                f'{indent}\t\tstatic StringRef comment() {{ return "{dec.comment}"_sr; }}\n'
-            )
-            writer.write(f"{indent}\t\ttypedef {dec.type} type;\n")
-            writer.write(
-                f"{indent}\t\tstatic inline type get({self.descr.name}& from);\n"
-            )
-            writer.write(f"{indent}\t}};\n")
-            lines += 7
-        writer.write(f"{indent}\ttypedef std::tuple<")
-        first = True
-        for dec in self.descr.body:
-            if not first:
-                writer.write(",")
-            writer.write(f"{dec.name}Descriptor")
-            first = False
-        writer.write("> fields;\n")
-        writer.write(
-            f"{indent}\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size<fields>::value>::type field_indexes;\n"
-        )
-        writer.write(f"{indent}}};\n")
-        if self.descr.super_class_list:
-            writer.write(
-                f"{indent}struct {self.descr.name} : {self.descr.super_class_list} {{\n"
-            )
-        else:
-            writer.write(f"{indent}struct {self.descr.name} {{\n")
-        lines += 4
-        for dec in self.descr.body:
-            writer.write(f"{indent}\t{dec.type} {dec.name}; //{dec.comment}\n")
-            lines += 1
-        writer.write(f"{indent}}};\n")
-        lines += 1
-        for dec in self.descr.body:
-            writer.write(
-                f"{indent}{dec.type} Descriptor<{self.descr.name}>::{dec.name}Descriptor::get({self.descr.name}& from) {{ return from.{dec.name}; }}\n"
-            )
-            lines += 1
-        return lines
-
-
 class ActorCompiler:
     member_indent_str = "\t"
     loop_depth_0 = "int loopDepth=0"
@@ -785,7 +718,7 @@ def writeCancelFunc(self, writer) -> None:
         cancel_func.specifiers = "override"
         cancel_func.indent(self.code_indent)
         cancel_func.write_line("auto wait_state = this->actor_wait_state;")
-        cancel_func.write_line("this->actor_wait_state = -1;")
+        cancel_func.write_line("this->actor_wait_state = ACTOR_WAIT_STATE_CANCELLED;")
         cancel_func.write_line("switch (wait_state) {")
         last_group = -1
         for cb in sorted(self.callbacks, key=lambda c: c.callback_group):
@@ -1412,7 +1345,8 @@ def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None:
         exit_func = self.get_function("exitChoose", "", [])
         exit_func.return_type = "void"
         exit_func.write_line(
-            "if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", self.this
+            "if (actorWaitStateIsWaiting({0}->actor_wait_state)) {0}->actor_wait_state = ACTOR_WAIT_STATE_NOT_WAITING;",
+            self.this,
         )
         for choice in choices:
             exit_func.write_line(
@@ -1554,7 +1488,7 @@ def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None:
                 self.line_numberFunction(cx.target, stmt.first_source_line)
                 if self.actor.is_cancellable():
                     cx.target.write_line(
-                        "if ({1}->actor_wait_state < 0) return {0};",
+                        "if (actorWaitStateIsCancelled({1}->actor_wait_state)) return {0};",
                         cx.catch_f_err.call(
                             "actor_cancelled()", self.adjust_loop_depth(cx.try_loop_depth)
                         ),
diff --git a/flow/actorcompiler_py/actor_parser.py b/flow/actorcompiler_py/actor_parser.py
index f6941f85d43..569d84420c2 100644
--- a/flow/actorcompiler_py/actor_parser.py
+++ b/flow/actorcompiler_py/actor_parser.py
@@ -7,7 +7,7 @@
 from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
 
 from .errors import ActorCompilerError
-from .actor_compiler import ActorCompiler, DescrCompiler
+from .actor_compiler import ActorCompiler
 
 from .parse_tree import (
     Actor,
@@ -15,8 +15,6 @@
     ChooseStatement,
     CodeBlock,
     ContinueStatement,
-    Declaration,
-    Descr,
     ForStatement,
     IfStatement,
     LoopStatement,
@@ -394,18 +392,6 @@ def write(self, writer: io.TextIOBase, destFileName: str) -> None:
                         f'#line {self.tokens[i].source_line} "{self.source_file}"\n'
                     )
                     outLine += 1
-            elif tok.value == "DESCR":
-                descr, end = self.parse_descr(i)
-                descr_writer = io.StringIO()
-                lines = DescrCompiler(descr, tok.brace_depth).write(descr_writer)
-                writer.write(descr_writer.getvalue())
-                outLine += lines
-                i = end
-                if i < len(self.tokens) and self.line_numbers_enabled:
-                    writer.write(
-                        f'#line {self.tokens[i].source_line} "{self.source_file}"\n'
-                    )
-                    outLine += 1
             elif tok.value in ("class", "struct", "union"):
                 writer.write(tok.value)
                 success, name = self.parse_class_context(
@@ -461,79 +447,6 @@ def parse_actor(self, pos: int) -> Actor:
             self._parse_end = body.end_pos + 1
         return actor
 
-    def parse_descr(self, pos: int) -> Tuple[Descr, int]:
-        descr = Descr()
-        toks = self.range(pos + 1, len(self.tokens))
-        heading = toks.take_while(lambda t: t.value != "{")
-        body = self.range(heading.end_pos + 1, len(self.tokens)).take_while(
-            lambda t: t.brace_depth > toks.first().brace_depth or t.value == ";"
-        )
-        self.parse_descrHeading(descr, heading)
-        descr.body = self.parse_descrCodeBlock(body)
-        end = body.end_pos + 1
-        return descr, end
-
-    def parse_descrHeading(self, descr: Descr, toks: TokenRange) -> None:
-        nonWhitespace = lambda t: not t.is_whitespace
-        toks.first(nonWhitespace).ensure(
-            "non-struct DESCR!", lambda t: t.value == "struct"
-        )
-        toks = (
-            toks.skipWhile(lambda t: t.is_whitespace)
-            .skip(1)
-            .skipWhile(lambda t: t.is_whitespace)
-        )
-        colon = next((t for t in toks if t.value == ":"), None)
-        if colon:
-            descr.super_class_list = self.str(
-                self.range(colon.position + 1, toks.end_pos)
-            ).strip()
-            toks = self.range(toks.begin_pos, colon.position)
-        descr.name = self.str(toks).strip()
-
-    def parse_descrCodeBlock(self, toks: TokenRange) -> List[Declaration]:
-        declarations: List[Declaration] = []
-        while True:
-            delim = next((t for t in toks if t.value == ";"), None)
-            if delim is None:
-                break
-            pos = delim.position + 1
-            potential_comment = self.range(pos, toks.end_pos).skipWhile(
-                lambda t: t.value in ("\t", " ")
-            )
-            if (
-                not potential_comment.is_empty()
-                and potential_comment.first().value.startswith("//")
-            ):
-                pos = potential_comment.first().position + 1
-            self.parse_declarationRange(self.range(toks.begin_pos, pos), declarations)
-            toks = self.range(pos, toks.end_pos)
-        if not toks.all_match(lambda t: t.is_whitespace):
-            raise ActorCompilerError(
-                toks.first(lambda t: not t.is_whitespace).source_line,
-                "Trailing unterminated statement in code block",
-            )
-        return declarations
-
-    def parse_declarationRange(
-        self, toks: TokenRange, declarations: List[Declaration]
-    ) -> None:
-        delim = toks.first(lambda t: t.value == ";")
-        nameRange = (
-            self.range(toks.begin_pos, delim.position)
-            .Revskip_while(lambda t: t.is_whitespace)
-            .Revtake_while(lambda t: not t.is_whitespace)
-        )
-        typeRange = self.range(toks.begin_pos, nameRange.begin_pos)
-        commentRange = self.range(delim.position + 1, toks.end_pos)
-        declarations.append(
-            Declaration(
-                name=self.str(nameRange).strip(),
-                type=self.str(typeRange).strip(),
-                comment=self.str(commentRange).strip().lstrip("/"),
-            )
-        )
-
     def parse_test_caseHeading(self, actor: Actor, toks: TokenRange) -> None:
         actor.is_static = True
         nonWhitespace = lambda t: not t.is_whitespace
diff --git a/flow/actorcompiler_py/parse_tree.py b/flow/actorcompiler_py/parse_tree.py
index 18d3cee0a11..8aa697b7bae 100644
--- a/flow/actorcompiler_py/parse_tree.py
+++ b/flow/actorcompiler_py/parse_tree.py
@@ -197,13 +197,6 @@ def contains_wait(self) -> bool:
         return False
 
 
-@dataclass
-class Declaration:
-    type: str = ""
-    name: str = ""
-    comment: str = ""
-
-
 @dataclass
 class Actor:
     attributes: List[str] = field(default_factory=list)
@@ -226,10 +219,3 @@ def is_cancellable(self) -> bool:
 
     def set_uncancellable(self) -> None:
         self._is_uncancellable = True
-
-
-@dataclass
-class Descr:
-    name: str = ""
-    super_class_list: Optional[str] = None
-    body: List[Declaration] = field(default_factory=list)
diff --git a/flow/bench/BenchAsyncResult.cpp b/flow/bench/BenchAsyncResult.cpp
new file mode 100644
index 00000000000..e94526b1954
--- /dev/null
+++ b/flow/bench/BenchAsyncResult.cpp
@@ -0,0 +1,96 @@
+/*
+ * BenchAsyncResult.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "flow/ThreadHelper.actor.h"
+#include "flow/flow.h"
+
+#include <cstdint>
+#include <numeric>
+#include <vector>
+
+namespace {
+
+struct ExpensivePayload {
+	std::vector<uint8_t> bytes;
+
+	explicit ExpensivePayload(size_t payloadBytes) : bytes(payloadBytes) {
+		std::iota(bytes.begin(), bytes.end(), uint8_t{ 0 });
+	}
+
+	uint64_t touch() const {
+		ASSERT(!bytes.empty());
+		return bytes.front() + bytes[bytes.size() / 2] + bytes.back();
+	}
+};
+
+Future<ExpensivePayload> returnFuturePayload(ExpensivePayload const& payload) {
+	co_return payload;
+}
+
+AsyncResult<ExpensivePayload> returnAsyncResultPayload(ExpensivePayload const& payload) {
+	co_return payload;
+}
+
+enum class AwaitImpl { Future, AsyncResult };
+
+template <AwaitImpl Impl>
+Future<Void> benchAwaitPayloadActor(benchmark::State* state) {
+	const size_t payloadBytes = state->range(0);
+	const ExpensivePayload source(payloadBytes);
+	uint64_t sink = 0;
+
+	while (state->KeepRunning()) {
+		if constexpr (Impl == AwaitImpl::Future) {
+			ExpensivePayload payload = co_await returnFuturePayload(source);
+			sink += payload.touch();
+			benchmark::DoNotOptimize(payload.bytes.data());
+		} else {
+			ExpensivePayload payload = co_await returnAsyncResultPayload(source);
+			sink += payload.touch();
+			benchmark::DoNotOptimize(payload.bytes.data());
+		}
+		benchmark::ClobberMemory();
+	}
+
+	state->SetBytesProcessed(static_cast<int64_t>(state->iterations()) * payloadBytes);
+	benchmark::DoNotOptimize(sink);
+	co_return;
+}
+
+template <AwaitImpl Impl>
+void benchAwaitPayload(benchmark::State& state) {
+	onMainThread([&state] { return benchAwaitPayloadActor<Impl>(&state); }).blockUntilReady();
+}
+
+BENCHMARK_TEMPLATE(benchAwaitPayload, AwaitImpl::Future)
+    ->Name("CoroutineAwaitPayload/Future")
+    ->RangeMultiplier(4)
+    ->Range(1 << 10, 1 << 20)
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchAwaitPayload, AwaitImpl::AsyncResult)
+    ->Name("CoroutineAwaitPayload/AsyncResult")
+    ->RangeMultiplier(4)
+    ->Range(1 << 10, 1 << 20)
+    ->ReportAggregatesOnly(true);
+
+} // namespace
diff --git a/flowbench/BenchCallback.cpp b/flow/bench/BenchCallback.cpp
similarity index 100%
rename from flowbench/BenchCallback.cpp
rename to flow/bench/BenchCallback.cpp
diff --git a/flowbench/BenchCallback.actor.cpp b/flow/bench/BenchCallbackActor.cpp
similarity index 84%
rename from flowbench/BenchCallback.actor.cpp
rename to flow/bench/BenchCallbackActor.cpp
index e5f370f2039..1e8e21072e7 100644
--- a/flowbench/BenchCallback.actor.cpp
+++ b/flow/bench/BenchCallbackActor.cpp
@@ -1,5 +1,5 @@
 /*
- * BenchCallback.actor.cpp
+ * BenchCallbackActor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,25 +22,21 @@
 
 #include <vector>
 
-#include "fdbclient/FDBTypes.h"
 #include "flow/flow.h"
 #include "flow/ThreadHelper.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR template <size_t Size>
+template <size_t Size>
 static Future<Void> increment(Future<Void> f, uint32_t* sum) {
-	state std::array<uint8_t, Size> arr;
-	wait(f);
+	std::array<uint8_t, Size> arr;
+	co_await f;
 	benchmark::DoNotOptimize(arr);
 	++(*sum);
-	return Void();
 }
 
-ACTOR template <size_t Size>
+template <size_t Size>
 static Future<Void> benchCallbackActor(benchmark::State* benchState) {
-	state size_t actorCount = benchState->range(0);
-	state uint32_t sum;
+	size_t actorCount = benchState->range(0);
+	uint32_t sum{ 0 };
 	while (benchState->KeepRunning()) {
 		sum = 0;
 		Promise<Void> trigger;
@@ -50,12 +46,11 @@ static Future<Void> benchCallbackActor(benchmark::State* benchState) {
 			futures.push_back(increment<Size>(trigger.getFuture(), &sum));
 		}
 		trigger.send(Void());
-		wait(waitForAll(futures));
+		co_await waitForAll(futures);
 		benchmark::DoNotOptimize(sum);
 	}
 	benchState->SetItemsProcessed(actorCount * static_cast<long>(benchState->iterations()));
 	benchState->SetBytesProcessed(actorCount * Size * static_cast<long>(benchState->iterations()));
-	return Void();
 }
 
 template <size_t Size>
diff --git a/flowbench/BenchConflictSet.cpp b/flow/bench/BenchConflictSet.cpp
similarity index 100%
rename from flowbench/BenchConflictSet.cpp
rename to flow/bench/BenchConflictSet.cpp
diff --git a/flow/bench/BenchCoroChooseRace.cpp b/flow/bench/BenchCoroChooseRace.cpp
new file mode 100644
index 00000000000..b67f688b355
--- /dev/null
+++ b/flow/bench/BenchCoroChooseRace.cpp
@@ -0,0 +1,278 @@
+/*
+ * BenchCoroChooseRace.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "flow/genericactors.actor.h"
+#include "flow/ThreadHelper.actor.h"
+
+namespace {
+
+enum class Impl { Choose, Race };
+enum class Scenario { ReadyFirst, ReadySecond, AfterFirst };
+
+// Measure the control-flow case where callers only need the winning branch to
+// perform side effects. This is the path Choose is designed for.
+template <Impl impl>
+void consumeReady(Future<int> const& first, Future<double> const& second, double* sink) {
+	if constexpr (impl == Impl::Choose) {
+		Future<Void> f = Choose()
+		                     .When(first, [sink](int const& value) { *sink += value; })
+		                     .When(second, [sink](double const& value) { *sink += value; })
+		                     .run();
+		ASSERT(f.isReady());
+		benchmark::DoNotOptimize(f);
+	} else {
+		Future<std::variant<int, double>> f = race(first, second);
+		ASSERT(f.isReady());
+		std::visit([sink](auto const& value) { *sink += value; }, f.get());
+		benchmark::DoNotOptimize(f);
+	}
+}
+
+template <Impl impl>
+void consumeAfter(Promise<int>& promise, Future<double> const& second, double* sink) {
+	if constexpr (impl == Impl::Choose) {
+		Future<Void> f = Choose()
+		                     .When(promise.getFuture(), [sink](int const& value) { *sink += value; })
+		                     .When(second, [sink](double const& value) { *sink += value; })
+		                     .run();
+		promise.send(1);
+		ASSERT(f.isReady());
+		benchmark::DoNotOptimize(f);
+	} else {
+		Future<std::variant<int, double>> f = race(promise.getFuture(), second);
+		promise.send(1);
+		ASSERT(f.isReady());
+		std::visit([sink](auto const& value) { *sink += value; }, f.get());
+		benchmark::DoNotOptimize(f);
+	}
+}
+
+// Measure the value-selection case where callers need "first result wins" as a
+// std::variant value. Choose has to emulate this with an extra Promise<Result>.
+template <Impl impl>
+void selectReadyAsValue(Future<int> const& first, Future<double> const& second, double* sink) {
+	using Result = std::variant<int, double>;
+	if constexpr (impl == Impl::Choose) {
+		Promise<Result> resultPromise;
+		Future<Result> resultFuture = resultPromise.getFuture();
+		Future<Void> f =
+		    Choose()
+		        .When(first,
+		              [&resultPromise](int const& value) { resultPromise.send(Result(std::in_place_index<0>, value)); })
+		        .When(second,
+		              [&resultPromise](double const& value) {
+			              resultPromise.send(Result(std::in_place_index<1>, value));
+		              })
+		        .run();
+		ASSERT(f.isReady());
+		ASSERT(resultFuture.isReady());
+		std::visit([sink](auto const& value) { *sink += value; }, resultFuture.get());
+		benchmark::DoNotOptimize(f);
+		benchmark::DoNotOptimize(resultFuture);
+	} else {
+		Future<Result> f = race(first, second);
+		ASSERT(f.isReady());
+		std::visit([sink](auto const& value) { *sink += value; }, f.get());
+		benchmark::DoNotOptimize(f);
+	}
+}
+
+template <Impl impl>
+void selectAfterAsValue(Promise<int>& promise, Future<double> const& second, double* sink) {
+	using Result = std::variant<int, double>;
+	if constexpr (impl == Impl::Choose) {
+		Promise<Result> resultPromise;
+		Future<Result> resultFuture = resultPromise.getFuture();
+		Future<Void> f =
+		    Choose()
+		        .When(promise.getFuture(),
+		              [&resultPromise](int const& value) { resultPromise.send(Result(std::in_place_index<0>, value)); })
+		        .When(second,
+		              [&resultPromise](double const& value) {
+			              resultPromise.send(Result(std::in_place_index<1>, value));
+		              })
+		        .run();
+		promise.send(1);
+		ASSERT(f.isReady());
+		ASSERT(resultFuture.isReady());
+		std::visit([sink](auto const& value) { *sink += value; }, resultFuture.get());
+		benchmark::DoNotOptimize(f);
+		benchmark::DoNotOptimize(resultFuture);
+	} else {
+		Future<Result> f = race(promise.getFuture(), second);
+		promise.send(1);
+		ASSERT(f.isReady());
+		std::visit([sink](auto const& value) { *sink += value; }, f.get());
+		benchmark::DoNotOptimize(f);
+	}
+}
+
+// Measure only the cost to construct a pending selector over unresolved
+// Futures. Cleanup is timed out so this isolates setup/registration overhead.
+template <Impl impl>
+static Future<Void> benchChooseRaceConstructPendingActor(benchmark::State* state) {
+	double sink = 0;
+	Promise<int> neverIntPromise;
+	Promise<double> neverDoublePromise;
+	Future<int> neverInt = neverIntPromise.getFuture();
+	Future<double> neverDouble = neverDoublePromise.getFuture();
+
+	for (auto _ : *state) {
+		benchmark::DoNotOptimize(_);
+		state->ResumeTiming();
+		if constexpr (impl == Impl::Choose) {
+			Future<Void> f = Choose()
+			                     .When(neverInt, [&sink](int const&) { benchmark::DoNotOptimize(sink); })
+			                     .When(neverDouble, [&sink](double const&) { benchmark::DoNotOptimize(sink); })
+			                     .run();
+			ASSERT(!f.isReady());
+			benchmark::DoNotOptimize(f);
+			state->PauseTiming();
+			f.cancel();
+		} else {
+			Future<std::variant<int, double>> f = race(neverInt, neverDouble);
+			ASSERT(!f.isReady());
+			benchmark::DoNotOptimize(f);
+			state->PauseTiming();
+			f.cancel();
+		}
+	}
+
+	benchmark::DoNotOptimize(sink);
+	co_return;
+}
+
+template <Impl impl, Scenario scenario>
+static Future<Void> benchChooseRaceActor(benchmark::State* state) {
+	double sink = 0;
+	Future<int> readyInt = 7;
+	Future<double> readyDouble = 8.0;
+	Promise<int> neverIntPromise;
+	Promise<double> neverDoublePromise;
+	Future<int> neverInt = neverIntPromise.getFuture();
+	Future<double> neverDouble = neverDoublePromise.getFuture();
+
+	while (state->KeepRunning()) {
+		if constexpr (scenario == Scenario::ReadyFirst) {
+			consumeReady<impl>(readyInt, readyDouble, &sink);
+		} else if constexpr (scenario == Scenario::ReadySecond) {
+			consumeReady<impl>(neverInt, readyDouble, &sink);
+		} else {
+			Promise<int> promise;
+			consumeAfter<impl>(promise, neverDouble, &sink);
+		}
+	}
+
+	benchmark::DoNotOptimize(sink);
+	co_return;
+}
+
+template <Impl impl, Scenario scenario>
+static Future<Void> benchChooseRaceValueActor(benchmark::State* state) {
+	double sink = 0;
+	Future<int> readyInt = 7;
+	Future<double> readyDouble = 8.0;
+	Promise<int> neverIntPromise;
+	Promise<double> neverDoublePromise;
+	Future<int> neverInt = neverIntPromise.getFuture();
+	Future<double> neverDouble = neverDoublePromise.getFuture();
+
+	while (state->KeepRunning()) {
+		if constexpr (scenario == Scenario::ReadyFirst) {
+			selectReadyAsValue<impl>(readyInt, readyDouble, &sink);
+		} else if constexpr (scenario == Scenario::ReadySecond) {
+			selectReadyAsValue<impl>(neverInt, readyDouble, &sink);
+		} else {
+			Promise<int> promise;
+			selectAfterAsValue<impl>(promise, neverDouble, &sink);
+		}
+	}
+
+	benchmark::DoNotOptimize(sink);
+	co_return;
+}
+
+template <Impl impl, Scenario scenario>
+static void benchChooseRace(benchmark::State& state) {
+	onMainThread([&state] { return benchChooseRaceActor<impl, scenario>(&state); }).blockUntilReady();
+}
+
+template <Impl impl, Scenario scenario>
+static void benchChooseRaceValue(benchmark::State& state) {
+	onMainThread([&state] { return benchChooseRaceValueActor<impl, scenario>(&state); }).blockUntilReady();
+}
+
+template <Impl impl>
+static void benchChooseRaceConstructPending(benchmark::State& state) {
+	onMainThread([&state] { return benchChooseRaceConstructPendingActor<impl>(&state); }).blockUntilReady();
+}
+
+} // namespace
+
+BENCHMARK_TEMPLATE(benchChooseRace, Impl::Choose, Scenario::ReadyFirst)
+    ->Name("coro_choose/ready_first")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRace, Impl::Race, Scenario::ReadyFirst)
+    ->Name("coro_race/ready_first")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchChooseRace, Impl::Choose, Scenario::ReadySecond)
+    ->Name("coro_choose/ready_second")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRace, Impl::Race, Scenario::ReadySecond)
+    ->Name("coro_race/ready_second")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchChooseRace, Impl::Choose, Scenario::AfterFirst)
+    ->Name("coro_choose/after_first")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRace, Impl::Race, Scenario::AfterFirst)
+    ->Name("coro_race/after_first")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchChooseRaceValue, Impl::Choose, Scenario::ReadyFirst)
+    ->Name("coro_choose_as_race/ready_first")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRaceValue, Impl::Race, Scenario::ReadyFirst)
+    ->Name("coro_race_as_value/ready_first")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchChooseRaceValue, Impl::Choose, Scenario::ReadySecond)
+    ->Name("coro_choose_as_race/ready_second")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRaceValue, Impl::Race, Scenario::ReadySecond)
+    ->Name("coro_race_as_value/ready_second")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchChooseRaceValue, Impl::Choose, Scenario::AfterFirst)
+    ->Name("coro_choose_as_race/after_first")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRaceValue, Impl::Race, Scenario::AfterFirst)
+    ->Name("coro_race_as_value/after_first")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchChooseRaceConstructPending, Impl::Choose)
+    ->Name("coro_choose_construct/pending")
+    ->ReportAggregatesOnly(true);
+BENCHMARK_TEMPLATE(benchChooseRaceConstructPending, Impl::Race)
+    ->Name("coro_race_construct/pending")
+    ->ReportAggregatesOnly(true);
diff --git a/flow/bench/BenchCoroutineOverhead.cpp b/flow/bench/BenchCoroutineOverhead.cpp
new file mode 100644
index 00000000000..4e1e0196c5f
--- /dev/null
+++ b/flow/bench/BenchCoroutineOverhead.cpp
@@ -0,0 +1,96 @@
+/*
+ * BenchCoroutineOverhead.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "flow/ThreadHelper.actor.h"
+#include "flow/flow.h"
+
+namespace {
+
+Future<int> returnReadyInt() {
+	co_return 1;
+}
+
+Future<Void> returnReadyVoid() {
+	co_return;
+}
+
+Future<Void> benchCreateReadyIntActor(benchmark::State* state) {
+	int sink = 0;
+	while (state->KeepRunning()) {
+		Future<int> f = returnReadyInt();
+		ASSERT(f.isReady());
+		sink += f.get();
+		benchmark::DoNotOptimize(f);
+	}
+	benchmark::DoNotOptimize(sink);
+	co_return;
+}
+
+Future<Void> benchCreateReadyVoidActor(benchmark::State* state) {
+	while (state->KeepRunning()) {
+		Future<Void> f = returnReadyVoid();
+		ASSERT(f.isReady());
+		benchmark::DoNotOptimize(f);
+	}
+	co_return;
+}
+
+Future<Void> benchAwaitReadyIntActor(benchmark::State* state) {
+	Future<int> ready = 1;
+	int sink = 0;
+	while (state->KeepRunning()) {
+		sink += co_await ready;
+	}
+	benchmark::DoNotOptimize(sink);
+	co_return;
+}
+
+Future<Void> benchAwaitReadyVoidActor(benchmark::State* state) {
+	Future<Void> ready = Void();
+	while (state->KeepRunning()) {
+		co_await ready;
+	}
+	co_return;
+}
+
+void benchCreateReadyInt(benchmark::State& state) {
+	onMainThread([&state] { return benchCreateReadyIntActor(&state); }).blockUntilReady();
+}
+
+void benchCreateReadyVoid(benchmark::State& state) {
+	onMainThread([&state] { return benchCreateReadyVoidActor(&state); }).blockUntilReady();
+}
+
+void benchAwaitReadyInt(benchmark::State& state) {
+	onMainThread([&state] { return benchAwaitReadyIntActor(&state); }).blockUntilReady();
+}
+
+void benchAwaitReadyVoid(benchmark::State& state) {
+	onMainThread([&state] { return benchAwaitReadyVoidActor(&state); }).blockUntilReady();
+}
+
+BENCHMARK(benchCreateReadyInt)->Name("coroutine_overhead/create_ready_int")->ReportAggregatesOnly(true);
+BENCHMARK(benchCreateReadyVoid)->Name("coroutine_overhead/create_ready_void")->ReportAggregatesOnly(true);
+BENCHMARK(benchAwaitReadyInt)->Name("coroutine_overhead/await_ready_int")->ReportAggregatesOnly(true);
+BENCHMARK(benchAwaitReadyVoid)->Name("coroutine_overhead/await_ready_void")->ReportAggregatesOnly(true);
+
+} // namespace
diff --git a/flowbench/BenchHash.cpp b/flow/bench/BenchHash.cpp
similarity index 84%
rename from flowbench/BenchHash.cpp
rename to flow/bench/BenchHash.cpp
index 9df8a457bbb..37308009295 100644
--- a/flowbench/BenchHash.cpp
+++ b/flow/bench/BenchHash.cpp
@@ -22,7 +22,8 @@
 #include "crc32/crc32c.h"
 #include "flow/Hash3.h"
 #include "flow/xxhash.h"
-#include "flowbench/GlobalData.h"
+
+#include "BenchSupport.h"
 
 #include <stdint.h>
 
@@ -33,10 +34,10 @@ enum class HashType {
 };
 
 template <HashType hashType>
-inline void hash(const KeyRef& key, size_t length) {}
+inline void hash(const StringRef& key, size_t length) {}
 
 template <>
-inline void hash<HashType::HashLittle2>(const KeyRef& key, size_t length) {
+inline void hash<HashType::HashLittle2>(const StringRef& key, size_t length) {
 	uint32_t part1;
 	uint32_t part2;
 	hashlittle2(key.begin(), length, &part1, &part2);
@@ -45,19 +46,19 @@ inline void hash<HashType::HashLittle2>(const KeyRef& key, size_t length) {
 }
 
 template <>
-inline void hash<HashType::CRC32C>(const KeyRef& key, size_t length) {
+inline void hash<HashType::CRC32C>(const StringRef& key, size_t length) {
 	benchmark::DoNotOptimize(crc32c_append(0xfdbeefdb, key.begin(), length));
 }
 
 template <>
-inline void hash<HashType::XXHash3>(const KeyRef& key, size_t length) {
+inline void hash<HashType::XXHash3>(const StringRef& key, size_t length) {
 	benchmark::DoNotOptimize(XXH3_64bits(key.begin(), length));
 }
 
 template <HashType hashType>
 static void bench_hash(benchmark::State& state) {
 	auto length = 1 << state.range(0);
-	auto key = getKey(length);
+	auto key = getString(length);
 	for (auto _ : state) {
 		hash<hashType>(key, length);
 	}
diff --git a/flow/bench/BenchMain.cpp b/flow/bench/BenchMain.cpp
new file mode 100644
index 00000000000..1502e9db3ed
--- /dev/null
+++ b/flow/bench/BenchMain.cpp
@@ -0,0 +1,25 @@
+/*
+ * BenchMain.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/BenchMain.h"
+
+int main(int argc, char** argv) {
+	return runBenchmarks(argc, argv);
+}
diff --git a/flowbench/BenchMem.cpp b/flow/bench/BenchMem.cpp
similarity index 100%
rename from flowbench/BenchMem.cpp
rename to flow/bench/BenchMem.cpp
diff --git a/flow/bench/BenchMemcpy.cpp b/flow/bench/BenchMemcpy.cpp
new file mode 100644
index 00000000000..9364ca645aa
--- /dev/null
+++ b/flow/bench/BenchMemcpy.cpp
@@ -0,0 +1,242 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <new>
+#include <string>
+
+#include "benchmark/benchmark.h"
+
+#include "flow/IRandom.h"
+#include "flow/Platform.h"
+
+void* rte_memcpy_noinline(void* dst, const void* src, size_t length);
+
+namespace {
+
+constexpr size_t kSmallBufferSize = 8192;
+constexpr size_t kLargeBufferSize = 100 * 1024 * 1024;
+constexpr size_t kAlignmentUnit = 64;
+constexpr size_t kAddressCount = 1 << 16;
+
+constexpr std::array<size_t, 63> kBufferSizes = { 1,    2,    3,    4,    5,    6,    7,    8,    9,    12,   15,
+	                                              16,   17,   31,   32,   33,   63,   64,   65,   127,  128,  129,
+	                                              191,  192,  193,  255,  256,  257,  319,  320,  321,  383,  384,
+	                                              385,  447,  448,  449,  511,  512,  513,  767,  768,  769,  1023,
+	                                              1024, 1025, 1518, 1522, 1536, 1600, 2048, 2560, 3072, 3584, 4096,
+	                                              4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 };
+
+enum class CopyFunction {
+	Rte,
+	Memcpy,
+};
+
+enum class CacheMode {
+	CacheToCache,
+	CacheToMem,
+	MemToCache,
+	MemToMem,
+};
+
+enum class CopyAlignment {
+	Aligned,
+	Unaligned,
+};
+
+static size_t roundUp(size_t size, size_t alignment) {
+	return (((size - 1) / alignment) + 1) * alignment;
+}
+
+struct AlignedFree {
+	void operator()(uint8_t* ptr) const { aligned_free(ptr); }
+};
+
+class AlignedBuffer {
+public:
+	explicit AlignedBuffer(size_t size)
+	  : ptr(static_cast<uint8_t*>(aligned_alloc(kAlignmentUnit, roundUp(size, kAlignmentUnit)))) {
+		if (ptr == nullptr) {
+			throw std::bad_alloc();
+		}
+	}
+
+	uint8_t* get() { return ptr.get(); }
+	const uint8_t* get() const { return ptr.get(); }
+
+private:
+	std::unique_ptr<uint8_t, AlignedFree> ptr;
+};
+
+class MemcpyBuffers {
+public:
+	MemcpyBuffers()
+	  : largeRead(kLargeBufferSize + kAlignmentUnit), largeWrite(kLargeBufferSize + kAlignmentUnit),
+	    smallRead(kSmallBufferSize + kAlignmentUnit), smallWrite(kSmallBufferSize + kAlignmentUnit) {
+		deterministicRandom()->randomBytes(largeRead.get(), static_cast<int>(kLargeBufferSize));
+		deterministicRandom()->randomBytes(smallRead.get(), static_cast<int>(kSmallBufferSize));
+
+		std::memset(largeWrite.get(), 0, kLargeBufferSize);
+		std::memset(smallWrite.get(), 0, kSmallBufferSize);
+
+		for (size_t i = 0; i < kAddressCount; ++i) {
+			largeReadOffsets[i] = randomLargeBufferOffset();
+			largeWriteOffsets[i] = randomLargeBufferOffset();
+		}
+	}
+
+	uint8_t* dstBuffer(bool cached) { return cached ? smallWrite.get() : largeWrite.get(); }
+	const uint8_t* srcBuffer(bool cached) const { return cached ? smallRead.get() : largeRead.get(); }
+
+	size_t dstOffset(bool cached, size_t index, size_t unalignedOffset) const {
+		return cached ? unalignedOffset : largeWriteOffsets[index] + unalignedOffset;
+	}
+
+	size_t srcOffset(bool cached, size_t index, size_t unalignedOffset) const {
+		return cached ? unalignedOffset : largeReadOffsets[index] + unalignedOffset;
+	}
+
+private:
+	static size_t randomLargeBufferOffset() {
+		return (deterministicRandom()->randomUInt32() % (kLargeBufferSize - kSmallBufferSize)) & ~(kAlignmentUnit - 1);
+	}
+
+	AlignedBuffer largeRead;
+	AlignedBuffer largeWrite;
+	AlignedBuffer smallRead;
+	AlignedBuffer smallWrite;
+	std::array<size_t, kAddressCount> largeReadOffsets;
+	std::array<size_t, kAddressCount> largeWriteOffsets;
+};
+
+static MemcpyBuffers& memcpyBuffers() {
+	static MemcpyBuffers buffers;
+	return buffers;
+}
+
+template <CacheMode Mode>
+constexpr bool isDstCached() {
+	return Mode == CacheMode::CacheToCache || Mode == CacheMode::MemToCache;
+}
+
+template <CacheMode Mode>
+constexpr bool isSrcCached() {
+	return Mode == CacheMode::CacheToCache || Mode == CacheMode::CacheToMem;
+}
+
+template <CopyFunction Function, size_t ConstantSize>
+void copy(uint8_t* dst, const uint8_t* src, size_t size) {
+	benchmark::DoNotOptimize(dst);
+	benchmark::DoNotOptimize(src);
+	if constexpr (Function == CopyFunction::Rte) {
+		if constexpr (ConstantSize != 0) {
+			benchmark::DoNotOptimize(rte_memcpy_noinline(dst, src, ConstantSize));
+		} else {
+			benchmark::DoNotOptimize(rte_memcpy_noinline(dst, src, size));
+		}
+	} else {
+		if constexpr (ConstantSize != 0) {
+			benchmark::DoNotOptimize(std::memcpy(dst, src, ConstantSize));
+		} else {
+			benchmark::DoNotOptimize(std::memcpy(dst, src, size));
+		}
+	}
+	benchmark::ClobberMemory();
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment, size_t ConstantSize>
+static void benchMemcpy(benchmark::State& state) {
+	const size_t size = ConstantSize == 0 ? state.range(0) : ConstantSize;
+	constexpr size_t dstUnalignedOffset = Alignment == CopyAlignment::Aligned ? 0 : 1;
+	constexpr size_t srcUnalignedOffset = Alignment == CopyAlignment::Aligned ? 0 : 5;
+	constexpr bool dstCached = isDstCached<Mode>();
+	constexpr bool srcCached = isSrcCached<Mode>();
+	auto& buffers = memcpyBuffers();
+	size_t addressIndex = 0;
+
+	for (auto _ : state) {
+		const size_t index = addressIndex & (kAddressCount - 1);
+		auto* dst = buffers.dstBuffer(dstCached) + buffers.dstOffset(dstCached, index, dstUnalignedOffset);
+		const auto* src = buffers.srcBuffer(srcCached) + buffers.srcOffset(srcCached, index, srcUnalignedOffset);
+		copy<Function, ConstantSize>(dst, src, size);
+		++addressIndex;
+	}
+
+	state.SetItemsProcessed(static_cast<int64_t>(state.iterations()));
+	state.SetBytesProcessed(static_cast<int64_t>(state.iterations()) * static_cast<int64_t>(size));
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment>
+static void benchMemcpyVariable(benchmark::State& state) {
+	benchMemcpy<Function, Mode, Alignment, 0>(state);
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment, size_t Size>
+static void benchMemcpyConstant(benchmark::State& state) {
+	benchMemcpy<Function, Mode, Alignment, Size>(state);
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment>
+static void registerVariableBenchmark(const std::string& name) {
+	auto* registeredBenchmark =
+	    benchmark::RegisterBenchmark(name.c_str(), &benchMemcpyVariable<Function, Mode, Alignment>);
+	for (const auto size : kBufferSizes) {
+		registeredBenchmark->Arg(size);
+	}
+	registeredBenchmark->MinTime(0.01);
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment, size_t Size>
+static void registerConstantBenchmark(const std::string& name) {
+	benchmark::RegisterBenchmark(name.c_str(), &benchMemcpyConstant<Function, Mode, Alignment, Size>)->MinTime(0.01);
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment>
+static void registerConstantBenchmarks(const std::string& prefix) {
+	registerConstantBenchmark<Function, Mode, Alignment, 6>(prefix + "/6");
+	registerConstantBenchmark<Function, Mode, Alignment, 64>(prefix + "/64");
+	registerConstantBenchmark<Function, Mode, Alignment, 128>(prefix + "/128");
+	registerConstantBenchmark<Function, Mode, Alignment, 192>(prefix + "/192");
+	registerConstantBenchmark<Function, Mode, Alignment, 256>(prefix + "/256");
+	registerConstantBenchmark<Function, Mode, Alignment, 512>(prefix + "/512");
+	registerConstantBenchmark<Function, Mode, Alignment, 768>(prefix + "/768");
+	registerConstantBenchmark<Function, Mode, Alignment, 1024>(prefix + "/1024");
+	registerConstantBenchmark<Function, Mode, Alignment, 1536>(prefix + "/1536");
+}
+
+template <CopyFunction Function, CacheMode Mode, CopyAlignment Alignment>
+static void registerMemcpyBenchmarksForCase(const std::string& functionName,
+                                            const std::string& cacheModeName,
+                                            const std::string& alignmentName) {
+	const auto prefix = "Memcpy/" + functionName + "/" + alignmentName + "/" + cacheModeName;
+	registerVariableBenchmark<Function, Mode, Alignment>(prefix + "/variable");
+	registerConstantBenchmarks<Function, Mode, Alignment>(prefix + "/constant");
+}
+
+template <CopyFunction Function, CacheMode Mode>
+static void registerMemcpyBenchmarksForCacheMode(const std::string& functionName, const std::string& cacheModeName) {
+	registerMemcpyBenchmarksForCase<Function, Mode, CopyAlignment::Aligned>(functionName, cacheModeName, "aligned");
+	registerMemcpyBenchmarksForCase<Function, Mode, CopyAlignment::Unaligned>(functionName, cacheModeName, "unaligned");
+}
+
+template <CopyFunction Function>
+static void registerMemcpyBenchmarksForFunction(const std::string& functionName) {
+	registerMemcpyBenchmarksForCacheMode<Function, CacheMode::CacheToCache>(functionName, "cache_to_cache");
+	registerMemcpyBenchmarksForCacheMode<Function, CacheMode::CacheToMem>(functionName, "cache_to_mem");
+	registerMemcpyBenchmarksForCacheMode<Function, CacheMode::MemToCache>(functionName, "mem_to_cache");
+	registerMemcpyBenchmarksForCacheMode<Function, CacheMode::MemToMem>(functionName, "mem_to_mem");
+}
+
+static bool registerMemcpyBenchmarks() {
+	registerMemcpyBenchmarksForFunction<CopyFunction::Rte>("rte_memcpy");
+	registerMemcpyBenchmarksForFunction<CopyFunction::Memcpy>("memcpy");
+	return true;
+}
+
+[[maybe_unused]] const bool memcpyBenchmarksRegistered = registerMemcpyBenchmarks();
+
+} // namespace
diff --git a/flowbench/BenchNet2.cpp b/flow/bench/BenchNet2.cpp
similarity index 98%
rename from flowbench/BenchNet2.cpp
rename to flow/bench/BenchNet2.cpp
index 194b9c5e8d5..c7abf4c761b 100644
--- a/flowbench/BenchNet2.cpp
+++ b/flow/bench/BenchNet2.cpp
@@ -41,7 +41,7 @@ static inline TaskPriority getRandomTaskPriority(DeterministicRandom& rand) {
 static Future<Void> benchNet2Actor(benchmark::State* benchState) {
 	size_t actorCount = benchState->range(0);
 	uint32_t sum;
-	int seed = platform::getRandomSeed();
+	uint64_t seed = platform::getRandomSeed();
 	while (benchState->KeepRunning()) {
 		sum = 0;
 		std::vector<Future<Void>> futures;
diff --git a/flowbench/BenchNet2.actor.cpp b/flow/bench/BenchNet2Actor.cpp
similarity index 67%
rename from flowbench/BenchNet2.actor.cpp
rename to flow/bench/BenchNet2Actor.cpp
index 1e0af848529..ac5d21e01a1 100644
--- a/flowbench/BenchNet2.actor.cpp
+++ b/flow/bench/BenchNet2Actor.cpp
@@ -1,5 +1,5 @@
 /*
- * BenchNet2.actor.cpp
+ * BenchNet2Actor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -26,22 +26,19 @@
 #include "flow/network.h"
 #include "flow/ThreadHelper.actor.h"
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR static Future<Void> increment(TaskPriority priority, uint32_t* sum) {
-	wait(delay(0, priority));
+static Future<Void> increment(TaskPriority priority, uint32_t* sum) {
+	co_await delay(0, priority);
 	++(*sum);
-	return Void();
 }
 
 static inline TaskPriority getRandomTaskPriority(DeterministicRandom& rand) {
 	return static_cast<TaskPriority>(rand.randomInt(0, 100));
 }
 
-ACTOR static Future<Void> benchNet2Actor(benchmark::State* benchState) {
-	state size_t actorCount = benchState->range(0);
-	state uint32_t sum;
-	state int seed = platform::getRandomSeed();
+static Future<Void> benchNet2Actor(benchmark::State* benchState) {
+	size_t actorCount = benchState->range(0);
+	uint32_t sum{ 0 };
+	uint64_t seed = platform::getRandomSeed();
 	while (benchState->KeepRunning()) {
 		sum = 0;
 		std::vector<Future<Void>> futures;
@@ -50,11 +47,10 @@ ACTOR static Future<Void> benchNet2Actor(benchmark::State* benchState) {
 		for (int i = 0; i < actorCount; ++i) {
 			futures.push_back(increment(getRandomTaskPriority(rand), &sum));
 		}
-		wait(waitForAll(futures));
+		co_await waitForAll(futures);
 		benchmark::DoNotOptimize(sum);
 	}
 	benchState->SetItemsProcessed(actorCount * static_cast<long>(benchState->iterations()));
-	return Void();
 }
 
 static void bench_net2(benchmark::State& benchState) {
@@ -66,27 +62,44 @@ BENCHMARK(bench_net2)->Range(1, 1 << 16)->ReportAggregatesOnly(true);
 static constexpr bool DELAY = false;
 static constexpr bool YIELD = true;
 
-ACTOR template <bool useYield>
-static Future<Void> benchDelay(benchmark::State* benchState) {
+static std::vector<Future<Void>> populateTimers(int64_t timerCount) {
 	// Number of random delays to start to just to populate the run loop
 	// priority queue
-	state int64_t timerCount = benchState->range(0);
-	state std::vector<Future<Void>> futures;
-	state DeterministicRandom rand(platform::getRandomSeed());
+	std::vector<Future<Void>> futures;
+	DeterministicRandom rand(platform::getRandomSeed());
 	while (--timerCount > 0) {
 		futures.push_back(delay(1.0 + rand.random01(), getRandomTaskPriority(rand)));
 	}
+	return futures;
+}
+
+static Future<Void> benchDelayLoop(benchmark::State* benchState) {
+	std::vector<Future<Void>> futures = populateTimers(benchState->range(0));
+
+	while (benchState->KeepRunning()) {
+		co_await delay(0);
+	}
+	benchState->SetItemsProcessed(static_cast<long>(benchState->iterations()));
+}
+
+static Future<Void> benchYieldLoop(benchmark::State* benchState) {
+	std::vector<Future<Void>> futures = populateTimers(benchState->range(0));
 
 	while (benchState->KeepRunning()) {
-		wait(useYield ? yield() : delay(0));
+		co_await yield();
 	}
 	benchState->SetItemsProcessed(static_cast<long>(benchState->iterations()));
-	return Void();
 }
 
 template <bool useYield>
 static void bench_delay(benchmark::State& benchState) {
-	onMainThread([&benchState] { return benchDelay<useYield>(&benchState); }).blockUntilReady();
+	onMainThread([&benchState] {
+		if constexpr (useYield) {
+			return benchYieldLoop(&benchState);
+		} else {
+			return benchDelayLoop(&benchState);
+		}
+	}).blockUntilReady();
 }
 
 BENCHMARK_TEMPLATE(bench_delay, DELAY)->Range(0, 1 << 16)->ReportAggregatesOnly(true);
diff --git a/flow/bench/BenchNoThrowOnCancel.cpp b/flow/bench/BenchNoThrowOnCancel.cpp
new file mode 100644
index 00000000000..8257da3c53a
--- /dev/null
+++ b/flow/bench/BenchNoThrowOnCancel.cpp
@@ -0,0 +1,158 @@
+/*
+ * BenchNoThrowOnCancel.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "flow/ThreadHelper.actor.h"
+#include "flow/flow.h"
+
+#include <cstdint>
+#include <vector>
+
+namespace {
+
+struct CleanupCounter {
+	uint64_t* cleanupCount;
+	~CleanupCounter() { ++*cleanupCount; }
+};
+
+// Baseline: cancellation resumes the coroutine through actor_cancelled(), so the
+// catch count should match the number of cancelled futures.
+Future<Void> cancelWithThrow(Future<Void> signal, uint64_t* cleanupCount, uint64_t* caughtCount) {
+	CleanupCounter cleanup{ cleanupCount };
+	try {
+		co_await signal;
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_actor_cancelled);
+		++*caughtCount;
+		throw;
+	}
+}
+
+// Treatment: cancellation destroys the frame directly. The catch block is kept
+// as a sentinel to prove that RAII cleanup ran without exception unwinding.
+Future<Void> cancelWithoutThrow(Future<Void> signal,
+                                uint64_t* cleanupCount,
+                                uint64_t* caughtCount,
+                                NoThrowOnCancel = {}) {
+	CleanupCounter cleanup{ cleanupCount };
+	try {
+		co_await signal;
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_actor_cancelled);
+		++*caughtCount;
+		throw;
+	}
+}
+
+enum class CancelImpl { Throwing, NoThrow };
+enum class CancelScenario { ConstructAndCancel, BatchCancel };
+
+template <CancelImpl Impl>
+Future<Void> makeCancelFuture(Future<Void> signal, uint64_t* cleanupCount, uint64_t* caughtCount) {
+	if constexpr (Impl == CancelImpl::Throwing) {
+		return cancelWithThrow(std::move(signal), cleanupCount, caughtCount);
+	} else {
+		return cancelWithoutThrow(std::move(signal), cleanupCount, caughtCount);
+	}
+}
+
+template <CancelImpl Impl, CancelScenario Scenario>
+Future<Void> benchNoThrowOnCancelActor(benchmark::State* state) {
+	Promise<Void> signal;
+	uint64_t cleanupCount = 0;
+	uint64_t caughtCount = 0;
+
+	if constexpr (Scenario == CancelScenario::ConstructAndCancel) {
+		while (state->KeepRunning()) {
+			Future<Void> f = makeCancelFuture<Impl>(signal.getFuture(), &cleanupCount, &caughtCount);
+			ASSERT(!f.isReady());
+			benchmark::DoNotOptimize(f);
+			f.cancel();
+			benchmark::DoNotOptimize(f);
+			benchmark::ClobberMemory();
+		}
+	} else {
+		const int batchSize = state->range(0);
+		std::vector<Future<Void>> futures;
+		futures.reserve(batchSize);
+		for (auto _ : *state) {
+			benchmark::DoNotOptimize(_);
+			// Exclude construction from this scenario so the timed region focuses
+			// on the cancellation path itself.
+			state->PauseTiming();
+			futures.clear();
+			for (int i = 0; i < batchSize; ++i) {
+				futures.push_back(makeCancelFuture<Impl>(signal.getFuture(), &cleanupCount, &caughtCount));
+				ASSERT(!futures.back().isReady());
+			}
+			benchmark::DoNotOptimize(futures.data());
+			state->ResumeTiming();
+			for (auto& f : futures) {
+				f.cancel();
+				benchmark::DoNotOptimize(f);
+			}
+			benchmark::ClobberMemory();
+			state->PauseTiming();
+			for (auto const& f : futures) {
+				ASSERT(f.isReady() && f.isError() && f.getError().code() == error_code_actor_cancelled);
+			}
+		}
+		state->ResumeTiming();
+	}
+
+	uint64_t cancels = state->iterations();
+	if constexpr (Scenario == CancelScenario::BatchCancel) {
+		cancels *= static_cast<uint64_t>(state->range(0));
+	}
+	ASSERT_EQ(cleanupCount, cancels);
+	if constexpr (Impl == CancelImpl::Throwing) {
+		ASSERT_EQ(caughtCount, cancels);
+	} else {
+		ASSERT_EQ(caughtCount, 0);
+	}
+	state->SetItemsProcessed(static_cast<int64_t>(cancels));
+	co_return;
+}
+
+template <CancelImpl Impl, CancelScenario Scenario>
+void benchNoThrowOnCancel(benchmark::State& state) {
+	onMainThread([&state] { return benchNoThrowOnCancelActor<Impl, Scenario>(&state); }).blockUntilReady();
+}
+
+BENCHMARK_TEMPLATE(benchNoThrowOnCancel, CancelImpl::Throwing, CancelScenario::ConstructAndCancel)
+    ->Name("NoThrowOnCancel/throwing/construct_and_cancel")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchNoThrowOnCancel, CancelImpl::NoThrow, CancelScenario::ConstructAndCancel)
+    ->Name("NoThrowOnCancel/no_throw/construct_and_cancel")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchNoThrowOnCancel, CancelImpl::Throwing, CancelScenario::BatchCancel)
+    ->Name("NoThrowOnCancel/throwing/batch_cancel")
+    ->Arg(1024)
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchNoThrowOnCancel, CancelImpl::NoThrow, CancelScenario::BatchCancel)
+    ->Name("NoThrowOnCancel/no_throw/batch_cancel")
+    ->Arg(1024)
+    ->ReportAggregatesOnly(true);
+
+} // namespace
diff --git a/flowbench/BenchPriorityMultiLock.actor.cpp b/flow/bench/BenchPriorityMultiLock.cpp
similarity index 76%
rename from flowbench/BenchPriorityMultiLock.actor.cpp
rename to flow/bench/BenchPriorityMultiLock.cpp
index 3223ab37945..e17aed0b735 100644
--- a/flowbench/BenchPriorityMultiLock.actor.cpp
+++ b/flow/bench/BenchPriorityMultiLock.cpp
@@ -1,5 +1,5 @@
 /*
- * BenchStream.actor.cpp
+ * BenchPriorityMultiLock.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -22,48 +22,47 @@
 
 #include "flow/flow.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/PriorityMultiLock.actor.h"
+#include "flow/PriorityMultiLock.h"
 #include <deque>
-#include "flow/actorcompiler.h" // This must be the last #include.
 #include "fmt/printf.h"
 
-ACTOR static Future<Void> benchPriorityMultiLock(benchmark::State* benchState) {
+static Future<Void> benchPriorityMultiLock(benchmark::State* benchState) {
 	// Arg1 is the number of active priorities to use
 	// Arg2 is the number of inactive priorities to use
-	state int active = benchState->range(0);
-	state int inactive = benchState->range(1);
+	int active = benchState->range(0);
+	int inactive = benchState->range(1);
 
 	// Set up priority list with limits 10, 20, 30, ...
-	state std::vector<int> priorities;
+	std::vector<int> priorities;
 	while (priorities.size() < active + inactive) {
 		priorities.push_back(10 * (priorities.size() + 1));
 	}
 
-	state int concurrency = priorities.size() * 10;
-	state Reference<PriorityMultiLock> pml = makeReference<PriorityMultiLock>(concurrency, priorities);
+	int concurrency = priorities.size() * 10;
+	auto pml = makeReference<PriorityMultiLock>(concurrency, priorities);
 
 	// Clog the lock buy taking n=concurrency locks
-	state std::deque<Future<PriorityMultiLock::Lock>> lockFutures;
+	std::deque<Future<PriorityMultiLock::Lock>> lockFutures;
 	for (int j = 0; j < concurrency; ++j) {
 		lockFutures.push_back(pml->lock(j % active));
 	}
 	// Wait for all of the initial locks to be taken
 	// This will work regardless of their priorities as there are only n = concurrency of them
-	wait(waitForAll(std::vector<Future<PriorityMultiLock::Lock>>(lockFutures.begin(), lockFutures.end())));
+	co_await waitForAll(std::vector<Future<PriorityMultiLock::Lock>>(lockFutures.begin(), lockFutures.end()));
 
 	// For each iteration of the loop, one new lock user is created, for a total of
 	// concurrency + 1 users.  The new user replaces an old one, which is then waited
 	// on.  This will succeed regardless of the lock priorities used because prior to
 	// new user there were only n = concurrency users so they will all be served before
 	// the new user.
-	state int p = 0;
-	state int i = 0;
+	int p = 0;
+	int i = 0;
 	while (benchState->KeepRunning()) {
 		// Get and replace the i'th lock future with a new lock waiter
 		Future<PriorityMultiLock::Lock> f = lockFutures[i];
 		lockFutures[i] = pml->lock(p);
 
-		PriorityMultiLock::Lock lock = wait(f);
+		PriorityMultiLock::Lock lock = co_await f;
 
 		// Rotate to another priority
 		if (++p == active) {
@@ -77,8 +76,6 @@ ACTOR static Future<Void> benchPriorityMultiLock(benchmark::State* benchState) {
 	}
 
 	benchState->SetItemsProcessed(static_cast<long>(benchState->iterations()));
-
-	return Void();
 }
 
 static void bench_priorityMultiLock(benchmark::State& benchState) {
diff --git a/flowbench/BenchRandom.cpp b/flow/bench/BenchRandom.cpp
similarity index 100%
rename from flowbench/BenchRandom.cpp
rename to flow/bench/BenchRandom.cpp
diff --git a/flowbench/BenchRef.cpp b/flow/bench/BenchRef.cpp
similarity index 100%
rename from flowbench/BenchRef.cpp
rename to flow/bench/BenchRef.cpp
diff --git a/flowbench/BenchStream.actor.cpp b/flow/bench/BenchStream.cpp
similarity index 71%
rename from flowbench/BenchStream.actor.cpp
rename to flow/bench/BenchStream.cpp
index 60fe3dc0f92..8949cc59020 100644
--- a/flowbench/BenchStream.actor.cpp
+++ b/flow/bench/BenchStream.cpp
@@ -1,5 +1,5 @@
 /*
- * BenchStream.actor.cpp
+ * BenchStream.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,34 +20,29 @@
 
 #include "benchmark/benchmark.h"
 
-#include "fdbclient/FDBTypes.h"
+#include "BenchSupport.h"
 #include "flow/flow.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 #include "flow/ThreadHelper.actor.h"
 #include "flow/network.h"
-#include "flowbench/GlobalData.h"
 
 #include <thread>
 
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR static Future<Void> benchStreamActor(benchmark::State* benchState) {
-	state size_t items = benchState->range(0);
+static Future<Void> benchStreamActor(benchmark::State* benchState) {
+	size_t items = benchState->range(0);
 	size_t size = benchState->range(1);
-	state KeyRef key = getKey(size);
-	state PromiseStream<Key> stream;
-	state int i;
+	StringRef key = getString(size);
+	PromiseStream<StringRef> stream;
 	while (benchState->KeepRunning()) {
-		for (i = 0; i < items; ++i) {
+		for (int i = 0; i < items; ++i) {
 			stream.send(key);
 		}
-		for (i = 0; i < items; ++i) {
-			Key receivedKey = waitNext(stream.getFuture());
+		for (int i = 0; i < items; ++i) {
+			StringRef receivedKey = co_await stream.getFuture();
 			benchmark::DoNotOptimize(receivedKey);
 		}
 	}
 	benchState->SetItemsProcessed(items * static_cast<long>(benchState->iterations()));
-	return Void();
 }
 
 static void bench_stream(benchmark::State& benchState) {
diff --git a/fdbclient/include/fdbclient/EventTypes.actor.h b/flow/bench/BenchSupport.h
similarity index 56%
rename from fdbclient/include/fdbclient/EventTypes.actor.h
rename to flow/bench/BenchSupport.h
index 1887c2afdf5..c3d9dfc4936 100644
--- a/fdbclient/include/fdbclient/EventTypes.actor.h
+++ b/flow/bench/BenchSupport.h
@@ -1,5 +1,5 @@
 /*
- * EventTypes.actor.h
+ * BenchSupport.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,24 +18,30 @@
  * limitations under the License.
  */
 
-#pragma once
+#ifndef FDB_FLOW_BENCH_SUPPORT_H
+#define FDB_FLOW_BENCH_SUPPORT_H
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
-#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
-#include "fdbclient/EventTypes.actor.g.h"
-#elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H)
-#define FDBCLIENT_EVENTTYPES_ACTOR_H
+#pragma once
 
+#include "flow/Arena.h"
+#include "flow/FastAlloc.h"
+#include "flow/IRandom.h"
 #include "flow/flow.h"
-#include "flow/TDMetric.actor.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
-DESCR struct GetValueComplete {
-	int64_t latency; // ns
-};
+static constexpr size_t flowBenchDataSize = 1 << 20;
+
+inline uint8_t* flowBenchData() {
+	static uint8_t* globalData = nullptr;
+	if (!globalData) {
+		globalData = static_cast<uint8_t*>(allocateFast(flowBenchDataSize));
+	}
+	deterministicRandom()->randomBytes(globalData, flowBenchDataSize);
+	return globalData;
+}
 
-#include "flow/unactorcompiler.h"
+inline StringRef getString(size_t size) {
+	ASSERT(size <= flowBenchDataSize);
+	return StringRef(flowBenchData(), size);
+}
 
 #endif
diff --git a/flow/bench/BenchTimeout.cpp b/flow/bench/BenchTimeout.cpp
new file mode 100644
index 00000000000..2f3c35d65cf
--- /dev/null
+++ b/flow/bench/BenchTimeout.cpp
@@ -0,0 +1,95 @@
+/*
+ * BenchTimeout.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "flow/ThreadHelper.actor.h"
+#include "flow/genericactors.actor.h"
+#include "genericcoros.h"
+
+namespace {
+
+enum class TimeoutImpl { Actor, Coroutine };
+enum class TimeoutScenario { Ready, ConstructPending };
+
+template <TimeoutImpl Impl, TimeoutScenario Scenario>
+Future<Void> benchTimeoutActor(benchmark::State* state) {
+	const int timedOutValue = -1;
+
+	if constexpr (Scenario == TimeoutScenario::Ready) {
+		const Future<int> readyFuture = Future<int>(7);
+		int64_t sink = 0;
+		while (state->KeepRunning()) {
+			Future<int> done;
+			if constexpr (Impl == TimeoutImpl::Actor) {
+				done = ::timeout<int>(readyFuture, 0.0, timedOutValue);
+			} else {
+				done = generic_coro::timeout<int>(readyFuture, 0.0, timedOutValue);
+			}
+			sink += co_await done;
+			benchmark::DoNotOptimize(done);
+			benchmark::ClobberMemory();
+		}
+		benchmark::DoNotOptimize(sink);
+	} else {
+		const Future<int> neverFuture = Future<int>(Never());
+		for (auto _ : *state) {
+			benchmark::DoNotOptimize(_);
+			state->ResumeTiming();
+			// Measure the cost of constructing the timeout race itself. Using a
+			// never-ready input keeps the returned future pending so timer delay
+			// does not dominate the comparison.
+			Future<int> done;
+			if constexpr (Impl == TimeoutImpl::Actor) {
+				done = ::timeout<int>(neverFuture, 1.0, timedOutValue);
+			} else {
+				done = generic_coro::timeout<int>(neverFuture, 1.0, timedOutValue);
+			}
+			ASSERT(!done.isReady());
+			benchmark::DoNotOptimize(done);
+			state->PauseTiming();
+			done.cancel();
+		}
+	}
+	state->SetItemsProcessed(static_cast<int64_t>(state->iterations()));
+}
+
+template <TimeoutImpl Impl, TimeoutScenario Scenario>
+void benchTimeout(benchmark::State& state) {
+	onMainThread([&state] { return benchTimeoutActor<Impl, Scenario>(&state); }).blockUntilReady();
+}
+
+BENCHMARK_TEMPLATE(benchTimeout, TimeoutImpl::Actor, TimeoutScenario::Ready)
+    ->Name("Timeout/actor/ready")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchTimeout, TimeoutImpl::Coroutine, TimeoutScenario::Ready)
+    ->Name("Timeout/coroutine/ready")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchTimeout, TimeoutImpl::Actor, TimeoutScenario::ConstructPending)
+    ->Name("Timeout/actor/construct_pending")
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchTimeout, TimeoutImpl::Coroutine, TimeoutScenario::ConstructPending)
+    ->Name("Timeout/coroutine/construct_pending")
+    ->ReportAggregatesOnly(true);
+
+} // namespace
diff --git a/flowbench/BenchTimer.cpp b/flow/bench/BenchTimer.cpp
similarity index 100%
rename from flowbench/BenchTimer.cpp
rename to flow/bench/BenchTimer.cpp
diff --git a/flow/bench/BenchWaitForAllReady.cpp b/flow/bench/BenchWaitForAllReady.cpp
new file mode 100644
index 00000000000..4320d214576
--- /dev/null
+++ b/flow/bench/BenchWaitForAllReady.cpp
@@ -0,0 +1,100 @@
+/*
+ * BenchWaitForAllReady.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "benchmark/benchmark.h"
+
+#include "flow/ThreadHelper.actor.h"
+#include "flow/genericactors.actor.h"
+#include "genericcoros.h"
+
+#include <vector>
+
+namespace {
+
+enum class WaitForAllReadyImpl { Actor, Coroutine };
+enum class WaitForAllReadyScenario { Ready, Error };
+
+template <WaitForAllReadyScenario Scenario>
+std::vector<Future<int>> makeResults(int futureCount) {
+	std::vector<Future<int>> results;
+	results.reserve(futureCount);
+	for (int i = 0; i < futureCount; ++i) {
+		if constexpr (Scenario == WaitForAllReadyScenario::Ready) {
+			results.emplace_back(i);
+		} else {
+			results.emplace_back(operation_failed());
+		}
+	}
+	return results;
+}
+
+template <WaitForAllReadyImpl Impl, WaitForAllReadyScenario Scenario>
+Future<Void> benchWaitForAllReadyActor(benchmark::State* state) {
+	const int futureCount = state->range(0);
+	// Prebuild the futures so the benchmark isolates waitForAllReady itself
+	// instead of vector growth or Promise/Future setup cost.
+	const std::vector<Future<int>> results = makeResults<Scenario>(futureCount);
+
+	while (state->KeepRunning()) {
+		if constexpr (Impl == WaitForAllReadyImpl::Actor) {
+			Future<Void> done = ::waitForAllReady<int>(results);
+			co_await done;
+			benchmark::DoNotOptimize(done);
+		} else {
+			Future<Void> done = generic_coro::waitForAllReady<int>(results);
+			co_await done;
+			benchmark::DoNotOptimize(done);
+		}
+		benchmark::ClobberMemory();
+	}
+
+	state->SetItemsProcessed(static_cast<int64_t>(state->iterations()) * futureCount);
+}
+
+template <WaitForAllReadyImpl Impl, WaitForAllReadyScenario Scenario>
+void benchWaitForAllReady(benchmark::State& state) {
+	onMainThread([&state] { return benchWaitForAllReadyActor<Impl, Scenario>(&state); }).blockUntilReady();
+}
+
+BENCHMARK_TEMPLATE(benchWaitForAllReady, WaitForAllReadyImpl::Actor, WaitForAllReadyScenario::Ready)
+    ->Name("WaitForAllReady/actor/ready")
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 12)
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchWaitForAllReady, WaitForAllReadyImpl::Coroutine, WaitForAllReadyScenario::Ready)
+    ->Name("WaitForAllReady/coroutine/ready")
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 12)
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchWaitForAllReady, WaitForAllReadyImpl::Actor, WaitForAllReadyScenario::Error)
+    ->Name("WaitForAllReady/actor/error")
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 12)
+    ->ReportAggregatesOnly(true);
+
+BENCHMARK_TEMPLATE(benchWaitForAllReady, WaitForAllReadyImpl::Coroutine, WaitForAllReadyScenario::Error)
+    ->Name("WaitForAllReady/coroutine/error")
+    ->RangeMultiplier(4)
+    ->Range(1, 1 << 12)
+    ->ReportAggregatesOnly(true);
+
+} // namespace
diff --git a/flowbench/BenchZstd.cpp b/flow/bench/BenchZstd.cpp
similarity index 99%
rename from flowbench/BenchZstd.cpp
rename to flow/bench/BenchZstd.cpp
index 596b645be2d..3910e81383f 100644
--- a/flowbench/BenchZstd.cpp
+++ b/flow/bench/BenchZstd.cpp
@@ -35,7 +35,7 @@
 
 // Benchmark zstd performance. To use it:
 //  # export BM_ZSTD_DATA=path/to/datafile
-//  # bin/flowbench --benchmark_filter=bench_zstd
+//  # bin/flow_bench --benchmark_filter=bench_zstd
 
 // Compress with raw ZSTD API
 static inline std::pair<std::unique_ptr<char[]>, size_t> compress(const std::string& data, int level) {
diff --git a/flow/bench/CMakeLists.txt b/flow/bench/CMakeLists.txt
new file mode 100644
index 00000000000..45060c04525
--- /dev/null
+++ b/flow/bench/CMakeLists.txt
@@ -0,0 +1,18 @@
+include(FDBBenchmark)
+
+fdb_find_sources(FLOW_BENCH_SRCS)
+add_flow_target(EXECUTABLE NAME flow_bench SRCS ${FLOW_BENCH_SRCS})
+
+fdb_setup_googlebenchmark()
+
+target_include_directories(
+  flow_bench
+  PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}"
+          "${CMAKE_CURRENT_SOURCE_DIR}/.."
+          "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+if(FLOW_USE_ZSTD)
+  target_include_directories(flow_bench PRIVATE ${ZSTD_LIB_INCLUDE_DIR})
+endif()
+
+target_link_libraries(flow_bench PRIVATE Threads::Threads fdb_google_benchmark flow)
diff --git a/flowbench/README.md b/flow/bench/README.md
similarity index 70%
rename from flowbench/README.md
rename to flow/bench/README.md
index a55b0e4c38a..f3c4b5aa52f 100644
--- a/flowbench/README.md
+++ b/flow/bench/README.md
@@ -1,7 +1,13 @@
 Summary
 =======
 
-`flowbench` is an executable that can be used to microbenchmark parts of the FoundationDB code. The goal is to make it easy to test the performance of various sub-millisecond operations using `flow` and `fdbrpc`. Specifically, this tool can be used to:
+The benchmark suite is split into three executables:
+
+- `flow_bench` for benchmarks that depend only on `flow`
+- `fdbrpc_bench` for benchmarks that depend on `fdbrpc`
+- `fdbclient_bench` for benchmarks that depend on `fdbclient`
+
+These binaries can be used to microbenchmark parts of the FoundationDB code without always pulling in higher-level dependencies. Specifically, they can be used to:
 
 - Test the performance effects of changes to the actor compiler or to the `flow` and `fdbrpc` libraries
 - Test the performance of various uses of the `flow` and `fdbrpc` libraries
@@ -11,16 +17,16 @@ Summary
 Usage
 =====
 
-- To build the `flowbench` executable, run `ninja flowbench` or `make flowbench` depending on which build system you're using.
-- Then you can run `bin/flowbench --help` to see possible uses of `flowbench`.
-- Running `bin/flowbench` directly will run all registered benchmarks, but you may want to limit your run to a subset of benchmarks. This can be done by running `bin/flowbench --benchmark_filter=<regex>`
-- All benchmark names can be listed with `bin/flowbench --benchmark_list_tests`
+- To build one of the benchmark executables, run `ninja flow_bench`, `ninja fdbrpc_bench`, or `ninja fdbclient_bench`.
+- Then you can run `bin/<target> --help` to see possible uses of that benchmark binary.
+- Running `bin/<target>` directly will run all registered benchmarks in that binary, but you may want to limit your run to a subset of benchmarks. This can be done by running `bin/<target> --benchmark_filter=<regex>`
+- All benchmark names can be listed with `bin/<target> --benchmark_list_tests`
 - Example output:
 
 ```
-$ bin/flowbench --benchmark_filter=bench_ref
+$ bin/flow_bench --benchmark_filter=bench_ref
 2020-08-04 21:49:40
-Running bin/flowbench
+Running bin/flow_bench
 Run on (7 X 2904 MHz CPU s)
 CPU Caches:
   L1 Data 32 KiB (x7)
@@ -49,6 +55,7 @@ Existing Benchmarks
 - `bench_stream` measures the performance of writing to and reading from a `PromiseStream`
 - `bench_random` measures the performance of `DeterministicRandom`.
 - `bench_timer` measures the performance of FoundationDB timers.
+- `Memcpy` compares `rte_memcpy_noinline` and `memcpy` across aligned/unaligned and cached/uncached copy cases.
 
 Future use cases
 ================
diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
index ccd4a7c2a79..5832e60d3cf 100644
--- a/flow/flat_buffers.cpp
+++ b/flow/flat_buffers.cpp
@@ -530,7 +530,7 @@ TEST_CASE("/flow/FlatBuffers/EmptyStrings") {
 	rd.deserialize(xs);
 	ASSERT(xs.size() == kSize);
 	for (const auto& x : xs) {
-		ASSERT(x.size() == 0);
+		ASSERT(x.empty());
 	}
 	return Void();
 }
@@ -543,7 +543,7 @@ TEST_CASE("/flow/FlatBuffers/EmptyVectors") {
 	rd.deserialize(xs);
 	ASSERT(xs.size() == kSize);
 	for (const auto& x : xs) {
-		ASSERT(x.size() == 0);
+		ASSERT(x.empty());
 	}
 	return Void();
 }
@@ -556,7 +556,7 @@ TEST_CASE("/flow/FlatBuffers/EmptyVectorRefs") {
 	rd.deserialize(xs);
 	ASSERT(xs.size() == kSize);
 	for (const auto& x : xs) {
-		ASSERT(x.size() == 0);
+		ASSERT(x.empty());
 	}
 	return Void();
 }
@@ -570,7 +570,7 @@ TEST_CASE("/flow/FlatBuffers/EmptyPreSerVectorRefs") {
 	rd.deserialize(xs);
 	ASSERT(xs.size() == kSize);
 	for (const auto& x : xs) {
-		ASSERT(x.size() == 0);
+		ASSERT(x.empty());
 	}
 	return Void();
 }
@@ -584,7 +584,7 @@ TEST_CASE("/flow/FlatBuffers/EmptyUnorderedSet") {
 	rd.deserialize(xs);
 	ASSERT(xs.size() == kSize);
 	for (const auto& x : xs) {
-		ASSERT(x.size() == 0);
+		ASSERT(x.empty());
 	}
 	return Void();
 }
diff --git a/flow/flow.cpp b/flow/flow.cpp
index 77002e61fd6..762d9e3eb1c 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -32,7 +32,8 @@
 #include "flow/DeterministicRandom.h"
 #include "flow/Error.h"
 #include "flow/Hostname.h"
-#include "flow/rte_memcpy.h"
+#include "flow/Util.h"
+#include "rte_memcpy.h"
 #include "flow/UnitTest.h"
 
 #ifdef WITH_FOLLY_MEMCPY
@@ -107,9 +108,9 @@ Reference<IRandom> seededDebugRandom;
 uint64_t debug_lastLoadBalanceResultEndpointToken = 0;
 bool noUnseed = false;
 
-void setThreadLocalDeterministicRandomSeed(uint32_t seed) {
-	seededRandom = Reference<IRandom>(new DeterministicRandom(seed, true));
-	seededDebugRandom = Reference<IRandom>(new DeterministicRandom(seed));
+void setThreadLocalDeterministicRandomSeed(uint64_t seed) {
+	seededRandom = makeReference<DeterministicRandom>(seed, true);
+	seededDebugRandom = makeReference<DeterministicRandom>(seed);
 }
 
 Reference<IRandom> debugRandom() {
@@ -118,7 +119,7 @@ Reference<IRandom> debugRandom() {
 
 Reference<IRandom> deterministicRandom() {
 	if (!seededRandom) {
-		seededRandom = Reference<IRandom>(new DeterministicRandom(platform::getRandomSeed(), true));
+		seededRandom = makeReference<DeterministicRandom>(platform::getRandomSeed(), true);
 	}
 	return seededRandom;
 }
@@ -126,7 +127,7 @@ Reference<IRandom> deterministicRandom() {
 Reference<IRandom> nondeterministicRandom() {
 	static thread_local Reference<IRandom> random;
 	if (!random) {
-		random = Reference<IRandom>(new DeterministicRandom(platform::getRandomSeed()));
+		random = makeReference<DeterministicRandom>(platform::getRandomSeed());
 	}
 	return random;
 }
@@ -148,9 +149,17 @@ UID UID::fromStringThrowsOnFailure(std::string const& s) {
 		// invalid string size
 		throw operation_failed();
 	}
-	uint64_t a = 0, b = 0;
-	int r = sscanf(s.c_str(), "%16" SCNx64 "%16" SCNx64, &a, &b);
-	if (r != 2) {
+	// Split into two 16-character hex strings and parse using strtoull
+	std::string first_half = s.substr(0, 16);
+	std::string second_half = s.substr(16, 16);
+
+	char* end1;
+	char* end2;
+	uint64_t a = strtoull(first_half.c_str(), &end1, 16);
+	uint64_t b = strtoull(second_half.c_str(), &end2, 16);
+
+	// Verify entire strings were parsed
+	if (end1 != first_half.c_str() + 16 || end2 != second_half.c_str() + 16) {
 		throw operation_failed();
 	}
 	return UID(a, b);
@@ -439,7 +448,7 @@ struct Int {
 	constexpr static FileIdentifier file_identifier = 12345;
 	uint32_t value;
 	Int() = default;
-	Int(uint32_t value) : value(value) {}
+	explicit Int(uint32_t value) : value(value) {}
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar, value);
@@ -703,3 +712,57 @@ TEST_CASE("/flow/ErrorOr/Map") {
 
 	return Void();
 }
+
+TEST_CASE("/flow/Util/formatBytesHumanReadable") {
+	// Test TB
+	ASSERT(formatBytesHumanReadable(1099511627776LL) == "1.00 TB");
+	ASSERT(formatBytesHumanReadable(2199023255552LL) == "2.00 TB");
+	ASSERT(formatBytesHumanReadable(1649267441664LL) == "1.50 TB");
+
+	// Test GB
+	ASSERT(formatBytesHumanReadable(1073741824LL) == "1.00 GB");
+	ASSERT(formatBytesHumanReadable(2147483648LL) == "2.00 GB");
+	ASSERT(formatBytesHumanReadable(536870912LL + 1073741824LL) == "1.50 GB");
+
+	// Test MB
+	ASSERT(formatBytesHumanReadable(1048576LL) == "1.00 MB");
+	ASSERT(formatBytesHumanReadable(10485760LL) == "10.00 MB");
+	ASSERT(formatBytesHumanReadable(1572864LL) == "1.50 MB");
+
+	// Test KB
+	ASSERT(formatBytesHumanReadable(1024LL) == "1.00 KB");
+	ASSERT(formatBytesHumanReadable(10240LL) == "10.00 KB");
+	ASSERT(formatBytesHumanReadable(1536LL) == "1.50 KB");
+
+	// Test bytes
+	ASSERT(formatBytesHumanReadable(0LL) == "0 bytes");
+	ASSERT(formatBytesHumanReadable(1LL) == "1 bytes");
+	ASSERT(formatBytesHumanReadable(512LL) == "512 bytes");
+	ASSERT(formatBytesHumanReadable(1023LL) == "1023 bytes");
+
+	return Void();
+}
+
+TEST_CASE("/flow/Util/formatDurationHumanReadable") {
+	// Test hours and minutes
+	ASSERT(formatDurationHumanReadable(3600) == "1 hours");
+	ASSERT(formatDurationHumanReadable(7200) == "2 hours");
+	ASSERT(formatDurationHumanReadable(3660) == "1 hours 1 minutes");
+	ASSERT(formatDurationHumanReadable(3720) == "1 hours 2 minutes");
+	ASSERT(formatDurationHumanReadable(5400) == "1 hours 30 minutes");
+	ASSERT(formatDurationHumanReadable(9000) == "2 hours 30 minutes");
+
+	// Test minutes only
+	ASSERT(formatDurationHumanReadable(60) == "1 minutes");
+	ASSERT(formatDurationHumanReadable(120) == "2 minutes");
+	ASSERT(formatDurationHumanReadable(300) == "5 minutes");
+	ASSERT(formatDurationHumanReadable(3540) == "59 minutes");
+
+	// Test seconds only
+	ASSERT(formatDurationHumanReadable(0) == "0 seconds");
+	ASSERT(formatDurationHumanReadable(1) == "1 seconds");
+	ASSERT(formatDurationHumanReadable(30) == "30 seconds");
+	ASSERT(formatDurationHumanReadable(59) == "59 seconds");
+
+	return Void();
+}
diff --git a/flow/genericactors.actor.cpp b/flow/genericactors.actor.cpp
index 9a8c227ccb7..fae76313b9e 100644
--- a/flow/genericactors.actor.cpp
+++ b/flow/genericactors.actor.cpp
@@ -20,6 +20,7 @@
 
 #include "flow/flow.h"
 #include "flow/UnitTest.h"
+#include "genericcoros.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ACTOR Future<bool> allTrue(std::vector<Future<bool>> all) {
@@ -199,6 +200,11 @@ ACTOR Future<Void> lowPriorityDelayAfterCleared(Reference<AsyncVar<bool>> condit
 	}
 }
 
+struct SetAsyncVarTrue {
+	Reference<AsyncVar<bool>> value;
+	void operator()() const { value->set(true); }
+};
+
 namespace {
 
 struct DummyState {
@@ -237,6 +243,25 @@ static Future<ErrorOr<Void>> badTestFuture(double duration, Error e) {
 	return tag(delay(duration), ErrorOr<Void>(e));
 }
 
+ACTOR Future<int> getErrorCode(Future<int> future) {
+	try {
+		int value = wait(future);
+		(void)value;
+		return 0;
+	} catch (Error& e) {
+		return e.code();
+	}
+}
+
+ACTOR Future<int> getVoidErrorCode(Future<Void> future) {
+	try {
+		wait(future);
+		return 0;
+	} catch (Error& e) {
+		return e.code();
+	}
+}
+
 } // namespace
 
 TEST_CASE("/flow/genericactors/AsyncListener") {
@@ -283,21 +308,110 @@ TEST_CASE("/flow/genericactors/WaitForMost") {
 	return Void();
 }
 
-#if false
-TEST_CASE("/flow/genericactors/generic/storeTuple") {
-	state std::vector<UID> resA;
-	state int resB;
-	state double resC;
+TEST_CASE("/flow/genericcoros/ThrowErrorOr") {
+	int value = wait(generic_coro::throwErrorOr<int>(Future<ErrorOr<int>>(ErrorOr<int>(7))));
+	ASSERT_EQ(value, 7);
+
+	int errorCode =
+	    wait(getErrorCode(generic_coro::throwErrorOr<int>(Future<ErrorOr<int>>(ErrorOr<int>(operation_failed())))));
+	ASSERT_EQ(errorCode, error_code_operation_failed);
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/TraceAfter") {
+	int value = wait(generic_coro::traceAfter<int>(Future<int>(7), "GenericCorosTraceAfter"));
+	ASSERT_EQ(value, 7);
+
+	int errorCode = wait(
+	    getErrorCode(generic_coro::traceAfter<int>(Future<int>(operation_failed()), "GenericCorosTraceAfterError")));
+	ASSERT_EQ(errorCode, error_code_operation_failed);
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/TransformErrors") {
+	int value = wait(generic_coro::transformErrors<int>(Future<int>(7), operation_failed()));
+	ASSERT_EQ(value, 7);
+
+	int errorCode =
+	    wait(getErrorCode(generic_coro::transformErrors<int>(Future<int>(transaction_too_old()), operation_failed())));
+	ASSERT_EQ(errorCode, error_code_operation_failed);
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/TransformError") {
+	int value = wait(generic_coro::transformError<int>(Future<int>(7), transaction_too_old(), operation_failed()));
+	ASSERT_EQ(value, 7);
+
+	int transformedErrorCode = wait(getErrorCode(generic_coro::transformError<int>(
+	    Future<int>(transaction_too_old()), transaction_too_old(), operation_failed())));
+	ASSERT_EQ(transformedErrorCode, error_code_operation_failed);
+
+	int preservedErrorCode = wait(getErrorCode(
+	    generic_coro::transformError<int>(Future<int>(process_behind()), transaction_too_old(), operation_failed())));
+	ASSERT_EQ(preservedErrorCode, error_code_process_behind);
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/WaitForAllReady") {
+	state std::vector<Future<int>> results;
+	results = { Future<int>(1), Future<int>(operation_failed()), Future<int>(3) };
+
+	wait(generic_coro::waitForAllReady<int>(results));
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/Timeout") {
+	int readyValue = wait(generic_coro::timeout<int>(Future<int>(7), 0.0, -1));
+	ASSERT_EQ(readyValue, 7);
+
+	int timedOutValue = wait(generic_coro::timeout<int>(Future<int>(Never()), 0.0, -1));
+	ASSERT_EQ(timedOutValue, -1);
+
+	Optional<int> readyOptional = wait(generic_coro::timeout<int>(Future<int>(7), 0.0));
+	ASSERT(readyOptional.present());
+	ASSERT_EQ(readyOptional.get(), 7);
+
+	Optional<int> timedOutOptional = wait(generic_coro::timeout<int>(Future<int>(Never()), 0.0));
+	ASSERT(!timedOutOptional.present());
+
+	int errorCode = wait(getErrorCode(generic_coro::timeoutError<int>(Future<int>(Never()), 0.0)));
+	ASSERT_EQ(errorCode, error_code_timed_out);
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/Delayed") {
+	int value = wait(generic_coro::delayed<int>(Future<int>(7)));
+	ASSERT_EQ(value, 7);
+
+	int errorCode = wait(getErrorCode(generic_coro::delayed<int>(Future<int>(operation_failed()))));
+	ASSERT_EQ(errorCode, error_code_operation_failed);
+
+	return Void();
+}
+
+TEST_CASE("/flow/genericcoros/Trigger") {
+	state Reference<AsyncVar<bool>> called = makeReference<AsyncVar<bool>>(false);
+	state Promise<Void> signal;
+	state Future<Void> triggered = generic_coro::trigger(SetAsyncVarTrue{ called }, signal.getFuture());
+
+	ASSERT(!called->get());
+	ASSERT(!triggered.isReady());
 
-	state Promise<std::tuple<std::vector<UID>, int, double>> promise;
+	signal.send(Void());
+	wait(triggered);
+	ASSERT(called->get());
 
-	auto future = storeTuple(promise.getFuture(), resA, resB, resC);
+	called->set(false);
+	int errorCode =
+	    wait(getVoidErrorCode(generic_coro::trigger(SetAsyncVarTrue{ called }, Future<Void>(operation_failed()))));
+	ASSERT_EQ(errorCode, error_code_operation_failed);
+	ASSERT(!called->get());
 
-	promise.send(std::make_tuple(std::vector<UID>(10), 15, 2.0));
-	wait(ready(future));
-	ASSERT(resA.size() == 10);
-	ASSERT(resB == 15);
-	ASSERT(resC == 2.0);
 	return Void();
 }
-#endif
diff --git a/flow/genericcoros.h b/flow/genericcoros.h
new file mode 100644
index 00000000000..a261fcae9b2
--- /dev/null
+++ b/flow/genericcoros.h
@@ -0,0 +1,215 @@
+/*
+ * genericcoros.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "flow/Coroutines.h"
+#include "flow/flow.h"
+
+#include <vector>
+
+namespace generic_coro {
+
+template <class T>
+Future<T> traceAfter(Future<T> what, std::string eventType, bool traceErrors = true, ExplicitVoid = {}) {
+	try {
+		T val = co_await what;
+		TraceEvent(eventType.c_str());
+		co_return val;
+	} catch (Error& e) {
+		// Don't trace operation_cancelled as it's a normal control flow mechanism, not an error
+		if (traceErrors && e.code() != error_code_operation_cancelled) {
+			TraceEvent(eventType.c_str()).errorUnsuppressed(e);
+		}
+		throw;
+	}
+}
+
+template <class T>
+Future<Optional<T>> stopAfter(Future<T> what, ExplicitVoid = {}) {
+	Optional<T> ret = T();
+	try {
+		T res = co_await what;
+		ret = Optional<T>(res);
+	} catch (Error& e) {
+		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete ||
+		          e.code() == error_code_actor_cancelled || e.code() == error_code_local_config_changed;
+		TraceEvent(ok ? SevInfo : SevError, "StopAfterError").error(e);
+		if (!ok) {
+			fprintf(stderr, "Fatal Error: %s\n", e.what());
+			ret = Optional<T>();
+		}
+	}
+	g_network->stop();
+	co_return ret;
+}
+
+template <class T>
+Future<T> throwErrorOr(Future<ErrorOr<T>> f, ExplicitVoid = {}) {
+	ErrorOr<T> t = co_await f;
+	if (t.isError()) {
+		throw t.getError();
+	}
+	co_return std::move(t).get();
+}
+
+template <class T>
+Future<T> transformErrors(Future<T> f, Error err, ExplicitVoid = {}) {
+	ErrorOr<T> t = co_await coro::errorOr(f);
+	if (t.present()) {
+		co_return std::move(t).get();
+	}
+	Error e = t.getError();
+	if (e.code() == error_code_actor_cancelled) {
+		throw e;
+	}
+	throw err;
+}
+
+template <class T>
+Future<T> transformError(Future<T> f, Error inErr, Error outErr, ExplicitVoid = {}) {
+	ErrorOr<T> t = co_await coro::errorOr(f);
+	if (t.present()) {
+		co_return std::move(t).get();
+	}
+	Error e = t.getError();
+	if (e.code() == inErr.code()) {
+		throw outErr;
+	}
+	throw e;
+}
+
+template <class T>
+Future<Void> waitForAllReady(std::vector<Future<T>> results) {
+	for (auto const& result : results) {
+		if (result.isReady()) {
+			continue;
+		}
+		// waitForAllReady only cares that each future completes; composing
+		// ignore() with errorOr() avoids both throwing on error and copying T.
+		co_await coro::errorOr(coro::ignore(result));
+	}
+}
+
+template <class T>
+Future<T> timeout(Future<T> what,
+                  double time,
+                  T timedoutValue,
+                  TaskPriority taskID = TaskPriority::DefaultDelay,
+                  ExplicitVoid = {}) {
+	if (what.canGet()) {
+		co_return what.get();
+	} else if (what.isError()) {
+		throw what.getError();
+	}
+	auto res = co_await race(what, delay(time, taskID));
+	if (res.index() == 0) {
+		co_return std::get<0>(std::move(res));
+	} else {
+		co_return timedoutValue;
+	}
+}
+
+template <class T>
+Future<Optional<T>> timeout(Future<T> what,
+                            double time,
+                            TaskPriority taskID = TaskPriority::DefaultDelay,
+                            ExplicitVoid = {}) {
+	if (what.canGet()) {
+		co_return what.get();
+	} else if (what.isError()) {
+		throw what.getError();
+	}
+	auto res = co_await race(what, delay(time, taskID));
+	if (res.index() == 0) {
+		co_return std::get<0>(std::move(res));
+	} else {
+		co_return Optional<T>();
+	}
+}
+
+template <class T>
+Future<T> timeoutError(Future<T> what,
+                       double time,
+                       TaskPriority taskID = TaskPriority::DefaultDelay,
+                       ExplicitVoid = {}) {
+	if (what.canGet()) {
+		co_return what.get();
+	} else if (what.isError()) {
+		throw what.getError();
+	}
+	auto res = co_await race(what, delay(time, taskID));
+	if (res.index() == 0) {
+		co_return std::get<0>(std::move(res));
+	} else {
+		throw timed_out();
+	}
+}
+
+template <class T>
+Future<T> delayed(Future<T> what,
+                  double time = 0.0,
+                  TaskPriority taskID = TaskPriority::DefaultDelay,
+                  ExplicitVoid = {}) {
+	ErrorOr<T> t = co_await coro::errorOr(what);
+	co_await delay(time, taskID);
+	if (t.present()) {
+		co_return std::move(t).get();
+	} else {
+		throw t.getError();
+	}
+}
+
+template <class Func>
+Future<Void> trigger(Func what, Future<Void> signal) {
+	co_await signal;
+	what();
+}
+
+template <class T>
+Future<Void> uncancellable(Uncancellable, Future<T> what, Promise<T> result) {
+	ErrorOr<T> res = co_await coro::errorOr(what);
+	if (res.present()) {
+		result.send(std::move(res).get());
+	} else {
+		result.sendError(res.getError());
+	}
+}
+
+template <class T>
+Future<T> uncancellable(Future<T> what, ExplicitVoid = {}) {
+	Promise<T> resultPromise;
+	Future<T> result = resultPromise.getFuture();
+
+	uncancellable(Uncancellable(), what, resultPromise);
+	co_return co_await result;
+}
+
+template <class T, class X>
+Future<T> holdWhile(X object, Future<T> what) {
+	co_return co_await what;
+}
+
+template <class T, class X>
+Future<Void> store(X& out, Future<T> what) {
+	out = co_await what;
+}
+
+} // namespace generic_coro
diff --git a/flow/include/flow/ActorContext.h b/flow/include/flow/ActorContext.h
index 17ef1a3222a..ad0ebd1de63 100644
--- a/flow/include/flow/ActorContext.h
+++ b/flow/include/flow/ActorContext.h
@@ -75,7 +75,7 @@ class ActiveActorHelper {
 public:
 	ActorID actorID;
 
-	ActiveActorHelper(const ActorIdentifier& actorIdentifier);
+	explicit ActiveActorHelper(const ActorIdentifier& actorIdentifier);
 	~ActiveActorHelper();
 };
 
@@ -127,7 +127,7 @@ using ActorBlockIdentifier = UID;
 struct ActorExecutionContext {};
 struct ActiveActor {};
 struct ActiveActorHelper {
-	ActiveActorHelper(const ActorIdentifier&) {}
+	explicit ActiveActorHelper(const ActorIdentifier&) {}
 };
 struct ActorExecutionContextHelper {
 	ActorExecutionContextHelper(const ActorID&, const ActorBlockIdentifier&);
diff --git a/flow/include/flow/Arena.h b/flow/include/flow/Arena.h
index 87326c17677..69aa35e8daf 100644
--- a/flow/include/flow/Arena.h
+++ b/flow/include/flow/Arena.h
@@ -77,8 +77,8 @@ struct TrackIt {
 	}
 
 	TrackIt() { printf("TrackItCreate\t%s\t%p\t%s\n", __trackit__type(), this, platform::get_backtrace().c_str()); }
-	TrackIt(const TrackIt& o) : TrackIt() {}
-	TrackIt(const TrackIt&& o) : TrackIt() {}
+	explicit(false) TrackIt(const TrackIt& o) : TrackIt() {}
+	explicit(false) TrackIt(const TrackIt&& o) : TrackIt() {}
 	TrackIt& operator=(const TrackIt& o) {
 		printf("TrackItAssigned\t%s\t%p<%p\t%s\n", __trackit__type(), this, &o, platform::get_backtrace().c_str());
 		return *this;
@@ -91,7 +91,7 @@ class NonCopyable {
 protected:
 	NonCopyable() = default;
 	~NonCopyable() = default; /// Protected non-virtual destructor
-	NonCopyable(NonCopyable&&) = default;
+	explicit(false) NonCopyable(NonCopyable&&) = default;
 	NonCopyable& operator=(NonCopyable&&) = default;
 	NonCopyable(const NonCopyable&) = delete;
 	NonCopyable& operator=(const NonCopyable&) = delete;
@@ -112,8 +112,8 @@ class Arena {
 	Arena();
 	explicit Arena(size_t reservedSize);
 	//~Arena();
-	Arena(const Arena&);
-	Arena(Arena&& r) noexcept;
+	explicit(false) Arena(const Arena&);
+	explicit(false) Arena(Arena&& r) noexcept;
 	Arena& operator=(const Arena&);
 	Arena& operator=(Arena&&) noexcept;
 
@@ -306,7 +306,7 @@ class Standalone : private Arena, public T {
 	T const& contents() const { return *(T const*)this; }
 
 	Standalone() {}
-	Standalone(const T& t) : Arena(t.expectedSize()), T(arena(), t) {}
+	explicit(false) Standalone(const T& t) : Arena(t.expectedSize()), T(arena(), t) {}
 	Standalone<T>& operator=(const T& t) {
 		Arena old = std::move(arena()); // We want to defer the destruction of the arena until after we have copied t,
 		                                // in case it cross-references our previous value
@@ -316,9 +316,9 @@ class Standalone : private Arena, public T {
 	}
 
 	Standalone(const T& t, const Arena& arena) : Arena(arena), T(t) {}
-	Standalone(const Standalone<T>&) = default;
+	explicit(false) Standalone(const Standalone<T>&) = default;
 	Standalone<T>& operator=(const Standalone<T>&) = default;
-	Standalone(Standalone<T>&&) = default;
+	explicit(false) Standalone(Standalone<T>&&) = default;
 	Standalone<T>& operator=(Standalone<T>&&) = default;
 	~Standalone() = default;
 
@@ -338,7 +338,7 @@ class Standalone : private Arena, public T {
 
 private:
 	template <class U>
-	Standalone(Standalone<U> const&); // unimplemented
+	explicit(false) Standalone(Standalone<U> const&); // unimplemented
 	template <class U>
 	Standalone<T> const& operator=(Standalone<U> const&); // unimplemented
 };
@@ -354,6 +354,12 @@ class StringRef {
 		return bytesCopied;
 	}
 
+	// Delegated-to by StringRef(const char*) so strlen runs once and overflow is checked before truncation.
+	StringRef(const char* str, size_t len)
+	  : data(reinterpret_cast<const uint8_t*>(str)), length(static_cast<int>(len)) {
+		UNSTOPPABLE_ASSERT(len <= std::numeric_limits<int>::max());
+	}
+
 public:
 	constexpr static FileIdentifier file_identifier = 13300811;
 	StringRef() : data(0), length(0) {}
@@ -378,7 +384,21 @@ class StringRef {
 		}
 	}
 	StringRef(const uint8_t* data, int length) : data(data), length(length) {}
-	StringRef(const std::string& s) : data((const uint8_t*)s.c_str()), length((int)s.size()) {
+	// For string literals prefer "foo"_sr, which captures the length at compile time and
+	// avoids the runtime strlen walk this ctor performs. This ctor is for const char*
+	// values whose length isn't known at compile time (e.g: from  C APIs or interop).
+	// Borrows the pointer and computes length via strlen, like std::string_view.
+	// Marked explicit so that existing `f("literal")` calls that resolve against
+	// overload pairs like f(std::string) / f(StringRef) keep picking std::string,
+	// avoiding ambiguity. Direct-init `StringRef s("literal")` still selects this
+	// constructor — which is the actual use-after-free case this guards against.
+	// Precondition: str is not null.
+	explicit StringRef(const char* str) : StringRef(str, strlen(str)) {}
+	StringRef(std::nullptr_t) = delete;
+	// Reject integer literals (e.g. StringRef(0)), which would otherwise pick the const char*
+	// overload via null-pointer conversion and crash in strlen.
+	StringRef(int) = delete;
+	explicit(false) StringRef(const std::string& s) : data((const uint8_t*)s.c_str()), length((int)s.size()) {
 		if (s.size() > std::numeric_limits<int>::max())
 			abort();
 	}
@@ -469,49 +489,11 @@ class StringRef {
 	static bool isPrintable(char c) { return c > 32 && c < 127; }
 	inline std::string printable() const;
 
-	std::string toHexString(int limit = -1) const {
-		if (limit < 0)
-			limit = length;
-		std::string rv;
-		if (length > limit) {
-			// If limit is high enough split it so that 2/3 of limit is used to show prefix bytes and the rest is used
-			// for suffix bytes
-			if (limit >= 9) {
-				int suffix = limit / 3;
-				return substr(0, limit - suffix).toHexString() + "..." + substr(length - suffix, suffix).toHexString() +
-				       format(" [%d bytes]", length);
-			}
-			rv = substr(0, limit).toHexString() + format("...[%d]", length);
-		} else {
-			rv.reserve(length * 7);
-			for (int i = 0; i < length; i++) {
-				uint8_t b = (*this)[i];
-				if (isalnum(b))
-					rv.append(format("%02x (%c) ", b, b));
-				else
-					rv.append(format("%02x ", b));
-			}
-			if (rv.size() > 0)
-				rv.resize(rv.size() - 1);
-		}
-		bytesCopied()->increment(rv.length());
-		return rv;
-	}
+	std::string toHexString(int limit = -1) const;
 
 	// Get string with full content in hex format. Different digits are splitted by a space.
 	// This is currently used for bulk dumping manifest text file when recording key ranges.
-	std::string toFullHexStringPlain() const {
-		std::string s;
-		s.reserve(length * 7);
-		for (int i = 0; i < length; i++) {
-			uint8_t b = (*this)[i];
-			s.append(format("%02x ", b));
-		}
-		if (s.size() > 0)
-			s.resize(s.size() - 1);
-		bytesCopied()->increment(s.length());
-		return s;
-	}
+	std::string toFullHexStringPlain() const;
 
 	int expectedSize() const { return size(); }
 
@@ -617,7 +599,7 @@ class StringRef {
 
 private:
 	// Unimplemented; blocks conversion through std::string
-	StringRef(char*);
+	explicit StringRef(char*);
 
 	const uint8_t* data;
 	int length;
@@ -651,16 +633,14 @@ struct hash<Optional<T>> {
 };
 } // namespace std
 
-template <class T, class V = std::void_t<>>
-struct boost_hashable : std::false_type {};
-
 template <class T>
-struct boost_hashable<T, std::void_t<decltype(boost::hash_value(std::declval<T>()))>> : std::true_type {};
+concept boost_hashable = requires(T const& value) { boost::hash_value(value); };
 
 // Using boost hash functions on types that depend on member hashes (e.g. std::pair) expect the members
 // to be boost hashable. This provides a default boost hash function based on std::hash.
 template <class T>
-std::enable_if_t<!boost_hashable<T>::value, std::size_t> hash_value(const T& v) {
+    requires(!boost_hashable<T>)
+std::size_t hash_value(const T& v) {
 	return std::hash<T>{}(v);
 }
 
@@ -704,7 +684,7 @@ inline static uintptr_t getAlignedUpperBound(uintptr_t value, uintptr_t alignmen
 
 // makeString is used to allocate a Standalone<StringRef> of a known length for later
 // mutation (via mutateString).  If you need to append to a string of unknown length,
-// consider factoring StringBuffer from DiskQueue.actor.cpp.
+// consider factoring StringBuffer from DiskQueue.cpp.
 inline static Standalone<StringRef> makeString(int length) {
 	Standalone<StringRef> returnString;
 	uint8_t* outData = new (returnString.arena()) uint8_t[length];
@@ -904,11 +884,11 @@ enum class VecSerStrategy { FlatBuffers, String };
 template <class T, VecSerStrategy>
 struct VectorRefPreserializer {
 	VectorRefPreserializer() {}
-	VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::FlatBuffers>&) noexcept {}
+	explicit(false) VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::FlatBuffers>&) noexcept {}
 	VectorRefPreserializer& operator=(const VectorRefPreserializer<T, VecSerStrategy::FlatBuffers>&) noexcept {
 		return *this;
 	}
-	VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::String>&) noexcept {}
+	explicit(false) VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::String>&) noexcept {}
 	VectorRefPreserializer& operator=(const VectorRefPreserializer<T, VecSerStrategy::String>&) noexcept {
 		return *this;
 	}
@@ -925,13 +905,14 @@ struct VectorRefPreserializer<T, VecSerStrategy::String> {
 	string_serialized_traits<T> _string_traits;
 
 	VectorRefPreserializer() : _cached_size(0) {}
-	VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::String>& other) noexcept
+	explicit(false) VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::String>& other) noexcept
 	  : _cached_size(other._cached_size) {}
 	VectorRefPreserializer& operator=(const VectorRefPreserializer<T, VecSerStrategy::String>& other) noexcept {
 		_cached_size = other._cached_size;
 		return *this;
 	}
-	VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::FlatBuffers>&) noexcept : _cached_size(-1) {}
+	explicit(false) VectorRefPreserializer(const VectorRefPreserializer<T, VecSerStrategy::FlatBuffers>&) noexcept
+	  : _cached_size(-1) {}
 	VectorRefPreserializer& operator=(const VectorRefPreserializer<T, VecSerStrategy::FlatBuffers>&) noexcept {
 		_cached_size = -1;
 		return *this;
@@ -970,7 +951,7 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 	VectorRef() : data(0), m_size(0), m_capacity(0) {}
 
 	template <VecSerStrategy S>
-	VectorRef(const VectorRef<T, S>& other)
+	explicit(false) VectorRef(const VectorRef<T, S>& other)
 	  : VPS(other), data(other.data), m_size(other.m_size), m_capacity(other.m_capacity) {}
 	template <VecSerStrategy S>
 	VectorRef& operator=(const VectorRef<T, S>& other) {
@@ -983,7 +964,8 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 
 	// Arena constructor for non-Ref types, identified by !flow_ref
 	template <class T2 = T, VecSerStrategy S>
-	VectorRef(Arena& p, const VectorRef<T, S>& toCopy, typename std::enable_if<!flow_ref<T2>::value, int>::type = 0)
+	    requires(!flow_ref<T2>::value)
+	VectorRef(Arena& p, const VectorRef<T, S>& toCopy)
 	  : VPS(toCopy), data((T*)new(p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()),
 	    m_capacity(toCopy.size()) {
 		if (m_size > 0) {
@@ -993,7 +975,8 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 
 	// Arena constructor for Ref types, which must have an Arena constructor
 	template <class T2 = T, VecSerStrategy S>
-	VectorRef(Arena& p, const VectorRef<T, S>& toCopy, typename std::enable_if<flow_ref<T2>::value, int>::type = 0)
+	    requires(flow_ref<T2>::value)
+	VectorRef(Arena& p, const VectorRef<T, S>& toCopy)
 	  : VPS(), data((T*)new(p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) {
 		for (int i = 0; i < m_size; i++) {
 			auto ptr = new (&data[i]) T(p, toCopy[i]);
@@ -1016,7 +999,8 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 	// toCopy.m_capacity ) {} VectorRef<T>& operator=( const VectorRef<T>& );
 
 	template <VecSerStrategy S = SerStrategy>
-	typename std::enable_if<S == VecSerStrategy::String, uint32_t>::type serializedSize() const {
+	    requires(S == VecSerStrategy::String)
+	uint32_t serializedSize() const {
 		uint32_t result = sizeof(uint32_t);
 		string_serialized_traits<T> t;
 		if (VPS::_cached_size >= 0) {
@@ -1047,7 +1031,8 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 	std::reverse_iterator<const T*> rend() const { return std::reverse_iterator<const T*>(begin()); }
 
 	template <VecSerStrategy S = SerStrategy>
-	typename std::enable_if<S == VecSerStrategy::FlatBuffers, VectorRef>::type slice(int begin, int end) const {
+	    requires(S == VecSerStrategy::FlatBuffers)
+	VectorRef slice(int begin, int end) const {
 		return VectorRef(data + begin, end - begin);
 	}
 
@@ -1181,13 +1166,15 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 
 	// expectedSize() for non-Ref types, identified by !flow_ref
 	template <class T2 = T>
-	typename std::enable_if<!flow_ref<T2>::value, size_t>::type expectedSize() const {
+	    requires(!flow_ref<T2>::value)
+	size_t expectedSize() const {
 		return sizeof(T) * m_size;
 	}
 
 	// expectedSize() for Ref types, which must in turn have expectedSize() implemented.
 	template <class T2 = T>
-	typename std::enable_if<flow_ref<T2>::value, size_t>::type expectedSize() const {
+	    requires(flow_ref<T2>::value)
+	size_t expectedSize() const {
 		size_t t = sizeof(T) * m_size;
 		for (int i = 0; i < m_size; i++)
 			t += data[i].expectedSize();
@@ -1325,7 +1312,7 @@ class SmallVectorRef {
 public: // Construction
 	static_assert(std::is_trivially_destructible_v<T>);
 	SmallVectorRef() {}
-	SmallVectorRef(const SmallVectorRef<T, InlineMembers>& other)
+	explicit(false) SmallVectorRef(const SmallVectorRef<T, InlineMembers>& other)
 	  : m_size(other.m_size), arr(other.arr), data(other.data) {}
 	SmallVectorRef& operator=(const SmallVectorRef<T, InlineMembers>& other) {
 		m_size = other.m_size;
@@ -1335,18 +1322,14 @@ class SmallVectorRef {
 	}
 
 	template <class T2 = T, int IM = InlineMembers>
-	SmallVectorRef(Arena& arena,
-	               const SmallVectorRef<T, IM>& toCopy,
-	               typename std::enable_if<!flow_ref<T2>::value, int>::type = 0)
-	  : m_size(0) {
+	    requires(!flow_ref<T2>::value)
+	SmallVectorRef(Arena& arena, const SmallVectorRef<T, IM>& toCopy, int = 0) : m_size(0) {
 		append(arena, toCopy.begin(), toCopy.size());
 	}
 
 	template <class T2 = T, int IM = InlineMembers>
-	SmallVectorRef(Arena& arena,
-	               const SmallVectorRef<T2, IM>& toCopy,
-	               typename std::enable_if<flow_ref<T2>::value, int>::type = 0)
-	  : m_size(0) {
+	    requires(flow_ref<T2>::value)
+	SmallVectorRef(Arena& arena, const SmallVectorRef<T2, IM>& toCopy, int = 0) : m_size(0) {
 		append_deep(arena, toCopy.begin(), toCopy.size());
 	}
 
diff --git a/flow/include/flow/AsioReactor.h b/flow/include/flow/AsioReactor.h
index 82ae2e656e7..d8a7e01f311 100644
--- a/flow/include/flow/AsioReactor.h
+++ b/flow/include/flow/AsioReactor.h
@@ -70,7 +70,7 @@ class ASIOReactor {
 		}
 
 	public:
-		EventFD(ASIOReactor* reactor) : sd(reactor->ios, open()) {}
+		explicit EventFD(ASIOReactor* reactor) : sd(reactor->ios, open()) {}
 		~EventFD() override {
 			sd.close(); // Also closes the fd, I assume...
 		}
diff --git a/flowbench/flowbench.actor.cpp b/flow/include/flow/BenchMain.h
similarity index 71%
rename from flowbench/flowbench.actor.cpp
rename to flow/include/flow/BenchMain.h
index 1cbd97ec24c..611410d4135 100644
--- a/flowbench/flowbench.actor.cpp
+++ b/flow/include/flow/BenchMain.h
@@ -1,5 +1,5 @@
 /*
- * flowbench.actor.cpp
+ * BenchMain.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,36 +18,52 @@
  * limitations under the License.
  */
 
+#ifndef FDB_FLOW_BENCH_MAIN_H
+#define FDB_FLOW_BENCH_MAIN_H
+
+#pragma once
+
 #include "benchmark/benchmark.h"
-#include "fdbclient/NativeAPI.actor.h"
-#include "fdbclient/ThreadSafeTransaction.h"
+#include "flow/Platform.h"
+#include "flow/TLSConfig.h"
 #include "flow/ThreadHelper.actor.h"
+#include "flow/network.h"
+
+#include <functional>
 #include <thread>
 
-ACTOR template <class T>
-Future<T> stopNetworkAfter(Future<T> what) {
+inline Future<Void> stopNetworkAfter(Future<Void> what) {
 	try {
-		T t = wait(what);
+		co_await what;
 		g_network->stop();
-		return t;
 	} catch (...) {
 		g_network->stop();
 		throw;
 	}
 }
 
-int main(int argc, char** argv) {
+inline int runBenchmarks(int argc, char** argv, const std::function<void()>& extraInit = {}) {
 	benchmark::Initialize(&argc, argv);
 	if (benchmark::ReportUnrecognizedArguments(argc, argv)) {
 		return 1;
 	}
-	setupNetwork();
+
+	platformInit();
+	Error::init();
+	g_network = newNet2(TLSConfig());
+	if (extraInit) {
+		extraInit();
+	}
+
 	Promise<Void> benchmarksDone;
 	std::thread benchmarkThread([&]() {
 		benchmark::RunSpecifiedBenchmarks();
 		onMainThreadVoid([&]() { benchmarksDone.send(Void()); });
 	});
 	auto f = stopNetworkAfter(benchmarksDone.getFuture());
-	runNetwork();
+	g_network->run();
 	benchmarkThread.join();
+	return 0;
 }
+
+#endif
diff --git a/flow/include/flow/CodeProbe.h b/flow/include/flow/CodeProbe.h
index 0255fded58a..1cf9351fa3b 100644
--- a/flow/include/flow/CodeProbe.h
+++ b/flow/include/flow/CodeProbe.h
@@ -53,9 +53,8 @@ struct OrContext {
 };
 
 template <class Left, class Right>
-constexpr std::enable_if_t<Left::type == AnnotationType::Context && Right::type == AnnotationType::Context,
-                           OrContext<Left, Right>>
-operator|(Left const& lhs, Right const& rhs) {
+    requires(Left::type == AnnotationType::Context && Right::type == AnnotationType::Context)
+constexpr OrContext<Left, Right> operator|(Left const& lhs, Right const& rhs) {
 	return OrContext<Left, Right>(lhs, rhs);
 }
 
@@ -92,20 +91,19 @@ struct AssertNot {
 };
 
 template <class Left, class Right>
-constexpr std::enable_if_t<Left::type == AnnotationType::Assertion && Right::type == AnnotationType::Assertion,
-                           AssertOr<Left, Right>>
-operator||(Left const& lhs, Right const& rhs) {
+    requires(Left::type == AnnotationType::Assertion && Right::type == AnnotationType::Assertion)
+constexpr AssertOr<Left, Right> operator||(Left const& lhs, Right const& rhs) {
 	return AssertOr<Left, Right>();
 }
 template <class Left, class Right>
-constexpr std::enable_if_t<Left::type == AnnotationType::Assertion && Right::type == AnnotationType::Assertion,
-                           AssertAnd<Left, Right>>
-operator&&(Left const& lhs, Right const& rhs) {
+    requires(Left::type == AnnotationType::Assertion && Right::type == AnnotationType::Assertion)
+constexpr AssertAnd<Left, Right> operator&&(Left const& lhs, Right const& rhs) {
 	return AssertAnd<Left, Right>();
 }
 
 template <class T>
-constexpr std::enable_if_t<T::type == AnnotationType::Assertion, AssertNot<T>> operator!(T const&) {
+    requires(T::type == AnnotationType::Assertion)
+constexpr AssertNot<T> operator!(T const&) {
 	return AssertNot<T>();
 }
 
diff --git a/flow/include/flow/CompressedInt.h b/flow/include/flow/CompressedInt.h
index c1b3d1c1f33..77e29dcf367 100644
--- a/flow/include/flow/CompressedInt.h
+++ b/flow/include/flow/CompressedInt.h
@@ -47,7 +47,7 @@
 //   1111 1111 1111 1111 110n nnnn N{15}  125 bits  18 bytes
 template <typename IntType>
 struct CompressedInt {
-	CompressedInt(IntType i = 0) : value(i) {}
+	explicit CompressedInt(IntType i = 0) : value(i) {}
 	IntType value;
 	template <class Ar>
 	void serialize(Ar& ar) {
diff --git a/flow/include/flow/CoroUtils.h b/flow/include/flow/CoroUtils.h
index eb51f923af0..c3bbbdc29af 100644
--- a/flow/include/flow/CoroUtils.h
+++ b/flow/include/flow/CoroUtils.h
@@ -25,22 +25,63 @@
 
 namespace coro {
 
-template <class Parent, int Idx, class F>
-using ConditionalActorCallback = std::conditional_t<GetFutureTypeV<F> == FutureType::Future,
-                                                    ActorCallback<Parent, Idx, FutureReturnTypeT<F>>,
-                                                    ActorSingleCallback<Parent, Idx, FutureReturnTypeT<F>>>;
+// Adapts `AsyncResult<T>` callbacks to the same callback interface used by
+// `choose()` and `race()` over Future/FutureStream inputs.
+template <class Parent, int Idx, class ValueType>
+struct ActorAsyncResultCallback : AsyncResultCallback<ValueType> {
+	AsyncResultState<ValueType>* state = nullptr;
+
+	void bind(AsyncResult<ValueType>& result) { state = result.resultState; }
+
+	void remove() {
+		if (state) {
+			state->clearCallback(this);
+			state = nullptr;
+		}
+	}
+
+	void fire(ValueType const& value) override {
+#ifdef ENABLE_SAMPLING
+		LineageScope _(static_cast<Parent*>(this)->lineageAddr());
+#endif
+		static_cast<Parent*>(this)->a_callback_fire(this, value);
+	}
+
+	void fire(ValueType&& value) override {
+#ifdef ENABLE_SAMPLING
+		LineageScope _(static_cast<Parent*>(this)->lineageAddr());
+#endif
+		static_cast<Parent*>(this)->a_callback_fire(this, std::move(value));
+	}
+
+	void error(Error e) override {
+#ifdef ENABLE_SAMPLING
+		LineageScope _(static_cast<Parent*>(this)->lineageAddr());
+#endif
+		static_cast<Parent*>(this)->a_callback_error(this, e);
+	}
+};
+
+template <class Parent, int Idx, class AwaitableType>
+using ConditionalActorCallback =
+    std::conditional_t<GetFutureTypeV<AwaitableType> == FutureType::Future,
+                       ActorCallback<Parent, Idx, FutureReturnTypeT<AwaitableType>>,
+                       std::conditional_t<GetFutureTypeV<AwaitableType> == FutureType::FutureStream,
+                                          ActorSingleCallback<Parent, Idx, FutureReturnTypeT<AwaitableType>>,
+                                          ActorAsyncResultCallback<Parent, Idx, FutureReturnTypeT<AwaitableType>>>>;
 
 template <class Parent, int Idx, class... Args>
 struct ChooseImplCallback;
 
-template <class Parent, int Idx, class F, class... Args>
-struct ChooseImplCallback<Parent, Idx, F, Args...>
-  : ConditionalActorCallback<ChooseImplCallback<Parent, Idx, F, Args...>, Idx, F>,
+template <class Parent, int Idx, class AwaitableType, class... Args>
+struct ChooseImplCallback<Parent, Idx, AwaitableType, Args...>
+  : ConditionalActorCallback<ChooseImplCallback<Parent, Idx, AwaitableType, Args...>, Idx, AwaitableType>,
     ChooseImplCallback<Parent, Idx + 1, Args...> {
 
-	using ThisCallback = ConditionalActorCallback<ChooseImplCallback<Parent, Idx, F, Args...>, Idx, F>;
-	using ValueType = FutureReturnTypeT<F>;
-	static constexpr FutureType futureType = GetFutureTypeV<F>;
+	using ThisCallback =
+	    ConditionalActorCallback<ChooseImplCallback<Parent, Idx, AwaitableType, Args...>, Idx, AwaitableType>;
+	using ValueType = FutureReturnTypeT<AwaitableType>;
+	static constexpr FutureType futureType = GetFutureTypeV<AwaitableType>;
 
 	[[nodiscard]] Parent* getParent() { return static_cast<Parent*>(this); }
 
@@ -58,7 +99,7 @@ struct ChooseImplCallback<Parent, Idx, F, Args...>
 	}
 
 	void a_callback_fire(ThisCallback*, ValueType const& value) {
-		getParent()->actor_wait_state = 0;
+		getParent()->actor_wait_state = ACTOR_WAIT_STATE_NOT_WAITING;
 		getParent()->removeCallbacks();
 		try {
 			std::get<Idx>(getParent()->functions)(value);
@@ -71,7 +112,7 @@ struct ChooseImplCallback<Parent, Idx, F, Args...>
 	}
 
 	void a_callback_error(ThisCallback*, Error e) {
-		getParent()->actor_wait_state = 0;
+		getParent()->actor_wait_state = ACTOR_WAIT_STATE_NOT_WAITING;
 		getParent()->removeCallbacks();
 		getParent()->SAV<Void>::sendErrorAndDelPromiseRef(e);
 	}
@@ -105,13 +146,13 @@ struct ChooseImplActor final : Actor<Void>,
 	                std::tuple<std::function<void(FutureReturnTypeT<Args> const&)>...>&& functions)
 	  : Actor<Void>(), futures(futures), functions(functions) {
 		ChooseImplCallback<ChooseImplActor<Args...>, 0, Args...>::registerCallbacks();
-		actor_wait_state = 1;
+		actor_wait_state = ACTOR_WAIT_STATE_WAITING;
 	}
 
 	void cancel() override {
 		const auto waitState = actor_wait_state;
-		actor_wait_state = -1;
-		if (waitState > 0) {
+		actor_wait_state = ACTOR_WAIT_STATE_CANCELLED;
+		if (actorWaitStateIsWaiting(waitState)) {
 			ChooseImplCallback<ChooseImplActor<Args...>, 0, Args...>::removeCallbacks();
 			SAV<Void>::sendErrorAndDelPromiseRef(actor_cancelled());
 		}
@@ -212,10 +253,147 @@ class ChooseClause {
 	}
 };
 
+template <class... Futures>
+using RaceResult = std::variant<FutureReturnTypeT<std::decay_t<Futures>>...>;
+
+template <std::size_t Idx, class Result, class AwaitableType>
+Future<Result> raceReadyResult(AwaitableType&& future) {
+	if (future.isError()) {
+		return future.getError();
+	}
+	if constexpr (GetFutureTypeV<std::remove_cvref_t<AwaitableType>> == FutureType::Future) {
+		return Result(std::in_place_index<Idx>, future.get());
+	} else if constexpr (GetFutureTypeV<std::remove_cvref_t<AwaitableType>> == FutureType::FutureStream) {
+		auto fs = future;
+		return Result(std::in_place_index<Idx>, fs.pop());
+	} else {
+		return Result(std::in_place_index<Idx>, std::forward<AwaitableType>(future).get());
+	}
+}
+
+template <std::size_t Idx, class Result, class First, class... Rest>
+Future<Result> raceReady(First&& first, Rest&&... rest) {
+	if (first.isReady()) {
+		return raceReadyResult<Idx, Result>(std::forward<First>(first));
+	}
+	if constexpr (sizeof...(Rest) > 0) {
+		return raceReady<Idx + 1, Result>(std::forward<Rest>(rest)...);
+	}
+	return Future<Result>();
+}
+
+template <class Parent, int Idx, class... Futures>
+struct RaceImplCallback;
+
+template <class Parent, int Idx, class AwaitableType, class... Futures>
+struct RaceImplCallback<Parent, Idx, AwaitableType, Futures...>
+  : ConditionalActorCallback<RaceImplCallback<Parent, Idx, AwaitableType, Futures...>, Idx, AwaitableType>,
+    RaceImplCallback<Parent, Idx + 1, Futures...> {
+	using ThisCallback =
+	    ConditionalActorCallback<RaceImplCallback<Parent, Idx, AwaitableType, Futures...>, Idx, AwaitableType>;
+	using ValueType = FutureReturnTypeT<AwaitableType>;
+	static constexpr FutureType futureType = GetFutureTypeV<AwaitableType>;
+
+	[[nodiscard]] Parent* getParent() { return static_cast<Parent*>(this); }
+
+	void registerCallbacks() {
+		if constexpr (futureType == FutureType::Future) {
+			StrictFuture<ValueType> sf = std::get<Idx>(getParent()->futures);
+			sf.addCallbackAndClear(static_cast<ThisCallback*>(this));
+		} else if constexpr (futureType == FutureType::FutureStream) {
+			auto sf = std::get<Idx>(getParent()->futures);
+			sf.addCallbackAndClear(static_cast<ThisCallback*>(this));
+		} else {
+			ThisCallback::bind(std::get<Idx>(getParent()->futures));
+			std::move(std::get<Idx>(getParent()->futures)).addCallbackAndClear(static_cast<ThisCallback*>(this));
+		}
+		if constexpr (sizeof...(Futures) > 0) {
+			RaceImplCallback<Parent, Idx + 1, Futures...>::registerCallbacks();
+		}
+	}
+
+	void a_callback_fire(ThisCallback*, ValueType const& value) { getParent()->template finish<Idx>(value); }
+	void a_callback_fire(ThisCallback*, ValueType&& value) { getParent()->template finish<Idx>(std::move(value)); }
+
+	void a_callback_error(ThisCallback*, Error e) { getParent()->fail(e); }
+
+	void removeCallbacks() {
+		ThisCallback::remove();
+		if constexpr (sizeof...(Futures) > 0) {
+			RaceImplCallback<Parent, Idx + 1, Futures...>::removeCallbacks();
+		}
+	}
+};
+
+template <class Parent, int Idx>
+struct RaceImplCallback<Parent, Idx> {
+#ifdef ENABLE_SAMPLING
+	LineageReference* lineageAddr() { return currentLineage; }
+#endif
+};
+
+template <class Result, class... Futures>
+struct RaceImplActor final : Actor<Result>,
+                             RaceImplCallback<RaceImplActor<Result, Futures...>, 0, Futures...>,
+                             FastAllocated<RaceImplActor<Result, Futures...>> {
+	std::tuple<Futures...> futures;
+
+	using FastAllocated<RaceImplActor<Result, Futures...>>::operator new;
+	using FastAllocated<RaceImplActor<Result, Futures...>>::operator delete;
+
+	explicit RaceImplActor(std::tuple<Futures...>&& futures) : Actor<Result>(), futures(std::move(futures)) {
+		this->actor_wait_state = ACTOR_WAIT_STATE_WAITING;
+		RaceImplCallback<RaceImplActor<Result, Futures...>, 0, Futures...>::registerCallbacks();
+	}
+
+	template <std::size_t Idx, class T>
+	void finish(T&& value) {
+		this->actor_wait_state = ACTOR_WAIT_STATE_NOT_WAITING;
+		RaceImplCallback<RaceImplActor<Result, Futures...>, 0, Futures...>::removeCallbacks();
+		this->SAV<Result>::sendAndDelPromiseRef(Result(std::in_place_index<Idx>, std::forward<T>(value)));
+	}
+
+	void fail(Error e) {
+		this->actor_wait_state = ACTOR_WAIT_STATE_NOT_WAITING;
+		RaceImplCallback<RaceImplActor<Result, Futures...>, 0, Futures...>::removeCallbacks();
+		this->SAV<Result>::sendErrorAndDelPromiseRef(e);
+	}
+
+	void cancel() override {
+		const auto waitState = this->actor_wait_state;
+		this->actor_wait_state = ACTOR_WAIT_STATE_CANCELLED;
+		if (actorWaitStateIsWaiting(waitState)) {
+			RaceImplCallback<RaceImplActor<Result, Futures...>, 0, Futures...>::removeCallbacks();
+			this->SAV<Result>::sendErrorAndDelPromiseRef(actor_cancelled());
+		}
+	}
+
+	void destroy() override { delete this; }
+};
+
 } // namespace coro
 
 using Choose = coro::ChooseClause<>;
 
+// Waits for the first input Future/FutureStream to become ready and returns its value in a variant whose index
+// matches the winning argument. If multiple inputs are already ready, the lowest-index argument wins; for streams,
+// the next queued element is consumed. Errors and explicit cancellation propagate to the returned Future, while
+// non-winning inputs are only detached from the race. They are not actively cancelled here, but may still cancel
+// if dropping the race also drops their last reference.
+template <class... Futures>
+[[nodiscard]] auto race(Futures&&... futures) -> Future<coro::RaceResult<std::decay_t<Futures>...>> {
+	static_assert(sizeof...(Futures) > 0, "race requires at least one Future argument");
+
+	using Result = coro::RaceResult<std::decay_t<Futures>...>;
+	Future<Result> ready = coro::raceReady<0, Result>(futures...);
+	if (ready.isValid()) {
+		return ready;
+	}
+
+	return Future<Result>(
+	    new coro::RaceImplActor<Result, std::decay_t<Futures>...>(std::make_tuple(std::forward<Futures>(futures)...)));
+}
+
 template <class T, class F>
 AsyncGenerator<T> map(AsyncGenerator<T> gen, F pred) {
 	while (gen) {
diff --git a/flow/include/flow/Coroutines.h b/flow/include/flow/Coroutines.h
index 233a65d94e0..8ba78750bbb 100644
--- a/flow/include/flow/Coroutines.h
+++ b/flow/include/flow/Coroutines.h
@@ -36,10 +36,164 @@ namespace n_coroutine = ::std::experimental;
 
 #include "flow/flow.h"
 #include "flow/Error.h"
-#include "flow/CoroutinesImpl.h"
 
 struct Uncancellable {};
 
+// Marker parameter for coroutines that should destroy the coroutine frame on
+// cancellation instead of resuming it to throw actor_cancelled(). Coroutine
+// locals are cleaned up by RAII and catch blocks inside the coroutine are not
+// run for cancellation.
+struct NoThrowOnCancel {};
+
+// Marker parameter for coroutines that want `co_await Future<Void>` to
+// produce a value convertible to `Void` instead of `void`:
+//
+//   Future<Void> f(Future<Void> ready, ExplicitVoid = {}) {
+//     Void v = co_await ready;
+//     co_return;
+//   }
+//
+// Unmarked coroutines resume with co_await Future<Void>` -> `void`
+struct ExplicitVoid {
+	operator Void() const { return Void(); }
+};
+
+template <class T>
+class AsyncResult;
+
+namespace coro {
+template <class T>
+struct FutureIgnore;
+
+template <class SourceValue, class ResultValue = std::conditional_t<std::is_void_v<SourceValue>, Void, SourceValue>>
+struct FutureErrorOr;
+
+template <class T>
+struct AsyncResultState;
+
+template <class T>
+struct AsyncResultCallback;
+
+template <class T, bool IsCancellable, bool ReturnsExplicitVoid = false, bool NoThrowOnCancel = false>
+struct AsyncResultPromise;
+
+template <class PromiseType, class ValueType>
+struct AwaitableAsyncResult;
+
+template <class ValueType>
+struct AsyncResultAwaiter;
+
+template <class Parent, int Idx, class ValueType>
+struct ActorAsyncResultCallback;
+
+template <class ValueType>
+struct QuorumAsyncResultCallback;
+
+template <class ValueType>
+struct GetAllAsyncResultCallback;
+
+template <class PromiseType, class ValueType>
+struct AwaitableFutureIgnore;
+
+template <class PromiseType, class SourceValue, class ResultValue>
+struct AwaitableFutureErrorOr;
+} // namespace coro
+
+namespace coro {
+
+template <class T>
+struct FutureIgnore {
+	// Wrap a Future<T> so coroutine await_transform can resume on completion
+	// without materializing the T payload at the await site.
+	Future<T> future;
+};
+
+template <class T>
+FutureIgnore<T> ignore(Future<T> future) {
+	return FutureIgnore<T>{ std::move(future) };
+}
+
+template <class SourceValue, class ResultValue>
+struct FutureErrorOr {
+	// Reuse Future<T> storage but request a non-throwing await_resume() that
+	// converts completion into ErrorOr<ResultValue>.
+	Future<std::conditional_t<std::is_void_v<SourceValue>, Void, SourceValue>> future;
+};
+
+template <class T>
+FutureErrorOr<T> errorOr(Future<T> future) {
+	return FutureErrorOr<T>{ std::move(future) };
+}
+
+template <class T>
+FutureErrorOr<T, Void> errorOr(FutureIgnore<T> future) {
+	return FutureErrorOr<T, Void>{ std::move(future.future) };
+}
+
+} // namespace coro
+
+// Move-only coroutine result that transfers ownership through co_await.
+// Unlike Future<T>, awaiting AsyncResult<T> produces T by value so expensive
+// payloads do not need an extra copy at the await site.
+template <class T>
+class SWIFT_SENDABLE AsyncResult {
+public:
+	static_assert(!std::is_void_v<T>, "Use AsyncResult<Void> instead of AsyncResult<void>");
+	using Element = T;
+	using StoredT = T;
+
+	AsyncResult() noexcept : resultState(nullptr) {}
+	AsyncResult(AsyncResult const&) = delete;
+	AsyncResult& operator=(AsyncResult const&) = delete;
+	AsyncResult(AsyncResult&& rhs) noexcept : resultState(rhs.resultState) { rhs.resultState = nullptr; }
+	AsyncResult& operator=(AsyncResult&& rhs) noexcept {
+		if (this != &rhs) {
+			release();
+			resultState = rhs.resultState;
+			rhs.resultState = nullptr;
+		}
+		return *this;
+	}
+	~AsyncResult() { release(); }
+
+	bool isValid() const { return resultState != nullptr; }
+	bool isReady() const;
+	bool isError() const;
+	bool canGet() const;
+	Error& getError() const;
+	void cancel() const;
+	void addCallbackAndClear(coro::AsyncResultCallback<StoredT>* cb) &&;
+
+	T const& get() const&;
+	T get() &&;
+	T getValue() const& { return get(); }
+	T getValue() && { return std::move(*this).get(); }
+
+	auto operator co_await() &;
+	auto operator co_await() &&;
+
+private:
+	explicit AsyncResult(coro::AsyncResultState<StoredT>* resultState) noexcept : resultState(resultState) {}
+	void release();
+
+	coro::AsyncResultState<StoredT>* resultState;
+
+	template <class U, bool IsCancellable, bool ReturnsExplicitVoid, bool NoThrowOnCancel>
+	friend struct coro::AsyncResultPromise;
+	template <class PromiseType, class ValueType>
+	friend struct coro::AwaitableAsyncResult;
+	template <class ValueType>
+	friend struct coro::AsyncResultAwaiter;
+	template <class Parent, int Idx, class ValueType>
+	friend struct coro::ActorAsyncResultCallback;
+	template <class ValueType>
+	friend struct coro::QuorumAsyncResultCallback;
+	template <class ValueType>
+	friend struct coro::GetAllAsyncResultCallback;
+};
+
+#include "flow/CoroutinesImpl.h"
+
 template <class T>
 class AsyncGenerator {
 	PromiseStream<T>* nextPromise;
@@ -87,12 +241,12 @@ class Generator {
 public:
 	explicit Generator(handle_type h) : handle(h) {}
 	Generator() {}
-	Generator(Generator const& other) : handle(other.handle) {
+	explicit(false) Generator(Generator const& other) : handle(other.handle) {
 		if (handle) {
 			handle.promise().addRef();
 		}
 	}
-	Generator(Generator&& other) : handle(std::move(other.handle)) { other.handle = handle_type{}; }
+	explicit(false) Generator(Generator&& other) : handle(std::move(other.handle)) { other.handle = handle_type{}; }
 	~Generator() {
 		if (handle) {
 			handle.promise().delRef();
@@ -155,13 +309,97 @@ class Generator {
 
 template <typename ReturnValue, typename... Args>
 struct [[maybe_unused]] n_coroutine::coroutine_traits<Future<ReturnValue>, Args...> {
-	using promise_type = coro::CoroPromise<ReturnValue, !coro::hasUncancellable<Args...>>;
+	static_assert(!(coro::hasUncancellable<Args...> && coro::hasNoThrowOnCancel<Args...>),
+	              "NoThrowOnCancel and Uncancellable are mutually exclusive");
+	using promise_type = coro::CoroPromise<ReturnValue,
+	                                       !coro::hasUncancellable<Args...>,
+	                                       coro::hasExplicitVoid<Args...>,
+	                                       coro::hasNoThrowOnCancel<Args...>>;
+};
+
+template <typename ReturnValue, typename... Args>
+struct [[maybe_unused]] n_coroutine::coroutine_traits<AsyncResult<ReturnValue>, Args...> {
+	static_assert(!(coro::hasUncancellable<Args...> && coro::hasNoThrowOnCancel<Args...>),
+	              "NoThrowOnCancel and Uncancellable are mutually exclusive");
+	using promise_type = coro::AsyncResultPromise<ReturnValue,
+	                                              !coro::hasUncancellable<Args...>,
+	                                              coro::hasExplicitVoid<Args...>,
+	                                              coro::hasNoThrowOnCancel<Args...>>;
 };
 
 template <typename ReturnValue, typename... Args>
 struct [[maybe_unused]] n_coroutine::coroutine_traits<AsyncGenerator<ReturnValue>, Args...> {
 	static_assert(!coro::hasUncancellable<Args...>, "AsyncGenerator can't be uncancellable");
-	using promise_type = coro::AsyncGeneratorPromise<ReturnValue>;
+	static_assert(!coro::hasNoThrowOnCancel<Args...>, "AsyncGenerator can't use NoThrowOnCancel");
+	using promise_type = coro::AsyncGeneratorPromise<ReturnValue, coro::hasExplicitVoid<Args...>>;
 };
 
+template <class T>
+bool AsyncResult<T>::isReady() const {
+	return resultState && resultState->isReady();
+}
+
+template <class T>
+bool AsyncResult<T>::isError() const {
+	return resultState && resultState->isError();
+}
+
+template <class T>
+bool AsyncResult<T>::canGet() const {
+	return resultState && resultState->canGet();
+}
+
+template <class T>
+Error& AsyncResult<T>::getError() const {
+	ASSERT(resultState);
+	return resultState->getError();
+}
+
+template <class T>
+void AsyncResult<T>::cancel() const {
+	if (resultState) {
+		resultState->cancelProducer();
+	}
+}
+
+template <class T>
+void AsyncResult<T>::addCallbackAndClear(coro::AsyncResultCallback<StoredT>* cb) && {
+	ASSERT(resultState);
+	resultState->registerCallback(cb);
+	resultState = nullptr;
+}
+
+template <class T>
+T const& AsyncResult<T>::get() const& {
+	ASSERT(resultState);
+	return resultState->get();
+}
+
+template <class T>
+T AsyncResult<T>::get() && {
+	ASSERT(resultState);
+	return resultState->take();
+}
+
+template <class T>
+auto AsyncResult<T>::operator co_await() & {
+	return coro::AsyncResultAwaiter<T>{ std::move(*this) };
+}
+
+template <class T>
+auto AsyncResult<T>::operator co_await() && {
+	return coro::AsyncResultAwaiter<T>{ std::move(*this) };
+}
+
+template <class T>
+void AsyncResult<T>::release() {
+	if (resultState) {
+		if (!resultState->isReady()) {
+			resultState->cancelProducer();
+		}
+		resultState->delRef();
+		resultState = nullptr;
+	}
+}
+
 #endif // FLOW_COROUTINES_H
diff --git a/flow/include/flow/CoroutinesImpl.h b/flow/include/flow/CoroutinesImpl.h
index c9ee5a903cb..e0b5838ed2c 100644
--- a/flow/include/flow/CoroutinesImpl.h
+++ b/flow/include/flow/CoroutinesImpl.h
@@ -30,10 +30,21 @@ class Generator;
 template <class T>
 class AsyncGenerator;
 
+template <class T>
+class AsyncResult;
+
 struct Uncancellable;
+struct NoThrowOnCancel;
 
 namespace coro {
 
+// Awaiters register one of these while suspended so NoThrowOnCancel can detach
+// the pending callback or continuation before destroying the coroutine frame.
+struct AwaitCancelHandler {
+	virtual ~AwaitCancelHandler() = default;
+	virtual void cancelWait() = 0;
+};
+
 template <class F>
 struct FutureReturnType;
 
@@ -57,10 +68,25 @@ struct FutureReturnType<FutureStream<T> const&> {
 	using type = T;
 };
 
+template <class T>
+struct FutureReturnType<AsyncResult<T>> {
+	using type = T;
+};
+
+template <class T>
+struct FutureReturnType<AsyncResult<T>&> {
+	using type = T;
+};
+
+template <class T>
+struct FutureReturnType<AsyncResult<T> const&> {
+	using type = T;
+};
+
 template <class F>
 using FutureReturnTypeT = typename FutureReturnType<F>::type;
 
-enum class FutureType { FutureStream, Future };
+enum class FutureType { FutureStream, Future, AsyncResult };
 
 template <class F>
 struct GetFutureType;
@@ -85,6 +111,21 @@ struct GetFutureType<FutureStream<T> const&> {
 	constexpr static FutureType value = FutureType::FutureStream;
 };
 
+template <class T>
+struct GetFutureType<AsyncResult<T>> {
+	constexpr static FutureType value = FutureType::AsyncResult;
+};
+
+template <class T>
+struct GetFutureType<AsyncResult<T>&> {
+	constexpr static FutureType value = FutureType::AsyncResult;
+};
+
+template <class T>
+struct GetFutureType<AsyncResult<T> const&> {
+	constexpr static FutureType value = FutureType::AsyncResult;
+};
+
 template <class F>
 inline constexpr FutureType GetFutureTypeV = GetFutureType<F>::value;
 
@@ -108,11 +149,10 @@ struct CoroActor final : Actor<std::conditional_t<std::is_void_v<T>, Void, T>> {
 		if constexpr (IsCancellable) {
 			auto prev_wait_state = Actor<ValType>::actor_wait_state;
 
-			// Set wait state to -1
-			Actor<ValType>::actor_wait_state = -1;
+			Actor<ValType>::actor_wait_state = ACTOR_WAIT_STATE_CANCELLED;
 
 			// If the actor is waiting, then resume the coroutine to throw actor_cancelled().
-			if (prev_wait_state > 0) {
+			if (actorWaitStateIsWaiting(prev_wait_state)) {
 				auto h = handle; // Copy to local — frame may be freed during resume
 				h.resume();
 			}
@@ -125,21 +165,246 @@ struct CoroActor final : Actor<std::conditional_t<std::is_void_v<T>, Void, T>> {
 	}
 };
 
+// Separate the Actor/SAV state from the coroutine frame because cancellation
+// destroys the frame while returned Future objects may still observe the
+// cancelled result.
+template <class T>
+struct NoThrowOnCancelCoroActor final : Actor<std::conditional_t<std::is_void_v<T>, Void, T>> {
+	using ValType = std::conditional_t<std::is_void_v<T>, Void, T>;
+
+	n_coroutine::coroutine_handle<> handle;
+	AwaitCancelHandler* cancelHandler = nullptr;
+
+	static void* operator new(size_t s) { return allocateFast(int(s)); }
+	static void operator delete(void* p, size_t s) { freeFast(int(s), p); }
+
+	int8_t& waitState() { return Actor<ValType>::actor_wait_state; }
+
+	template <class U>
+	void set(U&& value) {
+		new (&SAV<ValType>::value()) ValType(std::forward<U>(value));
+		SAV<ValType>::error_state = Error(SAV<ValType>::SET_ERROR_CODE);
+	}
+
+	void setError(Error const& e) { SAV<ValType>::error_state = e; }
+
+	void setCancelHandler(AwaitCancelHandler* handler) { cancelHandler = handler; }
+	void clearCancelHandler(AwaitCancelHandler* handler) {
+		if (cancelHandler == handler) {
+			cancelHandler = nullptr;
+		}
+	}
+
+	void destroyFrame() {
+		auto h = handle;
+		// The coroutine is gone after destroy(); leave no handle for a later
+		// Future drop to destroy again.
+		handle = {};
+		h.destroy();
+	}
+
+	void cancel() override {
+		if (!SAV<ValType>::canBeSet()) {
+			return;
+		}
+
+		if (cancelHandler) {
+			// The handler object is stored in the coroutine frame, so unregister
+			// it from its wait source before destroying the frame below.
+			auto handler = cancelHandler;
+			cancelHandler = nullptr;
+			handler->cancelWait();
+		}
+
+		destroyFrame();
+		if (SAV<ValType>::futures) {
+			SAV<ValType>::sendErrorAndDelPromiseRef(actor_cancelled());
+		} else {
+			SAV<ValType>::promises = 0;
+			delete this;
+		}
+	}
+
+	void destroy() override {
+		if (handle) {
+			destroyFrame();
+		}
+		delete this;
+	}
+};
+
+template <class T>
+struct AsyncResultCallback {
+	virtual ~AsyncResultCallback() = default;
+	virtual void fire(T const&) {}
+	virtual void fire(T&&) {}
+	virtual void error(Error) {}
+	virtual void unwait() {}
+};
+
+template <class T>
+struct AsyncResultState : FastAllocated<AsyncResultState<T>> {
+	int refs = 1;
+	bool ready = false;
+	bool hasValue = false;
+	bool cancellable = false;
+	bool noThrowOnCancel = false;
+
+private:
+	typename std::aligned_storage<sizeof(T), __alignof(T)>::type value_storage;
+
+public:
+	Error error;
+	n_coroutine::coroutine_handle<> continuation;
+	AsyncResultCallback<T>* callback = nullptr;
+	n_coroutine::coroutine_handle<> producerHandle;
+	int8_t* producerWaitState = nullptr;
+	// Only populated while the producer is suspended in an awaiter. The awaiter
+	// lives in the producer frame and must be detached before no-throw cancel
+	// destroys that frame.
+	AwaitCancelHandler* producerCancelHandler = nullptr;
+
+	~AsyncResultState() {
+		if (hasValue) {
+			value().~T();
+		}
+	}
+
+	T& value() { return *reinterpret_cast<T*>(&value_storage); }
+	T const& value() const { return *reinterpret_cast<T const*>(&value_storage); }
+
+	void addRef() { ++refs; }
+	void delRef() {
+		if (!--refs) {
+			delete this;
+		}
+	}
+
+	bool isReady() const { return ready; }
+	bool isError() const { return ready && error.isValid(); }
+	bool canGet() const { return ready && !error.isValid(); }
+
+	Error& getError() {
+		ASSERT(isError());
+		return error;
+	}
+
+	template <class U>
+	void setValue(U&& v) {
+		ASSERT(!hasValue && !error.isValid());
+		new (&value_storage) T(std::forward<U>(v));
+		hasValue = true;
+	}
+
+	void setError(Error const& e) {
+		ASSERT(!hasValue && !error.isValid());
+		error = e;
+	}
+
+	T const& get() const {
+		ASSERT(canGet());
+		return value();
+	}
+
+	T take() {
+		ASSERT(canGet());
+		return std::move(value());
+	}
+
+	void registerContinuation(n_coroutine::coroutine_handle<> h) {
+		ASSERT(!ready);
+		ASSERT(!callback);
+		ASSERT(!continuation);
+		continuation = h;
+	}
+
+	void clearContinuation() { continuation = {}; }
+
+	void registerCallback(AsyncResultCallback<T>* cb) {
+		ASSERT(!ready);
+		ASSERT(!continuation);
+		ASSERT(!callback);
+		callback = cb;
+	}
+
+	void clearCallback(AsyncResultCallback<T>* cb) {
+		if (callback == cb) {
+			callback = nullptr;
+		}
+	}
+
+	void cancelProducer() {
+		if (!cancellable || ready || !producerHandle || !producerWaitState) {
+			return;
+		}
+
+		if (noThrowOnCancel) {
+			if (producerCancelHandler) {
+				// The handler object is inside producerHandle's coroutine frame.
+				// Clear the wait source first so it cannot resume freed storage.
+				auto handler = producerCancelHandler;
+				producerCancelHandler = nullptr;
+				handler->cancelWait();
+			}
+			auto h = producerHandle;
+			producerHandle = {};
+			producerWaitState = nullptr;
+			// Hold the consumer-visible state, but discard the producer frame
+			// instead of resuming it through actor_cancelled().
+			h.destroy();
+			setError(actor_cancelled());
+			complete();
+			delRef();
+			return;
+		}
+
+		auto prevWaitState = *producerWaitState;
+		*producerWaitState = ACTOR_WAIT_STATE_CANCELLED;
+		if (actorWaitStateIsWaiting(prevWaitState)) {
+			auto h = producerHandle;
+			h.resume();
+		}
+	}
+
+	void complete() {
+		ready = true;
+		producerHandle = {};
+		producerWaitState = nullptr;
+		producerCancelHandler = nullptr;
+		if (callback) {
+			auto cb = callback;
+			callback = nullptr;
+			if (error.isValid()) {
+				cb->error(error);
+			} else {
+				cb->fire(std::move(value()));
+			}
+		} else if (continuation) {
+			auto h = continuation;
+			continuation = {};
+			h.resume();
+		}
+	}
+};
+
 template <class U>
 struct AwaitableFutureStore {
-	std::variant<Error, U> data;
+	std::variant<std::monostate, Error, U> data;
 
-	constexpr bool isSet() const noexcept { return data.index() != 0 || std::get<0>(data).isValid(); }
+	constexpr bool isSet() const noexcept { return data.index() != 0; }
 	void copy(U v) { data = std::move(v); }
 	void set(U&& v) { data = std::move(v); }
 	void set(U const& v) { data = v; }
+	void setError(Error error) { data = std::move(error); }
 
 	const U& getRef() const {
 		switch (data.index()) {
 		case 0:
-			throw std::get<0>(data);
+			throw internal_error();
 		case 1:
-			return std::get<1>(data);
+			throw std::get<1>(data);
+		case 2:
+			return std::get<2>(data);
 		}
 		UNREACHABLE();
 	}
@@ -147,36 +412,124 @@ struct AwaitableFutureStore {
 	U&& get() && {
 		switch (data.index()) {
 		case 0:
-			throw std::get<0>(data);
+			throw internal_error();
 		case 1:
-			return std::get<1>(std::move(data));
+			throw std::get<1>(data);
+		case 2:
+			return std::get<2>(std::move(data));
 		}
 		UNREACHABLE();
 	}
 };
 
+// Awaiter adapter for `AsyncResult<T>` inside coroutine promises.
+//
+// `PromiseType` is the coroutine promise object that owns the wait-state and
+// coroutine handle bookkeeping for the suspended actor.
+// `ValueType` is the logical `AsyncResult<T>` payload type being awaited.
+template <class PromiseType, class ValueType>
+struct AwaitableAsyncResult : AwaitCancelHandler {
+	using StateType = typename AsyncResult<ValueType>::StoredT;
+
+	AsyncResult<ValueType> result;
+	PromiseType* pt = nullptr;
+
+	AwaitableAsyncResult(AsyncResult<ValueType>&& result, PromiseType* pt) : result(std::move(result)), pt(pt) {}
+
+	[[nodiscard]] bool await_ready() const {
+		ASSERT(result.resultState);
+		if (actorWaitStateIsCancelled(pt->waitState())) {
+			pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
+			return true;
+		}
+		return result.resultState->isReady();
+	}
+
+	void await_suspend(n_coroutine::coroutine_handle<> h) {
+		ASSERT(result.resultState);
+		pt->setHandle(h);
+		pt->waitState() = ACTOR_WAIT_STATE_WAITING;
+		result.resultState->registerContinuation(h);
+		pt->setCancelHandler(this);
+	}
+
+	void cancelWait() override {
+		// AsyncResultState only stores one waiting continuation, so cancelling
+		// this await means the continuation must no longer be resumed.
+		if (result.resultState) {
+			result.resultState->clearContinuation();
+		}
+	}
+
+	bool resumeImpl() {
+		pt->clearCancelHandler(this);
+		switch (pt->waitState()) {
+		case ACTOR_WAIT_STATE_CANCELLED:
+			if (result.resultState) {
+				result.resultState->clearContinuation();
+			}
+		case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
+			throw actor_cancelled();
+		}
+
+		bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
+		if (actorWaitStateIsWaiting(pt->waitState())) {
+			result.resultState->clearContinuation();
+			pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
+		}
+		return wasReady;
+	}
+
+	void await_resume()
+	    requires(std::is_void_v<ValueType>)
+	{
+		if (pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			resumeImpl();
+		}
+		if (result.resultState->isError()) {
+			throw result.resultState->getError();
+		}
+	}
+
+	ValueType await_resume()
+	    requires(!std::is_void_v<ValueType>)
+	{
+		if (pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			resumeImpl();
+		}
+		if (result.resultState->isError()) {
+			throw result.resultState->getError();
+		}
+		return result.resultState->take();
+	}
+};
+
 template <class T>
 using ToFutureVal = std::conditional_t<std::is_void_v<T>, Void, T>;
 
-template <class F, class U, bool IsStream>
+template <class Awaiter, class ValueType, bool IsStream, bool ReturnsExplicitVoid = false>
 struct AwaitableResume;
 
-template <class F>
-struct AwaitableResume<F, Void, false> {
+template <class Awaiter>
+struct AwaitableResume<Awaiter, Void, /* IsStream = */ false, /* ReturnsExplicitVoid = */ false> {
 	[[maybe_unused]] void await_resume() {
-		auto self = static_cast<F*>(this);
-		self->resumeImpl();
+		auto self = static_cast<Awaiter*>(this);
+		if (self->pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			self->resumeImpl();
+		}
 		if (self->future.isError()) {
 			throw self->future.getError();
 		}
 	}
 };
 
-template <class F, class T>
-struct AwaitableResume<F, T, false> {
-	T const& await_resume() {
-		auto self = static_cast<F*>(this);
-		self->resumeImpl();
+template <class Awaiter, class ValueType>
+struct AwaitableResume<Awaiter, ValueType, /* IsStream = */ false, /* ReturnsExplicitVoid = */ false> {
+	ValueType const& await_resume() {
+		auto self = static_cast<Awaiter*>(this);
+		if (self->pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			self->resumeImpl();
+		}
 		if (self->future.isError()) {
 			throw self->future.getError();
 		}
@@ -184,10 +537,34 @@ struct AwaitableResume<F, T, false> {
 	}
 };
 
-template <class F, class T>
-struct AwaitableResume<F, T, true> {
-	T await_resume() {
-		auto self = static_cast<F*>(this);
+template <class Awaiter, class ValueType>
+struct AwaitableResume<Awaiter, ValueType, /* IsStream = */ false, /* ReturnsExplicitVoid = */ true>
+  : AwaitableResume<Awaiter, ValueType, /* IsStream = */ false, /* ReturnsExplicitVoid = */ false> {};
+
+template <class Awaiter>
+struct AwaitableResume<Awaiter, Void, /* IsStream = */ false, /* ReturnsExplicitVoid = */ true> {
+	[[maybe_unused]] ExplicitVoid await_resume() {
+		auto self = static_cast<Awaiter*>(this);
+		if (self->pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			self->resumeImpl();
+		}
+		if (self->future.isError()) {
+			throw self->future.getError();
+		}
+		return ExplicitVoid{};
+	}
+};
+
+template <class Awaiter, class ValueType>
+struct AwaitableResume<Awaiter, ValueType, /* IsStream = */ true, /* ReturnsExplicitVoid = */ false> {
+	ValueType await_resume() {
+		auto self = static_cast<Awaiter*>(this);
+		if (self->pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING) {
+			if (self->future.isError()) {
+				throw self->future.getError();
+			}
+			return self->future.pop();
+		}
 		if (self->resumeImpl()) {
 			if (self->future.isError()) {
 				throw self->future.getError();
@@ -198,19 +575,30 @@ struct AwaitableResume<F, T, true> {
 	}
 };
 
-template <class promise_type, class U, bool IsStream>
-struct AwaitableFuture : std::conditional_t<IsStream, SingleCallback<ToFutureVal<U>>, Callback<ToFutureVal<U>>>,
-                         AwaitableResume<AwaitableFuture<promise_type, U, IsStream>, U, IsStream> {
-	using FutureValue = ToFutureVal<U>;
+template <class Awaiter, class ValueType>
+struct AwaitableResume<Awaiter, ValueType, /* IsStream = */ true, /* ReturnsExplicitVoid = */ true>
+  : AwaitableResume<Awaiter, ValueType, /* IsStream = */ true, /* ReturnsExplicitVoid = */ false> {};
+
+// Awaiter for `Future<T>` and `FutureStream<T>` values transformed through a
+// coroutine promise.
+template <class PromiseType, class ValueType, bool IsStream, bool ReturnsExplicitVoid = false>
+struct AwaitableFuture
+  : std::conditional_t<IsStream, SingleCallback<ToFutureVal<ValueType>>, Callback<ToFutureVal<ValueType>>>,
+    AwaitCancelHandler,
+    AwaitableResume<AwaitableFuture<PromiseType, ValueType, IsStream, ReturnsExplicitVoid>,
+                    ValueType,
+                    IsStream,
+                    ReturnsExplicitVoid> {
+	using FutureValue = ToFutureVal<ValueType>;
 	using FutureType = std::conditional_t<IsStream, FutureStream<FutureValue>, Future<FutureValue> const&>;
 	FutureType future;
-	promise_type* pt = nullptr;
+	PromiseType* pt = nullptr;
 
 	// Store is only needed for streams — non-stream values are already in the SAV after fire()
 	struct Empty {};
 	[[no_unique_address]] std::conditional_t<IsStream, AwaitableFutureStore<FutureValue>, Empty> store;
 
-	AwaitableFuture(const FutureType& f, promise_type* pt) : future(f), pt(pt) {}
+	AwaitableFuture(const FutureType& f, PromiseType* pt) : future(f), pt(pt) {}
 
 	void fire(FutureValue const& value) override {
 		if constexpr (IsStream) {
@@ -227,15 +615,14 @@ struct AwaitableFuture : std::conditional_t<IsStream, SingleCallback<ToFutureVal
 
 	void error(Error error) override {
 		if constexpr (IsStream) {
-			store.data = error;
+			store.setError(error);
 		}
 		pt->resume();
 	}
 
 	[[maybe_unused]] [[nodiscard]] bool await_ready() const {
-		if (pt->waitState() < 0) {
-			pt->waitState() = -2;
-			// actor was cancelled
+		if (actorWaitStateIsCancelled(pt->waitState())) {
+			pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
 			return true;
 		}
 		return future.isReady();
@@ -246,7 +633,7 @@ struct AwaitableFuture : std::conditional_t<IsStream, SingleCallback<ToFutureVal
 		pt->setHandle(h);
 
 		// Set wait_state and add callback
-		pt->waitState() = 1;
+		pt->waitState() = ACTOR_WAIT_STATE_WAITING;
 
 		if constexpr (IsStream) {
 			auto sf = future;
@@ -255,41 +642,148 @@ struct AwaitableFuture : std::conditional_t<IsStream, SingleCallback<ToFutureVal
 			StrictFuture<FutureValue> sf = future;
 			sf.addCallbackAndClear(this);
 		}
+		pt->setCancelHandler(this);
 	}
 
+	// NoThrowOnCancel destroys the coroutine frame instead of resuming this
+	// awaiter, so detach the callback from the Future before that happens.
+	void cancelWait() override { this->remove(); }
+
 	bool resumeImpl() {
+		pt->clearCancelHandler(this);
 		// If actor is cancelled, then throw actor_cancelled()
 		switch (pt->waitState()) {
-		case -1:
+		case ACTOR_WAIT_STATE_CANCELLED:
 			this->remove();
-		case -2:
-			// -2 means that the `await_suspend` call returned `true`, so we shouldn't remove the callback.
-			// if the wait_state is -1 we still have to throw, so we fall through to the -2 case
+		case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
+			// await_ready() observed cancellation before await_suspend() registered a callback, so there is nothing to
+			// remove here.
 			throw actor_cancelled();
 		}
 
-		bool wasReady = pt->waitState() == 0;
+		bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
 		// Actor return from waiting, remove callback and reset wait_state.
-		if (pt->waitState() > 0) {
+		if (actorWaitStateIsWaiting(pt->waitState())) {
 			this->remove();
 
-			pt->waitState() = 0;
+			pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
 		}
 		return wasReady;
 	}
 };
 
+template <class PromiseType, class ValueType>
+struct AwaitableFutureOwning : Callback<ToFutureVal<ValueType>>, AwaitCancelHandler {
+	using FutureValue = ToFutureVal<ValueType>;
+	Future<FutureValue> future;
+	PromiseType* pt = nullptr;
+
+	AwaitableFutureOwning(Future<FutureValue> future, PromiseType* pt) : future(std::move(future)), pt(pt) {}
+
+	void fire(FutureValue const&) override { pt->resume(); }
+	void fire(FutureValue&&) override { pt->resume(); }
+	void error(Error) override { pt->resume(); }
+
+	[[maybe_unused]] [[nodiscard]] bool await_ready() const {
+		if (actorWaitStateIsCancelled(pt->waitState())) {
+			pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
+			return true;
+		}
+		return future.isReady();
+	}
+
+	[[maybe_unused]] void await_suspend(n_coroutine::coroutine_handle<> h) {
+		pt->setHandle(h);
+		pt->waitState() = ACTOR_WAIT_STATE_WAITING;
+
+		StrictFuture<FutureValue> sf = future;
+		sf.addCallbackAndClear(this);
+		pt->setCancelHandler(this);
+	}
+
+	// NoThrowOnCancel destroys the coroutine frame instead of resuming this
+	// awaiter, so detach the callback from the Future before that happens.
+	void cancelWait() override { this->remove(); }
+
+	bool resumeImpl() {
+		pt->clearCancelHandler(this);
+		switch (pt->waitState()) {
+		case ACTOR_WAIT_STATE_CANCELLED:
+			this->remove();
+		case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
+			throw actor_cancelled();
+		}
+
+		bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
+		if (actorWaitStateIsWaiting(pt->waitState())) {
+			this->remove();
+			pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
+		}
+		return wasReady;
+	}
+};
+
+template <class PromiseType, class ValueType>
+struct AwaitableFutureIgnore : AwaitableFutureOwning<PromiseType, ValueType> {
+	using Base = AwaitableFutureOwning<PromiseType, ValueType>;
+
+	AwaitableFutureIgnore(Future<ToFutureVal<ValueType>> future, PromiseType* pt) : Base(std::move(future), pt) {}
+
+	Void await_resume() {
+		auto self = static_cast<Base*>(this);
+		if (self->pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			self->resumeImpl();
+		}
+		// Preserve normal Future<T> error propagation while discarding any
+		// successful T payload.
+		if (self->future.isError()) {
+			throw self->future.getError();
+		}
+		return Void();
+	}
+};
+
+template <class PromiseType, class SourceValue, class ResultValue>
+struct AwaitableFutureErrorOr : AwaitableFutureOwning<PromiseType, SourceValue> {
+	using Base = AwaitableFutureOwning<PromiseType, SourceValue>;
+	using ResultType = ErrorOr<ResultValue>;
+
+	AwaitableFutureErrorOr(Future<ToFutureVal<SourceValue>> future, PromiseType* pt) : Base(std::move(future), pt) {}
+
+	ResultType await_resume() {
+		auto self = static_cast<Base*>(this);
+		if (self->pt->waitState() != ACTOR_WAIT_STATE_NOT_WAITING) {
+			self->resumeImpl();
+		}
+		// Convert failed Future<T> awaits into ErrorOr instead of throwing so
+		// callers can cheaply observe completion-only state.
+		if (self->future.isError()) {
+			return ResultType(self->future.getError());
+		}
+		if constexpr (std::is_same_v<ResultValue, Void>) {
+			return ResultType(Void());
+		} else {
+			return ResultType(self->future.get());
+		}
+	}
+};
+
 // TODO: This can be merged with AwaitableFutureStream by passing more template arguments.
-template <class promise_type, class U>
-struct ThreadAwaitableFutureStream : SingleCallback<ToFutureVal<U>>,
-                                     AwaitableResume<ThreadAwaitableFutureStream<promise_type, U>, U, true> {
-	using FutureValue = ToFutureVal<U>;
+template <class PromiseType, class ValueType, bool ReturnsExplicitVoid = false>
+struct ThreadAwaitableFutureStream
+  : SingleCallback<ToFutureVal<ValueType>>,
+    AwaitCancelHandler,
+    AwaitableResume<ThreadAwaitableFutureStream<PromiseType, ValueType, ReturnsExplicitVoid>,
+                    ValueType,
+                    true,
+                    ReturnsExplicitVoid> {
+	using FutureValue = ToFutureVal<ValueType>;
 	using FutureType = ThreadFutureStream<FutureValue>;
 	FutureType future;
-	promise_type* pt = nullptr;
+	PromiseType* pt = nullptr;
 	AwaitableFutureStore<FutureValue> store;
 
-	ThreadAwaitableFutureStream(const FutureType& f, promise_type* pt) : future(f), pt(pt) {}
+	ThreadAwaitableFutureStream(const FutureType& f, PromiseType* pt) : future(f), pt(pt) {}
 
 	void fire(FutureValue const& value) override {
 		store.set(value);
@@ -301,13 +795,13 @@ struct ThreadAwaitableFutureStream : SingleCallback<ToFutureVal<U>>,
 	}
 
 	void error(Error error) override {
-		store.data = error;
+		store.setError(error);
 		pt->resume();
 	}
 
 	[[maybe_unused]] [[nodiscard]] bool await_ready() const {
-		if (pt->waitState() < 0) {
-			pt->waitState() = -2;
+		if (actorWaitStateIsCancelled(pt->waitState())) {
+			pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
 			// actor was cancelled
 			return true;
 		}
@@ -319,72 +813,185 @@ struct ThreadAwaitableFutureStream : SingleCallback<ToFutureVal<U>>,
 		pt->setHandle(h);
 
 		// Set wait_state and add callback
-		pt->waitState() = 1;
+		pt->waitState() = ACTOR_WAIT_STATE_WAITING;
 
 		auto sf = future;
 		sf.addCallbackAndClear(this);
+		pt->setCancelHandler(this);
 	}
 
+	// NoThrowOnCancel destroys the coroutine frame instead of resuming this
+	// awaiter, so detach the callback from the FutureStream before that happens.
+	void cancelWait() override { this->remove(); }
+
 	bool resumeImpl() {
+		pt->clearCancelHandler(this);
 		// If actor is cancelled, then throw actor_cancelled()
 		switch (pt->waitState()) {
-		case -1:
+		case ACTOR_WAIT_STATE_CANCELLED:
 			this->remove();
-		case -2:
-			// -2 means that the `await_suspend` call returned `true`, so we shouldn't remove the callback.
+		case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
+			// await_ready() observed cancellation before await_suspend() registered a callback, so there is nothing to
+			// remove here.
 			// if the wait_state is -1 we still have to throw, so we fall through to the -2 case
 			throw actor_cancelled();
 		}
 
-		bool wasReady = pt->waitState() == 0;
+		bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
 		// Actor return from waiting, remove callback and reset wait_state.
-		if (pt->waitState() > 0) {
+		if (actorWaitStateIsWaiting(pt->waitState())) {
 			this->remove();
 
-			pt->waitState() = 0;
+			pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
 		}
 		return wasReady;
 	}
 };
 
-template <class T, class Promise>
+template <class T, class Promise, bool ReturnsExplicitVoid = false>
 struct CoroReturn {
 	template <class U>
 	void return_value(U&& value) {
-		static_cast<Promise*>(this)->coroActor.set(std::forward<U>(value));
+		static_cast<Promise*>(this)->setReturnValue(std::forward<U>(value));
 	}
 };
 
 template <class Promise>
-struct CoroReturn<Void, Promise> {
-	void return_void() { static_cast<Promise*>(this)->coroActor.set(Void()); }
+struct CoroReturn<Void, Promise, false> {
+	void return_void() { static_cast<Promise*>(this)->setReturnValue(Void()); }
 };
 
-template <class T, bool IsCancellable>
-struct CoroPromise : CoroReturn<T, CoroPromise<T, IsCancellable>> {
-	using promise_type = CoroPromise<T, IsCancellable>;
-	using ActorType = coro::CoroActor<T, IsCancellable>;
+template <class Promise>
+struct CoroReturn<Void, Promise, true> {
+	template <std::convertible_to<Void> U>
+	void return_value(U&&) {
+		static_cast<Promise*>(this)->setReturnValue(Void());
+	}
+};
+
+template <class T, class Promise, bool ReturnsExplicitVoid = false>
+struct AsyncResultReturn {
+	template <class U>
+	void return_value(U&& value) {
+		static_cast<Promise*>(this)->state->setValue(std::forward<U>(value));
+	}
+};
+
+template <class Promise>
+struct AsyncResultReturn<Void, Promise, false> {
+	void return_void() { static_cast<Promise*>(this)->state->setValue(Void()); }
+};
+
+template <class Promise>
+struct AsyncResultReturn<Void, Promise, true> {
+	template <std::convertible_to<Void> U>
+	void return_value(U&&) {
+		static_cast<Promise*>(this)->state->setValue(Void());
+	}
+};
+
+template <class T, bool IsCancellable, bool ReturnsExplicitVoid = false, bool NoThrowOnCancel = false>
+struct CoroPromise;
+
+template <class Derived, class T, bool ReturnsExplicitVoid>
+struct CoroPromiseBase : CoroReturn<T, Derived, ReturnsExplicitVoid> {
+	using promise_type = Derived;
 	using ReturnValue = std::conditional_t<std::is_void_v<T>, Void, T>;
 	using ReturnFutureType = Future<ReturnValue>;
 
-	ActorType coroActor; // Embedded in coroutine frame — single allocation
-
-	CoroPromise() {}
+	promise_type* self() { return static_cast<promise_type*>(this); }
 
 	n_coroutine::coroutine_handle<promise_type> handle() {
-		return n_coroutine::coroutine_handle<promise_type>::from_promise(*this);
+		return n_coroutine::coroutine_handle<promise_type>::from_promise(*self());
 	}
 
 	static void* operator new(size_t s) { return allocateFast(int(s)); }
 	static void operator delete(void* p, size_t s) { freeFast(int(s), p); }
 
-	ReturnFutureType get_return_object() noexcept {
-		coroActor.handle = handle();
-		return ReturnFutureType(&coroActor);
+	template <class U>
+	void setReturnValue(U&& value) {
+		self()->actor()->set(std::forward<U>(value));
 	}
 
 	[[nodiscard]] n_coroutine::suspend_never initial_suspend() const noexcept { return {}; }
 
+	void unhandled_exception() {
+		// The exception should always be type Error.
+		try {
+			std::rethrow_exception(std::current_exception());
+		} catch (const Error& error) {
+			self()->actor()->setError(error);
+		} catch (...) {
+			self()->actor()->setError(unknown_error());
+		}
+	}
+
+	void setHandle(n_coroutine::coroutine_handle<> h) { self()->actor()->handle = h; }
+
+	void resume() { self()->actor()->handle.resume(); }
+
+	int8_t& waitState() { return self()->actor()->waitState(); }
+	void setCancelHandler(AwaitCancelHandler*) {}
+	void clearCancelHandler(AwaitCancelHandler*) {}
+
+	template <class U>
+	auto await_transform(const Future<U>& future) {
+		return coro::AwaitableFuture<promise_type, U, false, ReturnsExplicitVoid>{ future, self() };
+	}
+
+	template <class U>
+	auto await_transform(coro::FutureIgnore<U> future) {
+		// Custom adapters compose through await_transform rather than wrapper
+		// coroutines so cancellation and wait-state handling stay identical to
+		// plain Future<T> awaits.
+		return coro::AwaitableFutureIgnore<promise_type, U>{ std::move(future.future), self() };
+	}
+
+	template <class U, class V>
+	auto await_transform(coro::FutureErrorOr<U, V> future) {
+		return coro::AwaitableFutureErrorOr<promise_type, U, V>{ std::move(future.future), self() };
+	}
+
+	template <class U>
+	auto await_transform(const FutureStream<U>& futureStream) {
+		return coro::AwaitableFuture<promise_type, U, true, ReturnsExplicitVoid>{ futureStream, self() };
+	}
+
+	template <class U>
+	auto await_transform(AsyncResult<U>& result) {
+		return coro::AwaitableAsyncResult<promise_type, U>{ std::move(result), self() };
+	}
+
+	template <class U>
+	auto await_transform(AsyncResult<U>&& result) {
+		return coro::AwaitableAsyncResult<promise_type, U>{ std::move(result), self() };
+	}
+
+	template <class U>
+	auto await_transform(const ThreadFutureStream<U>& futureStream) {
+		return coro::ThreadAwaitableFutureStream<promise_type, U, ReturnsExplicitVoid>{ futureStream, self() };
+	}
+};
+
+template <class T, bool IsCancellable, bool ReturnsExplicitVoid>
+struct CoroPromise<T, IsCancellable, ReturnsExplicitVoid, false>
+  : CoroPromiseBase<CoroPromise<T, IsCancellable, ReturnsExplicitVoid, false>, T, ReturnsExplicitVoid> {
+	using promise_type = CoroPromise<T, IsCancellable, ReturnsExplicitVoid, false>;
+	using Base = CoroPromiseBase<promise_type, T, ReturnsExplicitVoid>;
+	using ActorType = coro::CoroActor<T, IsCancellable>;
+	using ReturnFutureType = typename Base::ReturnFutureType;
+
+	ActorType coroActor; // Embedded in coroutine frame — single allocation
+
+	CoroPromise() {}
+
+	ActorType* actor() { return &coroActor; }
+
+	ReturnFutureType get_return_object() noexcept {
+		actor()->handle = this->handle();
+		return ReturnFutureType(actor());
+	}
+
 	auto final_suspend() noexcept {
 		struct FinalAwaitable {
 			ActorType* sav;
@@ -410,39 +1017,166 @@ struct CoroPromise : CoroReturn<T, CoroPromise<T, IsCancellable>> {
 				}
 			}
 		};
-		return FinalAwaitable(&coroActor);
+		return FinalAwaitable(actor());
+	}
+};
+
+template <class T, bool IsCancellable, bool ReturnsExplicitVoid>
+struct CoroPromise<T, IsCancellable, ReturnsExplicitVoid, true>
+  : CoroPromiseBase<CoroPromise<T, IsCancellable, ReturnsExplicitVoid, true>, T, ReturnsExplicitVoid> {
+	using promise_type = CoroPromise<T, IsCancellable, ReturnsExplicitVoid, true>;
+	using Base = CoroPromiseBase<promise_type, T, ReturnsExplicitVoid>;
+	using ActorType = coro::NoThrowOnCancelCoroActor<T>;
+	using ReturnFutureType = typename Base::ReturnFutureType;
+
+	// The normal promise embeds CoroActor in the coroutine frame. NoThrowOnCancel
+	// allocates it separately because cancel() intentionally destroys that frame.
+	ActorType* coroActor = new ActorType();
+
+	CoroPromise() {}
+
+	ActorType* actor() { return coroActor; }
+
+	ReturnFutureType get_return_object() noexcept {
+		actor()->handle = this->handle();
+		return ReturnFutureType(actor());
+	}
+
+	auto final_suspend() noexcept {
+		struct FinalAwaitable {
+			ActorType* sav;
+			explicit FinalAwaitable(ActorType* sav) : sav(sav) {}
+
+			[[nodiscard]] bool await_ready() const noexcept { return false; }
+			void await_resume() const noexcept {}
+
+			bool await_suspend(n_coroutine::coroutine_handle<>) const noexcept {
+				// Normal completion hands the result to the Future and leaves
+				// no frame for a later cancel/drop path to destroy.
+				sav->handle = {};
+				if (sav->isError()) {
+					sav->finishSendErrorAndDelPromiseRef();
+				} else {
+					sav->finishSendAndDelPromiseRef();
+				}
+				return false;
+			}
+		};
+		return FinalAwaitable(actor());
+	}
+
+	void setCancelHandler(AwaitCancelHandler* handler) { actor()->setCancelHandler(handler); }
+	void clearCancelHandler(AwaitCancelHandler* handler) { actor()->clearCancelHandler(handler); }
+};
+
+template <class T, bool IsCancellable, bool ReturnsExplicitVoid, bool NoThrowOnCancel>
+struct AsyncResultPromise
+  : AsyncResultReturn<T,
+                      AsyncResultPromise<T, IsCancellable, ReturnsExplicitVoid, NoThrowOnCancel>,
+                      ReturnsExplicitVoid> {
+	using promise_type = AsyncResultPromise<T, IsCancellable, ReturnsExplicitVoid, NoThrowOnCancel>;
+	using ReturnValue = std::conditional_t<std::is_void_v<T>, Void, T>;
+	using ReturnAsyncResultType = AsyncResult<T>;
+	using State = AsyncResultState<ReturnValue>;
+
+	State* state = new State();
+	n_coroutine::coroutine_handle<> producerHandle;
+	int8_t producerWaitState = ACTOR_WAIT_STATE_NOT_WAITING;
+
+	AsyncResultPromise() = default;
+
+	n_coroutine::coroutine_handle<promise_type> handle() {
+		return n_coroutine::coroutine_handle<promise_type>::from_promise(*this);
+	}
+
+	static void* operator new(size_t s) { return allocateFast(int(s)); }
+	static void operator delete(void* p, size_t s) { freeFast(int(s), p); }
+
+	ReturnAsyncResultType get_return_object() noexcept {
+		producerHandle = handle();
+		state->cancellable = IsCancellable;
+		state->noThrowOnCancel = NoThrowOnCancel;
+		state->producerHandle = producerHandle;
+		state->producerWaitState = &producerWaitState;
+		state->addRef();
+		return ReturnAsyncResultType(state);
+	}
+
+	[[nodiscard]] n_coroutine::suspend_never initial_suspend() const noexcept { return {}; }
+
+	auto final_suspend() noexcept {
+		struct FinalAwaitable {
+			State* state;
+
+			[[nodiscard]] bool await_ready() const noexcept { return false; }
+			void await_resume() const noexcept {}
+
+			bool await_suspend(n_coroutine::coroutine_handle<>) const noexcept {
+				state->complete();
+				state->delRef();
+				return false;
+			}
+		};
+		return FinalAwaitable{ state };
 	}
 
 	void unhandled_exception() {
-		// The exception should always be type Error.
 		try {
 			std::rethrow_exception(std::current_exception());
 		} catch (const Error& error) {
-			coroActor.setError(error);
+			state->setError(error);
 		} catch (...) {
-			coroActor.setError(unknown_error());
+			state->setError(unknown_error());
 		}
 	}
 
-	void setHandle(n_coroutine::coroutine_handle<> h) { coroActor.handle = h; }
+	void setHandle(n_coroutine::coroutine_handle<> h) { producerHandle = h; }
 
-	void resume() { coroActor.handle.resume(); }
+	void resume() { producerHandle.resume(); }
 
-	int8_t& waitState() { return coroActor.waitState(); }
+	int8_t& waitState() { return producerWaitState; }
+	void setCancelHandler(AwaitCancelHandler* handler) { state->producerCancelHandler = handler; }
+	void clearCancelHandler(AwaitCancelHandler* handler) {
+		if (state->producerCancelHandler == handler) {
+			state->producerCancelHandler = nullptr;
+		}
+	}
 
 	template <class U>
 	auto await_transform(const Future<U>& future) {
-		return coro::AwaitableFuture<promise_type, U, false>{ future, this };
+		return coro::AwaitableFuture<promise_type, U, false, ReturnsExplicitVoid>{ future, this };
+	}
+
+	template <class U>
+	auto await_transform(coro::FutureIgnore<U> future) {
+		// Mirror CoroPromise support so AsyncResult coroutines can use the same
+		// non-allocating await adapters.
+		return coro::AwaitableFutureIgnore<promise_type, U>{ std::move(future.future), this };
+	}
+
+	template <class U, class V>
+	auto await_transform(coro::FutureErrorOr<U, V> future) {
+		return coro::AwaitableFutureErrorOr<promise_type, U, V>{ std::move(future.future), this };
 	}
 
 	template <class U>
 	auto await_transform(const FutureStream<U>& futureStream) {
-		return coro::AwaitableFuture<promise_type, U, true>{ futureStream, this };
+		return coro::AwaitableFuture<promise_type, U, true, ReturnsExplicitVoid>{ futureStream, this };
+	}
+
+	template <class U>
+	auto await_transform(AsyncResult<U>& result) {
+		return coro::AwaitableAsyncResult<promise_type, U>{ std::move(result), this };
+	}
+
+	template <class U>
+	auto await_transform(AsyncResult<U>&& result) {
+		return coro::AwaitableAsyncResult<promise_type, U>{ std::move(result), this };
 	}
 
 	template <class U>
 	auto await_transform(const ThreadFutureStream<U>& futureStream) {
-		return coro::ThreadAwaitableFutureStream<promise_type, U>{ futureStream, this };
+		return coro::ThreadAwaitableFutureStream<promise_type, U, ReturnsExplicitVoid>{ futureStream, this };
 	}
 };
 
@@ -489,9 +1223,9 @@ struct GeneratorPromise {
 	void return_void() {}
 };
 
-template <class T>
+template <class T, bool ReturnsExplicitVoid = false>
 struct AsyncGeneratorPromise {
-	using promise_type = AsyncGeneratorPromise<T>;
+	using promise_type = AsyncGeneratorPromise<T, ReturnsExplicitVoid>;
 
 	static void* operator new(size_t s) { return allocateFast(int(s)); }
 	static void operator delete(void* p, size_t s) { freeFast(int(s), p); }
@@ -527,15 +1261,27 @@ struct AsyncGeneratorPromise {
 	void setHandle(n_coroutine::coroutine_handle<> h) { mHandle = h; }
 	int8_t& waitState() { return mWaitState; }
 	void resume() { mHandle.resume(); }
+	void setCancelHandler(AwaitCancelHandler*) {}
+	void clearCancelHandler(AwaitCancelHandler*) {}
 
 	template <class U>
 	auto await_transform(const Future<U>& future) {
-		return coro::AwaitableFuture<promise_type, U, false>{ future, this };
+		return coro::AwaitableFuture<promise_type, U, false, ReturnsExplicitVoid>{ future, this };
 	}
 
 	template <class U>
 	auto await_transform(const FutureStream<U>& futureStream) {
-		return coro::AwaitableFuture<promise_type, U, true>{ futureStream, this };
+		return coro::AwaitableFuture<promise_type, U, true, ReturnsExplicitVoid>{ futureStream, this };
+	}
+
+	template <class U>
+	auto await_transform(AsyncResult<U>& result) {
+		return coro::AwaitableAsyncResult<promise_type, U>{ std::move(result), this };
+	}
+
+	template <class U>
+	auto await_transform(AsyncResult<U>&& result) {
+		return coro::AwaitableAsyncResult<promise_type, U>{ std::move(result), this };
 	}
 
 	n_coroutine::coroutine_handle<> mHandle;
@@ -543,6 +1289,40 @@ struct AsyncGeneratorPromise {
 	int8_t mWaitState = 0;
 };
 
+// Awaiter used by plain C++ coroutines that directly `co_await AsyncResult<T>`
+// without an actor-style promise.
+template <class ValueType>
+struct AsyncResultAwaiter {
+	AsyncResult<ValueType> result;
+
+	[[nodiscard]] bool await_ready() const {
+		ASSERT(result.resultState);
+		return result.resultState->isReady();
+	}
+
+	void await_suspend(n_coroutine::coroutine_handle<> h) {
+		ASSERT(result.resultState);
+		result.resultState->registerContinuation(h);
+	}
+
+	void await_resume()
+	    requires(std::is_void_v<ValueType>)
+	{
+		if (result.resultState->isError()) {
+			throw result.resultState->getError();
+		}
+	}
+
+	ValueType await_resume()
+	    requires(!std::is_void_v<ValueType>)
+	{
+		if (result.resultState->isError()) {
+			throw result.resultState->getError();
+		}
+		return result.resultState->take();
+	}
+};
+
 template <class... Args>
 struct HasUncancellable;
 
@@ -564,6 +1344,48 @@ struct HasUncancellable<Uncancellable, Rest...> {
 template <class... Args>
 inline constexpr bool hasUncancellable = HasUncancellable<Args...>::value;
 
+template <class... Args>
+struct HasExplicitVoid;
+
+template <>
+struct HasExplicitVoid<> {
+	static constexpr bool value = false;
+};
+
+template <class First, class... Args>
+struct HasExplicitVoid<First, Args...> {
+	static constexpr bool value = HasExplicitVoid<Args...>::value;
+};
+
+template <class... Rest>
+struct HasExplicitVoid<ExplicitVoid, Rest...> {
+	static constexpr bool value = true;
+};
+
+template <class... Args>
+inline constexpr bool hasExplicitVoid = HasExplicitVoid<Args...>::value;
+
+template <class... Args>
+struct HasNoThrowOnCancel;
+
+template <>
+struct HasNoThrowOnCancel<> {
+	static constexpr bool value = false;
+};
+
+template <class First, class... Args>
+struct HasNoThrowOnCancel<First, Args...> {
+	static constexpr bool value = HasNoThrowOnCancel<Args...>::value;
+};
+
+template <class... Rest>
+struct HasNoThrowOnCancel<NoThrowOnCancel, Rest...> {
+	static constexpr bool value = true;
+};
+
+template <class... Args>
+inline constexpr bool hasNoThrowOnCancel = HasNoThrowOnCancel<Args...>::value;
+
 } // namespace coro
 
 #endif // FLOW_COROUTINESIMPL_H
diff --git a/flow/include/flow/Deque.h b/flow/include/flow/Deque.h
index 034ec150ebc..50bb0ba9ee7 100644
--- a/flow/include/flow/Deque.h
+++ b/flow/include/flow/Deque.h
@@ -41,7 +41,7 @@ class Deque {
 	Deque() : arr(0), begin(0), end(0), mask(-1) {}
 
 	// TODO: iterator construction, other constructors
-	Deque(Deque const& r) : arr(nullptr), begin(0), end(r.size()), mask(r.mask) {
+	explicit(false) Deque(Deque const& r) : arr(nullptr), begin(0), end(r.size()), mask(r.mask) {
 		if (r.capacity() > 0) {
 			arr = (T*)aligned_alloc(std::max(__alignof(T), sizeof(void*)), capacity() * sizeof(T));
 			if (arr == nullptr) {
@@ -85,7 +85,7 @@ class Deque {
 		}
 	}
 
-	Deque(Deque&& r) noexcept : arr(r.arr), begin(r.begin), end(r.end), mask(r.mask) {
+	explicit(false) Deque(Deque&& r) noexcept : arr(r.arr), begin(r.begin), end(r.end), mask(r.mask) {
 		r.arr = nullptr;
 		r.begin = r.end = 0;
 		r.mask = -1;
diff --git a/flow/include/flow/DeterministicRandom.h b/flow/include/flow/DeterministicRandom.h
index b38533a0381..298b2365e63 100644
--- a/flow/include/flow/DeterministicRandom.h
+++ b/flow/include/flow/DeterministicRandom.h
@@ -43,19 +43,15 @@ class SWIFT_CXX_REF_DETERMINISTICRANDOM DeterministicRandom final : public IRand
 private:
 	// Use boost::random::mt19937 to get consistent output across
 	// different compilers and therefore across different C++ standard
-	// library implementations. In other words, don't rely on the
-	// standard library for this.
-	//
-	// This is not expected to affect performance.  See e.g.
-	// https://chatgpt.com/share/68800ee9-3270-800b-aa84-4567167f02ab
-	boost::random::mt19937 rng;
+	// library implementations.
+	boost::random::mt19937_64 rng;
 	uint64_t next;
 	bool useRandLog;
 
 	uint64_t gen64();
 
 public:
-	DeterministicRandom(uint32_t seed, bool useRandLog = false);
+	DeterministicRandom(uint64_t seed, bool useRandLog = false);
 	double random01() override;
 	int randomInt(int min, int maxPlusOne) override;
 	int64_t randomInt64(int64_t min, int64_t maxPlusOne) override;
@@ -68,7 +64,7 @@ class SWIFT_CXX_REF_DETERMINISTICRANDOM DeterministicRandom final : public IRand
 	void randomBytes(uint8_t* buf, int length) override;
 	bool truePercent(const int percent) override;
 	uint64_t peek() const override;
-	void resetSeed(uint32_t seed) override; // Reset the random number generator with a new seed
+	void resetSeed(uint64_t seed) override; // Reset the random number generator with a new seed
 	void addref() override;
 	void delref() override;
 };
diff --git a/flow/include/flow/Error.h b/flow/include/flow/Error.h
index 1ef570b0c5f..444aabc001e 100644
--- a/flow/include/flow/Error.h
+++ b/flow/include/flow/Error.h
@@ -96,7 +96,7 @@ class AttributeNotFoundError : public Error {
 	std::string missingAttribute;
 
 public:
-	AttributeNotFoundError(const std::string&);
+	explicit AttributeNotFoundError(const std::string&);
 
 	const std::string& getMissingAttribute() const;
 };
@@ -150,7 +150,7 @@ extern bool isAssertDisabled(int line);
 		throw internal_error_impl("unreachable", __FILE__, __LINE__);                                                  \
 	}
 
-// TODO: magic so this works even if const-ness doesn't not match.
+// TODO: magic so this works even if const-ness doesn't match.
 template <typename T, typename U>
 void assert_impl(char const* a_nm,
                  T const& a,
diff --git a/flow/include/flow/FastRef.h b/flow/include/flow/FastRef.h
index 6c3c7ef28e7..1bc14d1cdf3 100644
--- a/flow/include/flow/FastRef.h
+++ b/flow/include/flow/FastRef.h
@@ -56,7 +56,7 @@ class ThreadSafeReferenceCounted {
 	int32_t debugGetReferenceCount() const { return referenceCount.load(); }
 
 private:
-	ThreadSafeReferenceCounted(const ThreadSafeReferenceCounted&) /* = delete*/;
+	explicit(false) ThreadSafeReferenceCounted(const ThreadSafeReferenceCounted&) /* = delete*/;
 	void operator=(const ThreadSafeReferenceCounted&) /* = delete*/;
 	mutable std::atomic<int32_t> referenceCount;
 };
@@ -76,7 +76,7 @@ class ThreadUnsafeReferenceCounted {
 	bool isSoleOwner() const { return referenceCount == 1; }
 
 private:
-	ThreadUnsafeReferenceCounted(const ThreadUnsafeReferenceCounted&) /* = delete*/;
+	explicit(false) ThreadUnsafeReferenceCounted(const ThreadUnsafeReferenceCounted&) /* = delete*/;
 	void operator=(const ThreadUnsafeReferenceCounted&) /* = delete*/;
 	mutable int32_t referenceCount;
 };
@@ -107,19 +107,19 @@ class Reference {
 		return Reference(ptr);
 	}
 
-	Reference(const Reference& r) : ptr(r.getPtr()) {
+	explicit(false) Reference(const Reference& r) : ptr(r.getPtr()) {
 		if (ptr)
 			addref(ptr);
 	}
-	Reference(Reference&& r) noexcept : ptr(r.getPtr()) { r.ptr = nullptr; }
+	explicit(false) Reference(Reference&& r) noexcept : ptr(r.getPtr()) { r.ptr = nullptr; }
 
 	template <class Q>
-	Reference(const Reference<Q>& r) : ptr(r.getPtr()) {
+	explicit(false) Reference(const Reference<Q>& r) : ptr(r.getPtr()) {
 		if (ptr)
 			addref(ptr);
 	}
 	template <class Q>
-	Reference(Reference<Q>&& r) : ptr(r.getPtr()) {
+	explicit(false) Reference(Reference<Q>&& r) : ptr(r.getPtr()) {
 		r.setPtrUnsafe(nullptr);
 	}
 
diff --git a/flow/include/flow/FlowThread.h b/flow/include/flow/FlowThread.h
index 909bf6cf33f..bb52c150c45 100644
--- a/flow/include/flow/FlowThread.h
+++ b/flow/include/flow/FlowThread.h
@@ -196,8 +196,8 @@ template <class T>
 class ThreadFutureStream {
 public:
 	ThreadFutureStream() : queue(nullptr) {}
-	ThreadFutureStream(const ThreadFutureStream& rhs) : queue(rhs.queue) { queue->addFutureRef(); }
-	ThreadFutureStream(ThreadFutureStream&& rhs) noexcept : queue(rhs.queue) { rhs.queue = 0; }
+	explicit(false) ThreadFutureStream(const ThreadFutureStream& rhs) : queue(rhs.queue) { queue->addFutureRef(); }
+	explicit(false) ThreadFutureStream(ThreadFutureStream&& rhs) noexcept : queue(rhs.queue) { rhs.queue = 0; }
 	explicit ThreadFutureStream(ThreadNotifiedQueue<T>* queue) : queue(queue) {}
 	~ThreadFutureStream() {
 		if (queue)
@@ -256,7 +256,9 @@ class ThreadReturnPromiseStream {
 public:
 	ThreadReturnPromiseStream() : queue(new ThreadNotifiedQueue<T>(0, 1)) {}
 	ThreadReturnPromiseStream(const ThreadReturnPromiseStream& rhs) = delete;
-	ThreadReturnPromiseStream(ThreadReturnPromiseStream&& rhs) noexcept : queue(rhs.queue) { rhs.queue = 0; }
+	explicit(false) ThreadReturnPromiseStream(ThreadReturnPromiseStream&& rhs) noexcept : queue(rhs.queue) {
+		rhs.queue = 0;
+	}
 	~ThreadReturnPromiseStream() {
 		if (queue)
 			queue->delPromiseRef();
diff --git a/flow/include/flow/Histogram.h b/flow/include/flow/Histogram.h
index c0f13312b46..5194f965dac 100644
--- a/flow/include/flow/Histogram.h
+++ b/flow/include/flow/Histogram.h
@@ -61,12 +61,12 @@ class Histogram final : public ReferenceCounted<Histogram> {
 	enum class Unit { milliseconds = 0, bytes, bytes_per_second, percentageLinear, countLinear, MAXHISTOGRAMUNIT };
 	static const char* const UnitToStringMapper[];
 
-	Histogram(Reference<HistogramRegistry> regis,
-	          std::string const& group = "",
-	          std::string const& op = "",
-	          Unit unit = Unit::MAXHISTOGRAMUNIT,
-	          uint32_t lower = 0,
-	          uint32_t upper = UINT32_MAX)
+	explicit Histogram(Reference<HistogramRegistry> regis,
+	                   std::string const& group = "",
+	                   std::string const& op = "",
+	                   Unit unit = Unit::MAXHISTOGRAMUNIT,
+	                   uint32_t lower = 0,
+	                   uint32_t upper = UINT32_MAX)
 	  : group(group), op(op), unit(unit), registry(regis), lowerBound(lower), upperBound(upper) {
 
 		ASSERT(unit <= Unit::MAXHISTOGRAMUNIT);
diff --git a/flow/include/flow/IConnection.h b/flow/include/flow/IConnection.h
index e7e85f97c6b..ea31309e1f8 100644
--- a/flow/include/flow/IConnection.h
+++ b/flow/include/flow/IConnection.h
@@ -55,8 +55,8 @@ class IConnection {
 	// returns when write() can write at least one byte (or may throw an error if the connection dies)
 	virtual Future<Void> onWritable() = 0;
 
-	// Precondition: read() has been called and last returned 0
 	// returns when read() can read at least one byte (or may throw an error if the connection dies)
+	// Typically called after read() has returned 0, but may also be called on a newly accepted connection.
 	virtual Future<Void> onReadable() = 0;
 
 	// Reads as many bytes as possible from the read buffer into [begin,end) and returns the number of bytes read (might
@@ -106,21 +106,28 @@ class IListener {
 class DNSCache {
 public:
 	DNSCache() = default;
-	explicit DNSCache(const std::map<std::string, std::vector<NetworkAddress>>& dnsCache)
-	  : hostnameToAddresses(dnsCache) {}
+	explicit DNSCache(const std::map<std::string, std::vector<NetworkAddress>>& dnsCache);
 
 	Optional<std::vector<NetworkAddress>> find(const std::string& host, const std::string& service);
 	void add(const std::string& host, const std::string& service, const std::vector<NetworkAddress>& addresses);
+	// Replace a cached entry's addresses without bumping its last-access time.
+	void update(const std::string& host, const std::string& service, const std::vector<NetworkAddress>& addresses);
 	void remove(const std::string& host, const std::string& service);
 	void clear();
+	std::vector<std::string> getKeys() const;
+	Optional<double> getLastAccess(const std::string& host, const std::string& service) const;
 
-	// Convert hostnameToAddresses to string. The format is:
+	// Convert the cache to a string. The format is:
 	// hostname1,host1Address1,host1Address2;hostname2,host2Address1,host2Address2...
 	std::string toString();
 	static DNSCache parseFromString(const std::string& s);
 
 private:
-	std::map<std::string, std::vector<NetworkAddress>> hostnameToAddresses;
+	struct Entry {
+		std::vector<NetworkAddress> addresses;
+		double lastAccess = 0.0;
+	};
+	std::map<std::string, Entry> entries;
 };
 
 class IUDPSocket;
diff --git a/flow/include/flow/IPAddress.h b/flow/include/flow/IPAddress.h
index f1ac34574b4..239ae936eac 100644
--- a/flow/include/flow/IPAddress.h
+++ b/flow/include/flow/IPAddress.h
@@ -29,7 +29,7 @@
 
 struct IPAddress {
 	// It should be the same to boost::asio::ip::address_v6::bytes_type. This is enforced by static_assert in
-	// Platform.actor.cpp.
+	// Platform.cpp.
 	using IPAddressStore = std::array<uint8_t, 16>;
 
 public:
diff --git a/flow/include/flow/IRandom.h b/flow/include/flow/IRandom.h
index 54536912dda..9d7eaf637ff 100644
--- a/flow/include/flow/IRandom.h
+++ b/flow/include/flow/IRandom.h
@@ -39,7 +39,8 @@
 // This is as good a place as any, I guess.
 
 template <typename T>
-typename std::enable_if<std::is_integral<T>::value, int>::type compare(T l, T r) {
+    requires(std::is_integral_v<T>)
+int compare(T l, T r) {
 	const int gt = l > r;
 	const int lt = l < r;
 	return gt - lt;
@@ -49,7 +50,8 @@ typename std::enable_if<std::is_integral<T>::value, int>::type compare(T l, T r)
 }
 
 template <typename T, typename U>
-typename std::enable_if<!std::is_integral<T>::value, int>::type compare(T const& l, U const& r) {
+    requires(!std::is_integral_v<T>)
+int compare(T const& l, U const& r) {
 	return l.compare(r);
 }
 
@@ -164,7 +166,7 @@ class IRandom {
 	                                   // Deterministic (and idempotent) for a deterministic generator.
 
 	// Reset the random number generator with a new seed (only supported by deterministic generators)
-	virtual void resetSeed(uint32_t seed) {}
+	virtual void resetSeed(uint64_t seed) {}
 
 	virtual void addref() = 0;
 	virtual void delref() = 0;
@@ -209,7 +211,7 @@ class IRandom {
 extern FILE* randLog;
 
 // Sets the seed for the deterministic random number generator on the current thread
-void setThreadLocalDeterministicRandomSeed(uint32_t seed);
+void setThreadLocalDeterministicRandomSeed(uint64_t seed);
 
 // Returns the random number generator that can be seeded. This generator should only
 // be used in contexts where the choice to call it is deterministic.
diff --git a/flow/include/flow/IThreadPool.h b/flow/include/flow/IThreadPool.h
index 019684a2eb4..44f747722e4 100644
--- a/flow/include/flow/IThreadPool.h
+++ b/flow/include/flow/IThreadPool.h
@@ -85,7 +85,7 @@ class ThreadReturnPromise : NonCopyable {
 public:
 	ThreadReturnPromise() {}
 	ThreadReturnPromise(const ThreadReturnPromise& p) = delete;
-	ThreadReturnPromise(ThreadReturnPromise&& other) : promise(std::move(other.promise)) {}
+	explicit(false) ThreadReturnPromise(ThreadReturnPromise&& other) : promise(std::move(other.promise)) {}
 
 	~ThreadReturnPromise() {
 		if (promise.isValid())
diff --git a/flow/include/flow/IndexedSet.actor.h b/flow/include/flow/IndexedSet.actor.h
deleted file mode 100644
index 147786303cc..00000000000
--- a/flow/include/flow/IndexedSet.actor.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * IndexedSet.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_INDEXEDSET_ACTOR_G_H)
-#define FLOW_INDEXEDSET_ACTOR_G_H
-#include "flow/IndexedSet.actor.g.h"
-#elif !defined(FLOW_INDEXEDSET_ACTOR_H)
-#define FLOW_INDEXEDSET_ACTOR_H
-
-#include "flow/flow.h"
-#include "flow/Platform.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-
-ACTOR template <class Node>
-[[flow_allow_discard]] Future<Void> ISFreeNodes(std::vector<Node*> toFree, bool synchronous) {
-	// Frees the forest of nodes in the 'toFree' vector.
-	// If 'synchronous' is true, then there can be no waits.
-
-	state int eraseCount = 0;
-
-	// Freeing many items from a large tree is bound by the memory latency to
-	// fetch each node from main memory.  This code does a largely depth first
-	// traversal of the forest to be destroyed (using a stack) but prefetches
-	// each node and puts it on a short queue before actually processing it, so
-	// that several memory transactions can be outstanding simultaneously.
-	state Deque<Node*> prefetchQueue;
-	while (!prefetchQueue.empty() || !toFree.empty()) {
-
-		while (prefetchQueue.size() < 10 && !toFree.empty()) {
-			_mm_prefetch((const char*)toFree.back(), _MM_HINT_T0);
-			prefetchQueue.push_back(toFree.back());
-			toFree.pop_back();
-		}
-
-		auto n = prefetchQueue.front();
-		prefetchQueue.pop_front();
-
-		if (n->child[0])
-			toFree.push_back(n->child[0]);
-		if (n->child[1])
-			toFree.push_back(n->child[1]);
-		n->child[0] = n->child[1] = 0;
-		delete n;
-		++eraseCount;
-
-		if (!synchronous && eraseCount % 1000 == 0)
-			wait(yield());
-	}
-
-	return Void();
-}
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/flow/include/flow/IndexedSet.h b/flow/include/flow/IndexedSet.h
index ce19336f122..c98141caea5 100644
--- a/flow/include/flow/IndexedSet.h
+++ b/flow/include/flow/IndexedSet.h
@@ -151,7 +151,7 @@ struct IndexedSet {
 
 	IndexedSet() : root(nullptr) {};
 	~IndexedSet() { delete root; }
-	IndexedSet(IndexedSet&& r) noexcept : root(r.root) { r.root = nullptr; }
+	explicit(false) IndexedSet(IndexedSet&& r) noexcept : root(r.root) { r.root = nullptr; }
 	IndexedSet& operator=(IndexedSet&& r) noexcept {
 		delete root;
 		root = r.root;
@@ -311,7 +311,7 @@ struct IndexedSet {
 
 private:
 	// Copy operations unimplemented.  SOMEDAY: Implement and make public.
-	IndexedSet(const IndexedSet&);
+	explicit(false) IndexedSet(const IndexedSet&);
 	IndexedSet& operator=(const IndexedSet&);
 
 	Node* root;
@@ -358,7 +358,7 @@ struct IndexedSet {
 class NoMetric {
 public:
 	NoMetric() {}
-	NoMetric(int) {} // NoMetric(1)
+	explicit(false) NoMetric(int) {} // NoMetric(1)
 	NoMetric operator+(NoMetric const&) const { return NoMetric(); }
 	NoMetric operator-(NoMetric const&) const { return NoMetric(); }
 	bool operator<(NoMetric const&) const { return false; }
@@ -376,9 +376,9 @@ class MapPair {
 		key = rhs.key;
 		value = rhs.value;
 	}
-	MapPair(MapPair const& rhs) : key(rhs.key), value(rhs.value) {}
+	explicit(false) MapPair(MapPair const& rhs) : key(rhs.key), value(rhs.value) {}
 
-	MapPair(MapPair&& r) noexcept : key(std::move(r.key)), value(std::move(r.value)) {}
+	explicit(false) MapPair(MapPair&& r) noexcept : key(std::move(r.key)), value(std::move(r.value)) {}
 	void operator=(MapPair&& r) noexcept {
 		key = std::move(r.key);
 		value = std::move(r.value);
@@ -531,13 +531,13 @@ class Map {
 
 	static int getElementBytes() { return IndexedSet<Pair, Metric>::getElementBytes(); }
 
-	Map(Map&& r) noexcept : set(std::move(r.set)) {}
+	explicit(false) Map(Map&& r) noexcept : set(std::move(r.set)) {}
 	void operator=(Map&& r) noexcept { set = std::move(r.set); }
 
 	Future<Void> clearAsync();
 
 private:
-	Map(Map<Key, Value, Pair> const&); // unimplemented
+	explicit(false) Map(Map<Key, Value, Pair> const&); // unimplemented
 	void operator=(Map<Key, Value, Pair> const&); // unimplemented
 
 	IndexedSet<Pair, Metric> set;
@@ -875,12 +875,13 @@ typename IndexedSet<T, Metric>::iterator IndexedSet<T, Metric>::insert(T_&& data
 		t = nextT;
 	}
 
+	Metric metricDelta = metric;
 	Node* newNode = new Node(std::forward<T_>(data), std::forward<Metric_>(metric), t);
 	t->child[d] = newNode;
 
 	while (true) {
 		t->balance += d ? 1 : -1;
-		t->total = t->total + metric;
+		t->total = t->total + metricDelta;
 		if (t->balance == 0)
 			break;
 		if (t->balance != 1 && t->balance != -1) {
@@ -909,7 +910,7 @@ typename IndexedSet<T, Metric>::iterator IndexedSet<T, Metric>::insert(T_&& data
 		t = t->parent;
 		if (!t)
 			break;
-		t->total = t->total + metric;
+		t->total = t->total + metricDelta;
 	}
 
 	return iterator{ newNode };
@@ -1383,7 +1384,60 @@ Metric IndexedSet<T, Metric>::sumTo(typename IndexedSet<T, Metric>::const_iterat
 }
 
 #include "flow/flow.h"
-#include "flow/IndexedSet.actor.h"
+
+template <class Node>
+bool ISFreeNodeImpl(std::vector<Node*>& toFree, Deque<Node*>& prefetchQueue) {
+	// Freeing many items from a large tree is bound by the memory latency to
+	// fetch each node from main memory.  This code does a largely depth first
+	// traversal of the forest to be destroyed (using a stack) but prefetches
+	// each node and puts it on a short queue before actually processing it, so
+	// that several memory transactions can be outstanding simultaneously.
+	if (prefetchQueue.empty() && toFree.empty()) {
+		return false;
+	}
+
+	while (prefetchQueue.size() < 10 && !toFree.empty()) {
+		_mm_prefetch((const char*)toFree.back(), _MM_HINT_T0);
+		prefetchQueue.push_back(toFree.back());
+		toFree.pop_back();
+	}
+
+	auto n = prefetchQueue.front();
+	prefetchQueue.pop_front();
+
+	if (n->child[0])
+		toFree.push_back(n->child[0]);
+	if (n->child[1])
+		toFree.push_back(n->child[1]);
+	n->child[0] = n->child[1] = 0;
+	delete n;
+
+	return true;
+}
+
+template <class Node>
+void ISFreeNodesSync(std::vector<Node*> toFree) {
+	// Frees the forest of nodes in the 'toFree' vector without waiting.
+
+	Deque<Node*> prefetchQueue;
+	while (ISFreeNodeImpl(toFree, prefetchQueue)) {
+	}
+}
+
+template <class Node>
+Future<Void> ISFreeNodes(std::vector<Node*> toFree) {
+	// Frees the forest of nodes in the 'toFree' vector, yielding periodically.
+
+	int eraseCount = 0;
+	Deque<Node*> prefetchQueue;
+	while (ISFreeNodeImpl(toFree, prefetchQueue)) {
+		++eraseCount;
+
+		if (eraseCount % 1000 == 0) {
+			co_await yield();
+		}
+	}
+}
 
 template <class T, class Metric>
 void IndexedSet<T, Metric>::erase(typename IndexedSet<T, Metric>::iterator begin,
@@ -1391,7 +1445,7 @@ void IndexedSet<T, Metric>::erase(typename IndexedSet<T, Metric>::iterator begin
 	std::vector<IndexedSet<T, Metric>::Node*> toFree;
 	erase(begin, end, toFree);
 
-	ISFreeNodes(toFree, true);
+	ISFreeNodesSync(toFree);
 }
 
 template <class T, class Metric>
@@ -1406,7 +1460,7 @@ Future<Void> IndexedSet<T, Metric>::eraseAsync(typename IndexedSet<T, Metric>::i
 	std::vector<IndexedSet<T, Metric>::Node*> toFree;
 	erase(begin, end, toFree);
 
-	return uncancellable(ISFreeNodes(toFree, false));
+	return uncancellable(ISFreeNodes(toFree));
 }
 
 template <class Key, class Value, class Pair, class Metric>
diff --git a/flow/include/flow/Knobs.h b/flow/include/flow/Knobs.h
index ab7fa3d2ed8..52165d133f0 100644
--- a/flow/include/flow/Knobs.h
+++ b/flow/include/flow/Knobs.h
@@ -104,6 +104,8 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 	double HOSTNAME_RECONNECT_INIT_INTERVAL;
 	double HOSTNAME_RECONNECT_MAX_INTERVAL;
 	bool ENABLE_COORDINATOR_DNS_CACHE;
+	double COORDINATOR_DNS_CACHE_REFRESH_INTERVAL;
+	double COORDINATOR_DNS_CACHE_TTL;
 	double CACHE_REFRESH_INTERVAL_WHEN_ALL_ALTERNATIVES_FAILED;
 
 	double DELAY_JITTER_OFFSET;
@@ -212,7 +214,6 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 	int64_t SIM_PAGE_CACHE_64K;
 	int64_t BUGGIFY_SIM_PAGE_CACHE_4K;
 	int64_t BUGGIFY_SIM_PAGE_CACHE_64K;
-	int64_t BLOB_WORKER_PAGE_CACHE;
 	std::string CACHE_EVICTION_POLICY; // for now, "random", "lru", are supported
 	int MAX_EVICT_ATTEMPTS;
 	double PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION;
@@ -225,10 +226,6 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 	int EIO_MAX_PARALLELISM;
 	int EIO_USE_ODIRECT;
 
-	// AsyncFileEncrypted
-	int ENCRYPTION_BLOCK_SIZE;
-	int MAX_DECRYPTED_BLOCKS;
-
 	// AsyncFileKAIO
 	int MAX_OUTSTANDING;
 	int MIN_SUBMIT;
diff --git a/flow/include/flow/MkCert.h b/flow/include/flow/MkCert.h
index 0ed777ad3ce..fe997971a1a 100644
--- a/flow/include/flow/MkCert.h
+++ b/flow/include/flow/MkCert.h
@@ -64,7 +64,7 @@ struct CertKind {
 	CertKind() noexcept = default;
 
 	template <class Kind>
-	CertKind(Kind kind) noexcept : value(std::in_place_type<Kind>, kind) {}
+	explicit CertKind(Kind kind) noexcept : value(std::in_place_type<Kind>, kind) {}
 
 	template <class Kind>
 	bool is() const noexcept {
diff --git a/flow/include/flow/OTELMetrics.h b/flow/include/flow/OTELMetrics.h
index 4bc534656a6..5b136ea578c 100644
--- a/flow/include/flow/OTELMetrics.h
+++ b/flow/include/flow/OTELMetrics.h
@@ -73,9 +73,9 @@ class NumberDataPoint {
 	DataPointFlags flags; // 1 byte in msgpack
 	// If we take the sum of above, we get 51 bytes
 	static const uint32_t MsgpackBytes = 51;
-	NumberDataPoint(int64_t v) : recordTime{ now() }, val{ v }, flags{ DataPointFlags::FLAG_NONE } {}
+	explicit NumberDataPoint(int64_t v) : recordTime{ now() }, val{ v }, flags{ DataPointFlags::FLAG_NONE } {}
 
-	NumberDataPoint(double v) : recordTime{ now() }, val{ v }, flags{ DataPointFlags::FLAG_NONE } {}
+	explicit NumberDataPoint(double v) : recordTime{ now() }, val{ v }, flags{ DataPointFlags::FLAG_NONE } {}
 
 	NumberDataPoint& addAttribute(const std::string& key, const std::string& value) {
 		attributes.emplace_back(Attribute(key, value));
@@ -92,7 +92,8 @@ class OTELSum {
 	AggregationTemporality aggregation;
 	bool isMonotonic;
 	OTELSum() : aggregation{ AGGREGATION_TEMPORALITY_CUMULATIVE }, isMonotonic{ true } {}
-	OTELSum(const std::string& n) : name{ n }, aggregation{ AGGREGATION_TEMPORALITY_CUMULATIVE }, isMonotonic{ true } {}
+	explicit OTELSum(const std::string& n)
+	  : name{ n }, aggregation{ AGGREGATION_TEMPORALITY_CUMULATIVE }, isMonotonic{ true } {}
 	OTELSum(const std::string& n, int64_t v)
 	  : name{ n }, aggregation{ AGGREGATION_TEMPORALITY_CUMULATIVE }, isMonotonic{ true } {
 		points.emplace_back(v);
@@ -113,7 +114,7 @@ class OTELGauge {
 	std::string name;
 	std::vector<NumberDataPoint> points;
 	OTELGauge() {}
-	OTELGauge(const std::string& n) : name{ n } {}
+	explicit OTELGauge(const std::string& n) : name{ n } {}
 	OTELGauge(const std::string& n, double v) : name{ n } { points.emplace_back(v); }
 };
 
diff --git a/flow/include/flow/ObjectSerializer.h b/flow/include/flow/ObjectSerializer.h
index 183b5a3cef8..641fcc451ea 100644
--- a/flow/include/flow/ObjectSerializer.h
+++ b/flow/include/flow/ObjectSerializer.h
@@ -32,7 +32,7 @@ template <class Ar>
 struct LoadContext {
 	Ar* ar;
 
-	LoadContext(Ar* ar) : ar(ar) {}
+	explicit LoadContext(Ar* ar) : ar(ar) {}
 
 	Arena& arena() { return ar->arena(); }
 
@@ -307,7 +307,7 @@ namespace detail {
 
 template <class T, class Context>
 struct LoadSaveHelper<Standalone<T>, Context> : Context {
-	LoadSaveHelper(const Context& context) : Context(context), helper(context) {}
+	explicit LoadSaveHelper(const Context& context) : Context(context), helper(context) {}
 
 	void load(Standalone<T>& member, const uint8_t* current) {
 		helper.load(member.contents(), current);
diff --git a/flow/include/flow/ObjectSerializerTraits.h b/flow/include/flow/ObjectSerializerTraits.h
index 523dcc7934f..ab0c0825812 100644
--- a/flow/include/flow/ObjectSerializerTraits.h
+++ b/flow/include/flow/ObjectSerializerTraits.h
@@ -32,7 +32,8 @@ template <class T, typename = void>
 struct is_fb_function_t : std::false_type {};
 
 template <class T>
-struct is_fb_function_t<T, typename std::enable_if<T::is_fb_visitor>::type> : std::true_type {};
+    requires(T::is_fb_visitor)
+struct is_fb_function_t<T> : std::true_type {};
 
 template <class T>
 constexpr bool is_fb_function = is_fb_function_t<T>::value;
@@ -60,8 +61,8 @@ template <class T, typename = void>
 struct fb_must_appear_last_t : std::false_type {};
 
 template <class T>
-struct fb_must_appear_last_t<T, typename std::enable_if<T::fb_must_appear_last>::type>
-  : std::conditional_t<T::fb_must_appear_last, std::true_type, std::false_type> {};
+    requires(T::fb_must_appear_last)
+struct fb_must_appear_last_t<T> : std::conditional_t<T::fb_must_appear_last, std::true_type, std::false_type> {};
 
 template <class T>
 constexpr bool fb_must_appear_last = fb_must_appear_last_t<T>::value;
@@ -84,7 +85,8 @@ constexpr bool fb_appears_last_property(pack<Items...>) {
 }
 
 template <class Visitor, class... Items>
-typename std::enable_if<is_fb_function<Visitor>, void>::type serializer(Visitor& visitor, Items&... items) {
+    requires(is_fb_function<Visitor>)
+void serializer(Visitor& visitor, Items&... items) {
 	static_assert(fb_appears_last_property(pack<Items...>{}),
 	              "An argument to a serializer call that must appear last (Arena?) does not appear last");
 	visitor(items...);
diff --git a/flow/include/flow/Optional.h b/flow/include/flow/Optional.h
index a207a60ab4b..31519c17a69 100644
--- a/flow/include/flow/Optional.h
+++ b/flow/include/flow/Optional.h
@@ -87,18 +87,16 @@ class
 	template <class F>
 	using MapRet = std::decay_t<std::invoke_result_t<F, T>>;
 
-	template <class F>
-	using EnableIfNotMemberPointer =
-	    std::enable_if_t<!std::is_member_object_pointer_v<F> && !std::is_member_function_pointer_v<F>>;
-
 public:
 	// If the optional is set, calls the function f on the value and returns the value. Otherwise, returns an empty
 	// optional.
-	template <class F, typename = EnableIfNotMemberPointer<F>>
+	template <class F>
+	    requires(!std::is_member_object_pointer_v<F> && !std::is_member_function_pointer_v<F>)
 	Optional<MapRet<F>> map(const F& f) const& {
 		return present() ? Optional<MapRet<F>>(f(get())) : Optional<MapRet<F>>();
 	}
-	template <class F, typename = EnableIfNotMemberPointer<F>>
+	template <class F>
+	    requires(!std::is_member_object_pointer_v<F> && !std::is_member_function_pointer_v<F>)
 	Optional<MapRet<F>> map(const F& f) && {
 		return present() ? Optional<MapRet<F>>(f(std::move(*this).get())) : Optional<MapRet<F>>();
 	}
@@ -107,13 +105,13 @@ class
 	//
 	// v.map(&T::member) is equivalent to v.map([](T v) { return v.member; })
 	template <class R, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, Optional<Rp>> map(
-	    R std::conditional_t<std::is_class_v<T>, T, Void>::* member) const& {
+	    requires(std::is_class_v<T>)
+	Optional<Rp> map(R std::conditional_t<std::is_class_v<T>, T, Void>::* member) const& {
 		return present() ? Optional<Rp>(get().*member) : Optional<Rp>();
 	}
 	template <class R, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, Optional<Rp>> map(
-	    R std::conditional_t<std::is_class_v<T>, T, Void>::* member) && {
+	    requires(std::is_class_v<T>)
+	Optional<Rp> map(R std::conditional_t<std::is_class_v<T>, T, Void>::* member) && {
 		return present() ? Optional<Rp>(std::move(*this).get().*member) : Optional<Rp>();
 	}
 
@@ -122,15 +120,15 @@ class
 	// v.map(&T::memberFunc, arg1, arg2, ...) is equivalent to
 	// v.map([](T v) { return v.memberFunc(arg1, arg2, ...); })
 	template <class R, class... Args, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, Optional<Rp>> map(
-	    R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
-	    Args&&... args) const& {
+	    requires(std::is_class_v<T>)
+	Optional<Rp> map(R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
+	                 Args&&... args) const& {
 		return present() ? Optional<Rp>((get().*memberFunc)(std::forward<Args>(args)...)) : Optional<Rp>();
 	}
 	template <class R, class... Args, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, Optional<Rp>> map(
-	    R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
-	    Args&&... args) && {
+	    requires(std::is_class_v<T>)
+	Optional<Rp> map(R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
+	                 Args&&... args) && {
 		return present() ? Optional<Rp>((std::move(*this).get().*memberFunc)(std::forward<Args>(args)...))
 		                 : Optional<Rp>();
 	}
@@ -141,7 +139,8 @@ class
 	//
 	// v.mapRef(&P::member) is equivalent to Optional<R>(v.get()->member) if v is present and non-null
 	template <class P, class R, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T> || std::is_pointer_v<T>, Optional<Rp>> mapRef(R P::* member) const& {
+	    requires(std::is_class_v<T> || std::is_pointer_v<T>)
+	Optional<Rp> mapRef(R P::* member) const& {
 		if (!present() || !get()) {
 			return Optional<Rp>();
 		}
@@ -157,8 +156,8 @@ class
 	// v.mapRef(&T::memberFunc, arg1, arg2, ...) is equivalent to Optional<R>(v.get()->memberFunc(arg1, arg2, ...)) if v
 	// is present and non-null
 	template <class P, class R, class... Args, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T> || std::is_pointer_v<T>, Optional<Rp>> mapRef(R (P::*memberFunc)(Args...) const,
-	                                                                                  Args&&... args) const& {
+	    requires(std::is_class_v<T> || std::is_pointer_v<T>)
+	Optional<Rp> mapRef(R (P::*memberFunc)(Args...) const, Args&&... args) const& {
 		if (!present() || !get()) {
 			return Optional<Rp>();
 		}
diff --git a/flow/include/flow/OwningResource.h b/flow/include/flow/OwningResource.h
index d5915508fc0..ebe2a202256 100644
--- a/flow/include/flow/OwningResource.h
+++ b/flow/include/flow/OwningResource.h
@@ -55,7 +55,7 @@ template <typename T>
 struct Resource : public ReferenceCounted<Resource<T>>, NonCopyable {
 	T* resource;
 
-	Resource(T* resource_) : resource(resource_) {}
+	explicit Resource(T* resource_) : resource(resource_) {}
 	~Resource() { delete resource; }
 
 	void reset(T* resource_) {
@@ -69,8 +69,8 @@ class ResourceRef {
 protected:
 	Reference<Resource<T>> resourceRef;
 
-	ResourceRef(const Reference<Resource<T>>& ref) : resourceRef(ref) {}
-	ResourceRef(Reference<Resource<T>>&& ref) : resourceRef(std::move(ref)) {}
+	explicit ResourceRef(const Reference<Resource<T>>& ref) : resourceRef(ref) {}
+	explicit ResourceRef(Reference<Resource<T>>&& ref) : resourceRef(std::move(ref)) {}
 	ResourceRef& operator=(const Reference<Resource<T>>& ref) {
 		resourceRef = ref.resourceRef;
 		return *this;
@@ -112,7 +112,7 @@ class ResourceOwningRef : public details::ResourceRef<T>, NonCopyable {
 	friend class ActorWeakSelfRef;
 
 public:
-	ResourceOwningRef(T* resource) : details::ResourceRef<T>(makeReference<details::Resource<T>>(resource)) {}
+	explicit ResourceOwningRef(T* resource) : details::ResourceRef<T>(makeReference<details::Resource<T>>(resource)) {}
 	virtual ~ResourceOwningRef() { details::ResourceRef<T>::resourceRef->reset(nullptr); }
 };
 
@@ -121,8 +121,8 @@ class ResourceOwningRef : public details::ResourceRef<T>, NonCopyable {
 template <typename T>
 class ResourceWeakRef : public details::ResourceRef<T> {
 public:
-	ResourceWeakRef(const ResourceOwningRef<T>& ref) : details::ResourceRef<T>(ref.resourceRef) {}
-	ResourceWeakRef(const ResourceWeakRef& ref) : details::ResourceRef<T>(ref.resourceRef) {}
+	explicit ResourceWeakRef(const ResourceOwningRef<T>& ref) : details::ResourceRef<T>(ref.resourceRef) {}
+	explicit(false) ResourceWeakRef(const ResourceWeakRef& ref) : details::ResourceRef<T>(ref.resourceRef) {}
 };
 
 // A unique reference that takes the ownership of the self object. The self object is widely used as the "global"
@@ -136,9 +136,9 @@ using ActorOwningSelfRef = ResourceOwningRef<T>;
 template <typename T>
 class ActorWeakSelfRef : public ResourceWeakRef<T> {
 public:
-	ActorWeakSelfRef(const ResourceOwningRef<T>& ref) : ResourceWeakRef<T>(ref) {}
-	ActorWeakSelfRef(const ResourceWeakRef<T>& ref) : ResourceWeakRef<T>(ref) {}
-	ActorWeakSelfRef(const ActorWeakSelfRef<T>& ref)
+	explicit ActorWeakSelfRef(const ResourceOwningRef<T>& ref) : ResourceWeakRef<T>(ref) {}
+	explicit ActorWeakSelfRef(const ResourceWeakRef<T>& ref) : ResourceWeakRef<T>(ref) {}
+	explicit(false) ActorWeakSelfRef(const ActorWeakSelfRef<T>& ref)
 	  : ResourceWeakRef<T>(static_cast<const ResourceWeakRef<T>&>(ref)) {}
 
 	// Retrieves the resource as a pointer, throws operation_cancelled if the resource is not available
diff --git a/flow/include/flow/PKey.h b/flow/include/flow/PKey.h
index 09b549eb88c..97dd80c5937 100644
--- a/flow/include/flow/PKey.h
+++ b/flow/include/flow/PKey.h
@@ -52,7 +52,7 @@ class PublicKey {
 	// d2i_PUBKEY
 	PublicKey(DerEncoded, StringRef der);
 
-	PublicKey(const PublicKey& other) noexcept = default;
+	explicit(false) PublicKey(const PublicKey& other) noexcept = default;
 
 	PublicKey& operator=(const PublicKey& other) noexcept = default;
 
@@ -87,7 +87,7 @@ class PrivateKey {
 	// d2i_AutoPrivateKey
 	PrivateKey(DerEncoded, StringRef der);
 
-	PrivateKey(const PrivateKey& other) noexcept = default;
+	explicit(false) PrivateKey(const PrivateKey& other) noexcept = default;
 
 	PrivateKey& operator=(const PrivateKey& other) noexcept = default;
 
diff --git a/flow/include/flow/Platform.actor.h b/flow/include/flow/Platform.actor.h
deleted file mode 100644
index b1ea30cd64d..00000000000
--- a/flow/include/flow/Platform.actor.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Platform.actor.h
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_PLATFORM_ACTOR_G_H)
-#define FLOW_PLATFORM_ACTOR_G_H
-#include "flow/Platform.actor.g.h"
-#elif !defined(FLOW_PLATFORM_ACTOR_H)
-#define FLOW_PLATFORM_ACTOR_H
-
-#include <stdlib.h>
-#include <string>
-#include <vector>
-
-#include "flow/flow.h"
-#include "flow/Platform.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
-//
-
-namespace platform {
-// e.g. extension==".fdb", returns filenames relative to directory
-Future<std::vector<std::string>> listFilesAsync(std::string const& directory, std::string const& extension = "");
-
-// returns directory names relative to directory
-Future<std::vector<std::string>> listDirectoriesAsync(std::string const& directory);
-
-ACTOR Future<Void> findFilesRecursivelyAsync(std::string path, std::vector<std::string>* out);
-} // namespace platform
-
-#include "flow/unactorcompiler.h"
-#endif
diff --git a/flow/include/flow/Platform.h b/flow/include/flow/Platform.h
index 63867039398..e0c2eaa50b3 100644
--- a/flow/include/flow/Platform.h
+++ b/flow/include/flow/Platform.h
@@ -136,6 +136,10 @@
 #include <functional>
 #endif
 
+template <class T>
+class AsyncResult;
+class Void;
+
 // g++ requires that non-dependent names have to be looked up at
 // template definition, which makes circular dependencies a royal
 // pain. (For whatever it's worth, g++ appears to be adhering to spec
@@ -426,6 +430,7 @@ std::vector<std::string> listFiles(std::string const& directory, std::string con
 std::vector<std::string> listDirectories(std::string const& directory);
 
 void findFilesRecursively(std::string const& path, std::vector<std::string>& out);
+AsyncResult<Void> findFilesRecursivelyAsync(std::string path, std::vector<std::string>* out);
 
 // Tag the given file as "temporary", i.e. not really needing commits to disk
 void makeTemporary(const char* filename);
@@ -435,7 +440,8 @@ void setCloseOnExec(int fd);
 // Logs an out of memory error and exits the program
 void outOfMemory();
 
-int getRandomSeed();
+uint64_t getRandomSeed();
+void getRandomBytes(void* buf, size_t len);
 
 bool getEnvironmentVar(const char* name, std::string& value);
 int setEnvironmentVar(const char* name, const char* value, int overwrite);
@@ -470,7 +476,7 @@ int eraseDirectoryRecursive(std::string const& directory);
 struct TmpFile {
 public:
 	TmpFile();
-	TmpFile(const std::string& tempDir);
+	explicit TmpFile(const std::string& tempDir);
 	TmpFile(const std::string& tempDir, std::string const& prefix);
 	~TmpFile();
 	size_t read(uint8_t* buff, size_t len);
diff --git a/flow/include/flow/PriorityMultiLock.actor.h b/flow/include/flow/PriorityMultiLock.h
similarity index 87%
rename from flow/include/flow/PriorityMultiLock.actor.h
rename to flow/include/flow/PriorityMultiLock.h
index b208f0ba82c..3ec0eeedd13 100644
--- a/flow/include/flow/PriorityMultiLock.actor.h
+++ b/flow/include/flow/PriorityMultiLock.h
@@ -1,5 +1,5 @@
 /*
- * PriorityMultiLock.actor.h
+ * PriorityMultiLock.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,17 +20,8 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_PRIORITYMULTILOCK_ACTOR_G_H)
-#define FLOW_PRIORITYMULTILOCK_ACTOR_G_H
-#include "flow/PriorityMultiLock.actor.g.h"
-#elif !defined(PRIORITYMULTILOCK_ACTOR_H)
-#define PRIORITYMULTILOCK_ACTOR_H
-
 #include "flow/flow.h"
 #include <boost/intrusive/list.hpp>
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 #define PRIORITYMULTILOCK_DEBUG 0
 
@@ -252,15 +243,7 @@ class PriorityMultiLock : public ReferenceCounted<PriorityMultiLock> {
 	Promise<Void> brokenOnDestruct;
 	bool killed;
 
-	ACTOR static void handleRelease(Reference<PriorityMultiLock> self, Priority* priority, Future<Void> holder) {
-		pml_debug_printf("%f handleRelease self=%p start\n", now(), self.getPtr());
-		try {
-			wait(holder);
-			pml_debug_printf("%f handleRelease self=%p success\n", now(), self.getPtr());
-		} catch (Error& e) {
-			pml_debug_printf("%f handleRelease self=%p error %s\n", now(), self.getPtr(), e.what());
-		}
-
+	static void releaseRunner(Reference<PriorityMultiLock> self, Priority* priority) {
 		pml_debug_printf("lock release priority %d  %s\n", (int)(priority->priority), self->toString().c_str());
 
 		pml_debug_printf("%f handleRelease self=%p releasing\n", now(), self.getPtr());
@@ -273,6 +256,49 @@ class PriorityMultiLock : public ReferenceCounted<PriorityMultiLock> {
 		}
 	}
 
+	// Keep this as a plain callback instead of a detached coroutine: this path runs once per granted
+	// lock, and the coroutine conversion added enough per-release overhead to regress the microbench.
+	struct ReleaseHandler final : Callback<Void>, FastAllocated<ReleaseHandler> {
+		Reference<PriorityMultiLock> self;
+		Priority* priority;
+
+		ReleaseHandler(Reference<PriorityMultiLock> self, Priority* priority)
+		  : self(std::move(self)), priority(priority) {}
+
+		void fire(Void const&) override {
+			Callback<Void>::remove();
+			pml_debug_printf("%f handleRelease self=%p success\n", now(), self.getPtr());
+			releaseRunner(std::move(self), priority);
+			delete this;
+		}
+
+		void error(Error e) override {
+			Callback<Void>::remove();
+			pml_debug_printf("%f handleRelease self=%p error %s\n", now(), self.getPtr(), e.what());
+			releaseRunner(std::move(self), priority);
+			delete this;
+		}
+	};
+
+	static void handleRelease(Reference<PriorityMultiLock> self, Priority* priority, Future<Void> holder) {
+		pml_debug_printf("%f handleRelease self=%p start\n", now(), self.getPtr());
+
+		// Handle immediately-ready releases inline so the hot path avoids allocating a callback object.
+		if (holder.isReady()) {
+			if (holder.isError()) {
+				pml_debug_printf("%f handleRelease self=%p error %s\n", now(), self.getPtr(), holder.getError().what());
+			} else {
+				holder.get();
+				pml_debug_printf("%f handleRelease self=%p success\n", now(), self.getPtr());
+			}
+
+			releaseRunner(std::move(self), priority);
+			return;
+		}
+
+		holder.addCallbackAndClear(new ReleaseHandler(std::move(self), priority));
+	}
+
 	void addRunner(Lock& lock, Priority* priority) {
 		priority->runners += 1;
 		--available;
@@ -287,18 +313,18 @@ class PriorityMultiLock : public ReferenceCounted<PriorityMultiLock> {
 		return ceil((float)weight / totalPendingWeights * concurrency);
 	}
 
-	ACTOR static Future<Void> runner(PriorityMultiLock* self) {
-		state Future<Void> error = self->brokenOnDestruct.getFuture();
+	static Future<Void> runner(PriorityMultiLock* self) {
+		Future<Void> error = self->brokenOnDestruct.getFuture();
 
 		// Priority to try to run tasks from next
-		state WaitingPrioritiesList::iterator p = self->waitingPriorities.end();
+		WaitingPrioritiesList::iterator p = self->waitingPriorities.end();
 
-		loop {
+		while (true) {
 			pml_debug_printf("runner loop start  priority=%d  %s\n", p->priority, self->toString().c_str());
 
 			// Wait for a runner to release its lock
 			pml_debug_printf("runner loop waitTrigger  priority=%d  %s\n", p->priority, self->toString().c_str());
-			wait(self->wakeRunner.onTrigger());
+			co_await self->wakeRunner.onTrigger();
 			pml_debug_printf("%f runner loop wake  priority=%d  %s\n", now(), p->priority, self->toString().c_str());
 
 			// While there are available slots and there are waiters, launch tasks
@@ -306,7 +332,7 @@ class PriorityMultiLock : public ReferenceCounted<PriorityMultiLock> {
 				pml_debug_printf("  launch loop start  priority=%d  %s\n", p->priority, self->toString().c_str());
 
 				// Find the next priority with waiters and capacity.  There must be at least one.
-				loop {
+				while (true) {
 					if (p == self->waitingPriorities.end()) {
 						p = self->waitingPriorities.begin();
 					}
@@ -357,7 +383,3 @@ class PriorityMultiLock : public ReferenceCounted<PriorityMultiLock> {
 		}
 	}
 };
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/flow/include/flow/ScopeExit.h b/flow/include/flow/ScopeExit.h
index c34fbdb6564..0bae74281cb 100644
--- a/flow/include/flow/ScopeExit.h
+++ b/flow/include/flow/ScopeExit.h
@@ -28,7 +28,7 @@ class ScopeExit {
 	std::decay_t<Func> fn;
 
 public:
-	ScopeExit(Func&& fn) : fn(std::forward<Func>(fn)) {}
+	explicit ScopeExit(Func&& fn) : fn(std::forward<Func>(fn)) {}
 
 	~ScopeExit() { fn(); }
 };
diff --git a/flow/include/flow/SendBufferIterator.h b/flow/include/flow/SendBufferIterator.h
index 9bd57994605..eab25e0c2af 100644
--- a/flow/include/flow/SendBufferIterator.h
+++ b/flow/include/flow/SendBufferIterator.h
@@ -36,7 +36,7 @@ class SendBufferIterator {
 	using pointer = boost::asio::const_buffer*;
 	using reference = boost::asio::const_buffer&;
 
-	SendBufferIterator(SendBuffer const* p = nullptr, int limit = std::numeric_limits<int>::max());
+	explicit SendBufferIterator(SendBuffer const* p = nullptr, int limit = std::numeric_limits<int>::max());
 
 	bool operator==(SendBufferIterator const& r) const { return p == r.p; }
 	bool operator!=(SendBufferIterator const& r) const { return p != r.p; }
diff --git a/flow/include/flow/SimpleCounter.h b/flow/include/flow/SimpleCounter.h
index c20bc93f098..0b2d068eafa 100644
--- a/flow/include/flow/SimpleCounter.h
+++ b/flow/include/flow/SimpleCounter.h
@@ -44,15 +44,10 @@
 // synchronous work in side threads, but is intended to generally be very
 // light weight.  `makeCounter` can be called in constructors of global objects.
 //
-// If you want to use hierarchical metric names (e.g., '/'-separated
-// components), please use ALL LOWER CASE METRIC NAMES AS PER THE EXAMPLE ABOVE.
-// This enables the implementation to smuggle path component
-// separators into the trace output by replacing path separaters like '/' with
-// carefully chosen capital letters.  This obtains compatibility with current FDB
-// "field name" naming conventions.
-//
+// Hierarchical counter names with '/' separaters are encouraged.
 // In the future we might replace '/' with '_' to obtain Prometheus-compatible
-// metric names that don't actually look terrible.
+// metric names that don't actually look terrible. For now software does this
+// conversion on back-end TraceEvent generation.
 //
 // If you don't want to use hierarchical metric names, then your counter
 // names should be ReallyVerboseConcatenatedNamesWithCaps and must be globally
@@ -79,7 +74,7 @@ class SimpleCounter {
 	static_assert(std::is_same_v<T, int64_t> || std::is_same_v<T, double>, "T must be int64_t or double");
 
 private:
-	SimpleCounter(std::string_view n) : value(T(0)), name_(n) {}
+	explicit SimpleCounter(std::string_view n) : value(T(0)), name_(n) {}
 
 	// Not copyable or movable.
 	SimpleCounter(const SimpleCounter&) = delete;
diff --git a/flow/include/flow/StreamCipher.h b/flow/include/flow/StreamCipher.h
index e112f999f0d..aa26aa74557 100644
--- a/flow/include/flow/StreamCipher.h
+++ b/flow/include/flow/StreamCipher.h
@@ -44,7 +44,7 @@ class StreamCipherKey : NonCopyable {
 	int keySize;
 
 public:
-	StreamCipherKey(int size);
+	explicit StreamCipherKey(int size);
 	~StreamCipherKey();
 
 	int size() const { return keySize; }
@@ -69,7 +69,7 @@ class StreamCipher final : NonCopyable {
 
 public:
 	StreamCipher();
-	StreamCipher(int KeySize);
+	explicit StreamCipher(int KeySize);
 	~StreamCipher();
 	EVP_CIPHER_CTX* getCtx();
 	HMAC_CTX* getHmacCtx();
diff --git a/flow/include/flow/SystemMonitor.h b/flow/include/flow/SystemMonitor.h
index 2b703654117..b05d36dbf9e 100644
--- a/flow/include/flow/SystemMonitor.h
+++ b/flow/include/flow/SystemMonitor.h
@@ -23,7 +23,7 @@
 #pragma once
 
 #include "flow/Platform.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 
 struct SystemMonitorMachineState {
 	Optional<std::string> folder;
@@ -96,48 +96,7 @@ struct NetworkData {
 	double countLaunchTime;
 	double countReactTime;
 
-	void init() {
-		bytesSent = Int64Metric::getValueOrDefault("Net2.BytesSent"_sr);
-		countPacketsReceived = Int64Metric::getValueOrDefault("Net2.CountPacketsReceived"_sr);
-		countPacketsGenerated = Int64Metric::getValueOrDefault("Net2.CountPacketsGenerated"_sr);
-		bytesReceived = Int64Metric::getValueOrDefault("Net2.BytesReceived"_sr);
-		countWriteProbes = Int64Metric::getValueOrDefault("Net2.CountWriteProbes"_sr);
-		countReadProbes = Int64Metric::getValueOrDefault("Net2.CountReadProbes"_sr);
-		countReads = Int64Metric::getValueOrDefault("Net2.CountReads"_sr);
-		countWouldBlock = Int64Metric::getValueOrDefault("Net2.CountWouldBlock"_sr);
-		countWrites = Int64Metric::getValueOrDefault("Net2.CountWrites"_sr);
-		countRunLoop = Int64Metric::getValueOrDefault("Net2.CountRunLoop"_sr);
-		countCantSleep = Int64Metric::getValueOrDefault("Net2.CountCantSleep"_sr);
-		countWontSleep = Int64Metric::getValueOrDefault("Net2.CountWontSleep"_sr);
-		countTimers = Int64Metric::getValueOrDefault("Net2.CountTimers"_sr);
-		countTasks = Int64Metric::getValueOrDefault("Net2.CountTasks"_sr);
-		countYields = Int64Metric::getValueOrDefault("Net2.CountYields"_sr);
-		countYieldBigStack = Int64Metric::getValueOrDefault("Net2.CountYieldBigStack"_sr);
-		countYieldCalls = Int64Metric::getValueOrDefault("Net2.CountYieldCalls"_sr);
-		countASIOEvents = Int64Metric::getValueOrDefault("Net2.CountASIOEvents"_sr);
-		countYieldCallsTrue = Int64Metric::getValueOrDefault("Net2.CountYieldCallsTrue"_sr);
-		countRunLoopProfilingSignals = Int64Metric::getValueOrDefault("Net2.CountRunLoopProfilingSignals"_sr);
-		countConnEstablished = Int64Metric::getValueOrDefault("Net2.CountConnEstablished"_sr);
-		countConnClosedWithError = Int64Metric::getValueOrDefault("Net2.CountConnClosedWithError"_sr);
-		countConnClosedWithoutError = Int64Metric::getValueOrDefault("Net2.CountConnClosedWithoutError"_sr);
-		countTLSPolicyFailures = Int64Metric::getValueOrDefault("Net2.CountTLSPolicyFailures"_sr);
-		countLaunchTime = DoubleMetric::getValueOrDefault("Net2.CountLaunchTime"_sr);
-		countReactTime = DoubleMetric::getValueOrDefault("Net2.CountReactTime"_sr);
-		countFileLogicalWrites = Int64Metric::getValueOrDefault("AsyncFile.CountLogicalWrites"_sr);
-		countFileLogicalReads = Int64Metric::getValueOrDefault("AsyncFile.CountLogicalReads"_sr);
-		countAIOSubmit = Int64Metric::getValueOrDefault("AsyncFile.CountAIOSubmit"_sr);
-		countAIOCollect = Int64Metric::getValueOrDefault("AsyncFile.CountAIOCollect"_sr);
-		countFileCacheWrites = Int64Metric::getValueOrDefault("AsyncFile.CountCacheWrites"_sr);
-		countFileCacheReads = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReads"_sr);
-		countFileCacheWritesBlocked = Int64Metric::getValueOrDefault("AsyncFile.CountCacheWritesBlocked"_sr);
-		countFileCacheReadsBlocked = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReadsBlocked"_sr);
-		countFileCachePageReadsMerged = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMerged"_sr);
-		countFileCacheFinds = Int64Metric::getValueOrDefault("AsyncFile.CountCacheFinds"_sr);
-		countFileCacheReadBytes = Int64Metric::getValueOrDefault("AsyncFile.CountCacheReadBytes"_sr);
-		countFilePageCacheHits = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsHit"_sr);
-		countFilePageCacheMisses = Int64Metric::getValueOrDefault("AsyncFile.CountCachePageReadsMissed"_sr);
-		countFilePageCacheEvictions = Int64Metric::getValueOrDefault("EvictablePageCache.CacheEvictions"_sr);
-	}
+	void init();
 };
 
 struct StatisticsState {
diff --git a/flow/include/flow/TDMetric.actor.h b/flow/include/flow/TDMetric.h
similarity index 91%
rename from flow/include/flow/TDMetric.actor.h
rename to flow/include/flow/TDMetric.h
index d4b7eeeeccd..929359b25f4 100644
--- a/flow/include/flow/TDMetric.actor.h
+++ b/flow/include/flow/TDMetric.h
@@ -1,5 +1,5 @@
 /*
- * TDMetric.actor.h
+ * TDMetric.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,28 +20,23 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
 #include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include <cstddef>
-#if defined(NO_INTELLISENSE) && !defined(FLOW_TDMETRIC_ACTOR_G_H)
-#define FLOW_TDMETRIC_ACTOR_G_H
-#include "flow/TDMetric.actor.g.h"
-#elif !defined(FLOW_TDMETRIC_ACTOR_H)
-#define FLOW_TDMETRIC_ACTOR_H
+#include <array>
 #include <string>
 #include <unordered_map>
 #include "flow/flow.h"
+#include "flow/CoroUtils.h"
 #include "flow/network.h"
 #include "flow/Knobs.h"
 #include "flow/genericactors.actor.h"
 #include "flow/CompressedInt.h"
 #include "flow/OTELMetrics.h"
 #include <algorithm>
-#include <functional>
 #include <cmath>
-#include "flow/actorcompiler.h" // This must be the last #include.
+#include <functional>
+#include <type_traits>
 
 enum MetricsDataModel { STATSD = 0, OTLP, NONE };
 MetricsDataModel knobToMetricModel(const std::string& knob);
@@ -99,7 +94,7 @@ struct KeyWithWriter {
 
 	KeyWithWriter(Standalone<StringRef> const& key, BinaryWriter& writer, int writerOffset = 0)
 	  : key(key), writer(std::move(writer)), writerOffset(writerOffset) {}
-	KeyWithWriter(KeyWithWriter&& r)
+	explicit(false) KeyWithWriter(KeyWithWriter&& r)
 	  : key(std::move(r.key)), writer(std::move(r.writer)), writerOffset(r.writerOffset) {}
 	void operator=(KeyWithWriter&& r) {
 		key = std::move(r.key);
@@ -148,9 +143,9 @@ struct MetricKeyRef {
 		return mk;
 	}
 
-	const Standalone<StringRef> packLatestKey() const;
-	const Standalone<StringRef> packDataKey(int64_t time = -1) const;
-	const Standalone<StringRef> packFieldRegKey() const;
+	Standalone<StringRef> packLatestKey() const;
+	Standalone<StringRef> packDataKey(int64_t time = -1) const;
+	Standalone<StringRef> packFieldRegKey() const;
 
 	bool isField() const { return fieldName.size() > 0 && fieldType.size() > 0; }
 	void writeField(BinaryWriter& wr) const;
@@ -176,7 +171,7 @@ struct MetricBatch {
 
 	MetricBatch() {}
 
-	MetricBatch(FDBScope* in) {
+	explicit MetricBatch(FDBScope* in) {
 		assert(in != nullptr);
 		scope.inserts = std::move(in->inserts);
 		scope.appends = std::move(in->appends);
@@ -276,7 +271,7 @@ struct MetricData {
 	  : start(0), rollTime(std::numeric_limits<uint64_t>::max()), appendStart(appendStart),
 	    writer(AssumeVersion(g_network->protocolVersion())) {}
 
-	MetricData(MetricData&& r) noexcept
+	explicit(false) MetricData(MetricData&& r) noexcept
 	  : start(r.start), rollTime(r.rollTime), appendStart(r.appendStart), writer(std::move(r.writer)) {}
 
 	void operator=(MetricData&& r) noexcept {
@@ -363,12 +358,12 @@ struct make_index_sequence_impl;
 
 template <size_t Start, size_t... Indices, size_t End>
 struct make_index_sequence_impl<Start, index_sequence<Indices...>, End> {
-	typedef typename make_index_sequence_impl<Start + 1, index_sequence<Indices..., Start>, End>::type type;
+	using type = typename make_index_sequence_impl<Start + 1, index_sequence<Indices..., Start>, End>::type;
 };
 
 template <size_t End, size_t... Indices>
 struct make_index_sequence_impl<End, index_sequence<Indices...>, End> {
-	typedef index_sequence<Indices...> type;
+	using type = index_sequence<Indices...>;
 };
 
 // The code that actually implements tuple_map
@@ -394,16 +389,105 @@ auto tuple_map(F f, const Tuple& t, const Tuples&... ts) -> decltype(tuple_map_i
 	    f, typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size<Tuple>::value>::type(), t, ts...);
 }
 
+template <class Tuple>
+using tuple_indexes_t = typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size<Tuple>::value>::type;
+
 template <class T>
 struct Descriptor {
+	// Specialize Descriptor<T> next to each metric payload struct, typically by inheriting from
+	// DescribeType<T, "...", DescribeField<&T::member, "...">, ...>.
 #ifndef NO_INTELLISENSE
 	using fields = std::tuple<>;
-	typedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size<fields>::value>::type field_indexes;
+	using field_indexes = tuple_indexes_t<fields>;
 
 	static StringRef typeName() { return ""_sr; }
 #endif
 };
 
+// String literals need a wrapper type before they can be used as non-type template parameters.
+template <size_t N>
+struct FixedString {
+	std::array<char, N> value{};
+
+	constexpr FixedString(const char (&str)[N]) {
+		for (size_t i = 0; i < N; ++i) {
+			value[i] = str[i];
+		}
+	}
+
+	constexpr size_t size() const { return N - 1; }
+};
+
+template <size_t N>
+FixedString(const char (&)[N]) -> FixedString<N>;
+
+template <FixedString Str>
+StringRef fixedStringRef() {
+	return StringRef(reinterpret_cast<const uint8_t*>(Str.value.data()), Str.size());
+}
+
+template <auto MemberPtr>
+struct MemberPointerTraits;
+
+// DescribeField only takes a member pointer at the call site; recover the owning class and field type here.
+template <class Class, class FieldType, FieldType Class::* MemberPtr>
+struct MemberPointerTraits<MemberPtr> {
+	using class_type = Class;
+	using field_type = FieldType;
+};
+
+template <typename T>
+inline StringRef describeFieldTypeName() {
+	return metricTypeName<T>();
+}
+
+template <>
+inline StringRef describeFieldTypeName<int64_t>() {
+	return "int64_t"_sr;
+}
+
+template <>
+inline StringRef describeFieldTypeName<double>() {
+	return "double"_sr;
+}
+
+template <>
+inline StringRef describeFieldTypeName<bool>() {
+	return "bool"_sr;
+}
+
+template <>
+inline StringRef describeFieldTypeName<Standalone<StringRef>>() {
+	return "Standalone<StringRef>"_sr;
+}
+
+// One field entry inside a Descriptor<T> specialization.
+// Use DescribeField<&T::member, "memberName", "optional comment">.
+template <auto MemberPtr, FixedString Name, FixedString Comment = "">
+struct DescribeField {
+	using traits = MemberPointerTraits<MemberPtr>;
+	using class_type = typename traits::class_type;
+	using type = typename traits::field_type;
+
+	static StringRef name() { return fixedStringRef<Name>(); }
+	static StringRef typeName() { return describeFieldTypeName<type>(); }
+	static StringRef comment() { return fixedStringRef<Comment>(); }
+	static inline type get(class_type& from) { return from.*MemberPtr; }
+};
+
+// Shared implementation for Descriptor<T> specializations. The specialization names the metric type
+// and lists its exported fields in the order they should appear in TDMetric output.
+template <class Self, FixedString TypeName, class... Fields>
+struct DescribeType {
+	static_assert((std::is_same_v<Self, typename Fields::class_type> && ...));
+
+	static StringRef typeName() { return fixedStringRef<TypeName>(); }
+
+	using type = Self;
+	using fields = std::tuple<Fields...>;
+	using field_indexes = tuple_indexes_t<fields>;
+};
+
 // FieldHeader is a serializable (FIXED SIZE!) and updatable Header type for Metric field levels.
 // Update is via += with either a T or another FieldHeader
 // Default implementation is sufficient for ints and doubles
@@ -631,13 +715,12 @@ class FieldLevel {
 		}
 	}
 
-	ACTOR static Future<Void> updatePreviousHeader(FieldLevel* self,
-	                                               IMetricDB* db,
-	                                               Standalone<MetricKeyRef> mk,
-	                                               uint64_t rollTime,
-	                                               FDBScope* scope) {
-
-		Optional<Standalone<StringRef>> block = wait(db->getLastBlock(mk.packDataKey(-1)));
+	static Future<Void> updatePreviousHeader(FieldLevel* self,
+	                                         IMetricDB* db,
+	                                         Standalone<MetricKeyRef> mk,
+	                                         uint64_t rollTime,
+	                                         FDBScope* scope) {
+		Optional<Standalone<StringRef>> block = co_await db->getLastBlock(mk.packDataKey(-1));
 
 		// If the block is present, use it
 		if (block.present()) {
@@ -666,8 +749,6 @@ class FieldLevel {
 		// lastTimeRequiringHeaderPatch
 		MetricBatch batch{ scope };
 		self->flushUpdates(mk, rollTime, batch);
-
-		return Void();
 	}
 
 	// Flush this level's data to the output batch.
@@ -703,11 +784,11 @@ template <class T, class Descriptor = NullDescriptor, class FieldLevelType = Fie
 struct EventField : public Descriptor {
 	std::vector<FieldLevelType> levels;
 
-	EventField(EventField&& r) noexcept : Descriptor(r), levels(std::move(r.levels)) {}
+	explicit(false) EventField(EventField&& r) noexcept : Descriptor(r), levels(std::move(r.levels)) {}
 
 	void operator=(EventField&& r) noexcept { levels = std::move(r.levels); }
 
-	EventField(Descriptor d = Descriptor()) : Descriptor(d) {}
+	explicit EventField(Descriptor d = Descriptor()) : Descriptor(d) {}
 
 	static StringRef typeName() { return metricTypeName<T>(); }
 
@@ -761,7 +842,8 @@ struct TimeDescriptor {
 };
 
 struct BaseMetric {
-	BaseMetric(MetricNameRef const& name) : metricName(name), enabled(false), pCollection(nullptr), registered(false) {
+	explicit BaseMetric(MetricNameRef const& name)
+	  : metricName(name), enabled(false), pCollection(nullptr), registered(false) {
 		setConfig(false);
 	}
 	virtual ~BaseMetric() {}
@@ -820,7 +902,7 @@ struct BaseMetric {
 
 struct BaseEventMetric : BaseMetric {
 
-	BaseEventMetric(MetricNameRef const& name) : BaseMetric(name) {}
+	explicit BaseEventMetric(MetricNameRef const& name) : BaseMetric(name) {}
 
 	// Needed for MetricUtil
 	alignas(8) static const StringRef metricType;
@@ -967,7 +1049,7 @@ struct EventMetric final : E, ReferenceCounted<EventMetric<E>>, MetricUtil<Event
 
 // A field Descriptor compatible with EventField but with name set at runtime
 struct DynamicDescriptor {
-	DynamicDescriptor(const char* name) : _name(StringRef((uint8_t*)name, strlen(name))) {}
+	explicit DynamicDescriptor(const char* name) : _name(StringRef((uint8_t*)name, strlen(name))) {}
 	StringRef name() const { return _name; }
 
 private:
@@ -1016,7 +1098,7 @@ struct DynamicFieldBase {
 template <typename T>
 struct DynamicField final : public DynamicFieldBase, EventField<T, DynamicDescriptor> {
 	typedef EventField<T, DynamicDescriptor> EventFieldType;
-	DynamicField(const char* name) : DynamicFieldBase(), EventFieldType(DynamicDescriptor(name)), value(T()) {}
+	explicit DynamicField(const char* name) : DynamicFieldBase(), EventFieldType(DynamicDescriptor(name)), value(T()) {}
 
 	StringRef fieldName() const override { return EventFieldType::name(); }
 
@@ -1096,7 +1178,7 @@ struct DynamicEventMetric final : ReferenceCounted<DynamicEventMetric>,
 	}
 
 public:
-	DynamicEventMetric(MetricNameRef const& name, Void = Void());
+	explicit DynamicEventMetric(MetricNameRef const& name, Void = Void());
 	~DynamicEventMetric() override = default;
 
 	void addref() override { ReferenceCounted<DynamicEventMetric>::addref(); }
@@ -1395,9 +1477,9 @@ template <typename T>
 struct MetricHandle {
 	using ValueType = typename T::ValueType;
 
-	MetricHandle(StringRef const& name = StringRef(),
-	             StringRef const& id = StringRef(),
-	             ValueType const& initial = ValueType())
+	explicit MetricHandle(StringRef const& name = StringRef(),
+	                      StringRef const& id = StringRef(),
+	                      ValueType const& initial = ValueType())
 	  : ref(T::getOrCreateInstance(name, id, true, initial)) {}
 
 	// Initialize this handle to point to a new or existing metric with (name, id).  If a new metric is created then the
@@ -1456,7 +1538,7 @@ class IMetric {
 public:
 	const UID id;
 	const MetricsDataModel model;
-	IMetric(MetricsDataModel m) : id{ deterministicRandom()->randomUniqueID() }, model{ m } {
+	explicit IMetric(MetricsDataModel m) : id{ deterministicRandom()->randomUniqueID() }, model{ m } {
 		MetricCollection* metrics = MetricCollection::getMetricCollection();
 		if (metrics != nullptr) {
 			if (metrics->map.count(id) > 0) {
@@ -1488,7 +1570,3 @@ bool verifyStatsdMessage(const std::string& msg);
 void createOtelGauge(UID id, const std::string& name, double value);
 
 void createOtelGauge(UID id, const std::string& name, double value, const std::vector<OTEL::Attribute>&);
-
-#include "flow/unactorcompiler.h"
-
-#endif
diff --git a/flow/include/flow/TLSConfig.actor.h b/flow/include/flow/TLSConfig.h
similarity index 95%
rename from flow/include/flow/TLSConfig.actor.h
rename to flow/include/flow/TLSConfig.h
index 5bd335b758a..e186c5d8600 100644
--- a/flow/include/flow/TLSConfig.actor.h
+++ b/flow/include/flow/TLSConfig.h
@@ -1,5 +1,5 @@
 /*
- * TLSConfig.actor.h
+ * TLSConfig.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,13 +18,8 @@
  * limitations under the License.
  */
 
-// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
-// version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_TLS_CONFIG_ACTOR_G_H)
-#define FLOW_TLS_CONFIG_ACTOR_G_H
-#include "flow/TLSConfig.actor.g.h"
-#elif !defined(FLOW_TLS_CONFIG_ACTOR_H)
-#define FLOW_TLS_CONFIG_ACTOR_H
+#ifndef FLOW_TLSCONFIG_H
+#define FLOW_TLSCONFIG_H
 
 #pragma once
 
@@ -42,7 +37,6 @@
 #include "flow/FastRef.h"
 #include "flow/Knobs.h"
 #include "flow/flow.h"
-#include "flow/actorcompiler.h" // This must be the last #include.
 
 typedef int NID;
 
@@ -219,7 +213,7 @@ class TLSConfig {
 #ifndef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP
 private:
 #endif
-	ACTOR static Future<LoadedTLSConfig> loadAsync(const TLSConfig* self); // FIXME
+	static Future<LoadedTLSConfig> loadAsync(const TLSConfig* self); // FIXME
 	template <typename T>
 	friend class LoadAsyncActorState;
 
@@ -315,5 +309,4 @@ class TLSPolicy : ReferenceCounted<TLSPolicy> {
 	"  " TLS_DISABLE_PLAINTEXT_CONNECTION_FLAG "\n"                                                                    \
 	"                 Disable non-TLS connections. All plaintext connection attempts will timeout.\n"
 
-#include "flow/unactorcompiler.h"
 #endif
diff --git a/flow/include/flow/TaskQueue.h b/flow/include/flow/TaskQueue.h
index 7cea68efbf6..5691d020dcb 100644
--- a/flow/include/flow/TaskQueue.h
+++ b/flow/include/flow/TaskQueue.h
@@ -24,7 +24,7 @@
 
 #include <queue>
 #include <vector>
-#include "flow/TDMetric.actor.h"
+#include "flow/TDMetric.h"
 #include "flow/network.h"
 #include "flow/ThreadSafeQueue.h"
 
@@ -139,7 +139,7 @@ class TaskQueue {
 	class ReadyQueue : public std::priority_queue<T, std::vector<T>> {
 	public:
 		typedef typename std::priority_queue<T, std::vector<T>>::size_type size_type;
-		ReadyQueue(size_type capacity = 0) { reserve(capacity); };
+		explicit ReadyQueue(size_type capacity = 0) { reserve(capacity); };
 		void reserve(size_type capacity) { this->c.reserve(capacity); }
 	};
 
diff --git a/flow/include/flow/ThreadHelper.actor.h b/flow/include/flow/ThreadHelper.actor.h
index b2f95f35058..36d734cfb96 100644
--- a/flow/include/flow/ThreadHelper.actor.h
+++ b/flow/include/flow/ThreadHelper.actor.h
@@ -300,7 +300,7 @@ class ThreadSingleAssignmentVarBase {
 	public:
 		Event ev;
 
-		BlockCallback(ThreadSingleAssignmentVarBase& sav) {
+		explicit BlockCallback(ThreadSingleAssignmentVarBase& sav) {
 			int ignore = 0;
 			sav.callOrSetAsCallback(this, ignore, 0);
 			ev.block();
@@ -326,7 +326,8 @@ class ThreadSingleAssignmentVarBase {
 		}
 	}
 
-	ThreadSingleAssignmentVarBase() : status(Unset), callback(NULL), valueReferenceCount(0) {} //, referenceCount(1) {}
+	ThreadSingleAssignmentVarBase()
+	  : status(Unset), callback(nullptr), valueReferenceCount(0) {} //, referenceCount(1) {}
 	~ThreadSingleAssignmentVarBase() {
 		this->mutex.assertNotEntered();
 
@@ -598,15 +599,19 @@ class ThreadFuture {
 	explicit ThreadFuture(ThreadSingleAssignmentVar<T>* sav) : sav(sav) {
 		// sav->addref();
 	}
-	ThreadFuture(const ThreadFuture<T>& rhs) : sav(rhs.sav) {
+	explicit(false) ThreadFuture(const ThreadFuture<T>& rhs) : sav(rhs.sav) {
 		if (sav)
 			sav->addref();
 	}
-	ThreadFuture(ThreadFuture<T>&& rhs) noexcept : sav(rhs.sav) { rhs.sav = 0; }
-	ThreadFuture(const T& presentValue) : sav(new ThreadSingleAssignmentVar<T>()) { sav->send(presentValue); }
-	ThreadFuture(Never) : sav(new ThreadSingleAssignmentVar<T>()) {}
-	ThreadFuture(const Error& error) : sav(new ThreadSingleAssignmentVar<T>()) { sav->sendError(error); }
-	ThreadFuture(const ErrorOr<T>& errorOr) : sav(new ThreadSingleAssignmentVar<T>()) {
+	explicit(false) ThreadFuture(ThreadFuture<T>&& rhs) noexcept : sav(rhs.sav) { rhs.sav = 0; }
+	explicit(false) ThreadFuture(const T& presentValue) : sav(new ThreadSingleAssignmentVar<T>()) {
+		sav->send(presentValue);
+	}
+	explicit(false) ThreadFuture(Never) : sav(new ThreadSingleAssignmentVar<T>()) {}
+	explicit(false) ThreadFuture(const Error& error) : sav(new ThreadSingleAssignmentVar<T>()) {
+		sav->sendError(error);
+	}
+	explicit(false) ThreadFuture(const ErrorOr<T>& errorOr) : sav(new ThreadSingleAssignmentVar<T>()) {
 		if (errorOr.isError()) {
 			sav->sendError(errorOr.getError());
 		} else {
@@ -661,7 +666,7 @@ struct CompletionCallback final : public ThreadCallback, ReferenceCounted<Comple
 	// Holds own reference to prevent deletion until callback is fired
 	Reference<CompletionCallback<T>> self;
 
-	CompletionCallback(ThreadFuture<T> threadFuture) { this->threadFuture = threadFuture; }
+	explicit CompletionCallback(ThreadFuture<T> threadFuture) { this->threadFuture = threadFuture; }
 
 	bool canFire(int notMadeActive) const override { return true; }
 
@@ -779,14 +784,14 @@ template <typename T>
 struct allow_anonymous_future<Optional<Standalone<T>>> : std::false_type {};
 
 template <class T>
-typename std::enable_if<allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
-    const ThreadFuture<T>& threadFuture) {
+    requires(allow_anonymous_future<T>::value)
+Future<T> safeThreadFutureToFuture(const ThreadFuture<T>& threadFuture) {
 	return safeThreadFutureToFutureImpl(threadFuture);
 }
 
 template <class T>
-typename std::enable_if<!allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
-    ThreadFuture<T>& threadFuture) {
+    requires(!allow_anonymous_future<T>::value)
+Future<T> safeThreadFutureToFuture(ThreadFuture<T>& threadFuture) {
 	Future<T> f = safeThreadFutureToFutureImpl(threadFuture);
 	if (BUGGIFY) {
 		return removeArenaFromStandalone(f);
@@ -795,15 +800,15 @@ typename std::enable_if<!allow_anonymous_future<T>::value, Future<T>>::type safe
 }
 
 template <class T>
-typename std::enable_if<allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
-    const Future<T>& future) {
+    requires(allow_anonymous_future<T>::value)
+Future<T> safeThreadFutureToFuture(const Future<T>& future) {
 	// Do nothing
 	return future;
 }
 
 template <class T>
-typename std::enable_if<!allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
-    Future<T>& future) {
+    requires(!allow_anonymous_future<T>::value)
+Future<T> safeThreadFutureToFuture(Future<T>& future) {
 	if (BUGGIFY) {
 		return removeArenaFromStandalone(future);
 	}
@@ -865,7 +870,7 @@ class ThreadSafeAsyncVar : NonCopyable, public ThreadSafeReferenceCounted<Thread
 	};
 
 	ThreadSafeAsyncVar() : value(), nextChange(new ThreadSingleAssignmentVar<Void>()) {}
-	ThreadSafeAsyncVar(V const& v) : value(v), nextChange(new ThreadSingleAssignmentVar<Void>()) {}
+	explicit ThreadSafeAsyncVar(V const& v) : value(v), nextChange(new ThreadSingleAssignmentVar<Void>()) {}
 
 	State get() {
 		ThreadSpinLockHolder holder(lock);
@@ -918,14 +923,18 @@ class ThreadResult {
 		ASSERT(sav->isReady());
 		// sav->addref();
 	}
-	ThreadResult(const ThreadResult<T>& rhs) : sav(rhs.sav) {
+	explicit(false) ThreadResult(const ThreadResult<T>& rhs) : sav(rhs.sav) {
 		if (sav)
 			sav->addref();
 	}
-	ThreadResult(ThreadResult<T>&& rhs) noexcept : sav(rhs.sav) { rhs.sav = 0; }
-	ThreadResult(const T& presentValue) : sav(new ThreadSingleAssignmentVar<T>()) { sav->send(presentValue); }
-	ThreadResult(const Error& error) : sav(new ThreadSingleAssignmentVar<T>()) { sav->sendError(error); }
-	ThreadResult(const ErrorOr<T> errorOr) : sav(new ThreadSingleAssignmentVar<T>()) {
+	explicit(false) ThreadResult(ThreadResult<T>&& rhs) noexcept : sav(rhs.sav) { rhs.sav = 0; }
+	explicit(false) ThreadResult(const T& presentValue) : sav(new ThreadSingleAssignmentVar<T>()) {
+		sav->send(presentValue);
+	}
+	explicit(false) ThreadResult(const Error& error) : sav(new ThreadSingleAssignmentVar<T>()) {
+		sav->sendError(error);
+	}
+	explicit(false) ThreadResult(const ErrorOr<T> errorOr) : sav(new ThreadSingleAssignmentVar<T>()) {
 		if (errorOr.isError()) {
 			sav->sendError(errorOr.getError());
 		} else {
diff --git a/flow/include/flow/ThreadPrimitives.h b/flow/include/flow/ThreadPrimitives.h
index 0e834c2444a..403b38d7288 100644
--- a/flow/include/flow/ThreadPrimitives.h
+++ b/flow/include/flow/ThreadPrimitives.h
@@ -100,7 +100,7 @@ class ThreadSpinLockHolder {
 	ThreadSpinLock& lock;
 
 public:
-	ThreadSpinLockHolder(ThreadSpinLock& lock) : lock(lock) { lock.enter(); }
+	explicit ThreadSpinLockHolder(ThreadSpinLock& lock) : lock(lock) { lock.enter(); }
 	ThreadSpinLockHolder(const ThreadSpinLockHolder& lock) = delete;
 	~ThreadSpinLockHolder() { lock.leave(); }
 };
@@ -113,7 +113,7 @@ class ThreadUnsafeSpinLock {
 };
 class ThreadUnsafeSpinLockHolder {
 public:
-	ThreadUnsafeSpinLockHolder(ThreadUnsafeSpinLock&) {};
+	explicit ThreadUnsafeSpinLockHolder(ThreadUnsafeSpinLock&) {};
 };
 
 #if FLOW_THREAD_SAFE
@@ -156,7 +156,7 @@ class MutexHolder {
 	Mutex& lock;
 
 public:
-	MutexHolder(Mutex& lock) : lock(lock) { lock.enter(); }
+	explicit MutexHolder(Mutex& lock) : lock(lock) { lock.enter(); }
 	~MutexHolder() { lock.leave(); }
 };
 
diff --git a/flow/include/flow/ThreadSafeQueue.h b/flow/include/flow/ThreadSafeQueue.h
index 12bef0cfea5..8f308defdd2 100644
--- a/flow/include/flow/ThreadSafeQueue.h
+++ b/flow/include/flow/ThreadSafeQueue.h
@@ -53,8 +53,8 @@ class ThreadSafeQueue : NonCopyable {
 
 	struct Node : BaseNode, FastAllocated<Node> {
 		T data;
-		Node(T const& data) : data(data) {}
-		Node(T&& data) : data(std::move(data)) {}
+		explicit Node(T const& data) : data(data) {}
+		explicit Node(T&& data) : data(std::move(data)) {}
 	};
 	std::atomic<BaseNode*> head;
 	BaseNode* tail;
diff --git a/flow/include/flow/Trace.h b/flow/include/flow/Trace.h
index 0f1431479ad..18efd505b74 100644
--- a/flow/include/flow/Trace.h
+++ b/flow/include/flow/Trace.h
@@ -145,7 +145,6 @@ class TraceEventFields {
 	Field& mutate(int index);
 
 	std::string toString() const;
-	void validateFormat() const;
 	template <class Archiver>
 	void serialize(Archiver& ar) {
 		static_assert(is_fb_function<Archiver>, "Streaming serializer has to use load/save");
@@ -287,8 +286,8 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 	static std::string printRealTime(double time);
 
 	template <class T>
-	typename std::enable_if<Traceable<T>::value && !std::is_enum_v<T>, BaseTraceEvent&>::type detail(std::string&& key,
-	                                                                                                 const T& value) {
+	    requires(Traceable<T>::value && !std::is_enum_v<T>)
+	BaseTraceEvent& detail(std::string&& key, const T& value) {
 		if (enabled && init()) {
 			auto s = Traceable<T>::toString(value);
 			addMetric(key.c_str(), value, s);
@@ -298,8 +297,8 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 	}
 
 	template <class T>
-	typename std::enable_if<Traceable<T>::value && !std::is_enum_v<T>, BaseTraceEvent&>::type detail(const char* key,
-	                                                                                                 const T& value) {
+	    requires(Traceable<T>::value && !std::is_enum_v<T>)
+	BaseTraceEvent& detail(const char* key, const T& value) {
 		if (enabled && init()) {
 			auto s = Traceable<T>::toString(value);
 			addMetric(key, value, s);
@@ -308,7 +307,8 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 		return *this;
 	}
 	template <class T>
-	typename std::enable_if<std::is_enum<T>::value, BaseTraceEvent&>::type detail(const char* key, T value) {
+	    requires(std::is_enum_v<T>)
+	BaseTraceEvent& detail(const char* key, T value) {
 		if (enabled && init()) {
 			setField(key, int64_t(value));
 			return detailImpl(std::string(key), format("%d", value), false);
@@ -316,6 +316,9 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 		return *this;
 	}
 	BaseTraceEvent& detailf(std::string key, const char* valueFormat, ...);
+	// Logs a formatted message with a fixed key ("LogMessage") for the case
+	// where all you want to do is log a message.
+	BaseTraceEvent& log(const char* format, ...);
 
 protected:
 	class State {
@@ -328,14 +331,14 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 
 	public:
 		constexpr State() noexcept : value(Type::DISABLED) {}
-		State(Severity severity) noexcept;
+		explicit State(Severity severity) noexcept;
 		State(Severity severity, AuditedEvent) noexcept : State(severity) {
 			if (*this)
 				value = Type::FORCED;
 		}
 
-		State(const State& other) noexcept = default;
-		State(State&& other) noexcept : value(other.value) { other.value = Type::DISABLED; }
+		explicit(false) State(const State& other) noexcept = default;
+		explicit(false) State(State&& other) noexcept : value(other.value) { other.value = Type::DISABLED; }
 		State& operator=(const State& other) noexcept = default;
 		State& operator=(State&& other) noexcept {
 			if (this != &other) {
@@ -368,16 +371,14 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 	BaseTraceEvent(Severity, const char* type, UID id = UID());
 
 	template <class T>
-	typename std::enable_if<SpecialTraceMetricType<T>::value, void>::type addMetric(const char* key,
-	                                                                                const T& value,
-	                                                                                const std::string&) {
+	    requires(SpecialTraceMetricType<T>::value)
+	void addMetric(const char* key, const T& value, const std::string&) {
 		setField(key, SpecialTraceMetricType<T>::getValue(value));
 	}
 
 	template <class T>
-	typename std::enable_if<!SpecialTraceMetricType<T>::value, void>::type addMetric(const char* key,
-	                                                                                 const T&,
-	                                                                                 const std::string& value) {
+	    requires(!SpecialTraceMetricType<T>::value)
+	void addMetric(const char* key, const T&, const std::string& value) {
 		setField(key, value);
 	}
 
@@ -416,7 +417,12 @@ struct SWIFT_CXX_IMPORT_OWNED BaseTraceEvent {
 
 	explicit operator bool() const { return static_cast<bool>(enabled); }
 
-	void log();
+	// Writes the event to the underlying log file.
+	void writeEvent();
+	// Legacy `log` method to force writing. This can be done by the destructor,
+	// but lots of legacy code (~500 calls) explicitly calls `log`, so leave this in
+	// to pacify those call sites.
+	void log(void) { writeEvent(); }
 
 	void disable() { enabled.suppress(); } // Disables the trace event so it doesn't get logged
 
@@ -491,7 +497,7 @@ struct SWIFT_CXX_IMPORT_OWNED TraceEvent : public BaseTraceEvent {
 	BaseTraceEvent& sample(double sampleRate, bool logSampleRate = true);
 	BaseTraceEvent& suppressFor(double duration, bool logSuppressedEventCount = true);
 
-	// Exposed for Swift which cannot use std::enable_if
+	// Exposed for Swift which cannot use constrained overloads.
 	template <class T>
 	void addDetail(std::string key, const T& value) {
 		if (enabled && init()) {
@@ -505,7 +511,7 @@ struct SWIFT_CXX_IMPORT_OWNED TraceEvent : public BaseTraceEvent {
 class StringRef;
 
 struct TraceInterval {
-	TraceInterval(const char* type, UID id = UID()) : type(type), pairID(id), count(-1), severity(SevInfo) {}
+	explicit TraceInterval(const char* type, UID id = UID()) : type(type), pairID(id), count(-1), severity(SevInfo) {}
 
 	TraceInterval& begin();
 	TraceInterval& end() { return *this; }
@@ -540,7 +546,7 @@ extern LatestEventCache latestEventCache;
 struct EventCacheHolder : public ReferenceCounted<EventCacheHolder> {
 	std::string trackingKey;
 
-	EventCacheHolder(const std::string& trackingKey) : trackingKey(trackingKey) {}
+	explicit EventCacheHolder(const std::string& trackingKey) : trackingKey(trackingKey) {}
 
 	~EventCacheHolder() { latestEventCache.clear(trackingKey); }
 };
diff --git a/flow/include/flow/Traceable.h b/flow/include/flow/Traceable.h
index a63c99b2b77..0ad6fb1df6e 100644
--- a/flow/include/flow/Traceable.h
+++ b/flow/include/flow/Traceable.h
@@ -109,7 +109,8 @@ FORMAT_TRACEABLE(volatile unsigned long long, "%llu");
 FORMAT_TRACEABLE(volatile double, "%g");
 
 template <class Enum>
-struct Traceable<Enum, std::enable_if_t<std::is_enum_v<Enum>>> : std::true_type {
+    requires(std::is_enum_v<Enum>)
+struct Traceable<Enum> : std::true_type {
 	static std::string toString(Enum e) { return format("%lld", (int64_t)e); }
 };
 
@@ -250,7 +251,8 @@ struct Traceable<std::atomic<T>> : std::true_type {
 };
 
 template <class BooleanParamSub>
-struct Traceable<BooleanParamSub, std::enable_if_t<std::is_base_of_v<BooleanParam, BooleanParamSub>>> : std::true_type {
+    requires(std::is_base_of_v<BooleanParam, BooleanParamSub>)
+struct Traceable<BooleanParamSub> : std::true_type {
 	static std::string toString(BooleanParamSub const& value) { return Traceable<bool>::toString(value); }
 };
 
diff --git a/flow/include/flow/TreeBenchmark.h b/flow/include/flow/TreeBenchmark.h
index 1769ab27b31..2ea2c441df4 100644
--- a/flow/include/flow/TreeBenchmark.h
+++ b/flow/include/flow/TreeBenchmark.h
@@ -53,7 +53,7 @@ struct MapHarness {
 	struct result {
 		typename map::const_iterator it;
 
-		result(typename map::const_iterator it) : it(it) {}
+		explicit result(typename map::const_iterator it) : it(it) {}
 
 		result& operator++() {
 			it++;
diff --git a/flow/include/flow/TxnCounters.h b/flow/include/flow/TxnCounters.h
new file mode 100644
index 00000000000..5331386e4db
--- /dev/null
+++ b/flow/include/flow/TxnCounters.h
@@ -0,0 +1,42 @@
+/*
+ * TxnCounters.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLOW_TXNCOUNTERS_H
+#define FLOW_TXNCOUNTERS_H
+#pragma once
+
+#include "flow/SimpleCounter.h"
+
+struct TxnCounters {
+	SimpleCounter<int64_t>* started;
+	SimpleCounter<int64_t>* committed;
+	SimpleCounter<int64_t>* aborted;
+};
+
+inline TxnCounters* makeCounters(const char* prefix) {
+	std::string p(prefix);
+	auto* c = new TxnCounters();
+	c->started = SimpleCounter<int64_t>::makeCounter(p + "/started");
+	c->committed = SimpleCounter<int64_t>::makeCounter(p + "/committed");
+	c->aborted = SimpleCounter<int64_t>::makeCounter(p + "/aborted");
+	return c;
+}
+
+#endif
diff --git a/flow/include/flow/Util.h b/flow/include/flow/Util.h
index b6c411ca841..b66ccf1bf66 100644
--- a/flow/include/flow/Util.h
+++ b/flow/include/flow/Util.h
@@ -24,8 +24,12 @@
 #pragma once
 
 #include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <ctime>
 #include <functional>
 #include <iosfwd>
+#include <string>
 
 // Read key/value pairs from stream. The stream is constituted by lines of text.
 // Each line contains a pair of key/value, separated by space/tab. e.g.
@@ -75,7 +79,7 @@ void swapAndPop(C* container, int index) {
 // Adds n to pCount upon construction, subtracts in upon destruction
 template <typename T>
 struct Hold {
-	Hold(T* pCount = nullptr, T n = 1) : pCount(pCount), n(n) {
+	explicit Hold(T* pCount = nullptr, T n = 1) : pCount(pCount), n(n) {
 		if (pCount != nullptr) {
 			*pCount += n;
 		}
@@ -86,7 +90,7 @@ struct Hold {
 		}
 	}
 
-	Hold(Hold&& other) {
+	explicit(false) Hold(Hold&& other) {
 		pCount = other.pCount;
 		other.pCount = nullptr;
 		n = other.n;
@@ -115,4 +119,115 @@ struct Hold {
 	void operator=(const Hold& other) = delete;
 };
 
+// Format byte count as human-readable string (e.g., "1.23 TB", "456.78 GB", "789.01 MB")
+// Uses binary units (1 KB = 1024 bytes)
+inline std::string formatBytesHumanReadable(int64_t bytes) {
+	char buf[64];
+	if (bytes >= 1099511627776LL) { // TB
+		std::snprintf(buf, sizeof(buf), "%.2f TB", bytes / 1099511627776.0);
+	} else if (bytes >= 1073741824LL) { // GB
+		std::snprintf(buf, sizeof(buf), "%.2f GB", bytes / 1073741824.0);
+	} else if (bytes >= 1048576LL) { // MB
+		std::snprintf(buf, sizeof(buf), "%.2f MB", bytes / 1048576.0);
+	} else if (bytes >= 1024LL) { // KB
+		std::snprintf(buf, sizeof(buf), "%.2f KB", bytes / 1024.0);
+	} else {
+		std::snprintf(buf, sizeof(buf), "%lld bytes", static_cast<long long>(bytes));
+	}
+	return std::string(buf);
+}
+
+// Format duration in seconds as human-readable string (e.g., "2 hours 30 minutes", "45 minutes", "30 seconds")
+inline std::string formatDurationHumanReadable(int seconds) {
+	char buf[64];
+	if (seconds >= 3600) {
+		int hours = seconds / 3600;
+		int mins = (seconds % 3600) / 60;
+		if (mins > 0) {
+			std::snprintf(buf, sizeof(buf), "%d hours %d minutes", hours, mins);
+		} else {
+			std::snprintf(buf, sizeof(buf), "%d hours", hours);
+		}
+	} else if (seconds >= 60) {
+		std::snprintf(buf, sizeof(buf), "%d minutes", seconds / 60);
+	} else {
+		std::snprintf(buf, sizeof(buf), "%d seconds", seconds);
+	}
+	return std::string(buf);
+}
+
+// Format byte progress as "completed (human) / total (human)"
+inline std::string formatBytesProgress(int64_t completedBytes, int64_t totalBytes) {
+	char buf[256];
+	std::snprintf(buf,
+	              sizeof(buf),
+	              "%lld (%s) / %lld (%s)",
+	              static_cast<long long>(completedBytes),
+	              formatBytesHumanReadable(completedBytes).c_str(),
+	              static_cast<long long>(totalBytes),
+	              formatBytesHumanReadable(totalBytes).c_str());
+	return std::string(buf);
+}
+
+// Overload to handle Optional totalBytes for bulk operations
+template <typename T>
+inline std::string formatBytesProgress(int64_t completedBytes, const T& totalBytes) {
+	if (totalBytes.present()) {
+		return formatBytesProgress(completedBytes, totalBytes.get());
+	} else {
+		char buf[128];
+		std::snprintf(buf,
+		              sizeof(buf),
+		              "%lld (%s)",
+		              static_cast<long long>(completedBytes),
+		              formatBytesHumanReadable(completedBytes).c_str());
+		return std::string(buf);
+	}
+}
+
+// Format throughput display line (returns empty string if throughput <= 0)
+inline std::string formatThroughputLine(double avgBytesPerSecond) {
+	if (avgBytesPerSecond <= 0) {
+		return "";
+	}
+	char buf[128];
+	std::snprintf(buf, sizeof(buf), " Throughput - %.1f MB/s", avgBytesPerSecond / 1048576.0);
+	return std::string(buf);
+}
+
+// Format ETA line (returns empty string if no ETA available)
+inline std::string formatETALine(double etaSeconds) {
+	if (etaSeconds <= 0) {
+		return "";
+	}
+	char buf[128];
+	std::snprintf(
+	    buf, sizeof(buf), " Estimated time remaining - %s", formatDurationHumanReadable((int)etaSeconds).c_str());
+	return std::string(buf);
+}
+
+// Format elapsed time line (returns empty string if elapsedSeconds <= 0)
+inline std::string formatElapsedTimeLine(double elapsedSeconds) {
+	if (elapsedSeconds <= 0) {
+		return "";
+	}
+	char buf[128];
+	std::snprintf(buf, sizeof(buf), " Elapsed time - %s", formatDurationHumanReadable((int)elapsedSeconds).c_str());
+	return std::string(buf);
+}
+
+// Format Unix timestamp as ISO8601 string in UTC (e.g., "2026-03-16T17:51:58Z")
+inline std::string formatTimeISO8601(double epochSeconds) {
+	time_t t = static_cast<time_t>(epochSeconds);
+	struct tm timeinfo;
+#ifdef _WIN32
+	gmtime_s(&timeinfo, &t);
+#else
+	gmtime_r(&t, &timeinfo);
+#endif
+	char buf[32];
+	std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &timeinfo);
+	return std::string(buf);
+}
+
 #endif // _FLOW_UTIL_H_
diff --git a/flow/include/flow/WatchFile.actor.h b/flow/include/flow/WatchFile.h
similarity index 60%
rename from flow/include/flow/WatchFile.actor.h
rename to flow/include/flow/WatchFile.h
index 4560781ef67..0ec2cacda52 100644
--- a/flow/include/flow/WatchFile.actor.h
+++ b/flow/include/flow/WatchFile.h
@@ -1,5 +1,5 @@
 /*
- * WatchFile.actor.h
+ * WatchFile.h
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -20,33 +20,24 @@
 
 #pragma once
 
-// When actually compiled (NO_INTELLISENSE), include the generated
-// version of this file.  In intellisense use the source version.
-#if defined(NO_INTELLISENSE) && !defined(FLOW_WATCH_FILE_ACTOR_G_H)
-#define FLOW_WATCH_FILE_ACTOR_G_H
-#include "flow/WatchFile.actor.g.h"
-#elif !defined(FLOW_WATCH_FILE_ACTOR_H)
-#define FLOW_WATCH_FILE_ACTOR_H
-
 #include <ctime>
 #include <string>
 #include "flow/IAsyncFile.h"
 #include "flow/genericactors.actor.h"
-#include "flow/actorcompiler.h"
 
-ACTOR static Future<Void> watchFileForChanges(std::string filename,
-                                              AsyncTrigger* fileChanged,
-                                              const int* intervalSeconds,
-                                              const char* errorType) {
-	if (filename == "") {
-		return Never();
+static Future<Void> watchFileForChanges(std::string filename,
+                                        AsyncTrigger* fileChanged,
+                                        const int* intervalSeconds,
+                                        const char* errorType) {
+	if (filename.empty()) {
+		co_await Future<Void>(Never());
 	}
-	state bool firstRun = true;
-	state bool statError = false;
-	state std::time_t lastModTime = 0;
-	loop {
+	bool firstRun = true;
+	bool statError = false;
+	std::time_t lastModTime = 0;
+	while (true) {
 		try {
-			std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename));
+			std::time_t modtime = co_await IAsyncFileSystem::filesystem()->lastWriteTime(filename);
 			if (firstRun) {
 				lastModTime = modtime;
 				firstRun = false;
@@ -68,10 +59,6 @@ ACTOR static Future<Void> watchFileForChanges(std::string filename,
 				throw;
 			}
 		}
-		wait(delay(*intervalSeconds));
+		co_await delay(*intervalSeconds);
 	}
 }
-
-#include "flow/unactorcompiler.h"
-
-#endif // FLOW_WATCH_FILE_ACTOR_H
diff --git a/flow/include/flow/WipedString.h b/flow/include/flow/WipedString.h
index 8f86e55b483..cac5bd29804 100644
--- a/flow/include/flow/WipedString.h
+++ b/flow/include/flow/WipedString.h
@@ -110,8 +110,8 @@ class WipedString {
 		}
 	}
 
-	WipedString(const WipedString& other) noexcept = default;
-	WipedString(WipedString&& other) noexcept = default;
+	explicit(false) WipedString(const WipedString& other) noexcept = default;
+	explicit(false) WipedString(WipedString&& other) noexcept = default;
 	WipedString& operator=(const WipedString& other) noexcept = default;
 	WipedString& operator=(WipedString&& other) noexcept = default;
 
diff --git a/flow/include/flow/actorcompiler.h b/flow/include/flow/actorcompiler.h
index 78596975256..3ceaae2828c 100644
--- a/flow/include/flow/actorcompiler.h
+++ b/flow/include/flow/actorcompiler.h
@@ -47,7 +47,6 @@ class FutureStream;
 #ifndef NO_INTELLISENSE
 #define ACTOR
 #define SWIFT_ACTOR
-#define DESCR
 #define state
 #define UNCANCELLABLE
 #define choose if (1)
diff --git a/flow/include/flow/error_definitions.h b/flow/include/flow/error_definitions.h
index 663f8d1f9a9..0e39a1684c0 100755
--- a/flow/include/flow/error_definitions.h
+++ b/flow/include/flow/error_definitions.h
@@ -87,7 +87,7 @@ ERROR( change_feed_cancelled, 1062, "Change feed was cancelled" )
 // 1064 removed
 // 1065 removed
 ERROR( change_feed_popped, 1066, "Tried to read a version older than what has been popped from the change feed" )
-ERROR( remote_kvs_cancelled, 1067, "The remote key-value store is cancelled" )
+// 1067 removed
 ERROR( page_header_wrong_page_id, 1068, "Page header does not match location on disk" )
 ERROR( page_header_checksum_failed, 1069, "Page header checksum failed" )
 ERROR( page_header_version_not_supported, 1070, "Page header version is not supported" )
@@ -139,7 +139,7 @@ ERROR( please_reboot_kv_store, 1219, "Need to reboot the storage engine")
 ERROR( incompatible_software_version, 1220, "Current software does not support database format" )
 ERROR( audit_storage_failed, 1221, "Validate storage consistency operation failed" )
 ERROR( audit_storage_exceeded_request_limit, 1222, "Exceeded the max number of allowed concurrent audit storage requests" )
-ERROR( proxy_tag_throttled, 1223, "Exceeded maximum proxy tag throttling duration" )
+// 1223 removed
 ERROR( key_value_store_deadline_exceeded, 1224, "Exceeded maximum time allowed to read or write.")
 // 1225 has been removed
 ERROR( audit_storage_error, 1226, "Found data corruption" )
@@ -166,6 +166,8 @@ ERROR( range_lock_reject, 1247, "Range lock is rejected" )
 ERROR( range_unlock_reject, 1248, "Range unlock is rejected" )
 ERROR( bulkload_dataset_not_cover_required_range, 1249, "Bulkload dataset does not cover the required range" )
 ERROR( bulkload_invalid_configuration, 1250, "BulkLoad requires cluster configuration with both shard_encode_location_metadata=1 and enable_read_lock_on_range=1" )
+ERROR( transaction_grv_queue_rejected, 1251, "GRV request rejected because estimated queue wait exceeds transaction limit" )
+ERROR( finish_move_keys_too_many_retries, 1252, "finishMoveKeys exceeded retry limit" )
 
 // 15xx Platform errors
 ERROR( platform_error, 1500, "Platform error" )
@@ -241,7 +243,7 @@ ERROR( key_not_tuple, 2041, "The key cannot be parsed as a tuple" );
 ERROR( value_not_tuple, 2042, "The value cannot be parsed as a tuple" );
 ERROR( mapper_not_tuple, 2043, "The mapper cannot be parsed as a tuple" );
 ERROR( invalid_checkpoint_format, 2044, "Invalid checkpoint format" )
-ERROR( invalid_throttle_quota_value, 2045, "Invalid quota value. Note that reserved_throughput cannot exceed total_throughput" )
+// 2045 removed
 ERROR( failed_to_create_checkpoint, 2046, "Failed to create a checkpoint" )
 ERROR( failed_to_restore_checkpoint, 2047, "Failed to restore a checkpoint" )
 ERROR( failed_to_create_checkpoint_shard_metadata, 2048, "Failed to dump shard metadata for a checkpoint to a sst file" )
@@ -320,7 +322,6 @@ ERROR( backup_does_not_exist, 2319, "Backup does not exist")
 ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges")
 ERROR( backup_not_overlapped_with_keys_filter, 2321, "Backup key ranges doesn't overlap with key ranges filter")
 ERROR( bucket_not_in_url, 2322, "bucket is not in the URL for backup" )
-ERROR( backup_parse_s3_response_failure, 2323, "cannot parse s3 response properly" )
 ERROR( restore_invalid_version, 2361, "Invalid restore version")
 ERROR( restore_corrupted_data, 2362, "Corrupted backup data")
 ERROR( restore_missing_data, 2363, "Missing backup data")
diff --git a/flow/include/flow/flat_buffers.h b/flow/include/flow/flat_buffers.h
index 6484c8389b5..5aab624417f 100644
--- a/flow/include/flow/flat_buffers.h
+++ b/flow/include/flow/flat_buffers.h
@@ -82,10 +82,8 @@ struct struct_like_traits<std::tuple<Ts...>> : std::true_type {
 };
 
 template <class T>
-struct scalar_traits<
-    T,
-    std::enable_if_t<std::is_integral<T>::value || std::is_floating_point<T>::value || std::is_enum<T>::value>>
-  : std::true_type {
+    requires(std::is_integral_v<T> || std::is_floating_point_v<T> || std::is_enum_v<T>)
+struct scalar_traits<T> : std::true_type {
 	constexpr static size_t size = sizeof(T);
 	template <class Context>
 	static void save(uint8_t* out, const T& t, Context&) {
@@ -500,7 +498,7 @@ void swapWithThreadLocalGlobal(std::vector<int>& writeToOffsets);
 
 template <class Context>
 struct PrecomputeSize : Context {
-	PrecomputeSize(const Context& context) : Context(context) {
+	explicit PrecomputeSize(const Context& context) : Context(context) {
 		swapWithThreadLocalGlobal(writeToOffsets);
 		writeToOffsets.clear();
 	}
@@ -510,7 +508,8 @@ struct PrecomputeSize : Context {
 	void write(const void*, int offset, int /*len*/) { current_buffer_size = std::max(current_buffer_size, offset); }
 
 	template <class T>
-	std::enable_if_t<is_dynamic_size<T>, bool> visitDynamicSize(const T& t) {
+	    requires(is_dynamic_size<T>)
+	bool visitDynamicSize(const T& t) {
 		uint32_t size = dynamic_size_traits<T>::size(t, this->context());
 		if (size == 0 && emptyVector.value != -1) {
 			return true;
@@ -609,7 +608,8 @@ struct WriteToBuffer : Context {
 	}
 
 	template <class T>
-	std::enable_if_t<is_dynamic_size<T>, bool> visitDynamicSize(const T& t) {
+	    requires(is_dynamic_size<T>)
+	bool visitDynamicSize(const T& t) {
 		uint32_t size = dynamic_size_traits<T>::size(t, this->context());
 		if (size == 0 && emptyVector.value != -1) {
 			return true;
@@ -700,12 +700,13 @@ struct InsertVTableLambda;
 
 template <class Context>
 struct TraverseMessageTypes : Context {
-	TraverseMessageTypes(InsertVTableLambda<Context>& context) : Context(context), f(context) {}
+	explicit TraverseMessageTypes(InsertVTableLambda<Context>& context) : Context(context), f(context) {}
 
 	InsertVTableLambda<Context>& f;
 
 	template <class Member>
-	std::enable_if_t<expect_serialize_member<Member>> operator()(const Member& member) {
+	    requires(expect_serialize_member<Member>)
+	void operator()(const Member& member) {
 		if constexpr (serializable_traits<Member>::value) {
 			serializable_traits<Member>::serialize(f, const_cast<Member&>(member));
 		} else {
@@ -714,10 +715,12 @@ struct TraverseMessageTypes : Context {
 	};
 
 	template <class T>
-	std::enable_if_t<!expect_serialize_member<T> && !is_vector_like<T> && !is_union_like<T>> operator()(const T&) {}
+	    requires(!expect_serialize_member<T> && !is_vector_like<T> && !is_union_like<T>)
+	void operator()(const T&) {}
 
 	template <class VectorLike>
-	std::enable_if_t<is_vector_like<VectorLike>> operator()(const VectorLike& members) {
+	    requires(is_vector_like<VectorLike>)
+	void operator()(const VectorLike& members) {
 		using VectorTraits = vector_like_traits<VectorLike>;
 		using T = typename VectorTraits::value_type;
 		// we don't need to check for recursion here because the next call
@@ -728,7 +731,8 @@ struct TraverseMessageTypes : Context {
 	}
 
 	template <class UnionLike>
-	std::enable_if_t<is_union_like<UnionLike>> operator()(const UnionLike& members) {
+	    requires(is_union_like<UnionLike>)
+	void operator()(const UnionLike& members) {
 		using UnionTraits = union_like_traits<UnionLike>;
 		static_assert(pack_size(typename UnionTraits::alternatives{}) <= 254,
 		              "Up to 254 alternatives are supported for unions");
@@ -1069,14 +1073,16 @@ void for_each_i(F&& f) {
 
 template <class, class Context>
 struct LoadSaveHelper : Context {
-	LoadSaveHelper(const Context& context) : Context(context) {}
+	explicit LoadSaveHelper(const Context& context) : Context(context) {}
 	template <class U>
-	std::enable_if_t<is_scalar<U>> load(U& member, const uint8_t* current) {
+	    requires(is_scalar<U>)
+	void load(U& member, const uint8_t* current) {
 		scalar_traits<U>::load(current, member, this->context());
 	}
 
 	template <class U>
-	std::enable_if_t<is_struct_like<U>> load(U& member, const uint8_t* current) {
+	    requires(is_struct_like<U>)
+	void load(U& member, const uint8_t* current) {
 		using StructTraits = struct_like_traits<U>;
 		using types = typename StructTraits::types;
 		for_each_i<pack_size(types{})>([&](auto i_type) {
@@ -1089,7 +1095,8 @@ struct LoadSaveHelper : Context {
 	}
 
 	template <class U>
-	std::enable_if_t<is_dynamic_size<U>> load(U& member, const uint8_t* current) {
+	    requires(is_dynamic_size<U>)
+	void load(U& member, const uint8_t* current) {
 		uint32_t current_offset = interpret_as<uint32_t>(current);
 		current += current_offset;
 		uint32_t size = interpret_as<uint32_t>(current);
@@ -1124,7 +1131,8 @@ struct LoadSaveHelper : Context {
 	};
 
 	template <class Member>
-	std::enable_if_t<expect_serialize_member<Member>> load(Member& member, const uint8_t* current) {
+	    requires(expect_serialize_member<Member>)
+	void load(Member& member, const uint8_t* current) {
 		SerializeFun fun(current, this->context());
 		if constexpr (serializable_traits<Member>::value) {
 			serializable_traits<Member>::serialize(fun, member);
@@ -1134,7 +1142,8 @@ struct LoadSaveHelper : Context {
 	}
 
 	template <class VectorLike>
-	std::enable_if_t<is_vector_like<VectorLike>> load(VectorLike& member, const uint8_t* current) {
+	    requires(is_vector_like<VectorLike>)
+	void load(VectorLike& member, const uint8_t* current) {
 		using VectorTraits = vector_like_traits<VectorLike>;
 		using T = typename VectorTraits::value_type;
 		uint32_t current_offset = interpret_as<uint32_t>(current);
@@ -1151,7 +1160,8 @@ struct LoadSaveHelper : Context {
 		}
 	}
 
-	template <class U, class Writer, typename = std::enable_if_t<is_scalar<U>>>
+	template <class U, class Writer>
+	    requires(is_scalar<U>)
 	auto save(const U& message, Writer& writer, const VTableSet*) {
 		constexpr auto size = scalar_traits<U>::size;
 		std::array<uint8_t, size> result = {};
@@ -1162,10 +1172,8 @@ struct LoadSaveHelper : Context {
 	}
 
 	template <class U, class Writer>
-	auto save(const U& message,
-	          Writer& writer,
-	          const VTableSet* vtables,
-	          std::enable_if_t<is_struct_like<U>, int> _ = 0) {
+	    requires(is_struct_like<U>)
+	auto save(const U& message, Writer& writer, const VTableSet* vtables) {
 		using StructTraits = struct_like_traits<U>;
 		using types = typename StructTraits::types;
 		constexpr auto size = struct_size(types{});
@@ -1179,21 +1187,17 @@ struct LoadSaveHelper : Context {
 		return struct_bytes;
 	}
 
-	template <class U, class Writer, typename = std::enable_if_t<is_dynamic_size<U>>>
-	RelativeOffset save(const U& message,
-	                    Writer& writer,
-	                    const VTableSet*,
-	                    std::enable_if_t<is_dynamic_size<U>, int> _ = 0) {
+	template <class U, class Writer>
+	    requires(is_dynamic_size<U>)
+	RelativeOffset save(const U& message, Writer& writer, const VTableSet*) {
 		if (writer.visitDynamicSize(message))
 			return writer.emptyVector;
 		return RelativeOffset{ writer.current_buffer_size };
 	}
 
 	template <class Member, class Writer>
-	RelativeOffset save(const Member& member,
-	                    Writer& writer,
-	                    const VTableSet* vtables,
-	                    std::enable_if_t<expect_serialize_member<Member>, int> _ = 0) {
+	    requires(expect_serialize_member<Member>)
+	RelativeOffset save(const Member& member, Writer& writer, const VTableSet* vtables) {
 		SaveVisitorLambda<Writer, Context> l{ *this, vtables, writer };
 		if constexpr (serializable_traits<Member>::value) {
 			serializable_traits<Member>::serialize(l, const_cast<Member&>(member));
@@ -1203,7 +1207,8 @@ struct LoadSaveHelper : Context {
 		return RelativeOffset{ writer.current_buffer_size };
 	}
 
-	template <class VectorLike, class Writer, typename = std::enable_if_t<is_vector_like<VectorLike>>>
+	template <class VectorLike, class Writer>
+	    requires(is_vector_like<VectorLike>)
 	RelativeOffset save(const VectorLike& members, Writer& writer, const VTableSet* vtables) {
 		using VectorTraits = vector_like_traits<VectorLike>;
 		using T = typename VectorTraits::value_type;
@@ -1236,7 +1241,7 @@ struct LoadSaveHelper : Context {
 template <class Alloc, class Context>
 struct LoadSaveHelper<std::vector<bool, Alloc>, Context> : Context {
 
-	LoadSaveHelper(const Context& context) : Context(context) {}
+	explicit LoadSaveHelper(const Context& context) : Context(context) {}
 
 	void load(std::vector<bool, Alloc>& member, const uint8_t* current) {
 		uint32_t current_offset = interpret_as<uint32_t>(current);
@@ -1288,7 +1293,7 @@ template <class... Members>
 struct FakeRoot {
 	std::tuple<Members&...> members;
 
-	FakeRoot(Members&... members) : members(members...) {}
+	explicit FakeRoot(Members&... members) : members(members...) {}
 
 	template <class Archive>
 	void serialize(Archive& archive) {
@@ -1368,7 +1373,7 @@ template <class T>
 struct EnsureTable
   : std::conditional_t<HasFileIdentifier<T>::value, detail::YesFileIdentifier<T>, detail::NoFileIdentifier> {
 	EnsureTable() : t() {}
-	EnsureTable(const T& t) : t(t) {}
+	explicit EnsureTable(const T& t) : t(t) {}
 	template <class Archive>
 	void serialize(Archive& ar) {
 		if constexpr (is_fb_function<Archive>) {
@@ -1398,7 +1403,7 @@ namespace detail {
 // Ensure if there's a LoadSaveHelper specialization available for T it gets used.
 template <class T, class Context>
 struct LoadSaveHelper<EnsureTable<T>, Context> : Context {
-	LoadSaveHelper(const Context& context) : Context(context), alreadyATable(context), wrapInTable(context) {}
+	explicit LoadSaveHelper(const Context& context) : Context(context), alreadyATable(context), wrapInTable(context) {}
 
 	void load(EnsureTable<T>& member, const uint8_t* current) {
 		if constexpr (expect_serialize_member<T>) {
diff --git a/flow/include/flow/flow.h b/flow/include/flow/flow.h
index f0c083a66a1..a52a1e035c0 100644
--- a/flow/include/flow/flow.h
+++ b/flow/include/flow/flow.h
@@ -47,7 +47,6 @@
 #include "flow/network.h"
 #include "flow/serialize.h"
 
-
 #ifdef WITH_SWIFT
 #include <swift/bridging>
 
@@ -68,7 +67,6 @@
 		static_assert(false, "TEST macros are deprecated, please use CODE_PROBE instead");                             \
 	} while (false)
 
-
 extern Optional<uint64_t> parse_with_suffix(std::string const& toparse, std::string const& default_unit = "");
 extern Optional<uint64_t> parseDuration(std::string const& str, std::string const& defaultUnit = "");
 extern std::string format(const char* form, ...);
@@ -128,10 +126,10 @@ class ErrorOr : public ComposedIdentifier<T, 2> {
 	using ValueType = T;
 
 	ErrorOr() : ErrorOr(default_error_or()) {}
-	ErrorOr(Error const& error) : value(std::in_place_type<Error>, error) {}
+	explicit(false) ErrorOr(Error const& error) : value(std::in_place_type<Error>, error) {}
 
 	template <class U>
-	ErrorOr(U const& t) : value(std::in_place_type<T>, t) {}
+	explicit(false) ErrorOr(U const& t) : value(std::in_place_type<T>, t) {}
 
 	ErrorOr(Arena& a, ErrorOr<T> const& o) {
 		if (o.present()) {
@@ -151,18 +149,16 @@ class ErrorOr : public ComposedIdentifier<T, 2> {
 	template <class F>
 	using MapRet = std::decay_t<std::invoke_result_t<F, T>>;
 
-	template <class F>
-	using EnableIfNotMemberPointer =
-	    std::enable_if_t<!std::is_member_object_pointer_v<F> && !std::is_member_function_pointer_v<F>>;
-
 public:
 	// If the ErrorOr is set, calls the function f on the value and returns the value. Otherwise, returns an ErrorOr
 	// with the same error value as this ErrorOr.
-	template <class F, typename = EnableIfNotMemberPointer<F>>
+	template <class F>
+	    requires(!std::is_member_object_pointer_v<F> && !std::is_member_function_pointer_v<F>)
 	ErrorOr<MapRet<F>> map(const F& f) const& {
 		return present() ? ErrorOr<MapRet<F>>(f(get())) : ErrorOr<MapRet<F>>(getError());
 	}
-	template <class F, typename = EnableIfNotMemberPointer<F>>
+	template <class F>
+	    requires(!std::is_member_object_pointer_v<F> && !std::is_member_function_pointer_v<F>)
 	ErrorOr<MapRet<F>> map(const F& f) && {
 		return present() ? ErrorOr<MapRet<F>>(f(std::move(*this).get())) : ErrorOr<MapRet<F>>(getError());
 	}
@@ -171,13 +167,13 @@ class ErrorOr : public ComposedIdentifier<T, 2> {
 	//
 	// v.map(&T::member) is equivalent to v.map<R>([](T t) { return t.member; })
 	template <class R, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, ErrorOr<Rp>> map(
-	    R std::conditional_t<std::is_class_v<T>, T, Void>::*member) const& {
+	    requires(std::is_class_v<T>)
+	ErrorOr<Rp> map(R std::conditional_t<std::is_class_v<T>, T, Void>::*member) const& {
 		return present() ? ErrorOr<Rp>(get().*member) : ErrorOr<Rp>(getError());
 	}
 	template <class R, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, ErrorOr<Rp>> map(
-	    R std::conditional_t<std::is_class_v<T>, T, Void>::*member) && {
+	    requires(std::is_class_v<T>)
+	ErrorOr<Rp> map(R std::conditional_t<std::is_class_v<T>, T, Void>::*member) && {
 		return present() ? ErrorOr<Rp>(std::move(*this).get().*member) : ErrorOr<Rp>(getError());
 	}
 
@@ -186,15 +182,15 @@ class ErrorOr : public ComposedIdentifier<T, 2> {
 	// v.map(&T::memberFunc, arg1, arg2, ...) is equivalent to
 	// v.map<R>([](T t) { return t.memberFunc(arg1, arg2, ...); })
 	template <class R, class... Args, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, ErrorOr<Rp>> map(
-	    R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
-	    Args&&... args) const& {
+	    requires(std::is_class_v<T>)
+	ErrorOr<Rp> map(R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
+	                Args&&... args) const& {
 		return present() ? ErrorOr<Rp>((get().*memberFunc)(std::forward<Args>(args)...)) : ErrorOr<Rp>(getError());
 	}
 	template <class R, class... Args, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T>, ErrorOr<Rp>> map(
-	    R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
-	    Args&&... args) && {
+	    requires(std::is_class_v<T>)
+	ErrorOr<Rp> map(R (std::conditional_t<std::is_class_v<T>, T, Void>::*memberFunc)(Args...) const,
+	                Args&&... args) && {
 		return present() ? ErrorOr<Rp>((std::move(*this).get().*memberFunc)(std::forward<Args>(args)...))
 		                 : ErrorOr<Rp>(getError());
 	}
@@ -205,7 +201,8 @@ class ErrorOr : public ComposedIdentifier<T, 2> {
 	//
 	// v.mapRef(&P::member) is equivalent to ErrorOr<R>(v.get()->member) if v is present and non-null
 	template <class P, class R, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T> || std::is_pointer_v<T>, ErrorOr<Rp>> mapRef(R P::*member) const& {
+	    requires(std::is_class_v<T> || std::is_pointer_v<T>)
+	ErrorOr<Rp> mapRef(R P::*member) const& {
 
 		if (!present()) {
 			return ErrorOr<Rp>(getError());
@@ -224,8 +221,8 @@ class ErrorOr : public ComposedIdentifier<T, 2> {
 	// v.map(&T::memberFunc, arg1, arg2, ...) is equivalent to ErrorOr<R>(v.get()->memberFunc(arg1, arg2, ...)) if v is
 	// present and non-null
 	template <class P, class R, class... Args, class Rp = std::decay_t<R>>
-	std::enable_if_t<std::is_class_v<T> || std::is_pointer_v<T>, ErrorOr<Rp>> mapRef(R (P::*memberFunc)(Args...) const,
-	                                                                                 Args&&... args) const& {
+	    requires(std::is_class_v<T> || std::is_pointer_v<T>)
+	ErrorOr<Rp> mapRef(R (P::*memberFunc)(Args...) const, Args&&... args) const& {
 		if (!present()) {
 			return ErrorOr<Rp>(getError());
 		} else if (!get()) {
@@ -436,7 +433,7 @@ namespace detail {
 
 template <class T, class Context>
 struct LoadSaveHelper<CachedSerialization<T>, Context> : Context {
-	LoadSaveHelper(const Context& context) : Context(context), helper(context) {}
+	explicit LoadSaveHelper(const Context& context) : Context(context), helper(context) {}
 
 	void load(CachedSerialization<T>& member, const uint8_t* current) { helper.load(member.mutate(), current); }
 
@@ -659,9 +656,11 @@ class LineageReference : public Reference<ActorLineage> {
 public:
 	LineageReference() : Reference<ActorLineage>(nullptr), actorName_(""), allocated_(false) {}
 	explicit LineageReference(ActorLineage* ptr) : Reference<ActorLineage>(ptr), actorName_(""), allocated_(false) {}
-	LineageReference(const LineageReference& r) : Reference<ActorLineage>(r), actorName_(""), allocated_(false) {}
-	LineageReference(LineageReference&& r)
-	  : Reference<ActorLineage>(std::forward<LineageReference>(r)), actorName_(r.actorName_), allocated_(r.allocated_) {
+	explicit(false) LineageReference(const LineageReference& r)
+	  : Reference<ActorLineage>(r), actorName_(""), allocated_(false) {}
+	explicit(false) LineageReference(LineageReference&& r)
+	  : Reference<ActorLineage>(r.getPtr()), actorName_(r.actorName_), allocated_(r.allocated_) {
+		r.setPtrUnsafe(nullptr);
 		r.actorName_ = "";
 		r.allocated_ = false;
 	}
@@ -702,7 +701,7 @@ struct StackLineage : LineageProperties<StackLineage> {
 #ifdef ENABLE_SAMPLING
 struct LineageScope {
 	LineageReference* oldLineage;
-	LineageScope(LineageReference* with) : oldLineage(currentLineage) { replaceLineage(with); }
+	explicit LineageScope(LineageReference* with) : oldLineage(currentLineage) { replaceLineage(with); }
 	~LineageScope() { replaceLineage(oldLineage); }
 };
 #endif
@@ -919,12 +918,12 @@ class Promise;
 #ifndef SWIFT_HIDE_CHECKED_CONTINUTATION
 using flow_swift::FlowCheckedContinuation;
 
-template<class T>
+template <class T>
 class
 #ifdef WITH_SWIFT
-SWIFT_CONFORMS_TO_PROTOCOL(flow_swift.FlowCallbackForSwiftContinuationT)
+    SWIFT_CONFORMS_TO_PROTOCOL(flow_swift.FlowCallbackForSwiftContinuationT)
 #endif
-FlowCallbackForSwiftContinuation : Callback<T> {
+        FlowCallbackForSwiftContinuation : Callback<T> {
 public:
 	using SwiftCC = flow_swift::FlowCheckedContinuation<T>;
 	using AssociatedFuture = Future<T>;
@@ -1011,7 +1010,8 @@ SWIFT_CONFORMS_TO_PROTOCOL(flow_swift.FlowFutureOps)
 
 #ifndef NO_INTELLISENSE
 	template <class U>
-	Future(const U&, typename std::enable_if<std::is_assignable<T, U>::value, int*>::type = 0) {}
+	    requires(std::is_assignable_v<T, U>)
+	Future(const U&) {}
 #endif
 
 	~Future() {
@@ -1158,7 +1158,8 @@ class SWIFT_SENDABLE Promise final {
 template <class T>
 struct NotifiedQueue : private SingleCallback<T>
 #ifndef WITH_SWIFT
-   , FastAllocated<NotifiedQueue<T>> // FIXME(swift): Swift can't deal with this type yet
+  ,
+                       FastAllocated<NotifiedQueue<T>> // FIXME(swift): Swift can't deal with this type yet
 #endif /* WITH_SWIFT */
 {
 	int promises; // one for each promise (and one for an active actor if this is an actor)
@@ -1480,16 +1481,30 @@ static inline void destruct(T& t) {
 	t.~T();
 }
 
+// These stay as int8_t constants rather than an enum class because actor_wait_state also stores generated callback
+// group numbers, so the code frequently mixes named sentinel states with arbitrary positive integer values.
+constexpr int8_t ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK = -2;
+constexpr int8_t ACTOR_WAIT_STATE_CANCELLED = -1;
+constexpr int8_t ACTOR_WAIT_STATE_NOT_WAITING = 0;
+constexpr int8_t ACTOR_WAIT_STATE_WAITING = 1;
+
+constexpr bool actorWaitStateIsCancelled(int8_t waitState) {
+	return waitState < ACTOR_WAIT_STATE_NOT_WAITING;
+}
+
+constexpr bool actorWaitStateIsWaiting(int8_t waitState) {
+	return waitState > ACTOR_WAIT_STATE_NOT_WAITING;
+}
+
 template <class ReturnValue>
 struct Actor : SAV<ReturnValue> {
 #ifdef ENABLE_SAMPLING
 	LineageReference lineage = *currentLineage;
 #endif
-	int8_t actor_wait_state; // -1 means actor is cancelled; 0 means actor is not waiting; 1-N mean waiting in callback
-	                         // group #
+	int8_t actor_wait_state; // Negative values mean cancellation, 0 means not waiting, positive values identify the
+	                         // waiting callback group.
 
-	Actor() : SAV<ReturnValue>(1, 1), actor_wait_state(0) { /*++actorCount;*/
-	}
+	Actor() : SAV<ReturnValue>(1, 1), actor_wait_state(ACTOR_WAIT_STATE_NOT_WAITING) { /*++actorCount;*/ }
 	// ~Actor() { --actorCount; }
 
 #ifdef ENABLE_SAMPLING
@@ -1504,10 +1519,9 @@ struct Actor<void> {
 #ifdef ENABLE_SAMPLING
 	LineageReference lineage = *currentLineage;
 #endif
-	int8_t actor_wait_state; // 0 means actor is not waiting; 1-N mean waiting in callback group #
+	int8_t actor_wait_state; // 0 means not waiting, positive values identify the waiting callback group.
 
-	Actor() : actor_wait_state(0) { /*++actorCount;*/
-	}
+	Actor() : actor_wait_state(ACTOR_WAIT_STATE_NOT_WAITING) { /*++actorCount;*/ }
 	// ~Actor() { --actorCount; }
 
 #ifdef ENABLE_SAMPLING
diff --git a/flow/include/flow/genericactors.actor.h b/flow/include/flow/genericactors.actor.h
index 9cef29962e6..ff3b3f12a23 100644
--- a/flow/include/flow/genericactors.actor.h
+++ b/flow/include/flow/genericactors.actor.h
@@ -38,6 +38,8 @@
 #define GENERICACTORS_ACTOR_H
 
 #include <list>
+#include <optional>
+#include <type_traits>
 #include <utility>
 
 #include "flow/flow.h"
@@ -127,6 +129,15 @@ ErrorOr<T> errorOr(T t) {
 	return ErrorOr<T>(t);
 }
 
+template <class T>
+AsyncResult<ErrorOr<T>> errorOr(AsyncResult<T> result) {
+	try {
+		co_return ErrorOr<T>(co_await std::move(result));
+	} catch (Error& e) {
+		co_return ErrorOr<T>(e);
+	}
+}
+
 ACTOR template <class T>
 Future<ErrorOr<T>> errorOr(Future<T> f) {
 	try {
@@ -221,8 +232,8 @@ Future<T> timeout(Future<T> what, double time, T timedoutValue, TaskPriority tas
 }
 
 ACTOR template <class T>
-Future<Optional<T>> timeout(Future<T> what, double time) {
-	Future<Void> end = delay(time);
+Future<Optional<T>> timeout(Future<T> what, double time, TaskPriority taskID = TaskPriority::DefaultDelay) {
+	Future<Void> end = delay(time, taskID);
 	choose {
 		when(T t = wait(what)) {
 			return t;
@@ -246,6 +257,24 @@ Future<T> timeoutError(Future<T> what, double time, TaskPriority taskID = TaskPr
 	}
 }
 
+template <class T>
+AsyncResult<T> timeoutError(AsyncResult<T> what,
+                            double time,
+                            TaskPriority taskID = TaskPriority::DefaultDelay,
+                            ExplicitVoid = {}) {
+	if (what.canGet()) {
+		co_return what.get();
+	} else if (what.isError()) {
+		throw what.getError();
+	}
+	auto res = co_await race(std::move(what), delay(time, taskID));
+	if (res.index() == 0) {
+		co_return std::get<0>(std::move(res));
+	} else {
+		throw timed_out();
+	}
+}
+
 ACTOR template <class T>
 Future<T> delayed(Future<T> what, double time = 0.0, TaskPriority taskID = TaskPriority::DefaultDelay) {
 	try {
@@ -266,17 +295,6 @@ Future<Void> trigger(Func what, Future<Void> signal) {
 	return Void();
 }
 
-ACTOR template <class Func>
-Future<Void> triggerOnError(Func what, Future<Void> signal) {
-	try {
-		wait(signal);
-	} catch (Error& e) {
-		what();
-	}
-
-	return Void();
-}
-
 // Waits for a future to complete and cannot be cancelled
 // Most situations will use the overload below, which does not require a promise
 ACTOR template <class T>
@@ -312,12 +330,6 @@ Future<T> holdWhile(X object, Future<T> what) {
 	return val;
 }
 
-ACTOR template <class T, class X>
-Future<Void> holdWhileVoid(X object, Future<T> what) {
-	T val = wait(what);
-	return Void();
-}
-
 // Assign the future value of what to out
 template <class T, class X>
 [[nodiscard]] Future<Void> store(X& out, Future<T> what) {
@@ -327,19 +339,6 @@ template <class T, class X>
 	});
 }
 
-#if false
-// NOTE: Think twice whether create a new struct for a complex return type is better before using tuple.
-// If we just use the return type once, is it worth to create a new struct?
-// And enable the unit test in genericactors.actor.cpp
-template <class A, class... Bs>
-Future<Void> storeTuple(Future<std::tuple<A, Bs...>> what, A& a, Bs&... b) {
-	return map(what, [&](std::tuple<A, Bs...> const& v) {
-		std::tie(a, b...) = v;
-		return Void();
-	});
-}
-#endif
-
 template <class T>
 Future<Void> storeOrThrow(T& out, Future<Optional<T>> what, Error e = key_not_found()) {
 	return map(what, [&out, e](Optional<T> const& o) {
@@ -465,67 +464,6 @@ Future<Void> returnIfTrue(Future<T> what, F pred) {
 	return returnIfTrue(map(what, pred));
 }
 
-// filters a stream
-ACTOR template <class T, class F>
-Future<Void> filter(FutureStream<T> input, F pred, PromiseStream<T> output) {
-	loop {
-		try {
-			T nextInput = waitNext(input);
-			if (pred(nextInput))
-				output.send(nextInput);
-		} catch (Error& e) {
-			if (e.code() == error_code_end_of_stream) {
-				break;
-			} else
-				throw;
-		}
-	}
-
-	output.sendError(end_of_stream());
-
-	return Void();
-}
-
-// filters a stream asynchronously
-ACTOR template <class T, class F>
-Future<Void> asyncFilter(FutureStream<T> input, F actorPred, PromiseStream<T> output) {
-	state Deque<std::pair<T, Future<bool>>> futures;
-	state std::pair<T, Future<bool>> p;
-
-	loop {
-		try {
-			choose {
-				when(T nextInput = waitNext(input)) {
-					futures.emplace_back(nextInput, actorPred(nextInput));
-				}
-				when(bool pass = wait(futures.size() == 0 ? Never() : futures.front().second)) {
-					if (pass)
-						output.send(futures.front().first);
-					futures.pop_front();
-				}
-			}
-		} catch (Error& e) {
-			if (e.code() == error_code_end_of_stream) {
-				break;
-			} else {
-				throw e;
-			}
-		}
-	}
-
-	while (futures.size()) {
-		p = futures.front();
-		bool pass = wait(p.second);
-		if (pass)
-			output.send(p.first);
-		futures.pop_front();
-	}
-
-	output.sendError(end_of_stream());
-
-	return Void();
-}
-
 template <class T>
 struct WorkerCache {
 	// SOMEDAY: Would we do better to use "unreliable" (at most once) transport for the initialize requests and get rid
@@ -718,9 +656,9 @@ template <class V>
 class ReferencedObject : NonCopyable, public ReferenceCounted<ReferencedObject<V>> {
 public:
 	ReferencedObject() : value() {}
-	ReferencedObject(V const& v) : value(v) {}
-	ReferencedObject(V&& v) : value(std::move(v)) {}
-	ReferencedObject(ReferencedObject&& r) : value(std::move(r.value)) {}
+	explicit ReferencedObject(V const& v) : value(v) {}
+	explicit ReferencedObject(V&& v) : value(std::move(v)) {}
+	explicit(false) ReferencedObject(ReferencedObject&& r) : value(std::move(r.value)) {}
 
 	void operator=(ReferencedObject&& r) { value = std::move(r.value); }
 
@@ -787,7 +725,7 @@ class SWIFT_CXX_REF_ASYNCVAR AsyncVar : NonCopyable, public ReferenceCounted<Asy
 class AsyncTrigger : NonCopyable {
 public:
 	AsyncTrigger() {}
-	AsyncTrigger(AsyncTrigger&& at) : v(std::move(at.v)) {}
+	explicit(false) AsyncTrigger(AsyncTrigger&& at) : v(std::move(at.v)) {}
 	void operator=(AsyncTrigger&& at) { v = std::move(at.v); }
 	Future<Void> onTrigger() const { return v.onChange(); }
 	void trigger() { v.trigger(); }
@@ -809,7 +747,7 @@ Future<Void> forward(Reference<AsyncVar<T> const> from, AsyncTrigger* to) {
 class Debouncer : NonCopyable {
 public:
 	explicit Debouncer(double delay) { worker = debounceWorker(this, delay); }
-	Debouncer(Debouncer&& at) = default;
+	explicit(false) Debouncer(Debouncer&& at) = default;
 	Debouncer& operator=(Debouncer&& at) = default;
 	Future<Void> onTrigger() { return output.onChange(); }
 	void trigger() { input.setUnconditional(Void()); }
@@ -850,14 +788,6 @@ Future<Void> asyncDeserialize(Reference<AsyncVar<Standalone<StringRef>>> input,
 	}
 }
 
-ACTOR template <class V, class T>
-void forwardVector(Future<V> values, std::vector<Promise<T>> out) {
-	V in = wait(values);
-	ASSERT(in.size() == out.size());
-	for (int i = 0; i < out.size(); i++)
-		out[i].send(in[i]);
-}
-
 ACTOR template <class T>
 Future<Void> delayedAsyncVar(Reference<AsyncVar<T>> in, Reference<AsyncVar<T>> out, double time) {
 	try {
@@ -943,24 +873,6 @@ ACTOR Future<Void> delayAfterCleared(Reference<AsyncVar<bool>> condition,
 // Same as delayAfterCleared, but use lowPriorityDelay.
 ACTOR Future<Void> lowPriorityDelayAfterCleared(Reference<AsyncVar<bool>> condition, double time);
 
-// Similar to timeoutError, but does not throw timed_out if condition is true.
-// Once condition becomes false again, reset the timer (e.g. if time is 10s, wait for 10s again before throwing
-// timed_out, if condition remains to be false).
-ACTOR template <class T>
-Future<T> timeoutErrorIfCleared(Future<T> what,
-                                Reference<AsyncVar<bool>> condition,
-                                double time,
-                                TaskPriority taskID = TaskPriority::DefaultDelay) {
-	choose {
-		when(T t = wait(what)) {
-			return t;
-		}
-		when(wait(delayAfterCleared(condition, time, taskID))) {
-			throw timed_out();
-		}
-	}
-}
-
 Future<bool> allTrue(const std::vector<Future<bool>>& all);
 Future<Void> anyTrue(std::vector<Reference<AsyncVar<bool>>> const& input, Reference<AsyncVar<bool>> const& output);
 Future<Void> cancelOnly(std::vector<Future<Void>> const& futures);
@@ -995,32 +907,34 @@ Future<Void> makeStream(const std::vector<Future<T>>& futures, PromiseStream<T>&
 template <class T>
 class QuorumCallback;
 
-template <class T>
-struct Quorum final : SAV<Void> {
+// SAV-backed quorum bookkeeping for Future callbacks. AsyncResult quorum uses
+// a separate state type because it owns producer cancellation.
+template <class CallbackType>
+struct QuorumState final : SAV<Void> {
 	int antiQuorum;
 	int count;
 
-	static inline int sizeFor(int count) { return sizeof(Quorum<T>) + sizeof(QuorumCallback<T>) * count; }
+	static inline int sizeFor(int count) { return sizeof(QuorumState<CallbackType>) + sizeof(CallbackType) * count; }
 
 	void destroy() override {
 		int size = sizeFor(this->count);
-		this->~Quorum();
+		this->~QuorumState();
 		freeFast(size, this);
 	}
 	void cancel() override {
-		int cancelled_callbacks = 0;
-		for (int i = 0; i < count; i++)
-			if (callbacks()[i].next) {
-				callbacks()[i].remove();
-				callbacks()[i].next = 0;
-				++cancelled_callbacks;
+		int cancelledCallbacks = 0;
+		for (int i = 0; i < count; i++) {
+			if (callbacks()[i].isRegistered()) {
+				callbacks()[i].detach();
+				++cancelledCallbacks;
 			}
+		}
 		if (canBeSet())
 			sendError(actor_cancelled());
-		for (int i = 0; i < cancelled_callbacks; i++)
+		for (int i = 0; i < cancelledCallbacks; i++)
 			delPromiseRef();
 	}
-	explicit Quorum(int quorum, int count) : SAV<Void>(1, count), antiQuorum(count - quorum + 1), count(count) {
+	explicit QuorumState(int quorum, int count) : SAV<Void>(1, count), antiQuorum(count - quorum + 1), count(count) {
 		if (!quorum)
 			this->send(Void());
 	}
@@ -1037,28 +951,40 @@ struct Quorum final : SAV<Void> {
 			delPromiseRef();
 	}
 
-	QuorumCallback<T>* callbacks() { return (QuorumCallback<T>*)(this + 1); }
+	CallbackType* callbacks() { return (CallbackType*)(this + 1); }
 };
 
+template <class T>
+using Quorum = QuorumState<QuorumCallback<T>>;
+
 template <class T>
 class QuorumCallback : public Callback<T> {
 public:
 	void fire(const T& value) override {
-		Callback<T>::remove();
-		Callback<T>::next = 0;
+		detach();
 		head->oneSuccess();
 	}
 	void error(Error error) override {
-		Callback<T>::remove();
-		Callback<T>::next = 0;
+		detach();
 		head->oneError(error);
 	}
+	bool isRegistered() const { return Callback<T>::next != nullptr; }
+	void detach() {
+		if (isRegistered()) {
+			Callback<T>::remove();
+			Callback<T>::prev = nullptr;
+			Callback<T>::next = nullptr;
+		}
+	}
 
 private:
 	template <class U>
 	friend Future<Void> quorum(const Future<U>* pItems, int itemCount, int n);
 	Quorum<T>* head;
-	QuorumCallback() = default;
+	QuorumCallback() {
+		Callback<T>::prev = nullptr;
+		Callback<T>::next = nullptr;
+	}
 	QuorumCallback(Future<T> future, Quorum<T>* head) : head(head) { future.addCallbackAndClear(this); }
 };
 
@@ -1074,7 +1000,6 @@ Future<Void> quorum(const Future<T>* pItems, int itemCount, int n) {
 		auto& r = pItems[i];
 		if (r.isReady()) {
 			new (nextCallback) QuorumCallback<T>();
-			nextCallback->next = 0;
 			if (r.isError())
 				q->oneError(r.getError());
 			else
@@ -1091,6 +1016,304 @@ Future<Void> quorum(std::vector<Future<T>> const& results, int n) {
 	return quorum(&results.front(), results.size(), n);
 }
 
+namespace coro {
+template <class T>
+struct QuorumAsyncResultCallback;
+
+template <class T>
+struct GetAllAsyncResultCallback;
+} // namespace coro
+
+template <class T>
+struct QuorumAsyncResult final : SAV<Void> {
+	int antiQuorum;
+	int count;
+
+	static inline int sizeFor(int count) {
+		return sizeof(QuorumAsyncResult<T>) + sizeof(coro::QuorumAsyncResultCallback<T>) * count;
+	}
+
+	int detachPendingCallbacks() {
+		int detachedCallbacks = 0;
+		for (int i = 0; i < count; ++i) {
+			if (callbacks()[i].isRegistered()) {
+				callbacks()[i].detach();
+				++detachedCallbacks;
+			}
+		}
+		return detachedCallbacks;
+	}
+
+	void destroy() override {
+		int size = sizeFor(this->count);
+		this->~QuorumAsyncResult();
+		freeFast(size, this);
+	}
+	void cancel() override {
+		int cancelledCallbacks = detachPendingCallbacks();
+		if (canBeSet())
+			sendError(actor_cancelled());
+		for (int i = 0; i < cancelledCallbacks; ++i)
+			delPromiseRef();
+	}
+	explicit QuorumAsyncResult(int quorum, int count)
+	  : SAV<Void>(1, count), antiQuorum(count - quorum + 1), count(count) {
+		if (!quorum)
+			this->send(Void());
+	}
+	void oneSuccess() {
+		if (getPromiseReferenceCount() == antiQuorum && canBeSet()) {
+			int cancelledCallbacks = detachPendingCallbacks();
+			this->sendAndDelPromiseRef(Void());
+			for (int i = 0; i < cancelledCallbacks; ++i)
+				delPromiseRef();
+		} else {
+			delPromiseRef();
+		}
+	}
+	void oneError(Error err) {
+		if (canBeSet()) {
+			int cancelledCallbacks = detachPendingCallbacks();
+			this->sendErrorAndDelPromiseRef(err);
+			for (int i = 0; i < cancelledCallbacks; ++i)
+				delPromiseRef();
+		} else {
+			delPromiseRef();
+		}
+	}
+
+	coro::QuorumAsyncResultCallback<T>* callbacks() { return (coro::QuorumAsyncResultCallback<T>*)(this + 1); }
+};
+
+namespace coro {
+template <class T>
+// AsyncResult has a single callback slot in its state, so quorum needs a
+// callback that can unregister directly from AsyncResultState and cancel the
+// producer if the aggregate no longer needs that result.
+struct QuorumAsyncResultCallback final : AsyncResultCallback<typename AsyncResult<T>::StoredT> {
+	using StoredT = typename AsyncResult<T>::StoredT;
+
+	AsyncResultState<StoredT>* resultState = nullptr;
+	QuorumAsyncResult<T>* head = nullptr;
+
+	QuorumAsyncResultCallback() = default;
+	QuorumAsyncResultCallback(AsyncResult<T>& result, QuorumAsyncResult<T>* head);
+
+	void fire(StoredT const&) override;
+	void fire(StoredT&&) override;
+	void error(Error error) override;
+	bool isRegistered() const { return resultState != nullptr; }
+	void detach();
+};
+} // namespace coro
+
+namespace coro {
+template <class StoredT>
+void detachAsyncResultStateCallback(AsyncResultState<StoredT>*& resultState, AsyncResultCallback<StoredT>* callback) {
+	if (resultState) {
+		auto* s = resultState;
+		resultState = nullptr;
+		s->clearCallback(callback);
+		if (!s->isReady()) {
+			s->cancelProducer();
+		}
+		s->delRef();
+	}
+}
+
+template <class T>
+QuorumAsyncResultCallback<T>::QuorumAsyncResultCallback(AsyncResult<T>& result, QuorumAsyncResult<T>* head)
+  : resultState(result.resultState), head(head) {
+	std::move(result).addCallbackAndClear(this);
+}
+
+template <class T>
+void QuorumAsyncResultCallback<T>::detach() {
+	detachAsyncResultStateCallback(resultState, this);
+}
+
+template <class T>
+void QuorumAsyncResultCallback<T>::fire(StoredT const&) {
+	detach();
+	head->oneSuccess();
+}
+
+template <class T>
+void QuorumAsyncResultCallback<T>::fire(StoredT&&) {
+	detach();
+	head->oneSuccess();
+}
+
+template <class T>
+void QuorumAsyncResultCallback<T>::error(Error error) {
+	detach();
+	head->oneError(error);
+}
+} // namespace coro
+
+template <class T>
+Future<Void> quorum(AsyncResult<T>* pItems, int itemCount, int n) {
+	ASSERT(n >= 0 && n <= itemCount);
+
+	int size = QuorumAsyncResult<T>::sizeFor(itemCount);
+	QuorumAsyncResult<T>* q = new (allocateFast(size)) QuorumAsyncResult<T>(n, itemCount);
+
+	coro::QuorumAsyncResultCallback<T>* nextCallback = q->callbacks();
+	for (int i = 0; i < itemCount; ++i) {
+		auto& r = pItems[i];
+		if (r.isReady()) {
+			new (nextCallback) coro::QuorumAsyncResultCallback<T>();
+			if (r.isError())
+				q->oneError(r.getError());
+			else
+				q->oneSuccess();
+			r = AsyncResult<T>();
+		} else
+			new (nextCallback) coro::QuorumAsyncResultCallback<T>(r, q);
+		++nextCallback;
+	}
+	return Future<Void>(q);
+}
+
+// AsyncResult is single-consumer, so quorum requires an explicit ownership
+// transfer from vector callers.
+template <class T>
+Future<Void> quorum(std::vector<AsyncResult<T>>& results, int n) = delete;
+
+template <class T>
+Future<Void> quorum(std::vector<AsyncResult<T>>&& results, int n) {
+	return quorum(results.data(), results.size(), n);
+}
+
+template <class T>
+// Collect AsyncResult values without first wrapping them in Future. Each slot
+// is optional so move-only payloads can be transferred exactly once into the
+// final output vector when the last callback fires.
+struct GetAllAsyncResult final : SAV<std::vector<T>> {
+	int remaining;
+	int count;
+	std::vector<std::optional<T>> values;
+
+	static inline int sizeFor(int count) {
+		return sizeof(GetAllAsyncResult<T>) + sizeof(coro::GetAllAsyncResultCallback<T>) * count;
+	}
+
+	int detachPendingCallbacks() {
+		int detachedCallbacks = 0;
+		for (int i = 0; i < count; ++i) {
+			if (callbacks()[i].isRegistered()) {
+				callbacks()[i].detach();
+				++detachedCallbacks;
+			}
+		}
+		return detachedCallbacks;
+	}
+
+	void destroy() override {
+		int size = sizeFor(this->count);
+		this->~GetAllAsyncResult();
+		freeFast(size, this);
+	}
+	void cancel() override {
+		int cancelledCallbacks = detachPendingCallbacks();
+		if (this->canBeSet())
+			this->sendError(actor_cancelled());
+		for (int i = 0; i < cancelledCallbacks; ++i)
+			this->delPromiseRef();
+	}
+
+	explicit GetAllAsyncResult(int count)
+	  : SAV<std::vector<T>>(1, count), remaining(count), count(count), values(count) {}
+
+	template <class U>
+	void oneSuccess(int idx, U&& value) {
+		values[idx].emplace(std::forward<U>(value));
+		if (--remaining == 0 && this->canBeSet()) {
+			std::vector<T> output;
+			output.reserve(count);
+			for (auto& item : values) {
+				output.push_back(std::move(*item));
+			}
+			this->sendAndDelPromiseRef(std::move(output));
+		} else {
+			this->delPromiseRef();
+		}
+	}
+	void oneError(Error err) {
+		if (this->canBeSet()) {
+			int cancelledCallbacks = detachPendingCallbacks();
+			this->sendErrorAndDelPromiseRef(err);
+			for (int i = 0; i < cancelledCallbacks; ++i)
+				this->delPromiseRef();
+		} else
+			this->delPromiseRef();
+	}
+
+	coro::GetAllAsyncResultCallback<T>* callbacks() { return (coro::GetAllAsyncResultCallback<T>*)(this + 1); }
+};
+
+namespace coro {
+template <class T>
+// Mirrors the Future-based getAll callback path, but owns AsyncResult-specific
+// detach/cancel behavior so pending producers are released when the aggregate
+// finishes early.
+struct GetAllAsyncResultCallback final : AsyncResultCallback<typename AsyncResult<T>::StoredT> {
+	using StoredT = typename AsyncResult<T>::StoredT;
+
+	AsyncResultState<StoredT>* resultState = nullptr;
+	GetAllAsyncResult<T>* head = nullptr;
+	int idx = -1;
+
+	GetAllAsyncResultCallback() = default;
+	GetAllAsyncResultCallback(AsyncResult<T>& result, GetAllAsyncResult<T>* head, int idx);
+
+	void fire(StoredT const& value) override;
+	void fire(StoredT&& value) override;
+	void error(Error error) override;
+	bool isRegistered() const { return resultState != nullptr; }
+	void detach();
+};
+} // namespace coro
+
+namespace coro {
+template <class T>
+GetAllAsyncResultCallback<T>::GetAllAsyncResultCallback(AsyncResult<T>& result, GetAllAsyncResult<T>* head, int idx)
+  : resultState(result.resultState), head(head), idx(idx) {
+	std::move(result).addCallbackAndClear(this);
+}
+
+template <class T>
+void GetAllAsyncResultCallback<T>::detach() {
+	detachAsyncResultStateCallback(resultState, this);
+}
+
+template <class T>
+void GetAllAsyncResultCallback<T>::fire(StoredT const& value) {
+	if constexpr (std::is_constructible_v<T, StoredT const&>) {
+		T copied(value);
+		detach();
+		head->oneSuccess(idx, std::move(copied));
+	} else {
+		// AsyncResultState::complete() currently dispatches through fire(T&&). If
+		// that ever changes, move-only AsyncResult payloads still need this path
+		// to stay unreachable.
+		UNREACHABLE();
+	}
+}
+
+template <class T>
+void GetAllAsyncResultCallback<T>::fire(StoredT&& value) {
+	detach();
+	head->oneSuccess(idx, std::move(value));
+}
+
+template <class T>
+void GetAllAsyncResultCallback<T>::error(Error error) {
+	detach();
+	head->oneError(error);
+}
+} // namespace coro
+
 ACTOR template <class T>
 Future<Void> smartQuorum(std::vector<Future<T>> results,
                          int required,
@@ -1152,6 +1375,41 @@ Future<std::vector<T>> getAll(std::vector<Future<T>> input) {
 	return output;
 }
 
+template <class T>
+// AsyncResult is single-consumer, so getAll requires an explicit ownership
+// transfer from vector callers.
+Future<std::vector<T>> getAll(std::vector<AsyncResult<T>>& input) = delete;
+
+template <class T>
+Future<std::vector<T>> getAll(std::vector<AsyncResult<T>>&& input) {
+	if (input.empty())
+		return std::vector<T>();
+
+	int size = GetAllAsyncResult<T>::sizeFor(input.size());
+	GetAllAsyncResult<T>* result = new (allocateFast(size)) GetAllAsyncResult<T>(input.size());
+
+	coro::GetAllAsyncResultCallback<T>* nextCallback = result->callbacks();
+	for (int i = 0; i < input.size(); ++i) {
+		auto& item = input[i];
+		if (item.isReady()) {
+			new (nextCallback) coro::GetAllAsyncResultCallback<T>();
+			if (item.isError()) {
+				Error err = item.getError();
+				item = AsyncResult<T>();
+				result->oneError(err);
+			} else {
+				T value = std::move(item).get();
+				item = AsyncResult<T>();
+				result->oneSuccess(i, std::move(value));
+			}
+		} else {
+			new (nextCallback) coro::GetAllAsyncResultCallback<T>(item, result, i);
+		}
+		++nextCallback;
+	}
+	return Future<std::vector<T>>(result);
+}
+
 ACTOR template <class T>
 Future<std::vector<T>> appendAll(std::vector<Future<std::vector<T>>> input) {
 	wait(quorum(input, input.size()));
@@ -1338,29 +1596,28 @@ inline Future<Void> operator||(Future<Void> const& lhs, Future<Void> const& rhs)
 	return chooseActor(lhs, rhs);
 }
 
-ACTOR template <class T>
+template <class T>
 Future<T> joinWith(Future<T> f, Future<Void> other) {
-	wait(other);
-	T t = wait(f);
-	return t;
+	co_await other;
+	co_return co_await f;
 }
 
 // wait <interval> then call what() in a loop forever
-ACTOR template <class Func>
+template <class Func>
 Future<Void> recurring(Func what, double interval, TaskPriority taskID = TaskPriority::DefaultDelay) {
-	loop choose {
-		when(wait(delay(interval, taskID))) {
-			what();
-		}
+	while (true) {
+		co_await delay(interval, taskID);
+		what();
 	}
 }
 
-ACTOR template <class Func>
+template <class Func>
 Future<Void> checkUntil(double checkInterval, Func f, TaskPriority taskID = TaskPriority::DefaultDelay) {
-	loop {
-		wait(delay(checkInterval, taskID));
-		if (f())
-			return Void();
+	while (true) {
+		co_await delay(checkInterval, taskID);
+		if (f()) {
+			co_return;
+		}
 	}
 }
 
@@ -1472,6 +1729,19 @@ Future<T> waitOrError(Future<T> f, Future<Void> errorSignal) {
 	}
 }
 
+ACTOR template <class T>
+Future<T> waitOrError(FutureStream<T> f, Future<Void> errorSignal) {
+	choose {
+		when(T val = waitNext(f)) {
+			return val;
+		}
+		when(wait(errorSignal)) {
+			ASSERT(false);
+			throw internal_error();
+		}
+	}
+}
+
 // A simple counter designed to track an ongoing count of something, such as how many actors are in a critical section,
 // how many bytes are currently being processed, etc... Can be explicitly released idempotently, or will automatically
 // release when destructed to handle actor ending or errors.
@@ -1496,7 +1766,8 @@ struct ActiveCounter {
 		  : parent(parent), delta(delta), releaseCallback(releaseCallback) {
 			parent->counter += delta;
 		}
-		Releaser(Releaser&& r) noexcept : parent(r.parent), delta(r.delta), releaseCallback(r.releaseCallback) {
+		explicit(false) Releaser(Releaser&& r) noexcept
+		  : parent(r.parent), delta(r.delta), releaseCallback(r.releaseCallback) {
 			r.parent = nullptr;
 		}
 		void operator=(Releaser&& r) {
@@ -1522,7 +1793,7 @@ struct ActiveCounter {
 
 	T counter;
 
-	ActiveCounter(T initialValue) : counter(initialValue) {}
+	explicit ActiveCounter(T initialValue) : counter(initialValue) {}
 
 	T getValue() { return counter; }
 
@@ -1547,7 +1818,9 @@ struct ActiveCounter {
 //   lock.error(e);   // Next waiter will get e, future waiters will see broken_promise
 //   lock = Lock();   // Or let Lock and any copies go out of scope.  All waiters will see broken_promise.
 struct FlowMutex {
-	FlowMutex(bool hangOnDroppedMutex = false) : hangOnDroppedMutex(hangOnDroppedMutex) { lastPromise.send(Void()); }
+	explicit FlowMutex(bool hangOnDroppedMutex = false) : hangOnDroppedMutex(hangOnDroppedMutex) {
+		lastPromise.send(Void());
+	}
 
 	bool available() const { return lastPromise.isSet(); }
 
@@ -1596,8 +1869,8 @@ struct FlowLock : NonCopyable, public ReferenceCounted<FlowLock> {
 		FlowLock* lock;
 		int remaining;
 		Releaser() : lock(0), remaining(0) {}
-		Releaser(FlowLock& lock, int64_t amount = 1) : lock(&lock), remaining(amount) {}
-		Releaser(Releaser&& r) noexcept : lock(r.lock), remaining(r.remaining) { r.remaining = 0; }
+		explicit(false) Releaser(FlowLock& lock, int64_t amount = 1) : lock(&lock), remaining(amount) {}
+		explicit(false) Releaser(Releaser&& r) noexcept : lock(r.lock), remaining(r.remaining) { r.remaining = 0; }
 		void operator=(Releaser&& r) {
 			if (remaining)
 				lock->release(remaining);
@@ -1735,7 +2008,7 @@ struct FlowLock : NonCopyable, public ReferenceCounted<FlowLock> {
 };
 
 struct NotifiedInt {
-	NotifiedInt(int64_t val = 0) : val(val) {}
+	explicit NotifiedInt(int64_t val = 0) : val(val) {}
 
 	Future<Void> whenAtLeast(int64_t limit) {
 		if (val >= limit)
@@ -1766,7 +2039,7 @@ struct NotifiedInt {
 
 	void operator=(int64_t v) { set(v); }
 
-	NotifiedInt(NotifiedInt&& r) noexcept : waiting(std::move(r.waiting)), val(r.val) {}
+	explicit(false) NotifiedInt(NotifiedInt&& r) noexcept : waiting(std::move(r.waiting)), val(r.val) {}
 	void operator=(NotifiedInt&& r) noexcept {
 		waiting = std::move(r.waiting);
 		val = r.val;
@@ -1792,7 +2065,9 @@ struct BoundedFlowLock : NonCopyable, public ReferenceCounted<BoundedFlowLock> {
 		int64_t permitNumber;
 		Releaser() : lock(nullptr), permitNumber(0) {}
 		Releaser(BoundedFlowLock* lock, int64_t permitNumber) : lock(lock), permitNumber(permitNumber) {}
-		Releaser(Releaser&& r) noexcept : lock(r.lock), permitNumber(r.permitNumber) { r.permitNumber = 0; }
+		explicit(false) Releaser(Releaser&& r) noexcept : lock(r.lock), permitNumber(r.permitNumber) {
+			r.permitNumber = 0;
+		}
 		void operator=(Releaser&& r) {
 			if (permitNumber)
 				lock->release(permitNumber);
@@ -1850,14 +2125,14 @@ struct BoundedFlowLock : NonCopyable, public ReferenceCounted<BoundedFlowLock> {
 	}
 };
 
-ACTOR template <class T>
+template <class T>
 Future<Void> yieldPromiseStream(FutureStream<T> input,
                                 PromiseStream<T> output,
                                 TaskPriority taskID = TaskPriority::DefaultYield) {
-	loop {
-		T f = waitNext(input);
+	while (true) {
+		T f = co_await input;
 		output.send(f);
-		wait(yield(taskID));
+		co_await yield(taskID);
 	}
 }
 
@@ -1871,7 +2146,7 @@ struct YieldedFutureActor final : SAV<Void>,
 	using FastAllocated<YieldedFutureActor>::operator new;
 	using FastAllocated<YieldedFutureActor>::operator delete;
 
-	YieldedFutureActor(Future<Void>&& f) : SAV<Void>(1, 1), in_error_state(Error::fromCode(UNSET_ERROR_CODE)) {
+	explicit YieldedFutureActor(Future<Void>&& f) : SAV<Void>(1, 1), in_error_state(Error::fromCode(UNSET_ERROR_CODE)) {
 		f.addYieldedCallbackAndClear(static_cast<ActorCallback<YieldedFutureActor, 1, Void>*>(this));
 	}
 
@@ -1973,24 +2248,17 @@ class YieldedAsyncMap : public AsyncMap<K, V> {
 	}
 };
 
-ACTOR template <class T>
-Future<T> delayActionJittered(Future<T> what, double time) {
-	wait(delayJittered(time));
-	T t = wait(what);
-	return t;
-}
-
 class AndFuture {
 public:
 	AndFuture() = default;
-	AndFuture(AndFuture const& f) = default;
-	AndFuture(AndFuture&& f) noexcept = default;
+	explicit(false) AndFuture(AndFuture const& f) = default;
+	explicit(false) AndFuture(AndFuture&& f) noexcept = default;
 	AndFuture& operator=(AndFuture const& f) = default;
 	AndFuture& operator=(AndFuture&& f) noexcept = default;
 
-	AndFuture(Future<Void> const& f) : futureCount(1), futures{ f } {}
+	explicit(false) AndFuture(Future<Void> const& f) : futureCount(1), futures{ f } {}
 
-	AndFuture(Error const& e) : futureCount(1), futures{ Future<Void>(e) } {}
+	explicit(false) AndFuture(Error const& e) : futureCount(1), futures{ Future<Void>(e) } {}
 
 	operator Future<Void>() { return getFuture(); }
 
@@ -2048,125 +2316,6 @@ class AndFuture {
 	std::vector<Future<Void>> futures;
 };
 
-// Performs an unordered merge of a and b.
-ACTOR template <class T>
-Future<Void> unorderedMergeStreams(FutureStream<T> a, FutureStream<T> b, PromiseStream<T> output) {
-	state Future<T> aFuture = waitAndForward(a);
-	state Future<T> bFuture = waitAndForward(b);
-	state bool aOpen = true;
-	state bool bOpen = true;
-
-	loop {
-		try {
-			choose {
-				when(T val = wait(aFuture)) {
-					output.send(val);
-					aFuture = waitAndForward(a);
-				}
-				when(T val = wait(bFuture)) {
-					output.send(val);
-					bFuture = waitAndForward(b);
-				}
-			}
-		} catch (Error& e) {
-			if (e.code() != error_code_end_of_stream) {
-				output.sendError(e);
-				break;
-			}
-
-			ASSERT(!aFuture.isError() || !bFuture.isError() || aFuture.getError().code() == bFuture.getError().code());
-
-			if (aFuture.isError()) {
-				aFuture = Never();
-				aOpen = false;
-			}
-			if (bFuture.isError()) {
-				bFuture = Never();
-				bOpen = false;
-			}
-
-			if (!aOpen && !bOpen) {
-				output.sendError(e);
-				break;
-			}
-		}
-	}
-
-	return Void();
-}
-
-// Returns the ordered merge of a and b, assuming that a and b are both already ordered (prefer a over b if keys are
-// equal). T must be a class that implements compare()
-ACTOR template <class T>
-Future<Void> orderedMergeStreams(FutureStream<T> a, FutureStream<T> b, PromiseStream<T> output) {
-	state Optional<T> savedKVa;
-	state bool aOpen;
-	state Optional<T> savedKVb;
-	state bool bOpen;
-
-	aOpen = bOpen = true;
-
-	loop {
-		if (aOpen && !savedKVa.present()) {
-			try {
-				T KVa = waitNext(a);
-				savedKVa = Optional<T>(KVa);
-			} catch (Error& e) {
-				if (e.code() == error_code_end_of_stream) {
-					aOpen = false;
-					if (!bOpen) {
-						output.sendError(e);
-					}
-				} else {
-					output.sendError(e);
-					break;
-				}
-			}
-		}
-		if (bOpen && !savedKVb.present()) {
-			try {
-				T KVb = waitNext(b);
-				savedKVb = Optional<T>(KVb);
-			} catch (Error& e) {
-				if (e.code() == error_code_end_of_stream) {
-					bOpen = false;
-					if (!aOpen) {
-						output.sendError(e);
-					}
-				} else {
-					output.sendError(e);
-					break;
-				}
-			}
-		}
-
-		if (!aOpen) {
-			output.send(savedKVb.get());
-			savedKVb = Optional<T>();
-		} else if (!bOpen) {
-			output.send(savedKVa.get());
-			savedKVa = Optional<T>();
-		} else {
-			int cmp = savedKVa.get().compare(savedKVb.get());
-
-			if (cmp == 0) {
-				// prefer a
-				output.send(savedKVa.get());
-				savedKVa = Optional<T>();
-				savedKVb = Optional<T>();
-			} else if (cmp < 0) {
-				output.send(savedKVa.get());
-				savedKVa = Optional<T>();
-			} else {
-				output.send(savedKVb.get());
-				savedKVb = Optional<T>();
-			}
-		}
-	}
-
-	return Void();
-}
-
 ACTOR template <class T>
 Future<Void> timeReply(Future<T> replyToTime, PromiseStream<double> timeOutput) {
 	state double startTime = now();
@@ -2198,47 +2347,6 @@ Future<T> forward(Future<T> from, Promise<T> to) {
 	}
 }
 
-ACTOR template <class Transaction>
-Future<Void> buggifiedCommit(Transaction tr, bool buggify, int maxDelayDuration = 60.0) {
-	state int buggifyUnknownResultPoint = 0;
-	state int buggifyDelayPoint = 0;
-
-	if (buggify) {
-		int choice = deterministicRandom()->randomInt(1, 9);
-		buggifyUnknownResultPoint = choice / 3;
-		buggifyDelayPoint = choice % 3;
-	}
-
-	// Simulate a delay before commit that could potentially trigger a timeout
-	if (buggifyDelayPoint == 1) {
-		wait(delay(deterministicRandom()->random01() * maxDelayDuration));
-	}
-
-	// Simulate an unknown result that didn't commit
-	if (buggifyUnknownResultPoint == 1) {
-		// The delay avoids a no-wait commit.
-		if (!BUGGIFY) {
-			wait(delay(0));
-		}
-
-		throw commit_unknown_result();
-	}
-
-	wait(safeThreadFutureToFuture(tr->commit()));
-
-	// Simulate a long delay after commit that could potentially trigger a timeout
-	if (buggifyDelayPoint == 2) {
-		wait(delay(deterministicRandom()->random01() * maxDelayDuration));
-	}
-
-	// Simulate an unknown result that did commit
-	if (buggifyUnknownResultPoint == 2) {
-		throw commit_unknown_result();
-	}
-
-	return Void();
-}
-
 // Monad
 
 ACTOR template <class Fun, class T>
@@ -2247,18 +2355,6 @@ Future<decltype(std::declval<Fun>()(std::declval<T>()))> fmap(Fun fun, Future<T>
 	return fun(val);
 }
 
-ACTOR template <class T, class Fun>
-Future<decltype(std::declval<Fun>()(std::declval<T>()).getValue())> runAfter(Future<T> lhs, Fun rhs) {
-	T val1 = wait(lhs);
-	decltype(std::declval<Fun>()(std::declval<T>()).getValue()) res = wait(rhs(val1));
-	return res;
-}
-
-template <class T, class Fun>
-auto operator>>=(Future<T> lhs, Fun&& rhs) -> Future<decltype(rhs(std::declval<T>()))> {
-	return runAfter(lhs, std::forward<Fun>(rhs));
-}
-
 /*
  * NOTE: This implementation can't guarantee the doesn't really enforce the ACTOR execution order. See issue #7708
 ACTOR template <class T, class U>
@@ -2299,9 +2395,9 @@ class AsyncListener final : public IAsyncListener<Output> {
 	// Order matters here, output must outlive monitorActor
 	AsyncVar<Output> output;
 	Future<Void> monitorActor;
-	ACTOR static Future<Void> monitor(Reference<AsyncVar<Input> const> input, AsyncVar<Output>* output, F f) {
-		loop {
-			wait(input->onChange());
+	static Future<Void> monitor(Reference<AsyncVar<Input> const> input, AsyncVar<Output>* output, F f) {
+		while (true) {
+			co_await input->onChange();
 			output->set(f(input->get()));
 		}
 	}
@@ -2344,7 +2440,7 @@ template <class T>
 class UnsafeWeakFutureReference {
 public:
 	UnsafeWeakFutureReference() {}
-	UnsafeWeakFutureReference(Future<Reference<T>> future) : data(new UnsafeWeakFutureReferenceData(future)) {}
+	explicit UnsafeWeakFutureReference(Future<Reference<T>> future) : data(new UnsafeWeakFutureReferenceData(future)) {}
 
 	// Returns a future to obtain a normal reference handle
 	// If the future is ready, this creates a Reference<T> to wrap the object
@@ -2370,7 +2466,7 @@ class UnsafeWeakFutureReference {
 		Future<Reference<T>> future;
 		Future<Void> moveResultFuture;
 
-		UnsafeWeakFutureReferenceData(Future<Reference<T>> future) : future(future) {
+		explicit UnsafeWeakFutureReferenceData(Future<Reference<T>> future) : future(future) {
 			moveResultFuture = moveResult(this);
 		}
 
diff --git a/flow/include/flow/network.h b/flow/include/flow/network.h
index a03063068cd..c083e5a061b 100644
--- a/flow/include/flow/network.h
+++ b/flow/include/flow/network.h
@@ -65,7 +65,7 @@ struct NetworkMetrics {
 		double windowedTimer = 0;
 		double maxDuration = 0;
 
-		PriorityStats(TaskPriority priority) : priority(priority) {}
+		explicit PriorityStats(TaskPriority priority) : priority(priority) {}
 	};
 
 	std::unordered_map<TaskPriority, struct PriorityStats> activeTrackers;
diff --git a/flow/include/flow/serialize.h b/flow/include/flow/serialize.h
index a76160dcefd..76bd03239cd 100644
--- a/flow/include/flow/serialize.h
+++ b/flow/include/flow/serialize.h
@@ -153,13 +153,15 @@ inline void save(Archive& ar, const std::string& value) {
 }
 
 template <class Archive, class T>
-class Serializer<Archive, T, typename std::enable_if_t<is_binary_serializable<T>::value>> {
+    requires(is_binary_serializable<T>::value)
+class Serializer<Archive, T> {
 public:
 	static void serialize(Archive& ar, T& t) { ar.serializeBinaryItem(t); }
 };
 
 template <class Archive, class T>
-class Serializer<Archive, T, typename std::enable_if_t<std::is_enum_v<T>>> {
+    requires(std::is_enum_v<T>)
+class Serializer<Archive, T> {
 public:
 	static void serialize(Archive& ar, T& t) {
 		static_assert(is_binary_serializable<std::underlying_type_t<T>>::value);
@@ -450,7 +452,7 @@ class BinaryWriter : NonCopyable {
 	explicit BinaryWriter(VersionOptions vo) : data(nullptr), size(0), allocated(0) {
 		vo.write(*this);
 	}
-	BinaryWriter(BinaryWriter&& rhs)
+	explicit(false) BinaryWriter(BinaryWriter&& rhs)
 	  : arena(std::move(rhs.arena)), data(rhs.data), size(rhs.size), allocated(rhs.allocated),
 	    m_protocolVersion(rhs.m_protocolVersion) {
 		rhs.size = 0;
@@ -647,7 +649,12 @@ class _Reader {
 		return begin;
 	}
 
-	void serializeBytes(void* data, int bytes) { memcpy(data, static_cast<Impl*>(this)->readBytes(bytes), bytes); }
+	void serializeBytes(void* data, int bytes) {
+		if (bytes == 0) {
+			return;
+		}
+		memcpy(data, static_cast<Impl*>(this)->readBytes(bytes), bytes);
+	}
 
 	template <class T>
 	void serializeBinaryItem(T& t) {
@@ -895,6 +902,9 @@ struct PacketWriter {
 	}
 
 	void serializeBytes(const void* data, int bytes) {
+		if (bytes == 0) {
+			return;
+		}
 		if (bytes <= buffer->bytes_unwritten()) {
 			memcpy(buffer->data() + buffer->bytes_written, data, bytes);
 			buffer->bytes_written += bytes;
@@ -979,7 +989,7 @@ template <class T>
 struct SerializeSource : MakeSerializeSource<SerializeSource<T>, T> {
 	using value_type = T;
 	T const& value;
-	SerializeSource(T const& value) : value(value) {}
+	explicit SerializeSource(T const& value) : value(value) {}
 	void serializeObjectWriter(ObjectWriter& w) const override { w.serialize(value); }
 	T const& get() const override { return value; }
 };
diff --git a/flow/include/flow/singleton.h b/flow/include/flow/singleton.h
index b059149f475..7d1e3627731 100644
--- a/flow/include/flow/singleton.h
+++ b/flow/include/flow/singleton.h
@@ -48,8 +48,8 @@ class WinLockGuard {
 
 public:
 	explicit WinLockGuard(HANDLE& mutex) : mutex(mutex) {
-		HANDLE result = CreateMutexA(NULL, FALSE, NULL);
-		if (result == NULL) {
+		HANDLE result = CreateMutexA(nullptr, FALSE, nullptr);
+		if (result == nullptr) {
 			throw std::system_error(GetLastError(), std::system_category(), nullptr);
 		}
 		mutex = result;
diff --git a/flow/include/flow/swift/ABI/MetadataValues.h b/flow/include/flow/swift/ABI/MetadataValues.h
index 7ef4f2e8930..18092d69fc9 100644
--- a/flow/include/flow/swift/ABI/MetadataValues.h
+++ b/flow/include/flow/swift/ABI/MetadataValues.h
@@ -120,7 +120,7 @@ class JobFlags : public FlagSet<uint32_t> {
 	};
 
 	explicit JobFlags(uint32_t bits) : FlagSet(bits) {}
-	JobFlags(JobKind kind) { setKind(kind); }
+	explicit JobFlags(JobKind kind) { setKind(kind); }
 	JobFlags(JobKind kind, JobPriority priority) {
 		setKind(kind);
 		setPriority(priority);
diff --git a/flow/include/flow/swift/Basic/FlagSet.h b/flow/include/flow/swift/Basic/FlagSet.h
index 591dfbcead4..b7d65114dc6 100644
--- a/flow/include/flow/swift/Basic/FlagSet.h
+++ b/flow/include/flow/swift/Basic/FlagSet.h
@@ -71,7 +71,7 @@ class FlagSet {
 
 	/// Assign to a multi-bit field.
 	template <unsigned FirstBit, unsigned BitWidth, typename FieldType = IntType>
-	void setField(typename std::enable_if<true, FieldType>::type value) {
+	void setField(std::type_identity_t<FieldType> value) {
 		// Note that we suppress template argument deduction for FieldType.
 		assert(IntType(value) <= lowMaskFor<BitWidth>() && "value out of range");
 		Bits = (Bits & ~maskFor<FirstBit, BitWidth>()) | (IntType(value) << FirstBit);
diff --git a/flow/include/flow/swift_concurrency_hooks.h b/flow/include/flow/swift_concurrency_hooks.h
index 051584b25d7..1c31ccd69d5 100644
--- a/flow/include/flow/swift_concurrency_hooks.h
+++ b/flow/include/flow/swift_concurrency_hooks.h
@@ -25,7 +25,7 @@
 #include "swift/ABI/Task.h"
 #include <stdint.h>
 #include "flow/AsioReactor.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 
 #if !defined(__has_feature)
 #define __has_feature(x) 0
diff --git a/flow/include/flow/unactorcompiler.h b/flow/include/flow/unactorcompiler.h
index 2ba5d0ef971..088e18be82c 100644
--- a/flow/include/flow/unactorcompiler.h
+++ b/flow/include/flow/unactorcompiler.h
@@ -23,7 +23,6 @@
 #ifndef NO_INTELLISENSE
 #undef ACTOR
 #undef SWIFT_ACTOR
-#undef DESCR
 #undef state
 #undef UNCANCELLABLE
 #undef choose
diff --git a/flow/network.cpp b/flow/network.cpp
index 0b609abb8f5..38f097ca2fb 100644
--- a/flow/network.cpp
+++ b/flow/network.cpp
@@ -253,37 +253,69 @@ std::string formatIpPort(const IPAddress& ip, uint16_t port) {
 	return format(patt, ip.toString().c_str(), port);
 }
 
+DNSCache::DNSCache(const std::map<std::string, std::vector<NetworkAddress>>& dnsCache) {
+	for (const auto& [key, addresses] : dnsCache) {
+		entries[key].addresses = addresses;
+	}
+}
+
 Optional<std::vector<NetworkAddress>> DNSCache::find(const std::string& host, const std::string& service) {
-	auto it = hostnameToAddresses.find(host + ":" + service);
-	if (it != hostnameToAddresses.end()) {
-		return it->second;
+	auto it = entries.find(host + ":" + service);
+	if (it != entries.end()) {
+		it->second.lastAccess = g_network->now();
+		return it->second.addresses;
 	}
 	return {};
 }
 
 void DNSCache::add(const std::string& host, const std::string& service, const std::vector<NetworkAddress>& addresses) {
-	hostnameToAddresses[host + ":" + service] = addresses;
+	Entry& entry = entries[host + ":" + service];
+	entry.addresses = addresses;
+	entry.lastAccess = g_network->now();
+}
+
+void DNSCache::update(const std::string& host,
+                      const std::string& service,
+                      const std::vector<NetworkAddress>& addresses) {
+	entries[host + ":" + service].addresses = addresses;
 }
 
 void DNSCache::remove(const std::string& host, const std::string& service) {
-	auto it = hostnameToAddresses.find(host + ":" + service);
-	if (it != hostnameToAddresses.end()) {
-		hostnameToAddresses.erase(it);
+	auto it = entries.find(host + ":" + service);
+	if (it != entries.end()) {
+		entries.erase(it);
 	}
 }
 
 void DNSCache::clear() {
-	hostnameToAddresses.clear();
+	entries.clear();
+}
+
+std::vector<std::string> DNSCache::getKeys() const {
+	std::vector<std::string> keys;
+	keys.reserve(entries.size());
+	for (const auto& [key, _] : entries) {
+		keys.push_back(key);
+	}
+	return keys;
+}
+
+Optional<double> DNSCache::getLastAccess(const std::string& host, const std::string& service) const {
+	auto it = entries.find(host + ":" + service);
+	if (it != entries.end()) {
+		return it->second.lastAccess;
+	}
+	return {};
 }
 
 std::string DNSCache::toString() {
 	std::string ret;
-	for (auto it = hostnameToAddresses.begin(); it != hostnameToAddresses.end(); ++it) {
-		if (it != hostnameToAddresses.begin()) {
+	for (auto it = entries.begin(); it != entries.end(); ++it) {
+		if (it != entries.begin()) {
 			ret += ';';
 		}
 		ret += it->first + ',';
-		const std::vector<NetworkAddress>& addresses = it->second;
+		const std::vector<NetworkAddress>& addresses = it->second.addresses;
 		for (int i = 0; i < addresses.size(); ++i) {
 			ret += addresses[i].toString();
 			if (i != addresses.size() - 1) {
@@ -390,7 +422,7 @@ Future<Reference<IConnection>> INetworkConnections::connect(const std::string& h
 	});
 }
 
-IUDPSocket::~IUDPSocket() {}
+IUDPSocket::~IUDPSocket() = default;
 
 const std::vector<int> NetworkMetrics::starvationBins = { 1, 3500, 7000, 7500, 8500, 8900, 10500 };
 
diff --git a/flow/include/flow/rte_memcpy.h b/flow/rte_memcpy.h
similarity index 100%
rename from flow/include/flow/rte_memcpy.h
rename to flow/rte_memcpy.h
diff --git a/flow/swift_concurrency_hooks.cpp b/flow/swift_concurrency_hooks.cpp
index 014919ac2ba..cbcc6a30abd 100644
--- a/flow/swift_concurrency_hooks.cpp
+++ b/flow/swift_concurrency_hooks.cpp
@@ -21,7 +21,7 @@
 #include "flow/swift_concurrency_hooks.h"
 #include "flow/swift.h"
 #include "flow/swift/ABI/Task.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 
 // ==== ----------------------------------------------------------------------------------------------------------------
 
diff --git a/flow/swift_task_priority.cpp b/flow/swift_task_priority.cpp
index 05e750abf82..f4172bbd51a 100644
--- a/flow/swift_task_priority.cpp
+++ b/flow/swift_task_priority.cpp
@@ -21,7 +21,7 @@
 #include "flow/swift.h"
 #include "flow/swift_concurrency_hooks.h"
 #include "flow/swift/ABI/Task.h"
-#include "flow/TLSConfig.actor.h"
+#include "flow/TLSConfig.h"
 
 // FIXME: surely there must be some more automatic way to maintain the mappings with Swift/C++ interop.
 TaskPriority swift_priority_to_net2(swift::JobPriority p) {
diff --git a/flow/test_memcpy.cpp b/flow/test_memcpy.cpp
index c5d3fd70920..8fb36f174ed 100644
--- a/flow/test_memcpy.cpp
+++ b/flow/test_memcpy.cpp
@@ -10,7 +10,7 @@
 #ifdef WITH_FOLLY_MEMCPY
 #include "folly_memcpy.h"
 #endif
-#include "flow/rte_memcpy.h"
+#include "rte_memcpy.h"
 #include "flow/IRandom.h"
 
 #include "flow/UnitTest.h"
diff --git a/flow/test_memcpy_perf.cpp b/flow/test_memcpy_perf.cpp
deleted file mode 100644
index efd2eb5abf3..00000000000
--- a/flow/test_memcpy_perf.cpp
+++ /dev/null
@@ -1,341 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
- */
-
-#include <stdint.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include "flow/rte_memcpy.h"
-#include "flow/IRandom.h"
-#include "flow/UnitTest.h"
-#include "flow/flow.h"
-
-#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__AVX__)
-
-#include <sys/time.h>
-
-extern "C" {
-void* folly_memcpy(void* dst, const void* src, uint32_t length);
-}
-
-void* rte_memcpy_noinline(void* dst, const void* src, size_t length); // for performance comparisons
-
-/*
- * Set this to the maximum buffer size you want to test. If it is 0, then the
- * values in the buf_sizes[] array below will be used.
- */
-#define TEST_VALUE_RANGE 0
-
-/* List of buffer sizes to test */
-#if TEST_VALUE_RANGE == 0
-static size_t buf_sizes[] = { 1,    2,    3,    4,    5,    6,    7,    8,    9,    12,   15,   16,   17,
-	                          31,   32,   33,   63,   64,   65,   127,  128,  129,  191,  192,  193,  255,
-	                          256,  257,  319,  320,  321,  383,  384,  385,  447,  448,  449,  511,  512,
-	                          513,  767,  768,  769,  1023, 1024, 1025, 1518, 1522, 1536, 1600, 2048, 2560,
-	                          3072, 3584, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 };
-/* MUST be as large as largest packet size above */
-#define SMALL_BUFFER_SIZE 8192
-#else /* TEST_VALUE_RANGE != 0 */
-static size_t buf_sizes[TEST_VALUE_RANGE];
-#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE
-#endif /* TEST_VALUE_RANGE == 0 */
-
-/*
- * Arrays of this size are used for measuring uncached memory accesses by
- * picking a random location within the buffer. Make this smaller if there are
- * memory allocation errors.
- */
-#define LARGE_BUFFER_SIZE (100 * 1024 * 1024)
-
-/* How many times to run timing loop for performance tests */
-#define TEST_ITERATIONS 1000000
-#define TEST_BATCH_SIZE 100
-
-/* Data is aligned on this many bytes (power of 2) */
-// #ifdef RTE_MACHINE_CPUFLAG_AVX512F
-#define ALIGNMENT_UNIT 64
-// #elif defined RTE_MACHINE_CPUFLAG_AVX2
-// #define ALIGNMENT_UNIT          32
-// #else /* RTE_MACHINE_CPUFLAG */
-// #define ALIGNMENT_UNIT          16
-// #endif /* RTE_MACHINE_CPUFLAG */
-
-/*
- * Pointers used in performance tests. The two large buffers are for uncached
- * access where random addresses within the buffer are used for each
- * memcpy. The two small buffers are for cached access.
- */
-static uint8_t *large_buf_read, *large_buf_write;
-static uint8_t *small_buf_read, *small_buf_write;
-
-static size_t round_up(size_t sz, size_t alignment) {
-	return (((sz - 1) / alignment) + 1) * alignment;
-}
-
-static uint8_t* rte_malloc(char const* ignored, size_t sz, size_t align) {
-	return (uint8_t*)aligned_alloc(align, round_up(sz, align));
-}
-
-static void rte_free(void* ptr) {
-	if (!!ptr) {
-		free(ptr);
-	}
-}
-
-/* Initialise data buffers. */
-static int init_buffers(void) {
-	unsigned i;
-
-	large_buf_read = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT);
-	if (large_buf_read == nullptr)
-		goto error_large_buf_read;
-
-	large_buf_write = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT);
-	if (large_buf_write == nullptr)
-		goto error_large_buf_write;
-
-	small_buf_read = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT);
-	if (small_buf_read == nullptr)
-		goto error_small_buf_read;
-
-	small_buf_write = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT);
-	if (small_buf_write == nullptr)
-		goto error_small_buf_write;
-
-	for (i = 0; i < LARGE_BUFFER_SIZE; i++)
-		large_buf_read[i] = deterministicRandom()->randomUInt32();
-	for (i = 0; i < SMALL_BUFFER_SIZE; i++)
-		small_buf_read[i] = deterministicRandom()->randomUInt32();
-
-	return 0;
-
-error_small_buf_write:
-	rte_free(small_buf_read);
-error_small_buf_read:
-	rte_free(large_buf_write);
-error_large_buf_write:
-	rte_free(large_buf_read);
-error_large_buf_read:
-	printf("ERROR: not enough memory\n");
-	return -1;
-}
-
-/* Cleanup data buffers */
-static void free_buffers(void) {
-	rte_free(large_buf_read);
-	rte_free(large_buf_write);
-	rte_free(small_buf_read);
-	rte_free(small_buf_write);
-}
-
-/*
- * Get a random offset into large array, with enough space needed to perform
- * max copy size. Offset is aligned, uoffset is used for unalignment setting.
- */
-static inline size_t get_rand_offset(size_t uoffset) {
-	return ((deterministicRandom()->randomUInt32() % (LARGE_BUFFER_SIZE - SMALL_BUFFER_SIZE)) & ~(ALIGNMENT_UNIT - 1)) +
-	       uoffset;
-}
-
-/* Fill in source and destination addresses. */
-static inline void fill_addr_arrays(size_t* dst_addr,
-                                    int is_dst_cached,
-                                    size_t dst_uoffset,
-                                    size_t* src_addr,
-                                    int is_src_cached,
-                                    size_t src_uoffset) {
-	unsigned int i;
-
-	for (i = 0; i < TEST_BATCH_SIZE; i++) {
-		dst_addr[i] = (is_dst_cached) ? dst_uoffset : get_rand_offset(dst_uoffset);
-		src_addr[i] = (is_src_cached) ? src_uoffset : get_rand_offset(src_uoffset);
-	}
-}
-
-/*
- * WORKAROUND: For some reason the first test doing an uncached write
- * takes a very long time (~25 times longer than is expected). So we do
- * it once without timing.
- */
-static void do_uncached_write(uint8_t* dst, int is_dst_cached, const uint8_t* src, int is_src_cached, size_t size) {
-	unsigned i, j;
-	size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE];
-
-	for (i = 0; i < (TEST_ITERATIONS / TEST_BATCH_SIZE); i++) {
-		fill_addr_arrays(dst_addrs, is_dst_cached, 0, src_addrs, is_src_cached, 0);
-		for (j = 0; j < TEST_BATCH_SIZE; j++) {
-			memcpy(dst + dst_addrs[j], src + src_addrs[j], size);
-		}
-	}
-}
-
-/*
- * Run a single memcpy performance test. This is a macro to ensure that if
- * the "size" parameter is a constant it won't be converted to a variable.
- */
-#define SINGLE_PERF_TEST(dst, is_dst_cached, dst_uoffset, src, is_src_cached, src_uoffset, size)                       \
-	do {                                                                                                               \
-		unsigned int iter, t;                                                                                          \
-		size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE];                                                 \
-		uint64_t start_time, total_time = 0;                                                                           \
-		uint64_t total_time2 = 0;                                                                                      \
-		for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) {                                           \
-			fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, src_addrs, is_src_cached, src_uoffset);            \
-			start_time = rte_rdtsc();                                                                                  \
-			for (t = 0; t < TEST_BATCH_SIZE; t++)                                                                      \
-				rte_memcpy_noinline(dst + dst_addrs[t], src + src_addrs[t], size);                                     \
-			total_time += rte_rdtsc() - start_time;                                                                    \
-		}                                                                                                              \
-		for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) {                                           \
-			fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, src_addrs, is_src_cached, src_uoffset);            \
-			start_time = rte_rdtsc();                                                                                  \
-			for (t = 0; t < TEST_BATCH_SIZE; t++)                                                                      \
-				memcpy(dst + dst_addrs[t], src + src_addrs[t], size);                                                  \
-			total_time2 += rte_rdtsc() - start_time;                                                                   \
-		}                                                                                                              \
-		printf("%3.0f -", (double)total_time / TEST_ITERATIONS);                                                       \
-		printf("%3.0f", (double)total_time2 / TEST_ITERATIONS);                                                        \
-		printf("(%6.2f%%) ", ((double)total_time - total_time2) * 100 / total_time2);                                  \
-	} while (0)
-
-/* Run aligned memcpy tests for each cached/uncached permutation */
-#define ALL_PERF_TESTS_FOR_SIZE(n)                                                                                     \
-	do {                                                                                                               \
-		if (__builtin_constant_p(n))                                                                                   \
-			printf("\nC%6u", (unsigned)n);                                                                             \
-		else                                                                                                           \
-			printf("\n%7u", (unsigned)n);                                                                              \
-		SINGLE_PERF_TEST(small_buf_write, 1, 0, small_buf_read, 1, 0, n);                                              \
-		SINGLE_PERF_TEST(large_buf_write, 0, 0, small_buf_read, 1, 0, n);                                              \
-		SINGLE_PERF_TEST(small_buf_write, 1, 0, large_buf_read, 0, 0, n);                                              \
-		SINGLE_PERF_TEST(large_buf_write, 0, 0, large_buf_read, 0, 0, n);                                              \
-	} while (0)
-
-/* Run unaligned memcpy tests for each cached/uncached permutation */
-#define ALL_PERF_TESTS_FOR_SIZE_UNALIGNED(n)                                                                           \
-	do {                                                                                                               \
-		if (__builtin_constant_p(n))                                                                                   \
-			printf("\nC%6u", (unsigned)n);                                                                             \
-		else                                                                                                           \
-			printf("\n%7u", (unsigned)n);                                                                              \
-		SINGLE_PERF_TEST(small_buf_write, 1, 1, small_buf_read, 1, 5, n);                                              \
-		SINGLE_PERF_TEST(large_buf_write, 0, 1, small_buf_read, 1, 5, n);                                              \
-		SINGLE_PERF_TEST(small_buf_write, 1, 1, large_buf_read, 0, 5, n);                                              \
-		SINGLE_PERF_TEST(large_buf_write, 0, 1, large_buf_read, 0, 5, n);                                              \
-	} while (0)
-
-/* Run memcpy tests for constant length */
-#define ALL_PERF_TEST_FOR_CONSTANT                                                                                     \
-	do {                                                                                                               \
-		TEST_CONSTANT(6U);                                                                                             \
-		TEST_CONSTANT(64U);                                                                                            \
-		TEST_CONSTANT(128U);                                                                                           \
-		TEST_CONSTANT(192U);                                                                                           \
-		TEST_CONSTANT(256U);                                                                                           \
-		TEST_CONSTANT(512U);                                                                                           \
-		TEST_CONSTANT(768U);                                                                                           \
-		TEST_CONSTANT(1024U);                                                                                          \
-		TEST_CONSTANT(1536U);                                                                                          \
-	} while (0)
-
-/* Run all memcpy tests for aligned constant cases */
-static inline void perf_test_constant_aligned(void) {
-#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE
-	ALL_PERF_TEST_FOR_CONSTANT;
-#undef TEST_CONSTANT
-}
-
-/* Run all memcpy tests for unaligned constant cases */
-static inline void perf_test_constant_unaligned(void) {
-#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE_UNALIGNED
-	ALL_PERF_TEST_FOR_CONSTANT;
-#undef TEST_CONSTANT
-}
-
-/* Run all memcpy tests for aligned variable cases */
-static inline void perf_test_variable_aligned(void) {
-	unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]);
-	unsigned i;
-	for (i = 0; i < n; i++) {
-		ALL_PERF_TESTS_FOR_SIZE((size_t)buf_sizes[i]);
-	}
-}
-
-/* Run all memcpy tests for unaligned variable cases */
-static inline void perf_test_variable_unaligned(void) {
-	unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]);
-	unsigned i;
-	for (i = 0; i < n; i++) {
-		ALL_PERF_TESTS_FOR_SIZE_UNALIGNED((size_t)buf_sizes[i]);
-	}
-}
-
-/* Run all memcpy tests */
-TEST_CASE("performance/memcpy/rte") {
-	int ret;
-	struct timeval tv_begin, tv_end;
-	double time_aligned, time_unaligned;
-	double time_aligned_const, time_unaligned_const;
-
-	ret = init_buffers();
-	ASSERT(ret == 0);
-
-#if TEST_VALUE_RANGE != 0
-	/* Set up buf_sizes array, if required */
-	unsigned i;
-	for (i = 0; i < TEST_VALUE_RANGE; i++)
-		buf_sizes[i] = i;
-#endif
-
-	/* See function comment */
-	do_uncached_write(large_buf_write, 0, small_buf_read, 1, SMALL_BUFFER_SIZE);
-
-	printf("\n** rte_memcpy() - memcpy perf. tests (C = compile-time constant) **\n"
-	       "======= ================= ================= ================= =================\n"
-	       "   Size   Cache to cache     Cache to mem      Mem to cache        Mem to mem\n"
-	       "(bytes)          (ticks)          (ticks)           (ticks)           (ticks)\n"
-	       "------- ----------------- ----------------- ----------------- -----------------");
-
-	printf("\n================================= %2dB aligned =================================", ALIGNMENT_UNIT);
-	/* Do aligned tests where size is a variable */
-	gettimeofday(&tv_begin, nullptr);
-	perf_test_variable_aligned();
-	gettimeofday(&tv_end, nullptr);
-	time_aligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + ((double)tv_end.tv_usec - tv_begin.tv_usec) / 1000000;
-	printf("\n------- ----------------- ----------------- ----------------- -----------------");
-	/* Do aligned tests where size is a compile-time constant */
-	gettimeofday(&tv_begin, nullptr);
-	perf_test_constant_aligned();
-	gettimeofday(&tv_end, nullptr);
-	time_aligned_const =
-	    (double)(tv_end.tv_sec - tv_begin.tv_sec) + ((double)tv_end.tv_usec - tv_begin.tv_usec) / 1000000;
-	printf("\n================================== Unaligned ==================================");
-	/* Do unaligned tests where size is a variable */
-	gettimeofday(&tv_begin, nullptr);
-	perf_test_variable_unaligned();
-	gettimeofday(&tv_end, nullptr);
-	time_unaligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + ((double)tv_end.tv_usec - tv_begin.tv_usec) / 1000000;
-	printf("\n------- ----------------- ----------------- ----------------- -----------------");
-	/* Do unaligned tests where size is a compile-time constant */
-	gettimeofday(&tv_begin, nullptr);
-	perf_test_constant_unaligned();
-	gettimeofday(&tv_end, nullptr);
-	time_unaligned_const =
-	    (double)(tv_end.tv_sec - tv_begin.tv_sec) + ((double)tv_end.tv_usec - tv_begin.tv_usec) / 1000000;
-	printf("\n======= ================= ================= ================= =================\n\n");
-
-	printf("Test Execution Time (seconds):\n");
-	printf("Aligned variable copy size   = %8.3f\n", time_aligned);
-	printf("Aligned constant copy size   = %8.3f\n", time_aligned_const);
-	printf("Unaligned variable copy size = %8.3f\n", time_unaligned);
-	printf("Unaligned constant copy size = %8.3f\n", time_unaligned_const);
-	free_buffers();
-
-	return Void();
-}
-
-#endif // defined (__linux__) || defined (__FreeBSD__)
-
-void forceLinkMemcpyPerfTests() {}
diff --git a/flowbench/CMakeLists.txt b/flowbench/CMakeLists.txt
deleted file mode 100644
index 4b0fe6b3e10..00000000000
--- a/flowbench/CMakeLists.txt
+++ /dev/null
@@ -1,66 +0,0 @@
-project(flowbench)
-
-fdb_find_sources(FLOWBENCH_SRCS)
-add_flow_target(EXECUTABLE NAME flowbench SRCS ${FLOWBENCH_SRCS})
-
-# This stub is kept to maintain the backward compatibility with the existing build
-# environment. It should be removed when a reasonable environment is established.
-if(NOT benchmark_ROOT)
-  if(EXISTS /opt/googlebenchmark-f91b6b AND CLANG AND USE_LIBCXX)
-    set(benchmark_ROOT /opt/googlebenchmark-f91b6b)
-  elseif(EXISTS /opt/googlebenchmark-f91b6b-g++ AND NOT USE_LIBCXX)
-    set(benchmark_ROOT /opt/googlebenchmark-f91b6b-g++)
-  endif()
-endif()
-
-find_package(benchmark)
-
-if(benchmark_FOUND)
-  target_include_directories(flowbench PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
-  target_include_directories(flowbench INTERFACE $<TARGET_PROPERTY:benchmark::benchmark, INTERFACE_INCLUDE_DIRECTORIES>)
-  target_link_libraries(flowbench benchmark::benchmark)
-else()
-  ## This seems to be copy-pasted from the the google benchmark documentation.
-  ## It breaks if you attempt to reuse a build of googlebenchmark across FDB
-  ## builds.
-
-  # include the configurations from benchmark.cmake
-  configure_file(benchmark.cmake googlebenchmark-download/CMakeLists.txt)
-  # executing the configuration step
-  execute_process(
-      COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
-      RESULT_VARIABLE results
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-download
-  )
-  # checking if the configuration step passed
-  if(results)
-      message(FATAL_ERROR "Configuration step for Benchmark has Failed. ${results}")
-  endif()
-  # executing the build step
-  execute_process(
-      COMMAND ${CMAKE_COMMAND} --build . --config Release
-      RESULT_VARIABLE results
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-download
-  )
-  # checking if the build step passed
-  if(results)
-    message(FATAL_ERROR "Build step for Benchmark has Failed. ${results}")
-  endif()
-
-  set(BENCHMARK_ENABLE_TESTING OFF)
-  # libc++ implicitly requires -pthread
-  set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS} -pthread")
-  add_subdirectory(
-    ${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-src
-    ${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-build
-    EXCLUDE_FROM_ALL
-  )
-  target_include_directories(flowbench PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include"  ${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-src/include)
-  target_link_libraries(flowbench benchmark)
-endif()
-
-
-if(FLOW_USE_ZSTD)
-  target_include_directories(flowbench PRIVATE ${ZSTD_LIB_INCLUDE_DIR})
-endif()
-target_link_libraries(flowbench pthread flow fdbclient)
diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index bc5bb04f0d6..f318149b307 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -60,7 +60,7 @@ RUN curl -Ls "https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TA
 
 WORKDIR /
 
-FROM golang:1.24.9-bookworm AS go-build
+FROM golang:1.25.8-bookworm AS go-build
 COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
 WORKDIR /fdbkubernetesmonitor
 RUN go build -o /fdb-kubernetes-monitor *.go
@@ -133,7 +133,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
     $CURL $FDB_WEBSITE/$FDB_VERSION/libfdb_c.${FDB_ARCH}.so -o /usr/lib/libfdb_c.so
 
 # Setup all symlinks for the other binaries that are a copy of fdbbackup
-RUN for file in fdbdr fdbrestore backup_agent dr_agent fastrestore_tool; do \
+RUN for file in fdbdr fdbrestore backup_agent dr_agent; do \
         ln -s /usr/bin/fdbbackup "/usr/bin/$file"; \
     done && \
     cd / && \
@@ -201,6 +201,31 @@ ENV FDB_CLUSTER_FILE_CONTENTS=""
 ENV FDB_PROCESS_CLASS=unset
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
 
+FROM foundationdb-base AS mako
+
+ARG FDB_VERSION=7.3.63
+ARG TARGETARCH
+COPY website/${FDB_VERSION}/mako.* /tmp/
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+        FDB_ARCH=x86_64; \
+    elif [ "$TARGETARCH" = "arm64" ]; then \
+        FDB_ARCH=aarch64; \
+    else \
+        echo "ERROR: unsupported architecture $TARGETARCH" 1>&2; \
+        exit 1; \
+    fi; \
+    cp "/tmp/mako.${FDB_ARCH}" /usr/bin/mako && \
+    chmod +x /usr/bin/mako && \
+    rm -rf /tmp/*
+
+RUN mkdir -p /var/log/fdb-trace-logs && \
+    chown -R fdb:fdb /var/log/fdb-trace-logs
+USER fdb
+WORKDIR /var/fdb
+ENV FDB_CLUSTER_FILE=/var/fdb/fdb.cluster
+ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/
+ENTRYPOINT ["/usr/bin/mako"]
+
 FROM foundationdb-base AS ycsb
 
 RUN microdnf -y install \
diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh
index 8fe7585e268..363b97495c5 100755
--- a/packaging/docker/build-images.sh
+++ b/packaging/docker/build-images.sh
@@ -35,6 +35,9 @@ function create_fake_website_directory () {
     fi
     local stripped_binaries_and_from_where="${1}"
     fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' )
+    if [ "${BUILD_MAKO:-0}" -eq 1 ]; then
+        fdb_binaries+=( 'mako' )
+    fi
     logg "PREPARING WEBSITE"
     website_directory="${script_dir}/website"
     rm -rf "${website_directory}"
@@ -208,6 +211,7 @@ function build_and_push_images () {
               [ "${image}" == 'foundationdb-kubernetes-sidecar' ] || \
               [ "${image}" == 'fdb-aws-s3-credentials-fetcher-sidecar' ] || \
               [ "${image}" == 'ycsb' ] || \
+              [ "${image}" == 'mako' ] || \
               [ "${image}" == 'fdb-kubernetes-monitor' ]; then
             tags_to_push+=("${image_tag}")
         fi
@@ -262,6 +266,9 @@ image_list=(
     'foundationdb-kubernetes-sidecar'
     'ycsb'
 )
+if [ "${BUILD_MAKO:-0}" -eq 1 ]; then
+    image_list+=( 'mako' )
+fi
 registry=""
 tag_base="foundationdb/"
 
diff --git a/packaging/docker/kubernetes/test_config.yaml b/packaging/docker/kubernetes/test_config.yaml
index 3d5cb42bdd3..baf91b9726b 100644
--- a/packaging/docker/kubernetes/test_config.yaml
+++ b/packaging/docker/kubernetes/test_config.yaml
@@ -43,7 +43,7 @@ spec:
     spec:
       containers:
         - name: foundationdb
-          image: foundationdb/fdb-kubernetes-monitor:7.1.5-local
+          image: foundationdb/fdb-kubernetes-monitor:7.3.73
           imagePullPolicy: IfNotPresent
           args:
             - --input-dir
@@ -91,13 +91,13 @@ spec:
             - name: logs
               mountPath: /var/fdb/logs
         - name: foundationdb-sidecar
-          image: foundationdb/fdb-kubernetes-monitor:7.1.6-local
+          image: foundationdb/fdb-kubernetes-monitor:7.3.73
           imagePullPolicy: IfNotPresent
           args:
             - --mode
             - sidecar
             - --main-container-version
-            - 7.1.5
+            - 7.3.73
             - --output-dir
             - /var/fdb/shared-binaries
             - --copy-binary
@@ -139,7 +139,7 @@ data:
   config.json: |
     {
       "runProcesses": false,
-      "version": "7.1.5",
+      "version": "7.3.73",
       "arguments": [
         {"value": "--cluster-file"},
         {"value": "/var/fdb/data/fdb.cluster"},
@@ -212,73 +212,3 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: fdb-kubernetes-example
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: fdb-kubernetes-example-client
-spec:
-  replicas: 2
-  revisionHistoryLimit: 1
-  selector:
-    matchLabels:
-      app: fdb-kubernetes-example-client
-  template:
-    metadata:
-      labels:
-        app: fdb-kubernetes-example-client
-      name: fdb-kubernetes-example-client
-    spec:
-      volumes:
-        - name: config-map
-          configMap:
-            name: fdb-kubernetes-example-config
-            items:
-              - key: fdb.cluster
-                path: fdb.cluster
-        - name: dynamic-conf
-          emptyDir: {}
-      initContainers:
-        - name: fdb-kubernetes-monitor-init
-          image: foundationdb/fdb-kubernetes-monitor:7.1.5-local
-          imagePullPolicy: IfNotPresent
-          args:
-            - "--mode"
-            - "init"
-            - "--input-dir"
-            - "/var/input-files"
-            - "--output-dir"
-            - "/var/output-files"
-            - "--copy-file"
-            - "fdb.cluster"
-            - "--require-not-empty"
-            - "fdb.cluster"
-            - "--copy-library"
-            - "6.3"
-            - "--copy-primary-library"
-            - "7.1"
-            - "--copy-binary"
-            - "fdbcli"
-          volumeMounts:
-            - name: config-map
-              mountPath: /var/input-files
-            - name: dynamic-conf
-              mountPath: /var/output-files
-      containers:
-        - name: client
-          image: foundationdb/foundationdb-sample-python-app:latest
-          imagePullPolicy: Always
-          env:
-            - name: FDB_CLUSTER_FILE
-              value: /var/dynamic-conf/fdb.cluster
-            - name: FDB_API_VERSION
-              value: "610"
-            - name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
-              value: fdb-kubernetes-example-client
-            - name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
-              value: /var/dynamic-conf/lib/multiversion
-            - name: LD_LIBRARY_PATH
-              value: /var/dynamic-conf/lib
-          volumeMounts:
-            - name: dynamic-conf
-              mountPath: /var/dynamic-conf
diff --git a/packaging/docker/samples/python/app/requirements.txt b/packaging/docker/samples/python/app/requirements.txt
index 3f7e9e71449..d88a261a88b 100644
--- a/packaging/docker/samples/python/app/requirements.txt
+++ b/packaging/docker/samples/python/app/requirements.txt
@@ -17,5 +17,5 @@
 # limitations under the License.
 #
 
-Flask==2.3.2
+Flask==3.1.3
 foundationdb==6.2.10
diff --git a/packaging/docker/sidecar_test.py b/packaging/docker/sidecar_test.py
index 97c94e3cc44..48e4def4e55 100755
--- a/packaging/docker/sidecar_test.py
+++ b/packaging/docker/sidecar_test.py
@@ -43,6 +43,11 @@ def setUp(self):
         self.mock_config = MagicMock()
         # We don't want to use TLS for the local tests for now.
         self.mock_config.enable_tls = False
+        self.mock_config.require_not_empty = []
+        self.mock_config.copy_files = []
+        self.mock_config.copy_binaries = []
+        self.mock_config.copy_libraries = []
+        self.mock_config.input_monitor_conf = None
         self.mock_config.output_dir = tempfile.mkdtemp()
 
         handler = partial(
@@ -146,8 +151,17 @@ def test_get_not_found(self):
         self.assertEqual(r.status_code, 404)
         self.assertRegex(r.text, "Path /foobar not found")
 
+    def test_post_copy_files(self):
+        r = requests.post(f"{self.server_url}/copy_files")
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.text, "OK\n")
+
+    def test_post_not_found(self):
+        r = requests.post(f"{self.server_url}/foobar")
+        self.assertEqual(r.status_code, 404)
+        self.assertRegex(r.text, "Path not found")
+
 
-# TODO(johscheuer): Add test cases for post requests.
 # TODO(johscheuer): Add test cases for TLS.
 if __name__ == "__main__":
     unittest.main()
diff --git a/packaging/multiversion/server/postinst b/packaging/multiversion/server/postinst
index 92f16550e6a..a53e64471dd 100644
--- a/packaging/multiversion/server/postinst
+++ b/packaging/multiversion/server/postinst
@@ -8,10 +8,12 @@ chown -R foundationdb:foundationdb /var/lib/foundationdb /var/log/foundationdb
 chmod -R 0750 /var/lib/foundationdb/data /var/log/foundationdb
 
 mkdir -p /usr/lib/foundationdb
+
 update-alternatives --install /usr/sbin/fdbserver fdbserver /usr/lib/foundationdb-@FDB_VERSION@/sbin/fdbserver @ALTERNATIVES_PRIORITY@ \
     --slave /usr/sbin/fdbmonitor fdbmonitor /usr/lib/foundationdb-@FDB_VERSION@/sbin/fdbmonitor \
     --slave /lib/systemd/system/foundationdb.service foundationdb.service /usr/lib/foundationdb-@FDB_VERSION@/lib/systemd/system/foundationdb.service
 
+
 if mkdir /etc/foundationdb
 then
     description=`LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 8`
diff --git a/packaging/osx/buildpkg.sh b/packaging/osx/buildpkg.sh
index 7eb13c47a6c..db210631178 100755
--- a/packaging/osx/buildpkg.sh
+++ b/packaging/osx/buildpkg.sh
@@ -17,10 +17,10 @@ fi
 #
 BUILDDIR="$1"
 SRCDIR="$2"
-
+HOST_ARCH="$(uname -m)"
 VERSION="$(cat "$BUILDDIR/version.txt")"
 
-PKGFILE="$BUILDDIR/packages/FoundationDB-$VERSION.pkg"
+PKGFILE="$BUILDDIR/packages/FoundationDB-$HOST_ARCH-$VERSION.pkg"
 
 CLIENTSDIR=$( mktemp -d -t fdb-clients-pkg )
 SERVERDIR=$( mktemp -d -t fdb-server-pkg )
@@ -68,6 +68,8 @@ pkgbuild --root $SERVERDIR --identifier FoundationDB-server --version "$VERSION"
 
 rm -rf $SERVERDIR
 
+sed -i '' "s|<options |<options hostArchitectures=\"$HOST_ARCH\" |" "$SRCDIR"/packaging/osx/Distribution.xml
+
 productbuild --distribution "$SRCDIR"/packaging/osx/Distribution.xml --resources "$SRCDIR"/packaging/osx/resources --package-path . "$PKGFILE"
 
 rm FoundationDB-clients.pkg FoundationDB-server.pkg
diff --git a/pull_request_template.md b/pull_request_template.md
index 416adbc743a..cb53c50ce0a 100644
--- a/pull_request_template.md
+++ b/pull_request_template.md
@@ -1,18 +1,18 @@
-Replace this text with your description here...
+## PLEASE DELETE THIS TEMPLATE WHEN YOU POST YOUR PR FOR REVIEW
 
-# Code-Reviewer Section
+Please describe:
+  * The problem (for bugs: how/where you encountered it).
+  * The solution.
+  * Testing done, if any.
 
-The general pull request guidelines can be found [here](https://github.com/apple/foundationdb/wiki/FoundationDB-Commit-Process).
+We encourage AI-assisted review. Save your diff:
+```
+git diff upstream/main -- . > /tmp/pr.diff
+```
 
-Please check each of the following things and check *all* boxes before accepting a PR.
+Then review it with a prompt like:
+```
+Review /tmp/pr.diff. What is it trying to do? Is it correct? Are there bugs? Are there omissions? Are there better ways of doing things? Should this CL be LGTMd?
+```
 
-- [ ] The PR has a description, explaining both the problem and the solution.
-- [ ] The description mentions which forms of testing were done and the testing seems reasonable.
-- [ ] Every function/class/actor that was touched is reasonably well documented.
-
-## For Release-Branches
-
-If this PR is made against a release-branch, please also check the following:
-
-- [ ] This change/bugfix is a cherry-pick from the next younger branch (younger `release-branch` or `main` if this is the youngest branch)
-- [ ] There is a good reason why this PR needs to go into a release branch and this reason is documented (either in the description above or in a linked GitHub issue)
+## END OF TEMPLATE TO DELETE
diff --git a/recipes/java-recipes/MicroSpatial.java b/recipes/java-recipes/MicroSpatial.java
index b7a2b86d260..21121bd9212 100644
--- a/recipes/java-recipes/MicroSpatial.java
+++ b/recipes/java-recipes/MicroSpatial.java
@@ -37,7 +37,7 @@ public class MicroSpatial {
 		zLabel = new Subspace(Tuple.from("Z"));
 	}
 
-	// TODO These three methods, xyToZ, zToXy, and setLocation, are allin the recipe book.
+	// TODO These three methods, xyToZ, zToXy, and setLocation, are all in the recipe book.
 	public long xyToZ(long[] p){
 		long x,y,z;
 		x = p[0]; y = p[1];
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 1ea5979513f..f0c44c528c0 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -12,11 +12,9 @@ set(TEST_INCLUDE ".*" CACHE STRING "Include only tests that match the given rege
 set(TEST_EXCLUDE ".^" CACHE STRING "Exclude all tests matching the given regex")
 set(SANITIZER_OPTIONS "UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1;TSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/contrib/tsan.suppressions;LSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/contrib/lsan.suppressions" CACHE STRING "Environment variables setting sanitizer options")
 
-# for the restart test we optimally want to use the last stable fdbserver
-# to test upgrades.
-# FIXME: explain what `last stable` means in the above comment.  Does it mean "last stable non-main release?"
-# Below we are using 7.1.* so that is pretty old stuff.  Does it mean "most recent patch release of the oldest X.Y release family"?
-# FIXME: Regardless of what "last stable" refers to, explain why this is important.
+# For the restart test, prefer an installed fdbserver from an older stable release
+# to exercise upgrade paths from existing on-disk state. If it is unavailable,
+# the current build is used instead.
 if(WITH_PYTHON)
   find_program(OLD_FDBSERVER_BINARY
     fdbserver-7.1.69 fdbserver fdbserver.exe
@@ -89,7 +87,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES KVStoreTestWrite.txt UNIT IGNORE)
   add_fdb_test(TEST_FILES KVStoreValueSize.txt UNIT IGNORE)
   add_fdb_test(TEST_FILES LayerStatusMerge.txt IGNORE)
-  add_fdb_test(TEST_FILES ParallelRestoreApiCorrectnessAtomicRestore.txt IGNORE)
   add_fdb_test(TEST_FILES PureNetwork.txt IGNORE)
   add_fdb_test(TEST_FILES RRW2500.txt IGNORE)
   add_fdb_test(TEST_FILES RandomRead.txt IGNORE)
@@ -172,8 +169,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/CycleTest.toml)
   add_fdb_test(TEST_FILES fast/RocksdbNondeterministicTest.toml)
   add_fdb_test(TEST_FILES fast/DataLossRecovery.toml)
-  # EncryptionUnitTests is only a convenience file to run the different types of encryption tests at once. Do not enable in general ensembles
-  add_fdb_test(TEST_FILES fast/EncryptionUnitTests.toml IGNORE)
   add_fdb_test(TEST_FILES fast/FuzzApiCorrectness.toml)
   add_fdb_test(TEST_FILES fast/FuzzApiCorrectnessClean.toml)
   add_fdb_test(TEST_FILES fast/HTTPKeyValueStore.toml)
@@ -184,6 +179,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/LocalRatekeeper.toml)
   add_fdb_test(TEST_FILES fast/LongStackWriteDuringRead.toml)
   add_fdb_test(TEST_FILES fast/LowLatency.toml)
+  add_fdb_test(TEST_FILES fast/MaxGrvQueueDelay.toml)
 
   # TODO: Fix failures and re-enable this test:
   add_fdb_test(TEST_FILES fast/LowLatencySingleClog.toml IGNORE)
@@ -209,6 +205,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/Sideband.toml)
   add_fdb_test(TEST_FILES fast/SidebandSingle.toml)
   add_fdb_test(TEST_FILES fast/SidebandWithStatus.toml)
+  add_fdb_test(TEST_FILES fast/StatusDuringOutage.toml)
   add_fdb_test(TEST_FILES fast/SimpleAtomicAdd.toml)
   add_fdb_test(TEST_FILES fast/SpecialKeySpaceCorrectness.toml)
   add_fdb_test(TEST_FILES fast/SpecialKeySpaceRobustness.toml)
@@ -284,7 +281,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/CycleWithKills.toml)
   add_fdb_test(TEST_FILES rare/CycleWithDeadHall.toml)
   add_fdb_test(TEST_FILES rare/DataDistributionMetrics.toml)
-  add_fdb_test(TEST_FILES rare/ExpectStableThroughput.toml)
   add_fdb_test(TEST_FILES rare/HighContentionPrefixAllocator.toml)
   add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectness.toml)
@@ -298,7 +294,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/RedwoodCorrectnessBTree.toml)
   add_fdb_test(TEST_FILES rare/RedwoodDeltaTree.toml)
   add_fdb_test(TEST_FILES rare/Throttling.toml)
-  add_fdb_test(TEST_FILES rare/ThroughputQuota.toml)
   add_fdb_test(TEST_FILES rare/TransactionCost.toml)
   add_fdb_test(TEST_FILES rare/TransactionTagApiCorrectness.toml)
   add_fdb_test(TEST_FILES rare/TransactionTagSwizzledApiCorrectness.toml)
@@ -435,21 +430,12 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/WriteDuringReadAtomicRestore.toml)
   add_fdb_test(TEST_FILES slow/WriteDuringReadSwitchover.toml)
   add_fdb_test(TEST_FILES slow/ddbalance.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupCorrectnessCycle.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessCycle.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml)
-  add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml)
   add_fdb_test(TEST_FILES negative/ResolverIgnoreTooOld.toml)
   add_fdb_test(TEST_FILES negative/ResolverIgnoreReads.toml)
   add_fdb_test(TEST_FILES negative/ResolverIgnoreWrites.toml)
   add_fdb_test(TEST_FILES negative/StorageCorruption.toml)
-  add_fdb_test(TEST_FILES ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml IGNORE)
 
-  # Note that status tests are not deterministic.
+# Note that status tests are not deterministic.
   add_fdb_test(TEST_FILES status/invalid_proc_addresses.txt)
   add_fdb_test(TEST_FILES status/local_6_machine_no_replicas_remain.txt)
   add_fdb_test(TEST_FILES status/separate_1_of_3_coordinators_remain.txt)
diff --git a/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt b/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
deleted file mode 100644
index 1ecb871d31f..00000000000
--- a/tests/ParallelRestoreApiCorrectnessAtomicRestore.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-testTitle=ApiCorrectnessTest
-clearAfterTest=false
-simBackupAgents=BackupToFile
-;timeout is in seconds
-timeout=360000
-runSetup=true
-
-    testName=ApiCorrectness
-    numKeys=5000
-    onlyLowerCase=true
-    shortKeysRatio=0.5
-    minShortKeyLength=1
-    maxShortKeyLength=3
-    minLongKeyLength=1
-    maxLongKeyLength=128
-    minValueLength=1
-    maxValueLength=1000
-    numGets=1000
-    numGetRanges=100
-    numGetRangeSelectors=100
-    numGetKeys=100
-    numClears=100
-    numClearRanges=10
-    maxTransactionBytes=500000
-    randomTestDuration=60
-
-    testName=AtomicRestore
-    startAfter=10.0
-    restoreAfter=50.0
-    fastRestore=true
-    usePartitionedLogs=true
-
-    ; Each testName=RunRestoreWorkerWorkload creates a restore worker
-    ; We need at least 3 restore workers: master, loader, and applier
-    testName=RunRestoreWorkerWorkload
-
diff --git a/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml b/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
deleted file mode 100644
index 15a00897874..00000000000
--- a/tests/ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml
+++ /dev/null
@@ -1,40 +0,0 @@
-[[test]]
-testTitle = 'ApiCorrectnessTest'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-runSetup = true
-
-    [[test.workload]]
-    testName = 'ApiCorrectness'
-    numKeys = 5000
-    onlyLowerCase = true
-    shortKeysRatio = 0.5
-    minShortKeyLength = 1
-    maxShortKeyLength = 3
-    minLongKeyLength = 1
-    maxLongKeyLength = 128
-    minValueLength = 1
-    maxValueLength = 1000
-    numGets = 1000
-    numGetRanges = 100
-    numGetRangeSelectors = 100
-    numGetKeys = 100
-    numClears = 100
-    numClearRanges = 10
-    maxTransactionBytes = 500000
-    randomTestDuration = 60
-    resetDBTimeout = 7200
-
-    [[test.workload]]
-    testName = 'AtomicRestore'
-    startAfter = 10.0
-    restoreAfter = 50.0
-    fastRestore = true
-    usePartitionedLogs = false
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
diff --git a/tests/TestRunner/fdb_test_runner/fdb_version.py b/tests/TestRunner/fdb_test_runner/fdb_version.py
new file mode 100644
index 00000000000..b1a0ac3ae83
--- /dev/null
+++ b/tests/TestRunner/fdb_test_runner/fdb_version.py
@@ -0,0 +1,4 @@
+CURRENT_VERSION = "8.0.0"
+FUTURE_VERSION = "8.1.0"
+PREV_RELEASE_VERSION = "7.4.5"
+PREV2_RELEASE_VERSION = "7.3.59"
diff --git a/tests/TestRunner/requirements.txt b/tests/TestRunner/requirements.txt
index bfa28cd46cd..dc2725ef11b 100644
--- a/tests/TestRunner/requirements.txt
+++ b/tests/TestRunner/requirements.txt
@@ -1,5 +1,4 @@
-Authlib==1.6.6
-cffi==1.15.1
-cryptography==44.0.1
+Authlib==1.6.11
+cryptography==46.0.7
 pycparser==2.21
 toml==0.10.2
diff --git a/tests/fast/BulkDumping.toml b/tests/fast/BulkDumping.toml
index 16a4deb9330..ae56180a0a8 100644
--- a/tests/fast/BulkDumping.toml
+++ b/tests/fast/BulkDumping.toml
@@ -1,5 +1,5 @@
 [configuration]
-storageEngineExcludeTypes = [5] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
 
 disableTss = true # TODO(BulkLoad): support TSS. finishMoveShard should wait for TSS if a data move is bulkload data move.
 
diff --git a/tests/fast/BulkLoading.toml b/tests/fast/BulkLoading.toml
index 68b42350733..501c0a7ec6f 100644
--- a/tests/fast/BulkLoading.toml
+++ b/tests/fast/BulkLoading.toml
@@ -1,5 +1,5 @@
 [configuration]
-storageEngineExcludeTypes = [5] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
 
 disableTss = true # TODO(BulkLoad): support TSS. finishMoveShard should wait for TSS if a data move is bulkload data move.
 
diff --git a/tests/fast/EncryptionUnitTests.toml b/tests/fast/EncryptionUnitTests.toml
deleted file mode 100644
index 66adf38c024..00000000000
--- a/tests/fast/EncryptionUnitTests.toml
+++ /dev/null
@@ -1,17 +0,0 @@
-[[test]]
-testTitle = 'BlobCipherUnitTest'
-useDB = false
-startDelay = 0
-
-    [[test.workload]]
-    testName = 'UnitTests'
-    testsMatching = '/blobCipher'
-
-[[test]]
-testTitle = 'EncryptKeyProxyUnitTest'
-useDB = false
-startDelay = 0
-
-    [[test.workload]]
-    testName = 'UnitTests'
-    testsMatching = '/EncryptKeyProxy'
diff --git a/tests/fast/MaxGrvQueueDelay.toml b/tests/fast/MaxGrvQueueDelay.toml
new file mode 100644
index 00000000000..e385a2465d4
--- /dev/null
+++ b/tests/fast/MaxGrvQueueDelay.toml
@@ -0,0 +1,19 @@
+[[knobs]]
+# Keep GRV ratekeeper capacity low enough that the warmup burst creates
+# queueing without throttling cluster setup.
+ratekeeper_default_limit = 100
+ratekeeper_max_rate = 100
+
+[[test]]
+testTitle = 'MaxGrvQueueDelay'
+
+    [[test.workload]]
+    testName = 'MaxGrvQueueDelay'
+    startAfter = 0.0
+    testTimeout = 30.0
+    requestCount = 250
+    minRejected = 1
+    maxQueueDelayMS = 0
+    permissiveMaxQueueDelayMS = 60000
+    warmupRequestCount = 10000
+    warmupDelay = 0.1
diff --git a/tests/fast/RESTUnit.toml b/tests/fast/RESTUnit.toml
index 3bd2071489b..c7f689805fd 100644
--- a/tests/fast/RESTUnit.toml
+++ b/tests/fast/RESTUnit.toml
@@ -1,6 +1,3 @@
-[[knobs]]
-rest_kms_connector_remove_trailing_newline = true
-
 [[test]]
 testTitle = 'RESTUtilsTest'
 startDelay = 0
@@ -9,21 +6,3 @@ useDB = false
     [[test.workload]]
     testName = 'UnitTests'
     testsMatching = '/RESTUtils'
-
-[[test]]
-testTitle = 'RESTSimKmsVaultTest'
-startDelay = 0
-useDB = false
-
-    [[test.workload]]
-    testName = 'UnitTests'
-    testsMatching = '/restSimKmsVault/'
-
-[[test]]
-testTitle = 'RESTKmsConnectorTest'
-useDB = false
-startDelay = 0
-
-    [[test.workload]]
-    testName = 'UnitTests'
-    testsMatching = '/KmsConnector/'
diff --git a/tests/fast/StatusDuringOutage.toml b/tests/fast/StatusDuringOutage.toml
new file mode 100644
index 00000000000..40ea6ac8fc1
--- /dev/null
+++ b/tests/fast/StatusDuringOutage.toml
@@ -0,0 +1,26 @@
+[[test]]
+testTitle = 'StatusDuringOutage'
+
+    [[test.workload]]
+    testName = 'Status'
+    testDuration = 60.0
+    requestsPerSecond = 0.5
+    # STATUS_TIMEOUT is 30s. The 2s buffer accounts for scheduling delay
+    # between when the timeout fires and when the latency-measuring code
+    # gets scheduled and calls now().
+    maxAcceptableStatusLatency = 32.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 1
+    reboot = true
+    testDuration = 60.0
+    suspendDuration = 60.0
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 60.0
+    swizzle = 0
+    scale = 1.0
+    clogginess = 2.0
diff --git a/tests/fast/TxnStateStoreCycleTest.toml b/tests/fast/TxnStateStoreCycleTest.toml
index 73ae7364fee..119860bd940 100644
--- a/tests/fast/TxnStateStoreCycleTest.toml
+++ b/tests/fast/TxnStateStoreCycleTest.toml
@@ -1,4 +1,6 @@
 [configuration]
+# FIXME: remove after bulkloading with fetchKey supports shardedrocksdb
+storageEngineExcludeTypes = [5]
 
 [[test]]
 testTitle = 'PreLoad'
diff --git a/tests/loopback_cluster/run_custom_cluster.sh b/tests/loopback_cluster/run_custom_cluster.sh
index 62cbe0a9f98..ec23cd65f89 100755
--- a/tests/loopback_cluster/run_custom_cluster.sh
+++ b/tests/loopback_cluster/run_custom_cluster.sh
@@ -60,6 +60,7 @@ function start_servers {
     # this native bash behavior, we printf the KNOBS string.
     ${2} "${FDB}" -p auto:"${port}" -p 127.0.0.1:"${gport}":grpc ${KNOBS:+$(printf "%s" "${KNOBS}")} -c "${3}" \
       -d "${datadir}" -L "${logdir}" -C "${CLUSTER}" \
+      --knob_blobstore_encryption_type=aws:kms \
       --datacenter_id="${4}" \
       --locality-zoneid "${zone}" \
       --locality-machineid M-$SERVER_COUNT &
diff --git a/tests/rare/ExpectStableThroughput.toml b/tests/rare/ExpectStableThroughput.toml
deleted file mode 100644
index a13360a48c6..00000000000
--- a/tests/rare/ExpectStableThroughput.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-[configuration]
-buggify = false
-
-[[test]]
-testTitle='ExpectStableThroughput'
-
-  [[test.workload]]
-  testName='ThroughputQuota'
-  transactionTag='testTag'
-  totalQuotaInPages=1000
-
-  [[test.workload]]
-  testName='ExpectStableThroughput'
-  throttlingTag='testTag'
-  expectedThroughputPagesRate=1000
-  # TODO: Reduce this error tolerance
-  errorTolerance=0.3
-  numActors=100
-  attemptedTransactionRatePerActor=4
-  opsPerTransaction=5
diff --git a/tests/rare/ThroughputQuota.toml b/tests/rare/ThroughputQuota.toml
deleted file mode 100644
index f135a226404..00000000000
--- a/tests/rare/ThroughputQuota.toml
+++ /dev/null
@@ -1,13 +0,0 @@
-[[test]]
-testTitle='ThroughputQuotaTest'
-
-  [[test.workload]]
-  testName='ThroughputQuota'
-  transactionTag='a'
-  totalQuotaInPages=4
-
-  [[test.workload]]
-  testName = 'Cycle'
-  transactionsPerSecond = 250.0
-  testDuration = 30.0
-  expectedRate = 0
diff --git a/tests/slow/ApiCorrectnessAtomicRestore.toml b/tests/slow/ApiCorrectnessAtomicRestore.toml
index 536599024b3..0711c42befb 100644
--- a/tests/slow/ApiCorrectnessAtomicRestore.toml
+++ b/tests/slow/ApiCorrectnessAtomicRestore.toml
@@ -37,4 +37,4 @@ runSetup = true
     startAfter = 10.0
     restoreAfter = 50.0
     # This test file uses old restore, which does not work on new backup format
-    usePartitionedLogs = false
+    mutationLogType = 0 # default logs
diff --git a/tests/slow/BackupAndRestore.toml b/tests/slow/BackupAndRestore.toml
index 1e49566d1a3..c89a93a1923 100644
--- a/tests/slow/BackupAndRestore.toml
+++ b/tests/slow/BackupAndRestore.toml
@@ -18,7 +18,7 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'Backup'
-    usePartitionedLog = true
+    mutationLogType = 1 # partitioned-log-experimental
     backupAfter = 10.0
     restoreAfter = 60.0
 
diff --git a/tests/slow/BackupNewAndOldRestore.toml b/tests/slow/BackupNewAndOldRestore.toml
index a5b3da8bf71..cd11579a1e2 100644
--- a/tests/slow/BackupNewAndOldRestore.toml
+++ b/tests/slow/BackupNewAndOldRestore.toml
@@ -16,7 +16,7 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'Backup'
-    usePartitionedLog = true
+    mutationLogType = 1 # partitioned-log-experimental
     backupTag = 'newBackup'
     backupAfter = 10.0
     restoreAfter = 60.0
@@ -38,7 +38,7 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'Backup'
-    usePartitionedLog = false
+    mutationLogType = 0 # default logs
     backupTag = 'oldBackup'
     backupAfter = 10.0
     restoreAfter = 60.0
diff --git a/tests/slow/BackupOldAndNewRestore.toml b/tests/slow/BackupOldAndNewRestore.toml
index 1cbcf8aa828..6718e512d3f 100644
--- a/tests/slow/BackupOldAndNewRestore.toml
+++ b/tests/slow/BackupOldAndNewRestore.toml
@@ -18,7 +18,7 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'Backup'
-    usePartitionedLog = false
+    mutationLogType = 0 # default logs
     backupTag = 'oldBackup'
     backupAfter = 10.0
     restoreAfter = 60.0
@@ -40,7 +40,7 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'Backup'
-    usePartitionedLog = true
+    mutationLogType = 1 # partitioned-log-experimental
     backupTag = 'newBackup'
     backupAfter = 10.0
     restoreAfter = 60.0
diff --git a/tests/slow/BackupS3BlobBulkLoadRestore.toml b/tests/slow/BackupS3BlobBulkLoadRestore.toml
index e58314ba51d..e35cd48954b 100644
--- a/tests/slow/BackupS3BlobBulkLoadRestore.toml
+++ b/tests/slow/BackupS3BlobBulkLoadRestore.toml
@@ -21,7 +21,7 @@
 testClass = "Backup"
 
 [configuration]
-storageEngineExcludeTypes = [5] # FIXME: remove after allowing bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] # FIXME: remove after allowing bulkloading with fetchKey and shardedrocksdb
 disableTss = true # TODO(BulkLoad): support TSS
 
 # HA (multi-region) configuration - randomly enabled for test coverage
diff --git a/tests/slow/BackupS3BlobBulkLoadRestoreMultiRange.toml b/tests/slow/BackupS3BlobBulkLoadRestoreMultiRange.toml
index 9a43858d468..41f9b9eefa2 100644
--- a/tests/slow/BackupS3BlobBulkLoadRestoreMultiRange.toml
+++ b/tests/slow/BackupS3BlobBulkLoadRestoreMultiRange.toml
@@ -20,7 +20,7 @@
 testClass = "Backup"
 
 [configuration]
-storageEngineExcludeTypes = [5] # TODO(BulkLoad): remove after allowing bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] # TODO(BulkLoad): remove after allowing bulkloading with fetchKey and shardedrocksdb
 disableTss = true # TODO(BulkLoad): support TSS
 
 # HA (multi-region) configuration - randomly enabled for test coverage
diff --git a/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml b/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml
index 725a4fdd41a..e850e6665cc 100644
--- a/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml
+++ b/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml
@@ -19,7 +19,7 @@
 testClass = "Backup"
 
 [configuration]
-storageEngineExcludeTypes = [5] # TODO(BulkLoad): remove after allowing bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] # TODO(BulkLoad): remove after allowing bulkloading with fetchKey and shardedrocksdb
 disableTss = true # TODO(BulkLoad): support TSS
 
 # HA (multi-region) configuration - randomly enabled for test coverage
diff --git a/tests/slow/BulkDumpingS3.toml b/tests/slow/BulkDumpingS3.toml
index d5a097ac9a8..3cb93ff3a14 100644
--- a/tests/slow/BulkDumpingS3.toml
+++ b/tests/slow/BulkDumpingS3.toml
@@ -1,5 +1,5 @@
 [configuration]
-storageEngineExcludeTypes = [5] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
 
 disableTss = true # TODO(BulkLoad): support TSS. finishMoveShard should wait for TSS if a data move is bulkload data move.
 
diff --git a/tests/slow/BulkDumpingS3WithChaos.toml b/tests/slow/BulkDumpingS3WithChaos.toml
index ec538b0b2ca..f7fbf7a0e28 100644
--- a/tests/slow/BulkDumpingS3WithChaos.toml
+++ b/tests/slow/BulkDumpingS3WithChaos.toml
@@ -20,7 +20,7 @@
 # - Don't cause FetchKeys/DataDistribution to hang under stress
 
 [configuration]
-storageEngineExcludeTypes = [5] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
+storageEngineExcludeTypes = ["ssd-sharded-rocksdb"] #FIXME: remove after allowing to do bulkloading with fetchKey and shardedrocksdb
 
 disableTss = true # TODO(BulkLoad): support TSS. finishMoveShard should wait for TSS if a data move is bulkload data move.
 
diff --git a/tests/slow/DiskFailureCycle.toml b/tests/slow/DiskFailureCycle.toml
index bbbb088b446..e26d2147a4e 100644
--- a/tests/slow/DiskFailureCycle.toml
+++ b/tests/slow/DiskFailureCycle.toml
@@ -3,8 +3,7 @@ buggify = false
 minimumReplication = 3
 minimumRegions = 3
 logAntiQuorum = 0
-storageEngineExcludeTypes = [1, 2, 4, 5]
-disableRemoteKVS = true
+storageEngineExcludeTypes = ["memory", "memory-radixtree", "ssd-rocksdb-v1", "ssd-sharded-rocksdb"]
 
 [[test]]
 testTitle = 'DiskFailureCycle'
diff --git a/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml b/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
deleted file mode 100644
index 5786e50af58..00000000000
--- a/tests/slow/ParallelRestoreNewBackupCorrectnessAtomicOp.toml
+++ /dev/null
@@ -1,62 +0,0 @@
-[configuration]
-
-[[test]]
-testTitle = 'BackupAndParallelRestoreWithAtomicOp'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'AtomicOps'
-    nodeCount = 30000
-    # Make ops space only 1 key per group
-    #    nodeCount=100
-    transactionsPerSecond = 2500.0
-    #    transactionsPerSecond=500.0
-    #    transactionsPerSecond=500.0
-    #    nodeCount=4
-    #    transactionsPerSecond=250.0
-    testDuration = 30.0
-    # Specify a type of atomicOp
-    #	opType=0
-    #	actorsPerClient=1
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
-    # Test case for parallel restore
-
-    [[test.workload]]
-    testName = 'BackupAndParallelRestoreCorrectness'
-    backupAfter = 10.0
-    restoreAfter = 60.0
-    backupRangesCount = -1
-    # use new backup
-    usePartitionedLogs = true
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 90.0
-    testDuration = 90.0
-    # Do NOT kill restore worker process yet
-    # Kill other process to ensure restore works when FDB cluster has faults
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
diff --git a/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml b/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
deleted file mode 100644
index 81f067e07af..00000000000
--- a/tests/slow/ParallelRestoreNewBackupCorrectnessCycle.toml
+++ /dev/null
@@ -1,57 +0,0 @@
-[configuration]
-
-[[test]]
-testTitle = 'BackupAndRestore'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'Cycle'
-    #    nodeCount=30000
-    nodeCount = 1000
-    #    transactionsPerSecond=500.0
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    #	keyPrefix=!
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
-    # Test case for parallel restore
-
-    [[test.workload]]
-    testName = 'BackupAndParallelRestoreCorrectness'
-    backupAfter = 10.0
-    restoreAfter = 60.0
-    # backupRangesCount<0 means backup the entire normal keyspace
-    backupRangesCount = -1
-    usePartitionedLogs = true
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 90.0
-    testDuration = 90.0
-    # Do NOT kill restore worker process yet
-    # Kill other process to ensure restore works when FDB cluster has faults
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
diff --git a/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml b/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
deleted file mode 100644
index 1d7182ace35..00000000000
--- a/tests/slow/ParallelRestoreNewBackupCorrectnessMultiCycles.toml
+++ /dev/null
@@ -1,80 +0,0 @@
-[configuration]
-
-[[test]]
-testTitle = 'BackupAndRestore'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'Cycle'
-    #    nodeCount=30000
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = '!'
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = 'z'
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = 'A'
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = 'Z'
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
-    # Test case for parallel restore
-
-    [[test.workload]]
-    testName = 'BackupAndParallelRestoreCorrectness'
-    backupAfter = 10.0
-    restoreAfter = 60.0
-    # backupRangesCount<0 means backup the entire normal keyspace
-    backupRangesCount = -1
-    usePartitionedLogs = true
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 90.0
-    testDuration = 90.0
-    # Do NOT kill restore worker process yet
-    # Kill other process to ensure restore works when FDB cluster has faults
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
diff --git a/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml b/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
deleted file mode 100644
index 4b305660bca..00000000000
--- a/tests/slow/ParallelRestoreNewBackupWriteDuringReadAtomicRestore.toml
+++ /dev/null
@@ -1,52 +0,0 @@
-[configuration]
-StderrSeverity = 30
-
-[[test]]
-testTitle = 'WriteDuringReadTest'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'WriteDuringRead'
-    maximumTotalData = 1000000
-    testDuration = 240.0
-    slowModeStart = 60.0
-    minNode = 1
-    useSystemKeys = false
-
-    [[test.workload]]
-    testName = 'AtomicRestore'
-    startAfter = 10.0
-    restoreAfter = 50.0
-    fastRestore = true
-    usePartitionedLogs = true
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 60.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 60.0
-    testDuration = 60.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 60.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 60.0
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
diff --git a/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml b/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
deleted file mode 100644
index 99c663f32b7..00000000000
--- a/tests/slow/ParallelRestoreOldBackupCorrectnessAtomicOp.toml
+++ /dev/null
@@ -1,60 +0,0 @@
-[configuration]
-
-[[test]]
-testTitle = 'BackupAndParallelRestoreWithAtomicOp'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'AtomicOps'
-    nodeCount = 30000
-    # Make ops space only 1 key per group
-    transactionsPerSecond = 2500.0
-    #    nodeCount=4
-    #    transactionsPerSecond=250.0
-    testDuration = 30.0
-    # Specify a type of atomicOp
-    # Unset the following two options for debug purpose
-    #	opType=0
-    #	actorsPerClient=1
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
-    # Test case for parallel restore
-
-    [[test.workload]]
-    testName = 'BackupAndParallelRestoreCorrectness'
-    backupAfter = 10.0
-    restoreAfter = 60.0
-    backupRangesCount = -1
-    #    use old backup
-    usePartitionedLogs = false
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 90.0
-    testDuration = 90.0
-    # Do NOT kill restore worker process yet
-    # Kill other process to ensure restore works when FDB cluster has faults
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
diff --git a/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml b/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
deleted file mode 100644
index 087ba163424..00000000000
--- a/tests/slow/ParallelRestoreOldBackupCorrectnessCycle.toml
+++ /dev/null
@@ -1,57 +0,0 @@
-[configuration]
-
-[[test]]
-testTitle = 'BackupAndRestore'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount=30000
-    #    nodeCount = 1000
-    #    transactionsPerSecond=500.0
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    #	keyPrefix=!
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
-    # Test case for parallel restore
-
-    [[test.workload]]
-    testName = 'BackupAndParallelRestoreCorrectness'
-    backupAfter = 10.0
-    restoreAfter = 60.0
-    # backupRangesCount<0 means backup the entire normal keyspace
-    backupRangesCount = -1
-    usePartitionedLogs = false
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 90.0
-    testDuration = 90.0
-    # Do NOT kill restore worker process yet
-    # Kill other process to ensure restore works when FDB cluster has faults
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
diff --git a/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml b/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
deleted file mode 100644
index 9a4e9865c9a..00000000000
--- a/tests/slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml
+++ /dev/null
@@ -1,80 +0,0 @@
-[configuration]
-
-[[test]]
-testTitle = 'BackupAndRestore'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'Cycle'
-    #    nodeCount=30000
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = '!'
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = 'z'
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = 'A'
-
-    [[test.workload]]
-    testName = 'Cycle'
-    nodeCount = 1000
-    transactionsPerSecond = 2500.0
-    testDuration = 30.0
-    expectedRate = 0
-    keyPrefix = 'Z'
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
-    # Test case for parallel restore
-
-    [[test.workload]]
-    testName = 'BackupAndParallelRestoreCorrectness'
-    backupAfter = 10.0
-    restoreAfter = 60.0
-    # backupRangesCount<0 means backup the entire normal keyspace
-    backupRangesCount = -1
-    usePartitionedLogs = false
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 90.0
-    testDuration = 90.0
-    # Do NOT kill restore worker process yet
-    # Kill other process to ensure restore works when FDB cluster has faults
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 90.0
diff --git a/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml b/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml
deleted file mode 100644
index 0479031d755..00000000000
--- a/tests/slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml
+++ /dev/null
@@ -1,52 +0,0 @@
-[configuration]
-StderrSeverity = 30
-
-[[test]]
-testTitle = 'WriteDuringReadTest'
-clearAfterTest = false
-simBackupAgents = 'BackupToFile'
-#timeout is in seconds
-timeout = 360000
-
-    [[test.workload]]
-    testName = 'WriteDuringRead'
-    maximumTotalData = 1000000
-    testDuration = 240.0
-    slowModeStart = 60.0
-    minNode = 1
-    useSystemKeys = false
-
-    [[test.workload]]
-    testName = 'AtomicRestore'
-    startAfter = 10.0
-    restoreAfter = 50.0
-    fastRestore = true
-    usePartitionedLogs = false
-
-    [[test.workload]]
-    testName = 'RandomClogging'
-    testDuration = 60.0
-
-    [[test.workload]]
-    testName = 'Rollback'
-    meanDelay = 60.0
-    testDuration = 60.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 60.0
-
-    [[test.workload]]
-    testName = 'Attrition'
-    machinesToKill = 10
-    machinesToLeave = 3
-    reboot = true
-    testDuration = 60.0
-    # Each testName=RunRestoreWorkerWorkload creates a restore worker
-    # We need at least 3 restore workers: master, loader, and applier
-
-    [[test.workload]]
-    testName = 'RunRestoreWorkerWorkload'
diff --git a/tests/slow/WriteDuringReadAtomicRestore.toml b/tests/slow/WriteDuringReadAtomicRestore.toml
index a148f0a1c92..1c9e8186676 100644
--- a/tests/slow/WriteDuringReadAtomicRestore.toml
+++ b/tests/slow/WriteDuringReadAtomicRestore.toml
@@ -18,7 +18,7 @@ simBackupAgents = 'BackupToFile'
     testName = 'AtomicRestore'
     startAfter = 10.0
     restoreAfter = 50.0
-    usePartitionedLogs = false
+    mutationLogType = 0 # default logs
 
     [[test.workload]]
     testName = 'RandomClogging'

From 2e393fab8390d762126caca382832ef87e41fef1 Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Fri, 22 May 2026 16:47:03 +0300
Subject: [PATCH 2/6] XDB-546 added pthread for fdbbenchmark

---
 cmake/FDBBenchmark.cmake | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cmake/FDBBenchmark.cmake b/cmake/FDBBenchmark.cmake
index 505ca64892b..5c0ddb4c04c 100644
--- a/cmake/FDBBenchmark.cmake
+++ b/cmake/FDBBenchmark.cmake
@@ -47,6 +47,12 @@ function(fdb_setup_googlebenchmark)
       message(FATAL_ERROR "Build step for Benchmark has Failed. ${results}")
     endif()
 
+    if(CMAKE_THREAD_LIBS_INIT)
+      list(APPEND BENCHMARK_CXX_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+    elseif(UNIX)
+      list(APPEND BENCHMARK_CXX_LIBRARIES -pthread)
+    endif()
+
     set(BENCHMARK_ENABLE_TESTING OFF)
     add_subdirectory(
       ${googlebenchmark_root}/googlebenchmark-src

From 3be2a24b8cdebec38fff97c0cbf9c7329bff2115 Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Fri, 19 Jun 2026 20:17:48 +0300
Subject: [PATCH 3/6] XDB-546 show keys data size and moving size when DD in
 initialization

---
 fdbserver/clustercontroller/Status.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/fdbserver/clustercontroller/Status.cpp b/fdbserver/clustercontroller/Status.cpp
index 0ff20ac85c6..5c74cbb89ee 100644
--- a/fdbserver/clustercontroller/Status.cpp
+++ b/fdbserver/clustercontroller/Status.cpp
@@ -1842,14 +1842,6 @@ static AsyncResult<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 		TraceEventFields startingStats = dataInfo[0];
 		TraceEventFields dataStats = dataInfo[1];
 
-		if (startingStats.size() && startingStats.getValue("State") != "Active") {
-			JsonBuilderObject stateSectionObj;
-			stateSectionObj["name"] = "initializing";
-			stateSectionObj["description"] = "(Re)initializing automatic data distribution";
-			statusObjData["state"] = stateSectionObj;
-			co_return statusObjData;
-		}
-
 		TraceEventFields md = dataInfo[2];
 
 		// If we have a MovingData message, parse it.
@@ -1880,6 +1872,14 @@ static AsyncResult<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 			statusObjData.setKeyRawNumber("partitions_count", dataStats.getValue("Shards"));
 		}
 
+		if (startingStats.size() && startingStats.getValue("State") != "Active") {
+			JsonBuilderObject stateSectionObj;
+			stateSectionObj["name"] = "initializing";
+			stateSectionObj["description"] = "Healthy (Re)initializing automatic data distribution";
+			statusObjData["state"] = stateSectionObj;
+			co_return statusObjData;
+		}
+
 		JsonBuilderArray teamTrackers;
 		for (int i = 3; i < 5; i++) {
 			const TraceEventFields& inFlight = dataInfo[i];

From 142396f05c2e8bd1b742e97303b88090fb6eaf86 Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Fri, 19 Jun 2026 21:22:13 +0300
Subject: [PATCH 4/6] XDB-546 added degraded status when multiregional failover

---
 fdbcli/StatusCommand.cpp               | 22 +++++++++++++++++-----
 fdbserver/clustercontroller/Status.cpp | 16 ++++++++++++++--
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/fdbcli/StatusCommand.cpp b/fdbcli/StatusCommand.cpp
index ff53f12e487..37aa789d92a 100644
--- a/fdbcli/StatusCommand.cpp
+++ b/fdbcli/StatusCommand.cpp
@@ -708,10 +708,19 @@ void printStatus(StatusObjectReader statusObj,
 							ASSERT_WE_THINK(availLoss == -1);
 							const bool possiblyLosingData = logEpochsMayBeLosingData(statusObjCluster);
 							if (possiblyLosingData) {
-								outputString += format(
-								    "\n\n  Warning: the database may have data loss and availability loss. Please "
-								    "restart following tlog interfaces, otherwise storage servers may never be able "
-								    "to catch up.\n");
+								const std::string baseMessage =
+								    "Please restart following tlog interfaces, otherwise storage servers "
+								    "may never be able to catch up.\n";
+
+								bool degradedMultiRegion = false;
+								statusObjCluster.get("degraded_multi_region", degradedMultiRegion);
+
+								const std::string header =
+								    !degradedMultiRegion
+								        ? "\n\n  Warning: the database may have data loss and availability loss. "
+								        : "\n\n  ";
+								
+								outputString += header + baseMessage;
 							} else {
 								outputString += format(
 								    "\n\n  Warning: the database may have availability loss. The current log state "
@@ -739,7 +748,10 @@ void printStatus(StatusObjectReader statusObj,
 											if (logInterface.get("healthy", healthy) && !healthy) {
 												logInterface.get("id", id);
 												logInterface.get("address", address);
-												missing_log_interfaces += format("%s,%s ", id.c_str(), address.c_str());
+												missing_log_interfaces +=
+												    format("%s,%s ",
+												           id.c_str(),
+												           address.empty() ? "unknown" : address.c_str());
 											}
 										}
 									}
diff --git a/fdbserver/clustercontroller/Status.cpp b/fdbserver/clustercontroller/Status.cpp
index 5c74cbb89ee..db7742d144c 100644
--- a/fdbserver/clustercontroller/Status.cpp
+++ b/fdbserver/clustercontroller/Status.cpp
@@ -1819,6 +1819,7 @@ static JsonBuilderObject configurationFetcher(Optional<DatabaseConfiguration> co
 
 static AsyncResult<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
                                                         DatabaseConfiguration configuration,
+                                                        bool degradedMultiRegion,
                                                         int* minStorageReplicasRemaining) {
 	JsonBuilderObject statusObjData;
 
@@ -1875,7 +1876,11 @@ static AsyncResult<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker,
 		if (startingStats.size() && startingStats.getValue("State") != "Active") {
 			JsonBuilderObject stateSectionObj;
 			stateSectionObj["name"] = "initializing";
-			stateSectionObj["description"] = "Healthy (Re)initializing automatic data distribution";
+			// When operating with a region down (data recovered in the surviving region but recovery is stuck at
+			// accepting_commits waiting for the other region's logs), make the degraded multiregional explicit.
+			stateSectionObj["description"] = degradedMultiRegion
+			                                     ? "Degraded multiregional (Re)initializing automatic data distribution"
+			                                     : "(Re)initializing automatic data distribution";
 			statusObjData["state"] = stateSectionObj;
 			co_return statusObjData;
 		}
@@ -3197,11 +3202,18 @@ static AsyncResult<Void> clusterGetStatusImpl(Reference<ClusterGetStatusState> s
 
 			int minStorageReplicasRemaining = -1;
 			int fullyReplicatedRegions = -1;
+			// The cluster is "degraded multi-region" when it is configured for two usable regions but recovery is
+			// stuck at accepting_commits: the primary region recovered and is committing, but the other region's
+			// logs are missing. Committed data is still safe in the surviving region (and its satellite), so this
+			// is a degraded-but-not-data-losing state.
+			bool degradedMultiRegion = configuration.present() && configuration.get().usableRegions > 1 &&
+			                                 statusCode == RecoveryStatus::accepting_commits;
+			statusObj["degraded_multi_region"] = degradedMultiRegion;
 			// NOTE: here we should start all the transaction before wait in order to overlay latency
 			Future<Optional<Value>> primaryDCFO = getActivePrimaryDC(cx, &fullyReplicatedRegions, &messages);
 			std::vector<AsyncResult<JsonBuilderObject>> statusSectionFetchers;
 			statusSectionFetchers.push_back(
-			    dataStatusFetcher(ddWorker, configuration.get(), &minStorageReplicasRemaining));
+			    dataStatusFetcher(ddWorker, configuration.get(), degradedMultiRegion, &minStorageReplicasRemaining));
 			statusSectionFetchers.push_back(workloadStatusFetcher(
 			    db, workers, mWorker, rkWorker, &qos, &dataOverlay, &status_incomplete_reasons, storageServerFuture));
 			statusSectionFetchers.push_back(layerStatusFetcher(cx, &messages, &status_incomplete_reasons));

From 2f70b2812f33be1217eae957d70adec86160e2dc Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Mon, 22 Jun 2026 10:23:15 +0300
Subject: [PATCH 5/6] XDB-546 added degraded_multi_region to Schemas to fix
 tests

---
 fdbclient/Schemas.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index d88ab83a70e..0a3e0248c88 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -612,6 +612,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
       },
       "active_tss_count":0,
       "degraded_processes":0,
+      "degraded_multi_region":true,
       "database_available":true,
       "database_lock_state": {
          "locked": true,
@@ -1526,6 +1527,7 @@ file is writable and has not been overwritten externally."
       },
       "maintenance_zone":"0ccb4e0fdbdb5583010f6b77d9d10ece",
       "maintenance_seconds_remaining":1.0,
+      "degraded_multi_region":true,
       "data":{
          "least_operating_space_bytes_log_server":0,
          "average_partition_size_bytes":0,

From a3eac468efc338f5f034686321ed9ed53b46674d Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Tue, 30 Jun 2026 16:20:43 +0300
Subject: [PATCH 6/6] XDB-546 Added proxyColocationOnStateless test

---
 .../ClusterController.actor.cpp               | 103 ++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/fdbserver/clustercontroller/ClusterController.actor.cpp b/fdbserver/clustercontroller/ClusterController.actor.cpp
index 644abd0d919..190bc9053e3 100644
--- a/fdbserver/clustercontroller/ClusterController.actor.cpp
+++ b/fdbserver/clustercontroller/ClusterController.actor.cpp
@@ -3837,4 +3837,107 @@ TEST_CASE("/fdbserver/clustercontroller/invalidateExcludedProcessComplaints") {
 	return Void();
 }
 
+// Test for the fix described in PR #10411.
+// Verifies that in a small cluster (3 stateless, 3 transaction, 3 storage processes),
+// the role allocation correctly assigns all 3 commit_proxy roles to the stateless nodes.
+// Previously, only 1 commit_proxy would be recruited due to a bug in the candidate
+// selection logic within ClusterControllerData::getWorkersForRoleInDatacenter.
+// This test ensures that the desired number of proxies (3) is now fully recruited
+// when sufficient stateless processes exist.
+TEST_CASE("/fdbserver/clustercontroller/proxyColocationOnStateless") {
+	ClusterControllerData data(ClusterControllerFullInterface(),
+	                           LocalityData(),
+	                           ServerCoordinators(Reference<IClusterConnectionRecord>(
+	                               new ClusterConnectionMemoryRecord(ClusterConnectionString()))),
+	                           makeReference<AsyncVar<Optional<UID>>>());
+
+	constexpr int numWorkersPerClass = 3;
+
+	auto makeWorkers = [&](ProcessClass::ClassType classType, StringRef prefix, int count) {
+		std::vector<WorkerInterface> workers;
+		for (int i = 0; i < count; i++) {
+			auto pid = prefix.toString() + std::to_string(i);
+			WorkerInterface wi;
+			wi.initEndpoints();
+			wi.locality.set(LocalityData::keyZoneId, Standalone<StringRef>(pid + "_zone"));
+			wi.locality.set(LocalityData::keyProcessId, Standalone<StringRef>(pid));
+			data.id_worker[wi.locality.processId()] =
+			    WorkerInfo(Future<Void>(),
+			               ReplyPromise<RegisterWorkerReply>(),
+			               0,
+			               wi,
+			               ProcessClass(classType, ProcessClass::CommandLineSource),
+			               ProcessClass(classType, ProcessClass::CommandLineSource),
+			               ClusterControllerPriorityInfo(
+			                   ProcessClass::UnsetFit, false, ClusterControllerPriorityInfo::FitnessUnknown),
+			               false,
+			               true,
+			               Standalone<VectorRef<StringRef>>());
+			workers.push_back(wi);
+		}
+		return workers;
+	};
+
+	auto statelessWorkers = makeWorkers(ProcessClass::StatelessClass, "sl"_sr, numWorkersPerClass);
+	auto transactionWorkers = makeWorkers(ProcessClass::TransactionClass, "tx"_sr, numWorkersPerClass);
+	auto storageWorkers = makeWorkers(ProcessClass::StorageClass, "ss"_sr, numWorkersPerClass);
+
+	data.masterProcessId = statelessWorkers[0].locality.processId();
+	data.clusterControllerProcessId = statelessWorkers[1].locality.processId();
+	data.startTime = 0;
+	data.gotFullyRecoveredConfig = true;
+	data.gotProcessClasses = true;
+
+	DatabaseConfiguration config;
+	config.initialized = true;
+	config.tLogReplicationFactor = numWorkersPerClass;
+	config.desiredTLogCount = numWorkersPerClass;
+	config.commitProxyCount = numWorkersPerClass;
+	config.grvProxyCount = numWorkersPerClass;
+	config.resolverCount = numWorkersPerClass;
+	config.tLogPolicy = makeReference<PolicyOne>();
+	data.db.config = config;
+	data.db.fullyRecoveredConfig = config;
+
+	// Use findWorkersForConfigurationDispatch — the production code path
+	// that handles full recruitment: TLogs + two-level proxy/resolver recruiting
+	RecruitFromConfigurationRequest req;
+	req.configuration = config;
+	req.recruitSeedServers = false;
+	req.maxOldLogRouters = 0;
+
+	auto reply = data.findWorkersForConfigurationDispatch(req, false);
+
+	ASSERT(reply.tLogs.size() == numWorkersPerClass);
+	ASSERT(reply.commitProxies.size() == numWorkersPerClass);
+	ASSERT(reply.grvProxies.size() == numWorkersPerClass);
+	ASSERT(reply.resolvers.size() == numWorkersPerClass);
+
+	// TLogs are on transaction processes
+	std::set<Optional<Standalone<StringRef>>> txPids;
+	for (const auto& w : transactionWorkers) {
+		txPids.insert(w.locality.processId());
+	}
+	for (const auto& tlog : reply.tLogs) {
+		ASSERT(txPids.contains(tlog.locality.processId()));
+	}
+
+	// check if proxy/resolvers are on stateless
+	std::set<Optional<Standalone<StringRef>>> slPids;
+	for (const auto& w : statelessWorkers) {
+		slPids.insert(w.locality.processId());
+	}
+	for (const auto& cp : reply.commitProxies) {
+		ASSERT(slPids.contains(cp.locality.processId()));
+	}
+	for (const auto& gp : reply.grvProxies) {
+		ASSERT(slPids.contains(gp.locality.processId()));
+	}
+	for (const auto& rs : reply.resolvers) {
+		ASSERT(slPids.contains(rs.locality.processId()));
+	}
+
+	return Void();
+}
+
 } // namespace